// Package actionlog records what CSM did to a host, as opposed to what it
// found. Findings already have one stream (the SIEM audit log); actions were
// spread across a firewall log, a web UI log, a mail-freeze log and, for
// everything the daemon did on its own, nothing at all.
//
// One record per action, one file, one schema. Each record names the
// privileged operation it belongs to (the IDs in internal/privops), so an
// operator can read the capability matrix and the action log with the same
// vocabulary, and carries the evidence that makes the action reviewable: the
// exact argv when CSM ran a command, and the file's digest before and after
// when it changed a file.
package actionlog
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/safepath"
)
// SchemaVersion is bumped only on an incompatible change. Additive fields do
// not bump it, so a parser can pin on v and ignore unknown keys.
const SchemaVersion = 1
// Result is the outcome of an action.
type Result string
const (
// Applied means the action changed the host.
Applied Result = "applied"
// DryRun means CSM decided to act and did not, because a dry-run or
// observe setting was in force. The record says what it would have done.
DryRun Result = "dry_run"
// Failed means the action was attempted and did not complete.
Failed Result = "failed"
// Refused means a safety rule stopped the action before it ran.
Refused Result = "refused"
)
// Actor is who started the action.
type Actor string
const (
// Daemon means CSM acted on its own.
Daemon Actor = "daemon"
// CLI means an operator ran a command.
CLI Actor = "cli"
// WebUI means an operator used the dashboard or the API.
WebUI Actor = "webui"
)
// FileState is a file's identity at one moment. Digest is a SHA-256 of the
// content; an empty digest with Exists false records a file that was not there.
type FileState struct {
Exists bool `json:"exists"`
Digest string `json:"sha256,omitempty"`
Size int64 `json:"size,omitempty"`
Mode string `json:"mode,omitempty"`
UID uint32 `json:"uid,omitempty"`
GID uint32 `json:"gid,omitempty"`
}
// Record is one action.
type Record struct {
V int `json:"v"`
Timestamp time.Time `json:"ts"`
Hostname string `json:"hostname,omitempty"`
// Op is the privileged-operation ID from internal/privops.
Op string `json:"op"`
// Action distinguishes changes within one operation, such as block and unblock.
Action string `json:"action,omitempty"`
// Actor and ActorDetail say who asked for it. ActorDetail carries the
// operator's source address for a web UI action, or the command name for
// a CLI action.
Actor Actor `json:"actor"`
ActorDetail string `json:"actor_detail,omitempty"`
// FindingID ties the action to the finding that caused it, using the same
// ID the SIEM audit log emits.
FindingID string `json:"finding_id,omitempty"`
IncidentID string `json:"incident_id,omitempty"`
// ActionID and ActionVersion identify a durable lifecycle event across retries.
ActionID string `json:"action_id,omitempty"`
ActionVersion uint64 `json:"action_version,omitempty"`
// UndoOf links a typed undo to its original action.
UndoOf string `json:"undo_of,omitempty"`
// Target is what was acted on: a path, an address, a message ID.
Target string `json:"target"`
Account string `json:"account,omitempty"`
Reason string `json:"reason,omitempty"`
// Digests holds full content identifiers separately from bounded prose.
Digests []string `json:"digests,omitempty"`
// Command is the exact argv when CSM ran a program. Empty when the action
// was performed through system calls.
Command []string `json:"command,omitempty"`
// Before and After are the target file's state around the change.
Before *FileState `json:"before,omitempty"`
After *FileState `json:"after,omitempty"`
Result Result `json:"result"`
Error string `json:"error,omitempty"`
// Undo is the command that reverses the action, when one exists.
Undo string `json:"undo,omitempty"`
// RecoveryPath identifies retained file content and its metadata sidecar.
RecoveryPath string `json:"recovery_path,omitempty"`
// Count is how many events a summary record stands for. Only a
// summary sets it, and a summary names no target.
Count uint64 `json:"count,omitempty"`
// Prediction is what a dry-run forecast said live admission would have
// done, from its fixed vocabulary. Only a dry_run summary sets it.
Prediction string `json:"prediction,omitempty"`
// Cause narrows a refusal summary's Error, from its fixed vocabulary.
// Only an attribution refusal summary sets it.
Cause string `json:"cause,omitempty"`
}
// Sink writes records. The daemon installs a file sink at startup; tests
// install their own.
type Sink interface {
Write(Record) error
}
var (
mu sync.RWMutex
sink Sink
host string
byActor = Daemon
)
// SetSink installs the process-wide sink. Passing nil disables recording,
// which is what a CLI that has not opened the log does.
func SetSink(s Sink, hostname string) {
mu.Lock()
defer mu.Unlock()
sink, host = s, hostname
}
// SetDefaultActor declares which process is recording. The daemon leaves it at
// Daemon; a CLI that opens the log sets CLI. Actions an operator starts through
// the web UI run inside the daemon, so they record as Daemon here and carry the
// operator's address in the web UI's own action log.
func SetDefaultActor(a Actor) {
mu.Lock()
defer mu.Unlock()
byActor = a
}
// DefaultActor returns the process-wide actor for call sites that have no more
// specific attribution.
func DefaultActor() Actor {
mu.RLock()
defer mu.RUnlock()
return byActor
}
// A stuck filesystem or a broken sink must not hold a response worker forever.
// Bound both the wait and outstanding writes; normal writes complete before
// returning, including in short-lived CLI processes.
const writeTimeout = 250 * time.Millisecond
// Write records one action without propagating sink errors or panics. Recording
// is best effort: saturation or an unresponsive sink can cost an action record.
func Write(r Record) {
mu.RLock()
s, h, a := sink, host, byActor
mu.RUnlock()
if s == nil {
return
}
actionWrites.write(s, prepareRecord(r, h, a))
}
func prepareRecord(r Record, h string, a Actor) Record {
r.V = SchemaVersion
if r.Timestamp.IsZero() {
r.Timestamp = time.Now().UTC()
}
if r.Hostname == "" {
r.Hostname = h
}
if r.Actor == "" {
r.Actor = a
}
// A timed-out write can outlive the caller's buffers.
r.Command = append([]string(nil), r.Command...)
r.Digests = append([]string(nil), r.Digests...)
if r.Before != nil {
before := *r.Before
r.Before = &before
}
if r.After != nil {
after := *r.After
r.After = &after
}
return r
}
// ErrDurableUnavailable means no acknowledging sink is installed.
var ErrDurableUnavailable = errors.New("durable action log sink unavailable")
// ErrDurableUnacknowledged means delivery did not finish within the bounded
// wait. The sink may still complete; retry using the same action identity.
var ErrDurableUnacknowledged = errors.New("durable action log delivery unacknowledged")
// WriteDurable acknowledges only a sink's durable write. Delivery is at least
// once: a timeout can leave an append running after this call returns.
func WriteDurable(r Record) error {
mu.RLock()
s, h, a := sink, host, byActor
mu.RUnlock()
durable, ok := s.(interface{ WriteDurable(Record) error })
if !ok {
return ErrDurableUnavailable
}
acknowledgement := make(chan error, 1)
actionWrites.write(durableWriteAdapter{write: durable.WriteDurable, acknowledgement: acknowledgement}, prepareRecord(r, h, a))
select {
case err := <-acknowledgement:
return err
default:
return ErrDurableUnacknowledged
}
}
// WriteDurableBatch acknowledges records only after a sink's one durable
// write of all of them: a delivery batch is synced once. Like
// WriteDurable it delivers at least once.
func WriteDurableBatch(rs []Record) error {
if len(rs) == 0 {
return nil
}
mu.RLock()
s, h, a := sink, host, byActor
mu.RUnlock()
durable, ok := s.(interface{ WriteDurableBatch([]Record) error })
if !ok {
return ErrDurableUnavailable
}
prepared := make([]Record, len(rs))
for i, r := range rs {
prepared[i] = prepareRecord(r, h, a)
}
acknowledgement := make(chan error, 1)
actionWrites.run(uint64(len(prepared)), func() error {
err := durable.WriteDurableBatch(prepared)
acknowledgement <- err
return err
})
select {
case err := <-acknowledgement:
return err
default:
return ErrDurableUnacknowledged
}
}
type durableWriteAdapter struct {
write func(Record) error
acknowledgement chan<- error
}
func (s durableWriteAdapter) Write(r Record) error {
err := s.write(r)
s.acknowledgement <- err
return err
}
// maxFileSize is the rotation threshold, matching the firewall and web UI
// logs this stream consolidates.
const maxFileSize = 10 * 1024 * 1024
// FileSink appends JSON lines to a file, rotating it once at the threshold.
type FileSink struct {
resolve func() string
mu sync.Mutex
path string
onErr func(error)
syncFile func(*os.File) error
}
// NewFileSink returns a sink writing to the file resolve names. The path is
// resolved on the first record and remembered, so installing a sink in a
// process that never records an action costs nothing: a CLI that only reads
// does not load config to work out where the log lives. onErr reports write
// failures; pass nil to discard them.
func NewFileSink(resolve func() string, onErr func(error)) *FileSink {
return &FileSink{resolve: resolve, onErr: onErr}
}
// logPath returns the resolved path, resolving it once. The caller holds mu.
func (f *FileSink) logPath() string {
if f.path == "" {
f.path = f.resolve()
}
return f.path
}
// DefaultPath is where the daemon keeps the action log.
func DefaultPath(logDir string) string { return filepath.Join(logDir, "actions.jsonl") }
func (f *FileSink) Write(r Record) error {
err := f.write([]Record{r}, false)
// Reporting outside the file lock lets a callback inspect or replace the
// sink without deadlocking a completed action.
if err != nil {
f.report(err)
}
return err
}
// WriteDurable appends and syncs the record and its directory entries while
// holding the same cross-process lock as ordinary writes and rotation.
func (f *FileSink) WriteDurable(r Record) error {
return f.WriteDurableBatch([]Record{r})
}
// WriteDurableBatch appends every record in one write and syncs once. An
// empty batch writes nothing.
func (f *FileSink) WriteDurableBatch(rs []Record) error {
if len(rs) == 0 {
return nil
}
err := f.write(rs, true)
if err != nil {
f.report(err)
}
return err
}
func (f *FileSink) sync(file *os.File) error {
if f.syncFile != nil {
return f.syncFile(file)
}
return file.Sync()
}
func (f *FileSink) write(rs []Record, durable bool) error {
var data []byte
for _, r := range rs {
// Cleaning explanations and command errors may contain
// attacker-controlled content. Keep individual lines readable by
// the bounded history reader.
r.Reason = boundedDetail(r.Reason)
r.Error = boundedDetail(r.Error)
line, marshalErr := json.Marshal(r)
if marshalErr != nil {
return marshalErr
}
data = append(append(data, line...), '\n')
}
f.mu.Lock()
defer f.mu.Unlock()
path := f.logPath()
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
return err
}
// The daemon and CLI are separate writers. Lock a stable sidecar inode so
// rotation cannot move another writer's newly opened log out from under it.
lock, lockErr := openLogFile(path+".lock", os.O_RDWR|os.O_CREATE, 0640)
if lockErr != nil {
return lockErr
}
defer func() { _ = lock.Close() }()
// #nosec G115 -- an open file descriptor fits in int on supported Unix hosts.
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil {
return err
}
defer func() { _ = unix.Flock(int(lock.Fd()), unix.LOCK_UN) }() // #nosec G115 -- open file descriptor.
if info, statErr := os.Lstat(path); statErr == nil {
if !info.Mode().IsRegular() {
return fmt.Errorf("action log is not a regular file: %s", path)
}
if info.Size() > maxFileSize {
if err := os.Rename(path, path+".1"); err != nil {
return err
}
}
}
fh, err := openLogFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640)
if err != nil {
return err
}
if _, err := fh.Write(data); err != nil {
_ = fh.Close()
return err
}
if durable {
if err := f.sync(fh); err != nil {
_ = fh.Close()
return err
}
}
if err := fh.Close(); err != nil {
return err
}
if durable {
return f.syncDirectories(filepath.Dir(path))
}
return nil
}
// Sync the whole ancestor chain because MkdirAll may have created multiple
// directories, and a retry must also acknowledge earlier uncertain creation.
func (f *FileSink) syncDirectories(dir string) error {
absolute, err := filepath.Abs(dir)
if err != nil {
return err
}
for {
// #nosec G304 -- operator-configured log directory, opened only for syncing.
directory, err := os.Open(absolute)
if err != nil {
return err
}
syncErr := f.sync(directory)
closeErr := directory.Close()
if syncErr != nil {
return syncErr
}
if closeErr != nil {
return closeErr
}
parent := filepath.Dir(absolute)
if parent == absolute {
return nil
}
absolute = parent
}
}
func boundedDetail(value string) string {
const limit = 4096
if len(value) > limit {
return strings.Clone(value[:limit]) + " [truncated]"
}
return value
}
func openLogFile(path string, flags int, mode os.FileMode) (*os.File, error) {
// #nosec G304 G302 -- operator-configured log path; no symlinks or special
// files. 0640 allows the log shipper's group to read the stream.
f, err := os.OpenFile(path, flags|unix.O_NOFOLLOW|unix.O_NONBLOCK, mode)
if err != nil {
return nil, err
}
info, err := f.Stat()
if err == nil && !info.Mode().IsRegular() {
err = fmt.Errorf("action log is not a regular file: %s", path)
}
if err != nil {
_ = f.Close()
return nil, err
}
return f, nil
}
// Read visits fixed snapshots of the rotated and current logs, oldest first.
// Both files are pinned before releasing the rotation lock, so a slow reader
// neither loses a rotated file nor holds up action writers.
func Read(path string, visit func(io.Reader) error) error {
for {
retry, err := readSnapshot(path, visit)
if !retry {
return err
}
}
}
func readSnapshot(path string, visit func(io.Reader) error) (bool, error) {
lock, err := openLogFile(path+".lock", os.O_RDONLY, 0)
if err != nil && !os.IsNotExist(err) {
return false, err
}
if lock != nil {
defer func() { _ = lock.Close() }()
// #nosec G115 -- an open file descriptor fits in int on supported Unix hosts.
if err := unix.Flock(int(lock.Fd()), unix.LOCK_SH); err != nil {
return false, err
}
}
var files []*os.File
defer func() {
for _, f := range files {
_ = f.Close()
}
}()
var readers []io.Reader
for _, name := range []string{path + ".1", path} {
f, openErr := openLogFile(name, os.O_RDONLY, 0)
if os.IsNotExist(openErr) {
continue
}
if openErr != nil {
return false, openErr
}
files = append(files, f)
info, statErr := f.Stat()
if statErr != nil {
return false, statErr
}
readers = append(readers, io.NewSectionReader(f, 0, info.Size()))
}
if lock == nil {
// The first writer may have created its lock while we opened the
// files. Retry under that lock before exposing a mixed snapshot.
if _, err := os.Lstat(path + ".lock"); err == nil {
return true, nil
} else if !os.IsNotExist(err) {
return false, err
}
} else if err := unix.Flock(int(lock.Fd()), unix.LOCK_UN); err != nil { // #nosec G115 -- open file descriptor.
return false, err
}
for _, reader := range readers {
if err := visit(reader); err != nil {
return false, err
}
}
return false, nil
}
func (f *FileSink) report(err error) {
if f.onErr != nil {
f.onErr(err)
}
}
// Metadata captures identity without opening or hashing the target. It is safe
// to use before a security action: evidence collection must not delay removal.
func Metadata(path string) *FileState {
info, err := os.Lstat(path)
if err != nil {
return &FileState{}
}
return FromInfo(info)
}
// FromInfo describes metadata already pinned by the operation itself.
func FromInfo(info os.FileInfo) *FileState {
if info == nil {
return &FileState{}
}
st := &FileState{Exists: true, Size: info.Size(), Mode: info.Mode().String()}
fillOwner(st, info)
return st
}
// Stat hashes only a bounded regular file reached without following symlinks.
// No digest is better than a digest of a different inode or a truncated prefix.
func Stat(path string) *FileState {
st := Metadata(path)
if !st.Exists {
return st
}
absolute, err := filepath.Abs(path)
if err != nil {
return st
}
dir, err := safepath.OpenDirNoFollow(filepath.Dir(absolute))
if err != nil {
return st
}
defer func() { _ = dir.Close() }()
f, err := dir.OpenFile(filepath.Base(absolute), os.O_RDONLY, 0)
if err != nil {
return st
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil {
return st
}
st = FromInfo(info)
if !info.Mode().IsRegular() || info.Size() > maxDigestBytes {
return st
}
h := sha256.New()
n, err := io.Copy(h, io.LimitReader(f, maxDigestBytes+1))
if err != nil || n != info.Size() || n > maxDigestBytes {
return st
}
after, err := f.Stat()
if err == nil && after.Size() == info.Size() && after.ModTime().Equal(info.ModTime()) {
st.Digest = hex.EncodeToString(h.Sum(nil))
}
return st
}
const maxDigestBytes = 64 * 1024 * 1024
// ContentState captures the exact bytes read or written through a pinned file.
func ContentState(info os.FileInfo, data []byte) *FileState {
st := FromInfo(info)
st.Size = int64(len(data))
if len(data) <= maxDigestBytes {
sum := sha256.Sum256(data)
st.Digest = hex.EncodeToString(sum[:])
}
return st
}
// Describe renders a record as one operator-readable line, used by `csm
// actions` and by the daemon log.
func (r Record) Describe() string {
line := fmt.Sprintf("%s %s %s target=%q", r.Timestamp.UTC().Format(time.RFC3339), r.Op, r.Result, r.Target)
if r.Action != "" {
line += fmt.Sprintf(" action=%q", r.Action)
}
if r.Account != "" {
line += fmt.Sprintf(" account=%q", r.Account)
}
if len(r.Command) > 0 {
line += " command=" + fmt.Sprintf("%q", r.Command)
}
if r.Before != nil && r.After != nil {
line += fmt.Sprintf(" sha256 %s -> %s", shortDigest(r.Before), shortDigest(r.After))
}
if r.Error != "" {
line += fmt.Sprintf(" error=%q", r.Error)
}
if r.Count != 0 {
line += fmt.Sprintf(" reason=%q via=%q count=%d", r.Reason, r.ActorDetail, r.Count)
}
return line
}
func shortDigest(s *FileState) string {
switch {
case s == nil || !s.Exists:
return "absent"
case s.Digest == "":
return "unhashed"
default:
return s.Digest[:min(12, len(s.Digest))]
}
}
//go:build unix
package actionlog
import (
"os"
"syscall"
)
func fillOwner(st *FileState, info os.FileInfo) {
if sys, ok := info.Sys().(*syscall.Stat_t); ok {
st.UID, st.GID = sys.Uid, sys.Gid
}
}
package actionlog
import (
"log"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
type writePool struct {
slots chan struct{}
stats *queuehealth.Tracker
}
// writeStallBudget is how long a sink write may run before the row reports a
// stall. The caller's own wait budget is far shorter, so a batch of actions
// serialising behind one file lock is normal rather than a degradation.
const writeStallBudget = time.Minute
var actionWrites = newWritePool(64)
func newWritePool(capacity int) *writePool {
return &writePool{
slots: make(chan struct{}, capacity),
stats: queuehealth.NewSharedCapacity(capacity, writeStallBudget),
}
}
// QueueStatus preserves process-wide work and loss across sink changes. A
// caller deadline does not discard a record that its sink may still write.
func QueueStatus(now time.Time) queuehealth.Status {
return actionWrites.stats.Snapshot(now)
}
func (p *writePool) write(s Sink, r Record) {
p.run(1, func() error { return s.Write(r) })
}
// run holds one slot while write runs; on failure its records count as
// lost.
func (p *writePool) run(records uint64, write func() error) {
timer := time.NewTimer(writeTimeout)
defer timer.Stop()
select {
case p.slots <- struct{}{}:
case <-timer.C:
p.stats.Lose(time.Now(), records)
return
}
ticket := p.stats.Begin(time.Now())
ticket.Start(time.Now())
done := make(chan struct{})
go func() {
failed := true
defer func() {
v := recover()
if failed {
p.stats.Lose(time.Now(), records)
}
defer func() {
<-p.slots
ticket.Finish(time.Now())
close(done)
}()
if v != nil {
log.Printf("action log sink panicked: %v", v)
}
}()
failed = write() != nil
}()
select {
case <-done:
case <-timer.C:
}
}
package admission
import (
"sort"
"time"
)
const (
// RootFreshness is how long a root observation can justify a response.
RootFreshness = 2 * time.Hour
// SupportLookback is how far back an independent root may corroborate.
// It never refreshes the fresh root it supports.
SupportLookback = 24 * time.Hour
// MaxRoots bounds the root references one assessment reads.
MaxRoots = 16
// maxFutureSkew tolerates producer and engine clocks disagreeing by
// less than a second.
maxFutureSkew = time.Second
)
// Assessment is the priority of one candidate at one instant. Admission
// recomputes it on dequeue, reload and recovery; it is never stored as
// proof.
type Assessment struct {
Tier Tier
// DirectC3: a fresh root carries reviewed compromise evidence.
DirectC3 bool
// Corroborated: an independent root of another family raised the class
// by one. Never set together with DirectC3.
Corroborated bool
// EvidenceExpiry is when the last fresh root stops being fresh; the
// candidate ages out then.
EvidenceExpiry time.Time
// ReassessBy is the earliest instant the class or severity drops, or
// all roots become stale, if no new evidence arrives.
ReassessBy time.Time
// NextChange is the first instant after now at which any root stops
// being fresh or stops supporting corroboration. The assessment cannot
// change before it, so a queued candidate is assessed again then.
NextChange time.Time
// Roots are the validated root IDs, sorted and deduplicated.
Roots []EvidenceID
}
// Reserved reports whether the candidate may use the reserved lane.
func (a Assessment) Reserved() bool { return a.DirectC3 || a.Corroborated }
func rootExpiry(e Evidence) time.Time {
exp := e.ObservedAt().Add(RootFreshness)
if intel, ok := e.Intel(); ok && intel.Expires.Before(exp) {
exp = intel.Expires
}
return exp
}
func sameObservation(a, b Evidence) bool {
return a.rec.Stream == b.rec.Stream && a.rec.Cursor == b.rec.Cursor
}
// Assess derives the tier of a candidate aimed at target from its roots at
// now. Every root must name target, or lie inside a prefix target. The base
// class is the highest class a fresh root supports; a fresh local attack
// root and an independent root of another family seen within the lookback
// raise an address candidate by one class, once. Prefix candidates are not
// raised: range corroboration has its own rules.
func Assess(target Target, roots []Evidence, now time.Time) (Assessment, error) {
if _, isService := target.Service(); isService {
return Assessment{}, refuse(ReasonUnsupportedContainment, "service targets cannot be assessed yet")
}
switch {
case target.IsZero():
return Assessment{}, refuse(ReasonInvalid, "assessment has no target")
case len(roots) == 0:
return Assessment{}, refuse(ReasonInvalid, "assessment has no evidence")
case len(roots) > MaxRoots:
return Assessment{}, refuse(ReasonInvalid, "assessment has too many roots")
}
now = now.UTC()
byID := make(map[EvidenceID]Evidence, len(roots))
for _, r := range roots {
if r.Family() == FamilyNone {
return Assessment{}, refuse(ReasonInvalid, "root is not minted evidence")
}
rt := r.Target()
if (target.IsAddress() && rt != target) || (!target.IsAddress() && !target.Covers(rt)) {
return Assessment{}, refuse(ReasonInvalid, "root names a different target")
}
if r.ObservedAt().After(now.Add(maxFutureSkew)) {
return Assessment{}, refuse(ReasonInvalid, "root is dated in the future")
}
if prev, dup := byID[r.ID()]; dup && !prev.Equal(r) {
return Assessment{}, refuse(ReasonInvalid, "two different roots share one ID")
}
byID[r.ID()] = r
}
ids := make([]EvidenceID, 0, len(byID))
for id := range byID {
ids = append(ids, id)
}
sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
all := make([]Evidence, len(ids))
var fresh []Evidence
for i, id := range ids {
all[i] = byID[id]
if now.Before(rootExpiry(all[i])) {
fresh = append(fresh, all[i])
}
}
if len(fresh) == 0 {
return Assessment{}, refuse(ReasonStale, "no root is fresh")
}
a := Assessment{Roots: ids}
for _, r := range all {
for _, boundary := range []time.Time{rootExpiry(r), supportExpiry(r)} {
if boundary.After(now) && (a.NextChange.IsZero() || boundary.Before(a.NextChange)) {
a.NextChange = boundary
}
}
}
for _, r := range fresh {
if c, _ := r.Basis().Class(); c > a.Tier.Class {
a.Tier.Class = c
}
if r.Severity() > a.Tier.Severity {
a.Tier.Severity = r.Severity()
}
exp := rootExpiry(r)
if exp.After(a.EvidenceExpiry) {
a.EvidenceExpiry = exp
}
}
a.DirectC3 = a.Tier.Class == ClassC3
var classEnd, severityEnd time.Time
if target.IsAddress() {
classEnd = corroborationExpiry(fresh, all, now)
if !classEnd.IsZero() && !a.DirectC3 {
a.Tier.Class = ClassC3
a.Corroborated = true
}
}
// Independent proofs can outlive each other, including corroboration
// that keeps C3 after direct compromise evidence stops being fresh.
for _, r := range fresh {
exp := rootExpiry(r)
if c, _ := r.Basis().Class(); c == a.Tier.Class && exp.After(classEnd) {
classEnd = exp
}
if r.Severity() == a.Tier.Severity && exp.After(severityEnd) {
severityEnd = exp
}
}
a.ReassessBy = classEnd
if severityEnd.Before(a.ReassessBy) {
a.ReassessBy = severityEnd
}
return a, nil
}
// corroborationExpiry is the last instant until which any fresh local attack
// root has independent support from a different family and observation.
// Each pair ends at the earlier of local freshness and support expiry.
// Only the deadline matters, so tied pairs need no evidence-ID selection.
func corroborationExpiry(fresh, all []Evidence, now time.Time) time.Time {
var last time.Time
for _, a := range fresh {
if !a.Family().LocalAttack() {
continue
}
for _, b := range all {
if !b.Family().Independent() || b.Family() == a.Family() || sameObservation(a, b) {
continue
}
end := supportExpiry(b)
if exp := rootExpiry(a); exp.Before(end) {
end = exp
}
if now.Before(end) && end.After(last) {
last = end
}
}
}
return last
}
func supportExpiry(e Evidence) time.Time {
exp := e.ObservedAt().Add(SupportLookback)
if intel, ok := e.Intel(); ok && intel.Expires.Before(exp) {
exp = intel.Expires
}
return exp
}
package admission
import (
"errors"
"fmt"
)
// Attribution is how a parser learned the address a finding names. The
// parser that read the address sets it; a response path never
// does. Parked findings keep these numbers: append, never renumber.
type Attribution uint8
const (
// AttributionUnknown is the zero value: no parser vouched for the
// address. A parked finding without the optional kind decodes with
// it. It never authorizes a new mint.
AttributionUnknown Attribution = iota
// AttributionPeer: the transport peer of an authenticated mail, SSH,
// FTP or panel exchange, never an address in a message, mail header,
// URL or username.
AttributionPeer
// AttributionHTTPPeer: the direct socket peer of an HTTP request.
AttributionHTTPPeer
// AttributionTrustedHTTPChain: the client a configured trusted proxy
// forwarded, verified hop by hop. It is not a packet source.
AttributionTrustedHTTPChain
// AttributionSocketDestination: the remote end of a local outbound
// connection. It is not an inbound source.
AttributionSocketDestination
// AttributionDerived: intel or history resting on retained roots.
AttributionDerived
attributionEnd
)
var attributionNames = [...]string{"unknown", "peer", "http_peer", "trusted_http_chain", "socket_destination", "derived"}
// Valid reports a known kind other than unknown.
func (a Attribution) Valid() bool { return a > AttributionUnknown && a < attributionEnd }
func (a Attribution) String() string {
if a < attributionEnd {
return attributionNames[a]
}
return fmt.Sprintf("attribution(%d)", uint8(a))
}
// AttributionSet is a set of attribution kinds: the kinds a check accepts,
// or the kinds a producer mints.
type AttributionSet uint8
// Attributions returns the set of kinds, or an invalid set if any kind is
// invalid: a kind past the bitset must not vanish into a valid set.
func Attributions(kinds ...Attribution) AttributionSet {
var s AttributionSet
for _, k := range kinds {
if !k.Valid() {
return 1
}
s |= 1 << k
}
return s
}
// Has reports whether s holds the valid kind k.
func (s AttributionSet) Has(k Attribution) bool { return k.Valid() && s&(1<<k) != 0 }
// Valid reports whether s holds only valid kinds.
func (s AttributionSet) Valid() bool {
return s&1 == 0 && s>>attributionEnd == 0
}
// Within reports whether every kind of s is in t.
func (s AttributionSet) Within(t AttributionSet) bool { return s&^t == 0 }
// Attributions returns the kinds an address of family f can be learned by.
func (f Family) Attributions() AttributionSet {
switch f {
case FamilyHTTP:
return Attributions(AttributionHTTPPeer, AttributionTrustedHTTPChain)
case FamilyPanel, FamilyMail, FamilySSH, FamilyFTP:
return Attributions(AttributionPeer)
case FamilyNetwork:
return Attributions(AttributionSocketDestination)
case FamilyReputation, FamilyDerived:
return Attributions(AttributionDerived)
}
return 0
}
// Cause names why an attribution refusal was refused. Persisted in refusal
// summaries: append, never renumber.
type Cause uint8
const (
// CauseNoObservation: the finding names no observation a producer
// stamped.
CauseNoObservation Cause = iota + 1
// CauseKindNotAllowed: the producer did not declare the kind, or the
// check does not accept it; unknown is never accepted.
CauseKindNotAllowed
// CauseTargetMismatch: the response names a target its evidence does
// not attribute.
CauseTargetMismatch
// CauseUnreachable: the address is a client behind a trusted proxy, not
// a packet source this host can contain.
CauseUnreachable
// CauseProxyPeer: the address is a known CDN or proxy edge the
// operator did not configure as trusted, which carries many visitors
// and cannot be blocked as one of them.
CauseProxyPeer
// CauseRewrittenLog: the web server may replace the logged address;
// this build reads no separate connecting peer.
CauseRewrittenLog
causeEnd
)
var causeNames = [...]string{"", "no_observation", "kind_not_allowed", "target_mismatch", "unreachable", "proxy_peer", "rewritten_log"}
func (c Cause) Valid() bool { return c > 0 && c < causeEnd }
func (c Cause) String() string {
if c.Valid() {
return causeNames[c]
}
return fmt.Sprintf("cause(%d)", uint8(c))
}
// CauseOf returns the cause of an attribution refusal in err, or zero.
func CauseOf(err error) Cause {
var e *Error
if errors.As(err, &e) && e.Reason == ReasonAttribution {
return e.Cause
}
return 0
}
func refuseAttribution(c Cause, detail string) error {
return &Error{Reason: ReasonAttribution, Cause: c, Detail: detail}
}
// BindTarget refuses a response target e does not attribute. A response
// acts on e's address, or a service on it; only a range entry, whose
// registered checks count the addresses of a range, may act on a prefix
// holding it. via zero is e's own entry.
func BindTarget(target Target, via Entry, e Evidence) error {
if via == 0 {
via = e.Entry()
}
if target.Covers(e.Target()) && (target.IsAddress() || via.Range()) {
return nil
}
return refuseAttribution(CauseTargetMismatch, "the response target is not the attributed address, or a range this entry may act on")
}
package admission
import (
"encoding/binary"
"sort"
"time"
)
// Audit bounds. Every admitted transition leaves one
// row in the outbox until the audit consumer acknowledges it; the ledger
// reserves each row as a fixed slot.
const (
// MaxAuditRowBytes bounds one encoded audit row.
MaxAuditRowBytes = 1088
// AuditKeyLen is the length of a row's key: a kind byte, the action
// ID and the candidate's transition number.
AuditKeyLen = 1 + actionIDLen + 4
// AuditSlotBytes is what one row holds in the outbox, with its key.
AuditSlotBytes = AuditKeyLen + MaxAuditRowBytes
// AuditStepsPerAttempt is the most rows one attempt writes: its
// reservation, its execution and its outcome.
AuditStepsPerAttempt = 3
// AttemptAuditBytes is what a reservation holds for its attempt's rows.
AttemptAuditBytes = AuditStepsPerAttempt * AuditSlotBytes
)
// auditKind is the outbox key kind of audit rows.
const auditKind = 'a'
const auditRowVersion = 2
// AuditRow is the audit record of one admitted transition of an attempt:
// its reservation, its execution or its outcome. (Attempt.ID, Transition)
// keys it, so a consumer delivers it idempotently. It carries
// what was true at the transition: the intended effect, its expiry, the
// original finding and the root evidence.
type AuditRow struct {
Attempt Attempt
// Transition is the candidate's transition count after this step.
Transition uint32
// State is the attempt's state after the step: reserved, executing or
// an outcome with its disposition.
State State
Disposition Disposition
Lane Lane
At time.Time
ExpiresAt time.Time
Kind Kind
Target Target
Check string
FindingID string
Roots []EvidenceID
// Tier is the candidate's last assessment; zero when it has none.
Tier Tier
}
// NewAuditRow is the row for c's transition to attempt a's current state
// at time at.
func NewAuditRow(c Candidate, a AttemptRecord, tier Tier, at time.Time) (AuditRow, error) {
id, err := c.ID()
if err != nil {
return AuditRow{}, err
}
if a.Attempt.Candidate != id {
return AuditRow{}, refuse(ReasonInvalid, "audit row joins an attempt to another candidate")
}
if a.Attempt.Seq != c.Attempts || !a.ExpiresAt.Equal(c.ExpiresAt) {
return AuditRow{}, refuse(ReasonInvalid, "audit row attempt does not match the candidate's current attempt")
}
r := AuditRow{
Attempt: a.Attempt, Transition: c.Transitions, State: a.State, Disposition: a.Disposition, Lane: a.Lane,
At: at, ExpiresAt: a.ExpiresAt, Kind: c.Key.Kind, Target: c.Key.Target, Check: c.Check,
FindingID: c.FindingID, Roots: append([]EvidenceID(nil), c.Roots...), Tier: tier,
}
_, err = r.record()
return r, err
}
type auditRowRecord struct {
V uint8 `json:"v"`
ID ActionID `json:"id"`
Candidate CandidateID `json:"candidate"`
Seq uint32 `json:"seq"`
Transition uint32 `json:"transition"`
State State `json:"state"`
Disposition Disposition `json:"disposition,omitempty"`
Lane Lane `json:"lane,omitempty"`
At int64 `json:"at"`
ExpiresAt int64 `json:"expires_at"`
Kind Kind `json:"kind"`
Target string `json:"target"`
// Base64 bounds printable check names without JSON escape expansion.
Check []byte `json:"check"`
FindingID string `json:"finding_id"`
Roots []EvidenceID `json:"roots"`
Class Class `json:"class,omitempty"`
Severity Severity `json:"severity,omitempty"`
}
func (r AuditRow) record() (auditRowRecord, error) {
bad := func(detail string) (auditRowRecord, error) { return auditRowRecord{}, refuse(ReasonInvalid, detail) }
if err := r.Attempt.Validate(); err != nil {
return auditRowRecord{}, err
}
if r.Attempt.Seq > MaxAttempts {
return bad("audit row names no transition of an admitted attempt")
}
switch r.State {
case StateReserved, StateExecuting, StateVerified, StateFailed, StateUnknown, StateObserved:
default:
return bad("audit row state is not an attempt phase")
}
retirement := r.State == StateUnknown && r.Disposition == DispositionExpiredUnproven
if !retirement && !terminalDisposition(r.State, r.Disposition) {
return bad("audit row state and disposition disagree")
}
minimumTransition := 2 * r.Attempt.Seq
switch {
case retirement:
minimumTransition += 3
case r.State == StateExecuting || r.State == StateFailed || r.State == StateObserved:
minimumTransition++
case r.State == StateVerified || r.State == StateUnknown:
minimumTransition += 2
}
if r.Transition < minimumTransition {
return bad("audit row has too few transitions for its attempt phase")
}
at, okAt := unixNano(r.At)
expires, okExp := unixNano(r.ExpiresAt)
if !okAt || !okExp {
return bad("audit row times are not representable")
}
if r.Lane != 0 && !r.Lane.Valid() {
return bad("audit row names an unknown lane")
}
if err := ValidateKindTarget(r.Kind, r.Target); err != nil {
return auditRowRecord{}, err
}
if r.Disposition == DispositionNarrowed && r.Kind != KindChallenge && r.Kind != KindBlockService {
return bad("only a challenge or service block narrows")
}
if !boundedToken(r.Check, 64) || !lowerHex(r.FindingID, 16) {
return bad("audit row check or finding link is malformed")
}
if len(r.Roots) == 0 || len(r.Roots) > MaxRoots || !sort.SliceIsSorted(r.Roots, func(i, j int) bool { return r.Roots[i] < r.Roots[j] }) {
return bad("audit row roots are empty, too many or unsorted")
}
for i, id := range r.Roots {
if _, err := ParseEvidenceID(string(id)); err != nil || (i > 0 && r.Roots[i-1] == id) {
return bad("audit row roots are malformed or repeated")
}
}
if r.Tier != (Tier{}) && !r.Tier.Valid() {
return bad("audit row tier is partial")
}
return auditRowRecord{
V: auditRowVersion, ID: r.Attempt.ID, Candidate: r.Attempt.Candidate, Seq: r.Attempt.Seq,
Transition: r.Transition, State: r.State, Disposition: r.Disposition, Lane: r.Lane, At: at,
ExpiresAt: expires, Kind: r.Kind, Target: r.Target.Key(), Check: []byte(r.Check), FindingID: r.FindingID,
Roots: r.Roots, Class: r.Tier.Class, Severity: r.Tier.Severity,
}, nil
}
func (r AuditRow) MarshalBinary() ([]byte, error) {
rec, err := r.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalAuditRow decodes a stored row and checks its invariants.
func UnmarshalAuditRow(data []byte) (AuditRow, error) {
var rec auditRowRecord
if err := openRecord(data, &rec); err != nil {
return AuditRow{}, err
}
if rec.V != auditRowVersion || rec.Roots == nil {
return AuditRow{}, ErrCorruptRecord
}
target, err := ParseTargetKey(rec.Target, Caps{IPv6: true})
if err != nil {
return AuditRow{}, ErrCorruptRecord
}
r := AuditRow{
Transition: rec.Transition, State: rec.State, Disposition: rec.Disposition, Lane: rec.Lane,
At: fromNano(rec.At), ExpiresAt: fromNano(rec.ExpiresAt), Kind: rec.Kind, Target: target,
Check: string(rec.Check), FindingID: rec.FindingID, Roots: rec.Roots, Tier: Tier{Class: rec.Class, Severity: rec.Severity},
}
r.Attempt, err = NewAttempt(rec.Candidate, rec.Seq)
if err != nil || r.Attempt.ID != rec.ID {
return AuditRow{}, ErrCorruptRecord
}
if again, err := r.MarshalBinary(); err != nil || string(again) != string(data) {
return AuditRow{}, ErrCorruptRecord
}
return r, nil
}
// AuditID names one row: the attempt and the candidate's transition.
type AuditID struct {
Action ActionID
Transition uint32
}
// ID is the row's name.
func (r AuditRow) ID() AuditID { return AuditID{Action: r.Attempt.ID, Transition: r.Transition} }
// AuditAck acknowledges the row ID names, written at At. The time fences an
// acknowledgement held past its row's retirement: a re-minted attempt
// writes a later row under the same ID, and the stale acknowledgement must
// not remove it.
type AuditAck struct {
ID AuditID
At time.Time
}
// Ack is the row's acknowledgement.
func (r AuditRow) Ack() AuditAck { return AuditAck{ID: r.ID(), At: r.At} }
// Key is the outbox key of the row id names.
func (id AuditID) Key() []byte {
return binary.BigEndian.AppendUint32(AuditPrefix(id.Action), id.Transition)
}
// AuditPrefix is the key prefix every row of one attempt shares.
func AuditPrefix(id ActionID) []byte { return append([]byte{auditKind}, id...) }
// Key is the row's outbox key. Rows of one attempt sort by transition.
func (r AuditRow) Key() []byte { return r.ID().Key() }
// ParseAuditKey splits a row key into its action and transition.
func ParseAuditKey(k []byte) (ActionID, uint32, error) {
if len(k) != AuditKeyLen || k[0] != auditKind {
return "", 0, ErrCorruptRecord
}
id, err := ParseActionID(string(k[1 : 1+actionIDLen]))
transition := binary.BigEndian.Uint32(k[1+actionIDLen:])
if err != nil || transition == 0 {
return "", 0, ErrCorruptRecord
}
return id, transition, nil
}
package admission
import (
"bytes"
"math"
"sort"
"strings"
"time"
)
// QueueAgeLimit is the longest a candidate waits. It ages out at
// min(FirstQueued+QueueAgeLimit, evidence expiry); a duplicate report never
// extends that.
const QueueAgeLimit = 2 * time.Hour
// MaxCandidateBytes bounds one encoded candidate record. Storage
// accounting reserves it per candidate; the invariants keep every valid
// record well inside it.
const MaxCandidateBytes = 4096
// MaxAttempts is how many proven-failure attempts one candidate generation
// may make. An unknown outcome is never retried.
const MaxAttempts = 3
// maxRetryBackoff caps the wait before a failed attempt is retried.
const maxRetryBackoff = time.Minute
const candidateVersion = 1
// linkedCandidateVersion encodes a candidate that carries an EffectLink.
// Older builds refuse it, and the ledger's schema says so first.
const linkedCandidateVersion = 2
const attemptVersion = 1
// RetryBackoff is the wait after the proven failure of attempt seq: one
// second, doubling, capped at a minute.
func RetryBackoff(seq uint32) time.Duration {
if seq == 0 {
return 0
}
if seq > 7 {
return maxRetryBackoff
}
return min(time.Second<<(seq-1), maxRetryBackoff)
}
// ParseEvidenceID accepts only the derived form.
func ParseEvidenceID(s string) (EvidenceID, error) {
rest, ok := strings.CutPrefix(s, "ev_")
if !ok || !lowerHex(rest, 32) {
return "", refuse(ReasonInvalid, "malformed evidence ID")
}
return EvidenceID(s), nil
}
// Candidate is the ledger's record of one queued response. The engine owns
// it; every field is checked on encode and decode. These local invariants
// bound possible histories; the ledger also checks the actual attempt rows.
type Candidate struct {
Key CandidateKey
Scope Scope
// Entry, Check and FindingID come from the primary root evidence.
Entry Entry
Check string
FindingID string
// Roots are the validated root evidence IDs, sorted and unique.
Roots []EvidenceID
FirstQueued time.Time
AgeOut time.Time
State State
// Reason is a deferral reason while queued, or the reason a queued
// candidate was refused, withheld or dropped.
Reason Reason
Disposition Disposition
// Attempts is the sequence of the latest attempt; zero before the
// first reservation.
Attempts uint32
// ExpiresAt is the absolute effect expiry fixed by the first
// reservation. Retries keep it.
ExpiresAt time.Time
// PreviewTTL is the legacy path's selected lifetime, fixed when the
// candidate is queued. Zero uses the configured lifetime at reservation.
PreviewTTL time.Duration
// NotBefore is when a candidate returned by a proven failure may be
// reserved again.
NotBefore time.Time
// Transitions counts recorded changes; (ID, Transitions) keys an
// idempotent audit event.
Transitions uint32
// Link names the block that already enforces the effect of a candidate
// refused as an existing effect; zero otherwise.
Link EffectLink
}
// EffectLink names the block that already enforces a refused candidate's
// effect (an existing exact effect with its verified action link): the
// admission action that wrote it, or the firewall's "eff_" name
// of an operator or earlier block, and when it ends (zero: permanent).
type EffectLink struct {
Effect string
ExpiresAt time.Time
}
type linkRecord struct {
Effect string `json:"effect"`
ExpiresAt int64 `json:"expires_at,omitempty"`
}
// validEffectName accepts "act_" or "eff_" and 32 lowercase hex digits.
func validEffectName(s string) bool {
rest, ok := strings.CutPrefix(s, "act_")
if !ok {
rest, ok = strings.CutPrefix(s, "eff_")
}
return ok && lowerHex(rest, 32)
}
// ID derives the candidate ID from its key.
func (c Candidate) ID() (CandidateID, error) { return c.Key.ID() }
type candidateRecord struct {
V uint8 `json:"v"`
Kind Kind `json:"kind"`
Target string `json:"target"`
Episode string `json:"episode"`
Generation uint32 `json:"generation"`
OwnerAccount string `json:"owner_account,omitempty"`
OwnerGeneration uint64 `json:"owner_generation,omitempty"`
Effect Effect `json:"effect"`
Entry Entry `json:"entry"`
Check string `json:"check"`
FindingID string `json:"finding_id"`
Roots []EvidenceID `json:"roots"`
FirstQueued int64 `json:"first_queued"`
AgeOut int64 `json:"age_out"`
State State `json:"state"`
Reason Reason `json:"reason,omitempty"`
Disposition Disposition `json:"disposition,omitempty"`
Attempts uint32 `json:"attempts,omitempty"`
ExpiresAt int64 `json:"expires_at,omitempty"`
PreviewTTL int64 `json:"preview_ttl,omitempty"`
NotBefore int64 `json:"not_before,omitempty"`
Transitions uint32 `json:"transitions"`
Link *linkRecord `json:"link,omitempty"`
}
// Validate checks the record's invariants.
func (c Candidate) Validate() error {
_, err := c.record()
return err
}
func (c Candidate) record() (candidateRecord, error) {
bad := func(detail string) (candidateRecord, error) {
return candidateRecord{}, refuse(ReasonInvalid, detail)
}
if _, err := c.Key.ID(); err != nil {
return candidateRecord{}, err
}
if c.Scope.Effect != c.Key.Kind.Effect() {
return bad("candidate scope is not its kind's action family")
}
if o := c.Scope.Owner; (o.IsHost() && o.generation != 0) || (!o.IsHost() && (!ValidAccountName(o.account) || o.generation == 0)) {
return bad("candidate owner is malformed")
}
if !c.Entry.Valid() || !boundedToken(c.Check, 64) || !lowerHex(c.FindingID, 16) {
return bad("candidate entry, check or finding link is malformed")
}
if len(c.Roots) == 0 || len(c.Roots) > MaxRoots || !sort.SliceIsSorted(c.Roots, func(i, j int) bool { return c.Roots[i] < c.Roots[j] }) {
return bad("candidate roots are empty, too many or unsorted")
}
for i, id := range c.Roots {
if _, err := ParseEvidenceID(string(id)); err != nil || (i > 0 && c.Roots[i-1] == id) {
return bad("candidate roots are malformed or repeated")
}
}
first, okFirst := unixNano(c.FirstQueued)
ageOut, okAge := unixNano(c.AgeOut)
if !okFirst || !okAge || !c.AgeOut.After(c.FirstQueued) || c.AgeOut.After(c.FirstQueued.Add(QueueAgeLimit)) {
return bad("candidate queue times are inconsistent")
}
if !c.State.Valid() || !terminalDisposition(c.State, c.Disposition) || !stateReason(c.State, c.Reason) {
return bad("candidate state, disposition and reason disagree")
}
if c.PreviewTTL < 0 {
return bad("candidate preview lifetime is negative")
}
if c.Disposition == DispositionNarrowed && c.Key.Kind != KindChallenge && c.Key.Kind != KindBlockService {
return bad("only a challenge or service block narrows")
}
switch {
case c.Attempts > MaxAttempts:
return bad("candidate exceeds its attempts")
case c.Attempts == 0 && c.State != StateQueued && !c.State.Terminal():
return bad("candidate is past the queue without an attempt")
case c.Attempts == 0 && (c.State == StateVerified || c.State == StateFailed || c.State == StateUnknown || c.State == StateObserved):
return bad("candidate has an outcome without an attempt")
case c.State == StateFailed && c.Attempts != MaxAttempts:
return bad("candidate failed before exhausting its attempts")
case c.State == StateQueued && c.Attempts > 0 && (c.Attempts == MaxAttempts || c.NotBefore.IsZero()):
return bad("queued retry has no delay or has exhausted its attempts")
case c.Attempts == MaxAttempts && (c.State == StateRefused || c.State == StateWithheld || c.State == StateDropped || c.State == StateDryRun):
return bad("exhausted candidate cannot end through the queue")
}
// Even immediate failures must wait out each preceding retry backoff.
// Use time arithmetic so a deadline near the nanosecond limit cannot wrap.
earliestReservation := c.FirstQueued
for seq := uint32(1); seq < c.Attempts; seq++ {
earliestReservation = earliestReservation.Add(RetryBackoff(seq))
}
var expires, notBefore int64
if c.Attempts == 0 {
if !c.ExpiresAt.IsZero() {
return bad("candidate has an expiry before its first reservation")
}
} else {
var ok bool
if expires, ok = unixNano(c.ExpiresAt); !ok || !c.ExpiresAt.After(earliestReservation) || !c.AgeOut.After(earliestReservation) {
return bad("candidate deadlines cannot accommodate its attempts")
}
}
if !c.NotBefore.IsZero() {
var ok bool
if notBefore, ok = unixNano(c.NotBefore); !ok || c.State != StateQueued || c.Attempts == 0 || c.NotBefore.Before(earliestReservation.Add(RetryBackoff(c.Attempts))) {
return bad("only a candidate returned by a failure waits for a retry time")
}
}
// Creation counts once; every preceding attempt needs a reservation and
// a failed outcome. Execute is optional only for a proven failure.
minimumTransitions := 1 + 2*c.Attempts
switch c.State {
case StateReserved:
minimumTransitions--
case StateVerified, StateUnknown, StateRefused, StateWithheld, StateDropped, StateDryRun:
minimumTransitions++
case StateQueued:
if c.Reason != 0 {
minimumTransitions++
}
}
if c.Transitions < minimumTransitions {
return bad("candidate has too few transitions for its state and attempts")
}
rec := candidateRecord{
V: candidateVersion, Kind: c.Key.Kind, Target: c.Key.Target.Key(), Episode: c.Key.Episode.String(),
Generation: c.Key.Generation, OwnerAccount: c.Scope.Owner.account, OwnerGeneration: c.Scope.Owner.generation,
Effect: c.Scope.Effect, Entry: c.Entry, Check: c.Check, FindingID: c.FindingID, Roots: c.Roots,
FirstQueued: first, AgeOut: ageOut, State: c.State, Reason: c.Reason, Disposition: c.Disposition,
Attempts: c.Attempts, ExpiresAt: expires, PreviewTTL: int64(c.PreviewTTL), NotBefore: notBefore, Transitions: c.Transitions,
}
if c.Link != (EffectLink{}) {
if c.State != StateRefused || c.Reason != ReasonExistingEffect {
return bad("only a refusal as an existing effect names a block")
}
link := linkRecord{Effect: c.Link.Effect}
if !c.Link.ExpiresAt.IsZero() {
var ok bool
if link.ExpiresAt, ok = unixNano(c.Link.ExpiresAt); !ok || link.ExpiresAt <= 0 {
return bad("linked block expiry is out of range")
}
}
if !validEffectName(link.Effect) {
return bad("linked block name is malformed")
}
rec.V, rec.Link = linkedCandidateVersion, &link
}
return rec, nil
}
// MarshalBinary validates and encodes c. The invariants keep every valid
// candidate well inside MaxCandidateBytes; the decoder repeats the same
// invariants on what it reads back.
func (c Candidate) MarshalBinary() ([]byte, error) {
rec, err := c.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// MaxBytes is the largest encoding c can reach while its identity, scope,
// roots and queue times stay as they are: its state, reason, outcome,
// attempts, deadlines and transition count at their widest. Storage
// accounting reserves it for a candidate whose roots are fixed.
func (c Candidate) MaxBytes() (int, error) {
rec, err := c.record()
if err != nil {
return 0, err
}
rec.State, rec.Reason, rec.Disposition = stateEnd-1, reasonEnd-1, dispositionEnd-1
rec.Attempts, rec.ExpiresAt, rec.NotBefore, rec.Transitions = MaxAttempts, math.MaxInt64, math.MaxInt64, math.MaxUint32
if rec.Link != nil {
rec.Link = &linkRecord{Effect: rec.Link.Effect, ExpiresAt: math.MaxInt64}
}
data, err := sealRecord(rec)
return len(data), err
}
// UnmarshalCandidate decodes a stored candidate and checks its invariants.
func UnmarshalCandidate(data []byte) (Candidate, error) {
var rec candidateRecord
if err := openRecord(data, &rec); err != nil {
return Candidate{}, err
}
if (rec.V != candidateVersion && rec.V != linkedCandidateVersion) || rec.Roots == nil {
return Candidate{}, ErrCorruptRecord
}
target, err := ParseTargetKey(rec.Target, Caps{IPv6: true})
if err != nil {
return Candidate{}, ErrCorruptRecord
}
episode, err := ParseEpisodeID(rec.Episode)
if err != nil {
return Candidate{}, ErrCorruptRecord
}
c := Candidate{
Key: CandidateKey{Kind: rec.Kind, Target: target, Episode: episode, Generation: rec.Generation},
Scope: Scope{Owner: Owner{account: rec.OwnerAccount, generation: rec.OwnerGeneration}, Effect: rec.Effect},
Entry: rec.Entry, Check: rec.Check, FindingID: rec.FindingID, Roots: rec.Roots,
FirstQueued: fromNano(rec.FirstQueued), AgeOut: fromNano(rec.AgeOut), State: rec.State, Reason: rec.Reason,
Disposition: rec.Disposition, Attempts: rec.Attempts, ExpiresAt: fromNano(rec.ExpiresAt), PreviewTTL: time.Duration(rec.PreviewTTL),
NotBefore: fromNano(rec.NotBefore), Transitions: rec.Transitions,
}
if rec.Link != nil {
c.Link = EffectLink{Effect: rec.Link.Effect, ExpiresAt: fromNano(rec.Link.ExpiresAt)}
}
// Re-encoding must reproduce the stored bytes: every field survived
// decoding and passes the invariants.
if again, err := c.MarshalBinary(); err != nil || !bytes.Equal(again, data) {
return Candidate{}, ErrCorruptRecord
}
return c, nil
}
// AttemptRecord is the ledger's record of one admitted attempt.
type AttemptRecord struct {
Attempt Attempt
// State is reserved, executing, verified, failed or unknown.
State State
Disposition Disposition
ExpiresAt time.Time
Reserved time.Time
// Finished is zero until the attempt has an outcome.
Finished time.Time
// Lane is the lane the reservation was admitted on and charged to. It
// is zero only on attempts reserved before the ledger kept a ceiling.
Lane Lane
// Retired marks an unknown outcome whose uncertainty recovery retired
// after its effect expired; the outcome itself stays unknown.
Retired bool
}
type attemptRecord struct {
V uint8 `json:"v"`
ID ActionID `json:"id"`
Candidate CandidateID `json:"candidate"`
Seq uint32 `json:"seq"`
Prev ActionID `json:"prev,omitempty"`
State State `json:"state"`
Disposition Disposition `json:"disposition,omitempty"`
ExpiresAt int64 `json:"expires_at"`
Reserved int64 `json:"reserved"`
Finished int64 `json:"finished,omitempty"`
Lane Lane `json:"lane,omitempty"`
Retired bool `json:"retired,omitempty"`
}
func (a AttemptRecord) record() (attemptRecord, error) {
bad := func(detail string) (attemptRecord, error) { return attemptRecord{}, refuse(ReasonInvalid, detail) }
if err := a.Attempt.Validate(); err != nil {
return attemptRecord{}, err
}
if a.Attempt.Seq > MaxAttempts {
return bad("attempt exceeds the candidate limit")
}
switch a.State {
case StateReserved, StateExecuting, StateVerified, StateFailed, StateUnknown, StateObserved:
default:
return bad("attempt state is not an attempt phase")
}
if !terminalDisposition(a.State, a.Disposition) {
return bad("attempt state and disposition disagree")
}
reserved, okR := unixNano(a.Reserved)
expires, okE := unixNano(a.ExpiresAt)
if !okR || !okE || !a.ExpiresAt.After(a.Reserved) {
return bad("attempt times are inconsistent")
}
var finished int64
if a.State.Terminal() {
var ok bool
if finished, ok = unixNano(a.Finished); !ok || a.Finished.Before(a.Reserved) {
return bad("attempt outcome has no valid finish time")
}
} else if !a.Finished.IsZero() {
return bad("an open attempt has a finish time")
}
if a.Lane != 0 && !a.Lane.Valid() {
return bad("attempt names an unknown lane")
}
if a.Retired && a.State != StateUnknown {
return bad("only an unknown outcome is retired")
}
return attemptRecord{
V: attemptVersion, ID: a.Attempt.ID, Candidate: a.Attempt.Candidate, Seq: a.Attempt.Seq, Prev: a.Attempt.Prev,
State: a.State, Disposition: a.Disposition, ExpiresAt: expires, Reserved: reserved, Finished: finished, Lane: a.Lane,
Retired: a.Retired,
}, nil
}
// Validate checks the record's invariants.
func (a AttemptRecord) Validate() error {
_, err := a.record()
return err
}
func (a AttemptRecord) MarshalBinary() ([]byte, error) {
rec, err := a.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalAttempt decodes a stored attempt and checks its invariants.
func UnmarshalAttempt(data []byte) (AttemptRecord, error) {
var rec attemptRecord
if err := openRecord(data, &rec); err != nil {
return AttemptRecord{}, err
}
if rec.V != attemptVersion {
return AttemptRecord{}, ErrCorruptRecord
}
a := AttemptRecord{
Attempt: Attempt{ID: rec.ID, Candidate: rec.Candidate, Seq: rec.Seq, Prev: rec.Prev},
State: rec.State, Disposition: rec.Disposition, ExpiresAt: fromNano(rec.ExpiresAt),
Reserved: fromNano(rec.Reserved), Finished: fromNano(rec.Finished), Lane: rec.Lane, Retired: rec.Retired,
}
if again, err := a.record(); err != nil || again != rec {
return AttemptRecord{}, ErrCorruptRecord
}
return a, nil
}
package admission
import (
"bytes"
"encoding/binary"
"time"
)
// MaxCeiling bounds the hourly ceiling the ledger accepts. Every charge
// retained in the window is one stored record, so it also bounds them.
const MaxCeiling = 20000
// CeilingWindow is the rolling window the ceiling counts charges over.
const CeilingWindow = time.Hour
// CeilingLanes splits the hourly ceiling L into the general allowance
// G = L-R and the reserved allowance R = ceil(L/5), which only direct
// compromise and independently corroborated work may spend. For
// L=1 only the reserved lane exists.
func CeilingLanes(limit uint32) (general, reserved uint32) {
reserved = limit / 5
if limit%5 != 0 {
reserved++
}
return limit - reserved, reserved
}
// BucketCap is the most credit a lane of size units per hour may save: ten
// minutes of its rate, and at least one unit so a small lane can run. An
// empty lane never runs.
func BucketCap(size uint32) uint32 {
if size == 0 {
return 0
}
return max(1, size/6)
}
// CeilingCost is what one attempt of kind k charges the ceiling. Each new
// address, prefix, promotion or service tuple costs one unit. Challenge
// work has its own bound and is never charged: it cannot
// create a block.
func (k Kind) CeilingCost() uint32 {
if k == KindChallenge {
return 0
}
return 1
}
const ceilingStateVersion = 1
// unitTicks is one unit of credit. A lane of S units per hour gains S ticks
// per nanosecond of elapsed time, so credit accrues exactly, without
// rounding, and an hour earns S units.
const unitTicks = uint64(time.Hour)
// Meter is one of the ceiling's two allowances.
type Meter struct {
// Credit is the saved pacing credit in ticks.
Credit uint64
// Used is the units charged inside the rolling window.
Used uint32
}
// Units is the whole units of saved credit.
func (m Meter) Units() uint32 { return uint32(min(m.Credit/unitTicks, MaxCeiling)) }
// CeilingState is the ledger's record of the emergency ceiling:
// the effective limit, each lane's token bucket and its usage inside the
// rolling window. The zero value is an upgraded ledger's first state.
type CeilingState struct {
// Limit is the effective hourly ceiling L; zero until the engine sets
// one, and nothing is charged before it does.
Limit uint32
// Fill makes the first limit fill each bucket to its cap. A new ledger
// starts with it; an upgraded one starts without credit, since its
// recent spend is unknown.
Fill bool
// Elapsed is the admission time elapsed within boots since metering
// began. Downtime between boots is never credited.
Elapsed time.Duration
// General and Reserved are the two allowances. The direct and
// corroborated lanes share the reserved one.
General, Reserved Meter
}
type allowance struct {
m *Meter
size uint32
}
// allowances pairs each meter with its lane size under the current limit.
func (s *CeilingState) allowances() [2]allowance {
g, r := CeilingLanes(s.Limit)
return [2]allowance{{&s.General, g}, {&s.Reserved, r}}
}
// meter is the allowance lane l spends: the direct and corroborated lanes
// share the reserved one.
func (s *CeilingState) meter(l Lane) allowance {
if l == LaneGeneral {
return s.allowances()[0]
}
return s.allowances()[1]
}
func room(size, used uint32) uint32 {
if used >= size {
return 0
}
return size - used
}
// Budget is the units lane l can charge now: its saved credit, bounded by
// what its own allowance and the whole ceiling leave in the window.
func (s CeilingState) Budget(l Lane) uint32 {
if !l.Valid() {
return 0
}
a := s.meter(l)
return min(a.m.Units(), room(a.size, a.m.Used), room(s.Limit, s.General.Used+s.Reserved.Used))
}
// SetLimit applies a new effective limit. The first limit of a new ledger
// fills each bucket to its cap, less the units its lane already spent in
// the window: the legacy spend imported with it. Every later limit clips
// saved credit to the new caps and never tops it up, so no restart or
// reload manufactures credit. Charges keep their lanes: a lane whose
// retained usage exceeds a reduced allowance waits for them to age out.
func (s CeilingState) SetLimit(limit uint32) (CeilingState, error) {
if limit == 0 || limit > MaxCeiling {
return s, refuse(ReasonInvalid, "ceiling is out of range")
}
s.Limit = limit
for _, a := range s.allowances() {
full := uint64(BucketCap(a.size)) * unitTicks
switch {
case s.Fill:
a.m.Credit = full - min(full, uint64(a.m.Used)*unitTicks)
case a.m.Credit > full:
a.m.Credit = full
}
}
s.Fill = false
return s, nil
}
// LegacySpend is what the legacy hourly counter recorded before a new
// ledger takes its first limit.
type LegacySpend struct {
// Units were spent no later than At, the end of the legacy hour that
// counted them.
Units uint32
At time.Time
// Unknown is set when the legacy counter could not be read and
// validated. A failed import is not an empty ledger: the first limit
// then grants no credit, as missing pacing history starts at zero.
Unknown bool
}
// Import takes the first limit together with the legacy spend before it.
// The units become charges dated at At, on the general lane up to its size
// and the rest on the reserved allowance, at most MaxCeiling in all: a
// count past every ceiling leaves no budget either way. The first fill is
// reduced by each lane's charges. A ledger that already has a limit
// imports nothing; a restart cannot import the same hour twice.
func (s CeilingState) Import(limit uint32, spend LegacySpend) (CeilingState, []Charge, error) {
if s.Limit != 0 {
return s, nil, ErrTransitionConflict
}
next := s
var charges []Charge
switch {
case spend.Unknown:
next.Fill = false
case spend.Units > 0:
if at, ok := unixNano(spend.At); !ok || at <= 0 {
return s, nil, refuse(ReasonInvalid, "legacy spend names no hour")
}
units := min(spend.Units, MaxCeiling)
g, _ := CeilingLanes(limit)
general := min(units, g)
// The reserved allowance is spent through either of its turns; a
// charge's lane only selects the allowance.
var seq uint32
for _, part := range []struct {
lane Lane
units uint32
}{{LaneGeneral, general}, {LaneDirect, units - general}} {
for part.units > 0 {
cost := min(part.units, MaxMemberCost)
seq++
charges = append(charges, Charge{At: spend.At, Action: LegacyActionID(spend.At, seq), Lane: part.lane, Cost: cost, Elapsed: s.Elapsed})
part.units -= cost
}
}
next.General.Used += general
next.Reserved.Used += units - general
}
next, err := next.SetLimit(limit)
if err != nil {
return s, nil, err
}
return next, charges, nil
}
// Advance credits elapsed admission time: it accumulates Elapsed and
// refills each running lane at its hourly rate, up to its cap. Exhausting
// the elapsed representation refuses the whole update, preserving charge ages.
func (s CeilingState) Advance(elapsed time.Duration) (CeilingState, error) {
if elapsed <= 0 {
return s, nil
}
if elapsed > time.Duration(1<<63-1)-s.Elapsed {
return s, refuse(ReasonEngineUnavailable, "ceiling elapsed time is exhausted")
}
s.Elapsed += elapsed
e := uint64(elapsed)
for _, a := range s.allowances() {
full := uint64(BucketCap(a.size)) * unitTicks
if a.m.Credit >= full {
continue
}
rate := uint64(a.size)
// Compare elapsed time with the time left to fill before
// multiplying, so a long gap saturates instead of overflowing.
if need := full - a.m.Credit; e >= (need+rate-1)/rate {
a.m.Credit = full
} else {
a.m.Credit += rate * e
}
}
return s, nil
}
// Charge spends cost units of lane l's budget. A refusal changes nothing.
func (s CeilingState) Charge(l Lane, cost uint32) (CeilingState, error) {
switch {
case !l.Valid() || cost == 0 || cost > MaxMemberCost:
return s, refuse(ReasonInvalid, "charge has no lane or an invalid cost")
case s.Limit == 0:
return s, refuse(ReasonEngineUnavailable, "ceiling is not set")
case cost > s.Budget(l):
return s, refuse(ReasonCeiling, "lane has no ceiling budget for the charge")
}
a := s.meter(l)
a.m.Credit -= uint64(cost) * unitTicks
a.m.Used += cost
return s, nil
}
// Release returns a charge's units to the window once it has aged out. It
// never refunds credit.
func (s CeilingState) Release(l Lane, cost uint32) (CeilingState, error) {
if !l.Valid() {
return s, ErrCorruptRecord
}
a := s.meter(l)
if a.m.Used < cost {
return s, ErrCorruptRecord
}
a.m.Used -= cost
return s, nil
}
// Uncharge removes a charge a model does not count: its units leave the
// window and its credit returns to its lane, never past the bucket's cap.
// A forecast seeds itself without preview charges this way; the committed
// ceiling never uncharges.
func (s CeilingState) Uncharge(l Lane, cost uint32) (CeilingState, error) {
next, err := s.Release(l, cost)
if err != nil {
return s, err
}
a := next.meter(l)
full := uint64(BucketCap(a.size)) * unitTicks
a.m.Credit = min(full, a.m.Credit+uint64(cost)*unitTicks)
return next, nil
}
type ceilingStateRecord struct {
V uint8 `json:"v"`
Limit uint32 `json:"limit,omitempty"`
Fill bool `json:"fill,omitempty"`
Elapsed int64 `json:"elapsed,omitempty"`
GeneralCredit uint64 `json:"general_credit,omitempty"`
GeneralUsed uint32 `json:"general_used,omitempty"`
ReservedCredit uint64 `json:"reserved_credit,omitempty"`
ReservedUsed uint32 `json:"reserved_used,omitempty"`
}
func (s CeilingState) record() (ceilingStateRecord, error) {
bad := func(detail string) (ceilingStateRecord, error) {
return ceilingStateRecord{}, refuse(ReasonInvalid, detail)
}
g, r := CeilingLanes(s.Limit)
switch {
case s.Limit > MaxCeiling:
return bad("ceiling is out of range")
case s.Fill && s.Limit != 0:
return bad("a set ceiling cannot still be waiting to fill")
case s.Elapsed < 0:
return bad("ceiling elapsed time is negative")
case s.General.Credit > uint64(BucketCap(g))*unitTicks, s.Reserved.Credit > uint64(BucketCap(r))*unitTicks:
return bad("ceiling credit exceeds its cap")
case uint64(s.General.Used)+uint64(s.Reserved.Used) > MaxCeiling:
return bad("ceiling usage exceeds any ceiling")
}
return ceilingStateRecord{
V: ceilingStateVersion, Limit: s.Limit, Fill: s.Fill, Elapsed: int64(s.Elapsed),
GeneralCredit: s.General.Credit, GeneralUsed: s.General.Used,
ReservedCredit: s.Reserved.Credit, ReservedUsed: s.Reserved.Used,
}, nil
}
// Validate checks the state's invariants.
func (s CeilingState) Validate() error {
_, err := s.record()
return err
}
func (s CeilingState) MarshalBinary() ([]byte, error) {
rec, err := s.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalCeilingState decodes a stored ceiling state and checks its
// invariants.
func UnmarshalCeilingState(data []byte) (CeilingState, error) {
var rec ceilingStateRecord
if err := openRecord(data, &rec); err != nil {
return CeilingState{}, err
}
s := CeilingState{
Limit: rec.Limit, Fill: rec.Fill, Elapsed: time.Duration(rec.Elapsed),
General: Meter{Credit: rec.GeneralCredit, Used: rec.GeneralUsed},
Reserved: Meter{Credit: rec.ReservedCredit, Used: rec.ReservedUsed},
}
if again, err := s.record(); rec.V != ceilingStateVersion || err != nil || again != rec {
return CeilingState{}, ErrCorruptRecord
}
return s, nil
}
const chargeVersion = 1
// chargeKeyLen is the length of a charge key: its time, then its action.
const chargeKeyLen = 8 + len("act_") + 32
// Charge is one spend of the ceiling, kept while it counts in the rolling
// window. At is the admission time of the reservation that spent
// it, and Elapsed the ceiling's elapsed time then.
type Charge struct {
At time.Time
Action ActionID
Lane Lane
Cost uint32
Elapsed time.Duration
}
// Releasable reports whether the charge has left the window at admission
// time now, with elapsed time metered: a full window of wall time and of
// elapsed time must both have passed, so neither a forward clock step nor
// downtime ends it early. A charge dated in the future counts until its own
// time plus the window.
func (c Charge) Releasable(now time.Time, elapsed time.Duration) bool {
return !now.Before(c.At.Add(CeilingWindow)) && elapsed-c.Elapsed >= CeilingWindow
}
type chargeRecord struct {
V uint8 `json:"v"`
Lane Lane `json:"lane"`
Cost uint32 `json:"cost"`
Elapsed int64 `json:"elapsed,omitempty"`
}
func (c Charge) record() ([]byte, chargeRecord, error) {
bad := func(detail string) ([]byte, chargeRecord, error) {
return nil, chargeRecord{}, refuse(ReasonInvalid, detail)
}
at, ok := unixNano(c.At)
switch {
case !ok || at < 0:
return bad("charge has no time after the epoch")
case !c.Lane.Valid() || c.Cost == 0 || c.Cost > MaxMemberCost:
return bad("charge has no lane or an invalid cost")
case c.Elapsed < 0:
return bad("charge elapsed time is negative")
}
if _, err := ParseActionID(string(c.Action)); err != nil {
return bad("charge names no action")
}
// Big-endian nanoseconds sort by time, so a walk in key order meets the
// oldest charges first.
key := binary.BigEndian.AppendUint64(make([]byte, 0, chargeKeyLen), uint64(at))
return append(key, c.Action...), chargeRecord{V: chargeVersion, Lane: c.Lane, Cost: c.Cost, Elapsed: int64(c.Elapsed)}, nil
}
// Validate checks the charge's invariants.
func (c Charge) Validate() error {
_, _, err := c.record()
return err
}
// Key is the charge's storage key.
func (c Charge) Key() ([]byte, error) {
key, _, err := c.record()
return key, err
}
func (c Charge) MarshalBinary() ([]byte, error) {
_, rec, err := c.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalCharge decodes a stored charge from its key and value and checks
// its invariants.
func UnmarshalCharge(key, data []byte) (Charge, error) {
var rec chargeRecord
if err := openRecord(data, &rec); err != nil {
return Charge{}, err
}
if len(key) != chargeKeyLen || rec.V != chargeVersion {
return Charge{}, ErrCorruptRecord
}
at := binary.BigEndian.Uint64(key[:8])
if at > 1<<63-1 {
return Charge{}, ErrCorruptRecord
}
c := Charge{At: fromNano(int64(at)), Action: ActionID(key[8:]), Lane: rec.Lane, Cost: rec.Cost, Elapsed: time.Duration(rec.Elapsed)}
again, againRec, err := c.record()
if err != nil || !bytes.Equal(again, key) || againRec != rec {
return Charge{}, ErrCorruptRecord
}
return c, nil
}
// UntilBudget is how much longer lane l must wait before it can charge one
// unit, given the retained charges in key order: until its credit reaches a
// unit and enough charges have left the window to make room in its
// allowance and in the whole ceiling. Charges leave in key order, so one
// waits for every charge before it. Zero means now; false means never under
// the current limit. A wall deadline is measured on the admission clock and
// an elapsed one on elapsed time; the owner's timer treats both as elapsed.
func (s CeilingState) UntilBudget(l Lane, charges []Charge, now time.Time) (time.Duration, bool) {
if !l.Valid() {
return 0, false
}
a := s.meter(l)
if a.size == 0 {
return 0, false
}
var wait time.Duration
if a.m.Credit < unitTicks {
short, rate := int64(unitTicks)-int64(a.m.Credit), int64(a.size)
wait = time.Duration((short + rate - 1) / rate)
}
// Units to release before the lane's allowance and the whole ceiling
// each have room for one more.
var laneNeed, totalNeed uint32
if a.m.Used >= a.size {
laneNeed = a.m.Used - a.size + 1
}
if total := s.General.Used + s.Reserved.Used; total >= s.Limit {
totalNeed = total - s.Limit + 1
}
for _, c := range charges {
if laneNeed == 0 && totalNeed == 0 {
break
}
wait = max(wait, c.At.Add(CeilingWindow).Sub(now), c.Elapsed+CeilingWindow-s.Elapsed)
totalNeed -= min(totalNeed, c.Cost)
if (c.Lane == LaneGeneral) == (l == LaneGeneral) {
laneNeed -= min(laneNeed, c.Cost)
}
}
return wait, true
}
package admission
import "time"
// maxClockSkew is how far wall time may drift from elapsed time between two
// readings before the clock reports degraded coverage.
const maxClockSkew = time.Second
const clockVersion = 1
// ClockReading is one sample of the host clocks: wall time, the kernel's
// boot identity and the time elapsed since that boot.
type ClockReading struct {
Wall time.Time
BootID string
SinceBoot time.Duration
}
// ClockTick is what one reading means for admission.
type ClockTick struct {
// Now is the admission time: the persisted high-water mark. A wall
// clock that steps back never lowers it, so rollback cannot release
// capacity or make old evidence fresh.
Now time.Time
// Elapsed is the time elapsed since the previous reading within the
// same boot. A new boot grants none: downtime is never credited.
Elapsed time.Duration
// Degraded is true while the wall clock is behind the high-water mark,
// disagrees with elapsed time by more than maxClockSkew, or elapsed
// time ran backward. Wall time alone proves nothing new while it holds.
Degraded bool
}
type clockRecord struct {
V uint8 `json:"v"`
HighWater int64 `json:"high_water"`
BootID string `json:"boot_id"`
CheckWall int64 `json:"check_wall"`
CheckSince int64 `json:"check_since"`
}
// Clock is the persisted admission clock. It is a value: Advance
// returns the next clock, so a failed transaction leaves the stored one.
// The zero value has seen no reading.
type Clock struct {
rec clockRecord
}
func (c Clock) Initialized() bool { return c.rec.V != 0 }
// Now is the high-water mark; zero before the first reading.
func (c Clock) Now() time.Time { return fromNano(c.rec.HighWater) }
// Proven is the latest time the saved clock and reading r prove has passed:
// the high-water mark, raised to the saved checkpoint plus the time elapsed
// since it when r comes from the same boot. A new boot proves no downtime.
// The firewall restores timed blocks no earlier than this. Zero before the
// first reading.
func (c Clock) Proven(r ClockReading) time.Time {
if !c.Initialized() {
return time.Time{}
}
at := fromNano(c.rec.HighWater)
if since := int64(r.SinceBoot); r.BootID == c.rec.BootID && since >= c.rec.CheckSince {
if checked := fromNano(c.rec.CheckWall).Add(time.Duration(since - c.rec.CheckSince)); checked.After(at) {
at = checked
}
}
return at
}
func validBootID(id string) bool {
if len(id) != 36 {
return false
}
for i := 0; i < len(id); i++ {
c := id[i]
switch i {
case 8, 13, 18, 23:
if c != '-' {
return false
}
default:
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
return false
}
}
}
return true
}
// Advance folds r into the clock. Within one boot, elapsed time is the
// truth and a wall step beyond maxClockSkew is degraded coverage. Across
// boots nothing elapsed is credited. The high-water mark only moves forward.
func (c Clock) Advance(r ClockReading) (Clock, ClockTick, error) {
wall, ok := unixNano(r.Wall)
switch {
case !ok:
return c, ClockTick{}, refuse(ReasonInvalid, "clock reading has no representable wall time")
case !validBootID(r.BootID):
return c, ClockTick{}, refuse(ReasonInvalid, "clock reading has a malformed boot ID")
case r.SinceBoot < 0:
return c, ClockTick{}, refuse(ReasonInvalid, "clock reading has negative time since boot")
}
since := int64(r.SinceBoot)
next := clockRecord{V: clockVersion, HighWater: c.rec.HighWater, BootID: r.BootID, CheckWall: wall, CheckSince: since}
var tick ClockTick
if c.Initialized() && r.BootID == c.rec.BootID {
if since < c.rec.CheckSince {
// Keep the elapsed checkpoint until it is reached again, so a
// later reading cannot credit the same interval twice.
next.CheckWall, next.CheckSince = c.rec.CheckWall, c.rec.CheckSince
tick.Degraded = true
} else {
tick.Elapsed = time.Duration(since - c.rec.CheckSince)
skew := r.Wall.Sub(fromNano(c.rec.CheckWall).Add(tick.Elapsed))
if skew > maxClockSkew || skew < -maxClockSkew {
tick.Degraded = true
}
}
}
switch {
case !c.Initialized() || wall > next.HighWater:
next.HighWater = wall
case wall < next.HighWater:
tick.Degraded = true
}
tick.Now = fromNano(next.HighWater)
return Clock{rec: next}, tick, nil
}
// MarshalBinary encodes an initialized clock.
func (c Clock) MarshalBinary() ([]byte, error) {
if !c.Initialized() {
return nil, refuse(ReasonInvalid, "clock has seen no reading")
}
return sealRecord(c.rec)
}
// UnmarshalClock decodes a stored clock and checks its invariants.
func UnmarshalClock(data []byte) (Clock, error) {
var rec clockRecord
if err := openRecord(data, &rec); err != nil {
return Clock{}, err
}
if rec.V != clockVersion || !validBootID(rec.BootID) || rec.CheckWall == 0 || rec.HighWater == 0 || rec.HighWater < rec.CheckWall || rec.CheckSince < 0 {
return Clock{}, ErrCorruptRecord
}
return Clock{rec: rec}, nil
}
package admission
import (
"bytes"
"crypto/sha256"
"encoding/json"
"errors"
"time"
)
// ErrCorruptRecord reports a stored admission record that fails its
// checksum, strict decoding, canonical form or invariants. A corrupt record
// refuses mutation; it is never repaired by guessing.
var ErrCorruptRecord = errors.New("admission record is corrupt")
// ErrArrivalsIsolated is returned with the damage when a drain discarded
// damaged arrivals and committed everything else. One damaged record is not
// proof the ledger failed, so admission stays open.
var ErrArrivalsIsolated = errors.New("damaged arrivals were isolated")
// sealRecord encodes v as JSON followed by the first 8 bytes of its SHA-256,
// the framing Generations and Evidence use.
func sealRecord(v any) ([]byte, error) {
body, err := json.Marshal(v)
if err != nil {
return nil, err
}
sum := sha256.Sum256(body)
return append(body, sum[:8]...), nil
}
// openRecord verifies the checksum, decodes strictly and requires the
// canonical encoding, so a stored record has exactly one byte form.
func openRecord(data []byte, v any) error {
if len(data) < 8 {
return ErrCorruptRecord
}
body, sum := data[:len(data)-8], data[len(data)-8:]
if want := sha256.Sum256(body); !bytes.Equal(sum, want[:8]) {
return ErrCorruptRecord
}
dec := json.NewDecoder(bytes.NewReader(body))
dec.DisallowUnknownFields()
if err := dec.Decode(v); err != nil {
return ErrCorruptRecord
}
canonical, err := json.Marshal(v)
if err != nil || !bytes.Equal(canonical, body) {
return ErrCorruptRecord
}
return nil
}
// unixNano returns t as nanoseconds, or false when t cannot round-trip
// through them.
func unixNano(t time.Time) (int64, bool) {
if t.IsZero() {
return 0, false
}
n := t.UnixNano()
return n, n != 0 && time.Unix(0, n).Equal(t)
}
func fromNano(n int64) time.Time {
if n == 0 {
return time.Time{}
}
return time.Unix(0, n).UTC()
}
package admission
import (
"bytes"
"encoding/binary"
"math"
"time"
)
// MaxEffectRecordBytes bounds one encoded effect record. With a known set
// of capacity C, slot admission bounds new records to C times the sum of
// this size and effectKeyLen, plus the evicted records kept until their
// expiry. A reduced limit or an unavailable snapshot can leave more
// records until their expiry sweep.
const MaxEffectRecordBytes = 512
// effectRecordVersion is the encoding of a record without an eviction;
// a record that names a victim or an evictor is evictionRecordVersion,
// and a gone record is goneRecordVersion.
const (
effectRecordVersion = 1
evictionRecordVersion = 2
goneRecordVersion = 3
)
// effectKeyLen is the length of an effect record's key: its expiry, then
// its action.
const effectKeyLen = 8 + actionIDLen
// EffectRecord is an automatic block's hold on a temporary set slot. The
// ledger writes it when the block's attempt is reserved, so
// pending and live reservations count together; the verified outcome
// records the first application, and the record leaves when the attempt
// fails, its uncertainty is retired or its expiry passes. The occupancy
// partition, the lane of the charge and the evidence eligibility are
// separate recorded fields.
type EffectRecord struct {
Action ActionID
Target Target
Partition Partition
Lane Lane
Tier Tier
// Scope is the fairness scope key of the block's candidate.
Scope string
Direct bool
Corroborated bool
ExpiresAt time.Time
// Applied is the admission time the verified outcome was recorded,
// zero while the attempt is pending; Elapsed is the ceiling's elapsed
// time then, so a minimum hold needs both wall and elapsed time, as a
// charge's window does.
Applied time.Time
Elapsed time.Duration
// Victim is the verified record a pending block claimed to evict;
// zero when the block took a free slot, and cleared when
// its application is verified.
Victim EffectRef
// EvictedBy is the pending block that claimed this verified record as
// its victim. Evicted is set when that block's application is
// verified: the record then holds no slot and stays, as the ledger's
// record of the eviction, until its expiry sweep.
EvictedBy ActionID
Evicted bool
// Gone is the admission time the ledger learned the verified block
// left the kernel before its expiry: at its verified eviction, or at
// the first of two consecutive firewall answers that no longer held it
// (an operator unblock, undo or flush, or a lost element). A gone
// record holds no slot and stays until its expiry sweep.
Gone time.Time
}
// EffectRef names an effect record by its action and expiry, which make
// its key, or an import, which has no record, by its effect name and
// expiry.
type EffectRef struct {
Action ActionID
ExpiresAt time.Time
}
// Key is the storage key of the record r names; an import has none.
func (r EffectRef) Key() ([]byte, error) { return EffectRecordKey(r.Action, r.ExpiresAt) }
// Imported reports whether r names an import rather than a record.
func (r EffectRef) Imported() bool { return ImportedVictim(r.Action) }
// Verified reports whether the block's application was verified.
func (e EffectRecord) Verified() bool { return !e.Applied.IsZero() }
// Holds reports whether the record holds a set slot: every record does
// until an eviction of it is verified or it is gone.
func (e EffectRecord) Holds() bool { return !e.Evicted && e.Gone.IsZero() }
type effectRecordRecord struct {
V uint8 `json:"v"`
Target string `json:"target"`
Partition Partition `json:"partition"`
Lane Lane `json:"lane"`
Class Class `json:"class"`
Severity Severity `json:"severity"`
Scope string `json:"scope"`
Direct bool `json:"direct,omitempty"`
Corroborated bool `json:"corroborated,omitempty"`
Applied int64 `json:"applied,omitempty"`
Elapsed int64 `json:"elapsed,omitempty"`
Victim ActionID `json:"victim,omitempty"`
VictimEnds int64 `json:"victim_expires,omitempty"`
EvictedBy ActionID `json:"evicted_by,omitempty"`
Evicted bool `json:"evicted,omitempty"`
Gone int64 `json:"gone,omitempty"`
}
func (e EffectRecord) record() ([]byte, effectRecordRecord, error) {
bad := func(detail string) ([]byte, effectRecordRecord, error) {
return nil, effectRecordRecord{}, refuse(ReasonInvalid, detail)
}
if _, err := ParseActionID(string(e.Action)); err != nil {
return bad("effect names no action")
}
if err := ValidateKindTarget(KindBlockIP, e.Target); err != nil {
return bad("effect has no single-address target")
}
expires, ok := unixNano(e.ExpiresAt)
switch {
case !ok || expires < 0:
return bad("effect has no expiry after the epoch")
case !e.Partition.Valid() || !e.Lane.Valid():
return bad("effect has no partition or lane")
case e.Partition == PartitionReserved && PartitionOf(e.Lane) != PartitionReserved:
return bad("general work never holds a reserved slot")
case !e.Tier.Valid():
return bad("effect tier is invalid")
case e.Direct && e.Corroborated, (e.Direct || e.Corroborated) && e.Tier.Class != ClassC3:
return bad("effect has impossible reserved evidence")
case !boundedToken(e.Scope, 128):
return bad("effect scope is malformed")
case e.Elapsed < 0, e.Elapsed > 0 && e.Applied.IsZero():
return bad("effect elapsed time has no application")
case e.Victim != EffectRef{} && e.Verified():
return bad("a verified block keeps no victim claim")
case e.EvictedBy != "" && !e.Verified():
return bad("only a verified block is a victim")
case e.Evicted && e.EvictedBy == "":
return bad("an evicted block names no evictor")
case e.Victim.Action != "" && e.Victim.Action == e.Action, e.EvictedBy != "" && e.EvictedBy == e.Action:
return bad("a block cannot evict itself")
case !e.Gone.IsZero() && !e.Verified():
return bad("only a verified block can be gone")
case !e.Gone.IsZero() && e.EvictedBy != "" && !e.Evicted:
return bad("a claimed victim waits for its evictor's outcome")
}
rec := effectRecordRecord{V: effectRecordVersion, Target: e.Target.Key(), Partition: e.Partition, Lane: e.Lane, Class: e.Tier.Class,
Severity: e.Tier.Severity, Scope: e.Scope, Direct: e.Direct, Corroborated: e.Corroborated, Elapsed: int64(e.Elapsed)}
if e.Victim != (EffectRef{}) {
ends, inRange := unixNano(e.Victim.ExpiresAt)
if _, err := ParseVictimID(string(e.Victim.Action)); err != nil || !inRange || ends <= 0 {
return bad("effect victim is malformed")
}
rec.Victim, rec.VictimEnds = e.Victim.Action, ends
}
if e.EvictedBy != "" {
if _, err := ParseActionID(string(e.EvictedBy)); err != nil {
return bad("effect evictor is malformed")
}
rec.EvictedBy, rec.Evicted = e.EvictedBy, e.Evicted
}
if rec.Victim != "" || rec.EvictedBy != "" {
rec.V = evictionRecordVersion
}
if !e.Gone.IsZero() {
if rec.Gone, ok = unixNano(e.Gone); !ok || rec.Gone < 0 {
return bad("effect is gone at no time after the epoch")
}
rec.V = goneRecordVersion
}
if e.Verified() {
if rec.Applied, ok = unixNano(e.Applied); !ok || rec.Applied < 0 {
return bad("effect has no application after the epoch")
}
}
key, err := EffectRecordKey(e.Action, e.ExpiresAt)
return key, rec, err
}
// Key is the effect record's storage key.
func (e EffectRecord) Key() ([]byte, error) {
key, _, err := e.record()
return key, err
}
// EffectRecordKey is the storage key of the record action's block holds
// until expiresAt. Big-endian nanoseconds sort by expiry, so a walk in key
// order meets the effects that end first.
func EffectRecordKey(action ActionID, expiresAt time.Time) ([]byte, error) {
expires, ok := unixNano(expiresAt)
if _, err := ParseActionID(string(action)); err != nil || !ok || expires < 0 {
return nil, refuse(ReasonInvalid, "effect key needs an action and an expiry after the epoch")
}
key := binary.BigEndian.AppendUint64(make([]byte, 0, effectKeyLen), uint64(expires))
return append(key, action...), nil
}
func (e EffectRecord) MarshalBinary() ([]byte, error) {
_, rec, err := e.record()
if err != nil {
return nil, err
}
data, err := sealRecord(rec)
if err == nil && len(data) > MaxEffectRecordBytes {
return nil, refuse(ReasonInvalid, "effect record is too large")
}
return data, err
}
// UnmarshalEffectRecord decodes a stored effect record from its key and
// value and checks its invariants.
func UnmarshalEffectRecord(key, data []byte) (EffectRecord, error) {
var rec effectRecordRecord
if len(data) > MaxEffectRecordBytes || openRecord(data, &rec) != nil || len(key) != effectKeyLen {
return EffectRecord{}, ErrCorruptRecord
}
expires := binary.BigEndian.Uint64(key[:8])
if expires > 1<<63-1 {
return EffectRecord{}, ErrCorruptRecord
}
target, err := ParseTargetKey(rec.Target, Caps{IPv6: true})
if err != nil {
return EffectRecord{}, ErrCorruptRecord
}
e := EffectRecord{Action: ActionID(key[8:]), Target: target, Partition: rec.Partition, Lane: rec.Lane,
Tier: Tier{Class: rec.Class, Severity: rec.Severity}, Scope: rec.Scope, Direct: rec.Direct, Corroborated: rec.Corroborated,
ExpiresAt: fromNano(int64(expires)), Applied: fromNano(rec.Applied), Elapsed: time.Duration(rec.Elapsed),
EvictedBy: rec.EvictedBy, Evicted: rec.Evicted, Gone: fromNano(rec.Gone)}
if rec.Victim != "" {
e.Victim = EffectRef{Action: rec.Victim, ExpiresAt: fromNano(rec.VictimEnds)}
}
again, againRec, err := e.record()
if err != nil || !bytes.Equal(again, key) || againRec != rec {
return EffectRecord{}, ErrCorruptRecord
}
return e, nil
}
const effectTallyVersion = 1
// EffectTally counts the effect records of each partition. The ledger
// changes it with every record it writes or removes, so occupancy never
// walks the records.
type EffectTally struct {
General, Reserved int
}
type effectTallyRecord struct {
V uint8 `json:"v"`
General int `json:"general"`
Reserved int `json:"reserved"`
}
// Add counts delta more records in partition p. Invalid counts, underflow
// and overflow return the original tally with an error.
func (t EffectTally) Add(p Partition, delta int) (EffectTally, error) {
n := &t.General
switch p {
case PartitionGeneral:
case PartitionReserved:
n = &t.Reserved
default:
return t, refuse(ReasonInvalid, "effect tally names no partition")
}
if t.General < 0 || t.Reserved < 0 || delta < -*n || delta > math.MaxInt-*n {
return t, ErrCorruptRecord
}
*n += delta
return t, nil
}
// Occupy adds the counted records to the automatic entries of o. Invalid
// counts or overflow return the original occupancy with an error.
func (t EffectTally) Occupy(o SetOccupancy) (SetOccupancy, error) {
if t.General < 0 || t.Reserved < 0 || o.General < 0 || o.Reserved < 0 ||
t.General > math.MaxInt-o.General || t.Reserved > math.MaxInt-o.Reserved {
return o, ErrCorruptRecord
}
o.General += t.General
o.Reserved += t.Reserved
return o, nil
}
func (t EffectTally) MarshalBinary() ([]byte, error) {
if t.General < 0 || t.Reserved < 0 {
return nil, refuse(ReasonInvalid, "effect tally is negative")
}
return sealRecord(effectTallyRecord{V: effectTallyVersion, General: t.General, Reserved: t.Reserved})
}
// UnmarshalEffectTally decodes a stored effect tally.
func UnmarshalEffectTally(data []byte) (EffectTally, error) {
var rec effectTallyRecord
if openRecord(data, &rec) != nil || rec.V != effectTallyVersion || rec.General < 0 || rec.Reserved < 0 {
return EffectTally{}, ErrCorruptRecord
}
return EffectTally{General: rec.General, Reserved: rec.Reserved}, nil
}
package admission
import (
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"io"
"math"
"slices"
"time"
)
// EpisodeQuiet is how long an episode without a verified response lasts
// after its last accepted observation.
const EpisodeQuiet = time.Hour
// Episode row bounds. The widest key is a service target's; the record
// holds at most one line per kind.
const (
MaxEpisodeKeyBytes = 64
MaxEpisodeBytes = 1024
)
const (
episodeVersion = 1
episodeSequenceVersion = 1
)
// EpisodeLine is the latest generation one kind queued in an episode.
type EpisodeLine struct {
Kind Kind
Generation uint32
Candidate CandidateID
// Observed is the latest qualifying observation of this kind. A new
// generation needs later evidence; a repeated root cannot reset age.
Observed time.Time
// Answered retains, after the candidate is retired, that its verified
// effect or observe preview answers the kind for the rest of the
// episode.
Answered bool
// VerifiedGeneration is the generation of this block kind's latest
// verified block in the episode, zero before one verifies. Its history
// outlives later generations, so the evidence gate can always read the
// latest removal.
VerifiedGeneration uint32
}
// Episode is the engine's record of the current offense episode at one
// target: its identity, the observations and response that bound it, and
// the latest candidate of each kind. Responses of different
// kinds at one target share the episode.
type Episode struct {
ID EpisodeID
// Last is the latest accepted observation.
Last time.Time
// Verified is the original expiry of the episode's verified block,
// zero until one.
Verified time.Time
// EndElapsed is the ceiling's elapsed time at which End passes on
// elapsed time too. Zero for a row written before elapsed ends, which
// ends on wall time alone.
EndElapsed time.Duration
// Previous is when the episode before this one ended, zero for the
// first. An observation older than it belongs to that episode.
Previous time.Time
// PriorLast is the latest observation accepted by the previous episode.
// An unresolved line or a degraded clock can keep it open beyond its
// scheduled verified boundary.
PriorLast time.Time
// Lines are sorted by kind, one per kind.
Lines []EpisodeLine
}
// End is when the episode ends on the wall: at a verified block's original
// expiry, whatever other kinds have queued or in flight, otherwise an hour
// after the last accepted observation once no work is queued or in flight.
// It ends only once EndElapsed has passed as well. An unresolved line or a
// degraded clock keeps it open either way.
func (e Episode) End() time.Time {
if !e.Verified.IsZero() {
return e.Verified
}
return e.Last.Add(EpisodeQuiet)
}
// Line is the line of kind.
func (e Episode) Line(kind Kind) (EpisodeLine, bool) {
i := slices.IndexFunc(e.Lines, func(l EpisodeLine) bool { return l.Kind == kind })
if i < 0 {
return EpisodeLine{}, false
}
return e.Lines[i], true
}
// WithLine returns e with l as the line of its kind.
func (e Episode) WithLine(l EpisodeLine) Episode {
lines := slices.DeleteFunc(slices.Clone(e.Lines), func(have EpisodeLine) bool { return have.Kind == l.Kind })
i, _ := slices.BinarySearchFunc(lines, l.Kind, func(have EpisodeLine, k Kind) int { return int(have.Kind) - int(k) })
e.Lines = slices.Insert(lines, i, l)
return e
}
// WithoutCandidate clears a deleted candidate but retains its generation
// and observation frontier while another candidate keeps the row alive.
func (e Episode) WithoutCandidate(id CandidateID, answered bool) (Episode, bool) {
if id == "" {
return e, false
}
i := slices.IndexFunc(e.Lines, func(l EpisodeLine) bool { return l.Candidate == id })
if i < 0 {
return e, false
}
e.Lines = slices.Clone(e.Lines)
e.Lines[i].Candidate, e.Lines[i].Answered = "", answered
return e, true
}
// HasCandidates says whether a stored candidate still anchors this row.
func (e Episode) HasCandidates() bool {
return slices.ContainsFunc(e.Lines, func(l EpisodeLine) bool { return l.Candidate != "" })
}
type episodeLineRecord struct {
Kind Kind `json:"kind"`
Generation uint32 `json:"generation"`
Candidate CandidateID `json:"candidate,omitempty"`
Observed int64 `json:"observed"`
Answered bool `json:"answered,omitempty"`
// VerifiedGeneration needs schema 15.
VerifiedGeneration uint32 `json:"verified_generation,omitempty"`
}
type episodeRecord struct {
V uint8 `json:"v"`
ID string `json:"id"`
Last int64 `json:"last"`
Verified int64 `json:"verified,omitempty"`
Elapsed int64 `json:"end_elapsed,omitempty"`
Previous int64 `json:"previous,omitempty"`
PriorLast int64 `json:"prior_last,omitempty"`
Lines []episodeLineRecord `json:"lines"`
}
func (e Episode) record() (episodeRecord, error) {
bad := func(detail string) (episodeRecord, error) {
return episodeRecord{}, refuse(ReasonInvalid, detail)
}
if e.ID.IsZero() {
return bad("episode has no ID")
}
rec := episodeRecord{V: episodeVersion, ID: e.ID.String()}
var ok bool
if rec.Last, ok = unixNano(e.Last); !ok {
return bad("episode has no representable last observation")
}
if !e.Verified.IsZero() {
if rec.Verified, ok = unixNano(e.Verified); !ok {
return bad("episode expiry is not representable")
}
}
if e.EndElapsed < 0 {
return bad("episode elapsed end is negative")
}
rec.Elapsed = int64(e.EndElapsed)
if !e.Previous.IsZero() {
if rec.Previous, ok = unixNano(e.Previous); !ok || e.Previous.After(e.Last) {
return bad("previous episode ends after this one's last observation")
}
}
if !e.PriorLast.IsZero() {
if rec.PriorLast, ok = unixNano(e.PriorLast); !ok || !e.PriorLast.Before(e.Last) {
return bad("previous observations overlap this episode")
}
}
if !e.HasCandidates() {
line, found := e.Line(KindBlockIP)
if e.Verified.IsZero() || e.EndElapsed <= 0 || !found || !line.Answered {
return bad("episode has no retained candidate or verified block frontier")
}
}
for i, l := range e.Lines {
switch _, err := ParseCandidateID(string(l.Candidate)); {
case !l.Kind.Valid() || l.Generation == 0 || (l.Candidate != "" && err != nil) || (l.Candidate != "" && l.Answered):
return bad("episode line is malformed")
case i > 0 && l.Kind <= e.Lines[i-1].Kind:
return bad("episode lines are not one per kind in kind order")
case l.VerifiedGeneration > l.Generation ||
(l.VerifiedGeneration > 0 && (e.Verified.IsZero() || (l.Kind != KindBlockIP && l.Kind != KindBlockSubnet))):
return bad("episode line names a verified block it cannot have")
}
observed, ok := unixNano(l.Observed)
if !ok || l.Observed.After(e.Last) {
return bad("episode line has no valid observation frontier")
}
rec.Lines = append(rec.Lines, episodeLineRecord{Kind: l.Kind, Generation: l.Generation, Candidate: l.Candidate, Observed: observed, Answered: l.Answered, VerifiedGeneration: l.VerifiedGeneration})
}
return rec, nil
}
func (e Episode) MarshalBinary() ([]byte, error) {
rec, err := e.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalEpisode decodes a stored episode and checks its invariants.
func UnmarshalEpisode(data []byte) (Episode, error) {
var rec episodeRecord
if err := openRecord(data, &rec); err != nil {
return Episode{}, err
}
id, err := ParseEpisodeID(rec.ID)
if err != nil {
return Episode{}, ErrCorruptRecord
}
e := Episode{ID: id, Last: fromNano(rec.Last), Verified: fromNano(rec.Verified), EndElapsed: time.Duration(rec.Elapsed), Previous: fromNano(rec.Previous), PriorLast: fromNano(rec.PriorLast)}
for _, l := range rec.Lines {
e.Lines = append(e.Lines, EpisodeLine{Kind: l.Kind, Generation: l.Generation, Candidate: l.Candidate, Observed: fromNano(l.Observed), Answered: l.Answered, VerifiedGeneration: l.VerifiedGeneration})
}
if again, err := e.record(); rec.V != episodeVersion || err != nil || !slices.Equal(again.Lines, rec.Lines) || again.ID != rec.ID ||
again.Last != rec.Last || again.Verified != rec.Verified || again.Elapsed != rec.Elapsed || again.Previous != rec.Previous || again.PriorLast != rec.PriorLast {
return Episode{}, ErrCorruptRecord
}
return e, nil
}
// LineState is what a line's candidate lets a new observation of its kind
// do.
type LineState uint8
const (
// LineQueued: the candidate waits; the observation coalesces into it.
LineQueued LineState = iota + 1
// LineRetry: the candidate is queued after an attempt; it stays live
// but accepts no further observations into the retry.
LineRetry
// LineInFlight: an attempt is reserved or running.
LineInFlight
// LineAnswered: the candidate's effect was verified or previewed; it
// answers the kind for the rest of the episode.
LineAnswered
// LineEnded: the candidate ended without an effect that answers the
// kind: before any attempt, after its last attempt failed, or after
// recovery retired its unknown outcome. A new generation may follow.
LineEnded
// LineUnresolved: the candidate's attempt ended unknown and recovery
// has not retired it. It answers the kind but never extends the
// episode, and it holds the episode open until it is retired. The ledger
// separately holds other kinds on the same target before execution.
LineUnresolved
)
// LineStateOf is the state of a line whose candidate is c; retired says
// whether recovery retired c's unknown outcome.
func LineStateOf(c Candidate, retired bool) LineState {
switch {
case c.State == StateQueued && c.Attempts > 0:
return LineRetry
case c.State == StateQueued:
return LineQueued
case !c.State.Terminal():
return LineInFlight
case c.State == StateUnknown && !retired:
return LineUnresolved
case c.State == StateVerified || c.State == StateObserved:
return LineAnswered
}
return LineEnded
}
// EpisodeClock is the reading episode boundaries are judged on.
type EpisodeClock struct {
// Now is admission time.
Now time.Time
// Elapsed is the ceiling's elapsed time at Now, credited within a boot
// only, so neither a wall step nor downtime advances it.
Elapsed time.Duration
// Degraded is set when the reading was degraded: wall time alone
// proves nothing new then.
Degraded bool
}
func (c EpisodeClock) check() error {
_, ok := unixNano(c.Now)
switch {
case !ok:
return refuse(ReasonInvalid, "episode clock has no representable reading")
case c.Elapsed < 0:
return refuse(ReasonInvalid, "episode clock has negative elapsed time")
}
return nil
}
// elapsedEnd is the elapsed time at which end passes, read on c: the time
// left until end added to c.Elapsed.
func elapsedEnd(end time.Time, c EpisodeClock) (time.Duration, error) {
left := max(end.Sub(c.Now), 0)
if left > 0 && !c.Now.Add(left).Equal(end) {
return 0, refuse(ReasonInvalid, "episode remaining duration is not representable")
}
if left > math.MaxInt64-c.Elapsed {
return 0, refuse(ReasonInvalid, "episode end is not representable in elapsed time")
}
return c.Elapsed + left, nil
}
// quietElapsedEnd is the elapsed end of a quiet hour that starts at the
// reading c. An observation's age is wall time, which a forward step
// inflates, so the hour runs on elapsed time from the placing reading; an
// observation processed late can lengthen its episode, never shorten it.
func quietElapsedEnd(c EpisodeClock) (time.Duration, error) {
return elapsedEnd(c.Now.Add(EpisodeQuiet), c)
}
// AnchorElapsed gives a legacy row its remaining elapsed duration on
// its next write. Existing anchors survive maintenance without moving.
func (e Episode) AnchorElapsed(c EpisodeClock) (Episode, error) {
if err := c.check(); err != nil {
return e, err
}
if e.EndElapsed != 0 {
return e, nil
}
end, err := elapsedEnd(e.End(), c)
if err != nil {
return e, err
}
e.EndElapsed = end
return e, nil
}
// Close ends e at verified, its verified block's original expiry in
// admission time, read on c: the elapsed end becomes the time left until
// then, so the episode lasts as long as the block on both clocks. An
// episode already verified keeps its scheduled boundary: a
// same-episode re-admission's block never moves it.
func (e Episode) Close(verified time.Time, c EpisodeClock) (Episode, error) {
if _, ok := unixNano(verified); !ok {
return e, refuse(ReasonInvalid, "episode expiry is not representable")
}
if err := c.check(); err != nil {
return e, err
}
if !e.Verified.IsZero() {
return e.AnchorElapsed(c)
}
at, err := elapsedEnd(verified, c)
if err != nil {
return e, err
}
e.Verified, e.EndElapsed = verified, at
return e, nil
}
// EpisodeChoice is where one observation belongs.
type EpisodeChoice struct {
// Episode is the record after accepting the observation. A newly opened
// one has no lines yet.
Episode Episode
// Generation is the generation of its kind the observation joins.
Generation uint32
// Answered is set when that kind's line already has an attempt: the
// observation extends the episode but queues nothing.
Answered bool
}
// Place decides where an observation of kind made at observed belongs,
// given the target's current episode (nil for none) and the state of each
// of its lines in order, read on clock. The episode ends at End:
// at a verified block's expiry even while other kinds' work is queued or in
// flight (I1), otherwise once no line is queued or in flight. It ends only
// once clock's elapsed time has passed EndElapsed too, so a forward wall
// step cannot end it early and downtime grants it no time (I3). An
// unresolved line holds it open, and a degraded clock never ends it, since
// wall time alone cannot prove a new episode. An observation older than the
// previous episode's end is refused. next names a newly opened episode.
func Place(cur *Episode, states []LineState, kind Kind, observed time.Time, clock EpisodeClock, next func() (EpisodeID, error)) (EpisodeChoice, error) {
if !kind.Valid() {
return EpisodeChoice{}, refuse(ReasonInvalid, "unknown response kind")
}
if _, ok := unixNano(observed); !ok {
return EpisodeChoice{}, refuse(ReasonInvalid, "observation time is not representable")
}
if err := clock.check(); err != nil {
return EpisodeChoice{}, err
}
if cur != nil {
if len(states) != len(cur.Lines) {
return EpisodeChoice{}, refuse(ReasonInvalid, "line states do not match the episode")
}
live, unresolved := false, false
for _, s := range states {
switch s {
case LineQueued, LineRetry, LineInFlight:
live = true
case LineUnresolved:
unresolved = true
case LineAnswered, LineEnded:
default:
return EpisodeChoice{}, refuse(ReasonInvalid, "unknown line state")
}
}
if observed.Before(cur.Previous) || (!cur.PriorLast.IsZero() && !observed.After(cur.PriorLast)) {
return EpisodeChoice{}, refuse(ReasonStale, "observation belongs to an earlier episode")
}
due := !observed.Before(cur.End()) && clock.Elapsed >= cur.EndElapsed
// A verified block's expiry ends the episode whatever else is
// queued (I1); only its own kind could hold it, and that line is
// the verified one.
if !cur.Verified.IsZero() && due {
live = false
}
if live || unresolved || clock.Degraded || !due {
return join(*cur, states, kind, observed, clock)
}
if !observed.After(cur.Last) {
return EpisodeChoice{}, refuse(ReasonStale, "observation was already inside the ending episode")
}
}
end, err := quietElapsedEnd(clock)
if err != nil {
return EpisodeChoice{}, err
}
id, err := next()
if err != nil {
return EpisodeChoice{}, err
}
e := Episode{ID: id, Last: observed, EndElapsed: end}
if cur != nil {
e.Previous, e.PriorLast = cur.End(), cur.Last
}
return EpisodeChoice{Episode: e, Generation: 1}, nil
}
func join(e Episode, states []LineState, kind Kind, observed time.Time, clock EpisodeClock) (EpisodeChoice, error) {
e.Lines = slices.Clone(e.Lines)
i := slices.IndexFunc(e.Lines, func(l EpisodeLine) bool { return l.Kind == kind })
// An unresolved line answers without extending the episode or its own
// frontier: an attack that outlasts the uncertainty is not stretched
// into it (I2).
if i >= 0 && states[i] == LineUnresolved {
return EpisodeChoice{Episode: e, Generation: e.Lines[i].Generation, Answered: true}, nil
}
if observed.After(e.Last) {
e.Last = observed
// The quiet end moves with the last observation; its elapsed end
// never moves earlier, so an observation delivered late after a
// forward step cannot shorten the episode. A verified end stays.
if e.Verified.IsZero() {
end, err := quietElapsedEnd(clock)
if err != nil {
return EpisodeChoice{}, err
}
e.EndElapsed = max(e.EndElapsed, end)
}
}
p := EpisodeChoice{Episode: e, Generation: 1}
if i < 0 {
return p, nil
}
p.Generation = e.Lines[i].Generation
switch states[i] {
case LineRetry, LineInFlight, LineAnswered:
p.Answered = true
case LineEnded:
if p.Generation == math.MaxUint32 {
return EpisodeChoice{}, refuse(ReasonInvalid, "episode has no generation left")
}
if !observed.After(e.Lines[i].Observed) {
return EpisodeChoice{}, refuse(ReasonStale, "an ended generation needs a later qualifying observation")
}
p.Generation++
}
if observed.After(p.Episode.Lines[i].Observed) {
p.Episode.Lines[i].Observed = observed
}
return p, nil
}
// EpisodeSequence names episodes: a random nonce chosen when the ledger is
// created, and a counter. IDs never repeat within a ledger, nor across a
// ledger created again after loss, whose earlier action IDs the action log
// still holds.
type EpisodeSequence struct {
Nonce [16]byte
// Next is the sequence number the next episode takes.
Next uint64
}
// NewEpisodeSequence starts a sequence with a nonce read from r.
func NewEpisodeSequence(r io.Reader) (EpisodeSequence, error) {
var s EpisodeSequence
if _, err := io.ReadFull(r, s.Nonce[:]); err != nil {
return EpisodeSequence{}, err
}
return s, nil
}
// Take names the next episode.
func (s *EpisodeSequence) Take() (EpisodeID, error) {
if s.Next == math.MaxUint64 {
return EpisodeID{}, refuse(ReasonInvalid, "episode sequence is exhausted")
}
h := sha256.New()
writeField(h, []byte{identityVersion})
writeField(h, []byte("episode"))
writeField(h, s.Nonce[:])
writeField(h, binary.BigEndian.AppendUint64(nil, s.Next))
var id EpisodeID
copy(id[:], h.Sum(nil))
if id.IsZero() {
return EpisodeID{}, ErrCorruptRecord
}
s.Next++
return id, nil
}
type episodeSequenceRecord struct {
V uint8 `json:"v"`
Nonce string `json:"nonce"`
Next uint64 `json:"next"`
}
func (s EpisodeSequence) MarshalBinary() ([]byte, error) {
if s.Nonce == ([16]byte{}) {
return nil, refuse(ReasonInvalid, "episode sequence has no nonce")
}
return sealRecord(episodeSequenceRecord{V: episodeSequenceVersion, Nonce: hex.EncodeToString(s.Nonce[:]), Next: s.Next})
}
// UnmarshalEpisodeSequence decodes a stored episode sequence.
func UnmarshalEpisodeSequence(data []byte) (EpisodeSequence, error) {
var rec episodeSequenceRecord
if err := openRecord(data, &rec); err != nil {
return EpisodeSequence{}, err
}
var s EpisodeSequence
if rec.V != episodeSequenceVersion || !lowerHex(rec.Nonce, 32) {
return EpisodeSequence{}, ErrCorruptRecord
}
_, _ = hex.Decode(s.Nonce[:], []byte(rec.Nonce))
if s.Nonce == ([16]byte{}) {
return EpisodeSequence{}, ErrCorruptRecord
}
s.Next = rec.Next
return s, nil
}
package admission
import (
"slices"
"time"
)
// EvictionVictim is a verified automatic block that strictly higher work
// may evict when the set has no free slot for it: an effect
// record or an import, whose Action is its effect name. Fresh work of the
// same tier may rotate a record out once its hold passed.
type EvictionVictim struct {
Action ActionID
Target Target
Tier Tier
Partition Partition
ExpiresAt time.Time
// Scope is the fairness scope of a record's block and Applied when its
// application was verified; an import has neither.
Scope string
Applied time.Time
// Rotatable is set for a record whose rotation hold passed on a
// trusted clock: equal-tier rotation may take only such a block.
Rotatable bool
}
// ImportTier is the tier of an import, a known automatic block an older
// producer wrote with explicit provenance: class C2, severity High.
var ImportTier = Tier{Class: ClassC2, Severity: SeverityHigh}
// Imported reports whether v is an import as the engine's list hands it
// over: an import name, an address, the import tier, the general
// partition it is counted in and an expiry after the epoch, with no scope,
// application or rotation: an import never rotates.
func (v EvictionVictim) Imported() bool {
ends, ok := unixNano(v.ExpiresAt)
return ImportedVictim(v.Action) && ValidateKindTarget(KindBlockIP, v.Target) == nil && v.Tier == ImportTier &&
v.Partition == PartitionGeneral && ok && ends > 0 && v.Scope == "" && v.Applied.IsZero() && !v.Rotatable
}
// victimOrder is the eviction order: the lowest tier first, then the
// earliest absolute expiry, then the lowest action ID.
func victimOrder(a, b EvictionVictim) int {
switch {
case a.Tier.Less(b.Tier):
return -1
case b.Tier.Less(a.Tier):
return 1
}
if c := a.ExpiresAt.Compare(b.ExpiresAt); c != 0 {
return c
}
switch {
case a.Action < b.Action:
return -1
case a.Action > b.Action:
return 1
}
return 0
}
// SlotDemand is what a pick asks of the set.
type SlotDemand struct {
Tier Tier
// Cost is the set slots it takes: its ceiling cost.
Cost uint32
// Evicts is set for one-unit block_ip work, the only work that may
// evict (one victim per charged unit).
Evicts bool
// Reserved is set for work eligible for a reserved lane.
Reserved bool
// Target is the work's address; the block of that address is never
// its victim.
Target Target
// Scope is the work's fairness scope; Fresh is set while its episode
// has had no verified response, the only work that may rotate an
// equal-tier block.
Scope string
Fresh bool
}
// SlotLimits bound block work by the temporary set: the free slots of each
// partition, the victims strictly higher work may evict and, once Share
// sets the scopes' shares, the equal-tier blocks fresh work may rotate. A
// nil *SlotLimits bounds nothing. Take consumes what it grants; Room only
// asks.
type SlotLimits struct {
freeGeneral, freeReserved uint32
// victims is sorted in victimOrder and holds what is still free to take.
victims []EvictionVictim
// size is each partition's slots, which its scopes share; shares, once
// set, lets fresh work rotate equal-tier blocks.
size [partitionEnd]int
shares *setShares
}
// NewSlotLimits holds the free slots of each partition and a sorted copy of
// the victim pool.
func NewSlotLimits(freeGeneral, freeReserved uint32, victims []EvictionVictim) *SlotLimits {
pool := slices.Clone(victims)
slices.SortFunc(pool, victimOrder)
return &SlotLimits{freeGeneral: freeGeneral, freeReserved: freeReserved, victims: pool}
}
// Limits is the slot budget of o, the one rule admission and its forecast
// share: each partition's free slots and size and, once the
// general partition is full and the set is not overfull, the victims pool
// returns, less those of an overfull partition. pool is read only then,
// and never for a set that evicts nothing.
func (o SetOccupancy) Limits(pool func() ([]EvictionVictim, error)) (*SlotLimits, error) {
s := o.Slots()
var victims []EvictionVictim
if !o.NoEviction && s.FreeGeneral == 0 && o.General+o.Reserved <= s.Available {
var err error
if victims, err = pool(); err != nil {
return nil, err
}
}
victims = slices.DeleteFunc(victims, func(v EvictionVictim) bool {
return (v.Partition == PartitionGeneral && o.General > s.General) || (v.Partition == PartitionReserved && o.Reserved > s.Reserved)
})
l := NewSlotLimits(slotUnits(s.FreeGeneral), slotUnits(s.FreeReserved), victims)
l.size[PartitionGeneral], l.size[PartitionReserved] = s.General, s.Reserved
return l, nil
}
// slotUnits is n free slots as block units; no lane budget exceeds the
// ceiling's largest.
func slotUnits(n int) uint32 {
return uint32(min(max(n, 0), int(MaxCeiling))) // #nosec G115 -- clamped to the ceiling's range.
}
// Clone is an independent copy; nil stays nil.
func (s *SlotLimits) Clone() *SlotLimits {
if s == nil {
return nil
}
return &SlotLimits{freeGeneral: s.freeGeneral, freeReserved: s.freeReserved, victims: slices.Clone(s.victims), size: s.size, shares: s.shares.clone()}
}
// Budget bounds a lane by its free slots or, under pressure, its victims:
// those below the highest valid tier, which no work may evict as lower,
// and, once shares are set, the blocks of the lane's partition rotation
// may take. Per-lane readiness and Take decide whether each pick can use
// that room.
func (s *SlotLimits) Budget(lane Lane) uint32 {
if s == nil {
return MaxCeiling
}
free, n := s.freeGeneral, uint32(0)
if PartitionOf(lane) == PartitionReserved {
free = s.freeReserved
}
if free > 0 || (PartitionOf(lane) == PartitionReserved && s.freeGeneral > 0) {
return min(free, MaxCeiling)
}
highest := Tier{Class: ClassC3, Severity: SeverityCritical}
for _, v := range s.victims {
lower := v.Tier.Less(highest) && (PartitionOf(lane) == PartitionReserved || v.Partition == PartitionGeneral)
if lower || (s.shares != nil && v.Rotatable && v.Partition == PartitionOf(lane)) {
n++
}
}
return min(free+n, MaxCeiling)
}
// Room reports how work d finds room on lane: a free slot of the lane's
// partition (a zero victim) or, under pressure, the first victim in
// eviction order that d strictly outranks, else the equal-tier block
// rotation gives it. Pressure means no free slot for d on any lane it may
// use: general work never takes a reserved slot or evicts a reserved
// entry, reserved work may evict a lower entry in either partition, and
// rotation stays in the lane's partition. ok is false when d must wait.
func (s *SlotLimits) Room(lane Lane, d SlotDemand) (EvictionVictim, bool) {
i, ok := s.room(lane, d)
if i < 0 {
return EvictionVictim{}, ok
}
return s.victims[i], ok
}
// Take is Room that consumes what it grants.
func (s *SlotLimits) Take(lane Lane, d SlotDemand) (EvictionVictim, bool) {
i, ok := s.room(lane, d)
switch {
case !ok || s == nil:
return EvictionVictim{}, ok
case i >= 0:
v := s.victims[i]
s.victims = slices.Delete(s.victims, i, i+1)
s.shares.move(v.Partition, d.Scope, v)
return v, true
case PartitionOf(lane) == PartitionReserved:
s.freeReserved -= d.Cost
default:
s.freeGeneral -= d.Cost
}
// Only one-unit block work writes a record, which then holds the slot.
if d.Evicts && d.Cost == 1 {
s.shares.move(PartitionOf(lane), d.Scope, EvictionVictim{})
}
return EvictionVictim{}, true
}
func (s *SlotLimits) room(lane Lane, d SlotDemand) (int, bool) {
if s == nil || d.Cost == 0 {
return -1, true
}
reserved := PartitionOf(lane) == PartitionReserved
own, other := s.freeGeneral, s.freeReserved
if reserved {
own, other = other, own
}
if d.Cost <= own {
return -1, true
}
// A free slot another lane of the work could take is no pressure.
if (reserved || d.Reserved) && d.Cost <= other {
return -1, false
}
if !d.Evicts || d.Cost != 1 {
return -1, false
}
for i, v := range s.victims {
if v.Tier.Less(d.Tier) && v.Target != d.Target && (reserved || v.Partition == PartitionGeneral) {
return i, true
}
}
if i := s.rotation(lane, d); i >= 0 {
return i, true
}
return -1, false
}
package admission
import (
"bytes"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
"slices"
"time"
)
// MaxEvidenceBytes bounds one encoded evidence record.
const MaxEvidenceBytes = 1024
const evidenceVersion = 1
// ObservationRef identifies the observation behind evidence by the
// producer's own cursor or sequence, never by a caller-chosen timestamp. A
// detector re-reporting the same observation keeps the same reference.
type ObservationRef struct {
// Stream is the producer's stream identity, such as a log file identity
// or a journal boot ID.
Stream string
// Cursor is the position or sequence number within Stream.
Cursor string
// Version is the producer's revision of this observation, from 1.
Version uint32
}
// ParserRef names the parser that read the observation.
type ParserRef struct {
Name string
Version uint32
}
// IntelRef is the external source behind reputation evidence and the time
// its verdict expires.
type IntelRef struct {
Source string
Expires time.Time
}
// EvidenceInput is what a producer supplies to Mint.
type EvidenceInput struct {
Check string
FindingID string
Severity Severity
Observation ObservationRef
ObservedAt time.Time
Parser ParserRef
// Target is the address the attribution names; never a prefix.
Target Target
// Attribution is how the producer's parser read Target. Mint checks
// it against the producer's declaration and the check's policy; the
// record does not store it.
Attribution Attribution
// Claims say which account the evidence concerns, as the finding read
// them. Only kinds the producer declared are accepted.
Claims []Claim
// Inventory is the server-owned snapshot that verifies Claims. Without
// one the evidence belongs to the host.
Inventory *Inventory
Intel *IntelRef
}
// EvidenceID names one evidence record. The same observation of the same
// check and target always has the same ID.
type EvidenceID string
// evidenceRecord is the persisted form. Field order is the encoding order.
type evidenceRecord struct {
Producer ProducerID `json:"producer"`
Entry Entry `json:"entry"`
Check string `json:"check"`
Family Family `json:"family"`
Basis Basis `json:"basis"`
FindingID string `json:"finding_id"`
Severity Severity `json:"severity"`
Stream string `json:"stream"`
Cursor string `json:"cursor"`
Version uint32 `json:"version"`
ObservedAt int64 `json:"observed_at"`
Parser string `json:"parser"`
ParserVersion uint32 `json:"parser_version"`
Target string `json:"target"`
OwnerAccount string `json:"owner_account,omitempty"`
OwnerGeneration uint64 `json:"owner_generation,omitempty"`
IntelSource string `json:"intel_source,omitempty"`
IntelExpires int64 `json:"intel_expires,omitempty"`
}
// Evidence is an immutable record minted by a registered producer. Its
// fields are only readable; the zero value is not valid evidence.
type Evidence struct {
rec evidenceRecord
}
func (e Evidence) Producer() ProducerID { return e.rec.Producer }
func (e Evidence) Entry() Entry { return e.rec.Entry }
func (e Evidence) Check() string { return e.rec.Check }
func (e Evidence) Family() Family { return e.rec.Family }
func (e Evidence) Basis() Basis { return e.rec.Basis }
func (e Evidence) FindingID() string { return e.rec.FindingID }
func (e Evidence) Severity() Severity { return e.rec.Severity }
func (e Evidence) Observation() ObservationRef {
return ObservationRef{Stream: e.rec.Stream, Cursor: e.rec.Cursor, Version: e.rec.Version}
}
func (e Evidence) ObservedAt() time.Time { return time.Unix(0, e.rec.ObservedAt).UTC() }
func (e Evidence) Parser() ParserRef {
return ParserRef{Name: e.rec.Parser, Version: e.rec.ParserVersion}
}
// Target returns the evidence target. It was validated when the record was
// minted or decoded.
func (e Evidence) Target() Target {
t, _ := ParseTargetKey(e.rec.Target, Caps{IPv6: true})
return t
}
func (e Evidence) Owner() Owner {
return Owner{account: e.rec.OwnerAccount, generation: e.rec.OwnerGeneration}
}
func (e Evidence) Intel() (IntelRef, bool) {
if e.rec.IntelSource == "" {
return IntelRef{}, false
}
return IntelRef{Source: e.rec.IntelSource, Expires: time.Unix(0, e.rec.IntelExpires).UTC()}, true
}
// Equal reports whether two records are identical. A producer publishing a
// different record under an existing ID is a conflict, never an update.
func (e Evidence) Equal(o Evidence) bool { return e.rec == o.rec }
// SameExceptFinding reports whether o records the same observation as e,
// field for field, apart from the finding that reported it. Such a record
// is a later report of the published original, not a conflicting one.
func (e Evidence) SameExceptFinding(o Evidence) bool {
a, b := e.rec, o.rec
a.FindingID, b.FindingID = "", ""
return a == b
}
// Conflict is the refusal of o under e's ID when o differs from e. Only a
// difference in owner alone identifies an inventory change between two
// mints of the observation; any additional difference is invalid.
func (e Evidence) Conflict(o Evidence) error {
a, b := e.rec, o.rec
owner := a.OwnerAccount != b.OwnerAccount || a.OwnerGeneration != b.OwnerGeneration
a.OwnerAccount, a.OwnerGeneration = b.OwnerAccount, b.OwnerGeneration
if owner && a == b {
return ErrEvidenceOwnerChanged
}
return ErrEvidenceConflict
}
// ID derives the evidence ID from producer, check, observation and target.
func (e Evidence) ID() EvidenceID {
h := sha256.New()
writeField(h, []byte{identityVersion})
writeField(h, []byte(e.rec.Producer))
writeField(h, []byte(e.rec.Check))
writeField(h, []byte(e.rec.Stream))
writeField(h, []byte(e.rec.Cursor))
writeField(h, binary.BigEndian.AppendUint32(nil, e.rec.Version))
writeField(h, []byte(e.rec.Target))
return EvidenceID("ev_" + hex.EncodeToString(h.Sum(nil)[:16]))
}
// Mint validates in and returns evidence bound to this producer. The
// producer's entry and the check's current family and basis are recorded;
// callers cannot choose them.
func (p *Producer) Mint(in EvidenceInput) (Evidence, error) {
if p == nil || p.reg == nil {
return Evidence{}, refuse(ReasonPolicy, "producer is not registered")
}
spec, ok := p.reg.Spec(p.id)
if !ok {
return Evidence{}, refuse(ReasonPolicy, "producer is not registered")
}
canonical, pol, ok := p.reg.lookup(in.Check)
if !ok || !spec.publishes(canonical) {
return Evidence{}, refuse(ReasonPolicy, "check is not registered for this producer")
}
if !spec.Attribution.Has(in.Attribution) || !pol.Attribution.Has(in.Attribution) {
return Evidence{}, refuseAttribution(CauseKindNotAllowed, "the producer or check does not accept this attribution kind")
}
// Stored records may still name a spray's prefix; a new record names
// the address its parser read.
if !in.Target.IsZero() && !in.Target.IsAddress() {
return Evidence{}, refuse(ReasonInvalid, "evidence names the address its parser read, not a prefix")
}
for _, c := range in.Claims {
if !slices.Contains(spec.Claims, c.Kind) {
return Evidence{}, refuse(ReasonPolicy, "claim kind is not declared for this producer")
}
}
owner := HostOwner()
if in.Inventory != nil {
owner = in.Inventory.Resolve(in.Claims...)
}
rec := evidenceRecord{
Producer: p.id,
Entry: spec.Entry,
Check: canonical,
Family: pol.Family,
Basis: pol.Basis,
FindingID: in.FindingID,
Severity: in.Severity,
Stream: in.Observation.Stream,
Cursor: in.Observation.Cursor,
Version: in.Observation.Version,
Parser: in.Parser.Name,
ParserVersion: in.Parser.Version,
Target: in.Target.Key(),
OwnerAccount: owner.account,
OwnerGeneration: owner.generation,
}
var err error
rec.ObservedAt, err = evidenceTime(in.ObservedAt)
if err != nil {
return Evidence{}, err
}
if in.Intel != nil {
rec.IntelSource = in.Intel.Source
rec.IntelExpires, err = evidenceTime(in.Intel.Expires)
if err != nil {
return Evidence{}, err
}
}
if err := validateRecord(rec); err != nil {
return Evidence{}, err
}
if rec.Severity < pol.MinSeverity {
return Evidence{}, refuse(ReasonPolicy, "finding severity is below the check's evidence floor")
}
e := Evidence{rec: rec}
if b, err := e.MarshalBinary(); err != nil || len(b) > MaxEvidenceBytes {
return Evidence{}, refuse(ReasonInvalid, "evidence record exceeds its size bound")
}
return e, nil
}
// UnixNano is undefined outside its representable range. Refuse such input
// instead of persisting a different observation time or intel expiry.
func evidenceTime(t time.Time) (int64, error) {
if t.IsZero() {
return 0, nil
}
n := t.UnixNano()
if !time.Unix(0, n).Equal(t) {
return 0, refuse(ReasonInvalid, "evidence time is outside the supported range")
}
return n, nil
}
// validateRecord checks everything that does not need the registry. Decode
// runs it too, so a stored record is held to the same rules as a new one.
func validateRecord(r evidenceRecord) error {
switch {
case !ValidProducerID(r.Producer) || !r.Entry.Valid():
return refuse(ReasonInvalid, "evidence producer or entry is malformed")
case r.Family == FamilyNone || ValidPolicy(r.Family, r.Basis) != nil:
return refuse(ReasonPolicy, "evidence family and basis are not an admissible pair")
case !lowerHex(r.FindingID, 16):
return refuse(ReasonInvalid, "finding ID is not 16 lowercase hex digits")
case !r.Severity.Valid():
return refuse(ReasonInvalid, "evidence severity is invalid")
case !boundedToken(r.Stream, 128) || !boundedToken(r.Cursor, 128) || r.Version == 0:
return refuse(ReasonInvalid, "observation reference is missing or malformed")
case r.ObservedAt <= 0:
return refuse(ReasonInvalid, "observation time is missing")
case !boundedToken(r.Parser, 64) || r.ParserVersion == 0:
return refuse(ReasonInvalid, "parser provenance is missing or malformed")
case !boundedToken(r.Check, 64):
return refuse(ReasonInvalid, "check name is malformed")
}
t, err := ParseTargetKey(r.Target, Caps{IPv6: true})
if err != nil {
return err
}
if _, isService := t.Service(); isService {
return refuse(ReasonInvalid, "evidence names an address or prefix, not a service")
}
if r.OwnerAccount == "" {
if r.OwnerGeneration != 0 {
return refuse(ReasonInvalid, "host owner carries a generation")
}
} else if !ValidAccountName(r.OwnerAccount) || r.OwnerGeneration == 0 {
return refuse(ReasonInvalid, "owner account is malformed")
}
intel := r.IntelSource != "" || r.IntelExpires != 0
switch {
case intel != (r.Family == FamilyReputation):
return refuse(ReasonInvalid, "intel source is required for reputation evidence and only for it")
case intel && (!boundedToken(r.IntelSource, 64) || r.IntelExpires <= r.ObservedAt):
return refuse(ReasonInvalid, "intel source or expiry is malformed")
}
return nil
}
// boundedToken accepts 1-max bytes of printable ASCII without spaces.
func boundedToken(s string, max int) bool {
if len(s) == 0 || len(s) > max {
return false
}
for i := 0; i < len(s); i++ {
if s[i] < 0x21 || s[i] > 0x7e {
return false
}
}
return true
}
// MarshalBinary encodes 'E', the version, the JSON record and the first 8
// bytes of the SHA-256 of everything before it.
func (e Evidence) MarshalBinary() ([]byte, error) {
body, err := json.Marshal(e.rec)
if err != nil {
return nil, err
}
out := append([]byte{'E', evidenceVersion}, body...)
sum := sha256.Sum256(out)
return append(out, sum[:8]...), nil
}
// UnmarshalEvidence checks record structure, checksum and canonical encoding.
// A checksum detects corruption, not authorship. Only the database owner's
// trusted evidence store may supply bytes; publication must reject an existing
// ID with different content using Equal.
func UnmarshalEvidence(data []byte) (Evidence, error) {
if len(data) > MaxEvidenceBytes || len(data) < 2+2+8 {
return Evidence{}, refuse(ReasonInvalid, "evidence record has an impossible length")
}
if data[0] != 'E' || data[1] != evidenceVersion {
return Evidence{}, refuse(ReasonInvalid, "evidence record version is not supported")
}
head, sum := data[:len(data)-8], data[len(data)-8:]
if want := sha256.Sum256(head); !bytes.Equal(sum, want[:8]) {
return Evidence{}, refuse(ReasonInvalid, "evidence record checksum mismatch")
}
dec := json.NewDecoder(bytes.NewReader(head[2:]))
dec.DisallowUnknownFields()
var rec evidenceRecord
if err := dec.Decode(&rec); err != nil || dec.More() {
return Evidence{}, refuse(ReasonInvalid, "evidence record body does not decode")
}
if err := validateRecord(rec); err != nil {
return Evidence{}, err
}
e := Evidence{rec: rec}
if again, err := e.MarshalBinary(); err != nil || !bytes.Equal(again, data) {
return Evidence{}, refuse(ReasonInvalid, "evidence record is not in canonical form")
}
return e, nil
}
package admission
import (
"fmt"
"strconv"
"time"
)
// Record bounds for storage accounting. The invariants keep every
// valid record inside its bound; tests pin each bound with the largest
// record the invariants allow.
const (
// MaxAttemptBytes bounds one encoded attempt record.
MaxAttemptBytes = 320
// MaxReportLinksBytes bounds one encoded set of report links.
MaxReportLinksBytes = 256
// MaxHistoryEntryBytes bounds one encoded history entry.
MaxHistoryEntryBytes = 192
// MaxEvidenceRefsBytes bounds one encoded evidence reference count.
MaxEvidenceRefsBytes = 64
// MaxHistoryBytes bounds the history bytes one candidate can be
// charged: its largest record, its attempts, its history rows and its
// roots at their largest.
MaxHistoryBytes = 32 << 10
)
// HistoryRetention is the review window: an admitted candidate's details
// are kept at least this long after it ends.
const HistoryRetention = 7 * 24 * time.Hour
// HistoryTarget is how long an ended candidate's details are kept when no
// quota needs their bytes.
const HistoryTarget = 30 * 24 * time.Hour
// candidateIDLen is the length of a derived candidate ID.
const candidateIDLen = len("cand_") + 32
// HistoryIndexKeyLen is the length of a history index key: a kind byte, a
// time as 19 decimal digits of nanoseconds and a candidate ID.
const HistoryIndexKeyLen = 1 + 19 + candidateIDLen
// HistoryTimes is when the details of an admitted candidate that ended at
// ended may first be retired, and when they are retired without pressure.
// They stay through the review window and, when holdsRow, for as long as
// the firewall row its attempt may have written can last.
func HistoryTimes(ended, expires time.Time, holdsRow bool) (eligible, target time.Time) {
eligible = ended.Add(HistoryRetention)
if holdsRow && expires.After(eligible) {
eligible = expires
}
return eligible, retireTarget(ended, eligible)
}
// HistoryUntilExpiry reports whether an ended admitted candidate keeps its
// history until its expiry: every attempt that ran may have written a
// firewall row, verified or not, and startup must still know its action
// while that row could be restored. Only a preview writes none, and a
// candidate previewed on a retry had a live attempt before it.
func HistoryUntilExpiry(c Candidate) bool { return c.State != StateObserved || c.Attempts > 1 }
func retireTarget(ended, eligible time.Time) time.Time {
if target := ended.Add(HistoryTarget); target.After(eligible) {
return target
}
return eligible
}
const historyEntryVersion = 2
// HistoryEntry is the ledger's storage record of an admitted candidate: the
// history bytes its reservations charged to each allowance and, once it
// ends, the earliest time its details may be retired.
type HistoryEntry struct {
// LegacyCost preserves the version 1 charge contract. It predates the
// episode allowance, so a candidate without an episode row need not
// have paid for one. New charges use version 2.
LegacyCost bool
// General and Reserved are the bytes charged to each allowance. The
// direct and corroborated lanes share the reserved one.
General, Reserved uint32
// RootMask names roots covered by the last reservation, in candidate order.
// Retry support stays queue data until a reservation pays for it.
RootMask uint32
// Ended is when the candidate ended; zero while it is live.
Ended time.Time
// Eligible is the earliest time its details may be retired. It is set
// when the candidate ends, unless the outcome is pinned.
Eligible time.Time
// Pinned: the outcome is unresolved. Its bytes are held in the recovery
// reserve, and it is not retired until recovery settles it.
Pinned bool
// Released: the live start made an observe preview's history eligible
// before its review window ended. A preview carries no
// review or undo.
Released bool
}
// Charged is the history bytes the entry holds.
func (h HistoryEntry) Charged() uint32 { return h.General + h.Reserved }
// Target is when an ended entry's details are retired without pressure;
// zero while it is live or pinned.
func (h HistoryEntry) Target() time.Time {
if h.Eligible.IsZero() {
return time.Time{}
}
return retireTarget(h.Ended, h.Eligible)
}
type historyEntryRecord struct {
V uint8 `json:"v"`
General uint32 `json:"general,omitempty"`
Reserved uint32 `json:"reserved,omitempty"`
Ended int64 `json:"ended,omitempty"`
Eligible int64 `json:"eligible,omitempty"`
Pinned bool `json:"pinned,omitempty"`
RootMask uint32 `json:"root_mask,omitempty"`
Released bool `json:"released,omitempty"`
}
func (h HistoryEntry) record() (historyEntryRecord, error) {
bad := func(detail string) (historyEntryRecord, error) {
return historyEntryRecord{}, refuse(ReasonInvalid, detail)
}
rec := historyEntryRecord{V: historyEntryVersion, General: h.General, Reserved: h.Reserved, Pinned: h.Pinned, RootMask: h.RootMask, Released: h.Released}
if h.LegacyCost {
rec.V = 1
}
if charged := uint64(h.General) + uint64(h.Reserved); charged == 0 || charged > MaxHistoryBytes {
return bad("history entry charges nothing or more than any candidate")
}
if h.RootMask>>MaxRoots != 0 {
return bad("history root mask exceeds the candidate bound")
}
if h.Ended.IsZero() {
if !h.Eligible.IsZero() || h.Pinned || h.Released {
return bad("a live history entry is pinned or has a retirement time")
}
return rec, nil
}
var ok bool
if rec.Ended, ok = unixNano(h.Ended); !ok || rec.Ended < 0 {
return bad("history entry has no end time after the epoch")
}
if h.Pinned {
if !h.Eligible.IsZero() || h.Released {
return bad("a pinned history entry has a retirement time")
}
return rec, nil
}
if rec.Eligible, ok = unixNano(h.Eligible); !ok {
return bad("history entry has no retirement time")
}
switch review := h.Ended.Add(HistoryRetention); {
case h.Released && (h.Eligible.Before(h.Ended) || !h.Eligible.Before(review)):
return bad("released history may be retired before its end or keeps its review window")
case !h.Released && h.Eligible.Before(review):
return bad("history entry may be retired inside its review window")
}
if _, ok = unixNano(h.Target()); !ok {
return bad("history entry target is not representable")
}
return rec, nil
}
// Index kinds of the history retirement keys.
const (
// RetireGeneral and RetireReserved order an allowance's ended entries
// by the earliest time they may be retired.
RetireGeneral = 'g'
RetireReserved = 'r'
// RetireTarget orders ended entries by their target.
RetireTarget = 't'
)
// RetireKeys are an ended, unpinned entry's index keys: one for each
// allowance it charged, ordered by when it may be retired, and one ordered
// by its target. Each is its kind, the time as 19 decimal digits of
// nanoseconds, which sort as the times do, and the candidate ID. A live or
// pinned entry has none.
func (h HistoryEntry) RetireKeys(id CandidateID) ([][]byte, error) {
if _, err := h.record(); err != nil {
return nil, err
}
if _, err := ParseCandidateID(string(id)); err != nil {
return nil, err
}
if h.Eligible.IsZero() {
return nil, nil
}
// Both times are after the end, which is after the epoch, so each has
// exactly 19 digits once padded.
key := func(kind byte, at time.Time) []byte {
return fmt.Appendf(make([]byte, 0, HistoryIndexKeyLen), "%c%019d%s", kind, at.UnixNano(), id)
}
var keys [][]byte
if h.General > 0 {
keys = append(keys, key(RetireGeneral, h.Eligible))
}
if h.Reserved > 0 {
keys = append(keys, key(RetireReserved, h.Eligible))
}
return append(keys, key(RetireTarget, h.Target())), nil
}
// ParseRetireKey splits a history index key into its kind, time and
// candidate.
func ParseRetireKey(k []byte) (byte, time.Time, CandidateID, error) {
if len(k) != HistoryIndexKeyLen || (k[0] != RetireGeneral && k[0] != RetireReserved && k[0] != RetireTarget) {
return 0, time.Time{}, "", ErrCorruptRecord
}
n, err := strconv.ParseInt(string(k[1:20]), 10, 64)
if err != nil || n <= 0 || fmt.Sprintf("%019d", n) != string(k[1:20]) {
return 0, time.Time{}, "", ErrCorruptRecord
}
id, err := ParseCandidateID(string(k[20:]))
if err != nil {
return 0, time.Time{}, "", ErrCorruptRecord
}
return k[0], fromNano(n), id, nil
}
// Validate checks the entry's invariants.
func (h HistoryEntry) Validate() error {
_, err := h.record()
return err
}
func (h HistoryEntry) MarshalBinary() ([]byte, error) {
rec, err := h.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalHistoryEntry decodes a stored history entry and checks its
// invariants.
func UnmarshalHistoryEntry(data []byte) (HistoryEntry, error) {
var rec historyEntryRecord
if err := openRecord(data, &rec); err != nil {
return HistoryEntry{}, err
}
h := HistoryEntry{LegacyCost: rec.V == 1, General: rec.General, Reserved: rec.Reserved, Ended: fromNano(rec.Ended), Eligible: fromNano(rec.Eligible), Pinned: rec.Pinned, RootMask: rec.RootMask, Released: rec.Released}
if again, err := h.record(); (rec.V != 1 && rec.V != historyEntryVersion) || err != nil || again != rec {
return HistoryEntry{}, ErrCorruptRecord
}
return h, nil
}
const evidenceRefsVersion = 1
// EvidenceRefs is how many stored candidates name one evidence record as a
// root. Evidence no candidate names is loose: it waits in a bounded ring,
// oldest out first, so rejected traffic allocates no unbounded rows.
type EvidenceRefs struct {
// Refs counts the stored candidates that name the evidence.
Refs uint32
// Loose is the evidence's position in the loose ring while no candidate
// names it; zero while one does.
Loose uint64
}
type evidenceRefsRecord struct {
V uint8 `json:"v"`
Refs uint32 `json:"refs,omitempty"`
Loose uint64 `json:"loose,omitempty"`
}
func (r EvidenceRefs) record() (evidenceRefsRecord, error) {
if (r.Refs == 0) == (r.Loose == 0) {
return evidenceRefsRecord{}, refuse(ReasonInvalid, "evidence must be named by a candidate or wait in the loose ring")
}
return evidenceRefsRecord{V: evidenceRefsVersion, Refs: r.Refs, Loose: r.Loose}, nil
}
func (r EvidenceRefs) MarshalBinary() ([]byte, error) {
rec, err := r.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalEvidenceRefs decodes a stored reference count and checks its
// invariants.
func UnmarshalEvidenceRefs(data []byte) (EvidenceRefs, error) {
var rec evidenceRefsRecord
if err := openRecord(data, &rec); err != nil {
return EvidenceRefs{}, err
}
r := EvidenceRefs{Refs: rec.Refs, Loose: rec.Loose}
if again, err := r.record(); rec.V != evidenceRefsVersion || err != nil || again != rec {
return EvidenceRefs{}, ErrCorruptRecord
}
return r, nil
}
package admission
import (
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"hash"
"strconv"
"strings"
"time"
)
// identityVersion is hashed into every derived ID. Changing the derivation
// requires a new version so old and new IDs never collide.
const identityVersion = 1
// EpisodeID names an offense episode. The engine assigns episodes; a
// producer, check or caller never does. The zero value is invalid.
type EpisodeID [16]byte
func (e EpisodeID) IsZero() bool { return e == EpisodeID{} }
func (e EpisodeID) String() string { return hex.EncodeToString(e[:]) }
// ParseEpisodeID accepts 32 lowercase hex digits naming a nonzero episode.
func ParseEpisodeID(s string) (EpisodeID, error) {
var e EpisodeID
if !lowerHex(s, 32) {
return e, refuse(ReasonInvalid, "episode ID is not 32 lowercase hex digits")
}
_, _ = hex.Decode(e[:], []byte(s))
if e.IsZero() {
return e, refuse(ReasonInvalid, "episode ID is zero")
}
return e, nil
}
// CandidateID is stable for one (kind, target, episode, root-evidence
// generation). Repeat reports of the same root set map to the same ID.
type CandidateID string
// ActionID is stable for one admitted attempt of a candidate, unchanged
// through crash recovery.
type ActionID string
// CandidateKey is everything a CandidateID is derived from.
type CandidateKey struct {
Kind Kind
Target Target
// Episode is assigned by the engine for the canonical address and
// logical service.
Episode EpisodeID
// Generation counts producer-verified new root sets within the
// episode, from 1. A new report ID is not a new generation.
Generation uint32
}
// ValidateKindTarget refuses a kind aimed at the wrong shape of target.
func ValidateKindTarget(k Kind, t Target) error {
_, hasService := t.Service()
switch {
case !k.Valid():
return refuse(ReasonInvalid, "unknown response kind")
case t.IsZero():
return refuse(ReasonInvalid, "no target")
case k == KindBlockService:
if !hasService {
return refuse(ReasonInvalid, "service block needs a service target")
}
case hasService:
return refuse(ReasonInvalid, "only a service block takes a service target")
case k == KindBlockSubnet:
if t.IsAddress() {
return refuse(ReasonInvalid, "subnet block needs a prefix wider than one address")
}
case !t.IsAddress():
return refuse(ReasonInvalid, "response needs a single-address target")
}
return nil
}
// ID derives the candidate ID.
func (k CandidateKey) ID() (CandidateID, error) {
if err := ValidateKindTarget(k.Kind, k.Target); err != nil {
return "", err
}
if k.Episode.IsZero() {
return "", refuse(ReasonInvalid, "candidate has no episode")
}
if k.Generation == 0 {
return "", refuse(ReasonInvalid, "candidate generation starts at 1")
}
h := sha256.New()
writeField(h, []byte{identityVersion, byte(k.Kind)})
writeField(h, []byte(k.Target.Key()))
writeField(h, k.Episode[:])
writeField(h, binary.BigEndian.AppendUint32(nil, k.Generation))
return CandidateID("cand_" + hex.EncodeToString(h.Sum(nil)[:16])), nil
}
// ParseCandidateID accepts only the derived form.
func ParseCandidateID(s string) (CandidateID, error) {
rest, ok := strings.CutPrefix(s, "cand_")
if !ok || !lowerHex(rest, 32) {
return "", refuse(ReasonInvalid, "malformed candidate ID")
}
return CandidateID(s), nil
}
// ParseActionID accepts only the derived form.
func ParseActionID(s string) (ActionID, error) {
rest, ok := strings.CutPrefix(s, "act_")
if !ok || !lowerHex(rest, 32) {
return "", refuse(ReasonInvalid, "malformed action ID")
}
return ActionID(s), nil
}
// ParseVictimID accepts the name of an eviction victim: an attempt's ID or
// an import's, the effect name the engine derives for a stamped legacy row
// ("eff_" and 32 lowercase hex digits).
func ParseVictimID(s string) (ActionID, error) {
if !validEffectName(s) {
return "", refuse(ReasonInvalid, "malformed victim ID")
}
return ActionID(s), nil
}
// ImportedVictim reports whether a names an import rather than an attempt.
func ImportedVictim(a ActionID) bool {
return strings.HasPrefix(string(a), "eff_") && validEffectName(string(a))
}
// Attempt links one admitted attempt to its candidate and to the attempt it
// follows. Replaying or recovering an admitted attempt keeps its ID. A later
// separately admitted attempt gets a new sequence and never overwrites an
// earlier outcome; the ledger enforces admission and predecessor state.
type Attempt struct {
ID ActionID
Candidate CandidateID
Seq uint32
Prev ActionID
}
func attemptID(c CandidateID, seq uint32) ActionID {
h := sha256.New()
writeField(h, []byte{identityVersion})
writeField(h, []byte(c))
writeField(h, binary.BigEndian.AppendUint32(nil, seq))
return ActionID("act_" + hex.EncodeToString(h.Sum(nil)[:16]))
}
// LegacyActionID names the seq-th charge imported from the legacy hourly
// counter for the hour ending at at. Its hash input has a field an
// attempt's never has, so the two never collide.
func LegacyActionID(at time.Time, seq uint32) ActionID {
h := sha256.New()
writeField(h, []byte{identityVersion})
writeField(h, []byte("legacy_hour"))
writeField(h, binary.BigEndian.AppendUint64(nil, uint64(at.UnixNano())))
writeField(h, binary.BigEndian.AppendUint32(nil, seq))
return ActionID("act_" + hex.EncodeToString(h.Sum(nil)[:16]))
}
// LiveStartActionID names the seq-th charge the live start carried from the
// legacy hour ending at at. It is named apart from the first import's
// charges, which may still be retained for the same hour.
func LiveStartActionID(at time.Time, seq uint32) ActionID {
h := sha256.New()
writeField(h, []byte{identityVersion})
writeField(h, []byte("live_start_hour"))
writeField(h, binary.BigEndian.AppendUint64(nil, uint64(at.UnixNano())))
writeField(h, binary.BigEndian.AppendUint32(nil, seq))
return ActionID("act_" + hex.EncodeToString(h.Sum(nil)[:16]))
}
// NewAttempt derives attempt seq of c. Sequences start at 1.
func NewAttempt(c CandidateID, seq uint32) (Attempt, error) {
if _, err := ParseCandidateID(string(c)); err != nil {
return Attempt{}, err
}
if seq == 0 {
return Attempt{}, refuse(ReasonInvalid, "attempt sequence starts at 1")
}
a := Attempt{ID: attemptID(c, seq), Candidate: c, Seq: seq}
if seq > 1 {
a.Prev = attemptID(c, seq-1)
}
return a, nil
}
// Validate recomputes the attempt's derived fields, so a stored attempt
// cannot be relinked to another candidate or sequence.
func (a Attempt) Validate() error {
want, err := NewAttempt(a.Candidate, a.Seq)
if err != nil {
return err
}
if a != want {
return refuse(ReasonInvalid, "attempt identity does not match its candidate and sequence")
}
return nil
}
// writeField writes b as a netstring ("<len>:<bytes>") so field
// boundaries cannot shift.
func writeField(h hash.Hash, b []byte) {
_, _ = h.Write(strconv.AppendInt(nil, int64(len(b)), 10))
_, _ = h.Write([]byte{':'})
_, _ = h.Write(b)
}
func lowerHex(s string, n int) bool {
if len(s) != n {
return false
}
for i := 0; i < len(s); i++ {
c := s[i]
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
return false
}
}
return true
}
// LiveStartPreviewActionID preserves a pre-start preview's charge as part
// of the conservative handover balance, separate from its preview attempt.
func LiveStartPreviewActionID(action ActionID) ActionID {
h := sha256.New()
writeField(h, []byte{identityVersion})
writeField(h, []byte("live_start_preview"))
writeField(h, []byte(action))
return ActionID("act_" + hex.EncodeToString(h.Sum(nil)[:16]))
}
package admission
import (
"bytes"
"errors"
"fmt"
"sync"
"time"
)
// Submission is the response kind, target and producer-minted evidence. Via
// names the derived entry, such as a challenge timeout or an incident,
// that answers the root evidence again; zero is the evidence's own entry.
type Submission struct {
Kind Kind
Target Target
Evidence Evidence
Via Entry
// PreviewTTL is a selected legacy lifetime; zero uses the owner fallback.
PreviewTTL time.Duration
}
// held identifies one held response: an observation may ask for several
// kinds, a derived entry answers the same root as its own response, and a
// range entry may name several prefixes holding it.
type held struct {
id EvidenceID
kind Kind
via Entry
target Target
}
func heldKeyOf(s Submission) held {
entry := s.Via
if entry == 0 {
entry = s.Evidence.Entry()
}
return held{id: s.Evidence.ID(), kind: s.Kind, via: entry, target: s.Target}
}
// IngressCheckpoint records the ingress decisions preceding a drain. Counters
// are a canonical QueueCounters record, copied at the handoff boundary.
// They count in-memory decisions, not durable candidate acceptance.
type IngressCheckpoint struct {
Generation uint64 `json:"generation"`
Sequence uint64 `json:"sequence"`
Cursors QueueCursors `json:"cursors"`
Counters []byte `json:"counters"`
}
// QueueSnapshot is an immutable owner publication. Revision orders committed
// database snapshots; Checkpoint acknowledges only the decisions it contains.
type QueueSnapshot struct {
Now time.Time
Inventory *Inventory
Items []QueueItem
Cursors QueueCursors
Revision int
Generation uint64
Checkpoint *IngressCheckpoint
}
// IngressItem is an accepted submission waiting for the owner. Reports and
// Dropped are the unacknowledged tail frozen by Take.
type IngressItem struct {
Submission Submission
Reports []string
Dropped uint32
// Selected is the unacknowledged count, including memory merges.
Selected uint64
ReportsOnly bool
key string
}
// IngressStats counts in-memory decisions. Its counters are checkpointed;
// accepted and duplicate totals and the Critical latch are process-local.
type IngressStats struct {
Accepted uint64
Duplicates uint64
Counters QueueCounters
CriticalLost uint64
}
type pending struct {
item IngressItem
pos QueueItem
taken bool
ackReports int
ackDropped uint32
ackSelected uint64
}
// Ingress holds at most IngressPositions items per partition. Stored work
// cannot borrow this allocation, including while a commit awaits completion.
// Submit takes only the ingress mutex and never calls a ledger method.
type Ingress struct {
reg *Registry
mu sync.Mutex
snap *QueueSnapshot
view *QueueView
items []*pending
byKey map[string]*pending
byEvidence map[held]*pending
seq uint64
stats IngressStats
cursors QueueCursors
generation uint64
revision int
initialized bool
// closed refuses every later submission and snapshot for good.
closed bool
// mono reads the monotonic clock. Submissions are judged at the
// snapshot's admission time plus the time elapsed since publication.
mono func() time.Time
published time.Time
// stoppedAt is when admission last closed, zero while a usable
// snapshot is published; criticalRefused counts the Critical arrivals
// refused since.
stoppedAt time.Time
criticalRefused uint64
// These observers are set before any submission or drain.
arrived func(DrainedArrival)
displaced func(DrainedArrival)
}
func NewIngress(reg *Registry) (*Ingress, error) {
if reg == nil || !reg.Sealed() {
return nil, fmt.Errorf("ingress needs a sealed producer registry")
}
return &Ingress{reg: reg, byKey: map[string]*pending{}, byEvidence: map[held]*pending{}, mono: time.Now, stoppedAt: time.Now()}, nil
}
// setSnapshot publishes s, or closes admission when s is nil. Closing an
// open ingress records when it stopped; a usable snapshot clears that.
func (in *Ingress) setSnapshot(s *QueueSnapshot) {
switch {
case s == nil && in.snap != nil:
in.stoppedAt, in.criticalRefused = in.mono(), 0
case s != nil:
in.stoppedAt, in.criticalRefused = time.Time{}, 0
}
in.snap = s
}
// Health reports whether the ingress admits and, while it does not, since
// when and how many Critical arrivals it refused. It is process-local, so
// it holds when the ledger is the damaged part.
func (in *Ingress) Health() IngressHealth {
in.mu.Lock()
defer in.mu.Unlock()
return IngressHealth{Admitting: in.snap != nil, StoppedSince: in.stoppedAt, CriticalRefused: in.criticalRefused}
}
func (in *Ingress) lose(event QueueEvent, reason Reason, tier Tier, sev Severity) {
_ = in.stats.Counters.Add(CountKey{Event: event, Reason: reason, Class: tier.Class, Severity: tier.Severity})
if sev == SeverityCritical {
in.stats.CriticalLost++
}
}
// Submit acknowledges memory acceptance only. An equal evidence record merges;
// a later finding is retained even while the owner persists an earlier tail.
func (in *Ingress) Submit(s Submission) error {
in.mu.Lock()
defer in.mu.Unlock()
in.seq++
e := s.Evidence
refused := func(err error, tier Tier) error {
reason, _ := ReasonOf(err)
in.lose(EventRefused, reason, tier, e.Severity())
return err
}
if in.snap == nil {
if e.Severity() == SeverityCritical {
in.criticalRefused++
}
return refused(refuse(ReasonEngineUnavailable, "the ledger owner has published no queue snapshot"), Tier{})
}
if err := in.reg.Validate(e); err != nil {
return refused(err, Tier{})
}
if s.PreviewTTL < 0 {
return refused(refuse(ReasonInvalid, "response lifetime is negative"), Tier{})
}
if s.Via != 0 {
if err := in.reg.ValidateVia(s.Via, e); err != nil {
return refused(err, Tier{})
}
}
if err := ValidateKindTarget(s.Kind, s.Target); err != nil {
return refused(err, Tier{})
}
// Validate the submission envelope before merging by evidence identity.
if err := BindTarget(s.Target, s.Via, e); err != nil {
return refused(err, Tier{})
}
if h := in.byEvidence[heldKeyOf(s)]; h != nil {
switch {
case h.item.Submission.Evidence.Equal(e):
in.stats.Duplicates++
case h.item.Submission.Evidence.SameExceptFinding(e):
in.stats.Duplicates++
h.addReport(e.FindingID())
default:
return refused(h.item.Submission.Evidence.Conflict(e), Tier{})
}
h.item.Selected++
return nil
}
// An idle owner must not make fresh evidence look future-dated, and
// wall-clock steps must not move this time: advance the snapshot's
// admission time by elapsed monotonic time only.
at := in.snap.Now
if elapsed := in.mono().Sub(in.published); elapsed > 0 {
at = at.Add(elapsed)
}
a, err := Assess(s.Target, []Evidence{e}, at)
if err != nil {
return refused(err, Tier{})
}
scope, eligible := Scope{Owner: e.Owner(), Effect: s.Kind.Effect()}, a.Reserved()
if in.snap.Inventory == nil || !in.snap.Inventory.Current(scope.Owner) {
scope.Owner, eligible = HostOwner(), false
}
p := &pending{item: IngressItem{Submission: s, Selected: 1, key: fmt.Sprintf("in:%020d", in.seq)}}
p.pos = QueueItem{Key: p.item.key, Scope: scope.Key(), Tier: a.Tier, Eligible: eligible, Queued: in.snap.Now, Seq: in.seq}
placed, ok := in.view.Admit(p.pos)
in.cursors = in.view.Cursors()
if !ok {
return refused(refuse(ReasonQueueOverflow, "queue has no position for the submission"), a.Tier)
}
p.pos.Partition = placed.Partition
if placed.Displaced {
victim := in.byKey[placed.Victim.Key]
in.drop(victim)
in.lose(EventEnded, ReasonQueueOverflow, victim.pos.Tier, victim.item.Submission.Evidence.Severity())
if in.displaced != nil {
in.displaced(DrainedArrival{Submission: victim.item.Submission,
Result: ArrivalResult{Err: refuse(ReasonQueueOverflow, "held submission displaced")},
Selected: victim.item.Selected - victim.ackSelected})
}
}
in.items = append(in.items, p)
in.byKey[p.item.key] = p
in.byEvidence[heldKeyOf(s)] = p
in.stats.Accepted++
return nil
}
// Inventory is the inventory of the published snapshot, nil while admission
// is closed. Producers resolve claims against it, as Submit scopes them.
func (in *Ingress) Inventory() *Inventory {
in.mu.Lock()
defer in.mu.Unlock()
if in.snap == nil {
return nil
}
return in.snap.Inventory
}
// Refuse counts a response whose evidence could not be minted, as Submit
// counts one it refuses before assessing it.
func (in *Ingress) Refuse(err error, sev Severity) {
in.mu.Lock()
defer in.mu.Unlock()
in.seq++
if in.snap == nil && sev == SeverityCritical {
in.criticalRefused++
}
reason, ok := ReasonOf(err)
if !ok {
reason = ReasonInvalid
}
in.lose(EventRefused, reason, Tier{}, sev)
}
// Close refuses every later submission and keeps admission closed whatever
// is published later. Held work can still be drained, so a stop persists a
// bounded set even while a producer keeps submitting.
func (in *Ingress) Close() {
in.mu.Lock()
defer in.mu.Unlock()
in.closed = true
in.setSnapshot(nil)
}
func (p *pending) addReport(finding string) {
for _, have := range p.item.Reports {
if have == finding {
return
}
}
if len(p.item.Reports) < MaxReportLinks {
p.item.Reports = append(p.item.Reports, finding)
} else if p.item.Dropped != ^uint32(0) {
p.item.Dropped++
}
}
func (in *Ingress) drop(p *pending) {
delete(in.byKey, p.item.key)
delete(in.byEvidence, heldKeyOf(p.item.Submission))
for i, have := range in.items {
if have == p {
in.items = append(in.items[:i], in.items[i+1:]...)
break
}
}
}
// Publish cannot overwrite newer local decisions or a newer durable snapshot.
// A new generation uses a new Ingress; retained work never crosses generations.
func (in *Ingress) Publish(snap *QueueSnapshot) {
in.mu.Lock()
defer in.mu.Unlock()
in.publish(snap)
}
func (in *Ingress) publish(snap *QueueSnapshot) {
if snap == nil || in.closed {
in.setSnapshot(nil)
return
}
if in.initialized && (snap.Generation != in.generation || snap.Revision < in.revision) {
return
}
if !in.initialized {
// A checkpoint that cannot be decoded applies nothing, not even
// the generation: admission stays closed until a usable snapshot.
if cp := snap.Checkpoint; cp != nil {
counts, err := UnmarshalQueueCounters(cp.Counters)
if err != nil {
return
}
in.cursors = cp.Cursors
// Generations retain lifetime counters and the last cursor, but start
// a new sequence. A same-generation restart resumes the checkpoint.
// Refusals before this first snapshot are additional local decisions.
if cp.Generation == snap.Generation {
in.seq += cp.Sequence
}
for k, n := range counts.counts {
before := in.stats.Counters.Count(k)
if in.stats.Counters.counts == nil {
in.stats.Counters.counts = map[CountKey]uint64{}
}
in.stats.Counters.counts[k] = n + min(before, ^uint64(0)-n)
}
}
in.generation = snap.Generation
in.initialized = true
}
// Every item must hold one position of the view: an item the view
// would reject still raises the partition limit in rebuild.
counts := [partitionEnd]int{}
keys := make(map[string]bool, len(snap.Items))
for _, it := range snap.Items {
if !it.Partition.Valid() || it.Key == "" || it.Scope == "" || keys[it.Key] {
in.setSnapshot(nil)
return
}
keys[it.Key] = true
counts[it.Partition]++
if counts[it.Partition] > it.Partition.DurableCapacity() {
in.setSnapshot(nil)
return
}
}
own := *snap
own.Items = append([]QueueItem(nil), snap.Items...)
in.setSnapshot(&own)
in.revision, in.published = snap.Revision, in.mono()
in.rebuild()
}
// Durable victims remain present and fixed. The view's capacity is the
// durable occupancy plus its transfer allocation, so pressure can replace
// only held work from an over-share scope or a strictly lower own tier.
func (in *Ingress) rebuild() {
if in.snap == nil {
return
}
v := NewQueueView(in.cursors)
for p := PartitionGeneral; p < partitionEnd; p++ {
v.limits[p] = IngressPositions
}
for _, it := range in.snap.Items {
it.Fixed = true
_ = v.Add(it)
v.limits[it.Partition]++
}
for _, p := range in.items {
owner := p.item.Submission.Evidence.Owner()
if in.snap.Inventory == nil || !in.snap.Inventory.Current(owner) {
p.pos.Scope = (Scope{Owner: HostOwner(), Effect: p.item.Submission.Kind.Effect()}).Key()
p.pos.Eligible = false
}
// Retain the transfer allocation through handoff, even when an
// inventory publication retires the owner of a taken item.
pos := p.pos
pos.Fixed = p.taken || p.item.ReportsOnly
_ = v.Add(pos)
}
in.view = v
}
// Take freezes an owned copy of each unacknowledged report tail. Taken work
// retains transfer capacity until commit and acknowledgement both finish.
func (in *Ingress) Take(n int) []IngressItem {
in.mu.Lock()
defer in.mu.Unlock()
var out []IngressItem
for _, p := range in.items {
if len(out) >= n {
break
}
if p.taken {
continue
}
p.taken = true
it := p.item
it.Reports = append([]string(nil), p.item.Reports[p.ackReports:]...)
it.Dropped = p.item.Dropped - p.ackDropped
it.Selected = p.item.Selected - p.ackSelected
out = append(out, it)
}
in.rebuild()
return out
}
func (in *Ingress) Release(items []IngressItem) {
in.mu.Lock()
defer in.mu.Unlock()
for _, it := range items {
if p := in.byKey[it.key]; p != nil {
p.taken = false
}
}
in.rebuild()
}
// Complete acknowledges only the taken tail. Reports accepted during the
// commit remain held for a report-only transaction and cannot create work.
// revision is the last committed group revision, including checkpoints.
func (in *Ingress) Complete(items []IngressItem, revision int, snap *QueueSnapshot) {
in.mu.Lock()
defer in.mu.Unlock()
for _, it := range items {
if p := in.byKey[it.key]; p != nil && p.taken {
p.ackReports += len(it.Reports)
p.ackDropped += it.Dropped
p.ackSelected += it.Selected
if p.ackReports < len(p.item.Reports) || p.ackDropped < p.item.Dropped || p.ackSelected < p.item.Selected {
p.taken = false
p.item.ReportsOnly = true
} else {
in.drop(p)
}
}
}
// A stale or other-generation snapshot must not leave the previous
// occupancy usable after acknowledgement, even if its read succeeded.
in.revision = max(in.revision, revision)
in.setSnapshot(nil)
in.publish(snap)
in.rebuild()
}
// Checkpoint copies the current cursor and fixed-dimensional counts. A
// failed transaction never acknowledges it; later submissions keep advancing.
func (in *Ingress) Checkpoint() IngressCheckpoint {
in.mu.Lock()
defer in.mu.Unlock()
counts, _ := in.stats.Counters.MarshalBinary()
return IngressCheckpoint{Generation: in.generation, Sequence: in.seq, Cursors: in.cursors, Counters: counts}
}
func (in *Ingress) Len() int {
in.mu.Lock()
defer in.mu.Unlock()
return len(in.items)
}
func (in *Ingress) Stats() IngressStats {
in.mu.Lock()
defer in.mu.Unlock()
out := in.stats
out.Counters = QueueCounters{counts: make(map[CountKey]uint64, len(in.stats.Counters.counts))}
for k, n := range in.stats.Counters.counts {
out.Counters.counts[k] = n
}
return out
}
// DrainReport counts durable decisions made by one drain.
type DrainReport struct{ Queued, Coalesced, Refused, Failed int }
// DrainedArrival is one submission and its committed, isolated or displaced decision.
type DrainedArrival struct {
Submission Submission
Result ArrivalResult
Selected uint64
}
// ObserveArrivals hands fn each committed or isolated submission with the ledger's
// decision on it, as every later drain commits them. Set it before the
// first drain; fn runs on the draining goroutine.
func (in *Ingress) ObserveArrivals(fn func(DrainedArrival)) { in.arrived = fn }
// ObserveDisplacements hands fn each held submission lost to queue pressure.
// Set it before the first submission. fn runs under the ingress mutex on the
// submitting goroutine and must not call the ingress or ledger.
func (in *Ingress) ObserveDisplacements(fn func(DrainedArrival)) { in.displaced = fn }
// Drain checkpoints ingress decisions even with no held items. Transient
// errors release work; only damaged arrivals are isolated and counted lost.
// When that damage is the only failure, admission stays open and the error
// is ErrArrivalsIsolated with the damage. A later failure retains both
// causes without ErrArrivalsIsolated. One owner serializes Drain with every
// other mutating ledger operation.
func (in *Ingress) Drain(l Ledger, limit int, request func(Submission) (CandidateRequest, error)) (DrainReport, error) {
items := in.Take(min(limit, MaxArrivalGroup))
return in.DrainTaken(l, items, request)
}
// DrainTaken persists the group frozen by Take. The owner can record a
// fresh clock reading after freezing it, so observations submitted during
// that reading stay held for a later group rather than looking future-dated.
// Errors release the group just as Drain does.
func (in *Ingress) DrainTaken(l Ledger, items []IngressItem, request func(Submission) (CandidateRequest, error)) (DrainReport, error) {
var report DrainReport
arrivals := make([]Arrival, 0, len(items))
for _, it := range items {
var req CandidateRequest
if !it.ReportsOnly {
var err error
req, err = request(it.Submission)
if err != nil {
in.Release(items)
return report, err
}
}
arrivals = append(arrivals, Arrival{Request: req, Evidence: it.Submission.Evidence, Reports: it.Reports, Dropped: it.Dropped, ReportsOnly: it.ReportsOnly})
}
checkpoint := in.Checkpoint()
results, revision, err := l.EnqueueGroup(arrivals, &checkpoint)
if err != nil && !errors.Is(err, ErrCorruptRecord) {
in.Release(items)
return report, err
}
done := items
var first, damage error
if err != nil {
// Damage an empty group also meets is shared, not an arrival's:
// every item goes back and none is counted lost.
if _, revision, err = l.EnqueueGroup(nil, &checkpoint); err != nil {
in.Release(items)
return report, err
}
done = nil
results = nil
isolating:
for i, a := range arrivals {
one, committed, oneErr := l.EnqueueGroup([]Arrival{a}, &checkpoint)
switch {
case errors.Is(oneErr, ErrCorruptRecord):
selected := in.discard(items[i])
report.Failed++
if in.arrived != nil {
in.arrived(DrainedArrival{Submission: items[i].Submission, Result: ArrivalResult{Err: oneErr}, Selected: selected})
}
if damage == nil {
damage = oneErr
}
case oneErr != nil:
in.Release(items[i:])
first = oneErr
break isolating
default:
revision = committed
done = append(done, items[i])
results = append(results, one...)
}
}
// Discard decisions belong to a separate checkpoint after the failed
// candidate transactions. A failure leaves them pending for the next drain.
checkpoint = in.Checkpoint()
if _, committed, checkpointErr := l.EnqueueGroup(nil, &checkpoint); checkpointErr != nil {
if first == nil {
first = checkpointErr
}
} else {
revision = committed
}
}
// Results follow the committed items in order.
for i, r := range results {
switch {
case r.Err != nil:
report.Refused++
case r.Created:
report.Queued++
default:
report.Coalesced++
}
if in.arrived != nil {
in.arrived(DrainedArrival{Submission: done[i].Submission, Result: r, Selected: done[i].Selected})
}
}
snap, err := l.QueueSnapshot()
if err != nil {
snap = nil
if first == nil {
first = err
}
}
// A readable snapshot does not mean an isolating drain succeeded.
// Reopening here would reset the stop on every failed retry before the
// owner can close admission again.
if first != nil {
snap = nil
}
in.Complete(done, revision, snap)
if damage != nil {
if first == nil {
// Everything else committed: one damaged record is not proof the
// ledger failed, so admission stays open.
first = fmt.Errorf("%w: %w", ErrArrivalsIsolated, damage)
} else {
// Recovery from the later failure cannot recover discarded work;
// retain the damage so the owner can keep its operator signal.
first = fmt.Errorf("%w; isolated arrival damage: %w", first, damage)
}
}
return report, first
}
func (in *Ingress) discard(it IngressItem) uint64 {
in.mu.Lock()
defer in.mu.Unlock()
var selected uint64
if p := in.byKey[it.key]; p != nil {
selected = p.item.Selected - p.ackSelected
in.seq++
in.lose(EventEnded, ReasonInvalid, p.pos.Tier, p.item.Submission.Evidence.Severity())
in.drop(p)
}
in.rebuild()
return selected
}
// Validate rejects a stale generation, regressed decision sequence or counters.
// Identical checkpoints are idempotent when damage isolation splits a group.
func (c IngressCheckpoint) Validate(previous *IngressCheckpoint, generation uint64) error {
if c.Generation == 0 || c.Generation != generation || !validCursor(c.Cursors.General) || !validCursor(c.Cursors.Reserved) {
return ErrTransitionConflict
}
counts, err := UnmarshalQueueCounters(c.Counters)
if err != nil {
return err
}
if previous == nil {
return nil
}
if previous.Generation == c.Generation {
if c.Sequence < previous.Sequence {
return ErrTransitionConflict
}
if c.Sequence == previous.Sequence && (c.Cursors != previous.Cursors || !bytes.Equal(c.Counters, previous.Counters)) {
return ErrTransitionConflict
}
}
old, err := UnmarshalQueueCounters(previous.Counters)
if err != nil {
return err
}
for k, n := range old.counts {
if counts.Count(k) < n {
return ErrTransitionConflict
}
}
return nil
}
package admission
import (
"crypto/sha256"
"encoding/hex"
"slices"
"strings"
"time"
)
// MaxIntentBytes bounds one encoded intent, with the victim of an
// evicting block.
const MaxIntentBytes = 384
// UnknownHoldBytes is what an unknown outcome holds in the recovery reserve
// besides its history and its rows: its intent and the slot of the row that
// retires it.
const UnknownHoldBytes = AuditSlotBytes + MaxIntentBytes
// AttemptRecoveryBytes is the room one reservation needs in the recovery
// reserve besides its history: its attempt's audit rows and what an unknown
// outcome holds.
const AttemptRecoveryBytes = AttemptAuditBytes + UnknownHoldBytes
// intentVersion is the encoding of an intent without a victim; an
// evicting intent is victimIntentVersion.
const (
intentVersion = 1
victimIntentVersion = 2
)
// BatchID names one execution batch.
type BatchID string
// NewBatchID derives the ID of the batch whose members are members: from
// the set, not its order, so executing the same members again names the
// same batch and any other set names another.
func NewBatchID(members []ActionID) (BatchID, error) {
if len(members) == 0 || len(members) > MaxBatchMembers {
return "", refuse(ReasonInvalid, "a batch has one to MaxBatchMembers members")
}
sorted := slices.Clone(members)
slices.Sort(sorted)
h := sha256.New()
writeField(h, []byte{identityVersion})
writeField(h, []byte("batch"))
for i, id := range sorted {
if _, err := ParseActionID(string(id)); err != nil {
return "", err
}
if i > 0 && sorted[i-1] == id {
return "", refuse(ReasonInvalid, "a batch names a member twice")
}
writeField(h, []byte(id))
}
return BatchID("bat_" + hex.EncodeToString(h.Sum(nil)[:16])), nil
}
// ParseBatchID accepts only the derived form.
func ParseBatchID(s string) (BatchID, error) {
rest, ok := strings.CutPrefix(s, "bat_")
if !ok || !lowerHex(rest, 32) {
return "", refuse(ReasonInvalid, "malformed batch ID")
}
return BatchID(s), nil
}
// Intent is what one attempt may apply, recorded before the attempt
// dispatches any host change: the batch it runs in, its kind,
// its target and the absolute expiry of the effect. Every member adds its
// target until that expiry; a member that evicts also removes its victim.
// The ledger keeps it while the attempt runs and, after an unknown
// outcome, until recovery retires the uncertainty, so a restart knows
// which effects are unproven.
type Intent struct {
Action ActionID
Batch BatchID
Kind Kind
Target Target
ExpiresAt time.Time
// Victim is the automatic block the member evicts; zero when it takes
// a free slot. Only a block_ip member evicts, one victim for its one
// unit.
Victim Victim
}
// Victim is an automatic block an evicting member removes: the attempt
// that applied it or, for an import, its effect name, its address and the
// absolute expiry it was applied with.
type Victim struct {
Action ActionID
Target Target
ExpiresAt time.Time
}
// VictimOutcome is what a verified block's attempt did to the victim it
// claimed, as the applier proved it.
type VictimOutcome uint8
const (
// VictimRemoved: the batch removed the victim and its readback proved
// it gone, so the victim is evicted.
VictimRemoved VictimOutcome = iota + 1
// VictimSpared: the batch never removed the victim, which returns to
// the pool with its slot.
VictimSpared
// VictimUnproven: the batch sent the removal without proving it. The
// victim may still be enforced: the attempt's outcome is unknown,
// which keeps the claim until the victim's expiry.
VictimUnproven
)
type intentRecord struct {
V uint8 `json:"v"`
Action ActionID `json:"action"`
Batch BatchID `json:"batch"`
Kind Kind `json:"kind"`
Target string `json:"target"`
ExpiresAt int64 `json:"expires_at"`
Victim ActionID `json:"victim,omitempty"`
VictimAt string `json:"victim_target,omitempty"`
VictimEnd int64 `json:"victim_expires,omitempty"`
}
func (in Intent) record() (intentRecord, error) {
if _, err := ParseActionID(string(in.Action)); err != nil {
return intentRecord{}, err
}
if _, err := ParseBatchID(string(in.Batch)); err != nil {
return intentRecord{}, err
}
if err := ValidateKindTarget(in.Kind, in.Target); err != nil {
return intentRecord{}, err
}
expires, ok := unixNano(in.ExpiresAt)
if !ok || expires < 0 {
return intentRecord{}, refuse(ReasonInvalid, "intent has no expiry after the epoch")
}
rec := intentRecord{V: intentVersion, Action: in.Action, Batch: in.Batch, Kind: in.Kind, Target: in.Target.Key(), ExpiresAt: expires}
if in.Victim == (Victim{}) {
return rec, nil
}
v := in.Victim
ends, ok := unixNano(v.ExpiresAt)
switch {
case in.Kind != KindBlockIP:
return intentRecord{}, refuse(ReasonInvalid, "only a block_ip member evicts")
case ValidateKindTarget(KindBlockIP, v.Target) != nil:
return intentRecord{}, refuse(ReasonInvalid, "intent victim is not an address")
case v.Target == in.Target || v.Action == in.Action:
return intentRecord{}, refuse(ReasonInvalid, "a member cannot evict itself")
case !ok || ends <= 0:
return intentRecord{}, refuse(ReasonInvalid, "intent victim has no expiry after the epoch")
}
if _, err := ParseVictimID(string(v.Action)); err != nil {
return intentRecord{}, err
}
rec.V, rec.Victim, rec.VictimAt, rec.VictimEnd = victimIntentVersion, v.Action, v.Target.Key(), ends
return rec, nil
}
// Validate checks the intent's invariants.
func (in Intent) Validate() error {
_, err := in.record()
return err
}
func (in Intent) MarshalBinary() ([]byte, error) {
rec, err := in.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalIntent decodes a stored intent and checks its invariants.
func UnmarshalIntent(data []byte) (Intent, error) {
var rec intentRecord
if err := openRecord(data, &rec); err != nil {
return Intent{}, err
}
target, err := ParseTargetKey(rec.Target, Caps{IPv6: true})
if err != nil {
return Intent{}, ErrCorruptRecord
}
in := Intent{Action: rec.Action, Batch: rec.Batch, Kind: rec.Kind, Target: target, ExpiresAt: fromNano(rec.ExpiresAt)}
if rec.Victim != "" {
victim, err := ParseTargetKey(rec.VictimAt, Caps{IPv6: true})
if err != nil {
return Intent{}, ErrCorruptRecord
}
in.Victim = Victim{Action: rec.Victim, Target: victim, ExpiresAt: fromNano(rec.VictimEnd)}
}
if again, err := in.record(); err != nil || again != rec {
return Intent{}, ErrCorruptRecord
}
return in, nil
}
package admission
import "time"
// LegacyHandover is what legacy blocking's state holds when the owner first
// executes: the current hour's count. Pending entries stay legacy's to
// retry: a finding admission takes never enters that queue.
type LegacyHandover struct {
Spend LegacySpend
}
const liveStartVersion = 1
// LiveStart is the ledger's record that its live start ran, at admission
// time At.
type LiveStart struct {
At time.Time
}
type liveStartRecord struct {
V uint8 `json:"v"`
At int64 `json:"at"`
}
func (s LiveStart) MarshalBinary() ([]byte, error) {
at, ok := unixNano(s.At)
if !ok {
return nil, refuse(ReasonInvalid, "live start has no time")
}
return sealRecord(liveStartRecord{V: liveStartVersion, At: at})
}
// UnmarshalLiveStart decodes a stored live start record.
func UnmarshalLiveStart(data []byte) (LiveStart, error) {
var rec liveStartRecord
if err := openRecord(data, &rec); err != nil {
return LiveStart{}, err
}
if rec.V != liveStartVersion || rec.At == 0 {
return LiveStart{}, ErrCorruptRecord
}
return LiveStart{At: fromNano(rec.At)}, nil
}
// CarryLegacy charges, at the live start, the legacy hour's units that the
// ledger's retained charges in that hour do not already count (design C7):
// observe previews and legacy blocking describe the same responses, so the
// hour counts the greater of the two, never their sum. The extra units are
// dated at the hour's end, as an import's are, on the general lane up to
// its room and the rest on the reserved allowance, and spend saved credit
// without a budget check: legacy blocking already spent them. Usage never
// passes MaxCeiling. An unreadable legacy counter keeps no saved credit, as
// a failed first import grants none.
func (s CeilingState) CarryLegacy(spend LegacySpend, retained uint32) (CeilingState, []Charge, error) {
if s.Limit == 0 {
return s, nil, refuse(ReasonEngineUnavailable, "ceiling is not set")
}
if spend.Unknown {
s.General.Credit, s.Reserved.Credit = 0, 0
return s, nil, nil
}
units := min(spend.Units, MaxCeiling)
if units == 0 {
return s, nil, nil
}
if at, ok := unixNano(spend.At); !ok || at <= 0 {
return s, nil, refuse(ReasonInvalid, "legacy spend names no hour")
}
used := s.General.Used + s.Reserved.Used
if units <= retained || used >= MaxCeiling {
return s, nil, nil
}
extra := min(units-retained, MaxCeiling-used)
g, _ := CeilingLanes(s.Limit)
general := min(extra, room(g, s.General.Used))
var charges []Charge
var seq uint32
for _, part := range []struct {
lane Lane
m *Meter
units uint32
}{{LaneGeneral, &s.General, general}, {LaneDirect, &s.Reserved, extra - general}} {
part.m.Used += part.units
part.m.Credit -= min(part.m.Credit, uint64(part.units)*unitTicks)
for part.units > 0 {
cost := min(part.units, MaxMemberCost)
seq++
charges = append(charges, Charge{At: spend.At, Action: LiveStartActionID(spend.At, seq), Lane: part.lane, Cost: cost, Elapsed: s.Elapsed})
part.units -= cost
}
}
return s, charges, nil
}
package admission
import (
"fmt"
"math"
"strconv"
"time"
)
// Notice bounds. Response notices coalesce into one record per
// key in the outbox; each record is charged a fixed slot.
const (
// MaxNoticeExamples bounds the example candidates one record keeps.
MaxNoticeExamples = 20
// MaxNoticeRecordBytes bounds one encoded notice record.
MaxNoticeRecordBytes = 2048
// NoticeKeyMaxLen is the longest notice key: a kind byte, the notice
// kind, reason, outcome and effect, and a check name of at most 64
// bytes.
NoticeKeyMaxLen = 5 + 64
// NoticeQuietKeyMaxLen is the longest quiet index key: a kind byte, a
// time as 19 decimal digits and the record's key.
NoticeQuietKeyMaxLen = 1 + 19 + NoticeKeyMaxLen
// NoticeSlotBytes is what one record holds in the outbox, with its key
// and its quiet index key.
NoticeSlotBytes = NoticeKeyMaxLen + NoticeQuietKeyMaxLen + MaxNoticeRecordBytes
// NoticeBytes is the share of the recovery and outbox reserve notice
// records may hold. Keys combine a reason with any registered check, so
// an attacker choosing which checks fire could otherwise fill the
// reserve and stop admission. An event whose key finds no room is
// counted in its kind's overflow record.
NoticeBytes = 8 << 20
// FixedNotices is the records every ledger keeps from its creation: an
// overflow record per keyed kind and the two summaries. They are
// charged in advance and never refused.
FixedNotices = 5
)
// noticeKind is the outbox key kind of notice records.
const noticeKind = 'n'
// NoticeKind is what a notice record reports.
type NoticeKind uint8
const (
// NoticeWithheld: Critical work did not receive its response.
NoticeWithheld NoticeKind = iota + 1
// NoticeWithheldWarning: C3 work below Critical did not receive it.
NoticeWithheldWarning
// NoticeCapacity: Critical work waits on capacity, or Critical or
// reserved work was lost to it.
NoticeCapacity
// NoticeCriticalSummary counts every Critical notice event, so a flood
// of other keys cannot delay the first Critical gap.
NoticeCriticalSummary
// NoticeAppliedSummary counts verified responses.
NoticeAppliedSummary
noticeKindEnd
)
var noticeKindNames = [...]string{"", "withheld", "withheld_warning", "capacity", "critical_summary", "applied_summary"}
func (k NoticeKind) Valid() bool { return k >= NoticeWithheld && k < noticeKindEnd }
func (k NoticeKind) String() string {
if k.Valid() {
return noticeKindNames[k]
}
return fmt.Sprintf("notice(%d)", uint8(k))
}
// Keyed reports whether records of k are kept per key rather than fixed.
func (k NoticeKind) Keyed() bool { return k >= NoticeWithheld && k <= NoticeCapacity }
// Critical reports whether an event of k is a Critical response gap.
func (k NoticeKind) Critical() bool { return k == NoticeWithheld || k == NoticeCapacity }
// Check is the registered check a notice of k is delivered as.
func (k NoticeKind) Check() string {
switch k {
case NoticeCapacity:
return "response_capacity_exhausted"
case NoticeAppliedSummary:
return "auto_block"
}
return "auto_response_withheld"
}
// AppliedSummaryMessage is the message a live NoticeAppliedSummary is
// delivered with. Alert classification names it a summary notice by it.
const AppliedSummaryMessage = "Automatic responses applied summary"
// Severity is the severity a notice of k is delivered with.
func (k NoticeKind) Severity() Severity {
if k == NoticeWithheldWarning || k == NoticeAppliedSummary {
return SeverityWarning
}
return SeverityCritical
}
// Interval is the least time between two deliveries of one record: an
// hour per key, a minute for each summary.
func (k NoticeKind) Interval() time.Duration {
if k.Keyed() {
return time.Hour
}
return time.Minute
}
// Capacity reports whether r is a capacity reason: the response waits on,
// or was lost to, a bound the ledger enforces.
func (r Reason) Capacity() bool {
switch r {
case ReasonCeiling, ReasonSetFull, ReasonStorageShare, ReasonPendingRecovery, ReasonQueueOverflow:
return true
}
return false
}
// GapEvent is the kind of event a notice can follow.
type GapEvent uint8
const (
// GapDeferred: a queued candidate took a new deferral reason.
GapDeferred GapEvent = iota + 1
// GapEnded: a candidate or arrival ended without an attempt outcome.
GapEnded
// GapOutcome: an attempt recorded its outcome.
GapOutcome
)
// GapNotice is the notice an event raises, or zero. tier is
// the last assessment, zero for unassessed work, which raises none. C3 is
// exactly the work the reserved lane serves, so a lost C3 candidate is
// lost reserved work. Capacity reasons raise only the capacity notice. Only a
// failure that ends the candidate or an unknown outcome is a gap; an
// existing verified effect is not. Once the ledger runs live, work a full
// set defers raises the capacity notice at any tier; a preview
// raises it for Critical work only.
func GapNotice(event GapEvent, reason Reason, outcome Disposition, tier Tier, live bool) NoticeKind {
critical, c3 := tier.Severity == SeverityCritical, tier.Class == ClassC3
switch event {
case GapDeferred:
switch {
case live && reason == ReasonSetFull && tier.Valid():
return NoticeCapacity
case !critical:
return 0
case reason.Capacity():
return NoticeCapacity
}
return NoticeWithheld
case GapEnded:
switch {
case reason == ReasonExistingEffect, reason == ReasonDisabled:
// An existing effect answers the work; the operator turned
// disabled work off. Neither withholds a wanted response.
return 0
case reason.Capacity():
if critical || c3 {
return NoticeCapacity
}
return 0
}
case GapOutcome:
if outcome != DispositionFailed && outcome != DispositionUnknown {
return 0
}
default:
return 0
}
switch {
case critical:
return NoticeWithheld
case c3:
return NoticeWithheldWarning
}
return 0
}
// NoticeKey coalesces notice events: one record per fixed reason,
// registered check and action family. An attempt that failed
// for good or ended unknown has no reason; its outcome takes the reason's
// place. Check and Effect are empty for an arrival lost before admission,
// which names no candidate. A keyed kind with neither reason nor outcome
// is that kind's overflow record; a summary has only its kind. The one key
// with both is the partial-containment record (PartialContainmentKey).
type NoticeKey struct {
Kind NoticeKind
Reason Reason
Outcome Disposition
Check string
Effect Effect
}
// OverflowKey is the record that counts events of kind k whose own key
// found no room.
func OverflowKey(k NoticeKind) NoticeKey { return NoticeKey{Kind: k} }
// FixedNoticeKeys are the records every ledger keeps.
func FixedNoticeKeys() []NoticeKey {
return []NoticeKey{
OverflowKey(NoticeWithheld), OverflowKey(NoticeWithheldWarning), OverflowKey(NoticeCapacity),
{Kind: NoticeCriticalSummary}, {Kind: NoticeAppliedSummary},
}
}
// PartialContainmentKey is the record of verified blocks of a destination
// that leave outbound-only traffic to it open.
func PartialContainmentKey(check string, effect Effect) NoticeKey {
return NoticeKey{Kind: NoticeWithheld, Reason: ReasonUnsupportedContainment, Outcome: DispositionApplied, Check: check, Effect: effect}
}
// Partial reports whether k is a partial-containment record.
func (k NoticeKey) Partial() bool {
return k.Kind == NoticeWithheld && k.Reason == ReasonUnsupportedContainment && k.Outcome == DispositionApplied
}
// Fixed reports whether the record under k is one every ledger keeps.
func (k NoticeKey) Fixed() bool { return k.Reason == 0 && k.Outcome == 0 }
func (k NoticeKey) validate() error {
bad := func(detail string) error { return refuse(ReasonInvalid, detail) }
switch {
case !k.Kind.Valid():
return bad("notice kind is unknown")
case k.Fixed():
if k.Check != "" || k.Effect != 0 {
return bad("a fixed notice record names a check or an effect")
}
return nil
case !k.Kind.Keyed():
return bad("a summary has no reason")
case k.Partial():
case k.Outcome != 0:
if k.Reason != 0 || k.Kind == NoticeCapacity || (k.Outcome != DispositionFailed && k.Outcome != DispositionUnknown) {
return bad("notice outcome is not a withheld response")
}
case !k.Reason.Valid() || k.Reason == ReasonExistingEffect || k.Reason == ReasonDisabled:
return bad("notice reason is not a response gap")
case k.Reason.Capacity() != (k.Kind == NoticeCapacity):
return bad("notice kind does not match its reason")
}
switch {
case k.Check != "" && !boundedToken(k.Check, 64):
return bad("notice check is malformed")
case k.Effect != 0 && !k.Effect.Valid():
return bad("notice effect is unknown")
}
return nil
}
// Bytes is the key's outbox key.
func (k NoticeKey) Bytes() ([]byte, error) {
if err := k.validate(); err != nil {
return nil, err
}
return append([]byte{noticeKind, byte(k.Kind), byte(k.Reason), byte(k.Outcome), byte(k.Effect)}, k.Check...), nil
}
// ParseNoticeKey decodes an outbox key written by Bytes.
func ParseNoticeKey(b []byte) (NoticeKey, error) {
if len(b) < 5 || len(b) > NoticeKeyMaxLen || b[0] != noticeKind {
return NoticeKey{}, ErrCorruptRecord
}
k := NoticeKey{Kind: NoticeKind(b[1]), Reason: Reason(b[2]), Outcome: Disposition(b[3]), Effect: Effect(b[4]), Check: string(b[5:])}
if k.validate() != nil {
return NoticeKey{}, ErrCorruptRecord
}
return k, nil
}
// NoticeExample is one event's candidate. Ordinal is the event's position
// in the record's count, so an acknowledgement drops exactly the examples
// it covered.
type NoticeExample struct {
Candidate CandidateID
Transitions uint32
Ordinal uint64
}
// NoticeRecord is one coalesced notice: a saturating count of events, the
// first and last event times, up to MaxNoticeExamples undelivered examples
// and the delivery state. Acked counts the events the last delivery
// covered; events after it stay pending.
type NoticeRecord struct {
Key NoticeKey
Count uint64
Acked uint64
First, Last time.Time
// Sent is when the last delivery was acknowledged; zero before one.
Sent time.Time
Examples []NoticeExample
}
// NewNoticeRecord is an empty record for k.
func NewNoticeRecord(k NoticeKey) NoticeRecord { return NoticeRecord{Key: k} }
// Unsent is the events no delivery has covered.
func (r NoticeRecord) Unsent() uint64 { return r.Count - r.Acked }
// Add records one event at time at. A zero candidate adds no example; so
// does a record whose examples are full or whose count has saturated.
func (r NoticeRecord) Add(at time.Time, candidate CandidateID, transitions uint32) (NoticeRecord, error) {
if _, err := r.record(true); err != nil {
return r, err
}
if _, ok := unixNano(at); !ok {
return r, refuse(ReasonInvalid, "notice event has no time")
}
if at.Before(r.Last) || at.Before(r.Sent) {
return r, refuse(ReasonInvalid, "notice event time moved backward")
}
if candidate != "" {
if _, err := ParseCandidateID(string(candidate)); err != nil || transitions == 0 {
return r, refuse(ReasonInvalid, "notice example is malformed")
}
}
if r.Count == 0 {
r.First = at
}
if at.After(r.Last) {
r.Last = at
}
if r.Count == math.MaxUint64 {
return r, nil
}
r.Count++
if candidate != "" && len(r.Examples) < MaxNoticeExamples {
r.Examples = append(append([]NoticeExample(nil), r.Examples...), NoticeExample{Candidate: candidate, Transitions: transitions, Ordinal: r.Count})
}
return r, nil
}
// AddCount records n events at time at without examples, as a checkpoint
// reports them. The count saturates.
func (r NoticeRecord) AddCount(at time.Time, n uint64) (NoticeRecord, error) {
if _, err := r.record(true); err != nil {
return r, err
}
if _, ok := unixNano(at); !ok {
return r, refuse(ReasonInvalid, "notice event has no time")
}
if at.Before(r.Last) || at.Before(r.Sent) {
return r, refuse(ReasonInvalid, "notice event time moved backward")
}
if n == 0 {
return r, nil
}
if r.Count == 0 {
r.First = at
}
if at.After(r.Last) {
r.Last = at
}
r.Count += min(n, math.MaxUint64-r.Count)
return r, nil
}
// Due reports whether the record has undelivered events and its kind's
// interval has passed since the last delivery.
func (r NoticeRecord) Due(now time.Time) bool {
return r.Count > r.Acked && (r.Sent.IsZero() || !now.Before(r.Sent.Add(r.Key.Kind.Interval())))
}
// Ack records a delivery that covered the first count events, at time at.
// A repeated or older acknowledgement changes nothing.
func (r NoticeRecord) Ack(count uint64, at time.Time) (NoticeRecord, error) {
if _, err := r.record(true); err != nil {
return r, err
}
if count > r.Count {
return r, refuse(ReasonInvalid, "acknowledgement covers events the record does not hold")
}
if count <= r.Acked {
return r, nil
}
if _, ok := unixNano(at); !ok {
return r, refuse(ReasonInvalid, "acknowledgement has no time")
}
if at.Before(r.Last) || at.Before(r.Sent) {
return r, refuse(ReasonInvalid, "notice acknowledgement time moved backward")
}
r.Acked, r.Sent = count, at
var kept []NoticeExample
for _, e := range r.Examples {
if e.Ordinal > count {
kept = append(kept, e)
}
}
r.Examples = kept
return r, nil
}
// QuietAt is when a keyed record whose events were all delivered becomes
// quiet; zero while an event is pending, before any delivery or for a
// fixed record.
func (r NoticeRecord) QuietAt() time.Time {
if r.Key.Fixed() || r.Count != r.Acked || r.Sent.IsZero() {
return time.Time{}
}
return r.Sent.Add(r.Key.Kind.Interval())
}
// quietKind is the outbox key kind of the quiet index.
const quietKind = 'q'
// QuietKey is the quiet index key of the record under k, quiet from at:
// the time as 19 decimal digits, so the index sorts by it, and the key.
func QuietKey(at time.Time, k NoticeKey) ([]byte, error) {
kb, err := k.Bytes()
if err != nil {
return nil, err
}
n, ok := unixNano(at)
if !ok || n <= 0 || k.Fixed() {
return nil, refuse(ReasonInvalid, "quiet key needs a keyed record and a time after the epoch")
}
return append(fmt.Appendf([]byte{quietKind}, "%019d", n), kb...), nil
}
// ParseQuietKey decodes a key written by QuietKey.
func ParseQuietKey(b []byte) (time.Time, NoticeKey, error) {
if len(b) < 20 || len(b) > NoticeQuietKeyMaxLen || b[0] != quietKind {
return time.Time{}, NoticeKey{}, ErrCorruptRecord
}
n, err := strconv.ParseInt(string(b[1:20]), 10, 64)
if err != nil || n <= 0 || fmt.Sprintf("%019d", n) != string(b[1:20]) {
return time.Time{}, NoticeKey{}, ErrCorruptRecord
}
k, err := ParseNoticeKey(b[20:])
if err != nil || k.Fixed() {
return time.Time{}, NoticeKey{}, ErrCorruptRecord
}
return time.Unix(0, n).UTC(), k, nil
}
// NoticeAck acknowledges Count events of the record read under Key.
// First fences a repeated acknowledgement from a later record under the
// same key: quiet removal requires the clock to advance past delivery.
type NoticeAck struct {
Key NoticeKey
First time.Time
Count uint64
}
// Quiet reports whether a keyed record may be removed: everything was
// delivered and its interval has passed since, so a new record for the
// key could not deliver sooner than this one would.
func (r NoticeRecord) Quiet(now time.Time) bool {
return !r.Key.Fixed() && r.Count == r.Acked && !r.Sent.IsZero() && !now.Before(r.Sent.Add(r.Key.Kind.Interval()))
}
const noticeRecordVersion = 2
type noticeExampleRecord struct {
Candidate CandidateID `json:"c"`
Transitions uint32 `json:"t"`
Ordinal uint64 `json:"o"`
}
type noticeRecordRecord struct {
V uint8 `json:"v"`
Kind NoticeKind `json:"kind"`
Reason Reason `json:"reason,omitempty"`
Outcome Disposition `json:"outcome,omitempty"`
// Base64 bounds printable check names without JSON escape expansion.
Check []byte `json:"check,omitempty"`
Effect Effect `json:"effect,omitempty"`
Count uint64 `json:"count,omitempty"`
Acked uint64 `json:"acked,omitempty"`
First int64 `json:"first,omitempty"`
Last int64 `json:"last,omitempty"`
Sent int64 `json:"sent,omitempty"`
Examples []noticeExampleRecord `json:"examples,omitempty"`
}
func (r NoticeRecord) record(allowEmpty bool) (noticeRecordRecord, error) {
bad := func(detail string) (noticeRecordRecord, error) {
return noticeRecordRecord{}, refuse(ReasonInvalid, detail)
}
if err := r.Key.validate(); err != nil {
return noticeRecordRecord{}, err
}
rec := noticeRecordRecord{V: noticeRecordVersion, Kind: r.Key.Kind, Reason: r.Key.Reason, Outcome: r.Key.Outcome, Check: []byte(r.Key.Check), Effect: r.Key.Effect, Count: r.Count, Acked: r.Acked}
if r.Count == 0 {
if !allowEmpty && !r.Key.Fixed() {
return bad("a keyed notice record has no event")
}
if r.Acked != 0 || !r.First.IsZero() || !r.Last.IsZero() || !r.Sent.IsZero() || len(r.Examples) != 0 {
return bad("an empty notice record has event or delivery state")
}
return rec, nil
}
var okFirst, okLast bool
rec.First, okFirst = unixNano(r.First)
rec.Last, okLast = unixNano(r.Last)
if !okFirst || !okLast || r.Last.Before(r.First) {
return bad("notice event times are inconsistent")
}
if r.Acked > r.Count || (r.Acked == 0) != r.Sent.IsZero() {
return bad("notice delivery state is inconsistent")
}
if r.Acked > 0 {
var ok bool
if rec.Sent, ok = unixNano(r.Sent); !ok || r.Sent.Before(r.First) {
return bad("notice delivery time is inconsistent")
}
}
if len(r.Examples) > MaxNoticeExamples {
return bad("notice record keeps too many examples")
}
last := r.Acked
for _, e := range r.Examples {
if _, err := ParseCandidateID(string(e.Candidate)); err != nil || e.Transitions == 0 || e.Ordinal <= last || e.Ordinal > r.Count {
return bad("notice examples are malformed, delivered or out of order")
}
last = e.Ordinal
rec.Examples = append(rec.Examples, noticeExampleRecord(e))
}
return rec, nil
}
func (r NoticeRecord) MarshalBinary() ([]byte, error) {
rec, err := r.record(false)
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalNoticeRecord decodes a stored record and checks its invariants.
func UnmarshalNoticeRecord(data []byte) (NoticeRecord, error) {
var rec noticeRecordRecord
if err := openRecord(data, &rec); err != nil {
return NoticeRecord{}, err
}
r := NoticeRecord{
Key: NoticeKey{Kind: rec.Kind, Reason: rec.Reason, Outcome: rec.Outcome, Check: string(rec.Check), Effect: rec.Effect},
Count: rec.Count, Acked: rec.Acked, First: fromNano(rec.First), Last: fromNano(rec.Last), Sent: fromNano(rec.Sent),
}
for _, e := range rec.Examples {
r.Examples = append(r.Examples, NoticeExample(e))
}
if again, err := r.MarshalBinary(); rec.V != noticeRecordVersion || err != nil || string(again) != string(data) {
return NoticeRecord{}, ErrCorruptRecord
}
return r, nil
}
package admission
// TemporaryCapacity is the finite temporary set capacity automatic
// admission works within when the configured temporary limit is zero,
// which leaves manual blocks unlimited. It is code policy, pending
// validation.
const TemporaryCapacity = 10000
// AutomaticCapacity is the temporary set capacity automatic admission
// works within under the configured temporary limit: the limit, or
// TemporaryCapacity for zero. Configuration loads a negative limit as
// zero; one set in code reads as zero too.
func AutomaticCapacity(limit int) int {
if limit <= 0 {
return TemporaryCapacity
}
return limit
}
// PartitionOf is the partition an entry admitted on lane occupies; zero
// for an invalid lane.
func PartitionOf(l Lane) Partition {
switch l {
case LaneGeneral:
return PartitionGeneral
case LaneDirect, LaneCorroborated:
return PartitionReserved
}
return 0
}
// SetOccupancy is the temporary set as automatic admission counts it: the
// configured temporary limit, the entries it can never evict (manual,
// unknown provenance and pending manual work), and the automatic entries
// of each partition.
type SetOccupancy struct {
Limit int
Unclaimed int
General, Reserved int
// NoEviction says no occupant may be replaced: the set offers its
// free slots only.
NoEviction bool
}
// SetSlots is how a SetOccupancy divides the set.
type SetSlots struct {
// Capacity is AutomaticCapacity of the limit; Available is what the
// unclaimed entries leave of it, never below zero.
Capacity, Available int
// General and Reserved size the partitions: ceil(Available/5) is
// reserved and the rest general, as the ceiling splits its lanes.
General, Reserved int
// FreeGeneral and FreeReserved are each partition's empty slots,
// bounded by shared headroom. An overfull partition has none.
FreeGeneral, FreeReserved int
}
// Slots divides o's set into its partitions and their empty slots.
func (o SetOccupancy) Slots() SetSlots {
s := SetSlots{Capacity: AutomaticCapacity(o.Limit)}
s.Available = max(s.Capacity-o.Unclaimed, 0)
s.Reserved = s.Available / 5
if s.Available%5 != 0 {
s.Reserved++
}
s.General = s.Available - s.Reserved
s.FreeGeneral = max(s.General-o.General, 0)
s.FreeReserved = max(s.Reserved-o.Reserved, 0)
// Existing occupants keep their partition after a limit or manual change.
// A partition's target room is also bounded by the shared physical room.
free := max(s.Available-o.General-o.Reserved, 0)
s.FreeGeneral = min(s.FreeGeneral, free)
s.FreeReserved = min(s.FreeReserved, free)
return s
}
// Free is the empty slots a pick on lane may take: those of the partition
// it would occupy.
func (s SetSlots) Free(l Lane) int {
switch PartitionOf(l) {
case PartitionGeneral:
return s.FreeGeneral
case PartitionReserved:
return s.FreeReserved
}
return 0
}
package admission
import (
"fmt"
"sort"
"strconv"
"time"
)
// OutcomeKey is one fixed dimension of the windowed outcome counts: a
// queue event with its reason, or an attempt outcome,
// by the candidate's last assessment. It never names an address, account
// or check.
type OutcomeKey struct {
Event QueueEvent
Reason Reason
Outcome Disposition
Class Class
Severity Severity
}
// QueueOutcome is the key of a queue event.
func QueueOutcome(e QueueEvent, r Reason, t Tier) OutcomeKey {
return OutcomeKey{Event: e, Reason: r, Class: t.Class, Severity: t.Severity}
}
// AttemptOutcome is the key of an attempt outcome.
func AttemptOutcome(d Disposition, t Tier) OutcomeKey {
return OutcomeKey{Outcome: d, Class: t.Class, Severity: t.Severity}
}
// Valid reports whether k is a queue event with a reason, or an attempt
// outcome, with a whole tier or none.
func (k OutcomeKey) Valid() bool {
unassessed := k.Class == 0 && k.Severity == 0
if !unassessed && (!k.Class.Valid() || !k.Severity.Valid()) {
return false
}
if k.Event != 0 {
return k.Event.Valid() && k.Reason.Valid() && k.Outcome == 0
}
switch k.Outcome {
case DispositionApplied, DispositionNarrowed, DispositionFailed, DispositionUnknown, DispositionObserve, DispositionDryRun:
return k.Reason == 0
}
return false
}
func (k OutcomeKey) less(o OutcomeKey) bool {
switch {
case k.Event != o.Event:
return k.Event < o.Event
case k.Reason != o.Reason:
return k.Reason < o.Reason
case k.Outcome != o.Outcome:
return k.Outcome < o.Outcome
case k.Class != o.Class:
return k.Class < o.Class
}
return k.Severity < o.Severity
}
// MaxOutcomeKeys is the size of the key space: every queue event with
// every reason and the six attempt and preview outcomes, unassessed or at
// each tier.
const MaxOutcomeKeys = (int(queueEventEnd-EventRefused)*int(reasonEnd-ReasonCeiling) + 6) * (1 + 3*3)
// MaxOutcomeBucketBytes bounds one encoded bucket: every key at its
// largest count.
const MaxOutcomeBucketBytes = 33 << 10
// OutcomeCounts are saturating counts by OutcomeKey. The zero value is
// empty and ready to use.
type OutcomeCounts struct {
counts map[OutcomeKey]uint64
}
// OutcomeRow is one count.
type OutcomeRow struct {
Key OutcomeKey
N uint64
}
// Add counts one event.
func (c *OutcomeCounts) Add(k OutcomeKey) error {
if !k.Valid() {
return refuse(ReasonInvalid, "outcome key is invalid")
}
if c.counts == nil {
c.counts = map[OutcomeKey]uint64{}
}
if c.counts[k] != ^uint64(0) {
c.counts[k]++
}
return nil
}
// AddCount counts n events, saturating.
func (c *OutcomeCounts) AddCount(k OutcomeKey, n uint64) error {
if !k.Valid() {
return refuse(ReasonInvalid, "outcome key is invalid")
}
if n == 0 {
return nil
}
if c.counts == nil {
c.counts = map[OutcomeKey]uint64{}
}
c.counts[k] += min(n, ^uint64(0)-c.counts[k])
return nil
}
// Count returns the count for k.
func (c OutcomeCounts) Count(k OutcomeKey) uint64 { return c.counts[k] }
// Merge adds o's counts, saturating.
func (c *OutcomeCounts) Merge(o OutcomeCounts) {
for k, n := range o.counts {
if c.counts == nil {
c.counts = map[OutcomeKey]uint64{}
}
c.counts[k] += min(n, ^uint64(0)-c.counts[k])
}
}
// Rows are the counts in key order.
func (c OutcomeCounts) Rows() []OutcomeRow {
rows := make([]OutcomeRow, 0, len(c.counts))
for k, n := range c.counts {
rows = append(rows, OutcomeRow{k, n})
}
sort.Slice(rows, func(i, j int) bool { return rows[i].Key.less(rows[j].Key) })
return rows
}
const outcomeCountsVersion = 1
type outcomeRowRecord struct {
Event QueueEvent `json:"e,omitempty"`
Reason Reason `json:"r,omitempty"`
Outcome Disposition `json:"o,omitempty"`
Class Class `json:"c,omitempty"`
Severity Severity `json:"s,omitempty"`
N uint64 `json:"n"`
}
type outcomeCountsRecord struct {
V uint8 `json:"v"`
Rows []outcomeRowRecord `json:"rows"`
}
func (c OutcomeCounts) MarshalBinary() ([]byte, error) {
rows := c.Rows()
rec := outcomeCountsRecord{V: outcomeCountsVersion, Rows: make([]outcomeRowRecord, 0, len(rows))}
for _, r := range rows {
if !r.Key.Valid() || r.N == 0 {
return nil, refuse(ReasonInvalid, "outcome count is invalid")
}
k := r.Key
rec.Rows = append(rec.Rows, outcomeRowRecord{Event: k.Event, Reason: k.Reason, Outcome: k.Outcome, Class: k.Class, Severity: k.Severity, N: r.N})
}
return sealRecord(rec)
}
// UnmarshalOutcomeCounts decodes a stored bucket.
func UnmarshalOutcomeCounts(data []byte) (OutcomeCounts, error) {
var rec outcomeCountsRecord
if err := openRecord(data, &rec); err != nil {
return OutcomeCounts{}, err
}
if rec.V != outcomeCountsVersion || rec.Rows == nil {
return OutcomeCounts{}, ErrCorruptRecord
}
c := OutcomeCounts{counts: make(map[OutcomeKey]uint64, len(rec.Rows))}
var prev OutcomeKey
for i, r := range rec.Rows {
k := OutcomeKey{Event: r.Event, Reason: r.Reason, Outcome: r.Outcome, Class: r.Class, Severity: r.Severity}
if !k.Valid() || r.N == 0 || (i > 0 && !prev.less(k)) {
return OutcomeCounts{}, ErrCorruptRecord
}
c.counts[k], prev = r.N, k
}
return c, nil
}
// Span is one resolution of the outcome windows.
type Span uint8
const (
// SpanFiveMinutes: twelve buckets, the last hour.
SpanFiveMinutes Span = iota + 1
// SpanHour: 24 buckets, the last day.
SpanHour
// SpanDay: 30 buckets, the last 30 days.
SpanDay
spanEnd
)
// Spans lists every resolution.
func Spans() []Span { return []Span{SpanFiveMinutes, SpanHour, SpanDay} }
func (s Span) Valid() bool { return s >= SpanFiveMinutes && s < spanEnd }
func (s Span) String() string {
switch s {
case SpanFiveMinutes:
return "5m"
case SpanHour:
return "1h"
case SpanDay:
return "1d"
}
return fmt.Sprintf("span(%d)", uint8(s))
}
// Width is one bucket's length.
func (s Span) Width() time.Duration {
switch s {
case SpanFiveMinutes:
return 5 * time.Minute
case SpanHour:
return time.Hour
}
return 24 * time.Hour
}
// Buckets is how many buckets the span keeps.
func (s Span) Buckets() int {
switch s {
case SpanFiveMinutes:
return 12
case SpanHour:
return 24
}
return 30
}
// Start is the start of the bucket holding at, aligned to the Unix epoch.
func (s Span) Start(at time.Time) time.Time { return at.UTC().Truncate(s.Width()) }
// Oldest is the start of the oldest bucket kept at time now.
func (s Span) Oldest(now time.Time) time.Time {
return s.Start(now).Add(-time.Duration(s.Buckets()-1) * s.Width())
}
// windowKind is the key kind of outcome buckets.
const windowKind = 'w'
// Key is the stored key of the bucket starting at start: the span and the
// start as 19 decimal digits of nanoseconds, so a span's buckets sort by
// time, as history retirement keys do.
func (s Span) Key(start time.Time) []byte {
return fmt.Appendf([]byte{windowKind, byte(s)}, "%019d", start.UnixNano())
}
// ParseSpanKey decodes a bucket key and checks its alignment.
func ParseSpanKey(k []byte) (Span, time.Time, error) {
if len(k) != 21 || k[0] != windowKind || !Span(k[1]).Valid() {
return 0, time.Time{}, ErrCorruptRecord
}
s := Span(k[1])
n, err := strconv.ParseInt(string(k[2:]), 10, 64)
if err != nil || n <= 0 || fmt.Sprintf("%019d", n) != string(k[2:]) {
return 0, time.Time{}, ErrCorruptRecord
}
start := time.Unix(0, n).UTC()
if !s.Start(start).Equal(start) {
return 0, time.Time{}, ErrCorruptRecord
}
return s, start, nil
}
package admission
import (
"bytes"
"crypto/sha256"
"encoding/json"
"errors"
"math"
"sort"
"strconv"
"strings"
)
// Owner is a verified victim identity: one hosting account at one inventory
// generation, or the shared host owner. Only an Inventory produces an
// account owner; the zero value is the host owner.
type Owner struct {
account string
generation uint64
}
// HostOwner is the owner of unknown, multi-account and host-level evidence.
func HostOwner() Owner { return Owner{} }
func (o Owner) IsHost() bool { return o.account == "" }
func (o Owner) Account() string { return o.account }
func (o Owner) Generation() uint64 { return o.generation }
// Key is a stable text form: "host" or "acct:<name>#<generation>".
func (o Owner) Key() string {
if o.IsHost() {
return "host"
}
return "acct:" + o.account + "#" + strconv.FormatUint(o.generation, 10)
}
// Scope is the fairness unit: a verified owner and an action family.
type Scope struct {
Owner Owner
Effect Effect
}
func (s Scope) Key() string { return s.Owner.Key() + "/" + s.Effect.String() }
// ClaimKind says where an ownership claim came from. Only server-owned
// context can verify an owner.
type ClaimKind uint8
const (
// ClaimAccount names a hosting account from server-owned identity: a
// process or file owner, or the account root a path lies under.
ClaimAccount ClaimKind = iota + 1
// ClaimDomain names the domain of the vhost or listener that served
// the request, taken from server configuration or a per-vhost log.
ClaimDomain
// ClaimMailbox names a mailbox from a protocol exchange. The client
// chose it, so it never verifies on its own.
ClaimMailbox
// ClaimRequestName names a host from request content such as a Host
// header. It never verifies.
ClaimRequestName
)
func (k ClaimKind) Valid() bool { return k >= ClaimAccount && k <= ClaimRequestName }
// Claim is an unverified statement of which account a piece of evidence
// concerns.
type Claim struct {
Kind ClaimKind `json:"kind"`
Value string `json:"value"`
}
// Inventory is an immutable snapshot of server-owned hosting accounts and
// the domains they serve.
type Inventory struct {
accounts map[string]uint64
domains map[string]string
}
// ValidAccountName is the account-name alphabet the scan control protocol
// accepts, bounded to 64 bytes.
func ValidAccountName(name string) bool {
if name == "" || name == "." || name == ".." || len(name) > 64 {
return false
}
for i := 0; i < len(name); i++ {
c := name[i]
if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') && c != '_' && c != '-' && c != '.' {
return false
}
}
return true
}
func canonicalDomain(d string) string {
return strings.TrimSuffix(strings.ToLower(d), ".")
}
// NewInventory copies accounts (name to generation) and domains (domain to
// owning account). Every generation is nonzero and every domain owner is a
// listed account.
func NewInventory(accounts map[string]uint64, domains map[string]string) (*Inventory, error) {
inv := &Inventory{accounts: make(map[string]uint64, len(accounts)), domains: make(map[string]string, len(domains))}
for name, gen := range accounts {
if !ValidAccountName(name) || gen == 0 {
return nil, errors.New("inventory account entry has an invalid name or zero generation")
}
inv.accounts[name] = gen
}
for domain, owner := range domains {
d := canonicalDomain(domain)
if d == "" || d != domain {
return nil, errors.New("inventory domain is empty or not canonical")
}
if _, ok := inv.accounts[owner]; !ok {
return nil, errors.New("inventory domain names an unlisted account")
}
inv.domains[d] = owner
}
return inv, nil
}
func (inv *Inventory) owner(account string) (Owner, bool) {
gen, ok := inv.accounts[account]
if !ok {
return Owner{}, false
}
return Owner{account: account, generation: gen}, true
}
// Resolve verifies claims against the inventory. Claims that cannot be
// verified are ignored. Exactly one verified account yields that account;
// none, or several different accounts, yields the host owner.
func (inv *Inventory) Resolve(claims ...Claim) Owner {
var found Owner
for _, c := range claims {
var o Owner
var ok bool
switch c.Kind {
case ClaimAccount:
o, ok = inv.owner(c.Value)
case ClaimDomain:
if account, hosted := inv.domains[canonicalDomain(c.Value)]; hosted {
o, ok = inv.owner(account)
}
}
if !ok {
continue
}
if !found.IsHost() && found != o {
return HostOwner()
}
found = o
}
return found
}
// Current reports whether o still names a present account at the same
// generation. The host owner is always current. Admission re-resolves
// owners with it before application.
func (inv *Inventory) Current(o Owner) bool {
if o.IsHost() {
return true
}
gen, ok := inv.accounts[o.account]
return ok && gen == o.generation
}
// Generations assigns inventory generations. A name keeps its generation
// while it appears in consecutive complete observations with the same
// server-owned incarnation token; a name that disappears and returns, or
// returns with another token, gets a new one. Recreation between observations
// is detected only when it changes the token; cPanel's zero-dated system
// entries keep theirs. A name observed without a token keeps its
// generation, and a stored name without one adopts the next token it is
// seen with. Generations are never reused. Initialize with NewGenerations
// or a successful UnmarshalBinary before observing or encoding; the zero
// value cannot allocate identities. It is not safe for concurrent use:
// callers must serialize every call.
type Generations struct {
next uint64
live map[string]uint64
incarnations map[string]string
}
func NewGenerations() *Generations {
return &Generations{next: 1, live: map[string]uint64{}, incarnations: map[string]string{}}
}
// maxIncarnationLen bounds one incarnation token.
const maxIncarnationLen = 64
func validIncarnation(token string) bool {
if token == "" || len(token) > maxIncarnationLen {
return false
}
for i := 0; i < len(token); i++ {
if token[i] < 0x21 || token[i] > 0x7e {
return false
}
}
return true
}
// Observe records one complete inventory observation and returns the
// current name-to-generation map. incarnations maps listed names to their
// incarnation tokens; a name may have none. Callers must not pass a partial
// observation: a transient read failure would retire every missing account.
func (g *Generations) Observe(names []string, incarnations map[string]string) (map[string]uint64, error) {
if g.next == 0 {
return nil, errors.New("inventory generation tracker is not initialized")
}
present := make(map[string]bool, len(names))
for _, name := range names {
if !ValidAccountName(name) {
return nil, errors.New("inventory observation has an invalid account name")
}
present[name] = true
}
for name, token := range incarnations {
if !present[name] || !validIncarnation(token) {
return nil, errors.New("inventory observation has an invalid incarnation")
}
}
replaced := func(name string) bool {
stored, observed := g.incarnations[name], incarnations[name]
return stored != "" && observed != "" && stored != observed
}
newCount := uint64(0)
for name := range present {
if _, exists := g.live[name]; !exists || replaced(name) {
newCount++
}
}
if newCount > math.MaxUint64-g.next {
return nil, errors.New("inventory generation counter is exhausted")
}
for name := range g.live {
if !present[name] || replaced(name) {
delete(g.live, name)
delete(g.incarnations, name)
}
}
sorted := make([]string, 0, len(present))
for name := range present {
sorted = append(sorted, name)
}
sort.Strings(sorted)
for _, name := range sorted {
if _, ok := g.live[name]; !ok {
g.live[name] = g.next
g.next++
}
if token := incarnations[name]; token != "" {
g.incarnations[name] = token
}
}
out := make(map[string]uint64, len(g.live))
for name, gen := range g.live {
out[name] = gen
}
return out, nil
}
type generationsRecord struct {
V int `json:"v"`
Next uint64 `json:"next"`
Live map[string]uint64 `json:"live"`
Incarnations map[string]string `json:"incarnations,omitempty"`
}
// A tracker without incarnation tokens keeps the version 1 encoding; one
// with them is version 2.
const (
generationsVersion = 1
generationsIncarnationsVersion = 2
)
// MarshalBinary encodes the tracker as versioned JSON followed by an 8-byte
// SHA-256 prefix of that JSON.
func (g *Generations) MarshalBinary() ([]byte, error) {
if g.next == 0 {
return nil, errors.New("inventory generation tracker is not initialized")
}
rec := generationsRecord{V: generationsVersion, Next: g.next, Live: g.live}
if len(g.incarnations) > 0 {
rec.V, rec.Incarnations = generationsIncarnationsVersion, g.incarnations
}
body, err := json.Marshal(rec)
if err != nil {
return nil, err
}
sum := sha256.Sum256(body)
return append(body, sum[:8]...), nil
}
// UnmarshalBinary restores a tracker. Corrupt, unknown-version or
// inconsistent state is an error, never an empty tracker: an empty tracker
// would hand every account a fresh generation.
func (g *Generations) UnmarshalBinary(data []byte) error {
if len(data) < 8 {
return errors.New("generations record is truncated")
}
body, sum := data[:len(data)-8], data[len(data)-8:]
if want := sha256.Sum256(body); !bytes.Equal(sum, want[:8]) {
return errors.New("generations record checksum mismatch")
}
dec := json.NewDecoder(bytes.NewReader(body))
dec.DisallowUnknownFields()
var rec generationsRecord
if err := dec.Decode(&rec); err != nil {
return errors.New("generations record does not decode")
}
canonical, err := json.Marshal(rec)
if err != nil || !bytes.Equal(canonical, body) {
return errors.New("generations record is not in canonical form")
}
if (rec.V != generationsVersion && rec.V != generationsIncarnationsVersion) || (rec.V == generationsIncarnationsVersion) != (len(rec.Incarnations) > 0) {
return errors.New("generations record version is not supported")
}
if rec.Next == 0 {
return errors.New("generations record has no next generation")
}
if rec.Live == nil {
return errors.New("generations record has no inventory")
}
used := make(map[uint64]bool, len(rec.Live))
for name, gen := range rec.Live {
if !ValidAccountName(name) || gen == 0 || gen >= rec.Next || used[gen] {
return errors.New("generations record has an invalid entry")
}
used[gen] = true
}
for name, token := range rec.Incarnations {
if _, ok := rec.Live[name]; !ok || !validIncarnation(token) {
return errors.New("generations record has an invalid incarnation")
}
}
if rec.Incarnations == nil {
rec.Incarnations = map[string]string{}
}
g.next, g.live, g.incarnations = rec.Next, rec.Live, rec.Incarnations
return nil
}
package admission
import "fmt"
// Prediction is what a dry-run forecast says live admission would have done
// with a pick. It is not a terminal reason: every forecast pick
// ends as a dry run, and the prediction only counts.
type Prediction uint8
const (
// PredictCapacityAdmit: the pick passes the modelled ceiling and
// temporary-capacity checks. It is not proof a block would apply.
PredictCapacityAdmit Prediction = iota + 1
// PredictExistingActual: a real proven block already covers it.
PredictExistingActual
// PredictExistingShadow: this session already forecast a covering block.
PredictExistingShadow
// PredictCeiling: it would wait for modelled usage or paced credit.
PredictCeiling
// PredictSetFull: it would wait for real-plus-forecast temporary room.
PredictSetFull
// PredictPendingRecovery: a shorter forecast block holds the address.
PredictPendingRecovery
// PredictForecastLimited: the forecast models no more active blocks.
PredictForecastLimited
// PredictNotModelled: the kind has no capacity model.
PredictNotModelled
// PredictCapacityEvict: the pick passes the modelled ceiling and takes
// the temporary slot of a strictly lower-tier automatic block, which it
// would replace.
PredictCapacityEvict
// PredictCapacityRotate: the pick passes the modelled ceiling and
// rotates an equal-tier forecast block whose hold passed out of its
// full partition, as fresh work may.
PredictCapacityRotate
// PredictEqualTierWait: it would wait as set_full while its partition
// holds an equal-tier block that rotation does not give it yet: the
// block's hold, the shares or an answered episode stop it.
PredictEqualTierWait
predictionEnd
)
// PredictionSlots sizes an array indexed by Prediction; slot zero is unused.
const PredictionSlots = int(predictionEnd)
var predictionNames = [...]string{"", "capacity_admit", "existing_actual", "existing_shadow", "ceiling", "set_full", "pending_recovery", "forecast_limited", "not_modelled", "capacity_evict", "capacity_rotate", "equal_tier_wait"}
func (p Prediction) Valid() bool { return p >= PredictCapacityAdmit && p < predictionEnd }
func (p Prediction) String() string {
if p.Valid() {
return predictionNames[p]
}
return fmt.Sprintf("prediction(%d)", uint8(p))
}
package admission
import (
"fmt"
"slices"
"sort"
"sync"
)
// Policy is the admission projection of a check's response policy.
type Policy struct {
Family Family
Basis Basis
// MinSeverity is the lowest finding severity that counts as evidence for
// the check; zero accepts every severity. It carries a Critical-only
// rule, so an advisory finding of such a check never becomes a root.
MinSeverity Severity
// Attribution is the set of kinds the check accepts its address from.
Attribution AttributionSet
}
// PolicyLookup resolves a check name, including a renamed producer's old
// name, to its canonical registered name and policy. ok is false for an
// unregistered check. Registration, minting and validation share this lookup;
// checks.AdmissionPolicy projects the production check registry. Calls from
// separate operations can overlap, so the lookup must be immutable or safe
// for concurrent use.
type PolicyLookup func(check string) (canonical string, p Policy, ok bool)
// ProducerID names a registered evidence producer.
type ProducerID string
// ValidProducerID accepts 1-48 bytes of lowercase letters, digits and
// underscores, starting with a letter.
func ValidProducerID(id ProducerID) bool {
if len(id) == 0 || len(id) > 48 || id[0] < 'a' || id[0] > 'z' {
return false
}
for i := 0; i < len(id); i++ {
c := id[i]
if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '_' {
return false
}
}
return true
}
// ObservationKind is how a producer identifies an observation.
type ObservationKind uint8
const (
// ObservationLogCursor: a position in a log file or journal.
ObservationLogCursor ObservationKind = iota + 1
// ObservationEventSeq: a sequence number of kernel, socket or IPC
// events.
ObservationEventSeq
// ObservationScanPass: one pass of a scan over server-owned state.
ObservationScanPass
observationEnd
)
func (k ObservationKind) Valid() bool { return k >= ObservationLogCursor && k < observationEnd }
// maxProducerChecks bounds one producer's check list.
const maxProducerChecks = 32
// ProducerSpec is what a producer registers. Checks are exact names; a
// dynamic prefix is never a catch-all. Claims are the ownership claim kinds
// the producer's findings may carry; Mint refuses any other. Attribution is
// the set of kinds its parsers read addresses by; a derived entry, which
// mints nothing, declares none.
type ProducerSpec struct {
ID ProducerID
Entry Entry
Observation ObservationKind
Checks []string
Claims []ClaimKind
Attribution AttributionSet
}
// Registry holds the producers allowed to mint evidence. Register all
// producers, then Seal; a sealed registry accepts none.
type Registry struct {
mu sync.Mutex
lookup PolicyLookup
producers map[ProducerID]ProducerSpec
sealed bool
}
func NewRegistry(lookup PolicyLookup) (*Registry, error) {
if lookup == nil {
return nil, fmt.Errorf("registry needs a policy lookup")
}
return &Registry{lookup: lookup, producers: map[ProducerID]ProducerSpec{}}, nil
}
// Producer is the capability to mint evidence as one registered producer.
// Only Register creates one.
type Producer struct {
reg *Registry
id ProducerID
}
func (p *Producer) ID() ProducerID { return p.id }
// Register validates spec and returns the producer's minting capability.
// Every check must be registered with an evidence family.
func (r *Registry) Register(spec ProducerSpec) (*Producer, error) {
r.mu.Lock()
defer r.mu.Unlock()
switch {
case r.sealed:
return nil, fmt.Errorf("producer registered after the registry was sealed")
case !ValidProducerID(spec.ID):
return nil, fmt.Errorf("invalid producer ID")
case !spec.Entry.Valid():
return nil, fmt.Errorf("producer has an unknown entry")
case !spec.Observation.Valid():
return nil, fmt.Errorf("producer has an unknown observation kind")
case len(spec.Checks) == 0 || len(spec.Checks) > maxProducerChecks:
return nil, fmt.Errorf("producer must publish 1-%d checks", maxProducerChecks)
}
if _, dup := r.producers[spec.ID]; dup {
return nil, fmt.Errorf("producer is already registered")
}
checks := make([]string, 0, len(spec.Checks))
seen := map[string]bool{}
for _, name := range spec.Checks {
canonical, p, ok := r.lookup(name)
if !ok || canonical != name {
return nil, fmt.Errorf("producer check is not a canonical registered check")
}
if p.Family == FamilyNone {
return nil, fmt.Errorf("producer check carries no admissible address evidence")
}
if err := ValidPolicy(p.Family, p.Basis); err != nil || (p.MinSeverity != 0 && !p.MinSeverity.Valid()) ||
p.Attribution == 0 || !p.Attribution.Valid() || !p.Attribution.Within(p.Family.Attributions()) {
return nil, fmt.Errorf("producer check has an invalid evidence policy")
}
if !spec.Attribution.Within(p.Attribution) {
return nil, fmt.Errorf("producer declares an attribution kind its check does not accept")
}
if seen[name] {
return nil, fmt.Errorf("producer check is repeated")
}
seen[name] = true
checks = append(checks, name)
}
sort.Strings(checks)
spec.Checks = checks
claims := make([]ClaimKind, 0, len(spec.Claims))
for _, kind := range spec.Claims {
if !kind.Valid() {
return nil, fmt.Errorf("producer declares an unknown claim kind")
}
if slices.Contains(claims, kind) {
return nil, fmt.Errorf("producer claim kind is repeated")
}
claims = append(claims, kind)
}
slices.Sort(claims)
spec.Claims = claims
r.producers[spec.ID] = spec
return &Producer{reg: r, id: spec.ID}, nil
}
// Seal stops further registration.
func (r *Registry) Seal() {
r.mu.Lock()
r.sealed = true
r.mu.Unlock()
}
// Sealed reports whether registration has ended. The ledger accepts only a
// sealed registry, so the set of producers cannot change under it.
func (r *Registry) Sealed() bool {
r.mu.Lock()
defer r.mu.Unlock()
return r.sealed
}
// Spec returns a copy of a registered producer's spec.
func (r *Registry) Spec(id ProducerID) (ProducerSpec, bool) {
r.mu.Lock()
defer r.mu.Unlock()
spec, ok := r.producers[id]
if ok {
spec.Checks = append([]string(nil), spec.Checks...)
spec.Claims = append([]ClaimKind(nil), spec.Claims...)
}
return spec, ok
}
func (spec ProducerSpec) publishes(check string) bool {
i := sort.SearchStrings(spec.Checks, check)
return i < len(spec.Checks) && spec.Checks[i] == check
}
// ValidateVia refuses a derived entry that no registered producer of that
// entry wraps e's check under: the engine binds every entry to its
// producers.
func (r *Registry) ValidateVia(via Entry, e Evidence) error {
if !via.Valid() {
return refuse(ReasonPolicy, "response entry is not valid")
}
r.mu.Lock()
defer r.mu.Unlock()
for _, spec := range r.producers {
if spec.Entry == via && spec.publishes(e.rec.Check) {
return nil
}
}
return refuse(ReasonPolicy, "response entry is not registered for the evidence check")
}
// Validate refuses evidence whose producer, entry, check or policy no
// longer matches the registry, including a severity below the check's
// current floor. Admission calls it on every use, so a policy change takes
// effect on evidence minted before it.
func (r *Registry) Validate(e Evidence) error {
spec, ok := r.Spec(e.rec.Producer)
if !ok {
return refuse(ReasonPolicy, "evidence producer is not registered")
}
if spec.Entry != e.rec.Entry || !spec.publishes(e.rec.Check) {
return refuse(ReasonPolicy, "evidence entry or check is not registered for its producer")
}
canonical, p, ok := r.lookup(e.rec.Check)
if !ok || canonical != e.rec.Check || p.Family != e.rec.Family || p.Basis != e.rec.Basis {
return refuse(ReasonPolicy, "check policy changed since the evidence was minted")
}
if e.rec.Severity < p.MinSeverity {
return refuse(ReasonPolicy, "finding severity is below the check's evidence floor")
}
// The record keeps no kind: its producer must still declare one, and
// the check must still accept every kind the producer declares.
if spec.Attribution == 0 || !spec.Attribution.Within(p.Attribution) {
return refuseAttribution(CauseKindNotAllowed, "the producer or check no longer accepts the evidence's attribution")
}
return nil
}
package admission
import (
"fmt"
"sort"
"time"
)
// Queue bounds. Ingress, queued and in-flight candidates share
// them. Every candidate record is at most MaxCandidateBytes, so the 4 MiB
// payload bound cannot bind before the count does.
const (
// QueueCapacity bounds ingress, queued and in-flight candidates together.
QueueCapacity = 1000
// ReservedPositions of the capacity are held for candidates that may use
// the reserved lane: direct compromise or independent corroboration.
ReservedPositions = 200
// IngressPositions reserves transfer space within each partition.
IngressPositions = 64
)
// Partition is the part of the queue a candidate holds its position in,
// and the part of the temporary set an automatic entry occupies. Values
// are persisted: append, never renumber.
type Partition uint8
const (
PartitionGeneral Partition = iota + 1
// PartitionReserved takes only candidates eligible for the reserved
// lane. General work never displaces it.
PartitionReserved
partitionEnd
)
var partitionNames = [...]string{"", "general", "reserved"}
func (p Partition) Valid() bool { return p >= PartitionGeneral && p < partitionEnd }
func (p Partition) String() string {
if p.Valid() {
return partitionNames[p]
}
return fmt.Sprintf("partition(%d)", uint8(p))
}
// Capacity is how many positions p holds.
func (p Partition) Capacity() int {
if p == PartitionReserved {
return ReservedPositions
}
return QueueCapacity - ReservedPositions
}
// DurableCapacity leaves room for an ingress group even when durable work
// fills the partition. In-flight candidates retain durable positions.
func (p Partition) DurableCapacity() int { return p.Capacity() - IngressPositions }
// QueueItem is one queue position as admission sees it.
type QueueItem struct {
// Key names the item: a candidate ID, or an ingress sequence.
Key string
// Scope is the fairness scope key.
Scope string
Partition Partition
Tier Tier
// Eligible: the candidate may hold a reserved position.
Eligible bool
// Queued, then Seq, then Key order items from oldest to newest.
Queued time.Time
// Seq orders items queued at the same instant. Stored items carry
// zero; arrivals being decided and ingress items take increasing
// values, so an arrival is newer than all work already queued.
Seq uint64
// Fixed items hold their position but are never displaced: they are in
// flight or not yet assessed.
Fixed bool
}
// newer reports whether a joined the queue after b.
func (a QueueItem) newer(b QueueItem) bool {
if !a.Queued.Equal(b.Queued) {
return a.Queued.After(b.Queued)
}
if a.Seq != b.Seq {
return a.Seq > b.Seq
}
return a.Key > b.Key
}
// goesFirst reports whether a scope that must lose an entry loses a before
// b: the lowest tier first, then the newest.
func goesFirst(a, b QueueItem) bool {
if a.Tier != b.Tier {
return a.Tier.Less(b.Tier)
}
return a.newer(b)
}
// QueueCursors are the persisted remainder cursors of the two partitions.
// Each is a scope key; the scopes after it take the remainder of an uneven
// share, so admission opportunities rotate when scopes outnumber positions.
type QueueCursors struct {
General string
Reserved string
}
type scopeQueue struct {
count int
// victims are the displaceable items, next victim first.
victims []QueueItem
}
type partitionQueue struct {
count int
cursor string
scopes map[string]*scopeQueue
}
// QueueView is the occupancy of the queue by partition and scope. Admit
// applies the queue's overflow rules to it. It is not safe for
// concurrent use.
type QueueView struct {
limits [partitionEnd]int
parts [partitionEnd]partitionQueue
items map[string]QueueItem
}
// NewQueueView returns an empty view with the given cursors.
func NewQueueView(c QueueCursors) *QueueView {
v := &QueueView{items: map[string]QueueItem{}}
for p := PartitionGeneral; p < partitionEnd; p++ {
v.parts[p].scopes = map[string]*scopeQueue{}
v.limits[p] = p.Capacity()
}
v.parts[PartitionGeneral].cursor, v.parts[PartitionReserved].cursor = c.General, c.Reserved
return v
}
// NewDurableQueueView keeps the transfer allocation unavailable to stored work.
func NewDurableQueueView(c QueueCursors) *QueueView {
v := NewQueueView(c)
for p := PartitionGeneral; p < partitionEnd; p++ {
v.limits[p] = p.DurableCapacity()
}
return v
}
// Cursors returns the current remainder cursors.
func (v *QueueView) Cursors() QueueCursors {
return QueueCursors{General: v.parts[PartitionGeneral].cursor, Reserved: v.parts[PartitionReserved].cursor}
}
// Len is the number of positions held.
func (v *QueueView) Len() int { return len(v.items) }
// Count is the number of positions held in p.
func (v *QueueView) Count(p Partition) int { return v.parts[p].count }
// Room reports whether p has a free position.
func (v *QueueView) Room(p Partition) bool { return v.parts[p].count < v.limits[p] }
// Item returns the item with key k.
func (v *QueueView) Item(k string) (QueueItem, bool) {
it, ok := v.items[k]
return it, ok
}
// Add records an item in its partition without applying admission rules:
// it already holds its position.
func (v *QueueView) Add(it QueueItem) error {
if !it.Partition.Valid() || it.Key == "" || it.Scope == "" {
return fmt.Errorf("queue item has no partition, key or scope")
}
if _, dup := v.items[it.Key]; dup {
return fmt.Errorf("queue item is already present")
}
pq := &v.parts[it.Partition]
sq := pq.scopes[it.Scope]
if sq == nil {
sq = &scopeQueue{}
pq.scopes[it.Scope] = sq
}
sq.count++
pq.count++
if !it.Fixed {
i := sort.Search(len(sq.victims), func(i int) bool { return goesFirst(it, sq.victims[i]) })
sq.victims = append(sq.victims, QueueItem{})
copy(sq.victims[i+1:], sq.victims[i:])
sq.victims[i] = it
}
v.items[it.Key] = it
return nil
}
// Remove releases the position of the item with key k.
func (v *QueueView) Remove(k string) (QueueItem, bool) {
it, ok := v.items[k]
if !ok {
return QueueItem{}, false
}
delete(v.items, k)
pq := &v.parts[it.Partition]
sq := pq.scopes[it.Scope]
sq.count--
pq.count--
for i := range sq.victims {
if sq.victims[i].Key == k {
sq.victims = append(sq.victims[:i], sq.victims[i+1:]...)
break
}
}
if sq.count == 0 {
delete(pq.scopes, it.Scope)
}
return it, true
}
// Placement is an admission decision: the partition the item joined and
// the item it displaced, if any.
type Placement struct {
Partition Partition
Displaced bool
Victim QueueItem
}
// Admit decides whether x may take a position and records it. x first
// takes a free position: an eligible item prefers the reserved partition,
// every other item uses the general one. When no position is free, each
// partition x may use is tried in the same order under the fair-share
// rules: a scope at or over its share can only replace its own lowest
// tier, newest entry, and only when that entry goes before x; a scope below
// its share reclaims one position from the scope with the greatest excess.
// Each full-partition decision advances that partition's cursor. A refused
// x changes nothing but the cursors.
func (v *QueueView) Admit(x QueueItem) (Placement, bool) {
order := []Partition{PartitionGeneral}
if x.Eligible {
order = []Partition{PartitionReserved, PartitionGeneral}
}
for _, p := range order {
if v.Room(p) {
return v.place(x, p, nil), true
}
}
for _, p := range order {
if victim, ok := v.overflow(x, p); ok {
return v.place(x, p, &victim), true
}
}
return Placement{}, false
}
func (v *QueueView) place(x QueueItem, p Partition, victim *QueueItem) Placement {
out := Placement{Partition: p}
if victim != nil {
v.Remove(victim.Key)
out.Displaced, out.Victim = true, *victim
}
x.Partition = p
// x is new to the view and its partition is valid, so Add cannot fail.
_ = v.Add(x)
return out
}
// overflow applies the fair-share rules of a full partition to x.
func (v *QueueView) overflow(x QueueItem, p Partition) (QueueItem, bool) {
pq := &v.parts[p]
keys := make([]string, 0, len(pq.scopes)+1)
for k := range pq.scopes {
keys = append(keys, k)
}
if pq.scopes[x.Scope] == nil {
keys = append(keys, x.Scope)
}
sort.Strings(keys)
shares := newFairShare(keys, pq.cursor, v.limits[p])
share := shares.of
pq.cursor = shares.first()
count := func(scope string) int {
if sq := pq.scopes[scope]; sq != nil {
return sq.count
}
return 0
}
if count(x.Scope) >= share(x.Scope) {
sq := pq.scopes[x.Scope]
if sq == nil || len(sq.victims) == 0 || !goesFirst(sq.victims[0], x) {
return QueueItem{}, false
}
return sq.victims[0], true
}
best, bestExcess := "", 0
for _, k := range shares.keys {
sq := pq.scopes[k]
if k == x.Scope || sq == nil || len(sq.victims) == 0 {
continue
}
if excess := sq.count - share(k); excess > bestExcess {
best, bestExcess = k, excess
}
}
if best == "" {
return QueueItem{}, false
}
return pq.scopes[best].victims[0], true
}
package admission
import (
"fmt"
"sort"
"time"
)
const queueEntryVersion = 1
const queueStateVersion = 1
const queueCountersVersion = 1
const ingressStateVersion = 1
// QueueEntry is the ledger's derived record of a live candidate: where it
// holds its queue position and how it was last assessed. Recovery rebuilds
// it from the candidate and its roots; it is never proof of anything.
type QueueEntry struct {
Partition Partition
// Tier, Direct, Corroborated and NextChange come from the last
// successful assessment. All are zero before one has succeeded; such
// an entry holds a general position.
Tier Tier
Direct bool
Corroborated bool
NextChange time.Time
}
// Assessed reports whether an assessment has succeeded for the entry.
func (q QueueEntry) Assessed() bool { return q.Tier != (Tier{}) }
// Eligible reports whether the last assessment allows the reserved lane.
func (q QueueEntry) Eligible() bool { return q.Direct || q.Corroborated }
type queueEntryRecord struct {
V uint8 `json:"v"`
Partition Partition `json:"partition"`
Class Class `json:"class,omitempty"`
Severity Severity `json:"severity,omitempty"`
Direct bool `json:"direct,omitempty"`
Corroborated bool `json:"corroborated,omitempty"`
NextChange int64 `json:"next_change,omitempty"`
}
func (q QueueEntry) record() (queueEntryRecord, error) {
bad := func(detail string) (queueEntryRecord, error) {
return queueEntryRecord{}, refuse(ReasonInvalid, detail)
}
if !q.Partition.Valid() {
return bad("queue entry has no partition")
}
rec := queueEntryRecord{V: queueEntryVersion, Partition: q.Partition, Class: q.Tier.Class, Severity: q.Tier.Severity, Direct: q.Direct, Corroborated: q.Corroborated}
if !q.Assessed() {
if q.Eligible() || !q.NextChange.IsZero() || q.Partition != PartitionGeneral {
return bad("an unassessed queue entry carries assessment fields")
}
return rec, nil
}
var ok bool
switch {
case !q.Tier.Valid():
return bad("queue entry tier is invalid")
case q.Direct && q.Corroborated:
return bad("queue entry is both direct and corroborated")
case q.Eligible() && q.Tier.Class != ClassC3:
return bad("only a C3 queue entry uses the reserved lane")
case q.Partition == PartitionReserved && !q.Eligible():
return bad("an ineligible queue entry holds a reserved position")
}
if rec.NextChange, ok = unixNano(q.NextChange); !ok {
return bad("queue entry has no reassessment time")
}
return rec, nil
}
// Validate checks the entry's invariants.
func (q QueueEntry) Validate() error {
_, err := q.record()
return err
}
func (q QueueEntry) MarshalBinary() ([]byte, error) {
rec, err := q.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalQueueEntry decodes a stored entry and checks its invariants.
func UnmarshalQueueEntry(data []byte) (QueueEntry, error) {
var rec queueEntryRecord
if err := openRecord(data, &rec); err != nil {
return QueueEntry{}, err
}
q := QueueEntry{Partition: rec.Partition, Tier: Tier{Class: rec.Class, Severity: rec.Severity}, Direct: rec.Direct,
Corroborated: rec.Corroborated, NextChange: fromNano(rec.NextChange)}
if again, err := q.record(); rec.V != queueEntryVersion || err != nil || again != rec {
return QueueEntry{}, ErrCorruptRecord
}
return q, nil
}
// QueueState is the ledger's queue bookkeeping.
type QueueState struct {
// NextSweep is the earliest time a queued candidate may need attention
// without a new report: an age-out, an effect expiry while it waits to
// retry, or a reassessment. Zero when nothing waits.
NextSweep time.Time
Cursors QueueCursors
// SetCursors are the remainder cursors of the temporary set's
// partitions, which a live ledger's schedule cycles move.
SetCursors QueueCursors
}
type queueStateRecord struct {
V uint8 `json:"v"`
NextSweep int64 `json:"next_sweep,omitempty"`
General string `json:"general,omitempty"`
Reserved string `json:"reserved,omitempty"`
SetGeneral string `json:"set_general,omitempty"`
SetReserved string `json:"set_reserved,omitempty"`
}
func validCursor(c string) bool { return c == "" || boundedToken(c, 128) }
func (s QueueState) record() (queueStateRecord, error) {
rec := queueStateRecord{V: queueStateVersion, General: s.Cursors.General, Reserved: s.Cursors.Reserved,
SetGeneral: s.SetCursors.General, SetReserved: s.SetCursors.Reserved}
if !s.NextSweep.IsZero() {
var ok bool
if rec.NextSweep, ok = unixNano(s.NextSweep); !ok {
return queueStateRecord{}, refuse(ReasonInvalid, "queue sweep time is not representable")
}
}
if !validCursor(rec.General) || !validCursor(rec.Reserved) || !validCursor(rec.SetGeneral) || !validCursor(rec.SetReserved) {
return queueStateRecord{}, refuse(ReasonInvalid, "queue cursor is malformed")
}
return rec, nil
}
func (s QueueState) MarshalBinary() ([]byte, error) {
rec, err := s.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalQueueState decodes stored queue bookkeeping.
func UnmarshalQueueState(data []byte) (QueueState, error) {
var rec queueStateRecord
if err := openRecord(data, &rec); err != nil {
return QueueState{}, err
}
s := QueueState{NextSweep: fromNano(rec.NextSweep), Cursors: QueueCursors{General: rec.General, Reserved: rec.Reserved},
SetCursors: QueueCursors{General: rec.SetGeneral, Reserved: rec.SetReserved}}
if again, err := s.record(); rec.V != queueStateVersion || err != nil || again != rec {
return QueueState{}, ErrCorruptRecord
}
return s, nil
}
// QueueEvent is what a queue counter counts. Values are persisted: append,
// never renumber.
type QueueEvent uint8
const (
// EventRefused counts arrivals refused before they held a position.
EventRefused QueueEvent = iota + 1
// EventDeferred counts queued candidates taking a new deferral reason.
EventDeferred
// EventEnded counts queued candidates ending as refused, withheld or
// dropped, including those displaced by overflow.
EventEnded
queueEventEnd
)
func (e QueueEvent) Valid() bool { return e >= EventRefused && e < queueEventEnd }
// CountKey is one counter. Class and Severity are both zero for a
// candidate without an assessment. No counter names an address, account or
// check: these dimensions are fixed.
type CountKey struct {
Event QueueEvent
Reason Reason
Class Class
Severity Severity
}
func (k CountKey) valid() bool {
unassessed := k.Class == 0 && k.Severity == 0
return k.Event.Valid() && k.Reason.Valid() && (unassessed || (k.Class.Valid() && k.Severity.Valid()))
}
// QueueCounters are durable, fixed-dimensional queue counts. Counts
// saturate instead of wrapping. The zero value is empty and ready to use.
type QueueCounters struct {
counts map[CountKey]uint64
}
// Add counts one event.
func (q *QueueCounters) Add(k CountKey) error {
if !k.valid() {
return refuse(ReasonInvalid, "queue counter key is invalid")
}
if q.counts == nil {
q.counts = map[CountKey]uint64{}
}
if q.counts[k] != ^uint64(0) {
q.counts[k]++
}
return nil
}
// AddCount counts n events of k at once, saturating as Add does.
func (q *QueueCounters) AddCount(k CountKey, n uint64) error {
if !k.valid() {
return refuse(ReasonInvalid, "queue counter key is invalid")
}
if n == 0 {
return nil
}
if q.counts == nil {
q.counts = map[CountKey]uint64{}
}
q.counts[k] += min(n, ^uint64(0)-q.counts[k])
return nil
}
// Count returns the count for k.
func (q QueueCounters) Count(k CountKey) uint64 { return q.counts[k] }
// QueueCount is one counter and its count.
type QueueCount struct {
Key CountKey
N uint64
}
// Rows are the counters in key order.
func (q QueueCounters) Rows() []QueueCount {
rows := make([]queueCountRow, 0, len(q.counts))
for k, n := range q.counts {
rows = append(rows, queueCountRow{Event: k.Event, Reason: k.Reason, Class: k.Class, Severity: k.Severity, N: n})
}
sort.Slice(rows, func(i, j int) bool { return rowLess(rows[i], rows[j]) })
out := make([]QueueCount, len(rows))
for i, r := range rows {
out[i] = QueueCount{Key: CountKey{Event: r.Event, Reason: r.Reason, Class: r.Class, Severity: r.Severity}, N: r.N}
}
return out
}
type queueCountRow struct {
Event QueueEvent `json:"e"`
Reason Reason `json:"r"`
Class Class `json:"c,omitempty"`
Severity Severity `json:"s,omitempty"`
N uint64 `json:"n"`
}
type queueCountersRecord struct {
V uint8 `json:"v"`
Rows []queueCountRow `json:"rows"`
}
func rowLess(a, b queueCountRow) bool {
switch {
case a.Event != b.Event:
return a.Event < b.Event
case a.Reason != b.Reason:
return a.Reason < b.Reason
case a.Class != b.Class:
return a.Class < b.Class
}
return a.Severity < b.Severity
}
func (q QueueCounters) MarshalBinary() ([]byte, error) {
rows := make([]queueCountRow, 0, len(q.counts))
for k, n := range q.counts {
if !k.valid() || n == 0 {
return nil, refuse(ReasonInvalid, "queue counter is invalid")
}
rows = append(rows, queueCountRow{Event: k.Event, Reason: k.Reason, Class: k.Class, Severity: k.Severity, N: n})
}
sort.Slice(rows, func(i, j int) bool { return rowLess(rows[i], rows[j]) })
return sealRecord(queueCountersRecord{V: queueCountersVersion, Rows: rows})
}
// UnmarshalQueueCounters decodes stored counters.
func UnmarshalQueueCounters(data []byte) (QueueCounters, error) {
var rec queueCountersRecord
if err := openRecord(data, &rec); err != nil {
return QueueCounters{}, err
}
if rec.V != queueCountersVersion || rec.Rows == nil {
return QueueCounters{}, ErrCorruptRecord
}
q := QueueCounters{counts: make(map[CountKey]uint64, len(rec.Rows))}
for i, row := range rec.Rows {
k := CountKey{Event: row.Event, Reason: row.Reason, Class: row.Class, Severity: row.Severity}
if !k.valid() || row.N == 0 || (i > 0 && !rowLess(rec.Rows[i-1], row)) {
return QueueCounters{}, ErrCorruptRecord
}
q.counts[k] = row.N
}
return q, nil
}
func (e QueueEvent) String() string {
switch e {
case EventRefused:
return "refused"
case EventDeferred:
return "deferred"
case EventEnded:
return "ended"
}
return fmt.Sprintf("event(%d)", uint8(e))
}
// IngressState is the ledger's record of ingress generations. Items the
// ingress held when the daemon stopped without a clean close are lost; the
// ledger cannot count them, only record that a generation was interrupted.
type IngressState struct {
// Generation numbers the current ingress generation, from 1; zero
// before the first.
Generation uint64
// Open is true from BeginIngress until a clean EndIngress.
Open bool
// Persisted counts arrivals decided in the current generation.
Persisted uint64
// Interrupted counts generations that ended without a clean close.
Interrupted uint64
// Resumed is the generation that began after the latest interruption;
// cleared when a clean generation follows. While it is the current
// generation, counts are lower bounds.
Resumed uint64
Checkpoint *IngressCheckpoint
}
type ingressStateRecord struct {
V uint8 `json:"v"`
Generation uint64 `json:"generation,omitempty"`
Open bool `json:"open,omitempty"`
Persisted uint64 `json:"persisted,omitempty"`
Interrupted uint64 `json:"interrupted,omitempty"`
Resumed uint64 `json:"resumed,omitempty"`
Checkpoint *IngressCheckpoint `json:"checkpoint,omitempty"`
}
func (s IngressState) record() (ingressStateRecord, error) {
if (s.Generation == 0 && (s.Open || s.Persisted != 0)) || s.Interrupted >= max(s.Generation, 1) {
return ingressStateRecord{}, refuse(ReasonInvalid, "ingress state is inconsistent")
}
if s.Resumed != 0 && (s.Interrupted == 0 || s.Resumed < 2 || s.Resumed > s.Generation) {
return ingressStateRecord{}, refuse(ReasonInvalid, "ingress state resumes a generation it has not reached")
}
if cp := s.Checkpoint; cp != nil {
if cp.Generation > s.Generation || cp.Validate(nil, cp.Generation) != nil {
return ingressStateRecord{}, refuse(ReasonInvalid, "ingress checkpoint is inconsistent")
}
}
return ingressStateRecord{V: ingressStateVersion, Generation: s.Generation, Open: s.Open, Persisted: s.Persisted, Interrupted: s.Interrupted, Resumed: s.Resumed, Checkpoint: s.Checkpoint}, nil
}
func (s IngressState) MarshalBinary() ([]byte, error) {
rec, err := s.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalIngressState decodes the stored ingress record.
func UnmarshalIngressState(data []byte) (IngressState, error) {
var rec ingressStateRecord
if err := openRecord(data, &rec); err != nil {
return IngressState{}, err
}
s := IngressState{Generation: rec.Generation, Open: rec.Open, Persisted: rec.Persisted, Interrupted: rec.Interrupted, Resumed: rec.Resumed, Checkpoint: rec.Checkpoint}
if _, err := s.record(); rec.V != ingressStateVersion || err != nil {
return IngressState{}, ErrCorruptRecord
}
return s, nil
}
package admission
import (
"maps"
"sort"
)
const inventoryVersion = 1
const reportLinksVersion = 1
// MaxReportLinks bounds the later findings kept for one evidence record.
// Further reports are only counted.
const MaxReportLinks = 8
type inventoryRecord struct {
V uint8 `json:"v"`
Accounts map[string]uint64 `json:"accounts"`
Domains map[string]string `json:"domains"`
}
// MarshalBinary encodes the inventory so the ledger can keep the last good
// snapshot when a later read fails.
func (inv *Inventory) MarshalBinary() ([]byte, error) {
if inv == nil || inv.accounts == nil || inv.domains == nil {
return nil, refuse(ReasonInvalid, "inventory is not initialized")
}
return sealRecord(inventoryRecord{V: inventoryVersion, Accounts: inv.accounts, Domains: inv.domains})
}
// UnmarshalInventory decodes a stored inventory and rebuilds it through
// NewInventory, so a stored snapshot obeys the same rules as a fresh one.
func UnmarshalInventory(data []byte) (*Inventory, error) {
var rec inventoryRecord
if err := openRecord(data, &rec); err != nil {
return nil, err
}
if rec.V != inventoryVersion || rec.Accounts == nil || rec.Domains == nil {
return nil, ErrCorruptRecord
}
inv, err := NewInventory(rec.Accounts, rec.Domains)
if err != nil {
return nil, ErrCorruptRecord
}
return inv, nil
}
// MatchesGenerations checks the persisted snapshot against its allocation
// tracker before either can be used to refresh account identities.
func (inv *Inventory) MatchesGenerations(g *Generations) bool {
return inv != nil && inv.accounts != nil && inv.domains != nil &&
g != nil && g.next != 0 && maps.Equal(inv.accounts, g.live)
}
// ReportLinks are the later findings that reported one evidence record. They
// are metadata: adding one never changes the evidence, its queue age or its
// freshness.
type ReportLinks struct {
// Evidence is the record the links belong to, so a stored copy under
// another evidence ID reads as damaged.
Evidence EvidenceID
// Links are finding IDs, sorted and unique.
Links []string
// Dropped counts reports beyond MaxReportLinks. It saturates.
Dropped uint32
}
type reportLinksRecord struct {
V uint8 `json:"v"`
Evidence EvidenceID `json:"evidence"`
Links []string `json:"links"`
Dropped uint32 `json:"dropped,omitempty"`
}
// Add returns the links with findingID recorded, and whether they changed.
func (r ReportLinks) Add(findingID string) (ReportLinks, bool, error) {
if !r.valid() {
return r, false, refuse(ReasonInvalid, "report links are inconsistent")
}
if !lowerHex(findingID, 16) {
return r, false, refuse(ReasonInvalid, "finding ID is not 16 lowercase hex digits")
}
i := sort.SearchStrings(r.Links, findingID)
if i < len(r.Links) && r.Links[i] == findingID {
return r, false, nil
}
if len(r.Links) >= MaxReportLinks {
if r.Dropped == ^uint32(0) {
return r, false, nil
}
r.Dropped++
return r, true, nil
}
links := make([]string, 0, len(r.Links)+1)
links = append(links, r.Links[:i]...)
links = append(links, findingID)
r.Links = append(links, r.Links[i:]...)
return r, true, nil
}
func (r ReportLinks) valid() bool {
if _, err := ParseEvidenceID(string(r.Evidence)); err != nil {
return false
}
if len(r.Links) > MaxReportLinks || (r.Dropped > 0 && len(r.Links) < MaxReportLinks) {
return false
}
for i, id := range r.Links {
if !lowerHex(id, 16) || (i > 0 && r.Links[i-1] >= id) {
return false
}
}
return true
}
func (r ReportLinks) MarshalBinary() ([]byte, error) {
if !r.valid() {
return nil, refuse(ReasonInvalid, "report links are inconsistent")
}
links := r.Links
if links == nil {
links = []string{}
}
return sealRecord(reportLinksRecord{V: reportLinksVersion, Evidence: r.Evidence, Links: links, Dropped: r.Dropped})
}
// UnmarshalReportLinks decodes stored links and checks their invariants.
func UnmarshalReportLinks(data []byte) (ReportLinks, error) {
var rec reportLinksRecord
if err := openRecord(data, &rec); err != nil {
return ReportLinks{}, err
}
r := ReportLinks{Evidence: rec.Evidence, Links: rec.Links, Dropped: rec.Dropped}
if rec.V != reportLinksVersion || rec.Links == nil || !r.valid() {
return ReportLinks{}, ErrCorruptRecord
}
return r, nil
}
package admission
import (
"maps"
"slices"
"sort"
"time"
)
// RotationHold is how long a verified automatic block keeps its slot
// before equal-tier rotation may take it, on both wall and elapsed
// admission time.
const RotationHold = 30 * time.Minute
// RotationHoldPassed reports whether a block applied at admission time
// applied, when the ceiling's elapsed time read from, has kept its slot
// for RotationHold on both clocks at admission time now with elapsed, as
// a charge's window does. Downtime never counts, so a restart cannot
// shorten a hold.
func RotationHoldPassed(applied time.Time, from time.Duration, now time.Time, elapsed time.Duration) bool {
return !now.Before(applied.Add(RotationHold)) && elapsed-from >= RotationHold
}
// HoldPassed reports whether e's rotation hold has passed at admission
// time now with the ceiling's elapsed time elapsed: its application is
// verified and RotationHoldPassed holds since.
func (e EffectRecord) HoldPassed(now time.Time, elapsed time.Duration) bool {
return e.Verified() && RotationHoldPassed(e.Applied, e.Elapsed, now, elapsed)
}
// fairShare is the integer share rule, the same for queue
// positions and set slots: capacity split evenly over the scopes, and the
// remainder one each to the scopes after the cursor in rotation order.
type fairShare struct {
// keys are the scopes in rotation order, from the one after the cursor.
keys []string
rank map[string]int
capacity int
}
// newFairShare shares capacity among keys, sorted, unique and not empty.
func newFairShare(keys []string, cursor string, capacity int) fairShare {
n := len(keys)
start := sort.SearchStrings(keys, cursor)
if start < n && keys[start] == cursor {
start++
}
start %= n
f := fairShare{keys: make([]string, n), rank: make(map[string]int, n), capacity: capacity}
for r := range n {
f.keys[r] = keys[(start+r)%n]
f.rank[f.keys[r]] = r
}
return f
}
// of is the share of key, one of the shared keys.
func (f fairShare) of(key string) int {
n := len(f.keys)
if f.rank[key] < f.capacity%n {
return f.capacity/n + 1
}
return f.capacity / n
}
// first is the scope after the cursor: where the cursor moves next.
func (f fairShare) first() string { return f.keys[0] }
// setShares is how the scopes of each partition share it under pressure.
type setShares struct {
holding [partitionEnd]map[string]int
waiting [partitionEnd][]string
cursor [partitionEnd]string
}
func (sh *setShares) clone() *setShares {
if sh == nil {
return nil
}
out := *sh
for p := range out.holding {
out.holding[p] = maps.Clone(sh.holding[p])
out.waiting[p] = slices.Clone(sh.waiting[p])
}
return &out
}
// active is the sorted scopes of partition p: those holding a slot, those
// waiting for one and extra, if not empty.
func (sh *setShares) active(p Partition, extra string) []string {
var keys []string
for k, n := range sh.holding[p] {
if n > 0 {
keys = append(keys, k)
}
}
keys = append(keys, sh.waiting[p]...)
if extra != "" {
keys = append(keys, extra)
}
slices.Sort(keys)
return slices.Compact(keys)
}
// move counts a slot of partition p for scope taken from victim, which
// leaves its own scope.
func (sh *setShares) move(p Partition, scope string, victim EvictionVictim) {
if sh == nil {
return
}
if scope != "" {
if sh.holding[p] == nil {
sh.holding[p] = map[string]int{}
}
sh.holding[p][scope]++
}
if victim.Scope != "" && sh.holding[victim.Partition][victim.Scope] > 0 {
sh.holding[victim.Partition][victim.Scope]--
}
}
// Share lets fresh work rotate equal-tier blocks out of a full partition.
// holding is the slots each scope's unclaimed blocks hold in
// each partition, waiting the scopes with work waiting for a slot there,
// and cursors the partitions' persisted remainder cursors. A partition is
// shared among its active scopes, those holding or waiting and the
// requester. Without it no work rotates.
func (s *SlotLimits) Share(holding map[Partition]map[string]int, waiting map[Partition][]string, cursors QueueCursors) {
if s == nil {
return
}
sh := &setShares{}
for _, p := range []Partition{PartitionGeneral, PartitionReserved} {
sh.holding[p] = maps.Clone(holding[p])
sh.waiting[p] = slices.Clone(waiting[p])
}
sh.cursor[PartitionGeneral], sh.cursor[PartitionReserved] = cursors.General, cursors.Reserved
s.shares = sh
}
// Advance moves each partition's remainder cursor to the first active
// scope after it and returns the cursors; a partition without an active
// scope keeps its cursor. A schedule cycle advances them once, so scopes
// that outnumber the slots take turns at the remainder.
func (s *SlotLimits) Advance() QueueCursors {
if s == nil || s.shares == nil {
return QueueCursors{}
}
sh := s.shares
for _, p := range []Partition{PartitionGeneral, PartitionReserved} {
if keys := sh.active(p, ""); len(keys) > 0 {
sh.cursor[p] = newFairShare(keys, sh.cursor[p], 0).first()
}
}
return QueueCursors{General: sh.cursor[PartitionGeneral], Reserved: sh.cursor[PartitionReserved]}
}
// rotation is the index of the block fresh work d may rotate out of its
// lane's partition, or -1. Only verified blocks of tier
// exactly d.Tier and of a scope are candidates. The scope comes first:
// zero share waits; at or over a positive share d rotates only its own
// blocks; below it, d reclaims only the greatest over-share borrowing,
// ties in cursor order, and never
// takes a scope below its share. Then the oldest application, then the
// lowest action, among the chosen scope's blocks whose hold passed; none
// means d waits.
func (s *SlotLimits) rotation(lane Lane, d SlotDemand) int {
p := PartitionOf(lane)
if s.shares == nil || !d.Fresh || d.Scope == "" || !p.Valid() {
return -1
}
sh := s.shares
candidate := func(v EvictionVictim) bool { return rotationCandidate(p, d, v) }
// The scopes with a candidate, whatever its age: one pass, not one per
// scope.
listed := map[string]bool{}
for _, v := range s.victims {
if candidate(v) {
listed[v.Scope] = true
}
}
share := newFairShare(sh.active(p, d.Scope), sh.cursor[p], s.size[p])
if share.of(d.Scope) == 0 {
return -1
}
scope := d.Scope
if sh.holding[p][d.Scope] < share.of(d.Scope) {
scope = ""
best := 0
for _, k := range share.keys {
if k == d.Scope || !listed[k] {
continue
}
if excess := sh.holding[p][k] - share.of(k); excess > best {
scope, best = k, excess
}
}
if scope == "" {
return -1
}
}
pick := -1
for i, v := range s.victims {
if !candidate(v) || v.Scope != scope || !v.Rotatable {
continue
}
if pick < 0 || v.Applied.Before(s.victims[pick].Applied) ||
(v.Applied.Equal(s.victims[pick].Applied) && v.Action < s.victims[pick].Action) {
pick = i
}
}
return pick
}
// rotationCandidate reports whether v is a block of partition p that
// rotation could give work d: a scoped block, never an import, of
// exactly d's tier at another address.
func rotationCandidate(p Partition, d SlotDemand, v EvictionVictim) bool {
return v.Partition == p && v.Tier == d.Tier && v.Target != d.Target && v.Scope != ""
}
// EqualTier reports whether lane's partition holds a block rotation could
// give one-unit block work d once the block's hold, d's freshness and the
// shares allow it. Work that finds no room while one does waits on an
// equal-tier block.
func (s *SlotLimits) EqualTier(lane Lane, d SlotDemand) bool {
if s == nil || !d.Evicts || d.Cost != 1 {
return false
}
p := PartitionOf(lane)
return slices.ContainsFunc(s.victims, func(v EvictionVictim) bool { return rotationCandidate(p, d, v) })
}
package admission
import (
"fmt"
"sort"
"time"
)
// MaxMemberCost bounds one candidate's cost in block units: one member
// spans at most 64 set elements.
const MaxMemberCost = 64
// MaxBatchMembers bounds the candidates one schedule picks and the members
// of one execution batch.
const MaxBatchMembers = 64
const scheduleStateVersion = 1
// Lane is the service a pick draws on. Values are persisted: append, never
// renumber.
type Lane uint8
const (
LaneGeneral Lane = iota + 1
// LaneDirect is the reserved lane's turn for direct compromise evidence.
LaneDirect
// LaneCorroborated is the reserved lane's turn for independently
// corroborated evidence.
LaneCorroborated
laneEnd
)
var laneNames = [...]string{"", "general", "direct", "corroborated"}
func (l Lane) Valid() bool { return l >= LaneGeneral && l < laneEnd }
func (l Lane) String() string {
if l.Valid() {
return laneNames[l]
}
return fmt.Sprintf("lane(%d)", uint8(l))
}
// Rings of the scheduler: one per general class and one per reserved
// sub-lane. Each rotates its scopes on its own.
const (
ringC1 = iota
ringC2
ringC3
ringDirect
ringCorroborated
ringCount
)
// patternSlots is the length of both 4:2:1 quantum patterns.
const patternSlots uint8 = 7
// classPattern is the repeating C3:C2:C1 quantum schedule of 4:2:1.
var classPattern = [patternSlots]Class{ClassC3, ClassC3, ClassC3, ClassC3, ClassC2, ClassC2, ClassC1}
// severityPattern is the Critical:High:Warning quantum schedule of 4:2:1
// inside a scope.
var severityPattern = [patternSlots]Severity{SeverityCritical, SeverityCritical, SeverityCritical, SeverityCritical, SeverityHigh, SeverityHigh, SeverityWarning}
// ScheduleItem is a queued candidate as the scheduler sees it.
type ScheduleItem struct {
ID CandidateID
Scope string
Tier Tier
Direct bool
Corroborated bool
Queued time.Time
// Cost is the fair-turn demand, 1 to MaxMemberCost.
Cost uint32
// CeilingCost is the actual block demand; zero still spends a fair turn.
CeilingCost uint32
// Bytes is the history a reservation of it charges now, at most
// MaxHistoryBytes.
Bytes uint32
// Recovery is the reserve room a reservation needs: the space an
// unresolved outcome would need and its attempt's audit rows.
Recovery uint32
// Ready is false while a proven failure's retry wait lasts. A waiting
// candidate is never picked, but its scope keeps its deficit.
Ready bool
// Slot readiness differs by lane; membership stays intact so blocked
// work retains its earned scope turns. Zero values add no constraint.
BlockedGeneral, BlockedReserved bool
}
func (it ScheduleItem) readyOn(reserved bool) bool {
return it.Ready && ((!reserved && !it.BlockedGeneral) || (reserved && !it.BlockedReserved))
}
// Answer is an outside answer a queued candidate waits on, such as the
// panel's verdict on a block.
type Answer uint8
const (
// AnswerPending: the question is outstanding. The candidate is never
// picked or deferred, and its scope keeps its turns.
AnswerPending Answer = iota + 1
// AnswerAllow: the answer ends the candidate without a charge. A
// schedule picks it as uncharged work, at no ceiling or history cost,
// whatever the live budgets; it still takes a member and a fair turn.
AnswerAllow
)
// ScheduleLimits bound one schedule.
type ScheduleLimits struct {
// General and Reserved are the block units each lane may serve. The
// ledger lowers them to what its ceiling can charge now.
General uint32
Reserved uint32
// Members bounds the picks, 1 to MaxBatchMembers.
Members int
// GeneralBytes and ReservedBytes are the history bytes each lane may
// admit. The ledger sets them from its history budget, replacing any
// value the caller passed.
GeneralBytes, ReservedBytes uint64
// RecoveryBytes is shared by both lanes and spent by each pick.
RecoveryBytes uint64
}
// Pick is one candidate chosen for service and the lane it draws on.
type Pick struct {
ID CandidateID
Lane Lane
Cost uint32
Bytes uint32
// Reserved is the reserved lane the candidate is eligible for, direct
// or corroborated, whichever lane served it; zero for general-only work.
Reserved Lane
// Tier is the item's (class, severity), which decides what work it may
// replace in a full set.
Tier Tier
}
// ScopeTurn is a scope's persisted place in one ring.
type ScopeTurn struct {
// Severity is the scope's position in the severity pattern.
Severity uint8
// Deficit is the block units the scope has earned toward a member that
// costs more than one turn; it never exceeds MaxMemberCost.
Deficit uint32
// Bytes is the history the scope has earned toward its next member,
// HistoryQuantum per turn while it is short; it never exceeds
// MaxHistoryBytes.
Bytes uint32
}
// Ring is the persisted rotation of one ring: the scope served last and
// the turns of scopes that still have work in it.
type Ring struct {
Last string
// Held names the scope promised the next storage budget. Last alone
// cannot preserve that promise when new scopes enter the rotation.
Held string
Scopes map[string]ScopeTurn
}
// ScheduleState is the persisted scheduler position. The zero value is a
// fresh scheduler.
type ScheduleState struct {
// ClassSlot is the general lane's position in the class pattern.
ClassSlot uint8
// NextCorroborated: the reserved lane serves corroboration next when
// both of its turns have work.
NextCorroborated bool
Rings [ringCount]Ring
GeneralHold, ReservedHold *CeilingHold
}
func (s ScheduleState) clone() ScheduleState {
out := s
if s.GeneralHold != nil {
h := *s.GeneralHold
out.GeneralHold = &h
}
if s.ReservedHold != nil {
h := *s.ReservedHold
out.ReservedHold = &h
}
for i := range out.Rings {
scopes := make(map[string]ScopeTurn, len(s.Rings[i].Scopes))
for k, v := range s.Rings[i].Scopes {
scopes[k] = v
}
out.Rings[i].Scopes = scopes
}
return out
}
func validateItems(items []ScheduleItem) error {
seen := make(map[CandidateID]bool, len(items))
for _, it := range items {
switch {
case seen[it.ID] || it.Scope == "":
return refuse(ReasonInvalid, "schedule item is repeated or has no scope")
case !it.Tier.Valid() || it.Cost == 0 || it.Cost > MaxMemberCost || it.CeilingCost > MaxMemberCost || it.Bytes > MaxHistoryBytes || it.Recovery > MaxRecoveryNeed:
return refuse(ReasonInvalid, "schedule item has an invalid tier or cost")
case it.Direct && it.Corroborated, (it.Direct || it.Corroborated) && it.Tier.Class != ClassC3:
return refuse(ReasonInvalid, "schedule item has an impossible reserved turn")
}
seen[it.ID] = true
}
return nil
}
// before orders candidates of one severity: oldest first, then by ID.
func before(a, b *ScheduleItem) bool {
if !a.Queued.Equal(b.Queued) {
return a.Queued.Before(b.Queued)
}
return a.ID < b.ID
}
type ringMembers struct {
// ready holds each scope's ready members by severity.
ready map[string]map[Severity][]*ScheduleItem
}
func memberOf(r int, it *ScheduleItem) bool {
switch r {
case ringDirect:
return it.Direct
case ringCorroborated:
return it.Corroborated
}
return int(it.Tier.Class)-1 == r
}
func buildRings(items []ScheduleItem) [ringCount]ringMembers {
var rings [ringCount]ringMembers
for r := range rings {
rings[r] = ringMembers{ready: map[string]map[Severity][]*ScheduleItem{}}
}
for i := range items {
it := &items[i]
for r := range rings {
if !memberOf(r, it) || !it.readyOn(r >= ringDirect) {
continue
}
bySeverity := rings[r].ready[it.Scope]
if bySeverity == nil {
bySeverity = map[Severity][]*ScheduleItem{}
rings[r].ready[it.Scope] = bySeverity
}
bySeverity[it.Tier.Severity] = append(bySeverity[it.Tier.Severity], it)
}
}
for r := range rings {
for _, bySeverity := range rings[r].ready {
for _, list := range bySeverity {
sort.Slice(list, func(i, j int) bool { return before(list[i], list[j]) })
}
}
}
return rings
}
type scheduler struct {
st ScheduleState
rings [ringCount]ringMembers
picked map[CandidateID]bool
}
// head is the member a scope serves next in ring r: the first severity in
// the pattern from the scope's position that has a ready member, and its
// oldest member. slot is that severity's pattern position.
func (s *scheduler) head(r int, scope string) (*ScheduleItem, uint8) {
bySeverity := s.rings[r].ready[scope]
start := s.st.Rings[r].Scopes[scope].Severity
for k := uint8(0); k < patternSlots; k++ {
slot := (start + k) % patternSlots
for _, it := range bySeverity[severityPattern[slot]] {
if !s.picked[it.ID] {
return it, slot
}
}
}
return nil, 0
}
// serve takes one turn of ring r within budget and the history bytes left.
// It rotates the scopes that have a ready member, from the one after the
// last served; each visit adds one block unit to the scope's deficit,
// capped at MaxMemberCost, and HistoryQuantum bytes while its earned bytes
// fall short of its head, capped at MaxHistoryBytes. The first scope whose
// deficit and earned bytes cover its head and whose head the budget covers
// is served. A head above the full budget can still earn bounded credit
// while other scopes run. A head that fits the full batch budget but not
// its remainder holds its turn for the next batch, and so does a head whose
// scope has earned its bytes when the bytes left cannot pay for it: lending
// that turn could leave it with too little budget on every call. The
// boolean reports this budget block; otherwise nil means no head fits.
func (s *scheduler) serve(r int, budget, fullBudget uint32, bytes, recovery uint64) (*ScheduleItem, bool) {
ring := &s.st.Rings[r]
if it, _ := s.head(r, ring.Held); it == nil {
ring.Held = ""
}
var scopes []string
fits := false
for scope := range s.rings[r].ready {
if it, _ := s.head(r, scope); it != nil {
scopes = append(scopes, scope)
fits = fits || it.CeilingCost <= fullBudget
}
}
if !fits {
return nil, false
}
sort.Strings(scopes)
start := sort.SearchStrings(scopes, ring.Last)
if start < len(scopes) && scopes[start] == ring.Last {
start++
}
if ring.Held != "" {
start = sort.SearchStrings(scopes, ring.Held)
}
// A fitting head is served or held for a fresh budget within
// MaxMemberCost rotations, since each visit earns a unit.
for visit := 0; ; visit++ {
scope := scopes[(start+visit)%len(scopes)]
it, slot := s.head(r, scope)
if it.CeilingCost > budget && it.CeilingCost <= fullBudget {
return it, true
}
turn := ring.Scopes[scope]
if turn.Deficit < MaxMemberCost {
turn.Deficit++
}
if turn.Bytes < it.Bytes {
turn.Bytes = min(turn.Bytes+HistoryQuantum, MaxHistoryBytes)
}
earned := it.Cost <= turn.Deficit && it.Bytes <= turn.Bytes
if earned && (uint64(it.Bytes) > bytes || uint64(it.Recovery) > recovery) {
ring.Scopes[scope] = turn
ring.Held = scope
return nil, true
}
ring.Held = ""
ring.Last = scope
if earned && it.CeilingCost <= budget {
turn.Deficit -= it.Cost
turn.Bytes -= it.Bytes
turn.Severity = (slot + 1) % patternSlots
ring.Scopes[scope] = turn
return it, false
}
ring.Scopes[scope] = turn
}
}
// Schedule picks up to lim.Members candidates, reserved lane first. The
// reserved lane alternates its direct and corroborated turns while both
// have work; the general lane follows the C3:C2:C1 pattern, lending an
// empty class's turn to the next class with work. Inside a ring, verified
// scopes rotate, a scope serves Critical:High:Warning 4:2:1, and a severity
// serves its oldest candidate. No candidate is picked twice. The returned
// state keeps turns only for scopes that still have work in a ring.
func Schedule(items []ScheduleItem, st ScheduleState, lim ScheduleLimits) ([]Pick, ScheduleState, error) {
if lim.Members < 1 || lim.Members > MaxBatchMembers {
return nil, st, refuse(ReasonInvalid, "schedule member bound is out of range")
}
if err := validateItems(items); err != nil {
return nil, st, err
}
if err := st.validate(); err != nil {
return nil, st, err
}
own := append([]ScheduleItem(nil), items...)
s := &scheduler{st: st.clone(), rings: buildRings(own), picked: map[CandidateID]bool{}}
var picks []Pick
take := func(it *ScheduleItem, lane Lane) {
lim.RecoveryBytes -= uint64(it.Recovery)
s.picked[it.ID] = true
var eligible Lane
switch {
case it.Direct:
eligible = LaneDirect
case it.Corroborated:
eligible = LaneCorroborated
}
picks = append(picks, Pick{ID: it.ID, Lane: lane, Cost: it.CeilingCost, Bytes: it.Bytes, Reserved: eligible, Tier: it.Tier})
}
for _, reserved := range []bool{true, false} {
budget, bytes := lim.General, lim.GeneralBytes
if reserved {
budget, bytes = lim.Reserved, lim.ReservedBytes
}
fullBudget := budget
for len(picks) < lim.Members {
it, lane, blocked := s.lane(own, reserved, budget, fullBudget, bytes, lim.RecoveryBytes)
if blocked || it == nil {
break
}
budget -= it.CeilingCost
bytes -= uint64(it.Bytes)
take(it, lane)
}
}
// A scope keeps its turn only while it has work left in the ring; a
// scope that emptied starts afresh, as in deficit round robin.
var left [ringCount]map[string]bool
for r := range left {
left[r] = map[string]bool{}
}
for i := range own {
for r := range left {
if !s.picked[own[i].ID] && memberOf(r, &own[i]) {
left[r][own[i].Scope] = true
}
}
}
for r := range s.st.Rings {
if !left[r][s.st.Rings[r].Held] {
s.st.Rings[r].Held = ""
}
for scope, turn := range s.st.Rings[r].Scopes {
if !left[r][scope] || turn == (ScopeTurn{}) {
delete(s.st.Rings[r].Scopes, scope)
}
}
}
return picks, s.st, nil
}
type scopeTurnRecord struct {
Scope string `json:"scope"`
Severity uint8 `json:"severity,omitempty"`
Deficit uint32 `json:"deficit,omitempty"`
Bytes uint32 `json:"bytes,omitempty"`
}
type ringRecord struct {
Last string `json:"last,omitempty"`
Held string `json:"held,omitempty"`
Scopes []scopeTurnRecord `json:"scopes"`
}
type scheduleStateRecord struct {
V uint8 `json:"v"`
ClassSlot uint8 `json:"class_slot,omitempty"`
NextCorroborated bool `json:"next_corroborated,omitempty"`
Rings []ringRecord `json:"rings"`
GeneralHold *CeilingHold `json:"general_hold,omitempty"`
ReservedHold *CeilingHold `json:"reserved_hold,omitempty"`
}
func (s ScheduleState) record() (scheduleStateRecord, error) {
bad := func(detail string) (scheduleStateRecord, error) {
return scheduleStateRecord{}, refuse(ReasonInvalid, detail)
}
if s.ClassSlot >= patternSlots || !s.GeneralHold.valid(false) || !s.ReservedHold.valid(true) {
return bad("schedule class position is out of range")
}
rec := scheduleStateRecord{V: scheduleStateVersion, ClassSlot: s.ClassSlot, NextCorroborated: s.NextCorroborated, Rings: make([]ringRecord, ringCount), GeneralHold: s.GeneralHold, ReservedHold: s.ReservedHold}
for r, ring := range s.Rings {
if !validCursor(ring.Last) || len(ring.Scopes) > QueueCapacity {
return bad("schedule ring is malformed")
}
if ring.Held != "" && ring.Scopes[ring.Held].Deficit == 0 {
return bad("held schedule scope has no earned turn")
}
rows := make([]scopeTurnRecord, 0, len(ring.Scopes))
for scope, turn := range ring.Scopes {
if !boundedToken(scope, 128) || turn.Severity >= patternSlots || turn.Deficit > MaxMemberCost || turn.Bytes > MaxHistoryBytes || turn == (ScopeTurn{}) {
return bad("schedule scope turn is malformed")
}
rows = append(rows, scopeTurnRecord{Scope: scope, Severity: turn.Severity, Deficit: turn.Deficit, Bytes: turn.Bytes})
}
sort.Slice(rows, func(i, j int) bool { return rows[i].Scope < rows[j].Scope })
rec.Rings[r] = ringRecord{Last: ring.Last, Held: ring.Held, Scopes: rows}
}
return rec, nil
}
func (s ScheduleState) validate() error {
_, err := s.record()
return err
}
func (s ScheduleState) MarshalBinary() ([]byte, error) {
rec, err := s.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalScheduleState decodes a stored scheduler position.
func UnmarshalScheduleState(data []byte) (ScheduleState, error) {
var rec scheduleStateRecord
if err := openRecord(data, &rec); err != nil {
return ScheduleState{}, err
}
if rec.V != scheduleStateVersion || len(rec.Rings) != ringCount {
return ScheduleState{}, ErrCorruptRecord
}
s := ScheduleState{ClassSlot: rec.ClassSlot, NextCorroborated: rec.NextCorroborated, GeneralHold: rec.GeneralHold, ReservedHold: rec.ReservedHold}
for r, ring := range rec.Rings {
if ring.Scopes == nil {
return ScheduleState{}, ErrCorruptRecord
}
s.Rings[r] = Ring{Last: ring.Last, Held: ring.Held, Scopes: make(map[string]ScopeTurn, len(ring.Scopes))}
for i, row := range ring.Scopes {
if i > 0 && ring.Scopes[i-1].Scope >= row.Scope {
return ScheduleState{}, ErrCorruptRecord
}
s.Rings[r].Scopes[row.Scope] = ScopeTurn{Severity: row.Severity, Deficit: row.Deficit, Bytes: row.Bytes}
}
}
if s.validate() != nil {
return ScheduleState{}, ErrCorruptRecord
}
return s, nil
}
package admission
import "sort"
// CeilingHold preserves the fair turn of a charged head while work with no
// ceiling demand borrows the lane. Each lane keeps at most one such head.
// Its credits are separate from Ring.Held, which promises history or recovery.
type CeilingHold struct {
ID CandidateID `json:"id"`
Scope string `json:"scope"`
Ring uint8 `json:"ring"`
Slot uint8 `json:"slot"`
ClassSlot uint8 `json:"class_slot"`
Turn ScopeTurn `json:"turn"`
}
func (h *CeilingHold) valid(reserved bool) bool {
if h == nil {
return true
}
return boundedToken(string(h.ID), 128) && boundedToken(h.Scope, 128) && h.Ring < ringCount && (h.Ring >= ringDirect) == reserved && h.Slot < patternSlots && h.ClassSlot < patternSlots && h.Turn.Severity < patternSlots && h.Turn.Deficit <= MaxMemberCost && h.Turn.Bytes <= MaxHistoryBytes && (reserved || int(classPattern[h.ClassSlot])-1 == int(h.Ring))
}
func (s *scheduler) freeReady(items []ScheduleItem, reserved bool) bool {
for _, it := range items {
if it.readyOn(reserved) && it.CeilingCost == 0 && !s.picked[it.ID] && (!reserved || it.Direct || it.Corroborated) {
return true
}
}
return false
}
func (s *scheduler) held(h **CeilingHold, items []ScheduleItem) *ScheduleItem {
if *h == nil {
return nil
}
for i := range items {
it := &items[i]
if it.ID != (*h).ID || s.picked[it.ID] || !it.Ready || !memberOf(int((*h).Ring), it) || it.Scope != (*h).Scope || it.Tier.Severity != severityPattern[(*h).Slot] {
continue
}
if !it.readyOn((*h).Ring >= ringDirect) {
// A blocked lane cannot spend the saved turn, but a later
// free slot must restore it ahead of new work.
return nil
}
return it
}
*h = nil
return nil
}
func (s *scheduler) classPosition(r int) uint8 {
for k := uint8(0); k < patternSlots; k++ {
slot := (s.st.ClassSlot + k) % patternSlots
if int(classPattern[slot])-1 == r {
return slot
}
}
return s.st.ClassSlot
}
func (s *scheduler) park(h **CeilingHold, it *ScheduleItem, r int, slot uint8) {
if *h != nil {
return
}
ring := &s.st.Rings[r]
// #nosec G115 -- r indexes the fixed scheduler rings.
*h = &CeilingHold{ID: it.ID, Scope: it.Scope, Ring: uint8(r), Slot: slot, ClassSlot: s.classPosition(r), Turn: ring.Scopes[it.Scope]}
delete(ring.Scopes, it.Scope)
if ring.Held == it.Scope {
ring.Held = ""
}
}
func (s *scheduler) historyHeld(reserved bool) (held, charged bool) {
first, last := ringC1, ringC3
if reserved {
first, last = ringDirect, ringCorroborated
}
for r := first; r <= last; r++ {
if scope := s.st.Rings[r].Held; scope != "" {
if it, _ := s.head(r, scope); it != nil {
held = true
charged = charged || it.CeilingCost > 0
}
}
}
return held, charged
}
// serveHeld spends the suspended head's own fair credits; borrowing never
// changes those credits or the independent history promise in the ring.
func (s *scheduler) serveHeld(h **CeilingHold, it *ScheduleItem, bytes, recovery uint64) (*ScheduleItem, bool) {
hold := *h
for hold.Turn.Deficit < it.Cost || hold.Turn.Bytes < it.Bytes {
if hold.Turn.Deficit < MaxMemberCost {
hold.Turn.Deficit++
}
if hold.Turn.Bytes < it.Bytes {
hold.Turn.Bytes = min(hold.Turn.Bytes+HistoryQuantum, MaxHistoryBytes)
}
}
if uint64(it.Bytes) > bytes || uint64(it.Recovery) > recovery {
return nil, true
}
ring := &s.st.Rings[hold.Ring]
turn := ring.Scopes[it.Scope]
turn.Deficit = min(turn.Deficit+hold.Turn.Deficit-it.Cost, MaxMemberCost)
turn.Bytes = min(turn.Bytes+hold.Turn.Bytes-it.Bytes, MaxHistoryBytes)
// An independent history hold keeps its severity position. Otherwise
// this fulfilled fair turn moves to the next severity as usual.
if ring.Held != it.Scope {
turn.Severity = (hold.Slot + 1) % patternSlots
}
ring.Scopes[it.Scope] = turn
ring.Last = it.Scope
if hold.Ring >= ringDirect {
s.st.NextCorroborated = hold.Ring == ringDirect
} else {
s.st.ClassSlot = (hold.ClassSlot + 1) % patternSlots
}
*h = nil
return it, false
}
// borrow serves only zero-demand work. It uses the same fair rotations and
// finite history, recovery and member bounds as ordinary work.
func (s *scheduler) borrow(items []ScheduleItem, reserved bool, bytes, recovery uint64) (*ScheduleItem, Lane, bool) {
if _, charged := s.historyHeld(reserved); charged {
// Filtering to challenges cannot change the owner of earned bytes.
return nil, 0, true
}
var free []ScheduleItem
for _, it := range items {
if it.CeilingCost == 0 && !s.picked[it.ID] {
free = append(free, it)
}
}
original := s.rings
s.rings = buildRings(free)
defer func() { s.rings = original }()
if reserved {
order := []int{ringDirect, ringCorroborated}
if s.st.NextCorroborated {
order = []int{ringCorroborated, ringDirect}
}
for _, r := range order {
it, blocked := s.serve(r, 0, 0, bytes, recovery)
if blocked {
s.st.NextCorroborated = r == ringCorroborated
return nil, 0, true
}
if it != nil {
s.st.NextCorroborated = r == ringDirect
lane := LaneDirect
if r == ringCorroborated {
lane = LaneCorroborated
}
return it, lane, false
}
}
} else {
for k := uint8(0); k < patternSlots; k++ {
slot := (s.st.ClassSlot + k) % patternSlots
it, blocked := s.serve(int(classPattern[slot])-1, 0, 0, bytes, recovery)
if blocked {
s.st.ClassSlot = slot
return nil, 0, true
}
if it != nil {
s.st.ClassSlot = (slot + 1) % patternSlots
return it, LaneGeneral, false
}
}
}
return nil, 0, false
}
func (s *scheduler) lane(items []ScheduleItem, reserved bool, budget, fullBudget uint32, bytes, recovery uint64) (*ScheduleItem, Lane, bool) {
h := &s.st.GeneralHold
if reserved {
h = &s.st.ReservedHold
}
owed := s.held(h, items)
history, chargedHistory := s.historyHeld(reserved)
if owed != nil && owed.CeilingCost <= budget && !history {
it, blocked := s.serveHeld(h, owed, bytes, recovery)
lane := LaneGeneral
if reserved {
lane = LaneDirect
if owed.Corroborated {
lane = LaneCorroborated
}
}
return it, lane, blocked
}
if budget == 0 && owed == nil && s.freeReady(items, reserved) {
first, last := ringC1, ringC3
if reserved {
first, last = ringDirect, ringCorroborated
}
// A charged history head keeps its earned promise before borrowing.
for r := first; r <= last; r++ {
if it, slot := s.head(r, s.st.Rings[r].Held); it != nil && it.CeilingCost > 0 {
s.park(h, it, r, slot)
owed = it
break
}
}
if owed == nil {
var order []int
if reserved {
order = []int{ringDirect, ringCorroborated}
if s.st.NextCorroborated {
order = []int{ringCorroborated, ringDirect}
}
} else {
for k := uint8(0); k < patternSlots; k++ {
order = append(order, int(classPattern[(s.st.ClassSlot+k)%patternSlots])-1)
}
}
for _, r := range order {
var scopes []string
for scope := range s.rings[r].ready {
scopes = append(scopes, scope)
}
sort.Strings(scopes)
start := sort.SearchStrings(scopes, s.st.Rings[r].Last)
if start < len(scopes) && scopes[start] == s.st.Rings[r].Last {
start++
}
for visit := 0; visit < len(scopes); visit++ {
scope := scopes[(start+visit)%len(scopes)]
if it, slot := s.head(r, scope); it != nil && it.CeilingCost > 0 {
s.park(h, it, r, slot)
owed = it
break
}
}
if owed != nil {
break
}
}
}
}
// A ceiling promise stops other charged work taking its remainder.
// A head above this call's entire budget does not stop affordable peers.
if budget == 0 || (owed != nil && owed.CeilingCost <= fullBudget && !chargedHistory) {
return s.borrow(items, reserved, bytes, recovery)
}
var order []int
if reserved {
order = []int{ringDirect, ringCorroborated}
if s.st.NextCorroborated {
order = []int{ringCorroborated, ringDirect}
}
} else {
for k := uint8(0); k < patternSlots; k++ {
order = append(order, int(classPattern[(s.st.ClassSlot+k)%patternSlots])-1)
}
}
original := s.rings
if owed != nil {
var rest []ScheduleItem
for _, it := range items {
if it.ID != owed.ID {
rest = append(rest, it)
}
}
s.rings = buildRings(rest)
defer func() { s.rings = original }()
}
for k, r := range order {
it, blocked := s.serve(r, budget, fullBudget, bytes, recovery)
if blocked {
if reserved {
s.st.NextCorroborated = r == ringCorroborated
} else {
s.st.ClassSlot = (s.st.ClassSlot + uint8(k)) % patternSlots
}
if it == nil {
return nil, 0, true
}
if !s.freeReady(items, reserved) {
return nil, 0, true
}
_, slot := s.head(r, it.Scope)
s.park(h, it, r, slot)
return s.borrow(items, reserved, bytes, recovery)
}
if it != nil {
lane := LaneGeneral
if reserved {
lane = LaneDirect
if r == ringCorroborated {
lane = LaneCorroborated
}
s.st.NextCorroborated = r == ringDirect
} else {
s.st.ClassSlot = (s.st.ClassSlot + uint8(k) + 1) % patternSlots
}
return it, lane, false
}
}
return s.borrow(items, reserved, bytes, recovery)
}
package admission
import "fmt"
// State is where a candidate is in its lifecycle. Queued work either becomes
// an attempt (reserved, executing, then an outcome) or ends before one
// (refused, withheld, dropped). Values are persisted: append, never renumber.
type State uint8
const (
StateQueued State = iota + 1
StateReserved
StateExecuting
StateVerified
StateFailed
StateUnknown
StateRefused
StateWithheld
StateDropped
// StateObserved ends an attempt that was reserved as live work would
// be and never ran: an observe preview, never an applied outcome.
StateObserved
// StateDryRun ends queued work that dry-run mode forecast instead of
// running. It makes no attempt and charges nothing; attempts that
// failed before it keep their records.
StateDryRun
stateEnd
)
var stateNames = [...]string{"", "queued", "reserved", "executing", "verified", "failed", "unknown", "refused", "withheld", "dropped", "observed", "dry_run"}
func (s State) Valid() bool { return s >= StateQueued && s < stateEnd }
func (s State) String() string {
if s.Valid() {
return stateNames[s]
}
return fmt.Sprintf("state(%d)", uint8(s))
}
// Terminal reports whether s ends the candidate. A verified effect can later
// expire or be evicted through linked events, but the candidate never runs
// again; a new root set needs a new generation.
func (s State) Terminal() bool { return s >= StateVerified && s < stateEnd }
// CanTransition reports whether the lifecycle allows from -> to. Queued to
// queued records a changed deferral reason. A proven failure with attempts
// left returns the candidate to the queue; an unknown outcome never does.
// Only a reservation can end as a preview: an attempt that ran may have
// applied something. A dry run ends only queued work, before any attempt
// of its own.
func CanTransition(from, to State) bool {
switch from {
case StateQueued:
return to == StateQueued || to == StateReserved || to == StateRefused || to == StateWithheld || to == StateDropped || to == StateDryRun
case StateReserved:
return to == StateExecuting || to == StateFailed || to == StateQueued || to == StateObserved
case StateExecuting:
return to == StateVerified || to == StateFailed || to == StateUnknown || to == StateQueued
}
return false
}
// terminalDisposition reports whether a record in state s may carry d. Only
// terminal states carry a disposition; a queued deferral is a reason alone.
func terminalDisposition(s State, d Disposition) bool {
switch s {
case StateQueued, StateReserved, StateExecuting:
return d == 0
case StateVerified:
return d == DispositionApplied || d == DispositionNarrowed
case StateFailed:
return d == DispositionFailed
case StateUnknown:
return d == DispositionUnknown
case StateRefused:
return d == DispositionRefused
case StateWithheld:
return d == DispositionWithheld
case StateDropped:
return d == DispositionDropped
case StateObserved:
return d == DispositionObserve
case StateDryRun:
return d == DispositionDryRun
}
return false
}
// stateReason reports whether a record in state s may carry reason r. A
// queued candidate may wait for a deferral reason; a pre-attempt ending
// needs a reason of its own group; outcomes carry none.
func stateReason(s State, r Reason) bool {
switch s {
case StateQueued:
return r == 0 || r.Disposition() == DispositionDeferred
case StateRefused, StateWithheld, StateDropped:
return r.Valid() && terminalDisposition(s, r.Disposition())
}
return r == 0
}
package admission
import (
"fmt"
"strings"
"time"
)
// MaxRecoveryNeed is the most room one reservation needs in the recovery
// and outbox reserve: its candidate's largest history, its attempt's audit
// rows and its intent.
const MaxRecoveryNeed = MaxHistoryBytes + AttemptRecoveryBytes
// LedgerStatus is a read-only view of the admission ledger. Each section is read on its own and carries its own error,
// so one damaged record never hides the others. Times are the ledger's
// stored admission time, never a current reading.
type LedgerStatus struct {
Clock ClockStatus `json:"clock"`
Queue QueueStatus `json:"queue"`
Counters CountersStatus `json:"counters"`
Outcomes OutcomesStatus `json:"outcomes"`
Ingress IngressSection `json:"ingress"`
Ceiling CeilingStatus `json:"ceiling"`
Storage StorageStatus `json:"storage"`
Outbox OutboxStatus `json:"outbox"`
Notices NoticesStatus `json:"notices"`
// Unresolved is the attempts whose outcome is unresolved.
Unresolved UnresolvedStatus `json:"unresolved"`
Effects EffectsStatus `json:"effects"`
}
// EffectsStatus is the temporary set slots the ledger's automatic blocks
// hold, pending and verified together, by partition, and when the first
// of them expires.
type EffectsStatus struct {
General int `json:"general"`
Reserved int `json:"reserved"`
Next time.Time `json:"next,omitempty"`
Error string `json:"error,omitempty"`
}
// ClockStatus is the stored high-water mark; zero before the first
// reading.
type ClockStatus struct {
Now time.Time `json:"now"`
Error string `json:"error,omitempty"`
}
// QueueStatus counts live candidates and their positions.
type QueueStatus struct {
Queued int `json:"queued"`
Reserved int `json:"reserved"`
Executing int `json:"executing"`
// Retrying counts queued candidates waiting out a retry backoff.
Retrying int `json:"retrying"`
// Oldest is the first queued time of the oldest queued candidate.
Oldest time.Time `json:"oldest,omitempty"`
Occupancy []QueueOccupancy `json:"occupancy,omitempty"`
Error string `json:"error,omitempty"`
}
// QueueOccupancy is the live candidates of one kind in one partition.
type QueueOccupancy struct {
Kind string `json:"kind"`
Partition string `json:"partition"`
Count int `json:"count"`
}
// CountRow is one lifetime queue counter.
type CountRow struct {
Event string `json:"event"`
Reason string `json:"reason"`
Class string `json:"class,omitempty"`
Severity string `json:"severity,omitempty"`
N uint64 `json:"n"`
}
// CountersStatus is the lifetime queue counters.
type CountersStatus struct {
Rows []CountRow `json:"rows,omitempty"`
Error string `json:"error,omitempty"`
}
// OutcomeStatusRow is one windowed outcome count.
type OutcomeStatusRow struct {
Event string `json:"event,omitempty"`
Reason string `json:"reason,omitempty"`
Outcome string `json:"outcome,omitempty"`
Class string `json:"class,omitempty"`
Severity string `json:"severity,omitempty"`
N uint64 `json:"n"`
}
// OutcomesStatus is the outcome counts of the last hour, day and 30 days,
// each to the resolution of its buckets.
type OutcomesStatus struct {
Hour []OutcomeStatusRow `json:"hour,omitempty"`
Day []OutcomeStatusRow `json:"day,omitempty"`
Month []OutcomeStatusRow `json:"month,omitempty"`
Error string `json:"error,omitempty"`
}
// IngressSection is the ledger's record of ingress generations.
type IngressSection struct {
Generation uint64 `json:"generation"`
Open bool `json:"open"`
Persisted uint64 `json:"persisted"`
Interrupted uint64 `json:"interrupted"`
// Resumed is the generation that began after the latest interruption;
// cleared when a clean generation follows.
Resumed uint64 `json:"resumed,omitempty"`
Error string `json:"error,omitempty"`
}
// LaneStatus is one ceiling allowance.
type LaneStatus struct {
Size uint32 `json:"size"`
Used uint32 `json:"used"`
Credit uint32 `json:"credit"`
// Next is how long until the lane can charge one more unit; zero when
// it can now.
Next time.Duration `json:"next"`
}
// CeilingStatus is the hourly ceiling.
type CeilingStatus struct {
Limit uint32 `json:"limit"`
General LaneStatus `json:"general"`
Reserved LaneStatus `json:"reserved"`
Error string `json:"error,omitempty"`
}
// AllowanceStatus is one history allowance.
type AllowanceStatus struct {
Size uint64 `json:"size"`
Used uint64 `json:"used"`
Credit uint64 `json:"credit"`
Room uint64 `json:"room"`
}
// StorageStatus is the history budget and the recovery reserve.
type StorageStatus struct {
General AllowanceStatus `json:"general"`
Reserved AllowanceStatus `json:"reserved"`
// Pinned is the history of unresolved outcomes.
Pinned uint64 `json:"pinned"`
// Outstanding is what reserved and executing attempts hold.
Outstanding uint64 `json:"outstanding"`
// RecoveryRoom is what the reserve has left after pinned history,
// outstanding attempts and the outbox.
RecoveryRoom uint64 `json:"recovery_room"`
Ended uint32 `json:"ended"`
Loose uint32 `json:"loose"`
// ReviewHorizon is when the oldest retained ended history ended: how
// far back review reaches.
ReviewHorizon time.Time `json:"review_horizon,omitempty"`
Error string `json:"error,omitempty"`
}
// UnresolvedStatus counts the attempts whose outcome is unresolved: those
// running and those that ended unknown and are not yet retired, with the
// earliest expiry after which one of the unknown ones can be retired.
type UnresolvedStatus struct {
Running int `json:"running"`
Unknown int `json:"unknown"`
Retires time.Time `json:"retires,omitempty"`
Error string `json:"error,omitempty"`
}
// OutboxStatus is what the outbox holds: unacknowledged audit rows and
// notice records, each at its slot. The slots outstanding attempts hold
// for rows they have not written count in the storage section's room.
type OutboxStatus struct {
AuditRows uint64 `json:"audit_rows"`
AuditBytes uint64 `json:"audit_bytes"`
NoticeRecords uint64 `json:"notice_records"`
NoticeBytes uint64 `json:"notice_bytes"`
Error string `json:"error,omitempty"`
}
// NoticeStatusRow is one notice record.
type NoticeStatusRow struct {
Kind string `json:"kind"`
Reason string `json:"reason,omitempty"`
Outcome string `json:"outcome,omitempty"`
Check string `json:"check,omitempty"`
Effect string `json:"effect,omitempty"`
Count uint64 `json:"count"`
Unsent uint64 `json:"unsent"`
Last time.Time `json:"last,omitempty"`
}
// NoticesStatus is the notice records and the latest Critical gap.
type NoticesStatus struct {
Records []NoticeStatusRow `json:"records,omitempty"`
// LastCriticalGap is the latest Critical response gap any record saw.
LastCriticalGap time.Time `json:"last_critical_gap,omitempty"`
Error string `json:"error,omitempty"`
}
// tierNames renders a tier; empty for an unassessed one.
func tierNames(c Class, s Severity) (string, string) {
if c == 0 && s == 0 {
return "", ""
}
return c.String(), s.String()
}
// CountRows renders queue counters for status.
func CountRows(rows []QueueCount) []CountRow {
out := make([]CountRow, 0, len(rows))
for _, r := range rows {
class, sev := tierNames(r.Key.Class, r.Key.Severity)
out = append(out, CountRow{Event: r.Key.Event.String(), Reason: r.Key.Reason.String(), Class: class, Severity: sev, N: r.N})
}
return out
}
// OutcomeRows renders outcome counts for status.
func OutcomeRows(c OutcomeCounts) []OutcomeStatusRow {
rows := c.Rows()
out := make([]OutcomeStatusRow, 0, len(rows))
for _, r := range rows {
row := OutcomeStatusRow{N: r.N}
row.Class, row.Severity = tierNames(r.Key.Class, r.Key.Severity)
if r.Key.Event != 0 {
row.Event, row.Reason = r.Key.Event.String(), r.Key.Reason.String()
} else {
row.Outcome = r.Key.Outcome.String()
}
out = append(out, row)
}
return out
}
// NoticeRow renders a notice record for status.
func NoticeRow(r NoticeRecord) NoticeStatusRow {
row := NoticeStatusRow{Kind: r.Key.Kind.String(), Check: r.Key.Check, Count: r.Count, Unsent: r.Unsent(), Last: r.Last}
if r.Key.Reason != 0 {
row.Reason = r.Key.Reason.String()
}
if r.Key.Outcome != 0 {
row.Outcome = r.Key.Outcome.String()
}
if r.Key.Effect != 0 {
row.Effect = r.Key.Effect.String()
}
return row
}
// IngressHealth is the in-memory ingress's admission state.
// It is process-local: the ledger may be the damaged part.
type IngressHealth struct {
Admitting bool `json:"admitting"`
// StoppedSince is when the ingress stopped admitting; zero while it
// admits.
StoppedSince time.Time `json:"stopped_since,omitempty"`
// CriticalRefused counts Critical arrivals refused since it stopped.
CriticalRefused uint64 `json:"critical_refused"`
}
// Doctor statuses, as `csm doctor` prints them.
const (
DoctorOK = "ok"
DoctorWarn = "warn"
DoctorFail = "fail"
)
// DoctorRow is one doctor check.
type DoctorRow struct {
Name string `json:"name"`
Status string `json:"status"`
Message string `json:"message,omitempty"`
Fix string `json:"fix,omitempty"`
}
// PreviewUnaffected labels preview rows: legacy blocking enforces
// while admission previews, so a healthy row says so and an outage only
// warns.
const PreviewUnaffected = "existing blocking is unaffected while admission previews"
// ExecutionAffected ends every admission row once admission runs automatic
// blocks: an admission outage is then an outage of automatic blocking, so
// an unavailable ledger, ingress, owner or clock fails.
const ExecutionAffected = "admission decides automatic blocks"
// UnresolvedOutcomesRow names the doctor row of unknown outcomes; doctor
// replaces its retire time with the cause when retirement cannot run.
const UnresolvedOutcomesRow = "admission unresolved outcomes"
// DoctorImpact is how admission rows close and how badly an outage rates:
// a preview only warns, since legacy blocking enforces; execution
// fails.
func DoctorImpact(executing bool) (closing, outage string) {
if executing {
return ExecutionAffected, DoctorFail
}
return PreviewUnaffected, DoctorWarn
}
// DoctorChecks turns a ledger status and the ingress health into fixed
// doctor rows. Either may be nil: then its rows are left out,
// so a daemon without an admission owner prints none. now is the time the
// Critical-gap rule looks back from; executing says whether admission runs
// automatic blocks (DoctorImpact).
func DoctorChecks(s *LedgerStatus, in *IngressHealth, now time.Time, executing bool) []DoctorRow {
closing, outage := DoctorImpact(executing)
var rows []DoctorRow
if s != nil {
rows = append(rows, ledgerRow(s, executing))
}
if in != nil {
rows = append(rows, ingressRow(s, in, executing))
}
if s == nil {
return rows
}
gap := DoctorRow{Name: "admission response gaps", Status: DoctorOK, Message: closing}
if last := s.Notices.LastCriticalGap; !last.IsZero() && now.Before(last.Add(time.Hour)) {
gap.Status = outage
gap.Message = "a Critical finding would not have received its response at " + last.UTC().Format(time.RFC3339) + "; " + closing
if executing {
gap.Message = "a Critical finding did not receive its response at " + last.UTC().Format(time.RFC3339) + "; " + closing
}
gap.Fix = "inspect the auto_response_withheld and response_capacity_exhausted notices for the reason"
}
reserve := DoctorRow{Name: "admission recovery reserve", Status: DoctorOK, Message: closing}
if s.Storage.Pinned > 0 || s.Storage.RecoveryRoom < MaxRecoveryNeed {
reserve.Status = DoctorWarn
reserve.Message = fmt.Sprintf("%d bytes of unresolved outcomes are pinned; %d bytes of room remain", s.Storage.Pinned, s.Storage.RecoveryRoom) + "; " + closing
reserve.Fix = "resolve unknown outcomes and confirm the audit consumer acknowledges rows; new reservations wait while the reserve is full"
}
outbox := DoctorRow{Name: "admission outbox", Status: DoctorOK, Message: closing}
if used := s.Outbox.AuditBytes + s.Outbox.NoticeBytes; used > RecoveryReserveBytes/2 {
outbox.Status = DoctorWarn
outbox.Message = fmt.Sprintf("the outbox holds %d bytes, over half the reserve", used) + "; " + closing
outbox.Fix = "confirm audit and notice delivery is running and acknowledging"
}
rows = append(rows, gap, reserve, outbox)
if u := s.Unresolved; u.Unknown > 0 {
rows = append(rows, DoctorRow{
Name: UnresolvedOutcomesRow, Status: DoctorWarn,
Message: fmt.Sprintf("%d automatic blocks have an unproven outcome; each holds its address until it has expired and the firewall shows nothing of it; the next can retire at %s", u.Unknown, u.Retires.UTC().Format(time.RFC3339)) + "; " + closing,
Fix: "none needed while the count falls; if it keeps growing, check the firewall rows and the admission execution row",
})
}
return rows
}
func ledgerRow(s *LedgerStatus, executing bool) DoctorRow {
closing, outage := DoctorImpact(executing)
var damaged []string
for _, section := range []struct{ name, err string }{
{"clock", s.Clock.Error}, {"queue", s.Queue.Error}, {"counters", s.Counters.Error},
{"outcomes", s.Outcomes.Error}, {"ingress", s.Ingress.Error}, {"ceiling", s.Ceiling.Error},
{"storage", s.Storage.Error}, {"outbox", s.Outbox.Error}, {"notices", s.Notices.Error},
{"unresolved", s.Unresolved.Error}, {"effects", s.Effects.Error},
} {
if section.err != "" {
damaged = append(damaged, section.name+": "+section.err)
}
}
if len(damaged) == 0 {
return DoctorRow{Name: "admission ledger", Status: DoctorOK, Message: closing}
}
fix := "previews are refused while the ledger is damaged; restart csm.service so opening the ledger proves every record, and keep the state database, which existing blocking still uses"
if executing {
fix = "automatic blocks are refused while the ledger is damaged; restart csm.service so opening the ledger proves every record, and keep the state database"
}
return DoctorRow{Name: "admission ledger", Status: outage, Message: strings.Join(damaged, "; ") + "; " + closing, Fix: fix}
}
func ingressRow(s *LedgerStatus, in *IngressHealth, executing bool) DoctorRow {
closing, outage := DoctorImpact(executing)
switch {
case !in.Admitting:
return DoctorRow{
Name: "admission ingress", Status: outage,
Message: fmt.Sprintf("refusing every automatic response since %s; %d Critical arrivals refused; %s", in.StoppedSince.UTC().Format(time.RFC3339), in.CriticalRefused, closing),
Fix: "the ledger owner has no usable queue snapshot; inspect the daemon log and the admission ledger row",
}
case s != nil && s.Ingress.Resumed != 0 && s.Ingress.Resumed == s.Ingress.Generation:
return DoctorRow{
Name: "admission ingress", Status: DoctorWarn,
Message: "the previous ingress generation was interrupted; its held arrivals were lost and counts are lower bounds; " + closing,
Fix: "stop csm.service cleanly next time; the warning clears after a clean restart",
}
}
return DoctorRow{Name: "admission ingress", Status: DoctorOK, Message: closing}
}
package admission
import (
"math"
"time"
)
// Storage budgets, internal constants rather than settings.
const (
// HistoryBytes is the terminal-history budget: the details the ledger
// keeps about admitted candidates. A fifth, rounded up, is reserved for
// direct compromise and corroborated work.
HistoryBytes = 256 << 20
// RecoveryReserveBytes holds the details of unresolved outcomes apart
// from history, until recovery settles them.
RecoveryReserveBytes = 64 << 20
// HistoryBurst is the most pacing credit an allowance saves: ten
// minutes of its rate.
HistoryBurst = 10 * time.Minute
// HistoryQuantum is the history bytes a scope earns per scheduler turn,
// so a scope of large records cannot take more than its share.
HistoryQuantum = 4 << 10
// MaxEndedCandidates bounds the candidates kept after ending before
// any attempt. They are not history: only the newest are kept.
MaxEndedCandidates = QueueCapacity
// MaxLooseEvidence bounds the evidence kept that no stored candidate
// names. The oldest leaves first.
MaxLooseEvidence = QueueCapacity
)
// Stored IDs are fixed-length derived strings.
const (
actionIDLen = 4 + 32 // "act_" and 32 hex digits, untyped so byte bounds mix freely
evidenceIDLen = len("ev_") + 32
)
// byteTicks is one byte of credit. An allowance admitting S bytes per
// second gains S ticks per nanosecond, so credit accrues without rounding.
const byteTicks = uint64(time.Second)
// HistoryLanes splits the history budget into the general allowance and
// the reserved one, ceil(HistoryBytes/5), as the ceiling splits its limit.
func HistoryLanes() (general, reserved uint64) {
reserved = (HistoryBytes + 4) / 5
return HistoryBytes - reserved, reserved
}
// HistoryRate is the bytes per second an allowance of size bytes admits:
// its size over the retention, so a week of admissions fits the budget
// however an attacker times them.
func HistoryRate(size uint64) uint64 { return size / uint64(HistoryRetention/time.Second) }
// HistoryCap is the most credit an allowance of size bytes saves.
func HistoryCap(size uint64) uint64 { return HistoryRate(size) * uint64(HistoryBurst/time.Second) }
// HistoryCost is the most ledger bytes an admitted candidate can retain,
// given the encoded size of each root's evidence in root order: its record
// at its widest, every attempt it may make, its history entry and index
// keys, the episode row that may name it, and each root's evidence with a
// full set of report links and its reference count. Every record counts
// with its key. Evidence several candidates share is counted for each of
// them, and so is an episode row.
func HistoryCost(c Candidate, evidence []int) (uint32, error) {
if len(evidence) != len(c.Roots) {
return 0, refuse(ReasonInvalid, "history cost needs the size of every root")
}
record, err := c.MaxBytes()
if err != nil {
return 0, err
}
n := 2*candidateIDLen + record + MaxHistoryEntryBytes + 3*HistoryIndexKeyLen + MaxAttempts*(actionIDLen+MaxAttemptBytes) + MaxEpisodeBytes
for _, size := range evidence {
if size <= 0 || size > MaxEvidenceBytes {
return 0, refuse(ReasonInvalid, "evidence size is out of range")
}
n += 3*evidenceIDLen + size + MaxReportLinksBytes + MaxEvidenceRefsBytes
}
if n <= 0 || n > MaxHistoryBytes {
return 0, refuse(ReasonInvalid, "history cost is outside any candidate's range")
}
return uint32(n), nil
}
// HistoryMeter is one allowance of the history budget.
type HistoryMeter struct {
// Credit is the saved pacing credit in ticks.
Credit uint64
// Used is the history bytes the allowance holds.
Used uint64
}
// Bytes is the whole bytes of saved credit.
func (m HistoryMeter) Bytes() uint64 { return m.Credit / byteTicks }
// RingState counts the entries of one bounded ring. Positions start at 1 and
// only grow, so the oldest entry has the smallest.
type RingState struct {
Count uint32
// Last is the position the newest entry took; zero before the first.
Last uint64
}
// Push takes the next position.
func (r RingState) Push() (RingState, uint64) {
r.Count++
r.Last++
return r, r.Last
}
// Remove drops one entry.
func (r RingState) Remove() (RingState, error) {
if r.Count == 0 {
return r, ErrCorruptRecord
}
r.Count--
return r, nil
}
// StorageState is the ledger's record of its storage budgets:
// each history allowance's pacing credit and retained bytes, the recovery
// reserve and the two bounded rings.
type StorageState struct {
General, Reserved HistoryMeter
// Recovery is the history bytes and intents of unresolved outcomes.
Recovery uint64
// Ended holds candidates that ended before any attempt; Loose holds
// evidence no stored candidate names.
Ended, Loose RingState
// AuditSlots is the audit row slots the outbox holds: rows not yet
// acknowledged and the rows outstanding attempts may still write.
AuditSlots uint64
// NoticeRecords is the notice records the outbox holds, the fixed
// ones included.
NoticeRecords uint64
}
// Outbox bounds: the notice share holds MaxNoticeRecords records, and the
// reserve holds no more than MaxAuditSlots rows.
const (
MaxNoticeRecords = uint64(NoticeBytes / NoticeSlotBytes)
MaxAuditSlots = uint64(RecoveryReserveBytes / AuditSlotBytes)
)
// HoldAudit holds n more audit slots.
func (s StorageState) HoldAudit(n uint64) (StorageState, error) {
if s.AuditSlots > MaxAuditSlots || n > MaxAuditSlots-s.AuditSlots {
return s, ErrCorruptRecord
}
s.AuditSlots += n
return s, nil
}
// ReleaseAudit returns n audit slots.
func (s StorageState) ReleaseAudit(n uint64) (StorageState, error) {
if s.AuditSlots > MaxAuditSlots || n > s.AuditSlots {
return s, ErrCorruptRecord
}
s.AuditSlots -= n
return s, nil
}
// AddNotice counts one more notice record, or reports false when the
// notice share is full.
func (s StorageState) AddNotice() (StorageState, bool) {
if s.NoticeRecords >= MaxNoticeRecords {
return s, false
}
s.NoticeRecords++
return s, true
}
// RemoveNotice counts one keyed record fewer; the fixed ones stay.
func (s StorageState) RemoveNotice() (StorageState, error) {
if s.NoticeRecords <= FixedNotices {
return s, ErrCorruptRecord
}
s.NoticeRecords--
return s, nil
}
// OutboxBytes is what the outbox holds of the recovery and outbox reserve.
func (s StorageState) OutboxBytes() uint64 {
return s.AuditSlots*AuditSlotBytes + s.NoticeRecords*NoticeSlotBytes
}
type historyAllowance struct {
m *HistoryMeter
size uint64
}
func (s *StorageState) allowances() [2]historyAllowance {
g, r := HistoryLanes()
return [2]historyAllowance{{&s.General, g}, {&s.Reserved, r}}
}
func (s *StorageState) allowance(l Lane) historyAllowance {
if l == LaneGeneral {
return s.allowances()[0]
}
return s.allowances()[1]
}
// NewStorageState is a new ledger's state: each allowance saves its full
// credit once, and the outbox holds the fixed notice records. An upgraded
// ledger starts from the zero value, without credit, since its recent
// spend is unknown.
func NewStorageState() StorageState {
s := StorageState{NoticeRecords: FixedNotices}
for _, a := range s.allowances() {
a.m.Credit = HistoryCap(a.size) * byteTicks
}
return s
}
// HistoryBudget is the history bytes lane l can charge now: its saved
// credit, bounded by the room its allowance has once retirable bytes of
// ended history are retired.
func (s StorageState) HistoryBudget(l Lane, retirable uint64) uint64 {
if !l.Valid() {
return 0
}
a := s.allowance(l)
used := a.m.Used - min(retirable, a.m.Used)
if used >= a.size {
return 0
}
return min(a.m.Bytes(), a.size-used)
}
// HistoryCredit is the whole bytes of credit lane l's allowance saves.
func (s StorageState) HistoryCredit(l Lane) uint64 {
if !l.Valid() {
return 0
}
return s.allowance(l).m.Bytes()
}
// HistoryRoom is the bytes lane l's allowance has left.
func (s StorageState) HistoryRoom(l Lane) uint64 {
if !l.Valid() {
return 0
}
a := s.allowance(l)
if a.m.Used >= a.size {
return 0
}
return a.size - a.m.Used
}
// UntilHistory is how long lane l's allowance waits for its saved credit
// to grow by one quantum, or to its cap if that is nearer. False when the
// credit is full: then only retiring history can make room.
func (s StorageState) UntilHistory(l Lane) (time.Duration, bool) {
if !l.Valid() {
return 0, false
}
a := s.allowance(l)
full := HistoryCap(a.size) * byteTicks
if a.m.Credit >= full {
return 0, false
}
need := min(full-a.m.Credit, uint64(HistoryQuantum)*byteTicks)
n, rate := int64(need), int64(min(HistoryRate(a.size), HistoryBytes))
return time.Duration((n + rate - 1) / rate), true
}
// Advance credits elapsed admission time, refilling each allowance at its
// rate up to its cap. A long gap saturates instead of overflowing.
func (s StorageState) Advance(elapsed time.Duration) (StorageState, error) {
if elapsed <= 0 {
return s, nil
}
e := uint64(elapsed)
for _, a := range s.allowances() {
full, rate := HistoryCap(a.size)*byteTicks, HistoryRate(a.size)
if a.m.Credit >= full {
continue
}
if need := full - a.m.Credit; e >= (need+rate-1)/rate {
a.m.Credit = full
} else {
a.m.Credit += rate * e
}
}
return s, nil
}
// ChargeHistory spends bytes of lane l's credit and allowance. The ledger
// retires ended history first when the allowance needs room. A refusal
// changes nothing.
func (s StorageState) ChargeHistory(l Lane, bytes uint32) (StorageState, error) {
if !l.Valid() || bytes == 0 || bytes > MaxHistoryBytes {
return s, refuse(ReasonInvalid, "history charge has no lane or an invalid size")
}
if uint64(bytes) > s.HistoryBudget(l, 0) {
return s, refuse(ReasonStorageShare, "lane has no history budget for the charge")
}
a := s.allowance(l)
a.m.Credit -= uint64(bytes) * byteTicks
a.m.Used += uint64(bytes)
return s, nil
}
// ReleaseHistory returns retired history bytes to each allowance. It never
// refunds credit.
func (s StorageState) ReleaseHistory(general, reserved uint32) (StorageState, error) {
if s.General.Used < uint64(general) || s.Reserved.Used < uint64(reserved) {
return s, ErrCorruptRecord
}
s.General.Used -= uint64(general)
s.Reserved.Used -= uint64(reserved)
return s, nil
}
// PinHistory moves an unresolved outcome's history bytes from its
// allowances to the recovery reserve.
func (s StorageState) PinHistory(general, reserved uint32) (StorageState, error) {
charged := uint64(general) + uint64(reserved)
if charged > math.MaxUint64-s.Recovery {
return s, ErrCorruptRecord
}
next, err := s.ReleaseHistory(general, reserved)
if err != nil {
return s, err
}
next.Recovery += charged
return next, nil
}
// HoldIntent holds an unresolved outcome's intent in the recovery reserve.
func (s StorageState) HoldIntent() (StorageState, error) {
if s.Recovery > math.MaxUint64-MaxIntentBytes {
return s, ErrCorruptRecord
}
s.Recovery += MaxIntentBytes
return s, nil
}
// UnpinHistory returns a retired outcome's history bytes from the recovery
// reserve to its allowances and releases its intent: the inverse of
// PinHistory and HoldIntent.
func (s StorageState) UnpinHistory(general, reserved uint32) (StorageState, error) {
held := uint64(general) + uint64(reserved) + MaxIntentBytes
if s.Recovery < held || s.General.Used > math.MaxUint64-uint64(general) || s.Reserved.Used > math.MaxUint64-uint64(reserved) {
return s, ErrCorruptRecord
}
s.Recovery -= held
s.General.Used += uint64(general)
s.Reserved.Used += uint64(reserved)
return s, nil
}
// CheckRecovery refuses a reservation whose candidate, if its outcome were
// unresolved, would not fit the recovery reserve: new work waits while
// unresolved outcomes fill it.
func (s StorageState) CheckRecovery(bytes uint32) error {
if s.Recovery > RecoveryReserveBytes || uint64(bytes) > RecoveryReserveBytes-s.Recovery {
return refuse(ReasonPendingRecovery, "unresolved outcomes fill the recovery reserve")
}
return nil
}
const storageStateVersion = 1
type storageStateRecord struct {
V uint8 `json:"v"`
GeneralCredit uint64 `json:"general_credit,omitempty"`
GeneralUsed uint64 `json:"general_used,omitempty"`
ReservedCredit uint64 `json:"reserved_credit,omitempty"`
ReservedUsed uint64 `json:"reserved_used,omitempty"`
Recovery uint64 `json:"recovery,omitempty"`
EndedCount uint32 `json:"ended_count,omitempty"`
EndedLast uint64 `json:"ended_last,omitempty"`
LooseCount uint32 `json:"loose_count,omitempty"`
LooseLast uint64 `json:"loose_last,omitempty"`
AuditSlots uint64 `json:"audit_slots,omitempty"`
NoticeRecords uint64 `json:"notice_records,omitempty"`
}
func (s StorageState) record() (storageStateRecord, error) {
bad := func(detail string) (storageStateRecord, error) {
return storageStateRecord{}, refuse(ReasonInvalid, detail)
}
for _, a := range s.allowances() {
if a.m.Credit > HistoryCap(a.size)*byteTicks {
return bad("history credit exceeds its cap")
}
}
for _, r := range []struct {
ring RingState
bound uint32
}{{s.Ended, MaxEndedCandidates}, {s.Loose, MaxLooseEvidence}} {
if r.ring.Count > r.bound || uint64(r.ring.Count) > r.ring.Last {
return bad("ring holds more entries than its bound or its positions")
}
}
if s.NoticeRecords > MaxNoticeRecords || s.AuditSlots > MaxAuditSlots {
return bad("outbox holds more than its bounds")
}
return storageStateRecord{
V: storageStateVersion, GeneralCredit: s.General.Credit, GeneralUsed: s.General.Used,
ReservedCredit: s.Reserved.Credit, ReservedUsed: s.Reserved.Used, Recovery: s.Recovery,
EndedCount: s.Ended.Count, EndedLast: s.Ended.Last, LooseCount: s.Loose.Count, LooseLast: s.Loose.Last,
AuditSlots: s.AuditSlots, NoticeRecords: s.NoticeRecords,
}, nil
}
// Validate checks the state's invariants.
func (s StorageState) Validate() error {
_, err := s.record()
return err
}
func (s StorageState) MarshalBinary() ([]byte, error) {
rec, err := s.record()
if err != nil {
return nil, err
}
return sealRecord(rec)
}
// UnmarshalStorageState decodes a stored storage state and checks its
// invariants.
func UnmarshalStorageState(data []byte) (StorageState, error) {
var rec storageStateRecord
if err := openRecord(data, &rec); err != nil {
return StorageState{}, err
}
s := StorageState{
General: HistoryMeter{Credit: rec.GeneralCredit, Used: rec.GeneralUsed},
Reserved: HistoryMeter{Credit: rec.ReservedCredit, Used: rec.ReservedUsed},
Recovery: rec.Recovery,
Ended: RingState{Count: rec.EndedCount, Last: rec.EndedLast},
Loose: RingState{Count: rec.LooseCount, Last: rec.LooseLast},
AuditSlots: rec.AuditSlots, NoticeRecords: rec.NoticeRecords,
}
if again, err := s.record(); rec.V != storageStateVersion || err != nil || again != rec {
return StorageState{}, ErrCorruptRecord
}
return s, nil
}
// UntilHistoryCost waits only for the remaining ticks of the selected
// head's charge. A quantum timer alone can sleep past a smaller ready head.
func (s StorageState) UntilHistoryCost(l Lane, cost uint32) (time.Duration, bool) {
if !l.Valid() || cost > MaxHistoryBytes {
return 0, false
}
a := s.allowance(l)
need := uint64(cost) * byteTicks
if a.m.Credit >= need {
return 0, false
}
n := int64(min(need-a.m.Credit, uint64(MaxHistoryBytes)*byteTicks))
rate := int64(min(HistoryRate(a.size), HistoryBytes))
return time.Duration((n + rate - 1) / rate), true
}
package admission
import (
"net/netip"
"strconv"
"strings"
)
// Proto is the transport of a service target.
type Proto uint8
const (
ProtoTCP Proto = iota + 1
ProtoUDP
)
func (p Proto) String() string {
switch p {
case ProtoTCP:
return "tcp"
case ProtoUDP:
return "udp"
}
return "proto(" + strconv.Itoa(int(p)) + ")"
}
// ParseProto accepts exactly "tcp" or "udp".
func ParseProto(s string) (Proto, error) {
switch s {
case "tcp":
return ProtoTCP, nil
case "udp":
return ProtoUDP, nil
}
return 0, refuse(ReasonInvalid, "unknown transport")
}
// Service is a (transport, port) tuple. The zero value means no service.
type Service struct {
Proto Proto
Port uint16
}
func (s Service) IsZero() bool { return s == Service{} }
// Caps is the firewall's address capability at canonicalization time.
type Caps struct {
IPv6 bool
}
// Target is a canonical response target: one address, one prefix, or one
// service on one address. Construct it only through CanonicalAddress,
// CanonicalPrefix, CanonicalService or ParseTargetKey; the zero value is no
// target.
type Target struct {
prefix netip.Prefix
service Service
}
// protectedPrefixes are never a target or part of one: unspecified,
// loopback, link-local, multicast and the limited broadcast address. The
// firewall engine refuses its own interface and infra addresses separately.
var protectedPrefixes = []netip.Prefix{
netip.MustParsePrefix("0.0.0.0/32"),
netip.MustParsePrefix("127.0.0.0/8"),
netip.MustParsePrefix("169.254.0.0/16"),
netip.MustParsePrefix("224.0.0.0/4"),
netip.MustParsePrefix("255.255.255.255/32"),
netip.MustParsePrefix("::/128"),
netip.MustParsePrefix("::1/128"),
netip.MustParsePrefix("fe80::/10"),
netip.MustParsePrefix("ff00::/8"),
}
var mappedIPv4Prefix = netip.MustParsePrefix("::ffff:0:0/96")
func (t Target) IsZero() bool { return !t.prefix.IsValid() }
// Prefix returns the masked prefix; a single address has full length.
func (t Target) Prefix() netip.Prefix { return t.prefix }
// IsAddress reports whether t names exactly one address. A service target
// names one address too, so a consumer that treats services differently
// checks Service first.
func (t Target) IsAddress() bool { return t.prefix.IsValid() && t.prefix.IsSingleIP() }
// Addr returns the address of a single-address or service target.
func (t Target) Addr() (netip.Addr, bool) {
if !t.IsAddress() {
return netip.Addr{}, false
}
return t.prefix.Addr(), true
}
// Service returns the service of a service target.
func (t Target) Service() (Service, bool) { return t.service, !t.service.IsZero() }
// Covers reports whether every address of u lies inside t.
func (t Target) Covers(u Target) bool {
return t.prefix.IsValid() && u.prefix.IsValid() &&
t.prefix.Bits() <= u.prefix.Bits() && t.prefix.Contains(u.prefix.Addr())
}
// Key is the canonical, reversible text form of t:
//
// ip:192.0.2.1
// net:198.51.100.0/24
// svc:2001:db8::1/tcp/22
func (t Target) Key() string {
switch {
case t.IsZero():
return ""
case !t.service.IsZero():
return "svc:" + t.prefix.Addr().String() + "/" + t.service.Proto.String() + "/" + strconv.Itoa(int(t.service.Port))
case t.IsAddress():
return "ip:" + t.prefix.Addr().String()
}
return "net:" + t.prefix.String()
}
func checkFamily(addr netip.Addr, caps Caps) error {
if addr.Is6() && !caps.IPv6 {
return refuse(ReasonUnsupportedContainment, "IPv6 is not enabled in the firewall")
}
return nil
}
// CanonicalAddress parses one address. IPv4-mapped IPv6 is unmapped; zones,
// protected addresses and IPv6 without firewall support are refused.
func CanonicalAddress(raw string, caps Caps) (Target, error) {
addr, err := netip.ParseAddr(raw)
if err != nil {
return Target{}, refuse(ReasonInvalid, "address does not parse")
}
if addr.Zone() != "" {
return Target{}, refuse(ReasonInvalid, "address carries a zone")
}
addr = addr.Unmap()
for _, p := range protectedPrefixes {
if p.Contains(addr) {
return Target{}, refuse(ReasonProtected, "unspecified, loopback, link-local, multicast or broadcast address")
}
}
if err := checkFamily(addr, caps); err != nil {
return Target{}, err
}
return Target{prefix: netip.PrefixFrom(addr, addr.BitLen())}, nil
}
// CanonicalPrefix parses one CIDR prefix and masks its host bits. A prefix
// that overlaps a protected range, the default route, or both native and
// IPv4-mapped IPv6 is refused.
func CanonicalPrefix(raw string, caps Caps) (Target, error) {
p, err := netip.ParsePrefix(raw)
if err != nil {
return Target{}, refuse(ReasonInvalid, "prefix does not parse")
}
addr, bits := p.Addr(), p.Bits()
if addr.Is4In6() {
if bits < 96 {
return Target{}, refuse(ReasonInvalid, "mapped prefix reaches outside IPv4")
}
addr, bits = addr.Unmap(), bits-96
}
p = netip.PrefixFrom(addr, bits).Masked()
if bits == 0 {
return Target{}, refuse(ReasonProtected, "default route")
}
for _, protected := range protectedPrefixes {
if protected.Overlaps(p) {
return Target{}, refuse(ReasonProtected, "prefix overlaps an unspecified, loopback, link-local, multicast or broadcast range")
}
}
// A native base address can still span mapped IPv4 addresses. Such a
// prefix cannot be represented by one target after unmapping.
if p.Overlaps(mappedIPv4Prefix) {
return Target{}, refuse(ReasonInvalid, "prefix spans native and IPv4-mapped IPv6")
}
if err := checkFamily(addr, caps); err != nil {
return Target{}, err
}
return Target{prefix: p}, nil
}
// CanonicalService parses one service on one address. The port must be
// 1-65535 and the transport tcp or udp.
func CanonicalService(rawAddr, proto string, port int, caps Caps) (Target, error) {
t, err := CanonicalAddress(rawAddr, caps)
if err != nil {
return Target{}, err
}
p, err := ParseProto(proto)
if err != nil {
return Target{}, err
}
if port < 1 || port > 65535 {
return Target{}, refuse(ReasonInvalid, "port outside 1-65535")
}
t.service = Service{Proto: p, Port: uint16(port)}
return t, nil
}
// ParseTargetKey parses a Key. Anything but the exact canonical form is
// refused, so a stored key cannot alias another target.
func ParseTargetKey(key string, caps Caps) (Target, error) {
kind, rest, ok := strings.Cut(key, ":")
if !ok {
return Target{}, refuse(ReasonInvalid, "target key has no kind")
}
var t Target
var err error
switch kind {
case "ip":
t, err = CanonicalAddress(rest, caps)
case "net":
t, err = CanonicalPrefix(rest, caps)
case "svc":
portAt := strings.LastIndexByte(rest, '/')
if portAt < 0 {
return Target{}, refuse(ReasonInvalid, "service key has no port")
}
protoAt := strings.LastIndexByte(rest[:portAt], '/')
if protoAt < 0 {
return Target{}, refuse(ReasonInvalid, "service key has no transport")
}
port, perr := strconv.Atoi(rest[portAt+1:])
if perr != nil {
return Target{}, refuse(ReasonInvalid, "service key port is not a number")
}
t, err = CanonicalService(rest[:protoAt], rest[protoAt+1:portAt], port, caps)
default:
return Target{}, refuse(ReasonInvalid, "unknown target key kind")
}
if err != nil {
return Target{}, err
}
if t.Key() != key {
return Target{}, refuse(ReasonInvalid, "target key is not canonical")
}
return t, nil
}
package admission
import "time"
// ActionView is one admitted candidate and its attempts as `csm action
// show` prints them: what the ledger decided and, for a candidate refused
// as an existing effect, the block that already enforced it.
type ActionView struct {
Candidate CandidateID `json:"candidate_id"`
Kind string `json:"kind"`
Target string `json:"target"`
Check string `json:"check,omitempty"`
FindingID string `json:"finding_id,omitempty"`
State string `json:"state"`
Reason string `json:"reason,omitempty"`
Disposition string `json:"disposition,omitempty"`
ExpiresAt time.Time `json:"expires_at,omitzero"`
Link *LinkView `json:"link,omitempty"`
Attempts []AttemptView `json:"attempts"`
}
// LinkView names the block that already enforced a refused candidate's
// effect, and when it ends (zero: permanent).
type LinkView struct {
Effect string `json:"effect"`
ExpiresAt time.Time `json:"expires_at,omitzero"`
}
// AttemptView is one attempt of an ActionView, in sequence order.
type AttemptView struct {
Action ActionID `json:"action_id"`
Seq uint32 `json:"seq"`
State string `json:"state"`
Disposition string `json:"disposition,omitempty"`
Lane string `json:"lane,omitempty"`
Reserved time.Time `json:"reserved"`
Finished time.Time `json:"finished,omitzero"`
ExpiresAt time.Time `json:"expires_at"`
Retired bool `json:"retired,omitempty"`
}
// NewActionView is the view of candidate c, named id, with its attempts
// in sequence order.
func NewActionView(id CandidateID, c Candidate, attempts []AttemptRecord) ActionView {
v := ActionView{
Candidate: id, Kind: c.Key.Kind.String(), Target: c.Key.Target.Key(), Check: c.Check, FindingID: c.FindingID,
State: c.State.String(), ExpiresAt: c.ExpiresAt, Attempts: make([]AttemptView, 0, len(attempts)),
}
if c.Reason != 0 {
v.Reason = c.Reason.String()
}
if c.Disposition != 0 {
v.Disposition = c.Disposition.String()
}
if c.Link != (EffectLink{}) {
v.Link = &LinkView{Effect: c.Link.Effect, ExpiresAt: c.Link.ExpiresAt}
}
for _, a := range attempts {
av := AttemptView{
Action: a.Attempt.ID, Seq: a.Attempt.Seq, State: a.State.String(),
Reserved: a.Reserved, Finished: a.Finished, ExpiresAt: a.ExpiresAt, Retired: a.Retired,
}
if a.Disposition != 0 {
av.Disposition = a.Disposition.String()
}
if a.Lane != 0 {
av.Lane = a.Lane.String()
}
v.Attempts = append(v.Attempts, av)
}
return v
}
// Package admission holds the typed vocabulary automatic firewall responses
// are admitted with: canonical targets, evidence records, verified owners,
// candidate identity and priority. It imports only the standard library so
// the checks, daemon, firewall and store packages can share it without an
// import cycle.
package admission
import (
"errors"
"fmt"
)
// Severity is a finding severity as admission orders it. The zero value is
// invalid; the checks package converts alert severities at its boundary.
type Severity uint8
const (
SeverityWarning Severity = iota + 1
SeverityHigh
SeverityCritical
)
func (s Severity) Valid() bool { return s >= SeverityWarning && s <= SeverityCritical }
func (s Severity) String() string {
switch s {
case SeverityWarning:
return "warning"
case SeverityHigh:
return "high"
case SeverityCritical:
return "critical"
}
return fmt.Sprintf("severity(%d)", uint8(s))
}
// Class is the priority class of a candidate; larger classes are served
// first. It is not the action risk tier of internal/privops.
type Class uint8
const (
// ClassC1: external intel or derived history about a locally sighted
// address.
ClassC1 Class = iota + 1
// ClassC2: local attack observations such as brute force, floods,
// scanners and WAF escalations.
ClassC2
// ClassC3: direct compromise evidence under a reviewed check policy, or
// C2 corroborated by an independent root.
ClassC3
)
func (c Class) Valid() bool { return c >= ClassC1 && c <= ClassC3 }
func (c Class) String() string {
if c.Valid() {
return fmt.Sprintf("c%d", uint8(c))
}
return fmt.Sprintf("class(%d)", uint8(c))
}
// Tier is the eviction pair. Only Tier decides strictly-lower-tier
// eviction; age, account and queue position never do.
type Tier struct {
Class Class
Severity Severity
}
func (t Tier) Valid() bool { return t.Class.Valid() && t.Severity.Valid() }
// Less reports whether t is strictly lower than u: class first, then
// severity.
func (t Tier) Less(u Tier) bool {
if t.Class != u.Class {
return t.Class < u.Class
}
return t.Severity < u.Severity
}
// Family is the observation channel a check's evidence comes from. Two roots
// of one family never corroborate each other, so one HTTP request seen by
// both the access log and the WAF stays one root. The zero value carries no
// admissible address evidence.
type Family uint8
const (
FamilyNone Family = iota
// FamilyHTTP: web server and WAF observations of HTTP requests.
FamilyHTTP
// FamilyPanel: cPanel, WHM, webmail and panel API authentication.
FamilyPanel
// FamilyMail: SMTP, IMAP and POP3 authentication and abuse.
FamilyMail
// FamilySSH: SSH and PAM authentication.
FamilySSH
// FamilyFTP: FTP authentication.
FamilyFTP
// FamilyNetwork: local socket observations of connections this host
// opened. Its address is a destination, never a source of traffic.
FamilyNetwork
// FamilyReputation: external intel about a locally sighted address.
FamilyReputation
// FamilyDerived: local history, threat database rows, central intel,
// incident wrappers and challenge timeouts. Never an independent root.
FamilyDerived
familyEnd
)
var familyNames = [...]string{"none", "http", "panel", "mail", "ssh", "ftp", "network", "reputation", "derived"}
func (f Family) Valid() bool { return f < familyEnd }
func (f Family) String() string {
if f.Valid() {
return familyNames[f]
}
return fmt.Sprintf("family(%d)", uint8(f))
}
// LocalAttack reports whether evidence of f is a local observation of the
// attack itself.
func (f Family) LocalAttack() bool {
switch f {
case FamilyHTTP, FamilyPanel, FamilyMail, FamilySSH, FamilyFTP:
return true
}
return false
}
// Destination reports whether an address of f is the far end of a
// connection this host opened. It proves compromise on this host, not that
// the address sent anything here, so it corroborates nothing else.
func (f Family) Destination() bool { return f == FamilyNetwork }
// Independent reports whether evidence of f can corroborate a root of a
// different family.
func (f Family) Independent() bool { return f.LocalAttack() || f == FamilyReputation }
// Basis is the class a check's own evidence supports before corroboration.
type Basis uint8
const (
BasisNone Basis = iota
// BasisIntel supports C1.
BasisIntel
// BasisLocal supports C2: a local observation of attack activity.
BasisLocal
// BasisCompromise supports C3: direct compromise evidence. Only checks
// on the reviewed compromise list may carry it.
BasisCompromise
basisEnd
)
var basisNames = [...]string{"none", "intel", "local", "compromise"}
func (b Basis) Valid() bool { return b < basisEnd }
func (b Basis) String() string {
if b.Valid() {
return basisNames[b]
}
return fmt.Sprintf("basis(%d)", uint8(b))
}
// Class returns the class b supports; BasisNone supports none.
func (b Basis) Class() (Class, bool) {
switch b {
case BasisIntel:
return ClassC1, true
case BasisLocal:
return ClassC2, true
case BasisCompromise:
return ClassC3, true
}
return 0, false
}
// ValidPolicy reports whether a family and basis may appear together in a
// check's policy. Intel needs a reputation or derived family; local evidence
// must be a local attack observation, and compromise evidence one of those
// or a destination.
func ValidPolicy(f Family, b Basis) error {
if !f.Valid() || !b.Valid() {
return fmt.Errorf("unknown family %d or basis %d", uint8(f), uint8(b))
}
switch {
case f == FamilyNone && b == BasisNone:
return nil
case f == FamilyNone || b == BasisNone:
return fmt.Errorf("family %s and basis %s must both be none or both be set", f, b)
case b == BasisIntel && f != FamilyReputation && f != FamilyDerived:
return fmt.Errorf("intel basis needs a reputation or derived family, not %s", f)
case b != BasisIntel && !f.LocalAttack() && (b != BasisCompromise || !f.Destination()):
return fmt.Errorf("%s basis needs a local attack family, not %s", b, f)
}
return nil
}
// Kind is the response a candidate asks for.
type Kind uint8
const (
KindBlockIP Kind = iota + 1
KindBlockService
KindBlockSubnet
KindPromote
KindChallenge
kindEnd
)
var kindNames = [...]string{"", "block_ip", "block_service", "block_subnet", "promote", "challenge"}
func (k Kind) Valid() bool { return k >= KindBlockIP && k < kindEnd }
func (k Kind) String() string {
if k.Valid() {
return kindNames[k]
}
return fmt.Sprintf("kind(%d)", uint8(k))
}
// Effect is the action family a kind belongs to. Fairness scopes are
// (owner, effect).
type Effect uint8
const (
EffectAddress Effect = iota + 1
EffectService
EffectPrefix
EffectChallenge
effectEnd
)
var effectNames = [...]string{"", "address", "service", "prefix", "challenge"}
func (e Effect) Valid() bool { return e >= EffectAddress && e < effectEnd }
func (e Effect) String() string {
if e.Valid() {
return effectNames[e]
}
return fmt.Sprintf("effect(%d)", uint8(e))
}
// Effect returns the action family of k.
func (k Kind) Effect() Effect {
switch k {
case KindBlockIP, KindPromote:
return EffectAddress
case KindBlockService:
return EffectService
case KindBlockSubnet:
return EffectPrefix
case KindChallenge:
return EffectChallenge
}
return 0
}
// Entry is the automatic path a candidate arrives on. The engine binds each
// entry to its registered producers.
type Entry uint8
const (
EntryScan Entry = iota + 1
EntryChallengeTimeout
EntryIncident
EntryIncidentSpray
EntryCentral
EntryNetblock
EntryASNCrawl
EntryMailSubnet
EntryPermblock
entryEnd
)
var entryNames = [...]string{"", "scan", "challenge_timeout", "incident", "incident_spray", "central", "netblock", "asn_crawl", "mail_subnet", "permblock"}
func (e Entry) Valid() bool { return e >= EntryScan && e < entryEnd }
// Range reports an entry that acts on a prefix holding its roots' addresses.
func (e Entry) Range() bool { return e == EntryMailSubnet || e == EntryASNCrawl }
func (e Entry) String() string {
if e.Valid() {
return entryNames[e]
}
return fmt.Sprintf("entry(%d)", uint8(e))
}
// Disposition is how a candidate ended or why it waits. The first four
// group the reasons a candidate did not receive its selected response; the
// rest are attempt and preview outcomes.
type Disposition uint8
const (
DispositionDeferred Disposition = iota + 1
DispositionRefused
DispositionWithheld
DispositionDropped
// Outcomes of an attempt or a preview. Persisted: append, never renumber.
DispositionApplied
DispositionNarrowed
DispositionDryRun
DispositionObserve
DispositionFailed
DispositionUnknown
// DispositionExpiredUnproven names the audit row of a linked event,
// never an outcome: recovery retired an unknown attempt's uncertainty
// after its effect expired, without proving success or failure.
DispositionExpiredUnproven
dispositionEnd
)
var dispositionNames = [...]string{"", "deferred", "refused", "withheld", "dropped", "applied", "narrowed", "dry_run", "observe", "failed", "unknown", "expired_unproven"}
func (d Disposition) Valid() bool { return d >= DispositionDeferred && d < dispositionEnd }
func (d Disposition) String() string {
if d.Valid() {
return dispositionNames[d]
}
return fmt.Sprintf("disposition(%d)", uint8(d))
}
// Reason is a fixed, bounded explanation. It never carries attacker text.
type Reason uint8
const (
ReasonCeiling Reason = iota + 1
ReasonSetFull
ReasonStorageShare
ReasonEngineUnavailable
ReasonPendingRecovery
ReasonProtected
ReasonAttribution
ReasonInvalid
ReasonPolicy
ReasonStaleIdentity
ReasonExistingEffect
ReasonCollateral
ReasonBreaker
ReasonEnvelopeNoAlternative
ReasonUnsupportedContainment
ReasonQueueOverflow
ReasonStale
ReasonIngressInterruption
// ReasonVerdictAllow: the panel's verdict callback allowed the target.
// It refuses, although appended after the dropped group.
ReasonVerdictAllow
// ReasonDisabled: the configuration no longer allows this automatic
// response, for example auto_response.enabled or block_ips was turned
// off while it was queued. It refuses, appended like verdict_allow.
ReasonDisabled
reasonEnd
)
var reasonNames = [...]string{
"", "ceiling", "set_full", "storage_share", "engine_unavailable",
"pending_recovery", "protected", "attribution", "invalid", "policy",
"stale_identity", "existing_effect", "collateral", "breaker",
"envelope_no_alternative", "unsupported_containment", "queue_overflow",
"stale", "ingress_interruption", "verdict_allow", "disabled",
}
func (r Reason) Valid() bool { return r >= ReasonCeiling && r < reasonEnd }
func (r Reason) String() string {
if r.Valid() {
return reasonNames[r]
}
return fmt.Sprintf("reason(%d)", uint8(r))
}
// Disposition returns the group r belongs to.
func (r Reason) Disposition() Disposition {
switch {
case r >= ReasonCeiling && r <= ReasonPendingRecovery:
return DispositionDeferred
case r >= ReasonProtected && r <= ReasonExistingEffect, r == ReasonVerdictAllow, r == ReasonDisabled:
return DispositionRefused
case r >= ReasonCollateral && r <= ReasonUnsupportedContainment:
return DispositionWithheld
case r >= ReasonQueueOverflow && r <= ReasonIngressInterruption:
return DispositionDropped
}
return 0
}
// Error is an admission refusal. Detail is engine-generated text; it never
// quotes the input that was refused.
type Error struct {
Reason Reason
// Cause narrows an attribution refusal; zero for every other reason.
Cause Cause
Detail string
}
func (e *Error) Error() string { return e.Reason.String() + ": " + e.Detail }
func refuse(r Reason, detail string) error { return &Error{Reason: r, Detail: detail} }
// ReasonOf returns the reason carried by err, if any.
func ReasonOf(err error) (Reason, bool) {
var e *Error
if errors.As(err, &e) {
return e.Reason, true
}
return 0, false
}
package admissionowner
import (
"errors"
"fmt"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/store"
)
var readActionAttempt = (*store.AdmissionLedger).Attempt
// Action reads one admitted candidate and its attempts for `csm action
// show`. ref is a candidate ID or the action ID of one of its attempts.
// The read runs on the owner's goroutine, so it sees no half-made change.
func (o *Owner) Action(ref string) (admission.ActionView, error) {
var view admission.ActionView
err := o.do(func() error {
if o.ledger == nil {
return errors.New("the admission ledger is not running")
}
id, err := admission.ParseCandidateID(ref)
if err != nil {
action, actionErr := admission.ParseActionID(ref)
if actionErr != nil {
return fmt.Errorf("%q is neither a candidate nor an action ID", ref)
}
a, readErr := o.ledger.Attempt(action)
if readErr != nil {
return missingRecord(ref, readErr)
}
id = a.Attempt.Candidate
}
c, err := o.ledger.Candidate(id)
if err != nil {
return missingRecord(ref, err)
}
attempts := make([]admission.AttemptRecord, 0, c.Attempts)
for seq := uint32(1); seq <= c.Attempts; seq++ {
next, idErr := admission.NewAttempt(id, seq)
if idErr != nil {
return idErr
}
a, readErr := readActionAttempt(o.ledger, next.ID)
if readErr != nil {
if reason, refused := admission.ReasonOf(readErr); refused && reason == admission.ReasonInvalid {
return fmt.Errorf("%w: candidate %s is missing attempt %s", admission.ErrCorruptRecord, id, next.ID)
}
return fmt.Errorf("reading attempt %s: %w", next.ID, readErr)
}
attempts = append(attempts, a)
}
view = admission.NewActionView(id, c, attempts)
return nil
})
return view, err
}
// missingRecord names a record the ledger refuses to read as one it does
// not hold: never recorded, or retired with its history.
func missingRecord(ref string, err error) error {
if reason, refused := admission.ReasonOf(err); refused && reason == admission.ReasonInvalid {
return fmt.Errorf("the admission ledger has no record of %s; it was never admitted or its history has retired", ref)
}
return fmt.Errorf("reading %s: %w", ref, err)
}
package admissionowner
import (
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/queuehealth"
)
const (
// auditBatchesPerRun bounds the writes one delivery makes, so a
// backlog cannot hold the owner goroutine.
auditBatchesPerRun = 16
// deliveryLag is how long pending work may go without progress before
// its queue reports a stall.
deliveryLag = time.Minute
)
// auditBatch bounds the rows one durable write carries; deliveryNow dates
// delivery progress. Tests shrink the one and move the other.
var (
auditBatch = 64
deliveryNow = time.Now
)
// deliverAudit writes pending audit rows to the action log, a batch at a
// time with one durable write each, and acknowledges each row by its ID
// and time only after its write. A failed or
// unacknowledged write acknowledges nothing: the rows are written again,
// with the same identity, and readers drop the copy. Rows this process wrote
// whose consumers then failed are not written again before their ack, so a
// long consumer outage cannot flood the action log; a restart writes them
// once more.
func (o *Owner) deliverAudit() (err error) {
defer func() { o.noteAuditError(err) }()
for range auditBatchesPerRun {
rows, err := o.ledger.PendingAudit(auditBatch)
if err != nil {
return err
}
o.audit.Observe(deliveryNow(), len(rows), o.auditAcked)
if len(rows) == 0 {
return nil
}
records := make([]actionlog.Record, 0, len(rows))
acks := make([]admission.AuditAck, len(rows))
for i, r := range rows {
acks[i] = r.Ack()
if !o.auditWritten[auditKeyOf(acks[i])] {
records = append(records, auditRecord(r))
}
}
if len(records) > 0 {
if err = o.opts.WriteAudit(records); err != nil {
return err
}
if o.auditWritten == nil {
o.auditWritten = make(map[auditKey]bool, len(acks))
}
for _, a := range acks {
o.auditWritten[auditKeyOf(a)] = true
}
}
if err = o.consumeApplied(rows); err != nil {
return err
}
if err = o.ledger.AckAudit(acks); err != nil {
return err
}
// New rows can sort before a failed batch and displace its written
// rows. Keep their write receipts until their own acknowledgements.
for _, a := range acks {
delete(o.auditWritten, auditKeyOf(a))
}
o.auditAcked += uint64(len(rows))
}
return nil
}
// auditKey names one audit row as its ack does: its identity and its time,
// which tells a re-minted attempt's rows apart.
type auditKey struct {
id admission.AuditID
at int64
}
func auditKeyOf(a admission.AuditAck) auditKey {
return auditKey{id: a.ID, at: a.At.UnixNano()}
}
// noteAuditError keeps the last delivery failure for status. Delivery runs
// every second and status reads the whole ledger on its own timer, so only a
// change is published, on a copy of the cached status.
func (o *Owner) noteAuditError(err error) {
if errorText(err) == errorText(o.auditErr) {
o.auditErr = err
return
}
o.auditErr = err
previous := o.current.Load()
if previous == nil || previous.Owner == nil {
return
}
snapshot, owner := *previous, *previous.Owner
owner.AuditError = errorText(err)
snapshot.Owner = &owner
o.current.Store(&snapshot)
}
func errorText(err error) string {
if err == nil {
return ""
}
return err.Error()
}
// consumeApplied hands the applied block_ip rows among rows to the
// configured consumer. Only a verified full-address block reaches it:
// reservations, executions, failures, unknown outcomes and other kinds
// never mark an address blocked.
func (o *Owner) consumeApplied(rows []admission.AuditRow) error {
if o.opts.Applied == nil {
return nil
}
var applied []admission.AuditRow
for _, r := range rows {
if r.Kind == admission.KindBlockIP && r.State == admission.StateVerified && r.Disposition == admission.DispositionApplied {
applied = append(applied, r)
}
}
if len(applied) == 0 {
return nil
}
return o.opts.Applied(applied)
}
// auditRecord is the action log record of one admitted transition: the
// attempt and transition name it, and the row's time tells a re-minted
// attempt's rows apart.
func auditRecord(r admission.AuditRow) actionlog.Record {
result := r.State.String()
if r.Disposition != 0 {
result = r.Disposition.String()
}
return actionlog.Record{
Timestamp: r.At, Op: "respond.block_ip", Action: r.Kind.String(), Actor: actionlog.Daemon,
FindingID: r.FindingID, ActionID: string(r.Attempt.ID), ActionVersion: uint64(r.Transition),
Target: r.Target.Key(), Reason: r.Lane.String() + " lane, expires " + r.ExpiresAt.UTC().Format(time.RFC3339),
Result: actionlog.Result(result),
}
}
// QueueStatuses reports the owner's deliveries through queue health, which
// does not depend on the ledger being writable. An executing owner also
// reports its verdict jobs.
func (o *Owner) QueueStatuses(now time.Time) map[string]queuehealth.Status {
out := map[string]queuehealth.Status{"audit": o.audit.Snapshot(now), "notices": o.notices.queue.Snapshot(now)}
if o.verdictQueue != nil {
out["verdicts"] = o.verdictQueue.Snapshot(now)
}
return out
}
// Package admissionowner is the daemon's owner of the admission ledger:
// the one handle on the state database's ledger, and the one
// goroutine that changes it.
package admissionowner
import (
"fmt"
"os"
"strings"
"time"
"github.com/pidginhost/csm/internal/admission"
)
// bootIDPath names the kernel's identity of the current boot.
const bootIDPath = "/proc/sys/kernel/random/boot_id"
// readClock samples wall time, the boot identity at path and the time since
// that boot. The ledger validates the identity.
func readClock(path string, wall func() time.Time, sinceBoot func() (time.Duration, error)) (admission.ClockReading, error) {
data, err := os.ReadFile(path) // #nosec G304 -- a fixed procfs path; tests pass a temporary file.
if err != nil {
return admission.ClockReading{}, fmt.Errorf("reading the boot identity: %w", err)
}
since, err := sinceBoot()
if err != nil {
return admission.ClockReading{}, fmt.Errorf("reading the time since boot: %w", err)
}
return admission.ClockReading{Wall: wall(), BootID: strings.TrimSpace(string(data)), SinceBoot: since}, nil
}
//go:build linux
package admissionowner
import (
"time"
"github.com/pidginhost/csm/internal/admission"
"golang.org/x/sys/unix"
)
// ReadClock reads the host clocks. Time since boot is the boot-time clock,
// which keeps counting while the host is suspended, so a same-boot restart
// resumes from where the ledger left off.
func ReadClock() (admission.ClockReading, error) {
return readClock(bootIDPath, time.Now, func() (time.Duration, error) {
var ts unix.Timespec
if err := unix.ClockGettime(unix.CLOCK_BOOTTIME, &ts); err != nil {
return 0, err
}
return time.Duration(ts.Nano()), nil
})
}
package admissionowner
import (
"sort"
"sync"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/checks"
)
// Decisions the comparison counts. A preview is the decision a live
// attempt would have been executed on; a dry run is a forecast ending,
// counted with its prediction.
const (
decisionRefused = "refused"
decisionQueued = "queued"
decisionCoalesced = "coalesced"
decisionObserve = "observe"
decisionDryRun = "dry_run"
)
// maxUnwrittenRows bounds history while the action log cannot be written.
const maxUnwrittenRows = 4096
type compareKey struct {
entry admission.Entry
check string
kind admission.Kind
decision string
reason admission.Reason
cause admission.Cause
prediction admission.Prediction
}
// Funnels count without waiting on the owner. The fixed check vocabulary
// keeps malformed findings and absent derived roots from growing the map.
type comparison struct {
mu sync.Mutex
start time.Time
counts map[compareKey]uint64
next uint64
unwritten []comparisonRow
// dropped counts the rows retention discarded unwritten.
dropped uint64
// discardedThrough keeps late rows from restoring partial hours after
// retention discarded them, even when acknowledgement makes room.
discardedThrough time.Time
}
type comparisonRow struct {
actionlog.Record
sequence uint64
}
// registeredCheck is name when it is a registered check and "unknown"
// otherwise, so no free text reaches a summary or an example.
func registeredCheck(name string) string {
if _, registered := checks.LookupCheck(name); !registered {
return "unknown"
}
return name
}
func (c *comparison) add(k compareKey) { c.addAt(k, deliveryNow()) }
func (c *comparison) addAt(k compareKey, now time.Time) { c.addCountAt(k, 1, now) }
func (c *comparison) addCountAt(k compareKey, n uint64, now time.Time) {
if n == 0 {
return
}
c.mu.Lock()
defer c.mu.Unlock()
c.addCountLocked(k, n, now)
}
// addLedgerRefusalAt counts a refusal in the hour of at, the ledger time it
// ended. When a funnel already opened a newer hour, the refusal keeps its
// own hour as a separate row; the open hour never moves back.
func (c *comparison) addLedgerRefusalAt(k compareKey, at time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
hour := at.UTC().Truncate(time.Hour)
if c.start.IsZero() || !hour.Before(c.start) {
c.addCountLocked(k, 1, at)
return
}
k.check = registeredCheck(k.check)
c.next++
c.unwritten = append(c.unwritten, comparisonRow{Record: k.row(hour.Add(time.Hour), 1), sequence: c.next})
c.retain()
}
// addCountLocked requires mu.
func (c *comparison) addCountLocked(k compareKey, n uint64, now time.Time) {
k.check = registeredCheck(k.check)
hour := now.UTC().Truncate(time.Hour)
if c.start.IsZero() {
c.start = hour
} else if hour.After(c.start) {
c.closeHour()
c.start = hour
}
if c.counts == nil {
c.counts = map[compareKey]uint64{}
}
c.counts[k] += n
}
// begin advances the hour floor after a committed admission clock reading,
// including readings made by a startup that later fails.
func (c *comparison) begin(now time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
if c.start.IsZero() {
c.start = now.UTC().Truncate(time.Hour)
} else if hour := now.UTC().Truncate(time.Hour); hour.After(c.start) {
c.closeHour()
c.start = hour
}
}
// closeHour requires mu.
func (c *comparison) closeHour() {
end := c.start.Add(time.Hour)
for k, n := range c.counts {
c.next++
c.unwritten = append(c.unwritten, comparisonRow{Record: k.row(end, n), sequence: c.next})
}
c.counts = nil
c.retain()
}
// row is the summary record of n decisions of k in the hour ending at end.
func (k compareKey) row(end time.Time, n uint64) actionlog.Record {
r := actionlog.Record{
Timestamp: end, Op: "respond.block_ip", Action: k.kind.String(), Actor: actionlog.Daemon,
ActorDetail: k.entry.String(), Reason: k.check, Result: actionlog.Result(k.decision), Count: n,
}
if k.reason != 0 {
r.Error = k.reason.String()
}
if k.cause != 0 {
r.Cause = k.cause.String()
}
if k.prediction != 0 {
r.Prediction = k.prediction.String()
}
return r
}
// retain requires mu. It sorts before trimming, so a backdated row can never
// evict newer evidence merely by append order.
func (c *comparison) retain() {
sort.SliceStable(c.unwritten, func(i, j int) bool { return c.unwritten[i].Timestamp.Before(c.unwritten[j].Timestamp) })
if extra := len(c.unwritten) - maxUnwrittenRows; extra > 0 {
if through := c.unwritten[extra-1].Timestamp; through.After(c.discardedThrough) {
c.discardedThrough = through
}
}
cut := 0
for cut < len(c.unwritten) && !c.unwritten[cut].Timestamp.After(c.discardedThrough) {
cut++
}
if cut > 0 {
copy(c.unwritten, c.unwritten[cut:])
c.unwritten = c.unwritten[:len(c.unwritten)-cut]
saturatingAdd(&c.dropped, uint64(cut))
}
}
// droppedRows is how many rows retention discarded unwritten.
func (c *comparison) droppedRows() uint64 {
c.mu.Lock()
defer c.mu.Unlock()
return c.dropped
}
// take returns the rows still to write and the sequence they end at: those
// of earlier failed writes and, once its hour has ended by now, or at once
// when final is set, the current hour's, stamped with that hour's end.
func (c *comparison) take(now time.Time, final bool) ([]actionlog.Record, uint64) {
c.mu.Lock()
defer c.mu.Unlock()
if !c.start.IsZero() && (final || !now.Before(c.start.Add(time.Hour))) {
c.closeHour()
if hour := now.UTC().Truncate(time.Hour); hour.After(c.start) {
c.start = hour
}
}
rows := make([]actionlog.Record, len(c.unwritten))
for i, r := range c.unwritten {
rows[i] = r.Record
}
sort.SliceStable(rows, func(i, j int) bool { return rows[i].Timestamp.Before(rows[j].Timestamp) })
return rows, c.next
}
// A writer acknowledges the immutable batch it took. Funnels can close
// newer hours while that write blocks, and retention can retire old rows.
func (c *comparison) written(through uint64) {
c.mu.Lock()
defer c.mu.Unlock()
kept := c.unwritten[:0]
for _, r := range c.unwritten {
if r.sequence > through {
kept = append(kept, r)
}
}
c.unwritten = kept
}
package admissionowner
import (
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/health"
)
// predictionScope names what the forecast models.
const predictionScope = "ceiling_and_temporary_capacity"
// dryRunStatus reads the forecast for status. While the owner
// forecasts it shows the session's model and counts, and no identity until
// a session starts; otherwise only the last session's identity. Loss and
// error counters cover the process.
func (o *Owner) dryRunStatus() *health.AdmissionDryRun {
d := &health.AdmissionDryRun{
Active: o.forecasting(), ModelledKinds: []string{admission.KindBlockIP.String()}, PredictionScope: predictionScope,
ExamplesLost: o.out.examplesLost, SummariesLost: o.out.summariesLost, RecordErrors: o.out.recordErrors, ExamplesStale: o.out.stale,
ComparisonIncomplete: o.out.restarted || o.out.examplesLost > 0 || o.out.summariesLost > 0 || o.compare.droppedRows() > 0,
}
var picks, recent [admission.PredictionSlots]uint64
// An earlier session's identity shows only once the owner stops
// forecasting; an active forecast without a session shows none.
var session sessionInfo
if !d.Active {
session = o.out.last
}
if s := o.shadow; s != nil {
session = sessionInfo{id: s.id, since: s.since, reset: s.reset}
if d.Active {
picks, recent = s.picks, s.recent.total(monoNow())
d.Occupancy, d.Ceiling = s.occupancyStatus(), s.ceilingStatus()
}
}
if session.id != "" {
id, since, reset := session.id, session.since, session.reset.String()
d.SessionID, d.Since, d.ResetReason = &id, &since, &reset
}
d.Picks, d.LastHour = predictionMap(picks), predictionMap(recent)
if o.forecastErr != "" {
code := string(o.forecastErr)
d.ForecastError = &code
}
if !o.forecastErrAt.IsZero() {
at := o.forecastErrAt
d.LastErrorAt = &at
}
return d
}
// predictionMap names every prediction's count, zeros included.
func predictionMap(counts [admission.PredictionSlots]uint64) map[string]uint64 {
out := make(map[string]uint64, admission.PredictionSlots-1)
for p := admission.Prediction(1); p.Valid(); p++ {
out[p.String()] = counts[p]
}
return out
}
// occupancyStatus is the temporary capacity as the last snapshot saw it,
// with the forecast blocks the real sets do not hold, the listed imports
// the forecast replaced and the modelled partitions of the automatic
// slots; nil without one.
func (s *shadow) occupancyStatus() *health.AdmissionDryRunOccupancy {
if s.adm == nil {
return nil
}
adm := *s.adm
occ := s.occupancy(adm)
slots := occ.Slots()
general, reserved := s.counted(adm)
only, replaced := general+reserved, s.replacedListed(adm)
imported := max(adm.Imported, len(adm.Imports))
actual := adm.Unclaimed + imported + adm.Admission
holding := s.holding(adm)
drain := adm.Drain
if occ.Unclaimed > adm.Unclaimed {
// The engine cannot bound the drain of unowned admission rows.
drain = -time.Second
}
return &health.AdmissionDryRunOccupancy{
Actual: actual, ShadowOnly: only, Replaced: replaced, Predicted: actual + only - replaced, Limit: adm.Limit,
Unlimited: adm.Limit <= 0, Limited: s.limited(),
Capacity: slots.Capacity, Unclaimed: occ.Unclaimed, Imported: imported, DrainSeconds: int64(drain / time.Second),
General: health.AdmissionDryRunPartition{Size: slots.General, Used: occ.General, Free: slots.FreeGeneral},
Reserved: health.AdmissionDryRunPartition{Size: slots.Reserved, Used: occ.Reserved, Free: slots.FreeReserved},
Shares: health.AdmissionDryRunShares{
General: shareOf(slots.General, holding[admission.PartitionGeneral], s.waiting[admission.PartitionGeneral]),
Reserved: shareOf(slots.Reserved, holding[admission.PartitionReserved], s.waiting[admission.PartitionReserved]),
},
}
}
// shareOf is a partition of size shared among the scopes holding its
// forecast blocks and those waiting for it.
func shareOf(size int, holding map[string]int, waiting []string) health.AdmissionDryRunShare {
scopes := map[string]bool{}
for scope := range holding {
scopes[scope] = true
}
for _, scope := range waiting {
scopes[scope] = true
}
if len(scopes) == 0 {
return health.AdmissionDryRunShare{}
}
return health.AdmissionDryRunShare{Scopes: len(scopes), Share: size / len(scopes)}
}
func (s *shadow) ceilingStatus() *health.AdmissionDryRunCeiling {
c := s.ceiling
return &health.AdmissionDryRunCeiling{
Limit: c.Limit, GeneralUsed: c.General.Used, ReservedUsed: c.Reserved.Used,
GeneralBudget: c.Budget(admission.LaneGeneral), ReservedBudget: c.Budget(admission.LaneDirect),
}
}
package admissionowner
import (
"errors"
"time"
"github.com/pidginhost/csm/internal/store"
)
// dryRunRetention is how long a forecast example is kept.
const dryRunRetention = 7 * 24 * time.Hour
// maintainEvery spaces the maintenance steps over the example bucket.
const maintainEvery = time.Second
// recordExamples writes one turn's forecast examples and maintainStore runs
// one maintenance step; tests make them fail.
var (
recordExamples = (*store.DB).RecordDryRunBlocks
maintainStore = (*store.DB).MaintainDryRunBlocks
)
// dryRunOutput is what an executable owner's dry runs leave outside their
// session: the turn's examples, the maintenance pass over
// the example bucket, the purge the last change to live asked for, and
// process counters a mode change never resets. Only the owner goroutine
// uses it.
type dryRunOutput struct {
batch []store.DryRunBlock
// cursor is where the maintenance pass resumes, nil to start one; keep
// is the session a purging pass keeps. maintainAt is the next step, on
// the monotonic clock.
cursor []byte
keep string
purging bool
maintainAt time.Time
// stale is set from a purge request until a purge pass completes.
stale bool
// examplesLost counts examples not written; recordErrors counts failed
// writes and maintenance steps; summariesLost counts summaries a stop
// discarded or a full count could not carry.
examplesLost, recordErrors, summariesLost uint64
// last is the session status shows once none runs; restarted is set
// when an earlier process opened the ledger.
last sessionInfo
restarted bool
}
// sessionInfo identifies a forecast session for status.
type sessionInfo struct {
id string
since time.Time
reset resetReason
}
// noteExample adds p's committed capacity admission to the turn's
// examples.
func (o *Owner) noteExample(p execPick, change *shadowChange) {
o.out.batch = append(o.out.batch, store.DryRunBlock{
At: o.now, IP: change.ip, Check: registeredCheck(p.c.Check), Candidate: p.pick.ID,
Session: o.shadow.id, Kind: change.entry.kind, ExpiresAt: change.entry.expires,
})
}
// flushExamples writes the turn's examples once, after its endings. A
// failed write loses them: the endings and the forecast stand, and nothing
// retries them. A bucket above its cap discards them without an error.
func (o *Owner) flushExamples() {
batch := o.out.batch
o.out.batch = o.out.batch[:0]
if len(batch) == 0 {
return
}
err := recordExamples(o.opts.DB, batch)
if err == nil {
return
}
saturatingAdd(&o.out.examplesLost, uint64(len(batch)))
if !errors.Is(err, store.ErrDryRunBlocksFull) {
saturating(&o.out.recordErrors)
}
}
// requestPurge asks the maintenance pass to remove every example but the
// active session's, starting over from the first key.
func (o *Owner) requestPurge() {
o.out.stale, o.out.purging, o.out.cursor = true, false, nil
}
// maintainExamples runs one bounded maintenance step a second at most: it
// trims a bucket above its cap, removes examples older than a week and,
// while a purge is pending, those of every session but the active one. A
// purge pass starts over when the active session changes, so no session
// it should remove is passed over. A failed step is counted and retried.
func (o *Owner) maintainExamples() {
if !o.executable() || !o.started || monoNow().Before(o.out.maintainAt) {
return
}
o.out.maintainAt = monoNow().Add(maintainEvery)
keep := ""
if o.shadow != nil {
keep = o.shadow.id
}
if o.out.stale && (!o.out.purging || keep != o.out.keep) {
o.out.cursor, o.out.purging, o.out.keep = nil, true, keep
}
res, err := maintainStore(o.opts.DB, store.DryRunMaintenance{
Cursor: o.out.cursor, Cutoff: o.now.Add(-dryRunRetention), Purge: o.out.purging, Keep: keep,
})
if err != nil {
saturating(&o.out.recordErrors)
return
}
o.out.cursor = res.Next
if res.Done {
o.out.stale = o.out.stale && !o.out.purging
o.out.cursor, o.out.purging = nil, false
}
}
// OwnsDryRunExamples reports whether the owner alone writes, prunes and
// purges its forecast examples: an owner that runs blocks does.
func (o *Owner) OwnsDryRunExamples() bool { return o.executable() }
func saturatingAdd(n *uint64, d uint64) {
*n += min(d, ^uint64(0)-*n)
}
package admissionowner
import (
"errors"
"fmt"
"slices"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/firewall"
csmlog "github.com/pidginhost/csm/internal/log"
)
// BlockEngine runs batches of timed full-address blocks. The firewall
// engine is one on every platform; only Linux runs them.
type BlockEngine interface {
CheckBlockBatchEffects(members []firewall.BlockMember) ([]firewall.BlockCheck, error)
// RetireBlockActions reports which unknown attempts left nothing in the
// kernel and removes their own rows.
RetireBlockActions(actions []firewall.BlockAction) ([]bool, error)
// HeldBlockActions reports which verified blocks the kernel may still
// hold, changing nothing.
HeldBlockActions(actions []firewall.BlockAction) ([]bool, error)
// BlockAdmissionSnapshot answers members' local checks and the
// temporary occupancy without changing anything or giving any member
// a slot.
BlockAdmissionSnapshot(members []firewall.BlockMember, shadow []string) (firewall.BlockAdmission, error)
ApplyBlockBatch(members []firewall.BlockMember) ([]firewall.BlockResult, error)
// LeaseNow is the clock the engine turns expiries into kernel
// timeouts against. A first attempt's expiry is set on it.
LeaseNow() time.Time
// MarkBlockActionsUnverified with no actions writes the safe state
// again while a failed state write holds batches, and so releases it.
MarkBlockActionsUnverified(actions []string) (int, error)
ProbeBlockState() error
}
// errNoEngine holds execution while no firewall engine runs.
var errNoEngine = errors.New("the firewall engine is not running")
// Probe waits while execution is held (1 s doubling to 60 s).
const (
firstProbeWait = time.Second
maxProbeWait = time.Minute
)
// batchEvery spaces batch starts (one a second at most).
const batchEvery = time.Second
// execPick is a block_ip pick of the turn with its candidate.
type execPick struct {
pick admission.Pick
c admission.Candidate
}
type pendingFinish struct {
id admission.ActionID
disposition admission.Disposition
rejected bool
// existing is a failure because a proven block already holds the
// address: after a fresh proof of member, the pick ends linked to that
// block instead of retrying against it.
existing bool
member firewall.BlockMember
// victim is what the batch did to the member's victim.
victim admission.VictimOutcome
// lead is how far the engine's lease clock ran ahead of admission
// time when the member ran: its block ends that much earlier in
// admission time.
lead time.Duration
}
// refusalReasons are the admission reasons for the engine's refusals of a
// member before it is reserved: a full set or an unproven hold
// defers it; the rest end it without a charge. An existing effect ends
// linked to its block (endExisting). A refusal missing here (a duplicate
// address in the batch) leaves the pick queued for a later turn.
var refusalReasons = map[firewall.BlockRefusal]admission.Reason{
firewall.BlockRefusedFull: admission.ReasonSetFull,
firewall.BlockRefusedPending: admission.ReasonPendingRecovery,
firewall.BlockRefusedInvalid: admission.ReasonInvalid,
firewall.BlockRefusedExpired: admission.ReasonStale,
firewall.BlockRefusedUnsupported: admission.ReasonUnsupportedContainment,
firewall.BlockRefusedProtected: admission.ReasonProtected,
firewall.BlockRefusedAllowed: admission.ReasonProtected,
}
// execute runs the turn's block picks as one batch (one batch
// start per turn, one in flight). The engine checks every member against
// one snapshot before anything is reserved; the members it accepts are
// reserved and charged, recorded as running with their intents, applied
// and finished by their own effect. Under an apply hold the batch is one
// member. Nothing runs again: a failure keeps its charge and the ledger
// retries it after its backoff, and an unknown outcome holds its address
// until recovery retires it.
func (o *Owner) execute(picks []execPick) error {
if !o.mayExecute() {
return nil
}
engine := o.opts.Blocks()
if engine == nil {
// The missing-engine wait must advance even when the clock cannot
// support execution; otherwise every due turn spends a selection.
o.ready(nil, true)
return o.deferBlocks(picks, admission.ReasonEngineUnavailable)
}
// A degraded clock cannot bound a lease: the admission time stays at
// its high-water mark while the wall clock is behind it or disagrees
// with elapsed time. The engine encodes against its own lease clock.
if o.degraded || !o.ready(engine, true) {
return o.deferBlocks(picks, admission.ReasonEngineUnavailable)
}
// Picks before the next batch start stay queued as they are.
if monoNow().Before(o.nextBatch) {
return nil
}
// After a step back the engine's clock leads the wall clock: an expiry
// set on the admission clock would lose that lead from its lifetime.
lease := engine.LeaseNow()
lead := lease.Sub(o.now)
if o.policy.VerdictEnabled {
var err error
if picks, err = o.gateVerdicts(engine, picks, lease); err != nil || len(picks) == 0 {
return err
}
}
// An apply probe already runs one member, so an isolated retry cannot
// endanger its peers. Keep every pick for checks and refusal endings.
if o.applyHold == nil {
picks = o.isolate(picks)
}
members := make([]firewall.BlockMember, len(picks))
for i, p := range picks {
m, err := o.memberOf(p, lease)
if err != nil {
return err
}
members[i] = m
}
checks, err := engine.CheckBlockBatchEffects(members)
if err == nil && len(checks) != len(members) {
err = fmt.Errorf("%d answers for %d members", len(checks), len(members))
o.probeCheck = true
}
if err != nil {
o.hold(fmt.Errorf("checking a block batch: %w", err), false)
return o.deferBlocks(picks, admission.ReasonEngineUnavailable)
}
o.probeCheck = false
if o.execHold != nil && o.applyHold == nil {
o.release()
}
var run []int
var stopped error
for i, p := range picks {
refused, endErr := o.endRefused(p, checks[i])
if endErr != nil {
stopped = endErr
break
}
if refused {
continue
}
// While an apply fault is unproven the probe is one member:
// the other picks stay queued, untouched, for the batch after it
// applies. Refusals still end this turn.
if o.applyHold != nil && len(run) > 0 {
continue
}
_, a, granted, reserveErr := reserveLedger(o.ledger, p.pick.ID, p.pick.Lane, members[i].ExpiresAt)
if reserveErr != nil {
if reason, refused := admission.ReasonOf(reserveErr); refused {
if err = o.endCounted(p, reason, admission.CauseOf(reserveErr)); err != nil {
stopped = err
break
}
continue
}
if errors.Is(reserveErr, admission.ErrNotReady) || errors.Is(reserveErr, admission.ErrLaneIneligible) {
continue
}
stopped = fmt.Errorf("reserving pick %s: %w", p.pick.ID, reserveErr)
break
}
if !granted {
stopped = fmt.Errorf("reserving pick %s: %w", p.pick.ID, admission.ErrTransitionConflict)
break
}
// The reservation consumes the panel's answer for this attempt.
delete(o.verdicts.ready, p.pick.ID)
members[i].Action = string(a.Attempt.ID)
run = append(run, i)
}
if len(run) == 0 {
if o.execHold != nil && o.applyHold != nil {
o.hold(o.execHold, true)
}
return stopped
}
ids := make([]admission.ActionID, len(run))
batch := make([]firewall.BlockMember, len(run))
for k, i := range run {
ids[k], batch[k] = admission.ActionID(members[i].Action), members[i]
}
// A batch the ledger will not record as running dispatches nothing.
intents, granted, executeErr := executeLedger(o.ledger, ids)
// Keep the intent's victim even when execution was already recorded:
// that outcome still finishes through the eviction path.
victims := make(map[admission.ActionID]admission.Victim, len(intents))
for _, in := range intents {
victims[in.Action] = in.Victim
}
for k := range batch {
if v := victims[ids[k]]; v != (admission.Victim{}) {
addr, _ := v.Target.Addr()
batch[k].Victim = firewall.BlockVictim{Action: string(v.Action), IP: addr.String(), ExpiresAt: v.ExpiresAt}
}
}
if executeErr != nil || !granted {
if o.execHold != nil && o.applyHold != nil {
o.hold(o.execHold, true)
}
err = executeErr
if executeErr == nil {
return errors.Join(stopped, admission.ErrTransitionConflict, o.finishAll(batch, admission.DispositionUnknown, admission.VictimUnproven, lead))
}
failErr := o.finishAll(batch, admission.DispositionFailed, admission.VictimSpared, lead)
if _, refused := admission.ReasonOf(err); refused {
return errors.Join(stopped, failErr)
}
return errors.Join(fmt.Errorf("running a block batch: %w", err), stopped, failErr)
}
o.nextBatch = monoNow().Add(batchEvery)
results, err := engine.ApplyBlockBatch(batch)
if err == nil && len(results) != len(batch) {
// Something may have run: no answer proves which members failed.
o.hold(fmt.Errorf("applying a block batch: %d results for %d members", len(results), len(batch)), true)
return errors.Join(stopped, o.finishAll(batch, admission.DispositionUnknown, admission.VictimUnproven, lead))
}
if err != nil {
// An error means nothing was written or sent.
// A confirmation window can begin after the member check. It
// needs no apply proof and must preserve any earlier apply fault.
applyFault := !errors.Is(err, firewall.ErrBlockBatchState) && !errors.Is(err, firewall.ErrConfirmPending) && !errors.Is(err, firewall.ErrBlockRules)
o.hold(fmt.Errorf("applying a block batch: %w", err), applyFault)
return errors.Join(stopped, o.finishAll(batch, admission.DispositionFailed, admission.VictimSpared, lead))
}
var stateErr, applyErr error
var applied bool
for k, r := range results {
applied = applied || r.Effect == firewall.BlockApplied
if errors.Is(r.Err, firewall.ErrBlockBatchState) {
stateErr = r.Err
} else if r.Err != nil && !errors.Is(r.Err, firewall.ErrBlockBatchRejected) {
applyErr = r.Err
}
o.finishes = append(o.finishes, pendingFinish{
id: ids[k], disposition: effectDisposition(r.Effect),
rejected: r.Effect == firewall.BlockFailed && errors.Is(r.Err, firewall.ErrBlockBatchRejected),
existing: r.Effect == firewall.BlockFailed && r.Refusal == firewall.BlockRefusedExisting,
member: batch[k], victim: victimOutcome(r), lead: lead,
})
}
switch {
case stateErr != nil:
// Repair the state first, retaining any independent apply fault.
if applyErr != nil {
o.applyHold = applyErr
}
o.hold(stateErr, false)
case applyErr != nil:
o.hold(applyErr, true)
case o.execHold != nil && o.applyHold != nil && !applied:
o.hold(o.execHold, true)
case o.execHold != nil:
o.release()
}
err = o.finishPending()
// A failed retirement waits for the next turn.
_ = o.retireEvicted()
return errors.Join(stopped, err)
}
// victimOutcome is what r says the batch did to its member's victim, as
// the ledger records it; a member without a victim spared none.
func victimOutcome(r firewall.BlockResult) admission.VictimOutcome {
switch r.Victim {
case firewall.VictimRemoved:
return admission.VictimRemoved
case firewall.VictimUnproven:
return admission.VictimUnproven
}
return admission.VictimSpared
}
// retireEvicted removes the rows of victims whose eviction is committed.
// Mark an admission victim's row before removing it so a reload between
// the calls cannot restore it; an import's row is never marked, since an
// unverified row without an action would never expire. Failed and held
// rows rotate behind the rest; startup is the durable cleanup path for
// admission victims. An import still enforced (a reload or restart
// restored its row) is a legacy block again and is forgotten. State faults
// use the existing execution hold and probe wait. A pass that clears no row
// backs the next one off; a row that leaves or arrives ends the wait.
func (o *Owner) retireEvicted() (err error) {
if len(o.evicted) == 0 || !o.mayRetire() {
return nil
}
if errors.Is(o.execHold, firewall.ErrBlockBatchState) && monoNow().Before(o.probeAt) {
o.noteRetirementRetry(o.probeAt)
return nil
}
if !o.evictedWait.due(monoNow()) {
o.noteRetirementRetry(o.evictedWait.at)
return nil
}
engine := o.opts.Blocks()
if engine == nil {
o.noteRetirementRetry(monoNow().Add(defaultScheduleEvery))
return nil
}
batch := o.evicted[:min(len(o.evicted), firewall.MaxBlockBatchMembers)]
var gone []bool
defer func() {
// An error (including a short answer) proves no row clear.
rest := o.evicted[len(batch):]
for i, a := range batch {
if err != nil || (!gone[i] && !admission.ImportedVictim(admission.ActionID(a.Action))) {
rest = append(rest, a)
}
}
if len(rest) < len(o.evicted) {
o.evictedWait = retryBackoff{}
} else {
o.evictedWait.fail(monoNow(), defaultScheduleEvery, maxProbeWait)
}
o.evicted = rest
if errors.Is(err, firewall.ErrBlockBatchState) {
o.hold(err, false)
}
// Unknown-attempt retirement owns retireErr. Keep these faults
// separate so a healthy pass cannot clear the other pass's fault.
if o.evictedErr != err {
o.evictedErr = err
o.refreshStatus()
}
if len(o.evicted) > 0 {
at := monoNow().Add(defaultScheduleEvery)
if at.Before(o.evictedWait.at) {
at = o.evictedWait.at
}
o.noteRetirementRetry(at)
}
}()
var ids []string
for _, a := range batch {
if !admission.ImportedVictim(admission.ActionID(a.Action)) {
ids = append(ids, a.Action)
}
}
// An import-only batch still needs the safe-state repair; otherwise
// each retirement failure postpones the execution probe that repairs it.
if len(ids) > 0 || errors.Is(o.execHold, firewall.ErrBlockBatchState) {
if _, markErr := engine.MarkBlockActionsUnverified(ids); markErr != nil {
return fmt.Errorf("marking evicted blocks: %w", markErr)
}
}
gone, err = engine.RetireBlockActions(batch)
if err == nil && len(gone) != len(batch) {
err = fmt.Errorf("%d retirement answers for %d evicted blocks", len(gone), len(batch))
}
if err != nil {
return fmt.Errorf("retiring evicted blocks: %w", err)
}
return nil
}
// queueEvicted adds a victim's row for retirement ahead of the rows
// already waiting, so the next pass tries it at once, whatever wait or
// backlog the rows before it built.
func (o *Owner) queueEvicted(a firewall.BlockAction) {
o.evicted = slices.Insert(o.evicted, 0, a)
o.evictedWait = retryBackoff{}
}
// memberOf is p's next attempt as a block member, lasting from lease.
func (o *Owner) memberOf(p execPick, lease time.Time) (firewall.BlockMember, error) {
next, err := admission.NewAttempt(p.pick.ID, p.c.Attempts+1)
if err != nil {
return firewall.BlockMember{}, fmt.Errorf("naming the attempt of pick %s: %w", p.pick.ID, err)
}
expires, err := o.cappedExpiry(p.c, lease)
if err != nil {
return firewall.BlockMember{}, fmt.Errorf("reading the episode of pick %s: %w", p.pick.ID, err)
}
addr, _ := p.c.Key.Target.Addr()
return firewall.BlockMember{Action: string(next.ID), IP: addr.String(), Reason: "CSM auto-block: " + p.c.Check, ExpiresAt: expires}, nil
}
// endRefused ends or defers a pick the engine refused before anything was
// reserved, without a charge, and reports whether it was refused. An
// existing effect ends linked to its block. A link the ledger refuses (no
// history budget for it, or no name) leaves the pick queued, deferred when
// the reason is a deferral; the next check tries again. A refusal the
// owner does not know (a repeated address) leaves the pick queued. Only a
// failed write is an error.
func (o *Owner) endRefused(p execPick, check firewall.BlockCheck) (bool, error) {
switch {
case check.Refusal == firewall.BlockRefusedExisting:
if err := o.endExisting(p, linkOf(check.Existing)); err != nil {
reason, refused := admission.ReasonOf(err)
if !refused {
return true, err
}
if reason.Disposition() == admission.DispositionDeferred {
return true, o.endPick(p, reason)
}
}
return true, nil
case check.Refusal != "":
if reason, known := refusalReasons[check.Refusal]; known {
// The dry-run comparison keeps its own rows; only a live
// refusal is counted here.
if o.forecasting() {
return true, o.endPick(p, reason)
}
return true, o.endCounted(p, reason, 0)
}
return true, nil
}
return false, nil
}
// gateVerdicts asks the panel about the turn's picks before anything is
// reserved and returns those ready to run: the picks with
// a default answer for this attempt. One read-only snapshot answers every
// pick's local checks without giving any pick a slot, so a pick waiting
// for its answer takes no room from a ready one. A refused or already
// enforced pick ends or waits as a batch check would have it, with no
// call; an allowed pick ends uncharged as verdict_allow, whatever the
// capacity; any other pick is asked about and stays queued until its
// answer arrives. Room is the ledger's: it scheduled the pick on a free
// slot or a victim it can claim, which a full set does not rule out. A
// failed write stops the turn.
func (o *Owner) gateVerdicts(engine BlockEngine, picks []execPick, lease time.Time) ([]execPick, error) {
members := make([]firewall.BlockMember, len(picks))
for i, p := range picks {
m, err := o.memberOf(p, lease)
if err != nil {
return nil, err
}
members[i] = m
}
adm, err := engine.BlockAdmissionSnapshot(members, nil)
if err == nil && !answersMatch(adm.Answers, members) {
err = fmt.Errorf("%d answers for %d members", len(adm.Answers), len(members))
o.probeCheck = true
}
if err != nil {
o.hold(fmt.Errorf("checking a block batch: %w", err), false)
return nil, o.deferBlocks(picks, admission.ReasonEngineUnavailable)
}
o.probeCheck = false
if o.execHold != nil && o.applyHold == nil {
o.release()
}
var ready []execPick
for i, p := range picks {
refused, err := o.endRefused(p, adm.Answers[i].BlockCheck)
if err != nil {
return nil, err
}
if refused {
continue
}
r, ok := o.readyVerdict(p.pick.ID, p.c, members[i].IP, members[i].Reason)
switch {
case ok && r.Answer == answerAllow:
err = o.endAllowed(p, r)
case ok:
ready = append(ready, p)
default:
o.askVerdict(p.pick.ID, p.c, members[i].IP, members[i].Reason)
}
if err != nil {
return nil, err
}
}
if len(ready) == 0 && o.execHold != nil && o.applyHold != nil {
o.hold(o.execHold, true)
}
return ready, nil
}
// answersMatch reports whether a snapshot answered exactly members, in
// order.
func answersMatch(answers []firewall.BlockAnswer, members []firewall.BlockMember) bool {
if len(answers) != len(members) {
return false
}
for i, a := range answers {
if a.Action != members[i].Action || a.IP != members[i].IP {
return false
}
}
return true
}
// endAllowed ends a pick the panel allowed: one verdict_allow refusal
// with no reservation, attempt or charge. It is counted and logged once,
// after the ending commits.
func (o *Owner) endAllowed(p execPick, r verdictResult) error {
if _, err := terminateLedger(o.ledger, p.pick.ID, admission.ReasonVerdictAllow); err != nil {
return fmt.Errorf("ending pick %s: %w", p.pick.ID, err)
}
delete(o.verdicts.ready, p.pick.ID)
o.verdicts.tally.add(&o.verdicts.tally.endings)
logVerdictAllow(p.pick.ID, p.c.Check, r)
return nil
}
// logVerdictAllow logs one committed allow; tests capture it. The panel's
// tenant and note arrive escaped and bounded, and never reach the ledger.
var logVerdictAllow = func(id admission.CandidateID, check string, r verdictResult) {
csmlog.Info("verdict callback allowed an automatic block; not blocking",
"candidate", string(id), "check", check, "ip", r.IP, "tenant", r.Tenant, "note", r.Note)
}
// ready reports whether automatic execution may run this turn.
// Without an engine it is held. While held, one probe runs per wait:
// state repair is followed by a read-only snapshot proof, including on an
// idle turn. Apply faults keep their wait through a successful snapshot
// and member check until an apply succeeds. A newly available engine can
// serve at once. When checking members next, keep the hold until that
// check succeeds too: a probe alone cannot reset a failing turn's wait.
func (o *Owner) ready(engine BlockEngine, checking bool) bool {
if !o.mayExecute() {
return false
}
if engine == nil {
// Holding again once the wait has passed spaces the next selecting
// turn of a missing engine, as a failed probe would.
if !errors.Is(o.execHold, errNoEngine) || !monoNow().Before(o.probeAt) {
o.hold(errNoEngine, false)
}
return false
}
switch {
case o.execHold == nil:
return true
case errors.Is(o.execHold, errNoEngine):
if o.applyHold == nil {
o.release()
return true
}
// A returning engine does not prove that an earlier apply fault
// recovered. Let it probe now, retaining the one-member hold.
o.execHold = o.applyHold
case monoNow().Before(o.probeAt):
return false
case errors.Is(o.execHold, firewall.ErrBlockBatchState):
if _, err := engine.MarkBlockActionsUnverified(nil); err != nil {
o.hold(err, false)
return false
}
}
if err := engine.ProbeBlockState(); err != nil {
o.hold(err, false)
return false
}
if o.applyHold == nil && !o.probeCheck && !checking {
o.release()
} else {
if o.applyHold != nil {
o.execHold = o.applyHold
o.publishExecutionHold()
}
o.probeAt = monoNow().Add(o.probeWait)
}
return true
}
// hold stops automatic execution for cause until a probe succeeds; each
// hold while already held doubles the wait for the next probe, up to a
// minute.
func (o *Owner) hold(cause error, apply bool) {
switch {
case o.execHold == nil:
o.probeWait = firstProbeWait
case o.probeWait < maxProbeWait:
o.probeWait = min(2*o.probeWait, maxProbeWait)
}
if apply {
o.applyHold = cause
}
o.execHold, o.probeAt = cause, monoNow().Add(o.probeWait)
o.publishExecutionHold()
}
func (o *Owner) release() {
o.execHold, o.probeAt, o.probeWait = nil, time.Time{}, 0
o.applyHold, o.probeCheck = nil, false
o.publishExecutionHold()
}
// publishExecutionHold preserves the age of the cached ledger sample.
func (o *Owner) publishExecutionHold() {
previous := o.current.Load()
snapshot, owner := *previous, *previous.Owner
owner.ExecutionHeld, owner.ExecutionHeldBy = "", ""
if cause := o.heldCause(); cause != nil {
owner.ExecutionHeld, owner.ExecutionHeldBy = cause.Error(), o.heldBy()
}
owner.RetirementHeldBy = ""
if snapshot.Ledger != nil && snapshot.Ledger.Unresolved.Unknown > 0 {
owner.RetirementHeldBy = o.retirementHeldBy()
}
snapshot.Owner = &owner
o.current.Store(&snapshot)
}
// isolate keeps a member the kernel rejected from failing healthy ones
// again: while a pick's last batch was rejected, the turn
// runs that pick alone and leaves the others queued for later turns.
// Only queued retries remain isolated; terminal, displaced and aged-out
// candidates leave the index at the next successful queue publication.
func (o *Owner) isolate(picks []execPick) []execPick {
for _, p := range picks {
if _, ok := o.isolated[p.pick.ID]; ok {
return []execPick{p}
}
}
return picks
}
// A displaced, terminal or aged-out candidate cannot retry alone, nor wait
// for a valid policy.
func (o *Owner) forgetIsolated(snap *admission.QueueSnapshot) {
if len(o.isolated) == 0 && len(o.policyHeld) == 0 {
return
}
live := make(map[admission.CandidateID]bool, len(snap.Items))
for _, item := range snap.Items {
live[admission.CandidateID(item.Key)] = true
}
for id, expires := range o.isolated {
if !live[id] || !o.now.Before(expires) {
delete(o.isolated, id)
}
}
for id := range o.policyHeld {
if !live[id] {
delete(o.policyHeld, id)
}
}
}
// effectDisposition is the outcome the ledger records for an effect; one
// the owner does not know proves nothing.
func effectDisposition(e firewall.BlockEffect) admission.Disposition {
switch e {
case firewall.BlockApplied:
return admission.DispositionApplied
case firewall.BlockFailed:
return admission.DispositionFailed
}
return admission.DispositionUnknown
}
// endExisting ends a pick before it is reserved because a proven block
// already enforces its effect, linked to that block.
func (o *Owner) endExisting(p execPick, link admission.EffectLink) error {
if _, err := terminateExistingLedger(o.ledger, p.pick.ID, link); err != nil {
return fmt.Errorf("ending pick %s: %w", p.pick.ID, err)
}
return nil
}
// linkOf is the ledger's link to the block the engine named.
func linkOf(b firewall.ExistingBlock) admission.EffectLink {
return admission.EffectLink{Effect: b.ID, ExpiresAt: b.ExpiresAt}
}
// endPick defers or ends a pick before it is reserved.
func (o *Owner) endPick(p execPick, reason admission.Reason) error {
var err error
switch reason.Disposition() {
case admission.DispositionDeferred:
// Deferring again for the same reason changes nothing but still
// commits; a long hold would pay that per pick every turn.
if p.c.Reason == reason {
return nil
}
_, err = o.ledger.Defer(p.pick.ID, reason)
case admission.DispositionRefused, admission.DispositionWithheld, admission.DispositionDropped:
_, err = terminateLedger(o.ledger, p.pick.ID, reason)
default:
err = fmt.Errorf("reason %s ends nothing", reason)
}
if err != nil {
return fmt.Errorf("ending pick %s: %w", p.pick.ID, err)
}
return nil
}
// endCounted ends or defers p as endPick does and counts a terminal ending
// as refused in the hourly comparison, in the hour it ended. The count
// follows the write, so a failed ending counts nothing, and a deferral is
// not a refusal.
func (o *Owner) endCounted(p execPick, reason admission.Reason, cause admission.Cause) error {
if err := o.endPick(p, reason); err != nil {
return err
}
switch reason.Disposition() {
case admission.DispositionRefused, admission.DispositionWithheld, admission.DispositionDropped:
c := p.c
o.compare.addLedgerRefusalAt(compareKey{entry: c.Entry, check: c.Check, kind: c.Key.Kind, decision: decisionRefused, reason: reason, cause: cause}, o.now)
}
return nil
}
func (o *Owner) deferBlocks(picks []execPick, reason admission.Reason) error {
for _, p := range picks {
if err := o.endPick(p, reason); err != nil {
return err
}
}
return nil
}
func (o *Owner) finishAll(members []firewall.BlockMember, d admission.Disposition, v admission.VictimOutcome, lead time.Duration) error {
for _, member := range members {
o.finishes = append(o.finishes, pendingFinish{
id: admission.ActionID(member.Action), disposition: d,
member: member, victim: v, lead: lead,
})
}
return o.finishPending()
}
// finishPending attempts every outstanding write even if another fails.
// Only writes retry; the engine is never called again for these attempts.
func (o *Owner) finishPending() error {
var err error
pending := o.finishes[:0]
for _, outcome := range o.finishes {
var c admission.Candidate
var a admission.AttemptRecord
var finishErr error
linked := false
evicting := outcome.member.Victim != (firewall.BlockVictim{})
if evicting {
// A link must not substitute for the intent's victim outcome.
c, a, finishErr = finishEvictionLedger(o.ledger, outcome.id, outcome.disposition, outcome.victim, outcome.lead)
}
if finishErr == nil && outcome.existing {
// Finishing can retry long after the apply-time proof expires.
// Only a fresh proof may end the pick as already covered. A
// failed link leaves the eviction outcome intact; a member
// without a victim records its plain failure below instead.
link, proofErr := o.proveExisting(outcome.member)
switch {
case proofErr != nil:
err = errors.Join(err, fmt.Errorf("checking existing effect for %s: %w", outcome.id, proofErr))
case link != (admission.EffectLink{}):
linkedC, linkedA, linkErr := finishExistingLedger(o.ledger, outcome.id, link)
if linkErr != nil {
// A refused link is expected when history has no room.
if _, refused := admission.ReasonOf(linkErr); !refused {
err = errors.Join(err, fmt.Errorf("ending %s as an existing effect: %w", outcome.id, linkErr))
}
} else {
c, a = linkedC, linkedA
linked = true
}
}
}
if !evicting && !linked {
c, a, finishErr = finishLedger(o.ledger, outcome.id, outcome.disposition, outcome.lead)
}
if finishErr != nil {
err = errors.Join(err, fmt.Errorf("finishing %s: %w", outcome.id, finishErr))
// A damaged record, an attempt already ended another way or a
// refused outcome cannot be written by retrying: the attempt
// stays unresolved and holds its target until recovery.
if _, refused := admission.ReasonOf(finishErr); !refused && !errors.Is(finishErr, admission.ErrCorruptRecord) && !errors.Is(finishErr, admission.ErrTransitionConflict) {
pending = append(pending, outcome)
}
continue
}
// Only a verified eviction retires the victim's row; after an
// unknown outcome the row stays until the victim's expiry.
if evicting && !linked && outcome.victim == admission.VictimRemoved && a.State == admission.StateVerified {
o.queueEvicted(firewall.BlockAction{Action: outcome.member.Victim.Action, IP: outcome.member.Victim.IP})
}
id := a.Attempt.Candidate
if c.State == admission.StateUnknown {
o.noteRetirement(c.ExpiresAt)
}
if c.State == admission.StateQueued && outcome.rejected {
if o.isolated == nil {
o.isolated = map[admission.CandidateID]time.Time{}
}
// Isolation lasts the block's lifetime in admission time.
expires := c.ExpiresAt.Add(-max(outcome.lead, 0))
if c.AgeOut.Before(expires) {
expires = c.AgeOut
}
o.isolated[id] = expires
} else {
delete(o.isolated, id)
}
}
o.finishes = pending
return err
}
// proveExisting checks member again and links the block that now proves
// it is already enforced; zero when nothing does (the engine names a block
// only for an existing effect).
func (o *Owner) proveExisting(member firewall.BlockMember) (admission.EffectLink, error) {
engine := o.opts.Blocks()
if engine == nil {
return admission.EffectLink{}, errNoEngine
}
checks, err := engine.CheckBlockBatchEffects([]firewall.BlockMember{member})
if err != nil {
return admission.EffectLink{}, err
}
if len(checks) != 1 {
return admission.EffectLink{}, fmt.Errorf("%d answers for one existing effect", len(checks))
}
return linkOf(checks[0].Existing), nil
}
package admissionowner
import (
"crypto/rand"
"errors"
"fmt"
"io"
"slices"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/firewall"
)
// sessionRand is where forecast session IDs come from; tests make it fail.
var sessionRand io.Reader = rand.Reader
// forecasting reports whether the turn forecasts its picks instead of
// running them: an executable owner under an installed dry-run policy.
func (o *Owner) forecasting() bool {
return o.executable() && o.policyErr == nil && o.policy.DryRun
}
// beginForecast starts a forecast session unless one runs.
// The turn has finished the live outcomes already returned and made a
// reading, so the session copies the committed ceiling and its charges,
// spent credit included; nothing is copied again while it runs. A failure
// holds forecasting and fails the turn.
func (o *Owner) beginForecast() error {
if o.shadow != nil {
return nil
}
id, err := newSessionID(sessionRand)
if err != nil {
o.noteForecastError(forecastSessionID)
return fmt.Errorf("naming a forecast session: %w", err)
}
ceiling, charges, err := o.ledger.ForecastSeed()
if err != nil {
o.noteForecastError(forecastLedgerUnavailable)
return fmt.Errorf("reading the ceiling for a forecast: %w", err)
}
s, err := newShadow(id, time.Now(), o.forecastReset, ceiling, charges)
if err != nil {
o.noteForecastError(forecastLedgerUnavailable)
return fmt.Errorf("starting a forecast: %w", err)
}
o.shadow, o.forecastReset = s, resetModeChange
o.forecastErr, o.forecastErrAt = "", time.Time{}
return nil
}
// dropForecast ends a session whose model cannot go on. The next session
// starts from the real state and names the failure as its reset.
func (o *Owner) dropForecast() {
o.endSession(resetModelError)
o.noteForecastError(forecastModelError)
}
// endSession discards the forecast session, keeping its identity for
// status, and names why the next one starts.
func (o *Owner) endSession(next resetReason) {
if s := o.shadow; s != nil {
o.out.last = sessionInfo{id: s.id, since: s.since, reset: s.reset}
}
o.shadow, o.forecastReset = nil, next
}
// noteForecastError records why forecasting failed, for status.
func (o *Owner) noteForecastError(e forecastError) {
o.forecastErr, o.forecastErrAt = e, time.Now().UTC()
}
// forecast serves the turn under a dry-run policy. The
// selection ignores live budgets and defers nothing for capacity. A
// block_ip pick is forecast against the session's model; another firewall
// kind ends as not_modelled; a challenge keeps its comparison preview.
// Nothing is reserved, charged or applied, and the engine is only read.
func (o *Owner) forecast() (err error) {
if err = o.refuseDisabled(); err != nil {
o.noteForecastError(forecastLedgerUnavailable)
return err
}
if err = o.beginForecast(); err != nil {
return err
}
// A turn that ends without naming another cause clears the error; one
// that fails anywhere else could not use the ledger.
o.forecastErr = ""
defer func() {
if err != nil && o.forecastErr == "" {
o.noteForecastError(forecastLedgerUnavailable)
}
}()
// Examples of committed endings are written even when a later pick
// stops the turn.
defer o.flushExamples()
picks, err := previewLedger(o.ledger, o.answers())
if err != nil {
return fmt.Errorf("selecting dry-run work: %w", err)
}
var blocks []execPick
for _, p := range picks {
c, readErr := o.ledger.Candidate(p.ID)
if readErr != nil {
return fmt.Errorf("reading pick %s: %w", p.ID, readErr)
}
pick := execPick{pick: p, c: c}
switch {
case c.Key.Kind == admission.KindBlockIP:
blocks = append(blocks, pick)
case c.Key.Kind == admission.KindChallenge:
// Until the ledger starts live the selection passes over a
// challenge the live history budget cannot pay for; one its
// reservation still cannot take waits as a schedule defers
// it. After the live start its preview spends no history.
err = o.preview(pick)
if reason, refused := admission.ReasonOf(err); refused {
err = o.endPick(pick, reason)
} else if errors.Is(err, admission.ErrNotReady) || errors.Is(err, admission.ErrLaneIneligible) {
err = nil
}
case o.degraded:
// A degraded clock freezes every prediction.
o.noteForecastError(forecastClockDegraded)
err = o.endPick(pick, admission.ReasonEngineUnavailable)
default:
err = o.endForecast(pick, admission.PredictNotModelled, nil)
}
if err != nil {
return err
}
}
return o.forecastBlocks(blocks)
}
// forecastBlocks forecasts the turn's block_ip picks in scheduler order.
// One read-only snapshot answers their local checks and the
// real temporary occupancy, with the real blocks of forecast addresses. A
// refused target ends or waits as a live check has it. A real block that
// already covers the address ends the pick as existing_actual, and a
// usable panel allow as verdict_allow, before any modelled capacity is
// judged. A covering or shorter forecast block, or a modelled set with
// neither a free slot in a partition the pick may occupy nor a strictly
// lower-tier block it may replace nor an equal-tier block it may rotate
// out, ends it without a call. The turn's picks share the modelled set
// as a live schedule cycle's queued work does.
// With the callback on, any other pick is asked about first and waits
// for its answer with no forecast change; then the forecast bound or the
// modelled ceiling can still end it. Only a capacity
// admission, with or without a replacement or rotation, changes the model. Without an engine, snapshot or sound clock
// the picks wait, with no call or forecast change.
func (o *Owner) forecastBlocks(picks []execPick) error {
if len(picks) == 0 {
return nil
}
engine := o.opts.Blocks()
if o.degraded || engine == nil {
cause := forecastEngineUnavailable
if o.degraded {
cause = forecastClockDegraded
}
o.noteForecastError(cause)
o.shadow.adm = nil
return o.deferBlocks(picks, admission.ReasonEngineUnavailable)
}
lease := engine.LeaseNow()
o.shadow.expire(lease)
members := make([]firewall.BlockMember, len(picks))
for i, p := range picks {
m, err := o.memberOf(p, lease)
if err != nil {
return err
}
members[i] = o.forecastMember(p.c, m)
}
adm, err := engine.BlockAdmissionSnapshot(members, o.shadow.ips())
if err != nil || !answersMatch(adm.Answers, members) {
cause := forecastSnapshotMismatch
if err != nil {
cause = forecastEngineUnavailable
}
o.noteForecastError(cause)
o.shadow.adm = nil
return o.deferBlocks(picks, admission.ReasonEngineUnavailable)
}
// The records of a live period keep their slots until they end.
if o.shadow.effects, err = o.ledger.EffectTally(); err != nil {
o.shadow.adm = nil
return fmt.Errorf("reading the ledger's set slots: %w", err)
}
occupancy := adm
occupancy.Answers = nil
o.shadow.adm = &occupancy
o.shadow.share(adm, o.now, waitingOf(picks))
for i, p := range picks {
m, check := members[i], adm.Answers[i].BlockCheck
if check.Refusal != "" && check.Refusal != firewall.BlockRefusedExisting {
if _, err = o.endRefused(p, check); err != nil {
return err
}
continue
}
// An earlier pick can replace this address in the same turn.
// Its committed boundary must also constrain this return.
m = o.forecastMember(p.c, m)
// The snapshot checked a longer lifetime. Shortening it can
// leave less than the engine's minimum encodable timeout.
if m.ExpiresAt.Sub(lease) < time.Millisecond {
if err = o.endPick(p, admission.ReasonStale); err != nil {
return err
}
continue
}
rank := rankOf(p, m, false)
r, answered := o.readyVerdict(p.pick.ID, p.c, m.IP, m.Reason)
var change *shadowChange
pred := o.shadow.covering(m.IP, m.ExpiresAt)
switch {
case check.Refusal == firewall.BlockRefusedExisting:
pred = admission.PredictExistingActual
case answered && r.Answer == answerAllow:
if err = o.endAllowed(p, r); err != nil {
return err
}
continue
case pred == 0:
// Existing coverage and a usable allow need no rotation proof.
if rank.fresh, err = o.freshPick(p, m.IP); err != nil {
return fmt.Errorf("reading the episode of pick %s: %w", p.pick.ID, err)
}
pred = o.shadow.wait(adm, lanesOf(p.pick), rank, o.now)
}
if pred == 0 {
if o.policy.VerdictEnabled && !answered {
o.askVerdict(p.pick.ID, p.c, m.IP, m.Reason)
continue
}
if pred, change, err = o.shadow.admit(m.IP, p.c.Key.Kind, m.ExpiresAt, lanesOf(p.pick), o.now, adm, rank); err != nil {
o.noteForecastError(forecastModelError)
return fmt.Errorf("forecasting pick %s: %w", p.pick.ID, err)
}
}
if err = o.endForecast(p, pred, change); err != nil {
return err
}
}
return nil
}
// forecastMember caps a first attempt at the end of the forecast block
// it returns to. Retries keep their already reserved absolute expiry.
func (o *Owner) forecastMember(c admission.Candidate, m firewall.BlockMember) firewall.BlockMember {
if until, ok := o.shadow.answered[m.IP]; ok && c.Attempts == 0 && until.Before(m.ExpiresAt) {
m.ExpiresAt = until
}
return m
}
// rankOf ranks the forecast block of pick p, block member m, as a victim:
// the pick's tier, the action of its next attempt and its address, with
// its scope and whether it is fresh.
func rankOf(p execPick, m firewall.BlockMember, fresh bool) shadowRank {
return shadowRank{tier: p.pick.Tier, action: admission.ActionID(m.Action), target: p.c.Key.Target, scope: p.c.Scope.Key(), fresh: fresh}
}
// freshPick reports whether forecast pick p for ip may rotate an
// equal-tier block: its episode row says so, as a live
// schedule reads it, and no forecast block of ip left the model before
// that block's expiry, so its episode has had no forecast response.
func (o *Owner) freshPick(p execPick, ip string) (bool, error) {
if _, answered := o.shadow.answered[ip]; answered {
return false, nil
}
return freshLedger(o.ledger, p.c)
}
// waitingOf is the scopes of a turn's block picks by the partitions they
// may occupy, as a live schedule cycle counts its queued block work:
// general, and reserved too for work with a reserved lane.
func waitingOf(picks []execPick) map[admission.Partition][]string {
out := map[admission.Partition][]string{}
for _, p := range picks {
for _, lane := range lanesOf(p.pick) {
part := admission.PartitionOf(lane)
out[part] = append(out[part], p.c.Scope.Key())
}
}
for part := range out {
slices.Sort(out[part])
out[part] = slices.Compact(out[part])
}
return out
}
// lanesOf is the order a forecast charges p: the lane the selection served
// it on, then the other allowance it is eligible for. General-only work
// never borrows reserved credit.
func lanesOf(p admission.Pick) []admission.Lane {
switch {
case p.Lane != admission.LaneGeneral:
return []admission.Lane{p.Lane, admission.LaneGeneral}
case p.Reserved != 0:
return []admission.Lane{admission.LaneGeneral, p.Reserved}
}
return []admission.Lane{admission.LaneGeneral}
}
// endForecast ends p as a dry run with its prediction and installs the
// decision in the forecast and the comparison once the ending commits, so
// a retried ending never counts or charges twice. A refusal the
// ending meets defers or ends p as a reservation would. A failed write
// stops the turn and leaves the forecast as it was.
func (o *Owner) endForecast(p execPick, pred admission.Prediction, change *shadowChange) error {
_, changed, err := endDryRunLedger(o.ledger, p.pick.ID)
if reason, refused := admission.ReasonOf(err); refused {
return o.endPick(p, reason)
}
switch {
case errors.Is(err, admission.ErrNotReady):
return nil
case err != nil:
return fmt.Errorf("ending pick %s as a dry run: %w", p.pick.ID, err)
case changed:
delete(o.verdicts.ready, p.pick.ID)
if change != nil {
o.noteExample(p, change)
}
o.noteSummary(p, pred, change)
o.shadow.commit(pred, change, monoNow())
c := p.c
o.compare.addAt(compareKey{entry: c.Entry, check: c.Check, kind: c.Key.Kind, decision: decisionDryRun, prediction: pred}, o.now)
}
return nil
}
package admissionowner
import (
"fmt"
"time"
"github.com/pidginhost/csm/internal/firewall"
)
// heldEvery spaces bounded held checks. Each batch gets two answered
// calls before the ledger advances to the next batch.
const heldEvery = time.Minute
// observeHeld asks the firewall, at most once every heldEvery and only
// while the ledger counts a verified block, whether each such block is
// still held, and hands the answer to the ledger. A block two consecutive
// answers no longer hold left early (an operator unblock, undo or flush,
// or a lost kernel element): the ledger marks it gone and stops counting
// its slot. A fault marks nothing and waits for the next
// check; like eviction cleanup it never fails the turn.
func (o *Owner) observeHeld() error {
if !o.mayRetire() || monoNow().Before(o.heldAt) {
return nil
}
asked, err := o.ledger.HeldEffects(firewall.MaxHeldBlockActions)
if err == nil && len(asked) == 0 {
o.heldAt = time.Time{}
return nil
}
o.heldAt = monoNow().Add(heldEvery)
if err != nil {
return err
}
engine := o.opts.Blocks()
if engine == nil {
return nil
}
actions := make([]firewall.BlockAction, len(asked))
for i, h := range asked {
addr, _ := h.Target.Addr()
actions[i] = firewall.BlockAction{Action: string(h.Action), IP: addr.String()}
}
held, err := engine.HeldBlockActions(actions)
if err != nil {
return fmt.Errorf("checking held blocks: %w", err)
}
// The ledger refuses a short answer whole.
gone, err := o.ledger.ObserveHeld(asked, held)
for _, h := range gone {
// An element lost outside CSM leaves its row behind. Retire it as
// an evicted victim's row, so a firewall reload cannot restore what
// the ledger no longer counts.
addr, _ := h.Target.Addr()
o.queueEvicted(firewall.BlockAction{Action: string(h.Action), IP: addr.String()})
}
if len(gone) > 0 {
// Do not leave restorable rows waiting for the next held check.
// Cleanup retains failures and schedules its own retry.
_ = o.retireEvicted()
}
return err
}
package admissionowner
import (
"fmt"
"slices"
"strings"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/store"
)
// ackNotices acknowledges delivered notices; tests make it fail.
var ackNotices = (*store.AdmissionLedger).AckNotices
// maxRetryWait caps the wait between attempts of failing work.
const maxRetryWait = time.Hour
// noticeMessages name each notice kind as its notification does.
var noticeMessages = map[admission.NoticeKind]string{
admission.NoticeWithheld: "Critical automatic response withheld",
admission.NoticeWithheldWarning: "Compromise response withheld",
admission.NoticeCapacity: "Automatic response capacity exhausted",
admission.NoticeCriticalSummary: "Critical automatic response gaps summary",
admission.NoticeAppliedSummary: admission.AppliedSummaryMessage,
}
// partialContainmentMessage names a partial-containment record.
const partialContainmentMessage = "Critical automatic response is partial containment: outbound-only traffic to the address still leaves"
// sender delivers the ledger's due notices, and the notice of a stopped
// ingress, through the daemon's independent health path. It runs on its
// own goroutine, since delivery may wait on mail or a webhook, and alone
// serializes reading, delivery and acknowledgement; the owner goroutine
// makes the acknowledgement.
type sender struct {
o *Owner
queue *queuehealth.Sampled
// unacked holds a delivery whose acknowledgement failed: it is
// acknowledged before another ledger notice batch, never delivered
// again. The independent stopped-ingress notice still runs.
unacked []admission.NoticeAck
unackedStarted bool
// unackedRead is the admission time the unacknowledged delivery was
// read at; ackedRead keeps it per key once acknowledged. Events after
// that read were not delivered, though they may predate the
// acknowledgement.
unackedRead time.Time
ackedRead map[admission.NoticeKey]time.Time
delivered uint64
// announced is set once the current stop of the ingress was sent.
announced bool
announcedSince time.Time
// A failed stop notification must survive recovery until delivery.
pendingStop *alert.Finding
pendingSince time.Time
ledgerPending int
// summaryWaiting is set while a dry-run summary waits for delivery.
summaryWaiting bool
// The independent stop alert must not inherit a ledger delivery's wait,
// and its successful delivery must not reset that ledger retry. Live
// and preview records go to different stores and back off apart.
ledgerRetry, previewRetry retryBackoff
// ledgerStarted binds the retry delay to the notice stream: clearing
// preview records at live start also discards their delivery failures.
ledgerStarted bool
stopRetry retryBackoff
done chan struct{}
}
// retryBackoff spaces the attempts of work that keeps failing: the next
// two attempts run at once, then each waits twice as long as the one before,
// from base up to ceiling. A success restores prompt attempts.
type retryBackoff struct {
failures int
at time.Time
}
func (r *retryBackoff) due(now time.Time) bool { return !now.Before(r.at) }
func (r *retryBackoff) fail(now time.Time, base, ceiling time.Duration) {
r.failures++
if r.failures > 2 {
r.at = now.Add(min(base<<min(r.failures-2, 20), ceiling))
}
}
func (s *sender) run() {
defer close(s.done)
t := time.NewTicker(s.o.opts.NoticeEvery)
defer t.Stop()
for {
select {
case <-s.o.done:
return
case <-t.C:
s.cycle()
}
}
}
func (s *sender) cycle() {
if s.o.stopping.Load() {
return
}
if s.o.opts.Deliver != nil {
s.announceStop()
}
s.sendSummary()
if s.o.opts.Deliver == nil {
return
}
if len(s.unacked) > 0 {
if err := s.o.do(func() error { return ackNoticeSnapshot(s.o.ledger, s.unackedStarted, s.unacked) }); err != nil {
return
}
s.noteAcked()
s.ledgerPending = 0
s.observe()
}
var l *store.AdmissionLedger
var live, started bool
var due []admission.NoticeRecord
var spans []liveSpan
var read time.Time
err := s.o.do(func() error {
l, live, read = s.o.ledger, s.o.executing(), s.o.now
if l == nil {
return nil
}
if !liveExecution {
return nil
}
s.o.noteMode()
spans = slices.Clone(s.o.liveSpans)
started = l.LiveStarted()
// The payload and its mode belong to the same owner boundary.
if live && !started {
return nil
}
var readErr error
due, readErr = l.PendingNotices()
if readErr == nil && len(due) > 0 && read.IsZero() {
// Startup can deliver saved notices before its first valid
// clock reading. Keep the persisted read boundary as well.
var clock admission.Clock
clock, readErr = store.ReadAdmissionClock(s.o.opts.DB)
read = clock.Now()
}
return readErr
})
if err != nil || l == nil {
return
}
if !liveExecution {
due, err = l.PendingNotices()
}
if err != nil {
return
}
if started != s.ledgerStarted {
s.ledgerRetry, s.previewRetry = retryBackoff{}, retryBackoff{}
s.ledgerStarted = started
}
s.ledgerPending = len(due)
s.observe()
if len(due) == 0 {
return
}
// Each record goes out in the mode it was raised in: live records
// first, then previews. A failed delivery leaves its group and those
// after it due.
now := time.Now()
var acks []admission.NoticeAck
for _, preview := range []bool{false, true} {
var findings []alert.Finding
var group []admission.NoticeAck
for _, r := range due {
if (liveExecution && raisedLive(r, spans, s.ackedRead[r.Key])) == preview {
continue
}
findings = append(findings, noticeFinding(r, now, !preview))
group = append(group, admission.NoticeAck{Key: r.Key, First: r.First, Count: r.Count})
}
if len(findings) == 0 {
continue
}
retry := &s.ledgerRetry
if preview {
retry = &s.previewRetry
}
if !s.deliver(retry, findings, preview) {
break
}
acks = append(acks, group...)
}
if len(acks) == 0 {
return
}
s.delivered += uint64(len(acks))
s.unacked, s.unackedStarted, s.unackedRead = acks, started, read
s.observe()
if err = s.o.do(func() error { return ackNoticeSnapshot(l, started, acks) }); err != nil {
return
}
s.noteAcked()
s.ledgerPending = len(due) - len(acks)
s.observe()
}
// noteAcked ends the unacknowledged delivery, keeping when each of its
// keys was read.
func (s *sender) noteAcked() {
if s.ackedRead == nil {
s.ackedRead = make(map[admission.NoticeKey]time.Time)
}
for _, a := range s.unacked {
s.ackedRead[a.Key] = s.unackedRead
}
s.unacked = nil
}
func (s *sender) observe() {
depth := s.ledgerPending
if s.pendingStop != nil {
depth++
}
if s.summaryWaiting {
depth++
}
s.queue.Observe(deliveryNow(), depth, s.delivered)
}
// announceStop sends one notice when the ingress stops admitting, naming
// the cause the owner recorded. The ledger may be the
// damaged part, so nothing here writes it. It warns while legacy blocking
// enforces and admission previews; while the owner executes, a stop
// stops automatic blocking and the notice is Critical.
func (s *sender) announceStop() bool {
if s.pendingStop != nil && s.sendStop() {
return true
}
h := admission.IngressHealth{}
cause := "the owner has not started"
// Read stop state and its cause together: a recovery between them
// must not report a healthy ingress as stopped.
if err := s.o.do(func() error {
if s.o.ingress != nil {
h = s.o.ingress.Health()
}
for _, err := range []error{s.o.startErr, s.o.drainErr, s.o.tickErr, s.o.snapshotErr} {
if err != nil {
cause = err.Error()
break
}
}
return nil
}); err != nil || s.o.stopping.Load() {
return false
}
if h.Admitting {
s.announced = false
return false
}
if s.announced && s.announcedSince.Equal(h.StoppedSince) {
return false
}
s.pendingStop = &alert.Finding{
Check: "auto_response_withheld", Severity: alert.Warning, Message: "Automatic response admission preview stopped; existing blocking is unaffected",
Details: fmt.Sprintf("since=%s critical_refused=%d cause=%s", h.StoppedSince.UTC().Format(time.RFC3339), h.CriticalRefused, cause),
Timestamp: time.Now(),
}
s.pendingSince = h.StoppedSince
return s.sendStop()
}
func (s *sender) sendStop() bool {
var live bool
if err := s.o.do(func() error { live = s.o.executing(); return nil }); err != nil {
return true
}
// A pending delivery can outlive a policy change. Keep its identity
// and details, but describe the mode in which it is sent.
s.pendingStop.Severity = alert.Warning
s.pendingStop.Message = "Automatic response admission preview stopped; existing blocking is unaffected"
if live {
s.pendingStop.Severity = alert.Critical
s.pendingStop.Message = "Automatic response admission stopped; automatic blocks are not applied"
}
s.observe()
if !s.deliver(&s.stopRetry, []alert.Finding{*s.pendingStop}, false) {
return true
}
s.announced, s.announcedSince = true, s.pendingSince
s.pendingStop = nil
s.delivered++
s.observe()
return false
}
// deliver sends findings unless the retry of a failing delivery is not yet
// due. Every attempt reaches every channel and history, those that accepted
// the last attempt included, so a failing one is backed off from the
// default notice period. A delivery restores prompt retries for its path.
func (s *sender) deliver(retry *retryBackoff, findings []alert.Finding, preview bool) bool {
if s.o.stopping.Load() || !retry.due(deliveryNow()) {
return false
}
if err := s.o.opts.Deliver(findings, preview); err != nil {
retry.fail(deliveryNow(), defaultNoticeEvery, maxRetryWait)
return false
}
*retry = retryBackoff{}
return true
}
// liveSpan is an admission time interval in which this process executed
// block picks; a zero until is still open.
type liveSpan struct{ since, until time.Time }
// noteMode opens or closes the live span at the owner's admission time
// when executing() changed. Records older than the process take the mode
// it started in. A span boundary shares its time with the events beside
// it, so an event at a flip counts as live.
func (o *Owner) noteMode() {
// Keep the initial mode for restored notices. A later policy cannot
// bound its span until startup gets its first admission clock reading.
if o.modeNoted && o.now.IsZero() {
return
}
open := len(o.liveSpans) > 0 && o.liveSpans[len(o.liveSpans)-1].until.IsZero()
switch live := o.executing(); {
case live && !open:
since := o.now
if !o.modeNoted {
since = time.Time{}
}
o.liveSpans = append(o.liveSpans, liveSpan{since: since})
case !live && open:
o.liveSpans[len(o.liveSpans)-1].until = o.now
}
o.modeNoted = true
}
// raisedLive reports whether r is a live notice: its key comes only from a
// real attempt (an outcome, or the applied summary), or an event it has not
// delivered may fall in one of spans. Undelivered events are no older than
// read, the admission time this process read r's last acknowledged
// delivery at, or without one than that acknowledgement (Sent).
func raisedLive(r admission.NoticeRecord, spans []liveSpan, read time.Time) bool {
if r.Key.Outcome != 0 || r.Key.Kind == admission.NoticeAppliedSummary {
return true
}
from, delivered := r.First, r.Sent
if !read.IsZero() {
delivered = read
}
if delivered.After(from) {
from = delivered
}
for _, s := range spans {
if !r.Last.Before(s.since) && (s.until.IsZero() || !from.After(s.until)) {
return true
}
}
return false
}
// noticeFinding is the record of one notice: its registered check, and in
// its details the events no delivery covered, their key and their example
// candidates. Legacy blocking enforces while admission previews, so a
// preview record warns, whatever its kind's severity. While the owner
// executes (live) a gap is an outage of automatic blocking: the record
// carries its kind's severity and no preview prefix.
func noticeFinding(r admission.NoticeRecord, now time.Time, live bool) alert.Finding {
var b strings.Builder
fmt.Fprintf(&b, "events=%d first=%s last=%s", r.Unsent(), r.First.UTC().Format(time.RFC3339), r.Last.UTC().Format(time.RFC3339))
if r.Key.Kind.Keyed() && r.Key.Fixed() {
b.WriteString(" overflow=true")
}
if r.Key.Reason != 0 {
b.WriteString(" reason=" + r.Key.Reason.String())
}
if r.Key.Outcome != 0 {
b.WriteString(" outcome=" + r.Key.Outcome.String())
}
if r.Key.Check != "" {
b.WriteString(" check=" + r.Key.Check)
}
if r.Key.Effect != 0 {
b.WriteString(" effect=" + r.Key.Effect.String())
}
if len(r.Examples) > 0 {
ids := make([]string, len(r.Examples))
for i, e := range r.Examples {
ids[i] = string(e.Candidate)
}
b.WriteString(" examples=" + strings.Join(ids, ","))
}
message := noticeMessages[r.Key.Kind]
if r.Key.Partial() {
message = partialContainmentMessage
}
if live {
return alert.Finding{Check: r.Key.Kind.Check(), Severity: alertSeverity(r.Key.Kind), Message: message, Details: b.String(), Timestamp: now}
}
return alert.Finding{Check: r.Key.Kind.Check(), Severity: alert.Warning, Message: "Admission preview: " + message, Details: b.String(), Timestamp: now}
}
// alertSeverity is the severity a notice of k alerts with.
func alertSeverity(k admission.NoticeKind) alert.Severity {
if k.Severity() == admission.SeverityCritical {
return alert.Critical
}
return alert.Warning
}
// ackNoticeSnapshot runs on the owner. Clearing preview notices at cutover
// invalidates their receipts, even if a new notice has the same first time.
func ackNoticeSnapshot(l *store.AdmissionLedger, started bool, acks []admission.NoticeAck) error {
if liveExecution && started != l.LiveStarted() {
return nil
}
return ackNotices(l, acks)
}
package admissionowner
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/health"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/store"
)
// Default periods of the owner's timers. A tick credits elapsed time and
// retires history in bounded batches, so it runs often even when nothing is
// queued; the status read walks the whole ledger, so it runs less often.
const (
defaultTickEvery = 10 * time.Second
defaultInventoryEvery = 5 * time.Minute
defaultStatusEvery = 30 * time.Second
defaultDeliverEvery = time.Second
defaultNoticeEvery = 5 * time.Second
defaultDrainEvery = time.Second
// defaultScheduleEvery bounds the owner to one batch start a second.
defaultScheduleEvery = time.Second
// turnEvery spaces an executable owner's selection, check and forecast
// turns, including those that dispatch nothing.
turnEvery = time.Second
)
// maxDrainGroups bounds the groups one drain persists, so a busy ingress
// cannot keep the owner from its other timers. Shutdown repeats this
// bounded turn until all held work is persisted.
const maxDrainGroups = 4
var _ checks.ResponseAdmission = (*Owner)(nil)
// drainGroup is the arrivals one drain group persists; tests shrink it.
var drainGroup = admission.MaxArrivalGroup
// drainGroupOf persists one frozen group of in's held work into l; tests make it
// fail.
var drainGroupOf = func(in *admission.Ingress, l *store.AdmissionLedger, items []admission.IngressItem) (admission.DrainReport, error) {
return in.DrainTaken(l, items, arrivalRequest)
}
// scheduleLedger picks queued work; tests make it fail. monoNow reads the
// monotonic clock the owner's wake time is set on; tests move it.
var (
scheduleLedger = (*store.AdmissionLedger).Schedule
monoNow = time.Now
// executeLedger records a batch as running; tests make it refuse.
executeLedger = (*store.AdmissionLedger).ExecuteBatch
reserveLedger = (*store.AdmissionLedger).Reserve
finishLedger = (*store.AdmissionLedger).FinishOnLease
// finishEvictionLedger finishes a member that claimed a victim, with
// what the batch did to it.
finishEvictionLedger = (*store.AdmissionLedger).FinishEviction
terminateLedger = (*store.AdmissionLedger).Terminate
// finishExistingLedger and terminateExistingLedger end a pick linked to
// the block that already enforces it; tests make them fail.
finishExistingLedger = (*store.AdmissionLedger).FinishExisting
terminateExistingLedger = (*store.AdmissionLedger).TerminateExisting
// retireLedger retires an unknown attempt; tests make it fail.
retireLedger = (*store.AdmissionLedger).RetireUnproven
// unresolvedIntents lists a retirement pass's intents; tests count it.
unresolvedIntents = (*store.AdmissionLedger).UnresolvedIntents
// previewLedger selects work for a dry-run forecast and endDryRunLedger
// ends a forecast pick; tests make them fail.
previewLedger = (*store.AdmissionLedger).SchedulePreview
endDryRunLedger = (*store.AdmissionLedger).EndDryRun
// observeLedger previews a pick and names the block whose slot it
// took; tests choose that block.
observeLedger = (*store.AdmissionLedger).ObserveVictim
// freshLedger says whether a forecast pick's episode may rotate an
// equal-tier block; tests answer it.
freshLedger = (*store.AdmissionLedger).Fresh
)
// previewLimits lets a schedule serve whatever the ledger's budgets allow,
// one batch of members at a time; tests shrink the batch.
var previewLimits = admission.ScheduleLimits{General: admission.MaxCeiling, Reserved: admission.MaxCeiling, Members: admission.MaxBatchMembers}
// Options are what the owner needs from the daemon.
type Options struct {
DB *store.DB
StatePath string
// Ceiling is the configured hourly ceiling and its source, read at
// start and at every reload. An executable owner reads its policy's.
Ceiling func() (uint32, string)
// Clock reads the host clocks (ReadClock).
Clock func() (admission.ClockReading, error)
// Inventory is one complete read of the hosting inventory (Inventory).
Inventory func() (admission.InventoryObservation, error)
// LegacySpend reads the legacy hourly counter (checks.LegacyBlockSpend).
LegacySpend func(statePath string, now time.Time) (admission.LegacySpend, error)
// LegacyHandover reads legacy blocking's state for the live start
// (checks.LegacyHandover).
LegacyHandover func(statePath string, now time.Time) (admission.LegacyHandover, error)
// WriteAudit writes audit rows to the action log in one durable write
// (actionlog.WriteDurableBatch).
WriteAudit func([]actionlog.Record) error
// Applied receives the applied block_ip rows of each delivered batch
// after their action log write and before their acknowledgement, so the
// consumers of a verified block see each one at least once. A
// failed call acknowledges nothing and the rows come again with
// the same identity: consumers must be idempotent by action and
// transition. Nil hands them to no one.
Applied func([]admission.AuditRow) error
// Deliver sends notices through the daemon's independent health path,
// as protection_queue_degraded is sent: history and direct dispatch,
// never the finding channel, suppressions or the routine rate limit.
// A preview is recorded in history only: legacy blocking enforces, so
// the gap it describes is not an outage.
Deliver func(findings []alert.Finding, preview bool) error
// DeliverSummary records a dry-run summary in history only. It must
// return promptly when ctx is cancelled, and an error must mean the
// record was not committed. The owner cancels it during shutdown.
DeliverSummary func(ctx context.Context, finding alert.Finding) error
// Expiry is how long the response a candidate asks for would last
// under the current configuration. A preview records the expiry a
// live attempt would have. An executable owner reads its policy's.
Expiry func(admission.Candidate) time.Duration
// BlockExpiry is auto_response.block_expiry, which bounds every
// response's lifetime; nil or not positive bounds nothing. An
// executable owner reads its policy's.
BlockExpiry func() time.Duration
// Caps are the firewall's containment capabilities block targets fit.
Caps func() admission.Caps
// Blocks returns the firewall engine that runs block_ip picks, or nil
// while none runs. Without a provider every pick is a preview. With
// one, the policy and liveExecution guard decide whether block picks
// forecast, preview or execute. The provider must be safe to call
// concurrently and return promptly: response funnels also read it.
Blocks func() BlockEngine
// Policy reads the published execution policy. An owner with
// Blocks binds to it at every turn and reload, and does not start
// without a valid one; a comparison-only owner never reads it.
Policy func() ExecutionPolicy
// Timer periods; zero selects the defaults.
TickEvery, InventoryEvery, StatusEvery, DeliverEvery, NoticeEvery, DrainEvery, ScheduleEvery time.Duration
}
var errStopped = errors.New("the admission owner has stopped")
type request struct {
fn func() error
err chan error
}
// Owner is the daemon's one handle on the admission ledger. One goroutine
// makes every call that changes the ledger, so sequences such as a tick
// followed by a new limit never interleave with another change.
// It serves its queue as observe previews unless an engine runs its block
// picks (Options.Blocks).
type Owner struct {
opts Options
stopping atomic.Bool
// executesBlocks publishes mayExecute() for the response funnels.
executesBlocks atomic.Bool
reg *admission.Registry
producers map[admission.ProducerID]*admission.Producer
ingress *admission.Ingress
requests chan request
quit chan bool
done chan struct{}
stopOnce sync.Once
current atomic.Pointer[health.AdmissionStatus]
audit *queuehealth.Sampled
notices *sender
// verdictQueue samples the verdict jobs of an executing owner.
verdictQueue *queuehealth.Sampled
compare comparison
// Owned by the owner goroutine.
ledger *store.AdmissionLedger
started bool
reloadPending bool
startErr error
snapshotErr error
tickErr error
scheduleErr error
// settled is set by the first turn that completes in this process.
settled bool
degraded bool
lastTick time.Time
// now is the admission time of the last reading.
now time.Time
// scheduleDue asks the next schedule turn to run; wakeAt, on the
// monotonic clock, is when the ledger said queued work changes next;
// scheduleWait spaces the turns while they keep failing.
scheduleDue bool
wakeAt time.Time
scheduleWait retryBackoff
limit uint32
source string
imported *health.AdmissionImport
inventoryAt time.Time
inventoryErr error
auditAcked uint64
auditErr error
// auditWritten holds the pending rows this process wrote whose
// acknowledgement has not landed yet.
auditWritten map[auditKey]bool
// drained is the ingress decision sequence the last drain persisted.
drained uint64
// drainFailed holds admission closed until a drain succeeds. drainErr
// retains its cause independently of later snapshot or reload errors.
drainFailed bool
drainErr error
// damageErr is the latest drain cause that discarded damaged arrivals,
// even if another failure stopped that drain. A damaged record stays
// damaged, so the cause is kept for the owner's life.
damageErr error
// isolated holds the candidates whose last batch the kernel rejected,
// with their absolute expiry: each runs alone.
isolated map[admission.CandidateID]time.Time
// finishes retains known outcomes until their writes commit. A turn
// retries these before selecting more work, bounding this to one batch.
finishes []pendingFinish
// execHold is why automatic execution is held, or nil. probeAt is
// when the next probe may run, on the monotonic clock, after probeWait.
execHold error
probeAt time.Time
probeWait time.Duration
// applyHold keeps an apply fault that snapshot repair cannot prove fixed.
applyHold error
probeCheck bool
// set is the temporary set the turn's schedule budgets block work by,
// and setErr why the turn could not read it (readSet).
set *firewall.BlockAdmission
setErr error
// policy is the installed execution policy and seenEpoch the newest
// epoch read, valid or not. policyErr holds execution while set.
policy ExecutionPolicy
seenEpoch uint64
policyErr error
verdicts verdicts
// liveSpans are the admission times this process executed in, for the
// mode of its notices (noteMode); modeNoted is set once the first is
// known. Spans change only with installed policies, so they stay few.
liveSpans []liveSpan
modeNoted bool
// shadow is the forecast session while a dry-run policy is installed,
// and forecastReset why the next session starts.
// forecastErr is why forecasting last failed, cleared once a session
// starts or a forecast turn succeeds; forecastErrAt is when, kept until
// the next session starts.
shadow *shadow
forecastReset resetReason
forecastErr forecastError
forecastErrAt time.Time
out dryRunOutput
summary summaryState
summaryCtx context.Context
summaryCancel context.CancelFunc
summaryResult chan summaryResult
// policyHeld is the firewall work a refused policy holds. It waits for
// a valid policy as pending work waits for its answer.
policyHeld map[admission.CandidateID]bool
// nextBatch is when the next block batch may start and nextTurn when an
// executable owner's next selection may, on the monotonic clock;
// arrived is set by a drain that queued work.
nextBatch time.Time
nextTurn time.Time
arrived bool
// retireDue is the next admission expiry; retireNext is the next
// monotonic retry. They must not be converted into each other when
// admission time stalls or an engine call takes time.
retireDue time.Time
retireNext time.Time
retireAt time.Time
retireWait time.Duration
// evicted names the victims whose eviction the ledger recorded and
// whose rows the engine has yet to retire. evictedWait spaces passes
// that clear none of them.
evicted []firewall.BlockAction
evictedErr error
evictedWait retryBackoff
// heldAt is the monotonic time of the next held check (observeHeld).
heldAt time.Time
// replacements counts this process's previews that took the slot of a
// lower-priority automatic block; replacementsHour keeps the last hour
// of them on the monotonic clock. rotations and rotationsHour count
// those that took an equal-priority block's slot.
replacements uint64
replacementsHour minuteTotals
rotations uint64
rotationsHour minuteTotals
// Per-action monotonic waits keep held entries from monopolizing a batch.
retireRetry map[admission.ActionID]time.Time
// retireDelay is each action's current wait while the ledger refuses
// its retirement; retireFaults names the actions whose fault was
// already reported.
retireDelay map[admission.ActionID]time.Duration
retireFaults map[admission.ActionID]bool
// An unreadable intent list has no readable action to key a fault by.
retireReadFault bool
// An engine check fault has no single attempt to key a fault by.
retireCheckFault bool
// retireErr is the last retirement fault reported while one stands.
// Retirement never fails a turn.
retireErr error
// Resume after the last checked action even if its retry is already due.
retireAfter admission.ActionID
}
// Start opens the ledger and runs the startup sequence before it returns,
// so no admission precedes it, then keeps the ledger on its timers. A
// failed start is retried at every tick; the status names the failure.
func Start(opts Options) *Owner {
if opts.TickEvery <= 0 {
opts.TickEvery = defaultTickEvery
}
if opts.InventoryEvery <= 0 {
opts.InventoryEvery = defaultInventoryEvery
}
if opts.StatusEvery <= 0 {
opts.StatusEvery = defaultStatusEvery
}
if opts.DeliverEvery <= 0 {
opts.DeliverEvery = defaultDeliverEvery
}
if opts.NoticeEvery <= 0 {
opts.NoticeEvery = defaultNoticeEvery
}
if opts.DrainEvery <= 0 {
opts.DrainEvery = defaultDrainEvery
}
if opts.ScheduleEvery <= 0 {
opts.ScheduleEvery = defaultScheduleEvery
}
o := &Owner{
opts: opts, requests: make(chan request), quit: make(chan bool), done: make(chan struct{}),
audit: queuehealth.NewSampled(int(admission.MaxAuditSlots), "rows", deliveryLag),
forecastReset: resetStartup,
summaryResult: make(chan summaryResult, 1),
}
o.summaryCtx, o.summaryCancel = context.WithCancel(context.Background())
if opts.Blocks != nil {
o.verdictQueue = queuehealth.NewSampled(verdictPending, "jobs", deliveryLag)
}
o.reg, o.producers, o.startErr = buildRegistry()
if o.startErr == nil {
o.ingress, o.startErr = admission.NewIngress(o.reg)
}
if o.startErr == nil {
o.ingress.ObserveArrivals(o.countArrival)
o.ingress.ObserveDisplacements(func(a admission.DrainedArrival) {
o.countArrivalAt(a, deliveryNow())
})
}
if o.startErr == nil {
o.startErr = o.start()
}
o.refreshStatus()
o.notices = &sender{o: o, queue: queuehealth.NewSampled(admission.FixedNotices+int(admission.NoticeBytes/admission.NoticeSlotBytes), "notices", deliveryLag), done: make(chan struct{})}
go o.run()
go o.notices.run()
return o
}
func (o *Owner) run() {
defer close(o.done)
tick := time.NewTicker(o.opts.TickEvery)
defer tick.Stop()
inventory := time.NewTicker(o.opts.InventoryEvery)
defer inventory.Stop()
status := time.NewTicker(o.opts.StatusEvery)
defer status.Stop()
deliver := time.NewTicker(o.opts.DeliverEvery)
defer deliver.Stop()
drain := time.NewTicker(o.opts.DrainEvery)
defer drain.Stop()
schedule := time.NewTicker(o.opts.ScheduleEvery)
defer schedule.Stop()
for {
select {
case clean := <-o.quit:
o.stopVerdicts()
if clean {
o.shutdown()
}
if o.ingress != nil {
o.ingress.Publish(nil)
}
o.refreshStatus()
return
case r := <-o.requests:
if o.stopping.Load() {
r.err <- errStopped
} else {
r.err <- r.fn()
}
case <-tick.C:
if o.started {
_ = o.tick()
} else if o.reg != nil {
o.startErr = o.start()
o.refreshStatus()
}
if !o.started {
o.writeComparison(false)
}
case <-inventory.C:
if o.started {
o.refreshInventory()
}
case <-status.C:
o.refreshStatus()
case <-deliver.C:
if o.started {
_ = o.deliverAudit()
}
case <-drain.C:
if o.started {
o.drainTick()
}
case <-schedule.C:
if o.started {
o.schedule()
o.maintainExamples()
}
case r := <-o.stageResults():
o.receiveVerdict(r)
case <-o.stageRetired():
o.verdictsRetired()
}
}
}
// do runs fn on the owner goroutine and returns its error.
func (o *Owner) do(fn func() error) error {
if o.stopping.Load() {
return errStopped
}
r := request{fn: fn, err: make(chan error, 1)}
select {
case o.requests <- r:
return <-r.err
case <-o.done:
return errStopped
}
}
// openLedger opens the ledger and buildRegistry builds its registry; tests
// count the handles and register their own producer.
var (
openLedger = store.OpenAdmissionLedger
buildRegistry = Registry
settleLedger = (*store.AdmissionLedger).Settle
)
// start runs the startup sequence. A retry
// keeps the handle it opened: one database has one ledger handle.
func (o *Owner) start() error {
// An executable owner never starts on settings it cannot bind to.
if o.executable() {
if err := o.refreshPolicy(); err != nil {
return err
}
}
if o.opts.DB == nil {
return errors.New("the state database is not open")
}
if o.ledger == nil {
l, err := openLedger(o.opts.DB, o.reg)
if err != nil {
return fmt.Errorf("opening the admission ledger: %w", err)
}
o.ledger = l
l.ObserveRefusals(o.countLaterRefusal)
}
o.syncPreviews()
// An existing ledger records elapsed time under its saved limit before
// a new limit applies.
reading, err := o.readTick()
if err != nil {
return err
}
if err = o.applyCeiling(reading.Wall); err != nil {
return err
}
// What a previous process left reserved or running ends before this
// handle reserves anything: nothing is dispatched again.
if _, _, err = settleLedger(o.ledger); err != nil {
return fmt.Errorf("settling outstanding attempts: %w", err)
}
if err = o.startLive(); err != nil {
return err
}
// Opted-out work must end silently before startup revalidation can
// report its expired evidence or changed ownership as a response gap.
if err = o.refuseDisabled(); err != nil {
return err
}
if err = o.endEarlierIncidents(); err != nil {
return err
}
o.refreshInventory()
generation, err := o.ledger.BeginIngress()
if err != nil {
return fmt.Errorf("beginning an ingress generation: %w", err)
}
// An earlier process's comparison counts and examples may be lost.
o.out.restarted = generation.Generation > 1
if err = o.ledger.Revalidate(); err != nil {
return fmt.Errorf("revalidating the queue: %w", err)
}
if err = o.publish(); err != nil {
return err
}
o.started, o.scheduleDue = true, true
return nil
}
// applyCeiling sets the configured limit. A new ledger takes its first
// limit together with the legacy hour's spend; an unreadable legacy
// counter starts it without credit. Once a limit is set, a retried start
// only applies the limit again.
func (o *Owner) applyCeiling(now time.Time) error {
limit, source := o.ceilingConfig()
c, err := o.ledger.Ceiling()
if err != nil {
return fmt.Errorf("reading the ceiling: %w", err)
}
if c.Limit != 0 {
if err = o.ledger.SetCeiling(limit); err != nil {
return fmt.Errorf("setting the ceiling: %w", err)
}
retained, restoreErr := o.ledger.ImportedLegacySpend()
if restoreErr != nil {
return fmt.Errorf("reading imported legacy spend: %w", restoreErr)
}
// A startup retry must keep a committed handover's read error.
if retained.Units > 0 && (o.imported == nil || o.imported.Error == "") {
o.imported = &health.AdmissionImport{Units: retained.Units, At: retained.At}
}
o.limit, o.source = limit, source
return nil
}
spend, readErr := o.opts.LegacySpend(o.opts.StatePath, now)
imported := &health.AdmissionImport{Units: min(spend.Units, admission.MaxCeiling), At: spend.At}
if readErr != nil {
spend = admission.LegacySpend{Unknown: true}
imported = &health.AdmissionImport{Error: readErr.Error()}
}
if err = o.ledger.ImportLegacySpend(limit, spend); err != nil {
return fmt.Errorf("importing the legacy hourly count: %w", err)
}
o.limit, o.source, o.imported = limit, source, imported
return nil
}
// startLiveLedger runs the ledger's live start; tests make it fail.
var startLiveLedger = (*store.AdmissionLedger).StartLive
// startLive runs the live start once per ledger, as soon as the
// installed policy executes. Every drain and turn reads the clock through
// tick first, so nothing is placed or served under that policy before it:
// legacy blocking's hour is carried over, its pending queue stays legacy's,
// and earlier previews stop answering episodes and sending notices. A
// legacy state it cannot read is not an empty one: the ledger keeps no saved
// credit, and status names the error.
func (o *Owner) startLive() error {
if !o.executing() || o.ledger.LiveStarted() {
return nil
}
h, readErr := o.opts.LegacyHandover(o.opts.StatePath, o.lastTick)
if readErr != nil {
h = admission.LegacyHandover{Spend: admission.LegacySpend{Unknown: true}}
}
if err := startLiveLedger(o.ledger, h); err != nil {
return fmt.Errorf("starting live execution: %w", err)
}
if readErr != nil {
o.imported = &health.AdmissionImport{Error: readErr.Error()}
}
return nil
}
// readTick records one clock reading.
func (o *Owner) readTick() (admission.ClockReading, error) {
reading, err := o.opts.Clock()
var t admission.ClockTick
if err == nil {
t, err = o.ledger.Tick(reading)
}
if err != nil {
o.tickErr = err
return reading, fmt.Errorf("reading the admission clock: %w", err)
}
o.tickErr, o.degraded, o.lastTick, o.now = nil, t.Degraded, reading.Wall, t.Now
o.noteMode()
// A forecast meters the committed clock as the ledger's ceiling does.
// One that cannot starts over at its next turn.
if o.shadow != nil && o.shadow.advance(t) != nil {
o.dropForecast()
}
o.compare.begin(o.now)
return reading, nil
}
// tick records a reading. A refused reading or live start stops admission:
// the ingress cannot use the unfinished tick's snapshot. The next
// good tick publishes a fresh snapshot.
func (o *Owner) tick() error {
defer o.writeComparison(false)
admitting := o.ingress.Health().Admitting
defer func() {
if !admitting || !o.ingress.Health().Admitting {
o.refreshStatus()
}
}()
if _, err := o.readTick(); err != nil {
o.ingress.Publish(nil)
return err
}
// A live policy installed since the last reading starts live before
// any later drain or turn; a tick that cannot start it fails, which
// closes admission.
if err := o.startLive(); err != nil {
o.tickErr = err
o.ingress.Publish(nil)
return err
}
// A ceiling changed by any config path, not only a reload, applies
// after this reading at the saved limit.
if limit, source := o.ceilingConfig(); limit != o.limit || source != o.source {
o.reloadPending = true
}
if o.reloadPending {
if err := o.reloadCeiling(); err != nil {
o.snapshotErr = err
o.ingress.Publish(nil)
return err
}
}
return o.publish()
}
var readSnapshot = (*store.AdmissionLedger).QueueSnapshot
// publish hands the ingress the durable queue; a failed read closes it.
func (o *Owner) publish() error {
if o.drainFailed {
// Only a successful drain reopens admission: a snapshot now would
// admit work the ledger cannot persist, and every retry would be
// announced as a new stop.
o.ingress.Publish(nil)
return nil
}
snap, err := readSnapshot(o.ledger)
o.snapshotErr = err
if err != nil {
o.ingress.Publish(nil)
return fmt.Errorf("reading the queue snapshot: %w", err)
}
o.forgetIsolated(snap)
o.ingress.Publish(snap)
return nil
}
// refreshInventory folds one complete inventory read into the ledger. A
// failed read keeps the committed inventory.
func (o *Owner) refreshInventory() {
obs, err := o.opts.Inventory()
if err == nil {
err = o.ledger.RefreshInventory(obs)
}
o.inventoryErr = err
if err == nil {
o.inventoryAt = time.Now()
if o.started && o.tickErr == nil && !o.reloadPending {
admitting := o.ingress.Health().Admitting
_ = o.publish()
if admitting != o.ingress.Health().Admitting {
o.refreshStatus()
}
}
}
}
// Reload records a reading under the saved limit, then applies the
// configured one and revalidates the queue against it.
// A failed reload is retried on ticks before admission can resume. An
// executable owner first installs the published policy: Reload returns
// once it is installed, or with the ErrPolicy that holds execution.
func (o *Owner) Reload() error {
return o.do(func() error {
if !o.started {
return fmt.Errorf("the admission owner has not started: %w", o.startErr)
}
defer o.refreshStatus()
var policyErr error
if o.executable() {
policyErr = o.refreshPolicy()
}
o.reloadPending = true
return errors.Join(policyErr, o.tick())
})
}
func (o *Owner) reloadCeiling() error {
limit, source := o.ceilingConfig()
if err := o.ledger.SetCeiling(limit); err != nil {
return fmt.Errorf("setting the ceiling: %w", err)
}
o.limit, o.source = limit, source
// A forecast takes the new limit without new credit.
if o.shadow != nil && o.shadow.setLimit(limit) != nil {
o.dropForecast()
}
if err := o.refuseDisabled(); err != nil {
return err
}
if err := o.ledger.Revalidate(); err != nil {
return err
}
o.reloadPending = false
o.scheduleDue = true
return nil
}
// Stop commits a final checkpoint, closes the ingress generation and stops
// the owner, after any notice delivery in flight. A failure leaves the
// generation open, so the next start counts it as interrupted.
func (o *Owner) Stop() { o.halt(true) }
// halt stops the owner goroutine, after the clean shutdown when clean is
// set; without it the ledger is left as a crash leaves it.
func (o *Owner) halt(clean bool) {
o.stopOnce.Do(func() {
// Close acceptance before returning new blocks to legacy, even
// while an owner turn is still running. Later snapshots stay closed.
if o.ingress != nil {
o.ingress.Close()
}
o.stopping.Store(true)
o.summaryCancel()
o.quit <- clean
<-o.done
<-o.notices.done
// Both writers have stopped. Settle any delivery whose receipt
// could no longer reach the owner, before the final flush.
select {
case result := <-o.summaryResult:
o.summaryDelivered(result.summary, result.err)
default:
}
if clean {
o.finishSummary()
o.refreshStatus()
}
})
}
func (o *Owner) shutdown() {
defer o.writeComparison(true)
if !o.started {
return
}
// An outcome still unwritten stays running for the next start to
// settle; the held arrivals are drained regardless.
if err := o.finishPending(); err != nil {
o.scheduleErr = err
}
held := o.ingress.Len() != 0
for {
if o.drainHeld() != nil {
return
}
if o.ingress.Len() == 0 {
break
}
}
// Producers are stopped before Stop. This empty drain commits their
// final decisions after every held group and before the clean close.
if held {
if o.drainHeld() != nil {
return
}
}
if err := o.ledger.EndIngress(); err != nil {
o.snapshotErr = fmt.Errorf("closing the ingress: %w", err)
}
}
// writeComparison writes the decision counts of each ended hour, or of the
// hour in progress at a stop, to the action log. Rows a failed write could
// not record are retried at the next tick.
func (o *Owner) writeComparison(final bool) {
now := o.now
// Refusals still happen during a ledger or clock outage. Their hourly
// summaries use wall time without moving the admission clock backward.
if !o.started || o.tickErr != nil {
if wall := deliveryNow(); wall.After(now) {
now = wall
}
}
rows, through := o.compare.take(now, final)
if len(rows) == 0 {
return
}
if err := o.opts.WriteAudit(rows); err == nil {
o.compare.written(through)
}
}
// drain persists what detectors handed the ingress since the last drain,
// after a fresh clock reading, so the ledger judges the arrivals at the
// current time. With no new ingress decision and no failed drain to
// retry it reads and writes nothing.
func (o *Owner) drain() error {
seq := o.ingress.Checkpoint().Sequence
if seq == o.drained && o.ingress.Len() == 0 && !o.drainFailed {
return nil
}
if err := o.drainHeld(); err != nil {
return err
}
o.drained = seq
return nil
}
// drainTick drains the ingress on its timer. With an engine, work the
// drain queued is served at once rather than on the next schedule tick:
// the first ready batch starts immediately and batchEvery
// spaces the rest. Without one, previews keep the schedule timer.
func (o *Owner) drainTick() {
o.arrived = false
_ = o.drain()
if o.arrived && o.opts.Blocks != nil {
o.schedule()
}
}
// drainHeld persists what the ingress holds, at most maxDrainGroups groups.
// Every group, even an empty one, checkpoints the ingress decisions. A
// failed drain closes admission and names its cause until a drain
// succeeds.
func (o *Owner) drainHeld() (err error) {
admitting := o.ingress.Health().Admitting
defer func() {
if err != nil {
o.drainFailed = true
o.drainErr = fmt.Errorf("draining the ingress: %w", err)
o.ingress.Publish(nil)
}
if err != nil || o.snapshotErr != nil || admitting != o.ingress.Health().Admitting {
o.refreshStatus()
}
}()
for range maxDrainGroups {
// Freeze the group before the clock read: Submit can otherwise
// hand us evidence newer than the reading used by the ledger.
items := o.ingress.Take(min(drainGroup, admission.MaxArrivalGroup))
if err := o.tick(); err != nil {
o.ingress.Release(items)
return err
}
report, err := drainGroupOf(o.ingress, o.ledger, items)
if report.Failed != 0 {
// Discarded work stays lost even if a later failure in this drain
// recovers, so its damage signal outlives the drain hold.
o.damageErr = fmt.Errorf("draining the ingress: %w", err)
o.refreshStatus()
}
if err != nil && !errors.Is(err, admission.ErrArrivalsIsolated) {
return err
}
if report.Queued+report.Coalesced > 0 {
o.scheduleDue, o.arrived = true, true
}
// The group's own snapshot reopened admission.
o.drainFailed = false
o.drainErr, o.snapshotErr = nil, nil
if o.ingress.Len() == 0 {
return nil
}
}
return nil
}
// schedule runs a turn when work was drained or the ledger's wake time has
// come, and keeps the cause of a failed turn for status until one
// succeeds. A turn that keeps failing is backed off from the default
// schedule period to a minute, as a held probe is: the turn
// also picks up policy changes and runs the probes.
func (o *Owner) schedule() {
o.expireVerdicts()
probeDue := o.mayExecute() && o.execHold != nil && o.heldProbeDue()
// A policy change can resume held work or probes without an arrival,
// reload or ledger deadline, including after an idle dry run.
policyDue := o.executable() && o.opts.Policy().Epoch > o.seenEpoch
if !o.scheduleDue && (o.wakeAt.IsZero() || monoNow().Before(o.wakeAt)) && !probeDue && !policyDue {
return
}
if !o.scheduleWait.due(monoNow()) {
return
}
o.scheduleDue, o.wakeAt = false, time.Time{}
// An executable owner selects, checks and forecasts once a second.
// A missing-engine preview must not delay the returning engine; the
// batch gate still bounds batch starts. Retirement keeps its own waits.
engineReturned := probeDue && errors.Is(o.execHold, errNoEngine)
selecting := engineReturned || !o.executable() || !monoNow().Before(o.nextTurn)
err := o.runTurn(selecting)
if err != nil {
o.scheduleDue = true
o.scheduleWait.fail(monoNow(), defaultScheduleEvery, maxProbeWait)
} else {
o.scheduleWait = retryBackoff{}
}
if (err == nil) != (o.scheduleErr == nil) {
o.scheduleErr = err
o.refreshStatus()
}
o.scheduleErr = err
}
// turn serves the queue's picks after a fresh reading. With an engine
// (Options.Blocks) the policy installed when the turn starts decides how:
// a dry-run policy forecasts its picks (forecast); otherwise block_ip picks
// run as one batch (execute). Every other pick is an observe preview. The
// ledger's wake time, converted by the admission clock's elapsed time,
// sets the next turn; a forecast wakes whatever the live budgets.
func (o *Owner) turn() error { return o.runTurn(true) }
// runTurn is a turn that selects work only when selecting is set; otherwise
// the queue stays due for a later turn.
func (o *Owner) runTurn(selecting bool) error {
// The next selection's deadline runs from the turn's start, so the
// turn's own durable writes cannot push it past the schedule ticker.
turnStart := monoNow()
// Recording an outcome needs only the saved ledger clock. A failed
// current reading must not abandon work that already ran.
if err := o.finishPending(); err != nil {
return err
}
if o.executable() {
// A refused policy holds execution; the turn goes on without it.
_ = o.refreshPolicy()
}
// Anchor before the reading: a slow clock or durable tick must not add
// its latency to an unknown attempt's expiry wake.
retireSample := monoNow()
if err := o.tick(); err != nil {
return err
}
// Retirement first: it can release the holds pending_recovery waits on.
// Its faults are reported apart and never fail the turn.
o.reportRetirement(o.retire())
_ = o.retireEvicted()
if o.executable() {
o.sweepVerdicts()
}
nextWake := (*store.AdmissionLedger).NextWake
if o.forecasting() || o.previewsOnly() {
nextWake = (*store.AdmissionLedger).PreviewWake
}
switch {
case o.batchGated():
// Nothing else wakes the owner at the batch deadline; a held probe
// wakes the schedule itself, and held work keeps the ledger's wake.
if monoNow().Before(o.nextBatch) {
o.scheduleDue = true
}
if err := o.deferBatchGate(selecting); err != nil {
return err
}
case !selecting:
o.scheduleDue = true
default:
// Only a selection under the newly installed policy spends the
// turn. A live batch gate must not delay a new dry-run policy.
if o.executable() {
o.nextTurn = turnStart.Add(turnEvery)
}
if o.forecasting() {
if err := o.forecast(); err != nil {
return err
}
} else if err := o.serve(); err != nil {
return err
}
}
_ = o.observeHeld()
if err := o.publish(); err != nil {
return err
}
wake, ok, err := nextWake(o.ledger, o.answers())
if err != nil {
return fmt.Errorf("reading the next wake time: %w", err)
}
if ok {
if d := wake.Sub(o.now); d > 0 {
o.wakeAt = monoNow().Add(d)
} else {
o.scheduleDue = true
}
}
if at := o.retirementWake(retireSample); !at.IsZero() {
if o.wakeAt.IsZero() || at.Before(o.wakeAt) {
o.wakeAt = at
}
}
// Retirement may clear its retries or delay them after a fault. The
// held check owns a separate monotonic deadline across those turns.
if o.mayRetire() && !o.heldAt.IsZero() && (o.wakeAt.IsZero() || o.heldAt.Before(o.wakeAt)) {
o.wakeAt = o.heldAt
}
if !o.settled {
// Publish recovery with settlement so doctor cannot observe the
// previous turn's error after the startup grace has ended.
o.scheduleErr = nil
o.settled = true
o.refreshStatus()
}
return nil
}
// batchGated keeps a turn that cannot start a batch from spending fair
// turns or postponing the next selection: one before the next batch may
// start, or while execution is held and its turn is not due. Forecasts
// and observe previews do not dispatch batches and retain their own
// cadence.
func (o *Owner) batchGated() bool {
return o.mayExecute() && (monoNow().Before(o.nextBatch) || o.execHold != nil && !o.heldTurnDue())
}
// heldTurnDue reports whether a held turn may select: when its probe is
// due, or, for a missing engine, which has no probe, once its wait has
// passed, so work that needs no engine is still previewed. That wait
// never wakes an idle owner by itself.
func (o *Owner) heldTurnDue() bool {
return o.heldProbeDue() || errors.Is(o.execHold, errNoEngine) && !monoNow().Before(o.probeAt)
}
// heldProbeDue reports whether held execution may probe now: at its retry
// deadline, or as soon as a missing engine has been published, without
// waiting for the old deadline.
func (o *Owner) heldProbeDue() bool {
if errors.Is(o.execHold, errNoEngine) {
return o.opts.Blocks() != nil
}
return !monoNow().Before(o.probeAt)
}
// deferBatchGate reports an execution outage without selecting work or
// running a probe that cannot be followed by a batch yet. Pending verdicts
// and allows need no execution; their state stays with the verdict stage.
func (o *Owner) deferBatchGate(selecting bool) error {
// Opt-outs are terminal even on maintenance turns, before an outage
// can report them as waiting for execution.
if err := o.refuseDisabled(); err != nil {
return err
}
// Maintenance turns keep the last scheduling snapshot.
if selecting {
if err := o.readSet(); err != nil {
return fmt.Errorf("reading gated set: %w", err)
}
}
if o.opts.Blocks() == nil {
o.ready(nil, false)
}
if !o.degraded && o.execHold == nil {
return nil
}
// Deferrals bypass selection, so they need its eligibility check:
// stale evidence and changed ownership must keep their own endings.
if err := o.ledger.Revalidate(); err != nil {
return fmt.Errorf("revalidating gated work: %w", err)
}
snap, err := o.ledger.QueueSnapshot()
if err != nil {
return fmt.Errorf("reading gated work: %w", err)
}
answers := o.answers()
for _, item := range snap.Items {
id := admission.CandidateID(item.Key)
c, err := o.ledger.Candidate(id)
if err != nil {
return fmt.Errorf("reading gated candidate %s: %w", id, err)
}
// A deferral already names why the work waits; relabelling it costs a
// write the next served turn undoes.
if c.State != admission.StateQueued || c.Key.Kind != admission.KindBlockIP || c.Reason != 0 ||
o.now.Before(c.NotBefore) || answers[id] == admission.AnswerPending || answers[id] == admission.AnswerAllow {
continue
}
if err := o.endPick(execPick{pick: admission.Pick{ID: id}, c: c}, admission.ReasonEngineUnavailable); err != nil {
return err
}
}
return nil
}
// serve runs the schedule's picks. With an engine its block_ip picks run
// as one batch, and a refused policy holds every firewall pick; any other
// pick is previewed. The schedule has already deferred the work its
// budgets and the set's free slots cannot serve and revalidated each pick
// under the same state.
func (o *Owner) serve() error {
if err := o.refuseDisabled(); err != nil {
return err
}
if err := o.readSet(); err != nil {
return fmt.Errorf("reading queued work: %w", err)
}
var picks []admission.Pick
var err error
if o.previewsOnly() {
picks, err = previewLedger(o.ledger, o.answers())
} else {
picks, err = scheduleLedger(o.ledger, previewLimits, o.answers())
}
if err != nil {
return fmt.Errorf("scheduling the queue: %w", err)
}
var blocks []execPick
for _, p := range picks {
c, readErr := o.ledger.Candidate(p.ID)
if readErr != nil {
return fmt.Errorf("reading pick %s: %w", p.ID, readErr)
}
pick := execPick{pick: p, c: c}
switch {
case !o.executable() || c.Key.Kind == admission.KindChallenge:
err = o.preview(pick)
case o.policyErr != nil:
err = o.holdForPolicy(pick)
case c.Key.Kind == admission.KindBlockIP && o.executing():
blocks = append(blocks, pick)
default:
err = o.preview(pick)
}
if err != nil {
return err
}
}
switch {
case len(blocks) > 0:
return o.execute(blocks)
case o.executing() && o.policyErr == nil && o.execHold != nil:
if o.ready(o.opts.Blocks(), false) && o.execHold != nil {
o.hold(o.execHold, o.applyHold != nil)
}
}
return nil
}
// previewsOnly reports whether every pick of a turn ends as an uncharged
// preview: an owner without an engine on a ledger that has started live.
// Its selection is then no admission, so the live budgets do not bound it.
func (o *Owner) previewsOnly() bool {
return !o.executable() && o.ledger.LiveStarted()
}
// holdForPolicy defers p while a refused policy holds execution, as an
// unavailable engine defers it: nothing is previewed, reserved or charged.
// Until a valid policy is installed the schedule passes p over, so it is
// not selected again at every turn.
func (o *Owner) holdForPolicy(p execPick) error {
if err := o.endPick(p, admission.ReasonEngineUnavailable); err != nil {
return err
}
if o.policyHeld == nil {
o.policyHeld = map[admission.CandidateID]bool{}
}
o.policyHeld[p.pick.ID] = true
return nil
}
// preview is an observe preview of p: reserved and charged as live work
// would be and ended in the same transaction without running. A
// terminal refusal ends only that pick, so an invalid selected lifetime
// cannot starve the queue; any other failure is returned. Once the ledger
// has started live a preview spends nothing.
func (o *Owner) preview(p execPick) error {
c := p.c
if o.ledger.LiveStarted() {
return o.previewLive(p)
}
expires, err := o.cappedExpiry(c, o.now)
if err != nil {
return fmt.Errorf("reading the episode of pick %s: %w", p.pick.ID, err)
}
_, _, _, victim, err := observeLedger(o.ledger, p.pick.ID, p.pick.Lane, expires)
if err != nil {
if reason, refused := admission.ReasonOf(err); refused {
switch reason.Disposition() {
case admission.DispositionRefused, admission.DispositionWithheld, admission.DispositionDropped:
if _, endErr := o.ledger.Terminate(p.pick.ID, reason); endErr != nil {
return fmt.Errorf("ending pick %s: %w", p.pick.ID, endErr)
}
o.compare.addLedgerRefusalAt(refusalKey(c, err), o.now)
return nil
}
}
return fmt.Errorf("previewing pick %s: %w", p.pick.ID, err)
}
switch {
case victim == (admission.EvictionVictim{}):
case victim.Tier == p.pick.Tier:
saturating(&o.rotations)
o.rotationsHour.add(monoNow())
default:
saturating(&o.replacements)
o.replacementsHour.add(monoNow())
}
o.compare.addAt(compareKey{entry: c.Entry, check: c.Check, kind: c.Key.Kind, decision: decisionObserve}, o.now)
return nil
}
// previewLive previews p on a ledger that has started live. Live work owns
// the ceiling, the set, the history and the episodes there, so the pick
// ends as a dry run ends one: nothing is reserved or charged, and its
// episode stays open to the next observation. A refusal the ending meets
// defers or ends p as a reservation would.
func (o *Owner) previewLive(p execPick) error {
_, changed, err := endDryRunLedger(o.ledger, p.pick.ID)
if reason, refused := admission.ReasonOf(err); refused {
return o.endCounted(p, reason, admission.CauseOf(err))
}
switch {
case errors.Is(err, admission.ErrNotReady):
return nil
case err != nil:
return fmt.Errorf("previewing pick %s: %w", p.pick.ID, err)
case changed:
c := p.c
o.compare.addAt(compareKey{entry: c.Entry, check: c.Check, kind: c.Key.Kind, decision: decisionObserve}, o.now)
}
return nil
}
// expiryOf is the absolute expiry of a pick's next attempt: a retry keeps
// its first attempt's; a first attempt lasts the selected legacy lifetime,
// or the configured response's, from now, and never longer than the block
// expiry: a longer request, such as an incident's second block or a
// central block, is cut to it, and a shorter one stays short.
func (o *Owner) expiryOf(c admission.Candidate, now time.Time) time.Time {
if c.Attempts > 0 {
return c.ExpiresAt
}
duration := c.PreviewTTL
if duration == 0 {
duration = o.expiry(c)
}
if lease := o.lease(); lease > 0 {
duration = min(duration, lease)
}
return now.Add(duration)
}
// cappedExpiry is expiryOf for a pick, capped for a same-episode
// re-admission of a block at its episode's original verified end:
// a block that returns after the first one left early lasts no
// longer than the first would have. The time left is admission time, so on
// the lease clock now the cap ends the lead later, as the first block's
// expiry did. A retry keeps its first attempt's expiry. A lost or
// replaced episode cannot grant an older queued return a new lease.
func (o *Owner) cappedExpiry(c admission.Candidate, now time.Time) (time.Time, error) {
expires := o.expiryOf(c, now)
if c.Attempts > 0 || c.Key.Kind != admission.KindBlockIP || c.Key.Generation < 2 {
return expires, nil
}
end, current, err := o.ledger.VerifiedEnd(c)
if err != nil || !current {
return now, err
}
if end.IsZero() {
return expires, nil
}
if capped := now.Add(max(end.Sub(o.now), 0)); capped.Before(expires) {
return capped, nil
}
return expires, nil
}
// countArrival counts the ledger's decision on one drained arrival.
func (o *Owner) countArrival(a admission.DrainedArrival) {
o.countArrivalAt(a, o.now)
}
func (o *Owner) countArrivalAt(a admission.DrainedArrival, at time.Time) {
k := compareKey{entry: entryOf(a.Submission.Evidence, a.Submission.Via), check: a.Submission.Evidence.Check(), kind: a.Submission.Kind}
switch {
case a.Result.Err != nil:
k.decision, k.reason, k.cause = decisionRefused, reasonOf(a.Result.Err), admission.CauseOf(a.Result.Err)
o.compare.addCountAt(k, a.Selected, at)
case a.Result.Created:
k.decision = decisionQueued
o.compare.addAt(k, at)
k.decision = decisionCoalesced
o.compare.addCountAt(k, a.Selected-1, at)
default:
k.decision = decisionCoalesced
o.compare.addCountAt(k, a.Selected, at)
}
}
// arrivalRequest asks for the response a submission names. The ledger
// assigns its episode and generation when it persists the arrival;
// a caller never chooses them.
func arrivalRequest(s admission.Submission) (admission.CandidateRequest, error) {
return admission.CandidateRequest{Kind: s.Kind, Target: s.Target, Primary: s.Evidence.ID(), Entry: s.Via, PreviewTTL: s.PreviewTTL}, nil
}
// Mint mints the evidence f's own observation supports, aimed at the
// address its parser read, with the producer that stamped it.
// It counts nothing: a path may mint a root before it knows it will
// answer it. Mint takes no owner lock: funnels call it on their own
// goroutines.
func (o *Owner) Mint(f alert.Finding) (admission.Evidence, error) {
if o.ingress == nil {
return admission.Evidence{}, o.startErr
}
producer, in, err := checks.AdmissionEvidence(f)
if err != nil {
return admission.Evidence{}, err
}
p := o.producers[producer]
if p == nil {
return admission.Evidence{}, &admission.Error{Reason: admission.ReasonPolicy, Detail: "finding names no registered producer"}
}
// Claims resolve against the inventory the ingress judges scopes by.
in.Inventory = o.ingress.Inventory()
return p.Mint(in)
}
// ExecutesBlocks reports whether legacy must leave new block picks to this
// owner: a valid live policy is installed and an engine is present. It is
// false once shutdown begins. It says nothing about one request: a request
// accepted earlier stays the owner's through later holds or displacement.
func (o *Owner) ExecutesBlocks() bool {
// A turn already in flight can publish its policy after Stop begins.
return !o.stopping.Load() && o.executesBlocks.Load() && o.opts.Blocks != nil && o.opts.Blocks() != nil
}
// Refuse counts a response of kind through via that could not be answered
// because f could not be minted, as the ingress counts a response it
// refuses itself.
func (o *Owner) Refuse(kind admission.Kind, f alert.Finding, via admission.Entry, err error) {
if o.ingress != nil {
o.ingress.Refuse(err, checks.AdmissionSeverity(f.Severity))
}
if via == 0 {
via = admission.EntryScan
}
o.compare.add(compareKey{entry: via, check: f.Check, kind: kind, decision: decisionRefused, reason: reasonOf(err), cause: admission.CauseOf(err)})
}
// countLaterRefusal adds a queued candidate that a later check refused to
// the hourly comparison, with the refusal's reason and cause, in the hour of
// the ledger time it ended at. The ledger calls it once the ending commits,
// on whichever goroutine ended it.
func (o *Owner) countLaterRefusal(c admission.Candidate, err error, at time.Time) {
o.compare.addLedgerRefusalAt(refusalKey(c, err), at)
}
// refusalKey is the hourly comparison key of candidate c refused with err.
func refusalKey(c admission.Candidate, err error) compareKey {
return compareKey{entry: c.Entry, check: c.Check, kind: c.Key.Kind, decision: decisionRefused, reason: reasonOf(err), cause: admission.CauseOf(err)}
}
// reasonOf is err's admission reason; any other error is an invalid
// request.
func reasonOf(err error) admission.Reason {
if r, ok := admission.ReasonOf(err); ok {
return r
}
return admission.ReasonInvalid
}
// entryOf is the entry a response answers through: via, or its evidence's.
func entryOf(e admission.Evidence, via admission.Entry) admission.Entry {
if via != 0 {
return via
}
return e.Entry()
}
// Respond hands the ingress a response of kind to target with e as its
// root, through the derived entry via when it is set. The ingress refuses a
// target e does not attribute. It acknowledges memory acceptance
// only and never waits for the ledger.
func (o *Owner) Respond(kind admission.Kind, target string, e admission.Evidence, via admission.Entry, ttl ...time.Duration) error {
if o.ingress == nil {
return o.startErr
}
var t admission.Target
if target != "" {
parsed, err := checks.AdmissionTarget(target, admission.Caps{IPv6: true})
if err != nil {
o.ingress.Refuse(err, e.Severity())
o.compare.add(compareKey{entry: entryOf(e, via), check: e.Check(), kind: kind, decision: decisionRefused, reason: reasonOf(err)})
return err
}
t = parsed
}
var caps admission.Caps
if o.opts.Caps != nil {
caps = o.opts.Caps()
}
if kind != admission.KindChallenge && !caps.IPv6 && t.Prefix().Addr().Is6() {
err := &admission.Error{Reason: admission.ReasonUnsupportedContainment, Detail: "firewall does not contain IPv6"}
o.ingress.Refuse(err, e.Severity())
o.compare.add(compareKey{entry: entryOf(e, via), check: e.Check(), kind: kind, decision: decisionRefused, reason: admission.ReasonUnsupportedContainment})
return err
}
var selected time.Duration
if len(ttl) > 1 {
err := &admission.Error{Reason: admission.ReasonInvalid, Detail: "response names several lifetimes"}
o.ingress.Refuse(err, e.Severity())
o.compare.add(compareKey{entry: entryOf(e, via), check: e.Check(), kind: kind, decision: decisionRefused, reason: admission.ReasonInvalid})
return err
}
if len(ttl) == 1 {
selected = ttl[0]
}
err := o.ingress.Submit(admission.Submission{Kind: kind, Target: t, Evidence: e, Via: via, PreviewTTL: selected})
if err != nil {
o.compare.add(compareKey{entry: entryOf(e, via), check: e.Check(), kind: kind, decision: decisionRefused, reason: reasonOf(err), cause: admission.CauseOf(err)})
}
return err
}
// Status copies the last status the owner read.
func (o *Owner) Status() *health.AdmissionStatus { return o.current.Load().Clone() }
// status reads the status now, on the owner goroutine.
func (o *Owner) status() *health.AdmissionStatus {
_ = o.do(func() error { o.refreshStatus(); return nil })
return o.Status()
}
// refreshStatus reads the ledger and the ingress for status and doctor. The
// ledger's read takes no lock and needs no current clock.
func (o *Owner) refreshStatus() {
s := &health.AdmissionStatus{CheckedAt: time.Now(), Owner: &health.AdmissionOwner{
Executing: o.executing(),
Settled: o.settled,
ClockDegraded: o.degraded, LastTick: o.lastTick, CeilingSource: o.source,
Import: o.imported, InventoryAt: o.inventoryAt,
}}
switch {
case o.startErr != nil:
s.Owner.Error = o.startErr.Error()
case o.drainErr != nil:
s.Owner.Error = o.drainErr.Error()
case o.snapshotErr != nil:
s.Owner.Error = o.snapshotErr.Error()
}
if o.scheduleErr != nil {
s.Owner.TurnError = o.scheduleErr.Error()
}
if err := errors.Join(o.retireErr, o.evictedErr); err != nil {
s.Owner.RetirementError = err.Error()
}
if o.auditErr != nil {
s.Owner.AuditError = o.auditErr.Error()
}
if o.tickErr != nil {
s.Owner.TickError = o.tickErr.Error()
}
if o.inventoryErr != nil {
s.Owner.InventoryError = o.inventoryErr.Error()
}
if o.damageErr != nil {
s.Owner.DamageError = o.damageErr.Error()
}
if cause := o.heldCause(); cause != nil {
s.Owner.ExecutionHeld, s.Owner.ExecutionHeldBy = cause.Error(), o.heldBy()
}
if o.executable() {
s.Owner.Verdict, s.Owner.DryRun = o.verdictStatus(), o.dryRunStatus()
}
s.Owner.PreviewReplacements, s.Owner.PreviewReplacementsLastHour = o.replacements, o.replacementsHour.total(monoNow())
s.Owner.PreviewRotations, s.Owner.PreviewRotationsLastHour = o.rotations, o.rotationsHour.total(monoNow())
if o.ledger != nil {
ls := o.ledger.Status()
s.Ledger = &ls
if ls.Unresolved.Unknown > 0 {
s.Owner.RetirementHeldBy = o.retirementHeldBy()
}
if o.set != nil && ls.Effects.Error == "" {
s.Owner.Set = setStatus(*o.set, ls.Effects)
}
}
if o.setErr != nil {
s.Owner.SetError = o.setErr.Error()
}
if o.ingress != nil {
in := o.ingress.Health()
s.Ingress = &in
}
o.current.Store(s)
}
package admissionowner
import (
"context"
"errors"
"fmt"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/verdict"
)
// Asker asks the panel's verdict callback about one block. It must return
// once ctx ends.
type Asker func(ctx context.Context, ip, reason string) (verdict.Response, error)
// ExecutionPolicy is one coherent reading of the configuration automatic
// execution depends on. Reading the settings one getter at a time
// could combine two reloads. The publisher numbers each reading: Epoch
// grows with every change of these settings, including the callback's
// endpoint and authentication and the response lifetimes the closures
// capture, so a repeated reading is the same policy.
type ExecutionPolicy struct {
Epoch uint64
// Observe is mode: observe, which forbids an executable owner.
Observe bool
// DryRun is auto_response.dry_run resolved, true when omitted.
DryRun bool
VerdictEnabled bool
VerdictTimeout time.Duration
CeilingLimit uint32
CeilingSource string
// Expiry is how long the response a candidate asks for lasts.
Expiry func(admission.Candidate) time.Duration
// BlockExpiry caps a new automatic response's lifetime.
// Earlier attempts keep their recorded absolute expiry.
BlockExpiry time.Duration
// AskVerdict asks the callback this policy configures, with the
// client's own validation.
AskVerdict Asker
// Allows reports whether the configuration still allows the response
// a queued candidate asks for. roots loads the candidate's root
// evidence; it returns nil if they cannot be read. The callback runs
// under the ledger write transaction: consume roots synchronously,
// never retain the loader or call back into the ledger.
Allows func(c admission.Candidate, roots func() []admission.Evidence) bool
}
// ErrPolicy marks an execution policy the owner refused to install.
var ErrPolicy = errors.New("execution policy refused")
const (
minVerdictTimeout = time.Second
maxVerdictTimeout = 30 * time.Second
)
func (p ExecutionPolicy) validate() error {
var problem string
switch {
case p.Epoch == 0:
problem = "no policy is published"
case p.Observe:
problem = "mode: observe forbids automatic execution"
case p.Expiry == nil || p.AskVerdict == nil || p.Allows == nil:
problem = "the policy has no expiry, verdict asker or allowed responses"
case p.CeilingLimit == 0 || p.CeilingLimit > admission.MaxCeiling:
problem = "the hourly ceiling is out of range"
case p.BlockExpiry <= 0:
problem = "the block expiry is not positive"
case p.VerdictTimeout < minVerdictTimeout || p.VerdictTimeout > maxVerdictTimeout:
problem = "the verdict timeout is out of range"
default:
return nil
}
return fmt.Errorf("%w: %s", ErrPolicy, problem)
}
// same reports whether q repeats p's settings. The closures cannot be
// compared; the publisher's epoch covers what they capture.
func (p ExecutionPolicy) same(q ExecutionPolicy) bool {
return p.Epoch == q.Epoch && p.Observe == q.Observe && p.DryRun == q.DryRun &&
p.VerdictEnabled == q.VerdictEnabled && p.VerdictTimeout == q.VerdictTimeout &&
p.CeilingLimit == q.CeilingLimit && p.CeilingSource == q.CeilingSource && p.BlockExpiry == q.BlockExpiry
}
// executable reports whether the owner runs block picks (Options.Blocks).
func (o *Owner) executable() bool { return o.opts.Blocks != nil }
// liveExecution lets an executable owner run block picks under a live
// policy. It stays false while legacy blocking still enforces, so the same
// observation cannot trigger both legacy and admission execution. A live
// policy previews block picks; a dry-run policy still forecasts. No
// configuration reaches it; only this package's tests enable it.
var liveExecution = false
// executing reports whether block picks run live: an engine, live
// execution and a policy that is not a dry run. A later refused policy
// holds that execution without ending it (heldCause).
func (o *Owner) executing() bool {
return liveExecution && o.executable() && !o.policy.DryRun
}
// mayExecute reports whether block picks, probes and repairs may run now:
// live execution under a valid installed policy.
func (o *Owner) mayExecute() bool { return o.executing() && o.policyErr == nil }
// mayRetire reports whether earlier real attempts retire now: an engine
// under a valid installed policy that runs live or forecasts.
func (o *Owner) mayRetire() bool {
return o.executable() && o.policyErr == nil && (liveExecution || o.forecasting())
}
// retirementHeldBy names what keeps earlier unknown outcomes from
// retiring: "engine" for an owner without the firewall engine, "policy"
// while a refused execution policy holds it, and nothing when retirement
// runs.
func (o *Owner) retirementHeldBy() string {
switch {
case !o.executable():
return "engine"
case o.policyErr != nil:
return "policy"
}
return ""
}
// asksVerdicts reports whether picks ask the installed policy's callback:
// only a forecast or live execution does.
func (o *Owner) asksVerdicts() bool {
return o.policyErr == nil && o.policy.VerdictEnabled && (o.forecasting() || o.executing())
}
// refreshPolicy reads the published policy at a turn boundary and installs
// a newer one; the installed policy then serves the whole turn. An older
// epoch is ignored and the same one again changes nothing. Different
// settings under an installed epoch, or an invalid newer policy, hold
// execution until a valid newer epoch is installed.
func (o *Owner) refreshPolicy() error {
before := o.policyErr
var p ExecutionPolicy
defer func() {
// A refusal that stops a dry run is a forecast error too.
if o.policyErr != nil && before == nil && (o.policy.DryRun || p.DryRun) {
o.noteForecastError(forecastInvalidPolicy)
}
// The new policy is visible even if the next clock reading fails.
// Discard the live set before publishing a dry-run policy.
if o.forecasting() && o.ledger != nil {
o.clearSet()
}
// Status shows the refusal while it holds execution.
if o.policyErr != before && o.current.Load() != nil {
o.publishExecutionHold()
}
o.syncPreviews()
o.executesBlocks.Store(o.mayExecute())
}()
if o.opts.Policy == nil {
o.policyErr = fmt.Errorf("%w: no policy provider", ErrPolicy)
return o.policyErr
}
p = o.opts.Policy()
switch {
case p.Epoch == 0:
o.policyErr = p.validate()
o.retireVerdicts()
return o.policyErr
case p.Epoch < o.seenEpoch:
return o.policyErr
case p.Epoch == o.seenEpoch:
if o.policyErr == nil {
o.policyErr = p.validate()
if o.policyErr == nil && !o.policy.same(p) {
o.policyErr = fmt.Errorf("%w: epoch %d was published again with other settings", ErrPolicy, p.Epoch)
}
if o.policyErr != nil {
o.retireVerdicts()
}
}
return o.policyErr
}
o.seenEpoch = p.Epoch
// Every new epoch, a mode change included, invalidates callback work.
o.retireVerdicts()
if err := p.validate(); err != nil {
o.policyErr = err
return err
}
prev := o.policy
o.policy, o.policyErr, o.policyHeld = p, nil, nil
o.noteMode()
if !p.DryRun {
// A live policy ends the forecast; a later dry run starts anew.
// Its examples are purged at this boundary, as are those a dry run
// before a restart left.
o.endSession(resetModeChange)
if prev.Epoch == 0 || prev.DryRun {
o.requestPurge()
}
}
o.startVerdicts()
return nil
}
// syncPreviews tells the ledger whether observe previews answer their
// episodes: they do unless the owner forecasts.
func (o *Owner) syncPreviews() {
if o.ledger != nil {
o.ledger.PreviewsAnswer(!o.forecasting())
}
}
// ceilingConfig is the configured hourly ceiling and its source. An
// executable owner reads it from its installed policy.
func (o *Owner) ceilingConfig() (uint32, string) {
if o.executable() {
return o.policy.CeilingLimit, o.policy.CeilingSource
}
return o.opts.Ceiling()
}
// expiry is how long c's response lasts under the configuration.
func (o *Owner) expiry(c admission.Candidate) time.Duration {
if o.executable() {
return o.policy.Expiry(c)
}
return o.opts.Expiry(c)
}
// lease is the longest a response may last under the configuration:
// its block expiry, or no bound without one.
func (o *Owner) lease() time.Duration {
if o.executable() {
return o.policy.BlockExpiry
}
if o.opts.BlockExpiry == nil {
return 0
}
return o.opts.BlockExpiry()
}
// refuseDisabled runs before selection: an opt-out must not wait for the
// capacity, retry timer or recovery hold that gates a new response.
func (o *Owner) refuseDisabled() error {
if o.executable() && o.policyErr == nil {
if err := o.ledger.RefuseDisabled(o.policy.Allows); err != nil {
return fmt.Errorf("refusing disabled responses: %w", err)
}
}
return nil
}
// endEarlierIncidents ends, as disabled, the incident and spray blocks an
// earlier process queued. The correlator reads its selectors when the
// process starts, so that work may no longer be selected; the incident's
// next block still applies. Only an owner that executes blocks ends it.
func (o *Owner) endEarlierIncidents() error {
if liveExecution && o.executable() && o.policyErr == nil {
if err := o.ledger.RefuseDisabled(func(c admission.Candidate, _ func() []admission.Evidence) bool {
return c.Entry != admission.EntryIncident && c.Entry != admission.EntryIncidentSpray
}); err != nil {
return fmt.Errorf("ending incident blocks queued before start: %w", err)
}
}
return nil
}
// heldCause is why automatic execution is held, or nil.
func (o *Owner) heldCause() error {
if o.policyErr != nil {
return o.policyErr
}
return o.execHold
}
// heldBy names what heldCause comes from, so doctor can say what to
// repair: the configuration or the firewall.
func (o *Owner) heldBy() string {
switch {
case o.policyErr != nil:
return "policy"
case o.execHold != nil:
return "firewall"
}
return ""
}
package admissionowner
import (
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/checks"
)
// Registry registers every producer of the producer table against the
// check registry's admission policy and seals it. The producer handles are
// the owner's: only it mints evidence.
func Registry() (*admission.Registry, map[admission.ProducerID]*admission.Producer, error) {
reg, err := admission.NewRegistry(checks.AdmissionPolicy)
if err != nil {
return nil, nil, err
}
producers := map[admission.ProducerID]*admission.Producer{}
for _, p := range checks.ProducerTable() {
handle, err := reg.Register(p.Spec)
if err != nil {
return nil, nil, err
}
producers[handle.ID()] = handle
}
for _, spec := range checks.DerivedEntries() {
if _, regErr := reg.Register(spec); regErr != nil {
return nil, nil, regErr
}
}
reg.Seal()
return reg, producers, nil
}
// Inventory is one complete read of the hosting inventory as the ledger
// observes it.
func Inventory() (admission.InventoryObservation, error) {
s, err := checks.HostingInventory()
if err != nil {
return admission.InventoryObservation{}, err
}
return admission.InventoryObservation{Accounts: s.Accounts, Domains: s.Domains, AmbiguousDomains: s.AmbiguousDomains, Incarnations: s.Incarnations}, nil
}
package admissionowner
import (
"errors"
"fmt"
"sort"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/firewall"
)
// retireRecheck is how long an unknown attempt whose element the kernel
// still holds waits before the engine is asked again.
const retireRecheck = time.Minute
// retire retires unknown attempts whose expiry has passed and of which the
// engine finds nothing left. The engine removes each
// clear attempt's own row first, then the ledger retires it: a stop between
// the two leaves an unknown attempt without a row, which the next pass
// finds clear again. Running attempts are never sent. A degraded clock or a
// missing engine retires nothing and tries again after a backoff from one
// second to a minute; a held attempt waits a minute on its own, so a held
// batch cannot starve the intents behind it.
func (o *Owner) retire() error {
if !o.mayRetire() || monoNow().Before(o.retireAt) {
return nil
}
// Nothing noted runs the pass; otherwise it waits until the noted
// expiry or retry, whichever comes first.
expiryAhead := o.retireDue.IsZero() || o.now.Before(o.retireDue)
retryAhead := o.retireNext.IsZero() || monoNow().Before(o.retireNext)
if expiryAhead && retryAhead && (!o.retireDue.IsZero() || !o.retireNext.IsZero()) {
return nil
}
intents, err := unresolvedIntents(o.ledger)
if err != nil {
if errors.Is(err, admission.ErrCorruptRecord) {
o.retireAt = monoNow().Add(retireRecheck)
o.noteRetirementRetry(o.retireAt)
} else {
o.backoffRetirement()
}
if o.retireReadFault {
return nil
}
o.retireReadFault = true
return fmt.Errorf("reading unresolved intents: %w", err)
}
o.retireReadFault = false
o.retireDue, o.retireNext = time.Time{}, time.Time{}
waiting, delays, faults := o.retireRetry, o.retireDelay, o.retireFaults
o.retireRetry = make(map[admission.ActionID]time.Time)
o.retireDelay = make(map[admission.ActionID]time.Duration)
o.retireFaults = make(map[admission.ActionID]bool)
var due []admission.Intent
var errs error
// Intents are in action order. Rotate the scan so due retries cannot
// starve the tail when a turn takes longer than their waits.
start := sort.Search(len(intents), func(i int) bool { return intents[i].Action > o.retireAfter })
for offset := range len(intents) {
in := intents[(start+offset)%len(intents)]
if in.Kind != admission.KindBlockIP {
continue
}
if d, ok := delays[in.Action]; ok {
o.retireDelay[in.Action] = d
}
if faults[in.Action] {
o.retireFaults[in.Action] = true
}
// A damaged attempt can fail to decode before retirement reaches
// the ledger write. Respect its wait before reading it again.
if at := waiting[in.Action]; monoNow().Before(at) {
o.retireRetry[in.Action] = at
o.noteRetirementRetry(at)
continue
}
a, readErr := o.ledger.Attempt(in.Action)
if readErr != nil {
if reason, refused := admission.ReasonOf(readErr); refused && reason == admission.ReasonInvalid {
readErr = fmt.Errorf("%w: intent is missing its attempt", admission.ErrCorruptRecord)
}
errs = errors.Join(errs, o.retirementResult(in.Action, fmt.Errorf("reading retirement attempt %s: %w", in.Action, readErr)))
continue
}
if a.State != admission.StateUnknown || a.Retired {
delete(o.retireDelay, in.Action)
delete(o.retireFaults, in.Action)
continue
}
if o.now.Before(in.ExpiresAt) {
// The attempt read cleanly, and nothing else can fault before
// its expiry: an earlier read fault is over.
delete(o.retireDelay, in.Action)
delete(o.retireFaults, in.Action)
o.noteRetirement(in.ExpiresAt)
continue
}
if len(due) == firewall.MaxBlockBatchMembers {
o.noteRetirementRetry(monoNow())
continue
}
due = append(due, in)
}
if len(due) == 0 {
if len(intents) == 0 {
o.retireCheckFault = false
}
return errs
}
if o.degraded {
o.backoffRetirement()
return errs
}
engine := o.opts.Blocks()
if engine == nil {
o.backoffRetirement()
return errs
}
members := make([]firewall.BlockAction, len(due))
for i, in := range due {
addr, _ := in.Target.Addr()
members[i] = firewall.BlockAction{Action: string(in.Action), IP: addr.String()}
}
gone, checked, checkErr := o.checkRetirements(engine, members)
if !checked {
return errors.Join(errs, checkErr)
}
o.retireAfter = due[len(due)-1].Action
for i, in := range due {
if !gone[i] {
// The attempt read cleanly and the engine answered: an earlier
// read fault is over while the element stays.
delete(o.retireDelay, in.Action)
delete(o.retireFaults, in.Action)
o.retryRetirement(in.Action, retireRecheck)
continue
}
_, _, _, err = retireLedger(o.ledger, in.Action)
if err != nil {
err = fmt.Errorf("retiring %s: %w", in.Action, err)
}
errs = errors.Join(errs, o.retirementResult(in.Action, err))
}
return errs
}
// Reads and writes share the same retry policy: a fault must not fail any
// turn or keep unrelated attempts from reaching the engine. Each faulted
// attempt is retried on its own and reported once.
func (o *Owner) retirementResult(id admission.ActionID, err error) error {
switch _, refused := admission.ReasonOf(err); {
case err == nil:
delete(o.retireDelay, id)
delete(o.retireFaults, id)
return nil
case errors.Is(err, admission.ErrNotReady), refused:
// Delivery or the ledger clock has not caught up. Keep a future
// wake even if no new candidates arrive.
o.retryRetirementLater(id)
return nil
case errors.Is(err, admission.ErrCorruptRecord), errors.Is(err, admission.ErrTransitionConflict):
// Retrying soon cannot repair damage.
o.retryRetirement(id, retireRecheck)
default:
o.retryRetirementLater(id)
}
if o.retireFaults[id] {
return nil
}
o.retireFaults[id] = true
return err
}
// reportRetirement keeps a newly reported retirement fault for status
// until no attempt, intent list read or engine check is left faulted.
func (o *Owner) reportRetirement(err error) {
was := o.retireErr
switch {
case err != nil:
o.retireErr = err
case len(o.retireFaults) == 0 && !o.retireReadFault && !o.retireCheckFault:
o.retireErr = nil
}
if o.retireErr != was {
o.refreshStatus()
}
}
// checkRetirements asks the engine about members. A failed or short answer
// backs off without failing the turn.
func (o *Owner) checkRetirements(engine BlockEngine, members []firewall.BlockAction) ([]bool, bool, error) {
gone, err := engine.RetireBlockActions(members)
if err == nil && len(gone) != len(members) {
err = fmt.Errorf("%d retirement answers for %d members", len(gone), len(members))
}
if err != nil {
o.backoffRetirement()
if o.retireCheckFault {
return nil, false, nil
}
o.retireCheckFault = true
return nil, false, fmt.Errorf("checking block retirements: %w", err)
}
o.retireCheckFault = false
o.retireWait, o.retireAt = 0, time.Time{}
return gone, true, nil
}
// backoffRetirement delays the next pass while nothing can be checked: an
// engine that fails or is missing, or a degraded clock. The wait starts at
// one second and doubles to a minute; a successful check resets it.
func (o *Owner) backoffRetirement() {
o.retireWait = min(max(2*o.retireWait, firstProbeWait), maxProbeWait)
o.retireAt = monoNow().Add(o.retireWait)
o.noteRetirementRetry(o.retireAt)
}
// retryRetirementLater holds one attempt back while the ledger refuses
// to retire it: one second, doubling to retireRecheck.
func (o *Owner) retryRetirementLater(id admission.ActionID) {
wait := min(max(2*o.retireDelay[id], firstProbeWait), retireRecheck)
o.retireDelay[id] = wait
o.retryRetirement(id, wait)
}
// retryRetirement holds one attempt back for wait on the monotonic clock.
func (o *Owner) retryRetirement(id admission.ActionID, wait time.Duration) {
at := monoNow().Add(wait)
o.retireRetry[id] = at
o.noteRetirementRetry(at)
}
// noteRetirement keeps the earliest admission time a retirement pass is due.
func (o *Owner) noteRetirement(at time.Time) {
if o.retireDue.IsZero() || at.Before(o.retireDue) {
o.retireDue = at
}
}
// Retry deadlines stay on the monotonic clock across slow calls and
// degraded admission readings. Only future expiries need conversion.
func (o *Owner) noteRetirementRetry(at time.Time) {
if o.retireNext.IsZero() || at.Before(o.retireNext) {
o.retireNext = at
}
}
func (o *Owner) retirementWake(sample time.Time) time.Time {
at := o.retireNext
if !o.retireDue.IsZero() {
expires := sample.Add(max(o.retireDue.Sub(o.now), 0))
if at.IsZero() || expires.Before(at) {
at = expires
}
}
if !at.IsZero() && o.retireAt.After(at) {
at = o.retireAt
}
return at
}
package admissionowner
import (
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/health"
)
// readSet reads the temporary set for the turn's schedule and reservations
// when queued work may take a slot: the firewall's entries
// outside admission's records go to the ledger, which adds its own.
// Without an engine the ledger has no set and block work has no slot
// budget, as before; a failed read leaves it so and shows in status. The
// batch check that follows reads the same firewall state and holds
// execution when it fails too. One read-only snapshot names no member and
// changes nothing, so a preview owner reads the set as well.
func (o *Owner) readSet() error {
o.clearSet()
var engine BlockEngine
if o.executable() {
engine = o.opts.Blocks()
}
if engine == nil {
return nil
}
if charged, err := o.ledger.HasChargedWork(); err != nil || !charged {
return err
}
adm, err := engine.BlockAdmissionSnapshot(nil, nil)
if err == nil && adm.Admission > 0 {
// Admission rows beyond the ledger's own records, as after a lost
// ledger, hold slots no record can give up.
var holds admission.EffectTally
if holds, err = o.ledger.EffectTally(); err != nil {
return err
}
if unknown := adm.Admission - holds.General - holds.Reserved; unknown > 0 {
adm.Unclaimed += unknown
// The engine's drain excludes admission rows, so it cannot
// bound the remaining time of these additional occupants.
adm.Drain = -time.Second
}
}
if o.setErr = err; err == nil {
o.set = &adm
// An engine that cannot prove admission outcomes removes no block
// for a batch, so the ledger plans none.
o.ledger.SetOccupants(&admission.SetOccupancy{Limit: adm.Limit, Unclaimed: adm.Unclaimed, General: adm.Imported, NoEviction: adm.Unproven})
o.ledger.SetImports(importVictims(adm.Imports))
}
return nil
}
// importVictims is the engine's import list as the ledger takes it: known
// automatic imports, (C2, High) victims of the general partition named by
// their effect.
func importVictims(imports []firewall.BlockImport) []admission.EvictionVictim {
out := make([]admission.EvictionVictim, 0, len(imports))
for _, imp := range imports {
target, err := admission.CanonicalAddress(imp.IP, admission.Caps{IPv6: true})
if err != nil {
continue
}
out = append(out, admission.EvictionVictim{Action: admission.ActionID(imp.ID), Target: target, Tier: admission.ImportTier,
Partition: admission.PartitionGeneral, ExpiresAt: imp.ExpiresAt})
}
return out
}
// clearSet forgets the set: a forecast turn models its own.
func (o *Owner) clearSet() {
o.set, o.setErr = nil, nil
o.ledger.SetOccupants(nil)
o.ledger.SetImports(nil)
}
// setStatus combines the last scheduling turn's non-admission occupants
// with the current ledger holds. Engine admission rows are already counted
// by effects and must not be added again.
func setStatus(adm firewall.BlockAdmission, effects admission.EffectsStatus) *health.AdmissionSet {
occ := admission.SetOccupancy{Limit: adm.Limit, Unclaimed: adm.Unclaimed, General: adm.Imported + effects.General, Reserved: effects.Reserved}
slots := occ.Slots()
return &health.AdmissionSet{
Limit: adm.Limit, Unlimited: adm.Limit <= 0, Capacity: slots.Capacity,
Unclaimed: adm.Unclaimed, Imported: adm.Imported, DrainSeconds: int64(adm.Drain / time.Second),
General: health.AdmissionDryRunPartition{Size: slots.General, Used: occ.General, Free: slots.FreeGeneral},
Reserved: health.AdmissionDryRunPartition{Size: slots.Reserved, Used: occ.Reserved, Free: slots.FreeReserved},
}
}
package admissionowner
import (
"encoding/hex"
"errors"
"io"
"maps"
"slices"
"sort"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/firewall"
)
// forecastEntries bounds active blocks plus retained answers to removed
// blocks. Both belong to one forecast; this is never a real set limit.
const forecastEntries = firewall.MaxShadowBlocks
// resetReason says why a forecast session started.
type resetReason uint8
const (
resetStartup resetReason = iota + 1
resetModeChange
// resetModelError: the previous session's model could not be metered.
resetModelError
)
func (r resetReason) String() string {
switch r {
case resetStartup:
return "startup"
case resetModeChange:
return "mode_change"
case resetModelError:
return "model_error"
}
return ""
}
// forecastError is why a forecast could not run or go on, a fixed code
// status shows.
type forecastError string
const (
forecastEngineUnavailable forecastError = "engine_unavailable"
forecastClockDegraded forecastError = "clock_degraded"
forecastLedgerUnavailable forecastError = "ledger_unavailable"
forecastSnapshotMismatch forecastError = "snapshot_mismatch"
forecastSessionID forecastError = "session_id_unavailable"
forecastInvalidPolicy forecastError = "invalid_policy"
// forecastModelError: the session's model could not be metered, so the
// session was dropped.
forecastModelError forecastError = "model_error"
)
// newSessionID reads a random 128-bit forecast session ID.
func newSessionID(r io.Reader) (string, error) {
var b [16]byte
if _, err := io.ReadFull(r, b[:]); err != nil {
return "", err
}
return hex.EncodeToString(b[:]), nil
}
// shadowEntry is one forecast block: its kind, absolute lease expiry, the
// partition of the set it occupies, its rank and its pick's scope, and
// the admission time and session elapsed time it was decided at, from
// which its rotation hold runs.
type shadowEntry struct {
kind admission.Kind
expires time.Time
partition admission.Partition
tier admission.Tier
action admission.ActionID
target admission.Target
scope string
applied time.Time
elapsed time.Duration
}
// shadowRank is what ranks a forecast block as a victim: the
// pick's tier, the action of its next attempt and its address. A block
// without a valid tier is never a victim, and a pick without one replaces
// nothing.
type shadowRank struct {
tier admission.Tier
action admission.ActionID
target admission.Target
// scope is the pick's fairness scope; fresh is set when its episode
// may still rotate an equal-tier block.
scope string
fresh bool
}
// shadowChange is a capacity admission a forecast decided and has not
// installed: one charge and one entry, and the lower-tier block it
// replaces or the equal-tier block it rotates out, zero for a free slot.
type shadowChange struct {
ip string
entry shadowEntry
charge admission.Charge
next admission.CeilingState
victim admission.EvictionVictim
}
// shadow is one dry-run forecast session. It copies the real ceiling when
// the session starts and then lives apart from it: simulated admissions
// charge only the copy, and the copy is metered by the same committed
// clock deltas as the ledger, never copied again. Only the owner goroutine
// uses it. A restart or a new session starts over from the real state.
type shadow struct {
// effects is the slots the ledger's records hold, read each turn.
effects admission.EffectTally
id string
since time.Time
reset resetReason
ceiling admission.CeilingState
// charges are the retained real and simulated charges in the order they
// leave the window, at most MaxCeiling.
charges []admission.Charge
entries map[string]shadowEntry
picks [admission.PredictionSlots]uint64
recent minuteCounts
// adm is the occupancy of the last admission snapshot, nil while none
// could be read; its answers are dropped.
adm *firewall.BlockAdmission
// replaced is the imports the session replaced, by name, until their
// lease expiry: the real set still holds and lists them.
replaced map[admission.ActionID]time.Time
// answered is the addresses whose forecast block the session replaced
// or rotated out, until that block's expiry: their episode had its
// response, so no later generation rotates.
answered map[string]time.Time
// waiting is the scopes of the turn's block picks by the partitions
// they may occupy, and cursors the session's remainder cursors: how
// the turn shares the set.
waiting map[admission.Partition][]string
cursors admission.QueueCursors
}
// newShadow starts session id at wall time since from the real ceiling and
// its retained charges, in key order.
func newShadow(id string, since time.Time, reset resetReason, ceiling admission.CeilingState, charges []admission.Charge) (*shadow, error) {
if raw, err := hex.DecodeString(id); err != nil || len(raw) != 16 || hex.EncodeToString(raw) != id {
return nil, errors.New("forecast session ID is malformed")
}
if ceiling.Limit == 0 || ceiling.Validate() != nil || len(charges) > admission.MaxCeiling {
return nil, errors.New("forecast needs a valid ceiling with a limit")
}
return &shadow{
id: id, since: since.UTC(), reset: reset, ceiling: ceiling,
charges: append([]admission.Charge(nil), charges...), entries: map[string]shadowEntry{},
replaced: map[admission.ActionID]time.Time{}, answered: map[string]time.Time{},
}, nil
}
// advance meters one committed clock reading as the ledger does: it credits
// the elapsed time and releases the charges that have left the window,
// oldest first. Release never refunds credit, and wall time alone earns or
// releases nothing.
func (s *shadow) advance(t admission.ClockTick) error {
next, err := s.ceiling.Advance(t.Elapsed)
if err != nil {
return err
}
released := 0
for _, c := range s.charges {
if !c.Releasable(t.Now, next.Elapsed) {
break
}
if next, err = next.Release(c.Lane, c.Cost); err != nil {
return err
}
released++
}
s.ceiling, s.charges = next, s.charges[released:]
return nil
}
// setLimit applies a new limit without refilling credit. Retained charges
// stay until they age out, even above a reduced limit.
func (s *shadow) setLimit(limit uint32) error {
next, err := s.ceiling.SetLimit(limit)
if err != nil {
return err
}
s.ceiling = next
return nil
}
// expire drops the forecast blocks, replaced imports and answered
// addresses whose lease has ended at lease, a reading of the engine's
// lease clock.
func (s *shadow) expire(lease time.Time) {
for ip, e := range s.entries {
if !lease.Before(e.expires) {
delete(s.entries, ip)
}
}
for name, expires := range s.replaced {
if !lease.Before(expires) {
delete(s.replaced, name)
}
}
for ip, expires := range s.answered {
if !lease.Before(expires) {
delete(s.answered, ip)
}
}
}
// ips are the forecast addresses, in order, for an admission snapshot to
// count the real blocks of.
func (s *shadow) ips() []string { return slices.Sorted(maps.Keys(s.entries)) }
// covering says how a forecast block holds ip for a pick lasting until
// expires: one that covers it as a real block would, within the engine's
// readback slack, answers it; a shorter one holds the address as live work
// would wait for it. Zero when none holds it.
func (s *shadow) covering(ip string, expires time.Time) admission.Prediction {
e, ok := s.entries[ip]
switch {
case !ok:
return 0
case !firewall.CoversExpiry(e.expires, expires):
return admission.PredictPendingRecovery
}
return admission.PredictExistingShadow
}
// occupancy is the modelled temporary set: the real timed occupants, with
// the slots the ledger's records hold in their partitions and admission
// rows beyond them unclaimed, as the live ledger reads the set; less the
// listed imports the session replaced; and in their own partitions the
// forecast blocks the real sets do not already hold as a timed or an
// untimed element. As in the ledger, every listed import counts even if
// the count is lower. adm is this turn's snapshot, taken after expire.
// Beside an engine that cannot prove its saved actions nothing is
// replaced, as in the ledger.
func (s *shadow) occupancy(adm firewall.BlockAdmission) admission.SetOccupancy {
general, reserved := s.counted(adm)
imported := max(adm.Imported, len(adm.Imports)) - s.replacedListed(adm)
unknown := max(adm.Admission-s.effects.General-s.effects.Reserved, 0)
return admission.SetOccupancy{Limit: adm.Limit, Unclaimed: adm.Unclaimed + unknown, General: imported + s.effects.General + general, Reserved: s.effects.Reserved + reserved, NoEviction: adm.Unproven}
}
// counted is the forecast blocks of each partition the real sets do not
// already hold.
func (s *shadow) counted(adm firewall.BlockAdmission) (general, reserved int) {
for ip, e := range s.entries {
switch {
case adm.ShadowHeld[ip]:
case e.partition == admission.PartitionReserved:
reserved++
default:
general++
}
}
return general, reserved
}
// replacedListed is how many imports adm lists that the session replaced.
// A replaced import the list no longer names counts again in Imported, as
// the forecast cannot tell whether it went or is merely unlisted.
func (s *shadow) replacedListed(adm firewall.BlockAdmission) int {
n := 0
for _, imp := range adm.Imports {
if _, ok := s.replaced[admission.ActionID(imp.ID)]; ok {
n++
}
}
return n
}
// limits is the modelled set's slot budget at admission time now, by the
// rule admission uses: its free slots and, once the general partition is
// full, the victims strictly higher work may replace: the ranked forecast
// blocks the real sets do not hold, and the listed imports not already
// replaced. Then the set is also shared among the scopes holding its
// forecast blocks and the turn's waiting scopes, from the session's
// cursors, so fresh work may rotate an equal-tier forecast block whose
// hold passed on the session's clocks; shared says so.
func (s *shadow) limits(adm firewall.BlockAdmission, now time.Time) (limits *admission.SlotLimits, shared bool) {
var holding map[admission.Partition]map[string]int
limits, _ = s.occupancy(adm).Limits(func() ([]admission.EvictionVictim, error) {
var pool []admission.EvictionVictim
holding = s.holding(adm)
for ip, e := range s.entries {
if !adm.ShadowHeld[ip] && e.partition.Valid() && e.tier.Valid() {
pool = append(pool, admission.EvictionVictim{Action: e.action, Target: e.target, Tier: e.tier, Partition: e.partition, ExpiresAt: e.expires,
Scope: e.scope, Applied: e.applied, Rotatable: admission.RotationHoldPassed(e.applied, e.elapsed, now, s.ceiling.Elapsed)})
}
}
for _, v := range importVictims(adm.Imports) {
if _, gone := s.replaced[v.Action]; !gone {
pool = append(pool, v)
}
}
return pool, nil
})
if holding == nil {
return limits, false
}
limits.Share(holding, s.waiting, s.cursors)
return limits, true
}
// holding is the forecast blocks each scope holds in each partition,
// those the real sets do not hold.
func (s *shadow) holding(adm firewall.BlockAdmission) map[admission.Partition]map[string]int {
holding := map[admission.Partition]map[string]int{}
for ip, e := range s.entries {
if adm.ShadowHeld[ip] || !e.partition.Valid() || e.scope == "" {
continue
}
if holding[e.partition] == nil {
holding[e.partition] = map[string]int{}
}
holding[e.partition][e.scope]++
}
return holding
}
// share begins a turn's shares: the scopes of its block picks wait for
// slots, and a turn that shares the set moves the session's remainder
// cursors once, before any pick, as a live schedule cycle does.
func (s *shadow) share(adm firewall.BlockAdmission, now time.Time, waiting map[admission.Partition][]string) {
s.waiting = waiting
if limits, shared := s.limits(adm, now); shared {
s.cursors = limits.Advance()
}
}
// demand is what a pick of kind ranked r on lanes asks of the set, as
// the ledger asks it: only one-unit block_ip work may replace another.
func demand(kind admission.Kind, lanes []admission.Lane, r shadowRank) admission.SlotDemand {
cost := kind.CeilingCost()
return admission.SlotDemand{Tier: r.tier, Cost: cost, Evicts: kind == admission.KindBlockIP && cost == 1, Target: r.target,
Reserved: slices.ContainsFunc(lanes, func(l admission.Lane) bool { return admission.PartitionOf(l) == admission.PartitionReserved }),
Scope: r.scope, Fresh: r.fresh}
}
// wait is why a block_ip pick ranked r on lanes waits for the set at
// admission time now, zero when a lane has room: a free slot in a
// partition it may occupy, a strictly lower-tier block it may replace or
// an equal-tier block it may rotate out. It waits as equal_tier_wait
// while a lane's partition holds an equal-tier block rotation does not
// give it yet, else as set_full. A zero limit is the finite automatic
// capacity.
func (s *shadow) wait(adm firewall.BlockAdmission, lanes []admission.Lane, r shadowRank, now time.Time) admission.Prediction {
limits, _ := s.limits(adm, now)
d := demand(admission.KindBlockIP, lanes, r)
equal := false
for _, lane := range lanes {
if _, ok := limits.Room(lane, d); ok {
return 0
}
equal = equal || limits.EqualTier(lane, d)
}
if equal {
return admission.PredictEqualTierWait
}
return admission.PredictSetFull
}
// limited counts retained answers too: forgetting one before expiry
// would grant a returning address another rotation privilege.
func (s *shadow) limited() bool { return len(s.entries)+len(s.answered) >= forecastEntries }
// admit decides a ready pick of kind for ip lasting until expires, ranked
// r, at admission time now, against this turn's snapshot: a full set
// first, then the modelling bound, then the ceiling on each of lanes in
// order that has room, the selected lane first. Room is a free slot of the
// lane's partition or, with none, the first strictly lower-tier block in
// eviction order, which the block replaces in that block's partition, or
// else the equal-tier block rotation takes in the lane's partition. A
// block on a free slot occupies the partition of the lane it is charged
// on. Only a capacity admission returns a change; nothing is installed
// until commit. Kinds outside the address model take no capacity.
func (s *shadow) admit(ip string, kind admission.Kind, expires time.Time, lanes []admission.Lane, now time.Time, adm firewall.BlockAdmission, r shadowRank) (admission.Prediction, *shadowChange, error) {
if kind != admission.KindBlockIP {
return admission.PredictNotModelled, nil, nil
}
if wait := s.wait(adm, lanes, r, now); wait != 0 {
return wait, nil, nil
}
if s.limited() {
return admission.PredictForecastLimited, nil, nil
}
limits, _ := s.limits(adm, now)
d := demand(kind, lanes, r)
cost := kind.CeilingCost()
for _, lane := range lanes {
victim, ok := limits.Room(lane, d)
if !ok {
continue
}
next, err := s.ceiling.Charge(lane, cost)
if reason, refused := admission.ReasonOf(err); refused && reason == admission.ReasonCeiling {
continue
}
if err != nil {
return 0, nil, err
}
pred, partition := admission.PredictCapacityAdmit, admission.PartitionOf(lane)
switch {
case victim == (admission.EvictionVictim{}):
case victim.Tier == r.tier:
pred, partition = admission.PredictCapacityRotate, victim.Partition
default:
pred, partition = admission.PredictCapacityEvict, victim.Partition
}
return pred, &shadowChange{
ip: ip, entry: shadowEntry{kind: kind, expires: expires, partition: partition, tier: r.tier, action: r.action, target: r.target,
scope: r.scope, applied: now, elapsed: s.ceiling.Elapsed}, next: next,
charge: admission.Charge{At: now, Lane: lane, Cost: cost, Elapsed: s.ceiling.Elapsed}, victim: victim,
}, nil
}
return admission.PredictCeiling, nil, nil
}
// commit installs a decision whose ending committed, and counts it at the
// process's monotonic time mono.
func (s *shadow) commit(p admission.Prediction, change *shadowChange, mono time.Time) {
if change != nil {
s.ceiling = change.next
// Imported legacy spend can be dated at the end of the current
// hour. Keep earlier predictions ahead of it for window release.
at := sort.Search(len(s.charges), func(i int) bool { return s.charges[i].At.After(change.charge.At) })
s.charges = slices.Insert(s.charges, at, change.charge)
// A replaced or rotated forecast block leaves the model, and its
// address is answered until that block's expiry; a replaced import
// stays in the real set, so the session remembers it.
switch v := change.victim; {
case v == (admission.EvictionVictim{}):
case admission.ImportedVictim(v.Action):
s.replaced[v.Action] = v.ExpiresAt
default:
addr, _ := v.Target.Addr()
if v.ExpiresAt.After(s.answered[addr.String()]) {
s.answered[addr.String()] = v.ExpiresAt
}
delete(s.entries, addr.String())
}
s.entries[change.ip] = change.entry
}
if s.picks[p] != ^uint64(0) {
s.picks[p]++
}
s.recent.add(mono, p)
}
// minuteCounts keeps sixty one-minute buckets of predictions on the
// process's monotonic clock, aligned to the first count, so a wall-clock
// change cannot age them.
type minuteCounts struct {
base time.Time
buckets [60]minuteBucket
}
type minuteBucket struct {
set bool
minute int64
counts [admission.PredictionSlots]uint64
}
func (m *minuteCounts) minute(mono time.Time) int64 {
return int64(max(mono.Sub(m.base), 0) / time.Minute)
}
func (m *minuteCounts) add(mono time.Time, p admission.Prediction) {
if m.base.IsZero() {
m.base = mono
}
minute := m.minute(mono)
b := &m.buckets[minute%int64(len(m.buckets))]
if !b.set || b.minute != minute {
*b = minuteBucket{set: true, minute: minute}
}
if b.counts[p] != ^uint64(0) {
b.counts[p]++
}
}
// total sums the sixty minutes up to and including mono's.
func (m *minuteCounts) total(mono time.Time) [admission.PredictionSlots]uint64 {
var out [admission.PredictionSlots]uint64
if m.base.IsZero() {
return out
}
now := m.minute(mono)
for _, b := range m.buckets {
if !b.set || b.minute > now || b.minute <= now-int64(len(m.buckets)) {
continue
}
for p, n := range b.counts {
out[p] += min(n, ^uint64(0)-out[p])
}
}
return out
}
package admissionowner
import (
"context"
"fmt"
"strings"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
)
// Bounds of the dry-run summary.
const (
// summaryEvery is the least time from a summary's first prediction to
// its delivery, and between two delivery attempts.
summaryEvery = time.Minute
// summaryExamples bounds the distinct examples one summary names.
summaryExamples = 20
// Shutdown bounds the final history attempt; normal delivery is
// cancelled through the owner's context before its sender is joined.
summaryStopTimeout = time.Second
)
type summaryResult struct {
summary *dryRunSummary
err error
}
// summaryExample is one capacity admission a summary names.
type summaryExample struct {
candidate admission.CandidateID
finding string
target, check string
}
// dryRunSummary counts the forecast's committed predictions between two
// deliveries. It is a fixed size: counts, at most summaryExamples examples
// in committed order and a session count.
type dryRunSummary struct {
counts [admission.PredictionSlots]uint64
examples []summaryExample
// sessions counts the sessions it covers, from first to last.
sessions int
first, last string
start, end time.Time
firstPrediction time.Time
// overflowed is set once a count could not grow, counted lost once.
overflowed bool
}
// summaryState is the owner's side of the summary: the one pending summary,
// the one in delivery, and the delivery slot that keeps summaries to one a
// minute whatever the mode. Only the owner goroutine uses it; the sender
// holds an immutable copy while it delivers.
type summaryState struct {
pending *dryRunSummary
inFlight bool
// lastAttempt is the monotonic time of the last delivery attempt, a
// failed one included; retry backs off failures.
lastAttempt time.Time
retry retryBackoff
}
// noteSummary counts p's committed prediction, and names a capacity
// admission as an example while the summary has room.
func (o *Owner) noteSummary(p execPick, pred admission.Prediction, change *shadowChange) {
s := o.summary.pending
if s == nil {
s = &dryRunSummary{start: time.Now().UTC(), firstPrediction: monoNow()}
o.summary.pending = s
}
s.end = time.Now().UTC()
if s.counts[pred] == ^uint64(0) {
o.summaryOverflow(s)
} else {
s.counts[pred]++
}
if id := o.shadow.id; id != s.last {
if s.sessions == 0 {
s.first = id
}
s.sessions++
s.last = id
}
if change != nil && len(s.examples) < summaryExamples && !s.names(change.ip) {
s.examples = append(s.examples, summaryExample{candidate: p.pick.ID, finding: p.c.FindingID, target: change.ip, check: registeredCheck(p.c.Check)})
}
}
// summaryOverflow counts one lost summary the first time a count of s
// cannot grow.
func (o *Owner) summaryOverflow(s *dryRunSummary) {
if !s.overflowed {
s.overflowed = true
saturating(&o.out.summariesLost)
}
}
func (s *dryRunSummary) names(target string) bool {
for _, e := range s.examples {
if e.target == target {
return true
}
}
return false
}
// summaryDue reports whether the pending summary may be delivered now: a
// minute after its first prediction and after the last attempt, once a
// failing delivery's backoff allows, and not while another is delivered.
func (o *Owner) summaryDue() bool {
st := &o.summary
now := monoNow()
return st.pending != nil && !st.inFlight && st.retry.due(now) &&
!now.Before(st.pending.firstPrediction.Add(summaryEvery)) &&
(st.lastAttempt.IsZero() || !now.Before(st.lastAttempt.Add(summaryEvery)))
}
// takeSummary hands the sender the due summary. The owner keeps no
// reference to it: later predictions start the next one.
func (o *Owner) takeSummary() *dryRunSummary {
if !o.summaryDue() {
return nil
}
st := &o.summary
s := st.pending
st.pending, st.inFlight, st.lastAttempt = nil, true, monoNow()
return s
}
// summaryDelivered settles a delivery attempt. A failed summary goes back
// in front of what accumulated meanwhile, as one summary.
func (o *Owner) summaryDelivered(s *dryRunSummary, err error) {
st := &o.summary
st.inFlight = false
if err == nil {
st.retry = retryBackoff{}
return
}
st.retry.fail(monoNow(), defaultNoticeEvery, maxRetryWait)
if later := st.pending; later != nil {
o.mergeSummary(s, later)
}
st.pending = s
}
// mergeSummary adds later, which followed s, into s.
func (o *Owner) mergeSummary(s, later *dryRunSummary) {
for p, n := range later.counts {
if n > ^uint64(0)-s.counts[p] {
o.summaryOverflow(s)
}
s.counts[p] += min(n, ^uint64(0)-s.counts[p])
}
for _, e := range later.examples {
if len(s.examples) < summaryExamples && !s.names(e.target) {
s.examples = append(s.examples, e)
}
}
s.sessions += later.sessions
if s.last == later.first {
s.sessions--
}
s.last, s.end = later.last, later.end
}
// finishSummary settles the pending summary when the owner stops: one its
// minute allows is delivered, any other is counted discarded. Stop never
// sends a summary early. The caller runs this after joining the owner and
// sender, so delivery never occupies the owner goroutine.
func (o *Owner) finishSummary() {
if o.summary.pending == nil {
return
}
if o.summaryDue() && o.opts.DeliverSummary != nil {
ctx, cancel := context.WithTimeout(context.Background(), summaryStopTimeout)
defer cancel()
if o.opts.DeliverSummary(ctx, summaryFinding(o.summary.pending, time.Now())) == nil {
o.summary.pending = nil
return
}
}
o.summary.pending = nil
saturating(&o.out.summariesLost)
}
// summaryFinding is the history record of s: the capacity admissions, with
// or without a replacement or rotation, in its message, every other prediction, its
// interval, its session count and its examples as
// candidate/finding/target/check.
func summaryFinding(s *dryRunSummary, now time.Time) alert.Finding {
var b strings.Builder
fmt.Fprintf(&b, "start=%s end=%s sessions=%d", s.start.Format(time.RFC3339), s.end.Format(time.RFC3339), s.sessions)
for p := admission.Prediction(1); p.Valid(); p++ {
if p != admission.PredictCapacityAdmit {
fmt.Fprintf(&b, " %s=%d", p, s.counts[p])
}
}
if len(s.examples) > 0 {
names := make([]string, len(s.examples))
for i, e := range s.examples {
names[i] = string(e.candidate) + "/" + e.finding + "/" + e.target + "/" + e.check
}
b.WriteString(" examples=" + strings.Join(names, ","))
}
passed := s.counts[admission.PredictCapacityAdmit]
passed += min(s.counts[admission.PredictCapacityEvict], ^uint64(0)-passed)
passed += min(s.counts[admission.PredictCapacityRotate], ^uint64(0)-passed)
return alert.Finding{
Check: "auto_block", Severity: alert.Warning, Details: b.String(), Timestamp: now,
Message: fmt.Sprintf("Dry-run capacity forecast: %d responses passed modelled capacity checks", passed),
}
}
// sendSummary delivers a due dry-run summary to history only, through the
// independent path, never to an alert channel. A summary waiting for its
// delivery counts in the notice queue, so a failing delivery shows there.
func (s *sender) sendSummary() {
if s.o.opts.DeliverSummary == nil {
return
}
var due *dryRunSummary
if s.o.do(func() error {
due = s.o.takeSummary()
s.summaryWaiting = s.o.summary.pending != nil || due != nil
return nil
}) != nil || due == nil {
s.observe()
return
}
s.observe()
err := s.o.opts.DeliverSummary(s.o.summaryCtx, summaryFinding(due, time.Now()))
if err == nil {
s.delivered++
}
if ackErr := s.o.do(func() error {
s.o.summaryDelivered(due, err)
s.summaryWaiting = s.o.summary.pending != nil
return nil
}); ackErr != nil {
s.o.summaryResult <- summaryResult{summary: due, err: err}
}
s.observe()
}
package admissionowner
import (
"context"
"encoding/json"
"fmt"
"maps"
"slices"
"strings"
"sync"
"time"
"unicode"
"unicode/utf8"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/health"
"github.com/pidginhost/csm/internal/verdict"
)
// Bounds of the verdict stage.
const (
verdictWorkers = 4
// verdictPending bounds the jobs queued, running or completed and not
// yet received; it is also the job channel's capacity.
verdictPending = 64
verdictResults = 64
verdictReady = 128
// verdictTTL is how long a completed answer may be used.
verdictTTL = 30 * time.Second
// verdictTextBytes bounds the panel's tenant and note, and
// verdictResultBytes one encoded result.
verdictTextBytes = 256
verdictResultBytes = 2048
)
// answerKind is what a completed job learned.
type answerKind uint8
const (
// answerAllow: a valid allow. The pick ends without a reservation.
answerAllow answerKind = iota + 1
// answerDefault: block, empty, or a failed exchange (Class set). The
// local path continues.
answerDefault
// answerCancelled: the call was cancelled; nothing was learned.
answerCancelled
// answerSkipped: the job was cancelled before its call.
answerSkipped
)
// jobKey is the attempt a job asks about, under one policy epoch.
type jobKey struct {
Candidate admission.CandidateID `json:"candidate"`
Attempt uint32 `json:"attempt"`
Epoch uint64 `json:"epoch"`
}
// verdictJob is immutable once submitted; workers only read it.
type verdictJob struct {
id uint64
key jobKey
ip, reason string
ctx context.Context
cancel context.CancelFunc
timeout time.Duration
ask Asker
}
// verdictResult is a completed job, normalized by its worker: fixed
// classes, the job's pinned identity and bounded panel text. No response
// body or error text is kept.
type verdictResult struct {
Job uint64 `json:"job"`
Key jobKey `json:"key"`
IP string `json:"ip"`
Reason string `json:"reason"`
Answer answerKind `json:"answer"`
Class verdict.ErrorClass `json:"class,omitempty"`
Tenant string `json:"tenant,omitempty"`
Note string `json:"note,omitempty"`
// completed includes time spent waiting for the owner to receive it.
completed time.Time
}
// verdictStage is one policy epoch's workers and channels. The owner alone
// submits jobs and receives results.
type verdictStage struct {
epoch uint64
jobs chan *verdictJob
results chan verdictResult
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
tally *verdictTally
// done closes once every worker has returned, after retirement began.
done chan struct{}
}
// verdictTally holds the process counters. Workers count the calls they
// start and finish, independently of when the owner receives their results.
type verdictTally struct {
mu sync.Mutex
asked, inFlight uint64
allow, deflt, failed, cancelled, discarded uint64
evicted, endings uint64
lastErr verdict.ErrorClass
lastErrAt time.Time
recent minuteTotals
}
func saturating(n *uint64) {
if *n != ^uint64(0) {
*n++
}
}
func (t *verdictTally) add(n *uint64) {
t.mu.Lock()
defer t.mu.Unlock()
saturating(n)
}
// complete counts one call when its worker finishes, at mono and wall.
func (t *verdictTally) complete(r verdictResult, mono, wall time.Time) {
t.mu.Lock()
defer t.mu.Unlock()
if r.Answer == answerSkipped {
return
}
t.inFlight--
switch {
case r.Answer == answerCancelled:
saturating(&t.cancelled)
case r.Class != "":
saturating(&t.failed)
t.lastErr, t.lastErrAt = r.Class, wall.UTC()
t.recent.add(mono)
case r.Answer == answerAllow:
saturating(&t.allow)
default:
saturating(&t.deflt)
}
}
// minuteTotals keeps sixty one-minute counts on the process's monotonic
// clock, aligned to the first count, so a wall-clock change cannot age
// them.
type minuteTotals struct {
base time.Time
buckets [60]struct {
set bool
minute int64
n uint64
}
}
func (m *minuteTotals) minute(mono time.Time) int64 {
return int64(max(mono.Sub(m.base), 0) / time.Minute)
}
func (m *minuteTotals) add(mono time.Time) {
if m.base.IsZero() {
m.base = mono
}
minute := m.minute(mono)
b := &m.buckets[minute%int64(len(m.buckets))]
if !b.set || b.minute != minute {
b.set, b.minute, b.n = true, minute, 0
}
saturating(&b.n)
}
// total sums the sixty minutes up to and including mono's.
func (m *minuteTotals) total(mono time.Time) uint64 {
if m.base.IsZero() {
return 0
}
now := m.minute(mono)
var out uint64
for _, b := range m.buckets {
if b.set && b.minute <= now && b.minute > now-int64(len(m.buckets)) {
out += min(b.n, ^uint64(0)-out)
}
}
return out
}
// verdicts is the owner's side of the stage, used only on its goroutine.
type verdicts struct {
stage *verdictStage
retiring *verdictStage
// pending holds each candidate's outstanding job until its completion
// is received, cancelled or not; ready holds received answers.
pending map[admission.CandidateID]*verdictJob
ready map[admission.CandidateID]verdictResult
nextJob uint64
// received counts the completions received: the queue's progress.
received uint64
tally verdictTally
}
// observeVerdicts samples the outstanding jobs for queue health.
func (o *Owner) observeVerdicts() {
if o.verdictQueue != nil {
o.verdictQueue.Observe(deliveryNow(), len(o.verdicts.pending), o.verdicts.received)
}
}
func (s *verdictStage) work() {
defer s.wg.Done()
for {
select {
case <-s.ctx.Done():
return
case j := <-s.jobs:
r := s.run(j)
// The owner keeps at most verdictPending jobs outstanding, so the
// result channel has room; a stopping stage still delivers it.
select {
case s.results <- r:
default:
select {
case s.results <- r:
case <-s.ctx.Done():
return
}
}
}
}
}
// run makes one call. Its timeout covers the exchange from the worker's
// start, not the time the job waited in the queue.
func (s *verdictStage) run(j *verdictJob) verdictResult {
r := verdictResult{Job: j.id, Key: j.key, IP: j.ip, Reason: j.reason}
// A stopping stage cancels its jobs one at a time after itself: check
// both, so no call starts once the stage is cancelled.
if s.ctx.Err() != nil || j.ctx.Err() != nil {
s.tally.add(&s.tally.discarded)
r.Answer = answerSkipped
return r
}
s.tally.mu.Lock()
saturating(&s.tally.asked)
s.tally.inFlight++
s.tally.mu.Unlock()
ctx, cancel := context.WithTimeout(j.ctx, j.timeout)
resp, err := j.ask(ctx, j.ip, j.reason)
timedOut := ctx.Err() == context.DeadlineExceeded
cancel()
switch {
case j.ctx.Err() != nil:
r.Answer = answerCancelled
case err != nil:
r.Answer, r.Class = answerDefault, verdict.ClassOf(err)
if timedOut {
r.Class = verdict.ClassTimeout
} else if r.Class == "" {
r.Class = verdict.ClassProtocol
}
case resp.Verdict == "allow":
r.Answer = answerAllow
case resp.Verdict == "" || resp.Verdict == "block":
r.Answer = answerDefault
default:
// An asker that returns a verdict the client would refuse made a
// protocol error; it never reads as an allow.
r.Answer, r.Class = answerDefault, verdict.ClassProtocol
}
if r.Class == "" && r.Answer != answerCancelled {
r.Tenant, r.Note = boundedText(resp.TenantID), boundedText(resp.Note)
fitResult(&r)
}
r.completed = monoNow()
s.tally.complete(r, r.completed, time.Now())
return r
}
// boundedText escapes control characters and invalid bytes, then keeps at
// most verdictTextBytes without splitting a character.
func boundedText(s string) string {
var b strings.Builder
for _, c := range strings.ToValidUTF8(s, "�") {
if unicode.IsControl(c) {
fmt.Fprintf(&b, "\\u%04x", c)
} else {
b.WriteRune(c)
}
if b.Len() > verdictTextBytes {
break
}
}
return truncateRunes(b.String(), verdictTextBytes)
}
// truncateRunes keeps at most n bytes of s, at a character boundary.
func truncateRunes(s string, n int) string {
if len(s) <= n {
return s
}
for n > 0 && !utf8.RuneStart(s[n]) {
n--
}
return s[:n]
}
// fitResult shortens the tenant and note, longer one first, until the
// encoded result fits verdictResultBytes. The fixed fields stay whole.
func fitResult(r *verdictResult) {
for {
data, err := json.Marshal(r)
if err == nil && len(data) <= verdictResultBytes {
return
}
field := &r.Tenant
if len(r.Note) > len(r.Tenant) {
field = &r.Note
}
if *field == "" {
return
}
_, size := utf8.DecodeLastRuneInString(*field)
*field = (*field)[:len(*field)-size]
}
}
// startVerdicts starts the stage for the installed policy when its callback
// is on. A stage still retiring starts its replacement once its workers
// have returned, so no more than verdictWorkers calls ever run.
func (o *Owner) startVerdicts() {
v := &o.verdicts
if v.stage != nil || v.retiring != nil || !o.asksVerdicts() {
return
}
ctx, cancel := context.WithCancel(context.Background())
s := &verdictStage{
epoch: o.policy.Epoch, jobs: make(chan *verdictJob, verdictPending), results: make(chan verdictResult, verdictResults),
ctx: ctx, cancel: cancel, tally: &v.tally, done: make(chan struct{}),
}
s.wg.Add(verdictWorkers)
for range verdictWorkers {
go s.work()
}
v.stage = s
}
// retireVerdicts invalidates every job and answer of the current stage and
// cancels it. Unrelated owner work goes on while its workers return; one
// joiner reports when they have.
func (o *Owner) retireVerdicts() {
v := &o.verdicts
for range v.ready {
v.tally.add(&v.tally.discarded)
}
clear(v.ready)
clear(v.pending)
o.observeVerdicts()
s := v.stage
if s == nil {
return
}
v.stage, v.retiring = nil, s
s.cancel()
go func() {
s.wg.Wait()
close(s.done)
}()
}
// verdictsRetired takes what the retired stage left in its channels,
// counts it and starts the stage of the current policy.
func (o *Owner) verdictsRetired() {
v := &o.verdicts
s := v.retiring
v.retiring = nil
o.drainStage(s)
o.startVerdicts()
o.scheduleDue = true
}
// drainStage counts the jobs no worker took and the completions nobody
// received, once its workers have returned.
func (o *Owner) drainStage(s *verdictStage) {
v := &o.verdicts
for {
select {
case <-s.jobs:
v.tally.add(&v.tally.discarded)
case r := <-s.results:
if r.Answer != answerSkipped && r.Answer != answerCancelled {
v.tally.add(&v.tally.discarded)
}
default:
return
}
}
}
// stopVerdicts cancels every call and joins the workers. A cancelled call
// is never read as an answer; its candidate stays queued.
func (o *Owner) stopVerdicts() {
v := &o.verdicts
for _, s := range []*verdictStage{v.stage, v.retiring} {
if s != nil {
s.cancel()
s.wg.Wait()
o.drainStage(s)
}
}
v.stage, v.retiring = nil, nil
clear(v.pending)
for range v.ready {
v.tally.add(&v.tally.discarded)
}
clear(v.ready)
o.observeVerdicts()
}
// stageResults is the current stage's result channel, nil without one.
func (o *Owner) stageResults() <-chan verdictResult {
if o.verdicts.stage == nil {
return nil
}
return o.verdicts.stage.results
}
// stageRetired closes once a retiring stage's workers have returned.
func (o *Owner) stageRetired() <-chan struct{} {
if o.verdicts.retiring == nil {
return nil
}
return o.verdicts.retiring.done
}
// askVerdict submits one job for pick c lasting as member, unless the
// stage is retiring or full: then the candidate stays as it is.
func (o *Owner) askVerdict(id admission.CandidateID, c admission.Candidate, ip, reason string) {
v := &o.verdicts
s := v.stage
if s == nil || len(v.pending) >= verdictPending || v.pending[id] != nil {
return
}
ctx, cancel := context.WithCancel(s.ctx)
j := &verdictJob{
id: v.nextJob + 1, key: jobKey{Candidate: id, Attempt: c.Attempts + 1, Epoch: s.epoch},
ip: ip, reason: reason, ctx: ctx, cancel: cancel, timeout: o.policy.VerdictTimeout, ask: o.policy.AskVerdict,
}
select {
case s.jobs <- j:
v.nextJob++
if v.pending == nil {
v.pending = map[admission.CandidateID]*verdictJob{}
}
v.pending[id] = j
o.observeVerdicts()
default:
cancel()
}
}
// receiveVerdict takes one completion of the current stage. Only the
// candidate's outstanding job is answered; a late, duplicate or swept
// completion is discarded. A received answer wakes the schedule.
func (o *Owner) receiveVerdict(r verdictResult) {
v := &o.verdicts
v.received++
defer o.observeVerdicts()
j := v.pending[r.Key.Candidate]
if j == nil || j.id != r.Job || j.key != r.Key || j.ip != r.IP || j.reason != r.Reason {
if r.Answer != answerSkipped && r.Answer != answerCancelled {
v.tally.add(&v.tally.discarded)
}
return
}
delete(v.pending, r.Key.Candidate)
swept := j.ctx.Err() != nil
j.cancel()
switch {
case r.Answer == answerSkipped || r.Answer == answerCancelled:
return
case swept || !monoNow().Before(r.completed.Add(verdictTTL)):
v.tally.add(&v.tally.discarded)
o.scheduleDue = true
return
}
if v.ready == nil {
v.ready = map[admission.CandidateID]verdictResult{}
}
if _, replacing := v.ready[r.Key.Candidate]; !replacing && len(v.ready) >= verdictReady {
o.evictReady()
}
v.ready[r.Key.Candidate] = r
o.scheduleDue = true
}
// evictReady makes room for one answer: expired answers go first, then the
// oldest completed, the earlier job first. The evicted candidate asks again
// when it is next selected.
func (o *Owner) evictReady() {
v := &o.verdicts
if o.expireVerdicts() {
return
}
oldest := slices.MinFunc(slices.Collect(maps.Values(v.ready)), func(a, b verdictResult) int {
if c := a.completed.Compare(b.completed); c != 0 {
return c
}
return cmpUint(a.Job, b.Job)
})
delete(v.ready, oldest.Key.Candidate)
v.tally.add(&v.tally.evicted)
}
func cmpUint(a, b uint64) int {
switch {
case a < b:
return -1
case a > b:
return 1
}
return 0
}
// expireVerdicts drops the answers whose lifetime has ended and wakes the
// schedule, so their candidates can ask again whatever the live budgets.
// It reports whether any was dropped.
func (o *Owner) expireVerdicts() bool {
v := &o.verdicts
now := monoNow()
expired := false
for id, r := range v.ready {
if !now.Before(r.completed.Add(verdictTTL)) {
delete(v.ready, id)
expired = true
}
}
if expired {
o.scheduleDue = true
}
return expired
}
// usable reports whether key still names c's next attempt under the
// installed policy, with c still waiting for it.
func (o *Owner) usable(c admission.Candidate, key jobKey) bool {
return c.State == admission.StateQueued && c.Attempts+1 == key.Attempt && key.Epoch == o.policy.Epoch &&
o.now.Before(c.AgeOut) && (c.Attempts == 0 || o.now.Before(c.ExpiresAt))
}
// sweepVerdicts cancels the jobs of candidates that ceased to be eligible
// and drops their answers. A cancelled job keeps its pending slot until
// its completion is received, so cancelled work stays bounded.
func (o *Owner) sweepVerdicts() {
v := &o.verdicts
for id, j := range v.pending {
if j.ctx.Err() != nil {
continue
}
if c, err := o.ledger.Candidate(id); err != nil || !o.usable(c, j.key) {
j.cancel()
}
}
for id, r := range v.ready {
if c, err := o.ledger.Candidate(id); err != nil || !o.usable(c, r.Key) {
delete(v.ready, id)
v.tally.add(&v.tally.discarded)
}
}
}
// readyVerdict is the usable answer for pick c lasting as member: same
// attempt, epoch, address and reason, completed less than verdictTTL ago.
// Any other answer for c is discarded.
func (o *Owner) readyVerdict(id admission.CandidateID, c admission.Candidate, ip, reason string) (verdictResult, bool) {
v := &o.verdicts
r, ok := v.ready[id]
if !ok {
return r, false
}
if !monoNow().Before(r.completed.Add(verdictTTL)) || !o.usable(c, r.Key) || r.Key.Candidate != id || r.IP != ip || r.Reason != reason {
delete(v.ready, id)
v.tally.add(&v.tally.discarded)
return r, false
}
return r, true
}
// answers are the outside answers the schedule judges queued work by:
// outstanding jobs and the work a refused policy holds wait, and allows
// are picked uncharged.
func (o *Owner) answers() map[admission.CandidateID]admission.Answer {
v := &o.verdicts
if len(v.pending) == 0 && len(v.ready) == 0 && len(o.policyHeld) == 0 {
return nil
}
out := make(map[admission.CandidateID]admission.Answer, len(v.pending)+len(v.ready)+len(o.policyHeld))
now := monoNow()
for id, r := range v.ready {
if r.Answer == answerAllow && now.Before(r.completed.Add(verdictTTL)) {
out[id] = admission.AnswerAllow
}
}
for id := range v.pending {
out[id] = admission.AnswerPending
}
for id := range o.policyHeld {
out[id] = admission.AnswerPending
}
return out
}
// verdictStatus reads the stage for status.
func (o *Owner) verdictStatus() *health.AdmissionVerdict {
v := &o.verdicts
t := &v.tally
t.mu.Lock()
defer t.mu.Unlock()
s := &health.AdmissionVerdict{
Enabled: o.asksVerdicts(), Epoch: o.policy.Epoch,
Pending: len(v.pending), InFlight: t.inFlight, Ready: len(v.ready), StageRetiring: v.retiring != nil,
Asked: t.asked, ResponsesAllow: t.allow, ResponsesDefault: t.deflt, Failed: t.failed, Cancelled: t.cancelled,
Discarded: t.discarded, ReadyEvicted: t.evicted, AllowEndings: t.endings,
LastHourFailed: t.recent.total(monoNow()),
}
if t.lastErr != "" {
class, at := string(t.lastErr), t.lastErrAt
s.LastError, s.LastErrorAt = &class, &at
}
return s
}
package alert
import (
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/processctx"
"github.com/pidginhost/csm/internal/queuehealth"
)
const alertDispatchFailuresMetric = "csm_alert_dispatch_failures_total"
// alertDispatchFailures counts individual channel send failures (email,
// webhook, phpanel) so an operator can see when alerts are silently failing
// to deliver instead of the daemon looking healthy while findings never
// reach anyone.
var alertDispatchFailures = metrics.NewCounter(
alertDispatchFailuresMetric,
"Alert deliveries that failed (email/webhook/phpanel). Sustained growth means findings are being detected but not reaching operators -- check SMTP/webhook reachability and credentials.",
)
func init() {
metrics.MustRegister(alertDispatchFailuresMetric, alertDispatchFailures)
}
func addDispatchError(errs *[]error, err error) {
*errs = append(*errs, err)
alertDispatchFailures.Inc()
}
// Severity levels for findings.
type Severity int
const (
Warning Severity = iota
High Severity = iota
Critical Severity = iota
)
func (s Severity) String() string {
switch s {
case Warning:
return "WARNING"
case High:
return "HIGH"
case Critical:
return "CRITICAL"
}
return "UNKNOWN"
}
// Finding represents a single security check result.
type Finding struct {
queueTicket queuehealth.Ticket
Severity Severity `json:"severity"`
// DemotedFrom retains the severity an automatically demoted finding came
// from, so a later positive re-check can restore it. It is deliberately not
// part of Key(): a finding's identity must not change when its severity
// does, or every dismissal and dedup entry keyed to it would be orphaned.
DemotedFrom Severity `json:"demoted_from,omitempty"`
Check string `json:"check"`
Message string `json:"message"`
Details string `json:"details,omitempty"`
// Context is background an operator needs beside the finding, such as the
// ModSecurity state of the account it concerns. It is outside Key,
// Fingerprint and every other identity on purpose: the background can
// change while the condition stays the same, and that change must not
// re-alert or undo a dismissal. Renderers show it after Details.
Context string `json:"context,omitempty"`
// CoverageScope identifies a scanner-owned unit that can be retired after
// complete coverage. Empty legacy scopes require whole-check completion.
// It is opaque, contains no credentials, and does not change dedup identity.
CoverageScope string `json:"coverage_scope,omitempty"`
// DedupKey, when set, pins the finding's dedup identity (Key and
// Fingerprint) regardless of Message/Details content. For findings whose
// details embed volatile values (pids, byte counts) that would otherwise
// mint a new identity for the same ongoing condition on every scan.
DedupKey string `json:"dedup_key,omitempty"`
FilePath string `json:"file_path,omitempty"`
ProcessInfo string `json:"process_info,omitempty"` // "pid=N cmd=name uid=N" from fanotify
PID int `json:"pid,omitempty"` // structured PID for auto-response
// Content fingerprint for re-verifiable content findings (PHP heuristics,
// signature, YARA). Set at emit time by content-family checks; empty for
// all other finding kinds and for findings emitted before this field
// existed. The re-verification re-check uses it to tell a superseded-
// heuristic false positive (identical bytes, current logic no longer
// flags) from a file edited after detection (never auto-cleared).
ContentSHA256 string `json:"content_sha256,omitempty"`
// DetectLogic is the checks.ContentDetectionVersion() token in effect when
// this content finding was emitted. Optional; used for sweep gating and
// audit explainability.
DetectLogic string `json:"detect_logic,omitempty"`
// ScanCarryForward marks an unchanged snapshot re-emitted only because the
// current scan could not examine its path. This internal provenance survives
// pending replay but is not part of the public finding contract.
ScanCarryForward bool `json:"-"`
// AutoFileResponseEvaluated records process-local delivery provenance.
// A detector or scan already considered automatic file remediation, so
// the alert dispatcher must not retry it, including a refused attempt.
// New detections and findings read from storage get a fresh evaluation.
AutoFileResponseEvaluated bool `json:"-"`
// FileResponseIdentity pins asynchronous content analysis to the inspected
// file version. A replacement must not receive the older file's response.
FileResponseIdentity string `json:"-"`
// AutoIPResponseEvaluated records that challenge routing and IP blocking
// already considered the finding in this process, so a later pass over
// the same finding neither acts again nor reaches admission twice.
AutoIPResponseEvaluated bool `json:"-"`
// AutoProcessResponseEvaluated records that automatic process termination
// already considered the finding in this process, so alert delivery does
// not signal the process again, including after a refused attempt.
AutoProcessResponseEvaluated bool `json:"-"`
// WriterChain lists, writer first, the processes that wrote back a file
// CSM had removed, named when the write was seen. Process-local: a
// finding replayed after a restart never signals a stale PID.
WriterChain []int `json:"-"`
// WriterObservedAt binds those PIDs to the event, not the later finding
// timestamp. A replacement born while analysis runs is not the writer.
WriterObservedAt time.Time `json:"-"`
// PHP-relay structured fields (Stage 1 email_php_relay_abuse). All optional;
// zero values mean "this finding does not carry that dimension".
Path string `json:"path,omitempty"` // path1 trigger label: "header" | "volume" | "volume_account" | "fanout" | "baseline" | "reputation"
MsgIDs []string `json:"msg_ids,omitempty"` // sample of in-flight msgIDs (auto-action acts on the live snapshot, not this list)
ScriptKey string `json:"script_key,omitempty"` // host:path from X-PHP-Script
SourceIP string `json:"source_ip,omitempty"` // IP after "for " in X-PHP-Script
CPUser string `json:"cp_user,omitempty"` // cPanel user from spool -H line 2
// RelayTotal is the trigger count for the PHP-relay path that fired
// (qualifying/volume/fanout/account-window count). RelayBreakdown lists
// the scripts that contributed, with per-script hit counts and a sample
// subject. Both optional; volume_account carries RelayTotal with no
// breakdown (account log-tail path has no trustworthy script key).
RelayTotal int `json:"relay_total,omitempty"`
RelayBreakdown []RelayScriptHit `json:"relay_breakdown,omitempty"`
// Tenant context (added v2.12.0). Optional - populated when the check
// has enough info to attribute the finding to a specific tenant within
// a multi-tenant host. Empty strings render as omitted JSON keys so
// existing webhook consumers see no diff.
TenantID string `json:"tenant_id,omitempty"`
Domain string `json:"domain,omitempty"`
Mailbox string `json:"mailbox,omitempty"`
// LoginAccount retains the authenticated login name independently of
// truncated log details and the hosting tenant that owns the account.
// Internal-only: customer history reads it, the public Finding JSON
// does not carry it.
LoginAccount string `json:"-"`
// SprayTargets carries the per-account targets for aggregate auth
// findings. It is internal-only so API payloads keep the public Finding
// contract while the incident correlator can count distinct targets.
SprayTargets []string `json:"-"`
// CIDRs carries the offending subnets of a subnet-scoped finding: the
// collapsed subnets of a distributed crawl. The subnet auto-response
// reads this, never the Message/Details text.
CIDRs []string `json:"cidrs,omitempty"`
// SprayConstituents lists the addresses a subnet spray counted and the
// line that last named each. Internal-only like SprayTargets.
SprayConstituents []SprayConstituent `json:"-"`
// Intel names the threat intelligence a reputation finding rests on and
// when that claim lapses. Internal-only like Claims.
Intel *admission.IntelRef `json:"-"`
// MailLoginFailure marks an address finding raised because the address
// failed mailbox logins when its check name does not say so, as for a
// reputation match. Incidents file it as mailbox brute force.
// Internal-only like Intel.
MailLoginFailure bool `json:"-"`
// BlockTTL is the finding's own auto-block length; set only by spray block verdicts.
BlockTTL time.Duration `json:"-"`
// Cause names the finding a derived finding was made from, such as the
// database finding behind a session-attacker block. Internal-only.
Cause *Cause `json:"-"`
// Claims say which account a finding concerns and where each value came
// from, as the producer read it. Admission resolves the owner from them,
// so a producer records only what it read, never a guess.
Claims []admission.Claim `json:"-"`
// Attribution is how the parser read SourceIP. The parser
// that read the address sets it; a response path never does. Unknown
// never authorizes an admission response.
Attribution admission.Attribution `json:"-"`
// ProxiedClient marks an address only a trusted proxy's forwarded header
// named: no record counted for it came from the address itself, so a
// firewall block cannot reach it. A count that mixes direct records is
// not one.
ProxiedClient bool `json:"-"`
// Observation is where the finding was read; zero when the producer has
// no position for it. Internal-only like Claims.
Observation Observation `json:"-"`
// Process context (Phase 1 process-ancestry enrichment). Optional.
// Populated by exec/connection live monitors when cache or enricher
// has data. Omitted from JSON when nil so existing webhook consumers
// see no diff.
Process *processctx.ProcessContext `json:"process,omitempty"`
Timestamp time.Time `json:"timestamp"`
// FirstSeen is when this condition was first observed, as opposed to
// when it was last reported. The latest-state merge carries it across
// re-reports; a scan that finds the same condition again refreshes
// Timestamp but not this. Correlation reads it so a months-old finding
// re-emitted by every scan cannot keep re-entering a recent-activity
// window. Zero on findings that never went through the merge, and on
// rows stored before the field existed; callers fall back to Timestamp.
FirstSeen time.Time `json:"first_seen,omitzero"`
// Full-scan quarantine outcome (Phase 2). Set ONLY on findings produced by a
// `--full --quarantine` job; empty for all report-only findings so existing
// consumers see no JSON diff.
RemediationStatus string `json:"remediation_status,omitempty"` // "quarantined" | "cleaned" | "left_for_review" | "failed"
RemediationDetail string `json:"remediation_detail,omitempty"` // action description or error
}
// RelayScriptHit is one script's contribution to a PHP-relay finding.
type RelayScriptHit struct {
ScriptKey string `json:"script_key"` // "host:/path" from X-PHP-Script
Hits int `json:"hits"` // messages counted in the path window
LastSeen time.Time `json:"last_seen"`
SampleSubject string `json:"sample_subject,omitempty"` // attacker-controlled; render escaped
}
func (f Finding) String() string {
ts := f.Timestamp.Format("2006-01-02 15:04:05")
s := fmt.Sprintf("[%s] %s - %s", f.Severity, f.Check, f.Message)
if f.Details != "" {
s += "\n " + strings.ReplaceAll(f.Details, "\n", "\n ")
}
if f.Context != "" {
s += "\n " + strings.ReplaceAll(f.Context, "\n", "\n ")
}
if f.ProcessInfo != "" {
s += fmt.Sprintf("\n Process: %s", f.ProcessInfo)
}
s += fmt.Sprintf("\n Time: %s", ts)
return s
}
// Key returns a unique key for deduplication.
func (f Finding) Key() string {
if f.DedupKey != "" {
// Keep explicit identities in a leading namespace. Putting the marker
// after Check would still collide with an ordinary finding from that
// check whose Message happens to start with "dedup:".
return fmt.Sprintf("dedup:%s:%s", f.Check, f.DedupKey)
}
if key := f.sourceIPKey(); key != "" {
return key
}
if f.Details == "" {
return fmt.Sprintf("%s:%s", f.Check, f.Message)
}
h := sha256.Sum256([]byte(f.Details))
return fmt.Sprintf("%s:%s:%x", f.Check, f.Message, h[:4])
}
// Fingerprint returns the content hash used by alert-state deduplication.
func (f Finding) Fingerprint() string {
if f.DedupKey != "" {
h := sha256.Sum256([]byte(f.Key()))
return fmt.Sprintf("%x", h[:8])
}
if key := f.sourceIPKey(); key != "" {
h := sha256.Sum256([]byte(key))
return fmt.Sprintf("%x", h[:8])
}
h := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", f.Check, f.Message, f.Details)))
return fmt.Sprintf("%x", h[:8])
}
func (f Finding) sourceIPKey() string {
severityScoped := false
switch f.Check {
case "admin_panel_bruteforce", "wp_login_bruteforce", "wp_user_enumeration", "xmlrpc_abuse",
"http_request_flood", "http_scanner_profile", "http_claimed_bot_unverified", "http_ua_spoof",
"ftp_bruteforce":
case "php_shield_webshell", "php_shield_block", "php_shield_eval":
// One scanner sweeping a shared host hits every account in the same
// second, which produced one alert per site instead of one per scanner.
// The collapse is per severity: the same check name carries both the
// Warning "observed a parameter" and the Critical "blocked a webshell",
// and a block from an address that was merely observed earlier is a new
// event, not a repeat, or it is never alerted, stored or correlated.
severityScoped = true
default:
return ""
}
ip := normalizeFindingIP(f.SourceIP)
if ip == "" {
ip = sourceIPFromFindingMessage(f.Message)
}
if ip == "" {
return ""
}
if severityScoped {
return fmt.Sprintf("%s:ip:%s:%s", f.Check, ip, f.Severity)
}
return fmt.Sprintf("%s:ip:%s", f.Check, ip)
}
func normalizeFindingIP(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
if host, _, err := net.SplitHostPort(raw); err == nil {
raw = host
}
raw = strings.Trim(raw, "[]")
ip := net.ParseIP(raw)
if ip == nil {
return ""
}
return ip.String()
}
func sourceIPFromFindingMessage(msg string) string {
for _, sep := range []string{" from ", ": "} {
idx := strings.LastIndex(msg, sep)
if idx < 0 {
continue
}
rest := msg[idx+len(sep):]
fields := strings.Fields(rest)
if len(fields) == 0 {
continue
}
candidate := strings.TrimRight(fields[0], ",:;)([]")
if ip := normalizeFindingIP(candidate); ip != "" {
return ip
}
}
return ""
}
// SplitEmail returns (localpart, domain) from an email address. Returns
// ("", "") when the input doesn't look like an email.
func SplitEmail(addr string) (localpart, domain string) {
at := strings.LastIndexByte(addr, '@')
if at <= 0 || at == len(addr)-1 {
return "", ""
}
return addr[:at], addr[at+1:]
}
// Deduplicate removes findings with the same Check+Message, keeping the first.
func Deduplicate(findings []Finding) []Finding {
seen := make(map[string]bool)
var result []Finding
for _, f := range findings {
key := f.Key()
if !seen[key] {
seen[key] = true
result = append(result, f)
}
}
return result
}
// FormatAlert formats a list of findings into a human-readable alert body.
// Sensitive data (passwords, tokens) is redacted before sending.
func FormatAlert(hostname string, findings []Finding) string {
var b strings.Builder
critCount := 0
highCount := 0
warnCount := 0
for _, f := range findings {
switch f.Severity {
case Critical:
critCount++
case High:
highCount++
case Warning:
warnCount++
}
}
fmt.Fprintf(&b, "SECURITY ALERT - %s\n", hostname)
fmt.Fprintf(&b, "Timestamp: %s\n", time.Now().Format("2006-01-02 15:04:05 MST"))
fmt.Fprintf(&b, "Findings: %d critical, %d high, %d warning\n", critCount, highCount, warnCount)
b.WriteString(strings.Repeat("─", 60) + "\n\n")
for _, sev := range []Severity{Critical, High, Warning} {
for _, f := range findings {
if f.Severity == sev {
b.WriteString(SanitizeFinding(f).String())
b.WriteString("\n\n")
}
}
}
b.WriteString(strings.Repeat("─", 60) + "\n")
b.WriteString("CSM - Continuous Security Monitor\n")
return b.String()
}
// SanitizeFinding returns a copy with recognized credentials redacted from
// Message, Details and Context. Call it at output and persistence boundaries
// so detection and identity calculations can still use the original finding.
// Other fields are unchanged; nested data is not modified.
func SanitizeFinding(f Finding) Finding {
f.Message = redactSensitive(f.Message)
f.Details = redactSensitive(f.Details)
f.Context = redactSensitive(f.Context)
return f
}
// redactSensitive replaces password values and tokens in text with [REDACTED].
func redactSensitive(s string) string {
if s == "" {
return s
}
s = redactCredentialFields(s)
// Normalize command-line text first: NUL-delimited arguments can expose
// session keywords once the argument separators become spaces.
s = RedactCommandLine(s)
// Gate each log line separately: a finding can also contain unrelated
// prose with NEW/PURGE and colons whose evidence must survive.
var lines strings.Builder
for line := range strings.SplitAfterSeq(s, "\n") {
lines.WriteString(redactSessionLogLine(line))
}
return lines.String()
}
// Scan the original text once so every field is covered without searching
// replacement markers or changing byte offsets. Log envelopes can quote a whole
// request, so command-line tokenization alone cannot find these nested fields.
func redactCredentialFields(s string) string {
lower := lowerASCII(s)
var b strings.Builder
last := 0
for i := 0; i < len(s); {
prefixLen := 0
for _, prefix := range []string{
"password=", "pass=", "passwd=", "new_password=",
"old_password=", "confirmpassword=", "token_value=", "api_token=",
} {
if strings.HasPrefix(lower[i:], prefix) {
prefixLen = len(prefix)
break
}
}
if prefixLen == 0 {
i++
continue
}
start := i + prefixLen
end := start
var quote byte
if end < len(s) && (s[end] == '\'' || s[end] == '"') {
quote = s[end]
end++
}
for end < len(s) {
c := s[end]
if c == '\\' && end+1 < len(s) {
end += 2
continue
}
if quote != 0 {
end++
if c == quote {
quote = 0
}
continue
}
if strings.ContainsRune(" &\t\n\r\v\f\x00\"',", rune(c)) {
break
}
end++
}
if end > start && s[start:end] != redactedToken {
b.WriteString(s[last:start])
b.WriteString(redactedToken)
last = end
}
i = end
}
if last == 0 {
return s
}
b.WriteString(s[last:])
return b.String()
}
// Credential names are ASCII. Unicode case folding can change byte lengths
// (including invalid UTF-8 from logs), invalidating offsets into the input.
func lowerASCII(s string) string {
b := []byte(s)
for i, c := range b {
if c >= 'A' && c <= 'Z' {
b[i] = c + ('a' - 'A')
}
}
return string(b)
}
func redactSessionLogLine(s string) string {
if !containsSessionLogTag(s) {
return s
}
// Scan the original text only, keeping replacements out of the search.
// Account names survive; only the credential after the colon is masked.
var b strings.Builder
last := 0
for i := 0; i < len(s); i++ {
keywordLen := 0
switch {
case strings.HasPrefix(s[i:], " NEW "):
keywordLen = len(" NEW ")
case strings.HasPrefix(s[i:], " PURGE "):
keywordLen = len(" PURGE ")
default:
continue
}
fieldStart := i + keywordLen
fieldEnd := fieldStart
for fieldEnd < len(s) && !strings.ContainsRune(" \t\n\r", rune(s[fieldEnd])) {
fieldEnd++
}
colon := strings.IndexByte(s[fieldStart:fieldEnd], ':')
// Keep the keyword's trailing space searchable: the malformed field
// may itself be NEW or PURGE, followed by a valid account:session.
i = fieldStart - 2
if colon < 0 {
continue
}
tokenStart := fieldStart + colon + 1
if tokenStart == fieldEnd {
continue
}
if s[tokenStart:fieldEnd] != redactedToken {
b.WriteString(s[last:tokenStart])
b.WriteString(redactedToken)
last = fieldEnd
}
i = fieldEnd - 1
}
if last == 0 {
return s
}
b.WriteString(s[last:])
return b.String()
}
// cPanel session logs use both frontend service names and the shared server
// daemon name. DAV and security purge logs also carry account:session pairs.
func containsSessionLogTag(s string) bool {
for _, tag := range []string{"[cpaneld]", "[webmaild]", "[whostmgr]", "[whostmgrd]", "[cpsrvd]", "[cpdavd]", "[security]"} {
if strings.Contains(s, tag) {
return true
}
}
return false
}
func filterChecks(findings []Finding, disabledChecks []string) []Finding {
if len(findings) == 0 || len(disabledChecks) == 0 {
return findings
}
disabled := make(map[string]bool, len(disabledChecks))
for _, check := range disabledChecks {
check = config.CanonicalCheckName(strings.TrimSpace(check))
if check != "" {
disabled[check] = true
}
}
if len(disabled) == 0 {
return findings
}
filtered := make([]Finding, 0, len(findings))
for _, f := range findings {
if !disabled[config.CanonicalCheckName(f.Check)] {
filtered = append(filtered, f)
}
}
return filtered
}
func buildSubject(hostname string, findings []Finding) string {
subject := fmt.Sprintf("[CSM] %s - %d security finding(s)", hostname, len(findings))
for _, f := range findings {
if f.Severity == Critical {
return fmt.Sprintf("[CSM] CRITICAL - %s - %d finding(s)", hostname, len(findings))
}
}
return subject
}
// rateLimitState tracks alerts sent per hour.
type rateLimitState struct {
Hour string `json:"hour"`
Count int `json:"count"`
}
// FindingBus is set by the daemon at startup to the broadcast.Bus that
// passive observers (e.g. SSE subscribers) drain. nil-safe: Dispatch
// only publishes if non-nil. Importing the broadcast package directly
// would create an import cycle (broadcast imports alert), so this is
// declared as an interface satisfied by *broadcast.Bus.
var FindingBus interface {
Publish(Finding)
}
// ReportHook, when set by the daemon at startup, is called once per
// deduplicated finding so the abuse reporter can consider it for submission to
// a central abuse database or collector. It must not block. Declared as a func
// to avoid an import cycle (the reporting package imports alert for the
// Finding type).
//
// Install or clear it with SetReportHook so Dispatch reads a consistent value.
var ReportHook func(Finding)
var reportHookMu sync.RWMutex
// SetReportHook installs or clears the abuse-reporting hook used by Dispatch.
func SetReportHook(h func(Finding)) {
reportHookMu.Lock()
ReportHook = h
reportHookMu.Unlock()
}
func currentReportHook() func(Finding) {
reportHookMu.RLock()
h := ReportHook
reportHookMu.RUnlock()
return h
}
func callReportHook(f Finding) {
h := currentReportHook()
if h == nil {
return
}
defer func() {
if r := recover(); r != nil {
fmt.Fprintln(os.Stderr, "alert: report hook panic")
}
}()
h(f)
}
// CentralHook, when set by the daemon, is called once per deduplicated finding
// so the central-intel consumer can escalate (challenge/block) when the
// finding's IP is in the verified central scored-set. A finding firing on an IP
// is the node's own local signal, so this is the local-corroboration path.
// Must not block. Install or clear with SetCentralHook.
var CentralHook func(Finding)
var centralHookMu sync.RWMutex
// SetCentralHook installs or clears the central-intel hook used by Dispatch.
func SetCentralHook(h func(Finding)) {
centralHookMu.Lock()
CentralHook = h
centralHookMu.Unlock()
}
func currentCentralHook() func(Finding) {
centralHookMu.RLock()
h := CentralHook
centralHookMu.RUnlock()
return h
}
func callCentralHook(f Finding) {
h := currentCentralHook()
if h == nil {
return
}
defer func() {
if r := recover(); r != nil {
fmt.Fprintln(os.Stderr, "alert: central hook panic")
}
}()
h(f)
}
// summaryHook receives the summary-class findings of every dispatch. Install
// or clear it with SetSummaryHook.
var (
summaryHookMu sync.RWMutex
summaryHook func(cfg *config.Config, summary []Finding)
)
// SetSummaryHook installs or clears the receiver of summary-class findings.
// Each dispatch calls it once, with no alert lock held, with the
// summary-class part of the findings it would otherwise notify, deduplicated
// by Key. That includes checks muted by alerts.email.disabled_checks, but not
// audit-only sources or findings the notification filter removed. It must
// not block. A dispatch with no hook installed, as in a CLI scan, leaves
// summary-class findings to audit, history and the data streams.
func SetSummaryHook(h func(cfg *config.Config, summary []Finding)) {
summaryHookMu.Lock()
summaryHook = h
summaryHookMu.Unlock()
}
// callSummaryHook reports a panicking hook as a delivery failure: the
// summary lost those findings, and some callers only log dispatch errors.
func callSummaryHook(cfg *config.Config, summary []Finding) (err error) {
summaryHookMu.RLock()
h := summaryHook
summaryHookMu.RUnlock()
if h == nil {
return nil
}
defer func() {
if r := recover(); r != nil {
fmt.Fprintln(os.Stderr, "alert: summary hook panic")
err = errors.New("summary: hook panicked")
}
}()
h(cfg, summary)
return nil
}
type rateLimitKey struct {
StatePath string
Hour string
}
type rateLimitReservation struct {
key rateLimitKey
active bool
}
var (
rateLimitMu sync.Mutex
rateLimitPending = make(map[rateLimitKey]int)
// rateLimitNow is the hourly budget's clock, read under rateLimitMu.
rateLimitNow = time.Now
)
// SetRateLimitClockForTest pins the clock that picks the hourly budget, so a
// test cannot straddle an hour boundary.
func SetRateLimitClockForTest(now func() time.Time) (restore func()) {
rateLimitMu.Lock()
defer rateLimitMu.Unlock()
previous := rateLimitNow
rateLimitNow = now
return func() {
rateLimitMu.Lock()
defer rateLimitMu.Unlock()
rateLimitNow = previous
}
}
// reserveRateLimit reports whether the per-hour alert budget can absorb
// another send without committing the slot. The in-memory reservation
// prevents concurrent dispatches from all taking the same final slot while
// the outbound channel is still blocked on SMTP or webhook I/O.
func reserveRateLimit(statePath string, maxPerHour int) (*rateLimitReservation, bool) {
rateLimitMu.Lock()
defer rateLimitMu.Unlock()
if maxPerHour <= 0 {
return nil, false
}
currentHour := rateLimitNow().Format("2006-01-02T15")
rlPath := filepath.Join(statePath, "ratelimit.json")
var rl rateLimitState
// #nosec G304 -- filepath.Join(statePath, "ratelimit.json"); statePath from operator config.
data, err := os.ReadFile(rlPath)
if err == nil {
_ = json.Unmarshal(data, &rl)
}
count := 0
if rl.Hour != currentHour {
count = 0
} else {
count = rl.Count
}
key := rateLimitKey{StatePath: statePath, Hour: currentHour}
if count+rateLimitPending[key] >= maxPerHour {
return nil, false
}
rateLimitPending[key]++
return &rateLimitReservation{key: key, active: true}, true
}
func releaseRateLimit(reservation *rateLimitReservation) {
if reservation == nil {
return
}
rateLimitMu.Lock()
defer rateLimitMu.Unlock()
releaseRateLimitLocked(reservation)
}
func releaseRateLimitLocked(reservation *rateLimitReservation) {
if reservation == nil || !reservation.active {
return
}
if pending := rateLimitPending[reservation.key]; pending <= 1 {
delete(rateLimitPending, reservation.key)
} else {
rateLimitPending[reservation.key] = pending - 1
}
reservation.active = false
}
// commitRateLimit records one successful dispatch toward the hourly
// budget. Logs the WriteFile error so a disk-full or perm regression
// surfaces in the daemon log instead of silently letting the counter
// drift from the on-disk record.
func commitRateLimit(statePath string, reservation *rateLimitReservation) {
rateLimitMu.Lock()
defer rateLimitMu.Unlock()
releaseRateLimitLocked(reservation)
currentHour := rateLimitNow().Format("2006-01-02T15")
rlPath := filepath.Join(statePath, "ratelimit.json")
var rl rateLimitState
// #nosec G304 -- filepath.Join under operator-configured statePath.
if data, err := os.ReadFile(rlPath); err == nil {
_ = json.Unmarshal(data, &rl)
}
if rl.Hour != currentHour {
rl = rateLimitState{Hour: currentHour}
}
rl.Count++
newData, err := json.Marshal(rl)
if err != nil {
fmt.Fprintf(os.Stderr, "alert: rate-limit marshal failed: %v\n", err)
return
}
if err := os.WriteFile(rlPath, newData, 0600); err != nil {
fmt.Fprintf(os.Stderr, "alert: rate-limit write failed for %s: %v\n", rlPath, err)
}
}
// checkRateLimit returns true if we can send more alerts this hour.
//
// Deprecated: kept for callers that expect the check-and-increment pattern.
// New code should use reserveRateLimit and commitRateLimit so a failed
// dispatch does not consume the operator's budget.
func checkRateLimit(statePath string, maxPerHour int) bool {
rateLimitMu.Lock()
defer rateLimitMu.Unlock()
rlPath := filepath.Join(statePath, "ratelimit.json")
currentHour := rateLimitNow().Format("2006-01-02T15")
var rl rateLimitState
// #nosec G304 -- filepath.Join(statePath, "ratelimit.json"); statePath from operator config.
data, err := os.ReadFile(rlPath)
if err == nil {
_ = json.Unmarshal(data, &rl)
}
// Reset if new hour
if rl.Hour != currentHour {
rl = rateLimitState{Hour: currentHour, Count: 0}
}
if rl.Count >= maxPerHour {
return false
}
rl.Count++
newData, _ := json.Marshal(rl)
_ = os.WriteFile(rlPath, newData, 0600)
return true
}
func formatDispatchErrors(errs []error) error {
if len(errs) == 0 {
return nil
}
msgs := make([]string, len(errs))
for i, e := range errs {
msgs[i] = e.Error()
}
return fmt.Errorf("alert dispatch errors: %s", strings.Join(msgs, "; "))
}
// FillTimestamps stamps now on every finding that carries no Timestamp.
// Realtime producers build findings without one; a zero time sorts before
// every real event in a SIEM and makes the audit finding id collide across
// occurrences, so each sink boundary fills it in.
func FillTimestamps(findings []Finding, now time.Time) {
for i := range findings {
if findings[i].Timestamp.IsZero() {
findings[i].Timestamp = now
}
}
}
// Dispatch sends alerts via all configured channels without modifying findings.
func Dispatch(cfg *config.Config, findings []Finding) error {
return DispatchWithSources(cfg, findings, nil)
}
// DispatchWithSources audits source observations even when notification policy
// filters them out. Only findings reach notification channels and observers;
// sources add audit records without changing alert or auto-response policy.
// Both inputs remain caller-owned and must already carry the times used by
// actions that reference them. Missing times are filled on copies for ad-hoc use.
func DispatchWithSources(cfg *config.Config, findings, sources []Finding) error {
return dispatchWithSources(cfg, findings, sources, findings, nil, nil, true, nil)
}
// DispatchNotices delivers notices whose producer already bounds their rate.
// They keep their severity and notification settings, but do not spend or
// wait on the routine alert budget.
func DispatchNotices(cfg *config.Config, findings []Finding) error {
return DispatchNoticesWithNotificationFilter(cfg, findings, nil)
}
// DispatchNoticesWithNotificationFilter applies operator notification policy
// without hiding notices from audit, phpanel or SSE, or spending the routine
// alert budget.
func DispatchNoticesWithNotificationFilter(cfg *config.Config, findings []Finding, filter func([]Finding) []Finding) error {
return dispatchWithSources(cfg, findings, nil, findings, nil, filter, false, nil)
}
// DispatchWithEnforcement offers central IP enforcement its own finding set
// instead of the notification set. Suppression rules mute notifications but
// must not exempt an attacker from central challenges and blocks.
func DispatchWithEnforcement(cfg *config.Config, findings, sources, enforcement []Finding) error {
return dispatchWithSources(cfg, findings, sources, enforcement, nil, nil, true, nil)
}
// RepeatGate holds back findings alert email and the generic webhook already
// delivered with the same content. *state.Store implements it; alert cannot
// import state.
type RepeatGate interface {
// The returned callback must be called once, including on failed or
// capped sends, with only the successfully delivered findings.
BeginDelivery([]Finding) ([]Finding, func([]Finding))
}
// DispatchWithNotificationFilter keeps operator policy separate from the
// phpanel and SSE data streams. Notification observers, audit sources and
// central enforcement retain their independent policy boundaries. Delivery
// candidates may include reobserved findings whose history reminder is not due.
// A non-nil gate sends each candidate once per content to email and the generic
// webhook; candidates alone do not repeat observers or enforcement.
// Nil candidates reuse the findings and their filtered notification set.
func DispatchWithNotificationFilter(cfg *config.Config, findings, sources, enforcement, candidates []Finding, filter func([]Finding) []Finding, gate RepeatGate) error {
return dispatchWithSources(cfg, findings, sources, enforcement, candidates, filter, true, gate)
}
func dispatchWithSources(cfg *config.Config, findings, sources, enforcement, candidates []Finding, notificationFilter func([]Finding) []Finding, routineBudget bool, gate RepeatGate) error {
// Deduplicate owns a copy, so stamping cannot race with callers sharing
// the input or pin a reused unstamped finding to its first dispatch time.
findings = Deduplicate(findings)
now := auditNow()
FillTimestamps(findings, now)
sources = append([]Finding(nil), sources...)
FillTimestamps(sources, now)
notifications := findings
if notificationFilter != nil {
notifications = notificationFilter(findings)
// Audit every observation, while registered notification observers
// retain the same suppression policy as operator notifications.
sources = append(sources, findings...)
}
// Audit log captures every (deduplicated) finding before the class
// split and the rate limiter, so SIEMs see the complete picture even
// when email/webhook are throttled or a handled attack goes to the
// summary instead of the operator-facing channels.
emitAuditWithSources(cfg, notifications, sources)
// Every observation, audit sources included, may prove a response
// applied. Only the delivery set below is classified and summarized.
evidence := slices.Concat(sources, findings, candidates)
// The central-intel consumer escalates findings whose IP is in the
// verified central scored-set.
enforcement = Deduplicate(enforcement)
FillTimestamps(enforcement, now)
for _, f := range enforcement {
callCentralHook(f)
}
if len(findings) == 0 && len(candidates) == 0 {
return nil
}
// Publish to passive observers (e.g. SSE subscribers) immediately after
// auditing, before rate-limit and webhook delivery, so subscribers see
// the complete picture even when operator-facing channels are throttled.
if FindingBus != nil {
for _, f := range findings {
FindingBus.Publish(f)
}
}
// Offer every finding to the abuse reporter (it gates and minimizes
// internally, queueing only confirmed-abuse findings for the drain loop).
for _, f := range notifications {
callReportHook(f)
}
var errs []error
// Phpanel consumes this webhook as a signed data-plane stream. Send the
// full deduplicated stream before operator notification suppression and
// rate limiting, otherwise fleet correlation can miss attacker spread.
phpanelWebhook := cfg.Alerts.Webhook.Enabled && cfg.Alerts.Webhook.Type == "phpanel"
if phpanelWebhook {
if err := enqueuePhpanelFindings(cfg, findings); err != nil {
addDispatchError(&errs, err)
}
}
findings = notifications
if candidates != nil {
findings = Deduplicate(candidates)
FillTimestamps(findings, now)
if notificationFilter != nil {
findings = notificationFilter(findings)
}
}
// Summary-class findings report attacks CSM already handled. They leave
// before disabled_checks, the repeat gate and the hourly cap and go to
// the summary, muted checks included, so muting never hides activity.
// Each delivery finding is classified on its own fields: an older audit
// copy of the same key must not decide the class of an escalated one.
var summary []Finding
findings, summary = splitByClass(cfg, findings, evidence)
if len(summary) > 0 {
if err := callSummaryHook(cfg, summary); err != nil {
addDispatchError(&errs, err)
}
}
if len(findings) == 0 {
return formatDispatchErrors(errs)
}
emailFindings := []Finding(nil)
if cfg.Alerts.Email.Enabled {
emailFindings = filterChecks(findings, cfg.Alerts.Email.DisabledChecks)
}
webhookFindings := []Finding(nil)
if cfg.Alerts.Webhook.Enabled && !phpanelWebhook {
webhookFindings = findings
}
// The gate runs before the budget so a delivered finding neither spends
// nor waits on the hourly cap.
var delivered []Finding
if gate != nil && (len(emailFindings) > 0 || len(webhookFindings) > 0) {
eligible, finish := gate.BeginDelivery(Deduplicate(append(append([]Finding(nil), emailFindings...), webhookFindings...)))
defer func() { finish(Deduplicate(delivered)) }()
undelivered := make(map[string]bool)
for _, f := range eligible {
undelivered[f.Key()] = true
}
emailFindings = findingsWithKeys(emailFindings, undelivered)
webhookFindings = findingsWithKeys(webhookFindings, undelivered)
}
// Only routine findings spend the hourly budget. Urgent ones always go
// out, but they never carry routine findings from the same batch past the
// cap with them.
var reservation *rateLimitReservation
if routineBudget && (hasRoutineFinding(emailFindings) || hasRoutineFinding(webhookFindings)) {
var ok bool
reservation, ok = reserveRateLimit(cfg.StatePath, cfg.Alerts.MaxPerHour)
if ok {
defer releaseRateLimit(reservation)
} else {
fmt.Fprintf(os.Stderr, "Alert rate limit reached (%d/hour), skipping non-critical alert dispatch\n", cfg.Alerts.MaxPerHour)
emailFindings = urgentFindings(emailFindings)
webhookFindings = urgentFindings(webhookFindings)
}
}
if len(emailFindings) == 0 && len(webhookFindings) == 0 {
return formatDispatchErrors(errs)
}
routineDispatched := false
if len(emailFindings) > 0 {
subject := buildSubject(cfg.Hostname, emailFindings)
body := FormatAlert(cfg.Hostname, emailFindings)
if err := SendEmail(cfg, subject, body); err != nil {
addDispatchError(&errs, fmt.Errorf("email: %w", err))
} else {
delivered = append(delivered, emailFindings...)
routineDispatched = routineDispatched || hasRoutineFinding(emailFindings)
}
}
if len(webhookFindings) > 0 {
subject := buildSubject(cfg.Hostname, webhookFindings)
body := FormatAlert(cfg.Hostname, webhookFindings)
if err := SendWebhook(cfg, subject, body); err != nil {
addDispatchError(&errs, fmt.Errorf("webhook: %w", err))
} else {
delivered = append(delivered, webhookFindings...)
routineDispatched = routineDispatched || hasRoutineFinding(webhookFindings)
}
}
// An urgent-only email can succeed while the webhook carrying the routine
// findings fails. Spend the slot only if routine findings were delivered.
if routineDispatched && reservation != nil {
commitRateLimit(cfg.StatePath, reservation)
}
return formatDispatchErrors(errs)
}
// bypassesRateLimit reports whether a finding is delivered regardless of the
// hourly budget. Reputation delivery is check-keyed: its surface-based severity
// is presentation metadata and must not make sightings that previously
// bypassed the budget disappear.
func bypassesRateLimit(f Finding) bool {
return f.Severity == Critical || f.Check == "ip_reputation"
}
func findingsWithKeys(findings []Finding, keys map[string]bool) []Finding {
var out []Finding
for _, f := range findings {
if keys[f.Key()] {
out = append(out, f)
}
}
return out
}
func hasRoutineFinding(findings []Finding) bool {
for _, f := range findings {
if !bypassesRateLimit(f) {
return true
}
}
return false
}
func urgentFindings(findings []Finding) []Finding {
var out []Finding
for _, f := range findings {
if bypassesRateLimit(f) {
out = append(out, f)
}
}
return out
}
// SendHeartbeat pings a dead man's switch URL.
func SendHeartbeat(cfg *config.Config) {
if !cfg.Alerts.Heartbeat.Enabled || cfg.Alerts.Heartbeat.URL == "" {
return
}
client := httpClient(10 * time.Second)
resp, err := client.Get(cfg.Alerts.Heartbeat.URL)
if err != nil {
fmt.Fprintf(os.Stderr, "Heartbeat failed: %v\n", err)
return
}
defer closeWebhookResponseBody(resp)
}
package alert
// SubnetSprayCIDR returns the subnet a mail or SMTP password spray names in
// its structured CIDRs. Other findings, and a spray without exactly one
// subnet, return "": a crawl's subnets are block targets, not an attacker
// identity.
func SubnetSprayCIDR(f Finding) string {
switch f.Check {
case "mail_subnet_spray", "smtp_subnet_spray":
default:
return ""
}
if len(f.CIDRs) != 1 {
return ""
}
return f.CIDRs[0]
}
// AttackerAddress returns what a finding names as the attack source:
// SourceIP, or the subnet of a password spray. SourceIP comes first so a
// spray parked by an older daemon, with its subnet in SourceIP, reads the
// same as a new one.
func AttackerAddress(f Finding) string {
if f.SourceIP != "" {
return f.SourceIP
}
return SubnetSprayCIDR(f)
}
package alert
import (
"container/list"
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"runtime/debug"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/config"
)
// Audit-log dispatching is layered on top of the existing email +
// webhook fork in Dispatch(). Sinks live behind a package-level
// manager so the daemon's per-call Dispatch path does not pay the
// cost of opening a JSONL file or dialling syslog on every alert.
//
// The manager keys its sink set by a fingerprint of the relevant
// config sub-block; on hot reload the fingerprint changes and the
// manager closes the old sinks and rebuilds.
var (
auditMu sync.Mutex
auditSinks []*managedAuditSink
auditFingerprint string
auditNow = time.Now
openJSONLAuditSink = func(path string) (AuditSink, error) { return NewJSONLSink(path) }
openSyslogAuditSink = func(cfg SyslogConfig) (AuditSink, error) { return NewSyslogSink(cfg) }
)
type managedAuditSink struct {
name string
open func() (AuditSink, error)
sink AuditSink
retryAt time.Time
retryDelay time.Duration
failed bool
// Keep successful observation IDs across batches and transient sink
// failures. Each destination owns its receipts so a failed destination
// can receive a replay without duplicating the healthy destination.
delivered map[[sha256.Size]byte]*list.Element
recent list.List
}
// Bound replay receipts independently of traffic volume. Evicted observations
// can be emitted again; distinct observations are never dropped by this cache.
const auditReceiptCap = 16384
func (s *managedAuditSink) remember(id [sha256.Size]byte) {
if s.delivered == nil {
s.delivered = make(map[[sha256.Size]byte]*list.Element)
}
if s.recent.Len() == auditReceiptCap {
oldest := s.recent.Back()
delete(s.delivered, oldest.Value.([sha256.Size]byte))
s.recent.Remove(oldest)
}
s.delivered[id] = s.recent.PushFront(id)
}
// auditObservationKey distinguishes original evidence that the legacy action
// ID omits. For example, a process scan can report several PIDs with the same
// message and timestamp. Keep action IDs stable and hash scalar source evidence
// before redaction, without retaining its raw text or mutable enrichment.
func auditObservationKey(f Finding) [sha256.Size]byte {
var data []byte
for _, field := range []string{
f.Timestamp.UTC().Format(time.RFC3339Nano), f.Check, f.Severity.String(),
f.Message, f.FilePath, f.Details, strconv.Itoa(f.PID), f.SourceIP,
f.TenantID, f.Domain, f.Mailbox, f.DedupKey, f.CoverageScope,
} {
// Quote preserves boundaries and invalid UTF-8 from raw log input.
data = strconv.AppendQuote(data, field)
}
return sha256.Sum256(data)
}
// emitAudit records findings before email/webhook throttling. The manager lock
// covers the whole batch, including reconfiguration, so Close cannot invalidate
// a sink held by another dispatcher. Observers run outside the lock because
// they can dispatch findings themselves.
func emitAudit(cfg *config.Config, findings []Finding) {
emitAuditWithSources(cfg, findings, nil)
}
func emitAuditWithSources(cfg *config.Config, findings, sources []Finding) {
if cfg == nil {
return
}
for _, f := range findings {
notifyFindingObservers(f)
}
if len(sources) > 0 {
// Notification dedup uses condition keys. Audit joins need every
// distinct observation, including repeats with a new timestamp.
combined := make([]Finding, 0, len(sources)+len(findings))
seen := make(map[[sha256.Size]byte]bool, len(sources)+len(findings))
for _, batch := range [][]Finding{sources, findings} {
for _, f := range batch {
id := auditObservationKey(f)
if !seen[id] {
seen[id] = true
combined = append(combined, f)
}
}
}
findings = combined
}
auditMu.Lock()
defer auditMu.Unlock()
ensureAuditSinksLocked(cfg)
for _, f := range findings {
ev := NewAuditEvent(cfg.Hostname, f)
id := auditObservationKey(f)
for _, s := range auditSinks {
if receipt, delivered := s.delivered[id]; delivered {
// Retained findings can be replayed every scan amid realtime
// churn. Keep their receipts hot without growing the cache.
s.recent.MoveToFront(receipt)
continue
}
if s.sink == nil {
auditEventsDropped.With(s.name).Inc()
continue
}
if err := s.sink.Emit(ev); err != nil {
auditEventsDropped.With(s.name).Inc()
_ = s.sink.Close()
s.sink = nil
s.fail("emit", err)
} else {
s.retryDelay = 0
s.remember(id)
}
}
}
}
func ensureAuditSinks(cfg *config.Config) {
auditMu.Lock()
defer auditMu.Unlock()
ensureAuditSinksLocked(cfg)
}
func ensureAuditSinksLocked(cfg *config.Config) {
fp := auditConfigFingerprint(cfg)
if fp != auditFingerprint {
closeAuditSinksLocked()
if cfg.Alerts.AuditLog.File.Enabled {
path := cfg.Alerts.AuditLog.File.Path
auditSinks = append(auditSinks, &managedAuditSink{name: "jsonl", open: func() (AuditSink, error) { return openJSONLAuditSink(path) }})
}
if cfg.Alerts.AuditLog.Syslog.Enabled {
sc := SyslogConfig{
Network: cfg.Alerts.AuditLog.Syslog.Network,
Address: cfg.Alerts.AuditLog.Syslog.Address,
Facility: cfg.Alerts.AuditLog.Syslog.Facility,
Hostname: cfg.Hostname,
TLSCAFile: cfg.Alerts.AuditLog.Syslog.TLSCAFile,
}
auditSinks = append(auditSinks, &managedAuditSink{name: "syslog", open: func() (AuditSink, error) { return openSyslogAuditSink(sc) }})
}
auditFingerprint = fp
}
for _, s := range auditSinks {
if s.sink != nil || auditNow().Before(s.retryAt) {
continue
}
sink, err := s.open()
if err != nil {
s.fail("init", err)
continue
}
s.sink = sink
auditSinkDegraded.With(s.name).Set(0)
if s.failed {
fmt.Fprintf(os.Stderr, "[audit-log] %s recovered\n", s.name)
}
s.failed = false
}
}
func (s *managedAuditSink) fail(phase string, err error) {
if s.retryDelay == 0 {
s.retryDelay = time.Second
} else {
s.retryDelay = min(2*s.retryDelay, time.Minute)
}
s.retryAt = auditNow().Add(s.retryDelay)
s.failed = true
auditSinkDegraded.With(s.name).Set(1)
fmt.Fprintf(os.Stderr, "[audit-log] %s %s failed; retry after %s: %v\n", s.name, phase, s.retryDelay, err)
}
// auditConfigFingerprint reduces the audit-log sub-block to a stable
// hash so ensureAuditSinks can detect config changes without a deep
// reflect-based diff. Hostname is included because it appears in
// every emitted event.
func auditConfigFingerprint(cfg *config.Config) string {
h := sha256.New()
_, _ = fmt.Fprintf(h, "host=%s|", cfg.Hostname)
_, _ = fmt.Fprintf(h, "file.enabled=%t|file.path=%s|",
cfg.Alerts.AuditLog.File.Enabled,
cfg.Alerts.AuditLog.File.Path,
)
_, _ = fmt.Fprintf(h, "syslog.enabled=%t|syslog.network=%s|syslog.address=%s|syslog.facility=%s|syslog.tls=%s",
cfg.Alerts.AuditLog.Syslog.Enabled,
cfg.Alerts.AuditLog.Syslog.Network,
cfg.Alerts.AuditLog.Syslog.Address,
cfg.Alerts.AuditLog.Syslog.Facility,
cfg.Alerts.AuditLog.Syslog.TLSCAFile,
)
return hex.EncodeToString(h.Sum(nil))
}
// CloseAuditSinks waits for in-flight emissions and releases active sinks.
// A later dispatch can initialize them again.
func CloseAuditSinks() {
auditMu.Lock()
defer auditMu.Unlock()
closeAuditSinksLocked()
}
func closeAuditSinksLocked() {
for _, s := range auditSinks {
if s.sink != nil {
_ = s.sink.Close()
}
auditSinkDegraded.With(s.name).Set(0)
}
auditSinks = nil
auditFingerprint = ""
}
// resetAuditSinksForTest is the test-only seam to wipe the package
// state between cases. Production code never needs this -- live
// daemons run a single Dispatch path with a single config object.
func resetAuditSinksForTest() {
CloseAuditSinks()
}
// findingObservers registry. Used by the daemon to feed the incident
// correlator without making the alert package depend on internal/incident.
var (
findingObserversMu sync.RWMutex
findingObservers []findingObserver
findingObserverSeq atomic.Uint64
)
type findingObserver struct {
id uint64
fn func(Finding)
}
// RegisterFindingObserver registers fn to be called for every finding
// dispatched through emitAudit. Returns a cancel func that removes the
// observer. Safe for concurrent use; observer panics are recovered so
// one bad observer cannot stop dispatch.
func RegisterFindingObserver(fn func(Finding)) func() {
id := findingObserverSeq.Add(1)
findingObserversMu.Lock()
findingObservers = append(findingObservers, findingObserver{id: id, fn: fn})
findingObserversMu.Unlock()
return func() {
findingObserversMu.Lock()
defer findingObserversMu.Unlock()
out := findingObservers[:0]
for _, o := range findingObservers {
if o.id != id {
out = append(out, o)
}
}
findingObservers = out
}
}
// notifyFindingObservers fans a finding out to every registered observer.
// Each observer runs in a recover scope so a panic in one cannot stop
// dispatch to the rest, the audit-log sinks, or future ones.
func notifyFindingObservers(f Finding) {
findingObserversMu.RLock()
obs := append([]findingObserver(nil), findingObservers...)
findingObserversMu.RUnlock()
for _, o := range obs {
func(o findingObserver) {
defer func() {
if r := recover(); r != nil {
fmt.Fprintf(os.Stderr,
"alert: finding observer id=%d panic for check=%q: %s\n%s",
o.id, f.Check, formatRecoverValue(r), debug.Stack())
}
}()
o.fn(f)
}(o)
}
}
func formatRecoverValue(v any) (out string) {
defer func() {
if recover() != nil {
out = strconv.Quote(fmt.Sprintf("<unprintable panic value of type %T>", v))
}
}()
return strconv.QuoteToASCII(fmt.Sprint(v))
}
package alert
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
)
// JSONLSink appends one JSON object per finding to a file. Designed
// for SIEM ingest via standard log shippers (Vector, Filebeat,
// Fluentbit). Writes are mutex-serialised so concurrent emits cannot
// interleave bytes inside a single line; logrotate's copytruncate
// rotation works without daemon restart because the file's offset is
// reset by the truncate, which the open fd then writes past.
type JSONLSink struct {
path string
mu sync.Mutex
f *os.File
}
// NewJSONLSink opens (or creates) the JSONL file. Permissions are
// 0640 -- group-readable so an operator running a log shipper under
// a non-root user in the appropriate group can tail it. The parent
// directory is created with 0750 so packaging (logrotate) sees a
// reasonable default.
func NewJSONLSink(path string) (*JSONLSink, error) {
if path == "" {
return nil, errors.New("jsonl sink: path is empty")
}
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
return nil, fmt.Errorf("jsonl sink: creating dir: %w", err)
}
// #nosec G304 G302 -- G304: path comes from cfg.Alerts.AuditLog.File.Path, which is operator-controlled (root-owned daemon config), not attacker input. G302: 0640 is intentional; SIEM log shippers (Vector, Filebeat, Fluentbit) commonly run as a non-root user that needs group-read access. 0600 would force the shipper to run as root.
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
if err != nil {
return nil, fmt.Errorf("jsonl sink: opening %s: %w", path, err)
}
return &JSONLSink{path: path, f: f}, nil
}
// Name returns the sink identifier used in error messages.
func (s *JSONLSink) Name() string { return "jsonl" }
// Emit appends one JSON line. The trailing newline is written as part
// of the same Write call so a partial write at EOL boundaries cannot
// leave a half-finished line in the file.
func (s *JSONLSink) Emit(event AuditEvent) error {
line, err := json.Marshal(event)
if err != nil {
return fmt.Errorf("jsonl sink: marshal: %w", err)
}
line = append(line, '\n')
s.mu.Lock()
defer s.mu.Unlock()
if s.f == nil {
return errors.New("jsonl sink: closed")
}
if _, err := s.f.Write(line); err != nil {
return fmt.Errorf("jsonl sink: write: %w", err)
}
return nil
}
// Close releases the file descriptor. Safe to call multiple times.
func (s *JSONLSink) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.f == nil {
return nil
}
err := s.f.Close()
s.f = nil
return err
}
package alert
import "github.com/pidginhost/csm/internal/metrics"
var (
auditSinkDegraded = metrics.NewGaugeVec("csm_audit_sink_degraded",
"Configured audit destination unavailable or last delivery failed; 1 means degraded, 0 means healthy or disabled.", []string{"sink"})
auditEventsDropped = metrics.NewCounterVec("csm_audit_events_dropped_total",
"Audit events with unavailable destinations or failed writes.", []string{"sink"})
)
func init() {
metrics.MustRegister("csm_audit_sink_degraded", auditSinkDegraded)
metrics.MustRegister("csm_audit_events_dropped_total", auditEventsDropped)
}
package alert
import (
"crypto/sha256"
"encoding/hex"
"time"
"github.com/pidginhost/csm/internal/processctx"
)
// AuditSchemaVersion is the value emitted in every AuditEvent's "v"
// field. Frozen contract -- downstream JSONL / syslog parsers pin on
// it. Bump only on incompatible schema changes; additive fields stay
// at the same version.
const AuditSchemaVersion = 1
// AuditEvent is the wire-stable shape every audit-log sink emits. JSON
// keys match what downstream SIEMs expect; fields are added at the
// end so older parsers ignore unknown ones.
type AuditEvent struct {
V int `json:"v"`
Timestamp time.Time `json:"ts"`
FindingID string `json:"finding_id"`
Severity string `json:"severity"`
Check string `json:"check"`
Message string `json:"message"`
Details string `json:"details,omitempty"`
Context string `json:"context,omitempty"`
FilePath string `json:"file_path,omitempty"`
Hostname string `json:"hostname"`
TenantID string `json:"tenant_id,omitempty"`
Domain string `json:"domain,omitempty"`
Mailbox string `json:"mailbox,omitempty"`
Process *processctx.ProcessContext `json:"process,omitempty"`
CIDRs []string `json:"cidrs,omitempty"`
}
// AuditSink is what every audit-log destination implements. Emit must
// be safe for concurrent calls; the alert dispatcher fans out to
// multiple sinks per finding.
type AuditSink interface {
// Name identifies the sink for diagnostics (e.g. "jsonl", "syslog").
Name() string
// Emit ships one event. Should return promptly; sinks that need
// long-haul I/O are expected to handle their own buffering.
Emit(event AuditEvent) error
// Close releases any held resources. Safe to call multiple times.
Close() error
}
// NewAuditEvent builds a versioned audit event from a Finding. hostname
// comes from cfg.Hostname (or os.Hostname() fallback); the caller is
// responsible for picking a stable value across emits.
//
// The finding is redacted here because this is the one constructor both
// audit sinks build from. Redaction used to run only while rendering
// the email digest, so secrets a watcher had copied out of a raw log
// line -- cPanel session identifiers, password fields -- were written
// to audit.jsonl and shipped to syslog in the clear.
func NewAuditEvent(hostname string, f Finding) AuditEvent {
// Remediation records hash the original finding, so redaction must not
// change the ID used to join those records to this event.
id := FindingID(f)
f = SanitizeFinding(f)
return AuditEvent{
V: AuditSchemaVersion,
Timestamp: f.Timestamp.UTC(),
FindingID: id,
Severity: f.Severity.String(),
Check: f.Check,
Message: f.Message,
Details: f.Details,
Context: f.Context,
FilePath: f.FilePath,
Hostname: hostname,
TenantID: f.TenantID,
Domain: f.Domain,
Mailbox: f.Mailbox,
Process: f.Process,
CIDRs: f.CIDRs,
}
}
// FindingID hashes the canonical fields of a Finding to a stable
// 16-hex-char ID. Two emits of the same finding (same timestamp + the
// same other fields) produce the same ID, so downstream dedup works
// across re-runs.
//
// The hash inputs use a "|" separator so the byte-for-byte
// concatenation cannot collide via field-boundary ambiguity (e.g. a
// Check name that ends in the same chars another field starts with).
func FindingID(f Finding) string {
h := sha256.New()
_, _ = h.Write([]byte(f.Timestamp.UTC().Format(time.RFC3339Nano)))
_, _ = h.Write([]byte("|"))
_, _ = h.Write([]byte(f.Check))
_, _ = h.Write([]byte("|"))
_, _ = h.Write([]byte(f.Severity.String()))
_, _ = h.Write([]byte("|"))
_, _ = h.Write([]byte(f.Message))
_, _ = h.Write([]byte("|"))
_, _ = h.Write([]byte(f.FilePath))
return hex.EncodeToString(h.Sum(nil))[:16]
}
package alert
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"net"
"os"
"strings"
"sync"
"time"
)
// SyslogConfig drives the syslog sink. Network is one of "udp",
// "tcp", "unix", "unixgram", or "tls"; Address is host:port (or a
// filesystem path for the unix variants). Facility names follow the
// classic syslog set; default is "local0".
type SyslogConfig struct {
Network string
Address string
Facility string
Hostname string // typically cfg.Hostname; falls back to os.Hostname()
TLSCAFile string // optional CA cert path for tls; empty = system roots
}
const (
syslogDialTimeout = 5 * time.Second
syslogWriteTimeout = 2 * time.Second
)
// SyslogSink is an RFC 5424 syslog client. The wire payload is the
// AuditEvent JSON so SIEMs that parse our JSONL file have a single
// schema regardless of transport. Messages bigger than the legacy
// 1024-byte limit are still sent -- modern receivers (rsyslog,
// syslog-ng) accept the full RFC 5424 max of 8192 bytes; if the
// operator's receiver caps shorter, syslog truncation is the
// expected behaviour.
type SyslogSink struct {
cfg SyslogConfig
priority int // pre-computed PRI byte; severity is OR'd in per emit
mu sync.Mutex
conn net.Conn
}
// facilityCodes is the standard syslog facility number set. local0..7
// is the customary range for application-level audit traffic; we
// default to local0 if the operator leaves the field blank.
var facilityCodes = map[string]int{
"kern": 0, "user": 1, "mail": 2, "daemon": 3, "auth": 4,
"syslog": 5, "lpr": 6, "news": 7, "uucp": 8, "cron": 9,
"authpriv": 10, "ftp": 11,
"local0": 16, "local1": 17, "local2": 18, "local3": 19,
"local4": 20, "local5": 21, "local6": 22, "local7": 23,
}
// NewSyslogSink validates the config, dials the destination, and
// returns a ready-to-emit sink. The dispatch manager reports dial failures
// and retries with backoff. Direct callers can retry a failed write with
// another Emit, which redials when the connection was dropped.
func NewSyslogSink(cfg SyslogConfig) (*SyslogSink, error) {
if cfg.Network == "" || cfg.Address == "" {
return nil, errors.New("syslog sink: network and address required")
}
switch cfg.Network {
case "udp", "tcp", "unix", "unixgram", "tls":
default:
return nil, fmt.Errorf("syslog sink: unknown network %q (want udp|tcp|unix|unixgram|tls)", cfg.Network)
}
facilityName := strings.ToLower(strings.TrimSpace(cfg.Facility))
if facilityName == "" {
facilityName = "local0"
}
facility, ok := facilityCodes[facilityName]
if !ok {
return nil, fmt.Errorf("syslog sink: unknown facility %q", cfg.Facility)
}
if cfg.Hostname == "" {
if h, err := os.Hostname(); err == nil {
cfg.Hostname = h
} else {
cfg.Hostname = "localhost"
}
}
s := &SyslogSink{cfg: cfg, priority: facility * 8}
if err := s.dial(); err != nil {
return nil, err
}
return s, nil
}
// Name identifies the sink in error messages.
func (s *SyslogSink) Name() string { return "syslog" }
// Emit formats the event as RFC 5424 and writes it to the
// destination. Mutex-serialised so concurrent calls do not interleave
// bytes on stream-oriented transports (TCP, TLS).
func (s *SyslogSink) Emit(event AuditEvent) error {
line, err := s.format(event)
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
if s.conn == nil {
if redialErr := s.dialLocked(); redialErr != nil {
return redialErr
}
}
// Emit runs on the single alert-dispatch goroutine. A receiver that stops
// reading fills the socket buffer, and a write with no deadline would then
// hold this mutex forever and stall every alert on the host. Give up and
// drop the connection instead; the next Emit redials.
if err := s.conn.SetWriteDeadline(time.Now().Add(syslogWriteTimeout)); err != nil {
_ = s.conn.Close()
s.conn = nil
return fmt.Errorf("syslog sink: set write deadline: %w", err)
}
if _, err := s.conn.Write(line); err != nil {
_ = s.conn.Close()
s.conn = nil
return fmt.Errorf("syslog sink: write: %w", err)
}
return nil
}
// Close releases the connection.
func (s *SyslogSink) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.conn == nil {
return nil
}
err := s.conn.Close()
s.conn = nil
return err
}
func (s *SyslogSink) dial() error {
s.mu.Lock()
defer s.mu.Unlock()
return s.dialLocked()
}
// dialLocked establishes the connection. Caller must hold s.mu.
func (s *SyslogSink) dialLocked() error {
if s.cfg.Network == "tls" {
tlsCfg, err := s.tlsConfig()
if err != nil {
return err
}
// The context bounds the TLS handshake as well as the TCP connect; a
// blackholed receiver would otherwise hold the dispatch goroutine for
// the kernel's SYN retry budget on every redial.
ctx, cancel := context.WithTimeout(context.Background(), syslogDialTimeout)
defer cancel()
dialer := &tls.Dialer{NetDialer: &net.Dialer{Timeout: syslogDialTimeout}, Config: tlsCfg}
conn, err := dialer.DialContext(ctx, "tcp", s.cfg.Address)
if err != nil {
return fmt.Errorf("syslog sink: tls dial %s: %w", s.cfg.Address, err)
}
s.conn = conn
return nil
}
conn, err := net.DialTimeout(s.cfg.Network, s.cfg.Address, syslogDialTimeout)
if err != nil {
return fmt.Errorf("syslog sink: %s dial %s: %w", s.cfg.Network, s.cfg.Address, err)
}
s.conn = conn
return nil
}
func (s *SyslogSink) tlsConfig() (*tls.Config, error) {
cfg := &tls.Config{MinVersion: tls.VersionTLS12}
if s.cfg.TLSCAFile == "" {
return cfg, nil
}
// #nosec G304 -- TLSCAFile is operator-supplied via cfg.Alerts.AuditLog.Syslog.TLSCAFile; the operator owns the daemon config. Not attacker-controlled.
pem, err := os.ReadFile(s.cfg.TLSCAFile)
if err != nil {
return nil, fmt.Errorf("syslog sink: reading TLS CA: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, fmt.Errorf("syslog sink: TLS CA file %s is not a valid PEM bundle", s.cfg.TLSCAFile)
}
cfg.RootCAs = pool
return cfg, nil
}
// format produces an RFC 5424 line. The MSG body is the JSON-encoded
// event so receivers can parse the structured payload directly.
//
// <PRI>1 TIMESTAMP HOSTNAME APP-NAME PROCID MSGID - MSG
//
// PRI = facility * 8 + severity-as-syslog-level. STRUCTURED-DATA is
// "-" (we lift everything into the JSON body to avoid duplicate
// representation).
func (s *SyslogSink) format(event AuditEvent) ([]byte, error) {
body, err := json.Marshal(event)
if err != nil {
return nil, fmt.Errorf("syslog sink: marshal: %w", err)
}
pri := s.priority + severityToSyslogLevel(event.Severity)
ts := event.Timestamp.UTC().Format(time.RFC3339Nano)
msgID := event.Check
if msgID == "" {
msgID = "-"
}
procID := os.Getpid()
line := fmt.Sprintf("<%d>1 %s %s csm %d %s - %s",
pri, ts, s.cfg.Hostname, procID, msgID, body)
// RFC 5424 over UDP / unixgram is one datagram per message; over
// TCP / TLS / unix-stream the receiver expects either octet
// counting ("nnn ") or LF framing. LF is the common rsyslog
// default; emit it for stream transports.
if s.cfg.Network == "tcp" || s.cfg.Network == "tls" || s.cfg.Network == "unix" {
line += "\n"
}
return []byte(line), nil
}
// severityToSyslogLevel maps CSM severity strings onto the standard
// syslog level codes. Critical -> 2 (crit), High -> 3 (err),
// Warning -> 4 (warning), default -> 6 (info).
func severityToSyslogLevel(s string) int {
switch s {
case "CRITICAL":
return 2
case "HIGH":
return 3
case "WARNING":
return 4
}
return 6
}
package alert
// Cause names the finding a derived finding was made from: its audit
// identity and its check.
type Cause struct {
FindingID string `json:"finding_id"`
Check string `json:"check"`
}
// CauseOf names f as the cause of a finding derived from it.
func CauseOf(f Finding) Cause {
return Cause{FindingID: FindingID(f), Check: f.Check}
}
package alert
import (
"crypto/tls"
"fmt"
"net"
"net/smtp"
"strings"
"time"
"github.com/pidginhost/csm/internal/config"
)
var emailSendTimeout = 10 * time.Second
var smtpDial = func(timeout time.Duration, addr string) (net.Conn, error) {
return (&net.Dialer{Timeout: timeout}).Dial("tcp", addr)
}
func SendEmail(cfg *config.Config, subject, body string) error {
to := cfg.Alerts.Email.To
from := cfg.Alerts.Email.From
smtpAddr := cfg.Alerts.Email.SMTP
if len(to) == 0 {
return fmt.Errorf("no email recipients configured")
}
msg := fmt.Sprintf("From: %s\r\nTo: %s\r\nSubject: %s\r\nContent-Type: text/plain; charset=UTF-8\r\n\r\n%s",
from,
strings.Join(to, ", "),
subject,
body,
)
host := smtpHost(smtpAddr)
deadline := time.Now().Add(emailSendTimeout)
if err := sendSMTPWithDeadline(smtpAddr, host, from, to, []byte(msg), deadline, true); err != nil {
if fallbackErr := sendSMTPWithDeadline(smtpAddr, host, from, to, []byte(msg), deadline, false); fallbackErr != nil {
return fmt.Errorf("%w (original: %v)", fallbackErr, err)
}
}
return nil
}
func smtpHost(addr string) string {
host, _, err := net.SplitHostPort(addr)
if err == nil {
return strings.Trim(host, "[]")
}
return strings.Split(addr, ":")[0]
}
func sendSMTPWithDeadline(addr, host, from string, to []string, msg []byte, deadline time.Time, tryStartTLS bool) error {
timeout := time.Until(deadline)
if timeout <= 0 {
return fmt.Errorf("smtp send timed out after %s", emailSendTimeout)
}
conn, dialErr := smtpDial(timeout, addr)
if dialErr != nil {
return fmt.Errorf("smtp dial %s: %w", addr, dialErr)
}
if err := conn.SetDeadline(deadline); err != nil {
_ = conn.Close()
return fmt.Errorf("smtp deadline: %w", err)
}
c, clientErr := smtp.NewClient(conn, host)
if clientErr != nil {
_ = conn.Close()
return fmt.Errorf("smtp connect %s: %w", addr, clientErr)
}
defer func() { _ = c.Close() }()
if err := c.Hello(host); err != nil {
return fmt.Errorf("smtp hello: %w", err)
}
if tryStartTLS {
if ok, _ := c.Extension("STARTTLS"); ok {
if err := c.StartTLS(&tls.Config{ServerName: host, MinVersion: tls.VersionTLS12}); err != nil {
return fmt.Errorf("smtp starttls: %w", err)
}
}
}
if err := c.Mail(from); err != nil {
return fmt.Errorf("smtp mail from: %w", err)
}
for _, addr := range to {
if err := c.Rcpt(addr); err != nil {
return fmt.Errorf("smtp rcpt %s: %w", addr, err)
}
}
w, dataErr := c.Data()
if dataErr != nil {
return fmt.Errorf("smtp data: %w", dataErr)
}
if _, writeErr := w.Write(msg); writeErr != nil {
return fmt.Errorf("smtp write: %w", writeErr)
}
if closeErr := w.Close(); closeErr != nil {
return fmt.Errorf("smtp close: %w", closeErr)
}
if quitErr := c.Quit(); quitErr != nil {
return fmt.Errorf("smtp quit: %w", quitErr)
}
return nil
}
package alert
import (
"encoding/json"
"net"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
csmlog "github.com/pidginhost/csm/internal/log"
)
// State files tell SplitByClass which sources CSM already blocked. Parse
// failures must degrade open so corrupt state never moves an operator-facing
// finding to the summary; warning logs keep the corruption visible. Missing
// state files stay silent because they are normal before the first block.
// SplitByClass separates the findings an operator must see now from the
// ones that report an attack CSM already handled, which belong in the daily
// summary. A finding is summary class when the summary-notice policy names
// it (an applied response or an observation that never acts), or when a
// block or challenge on its source address answers it. A block counts only
// once it is in effect: blocked state on disk, or an applied block notice in
// the same batch. Pending, unverified and dry-run blocks keep the finding
// immediate. Both halves keep their input order.
func SplitByClass(cfg *config.Config, findings []Finding) (immediate, summary []Finding) {
return splitByClass(cfg, findings, findings)
}
// splitByClass classifies each of findings on its own fields, while every
// observation in evidence may prove a block or challenge applied. Dispatch
// passes its audit sources as evidence only: a suppressed block notice still
// blocked its address, but it is not delivered to the summary itself.
func splitByClass(cfg *config.Config, findings, evidence []Finding) (immediate, summary []Finding) {
if len(findings) == 0 {
return nil, nil
}
notice := currentSummaryNoticePolicy()
policy := currentIPResponsePolicy()
web := currentWebTrafficPolicy()
// The split runs on every dispatch: block state is read once, and only
// for a batch where a block could answer a finding.
var handled *handledSet
for _, f := range findings {
answered := notice != nil && notice(f)
if !answered {
if ip := ClassSourceIP(f); ip != nil {
if handled == nil {
handled = handledSources(cfg.StatePath, evidence)
}
answered = handled.answers(policy, web, cfg, f, ip)
}
}
if answered {
summary = append(summary, f)
} else {
immediate = append(immediate, f)
}
}
return immediate, summary
}
// handledSet holds the canonical addresses and subnets CSM blocked, and the
// addresses it routed to the challenge.
type handledSet struct {
blocked map[string]bool
subnets []*net.IPNet
challenged map[string]bool
// The Cloudflare ranges the firewall accepts on its web ports, read at
// the first blocked web finding.
statePath string
cfLoaded bool
cfV4, cfV6 []string
}
// loadCloudflareRanges reads the firewall's cached Cloudflare ranges.
var loadCloudflareRanges = firewall.LoadCFState
func handledSources(statePath string, findings []Finding) *handledSet {
// Load only IPs with actual block state. Pending entries have not reached
// the firewall and must remain visible to the operator.
blockedIPs := loadBlockedIPs(statePath)
// Also collect IPs and subnets blocked in this batch, and IPs routed
// to the challenge in this batch. A challenge-routed IP is handled
// automatically the same way a blocked one is: it either solves the
// PoW (legitimate) or gets escalated to a block by the port gate, so
// the operator needs no notification either way. Dry-run notices put
// "[dry-run]" after the action word, so they never match these tokens.
var blockedSubnets []*net.IPNet
challengedIPs := make(map[string]bool)
for _, f := range findings {
switch f.Check {
case "auto_block":
parts := strings.Fields(f.Message)
for i, p := range parts {
if p == "AUTO-BLOCK:" && i+1 < len(parts) {
blockedIPs[parts[i+1]] = true
break
}
if (p == "AUTO-BLOCK-SUBNET:" || p == "AUTO-NETBLOCK:") && i+1 < len(parts) {
if _, ipnet, err := net.ParseCIDR(parts[i+1]); err == nil {
blockedSubnets = append(blockedSubnets, ipnet)
}
break
}
}
case "challenge_route":
parts := strings.Fields(f.Message)
for i, p := range parts {
if p == "CHALLENGE:" && i+1 < len(parts) {
challengedIPs[parts[i+1]] = true
break
}
}
}
}
// Canonicalize blocked-IP keys via net.ParseIP so equality survives
// notation differences (2001:db8::1 vs 2001:db8:0:0:0:0:0:1,
// IPv4-mapped IPv6). Keys that do not parse as an IP cannot
// exact-match a finding's IP and are dropped.
h := &handledSet{
blocked: make(map[string]bool, len(blockedIPs)),
subnets: blockedSubnets,
challenged: make(map[string]bool, len(challengedIPs)),
statePath: statePath,
}
for ip := range blockedIPs {
if parsed := net.ParseIP(ip); parsed != nil {
h.blocked[parsed.String()] = true
}
}
for ip := range challengedIPs {
if parsed := net.ParseIP(ip); parsed != nil {
h.challenged[parsed.String()] = true
}
}
return h
}
// answers reports whether a block or challenge on ip, f's source, answers f.
func (h *handledSet) answers(policy IPResponsePolicy, web func(Finding) bool, cfg *config.Config, f Finding, ip net.IP) bool {
// The class keys on the actual block state, never on auto-response
// intent: an enabled block_ips used to hide every reputation finding,
// including the IPs auto-block did NOT handle (dry-run, rate-limited
// queue drops, verdict-allowed). Structured SourceIP wins when present;
// older findings fall back to the message token. The address is
// compared canonically: substring matching used to let a blocked
// 192.0.2.1 answer a finding about the unrelated 192.0.2.10. A finding
// with no parseable IP is never handled (fail open to alerting).
// The Cloudflare accept on TCP 80/443 precedes the blocked drop, so a
// block of a covered address leaves its web traffic flowing.
if ipBlocked(ip, h.blocked, h.subnets) && ipResponseAnswers(policy, cfg, f, true) &&
(web == nil || !web(f) || !h.cloudflareCovers(ip)) {
return true
}
// A challenge counts only for findings a challenge answers. Critical
// ip_reputation is a browserless mail/SSH/FTP sighting and must stay
// immediate until a hard block lands; otherwise an old challenge entry
// hides the exact finding that is supposed to escalate it.
return ipResponseAnswers(policy, cfg, f, false) && ipChallenged(ip, h.challenged)
}
func (h *handledSet) cloudflareCovers(ip net.IP) bool {
if !h.cfLoaded {
h.cfV4, h.cfV6 = loadCloudflareRanges(h.statePath)
h.cfLoaded = true
}
return firewall.CloudflareRangesCover(h.cfV4, h.cfV6, ip.String())
}
func ipBlocked(ip net.IP, blocked map[string]bool, subnets []*net.IPNet) bool {
if blocked[ip.String()] {
return true
}
// Same-batch subnet and netblock blocks answer per-IP findings for
// addresses inside them.
for _, subnet := range subnets {
if subnet.Contains(ip) {
return true
}
}
return false
}
func ipChallenged(ip net.IP, challenged map[string]bool) bool {
if challenged[ip.String()] {
return true
}
return ChallengedIPFunc != nil && ChallengedIPFunc(ip.String())
}
// ipResponseAnswers reports whether a block (blocked) or a challenge answers
// the finding. With no policy wired it keeps the reputation-only rule.
func ipResponseAnswers(policy IPResponsePolicy, cfg *config.Config, f Finding, blocked bool) bool {
if policy != nil {
return policy(cfg, f, blocked)
}
if f.Check != "ip_reputation" {
return false
}
return blocked || challengeHandlesFinding(f)
}
// challengeHandlesFinding mirrors the only finding-level exception to the
// challenge response policy. ip_reputation has one production grading source:
// High is browser-facing HTTP/cPanel traffic, while Critical is a browserless
// vector that resolves to a hard block.
func challengeHandlesFinding(f Finding) bool {
return f.Check == "ip_reputation" && f.Severity != Critical
}
// ClassSourceIP is the address SplitByClass checks against block and
// challenge state: the structured SourceIP, or for an older finding the
// first address in its message. nil means the finding has none.
func ClassSourceIP(f Finding) net.IP {
if strings.TrimSpace(f.SourceIP) != "" {
if normalized := normalizeFindingIP(f.SourceIP); normalized != "" {
return net.ParseIP(normalized)
}
return nil
}
return extractIPFromFindingMessage(f.Message)
}
// extractIPFromFindingMessage scans a finding message for the first token
// that parses as a valid IP address and returns it. Returns nil if no IP
// is found. Used to match reputation findings against blocked IPs and
// blocked-subnet CIDRs.
func extractIPFromFindingMessage(msg string) net.IP {
for _, field := range strings.Fields(msg) {
if ip := parseIPMessageField(field); ip != nil {
return ip
}
}
return nil
}
func parseIPMessageField(field string) net.IP {
if ip := parseIPMessageToken(field); ip != nil {
return ip
}
// Accept key=value tokens such as "ip=5.5.5.5". IP literals never
// contain '=', so taking the value side cannot mangle a real IP.
if idx := strings.LastIndexByte(field, '='); idx >= 0 {
return parseIPMessageToken(field[idx+1:])
}
return nil
}
func parseIPMessageToken(token string) net.IP {
token = strings.TrimSpace(token)
if token == "" {
return nil
}
if ip := parseNormalizedIP(token); ip != nil {
return ip
}
unquoted := strings.Trim(token, "\"'`")
if ip := parseNormalizedIP(unquoted); ip != nil {
return ip
}
withoutTrailing := strings.TrimRight(unquoted, ",;")
if ip := parseNormalizedIP(withoutTrailing); ip != nil {
return ip
}
unwrapped := strings.Trim(withoutTrailing, "()[]{}<>")
if ip := parseNormalizedIP(unwrapped); ip != nil {
return ip
}
if strings.HasSuffix(unwrapped, ":") {
withoutColon := strings.TrimSuffix(unwrapped, ":")
if ip := parseNormalizedIP(withoutColon); ip != nil {
return ip
}
if ip := parseNormalizedIP(strings.Trim(withoutColon, "()[]{}<>")); ip != nil {
return ip
}
}
return nil
}
func parseNormalizedIP(raw string) net.IP {
if normalized := normalizeFindingIP(raw); normalized != "" {
return net.ParseIP(normalized)
}
return nil
}
func loadBlockedIPSource(statePath string, now time.Time, ips map[string]bool) {
// Use injected loader (bbolt-backed) when available.
loadVerified := BlockedIPsFunc == nil
if !loadVerified {
for ip, v := range BlockedIPsFunc() {
ips[ip] = v
}
}
// Unverified state overrides legacy tracker rows even with an injected
// loader, which only returns the addresses it considers blocked.
loadFirewallStateFile(statePath, now, ips, loadVerified)
}
// BlockedIPsFunc is an optional callback that returns currently blocked IPs.
// Set by the daemon (or tests) to provide blocked IPs from bbolt store,
// avoiding a circular import between alert and store packages.
// When nil, loadBlockedIPs falls back to reading flat files.
var BlockedIPsFunc func() map[string]bool
// ChallengedIPFunc reports whether an IP is currently on the challenge
// list. Set by the daemon from the live challenge IP list, mirroring
// BlockedIPsFunc; nil means challenge membership is unknown and no listed
// challenge makes a finding summary class (fail open to alerting).
var ChallengedIPFunc func(ip string) bool
// loadBlockedIPs reads blocked IPs from both the firewall engine state
// and the legacy blocked_ips.json file.
func loadBlockedIPs(statePath string) map[string]bool {
ips := make(map[string]bool)
now := time.Now()
loadBlockFileEntries(statePath, now, ips, nil, blockFileIPsSection)
loadBlockedIPSource(statePath, now, ips)
return ips
}
func loadFirewallStateFile(statePath string, now time.Time, ips map[string]bool, loadVerified bool) {
fwPath := filepath.Join(statePath, "firewall", "state.json")
fwData, err := os.ReadFile(fwPath) // #nosec G304 -- filepath.Join under operator-configured statePath.
if err != nil {
return
}
var fwState struct {
Blocked []struct {
IP string `json:"ip"`
ExpiresAt time.Time `json:"expires_at"`
Unverified bool `json:"unverified"`
} `json:"blocked"`
}
if err := json.Unmarshal(fwData, &fwState); err != nil {
csmlog.Warn("alert class split: firewall state.json unparseable, its blocks count as unhandled",
"path", fwPath, "err", err)
return
}
unverified := make(map[string]bool)
for _, entry := range fwState.Blocked {
if entry.Unverified {
if ip := net.ParseIP(entry.IP); ip != nil {
unverified[ip.String()] = true
}
} else if loadVerified && (entry.ExpiresAt.IsZero() || now.Before(entry.ExpiresAt)) {
ips[entry.IP] = true
}
}
for ip := range ips {
if parsed := net.ParseIP(ip); parsed != nil && unverified[parsed.String()] {
delete(ips, ip)
}
}
}
type blockFile struct {
IPs []blockFileIP
Pending []blockFilePendingIP
}
type blockFileIP struct {
IP string `json:"ip"`
ExpiresAt time.Time `json:"expires_at"`
}
type blockFilePendingIP struct {
IP string `json:"ip"`
}
type blockFileSection uint8
const (
blockFileIPsSection blockFileSection = 1 << iota
blockFilePendingSection
)
func loadBlockFileEntries(statePath string, now time.Time, ips map[string]bool, pending map[string]bool, sections blockFileSection) {
bf, ok := loadBlockFile(statePath, sections)
if !ok {
return
}
for _, entry := range bf.IPs {
if ips != nil && (entry.ExpiresAt.IsZero() || now.Before(entry.ExpiresAt)) {
ips[entry.IP] = true
}
}
for _, entry := range bf.Pending {
if pending != nil {
pending[entry.IP] = true
}
}
}
func loadBlockFile(statePath string, sections blockFileSection) (blockFile, bool) {
blockedPath := filepath.Join(statePath, "blocked_ips.json")
data, err := os.ReadFile(blockedPath) // #nosec G304 -- filepath.Join under operator-configured statePath.
if err != nil {
return blockFile{}, false
}
var raw struct {
IPs json.RawMessage `json:"ips"`
Pending json.RawMessage `json:"pending"`
}
if err := json.Unmarshal(data, &raw); err != nil {
csmlog.Warn("alert class split: blocked_ips.json unparseable, its blocks count as unhandled",
"path", blockedPath, "err", err)
return blockFile{}, false
}
var bf blockFile
if sections&blockFileIPsSection != 0 && len(raw.IPs) > 0 && string(raw.IPs) != "null" {
if err := json.Unmarshal(raw.IPs, &bf.IPs); err != nil {
csmlog.Warn("alert class split: blocked_ips.json blocked entries unparseable, they count as unhandled",
"path", blockedPath, "err", err)
}
}
if sections&blockFilePendingSection != 0 && len(raw.Pending) > 0 && string(raw.Pending) != "null" {
if err := json.Unmarshal(raw.Pending, &bf.Pending); err != nil {
csmlog.Warn("alert class split: blocked_ips.json pending entries unparseable, they count as unhandled",
"path", blockedPath, "err", err)
}
}
return bf, true
}
// IPResponsePolicy reports whether an IP disposition on a finding's source
// fully answers the finding, so SplitByClass moves it to the summary. blocked
// is true for a firewall block and false for a challenge.
type IPResponsePolicy func(cfg *config.Config, f Finding, blocked bool) bool
var (
ipResponsePolicyMu sync.RWMutex
ipResponsePolicy IPResponsePolicy
summaryNoticeMu sync.RWMutex
summaryNoticePolicy func(Finding) bool
webTrafficMu sync.RWMutex
webTrafficPolicy func(Finding) bool
)
// SetIPResponsePolicy installs or clears the policy SplitByClass consults
// and returns the previous one. The check classification lives in the
// checks package, which imports this one and registers it at package
// initialization so every dispatch path uses the same policy.
func SetIPResponsePolicy(p IPResponsePolicy) IPResponsePolicy {
ipResponsePolicyMu.Lock()
defer ipResponsePolicyMu.Unlock()
previous := ipResponsePolicy
ipResponsePolicy = p
return previous
}
func currentIPResponsePolicy() IPResponsePolicy {
ipResponsePolicyMu.RLock()
p := ipResponsePolicy
ipResponsePolicyMu.RUnlock()
return p
}
// SetSummaryNoticePolicy installs or clears the policy that names summary
// notices by their own content, and returns the previous one. The checks
// package registers it at package initialization, like SetIPResponsePolicy.
// With no policy no finding is a summary notice, so every one stays
// immediate unless its source is handled.
func SetSummaryNoticePolicy(p func(Finding) bool) func(Finding) bool {
summaryNoticeMu.Lock()
defer summaryNoticeMu.Unlock()
previous := summaryNoticePolicy
summaryNoticePolicy = p
return previous
}
func currentSummaryNoticePolicy() func(Finding) bool {
summaryNoticeMu.RLock()
p := summaryNoticePolicy
summaryNoticeMu.RUnlock()
return p
}
// SetWebTrafficPolicy installs or clears the policy that names findings about
// requests on the web ports, and returns the previous one. A block of a
// Cloudflare-covered address does not answer them. The checks package
// registers it at package initialization, like SetIPResponsePolicy.
func SetWebTrafficPolicy(p func(Finding) bool) func(Finding) bool {
webTrafficMu.Lock()
defer webTrafficMu.Unlock()
previous := webTrafficPolicy
webTrafficPolicy = p
return previous
}
func currentWebTrafficPolicy() func(Finding) bool {
webTrafficMu.RLock()
p := webTrafficPolicy
webTrafficMu.RUnlock()
return p
}
package alert
import (
"crypto/sha256"
"encoding/hex"
"strconv"
"strings"
"time"
)
// Observation names where a finding was read: the producer that read it,
// the stream (a log file, or a journal), the position of the triggering
// record in that stream, and when it was observed. Admission mints evidence
// from it; the zero value means the producer has none.
type Observation struct {
Producer string `json:"producer"`
Stream string `json:"stream"`
Cursor string `json:"cursor"`
ObservedAt time.Time `json:"observed_at"`
}
// FileStream names a log file by device and inode. It depends on the file
// alone, so every reader of a file names it alike, across restarts.
func FileStream(dev, ino uint64) string {
return "f:" + strconv.FormatUint(dev, 16) + ":" + strconv.FormatUint(ino, 16)
}
// SocketStream separates boots. Inodes can be reused within a boot;
// rereads retain their first sighting rather than refreshing evidence.
func SocketStream(bootID string) string {
return "s:" + bootID
}
// RecordCursor names one record of a file stream by its start offset and its
// bytes, so a record rewritten at a reused offset is never an earlier one,
// while the same bytes read again keep their cursor. record is the line as
// read, with or without its line ending.
func RecordCursor(offset int64, record string) string {
record = strings.TrimRight(strings.TrimSuffix(record, "\n"), "\r")
sum := sha256.Sum256([]byte(record))
var buf [20 + 1 + 16]byte
b := strconv.AppendInt(buf[:0], offset, 10)
b = append(b, ':')
return string(hex.AppendEncode(b, sum[:8]))
}
package alert
import (
"bytes"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"time"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/queuehealth"
bolt "go.etcd.io/bbolt"
)
const (
phpanelQueueLimit = 100000
phpanelQuarantineLimit = 1000
// phpanelMaxRefusalRun bounds how many findings in a row the panel may
// refuse, with none accepted between them, before refusals stop
// archiving records. A panel refusing every request (a wrong URL or
// node, a proxy answering 400) looks like a run of bad records; past
// this run the queue keeps its findings and retries as for any other
// failure, so a misconfiguration cannot empty it.
phpanelMaxRefusalRun = 10
)
var phpanelQueueBucket = []byte("findings")
var phpanelQuarantineBucket = []byte("quarantine")
var phpanelDeliveryStateBucket = []byte("delivery_state")
var phpanelRefusalsKey = []byte("refusals")
var errPhpanelRefusalRun = errors.New("panel refusal run reached; keeping the queue")
type queuedPhpanelFinding struct {
Finding Finding `json:"finding"`
Timestamp time.Time `json:"timestamp"`
}
type quarantinedPhpanelFinding struct {
Payload []byte `json:"payload"`
Error string `json:"error"`
QuarantinedAt time.Time `json:"quarantined_at"`
}
type phpanelDeliveryConfig struct {
hostname string
url string
hmacSecret string
hmacSecretEnv string
}
type phpanelQueue struct {
db *bolt.DB
cfgMu sync.RWMutex
cfg phpanelDeliveryConfig
wake chan struct{}
stop chan struct{}
done chan struct{}
drain sync.Mutex
retryMu sync.Mutex
retryAt time.Time
retryDelay time.Duration
closed bool
mu sync.Mutex
mutation sync.Mutex
limit int
health phpanelQueueHealth
}
var phpanelQueues = struct {
sync.Mutex
byState map[string]*phpanelQueue
}{byState: make(map[string]*phpanelQueue)}
func enqueuePhpanelFindings(cfg *config.Config, findings []Finding) error {
queue, err := phpanelQueueForMode(cfg, false)
if err != nil {
return err
}
queued := make([]queuedPhpanelFinding, 0, len(findings))
for _, finding := range findings {
queued = append(queued, queuedPhpanelFinding{Finding: finding, Timestamp: time.Now().UTC()})
}
dropped, err := queue.enqueueBatch(queued)
if err != nil {
return fmt.Errorf("queueing phpanel webhook: %w", err)
}
select {
case queue.wake <- struct{}{}:
default:
}
if dropped > 0 {
alertDispatchFailures.Add(float64(dropped))
return fmt.Errorf("phpanel webhook queue reached %d entries and dropped %d oldest findings", phpanelQueueLimit, dropped)
}
return nil
}
// ConfigurePhpanelQueue opens and wakes the durable queue during daemon
// startup and safe config reloads. This lets persisted findings resume delivery
// without waiting for a new finding to arrive after a restart.
func ConfigurePhpanelQueue(cfg *config.Config) error {
if cfg == nil || !cfg.Alerts.Webhook.Enabled || cfg.Alerts.Webhook.Type != "phpanel" {
closePhpanelQueue(cfg)
return nil
}
queue, err := phpanelQueueFor(cfg)
if err != nil {
return err
}
select {
case queue.wake <- struct{}{}:
default:
}
return nil
}
func closePhpanelQueue(cfg *config.Config) {
if cfg == nil || cfg.StatePath == "" {
return
}
statePath, err := filepath.Abs(cfg.StatePath)
if err != nil {
return
}
phpanelQueues.Lock()
queue := phpanelQueues.byState[statePath]
delete(phpanelQueues.byState, statePath)
phpanelQueues.Unlock()
if queue != nil {
queue.close()
}
}
func phpanelQueueFor(cfg *config.Config) (*phpanelQueue, error) {
return phpanelQueueForMode(cfg, true)
}
func phpanelQueueForMode(cfg *config.Config, updateExisting bool) (*phpanelQueue, error) {
if cfg.StatePath == "" {
return nil, fmt.Errorf("phpanel webhook requires state_path for its durable queue")
}
statePath, err := filepath.Abs(cfg.StatePath)
if err != nil {
return nil, fmt.Errorf("resolving phpanel queue state path: %w", err)
}
phpanelQueues.Lock()
defer phpanelQueues.Unlock()
if queue := phpanelQueues.byState[statePath]; queue != nil {
if updateExisting {
queue.updateConfig(cfg)
}
return queue, nil
}
if mkdirErr := os.MkdirAll(statePath, 0o700); mkdirErr != nil {
return nil, fmt.Errorf("creating phpanel queue state directory: %w", mkdirErr)
}
db, err := bolt.Open(filepath.Join(statePath, "phpanel-webhook.db"), 0o600, &bolt.Options{Timeout: 2 * time.Second})
if err != nil {
return nil, fmt.Errorf("opening phpanel webhook queue: %w", err)
}
queue, err := newPhpanelQueue(db, phpanelQueueLimit)
if err != nil {
_ = db.Close()
return nil, fmt.Errorf("creating phpanel webhook queue: %w", err)
}
queue.updateConfig(cfg)
phpanelQueues.byState[statePath] = queue
phpanelHealth.Lock()
phpanelHealth.active[queue] = struct{}{}
phpanelHealth.Unlock()
obs.Go("phpanel-webhook-queue", queue.run)
return queue, nil
}
func newPhpanelQueue(db *bolt.DB, limit int) (*phpanelQueue, error) {
q := &phpanelQueue{
db: db, limit: limit,
wake: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}),
health: phpanelQueueHealth{
// Two normal drain intervals without completion warrant attention.
stats: queuehealth.NewSharedCapacity(limit, time.Minute), pending: make(map[string]*phpanelWork),
},
}
if err := db.Update(func(tx *bolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists(phpanelQueueBucket); err != nil {
return err
}
if _, err := tx.CreateBucketIfNotExists(phpanelQuarantineBucket); err != nil {
return err
}
_, err := tx.CreateBucketIfNotExists(phpanelDeliveryStateBucket)
return err
}); err != nil {
return nil, err
}
now := time.Now()
if err := db.View(func(tx *bolt.Tx) error {
return tx.Bucket(phpanelQueueBucket).ForEach(func(key, payload []byte) error {
q.health.pending[string(key)] = &phpanelWork{ticket: q.health.stats.BeginAt(phpanelRecordTime(payload, now), now)}
return nil
})
}); err != nil {
return nil, err
}
return q, nil
}
func phpanelRecordTime(payload []byte, now time.Time) time.Time {
var item queuedPhpanelFinding
if err := json.Unmarshal(payload, &item); err == nil && !item.Timestamp.IsZero() && item.Timestamp.Before(now) {
return item.Timestamp
}
// Damaged or future timestamps supply no reliable elapsed age.
// The record still enters normal delivery/quarantine processing.
return now
}
func (q *phpanelQueue) updateConfig(cfg *config.Config) {
q.cfgMu.Lock()
q.cfg = phpanelDeliveryConfig{
hostname: cfg.Hostname,
url: cfg.Alerts.Webhook.URL,
hmacSecret: cfg.Alerts.Webhook.HMACSecret,
hmacSecretEnv: cfg.Alerts.Webhook.HMACSecretEnv,
}
q.cfgMu.Unlock()
q.retryMu.Lock()
q.retryAt = time.Time{}
q.retryDelay = 0
q.retryMu.Unlock()
}
func (q *phpanelQueue) enqueueBatch(items []queuedPhpanelFinding) (int, error) {
if len(items) == 0 {
return 0, nil
}
now := time.Now()
work := make([]*phpanelWork, len(items))
for i, item := range items {
queuedAt := item.Timestamp
if queuedAt.IsZero() || queuedAt.After(now) {
queuedAt = now
}
work[i] = &phpanelWork{ticket: q.health.stats.BeginAt(queuedAt, now)}
work[i].ticket.Start(now)
}
committed := false
defer func() {
if !committed {
for _, entry := range work {
q.discardWork(entry, time.Now())
}
}
}()
bodies := make([][]byte, 0, len(items))
for _, item := range items {
body, err := json.Marshal(item)
if err != nil {
q.health.enqueueFailed.Store(true)
return 0, err
}
bodies = append(bodies, body)
}
q.mutation.Lock()
defer q.mutation.Unlock()
select {
case <-q.stop:
return 0, fmt.Errorf("phpanel webhook queue is stopped")
default:
}
var added, evicted []string
err := q.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(phpanelQueueBucket)
count := queuedFindingCount(bucket) + len(bodies)
for _, body := range bodies {
seq, err := bucket.NextSequence()
if err != nil {
return err
}
var key [8]byte
binary.BigEndian.PutUint64(key[:], seq)
if err := bucket.Put(key[:], body); err != nil {
return err
}
added = append(added, string(key[:]))
}
for count > q.limit {
oldest, _ := bucket.Cursor().First()
if oldest == nil {
// The live span is empty, so the count came from somewhere
// other than this bucket. Nothing is left to evict.
break
}
evictedKey := string(oldest)
if err := bucket.Delete(oldest); err != nil {
return err
}
evicted = append(evicted, evictedKey)
count--
}
return nil
})
if err != nil {
q.health.enqueueFailed.Store(true)
return 0, err
}
now = time.Now()
for i, key := range added {
work[i].ticket.Requeue(now)
q.health.pending[key] = work[i]
}
for _, key := range evicted {
entry := q.health.pending[key]
delete(q.health.pending, key)
switch entry {
case nil:
// A record evicted from the queue file with no accounting cannot
// be attributed to a caller; count the finding it carried as lost.
phpanelHealth.losses.Lose(now, 1)
case q.health.active:
entry.evicted = true
default:
q.discardWork(entry, now)
}
}
q.health.enqueueFailed.Store(false)
committed = true
return len(evicted), nil
}
// queuedFindingCount returns the number of live entries without walking every
// page. Deliveries, quarantine, and overflow trimming only ever remove the
// current oldest entry, so live keys stay a contiguous span of the monotonic
// sequence numbers assigned by NextSequence; the count is that span.
func queuedFindingCount(bucket *bolt.Bucket) int {
cursor := bucket.Cursor()
firstKey, _ := cursor.First()
if firstKey == nil {
return 0
}
lastKey, _ := cursor.Last()
// last >= first (bbolt key order) and the live span is bounded by
// phpanelQueueLimit, so the difference always fits in an int.
// #nosec G115 -- bounded span (<= phpanelQueueLimit); cannot overflow int.
return int(binary.BigEndian.Uint64(lastKey)-binary.BigEndian.Uint64(firstKey)) + 1
}
func (q *phpanelQueue) run() {
defer close(q.done)
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case <-q.wake:
q.drainQueued()
case <-ticker.C:
q.drainQueued()
case <-q.stop:
return
}
}
}
func (q *phpanelQueue) drainQueued() {
q.drain.Lock()
defer q.drain.Unlock()
q.retryMu.Lock()
if time.Now().Before(q.retryAt) {
q.retryMu.Unlock()
return
}
q.retryMu.Unlock()
for {
select {
case <-q.stop:
return
default:
}
key, payload, work, err := q.takeDelivery()
if err != nil {
fmt.Fprintf(os.Stderr, "alert: reading phpanel webhook queue: %v\n", err)
alertDispatchFailures.Inc()
return
}
if key == nil {
return
}
if !q.deliverQueued(key, payload, work) {
return
}
}
}
func (q *phpanelQueue) takeDelivery() ([]byte, []byte, *phpanelWork, error) {
q.mutation.Lock()
defer q.mutation.Unlock()
var key, payload []byte
err := q.db.View(func(tx *bolt.Tx) error {
firstKey, value := tx.Bucket(phpanelQueueBucket).Cursor().First()
if firstKey != nil {
key = append([]byte(nil), firstKey...)
payload = append([]byte(nil), value...)
}
return nil
})
q.health.readFailed.Store(err != nil)
if err != nil || key == nil {
return key, payload, nil, err
}
work := q.health.pending[string(key)]
if work == nil {
// A boundary record absent from the rebuilt map still needs delivery
// and must retain any reliable waiting age across retries.
now := time.Now()
work = &phpanelWork{ticket: q.health.stats.BeginAt(phpanelRecordTime(payload, now), now)}
q.health.pending[string(key)] = work
}
work.ticket.Start(time.Now())
q.health.active = work
return key, payload, work, nil
}
func (q *phpanelQueue) deliverQueued(key, payload []byte, work *phpanelWork) bool {
sent, removed, refused := false, false, false
attempted := false
defer func() {
// A refused record is a counted loss, not a failing panel.
if attempted && !sent && !refused {
q.health.sendFailed.Store(true)
}
q.finishDelivery(work, sent, removed)
}()
var item queuedPhpanelFinding
if err := json.Unmarshal(payload, &item); err != nil {
return q.archive(key, payload, err, false) == nil
}
q.cfgMu.RLock()
delivery := q.cfg
q.cfgMu.RUnlock()
attempted = true
err := sendQueuedPhpanelWebhookFinding(delivery, item)
if permanentPhpanelRejection(err) {
archiveErr := q.archive(key, payload, err, true)
if archiveErr == nil {
refused = true
q.health.sendFailed.Store(false)
q.clearRetryFailure()
return true
}
if !errors.Is(archiveErr, errPhpanelRefusalRun) {
return false
}
err = fmt.Errorf("%w: %w", err, archiveErr)
}
if err != nil {
fmt.Fprintf(os.Stderr, "alert: phpanel webhook delivery failed: %v\n", err)
alertDispatchFailures.Inc()
q.recordRetryFailure()
return false
}
sent = true
q.health.sendFailed.Store(false)
removed, err = q.removeDelivered(key)
if err != nil {
fmt.Fprintf(os.Stderr, "alert: deleting delivered phpanel webhook: %v\n", err)
alertDispatchFailures.Inc()
return false
}
q.clearRetryFailure()
return true
}
func (q *phpanelQueue) removeDelivered(key []byte) (bool, error) {
q.mutation.Lock()
defer q.mutation.Unlock()
err := q.db.Update(func(tx *bolt.Tx) error {
if err := tx.Bucket(phpanelQueueBucket).Delete(key); err != nil {
return err
}
// Only an accepted delivery ends the refusal run. Persist this with
// its acknowledgment so reloads and restarts cannot bypass a hold.
return tx.Bucket(phpanelDeliveryStateBucket).Delete(phpanelRefusalsKey)
})
q.health.removeFailed.Store(err != nil)
if err != nil {
return false, err
}
delete(q.health.pending, string(key))
return true, nil
}
// archive moves a record no retry can deliver into the bounded quarantine,
// counted as a lost finding, so the records behind it keep moving.
func (q *phpanelQueue) archive(key, payload []byte, cause error, refused bool) error {
what := "malformed"
if refused {
what = "rejected"
}
if err := q.quarantineRecord(key, payload, cause, phpanelQuarantineLimit, refused); err != nil {
if errors.Is(err, errPhpanelRefusalRun) {
return err
}
fmt.Fprintf(os.Stderr, "alert: quarantining %s phpanel webhook: %v\n", what, err)
alertDispatchFailures.Inc()
return err
}
fmt.Fprintf(os.Stderr, "alert: quarantined %s phpanel webhook: %v\n", what, cause)
alertDispatchFailures.Inc()
return nil
}
func (q *phpanelQueue) quarantine(key, payload []byte, cause error) error {
return q.quarantineWithLimit(key, payload, cause, phpanelQuarantineLimit)
}
func (q *phpanelQueue) quarantineWithLimit(key, payload []byte, cause error, limit int) error {
return q.quarantineRecord(key, payload, cause, limit, false)
}
func (q *phpanelQueue) quarantineRecord(key, payload []byte, cause error, limit int, refused bool) error {
if limit <= 0 {
return fmt.Errorf("phpanel quarantine limit must be positive")
}
record, marshalErr := json.Marshal(quarantinedPhpanelFinding{
Payload: payload,
Error: cause.Error(),
QuarantinedAt: time.Now().UTC(),
})
if marshalErr != nil {
return marshalErr
}
q.mutation.Lock()
defer q.mutation.Unlock()
removed := false
updateErr := q.db.Update(func(tx *bolt.Tx) error {
active := tx.Bucket(phpanelQueueBucket)
current := active.Get(key)
if current == nil {
return nil
}
if !bytes.Equal(current, payload) {
return fmt.Errorf("phpanel queue entry changed while being quarantined")
}
if refused {
state := tx.Bucket(phpanelDeliveryStateBucket)
var count uint64
if raw := state.Get(phpanelRefusalsKey); raw != nil {
if len(raw) != 8 {
return errors.New("invalid phpanel refusal count")
}
count = binary.BigEndian.Uint64(raw)
}
if count >= phpanelMaxRefusalRun {
return errPhpanelRefusalRun
}
var next [8]byte
binary.BigEndian.PutUint64(next[:], count+1)
if err := state.Put(phpanelRefusalsKey, next[:]); err != nil {
return err
}
}
quarantine := tx.Bucket(phpanelQuarantineBucket)
if quarantine.Get(key) == nil {
count := quarantine.Stats().KeyN
for count >= limit {
oldest, _ := quarantine.Cursor().First()
if oldest == nil {
break
}
if err := quarantine.Delete(oldest); err != nil {
return err
}
count--
}
}
if err := quarantine.Put(key, record); err != nil {
return err
}
if err := active.Delete(key); err != nil {
return err
}
removed = true
return nil
})
q.health.removeFailed.Store(updateErr != nil && !errors.Is(updateErr, errPhpanelRefusalRun))
if updateErr == nil && removed {
work := q.health.pending[string(key)]
delete(q.health.pending, string(key))
switch work {
case nil:
// Quarantined a record this process never accounted for.
phpanelHealth.losses.Lose(time.Now(), 1)
case q.health.active:
work.evicted = true
default:
q.discardWork(work, time.Now())
}
}
return updateErr
}
func (q *phpanelQueue) recordRetryFailure() {
q.retryMu.Lock()
defer q.retryMu.Unlock()
if q.retryDelay == 0 {
q.retryDelay = 30 * time.Second
} else {
q.retryDelay *= 2
if q.retryDelay > 15*time.Minute {
q.retryDelay = 15 * time.Minute
}
}
q.retryAt = time.Now().Add(q.retryDelay)
}
func (q *phpanelQueue) clearRetryFailure() {
q.retryMu.Lock()
q.retryAt = time.Time{}
q.retryDelay = 0
q.retryMu.Unlock()
}
func (q *phpanelQueue) close() {
q.mu.Lock()
if q.closed {
q.mu.Unlock()
return
}
q.closed = true
close(q.stop)
q.mu.Unlock()
<-q.done
q.drain.Lock()
q.mutation.Lock()
phpanelHealth.Lock()
delete(phpanelHealth.active, q)
phpanelHealth.Unlock()
for _, work := range q.health.pending {
// These findings remain durable for a later start.
work.ticket.Finish(time.Now())
}
clear(q.health.pending)
_ = q.db.Close()
q.mutation.Unlock()
q.drain.Unlock()
}
func closePhpanelQueuesForTest() {
ClosePhpanelQueues()
}
// ClosePhpanelQueues stops delivery workers and closes their durable databases.
func ClosePhpanelQueues() {
phpanelQueues.Lock()
queues := make([]*phpanelQueue, 0, len(phpanelQueues.byState))
for _, queue := range phpanelQueues.byState {
queues = append(queues, queue)
}
phpanelQueues.byState = make(map[string]*phpanelQueue)
phpanelQueues.Unlock()
for _, queue := range queues {
queue.close()
}
}
func phpanelQueueDepthForTest(statePath string) int {
absolute, _ := filepath.Abs(statePath)
phpanelQueues.Lock()
queue := phpanelQueues.byState[absolute]
phpanelQueues.Unlock()
if queue == nil {
return 0
}
depth := 0
_ = queue.db.View(func(tx *bolt.Tx) error {
depth = tx.Bucket(phpanelQueueBucket).Stats().KeyN
return nil
})
return depth
}
package alert
import (
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
type phpanelWork struct {
ticket queuehealth.Ticket
evicted bool
delivered bool
}
type phpanelQueueHealth struct {
stats *queuehealth.Tracker
pending map[string]*phpanelWork // guarded by phpanelQueue.mutation
active *phpanelWork
enqueueFailed atomic.Bool
readFailed atomic.Bool
removeFailed atomic.Bool
sendFailed atomic.Bool
}
// Factory locking spans database open. This registry only publishes memory
// snapshots, so status remains available during a stalled open, write or send.
var phpanelHealth = struct {
sync.RWMutex
active map[*phpanelQueue]struct{}
losses *queuehealth.Tracker
}{active: make(map[*phpanelQueue]struct{}), losses: queuehealth.New(0, time.Minute)}
// PhpanelQueueStatus includes cumulative loss from queues disabled or replaced
// during this process. It retains no retired queues or state-path labels.
func PhpanelQueueStatus(now time.Time) queuehealth.Status {
phpanelHealth.RLock()
queues := make([]*phpanelQueue, 0, len(phpanelHealth.active))
for q := range phpanelHealth.active {
queues = append(queues, q)
}
phpanelHealth.RUnlock()
status := phpanelHealth.losses.Snapshot(now)
for _, q := range queues {
local := q.queueStatus(now)
status.Depth += local.Depth
status.InFlight += local.InFlight
status.Capacity += local.Capacity
status.LagSeconds = max(status.LagSeconds, local.LagSeconds)
status.ProcessingSeconds = max(status.ProcessingSeconds, local.ProcessingSeconds)
if phpanelReasonRank(local.Reason) > phpanelReasonRank(status.Reason) {
status.Status, status.Reason = local.Status, local.Reason
}
}
return status
}
func phpanelReasonRank(reason string) int {
switch reason {
case "spool_io":
return 6
case "delivery_failed":
return 5
case "backlog_lag":
return 4
case "processing_lag":
return 3
case "queue_full":
return 2
case "dropped_work":
return 1
default:
return 0
}
}
func (q *phpanelQueue) QueueStatuses(now time.Time) map[string]queuehealth.Status {
return map[string]queuehealth.Status{"spool": q.queueStatus(now)}
}
func (q *phpanelQueue) queueStatus(now time.Time) queuehealth.Status {
status := q.health.stats.Snapshot(now)
switch {
case q.health.enqueueFailed.Load() || q.health.readFailed.Load() || q.health.removeFailed.Load():
status.Status, status.Reason = "degraded", "spool_io"
case q.health.sendFailed.Load():
status.Status, status.Reason = "degraded", "delivery_failed"
}
return status
}
func (q *phpanelQueue) discardWork(work *phpanelWork, now time.Time) {
if work.delivered {
work.ticket.Finish(now)
return
}
work.ticket.Reject(now)
phpanelHealth.losses.Lose(now, 1)
}
// The send owns an evicted record until its result is known. Evicting its
// durable copy is not a lost finding if the collector already received it.
func (q *phpanelQueue) finishDelivery(work *phpanelWork, sent, removed bool) {
q.mutation.Lock()
defer q.mutation.Unlock()
now := time.Now()
// An acknowledged send may remain queued after database removal fails.
// Later retries cannot undo the collector's earlier receipt.
work.delivered = work.delivered || sent
switch {
case removed:
work.ticket.Finish(now)
case work.evicted:
q.discardWork(work, now)
default:
work.ticket.Requeue(now)
}
q.health.active = nil
}
package alert
import (
"errors"
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
var (
ErrQueueTimeout = errors.New("finding queue deadline exceeded")
ErrQueueStopped = errors.New("finding queue stopped")
)
var ingestQueues = struct {
sync.RWMutex
queues map[chan<- Finding]*queuehealth.Tracker
}{queues: make(map[chan<- Finding]*queuehealth.Tracker)}
// RegisterQueue attaches accounting to a daemon's ingest channel for its
// lifetime. Standalone consumers have no daemon health snapshot and continue
// to use the same watcher APIs without a registered ingest channel.
func RegisterQueue(ch chan<- Finding, q *queuehealth.Tracker) func() {
ingestQueues.Lock()
ingestQueues.queues[ch] = q
ingestQueues.Unlock()
return func() {
ingestQueues.Lock()
defer ingestQueues.Unlock()
if ingestQueues.queues[ch] == q {
delete(ingestQueues.queues, ch)
}
}
}
func queuedFinding(ch chan<- Finding, f Finding) Finding {
f.queueTicket = queuehealth.Ticket{}
ingestQueues.RLock()
q := ingestQueues.queues[ch]
ingestQueues.RUnlock()
if q != nil {
f.queueTicket = q.Begin(time.Now())
}
return f
}
// TryEnqueue preserves the realtime producers' nonblocking contract while
// counting every lost finding in the shared ingest queue's health evidence.
func TryEnqueue(ch chan<- Finding, f Finding) bool {
f = queuedFinding(ch, f)
select {
case ch <- f:
return true
default:
f.queueTicket.Reject(time.Now())
return false
}
}
// Enqueue waits for capacity or shutdown. Callers that cannot block use
// TryEnqueue; bounded queues must never spawn a goroutine for each send.
func Enqueue(ch chan<- Finding, f Finding, stop <-chan struct{}) bool {
f = queuedFinding(ch, f)
select {
case ch <- f:
return true
case <-stop:
f.queueTicket.Reject(time.Now())
return false
}
}
// EnqueueWithin admits one finding with a bounded wait. The initial full
// channel is backpressure, not loss: its ticket is rejected only if the
// deadline or shutdown wins before a send succeeds.
func EnqueueWithin(ch chan<- Finding, f Finding, stop <-chan struct{}, timeout time.Duration) error {
f = queuedFinding(ch, f)
select {
case ch <- f:
return nil
default:
}
timer := time.NewTimer(timeout)
defer timer.Stop()
select {
case ch <- f:
return nil
case <-stop:
f.queueTicket.Reject(time.Now())
return ErrQueueStopped
case <-timer.C:
f.queueTicket.Reject(time.Now())
return ErrQueueTimeout
}
}
// RecordQueueLoss covers work abandoned with the rest of a batch before
// individual tickets were created.
func RecordQueueLoss(ch chan<- Finding, count uint64) {
ingestQueues.RLock()
q := ingestQueues.queues[ch]
ingestQueues.RUnlock()
if q != nil {
q.Lose(time.Now(), count)
}
}
func RejectQueued(f Finding) { f.queueTicket.Reject(time.Now()) }
func StartQueued(f Finding) { f.queueTicket.Start(time.Now()) }
// FinishQueued receives the original input batch before filtering or
// deduplication, so discarded duplicates also release their queue tickets.
func FinishQueued(findings []Finding) {
now := time.Now()
for _, f := range findings {
f.queueTicket.Finish(now)
}
}
package alert
import (
"path"
"strings"
)
const redactedToken = "[REDACTED]"
// RedactCommandLine masks credential-bearing arguments in a command line
// (or any text quoting one): the MySQL family's attached -pSECRET, KEY=VALUE
// tokens whose key names a password, token, secret or API key (on options
// and on environment assignments alike), separated --password VALUE forms,
// curl-style user:password pairs, and URL userinfo or query credentials.
// Whitespace is preserved so the rest of the line stays readable.
func RedactCommandLine(s string) string {
if s == "" {
return s
}
display, tokens := splitTokens(s)
if len(tokens) == 0 {
return display
}
mysqlAt := -1
sshpassAt := -1
for i, tok := range tokens {
base := strings.ToLower(path.Base(strings.Trim(tok.text, "\"'")))
if mysqlAt < 0 && mysqlFamily[base] {
mysqlAt = i
}
if sshpassAt < 0 && base == "sshpass" {
sshpassAt = i
}
}
changed := false
redactNext := false
redactNextPair := false
sshpassPending := false
for i := range tokens {
tok := &tokens[i]
text := tok.text
switch {
case redactNext:
redactNext = false
tok.text = redactedToken
changed = changed || tok.text != text
continue
case redactNextPair:
redactNextPair = false
if r, ok := redactPair(text); ok {
tok.text = r
changed = true
continue
}
}
if mysqlAt >= 0 && i > mysqlAt && len(text) > 2 && strings.HasPrefix(text, "-p") && text[2] != '-' {
tok.text = "-p" + redactedToken
changed = true
continue
}
if sshpassAt >= 0 && i == sshpassAt {
sshpassPending = true
}
if sshpassPending && i > sshpassAt && len(text) > 2 && strings.HasPrefix(text, "-p") && text[2] != '-' {
sshpassPending = false
tok.text = "-p" + redactedToken
changed = true
continue
}
if sshpassPending && i > sshpassAt && text == "-p" {
sshpassPending = false
redactNext = true
continue
}
if separatedSecretFlags[strings.ToLower(text)] {
redactNext = true
continue
}
if separatedPairFlags[strings.ToLower(text)] {
redactNextPair = true
continue
}
hasURL := strings.Contains(text, "://")
// An assignment can contain a URL as its value. An equals sign
// inside the URL itself must never make its authority a secret key.
eq := strings.IndexByte(text, '=')
if !hasURL || (eq >= 0 && eq < strings.Index(text, "://")) {
if r, ok := redactAssignments(text); ok {
tok.text = r
changed = true
continue
}
}
if tok.argv && strings.ContainsAny(text, " \t\n\r\v\f") {
if r := RedactCommandLine(text); r != text {
tok.text = r
changed = true
continue
}
}
if hasURL {
if r := redactURLToken(text); r != text {
tok.text = r
changed = true
}
continue
}
if r, ok := redactAssignments(text); ok {
tok.text = r
changed = true
}
}
if changed {
var b strings.Builder
b.Grow(len(display))
last := 0
for _, tok := range tokens {
b.WriteString(display[last:tok.start])
b.WriteString(tok.text)
last = tok.end
}
b.WriteString(display[last:])
display = b.String()
}
// Flattening argv changes how quotes group the displayed text. Redact
// that representation too, after protecting values with known argv
// boundaries. The display has no NULs, so this cannot recurse again.
if strings.IndexByte(s, 0) >= 0 {
return RedactCommandLine(display)
}
return display
}
type cmdToken struct {
start, end int
text string
argv bool
}
func splitTokens(s string) (string, []cmdToken) {
if strings.IndexByte(s, 0) >= 0 {
for strings.HasSuffix(s, "\x00") {
s = strings.TrimSuffix(s, "\x00")
}
display := strings.ReplaceAll(s, "\x00", " ")
var tokens []cmdToken
start := 0
for start <= len(s) {
end := strings.IndexByte(s[start:], 0)
if end < 0 {
end = len(s)
} else {
end += start
}
if end > start {
tokens = append(tokens, cmdToken{start: start, end: end, text: display[start:end], argv: true})
}
if end == len(s) {
break
}
start = end + 1
}
return display, tokens
}
var tokens []cmdToken
start := -1
var quote byte
escaped := false
for i := 0; i < len(s); i++ {
c := s[i]
if escaped {
escaped = false
continue
}
if c == '\\' {
escaped = true
if start < 0 {
start = i
}
continue
}
if quote != 0 {
if c == quote {
quote = 0
}
continue
}
if c == '\'' || c == '"' {
quote = c
if start < 0 {
start = i
}
continue
}
if c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f' {
if start >= 0 {
tokens = append(tokens, cmdToken{start: start, end: i, text: s[start:i]})
start = -1
}
continue
}
if start < 0 {
start = i
}
}
if start >= 0 {
tokens = append(tokens, cmdToken{start: start, end: len(s), text: s[start:]})
}
return s, tokens
}
var mysqlFamily = map[string]bool{
"mysql": true, "mysqldump": true, "mysqladmin": true, "mysqlimport": true,
"mysqlcheck": true, "mysqlshow": true, "mysqlbinlog": true, "mysqlpump": true,
"mysqlslap": true, "mariadb": true, "mariadb-dump": true, "mariadb-admin": true,
"mariadb-import": true, "mariadb-check": true, "mariadb-show": true,
"mariadb-binlog": true, "mydumper": true, "myloader": true,
}
// separatedSecretFlags take their secret as the following argument.
var separatedSecretFlags = map[string]bool{
"--password": true, "--passwd": true, "--pass": true, "--pw": true, "-pw": true,
"--token": true, "--secret": true, "--api-key": true, "--apikey": true,
"--access-token": true, "--auth-token": true, "--client-secret": true,
}
// separatedPairFlags take a user:password (or user%password) pair next.
var separatedPairFlags = map[string]bool{
"-u": true, "--user": true, "-U": true, "--username": true, "--credentials": true,
}
// redactAssignments applies the KEY=VALUE rules to a token, treating a
// form-encoded token (a=1&b=2) segment by segment so the other fields
// survive. An empty value is left alone.
func redactAssignments(text string) (string, bool) {
segments := strings.Split(text, "&")
changed := false
for i, seg := range segments {
eq := strings.IndexByte(seg, '=')
if eq <= 0 || eq == len(seg)-1 {
continue
}
key := strings.TrimLeft(seg[:eq], "-")
value := seg[eq+1:]
if value == redactedToken {
continue
}
if sensitiveKey(key) {
segments[i] = seg[:eq+1] + redactedToken
changed = true
continue
}
if pairKey(key) {
if r, ok := redactPair(value); ok {
segments[i] = seg[:eq+1] + r
changed = true
}
}
}
if !changed {
return text, false
}
return strings.Join(segments, "&"), true
}
func pairKey(key string) bool {
switch strings.ToLower(key) {
case "u", "user", "username", "credentials", "creds", "userpwd", "auth":
return true
}
return false
}
// redactPair masks the password half of user:password or user%password.
func redactPair(v string) (string, bool) {
sep := strings.IndexAny(v, ":%")
if sep <= 0 || sep == len(v)-1 {
return v, false
}
return v[:sep+1] + redactedToken, true
}
// sensitiveKey reports whether an option or variable name denotes a secret.
func sensitiveKey(key string) bool {
lower := strings.ToLower(key)
parts := strings.FieldsFunc(lower, func(r rune) bool { return r == '_' || r == '-' || r == '.' })
hasKey := false
hasKeyQualifier := false
for i, p := range parts {
switch {
case p == "password", strings.HasSuffix(p, "password"),
p == "passwd", strings.HasSuffix(p, "passwd"),
p == "pass" && i == len(parts)-1, p == "pwd", strings.HasSuffix(p, "pwd"),
p == "secret", strings.HasSuffix(p, "secret"),
p == "token", strings.HasSuffix(p, "token"),
p == "apikey", p == "credential", p == "credentials":
return true
case p == "key":
hasKey = true
case p == "api", p == "access", p == "private", p == "auth", p == "license", p == "signing":
hasKeyQualifier = true
}
}
return hasKey && hasKeyQualifier
}
// redactURLToken masks the password in userinfo and the values of sensitive
// query parameters inside a URL token.
func redactURLToken(tok string) string {
schemeEnd := strings.Index(tok, "://")
if schemeEnd < 0 {
return tok
}
rest := tok[schemeEnd+3:]
authorityEnd := strings.IndexAny(rest, "/?#")
if authorityEnd < 0 {
authorityEnd = len(rest)
}
authority := rest[:authorityEnd]
tail := rest[authorityEnd:]
if at := strings.LastIndexByte(authority, '@'); at > 0 {
userinfo := authority[:at]
if colon := strings.IndexByte(userinfo, ':'); colon >= 0 && colon < len(userinfo)-1 {
authority = userinfo[:colon+1] + redactedToken + authority[at:]
}
}
if q := strings.IndexByte(tail, '?'); q >= 0 {
query := tail[q+1:]
fragment := ""
if h := strings.IndexByte(query, '#'); h >= 0 {
fragment = query[h:]
query = query[:h]
}
params := strings.Split(query, "&")
for i, p := range params {
eq := strings.IndexByte(p, '=')
if eq <= 0 || eq == len(p)-1 {
continue
}
if sensitiveKey(p[:eq]) {
params[i] = p[:eq+1] + redactedToken
}
}
tail = tail[:q+1] + strings.Join(params, "&") + fragment
}
return tok[:schemeEnd+3] + authority + tail
}
package alert
import "github.com/pidginhost/csm/internal/config"
// EmitForTest drives emitAudit with no audit sinks so callers (often in
// other packages' tests) can trigger observer fan-out without setting up
// jsonl/syslog. Lives in a non-test file because Go tests cannot import
// _test.go symbols across packages.
func EmitForTest(f Finding) {
emitAudit(&config.Config{Hostname: "test"}, []Finding{f})
}
package alert
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"time"
"github.com/pidginhost/csm/internal/config"
)
// webhookTransport is the shared *http.Transport reused across every
// webhook dispatch. Reusing one transport keeps the underlying TCP /
// TLS connections in its keepalive pool so hosts that fire hundreds
// of webhooks per hour avoid a new handshake per alert. Per-call
// timeouts stay configurable via httpClient: each call wraps the
// shared transport in a fresh *http.Client carrying the requested
// timeout. http.DefaultTransport already configures sensible
// defaults; reuse it directly rather than instantiating a separate
// pool that would shadow Go's HTTP/2 / proxy plumbing.
var webhookTransport http.RoundTripper = http.DefaultTransport
const maxWebhookResponseDrainBytes int64 = 512 << 10
const maxWebhookResponseDrainDuration = 250 * time.Millisecond
// httpClient returns a webhook client with the requested timeout
// backed by the shared transport, so the keepalive pool is shared
// across dispatches without losing per-call timeout configurability.
func httpClient(timeout time.Duration) *http.Client {
return &http.Client{Timeout: timeout, Transport: webhookTransport}
}
// SetWebhookTransportForTest lets tests inject a fake RoundTripper.
// Not safe for concurrent calls; tests should set up before parallel
// dispatch and restore after.
func SetWebhookTransportForTest(rt http.RoundTripper) (restore func()) {
prev := webhookTransport
webhookTransport = rt
return func() { webhookTransport = prev }
}
func closeWebhookResponseBody(resp *http.Response) {
if resp == nil || resp.Body == nil {
return
}
if resp.Close || resp.ContentLength == 0 || resp.ContentLength > maxWebhookResponseDrainBytes {
_ = resp.Body.Close()
return
}
done := make(chan struct{})
go func() {
// Read one sentinel byte past the reuse limit so an exactly-at-limit
// response still reaches the underlying EOF and can be pooled.
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxWebhookResponseDrainBytes+1))
close(done)
}()
select {
case <-done:
case <-time.After(maxWebhookResponseDrainDuration):
// A slow or streaming response is not worth holding alert dispatch
// open just to preserve a keepalive connection.
}
_ = resp.Body.Close()
}
func SendWebhook(cfg *config.Config, subject, body string) error {
url := cfg.Alerts.Webhook.URL
if url == "" {
return fmt.Errorf("no webhook URL configured")
}
var payload []byte
var err error
switch cfg.Alerts.Webhook.Type {
case "slack":
payload, err = json.Marshal(map[string]string{
"text": fmt.Sprintf("*%s*\n```\n%s\n```", subject, body),
})
case "discord":
payload, err = json.Marshal(map[string]string{
"content": fmt.Sprintf("**%s**\n```\n%s\n```", subject, body),
})
default:
payload, err = json.Marshal(map[string]string{
"subject": subject,
"body": body,
})
}
if err != nil {
return fmt.Errorf("marshaling webhook payload: %w", err)
}
client := httpClient(10 * time.Second)
resp, err := client.Post(url, "application/json", bytes.NewReader(payload))
if err != nil {
return fmt.Errorf("webhook POST: %w", err)
}
defer closeWebhookResponseBody(resp)
if resp.StatusCode >= 400 {
return fmt.Errorf("webhook returned %d", resp.StatusCode)
}
return nil
}
// SendWebhookJSON posts a pre-built JSON payload to the configured webhook
// URL. Senders that need a structured body (not the slack/discord subject
// envelope) use this. No-op when no URL is configured.
func SendWebhookJSON(cfg *config.Config, payload any) error {
url := cfg.Alerts.Webhook.URL
if url == "" {
return nil
}
body, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshaling webhook payload: %w", err)
}
client := httpClient(10 * time.Second)
resp, err := client.Post(url, "application/json", bytes.NewReader(body))
if err != nil {
return fmt.Errorf("webhook POST: %w", err)
}
defer closeWebhookResponseBody(resp)
if resp.StatusCode >= 400 {
return fmt.Errorf("webhook returned %d", resp.StatusCode)
}
return nil
}
// SendPhpanelWebhookFinding posts a single finding to the configured phpanel
// endpoint, signing the body with HMAC-SHA256 in X-CSM-Signature. Stateless;
// caller is responsible for filtering / batching.
func SendPhpanelWebhookFinding(cfg *config.Config, f Finding) error {
delivery := phpanelDeliveryConfig{
hostname: cfg.Hostname,
url: cfg.Alerts.Webhook.URL,
hmacSecret: cfg.Alerts.Webhook.HMACSecret,
hmacSecretEnv: cfg.Alerts.Webhook.HMACSecretEnv,
}
return sendQueuedPhpanelWebhookFinding(delivery, queuedPhpanelFinding{Finding: f, Timestamp: time.Now().UTC()})
}
func sendQueuedPhpanelWebhookFinding(delivery phpanelDeliveryConfig, queued queuedPhpanelFinding) error {
cfg := &config.Config{Hostname: delivery.hostname}
cfg.Alerts.Webhook.URL = delivery.url
cfg.Alerts.Webhook.HMACSecret = delivery.hmacSecret
cfg.Alerts.Webhook.HMACSecretEnv = delivery.hmacSecretEnv
if cfg.Alerts.Webhook.URL == "" {
return fmt.Errorf("phpanel webhook URL not set")
}
secret := phpanelWebhookSecret(cfg)
if secret == "" {
return fmt.Errorf("phpanel webhook HMAC secret not configured")
}
payload := map[string]interface{}{
"hostname": cfg.Hostname,
"timestamp": queued.Timestamp.Format(time.RFC3339),
"finding": queued.Finding,
}
body, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshaling phpanel payload: %w", err)
}
mac := hmac.New(sha256.New, []byte(secret))
mac.Write(body)
sig := "sha256=" + hex.EncodeToString(mac.Sum(nil))
req, err := http.NewRequest(http.MethodPost, cfg.Alerts.Webhook.URL, bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-CSM-Signature", sig)
req.Header.Set("X-CSM-Hostname", cfg.Hostname)
req.Header.Set("User-Agent", "csm")
client := httpClient(10 * time.Second)
// A redirect may turn this POST into a login-page GET. Only the
// configured collector can acknowledge receipt of a queued finding.
client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse }
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("phpanel webhook POST: %w", err)
}
defer closeWebhookResponseBody(resp)
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return &phpanelHTTPError{status: resp.StatusCode}
}
return nil
}
// phpanelHTTPError is a response that does not acknowledge receipt.
type phpanelHTTPError struct {
status int
}
func (e *phpanelHTTPError) Error() string {
return fmt.Sprintf("phpanel webhook HTTP %d", e.status)
}
// permanentPhpanelRejection reports whether the panel refused the record
// itself as malformed (400), too large (413) or unprocessable (422). Any other
// failure, including auth, rate limits and server errors, can clear on retry.
func permanentPhpanelRejection(err error) bool {
var httpErr *phpanelHTTPError
if !errors.As(err, &httpErr) {
return false
}
switch httpErr.status {
case http.StatusBadRequest, http.StatusRequestEntityTooLarge, http.StatusUnprocessableEntity:
return true
}
return false
}
func phpanelWebhookSecret(cfg *config.Config) string {
if cfg.Alerts.Webhook.HMACSecretEnv != "" {
if v := os.Getenv(cfg.Alerts.Webhook.HMACSecretEnv); v != "" {
return v
}
}
return cfg.Alerts.Webhook.HMACSecret
}
package antispam
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"slices"
"strings"
)
// CodeRspamdConfig reports rspamd's loaded settings defeating CSM's files.
const CodeRspamdConfig = "rspamd_config"
// Action is what reconcile does next.
type Action int
const (
// Keep changes nothing: the host matches the configuration, or a
// problem prevents a safe change.
Keep Action = iota
// Install applies CSM's integration for the configured settings.
Install
// Uninstall removes CSM's integration.
Uninstall
)
// Assessment is reconcile's view of one host.
type Assessment struct {
Action Action
// Installed is true when any CSM file, block or record is on the host.
Installed bool
// Problems are reported whatever the action.
Problems []Problem
}
// Assess decides what reconcile does. For a disabled integration f needs
// only Stack, Targets, GeneratedConf and Record, as hostfacts.CollectInstalled
// reads them; enabled, f is a full collection.
func Assess(enabled bool, s Settings, f HostFacts) Assessment {
a := Assessment{Installed: leftovers(f)}
if !enabled {
if !a.Installed {
return a
}
a.Problems = CheckRemoval(f.Stack, f.Targets)
if !blocking(a.Problems) {
a.Action = Uninstall
}
return a
}
a.Problems = CheckHost(f)
a.report(checkActionsTarget(s, f))
a.report(checkDryRunRejects(s, f))
a.report(CheckQuietRejects(s, f))
isCurrent := a.Installed && current(s, f)
if isCurrent {
// A temporary host probe failure cannot hide a confirmed override.
a.Problems = append(a.Problems, loadedConflicts(s, f)...)
}
switch {
case hasSeverity(a.Problems, Definite):
a.uninstall(f)
case hasSeverity(a.Problems, Temporary):
return a
case !isCurrent:
// Replacing a shared block must be just as safe as removing it.
if removal := CheckRemoval(f.Stack, f.Targets); blocking(removal) {
a.report(removal)
return a
}
a.Action = Install
}
return a
}
// uninstall removes an installation when CSM can edit every file safely,
// and otherwise reports why it cannot.
func (a *Assessment) uninstall(f HostFacts) {
if !a.Installed {
return
}
removal := CheckRemoval(f.Stack, f.Targets)
if blocking(removal) {
a.report(removal)
return
}
a.Action = Uninstall
}
// report adds the problems not already reported: CheckHost, CheckRemoval
// and checkActionsTarget share the path rules.
func (a *Assessment) report(problems []Problem) {
for _, p := range problems {
if !slices.Contains(a.Problems, p) {
a.Problems = append(a.Problems, p)
}
}
}
// leftovers reports whether any part of CSM's integration is on the host.
// An unreadable record counts: only a remove can settle it.
func leftovers(f HostFacts) bool {
if !f.Record.Missing || hasMarker(f.GeneratedConf) {
return true
}
for _, t := range f.Targets {
if t.Kind.WholeFile() && ManagedFor(t.Kind, t.Content) || !t.Kind.WholeFile() && hasMarker(t.Content) {
return true
}
}
return false
}
func hasMarker(s string) bool {
return strings.Contains(s, MarkerBegin) || strings.Contains(s, MarkerEnd)
}
// checkActionsTarget judges a Postfix host's actions.conf while CSM
// enforces: CSM then owns it like a core file, and the worker refuses one it
// cannot replace safely. As for a core file, a check that could not run is
// temporary, so a passing read error never removes an installation, and
// unsafe metadata collected before it stays definite. A dry run leaves the
// file to the host.
func checkActionsTarget(s Settings, f HostFacts) []Problem {
if f.Stack != StackPostfix || s.HostActions() {
return nil
}
for _, t := range f.Targets {
if t.Path != ActionsPath {
continue
}
err := OptionalUsable(t)
if err == nil {
return nil
}
sev := Definite
if !errors.Is(t.Err, fs.ErrNotExist) && (t.Err != nil || len(t.Parents) == 0) {
sev = Temporary
// Metadata collected before the error still counts: a symlink,
// loose mode, foreign owner or loose parent stays definite. A
// file whose lstat failed has zero facts, which pass.
collected := t
collected.Err = nil
if unsafe := OptionalUsable(collected); unsafe != nil && len(t.Parents) > 0 {
err, sev = unsafe, Definite
}
}
return []Problem{{Code: CodeTargets, Severity: sev, Detail: err.Error()}}
}
return nil
}
// checkDryRunRejects warns that a Postfix dry run still rejects mail: the
// host's own actions apply to CSM's scores, and a reject threshold at or
// below CSM's turns away what CSM would. rspamd's built-in actions, loaded
// without an actions.conf, reject at 15. CSM's own actions.conf, which this
// pass restores or removes, says nothing about the host's.
func checkDryRunRejects(s Settings, f HostFacts) []Problem {
if f.Stack != StackPostfix || !s.DryRun || f.RspamdActions.Err != nil {
return nil
}
for _, t := range f.Targets {
if t.Path == ActionsPath && !t.Missing && ManagedFor(t.Kind, t.Content) {
return nil
}
}
a, err := ParseActions([]byte(f.RspamdActions.Stdout))
if err != nil || a.Reject == nil || *a.Reject > float64(s.RejectScore) {
return nil
}
return []Problem{{Code: CodeDryRunRejects, Severity: Warning, Detail: fmt.Sprintf(
"dry run on this host still rejects: rspamd rejects at %g, at or below CSM's reject score of %d", *a.Reject, s.RejectScore)}}
}
// quietRejectsFix tells the operator how to make rspamd refuse again.
const quietRejectsFix = "set discard_on_reject and quarantine_on_reject to false, once each, in the proxy worker settings, for example /etc/rspamd/local.d/worker-proxy.inc"
// CheckQuietRejects reports a Postfix host whose rspamd milter accepts the
// mail it rejects, so the sender is never refused: discard_on_reject drops
// it, and quarantine_on_reject leaves it in Postfix's hold queue until
// someone releases or deletes it. CSM's rejects must be refusals, so CSM
// does not enforce there. In a dry run the host's own rejects go the same
// way, which is a warning. Unwired or unreadable settings are CheckHost's.
func CheckQuietRejects(s Settings, f HostFacts) []Problem {
if f.Stack != StackPostfix {
return nil
}
set, err := QuietRejects(f)
if err != nil || len(set) == 0 {
return nil
}
what := "puts the mail it rejects in Postfix's hold queue"
if set[0] == "discard_on_reject" {
what = "drops the mail it rejects"
}
sev := Definite
if s.DryRun {
sev = Warning
}
return []Problem{{Code: CodeMilterRejects, Severity: sev, Detail: fmt.Sprintf(
"rspamd's milter %s instead of refusing it (%s); %s", what, strings.Join(set, ", "), quietRejectsFix)}}
}
// current reports whether the host holds exactly s: CSM's files, a
// completed record for them, and on cPanel the hooks active in the
// generated configuration.
func current(s Settings, f HostFacts) bool {
managed, err := ManagedContent(s)
if err != nil {
return false
}
want := Generation(managed)
if Generation(InstalledContent(f.Targets)) != want || f.Record.Err != nil {
return false
}
r, err := DecodeRecord([]byte(f.Record.Content))
if err != nil || !r.Complete() || *r.Generation != want {
return false
}
for _, p := range s.SecretTargets() {
if !secretFileOK(f, p) {
return false
}
}
if f.Stack == StackPostfix {
return true
}
hooks, err := RenderExim(s)
if err != nil || f.GeneratedErr != nil || f.EximSettings.Err != nil {
return false
}
opts := ParseEximOptions(f.EximSettings.Stdout)
return CheckGenerated(f.GeneratedConf, hooks, opts["acl_smtp_rcpt"], opts["acl_smtp_data"]) == nil
}
// loadedConflicts compares the settings rspamd loaded with CSM's files,
// as the apply transaction does after staging them.
func loadedConflicts(s Settings, f HostFacts) []Problem {
var out []Problem
var conflicts []string
unreadable := false
modules, err := EnabledModules(f.RspamdModules.Stdout)
if f.RspamdModules.Err != nil || err != nil {
unreadable = true
} else {
conflicts = append(conflicts, checkRspamdModules(modules, s)...)
if len(s.DNSFeeds()) > 0 {
if rules, err := LoadedRBLRules([]byte(f.RspamdRBL.Stdout)); f.RspamdRBL.Err != nil || err != nil {
out = append(out, Problem{Code: CodeFeed, Severity: Warning, Key: "rbl",
Detail: "cannot read the blocklist rules rspamd loaded; check rspamadm configdump"})
} else {
for _, c := range CheckFeedsLoaded(modules, rules, s) {
// Feed drift never removes core.
out = append(out, Problem{Code: CodeFeed, Severity: Warning, Key: "rbl", Detail: c})
}
}
}
if s.HasFeed(FeedCSMIPMap) {
var section []byte
if f.RspamdConfig.Err == nil {
if root, err := dumpObjectAs[json.RawMessage]([]byte(f.RspamdConfig.Stdout), "config"); err == nil {
section = root["multimap"]
}
}
for _, c := range CheckIPMapLoaded(modules, section) {
out = append(out, Problem{Code: CodeFeed, Severity: Warning, Key: FeedCSMIPMap, Detail: c})
}
}
}
data := []byte(f.RspamdActions.Stdout)
switch actions, err := ParseActions(data); {
case s.HostActions():
// The host's own actions keep enforcing, whatever their form.
case f.RspamdActions.Err != nil || !json.Valid(data):
unreadable = true
case err != nil:
out = append(out, Problem{Code: CodeRspamdConfig, Severity: Definite,
Detail: "rspamd actions use a form CSM cannot compare; set them only through CSM's local.d/actions.conf"})
default:
conflicts = append(conflicts, checkRspamdActions(actions, s)...)
}
data = []byte(f.RspamdPhishing.Stdout)
if f.RspamdPhishing.Err != nil || !json.Valid(data) {
unreadable = true
} else if on, err := PhishtankEnabled(data); err != nil {
// Lua treats strings, numbers and tables as true, so an override
// cannot be left active as though collection had merely failed.
out = append(out, Problem{Code: CodeRspamdConfig, Severity: Definite,
Detail: "rspamd phishtank setting is missing or not a single boolean; remove overrides of CSM's phishing.conf"})
} else if on {
conflicts = append(conflicts, "rspamd phishtank lookups are enabled; an override defeats CSM's phishing.conf")
}
for _, c := range conflicts {
out = append(out, Problem{Code: CodeRspamdConfig, Severity: Definite, Detail: c})
}
if unreadable {
out = append(out, Problem{Code: CodeRspamdConfig, Severity: Temporary,
Detail: "cannot read the settings rspamd loaded; check rspamadm configdump"})
}
return out
}
func hasSeverity(problems []Problem, sev Severity) bool {
for _, p := range problems {
if p.Severity == sev {
return true
}
}
return false
}
// blocking reports a problem that stops a change.
func blocking(problems []Problem) bool {
return hasSeverity(problems, Definite) || hasSeverity(problems, Temporary)
}
package antispam
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/fs"
"math"
"net/netip"
"path/filepath"
"strconv"
"strings"
)
// Bound allocations before decoding the full dump, including unrelated sections.
const maxControllerDumpBytes = 16 << 20
// ControllerListener is one bind_socket entry of the loaded controller
// worker. Raw is the entry as rspamd printed it, used as the finding key.
type ControllerListener struct {
Raw string
Unix bool
Path string
Mode fs.FileMode
HostPort string
}
// ControllerConfig is the loaded controller worker. Password values never
// leave ParseControllerConfig; only whether each one is set.
type ControllerConfig struct {
Present bool
Listeners []ControllerListener
TrustEntries []string
PasswordSet bool
EnablePasswordSet bool
// PasswordEmpty and EnablePasswordEmpty mark a password rspamd reads
// as an empty string: present, but no secret.
PasswordEmpty bool
EnablePasswordEmpty bool
}
// ControllerUnverifiableProblem reports a loaded controller configuration
// CSM read but could not understand. A form rspamd accepts but CSM cannot
// parse must not read as a safe controller.
func ControllerUnverifiableProblem() Problem {
return Problem{Code: CodeControllerOpen, Severity: Warning, Key: "controller", Detail: exposureUnverifiable}
}
// ParseControllerConfig reads the controller worker from the full
// `rspamadm configdump -j` output. Errors name the field, never a value.
func ParseControllerConfig(fullDump []byte) (ControllerConfig, error) {
var cfg ControllerConfig
if len(fullDump) > maxControllerDumpBytes {
return cfg, errors.New("configdump worker controller: dump is too large")
}
root, err := dumpObjectAs[json.RawMessage](fullDump, "worker")
if err != nil {
return cfg, err
}
section, ok := root["worker"]
if !ok {
return cfg, nil
}
controller, err := controllerSection(section)
if err != nil || controller == nil {
return cfg, err
}
fields, err := dumpObjectAs[json.RawMessage](controller, "worker controller")
if err != nil {
return cfg, err
}
// rspamd skips a worker with enabled = false or count <= 0.
if disabled, workerErr := disabledWorker(fields); workerErr != nil || disabled {
return cfg, workerErr
}
cfg.Present = true
binds, err := workerBinds(fields)
if err != nil {
return ControllerConfig{}, err
}
for _, b := range binds {
l, bindErr := parseControllerBind(b)
if bindErr != nil {
return ControllerConfig{}, bindErr
}
cfg.Listeners = append(cfg.Listeners, l)
}
// trusted_ips is an alias that writes the same setting; rspamd keeps
// whichever it parses last, so both count.
for _, key := range []string{"secure_ip", "trusted_ips"} {
entries, listErr := stringOrList(fields, key)
if listErr != nil {
return ControllerConfig{}, listErr
}
cfg.TrustEntries = append(cfg.TrustEntries, entries...)
}
if cfg.PasswordSet, cfg.PasswordEmpty, err = passwordState(fields, "password"); err != nil {
return ControllerConfig{}, err
}
if cfg.EnablePasswordSet, cfg.EnablePasswordEmpty, err = passwordState(fields, "enable_password"); err != nil {
return ControllerConfig{}, err
}
return cfg, nil
}
// workerBinds returns a worker's bind addresses. rspamd looks them up as
// bind_socket, then listen, then bind.
func workerBinds(fields map[string]json.RawMessage) ([]string, error) {
for _, key := range []string{"bind_socket", "listen", "bind"} {
if _, ok := fields[key]; !ok {
continue
}
binds, err := stringOrList(fields, key)
if err != nil || len(binds) > 0 {
return binds, err
}
break
}
return nil, errors.New("configdump worker controller: bind_socket is missing")
}
// workerBool reads a boolean worker field, or returns value when it is
// absent.
func workerBool(fields map[string]json.RawMessage, name string, value bool) (bool, error) {
raw, ok := fields[name]
if !ok {
return value, nil
}
var flag *bool
if json.Unmarshal(raw, &flag) == nil && flag != nil {
return *flag, nil
}
// Rspamd also accepts integers in its boolean fields.
var n *int32
if json.Unmarshal(raw, &n) != nil || n == nil {
return false, fmt.Errorf("configdump worker controller: %s is not a boolean or supported integer", name)
}
return *n != 0, nil
}
func disabledWorker(fields map[string]json.RawMessage) (bool, error) {
enabled, err := workerBool(fields, "enabled", true)
if err != nil {
return false, err
}
count := 1.0
if raw, ok := fields["count"]; ok {
var value *float64
if json.Unmarshal(raw, &value) != nil || value == nil {
return false, errors.New("configdump worker controller: count is not a number")
}
// Rspamd truncates fractions and stores count in a signed short.
// Values outside that range cannot prove a worker is disabled.
count = math.Trunc(*value)
if count < math.MinInt16 || count > math.MaxInt16 {
return false, errors.New("configdump worker controller: count is outside the supported range")
}
}
return !enabled || count <= 0, nil
}
// controllerSection returns the controller object from the worker section,
// which rspamd prints as a list of single-key objects (4.2) or as an object
// keyed by worker type.
func controllerSection(section json.RawMessage) (json.RawMessage, error) {
var entries []json.RawMessage
switch trimmed := bytes.TrimSpace(section); {
case len(trimmed) > 0 && trimmed[0] == '[':
if err := json.Unmarshal(trimmed, &entries); err != nil {
return nil, errors.New("configdump worker: invalid JSON")
}
default:
entries = []json.RawMessage{section}
}
var found json.RawMessage
for _, entry := range entries {
c, err := workerOfKind(entry, "controller")
if err != nil {
return nil, err
}
if c == nil {
continue
}
if found != nil {
return nil, errors.New("configdump worker: more than one controller worker")
}
found = c
}
return found, nil
}
// workerOfKind returns the settings of entry when it is a worker of the
// given type, else nil. rspamd prints a worker with a type attribute or as
// a single object keyed by its type.
func workerOfKind(entry json.RawMessage, kind string) (json.RawMessage, error) {
workers, err := dumpObjectAs[json.RawMessage](entry, "worker")
if err != nil {
return nil, err
}
raw, typed := workers["type"]
if !typed {
return workers[kind], nil
}
// A type attribute makes this the worker itself, even if it also has a
// field named after a worker type.
var k string
if json.Unmarshal(raw, &k) != nil || k == "" || strings.ContainsRune(k, '\x00') {
return nil, errors.New("configdump worker: type is not a non-empty string without NUL bytes")
}
if k != kind {
return nil, nil
}
return entry, nil
}
func stringOrList(fields map[string]json.RawMessage, name string) ([]string, error) {
raw, ok := fields[name]
if !ok {
return nil, nil
}
var one *string
if err := json.Unmarshal(raw, &one); err == nil && one != nil {
return []string{*one}, nil
}
var many []*string
if err := json.Unmarshal(raw, &many); err != nil || many == nil {
return nil, fmt.Errorf("configdump worker controller: %s is not a string or a list of strings", name)
}
values := make([]string, 0, len(many))
for _, value := range many {
if value == nil {
return nil, fmt.Errorf("configdump worker controller: %s contains a non-string entry", name)
}
values = append(values, *value)
}
return values, nil
}
// passwordState follows rspamd's reading: null is unset, a number or a
// boolean is a set password, and an empty string is present but empty.
func passwordState(fields map[string]json.RawMessage, name string) (set, empty bool, err error) {
raw, ok := fields[name]
if !ok {
return false, false, nil
}
var value any
if json.Unmarshal(raw, &value) != nil {
return false, false, fmt.Errorf("configdump worker controller: %s is not readable", name)
}
switch v := value.(type) {
case nil:
return false, false, nil
case string:
// Rspamd copies the value as a NUL-terminated C string.
empty = v == "" || v[0] == 0
return !empty, empty, nil
case float64, bool:
return true, false, nil
}
return false, false, fmt.Errorf("configdump worker controller: %s is not a string", name)
}
func parseControllerBind(raw string) (ControllerListener, error) {
l := ControllerListener{Raw: raw}
for _, c := range raw {
if c < ' ' || c == 0x7f {
return ControllerListener{}, errors.New("configdump worker controller: bind_socket contains a control character")
}
}
addr := strings.Trim(raw, " ")
// TLS does not change the listener address or the controller's auth policy.
if len(addr) > 4 && strings.EqualFold(addr[len(addr)-4:], " ssl") {
addr = addr[:len(addr)-4]
}
if addr == "" {
return l, errors.New("configdump worker controller: empty bind_socket entry")
}
if strings.HasPrefix(addr, "systemd:") {
// The dump does not say whether the inherited descriptor is TCP or Unix.
return ControllerListener{}, errors.New("configdump worker controller: bind_socket systemd endpoint is unresolved")
}
if !strings.HasPrefix(addr, "/") {
if !controllerTCPAddress(addr) {
return l, errors.New("configdump worker controller: bind_socket entry is neither an absolute Unix path nor host:port")
}
l.HostPort = addr
return l, nil
}
// Rspamd separates Unix options with spaces or commas. Cleaning the path
// would change its target when a component before ".." is a symlink.
parts := strings.FieldsFunc(addr, func(r rune) bool { return r == ' ' || r == ',' })
if len(parts[0]) > maxSocketPath {
return ControllerListener{}, errors.New("configdump worker controller: bind_socket Unix path is too long")
}
l.Unix, l.Path = true, parts[0]
for _, opt := range parts[1:] {
name, value, ok := strings.Cut(opt, "=")
if !ok || !strings.EqualFold(name, "mode") {
continue
}
// Rspamd uses strtoul with base 0: only a leading zero means octal.
base := 10
if strings.HasPrefix(value, "0x") || strings.HasPrefix(value, "0X") {
base, value = 16, value[2:]
} else if strings.HasPrefix(value, "0") {
base = 8
}
mode, err := strconv.ParseUint(value, base, 32)
if err != nil || mode == 0 || mode > 0o7777 {
return ControllerListener{}, errors.New("configdump worker controller: bind_socket mode is not a valid file mode")
}
l.Mode = fs.FileMode(mode & 0o777)
if mode&0o4000 != 0 {
l.Mode |= fs.ModeSetuid
}
if mode&0o2000 != 0 {
l.Mode |= fs.ModeSetgid
}
if mode&0o1000 != 0 {
l.Mode |= fs.ModeSticky
}
}
return l, nil
}
// ControllerHostFacts is what exposure needs beyond the loaded worker:
// host addresses a trust entry could name, the facts of each Unix
// controller socket, and the members of each group that could own one.
type ControllerHostFacts struct {
// Addrs retains each assigned interface address, not its masked subnet.
Addrs []netip.Prefix
Sockets map[string]SocketFacts
// GroupMembers includes primary and supplementary members. Omit a group
// when either inventory is unavailable; an empty list means no members.
GroupMembers map[int][]string
RspamdUID int
RspamdGID int
RspamdUser string
}
// Exposure reasons. They are fixed strings so a finding never carries
// configuration values.
const (
exposureTrusted = "trusted address reachable by local processes"
exposureUnresolved = "unresolved trust entry"
exposureNoPasswords = "no passwords configured"
exposureReadOnly = "read-only access without password"
exposureSocket = "socket reachable by other users"
exposureSocketParent = "socket parent writable by other users"
exposureSocketUnknown = "socket facts unavailable"
exposureEmptyPassword = "empty password configured"
exposureUnverifiable = "controller configuration CSM cannot verify"
)
// Local processes reach these through loopback, whatever the bind address.
var localPrefixes = []netip.Prefix{
netip.MustParsePrefix("127.0.0.0/8"),
netip.MustParsePrefix("::1/128"),
netip.MustParsePrefix("::ffff:127.0.0.0/104"),
}
// ControllerExposure returns one Warning per controller listener that a
// tenant process can use. Every TCP listener is reachable from the host, so
// trust entries and passwords decide; a Unix listener grants admin to any
// process that can connect, so its file access decides.
func ControllerExposure(c ControllerConfig, f ControllerHostFacts) []Problem {
if !c.Present {
return nil
}
var out []Problem
for _, l := range c.Listeners {
var reason string
if l.Unix {
reason = unixExposure(l, f)
} else {
reason = tcpExposure(c, f)
}
if reason != "" {
out = append(out, Problem{Code: CodeControllerOpen, Severity: Warning, Key: l.Raw, Detail: reason})
}
}
return out
}
func tcpExposure(c ControllerConfig, f ControllerHostFacts) string {
unresolved := false
for _, entry := range c.TrustEntries {
prefix, ok := trustPrefix(entry)
if !ok {
unresolved = true
continue
}
if reachable(prefix, f.Addrs) {
return exposureTrusted
}
}
switch {
case !c.PasswordSet && !c.EnablePasswordSet && !c.PasswordEmpty && !c.EnablePasswordEmpty:
return exposureNoPasswords
case c.PasswordEmpty || c.EnablePasswordEmpty:
// rspamd compares an empty configured password by length; CSM cannot
// show that an empty client password is refused.
return exposureEmptyPassword
case unresolved:
return exposureUnresolved
case !c.PasswordSet:
return exposureReadOnly
}
return ""
}
// trustPrefix parses a secure_ip entry as an address or CIDR. Maps,
// hostnames and file paths do not parse: CSM cannot see what they hold.
func trustPrefix(entry string) (netip.Prefix, bool) {
if p, err := netip.ParsePrefix(entry); err == nil {
return p.Masked(), true
}
// PrefixFrom discards zones, including arbitrary text after '%'. Such
// an entry is not an unambiguous address in a secure_ip network map.
if a, err := netip.ParseAddr(entry); err == nil && a.Zone() == "" {
return netip.PrefixFrom(a, a.BitLen()), true
}
return netip.Prefix{}, false
}
// reachable compares IPv4 addresses in both spellings: netip never matches
// IPv4 against IPv4-mapped IPv6, and a trust entry may use either.
func reachable(p netip.Prefix, hostAddrs []netip.Prefix) bool {
for _, local := range localPrefixes {
if p.Overlaps(local) {
return true
}
if local.Addr().Is4() && p.Overlaps(netip.PrefixFrom(netip.AddrFrom16(local.Addr().As16()), local.Bits()+96)) {
return true
}
}
for _, host := range hostAddrs {
// A local process can use the assigned address, not every neighbor
// on its subnet. netip treats IPv4 and mapped IPv6 as separate
// families, but both spellings identify the same local endpoint.
a := host.Addr().Unmap()
if p.Contains(a) || a.Is4() && p.Contains(netip.AddrFrom16(a.As16())) {
return true
}
}
return false
}
func unixExposure(l ControllerListener, f ControllerHostFacts) string {
s, ok := f.Sockets[l.Path]
switch {
case !ok || s.Err != nil:
return exposureSocketUnknown
case s.Missing:
return ""
}
if s.Socket.Mode.Type() != fs.ModeSocket || len(s.Parents) == 0 {
return exposureSocketUnknown
}
trusted := func(uid int) bool { return uid == 0 || uid == f.RspamdUID }
var privateDirs []string
parents:
for _, p := range s.Parents {
if !p.Mode.IsDir() {
return exposureSocketUnknown
}
// A private detour still hides its own descendants, even when it
// does not hide the socket at the end of the resolved path.
for _, dir := range privateDirs {
if ancestorOf(dir, p.Path) {
continue parents
}
}
if !trusted(p.UID) || p.ExtACL {
return exposureSocketParent
}
if !f.othersPermit(p, 0o1) {
// Only a real ancestor hides the socket. A directory met on a
// symlink detour does not: the real path stays usable.
if ancestorOf(p.Path, s.Socket.Path) {
return ""
}
privateDirs = append(privateDirs, p.Path)
continue
}
if f.othersPermit(p, 0o3) {
if p.Mode&fs.ModeSticky == 0 {
return exposureSocketParent
}
// Sticky directories protect trusted children from replacement.
// Parents omits symlink ownership, so a redirected walk cannot
// prove that every child looked up here has a trusted owner.
if !directSocketPath(l, s) {
return exposureSocketUnknown
}
// The next directory or socket owner is checked below.
}
}
sock := s.Socket
if !trusted(sock.UID) || sock.ExtACL || f.othersPermit(sock, 0o2) {
return exposureSocket
}
return ""
}
func ancestorOf(dir, p string) bool {
for parent := filepath.Dir(p); ; parent = filepath.Dir(parent) {
if parent == dir {
return true
}
if parent == "/" || parent == "." {
return false
}
}
}
func directSocketPath(l ControllerListener, s SocketFacts) bool {
if l.Path != s.Socket.Path {
return false
}
parent := filepath.Dir(s.Socket.Path)
for i := len(s.Parents) - 1; i >= 0; i-- {
if s.Parents[i].Path != parent {
return false
}
parent = filepath.Dir(parent)
}
return s.Parents[0].Path == "/"
}
// othersPermit requires all requested bits in the same permission class:
// group write and other search do not combine into directory access.
func (f ControllerHostFacts) othersPermit(p PathFacts, want fs.FileMode) bool {
perm := p.Mode.Perm()
return perm&want == want || perm>>3&want == want && f.groupHasOthers(p.GID)
}
// groupHasOthers reports whether a group could hold a tenant: any member
// other than root and the rspamd user, or members CSM could not read.
func (f ControllerHostFacts) groupHasOthers(gid int) bool {
members, ok := f.GroupMembers[gid]
if !ok {
return true
}
for _, m := range members {
if m != "root" && m != f.RspamdUser {
return true
}
}
return false
}
// Validate syntax without resolving hostnames or opening sockets. Rspamd permits
// omitted ports and an optional numeric priority after an explicit port.
func controllerTCPAddress(addr string) bool {
var host, suffix string
if strings.HasPrefix(addr, "[") {
var ok bool
host, suffix, ok = strings.Cut(addr[1:], "]")
ip, err := netip.ParseAddr(host)
if !ok || err != nil || !ip.Is6() {
return false
}
} else {
host, suffix, _ = strings.Cut(addr, ":")
if len(host) != len(addr) {
suffix = ":" + suffix
}
switch strings.ToLower(host) {
case "*", "*v4", "*v6":
default:
if host == "" || strings.HasPrefix(host, ".") {
return false
}
for _, c := range host {
switch {
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', c == '-', c == '_', c == '.':
default:
return false
}
}
}
}
if suffix == "" {
return true
}
if !strings.HasPrefix(suffix, ":") {
return false
}
port, priority, hasPriority := strings.Cut(suffix[1:], ":")
if _, err := strconv.ParseUint(port, 10, 16); err != nil {
return false
}
if hasPriority {
if _, err := strconv.ParseUint(priority, 10, 32); err != nil {
return false
}
}
return true
}
// Package dovecot lists the users of a Dovecot 2.3 host and the Maildir++
// folders of their \Junk mailboxes, checking directory aliases on disk.
// It imports none of the learning packages that consume it.
package dovecot
import (
"bytes"
"cmp"
"context"
"encoding/base64"
"encoding/binary"
"errors"
"fmt"
"io/fs"
"os"
"os/exec"
"path/filepath"
"slices"
"strconv"
"strings"
"syscall"
"time"
"unicode"
"unicode/utf16"
"unicode/utf8"
)
const (
// MaxMailboxes bounds the users one listing may name.
MaxMailboxes = 10000
// MaxListBytes bounds the output of every command.
MaxListBytes = 4 << 20
// commandTimeout bounds each command.
commandTimeout = 10 * time.Second
// batchSize users are looked up per doveadm call.
batchSize = 100
)
// Skip reasons. A reason is a fixed string and never carries command output.
const (
SkipInvalidName = "invalid_name"
SkipLookupFailed = "lookup_failed"
SkipUIDRoot = "uid_root"
SkipPluginZlib = "plugin_zlib"
SkipPluginMailCrypt = "plugin_mail_crypt"
SkipDriver = "driver"
SkipLayoutFS = "layout_fs"
SkipLocation = "location"
SkipInvalidHome = "invalid_home"
SkipNoJunk = "no_junk"
// SkipSharedPath: the mail directories are, hold or lie inside another
// listed user's.
SkipSharedPath = "shared_path"
)
var (
// ErrCannotIterate reports a configuration with no userdb that can list
// its users.
ErrCannotIterate = errors.New("the Dovecot userdb cannot list users")
// ErrDovecot24 reports Dovecot 2.4, whose mail settings differ.
ErrDovecot24 = errors.New("dovecot 2.4 mail settings are not supported yet")
// errListFailed is Dovecot's answer to any failed listing: a userdb that
// cannot iterate, or an SQL, LDAP or passwd-file backend that failed to.
errListFailed = errors.New("the Dovecot userdb did not list its users")
)
// DovecotMailbox is one user whose \Junk folders can be watched. User is the
// userdb name in lower case and Domain its domain, empty for a domainless
// system user. Home is empty when the userdb has none. Junk holds the
// absolute Maildir++ folder of every \Junk mailbox, sorted.
type DovecotMailbox struct {
User, Domain string
UID, GID int
Home string
Junk []string
}
// Skip is a listed user that is not watched.
type Skip struct {
User, Reason string
// UID is retained after a successful lookup, even when mail cannot be watched.
UID int
}
// Snapshot is one discovery pass. Complete is set only when the listing,
// every identity and a second listing that matches the first were checked.
// A user confirmed absent is skipped; temporary lookup failures abort the pass.
// Skipped users are present, not removed.
type Snapshot struct {
Mailboxes []DovecotMailbox
Skips []Skip
Complete bool
}
// Dovecot discovers mailboxes by running dovecot, doveconf and doveadm
// through Run; a nil Run executes them.
type Dovecot struct {
Run func(ctx context.Context, name string, args ...string) ([]byte, error)
}
// Snapshot lists every user and resolves its \Junk folders. Any failure that
// leaves the listing in doubt returns an error and an empty Snapshot. A user
// the listing names but doveadm confirms absent is skipped, not dropped.
func (d Dovecot) Snapshot(ctx context.Context) (Snapshot, error) {
if err := d.version(ctx); err != nil {
return Snapshot{}, err
}
out, err := d.cmd(ctx, "doveconf", "-n")
if err != nil {
return Snapshot{}, err
}
conf, err := parseConfig(out)
if err != nil {
return Snapshot{}, err
}
listed, err := d.users(ctx)
if errors.Is(err, errListFailed) && !conf.iterable() {
return Snapshot{}, ErrCannotIterate
}
if err != nil {
return Snapshot{}, err
}
var snap Snapshot
var lookup []string
seen := map[string]bool{}
for _, name := range listed {
user, _, ok := canonical(name)
switch {
case !ok || seen[user]:
return Snapshot{}, errors.New("the Dovecot user list has unsupported or ambiguous identities")
default:
seen[user] = true
lookup = append(lookup, name)
}
}
lookupConf := conf
conditional := conf.conditional()
if conditional {
// Read identities without expanding a conditional that could suppress
// a record. Skipped mailboxes still determine service UID isolation.
lookupConf.location, lookupConf.locs = "", nil
}
var watch []DovecotMailbox
paths := map[string][]string{}
unknownPaths := map[int]bool{}
for batch := range slices.Chunk(lookup, batchSize) {
found, derr := d.details(ctx, lookupConf, batch)
if derr != nil {
return Snapshot{}, derr
}
for _, name := range batch {
user, domain, _ := canonical(name)
det, ok := found[name]
if !ok {
snap.Skips = append(snap.Skips, Skip{User: user, Reason: SkipLookupFailed})
continue
}
if conditional {
snap.Skips = append(snap.Skips, Skip{User: user, Reason: SkipLocation, UID: det.uid})
continue
}
dirs, complete := conf.paths(det)
if !complete {
unknownPaths[det.uid] = true
}
paths[user], err = resolveMailPaths(dirs)
if err != nil {
return Snapshot{}, err
}
m, reason := conf.mailbox(user, domain, det)
if reason != "" {
snap.Skips = append(snap.Skips, Skip{User: user, Reason: reason, UID: det.uid})
continue
}
watch = append(watch, m)
}
}
shared := sharedPaths(paths)
for _, m := range watch {
if unknownPaths[m.UID] {
snap.Skips = append(snap.Skips, Skip{User: m.User, Reason: SkipLocation, UID: m.UID})
continue
}
if shared[m.User] {
snap.Skips = append(snap.Skips, Skip{User: m.User, Reason: SkipSharedPath, UID: m.UID})
continue
}
snap.Mailboxes = append(snap.Mailboxes, m)
}
final, err := d.users(ctx)
if err != nil {
return Snapshot{}, err
}
if !slices.Equal(listed, final) {
return Snapshot{}, errors.New("the Dovecot user list changed during discovery")
}
slices.SortFunc(snap.Mailboxes, func(a, b DovecotMailbox) int { return strings.Compare(a.User, b.User) })
slices.SortFunc(snap.Skips, func(a, b Skip) int { return strings.Compare(a.User, b.User) })
snap.Complete = true
return snap, nil
}
// cmd runs one command within commandTimeout and MaxListBytes. On a non-zero
// exit it returns the output with the error.
func (d Dovecot) cmd(ctx context.Context, name string, args ...string) ([]byte, error) {
run := d.Run
if run == nil {
run = runCommand
}
ctx, cancel := context.WithTimeout(ctx, commandTimeout)
defer cancel()
out, err := run(ctx, name, args...)
switch {
case ctx.Err() != nil:
return nil, fmt.Errorf("%s: %w", name, ctx.Err())
case len(out) > MaxListBytes:
return nil, fmt.Errorf("%s: %w", name, errTooLarge)
case err != nil:
return out, fmt.Errorf("%s: %w", name, err)
}
return out, nil
}
// version accepts Dovecot 2.3 only: 2.4 renamed the mail settings read here.
func (d Dovecot) version(ctx context.Context) error {
out, err := d.cmd(ctx, "dovecot", "--version")
if err != nil {
return err
}
v, _, _ := strings.Cut(strings.TrimSpace(string(out)), " ")
major, rest, _ := strings.Cut(v, ".")
minor, _, _ := strings.Cut(rest, ".")
maj, err1 := strconv.Atoi(major)
low, err2 := strconv.Atoi(minor)
switch {
case err1 != nil || err2 != nil:
return errors.New("unrecognised Dovecot version")
case maj == 2 && low == 3:
return nil
case maj > 2 || maj == 2 && low > 3:
return ErrDovecot24
}
return errors.New("dovecot before 2.3 is not supported")
}
// Users is the identity-only listing: the names Snapshot would look up, in
// lower case, sorted and distinct. It runs one doveadm command and looks no
// user up, so a caller can recheck a removal without another Snapshot.
func (d Dovecot) Users(ctx context.Context) ([]string, error) {
listed, err := d.users(ctx)
if err != nil {
return nil, err
}
var out []string
seen := map[string]bool{}
for _, name := range listed {
user, _, ok := canonical(name)
if !ok || seen[user] {
return nil, errors.New("the Dovecot user list has unsupported or ambiguous identities")
}
seen[user] = true
out = append(out, user)
}
slices.Sort(out)
return slices.Compact(out), nil
}
// users returns the sorted distinct names the userdb lists.
func (d Dovecot) users(ctx context.Context) ([]string, error) {
out, err := d.cmd(ctx, "doveadm", "user", "*")
var exit *exec.ExitError
if errors.As(err, &exit) && bytes.Contains(exit.Stderr, []byte("User listing returned failure")) {
return nil, errListFailed
}
if err != nil {
return nil, err
}
return parseUsers(out)
}
// doveadm user goes on past a user it cannot look up, prints no record for
// it and exits EX_NOUSER (unknown user) or EX_TEMPFAIL (lookup failed).
const (
exNoUser = 67
exTempFail = 75
)
// details accepts a successful batch only with one record per user in order.
// EX_NOUSER can mask an earlier temporary failure, so every requested user
// is rechecked alone on that exit. Unescaped userdb fields can also forge
// another user's batch record; none is trusted without its own lookup.
func (d Dovecot) details(ctx context.Context, conf config, batch []string) (map[string]detail, error) {
out, err := d.lookup(ctx, conf, batch)
var exit *exec.ExitError
partial := errors.As(err, &exit) && exit.ExitCode() == exNoUser
if err != nil && !partial {
return nil, err
}
recs, err := parseDetails(out, batch, len(conf.locs))
if err != nil {
return nil, err
}
if !partial {
return inOrder(recs, batch)
}
found := make(map[string]detail, len(batch))
for _, name := range batch {
out, lookupErr := d.lookup(ctx, conf, []string{name})
if lookupErr != nil && (!errors.As(lookupErr, &exit) || exit.ExitCode() != exNoUser) {
return nil, lookupErr
}
one, err := parseDetails(out, []string{name}, len(conf.locs))
if err != nil {
return nil, err
}
if lookupErr != nil {
if len(one) != 0 || slices.ContainsFunc(recs, func(r named) bool { return r.name == name }) {
return nil, lookupErr
}
continue
}
confirmed, err := inOrder(one, []string{name})
if err != nil {
return nil, err
}
found[name] = confirmed[name]
}
return found, nil
}
// lookup runs doveadm user for names. They come after "--" so none is read
// as an option, and none contains a mask character.
func (d Dovecot) lookup(ctx context.Context, conf config, names []string) ([]byte, error) {
return d.cmd(ctx, "doveadm", append([]string{"user", "-e", conf.template(), "--"}, names...)...)
}
// inOrder accepts the records of a lookup that exited 0: exactly one per
// requested user, in request order.
func inOrder(recs []named, names []string) (map[string]detail, error) {
if !slices.EqualFunc(recs, names, func(r named, name string) bool { return r.name == name }) {
return nil, errors.New("doveadm user did not print one record per user in order")
}
found := make(map[string]detail, len(recs))
for _, r := range recs {
found[r.name] = r.detail
}
return found, nil
}
// canonical validates a listed name and returns it in lower case with its
// domain. doveadm reads names holding * or ? as masks.
func canonical(name string) (user, domain string, ok bool) {
if strings.ContainsAny(name, "*?") {
return "", "", false
}
local, domain, found := strings.Cut(name, "@")
if !ValidMailbox(local) || found && !ValidDomain(domain) {
return "", "", false
}
return strings.ToLower(name), strings.ToLower(domain), true
}
var errTooLarge = errors.New("output exceeds the bound")
// detail is one user's record: its effective uid, gid and home, the userdb
// mail and mail_plugins fields, and the configured locations expanded for
// that user.
type detail struct {
uid, gid int
home, mail, plugins string
locs []string
}
// junkBox is a \Junk mailbox of a private namespace; loc 0 is the user's
// mail location, n the namespace location config.locs[n-1].
type junkBox struct {
name, sep string
loc int
}
// config is what Snapshot reads from doveconf -n: the global mail_location,
// mail_plugins of every protocol, the \Junk mailboxes with the locations of
// every private namespace that sets its own location, and the userdbs.
type config struct {
location string
plugins []string
locs []string
junk []junkBox
userdbs []userdb
}
type userdb struct{ driver, args string }
// iterable reports whether a userdb can list its users. In Dovecot 2.3
// static, checkpassword and prefetch cannot, nor can passwd-file with a
// %variable in its path. Any other driver counts as able, so its failure
// is an outage that keeps learning, not a reason to remove it.
func (c config) iterable() bool {
return slices.ContainsFunc(c.userdbs, func(u userdb) bool {
switch u.driver {
case "static", "checkpassword", "prefetch":
return false
case "passwd-file":
// The only option a userdb passwd-file takes comes first.
path := u.args
if strings.HasPrefix(path, "username_format=") {
_, path, _ = strings.Cut(path, " ")
}
return !hasVariable(path)
}
return true
})
}
// hasVariable reports a % that starts a variable; %% is a literal %.
func hasVariable(s string) bool {
for i := 0; i+1 < len(s); i++ {
if s[i] == '%' {
if s[i+1] != '%' {
return true
}
i++
}
}
return false
}
// conditional reports a location in the lookup template that uses %{if}.
// doveadm prints nothing, and still exits 0, for a user whose %{if} fails
// to expand, so such a lookup cannot vouch for one record per user.
func (c config) conditional() bool {
return slices.ContainsFunc(append([]string{c.location}, c.locs...), usesIf)
}
// usesIf matches %{if} as Dovecot does: in any case and after modifiers
// such as %L.
func usesIf(loc string) bool {
return strings.Contains(strings.ToLower(loc), "{if")
}
// template has doveadm print, per user, one tab-separated line: %u, uid,
// gid, home, the userdb mail and mail_plugins fields, every location
// expanded for that user, then %u again. doveadm escapes nothing, and %u is
// the name after lookup, which a userdb may rename. So a line counts as a
// record only with the exact field count, no control byte, and a first and
// last field equal to a requested name, as listed or in lower case. Even
// then a newline inside one user's fields can make lines that pass for other
// users' records; details trusts records only from a lookup that exited 0
// and printed exactly one per user in request order. Any extra line or tab
// breaks that count.
func (c config) template() string {
f := append([]string{"%u", "%{uid}", "%{gid}", "%h", "%{userdb:mail}", "%{userdb:mail_plugins}", c.location}, c.locs...)
return strings.Join(append(f, "%u"), "\t")
}
func (c config) mailbox(user, domain string, d detail) (DovecotMailbox, string) {
plugins := append(strings.Fields(d.plugins), c.plugins...)
switch {
case d.uid == 0:
return DovecotMailbox{}, SkipUIDRoot
case slices.Contains(plugins, "zlib"):
return DovecotMailbox{}, SkipPluginZlib
case slices.Contains(plugins, "mail_crypt") || slices.Contains(plugins, "mail_crypt_acl"):
return DovecotMailbox{}, SkipPluginMailCrypt
case d.home != "" && !absClean(d.home):
return DovecotMailbox{}, SkipInvalidHome
case len(c.junk) == 0:
return DovecotMailbox{}, SkipNoJunk
}
var junk []string
for _, j := range c.junk {
loc := cmp.Or(d.mail, d.locs[0])
if j.loc > 0 {
loc = d.locs[j.loc]
}
root, utf8Names, reason := parseLocation(loc, d.home)
if reason != "" {
return DovecotMailbox{}, reason
}
dir, ok := folderDir(j.name, j.sep, utf8Names)
if !ok {
return DovecotMailbox{}, SkipInvalidName
}
junk = append(junk, filepath.Join(root, dir))
}
slices.Sort(junk)
home := d.home
if home != "" {
home = filepath.Clean(home)
}
return DovecotMailbox{User: user, Domain: domain, UID: d.uid, GID: d.gid, Home: home, Junk: slices.Compact(junk)}, ""
}
// paths are the directories a user's mail lives in, as far as CSM can tell:
// the root of the location its INBOX and each private namespace use, and any
// INBOX override. An unresolved path prevents other users of the same mail
// UID from borrowing it undetected.
func (c config) paths(d detail) ([]string, bool) {
var out []string
complete := true
for _, loc := range append([]string{cmp.Or(d.mail, d.locs[0])}, d.locs[1:]...) {
_, rest, hasDriver := strings.Cut(loc, ":")
if !hasDriver {
rest = loc
}
params := strings.Split(rest, ":")
dirs := []string{params[0]}
for _, p := range params[1:] {
if inbox, ok := strings.CutPrefix(p, "INBOX="); ok {
dirs = append(dirs, inbox)
}
}
for _, dir := range dirs {
if dir == "~" || strings.HasPrefix(dir, "~/") {
if !absClean(d.home) {
complete = false
continue
}
dir = d.home + dir[1:]
}
if absClean(dir) && !strings.Contains(dir, "%") {
out = append(out, filepath.Clean(dir))
} else {
complete = false
}
}
}
return out, complete
}
// Aliased directories belong to the same mailbox even when only one path
// can be opened without following links. Nonexistent roots hold no mail;
// other lookup errors leave the snapshot unable to prove isolation.
func resolveMailPaths(paths []string) ([]string, error) {
out := slices.Clone(paths)
for _, path := range paths {
resolved, err := filepath.EvalSymlinks(path)
switch {
case err == nil:
if resolved != path {
out = append(out, resolved)
}
case errors.Is(err, fs.ErrNotExist), errors.Is(err, syscall.ENOTDIR):
default:
return nil, errors.New("a Dovecot mail path could not be checked")
}
}
return out, nil
}
// sharedPaths returns the users with a mail directory that is, holds or
// lies inside one of another user's. On a shared mail uid file owners
// cannot tell such users apart.
func sharedPaths(paths map[string][]string) map[string]bool {
owners := map[string][]string{}
for user, dirs := range paths {
for _, dir := range dirs {
if !slices.Contains(owners[dir], user) {
owners[dir] = append(owners[dir], user)
}
}
}
shared := map[string]bool{}
for user, dirs := range paths {
for _, dir := range dirs {
for up := dir; ; up = filepath.Dir(up) {
for _, other := range owners[up] {
if other != user {
shared[user], shared[other] = true, true
}
}
if up == "/" {
break
}
}
}
}
return shared
}
// parseUsers reads one name per line, at most MaxMailboxes lines.
func parseUsers(out []byte) ([]string, error) {
var names []string
for line := range strings.SplitSeq(string(out), "\n") {
if line != "" {
names = append(names, line)
}
}
switch {
case len(names) == 0:
return nil, errors.New("doveadm listed no users")
case len(names) > MaxMailboxes:
return nil, fmt.Errorf("doveadm listed more than %d users", MaxMailboxes)
}
slices.Sort(names)
return slices.Compact(names), nil
}
// named is a record under the requested name it matched.
type named struct {
name string
detail
}
// parseDetails returns the records in output order. Every non-empty line
// must be a record, as template describes, of a distinct requested user;
// anything else fails the batch.
func parseDetails(out []byte, batch []string, nloc int) ([]named, error) {
errFraming := errors.New("doveadm user printed a malformed record")
want := make(map[string]string, 2*len(batch))
for _, name := range batch {
want[name] = name
want[strings.ToLower(name)] = name
}
var recs []named
seen := map[string]bool{}
for line := range strings.SplitSeq(string(out), "\n") {
if line == "" {
continue
}
f := strings.Split(line, "\t")
if len(f) != 8+nloc || f[len(f)-1] != f[0] || strings.ContainsFunc(line, func(r rune) bool { return r != '\t' && control(r) }) {
return nil, errFraming
}
name, ok := want[f[0]]
uid, err1 := strconv.ParseUint(f[1], 10, 32)
gid, err2 := strconv.ParseUint(f[2], 10, 32)
if !ok || seen[name] || err1 != nil || err2 != nil {
return nil, errFraming
}
seen[name] = true
recs = append(recs, named{name, detail{uid: int(uid), gid: int(gid), home: f[3], mail: f[4], plugins: f[5], locs: f[6 : len(f)-1]}})
}
return recs, nil
}
// parseConfig reads doveconf -n output: settings as "key = value", sections
// as "kind [name] {" ... "}". Values and names may be double-quoted.
func parseConfig(out []byte) (config, error) {
errSyntax := errors.New("doveconf -n printed a malformed configuration")
var c config
var stack []string
var nsType, nsSep, nsLoc string
var boxes []string
var box string
var junk bool
for line := range strings.SplitSeq(string(out), "\n") {
line = strings.TrimSpace(line)
inNS := len(stack) > 0 && stack[0] == "namespace"
switch {
case line == "" || line[0] == '#':
case line == "}":
if len(stack) == 0 {
return config{}, errSyntax
}
top := stack[len(stack)-1]
stack = stack[:len(stack)-1]
switch {
case len(stack) == 1 && inNS && top == "mailbox" && junk:
boxes = append(boxes, box)
case len(stack) == 0 && inNS && (nsType == "" || nsType == "private"):
c.addNamespace(boxes, cmp.Or(nsSep, "."), nsLoc)
}
case strings.HasSuffix(line, "{"):
kind, name, _ := strings.Cut(strings.TrimSpace(strings.TrimSuffix(line, "{")), " ")
stack = append(stack, kind)
switch {
case len(stack) == 1 && kind == "namespace":
nsType, nsSep, nsLoc, boxes = "", "", "", nil
case len(stack) == 2 && inNS && kind == "mailbox":
box, junk = unquote(strings.TrimSpace(name)), false
case kind == "userdb":
c.userdbs = append(c.userdbs, userdb{})
}
default:
key, value, ok := strings.Cut(line, "=")
if !ok {
return config{}, errSyntax
}
key, value = strings.TrimSpace(key), unquote(strings.TrimSpace(value))
inUserdb := len(stack) > 0 && stack[len(stack)-1] == "userdb"
switch {
case inUserdb && key == "driver":
c.userdbs[len(c.userdbs)-1].driver = value
case inUserdb && key == "args":
c.userdbs[len(c.userdbs)-1].args = value
case key == "mail_plugins":
c.plugins = append(c.plugins, strings.Fields(value)...)
case len(stack) == 0 && key == "mail_location":
c.location = value
case len(stack) == 1 && inNS && key == "type":
nsType = value
case len(stack) == 1 && inNS && key == "separator":
nsSep = value
case len(stack) == 1 && inNS && key == "location":
nsLoc = value
case len(stack) == 2 && inNS && stack[1] == "mailbox" && key == "name":
box = value
case len(stack) == 2 && inNS && stack[1] == "mailbox" && key == "special_use":
junk = slices.ContainsFunc(strings.Fields(value), func(f string) bool { return strings.EqualFold(f, `\Junk`) })
}
}
}
if len(stack) != 0 {
return config{}, errSyntax
}
return c, nil
}
func (c *config) addNamespace(boxes []string, sep, location string) {
loc := 0
if location != "" {
c.locs = append(c.locs, location)
loc = len(c.locs)
}
for _, b := range boxes {
c.junk = append(c.junk, junkBox{name: b, sep: sep, loc: loc})
}
}
// unquote strips the quotes doveconf puts around names and values with
// spaces. doveconf prints backslashes as they are.
func unquote(s string) string {
if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
return s[1 : len(s)-1]
}
return s
}
// harmlessParams are maildir location parameters that leave folder paths
// unchanged.
var harmlessParams = map[string]bool{
"INBOX": true, "INDEX": true, "INDEXPVT": true, "CONTROL": true, "VOLATILEDIR": true,
"LISTINDEX": true, "SUBSCRIPTIONS": true, "BROKENCHAR": true, "ITERINDEX": true, "NO-NOSELECT": true,
}
// parseLocation returns the Maildir++ root of a mail location and whether
// folder names are stored as UTF-8. A leading ~ expands to home only.
func parseLocation(loc, home string) (root string, utf8Names bool, reason string) {
driver, rest, ok := strings.Cut(loc, ":")
if !ok || !strings.EqualFold(driver, "maildir") {
return "", false, SkipDriver
}
params := strings.Split(rest, ":")
root = params[0]
for _, p := range params[1:] {
key, value, _ := strings.Cut(p, "=")
switch {
case key == "LAYOUT" && value == "fs":
return "", false, SkipLayoutFS
case key == "LAYOUT" && value == "maildir++":
case key == "UTF-8":
utf8Names = true
case !harmlessParams[key]:
// Includes "" from a "::" escape, DIRNAME and MAILBOXDIR.
return "", false, SkipLocation
}
}
if root == "~" || strings.HasPrefix(root, "~/") {
if !absClean(home) {
return "", false, SkipInvalidHome
}
root = home + root[1:]
}
// doveadm expands every location but a userdb mail field, which Dovecot
// expands later: a % left there is a variable, not a path, and a
// conditional may put any parameter anywhere.
if !absClean(root) || strings.Contains(root, "%") || usesIf(loc) {
return "", false, SkipLocation
}
return filepath.Clean(root), utf8Names, ""
}
// absClean accepts absolute paths without ".." components. Control bytes
// never get here: parseDetails refuses them in every field.
func absClean(p string) bool {
return strings.HasPrefix(p, "/") && !slices.Contains(strings.Split(p, "/"), "..")
}
func control(r rune) bool { return r < 0x20 || r == 0x7f }
// folderDir is the Maildir++ directory of a namespace-relative mailbox name:
// a dot, then the hierarchy levels joined by dots. A level holding a dot or a
// slash cannot be stored, so the name is refused.
func folderDir(name, sep string, utf8Names bool) (string, bool) {
if len(sep) != 1 || !utf8.ValidString(name) || strings.EqualFold(name, "INBOX") {
return "", false
}
levels := strings.Split(name, sep)
for i, l := range levels {
if l == "" || strings.ContainsAny(l, "./") || strings.ContainsFunc(l, unicode.IsControl) {
return "", false
}
if !utf8Names {
levels[i] = mutf7(l)
}
}
return "." + strings.Join(levels, "."), true
}
var imapBase64 = base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,").WithPadding(base64.NoPadding)
// mutf7 is IMAP modified UTF-7 (RFC 3501 5.1.3), the encoding Dovecot uses
// for folder names on disk unless the location says UTF-8.
func mutf7(s string) string {
var b strings.Builder
var wide []rune
flush := func() {
if len(wide) == 0 {
return
}
var raw []byte
for _, u := range utf16.Encode(wide) {
raw = binary.BigEndian.AppendUint16(raw, u)
}
b.WriteString("&" + imapBase64.EncodeToString(raw) + "-")
wide = wide[:0]
}
for _, r := range s {
if r < 0x20 || r > 0x7e {
wide = append(wide, r)
continue
}
flush()
b.WriteRune(r)
if r == '&' {
b.WriteByte('-')
}
}
flush()
return b.String()
}
// runCommand executes a command with stdin closed. Stdout over MaxListBytes
// stops it with an error; the first stderr bytes ride on an exec.ExitError.
// Cancelling ctx kills its whole process group, since doveconf forks.
func runCommand(ctx context.Context, name string, args ...string) ([]byte, error) {
stdout := &capped{limit: MaxListBytes}
stderr := &head{limit: 4096}
cmd := exec.CommandContext(ctx, name, args...) // #nosec G204 -- fixed Dovecot tools; arguments are constants, doveconf locations and validated user names after "--"
cmd.Stdout, cmd.Stderr = stdout, stderr
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
cmd.Cancel = func() error {
err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
if errors.Is(err, syscall.ESRCH) {
return os.ErrProcessDone
}
return err
}
cmd.WaitDelay = time.Second
err := cmd.Run()
if stdout.over {
return nil, errTooLarge
}
var exit *exec.ExitError
if errors.As(err, &exit) {
exit.Stderr = stderr.buf.Bytes()
}
return stdout.buf.Bytes(), err
}
// capped fails a write past limit, which closes the pipe on the child. The
// buffer is a field, not embedded: an embedded ReadFrom would bypass Write.
type capped struct {
buf bytes.Buffer
limit int
over bool
}
func (c *capped) Write(p []byte) (int, error) {
if c.buf.Len()+len(p) > c.limit {
c.over = true
return 0, errTooLarge
}
return c.buf.Write(p)
}
// head keeps the first limit bytes and drops the rest.
type head struct {
buf bytes.Buffer
limit int
}
func (h *head) Write(p []byte) (int, error) {
h.buf.Write(p[:min(len(p), max(h.limit-h.buf.Len(), 0))])
return len(p), nil
}
func alnum(c byte) bool {
return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9'
}
// ValidDomain accepts ASCII DNS names of at most 253 bytes with at least two
// nonempty labels of at most 63 letters, digits or hyphens, no edge hyphens.
// Single-label names such as the virtual namespace are not domains.
func ValidDomain(s string) bool {
if len(s) > 253 {
return false
}
labels := strings.Split(s, ".")
if len(labels) < 2 {
return false
}
for _, l := range labels {
if l == "" || len(l) > 63 || l[0] == '-' || l[len(l)-1] == '-' {
return false
}
for i := 0; i < len(l); i++ {
if !alnum(l[i]) && l[i] != '-' {
return false
}
}
}
return true
}
// atextSpecials is RFC 5322 atext punctuation without the slash.
const atextSpecials = "!#$%&'*+-=?^_`{|}~"
// ValidMailbox accepts ASCII dot-atoms of at most 64 bytes with no leading,
// trailing or repeated dot. Quoted local parts, whitespace, controls,
// slashes, backslashes and non-ASCII bytes are refused.
func ValidMailbox(s string) bool {
if s == "" || len(s) > 64 || s[0] == '.' || s[len(s)-1] == '.' || strings.Contains(s, "..") {
return false
}
for i := 0; i < len(s); i++ {
c := s[i]
if !alnum(c) && c != '.' && strings.IndexByte(atextSpecials, c) < 0 {
return false
}
}
return true
}
package antispam
import (
"fmt"
"io/fs"
)
// Secret holds a Spamhaus DQS key with redacted formatting and encoding.
// Its zero value is empty; Reveal is for rendering.
type Secret struct {
// fmt bypasses methods for invalid verbs and unexported fields. An
// indirect value keeps those reflection fallbacks from printing the key.
value *string
}
// NewSecret wraps a key without validating it. LoadDQSKey validates file facts.
func NewSecret(key string) Secret {
if key == "" {
return Secret{}
}
return Secret{value: &key}
}
const redacted = "[redacted]"
func (Secret) String() string { return redacted }
func (Secret) GoString() string { return redacted }
func (Secret) MarshalJSON() ([]byte, error) { return []byte(`"` + redacted + `"`), nil }
func (Secret) MarshalText() ([]byte, error) { return []byte(redacted), nil }
// Reveal returns the key for rendering into rspamd configuration.
func (s Secret) Reveal() string {
if s.value == nil {
return ""
}
return *s.value
}
// KeyFacts is the DQS key file as a collector found it. Key is set only for
// a regular file small enough to hold a key.
type KeyFacts struct {
Path string
Missing bool
UID int
Mode fs.FileMode
Key Secret
Err error
}
// dqsKeyLen is the DQS key length Spamhaus documents.
const dqsKeyLen = 26
// LoadDQSKey checks the key file's owner and mode and the key's format, and
// returns the key. Errors name the path and the rule, never the content.
func LoadDQSKey(k KeyFacts) (Secret, error) {
switch {
case k.Missing:
return Secret{}, fmt.Errorf("the Spamhaus DQS key file %s does not exist", k.Path)
case k.Err != nil:
return Secret{}, fmt.Errorf("cannot read the Spamhaus DQS key file %s; it must be a regular file", k.Path)
case k.UID != 0:
return Secret{}, fmt.Errorf("the Spamhaus DQS key file %s is owned by uid %d; want root", k.Path, k.UID)
case k.Mode.Perm()&0o022 != 0:
return Secret{}, fmt.Errorf("the Spamhaus DQS key file %s is writable by group or others", k.Path)
case k.Mode.Perm()&0o004 != 0:
return Secret{}, fmt.Errorf("the Spamhaus DQS key file %s is readable by others; make it mode 600", k.Path)
case !validDQSKey(k.Key.Reveal()):
return Secret{}, fmt.Errorf("the Spamhaus DQS key file %s does not hold a %d-character key of lowercase letters and digits", k.Path, dqsKeyLen)
}
return k.Key, nil
}
// validDQSKey accepts exactly dqsKeyLen lowercase ASCII letters and digits,
// so the key can sit inside a quoted UCL string and a DNS label.
func validDQSKey(key string) bool {
if len(key) != dqsKeyLen {
return false
}
for i := 0; i < len(key); i++ {
if c := key[i]; (c < 'a' || c > 'z') && (c < '0' || c > '9') {
return false
}
}
return true
}
package antispam
import (
"errors"
"slices"
"strings"
)
// fileBlocks finds the CSM blocks of one shared Exim file. Hook files parse
// as the ACL fragment cPanel inlines; exim.conf.local as main configuration.
func fileBlocks(kind TargetKind, content string) ([]Placement, error) {
if kind == TargetEximHook {
return locateBlocks(content, "acl", "csm_hook", false)
}
blocks, err := LocateBlocks(content)
if err != nil {
return nil, err
}
// cPanel section markers are not Exim section headers. A block that
// spans one cannot be removed without taking operator configuration.
lines := strings.Split(content, "\n")
for _, b := range blocks {
for _, line := range lines[b.Begin : b.End-1] {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "@") && strings.HasSuffix(line, "@") {
return nil, errors.New("a cPanel section starts inside a CSM block")
}
}
}
return blocks, nil
}
// BlockText returns the one CSM block in content, markers included, exactly
// as it appears there.
func BlockText(kind TargetKind, content string) (string, bool) {
blocks, err := fileBlocks(kind, content)
if err != nil || len(blocks) != 1 {
return "", false
}
lines := strings.SplitAfter(content, "\n")
return strings.Join(lines[blocks[0].Begin-1:blocks[0].End], ""), true
}
// SetBlock returns content with its CSM block replaced by block, or removed
// when block is empty. A hook file gets the block first, so no earlier
// statement in it can skip CSM's hook; exim.conf.local gets it right after
// its @CONFIG@ line. Content outside the block is kept byte for byte.
func SetBlock(kind TargetKind, content, block string) (string, error) {
blocks, err := fileBlocks(kind, content)
if err != nil {
return "", err
}
if len(blocks) > 1 {
return "", errors.New("the file has more than one CSM block")
}
lines := strings.SplitAfter(content, "\n")
if len(blocks) == 1 {
lines = slices.Delete(lines, blocks[0].Begin-1, blocks[0].End)
}
if block != "" {
at := 0
if kind == TargetEximLocal {
at = slices.IndexFunc(lines, func(l string) bool { return strings.TrimSuffix(l, "\n") == "@CONFIG@" })
if at < 0 {
return "", errors.New("exim.conf.local has no @CONFIG@ line")
}
lines[at] = "@CONFIG@\n"
at++
}
lines = slices.Insert(lines, at, block)
}
out := strings.Join(lines, "")
want := 0
if block != "" {
want = 1
}
// Content after the block must not continue CSM's last statement.
if got, err := fileBlocks(kind, out); err != nil || len(got) != want {
return "", errors.New("the edited file would not hold the CSM block on its own")
}
return out, nil
}
package antispam
import (
"errors"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
)
// EximVersion is the release and content-scanning support from `exim -bV`.
type EximVersion struct {
Major, Minor, Patch int
ContentScanning bool
}
var eximVersionRe = regexp.MustCompile(`^Exim version (\d{1,4})\.(\d{1,4})(?:\.(\d{1,4}))?\b`)
// ParseEximVersion reads `exim -bV` output.
func ParseEximVersion(out string) (EximVersion, error) {
var v EximVersion
found := false
for _, line := range strings.Split(out, "\n") {
line = strings.TrimRight(line, "\r")
if m := eximVersionRe.FindStringSubmatch(line); m != nil && !found {
v.Major, _ = strconv.Atoi(m[1])
v.Minor, _ = strconv.Atoi(m[2])
if m[3] != "" {
v.Patch, _ = strconv.Atoi(m[3])
}
found = true
continue
}
if rest, ok := strings.CutPrefix(line, "Support for:"); ok {
for _, feature := range strings.Fields(rest) {
if feature == "Content_Scanning" {
v.ContentScanning = true
}
}
}
}
if !found {
return EximVersion{}, errors.New("exim -bV printed no version line")
}
return v, nil
}
// AtLeast compares numerically, so 4.100 is newer than 4.97.
func (v EximVersion) AtLeast(major, minor int) bool {
return v.Major > major || v.Major == major && v.Minor >= minor
}
// NamedListProbes are the `exim -be` expressions proving every named list the
// hooks reference is defined. Each must print exactly "y" or "n".
func NamedListProbes() []string {
return []string{
"${if match_domain{example.invalid}{+local_domains}{y}{n}}",
"${if match_ip{192.0.2.1}{+trustedmailhosts}{y}{n}}",
"${if match_ip{2001:db8::1}{+cpanel_mail_netblocks}{y}{n}}",
}
}
// ErrUndefinedList is the probe result for a named list that does not exist.
// Exim 4.100.1 exits 0 for it, prints "Failed: ..." and logs "unknown named
// <kind> list" on stderr.
var ErrUndefinedList = errors.New("named list is undefined")
// errBeUnexpected covers any other output, such as a warning on stderr.
var errBeUnexpected = errors.New("exim -be printed an unexpected result")
var undefinedListRe = regexp.MustCompile(`unknown named (?:domain|host|address|local part) list "\+`)
// ParseBeResult checks one `exim -be` probe. Exim exits 0 even when a list is
// undefined, so only the printed text separates success from failure. Errors
// never repeat the output: a list's lookup text can hold credentials.
func ParseBeResult(stdout, stderr string) error {
switch {
case strings.TrimSpace(stderr) == "" && (stdout == "y\n" || stdout == "n\n"):
return nil
case strings.HasPrefix(stdout, "Failed: ") && undefinedListRe.MatchString(stderr):
return ErrUndefinedList
default:
return errBeUnexpected
}
}
// ParseEximOptions reads `exim -bP name...` output. A "name = value" line maps
// to value, "no_name" to "false" and a bare "name" to "true".
func ParseEximOptions(out string) map[string]string {
opts := make(map[string]string)
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if name, value, ok := strings.Cut(line, "="); ok {
opts[strings.TrimSpace(name)] = strings.TrimSpace(value)
continue
}
if name, ok := strings.CutPrefix(line, "no_"); ok {
opts[name] = "false"
continue
}
opts[line] = "true"
}
return opts
}
// cpanelChunkingOff is the documentation address cPanel puts in
// chunking_advertise_hosts so that no real client is offered CHUNKING.
const cpanelChunkingOff = "198.51.100.1"
// ChunkingOff reports whether Exim offers CHUNKING to no real client.
func ChunkingOff(hosts string) bool {
h := strings.TrimSpace(hosts)
return h == "" || h == cpanelChunkingOff
}
var aclVerbs = map[string]bool{
"accept": true, "defer": true, "deny": true, "discard": true,
"drop": true, "require": true, "warn": true,
}
var (
aclNameRe = regexp.MustCompile(`^[A-Za-z0-9_]+\s*:$`)
spamCondRe = regexp.MustCompile(`^!?\s*spam\s*=`)
spamdAddrRe = regexp.MustCompile(`^(?:hide\s+)?spamd_address\s*=`)
sectionRe = regexp.MustCompile(`(?i)^begin\s+([a-z_]+)$`)
cutthroughRe = regexp.MustCompile(`\bcutthrough_delivery\b`)
macroNameRe = regexp.MustCompile(`^[A-Z][A-Za-z0-9_]*$`)
)
// ErrUnverifiable marks an Exim configuration CSM cannot check: an included
// file it does not follow, macros that expand past its parsing bounds, or
// conditional placement of generated hooks.
var ErrUnverifiable = errors.New("exim configuration cannot be verified")
const (
// maxMacroVariants bounds the lines one configuration line expands to.
// Each conditional redefinition adds a variant, so they multiply.
maxMacroVariants = 64
// maxExpandedLine bounds one expanded line, so a macro redefined in terms
// of itself cannot double without limit.
maxExpandedLine = 64 << 10
)
type eximLine struct {
number int
text string
continued bool
physical []string
}
// Exim joins continuations without adding spaces and ignores intervening
// comments. Keep comments and physical line numbers for the ownership markers.
// An .include makes the file unverifiable: CSM does not follow it.
func eximLines(conf string) ([]eximLine, error) {
var out []eximLine
var text strings.Builder
current := -1
for i, raw := range strings.Split(conf, "\n") {
line := strings.TrimSpace(raw)
if strings.HasPrefix(line, "#") {
out = append(out, eximLine{number: i + 1, text: line, continued: current != -1})
continue
}
if strings.HasPrefix(line, ".include") {
return nil, fmt.Errorf("line %d: %w: .include is not followed", i+1, ErrUnverifiable)
}
if current == -1 {
if line == "" {
continue
}
current = len(out)
out = append(out, eximLine{number: i + 1})
}
out[current].physical = append(out[current].physical, line)
part, continued := strings.CutSuffix(line, `\`)
text.WriteString(part)
if !continued {
out[current].text = strings.TrimSpace(text.String())
text.Reset()
current = -1
}
}
if current != -1 {
out[current].text = strings.TrimSpace(text.String())
}
return out, nil
}
// Exim expands macros and processes includes on physical lines, even inside
// continuations. Checking only the joined line can hide an included file.
func checkEximIncludes(line eximLine, macros []eximMacro) error {
for i, physical := range line.physical {
if name, _, ok := strings.Cut(physical, "="); i == 0 && ok && macroNameRe.MatchString(strings.TrimSpace(name)) {
continue
}
lines, err := expandEximMacros(physical, macros)
if err != nil {
return err
}
for _, expanded := range lines {
if strings.HasPrefix(strings.TrimSpace(expanded), ".include") {
return fmt.Errorf("%w: .include is not followed", ErrUnverifiable)
}
}
}
return nil
}
func eximSection(line string) (string, bool) {
m := sectionRe.FindStringSubmatch(line)
if m == nil {
return "", false
}
section := strings.ToLower(m[1])
if section == "acls" {
section = "acl"
}
return section, true
}
func splitACLVerb(line string) (string, string) {
word, rest := line, ""
if i := strings.IndexAny(line, " \t\r\n\f\v"); i >= 0 {
word, rest = line[:i], strings.TrimSpace(line[i:])
}
if aclVerbs[word] {
return word, rest
}
return "", line
}
// UngatedSpamUses returns the 1-based line numbers, outside CSM blocks, of ACL
// statements that run a spam scan or read $spam_* without cPanel's
// ${perl{spamd_is_available}} gate, and of any other line that reads $spam_*.
// cPanel's spam-score deny, ratelimit and forwarding options generate such
// statements and routers; they would act on CSM's scan with their own
// threshold and ignore an account's opt-out.
func UngatedSpamUses(conf string) ([]int, error) {
var out []int
var macros []eximMacro
inACL, inCSM := false, false
stmt, unsafe, verb := 0, false, ""
depth, gateDepth := 0, -1
flush := func() {
if stmt != 0 && unsafe {
out = append(out, stmt)
}
stmt, unsafe, verb, gateDepth = 0, false, "", -1
}
parsed, err := eximLines(conf)
if err != nil {
return nil, err
}
for _, raw := range parsed {
line := raw.text
if err := checkEximIncludes(raw, macros); err != nil {
flush()
return out, fmt.Errorf("line %d: %w", raw.number, err)
}
if ok, err := recordEximMacro(line, depth, ¯os); err != nil {
flush()
return out, fmt.Errorf("line %d: %w", raw.number, err)
} else if ok {
flush()
continue
}
switch line {
case MarkerBegin:
flush()
inCSM = true
continue
case MarkerEnd:
inCSM = false
continue
}
if inCSM || line == "" || line[0] == '#' {
continue
}
if line[0] == '.' {
// A gate in an optional branch cannot protect uses after that
// branch, but an enclosing gate still protects nested branches.
switch strings.Fields(line)[0] {
case ".if", ".ifdef", ".ifndef":
depth++
case ".else", ".elifdef", ".elifndef", ".endif":
if gateDepth >= depth {
gateDepth = -1
}
if strings.HasPrefix(line, ".endif") {
depth--
}
}
continue
}
lines, err := expandEximMacros(line, macros)
if err != nil {
flush()
return out, fmt.Errorf("line %d: %w", raw.number, err)
}
sectionFound, aclSection := false, false
for _, expanded := range lines {
if section, ok := eximSection(expanded); ok {
sectionFound = true
aclSection = aclSection || section == "acl"
}
}
if sectionFound {
flush()
inACL = aclSection
continue
}
if !inACL {
// Nothing gates a router, transport or main option, so any spam
// read there acts on CSM's score for every account.
for _, expanded := range lines {
if readsSpamVariable(expanded) {
out = append(out, raw.number)
break
}
}
continue
}
newVerb, uses, gate := "", false, true
boundary := false
for _, expanded := range lines {
if aclNameRe.MatchString(expanded) {
boundary = true
}
word, rest := splitACLVerb(expanded)
if word != "" && (newVerb == "" || word == "require") {
newVerb = word
}
uses = uses || spamCondRe.MatchString(rest) || readsSpamVariable(rest)
name, value, ok := strings.Cut(rest, "=")
gate = gate && ok && strings.TrimSpace(name) == "condition" && strings.TrimSpace(value) == "${perl{spamd_is_available}}"
}
if boundary {
flush()
continue
}
if newVerb != "" {
flush()
stmt, verb = raw.number, newVerb
}
if stmt == 0 {
continue
}
if uses && (gateDepth == -1 || verb == "require") {
unsafe = true
}
if gate && gateDepth == -1 {
gateDepth = depth
}
}
flush()
sort.Ints(out)
return out, nil
}
func readsSpamVariable(s string) bool {
return strings.Contains(s, "$spam_") || strings.Contains(s, "${spam_")
}
// HasCutthrough reports whether a generated exim.conf enables cutthrough
// delivery anywhere outside a comment.
func HasCutthrough(conf string) (bool, error) {
var macros []eximMacro
depth := 0
parsed, err := eximLines(conf)
if err != nil {
return false, err
}
for _, raw := range parsed {
line := raw.text
if err := checkEximIncludes(raw, macros); err != nil {
return false, fmt.Errorf("line %d: %w", raw.number, err)
}
if line == "" || line[0] == '#' {
continue
}
if line[0] == '.' {
switch strings.Fields(line)[0] {
case ".if", ".ifdef", ".ifndef":
depth++
case ".endif":
depth--
}
continue
}
if ok, err := recordEximMacro(line, depth, ¯os); err != nil {
return false, fmt.Errorf("line %d: %w", raw.number, err)
} else if ok {
continue
}
lines, err := expandEximMacros(line, macros)
if err != nil {
return false, fmt.Errorf("line %d: %w", raw.number, err)
}
for _, expanded := range lines {
_, condition := splitACLVerb(expanded)
if name, value, ok := strings.Cut(condition, "="); ok && strings.TrimSpace(name) == "control" && cutthroughRe.MatchString(value) {
return true, nil
}
}
}
return false, nil
}
type eximMacro struct {
name string
values []string
}
func recordEximMacro(line string, depth int, macros *[]eximMacro) (bool, error) {
name, value, ok := strings.Cut(line, "=")
name = strings.TrimSpace(name)
if !ok || !macroNameRe.MatchString(name) {
return false, nil
}
value = strings.TrimSpace(strings.TrimPrefix(value, "="))
values, err := expandEximMacros(value, *macros)
if err != nil {
return false, err
}
for i := range *macros {
if (*macros)[i].name == name {
if depth == 0 {
(*macros)[i].values = values
} else {
// Host-dependent conditionals can select any definition.
(*macros)[i].values = append((*macros)[i].values, values...)
}
if len((*macros)[i].values) > maxMacroVariants {
return false, fmt.Errorf("%w: a macro has more than %d definitions", ErrUnverifiable, maxMacroVariants)
}
return true, nil
}
}
*macros = append(*macros, eximMacro{name: name, values: values})
return true, nil
}
func eximConditionalDepth(line string, depth int) int {
if strings.HasPrefix(line, ".") {
switch strings.Fields(line)[0] {
case ".if", ".ifdef", ".ifndef":
return depth + 1
case ".endif":
return depth - 1
}
}
return depth
}
func expandEximMacros(line string, macros []eximMacro) ([]string, error) {
lines := []string{line}
for _, macro := range macros {
var expanded []string
for _, line := range lines {
n := strings.Count(line, macro.name)
if n == 0 {
expanded = append(expanded, line)
continue
}
for _, value := range macro.values {
if len(line)+n*(len(value)-len(macro.name)) > maxExpandedLine {
return nil, fmt.Errorf("%w: a macro expands one line past %d bytes", ErrUnverifiable, maxExpandedLine)
}
expanded = append(expanded, strings.ReplaceAll(line, macro.name, value))
}
if len(expanded) > maxMacroVariants {
return nil, fmt.Errorf("%w: macros expand one line into more than %d variants", ErrUnverifiable, maxMacroVariants)
}
}
lines = expanded
}
return lines, nil
}
// ForeignSpamdAddress reports whether /etc/exim.conf.local sets spamd_address
// outside CSM's own block.
func ForeignSpamdAddress(local string) (bool, error) {
inCSM := false
var macros []eximMacro
depth := 0
parsed, err := eximLines(local)
if err != nil {
return false, err
}
for _, raw := range parsed {
line := raw.text
if err := checkEximIncludes(raw, macros); err != nil {
return false, fmt.Errorf("line %d: %w", raw.number, err)
}
depth = eximConditionalDepth(line, depth)
if ok, err := recordEximMacro(line, depth, ¯os); err != nil {
return false, fmt.Errorf("line %d: %w", raw.number, err)
} else if ok {
continue
}
switch line {
case MarkerBegin:
inCSM = true
case MarkerEnd:
inCSM = false
default:
if inCSM {
continue
}
lines, err := expandEximMacros(line, macros)
if err != nil {
return false, fmt.Errorf("line %d: %w", raw.number, err)
}
for _, expanded := range lines {
if spamdAddrRe.MatchString(expanded) {
return true, nil
}
}
}
}
return false, nil
}
// Placement is one CSM block found in an Exim config file.
type Placement struct {
// ACL is the ACL holding the block, or "" in the main configuration.
ACL string
Begin int // 1-based line of the BEGIN marker
End int // 1-based line of the END marker
}
// LocateBlocks finds every CSM block and the ACL that holds it. Nested,
// unpaired or boundary-crossing markers are an error: CSM never edits a file
// whose blocks it cannot identify.
func LocateBlocks(conf string) ([]Placement, error) {
return locateBlocks(conf, "", "", false)
}
// locateBlocks starts in section and acl, so a hook file parses as the ACL
// fragment cPanel inlines. Verification needs stronger evidence than
// ownership: with verify set, it refuses blocks Exim could skip or rewrite,
// since comments can still identify a block whose statements Exim skips.
func locateBlocks(conf, section, acl string, verify bool) ([]Placement, error) {
var out []Placement
var macros []eximMacro
depth := 0
open := 0
conditionalSection, conditionalACL := false, false
// afterEnd is set when a block closed inside an ACL: the next line must
// start something new, or Exim would join it to CSM's last statement.
afterEnd := false
type branch struct {
entry, pending, hasElse bool
}
var branches []branch
parsed, err := eximLines(conf)
if err != nil {
return nil, err
}
for _, raw := range parsed {
line, number := raw.text, raw.number
if err := checkEximIncludes(raw, macros); err != nil {
return nil, fmt.Errorf("line %d: %w", number, err)
}
if raw.continued && (line == MarkerBegin || line == MarkerEnd) {
return nil, fmt.Errorf("line %d: CSM marker splits a continued configuration line", number)
}
if verify {
// Exim handles directives before joining continuation lines.
for _, physical := range raw.physical {
lines, err := expandEximMacros(physical, macros)
if err != nil {
return nil, fmt.Errorf("line %d: %w", number, err)
}
nextDepth := eximConditionalDepth(lines[0], depth)
for _, expanded := range lines[1:] {
if eximConditionalDepth(expanded, depth) != nextDepth {
return nil, fmt.Errorf("line %d: %w: ambiguous conditional directive", number, ErrUnverifiable)
}
}
depth = nextDepth
}
} else {
depth = eximConditionalDepth(line, depth)
}
if verify && (line == MarkerBegin || open != 0) && (depth != 0 || conditionalSection || conditionalACL) {
return nil, fmt.Errorf("line %d: %w: a CSM block or its section or ACL is conditional", number, ErrUnverifiable)
}
if strings.HasPrefix(line, ".") {
switch strings.Fields(line)[0] {
case ".if", ".ifdef", ".ifndef":
branches = append(branches, branch{entry: afterEnd})
case ".else", ".elifdef", ".elifndef":
if len(branches) > 0 {
b := &branches[len(branches)-1]
b.pending = b.pending || afterEnd
b.hasElse = b.hasElse || strings.Fields(line)[0] == ".else"
afterEnd = b.entry
}
case ".endif":
if len(branches) > 0 {
b := branches[len(branches)-1]
afterEnd = afterEnd || b.pending || !b.hasElse && b.entry
branches = branches[:len(branches)-1]
}
}
}
if afterEnd && line != "" && line[0] != '#' && line[0] != '.' {
if err := startsStatement(line, macros); err != nil {
return nil, fmt.Errorf("line %d: %w", number, err)
}
afterEnd = false
}
if ok, err := recordEximMacro(line, depth, ¯os); err != nil {
return nil, fmt.Errorf("line %d: %w", number, err)
} else if ok {
if section == "acl" {
if open != 0 {
return nil, fmt.Errorf("line %d: macro ends an ACL inside a CSM block", number)
}
acl = ""
}
continue
}
boundary := ""
lines, err := expandEximMacros(line, macros)
if err != nil {
return nil, fmt.Errorf("line %d: %w", number, err)
}
boundaries := 0
for _, expanded := range lines {
if verify && open != 0 && line != "" && line[0] != '#' && expanded != line {
return nil, fmt.Errorf("line %d: a macro changes the rendered CSM block", number)
}
if name, ok := eximSection(expanded); ok {
boundaries++
if open != 0 {
return nil, fmt.Errorf("line %d: section starts inside a CSM block", number)
}
if boundary != "" && boundary != "section:"+name {
return nil, fmt.Errorf("line %d: ambiguous section", number)
}
boundary = "section:" + name
section, acl = name, ""
conditionalSection, conditionalACL = depth != 0, false
} else if section == "acl" && aclNameRe.MatchString(expanded) {
boundaries++
if open != 0 {
return nil, fmt.Errorf("line %d: ACL starts inside a CSM block", number)
}
name := strings.TrimSpace(strings.TrimSuffix(expanded, ":"))
if boundary != "" && boundary != "acl:"+name {
return nil, fmt.Errorf("line %d: ambiguous ACL", number)
}
boundary, acl = "acl:"+name, name
conditionalACL = depth != 0
}
}
if boundaries != 0 && boundaries != len(lines) {
return nil, fmt.Errorf("line %d: optional section or ACL boundary", number)
}
switch line {
case MarkerBegin:
if open != 0 {
return nil, fmt.Errorf("line %d: nested %s", number, MarkerBegin)
}
if section != "" && (section != "acl" || acl == "") {
return nil, fmt.Errorf("line %d: CSM block is outside the main configuration or a named ACL", number)
}
open = number
case MarkerEnd:
if open == 0 {
return nil, fmt.Errorf("line %d: %s without %s", number, MarkerEnd, MarkerBegin)
}
out = append(out, Placement{ACL: acl, Begin: open, End: number})
open = 0
afterEnd = acl != ""
}
}
if open != 0 {
return nil, fmt.Errorf("line %d: %s without %s", open, MarkerBegin, MarkerEnd)
}
return out, nil
}
// errContinuesBlock is a line after a CSM block in an ACL that would extend
// the block's last statement.
var errContinuesBlock = errors.New("a line after the CSM block continues its last statement")
// startsStatement accepts a line after a CSM block that begins something new:
// an ACL verb, an ACL name, a section, a macro or another CSM block.
func startsStatement(line string, macros []eximMacro) error {
if line == MarkerBegin {
return nil
}
if name, _, ok := strings.Cut(line, "="); ok && macroNameRe.MatchString(strings.TrimSpace(name)) {
return nil
}
lines, err := expandEximMacros(line, macros)
if err != nil {
return err
}
for _, expanded := range lines {
word, _ := splitACLVerb(expanded)
_, section := eximSection(expanded)
if word == "" && !section && !aclNameRe.MatchString(expanded) {
return errContinuesBlock
}
}
return nil
}
// CheckPlacement verifies a generated exim.conf holds exactly one scanner
// block in the main configuration, one hook in the RCPT ACL and two hooks in
// the DATA ACL. rcptACL and dataACL come from
// `exim -bP acl_smtp_rcpt acl_smtp_data`.
func CheckPlacement(blocks []Placement, rcptACL, dataACL string) error {
var main, rcpt, data int
for _, b := range blocks {
switch b.ACL {
case "":
main++
case rcptACL:
rcpt++
case dataACL:
data++
default:
return fmt.Errorf("CSM block at line %d is in unexpected ACL %q", b.Begin, b.ACL)
}
}
if main != 1 || rcpt != 1 || data != 2 {
return fmt.Errorf("found %d config, %d RCPT and %d DATA CSM blocks; want 1, 1 and 2", main, rcpt, data)
}
return nil
}
package antispam
import (
"errors"
"fmt"
"maps"
"path"
"slices"
"strconv"
"strings"
)
// Feed names an operator can enable.
const (
FeedMailspike = "mailspike"
FeedPSBL = "psbl"
FeedUCEPROTECT1 = "uceprotect1"
FeedNordspam = "nordspam"
FeedSEM = "sem"
FeedSpamCop = "spamcop"
FeedBarracuda = "barracuda"
FeedDNSWL = "dnswl"
FeedURIBL = "uribl"
FeedSURBL = "surbl"
FeedSpamhausDQS = "spamhaus_dqs"
FeedCSMIPMap = "csm_ip_map"
FeedRspamdMaps = "rspamd_maps"
)
// dqsKeyMark is replaced by the operator's Spamhaus DQS key in a zone.
const dqsKeyMark = "{key}"
// Code maps exact DNS answers to one symbol.
type Code struct {
Symbol string
Answers []string
Weight float64
Description string
}
// Zone is one DNS list, rendered as one rspamd rbl rule.
type Zone struct {
// Rule is the rbls key; the rule's base symbol is Rule upper-cased with
// a _RULE suffix and weighs 0.
Rule string
Zone string
// Domain lists check message domains (URLs, emails, HELO, rDNS, DKIM);
// the others check only the connecting address.
Domain bool
IPv4, IPv6 bool
FullHostname bool
Codes []Code
// TestName is the listed test point, queried as TestName.Zone; empty
// means the zone has no documented test point and is not probed.
TestName string
TestAnswers []string
}
// Feed is one catalogue entry.
type Feed struct {
Name string
// Zones is empty for rspamd_maps, which only keeps rspamd's map URLs.
Zones []Zone
NeedsKey bool
// Errors maps answers that mean refusal or misuse to their meaning.
// They never score; the probe reports them.
Errors map[string]string
Terms string
}
// DNS reports whether f adds rbl rules.
func (f Feed) DNS() bool { return len(f.Zones) > 0 }
func code(symbol string, weight float64, description string, answers ...string) Code {
return Code{Symbol: symbol, Answers: answers, Weight: weight, Description: description}
}
// ipZone is an IP list with the common RFC 5782 test entry 127.0.0.2.
func ipZone(rule, zone string, ipv6 bool, codes ...Code) Zone {
return Zone{Rule: rule, Zone: zone, IPv4: true, IPv6: ipv6, Codes: codes,
TestName: "2.0.0.127", TestAnswers: []string{"127.0.0.2"}}
}
func answerRange(prefix string, from, to int) []string {
var out []string
for n := from; n <= to; n++ {
out = append(out, prefix+strconv.Itoa(n))
}
return out
}
type bit struct {
mask int
symbol string
weight float64
description string
}
// exclusiveBits maps each answer 127.0.0.N whose low (blocked) bit is clear
// to the first bit in order that N sets, so one answer fires one symbol.
// Bits not in order are ignored.
func exclusiveBits(order ...bit) []Code {
codes := make([]Code, len(order))
for i, b := range order {
codes[i] = Code{Symbol: b.symbol, Weight: b.weight, Description: b.description}
}
for n := 2; n <= 254; n += 2 {
for i, b := range order {
if n&b.mask != 0 {
codes[i].Answers = append(codes[i].Answers, "127.0.0."+strconv.Itoa(n))
break
}
}
}
return codes
}
// dnswlCodes maps 127.0.<category>.<trust> to one symbol per trust level.
// Category 10 holds special cases, including the test entry, and never
// scores.
func dnswlCodes() []Code {
levels := []Code{
{Symbol: "CSM_DNSWL_NONE", Weight: 0, Description: "DNSWL: listed, no trust"},
{Symbol: "CSM_DNSWL_LOW", Weight: -0.1, Description: "DNSWL: low trust"},
{Symbol: "CSM_DNSWL_MED", Weight: -0.2, Description: "DNSWL: medium trust"},
{Symbol: "CSM_DNSWL_HI", Weight: -0.5, Description: "DNSWL: high trust"},
}
for trust := range levels {
for category := 1; category <= 20; category++ {
if category != 10 {
levels[trust].Answers = append(levels[trust].Answers,
"127.0."+strconv.Itoa(category)+"."+strconv.Itoa(trust))
}
}
}
return levels
}
var spamhausErrors = map[string]string{
"127.255.255.250": "the DQS key is disabled",
"127.255.255.251": "the DQS key is used from an address Spamhaus does not allow",
"127.255.255.252": "the query names a zone that does not exist",
"127.255.255.254": "the query came through a public or open resolver",
"127.255.255.255": "the query volume is over the allowed limit",
"127.0.1.255": "the domain list was asked about an IP address",
"127.0.2.255": "the domain age list was asked about an IP address",
}
// catalogue lists every feed. Weights start from
// rspamd's stock weights, or Spamhaus's for DQS, capped at 5. SURBL and DQS
// are reduced further so all positive symbols together stay below the
// default reject score. Domain lists can hit several symbols per message;
// consumers must also make each symbol one_shot to bound repeated hits.
var catalogue = []Feed{
{
Name: FeedMailspike,
Zones: []Zone{{Rule: "csm_mailspike", Zone: "rep.mailspike.net", IPv4: true,
Codes: []Code{
code("CSM_MAILSPIKE_WORST", 2.0, "Mailspike: worst reputation", "127.0.0.10"),
code("CSM_MAILSPIKE_VERYBAD", 1.5, "Mailspike: very bad reputation", "127.0.0.11"),
code("CSM_MAILSPIKE_BAD", 1.0, "Mailspike: bad reputation", "127.0.0.12"),
},
TestName: "2.0.0.127", TestAnswers: []string{"127.0.0.2"}}},
Terms: "Free for sites under 100,000 messages and 100,000 DNS queries a day; contact Mailspike above that.",
},
{
Name: FeedPSBL,
Zones: []Zone{ipZone("csm_psbl", "psbl.surriel.com", false, code("CSM_PSBL", 2.0, "PSBL: listed", "127.0.0.2"))},
Terms: "Free for anyone.",
},
{
Name: FeedUCEPROTECT1,
Zones: []Zone{ipZone("csm_uceprotect1", "dnsbl-1.uceprotect.net", false, code("CSM_UCEPROTECT1", 1.5, "UCEPROTECT level 1: listed", "127.0.0.2"))},
Terms: "Free for private and commercial use; above 100,000 requests a day UCEPROTECT asks for its rsync feed. Level 1 also lists some forwarders that break SPF.",
},
{
Name: FeedNordspam,
Zones: []Zone{ipZone("csm_nordspam", "bl.nordspam.com", true, code("CSM_NORDSPAM", 2.0, "Nordspam: listed", "127.0.0.2"))},
Terms: "Free for commercial and non-commercial use; contact Nordspam above 10,000 queries a day.",
},
{
Name: FeedSEM,
Zones: []Zone{
ipZone("csm_sem", "bl.spameatingmonkey.net", false, code("CSM_SEM", 1.0, "Spam Eating Monkey: listed", "127.0.0.2")),
{Rule: "csm_sem_ipv6", Zone: "bl.ipv6.spameatingmonkey.net", IPv6: true,
Codes: []Code{code("CSM_SEM_IPV6", 1.0, "Spam Eating Monkey IPv6: listed", "127.0.0.2")}},
},
Terms: "Free; for scoring only, never to reject mail on its own. Tell Spam Eating Monkey before going above 100,000 queries a day.",
},
{
Name: FeedSpamCop,
Zones: []Zone{ipZone("csm_spamcop", "bl.spamcop.net", false, code("CSM_SPAMCOP", 1.5, "SpamCop: listed", "127.0.0.2"))},
Terms: "Free; SpamCop calls its list aggressive and recommends tagging, not rejecting.",
},
{
Name: FeedBarracuda,
Zones: []Zone{ipZone("csm_barracuda", "b.barracudacentral.org", false, code("CSM_BARRACUDA", 2.0, "Barracuda: listed", "127.0.0.2"))},
Terms: "Free after registering the addresses of the resolvers that query it with Barracuda.",
},
{
Name: FeedDNSWL,
Zones: []Zone{{Rule: "csm_dnswl", Zone: "list.dnswl.org", IPv4: true, IPv6: true, Codes: dnswlCodes(),
TestName: "2.0.0.127", TestAnswers: []string{"127.0.10.0", "127.0.10.1", "127.0.10.2", "127.0.10.3"}}},
Errors: map[string]string{"127.0.0.255": "DNSWL refuses this resolver: public resolvers and sites over 100,000 queries a day are blocked"},
Terms: "Free through DNS under 100,000 queries a day; its pages conflict on non-commercial use. A listing lowers the score slightly.",
},
{
Name: FeedURIBL,
Zones: []Zone{{Rule: "csm_uribl", Zone: "multi.uribl.com", Domain: true,
Codes: exclusiveBits(
bit{2, "CSM_URIBL_BLACK", 5.0, "URIBL: black list"},
bit{4, "CSM_URIBL_GREY", 0.5, "URIBL: grey list"},
bit{8, "CSM_URIBL_RED", 0.5, "URIBL: red list"},
),
TestName: "test.uribl.com", TestAnswers: []string{"127.0.0.14"}}},
Errors: map[string]string{"127.0.0.1": "URIBL refuses this resolver", "127.0.0.255": "URIBL refuses this resolver"},
Terms: "Free for low-volume use only; URIBL blocks public and large shared resolvers.",
},
{
Name: FeedSURBL,
Zones: []Zone{{Rule: "csm_surbl", Zone: "multi.surbl.org", Domain: true, FullHostname: true,
Codes: exclusiveBits(
bit{8, "CSM_SURBL_PH", 3.5, "SURBL: phishing"},
bit{16, "CSM_SURBL_MW", 3.5, "SURBL: malware"},
bit{64, "CSM_SURBL_ABUSE", 3.5, "SURBL: abuse"},
bit{128, "CSM_SURBL_CRACKED", 3.5, "SURBL: cracked site"},
bit{32, "CSM_SURBL_CT", 0, "SURBL: click tracker"},
bit{4, "CSM_SURBL_DM", 0, "SURBL: disposable mail"},
),
TestName: "test.surbl.org", TestAnswers: []string{"127.0.0.254"}}},
Errors: map[string]string{"127.0.0.1": "SURBL refuses this resolver"},
Terms: "Free only for organisations under 1,000 users or 250,000 messages a day, and never inside a product or service sold for a fee; other use needs SURBL's paid data service.",
},
{
Name: FeedSpamhausDQS,
NeedsKey: true,
Zones: []Zone{
{Rule: "csm_dqs_zen", Zone: dqsKeyMark + ".zen.dq.spamhaus.net", IPv4: true, IPv6: true,
Codes: []Code{code("CSM_DQS_ZEN", 5.0, "Spamhaus ZEN: listed",
"127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5", "127.0.0.6", "127.0.0.7", "127.0.0.9", "127.0.0.10", "127.0.0.11", "127.0.0.30")},
TestName: "2.0.0.127", TestAnswers: []string{"127.0.0.2"}},
{Rule: "csm_dqs_dbl", Zone: dqsKeyMark + ".dbl.dq.spamhaus.net", Domain: true, Codes: []Code{
code("CSM_DQS_DBL", 3.0, "Spamhaus DBL: spam, phishing, malware or botnet domain", "127.0.1.2", "127.0.1.4", "127.0.1.5", "127.0.1.6"),
code("CSM_DQS_DBL_ABUSED", 2.0, "Spamhaus DBL: abused legitimate domain", answerRange("127.0.1.", 102, 106)...),
}},
{Rule: "csm_dqs_zrd", Zone: dqsKeyMark + ".zrd.dq.spamhaus.net", Domain: true, Codes: []Code{
code("CSM_DQS_ZRD_VERY_FRESH", 3.0, "Spamhaus ZRD: domain first seen in the last 4 hours", answerRange("127.0.2.", 2, 4)...),
code("CSM_DQS_ZRD_FRESH", 1.0, "Spamhaus ZRD: domain first seen in the last 24 hours", answerRange("127.0.2.", 5, 24)...),
}},
},
Errors: spamhausErrors,
Terms: "Needs a DQS key. The free key covers non-commercial use under 100,000 queries a day; commercial use needs a paid key.",
},
{
Name: FeedCSMIPMap,
Terms: "Local data only: CSM's block history, its threat feeds and the spam evidence it records.",
},
{
Name: FeedRspamdMaps,
Terms: "Keeps rspamd's own map downloads from maps.rspamd.com; free use is non-commercial and under 5,000 queries a day, other use needs an agreement with rspamd.",
},
}
// FeedByName returns an independent copy of the catalogue entry named name.
func FeedByName(name string) (Feed, bool) {
i := slices.IndexFunc(catalogue, func(f Feed) bool { return f.Name == name })
if i < 0 {
return Feed{}, false
}
f := catalogue[i]
f.Errors = maps.Clone(f.Errors)
f.Zones = slices.Clone(f.Zones)
for i := range f.Zones {
z := &f.Zones[i]
z.TestAnswers = slices.Clone(z.TestAnswers)
z.Codes = slices.Clone(z.Codes)
for j := range z.Codes {
z.Codes[j].Answers = slices.Clone(z.Codes[j].Answers)
}
}
return f, true
}
// FeedNames lists every catalogue name in catalogue order.
func FeedNames() []string {
out := make([]string, len(catalogue))
for i, f := range catalogue {
out[i] = f.Name
}
return out
}
// ValidateFeeds checks feed names and the DQS key path: every name is in
// the catalogue once, and spamhaus_dqs needs a clean absolute key file path.
func ValidateFeeds(names []string, keyFile string) error {
seen := make(map[string]bool, len(names))
for _, name := range names {
if _, ok := FeedByName(name); !ok {
return fmt.Errorf("unknown feed; known feeds: %s", strings.Join(FeedNames(), ", "))
}
if seen[name] {
return fmt.Errorf("feed %q is listed twice", name)
}
seen[name] = true
}
if !seen[FeedSpamhausDQS] {
return nil
}
switch {
case keyFile == "":
return errors.New("feed spamhaus_dqs needs spamhaus_dqs_key_file")
case !path.IsAbs(keyFile) || path.Clean(keyFile) != keyFile:
return errors.New("spamhaus_dqs_key_file must be a clean absolute path")
case keyFile == "/" || strings.ContainsRune(keyFile, 0):
return errors.New("spamhaus_dqs_key_file must be a file path without NUL bytes")
}
return nil
}
package antispam
import (
"slices"
"strings"
)
// ProbeQuery is one DNS question the feed probe asks. Name can hold the
// DQS key: it must never reach a log line or a finding.
type ProbeQuery struct {
Feed string
Zone Zone
Name string
Listed bool
}
// ProbeAnswer is what a resolver said, without any error text: Go's DNS
// errors print the query name.
type ProbeAnswer struct {
Addrs []string
NXDomain bool
// Failed is a timeout, SERVFAIL, REFUSED or network error.
Failed bool
}
// ProbeStatus classifies one probe answer.
type ProbeStatus int
const (
ProbeOK ProbeStatus = iota
// ProbeNoAnswer: the listed test point got no answer, or a query failed.
ProbeNoAnswer
// ProbeRefused: the list answered with one of its error codes.
ProbeRefused
// ProbeListsAll: the never-listed point got a listing code.
ProbeListsAll
// ProbeUnexpected: any other answer.
ProbeUnexpected
)
// unlistedDomain is a name no domain list can list: .invalid never
// resolves (RFC 6761).
const unlistedDomain = "csm-probe.invalid"
// ProbeQueries lists the questions for s's probed zones: the listed test
// point and the unlisted point, which for IP lists is also rspamd's own
// monitored address.
func ProbeQueries(s Settings) []ProbeQuery {
var out []ProbeQuery
for _, f := range s.DNSFeeds() {
for _, z := range f.Zones {
if z.TestName == "" {
continue
}
zone := strings.ReplaceAll(z.Zone, dqsKeyMark, s.DQSKey.Reveal())
unlisted := "1.0.0.127"
if z.Domain {
unlisted = unlistedDomain
}
out = append(out,
ProbeQuery{Feed: f.Name, Zone: z, Name: z.TestName + "." + zone, Listed: true},
ProbeQuery{Feed: f.Name, Zone: z, Name: unlisted + "." + zone})
}
}
return out
}
// ClassifyProbe judges one answer. The second result is the meaning of an
// error answer, from the catalogue.
func ClassifyProbe(f Feed, q ProbeQuery, a ProbeAnswer) (ProbeStatus, string) {
scores := func(addr string) bool {
return slices.ContainsFunc(q.Zone.Codes, func(c Code) bool { return slices.Contains(c.Answers, addr) })
}
// Rspamd scores each returned address independently. A refusal alongside
// a listing does not make a listing of the negative test point harmless.
hasListing := slices.ContainsFunc(a.Addrs, scores)
if !q.Listed && hasListing {
return ProbeListsAll, ""
}
for _, addr := range a.Addrs {
if meaning, ok := f.Errors[addr]; ok {
if hasListing {
return ProbeUnexpected, ""
}
return ProbeRefused, meaning
}
}
switch {
case a.Failed:
return ProbeNoAnswer, ""
case q.Listed && (a.NXDomain || len(a.Addrs) == 0):
return ProbeNoAnswer, ""
case q.Listed && slices.ContainsFunc(a.Addrs, func(addr string) bool { return slices.Contains(q.Zone.TestAnswers, addr) }):
return ProbeOK, ""
case !q.Listed && a.NXDomain:
return ProbeOK, ""
}
return ProbeUnexpected, ""
}
package antispam
import (
"errors"
"fmt"
"path"
"slices"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/ipmap"
)
// Map redirect names, for Settings.SkipMaps.
const (
RedirectForgedRecipients = "forged_recipients"
RedirectMID = "mid"
RedirectMimeTypes = "mime_types"
RedirectWhitelist = "whitelist"
RedirectMultimap = "multimap"
)
// MapRedirectNames lists the map redirects in write order.
func MapRedirectNames() []string {
return []string{RedirectForgedRecipients, RedirectMID, RedirectMimeTypes, RedirectWhitelist, RedirectMultimap}
}
// Rspamd paths for feed rules and symbol weights.
const (
RspamdOverrideDir = "/etc/rspamd/override.d"
FeedRBLPath = RspamdOverrideDir + "/rbl.conf"
FeedRBLGroupPath = RspamdOverrideDir + "/rbl_group.conf"
// ScoresPath sets the weight of rspamd's own symbols. CSM leaves it out
// when the operator owns it.
ScoresPath = RspamdLocalDir + "/groups.conf"
)
// mapRedirects point rspamd.com maps at rspamd's package copies, or at
// operator files that may not exist; rspamd loads both cleanly. String
// values replace the stock arrays in local.d, where arrays would append.
var mapRedirects = map[string]string{
RedirectForgedRecipients: "equivalent_domains = \"/etc/rspamd/maps.d/equivalent_domains.inc\";\n",
RedirectMID: "source {\n url = \"/etc/rspamd/maps.d/mid.inc\";\n}\n",
RedirectMimeTypes: "file = \"/etc/rspamd/maps.d/mime_types.inc\";\n",
RedirectWhitelist: "rules {\n" +
" WHITELIST_SPF_DKIM {\n domains = \"/etc/rspamd/maps.d/spf_dkim_whitelist.inc\";\n }\n" +
" WHITELIST_DMARC {\n domains = \"/etc/rspamd/maps.d/dmarc_whitelist.inc\";\n }\n}\n",
RedirectMultimap: multimapRedirects(),
}
func multimapRedirects() string {
var b strings.Builder
b.WriteString("redirector {\n map = \"/etc/rspamd/maps.d/redirectors.inc\";\n}\n")
for _, kind := range []string{"freemail", "disposable"} {
for _, part := range []string{"envfrom", "envrcpt", "from", "to", "cc", "replyto", "mdn"} {
fmt.Fprintf(&b, "%s_%s {\n map = \"/etc/rspamd/local.d/maps.d/%s.map\";\n}\n", kind, part, kind)
}
}
return b.String()
}
// phishingExceptions points the phishing module's rspamd.com maps at local
// files; it goes into core's phishing.conf.
const phishingExceptions = "exceptions {\n" +
" REDIRECTOR_FALSE = \"/etc/rspamd/maps.d/redirectors.inc\";\n" +
" PHISHED_WHITELISTED = \"glob;/etc/rspamd/local.d/maps.d/phishing_whitelist.inc\";\n" +
"}\n"
// RedirectPath is the local.d file of map redirect name.
func RedirectPath(name string) string { return path.Join(RspamdLocalDir, name+".conf") }
// OptionalRspamdPaths lists every optional file in write order: the DNS
// feed files, the map redirects, then the symbol weights.
func OptionalRspamdPaths() []string {
out := []string{FeedRBLPath, FeedRBLGroupPath}
for _, name := range MapRedirectNames() {
out = append(out, RedirectPath(name))
}
return append(out, ScoresPath)
}
// RenderFeeds returns the optional files s needs, keyed by path. A path
// absent from the result is one CSM deletes if it wrote it.
func RenderFeeds(s Settings) (map[string]string, error) {
if err := s.Validate(); err != nil {
return nil, err
}
files := make(map[string]string)
if feeds := s.DNSFeeds(); len(feeds) > 0 {
if s.HasFeed(FeedSpamhausDQS) && !validDQSKey(s.DQSKey.Reveal()) {
return nil, errors.New("feed spamhaus_dqs has no valid key loaded")
}
files[FeedRBLPath] = renderRBL(s, feeds)
files[FeedRBLGroupPath] = renderRBLGroup(feeds)
}
for _, name := range MapRedirectNames() {
text := ""
if s.Redirects() && !slices.Contains(s.SkipMaps, name) {
text = mapRedirects[name]
}
if name == RedirectMultimap && s.HasFeed(FeedCSMIPMap) {
text += ipMapRule()
}
if text != "" {
files[RedirectPath(name)] = ManagedHeader + "\n" + text
}
}
return files, nil
}
// ipMapRule is the multimap rule that reads the CSM IP map: each line names
// one of these symbols with a multiplier on the rule's score.
func ipMapRule() string {
var names []string
for _, s := range ipmap.Symbols() {
names = append(names, strconv.Quote(s.Name))
}
return "CSM_IP {\n type = \"ip\";\n map = \"" + ipmap.DefaultPath + "\";\n" +
" symbols = [" + strings.Join(names, ", ") + "];\n" +
" score = " + formatWeight(ipmap.RuleScore) + ";\n description = \"CSM IP map\";\n}\n"
}
// renderRBL writes only CSM's lists: override.d replaces rspamd's whole
// rbls object. IP rules set every default explicitly, since stock rbl.conf
// adds unknown = true; domain rules follow the stock DBL rule.
func renderRBL(s Settings, feeds []Feed) string {
var b strings.Builder
b.WriteString(ManagedHeader + "\nrbls {\n")
for _, f := range feeds {
for _, z := range f.Zones {
zone := strings.ReplaceAll(z.Zone, dqsKeyMark, s.DQSKey.Reveal())
fmt.Fprintf(&b, " %s {\n symbol = \"%s_RULE\";\n rbl = \"%s\";\n", z.Rule, strings.ToUpper(z.Rule), zone)
if z.Domain {
b.WriteString(" ignore_defaults = true;\n no_ip = true;\n exclude_users = false;\n emails_domainonly = true;\n" +
" checks = [\"emails\", \"dkim\", \"helo\", \"rdns\", \"replyto\", \"urls\", \"content_urls\"];\n")
// The emails check extracts body addresses, not sender headers.
b.WriteString(" selector {\n" +
" from_smtp = \"from('smtp'):domain\";\n" +
" from_mime = \"from('mime'):domain\";\n" +
" mid = \"header(Message-Id).regexp('@([^\\\\.]+\\\\.[^>]+)').last\";\n" +
" }\n")
if z.FullHostname {
b.WriteString(" url_full_hostname = true;\n")
}
} else {
fmt.Fprintf(&b, " checks = [\"from\"];\n ipv4 = %t;\n ipv6 = %t;\n"+
" exclude_local = true;\n exclude_users = true;\n unknown = false;\n", z.IPv4, z.IPv6)
}
if f.NeedsKey {
// Keep lookup names, which contain the key, out of the monitor log.
b.WriteString(" disable_monitoring = true;\n")
}
b.WriteString(" returncodes {\n")
for _, c := range z.Codes {
fmt.Fprintf(&b, " %s = [\"%s\"];\n", c.Symbol, strings.Join(c.Answers, "\", \""))
}
b.WriteString(" }\n }\n")
}
}
// The legacy rules alias would otherwise merge unselected lists back in.
b.WriteString("}\nrules {};\n")
// attached_maps is SURBL's hashbl map, fetched over plain HTTP whenever
// rbl runs; only override.d can clear it.
b.WriteString("attached_maps = [];\n")
if s.Redirects() {
b.WriteString("url_whitelist = \"/etc/rspamd/maps.d/surbl-whitelist.inc\";\n" +
"disabled_rbl_suffixes_map = \"/etc/rspamd/local.d/maps.d/disabled_rbls.inc\";\n")
}
return b.String()
}
// renderRBLGroup weights every CSM feed symbol; override.d replaces the
// stock rbl group's symbols, which belong to the lists renderRBL removed.
func renderRBLGroup(feeds []Feed) string {
var b strings.Builder
b.WriteString(ManagedHeader + "\nsymbols {\n")
for _, f := range feeds {
for _, z := range f.Zones {
for _, c := range z.Codes {
fmt.Fprintf(&b, " \"%s\" {\n weight = %s;\n description = \"%s\";\n one_shot = true;\n }\n",
c.Symbol, formatWeight(c.Weight), c.Description)
}
}
}
b.WriteString("}\n")
return b.String()
}
func formatWeight(w float64) string {
s := strconv.FormatFloat(w, 'f', -1, 64)
if !strings.Contains(s, ".") {
s += ".0"
}
return s
}
package antispam
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"net/netip"
"path"
"slices"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/netutil"
)
// OptionalUsable reports why CSM cannot write optional target t, or nil:
// its directory must exist, and every parent and the file must be protected
// root-owned objects. The worker backs up and replaces a protected file that
// is not CSM's. Otherwise the daemon leaves out the feed or redirect that
// needs t, and the worker refuses to write it.
func OptionalUsable(t TargetFacts) error {
if t.Err != nil {
if errors.Is(t.Err, fs.ErrNotExist) {
return fmt.Errorf("the directory %s is missing", path.Dir(t.Path))
}
return fmt.Errorf("cannot check %s; check file access", t.Path)
}
if len(t.Parents) == 0 {
return fmt.Errorf("the parent directories of %s were not checked", t.Path)
}
for _, p := range t.Parents {
if !p.Mode.IsDir() {
return fmt.Errorf("%s is not a directory", p.Path)
}
if problem := unprotected(p); problem != "" {
return fmt.Errorf("%s %s", p.Path, problem)
}
}
if t.Missing {
return nil
}
if !t.File.Mode.IsRegular() {
return fmt.Errorf("%s is not a regular file", t.Path)
}
if problem := unprotected(t.File); problem != "" {
return fmt.Errorf("%s %s", t.Path, problem)
}
return nil
}
// SecretFileMode is the mode of a file holding a secret, such as
// override.d/rbl.conf with the DQS key: rspamd reads it through its group,
// nobody else can.
const SecretFileMode fs.FileMode = 0o640
// SecretPaths lists every file that can hold a secret: the DQS key in the
// rbl override, the learning keys in the fuzzy rules and worker.
func SecretPaths() []string {
return []string{FeedRBLPath, LearningCheckPath, LearningWorkerPath}
}
// SecretTargets lists the SecretPaths that hold a secret under s.
func (s Settings) SecretTargets() []string {
var out []string
if s.HasFeed(FeedSpamhausDQS) {
out = append(out, FeedRBLPath)
}
if s.Learning {
out = append(out, LearningCheckPath, LearningWorkerPath)
}
return out
}
// secretFileOK reports whether secret file p is root-owned, in rspamd's
// group and exactly mode SecretFileMode, without set-id or sticky bits.
func secretFileOK(f HostFacts, p string) bool {
for _, t := range f.Targets {
if t.Path == p {
return t.Err == nil && !t.Missing && t.File.UID == 0 && f.RspamdGID >= 0 &&
t.File.GID == f.RspamdGID && t.File.Mode == SecretFileMode
}
}
return false
}
var errNoSystemNameserver = errors.New("rspamd has no nameserver in /etc/resolv.conf")
// Nameservers returns the resolvers rspamd uses: options.dns.nameserver
// (or its aliases) when set, else the nameserver lines of resolv.conf. An
// entry CSM cannot parse is an error, never proof of a private resolver.
func Nameservers(options []byte, resolvConf string) ([]netip.AddrPort, error) {
opts, err := dumpObjectAs[json.RawMessage](options, "options")
if err != nil {
return nil, errors.New("rspamd's options are not readable")
}
var settings []json.RawMessage
if raw, ok := opts["dns_nameserver"]; ok {
settings = append(settings, raw)
}
if raw, ok := opts["dns"]; ok {
dns, err := dumpObjectAs[json.RawMessage](raw, "dns")
if err != nil {
return nil, errors.New("rspamd's DNS options are not readable")
}
for _, name := range []string{"nameserver", "server"} {
if raw, ok := dns[name]; ok {
settings = append(settings, raw)
}
}
}
// configdump omits UCL priorities, so competing aliases cannot establish
// which resolver rspamd selected.
if len(settings) > 1 {
return nil, errors.New("rspamd has multiple nameserver settings; use only dns.nameserver")
}
var entries []string
if len(settings) == 1 {
raw := settings[0]
var one string
if err := json.Unmarshal(raw, &one); err == nil {
entries = []string{one}
} else if err := json.Unmarshal(raw, &entries); err != nil {
return nil, errors.New("rspamd's dns.nameserver is neither a string nor a list of strings")
}
} else if entries = resolvConfEntries(resolvConf); len(entries) == 0 {
return nil, errNoSystemNameserver
}
var out []netip.AddrPort
for _, entry := range entries {
// rspamd compares rotation names without case.
for _, prefix := range []string{"master-slave:", "round-robin:", "random:", "hash:"} {
if len(entry) >= len(prefix) && strings.EqualFold(entry[:len(prefix)], prefix) {
entry = entry[len(prefix):]
break
}
}
parts := strings.FieldsFunc(entry, func(r rune) bool { return strings.ContainsRune(";, \n\r\t", r) })
if len(parts) == 0 {
return nil, errors.New("rspamd has an empty nameserver entry")
}
for _, part := range parts {
addr, err := nameserverEndpoint(part)
if err != nil {
return nil, err
}
out = append(out, addr)
}
}
if len(out) == 0 {
return nil, errors.New("rspamd has no nameserver")
}
return out, nil
}
// resolvConfEntries returns the nameserver addresses a resolv.conf lists.
func resolvConfEntries(text string) []string {
var out []string
for _, line := range strings.Split(text, "\n") {
line, _, _ = strings.Cut(line, "#")
if fields := strings.Fields(line); len(fields) >= 2 && fields[0] == "nameserver" {
out = append(out, fields[1])
}
}
return out
}
// nameserverEndpoint reads one rspamd nameserver entry: an address, optionally
// with a port and a weight (host:port:weight), IPv6 in brackets when a port
// follows. Keep the port and IPv6 scope so probes reach the same resolver.
func nameserverEndpoint(s string) (netip.AddrPort, error) {
if addr, err := netip.ParseAddr(s); err == nil {
return netip.AddrPortFrom(addr, 53), nil
}
invalid := errors.New("rspamd has an invalid nameserver; use an IP address with an optional numeric port and weight")
host, suffix := s, ""
port := uint16(53)
if strings.HasPrefix(s, "[") {
end := strings.IndexByte(s, ']')
if end < 0 {
return netip.AddrPort{}, invalid
}
host, suffix = s[1:end], s[end+1:]
} else if i := strings.Index(s, ":"); i > 0 {
host, suffix = s[:i], s[i:]
}
if suffix != "" {
if suffix[0] != ':' {
return netip.AddrPort{}, invalid
}
parts := strings.Split(suffix[1:], ":")
if len(parts) > 2 {
return netip.AddrPort{}, invalid
}
value, err := strconv.ParseUint(parts[0], 10, 16)
if err != nil {
return netip.AddrPort{}, invalid
}
port = uint16(value)
if len(parts) == 2 {
// rspamd weights are unsigned integers.
if _, err := strconv.ParseUint(parts[1], 10, 32); err != nil {
return netip.AddrPort{}, invalid
}
}
}
addr, err := netip.ParseAddr(host)
if err != nil {
return netip.AddrPort{}, invalid
}
return netip.AddrPortFrom(addr, port), nil
}
// ownResolver reports whether rspamd's queries stay on this host, a private
// network, or a resolver the operator lists in infra_ips. Lists refuse public
// resolvers. The test is a plain prefix match: the wider
// infra test also trusts Cloudflare's ranges, which hold a public resolver.
func ownResolver(a netip.Addr, infra []netip.Prefix) bool {
a = a.WithZone("").Unmap()
return a.IsLoopback() || a.IsPrivate() || slices.ContainsFunc(infra, func(p netip.Prefix) bool { return p.Contains(a) })
}
// ParseInfraNets reads infra_ips entries that are addresses or CIDRs.
// Hostnames and invalid entries are skipped: the resolver check never
// resolves names.
func ParseInfraNets(entries []string) []netip.Prefix {
var out []netip.Prefix
for _, e := range entries {
if n := netutil.ParseCIDROrIP(e); n != nil {
a, _ := netip.AddrFromSlice(n.IP)
bits, _ := n.Mask.Size()
out = append(out, netip.PrefixFrom(a.Unmap(), bits))
}
}
return out
}
// ResolvedStub identifies systemd-resolved's standard stub endpoints.
// Other ports on those addresses can belong to an independent resolver.
func ResolvedStub(server netip.AddrPort) bool {
a := server.Addr().Unmap()
return server.Port() == 53 && (a == netip.AddrFrom4([4]byte{127, 0, 0, 53}) || a == netip.AddrFrom4([4]byte{127, 0, 0, 54}))
}
// resolverProblem returns why rspamd's nameservers would be refused by
// blocklists, or "". A systemd-resolved stub is judged by its upstreams.
func resolverProblem(servers []netip.AddrPort, f HostFacts) string {
public := func(a netip.AddrPort) bool { return !ownResolver(a.Addr(), f.InfraNets) }
if slices.ContainsFunc(servers, ResolvedStub) {
if f.ResolvedDNSErr != nil {
return "CSM cannot read systemd-resolved's live DNS servers through busctl. " + resolverFix
}
if len(f.ResolvedDNS) == 0 || slices.ContainsFunc(f.ResolvedDNS, func(a netip.Addr) bool { return !ownResolver(a, f.InfraNets) }) {
return "rspamd uses systemd-resolved, which forwards to a public or unknown nameserver. " + resolverFix
}
servers = slices.DeleteFunc(slices.Clone(servers), ResolvedStub)
}
if slices.ContainsFunc(servers, public) {
return "rspamd uses a public nameserver. " + resolverFix
}
return ""
}
const resolverFix = "DNS blocklists refuse queries from public resolvers. Point rspamd at a local recursive resolver or at your own resolver listed in infra_ips, for example dns { nameserver = ['127.0.0.1']; } in /etc/rspamd/local.d/options.inc"
// ResolveFeeds returns the settings this host can take. Map redirects whose
// file CSM cannot own are skipped; DNS feeds go off when their files cannot
// be written or rspamd's resolver is not local; spamhaus_dqs goes off when
// its key file or rspamd's group is unusable. Each is a Warning: a feed
// problem never blocks core.
func ResolveFeeds(s Settings, f HostFacts) (Settings, []Problem) {
// Dropping a temporarily unusable feed must not alter the desired settings
// that callers retain for the next pass.
s.Feeds = slices.Clone(s.Feeds)
if ok, why := FeedsSupported(f.RspamdVersion); !ok {
return withoutFeeds(s, why)
}
var problems []Problem
targets := make(map[string]TargetFacts, len(f.Targets))
for _, t := range f.Targets {
targets[t.Path] = t
}
usable := func(p string) error {
t, ok := targets[p]
if !ok {
return fmt.Errorf("%s was not checked", p)
}
return OptionalUsable(t)
}
s.SkipMaps = nil
if s.Redirects() {
for _, name := range MapRedirectNames() {
if err := usable(RedirectPath(name)); err != nil {
s.SkipMaps = append(s.SkipMaps, name)
problems = append(problems, Problem{Code: CodeFeed, Severity: Warning, Key: FeedRspamdMaps,
Detail: fmt.Sprintf("%v, so rspamd keeps fetching that map from rspamd.com; remove or rename it so CSM can point it at a local copy", err)})
}
}
}
drop := func(feeds []string, detail string) {
for _, name := range feeds {
problems = append(problems, Problem{Code: CodeFeed, Severity: Warning, Key: name,
Detail: "feed " + name + " is off: " + detail})
}
s.Feeds = slices.DeleteFunc(s.Feeds, func(n string) bool { return slices.Contains(feeds, n) })
}
var dns []string
for _, feed := range s.DNSFeeds() {
dns = append(dns, feed.Name)
}
if len(dns) > 0 {
reason := ""
for _, p := range []string{FeedRBLPath, FeedRBLGroupPath} {
if err := usable(p); err != nil && reason == "" {
reason = err.Error()
}
}
if reason == "" {
servers, err := Nameservers([]byte(f.RspamdOptions.Stdout), f.ResolvConf)
switch {
case f.RspamdOptions.Err != nil:
reason = "CSM cannot read which nameservers rspamd uses. " + resolverFix
case errors.Is(err, errNoSystemNameserver) && f.ResolvConfErr != nil:
reason = "CSM cannot read /etc/resolv.conf, which rspamd uses without dns.nameserver. " + resolverFix
case err != nil:
reason = err.Error() + ". " + resolverFix
default:
reason = resolverProblem(servers, f)
}
}
if reason != "" {
drop(dns, reason)
}
}
if s.HasFeed(FeedCSMIPMap) {
if err := usable(RedirectPath(RedirectMultimap)); err != nil {
drop([]string{FeedCSMIPMap}, err.Error())
} else if err := f.IPMap.UsableByRspamd(f.RspamdGID); err != nil {
drop([]string{FeedCSMIPMap}, err.Error())
}
}
if s.HasFeed(FeedSpamhausDQS) {
key, err := LoadDQSKey(f.DQSKey)
exposed, why := f.RspamdLog.Exposed(f.Socket.RspamdUID, f.RspamdGID)
switch {
case err != nil:
drop([]string{FeedSpamhausDQS}, err.Error())
case exposed:
drop([]string{FeedSpamhausDQS}, why+", and rspamd writes failed DQS lookups, which contain the key, to it; "+
"use a private file log owned by root or rspamd, with only root or rspamd group access and no extra ACLs")
case f.RspamdGID < 0:
drop([]string{FeedSpamhausDQS}, "rspamd's group is unknown, so CSM cannot make the key file readable by rspamd only")
default:
s.DQSKey = key
}
}
if !s.HasFeed(FeedSpamhausDQS) {
s.DQSKey = Secret{}
}
return s, problems
}
// FileFacts is one file as the collector found it; File is set unless
// Missing or Err.
type FileFacts struct {
Missing bool
File PathFacts
Parents []PathFacts
Err error
}
// UsableByRspamd checks the IP map's private, root-owned file contract.
// Both resolution and the worker check it because the file can change
// while a request waits to apply.
func (f FileFacts) UsableByRspamd(gid int) error {
switch {
case gid < 0:
return errors.New("rspamd's group is unknown, so CSM cannot make the IP map readable by rspamd only")
case f.Missing:
return errors.New("CSM has not written the IP map yet")
case f.Err != nil || f.File.Path == "":
return errors.New("CSM cannot check the IP map; check file access")
case f.File.Mode != 0o640 || f.File.UID != 0 || f.File.GID != gid || f.File.ExtACL:
return errors.New("the IP map must be a regular root-owned file, mode 0640, in rspamd's group, with no extra ACLs")
}
if len(f.Parents) == 0 {
return errors.New("the IP map's parent directories were not checked")
}
for _, p := range f.Parents {
if !p.Mode.IsDir() {
return fmt.Errorf("IP map parent %s is not a directory", p.Path)
}
if why := unprotected(p); why != "" {
return fmt.Errorf("IP map parent %s %s", p.Path, why)
}
search := fs.FileMode(0o001)
if p.GID == gid {
search = 0o010
}
if p.Mode&search == 0 {
return fmt.Errorf("rspamd cannot traverse IP map parent %s", p.Path)
}
}
return nil
}
// LogFacts is rspamd's log destination as the collector found it. Type is
// rspamd's logging.type; for a file, File and Parents describe the log
// file (unset when Missing) and its directories.
type LogFacts struct {
Type string
Path string
Missing bool
File PathFacts
Parents []PathFacts
Err error
}
// Exposed reports whether users other than root and rspamd may read
// rspamd's log: rspamd writes every failed lookup name there, and DQS
// lookup names hold the key. A log CSM cannot check counts as exposed; a
// file rspamd has yet to create counts as readable unless a private parent
// protects it. Console and syslog routing cannot establish file privacy.
func (l LogFacts) Exposed(uid, gid int) (bool, string) {
return l.exposed(uid, func(id int) bool { return id == 0 || gid >= 0 && id == gid })
}
func (l LogFacts) exposed(uid int, trustedGroup func(int) bool) (bool, string) {
unknown := "CSM cannot check who can read rspamd's log"
switch {
case l.Err != nil:
return true, unknown
case l.Type != "file" || !path.IsAbs(l.Path) || len(l.Parents) == 0:
return true, unknown
case path.Clean(l.Path) != l.Path:
// Cleaned parent facts can omit a symlink traversed before "..".
return true, unknown
}
protected := func(p PathFacts) bool {
return (p.UID == 0 || uid >= 0 && p.UID == uid) && !p.ExtACL &&
p.Mode.Perm()&0o002 == 0 && (p.Mode.Perm()&0o020 == 0 || trustedGroup(p.GID))
}
private := func(p PathFacts, other, group fs.FileMode) bool {
return p.Mode.Perm()&other == 0 && (p.Mode.Perm()&group == 0 || trustedGroup(p.GID))
}
closed := false
for _, p := range l.Parents {
if !p.Mode.IsDir() || !protected(p) {
return true, unknown
}
closed = closed || private(p, 0o001, 0o010)
}
if !l.Missing && (!l.File.Mode.IsRegular() || !protected(l.File)) {
return true, unknown
}
if !closed && (l.Missing || !private(l.File, 0o004, 0o040)) {
return true, "rspamd's log " + l.Path + " is readable by other users"
}
return false, ""
}
// minFeedsRspamdMajor is the rspamd release CSM's feed files and map
// redirects were verified on.
const minFeedsRspamdMajor = 4
// ResolveScores leaves out the symbol weights file when CSM cannot write it
// safely, with a Warning: rspamd then keeps its own weights. Otherwise it
// leaves out each weight the host's own group files set.
func ResolveScores(s Settings, f HostFacts) (Settings, []Problem) {
s.SkipScores, s.HostScores = false, nil
err := fmt.Errorf("%s was not checked", ScoresPath)
for _, t := range f.Targets {
if t.Path == ScoresPath {
err = OptionalUsable(t)
break
}
}
if err == nil {
s.HostScores = hostScores(f.GroupFiles)
return s, nil
}
s.SkipScores = true
return s, []Problem{{Code: CodeScores, Severity: Warning,
Detail: fmt.Sprintf("%v, so rspamd keeps its full weight for hidden text, which can tag newsletters", err)}}
}
// FeedsSupported reports whether rspamd is recent enough for feed files
// and map redirects, and why not.
func FeedsSupported(version CommandResult) (bool, string) {
if version.Err != nil {
return false, "rspamd's version is unknown"
}
v, major, err := ParseRspamdVersion(version.Stdout)
switch {
case err != nil:
return false, "rspamd's version is unknown"
case major < minFeedsRspamdMajor:
return false, "rspamd " + v + " is older than 4"
}
return true, ""
}
// withoutFeeds leaves rspamd's own lists and maps alone, as rspamd_maps
// does, rather than risk files rspamd rejects, which would block core.
func withoutFeeds(s Settings, why string) (Settings, []Problem) {
var problems []Problem
for _, feed := range s.DNSFeeds() {
problems = append(problems, Problem{Code: CodeFeed, Severity: Warning, Key: feed.Name,
Detail: "feed " + feed.Name + " is off: " + why + "; CSM's feed rules need rspamd 4 or later"})
}
if s.HasFeed(FeedCSMIPMap) {
problems = append(problems, Problem{Code: CodeFeed, Severity: Warning, Key: FeedCSMIPMap,
Detail: "feed " + FeedCSMIPMap + " is off: " + why + "; CSM's feed rules need rspamd 4 or later"})
}
if s.Redirects() {
problems = append(problems, Problem{Code: CodeFeed, Severity: Warning, Key: FeedRspamdMaps,
Detail: why + ", so CSM leaves rspamd's own maps alone and they keep downloading from rspamd.com; upgrade rspamd to 4 or later"})
}
s.Feeds = slices.DeleteFunc(slices.Clone(s.Feeds), func(name string) bool {
f, _ := FeedByName(name)
return f.DNS() || name == FeedCSMIPMap
})
if !s.HasFeed(FeedRspamdMaps) {
s.Feeds = append(s.Feeds, FeedRspamdMaps)
}
s.SkipMaps, s.DQSKey = nil, Secret{}
return s, problems
}
package antispam
import (
"encoding/json"
"errors"
"fmt"
"net/url"
"slices"
"sort"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/ipmap"
)
// LoadedRBLRules reads `rspamadm configdump -j rbl` and returns enabled
// rbls rule names, sorted. Ambiguous entries and legacy overrides are errors.
func LoadedRBLRules(out []byte) ([]string, error) {
dump, err := dumpObjectAs[json.RawMessage](out, "rbl")
if err != nil {
return nil, err
}
rules, err := dumpObjectAs[json.RawMessage](dump["rbls"], "rbl rules")
if err != nil {
return nil, err
}
if raw, present := dump["rules"]; present {
// Rspamd merges the legacy alias into rbls, including overrides
// of names already there. CSM renders this object empty.
legacy, err := dumpObjectAs[json.RawMessage](raw, "rbl legacy rules")
if err != nil || len(legacy) != 0 {
return nil, errors.New("configdump rbl has legacy rule overrides")
}
}
names := make([]string, 0, len(rules))
for name, raw := range rules {
rule, err := dumpObjectAs[json.RawMessage](raw, "rbl rule")
if err != nil {
return nil, err
}
enabled, disabled := true, false
for _, flag := range []struct {
key string
value *bool
}{{"enabled", &enabled}, {"disabled", &disabled}} {
if raw, present := rule[flag.key]; present {
// A null or non-boolean flag cannot prove the rule is on.
var value *bool
if err := json.Unmarshal(raw, &value); err != nil || value == nil {
return nil, errors.New("configdump rbl has a non-boolean rule switch")
}
*flag.value = *value
}
}
if enabled && !disabled {
names = append(names, name)
}
}
sort.Strings(names)
return names, nil
}
// CheckFeedsLoaded compares rspamd's loaded rbl state with s: the module
// is on exactly when s has a DNS feed, and its rules are exactly CSM's.
// Each string names one difference; none repeats configuration text.
func CheckFeedsLoaded(enabled map[string]bool, rules []string, s Settings) []string {
var want []string
for _, f := range s.DNSFeeds() {
for _, z := range f.Zones {
want = append(want, z.Rule)
}
}
if len(want) == 0 {
return nil // rbl must be off; checkRspamdModules reports it
}
if !enabled["rbl"] {
return []string{"rspamd's rbl module is off, so no DNS feed runs; an override defeats CSM's local.d/rbl.conf"}
}
sort.Strings(want)
var out []string
for _, r := range want {
if !slices.Contains(rules, r) {
out = append(out, fmt.Sprintf("rspamd did not load CSM's rule %s", r))
}
}
extra := 0
for _, r := range rules {
if !slices.Contains(want, r) {
extra++
}
}
if extra > 0 {
out = append(out, fmt.Sprintf("rspamd loaded %d blocklist rules CSM did not write; an override adds them next to CSM's override.d/rbl.conf", extra))
}
return out
}
// RspamdComPaths walks `rspamadm configdump -j` and returns sorted paths
// of remote rspamd.com references inside enabled modules. Sensitive object
// keys are redacted. Symbol descriptions under group are not modules.
func RspamdComPaths(dump []byte, enabled map[string]bool) ([]string, error) {
var root map[string]any
if err := json.Unmarshal(dump, &root); err != nil || root == nil {
return nil, errors.New("configdump is not a JSON object")
}
var out []string
var walk func(prefix string, v any)
walk = func(prefix string, v any) {
switch x := v.(type) {
case map[string]any:
for k, child := range x {
walk(prefix+"."+configPathKey(k), child)
}
case []any:
for i, child := range x {
walk(prefix+"."+strconv.Itoa(i), child)
}
case string:
if rspamdComReference(x) {
out = append(out, prefix)
}
}
}
for module, v := range root {
if module != "group" && enabled[module] {
walk(configPathKey(module), v)
}
}
sort.Strings(out)
return out, nil
}
// Object keys can hold DQS credentials or whole URLs. Keep short plain
// identifiers so neither is copied into a finding.
func configPathKey(key string) string {
if len(key) == 0 || len(key) >= dqsKeyLen {
return redacted
}
for _, c := range key {
if c != '_' && c != '-' && (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') {
return redacted
}
}
return key
}
// Map types and signing/fallback prefixes precede the URL. Match its host,
// not a local filename, description, URL path, or similarly named domain.
// Bare hosts and service discovery references cover DNS and fuzzy servers.
func rspamdComReference(value string) bool {
value = strings.TrimSpace(value)
if kind, rest, ok := strings.Cut(value, ";"); ok && slices.Contains([]string{
"set", "hash", "radix", "regexp", "regexp_multi", "glob", "glob_multi", "plain", "cdb",
}, kind) {
value = rest
}
for {
prefix, rest, ok := strings.Cut(value, "+")
if !ok || prefix != "sign" && prefix != "fallback" && !strings.HasPrefix(prefix, "key=") && !strings.HasPrefix(prefix, "service=") {
break
}
value = rest
}
if !strings.Contains(value, "://") {
value = "//" + value
}
u, err := url.Parse(value)
if err != nil {
return false
}
switch u.Scheme {
case "http", "https":
case "":
if u.Path != "" || u.RawQuery != "" || u.Fragment != "" || u.User != nil {
return false
}
default:
return false
}
host := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".")
return host == "rspamd.com" || strings.HasSuffix(host, ".rspamd.com")
}
// CheckIPMapLoaded compares rspamd's loaded multimap rules with the CSM_IP
// rule CSM writes. Each string names one difference; none repeats
// configuration text.
func CheckIPMapLoaded(enabled map[string]bool, multimap []byte) []string {
if !enabled["multimap"] {
return []string{"rspamd's multimap module is off, so CSM's IP map cannot score mail"}
}
rules, err := dumpObjectAs[json.RawMessage](multimap, "multimap")
if err != nil {
return []string{"rspamd's multimap rules are not readable"}
}
raw, ok := rules["CSM_IP"]
if !ok {
return []string{"rspamd did not load CSM's CSM_IP rule"}
}
fields, err := dumpObjectAs[json.RawMessage](raw, "CSM_IP")
if err != nil {
return []string{"rspamd's CSM_IP rule is not a rule CSM wrote"}
}
// Extra options can change matching or force an action independently
// of the score. Only the description is free to differ from CSM's rule.
for key := range fields {
switch key {
case "type", "map", "symbols", "score", "description", "enabled":
default:
return []string{"rspamd's CSM_IP rule has options CSM did not write; an override changes it"}
}
}
var rule struct {
Type string `json:"type"`
Map json.RawMessage `json:"map"`
Symbols []string `json:"symbols"`
Score *float64 `json:"score"`
Enabled *bool `json:"enabled"`
}
if json.Unmarshal(raw, &rule) != nil {
return []string{"rspamd's CSM_IP rule is not a rule CSM wrote"}
}
var path string
if json.Unmarshal(rule.Map, &path) != nil {
var paths []string
if json.Unmarshal(rule.Map, &paths) == nil && len(paths) == 1 {
path = paths[0]
}
}
var want []string
for _, s := range ipmap.Symbols() {
want = append(want, s.Name)
}
// Rspamd registers each symbol independently; array order has no effect.
slices.Sort(want)
slices.Sort(rule.Symbols)
if rule.Type != "ip" || path != ipmap.DefaultPath || !slices.Equal(rule.Symbols, want) ||
rule.Score == nil || *rule.Score != ipmap.RuleScore ||
fields["enabled"] != nil && (rule.Enabled == nil || !*rule.Enabled) {
return []string{"rspamd's CSM_IP rule differs from the one CSM wrote; an override changes it"}
}
return nil
}
package antispam
import (
"strconv"
"strings"
)
// FilterSpamUses returns the 1-based lines of Exim system filter data items
// that read $spam_*. The filter runs after CSM's scan for every account.
func FilterSpamUses(filter string) []int {
var out []int
var token strings.Builder
line, start := 1, 1
quoted := false
flush := func() {
if readsExpandedSpamVariable(token.String()) && (len(out) == 0 || out[len(out)-1] != start) {
out = append(out, start)
}
token.Reset()
}
for i := 0; i < len(filter); i++ {
c := filter[i]
if !quoted && filterSeparator(c) {
flush()
if c == '\n' {
line++
}
continue
}
if !quoted && c == '#' && token.Len() == 0 {
for i < len(filter) && filter[i] != '\n' {
i++
}
if i < len(filter) {
line++
}
continue
}
if token.Len() == 0 && !quoted {
start = line
}
if c == '"' {
if quoted {
flush()
}
quoted = !quoted
continue
}
if quoted && c == '\\' && i+1 < len(filter) {
i++
c = filter[i]
if c == '\n' {
line++
for i+1 < len(filter) && (filter[i+1] == ' ' || filter[i+1] == '\t' || filter[i+1] == '\r') {
i++
}
continue
}
// Quoted data is unescaped before expansion. A decoded dollar
// can read a variable; a decoded backslash can suppress it.
switch c {
case 'n':
c = '\n'
case 'r':
c = '\r'
case 't':
c = '\t'
default:
base, limit, first := 8, 3, i
if c == 'x' {
base, limit, first = 16, 2, i+1
}
end := first
for end < len(filter) && end-first < limit && filterDigit(filter[end], base) {
end++
}
if end > first {
n, _ := strconv.ParseUint(filter[first:end], base, 16)
c, i = byte(n&0xff), end-1
}
}
} else if c == '\n' {
line++
}
token.WriteByte(c)
}
flush()
return out
}
func filterSeparator(c byte) bool {
return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' || c == '(' || c == ')'
}
func filterDigit(c byte, base int) bool {
if base == 8 {
return c >= '0' && c <= '7'
}
return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F'
}
func readsExpandedSpamVariable(s string) bool {
literal := false
for i := 0; i < len(s); i++ {
if s[i] == '\\' && i+1 < len(s) {
if s[i+1] == 'N' {
literal = !literal
}
i++
continue
}
if !literal && s[i] == '$' && (strings.HasPrefix(s[i+1:], "spam_") || strings.HasPrefix(s[i+1:], "{spam_")) {
return true
}
}
return false
}
package antispam
import (
"errors"
"fmt"
"slices"
"strings"
)
// CheckGenerated verifies that a generated exim.conf carries CSM's hooks
// exactly as rendered: the scanner block in the main configuration, the RCPT
// hook in the RCPT ACL, and the scan hook before the post hook in the DATA
// ACL. rcptACL and dataACL come
// from `exim -bP acl_smtp_rcpt acl_smtp_data`. Errors name blocks, never
// configuration text. Conditional blocks or boundaries cannot prove that the
// hooks are active; macro substitution must leave their statements unchanged.
func CheckGenerated(conf string, h EximHooks, rcptACL, dataACL string) error {
if rcptACL == "" || dataACL == "" || rcptACL == dataACL {
return errors.New("the RCPT and DATA ACL names must be two different names")
}
blocks, err := locateBlocks(conf, "", "", true)
if err != nil {
return err
}
// Counts name what is missing without repeating ACL names from the file.
var nMain, nRCPT, nData, nOther int
for _, b := range blocks {
switch b.ACL {
case "":
nMain++
case rcptACL:
nRCPT++
case dataACL:
nData++
default:
nOther++
}
}
if nMain != 1 || nRCPT != 1 || nData != 2 || nOther != 0 {
return fmt.Errorf("found %d scanner, %d RCPT, %d DATA and %d misplaced CSM blocks; want 1, 1, 2 and 0", nMain, nRCPT, nData, nOther)
}
lines := strings.Split(conf, "\n")
text := func(b Placement) string { return strings.Join(lines[b.Begin-1:b.End], "\n") + "\n" }
var data []string
for _, b := range blocks {
switch b.ACL {
case "":
if text(b) != h.Config {
return errors.New("the scanner block differs from the rendered spamd_address")
}
case rcptACL:
if text(b) != h.RCPT {
return errors.New("the RCPT ACL hook differs from the rendered hook")
}
default:
data = append(data, text(b))
}
}
if data[0] != h.Scan || data[1] != h.Post {
return errors.New("the DATA ACL hooks differ from the rendered scan and post hooks, or are out of order")
}
return nil
}
// CheckGeneratedBlocks verifies prior blocks after a rollback, including
// partial installations that cannot satisfy CheckGenerated's full layout.
// Owned markers alone do not prove that Exim executes the restored text.
func CheckGeneratedBlocks(conf string, want map[string][]string) error {
blocks, err := locateBlocks(conf, "", "", true)
if err != nil {
return errors.New("cannot verify CSM blocks in the restored Exim configuration")
}
got := make(map[string][]string)
lines := strings.SplitAfter(conf, "\n")
for _, b := range blocks {
got[b.ACL] = append(got[b.ACL], strings.Join(lines[b.Begin-1:b.End], ""))
}
if len(got) != len(want) {
return errors.New("the restored Exim configuration differs from the prior CSM blocks")
}
for acl, prior := range want {
if !slices.Equal(got[acl], prior) {
return errors.New("the restored Exim configuration differs from the prior CSM blocks")
}
}
return nil
}
package antispam
import (
"encoding/json"
"strings"
)
// hostScores lists the symbols CSM weighs that the host's group files
// define. rspamd loads those files at the same priority as CSM's
// groups.conf and lets the later include, CSM's, win; the operator's
// weight must stay. A file CSM cannot read may define any of them.
func hostScores(files map[string]GroupFile) []string {
var out []string
for _, w := range scoreWeights {
for _, f := range files {
if f.Err != nil || definesSymbol(f.Content, w.symbol) {
out = append(out, w.symbol)
break
}
}
}
return out
}
// definesSymbol conservatively recognizes symbol tokens, including escaped
// keys and array definitions. Macros and unfamiliar string syntax may hide
// definitions, so leave their weights to the host without evaluating UCL or
// following includes. Comments must be lexed with strings, not stripped
// first: a quoted comment marker must never erase a later definition.
func definesSymbol(content, symbol string) bool {
for i := 0; i < len(content); {
switch {
case strings.ContainsRune(" \t\r\n\v\f{}[]=:;,()", rune(content[i])):
i++
case content[i] == '#':
for i < len(content) && content[i] != '\n' {
i++
}
case strings.HasPrefix(content[i:], "/*"):
i += 2
depth := 1
quoted := false
for i < len(content) && depth > 0 {
switch {
case content[i] == '"' && content[i-1] != '\\':
// UCL ignores comment delimiters inside quoted examples.
quoted = !quoted
i++
case quoted:
i++
case strings.HasPrefix(content[i:], "/*"):
depth++
i += 2
case strings.HasPrefix(content[i:], "*/"):
depth--
i += 2
default:
i++
}
}
if depth != 0 {
return true
}
case content[i] == '"' || content[i] == '\'':
start, quote := i, content[i]
i++
for i < len(content) && content[i] != quote {
if content[i] == '\\' {
i++
}
i++
}
if i >= len(content) {
return true
}
i++
value := content[start+1 : i-1]
if quote == '"' {
if json.Unmarshal([]byte(content[start:i]), &value) != nil {
return true
}
} else {
value = strings.NewReplacer("\\'", "'", "\\\n", "").Replace(value)
}
if value == symbol {
return true
}
default:
start := i
for i < len(content) && !strings.ContainsRune(" \t\r\n\v\f{}[]=:;,()#\"'", rune(content[i])) && !strings.HasPrefix(content[i:], "/*") {
i++
}
token := content[start:i]
macro := len(token) > 1 && token[0] == '.' && (token[1] >= 'a' && token[1] <= 'z' || token[1] >= 'A' && token[1] <= 'Z')
if token == symbol || macro || strings.HasPrefix(token, "<<") {
return true
}
}
}
return false
}
package hostfacts
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"strings"
"time"
)
const (
canaryTimeout = 5 * time.Second
maxCanaryBody = 64 << 10
maxCanaryReply = 2 * maxCanaryBody
// canaryMessage is the smallest message rspamd scans.
canaryMessage = "From: <canary@csm.invalid>\r\nSubject: csm canary\r\n\r\ncanary\r\n"
)
// Canary scans one message from the canary address with only the CSM_IP
// rule enabled and returns the generation its CSM_IP_CANARY option names
// ("" when the symbol is absent). no_log and no_stat keep the scan out of
// rspamd's history and statistics. Errors never repeat the reply.
func Canary(ctx context.Context, socket string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, canaryTimeout)
defer cancel()
var d net.Dialer
conn, err := d.DialContext(ctx, "unix", socket)
if err != nil {
return "", err
}
defer func() { _ = conn.Close() }()
if deadline, ok := ctx.Deadline(); ok {
_ = conn.SetDeadline(deadline)
}
stop := context.AfterFunc(ctx, func() { _ = conn.SetDeadline(time.Now()) })
defer stop()
req := "POST /checkv2 HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n" +
"IP: 192.0.2.255\r\nSettings: {symbols_enabled = [\"CSM_IP\"]}\r\nFlags: no_log,no_stat\r\n" +
fmt.Sprintf("Content-Length: %d\r\n\r\n", len(canaryMessage)) + canaryMessage
if _, err = io.WriteString(conn, req); err != nil {
return "", err
}
return readCanaryReply(conn)
}
func readCanaryReply(conn io.ReadCloser) (string, error) {
reply := &io.LimitedReader{R: conn, N: maxCanaryReply + 1}
resp, err := http.ReadResponse(bufio.NewReader(reply), nil)
if err != nil {
return "", errors.New("rspamd /checkv2 returned an invalid HTTP response")
}
defer func() {
// Closing an unread HTTP body can drain it. This connection is never
// reused, so close it first rather than wait for a rejected reply.
_ = conn.Close()
_ = resp.Body.Close()
}()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("rspamd /checkv2 returned HTTP %d", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxCanaryBody+1))
if err != nil || len(body) > maxCanaryBody {
return "", errors.New("rspamd /checkv2 reply could not be read")
}
if reply.N == 0 {
return "", errors.New("rspamd /checkv2 reply is too large")
}
var result struct {
Symbols map[string]struct {
Options []string `json:"options"`
} `json:"symbols"`
}
if json.Unmarshal(body, &result) != nil {
return "", errors.New("rspamd /checkv2 reply is not JSON")
}
for _, o := range result.Symbols["CSM_IP_CANARY"].Options {
if gen, ok := strings.CutPrefix(o, "g"); ok {
// The writer emits a truncated SHA-256, not arbitrary reply text
// that could otherwise be repeated in a stale-map finding.
if len(gen) != 12 || strings.IndexFunc(gen, func(c rune) bool {
return (c < '0' || c > '9') && (c < 'a' || c > 'f')
}) >= 0 {
return "", errors.New("rspamd /checkv2 returned an invalid map generation")
}
return gen, nil
}
}
return "", nil
}
package hostfacts
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"net"
"os"
"os/exec"
"os/user"
"path/filepath"
"slices"
"strconv"
"strings"
"syscall"
"time"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/antispam/dovecot"
"github.com/pidginhost/csm/internal/ipmap"
"github.com/pidginhost/csm/internal/platform"
)
// commandTimeout bounds the collector's runCommand calls.
const commandTimeout = 10 * time.Second
// maxConfigBytes bounds each configuration file the collector reads; cPanel's
// generated exim.conf is under 100 KiB.
const maxConfigBytes = 16 << 20
// Layout holds the host files Collect reads.
type Layout struct {
GeneratedConf string
LocalConf string
SpamdDisabled string
GlobalEnable string
Record string
ResolvConf string
// Targets are the files CSM writes on a cPanel host, PostfixTargets
// those on a Postfix host.
Targets []antispam.Target
PostfixTargets []antispam.Target
// IPMap is the CSM IP map file rspamd reads.
IPMap string
// RspamdLocalDir holds the operator's group files.
RspamdLocalDir string
// Learning fact sources.
ProcNetUDP string
ProcNetUDP6 string
CageFSSkeletonLog string
RoundcubeConfig string
CPanelPHP string
CPanelUsers string
}
type collector struct {
layout Layout
stack func() antispam.MailStack
run func(ctx context.Context, name string, args ...string) antispam.CommandResult
lookPath func(string) (string, error)
uid func(name string) (int, error)
gid func(name string) (int, error)
userName func(uid string) (string, error)
// primaryGID is a user's primary group.
primaryGID func(user string) (int, error)
// interfaceAddrs, readGroup and readPasswd feed controller exposure.
interfaceAddrs func() ([]net.Addr, error)
readGroup func() ([]byte, error)
readPasswd func() ([]byte, error)
// redisProbe reads the live Redis values learning depends on.
redisProbe func(ctx context.Context, endpoint string, password antispam.Secret, peer int) antispam.RedisFacts
// controllerGet reads one path from rspamd's private controller socket.
controllerGet func(ctx context.Context, socket, path string, limit int64, peer int) ([]byte, error)
// dovecot lists a Postfix host's mailboxes for learning.
dovecot func(context.Context) (dovecot.Snapshot, error)
}
func live() collector {
return collector{
layout: Layout{
GeneratedConf: antispam.GeneratedEximConfPath,
LocalConf: antispam.LocalEximConfPath,
SpamdDisabled: "/etc/spamddisable",
GlobalEnable: "/etc/global_spamassassin_enable",
Record: antispam.RecordPath,
ResolvConf: "/etc/resolv.conf",
Targets: antispam.MutationTargets(antispam.StackExim),
PostfixTargets: antispam.MutationTargets(antispam.StackPostfix),
IPMap: ipmap.DefaultPath,
RspamdLocalDir: antispam.RspamdLocalDir,
ProcNetUDP: procNetUDP,
ProcNetUDP6: procNetUDP6,
CageFSSkeletonLog: cageFSSkeletonLog,
RoundcubeConfig: roundcubeConfig,
CPanelPHP: cpanelPHP,
CPanelUsers: cpanelUsers,
},
redisProbe: RedisProbe,
controllerGet: ControllerGet,
dovecot: dovecot.Dovecot{}.Snapshot,
stack: DetectStack,
run: runCommand,
lookPath: exec.LookPath,
uid: lookupUID,
gid: lookupGID,
userName: lookupUserName,
primaryGID: lookupPrimaryGID,
interfaceAddrs: net.InterfaceAddrs,
readGroup: func() ([]byte, error) { return readAccountFile("/etc/group") },
readPasswd: func() ([]byte, error) { return readAccountFile("/etc/passwd") },
}
}
// DetectStack reports the mail stack antispam integrates with: Exim on a
// cPanel host, Postfix on a host without cPanel, else none.
func DetectStack() antispam.MailStack {
return stackFor(platform.Detect().IsCPanel(), platform.DetectMTA())
}
func stackFor(cpanel bool, mta platform.MTAKind) antispam.MailStack {
switch {
case cpanel && mta == platform.MTAExim:
return antispam.StackExim
case !cpanel && mta == platform.MTAPostfix:
return antispam.StackPostfix
}
return antispam.StackNone
}
// RspamdGroup is rspamd's service group, or -1 when unknown (a root-started
// rspamd, or systemctl failing).
func RspamdGroup(ctx context.Context) int {
c := live()
return c.serviceGroup(c.run(ctx, "systemctl", antispam.UnitShowArgs(antispam.RspamdUnit)...))
}
// postfixService reads the unit running postfix's master: the Debian
// instance unit where it exists, else postfix.service.
func (c collector) postfixService(ctx context.Context) antispam.CommandResult {
show := func(unit string) antispam.CommandResult {
return c.run(ctx, "systemctl", "show", "-p", "LoadState", "-p", "ActiveState", unit)
}
r := show(antispam.PostfixInstanceUnit)
if r.Err == nil && antispam.ParseUnitShow(r.Stdout)["LoadState"] == "not-found" {
return show(antispam.PostfixUnit)
}
return r
}
// Collect reads every fact antispam.CheckHost evaluates for the scan socket.
// It changes nothing on the host.
func Collect(ctx context.Context, socket string) antispam.HostFacts {
return live().collect(ctx, socket)
}
func (c collector) collect(ctx context.Context, socket string) antispam.HostFacts {
f := antispam.HostFacts{Stack: c.stack()}
exim := f.Stack == antispam.StackExim
if exim {
f.MonitoredLog = platform.EximMainLog
f.EximVersion = c.run(ctx, "exim", "-bV")
f.EximOptions = c.run(ctx, "exim", "-bP", "prdr_enable", "chunking_advertise_hosts")
f.EximSettings = c.run(ctx, "exim", append([]string{"-bP"}, antispam.EximSettingNames()...)...)
for _, probe := range antispam.NamedListProbes() {
f.NamedLists = append(f.NamedLists, c.run(ctx, "exim", "-C", c.layout.GeneratedConf, "-be", probe))
}
}
f.RspamdVersion = c.run(ctx, "rspamd", "--version")
f.RspamadmVersion = c.run(ctx, "rspamadm", "--version")
f.RspamdOptions = c.run(ctx, "rspamadm", "configdump", "-j", "options")
f.RspamdActions = c.run(ctx, "rspamadm", "configdump", "-j", "actions")
f.RspamdModules = c.run(ctx, "rspamadm", "configdump", "-m")
f.RspamdPhishing = c.run(ctx, "rspamadm", "configdump", "-j", "phishing")
f.RspamdRBL = c.run(ctx, "rspamadm", "configdump", "-j", "rbl")
f.RspamdConfig = c.run(ctx, "rspamadm", "configdump", "-j")
f.RspamdLog = rspamdLog(f.RspamdConfig)
f.IPMap = fileFacts(c.layout.IPMap)
f.GroupFiles = groupFiles(c.layout.RspamdLocalDir)
f.RspamdService = c.run(ctx, "systemctl", antispam.UnitShowArgs(antispam.RspamdUnit)...)
switch f.Stack {
case antispam.StackExim:
f.EximService = c.run(ctx, "systemctl", antispam.UnitShowArgs(antispam.EximUnit)...)
case antispam.StackPostfix:
// -x expands $name references, so the values are the ones postfix uses.
f.Postconf = c.run(ctx, "postconf", append([]string{"-x", "-h"}, antispam.PostconfNames()...)...)
f.PostfixMaster = c.run(ctx, "postconf", append([]string{"-F"}, antispam.SmtpdChrootFields()...)...)
f.PostfixService = c.postfixService(ctx)
f.RspamdWorkers = c.run(ctx, "rspamadm", "configdump", "-j", "worker")
}
_, f.SystemdRunErr = c.lookPath("systemd-run")
if exim {
f.GeneratedConf, f.GeneratedErr = readText(c.layout.GeneratedConf)
if f.LocalConf, f.LocalErr = readText(c.layout.LocalConf); errors.Is(f.LocalErr, fs.ErrNotExist) {
f.LocalConf, f.LocalErr = "", nil
}
f.SpamdDisabled, f.SpamdDisabledErr = exists(c.layout.SpamdDisabled)
f.GlobalEnable, f.GlobalEnableErr = exists(c.layout.GlobalEnable)
}
f.ResolvConf, f.ResolvConfErr = readText(c.layout.ResolvConf)
servers, err := antispam.Nameservers([]byte(f.RspamdOptions.Stdout), f.ResolvConf)
if f.RspamdOptions.Err == nil && err == nil && slices.ContainsFunc(servers, antispam.ResolvedStub) {
f.ResolvedDNS, f.ResolvedDNSErr = c.resolvedDNS(ctx)
}
settings := map[string]string{}
if exim {
if f.EximSettings.Err == nil {
settings = antispam.ParseEximOptions(f.EximSettings.Stdout)
}
name, known := settings["system_filter"]
switch {
case !known && f.EximSettings.Err == nil:
f.SystemFilterErr = errors.New("exim -bP did not report system_filter")
case name != "" && (!filepath.IsAbs(name) || strings.Contains(name, "$")):
// The collector cannot reproduce Exim's delivery-time expansion
// or working directory.
f.SystemFilterErr = errors.New("system_filter is not a literal absolute path")
case name != "":
f.SystemFilter, f.SystemFilterErr = readText(name)
}
}
f.Targets = Targets(c.targets(f.Stack))
f.Record = c.record()
f.Socket = socketFacts(ctx, socket)
f.Socket.RspamdUID = c.serviceUser(f.RspamdService)
f.RspamdGID = c.serviceGroup(f.RspamdService)
f.Socket.EximUID = id(settings["exim_user"], c.uid)
f.Socket.EximGID = id(settings["exim_group"], c.gid)
c.controllerFacts(&f)
return f
}
// CollectInstalled reads only what shows whether CSM's integration is on
// the host: its mail stack, the files CSM writes on that stack, the
// generated Exim configuration and the completion record. Detecting the
// stack is the only step that can run commands. Other facts stay unset.
func CollectInstalled() antispam.HostFacts {
return live().installed()
}
func (c collector) installed() antispam.HostFacts {
f := antispam.HostFacts{Stack: c.stack()}
f.GeneratedConf, f.GeneratedErr = readText(c.layout.GeneratedConf)
f.Targets = Targets(c.targets(f.Stack))
f.Record = c.record()
return f
}
// targets are the files CSM writes on a host of stack.
func (c collector) targets(stack antispam.MailStack) []antispam.Target {
if stack == antispam.StackPostfix {
return c.layout.PostfixTargets
}
return c.layout.Targets
}
func (c collector) record() antispam.RecordFacts {
content, err := readRegularContent(c.layout.Record, antispam.MaxRecordBytes)
if errors.Is(err, fs.ErrNotExist) {
return antispam.RecordFacts{Missing: true}
}
return antispam.RecordFacts{Content: content, Err: err}
}
// serviceUser returns the scanner uid, or -1 when it cannot be inferred.
// A root-started rspamd can select a different worker user with -u.
func (c collector) serviceUser(show antispam.CommandResult) int {
props := antispam.ParseUnitShow(show.Stdout)
if show.Err != nil || props["LoadState"] != "loaded" {
return -1
}
uid := id(props["User"], c.uid)
if uid == 0 {
return -1
}
return uid
}
// serviceGroup returns the rspamd service's group: the unit's Group, else
// the primary group of its User, else -1. A root-started rspamd drops to a
// user and group of its own choosing, so its group is unknown.
func (c collector) serviceGroup(show antispam.CommandResult) int {
if c.serviceUser(show) < 0 {
return -1
}
props := antispam.ParseUnitShow(show.Stdout)
if g := props["Group"]; g != "" {
return id(g, c.gid)
}
// A numeric User is a uid, not its primary gid.
gid, err := c.primaryGID(props["User"])
if err != nil || gid < 0 {
return -1
}
return gid
}
func lookupPrimaryGID(name string) (int, error) {
u, err := user.Lookup(name)
if err != nil {
u, err = user.LookupId(name)
}
if err != nil {
return -1, err
}
return strconv.Atoi(u.Gid)
}
// id resolves a user or group given by name or number, or returns -1.
func id(value string, lookup func(string) (int, error)) int {
if value == "" {
return -1
}
n, err := strconv.Atoi(value)
if err != nil {
n, err = lookup(value)
}
if err != nil || n < 0 {
return -1
}
return n
}
func lookupUID(name string) (int, error) {
u, err := user.Lookup(name)
if err != nil {
return -1, err
}
return strconv.Atoi(u.Uid)
}
func lookupUserName(uid string) (string, error) {
u, err := user.LookupId(uid)
if err != nil {
return "", err
}
return u.Username, nil
}
func lookupGID(name string) (int, error) {
g, err := user.LookupGroup(name)
if err != nil {
return -1, err
}
return strconv.Atoi(g.Gid)
}
func runCommand(ctx context.Context, name string, args ...string) antispam.CommandResult {
return RunCommand(ctx, commandTimeout, name, args...)
}
// RunCommand runs one command with its own time limit. Standard input is
// /dev/null and output goes to fresh pipes, never the caller's streams.
func RunCommand(ctx context.Context, limit time.Duration, name string, args ...string) antispam.CommandResult {
ctx, cancel := context.WithTimeout(ctx, limit)
defer cancel()
var stdout, stderr strings.Builder
cmd := exec.CommandContext(ctx, name, args...) // #nosec G204 -- fixed tool names; arguments are constants, the validated config path or fixed probe expressions
cmd.Stdout, cmd.Stderr = &stdout, &stderr
// A timed-out rebuild must not leave children writing during rollback.
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
cmd.Cancel = func() error {
err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
if errors.Is(err, syscall.ESRCH) {
return os.ErrProcessDone
}
return err
}
cmd.WaitDelay = time.Second
err := cmd.Run()
if err != nil && cmd.Process != nil {
// A failing parent can exit while descendants still hold its pipes.
if stopErr := cmd.Cancel(); stopErr != nil && !errors.Is(stopErr, os.ErrProcessDone) {
err = errors.Join(err, stopErr)
}
}
if err != nil && ctx.Err() != nil {
err = fmt.Errorf("%s: %w", name, ctx.Err())
}
return antispam.CommandResult{Stdout: stdout.String(), Stderr: stderr.String(), Err: err}
}
// groupFiles also checks other groups: rspamd symbol names are global, so
// an operator can define a symbol outside its stock group.
func groupFiles(dir string) map[string]antispam.GroupFile {
out := make(map[string]antispam.GroupFile)
groups := antispam.ScoreGroups()
entries, listErr := os.ReadDir(dir)
for _, entry := range entries {
if group, ok := strings.CutSuffix(entry.Name(), "_group.conf"); ok && !slices.Contains(groups, group) {
groups = append(groups, group)
}
}
for _, group := range groups {
if listErr != nil && !errors.Is(listErr, fs.ErrNotExist) {
out[group] = antispam.GroupFile{Err: listErr}
continue
}
content, err := readText(filepath.Join(dir, group+"_group.conf"))
if errors.Is(err, fs.ErrNotExist) {
content, err = "", nil
}
out[group] = antispam.GroupFile{Content: content, Err: err}
}
return out
}
func readText(p string) (string, error) {
// Opening a FIFO must not stall collection before its type is checked.
file, err := os.OpenFile(p, os.O_RDONLY|syscall.O_NONBLOCK, 0) // #nosec G304 -- paths come from the fixed layout or trusted Exim settings
if err != nil {
return "", err
}
defer func() { _ = file.Close() }()
info, err := file.Stat()
if err != nil {
return "", err
}
if !info.Mode().IsRegular() {
return "", errors.New("configuration is not a regular file")
}
return readConfigContent(file, maxConfigBytes)
}
func readConfigContent(r io.Reader, maxBytes int64) (string, error) {
data, err := io.ReadAll(io.LimitReader(r, maxBytes+1))
switch {
case err != nil:
return "", err
case int64(len(data)) > maxBytes:
return "", fmt.Errorf("file is larger than %d bytes", maxBytes)
}
return string(data), nil
}
// exists reports whether p exists, following symlinks as cPanel's own -e test does.
func exists(p string) (bool, error) {
_, err := os.Stat(p)
switch {
case err == nil:
return true, nil
case errors.Is(err, fs.ErrNotExist):
return false, nil
}
return false, err
}
// rspamdLog reads rspamd's log destination from `rspamadm configdump -j`
// and, for a file, the file and its directories.
func rspamdLog(dump antispam.CommandResult) antispam.LogFacts {
var out antispam.LogFacts
var d struct {
Logging struct {
Type string `json:"type"`
Filename string `json:"filename"`
} `json:"logging"`
}
if dump.Err != nil || json.Unmarshal([]byte(dump.Stdout), &d) != nil || d.Logging.Type == "" {
out.Err = errors.New("rspamd's logging settings are not readable")
return out
}
out.Type, out.Path = d.Logging.Type, d.Logging.Filename
if out.Type != "file" {
return out
}
if !filepath.IsAbs(out.Path) {
out.Err = errors.New("rspamd's log file is not an absolute path")
return out
}
if out.Parents, out.Err = chain(filepath.Dir(out.Path)); out.Err != nil {
return out
}
switch facts, err := pathFacts(out.Path); {
case errors.Is(err, fs.ErrNotExist):
out.Missing = true
case err != nil:
out.Err = err
default:
out.File = facts
}
return out
}
// fileFacts records the file and its directories without following links.
func fileFacts(p string) antispam.FileFacts {
var out antispam.FileFacts
if out.Parents, out.Err = chain(filepath.Dir(p)); out.Err != nil {
if errors.Is(out.Err, fs.ErrNotExist) {
out.Missing, out.Err = true, nil
}
return out
}
if last := out.Parents[len(out.Parents)-1]; !last.Mode.IsDir() {
out.Err = fmt.Errorf("%s is not a directory", last.Path)
return out
}
switch facts, err := pathFacts(p); {
case errors.Is(err, fs.ErrNotExist):
out.Missing = true
case err != nil:
out.Err = err
default:
out.File = facts
}
return out
}
package hostfacts
import (
"errors"
"io/fs"
"net"
"net/netip"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/antispam"
)
// maxAccountFileBytes bounds /etc/group and /etc/passwd reads.
const maxAccountFileBytes = 8 << 20
func readAccountFile(p string) ([]byte, error) {
content, err := readRegularContent(p, maxAccountFileBytes)
return []byte(content), err
}
// controllerFacts parses the loaded controller from the full configdump the
// collector already read and gathers what exposure needs: host addresses,
// each Unix controller socket and the members of the groups owning them.
func (c collector) controllerFacts(f *antispam.HostFacts) {
h := antispam.ControllerHostFacts{
Sockets: map[string]antispam.SocketFacts{},
GroupMembers: map[int][]string{},
RspamdUID: f.Socket.RspamdUID,
RspamdGID: f.RspamdGID,
}
if props := antispam.ParseUnitShow(f.RspamdService.Stdout); f.RspamdService.Err == nil && h.RspamdUID >= 0 {
h.RspamdUser = props["User"]
if _, err := strconv.Atoi(h.RspamdUser); err == nil {
h.RspamdUser, _ = c.userName(strconv.Itoa(h.RspamdUID))
}
}
defer func() { f.ControllerHost = h }()
if f.RspamdConfig.Err != nil {
f.ControllerErr = errors.New("rspamadm configdump failed; controller exposure is unknown")
return
}
cfg, err := antispam.ParseControllerConfig([]byte(f.RspamdConfig.Stdout))
if err != nil {
f.ControllerUnverifiable = err
return
}
f.Controller = cfg
f.ControllerErr = nil
gids := map[int]bool{}
hasTCP := false
for _, l := range cfg.Listeners {
if !l.Unix {
hasTCP = true
continue
}
s := unixSocketFacts(l.Path)
h.Sockets[l.Path] = s
if s.Err == nil && !s.Missing {
gids[s.Socket.GID] = true
}
}
if len(gids) > 0 {
h.GroupMembers = c.groupMembers(gids)
}
if hasTCP {
addrs, err := c.interfaceAddrs()
if err != nil {
f.ControllerErr = errors.New("host interface addresses unavailable; controller exposure is unknown")
return
}
for _, a := range addrs {
if n, ok := a.(*net.IPNet); ok {
if ip, ok := netip.AddrFromSlice(n.IP); ok {
ip = ip.Unmap()
h.Addrs = append(h.Addrs, netip.PrefixFrom(ip, ip.BitLen()))
}
}
}
}
}
// unixSocketFacts lstats a Unix socket and every directory its path walks
// through, without probing it.
func unixSocketFacts(p string) antispam.SocketFacts {
s := antispam.SocketFacts{Socket: antispam.PathFacts{Path: p}, RspamdUID: -1, PeerUID: -1, EximUID: -1, EximGID: -1}
final, dirs, walkErr := resolve(p)
for _, d := range dirs {
facts, err := pathFacts(d)
if err != nil {
s.Err = err
return s
}
s.Parents = append(s.Parents, facts)
}
switch {
case errors.Is(walkErr, fs.ErrNotExist):
s.Missing = true
return s
case walkErr != nil:
s.Err = walkErr
return s
}
s.Socket, s.Err = pathFacts(final)
if s.Err == nil && s.Socket.Mode.Type() != fs.ModeSocket {
s.Err = errors.New("the path is not a Unix socket")
}
return s
}
// groupMembers lists, for each wanted gid, the names in its /etc/group
// entry plus every user whose primary group it is. A gid that cannot be
// read is left out, which exposure treats as possibly holding a tenant.
func (c collector) groupMembers(want map[int]bool) map[int][]string {
groups, err := c.readGroup()
if err != nil {
return nil
}
passwd, err := c.readPasswd()
if err != nil {
return nil
}
return parseGroupMembers(string(groups), string(passwd), want)
}
// A malformed record may hide a member, so no group is known complete
// unless both account files parse in full. Split lazily to bound temporary
// allocations even when a file is mostly delimiters.
func parseGroupMembers(groups, passwd string, want map[int]bool) map[int][]string {
out := map[int][]string{}
for line := range strings.SplitSeq(groups, "\n") {
if strings.TrimSpace(line) == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.SplitN(line, ":", 5)
if len(fields) != 4 || !accountName(fields[0]) {
return nil
}
gid, ok := accountID(fields[2])
if !ok {
return nil
}
// Several group names can share a gid; keep every list.
if _, listed := out[gid]; want[gid] && !listed {
out[gid] = []string{}
}
if fields[3] == "" {
continue
}
for m := range strings.SplitSeq(fields[3], ",") {
if !accountName(m) {
return nil
}
if want[gid] {
out[gid] = append(out[gid], m)
}
}
}
for line := range strings.SplitSeq(passwd, "\n") {
if strings.TrimSpace(line) == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.SplitN(line, ":", 8)
if len(fields) != 7 || !accountName(fields[0]) {
return nil
}
if _, ok := accountID(fields[2]); !ok {
return nil
}
gid, ok := accountID(fields[3])
if !ok {
return nil
}
if want[gid] {
out[gid] = append(out[gid], fields[0])
}
}
return out
}
func accountName(name string) bool {
return name != "" && !strings.ContainsAny(name, ":,\x00 \t\r\n\v\f")
}
func accountID(raw string) (int, bool) {
if _, err := strconv.ParseUint(raw, 10, 32); err != nil {
return 0, false
}
id, err := strconv.Atoi(raw)
return id, err == nil
}
package hostfacts
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"sort"
"strconv"
"strings"
"time"
)
const controllerTimeout = 5 * time.Second
// Controller request errors. ErrControllerNotSent means no request byte left
// CSM; ErrControllerNoReply means a request was written and no complete
// reply came back, so its outcome is unknown. ErrControllerPeer also wraps
// ErrControllerNotSent.
var (
ErrControllerNotSent = errors.New("request not sent to rspamd's controller")
ErrControllerNoReply = errors.New("no complete reply from rspamd's controller")
ErrControllerPeer = fmt.Errorf("rspamd's controller socket is not served by rspamd: %w", ErrControllerNotSent)
)
// ControllerGet sends GET path to rspamd's controller over its private Unix
// socket and returns the body of an HTTP 200 reply of at most limit bytes.
// Errors are fixed text; they never repeat the reply.
func ControllerGet(ctx context.Context, socket, path string, limit int64, peer int) ([]byte, error) {
status, body, err := ControllerDo(ctx, socket, peer, http.MethodGet, path, nil, nil, limit)
if err != nil {
return nil, err
}
if status != http.StatusOK {
return nil, fmt.Errorf("rspamd's controller returned HTTP %d", status)
}
return body, nil
}
// ControllerDo sends one request to rspamd's controller over its private
// Unix socket and returns the reply status and a body of at most limit
// bytes. The socket must be served by uid peer (SO_PEERCRED), so a process
// squatting the path after a restart gets nothing. It sends no forwarded
// header, so rspamd treats the request as local root access, and never a
// password. Errors are fixed text; they never repeat the reply.
func ControllerDo(ctx context.Context, socket string, peer int, method, path string, hdr map[string]string, body []byte, limit int64) (int, []byte, error) {
request, err := controllerRequest(method, path, hdr, body)
if err != nil {
return 0, nil, fmt.Errorf("%w: %w", ErrControllerNotSent, err)
}
ctx, cancel := context.WithTimeout(ctx, controllerTimeout)
defer cancel()
var d net.Dialer
conn, err := d.DialContext(ctx, "unix", socket)
if err != nil {
return 0, nil, fmt.Errorf("%w: cannot connect to rspamd's controller socket", ErrControllerNotSent)
}
defer func() { _ = conn.Close() }()
if peerUID(conn) != peer {
return 0, nil, ErrControllerPeer
}
if deadline, ok := ctx.Deadline(); ok {
_ = conn.SetDeadline(deadline)
}
stop := context.AfterFunc(ctx, func() { _ = conn.SetDeadline(time.Now()) })
defer stop()
if err = writeControllerRequest(conn, request); err != nil {
return 0, nil, err
}
// Framing gets its own allowance, as for /ping.
reply := &io.LimitedReader{R: conn, N: 2*limit + 1}
reader := bufio.NewReader(reply)
var resp *http.Response
for {
resp, err = http.ReadResponse(reader, nil)
if err != nil {
return 0, nil, fmt.Errorf("%w: invalid HTTP response", ErrControllerNoReply)
}
if resp.StatusCode >= http.StatusOK {
break
}
_ = resp.Body.Close()
if resp.StatusCode < http.StatusContinue || resp.StatusCode == http.StatusSwitchingProtocols {
return 0, nil, fmt.Errorf("%w: unexpected HTTP status", ErrControllerNoReply)
}
// Informational responses do not settle a mutation's outcome. Keep
// the same reader and byte budget until a final response arrives.
}
defer func() { _ = resp.Body.Close() }()
data, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
switch {
case err != nil:
return 0, nil, fmt.Errorf("%w: the reply could not be read", ErrControllerNoReply)
case reply.N == 0 || int64(len(data)) > limit:
return 0, nil, fmt.Errorf("%w: the reply is too large", ErrControllerNoReply)
}
return resp.StatusCode, data, nil
}
func writeControllerRequest(w io.Writer, request []byte) error {
if n, err := w.Write(request); err != nil {
if n == 0 {
return fmt.Errorf("%w: cannot send the request", ErrControllerNotSent)
}
// A partial write is not safe to repeat: rspamd may have received
// enough of the request to act before the connection failed.
return fmt.Errorf("%w: cannot send the request", ErrControllerNoReply)
}
return nil
}
// controllerRequest builds the request bytes, refusing anything that could
// split into a second request or header.
func controllerRequest(method, path string, hdr map[string]string, body []byte) ([]byte, error) {
plain := func(s string) bool {
return !strings.ContainsFunc(s, func(r rune) bool { return r < ' ' || r == 0x7f })
}
if method != http.MethodPost && (method != http.MethodGet || len(body) != 0) {
return nil, errors.New("unsupported method")
}
if !strings.HasPrefix(path, "/") || !plain(path) || strings.Contains(path, " ") {
return nil, errors.New("controller path is not a plain absolute path")
}
var b strings.Builder
b.WriteString(method + " " + path + " HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n")
names := make([]string, 0, len(hdr))
for name := range hdr {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
if name == "" || strings.ContainsFunc(name, func(r rune) bool {
return (r < 'A' || r > 'Z') && (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-'
}) || !plain(hdr[name]) {
return nil, errors.New("invalid request header")
}
switch strings.ToLower(name) {
case "host", "connection", "content-length", "transfer-encoding",
"x-forwarded-for", "x-real-ip", "forwarded", "password", "authorization", "proxy-authorization":
return nil, errors.New("reserved request header")
}
b.WriteString(name + ": " + hdr[name] + "\r\n")
}
if method == http.MethodPost {
b.WriteString("Content-Length: " + strconv.Itoa(len(body)) + "\r\n")
}
b.WriteString("\r\n")
return append([]byte(b.String()), body...), nil
}
package hostfacts
import (
"context"
"crypto/rand"
"encoding/binary"
"io"
"net"
"net/netip"
"strings"
"time"
"github.com/pidginhost/csm/internal/antispam"
"golang.org/x/net/dns/dnsmessage"
)
// probeTimeout bounds one probe query, including a TCP retry.
const probeTimeout = 3 * time.Second
// QueryDNS asks server for A records of an absolute name. Use DNS directly:
// net.Resolver consults host files and can print the DQS key with netdns=2.
func QueryDNS(ctx context.Context, server netip.AddrPort, name string) antispam.ProbeAnswer {
failed := antispam.ProbeAnswer{Failed: true}
qname, err := dnsmessage.NewName(strings.TrimSuffix(name, ".") + ".")
if err != nil {
return failed
}
question := dnsmessage.Question{Name: qname, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}
var id [2]byte
if _, err = rand.Read(id[:]); err != nil {
return failed
}
query := dnsmessage.Message{
Header: dnsmessage.Header{ID: binary.BigEndian.Uint16(id[:]), RecursionDesired: true},
Questions: []dnsmessage.Question{question},
}
packet, err := query.Pack()
if err != nil {
return failed
}
ctx, cancel := context.WithTimeout(ctx, probeTimeout)
defer cancel()
for _, network := range []string{"udp", "tcp"} {
answer, truncated := exchangeProbe(ctx, network, server, packet, query.ID, question)
if !truncated {
return answer
}
}
return failed
}
func exchangeProbe(ctx context.Context, network string, server netip.AddrPort, packet []byte, id uint16, question dnsmessage.Question) (antispam.ProbeAnswer, bool) {
failed := antispam.ProbeAnswer{Failed: true}
var dialer net.Dialer
conn, err := dialer.DialContext(ctx, network, server.String())
if err != nil {
return failed, false
}
defer func() { _ = conn.Close() }()
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
defer stop()
deadline, _ := ctx.Deadline()
if err = conn.SetDeadline(deadline); err != nil {
return failed, false
}
if network == "tcp" {
// #nosec G115 -- QueryDNS packs one DNS question, at most 271 bytes.
packet = append(binary.BigEndian.AppendUint16(nil, uint16(len(packet))), packet...)
}
if _, err = conn.Write(packet); err != nil {
return failed, false
}
buf := make([]byte, 65535)
for {
var n int
if network == "tcp" {
if _, err = io.ReadFull(conn, buf[:2]); err == nil {
n, err = io.ReadFull(conn, buf[:binary.BigEndian.Uint16(buf[:2])])
}
} else {
n, err = conn.Read(buf)
}
if err != nil {
return failed, false
}
reply := parseProbeReply(buf[:n], id, question)
if reply.matched {
return reply.answer, reply.truncated
}
// Ignore unrelated packets on the connected socket until the deadline.
}
}
type probeReply struct {
answer antispam.ProbeAnswer
matched bool
truncated bool
}
func parseProbeReply(packet []byte, id uint16, question dnsmessage.Question) probeReply {
var p dnsmessage.Parser
h, err := p.Start(packet)
if err != nil || h.ID != id || !h.Response || h.OpCode != 0 {
return probeReply{}
}
q, err := p.Question()
if err != nil || q.Type != question.Type || q.Class != question.Class || !strings.EqualFold(q.Name.String(), question.Name.String()) {
return probeReply{}
}
if _, err := p.Question(); err != dnsmessage.ErrSectionDone {
return probeReply{}
}
reply := probeReply{matched: true, answer: antispam.ProbeAnswer{Failed: true}, truncated: h.Truncated}
if h.Truncated {
return reply
}
var m dnsmessage.Message
if m.Unpack(packet) != nil {
return reply
}
switch h.RCode {
case dnsmessage.RCodeNameError:
reply.answer = antispam.ProbeAnswer{NXDomain: true}
return reply
case dnsmessage.RCodeSuccess:
default:
return reply
}
// Recursive resolvers include CNAME targets in the answer. Follow only
// that chain, so unrelated A records cannot turn into listing codes.
aliases := make(map[string]string)
for _, rr := range m.Answers {
if cname, ok := rr.Body.(*dnsmessage.CNAMEResource); ok && rr.Header.Class == dnsmessage.ClassINET {
from, to := strings.ToLower(rr.Header.Name.String()), strings.ToLower(cname.CNAME.String())
if previous, ok := aliases[from]; ok && previous != to {
return reply
}
aliases[from] = to
}
}
name := strings.ToLower(question.Name.String())
seen := make(map[string]bool)
for aliases[name] != "" {
if seen[name] {
return reply
}
seen[name] = true
name = aliases[name]
}
reply.answer = antispam.ProbeAnswer{}
for _, rr := range m.Answers {
if a, ok := rr.Body.(*dnsmessage.AResource); ok && rr.Header.Class == dnsmessage.ClassINET && strings.EqualFold(rr.Header.Name.String(), name) {
reply.answer.Addrs = append(reply.answer.Addrs, netip.AddrFrom4(a.A).String())
}
}
return reply
}
package hostfacts
import (
"errors"
"io"
"io/fs"
"os"
"strings"
"syscall"
"github.com/pidginhost/csm/internal/antispam"
)
// maxKeyBytes bounds the key file read: a 26-character key, a newline and
// some slack for white space.
const maxKeyBytes = 128
// ReadDQSKey reads the Spamhaus DQS key file without following a final
// symlink or blocking on a FIFO. Surrounding white space is trimmed.
func ReadDQSKey(p string) antispam.KeyFacts {
k := antispam.KeyFacts{Path: p, UID: -1}
// #nosec G304 -- the operator's configured key path, opened read-only without following a final link.
f, err := os.OpenFile(p, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if errors.Is(err, fs.ErrNotExist) {
k.Missing = true
return k
}
if err != nil {
k.Err = err
return k
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil {
k.Err = err
return k
}
if !info.Mode().IsRegular() {
k.Err = errors.New("not a regular file")
return k
}
k.Mode = info.Mode()
if st, ok := info.Sys().(*syscall.Stat_t); ok {
k.UID = int(st.Uid)
}
data, err := io.ReadAll(io.LimitReader(f, maxKeyBytes+1))
switch {
case err != nil:
k.Err = err
case len(data) > maxKeyBytes:
k.Err = errors.New("larger than a DQS key")
default:
k.Key = antispam.NewSecret(strings.TrimSpace(string(data)))
}
return k
}
package hostfacts
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"io/fs"
"os"
"os/exec"
"os/user"
"slices"
"strconv"
"strings"
"syscall"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/antispam"
)
// Learning fact sources on a live cPanel host.
const (
procNetUDP = "/proc/net/udp"
procNetUDP6 = "/proc/net/udp6"
cageFSSkeletonLog = "/usr/share/cagefs-skeleton/var/log/rspamd"
roundcubeConfig = "/usr/local/cpanel/base/3rdparty/roundcube/config/config.inc.php"
cpanelPHP = "/usr/local/cpanel/3rdparty/bin/php"
cpanelUsers = "/var/cpanel/users"
// roundcubeIncludeGlob is the line cPanel's Roundcube configuration uses
// to load inc.d.
roundcubeIncludeGlob = `glob("/usr/local/cpanel/base/3rdparty/roundcube/config/inc.d/*.php")`
// fuzzyListenAddr is 127.0.0.1:11335 as /proc/net/udp prints it.
fuzzyListenAddr = "0100007F:2C47"
// fuzzyWildcardAddr is 0.0.0.0:11335, which also receives its datagrams.
fuzzyWildcardAddr = "00000000:2C47"
)
// CollectLearning adds the learning prerequisites for the desired settings
// s to f, which must already hold the core facts (full rspamd dump, options
// and targets).
func CollectLearning(ctx context.Context, s antispam.Settings, f *antispam.HostFacts) {
live().learningFacts(ctx, s, f)
}
func (c collector) learningFacts(ctx context.Context, s antispam.Settings, f *antispam.HostFacts) {
c.learningDumps(ctx, f)
c.learningSymbols(ctx, f)
c.controlSocket(f)
c.controlSocketGroup(f)
c.logGroups(f)
f.FuzzyListening, f.FuzzyListenForeign, f.FuzzyListenErr = c.fuzzyListening(f.Socket.RspamdUID)
f.CageFS = c.cageFS(ctx)
if f.Stack != antispam.StackPostfix {
f.Roundcube = c.roundcube(*f)
}
c.learningRedis(ctx, s.LearningRedisSocket, f)
f.RspamdUIDShared, f.RspamdUIDErr = false, nil
if f.ControllerHost.RspamdUID > 0 {
f.RspamdUIDShared, f.RspamdUIDErr = c.sharesUntrustedUID(f.ControllerHost.RspamdUID, *f)
}
}
// dovecotDeadline bounds the pass's Dovecot listing, which runs while the
// learner is paused. It is learnwatch.PassDeadline, the bound the watcher's
// pass puts on the same listing: one that cannot finish within it cannot
// serve the watcher either.
var dovecotDeadline = 4 * time.Minute
// dovecotFacts lists a Postfix host's mailboxes for the pass: the uids mail
// is stored as and the users discovery skips. Only a complete snapshot
// counts.
func (c collector) dovecotFacts(ctx context.Context) antispam.DovecotFacts {
ctx, cancel := context.WithTimeout(ctx, dovecotDeadline)
defer cancel()
snap, err := c.dovecot(ctx)
if err == nil && !snap.Complete {
err = errors.New("the Dovecot mailbox listing is incomplete")
}
if err != nil {
return antispam.DovecotFacts{Err: err}
}
var uids []int
for _, m := range snap.Mailboxes {
uids = append(uids, m.UID)
}
for _, sk := range snap.Skips {
if sk.UID > 0 {
uids = append(uids, sk.UID)
}
}
slices.Sort(uids)
return antispam.DovecotFacts{UIDs: slices.Compact(uids), Skips: snap.Skips}
}
// CollectLearningRedis reads learning's Redis at socket, the dedicated one
// named in csm.yaml, into f, which must hold the core facts. The worker
// calls it under its lock: the socket can change after the pass judged it.
func CollectLearningRedis(ctx context.Context, socket string, f *antispam.HostFacts) {
live().learningRedis(ctx, socket, f)
}
func (c collector) learningRedis(ctx context.Context, socket string, f *antispam.HostFacts) {
if f.Stack == antispam.StackPostfix {
// Both the pass and the locked worker must use current mail UIDs.
f.Dovecot = c.dovecotFacts(ctx)
f.AccountNames, f.AccountNamesErr = c.untrustedLocalUsers(f.ControllerHost.RspamdUser, f.Dovecot.UIDs)
} else {
f.AccountNames, f.AccountNamesErr = c.accountNames()
}
f.Redis = c.redis(ctx, *f, socket)
}
// learningDumps reads the rspamd sections learning verifies. Any failure is
// one error; a partial set is never judged.
func (c collector) learningDumps(ctx context.Context, f *antispam.HostFacts) {
var d antispam.LearningDumps
var errs []error
read := func(section string) []byte {
r := c.run(ctx, "rspamadm", "configdump", "-j", section)
if r.Err != nil {
errs = append(errs, errors.New("rspamadm configdump -j "+section+" failed"))
return nil
}
if !readableDump(r.Stdout, section == "group") {
errs = append(errs, errors.New("rspamadm configdump -j "+section+" is not readable"))
}
return []byte(r.Stdout)
}
d.FuzzyCheck = read("fuzzy_check")
d.Group = read("group")
d.Redis = read("redis")
root, parseErr := antispam.ParseRspamdDumpObject([]byte(f.RspamdConfig.Stdout))
switch {
case f.RspamdConfig.Err != nil:
errs = append(errs, errors.New("rspamadm configdump -j failed"))
case parseErr != nil:
errs = append(errs, errors.New("rspamadm configdump -j is not readable"))
default:
d.Worker = root["worker"]
if _, err := workerEntries(d.Worker); err != nil {
errs = append(errs, err)
}
}
d.Err = errors.Join(errs...)
f.LearningDumps = d
}
// The group command emits a stream of objects, not a JSON array. Validate
// the envelopes without rewriting bytes or interpreting module settings.
func readableDump(data string, stream bool) bool {
d := json.NewDecoder(strings.NewReader(data))
count := 0
for {
var object json.RawMessage
err := d.Decode(&object)
if errors.Is(err, io.EOF) {
return count > 0
}
if err != nil || (!stream && count > 0) {
return false
}
if _, err := antispam.ParseRspamdDumpObject(object); err != nil {
return false
}
count++
}
}
// maxSymbolsReply bounds the controller's /symbols reply.
const maxSymbolsReply = 4 << 20
// learningSymbols reads the controller's effective weights while CSM's
// learning worker file is installed, so a pass can see a runtime change.
func (c collector) learningSymbols(ctx context.Context, f *antispam.HostFacts) {
installed := false
for _, t := range f.Targets {
if t.Path == antispam.LearningWorkerPath {
installed = !t.Missing && t.Err == nil && antispam.ManagedFor(t.Kind, t.Content)
}
}
if !installed {
return
}
socket, ok := antispam.PrivateControllerSocket(*f)
if !ok {
f.LearningDumps.SymbolsErr = errors.New("learning has no private controller socket to read weights from")
return
}
f.LearningDumps.Symbols, f.LearningDumps.SymbolsErr = c.controllerGet(ctx, socket, "/symbols", maxSymbolsReply, f.ControllerHost.RspamdUID)
}
// controlSocket reads options.control_socket, without its mode option.
func (c collector) controlSocket(f *antispam.HostFacts) {
f.ControlSocket = antispam.SocketFacts{}
opts, parseErr := antispam.ParseRspamdDumpObject([]byte(f.RspamdOptions.Stdout))
var value string
switch {
case f.RspamdOptions.Err != nil:
f.ControlSocket.Err = errors.New("rspamadm configdump -j options failed")
case parseErr != nil:
f.ControlSocket.Err = errors.New("rspamd's options are not readable")
case opts["control_socket"] == nil:
f.ControlSocket.Err = errors.New("rspamd reports no control_socket")
case json.Unmarshal(opts["control_socket"], &value) != nil:
f.ControlSocket.Err = errors.New("control_socket is not a string")
default:
fields := strings.Fields(value)
if len(fields) == 0 || !strings.HasPrefix(fields[0], "/") {
f.ControlSocket.Err = errors.New("control_socket is not a Unix socket path")
return
}
f.ControlSocket = unixSocketFacts(fields[0])
}
}
// controlSocketGroup adds the control socket's group members to the
// controller facts, so one exposure check judges both sockets.
func (c collector) controlSocketGroup(f *antispam.HostFacts) {
if f.ControlSocket.Err != nil || f.ControlSocket.Missing {
return
}
gid := f.ControlSocket.Socket.GID
if _, known := f.ControllerHost.GroupMembers[gid]; known {
return
}
members, ok := c.groupMembers(map[int]bool{gid: true})[gid]
if !ok {
return
}
if f.ControllerHost.GroupMembers == nil {
f.ControllerHost.GroupMembers = map[int][]string{}
}
f.ControllerHost.GroupMembers[gid] = members
}
// Log privacy depends on group membership, including the groups of every
// parent that can grant traversal or replacement access.
func (c collector) logGroups(f *antispam.HostFacts) {
gids := map[int]bool{f.RspamdLog.File.GID: true}
for _, p := range f.RspamdLog.Parents {
gids[p.GID] = true
}
if f.ControllerHost.GroupMembers == nil {
f.ControllerHost.GroupMembers = map[int][]string{}
}
for gid, members := range c.groupMembers(gids) {
f.ControllerHost.GroupMembers[gid] = members
}
}
// fuzzyListening reports whether rspamd (or root) has a UDP socket bound to
// 127.0.0.1:11335, and whether any other user's socket sits on that port of
// the loopback or wildcard address. The port is unprivileged: a tenant socket
// there would receive scan and learning datagrams. rspamd does not respawn a
// fuzzy worker that exited.
func (c collector) fuzzyListening(rspamdUID int) (listening, foreign bool, err error) {
if rspamdUID < 0 {
return false, false, errors.New("rspamd's service user is unknown, so the fuzzy worker's socket cannot be judged")
}
trusted := func(uid int) bool { return uid == 0 || uid == rspamdUID }
data, readErr := readText(c.layout.ProcNetUDP)
if readErr != nil {
err = errors.New("cannot read the UDP socket table")
} else {
listening, foreign, err = parseFuzzyListening(data, trusted)
}
// Each table is independent evidence: a failed read of one must not
// hide another user's listener in the other.
// Without IPv6 there is no udp6 table; any other failure is unknown.
data6, readErr := readText(c.layout.ProcNetUDP6)
switch {
case errors.Is(readErr, fs.ErrNotExist):
return listening, foreign, err
case readErr != nil:
return listening, foreign, errors.Join(err, errors.New("cannot read the UDP6 socket table"))
}
foreign6, err6 := parseUDP6Foreign(data6, trusted)
return listening, foreign || foreign6, errors.Join(err, err6)
}
// IPv6 bindings on the fuzzy port, as /proc prints them, that can receive
// traffic for 127.0.0.1:11335: the wildcard (dual-stack), IPv4-mapped
// 127.0.0.1 and 0.0.0.0. Native IPv6 and other 127/8 addresses receive none.
var fuzzyPort6Addrs = map[string]bool{
"00000000000000000000000000000000:2C47": true,
"0000000000000000FFFF00000100007F:2C47": true,
"0000000000000000FFFF000000000000:2C47": true,
}
// parseUDP6Foreign reports whether another user's IPv6 socket sits on one of
// the fuzzy port's addresses.
func parseUDP6Foreign(data string, trusted func(uid int) bool) (bool, error) {
lines := strings.Split(data, "\n")
header := strings.Fields(lines[0])
bad := errors.New("the UDP6 socket table is not readable")
if len(header) < 4 || strings.Join(header[:4], " ") != "sl local_address remote_address st" {
return false, bad
}
foreign := false
for _, line := range lines[1:] {
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
if len(fields) < 10 || !udp6Address(fields[1]) || !udp6Address(fields[2]) {
return false, bad
}
uid, err := strconv.Atoi(fields[7])
if err != nil || uid < 0 {
return false, bad
}
if fuzzyPort6Addrs[fields[1]] && !trusted(uid) {
foreign = true
}
}
return foreign, nil
}
func udp6Address(value string) bool {
if len(value) != 37 || value[32] != ':' {
return false
}
for _, c := range value[:32] + value[33:] {
if !strings.ContainsRune("0123456789ABCDEF", c) {
return false
}
}
return true
}
func parseFuzzyListening(data string, trusted func(uid int) bool) (listening, foreign bool, err error) {
lines := strings.Split(data, "\n")
header := strings.Fields(lines[0])
bad := errors.New("the UDP socket table is not readable")
if len(header) < 4 || strings.Join(header[:4], " ") != "sl local_address rem_address st" {
return false, false, bad
}
for _, line := range lines[1:] {
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
if len(fields) < 10 || !strings.HasSuffix(fields[0], ":") ||
!udpAddress(fields[1]) || !udpAddress(fields[2]) || len(fields[3]) != 2 {
return false, false, bad
}
if _, err := strconv.ParseUint(strings.TrimSuffix(fields[0], ":"), 10, 64); err != nil {
return false, false, bad
}
if _, err := strconv.ParseUint(fields[3], 16, 8); err != nil {
return false, false, bad
}
uid, err := strconv.Atoi(fields[7])
if err != nil || uid < 0 {
return false, false, bad
}
onPort := fields[1] == fuzzyListenAddr || fields[1] == fuzzyWildcardAddr
switch {
case onPort && !trusted(uid):
foreign = true
case fields[1] == fuzzyListenAddr && fields[3] == "07":
listening = true
}
}
return listening, foreign, nil
}
func udpAddress(value string) bool {
if len(value) != 13 || value[8] != ':' {
return false
}
_, addrErr := strconv.ParseUint(value[:8], 16, 32)
_, portErr := strconv.ParseUint(value[9:], 16, 16)
return addrErr == nil && portErr == nil
}
// cageFS reads whether CageFS confines every account: enabled, in Enable
// All mode, and how many accounts are excluded. A disabled-list line that is
// not an account name is an error: CSM cannot count it.
func (c collector) cageFS(ctx context.Context) antispam.CageFSFacts {
var out antispam.CageFSFacts
if _, err := os.Lstat(c.layout.CageFSSkeletonLog); err == nil {
out.SkeletonHasRspamdLog = true
} else if !errors.Is(err, fs.ErrNotExist) {
out.Err = errors.New("cannot check the CageFS skeleton")
return out
}
status := c.run(ctx, "cagefsctl", "--cagefs-status")
if errors.Is(status.Err, exec.ErrNotFound) {
// No cagefsctl: CageFS is not installed, which is a known fact.
return out
}
if status.Err != nil {
out.Err = errors.New("cagefsctl --cagefs-status failed")
return out
}
switch strings.TrimSpace(status.Stdout) {
case "Disabled":
return out
case "Enabled":
default:
out.Err = errors.New("cagefsctl --cagefs-status is not readable")
return out
}
mode := c.run(ctx, "cagefsctl", "--display-user-mode")
if mode.Err != nil {
out.Err = errors.New("cagefsctl --display-user-mode failed")
return out
}
switch strings.TrimSpace(mode.Stdout) {
case "Mode: Disable All":
return out
case "Mode: Enable All":
default:
out.Err = errors.New("cagefsctl --display-user-mode is not readable")
return out
}
disabled := c.run(ctx, "cagefsctl", "--list-disabled")
if disabled.Err != nil {
out.Err = errors.New("cagefsctl --list-disabled failed")
return out
}
for _, line := range strings.Split(disabled.Stdout, "\n") {
switch name := strings.TrimSpace(line); {
case name == "":
case admission.ValidAccountName(name):
out.DisabledUsers++
default:
out.Err = errors.New("cagefsctl --list-disabled printed a line CSM cannot read")
return out
}
}
out.Enabled = true
out.ExcludedLogDir = strings.TrimPrefix(cageFSSkeletonLog, "/usr/share/cagefs-skeleton")
return out
}
// roundcube reads whether cPanel's Roundcube loads inc.d and whether
// cPanel's PHP can verify the include.
func (c collector) roundcube(f antispam.HostFacts) antispam.RoundcubeFacts {
var out antispam.RoundcubeFacts
for _, t := range f.Targets {
if t.Path == antispam.RoundcubeIncludePath {
out.Include = t
}
}
if out.Include.Path == "" {
out.Err = errors.New("the Roundcube include was not checked")
}
conf, err := readText(c.layout.RoundcubeConfig)
if err != nil {
out.Err = errors.New("cannot read cPanel's Roundcube configuration")
} else {
out.IncludeMechanism = strings.Contains(conf, roundcubeIncludeGlob)
}
info, err := os.Stat(c.layout.CPanelPHP)
switch {
case err == nil:
out.PHP = info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0
case !errors.Is(err, fs.ErrNotExist):
out.Err = errors.Join(out.Err, errors.New("cannot check cPanel's PHP executable"))
}
return out
}
// accountNames lists the cPanel accounts by their user files.
func (c collector) accountNames() (map[string]bool, error) {
entries, err := os.ReadDir(c.layout.CPanelUsers)
if err != nil {
return nil, errors.New("cannot list cPanel accounts")
}
out := make(map[string]bool, len(entries))
for _, e := range entries {
if admission.ValidAccountName(e.Name()) {
out[e.Name()] = true
}
}
return out, nil
}
// redis identifies learning's Redis and probes it: the dedicated socket when
// one is configured, which gets no password, else on cPanel the fuzzy
// worker's backend. CSM sends the password only to a Unix socket nobody but
// its owner or root could have put there, owned by a pinned service user; a
// TCP endpoint is never probed, it fails isolation anyway.
func (c collector) redis(ctx context.Context, f antispam.HostFacts, socket string) antispam.RedisFacts {
endpoint, unix, password := socket, true, antispam.Secret{}
switch {
case socket != "":
case f.Stack == antispam.StackPostfix:
// rspamd's own Redis serves every module on Postfix; learning
// there uses only a dedicated one.
return antispam.RedisFacts{ServiceUID: -1, Err: errors.New("learning on Postfix has no dedicated Redis")}
case f.LearningDumps.Err != nil:
return antispam.RedisFacts{ServiceUID: -1, Err: errors.New("rspamd's redis settings were not read")}
default:
var err error
if endpoint, unix, password, err = redisEndpoint(f.LearningDumps, []byte(f.RspamdConfig.Stdout)); err != nil {
return antispam.RedisFacts{ServiceUID: -1, Err: err}
}
}
r := antispam.RedisFacts{Endpoint: endpoint, Unix: unix, ServiceUID: -1}
if socket != "" {
r.Inherits = globalRedisRefusal(f.RspamdConfig)
}
if !unix {
return r
}
r.Socket = unixSocketFacts(endpoint)
if r.Socket.Err != nil || r.Socket.Missing {
r.Err = errors.New("redis socket cannot be checked")
return r
}
if f.Stack == antispam.StackPostfix && f.Dovecot.Err != nil {
r.Err = errors.New("the Dovecot mail UIDs could not be checked")
return r
}
owner := r.Socket.Socket.UID
gid := r.Socket.Socket.GID
r.GroupMembers = c.groupMembers(map[int]bool{gid: true})[gid]
trusted, trustErr := c.trustedRedisOwner(owner, f)
if trustErr != nil && f.Stack == antispam.StackPostfix {
r.Err = errors.New("the Redis owner isolation could not be checked")
return r
}
if !trusted || !protectedRedisPath(endpoint, owner, pathFacts) {
r.Unprotected = true
r.Err = errors.New("redis socket is not protected; CSM sends it no password")
return r
}
if socket != "" {
r.Shared, r.SharedErr = sharedRedis(f.RspamdConfig, endpoint)
}
probed := c.redisProbe(ctx, endpoint, password, owner)
probed.Endpoint, probed.Unix, probed.Socket, probed.GroupMembers = endpoint, unix, r.Socket, r.GroupMembers
probed.Shared, probed.SharedErr, probed.Inherits = r.Shared, r.SharedErr, r.Inherits
return probed
}
// globalRedisRefusal refuses rspamd's global redis settings that rspamd
// merges into a fuzzy worker with its own servers, CSM's on the dedicated
// Redis, and that an empty password cannot clear. redis.fuzzy is not
// merged into such a worker. An unreadable dump is sharedRedis's to report.
func globalRedisRefusal(dump antispam.CommandResult) error {
root, rootErr := antispam.ParseRspamdDumpObject([]byte(dump.Stdout))
global, globalErr := antispam.ParseRspamdDumpObject(root["redis"])
if dump.Err == nil && rootErr == nil && globalErr == nil {
return unprobedRedisOptions(func(name string) json.RawMessage { return global[name] })
}
return nil
}
// sharedRedis reports whether a Redis consumer in rspamd's loaded
// configuration, other than fuzzy storage, names the socket at endpoint.
// Paths are compared as files, so a symlink, a bind mount or another
// spelling of the path is the same endpoint. Fuzzy workers and the
// redis.fuzzy settings are left out: CSM's worker selects the socket itself,
// and any other fuzzy worker is a configuration conflict.
func sharedRedis(dump antispam.CommandResult, endpoint string) (bool, error) {
self, err := os.Stat(endpoint)
if err != nil {
return false, errors.New("cannot read the learning Redis socket")
}
root, err := antispam.ParseRspamdDumpObject([]byte(dump.Stdout))
if dump.Err != nil || err != nil {
return false, errors.New("rspamd's loaded configuration is not readable")
}
var paths []string
for key, raw := range root {
switch key {
case "worker":
entries, err := workerEntries(raw)
if err != nil {
return false, err
}
for _, entry := range entries {
var kind string
if _, named := entry["fuzzy"]; named || json.Unmarshal(entry["type"], &kind) == nil && kind == "fuzzy" {
continue
}
if err = redisPaths(entry, &paths); err != nil {
return false, err
}
}
case "redis":
global, err := antispam.ParseRspamdDumpObject(raw)
if err != nil {
return false, errors.New("rspamd's redis settings are not readable")
}
delete(global, "fuzzy")
delete(global, "fuzzy_storage")
if err = redisPaths(global, &paths); err != nil {
return false, err
}
default:
if err := redisPathsIn(raw, &paths); err != nil {
return false, err
}
}
}
for _, p := range paths {
if !strings.HasPrefix(p, "/") {
return false, errors.New("rspamd names a Redis socket by a relative path")
}
info, err := os.Stat(p)
switch {
case err == nil:
if os.SameFile(self, info) {
return true, nil
}
case errors.Is(err, fs.ErrNotExist), errors.Is(err, syscall.ENOTDIR):
default:
return false, errors.New("cannot check a Redis socket rspamd names")
}
}
return false, nil
}
// redisServerKeys select Redis servers in an rspamd section.
var redisServerKeys = map[string]bool{"servers": true, "server": true, "read_servers": true, "write_servers": true}
// redisPaths adds the socket paths the server keys of section name, and
// those below it.
func redisPaths(section map[string]json.RawMessage, out *[]string) error {
for key, raw := range section {
if redisServerKeys[key] {
*out = append(*out, socketEntries(raw)...)
}
if err := redisPathsIn(raw, out); err != nil {
return err
}
}
return nil
}
func redisPathsIn(raw json.RawMessage, out *[]string) error {
switch trimmed := bytes.TrimSpace(raw); {
case len(trimmed) > 0 && trimmed[0] == '{':
section, err := antispam.ParseRspamdDumpObject(trimmed)
if err != nil {
return errors.New("rspamd's loaded configuration is not readable")
}
return redisPaths(section, out)
case len(trimmed) > 0 && trimmed[0] == '[':
var items []json.RawMessage
if json.Unmarshal(trimmed, &items) != nil {
return errors.New("rspamd's loaded configuration is not readable")
}
for _, item := range items {
if err := redisPathsIn(item, out); err != nil {
return err
}
}
}
return nil
}
// socketEntries lists the Unix socket entries of an rspamd upstream list: a
// string or list of strings, each with an optional rotation prefix and
// entries separated by commas, semicolons or spaces. Unix sockets start
// with '/' or '.'; other entries are network addresses.
func socketEntries(raw json.RawMessage) []string {
var lines []string
var line string
if json.Unmarshal(raw, &line) == nil {
lines = []string{line}
} else {
var list []json.RawMessage
_ = json.Unmarshal(raw, &list)
for _, item := range list {
if json.Unmarshal(item, &line) == nil {
lines = append(lines, line)
}
}
}
var out []string
for _, line := range lines {
if mode, rest, ok := strings.Cut(line, ":"); ok && upstreamRotations[mode] {
line = rest
}
for _, entry := range strings.FieldsFunc(line, func(r rune) bool { return strings.ContainsRune(",; \t\r\n", r) }) {
if strings.HasPrefix(entry, "/") || strings.HasPrefix(entry, ".") {
out = append(out, entry)
}
}
}
return out
}
// upstreamRotations are the prefixes rspamd accepts before an upstream list.
var upstreamRotations = map[string]bool{"random": true, "master-slave": true, "round-robin": true, "hash": true, "sequential": true}
// trustedRedisOwner reports whether uid may serve the Redis socket: root,
// rspamd's service user or the system user named redis, and no untrusted
// user (f.AccountNames) shares the uid. Any other owner, including shared
// users such as nobody that can run tenant code, gets no password.
func (c collector) trustedRedisOwner(uid int, f antispam.HostFacts) (bool, error) {
if uid == 0 {
return true, nil
}
// A pinned service name does not make a shared UID safe.
shared, err := c.sharesUntrustedUID(uid, f)
if f.AccountNames["redis"] || shared {
return false, nil
}
if err != nil {
return false, err
}
if f.Socket.RspamdUID >= 0 && uid == f.Socket.RspamdUID {
return true, nil
}
redisUID, err := c.uid("redis")
return err == nil && redisUID >= 0 && uid == redisUID, err
}
// sharesUntrustedUID reports whether a user in f.AccountNames, which may
// run tenant code, has uid. The error says CSM cannot rule that out; a
// proven share wins over it.
func (c collector) sharesUntrustedUID(uid int, f antispam.HostFacts) (bool, error) {
if f.AccountNamesErr != nil {
return false, f.AccountNamesErr
}
var local map[string]int
var localErr, unknown error
for account := range f.AccountNames {
accountUID, err := c.uid(account)
// cPanel keeps registry records with no system user, such as its own
// "system" record. cPanel accounts are local users, so a name absent
// from /etc/passwd has no uid to share. NSS can report an unavailable
// backend as an unknown user, so the unknown reply alone is not proof.
if errors.As(err, new(user.UnknownUserError)) {
if local == nil && localErr == nil {
local, localErr = c.localAccounts()
}
switch localUID, listed := local[account]; {
case localErr != nil:
unknown = localErr
case listed && localUID == uid:
return true, nil
case listed:
unknown = errors.New("an account NSS does not know is listed in /etc/passwd")
}
continue
}
switch {
case err != nil:
unknown = err
case accountUID == uid:
return true, nil
}
}
return false, unknown
}
// protectedRedisPath verifies each lookup, including symlinks and repeated
// visits to a directory. Sticky protection depends on the entry being
// looked up, not the directory or socket it eventually resolves to.
func protectedRedisPath(endpoint string, owner int, stat func(string) (antispam.PathFacts, error)) bool {
trusted := func(uid int) bool { return uid == 0 || uid == owner }
unsafe := errors.New("redis socket path is not protected")
_, _, err := resolveChecked(endpoint, func(dir, next string) error {
p, err := stat(dir)
if err != nil || !p.Mode.IsDir() || p.ExtACL || !trusted(p.UID) {
return unsafe
}
if p.Mode.Perm()&0o022 == 0 {
return nil
}
entry, err := stat(next)
if err != nil || p.Mode&fs.ModeSticky == 0 || !trusted(entry.UID) {
return unsafe
}
return nil
})
return err == nil
}
package hostfacts
import (
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"regexp"
"strings"
"syscall"
"time"
"github.com/pidginhost/csm/internal/antispam"
)
// keyOwner is the uid every key file and the keys directory must have.
// Tests set it to their own uid.
var keyOwner = 0
// maxLearningKeyBytes bounds one key file read.
const maxLearningKeyBytes = 4 << 10
var (
hashKeyPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
prefixPattern = regexp.MustCompile(`^csmfz_[0-9a-f]{32}$`)
)
// storedKeypair is a keypair file: the two keys `rspamadm keypair -j` printed.
type storedKeypair struct {
Pubkey string `json:"pubkey"`
Privkey string `json:"privkey"`
}
// LoadLearningKeys reads and checks the key set under dir/keys: the
// directories must be root's, mode 0700 and not links; every file root's,
// regular, mode 0600 and small. Errors name the file and the rule, never
// the content.
func LoadLearningKeys(dir string) (antispam.LearningKeys, error) {
keys, err := openLearningKeysDir(dir, false)
if err != nil {
return antispam.LearningKeys{}, err
}
defer func() { _ = keys.Close() }()
return loadLearningKeys(keys)
}
func loadLearningKeys(dir *os.Root) (antispam.LearningKeys, error) {
var k antispam.LearningKeys
values := make(map[string]string, len(antispam.LearningKeyNames))
for _, name := range antispam.LearningKeyNames {
data, err := readKeyFile(dir, name)
if err != nil {
return antispam.LearningKeys{}, fmt.Errorf("learning key %s: %w", name, err)
}
values[name] = strings.TrimSuffix(data, "\n")
}
if !hashKeyPattern.MatchString(values[antispam.KeyFuzzy]) {
return antispam.LearningKeys{}, errors.New("learning key fuzzy.key is not 64 lowercase hex characters")
}
if !hashKeyPattern.MatchString(values[antispam.KeyShingles]) {
return antispam.LearningKeys{}, errors.New("learning key shingles.key is not 64 lowercase hex characters")
}
if !prefixPattern.MatchString(values[antispam.KeyPrefix]) {
return antispam.LearningKeys{}, errors.New("learning key prefix is not csmfz_ and 32 lowercase hex characters")
}
k.Fuzzy = antispam.NewSecret(values[antispam.KeyFuzzy])
k.Shingles = antispam.NewSecret(values[antispam.KeyShingles])
k.Prefix = values[antispam.KeyPrefix]
var err error
if k.Read, err = parseStoredKeypair(values[antispam.KeyRead]); err != nil {
return antispam.LearningKeys{}, fmt.Errorf("learning key read.keypair: %w", err)
}
if k.Write, err = parseStoredKeypair(values[antispam.KeyWrite]); err != nil {
return antispam.LearningKeys{}, fmt.Errorf("learning key write.keypair: %w", err)
}
if err := distinctLearningKeypairs(k.Read, k.Write); err != nil {
return antispam.LearningKeys{}, err
}
return k, nil
}
func openLearningKeysDir(dir string, create bool) (*os.Root, error) {
if create {
if err := os.MkdirAll(filepath.Dir(dir), 0o700); err != nil {
return nil, errors.New("cannot create learning parent directory")
}
}
learning, err := openPrivateKeyDir(nil, dir, "learning directory", create)
if err != nil {
return nil, err
}
defer func() { _ = learning.Close() }()
return openPrivateKeyDir(learning, "keys", "learning keys directory", create)
}
// Pin each checked directory so later I/O cannot follow a replacement link.
func openPrivateKeyDir(parent *os.Root, name, label string, create bool) (*os.Root, error) {
mkdir, lstat, open := os.Mkdir, os.Lstat, os.OpenRoot
if parent != nil {
mkdir, lstat, open = parent.Mkdir, parent.Lstat, parent.OpenRoot
}
if create {
if err := mkdir(name, 0o700); err != nil && !errors.Is(err, fs.ErrExist) {
return nil, fmt.Errorf("cannot create %s", label)
}
}
info, err := lstat(name)
switch {
case err != nil:
return nil, fmt.Errorf("%s cannot be read", label)
case !info.IsDir():
return nil, fmt.Errorf("%s is not a directory", label)
case info.Mode().Perm() != 0o700:
return nil, fmt.Errorf("%s is not mode 700", label)
case fileUID(info) != keyOwner:
return nil, fmt.Errorf("%s is not owned by root", label)
}
root, err := open(name)
if err != nil {
return nil, fmt.Errorf("cannot open %s", label)
}
current, err := root.Stat(".")
if err != nil || !os.SameFile(info, current) {
_ = root.Close()
return nil, fmt.Errorf("%s changed while opening", label)
}
return root, nil
}
func readKeyFile(dir *os.Root, name string) (string, error) {
entry, err := dir.Lstat(name)
if errors.Is(err, fs.ErrNotExist) {
return "", errors.New("is missing")
}
if err != nil {
return "", errors.New("cannot be checked")
}
if !entry.Mode().IsRegular() {
return "", errors.New("is not a regular file")
}
// Root.OpenFile follows links within the root, even with O_NOFOLLOW.
// Match the opened inode to the checked entry before reading any bytes.
f, err := dir.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK, 0)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return "", errors.New("is missing")
}
return "", errors.New("cannot be opened as a regular file")
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
switch {
case err != nil:
return "", errors.New("cannot be read")
case !info.Mode().IsRegular():
return "", errors.New("is not a regular file")
case !os.SameFile(entry, info):
return "", errors.New("changed while opening")
case info.Mode().Perm() != 0o600:
return "", errors.New("is not mode 600")
case fileUID(info) != keyOwner:
return "", errors.New("is not owned by root")
}
data, err := io.ReadAll(io.LimitReader(f, maxLearningKeyBytes+1))
switch {
case err != nil:
return "", errors.New("cannot be read")
case len(data) > maxLearningKeyBytes:
return "", errors.New("is larger than a key file")
}
return string(data), nil
}
func fileUID(info fs.FileInfo) int {
if st, ok := info.Sys().(*syscall.Stat_t); ok {
return int(st.Uid)
}
return -1
}
func parseStoredKeypair(data string) (antispam.Keypair, error) {
var stored storedKeypair
decoder := json.NewDecoder(strings.NewReader(data))
decoder.DisallowUnknownFields()
if decoder.Decode(&stored) != nil || decoder.Decode(new(any)) != io.EOF {
return antispam.Keypair{}, errors.New("is not a keypair object")
}
kp := antispam.Keypair{Public: stored.Pubkey, Private: antispam.NewSecret(stored.Privkey)}
if err := kp.Validate(); err != nil {
return antispam.Keypair{}, err
}
return kp, nil
}
// NewFuzzyKeypair runs `rspamadm keypair -j` and returns its keypair.
// rspamd prints it either wrapped in a "keypair" object or bare.
func NewFuzzyKeypair(ctx context.Context, run func(context.Context, time.Duration, string, ...string) antispam.CommandResult) (antispam.Keypair, error) {
res := run(ctx, commandTimeout, "rspamadm", "keypair", "-j")
if res.Err != nil {
return antispam.Keypair{}, errors.New("rspamadm keypair failed")
}
var out struct {
Keypair *storedKeypair `json:"keypair"`
storedKeypair
}
if json.Unmarshal([]byte(res.Stdout), &out) != nil {
return antispam.Keypair{}, errors.New("rspamadm keypair printed no JSON keypair")
}
stored := out.storedKeypair
if out.Keypair != nil {
stored = *out.Keypair
}
kp := antispam.Keypair{Public: stored.Pubkey, Private: antispam.NewSecret(stored.Privkey)}
if err := kp.Validate(); err != nil {
return antispam.Keypair{}, fmt.Errorf("rspamadm keypair: %w", err)
}
return kp, nil
}
// EnsureLearningKeys creates the key set under dir/keys once. An existing
// complete set is checked and kept; a partial set is an error and nothing is
// written, because new keys would silently orphan every published hash.
func EnsureLearningKeys(ctx context.Context, dir string, newKeypair func(context.Context) (antispam.Keypair, error), rnd io.Reader) (bool, error) {
keys, err := openLearningKeysDir(dir, true)
if err != nil {
return false, err
}
defer func() { _ = keys.Close() }()
var present, missing []string
for _, name := range antispam.LearningKeyNames {
if _, statErr := keys.Lstat(name); statErr == nil {
present = append(present, name)
} else if errors.Is(statErr, fs.ErrNotExist) {
missing = append(missing, name)
} else {
return false, fmt.Errorf("learning key %s cannot be checked", name)
}
}
switch {
case len(missing) == 0:
_, err = loadLearningKeys(keys)
return false, err
case len(present) > 0:
return false, fmt.Errorf("learning keys are incomplete: missing %s; CSM never regenerates part of a key set", strings.Join(missing, ", "))
}
values := make(map[string]string, len(antispam.LearningKeyNames))
for _, name := range []string{antispam.KeyFuzzy, antispam.KeyShingles} {
if values[name], err = randomHex(rnd, 32); err != nil {
return false, err
}
}
prefix, err := randomHex(rnd, 16)
if err != nil {
return false, err
}
values[antispam.KeyPrefix] = "csmfz_" + prefix
var read antispam.Keypair
for _, name := range []string{antispam.KeyRead, antispam.KeyWrite} {
kp, kpErr := newKeypair(ctx)
if kpErr != nil {
return false, fmt.Errorf("learning key %s: keypair generation failed", name)
}
if err = kp.Validate(); err != nil {
return false, fmt.Errorf("learning key %s: %w", name, err)
}
if name == antispam.KeyRead {
read = kp
} else if err = distinctLearningKeypairs(read, kp); err != nil {
return false, err
}
data, _ := json.Marshal(storedKeypair{Pubkey: kp.Public, Privkey: kp.Private.Reveal()})
values[name] = string(data)
}
for _, name := range antispam.LearningKeyNames {
if err = writeNewKeyFile(keys, name, values[name]+"\n"); err != nil {
return false, fmt.Errorf("learning key %s: %w", name, err)
}
}
if err = syncKeysDir(keys); err != nil {
return false, err
}
if _, err = LoadLearningKeys(dir); err != nil {
return false, err
}
return true, nil
}
func distinctLearningKeypairs(read, write antispam.Keypair) error {
if read.Public == write.Public || read.Private.Reveal() == write.Private.Reveal() {
return errors.New("learning read and write keypairs must be distinct")
}
return nil
}
func randomHex(rnd io.Reader, n int) (string, error) {
b := make([]byte, n)
if _, err := io.ReadFull(rnd, b); err != nil {
return "", errors.New("cannot read random bytes for learning keys")
}
return hex.EncodeToString(b), nil
}
func writeNewKeyFile(dir *os.Root, name, content string) error {
f, err := dir.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600)
if err != nil {
return errors.New("cannot be created")
}
if _, err := f.WriteString(content); err != nil {
_ = f.Close()
return errors.New("cannot be written")
}
if err := f.Sync(); err != nil {
_ = f.Close()
return errors.New("cannot be synced")
}
if err := f.Close(); err != nil {
return errors.New("cannot be closed")
}
return nil
}
func syncKeysDir(dir *os.Root) error {
d, err := dir.Open(".")
if err != nil {
return errors.New("cannot open learning keys directory to sync it")
}
defer func() { _ = d.Close() }()
if err := d.Sync(); err != nil {
return errors.New("cannot sync learning keys directory")
}
return nil
}
// Package hostfacts reads the facts antispam.CheckHost evaluates. It only
// reads: files, lstat results, command output and one rspamd /ping.
package hostfacts
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/safepath"
)
// maxSymlinks matches the kernel's limit on links followed in one lookup.
const maxSymlinks = 40
// pathFacts is one lstat of p, with its extended ACL state.
func pathFacts(p string) (antispam.PathFacts, error) {
info, err := os.Lstat(p)
if err != nil {
return antispam.PathFacts{}, err
}
st, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return antispam.PathFacts{}, fmt.Errorf("%s: no ownership information", p)
}
acl := false
if info.Mode()&fs.ModeSymlink == 0 {
if acl, err = hasExtACL(p); err != nil {
return antispam.PathFacts{}, fmt.Errorf("%s: %w", p, err)
}
}
return antispam.PathFacts{Path: p, UID: int(st.Uid), GID: int(st.Gid), Mode: info.Mode(), ExtACL: acl}, nil
}
// chain lstats every directory from "/" down to dir without following
// symbolic links. It stops after an entry that is not a directory, which
// the checks reject.
func chain(dir string) ([]antispam.PathFacts, error) {
paths := []string{"/"}
for _, name := range names(dir) {
paths = append(paths, filepath.Join(paths[len(paths)-1], name))
}
var out []antispam.PathFacts
for _, p := range paths {
facts, err := pathFacts(p)
if err != nil {
return out, err
}
out = append(out, facts)
if !facts.Mode.IsDir() {
break
}
}
return out, nil
}
// resolve follows p the way connect(2) does. It returns the object p names
// and every directory a name was looked up in, in lookup order without
// repeats. A followed symlink's own directory is among them: whoever can
// write there can redirect the path.
func resolve(p string) (string, []string, error) {
return resolveChecked(p, nil)
}
// resolveChecked checks each directory entry before following it. The
// deduplicated directory list alone cannot prove ownership of symlinks.
func resolveChecked(p string, check func(dir, next string) error) (string, []string, error) {
var dirs []string
seen := make(map[string]bool)
cur, rest, links := "/", lookupNames(p), 0
for len(rest) > 0 {
name := rest[0]
rest = rest[1:]
if !seen[cur] {
seen[cur] = true
dirs = append(dirs, cur)
}
next := filepath.Join(cur, name)
if check != nil {
if err := check(cur, next); err != nil {
return next, dirs, err
}
}
switch name {
case ".":
continue
case "..":
cur = filepath.Dir(cur)
continue
}
info, err := os.Lstat(next)
if err != nil {
return next, dirs, err
}
if info.Mode()&fs.ModeSymlink != 0 {
if links++; links > maxSymlinks {
return next, dirs, errors.New("too many levels of symbolic links")
}
target, err := os.Readlink(next)
if err != nil {
return next, dirs, err
}
if filepath.IsAbs(target) {
cur = "/"
}
rest = append(lookupNames(target), rest...)
continue
}
if len(rest) > 0 && !info.IsDir() {
return next, dirs, fmt.Errorf("%s: %w", next, syscall.ENOTDIR)
}
cur = next
}
return cur, dirs, nil
}
func names(p string) []string {
return strings.FieldsFunc(p, func(r rune) bool { return r == '/' })
}
func lookupNames(p string) []string {
out := names(p)
if strings.HasSuffix(p, "/") {
// A trailing slash requires a directory even in a symlink target.
out = append(out, ".")
}
return out
}
func readRegularContent(p string, maxBytes int64) (string, error) {
dir, err := safepath.OpenDirNoFollow(filepath.Dir(p))
if err != nil {
return "", err
}
defer func() { _ = dir.Close() }()
file, err := dir.OpenFile(filepath.Base(p), os.O_RDONLY, 0)
if err != nil {
return "", err
}
defer func() { _ = file.Close() }()
info, err := file.Stat()
if err != nil {
return "", err
}
if !info.Mode().IsRegular() {
return "", errors.New("file is not a regular file")
}
return readConfigContent(file, maxBytes)
}
// Targets describes each mutation target, in order, as Collect does.
func Targets(targets []antispam.Target) []antispam.TargetFacts {
out := make([]antispam.TargetFacts, 0, len(targets))
for _, t := range targets {
out = append(out, targetFacts(t))
}
return out
}
// targetFacts describes one mutation target.
func targetFacts(t antispam.Target) antispam.TargetFacts {
f := antispam.TargetFacts{Target: t}
if f.Parents, f.Err = chain(filepath.Dir(t.Path)); f.Err != nil {
return f
}
if last := f.Parents[len(f.Parents)-1]; !last.Mode.IsDir() {
return f
}
facts, err := pathFacts(t.Path)
switch {
case errors.Is(err, fs.ErrNotExist):
f.Missing = true
return f
case err != nil:
f.Err = err
return f
}
f.File = facts
if ContentReadable(f) {
f.Content, f.Err = readRegularContent(t.Path, maxConfigBytes)
}
return f
}
// ContentReadable reports whether the collector reads target f's content.
// An optional file is read whenever its directories are protected, even if
// the file itself was loosened: CSM must still recognise its own file, which
// can hold a secret, to delete or replace it. Deciding that needs only a
// safe directory, and the file is never edited in place.
func ContentReadable(f antispam.TargetFacts) bool {
if !f.Kind.Optional() {
return targetSafeToRead(f)
}
relaxed := f
relaxed.File.UID, relaxed.File.ExtACL = 0, false
relaxed.File.Mode = f.File.Mode &^ 0o022
return targetSafeToRead(relaxed)
}
func targetSafeToRead(f antispam.TargetFacts) bool {
protected := func(p antispam.PathFacts) bool {
return p.UID == 0 && p.Mode.Perm()&0o022 == 0 && !p.ExtACL
}
if !f.File.Mode.IsRegular() || !protected(f.File) {
return false
}
for _, parent := range f.Parents {
if !parent.Mode.IsDir() || !protected(parent) {
return false
}
}
return true
}
package hostfacts
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"io/fs"
"net"
"net/http"
"strings"
"time"
"github.com/pidginhost/csm/internal/antispam"
)
// Probe limits.
const (
pingTimeout = 3 * time.Second
maxPingBody = 4 << 10
// Framing has its own allowance so headers and chunk metadata cannot
// consume unbounded memory before the decoded body limit applies.
maxPingReply = 2 * maxPingBody
)
// Ping sends GET /ping over the rspamd socket and wants HTTP 200 with "pong"
// (rspamd ends it with CRLF). It returns the listener's uid from SO_PEERCRED,
// or -1 when no connection was made; the uid stays known when the reply is
// wrong. Errors never repeat the reply.
func Ping(ctx context.Context, socket string) (int, error) {
ctx, cancel := context.WithTimeout(ctx, pingTimeout)
defer cancel()
var d net.Dialer
conn, err := d.DialContext(ctx, "unix", socket)
if err != nil {
return -1, err
}
defer func() { _ = conn.Close() }()
uid := peerUID(conn)
if deadline, ok := ctx.Deadline(); ok {
_ = conn.SetDeadline(deadline)
}
// Install cancellation last so an expired deadline cannot be replaced
// by the later timeout when the caller cancels during setup.
stop := context.AfterFunc(ctx, func() { _ = conn.SetDeadline(time.Now()) })
defer stop()
if _, err = io.WriteString(conn, "GET /ping HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"); err != nil {
return uid, err
}
return uid, readPingReply(conn)
}
func readPingReply(conn io.ReadCloser) error {
reply := &io.LimitedReader{R: conn, N: maxPingReply + 1}
resp, err := http.ReadResponse(bufio.NewReader(reply), nil)
if err != nil {
return errors.New("rspamd /ping returned an invalid HTTP response")
}
defer func() {
// ReadResponse bodies can drain unread data on Close. This probe
// never reuses its connection, so close it before closing the body.
_ = conn.Close()
_ = resp.Body.Close()
}()
body, err := io.ReadAll(io.LimitReader(resp.Body, maxPingBody+1))
switch {
case err != nil:
return errors.New("rspamd /ping reply could not be read")
case reply.N == 0:
return errors.New("rspamd /ping reply is too large")
case resp.StatusCode != http.StatusOK:
return fmt.Errorf("rspamd /ping returned HTTP %d", resp.StatusCode)
case len(body) > maxPingBody:
return errors.New("rspamd /ping reply is too large")
case strings.Trim(string(body), " \t\r\n") != "pong":
return errors.New("rspamd /ping did not answer pong")
}
return nil
}
// socketFacts describes the scan socket, the directories its path walks
// through and one /ping. The caller fills the uids it knows.
func socketFacts(ctx context.Context, socket string) antispam.SocketFacts {
f := antispam.SocketFacts{Socket: antispam.PathFacts{Path: socket}, RspamdUID: -1, PeerUID: -1, EximUID: -1, EximGID: -1}
if err := antispam.ValidateSocket(socket); err != nil {
f.Err = err
return f
}
final, dirs, walkErr := resolve(socket)
for _, d := range dirs {
facts, err := pathFacts(d)
if err != nil {
f.Err = err
return f
}
f.Parents = append(f.Parents, facts)
}
switch {
case errors.Is(walkErr, fs.ErrNotExist):
f.Missing = true
return f
case walkErr != nil:
f.Err = walkErr
return f
}
facts, err := pathFacts(final)
if err != nil {
f.Err = err
return f
}
f.Socket = facts
f.PeerUID, f.PingErr = Ping(ctx, socket)
return f
}
package hostfacts
import (
"bufio"
"context"
"encoding/json"
"errors"
"net"
"net/netip"
"slices"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
"github.com/pidginhost/csm/internal/antispam"
)
const redisTimeout = 500 * time.Millisecond
// RedisProbe reads the live settings learning depends on from the Redis
// Unix socket at endpoint. The password is sent only on a connection whose
// peer is uid peer, the socket's verified owner: a process squatting the path
// gets nothing. It only reads: CONFIG GET and INFO. Errors are fixed text
// naming the step; the password never appears in them.
func RedisProbe(ctx context.Context, endpoint string, password antispam.Secret, peer int) antispam.RedisFacts {
r := antispam.RedisFacts{Endpoint: endpoint, Unix: strings.HasPrefix(endpoint, "/"), ServiceUID: -1}
if !r.Unix {
r.Err = errors.New("redis: CSM probes only a Unix socket")
return r
}
errPeer := errors.New("redis: the socket is not served by its owner")
client := redis.NewClient(&redis.Options{
Network: "unix", Addr: endpoint, Password: password.Reveal(),
Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
conn, err := (&net.Dialer{Timeout: redisTimeout}).DialContext(ctx, network, addr)
if err != nil {
return nil, errors.New("redis connection failed")
}
if peerUID(conn) != peer {
_ = conn.Close()
return nil, errPeer
}
return conn, nil
},
// RESP2 without CLIENT SETINFO: the probe must work on Redis 5 and
// send nothing but reads.
Protocol: 2, DisableIdentity: true,
DialTimeout: redisTimeout, ReadTimeout: redisTimeout, WriteTimeout: redisTimeout,
MaxRetries: -1, PoolSize: 1, DialerRetries: 1, ContextTimeoutEnabled: true,
})
defer func() { _ = client.Close() }()
probe := client.Conn()
defer func() { _ = probe.Close() }()
// One connection supplies the peer check and every value, so a replaced
// socket cannot answer part of the probe.
if err := probe.Ping(ctx).Err(); err != nil {
r.Err = errors.New("redis: cannot connect to the socket as its owner")
if errors.Is(err, errPeer) {
r.Err = errPeer
r.Unprotected = true
}
return r
}
r.ServiceUID = peer
// Each value is kept as soon as it is read: a later failure must not
// erase evidence such as a TCP listener.
for _, name := range []string{"port", "tls-port", "maxmemory", "maxmemory-policy", "appendonly", "save"} {
got, err := probe.ConfigGet(ctx, name).Result()
if err != nil {
r.Err = errors.New("redis: CONFIG GET " + name + " failed")
return r
}
value, ok := got[name]
// Redis builds without TLS omit this option entirely.
if name == "tls-port" && !ok {
value, ok = "0", true
}
if !ok {
r.Err = errors.New("redis: CONFIG GET " + name + " returned no value")
return r
}
switch name {
case "port":
if _, err = strconv.ParseUint(value, 10, 16); err != nil {
r.Err = errors.New("redis: port is not a port number")
return r
}
r.Port = value
case "tls-port":
if _, err = strconv.ParseUint(value, 10, 16); err != nil {
r.Err = errors.New("redis: tls-port is not a port number")
return r
}
r.TLSPort = value
case "maxmemory":
if r.MaxMemory, err = strconv.ParseUint(value, 10, 64); err != nil {
r.Err = errors.New("redis: maxmemory is not a number")
return r
}
r.MaxMemoryKnown = true
case "maxmemory-policy":
if strings.TrimSpace(value) == "" {
r.Err = errors.New("redis: memory policy is not readable")
return r
}
r.Policy = value
case "appendonly":
if value != "yes" && value != "no" {
r.Err = errors.New("redis: persistence setting is not readable")
return r
}
r.AppendOnly = value
case "save":
r.Save, r.SaveKnown = value, true
}
}
memory, err := probe.Info(ctx, "memory").Result()
if err != nil {
r.Err = errors.New("redis: INFO memory failed")
return r
}
used, ok := infoField(memory, "used_memory")
if r.Used, err = strconv.ParseUint(used, 10, 64); !ok || err != nil {
r.Err = errors.New("redis: INFO memory has no used_memory")
return r
}
server, err := probe.Info(ctx, "server").Result()
if err != nil {
r.Err = errors.New("redis: INFO server failed")
return r
}
if r.Version, ok = infoField(server, "redis_version"); !ok {
r.Err = errors.New("redis: INFO server has no redis_version")
}
return r
}
func infoField(info, name string) (string, bool) {
s := bufio.NewScanner(strings.NewReader(info))
var found string
seen := false
for s.Scan() {
if key, value, ok := strings.Cut(strings.TrimSpace(s.Text()), ":"); ok && key == name {
if seen || value == "" {
return "", false
}
found, seen = value, true
}
}
return found, seen && s.Err() == nil
}
// redisEndpoint follows the fuzzy backend's worker, redis.fuzzy (or
// fuzzy_storage), then global selection. Unsupported or ambiguous settings
// must stop collection before a different Redis instance can be probed.
func redisEndpoint(d antispam.LearningDumps, fullDump []byte) (endpoint string, unix bool, password antispam.Secret, err error) {
global, err := antispam.ParseRspamdDumpObject(d.Redis)
if err != nil {
return "", false, antispam.Secret{}, errors.New("rspamd's redis settings are not readable")
}
worker, err := fuzzyWorker(fullDump)
if err != nil {
return "", false, antispam.Secret{}, err
}
section := worker
if !hasServers(section) {
section = global
for _, name := range []string{"fuzzy", "fuzzy_storage"} {
if raw, ok := global[name]; ok {
specific, parseErr := antispam.ParseRspamdDumpObject(raw)
if parseErr != nil {
return "", false, antispam.Secret{}, errors.New("fuzzy Redis settings are not readable")
}
section = specific
break
}
}
}
// Rspamd inherits authentication and other options from global Redis
// settings even when the worker selects its own servers.
option := func(name string) json.RawMessage {
if raw, ok := section[name]; ok {
return raw
}
return global[name]
}
if err := unprobedRedisOptions(option); err != nil {
return "", false, antispam.Secret{}, err
}
var servers []string
readKey := ""
for _, key := range []string{"read_servers", "servers", "server"} {
if _, ok := section[key]; ok {
readKey = key
break
}
}
for _, key := range []string{readKey, "write_servers"} {
raw, ok := section[key]
if !ok {
continue
}
list, err := redisServers(raw)
if err != nil {
return "", false, antispam.Secret{}, err
}
for _, server := range list {
if !slices.Contains(servers, server) {
servers = append(servers, server)
}
}
}
if readKey == "" || len(servers) == 0 {
return "", false, antispam.Secret{}, errors.New("rspamd has no redis server for fuzzy storage")
}
if len(servers) != 1 {
return "", false, antispam.Secret{}, errors.New("rspamd names more than one redis server for fuzzy storage")
}
if raw := option("password"); raw != nil {
var value string
if string(raw) == "null" || json.Unmarshal(raw, &value) != nil {
return "", false, antispam.Secret{}, errors.New("redis password is not a string")
}
password = antispam.NewSecret(value)
}
return servers[0], strings.HasPrefix(servers[0], "/"), password, nil
}
// unprobedRedisOptions refuses Redis settings that make rspamd reach a
// server other than the one CSM probes, or reach it another way: sentinels
// pick the server at runtime, TLS and named users differ from CSM's plain
// connection.
func unprobedRedisOptions(option func(name string) json.RawMessage) error {
if raw := option("sentinels"); raw != nil {
return errors.New("redis sentinel backend cannot be identified from config alone")
}
if raw := option("ssl"); raw != nil && string(raw) != "false" {
return errors.New("redis TLS backend cannot be probed")
}
if raw := option("username"); raw != nil {
var username string
if string(raw) == "null" || json.Unmarshal(raw, &username) != nil || (username != "" && username != "default") {
return errors.New("redis named-user authentication cannot be probed")
}
}
return nil
}
func redisServers(raw json.RawMessage) ([]string, error) {
var values []string
var single string
if string(raw) == "null" {
return nil, errors.New("redis servers are not readable")
}
if json.Unmarshal(raw, &single) == nil {
values = []string{single}
} else if json.Unmarshal(raw, &values) != nil {
return nil, errors.New("redis servers are not readable")
}
var servers []string
for _, value := range values {
for _, rawServer := range strings.Split(value, ",") {
server, err := redisAddress(strings.TrimSpace(rawServer))
if err != nil {
return nil, err
}
servers = append(servers, server)
}
}
if len(servers) == 0 {
return nil, errors.New("redis servers are empty")
}
return servers, nil
}
func redisAddress(server string) (string, error) {
bad := errors.New("redis server address is not supported")
if server == "" || strings.ContainsAny(server, "@ \t\r\n\x00") {
return "", bad
}
if strings.HasPrefix(server, "/") {
return server, nil
}
if addr, err := netip.ParseAddr(server); err == nil {
return net.JoinHostPort(addr.String(), "6379"), nil
}
if !strings.ContainsAny(server, ":/[]") {
return net.JoinHostPort(server, "6379"), nil
}
host, port, err := net.SplitHostPort(server)
if err != nil || host == "" || strings.ContainsAny(host, "/[]") {
return "", bad
}
if _, err := strconv.ParseUint(port, 10, 16); err != nil || port == "0" {
return "", bad
}
return net.JoinHostPort(host, port), nil
}
func hasServers(section map[string]json.RawMessage) bool {
for _, key := range []string{"servers", "server", "read_servers", "write_servers"} {
if _, ok := section[key]; ok {
return true
}
}
return false
}
func workerEntries(raw json.RawMessage) ([]map[string]json.RawMessage, error) {
bad := errors.New("rspamd's worker settings are not readable")
var objects []json.RawMessage
if json.Unmarshal(raw, &objects) != nil {
objects = []json.RawMessage{raw}
}
if objects == nil {
return nil, bad
}
entries := make([]map[string]json.RawMessage, 0, len(objects))
for _, object := range objects {
entry, err := antispam.ParseRspamdDumpObject(object)
if err != nil {
return nil, bad
}
entries = append(entries, entry)
}
return entries, nil
}
// An absent fuzzy worker is legitimate before installation; malformed or
// repeated workers must never silently select the global Redis instead.
func fuzzyWorker(fullDump []byte) (map[string]json.RawMessage, error) {
root, err := antispam.ParseRspamdDumpObject(fullDump)
if err != nil {
return nil, errors.New("rspamd's worker settings are not readable")
}
entries, err := workerEntries(root["worker"])
if err != nil {
return nil, err
}
var found map[string]json.RawMessage
for _, entry := range entries {
var worker map[string]json.RawMessage
if raw, ok := entry["type"]; ok {
var kind string
if json.Unmarshal(raw, &kind) != nil || kind == "" || strings.ContainsRune(kind, '\x00') {
return nil, errors.New("rspamd's worker type is not readable")
}
if kind == "fuzzy" {
worker = entry
}
} else if raw, ok := entry["fuzzy"]; ok {
worker, err = antispam.ParseRspamdDumpObject(raw)
if err != nil {
return nil, errors.New("rspamd's fuzzy worker is not readable")
}
}
if worker != nil {
if found != nil {
return nil, errors.New("rspamd names more than one fuzzy worker")
}
found = worker
}
}
return found, nil
}
package hostfacts
import (
"errors"
"slices"
"strings"
)
// localAccounts maps the users in /etc/passwd to their uids, the first
// entry of a name winning as with NSS. A record that does not parse may
// hide a user, so the whole file must parse.
func (c collector) localAccounts() (map[string]int, error) {
passwd, err := c.readPasswd()
if err != nil {
return nil, err
}
users := make(map[string]int)
for line := range strings.SplitSeq(string(passwd), "\n") {
line = strings.Trim(line, " \t\r\v\f")
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.SplitN(line, ":", 8)
if len(fields) != 7 || !accountName(fields[0]) {
return nil, errors.New("local account database is not readable")
}
uid, ok := accountID(fields[2])
if !ok {
return nil, errors.New("local account uid is not readable")
}
if _, ok := accountID(fields[3]); !ok {
return nil, errors.New("local account gid is not readable")
}
if _, seen := users[fields[0]]; !seen {
users[fields[0]] = uid
}
}
return users, nil
}
// postfixServiceUsers run no tenant code on a Postfix host. The mail uid
// user (vmail) is not one of them: tenants' sieve scripts and Dovecot's mail
// processes run as it. Being pinned grants no access: Redis trusts only
// root, redis and rspamd's user, the controller only root and rspamd's user.
var postfixServiceUsers = []string{"root", "redis", "dovecot", "dovenull", "postfix"}
// untrustedLocalUsers lists the users that may run tenant code on a Postfix
// host, which has no account registry: every user in /etc/passwd but
// rspamd's service user and postfixServiceUsers. A name whose uid Dovecot
// stores mail as stays untrusted even when pinned.
func (c collector) untrustedLocalUsers(rspamdUser string, mailUIDs []int) (map[string]bool, error) {
users, err := c.localAccounts()
if err != nil {
return nil, err
}
names := make(map[string]bool, len(users))
for name, uid := range users {
if slices.Contains(mailUIDs, uid) || !slices.Contains(postfixServiceUsers, name) && name != rspamdUser {
names[name] = true
}
}
return names, nil
}
package hostfacts
import (
"errors"
"io/fs"
"os"
"path/filepath"
"github.com/pidginhost/csm/internal/antispam"
)
// Replaced lists, in name order, the operator files the worker backed up in
// dir before replacing them. Only regular files with a backup name count:
// entries are judged by their own type, so a link is never followed. A
// missing dir means nothing was replaced.
func Replaced(dir string) ([]antispam.Replacement, error) {
entries, err := os.ReadDir(dir)
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
if err != nil {
return nil, err
}
var out []antispam.Replacement
for _, e := range entries {
if target, ok := antispam.ParseBackupName(e.Name()); ok && e.Type().IsRegular() {
out = append(out, antispam.Replacement{Path: target, Backup: filepath.Join(dir, e.Name())})
}
}
return out, nil
}
package hostfacts
import (
"context"
"errors"
"maps"
"net/netip"
"slices"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/antispam"
)
type resolvedServer struct {
link int
addr netip.Addr
}
// resolvedDNS uses the live inventory because the generated resolv.conf
// omits route-only DNS servers and servers using nonstandard ports.
func (c collector) resolvedDNS(ctx context.Context) ([]netip.Addr, error) {
read := func(property string) ([]resolvedServer, error) {
result := c.resolvedProperty(ctx, "", property)
if result.Err != nil {
return nil, result.Err
}
return parseResolvedDNS(result.Stdout)
}
servers, err := read("DNS")
if err != nil {
return nil, err
}
var out []netip.Addr
links := make(map[int]bool)
defaultRoute := false
for _, server := range servers {
out = append(out, server.addr)
if server.link == 0 {
defaultRoute = true
} else {
links[server.link] = true
}
}
var routeErr error
for _, link := range slices.Sorted(maps.Keys(links)) {
if defaultRoute {
break
}
result := c.resolvedProperty(ctx, resolvedLinkPath(link), "DefaultRoute")
if result.Err != nil {
routeErr = errors.Join(routeErr, result.Err)
continue
}
switch strings.TrimSpace(result.Stdout) {
case "b true":
defaultRoute = true
case "b false":
default:
routeErr = errors.Join(routeErr, errors.New("unreadable systemd-resolved default route"))
}
}
if !defaultRoute {
if routeErr != nil {
return nil, routeErr
}
fallback, err := read("FallbackDNS")
if err != nil {
return nil, err
}
for _, server := range fallback {
out = append(out, server.addr)
}
}
return out, nil
}
// resolvedLinkPath is systemd's bus label for a link: a leading digit is
// escaped as _ and its hex code, so link 2 is /link/_32. Link indexes here
// are positive, so the first character is always a digit from 1 to 9.
func resolvedLinkPath(link int) string {
return "/link/_3" + strconv.Itoa(link)
}
func (c collector) resolvedProperty(ctx context.Context, link, property string) antispam.CommandResult {
iface := "org.freedesktop.resolve1.Manager"
if link != "" {
iface = "org.freedesktop.resolve1.Link"
}
return c.run(ctx, "busctl", "--system", "--auto-start=no", "--no-pager", "get-property",
"org.freedesktop.resolve1", "/org/freedesktop/resolve1"+link, iface, property)
}
// parseResolvedDNS reads busctl's typed a(iiay) output. DNS includes all
// server addresses even when their ports cannot be written to resolv.conf.
func parseResolvedDNS(output string) ([]resolvedServer, error) {
invalid := errors.New("unreadable systemd-resolved DNS inventory")
fields := strings.Fields(output)
if len(fields) < 2 || fields[0] != "a(iiay)" {
return nil, invalid
}
count, err := strconv.ParseUint(fields[1], 10, 32)
fields = fields[2:]
if err != nil || count > uint64(len(fields)/7) {
return nil, invalid
}
var out []resolvedServer
for range count {
if len(fields) < 3 {
return nil, invalid
}
link, err := strconv.ParseInt(fields[0], 10, 32)
if err != nil || link < 0 {
return nil, invalid
}
// These are Linux's wire address families, independent of the
// platform compiling this parser.
size := 0
switch fields[1] {
case "2":
size = 4
case "10":
size = 16
default:
return nil, invalid
}
if fields[2] != strconv.Itoa(size) || len(fields) < 3+size {
return nil, invalid
}
var bytes [16]byte
for i, field := range fields[3 : 3+size] {
value, err := strconv.ParseUint(field, 10, 8)
if err != nil {
return nil, invalid
}
bytes[i] = byte(value)
}
addr, _ := netip.AddrFromSlice(bytes[:size])
out = append(out, resolvedServer{link: int(link), addr: addr})
fields = fields[3+size:]
}
if len(fields) != 0 {
return nil, invalid
}
return out, nil
}
package hostfacts
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"github.com/pidginhost/csm/internal/antispam"
)
// Reply bounds for the stats card, and how many rules it lists.
const (
maxStatReply = 1 << 20
maxCountersReply = 4 << 20
statsTopRules = 8
)
// loopbackClient talks only to the address it is given: no proxy from the
// environment, no redirects, no connection reuse.
var loopbackClient = &http.Client{
Timeout: controllerTimeout,
Transport: &http.Transport{
Proxy: nil,
DisableKeepAlives: true,
DisableCompression: true,
MaxResponseHeaderBytes: 64 << 10,
},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
// ReadRspamdStats reads /stat and /counters from the endpoint
// antispam.StatsEndpointFor chose. It never sends a password. A failed
// /stat fails the read; a failed /counters only leaves TopRules nil.
// Errors are fixed text; they never repeat a reply or the endpoint.
func ReadRspamdStats(ctx context.Context, ep antispam.StatsEndpoint) (antispam.RspamdStats, error) {
if ep.Address == "" {
return antispam.RspamdStats{}, errors.New("no rspamd controller endpoint to read")
}
get := func(path string, limit int64) ([]byte, error) {
if ep.Unix {
return ControllerGet(ctx, ep.Address, path, limit, ep.RspamdUID)
}
return loopbackGet(ctx, ep.Address, path, limit)
}
body, err := get("/stat", maxStatReply)
if err != nil {
return antispam.RspamdStats{}, err
}
stats, err := antispam.ParseRspamdStat(body)
if err != nil {
return antispam.RspamdStats{}, err
}
if body, err = get("/counters", maxCountersReply); err == nil {
// ParseRspamdCounters returns nil with its error.
stats.TopRules, _ = antispam.ParseRspamdCounters(body, statsTopRules)
}
if stats.Version == "" {
stats.Version = ep.Version
}
return stats, nil
}
// loopbackGet sends a plain GET to a loopback controller listener that
// trusts its peer address, with the same Host header ControllerDo sends.
func loopbackGet(ctx context.Context, addr, path string, limit int64) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+addr+path, nil)
if err != nil {
return nil, errors.New("invalid rspamd controller address")
}
req.Host = "localhost"
resp, err := loopbackClient.Do(req)
if err != nil {
return nil, errors.New("cannot reach rspamd's controller")
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("rspamd's controller returned HTTP %d", resp.StatusCode)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
switch {
case err != nil:
return nil, errors.New("rspamd's controller reply could not be read")
case int64(len(data)) > limit:
return nil, errors.New("rspamd's controller reply is too large")
}
return data, nil
}
//go:build linux
package hostfacts
import (
"errors"
"net"
"golang.org/x/sys/unix"
)
// Default ACLs also matter: they can grant access to the files CSM creates
// even when the directory's current access permissions are safe.
func hasExtACL(p string) (bool, error) {
for _, attribute := range []string{"system.posix_acl_access", "system.posix_acl_default"} {
_, err := unix.Lgetxattr(p, attribute, nil)
switch {
case err == nil:
return true, nil
case errors.Is(err, unix.ENODATA), errors.Is(err, unix.EOPNOTSUPP):
continue
default:
return false, err
}
}
return false, nil
}
// peerUID is the uid SO_PEERCRED reports for the listener behind conn, or -1.
func peerUID(conn net.Conn) int {
uc, ok := conn.(*net.UnixConn)
if !ok {
return -1
}
raw, err := uc.SyscallConn()
if err != nil {
return -1
}
uid := -1
_ = raw.Control(func(fd uintptr) {
cred, err := unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED) // #nosec G115 -- a socket fd fits in int
if err == nil {
uid = int(cred.Uid)
}
})
return uid
}
package learn
import (
"math"
"slices"
)
// JoinBoundary is the probability at which a near match joins a live anchor.
const JoinBoundary = 0.75
// Contribution weights.
const (
UIDWeight = 2
OperatorWeight = 18
)
// Per-account and operator state allowances. BenchmarkClusterBytes measured
// 3317 logical ledger bytes per cluster of 8 aliases, charges included, for
// one contributor mix; shared clusters and bbolt overhead vary independently.
const (
// AccountBodyBytes holds six anchors at the 10 MiB message limit, or
// thousands of typical 4 to 20 KiB spam messages.
AccountBodyBytes = 64 << 20
// AccountAliases limits charged aliases, not the ledger's byte size.
AccountAliases = 4096
// The operator's allowances are eight times an account's.
OperatorBodyBytes = 512 << 20
OperatorAliases = 8 * AccountAliases
)
// Skip reasons: terminal outcomes for one digest of a report.
const (
SkipDuplicate = "duplicate"
SkipBelowJoin = "below_join"
SkipMultipartSeed = "multipart_seed"
SkipNoDigest = "no_digest"
SkipAccountCapacity = "account_capacity"
SkipOperatorCapacity = "operator_capacity"
SkipOverflow = "overflow"
)
// Source is who reports and how large the snapshot is. Operator is a source
// kind, never a tenant UID.
type Source struct {
UID int
Operator bool
MarkID string
BodyLen int64
}
// Join is one new contribution to an existing cluster.
type Join struct {
ClusterID uint64
Aliases []Digest
NewT int32
}
// Skip is a terminal outcome for one digest; an empty digest means the
// whole report.
type Skip struct {
Digest Digest
Reason string
}
// Admission is what a report may reserve. Duplicates names clusters the
// reporter already voted on. Defer means reserve nothing and re-inspect
// later; Mismatch names unknown stored digests (state_mismatch); DeleteZero
// names zero-target digests that reappeared.
type Admission struct {
Joins []Join
Seed *Seed
Skips []Skip
Duplicates []uint64
Defer bool
Mismatch []Digest
DeleteZero []Digest
}
// LedgerReader is what assignment reads; LedgerView implements it.
type LedgerReader interface {
Alias(d Digest) (uint64, bool, error)
Cluster(id uint64) (Cluster, bool, error)
Target(d Digest) (Target, bool, error)
Usage(uid int, operator bool) (Usage, error)
}
var _ LedgerReader = LedgerView{}
// resolved is one digest's assignment before votes are counted.
type resolved struct {
digest Digest
cluster uint64
alias bool // an accepted alias, not a fresh match
skip string
miss bool
}
// Assign applies the assignment rules to a complete inspection.
// gaps holds clusters with a known publication gap (anchor missing, being
// deleted and re-added, or awaiting first publication); while any exists no
// fresh match or seed is resolved, since a miss may be that anchor. A ledger
// read error defers the whole report.
func Assign(insp Inspection, src Source, v LedgerReader, gaps map[uint64]bool) (Admission, error) {
var a Admission
if insp.Distinct == 0 {
a.Skips = []Skip{{Reason: SkipNoDigest}}
return a, nil
}
parts := make([]resolved, 0, len(insp.Results))
for _, r := range insp.Results {
p, err := resolve(r, v, &a)
if err != nil {
return Admission{}, err
}
parts = append(parts, p)
}
if len(a.Mismatch) > 0 || len(a.DeleteZero) > 0 {
return Admission{Mismatch: a.Mismatch, DeleteZero: a.DeleteZero, Defer: true}, nil
}
hasGap := false
for _, gap := range gaps {
if gap {
hasGap = true
break
}
}
for _, p := range parts {
// A below-boundary hit may be a fallback for a missing anchor.
if gaps[p.cluster] || (!p.alias && hasGap) {
return Admission{Defer: true}, nil
}
}
if len(parts) == 1 && parts[0].miss {
t := int32(UIDWeight)
if src.Operator {
t = OperatorWeight
}
a.Seed = &Seed{Anchor: parts[0].digest, T: t}
return capacity(a, src, v, 1, src.BodyLen, parts)
}
newAliases := 0
for _, id := range clusterOrder(parts) {
c, ok, err := v.Cluster(id)
if err != nil {
return Admission{}, err
}
if !ok {
// An alias or target names a cluster the ledger lacks.
var named []Digest
for _, p := range parts {
if p.cluster == id {
named = append(named, p.digest)
}
}
return Admission{Mismatch: named, Defer: true}, nil
}
voted := slices.Contains(c.UIDs, src.UID) && !src.Operator || src.Operator && slices.Contains(c.Marks, src.MarkID)
if voted {
a.Duplicates = append(a.Duplicates, id)
for _, p := range parts {
if p.cluster == id {
a.Skips = append(a.Skips, Skip{Digest: p.digest, Reason: SkipDuplicate})
}
}
continue
}
t, ok, err := v.Target(c.Anchor)
if err != nil {
return Admission{}, err
}
if !ok || t.T <= 0 {
// Retirement or forget is under way: wait for it to finish.
return Admission{Defer: true}, nil
}
weight := int64(UIDWeight)
if src.Operator {
weight = OperatorWeight
}
if int64(t.T)+weight > math.MaxInt32 {
return rejectAll(parts, SkipOverflow), nil
}
j := Join{ClusterID: id, NewT: t.T + int32(weight)}
for _, p := range parts {
if p.cluster == id && !p.alias {
j.Aliases = append(j.Aliases, p.digest)
}
}
newAliases += len(j.Aliases)
a.Joins = append(a.Joins, j)
}
for _, p := range parts {
switch {
case p.skip != "":
a.Skips = append(a.Skips, Skip{Digest: p.digest, Reason: p.skip})
case p.miss:
a.Skips = append(a.Skips, Skip{Digest: p.digest, Reason: SkipMultipartSeed})
}
}
return capacity(a, src, v, newAliases, 0, parts)
}
// resolve maps one digest: accepted aliases first, then a live anchor
// matched at or above the join boundary.
func resolve(r Result, v LedgerReader, a *Admission) (resolved, error) {
p := resolved{digest: r.Queried}
id, ok, err := v.Alias(r.Queried)
if err != nil || ok {
p.cluster, p.alias = id, ok
return p, err
}
if !r.Found {
p.miss = true
return p, nil
}
t, ok, err := v.Target(r.Stored)
switch {
case err != nil:
return p, err
case !ok:
a.Mismatch = append(a.Mismatch, r.Stored)
case t.T == 0:
a.DeleteZero = append(a.DeleteZero, r.Stored)
case r.Prob < JoinBoundary:
p.skip = SkipBelowJoin
default:
p.cluster = t.ClusterID
}
return p, nil
}
func clusterOrder(parts []resolved) []uint64 {
var ids []uint64
for _, p := range parts {
if p.cluster != 0 && !slices.Contains(ids, p.cluster) {
ids = append(ids, p.cluster)
}
}
return ids
}
func rejectAll(parts []resolved, reason string) Admission {
var a Admission
for _, p := range parts {
a.Skips = append(a.Skips, Skip{Digest: p.digest, Reason: reason})
}
return a
}
// capacity rejects the whole admission when it would pass the reporter's
// allowance; existing guards stay.
func capacity(a Admission, src Source, v LedgerReader, aliases int, bodyBytes int64, parts []resolved) (Admission, error) {
if len(a.Joins) == 0 && a.Seed == nil {
return a, nil
}
u, err := v.Usage(src.UID, src.Operator)
if err != nil {
return Admission{}, err
}
maxBody, maxAliases, reason := int64(AccountBodyBytes), AccountAliases, SkipAccountCapacity
if src.Operator {
maxBody, maxAliases, reason = OperatorBodyBytes, OperatorAliases, SkipOperatorCapacity
}
if u.BodyBytes+bodyBytes > maxBody || (aliases > 0 && u.Aliases+aliases > maxAliases) {
return rejectAll(parts, reason), nil
}
return a, nil
}
package learn
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"time"
"github.com/pidginhost/csm/internal/store"
)
// AuditBatch is the number of targets one audit turn checks, in one exact
// inspection. BenchmarkAuditRotation took 400 turns for 10000 live and
// 10000 zero targets, about 67 minutes at one turn per EngineTick.
// OldestUnchecked can span two rotations, so single-rotation throughput
// alone does not bound the age used by AuditOverdueAfter.
const AuditBatch = 50
// Retention is how long a cluster lives after its latest committed or
// reserved admission.
const Retention = 90 * 24 * time.Hour
const (
auditCursorKey = "audit"
repairCursorKey = "repair"
watermarkKey = "retire_watermark"
)
// AuditProgress reports one audit turn. Repaired counts targets scheduled
// for repair; OldestUnchecked is the start of the previous rotation, the
// latest time every target not yet checked in this one was checked.
type AuditProgress struct {
Checked, Repaired int
RotationDone bool
OldestUnchecked time.Time
}
type auditCursor struct {
After Digest `json:"after"`
RotationStart time.Time `json:"rotation_start"`
PrevStart time.Time `json:"prev_start"`
}
// auditState is in memory on purpose: after a restart fresh matching waits
// for a complete rotation again.
type auditState struct {
init bool
coverPending bool // a full rotation is still owed
coverFrom Digest // cursor when the rotation became owed
wrapped bool
coverReached bool
covered bool // fresh matching allowed
rebuilding bool
drift map[Digest]int
reconstructed uint64 // missing live anchors newly scheduled for repair
}
// FreshMatchingAllowed reports whether a full audit and repair rotation has
// completed since start or since anchors were found missing.
func (p *Publisher) FreshMatchingAllowed() bool { return p.audit.covered }
// RebuildNeeded reports live anchors found missing and not yet restored by a
// full rotation (backend_rebuilding).
func (p *Publisher) RebuildNeeded() bool { return p.audit.rebuilding }
// RepeatedDrift counts digests found drifted more than once since start.
func (p *Publisher) RepeatedDrift() int {
n := 0
for _, c := range p.audit.drift {
if c > 1 {
n++
}
}
return n
}
// owe makes fresh matching wait for a rotation that starts at from.
func (p *Publisher) owe(from Digest) {
p.audit.coverPending, p.audit.coverFrom, p.audit.wrapped = true, from, false
p.audit.coverReached, p.audit.covered = false, false
}
// AuditTurn checks the next batch of targets, live and zero, in a persisted
// round-robin order with one exact inspection. Drifted or unfinished
// targets become repair work; nothing is reset unconditionally.
func (p *Publisher) AuditTurn(ctx context.Context, batch int, now time.Time) (AuditProgress, error) {
if batch <= 0 || batch > MaxExplicitDigests {
batch = AuditBatch
}
var cur auditCursor
var targets []Target
err := p.ledger.View(func(v LedgerView) error {
if _, err := v.get(store.LearnCursorBucket, []byte(auditCursorKey), &cur); err != nil {
return err
}
return v.tx.Scan(store.LearnTargetsBucket, []byte(cur.After), func(_, data []byte) (bool, error) {
var t Target
if err := json.Unmarshal(data, &t); err != nil {
return false, errors.New("learning ledger target is unreadable")
}
targets = append(targets, t)
return len(targets) <= batch, nil
})
})
if err != nil {
return AuditProgress{}, fmt.Errorf("%w: %w", ErrStateIO, err)
}
if !p.audit.init {
p.audit.init = true
p.owe(cur.After)
}
more := len(targets) > batch
if more {
targets = targets[:batch]
}
if cur.RotationStart.IsZero() {
cur.RotationStart = now
}
prog := AuditProgress{OldestUnchecked: cur.PrevStart, RotationDone: !more}
if len(targets) > 0 {
lost, err := p.check(ctx, targets, &prog)
if lost {
p.audit.rebuilding = true
p.owe(cur.After)
}
if err != nil {
return AuditProgress{}, err
}
}
next := cur
if more {
next.After = targets[len(targets)-1].Digest
} else {
next = auditCursor{RotationStart: now, PrevStart: cur.RotationStart}
}
if err := p.ledger.Update(func(tx LedgerTx) error {
return tx.put(store.LearnCursorBucket, []byte(auditCursorKey), next)
}); err != nil {
return AuditProgress{}, fmt.Errorf("%w: %w", ErrStateIO, err)
}
if p.audit.coverPending {
switch {
case !more && (p.audit.coverFrom == "" || p.audit.wrapped):
p.audit.coverPending = false
case !more:
p.audit.wrapped = true
case p.audit.wrapped && next.After >= p.audit.coverFrom:
p.audit.coverPending = false
}
if !p.audit.coverPending {
p.audit.coverReached = true
if err := p.settle(); err != nil {
return prog, err
}
}
}
return prog, nil
}
// check inspects targets exactly and marks the unclean ones for repair.
// lost reports a live anchor missing from storage.
func (p *Publisher) check(ctx context.Context, targets []Target, prog *AuditProgress) (lost bool, err error) {
if err = ctx.Err(); err != nil {
return false, err
}
ds := make([]Digest, len(targets))
for i, t := range targets {
ds[i] = t.Digest
}
insp, err := p.client.InspectDigests(ctx, ds)
if err != nil {
return false, fmt.Errorf("%w: %w", ErrReadFailed, err)
}
// Observe the entire batch before journaling repairs. A failed commit
// must not hide a missing anchor from the fresh-matching gate.
var drifted []Target
missing := map[Digest]bool{}
for _, t := range targets {
v, present, err := insp.ExactValue(t.Digest)
if err != nil {
return lost, fmt.Errorf("%w: %w", ErrReadFailed, err)
}
prog.Checked++
exact := t.T == 0 && !present || t.T > 0 && present && v == int64(t.T)
if t.Phase == PhaseIdle && exact {
continue
}
prog.Repaired++
if t.T > 0 && !present {
live, err := p.missingLiveAnchor(t)
if err != nil {
return lost, err
}
lost = lost || live
missing[t.Digest] = live
}
if t.Phase != PhaseIdle {
continue // already pending repair or publication
}
drifted = append(drifted, t)
}
for _, t := range drifted {
if err := p.setPhase(t, PhaseRead, nil); err != nil {
return lost, err
}
if missing[t.Digest] {
p.audit.reconstructed++
}
if p.audit.drift == nil {
p.audit.drift = map[Digest]int{}
}
p.audit.drift[t.Digest]++
}
return lost, nil
}
// A pending contribution can lose a previously published anchor too.
// Reset gaps and a seed's first add are expected absences; committed
// contributors distinguish a live anchor from an unpublished seed.
func (p *Publisher) missingLiveAnchor(t Target) (bool, error) {
if t.Phase == PhaseIdle {
return true, nil
}
if t.ResetPending || t.RestoreT != 0 || t.Phase != PhaseRead && t.Phase != PhaseAddSent {
return false, nil
}
var c Cluster
var ok bool
if err := p.ledger.View(func(v LedgerView) error {
var err error
c, ok, err = v.Cluster(t.ClusterID)
return err
}); err != nil {
return false, fmt.Errorf("%w: %w", ErrStateIO, err)
}
if !ok || c.Anchor != t.Digest {
return false, fmt.Errorf("%w: target has no cluster for its anchor", ErrStateMismatch)
}
return len(c.UIDs) > 0 || len(c.Marks) > 0, nil
}
// settle allows fresh matching once the owed rotation is over and no target
// is still pending.
func (p *Publisher) settle() error {
pending, err := p.pending()
if err != nil {
return err
}
if p.audit.coverReached && len(pending) == 0 {
p.audit.covered, p.audit.rebuilding = true, false
}
return nil
}
// pending lists targets not idle, in key order.
// ponytail: full scan per call; index pending digests if a benchmark
// shows the scan in the turn budget.
func (p *Publisher) pending() ([]Digest, error) {
var out []Digest
err := p.ledger.View(func(v LedgerView) error {
return v.tx.ForEach(store.LearnTargetsBucket, func(_, data []byte) error {
var t Target
if err := json.Unmarshal(data, &t); err != nil {
return errors.New("learning ledger target is unreadable")
}
if t.Phase != PhaseIdle {
out = append(out, t.Digest)
}
return nil
})
})
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrStateIO, err)
}
return out, nil
}
// RepairTurn runs one publication turn for up to max pending targets,
// continuing from a persisted cursor. It returns how many it turned.
func (p *Publisher) RepairTurn(ctx context.Context, max int) (int, error) {
if max <= 0 || max > MaxExplicitDigests {
max = AuditBatch
}
// Retention supersedes pending publications before any body can be
// used to reconstruct an expired anchor.
if _, err := p.retireDue(p.now()); err != nil {
return 0, err
}
pending, err := p.pending()
if err != nil {
return 0, err
}
if len(pending) == 0 {
if err := p.cleanup(ctx); err != nil {
return 0, err
}
return 0, p.settle()
}
var cur auditCursor
if err := p.ledger.View(func(v LedgerView) error {
_, err := v.get(store.LearnCursorBucket, []byte(repairCursorKey), &cur)
return err
}); err != nil {
return 0, fmt.Errorf("%w: %w", ErrStateIO, err)
}
start, _ := slices.BinarySearch(pending, cur.After+"\x00")
order := slices.Concat(pending[start:], pending[:start])
if len(order) > max {
order = order[:max]
}
var errs []error
for _, d := range order {
if _, err := p.Turn(ctx, d); err != nil {
errs = append(errs, err)
}
}
cur.After = order[len(order)-1]
if err := p.ledger.Update(func(tx LedgerTx) error {
return tx.put(store.LearnCursorBucket, []byte(repairCursorKey), cur)
}); err != nil {
errs = append(errs, fmt.Errorf("%w: %w", ErrStateIO, err))
}
if err := p.cleanup(ctx); err != nil {
errs = append(errs, err)
}
if len(errs) == 0 {
errs = append(errs, p.settle())
}
return len(order), errors.Join(errs...)
}
// RetireDue supersedes every cluster whose latest committed or reserved
// admission is Retention old with a zero target, removes its aliases, then
// unlinks its body and credits its charges. Storage TTLs play no part. A
// wall clock behind the persisted watermark retires nothing new.
func (p *Publisher) RetireDue(ctx context.Context, now time.Time) ([]uint64, error) {
retired, err := p.retireDue(now)
if err != nil {
return nil, err
}
return retired, p.cleanup(ctx)
}
// Keep the durable retirement separate from filesystem cleanup so a body
// unlink failure cannot prevent publication of the zero target.
func (p *Publisher) retireDue(now time.Time) ([]uint64, error) {
now = now.UTC()
var retired []uint64
err := p.ledger.Update(func(tx LedgerTx) error {
var mark time.Time
if _, err := tx.get(store.LearnMetaBucket, []byte(watermarkKey), &mark); err != nil {
return err
}
if now.Before(mark) {
return nil
}
if err := tx.put(store.LearnMetaBucket, []byte(watermarkKey), now); err != nil {
return err
}
latest, err := reservedAdmissions(tx)
if err != nil {
return err
}
var due []Cluster
if err := tx.tx.ForEach(store.LearnClustersBucket, func(_, data []byte) error {
var c Cluster
if err := json.Unmarshal(data, &c); err != nil {
return errors.New("learning ledger cluster is unreadable")
}
last := c.LastReport
if r := latest[c.ID]; r.After(last) {
last = r
}
if !c.Retired && !now.Before(last.Add(Retention)) {
due = append(due, c)
}
return nil
}); err != nil {
return err
}
for _, c := range due {
if err := supersede(tx, c, now); err != nil {
return err
}
retired = append(retired, c.ID)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrStateIO, err)
}
return retired, nil
}
// reservedAdmissions maps each cluster to its latest open reservation.
func reservedAdmissions(tx LedgerTx) (map[uint64]time.Time, error) {
latest := map[uint64]time.Time{}
err := tx.tx.ForEach(store.LearnIntentsBucket, func(_, data []byte) error {
var i Intent
if err := json.Unmarshal(data, &i); err != nil {
return errors.New("learning ledger intent is unreadable")
}
if i.Done || i.Kind != IntentReport && i.Kind != IntentOperator {
return nil
}
for _, ic := range i.Clusters {
if !ic.Done && i.Admitted.After(latest[ic.ClusterID]) {
latest[ic.ClusterID] = i.Admitted
}
}
return nil
})
return latest, err
}
// supersede durably gives a cluster's anchor a zero target under a new
// revision, drops its aliases and marks the record as a cleanup reference.
// Old intents no longer match the revision and cannot commit.
func supersede(tx LedgerTx, c Cluster, now time.Time) error {
t, ok, err := tx.Target(c.Anchor)
if err != nil {
return err
}
if !ok {
t = Target{Digest: c.Anchor, ClusterID: c.ID}
}
if len(c.Charges) == 0 {
if err := retainSeedCharge(tx, &c); err != nil {
return err
}
}
c.Revision = max(c.Revision, t.Revision) + 1
t.T, t.Revision, t.Phase, t.ResetPending, t.RestoreT, t.Updated = 0, c.Revision, PhaseRead, false, 0, now
// The anchor's permanent guard still consumes the creator's allowance.
// Transfer one alias charge to it; cleanup refunds only the remainder.
for i := range c.Charges {
ch := &c.Charges[i]
if ch.Usage.Aliases > 0 {
t.GuardCharge = &Charge{UID: ch.UID, Operator: ch.Operator, Usage: Usage{Aliases: 1}}
ch.Usage.Aliases--
break
}
}
if err := tx.PutTarget(t); err != nil {
return err
}
for _, a := range c.Aliases {
if id, ok, err := tx.Alias(a); err != nil {
return err
} else if ok && id == c.ID {
if err := tx.DeleteAlias(a); err != nil {
return err
}
}
}
c.Retired = true
return tx.PutCluster(c)
}
// An unpublished seed owns a body and an alias through its reservation.
// Move that charge to retirement so cleanup refunds the body only after
// unlinking it, and the permanent guard keeps consuming its allowance.
func retainSeedCharge(tx LedgerTx, c *Cluster) error {
var seed Intent
err := tx.tx.ForEach(store.LearnIntentsBucket, func(_, data []byte) error {
var i Intent
if err := json.Unmarshal(data, &i); err != nil {
return errors.New("learning ledger intent is unreadable")
}
if i.Seed != nil && !i.Done && len(i.Clusters) > 0 && !i.Clusters[0].Done && i.Clusters[0].ClusterID == c.ID {
seed = i
return errStop
}
return nil
})
if err != nil && !errors.Is(err, errStop) {
return err
}
if seed.Seed == nil {
return nil
}
c.Charges = append(c.Charges, Charge{UID: seed.UID, Operator: seed.Kind == IntentOperator,
Usage: Usage{BodyBytes: seed.Seed.BodyLen, Aliases: len(seed.Clusters[0].Aliases)}})
// The superseded intent must not refund the transferred charge again.
seed.Seed, seed.Clusters[0].Aliases = nil, nil
return tx.PutIntent(seed)
}
// cleanup finishes retired clusters: unlink and sync the body, then credit
// charges and drop the record. A crash in between repeats harmlessly.
func (p *Publisher) cleanup(ctx context.Context) error {
var refs []Cluster
if err := p.ledger.View(func(v LedgerView) error {
return v.tx.ForEach(store.LearnClustersBucket, func(_, data []byte) error {
var c Cluster
if err := json.Unmarshal(data, &c); err != nil {
return errors.New("learning ledger cluster is unreadable")
}
if c.Retired {
refs = append(refs, c)
}
return nil
})
}); err != nil {
return fmt.Errorf("%w: %w", ErrStateIO, err)
}
for _, c := range refs {
if err := ctx.Err(); err != nil {
return err
}
if err := p.bodies.Remove(c.ID); err != nil {
return fmt.Errorf("%w: %w", ErrStateIO, err)
}
if err := p.ledger.Update(func(tx LedgerTx) error {
for _, ch := range c.Charges {
if err := tx.credit(ch); err != nil {
return err
}
}
return tx.DeleteCluster(c.ID)
}); err != nil {
return fmt.Errorf("%w: %w", ErrStateIO, err)
}
}
return nil
}
// credit returns a charge, never below zero.
func (tx LedgerTx) credit(ch Charge) error {
u, err := tx.Usage(ch.UID, ch.Operator)
if err != nil {
return err
}
u.BodyBytes = max(0, u.BodyBytes-ch.Usage.BodyBytes)
u.Aliases = max(0, u.Aliases-ch.Usage.Aliases)
return tx.put(store.LearnUsageBucket, usageKey(ch.UID, ch.Operator), u)
}
package learn
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"syscall"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/safepath"
)
// DefaultBodiesDir holds one anchor body per cluster.
const DefaultBodiesDir = antispam.DefaultLearningDir + "/anchors"
const stagePrefix = ".stage-"
// ErrBodyInvalid: an anchor body is missing, unsafe or no longer the bytes
// its cluster recorded. Publication of that cluster stops.
var ErrBodyInvalid = errors.New("anchor body is missing or changed")
// Bodies stores immutable anchor bodies: stored mail, root-only, never
// logged. Errors never carry content or paths.
type Bodies struct {
Dir string
// owner is the uid that must own the directory and files: root, except
// in tests.
owner int
// writeFile writes and syncs a staged body; tests inject failures.
writeFile func(*os.File, []byte) error
}
// Staged is a body written under a staging name, not yet committed.
type Staged struct {
Name string
Len int64
SHA string
}
// BodyName is the committed file name of a cluster's anchor body.
func BodyName(clusterID uint64) string {
return strconv.FormatUint(clusterID, 10) + ".msg"
}
// Keep the checked directory open so cleanup, rename and sync cannot be
// redirected by a replacement of its path.
func (b *Bodies) dir() (*safepath.Dir, error) {
if err := os.MkdirAll(b.Dir, 0o700); err != nil {
return nil, errors.New("cannot create the anchor body directory")
}
// A trailing slash makes Lstat follow a directory symlink.
path := filepath.Clean(b.Dir)
info, err := os.Lstat(path)
if err != nil || !info.IsDir() || info.Mode().Perm() != 0o700 || ownerOf(info) != b.owner {
return nil, errors.New("anchor body directory is not private to CSM")
}
dir, err := safepath.OpenDir(path)
if err != nil {
return nil, errors.New("cannot open the anchor body directory")
}
current, err := dir.Info()
if err != nil || !os.SameFile(info, current) || current.Mode().Perm() != 0o700 || ownerOf(current) != b.owner {
_ = dir.Close()
return nil, errors.New("anchor body directory changed while opening")
}
return dir, nil
}
func ownerOf(info os.FileInfo) int {
if st, ok := info.Sys().(*syscall.Stat_t); ok {
return int(st.Uid)
}
return -1
}
// Stage writes body under a fresh exclusive staging name, 0600, and syncs
// it. On any failure no file is left and no Staged is returned.
func (b *Bodies) Stage(body []byte) (Staged, error) {
if len(body) > MaxBody {
return Staged{}, errors.New("anchor body is larger than the learning limit")
}
dir, err := b.dir()
if err != nil {
return Staged{}, err
}
defer func() { _ = dir.Close() }()
name := stagePrefix + rand.Text()
f, err := dir.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return Staged{}, fmt.Errorf("cannot stage an anchor body: %w", unwrapErrno(err))
}
write := b.writeFile
if write == nil {
write = writeSync
}
// The process umask may remove owner permissions at creation.
err = f.Chmod(0o600)
if err == nil {
err = write(f, body)
}
if closeErr := f.Close(); err == nil {
err = closeErr
}
if err != nil {
stageErr := fmt.Errorf("cannot stage an anchor body: %w", unwrapErrno(err))
if removeErr := dir.Remove(name); removeErr != nil {
return Staged{}, errors.Join(stageErr, fmt.Errorf("cannot remove a staged anchor body: %w", unwrapErrno(removeErr)))
}
return Staged{}, errors.Join(stageErr, syncDir(dir))
}
sum := sha256.Sum256(body)
return Staged{Name: name, Len: int64(len(body)), SHA: hex.EncodeToString(sum[:])}, nil
}
func writeSync(f *os.File, body []byte) error {
if _, err := f.Write(body); err != nil {
return err
}
return f.Sync()
}
// unwrapErrno keeps the errno (ENOSPC, EIO) and drops paths.
func unwrapErrno(err error) error {
var errno syscall.Errno
if errors.As(err, &errno) {
return errno
}
return errors.New("file system error")
}
// Commit gives a staged body its cluster's name and syncs the directory.
// It never replaces an existing body.
func (b *Bodies) Commit(s Staged, clusterID uint64) error {
if !isStagedName(s.Name) {
return errors.New("not a staged anchor body")
}
dir, err := b.dir()
if err != nil {
return err
}
defer func() { _ = dir.Close() }()
if err := dir.RenameTo(s.Name, dir, BodyName(clusterID)); err != nil {
return fmt.Errorf("cannot commit an anchor body: %w", unwrapErrno(err))
}
return syncDir(dir)
}
// Read returns a cluster's body after checking it is a private regular
// file of exactly the recorded length and SHA-256.
func (b *Bodies) Read(clusterID uint64, wantLen int64, wantSHA string) ([]byte, error) {
dir, err := b.dir()
if err != nil {
return nil, ErrBodyInvalid
}
defer func() { _ = dir.Close() }()
if wantLen < 0 || wantLen > MaxBody {
return nil, ErrBodyInvalid
}
f, err := dir.OpenFile(BodyName(clusterID), os.O_RDONLY, 0)
if err != nil {
return nil, ErrBodyInvalid
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 || ownerOf(info) != b.owner || info.Size() != wantLen {
return nil, ErrBodyInvalid
}
body, err := io.ReadAll(io.LimitReader(f, wantLen+1))
sum := sha256.Sum256(body)
if err != nil || int64(len(body)) != wantLen || hex.EncodeToString(sum[:]) != wantSHA {
return nil, ErrBodyInvalid
}
return body, nil
}
// Remove unlinks a cluster's body and syncs the directory. A missing body
// is already removed.
func (b *Bodies) Remove(clusterID uint64) error {
dir, err := b.dir()
if err != nil {
return err
}
defer func() { _ = dir.Close() }()
if err := dir.Remove(BodyName(clusterID)); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("cannot remove an anchor body: %w", unwrapErrno(err))
}
return syncDir(dir)
}
// Orphans lists staged and committed bodies that referenced does not claim.
// Callers remove them only after ledger recovery.
func (b *Bodies) Orphans(referenced func(name string) bool) ([]string, error) {
dir, err := b.dir()
if err != nil {
return nil, err
}
defer func() { _ = dir.Close() }()
entries, err := dir.ReadDir()
if err != nil {
return nil, errors.New("cannot list anchor bodies")
}
var out []string
for _, e := range entries {
name := e.Name()
if !e.IsDir() && isBodyName(name) && !referenced(name) {
out = append(out, name)
}
}
slices.Sort(out)
return out, nil
}
// RemoveOrphan unlinks one name Orphans returned.
func (b *Bodies) RemoveOrphan(name string) error {
if !isBodyName(name) {
return errors.New("not an anchor body name")
}
dir, err := b.dir()
if err != nil {
return err
}
defer func() { _ = dir.Close() }()
if err := dir.Remove(name); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("cannot remove an anchor body: %w", unwrapErrno(err))
}
return syncDir(dir)
}
func isStagedName(name string) bool {
return strings.HasPrefix(name, stagePrefix) && len(name) > len(stagePrefix) &&
filepath.Base(name) == name && !strings.ContainsRune(name, '\x00')
}
func isBodyName(name string) bool {
if isStagedName(name) {
return true
}
id, ok := strings.CutSuffix(name, ".msg")
n, err := strconv.ParseUint(id, 10, 64)
return ok && err == nil && BodyName(n) == name
}
func syncDir(dir *safepath.Dir) error {
if err := dir.Sync(); err != nil {
return fmt.Errorf("cannot sync the anchor directory: %w", unwrapErrno(err))
}
return nil
}
package learn
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"time"
"github.com/pidginhost/csm/internal/store"
)
// CancelUnsent cancels uid's Spam folder reservations that sent nothing
// under their revision (an observed opt-out cancels unsent work).
// Their targets return to what they replaced. Sent clusters keep
// publishing and committed ones keep their votes. Only the ledger changes,
// so it runs whatever the gate. When supplied, confirm rechecks eligibility
// on the owner immediately before cancellation; false or an error keeps work.
func (e *Engine) CancelUnsent(ctx context.Context, uid int, confirm func() (bool, error)) error {
// Accounts with no cancellable work need no owner turn. In particular,
// sent work must not hold up each opt-out pass on native request delays.
ids, err := e.unsentReportIntents(ctx, uid)
if err != nil || len(ids) == 0 {
return err
}
var cancelErr error
if err := e.do(ctx, func() {
if confirm != nil {
var cancel bool
cancel, cancelErr = confirm()
if cancelErr != nil || !cancel {
return
}
}
if cancelErr = ctx.Err(); cancelErr == nil {
cancelErr = e.cancelUnsent(ids)
}
}); err != nil {
return err
}
return cancelErr
}
func (e *Engine) unsentReportIntents(ctx context.Context, uid int) ([]string, error) {
candidates, lookupErr := e.cfg.Ledger.reportIntents(ctx, uid)
if lookupErr != nil {
return nil, fmt.Errorf("%w: %w", ErrStateIO, lookupErr)
}
if len(candidates) == 0 {
return nil, nil
}
var ids []string
err := e.cfg.Ledger.View(func(v LedgerView) error {
for _, id := range candidates {
if err := ctx.Err(); err != nil {
return err
}
i, ok, err := v.Intent(id)
if err != nil {
return err
}
if ok {
for _, ic := range i.Clusters {
if ic.Done {
continue
}
t, ok, err := v.Target(ic.Anchor)
if err != nil {
return err
}
if ok && unsentTarget(t, ic) {
ids = append(ids, id)
break
}
}
}
}
return nil
})
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrStateIO, err)
}
return ids, nil
}
// ReportUIDs lists the accounts holding Spam folder reservations, sent or
// not, so the watcher can cancel those of removed accounts.
func (e *Engine) ReportUIDs(ctx context.Context) ([]int, error) {
e.cfg.Ledger.intentsMu.Lock()
defer e.cfg.Ledger.intentsMu.Unlock()
ids, err := e.cfg.Ledger.reportIndex(ctx)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrStateIO, err)
}
return slices.Sorted(maps.Keys(ids)), nil
}
func (l *Ledger) reportIntents(ctx context.Context, uid int) ([]string, error) {
l.intentsMu.Lock()
defer l.intentsMu.Unlock()
ids, err := l.reportIndex(ctx)
return slices.Clone(ids[uid]), err
}
// reportIndex maps UIDs to their report intents; the caller holds
// intentsMu. It indexes only ownership. Targets are read again on every
// check and on the owner so a cached entry cannot authorize cancellation.
func (l *Ledger) reportIndex(ctx context.Context) (map[int][]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if l.reportIDs == nil {
ids := map[int][]string{}
if err := l.View(func(v LedgerView) error {
return v.tx.ForEach(store.LearnIntentsBucket, func(k, data []byte) error {
if err := ctx.Err(); err != nil {
return err
}
var owner struct {
Kind IntentKind `json:"kind"`
UID int `json:"uid"`
}
if err := json.Unmarshal(data, &owner); err != nil {
return errors.New("learning ledger intent is unreadable")
}
if owner.Kind == IntentReport {
ids[owner.UID] = append(ids[owner.UID], string(k))
}
return nil
})
}); err != nil {
return nil, err
}
l.reportIDs = ids
}
return l.reportIDs, nil
}
func unsentTarget(t Target, ic IntentCluster) bool {
return t.ClusterID == ic.ClusterID && t.Revision == ic.Revision && t.SentRevision != t.Revision
}
// cancelUnsent runs on the owner, so no turn can send between its check
// and its restore.
func (e *Engine) cancelUnsent(ids []string) error {
e.start()
if e.bindErr != nil {
return e.bindErr
}
now := e.cfg.Now().UTC()
err := e.cfg.Ledger.Update(func(tx LedgerTx) error {
for _, id := range ids {
if err := cancelIntent(tx, id, now); err != nil {
return err
}
}
return nil
})
if err != nil {
err = fmt.Errorf("%w: %w", ErrStateIO, err)
e.note(err)
}
return err
}
func cancelIntent(tx LedgerTx, id string, now time.Time) error {
intent, ok, err := tx.Intent(id)
if err != nil || !ok {
return err
}
for k := range intent.Clusters {
if intent.Clusters[k].Done {
continue
}
cancelled, err := cancelCluster(tx, intent, k, now)
if err != nil {
return err
}
intent.Clusters[k].Done = cancelled
}
if !slices.ContainsFunc(intent.Clusters, func(ic IntentCluster) bool { return !ic.Done }) {
return tx.DeleteIntent(id)
}
return tx.PutIntent(intent)
}
// cancelCluster restores the target that cluster k of intent replaced and
// returns its charge, unless a native write was sent under its revision.
func cancelCluster(tx LedgerTx, intent Intent, k int, now time.Time) (bool, error) {
ic := intent.Clusters[k]
t, ok, err := tx.Target(ic.Anchor)
if err != nil {
return false, err
}
// A superseded target ends the contribution when it is published.
if !ok || !unsentTarget(t, ic) {
return false, nil
}
seed := intent.Seed != nil && k == 0
switch {
case ic.Prior != nil:
p := *ic.Prior
// A new revision keeps the restored target apart from this intent;
// a read verifies it before the next fresh match.
p.Revision, p.Phase, p.Updated = t.Revision+1, PhaseRead, now
p.ResetPending, p.RestoreT = t.ResetPending, 0
if g := p.GuardCharge; seed && g != nil {
// The seed returned the guard's charge; it is a guard again.
if err = tx.AddUsage(g.UID, g.Operator, g.Usage); err != nil {
return false, err
}
}
err = tx.PutTarget(p)
case seed:
// Nothing was ever sent for a target the seed created.
err = tx.DeleteTarget(ic.Anchor)
default:
return false, nil
}
if err != nil {
return false, err
}
ch := Charge{UID: intent.UID, Usage: Usage{Aliases: len(ic.Aliases)}}
if seed {
ch.Usage.BodyBytes = intent.Seed.BodyLen
// Cleanup unlinks the unpublished body and drops the record.
c, found, err := tx.Cluster(ic.ClusterID)
if err != nil {
return false, err
}
if found {
c.Retired = true
if err = tx.PutCluster(c); err != nil {
return false, err
}
}
}
return true, tx.credit(ch)
}
package learn
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/antispam/hostfacts"
)
// Limits of the native calls. RequestTimeout is provisional pending
// benchmark measurements.
const (
RequestTimeout = 5 * time.Second
// SyncTimeout bounds rspamadm control fuzzy_sync.
SyncTimeout = 10 * time.Second
// MaxBody is the snapshot limit, headers and attachments included.
MaxBody = 10 << 20
// MaxExplicitDigests matches the inspection plugin's own cap.
MaxExplicitDigests = 64
maxInspectReply = 256 << 10
maxMutationReply = 64 << 10
)
// InspectPath is the CSM inspection plugin's controller path.
const InspectPath = "/plugins/csm_learning/inspect"
// SyntheticMessage is the task body of an exact-only inspection; explicit
// digest checks ignore the body's own hashes, so no retained mail is sent.
const SyntheticMessage = "From: csm@localhost\r\nSubject: csm learning audit\r\n\r\naudit\r\n"
// Client errors. A rejected reply is a definite answer from the controller,
// yet it does not prove storage left the value unchanged (an IO timeout
// reply can follow an applied write), so callers reconcile by reading.
var (
ErrUncertain = errors.New("request sent; outcome unknown")
ErrRejected = errors.New("controller refused the request")
ErrPeer = errors.New("controller socket is not served by rspamd")
)
// Client speaks the native learning calls over rspamd's private
// controller socket. Mutations are never repeated automatically.
type Client struct {
Controller string
RspamdUID int
// Run runs rspamadm; nil means hostfacts.RunCommand.
Run func(ctx context.Context, limit time.Duration, name string, args ...string) antispam.CommandResult
}
// Inspect inspects a message: one result per distinct text digest, hits
// carrying the stored anchor they matched.
func (c Client) Inspect(ctx context.Context, body []byte) (Inspection, error) {
if len(body) > MaxBody {
return Inspection{}, errors.New("message is larger than the learning limit")
}
reply, err := c.inspect(ctx, InspectPath, body)
if errors.Is(err, ErrRejected) {
return Inspection{}, fmt.Errorf("%w: %w", ErrInspectionFailed, err)
}
if err != nil {
return Inspection{}, err
}
insp, err := ParseInspection(reply, nil)
if err != nil && !errors.Is(err, ErrMessageRejected) {
err = fmt.Errorf("%w: %w", ErrInspectionFailed, err)
}
return insp, err
}
// InspectDigests runs an exact-only check of full digests.
func (c Client) InspectDigests(ctx context.Context, ds []Digest) (Inspection, error) {
if len(ds) == 0 || len(ds) > MaxExplicitDigests {
return Inspection{}, errors.New("exact inspection needs 1 to 64 digests")
}
list := make([]string, len(ds))
for i, d := range ds {
if _, err := ParseDigest(string(d)); err != nil {
return Inspection{}, err
}
list[i] = string(d)
}
reply, err := c.inspect(ctx, InspectPath+"?digests="+strings.Join(list, ","), []byte(SyntheticMessage))
if err != nil {
return Inspection{}, err
}
return ParseInspection(reply, ds)
}
func (c Client) inspect(ctx context.Context, path string, body []byte) ([]byte, error) {
status, reply, err := c.do(ctx, path, nil, body, maxInspectReply)
if err != nil {
return nil, err
}
if status != http.StatusOK {
return nil, fmt.Errorf("%w: inspection returned HTTP %d", ErrRejected, status)
}
return reply, nil
}
// Add sends one /fuzzyadd of the anchor body with weight w.
func (c Client) Add(ctx context.Context, body []byte, w int32) error {
if w <= 0 {
return errors.New("weight must be positive")
}
if len(body) > MaxBody {
return errors.New("message is larger than the learning limit")
}
return c.mutate(ctx, "/fuzzyadd", map[string]string{"Flag": "1", "Weight": strconv.Itoa(int(w))}, body)
}
// DeleteHash sends one /fuzzydelhash of a full digest. Success alone proves
// nothing; a later exact check must observe absence.
func (c Client) DeleteHash(ctx context.Context, d Digest) error {
if _, err := ParseDigest(string(d)); err != nil {
return err
}
return c.mutate(ctx, "/fuzzydelhash", map[string]string{"Flag": "1", "Hash": string(d)}, nil)
}
func (c Client) mutate(ctx context.Context, path string, hdr map[string]string, body []byte) error {
status, reply, err := c.do(ctx, path, hdr, body, maxMutationReply)
if err != nil {
return err
}
if status != http.StatusOK {
return fmt.Errorf("%w: mutation returned HTTP %d", ErrRejected, status)
}
fields, err := objectFields(reply, "success", "hashes", "error")
if err != nil {
return ErrUncertain
}
if fields["error"] != nil {
var message string
if fields["success"] == nil && json.Unmarshal(fields["error"], &message) == nil && message != "" {
return ErrRejected
}
return ErrUncertain
}
var success bool
if json.Unmarshal(fields["success"], &success) != nil {
return ErrUncertain
}
if !success {
return ErrRejected
}
if raw := fields["hashes"]; raw != nil {
var hashes []json.RawMessage
if uclList(raw, &hashes) != nil {
return ErrUncertain
}
for _, rawHash := range hashes {
var hash string
if json.Unmarshal(rawHash, &hash) != nil {
return ErrUncertain
}
if _, err := ParseDigest(hash); err != nil {
return ErrUncertain
}
}
}
return nil
}
func (c Client) do(ctx context.Context, path string, hdr map[string]string, body []byte, limit int64) (int, []byte, error) {
ctx, cancel := context.WithTimeout(ctx, RequestTimeout)
defer cancel()
status, reply, err := hostfacts.ControllerDo(ctx, c.Controller, c.RspamdUID, http.MethodPost, path, hdr, body, limit)
if err != nil {
// The sanitized transport error keeps send status but drops the
// cause. Cancellation must not spend a message's retry allowance.
err = errors.Join(err, ctx.Err())
}
switch {
case errors.Is(err, hostfacts.ErrControllerPeer):
return 0, nil, ErrPeer
case errors.Is(err, hostfacts.ErrControllerNoReply):
return 0, nil, fmt.Errorf("%w: %w", ErrUncertain, err)
case err != nil:
return 0, nil, err
}
return status, bytes.Clone(reply), nil
}
// Sync asks the fuzzy worker to flush queued updates. It does not wait for
// Redis; callers verify with exact checks.
func (c Client) Sync(ctx context.Context) error {
run := c.Run
if run == nil {
run = hostfacts.RunCommand
}
if res := run(ctx, SyncTimeout, "rspamadm", "control", "fuzzy_sync"); res.Err != nil {
return errors.New("rspamadm control fuzzy_sync failed")
}
return nil
}
// Package learn turns Spam folder reports and operator requests into anchor
// values in rspamd's local fuzzy storage: a strict controller
// client, the ledger, anchor bodies, assignment, publication and audits.
package learn
import (
"errors"
"strings"
)
// Digest is a full fuzzy digest: 128 lowercase hex digits.
type Digest string
// ParseDigest accepts only a full digest; a prefix never names a hash.
func ParseDigest(s string) (Digest, error) {
if len(s) != 128 || strings.ContainsFunc(s, func(r rune) bool { return (r < '0' || r > '9') && (r < 'a' || r > 'f') }) {
return "", errors.New("not a 128-digit lowercase hex digest")
}
return Digest(s), nil
}
package learn
import (
"cmp"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/store"
)
// Engine limits. BenchmarkPublishTurn took 2.5 to 3.4 ms of ledger work for
// a report joining a single-anchor ledger; native calls measured about 2 ms
// per report. Larger ledgers need separate measurements.
const (
EngineTick = 10 * time.Second
RepairBatch = 10
AuditOverdueAfter = 24 * time.Hour
RequestErrorPasses = 3
)
// Operator actions.
const (
ActionSpam = "spam"
ActionForget = "forget"
)
// Per-cluster and overall operator results. A skipped part reads
// "skipped:<reason>".
const (
ResultCommitted = "committed"
ResultDuplicate = "duplicate"
ResultPending = "pending"
// ResultDeferred: nothing was reserved or changed, so nothing finishes
// the request in the background; a new run is needed.
ResultDeferred = "deferred"
ResultForgotten = "forgotten"
ResultNoMatch = "no-match"
ResultPartial = "partial"
ResultSkipped = "skipped"
)
// Engine errors.
var (
ErrLearningOff = errors.New("learning is not active")
ErrRequestReused = errors.New("request ID was used with a different action or file")
ErrNoDigest = errors.New("the file has no usable text to learn from")
ErrUnknownAnchor = fmt.Errorf("%w: storage returned an anchor the ledger does not know", ErrStateMismatch)
errEngineStopped = errors.New("the learning engine has stopped")
errDeferSeed = errors.New("the digest's earlier cluster is still being removed")
errTargetNotReady = errors.New("a joined cluster has an unfinished publication")
)
// Candidate is a report: the reporting tenant's UID and a bounded message
// snapshot. Ref is opaque to the engine.
type Candidate struct {
UID int
Snapshot []byte
Ref string
}
// Outcome is what became of a candidate. Pending clusters were reserved and
// keep publishing in the background; Deferred means nothing was reserved
// and the candidate should come back later.
type Outcome struct {
Ref string
Committed []uint64
Duplicates []uint64
Pending []uint64
Skips []Skip
Deferred bool
Err error
}
// OperatorRequest is one file of csm antispam learn.
type OperatorRequest struct {
Action string
RequestID string
Snapshot []byte
SnapshotSHA string
}
// OperatorResult is the per-cluster outcome of one operator request.
type OperatorResult struct {
RequestID string
Clusters []ClusterOutcome
// Digests are the file's full text digests, for the audit record.
Digests []Digest
Outcome string
ExitOK bool
}
// EngineConfig is what the engine needs from the daemon.
type EngineConfig struct {
Client nativeClient
Ledger *Ledger
Bodies *Bodies
HashGeneration string
Prefix string
// Now reads the wall clock; nil means time.Now.
Now func() time.Time
// TickEvery spaces background turns; zero means EngineTick.
TickEvery time.Duration
// OnReportCommit observes a durable Spam folder contribution, including
// one completed by a background retry. It must not block the owner.
OnReportCommit func()
// OnStateMismatch observes the moment learning freezes because its
// ledger and storage disagree, with the evidence. It must not block the
// owner.
OnStateMismatch func(detail string)
// OnUnstableAnchor observes a cluster retired because rspamd no longer
// hashes its stored message to its anchor. It must not block the owner.
OnUnstableAnchor func(detail string)
}
type request struct {
fn func()
done chan struct{}
}
// Engine is the one owner of the learning ledger and of every native
// mutation: one goroutine serializes assignment, reservation,
// publication, audits, retirement and operator requests.
type Engine struct {
cfg EngineConfig
pub *Publisher
requests chan request
done chan struct{}
mu sync.Mutex
gate antispam.LearningGate
gateEpoch uint64
// paused: the daemon holds native work for a reconcile pass. Unlike a
// closed gate it owes no audit rotation; the next SetGate lifts it.
paused bool
runCtx context.Context
cancel context.CancelFunc
published []antispam.Problem
// Audit progress for Stats.
statOldest, statAudit time.Time
statRepaired uint64
// Owned by the engine goroutine.
seenEpoch uint64
started bool
recovered bool
startedAt time.Time
bindErr error
frozen bool
stateErr error
bodyErr bool
requestFails int
oldest time.Time
capacity map[int]bool
}
// NewEngine returns an engine; Run starts it.
func NewEngine(cfg EngineConfig) *Engine {
if cfg.Now == nil {
cfg.Now = time.Now
}
if cfg.TickEvery <= 0 {
cfg.TickEvery = EngineTick
}
p := NewPublisher(cfg.Client, cfg.Ledger, cfg.Bodies)
p.now = cfg.Now
p.onUnstable = cfg.OnUnstableAnchor
return &Engine{cfg: cfg, pub: p, requests: make(chan request), done: make(chan struct{}), capacity: map[int]bool{}}
}
// Run serves requests and background turns until ctx ends.
func (e *Engine) Run(ctx context.Context) {
defer close(e.done)
e.mu.Lock()
e.runCtx = ctx
e.mu.Unlock()
e.start()
tick := time.NewTicker(e.cfg.TickEvery)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case r := <-e.requests:
r.fn()
close(r.done)
case <-tick.C:
if turn, end, ok := e.turnCtx(ctx); ok {
e.background(turn)
end()
}
e.refresh()
}
}
}
// SetGate applies the reconcile pass's gate and lifts a pause. Closing it
// cancels native calls and waits for their turn to finish within their own
// deadlines; reopening a closed gate owes a full audit rotation before
// fresh matching, as after a restart.
func (e *Engine) SetGate(g antispam.LearningGate) {
e.mu.Lock()
if e.gate.Install && e.gate.Publish && (!g.Install || !g.Publish) {
e.gateEpoch++
}
e.gate, e.paused = g, false
e.stopLocked(!g.Install || !g.Publish)
}
// Pause holds native work while the daemon reconciles rspamd, draining
// calls in flight like a closed gate. A pause loses no verified target, so
// it owes no audit rotation.
func (e *Engine) Pause() {
e.mu.Lock()
e.paused = true
e.stopLocked(true)
}
// stopLocked cancels native work when stop is set, unlocks e.mu, and waits
// for the owner to finish its current turn.
func (e *Engine) stopLocked(stop bool) {
if stop && e.cancel != nil {
e.cancel()
e.cancel = nil
}
running := e.runCtx != nil
e.mu.Unlock()
if stop && running {
// A barrier on the existing owner queue waits for the journal and
// problem snapshot as well as the canceled call. A stopped owner
// has already drained its work.
_ = e.do(context.Background(), func() {})
}
}
// Publishing reports whether the reconcile gate allows native work and
// no reconciliation pause is holding it.
func (e *Engine) Publishing() bool {
e.mu.Lock()
defer e.mu.Unlock()
return e.gate.Install && e.gate.Publish && !e.paused
}
// turnCtx starts native work if the gate allows it.
func (e *Engine) turnCtx(caller context.Context) (context.Context, func(), bool) {
e.mu.Lock()
defer e.mu.Unlock()
if !e.gate.Install || !e.gate.Publish || e.paused || caller.Err() != nil {
return nil, nil, false
}
parent := e.runCtx
if parent == nil {
parent = context.Background()
}
ctx, cancel := context.WithCancel(parent)
stop := context.AfterFunc(caller, cancel)
e.cancel = cancel
return ctx, func() { stop(); cancel() }, true
}
// Problems returns the learner's current problems, one per reason, in a
// fixed order.
func (e *Engine) Problems() []antispam.Problem {
e.mu.Lock()
defer e.mu.Unlock()
return slices.Clone(e.published)
}
func (e *Engine) do(ctx context.Context, fn func()) error {
if err := ctx.Err(); err != nil {
return err
}
r := request{fn: fn, done: make(chan struct{})}
select {
case e.requests <- r:
case <-ctx.Done():
return ctx.Err()
case <-e.done:
return errEngineStopped
}
// Once handed off, the owner still borrows the snapshot and result.
// Cancellation stops native work, but cannot release those objects early.
<-r.done
return ctx.Err()
}
// start binds the ledger without touching retained bodies while suspended.
func (e *Engine) start() {
if e.started && e.bindErr == nil {
return
}
if !e.started {
e.started, e.startedAt = true, e.cfg.Now()
}
e.bindErr = e.cfg.Ledger.Bind(e.cfg.HashGeneration, e.cfg.Prefix)
if e.bindErr != nil && !errors.Is(e.bindErr, ErrStateMismatch) {
e.bindErr = fmt.Errorf("%w: %w", ErrStateIO, e.bindErr)
}
e.refresh()
}
// prepare recovers and retires before admitting work, including between ticks.
func (e *Engine) prepare(ctx context.Context) error {
if err := e.prepareState(ctx); err != nil {
return err
}
if err := e.recover(ctx); err != nil {
return err
}
return e.pub.cleanup(ctx)
}
// Durable retirement must precede publication even when retained bodies
// cannot be cleaned up. Otherwise expired anchors keep scoring mail.
func (e *Engine) prepareState(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
e.mu.Lock()
epoch := e.gateEpoch
e.mu.Unlock()
if e.seenEpoch != epoch {
e.seenEpoch = epoch
e.pub.audit.init = false
e.pub.owe("")
}
e.start()
if e.bindErr != nil {
return e.bindErr
}
if e.frozen {
return ErrStateMismatch
}
if err := ctx.Err(); err != nil {
return err
}
_, err := e.pub.retireDue(e.cfg.Now())
return err
}
func (e *Engine) recover(ctx context.Context) error {
if !e.recovered {
if err := e.recoverBodies(ctx); err != nil {
return err
}
e.recovered = true
}
return nil
}
// note turns an error into the problem state it reveals.
func (e *Engine) note(err error) {
if errors.Is(err, ErrStateIO) {
e.stateErr = err
}
if errors.Is(err, ErrStateMismatch) {
e.freeze(err.Error())
}
if errors.Is(err, ErrBodyInvalid) {
e.bodyErr = true
}
if errors.Is(err, ErrReadFailed) && !errors.Is(err, context.Canceled) {
e.requestFails++
}
}
// freeze holds admissions and reports why, once per engine.
func (e *Engine) freeze(detail string) {
if e.frozen {
return
}
e.frozen = true
if e.cfg.OnStateMismatch != nil {
e.cfg.OnStateMismatch(detail)
}
}
// mismatchDetail names the digests the ledger could not resolve and the
// lookup that found them, by 16-digit prefix.
func mismatchDetail(unknown []Digest, insp Inspection) string {
var b strings.Builder
b.WriteString("lookup named digests the ledger cannot resolve:")
for _, d := range unknown {
b.WriteString(" " + shortDigest(d))
}
b.WriteString("; lookup:")
for _, r := range insp.Results {
fmt.Fprintf(&b, " queried=%s found=%t stored=%s prob=%.2f;", shortDigest(r.Queried), r.Found, shortDigest(r.Stored), r.Prob)
}
return b.String()
}
func shortDigest(d Digest) string {
if len(d) > 16 {
return string(d[:16])
}
return string(d)
}
// recoverBodies removes staged and committed bodies that no cluster or
// open intent references.
func (e *Engine) recoverBodies(ctx context.Context) error {
refs := map[string]bool{}
err := e.cfg.Ledger.View(func(v LedgerView) error {
if err := v.tx.ForEach(store.LearnClustersBucket, func(_, data []byte) error {
var c Cluster
if err := json.Unmarshal(data, &c); err != nil {
return errors.New("learning ledger cluster is unreadable")
}
refs[BodyName(c.ID)] = true
return nil
}); err != nil {
return err
}
return v.tx.ForEach(store.LearnIntentsBucket, func(_, data []byte) error {
var i Intent
if err := json.Unmarshal(data, &i); err != nil {
return errors.New("learning ledger intent is unreadable")
}
if i.Seed != nil {
refs[i.Seed.Staged] = true
}
return nil
})
})
if err != nil {
return fmt.Errorf("%w: %w", ErrStateIO, err)
}
if err = ctx.Err(); err != nil {
return err
}
orphans, err := e.cfg.Bodies.Orphans(func(name string) bool { return refs[name] })
if err != nil {
return fmt.Errorf("%w: %w", ErrStateIO, err)
}
for _, name := range orphans {
if err := ctx.Err(); err != nil {
return err
}
if err := e.cfg.Bodies.RemoveOrphan(name); err != nil {
return fmt.Errorf("%w: %w", ErrStateIO, err)
}
}
return nil
}
// background runs retirement before reconstruction, one audit turn, one
// repair turn and the open intents.
func (e *Engine) background(ctx context.Context) {
if err := e.prepareState(ctx); err != nil {
e.note(err)
return
}
// Cleanup failures hold admissions, but must not hold native deletion.
errs := []error{e.recover(ctx)}
now := e.cfg.Now()
prog, err := e.pub.AuditTurn(ctx, AuditBatch, now)
errs = append(errs, err)
e.mu.Lock()
e.statRepaired = e.pub.audit.reconstructed
if err == nil {
e.oldest = prog.OldestUnchecked
e.statOldest = prog.OldestUnchecked
if prog.RotationDone {
e.statAudit = now
}
}
e.mu.Unlock()
_, err = e.pub.RepairTurn(ctx, RepairBatch)
errs = append(errs, err, e.resume(ctx))
err = errors.Join(errs...)
e.note(err)
if err == nil {
e.bodyErr = false
e.stateErr = nil
e.requestFails = 0
}
}
// resume publishes and commits open intents.
func (e *Engine) resume(ctx context.Context) error {
var ids []string
if err := e.cfg.Ledger.View(func(v LedgerView) error {
return v.tx.ForEach(store.LearnIntentsBucket, func(k, _ []byte) error {
ids = append(ids, string(k))
return nil
})
}); err != nil {
return fmt.Errorf("%w: %w", ErrStateIO, err)
}
var errs []error
for _, id := range ids {
if _, _, err := e.publishIntent(ctx, id); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
// probeMessage is plain text long enough to hash to one digest. It is
// inspected, never learned.
const probeMessage = "From: csm@localhost\r\nSubject: csm learning probe\r\n\r\n" +
"This fixed message lets the security monitor confirm that rspamd can inspect a plain text message and query its fuzzy storage. " +
"It is never learned and never stored. The words below only make the text long enough to be hashed: " +
"alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa quebec romeo sierra tango uniform victor whiskey xray yankee zulu.\r\n"
// ProbeStorage inspects probeMessage. It returns nil only when storage
// answered the probe's one digest, which shows the controller, the plugin
// and fuzzy storage serve an ordinary body.
func (e *Engine) ProbeStorage(ctx context.Context) error {
var err error
if doErr := e.do(ctx, func() { err = e.probe(ctx) }); doErr != nil {
return doErr
}
return err
}
func (e *Engine) probe(caller context.Context) error {
ctx, end, ok := e.turnCtx(caller)
if !ok {
return errors.New("learning is not publishing")
}
defer end()
insp, err := e.cfg.Client.Inspect(ctx, []byte(probeMessage))
if err != nil {
return err
}
if insp.Distinct != 1 {
return fmt.Errorf("probe inspection queried %d digests, want 1", insp.Distinct)
}
return nil
}
// SubmitReport admits one report. A closed gate or frozen ledger defers it
// without reading the snapshot.
func (e *Engine) SubmitReport(ctx context.Context, c Candidate) (Outcome, error) {
out := Outcome{Ref: c.Ref, Deferred: true}
if !e.Publishing() {
return out, nil
}
if err := e.do(ctx, func() { out = e.report(ctx, c) }); err != nil {
return Outcome{Ref: c.Ref, Deferred: true}, err
}
return out, nil
}
func (e *Engine) report(caller context.Context, c Candidate) Outcome {
out := Outcome{Ref: c.Ref, Deferred: true}
defer e.refresh()
ctx, end, ok := e.turnCtx(caller)
if !ok {
return out
}
defer end()
if err := e.prepare(ctx); err != nil {
e.note(err)
out.Err = err
return out
}
res, err := e.admit(ctx, Source{UID: c.UID, BodyLen: int64(len(c.Snapshot))}, c.Snapshot, "")
e.note(err)
if err == nil {
e.requestFails = 0
}
return Outcome{Ref: c.Ref, Committed: res.committed, Duplicates: res.duplicates, Pending: res.pending, Skips: res.skips, Deferred: res.deferred, Err: err}
}
type admitResult struct {
committed, duplicates, pending []uint64
skips []Skip
digests []Digest
deferred bool
}
func digestsOf(insp Inspection) []Digest {
ds := make([]Digest, 0, len(insp.Results))
for _, r := range insp.Results {
ds = append(ds, r.Queried)
}
return ds
}
// admit inspects, assigns, reserves and publishes one snapshot.
func (e *Engine) admit(ctx context.Context, src Source, snapshot []byte, intentID string) (admitResult, error) {
if len(snapshot) > MaxBody {
return admitResult{skips: []Skip{{Reason: SkipNoDigest}}}, errors.New("message is larger than the learning limit")
}
if snapshot = textOnlyMessage(snapshot); snapshot == nil {
return admitResult{skips: []Skip{{Reason: SkipNoDigest}}}, nil
}
src.BodyLen = int64(len(snapshot))
if err := ctx.Err(); err != nil {
return admitResult{deferred: true}, err
}
insp, err := e.cfg.Client.Inspect(ctx, snapshot)
if err != nil {
return admitResult{deferred: true}, fmt.Errorf("%w: %w", ErrReadFailed, err)
}
digests := digestsOf(insp)
if err = ctx.Err(); err != nil {
return admitResult{deferred: true, digests: digests}, err
}
if src.Operator {
if recordErr := e.recordOperatorDigests(intentID, digests); recordErr != nil {
return admitResult{deferred: true, digests: digests}, recordErr
}
}
gaps, err := e.gaps()
if err != nil {
return admitResult{deferred: true, digests: digests}, err
}
var a Admission
if viewErr := e.cfg.Ledger.View(func(v LedgerView) error {
var assignErr error
a, assignErr = Assign(insp, src, v, gaps)
return assignErr
}); viewErr != nil {
err = fmt.Errorf("%w: %w", ErrStateIO, viewErr)
return admitResult{deferred: true, digests: digests}, err
}
if allNewParts(a, insp) {
pa, part, partErr := e.seedFromPart(ctx, src, snapshot, insp, gaps)
if partErr != nil {
return admitResult{deferred: true, digests: digests}, partErr
}
if part != nil {
a, snapshot = pa, part
}
}
res := admitResult{skips: a.Skips, duplicates: a.Duplicates, digests: digests}
switch {
case len(a.Mismatch) > 0:
e.freeze(mismatchDetail(a.Mismatch, insp))
return admitResult{deferred: true, digests: digests}, nil
case len(a.DeleteZero) > 0:
return admitResult{deferred: true, digests: digests}, e.scheduleZero(a.DeleteZero)
case a.Defer:
return admitResult{deferred: true, digests: digests}, nil
}
key := src.UID
if src.Operator {
key = -1
}
full := slices.ContainsFunc(a.Skips, func(s Skip) bool {
return s.Reason == SkipAccountCapacity || s.Reason == SkipOperatorCapacity
})
if full {
e.capacity[key] = true
} else if len(a.Joins) > 0 || a.Seed != nil {
delete(e.capacity, key)
}
if len(a.Joins) == 0 && a.Seed == nil {
return res, nil
}
id, err := e.reserve(ctx, src, a, snapshot, intentID, digests)
if errors.Is(err, errDeferSeed) || errors.Is(err, errTargetNotReady) {
return admitResult{deferred: true, digests: digests}, nil
}
if err != nil {
return admitResult{deferred: true, digests: digests}, err
}
e.stateErr = nil
res.committed, res.pending, err = e.publishIntent(ctx, id)
return res, err
}
// allNewParts reports an admission that refused every digest of a report
// with several parts because none of them is learned yet.
func allNewParts(a Admission, insp Inspection) bool {
if insp.Distinct < 2 || len(a.Skips) != insp.Distinct {
return false
}
for _, s := range a.Skips {
if s.Reason != SkipMultipartSeed {
return false
}
}
return true
}
// seedFromPart retries a report whose parts are all new with one text part
// at a time, the largest first. A single-part message qualifies when its own
// inspection is exactly one still-new digest of the report, so publishing it
// adds only that digest. It returns the admission and the body to store, or
// a nil body when no part qualifies.
func (e *Engine) seedFromPart(ctx context.Context, src Source, snapshot []byte, insp Inspection, gaps map[uint64]bool) (Admission, []byte, error) {
report := map[Digest]bool{}
for _, r := range insp.Results {
report[r.Queried] = true
}
for part := range textPartMessages(snapshot, insp.Distinct) {
pi, err := e.cfg.Client.Inspect(ctx, part)
if errors.Is(err, ErrMessageRejected) {
continue
}
if err != nil {
return Admission{}, nil, fmt.Errorf("%w: %w", ErrReadFailed, err)
}
if pi.Distinct != 1 || pi.Results[0].Found || !report[pi.Results[0].Queried] {
continue
}
src.BodyLen = int64(len(part))
var a Admission
if err := e.cfg.Ledger.View(func(v LedgerView) error {
var assignErr error
a, assignErr = Assign(pi, src, v, gaps)
return assignErr
}); err != nil {
return Admission{}, nil, fmt.Errorf("%w: %w", ErrStateIO, err)
}
for _, r := range insp.Results {
if r.Queried != pi.Results[0].Queried {
a.Skips = append(a.Skips, Skip{Digest: r.Queried, Reason: SkipMultipartSeed})
}
}
return a, part, nil
}
return Admission{}, nil, nil
}
// gaps lists clusters whose positive anchor has a publication in progress;
// key 0 stands for every fresh match while the startup or rebuild rotation
// is owed.
func (e *Engine) gaps() (map[uint64]bool, error) {
gaps := map[uint64]bool{}
if !e.pub.FreshMatchingAllowed() {
gaps[0] = true
}
err := e.scanTargets(func(t Target) {
if t.T > 0 && t.Phase != PhaseIdle {
gaps[t.ClusterID] = true
}
})
if err != nil {
return nil, err
}
// An idle target may still have an uncommitted contribution. Do not
// count that UID or mark again before its reservation is resolved.
err = e.cfg.Ledger.View(func(v LedgerView) error {
return v.tx.ForEach(store.LearnIntentsBucket, func(_, data []byte) error {
var i Intent
if decodeErr := json.Unmarshal(data, &i); decodeErr != nil {
return errors.New("learning ledger intent is unreadable")
}
for _, c := range i.Clusters {
if !c.Done {
gaps[c.ClusterID] = true
}
}
return nil
})
})
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrStateIO, err)
}
return gaps, nil
}
func (e *Engine) scanTargets(fn func(Target)) error {
err := e.cfg.Ledger.View(func(v LedgerView) error {
return v.tx.ForEach(store.LearnTargetsBucket, func(_, data []byte) error {
var t Target
if err := json.Unmarshal(data, &t); err != nil {
return errors.New("learning ledger target is unreadable")
}
fn(t)
return nil
})
})
if err != nil {
return fmt.Errorf("%w: %w", ErrStateIO, err)
}
return nil
}
// scheduleZero queues deletion of zero-target digests that reappeared.
func (e *Engine) scheduleZero(ds []Digest) error {
for _, d := range ds {
t, err := e.pub.target(d)
if err != nil {
return err
}
if t.Phase == PhaseIdle {
if err := e.pub.setPhase(t, PhaseRead, nil); err != nil && !errors.Is(err, errTargetMoved) {
return err
}
}
}
return nil
}
func newIntentID() string {
var b [16]byte
_, _ = rand.Read(b[:])
return hex.EncodeToString(b[:])
}
// reserve durably records an admission before any native send: for a seed,
// the body is staged, committed under its cluster ID and synced first.
func (e *Engine) reserve(ctx context.Context, src Source, a Admission, snapshot []byte, id string, digests []Digest) (reservedID string, retErr error) {
bodyStaged := false
defer func() {
if retErr != nil && bodyStaged {
// Recovery retries only on an active turn, never after disable.
e.recovered = false
}
}()
if err := ctx.Err(); err != nil {
return "", err
}
if id == "" {
id = newIntentID()
}
now := e.cfg.Now().UTC()
kind := IntentReport
if src.Operator {
kind = IntentOperator
}
intent := Intent{ID: id, Kind: kind, UID: src.UID, MarkID: src.MarkID, Admitted: now}
var charge Usage
var seed *Seed
var seedID uint64
if a.Seed != nil {
if err := e.cfg.Ledger.View(func(v LedgerView) error {
t, ok, err := v.Target(a.Seed.Anchor)
if err == nil && ok && (t.T > 0 || t.Phase != PhaseIdle) {
return errDeferSeed
}
return err
}); err != nil {
if errors.Is(err, errDeferSeed) {
return "", err
}
return "", fmt.Errorf("%w: %w", ErrStateIO, err)
}
if err := e.cfg.Ledger.Update(func(tx LedgerTx) error {
var err error
seedID, err = tx.NextClusterID()
return err
}); err != nil {
return "", fmt.Errorf("%w: %w", ErrStateIO, err)
}
if err := ctx.Err(); err != nil {
return "", err
}
bodyStaged = true
staged, err := e.cfg.Bodies.Stage(snapshot)
if err != nil {
return "", fmt.Errorf("%w: %w", ErrStateIO, err)
}
if err := ctx.Err(); err != nil {
return "", err
}
if err := e.cfg.Bodies.Commit(staged, seedID); err != nil {
return "", fmt.Errorf("%w: %w", ErrStateIO, err)
}
s := *a.Seed
s.Staged, s.BodyLen, s.BodySHA = BodyName(seedID), staged.Len, staged.SHA
seed = &s
intent.Seed = seed
intent.Clusters = append(intent.Clusters, IntentCluster{ClusterID: seedID, Anchor: s.Anchor, Aliases: []Digest{s.Anchor}, NewT: s.T})
charge = Usage{BodyBytes: staged.Len, Aliases: 1}
}
if err := ctx.Err(); err != nil {
return "", err
}
err := e.cfg.Ledger.Update(func(tx LedgerTx) error {
if _, ok, err := tx.Intent(id); err != nil || ok {
return err
}
for _, j := range a.Joins {
c, ok, err := tx.Cluster(j.ClusterID)
if err != nil {
return err
}
if !ok {
return ErrStateMismatch
}
t, ok, err := tx.Target(c.Anchor)
if err != nil {
return err
}
if !ok || t.Phase != PhaseIdle {
return errTargetNotReady
}
prior := t
t.T, t.Revision, t.Phase, t.Updated = j.NewT, t.Revision+1, PhaseRead, now
if err := tx.PutTarget(t); err != nil {
return err
}
intent.Clusters = append(intent.Clusters, IntentCluster{ClusterID: j.ClusterID, Anchor: c.Anchor, Revision: t.Revision, Aliases: j.Aliases, NewT: j.NewT, Prior: &prior})
charge.Aliases += len(j.Aliases)
}
if seed != nil {
t, ok, err := tx.Target(seed.Anchor)
if err != nil {
return err
}
rev := uint64(1)
if ok {
prior := t
intent.Clusters[0].Prior = &prior
rev = t.Revision + 1
// The live target replaces the zero-target guard and its charge.
if t.GuardCharge != nil {
if err := tx.credit(*t.GuardCharge); err != nil {
return err
}
}
}
if err := tx.PutCluster(Cluster{ID: seedID, Anchor: seed.Anchor, BodyLen: seed.BodyLen, BodySHA: seed.BodySHA, Revision: 1, Created: now, Operator: src.Operator}); err != nil {
return err
}
if err := tx.PutTarget(Target{Digest: seed.Anchor, ClusterID: seedID, T: seed.T, Revision: rev, Phase: PhaseRead, Updated: now}); err != nil {
return err
}
intent.Clusters[0].Revision = rev
}
if src.Operator {
out := admissionOutcomes(admitResult{duplicates: a.Duplicates, skips: a.Skips})
for _, c := range intent.Clusters {
out = append(out, ClusterOutcome{ClusterID: c.ClusterID, Result: ResultPending})
}
if _, err := recordOutcome(tx, id, out, digests); err != nil {
return err
}
}
_, err := tx.Reserve(intent, src.UID, src.Operator, charge)
return err
})
switch {
case err == nil:
return id, nil
case errors.Is(err, errTargetNotReady):
return "", err
case errors.Is(err, ErrStateMismatch):
return "", err
}
// Nothing references the seed body; the next active turn removes it.
return "", fmt.Errorf("%w: %w", ErrStateIO, err)
}
// publishIntent runs a turn for each unfinished cluster of an intent and
// commits those verified at their target.
func (e *Engine) publishIntent(ctx context.Context, id string) (committed, pending []uint64, err error) {
var intent Intent
var ok bool
if err := e.cfg.Ledger.View(func(v LedgerView) error {
var err error
intent, ok, err = v.Intent(id)
return err
}); err != nil {
return nil, nil, fmt.Errorf("%w: %w", ErrStateIO, err)
}
if !ok {
return nil, nil, nil
}
if intent.Kind == IntentForget {
_, err := e.finishForget(ctx, intent)
return nil, nil, err
}
var errs []error
for k, ic := range intent.Clusters {
if ic.Done {
continue
}
target, err := e.pub.target(ic.Anchor)
if err != nil {
errs = append(errs, err)
pending = append(pending, ic.ClusterID)
continue
}
// Superseded contributions are terminal even if their replacement
// target is still awaiting publication.
res := TurnResult{Done: target.Revision != ic.Revision || target.ClusterID != ic.ClusterID}
if !res.Done {
res, err = e.pub.Turn(ctx, ic.Anchor)
}
if err != nil {
errs = append(errs, err)
}
if !res.Done {
pending = append(pending, ic.ClusterID)
continue
}
done, err := e.commit(id, k)
if err != nil {
errs = append(errs, err)
pending = append(pending, ic.ClusterID)
} else if done {
committed = append(committed, ic.ClusterID)
}
}
return committed, pending, errors.Join(errs...)
}
// commit records one verified contribution: votes or mark, aliases, last
// report time and charge. A cluster whose target moved on (forget,
// retirement) ends the contribution without a vote and returns its charge.
func (e *Engine) commit(id string, k int) (bool, error) {
committed := false
report := false
err := e.cfg.Ledger.Update(func(tx LedgerTx) error {
intent, ok, err := tx.Intent(id)
if err != nil || !ok || k >= len(intent.Clusters) || intent.Clusters[k].Done {
return err
}
ic := &intent.Clusters[k]
report = intent.Kind == IntentReport
ch := Charge{UID: intent.UID, Operator: intent.Kind == IntentOperator, Usage: Usage{Aliases: len(ic.Aliases)}}
if intent.Seed != nil && k == 0 {
ch.Usage.BodyBytes = intent.Seed.BodyLen
}
t, tok, err := tx.Target(ic.Anchor)
if err != nil {
return err
}
c, cok, err := tx.Cluster(ic.ClusterID)
if err != nil {
return err
}
ic.Done = true
if !tok || !cok || c.Retired || t.ClusterID != ic.ClusterID || t.Revision != ic.Revision || t.T != ic.NewT || t.Phase != PhaseIdle {
if err := tx.credit(ch); err != nil {
return err
}
} else {
if ch.Operator {
c.Marks = append(c.Marks, intent.MarkID)
} else {
c.UIDs = append(c.UIDs, intent.UID)
}
for _, a := range ic.Aliases {
if !slices.Contains(c.Aliases, a) {
c.Aliases = append(c.Aliases, a)
}
if err := tx.PutAlias(a, c.ID); err != nil {
return err
}
}
if intent.Admitted.After(c.LastReport) {
c.LastReport = intent.Admitted
}
c.Charges = append(c.Charges, ch)
c.Revision++
if err := tx.PutCluster(c); err != nil {
return err
}
committed = true
}
if intent.Kind == IntentOperator {
result := ResultSkipped + ":superseded"
if committed {
result = ResultCommitted
}
if _, err := recordOutcome(tx, id, []ClusterOutcome{{ClusterID: ic.ClusterID, Result: result}}, nil); err != nil {
return err
}
}
if !slices.ContainsFunc(intent.Clusters, func(ic IntentCluster) bool { return !ic.Done }) {
return tx.DeleteIntent(id)
}
return tx.PutIntent(intent)
})
if err != nil {
return false, fmt.Errorf("%w: %w", ErrStateIO, err)
}
if committed && report && e.cfg.OnReportCommit != nil {
e.cfg.OnReportCommit()
}
return committed, nil
}
// Operator runs one operator request through the owner.
func (e *Engine) Operator(ctx context.Context, r OperatorRequest) (OperatorResult, error) {
res := OperatorResult{RequestID: r.RequestID}
if !e.Publishing() {
return res, ErrLearningOff
}
if err := validateRequest(r); err != nil {
return res, err
}
var opErr error
if err := e.do(ctx, func() { res, opErr = e.operator(ctx, r) }); err != nil {
return res, err
}
return res, opErr
}
// Keep the inspected digests before native writes. A pending intent can
// outlive a failed outcome commit and resume without another inspection.
func (e *Engine) recordOperatorDigests(id string, digests []Digest) error {
if err := e.cfg.Ledger.Update(func(tx LedgerTx) error {
o, _, err := tx.Op(id)
if err != nil {
return err
}
if o.Outcome == nil {
o.Outcome = &OpOutcome{}
}
o.Outcome.Digests = digests
return tx.PutOp(o)
}); err != nil {
return fmt.Errorf("%w: %w", ErrStateIO, err)
}
return nil
}
func validateRequest(r OperatorRequest) error {
if r.Action != ActionSpam && r.Action != ActionForget {
return errors.New("unknown learning action")
}
if len(r.RequestID) != 32 || !lowerHex(r.RequestID) {
return errors.New("request ID is not 32 lowercase hex digits")
}
if len(r.Snapshot) > MaxBody {
return errors.New("message is larger than the learning limit")
}
sum := sha256.Sum256(r.Snapshot)
if hex.EncodeToString(sum[:]) != r.SnapshotSHA {
return errors.New("the file changed in transit")
}
return nil
}
func lowerHex(s string) bool {
for _, r := range s {
if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
return false
}
}
return true
}
func (e *Engine) operator(caller context.Context, r OperatorRequest) (OperatorResult, error) {
res := OperatorResult{RequestID: r.RequestID}
defer e.refresh()
ctx, end, ok := e.turnCtx(caller)
if !ok {
return res, ErrLearningOff
}
defer end()
e.start()
if e.bindErr != nil {
return res, e.bindErr
}
// Returning a terminal result needs no write or backend availability.
var prior OpRecord
var found bool
if err := e.cfg.Ledger.View(func(v LedgerView) error {
var err error
prior, found, err = v.Op(r.RequestID)
return err
}); err != nil {
err = fmt.Errorf("%w: %w", ErrStateIO, err)
e.note(err)
return res, err
}
if found {
if prior.Action != r.Action || prior.SnapshotSHA != r.SnapshotSHA {
return res, ErrRequestReused
}
if prior.Outcome != nil {
// A failing retry still reports what the first attempt inspected.
res.Digests = prior.Outcome.Digests
}
if prior.Done && prior.Outcome != nil {
o := prior.Outcome
return OperatorResult{RequestID: r.RequestID, Clusters: o.Clusters, Digests: o.Digests, Outcome: o.Outcome, ExitOK: o.ExitOK}, nil
}
}
if err := e.prepare(ctx); err != nil {
e.note(err)
return res, err
}
if !found {
if err := e.cfg.Ledger.Update(func(tx LedgerTx) error {
return tx.PutOp(OpRecord{RequestID: r.RequestID, Action: r.Action, SnapshotSHA: r.SnapshotSHA})
}); err != nil {
err = fmt.Errorf("%w: %w", ErrStateIO, err)
e.note(err)
return res, err
}
}
var update []ClusterOutcome
var digests []Digest
var err error
if r.Action == ActionSpam {
update, digests, err = e.operatorSpam(ctx, r)
} else {
update, digests, err = e.forget(ctx, r)
}
if len(digests) > 0 {
res.Digests = digests
}
if err != nil {
e.note(err)
return res, err
}
if r.Action == ActionSpam {
e.requestFails = 0
}
if err := e.cfg.Ledger.Update(func(tx LedgerTx) error {
var err error
res, err = recordOutcome(tx, r.RequestID, update, digests)
return err
}); err != nil {
err = fmt.Errorf("%w: %w", ErrStateIO, err)
e.note(err)
return res, err
}
return res, nil
}
// recordOutcome is committed with the reservation or contribution it
// describes. A crash cannot leave a completed mark eligible for admission.
// digests, when given, replace the recorded file digests.
func recordOutcome(tx LedgerTx, id string, update []ClusterOutcome, digests []Digest) (OperatorResult, error) {
o, ok, err := tx.Op(id)
if err != nil {
return OperatorResult{}, err
}
if !ok {
return OperatorResult{}, ErrStateMismatch
}
var prior []ClusterOutcome
var priorDigests []Digest
if o.Outcome != nil {
prior, priorDigests = o.Outcome.Clusters, o.Outcome.Digests
}
if len(digests) == 0 {
digests = priorDigests
}
if !o.Done {
cs := mergeOutcomes(prior, update)
outcome, exitOK := summarize(o.Action, cs)
o.Outcome = &OpOutcome{Clusters: cs, Digests: digests, Outcome: outcome, ExitOK: exitOK}
o.Done = outcome != ResultPending && outcome != ResultDeferred
if err := tx.PutOp(o); err != nil {
return OperatorResult{}, err
}
}
return OperatorResult{RequestID: id, Clusters: o.Outcome.Clusters, Digests: o.Outcome.Digests, Outcome: o.Outcome.Outcome, ExitOK: o.Outcome.ExitOK}, nil
}
// mergeOutcomes applies a retry's per-cluster results over the recorded
// ones; parts without a cluster stay as first recorded.
func mergeOutcomes(prior, update []ClusterOutcome) []ClusterOutcome {
out := slices.Clone(prior)
for _, u := range update {
i := slices.IndexFunc(out, func(c ClusterOutcome) bool { return u.ClusterID != 0 && c.ClusterID == u.ClusterID })
switch {
case i >= 0:
out[i] = u
case u.ClusterID != 0 || len(prior) == 0:
out = append(out, u)
}
}
slices.SortStableFunc(out, func(a, b ClusterOutcome) int {
// Clusters first, in ID order; skipped parts after them.
switch {
case a.ClusterID == 0 && b.ClusterID != 0:
return 1
case a.ClusterID != 0 && b.ClusterID == 0:
return -1
}
return cmp.Compare(a.ClusterID, b.ClusterID)
})
return out
}
// summarize derives the overall outcome: zero exit only for fully committed
// or all-duplicate spam and completely forgotten files.
func summarize(action string, cs []ClusterOutcome) (string, bool) {
count := map[string]int{}
for _, c := range cs {
r := c.Result
if strings.HasPrefix(r, ResultSkipped+":") {
r = ResultSkipped
}
count[r]++
}
done := count[ResultCommitted] + count[ResultDuplicate] + count[ResultForgotten] + count[ResultNoMatch]
switch {
case len(cs) == 0:
return ResultDeferred, false
case count[ResultPending] > 0:
return ResultPending, false
case count[ResultSkipped] > 0 && done > 0:
return ResultPartial, false
case count[ResultSkipped] > 0:
return ResultSkipped, false
case action == ActionForget && count[ResultNoMatch] == len(cs):
return ResultNoMatch, true
case action == ActionForget:
return ResultForgotten, true
case count[ResultCommitted] > 0:
return ResultCommitted, true
}
return ResultDuplicate, true
}
// operatorSpam admits the file with one operator mark per resolved cluster,
// or resumes the request's pending intent.
func (e *Engine) operatorSpam(ctx context.Context, r OperatorRequest) ([]ClusterOutcome, []Digest, error) {
var resumed bool
if err := e.cfg.Ledger.View(func(v LedgerView) error {
_, ok, err := v.Intent(r.RequestID)
resumed = ok
return err
}); err != nil {
return nil, nil, fmt.Errorf("%w: %w", ErrStateIO, err)
}
var a admitResult
if resumed {
var err error
a.committed, a.pending, err = e.publishIntent(ctx, r.RequestID)
if err != nil {
return nil, nil, err
}
} else {
var err error
a, err = e.admit(ctx, Source{Operator: true, MarkID: r.RequestID, BodyLen: int64(len(r.Snapshot))}, r.Snapshot, r.RequestID)
if err != nil {
return nil, a.digests, err
}
}
return admissionOutcomes(a), a.digests, nil
}
func admissionOutcomes(a admitResult) []ClusterOutcome {
var out []ClusterOutcome
add := func(ids []uint64, result string) {
for _, id := range ids {
out = append(out, ClusterOutcome{ClusterID: id, Result: result})
}
}
add(a.committed, ResultCommitted)
add(a.duplicates, ResultDuplicate)
add(a.pending, ResultPending)
for _, s := range a.skips {
if s.Reason == SkipDuplicate {
continue // The cluster already reports this as a successful duplicate.
}
out = append(out, ClusterOutcome{Result: ResultSkipped + ":" + s.Reason})
}
return out
}
// forget resolves accepted aliases first, then live anchors matched at or
// above the join boundary, supersedes each resolved cluster with T=0 and
// waits for verified absence and body removal.
func (e *Engine) forget(ctx context.Context, r OperatorRequest) ([]ClusterOutcome, []Digest, error) {
var out []ClusterOutcome
var digests []Digest
var intent Intent
var resumed bool
if err := e.cfg.Ledger.View(func(v LedgerView) error {
var err error
intent, resumed, err = v.Intent(r.RequestID)
return err
}); err != nil {
return nil, digests, fmt.Errorf("%w: %w", ErrStateIO, err)
}
if !resumed {
if err := ctx.Err(); err != nil {
return nil, digests, err
}
// Learning inspects the text-only form; forget must match it.
var insp Inspection
if body := textOnlyMessage(r.Snapshot); body != nil {
var err error
if insp, err = e.cfg.Client.Inspect(ctx, body); err != nil {
return nil, digests, fmt.Errorf("%w: %w", ErrReadFailed, err)
}
}
digests = digestsOf(insp)
if recordErr := e.recordOperatorDigests(r.RequestID, digests); recordErr != nil {
return nil, digests, recordErr
}
if insp.Distinct == 0 {
return nil, digests, ErrNoDigest
}
ids, skipped, err := e.resolveForget(insp)
if errors.Is(err, errTargetNotReady) {
return nil, digests, nil
}
if err != nil {
return nil, digests, err
}
if err := ctx.Err(); err != nil {
return nil, digests, err
}
out = skipped
if len(ids) == 0 {
e.requestFails = 0
return out, digests, nil
}
intent = Intent{ID: r.RequestID, Kind: IntentForget}
if err := e.cfg.Ledger.Update(func(tx LedgerTx) error {
now := e.cfg.Now().UTC()
for _, id := range ids {
c, ok, err := tx.Cluster(id.cluster)
if err != nil {
return err
}
if !ok {
c = Cluster{ID: id.cluster, Anchor: id.anchor}
}
if serr := supersede(tx, c, now); serr != nil {
return serr
}
if !ok {
// Nothing to clean up: only the zero target remains.
if derr := tx.DeleteCluster(id.cluster); derr != nil {
return derr
}
}
t, _, err := tx.Target(c.Anchor)
if err != nil {
return err
}
intent.Clusters = append(intent.Clusters, IntentCluster{ClusterID: id.cluster, Anchor: c.Anchor, Revision: t.Revision})
}
pending := slices.Clone(skipped)
for _, c := range intent.Clusters {
pending = append(pending, ClusterOutcome{ClusterID: c.ClusterID, Result: ResultPending})
}
if _, err := recordOutcome(tx, intent.ID, pending, digests); err != nil {
return err
}
return tx.PutIntent(intent)
}); err != nil {
return nil, digests, fmt.Errorf("%w: %w", ErrStateIO, err)
}
}
out, err := e.finishForget(ctx, intent)
// The durable pending outcome survives native and cleanup failures.
// They are also exposed through Problems until a later turn succeeds.
e.note(err)
if err == nil {
e.requestFails = 0
}
return out, digests, nil
}
func (e *Engine) finishForget(ctx context.Context, intent Intent) ([]ClusterOutcome, error) {
var errs []error
allDone := true
for k, ic := range intent.Clusters {
if !ic.Done {
t, err := e.pub.target(ic.Anchor)
switch {
case err != nil:
errs = append(errs, err)
case t.T != 0 || t.ClusterID != ic.ClusterID:
errs = append(errs, ErrStateMismatch)
default:
tr, err := e.pub.Turn(ctx, ic.Anchor)
if err != nil {
errs = append(errs, err)
}
intent.Clusters[k].Done = tr.Done
}
}
allDone = allDone && intent.Clusters[k].Done
}
if ctx.Err() != nil {
return nil, errors.Join(append(errs, ctx.Err())...)
}
if err := e.pub.cleanup(ctx); err != nil {
return nil, errors.Join(append(errs, err)...)
}
var out []ClusterOutcome
for _, ic := range intent.Clusters {
result := ResultPending
if ic.Done {
result = ResultForgotten
}
out = append(out, ClusterOutcome{ClusterID: ic.ClusterID, Result: result})
}
if err := e.cfg.Ledger.Update(func(tx LedgerTx) error {
if _, err := recordOutcome(tx, intent.ID, out, nil); err != nil {
return err
}
if allDone {
return tx.DeleteIntent(intent.ID)
}
return tx.PutIntent(intent)
}); err != nil {
return nil, errors.Join(append(errs, fmt.Errorf("%w: %w", ErrStateIO, err))...)
}
return out, errors.Join(errs...)
}
type forgetTarget struct {
cluster uint64
anchor Digest
}
// resolveForget maps every part before anything changes: aliases first,
// then live anchors at or above the boundary. Unknown
// anchors abort; low-probability hits are skipped parts; misses are
// no-match parts.
func (e *Engine) resolveForget(insp Inspection) ([]forgetTarget, []ClusterOutcome, error) {
var out []forgetTarget
var skipped []ClusterOutcome
fresh := false
waiting := false
err := e.cfg.Ledger.View(func(v LedgerView) error {
for _, r := range insp.Results {
id, ok, err := v.Alias(r.Queried)
if err != nil {
return err
}
var anchor Digest
fresh = fresh || !ok
switch {
case ok:
c, cok, err := v.Cluster(id)
if err != nil {
return err
}
if !cok {
return ErrUnknownAnchor
}
anchor = c.Anchor
case r.Found:
t, tok, err := v.Target(r.Stored)
if err != nil {
return err
}
if !tok {
return ErrUnknownAnchor
}
if t.T == 0 {
waiting = true
continue
}
c, found, err := v.Cluster(t.ClusterID)
if err != nil {
return err
}
if !found || c.Retired || c.Anchor != t.Digest {
return ErrUnknownAnchor
}
if r.Prob < JoinBoundary {
skipped = append(skipped, ClusterOutcome{Result: ResultSkipped + ":" + SkipBelowJoin})
continue
}
id, anchor = t.ClusterID, r.Stored
default:
skipped = append(skipped, ClusterOutcome{Result: ResultNoMatch})
continue
}
if !slices.ContainsFunc(out, func(f forgetTarget) bool { return f.cluster == id }) {
out = append(out, forgetTarget{cluster: id, anchor: anchor})
}
}
return nil
})
if err != nil && !errors.Is(err, ErrUnknownAnchor) {
err = fmt.Errorf("%w: %w", ErrStateIO, err)
}
if err != nil {
return nil, nil, err
}
if waiting {
return nil, nil, errTargetNotReady
}
if fresh {
gaps, err := e.gaps()
if err != nil {
return nil, nil, err
}
if len(gaps) > 0 {
return nil, nil, errTargetNotReady
}
}
return out, skipped, nil
}
// refresh publishes the learner's problems.
func (e *Engine) refresh() {
var ps []antispam.Problem
add := func(reason, detail string) {
ps = append(ps, antispam.Problem{Code: antispam.CodeLearning, Severity: antispam.Warning, Key: reason, Detail: detail})
}
uncertain := 0
scanErr := e.scanTargets(func(t Target) {
if t.Phase == PhaseAddSent || t.Phase == PhaseDeleteSent {
uncertain++
}
})
if errors.Is(e.bindErr, ErrStateMismatch) || e.frozen {
add(antispam.ReasonStateMismatch, "learning state does not match its storage; new reports are held and nothing is adopted or deleted")
}
if e.stateErr != nil || scanErr != nil || e.bindErr != nil && !errors.Is(e.bindErr, ErrStateMismatch) {
add(antispam.ReasonStateIO, "learning state could not be written; no unjournaled change is sent and installed scores stay")
}
if uncertain > 0 {
add(antispam.ReasonMutationUncertain, strconv.Itoa(uncertain)+" anchor writes have an unknown outcome and are being reconciled")
}
if e.bodyErr {
add(antispam.ReasonAnchorBodyInvalid, "a stored anchor body is missing or changed; that cluster's publication is held")
}
if n := e.pub.RepeatedDrift(); n > 0 {
add(antispam.ReasonAnchorDrift, strconv.Itoa(n)+" anchors drifted from their target more than once")
}
if e.requestFails >= RequestErrorPasses {
add(antispam.ReasonRequestErrors, "learning requests to rspamd failed repeatedly")
}
if e.pub.RebuildNeeded() {
add(antispam.ReasonBackendRebuilding, "anchors are missing from storage; they are being rebuilt and new clusters wait")
}
since := e.oldest
if since.IsZero() {
since = e.startedAt
}
if !since.IsZero() && e.cfg.Now().Sub(since) > AuditOverdueAfter {
add(antispam.ReasonAuditOverdue, "the anchor audit has not completed a rotation within "+AuditOverdueAfter.String())
}
if len(e.capacity) > 0 {
add(antispam.ReasonAccountCapacity, strconv.Itoa(len(e.capacity))+" reporters reached their learning allowance; their new reports are skipped")
}
e.mu.Lock()
e.published = ps
e.mu.Unlock()
}
package learn
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"math/big"
)
// Result is one digest's reply from the inspection plugin. Stored is the
// full stored digest a hit matched; it is empty for a miss.
type Result struct {
Queried, Stored Digest
Found, Confirmed bool
Prob float64
Value int64
Flag int
}
// Inspection is a complete, validated inspection: one result per distinct
// queried digest.
type Inspection struct {
Results []Result
Distinct int
}
// ErrMessageRejected identifies a body the plugin could not process. It
// still invalidates the inspection, but says nothing about backend health.
var ErrMessageRejected = errors.New("inspection rejected the message")
// ErrInspectionFailed marks a body inspection the controller answered with
// an error status or an incomplete, failed or malformed reply. Storage or
// the body can cause it, so blaming the message needs other evidence.
var ErrInspectionFailed = errors.New("inspection reply failed")
// rawResult keeps every field optional so absence is detectable.
type rawResult struct {
Queried *string `json:"queried"`
Found *bool `json:"found"`
Digest *string `json:"digest"`
Prob json.RawMessage `json:"prob"`
Value json.RawMessage `json:"value"`
Flag json.RawMessage `json:"flag"`
Confirmed *bool `json:"confirmed"`
}
// ParseInspection validates a plugin reply. explicit is the
// digest list of an exact-only check, nil for a body inspection. Anything
// missing, unexpected, duplicate, malformed or failed invalidates the whole
// inspection; it never reads as a miss. Errors are fixed text.
func ParseInspection(body []byte, explicit []Digest) (Inspection, error) {
fields, err := objectFields(body, "results", "errors", "complete", "distinct")
if err != nil || len(fields) != 4 {
return Inspection{}, errors.New("inspection reply is malformed")
}
var complete bool
var distinct int
if json.Unmarshal(fields["complete"], &complete) != nil || !complete {
if bytes.Equal(fields["complete"], []byte("false")) && explicit == nil && rejectedMessage(fields) {
return Inspection{}, ErrMessageRejected
}
return Inspection{}, errors.New("inspection is incomplete")
}
if json.Unmarshal(fields["distinct"], &distinct) != nil || distinct < 0 || distinct > MaxExplicitDigests {
return Inspection{}, errors.New("inspection has an invalid digest count")
}
var problems []json.RawMessage
if err := uclList(fields["errors"], &problems); err != nil || len(problems) != 0 {
return Inspection{}, errors.New("inspection reported errors")
}
var raws []json.RawMessage
if err := uclList(fields["results"], &raws); err != nil {
return Inspection{}, errors.New("inspection results are not a list")
}
var want map[Digest]bool
if explicit != nil {
want = map[Digest]bool{}
for _, d := range explicit {
if _, err := ParseDigest(string(d)); err != nil {
return Inspection{}, errors.New("inspection request holds an invalid digest")
}
want[d] = true
}
if distinct != len(want) {
return Inspection{}, errors.New("inspection answered a different digest count")
}
}
if len(raws) != distinct {
return Inspection{}, errors.New("inspection results do not match its digest count")
}
insp := Inspection{Results: make([]Result, 0, len(raws)), Distinct: distinct}
seen := map[Digest]bool{}
for _, raw := range raws {
r, err := parseResult(raw)
if err != nil {
return Inspection{}, err
}
if seen[r.Queried] {
return Inspection{}, errors.New("inspection answered a digest twice")
}
seen[r.Queried] = true
if want != nil && (!want[r.Queried] || r.Found && r.Stored != r.Queried) {
return Inspection{}, errors.New("exact inspection returned an unrequested digest")
}
insp.Results = append(insp.Results, r)
}
return insp, nil
}
// Only the plugin's pre-query body failures are local to one report.
// Exact checks, storage errors and malformed replies still stop submissions.
func rejectedMessage(fields map[string]json.RawMessage) bool {
var problems, results []json.RawMessage
var distinct int
if uclList(fields["errors"], &problems) != nil || len(problems) != 1 ||
uclList(fields["results"], &results) != nil || len(results) != 0 ||
json.Unmarshal(fields["distinct"], &distinct) != nil {
return false
}
var reason string
if json.Unmarshal(problems[0], &reason) != nil {
return false
}
switch reason {
case "cannot process message", "cannot hash message":
return distinct == 0
case "too many digests":
return distinct > MaxExplicitDigests
default:
return false
}
}
func parseResult(raw json.RawMessage) (Result, error) {
fields, err := objectFields(raw, "queried", "found", "digest", "prob", "value", "flag", "confirmed")
if err != nil {
return Result{}, err
}
var rr rawResult
if err = json.Unmarshal(raw, &rr); err != nil || rr.Queried == nil || rr.Found == nil || rr.Prob == nil || rr.Value == nil || rr.Flag == nil || rr.Confirmed == nil {
return Result{}, errors.New("inspection result is malformed")
}
q, err := ParseDigest(*rr.Queried)
if err != nil {
return Result{}, errors.New("inspection result has an invalid digest")
}
value, okValue := wholeNumber(rr.Value)
flag, okFlag := wholeNumber(rr.Flag)
prob, okProb := inspectionNumber(rr.Prob)
if !okValue || !okFlag || !okProb || prob.Sign() < 0 || prob.Cmp(big.NewRat(1, 1)) > 0 || value < 0 {
return Result{}, errors.New("inspection result has invalid values")
}
p, _ := prob.Float64()
r := Result{Queried: q, Found: *rr.Found, Prob: p, Value: value}
if !r.Found {
if value != 0 || prob.Sign() != 0 || flag != 0 || *rr.Confirmed || fields["digest"] != nil {
return Result{}, errors.New("inspection miss carries a value")
}
return r, nil
}
if rr.Digest == nil || !*rr.Confirmed || flag != 1 {
return Result{}, errors.New("inspection hit is not a confirmed CSM digest")
}
if r.Stored, err = ParseDigest(*rr.Digest); err != nil {
return Result{}, errors.New("inspection hit has an invalid stored digest")
}
r.Confirmed, r.Flag = true, 1
return r, nil
}
// wholeNumber reads a nonfractional JSON number; Lua numbers may arrive as
// 2 or 2.0.
func wholeNumber(raw json.RawMessage) (int64, bool) {
n, ok := inspectionNumber(raw)
if !ok || !n.IsInt() || !n.Num().IsInt64() {
return 0, false
}
return n.Num().Int64(), true
}
// Validate the decimal value before rounding it to float64; rounding can
// turn a fractional value into an integer or a nonzero miss into zero.
func inspectionNumber(raw json.RawMessage) (*big.Rat, bool) {
if len(raw) == 0 || (raw[0] != '-' && (raw[0] < '0' || raw[0] > '9')) {
return nil, false
}
return new(big.Rat).SetString(string(raw))
}
// uclList decodes a list, reading rspamd's {} encoding of an empty Lua table
// as an empty list.
func uclList(raw json.RawMessage, out *[]json.RawMessage) error {
if fields, err := objectFields(raw); err == nil && len(fields) == 0 {
*out = nil
return nil
}
if len(raw) == 0 || raw[0] != '[' {
return errors.New("not a list")
}
return json.Unmarshal(raw, out)
}
// objectFields reads one JSON object allowing only the named fields, with
// no duplicates, null values or trailing data. Callers check required fields.
func objectFields(data []byte, names ...string) (map[string]json.RawMessage, error) {
malformed := errors.New("inspection reply is malformed")
dec := json.NewDecoder(bytes.NewReader(data))
if tok, err := dec.Token(); err != nil || tok != json.Delim('{') {
return nil, malformed
}
allowed := map[string]bool{}
for _, n := range names {
allowed[n] = true
}
fields := map[string]json.RawMessage{}
for dec.More() {
tok, err := dec.Token()
key, ok := tok.(string)
if err != nil || !ok || !allowed[key] || fields[key] != nil {
return nil, malformed
}
var v json.RawMessage
if err := dec.Decode(&v); err != nil || bytes.Equal(v, []byte("null")) {
return nil, malformed
}
fields[key] = v
}
if tok, err := dec.Token(); err != nil || tok != json.Delim('}') {
return nil, malformed
}
if _, err := dec.Token(); err != io.EOF {
return nil, malformed
}
return fields, nil
}
// ExactValue reads d's stored value from an exact-only inspection. A miss is
// value 0, not present; a hit on another stored digest is an error.
func (i Inspection) ExactValue(d Digest) (value int64, present bool, err error) {
for _, r := range i.Results {
if r.Queried != d {
continue
}
if !r.Found {
return 0, false, nil
}
if r.Stored != d {
return 0, false, fmt.Errorf("digest matched another stored digest")
}
return r.Value, true, nil
}
return 0, false, errors.New("digest was not inspected")
}
package learn
import (
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"strconv"
"sync"
"time"
"github.com/pidginhost/csm/internal/store"
)
// ErrStateMismatch: the ledger does not belong to the current hash keys and
// storage, or holds records without a binding.
var ErrStateMismatch = errors.New("learning ledger does not match the current storage")
const bindingKey = "binding"
// declaredRestoreKey marks a ledger that csm restore brought back. It holds
// admissions with state_mismatch until learning has been seen disabled,
// the operator's confirmation, or a fresh namespace replaces the ledger.
const declaredRestoreKey = "declared_restore"
// kv is the store's learning transaction API; tests wrap it to inject
// commit failures.
type kv interface {
LearnUpdate(func(store.LearnTx) error) error
LearnView(func(store.LearnTx) error) error
}
// Ledger owns the learning buckets. Only the engine writes it.
type Ledger struct {
db kv
// Opt-out checks share the report lookup until an intent transaction
// commits. Native phase changes do not change report ownership.
intentsMu sync.Mutex
reportIDs map[int][]string
}
// Open returns the ledger stored in db.
func Open(db *store.DB) *Ledger {
return &Ledger{db: db}
}
// LedgerView reads the ledger.
type LedgerView struct {
tx store.LearnTx
}
// LedgerTx reads and writes the ledger in one transaction.
type LedgerTx struct {
LedgerView
intentsChanged *bool
}
// View runs fn in a read-only transaction.
func (l *Ledger) View(fn func(LedgerView) error) error {
return l.db.LearnView(func(tx store.LearnTx) error { return fn(LedgerView{tx: tx}) })
}
// Update runs fn in one read-write transaction; an error rolls it back.
func (l *Ledger) Update(fn func(LedgerTx) error) error {
l.intentsMu.Lock()
defer l.intentsMu.Unlock()
changed := false
err := l.db.LearnUpdate(func(tx store.LearnTx) error {
return fn(LedgerTx{LedgerView: LedgerView{tx: tx}, intentsChanged: &changed})
})
if err == nil && changed {
l.reportIDs = nil
}
return err
}
// Bind records the hash generation and prefix on first use and refuses a
// different one later. Records without a binding are ErrStateMismatch: they
// belong to storage CSM can no longer identify.
func (l *Ledger) Bind(gen, prefix string) error {
if gen == "" || prefix == "" {
return errors.New("learning binding needs a hash generation and prefix")
}
return l.Update(func(tx LedgerTx) error {
var at time.Time
if held, err := tx.get(store.LearnMetaBucket, []byte(declaredRestoreKey), &at); err != nil || held {
return errors.Join(err, fmt.Errorf("%w: restored learning state awaits confirmation", ErrStateMismatch))
}
b, ok, err := tx.Binding()
if err != nil {
return err
}
if ok {
if b.HashGeneration != gen || b.Prefix != prefix {
return ErrStateMismatch
}
return nil
}
for _, name := range store.LearnBuckets {
empty := true
if err := tx.tx.ForEach(name, func(_, _ []byte) error { empty = false; return errStop }); err != nil && !errors.Is(err, errStop) {
return err
}
if !empty {
return ErrStateMismatch
}
}
return tx.put(store.LearnMetaBucket, []byte(bindingKey), Binding{HashGeneration: gen, Prefix: prefix, Created: time.Now().UTC()})
})
}
var errStop = errors.New("stop")
func clusterKey(id uint64) []byte { return binary.BigEndian.AppendUint64(nil, id) }
func usageKey(uid int, operator bool) []byte {
if operator {
return []byte("operator")
}
return []byte("uid:" + strconv.Itoa(uid))
}
// get decodes the record at key; ok is false when it is absent.
func (v LedgerView) get(bucket string, key []byte, out any) (bool, error) {
data, err := v.tx.Get(bucket, key)
if err != nil || data == nil {
return false, err
}
if err := json.Unmarshal(data, out); err != nil {
return false, fmt.Errorf("learning ledger record in %s is unreadable", bucket)
}
return true, nil
}
func (tx LedgerTx) put(bucket string, key []byte, rec any) error {
data, err := json.Marshal(rec)
if err != nil {
return err
}
return tx.tx.Put(bucket, key, data)
}
// Binding returns the stored binding.
func (v LedgerView) Binding() (Binding, bool, error) {
var b Binding
ok, err := v.get(store.LearnMetaBucket, []byte(bindingKey), &b)
return b, ok, err
}
// Cluster returns cluster id.
func (v LedgerView) Cluster(id uint64) (Cluster, bool, error) {
var c Cluster
ok, err := v.get(store.LearnClustersBucket, clusterKey(id), &c)
return c, ok, err
}
// Alias returns the cluster an accepted alias belongs to.
func (v LedgerView) Alias(d Digest) (uint64, bool, error) {
data, err := v.tx.Get(store.LearnAliasesBucket, []byte(d))
if err != nil || data == nil {
return 0, false, err
}
if len(data) != 8 {
return 0, false, errors.New("learning ledger alias is unreadable")
}
return binary.BigEndian.Uint64(data), true, nil
}
// Target returns the current target of d.
func (v LedgerView) Target(d Digest) (Target, bool, error) {
var t Target
ok, err := v.get(store.LearnTargetsBucket, []byte(d), &t)
return t, ok, err
}
// Intent returns intent id.
func (v LedgerView) Intent(id string) (Intent, bool, error) {
var i Intent
ok, err := v.get(store.LearnIntentsBucket, []byte(id), &i)
return i, ok, err
}
// Op returns the operator request record.
func (v LedgerView) Op(requestID string) (OpRecord, bool, error) {
var o OpRecord
ok, err := v.get(store.LearnOpsBucket, []byte(requestID), &o)
return o, ok, err
}
// Usage returns what an account (or the operator) has been charged.
func (v LedgerView) Usage(uid int, operator bool) (Usage, error) {
var u Usage
_, err := v.get(store.LearnUsageBucket, usageKey(uid, operator), &u)
return u, err
}
// NextClusterID allocates a cluster ID; IDs are never reused.
func (tx LedgerTx) NextClusterID() (uint64, error) {
return tx.tx.NextSequence(store.LearnClustersBucket)
}
// PutCluster stores c.
func (tx LedgerTx) PutCluster(c Cluster) error {
if c.ID == 0 {
return errors.New("cluster has no ID")
}
return tx.put(store.LearnClustersBucket, clusterKey(c.ID), c)
}
// DeleteCluster removes cluster id.
func (tx LedgerTx) DeleteCluster(id uint64) error {
return tx.tx.Delete(store.LearnClustersBucket, clusterKey(id))
}
// PutAlias maps an accepted alias to its cluster.
func (tx LedgerTx) PutAlias(d Digest, clusterID uint64) error {
return tx.tx.Put(store.LearnAliasesBucket, []byte(d), clusterKey(clusterID))
}
// DeleteAlias removes an alias.
func (tx LedgerTx) DeleteAlias(d Digest) error {
return tx.tx.Delete(store.LearnAliasesBucket, []byte(d))
}
// PutTarget stores the current target of t.Digest.
func (tx LedgerTx) PutTarget(t Target) error {
return tx.put(store.LearnTargetsBucket, []byte(t.Digest), t)
}
// DeleteTarget removes the target of d.
func (tx LedgerTx) DeleteTarget(d Digest) error {
return tx.tx.Delete(store.LearnTargetsBucket, []byte(d))
}
// PutIntent stores i.
func (tx LedgerTx) PutIntent(i Intent) error {
*tx.intentsChanged = true
return tx.put(store.LearnIntentsBucket, []byte(i.ID), i)
}
// DeleteIntent removes intent id.
func (tx LedgerTx) DeleteIntent(id string) error {
*tx.intentsChanged = true
return tx.tx.Delete(store.LearnIntentsBucket, []byte(id))
}
// PutOp stores an operator request record.
func (tx LedgerTx) PutOp(o OpRecord) error {
return tx.put(store.LearnOpsBucket, []byte(o.RequestID), o)
}
// AddUsage adds delta (which may be negative) to an account's charge.
func (tx LedgerTx) AddUsage(uid int, operator bool, delta Usage) error {
u, err := tx.Usage(uid, operator)
if err != nil {
return err
}
u.BodyBytes += delta.BodyBytes
u.Aliases += delta.Aliases
if u.BodyBytes < 0 || u.Aliases < 0 {
return errors.New("learning usage would go negative")
}
return tx.put(store.LearnUsageBucket, usageKey(uid, operator), u)
}
// Reserve stores a new intent and charges its admission once. A retry with
// an existing intent ID charges nothing and returns the stored intent.
func (tx LedgerTx) Reserve(i Intent, uid int, operator bool, charge Usage) (Intent, error) {
stored, ok, err := tx.Intent(i.ID)
if err != nil || ok {
return stored, err
}
if err := tx.AddUsage(uid, operator, charge); err != nil {
return Intent{}, err
}
return i, tx.PutIntent(i)
}
// MarkDeclaredRestore records that this ledger was restored from a backup.
func (l *Ledger) MarkDeclaredRestore(at time.Time) error {
return l.Update(func(tx LedgerTx) error {
return tx.put(store.LearnMetaBucket, []byte(declaredRestoreKey), at.UTC())
})
}
// DeclaredRestore reports whether a restore still awaits confirmation.
func (l *Ledger) DeclaredRestore() (bool, error) {
var held bool
err := l.View(func(v LedgerView) error {
var at time.Time
var err error
held, err = v.get(store.LearnMetaBucket, []byte(declaredRestoreKey), &at)
return err
})
return held, err
}
// ClearDeclaredRestore confirms a restore; it writes only when marked.
func (l *Ledger) ClearDeclaredRestore() error {
held, err := l.DeclaredRestore()
if err != nil || !held {
return err
}
return l.Update(func(tx LedgerTx) error {
return tx.tx.Delete(store.LearnMetaBucket, []byte(declaredRestoreKey))
})
}
package learn
import (
"bufio"
"bytes"
"crypto/sha256"
"fmt"
"io"
"iter"
"mime"
"mime/multipart"
"net/textproto"
"sort"
"strings"
)
// maxPartDepth bounds multipart nesting. Every level rescans the bytes it
// encloses, so a tenant-built message cannot make splitting quadratic; real
// mail nests three or four levels.
const maxPartDepth = 8
type textPart struct {
header textproto.MIMEHeader
body []byte
}
// textPartMessages yields up to limit single-part messages from inline
// text/plain or text/html parts, the largest first. Each keeps the
// original header fields except Content-* and MIME-Version, then the part's
// own content type and transfer encoding, then its undecoded body. A message
// that is not multipart, has no header break, carries an unreadable header or
// nests deeper than maxPartDepth yields none. Parts after a broken boundary
// are dropped. The caller supplies its inspection budget as limit. Shared
// headers are copied only when a message is requested, so many small parts
// cannot multiply a large header into an unbounded allocation.
func textPartMessages(msg []byte, limit int) iter.Seq[[]byte] {
return func(yield func([]byte) bool) {
if limit <= 0 {
return
}
head, body, eol, ok := splitHeader(msg)
if !ok {
return
}
h, err := readHeader(head, eol)
if err != nil {
return
}
parts := make([]textPart, 0, limit)
var dropped bool
if !collectTextParts(h, body, 1, limit, inlinePlainOrHTML, &parts, &dropped) || len(parts) == 0 {
return
}
fields := keptFields(head)
for _, p := range parts {
var b bytes.Buffer
b.Write(fields)
b.WriteString("MIME-Version: 1.0" + eol)
writePartHeader(&b, p, eol)
b.WriteString(eol)
b.Write(p.body)
if !yield(b.Bytes()) {
return
}
}
}
}
// textOnlyMessage returns msg with only its text parts: the form the learner
// inspects, publishes and stores. rspamd hashes every text/* part, attached
// or inline, so those stay; a non-text part that rspamd hashes on its own,
// such as a PDF with scripts, can then neither break an inspection nor
// publish a digest the ledger does not track. A message with nothing to drop
// comes back unchanged. Unreadable MIME, no text or a rebuild exceeding
// MaxBody yields nil.
func textOnlyMessage(msg []byte) []byte {
head, body, eol, ok := splitHeader(msg)
if !ok {
return msg
}
h, err := readHeader(head, eol)
if err != nil {
return nil
}
ct := h.Get("Content-Type")
if ct == "" {
return msg
}
mt, _, err := mime.ParseMediaType(ct)
switch {
case err != nil:
return nil
case strings.HasPrefix(mt, "text/"):
return msg
case !strings.HasPrefix(mt, "multipart/"):
return nil
}
var parts []textPart
var dropped bool
switch {
case !collectTextParts(h, body, 1, MaxExplicitDigests, anyText, &parts, &dropped):
return nil
case len(parts) == 0:
return nil
case !dropped:
return msg
}
var b bytes.Buffer
b.Write(keptFields(head))
b.WriteString("MIME-Version: 1.0" + eol)
if len(parts) == 1 {
writePartHeader(&b, parts[0], eol)
b.WriteString(eol)
b.Write(parts[0].body)
} else {
boundary := textBoundary(msg, parts)
b.WriteString("Content-Type: multipart/mixed; boundary=\"" + boundary + "\"" + eol + eol)
for _, p := range parts {
b.WriteString("--" + boundary + eol)
writePartHeader(&b, p, eol)
b.WriteString(eol)
b.Write(p.body)
b.WriteString(eol)
}
b.WriteString("--" + boundary + "--" + eol)
}
// Longer MIME boundaries can grow an otherwise admissible snapshot.
// Do not send a body the client and retained-body store would reject.
if b.Len() > MaxBody {
return nil
}
return b.Bytes()
}
// textBoundary derives the rebuilt message's boundary from its source, so a
// retried request stores the same bytes, and avoids any part's content.
func textBoundary(msg []byte, parts []textPart) string {
sum := sha256.Sum256(msg)
for n := 0; ; n++ {
boundary := fmt.Sprintf("csm-text-%x-%d", sum[:12], n)
clash := false
for _, p := range parts {
if bytes.Contains(p.body, []byte(boundary)) {
clash = true
break
}
}
if !clash {
return boundary
}
}
}
func writePartHeader(b *bytes.Buffer, p textPart, eol string) {
if ct := p.header.Get("Content-Type"); ct != "" {
b.WriteString("Content-Type: " + ct + eol)
}
if cte := p.header.Get("Content-Transfer-Encoding"); cte != "" {
b.WriteString("Content-Transfer-Encoding: " + cte + eol)
}
}
func readHeader(head []byte, eol string) (textproto.MIMEHeader, error) {
return textproto.NewReader(bufio.NewReader(io.MultiReader(bytes.NewReader(head), strings.NewReader(eol)))).ReadMIMEHeader()
}
// keepPart says whether collectTextParts keeps a leaf part.
type keepPart func(mt string, h textproto.MIMEHeader) bool
// inlinePlainOrHTML selects seed candidates.
func inlinePlainOrHTML(mt string, h textproto.MIMEHeader) bool {
return (mt == "text/plain" || mt == "text/html") && !attachment(h)
}
// anyText selects what rspamd hashes as text, attachments included.
func anyText(mt string, _ textproto.MIMEHeader) bool {
return strings.HasPrefix(mt, "text/")
}
// collectTextParts keeps the largest limit leaf parts that keep accepts, in
// descending size and wire order for ties, and sets dropped when it leaves
// any part out. It reports false when h is not multipart or the nesting is
// too deep.
func collectTextParts(h textproto.MIMEHeader, body []byte, depth, limit int, keep keepPart, out *[]textPart, dropped *bool) bool {
mt, params, err := mime.ParseMediaType(h.Get("Content-Type"))
if err != nil || !strings.HasPrefix(mt, "multipart/") || params["boundary"] == "" {
return false
}
if attachment(h) {
*dropped = true
return true
}
r := multipart.NewReader(bytes.NewReader(body), params["boundary"])
for {
p, err := r.NextRawPart()
if err != nil {
*dropped = *dropped || err != io.EOF
return true
}
pmt := "text/plain"
// RFC 2046 gives digest children an attached-message default.
if mt == "multipart/digest" {
pmt = "message/rfc822"
}
if ct := p.Header.Get("Content-Type"); ct != "" {
if pmt, _, err = mime.ParseMediaType(ct); err != nil {
*dropped = true
continue
}
}
nested := strings.HasPrefix(pmt, "multipart/")
if nested && attachment(p.Header) || !nested && !keep(pmt, p.Header) {
*dropped = true
continue
}
raw, err := io.ReadAll(p)
if err != nil {
*dropped = true
return true
}
if nested {
if depth == maxPartDepth || !collectTextParts(p.Header, raw, depth+1, limit, keep, out, dropped) {
return false
}
continue
}
i := sort.Search(len(*out), func(i int) bool { return len((*out)[i].body) < len(raw) })
if i == limit {
*dropped = true
continue
}
if len(*out) < limit {
*out = append(*out, textPart{})
} else {
*dropped = true
}
copy((*out)[i+1:], (*out)[i:])
(*out)[i] = textPart{header: p.Header, body: raw}
}
}
func attachment(h textproto.MIMEHeader) bool {
// ParseMediaType still returns the disposition when only its parameters
// are malformed. A broken filename must not turn an attachment inline.
d, _, _ := mime.ParseMediaType(h.Get("Content-Disposition"))
return d == "attachment"
}
// splitHeader returns the header block with its last line break, the body,
// and the line break that ends the header.
func splitHeader(msg []byte) (head, body []byte, eol string, ok bool) {
lf := bytes.Index(msg, []byte("\n\n"))
crlf := bytes.Index(msg, []byte("\r\n\r\n"))
switch {
case crlf >= 0 && (lf < 0 || crlf < lf):
return msg[:crlf+2], msg[crlf+4:], "\r\n", true
case lf >= 0:
return msg[:lf+1], msg[lf+2:], "\n", true
}
return nil, nil, "", false
}
// keptFields returns head's raw fields, folded lines included, without
// Content-* and MIME-Version.
func keptFields(head []byte) []byte {
var out []byte
keep := false
for line := range bytes.Lines(head) {
if line[0] != ' ' && line[0] != '\t' {
name, _, _ := bytes.Cut(line, []byte(":"))
n := strings.ToLower(strings.TrimSpace(string(name)))
keep = !strings.HasPrefix(n, "content-") && n != "mime-version"
}
if keep {
out = append(out, line...)
}
}
return out
}
package learn
import (
"bytes"
"context"
"errors"
"fmt"
"strings"
"time"
)
// Publication limits, provisional pending benchmark measurements.
const (
PublishTurnAttempts = 3
VerifyPollEvery = 100 * time.Millisecond
VerifyDeadline = 5 * time.Second
)
// Publication errors. A failed inspection or phase commit prevents a send.
// Ledger failures after a send leave its recorded phase unresolved.
var (
ErrReadFailed = errors.New("cannot read the anchor's stored value")
ErrStateIO = errors.New("cannot record learning state")
)
// nativeClient is the subset of Client publication uses.
type nativeClient interface {
Inspect(ctx context.Context, body []byte) (Inspection, error)
InspectDigests(ctx context.Context, ds []Digest) (Inspection, error)
Add(ctx context.Context, body []byte, w int32) error
DeleteHash(ctx context.Context, d Digest) error
Sync(ctx context.Context) error
}
// Publisher drives one target at a time toward its value through native
// calls, persisting the phase before every send. Only the engine goroutine
// uses it.
type Publisher struct {
client nativeClient
ledger *Ledger
bodies *Bodies
now func() time.Time
poll time.Duration
deadline time.Duration
attempts int
audit auditState
// onUnstable observes a cluster retired because its stored body no
// longer hashes to its anchor.
onUnstable func(detail string)
}
// NewPublisher returns a publisher with the default verify and retry limits.
func NewPublisher(c nativeClient, l *Ledger, b *Bodies) *Publisher {
return &Publisher{client: c, ledger: l, bodies: b, now: time.Now, poll: VerifyPollEvery, deadline: VerifyDeadline, attempts: PublishTurnAttempts}
}
// TurnResult reports where a target stands after one turn. Done: an exact
// check showed T (or absence for T=0) and the target is idle. Uncertain: a
// mutation's outcome is unknown; the next turn starts by reading.
type TurnResult struct {
Done bool
Value int64
Phase Phase
Uncertain bool
}
// Turn runs at most the publisher's attempts of native mutations for d.
// It never computes an add from a value read before an unresolved send.
func (p *Publisher) Turn(ctx context.Context, d Digest) (TurnResult, error) {
sends := 0
for step := 0; step < 4*p.attempts; step++ {
t, err := p.target(d)
if err != nil {
return TurnResult{}, err
}
res := TurnResult{Phase: t.Phase}
if ctx.Err() != nil {
return res, ctx.Err()
}
v, present, err := p.read(ctx, d)
if err != nil {
return res, err
}
res.Value = v
reset := t.ResetPending || present && v <= 0
switch {
case t.T == 0 && !present:
return p.complete(t, 0)
case t.T > 0 && present && v == int64(t.T) && t.Phase != PhaseAddPending &&
(!reset || t.Phase == PhaseAddSent && t.ResetPending):
// A recorded reset add followed by exact T finishes the reset;
// merely entering the reset phase does not. AddPending must
// still delete a digest that reappeared before its reset add.
return p.complete(t, v)
}
switch t.Phase {
case PhaseDeletePending, PhaseDeleteSent:
if !present {
if err = p.setPhase(t, PhaseAddPending, nil); err != nil {
return res, err
}
continue
}
case PhaseAddPending:
if present {
// A late write arrived before the re-add: delete it first.
if err = p.setPhase(t, PhaseDeletePending, func(t *Target) {
t.ResetPending, t.RestoreT = true, t.T
}); err != nil {
return res, err
}
continue
}
default:
if t.T == 0 || reset || v > int64(t.T) {
if err = p.setPhase(t, PhaseDeletePending, func(t *Target) {
t.ResetPending = t.ResetPending || present && v <= 0
t.RestoreT = t.T
}); err != nil {
return res, err
}
continue
}
}
if sends == p.attempts {
return res, nil
}
sends++
var verified bool
if t.Phase == PhaseDeletePending || t.Phase == PhaseDeleteSent {
verified, err = p.sendDelete(ctx, t)
} else {
// Idle, Read and AddSent with 0 <= V < T add T-V; AddPending
// re-adds the whole target after verified absence.
verified, err = p.sendAdd(ctx, t, v)
}
if errors.Is(err, errAnchorRetired) {
continue
}
if err != nil || !verified {
var stateErr error
res, stateErr = p.result(d, res)
res.Uncertain = err == nil || errors.Is(err, ErrUncertain) && !errors.Is(err, ErrReadFailed)
if res.Uncertain {
err = nil
}
return res, errors.Join(err, stateErr)
}
}
t, err := p.target(d)
return TurnResult{Phase: t.Phase}, err
}
func (p *Publisher) result(d Digest, res TurnResult) (TurnResult, error) {
t, err := p.target(d)
if err == nil {
res.Phase = t.Phase
}
return res, err
}
func (p *Publisher) target(d Digest) (Target, error) {
var t Target
var ok bool
err := p.ledger.View(func(v LedgerView) error {
var err error
t, ok, err = v.Target(d)
return err
})
if err != nil {
return Target{}, fmt.Errorf("%w: %w", ErrStateIO, err)
}
if !ok {
return Target{}, fmt.Errorf("%w: no target for digest", ErrStateMismatch)
}
return t, nil
}
// read returns d's exact stored value; a verified miss is absence.
func (p *Publisher) read(ctx context.Context, d Digest) (int64, bool, error) {
if err := ctx.Err(); err != nil {
return 0, false, err
}
insp, err := p.client.InspectDigests(ctx, []Digest{d})
if err != nil {
return 0, false, fmt.Errorf("%w: %w", ErrReadFailed, err)
}
v, present, err := insp.ExactValue(d)
if err != nil {
return 0, false, fmt.Errorf("%w: %w", ErrReadFailed, err)
}
return v, present, nil
}
// setPhase records a phase change, refusing if the target moved since it
// was read (a new revision or target supersedes this turn's plan).
func (p *Publisher) setPhase(seen Target, phase Phase, edit func(*Target)) error {
err := p.ledger.Update(func(tx LedgerTx) error {
cur, ok, err := tx.Target(seen.Digest)
if err != nil {
return err
}
if !ok || cur.Revision != seen.Revision || cur.T != seen.T || cur.Phase != seen.Phase {
return errTargetMoved
}
cur.Phase, cur.Updated = phase, p.now().UTC()
if edit != nil {
edit(&cur)
}
return tx.PutTarget(cur)
})
if err != nil && !errors.Is(err, errTargetMoved) {
return fmt.Errorf("%w: %w", ErrStateIO, err)
}
return err
}
var errTargetMoved = errors.New("target changed during the turn")
// errAnchorUnreproducible: the stored body now inspects to other digests
// than its anchor. rspamd hashes stemmed words and stems by the detected
// language, which can differ between inspections of the same bytes.
var errAnchorUnreproducible = errors.New("the anchor body no longer hashes to its digest")
// errAnchorRetired ends a send whose cluster was retired instead; the turn
// reads the new zero target and continues.
var errAnchorRetired = errors.New("the anchor's cluster was retired")
// markSent journals that the target's current revision has a native write.
func markSent(t *Target) { t.SentRevision = t.Revision }
func (p *Publisher) complete(t Target, v int64) (TurnResult, error) {
err := p.setPhase(t, PhaseIdle, func(t *Target) {
t.ResetPending, t.RestoreT = false, 0
})
if err != nil {
return TurnResult{Phase: t.Phase, Value: v}, err
}
return TurnResult{Done: true, Value: v, Phase: PhaseIdle}, nil
}
// sendDelete journals delete_sent, deletes by full digest, flushes and
// waits for exact absence.
func (p *Publisher) sendDelete(ctx context.Context, t Target) (bool, error) {
if err := p.setPhase(t, PhaseDeleteSent, markSent); err != nil {
return false, err
}
if err := ctx.Err(); err != nil {
return false, err
}
if err := p.client.DeleteHash(ctx, t.Digest); err != nil {
if errors.Is(err, ErrRejected) {
sent := t
sent.Phase = PhaseDeleteSent
if perr := p.setPhase(sent, PhaseDeletePending, func(t *Target) {
t.ResetPending, t.RestoreT = true, t.T
}); perr != nil {
return false, errors.Join(err, perr)
}
}
return false, err
}
return p.verify(ctx, t.Digest, func(_ int64, present bool) bool { return !present })
}
// sendAdd checks the anchor body, journals add_sent and sends one add: T-V
// on the plain path, the whole T after a verified deletion.
func (p *Publisher) sendAdd(ctx context.Context, t Target, v int64) (bool, error) {
resetAdd := t.Phase == PhaseAddPending
weight := t.T
if !resetAdd {
// The caller sends T-V only for 0 <= V < T; anything else is never
// a weight.
if v < 0 || v >= int64(t.T) {
return false, fmt.Errorf("%w: stored value outside the add range", ErrStateMismatch)
}
weight = t.T - int32(v) // #nosec G115 -- 0 <= v < t.T, so v fits in int32
}
body, err := p.anchorBody(ctx, t)
if errors.Is(err, errAnchorUnreproducible) {
return false, p.retireUnstable(t, err)
}
if err != nil {
return false, err
}
if err := p.setPhase(t, PhaseAddSent, markSent); err != nil {
return false, err
}
sent := t
sent.Phase = PhaseAddSent
if err := ctx.Err(); err != nil {
return false, err
}
if err := p.client.Add(ctx, body, weight); err != nil {
if errors.Is(err, ErrRejected) {
// An observed backend write failure: rebuild the anchor rather
// than trust whatever part of it storage kept.
next := PhaseDeletePending
if resetAdd {
next = PhaseAddPending
}
if perr := p.setPhase(sent, next, func(t *Target) {
t.ResetPending, t.RestoreT = true, t.T
}); perr != nil {
return false, errors.Join(err, perr)
}
}
return false, err
}
return p.verify(ctx, t.Digest, func(v int64, present bool) bool { return present && v >= int64(t.T) })
}
// anchorBody verifies the stored bytes and returns the text-only message
// only if the current hash generation still yields the recorded anchor.
func (p *Publisher) anchorBody(ctx context.Context, t Target) ([]byte, error) {
var c Cluster
var ok bool
if err := p.ledger.View(func(v LedgerView) error {
var err error
c, ok, err = v.Cluster(t.ClusterID)
return err
}); err != nil {
return nil, fmt.Errorf("%w: %w", ErrStateIO, err)
}
if !ok || c.Anchor != t.Digest {
return nil, fmt.Errorf("%w: target has no cluster for its anchor", ErrStateMismatch)
}
if err := ctx.Err(); err != nil {
return nil, err
}
body, err := p.bodies.Read(c.ID, c.BodyLen, c.BodySHA)
if err != nil {
return nil, err
}
if err = ctx.Err(); err != nil {
return nil, err
}
// A clean inspection proves the stored bytes publish only the anchor.
// Bodies kept before text-only learning may carry an attachment that
// rspamd now hashes on its own; their text-only form, verified the same
// way, is published instead. The stored body and its ledger record stay.
// Bodies rspamd cannot inspect as-is (a PDF with scripts) also fall
// back to their text-only form.
err = p.yieldsAnchor(ctx, body, t.Digest)
if err == nil {
return body, nil
}
text := textOnlyMessage(body)
if text == nil || bytes.Equal(text, body) {
return nil, err
}
textErr := p.yieldsAnchor(ctx, text, t.Digest)
if textErr == nil {
return text, nil
}
// Only answered inspections prove the anchor is unreproducible: a failed
// inspection of the stored bytes says nothing about their digest.
if errors.Is(textErr, errAnchorUnreproducible) && !errors.Is(err, errAnchorUnreproducible) {
return nil, err
}
return nil, textErr
}
// yieldsAnchor inspects body and checks it hashes to exactly d.
func (p *Publisher) yieldsAnchor(ctx context.Context, body []byte, d Digest) error {
insp, err := p.client.Inspect(ctx, body)
if err != nil {
return fmt.Errorf("%w: %w", ErrReadFailed, err)
}
if insp.Distinct != 1 || insp.Results[0].Queried != d {
got := make([]string, 0, len(insp.Results))
for _, r := range insp.Results {
got = append(got, shortDigest(r.Queried))
}
return fmt.Errorf("%w: now %s", errAnchorUnreproducible, strings.Join(got, " "))
}
return nil
}
// retireUnstable retires the cluster of a target whose stored body no
// longer reproduces its anchor: the body can never publish or rebuild it.
// Only that cluster stops; its reports end without a vote. It returns
// errAnchorRetired when the turn should read the target again.
func (p *Publisher) retireUnstable(seen Target, cause error) error {
retired := false
err := p.ledger.Update(func(tx LedgerTx) error {
cur, ok, err := tx.Target(seen.Digest)
if err != nil || !ok || cur.Revision != seen.Revision || cur.T != seen.T || cur.Phase != seen.Phase {
return err
}
c, ok, err := tx.Cluster(cur.ClusterID)
if err != nil || !ok || c.Retired || c.Anchor != cur.Digest {
return err
}
retired = true
return supersede(tx, c, p.now().UTC())
})
if err != nil {
return fmt.Errorf("%w: %w", ErrStateIO, err)
}
if retired && p.onUnstable != nil {
p.onUnstable(fmt.Sprintf("cluster %d retired: anchor %s, %v", seen.ClusterID, shortDigest(seen.Digest), cause))
}
return errAnchorRetired
}
// verify flushes and polls exact checks until done reports true or the
// deadline passes. A failed flush or read leaves the outcome open.
func (p *Publisher) verify(ctx context.Context, d Digest, done func(v int64, present bool) bool) (bool, error) {
if ctx.Err() != nil {
return false, nil
}
if err := p.client.Sync(ctx); err != nil {
return false, nil
}
ctx, cancel := context.WithTimeout(ctx, p.deadline)
defer cancel()
for {
if v, present, err := p.read(ctx, d); err == nil && done(v, present) {
return true, nil
}
select {
case <-ctx.Done():
return false, nil
case <-time.After(p.poll):
}
}
}
package learn
import (
"encoding/json"
"fmt"
"time"
"github.com/pidginhost/csm/internal/store"
)
// Stats is the learner's state for metrics and health. Counts come from
// one ledger read; the audit fields from the engine's last turns.
type Stats struct {
LiveClusters int
ZeroTargets int
UncertainIntents int
BodyBytes int64
OldestUnchecked time.Time
AuditCompleted time.Time
// Reconstructed counts missing live anchors newly scheduled for repair
// since the engine started.
Reconstructed uint64
}
// Stats reads the ledger counts and the engine's audit progress. It is
// safe from any goroutine.
func (e *Engine) Stats() (Stats, error) {
e.mu.Lock()
s := Stats{OldestUnchecked: e.statOldest, AuditCompleted: e.statAudit, Reconstructed: e.statRepaired}
e.mu.Unlock()
err := e.cfg.Ledger.View(func(v LedgerView) error {
if err := v.tx.ForEach(store.LearnClustersBucket, func(_, raw []byte) error {
var c Cluster
if err := json.Unmarshal(raw, &c); err != nil {
return err
}
if !c.Retired {
s.LiveClusters++
}
s.BodyBytes += c.BodyLen
return nil
}); err != nil {
return err
}
return v.tx.ForEach(store.LearnTargetsBucket, func(_, raw []byte) error {
var t Target
if err := json.Unmarshal(raw, &t); err != nil {
return err
}
if t.T == 0 {
s.ZeroTargets++
}
if t.Phase == PhaseAddSent || t.Phase == PhaseDeleteSent {
s.UncertainIntents++
}
return nil
})
})
if err != nil {
return Stats{}, fmt.Errorf("%w: %w", ErrStateIO, err)
}
return s, nil
}
package learn
import (
"encoding/json"
"errors"
"fmt"
"github.com/pidginhost/csm/internal/store"
)
// Status is what the ledger holds for one operator request ID: its
// recorded outcome and, for clusters still publishing, their phase.
type Status struct {
RequestID string
Action string
Found, Done, ExitOK bool
Outcome string
Digests []Digest
Clusters []StatusCluster
}
// StatusCluster is one cluster's recorded result; Phase is set while its
// publication is unfinished.
type StatusCluster struct {
ClusterID uint64
Result string
Phase Phase
}
// RequestStatus reads the given request IDs, or every unfinished request
// when ids is empty. It only reads, so it is safe beside a running
// engine and when none runs.
func RequestStatus(l *Ledger, ids []string) ([]Status, error) {
var out []Status
err := l.View(func(v LedgerView) error {
if len(ids) == 0 {
if err := v.tx.ForEach(store.LearnOpsBucket, func(k, data []byte) error {
var o OpRecord
if err := json.Unmarshal(data, &o); err != nil {
return errors.New("learning ledger request record is unreadable")
}
if !o.Done {
ids = append(ids, string(k))
}
return nil
}); err != nil {
return err
}
}
for _, id := range ids {
s, err := requestStatus(v, id)
if err != nil {
return err
}
out = append(out, s)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrStateIO, err)
}
return out, nil
}
func requestStatus(v LedgerView, id string) (Status, error) {
s := Status{RequestID: id}
o, ok, err := v.Op(id)
if err != nil || !ok {
return s, err
}
s.Action, s.Found, s.Done = o.Action, true, o.Done
if o.Outcome == nil {
return s, nil
}
s.Outcome, s.ExitOK, s.Digests = o.Outcome.Outcome, o.Outcome.ExitOK, o.Outcome.Digests
intent, _, err := v.Intent(id)
if err != nil {
return s, err
}
for _, c := range o.Outcome.Clusters {
sc := StatusCluster{ClusterID: c.ClusterID, Result: c.Result}
if c.Result == ResultPending {
for _, ic := range intent.Clusters {
if ic.ClusterID != c.ClusterID || ic.Done {
continue
}
t, found, err := v.Target(ic.Anchor)
if err != nil {
return s, err
}
if found {
sc.Phase = t.Phase
}
}
}
s.Clusters = append(s.Clusters, sc)
}
return s, nil
}
package antispam
import (
_ "embed"
"errors"
"fmt"
"path"
"regexp"
"strings"
)
// RspamdLuaDir holds local Lua plugins rspamd loads at startup.
const RspamdLuaDir = "/etc/rspamd/lua.local.d"
// Learning files. The check and worker files hold secrets.
var (
LearningCheckPath = path.Join(RspamdOverrideDir, "fuzzy_check.conf")
LearningWorkerPath = path.Join(RspamdOverrideDir, "worker-fuzzy.inc")
LearningGroupPath = path.Join(RspamdLocalDir, "fuzzy_group.conf")
LearningLuaPath = path.Join(RspamdLuaDir, "csm_learning.lua")
)
// LuaManagedHeader is the first line of the Lua file CSM owns.
const LuaManagedHeader = "-- csm-managed-version: 1"
// LearningPlugin is the inspection plugin CSM installs as LearningLuaPath.
//
//go:embed learning_plugin.lua
var LearningPlugin string
// Values the learning rules and worker pin.
const (
learningServer = "127.0.0.1:11335"
learningMaxScore = "18.0"
learningGroupCap = "10.0"
learningWeight = "10.0"
learningExpire = "90d"
// LearningFlag is the only fuzzy flag CSM maps.
LearningFlag = 1
)
var learningPrefixPattern = regexp.MustCompile(`^csmfz_[0-9a-f]{32}$`)
var learningHashKeyPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
// LearningPaths lists the learning files in write order.
func LearningPaths() []string {
return []string{LearningCheckPath, LearningWorkerPath, LearningGroupPath, LearningLuaPath}
}
// validateLearningKeys checks every key that reaches a rendered file, so a
// quote or newline can never leave its UCL string.
func validateLearningKeys(k LearningKeys) error {
if !learningHashKeyPattern.MatchString(k.Fuzzy.Reveal()) || !learningHashKeyPattern.MatchString(k.Shingles.Reveal()) {
return errors.New("learning hash keys are not loaded")
}
if !learningPrefixPattern.MatchString(k.Prefix) {
return errors.New("learning prefix is not loaded")
}
if err := k.Read.Validate(); err != nil {
return fmt.Errorf("learning read keypair: %w", err)
}
if err := k.Write.Validate(); err != nil {
return fmt.Errorf("learning write keypair: %w", err)
}
return nil
}
// RenderLearning returns the learning files keyed by path, or none when
// learning is off. The keys come from s.LearningKeys, which the caller loads.
func RenderLearning(s Settings) (map[string]string, error) {
if err := s.Validate(); err != nil {
return nil, err
}
if !s.Learning {
return map[string]string{}, nil
}
k := s.LearningKeys
if err := validateLearningKeys(k); err != nil {
return nil, err
}
return map[string]string{
LearningCheckPath: renderLearningRules(k),
LearningWorkerPath: renderLearningWorker(k, s.LearningRedisSocket),
LearningGroupPath: ManagedHeader + "\nmax_score = " + learningGroupCap + ";\nsymbols {\n" +
" CSM_FUZZY_SPAM { weight = " + learningWeight + "; }\n" +
" CSM_FUZZY_LEARN { weight = 0.0; }\n}\n",
LearningLuaPath: LearningPlugin,
}, nil
}
// renderLearningRules writes the scan rule csm and the learning rule
// csm_learn. Both share the hash keys and options, so they compute the same
// digests; only csm_learn carries the write key and only csm scores.
func renderLearningRules(k LearningKeys) string {
common := func(b *strings.Builder) {
fmt.Fprintf(b, " servers = %q;\n", learningServer)
fmt.Fprintf(b, " fuzzy_key = %q;\n", k.Fuzzy.Reveal())
fmt.Fprintf(b, " fuzzy_shingles_key = %q;\n", k.Shingles.Reveal())
fmt.Fprintf(b, " read_encryption_key = %q;\n", k.Read.Public)
b.WriteString(" skip_unknown = true;\n")
b.WriteString(" max_score = " + learningMaxScore + ";\n")
fmt.Fprintf(b, " min_length = %d;\n", LearningMinLength)
b.WriteString(" text_hashes = true;\n text_shingles = true;\n html_shingles = false;\n")
b.WriteString(" skip_images = true;\n mime_types = [];\n scan_archives = false;\n")
}
var b strings.Builder
b.WriteString(ManagedHeader + "\nrule \"csm\" {\n")
common(&b)
b.WriteString(" read_only = true;\n")
fmt.Fprintf(&b, " fuzzy_map = { CSM_FUZZY_SPAM { flag = %d; max_score = %s; } }\n}\n", LearningFlag, learningMaxScore)
b.WriteString("rule \"csm_learn\" {\n")
common(&b)
fmt.Fprintf(&b, " write_encryption_key = %q;\n", k.Write.Public)
b.WriteString(" mode = \"write_only\";\n retransmits = 0;\n")
fmt.Fprintf(&b, " fuzzy_map = { CSM_FUZZY_LEARN { flag = %d; max_score = %s; } }\n}\n", LearningFlag, learningMaxScore)
return b.String()
}
// renderLearningWorker writes the loopback fuzzy worker: encrypted traffic
// only, no address-based updates, and one keypair per operation set. A
// dedicated Redis is selected by its socket, with an empty password: rspamd
// fills a worker's missing password from the global redis section and would
// send that to the dedicated socket. Without one the worker keeps rspamd's
// Redis settings.
func renderLearningWorker(k LearningKeys, redisSocket string) string {
var b strings.Builder
b.WriteString(ManagedHeader + "\ncount = 1;\n")
fmt.Fprintf(&b, "bind_socket = %q;\n", learningServer)
b.WriteString("backend = \"redis\";\n")
if redisSocket != "" {
fmt.Fprintf(&b, "servers = %q;\npassword = \"\";\n", redisSocket)
}
fmt.Fprintf(&b, "prefix = %q;\n", k.Prefix)
b.WriteString("expire = " + learningExpire + ";\nallow_update = [];\nencrypted_only = true;\n")
for _, kp := range []struct {
pair Keypair
ops string
}{{k.Read, `"read"`}, {k.Write, `"read", "write", "delete"`}} {
fmt.Fprintf(&b, "keypair {\n privkey = %q;\n pubkey = %q;\n extensions { allowed_ops = [%s]; }\n}\n",
kp.pair.Private.Reveal(), kp.pair.Public, kp.ops)
}
return b.String()
}
package antispam
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/antispam/dovecot"
)
// CodeLearning is the problem code of every learning finding.
const CodeLearning = "learning"
// Learning reason keys. Plans B and C add the learner's own.
const (
ReasonUnsupported = "unsupported"
ReasonUnsafe = "unsafe_prerequisite"
ReasonStateMismatch = "state_mismatch"
ReasonConfigConflict = "config_conflict"
ReasonApplyFailed = "apply_failed"
ReasonBackendUnavailable = "backend_unavailable"
ReasonWorkerAbsent = "worker_absent"
ReasonRedisCapacity = "redis_capacity"
ReasonRedisPersistence = "redis_persistence"
ReasonRoundcube = "roundcube_unavailable"
ReasonRemovalIncomplete = "removal_incomplete"
// The learner's own.
ReasonAnchorDrift = "anchor_drift"
ReasonMutationUncertain = "mutation_uncertain"
ReasonAnchorBodyInvalid = "anchor_body_invalid"
ReasonStateIO = "state_io"
ReasonAuditOverdue = "audit_overdue"
ReasonRequestErrors = "request_errors"
ReasonBackendRebuilding = "backend_rebuilding"
ReasonAccountCapacity = "account_capacity"
// ReasonMailboxesSkipped: Dovecot mailboxes learning does not watch on
// a Postfix host, counted by reason.
ReasonMailboxesSkipped = "mailboxes_skipped"
)
// LearningGate says what learning may do this pass. Install: CSM's learning
// files may be installed. Publish: the worker and Redis can take new
// hashes. Roundcube: the Junk-button include may be installed.
type LearningGate struct {
Install bool
Publish bool
Roundcube bool
}
// stockFuzzyRule is the public rule rspamd ships; CSM's override replaces it
// on purpose.
const stockFuzzyRule = "rspamd.com"
var (
learningRspamdVersion = regexp.MustCompile(`^4\.2\.\d+$`)
learningRedisVersion = regexp.MustCompile(`^(5\.0|6\.2|7\.\d+)\.\d+$`)
)
// learningCheck collects learning problems, one per reason, in the order
// found.
type learningCheck struct {
problems []Problem
seen map[string]bool
}
func (c *learningCheck) add(reason, detail string) {
if c.seen[reason] {
return
}
if c.seen == nil {
c.seen = map[string]bool{}
}
c.seen[reason] = true
c.problems = append(c.problems, Problem{Code: CodeLearning, Severity: Warning, Key: reason, Detail: detail})
}
func (c *learningCheck) has(reasons ...string) bool {
for _, r := range reasons {
if c.seen[r] {
return true
}
}
return false
}
// ResolveLearning decides from host facts what learning may do. A definite
// failure (unsupported version, isolation, key state, configuration
// conflict) turns learning off so the pass reconciles core alone. A
// temporary probe failure keeps whatever is installed and stops new
// publications. Warnings keep learning on.
func ResolveLearning(s Settings, f HostFacts) (Settings, LearningGate, []Problem) {
if !s.Learning {
s.Roundcube = false
return s, LearningGate{}, nil
}
return resolveLearning(s, f)
}
// resolveLearning judges requested learning.
func resolveLearning(s Settings, f HostFacts) (Settings, LearningGate, []Problem) {
var c learningCheck
installed := learningInstalled(f)
if f.LearningKeysErr != nil {
c.add(ReasonStateMismatch, "the learning keys are missing or invalid; CSM never replaces part of a key set")
}
if f.LearnerMismatch {
c.add(ReasonStateMismatch, "learning state does not match its storage; new reports are held and nothing is adopted or deleted")
}
checkLearningDovecot(&c, s, f)
checkLearningVersions(&c, f)
checkLearningController(&c, f)
checkLearningRedis(&c, s, f)
checkLearningLogs(&c, f)
checkLearningConflicts(&c, s, f)
if f.FuzzyListenErr != nil {
c.add(ReasonBackendUnavailable, "CSM cannot read the UDP socket table to find the fuzzy worker")
}
if f.FuzzyListenForeign {
c.add(ReasonUnsafe, "a process of another user listens on the fuzzy worker's port 11335")
}
gate := LearningGate{}
switch {
case c.has(ReasonStateMismatch, ReasonUnsupported, ReasonUnsafe, ReasonConfigConflict):
case c.has(ReasonBackendUnavailable):
gate.Install = installed
// An outage keeps what is installed, including its Redis: a new
// redis_socket is used once a pass has checked it.
s.LearningRedisSocket = installedLearningRedis(f)
default:
gate.Install, gate.Publish = true, true
}
if gate.Install && gate.Publish {
if installed && !f.FuzzyListening {
c.add(ReasonWorkerAbsent, "the fuzzy worker is not listening on 127.0.0.1:11335; restart rspamd")
gate.Publish = false
}
}
if gate.Install && gate.Publish && RedisGuard(f.Redis.Used, f.Redis.MaxMemory, f.RedisGuardPaused) {
c.add(ReasonRedisCapacity, "Redis is at the learning memory guard; new hashes wait until usage falls below 70%")
gate.Publish = false
}
// The Junk-button include is for cPanel's Roundcube.
if gate.Install && f.Stack != StackPostfix {
gate.Roundcube = roundcubeUsable(&c, f)
}
s.Learning, s.Roundcube = gate.Install, gate.Roundcube
return s, gate, c.problems
}
// RedisGuard reports whether new publications pause: at 75% of maxmemory,
// staying paused until usage falls below 70%. No limit always pauses. The
// reserve is headroom for synchronization peaks, not a fixed message count:
// a measured peak was about four times a batch's steady growth.
func RedisGuard(used, max uint64, paused bool) bool {
if max == 0 {
return true
}
// Compare against the ceiling of the threshold, without overflowing
// uint64 or rounding away a byte near the boundary.
numerator, denominator := uint64(3), uint64(4)
if paused {
numerator, denominator = 7, 10
}
limit := max/denominator*numerator + (max%denominator*numerator+denominator-1)/denominator
return used >= limit
}
// learningInstalled reports whether CSM's learning worker file is on the
// host, the file whose presence makes CSM's scoring active.
func learningInstalled(f HostFacts) bool {
for _, t := range f.Targets {
if t.Path == LearningWorkerPath {
return !t.Missing && t.Err == nil && ManagedFor(t.Kind, t.Content)
}
}
return false
}
// installedLearningRedis returns the dedicated Redis socket CSM's installed
// fuzzy worker selects, "" when it uses rspamd's Redis or none is installed.
func installedLearningRedis(f HostFacts) string {
if !learningInstalled(f) {
return ""
}
for _, t := range f.Targets {
if t.Path != LearningWorkerPath {
continue
}
for _, line := range strings.Split(t.Content, "\n") {
if quoted, ok := strings.CutPrefix(line, "servers = "); ok {
// CSM rendered it with %q from a validated socket path.
socket, _ := strconv.Unquote(strings.TrimSuffix(quoted, ";"))
return socket
}
}
}
return ""
}
// DovecotProblem is the learning problem of a Dovecot listing that failed:
// definite when Dovecot cannot serve learning as configured, temporary
// otherwise. Its detail is fixed text, never command output.
func DovecotProblem(err error) Problem {
p := Problem{Code: CodeLearning, Severity: Warning, Key: ReasonBackendUnavailable,
Detail: "CSM cannot list Dovecot's mailboxes; learning waits for the next check"}
switch {
case errors.Is(err, dovecot.ErrCannotIterate):
p.Key, p.Detail = ReasonUnsupported, "Dovecot cannot list its users; learning needs a userdb that supports iteration"
case errors.Is(err, dovecot.ErrDovecot24):
p.Key, p.Detail = ReasonUnsupported, "Dovecot 2.4 mail settings are not supported yet"
case errors.Is(err, context.DeadlineExceeded):
p.Detail = "listing Dovecot's mailboxes timed out; learning waits for the next check"
}
return p
}
// checkLearningDovecot judges what Dovecot told a Postfix host's pass: a
// failed listing, mail stored as rspamd's or Redis's uid, which tenants'
// sieve scripts and mail processes run as (L12), and the mailboxes learning
// skips. Exempt mailboxes are not watched, so their skips are no problem.
func checkLearningDovecot(c *learningCheck, s Settings, f HostFacts) {
if f.Stack != StackPostfix {
return
}
d := f.Dovecot
if d.Err != nil {
p := DovecotProblem(d.Err)
c.add(p.Key, p.Detail)
return
}
for _, uid := range d.UIDs {
// Mail uids are never 0, so an unread socket owner never matches.
if uid == f.ControllerHost.RspamdUID || uid == f.Redis.Socket.Socket.UID {
c.add(ReasonUnsafe, "Dovecot stores mail as rspamd's or Redis's user; learning needs a separate mail user")
break
}
}
skipped := map[string]int{}
total := 0
for _, sk := range d.Skips {
if !IsExempt(s.Exempt, sk.User) {
skipped[sk.Reason]++
total++
}
}
if total == 0 {
return
}
var parts []string
for _, reason := range slices.Sorted(maps.Keys(skipped)) {
parts = append(parts, fmt.Sprintf("%s %d", reason, skipped[reason]))
}
c.add(ReasonMailboxesSkipped, fmt.Sprintf("mailboxes not watched for learning: %d (%s)", total, strings.Join(parts, ", ")))
}
func checkLearningVersions(c *learningCheck, f HostFacts) {
if f.RspamdVersion.Err != nil {
c.add(ReasonBackendUnavailable, "rspamd's version is unknown")
} else if v, _, err := ParseRspamdVersion(f.RspamdVersion.Stdout); err != nil {
c.add(ReasonBackendUnavailable, "rspamd's version is unknown")
} else if !learningRspamdVersion.MatchString(v) {
c.add(ReasonUnsupported, "learning is qualified on rspamd 4.2 only")
}
checkLearningRedisVersion(c, f.Redis)
}
func checkLearningRedisVersion(c *learningCheck, r RedisFacts) {
if r.Version == "" && r.Err == nil && r.Unix {
c.add(ReasonBackendUnavailable, "Redis's version is unknown")
} else if r.Version != "" && !learningRedisVersion.MatchString(r.Version) {
c.add(ReasonUnsupported, "learning is qualified on Redis 5.0, 6.2 and 7 only")
}
}
// checkLearningController requires a clean controller, one private
// Unix controller socket owned by rspamd with mode 0600, and a private
// rspamadm control socket.
func checkLearningController(c *learningCheck, f HostFacts) {
// The control socket is independent of controller config discovery. A
// failed controller probe must not mask a known unsafe control socket.
checkLearningControlSocket(c, f)
if f.ControllerUnverifiable != nil {
c.add(ReasonUnsafe, "CSM cannot verify rspamd's controller configuration")
return
}
if f.ControllerErr != nil {
c.add(ReasonBackendUnavailable, "CSM cannot read rspamd's controller configuration")
}
host := f.ControllerHost
if host.RspamdUID < 0 {
c.add(ReasonBackendUnavailable, "rspamd's service user is unknown")
}
unknownSocket := false
for _, l := range f.Controller.Listeners {
if !l.Unix {
if tcpExposure(f.Controller, host) != "" {
c.add(ReasonUnsafe, "rspamd's controller is reachable by other users; see antispam_controller_open")
}
continue
}
if host.RspamdUID < 0 {
continue
}
s, ok := host.Sockets[l.Path]
if !ok || s.Err != nil || s.Missing {
unknownSocket = true
c.add(ReasonBackendUnavailable, "CSM cannot check rspamd's controller socket")
continue
}
if unixExposure(l, host) != "" {
c.add(ReasonUnsafe, "rspamd's controller is reachable by other users; see antispam_controller_open")
}
}
private := len(privateControllerSockets(f))
if private > 1 || private == 0 && !unknownSocket && f.ControllerErr == nil && host.RspamdUID >= 0 {
c.add(ReasonUnsafe, "learning needs exactly one private controller socket owned by rspamd with mode 0600")
}
// The controller trusts rspamd's uid, so no tenant may hold it. Not
// knowing only pauses learning: creating an alias takes root (P6).
switch {
case f.RspamdUIDShared:
c.add(ReasonUnsafe, "rspamd's uid is shared with a user that can run tenant code")
case f.RspamdUIDErr != nil:
c.add(ReasonBackendUnavailable, "CSM cannot rule out that another user shares rspamd's uid; learning waits for the next check")
}
}
func learningSocketProtected(l ControllerListener, s SocketFacts, host ControllerHostFacts) bool {
if s.Socket.ExtACL || len(s.Parents) == 0 || !directSocketPath(l, s) {
return false
}
for _, p := range s.Parents {
if p.ExtACL || !p.Mode.IsDir() || p.UID != 0 && p.UID != host.RspamdUID {
return false
}
}
return unixExposure(l, host) == ""
}
func checkLearningControlSocket(c *learningCheck, f HostFacts) {
host := f.ControllerHost
// The control socket gets the same checks as the controller socket:
// root or rspamd's, mode 0600, no ACL, protected directories.
cs := f.ControlSocket
switch {
case cs.Err != nil || cs.Missing:
c.add(ReasonBackendUnavailable, "CSM cannot check rspamd's control socket")
case cs.Socket.Mode.Perm()&0o077 != 0, cs.Socket.ExtACL:
c.add(ReasonUnsafe, "rspamd's control socket is not private to root and rspamd")
case host.RspamdUID < 0:
c.add(ReasonBackendUnavailable, "rspamd's service user is unknown")
case cs.Socket.UID != 0 && cs.Socket.UID != host.RspamdUID,
!learningSocketProtected(ControllerListener{Unix: true, Path: cs.Socket.Path}, cs, withSocket(host, cs.Socket.Path, cs)):
c.add(ReasonUnsafe, "rspamd's control socket is not private to root and rspamd")
}
}
// privateControllerSockets lists the controller's Unix sockets that only
// root and rspamd can use: owned by rspamd, mode 0600, behind protected
// directories.
func privateControllerSockets(f HostFacts) []string {
host := f.ControllerHost
var out []string
for _, l := range f.Controller.Listeners {
if !l.Unix || host.RspamdUID < 0 {
continue
}
s, ok := host.Sockets[l.Path]
if ok && s.Err == nil && !s.Missing && s.Socket.UID == host.RspamdUID &&
s.Socket.Mode.Perm() == 0o600 && learningSocketProtected(l, s, host) {
out = append(out, l.Path)
}
}
return out
}
// LearnerEndpoint returns the private controller socket and the uid that
// must serve it, for the learner's native calls.
func LearnerEndpoint(f HostFacts) (socket string, uid int, ok bool) {
socket, ok = PrivateControllerSocket(f)
if !ok || f.ControllerHost.RspamdUID < 0 {
return "", 0, false
}
return socket, f.ControllerHost.RspamdUID, true
}
// PrivateControllerSocket returns the one private controller socket learning
// talks to, or false when there is none or more than one.
func PrivateControllerSocket(f HostFacts) (string, bool) {
paths := privateControllerSockets(f)
if len(paths) != 1 {
return "", false
}
return paths[0], true
}
func withSocket(host ControllerHostFacts, p string, s SocketFacts) ControllerHostFacts {
sockets := make(map[string]SocketFacts, len(host.Sockets)+1)
for k, v := range host.Sockets {
sockets[k] = v
}
sockets[p] = s
host.Sockets = sockets
return host
}
// checkLearningRedis requires a socket-only Redis no tenant can reach, with
// a finite maxmemory and noeviction. On Postfix it must be a
// Redis dedicated to learning: rspamd's own serves tenants'
// modules too.
func checkLearningRedis(c *learningCheck, s Settings, f HostFacts) {
switch {
case f.Stack == StackPostfix && s.LearningRedisSocket == "":
c.add(ReasonUnsupported, "learning on Postfix needs a dedicated Redis: set email_protection.antispam.learning.redis_socket")
case f.LearningDumps.Err != nil:
c.add(ReasonBackendUnavailable, "CSM cannot read rspamd's loaded configuration")
default:
checkLearningRedisEndpoint(c, f)
}
}
// CheckLearningRedis rechecks s's dedicated learning Redis for the worker,
// which applies under its lock after the pass judged it: the socket can be
// replaced or opened in between. A definite failure is an error. So is an
// outage, unless the installed worker already uses that socket: a worker
// file never names a Redis no check has passed.
func CheckLearningRedis(s Settings, f HostFacts) error {
var c learningCheck
checkLearningDovecot(&c, s, f)
checkLearningRedisVersion(&c, f.Redis)
checkLearningRedisEndpoint(&c, f)
switching := installedLearningRedis(f) != s.LearningRedisSocket
for _, p := range c.problems {
if p.Key == ReasonUnsafe || p.Key == ReasonUnsupported || switching && p.Key == ReasonBackendUnavailable {
return errors.New(p.Detail)
}
}
return nil
}
func checkLearningRedisEndpoint(c *learningCheck, f HostFacts) {
r := f.Redis
switch {
case r.Endpoint == "":
c.add(ReasonUnsafe, "CSM cannot identify the one Redis server fuzzy storage uses")
return
case !r.Unix:
c.add(ReasonUnsafe, "fuzzy storage uses Redis over TCP; learning needs a Unix socket only")
return
case r.Inherits != nil:
c.add(ReasonUnsafe, "rspamd's global redis settings would reach the learning Redis: "+r.Inherits.Error())
return
case r.Unprotected:
c.add(ReasonUnsafe, "the Redis socket's owner, directories or peer do not prove that Redis serves it")
return
case r.Socket.Err != nil || r.Socket.Missing:
c.add(ReasonBackendUnavailable, "the Redis socket cannot be checked")
return
case r.Shared:
c.add(ReasonUnsafe, "the learning Redis is shared with another rspamd module")
return
}
if r.SharedErr != nil {
c.add(ReasonBackendUnavailable, "CSM cannot tell whether another rspamd module uses the learning Redis")
}
// A protected socket as well as protected directories:
// no access for others and no ACL CSM cannot evaluate.
if r.Socket.Socket.Mode.Perm()&0o007 != 0 || r.Socket.Socket.ExtACL {
c.add(ReasonUnsafe, "the Redis socket is open to other users or carries extra ACL entries")
return
}
if why := redisExposure(r, f.ControllerHost.RspamdUser, f.AccountNames, f.AccountNamesErr); why != "" {
if f.Stack == StackPostfix && (f.AccountNamesErr != nil || r.GroupMembers == nil) {
c.add(ReasonBackendUnavailable, "CSM cannot check the Redis socket's account and group membership")
} else {
c.add(ReasonUnsafe, why)
return
}
}
// Successful CONFIG reads remain evidence even when a later INFO
// request fails. Definite exposure takes precedence over that outage.
switch {
case r.Port != "" && r.Port != "0":
c.add(ReasonUnsafe, "Redis also listens on TCP; set port 0")
case r.TLSPort != "" && r.TLSPort != "0":
c.add(ReasonUnsafe, "Redis also listens on TLS TCP; set tls-port 0")
case r.MaxMemoryKnown && r.MaxMemory == 0:
c.add(ReasonUnsafe, "Redis has no maxmemory limit")
case r.Policy != "" && r.Policy != "noeviction":
c.add(ReasonUnsafe, "Redis can evict keys; set maxmemory-policy noeviction")
}
if r.Err != nil {
c.add(ReasonBackendUnavailable, "Redis did not answer CSM's read-only checks")
return
}
switch {
case r.Port != "0":
c.add(ReasonUnsafe, "Redis also listens on TCP; set port 0")
case r.TLSPort != "" && r.TLSPort != "0":
c.add(ReasonUnsafe, "Redis also listens on TLS over TCP; set tls-port 0")
case r.MaxMemory == 0:
c.add(ReasonUnsafe, "Redis has no maxmemory limit")
case r.Policy != "noeviction":
c.add(ReasonUnsafe, "Redis can evict keys; set maxmemory-policy noeviction")
case r.AppendOnly != "yes" && (!r.SaveKnown || r.Save == ""):
c.add(ReasonRedisPersistence, "Redis keeps no data across a restart; learned hashes are rebuilt from stored anchors")
}
}
// redisExposure judges the Redis socket like a controller socket: its owner
// and root are trusted, and a group member counts as another user unless it
// is root, redis or rspamd's service user. Shared users such as nobody can
// run tenant code, so only these names are trusted, and redis or rspamd's
// user only while the untrusted users are known and none carries the name.
// Unknown group members count as other users.
func redisExposure(r RedisFacts, rspamdUser string, untrusted map[string]bool, untrustedErr error) string {
owner := r.Socket.Socket.UID
host := ControllerHostFacts{Sockets: map[string]SocketFacts{r.Endpoint: r.Socket}, RspamdUID: owner, GroupMembers: map[int][]string{}}
trusted := func(m string) bool {
if m == "root" {
return true
}
if untrustedErr != nil || untrusted[m] {
return false
}
return m == "redis" || (rspamdUser != "" && m == rspamdUser)
}
if r.GroupMembers != nil {
others := []string{}
for _, m := range r.GroupMembers {
if !trusted(m) {
others = append(others, m)
}
}
host.GroupMembers[r.Socket.Socket.GID] = others
}
why := unixExposure(ControllerListener{Unix: true, Path: r.Endpoint}, host)
if why == "" {
return ""
}
return fmt.Sprintf("the Redis socket: %s", why)
}
// checkLearningLogs requires rspamd's log to be unreadable to tenants:
// file permissions, or CageFS confining every account with no rspamd
// log in its skeleton.
func checkLearningLogs(c *learningCheck, f HostFacts) {
exposed, _ := f.RspamdLog.exposed(f.Socket.RspamdUID, func(gid int) bool {
return !f.ControllerHost.groupHasOthers(gid)
})
if !exposed {
return
}
cage := f.CageFS
if cage.Err == nil && cage.Enabled && cage.DisabledUsers == 0 && !cage.SkeletonHasRspamdLog && cageFSProtectsLog(cage, f.RspamdLog) {
return
}
if f.RspamdLog.Err != nil || cage.Err != nil {
c.add(ReasonBackendUnavailable, "CSM cannot check who can read rspamd's log")
return
}
c.add(ReasonUnsafe, "rspamd's log, which records learned digests, is readable by other users")
}
func cageFSProtectsLog(cage CageFSFacts, log LogFacts) bool {
// The skeleton probe proves exclusion of one directory, not syslog,
// journal output, arbitrary log files or symlink destinations.
if log.Err != nil || log.Type != "file" || cage.ExcludedLogDir == "" ||
!filepath.IsAbs(log.Path) || filepath.Clean(log.Path) != log.Path ||
!ancestorOf(cage.ExcludedLogDir, log.Path) || len(log.Parents) == 0 {
return false
}
for _, p := range log.Parents {
if !p.Mode.IsDir() {
return false
}
}
return log.Missing || log.File.Mode.IsRegular()
}
// checkLearningConflicts finds operator fuzzy configuration CSM's override
// would hide. Once CSM's files are installed, verification of the
// loaded rules takes over.
func checkLearningConflicts(c *learningCheck, s Settings, f HostFacts) {
for _, t := range f.Targets {
if t.Path == RoundcubeIncludePath || !isLearningPath(t.Path) {
continue
}
if err := OptionalUsable(t); err != nil {
c.add(ReasonConfigConflict, "a learning file CSM would write cannot be written safely")
return
}
}
if f.LearningDumps.Err != nil {
return
}
if learningInstalled(f) {
checkLearningStillLoaded(c, s, f)
return
}
rules, err := loadedFuzzyRules(f.LearningDumps.FuzzyCheck)
if err != nil {
c.add(ReasonConfigConflict, "CSM cannot read rspamd's loaded fuzzy rules")
return
}
for name := range rules {
if name != stockFuzzyRule {
c.add(ReasonConfigConflict, "rspamd loads a local fuzzy rule that CSM's override would hide")
return
}
}
enabled, err := fuzzyWorkerEnabled(f.LearningDumps.Worker)
if err != nil {
c.add(ReasonConfigConflict, "CSM cannot read rspamd's loaded fuzzy worker")
} else if enabled {
c.add(ReasonConfigConflict, "rspamd runs a local fuzzy worker that CSM's override would replace")
}
}
// checkLearningStillLoaded compares what rspamd loads now with CSM's files,
// on every pass after install: a runtime weight change, a
// higher-priority group cap or score rescaling is a conflict. Files that
// drifted from CSM's text are left to the next apply, whose own check
// verifies the result.
func checkLearningStillLoaded(c *learningCheck, s Settings, f HostFacts) {
if !learningFilesCurrent(s, f) {
return
}
d := f.LearningDumps
if d.SymbolsErr != nil {
c.add(ReasonBackendUnavailable, "CSM cannot read the effective learning weights from rspamd's controller")
return
}
actions, err := ParseActions([]byte(f.RspamdActions.Stdout))
if f.RspamdActions.Err != nil || err != nil {
c.add(ReasonBackendUnavailable, "rspamd's actions are not readable")
return
}
if err := CheckLearningLoaded(d.FuzzyCheck, d.Worker, d.Group, d.Symbols, actions, s.LearningKeys, s.LearningRedisSocket); err != nil {
c.add(ReasonConfigConflict, "rspamd no longer loads CSM's learning configuration as written: "+err.Error())
}
}
// learningFilesCurrent reports whether every learning file holds exactly the
// text CSM renders for s.
func learningFilesCurrent(s Settings, f HostFacts) bool {
files, err := RenderLearning(s)
if err != nil {
return false
}
for _, t := range f.Targets {
if want, ok := files[t.Path]; ok && (t.Missing || t.Err != nil || t.Content != want) {
return false
}
}
return true
}
// loadedFuzzyRules reads the rules of `rspamadm configdump -j fuzzy_check`:
// an object keyed by rule name with one rule, a list of single-key objects
// with several. A rule name seen twice is an error.
func loadedFuzzyRules(dump []byte) (map[string]map[string]json.RawMessage, error) {
out := map[string]map[string]json.RawMessage{}
if len(dump) == 0 {
return out, nil
}
root, err := dumpObjectAs[json.RawMessage](dump, "fuzzy_check")
if err != nil {
return nil, err
}
raw, ok := root["rule"]
if !ok {
return out, nil
}
if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
return nil, errors.New("fuzzy_check rule is null")
}
var entries []json.RawMessage
if json.Unmarshal(raw, &entries) != nil {
entries = []json.RawMessage{raw}
}
for _, entry := range entries {
named, err := dumpObjectAs[json.RawMessage](entry, "fuzzy_check rule")
if err != nil {
return nil, err
}
for name, body := range named {
fields, err := dumpObjectAs[json.RawMessage](body, "fuzzy_check rule")
if err != nil {
return nil, err
}
if _, dup := out[name]; dup {
return nil, errors.New("fuzzy_check names a rule twice")
}
out[name] = fields
}
}
return out, nil
}
func isLearningPath(p string) bool {
for _, lp := range LearningPaths() {
if p == lp {
return true
}
}
return false
}
// fuzzyWorkerEnabled reports whether the worker section holds an enabled
// fuzzy worker, in either form rspamd prints.
func fuzzyWorkerEnabled(section json.RawMessage) (bool, error) {
if len(section) == 0 {
return false, nil
}
var entries []json.RawMessage
if trimmed := bytes.TrimSpace(section); len(trimmed) > 0 && trimmed[0] == '[' {
if json.Unmarshal(section, &entries) != nil {
return false, errors.New("worker section is not readable")
}
} else {
entries = []json.RawMessage{section}
}
for _, entry := range entries {
fields, err := dumpObjectAs[json.RawMessage](entry, "worker")
if err != nil {
return false, err
}
if raw, typed := fields["type"]; typed {
var kind string
if json.Unmarshal(raw, &kind) != nil || kind == "" || strings.ContainsRune(kind, '\x00') {
return false, errors.New("worker type is not readable")
}
if kind != "fuzzy" {
continue
}
} else if raw, ok := fields["fuzzy"]; ok {
fields, err = dumpObjectAs[json.RawMessage](raw, "fuzzy worker")
if err != nil {
return false, err
}
} else {
continue
}
disabled, err := disabledWorker(fields)
if err != nil {
return false, err
}
if !disabled {
return true, nil
}
}
return false, nil
}
// roundcubeUsable decides the Junk-button include alone: any
// failure reports it and leaves learning on.
func roundcubeUsable(c *learningCheck, f HostFacts) bool {
rc := f.Roundcube
if rc.Err != nil || !rc.IncludeMechanism || !rc.PHP || OptionalUsable(rc.Include) != nil {
c.add(ReasonRoundcube, "the webmail Junk-button include cannot be installed or verified; learning continues without it")
return false
}
return true
}
// LearningScoringLoaded reports whether the full `rspamadm configdump -j`
// shows CSM's learning rules loaded in an enabled fuzzy_check module. An
// unreadable dump says nothing.
func LearningScoringLoaded(fullDump []byte) bool {
root, err := dumpObjectAs[json.RawMessage](fullDump, "configdump")
if err != nil {
return false
}
section, ok := root["fuzzy_check"]
if !ok {
return false
}
fields, err := dumpObjectAs[json.RawMessage](section, "fuzzy_check")
if err != nil {
return false
}
var enabled bool
if raw, ok := fields["enabled"]; ok && json.Unmarshal(raw, &enabled) == nil && !enabled {
return false
}
rules, err := loadedFuzzyRules(section)
if err != nil {
return false
}
_, scan := rules["csm"]
_, learn := rules["csm_learn"]
return scan || learn
}
package antispam
import (
"encoding/json"
"errors"
"fmt"
"io"
"reflect"
"slices"
"strings"
)
// LearningCheckStep is the failed step of an apply whose loaded learning
// configuration differs from CSM's files; the daemon then holds learning
// off until the learning configuration changes.
const LearningCheckStep = "learning_check"
// LearningProbeStep is the failed step of an apply that could not read
// rspamd's loaded learning configuration (a dump command or the controller
// failed). It is an outage, so learning is not held.
const LearningProbeStep = "learning_probe"
// learningExpireSeconds is the worker's expire = 90d as configdump prints it.
const learningExpireSeconds = 90 * 24 * 3600
// Rule attributes CSM renders. A loaded rule with any other attribute, such
// as a different hash algorithm, is a conflict.
var (
learningRuleCommon = []string{"servers", "fuzzy_key", "fuzzy_shingles_key", "read_encryption_key", "skip_unknown",
"max_score", "min_length", "text_hashes", "text_shingles", "html_shingles", "skip_images", "mime_types",
"scan_archives", "fuzzy_map"}
learningRuleScan = append(slices.Clone(learningRuleCommon), "read_only")
learningRuleLearn = append(slices.Clone(learningRuleCommon), "mode", "retransmits", "write_encryption_key")
)
// CheckLearningLoaded compares what rspamd loaded with CSM's learning files:
// exactly the rules csm and csm_learn with the pinned
// attributes and only flag 1, one fuzzy worker with the pinned transport
// and two keypairs, the fuzzy group cap, the effective CSM_FUZZY_SPAM weight
// from the controller's /symbols, and no score rescaling. With a dedicated
// learning Redis the worker must select exactly redisSocket. Errors name
// the attribute, never a value that could be a key.
func CheckLearningLoaded(fuzzyCheck, worker, group, symbols []byte, a Actions, keys LearningKeys, redisSocket string) error {
scan, learn, err := checkLearningRules(fuzzyCheck, keys)
if err != nil {
return err
}
if err := checkLearningWorker(worker, scan, learn, keys, redisSocket); err != nil {
return err
}
if err := checkLearningGroup(group); err != nil {
return err
}
if err := checkLearningSymbols(symbols); err != nil {
return err
}
if a.GrowFactor != nil && *a.GrowFactor > 1 {
return errors.New("rspamd rescales scores with grow_factor above 1")
}
return nil
}
func value(raw json.RawMessage) any {
var v any
if json.Unmarshal(raw, &v) != nil {
return nil
}
return v
}
func isString(raw json.RawMessage) (string, bool) {
s, ok := value(raw).(string)
return s, ok && s != ""
}
// checkLearningRules returns the scan and learning rules once both match.
func checkLearningRules(dump []byte, keys LearningKeys) (scan, learn map[string]json.RawMessage, err error) {
root, err := dumpObjectAs[json.RawMessage](dump, "fuzzy_check")
if err != nil {
return nil, nil, errors.New("loaded fuzzy_check is not readable")
}
if raw, ok := root["enabled"]; ok && value(raw) != true {
return nil, nil, errors.New("loaded fuzzy_check is disabled")
}
rules, err := loadedFuzzyRules(dump)
if err != nil {
return nil, nil, errors.New("loaded fuzzy rules are not readable")
}
scan, learn = rules["csm"], rules["csm_learn"]
if len(rules) != 2 || scan == nil || learn == nil {
return nil, nil, errors.New("rspamd must load exactly the fuzzy rules csm and csm_learn")
}
if err := checkRule("csm", scan, learningRuleScan, "CSM_FUZZY_SPAM"); err != nil {
return nil, nil, err
}
if err := checkRule("csm_learn", learn, learningRuleLearn, "CSM_FUZZY_LEARN"); err != nil {
return nil, nil, err
}
if value(scan["read_only"]) != true {
return nil, nil, errors.New("fuzzy rule csm: read_only is not true")
}
if value(learn["mode"]) != "write_only" {
return nil, nil, errors.New("fuzzy rule csm_learn: mode is not write_only")
}
if value(learn["retransmits"]) != 0.0 {
return nil, nil, errors.New("fuzzy rule csm_learn: retransmits is not 0")
}
if key, ok := isString(learn["write_encryption_key"]); !ok || key != keys.Write.Public {
return nil, nil, errors.New("fuzzy rule csm_learn: write_encryption_key differs from CSM's")
}
// Agreement between rules cannot detect an override replacing both keys.
for key, expected := range map[string]string{
"fuzzy_key": keys.Fuzzy.Reveal(), "fuzzy_shingles_key": keys.Shingles.Reveal(), "read_encryption_key": keys.Read.Public,
} {
if value(scan[key]) != expected || value(learn[key]) != expected {
return nil, nil, fmt.Errorf("fuzzy rules: %s differs from CSM's", key)
}
}
return scan, learn, nil
}
func checkRule(name string, rule map[string]json.RawMessage, attrs []string, symbol string) error {
if len(rule) != len(attrs) {
return fmt.Errorf("fuzzy rule %s carries attributes CSM did not set", name)
}
for _, attr := range attrs {
if _, ok := rule[attr]; !ok {
return fmt.Errorf("fuzzy rule %s: %s is missing", name, attr)
}
}
want := map[string]any{
"servers": learningServer, "skip_unknown": true, "max_score": 18.0, "min_length": float64(LearningMinLength),
"text_hashes": true, "text_shingles": true, "html_shingles": false, "skip_images": true,
"mime_types": []any{}, "scan_archives": false,
}
for attr, w := range want {
if !reflect.DeepEqual(value(rule[attr]), w) {
return fmt.Errorf("fuzzy rule %s: %s differs from CSM's", name, attr)
}
}
for _, key := range []string{"fuzzy_key", "fuzzy_shingles_key", "read_encryption_key"} {
if _, ok := isString(rule[key]); !ok {
return fmt.Errorf("fuzzy rule %s: %s is missing", name, key)
}
}
m, err := dumpObjectAs[json.RawMessage](rule["fuzzy_map"], "fuzzy_map")
if err != nil || len(m) != 1 {
return fmt.Errorf("fuzzy rule %s: fuzzy_map must map only flag 1 to %s", name, symbol)
}
mapping, err := dumpObjectAs[float64](m[symbol], "fuzzy mapping")
wantMap := map[string]float64{"flag": float64(LearningFlag), "max_score": 18.0}
if err != nil || !reflect.DeepEqual(mapping, wantMap) {
return fmt.Errorf("fuzzy rule %s: fuzzy_map must map only flag 1 to %s", name, symbol)
}
return nil
}
// checkLearningWorker requires exactly one fuzzy worker with the pinned
// transport and the two keypairs the rules use.
func checkLearningWorker(section []byte, scan, learn map[string]json.RawMessage, keys LearningKeys, redisSocket string) error {
workers, err := fuzzyWorkers(section)
if err != nil {
return errors.New("loaded fuzzy worker is not readable")
}
if len(workers) != 1 {
return errors.New("rspamd must load exactly one fuzzy worker")
}
w := workers[0]
if disabled, err := disabledWorker(w); err != nil || disabled {
return errors.New("the fuzzy worker is not enabled")
}
want := map[string]any{"bind_socket": learningServer, "count": 1.0, "backend": "redis",
"encrypted_only": true, "allow_update": []any{}, "expire": float64(learningExpireSeconds)}
for attr, v := range want {
if !reflect.DeepEqual(value(w[attr]), v) {
return fmt.Errorf("fuzzy worker: %s differs from CSM's", attr)
}
}
if p, ok := isString(w["prefix"]); !ok || p != keys.Prefix {
return errors.New("fuzzy worker: prefix differs from CSM's")
}
if redisSocket != "" {
// Any other server key would send reads or writes elsewhere.
for _, key := range []string{"server", "read_servers", "write_servers"} {
if _, ok := w[key]; ok {
return fmt.Errorf("fuzzy worker: %s is set besides CSM's servers", key)
}
}
if value(w["servers"]) != redisSocket {
return errors.New("fuzzy worker: servers differs from CSM's")
}
if value(w["password"]) != "" {
return errors.New("fuzzy worker: password differs from CSM's")
}
}
var pairs []json.RawMessage
if json.Unmarshal(w["keypair"], &pairs) != nil || len(pairs) != 2 {
return errors.New("fuzzy worker: want exactly two keypairs")
}
readKey, _ := isString(scan["read_encryption_key"])
writeKey, _ := isString(learn["write_encryption_key"])
seen := map[string]bool{}
for _, raw := range pairs {
kp, err := dumpObjectAs[json.RawMessage](raw, "fuzzy keypair")
if err != nil {
return errors.New("fuzzy worker: a keypair is not readable")
}
pub, ok := isString(kp["pubkey"])
if _, priv := isString(kp["privkey"]); !ok || !priv {
return errors.New("fuzzy worker: a keypair is incomplete")
}
ext, err := dumpObjectAs[json.RawMessage](kp["extensions"], "keypair extensions")
var ops []string
if err != nil || json.Unmarshal(ext["allowed_ops"], &ops) != nil {
return errors.New("fuzzy worker: keypair extensions are not readable")
}
switch {
case slices.Equal(ops, []string{"read"}) && pub == readKey && value(kp["privkey"]) == keys.Read.Private.Reveal():
seen["read"] = true
case slices.Equal(ops, []string{"read", "write", "delete"}) && pub == writeKey && value(kp["privkey"]) == keys.Write.Private.Reveal():
seen["write"] = true
default:
return errors.New("fuzzy worker: a keypair's allowed_ops or keys differ from CSM's")
}
}
if !seen["read"] || !seen["write"] {
return errors.New("fuzzy worker: want one read keypair and one write keypair")
}
return nil
}
// fuzzyWorkers returns every fuzzy worker in the worker section, in either
// form rspamd prints.
func fuzzyWorkers(section []byte) ([]map[string]json.RawMessage, error) {
var entries []json.RawMessage
if json.Unmarshal(section, &entries) != nil {
entries = []json.RawMessage{section}
}
var out []map[string]json.RawMessage
for _, entry := range entries {
obj, err := dumpObjectAs[json.RawMessage](entry, "worker")
if err != nil {
return nil, err
}
if raw, ok := obj["fuzzy"]; ok {
w, err := dumpObjectAs[json.RawMessage](raw, "fuzzy worker")
if err != nil {
return nil, err
}
out = append(out, w)
continue
}
if value(obj["type"]) == "fuzzy" {
out = append(out, obj)
}
}
return out, nil
}
// checkLearningGroup reads `configdump -j group`, a stream of objects, and
// requires the fuzzy group's cap and CSM's configured weights.
func checkLearningGroup(dump []byte) error {
d := json.NewDecoder(strings.NewReader(string(dump)))
var fuzzy map[string]json.RawMessage
found := 0
for {
var entry json.RawMessage
err := d.Decode(&entry)
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return errors.New("loaded symbol groups are not readable")
}
obj, err := dumpObjectAs[json.RawMessage](entry, "symbol group")
if err != nil {
return errors.New("loaded symbol groups are not readable")
}
if raw, ok := obj["fuzzy"]; ok {
found++
if fuzzy, err = dumpObjectAs[json.RawMessage](raw, "fuzzy group"); err != nil {
return errors.New("loaded fuzzy group is not readable")
}
}
}
if found != 1 {
return errors.New("rspamd must load one fuzzy group")
}
if value(fuzzy["max_score"]) != 10.0 {
return errors.New("fuzzy group: max_score is not 10")
}
syms, err := dumpObjectAs[json.RawMessage](fuzzy["symbols"], "fuzzy symbols")
if err != nil {
return errors.New("fuzzy group: symbols are not readable")
}
for symbol, weight := range map[string]float64{"CSM_FUZZY_SPAM": 10, "CSM_FUZZY_LEARN": 0} {
s, err := dumpObjectAs[json.RawMessage](syms[symbol], "fuzzy symbol")
if err != nil || value(s["weight"]) != weight {
return fmt.Errorf("fuzzy group: %s weight is not %g", symbol, weight)
}
}
return nil
}
// checkLearningSymbols reads the controller's /symbols reply, which shows
// dynamic weight overrides configdump cannot.
func checkLearningSymbols(reply []byte) error {
var groups []json.RawMessage
if json.Unmarshal(reply, &groups) != nil {
return errors.New("the controller's /symbols reply is not readable")
}
found := 0
for _, raw := range groups {
g, err := dumpObjectAs[json.RawMessage](raw, "controller group")
var rules []json.RawMessage
if err != nil || json.Unmarshal(g["rules"], &rules) != nil {
return errors.New("the controller's /symbols group is not readable")
}
for _, raw := range rules {
r, err := dumpObjectAs[json.RawMessage](raw, "controller symbol")
if err != nil {
return errors.New("the controller's /symbols rule is not readable")
}
if value(r["symbol"]) != "CSM_FUZZY_SPAM" {
continue
}
found++
if value(r["weight"]) != 10.0 {
return errors.New("the effective CSM_FUZZY_SPAM weight is not 10")
}
}
}
if found != 1 {
return errors.New("the controller does not list CSM_FUZZY_SPAM once")
}
return nil
}
package antispam
import (
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"path"
"strconv"
"strings"
)
// Learning key file names under the learning directory's keys/ subdirectory.
const (
KeyFuzzy = "fuzzy.key"
KeyShingles = "shingles.key"
KeyRead = "read.keypair"
KeyWrite = "write.keypair"
KeyPrefix = "prefix"
)
// LearningKeyNames lists the key files in creation order: the prefix comes
// last, so a crash during first generation leaves an incomplete set.
var LearningKeyNames = []string{KeyFuzzy, KeyShingles, KeyRead, KeyWrite, KeyPrefix}
// Parameters shared by both CSM fuzzy rules that affect digests or which
// content is hashed. They are part of the hash generation.
const (
LearningMinLength = 32
learningTextHashOpts = "text_hashes=true;text_shingles=true;html_shingles=false;skip_images=true;mime_types=[];scan_archives=false"
)
// Keypair is an rspamd curve25519 keypair from `rspamadm keypair -j`, both
// halves in rspamd's base32 encoding.
type Keypair struct {
Public string
Private Secret
}
// LearningKeys are the secrets of one learning installation: the two hash
// keys and the Redis prefix define the hash generation; the keypairs only
// authorize reads and writes to the fuzzy storage.
type LearningKeys struct {
Fuzzy, Shingles Secret
Read, Write Keypair
Prefix string
}
// rspamdKeyLen is the base32 length of a 32-byte curve25519 key.
const rspamdKeyLen = 52
// rspamdBase32 is rspamd's base32 alphabet.
const rspamdBase32 = "ybndrfg8ejkmcpqxot1uwisza345h769"
// Validate checks both halves are rspamd base32 keys. Errors never carry
// the private key.
func (k Keypair) Validate() error {
if !validRspamdKey(k.Public) {
return errors.New("public key is not a 52-character rspamd base32 key")
}
if !validRspamdKey(k.Private.Reveal()) {
return errors.New("private key is not a 52-character rspamd base32 key")
}
return nil
}
func validRspamdKey(s string) bool {
if len(s) != rspamdKeyLen {
return false
}
for i := 0; i < len(s); i++ {
if strings.IndexByte(rspamdBase32, s[i]) < 0 {
return false
}
}
return true
}
// LearningKeyPaths maps each key file name to its path under dir/keys.
func LearningKeyPaths(dir string) map[string]string {
out := make(map[string]string, len(LearningKeyNames))
for _, name := range LearningKeyNames {
out[name] = path.Join(dir, "keys", name)
}
return out
}
// HashGeneration identifies the digests a key set produces: hex SHA-256 over
// the hash keys, the prefix and the pinned hash options, each length
// prefixed. The transport keypairs are left out; they change no digest.
func HashGeneration(k LearningKeys) string {
h := sha256.New()
for _, field := range []string{
"csm-learning-generation-v1",
k.Fuzzy.Reveal(), k.Shingles.Reveal(), k.Prefix,
strconv.Itoa(LearningMinLength), learningTextHashOpts,
} {
var n [8]byte
binary.BigEndian.PutUint64(n[:], uint64(len(field)))
h.Write(n[:])
h.Write([]byte(field))
}
return hex.EncodeToString(h.Sum(nil))
}
// Package learnwatch finds new reports in tenants' Spam folders and hands
// them to the learning engine.
package learnwatch
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"math"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/admission"
)
// MaxPasswdBytes bounds the password database read.
const MaxPasswdBytes = 8 << 20
// Account skip reasons.
const (
SkipNoPasswd = "no_passwd_entry"
SkipBadHome = "bad_home"
SkipRootUID = "root_uid"
)
// Account is one cPanel account with its passwd identity, or a Postfix
// reporter with only its id. Home may sit on any partition; its owner is
// checked when it is opened.
type Account struct {
Name string
UID, GID int
Home string
}
// Skip is a name left out, with the reason. UID is the passwd UID when
// known: a skipped account still exists and holds it.
type Skip struct {
Name string
Reason string
UID int
}
// Source supplies the account registry and the password database.
type Source interface {
CPanelUsers() ([]string, error)
Passwd() ([]byte, error)
}
// LiveSource reads /var/cpanel/users and /etc/passwd.
type LiveSource struct {
UsersDir string
PasswdPath string
}
// DefaultSource is the host's registry.
var DefaultSource = LiveSource{UsersDir: "/var/cpanel/users", PasswdPath: "/etc/passwd"}
// CPanelUsers lists regular account records without following symlinks.
func (s LiveSource) CPanelUsers() ([]string, error) {
entries, err := os.ReadDir(s.UsersDir)
if err != nil {
return nil, err
}
names := make([]string, 0, len(entries))
for _, e := range entries {
if e.Type().IsRegular() {
names = append(names, e.Name())
}
}
return names, nil
}
// Passwd reads the password database, one byte past the bound so an
// oversized file is detected.
func (s LiveSource) Passwd() ([]byte, error) {
f, err := os.Open(s.PasswdPath)
if err != nil {
return nil, err
}
defer f.Close()
return io.ReadAll(io.LimitReader(f, MaxPasswdBytes+1))
}
type pwEntry struct {
uid, gid int
home string
}
// registry is one pass's reporters: cPanel accounts with their homes, or
// Postfix reporters with their Junk folders.
type registry struct {
accounts []Account
// skips holds the reason of every name left out.
skips []string
// held are the ids that keep their reservations: every reporter, and
// those that only skipped names still hold.
held map[int]bool
// junk maps each Postfix reporter's Junk folders to the uid that owns
// their mailbox's files.
junk map[int]map[FolderKey]int
// unchecked ids keep their folder state; complete false keeps every
// reporter's.
unchecked []int
complete bool
// removed rechecks with a fresh read that uid is gone.
removed func(ctx context.Context, uid int) (bool, error)
}
func cpanelRegistry(src Source) (registry, error) {
accounts, skips, err := Accounts(src)
if err != nil {
return registry{}, err
}
// A registry name without a usable UID cannot be distinguished from a
// deleted account by the UID-keyed persisted state.
reg := registry{accounts: accounts, held: map[int]bool{}, complete: len(skips) == 0}
for _, a := range accounts {
reg.held[a.UID] = true
}
// An account CSM cannot read still holds its UID.
for _, s := range skips {
reg.skips = append(reg.skips, s.Reason)
if s.Reason == SkipBadHome {
reg.held[s.UID] = true
}
}
reg.removed = func(_ context.Context, uid int) (bool, error) {
current, skipped, err := Accounts(src)
if err != nil {
return false, err
}
return !slices.ContainsFunc(current, func(a Account) bool { return a.UID == uid }) &&
!slices.ContainsFunc(skipped, func(s Skip) bool { return s.Reason == SkipBadHome && s.UID == uid }), nil
}
return reg, nil
}
// Accounts joins registry names with passwd. Names outside the account
// alphabet and root are not accounts and are dropped silently.
func Accounts(src Source) ([]Account, []Skip, error) {
names, err := src.CPanelUsers()
if err != nil {
return nil, nil, fmt.Errorf("cannot list cPanel accounts: %w", err)
}
names = slices.Clone(names)
data, err := src.Passwd()
if err != nil {
return nil, nil, fmt.Errorf("cannot read the password database: %w", err)
}
if len(data) > MaxPasswdBytes {
return nil, nil, errors.New("the password database is larger than the limit")
}
pw := parsePasswd(data)
var out []Account
var skips []Skip
for _, name := range names {
if name == "root" || !admission.ValidAccountName(name) {
continue
}
e, ok := pw[name]
switch {
case !ok:
skips = append(skips, Skip{Name: name, Reason: SkipNoPasswd})
case e.uid == 0:
skips = append(skips, Skip{Name: name, Reason: SkipRootUID})
case !filepath.IsAbs(e.home) || e.home == "/" || filepath.Clean(e.home) != e.home || strings.ContainsRune(e.home, 0):
skips = append(skips, Skip{Name: name, Reason: SkipBadHome, UID: e.uid})
default:
out = append(out, Account{Name: name, UID: e.uid, GID: e.gid, Home: e.home})
}
}
// A rename or UID reuse between the two reads must not make a held
// UID appear removed. Retry the pass instead of joining mixed lists.
after, err := src.CPanelUsers()
if err != nil {
return nil, nil, fmt.Errorf("cannot recheck cPanel accounts: %w", err)
}
after = slices.Clone(after)
before := slices.Clone(names)
slices.Sort(before)
slices.Sort(after)
if !slices.Equal(before, after) {
return nil, nil, errors.New("cPanel account registry changed during read")
}
return out, skips, nil
}
// parsePasswd keeps the first well-formed entry per name, as getpwnam does.
func parsePasswd(data []byte) map[string]pwEntry {
out := map[string]pwEntry{}
for _, line := range bytes.Split(data, []byte{'\n'}) {
f := strings.Split(string(line), ":")
if len(f) != 7 {
continue
}
if _, dup := out[f[0]]; dup {
continue
}
uid, err1 := strconv.Atoi(f[2])
gid, err2 := strconv.Atoi(f[3])
if err1 != nil || err2 != nil || uid < 0 || gid < 0 {
continue
}
// Linux IDs are 32-bit; the all-ones value is a syscall sentinel.
if int64(uid) >= math.MaxUint32 || int64(gid) >= math.MaxUint32 {
continue
}
out[f[0]] = pwEntry{uid: uid, gid: gid, home: f[5]}
}
return out
}
package learnwatch
import (
"bytes"
"errors"
"io"
"io/fs"
"os"
"slices"
"sort"
"strings"
"syscall"
"github.com/pidginhost/csm/internal/antispam/dovecot"
"github.com/pidginhost/csm/internal/safepath"
)
// Mailbox discovery reads each passwd file up to MaxMailboxListBytes and
// inspects at most MaxMailboxes records across the account. Invalid, empty
// and duplicate records consume the same budget as accepted names. Directory
// enumeration has a separate MaxMailboxes bound.
const (
MaxMailboxListBytes = 1 << 20
MaxMailboxes = dovecot.MaxMailboxes
// MaxAccountMailboxListBytes bounds all of one account's lists; real
// lists take a few hundred bytes per mailbox.
MaxAccountMailboxListBytes = dovecot.MaxListBytes
)
// Mailbox discovery skip reasons. Skips carry no tenant-supplied names.
const (
SkipUnsupportedName = "unsupported_name"
SkipMailboxListTooLarge = "mailbox_list_too_large"
SkipMailboxLimit = "mailbox_limit"
SkipUnsafePath = "unsafe_path"
)
// ParseMailboxPasswd returns the sorted distinct valid names from at most
// limit records of a cPanel mailbox passwd file, including empty lines.
func ParseMailboxPasswd(data []byte, limit int) ([]string, []Skip) {
return parseMailboxPasswd(data, &limit)
}
func parseMailboxPasswd(data []byte, remaining *int) ([]string, []Skip) {
seen := map[string]bool{}
var names []string
var skips []Skip
data = bytes.TrimSuffix(data, []byte{'\n'})
if len(data) == 0 {
return nil, nil
}
// Every line costs a record, empty ones included, so a list of
// newlines cannot buy unbounded parsing.
for line := range bytes.SplitSeq(data, []byte{'\n'}) {
if *remaining <= 0 {
skips = append(skips, Skip{Reason: SkipMailboxLimit})
break
}
*remaining--
if len(line) == 0 {
continue
}
raw, _, found := bytes.Cut(line, []byte{':'})
name := string(raw)
switch {
case !found || !dovecot.ValidMailbox(name):
skips = append(skips, Skip{Reason: SkipUnsupportedName})
continue
case seen[name]:
continue
}
seen[name] = true
names = append(names, name)
}
sort.Strings(names)
return names, skips
}
// unsafeErr: a component was a symlink or not a directory.
func unsafeErr(err error) bool {
return errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.ENOTDIR) || errors.Is(err, syscall.ENXIO)
}
// Mailboxes lists the main mailbox and every valid mailbox of every valid
// domain directory under ~/etc, within the discovery bounds. It never lists
// ~/mail, so the virtual namespace is never followed. Entries of ~/etc that
// are not directories, links included, are not domains. Unsafe or
// unsupported entries are skipped; a transient failure is an error.
func (h *Home) Mailboxes() ([]Mailbox, []Skip, error) {
boxes, skips, err := h.listMailboxes(domainDirs)
return append([]Mailbox{{}}, boxes...), skips, err
}
// DomainMailboxes lists the valid mailboxes of exactly the given domains,
// within the discovery bounds. It does not list ~/etc, so an entry there that
// is not one of them costs nothing. The caller names real domains, so a link
// or a file at one is an unsafe path, not an entry to ignore. A domain with no
// directory has no mailboxes. Names come from cPanel's files, validated by the
// caller, and the main mailbox is not listed.
func (h *Home) DomainMailboxes(domains []string) ([]Mailbox, []Skip, error) {
return h.listMailboxes(func(*safepath.Dir) ([]string, []Skip, error) { return domains, nil, nil })
}
// listMailboxes reads the lists of the domains that find names under ~/etc,
// charging every list to one account budget.
func (h *Home) listMailboxes(find func(*safepath.Dir) ([]string, []Skip, error)) ([]Mailbox, []Skip, error) {
var out []Mailbox
etc, err := h.dir.Walk("etc")
switch {
case errors.Is(err, fs.ErrNotExist):
return out, nil, nil
case unsafeErr(err):
return out, []Skip{{Reason: SkipUnsafePath}}, nil
case err != nil:
return out, nil, err
}
defer func() { _ = etc.Close() }()
domains, skips, err := find(etc)
if err != nil {
return out, skips, err
}
remaining, bytesLeft := MaxMailboxes, int64(MaxAccountMailboxListBytes)
for i, d := range domains {
// Empty and absent domains cost opens even when they have no records.
if i >= MaxMailboxes || remaining == 0 {
skips = append(skips, Skip{Reason: SkipMailboxLimit})
break
}
boxes, s, err := readMailboxList(etc, d, &remaining, &bytesLeft)
skips = append(skips, s...)
if err != nil {
return out, skips, err
}
for _, b := range boxes {
out = append(out, Mailbox{Domain: d, Box: b})
}
if slices.Contains(s, Skip{Reason: SkipMailboxLimit}) {
break
}
}
return out, skips, nil
}
func domainDirs(etc *safepath.Dir) ([]string, []Skip, error) {
var domains []string
var skips []Skip
for seen := 0; ; {
entries, err := etc.ReadDirN(512)
for _, e := range entries {
if seen++; seen > MaxMailboxes {
sort.Strings(domains)
return domains, append(skips, Skip{Reason: SkipMailboxLimit}), nil
}
name, typ := e.Name(), e.Type()
// Only directories are domains. Links are never followed and,
// like files, are ignored: cPanel keeps non-domain links here.
switch {
case strings.HasPrefix(name, "."):
case !dovecot.ValidDomain(name):
if typ.IsDir() {
skips = append(skips, Skip{Reason: SkipUnsupportedName})
}
case typ.IsDir():
domains = append(domains, name)
}
}
if err == io.EOF {
break
}
if err != nil {
return nil, skips, err
}
}
sort.Strings(domains)
return domains, skips, nil
}
// readMailboxList reads ~/etc/<domain>/passwd without following links.
func readMailboxList(etc *safepath.Dir, domain string, remaining *int, bytesLeft *int64) ([]string, []Skip, error) {
dir, err := etc.Walk(domain)
switch {
case errors.Is(err, fs.ErrNotExist):
return nil, nil, nil
case unsafeErr(err):
return nil, []Skip{{Reason: SkipUnsafePath}}, nil
case err != nil:
return nil, nil, err
}
defer func() { _ = dir.Close() }()
f, err := dir.OpenFile("passwd", os.O_RDONLY|syscall.O_NOCTTY, 0)
switch {
case errors.Is(err, fs.ErrNotExist):
return nil, nil, nil
case unsafeErr(err):
return nil, []Skip{{Reason: SkipUnsafePath}}, nil
case err != nil:
return nil, nil, err
}
defer func() { _ = f.Close() }()
fi, err := f.Stat()
if err != nil {
return nil, nil, err
}
if !fi.Mode().IsRegular() {
return nil, []Skip{{Reason: SkipUnsafePath}}, nil
}
if fi.Size() > MaxMailboxListBytes {
return nil, []Skip{{Reason: SkipMailboxListTooLarge}}, nil
}
if fi.Size() > *bytesLeft {
return nil, []Skip{{Reason: SkipMailboxLimit}}, nil
}
return readMailboxData(f, remaining, bytesLeft)
}
func readMailboxData(r io.Reader, remaining *int, bytesLeft *int64) ([]string, []Skip, error) {
data, err := io.ReadAll(io.LimitReader(r, min(MaxMailboxListBytes, *bytesLeft)+1))
// A tenant can grow a list after stat; rejected reads cost bytes too.
*bytesLeft -= int64(len(data))
if err != nil {
return nil, nil, err
}
switch {
case *bytesLeft < 0:
return nil, []Skip{{Reason: SkipMailboxLimit}}, nil
case len(data) > MaxMailboxListBytes:
return nil, []Skip{{Reason: SkipMailboxListTooLarge}}, nil
}
names, skips := parseMailboxPasswd(data, remaining)
return names, skips, nil
}
package learnwatch
import (
"errors"
"io/fs"
"os"
)
// GlobalOptInPath makes every account eligible; core then warns that no
// account can opt out.
const GlobalOptInPath = "/etc/global_spamassassin_enable"
// optInName is the per-account Spam Filters switch in the home directory.
const optInName = ".spamassassinenable"
// Eligibility skip reasons.
const (
SkipOptedOut = "opted_out"
SkipOptInSymlink = "optin_symlink"
SkipOptInNotRegular = "optin_not_regular"
)
// GlobalOptIn reports whether the host-wide override exists. Errors other
// than absence are returned, never read as off.
func GlobalOptIn(path string) (bool, error) {
_, err := os.Stat(path)
switch {
case err == nil:
return true, nil
case errors.Is(err, fs.ErrNotExist):
return false, nil
}
return false, err
}
// Eligible applies core's Spam Filters rule under the pinned home. The
// switch must be a regular file; a symlink or other type is off with a
// reason. A stat failure is an error so the caller retries.
func (h *Home) Eligible(global bool) (bool, string, error) {
if global {
return true, "", nil
}
fi, err := lstatAt(h.dir, optInName)
switch {
case errors.Is(err, fs.ErrNotExist):
return false, SkipOptedOut, nil
case err != nil:
return false, "", err
case fi.Mode()&fs.ModeSymlink != 0:
return false, SkipOptInSymlink, nil
case !fi.Mode().IsRegular():
return false, SkipOptInNotRegular, nil
}
return true, "", nil
}
package learnwatch
import (
"context"
"errors"
"hash/fnv"
"math"
"strings"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/antispam/dovecot"
)
// Reporter ids of domains and mailboxes lie in [1<<32, 1<<62): above every
// 32-bit UID, so they never meet an account's or a domainless user's, and
// never 0 or -1, which the scheduler and the learner reserve.
const (
minReporterID = 1 << 32
reporterMask = 1<<62 - 1
)
// Reporter ids need a 64-bit int, as every shipped target has; this
// constant does not compile where int is narrower.
const _ uint64 = math.MaxInt - reporterMask
// ReporterID is the learning reporter of a Postfix mailbox: its domain in
// domain mode, the mailbox itself in mailbox mode. Both are lower case, as
// Dovecot discovery and exempt matching give them. The id is a hash, so it
// survives restarts and watcher resets. A domainless user is not passed
// here: it votes under its own userdb uid.
func ReporterID(mode, domain, mailbox string) int {
key := "d:" + domain
if mode == antispam.ReporterMailbox {
key = "m:" + mailbox
}
h := fnv.New64a()
_, _ = h.Write([]byte(key))
return int(minReporterID | h.Sum64()&reporterMask) // #nosec G115 -- below 1<<62; int is 64-bit, asserted above
}
// DovecotSource is the discovery a Postfix registry reads; dovecot.Dovecot
// is one.
type DovecotSource interface {
Snapshot(context.Context) (dovecot.Snapshot, error)
Users(context.Context) ([]string, error)
}
// Postfix is the reporter registry of a Postfix and Dovecot host. Each mail
// domain, or each mailbox in mailbox mode, is one reporter whose mailboxes
// bring their own Junk folders and file owners. Exempt mailboxes and
// domains are not watched, and no opt-in file is read.
type Postfix struct {
Dovecot DovecotSource
// Reporter is antispam.ReporterDomain or antispam.ReporterMailbox.
Reporter string
// Exempt holds lower-case mailboxes and domains, as antispam.exempt.
Exempt []string
}
var (
errIncomplete = errors.New("the Dovecot mailbox listing is incomplete")
errNoUsers = errors.New("the Dovecot user listing is empty")
// errUnconfirmed: the recheck can neither confirm nor refute a removal.
errUnconfirmed = errors.New("the removal of a reporter cannot be confirmed")
)
// registry reads one complete snapshot. Only a complete snapshot says who
// is gone; anything less is an error, so the pass reports, cancels and
// prunes nothing.
func (p *Postfix) registry(ctx context.Context) (registry, error) {
snap, err := p.Dovecot.Snapshot(ctx)
if err == nil && !snap.Complete {
err = errIncomplete
}
if err != nil {
return registry{}, err
}
reg := registry{held: map[int]bool{}, junk: map[int]map[FolderKey]int{}, complete: true}
// known holds each domainless user's uid, its reporter, which the
// recheck listing does not show; 0 for one refused as root.
known := map[string]int{}
// A skipped user is present, not removed: its reporter keeps its
// reservations and its folders keep their state.
skip := func(user, reason string) {
reg.skips = append(reg.skips, reason)
_, domain, found := strings.Cut(user, "@")
switch {
case user == "" || antispam.IsExempt(p.Exempt, user):
// A refused name or an exempt mailbox is never watched.
case found:
id := ReporterID(p.Reporter, domain, user)
reg.held[id] = true
reg.unchecked = append(reg.unchecked, id)
case reason == dovecot.SkipUIDRoot:
known[user] = 0
default:
// Its reporter is a uid the skip does not carry.
reg.complete = false
}
}
for _, m := range snap.Mailboxes {
switch {
case antispam.IsExempt(p.Exempt, m.User):
continue
case m.UID <= 0:
skip(m.User, dovecot.SkipUIDRoot)
continue
}
id := m.UID
if m.Domain != "" {
id = ReporterID(p.Reporter, m.Domain, m.User)
} else {
known[m.User] = m.UID
}
if reg.junk[id] == nil {
reg.junk[id] = map[FolderKey]int{}
reg.accounts = append(reg.accounts, Account{UID: id})
}
reg.held[id] = true
for _, path := range m.Junk {
reg.junk[id][FolderKey{UID: id, Domain: m.Domain, Box: m.User, Folder: path}] = m.UID
}
}
for _, s := range snap.Skips {
skip(s.User, s.Reason)
}
reg.removed = func(ctx context.Context, uid int) (bool, error) { return p.removed(ctx, known, uid) }
return reg, nil
}
// removed rechecks with a fresh identity-only listing that reporter uid is
// gone: a domain or mailbox listed again keeps it. A domainless user's
// reporter is its uid, which the listing does not show, so a domainless
// name the pass did not resolve leaves the removal of a uid unconfirmed.
func (p *Postfix) removed(ctx context.Context, known map[string]int, uid int) (bool, error) {
users, err := p.Dovecot.Users(ctx)
if err == nil && len(users) == 0 {
err = errNoUsers
}
if err != nil {
return false, err
}
for _, u := range users {
_, domain, found := strings.Cut(u, "@")
switch {
case antispam.IsExempt(p.Exempt, u):
case found:
if ReporterID(p.Reporter, domain, u) == uid {
return false, nil
}
case uid < minReporterID:
if _, ok := known[u]; !ok {
return false, errUnconfirmed
}
}
}
return true, nil
}
package learnwatch
import (
"cmp"
"context"
"errors"
"io"
"io/fs"
"maps"
"slices"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/antispam/hostfacts"
"github.com/pidginhost/csm/internal/antispam/learn"
)
// Scheduler constants. bench_test.go measures fixed synthetic
// workloads on an 8-core development host with SSD storage; listing
// latency depends on the host and filesystem.
const (
// PassEvery sets the pass interval; MaxAttempts caps submission attempts
// per account per pass, shared across its folders.
PassEvery = 5 * time.Minute
MaxAttempts = 20
// MaxEntryErrors bounds the failed submissions of one entry, so a
// message that breaks the backend cannot stop every pass.
MaxEntryErrors = 3
// PassDeadline leaves time between passes even when discovery runs long;
// baselining a 51217-entry folder took 1.5 s.
PassDeadline = 4 * time.Minute
// DiscoveryChunk bounds the listing work before another account's turn:
// about 15 ms per chunk at 29 microseconds per entry; during a
// 51217-entry flood a second account was submitted after 15 to 19 ms.
DiscoveryChunk = 512
// AccountSeenAllowance limits identities, not database bytes; it covers
// S8's 51217-entry folder. A synthetic entry took 353 logical bytes;
// names, statuses and bbolt page overhead change the cost per entry.
AccountSeenAllowance = 65536
// FullRescanEvery relists unchanged folders daily so presence stays
// fresh for the 90-day prune; relisting the large folder takes seconds.
FullRescanEvery = 24 * time.Hour
pruneEvery = 24 * time.Hour
// FolderOpensPerTurn bounds the folders one account turn opens, so many
// mailboxes cost turns like a large folder does; an open walks a few
// path components and settling an unchanged folder writes nothing.
FolderOpensPerTurn = 256
)
// Pass skip reasons beyond the registry, eligibility and discovery ones.
const (
SkipHomeUnavailable = "home_unavailable"
SkipOptInUnreadable = "optin_unreadable"
SkipFolderUnreadable = "folder_unreadable"
SkipRegistryUnreadable = "registry_unreadable"
SkipStateUnavailable = "state_unavailable"
// SkipFailed: an entry failed MaxEntryErrors submissions.
SkipFailed = "failed"
// SkipCancelUnconfirmed: a removal could not be confirmed, so the pass
// cancelled and pruned nothing more.
SkipCancelUnconfirmed = "cancel_unconfirmed"
)
type submitter interface {
SubmitReport(context.Context, learn.Candidate) (learn.Outcome, error)
ProbeStorage(context.Context) error
CancelUnsent(ctx context.Context, uid int, confirm func() (bool, error)) error
ReportUIDs(context.Context) ([]int, error)
}
// WatcherConfig is what a watcher needs. Zero Chunk and SeenAllowance mean
// the constants; nil CanSubmit always submits.
type WatcherConfig struct {
Source Source
GlobalPath string
State *State
Engine submitter
Now func() time.Time
// CanDiscover false makes passes idle: nothing is listed or read.
// CanSubmit false keeps discovery and baselines but reads no message
// content and submits nothing.
CanDiscover func() bool
CanSubmit func() bool
// OnPass sees every completed pass, not idle or busy ones.
OnPass func(PassReport)
Chunk int
SeenAllowance int
// FolderOpens bounds folder opens per account turn; zero means
// FolderOpensPerTurn.
FolderOpens int
// Postfix, when set, replaces Source and GlobalPath: reporters come
// from Dovecot and nothing opts in or out.
Postfix *Postfix
}
// PassReport summarizes one pass. Skipped counts reasons, never names.
// Pending counts the reports known to wait; with Unfinished set some
// folders were not enumerated, so it is a lower bound.
type PassReport struct {
Accounts int
Submitted int
// Committed counts the cluster contributions the engine committed.
Committed int
Pending int
Skipped map[string]int
Unfinished bool
Busy bool
OldestPending time.Time
// RegistryErr is why the registry could not be read: the pass then
// submitted, cancelled and pruned nothing.
RegistryErr error
}
// ownerUID is the UID an account's files must belong to; for a Postfix
// mailbox it gets an Account holding the mailbox's userdb uid. Tests map
// their synthetic accounts to the test process's own UID.
var ownerUID = func(a Account) int { return a.UID }
// turnHook observes scheduling for tests: kinds turn, chunk, attempt and
// submit.
var turnHook = func(uid int, kind string) {}
// Watcher runs Spam folder passes, one at a time.
type Watcher struct {
cfg WatcherConfig
busy atomic.Bool
// submitStopped: a backend error this pass; the pass keeps discovering
// but submits nothing more, so a hung controller costs one request
// timeout per pass. Owned by the single active pass.
submitStopped bool
stoppedBy int
// answered: storage answered a digest query earlier this pass, so a
// later failure can be the message's fault (a backend
// failure leaves work pending).
answered bool
mu sync.Mutex
last PassReport
at time.Time
}
// NewWatcher fills defaults; nil gates are open.
func NewWatcher(cfg WatcherConfig) *Watcher {
if cfg.Now == nil {
cfg.Now = time.Now
}
if cfg.CanDiscover == nil {
cfg.CanDiscover = func() bool { return true }
}
if cfg.CanSubmit == nil {
cfg.CanSubmit = func() bool { return true }
}
if cfg.Chunk <= 0 {
cfg.Chunk = DiscoveryChunk
}
if cfg.FolderOpens <= 0 {
cfg.FolderOpens = FolderOpensPerTurn
}
if cfg.SeenAllowance <= 0 {
cfg.SeenAllowance = AccountSeenAllowance
}
if cfg.GlobalPath == "" {
cfg.GlobalPath = GlobalOptInPath
}
return &Watcher{cfg: cfg}
}
// Run passes every PassEvery until ctx ends, starting with one now.
func (w *Watcher) Run(ctx context.Context) {
t := time.NewTicker(PassEvery)
defer t.Stop()
for {
w.Pass(ctx)
select {
case <-ctx.Done():
return
case <-t.C:
}
}
}
// Last returns the last completed pass report and when it ended.
func (w *Watcher) Last() (PassReport, time.Time) {
w.mu.Lock()
defer w.mu.Unlock()
r := w.last
r.Skipped = maps.Clone(r.Skipped)
return r, w.at
}
// Pass runs one rotation-fair pass. A concurrent call returns Busy; a
// closed discovery gate makes it idle and unrecorded.
func (w *Watcher) Pass(ctx context.Context) PassReport {
if !w.busy.CompareAndSwap(false, true) {
return PassReport{Busy: true}
}
defer w.busy.Store(false)
if ctx.Err() != nil || !w.cfg.CanDiscover() {
return PassReport{}
}
ctx, cancel := context.WithTimeout(ctx, PassDeadline)
defer cancel()
r := w.pass(ctx)
w.mu.Lock()
w.last, w.at = r, w.cfg.Now()
w.last.Skipped = maps.Clone(r.Skipped)
w.mu.Unlock()
if w.cfg.OnPass != nil {
w.cfg.OnPass(r)
}
return r
}
// cancelRemoved cancels the unsent reservations of accounts gone from the
// registry: a removed account has lost its eligibility. Only a
// complete registry read gets here. An account CSM cannot read still holds
// its UID, and a reused UID keeps the reservation. Failures retry next pass.
// False stops the pass so stale account data cannot authorize later opt-outs.
// A removal the recheck cannot settle ends cancelling and pruning for the
// pass.
func (w *Watcher) cancelRemoved(ctx context.Context, reg *registry, r *PassReport) bool {
uids, err := w.cfg.Engine.ReportUIDs(ctx)
if err != nil {
r.Skipped[SkipStateUnavailable]++
r.Unfinished = true
return true
}
proceed := true
for _, uid := range uids {
if ctx.Err() != nil || !w.cfg.CanDiscover() {
r.Unfinished = true
proceed = false
break
}
if !reg.held[uid] {
var registryErr error
unconfirmed := false
cancelErr := w.cfg.Engine.CancelUnsent(ctx, uid, func() (bool, error) {
// The owner may have been busy publishing. Do not cancel a
// newly created account or a reused UID from the pass snapshot.
removed, recheckErr := reg.removed(ctx, uid)
if errors.Is(recheckErr, errUnconfirmed) {
unconfirmed = true
return false, nil
}
registryErr = recheckErr
return removed, recheckErr
})
if registryErr != nil {
r.Skipped[SkipRegistryUnreadable]++
r.Unfinished = true
r.RegistryErr = registryErr
proceed = false
break
}
if unconfirmed {
r.Skipped[SkipCancelUnconfirmed]++
reg.complete = false
break
}
err = errors.Join(err, cancelErr)
}
}
if err != nil {
r.Skipped[SkipStateUnavailable]++
r.Unfinished = true
}
return proceed
}
func (w *Watcher) pass(ctx context.Context) PassReport {
w.submitStopped, w.stoppedBy, w.answered = false, 0, false
r := PassReport{Skipped: map[string]int{}}
deadline := w.cfg.Now().Add(PassDeadline)
reg, err := w.registry(ctx)
if err != nil {
r.Skipped[SkipRegistryUnreadable]++
r.Unfinished = true
r.RegistryErr = err
return r
}
for _, reason := range reg.skips {
r.Skipped[reason]++
}
if ctx.Err() != nil || !w.cfg.CanDiscover() {
r.Unfinished = true
return r
}
if !w.cancelRemoved(ctx, ®, &r) {
return r
}
if w.cfg.Postfix == nil {
if _, err = GlobalOptIn(w.cfg.GlobalPath); err != nil {
r.Skipped[SkipOptInUnreadable]++
r.Unfinished = true
return r
}
}
accounts := reg.accounts
slices.SortFunc(accounts, func(a, b Account) int { return cmp.Compare(a.UID, b.UID) })
var cursor int
if err = w.cfg.State.View(func(tx StateTx) (e error) { cursor, e = tx.Cursor(); return e }); err != nil {
r.Skipped[SkipStateUnavailable]++
r.Unfinished = true
return r
}
start, _ := slices.BinarySearchFunc(accounts, cursor, func(a Account, uid int) int { return cmp.Compare(a.UID, uid) })
accounts = append(accounts[start:], accounts[:start]...)
runs := make([]*accountRun, 0, len(accounts))
for _, a := range accounts {
run := &accountRun{w: w, a: a, deadline: deadline, junk: reg.junk[a.UID]}
if cursor != 0 && len(runs) > 0 {
// Rotation must govern the first attempts too: a small account
// must not overtake accounts still discovering their mailboxes.
run.waitFor = runs[len(runs)-1]
}
runs = append(runs, run)
}
defer func() {
for _, run := range runs {
run.close()
}
}()
for active := slices.Clone(runs); len(active) > 0; {
next := active[:0]
for _, run := range active {
if ctx.Err() != nil || !w.cfg.CanDiscover() || !w.cfg.Now().Before(deadline) {
r.Unfinished = true
nextUID := nextAfter(runs, w.stoppedBy)
if nextUID == 0 {
nextUID = run.a.UID
}
w.saveCursor(nextUID, &r)
w.tally(runs, &r)
return r
}
turnHook(run.a.UID, "turn")
run.turn(ctx, &r)
if run.done {
run.close()
} else {
next = append(next, run)
}
}
active = next
}
w.saveCursor(nextAfter(runs, w.stoppedBy), &r)
w.tally(runs, &r)
w.prune(runs, &r, reg.complete, reg.unchecked)
return r
}
func (w *Watcher) registry(ctx context.Context) (registry, error) {
if w.cfg.Postfix != nil {
return w.cfg.Postfix.registry(ctx)
}
return cpanelRegistry(w.cfg.Source)
}
// hostSideFailure reports failures a message cannot cause: cancellation by
// the engine or daemon, a request never sent, a wrong controller peer and
// local learning-state errors.
func hostSideFailure(err error) bool {
for _, local := range []error{context.Canceled, hostfacts.ErrControllerNotSent, learn.ErrPeer,
learn.ErrStateIO, learn.ErrStateMismatch, learn.ErrBodyInvalid} {
if errors.Is(err, local) {
return true
}
}
return false
}
// nextAfter is the UID of the account after uid in the pass order, so a
// pass stopped by one account's failing entry starts the next pass with
// the others; zero when no account stopped the pass.
func nextAfter(runs []*accountRun, uid int) int {
if uid == 0 {
return 0
}
for i, run := range runs {
if run.a.UID == uid {
return runs[(i+1)%len(runs)].a.UID
}
}
return 0
}
func (w *Watcher) saveCursor(uid int, r *PassReport) {
var cur int
err := w.cfg.State.View(func(tx StateTx) (e error) { cur, e = tx.Cursor(); return e })
if err == nil && cur == uid {
return
}
if err == nil {
err = w.cfg.State.Update(func(tx StateTx) error { return tx.SetCursor(uid) })
}
if err != nil {
r.Skipped[SkipStateUnavailable]++
r.Unfinished = true
}
}
func (w *Watcher) prune(runs []*accountRun, r *PassReport, accountsComplete bool, unchecked []int) {
var folders []FolderKey
uncheckedAccounts := slices.Clone(unchecked)
var unreadableFolders []FolderKey
for _, run := range runs {
if !run.inventoryComplete {
uncheckedAccounts = append(uncheckedAccounts, run.a.UID)
}
for k, done := range run.subsDone {
if done == len(Subdirs) && !run.subsFailed[k] {
folders = append(folders, k)
}
if run.subsFailed[k] {
unreadableFolders = append(unreadableFolders, k)
}
}
}
due := false
err := w.cfg.State.View(func(tx StateTx) error {
last, err := tx.LastPrune()
due = err == nil && tx.now.Sub(last) >= pruneEvery
return err
})
if err == nil && due {
err = w.cfg.State.Update(func(tx StateTx) error {
if _, pruneErr := tx.Prune(folders); pruneErr != nil {
return pruneErr
}
// Absence is proven only when every registry name has a
// reporter the state can be checked against.
if accountsComplete {
if _, pruneErr := tx.PruneAbsentFolders(uncheckedAccounts, unreadableFolders); pruneErr != nil {
return pruneErr
}
}
return tx.SetLastPrune(tx.now)
})
}
if err != nil {
r.Skipped[SkipStateUnavailable]++
r.Unfinished = true
}
}
// tally adds every run's waiting reports; a run cut short leaves folders
// unenumerated, which Unfinished reports.
func (w *Watcher) tally(runs []*accountRun, r *PassReport) {
for _, run := range runs {
for _, q := range run.queues {
for _, items := range [][]PendingItem{q.fresh, q.retry, q.waiting} {
for _, it := range items {
r.Pending++
if r.OldestPending.IsZero() || it.FirstSeen.Before(r.OldestPending) {
r.OldestPending = it.FirstSeen
}
}
}
}
if !run.done {
r.Unfinished = true
}
}
}
type folderQueue struct {
key FolderKey
fresh, retry, waiting []PendingItem
preferRetry bool
}
// accountRun is one account's progress through a pass: discovery walks its
// folders one chunk per turn; candidates alternate fresh and retry queues
// across folders, one attempt per turn, MaxAttempts per pass.
type accountRun struct {
w *Watcher
a Account
deadline time.Time
// junk is a Postfix reporter's folders with their owners' uids.
junk map[FolderKey]int
home *Home
started, done bool
paused bool
folders []FolderKey
fi, si int
cur *Folder
curKey FolderKey
curSub string
curID DirIdent
curIncomplete bool
subsDone map[FolderKey]int
subsFailed map[FolderKey]bool
cappedFolders map[FolderKey]bool
queues []*folderQueue
qi int
attempts int
waitFor *accountRun
turnOpens int
seenCount int
capReported bool
inventoryComplete bool
}
func (r *accountRun) close() {
if r.cur != nil {
_ = r.cur.Close()
r.cur = nil
}
if r.home != nil {
_ = r.home.Close()
r.home = nil
}
}
func (r *accountRun) state() *State { return r.w.cfg.State }
func (r *accountRun) stateFailed(err error, rep *PassReport) bool {
if err == nil {
return false
}
rep.Skipped[SkipStateUnavailable]++
rep.Unfinished = true
r.done = true
r.inventoryComplete = false
return true
}
func (r *accountRun) expired(ctx context.Context) bool {
return ctx.Err() != nil || !r.w.cfg.Now().Before(r.deadline)
}
func (r *accountRun) turn(ctx context.Context, rep *PassReport) {
if ctx.Err() != nil || !r.w.cfg.CanDiscover() {
return
}
r.turnOpens = 0
if !r.started {
r.started = true
if !r.start(ctx, rep) {
r.done = true
return
}
}
if r.discovering() {
r.discover(rep)
}
if r.expired(ctx) {
rep.Unfinished = true
return
}
if !r.discovering() && r.attempts == 0 {
// Persisted attempt times rotate folders across passes even when
// there are more nonempty folders than the per-account budget.
slices.SortStableFunc(r.queues, func(a, b *folderQueue) int {
if order := a.oldestTurn().Compare(b.oldestTurn()); order != 0 {
return order
}
if a.preferRetry == b.preferRetry {
return 0
}
if a.preferRetry {
return 1
}
return -1
})
}
if !r.done && r.canAttempt() {
for r.waitFor != nil {
if !r.waitFor.done && r.waitFor.attempts == 0 {
return
}
r.waitFor = r.waitFor.waitFor
}
r.attempt(ctx, rep)
}
if !r.done && !r.discovering() && !r.canAttempt() {
r.done = true
}
}
func (r *accountRun) start(ctx context.Context, rep *PassReport) bool {
if r.w.cfg.Postfix != nil {
// A Postfix reporter's folders come from a complete snapshot; each
// mail root is pinned and owner-checked when opened.
rep.Accounts++
r.folders = slices.SortedFunc(maps.Keys(r.junk), func(a, b FolderKey) int {
return cmp.Or(strings.Compare(a.Box, b.Box), strings.Compare(a.Folder, b.Folder))
})
r.inventoryComplete = true
} else if !r.startHome(ctx, rep) {
return false
}
r.subsDone, r.subsFailed = map[FolderKey]int{}, map[FolderKey]bool{}
r.cappedFolders = map[FolderKey]bool{}
return !r.stateFailed(r.state().View(func(tx StateTx) (e error) { r.seenCount, e = tx.CountSeen(r.a.UID); return e }), rep)
}
// startHome pins a cPanel account's home and lists its mailboxes.
func (r *accountRun) startHome(ctx context.Context, rep *PassReport) bool {
fsAccount := r.a
fsAccount.UID = ownerUID(r.a)
home, err := OpenHome(fsAccount)
if err != nil {
rep.Skipped[SkipHomeUnavailable]++
rep.Unfinished = true
return false
}
r.home = home
if !r.eligible(ctx, rep) {
return false
}
rep.Accounts++
boxes, skips, err := home.Mailboxes()
r.inventoryComplete = err == nil && len(skips) == 0
for _, s := range skips {
rep.Skipped[s.Reason]++
if s.Reason == SkipMailboxLimit || s.Reason == SkipMailboxListTooLarge || s.Reason == SkipUnsafePath {
rep.Unfinished = true
}
}
if err != nil {
rep.Skipped[SkipFolderUnreadable]++
rep.Unfinished = true
}
for _, m := range boxes {
for _, f := range Folders {
r.folders = append(r.folders, FolderKey{UID: r.a.UID, Domain: m.Domain, Box: m.Box, Folder: f})
}
}
return true
}
// open opens one subdirectory of an observed folder.
func (r *accountRun) open(k FolderKey, sub string) (*Folder, error) {
if r.w.cfg.Postfix != nil {
return OpenJunk(k.Folder, sub, r.owner(k))
}
return r.home.OpenFolder(mailboxOf(k), k.Folder, sub)
}
// owner is the UID the files of k must belong to: the account's, or the
// userdb uid of a Postfix mailbox, never its reporter id. A Postfix folder
// that is not one of r.junk's gets -1, which owns no file, so it is refused
// as unsafe.
func (r *accountRun) owner(k FolderKey) int {
if r.w.cfg.Postfix == nil {
return ownerUID(r.a)
}
uid, ok := r.junk[k]
if !ok {
return -1
}
return ownerUID(Account{UID: uid})
}
// eligible rechecks the Spam Filters switch. An opt-out cancels the
// account's unsent work, the learner's reservations included, and starts
// new baselines on re-entry.
func (r *accountRun) eligible(ctx context.Context, rep *PassReport) bool {
if r.w.cfg.Postfix != nil {
return true // Postfix has no Spam Filters switch.
}
global, err := GlobalOptIn(r.w.cfg.GlobalPath)
if err != nil {
rep.Skipped[SkipOptInUnreadable]++
rep.Unfinished = true
return false
}
on, reason, err := r.home.Eligible(global)
if err != nil {
rep.Skipped[SkipOptInUnreadable]++
rep.Unfinished = true
return false
}
if on {
return true
}
rep.Skipped[reason]++
// Asked on every pass, since an explicit disable can drop the folder
// state of an account the learner still holds work for. A failure is
// retried by the next pass.
// The folder baseline must reset even if the learner cannot cancel its
// reservations. Otherwise re-entry could submit mail from the opt-out.
r.stateFailed(r.w.cfg.Engine.CancelUnsent(ctx, r.a.UID, nil), rep)
var has bool
if r.stateFailed(r.state().View(func(tx StateTx) (e error) { has, e = tx.HasState(r.a.UID); return e }), rep) {
return false
}
if has && r.stateFailed(r.state().Update(func(tx StateTx) error { return tx.ResetOnOptOut(r.a.UID) }), rep) {
return false
}
r.queues = nil
return false
}
func (r *accountRun) discovering() bool { return r.cur != nil || r.fi < len(r.folders) }
func (r *accountRun) advance() {
if r.si++; r.si == len(Subdirs) {
r.si = 0
r.fi++
}
}
func mailboxOf(k FolderKey) Mailbox { return Mailbox{Domain: k.Domain, Box: k.Box} }
// discover lists one chunk. Opening folders and settling unchanged or
// missing ones costs no listing, so it runs before and after the chunk.
func (r *accountRun) discover(rep *PassReport) {
r.openNext(rep)
if r.cur == nil {
return
}
turnHook(r.a.UID, "chunk")
if !r.w.cfg.CanDiscover() {
return
}
names, err := r.cur.Next(r.w.cfg.Chunk)
r.observe(names, rep)
if r.done {
return
}
if err == nil && len(names) < r.w.cfg.Chunk {
// A short batch ends the directory; confirm it in this turn.
if !r.w.cfg.CanDiscover() {
return
}
names, err = r.cur.Next(r.w.cfg.Chunk)
r.observe(names, rep)
}
if r.done {
return
}
switch {
case err == io.EOF:
k, sub, id, incomplete := r.curKey, r.curSub, r.curID, r.curIncomplete
_ = r.cur.Close()
r.cur = nil
if !incomplete && r.stateFailed(r.state().Update(func(tx StateTx) error { return tx.Listed(k, sub, id) }), rep) {
return
}
r.subComplete(k, rep)
case err != nil:
_ = r.cur.Close()
r.cur = nil
rep.Skipped[SkipFolderUnreadable]++
r.subFailed(r.curKey, rep)
}
r.openNext(rep)
}
// openNext opens the next folder that needs listing, settling missing,
// unsafe and unchanged ones on the way.
func (r *accountRun) openNext(rep *PassReport) {
for !r.done && r.cur == nil && r.fi < len(r.folders) && r.w.cfg.CanDiscover() {
if r.turnOpens >= r.w.cfg.FolderOpens {
return
}
r.turnOpens++
k, sub := r.folders[r.fi], Subdirs[r.si]
r.advance()
turnHook(r.a.UID, "open")
f, err := r.open(k, sub)
switch {
case errors.Is(err, fs.ErrNotExist):
r.subComplete(k, rep)
continue
case errors.Is(err, ErrUnsafePath):
rep.Skipped[SkipUnsafePath]++
r.subFailed(k, rep)
continue
case err != nil:
rep.Skipped[SkipFolderUnreadable]++
r.subFailed(k, rep)
continue
}
dev, ino, mtime, ctime := f.Identity()
id := DirIdent{Dev: dev, Ino: ino, MtimeNs: mtime, CtimeNs: ctime}
var unchanged bool
err = r.state().View(func(tx StateTx) (e error) { unchanged, e = tx.Unchanged(k, sub, id); return e })
if err == nil && !unchanged {
err = r.state().Update(func(tx StateTx) (e error) { _, unchanged, e = tx.Open(k, sub, id); return e })
}
if err != nil {
_ = f.Close()
r.stateFailed(err, rep)
return
}
if unchanged {
_ = f.Close()
r.subComplete(k, rep)
continue
}
r.cur, r.curKey, r.curSub, r.curID, r.curIncomplete = f, k, sub, id, false
}
}
type observed struct {
name, base string
meta Meta
status ReadStatus
}
// observe stats a chunk's entries, then records them in one transaction.
func (r *accountRun) observe(names []string, rep *PassReport) {
var batch []observed
for _, name := range names {
base, ok := BaseName(name)
if !ok {
rep.Skipped[SkipUnsupportedName]++
continue
}
m, st, err := r.cur.Stat(name)
if err != nil {
r.curIncomplete = true
r.subsFailed[r.curKey] = true
rep.Skipped[SkipFolderUnreadable]++
rep.Unfinished = true
continue
}
if st == ReadGone {
continue
}
batch = append(batch, observed{name: name, base: base, meta: m, status: st})
}
if len(batch) == 0 {
return
}
k, sub := r.curKey, r.curSub
seenCount, capped := r.seenCount, false
skips := map[string]int{}
err := r.state().Update(func(tx StateTx) error {
for _, o := range batch {
if seenCount >= r.w.cfg.SeenAllowance {
if _, found, err := tx.Seen(k, o.base); err != nil {
return err
} else if !found {
capped = true
continue
}
}
cand, created, err := tx.Observe(k, sub, o.name, o.meta)
if err != nil {
return err
}
if created {
seenCount++
}
if cand && o.status != ReadOK {
skips[o.status.String()]++
if err = tx.Mark(k, o.base, SkippedStatus(o.status.String())); err != nil {
return err
}
}
}
return nil
})
if r.stateFailed(err, rep) {
return
}
r.seenCount = seenCount
for reason, count := range skips {
rep.Skipped[reason] += count
}
if capped {
r.capped(rep)
}
}
// capped records that the account's seen allowance stopped discovery of
// new entries; listing goes on so pending reports are still served.
func (r *accountRun) capped(rep *PassReport) {
r.curIncomplete = true
r.cappedFolders[r.curKey] = true
rep.Unfinished = true
r.reportCapacity(rep)
}
func (r *accountRun) reportCapacity(rep *PassReport) {
if !r.capReported {
r.capReported = true
rep.Skipped[learn.SkipAccountCapacity]++
}
}
func (r *accountRun) subFailed(k FolderKey, rep *PassReport) {
r.subsFailed[k] = true
rep.Unfinished = true
r.subComplete(k, rep)
}
// subComplete counts a finished subdirectory; once both are done the
// folder's baseline may finish and its pending reports join the queues.
func (r *accountRun) subComplete(k FolderKey, rep *PassReport) {
if r.subsDone[k]++; r.subsDone[k] < len(Subdirs) {
return
}
var items []PendingItem
finish := false
err := r.state().View(func(tx StateTx) error {
st, ok, err := tx.Folder(k)
if err != nil || !ok {
return err
}
finish = !st.BaselineDone && !r.subsFailed[k] && !r.cappedFolders[k]
items, err = tx.Pending(k)
return err
})
if err == nil && finish {
err = r.state().Update(func(tx StateTx) error { return tx.FinishBaseline(k) })
}
if r.stateFailed(err, rep) || len(items) == 0 {
return
}
q := &folderQueue{key: k}
for _, it := range items {
if it.Attempts == 0 {
q.fresh = append(q.fresh, it)
} else {
q.retry = append(q.retry, it)
}
}
sort.SliceStable(q.fresh, func(i, j int) bool { return q.fresh[i].FirstSeen.Before(q.fresh[j].FirstSeen) })
sort.SliceStable(q.retry, func(i, j int) bool { return q.retry[i].LastAttempt.Before(q.retry[j].LastAttempt) })
q.preferRetry = len(q.retry) > 0 && (len(q.fresh) == 0 || q.retry[0].LastAttempt.Before(q.fresh[0].FirstSeen))
r.queues = append(r.queues, q)
}
func (q *folderQueue) oldestTurn() time.Time {
if len(q.retry) == 0 {
return q.fresh[0].FirstSeen
}
if len(q.fresh) == 0 || q.retry[0].LastAttempt.Before(q.fresh[0].FirstSeen) {
return q.retry[0].LastAttempt
}
return q.fresh[0].FirstSeen
}
func (r *accountRun) canAttempt() bool {
// Complete discovery before spending the attempt budget, so a slow
// later folder gets the same share as the first one.
if r.discovering() || r.paused || r.w.submitStopped || r.attempts >= MaxAttempts || !r.w.cfg.CanDiscover() || !r.w.cfg.CanSubmit() {
return false
}
for _, q := range r.queues {
if len(q.fresh)+len(q.retry) > 0 {
return true
}
}
return false
}
// next takes the next item round-robin across folders, alternating fresh
// and retry queues; each item is taken at most once per pass.
func (r *accountRun) next() (*folderQueue, PendingItem) {
for range r.queues {
q := r.queues[r.qi%len(r.queues)]
r.qi++
first, second := &q.fresh, &q.retry
if q.preferRetry {
first, second = second, first
}
for _, list := range []*[]PendingItem{first, second} {
if len(*list) > 0 {
it := (*list)[0]
*list = (*list)[1:]
q.preferRetry = !q.preferRetry
return q, it
}
}
}
return nil, PendingItem{}
}
func (r *accountRun) attempt(ctx context.Context, rep *PassReport) {
q, it := r.next()
if q == nil {
return
}
r.attempts++
turnHook(r.a.UID, "attempt")
hold := func() { q.waiting = append(q.waiting, it) }
k := q.key
retry := func() {
hold()
r.stateFailed(r.state().Update(func(tx StateTx) error { return tx.Retry(k, it.Base) }), rep)
}
mark := func(status string) {
if r.stateFailed(r.state().Update(func(tx StateTx) error { return tx.Mark(k, it.Base, status) }), rep) {
hold()
}
}
forget := func() {
if r.stateFailed(r.state().Update(func(tx StateTx) error { return tx.Forget(k, it.Base) }), rep) {
hold()
}
}
allowed := func() bool {
if r.expired(ctx) {
rep.Unfinished = true
hold()
return false
}
if !r.w.cfg.CanDiscover() || !r.w.cfg.CanSubmit() {
hold()
r.paused = true
return false
}
return true
}
if !allowed() {
return
}
f, err := r.open(k, it.Sub)
if errors.Is(err, fs.ErrNotExist) {
// The folder went with the message; an unsent report is not history.
forget()
return
}
if err != nil {
retry()
return
}
dev, ino, mtime, ctime := f.Identity()
// Both subdirectories share a baseline. Replacing the sibling also
// cancels this report, even if its own directory is unchanged.
ids := map[string]DirIdent{it.Sub: {Dev: dev, Ino: ino, MtimeNs: mtime, CtimeNs: ctime}}
for _, sub := range Subdirs {
if sub == it.Sub {
continue
}
sibling, openErr := r.open(k, sub)
if errors.Is(openErr, fs.ErrNotExist) {
continue
}
if openErr != nil {
_ = f.Close()
retry()
return
}
dev, ino, mtime, ctime := sibling.Identity()
ids[sub] = DirIdent{Dev: dev, Ino: ino, MtimeNs: mtime, CtimeNs: ctime}
_ = sibling.Close()
}
var baseline bool
err = r.state().Update(func(tx StateTx) error {
for _, sub := range Subdirs {
id, exists := ids[sub]
if !exists {
continue
}
b, _, openErr := tx.Open(k, sub, id)
if openErr != nil {
return openErr
}
baseline = baseline || b
}
return nil
})
if r.stateFailed(err, rep) {
_ = f.Close()
hold()
return
}
if baseline {
_ = f.Close()
q.fresh, q.retry, q.waiting = nil, nil, nil
rep.Unfinished = true
return
}
if !r.eligible(ctx, rep) {
_ = f.Close()
if len(r.queues) > 0 {
hold()
}
r.done = true
return
}
if !allowed() {
_ = f.Close()
return
}
snap, meta, st, err := f.ReadMessage(it.Name, r.owner(k))
_ = f.Close()
if !allowed() {
return
}
switch {
case err != nil, st == ReadPendingChanged:
retry()
return
case st == ReadGone:
forget()
return
case st != ReadOK && st != ReadPendingLinked:
rep.Skipped[st.String()]++
mark(SkippedStatus(st.String()))
return
}
if v := ParseVerdict(snap); v != VerdictEligible {
rep.Skipped[v.String()]++
mark(SkippedStatus(v.String()))
return
}
// A move links the message into its new folder before unlinking the old
// one, in either direction. Count a linked message only once its links
// and content have stayed unchanged for a pass interval; a lasting copy,
// such as one delivery shared by several mailboxes, then counts.
if st == ReadPendingLinked {
var settled bool
err = r.state().Update(func(tx StateTx) (e error) {
settled, e = tx.LinkedSettled(k, it.Base, meta)
return e
})
if r.stateFailed(err, rep) {
hold()
return
}
if !settled {
retry()
return
}
}
if !r.eligible(ctx, rep) {
if len(r.queues) > 0 {
hold()
}
r.done = true
return
}
if !allowed() {
return
}
turnHook(r.a.UID, "submit")
out, err := r.w.cfg.Engine.SubmitReport(ctx, learn.Candidate{UID: r.a.UID, Snapshot: snap})
switch {
case err != nil || out.Err != nil:
rejected := err == nil && errors.Is(out.Err, learn.ErrMessageRejected)
if !rejected {
r.w.submitStopped, r.w.stoppedBy = true, r.a.UID
}
if !allowed() {
return
}
failure := errors.Join(err, out.Err)
// A failed inspection reply while storage answers the probe comes
// from this body, as a rejection does, so the pass goes on.
if !rejected && ctx.Err() == nil && !hostSideFailure(failure) &&
errors.Is(failure, learn.ErrInspectionFailed) {
probeErr := r.w.cfg.Engine.ProbeStorage(ctx)
if !allowed() {
return
}
// A failed probe overrides earlier answers: storage may have
// stopped responding since those reports were processed.
if probeErr != nil {
retry()
return
}
rejected, r.w.answered = true, true
r.w.submitStopped, r.w.stoppedBy = false, 0
}
// Only failures known to come from the pass, the host or local
// learning state always spare the message's allowance. A message
// cannot cancel a context or break the ledger; it can cause any
// other error, but so can an outage. Charge an ambiguous failure
// only when storage answered earlier in this pass; pre-query
// rejections say nothing about its health. The next pass starts
// after this account, so other accounts' answers come first.
if ctx.Err() != nil || hostSideFailure(failure) || !rejected && !r.w.answered {
retry()
return
}
// Each entry gets MaxEntryErrors charged tries.
var gaveUp bool
if r.stateFailed(r.state().Update(func(tx StateTx) (e error) { gaveUp, e = tx.Fail(k, it.Base); return e }), rep) || !gaveUp {
hold()
} else {
rep.Skipped[SkipFailed]++
}
case out.Deferred:
retry()
case slices.ContainsFunc(out.Skips, func(s learn.Skip) bool { return s.Reason == learn.SkipAccountCapacity }):
r.w.answered = true
retry()
r.paused = true
r.reportCapacity(rep)
default:
// An empty inspection needs no storage query. Preserve any earlier
// answer, but do not infer storage health from this report.
if !slices.ContainsFunc(out.Skips, func(s learn.Skip) bool { return s.Reason == learn.SkipNoDigest }) {
r.w.answered = true
}
rep.Submitted++
rep.Committed += len(out.Committed)
for _, s := range out.Skips {
rep.Skipped[s.Reason]++
}
mark(StatusSubmitted)
}
}
package learnwatch
import (
"bytes"
"encoding/binary"
"encoding/json"
"errors"
"time"
"github.com/pidginhost/csm/internal/store"
)
// SeenRetention keeps entries absent from their folder this long;
// lastSeenRefresh bounds presence writes to one per entry per day.
const (
SeenRetention = 90 * 24 * time.Hour
lastSeenRefresh = 24 * time.Hour
)
// Seen statuses. A skip is "skipped:<reason>".
const (
StatusBaseline = "baseline"
StatusPending = "pending"
StatusSubmitted = "submitted"
)
// SkippedStatus is the terminal status for reason.
func SkippedStatus(reason string) string { return "skipped:" + reason }
var errNoFolder = errors.New("learnwatch: folder has no state; open it first")
// FolderKey names one observed folder of one mailbox; its new and cur
// subdirectories share the seen identities. UID is the account's UID or a
// Postfix reporter id; a Postfix Folder is the folder's absolute path.
type FolderKey struct {
UID int
Domain, Box, Folder string
}
// uidPrefix is 8 bytes big-endian, so account UIDs keep their keys and
// reporter ids, below 2^62, start below the meta and pending keys.
func uidPrefix(uid int) []byte {
// #nosec G115 -- UIDs and reporter ids are never negative
return binary.BigEndian.AppendUint64(nil, uint64(uid))
}
// Names never contain NUL, so the separators keep keys unambiguous.
func (k FolderKey) key() []byte {
b := append(uidPrefix(k.UID), 0)
b = append(append(b, k.Domain...), 0)
b = append(append(b, k.Box...), 0)
return append(b, k.Folder...)
}
func seenKey(k FolderKey, base string) []byte {
return append(append(k.key(), 0), base...)
}
// DirIdent is a subdirectory's identity and timestamps.
type DirIdent struct {
Dev, Ino uint64
MtimeNs, CtimeNs int64
}
// FolderState is one folder's baseline. Listed holds each subdirectory's
// identity as of its last complete enumeration; Dirs its current one.
type FolderState struct {
Dirs map[string]DirIdent `json:"dirs"`
Listed map[string]DirIdent `json:"listed,omitempty"`
ListedAt map[string]time.Time `json:"listed_at,omitempty"`
// PresentAt is when the folder was last opened, refreshed at most
// daily; a folder absent for SeenRetention is pruned with its entries.
PresentAt time.Time `json:"present_at"`
BaselineDone bool `json:"baseline_done"`
Epoch uint64 `json:"epoch"`
}
// SeenEntry is one message identity of a folder. Sub and Name are where
// the last listing saw it; Attempts and LastAttempt order the retry queue.
type SeenEntry struct {
Sub string `json:"sub,omitempty"`
Name string `json:"name,omitempty"`
Attempts int `json:"attempts,omitempty"`
Errors int `json:"errors,omitempty"`
LastAttempt time.Time `json:"last_attempt,omitzero"`
Dev uint64 `json:"dev"`
Ino uint64 `json:"ino"`
Size int64 `json:"size"`
MtimeNs int64 `json:"mtime_ns"`
CtimeNs int64 `json:"ctime_ns"`
Status string `json:"status"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
// StableSince tracks the current version, independently of queue age.
// Ctime changes when a move adds or removes a hard link elsewhere.
StableSince time.Time `json:"stable_since,omitzero"`
}
func (e *SeenEntry) setMeta(m Meta, now time.Time) {
if e.StableSince.IsZero() || !e.sameContent(m) {
e.StableSince = now
}
e.Dev, e.Ino, e.Size, e.MtimeNs, e.CtimeNs = m.Dev, m.Ino, m.Size, m.MtimeNs, m.CtimeNs
}
func (e SeenEntry) sameContent(m Meta) bool {
return e.Dev == m.Dev && e.Ino == m.Ino && e.Size == m.Size && e.MtimeNs == m.MtimeNs && e.CtimeNs == m.CtimeNs
}
type kv interface {
LearnUpdate(func(store.LearnTx) error) error
LearnView(func(store.LearnTx) error) error
}
// State is the watcher's persisted folder and seen state.
type State struct {
db kv
now func() time.Time
}
// NewState returns the state over db; now reads the clock.
func NewState(db kv, now func() time.Time) *State {
if now == nil {
now = time.Now
}
return &State{db: db, now: now}
}
// Update runs fn in one write transaction; callers batch a discovery chunk.
func (s *State) Update(fn func(StateTx) error) error {
now := s.now()
return s.db.LearnUpdate(func(tx store.LearnTx) error { return fn(StateTx{tx: tx, now: now}) })
}
// View runs fn in one read transaction.
func (s *State) View(fn func(StateTx) error) error {
now := s.now()
return s.db.LearnView(func(tx store.LearnTx) error { return fn(StateTx{tx: tx, now: now}) })
}
// StateTx is one transaction over the watcher state.
type StateTx struct {
tx store.LearnTx
now time.Time
}
func getJSON[T any](tx store.LearnTx, bucket string, key []byte) (T, bool, error) {
var v T
raw, err := tx.Get(bucket, key)
if err != nil || raw == nil {
return v, false, err
}
return v, true, json.Unmarshal(raw, &v)
}
func putJSON(tx store.LearnTx, bucket string, key []byte, v any) error {
raw, err := json.Marshal(v)
if err != nil {
return err
}
return tx.Put(bucket, key, raw)
}
// Folder returns k's state.
func (t StateTx) Folder(k FolderKey) (FolderState, bool, error) {
return getJSON[FolderState](t.tx, store.LearnWatchFoldersBucket, k.key())
}
// Open records sub's current identity. A folder without state, a new
// subdirectory or a replaced one (device or inode changed) starts a new
// baseline. Unchanged means the baseline is done and sub has not changed
// since its last complete enumeration.
func (t StateTx) Open(k FolderKey, sub string, id DirIdent) (baseline, unchanged bool, err error) {
st, ok, err := t.Folder(k)
if err != nil {
return false, false, err
}
prev, known := st.Dirs[sub]
switch {
case !ok:
st = FolderState{Epoch: 1}
baseline = true
case !known || prev.Dev != id.Dev || prev.Ino != id.Ino:
// Both Maildir subdirectories belong to this baseline. Forgetting
// the sibling would make alternating opens restart it forever.
st.Epoch++
st.BaselineDone = false
st.Listed, st.ListedAt = nil, nil
baseline = true
// The sibling may already have been listed or skipped this pass.
// Cancel its old pending work too, before completing this baseline.
items, e := t.Pending(k)
if e != nil {
return false, false, e
}
for _, it := range items {
if e = t.Mark(k, it.Base, StatusBaseline); e != nil {
return false, false, e
}
}
default:
baseline = !st.BaselineDone
unchanged = st.BaselineDone && st.Listed[sub] == id && t.now.Sub(st.ListedAt[sub]) < FullRescanEvery
}
if st.Dirs == nil {
st.Dirs = map[string]DirIdent{}
}
st.Dirs[sub] = DirIdent{Dev: id.Dev, Ino: id.Ino}
st.PresentAt = t.now
return baseline, unchanged, putJSON(t.tx, store.LearnWatchFoldersBucket, k.key(), st)
}
// Unchanged is Open's unchanged answer without a write: the baseline is
// done, sub has its recorded identity and timestamps since a recent
// complete listing, and the folder's presence was recorded within a day.
// Callers write through Open only when it is false, so a pass over
// unchanged folders writes nothing.
func (t StateTx) Unchanged(k FolderKey, sub string, id DirIdent) (bool, error) {
st, ok, err := t.Folder(k)
if err != nil || !ok {
return false, err
}
known, listed := st.Dirs[sub]
return listed && st.BaselineDone && known.Dev == id.Dev && known.Ino == id.Ino &&
st.Listed[sub] == id && t.now.Sub(st.ListedAt[sub]) < FullRescanEvery &&
t.now.Sub(st.PresentAt) < lastSeenRefresh, nil
}
// Listed records a complete enumeration of sub at identity id.
func (t StateTx) Listed(k FolderKey, sub string, id DirIdent) error {
st, ok, err := t.Folder(k)
if err != nil || !ok {
return errors.Join(err, errNoFolder)
}
if cur := st.Dirs[sub]; cur.Dev != id.Dev || cur.Ino != id.Ino {
return nil
}
if st.Listed == nil {
st.Listed = map[string]DirIdent{}
}
if st.ListedAt == nil {
st.ListedAt = map[string]time.Time{}
}
st.Listed[sub], st.ListedAt[sub] = id, t.now
return putJSON(t.tx, store.LearnWatchFoldersBucket, k.key(), st)
}
// FinishBaseline marks the observation boundary: later new identities are
// candidates.
func (t StateTx) FinishBaseline(k FolderKey) error {
st, ok, err := t.Folder(k)
if err != nil || !ok {
return errors.Join(err, errNoFolder)
}
st.BaselineDone = true
return putJSON(t.tx, store.LearnWatchFoldersBucket, k.key(), st)
}
// Seen returns the entry for base.
func (t StateTx) Seen(k FolderKey, base string) (SeenEntry, bool, error) {
return getJSON[SeenEntry](t.tx, store.LearnWatchSeenBucket, seenKey(k, base))
}
// Observe records a listed message and reports whether it is a candidate:
// a new identity after the baseline, a different inode under a seen name,
// or an entry still pending. An edit to a seen inode is not a candidate.
// Created reports a new entry, which counts against the seen allowance.
func (t StateTx) Observe(k FolderKey, sub, name string, m Meta) (candidate, created bool, err error) {
base, ok := BaseName(name)
if !ok {
return false, false, errors.New("learnwatch: unusable message name")
}
st, ok, err := t.Folder(k)
if err != nil || !ok {
return false, false, errors.Join(err, errNoFolder)
}
e, found, err := t.Seen(k, base)
if err != nil {
return false, false, err
}
fresh := !found || (e.Sub == sub && e.Name == name && t.now.Sub(e.LastSeen) < lastSeenRefresh)
switch {
case found && e.Status == StatusPending && !st.BaselineDone:
// A new baseline must not replay pending work from the old folder,
// even when a basename or inode is reused in its initial corpus.
e = SeenEntry{Status: StatusBaseline, FirstSeen: t.now}
case found && e.Status == StatusPending && e.Dev == m.Dev && e.Ino == m.Ino:
if fresh && e.sameContent(m) && !e.StableSince.IsZero() {
return true, false, nil
}
case found && e.Dev == m.Dev && e.Ino == m.Ino:
if fresh {
return false, false, nil
}
default:
e = SeenEntry{Status: StatusPending, FirstSeen: t.now}
if !st.BaselineDone {
e.Status = StatusBaseline
}
}
e.setMeta(m, t.now)
e.Sub, e.Name, e.LastSeen = sub, name, t.now
return e.Status == StatusPending, !found, t.putSeen(k, base, e)
}
// LinkedSettled checks the read snapshot against the last observed version.
// Reads must also refresh it: links outside Spam do not change the watched
// directory, so discovery may have skipped its listing this pass.
func (t StateTx) LinkedSettled(k FolderKey, base string, m Meta) (bool, error) {
e, found, err := t.Seen(k, base)
if err != nil || !found {
return false, errors.Join(err, errors.New("learnwatch: no seen entry to settle"))
}
if e.StableSince.IsZero() || !e.sameContent(m) {
_, _, err = t.Observe(k, e.Sub, e.Name, m)
return false, err
}
return t.now.Sub(e.StableSince) >= PassEvery, nil
}
// Pending index keys sit in the folders bucket after every folder key,
// whose first byte is the high byte of a UID or reporter id below 2^62;
// meta keys too.
var (
cursorKey = []byte("\xfecursor")
pruneKey = []byte("\xfeprune")
)
func pendingKey(k FolderKey, base string) []byte {
return append([]byte{0xff}, seenKey(k, base)...)
}
// putSeen stores e and keeps the pending index in step with its status.
func (t StateTx) putSeen(k FolderKey, base string, e SeenEntry) error {
if err := putJSON(t.tx, store.LearnWatchSeenBucket, seenKey(k, base), e); err != nil {
return err
}
if e.Status == StatusPending {
return t.tx.Put(store.LearnWatchFoldersBucket, pendingKey(k, base), []byte{1})
}
return t.tx.Delete(store.LearnWatchFoldersBucket, pendingKey(k, base))
}
// Mark sets an existing entry's status.
func (t StateTx) Mark(k FolderKey, base, status string) error {
e, found, err := t.Seen(k, base)
if err != nil || !found {
return errors.Join(err, errors.New("learnwatch: no seen entry to mark"))
}
e.Status = status
return t.putSeen(k, base, e)
}
// Retry records a failed attempt on a pending entry: it moves to the tail
// of the retry queue.
func (t StateTx) Retry(k FolderKey, base string) error {
e, found, err := t.Seen(k, base)
if err != nil || !found {
return errors.Join(err, errors.New("learnwatch: no seen entry to retry"))
}
e.Attempts++
e.LastAttempt = t.now
return t.putSeen(k, base, e)
}
// Fail records a failed submission. After MaxEntryErrors failures the
// entry becomes skipped:failed and gaveUp is true; otherwise it moves to the
// tail of the retry queue.
func (t StateTx) Fail(k FolderKey, base string) (gaveUp bool, err error) {
e, found, err := t.Seen(k, base)
if err != nil || !found {
return false, errors.Join(err, errors.New("learnwatch: no seen entry to fail"))
}
e.Attempts++
e.Errors++
e.LastAttempt = t.now
if e.Errors >= MaxEntryErrors {
e.Status = SkippedStatus(SkipFailed)
gaveUp = true
}
return gaveUp, t.putSeen(k, base, e)
}
// Forget drops an entry, as for a vanished message.
func (t StateTx) Forget(k FolderKey, base string) error {
if err := t.tx.Delete(store.LearnWatchSeenBucket, seenKey(k, base)); err != nil {
return err
}
return t.tx.Delete(store.LearnWatchFoldersBucket, pendingKey(k, base))
}
// PendingItem is a pending entry with its base name.
type PendingItem struct {
Base string
SeenEntry
}
// Pending lists k's pending entries in base-name order.
func (t StateTx) Pending(k FolderKey) ([]PendingItem, error) {
prefix := pendingKey(k, "")
var bases []string
err := t.tx.Scan(store.LearnWatchFoldersBucket, prefix[:len(prefix)-1], func(key, _ []byte) (bool, error) {
if !bytes.HasPrefix(key, prefix) {
return false, nil
}
bases = append(bases, string(key[len(prefix):]))
return true, nil
})
if err != nil {
return nil, err
}
items := make([]PendingItem, 0, len(bases))
for _, base := range bases {
e, found, err := t.Seen(k, base)
if err != nil {
return nil, err
}
if found && e.Status == StatusPending {
items = append(items, PendingItem{Base: base, SeenEntry: e})
}
}
return items, nil
}
// CountSeen counts uid's seen entries, the measure of its allowance.
func (t StateTx) CountSeen(uid int) (int, error) {
n := 0
prefix := append(uidPrefix(uid), 0)
err := t.tx.Scan(store.LearnWatchSeenBucket, prefix[:len(prefix)-1], func(k, _ []byte) (bool, error) {
if !bytes.HasPrefix(k, prefix) {
return false, nil
}
n++
return true, nil
})
return n, err
}
// HasState reports whether uid has any folder record.
func (t StateTx) HasState(uid int) (bool, error) {
has := false
prefix := append(uidPrefix(uid), 0)
err := t.tx.Scan(store.LearnWatchFoldersBucket, prefix[:len(prefix)-1], func(k, _ []byte) (bool, error) {
has = bytes.HasPrefix(k, prefix)
return false, nil
})
return has, err
}
// Cursor is the UID the next pass starts at; zero starts at the lowest.
func (t StateTx) Cursor() (int, error) {
raw, err := t.tx.Get(store.LearnWatchFoldersBucket, cursorKey)
if err != nil || len(raw) != 8 {
return 0, err
}
// #nosec G115 -- stored by SetCursor from a UID or reporter id below 2^62
return int(binary.BigEndian.Uint64(raw)), nil
}
// SetCursor persists the rotation cursor.
func (t StateTx) SetCursor(uid int) error {
return t.tx.Put(store.LearnWatchFoldersBucket, cursorKey, uidPrefix(uid))
}
// LastPrune is when seen entries were last pruned.
func (t StateTx) LastPrune() (time.Time, error) {
v, _, err := getJSON[time.Time](t.tx, store.LearnWatchFoldersBucket, pruneKey)
return v, err
}
// SetLastPrune records a prune.
func (t StateTx) SetLastPrune(at time.Time) error {
return putJSON(t.tx, store.LearnWatchFoldersBucket, pruneKey, at)
}
// Prune drops entries absent for SeenRetention from folders whose presence
// checks completed this pass. Pending entries are never pruned.
func (t StateTx) Prune(folders []FolderKey) (int, error) {
var stale [][]byte
for _, folder := range folders {
prefix := seenKey(folder, "")
err := t.tx.Scan(store.LearnWatchSeenBucket, folder.key(), func(k, v []byte) (bool, error) {
if !bytes.HasPrefix(k, prefix) {
return false, nil
}
var e SeenEntry
if err := json.Unmarshal(v, &e); err != nil {
return false, err
}
if e.Status != StatusPending && t.now.Sub(e.LastSeen) > SeenRetention {
stale = append(stale, bytes.Clone(k))
}
return true, nil
})
if err != nil {
return 0, err
}
}
for _, k := range stale {
if err := t.tx.Delete(store.LearnWatchSeenBucket, k); err != nil {
return 0, err
}
}
return len(stale), nil
}
// PruneAbsentFolders drops folders not opened for SeenRetention, such as
// deleted mailboxes and accounts, with every seen and pending entry: they
// can never be served and would hold the account's allowance. Failed or
// incomplete inventories cannot establish absence and keep their state.
func (t StateTx) PruneAbsentFolders(uncheckedAccounts []int, unreadableFolders []FolderKey) (int, error) {
accounts := make(map[string]bool, len(uncheckedAccounts))
for _, uid := range uncheckedAccounts {
accounts[string(uidPrefix(uid))] = true
}
folders := make(map[string]bool, len(unreadableFolders))
for _, k := range unreadableFolders {
folders[string(k.key())] = true
}
var stale [][]byte
undated := map[string]FolderState{}
err := t.tx.ForEach(store.LearnWatchFoldersBucket, func(k, v []byte) error {
if len(k) == 0 || k[0] == 0xfe || k[0] == 0xff {
return nil
}
if (len(k) >= 8 && accounts[string(k[:8])]) || folders[string(k)] {
return nil
}
var st FolderState
if err := json.Unmarshal(v, &st); err != nil {
return err
}
// Older state has no presence timestamp. Start its retention clock
// here instead of treating the zero time as proven absence.
if st.PresentAt.IsZero() {
st.PresentAt = t.now
undated[string(k)] = st
return nil
}
if t.now.Sub(st.PresentAt) > SeenRetention {
stale = append(stale, bytes.Clone(k))
}
return nil
})
if err != nil {
return 0, err
}
for k, st := range undated {
if err = putJSON(t.tx, store.LearnWatchFoldersBucket, []byte(k), st); err != nil {
return 0, err
}
}
for _, k := range stale {
if err = t.tx.Delete(store.LearnWatchFoldersBucket, k); err != nil {
return 0, err
}
if err = t.deletePrefix(store.LearnWatchSeenBucket, append(bytes.Clone(k), 0)); err != nil {
return 0, err
}
if err = t.deletePrefix(store.LearnWatchFoldersBucket, append(append([]byte{0xff}, k...), 0)); err != nil {
return 0, err
}
}
return len(stale), nil
}
// ResetOnOptOut drops every folder and seen record of uid: unsent work is
// cancelled and re-entry starts new baselines. Contribution guards live in
// the learning ledger and are untouched.
func (t StateTx) ResetOnOptOut(uid int) error {
for _, bucket := range store.WatchBuckets {
if err := t.deletePrefix(bucket, append(uidPrefix(uid), 0)); err != nil {
return err
}
}
return t.deletePrefix(store.LearnWatchFoldersBucket, append(append([]byte{0xff}, uidPrefix(uid)...), 0))
}
func (t StateTx) deletePrefix(bucket string, prefix []byte) error {
var keys [][]byte
err := t.tx.Scan(bucket, prefix[:len(prefix)-1], func(k, _ []byte) (bool, error) {
if !bytes.HasPrefix(k, prefix) {
return false, nil
}
keys = append(keys, bytes.Clone(k))
return true, nil
})
if err != nil {
return err
}
for _, k := range keys {
if err := t.tx.Delete(bucket, k); err != nil {
return err
}
}
return nil
}
package learnwatch
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
"syscall"
"github.com/pidginhost/csm/internal/antispam/dovecot"
"github.com/pidginhost/csm/internal/antispam/learn"
"github.com/pidginhost/csm/internal/safepath"
)
// Folders are the observed folders of each cPanel mailbox: .spam is
// Dovecot's Junk special-use folder, .Junk the folder cPanel creates.
var Folders = []string{".spam", ".Junk"}
// Subdirs are the Maildir subdirectories read; IMAP moves land in cur.
var Subdirs = []string{"new", "cur"}
// ErrUnsafePath: a mail path component is a symlink, not a directory, or
// a name CSM does not traverse. The layout is reported, never followed.
var ErrUnsafePath = errors.New("mail path is not a plain directory tree")
var errHomeOwner = errors.New("home directory is not owned by the account")
// Mailbox is the main mailbox (both empty) or Box@Domain.
type Mailbox struct {
Domain, Box string
}
// Meta is descriptor metadata compared before and after a read.
type Meta struct {
Dev, Ino uint64
UID int
Nlink uint64
Size int64
MtimeNs int64
CtimeNs int64
}
// ReadStatus says what a message read found. Pending statuses retry next
// pass; skip statuses are terminal; ReadGone is not a report.
type ReadStatus int
// Read statuses.
const (
ReadOK ReadStatus = iota
ReadPendingLinked
ReadPendingChanged
ReadGone
ReadSkipNotRegular
ReadSkipWrongOwner
ReadSkipOversize
ReadSkipSymlink
)
var readStatusNames = [...]string{"ok", "linked", "changed", "gone", "not_regular", "wrong_owner", "oversize", "symlink"}
func (s ReadStatus) String() string {
if s >= 0 && int(s) < len(readStatusNames) {
return readStatusNames[s]
}
return fmt.Sprintf("status(%d)", int(s))
}
// afterRead runs between the read and the second fstat; tests inject
// writers here.
var afterRead = func() {}
// Home is an account's home directory, pinned without following links and
// owned by the account.
type Home struct {
dir *safepath.Dir
path string
uid int
}
// OpenHome pins a's home from the filesystem root and checks its owner.
func OpenHome(a Account) (*Home, error) {
d, err := safepath.OpenDirNoFollow(a.Home)
if err != nil {
return nil, err
}
fi, err := d.Info()
if err == nil {
if m, ok := metaOf(fi); !ok || m.UID != a.UID {
err = errHomeOwner
}
}
if err != nil {
_ = d.Close()
return nil, err
}
return &Home{dir: d, path: filepath.Clean(a.Home), uid: a.UID}, nil
}
// Close releases the home descriptor.
func (h *Home) Close() error { return h.dir.Close() }
// plainName is one directory component CSM may traverse.
func plainName(s string) bool {
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\x00")
}
func mailboxPath(m Mailbox, folder, sub string) (string, error) {
if !slices.Contains(Folders, folder) || !slices.Contains(Subdirs, sub) {
return "", fmt.Errorf("%w: folder %q/%q is not observed", ErrUnsafePath, folder, sub)
}
switch {
case m == Mailbox{}:
return "mail", nil
case dovecot.ValidDomain(m.Domain) && dovecot.ValidMailbox(m.Box):
return "mail/" + m.Domain + "/" + m.Box, nil
}
return "", fmt.Errorf("%w: unsupported mailbox name", ErrUnsafePath)
}
// siblingFolder accepts a folder link that names another unobserved Maildir++
// folder of the same mailbox, as ".name" or as that exact absolute path. Some
// older cPanel accounts keep .spam as such a link; nothing else is followed.
func siblingFolder(target, mailboxAbs string) (string, bool) {
name := target
if strings.HasPrefix(target, "/") {
name = strings.TrimPrefix(target, mailboxAbs+"/")
if name == target {
return "", false
}
}
return name, plainName(name) && strings.HasPrefix(name, ".") && name != ".." && !slices.Contains(Folders, name)
}
// OpenFolder opens one observed Maildir subdirectory beneath the home, one
// no-follow component at a time. A missing folder wraps fs.ErrNotExist;
// symlinks and non-directories wrap ErrUnsafePath, except a folder linked to
// a sibling folder (siblingFolder). Other errors are retryable.
func (h *Home) OpenFolder(m Mailbox, folder, sub string) (*Folder, error) {
rel, err := mailboxPath(m, folder, sub)
if err != nil {
return nil, err
}
d, err := h.dir.Walk(rel + "/" + folder + "/" + sub)
if errors.Is(err, syscall.ENOTDIR) || errors.Is(err, syscall.ELOOP) {
d, err = h.openSibling(rel, folder, sub, err)
}
return openedFolder(d, err)
}
// OpenJunk opens subdirectory sub of a Postfix Junk folder: path is the
// absolute Maildir++ folder, directly under its mail root. The root is
// pinned from the filesystem root without following links and must belong
// to uid; the folder is walked as OpenFolder walks one, with no sibling
// link allowed. Errors are classified as OpenFolder's; a root another uid
// owns wraps ErrUnsafePath.
func OpenJunk(path, sub string, uid int) (*Folder, error) {
name := filepath.Base(path)
if !filepath.IsAbs(path) || filepath.Clean(path) != path || !strings.HasPrefix(name, ".") || !plainName(name) || !slices.Contains(Subdirs, sub) {
return nil, fmt.Errorf("%w: folder is not observed", ErrUnsafePath)
}
root, err := OpenHome(Account{UID: uid, Home: filepath.Dir(path)})
if errors.Is(err, errHomeOwner) {
return nil, fmt.Errorf("%w: %w", ErrUnsafePath, err)
}
if err != nil {
return openedFolder(nil, err)
}
defer func() { _ = root.Close() }()
return openedFolder(root.dir.Walk(name + "/" + sub))
}
// openedFolder classifies a walk to a folder's subdirectory and records the
// subdirectory's identity.
func openedFolder(d *safepath.Dir, err error) (*Folder, error) {
if err != nil {
if errors.Is(err, syscall.ENOTDIR) || errors.Is(err, syscall.ELOOP) {
return nil, fmt.Errorf("%w: %w", ErrUnsafePath, err)
}
return nil, err
}
fi, err := d.Info()
var ident Meta
if err == nil {
var ok bool
if ident, ok = metaOf(fi); !ok {
err = errors.ErrUnsupported
}
}
if err != nil {
_ = d.Close()
return nil, err
}
return &Folder{dir: d, ident: ident}, nil
}
// openSibling preserves the original unsafe-path error only when the folder
// is not a permitted link. Lookup failures must keep their retry semantics.
func (h *Home) openSibling(rel, folder, sub string, cause error) (*safepath.Dir, error) {
mbox, err := h.dir.Walk(rel)
if err != nil {
return nil, err
}
defer func() { _ = mbox.Close() }()
target, err := mbox.Readlink(folder)
if errors.Is(err, syscall.EINVAL) {
return nil, cause
}
if err != nil {
return nil, err
}
name, ok := siblingFolder(target, h.path+"/"+rel)
if !ok {
return nil, cause
}
return mbox.Walk(name + "/" + sub)
}
// Folder is an open Maildir subdirectory.
type Folder struct {
dir *safepath.Dir
ident Meta
}
// Close releases the folder descriptor.
func (f *Folder) Close() error { return f.dir.Close() }
// Identity is the folder's device, inode and timestamps when it was opened.
func (f *Folder) Identity() (dev, ino uint64, mtimeNs, ctimeNs int64) {
return f.ident.Dev, f.ident.Ino, f.ident.MtimeNs, f.ident.CtimeNs
}
// Next returns at most n entry names; io.EOF follows the last batch.
func (f *Folder) Next(n int) ([]string, error) {
entries, err := f.dir.ReadDirN(n)
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, e.Name())
}
return names, err
}
// Stat reads an entry's metadata without opening it. A non-nil error is
// transient and its status is meaningless.
func (f *Folder) Stat(name string) (Meta, ReadStatus, error) {
fi, err := lstatAt(f.dir, name)
if errors.Is(err, fs.ErrNotExist) {
return Meta{}, ReadGone, nil
}
if err != nil {
return Meta{}, ReadOK, err
}
m, ok := metaOf(fi)
switch {
case !ok:
return Meta{}, ReadOK, errors.ErrUnsupported
case fi.Mode()&fs.ModeSymlink != 0:
return m, ReadSkipSymlink, nil
case !fi.Mode().IsRegular():
return m, ReadSkipNotRegular, nil
}
return m, ReadOK, nil
}
// classify applies the per-file read rules to descriptor metadata.
func classify(fi os.FileInfo, m Meta, uid int) ReadStatus {
switch {
case !fi.Mode().IsRegular():
return ReadSkipNotRegular
case m.UID != uid:
return ReadSkipWrongOwner
case m.Nlink == 0:
return ReadGone
case m.Size > learn.MaxBody:
return ReadSkipOversize
case m.Nlink > 1:
return ReadPendingLinked
}
return ReadOK
}
// ReadMessage takes a bounded snapshot of one message owned by uid. The
// snapshot is returned only with ReadOK or ReadPendingLinked, when the
// descriptor metadata did not change across the read; a link elsewhere does
// not change the content, only when it may be reported. A non-nil error is
// transient; retry later.
func (f *Folder) ReadMessage(name string, uid int) ([]byte, Meta, ReadStatus, error) {
named, status, err := f.Stat(name)
if err != nil || status != ReadOK {
return nil, named, status, err
}
file, err := f.dir.OpenFile(name, os.O_RDONLY|syscall.O_NOCTTY, 0)
switch {
case errors.Is(err, fs.ErrNotExist):
return nil, Meta{}, ReadGone, nil
case errors.Is(err, syscall.ELOOP):
return nil, Meta{}, ReadSkipSymlink, nil
case errors.Is(err, syscall.ENXIO):
return nil, Meta{}, ReadSkipNotRegular, nil
case err != nil:
return nil, Meta{}, ReadOK, err
}
defer file.Close()
before, status, err := fstatMeta(file, uid)
if err != nil || status != ReadOK && status != ReadPendingLinked {
return nil, before, status, err
}
if before.Dev != named.Dev || before.Ino != named.Ino {
return nil, before, ReadPendingChanged, nil
}
data, err := io.ReadAll(io.LimitReader(file, learn.MaxBody+1))
if err != nil {
return nil, before, ReadOK, err
}
afterRead()
after, status, err := fstatMeta(file, uid)
switch {
case err != nil:
return nil, before, ReadOK, err
case status == ReadGone:
return nil, after, ReadGone, nil
case after != before || int64(len(data)) != before.Size:
return nil, after, ReadPendingChanged, nil
case len(data) > learn.MaxBody:
return nil, after, ReadSkipOversize, nil
}
// after equals before, so status is still OK or linked.
return data, before, status, nil
}
func fstatMeta(file *os.File, uid int) (Meta, ReadStatus, error) {
fi, err := file.Stat()
if err != nil {
return Meta{}, ReadOK, err
}
m, ok := metaOf(fi)
if !ok {
return Meta{}, ReadOK, errors.ErrUnsupported
}
return m, classify(fi, m, uid), nil
}
// BaseName is the folder-scoped Maildir identity: the name before the
// ":2," info suffix, which flag renames and new-to-cur moves keep.
func BaseName(name string) (string, bool) {
if !plainName(name) {
return "", false
}
base, _, _ := strings.Cut(name, ":2,")
if !plainName(base) {
return "", false
}
return base, true
}
//go:build linux
package learnwatch
import (
"os"
"syscall"
"github.com/pidginhost/csm/internal/safepath"
)
func metaOf(fi os.FileInfo) (Meta, bool) {
st, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return Meta{}, false
}
return Meta{
Dev: st.Dev, Ino: st.Ino, UID: int(st.Uid), Nlink: uint64(st.Nlink),
Size: st.Size, MtimeNs: st.Mtim.Nano(), CtimeNs: st.Ctim.Nano(),
}, true
}
func lstatAt(d *safepath.Dir, name string) (os.FileInfo, error) { return d.Lstat(name) }
package learnwatch
import "bytes"
// Header parser bounds; exceeding either makes the message malformed.
const (
MaxHeaderBytes = 256 << 10
MaxHeaderFields = 2048
)
// Verdict classifies learning eligibility from the message's header block
// and any X-Spam-Status fields, including missing or malformed headers.
type Verdict int
// Verdicts. Only VerdictEligible may become a report.
const (
VerdictEligible Verdict = iota
VerdictTagged
VerdictMalformed
)
func (v Verdict) String() string {
switch v {
case VerdictEligible:
return "eligible"
case VerdictTagged:
return "tagged"
}
return "malformed"
}
// ParseVerdict reads the bounded header block. Any X-Spam-Status that
// parses as Yes tags the message; every present one must parse as No for
// eligibility. Missing headers are eligible.
func ParseVerdict(snapshot []byte) Verdict {
values, ok := spamStatuses(snapshot)
if !ok {
return VerdictMalformed
}
v := VerdictEligible
for _, value := range values {
switch spamToken(value) {
case "yes":
return VerdictTagged
case "no":
default:
v = VerdictMalformed
}
}
return v
}
// spamStatuses returns the unfolded X-Spam-Status values of the header
// block, or false for a header that breaks the syntax or the bounds.
// Whitespace-only lines continue the previous field, so a field that some
// parser might still read is never hidden behind one.
func spamStatuses(b []byte) ([][]byte, bool) {
// Bound the newline search too, retaining one byte to detect overflow.
if len(b) > MaxHeaderBytes {
b = b[:MaxHeaderBytes+1]
}
var values [][]byte
fields, consumed := 0, 0
inStatus := false
for len(b) > 0 {
line, rest, found := bytes.Cut(b, []byte{'\n'})
consumed += len(line)
if found {
consumed++
}
if consumed > MaxHeaderBytes {
return nil, false
}
b = rest
if found {
line = bytes.TrimSuffix(line, []byte{'\r'})
}
// Bare CR and NUL can hide a field from this line-oriented parser.
if bytes.ContainsAny(line, "\r\x00") {
return nil, false
}
if len(line) == 0 {
break
}
if line[0] == ' ' || line[0] == '\t' {
if fields == 0 {
return nil, false
}
if inStatus {
values[len(values)-1] = append(values[len(values)-1], line...)
}
continue
}
name, value, ok := bytes.Cut(line, []byte{':'})
name = bytes.TrimRight(name, " \t")
if !ok || !fieldName(name) {
return nil, false
}
if fields++; fields > MaxHeaderFields {
return nil, false
}
inStatus = asciiEqualFold(name, "x-spam-status")
if inStatus {
values = append(values, bytes.Clone(value))
}
}
return values, true
}
// fieldName: printable ASCII without the colon (RFC 5322 ftext).
func fieldName(name []byte) bool {
if len(name) == 0 {
return false
}
for _, c := range name {
if c < 33 || c > 126 {
return false
}
}
return true
}
// asciiEqualFold compares ASCII letters case-insensitively and nothing
// else, so Unicode folding cannot turn other bytes into a match.
func asciiEqualFold(b []byte, lower string) bool {
if len(b) != len(lower) {
return false
}
for i, c := range b {
if c >= 'A' && c <= 'Z' {
c += 'a' - 'A'
}
if c != lower[i] {
return false
}
}
return true
}
// spamToken returns "yes" or "no" for a value that is exactly one of them
// after leading horizontal whitespace, followed by optional horizontal
// whitespace and a comma or the end; otherwise "".
func spamToken(v []byte) string {
v = bytes.TrimLeft(v, " \t")
for _, tok := range []string{"yes", "no"} {
if len(v) < len(tok) || !asciiEqualFold(v[:len(tok)], tok) {
continue
}
if rest := bytes.TrimLeft(v[len(tok):], " \t"); len(rest) == 0 || rest[0] == ',' {
return tok
}
}
return ""
}
package antispam
import (
"net/netip"
"regexp"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/eximlog"
)
// scanMarker is the text renderScan logs for one scan, in full. The sender
// field is optional so markers Exim wrote before the upgrade still count.
var scanMarker = regexp.MustCompile(`^CSM-ANTISPAM event=scan mode=(enforce|dry-run) policy=(on|off) ` +
`(?:result=ok score=(0|-?[1-9][0-9]{0,9}) action="[a-z_ ]*" decision=(unfiltered|reject|tag|none)|` +
`result=error score=none action="unavailable" decision=none)(?: ip=([0-9A-Fa-f.:]*))?$`)
// rspamdScanMarker is the record CSM's Postfix Lua logs for one scan, in
// full, from the "; " that ends rspamd's own prefix. Its submatches 1 to 5
// are scanMarker's; 6 is the queue ID.
var rspamdScanMarker = regexp.MustCompile(`^; lua; csm_scope\.lua:[0-9]+: CSM-ANTISPAM event=scan mode=(enforce|dry-run) policy=(on|off) ` +
`result=ok score=(0|-?[1-9][0-9]{0,9}) action="[a-z_ ]*" decision=(unfiltered|reject|tag|none) ip=([0-9A-Fa-f.:]*) qid=([0-9A-Za-z_]*)$`)
// Scan is one scan marker. Score is in integer tenths and valid only when
// OK; IP is invalid when the sender field is absent or empty. MsgID is
// the Exim message ID as the line wrote it, or Postfix's queue ID.
type Scan struct {
OK bool
Mode string
Policy string
Score int
Decision string
IP netip.Addr
MsgID string
}
// ParseScan reads the marker CSM's scan hook logs. The
// marker must follow the line's timestamp and message ID directly and match
// the rendered grammar in full, so text a sender controls cannot pose as one.
func ParseScan(line string) (Scan, bool) {
i := strings.Index(line, " CSM-ANTISPAM ")
// Exim writes a printable ASCII prefix; the fields check alone would
// also accept Unicode whitespace, including the NEL control character.
if i < 0 || strings.ContainsFunc(line[:i], notPrintable) || !eximlog.MessagePrefix(line[:i]) {
return Scan{}, false
}
m := scanMarker.FindStringSubmatch(line[i+1:])
if m == nil {
return Scan{}, false
}
fields := strings.Fields(line[:i])
return scanOf(m, fields[len(fields)-1])
}
// ParseRspamdScan reads the record CSM's Postfix Lua logs into rspamd's log.
// rspamd writes "<time> #<pid>(<process>) <<task>>; ",
// then "<module>; <function>: " and the message. The record must take the
// module and function places, so only csm_scope.lua logs it: before the
// first "; " a message would follow a function's ": ", and what follows
// must match the grammar in full.
func ParseRspamdScan(line string) (Scan, bool) {
i := strings.Index(line, "; ")
if i < 0 || strings.Contains(line[:i], ": ") || strings.ContainsFunc(line[:i], notPrintable) {
return Scan{}, false
}
m := rspamdScanMarker.FindStringSubmatch(line[i:])
if m == nil {
return Scan{}, false
}
return scanOf(m, m[6])
}
// notPrintable reports a rune outside printable ASCII.
func notPrintable(r rune) bool { return r < ' ' || r > '~' }
// scanOf builds the Scan of a marker's mode, policy, score, decision and
// sender submatches.
func scanOf(m []string, msgID string) (Scan, bool) {
s := Scan{Mode: m[1], Policy: m[2], Decision: "none", MsgID: msgID}
if m[5] != "" {
ip, err := netip.ParseAddr(m[5])
if err != nil {
return Scan{}, false
}
s.IP = ip.Unmap()
}
if m[4] == "" {
return s, true
}
if (m[2] == "off") != (m[4] == "unfiltered") {
// An opted-out recipient's scan never decides; an opted-in one does.
return Scan{}, false
}
// Exim formats $spam_score_int with %d from a signed C int. A digit
// count alone would also accept values outside its range.
score, err := strconv.ParseInt(m[3], 10, 32)
if err != nil {
return Scan{}, false
}
s.OK, s.Score, s.Decision = true, int(score), m[4]
return s, true
}
// ScanMarker reports whether line is the marker CSM's scan hook logs, and
// whether that scan succeeded.
func ScanMarker(line string) (marker, ok bool) {
s, marker := ParseScan(line)
return marker, s.OK
}
package antispam
import (
"bytes"
"encoding/json"
"errors"
"io"
"net"
"net/netip"
"path"
"slices"
"strconv"
"strings"
)
// Postconf is what CSM reads from postfix's main.cf.
type Postconf struct {
// Milters are smtpd_milters as postfix names them, such as
// inet:127.0.0.1:11332, without per-milter settings.
Milters []string
DefaultAction string
// SizeLimit is message_size_limit; 0 means no limit.
SizeLimit int64
QueueDirectory string
Version string
// AuthUser is set when milter_mail_macros sends {auth_authen}, the
// name a client authenticated as. rspamd knows authenticated mail by it.
AuthUser bool
}
// PostconfNames are the parameters the collector reads with one
// `postconf -x -h`, which prints one value per line in this order.
func PostconfNames() []string {
return []string{"smtpd_milters", "milter_default_action", "message_size_limit", "queue_directory", "mail_version", "milter_mail_macros"}
}
// ParsePostconf reads `postconf -x -h` for PostconfNames.
func ParsePostconf(out string) (Postconf, error) {
lines := strings.Split(strings.TrimSuffix(out, "\n"), "\n")
if len(lines) != len(PostconfNames()) {
return Postconf{}, errors.New("postconf did not print one line per parameter")
}
size, err := strconv.ParseInt(strings.TrimSpace(lines[2]), 10, 64)
if err != nil || size < 0 {
return Postconf{}, errors.New("postconf message_size_limit is not a byte count")
}
queue := strings.TrimSpace(lines[3])
if !strings.HasPrefix(queue, "/") {
return Postconf{}, errors.New("postconf queue_directory is not an absolute path")
}
p := Postconf{DefaultAction: strings.TrimSpace(lines[1]), SizeLimit: size, QueueDirectory: queue, Version: strings.TrimSpace(lines[4])}
// Postfix sends a macro under the name the list gives, and rspamd reads
// the user only from the braced name.
macros := strings.FieldsFunc(lines[5], func(r rune) bool { return strings.ContainsRune(", \t\r", r) })
p.AuthUser = slices.Contains(macros, "{auth_authen}")
// Postfix separates list items with whitespace or commas; braces group
// a milter with its name=value settings.
for _, item := range strings.FieldsFunc(lines[0], func(r rune) bool { return strings.ContainsRune(", \t\r{}", r) }) {
if !strings.Contains(item, "=") {
p.Milters = append(p.Milters, item)
}
}
return p, nil
}
// SmtpdChrootFields are the master.cf fields the collector reads with
// `postconf -F`: which program serves port 25, and the chroot flag of that
// smtpd, or of the one postscreen passes connections to.
func SmtpdChrootFields() []string {
return []string{"smtp/inet/command", "smtp/inet/chroot", "smtpd/pass/chroot"}
}
// SmtpdChroot reads `postconf -F` for SmtpdChrootFields and reports whether
// the smtpd that receives port-25 mail runs chrooted. version is
// mail_version: "-" selects the default, which became n in Postfix 3.0.
func SmtpdChroot(out, version string) (bool, error) {
fields := make(map[string]string)
for _, line := range strings.Split(out, "\n") {
if name, value, ok := strings.Cut(line, "="); ok {
fields[strings.TrimSpace(name)] = strings.TrimSpace(value)
}
}
command := strings.Fields(fields["smtp/inet/command"])
if len(command) == 0 {
return false, errors.New("postconf -F did not report the port 25 service")
}
var flag string
switch path.Base(command[0]) {
case "smtpd":
flag = "smtp/inet/chroot"
case "postscreen":
flag = "smtpd/pass/chroot"
default:
return false, errors.New("port 25 is served by neither smtpd nor postscreen")
}
switch fields[flag] {
case "y":
return true, nil
case "n":
return false, nil
case "-":
major, _, _ := strings.Cut(version, ".")
if n, err := strconv.Atoi(major); err == nil {
return n < 3, nil
}
return false, errors.New("postfix mail_version is not readable")
}
return false, errors.New("postconf -F did not report a readable chroot flag")
}
// MilterAddrs returns the address CSM, which runs outside postfix's chroot,
// reaches each milter at: inet: milters as written, unix: paths under
// queue_directory when smtpd reads them there. Postfix daemons run in the
// queue directory, so a relative path always is; an absolute path is when
// smtpd runs chrooted, which master, `postconf -F` for SmtpdChrootFields,
// shows. When it does not, absolute unix: milters are left out and complete
// is false. Milters of other transports are left out.
func (p Postconf) MilterAddrs(master CommandResult) (addrs []string, complete bool) {
complete = true
queue := strings.TrimRight(p.QueueDirectory, "/")
for _, m := range p.Milters {
if strings.HasPrefix(m, "inet:") {
addrs = append(addrs, m)
continue
}
sock, ok := strings.CutPrefix(m, "unix:")
if !ok || sock == "" {
continue
}
if strings.HasPrefix(sock, "/") {
chroot, err := SmtpdChroot(master.Stdout, p.Version)
if master.Err != nil || err != nil {
complete = false
continue
}
if !chroot {
addrs = append(addrs, m)
continue
}
}
// The kernel resolves the joined path; cleaning it could change
// its target where a symlink precedes "..".
addrs = append(addrs, "unix:"+queue+"/"+strings.TrimPrefix(sock, "/"))
}
return addrs, complete
}
// ProxyMilterAddrs reads `rspamadm configdump -j worker`, which prints one
// object per worker, and returns the bind addresses of the running
// rspamd_proxy workers that speak the milter protocol. The dump also holds
// the controller password, so errors never quote it.
func ProxyMilterAddrs(workerDump []byte) ([]string, error) {
proxies, err := milterProxies(workerDump)
if err != nil {
return nil, err
}
var addrs []string
for _, p := range proxies {
addrs = append(addrs, p.binds...)
}
return addrs, nil
}
// milterProxy is one running rspamd_proxy worker that speaks the milter
// protocol: its settings and bind addresses.
type milterProxy struct {
fields map[string]json.RawMessage
binds []string
}
// milterProxies returns the milter proxy workers in a worker dump.
func milterProxies(workerDump []byte) ([]milterProxy, error) {
dec := json.NewDecoder(bytes.NewReader(workerDump))
var out []milterProxy
for n := 0; ; n++ {
var entry json.RawMessage
if err := dec.Decode(&entry); errors.Is(err, io.EOF) && n > 0 {
return out, nil
} else if err != nil {
return nil, errors.New("configdump worker: invalid JSON")
}
proxy, err := workerOfKind(entry, "rspamd_proxy")
if err != nil {
return nil, err
}
if proxy == nil {
continue
}
fields, binds, err := milterBinds(proxy)
if err != nil {
return nil, errors.New("configdump worker rspamd_proxy: settings are not readable")
}
if len(binds) > 0 {
out = append(out, milterProxy{fields: fields, binds: binds})
}
}
}
// milterBinds returns a proxy worker's settings and, when rspamd runs it as
// a milter, its bind addresses.
func milterBinds(proxy json.RawMessage) (map[string]json.RawMessage, []string, error) {
fields, err := dumpObjectAs[json.RawMessage](proxy, "worker rspamd_proxy")
if err != nil {
return nil, nil, err
}
disabled, err := disabledWorker(fields)
if err != nil || disabled {
return fields, nil, err
}
milter, err := workerBool(fields, "milter", false)
if err != nil || !milter {
return fields, nil, err
}
binds, err := workerBinds(fields)
return fields, binds, err
}
// quietRejectOptions make rspamd's milter accept the mail it rejects:
// discard_on_reject drops it, quarantine_on_reject puts it in Postfix's
// hold queue. rspamd checks them in this order.
var quietRejectOptions = []string{"discard_on_reject", "quarantine_on_reject"}
// QuietRejects returns the quietRejectOptions set on an rspamd proxy worker
// that Postfix passes mail to. An option given more than once counts as
// set unless every value is false: CSM cannot tell which value rspamd uses.
// A value that is not a boolean counts as set too. Errors are those of
// RspamdMilters.
func QuietRejects(f HostFacts) ([]string, error) {
milters, _, err := RspamdMilters(f)
if err != nil {
return nil, err
}
proxies, err := milterProxies([]byte(f.RspamdWorkers.Stdout))
if err != nil {
return nil, err
}
var set []string
for _, name := range quietRejectOptions {
for _, p := range proxies {
if MilterWired(milters, p.binds) && optionSet(p.fields, name) {
set = append(set, name)
break
}
}
}
return set, nil
}
// optionSet reports whether a boolean worker option may be on.
func optionSet(fields map[string]json.RawMessage, name string) bool {
raw, ok := fields[name]
if !ok {
return false
}
values := []json.RawMessage{raw}
var many []json.RawMessage
if json.Unmarshal(raw, &many) == nil {
// JSON null also decodes into a slice. Neither it nor an empty
// list proves that a present option is explicitly false.
if len(many) == 0 {
return true
}
values = many
}
for _, v := range values {
if on, err := workerBool(map[string]json.RawMessage{name: v}, name, false); err != nil || on {
return true
}
}
return false
}
var errNotWired = errors.New("postfix does not pass mail to rspamd's proxy worker")
// RspamdMilters returns the postfix milters that reach rspamd's proxy
// worker, as MilterAddrs resolves them and as smtpd_milters names them.
// Errors quote no command output.
func RspamdMilters(f HostFacts) (addrs, names []string, err error) {
pc, err := ParsePostconf(f.Postconf.Stdout)
if f.Postconf.Err != nil || err != nil {
return nil, nil, errors.New("postconf did not report postfix's milters")
}
binds, err := ProxyMilterAddrs([]byte(f.RspamdWorkers.Stdout))
if f.RspamdWorkers.Err != nil || err != nil {
return nil, nil, errors.New("rspamadm configdump did not report rspamd's proxy worker")
}
for _, m := range pc.Milters {
one := pc
one.Milters = []string{m}
resolved, _ := one.MilterAddrs(f.PostfixMaster)
if MilterWired(resolved, binds) {
addrs, names = append(addrs, resolved...), append(names, m)
}
}
if len(addrs) == 0 {
return nil, nil, errNotWired
}
return addrs, names, nil
}
// ScanSource is where CSM reads a Postfix host's scan records:
// Log is the rspamd log file, "" when rspamd logs elsewhere, and
// Milters are the milters that reach rspamd as smtpd_milters names them,
// the way postfix's warnings name a milter that failed. MilterAction is
// milter_default_action, what postfix does with mail while a milter fails.
// Stack is StackPostfix; the zero ScanSource stands for any other host.
type ScanSource struct {
Stack MailStack
Log string
Milters []string
MilterAction string
}
// PostfixScanSource reads the ScanSource of a Postfix host. ok is false
// when rspamd's logging settings, postconf or rspamd's worker dump could
// not be read.
func PostfixScanSource(f HostFacts) (src ScanSource, ok bool) {
switch {
case f.RspamdLog.Type == "":
return ScanSource{}, false
// A log CSM could not inspect is still the file rspamd writes.
case f.RspamdLog.Type == "file" && path.IsAbs(f.RspamdLog.Path):
src.Log = f.RspamdLog.Path
}
_, names, err := RspamdMilters(f)
if err != nil && !errors.Is(err, errNotWired) {
return ScanSource{}, false
}
// RspamdMilters has read postconf.
pc, _ := ParsePostconf(f.Postconf.Stdout)
src.Stack, src.Milters, src.MilterAction = StackPostfix, names, pc.DefaultAction
return src, true
}
// MilterWired reports whether any postfix milter reaches one of the proxy
// worker's bind addresses.
func MilterWired(milters, proxyAddrs []string) bool {
for _, m := range milters {
for _, b := range proxyAddrs {
if milterReaches(m, b) {
return true
}
}
}
return false
}
// milterReaches matches a unix: milter to the same socket path, and an
// inet: milter to a listener on the same port that accepts its address.
func milterReaches(milter, bind string) bool {
// rspamd puts socket options after the address, split by spaces or commas.
fields := strings.FieldsFunc(bind, func(r rune) bool { return r == ' ' || r == ',' })
if len(fields) == 0 {
return false
}
if sock, ok := strings.CutPrefix(milter, "unix:"); ok {
return strings.HasPrefix(fields[0], "/") && sock == fields[0]
}
hostPort, ok := strings.CutPrefix(milter, "inet:")
if !ok {
return false
}
mh, mp, err := net.SplitHostPort(hostPort)
if err != nil {
return false
}
bh, bp, err := net.SplitHostPort(fields[0])
if err != nil || mp != bp {
return false
}
if strings.EqualFold(mh, bh) {
return true
}
for _, a := range hostAddrs(mh) {
if slices.Contains(hostAddrs(bh), a) {
return true
}
// A wildcard listener here says nothing about another host's
// rspamd, so only loopback milters match one.
if a.IsLoopback() && anyAddress(bh, a) {
return true
}
}
return false
}
// hostAddrs returns the addresses host stands for: itself, or localhost's
// loopback addresses. Other names are not resolved.
func hostAddrs(host string) []netip.Addr {
if strings.EqualFold(host, "localhost") {
return []netip.Addr{netip.AddrFrom4([4]byte{127, 0, 0, 1}), netip.IPv6Loopback()}
}
if a, err := netip.ParseAddr(host); err == nil {
return []netip.Addr{a.Unmap()}
}
return nil
}
// anyAddress reports whether a listener bound to host accepts every address
// of a's family: rspamd's *, *v4 and *v6, or an unspecified address.
func anyAddress(host string, a netip.Addr) bool {
switch host {
case "*":
return true
case "*v4":
return a.Is4()
case "*v6":
return a.Is6()
}
b, err := netip.ParseAddr(host)
return err == nil && b.IsUnspecified() && b.Unmap().Is4() == a.Is4()
}
package antispam
import (
"errors"
"fmt"
"io/fs"
"net/netip"
"path/filepath"
"slices"
"github.com/pidginhost/csm/internal/antispam/dovecot"
)
// Severity says what a failed precondition means for an installation.
type Severity int
const (
// Temporary failures keep an installed integration and are retried.
Temporary Severity = iota
// Definite failures refuse a new install and remove an owned one.
Definite
// Warning failures are reported only.
Warning
)
// Problem is one failed precondition. Code is stable for findings; Detail is
// for the operator and never carries file contents or secrets.
type Problem struct {
Code string
Severity Severity
Detail string
// Key separates findings of one code: the feed name, or rspamd_maps
// for a map redirect.
Key string
}
// Problem codes.
const (
CodePlatform = "platform"
CodeEximVersion = "exim_version"
CodeSpamAssassin = "spamassassin_enabled"
CodeGlobalEnable = "global_enable"
CodeSocket = "rspamd_socket"
CodeRspamdVersion = "rspamd_version"
CodeEximOptions = "exim_options"
CodeSpamScoreRules = "spam_score_rules"
CodeCutthrough = "cutthrough"
CodeForeignScanner = "foreign_spamd_address"
CodeMarkers = "markers"
CodeNamedList = "named_list"
CodeUnverifiable = "exim_config_unverifiable"
CodeTargets = "mutation_target"
CodeMessageSize = "message_size"
CodeEximLog = "exim_log"
CodeSystemd = "systemd"
CodeFeed = "feed"
CodeScores = "rspamd_scores"
CodeControllerOpen = "controller_open"
CodeMilter = "postfix_milter"
CodeRspamdLog = "rspamd_log"
CodeDryRunRejects = "dry_run_rejects"
CodeMilterRejects = "milter_rejects"
)
const (
minEximMajor, minEximMinor = 4, 97
minRspamdMajor = 3
)
// CommandResult is one command's captured output.
type CommandResult struct {
Stdout string
Stderr string
// Err is a start failure, timeout or non-zero exit.
Err error
}
// PathFacts is one lstat result.
type PathFacts struct {
Path string
UID int
GID int
// Mode is the full lstat mode, including type bits.
Mode fs.FileMode
// ExtACL is true when the object carries a POSIX access or default ACL
// beyond its mode bits.
ExtACL bool
}
// SocketFacts describes the scan socket, its parent directories and a probe.
type SocketFacts struct {
// Err is a stat failure other than absence.
Err error
Missing bool
Socket PathFacts
// Parents are every directory the path walk looked a name up in, in
// lookup order without repeats: "/" first, including the directory of
// each symlink followed on the way.
Parents []PathFacts
// RspamdUID is the rspamd service user's uid, or -1 when unknown.
RspamdUID int
// PingErr is nil when GET /ping over the socket answered pong.
PingErr error
// PeerUID is the listener's SO_PEERCRED uid, read during the probe, or -1
// when unavailable. A failed /ping does not invalidate known credentials.
PeerUID int
// EximUID and EximGID are Exim's runtime user and group (exim_user and
// exim_group, mailnull and mail on cPanel), or -1 when unknown.
EximUID int
EximGID int
}
// GroupFile is one rspamd group file as read from the host.
type GroupFile struct {
Content string
// Err is a read failure other than the file's absence.
Err error
}
// HostFacts is everything CheckHost reads. A collector fills it; the checks
// stay pure.
type HostFacts struct {
// Stack selects the preconditions CheckHost applies.
Stack MailStack
// EximVersion is `exim -bV`.
EximVersion CommandResult
// EximOptions is `exim -bP prdr_enable chunking_advertise_hosts`.
EximOptions CommandResult
// EximSettings is `exim -bP` for EximSettingNames.
EximSettings CommandResult
// RspamdOptions is `rspamadm configdump -j options`.
RspamdOptions CommandResult
// RspamdActions is `rspamadm configdump -j actions` and RspamdModules
// `rspamadm configdump -m`: the loaded settings reconcile compares with
// CSM's once CSM's files are installed.
RspamdActions CommandResult
RspamdModules CommandResult
// RspamdPhishing is `rspamadm configdump -j phishing`, compared the same
// way.
RspamdPhishing CommandResult
// RspamdRBL is `rspamadm configdump -j rbl`, compared with CSM's feeds.
RspamdRBL CommandResult
// RspamdConfig is `rspamadm configdump -j`, read for rspamd.com paths.
RspamdConfig CommandResult
// ResolvConf is /etc/resolv.conf: rspamd's nameservers unless
// options.dns.nameserver is set.
ResolvConf string
ResolvConfErr error
// ResolvedDNS includes systemd-resolved's global and per-link DNS
// servers, plus fallback servers when no default DNS route exists.
ResolvedDNS []netip.Addr
ResolvedDNSErr error
// InfraNets are the infra_ips addresses and CIDRs. The pass supplies
// them; a nameserver inside counts as the operator's own.
InfraNets []netip.Prefix
// DQSKey is the Spamhaus DQS key file. Collect leaves it unset; the
// caller reads it only when spamhaus_dqs is enabled.
DQSKey KeyFacts
// RspamdLog is where rspamd writes its log, which can hold DQS lookup
// names.
RspamdLog LogFacts
// IPMap is the CSM IP map file rspamd reads.
IPMap FileFacts
// GroupFiles are the operator's local.d/*_group.conf files, keyed by
// group; an absent file reads as empty.
GroupFiles map[string]GroupFile
// Record is CSM's completion record as found on disk.
Record RecordFacts
// MonitoredLog is the Exim main log CSM reads markers from.
MonitoredLog string
// SystemdRunErr is nil when systemd-run is on PATH.
SystemdRunErr error
// Read only on Postfix: Postconf is `postconf -x -h` for PostconfNames,
// PostfixMaster `postconf -F` for SmtpdChrootFields, PostfixService the
// LoadState and ActiveState of the unit running postfix's master and
// RspamdWorkers `rspamadm configdump -j worker`. The worker dump holds
// the controller password, so it never reaches a finding or log.
Postconf CommandResult
PostfixMaster CommandResult
PostfixService CommandResult
RspamdWorkers CommandResult
// RspamdService and EximService are `systemctl` with UnitShowArgs.
RspamdService CommandResult
EximService CommandResult
GeneratedConf string // /etc/exim.conf
GeneratedErr error
LocalConf string // /etc/exim.conf.local; "" when absent
LocalErr error
// SystemFilter is the file `exim -bP system_filter` names; "" when none.
SystemFilter string
SystemFilterErr error
// NamedLists holds one result per NamedListProbes entry, in order.
NamedLists []CommandResult
SpamdDisabled bool // /etc/spamddisable exists
// SpamdDisabledErr is a stat failure other than absence.
SpamdDisabledErr error
GlobalEnable bool // /etc/global_spamassassin_enable exists
// GlobalEnableErr is a stat failure other than absence.
GlobalEnableErr error
// RspamdVersion is `rspamd --version`; RspamadmVersion `rspamadm --version`.
// RspamdGID is the rspamd service's group, or -1 when unknown.
RspamdGID int
RspamdVersion CommandResult
RspamadmVersion CommandResult
Socket SocketFacts
// Targets describe the files CSM writes, in MutationTargets order.
Targets []TargetFacts
// Controller is the loaded controller worker, parsed from RspamdConfig.
// ControllerErr is set when it could not be read, so exposure is unknown.
Controller ControllerConfig
ControllerErr error
ControllerHost ControllerHostFacts
// ControllerUnverifiable is set when the dump was read but the
// controller in it could not be understood; unlike ControllerErr it is
// reported, not waited out.
ControllerUnverifiable error
// LearningKeys are loaded from the learning directory when learning is
// requested; LearningKeysErr says why they could not be.
LearningKeys LearningKeys
LearningKeysErr error
// LearnerMismatch: the running learner found its ledger out of step
// with storage (state_mismatch) and froze admissions.
LearnerMismatch bool
// Learning prerequisites, collected only while learning is requested.
LearningDumps LearningDumps
ControlSocket SocketFacts
Redis RedisFacts
FuzzyListening bool
// FuzzyListenForeign is set when another user's socket shares the
// fuzzy worker's port.
FuzzyListenForeign bool
FuzzyListenErr error
CageFS CageFSFacts
Roundcube RoundcubeFacts
// AccountNames are the local users that may run tenant code: the cPanel
// accounts on Exim; on Postfix every passwd user but root and the
// pinned service users, and every name of a uid Dovecot stores mail as.
AccountNames map[string]bool
AccountNamesErr error
// RspamdUIDShared is set when a user in AccountNames has rspamd's uid;
// RspamdUIDErr when CSM cannot rule that out.
RspamdUIDShared bool
RspamdUIDErr error
// Dovecot is what learning reads from Dovecot on a Postfix host.
Dovecot DovecotFacts
// RedisGuardPaused is the pass's last memory-guard state, so the guard
// keeps its hysteresis between passes.
RedisGuardPaused bool
}
// CheckHost evaluates the host preconditions that hold
// before CSM writes anything, for the host's mail stack. Staged rspamd
// settings are checked separately with CheckRspamd.
func CheckHost(f HostFacts) []Problem {
switch {
case f.Stack == StackPostfix:
return checkPostfixHost(f)
case f.Stack != StackExim && leftovers(f):
return []Problem{stackUnknown}
case f.Stack != StackExim:
// The collector gathers no Exim facts here.
return []Problem{{Code: CodePlatform, Severity: Definite, Detail: "needs a cPanel host running Exim, or a Postfix host"}}
}
var out []Problem
add := func(code string, sev Severity, format string, args ...any) {
out = append(out, Problem{Code: code, Severity: sev, Detail: fmt.Sprintf(format, args...)})
}
switch v, err := ParseEximVersion(f.EximVersion.Stdout); {
case f.EximVersion.Err != nil:
add(CodeEximVersion, Temporary, "exim -bV failed; check Exim availability and configuration")
case err != nil:
add(CodeEximVersion, Temporary, "exim -bV did not report a readable version")
case !v.AtLeast(minEximMajor, minEximMinor) || !v.ContentScanning:
add(CodeEximVersion, Definite, "Exim %d.%d.%d (content scanning: %t) is unsupported; need 4.97 or later with content scanning",
v.Major, v.Minor, v.Patch, v.ContentScanning)
}
switch {
case f.SpamdDisabledErr != nil:
add(CodeSpamAssassin, Temporary, "cannot check /etc/spamddisable; check file access")
case !f.SpamdDisabled:
add(CodeSpamAssassin, Definite, "SpamAssassin is enabled; disable spamd in WHM so /etc/spamddisable exists")
}
if f.GlobalEnableErr != nil {
add(CodeGlobalEnable, Warning, "cannot check /etc/global_spamassassin_enable; account opt-out availability is unknown")
} else if f.GlobalEnable {
add(CodeGlobalEnable, Warning, "/etc/global_spamassassin_enable exists, so no account can opt out")
}
out = append(out, checkSocket(f.Socket, true)...)
out = append(out, checkRspamdVersions(f.RspamdVersion, f.RspamadmVersion)...)
if f.EximOptions.Err != nil {
add(CodeEximOptions, Temporary, "exim -bP failed; check Exim availability and configuration")
} else {
opts := ParseEximOptions(f.EximOptions.Stdout)
prdr, okPRDR := opts["prdr_enable"]
chunking, okChunking := opts["chunking_advertise_hosts"]
if !okPRDR || !okChunking {
add(CodeEximOptions, Temporary, "exim -bP did not report prdr_enable and chunking_advertise_hosts")
}
if okPRDR && prdr != "false" {
add(CodeEximOptions, Definite, "prdr_enable must be off; per-recipient DATA handling is untested with these hooks")
}
if okChunking && !ChunkingOff(chunking) {
add(CodeEximOptions, Definite, "chunking_advertise_hosts must be empty; BDAT is untested with these hooks")
}
}
if f.EximSettings.Err != nil {
add(CodeEximOptions, Temporary, "exim -bP for the message size limit, log path and Exim user failed; check Exim availability and configuration")
} else {
settings := ParseEximOptions(f.EximSettings.Stdout)
out = append(out, checkMessageSize(settings, f.RspamdOptions)...)
out = append(out, checkEximLog(settings, f.MonitoredLog)...)
}
out = append(out, checkServices(f)...)
if f.GeneratedErr != nil {
add(CodeSpamScoreRules, Temporary, "cannot read the generated Exim configuration; check file access")
} else {
lines, errU := UngatedSpamUses(f.GeneratedConf)
cut, errC := HasCutthrough(f.GeneratedConf)
_, errL := LocateBlocks(f.GeneratedConf)
if err := firstUnverifiable(errU, errC, errL); err != nil {
add(CodeUnverifiable, Temporary, "cannot verify the generated Exim configuration: %v", err)
}
if len(lines) > 0 {
add(CodeSpamScoreRules, Definite, "Exim configuration lines %v act on spam scores outside cPanel's SpamAssassin gate; turn off cPanel's spam-score deny, ratelimit, forwarding and outgoing-scan options", lines)
}
if cut {
add(CodeCutthrough, Definite, "cutthrough delivery is enabled; DATA-time header changes need it off")
}
if errL != nil && !errors.Is(errL, ErrUnverifiable) {
add(CodeMarkers, Definite, "generated Exim configuration: %v", errL)
}
}
if f.SystemFilterErr != nil {
add(CodeSpamScoreRules, Temporary, "cannot read or verify Exim's system filter; check system_filter and file access")
} else if lines := FilterSpamUses(f.SystemFilter); len(lines) > 0 {
add(CodeSpamScoreRules, Definite, "Exim's system filter reads spam scores at lines %v; remove those rules so opted-out accounts are not affected", lines)
}
if f.LocalErr != nil {
add(CodeForeignScanner, Temporary, "cannot read /etc/exim.conf.local; check file access")
} else {
found, errF := ForeignSpamdAddress(f.LocalConf)
blocks, errL := LocateBlocks(f.LocalConf)
if err := firstUnverifiable(errF, errL); err != nil {
add(CodeUnverifiable, Temporary, "cannot verify /etc/exim.conf.local: %v", err)
}
if found {
add(CodeForeignScanner, Definite, "/etc/exim.conf.local sets its own spamd_address; remove it so CSM's scanner is the only one")
}
if errL != nil {
if !errors.Is(errL, ErrUnverifiable) {
add(CodeMarkers, Definite, "/etc/exim.conf.local: %v", errL)
}
} else if len(blocks) > 1 {
add(CodeMarkers, Definite, "/etc/exim.conf.local has multiple CSM blocks; only one scanner block is allowed")
}
}
out = append(out, checkTargets(f.Stack, f.Targets, false)...)
probes := NamedListProbes()
if len(f.NamedLists) != len(probes) {
add(CodeNamedList, Temporary, "named-list checks incomplete: %d of %d results", len(f.NamedLists), len(probes))
}
for i, r := range f.NamedLists[:min(len(f.NamedLists), len(probes))] {
if r.Err != nil {
add(CodeNamedList, Temporary, "exim -be %s failed; check Exim availability and configuration", probes[i])
continue
}
switch err := ParseBeResult(r.Stdout, r.Stderr); {
case errors.Is(err, ErrUndefinedList):
add(CodeNamedList, Definite, "exim -be %s: a named list it uses is undefined", probes[i])
case err != nil:
add(CodeNamedList, Temporary, "exim -be %s printed an unexpected result", probes[i])
}
}
return out
}
// checkPostfixHost applies the Postfix preconditions:
// rspamd's side of the scan socket, rspamd itself, the milter wiring, the
// message size limits, the services and CSM's targets. CSM only reads
// postfix's configuration.
func checkPostfixHost(f HostFacts) []Problem {
out := checkSocket(f.Socket, false)
out = append(out, checkRspamdVersions(f.RspamdVersion, f.RspamadmVersion)...)
add := func(sev Severity, detail string) {
out = append(out, Problem{Code: CodeMilter, Severity: sev, Detail: detail})
}
pc, err := ParsePostconf(f.Postconf.Stdout)
switch {
case f.Postconf.Err != nil:
add(Temporary, "postconf failed; check postfix")
case err != nil:
add(Temporary, "postconf did not report readable smtpd_milters, milter_default_action, message_size_limit, queue_directory, mail_version and milter_mail_macros")
default:
milters, complete := pc.MilterAddrs(f.PostfixMaster)
addrs, err := ProxyMilterAddrs([]byte(f.RspamdWorkers.Stdout))
switch {
case f.RspamdWorkers.Err != nil:
add(Temporary, "rspamadm configdump worker failed; check rspamd")
case err != nil:
add(Temporary, "rspamadm configdump did not report readable worker settings")
case MilterWired(milters, addrs):
case !complete:
add(Temporary, "cannot tell whether postfix smtpd runs chrooted")
default:
add(Definite, "postfix does not pass mail to rspamd's proxy worker")
}
// Without it rspamd scores authenticated mail as remote mail, so
// CSM would reject customers' outbound mail.
if !pc.AuthUser {
add(Definite, "postfix does not pass the authenticated user to rspamd; add {auth_authen} to milter_mail_macros")
}
out = append(out, checkPostfixMessageSize(pc.SizeLimit, f.RspamdOptions)...)
}
out = append(out, checkRspamdLog(f.RspamdLog)...)
out = append(out, checkServices(f)...)
return append(out, checkTargets(f.Stack, f.Targets, false)...)
}
// checkRspamdLog warns when CSM cannot read the scan records rspamd logs on
// a Postfix host. Unreadable logging settings leave Type empty and say
// nothing about where rspamd logs.
func checkRspamdLog(l LogFacts) []Problem {
var detail string
switch {
case l.Type == "":
return nil
case l.Type == "console":
detail = "rspamd logs to the journal; CSM reads scan records from a file log"
case l.Type == "syslog":
detail = "rspamd logs to syslog; CSM reads scan records from a file log"
case l.Type != "file":
detail = fmt.Sprintf("rspamd's log type %q is not a file; CSM reads scan records from a file log", l.Type)
case !filepath.IsAbs(l.Path):
detail = "rspamd's log file is not an absolute path; CSM reads scan records only from one"
case l.Missing:
detail = fmt.Sprintf("rspamd's log file %s does not exist; CSM reads scan records from it once rspamd writes it", l.Path)
default:
return nil
}
return []Problem{{Code: CodeRspamdLog, Severity: Warning, Detail: detail}}
}
// firstUnverifiable returns the first error that marks an Exim file CSM
// cannot check, so one unparseable file yields one temporary problem.
func firstUnverifiable(errs ...error) error {
for _, err := range errs {
if errors.Is(err, ErrUnverifiable) {
return err
}
}
return nil
}
// checkSocket checks the scan socket. eximClient is set when Exim connects
// to it; on Postfix only CSM does, as root, and mail reaches rspamd through
// the proxy worker instead.
func checkSocket(s SocketFacts, eximClient bool) []Problem {
var out []Problem
add := func(sev Severity, format string, args ...any) {
out = append(out, Problem{Code: CodeSocket, Severity: sev, Detail: fmt.Sprintf(format, args...)})
}
trusted := func(uid int) bool { return uid == 0 || uid == s.RspamdUID }
eximKnown := s.EximUID >= 0 && s.EximGID >= 0
// Without the rspamd uid, one non-root owner or listener may be rspamd's;
// two different ones cannot both be. That pair is enough evidence, and
// keeping only it bounds the diagnostic even after a long symlink walk.
var owners []int
if s.RspamdUID < 0 {
rememberOwner := func(uid int) {
if uid > 0 && len(owners) < 2 && !slices.Contains(owners, uid) {
owners = append(owners, uid)
}
}
if s.Err == nil && !s.Missing {
rememberOwner(s.Socket.UID)
rememberOwner(s.PeerUID)
}
for _, p := range s.Parents {
rememberOwner(p.UID)
}
slices.Sort(owners)
}
switch {
case s.Err != nil:
add(Temporary, "cannot stat the rspamd socket; check file access")
case s.Missing:
add(Temporary, "rspamd socket %s does not exist; is rspamd running with CSM's bind_socket?", s.Socket.Path)
default:
switch {
case s.RspamdUID >= 0:
case len(owners) == 0:
add(Temporary, "the rspamd service user is unknown")
case len(owners) == 1:
add(Temporary, "the rspamd service user is unknown, so CSM cannot confirm that uid %d, which owns the socket path or runs its listener, is that user", owners[0])
}
if s.Socket.Mode.Type() != fs.ModeSocket {
add(Definite, "%s is not a socket", s.Socket.Path)
}
if s.RspamdUID >= 0 && !trusted(s.Socket.UID) {
add(Definite, "%s is owned by uid %d; want root or the rspamd user", s.Socket.Path, s.Socket.UID)
}
if len(s.Parents) == 0 {
add(Temporary, "the socket's parent directories were not checked")
}
switch {
case !eximClient:
case !eximKnown:
add(Temporary, "Exim's runtime user is unknown")
case s.Socket.ExtACL:
add(Temporary, "Exim's access to the socket cannot be verified from mode bits while extra ACL entries are present")
case !permits(s.Socket, s.EximUID, s.EximGID, 0o2):
add(Definite, "Exim's runtime user (uid %d) cannot write to %s; bind it with mode=0666", s.EximUID, s.Socket.Path)
}
}
for _, p := range s.Parents {
switch {
case !p.Mode.IsDir():
add(Definite, "%s is not a directory", p.Path)
case s.RspamdUID >= 0 && !trusted(p.UID):
add(Definite, "%s is owned by uid %d; want root or the rspamd user", p.Path, p.UID)
case p.Mode.Perm()&0o022 != 0:
add(Definite, "%s is writable by group or others", p.Path)
case p.ExtACL:
add(Definite, "%s carries extra ACL entries", p.Path)
case eximClient && eximKnown && !permits(p, s.EximUID, s.EximGID, 0o1):
add(Definite, "Exim's runtime user (uid %d) cannot search %s", s.EximUID, p.Path)
}
}
if len(owners) > 1 {
add(Definite, "uids %v own the socket path or run its listener; only root and the rspamd user may, and at most one of them is the rspamd user", owners)
}
if s.Err != nil || s.Missing {
return out
}
if s.PingErr != nil {
add(Temporary, "rspamd did not answer /ping on the socket; check the rspamd service")
}
if s.PeerUID < 0 {
if s.PingErr == nil {
add(Temporary, "the socket listener's peer credentials are unavailable")
}
} else if s.RspamdUID >= 0 && !trusted(s.PeerUID) {
add(Definite, "the socket listener runs as uid %d; want root or the rspamd user", s.PeerUID)
}
return out
}
// permits reports whether uid and gid get the want bits (2 write, 1 search)
// on p from its mode bits. CSM's own probe runs as root, so this is how it
// knows Exim's user can connect. Extra ACL entries are not read here.
func permits(p PathFacts, uid, gid int, want fs.FileMode) bool {
perm := p.Mode.Perm()
switch {
case uid == 0:
return true
case uid == p.UID:
return perm>>6&want == want
case gid == p.GID:
return perm>>3&want == want
default:
return perm&want == want
}
}
func checkRspamdVersions(daemon, adm CommandResult) []Problem {
var out []Problem
add := func(sev Severity, format string, args ...any) {
out = append(out, Problem{Code: CodeRspamdVersion, Severity: sev, Detail: fmt.Sprintf(format, args...)})
}
if daemon.Err != nil || adm.Err != nil {
add(Temporary, "rspamd --version or rspamadm --version failed; check both commands are available")
}
version := func(r CommandResult, name string) (string, int) {
if r.Err != nil {
return "", 0
}
v, major, err := ParseRspamdVersion(r.Stdout)
if err != nil {
add(Temporary, "%s --version did not report a readable version", name)
return "", 0
}
return v, major
}
dv, dm := version(daemon, "rspamd")
av, am := version(adm, "rspamadm")
if dv != "" && dm < minRspamdMajor {
add(Definite, "rspamd %s is unsupported; need 3 or later", dv)
} else if av != "" && am < minRspamdMajor {
add(Definite, "rspamadm %s is unsupported; need 3 or later", av)
}
if dv != "" && av != "" && dv != av {
// A package upgrade replaces the two binaries one after the other.
add(Temporary, "rspamd %s and rspamadm %s differ; wait for any upgrade to finish", dv, av)
}
return out
}
// LearningDumps are the loaded rspamd sections learning checks: the
// fuzzy_check module, the full dump's worker section, the symbol groups and
// the Redis settings. They can hold keys and passwords, so they stay in
// memory and never reach a finding or log.
type LearningDumps struct {
FuzzyCheck, Worker, Group, Redis []byte
Err error
// Symbols is the controller's /symbols reply, read while CSM's learning
// worker file is installed; it shows runtime weight changes.
Symbols []byte
SymbolsErr error
}
// RedisFacts are the live values of the Redis instance the fuzzy worker
// stores hashes in. Save is meaningful only when SaveKnown.
type RedisFacts struct {
Endpoint string
Unix bool
Socket SocketFacts
Port string
// TLSPort is empty when CONFIG GET reports no TLS port setting (Redis 5).
TLSPort string
Policy string
AppendOnly string
Save string
SaveKnown bool
Version string
MaxMemory uint64
// MaxMemoryKnown distinguishes a measured zero limit from a failed probe.
MaxMemoryKnown bool
Used uint64
GroupMembers []string
// ServiceUID is the uid of the process serving the socket, or -1.
ServiceUID int
// Unprotected is set when the collector refused to send the password:
// the socket's owner, directories or peer do not prove who serves it.
Unprotected bool
Err error
// Shared is set when another rspamd Redis consumer names the dedicated
// learning Redis; SharedErr when CSM could not tell.
Shared bool
SharedErr error
// Inherits names a setting of rspamd's global redis section that rspamd
// merges into a fuzzy worker on the dedicated Redis and CSM cannot
// probe with.
Inherits error
}
// DovecotFacts are what learning on a Postfix host reads from Dovecot: the
// uids mail is stored as, the users discovery does not watch, and why it
// could not list them.
type DovecotFacts struct {
UIDs []int
Skips []dovecot.Skip
Err error
}
// CageFSFacts say whether CloudLinux CageFS confines every account and
// whether its skeleton exposes rspamd's log directory.
type CageFSFacts struct {
Enabled bool
DisabledUsers int
SkeletonHasRspamdLog bool
// ExcludedLogDir is the host log directory checked in the skeleton.
ExcludedLogDir string
Err error
}
// RoundcubeFacts describe cPanel's Roundcube include mechanism and the
// include CSM owns.
type RoundcubeFacts struct {
Include TargetFacts
IncludeMechanism bool
PHP bool
Err error
}
package antispam
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/netip"
"strings"
)
// MaxRequestBytes bounds one worker request.
const MaxRequestBytes = 16 << 10
// Worker operations.
const (
OpApply = "apply"
OpRemove = "remove"
)
// Rollback states a Result reports.
const (
// RollbackNone: the worker changed nothing.
RollbackNone = "none"
// RollbackVerified: every change was undone, rebuilt, reloaded and probed.
RollbackVerified = "verified"
// RollbackFailed: the prior state is not confirmed; reconcile must repair.
RollbackFailed = "failed"
)
// FeedsCheckStep is the failed step of an apply whose rspamd blocklist rules
// could not be verified against CSM's settings; the daemon then holds feeds off.
const FeedsCheckStep = "feeds_check"
// Request is the one JSON object the daemon sends to the antispam worker. It
// carries settings only: no command or Exim text. Its paths are the scan
// socket, DQS key file, learning directory and learning Redis socket; keys
// never travel, because the request is recorded on disk.
type Request struct {
Operation string `json:"operation"`
Enabled *bool `json:"enabled,omitempty"`
DryRun *bool `json:"dry_run,omitempty"`
RspamdSocket *string `json:"rspamd_socket,omitempty"`
RejectScore *int `json:"reject_score,omitempty"`
TagScore *int `json:"tag_score,omitempty"`
// Feeds and SkipMaps are comma-separated names.
Feeds *string `json:"feeds,omitempty"`
SkipMaps *string `json:"skip_maps,omitempty"`
SkipScores *bool `json:"skip_scores,omitempty"`
// HostScores is comma-separated symbol names.
HostScores *string `json:"host_scores,omitempty"`
DQSKeyFile *string `json:"dqs_key_file,omitempty"`
// Learning fields are omitted when learning is off, so older records
// keep their form.
Learning *bool `json:"learning,omitempty"`
LearningDir *string `json:"learning_dir,omitempty"`
LearningRedisSocket *string `json:"learning_redis_socket,omitempty"`
Roundcube *bool `json:"roundcube,omitempty"`
// Exempt and ScopeNets are comma-separated. Exim renders neither, so
// they never travel from an Exim host and its requests keep their form.
Exempt *string `json:"exempt,omitempty"`
ScopeNets *string `json:"scope_nets,omitempty"`
}
// ApplyRequest is the request that installs s.
func ApplyRequest(s Settings) Request {
enabled := true
r := Request{
Operation: OpApply,
Enabled: &enabled,
DryRun: &s.DryRun,
RspamdSocket: &s.Socket,
RejectScore: &s.RejectScore,
TagScore: &s.TagScore,
}
if len(s.Feeds) > 0 {
v := strings.Join(s.Feeds, ",")
r.Feeds = &v
}
if len(s.SkipMaps) > 0 {
v := strings.Join(s.SkipMaps, ",")
r.SkipMaps = &v
}
if s.SkipScores {
skip := true
r.SkipScores = &skip
}
if len(s.HostScores) > 0 {
v := strings.Join(s.HostScores, ",")
r.HostScores = &v
}
if s.DQSKeyFile != "" {
v := s.DQSKeyFile
r.DQSKeyFile = &v
}
if s.Learning {
learning, dir := true, s.LearningDir
r.Learning, r.LearningDir = &learning, &dir
if s.LearningRedisSocket != "" {
socket := s.LearningRedisSocket
r.LearningRedisSocket = &socket
}
}
if s.Roundcube {
roundcube := true
r.Roundcube = &roundcube
}
if s.Stack != StackExim && len(s.Exempt) > 0 {
v := strings.Join(s.Exempt, ",")
r.Exempt = &v
}
if s.Stack != StackExim && len(s.ScopeNets) > 0 {
nets := make([]string, len(s.ScopeNets))
for i, p := range s.ScopeNets {
nets[i] = p.String()
}
v := strings.Join(nets, ",")
r.ScopeNets = &v
}
return r
}
// RemoveRequest is the request that removes CSM's antispam integration.
func RemoveRequest() Request {
return Request{Operation: OpRemove}
}
var requestKeys = map[string]bool{
"operation": true, "enabled": true, "dry_run": true,
"rspamd_socket": true, "reject_score": true, "tag_score": true,
"feeds": true, "skip_maps": true, "skip_scores": true, "host_scores": true, "dqs_key_file": true,
"learning": true, "learning_dir": true, "learning_redis_socket": true, "roundcube": true,
"exempt": true, "scope_nets": true,
}
// splitNames reads a comma-separated request field.
func splitNames(p *string) []string {
if p == nil || *p == "" {
return nil
}
return strings.Split(*p, ",")
}
// DecodeRequest validates one request before any side effect. Settings is
// set for an apply. Error messages repeat no input except the one invalid
// socket character ValidateSocket names.
func DecodeRequest(data []byte) (Request, Settings, error) {
if len(data) > MaxRequestBytes {
return Request{}, Settings{}, fmt.Errorf("request exceeds %d bytes", MaxRequestBytes)
}
fields, err := objectFields(data, requestKeys, "request")
if err != nil {
return Request{}, Settings{}, err
}
for _, key := range []string{"feeds", "skip_maps", "skip_scores", "host_scores", "dqs_key_file", "learning_dir", "learning_redis_socket", "learning", "roundcube", "exempt", "scope_nets"} {
if value, present := fields[key]; present {
var valid bool
if key == "skip_scores" || key == "learning" || key == "roundcube" {
_, valid = value.(bool)
} else {
_, valid = value.(string)
}
if !valid {
return Request{}, Settings{}, errors.New("request has a value of the wrong type")
}
}
}
var r Request
if err := json.Unmarshal(data, &r); err != nil {
return Request{}, Settings{}, errors.New("request has a value of the wrong type")
}
switch r.Operation {
case OpApply:
if r.Enabled == nil || r.DryRun == nil || r.RspamdSocket == nil || r.RejectScore == nil || r.TagScore == nil {
return Request{}, Settings{}, errors.New("apply needs enabled, dry_run, rspamd_socket, reject_score and tag_score")
}
if !*r.Enabled {
return Request{}, Settings{}, errors.New("apply needs enabled true")
}
s := Settings{DryRun: *r.DryRun, Socket: *r.RspamdSocket, RejectScore: *r.RejectScore, TagScore: *r.TagScore,
Feeds: splitNames(r.Feeds), SkipMaps: splitNames(r.SkipMaps)}
if r.DQSKeyFile != nil {
s.DQSKeyFile = *r.DQSKeyFile
}
s.SkipScores = r.SkipScores != nil && *r.SkipScores
s.HostScores = splitNames(r.HostScores)
s.Learning = r.Learning != nil && *r.Learning
s.Roundcube = r.Roundcube != nil && *r.Roundcube
if s.Learning && r.LearningDir != nil {
s.LearningDir = *r.LearningDir
}
if s.Learning && r.LearningRedisSocket != nil {
s.LearningRedisSocket = *r.LearningRedisSocket
}
s.Exempt = splitNames(r.Exempt)
for _, n := range splitNames(r.ScopeNets) {
p, err := netip.ParsePrefix(n)
if err != nil {
return Request{}, Settings{}, errors.New("request has an invalid infra_ips network")
}
s.ScopeNets = append(s.ScopeNets, p)
}
if err := s.Validate(); err != nil {
return Request{}, Settings{}, err
}
return r, s, nil
case OpRemove:
if len(fields) != 1 {
return Request{}, Settings{}, errors.New("remove carries only the operation")
}
return r, Settings{}, nil
default:
return Request{}, Settings{}, errors.New("unsupported operation")
}
}
// objectFields rejects ambiguous keys before encoding/json can match them
// case-insensitively or keep only the last duplicate.
func objectFields(data []byte, allowed map[string]bool, label string) (map[string]any, error) {
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.UseNumber()
if token, err := decoder.Token(); err != nil || token != json.Delim('{') {
return nil, fmt.Errorf("%s is not a JSON object", label)
}
fields := make(map[string]any)
for decoder.More() {
token, err := decoder.Token()
if err != nil {
return nil, fmt.Errorf("%s is not valid JSON", label)
}
key, _ := token.(string)
if !allowed[key] {
return nil, fmt.Errorf("%s has an unknown field", label)
}
if _, seen := fields[key]; seen {
return nil, fmt.Errorf("%s repeats a field", label)
}
value, err := decoder.Token()
if err != nil {
return nil, fmt.Errorf("%s is not valid JSON", label)
}
if _, nested := value.(json.Delim); nested {
return nil, fmt.Errorf("%s has a nested value", label)
}
fields[key] = value
}
if token, err := decoder.Token(); err != nil || token != json.Delim('}') {
return nil, fmt.Errorf("%s is not valid JSON", label)
}
if err := decoder.Decode(new(any)); !errors.Is(err, io.EOF) {
return nil, fmt.Errorf("%s has trailing data", label)
}
return fields, nil
}
// Result is the one JSON line the worker writes to stdout.
type Result struct {
Operation string `json:"operation,omitempty"`
OK bool `json:"ok"`
// Busy means another CSM transaction held the Exim lock: retry later.
Busy bool `json:"busy,omitempty"`
FailedStep string `json:"failed_step,omitempty"`
Rollback string `json:"rollback,omitempty"`
// Generation identifies the installed content after a successful apply.
Generation string `json:"generation,omitempty"`
Error string `json:"error,omitempty"`
// Roundcube is set when a successful transaction could not verify the
// webmail include; the rest of the change stands.
Roundcube string `json:"roundcube,omitempty"`
}
// ErrorSummary keeps Result.Error on one line for logs and findings.
func ErrorSummary(s string) string {
return strings.NewReplacer("\r\n", "; ", "\r", "; ", "\n", "; ").Replace(s)
}
var resultKeys = map[string]bool{
"operation": true, "ok": true, "busy": true, "failed_step": true,
"rollback": true, "generation": true, "error": true, "roundcube": true,
}
// ParseResult reads one unambiguous worker report. Invalid or contradictory
// reports return no result, so callers cannot mistake them for completed work.
func ParseResult(out []byte) (Result, error) {
fields, err := objectFields(out, resultKeys, "worker result")
if err != nil {
return Result{}, err
}
if _, present := fields["ok"]; !present {
return Result{}, errors.New("worker result needs ok")
}
for key, value := range fields {
var valid bool
if key == "ok" || key == "busy" {
_, valid = value.(bool)
} else {
_, valid = value.(string)
}
if !valid {
return Result{}, errors.New("worker result has a value of the wrong type")
}
}
var r Result
if err := json.Unmarshal(out, &r); err != nil {
return Result{}, errors.New("worker result is not valid JSON")
}
if err := r.Validate(); err != nil {
return Result{}, err
}
return r, nil
}
// Validate checks a result's invariants: success carries a generation only
// for apply, failure names its step and rollback, busy only at the lock.
func (r Result) Validate() error {
if r.Operation != OpApply && r.Operation != OpRemove &&
(r.Operation != "" || r.OK || r.FailedStep != "request") {
return errors.New("worker result has an unsupported operation")
}
if r.Roundcube != "" && (!r.OK || !strings.HasPrefix(r.Roundcube, RoundcubeUnavailable)) {
return errors.New("worker result reports an unexpected Roundcube state")
}
if r.OK {
if r.Busy || r.FailedStep != "" || r.Error != "" || (r.Rollback != "" && r.Rollback != RollbackNone) {
return errors.New("worker result reports both success and failure")
}
if (r.Operation == OpApply) != (r.Generation != "") {
return errors.New("worker result needs a generation only for successful apply")
}
return nil
}
if r.FailedStep == "" || r.Generation != "" {
return errors.New("worker failure needs a failed step and no generation")
}
if r.Rollback != RollbackNone && r.Rollback != RollbackVerified && r.Rollback != RollbackFailed {
return errors.New("worker failure needs a valid rollback state")
}
if r.Busy && r.FailedStep != "lock" {
return errors.New("worker result reports busy outside the lock step")
}
if (r.FailedStep == "request" || r.FailedStep == "lock") && r.Rollback != RollbackNone {
return errors.New("worker result reports changes before the transaction")
}
return nil
}
package reconcile
import (
"context"
"crypto/sha256"
"fmt"
"net/netip"
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/antispam"
)
// FeedState is what feed handling remembers between passes. Passes run one
// at a time; the mutex guards readers in tests and future callers.
type FeedState struct {
mu sync.Mutex
// held is the feed configuration whose apply failed the feeds check;
// Scoring feeds stay off while the configuration is unchanged.
held string
// scope fingerprints the resolver and queries without retaining a key.
scope [32]byte
// noAnswer counts consecutive completed probes, per zone and query kind.
noAnswer map[string]int
// ipMapActive is set while the installed integration holds the CSM_IP
// rule, so the IP map loop knows rspamd should answer its canary.
ipMapActive atomic.Bool
}
// IPMapHooks keep the CSM IP map in step with the CSM_IP rule.
type IPMapHooks struct {
// Write builds and writes the map now, so a pass that adds the rule
// never applies it without its file.
Write func(ctx context.Context) error
// Remove deletes the map once no rule reads it.
Remove func() error
}
// IPMapActive reports whether the installed integration holds the CSM_IP
// rule.
func (st *FeedState) IPMapActive() bool { return st != nil && st.ipMapActive.Load() }
func (st *FeedState) setIPMapActive(v bool) {
if st != nil {
st.ipMapActive.Store(v)
}
}
// NewFeedState returns empty feed memory.
func NewFeedState() *FeedState { return &FeedState{noAnswer: map[string]int{}} }
func feedConfig(s antispam.Settings) string {
return strings.Join(s.Feeds, ",") + "|" + s.DQSKeyFile
}
func (st *FeedState) hold(configured antispam.Settings) {
if st == nil {
return
}
st.mu.Lock()
defer st.mu.Unlock()
st.held = feedConfig(configured)
}
// apply drops scoring feeds from resolved while the configured feeds are held,
// with one Warning per feed. A changed configuration lifts the hold.
func (st *FeedState) apply(configured, resolved antispam.Settings, problems []antispam.Problem) (antispam.Settings, []antispam.Problem) {
if st == nil {
return resolved, problems
}
st.mu.Lock()
held := st.held != "" && st.held == feedConfig(configured)
if !held {
st.held = ""
}
st.mu.Unlock()
if !held {
return resolved, problems
}
var keep []string
for _, name := range resolved.Feeds {
if name == antispam.FeedCSMIPMap {
problems = append(problems, antispam.Problem{Code: antispam.CodeFeed, Severity: antispam.Warning, Key: name,
Detail: "feed " + name + " is off: at the last apply CSM could not confirm the CSM_IP rule rspamd loaded, or it differed from CSM's; CSM keeps the IP map off until the feed configuration changes or CSM restarts"})
continue
}
if f, _ := antispam.FeedByName(name); f.DNS() {
problems = append(problems, antispam.Problem{Code: antispam.CodeFeed, Severity: antispam.Warning, Key: name,
Detail: "feed " + name + " is off: at the last apply CSM could not confirm the blocklist rules rspamd loaded, or they differed from CSM's; CSM keeps DNS feeds off until the feed configuration changes or CSM restarts"})
continue
}
keep = append(keep, name)
}
resolved.Feeds = keep
if !resolved.HasFeed(antispam.FeedSpamhausDQS) {
resolved.DQSKey = antispam.Secret{}
}
return resolved, problems
}
// mergeByKey joins feed problems that share a key, in first-seen order.
// Findings with one check and dedup key collapse into one alert, so each
// key must carry every cause in its details, each cause once.
func mergeByKey(problems []antispam.Problem) []antispam.Problem {
var out []antispam.Problem
index := make(map[string]int)
seen := make(map[[2]string]bool)
for _, p := range problems {
if seen[[2]string{p.Key, p.Detail}] {
continue
}
seen[[2]string{p.Key, p.Detail}] = true
if i, ok := index[p.Key]; ok {
out[i].Detail += "; " + p.Detail
continue
}
index[p.Key] = len(out)
out = append(out, p)
}
return out
}
// blockingProblems reports a Definite or Temporary problem: the host is
// not known to be current.
func blockingProblems(problems []antispam.Problem) bool {
return slices.ContainsFunc(problems, func(p antispam.Problem) bool { return p.Severity != antispam.Warning })
}
// leftoverMaps reports rspamd.com paths still loaded in enabled modules.
// Unreadable dumps say nothing: the next pass retries.
func leftoverMaps(f antispam.HostFacts) (antispam.Problem, bool) {
if f.RspamdConfig.Err != nil || f.RspamdModules.Err != nil {
return antispam.Problem{}, false
}
enabled, err := antispam.EnabledModules(f.RspamdModules.Stdout)
if err != nil {
return antispam.Problem{}, false
}
paths, err := antispam.RspamdComPaths([]byte(f.RspamdConfig.Stdout), enabled)
if err != nil || len(paths) == 0 {
return antispam.Problem{}, false
}
shown := paths[:min(len(paths), 10)]
detail := "rspamd configuration still references rspamd.com at " + strings.Join(shown, ", ")
if len(paths) > len(shown) {
detail += fmt.Sprintf(" and %d more", len(paths)-len(shown))
}
detail += "; CSM points only the maps it knows at local copies. Enable the rspamd_maps feed to accept rspamd.com's terms, or redirect these maps yourself"
return antispam.Problem{Code: antispam.CodeFeed, Severity: antispam.Warning, Key: antispam.FeedRspamdMaps, Detail: detail}, true
}
// probeFeeds asks every probed zone's test points through rspamd's first
// nameserver, concurrently, and reports what is wrong. No-answer counts
// carry across passes; three in a row raise a finding.
func (r Reconciler) probeFeeds(ctx context.Context, s antispam.Settings, f antispam.HostFacts, now time.Time) []alert.Finding {
queries := antispam.ProbeQueries(s)
if r.Probe == nil || len(queries) == 0 {
r.Feeds.setProbeScope([32]byte{})
return nil
}
servers, err := antispam.Nameservers([]byte(f.RspamdOptions.Stdout), f.ResolvConf)
if err != nil {
r.Feeds.setProbeScope([32]byte{})
return nil // ResolveFeeds already turned DNS feeds off
}
answers := make([]antispam.ProbeAnswer, len(queries))
var wg sync.WaitGroup
for i, q := range queries {
wg.Add(1)
go func() {
defer wg.Done()
answers[i] = r.Probe(ctx, servers[0], q.Name)
}()
}
wg.Wait()
if ctx.Err() != nil {
return nil
}
var scope strings.Builder
scope.WriteString(servers[0].String())
for _, q := range queries {
scope.WriteByte(0)
scope.WriteString(q.Name)
}
r.Feeds.setProbeScope(sha256.Sum256([]byte(scope.String())))
var out []alert.Finding
seen := make(map[string]bool)
for i, q := range queries {
feed, _ := antispam.FeedByName(q.Feed)
status, meaning := antispam.ClassifyProbe(feed, q, answers[i])
// Both queries of a zone can fail alike; findings with one key
// collapse, so keep the first.
if finding, ok := r.Feeds.probeFinding(feed, q, status, meaning, answers[i], now); ok && !seen[finding.DedupKey] {
seen[finding.DedupKey] = true
out = append(out, finding)
}
}
return out
}
// A changed resolver or query set starts a new sequence of observations.
func (st *FeedState) setProbeScope(scope [32]byte) {
if st == nil {
return
}
st.mu.Lock()
defer st.mu.Unlock()
if st.scope != scope {
clear(st.noAnswer)
st.scope = scope
}
}
// probeFinding turns one probe status into a finding. Messages name the
// feed and the answer addresses only, never the query name, which can hold
// the DQS key.
func (st *FeedState) probeFinding(feed antispam.Feed, q antispam.ProbeQuery, status antispam.ProbeStatus, meaning string, a antispam.ProbeAnswer, now time.Time) (alert.Finding, bool) {
kind := "unlisted"
if q.Listed {
kind = "listed"
}
counter := feed.Name + ":" + q.Zone.Rule + ":" + kind
f := alert.Finding{Severity: alert.Warning, Check: CheckFeed, Timestamp: now}
switch status {
case antispam.ProbeOK:
st.count(counter, false)
return alert.Finding{}, false
case antispam.ProbeNoAnswer:
if st.count(counter, true) < 3 {
return alert.Finding{}, false
}
f.DedupKey = feed.Name + ":no_answer"
f.Message = "Antispam feed " + feed.Name + " does not answer"
f.Details = "Three checks in a row got no answer from the list through rspamd's resolver. The list adds no score meanwhile; check that rspamd's resolver recurses and can reach the list."
if feed.NeedsKey {
f.Details += " Spamhaus also answers nothing for a wrong or disabled DQS key; check the key."
}
case antispam.ProbeRefused:
f.DedupKey = feed.Name + ":refused"
f.Message = "Antispam feed " + feed.Name + " refuses CSM's queries"
f.Details = clean(meaning) + ". The list adds no score while it refuses."
case antispam.ProbeListsAll:
f.Severity = alert.Critical
f.DedupKey = feed.Name + ":lists_all"
f.Message = "Antispam feed " + feed.Name + " answers for addresses it never lists"
f.Details = "The list returned a listing for its never-listed test point. rspamd stops querying a list that does this once its own check sees it; turn the feed off if this persists."
if feed.NeedsKey {
f.Details = "The list returned a listing for its never-listed test point. DQS monitoring in rspamd is disabled to keep the key out of its monitor log; turn the feed off now. CSM's probe reports this condition but does not stop lookups."
}
default:
f.DedupKey = feed.Name + ":unexpected"
f.Message = "Antispam feed " + feed.Name + " gave an unexpected answer"
addrs := slices.DeleteFunc(slices.Clone(a.Addrs), func(s string) bool { _, err := netip.ParseAddr(s); return err != nil })
f.Details = "Answer: " + clean(strings.Join(addrs, ", ")) + ". CSM scores only the list's documented codes."
if len(addrs) == 0 {
f.Details = "The list answered with no address. CSM scores only the list's documented codes."
}
}
if status != antispam.ProbeNoAnswer {
st.count(counter, false)
}
return f, true
}
// count updates the consecutive no-answer count and returns it.
func (st *FeedState) count(key string, failed bool) int {
if st == nil {
return 0
}
st.mu.Lock()
defer st.mu.Unlock()
if !failed {
delete(st.noAnswer, key)
return 0
}
st.noAnswer[key]++
return st.noAnswer[key]
}
// Package reconcile runs CSM's antispam integration from the daemon: it
// decides on each pass whether to install, remove or keep the integration,
// asks the root worker to do it, and turns preconditions, worker results
// and rspamd health into findings.
package reconcile
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/antispam"
)
// Finding checks.
const (
CheckPrecondition = "antispam_precondition"
CheckApplyFailed = "antispam_apply_failed"
CheckRspamdDown = "antispam_rspamd_down"
CheckRspamdRecovered = "antispam_rspamd_recovered"
CheckScanError = "antispam_scan_error"
CheckFeed = "antispam_feed"
CheckIPMap = "antispam_ip_map"
CheckControllerOpen = "antispam_controller_open"
CheckLearning = "antispam_learning"
CheckFileReplaced = "antispam_file_replaced"
)
// clean escapes control and non-ASCII characters: problem details and
// worker errors can name paths a tenant controls.
func clean(s string) string {
q := strconv.QuoteToASCII(s)
return q[1 : len(q)-1]
}
// feedFinding reports a feed or map redirect CSM left out, or loaded
// blocklist rules that differ from CSM's. Feed problems never block core.
func feedFinding(p antispam.Problem, now time.Time) alert.Finding {
return alert.Finding{
Severity: alert.Warning,
Check: CheckFeed,
DedupKey: p.Key,
Message: "Antispam feed problem: " + clean(p.Key),
Details: clean(p.Detail) + ". Core spam filtering is unaffected.",
Timestamp: now,
}
}
// controllerFinding reports one rspamd controller listener that tenant
// processes can use for admin or read-only requests.
func controllerFinding(p antispam.Problem, now time.Time) alert.Finding {
return alert.Finding{
Severity: alert.Warning,
Check: CheckControllerOpen,
DedupKey: clean(p.Key),
Message: "rspamd controller may be reachable by other users: " + clean(p.Key),
Details: clean(p.Detail) + ". Local processes, including customer scripts, may use this listener to read " +
"rspamd's history or change its settings. For TCP, set secure_ip = []; in override.d/worker-controller.inc " +
"while a controller password is set. For Unix sockets, use mode 0600 and root or rspamd ownership " +
"for the socket and parent directories. Keep those directories unwritable by other users and remove extended ACLs.",
Timestamp: now,
}
}
// IPMapFinding reports that rspamd is not using CSM's current IP map, or
// that the map could not be written.
func IPMapFinding(detail string, now time.Time) alert.Finding {
return alert.Finding{
Severity: alert.Warning,
Check: CheckIPMap,
DedupKey: "ip_map",
Message: "Antispam IP map problem",
Details: clean(detail) + ". Core spam filtering is unaffected.",
Timestamp: now,
}
}
// problemFinding reports one failed precondition and what CSM does about it.
func problemFinding(p antispam.Problem, now time.Time) alert.Finding {
details := clean(p.Detail)
switch p.Severity {
case antispam.Temporary:
details += ". CSM keeps any installed antispam integration and checks again on its next pass."
case antispam.Definite:
details += ". CSM does not install its antispam integration, and removes an installed one when it can do so safely."
}
return alert.Finding{
Severity: alert.Warning,
Check: CheckPrecondition,
Message: "Antispam precondition not met: " + p.Code,
Details: details,
Timestamp: now,
}
}
// resultFinding reports a worker run that did not complete. A busy lock
// is retried, not reported. Worker errors stay in the daemon log: they
// can carry the worker's diagnostics.
func resultFinding(op string, r antispam.Result, err error, now time.Time) (alert.Finding, bool) {
f := alert.Finding{Severity: alert.Warning, Check: CheckApplyFailed, Timestamp: now}
switch {
case err != nil:
f.Message = "Antispam " + op + " returned no result"
f.Details = "CSM could not confirm the outcome of the antispam " + op + "; the daemon log has the worker's error. CSM runs it again on its next pass."
return f, true
case r.OK || r.Busy:
return alert.Finding{}, false
}
f.Message = fmt.Sprintf("Antispam %s failed at %s", op, r.FailedStep)
f.Details = clean(r.Error)
switch r.Rollback {
case antispam.RollbackVerified:
f.Details += ". CSM restored the previous state and tries again on its next pass."
case antispam.RollbackFailed:
f.Severity = alert.Critical
f.Details += ". CSM could not confirm that the previous state was restored; check Exim and rspamd now. CSM repairs the integration on its next pass."
default:
f.Details += ". CSM changed nothing."
}
return f, true
}
// The health findings. Down and scan errors keep one identity each while
// they last; recovery is a one-time notice.
func rspamdDownFinding(socket string, since time.Time) alert.Finding {
return alert.Finding{
Severity: alert.Warning, Check: CheckRspamdDown, DedupKey: "rspamd", Timestamp: since,
Message: "rspamd does not answer on the antispam scan socket",
Details: fmt.Sprintf("%d /ping probes in a row over %s failed. Exim delivers mail without spam filtering until rspamd answers again.", pingDown, clean(socket)),
}
}
func rspamdRecoveredFinding(socket string, now time.Time) alert.Finding {
return alert.Finding{
Severity: alert.Warning, Check: CheckRspamdRecovered, Timestamp: now,
Message: "rspamd answers on the antispam scan socket again",
Details: fmt.Sprintf("%d /ping probes in a row over %s succeeded.", pingUp, clean(socket)),
}
}
func scanErrorFinding(count int, first, last time.Time, stack antispam.MailStack, milterAction string) alert.Finding {
scans := "scans"
if count == 1 {
scans = "scan"
}
details := fmt.Sprintf("Exim logged %d failed %s since %s, the last at %s. Mail is delivered without spam filtering while scans fail; check rspamd's log.",
count, scans, first.UTC().Format(time.RFC3339), last.UTC().Format(time.RFC3339))
if stack == antispam.StackPostfix {
details = fmt.Sprintf("Postfix logged %d failed %s since %s, the last at %s. %s; check rspamd's log.",
count, scans, first.UTC().Format(time.RFC3339), last.UTC().Format(time.RFC3339), milterEffect(milterAction))
}
return alert.Finding{
Severity: alert.Warning, Check: CheckScanError, DedupKey: "scan", Timestamp: last,
Message: "Antispam scans are failing",
Details: details,
}
}
// milterEffect says what postfix does with mail while rspamd's milter
// fails, by milter_default_action, which postfix reads without regard to
// case.
func milterEffect(action string) string {
switch strings.ToLower(action) {
case "accept":
return "Mail is delivered without spam filtering while scans fail"
case "tempfail":
return "Mail is deferred until rspamd answers"
case "reject":
return "Mail is rejected while scans fail"
case "quarantine":
return "Mail is held in postfix's hold queue while scans fail"
}
return "Postfix's milter_default_action is unknown, so what happens to that mail is unknown"
}
// replacedFinding reports one operator file CSM backed up and replaced,
// until the backup is gone. On a Postfix host removal puts the newest copy
// of each file back over CSM's own, and so does a return to dry run for
// actions.conf; cPanel restores nothing.
func replacedFinding(rep antispam.Replacement, postfix, newest bool, now time.Time) alert.Finding {
var advice string
switch {
case !postfix:
advice = ". Move any settings you still need into a file CSM does not manage, then delete the backup to clear this warning."
case !newest:
advice = ". CSM restores only the newest copy of a file, so this older one stays. Copy it back by hand if you need it, or delete it to clear this warning."
case rep.Path == antispam.ActionsPath:
advice = ". If CSM's own file is still there, CSM puts this copy back when antispam is disabled, and also on returning antispam to dry run, which clears this warning. Delete the backup if you do not want it back."
default:
advice = ". If CSM's own file is still there, CSM puts this copy back when antispam is disabled, which clears this warning. Delete the backup if you do not want it back."
}
return alert.Finding{
Severity: alert.Warning,
Check: CheckFileReplaced,
DedupKey: rep.Backup,
Message: "Antispam replaced " + clean(rep.Path),
Details: "CSM replaced " + clean(rep.Path) + " and kept your previous file at " + clean(rep.Backup) + advice,
Timestamp: now,
}
}
// learningFinding reports one learning condition, one finding per reason.
func learningFinding(p antispam.Problem, now time.Time) alert.Finding {
return alert.Finding{
Severity: alert.Warning,
Check: CheckLearning,
Message: "Antispam learning: " + p.Key,
Details: clean(p.Detail),
DedupKey: "learning:" + clean(p.Key),
Timestamp: now,
}
}
package reconcile
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/antispam"
)
// Consecutive probe results that change rspamd's state.
const (
pingDown = 3
pingUp = 2
)
// Health tracks rspamd's scan socket and the scans the mail logs record
// while antispam is enabled. The probe loop and the log
// watchers use it concurrently.
type Health struct {
mu sync.Mutex
enabled bool
socket string
// fails and oks count consecutive probe results; down is an outage
// already reported, since downSince.
fails, oks int
down bool
downSince time.Time
// scanErrors counts failed scans since the last successful one.
scanErrors int
firstErr, lastErr time.Time
// stack and milterAction word the scan error finding.
stack antispam.MailStack
milterAction string
}
// SetEnabled starts or stops tracking. Stopping forgets every count and
// active finding without a recovery notice; a new socket forgets the
// probe state of the old one.
func (h *Health) SetEnabled(enabled bool, socket string) {
h.mu.Lock()
defer h.mu.Unlock()
if !enabled {
h.scanErrors = 0
}
if !enabled || socket != h.socket {
h.fails, h.oks, h.down = 0, 0, false
}
h.enabled, h.socket = enabled, socket
}
// SetStack words failed scans for the host's mail stack: on Postfix by
// what milterAction, postfix's milter_default_action, does with mail while
// rspamd's milter fails.
func (h *Health) SetStack(stack antispam.MailStack, milterAction string) {
h.mu.Lock()
defer h.mu.Unlock()
h.stack, h.milterAction = stack, milterAction
}
// Ping records one /ping result for socket and returns the finding to
// report: rspamd down after pingDown failures in a row, once per outage, and
// recovered after pingUp successes in a row. A result for a socket no
// longer tracked, from a probe that outlived a reload, is ignored.
func (h *Health) Ping(socket string, err error, now time.Time) []alert.Finding {
h.mu.Lock()
defer h.mu.Unlock()
if !h.enabled || socket != h.socket {
return nil
}
if err != nil {
h.oks = 0
h.fails++
if h.fails >= pingDown && !h.down {
h.down, h.downSince = true, now
return []alert.Finding{rspamdDownFinding(h.socket, now)}
}
return nil
}
h.fails = 0
h.oks++
if h.down && h.oks >= pingUp {
h.down = false
return []alert.Finding{rspamdRecoveredFinding(h.socket, now)}
}
return nil
}
// Line records one Exim main-log line: a scan marker Exim wrote counts as
// Scan counts it, any other line not at all.
func (h *Health) Line(line string, now time.Time) (findings []alert.Finding, recovered bool) {
s, marker := antispam.ParseScan(line)
if !marker {
return nil, false
}
return h.Scan(s, now)
}
// Scan records one scan and returns the finding to report when scans start
// failing, and whether a successful scan cleared a failure.
func (h *Health) Scan(s antispam.Scan, now time.Time) (findings []alert.Finding, recovered bool) {
h.mu.Lock()
defer h.mu.Unlock()
switch {
case !h.enabled:
return nil, false
case s.OK:
recovered = h.scanErrors > 0
h.scanErrors = 0
return nil, recovered
}
h.scanErrors++
h.lastErr = now
if h.scanErrors > 1 {
return nil, false
}
h.firstErr = now
return []alert.Finding{scanErrorFinding(1, now, now, h.stack, h.milterAction)}, false
}
// Active returns the health conditions true now.
func (h *Health) Active() []alert.Finding {
h.mu.Lock()
defer h.mu.Unlock()
var out []alert.Finding
if h.down {
out = append(out, rspamdDownFinding(h.socket, h.downSince))
}
if h.scanErrors > 0 {
out = append(out, scanErrorFinding(h.scanErrors, h.firstErr, h.lastErr, h.stack, h.milterAction))
}
return out
}
package reconcile
import (
"sync"
"github.com/pidginhost/csm/internal/antispam"
)
// LearningState remembers a learning configuration whose apply failed the
// learning check, so learning stays off instead of reloading rspamd with the
// same failing configuration every pass.
type LearningState struct {
mu sync.Mutex
// generation is the held learning configuration; empty holds nothing.
generation string
// roundcube is the learning configuration whose webmail include did not
// take effect, and why.
roundcube, roundcubeWhy string
// paused is the Redis memory guard's last state.
paused bool
}
func (st *LearningState) setRedisPaused(paused bool) {
if st == nil {
return
}
st.mu.Lock()
defer st.mu.Unlock()
st.paused = paused
}
func (st *LearningState) redisPaused() bool {
if st == nil {
return false
}
st.mu.Lock()
defer st.mu.Unlock()
return st.paused
}
// NewLearningState returns empty learning memory.
func NewLearningState() *LearningState { return &LearningState{} }
func (st *LearningState) hold(generation string) {
if st == nil {
return
}
st.mu.Lock()
defer st.mu.Unlock()
st.generation = generation
}
// held reports whether generation is held; any other generation lifts the
// hold.
func (st *LearningState) held(generation string) bool {
if st == nil {
return false
}
st.mu.Lock()
defer st.mu.Unlock()
if st.generation != generation {
st.generation = ""
return false
}
return generation != ""
}
func (st *LearningState) holdRoundcube(generation, why string) {
if st == nil {
return
}
st.mu.Lock()
defer st.mu.Unlock()
st.roundcube, st.roundcubeWhy = generation, why
}
// roundcubeHeld returns why the include is held for generation; any other
// generation lifts the hold.
func (st *LearningState) roundcubeHeld(generation string) (string, bool) {
if st == nil {
return "", false
}
st.mu.Lock()
defer st.mu.Unlock()
if st.roundcube != generation || generation == "" {
st.roundcube, st.roundcubeWhy = "", ""
return "", false
}
return st.roundcubeWhy, true
}
// learningGeneration identifies the desired learning configuration: the
// rendered learning files when their keys are loaded, else the directory
// they come from and the dedicated Redis, a new installation target.
func learningGeneration(s antispam.Settings) string {
if !s.Learning {
return ""
}
if files, err := antispam.RenderLearning(s); err == nil {
return antispam.Generation(files)
}
if s.LearningRedisSocket != "" {
return "dir:" + s.LearningDir + "\nredis:" + s.LearningRedisSocket
}
return "dir:" + s.LearningDir
}
// applyLearningHold turns learning off while its configuration is held.
func (st *LearningState) applyLearningHold(s antispam.Settings) (antispam.Settings, []antispam.Problem) {
if !st.held(learningGeneration(s)) {
return s, nil
}
s.Learning, s.Roundcube = false, false
return s, []antispam.Problem{learningHoldProblem()}
}
// applyRoundcubeHold turns the webmail include off while it is held.
func (st *LearningState) applyRoundcubeHold(s antispam.Settings, generation string) (antispam.Settings, []antispam.Problem) {
// A prerequisite failure can turn learning off without changing the
// desired configuration. It must not lift an unchanged include's hold.
why, held := st.roundcubeHeld(generation)
if !held || !s.Roundcube {
return s, nil
}
s.Roundcube = false
return s, []antispam.Problem{roundcubeHoldProblem(why)}
}
func learningHoldProblem() antispam.Problem {
return antispam.Problem{Code: antispam.CodeLearning, Severity: antispam.Warning, Key: antispam.ReasonApplyFailed,
Detail: "rspamd did not load CSM's learning configuration as written; learning stays off until its configuration changes or CSM restarts"}
}
func learningProbeProblem() antispam.Problem {
return antispam.Problem{Code: antispam.CodeLearning, Severity: antispam.Warning, Key: antispam.ReasonBackendUnavailable,
Detail: "CSM could not read rspamd's loaded learning configuration after the reload; the next pass retries"}
}
func roundcubeHoldProblem(why string) antispam.Problem {
return antispam.Problem{Code: antispam.CodeLearning, Severity: antispam.Warning, Key: antispam.ReasonRoundcube,
Detail: why + "; the webmail Junk-button include stays off until the learning configuration changes or CSM restarts"}
}
package reconcile
import (
"context"
"time"
)
// Interval is how often a pass runs without a trigger.
const Interval = 15 * time.Minute
// Loop runs passes one at a time: at start, after each trigger, every
// Interval, and sooner when a pass asks to retry.
type Loop struct {
pass func(context.Context) Outcome
report func(Outcome)
wake chan struct{}
interval time.Duration
}
// NewLoop returns a loop that calls pass and hands each outcome to report.
// pass reads the configuration itself, so each pass sees the latest one.
func NewLoop(pass func(context.Context) Outcome, report func(Outcome)) *Loop {
return &Loop{pass: pass, report: report, wake: make(chan struct{}, 1), interval: Interval}
}
// Trigger asks for a pass. It never blocks: triggers that arrive while a
// pass runs become one more pass after it.
func (l *Loop) Trigger() {
select {
case l.wake <- struct{}{}:
default:
}
}
// Run passes until ctx ends. A running pass still reports its outcome
// after cancellation.
func (l *Loop) Run(ctx context.Context) {
// A ready wake or timer can win the select even after ctx is canceled.
for ctx.Err() == nil {
out := l.pass(ctx)
l.report(out)
wait := l.interval
if out.Retry > 0 && out.Retry < wait {
wait = out.Retry
}
select {
case <-ctx.Done():
return
case <-l.wake:
case <-time.After(wait):
}
}
}
package reconcile
import (
"context"
"errors"
"fmt"
"net/netip"
"slices"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/eximlock"
)
// BusyRetry is how soon a pass runs again while another CSM transaction
// holds the Exim lock.
const BusyRetry = 20 * time.Second
// codeObserve reports an integration that observe mode leaves in place.
const codeObserve = "observe_mode"
// Desired is the antispam configuration a pass works towards.
type Desired struct {
Enabled bool
Observe bool
Settings antispam.Settings
// InfraNets are infra_ips as prefixes; DNS feeds trust a nameserver
// inside them, and a Postfix host never enforces on mail they send.
InfraNets []netip.Prefix
}
// DesiredFromConfig reads the antispam settings and mode from cfg.
func DesiredFromConfig(cfg *config.Config) Desired {
as := cfg.EmailProtection.Antispam
d := Desired{
Enabled: as.Enabled,
Observe: cfg.ObserveMode(),
InfraNets: antispam.ParseInfraNets(cfg.InfraIPs),
Settings: antispam.Settings{
DryRun: as.DryRun, Socket: as.RspamdSocket,
RejectScore: as.RejectScore, TagScore: as.TagScore,
Feeds: slices.Sorted(slices.Values(as.Feeds.Enabled)),
DQSKeyFile: as.Feeds.SpamhausDQSKeyFile,
Exempt: as.Exempt,
},
}
if as.Learning.Enabled {
d.Settings.Learning, d.Settings.LearningDir = true, antispam.DefaultLearningDir
d.Settings.LearningRedisSocket = as.Learning.RedisSocket
}
return d
}
// Reconciler runs one pass at a time against the host its functions read
// and change.
type Reconciler struct {
// Invalidate stops pending workers before the host is read, including
// in observe mode. It changes only CSM's request record, not mail config.
Invalidate func(context.Context) error
// Collect reads every precondition fact for the scan socket.
Collect func(ctx context.Context, socket string) antispam.HostFacts
// Installed reads only what shows CSM's integration on the host.
Installed func() antispam.HostFacts
// Run sends one request to the root worker.
Run func(ctx context.Context, req antispam.Request) (antispam.Result, error)
// Now stamps findings.
Now func() time.Time
// ReadKey reads the Spamhaus DQS key file; nil disables spamhaus_dqs.
ReadKey func(path string) antispam.KeyFacts
// Feeds remembers feed holds and probe counts between passes.
Feeds *FeedState
// Probe asks server one DNS question; nil disables the feed probe.
Probe func(ctx context.Context, server netip.AddrPort, name string) antispam.ProbeAnswer
// IPMap writes and removes the CSM IP map; unset hooks do nothing.
IPMap IPMapHooks
// Learning remembers a learning configuration held after a failed check.
Learning *LearningState
// EnsureKeys creates the learning keys once and LoadKeys reads them;
// CollectLearning adds the learning prerequisites to the core facts.
// All run only while learning is requested.
EnsureKeys func(ctx context.Context, dir string) (bool, error)
LoadKeys func(dir string) (antispam.LearningKeys, error)
CollectLearning func(ctx context.Context, s antispam.Settings, f *antispam.HostFacts)
// Learner returns the running learning engine's problems; nil when no
// engine runs.
Learner func() []antispam.Problem
// Replaced lists the operator files the worker backed up and replaced;
// nil reports none.
Replaced func() ([]antispam.Replacement, error)
// resolveLearning replaces antispam.ResolveLearning in tests.
resolveLearning func(s antispam.Settings, f antispam.HostFacts) (antispam.Settings, antispam.LearningGate, []antispam.Problem)
// assess replaces antispam.Assess in tests.
assess func(enabled bool, s antispam.Settings, f antispam.HostFacts) antispam.Assessment
}
// Outcome is what one pass found and did.
type Outcome struct {
// Findings are the antispam conditions true after this pass.
Findings []alert.Finding
// Retry, when set, asks for another pass this soon.
Retry time.Duration
// Err is the cause of a pass that did not complete, for the daemon log.
Err error
// Checks are the checks this pass settled: a reporter replaces their
// latest findings with Findings. A canceled pass settles none, and a
// busy one leaves the last apply result until its retry.
Checks []string
// LearningProblems are this pass's learning conditions.
LearningProblems []antispam.Problem
// LearningGate says what learning may do after this pass.
LearningGate antispam.LearningGate
// Learner is where the learning engine runs, zero when no engine
// should: learning not requested, observe mode, unreadable keys or no
// private controller socket.
Learner LearnerTarget
// Stats is where the web UI reads rspamd's counters, and StatsReason
// why there is no such place. Zero in a pass that did not collect the
// controller.
Stats antispam.StatsEndpoint
StatsReason string
// Stack is the mail stack this pass found; the daemon picks the Spam
// folder watcher by it.
Stack antispam.MailStack
// ScanSource is where the daemon reads scan records on a Postfix host,
// empty elsewhere and while antispam is disabled. Nil when this pass
// could not tell; the last one stands.
ScanSource *antispam.ScanSource
}
// LearnerTarget is what a learning engine is built for. A change means a
// new engine.
type LearnerTarget struct {
Controller string
RspamdUID int
HashGeneration string
Prefix string
}
// Pass brings the host towards d once. A
// disabled pass reads only the host's mail stack and CSM's own files, and
// runs no other command unless it finds something to remove.
func (r Reconciler) Pass(ctx context.Context, d Desired) (out Outcome) {
if err := ctx.Err(); err != nil {
return Outcome{Err: err}
}
if !d.Enabled || d.Observe {
r.Feeds.setProbeScope([32]byte{})
}
if !d.Enabled || !d.Settings.Learning {
r.Learning.held("")
r.Learning.roundcubeHeld("")
}
invalidateErr := r.Invalidate(ctx)
if err := ctx.Err(); err != nil {
return Outcome{Err: err}
}
if err := invalidateErr; err != nil {
out = Outcome{Retry: BusyRetry}
if !d.Enabled {
// Disabling this warning depends only on configuration, not
// on whether pending mail configuration changes can be canceled.
// Learning findings wait for a completed pass: until removal
// runs they can still describe installed scoring.
out.Checks = []string{CheckControllerOpen}
}
if !errors.Is(err, eximlock.ErrBusy) {
retryDetails := "CSM retries shortly."
if !d.Enabled {
// A broken state directory does not heal in seconds. With
// antispam off, retry on the normal interval instead.
out.Retry = 0
retryDetails = "CSM retries at the next regular check or after a config reload."
}
out.Err = fmt.Errorf("invalidating pending antispam requests: %w", err)
out.Findings = []alert.Finding{{
Severity: alert.Warning, Check: CheckApplyFailed, Timestamp: r.Now(),
Message: "Antispam could not cancel pending changes",
Details: "Pending antispam workers may still change mail configuration. Check the daemon log for the recording error; " + retryDetails,
}}
}
return out
}
var f antispam.HostFacts
wantLearning := d.Enabled && d.Settings.Learning
var keys antispam.LearningKeys
var keysErr error
var learnerProblems []antispam.Problem
if wantLearning {
keys, keysErr = r.learningKeys(ctx, d.Settings.LearningDir)
if r.Learner != nil {
learnerProblems = r.Learner()
}
}
if d.Enabled {
writable, _ := r.Feeds.apply(d.Settings, d.Settings, nil)
if !d.Observe && writable.HasFeed(antispam.FeedCSMIPMap) && r.IPMap.Write != nil {
// A write failure is the IP map loop's to report; the feed's
// preconditions keep the rule off without its file.
_ = r.IPMap.Write(ctx)
}
f = r.Collect(ctx, d.Settings.Socket)
f.InfraNets = d.InfraNets
if d.Settings.HasFeed(antispam.FeedSpamhausDQS) && r.ReadKey != nil {
f.DQSKey = r.ReadKey(d.Settings.DQSKeyFile)
}
if wantLearning {
if r.CollectLearning != nil {
r.CollectLearning(ctx, d.Settings, &f)
}
f.LearningKeys, f.LearningKeysErr = keys, keysErr
f.RedisGuardPaused = r.Learning.redisPaused()
f.LearnerMismatch = slices.ContainsFunc(learnerProblems, func(p antispam.Problem) bool {
return p.Key == antispam.ReasonStateMismatch
})
}
} else {
f = r.Installed()
}
// Interrupted probes do not describe the host's preconditions.
if err := ctx.Err(); err != nil {
return Outcome{Err: err}
}
// The host's mail stack picks the files CSM renders.
d.Settings.Stack, d.Settings.ScopeNets = f.Stack, d.InfraNets
assess := r.assess
if assess == nil {
assess = antispam.Assess
}
settings, feedProblems := d.Settings, []antispam.Problem(nil)
var scoreProblems, learningProblems []antispam.Problem
var gate antispam.LearningGate
if d.Enabled {
settings, feedProblems = antispam.ResolveFeeds(d.Settings, f)
settings, feedProblems = r.Feeds.apply(d.Settings, settings, feedProblems)
settings, scoreProblems = antispam.ResolveScores(settings, f)
settings, gate, learningProblems = r.resolveLearningState(settings, f, keys)
}
a := assess(d.Enabled, settings, f)
now := r.Now()
out = Outcome{Checks: []string{CheckPrecondition, CheckApplyFailed, CheckFeed, CheckLearning}, LearningGate: gate, Stack: f.Stack}
if d.Enabled {
out.Stats, out.StatsReason = antispam.StatsEndpointFor(f)
}
out.ScanSource = &antispam.ScanSource{}
// A pass that cannot tell the stack, as while postfix restarts, keeps the
// last source.
if d.Enabled && f.Stack != antispam.StackExim {
out.ScanSource = nil
if f.Stack == antispam.StackPostfix {
if src, ok := antispam.PostfixScanSource(f); ok {
out.ScanSource = &src
}
}
}
// List backups after the worker finishes, including failed rollbacks.
// An unreadable directory or an abandoned pass keeps the last findings.
defer func() {
if r.Replaced == nil || len(out.Checks) == 0 {
return
}
if replaced, err := r.Replaced(); err == nil {
out.Checks = append(out.Checks, CheckFileReplaced)
// Names start with the UTC time, so the newest copy sorts last.
newest := map[string]string{}
for _, rep := range replaced {
newest[rep.Path] = max(newest[rep.Path], rep.Backup)
}
for _, rep := range replaced {
out.Findings = append(out.Findings, replacedFinding(rep, f.Stack == antispam.StackPostfix, newest[rep.Path] == rep.Backup, now))
}
}
}()
// Prerequisites alone do not prove that the requested integration is
// active. A worker must finish an apply, or the installed state must
// already be current, before the learner may publish.
out.LearningGate.Publish = gate.Publish && a.Action == antispam.Keep && a.Installed && !blockingProblems(a.Problems)
if d.Observe || a.Action == antispam.Uninstall {
out.LearningGate = antispam.LearningGate{}
}
// On Postfix the learner starts only for learning that resolved on.
if wantLearning && !d.Observe && keysErr == nil && (f.Stack != antispam.StackPostfix || out.LearningGate.Install) {
if socket, uid, ok := antispam.LearnerEndpoint(f); ok {
out.Learner = LearnerTarget{Controller: socket, RspamdUID: uid, HashGeneration: antispam.HashGeneration(keys), Prefix: keys.Prefix}
}
}
defer func() {
// Worker verification can add problems. Busy and canceled passes
// must not replace or add to the last completed learning findings.
if !slices.Contains(out.Checks, CheckLearning) {
return
}
out.LearningProblems = mergeByKey(append(learningProblems, learnerProblems...))
for _, p := range out.LearningProblems {
out.Findings = append(out.Findings, learningFinding(p, now))
}
}()
// Controller exposure needs no install: it is reported whenever
// antispam is on and cleared when it is off. An unreadable controller
// keeps the last result.
controllerSettled := !d.Enabled || f.ControllerErr == nil
if controllerSettled {
out.Checks = append(out.Checks, CheckControllerOpen)
}
switch {
case !d.Enabled || f.ControllerErr != nil:
case f.ControllerUnverifiable != nil:
out.Findings = append(out.Findings, controllerFinding(antispam.ControllerUnverifiableProblem(), now))
default:
for _, p := range antispam.ControllerExposure(f.Controller, f.ControllerHost) {
out.Findings = append(out.Findings, controllerFinding(p, now))
}
}
for _, p := range a.Problems {
if p.Code == antispam.CodeFeed {
feedProblems = append(feedProblems, p)
}
}
// Leftover maps mean something only once CSM's files are installed and
// current, which a Keep without blocking problems shows.
if d.Enabled && !d.Observe && settings.Redirects() && a.Action == antispam.Keep && a.Installed && !blockingProblems(a.Problems) {
if p, ok := leftoverMaps(f); ok {
feedProblems = append(feedProblems, p)
}
}
for _, p := range mergeByKey(feedProblems) {
out.Findings = append(out.Findings, feedFinding(p, now))
}
if d.Enabled && !d.Observe && a.Action == antispam.Keep && a.Installed && !blockingProblems(a.Problems) {
out.Findings = append(out.Findings, r.probeFeeds(ctx, settings, f, now)...)
if err := ctx.Err(); err != nil {
return Outcome{Err: err}
}
} else {
r.Feeds.setProbeScope([32]byte{})
}
for _, p := range slices.Concat(a.Problems, scoreProblems) {
if p.Code == antispam.CodeFeed {
continue
}
finding := problemFinding(p, now)
if d.Observe {
finding.Details = clean(p.Detail) + ". CSM changes nothing in observe mode."
}
out.Findings = append(out.Findings, finding)
}
// Learning off while rspamd still loads CSM's rules means a removal did
// not finish, for example because a foreign file replaced one CSM
// wrote. Judged from this pass's facts, so only when no removal succeeds.
scoringLeft := d.Enabled && !settings.Learning && f.RspamdConfig.Err == nil && antispam.LearningScoringLoaded([]byte(f.RspamdConfig.Stdout))
if scoringLeft && (d.Observe || a.Action == antispam.Keep) {
learningProblems = append(learningProblems, removalIncomplete())
}
if d.Observe {
if a.Installed {
out.Findings = append(out.Findings, problemFinding(antispam.Problem{
Code: codeObserve, Severity: antispam.Warning,
Detail: "CSM's antispam integration is installed and observe mode leaves it in place; leave observe mode so CSM can update or remove it",
}, now))
}
r.settleIPMap(d, settings, a, false, &out)
return out
}
var req antispam.Request
switch a.Action {
case antispam.Install:
req = antispam.ApplyRequest(settings)
case antispam.Uninstall:
req = antispam.RemoveRequest()
default:
r.settleIPMap(d, settings, a, false, &out)
return out
}
result, err := r.Run(ctx, req)
// A canceled caller needs no failure alert. Keep completed worker
// reports, which can still carry a failed rollback requiring attention.
if err != nil && ctx.Err() != nil {
return Outcome{Err: err}
}
if result.Busy {
out.Retry = BusyRetry
// Learning findings depend on the apply as well; keep the last.
out.Checks = []string{CheckPrecondition, CheckFeed}
if controllerSettled {
out.Checks = append(out.Checks, CheckControllerOpen)
}
}
if result.FailedStep == antispam.FeedsCheckStep {
r.Feeds.hold(d.Settings)
}
out.LearningGate.Publish = gate.Publish && req.Operation == antispam.OpApply && err == nil && result.OK
switch result.FailedStep {
case antispam.LearningCheckStep:
r.Learning.hold(learningGeneration(settings))
out.LearningGate = antispam.LearningGate{}
learningProblems = append(learningProblems, learningHoldProblem())
case antispam.LearningProbeStep:
learningProblems = append(learningProblems, learningProbeProblem())
}
if strings.HasPrefix(result.Roundcube, antispam.RoundcubeUnavailable) {
why := strings.TrimPrefix(result.Roundcube, antispam.RoundcubeUnavailable)
p := antispam.Problem{Code: antispam.CodeLearning, Severity: antispam.Warning, Key: antispam.ReasonRoundcube, Detail: why}
if req.Operation == antispam.OpApply && settings.Roundcube {
r.Learning.holdRoundcube(learningGeneration(settings), why)
p = roundcubeHoldProblem(why)
}
out.LearningGate.Roundcube = false
learningProblems = append(learningProblems, p)
}
if finding, ok := resultFinding(req.Operation, result, err, now); ok {
out.Findings = append(out.Findings, finding)
}
if scoringLeft && !result.Busy && (err != nil || !result.OK) {
learningProblems = append(learningProblems, removalIncomplete())
}
out.Err = err
r.settleIPMap(d, settings, a, err == nil && result.OK, &out)
return out
}
// settleIPMap records whether the CSM_IP rule is in use once a pass leaves
// the host matching settings: a Keep without blocking problems, or a
// successful apply or removal. The map is deleted once the host holds
// settings without the rule; observe mode changes nothing.
func (r Reconciler) settleIPMap(d Desired, settings antispam.Settings, a antispam.Assessment, applied bool, out *Outcome) {
if !applied && (a.Action != antispam.Keep || blockingProblems(a.Problems)) {
return
}
if a.Action != antispam.Uninstall && d.Enabled && !d.Observe && settings.HasFeed(antispam.FeedCSMIPMap) && (a.Installed || applied) {
r.Feeds.setIPMapActive(true)
return
}
r.Feeds.setIPMapActive(false)
if !d.Observe && r.IPMap.Remove != nil {
if err := r.IPMap.Remove(); err != nil {
out.Err = errors.Join(out.Err, fmt.Errorf("removing the antispam IP map: %w", err))
out.Findings = append(out.Findings, alert.Finding{
Severity: alert.Warning, Check: CheckApplyFailed, Timestamp: r.Now(),
Message: "Antispam IP map removal failed",
Details: "The unused IP map remains on disk. Check the daemon log for the removal error; CSM retries on its next pass.",
})
}
}
}
// learningKeys creates the keys on first use and reads them. A failure is
// returned for the resolver, which reports it as state_mismatch.
func (r Reconciler) learningKeys(ctx context.Context, dir string) (antispam.LearningKeys, error) {
if r.EnsureKeys == nil || r.LoadKeys == nil {
return antispam.LearningKeys{}, errors.New("learning keys are not available in this build")
}
if _, err := r.EnsureKeys(ctx, dir); err != nil {
return antispam.LearningKeys{}, err
}
return r.LoadKeys(dir)
}
// resolveLearningState applies the learning hold, the prerequisites and the
// Roundcube hold, in that order, and remembers the Redis guard state.
func (r Reconciler) resolveLearningState(s antispam.Settings, f antispam.HostFacts, keys antispam.LearningKeys) (antispam.Settings, antispam.LearningGate, []antispam.Problem) {
if !s.Learning {
s.Roundcube = false
return s, antispam.LearningGate{}, nil
}
s.LearningKeys = keys
generation := learningGeneration(s)
s, problems := r.Learning.applyLearningHold(s)
if !s.Learning {
return s, antispam.LearningGate{}, problems
}
resolve := r.resolveLearning
if resolve == nil {
resolve = antispam.ResolveLearning
}
s, gate, found := resolve(s, f)
problems = append(problems, found...)
if f.Redis.Err == nil && f.Redis.MaxMemoryKnown {
r.Learning.setRedisPaused(antispam.RedisGuard(f.Redis.Used, f.Redis.MaxMemory, f.RedisGuardPaused))
}
s, held := r.Learning.applyRoundcubeHold(s, generation)
if len(held) > 0 {
gate.Roundcube = false
problems = append(problems, held...)
}
return s, gate, problems
}
func removalIncomplete() antispam.Problem {
return antispam.Problem{Code: antispam.CodeLearning, Severity: antispam.Warning, Key: antispam.ReasonRemovalIncomplete,
Detail: "learning is off but rspamd still loads CSM's learned scoring; a learning file CSM wrote may have been replaced, so CSM cannot remove it"}
}
package antispam
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"maps"
"path"
"strings"
)
// RecordPath is CSM's completion record. Its directory is root-only 0700.
const RecordPath = "/var/lib/csm/antispam/installed.json"
// RecordFormat is the format of installed.json.
const RecordFormat = 1
// MaxRecordBytes bounds a completion record read from disk.
const MaxRecordBytes = 4 << 10
// RecordFacts is installed.json as a collector found it.
type RecordFacts struct {
Missing bool
Content string
Err error
}
// Record is CSM's completion record. A nil Generation means an apply or
// remove started and did not finish, so reconcile must repeat it.
type Record struct {
Format int `json:"format"`
Generation *string `json:"generation"`
}
// Complete reports whether the record names an installed generation.
func (r Record) Complete() bool { return r.Generation != nil }
// EncodeRecord renders a record. An empty generation marks work in progress.
func EncodeRecord(generation string) []byte {
r := Record{Format: RecordFormat}
if generation != "" {
r.Generation = &generation
}
data, _ := json.Marshal(r)
return append(data, '\n')
}
var recordKeys = map[string]bool{"format": true, "generation": true}
// DecodeRecord reads installed.json strictly: exactly format 1 and a
// generation that is null or a SHA-256 hex digest.
func DecodeRecord(data []byte) (Record, error) {
if len(data) > MaxRecordBytes {
return Record{}, errors.New("installed record is too large")
}
fields, err := objectFields(data, recordKeys, "installed record")
if err != nil {
return Record{}, err
}
if len(fields) != len(recordKeys) {
return Record{}, errors.New("installed record needs format and generation")
}
var r Record
if err := json.Unmarshal(data, &r); err != nil {
return Record{}, errors.New("installed record has a value of the wrong type")
}
if r.Format != RecordFormat {
return Record{}, errors.New("installed record has an unsupported format")
}
if r.Generation != nil && !isDigest(*r.Generation) {
return Record{}, errors.New("installed record has an invalid generation")
}
return r, nil
}
func isDigest(s string) bool {
return len(s) == 2*sha256.Size && strings.Trim(s, "0123456789abcdef") == ""
}
// Generation identifies managed content: the SHA-256 of canonical JSON that
// maps each managed destination to its text, keys sorted, no HTML escaping.
func Generation(content map[string]string) string {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
enc.SetEscapeHTML(false)
_ = enc.Encode(content) // a map of strings always encodes
sum := sha256.Sum256(bytes.TrimSuffix(buf.Bytes(), []byte("\n")))
return hex.EncodeToString(sum[:])
}
// ManagedContent maps each file CSM writes on s.Stack to the text it owns
// there: a whole rspamd file, or the CSM block of a shared Exim file.
// Optional files appear only while a feed, map redirect, learning or the
// Roundcube include needs them; the symbol weights appear unless the
// operator owns that file. A Postfix host has no Exim file or Roundcube
// include, and in dry run its own actions.conf keeps enforcing.
func ManagedContent(s Settings) (map[string]string, error) {
out := make(map[string]string)
if s.Stack == StackPostfix {
postfix, err := RenderPostfix(s)
if err != nil {
return nil, err
}
maps.Copy(out, postfix)
} else {
hooks, err := RenderExim(s)
if err != nil {
return nil, err
}
out[HookRCPTPath], out[HookScanPath], out[HookPostPath], out[LocalEximConfPath] = hooks.RCPT, hooks.Scan, hooks.Post, hooks.Config
if s.Roundcube {
out[RoundcubeIncludePath] = RenderRoundcube()
}
}
files, err := RenderRspamd(s)
if err != nil {
return nil, err
}
if s.HostActions() {
delete(files, "actions.conf")
}
for name, text := range files {
out[path.Join(RspamdLocalDir, name)] = text
}
feeds, err := RenderFeeds(s)
if err != nil {
return nil, err
}
maps.Copy(out, feeds)
if !s.SkipScores {
out[ScoresPath] = renderScores(s.HostScores)
}
learning, err := RenderLearning(s)
if err != nil {
return nil, err
}
maps.Copy(out, learning)
return out, nil
}
// InstalledContent maps each target to the CSM text found in it, in the
// form ManagedContent uses. Unreadable, inactive or misplaced blocks are
// absent, so they cannot match a completed installation.
func InstalledContent(targets []TargetFacts) map[string]string {
out := make(map[string]string)
for _, t := range targets {
if t.Err != nil || t.Missing {
continue
}
if t.Kind.WholeFile() {
if ManagedFor(t.Kind, t.Content) {
out[t.Path] = t.Content
}
} else if text, ok := installedBlockText(t.Kind, t.Content); ok {
out[t.Path] = text
}
}
return out
}
func installedBlockText(kind TargetKind, content string) (string, bool) {
section, acl := "", ""
if kind == TargetEximHook {
section, acl = "acl", "csm_hook"
}
// Ownership markers alone cannot prove that Exim runs the rendered text.
blocks, err := locateBlocks(content, section, acl, true)
if err != nil || len(blocks) != 1 {
return "", false
}
b := blocks[0]
lines := strings.SplitAfter(content, "\n")
if kind == TargetEximHook {
if b.Begin != 1 {
return "", false
}
} else if !oneConfigSection(content) || b.Begin < 2 || lines[b.Begin-2] != "@CONFIG@\n" {
return "", false
}
return strings.Join(lines[b.Begin-1:b.End], ""), true
}
package antispam
import (
"strconv"
"strings"
)
// Marker lines bracket every block CSM writes into a shared Exim file.
const (
MarkerBegin = "# CSM-ANTISPAM BEGIN"
MarkerEnd = "# CSM-ANTISPAM END"
)
// EximHooks is the rendered Exim side of the integration.
type EximHooks struct {
// RCPT is ACL_SPAM_SCAN_CHECK_BLOCK/custom_begin_spam_scan_check.
RCPT string
// Scan is ACL_SPAM_SCAN_BLOCK/custom_begin_spam_scan.
Scan string
// Post is ACL_CHECK_MESSAGE_POST_BLOCK/custom_begin_check_message_post.
Post string
// Config is the block CSM inserts after @CONFIG@ in /etc/exim.conf.local.
Config string
}
// eligible skips authenticated sessions and the peers cPanel exempts from
// its own spam scan.
const eligible = " !authenticated = *\n" +
" !hosts = <; 127.0.0.0/8 ; ::1 ; +trustedmailhosts ; +cpanel_mail_netblocks\n"
// optIn resolves the recipient's account with cPanel's own owner rule and
// reads its native Spam Filters toggle. A missing owner or home is "off".
const optIn = "${if exists{/etc/global_spamassassin_enable}{on}" +
"{${lookup passwd{${if eq{$domain}{$primary_hostname}{${sg{$local_part}{\\N[/+].*\\N}{}}}" +
"{${lookup{$domain}lsearch{/etc/userdomains}}}}}" +
"{${if exists{${extract{5}{:}{$value}}/.spamassassinenable}{on}{off}}}{off}}}}"
const hasPolicy = " condition = ${if or{{eq{$acl_m_csm_policy}{on}}{eq{$acl_m_csm_policy}{off}}}}\n"
const differs = " condition = ${if and{{!eq{$acl_m_csm_rcpt}{}}{>{$recipients_count}{0}}{!eq{$acl_m_csm_rcpt}{$acl_m_csm_policy}}}}\n"
// RenderExim renders the three ACL hooks and the scanner address. Only
// validated integers and the validated socket path enter the text.
func RenderExim(s Settings) (EximHooks, error) {
if err := s.Validate(); err != nil {
return EximHooks{}, err
}
return EximHooks{
RCPT: renderRCPT(s.DryRun),
Scan: renderScan(s),
Post: renderPost(s),
Config: MarkerBegin + "\nspamd_address = " + s.Socket + " variant=rspamd tmo=30s\n" + MarkerEnd + "\n",
}, nil
}
func renderRCPT(dryRun bool) string {
var b strings.Builder
b.WriteString(MarkerBegin + "\n")
b.WriteString("warn\n set acl_m_csm_rcpt =\n\n")
b.WriteString("warn\n" + eligible + " set acl_m_csm_rcpt = skip\n\n")
b.WriteString("warn\n condition = ${if eq{$acl_m_csm_rcpt}{skip}}\n domains = +local_domains\n" +
" set acl_m_csm_rcpt = " + optIn + "\n\n")
b.WriteString("warn\n condition = ${if and{{!eq{$acl_m_csm_rcpt}{}}{=={$recipients_count}{0}}}}\n" +
" set acl_m_csm_policy = $acl_m_csm_rcpt\n\n")
if dryRun {
b.WriteString("warn\n" + differs +
" logwrite = :main: CSM-ANTISPAM event=split mode=dry-run policy=$acl_m_csm_policy rcpt_class=$acl_m_csm_rcpt\n")
} else {
b.WriteString("defer\n" + differs +
" message = 452 4.5.3 Different spam policy, retry this recipient separately\n")
}
b.WriteString(MarkerEnd + "\n")
return b.String()
}
func renderScan(s Settings) string {
mode := "enforce"
if s.DryRun {
mode = "dry-run"
}
reject, tag := strconv.Itoa(s.RejectScore*10), strconv.Itoa(s.TagScore*10)
var b strings.Builder
b.WriteString(MarkerBegin + "\n")
b.WriteString("warn\n set acl_m_csm_scan_ok = 0\n set acl_m_csm_score = 0\n\n")
if !s.DryRun {
b.WriteString("warn\n" + eligible + hasPolicy + " remove_header = ^(?i)x-(spam|rspamd|spamd)-\n\n")
}
b.WriteString("warn\n" + eligible + hasPolicy + " spam = nobody:true/defer_ok\n" +
" set acl_m_csm_scan_ok = 1\n set acl_m_csm_score = $spam_score_int\n\n")
b.WriteString("warn\n" + eligible + hasPolicy +
" logwrite = :main: CSM-ANTISPAM event=scan mode=" + mode + " policy=$acl_m_csm_policy " +
"${if eq{$acl_m_csm_scan_ok}{1}{result=ok score=$acl_m_csm_score action=\"${sg{$spam_action}{[^a-z ]}{_}}\" " +
"decision=${if eq{$acl_m_csm_policy}{off}{unfiltered}{${if >={$acl_m_csm_score}{" + reject + "}{reject}" +
"{${if >={$acl_m_csm_score}{" + tag + "}{tag}{none}}}}}}}" +
"{result=error score=none action=\"unavailable\" decision=none}} ip=$sender_host_address\n")
b.WriteString(MarkerEnd + "\n")
return b.String()
}
func renderPost(s Settings) string {
if s.DryRun {
return MarkerBegin + "\n" + MarkerEnd + "\n"
}
reject, tag := strconv.Itoa(s.RejectScore*10), strconv.Itoa(s.TagScore*10)
return MarkerBegin + "\n" +
"deny\n" + eligible +
" condition = ${if and{{eq{$acl_m_csm_policy}{on}}{eq{$acl_m_csm_scan_ok}{1}}{>={$acl_m_csm_score}{" + reject + "}}}}\n" +
" message = 550 5.7.1 Message rejected as spam\n\n" +
"warn\n" + eligible +
" condition = ${if and{{eq{$acl_m_csm_policy}{on}}{eq{$acl_m_csm_scan_ok}{1}}{>={$acl_m_csm_score}{" + tag + "}}}}\n" +
" add_header = X-Spam-Status: Yes, score=${eval:$acl_m_csm_score/10}.${eval:$acl_m_csm_score%10}\n" +
MarkerEnd + "\n"
}
package antispam
import (
_ "embed"
"strconv"
"strings"
)
// milterHeaders makes rspamd's milter add X-Spam-Status, which learning
// reads to skip mail CSM tagged, and strip inbound copies so a sender
// cannot forge it. It also adds the host's Authentication-Results and strips
// inbound copies, which a sender could forge too. rspamd adds neither header
// to authenticated or local mail, so no login name reaches outbound mail.
// X-Spam stays: the proxy worker's own X-Spam: Yes is what the host's sieve
// rules file into Junk.
const milterHeaders = ManagedHeader + "\nuse = [\"x-spam-status\", \"authentication-results\"];\nremove = { \"X-Spam-Status\" = 0; };\n"
// postfixScope is the scope Lua with $ placeholders for validated values.
// Its file name is fixed: scan records are read from rspamd log lines that
// name it.
//
//go:embed postfix_scope.lua
var postfixScope string
// RenderPostfix returns the files CSM owns only on a Postfix host, keyed by
// path: the milter headers and the scope Lua.
func RenderPostfix(s Settings) (map[string]string, error) {
if err := s.Validate(); err != nil {
return nil, err
}
return map[string]string{
MilterHeadersPath: milterHeaders,
PostfixScopeLuaPath: renderScope(s),
}, nil
}
// renderScope fills the scope Lua. Each operator string is validated ASCII
// and becomes a quoted Lua string; thresholds are in tenths.
func renderScope(s Settings) string {
mode := "enforce"
if s.DryRun {
mode = "dry-run"
}
nets := make([]string, len(s.ScopeNets))
for i, p := range s.ScopeNets {
nets[i] = strconv.Quote(p.Masked().String())
}
exempt := make([]string, len(s.Exempt))
for i, e := range s.Exempt {
exempt[i] = strconv.Quote(e)
}
return strings.NewReplacer(
"$MODE", strconv.Quote(mode),
"$REJECT", strconv.Itoa(s.RejectScore*10),
"$TAG", strconv.Itoa(s.TagScore*10),
"$NETS", strings.Join(nets, ", "),
"$EXEMPT", strings.Join(exempt, ", "),
).Replace(postfixScope)
}
package antispam
import (
"fmt"
"slices"
"strconv"
"strings"
)
// ManagedHeader is the first line of every rspamd file CSM owns.
const ManagedHeader = "# csm-managed-version: 1"
// RspamdFiles lists the local.d files CSM owns, in write order.
var RspamdFiles = []string{"actions.conf", "greylist.conf", "rbl.conf", "emails.conf", "fuzzy_check.conf", "asn.conf", "phishing.conf", "regexp.conf"}
// disabledModule turns one rspamd module off. Core makes no third-party
// lookups: rbl, emails and fuzzy_check query outside lists, asn sends sender
// IPs to rspamd.com, and greylist would defer mail. rbl stays off unless the
// operator enables a DNS feed.
const disabledModule = ManagedHeader + "\nenabled = false;\n"
// invisibleCJK matches an invisible character next to a Chinese, Japanese
// or Korean letter. Spam puts them inside subject words to split them for
// filters; legitimate subjects use them beside Latin text or inside emoji.
// Explicit Unicode 17.0 letter ranges keep RE2, PCRE and Hyperscan independent
// of the engine's Unicode version. Gaps must stay excluded: they contain
// unassigned code points, not letters. Combining accents, tone marks, kana
// voicing marks and variation selectors may follow a letter. The lists are
// static so the rendered file never changes with the Go toolchain.
const (
cjkLetters = `\x{1100}-\x{11FF}\x{3005}\x{303B}\x{3041}-\x{3096}\x{309D}-\x{309F}` +
`\x{30A1}-\x{30FA}\x{30FC}-\x{30FF}\x{3131}-\x{318E}\x{31F0}-\x{31FF}` +
`\x{3400}-\x{4DBF}\x{4E00}-\x{9FFF}\x{A960}-\x{A97C}\x{AC00}-\x{D7A3}` +
`\x{D7B0}-\x{D7C6}\x{D7CB}-\x{D7FB}\x{F900}-\x{FA6D}\x{FA70}-\x{FAD9}` +
`\x{FF66}-\x{FF9D}\x{FFA0}-\x{FFBE}\x{FFC2}-\x{FFC7}\x{FFCA}-\x{FFCF}\x{FFD2}-\x{FFD7}\x{FFDA}-\x{FFDC}` +
`\x{16FE3}\x{16FF2}-\x{16FF3}\x{1AFF0}-\x{1AFF3}\x{1AFF5}-\x{1AFFB}\x{1AFFD}-\x{1AFFE}` +
`\x{1B000}-\x{1B122}\x{1B132}\x{1B150}-\x{1B152}\x{1B155}\x{1B164}-\x{1B167}` +
`\x{20000}-\x{2A6DF}\x{2A700}-\x{2B81D}\x{2B820}-\x{2CEAD}\x{2CEB0}-\x{2EBE0}` +
`\x{2EBF0}-\x{2EE5D}\x{2F800}-\x{2FA1D}\x{30000}-\x{3134A}\x{31350}-\x{33479}`
letterMarks = `\x{0300}-\x{036F}\x{302A}-\x{302F}\x{3099}-\x{309A}\x{FE00}-\x{FE0F}\x{FE20}-\x{FE2F}\x{E0100}-\x{E01EF}`
invisibleChars = `\x{200B}-\x{200D}\x{2060}\x{FEFF}`
invisibleCJK = `[` + cjkLetters + `][` + letterMarks + `]*[` + invisibleChars + `]|[` + invisibleChars + `][` + cjkLetters + `]`
)
// subjectRules scores the decoded Subject. On its own the weight lifts an
// authenticated message over the default tag score, not to the default
// reject score.
const subjectRules = ManagedHeader + "\nCSM_INVISIBLE_CJK_SUBJECT {\n" +
" re = 'Subject=/" + invisibleCJK + "/u';\n" +
" score = 8.0;\n" +
" description = \"Invisible character inside CJK text in Subject\";\n}\n"
// scoreWeights lowers rspamd's HIDDEN_TEXT, which gives up to 5 points for
// hidden text. Newsletters hide their preview line that way, so at the
// default tag score the rule and one minor symbol were enough to tag them.
// URL_VERY_LONG adds its weight once per long link, so a newsletter full of
// tracking links reached the tag score on that rule alone; one_shot counts
// it once per message. A configured symbol loses rspamd's one_shot unless
// the entry repeats it. DMARC_POLICY_REJECT gains a point: personalized
// phishing that forges a domain with a reject policy stayed just under the
// default tag score, while mail that fails such a policy without forgery
// scores far lower. The symbol keeps its dmarc group.
var scoreWeights = []scoreWeight{
{"html", "HIDDEN_TEXT", "2.0", true},
{"url", "URL_VERY_LONG", "1.5", true},
{"policies", "DMARC_POLICY_REJECT", "3.0", false},
}
// scoreWeight is one rspamd symbol weight CSM sets.
type scoreWeight struct {
group, symbol, weight string
oneShot bool
}
// ScoreGroups are the rspamd groups of the symbols CSM weighs, in order.
func ScoreGroups() []string {
out := make([]string, 0, len(scoreWeights))
for _, w := range scoreWeights {
if !slices.Contains(out, w.group) {
out = append(out, w.group)
}
}
return out
}
// renderScores writes CSM's weights except for the symbols in host, whose
// weight the host's own group files set.
func renderScores(host []string) string {
var b strings.Builder
b.WriteString(ManagedHeader + "\n")
for _, w := range scoreWeights {
if slices.Contains(host, w.symbol) {
continue
}
fmt.Fprintf(&b, "group %q {\n symbols {\n %q {\n weight = %s;\n", w.group, w.symbol, w.weight)
if w.oneShot {
b.WriteString(" one_shot = true;\n")
}
b.WriteString(" }\n }\n}\n")
}
return b.String()
}
// RenderRspamd returns the content of each file in RspamdFiles, keyed by its
// name under /etc/rspamd/local.d.
func RenderRspamd(s Settings) (map[string]string, error) {
if err := s.Validate(); err != nil {
return nil, err
}
files := make(map[string]string, len(RspamdFiles))
for _, name := range RspamdFiles {
files[name] = disabledModule
}
// The phishing module keeps its local checks; only its URL lookups in
// rspamd.com's phishtank zone go off.
files["phishing.conf"] = ManagedHeader + "\nphishtank_enabled = false;\n"
if s.Redirects() {
files["phishing.conf"] += phishingExceptions
}
files["regexp.conf"] = subjectRules
if len(s.DNSFeeds()) > 0 {
files["rbl.conf"] = ManagedHeader + "\nenabled = true;\n"
}
// Learning queries only CSM's loopback worker; the override drops every
// other fuzzy rule, including rspamd.com's.
if s.Learning {
files["fuzzy_check.conf"] = ManagedHeader + "\nenabled = true;\n"
}
files["actions.conf"] = ManagedHeader + "\n" +
"reject = " + strconv.Itoa(s.RejectScore) + ";\n" +
"add_header = " + strconv.Itoa(s.TagScore) + ";\n" +
"greylist = null;\n" +
"rewrite_subject = null;\n"
return files, nil
}
// IsManaged reports whether content is a file CSM wrote: its first line is
// exactly ManagedHeader. Any other first line belongs to the operator.
func IsManaged(content string) bool {
first, _, _ := strings.Cut(content, "\n")
return first == ManagedHeader
}
package antispam
import (
"net/url"
"path"
"strings"
"time"
)
// ReplacedDir holds root-only backups of operator files CSM replaces.
// Retained copies are durable records: the daemon reports one warning per
// file found here.
const ReplacedDir = "/var/lib/csm/antispam/replaced"
// backupStamp is the UTC time that starts a backup name.
const backupStamp = "20060102T150405Z"
// Replacement is one operator file CSM replaced and the copy it kept.
type Replacement struct {
Path, Backup string
}
// BackupName names the copy of target taken at at. The escaped path holds
// no "/", so every copy sits directly in ReplacedDir.
func BackupName(target string, at time.Time) string {
return at.UTC().Format(backupStamp) + "_" + url.QueryEscape(target)
}
// ParseBackupName returns the absolute, clean target path a backup name was
// made for.
func ParseBackupName(name string) (string, bool) {
stamp, escaped, ok := strings.Cut(name, "_")
if !ok {
return "", false
}
if _, err := time.Parse(backupStamp, stamp); err != nil {
return "", false
}
target, err := url.QueryUnescape(escaped)
if err != nil || !path.IsAbs(target) || path.Clean(target) != target {
return "", false
}
return target, true
}
package antispam
import (
"context"
"errors"
"strconv"
"strings"
"time"
)
// RoundcubeUnavailable prefixes Result.Roundcube when the include could not
// be installed, verified or removed cleanly.
const RoundcubeUnavailable = "unavailable: "
// RoundcubeJunkFolder is the Maildir folder cPanel delivers spam to and
// learning reads.
const RoundcubeJunkFolder = "INBOX.spam"
// RenderRoundcube returns the include that points Roundcube's Junk button
// at the Spam folder. Its marker follows the PHP opening tag.
func RenderRoundcube() string {
return "<?php\n" + ManagedHeader + "\n" +
"$config['junk_mbox'] = '" + RoundcubeJunkFolder + "';\n" +
"$config['markasjunk_spam_mbox'] = '" + RoundcubeJunkFolder + "';\n"
}
// RoundcubeState is what a fresh Roundcube request loads: the Junk folder
// settings and whether the markasjunk plugin is on.
type RoundcubeState struct {
JunkMbox, SpamMbox string
MarkAsJunk bool
}
// Describe names the loaded settings for a finding: each value quoted, kept
// to printable ASCII and cut to 64 characters.
func (r RoundcubeState) Describe() string {
quote := func(v string) string {
if len(v) > 64 {
v = v[:64]
}
return strconv.QuoteToASCII(v)
}
plugin := "off"
if r.MarkAsJunk {
plugin = "on"
}
return "Roundcube loads junk_mbox " + quote(r.JunkMbox) + ", markasjunk_spam_mbox " + quote(r.SpamMbox) + ", markasjunk " + plugin
}
// Applied reports whether the Junk button reaches the Spam folder.
func (r RoundcubeState) Applied() bool {
return r.MarkAsJunk && r.JunkMbox == RoundcubeJunkFolder && r.SpamMbox == RoundcubeJunkFolder
}
// cPanel's Roundcube and its PHP runtime.
const (
roundcubeDir = "/usr/local/cpanel/base/3rdparty/roundcube/"
cpanelPHP = "/usr/local/cpanel/3rdparty/bin/php"
)
// roundcubeScript loads Roundcube's configuration the way a request does,
// defaults first, and prints only the three settings CSM checks. It never
// dumps the configuration.
const roundcubeScript = `define('RCMAIL_VERSION', '1.6');
define('RCUBE_INSTALL_PATH', '` + roundcubeDir + `');
$config = array();
require RCUBE_INSTALL_PATH . 'config/defaults.inc.php';
require RCUBE_INSTALL_PATH . 'config/config.inc.php';
$line = function ($v) {
if (!is_string($v)) { return ''; }
if (strpbrk($v, "\r\n") !== false) { exit(1); }
return $v;
};
$plugins = isset($config['plugins']) && is_array($config['plugins']) ? $config['plugins'] : array();
echo 'junk_mbox=', $line(isset($config['junk_mbox']) ? $config['junk_mbox'] : ''), "\n";
echo 'spam_mbox=', $line(isset($config['markasjunk_spam_mbox']) ? $config['markasjunk_spam_mbox'] : ''), "\n";
echo 'markasjunk=', in_array('markasjunk', $plugins, true) ? '1' : '0', "\n";
`
// RoundcubeEffective evaluates Roundcube's configuration with cPanel's PHP
// the way a webmail request loads it, as the unprivileged uid and gid: the
// includes it loads are never run as root. Webmail itself runs them as each
// account's user.
func RoundcubeEffective(ctx context.Context, run func(context.Context, time.Duration, string, ...string) CommandResult, uid, gid int) (RoundcubeState, error) {
if uid <= 0 || gid <= 0 {
return RoundcubeState{}, errors.New("CSM has no unprivileged account to run cPanel's PHP as")
}
r := run(ctx, 30*time.Second, "setpriv", "--reuid="+strconv.Itoa(uid), "--regid="+strconv.Itoa(gid), "--clear-groups", "--no-new-privs",
"env", "DNS=localhost", cpanelPHP, "-d", "display_errors=0", "-r", roundcubeScript)
if r.Err != nil {
return RoundcubeState{}, errors.New("cPanel's PHP could not load the Roundcube configuration")
}
return ParseRoundcubeState(r.Stdout)
}
// ParseRoundcubeState reads exactly the three key=value lines the script
// prints, in any order.
func ParseRoundcubeState(out string) (RoundcubeState, error) {
var st RoundcubeState
seen := map[string]bool{}
for _, line := range strings.Split(strings.TrimSuffix(out, "\n"), "\n") {
key, value, ok := strings.Cut(line, "=")
if !ok || seen[key] {
return RoundcubeState{}, errors.New("the Roundcube check printed unexpected output")
}
seen[key] = true
switch key {
case "junk_mbox":
st.JunkMbox = value
case "spam_mbox":
st.SpamMbox = value
case "markasjunk":
if value != "0" && value != "1" {
return RoundcubeState{}, errors.New("the Roundcube check printed unexpected output")
}
st.MarkAsJunk = value == "1"
default:
return RoundcubeState{}, errors.New("the Roundcube check printed unexpected output")
}
}
if len(seen) != 3 {
return RoundcubeState{}, errors.New("the Roundcube check printed unexpected output")
}
return st, nil
}
package antispam
import (
"encoding/json"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
)
var rspamdVersionRe = regexp.MustCompile(`(?i)\brspam(?:d daemon version|adm)\s+(\d{1,4})\.(\d{1,4})(?:\.(\d{1,4}))?(?:[ \t\r\n+-]|$)`)
// ParseRspamdDumpObject rejects ambiguous fields while preserving nested
// settings for the caller. Parse errors never include the dump's contents.
func ParseRspamdDumpObject(out []byte) (map[string]json.RawMessage, error) {
return dumpObjectAs[json.RawMessage](out, "object")
}
// ParseRspamdVersion reads `rspamd --version` or `rspamadm --version` and
// returns the dotted version and its major number.
func ParseRspamdVersion(out string) (string, int, error) {
m := rspamdVersionRe.FindStringSubmatch(out)
if m == nil {
return "", 0, errors.New("no rspamd version in output")
}
major, _ := strconv.Atoi(m[1])
version := m[1] + "." + m[2]
if m[3] != "" {
version += "." + m[3]
}
return version, major, nil
}
// EnabledModules reads `rspamadm configdump -m` and returns the modules on its
// "Modules enabled:" line.
func EnabledModules(out string) (map[string]bool, error) {
for _, line := range strings.Split(out, "\n") {
rest, ok := strings.CutPrefix(strings.TrimSpace(line), "Modules enabled:")
if !ok {
continue
}
set := make(map[string]bool)
for _, name := range strings.Split(rest, ",") {
if name = strings.TrimSpace(name); name != "" {
set[name] = true
}
}
return set, nil
}
return nil, errors.New(`configdump output has no "Modules enabled:" line`)
}
// Actions are the loaded rspamd action thresholds. A nil field means the
// action is disabled or absent.
type Actions struct {
Reject, AddHeader, Greylist, RewriteSubject *float64
// GrowFactor rescales scores above the reject threshold; learning's
// score table assumes none.
GrowFactor *float64
}
// ParseActions reads `rspamadm configdump -j actions`. Keys other than the
// four CSM sets and grow_factor are ignored. Underscored and spaced names
// are accepted; duplicate fields, ambiguous aliases and non-numeric,
// non-null values are errors.
func ParseActions(out []byte) (Actions, error) {
raw, err := dumpObjectAs[json.RawMessage](out, "actions")
if err != nil {
return Actions{}, err
}
get := func(key string) (*float64, error) {
names := []string{key}
if alias := strings.ReplaceAll(key, "_", " "); alias != key {
names = append(names, alias)
}
var value *float64
seen := false
for _, name := range names {
v, ok := raw[name]
if !ok {
continue
}
var f *float64
if decodeErr := json.Unmarshal(v, &f); decodeErr != nil {
return nil, fmt.Errorf("configdump action %s: %w", name, decodeErr)
}
same := value == nil && f == nil || value != nil && f != nil && *value == *f
if seen && !same {
// The dump omits UCL priorities, so alias precedence is unknown.
return nil, fmt.Errorf("configdump action %s: conflicting aliases %q and %q", key, key, name)
}
value, seen = f, true
}
return value, nil
}
var a Actions
for _, field := range []struct {
key string
dst **float64
}{{"reject", &a.Reject}, {"add_header", &a.AddHeader}, {"greylist", &a.Greylist}, {"rewrite_subject", &a.RewriteSubject}, {"grow_factor", &a.GrowFactor}} {
if *field.dst, err = get(field.key); err != nil {
return Actions{}, err
}
}
return a, nil
}
// PhishtankEnabled reads `rspamadm configdump -j phishing` and reports
// whether the phishing module looks up message URLs in rspamd.com's
// phishtank zone. A missing or non-boolean setting is an error.
func PhishtankEnabled(out []byte) (bool, error) {
fields, err := dumpObject(out, "phishing")
if err != nil {
return false, err
}
on, ok := fields["phishtank_enabled"].(bool)
if !ok {
return false, errors.New("configdump phishing: phishtank_enabled is not a boolean")
}
return on, nil
}
// modulesOff must be inactive in the loaded configuration: core makes no
// third-party lookups and never defers mail.
var modulesOff = []string{"asn", "emails", "fuzzy_check", "greylist", "rbl"}
// CheckRspamd compares staged rspamd state with what CSM wrote. Each string
// names one conflict, such as an override.d file defeating a CSM setting.
func CheckRspamd(enabled map[string]bool, a Actions, s Settings) []string {
return append(checkRspamdModules(enabled, s), checkRspamdActions(a, s)...)
}
func checkRspamdModules(enabled map[string]bool, s Settings) []string {
var out []string
dns := len(s.DNSFeeds()) > 0
for _, m := range modulesOff {
// With a DNS feed rbl runs CSM's rules; CheckFeedsLoaded checks them.
if m == "rbl" && dns {
continue
}
// With learning fuzzy_check runs CSM's rules; learning verifies it,
// and a learning fault never removes core filtering.
if m == "fuzzy_check" && s.Learning {
continue
}
if enabled[m] {
out = append(out, fmt.Sprintf("rspamd module %s is enabled; an override defeats CSM's %s.conf", m, m))
}
}
return out
}
func checkRspamdActions(a Actions, s Settings) []string {
if s.HostActions() {
return nil
}
var out []string
if !equals(a.Reject, s.RejectScore) {
out = append(out, fmt.Sprintf("rspamd reject threshold is %s, want %d", show(a.Reject), s.RejectScore))
}
if !equals(a.AddHeader, s.TagScore) {
out = append(out, fmt.Sprintf("rspamd add_header threshold is %s, want %d", show(a.AddHeader), s.TagScore))
}
if a.Greylist != nil {
out = append(out, "rspamd greylist action is enabled")
}
if a.RewriteSubject != nil {
out = append(out, "rspamd rewrite_subject action is enabled")
}
return out
}
func equals(v *float64, want int) bool { return v != nil && *v == float64(want) }
func show(v *float64) string {
if v == nil {
return "disabled"
}
return strconv.FormatFloat(*v, 'g', -1, 64)
}
package antispam
import (
"bytes"
"cmp"
"encoding/json"
"errors"
"maps"
"math"
"net"
"net/netip"
"slices"
"strconv"
"strings"
"unicode/utf8"
)
// The controller on a TCP port can be spoofed, so the strings it hands the
// web UI are bounded, in characters.
const (
maxStatsVersion = 32
maxStatsAction = 64
maxStatsSymbol = 128
)
// Stats endpoint reasons. They are fixed strings so the web UI never shows
// configuration values.
const (
statsNoListener = "rspamd's controller has no listener CSM can read without a password; add a private Unix socket (mode 0600, owned by rspamd)"
statsControllerUnknown = "CSM cannot read rspamd's controller configuration"
)
// StatsEndpoint is where CSM reads rspamd's /stat and /counters without a
// password. Address is empty when there is no such listener; Version is set
// either way.
type StatsEndpoint struct {
Unix bool
Address string // socket path, or loopback host:port
RspamdUID int // uid that must serve the socket; Unix only
Version string // from `rspamd --version`
}
// StatsEndpointFor picks rspamd's private controller socket, or else a
// loopback TCP listener that secure_ip trusts. The reason is fixed text,
// set when there is neither.
func StatsEndpointFor(f HostFacts) (StatsEndpoint, string) {
ep := StatsEndpoint{Version: statsVersion(f.RspamdVersion)}
if !f.Controller.Present && (f.ControllerErr != nil || f.ControllerUnverifiable != nil) {
return ep, statsControllerUnknown
}
host := f.ControllerHost
for _, l := range f.Controller.Listeners {
if !l.Unix || host.RspamdUID < 0 {
continue
}
if s, ok := host.Sockets[l.Path]; ok && s.Err == nil && !s.Missing && s.Socket.UID == host.RspamdUID {
ep.Unix, ep.Address, ep.RspamdUID = true, l.Path, host.RspamdUID
return ep, ""
}
}
for _, l := range f.Controller.Listeners {
if l.Unix {
continue
}
if addr, ok := trustedLoopback(l.HostPort, f.Controller.TrustEntries); ok {
ep.Address = addr
return ep, ""
}
}
return ep, statsNoListener
}
var statsLoopbackV4 = netip.MustParseAddr("127.0.0.1")
// trustedLoopback returns the address to dial for a listener bound to a
// loopback literal that a trust entry covers. For localhost CSM dials
// 127.0.0.1. A bind without an explicit port is skipped rather than guessed.
func trustedLoopback(hostPort string, trust []string) (string, bool) {
host, port, err := net.SplitHostPort(hostPort)
n, portErr := strconv.ParseUint(port, 10, 16)
if err != nil || portErr != nil || n == 0 {
return "", false
}
if strings.EqualFold(host, "localhost") {
host = statsLoopbackV4.String()
}
addr, err := netip.ParseAddr(host)
if err != nil || addr != statsLoopbackV4 && addr != netip.IPv6Loopback() {
return "", false
}
for _, entry := range trust {
// Both IPv4 spellings name the same peer, as in reachable.
if p, ok := trustPrefix(entry); ok && (p.Contains(addr) || addr.Is4() && p.Contains(netip.AddrFrom16(addr.As16()))) {
return net.JoinHostPort(addr.String(), port), true
}
}
return "", false
}
// statsVersion is the first line of `rspamd --version`, trimmed, reduced to
// the version number when the line has the usual form.
func statsVersion(r CommandResult) string {
if r.Err != nil {
return ""
}
line, _, _ := strings.Cut(r.Stdout, "\n")
line = strings.TrimSpace(line)
if v, _, err := ParseRspamdVersion(line); err == nil {
return v
}
if utf8.RuneCountInString(line) > maxStatsVersion {
return ""
}
return line
}
// RuleHits is one symbol's hit count from rspamd's /counters.
type RuleHits struct {
Symbol string `json:"symbol"`
Hits int64 `json:"hits"`
}
// RspamdStats is what the web UI shows from rspamd's /stat and /counters.
type RspamdStats struct {
Scanned int64 `json:"scanned"`
Learned int64 `json:"learned"`
HamCount int64 `json:"ham_count"`
SpamCount int64 `json:"spam_count"`
UptimeSeconds int64 `json:"uptime_seconds"` // 0 = unknown
Version string `json:"version"`
Actions map[string]int64 `json:"actions"`
TopRules []RuleHits `json:"top_rules"`
}
// ParseRspamdStat reads the controller's /stat reply. Absent fields stay
// zero and unknown fields are ignored; a known field of the wrong type is
// an error. An overlong version is dropped and an overlong action name
// skipped. Errors are fixed text; they never repeat the reply.
func ParseRspamdStat(body []byte) (RspamdStats, error) {
if t := bytes.TrimSpace(body); len(t) == 0 || t[0] != '{' {
return RspamdStats{}, errors.New("rspamd's /stat reply is not a JSON object")
}
var raw struct {
Scanned int64 `json:"scanned"`
Learned int64 `json:"learned"`
HamCount int64 `json:"ham_count"`
SpamCount int64 `json:"spam_count"`
// Some rspamd versions report uptime with a fraction.
Uptime float64 `json:"uptime"`
Version string `json:"version"`
Actions map[string]int64 `json:"actions"`
}
if err := json.Unmarshal(body, &raw); err != nil {
return RspamdStats{}, errors.New("rspamd's /stat reply has an invalid field")
}
s := RspamdStats{Scanned: raw.Scanned, Learned: raw.Learned, HamCount: raw.HamCount, SpamCount: raw.SpamCount, Actions: raw.Actions}
if utf8.RuneCountInString(raw.Version) <= maxStatsVersion {
s.Version = raw.Version
}
maps.DeleteFunc(s.Actions, func(name string, _ int64) bool { return utf8.RuneCountInString(name) > maxStatsAction })
if raw.Uptime > 0 && raw.Uptime < math.MaxInt64 {
s.UptimeSeconds = int64(raw.Uptime)
}
return s, nil
}
// ParseRspamdCounters reads the controller's /counters reply and returns the
// top symbols by hits, ties by name. An entry without a symbol, with an
// overlong one, or without a whole positive hit count is skipped. Errors
// are fixed text; they never repeat the reply.
func ParseRspamdCounters(body []byte, top int) ([]RuleHits, error) {
if t := bytes.TrimSpace(body); len(t) == 0 || t[0] != '[' {
return nil, errors.New("rspamd's /counters reply is not a JSON array")
}
// rspamd prints some hit counts as doubles: 0.0 for virtual symbols.
var entries []struct {
Symbol string `json:"symbol"`
Hits float64 `json:"hits"`
}
if err := json.Unmarshal(body, &entries); err != nil {
return nil, errors.New("rspamd's /counters reply has an invalid entry")
}
var rules []RuleHits
for _, e := range entries {
if e.Symbol == "" || utf8.RuneCountInString(e.Symbol) > maxStatsSymbol ||
e.Hits < 1 || e.Hits >= math.MaxInt64 || e.Hits != math.Trunc(e.Hits) {
continue
}
rules = append(rules, RuleHits{Symbol: e.Symbol, Hits: int64(e.Hits)})
}
slices.SortFunc(rules, func(a, b RuleHits) int {
return cmp.Or(cmp.Compare(b.Hits, a.Hits), strings.Compare(a.Symbol, b.Symbol))
})
return slices.Clone(rules[:min(len(rules), max(top, 0))]), nil
}
package antispam
import (
"fmt"
"strings"
)
// The systemd units CSM reloads. It never restarts either.
const (
RspamdUnit = "rspamd.service"
EximUnit = "exim.service"
)
// The units postfix runs in; CSM only reads them and never changes postfix.
// Debian and Ubuntu run the master in PostfixInstanceUnit and make
// PostfixUnit a oneshot wrapper that stays active while postfix is down.
const (
PostfixUnit = "postfix.service"
PostfixInstanceUnit = "postfix@-.service"
)
// UnitShowArgs are the `systemctl` arguments the collector runs per unit.
func UnitShowArgs(unit string) []string {
return []string{"show", "-p", "LoadState", "-p", "User", "-p", "Group", "-p", "CanReload", unit}
}
// ParseUnitShow reads `systemctl show -p ...` output: one Key=Value per line.
func ParseUnitShow(out string) map[string]string {
props := make(map[string]string)
for _, line := range strings.Split(out, "\n") {
if key, value, ok := strings.Cut(strings.TrimSpace(line), "="); ok {
props[key] = value
}
}
return props
}
// checkServices requires systemd-run to be available and the services CSM
// reloads to support reloading in place. Postfix, which CSM only reads,
// must be running.
func checkServices(f HostFacts) []Problem {
var out []Problem
add := func(sev Severity, format string, args ...any) {
out = append(out, Problem{Code: CodeSystemd, Severity: sev, Detail: fmt.Sprintf(format, args...)})
}
if f.SystemdRunErr != nil {
add(Temporary, "systemd-run is not available; CSM changes mail configuration only through it")
}
reloaded := []struct {
unit string
show CommandResult
}{{RspamdUnit, f.RspamdService}, {EximUnit, f.EximService}}
if f.Stack == StackPostfix {
reloaded = reloaded[:1]
}
for _, s := range reloaded {
props := ParseUnitShow(s.show.Stdout)
switch {
case s.show.Err != nil:
add(Temporary, "systemctl show %s failed; check systemd", s.unit)
case props["LoadState"] != "loaded":
add(Temporary, "%s is not loaded; check that it is installed", s.unit)
case props["CanReload"] == "no":
add(Definite, "%s cannot reload; CSM applies changes only by reload, never by restart", s.unit)
case props["CanReload"] != "yes":
add(Temporary, "systemctl show %s did not report a readable CanReload property", s.unit)
}
}
if f.Stack == StackPostfix {
props := ParseUnitShow(f.PostfixService.Stdout)
switch {
case f.PostfixService.Err != nil:
add(Temporary, "systemctl show for postfix failed; check systemd")
case props["LoadState"] != "loaded":
add(Temporary, "the postfix service is not loaded; check that postfix is installed")
case props["ActiveState"] != "active":
add(Temporary, "the postfix service is not active; check postfix")
}
}
return out
}
// checkEximLog checks that Exim's main log destination matches the log CSM
// already monitors for scan markers.
func checkEximLog(settings map[string]string, monitored string) []Problem {
var out []Problem
add := func(sev Severity, format string, args ...any) {
out = append(out, Problem{Code: CodeEximLog, Severity: sev, Detail: fmt.Sprintf(format, args...)})
}
raw, ok := settings["log_file_path"]
switch {
case !ok:
add(Temporary, "exim -bP did not report log_file_path")
return out
case monitored == "":
add(Temporary, "the Exim log CSM monitors is unknown")
return out
case strings.ContainsAny(raw, "$<"):
add(Temporary, "Exim's log_file_path uses an expansion or list separator CSM does not evaluate")
return out
}
path, found := firstEximLogPath(raw)
if found && path == "" {
add(Temporary, "Exim uses its default log destination; set an explicit log_file_path so CSM can verify where antispam markers are written")
return out
}
if found && strings.ReplaceAll(path, "%s", "main") == monitored {
return out
}
add(Definite, "Exim does not write its main log to %s, where CSM reads antispam markers; restore log_file_path", monitored)
return out
}
// Exim uses only the first non-syslog item, with doubled colons denoting a
// literal colon. An empty path selects a build-time default we cannot verify.
func firstEximLogPath(raw string) (string, bool) {
const space = " \t\r\n\v\f"
if raw == "" {
return "", true
}
for raw != "" {
raw = strings.TrimLeft(raw, space)
if raw == "" {
break
}
var entry strings.Builder
for raw != "" {
c := raw[0]
raw = raw[1:]
if c == ':' {
if !strings.HasPrefix(raw, ":") {
break
}
raw = raw[1:]
}
entry.WriteByte(c)
}
path := strings.TrimRight(entry.String(), space)
if path != "syslog" {
return path, true
}
}
return "", false
}
// Package antispam renders the Exim and rspamd configuration CSM owns for
// inbound spam filtering and evaluates the host preconditions for it. Every
// function here is pure: callers supply command output and file facts.
package antispam
import (
"errors"
"fmt"
"net/netip"
"path"
"regexp"
"slices"
"strings"
"unicode"
)
const (
// DefaultSocket is the rspamd normal-worker socket CSM scans through.
DefaultSocket = "/run/rspamd/csm-normal.sock"
// DefaultRejectScore and DefaultTagScore are the score thresholds used
// when the operator sets none.
DefaultRejectScore = 15
DefaultTagScore = 6
maxScore = 1000
// maxSocketPath is the sun_path size (108 bytes) minus its NUL.
maxSocketPath = 107
)
// Settings are the validated values rendered into Exim and rspamd files.
type Settings struct {
DryRun bool
Socket string
RejectScore int
TagScore int
// Feeds are the feed names to render; the daemon
// has already dropped those whose preconditions fail.
Feeds []string
// SkipMaps names map redirects left out because CSM cannot own their
// file.
SkipMaps []string
// SkipScores leaves out the symbol weights file because CSM cannot own
// it.
SkipScores bool
// HostScores names the symbols whose weight the host's own rspamd group
// files set; CSM leaves them out of its weights file.
HostScores []string
// DQSKeyFile holds the Spamhaus DQS key. DQSKey is its validated
// content: the caller loads it, and it never travels in a request.
DQSKeyFile string
DQSKey Secret
// Learning installs CSM's fuzzy rules and worker; LearningDir holds
// their keys, read by the worker and never carried in a request.
// Roundcube adds the webmail Junk-button include and needs Learning.
Learning bool
LearningDir string
Roundcube bool
// LearningRedisSocket is the Unix socket of a Redis used only by
// learning; empty means none. Like LearningDir it is ignored while
// Learning is off.
LearningRedisSocket string
// LearningKeys are loaded from LearningDir by the caller, like DQSKey;
// they never travel in a request.
LearningKeys LearningKeys
// Stack is the host's mail stack, which picks the files CSM writes. The
// daemon and the worker each read it from the host; it never travels
// in a request.
Stack MailStack
// Exempt lists recipient mailboxes and domains, in lower case, whose
// mail a Postfix host scores and logs but never enforces on. Exim
// ignores it: each cPanel account's own toggle decides there.
Exempt []string
// ScopeNets are the infra_ips networks. A Postfix host never enforces
// on mail they send.
ScopeNets []netip.Prefix
}
// DefaultLearningDir holds learning state, inside CSM's state directory.
const DefaultLearningDir = "/var/lib/csm/antispam/learning"
// Validate checks every value that reaches a rendered file.
func (s Settings) Validate() error {
if err := ValidateSocket(s.Socket); err != nil {
return err
}
if err := ValidateScores(s.RejectScore, s.TagScore); err != nil {
return err
}
if err := ValidateFeeds(s.Feeds, s.DQSKeyFile); err != nil {
return err
}
if err := validateLearning(s); err != nil {
return err
}
if err := ValidateExempt(s.Exempt); err != nil {
return err
}
for _, p := range s.ScopeNets {
if !p.IsValid() {
return errors.New("an infra_ips network is not valid")
}
}
if err := ValidateSkipMaps(s.SkipMaps); err != nil {
return err
}
return ValidateHostScores(s.HostScores)
}
var (
exemptMailbox = regexp.MustCompile(`^[a-z0-9._+-]+@[a-z0-9.-]+$`)
exemptDomain = regexp.MustCompile(`^[a-z0-9.-]+$`)
)
// Reporter modes of learning on a Postfix host: one vote per mail domain or
// one per mailbox.
const (
ReporterDomain = "domain"
ReporterMailbox = "mailbox"
)
// IsExempt reports whether mailbox is exempt, matched as the scope Lua
// matches a recipient: in lower case, as the whole address or as the part
// after its last @. A name without an @ is never exempt.
func IsExempt(exempt []string, mailbox string) bool {
mailbox = strings.ToLower(mailbox)
at := strings.LastIndexByte(mailbox, '@')
return at >= 0 && (slices.Contains(exempt, mailbox) || slices.Contains(exempt, mailbox[at+1:]))
}
// ValidateExempt accepts lower-case mailboxes and domains only. These are
// the only operator strings rendered into the scope Lua, so none may hold a
// quote, backslash, space or wildcard.
func ValidateExempt(entries []string) error {
for i, e := range entries {
if !exemptMailbox.MatchString(e) && !exemptDomain.MatchString(e) {
return fmt.Errorf("entry %d is not a lower-case mailbox or domain, such as user@example.com or example.com", i+1)
}
}
return nil
}
func validateLearning(s Settings) error {
if s.Roundcube && !s.Learning {
return errors.New("roundcube needs learning")
}
if !s.Learning {
return nil
}
dir := s.LearningDir
if !path.IsAbs(dir) || path.Clean(dir) != dir || strings.ContainsFunc(dir, unicode.IsControl) {
return errors.New("learning directory must be an absolute clean path")
}
if s.LearningRedisSocket != "" {
return ValidateLearningRedisSocket(s.LearningRedisSocket)
}
return nil
}
// HasFeed reports whether name is among s.Feeds.
func (s Settings) HasFeed(name string) bool { return slices.Contains(s.Feeds, name) }
// HostActions reports whether the host's own actions decide, not CSM's
// actions.conf: a Postfix dry run, so the host keeps enforcing.
func (s Settings) HostActions() bool { return s.Stack == StackPostfix && s.DryRun }
// DNSFeeds returns the catalogue entries of s's DNS feeds in catalogue
// order, so rendering never depends on the order the operator wrote.
func (s Settings) DNSFeeds() []Feed {
var out []Feed
for _, f := range catalogue {
if f.DNS() && s.HasFeed(f.Name) {
out = append(out, f)
}
}
return out
}
// Redirects reports whether rspamd.com maps are pointed at local copies.
func (s Settings) Redirects() bool { return !s.HasFeed(FeedRspamdMaps) }
// ValidateSkipMaps accepts known map redirect names, once each.
func ValidateSkipMaps(names []string) error {
seen := make(map[string]bool, len(names))
for _, name := range names {
if !slices.Contains(MapRedirectNames(), name) {
return errors.New("unknown map redirect")
}
if seen[name] {
return fmt.Errorf("map redirect %q is listed twice", name)
}
seen[name] = true
}
return nil
}
// ValidateHostScores accepts symbols CSM weighs, once each.
func ValidateHostScores(names []string) error {
seen := make(map[string]bool, len(names))
for _, name := range names {
if !slices.ContainsFunc(scoreWeights, func(w scoreWeight) bool { return w.symbol == name }) {
return errors.New("unknown score symbol")
}
if seen[name] {
return fmt.Errorf("score symbol %q is listed twice", name)
}
seen[name] = true
}
return nil
}
// ValidateSocket accepts a clean absolute path made only of ASCII letters,
// digits, '/', '.', '_' and '-'. The socket path is the only operator string
// rendered into Exim configuration, so it may hold no expansion, list or
// whitespace character.
func ValidateSocket(p string) error { return validateSocketPath("rspamd_socket", p) }
// ValidateLearningRedisSocket applies the same rules to the learning Redis
// socket, which is rendered into a quoted rspamd UCL string.
func ValidateLearningRedisSocket(p string) error { return validateSocketPath("redis_socket", p) }
func validateSocketPath(key, p string) error {
if p == "" {
return fmt.Errorf("%s is empty", key)
}
if len(p) > maxSocketPath {
return fmt.Errorf("%s is longer than %d bytes", key, maxSocketPath)
}
if p[0] != '/' || p == "/" {
return fmt.Errorf("%s must be an absolute path to a socket file", key)
}
for i := 0; i < len(p); i++ {
c := p[i]
switch {
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9',
c == '/', c == '.', c == '_', c == '-':
default:
return fmt.Errorf("%s contains %q; only letters, digits, '/', '.', '_' and '-' are allowed", key, c)
}
}
if path.Clean(p) != p {
return fmt.Errorf("%s must be a clean path: no empty, '.' or '..' elements and no trailing slash", key)
}
return nil
}
// ValidateScores requires 0 < tag < reject <= 1000. Exim compares scores in
// tenths, so the bound keeps every rendered threshold a small integer.
func ValidateScores(reject, tag int) error {
if reject < 1 || reject > maxScore {
return fmt.Errorf("reject_score must be between 1 and %d", maxScore)
}
if tag < 1 || tag > maxScore {
return fmt.Errorf("tag_score must be between 1 and %d", maxScore)
}
if tag >= reject {
return errors.New("tag_score must be lower than reject_score")
}
return nil
}
package antispam
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"strconv"
"strings"
)
// DefaultMaxMessage is rspamd's built-in max_message (DEFAULT_MAX_MESSAGE in
// rspamd 3.0 and 4.2.1), in effect when configdump omits the setting.
const DefaultMaxMessage = 50 << 20
// sizeHeadroom is what Exim adds around a message before rspamd reads it:
// the mbox From line, X-Envelope-From, an X-Envelope-To listing every
// recipient, and Exim's Received header. A larger envelope makes rspamd
// refuse the scan, which fails open with a scan error.
const sizeHeadroom = 1 << 20
// EximSettingNames are the options the collector reads with one `exim -bP`.
// CheckHost needs message_size_limit and log_file_path; the collector uses
// exim_user, exim_group and system_filter, and reconcile the ACL names to
// verify the generated configuration.
func EximSettingNames() []string {
return []string{"message_size_limit", "log_file_path", "exim_user", "exim_group", "system_filter", "acl_smtp_rcpt", "acl_smtp_data"}
}
// ParseEximSize reads a non-negative Exim integer with an optional sign
// and K, M or G suffix in either case (powers of 1024).
func ParseEximSize(s string) (int64, error) {
s = strings.TrimSpace(s)
shift := 0
if s != "" {
switch s[len(s)-1] {
case 'k', 'K':
shift = 10
case 'm', 'M':
shift = 20
case 'g', 'G':
shift = 30
}
}
if shift != 0 {
s = s[:len(s)-1]
}
digits := s
if digits != "" && (digits[0] == '+' || digits[0] == '-') {
digits = digits[1:]
}
if digits == "" || strings.Trim(digits, "0123456789") != "" {
return 0, errors.New("not a plain Exim size")
}
n, err := strconv.ParseInt(s, 10, 64)
if err != nil || n < 0 || n > (1<<62)>>shift {
return 0, errors.New("size out of range")
}
return n << shift, nil
}
// MaxMessage reads max_message from `rspamadm configdump -j options`. Zero
// means unlimited; an absent setting is DefaultMaxMessage. Rspamd converts
// numeric values to bytes, discarding any fractional part of a float.
func MaxMessage(out []byte) (int64, error) {
opts, err := dumpObject(out, "options")
if err != nil {
return 0, err
}
raw, ok := opts["max_message"]
if !ok {
return DefaultMaxMessage, nil
}
num, ok := raw.(json.Number)
if !ok {
return 0, errors.New("configdump options: max_message is not a number")
}
n, err := num.Int64()
if err == nil && n >= 0 {
return n, nil
}
// Rspamd's UCL integer conversion also accepts floating-point values.
// Keep plain integers exact and use the same truncation for floats.
size, err := strconv.ParseFloat(num.String(), 64)
if err != nil || size < 0 || size >= 1<<63 {
return 0, errors.New("configdump options: max_message is not a byte count")
}
return int64(size), nil
}
// dumpObject decodes one `rspamadm configdump -j <section>` object. A
// duplicate can hide one value behind another, so fields are decoded
// separately and a repeated name is an error, not the last one winning.
func dumpObject(out []byte, section string) (map[string]any, error) {
return dumpObjectAs[any](out, section)
}
// RawMessage values let nested sections receive the same duplicate checks.
func dumpObjectAs[T any](out []byte, section string) (map[string]T, error) {
dec := json.NewDecoder(bytes.NewReader(out))
dec.UseNumber()
if token, err := dec.Token(); err != nil || token != json.Delim('{') {
return nil, fmt.Errorf("configdump %s: expected an object", section)
}
fields := make(map[string]T)
for dec.More() {
token, err := dec.Token()
if err != nil {
return nil, fmt.Errorf("configdump %s: invalid JSON", section)
}
key, _ := token.(string)
if _, seen := fields[key]; seen {
return nil, fmt.Errorf("configdump %s: duplicate field", section)
}
var value T
if err := dec.Decode(&value); err != nil {
return nil, fmt.Errorf("configdump %s: invalid JSON", section)
}
fields[key] = value
}
if token, err := dec.Token(); err != nil || token != json.Delim('}') {
return nil, fmt.Errorf("configdump %s: invalid JSON", section)
}
if err := dec.Decode(new(any)); !errors.Is(err, io.EOF) {
return nil, fmt.Errorf("configdump %s: trailing data", section)
}
return fields, nil
}
// checkMessageSize checks that rspamd accepts every message Exim accepts,
// with the envelope Exim adds.
func checkMessageSize(settings map[string]string, options CommandResult) []Problem {
var out []Problem
add := func(sev Severity, format string, args ...any) {
out = append(out, Problem{Code: CodeMessageSize, Severity: sev, Detail: fmt.Sprintf(format, args...)})
}
raw, ok := settings["message_size_limit"]
if !ok {
add(Temporary, "exim -bP did not report message_size_limit")
return out
}
limit, err := ParseEximSize(raw)
switch {
case err != nil:
add(Temporary, "Exim's message_size_limit is not a plain size; CSM cannot compare it with rspamd's max_message")
return out
case limit == 0:
add(Definite, "Exim's message_size_limit is unlimited; set a finite limit so rspamd can scan every accepted message")
return out
}
if maxMessage, ok := rspamdMaxMessage(options, add); ok && maxMessage != 0 && maxMessage < limit+sizeHeadroom {
add(Definite, "rspamd max_message is %d bytes, below Exim's message_size_limit of %d bytes plus %d bytes for the envelope Exim adds; raise max_message in /etc/rspamd/local.d/options.inc",
maxMessage, limit, sizeHeadroom)
}
return out
}
// checkPostfixMessageSize warns when postfix accepts messages rspamd does
// not scan. The proxy worker passes them on, so mail keeps flowing; raising
// the limit is the operator's call. A limit of 0 is unlimited on both sides.
func checkPostfixMessageSize(limit int64, options CommandResult) []Problem {
var out []Problem
add := func(sev Severity, format string, args ...any) {
out = append(out, Problem{Code: CodeMessageSize, Severity: sev, Detail: fmt.Sprintf(format, args...)})
}
maxMessage, ok := rspamdMaxMessage(options, add)
switch {
case !ok || maxMessage == 0:
case limit == 0:
add(Warning, "postfix's message_size_limit is unlimited and rspamd max_message is %d bytes; rspamd does not scan larger messages", maxMessage)
case maxMessage < limit:
add(Warning, "rspamd max_message is %d bytes, below postfix's message_size_limit of %d bytes; rspamd does not scan larger messages; raise max_message in /etc/rspamd/local.d/options.inc",
maxMessage, limit)
}
return out
}
// rspamdMaxMessage reads rspamd's loaded max_message, or reports why it
// cannot.
func rspamdMaxMessage(options CommandResult, add func(Severity, string, ...any)) (int64, bool) {
if options.Err != nil {
add(Temporary, "rspamadm configdump options failed; check rspamd")
return 0, false
}
maxMessage, err := MaxMessage([]byte(options.Stdout))
if err != nil {
add(Temporary, "rspamadm configdump did not report a readable max_message")
return 0, false
}
return maxMessage, true
}
package antispam
import (
"errors"
"fmt"
"io/fs"
"path"
"strings"
)
// GeneratedEximConfPath is the configuration cPanel's buildeximconf writes.
const GeneratedEximConfPath = "/etc/exim.conf"
// Files CSM writes on a cPanel host.
const (
HookRCPTPath = "/usr/local/cpanel/etc/exim/acls/ACL_SPAM_SCAN_CHECK_BLOCK/custom_begin_spam_scan_check"
HookScanPath = "/usr/local/cpanel/etc/exim/acls/ACL_SPAM_SCAN_BLOCK/custom_begin_spam_scan"
HookPostPath = "/usr/local/cpanel/etc/exim/acls/ACL_CHECK_MESSAGE_POST_BLOCK/custom_begin_check_message_post"
LocalEximConfPath = "/etc/exim.conf.local"
RspamdLocalDir = "/etc/rspamd/local.d"
// ActionsPath holds rspamd's action thresholds.
ActionsPath = RspamdLocalDir + "/actions.conf"
)
// Files CSM writes only on a Postfix host: the headers
// rspamd's milter adds and removes, and the Lua that scopes enforcement.
const (
MilterHeadersPath = RspamdLocalDir + "/milter_headers.conf"
PostfixScopeLuaPath = RspamdLuaDir + "/csm_scope.lua"
)
// TargetKind says which ownership rule a mutation target follows.
type TargetKind int
const (
// TargetRspamd is a local.d file CSM owns outright.
TargetRspamd TargetKind = iota
// TargetEximHook is a shared cPanel ACL hook file: at most one CSM block.
TargetEximHook
// TargetEximLocal is /etc/exim.conf.local: shared, with one @CONFIG@ line.
TargetEximLocal
// TargetRspamdOptional is an rspamd file CSM owns outright while a feed
// or map redirect needs it. A file CSM cannot own never blocks core: the
// daemon leaves the feed or redirect out instead. On a
// Postfix host actions.conf is one too, needed only to enforce.
TargetRspamdOptional
// TargetLuaOptional is an rspamd Lua plugin CSM owns outright while
// learning needs it. Its marker is a Lua comment.
TargetLuaOptional
// TargetRoundcube is the webmail include CSM owns outright while
// learning sets the Junk button's folder. It needs no service reload.
TargetRoundcube
// TargetRspamdLua is an rspamd Lua file CSM owns outright on every
// install. Its marker is a Lua comment.
TargetRspamdLua
)
// RoundcubeIncludePath is the Roundcube include that points the Junk button
// at the Spam folder learning reads.
const RoundcubeIncludePath = "/usr/local/cpanel/base/3rdparty/roundcube/config/inc.d/zz-csm-antispam.inc.php"
// WholeFile reports whether CSM owns the target's entire content.
func (k TargetKind) WholeFile() bool {
return k == TargetRspamd || k == TargetRspamdLua || k.Optional()
}
// Optional reports whether CSM writes the target only while a feature
// needs it; the daemon decides whether it may (OptionalUsable).
func (k TargetKind) Optional() bool {
return k == TargetRspamdOptional || k == TargetLuaOptional || k == TargetRoundcube
}
// ReloadsRspamd reports whether a change to the target needs an rspamd
// reload.
func (k TargetKind) ReloadsRspamd() bool {
return k == TargetRspamd || k == TargetRspamdOptional || k == TargetLuaOptional || k == TargetRspamdLua
}
// ManagedFor reports whether content is a whole file CSM wrote for a target
// of kind k, by the marker that kind's syntax allows. Shared Exim files
// carry blocks, never a whole-file marker.
func ManagedFor(k TargetKind, content string) bool {
switch k {
case TargetRspamd, TargetRspamdOptional:
return IsManaged(content)
case TargetLuaOptional, TargetRspamdLua:
first, _, _ := strings.Cut(content, "\n")
return first == LuaManagedHeader
case TargetRoundcube:
first, rest, _ := strings.Cut(content, "\n")
second, _, _ := strings.Cut(rest, "\n")
return first == "<?php" && second == ManagedHeader
}
return false
}
// Target is one file CSM writes.
type Target struct {
Path string
Kind TargetKind
}
// MutationTargets lists every file CSM writes on a host of stack, rspamd
// first as in the apply transaction. A Postfix host gets
// the milter headers and the scope Lua and no cPanel file; its actions.conf
// is optional, because a dry run leaves it to the host. Any other
// stack gets the cPanel list.
func MutationTargets(stack MailStack) []Target {
postfix := stack == StackPostfix
optional := OptionalRspamdPaths()
learning := LearningPaths()
out := make([]Target, 0, len(RspamdFiles)+len(optional)+len(learning)+5)
for _, name := range RspamdFiles {
kind := TargetRspamd
if postfix && name == "actions.conf" {
kind = TargetRspamdOptional
}
out = append(out, Target{Path: path.Join(RspamdLocalDir, name), Kind: kind})
}
if postfix {
out = append(out, Target{Path: MilterHeadersPath, Kind: TargetRspamd}, Target{Path: PostfixScopeLuaPath, Kind: TargetRspamdLua})
}
for _, p := range optional {
out = append(out, Target{Path: p, Kind: TargetRspamdOptional})
}
for _, p := range learning {
kind := TargetRspamdOptional
if p == LearningLuaPath {
kind = TargetLuaOptional
}
out = append(out, Target{Path: p, Kind: kind})
}
if postfix {
return out
}
return append(out,
Target{Path: RoundcubeIncludePath, Kind: TargetRoundcube},
Target{Path: HookRCPTPath, Kind: TargetEximHook},
Target{Path: HookScanPath, Kind: TargetEximHook},
Target{Path: HookPostPath, Kind: TargetEximHook},
Target{Path: LocalEximConfPath, Kind: TargetEximLocal},
)
}
// TargetFacts describes one mutation target before CSM writes it.
type TargetFacts struct {
Target
// Err is a failure other than the target's absence. It wraps
// fs.ErrNotExist when a parent directory is missing.
Err error
// Missing is true when the target does not exist; CSM creates it.
Missing bool
// File is the target's own lstat, including Path, set after a successful
// stat. It remains available if reading Content then fails.
File PathFacts
// Parents run from "/" down to the target's directory, without following
// symbolic links. The walk stops at the first entry that is not a directory.
Parents []PathFacts
// Content is the text of an existing protected regular file. Unsafe
// targets are rejected from their path facts without reading content.
Content string
}
// CheckRemoval applies the target rules that still hold when CSM removes its
// integration: it never writes through an unsafe path or
// edits a file whose markers it cannot identify. A foreign local.d file stays
// in place and a missing directory has nothing to remove. targets must be
// the facts of MutationTargets(stack).
func CheckRemoval(stack MailStack, targets []TargetFacts) []Problem {
if stack != StackExim && stack != StackPostfix {
return []Problem{stackUnknown}
}
return checkTargets(stack, targets, true)
}
// stackUnknown stops changes to an installed integration on a host whose
// mail stack CSM cannot tell. A cPanel host always runs Exim, so this is a
// Postfix host, for example one with both MTAs installed while postfix
// restarts. Removal needs the stack's own files, and the Exim path would
// rebuild a configuration the host does not have.
var stackUnknown = Problem{Code: CodePlatform, Severity: Temporary, Detail: "CSM cannot tell which mail server this host runs, for example while Postfix restarts"}
// checkTargets checks required targets for protected root-owned directories
// and regular, protected files when present. Shared files must accept CSM's
// block. Optional targets are checked separately on apply; removal checks
// their parents when they contain CSM content. The worker backs up foreign
// whole files before replacing them. Removal checks only what protects an edit.
func checkTargets(stack MailStack, targets []TargetFacts, removal bool) []Problem {
var out []Problem
add := func(code string, sev Severity, format string, args ...any) {
out = append(out, Problem{Code: code, Severity: sev, Detail: fmt.Sprintf(format, args...)})
}
if len(targets) != len(MutationTargets(stack)) {
add(CodeTargets, Temporary, "the files CSM writes were not checked")
}
reported := make(map[string]bool)
for _, t := range targets {
// Optional files are the daemon's to judge (OptionalUsable). Removal
// edits only files CSM wrote, and must not do so through an unsafe path.
if t.Kind.Optional() && (!removal || !ManagedFor(t.Kind, t.Content)) {
continue
}
safe := true
for _, p := range t.Parents {
problem := unprotected(p)
if !p.Mode.IsDir() {
problem = "is not a directory; CSM does not write through symbolic links"
}
if problem == "" {
continue
}
safe = false
if !reported[p.Path] {
reported[p.Path] = true
add(CodeTargets, Definite, "%s %s", p.Path, problem)
}
}
if !safe {
continue
}
// A later read error must not hide unsafe metadata already collected.
// Removal only deletes an optional file, which needs a safe
// directory, not a safe file: a loosened one must still go.
if !t.Missing && t.File.Path != "" && (!removal || !t.Kind.Optional()) {
if !t.File.Mode.IsRegular() {
add(CodeTargets, Definite, "%s is not a regular file; CSM does not write through symbolic links", t.Path)
continue
}
if problem := unprotected(t.File); problem != "" {
add(CodeTargets, Definite, "%s %s", t.Path, problem)
continue
}
}
switch {
case errors.Is(t.Err, fs.ErrNotExist):
if dir := path.Dir(t.Path); !removal && !reported[dir] {
reported[dir] = true
add(CodeTargets, Definite, "the directory %s is missing", dir)
}
continue
case t.Err != nil:
add(CodeTargets, Temporary, "cannot check %s; check file access", t.Path)
continue
case len(t.Parents) == 0:
add(CodeTargets, Temporary, "the parent directories for %s were not checked", t.Path)
continue
case t.Missing:
continue
case t.File.Path == "":
add(CodeTargets, Temporary, "the file metadata for %s was not checked", t.Path)
continue
}
switch t.Kind {
case TargetEximLocal:
if !removal {
// A blank file is filled with cPanel's skeleton, as forward
// guard does.
if strings.TrimSpace(t.Content) != "" && !oneConfigSection(t.Content) {
add(CodeTargets, Definite, "%s must hold exactly one @CONFIG@ line, alone and unindented", t.Path)
}
continue
}
fallthrough
case TargetEximHook:
// Hook files are ACL fragments, so trailing options can extend
// CSM's last statement even without a section header in the file.
switch blocks, err := fileBlocks(t.Kind, t.Content); {
case errors.Is(err, ErrUnverifiable):
add(CodeUnverifiable, Temporary, "cannot verify %s: %v", t.Path, err)
case err != nil:
add(CodeMarkers, Definite, "%s: %v", t.Path, err)
case len(blocks) > 1:
add(CodeMarkers, Definite, "%s has multiple CSM blocks; only one is allowed", t.Path)
}
}
}
return out
}
// unprotected returns why another user could change p, or "".
func unprotected(p PathFacts) string {
switch {
case p.UID != 0:
return fmt.Sprintf("is owned by uid %d; want root", p.UID)
case p.Mode.Perm()&0o022 != 0:
return "is writable by group or others"
case p.ExtACL:
return "carries extra ACL entries"
}
return ""
}
// oneConfigSection reports whether conf has exactly one line that reads
// @CONFIG@ once surrounding blanks are trimmed, and that line is exact.
func oneConfigSection(conf string) bool {
n, exact := 0, false
for _, line := range strings.Split(conf, "\n") {
if strings.TrimSpace(line) == "@CONFIG@" {
n++
exact = line == "@CONFIG@"
}
}
return n == 1 && exact
}
package txn
import (
"context"
"errors"
"slices"
"strings"
"github.com/pidginhost/csm/internal/antispam"
)
var errEximInactive = errors.New("exim is not running; CSM changes its configuration only while it can confirm the reload")
// Apply installs or updates CSM's integration for s. An
// equal installation changes nothing and reloads nothing.
func (h *Host) Apply(s antispam.Settings) antispam.Result {
t := h.begin()
defer t.cancel()
t.rspamdProbe = func(ctx context.Context) error {
return retry(ctx, h.rspamdWait, func(ctx context.Context) error { return h.ping(ctx, s.Socket) })
}
f, problems := h.preflight(t.fwd, s)
if err := t.fwd.Err(); err != nil {
return failed(StepPreconditions, err)
}
if err := blocking(problems); err != nil {
return failed(StepPreconditions, err)
}
// Takeovers re-check the files of the worker's own detection, so it
// must match the stack preflight collected.
if f.Stack != h.stack() {
return failed(StepPreconditions, errors.New("the host's mail stack changed while CSM checked it"))
}
// The host's mail stack picks the files CSM renders; a request never
// names it.
s.Stack = f.Stack
// The host can change while the resolved request waits for the worker.
if !s.SkipScores {
resolved, _ := antispam.ResolveScores(s, f)
for _, symbol := range resolved.HostScores {
if !slices.Contains(s.HostScores, symbol) {
return failed(StepPreconditions, errors.New("rspamd operator weights changed; retry with current host settings"))
}
}
}
if len(s.DNSFeeds()) > 0 || s.Redirects() || s.HasFeed(antispam.FeedCSMIPMap) {
if ok, why := antispam.FeedsSupported(f.RspamdVersion); !ok {
return failed(StepPreconditions, errors.New(why+"; CSM's feeds and map redirects need rspamd 4 or later"))
}
}
if s.HasFeed(antispam.FeedCSMIPMap) {
if err := f.IPMap.UsableByRspamd(f.RspamdGID); err != nil {
return failed(StepPreconditions, err)
}
}
if s.HasFeed(antispam.FeedSpamhausDQS) {
key, err := antispam.LoadDQSKey(f.DQSKey)
if err != nil {
return failed(StepPreconditions, err)
}
if exposed, why := f.RspamdLog.Exposed(f.Socket.RspamdUID, f.RspamdGID); exposed {
return failed(StepPreconditions, errors.New(why+"; refusing to enable DQS lookups"))
}
s.DQSKey = key
}
if s.Learning {
if f.LearningKeysErr != nil {
return failed(StepPreconditions, f.LearningKeysErr)
}
// The pass judged the dedicated Redis before this lock was taken.
if s.LearningRedisSocket != "" {
if err := antispam.CheckLearningRedis(s, f); err != nil {
return failed(StepPreconditions, err)
}
}
s.LearningKeys = f.LearningKeys
}
installed := antispam.InstalledContent(f.Targets)
if f.Stack == antispam.StackPostfix {
return t.applyPostfix(s, f, installed)
}
rollbackSocket := s.Socket
// An update can select a new socket. Rollback must probe the scanner
// referenced by the hooks it restores, rather than the failed update.
address := antispam.ParseEximOptions(installed[antispam.LocalEximConfPath])["spamd_address"]
if fields := strings.Fields(address); len(fields) > 0 && antispam.ValidateSocket(fields[0]) == nil {
rollbackSocket = fields[0]
}
t.rspamdRollbackProbe = func(ctx context.Context) error {
return retry(ctx, h.rspamdWait, func(ctx context.Context) error { return h.ping(ctx, rollbackSocket) })
}
hooks, err := antispam.RenderExim(s)
if err != nil {
return failed(StepPreconditions, err)
}
managed, changes, err := changesFor(s, f)
if err != nil {
return failed(StepPreconditions, err)
}
generation := antispam.Generation(managed)
rspamdChanges, eximChanges, roundcubeChanges := split(changes)
matches, err := t.recorded(installed, generation)
if err != nil {
return failed(StepRecord, err)
}
opts, err := t.eximSettings(t.fwd)
if err != nil {
return failed(StepPreconditions, err)
}
t.rememberEximBlocks(f.Targets, opts)
current, err := readFile(h.path(antispam.GeneratedEximConfPath))
generated := err == nil && antispam.CheckGenerated(current.content, hooks, opts["acl_smtp_rcpt"], opts["acl_smtp_data"]) == nil
reloadRspamd := len(rspamdChanges) > 0 || !matches
rebuildExim := len(eximChanges) > 0 || !matches || !generated
if !reloadRspamd && !rebuildExim && len(roundcubeChanges) == 0 {
return antispam.Result{OK: true, Generation: generation}
}
if err := t.listenerProbe(opts); err != nil {
return failed(StepPreconditions, err)
}
if !t.eximActive {
return failed(StepPreconditions, errEximInactive)
}
if err := t.writeRecord(""); err != nil {
return t.fail(StepRecord, err)
}
if reloadRspamd {
if step, err := t.rspamd(rspamdChanges, &s, f); err != nil {
return t.fail(step, err)
}
}
if rebuildExim {
verify := func(conf, rcptACL, dataACL string) error {
return antispam.CheckGenerated(conf, hooks, rcptACL, dataACL)
}
if step, err := t.exim(eximChanges, true, verify); err != nil {
return t.fail(step, err)
}
}
roundcube, rcErr := t.roundcube(roundcubeChanges)
if rcErr != nil {
return t.fail(StepRoundcube, rcErr)
}
if roundcube != "" && s.Roundcube {
// Verification restored the snapshot, which may hold an older
// owned include rather than absence. Record that exact state.
if prior, present := installed[antispam.RoundcubeIncludePath]; present {
managed[antispam.RoundcubeIncludePath] = prior
} else {
delete(managed, antispam.RoundcubeIncludePath)
}
generation = antispam.Generation(managed)
}
if err := t.writeRecord(generation); err != nil {
return t.fail(StepComplete, err)
}
return antispam.Result{OK: true, Generation: generation, Roundcube: roundcube}
}
// applyPostfix installs or updates CSM's rspamd files on a Postfix host.
// rspamd's milter enforces there, so the transaction never touches Exim or
// postfix's configuration; probes after each reload include a connection to
// every milter postfix passes mail to rspamd through.
func (t *tx) applyPostfix(s antispam.Settings, f antispam.HostFacts, installed map[string]string) antispam.Result {
milters, _, err := antispam.RspamdMilters(f)
if err != nil {
return failed(StepPreconditions, err)
}
// Proxy settings can change after the daemon assesses the host.
if rejectErr := blocking(antispam.CheckQuietRejects(s, f)); rejectErr != nil {
return failed(StepPreconditions, rejectErr)
}
reachable := func(ctx context.Context) error {
for _, m := range milters {
if t.h.milter(ctx, m) != nil {
return errors.New("rspamd's milter does not accept connections")
}
}
return nil
}
ping := t.rspamdProbe
t.rspamdProbe = func(ctx context.Context) error {
if pingErr := ping(ctx); pingErr != nil {
return pingErr
}
return retry(ctx, t.h.rspamdWait, reachable)
}
t.rspamdRollbackProbe = t.rspamdProbe
managed, changes, err := changesFor(s, f)
if err != nil {
return failed(StepPreconditions, err)
}
if s.HostActions() {
// The host's own thresholds decide again, not rspamd's built-in ones.
if err = t.restoreCopies(changes, func(p string) bool { return p == antispam.ActionsPath }); err != nil {
return failed(StepPreconditions, err)
}
}
generation := antispam.Generation(managed)
matches, err := t.recorded(installed, generation)
if err != nil {
return failed(StepRecord, err)
}
if len(changes) == 0 && matches {
return antispam.Result{OK: true, Generation: generation}
}
// As with Exim's listener, the probe after the reload compares like
// with like.
if err := retry(t.fwd, t.h.rspamdWait, reachable); err != nil {
return failed(StepPreconditions, err)
}
if err := t.writeRecord(""); err != nil {
return t.fail(StepRecord, err)
}
if step, err := t.rspamd(changes, &s, f); err != nil {
return t.fail(step, err)
}
if err := t.consumeRestored(); err != nil {
return t.fail(StepComplete, err)
}
if err := t.writeRecord(generation); err != nil {
return t.fail(StepComplete, err)
}
return antispam.Result{OK: true, Generation: generation}
}
// changesFor renders s and plans the edits that bring f's targets to it.
func changesFor(s antispam.Settings, f antispam.HostFacts) (map[string]string, []change, error) {
managed, err := antispam.ManagedContent(s)
if err != nil {
return nil, nil, err
}
changes, err := plan(f.Targets, managed)
if err != nil {
return nil, nil, err
}
if secrets := s.SecretTargets(); len(secrets) > 0 {
if f.RspamdGID < 0 {
return nil, nil, errors.New("rspamd's group is unknown, so secret files cannot be restricted to it")
}
changes = secureSecret(changes, f.Targets, f.RspamdGID, secrets)
}
return managed, changes, nil
}
// recorded reads the record and reports whether it completed installed or
// generation. Either proves the unchanged services were already reloaded,
// so repairing drift in just one target must not reload the others. An
// incomplete record proves neither.
func (t *tx) recorded(installed map[string]string, generation string) (bool, error) {
record, valid, err := t.readRecord()
if err != nil {
return false, err
}
return valid && record.Complete() &&
(*record.Generation == antispam.Generation(installed) || *record.Generation == generation), nil
}
// rspamd writes the rspamd edits, validates them against s, reloads rspamd
// and probes it, then verifies learning as loaded.
func (t *tx) rspamd(changes []change, s *antispam.Settings, f antispam.HostFacts) (string, error) {
for _, c := range changes {
if err := t.write(c); err != nil {
return StepRspamdFiles, err
}
}
if err := t.checkRspamd(t.fwd, s); err != nil {
step := StepRspamdCheck
if errors.Is(err, errFeedsLoaded) {
step = StepFeedsCheck
} else if errors.Is(err, errLearningLoaded) {
step = StepLearningCheck
}
return step, err
}
if err := t.reloadRspamd(t.fwd, t.rspamdProbe); err != nil {
return StepRspamdReload, err
}
if s.Learning {
if err := t.checkLearning(t.fwd, f, s.LearningKeys, s.LearningRedisSocket); err != nil {
step := StepLearningCheck
if errors.Is(err, errLearningProbe) {
step = StepLearningProbe
}
return step, err
}
}
return "", nil
}
// Package txn runs CSM's antispam apply and remove transactions on cPanel
// and Postfix hosts. It runs as root in the antispam worker,
// under the shared Exim mutation lock and outside the daemon's sandbox.
package txn
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"syscall"
"github.com/pidginhost/csm/internal/safepath"
)
// maxFileBytes bounds every file a transaction reads; cPanel's generated
// exim.conf is under 100 KiB.
const maxFileBytes = 16 << 20
// fileState is one file as a transaction saw it.
type fileState struct {
exists bool
content string
uid, gid int
perm fs.FileMode
// metadata marks perm, uid and gid as the state to write, so a restore
// reinstates a deleted file's owner and mode, including mode 0000.
metadata bool
}
// sameContent reports whether two states hold the same bytes, or are both
// absent. Owner and mode changes are not content edits.
func sameContent(a, b fileState) bool {
return a.exists == b.exists && a.content == b.content
}
// readFile reads p without following a final symbolic link or blocking on a
// FIFO. The caller has checked p's parent directories.
func readFile(p string) (fileState, error) {
// #nosec G304 -- fixed cPanel, rspamd and CSM paths from antispam.MutationTargets and the record path.
f, err := os.OpenFile(p, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if errors.Is(err, fs.ErrNotExist) {
return fileState{}, nil
}
if err != nil {
return fileState{}, err
}
defer func() { _ = f.Close() }()
return readOpenFile(f)
}
func readOpenFile(f *os.File) (fileState, error) {
info, err := f.Stat()
if err != nil {
return fileState{}, err
}
if !info.Mode().IsRegular() {
return fileState{}, fmt.Errorf("%s is not a regular file", f.Name())
}
data, err := io.ReadAll(io.LimitReader(f, maxFileBytes+1))
if err != nil {
return fileState{}, err
}
if len(data) > maxFileBytes {
return fileState{}, fmt.Errorf("%s is larger than %d bytes", f.Name(), maxFileBytes)
}
st := fileState{exists: true, content: string(data), uid: -1, gid: -1, perm: fileMode(info.Mode()), metadata: true}
if sys, ok := info.Sys().(*syscall.Stat_t); ok {
st.uid, st.gid = int(sys.Uid), int(sys.Gid)
}
return st, nil
}
func fileMode(mode fs.FileMode) fs.FileMode {
return mode & (fs.ModePerm | fs.ModeSetuid | fs.ModeSetgid | fs.ModeSticky)
}
// errConflict marks a file someone else changed during the transaction. It
// is never overwritten from a stale snapshot.
var errConflict = errors.New("changed by another writer during the transaction")
// errDurability means the edit reached the destination but was not fsynced.
var errDurability = errors.New("file changed but directory sync failed")
// replaceFile makes p hold next, provided p still holds want. An existing
// file keeps its owner and mode and a new one is 0644 and owned by the
// worker, unless next carries metadata. next.exists false removes p.
func replaceFile(p string, want, next fileState) error {
cur, err := readFile(p)
if err != nil {
return err
}
if !sameContent(cur, want) {
return fmt.Errorf("%s %w", p, errConflict)
}
if !next.exists {
return deleteFile(p)
}
tmp, err := stageFile(p, next.content)
if err != nil {
return err
}
defer discardTemp(tmp)
return replaceStagedFile(p, want, tmp, next)
}
// Staging and syncing may take long enough for another writer to change p.
// Recheck its content and refresh its metadata before publishing.
func replaceStagedFile(p string, want fileState, tmp *os.File, next fileState) error {
cur, err := readFile(p)
if err != nil {
return err
}
if !sameContent(cur, want) {
return fmt.Errorf("%s %w", p, errConflict)
}
perm, uid, gid := fs.FileMode(0o644), -1, -1
if cur.exists {
perm, uid, gid = cur.perm, cur.uid, cur.gid
}
if next.metadata {
perm, uid, gid = next.perm, next.uid, next.gid
}
return publishFile(p, tmp, perm, uid, gid, cur.exists)
}
// writeAtomic writes p through a temporary file in its directory, fsyncs it,
// renames it into place and fsyncs the directory. IDs of -1 keep the worker's
// ownership of the temporary file.
func writeAtomic(p, content string, perm fs.FileMode, uid, gid int) error {
tmp, err := stageFile(p, content)
if err != nil {
return err
}
defer discardTemp(tmp)
return publishFile(p, tmp, perm, uid, gid, true)
}
// removeTemp deletes a temporary file; tests replace it.
var removeTemp = os.Remove
func discardTemp(tmp *os.File) {
_ = tmp.Close()
_ = removeTemp(tmp.Name())
}
func stageFile(p, content string) (*os.File, error) {
if len(content) > maxFileBytes {
return nil, fmt.Errorf("%s is larger than %d bytes", p, maxFileBytes)
}
dir := filepath.Dir(p)
tmp, err := os.CreateTemp(dir, "."+filepath.Base(p)+".*.csmtmp")
if err != nil {
return nil, err
}
if _, err := io.WriteString(tmp, content); err != nil {
discardTemp(tmp)
return nil, err
}
if err := tmp.Sync(); err != nil {
discardTemp(tmp)
return nil, err
}
return tmp, nil
}
func publishFile(p string, tmp *os.File, perm fs.FileMode, uid, gid int, replace bool) error {
if uid >= 0 || gid >= 0 {
if err := tmp.Chown(uid, gid); err != nil {
return err
}
}
// Writing and chown can clear set-id bits, so set the final mode last.
if err := tmp.Chmod(perm); err != nil {
return err
}
if err := tmp.Sync(); err != nil {
return err
}
if err := tmp.Close(); err != nil {
return err
}
if replace {
if err := os.Rename(tmp.Name(), p); err != nil {
return err
}
} else {
// Linking publishes a new file atomically without overwriting a
// destination created after the content check.
if err := os.Link(tmp.Name(), p); err != nil {
if errors.Is(err, fs.ErrExist) {
return fmt.Errorf("%s %w", p, errConflict)
}
return err
}
// The file is published; the caller's discardTemp removes the
// temporary name, and a failure there leaves only a dotfile.
}
if err := syncDir(filepath.Dir(p)); err != nil {
return fmt.Errorf("%w: %w", errDurability, err)
}
return nil
}
// deleteFile removes CSM's own file p, if present, durably.
func deleteFile(p string) error {
if err := os.Remove(p); err != nil && !errors.Is(err, fs.ErrNotExist) {
return err
}
// A retry after a sync failure sees absence, but still has to make the
// earlier removal durable before rollback can be called verified.
if err := syncDir(filepath.Dir(p)); err != nil {
return fmt.Errorf("%w: %w", errDurability, err)
}
return nil
}
func syncDir(dir string) error {
d, err := os.Open(dir) // #nosec G304 -- directory of a fixed target path
if err != nil {
return err
}
defer func() { _ = d.Close() }()
return d.Sync()
}
// privateDir requires worker ownership; the worker runs as root. Ancestors
// must be real directories, and chmod uses the opened directory's inode.
func privateDir(dir string) error {
return privateDirOwned(dir, os.Geteuid())
}
func privateDirOwned(dir string, uid int) error {
parent, err := safepath.OpenDirNoFollow(filepath.Dir(dir))
if err != nil {
return err
}
defer func() { _ = parent.Close() }()
if mkdirErr := os.Mkdir(dir, 0o700); mkdirErr != nil && !errors.Is(mkdirErr, fs.ErrExist) {
return mkdirErr
}
d, err := parent.OpenFile(filepath.Base(dir), os.O_RDONLY|syscall.O_DIRECTORY, 0)
if err != nil {
return err
}
defer func() { _ = d.Close() }()
info, err := d.Stat()
if err != nil {
return err
}
owner, ok := info.Sys().(*syscall.Stat_t)
if !ok || int(owner.Uid) != uid {
return fmt.Errorf("%s is not owned by the worker", dir)
}
if fileMode(info.Mode()) != 0o700 {
// #nosec G302 -- a directory needs its search bit; 0700 is root-only.
if err := d.Chmod(0o700); err != nil {
return err
}
}
if err := d.Sync(); err != nil {
return err
}
// Sync the entry as well as the directory, including a concurrent
// creator's entry that might not yet be durable.
return parent.Sync()
}
package txn
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"strings"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/safepath"
)
func replacementFor(stack antispam.MailStack) func(string, fileState, fileState) error {
if stack == antispam.StackPostfix {
return replacePostfixFile
}
return replaceFile
}
// Postfix restores operator content as root. Pin every parent so a rename
// or symlink swap cannot redirect the restore or its rollback.
func replacePostfixFile(p string, want, next fileState) error {
target, err := safepath.OpenTarget("/", strings.TrimPrefix(p, "/"), false)
if err != nil {
return err
}
defer target.Close()
if err := target.Check(); err != nil {
return err
}
if err := replacePinned(target, want, next); err != nil {
return err
}
if err := target.Check(); err != nil {
return fmt.Errorf("%w: %w", errDurability, err)
}
return nil
}
func readPinned(dir *safepath.Dir, name string) (fileState, error) {
f, err := dir.OpenFile(name, os.O_RDONLY, 0)
if errors.Is(err, fs.ErrNotExist) {
return fileState{}, nil
}
if err != nil {
return fileState{}, err
}
defer func() { _ = f.Close() }()
return readOpenFile(f)
}
func replacePinned(target *safepath.Target, want, next fileState) error {
dir, name := target.Parent, target.Name
cur, err := readPinned(dir, name)
if err != nil {
return err
}
if !sameContent(cur, want) {
return errConflict
}
if !next.exists {
if err = dir.Remove(name); err != nil && !errors.Is(err, fs.ErrNotExist) {
return err
}
return syncPinned(dir)
}
if len(next.content) > maxFileBytes {
return errors.New("replacement is too large")
}
staging, stageName, err := dir.CreatePrivateTemp()
if err != nil {
return err
}
defer func() {
_ = staging.Close()
_ = dir.RemoveDir(stageName)
}()
tmp, err := staging.CreateTemp()
if err != nil {
return err
}
defer func() {
_ = tmp.Close()
_ = staging.Remove(tmp.Name())
}()
if _, err = io.WriteString(tmp, next.content); err != nil {
return err
}
cur, err = readPinned(dir, name)
if err != nil {
return err
}
if !sameContent(cur, want) {
return errConflict
}
meta := fileState{uid: -1, gid: -1, perm: 0o644}
if cur.exists {
meta = cur
}
if next.metadata {
meta = next
}
if err = tmp.Chown(meta.uid, meta.gid); err != nil {
return err
}
if err = tmp.Chmod(meta.perm); err != nil {
return err
}
if err = tmp.Sync(); err != nil {
return err
}
if cur.exists {
err = staging.ReplaceTo(tmp.Name(), dir, name)
} else {
err = staging.RenameTo(tmp.Name(), dir, name)
}
if err != nil {
return err
}
return syncPinned(dir)
}
func syncPinned(dir *safepath.Dir) error {
if err := dir.Sync(); err != nil {
return fmt.Errorf("%w: %w", errDurability, err)
}
return nil
}
package txn
import (
"bufio"
"context"
"errors"
"io"
"net"
"net/netip"
"slices"
"strconv"
"strings"
"time"
)
// listeners returns the local addresses where Exim's daemon greets in
// plain text, from daemon_smtp_ports, local_interfaces and
// tls_on_connect_ports. Wildcard interfaces map to loopback. Service
// names resolve through the host's TCP service database, as in Exim.
func listeners(settings map[string]string) []string {
tls := map[int]bool{}
for _, item := range eximList(settings["tls_on_connect_ports"]) {
if port, ok := eximPort(item, 10); ok {
tls[port] = true
}
}
var ports []int
for _, item := range eximList(settings["daemon_smtp_ports"]) {
if port, ok := eximPort(item, 0); ok && !tls[port] {
ports = append(ports, port)
}
}
interfaces := eximList(settings["local_interfaces"])
var out []string
add := func(addr netip.Addr, port int) {
switch {
case addr.IsUnspecified() && addr.Is4():
addr = netip.AddrFrom4([4]byte{127, 0, 0, 1})
case addr.IsUnspecified():
addr = netip.IPv6Loopback()
}
hostport := net.JoinHostPort(addr.String(), strconv.Itoa(port))
if !tls[port] && !slices.Contains(out, hostport) {
out = append(out, hostport)
}
}
for _, item := range interfaces {
addr, port, ok := splitInterface(item)
switch {
case !ok:
case port != 0:
add(addr, port)
default:
for _, p := range ports {
add(addr, p)
}
}
}
return out
}
// Exim uses base zero for numeric daemon ports and decimal for TLS ports.
// Non-numeric items are TCP service names. eximList omits empty items.
func eximPort(item string, base int) (int, bool) {
if item[0] >= '0' && item[0] <= '9' {
port, err := strconv.ParseInt(item, base, 32)
return int(port), err == nil && port > 0 && port < 65536
}
port, err := net.LookupPort("tcp", item)
return port, err == nil && port > 0
}
// eximList splits an Exim list: ':' separates items unless the list starts
// with '<' and a punctuation or control separator. Doubled punctuation
// separators are literal; doubled control separators separate empty items.
func eximList(raw string) []string {
raw = strings.TrimSpace(raw)
sep := byte(':')
if len(raw) > 1 && raw[0] == '<' && (raw[1] < ' ' || raw[1] == 127 ||
strings.ContainsRune("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~", rune(raw[1]))) {
sep, raw = raw[1], raw[2:]
}
var out []string
var item strings.Builder
flush := func() {
if s := strings.TrimSpace(item.String()); s != "" {
out = append(out, s)
}
item.Reset()
}
for i := 0; i < len(raw); i++ {
switch {
case raw[i] != sep:
item.WriteByte(raw[i])
case sep >= ' ' && sep != 127 && i+1 < len(raw) && raw[i+1] == sep:
item.WriteByte(sep)
i++
default:
flush()
}
}
flush()
return out
}
// splitInterface reads one local_interfaces item: an address, optionally
// followed by a dot and a port, or [address] with an optional :port.
func splitInterface(item string) (netip.Addr, int, bool) {
if strings.HasPrefix(item, "[") {
if strings.HasSuffix(item, "]") {
addr, err := netip.ParseAddr(item[1 : len(item)-1])
return addr, 0, err == nil
}
host, port, err := net.SplitHostPort(item)
addr, errA := netip.ParseAddr(host)
n, errP := strconv.Atoi(port)
return addr, n, err == nil && errA == nil && errP == nil && n >= 0 && n < 65536
}
// Exim's first-dot rule also applies to IPv6 scope identifiers, so try
// the port before ParseAddr accepts it as part of a zone.
i := strings.LastIndexByte(item, '.')
if strings.Contains(item, ":") {
i = strings.IndexByte(item, '.')
}
if i >= 0 {
addr, errA := netip.ParseAddr(item[:i])
n, errP := strconv.Atoi(item[i+1:])
if errA == nil && errP == nil && n >= 0 && n < 65536 {
return addr, n, true
}
}
if addr, err := netip.ParseAddr(item); err == nil {
return addr, 0, true
}
return netip.Addr{}, 0, false
}
// maxGreeting bounds what the probe reads from one SMTP session.
const maxGreeting = 4 << 10
// smtpGreeting reads Exim's complete 220 greeting from addr and ends the
// session with QUIT, so the probe never counts as a dropped connection.
// Errors never repeat the server's text.
func smtpGreeting(ctx context.Context, addr string) error {
var d net.Dialer
conn, err := d.DialContext(ctx, "tcp", addr)
if err != nil {
return err
}
defer func() { _ = conn.Close() }()
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
defer stop()
if deadline, ok := ctx.Deadline(); ok {
if err := conn.SetDeadline(deadline); err != nil {
return err
}
}
r := bufio.NewReader(io.LimitReader(conn, maxGreeting))
if err := readReply(r, "220"); err != nil {
return err
}
if _, err := io.WriteString(conn, "QUIT\r\n"); err != nil {
return err
}
return readReply(r, "221")
}
func readReply(r *bufio.Reader, code string) error {
for {
line, err := r.ReadString('\n')
if err != nil {
return errors.New("incomplete SMTP reply")
}
if !strings.HasSuffix(line, "\r\n") {
return errors.New("malformed SMTP reply")
}
line = strings.TrimSuffix(line, "\r\n")
if len(line) < 3 || line[:3] != code {
return errors.New("SMTP reply is not " + code)
}
if len(line) == 3 || line[3] == ' ' {
return nil
}
if line[3] != '-' {
return errors.New("malformed SMTP reply")
}
}
}
// connectMilter opens one connection to a postfix milter address,
// inet:host:port or unix:path, as postfix does before each message, and
// closes it.
func connectMilter(ctx context.Context, milter string) error {
network, addr := "tcp", strings.TrimPrefix(milter, "inet:")
if sock, ok := strings.CutPrefix(milter, "unix:"); ok {
network, addr = "unix", sock
}
var d net.Dialer
conn, err := d.DialContext(ctx, network, addr)
if err != nil {
return err
}
return conn.Close()
}
// retry calls probe until it succeeds or window ends, pausing between
// attempts. Each attempt gets the time left.
func retry(ctx context.Context, window time.Duration, probe func(context.Context) error) error {
ctx, cancel := context.WithTimeout(ctx, window)
defer cancel()
for {
if err := ctx.Err(); err != nil {
return err
}
err := probe(ctx)
if err == nil {
return ctx.Err()
}
select {
case <-ctx.Done():
return err
case <-time.After(200 * time.Millisecond):
}
}
}
package txn
import (
"context"
"errors"
"strings"
"github.com/pidginhost/csm/internal/antispam"
)
// Remove takes out CSM's integration: Exim first, then
// CSM's rspamd files, then the record. Foreign local.d files stay. A stopped
// service is not reloaded, and the scan socket is not needed. Postfix mail
// never passes through Exim, so there it removes rspamd files only, and it
// puts back the operator files CSM took over: rspamd enforces there, and
// its built-in actions and greylisting would change the host's mail flow.
func (h *Host) Remove() antispam.Result {
t := h.begin()
defer t.cancel()
t.rspamdProbe = func(ctx context.Context) error {
if r := t.cmd(ctx, commandLimit, "systemctl", "is-active", antispam.RspamdUnit); r.Err != nil {
return errors.New("rspamd is not active after reload")
}
return nil
}
// A remove request carries no socket, so rollback checks rspamd the same way.
t.rspamdRollbackProbe = t.rspamdProbe
stack := h.stack()
exim := stack != antispam.StackPostfix
targets := h.targets()
if err := blocking(antispam.CheckRemoval(stack, targets)); err != nil {
return failed(StepPreconditions, err)
}
changes, err := plan(targets, nil)
if err != nil {
return failed(StepPreconditions, err)
}
if !exim {
if err = t.restoreCopies(changes, func(string) bool { return true }); err != nil {
return failed(StepPreconditions, err)
}
}
rspamdChanges, eximChanges, roundcubeChanges := split(changes)
if _, _, err = t.readRecord(); err != nil {
return failed(StepRecord, err)
}
// A record, complete or not, means an earlier transaction may have
// changed files without a confirmed reload.
rebuildExim := false
if exim {
conf, readErr := readFile(h.path(antispam.GeneratedEximConfPath))
generated := readErr == nil && conf.exists && !strings.Contains(conf.content, antispam.MarkerBegin) && !strings.Contains(conf.content, antispam.MarkerEnd)
rebuildExim = len(eximChanges) > 0 || t.record.exists || !generated
}
reloadRspamd := len(rspamdChanges) > 0 || t.record.exists
if !rebuildExim && !reloadRspamd && len(roundcubeChanges) == 0 {
return antispam.Result{OK: true}
}
if exim {
// A broken generated configuration must not prevent removing its hooks:
// without listener settings the probe after reload checks activity only.
opts, _ := t.eximSettings(t.fwd)
t.rememberEximBlocks(targets, opts)
if t.eximActive, err = t.serviceActive(t.fwd, antispam.EximUnit); err != nil {
return failed(StepPreconditions, err)
}
if t.eximActive {
_ = t.listenerGreeting(opts)
}
}
if t.rspamdActive, err = t.serviceActive(t.fwd, antispam.RspamdUnit); err != nil {
return failed(StepPreconditions, err)
}
if err := t.writeRecord(""); err != nil {
return t.fail(StepRecord, err)
}
if rebuildExim {
verify := func(conf, _, _ string) error {
if strings.Contains(conf, antispam.MarkerBegin) || strings.Contains(conf, antispam.MarkerEnd) {
return errors.New("the generated configuration still holds CSM blocks")
}
return nil
}
if step, err := t.exim(eximChanges, false, verify); err != nil {
return t.fail(step, err)
}
}
if reloadRspamd {
for _, c := range rspamdChanges {
if err := t.write(c); err != nil {
return t.fail(StepRspamdFiles, err)
}
}
if t.rspamdActive {
if err := t.checkRspamd(t.fwd, nil); err != nil {
return t.fail(StepRspamdCheck, err)
}
if err := t.reloadRspamd(t.fwd, t.rspamdProbe); err != nil {
return t.fail(StepRspamdReload, err)
}
}
}
roundcube, rcErr := t.roundcube(roundcubeChanges)
if rcErr != nil {
return t.fail(StepRoundcube, rcErr)
}
if err := t.consumeRestored(); err != nil {
return t.fail(StepComplete, err)
}
if err := t.removeRecord(); err != nil {
return t.fail(StepComplete, err)
}
return antispam.Result{OK: true, Roundcube: roundcube}
}
package txn
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
"syscall"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/safepath"
)
// restoredCopy retains the source directory and snapshot through completion
// and rollback. A pathname alone cannot identify the copy after a rename.
type restoredCopy struct {
path string
target *safepath.Target
saved fileState
consumed bool
}
// restoreCopies substitutes the operator's newest backup for each selected
// deletion. A matching unsafe backup must fail the transaction, not silently
// turn a restore into a deletion or select an older backup.
func (t *tx) restoreCopies(changes []change, want func(path string) bool) error {
selected := func(c change) bool { return !c.next.exists && c.target.Kind.WholeFile() && want(c.target.Path) }
if !slices.ContainsFunc(changes, selected) {
return nil
}
path := t.h.path(antispam.ReplacedDir)
dir, err := safepath.OpenDirNoFollow(path)
if errors.Is(err, fs.ErrNotExist) {
return nil
}
if err != nil {
return err
}
defer func() { _ = dir.Close() }()
info, err := dir.Info()
if err != nil {
return err
}
owner, ok := info.Sys().(*syscall.Stat_t)
if !ok || int(owner.Uid) != os.Geteuid() || info.Mode().Perm()&0o077 != 0 {
return errors.New("operator backup directory is not private and worker-owned")
}
entries, err := dir.ReadDir()
if err != nil {
return err
}
newest := map[string]string{}
for _, e := range entries {
if p, ok := antispam.ParseBackupName(e.Name()); ok {
newest[p] = max(newest[p], e.Name())
}
}
for i, c := range changes {
name, ok := newest[c.target.Path]
if !ok || !selected(c) {
continue
}
// Deleting a loosened CSM file is safe, but restoring private
// operator content needs metadata we can trust. Its original
// permissions cannot be recovered from the root-only backup.
if usableErr := antispam.OptionalUsable(c.target); usableErr != nil {
return usableErr
}
p := filepath.Join(path, name)
target, err := safepath.OpenTarget("/", strings.TrimPrefix(p, "/"), false)
if err != nil {
return err
}
r := &restoredCopy{path: p, target: target}
t.restored = append(t.restored, r)
if same, compareErr := dir.SameFile(target.Parent); compareErr != nil || !same {
return fmt.Errorf("operator backup directory %w", errConflict)
}
saved, err := readPinned(target.Parent, name)
if err != nil {
return err
}
if !saved.exists || saved.uid != os.Geteuid() || saved.perm != 0o600 {
return errors.New("operator backup is not a private worker-owned regular file")
}
r.saved = saved
// Keep only metadata checked before the restore. Refreshing it at
// rename time could publish private operator content with a mode
// another writer just loosened on CSM's file.
next := fileState{exists: true, content: saved.content, metadata: true,
uid: os.Geteuid(), gid: c.target.File.GID, perm: fileMode(c.target.File.Mode)}
changes[i].next = next
}
return nil
}
// consume first moves the source to a private name, then checks the exact
// file it will delete. A changed copy goes back without replacing a new
// operator entry. All operations stay in the pinned backup directory.
func (r *restoredCopy) consume() error {
if err := r.target.Check(); err != nil {
return err
}
dir, name := r.target.Parent, r.target.Name
held, heldName, err := dir.CreatePrivateTemp()
if err != nil {
return err
}
defer func() {
_ = held.Close()
_ = dir.RemoveDir(heldName)
}()
if err = dir.RenameTo(name, held, name); err != nil {
return err
}
cur, err := readPinned(held, name)
if err != nil || cur != r.saved {
return errors.Join(errConflict, err, held.RenameTo(name, dir, name), syncPinned(dir))
}
if err := held.Remove(name); err != nil {
return errors.Join(err, held.RenameTo(name, dir, name), syncPinned(dir))
}
r.consumed = true
return errors.Join(syncPinned(held), syncPinned(dir))
}
func (t *tx) consumeRestored() error {
for _, r := range t.restored {
if err := r.consume(); err != nil {
return err
}
}
return t.fwd.Err()
}
func (t *tx) restoreConsumedCopies() error {
var errs []error
for _, r := range t.restored {
if !r.consumed {
continue
}
if err := replacePinned(r.target, fileState{}, r.saved); err != nil {
errs = append(errs, fmt.Errorf("restoring backup %s: %w", strings.TrimPrefix(r.path, t.h.root), err))
continue
}
r.consumed = false
if err := r.target.Check(); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
package txn
import (
"errors"
"slices"
"github.com/pidginhost/csm/internal/antispam"
)
// roundcube applies the webmail include change after rspamd and Exim are
// verified, then checks what a fresh Roundcube request loads through
// cPanel's PHP. An include that does not take effect, or whose
// check fails, is put back to its prior state alone and reported; learning
// and the rest of the transaction stand. A removal is reported, never
// undone. The include still rolls back with an enclosing failure.
func (t *tx) roundcube(changes []change) (string, error) {
for _, c := range changes {
if err := t.write(c); err != nil {
return "", err
}
// A failed check is reported, not returned: it never fails the
// transaction.
uid, gid, idErr := t.h.unprivileged()
if idErr != nil {
uid, gid = -1, -1
}
state, checkErr := antispam.RoundcubeEffective(t.fwd, t.cmd, uid, gid)
checked, install := checkErr == nil, c.next.exists
switch {
case !install && !checked:
return antispam.RoundcubeUnavailable + "CSM could not confirm that Roundcube stopped loading the include", nil
case !install && (state.JunkMbox == antispam.RoundcubeJunkFolder || state.SpamMbox == antispam.RoundcubeJunkFolder):
return antispam.RoundcubeUnavailable + "Roundcube still sends Junk to INBOX.spam through another include", nil
case !install:
continue
case checked && state.Applied():
continue
}
reason := "Roundcube does not load the include's Junk folder, another include may override it; " + state.Describe()
if !checked {
reason = "cPanel's PHP could not verify the include"
}
if restoreErr := t.restoreLast(); restoreErr != nil {
return "", restoreErr
}
return antispam.RoundcubeUnavailable + reason, nil
}
return "", nil
}
// restoreLast puts the last written file back to its snapshot and drops it
// from the rollback list, so an enclosing rollback does not find it changed.
// Once an operator file is back, its backup goes too.
func (t *tx) restoreLast() error {
last := len(t.writes) - 1
w := t.writes[last]
if err := t.h.replace(w.path, w.next, w.before); err != nil {
if errors.Is(err, errDurability) {
// The snapshot reached disk. An enclosing rollback must retry
// against it, not mistake our own restore for a foreign edit.
t.writes[last].next = w.before
}
return err
}
t.writes = slices.Delete(t.writes, last, len(t.writes))
if w.backup == "" {
return nil
}
// On failure the backup stays listed, so an enclosing rollback retries.
t.backups[w.backup] = false
if err := deleteFile(w.backup); err != nil {
return err
}
delete(t.backups, w.backup)
return nil
}
package txn
import (
"context"
"errors"
"fmt"
"os/user"
"path/filepath"
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/antispam/hostfacts"
"github.com/pidginhost/csm/internal/platform"
)
// Transaction time limits.
const (
// forwardBudget is when rollback must start at the latest.
forwardBudget = 150 * time.Second
// totalBudget ends rollback before the worker's 5-minute unit limit,
// leaving time to report.
totalBudget = 290 * time.Second
buildLimit = 120 * time.Second
commandLimit = 10 * time.Second
rspamdWait = 20 * time.Second
eximWait = 3 * time.Second
)
// BuildEximConf is cPanel's Exim configuration builder.
const BuildEximConf = "/scripts/buildeximconf"
// Steps a failed result names.
const (
StepPreconditions = "preconditions"
StepRecord = "record"
StepRspamdFiles = "rspamd_files"
StepRspamdCheck = "rspamd_check"
StepRspamdReload = "rspamd_reload"
StepEximFiles = "exim_files"
StepEximBuild = "exim_build"
StepEximReload = "exim_reload"
StepEximProbe = "exim_probe"
StepRoundcube = "roundcube_files"
StepComplete = "complete"
StepFeedsCheck = antispam.FeedsCheckStep
StepLearningCheck = antispam.LearningCheckStep
StepLearningProbe = antispam.LearningProbeStep
)
// errLearningLoaded distinguishes learning verification failures.
var errLearningLoaded = errors.New("rspamd's loaded learning configuration could not be verified against CSM's files")
// errLearningProbe marks a learning check that could not read rspamd's
// loaded configuration at all.
var errLearningProbe = errors.New("rspamd's loaded learning configuration could not be read")
// maxSymbolsReply bounds the controller's /symbols reply.
const maxSymbolsReply = 4 << 20
// errFeedsLoaded distinguishes feed verification failures from core failures.
var errFeedsLoaded = errors.New("rspamd feed rules could not be verified against CSM's settings")
// Host runs transactions against the host. Every function field is a side
// effect; tests replace them and set root to a temporary directory.
type Host struct {
// root prefixes every host path; it is "" on a live host.
root string
run func(ctx context.Context, limit time.Duration, name string, args ...string) antispam.CommandResult
replace func(p string, want, next fileState) error
preflight func(ctx context.Context, s antispam.Settings) (antispam.HostFacts, []antispam.Problem)
targets func() []antispam.TargetFacts
// stack is the host's mail stack, which picks targets' files.
stack func() antispam.MailStack
ping func(ctx context.Context, socket string) error
// milter connects to a postfix milter address, inet:host:port or
// unix:path, as postfix would.
milter func(ctx context.Context, addr string) error
// unprivileged returns the uid and gid Roundcube's PHP runs as.
unprivileged func() (uid, gid int, err error)
// controllerGet reads one controller path over a private Unix socket.
controllerGet func(ctx context.Context, socket, path string, limit int64, peer int) ([]byte, error)
greet func(ctx context.Context, addr string) error
now func() time.Time
// rspamdWait and eximWait bound the service probes after a reload.
rspamdWait, eximWait time.Duration
}
// New returns a Host for the live paths of the host's mail stack, which it
// detects once.
func New() *Host {
stack := sync.OnceValue(hostfacts.DetectStack)
return &Host{
run: hostfacts.RunCommand,
replace: func(p string, want, next fileState) error {
return replacementFor(stack())(p, want, next)
},
stack: stack,
preflight: func(ctx context.Context, s antispam.Settings) (antispam.HostFacts, []antispam.Problem) {
f := hostfacts.Collect(ctx, s.Socket)
if s.HasFeed(antispam.FeedSpamhausDQS) {
f.DQSKey = hostfacts.ReadDQSKey(s.DQSKeyFile)
}
if s.Learning {
f.LearningKeys, f.LearningKeysErr = hostfacts.LoadLearningKeys(s.LearningDir)
if s.LearningRedisSocket != "" {
hostfacts.CollectLearningRedis(ctx, s.LearningRedisSocket, &f)
}
}
return f, antispam.CheckHost(f)
},
targets: func() []antispam.TargetFacts { return hostfacts.Targets(antispam.MutationTargets(stack())) },
ping: func(ctx context.Context, socket string) error {
_, err := hostfacts.Ping(ctx, socket)
return err
},
milter: connectMilter,
greet: smtpGreeting,
controllerGet: hostfacts.ControllerGet,
unprivileged: nobody,
now: time.Now,
rspamdWait: rspamdWait,
eximWait: eximWait,
}
}
func (h *Host) path(p string) string { return h.root + p }
// change is one planned file edit.
type change struct {
target antispam.TargetFacts
next fileState
}
// written is one file edit made, for rollback. backup is the copy of the
// operator file the edit replaced, if any.
type written struct {
path string
before, next fileState
backup string
}
// tx is one transaction's progress, which rollback reads.
type tx struct {
h *Host
start time.Time
// fwd ends at forwardBudget; every forward step runs under it.
fwd context.Context
cancel context.CancelFunc
record fileState
recordCurrent fileState
recordWritten bool
writes []written
eximTouched bool
eximReloaded bool
eximActive bool
// eximBlocks are the prior CSM blocks per ACL, for rollback.
eximBlocks map[string][]string
eximRCPTACL, eximDataACL string
// The scanner address alone uses no named lists.
eximHadHooks bool
// greeting is the listener that answered before any change; "" when
// Exim was not running.
greeting string
rspamdReloaded bool
rspamdActive bool
// rspamdProbe confirms rspamd after a reload.
rspamdProbe func(context.Context) error
rspamdRollbackProbe func(context.Context) error
// backups maps each copy to whether its original still needs a durable
// restore. A failed replacement leaves its copy eligible for cleanup.
backups map[string]bool
// restored are the copies a Postfix transaction puts back.
restored []*restoredCopy
}
func (h *Host) begin() *tx {
t := &tx{h: h, start: h.now()}
t.fwd, t.cancel = context.WithDeadline(context.Background(), t.start.Add(forwardBudget))
cancel := t.cancel
t.cancel = func() {
cancel()
for _, r := range t.restored {
r.target.Close()
}
}
return t
}
func (t *tx) cmd(ctx context.Context, limit time.Duration, name string, args ...string) antispam.CommandResult {
if err := ctx.Err(); err != nil {
return antispam.CommandResult{Err: err}
}
r := t.h.run(ctx, limit, name, args...)
if err := ctx.Err(); err != nil {
r.Err = err
}
return r
}
// cmdErr describes a failed command by name and timeout only: command
// output can carry configuration text.
func cmdErr(what string, r antispam.CommandResult) error {
switch {
case errors.Is(r.Err, context.DeadlineExceeded):
return fmt.Errorf("%s failed: %w", what, context.DeadlineExceeded)
case errors.Is(r.Err, context.Canceled):
return fmt.Errorf("%s failed: %w", what, context.Canceled)
default:
return fmt.Errorf("%s failed", what)
}
}
func failed(step string, err error) antispam.Result {
return antispam.Result{FailedStep: step, Rollback: antispam.RollbackNone, Error: antispam.ErrorSummary(err.Error())}
}
// blocking returns the Definite and Temporary problem codes as one error.
// Warnings never block a transaction.
func blocking(problems []antispam.Problem) error {
var codes []string
for _, p := range problems {
if p.Severity != antispam.Warning && !slices.Contains(codes, p.Code) {
codes = append(codes, p.Code)
}
}
if len(codes) == 0 {
return nil
}
return fmt.Errorf("preconditions not met: %s", strings.Join(codes, ", "))
}
// plan lists the edits that bring targets to managed, keyed by target
// path. A nil managed map plans removal: CSM blocks are stripped, managed
// rspamd files deleted and foreign ones left in place. Optional files CSM
// wrote but no longer needs are deleted on apply too.
func plan(targets []antispam.TargetFacts, managed map[string]string) ([]change, error) {
var out []change
for _, t := range targets {
before := fileState{exists: !t.Missing, content: t.Content}
switch {
case t.Kind.Optional() && managed != nil:
want, wanted := managed[t.Path]
switch {
case wanted:
// The daemon left out what CSM cannot own; a file that
// changed since then is refused, never overwritten.
if err := antispam.OptionalUsable(t); err != nil {
return nil, err
}
if !sameContent(before, fileState{exists: true, content: want}) {
out = append(out, change{t, fileState{exists: true, content: want}})
}
case !t.Missing && antispam.ManagedFor(t.Kind, t.Content):
out = append(out, change{t, fileState{}})
}
case t.Kind.WholeFile() && managed != nil:
if want := managed[t.Path]; !sameContent(before, fileState{exists: true, content: want}) {
out = append(out, change{t, fileState{exists: true, content: want}})
}
case t.Kind.WholeFile():
if !t.Missing && antispam.ManagedFor(t.Kind, t.Content) {
out = append(out, change{t, fileState{}})
}
case t.Missing && managed == nil:
default:
base := t.Content
if t.Kind == antispam.TargetEximLocal && managed != nil && strings.TrimSpace(base) == "" {
base = platform.EximLocalSkeleton
}
next, err := antispam.SetBlock(t.Kind, base, managed[t.Path])
if err != nil {
return nil, fmt.Errorf("%s: %w", t.Path, err)
}
if !sameContent(before, fileState{exists: true, content: next}) {
out = append(out, change{t, fileState{exists: true, content: next}})
}
}
}
return out, nil
}
// secureSecret gives each secret file in paths the group and mode that keep
// its secret from other users, also when only its mode or group is wrong.
// The worker runs as root, so uid -1 keeps root as owner.
func secureSecret(changes []change, targets []antispam.TargetFacts, gid int, paths []string) []change {
meta := func(content string) fileState {
return fileState{exists: true, content: content, uid: -1, gid: gid, perm: antispam.SecretFileMode, metadata: true}
}
for _, p := range paths {
changes = secureOne(changes, targets, p, meta)
}
return changes
}
func secureOne(changes []change, targets []antispam.TargetFacts, p string, meta func(string) fileState) []change {
for i, c := range changes {
if c.target.Path == p && c.next.exists {
changes[i].next = meta(c.next.content)
return changes
}
}
for _, t := range targets {
if t.Path == p && !t.Missing {
if want := meta(t.Content); t.File.GID != want.gid || t.File.Mode != antispam.SecretFileMode {
return append(changes, change{t, want})
}
}
}
return changes
}
// split separates edits that need an rspamd reload, edits that need an Exim
// rebuild, and Roundcube edits that need neither.
func split(changes []change) (rspamd, exim, roundcube []change) {
for _, c := range changes {
switch {
case c.target.Kind.ReloadsRspamd():
rspamd = append(rspamd, c)
case c.target.Kind == antispam.TargetRoundcube:
roundcube = append(roundcube, c)
default:
exim = append(exim, c)
}
}
return rspamd, exim, roundcube
}
// write applies one planned edit unless another writer changed the file
// since the snapshot.
func (t *tx) write(c change) error {
if err := t.fwd.Err(); err != nil {
return err
}
p := t.h.path(c.target.Path)
takeover := c.target.Kind.WholeFile() && !c.target.Missing && !antispam.ManagedFor(c.target.Kind, c.target.Content)
if takeover {
if err := t.checkTakeover(c.target.Path); err != nil {
return err
}
}
// Read the file again for the owner and mode a rollback must restore,
// including those of a file the transaction deletes.
before, err := readFile(p)
if err != nil {
return err
}
if !sameContent(before, fileState{exists: !c.target.Missing, content: c.target.Content}) {
return fmt.Errorf("%s %w", p, errConflict)
}
// An operator file at a path CSM owns outright is kept before it is
// replaced.
backup := ""
if takeover {
if backup, err = t.backup(c.target.Path, before.content); err != nil {
return err
}
// Copying and syncing the backup can take time. Content checks alone
// do not catch an operator loosening the file or its parents meanwhile.
if err = t.checkTakeover(c.target.Path); err != nil {
return err
}
}
err = t.h.replace(p, before, c.next)
if err == nil || errors.Is(err, errDurability) {
t.writes = append(t.writes, written{path: p, before: before, next: c.next, backup: backup})
if backup != "" {
t.backups[backup] = true
}
if !c.target.Kind.WholeFile() {
t.eximTouched = true
}
}
if err == nil {
return t.fwd.Err()
}
return err
}
// checkTakeover reuses the same protection requirements as preflight before
// reading an operator file and again before publishing its replacement.
func (t *tx) checkTakeover(p string) error {
for _, target := range t.h.targets() {
if target.Path == p {
return antispam.OptionalUsable(target)
}
}
return fmt.Errorf("%s was not checked", p)
}
// backup copies the operator file at target into ReplacedDir, root-only
// and durable. It never overwrites an earlier copy: a name already taken
// fails the write, and the next pass tries again with a new time.
func (t *tx) backup(target, content string) (string, error) {
dir := t.h.path(antispam.ReplacedDir)
if err := privateDir(dir); err != nil {
return "", err
}
p := filepath.Join(dir, antispam.BackupName(target, t.h.now()))
err := publishCopy(p, content)
if err == nil || errors.Is(err, errDurability) {
// The target is still unchanged, so a rollback may delete this copy.
if t.backups == nil {
t.backups = make(map[string]bool)
}
t.backups[p] = false
}
return p, err
}
// publishCopy writes a root-only copy at p, never over an existing file.
func publishCopy(p, content string) error {
tmp, err := stageFile(p, content)
if err != nil {
return err
}
defer discardTemp(tmp)
return publishFile(p, tmp, 0o600, -1, -1, false)
}
// dropBackups deletes only copies whose originals are durably back in
// place or were never replaced. Other failed restores must not delay them.
func (t *tx) dropBackups() error {
var errs []error
for p, pending := range t.backups {
if pending {
continue
}
if err := deleteFile(p); err != nil {
errs = append(errs, fmt.Errorf("removing backup %s: %w", strings.TrimPrefix(p, t.h.root), err))
} else {
delete(t.backups, p)
}
}
return errors.Join(errs...)
}
// readRecord reads the completion record into t.record.
func (t *tx) readRecord() (antispam.Record, bool, error) {
st, err := readFile(t.h.path(antispam.RecordPath))
if err != nil {
return antispam.Record{}, false, fmt.Errorf("cannot read %s: %w", antispam.RecordPath, err)
}
t.record = st
t.recordCurrent = st
if !st.exists {
return antispam.Record{}, false, nil
}
r, err := antispam.DecodeRecord([]byte(st.content))
return r, err == nil, nil
}
// writeRecord persists a record; an empty generation marks work in progress.
func (t *tx) writeRecord(generation string) error {
return t.replaceRecord(t.fwd, generation)
}
func (t *tx) replaceRecord(ctx context.Context, generation string) error {
if err := ctx.Err(); err != nil {
return err
}
p := t.h.path(antispam.RecordPath)
if err := privateDir(filepath.Dir(p)); err != nil {
return err
}
next := fileState{exists: true, content: string(antispam.EncodeRecord(generation)), perm: 0o600, uid: -1, gid: -1, metadata: true}
err := t.h.replace(p, t.recordCurrent, next)
if err == nil || errors.Is(err, errDurability) {
t.recordWritten = true
t.recordCurrent = next
}
if err == nil {
return ctx.Err()
}
return err
}
func (t *tx) removeRecord() error {
if err := t.fwd.Err(); err != nil {
return err
}
err := t.h.replace(t.h.path(antispam.RecordPath), t.recordCurrent, fileState{})
if err == nil || errors.Is(err, errDurability) {
t.recordWritten = true
t.recordCurrent = fileState{}
}
if err == nil {
return t.fwd.Err()
}
return err
}
// listenerProbe reads where Exim listens and checks that one listener greets
// before anything changes, so the probe after reload compares like with like.
func (t *tx) listenerProbe(opts map[string]string) error {
var err error
if t.eximActive, err = t.serviceActive(t.fwd, antispam.EximUnit); err != nil || !t.eximActive {
return err
}
return t.listenerGreeting(opts)
}
// listenerGreeting records the first configured listener that greets.
func (t *tx) listenerGreeting(opts map[string]string) error {
for _, addr := range listeners(opts) {
ctx, cancel := context.WithTimeout(t.fwd, t.h.eximWait)
err := t.h.greet(ctx, addr)
cancel()
if err == nil {
t.greeting = addr
return nil
}
}
return errors.New("no configured Exim listener answered with an SMTP greeting")
}
// serviceActive reads `systemctl is-active`. Exit 3 with "inactive" or
// "failed", and exit 4 for an unknown unit, mean stopped. A unit that is
// activating or deactivating, or a failed query, leaves its loaded state
// unknown, so no transaction may build on it.
func (t *tx) serviceActive(ctx context.Context, unit string) (bool, error) {
r := t.cmd(ctx, commandLimit, "systemctl", "is-active", unit)
if r.Err == nil {
return true, nil
}
var exit interface{ ExitCode() int }
if errors.As(r.Err, &exit) {
state := strings.TrimSpace(r.Stdout)
if exit.ExitCode() == 4 || exit.ExitCode() == 3 && (state == "inactive" || state == "failed") {
return false, nil
}
}
return false, cmdErr("systemctl is-active "+unit, r)
}
// listenerSettings are the `exim -bP` options that say where the daemon
// accepts SMTP, read with the ACL names before any change.
var listenerSettings = []string{"daemon_smtp_ports", "local_interfaces", "tls_on_connect_ports"}
// eximSettings reads the ACL names and listener options from the
// configuration Exim would load now.
func (t *tx) eximSettings(ctx context.Context) (map[string]string, error) {
r := t.cmd(ctx, commandLimit, "exim", append([]string{"-bP", "acl_smtp_rcpt", "acl_smtp_data"}, listenerSettings...)...)
if r.Err != nil {
return nil, cmdErr("exim -bP", r)
}
return antispam.ParseEximOptions(r.Stdout), nil
}
// exim writes the Exim edits, rebuilds and validates the configuration, then
// reloads Exim and probes it. hooks says
// whether the result holds CSM ACL hooks, which need the named lists.
func (t *tx) exim(changes []change, hooks bool, verify func(conf, rcptACL, dataACL string) error) (string, error) {
for _, c := range changes {
if err := t.write(c); err != nil {
return StepEximFiles, err
}
}
t.eximTouched = true
if r := t.cmd(t.fwd, buildLimit, BuildEximConf); r.Err != nil {
return StepEximBuild, cmdErr("buildeximconf", r)
}
if err := t.validateExim(t.fwd, hooks); err != nil {
return StepEximBuild, err
}
opts, err := t.eximSettings(t.fwd)
if err != nil {
return StepEximBuild, err
}
conf, err := t.generatedConf()
if err != nil {
return StepEximBuild, err
}
if err := verify(conf.content, opts["acl_smtp_rcpt"], opts["acl_smtp_data"]); err != nil {
return StepEximBuild, err
}
if !t.eximActive {
return "", nil
}
t.eximReloaded = true
if r := t.cmd(t.fwd, commandLimit, "systemctl", "reload", antispam.EximUnit); r.Err != nil {
return StepEximReload, cmdErr("systemctl reload exim", r)
}
if err := t.probeExim(t.fwd); err != nil {
return StepEximProbe, err
}
return "", nil
}
// generatedConf reads /etc/exim.conf after a build, which must have written it.
func (t *tx) generatedConf() (fileState, error) {
conf, err := readFile(t.h.path(antispam.GeneratedEximConfPath))
if err != nil {
return fileState{}, err
}
if !conf.exists {
return fileState{}, errors.New("generated Exim configuration is missing after buildeximconf")
}
return conf, nil
}
// rememberEximBlocks keeps the CSM blocks of the shared Exim files before any
// change, keyed by the ACL that inlines them ("" for the main configuration),
// so rollback can prove the rebuilt configuration holds them again.
func (t *tx) rememberEximBlocks(targets []antispam.TargetFacts, opts map[string]string) {
t.eximBlocks = make(map[string][]string)
t.eximHadHooks = false
t.eximRCPTACL, t.eximDataACL = opts["acl_smtp_rcpt"], opts["acl_smtp_data"]
for _, target := range targets {
if target.Kind.WholeFile() {
continue
}
block, ok := antispam.BlockText(target.Kind, target.Content)
if !ok {
continue
}
if target.Kind == antispam.TargetEximHook {
t.eximHadHooks = true
}
acl := ""
switch target.Path {
case antispam.HookRCPTPath:
acl = opts["acl_smtp_rcpt"]
case antispam.HookScanPath, antispam.HookPostPath:
acl = opts["acl_smtp_data"]
}
t.eximBlocks[acl] = append(t.eximBlocks[acl], block)
}
}
// checkRollbackExim compares the rebuilt configuration's CSM blocks with the
// remembered ones. A builder can exit 0 and leave stale output, and service
// probes alone cannot show which hooks the reloaded configuration holds.
func (t *tx) checkRollbackExim(ctx context.Context) error {
if len(t.eximBlocks) > 0 {
opts, err := t.eximSettings(ctx)
if err != nil {
return err
}
if opts["acl_smtp_rcpt"] != t.eximRCPTACL || opts["acl_smtp_data"] != t.eximDataACL {
return errors.New("the restored Exim configuration selects different SMTP ACLs")
}
}
conf, err := t.generatedConf()
if err != nil {
return err
}
return antispam.CheckGeneratedBlocks(conf.content, t.eximBlocks)
}
// validateExim checks the generated configuration and, when it holds CSM
// ACL hooks, the named lists they use. A missing list
// makes the hooks defer mail, so taking them out must not depend on it.
func (t *tx) validateExim(ctx context.Context, hooks bool) error {
if r := t.cmd(ctx, commandLimit, "exim", "-bV"); r.Err != nil {
return cmdErr("exim -bV", r)
}
if !hooks {
return nil
}
for _, probe := range antispam.NamedListProbes() {
r := t.cmd(ctx, commandLimit, "exim", "-C", t.h.path(antispam.GeneratedEximConfPath), "-be", probe)
if r.Err != nil || antispam.ParseBeResult(r.Stdout, r.Stderr) != nil {
return fmt.Errorf("exim -be %s did not print y or n", probe)
}
}
return nil
}
// probeExim confirms Exim is active and greets on the listener that
// answered before the change.
func (t *tx) probeExim(ctx context.Context) error {
if r := t.cmd(ctx, commandLimit, "systemctl", "is-active", antispam.EximUnit); r.Err != nil {
return errors.New("exim is not active after reload")
}
if t.greeting == "" {
return nil
}
if err := retry(ctx, t.h.eximWait, func(ctx context.Context) error { return t.h.greet(ctx, t.greeting) }); err != nil {
return errors.New("exim SMTP greeting probe failed")
}
return nil
}
// checkRspamd validates staged rspamd files and, for an apply, compares the
// loaded thresholds, modules and phishtank switch with CSM's settings.
func (t *tx) checkRspamd(ctx context.Context, s *antispam.Settings) error {
if r := t.cmd(ctx, commandLimit, "rspamadm", "configtest"); r.Err != nil {
return cmdErr("rspamadm configtest", r)
}
if s == nil {
return nil
}
mods := t.cmd(ctx, commandLimit, "rspamadm", "configdump", "-m")
if mods.Err != nil {
return cmdErr("rspamadm configdump -m", mods)
}
enabled, err := antispam.EnabledModules(mods.Stdout)
if err != nil {
return errors.New("rspamadm configdump -m: rspamd configuration conflict")
}
if s.Learning && !enabled["fuzzy_check"] {
return errors.Join(errLearningLoaded, errors.New("rspamd module fuzzy_check is disabled"))
}
var actions antispam.Actions
// The host's own actions keep enforcing in a Postfix dry run.
if !s.HostActions() {
acts := t.cmd(ctx, commandLimit, "rspamadm", "configdump", "-j", "actions")
if acts.Err != nil {
return cmdErr("rspamadm configdump -j actions", acts)
}
if actions, err = antispam.ParseActions([]byte(acts.Stdout)); err != nil {
// Object-form actions hide their thresholds: a conflict, not drift.
return errors.New("rspamadm configdump -j actions: rspamd configuration conflict")
}
}
if conflicts := antispam.CheckRspamd(enabled, actions, *s); len(conflicts) > 0 {
return errors.New("rspamadm configdump -j actions: rspamd configuration conflict")
}
phishing := t.cmd(ctx, commandLimit, "rspamadm", "configdump", "-j", "phishing")
if phishing.Err != nil {
return cmdErr("rspamadm configdump -j phishing", phishing)
}
if on, err := antispam.PhishtankEnabled([]byte(phishing.Stdout)); err != nil || on {
return errors.New("rspamadm configdump -j phishing: rspamd configuration conflict")
}
if len(s.DNSFeeds()) > 0 {
dump := t.cmd(ctx, commandLimit, "rspamadm", "configdump", "-j", "rbl")
if dump.Err != nil {
return errors.Join(errFeedsLoaded, cmdErr("rspamadm configdump -j rbl", dump))
}
rules, err := antispam.LoadedRBLRules([]byte(dump.Stdout))
if err != nil || len(antispam.CheckFeedsLoaded(enabled, rules, *s)) > 0 {
return errFeedsLoaded
}
}
if s.HasFeed(antispam.FeedCSMIPMap) {
dump := t.cmd(ctx, commandLimit, "rspamadm", "configdump", "-j", "multimap")
if dump.Err != nil {
return errors.Join(errFeedsLoaded, cmdErr("rspamadm configdump -j multimap", dump))
}
if len(antispam.CheckIPMapLoaded(enabled, []byte(dump.Stdout))) > 0 {
return errFeedsLoaded
}
}
return nil
}
// checkLearning compares what rspamd loaded after the reload with CSM's
// learning files, including the effective weight only the controller shows.
// Errors carry no command output: the dumps hold keys.
func (t *tx) checkLearning(ctx context.Context, f antispam.HostFacts, keys antispam.LearningKeys, redisSocket string) error {
socket, ok := antispam.PrivateControllerSocket(f)
if !ok {
return errors.Join(errLearningLoaded, errors.New("learning needs one private controller socket"))
}
dump := func(args ...string) (string, error) {
r := t.cmd(ctx, commandLimit, "rspamadm", append([]string{"configdump", "-j"}, args...)...)
if r.Err != nil {
return "", errors.Join(errLearningProbe, cmdErr("rspamadm configdump -j "+strings.Join(args, " "), r))
}
return r.Stdout, nil
}
fuzzyCheck, err := dump("fuzzy_check")
if err != nil {
return err
}
full, err := dump()
if err != nil {
return err
}
group, err := dump("group")
if err != nil {
return err
}
acts, err := dump("actions")
if err != nil {
return err
}
root, err := antispam.ParseRspamdDumpObject([]byte(full))
if err != nil {
return errors.Join(errLearningLoaded, errors.New("rspamadm configdump -j is not readable"))
}
actions, err := antispam.ParseActions([]byte(acts))
if err != nil {
return errors.Join(errLearningLoaded, errors.New("rspamd's actions are not readable"))
}
// A successful scan ping can still come from the old workers while the
// controller is reloading. Use the same bounded service-probe window.
err = retry(ctx, t.h.rspamdWait, func(ctx context.Context) error {
symbols, getErr := t.h.controllerGet(ctx, socket, "/symbols", maxSymbolsReply, f.ControllerHost.RspamdUID)
if getErr != nil {
return errors.Join(errLearningProbe, getErr)
}
return antispam.CheckLearningLoaded([]byte(fuzzyCheck), root["worker"], []byte(group), symbols, actions, keys, redisSocket)
})
switch {
case err == nil:
return nil
case errors.Is(err, errLearningProbe):
return err
}
return errors.Join(errLearningLoaded, err)
}
// reloadRspamd reloads rspamd and waits for its probe.
func (t *tx) reloadRspamd(ctx context.Context, probe func(context.Context) error) error {
t.rspamdReloaded = true
if r := t.cmd(ctx, commandLimit, "systemctl", "reload", antispam.RspamdUnit); r.Err != nil {
return cmdErr("systemctl reload rspamd", r)
}
if err := probe(ctx); err != nil {
return errors.New("rspamd did not pass its probe after reload")
}
return ctx.Err()
}
// fail rolls back and reports step as the failed step.
func (t *tx) fail(step string, err error) antispam.Result {
state, rbErr := t.rollback()
msg := err.Error()
if rbErr != nil {
msg += "; rollback: " + rbErr.Error()
}
return antispam.Result{FailedStep: step, Rollback: state, Error: antispam.ErrorSummary(msg)}
}
// rollback restores every change in reverse and reloads each service whose
// reload was attempted. "verified" needs every restore, rebuild, reload and
// probe to succeed; otherwise the incomplete record stays for reconcile.
func (t *tx) rollback() (state string, rollbackErr error) {
if !t.recordWritten && len(t.writes) == 0 && len(t.backups) == 0 && !t.eximTouched && !t.rspamdReloaded {
return antispam.RollbackNone, nil
}
ctx, cancel := context.WithDeadline(context.Background(), t.start.Add(totalBudget))
defer cancel()
defer func() {
if state == antispam.RollbackFailed && t.recordWritten &&
t.recordCurrent.content != string(antispam.EncodeRecord("")) {
rollbackErr = errors.Join(rollbackErr, t.replaceRecord(ctx, ""))
}
}()
// Cleanup must also run when the rollback deadline interrupts later
// restores. A failed cleanup prevents a verified completion record.
defer func() {
if err := t.dropBackups(); err != nil {
state = antispam.RollbackFailed
rollbackErr = errors.Join(rollbackErr, err)
}
}()
var errs []error
// Completion can reach disk before its fsync fails. Invalidate it
// before recovery, which may consume the rest of the rollback budget.
if t.recordWritten && t.recordCurrent.content != string(antispam.EncodeRecord("")) {
if err := t.replaceRecord(ctx, ""); err != nil {
errs = append(errs, err)
}
}
// Recreate consumed operator copies before overwriting their restored
// files. This preservation must run even after the rollback deadline.
if err := t.restoreConsumedCopies(); err != nil {
return antispam.RollbackFailed, errors.Join(append(errs, err)...)
}
for i := len(t.writes) - 1; i >= 0; i-- {
if err := ctx.Err(); err != nil {
return antispam.RollbackFailed, err
}
w := t.writes[i]
before := w.before
switch {
case w.backup != "":
// An operator's private file must not inherit the replacement's
// service group or more permissive mode.
case slices.Contains(antispam.SecretPaths(), strings.TrimPrefix(w.path, t.h.root)):
// Rollback can put secrets back into this file, so it never
// keeps a mode someone loosened meanwhile: it gets the private
// metadata this apply wrote, or the snapshot's.
if w.next.metadata {
before.uid, before.gid, before.perm = w.next.uid, w.next.gid, w.next.perm
}
case w.next.exists:
// Other surviving files keep later owner and mode changes. Only
// a deleted file needs its snapshot metadata reinstated.
before.metadata = false
}
if err := t.h.replace(w.path, w.next, before); err != nil {
errs = append(errs, fmt.Errorf("restoring %s: %w", strings.TrimPrefix(w.path, t.h.root), err))
} else if w.backup != "" {
t.backups[w.backup] = false
}
}
if err := t.dropBackups(); err != nil {
errs = append(errs, err)
}
loadable := false
if t.eximTouched {
if r := t.cmd(ctx, buildLimit, BuildEximConf); r.Err != nil {
errs = append(errs, cmdErr("buildeximconf", r))
} else if err := t.validateExim(ctx, t.eximHadHooks); err != nil {
errs = append(errs, err)
} else {
loadable = true
if err := t.checkRollbackExim(ctx); err != nil {
errs = append(errs, err)
}
}
}
if t.eximReloaded {
// A HUP re-executes Exim with the file on disk. A configuration that
// failed its build or validation could stop a running daemon, so
// Exim keeps the configuration it loaded and the rollback fails.
var reload antispam.CommandResult
if loadable {
reload = t.cmd(ctx, commandLimit, "systemctl", "reload", antispam.EximUnit)
}
switch active, err := t.serviceActive(ctx, antispam.EximUnit); {
case err != nil:
errs = append(errs, err)
case !active:
// An Exim that stopped is started on the restored
// configuration; this is not a restart of a running daemon.
if r := t.cmd(ctx, commandLimit, "systemctl", "start", antispam.EximUnit); r.Err != nil {
errs = append(errs, cmdErr("systemctl start exim", r))
}
case reload.Err != nil:
errs = append(errs, cmdErr("systemctl reload exim", reload))
}
if err := t.probeExim(ctx); err != nil {
errs = append(errs, err)
}
}
if t.rspamdReloaded {
if err := t.checkRspamd(ctx, nil); err != nil {
errs = append(errs, err)
}
if err := t.reloadRspamd(ctx, t.rspamdRollbackProbe); err != nil {
errs = append(errs, err)
}
}
if len(errs) > 0 {
return antispam.RollbackFailed, errors.Join(errs...)
}
if t.recordWritten {
if err := t.restoreRecord(ctx); err != nil {
return antispam.RollbackFailed, err
}
}
return antispam.RollbackVerified, nil
}
func (t *tx) restoreRecord(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
p := t.h.path(antispam.RecordPath)
before := t.record
if before.exists {
before.perm = 0o600
}
err := t.h.replace(p, t.recordCurrent, before)
if err == nil || errors.Is(err, errDurability) {
t.recordCurrent = before
}
if err == nil {
return ctx.Err()
}
return err
}
// nobody returns the nobody account's uid and gid.
func nobody() (int, int, error) {
u, err := user.Lookup("nobody")
if err != nil {
return -1, -1, err
}
uid, uidErr := strconv.Atoi(u.Uid)
gid, gidErr := strconv.Atoi(u.Gid)
if uidErr != nil || gidErr != nil {
return -1, -1, errors.New("the nobody account has a non-numeric id")
}
return uid, gid, nil
}
// Package worker runs CSM's antispam transactions at the root boundary and
// launches them from the daemon: one JSON request in, one JSON result out,
// under the shared Exim mutation lock.
package worker
import (
"encoding/json"
"errors"
"io"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/eximlock"
)
// Transactor performs the apply and remove transactions. Handle
// calls it only with a validated request, under the lock.
type Transactor interface {
Apply(antispam.Settings) antispam.Result
Remove() antispam.Result
}
// Locker runs mutate for request under the shared Exim mutation lock.
type Locker func(request []byte, mutate func() error) error
// Handle reads one request from input, runs it under lock and writes one
// JSON result line to output. It returns the process exit code: 0 when the
// result is OK, 1 otherwise.
func Handle(input io.Reader, output io.Writer, t Transactor, lock Locker) int {
result := handle(input, t, lock)
result.Error = antispam.ErrorSummary(result.Error)
line, err := json.Marshal(result)
if err != nil {
return 1
}
line = append(line, '\n')
if n, err := output.Write(line); err != nil || n != len(line) || !result.OK {
return 1
}
return 0
}
func handle(input io.Reader, t Transactor, lock Locker) antispam.Result {
data, err := io.ReadAll(io.LimitReader(input, antispam.MaxRequestBytes+1))
if err != nil {
return antispam.Result{FailedStep: "request", Rollback: antispam.RollbackNone, Error: "cannot read request"}
}
req, settings, err := antispam.DecodeRequest(data)
if err != nil {
return antispam.Result{FailedStep: "request", Rollback: antispam.RollbackNone, Error: err.Error()}
}
var result antispam.Result
err = lock(data, func() error {
if req.Operation == antispam.OpApply {
result = t.Apply(settings)
} else {
result = t.Remove()
}
return nil
})
if err != nil {
busy := errors.Is(err, eximlock.ErrBusy) || errors.Is(err, errSuperseded)
result = antispam.Result{Busy: busy, FailedStep: "lock", Rollback: antispam.RollbackNone, Error: err.Error()}
}
result.Operation = req.Operation
if err := result.Validate(); err != nil {
// The transaction ran but cannot say what it changed.
return antispam.Result{Operation: req.Operation, FailedStep: "result", Rollback: antispam.RollbackFailed,
Error: "the transaction returned an invalid result: " + err.Error()}
}
return result
}
package worker
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"strings"
"time"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/eximlock"
"github.com/pidginhost/csm/internal/systemdrun"
)
// Timeout bounds one worker run. systemd stops the transient unit at the
// same limit.
const Timeout = 5 * time.Minute
// Command is the hidden csm subcommand that runs Handle as root.
const Command = "antispam-worker"
// maxStderr bounds the worker diagnostics copied into an error.
const maxStderr = 512
// Results contain summaries, not command transcripts. Reject excess stdout
// rather than silently accepting a complete JSON prefix of truncated output.
const maxResultBytes = 1 << 20
type cappedBuffer struct {
data []byte
limit int
exceeded bool
}
func (b *cappedBuffer) Write(p []byte) (int, error) {
n := len(p)
remaining := b.limit - len(b.data)
if n > remaining {
b.exceeded = true
p = p[:remaining]
}
b.data = append(b.data, p...)
// Drain excess output so a full pipe cannot block the worker's rollback.
return n, nil
}
// Run records req as the latest request under the shared Exim lock, then
// sends it to `csm antispam-worker` in a transient systemd unit, outside the
// daemon's sandbox, and returns the worker's result. A failed or busy result
// is a Result, not an error; an error means no result came back. A busy lock
// returns a busy result without starting the worker. A request that cannot be
// recorded, or a failed systemd-run lookup or availability probe (wrapping
// systemdrun.ErrUnavailable), means the worker was not attempted. Other
// errors may follow state changes.
func Run(ctx context.Context, req antispam.Request) (antispam.Result, error) {
if err := ctx.Err(); err != nil {
return antispam.Result{}, err
}
binary, err := os.Executable()
if err != nil {
return antispam.Result{}, err
}
body, err := json.Marshal(req)
if err != nil {
return antispam.Result{}, err
}
recorded, err := recordRequest(ctx, requestDir, body)
if err != nil {
return antispam.Result{}, fmt.Errorf("recording the antispam request: %w", err)
}
if !recorded {
return antispam.Result{Operation: req.Operation, Busy: true, FailedStep: "lock", Rollback: antispam.RollbackNone, Error: eximlock.ErrBusy.Error()}, nil
}
ctx, cancel := context.WithTimeout(ctx, Timeout)
defer cancel()
run := func(ctx context.Context, name string, args ...string) ([]byte, error) {
// #nosec G204 -- this executable with a fixed subcommand, or systemd-run with fixed flags.
cmd := exec.CommandContext(ctx, name, args...)
cmd.Stdin = bytes.NewReader(body)
// The transient unit's descendants may retain the
// pipes after CommandContext kills the process we are waiting for.
cmd.WaitDelay = time.Second
stdout := cappedBuffer{limit: maxResultBytes}
stderr := cappedBuffer{limit: maxStderr}
cmd.Stdout, cmd.Stderr = &stdout, &stderr
err := cmd.Run()
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
err = ctxErr
}
if detail := clip(string(stderr.data)); detail != "" {
err = fmt.Errorf("%w: %s", err, detail)
}
}
if stdout.exceeded {
return nil, errors.Join(fmt.Errorf("antispam worker output exceeds %d bytes", maxResultBytes), err)
}
return stdout.data, err
}
return execute(ctx, binary, exec.LookPath, run)
}
func execute(ctx context.Context, binary string, lookup systemdrun.LookPathFunc, run systemdrun.RunnerFunc) (antispam.Result, error) {
out, runErr := systemdrun.Run(ctx, lookup, run, systemdrun.Options{Pipe: true, RuntimeMax: Timeout, Required: true}, binary, Command)
// The worker exits 1 for a failed or busy result and still prints it.
if result, err := antispam.ParseResult(out); err == nil {
return result, nil
} else if runErr == nil {
return antispam.Result{}, fmt.Errorf("antispam worker returned no result: %w", err)
}
return antispam.Result{}, fmt.Errorf("antispam worker failed: %w", runErr)
}
func clip(s string) string {
s = strings.TrimSpace(s)
if len(s) > maxStderr {
s = s[:maxStderr]
}
return s
}
package worker
import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"github.com/pidginhost/csm/internal/eximlock"
)
// RequestFile holds the latest request, or null when reconciliation has
// invalidated pending workers. Publication atomically replaces the record.
const RequestFile = "antispam-request.json"
// requestDir holds the lock and the request record for Run. Tests point it
// at a temporary directory.
var requestDir = eximlock.Dir
// errSuperseded reports a request the daemon replaced before this worker
// reached the lock, for example one from before a daemon restart. The newer
// request decides, so the worker reports busy and changes nothing.
var errSuperseded = errors.New("a newer antispam request replaced this one")
// Invalidate stops pending workers before reconcile reads the host. Even a
// pass that needs no worker must supersede requests from an older daemon.
// The cancellation is published without waiting for the lock, so a stale
// worker waiting behind a running transaction finds it once that transaction
// ends. A running transaction is still reported as busy, so the pass reads
// the host only after it ends.
func Invalidate(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
// #nosec G301 -- the shared lock directory, created with the mode eximlock uses.
if err := os.MkdirAll(requestDir, 0o755); err != nil {
return err
}
path := filepath.Join(requestDir, RequestFile)
tmp, err := stageRecord(path, []byte("null"))
if err != nil {
return err
}
if err := ctx.Err(); err != nil {
_ = os.Remove(tmp)
return err
}
if err := os.Rename(tmp, path); err != nil {
_ = os.Remove(tmp)
return err
}
if err := syncRecordDir(requestDir); err != nil {
return err
}
// A brief record publication must not defer the pass for a full retry
// interval. Pending workers are already canceled while this waits.
return eximlock.WithRetry(ctx, requestDir, eximlock.RecordWait, func() error { return nil })
}
// Lock is the worker's Locker: it holds the shared lock in dir and runs
// mutate only while request is the latest one recorded there. A missing
// record comes from a daemon that predates it, so the request runs.
func Lock(dir string) Locker {
return func(request []byte, mutate func() error) error {
return eximlock.WithRetry(context.Background(), dir, eximlock.RecordWait, func() error {
// #nosec G304 -- fixed CSM state path next to the lock.
recorded, err := os.ReadFile(filepath.Join(dir, RequestFile))
if err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if err == nil && !bytes.Equal(recorded, request) {
return errSuperseded
}
return mutate()
})
}
}
// recordRequest records body as the latest request under the shared lock in
// dir. It reports false without an error while another transaction holds
// the lock for longer than eximlock.RecordWait. The record is written and
// flushed before the lock is taken; under the lock it is only renamed into
// place. The directory is flushed after unlocking, before reporting success.
func recordRequest(ctx context.Context, dir string, body []byte) (bool, error) {
if err := ctx.Err(); err != nil {
return false, err
}
// #nosec G301 -- the shared lock directory, created with the mode eximlock uses.
if err := os.MkdirAll(dir, 0o755); err != nil {
return false, err
}
path := filepath.Join(dir, RequestFile)
tmp, err := stageRecord(path, body)
if err != nil {
return false, err
}
published := false
err = eximlock.WithRetry(ctx, dir, eximlock.RecordWait, func() error {
if renameErr := os.Rename(tmp, path); renameErr != nil {
return renameErr
}
published = true
return nil
})
if !published {
_ = os.Remove(tmp)
}
if errors.Is(err, eximlock.ErrBusy) {
return false, nil
}
if err != nil {
return false, err
}
err = syncRecordDir(dir)
return err == nil, err
}
// stageRecord writes body to a new 0600 temporary file next to path and
// flushes it. An interrupted write leaves the previous record in place.
// Tests replace it.
var stageRecord = func(path string, body []byte) (string, error) {
f, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*.tmp")
if err != nil {
return "", err
}
tmp := f.Name()
_, err = f.Write(body)
if err == nil {
err = f.Sync()
}
if closeErr := f.Close(); err == nil {
err = closeErr
}
if err != nil {
_ = os.Remove(tmp)
return "", err
}
return tmp, nil
}
// A file flush alone does not persist the rename. Keep the directory flush
// outside the shared lock, too, so it cannot delay another record writer.
var syncRecordDir = func(dir string) error {
// #nosec G304 -- fixed CSM state directory, opened only to persist the rename.
d, err := os.Open(dir)
if err != nil {
return err
}
err = d.Sync()
if closeErr := d.Close(); err == nil {
err = closeErr
}
return err
}
// Package atomicio implements atomic file writes used by state-bearing
// callers (firewall engine, autoblock tracker, etc.). The package is a
// dependency leaf: it imports only the standard library so any caller
// can use AtomicWriteJSON without risking an import cycle through the
// existing state / store packages.
package atomicio
import (
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
)
// AtomicWriteJSON marshals v to JSON and writes it to path atomically:
// MarshalIndent, write tmp, fsync tmp, rename. Returns the first error
// encountered. On rename failure the tmp file is removed best-effort.
//
// Reserved for state files that callers re-read on the next startup or
// the next tick - a torn write would leave the daemon with stale or
// corrupt state. Hot-path callers that only need a best-effort cache
// dump should not use this helper.
func AtomicWriteJSON(path string, perm os.FileMode, v any) error {
data, err := json.MarshalIndent(v, "", " ")
if err != nil {
return fmt.Errorf("marshal: %w", err)
}
legacyTmp := path + ".tmp"
if removeErr := os.Remove(legacyTmp); removeErr != nil && !os.IsNotExist(removeErr) {
return fmt.Errorf("remove stale tmp: %w", removeErr)
}
return atomicWrite(path, perm, data)
}
// AtomicWrite writes already-serialized bytes to path atomically with the
// same write-tmp, fsync, rename, dir-fsync sequence as AtomicWriteJSON.
func AtomicWrite(path string, perm os.FileMode, data []byte) error {
return atomicWrite(path, perm, data)
}
func atomicWrite(path string, perm os.FileMode, data []byte) error {
dir := filepath.Dir(path)
// #nosec G304 -- caller owns the destination path; tmp lives in
// the same operator-owned state directory.
f, err := os.CreateTemp(dir, "."+filepath.Base(path)+".*.tmp")
if err != nil {
return fmt.Errorf("open tmp: %w", err)
}
tmp := f.Name()
if err := f.Chmod(perm); err != nil {
_ = f.Close()
_ = os.Remove(tmp)
return fmt.Errorf("chmod tmp: %w", err)
}
if n, err := f.Write(data); err != nil {
_ = f.Close()
_ = os.Remove(tmp)
return fmt.Errorf("write tmp: %w", err)
} else if n != len(data) {
_ = f.Close()
_ = os.Remove(tmp)
return fmt.Errorf("write tmp: %w", io.ErrShortWrite)
}
if err := f.Sync(); err != nil {
_ = f.Close()
_ = os.Remove(tmp)
return fmt.Errorf("fsync tmp: %w", err)
}
if err := f.Close(); err != nil {
_ = os.Remove(tmp)
return fmt.Errorf("close tmp: %w", err)
}
if err := os.Rename(tmp, path); err != nil {
_ = os.Remove(tmp)
return fmt.Errorf("rename: %w", err)
}
// #nosec G304 -- dir is filepath.Dir of caller-owned path; opened
// read-only solely to fsync the directory after rename so the
// new dentry survives a power-loss.
d, openErr := os.Open(dir)
if openErr != nil {
return fmt.Errorf("open dir: %w", openErr)
}
if err := d.Sync(); err != nil {
_ = d.Close()
return fmt.Errorf("fsync dir: %w", err)
}
if err := d.Close(); err != nil {
return fmt.Errorf("close dir: %w", err)
}
return nil
}
package attackcorpus
import (
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"runtime"
"slices"
"strings"
"testing"
"github.com/pidginhost/csm/internal/config"
)
// Config is the config under test: the shipped template as a fresh install
// loads it, with auto-response and file quarantine on, observe mode off and
// state in stateDir. Nothing else differs from the template.
func Config(stateDir string) (*config.Config, error) {
_, file, _, _ := runtime.Caller(0)
// #nosec G304 -- the shipped template at a fixed path in this source tree.
data, err := os.ReadFile(filepath.Join(filepath.Dir(file), "..", "..", "configs", "csm.yaml"))
if err != nil {
return nil, err
}
cfg, err := config.LoadBytes(data)
if err != nil {
return nil, err
}
cfg.AutoResponse.Enabled = true
cfg.AutoResponse.QuarantineFiles = true
cfg.Mode = config.ModeEnforce
cfg.StatePath = stateDir
return cfg, nil
}
// PrivateCommit returns the private corpus commit pinned in private.commit
// beside this package's source.
func PrivateCommit() (string, error) {
_, file, _, _ := runtime.Caller(0)
return readCommit(filepath.Join(filepath.Dir(file), "private.commit"))
}
// A full commit hash only: a short hash or ref name could resolve to another
// commit than the one reviewed.
var commitPin = regexp.MustCompile(`^[0-9a-f]{40}\n?$`)
func readCommit(path string) (string, error) {
// #nosec G304 -- the pin file at a fixed path in this source tree, or a test fixture.
data, err := os.ReadFile(path)
if err != nil {
return "", err
}
if !commitPin.Match(data) {
return "", fmt.Errorf("%s: want one line of 40 lowercase hex digits", path)
}
return strings.TrimSuffix(string(data), "\n"), nil
}
// gate decides whether the corpus runs. It runs only with a report dir, and a
// pipeline that requires it fails instead of skipping when the dir is unset.
func gate(getenv func(string) string) (reportDir, skipReason string, err error) {
if dir := getenv("CSM_ATTACK_CORPUS_REPORT_DIR"); dir != "" {
return dir, "", nil
}
if getenv("CSM_ATTACK_CORPUS_REQUIRED") == "1" || getenv("CSM_ATTACK_CORPUS_PRIVATE_REQUIRED") == "1" {
return "", "", errors.New("a corpus run is required but CSM_ATTACK_CORPUS_REPORT_DIR is not set")
}
return "", "CSM_ATTACK_CORPUS_REPORT_DIR is not set", nil
}
// prepare makes every check that must stop a run before its first sample and
// returns the report dir and samples, or a reason to skip. The private half is
// checked first, so its errors surface even when the public half is unusable.
func prepare(p Path, getenv func(string) string) (reportDir, skip string, samples []Sample, err error) {
if !slices.Contains(Paths, p) {
return "", "", nil, fmt.Errorf("unknown path %q", p)
}
dir := getenv("CSM_ATTACK_CORPUS_PRIVATE")
// Checked before the gate: a required private half must never skip silently.
if dir == "" && getenv("CSM_ATTACK_CORPUS_PRIVATE_REQUIRED") == "1" {
return "", "", nil, errors.New("CSM_ATTACK_CORPUS_PRIVATE_REQUIRED=1 but CSM_ATTACK_CORPUS_PRIVATE is not set")
}
if reportDir, skip, err = gate(getenv); err != nil || skip != "" {
return "", skip, nil, err
}
if dir != "" {
if samples, err = Load(dir, "private"); err != nil {
return "", "", nil, fmt.Errorf("private corpus: %w", err)
}
// A checkout that came up empty must not pass as a private run.
if len(samples) == 0 {
return "", "", nil, fmt.Errorf("private corpus %s has no samples", dir)
}
}
public, err := Load(PublicDir(), "public")
if err != nil {
return "", "", nil, fmt.Errorf("public corpus: %w", err)
}
// Reports and subtests identify a sample by name.
for _, s := range samples {
if slices.ContainsFunc(public, func(x Sample) bool { return x.Name == s.Name }) {
return "", "", nil, fmt.Errorf("private sample %s/%s: name also used in the public corpus", s.Category, s.Name)
}
}
if err = ValidateCoverage(public, true); err != nil {
return "", "", nil, fmt.Errorf("public corpus: %w", err)
}
return reportDir, "", append(public, samples...), nil
}
// Drive replays every corpus sample through path p, one subtest per sample,
// and writes the report even when samples fail. run must not call t.Parallel:
// drivers swap package globals, and the report is written after the loop.
func Drive(t *testing.T, p Path, run func(t *testing.T, s Sample) Outcome) {
t.Helper()
reportDir, skip, samples, err := prepare(p, os.Getenv)
if err != nil {
t.Fatal(err)
}
if skip != "" {
t.Skip(skip)
}
results := replay(t, p, samples, run)
if err := WriteReport(reportDir, p, results); err != nil {
t.Errorf("report: %v", err)
}
}
// DriverErrorText describes a driver error for a test log. Test logs become CI
// artifacts and an error can quote decoded sample bytes, so only its type is
// shown unless CSM_ATTACK_CORPUS_SHOW_ERRORS=1 asks for the reason locally.
func DriverErrorText(err error) string {
if os.Getenv("CSM_ATTACK_CORPUS_SHOW_ERRORS") == "1" {
return err.Error()
}
return fmt.Sprintf("%T (details withheld; set CSM_ATTACK_CORPUS_SHOW_ERRORS=1 locally)", err)
}
func replay(t *testing.T, p Path, samples []Sample, run func(t *testing.T, s Sample) Outcome) []Result {
t.Helper()
attempts := Offline(t)
results := make([]Result, 0, len(samples))
for _, s := range samples {
ran := false
t.Run(s.Half+"/"+s.Category+"/"+s.Name, func(t *testing.T) {
ran = true
// A driver that stops the subtest still leaves an error result.
o := Outcome{Err: errors.New("driver stopped before returning an outcome")}
before := attempts.Load()
// Register first so driver cleanups finish before the verdict is recorded.
t.Cleanup(func() {
if o.Err == nil && (t.Failed() || t.Skipped()) {
o.Err = errors.New("driver subtest did not pass")
}
o.NetworkAttempts += attempts.Load() - before
r := Judge(s, p, o)
for _, f := range r.Failures {
t.Error(f)
}
results = append(results, r)
})
o = run(t, s)
if o.Err != nil {
t.Logf("driver error: %s", DriverErrorText(o.Err))
}
})
if !ran {
r := Judge(s, p, Outcome{Err: errors.New("sample excluded by test filter")})
results = append(results, r)
t.Errorf("sample %s/%s/%s excluded by test filter", s.Half, s.Category, s.Name)
}
}
return results
}
package attackcorpus
import (
"cmp"
"fmt"
"slices"
"strings"
"github.com/pidginhost/csm/internal/alert"
)
// Result is the verdict for one sample on one path. It goes into the report,
// so it holds names, paths and verdicts, never sample content.
type Result struct {
Sample string `json:"sample"`
Category string `json:"category"`
Kind string `json:"kind"`
Half string `json:"half"`
Path Path `json:"path"`
Pass bool `json:"pass"`
Failures []string `json:"failures,omitempty"`
Findings []Observed `json:"findings,omitempty"`
Remediation map[string]Remediation `json:"remediation"`
Gap string `json:"gap,omitempty"`
NotAdmitted []string `json:"not_admitted,omitempty"`
NetworkAttempts int64 `json:"network_attempts"`
}
// Judge compares what a path did with a sample against the sample's manifest.
// Each failure starts with its kind: "missed:", "severity:", "false
// positive:", "remediation:", "gap closed:" or "error:". An outcome with an
// error is not judged further. Extra findings on a malicious sample are kept
// in Findings and never fail it. For malicious samples, findings on a file the
// admission filter dropped are discarded, so that file counts as not detected.
// A control's findings are always retained: admission cannot excuse a false positive.
//
// An unattributed control finding is a false positive when its text names a
// sample path, and an attribution error otherwise. On a path with a gap, only
// a finding on a planted file, or an unattributed finding whose text names an
// admitted sample path, closes the gap. A finding on a directory or any other
// path is an extra finding. @crontab is not matched in text because its real
// name is the account, which Judge is not given. Drivers must attribute crontab
// findings and exclude proven unrelated host findings before returning an
// outcome.
//
// Findings are sorted by file, check, severity and text, so reports and
// failures do not depend on the order scanners report in.
func Judge(s Sample, p Path, o Outcome) Result {
e := s.Expect[p]
r := Result{
Sample: s.Name, Category: s.Category, Kind: s.Kind, Half: s.Half, Path: p,
Remediation: o.Remediation, Gap: e.Gap, NotAdmitted: o.NotAdmitted, NetworkAttempts: o.NetworkAttempts,
}
for _, f := range o.Findings {
if s.Kind == "control" || !slices.Contains(o.NotAdmitted, f.File) {
r.Findings = append(r.Findings, f)
}
}
slices.SortFunc(r.Findings, func(a, b Observed) int {
return cmp.Or(cmp.Compare(a.File, b.File), cmp.Compare(a.Check, b.Check), cmp.Compare(a.Severity, b.Severity), cmp.Compare(a.Text, b.Text))
})
r.Failures = failures(s, e, r.Findings, o)
r.Pass = len(r.Failures) == 0
return r
}
func failures(s Sample, e PathExpect, findings []Observed, o Outcome) []string {
if o.Err != nil {
// Parser and scanner errors can quote decoded content, just like findings.
return []string{"error: driver did not complete"}
}
var out []string
add := func(format string, args ...any) { out = append(out, fmt.Sprintf(format, args...)) }
for _, want := range e.Findings {
best := alert.Severity(-1)
for _, f := range findings {
if f.Check == want.Check && f.File == want.File && f.Severity > best {
best = f.Severity
}
}
floor, _ := severityLevel(want.MinSeverity)
switch {
case best < 0:
add("missed: %s on %s", want.Check, want.File)
case best < floor:
add("severity: %s on %s is %s, want %s or above", want.Check, want.File, best, floor)
}
}
for _, f := range findings {
named := -1
if f.File == "" {
named = slices.IndexFunc(s.Files, func(sf File) bool {
return sf.Path != "@crontab" && strings.Contains(f.Text, sf.Path) &&
(s.Kind == "control" || !slices.Contains(o.NotAdmitted, sf.Path))
})
}
switch {
case f.File == "" && s.Kind == "control" && named >= 0:
add("false positive: %s names %s", f.Check, s.Files[named].Path)
case f.File == "" && s.Kind == "control":
add("error: unattributed finding %s on control", f.Check)
case f.File == "" && e.Gap != "" && named >= 0:
add("gap closed: %s names %s", f.Check, s.Files[named].Path)
case f.File == "":
// An unattributed extra finding on a malicious sample.
case s.Kind == "control":
add("false positive: %s on %s", f.Check, f.File)
case e.Gap != "" && slices.ContainsFunc(s.Files, func(sf File) bool { return sf.Path == f.File }):
add("gap closed: %s on %s", f.Check, f.File)
}
}
for _, f := range s.Files {
want, got := e.Remediation[f.Path], o.Remediation[f.Path]
if want == "" {
want = Untouched
}
switch {
case got != want:
add("remediation: %s is %s, want %s", f.Path, got, want)
case want == Cleaned && o.CleanedSHA256[f.Path] != e.CleanedSHA256[f.Path]:
add("remediation: %s cleaned to sha256 %s, want %s", f.Path, o.CleanedSHA256[f.Path], e.CleanedSHA256[f.Path])
}
}
return out
}
package attackcorpus
import (
"bytes"
"encoding/base64"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"slices"
"strings"
"github.com/pidginhost/csm/internal/safepath"
"github.com/pidginhost/csm/internal/selftest"
)
// Load reads every <dir>/<category>/<name>/sample.json and decodes each file
// once to validate it. Hidden entries and top-level files are skipped, so a
// git checkout with a README loads as a corpus. A missing or unreadable dir is
// an error: a mistyped private corpus path must not pass as an empty corpus.
func Load(dir, half string) ([]Sample, error) {
if half != "public" && half != "private" {
return nil, fmt.Errorf("unknown corpus half %q", half)
}
corpusDir, err := filepath.Abs(dir)
if err != nil {
return nil, err
}
root, err := safepath.OpenDir(corpusDir)
if err != nil {
return nil, err
}
defer func() { _ = root.Close() }()
categories, err := readDir(root, ".")
if err != nil {
return nil, err
}
var samples []Sample
seen := map[string]string{}
for _, c := range categories {
if strings.HasPrefix(c.Name(), ".") {
continue
}
if c.Type()&os.ModeSymlink != 0 {
return nil, fmt.Errorf("category %q: symlinks are not allowed", c.Name())
}
if !c.IsDir() {
continue
}
entries, err := readDir(root, c.Name())
if err != nil {
return nil, err
}
for _, e := range entries {
if strings.HasPrefix(e.Name(), ".") {
continue
}
rel := c.Name() + "/" + e.Name()
if !e.IsDir() {
return nil, fmt.Errorf("sample %s: expected a directory, not a file or symlink", rel)
}
s, err := loadSample(root, corpusDir, rel, half)
if err != nil {
return nil, fmt.Errorf("sample %s: %w", rel, err)
}
if prev, dup := seen[s.Name]; dup {
return nil, fmt.Errorf("sample %s: duplicate name, also in %s", rel, prev)
}
seen[s.Name] = rel
s.Dir = filepath.Join(dir, c.Name(), e.Name())
samples = append(samples, s)
}
}
return samples, nil
}
func readDir(root *safepath.Dir, relative string) ([]os.DirEntry, error) {
dir, err := root.Walk(relative)
if err != nil {
return nil, err
}
defer func() { _ = dir.Close() }()
entries, err := dir.ReadDir()
if err != nil {
return nil, err
}
slices.SortFunc(entries, func(a, b os.DirEntry) int { return strings.Compare(a.Name(), b.Name()) })
return entries, nil
}
func readRegularFile(dir *safepath.Dir, name string) ([]byte, error) {
// safepath rejects symlinks and opens nonblocking, so a FIFO cannot stall a run.
f, err := dir.OpenFile(name, os.O_RDONLY, 0)
if err != nil {
return nil, err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() {
return nil, fmt.Errorf("%q is not a regular file", name)
}
data, err := io.ReadAll(f)
if err != nil {
return nil, err
}
return data, nil
}
func loadSample(root *safepath.Dir, corpusDir, relative, half string) (Sample, error) {
dir, err := root.Walk(relative)
if err != nil {
return Sample{}, err
}
defer func() { _ = dir.Close() }()
data, err := readRegularFile(dir, "sample.json")
if err != nil {
return Sample{}, err
}
s, err := decodeManifest(data)
if err != nil {
return Sample{}, err
}
s.Half, s.corpusDir, s.relativeDir = half, corpusDir, relative
if err := s.validate(dir); err != nil {
return Sample{}, err
}
if s.Name != filepath.Base(relative) || s.Category != filepath.Base(filepath.Dir(relative)) {
return Sample{}, fmt.Errorf("manifest %s/%s does not match its directory", s.Category, s.Name)
}
return s, nil
}
func decodeManifest(data []byte) (Sample, error) {
// JSON v2 rejects duplicate names and case aliases. Parser errors can quote
// input bytes, so only fixed diagnostics leave this boundary.
dec := jsontext.NewDecoder(bytes.NewReader(data))
if _, err := dec.ReadValue(); err != nil {
return Sample{}, errors.New("invalid manifest JSON")
}
if _, err := dec.ReadValue(); err != io.EOF {
return Sample{}, errors.New("trailing data after the manifest")
}
var s Sample
if err := json.Unmarshal(data, &s, json.RejectUnknownMembers(true)); err != nil {
return Sample{}, errors.New("invalid manifest: unknown field or wrong field type")
}
return s, nil
}
// Content returns the decoded bytes of a file listed in a loaded sample.
// Source directories and files must still be present without symlinks.
func (s Sample) Content(f File) ([]byte, error) {
if !slices.Contains(s.Files, f) {
return nil, errors.New("file is not in the loaded sample")
}
if s.corpusDir == "" {
return nil, errors.New("sample has not been loaded")
}
root, err := safepath.OpenDir(s.corpusDir)
if err != nil {
return nil, err
}
defer func() { _ = root.Close() }()
dir, err := root.Walk(s.relativeDir)
if err != nil {
return nil, err
}
defer func() { _ = dir.Close() }()
return content(dir, f)
}
func content(dir *safepath.Dir, f File) ([]byte, error) {
if f.Selftest != "" {
for _, b := range selftest.Samples() {
if b.Name == f.Selftest {
return b.Content()
}
}
return nil, fmt.Errorf("unknown selftest sample %q", f.Selftest)
}
encoded, err := readRegularFile(dir, f.B64)
if err != nil {
return nil, err
}
data, err := base64.StdEncoding.DecodeString(string(encoded))
if err != nil {
return nil, fmt.Errorf("%s: %w", f.B64, err)
}
if len(data) == 0 {
return nil, fmt.Errorf("%s: empty content", f.B64)
}
return data, nil
}
// PublicDir is the public corpus root inside this package's source tree.
func PublicDir() string {
_, file, _, _ := runtime.Caller(0)
return filepath.Join(filepath.Dir(file), "testdata", "corpus")
}
// ValidateCoverage requires a malicious sample and a control in every
// category and, with requireSelftest, every self-test bundle sample referenced
// exactly once, so the bundle and the corpus cannot drift.
func ValidateCoverage(samples []Sample, requireSelftest bool) error {
var errs []error
for _, c := range Categories {
var malicious, control bool
for _, s := range samples {
if s.Category == string(c) {
malicious = malicious || s.Kind == "malicious"
control = control || s.Kind == "control"
}
}
if !malicious {
errs = append(errs, fmt.Errorf("category %s has no malicious sample", c))
}
if !control {
errs = append(errs, fmt.Errorf("category %s has no control sample", c))
}
}
if requireSelftest {
refs := map[string]int{}
for _, s := range samples {
for _, f := range s.Files {
if f.Selftest != "" {
refs[f.Selftest]++
}
}
}
for _, b := range selftest.Samples() {
if n := refs[b.Name]; n != 1 {
errs = append(errs, fmt.Errorf("selftest sample %s referenced %d times, want 1", b.Name, n))
}
}
}
return errors.Join(errs...)
}
// Package attackcorpus loads the attack replay corpus: malicious samples and
// near-miss controls, each with what every detection path must report for it.
//
// Sample content stays base64-encoded at rest and is decoded in memory only,
// because endpoint antivirus deletes decoded web shells from a checkout.
package attackcorpus
import (
"errors"
"fmt"
"path/filepath"
"regexp"
"slices"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/safepath"
)
// Category groups samples by attack type.
type Category string
const (
Webshell Category = "webshell"
Dropper Category = "dropper"
MaliciousPlugin Category = "malicious_plugin"
CredentialStealer Category = "credential_stealer" // #nosec G101 -- category name, not a credential.
InjectedJS Category = "injected_js"
PhishingKit Category = "phishing_kit"
SpamScript Category = "spam_script"
Persistence Category = "persistence"
Binary Category = "binary"
Archive Category = "archive"
)
// Categories lists every category; each needs a malicious sample and a control.
var Categories = []Category{
Webshell, Dropper, MaliciousPlugin, CredentialStealer, InjectedJS,
PhishingKit, SpamScript, Persistence, Binary, Archive,
}
// Path is a detection path a sample is replayed through.
type Path string
const (
Realtime Path = "realtime"
Tier Path = "tier"
FullScan Path = "fullscan"
)
// Paths lists every detection path; a manifest states its expectation for each.
var Paths = []Path{Realtime, Tier, FullScan}
// Remediation is what a path leaves on disk for one planted file.
type Remediation string
const (
Quarantined Remediation = "quarantined"
Cleaned Remediation = "cleaned"
Untouched Remediation = "untouched"
// Removed is only ever observed, never a valid expectation in a manifest.
Removed Remediation = "removed"
)
// File is one planted file. Exactly one of B64 or Selftest is set.
type File struct {
// Path is relative to the account home, or "@crontab" for the account's
// crontab in the cron spool.
Path string `json:"path"`
// B64 names a base64 file beside the manifest.
B64 string `json:"b64,omitempty"`
// Selftest names a sample in the self-test bundle.
Selftest string `json:"selftest,omitempty"`
// Mode is permission bits only. JSON has no octal, so 0644 is written 420.
Mode uint32 `json:"mode,omitempty"`
}
// ExpectedFinding is a finding a path must report.
type ExpectedFinding struct {
Check string `json:"check"`
File string `json:"file"`
// MinSeverity is warning, high or critical, in any case.
MinSeverity string `json:"min_severity"`
}
// PathExpect is what one path must do with a sample. Files missing from
// Remediation are expected untouched.
type PathExpect struct {
Findings []ExpectedFinding `json:"findings"`
Gap string `json:"gap"`
Remediation map[string]Remediation `json:"remediation"`
CleanedSHA256 map[string]string `json:"cleaned_sha256"`
}
// Sample is one corpus entry, read from <category>/<name>/sample.json.
type Sample struct {
Name string `json:"name"`
Category string `json:"category"`
// Kind is malicious or control.
Kind string `json:"kind"`
// Source is synthetic, fixture or real.
Source string `json:"source"`
Description string `json:"description"`
Provenance string `json:"provenance"`
Files []File `json:"files"`
Expect map[Path]PathExpect `json:"expect"`
// Dir is the sample directory the manifest was read from.
Dir string `json:"-"`
// Half is public or private.
Half string `json:"-"`
// Keep later reads anchored to the original corpus even if Dir or cwd changes.
corpusDir string
relativeDir string
}
var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`)
// severityLevel maps a manifest severity to the alert level it names.
func severityLevel(s string) (alert.Severity, bool) {
switch strings.ToLower(s) {
case "warning":
return alert.Warning, true
case "high":
return alert.High, true
case "critical":
return alert.Critical, true
}
return 0, false
}
// checkPath accepts a clean path inside the account, or the one reserved name.
func checkPath(p string) error {
if p == "@crontab" {
return nil
}
if !filepath.IsLocal(p) || filepath.Clean(p) != p || p == "." || strings.HasPrefix(p, "@") || strings.ContainsRune(p, '\x00') {
return fmt.Errorf("unsafe path %q", p)
}
return nil
}
func (s Sample) validate(dir *safepath.Dir) error {
switch {
case !slices.Contains(Categories, Category(s.Category)):
return fmt.Errorf("unknown category %q", s.Category)
case s.Kind != "malicious" && s.Kind != "control":
return fmt.Errorf("unknown kind %q", s.Kind)
case s.Source != "synthetic" && s.Source != "fixture" && s.Source != "real":
return fmt.Errorf("unknown source %q", s.Source)
case len(s.Files) == 0:
return errors.New("no files")
}
files := make(map[string]bool, len(s.Files))
for _, f := range s.Files {
if err := checkPath(f.Path); err != nil {
return err
}
if files[f.Path] {
return fmt.Errorf("duplicate path %q", f.Path)
}
files[f.Path] = true
if (f.B64 == "") == (f.Selftest == "") {
return fmt.Errorf("file %q: set exactly one of b64 or selftest", f.Path)
}
if f.B64 != "" && (filepath.Base(f.B64) != f.B64 || !filepath.IsLocal(f.B64)) {
return fmt.Errorf("file %q: b64 %q must be a file name beside the manifest", f.Path, f.B64)
}
// fixturecheck scans sample data decoded; a .b64 name keeps the two in step.
if f.B64 != "" && !strings.HasSuffix(f.B64, ".b64") {
return fmt.Errorf("file %q: b64 %q must end in \".b64\"", f.Path, f.B64)
}
if f.Mode&^0o777 != 0 {
return fmt.Errorf("file %q: mode %#o has bits beyond the permission bits", f.Path, f.Mode)
}
if _, err := content(dir, f); err != nil {
return fmt.Errorf("file %q: %w", f.Path, err)
}
}
for file := range files {
for parent := filepath.Dir(file); parent != "."; parent = filepath.Dir(parent) {
if files[parent] {
return fmt.Errorf("conflicting paths %q and %q", parent, file)
}
}
}
for p := range s.Expect {
if !slices.Contains(Paths, p) {
return fmt.Errorf("expect: unknown path %q", p)
}
}
for _, p := range Paths {
e, ok := s.Expect[p]
if !ok {
return fmt.Errorf("missing expect for %s", p)
}
if err := s.validateExpect(e, files); err != nil {
return fmt.Errorf("expect %s: %w", p, err)
}
}
return nil
}
func (s Sample) validateExpect(e PathExpect, files map[string]bool) error {
switch {
case e.Gap != "" && len(e.Findings) > 0:
return errors.New("gap and findings are exclusive")
case s.Kind == "control" && (e.Gap != "" || len(e.Findings) > 0):
return errors.New("a control expects no gap or findings")
case s.Kind == "malicious" && strings.TrimSpace(e.Gap) == "" && len(e.Findings) == 0:
return errors.New("a malicious sample needs findings or a gap")
}
for _, f := range e.Findings {
if f.Check == "" {
return fmt.Errorf("finding for %q has no check", f.File)
}
if !files[f.File] {
return fmt.Errorf("finding file %q is not in files", f.File)
}
if _, ok := severityLevel(f.MinSeverity); !ok {
return fmt.Errorf("finding %s: unknown severity %q", f.Check, f.MinSeverity)
}
}
for file, r := range e.Remediation {
if !files[file] {
return fmt.Errorf("remediation file %q is not in files", file)
}
if r != Quarantined && r != Cleaned && r != Untouched {
return fmt.Errorf("file %q: remediation %q is not allowed", file, r)
}
if s.Kind == "control" && r != Untouched {
return fmt.Errorf("a control leaves every file untouched, %q is %s", file, r)
}
if r == Cleaned && !sha256Hex.MatchString(e.CleanedSHA256[file]) {
return fmt.Errorf("file %q is cleaned but has no valid cleaned_sha256", file)
}
}
for file := range e.CleanedSHA256 {
if e.Remediation[file] != Cleaned {
return fmt.Errorf("cleaned_sha256 for %q, which is not cleaned", file)
}
}
return nil
}
package attackcorpus
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/safepath"
)
// Observed is one finding a path reported, as the judge sees it.
type Observed struct {
Check string `json:"check"`
Severity alert.Severity `json:"severity"`
// File is the sample path the finding names, "" when unattributed.
File string `json:"file"`
// Text is the finding's message and details. Details can quote sample
// content, so Text is for the judge only and never reaches a report.
Text string `json:"-"`
}
// MarshalJSON writes the severity by name: alert.Severity has no JSON form of
// its own and would appear in reports as 0, 1 or 2.
func (o Observed) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
Check string `json:"check"`
Severity string `json:"severity"`
File string `json:"file"`
}{o.Check, o.Severity.String(), o.File})
}
// ObservedFrom converts a finding, so every driver builds Text the same way.
func ObservedFrom(f alert.Finding, file string) Observed {
return Observed{Check: f.Check, Severity: f.Severity, File: file, Text: f.Message + "\n" + f.Details}
}
// Outcome is what a driver saw when it ran one sample through one path.
type Outcome struct {
Findings []Observed
Remediation map[string]Remediation
CleanedSHA256 map[string]string
// NotAdmitted lists sample paths the realtime admission filter dropped.
NotAdmitted []string
// NetworkAttempts counts lookups a driver's own seams answered instead of
// the network. Drive adds the requests that reached http.DefaultTransport.
NetworkAttempts int64
Err error
}
// ObserveRemediation reports what happened to each planted file. A missing
// file is quarantined when a quarantine entry records its path or a parent
// directory's and retains the original bytes, and removed otherwise. A file
// with changed bytes is cleaned, with the SHA-256 of what is left. Missing
// baselines or damaged quarantine entries are errors. quarantineDir must be
// the sample's own, so every entry in it is new.
func ObserveRemediation(planted map[string]string, original map[string][]byte, quarantineDir string) (map[string]Remediation, map[string]string, error) {
quarantine, err := safepath.OpenDir(quarantineDir)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return nil, nil, err
}
if quarantine != nil {
defer func() { _ = quarantine.Close() }()
}
moved, err := quarantinedPaths(quarantine)
if err != nil {
return nil, nil, err
}
remediation := make(map[string]Remediation, len(planted))
sums := map[string]string{}
for p, abs := range planted {
before := original[p]
if len(before) == 0 {
return nil, nil, fmt.Errorf("%s: missing original content", p)
}
// #nosec G304 -- abs is a file Plant wrote under the test's temp tree.
data, err := os.ReadFile(abs)
switch {
case errors.Is(err, fs.ErrNotExist):
remediation[p] = Removed
for _, entry := range moved {
rel, relErr := filepath.Rel(entry.original, abs)
if relErr != nil || (rel != "." && !filepath.IsLocal(rel)) {
continue
}
retained := filepath.Join(entry.archived, rel)
dir, openErr := quarantine.Walk(filepath.Dir(retained))
if openErr != nil {
return nil, nil, fmt.Errorf("%s: open retained quarantine: %w", p, openErr)
}
retainedData, readErr := readRegularFile(dir, filepath.Base(retained))
_ = dir.Close()
if readErr != nil {
return nil, nil, fmt.Errorf("%s: read retained quarantine: %w", p, readErr)
}
if !bytes.Equal(retainedData, before) {
return nil, nil, fmt.Errorf("%s: quarantined content differs from original", p)
}
remediation[p] = Quarantined
}
case err != nil:
return nil, nil, fmt.Errorf("%s: %w", p, err)
case bytes.Equal(data, before):
remediation[p] = Untouched
default:
remediation[p] = Cleaned
sum := sha256.Sum256(data)
sums[p] = hex.EncodeToString(sum[:])
}
}
return remediation, sums, nil
}
type quarantineEntry struct{ original, archived string }
// quarantinedPaths reads the original path from each quarantine entry's .meta
// sidecar. Only top-level entries count: pre_clean below it holds the backups
// a cleaning takes, not quarantined files.
func quarantinedPaths(dir *safepath.Dir) ([]quarantineEntry, error) {
if dir == nil {
return nil, nil
}
entries, err := dir.ReadDir()
if err != nil {
return nil, err
}
var paths []quarantineEntry
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".meta") {
continue
}
data, err := readRegularFile(dir, e.Name())
if err != nil {
return nil, err
}
var meta struct {
OriginalPath string `json:"original_path"`
}
if err := json.Unmarshal(data, &meta); err != nil {
return nil, fmt.Errorf("quarantine sidecar %s: invalid metadata JSON", e.Name())
}
if !filepath.IsAbs(meta.OriginalPath) || filepath.Clean(meta.OriginalPath) != meta.OriginalPath {
return nil, fmt.Errorf("quarantine sidecar %s has no absolute clean original path", e.Name())
}
paths = append(paths, quarantineEntry{original: meta.OriginalPath, archived: strings.TrimSuffix(e.Name(), ".meta")})
}
return paths, nil
}
package attackcorpus
import (
"errors"
"net/http"
"sync/atomic"
"testing"
)
var errOffline = errors.New("attack corpus runs offline")
type offlineTransport struct{ attempts *atomic.Int64 }
func (o offlineTransport) RoundTrip(req *http.Request) (*http.Response, error) {
o.attempts.Add(1)
if req.Body != nil {
_ = req.Body.Close()
}
return nil, errOffline
}
// Offline fails and counts every request through http.DefaultTransport, which
// every client without a transport of its own uses, until the test ends.
func Offline(t testing.TB) *atomic.Int64 {
attempts := new(atomic.Int64)
prev := http.DefaultTransport
http.DefaultTransport = offlineTransport{attempts}
t.Cleanup(func() { http.DefaultTransport = prev })
return attempts
}
package attackcorpus
import (
"fmt"
"os"
"path/filepath"
)
// Plant writes the sample's files into an account tree: root/account/<path>,
// and the @crontab file to cronDir/account. Directories in the account tree
// are set to 0755; root and cronDir are the caller's and are only created
// when missing. It returns each sample path's absolute path. It never writes
// onto an existing file or symlink, nor through a symlink that leads out of
// either tree, because a private corpus is input this harness does not control.
func (s Sample) Plant(root, cronDir, account string) (map[string]string, error) {
planted := make(map[string]string, len(s.Files))
for _, f := range s.Files {
data, err := s.Content(f)
if err != nil {
return nil, fmt.Errorf("plant %s: %w", f.Path, err)
}
base, rel := root, filepath.Join(account, f.Path)
if f.Path == "@crontab" {
base, rel = cronDir, account
}
base, err = filepath.Abs(base)
if err != nil {
return nil, fmt.Errorf("plant %s: %w", f.Path, err)
}
if err := writeNew(base, rel, data, os.FileMode(f.Mode)); err != nil {
return nil, fmt.Errorf("plant %s: %w", f.Path, err)
}
planted[f.Path] = filepath.Join(base, rel)
}
return planted, nil
}
// writeNew creates base/rel through an os.Root, which refuses symlinks that
// leave base, and O_EXCL refuses a symlink or file already at rel.
func writeNew(base, rel string, data []byte, mode os.FileMode) error {
if mode == 0 {
mode = 0o644
}
// #nosec G301 -- account trees get the directory mode a real home has.
if err := os.MkdirAll(base, 0o755); err != nil {
return err
}
r, err := os.OpenRoot(base)
if err != nil {
return err
}
defer func() { _ = r.Close() }()
dir := filepath.Dir(rel)
if err = r.MkdirAll(dir, 0o755); err != nil {
return err
}
// The umask narrows MkdirAll's mode, so set it on each account directory.
for d := dir; d != "."; d = filepath.Dir(d) {
if err = r.Chmod(d, 0o755); err != nil {
return err
}
}
f, err := r.OpenFile(rel, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)
if err != nil {
return err
}
_, err = f.Write(data)
// The umask would otherwise drop bits such as the world-write a sample sets.
if err == nil {
err = f.Chmod(mode)
}
if cerr := f.Close(); err == nil {
err = cerr
}
return err
}
// Attribute maps an absolute path to its sample path: "@crontab" for the
// account's crontab, the path relative to the account home for anything
// inside it, and "" for everything else.
func Attribute(abs, root, cronDir, account string) string {
cron, err := filepath.Abs(filepath.Join(cronDir, account))
if err != nil {
return ""
}
if abs == cron {
return "@crontab"
}
home, err := filepath.Abs(filepath.Join(root, account))
if err != nil {
return ""
}
rel, err := filepath.Rel(home, abs)
if err != nil || rel == "." || !filepath.IsLocal(rel) {
return ""
}
return filepath.ToSlash(rel)
}
package attackcorpus
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"slices"
"strings"
)
// Counts summarizes one category on one path. Samples counts every result.
// Every other kind counts results, not failures: a result counts once toward
// each kind its failures name, however many failures of that kind it has.
//
// - Detected: a malicious result with no failures on a path without a gap,
// so its expected findings arrived at their severity.
// - Gaps: a result on a path with a gap and no "gap closed:" failure.
// - Missed: a "missed:" or "severity:" failure; an expected finding absent
// and one below its minimum both leave the path short of its contract.
// - FalsePositives, ClosedGaps, RemediationMismatches, Errors: a "false
// positive:", "gap closed:", "remediation:" or "error:" failure.
type Counts struct {
Samples int `json:"samples"`
Detected int `json:"detected"`
Gaps int `json:"gaps"`
Missed int `json:"missed"`
FalsePositives int `json:"false_positives"`
ClosedGaps int `json:"closed_gaps"`
RemediationMismatches int `json:"remediation_mismatches"`
Errors int `json:"errors"`
}
// Summarize counts results per category.
func Summarize(results []Result) map[Category]Counts {
out := map[Category]Counts{}
for _, r := range results {
has := func(kinds ...string) int {
for _, f := range r.Failures {
if slices.ContainsFunc(kinds, func(k string) bool { return strings.HasPrefix(f, k+":") }) {
return 1
}
}
return 0
}
c := out[Category(r.Category)]
c.Samples++
if r.Kind == "malicious" && len(r.Failures) == 0 && r.Gap == "" {
c.Detected++
}
if r.Gap != "" {
c.Gaps += 1 - has("gap closed")
}
c.Missed += has("missed", "severity")
c.FalsePositives += has("false positive")
c.ClosedGaps += has("gap closed")
c.RemediationMismatches += has("remediation")
c.Errors += has("error")
out[Category(r.Category)] = c
}
return out
}
// WriteReport writes <dir>/<path>.json with the results and their summary.
// Nothing in a Result carries sample content; finding text stays out through
// its JSON tag.
func WriteReport(dir string, p Path, results []Result) error {
if len(results) == 0 {
return errors.New("cannot report an empty corpus run")
}
data, err := json.MarshalIndent(struct {
Path Path `json:"path"`
Results []Result `json:"results"`
Summary map[Category]Counts `json:"summary"`
}{p, results, Summarize(results)}, "", " ")
if err != nil {
return err
}
if err := os.MkdirAll(dir, 0o750); err != nil {
return err
}
return os.WriteFile(filepath.Join(dir, string(p)+".json"), append(data, '\n'), 0o600)
}
package attackdb
import (
"bufio"
"fmt"
"io"
"maps"
"net"
"net/netip"
"os"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/store"
)
// AttackType categorises observed attacks for grouping and scoring.
type AttackType string
const (
AttackBruteForce AttackType = "brute_force"
AttackWAFBlock AttackType = "waf_block"
AttackWebshell AttackType = "webshell"
AttackPhishing AttackType = "phishing"
AttackC2 AttackType = "c2"
AttackRecon AttackType = "recon"
AttackSPAM AttackType = "spam"
AttackCPanelLogin AttackType = "cpanel_login"
AttackFileUpload AttackType = "file_upload"
// AttackAuthSuccess marks an event that followed a SUCCESSFUL
// authentication. Recorded for correlation; carries no score.
AttackAuthSuccess AttackType = "auth_success"
AttackReputation AttackType = "reputation"
AttackOther AttackType = "other"
)
// attackTypeLabels is how the Web UI names each attack type.
var attackTypeLabels = map[AttackType]string{
AttackBruteForce: "Brute Force",
AttackWAFBlock: "WAF Block",
AttackWebshell: "Web Shell",
AttackPhishing: "Phishing",
AttackC2: "C2 Communication",
AttackRecon: "Reconnaissance",
AttackSPAM: "Spam",
AttackCPanelLogin: "cPanel Login",
AttackFileUpload: "File Upload",
AttackAuthSuccess: "Authenticated Activity",
AttackReputation: "Known Malicious IP",
AttackOther: "Other",
}
// AttackTypeLabels returns the display label of every attack type, keyed
// by the type's string value.
func AttackTypeLabels() map[string]string {
out := make(map[string]string, len(attackTypeLabels))
for typ, label := range attackTypeLabels {
out[string(typ)] = label
}
return out
}
// checkToAttack maps alert.Finding.Check values to attack types.
var checkToAttack = map[string]AttackType{
// Brute force
"wp_login_bruteforce": AttackBruteForce,
"xmlrpc_abuse": AttackBruteForce,
"ftp_bruteforce": AttackBruteForce,
"ssh_login_unknown_ip": AttackBruteForce,
"webmail_bruteforce": AttackBruteForce,
"api_auth_failure": AttackBruteForce,
"api_auth_failure_realtime": AttackBruteForce,
"ftp_auth_failure_realtime": AttackBruteForce,
"credential_stuffing": AttackBruteForce,
"pam_bruteforce": AttackBruteForce,
"smtp_bruteforce": AttackBruteForce,
"smtp_probe_abuse": AttackBruteForce,
"smtp_subnet_spray": AttackBruteForce,
"mail_bruteforce": AttackBruteForce,
"mail_subnet_spray": AttackBruteForce,
"mail_account_compromised": AttackBruteForce,
"stalwart_ban": AttackBruteForce,
"admin_panel_bruteforce": AttackBruteForce,
// File, process, outbound-connection and mail-volume findings carry no
// attacker source address (their producers set no SourceIP), so they
// are not mapped: records are keyed on the structured source only.
// Recon
"wp_user_enumeration": AttackRecon,
// HTTP-layer abuse from a single source IP: URL enumeration, request
// floods, and scanner UA spoofing. Recorded so repeat offenders build
// local reputation; recon-classed (volume-scored, no brute-force bonus).
// "http_distributed_flood" is intentionally excluded: it is an aggregate
// per-vhost finding with no single source IP, so RecordFinding has nothing
// to attribute it to.
"http_scanner_profile": AttackRecon,
"http_request_flood": AttackRecon,
"http_claimed_bot_unverified": AttackRecon,
"http_ua_spoof": AttackRecon,
// WAF: no emitted check maps here today. The two names this table once
// listed were never emitted by any release, so WAF blocks have never
// built local reputation through this database; mapping the real
// ModSecurity block names is a scoring decision recorded in the roadmap.
// cPanel/webmail login
// Successful, post-authentication events. They are RECORDED, because they
// are evidence when correlated with other findings on the same account,
// but they carry no attack weight: scoring them made an account owner an
// attacker for using cPanel, FTP or File Manager normally. One successful
// File Manager upload alone added 20 points that never decayed, and the
// resulting score fed the reputation path that kept re-blocking the owner.
"cpanel_login": AttackAuthSuccess,
"cpanel_login_realtime": AttackAuthSuccess,
"webmail_login_realtime": AttackAuthSuccess,
"ftp_login": AttackAuthSuccess,
"pam_login": AttackAuthSuccess,
"cpanel_file_upload_realtime": AttackAuthSuccess,
// cpanel_multi_ip_login names its addresses only in Details, so it has no
// source to key on.
// Reputation - known malicious IPs from threat database
"ip_reputation": AttackReputation,
// NOTE: "local_threat_score" is intentionally excluded - it is a derived
// finding, not a raw attack. Recording it would create a feedback loop
// that inflates EventCount by +1 every 10-minute cycle.
}
// MappedChecks lists every check name the attack database records, sorted.
// The slice is the caller's own copy.
func MappedChecks() []string {
out := make([]string, 0, len(checkToAttack))
for name := range checkToAttack {
out = append(out, name)
}
sort.Strings(out)
return out
}
// AttackTypeFor reports the attack type a check name records under, and
// whether the name is mapped at all.
func AttackTypeFor(check string) (AttackType, bool) {
kind, ok := checkToAttack[config.CanonicalCheckName(check)]
return kind, ok
}
// Event is a single observed attack incident.
type Event struct {
Timestamp time.Time `json:"ts"`
IP string `json:"ip"`
AttackType AttackType `json:"type"`
CheckName string `json:"check"`
Severity int `json:"sev"`
Account string `json:"account,omitempty"`
Message string `json:"msg,omitempty"`
}
// IPRecord is the per-IP aggregated intelligence record.
type IPRecord struct {
IP string `json:"ip"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
EventCount int `json:"event_count"`
AttackCounts map[AttackType]int `json:"attack_counts"`
Accounts map[string]int `json:"accounts"`
AuthSuccessAccounts map[string]int `json:"auth_success_accounts,omitempty"`
ThreatScore int `json:"threat_score"`
AutoBlocked bool `json:"auto_blocked"`
Mail *MailEvidence `json:"mail,omitempty"`
// SpamNetwork names the /64 record a lookup took the spam evidence from.
// Only LookupIP sets it; it is never stored.
SpamNetwork string `json:"spam_network,omitempty"`
}
// DB is the in-memory attack database backed by JSON files.
type DB struct {
flushMu sync.Mutex
mu sync.RWMutex
records map[string]*IPRecord
deletedIPs map[string]struct{}
dirtyIPs map[string]struct{}
pendingEvents []Event
eventHealthOnce sync.Once
eventQueue *eventQueue
openEvents func(string) (io.WriteCloser, error)
recordHealthOnce sync.Once
recordQueue *recordQueue
saveRecord func(*store.IPRecordTx, store.IPRecord) error
deleteRecord func(*store.IPRecordTx, string) error
updateRecords func(*store.DB, func(*store.IPRecordTx) error) error
writeRecords func(string, []byte) error
dbPath string
dirty bool
stopCh chan struct{}
wg sync.WaitGroup
// Spam-evidence budget, under mu. mailOnlyCount is never below the number
// of mail-only records. It also counts records with temporary login
// protection; mailAuthTimes indexes their clocks. Operator removals leave
// both counts too high until a pass, a load or a prune recounts.
mailBudget int
mailAuthTimes []time.Time
mailOnlyCount int
mailEvicted int
mailEvictPending int // evicted since the last report
mailEvictReport time.Time // when an eviction was last reported
mailEvictNetwork netip.Prefix // most crowded covering network when the last evicting pass began
mailEvictCrowd int // mail-only records it held
}
// markDirtyLocked records that ip's record changed and must be persisted on the
// next flush. The caller must hold db.mu. dirty stays as the flush gate so
// Flush keeps its existing "anything to write?" check.
func (db *DB) markDirtyLocked(ip string) {
if db.dirtyIPs == nil {
db.dirtyIPs = make(map[string]struct{})
}
db.dirtyIPs[ip] = struct{}{}
db.dirty = true
db.queueRecordLocked(ip)
}
func (db *DB) markDeletedLocked(ip string) {
if db.deletedIPs == nil {
db.deletedIPs = make(map[string]struct{})
}
db.deletedIPs[ip] = struct{}{}
delete(db.dirtyIPs, ip)
db.dirty = true
db.queueRecordLocked(ip)
}
var (
globalDB *DB
globalMu sync.Mutex
dbInitOnce sync.Once
)
// Init initializes the global attack database.
func Init(statePath string) *DB {
dbInitOnce.Do(func() {
dbPath := statePath + "/attack_db"
_ = os.MkdirAll(dbPath, 0700)
db := &DB{
records: make(map[string]*IPRecord),
deletedIPs: make(map[string]struct{}),
dbPath: dbPath,
stopCh: make(chan struct{}),
}
db.SetMailBudget(DefaultMailBudget)
db.load()
db.pruneExpired()
// Background saver - flush dirty records every 30 seconds
db.wg.Add(1)
go db.backgroundSaver()
globalDB = db
})
return globalDB
}
// SeedFromPermanentBlocklist imports IPs from the threat DB permanent blocklist
// into the attack database. These are IPs that already attacked and were auto-blocked.
// Only imports IPs not already in the attack DB.
func (db *DB) SeedFromPermanentBlocklist(statePath string) int {
path := statePath + "/threat_db/permanent.txt"
// #nosec G304 -- fixed filename under operator-configured statePath.
f, err := os.Open(path)
if err != nil {
return 0
}
defer func() { _ = f.Close() }()
imported := 0
now := time.Now()
scanner := bufio.NewScanner(f)
db.mu.Lock()
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
ip := fields[0]
if net.ParseIP(ip) == nil {
continue
}
if _, exists := db.records[ip]; exists {
continue // already tracked
}
// Extract reason from comment: "1.2.3.4 # reason [date]"
reason := "auto-blocked (historical)"
if idx := strings.Index(line, "# "); idx > 0 {
reason = strings.TrimSpace(line[idx+2:])
}
db.records[ip] = &IPRecord{
IP: ip,
FirstSeen: now,
LastSeen: now,
EventCount: 1,
AttackCounts: map[AttackType]int{AttackOther: 1},
Accounts: make(map[string]int),
AutoBlocked: true,
}
db.records[ip].ThreatScore = ComputeScore(db.records[ip])
db.queueEventLocked(Event{
Timestamp: now,
IP: ip,
AttackType: AttackOther,
CheckName: "permanent_blocklist_import",
Severity: 2,
Message: truncate(reason, 200),
})
delete(db.deletedIPs, ip)
db.markDirtyLocked(ip)
imported++
}
db.mu.Unlock()
return imported
}
// Global returns the global attack database instance.
func Global() *DB {
globalMu.Lock()
defer globalMu.Unlock()
return globalDB
}
// SetGlobal overrides the global attack database. Mirrors
// store.SetGlobal: production wires globalDB exactly once via Init;
// tests use this to install a pre-seeded DB without touching the
// sync.Once-guarded Init path.
func SetGlobal(db *DB) {
globalMu.Lock()
globalDB = db
globalMu.Unlock()
}
// NewForTest builds a bare in-memory DB pre-populated with the given
// records. No backgroundSaver is started (no goroutines to clean up),
// no disk path is configured. Reserved for unit tests; production
// wiring stays on Init. Records are deep-copied so a later mutation
// to the caller's map (including its nested AttackCounts / Accounts
// maps) cannot bleed into the DB.
func NewForTest(records map[string]*IPRecord) *DB {
db := &DB{
records: make(map[string]*IPRecord, len(records)),
deletedIPs: make(map[string]struct{}),
stopCh: make(chan struct{}),
}
for k, v := range records {
db.records[k] = cloneIPRecord(v)
}
db.recountMailOnlyLocked()
return db
}
// RecordFinding records an attack event from a finding.
// Fire-and-forget: never blocks, never panics.
func (db *DB) RecordFinding(f alert.Finding) {
attackType, ok := AttackTypeFor(f.Check)
if !ok {
return // not an attack-related check
}
ip := extractFindingIP(f)
if ip == "" {
return
}
account := extractFindingAccount(f)
// Attribute the original finding, then redact before truncation can remove
// the service tag or other context needed to recognize a credential.
f = alert.SanitizeFinding(f)
event := Event{
Timestamp: f.Timestamp,
IP: ip,
AttackType: attackType,
CheckName: f.Check,
Severity: int(f.Severity),
Account: account,
Message: truncate(f.Message, 200),
}
now := f.Timestamp
if now.IsZero() {
now = time.Now()
}
db.mu.Lock()
rec, exists := db.records[ip]
if !exists {
rec = &IPRecord{
IP: ip,
FirstSeen: now,
AttackCounts: make(map[AttackType]int),
Accounts: make(map[string]int),
}
db.records[ip] = rec
}
// An event takes a record out of the mail-only records for good.
db.uncountMailLocked(rec)
rec.LastSeen = now
rec.EventCount++
rec.AttackCounts[attackType]++
recordMailAuthLocked(rec, f.Check, now)
if account != "" {
rec.Accounts[account]++
if attackType == AttackAuthSuccess {
if rec.AuthSuccessAccounts == nil {
rec.AuthSuccessAccounts = make(map[string]int)
}
rec.AuthSuccessAccounts[account]++
}
}
rec.ThreatScore = ComputeScore(rec)
db.queueEventLocked(event)
delete(db.deletedIPs, ip)
db.markDirtyLocked(ip)
db.mu.Unlock()
}
// MarkBlocked sets the auto-blocked flag on an IP record.
func (db *DB) MarkBlocked(ip string) {
db.mu.Lock()
if rec, ok := db.records[ip]; ok && !rec.AutoBlocked {
db.uncountMailLocked(rec)
rec.AutoBlocked = true
rec.ThreatScore = ComputeScore(rec)
delete(db.deletedIPs, ip)
db.markDirtyLocked(ip)
}
db.mu.Unlock()
}
// LookupIP returns a copy of the record stored under ip, or nil if there is
// none. An IPv6 address also reads the spam evidence of its /64 record: the
// result carries whichever of the two scores higher now, the network's on a
// tie, and SpamNetwork names the network when its evidence was used.
func (db *DB) LookupIP(ip string) *IPRecord {
db.mu.RLock()
defer db.mu.RUnlock()
var out *IPRecord
if rec, ok := db.records[ip]; ok {
out = cloneIPRecord(rec)
}
key := mailRecordKey(ip)
if key == ip {
return out
}
network, ok := db.records[key]
if !ok || network.Mail == nil {
return out
}
if out == nil {
out = &IPRecord{
IP: ip, FirstSeen: network.FirstSeen, LastSeen: network.LastSeen,
AttackCounts: make(map[AttackType]int), Accounts: make(map[string]int),
}
} else if now := time.Now(); SpamScore(network.Mail, now) < SpamScore(out.Mail, now) {
// Evidence an earlier release stored for this one address is stronger.
return out
}
// Only the spam fields move: login attacks are counted per address.
m := out.mail()
authAttacks, lastAuthAttack := m.AuthAttacks, m.LastAuthAttack
*m = *network.Mail
m.AuthAttacks, m.LastAuthAttack = authAttacks, lastAuthAttack
out.SpamNetwork = key
return out
}
// TopAttackers returns the top N IPs by threat score.
func (db *DB) TopAttackers(n int) []*IPRecord {
db.mu.RLock()
defer db.mu.RUnlock()
all := make([]*IPRecord, 0, len(db.records))
for _, rec := range db.records {
all = append(all, cloneIPRecord(rec))
}
// Sort by threat score descending, then event count
sortRecords(all)
if n > len(all) {
n = len(all)
}
return all[:n]
}
// Flush saves all pending data to disk. Called on daemon shutdown.
func (db *DB) Flush() error {
// Keep snapshots and disk writes in the same order. A command can flush
// alongside the background saver; an older write must not undo its delete.
db.flushMu.Lock()
defer db.flushMu.Unlock()
db.mu.Lock()
events := db.pendingEvents
eventBatch := db.eventHealth().detach()
db.pendingEvents = nil
dirty := db.dirty
db.dirty = false
db.mu.Unlock()
if len(events) > 0 {
db.appendEvents(events, eventBatch)
}
if dirty {
db.saveRecords()
}
return nil
}
// Stop stops the background saver and flushes.
func (db *DB) Stop() {
close(db.stopCh)
db.wg.Wait()
_ = db.Flush()
}
func (db *DB) backgroundSaver() {
defer db.wg.Done()
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case <-db.stopCh:
return
case <-ticker.C:
_ = db.Flush()
}
}
}
// extractFindingIP is the attacker address of a finding: its structured
// source only. Message text can name a connection's destination, a process
// name or a file path an attacker chose, none of which attacked the host.
func extractFindingIP(f alert.Finding) string {
return normalizeRecordIP(f.SourceIP)
}
func normalizeRecordIP(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
if host, _, err := net.SplitHostPort(raw); err == nil {
raw = host
}
raw = strings.Trim(raw, "[]")
ip := net.ParseIP(raw)
if ip == nil {
return ""
}
return ip.String()
}
func extractFindingAccount(f alert.Finding) string {
// SSH tenants already reflect the authenticated hosting-account lookup.
// Raw log details can contain a client-chosen certificate ID.
if config.CanonicalCheckName(f.Check) == "ssh_login_unknown_ip" {
return strings.TrimSpace(f.TenantID)
}
mailbox := strings.TrimSpace(f.Mailbox)
domain := strings.TrimSpace(f.Domain)
if mailbox != "" {
if strings.Contains(mailbox, "@") || domain == "" {
return mailbox
}
return mailbox + "@" + strings.ToLower(domain)
}
if tenant := strings.TrimSpace(f.TenantID); tenant != "" {
return tenant
}
return extractAccount(f.Message, f.Details)
}
// extractAccount tries to pull a cPanel account name from message or details.
func extractAccount(message, details string) string {
// Check details first: "Account: username"
for _, text := range []string{details, message} {
if idx := strings.Index(text, "Account: "); idx >= 0 {
rest := text[idx+9:]
fields := strings.Fields(rest)
if len(fields) > 0 {
return fields[0]
}
}
}
// Try /home/username/ pattern
if idx := strings.Index(message, "/home/"); idx >= 0 {
rest := message[idx+6:]
if slash := strings.Index(rest, "/"); slash > 0 {
return rest[:slash]
}
}
return ""
}
func truncate(s string, n int) string {
r := []rune(s)
if len(r) <= n {
return s
}
return string(r[:n])
}
// RemoveIP removes an IP from the attack database entirely.
func (db *DB) RemoveIP(ip string) {
db.mu.Lock()
delete(db.records, ip)
db.markDeletedLocked(ip)
db.mu.Unlock()
}
// ForgetIP atomically removes all scoring records for a parsed IP, including
// legacy imports stored under equivalent spellings. Event history is retained.
// The returned records are detached, so later findings cannot change them.
// An IPv6 address also takes its /64 spam record: that evidence cannot be
// split per address.
func (db *DB) ForgetIP(ip net.IP) []*IPRecord {
// Empty IP slices also equal a failed parse of a network key.
if ip.To16() == nil {
return nil
}
network := mailRecordKey(ip.String())
db.mu.Lock()
defer db.mu.Unlock()
var removed []*IPRecord
for key, rec := range db.records {
if key == network || ip.Equal(net.ParseIP(key)) {
removed = append(removed, rec)
delete(db.records, key)
db.markDeletedLocked(key)
}
}
return removed
}
// mailMassFloor is the decayed mass below which mail evidence counts as gone.
const mailMassFloor = 0.01
// PruneExpired removes records older than 90 days, and records kept only for
// spam evidence once that evidence has decayed away.
func (db *DB) PruneExpired() {
db.pruneExpired()
}
func (db *DB) pruneExpired() {
db.pruneExpiredAt(time.Now())
}
func (db *DB) pruneExpiredAt(now time.Time) {
cutoff := now.Add(-90 * 24 * time.Hour)
db.mu.Lock()
for ip, rec := range db.records {
expired := rec.LastSeen.Before(cutoff)
if !expired && mailOnly(rec, now) {
// Match scoring's missing-clock behavior without changing stored evidence.
m := *rec.Mail
decayTo(&m, now)
// Ham holds the record too: it keeps later spam from listing the sender.
expired = m.SpamMass < mailMassFloor && m.HamMass < mailMassFloor
}
if expired {
delete(db.records, ip)
db.markDeletedLocked(ip)
}
}
db.recountMailOnlyLocked()
db.mu.Unlock()
}
// TotalIPs returns the number of tracked IPs.
func (db *DB) TotalIPs() int {
db.mu.RLock()
defer db.mu.RUnlock()
return len(db.records)
}
// AllRecords returns a deep-copy snapshot of all records.
func (db *DB) AllRecords() []*IPRecord {
db.mu.RLock()
defer db.mu.RUnlock()
result := make([]*IPRecord, 0, len(db.records))
for _, rec := range db.records {
result = append(result, cloneIPRecord(rec))
}
return result
}
// FormatTopLine returns a summary string for stderr logging.
func (db *DB) FormatTopLine() string {
db.mu.RLock()
defer db.mu.RUnlock()
now := time.Now()
cutoff24h := now.Add(-24 * time.Hour)
total := len(db.records)
blocked, mailOnlyIPs, mailOnlyNew := 0, 0, 0
for _, r := range db.records {
if r.AutoBlocked {
blocked++
}
if mailOnly(r, now) {
mailOnlyIPs++
if r.FirstSeen.After(cutoff24h) {
mailOnlyNew++
}
}
}
return fmt.Sprintf("%d IPs tracked, %d auto-blocked, %d spam evidence only (%d new in 24h)", total, blocked, mailOnlyIPs, mailOnlyNew)
}
// Snapshots must detach all count maps from concurrent recording.
func cloneIPRecord(rec *IPRecord) *IPRecord {
cp := *rec
cp.AttackCounts = maps.Clone(rec.AttackCounts)
cp.Accounts = maps.Clone(rec.Accounts)
if cp.AttackCounts == nil {
cp.AttackCounts = make(map[AttackType]int)
}
if cp.Accounts == nil {
cp.Accounts = make(map[string]int)
}
cp.AuthSuccessAccounts = maps.Clone(rec.AuthSuccessAccounts)
if rec.Mail != nil {
m := *rec.Mail
cp.Mail = &m
}
return &cp
}
package attackdb
import (
"bytes"
"io"
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/store"
)
// Event batches are detached together under db.mu. Health keeps only their
// counts and oldest arrival, independently of state locks and persistence I/O.
type eventQueue struct {
mu sync.Mutex
waiting int
oldest time.Time
active *eventBatch
losses *queuehealth.Tracker
uncertain bool
uncertainAt time.Time
}
type eventBatch struct {
queue *eventQueue
count, offered, saved, lost int
progress time.Time
}
func (db *DB) eventHealth() *eventQueue {
db.eventHealthOnce.Do(func() { db.eventQueue = &eventQueue{losses: queuehealth.New(0, time.Minute)} })
return db.eventQueue
}
func (db *DB) queueEventLocked(event Event) {
db.pendingEvents = append(db.pendingEvents, event)
if db.dbPath == "" && store.Global() == nil {
return
}
q := db.eventHealth()
q.mu.Lock()
if q.waiting == 0 {
q.oldest = time.Now()
}
q.waiting++
q.mu.Unlock()
}
func (q *eventQueue) detach() *eventBatch {
q.mu.Lock()
defer q.mu.Unlock()
if q.waiting == 0 {
return nil
}
batch := &eventBatch{queue: q, count: q.waiting, progress: time.Now()}
q.waiting = 0
q.oldest = time.Time{}
q.active = batch
return batch
}
func (b *eventBatch) beginWrite(count int) {
if b == nil {
return
}
b.queue.mu.Lock()
b.offered = count
b.queue.mu.Unlock()
}
// Only complete records offered to the current write can have an unknown
// outcome. Everything still buffered or not yet encoded is a confirmed loss.
func (b *eventBatch) settleInterrupted() {
if b == nil {
return
}
q := b.queue
q.mu.Lock()
defer q.mu.Unlock()
count := b.count - b.saved - b.lost - b.offered
if count > 0 {
b.lost += count
q.losses.Lose(time.Now(), uint64(count))
}
if b.count > b.saved+b.lost {
q.uncertain = true
q.uncertainAt = time.Now()
}
}
func (b *eventBatch) advance(saved, lost int) {
if b == nil {
return
}
q := b.queue
q.mu.Lock()
defer q.mu.Unlock()
b.offered = 0
b.saved += saved
b.lost += lost
now := time.Now()
if lost > 0 {
q.losses.Lose(now, uint64(lost))
}
b.progress = now
}
func (b *eventBatch) uncertainOutcome() {
if b == nil {
return
}
q := b.queue
q.mu.Lock()
q.uncertain = true
q.uncertainAt = time.Now()
q.mu.Unlock()
}
func (b *eventBatch) discardRemaining() {
if b == nil {
return
}
q := b.queue
q.mu.Lock()
defer q.mu.Unlock()
b.offered = 0
remaining := b.count - b.saved - b.lost
if remaining > 0 {
q.losses.Lose(time.Now(), uint64(remaining))
b.lost += remaining
}
}
func (b *eventBatch) finish() {
if b == nil {
return
}
b.settleInterrupted()
q := b.queue
q.mu.Lock()
q.active = nil
q.mu.Unlock()
}
func (db *DB) eventQueueStatus(now time.Time) queuehealth.Status {
q := db.eventHealth()
q.mu.Lock()
defer q.mu.Unlock()
row := q.losses.Snapshot(now)
row.CapacityUnavailable = true
row.DepthUnit = "events"
row.LagBasis = "operation_progress"
row.Depth = q.waiting
row.DroppedLowerBound = q.uncertain
if q.waiting > 0 {
row.LagSeconds = max(0, now.Sub(q.oldest).Seconds())
}
if b := q.active; b != nil {
row.InFlight = b.count
row.ProcessingSeconds = max(0, now.Sub(b.progress).Seconds())
}
// An uncertain write may have lost evidence already accepted; a backlog
// has not. The record queue orders its reasons the same way.
switch {
case !q.uncertainAt.IsZero() && now.Sub(q.uncertainAt) < time.Minute:
row.Status, row.Reason = "degraded", "persistence_uncertain"
case row.LagSeconds >= time.Minute.Seconds():
row.Status, row.Reason = "degraded", "backlog_lag"
case row.ProcessingSeconds >= time.Minute.Seconds():
row.Status, row.Reason = "degraded", "processing_lag"
}
return row
}
// The encoder escapes embedded newlines. Only a complete JSONL delimiter
// accepted by the file writer counts as a persisted event; buffered bytes do not.
type eventWriter struct {
writer io.Writer
batch *eventBatch
}
func (w eventWriter) Write(p []byte) (int, error) {
w.batch.beginWrite(bytes.Count(p, []byte{'\n'}))
n, err := w.writer.Write(p)
w.batch.advance(bytes.Count(p[:n], []byte{'\n'}), 0)
return n, err
}
// An encoder error before Write confirms this event was never submitted, even
// if a later file operation exits without returning an observable byte count.
type eventEncoderWriter struct {
writer io.Writer
called bool
}
func (w *eventEncoderWriter) Write(p []byte) (int, error) {
w.called = true
return w.writer.Write(p)
}
package attackdb
import (
"net"
"net/netip"
"github.com/pidginhost/csm/internal/netutil"
)
// ForgetNetworks removes scoring records inside the supplied networks and
// returns the number removed. A network record goes when it shares any
// address with one of them. Event history is retained. Scanning once under
// the lock avoids a full database scan per address during startup cleanup.
func (db *DB) ForgetNetworks(nets []*net.IPNet) int {
if len(nets) == 0 {
return 0
}
db.mu.Lock()
defer db.mu.Unlock()
dropped := 0
for key := range db.records {
if netutil.IPInAnyNet(net.ParseIP(key), nets) || isNetworkKey(key) && networkKeyOverlaps(key, nets) {
delete(db.records, key)
db.markDeletedLocked(key)
dropped++
}
}
return dropped
}
func networkKeyOverlaps(key string, nets []*net.IPNet) bool {
network, err := netip.ParsePrefix(key)
if err != nil {
return false
}
for _, n := range nets {
addr, _ := netip.AddrFromSlice(n.IP)
addr = addr.Unmap()
// A mask that does not fit the address would read as a /0 and match all.
if ones, bits := n.Mask.Size(); bits == addr.BitLen() && network.Overlaps(netip.PrefixFrom(addr, ones)) {
return true
}
}
return false
}
package attackdb
import (
"net/netip"
"slices"
"strings"
"time"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/spamscore"
"github.com/pidginhost/csm/internal/store"
)
// MailEvidence is the attack DB record's mail section.
type MailEvidence = store.MailEvidence
const (
// SpamListScore is the spam score at which the CSM IP map lists a
// sender: two recent spam verdicts with no ham.
SpamListScore = 50
// mailAuthWindow is how long a mail login attack lists its source.
mailAuthWindow = 30 * 24 * time.Hour
)
// mailAuthChecks name one source address attacking mail logins. The spray
// findings name networks, not an address, so RecordFinding never sees them.
var mailAuthChecks = map[string]bool{
"smtp_bruteforce": true, "smtp_probe_abuse": true,
"mail_bruteforce": true, "mail_account_compromised": true,
}
func (r *IPRecord) mail() *MailEvidence {
if r.Mail == nil {
r.Mail = &MailEvidence{}
}
return r.Mail
}
// recordMailAuthLocked counts a mail login attack in rec's mail section.
// Existing scoring is unchanged; the caller holds db.mu.
func recordMailAuthLocked(rec *IPRecord, check string, at time.Time) {
if !mailAuthChecks[config.CanonicalCheckName(check)] {
return
}
m := rec.mail()
m.AuthAttacks++
if at.After(m.LastAuthAttack) {
m.LastAuthAttack = at
}
}
// mailOnly reports whether rec exists only because of spam verdicts.
func mailOnly(rec *IPRecord, now time.Time) bool {
m := rec.Mail
if m == nil || rec.EventCount != 0 || rec.AutoBlocked {
return false
}
return m.AuthAttacks == 0 || now.Sub(m.LastAuthAttack) > mailAuthWindow
}
// decayTo brings m's masses forward to at.
func decayTo(m *MailEvidence, at time.Time) {
if !m.MassAt.IsZero() && at.After(m.MassAt) {
f := spamscore.Recency(at.Sub(m.MassAt))
m.SpamMass *= f
m.HamMass *= f
}
if at.After(m.MassAt) {
m.MassAt = at
}
}
// mailRecordKey is the record a spam verdict about ip counts on: the address
// itself for IPv4, the /64 for IPv6, where one sender holds the whole
// network and can send each message from a new address.
func mailRecordKey(ip string) string {
addr, err := netip.ParseAddr(ip)
if err != nil || !addr.Unmap().Is6() {
return ip
}
return netip.PrefixFrom(addr, 64).Masked().String()
}
// isNetworkKey reports whether key names a network record. Nothing that
// parses a key as an address accepts one, so no block, whitelist or
// per-address path can act on a network.
func isNetworkKey(key string) bool {
return strings.Contains(key, "/")
}
// RecordMailVerdict adds one scan verdict for ip to the record mailRecordKey
// names. Spam creates the record; ham only updates a record that exists, so
// large legitimate senders never fill the attack DB. Neither touches
// EventCount, AttackCounts or ThreatScore, and no event is queued. Invalid
// addresses are ignored; a zero timestamp uses the current time, as
// RecordFinding does. A verdict that puts the mail-only records over the
// budget starts an eviction pass; the result is non-nil when that pass has
// evictions to report.
func (db *DB) RecordMailVerdict(ip string, spam bool, at time.Time) *Eviction {
ip = normalizeRecordIP(ip)
if ip == "" {
return nil
}
key := mailRecordKey(ip)
if at.IsZero() {
at = time.Now()
}
db.mu.Lock()
defer db.mu.Unlock()
rec, ok := db.records[key]
if !ok {
if !spam {
return nil
}
rec = &IPRecord{IP: key, FirstSeen: at, AttackCounts: make(map[AttackType]int), Accounts: make(map[string]int)}
db.records[key] = rec
}
// A stored record with no mail section joins the mail-only records too.
counted := ok && rec.Mail != nil
m := rec.mail()
decayTo(m, at)
// Delayed verdicts must have the same age as the accumulated evidence.
weight := spamscore.Recency(m.MassAt.Sub(at))
if spam {
m.Spam++
m.SpamMass += weight
if at.After(m.LastSpam) {
m.LastSpam = at
}
// Spam is activity: it keeps the 90-day prune correct. Ham is not.
if at.After(rec.LastSeen) {
rec.LastSeen = at
}
} else {
m.Ham++
m.HamMass += weight
}
delete(db.deletedIPs, key)
db.markDirtyLocked(key)
if !counted && rec.EventCount == 0 && !rec.AutoBlocked {
db.mailOnlyCount++
}
if db.mailBudget <= 0 || db.mailCountAtLocked(at) <= db.mailBudget {
return nil
}
return db.evictMailLocked(at)
}
// SpamScore is m's spam score at now, 0 to 100.
func SpamScore(m *MailEvidence, now time.Time) int {
if m == nil {
return 0
}
f := 1.0
if !m.MassAt.IsZero() && now.After(m.MassAt) {
f = spamscore.Recency(now.Sub(m.MassAt))
}
spam, ham := m.SpamMass*f, m.HamMass*f
return spamscore.Score(spam, spam+ham)
}
// MapSources lists the entries the CSM IP map takes from the attack DB,
// sorted: addresses with mail login attacks in the last 30 days, and
// addresses and IPv6 /64 networks with spam scores of SpamListScore or more.
func (db *DB) MapSources(now time.Time) (auth, spam []string) {
db.mu.RLock()
defer db.mu.RUnlock()
for ip, rec := range db.records {
m := rec.Mail
if m == nil {
continue
}
if m.AuthAttacks > 0 && now.Sub(m.LastAuthAttack) <= mailAuthWindow {
auth = append(auth, ip)
}
if SpamScore(m, now) >= SpamListScore {
spam = append(spam, ip)
}
}
slices.Sort(auth)
slices.Sort(spam)
return auth, spam
}
// TopSpamSenders returns copies of up to n records with spam verdicts,
// highest spam score first, then most spam, then record name. Non-positive n
// returns no records.
func (db *DB) TopSpamSenders(n int, now time.Time) []*IPRecord {
if n <= 0 {
return nil
}
db.mu.RLock()
var out []*IPRecord
for _, rec := range db.records {
if rec.Mail != nil && rec.Mail.Spam > 0 {
out = append(out, cloneIPRecord(rec))
}
}
db.mu.RUnlock()
slices.SortFunc(out, func(a, b *IPRecord) int {
if d := SpamScore(b.Mail, now) - SpamScore(a.Mail, now); d != 0 {
return d
}
if d := b.Mail.Spam - a.Mail.Spam; d != 0 {
return d
}
return strings.Compare(a.IP, b.IP)
})
return out[:min(n, len(out))]
}
package attackdb
import (
"bytes"
"cmp"
"math"
"net/netip"
"slices"
"sort"
"strings"
"time"
)
// mailReportEvery is the least time between two eviction reports.
const mailReportEvery = 24 * time.Hour
// DefaultMailBudget is the budget Init applies: more than ten times the
// spam-only records a busy host is projected to hold, from counts measured
// on a production cPanel host. It bounds them to about 22 MB of memory.
const DefaultMailBudget = 50000
// Eviction reports spam-evidence records dropped to keep the budget.
type Eviction struct {
Dropped int // records dropped since the last report
Network string // covering network with the most such records when the last pass began
NetworkRecords int // how many it held
}
// SetMailBudget sets how many records the database keeps only for spam
// evidence. Zero or less means no limit.
func (db *DB) SetMailBudget(n int) {
db.mu.Lock()
db.mailBudget = n
db.mu.Unlock()
}
// coveringNetwork is the network a record shares with its likely
// co-senders: the /48 around an IPv6 record, the /16 around an IPv4 one.
func coveringNetwork(key string) netip.Prefix {
var addr netip.Addr
if isNetworkKey(key) {
network, _ := netip.ParsePrefix(key)
addr = network.Addr()
} else {
addr, _ = netip.ParseAddr(key)
}
addr = addr.Unmap()
bits := 48
if addr.Is4() {
bits = 16
}
return netip.PrefixFrom(addr, bits).Masked()
}
// Count records that can become mail-only, including stored login evidence
// without events. Its protection can expire between passes, so excluding it
// here would let the running count fall below the eligible population.
func (db *DB) recountMailOnlyLocked() {
db.mailOnlyCount = 0
db.mailAuthTimes = db.mailAuthTimes[:0]
for _, rec := range db.records {
if rec.Mail == nil || rec.EventCount != 0 || rec.AutoBlocked {
continue
}
db.mailOnlyCount++
if rec.Mail.AuthAttacks > 0 {
db.mailAuthTimes = append(db.mailAuthTimes, rec.Mail.LastAuthAttack)
}
}
slices.SortFunc(db.mailAuthTimes, time.Time.Compare)
}
// Subtract still-protected records without scanning the database on each
// verdict. A sorted clock index also handles verdicts arriving out of order.
func (db *DB) mailCountAtLocked(at time.Time) int {
firstProtected := sort.Search(len(db.mailAuthTimes), func(i int) bool {
return at.Sub(db.mailAuthTimes[i]) <= mailAuthWindow
})
return db.mailOnlyCount - (len(db.mailAuthTimes) - firstProtected)
}
func (db *DB) uncountMailLocked(rec *IPRecord) {
if rec.Mail == nil || rec.EventCount != 0 || rec.AutoBlocked {
return
}
if rec.Mail.AuthAttacks > 0 {
i, found := slices.BinarySearchFunc(db.mailAuthTimes, rec.Mail.LastAuthAttack, time.Time.Compare)
if !found {
// Keep the count with the stale index entry until recounting,
// or subtracting that entry can undercount eligible records.
return
}
db.mailAuthTimes = slices.Delete(db.mailAuthTimes, i, i+1)
}
db.mailOnlyCount--
}
// mailLevel is the evidence level of m at the time its masses are decayed
// to: the mass in whole verdicts, rounded up, at least 1. Two counted
// verdicts hold level 2 however far they decayed, so senders of one message
// each never outrank a sender seen twice.
func mailLevel(m *MailEvidence) float64 {
least := 1.0
if m.Spam >= 2 || m.Ham >= 2 || m.Spam == 1 && m.Ham == 1 {
least = 2
}
return max(least, math.Ceil(m.SpamMass+m.HamMass))
}
// evictMailLocked counts the mail-only records at at and, if they are over
// the budget, drops the weakest until nine tenths of it are left, so a flood
// pays for one pass per tenth of the budget and not per message.
//
// The weakest are those of the lowest evidence level. Inside a level the
// record whose covering network holds the most mail-only records goes
// first, then the one of the lowest mass, then the smallest key. So a
// record goes only when none of a lower level is left, and among equals a
// sender spread over one network pays before anyone else. A crowd never
// outranks a level: a network with many strong senders keeps them while
// weaker records are left. A pass always trims to nine tenths, so senders
// of one message each remove no record with two or more verdicts while such
// records fill less than nine tenths of the budget; above that, one pass
// trims the weakest of them once.
//
// It returns a report at most once in mailReportEvery; PendingEviction
// hands out what a pass could not report. The caller holds db.mu.
func (db *DB) evictMailLocked(at time.Time) *Eviction {
defer db.recountMailOnlyLocked()
type candidate struct {
key string
network netip.Prefix
crowd int
level float64
mass float64
}
all := make([]candidate, 0, db.mailOnlyCount)
crowds := make(map[netip.Prefix]int)
for key, rec := range db.records {
if !mailOnly(rec, at) {
continue
}
// A copy: the pass ranks evidence and must not change what is stored.
m := *rec.Mail
decayTo(&m, at)
network := coveringNetwork(key)
crowds[network]++
all = append(all, candidate{key: key, network: network, level: mailLevel(&m), mass: m.SpamMass + m.HamMass})
}
if len(all) <= db.mailBudget {
return nil
}
for i := range all {
all[i].crowd = crowds[all[i].network]
}
slices.SortFunc(all, func(a, b candidate) int {
if c := cmp.Compare(a.level, b.level); c != 0 {
return c
}
if a.crowd != b.crowd {
return b.crowd - a.crowd
}
if c := cmp.Compare(a.mass, b.mass); c != 0 {
return c
}
return strings.Compare(a.key, b.key)
})
dropped := len(all) - db.mailBudget*9/10
for _, c := range all[:dropped] {
delete(db.records, c.key)
db.markDeletedLocked(c.key)
}
db.mailEvicted += dropped
db.mailEvictPending += dropped
// Every pass keeps its figures: a later report may be the one to carry
// them. Ties go to the smaller prefix text, compared in stack buffers to
// keep a wide sender spread from allocating one string per network.
var best, scratch [64]byte
bestLen := 0
db.mailEvictCrowd = 0
for network, n := range crowds {
text := network.AppendTo(scratch[:0])
if n > db.mailEvictCrowd || n == db.mailEvictCrowd && bytes.Compare(text, best[:bestLen]) < 0 {
bestLen = copy(best[:], text)
db.mailEvictNetwork, db.mailEvictCrowd = network, n
}
}
return db.pendingEvictionLocked(at)
}
// PendingEviction reports the drops no verdict has reported yet, once
// mailReportEvery has passed since the last report. Without it a flood that
// stops would leave its last drops unreported.
func (db *DB) PendingEviction(now time.Time) *Eviction {
db.mu.Lock()
defer db.mu.Unlock()
return db.pendingEvictionLocked(now)
}
// pendingEvictionLocked issues every report, so drops are reported once and
// all reports share one clock. The caller holds db.mu.
func (db *DB) pendingEvictionLocked(at time.Time) *Eviction {
if db.mailEvictPending == 0 || !db.mailEvictReport.IsZero() && at.Sub(db.mailEvictReport) < mailReportEvery {
return nil
}
report := &Eviction{Dropped: db.mailEvictPending, Network: db.mailEvictNetwork.String(), NetworkRecords: db.mailEvictCrowd}
db.mailEvictPending, db.mailEvictReport = 0, at
return report
}
package attackdb
import (
"bufio"
"encoding/json"
"fmt"
"io"
"maps"
"os"
"path/filepath"
"time"
"github.com/pidginhost/csm/internal/store"
)
const (
recordsFile = "records.json"
eventsFile = "events.jsonl"
maxEventsBytes = 10 * 1024 * 1024 // 10 MB
)
// load reads IP records from the bbolt store (if available) or from
// the flat-file records.json.
func (db *DB) load() {
if sdb := store.Global(); sdb != nil {
storeRecords := sdb.LoadAllIPRecords()
db.mu.Lock()
defer db.mu.Unlock()
if db.records == nil {
db.records = make(map[string]*IPRecord, len(storeRecords))
}
for ip, sr := range storeRecords {
rec := fromStoreIPRecord(sr)
changed, emptied := normalizeLoadedRecord(rec)
if emptied {
db.markDeletedLocked(ip)
continue
}
if changed || sr.NeedsRewrite {
db.markDirtyLocked(ip)
}
db.records[ip] = rec
}
db.recountMailOnlyLocked()
return
}
// No configured directory means there is nowhere to persist to. Joining
// an empty dbPath yields a relative path, which would read and write
// state in whatever directory the process was started from.
if db.dbPath == "" {
return
}
// Fallback: flat-file records.json.
path := filepath.Join(db.dbPath, recordsFile)
// #nosec G304 -- filepath.Join under operator-configured db.dbPath.
data, err := os.ReadFile(path)
if err != nil {
return
}
var storedRecords map[string]*store.IPRecord
if err := json.Unmarshal(data, &storedRecords); err != nil {
fmt.Fprintf(os.Stderr, "attackdb: error loading %s: %v\n", path, err)
return
}
db.mu.Lock()
records := make(map[string]*IPRecord, len(storedRecords))
for ip, sr := range storedRecords {
rec := fromStoreIPRecord(sr)
changed, emptied := normalizeLoadedRecord(rec)
if emptied {
db.markDeletedLocked(ip)
continue
}
if changed || sr.NeedsRewrite {
db.markDirtyLocked(ip)
}
records[ip] = rec
}
db.records = records
db.recountMailOnlyLocked()
db.mu.Unlock()
}
func fromStoreIPRecord(sr *store.IPRecord) *IPRecord {
rec := &IPRecord{
IP: sr.IP,
FirstSeen: sr.FirstSeen,
LastSeen: sr.LastSeen,
EventCount: sr.EventCount,
ThreatScore: sr.ThreatScore,
AutoBlocked: sr.AutoBlocked,
AttackCounts: make(map[AttackType]int, len(sr.AttackCounts)),
Accounts: maps.Clone(sr.Accounts),
AuthSuccessAccounts: maps.Clone(sr.AuthSuccessAccounts),
}
for k, v := range sr.AttackCounts {
rec.AttackCounts[AttackType(k)] = v
}
if sr.Mail != nil {
m := *sr.Mail
rec.Mail = &m
}
return rec
}
// textDerivedAttackTypes are attack types no producer counts against a
// structured source address. Earlier builds counted them against addresses
// read from message text (connection destinations, process names, file
// paths), so stored counts of them prove nothing about an attacker.
var textDerivedAttackTypes = []AttackType{AttackC2, AttackWebshell, AttackPhishing, AttackSPAM, AttackCPanelLogin}
// normalizeLoadedRecord repairs a stored record in place. It reports whether
// the record changed, and whether nothing but text-derived events was left
// in it, so the caller drops the record.
func normalizeLoadedRecord(rec *IPRecord) (changed, emptied bool) {
// Empty maps are an in-memory invariant; bbolt omits them, so nil-to-empty
// alone should not dirty every account-free record on each startup.
if rec.AttackCounts == nil {
rec.AttackCounts = make(map[AttackType]int)
}
if rec.Accounts == nil {
rec.Accounts = make(map[string]int)
}
dropped := 0
for _, t := range textDerivedAttackTypes {
if n := rec.AttackCounts[t]; n > 0 {
dropped += n
delete(rec.AttackCounts, t)
}
}
if dropped > 0 {
rec.EventCount = max(0, rec.EventCount-dropped)
changed = true
if rec.EventCount == 0 && rec.Mail == nil {
return true, true
}
}
score := ComputeScore(rec)
if rec.ThreatScore != score {
rec.ThreatScore = score
changed = true
}
return changed, false
}
// saveRecords writes records to the bbolt store (if available) or to
// the flat-file records.json.
func (db *DB) saveRecords() {
if sdb := store.Global(); sdb != nil {
// Incremental: only records changed since the last flush are
// re-serialized. On a host tracking tens of thousands of IPs the old
// full rewrite cost seconds of CPU on every 30s flush and on shutdown.
// Snapshot and clear the dirty set under the lock so concurrent
// mutations land in the fresh set for the next flush; a write that
// fails is re-marked dirty below so it retries.
db.mu.Lock()
dirty := db.dirtyIPs
db.dirtyIPs = make(map[string]struct{})
records := make([]store.IPRecord, 0, len(dirty))
for ip := range dirty {
rec, ok := db.records[ip]
if !ok {
continue // removed since marked; deletedIPs carries the removal
}
records = append(records, toStoreIPRecord(rec))
}
var deleted []string
for ip := range db.deletedIPs {
deleted = append(deleted, ip)
}
batch := db.detachRecordsLocked()
db.mu.Unlock()
defer batch.finish(db)
if len(records) == 0 && len(deleted) == 0 {
return
}
save := db.saveRecord
if save == nil {
save = (*store.IPRecordTx).Save
}
remove := db.deleteRecord
if remove == nil {
remove = (*store.IPRecordTx).Delete
}
update := db.updateRecords
if update == nil {
update = (*store.DB).UpdateIPRecords
}
// One transaction for the whole flush: a sync per record made a large
// flush take minutes on a slow disk. A record the store refuses fails
// alone; the rest are written when the transaction commits.
var saved, removed, failed, failedDeletes []string
refused := make(map[string]error)
batch.beginTransaction()
err := update(sdb, func(w *store.IPRecordTx) error {
for _, sr := range records {
batch.begin(sr.IP)
if err := save(w, sr); err != nil {
refused[sr.IP] = err
batch.result(sr.IP, err)
fmt.Fprintf(os.Stderr, "attackdb: store save %s: %v\n", sr.IP, err)
failed = append(failed, sr.IP)
continue
}
batch.staged()
saved = append(saved, sr.IP)
}
for _, ip := range deleted {
batch.begin(ip)
if err := remove(w, ip); err != nil {
refused[ip] = err
batch.result(ip, err)
fmt.Fprintf(os.Stderr, "attackdb: store delete %s: %v\n", ip, err)
failedDeletes = append(failedDeletes, ip)
continue
}
batch.staged()
removed = append(removed, ip)
}
return nil
})
batch.endTransaction()
if err != nil {
// Nothing was written, whether or not the transaction got as
// far as the records: every one of them retries.
fmt.Fprintf(os.Stderr, "attackdb: store commit: %v\n", err)
saved, removed, failed = nil, nil, failed[:0]
for _, sr := range records {
failed = append(failed, sr.IP)
}
failedDeletes = deleted
for _, ip := range append(failed, deleted...) {
batch.result(ip, err)
}
}
for _, ip := range append(saved, removed...) {
// A save and delete share one queue key. Either refusal must
// remain visible even if the other operation committed.
batch.result(ip, refused[ip])
}
if len(removed) > 0 {
db.mu.Lock()
for _, ip := range removed {
delete(db.deletedIPs, ip)
}
db.mu.Unlock()
}
if len(failedDeletes) > 0 {
db.mu.Lock()
db.dirty = true
db.mu.Unlock()
}
if len(failed) > 0 {
db.mu.Lock()
for _, ip := range failed {
db.markDirtyLocked(ip)
}
db.mu.Unlock()
}
return
}
// No configured directory means there is nowhere to persist to. Joining
// an empty dbPath yields a relative path, which would read and write
// state in whatever directory the process was started from.
if db.dbPath == "" {
return
}
// Fallback: flat-file records.json. The whole records map is rewritten
// each flush, so removals are reflected by absence and deletedIPs is
// redundant here -- but it must still be drained or it grows for the
// process lifetime on a host with no bbolt store. Snapshot under the same
// lock as the marshal and swap dirtyIPs before disk I/O, so mutations
// during the write land in a fresh set. Failed writes requeue the snapshot.
db.mu.Lock()
data, err := json.Marshal(db.records)
var drained []string
for ip := range db.deletedIPs {
drained = append(drained, ip)
}
flushedDirty := db.dirtyIPs
db.dirtyIPs = make(map[string]struct{})
batch := db.detachRecordsLocked()
db.mu.Unlock()
defer batch.finish(db)
if err != nil {
batch.resultAll(err)
fmt.Fprintf(os.Stderr, "attackdb: error marshaling records: %v\n", err)
db.requeueDirty(flushedDirty, len(drained) > 0)
return
}
path := filepath.Join(db.dbPath, recordsFile)
write := db.writeRecords
if write == nil {
write = writeRecordFile
}
batch.beginAll()
err = write(path, data)
batch.resultAll(err)
if err != nil {
fmt.Fprintf(os.Stderr, "attackdb: %v\n", err)
db.requeueDirty(flushedDirty, len(drained) > 0)
return
}
db.mu.Lock()
for _, ip := range drained {
delete(db.deletedIPs, ip)
}
db.mu.Unlock()
}
func (db *DB) requeueDirty(dirty map[string]struct{}, hasPendingDelete bool) {
if len(dirty) == 0 && !hasPendingDelete {
return
}
db.mu.Lock()
if hasPendingDelete {
db.dirty = true
}
for ip := range dirty {
db.markDirtyLocked(ip)
}
db.mu.Unlock()
}
// toStoreIPRecord projects an in-memory record into the store's persistence
// shape, copying the count maps so the store never aliases live maps.
func toStoreIPRecord(rec *IPRecord) store.IPRecord {
sr := store.IPRecord{
IP: rec.IP,
FirstSeen: rec.FirstSeen,
LastSeen: rec.LastSeen,
EventCount: rec.EventCount,
ThreatScore: rec.ThreatScore,
AutoBlocked: rec.AutoBlocked,
AttackCounts: make(map[string]int, len(rec.AttackCounts)),
Accounts: make(map[string]int, len(rec.Accounts)),
AuthSuccessAccounts: maps.Clone(rec.AuthSuccessAccounts),
}
for k, v := range rec.AttackCounts {
sr.AttackCounts[string(k)] = v
}
for k, v := range rec.Accounts {
sr.Accounts[k] = v
}
if rec.Mail != nil {
m := *rec.Mail
sr.Mail = &m
}
return sr
}
// appendEvents writes events to the bbolt store (if available) or appends
// to the JSONL file, rotating if needed.
func (db *DB) appendEvents(events []Event, batch *eventBatch) {
defer batch.finish()
if sdb := store.Global(); sdb != nil {
for i, ev := range events {
batch.beginWrite(1)
ts := ev.Timestamp
if ts.IsZero() {
ts = time.Now()
}
se := store.AttackEvent{
Timestamp: ts,
IP: ev.IP,
AttackType: string(ev.AttackType),
CheckName: ev.CheckName,
Severity: ev.Severity,
Account: ev.Account,
Message: ev.Message,
}
if err := sdb.RecordAttackEvent(se, i); err != nil {
batch.advance(0, 1)
fmt.Fprintf(os.Stderr, "attackdb: store event: %v\n", err)
} else {
batch.advance(1, 0)
}
}
return
}
// No configured directory means there is nowhere to persist to. Joining
// an empty dbPath yields a relative path, which would read and write
// state in whatever directory the process was started from.
if db.dbPath == "" {
batch.discardRemaining()
return
}
// Fallback: flat-file JSONL.
path := filepath.Join(db.dbPath, eventsFile)
// Check file size and rotate if needed
if info, err := os.Stat(path); err == nil && info.Size() > maxEventsBytes {
rotateEventsFile(path)
}
open := db.openEvents
if open == nil {
open = openEventsFile
}
f, err := open(path)
if err != nil {
batch.discardRemaining()
fmt.Fprintf(os.Stderr, "attackdb: error opening %s: %v\n", path, err)
return
}
defer func() {
returned := false
defer func() {
if !returned {
batch.uncertainOutcome()
}
}()
err := f.Close()
returned = true
if err != nil {
batch.uncertainOutcome()
}
}()
// Settle buffered and unencoded work before file cleanup, which may block.
// Only complete records offered to an interrupted write remain uncertain.
defer batch.settleInterrupted()
w := bufio.NewWriter(eventWriter{writer: f, batch: batch})
output := &eventEncoderWriter{writer: w}
enc := json.NewEncoder(output)
for _, ev := range events {
output.called = false
if err := enc.Encode(ev); err != nil && !output.called {
batch.advance(0, 1)
}
}
_ = w.Flush()
batch.discardRemaining()
}
// rotateEventsFile keeps the newest half of the file.
func rotateEventsFile(path string) {
// #nosec G304 -- path is filepath.Join under operator-configured db.dbPath.
data, err := os.ReadFile(path)
if err != nil {
return
}
// Find the midpoint newline
mid := len(data) / 2
for mid < len(data) {
if data[mid] == '\n' {
mid++
break
}
mid++
}
if mid >= len(data) {
return
}
tmpPath := path + ".tmp"
// #nosec G703 -- path is db.path, derived from the operator-configured
// statePath at DB open time (see Open / NewDB).
if err := os.WriteFile(tmpPath, data[mid:], 0600); err != nil {
return
}
_ = os.Rename(tmpPath, path)
}
// QueryEvents reads events for a specific IP from the bbolt store (if
// available) or from the JSONL file. Returns the most recent `limit`
// events in reverse chronological order.
func (db *DB) QueryEvents(ip string, limit int) []Event {
if sdb := store.Global(); sdb != nil {
storeEvents := sdb.QueryAttackEvents(ip, limit)
result := make([]Event, len(storeEvents))
for i, se := range storeEvents {
result[i] = Event{
Timestamp: se.Timestamp,
IP: se.IP,
AttackType: AttackType(se.AttackType),
CheckName: se.CheckName,
Severity: se.Severity,
Account: se.Account,
Message: se.Message,
}
}
return result
}
// See the note in saveRecords: an empty dbPath would resolve to a
// relative path and read an unrelated file from the working directory.
if db.dbPath == "" {
return nil
}
// Fallback: flat-file JSONL.
path := filepath.Join(db.dbPath, eventsFile)
// #nosec G304 -- filepath.Join under operator-configured db.dbPath.
f, err := os.Open(path)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
var all []Event
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 1024*1024), 1024*1024)
for scanner.Scan() {
var ev Event
if err := json.Unmarshal(scanner.Bytes(), &ev); err != nil {
continue
}
if ev.IP == ip {
all = append(all, ev)
}
}
// Return most recent first
if len(all) > limit && limit > 0 {
all = all[len(all)-limit:]
}
// Reverse
for i, j := 0, len(all)-1; i < j; i, j = i+1, j-1 {
all[i], all[j] = all[j], all[i]
}
return all
}
func openEventsFile(path string) (io.WriteCloser, error) {
// #nosec G304 -- path is eventsFile under the operator-configured db.dbPath.
return os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
}
func writeRecordFile(path string, data []byte) error {
tmpPath := path + ".tmp"
if err := os.WriteFile(tmpPath, data, 0600); err != nil {
return fmt.Errorf("error writing %s: %w", tmpPath, err)
}
if err := os.Rename(tmpPath, path); err != nil {
return fmt.Errorf("error renaming %s: %w", path, err)
}
return nil
}
package attackdb
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/store"
)
type recordPhase uint8
const (
recordQueued recordPhase = iota
recordWriting
recordSucceeded
recordFailed
)
type recordDemand struct {
arrival time.Time
failed bool
phase recordPhase
}
type recordQueue struct {
mu sync.Mutex
waiting map[string]recordDemand
active *recordBatch
losses *queuehealth.Tracker
uncertain bool
uncertainAt time.Time
}
type recordBatch struct {
queue *recordQueue
items map[string]recordDemand
progress time.Time
transactionPending bool
}
func (db *DB) recordHealth() *recordQueue {
db.recordHealthOnce.Do(func() {
db.recordQueue = &recordQueue{waiting: make(map[string]recordDemand), losses: queuehealth.New(0, time.Minute)}
})
return db.recordQueue
}
func (db *DB) queueRecordLocked(ip string) {
if db.dbPath == "" && store.Global() == nil {
return
}
q := db.recordHealth()
q.mu.Lock()
if _, exists := q.waiting[ip]; !exists {
q.waiting[ip] = recordDemand{arrival: time.Now()}
}
q.mu.Unlock()
}
// The caller detaches the actual dirty and deletion snapshot under db.mu.
// Subsequent mutations then own a separate pending generation for the same IP.
func (db *DB) detachRecordsLocked() *recordBatch {
q := db.recordHealth()
q.mu.Lock()
defer q.mu.Unlock()
if len(q.waiting) == 0 {
return nil
}
for ip, demand := range q.waiting {
demand.phase = recordQueued
q.waiting[ip] = demand
}
b := &recordBatch{queue: q, items: q.waiting, progress: time.Now()}
q.waiting = make(map[string]recordDemand)
q.active = b
return b
}
func (b *recordBatch) begin(ip string) {
if b == nil {
return
}
q := b.queue
q.mu.Lock()
if demand, exists := b.items[ip]; exists {
demand.phase = recordWriting
b.items[ip] = demand
}
q.mu.Unlock()
}
func (b *recordBatch) beginAll() {
if b == nil {
return
}
q := b.queue
q.mu.Lock()
for ip, demand := range b.items {
demand.phase = recordWriting
b.items[ip] = demand
}
q.mu.Unlock()
}
func (b *recordBatch) result(ip string, err error) {
if b == nil {
return
}
q := b.queue
q.mu.Lock()
if demand, exists := b.items[ip]; exists {
demand.failed = err != nil
demand.phase = recordSucceeded
if err != nil {
demand.phase = recordFailed
}
b.items[ip] = demand
}
b.progress = time.Now()
q.mu.Unlock()
}
// staged notes progress on a write whose outcome waits for its transaction.
func (b *recordBatch) staged() {
if b == nil {
return
}
b.queue.mu.Lock()
b.progress = time.Now()
b.queue.mu.Unlock()
}
// Track the transaction separately: it can be interrupted before any record
// starts, or after every individual operation has returned a refusal.
func (b *recordBatch) beginTransaction() {
if b == nil {
return
}
b.queue.mu.Lock()
b.transactionPending = true
b.queue.mu.Unlock()
}
func (b *recordBatch) endTransaction() {
if b == nil {
return
}
b.queue.mu.Lock()
b.transactionPending = false
b.progress = time.Now()
b.queue.mu.Unlock()
}
func (b *recordBatch) resultAll(err error) {
if b == nil {
return
}
q := b.queue
q.mu.Lock()
for ip, demand := range b.items {
demand.failed = err != nil
demand.phase = recordSucceeded
if err != nil {
demand.phase = recordFailed
}
b.items[ip] = demand
}
b.progress = time.Now()
q.mu.Unlock()
}
// Reconcile at the real requeue boundary. An older successful delete may also
// satisfy a newer delete, while an intervening record mutation must stay queued.
func (b *recordBatch) finish(db *DB) {
if b == nil {
return
}
q := b.queue
// Publish unreturned I/O before retry reconciliation can wait for db.mu.
q.mu.Lock()
if b.transactionPending {
q.uncertain = true
q.uncertainAt = time.Now()
}
for _, demand := range b.items {
if demand.phase == recordWriting {
q.uncertain = true
q.uncertainAt = time.Now()
}
}
q.mu.Unlock()
db.mu.Lock()
defer db.mu.Unlock()
q.mu.Lock()
defer q.mu.Unlock()
pending := func(ip string) bool {
_, dirty := db.dirtyIPs[ip]
_, deleted := db.deletedIPs[ip]
return dirty || deleted
}
for ip := range q.waiting {
if !pending(ip) {
delete(q.waiting, ip)
}
}
for ip, demand := range b.items {
if !pending(ip) {
if demand.phase == recordQueued || demand.phase == recordFailed {
q.losses.Lose(time.Now(), 1)
}
continue
}
next, exists := q.waiting[ip]
if !exists || demand.failed && demand.arrival.Before(next.arrival) {
next.arrival = demand.arrival
}
next.failed = next.failed || demand.failed
q.waiting[ip] = next
}
q.active = nil
}
func (q *recordQueue) snapshot(now time.Time) queuehealth.Status {
q.mu.Lock()
defer q.mu.Unlock()
row := q.losses.Snapshot(now)
row.CapacityUnavailable = true
row.DepthUnit = "records"
row.LagBasis = "operation_progress"
row.Depth = len(q.waiting)
row.DroppedLowerBound = q.uncertain
failed := false
for _, demand := range q.waiting {
row.LagSeconds = max(row.LagSeconds, now.Sub(demand.arrival).Seconds())
failed = failed || demand.failed
}
if b := q.active; b != nil {
row.InFlight = len(b.items)
row.ProcessingSeconds = max(0, now.Sub(b.progress).Seconds())
for _, demand := range b.items {
failed = failed || demand.failed
}
}
switch {
case !q.uncertainAt.IsZero() && now.Sub(q.uncertainAt) < time.Minute:
row.Status, row.Reason = "degraded", "persistence_uncertain"
case failed:
row.Status, row.Reason = "degraded", "retry_failed"
case row.LagSeconds >= time.Minute.Seconds():
row.Status, row.Reason = "degraded", "backlog_lag"
case row.ProcessingSeconds >= time.Minute.Seconds():
row.Status, row.Reason = "degraded", "processing_lag"
}
return row
}
// QueueStatuses never acquires database state or filesystem locks.
func (db *DB) QueueStatuses(now time.Time) map[string]queuehealth.Status {
return map[string]queuehealth.Status{"events": db.eventQueueStatus(now), "records": db.recordHealth().snapshot(now)}
}
package attackdb
import "sort"
// ComputeScore returns a 0-100 local threat score from an IPRecord.
//
// Scoring logic:
// - Volume: min(attack event count * 2, 30); authenticated activity excluded
// - Attack type bonuses (non-cumulative per type)
// - Multi-account targeting: +10
// - Auto-blocked floor: 50
// - Hard cap: 100
func ComputeScore(r *IPRecord) int {
score := 0
// Audit events stay in the record and history, but cannot increase the
// volume score or turn successful access into multi-account targeting.
attackEvents := max(0, r.EventCount-r.AttackCounts[AttackAuthSuccess])
vol := attackEvents * 2
if vol > 30 {
vol = 30
}
score += vol
// Attack type bonuses
if r.AttackCounts[AttackBruteForce] > 0 {
score += 15
}
if r.AttackCounts[AttackWAFBlock] > 5 {
score += 10
}
if r.AttackCounts[AttackFileUpload] > 0 {
score += 20
}
// Count accounts with attack evidence, including accounts that also
// have successful activity from this address.
targetedAccounts := 0
for account, count := range r.Accounts {
if count > r.AuthSuccessAccounts[account] {
targetedAccounts++
}
}
if targetedAccounts > 1 && attackEvents > 0 {
score += 10
}
// Auto-blocked floor
if r.AutoBlocked && score < 50 {
score = 50
}
// Cap at 100
if score > 100 {
score = 100
}
return score
}
// sortRecords sorts by threat score descending, then event count descending.
func sortRecords(recs []*IPRecord) {
sort.Slice(recs, func(i, j int) bool {
if recs[i].ThreatScore != recs[j].ThreatScore {
return recs[i].ThreatScore > recs[j].ThreatScore
}
return recs[i].EventCount > recs[j].EventCount
})
}
package attackdb
import (
"bufio"
"encoding/json"
"os"
"path/filepath"
"sync"
"time"
"github.com/pidginhost/csm/internal/store"
)
const statsCacheTTL = 30 * time.Second
// AttackStats contains aggregate statistics for the API and dashboard.
type AttackStats struct {
TotalIPs int `json:"total_ips"`
TotalEvents int `json:"total_events"`
Last24hEvents int `json:"last_24h_events"`
Last7dEvents int `json:"last_7d_events"`
BlockedIPs int `json:"blocked_ips"`
MailOnlyIPs int `json:"mail_only_ips"` // records held only for spam verdicts
MailOnlyNew24h int `json:"mail_only_new_24h"` // of those, first seen in the last 24h
MailEvicted int `json:"mail_evicted"` // spam-evidence records evicted since the daemon started
ByType map[AttackType]int `json:"by_type"` // lifetime, aggregated from IPRecord.AttackCounts
ByType24h map[AttackType]int `json:"by_type_24h"` // last 24h, aggregated from the events log
TopAttackers []*IPRecord `json:"top_attackers"`
HourlyBuckets [24]int `json:"hourly_buckets"` // last 24h, index 0 = oldest hour
DailyBuckets [7]int `json:"daily_buckets"` // last 7 days, index 0 = oldest day
}
var (
cachedStats AttackStats
cachedStatsTime time.Time
cachedStatsMu sync.Mutex
)
// Stats returns aggregate statistics, cached for 30 seconds to avoid
// re-scanning the full events.jsonl on every API call.
func (db *DB) Stats() AttackStats {
cachedStatsMu.Lock()
if time.Since(cachedStatsTime) < statsCacheTTL {
s := cachedStats
cachedStatsMu.Unlock()
return s
}
cachedStatsMu.Unlock()
stats := db.computeStats()
cachedStatsMu.Lock()
cachedStats = stats
cachedStatsTime = time.Now()
cachedStatsMu.Unlock()
return stats
}
func (db *DB) computeStats() AttackStats {
now := time.Now()
cutoff24h := now.Add(-24 * time.Hour)
cutoff7d := now.Add(-7 * 24 * time.Hour)
db.mu.RLock()
stats := AttackStats{
TotalIPs: len(db.records),
MailEvicted: db.mailEvicted,
ByType: make(map[AttackType]int),
ByType24h: make(map[AttackType]int),
}
for _, rec := range db.records {
stats.TotalEvents += rec.EventCount
if rec.AutoBlocked {
stats.BlockedIPs++
}
if mailOnly(rec, now) {
stats.MailOnlyIPs++
if rec.FirstSeen.After(cutoff24h) {
stats.MailOnlyNew24h++
}
}
for atype, count := range rec.AttackCounts {
stats.ByType[atype] += count
}
}
db.mu.RUnlock()
// Compute time-based stats from events log
events := db.readAllEvents()
for _, ev := range events {
if ev.Timestamp.After(cutoff24h) {
stats.Last24hEvents++
// Skip events with no attack type — a malformed or legacy JSONL
// line would otherwise surface as a "" key in the JSON response.
if ev.AttackType != "" {
stats.ByType24h[ev.AttackType]++
}
hoursAgo := int(now.Sub(ev.Timestamp).Hours())
if hoursAgo >= 0 && hoursAgo < 24 {
stats.HourlyBuckets[23-hoursAgo]++
}
}
if ev.Timestamp.After(cutoff7d) {
stats.Last7dEvents++
daysAgo := int(now.Sub(ev.Timestamp).Hours() / 24)
if daysAgo >= 0 && daysAgo < 7 {
stats.DailyBuckets[6-daysAgo]++
}
}
}
stats.TopAttackers = db.TopAttackers(10)
return stats
}
// readAllEvents reads all events for stats computation.
// Uses bbolt store when available, falls back to JSONL file.
func (db *DB) readAllEvents() []Event {
if sdb := store.Global(); sdb != nil {
storeEvents := sdb.ReadAllAttackEvents()
events := make([]Event, 0, len(storeEvents))
for _, se := range storeEvents {
events = append(events, Event{
Timestamp: se.Timestamp,
IP: se.IP,
AttackType: AttackType(se.AttackType),
CheckName: se.CheckName,
Severity: se.Severity,
Account: se.Account,
Message: se.Message,
})
}
return events
}
// An unset directory must not let statistics ingest an unrelated log
// from the process's working directory.
if db.dbPath == "" {
return nil
}
// Fallback: flat-file events.jsonl
path := filepath.Join(db.dbPath, eventsFile)
// #nosec G304 -- filepath.Join under operator-configured db.dbPath.
f, err := os.Open(path)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
var events []Event
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 1024*1024), 1024*1024)
for scanner.Scan() {
var ev Event
if err := json.Unmarshal(scanner.Bytes(), &ev); err != nil {
continue
}
events = append(events, ev)
}
return events
}
package auditd
import (
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
)
type commandRunner func(name string, args ...string) error
func runCommand(name string, args ...string) error {
// #nosec G204 -- production callers supply the fixed augenrules command or
// the absolute executable returned by exec.LookPath, never user input.
return exec.Command(name, args...).Run()
}
const rulesPath = "/etc/audit/rules.d/csm.rules"
const rules = `## Continuous Security Monitor - auditd rules
# Password/auth file changes
-w /etc/shadow -p wa -k csm_shadow_change
-w /etc/passwd -p wa -k csm_passwd_change
-w /etc/group -p wa -k csm_group_change
# SSH config and keys
-w /etc/ssh/sshd_config -p wa -k csm_sshd_change
-w /root/.ssh/authorized_keys -p wa -k csm_root_ssh_keys
# WHM API tokens
-w /var/cpanel/authn/api_tokens_v2/ -p wa -k csm_whm_api_tokens
# Crontab modifications
-w /var/spool/cron/ -p wa -k csm_crontab_change
-w /etc/cron.d/ -p wa -k csm_crond_change
# Password change commands
-w /usr/bin/passwd -p x -k csm_passwd_exec
-w /usr/sbin/chpasswd -p x -k csm_chpasswd_exec
# CSM binary self-protection
-w /opt/csm/csm -p wa -k csm_binary_tamper
-w /etc/csm/csm.yaml -p wa -k csm_config_tamper
-w /opt/csm/csm.yaml -p wa -k csm_config_tamper
# Execution from suspicious locations
-a always,exit -F arch=b64 -S execve -F dir=/tmp -k csm_exec_tmp
-a always,exit -F arch=b64 -S execve -F dir=/dev/shm -k csm_exec_shm
# User account modifications
-w /usr/sbin/useradd -p x -k csm_useradd
-w /usr/sbin/usermod -p x -k csm_usermod
-w /usr/sbin/userdel -p x -k csm_userdel
# AF_ALG socket creation — CVE-2026-31431 "Copy Fail" exploit signature.
# AF_ALG (numeric family 38) is essentially never used by cPanel/PHP
# workloads, so any non-system UID hitting socket(AF_ALG, ...) is suspicious.
# Filter on uid, not auid: service-launched PHP/cPanel workers commonly have
# unset audit login UID while still running as the account user.
# Two rules — b64 covers native 64-bit binaries, b32 closes the i386
# emulation evasion path on x86_64 hosts with 32-bit compat enabled.
-a always,exit -F arch=b64 -S socket -F a0=38 -F uid>=1000 -k csm_af_alg_socket
-a always,exit -F arch=b32 -S socket -F a0=38 -F uid>=1000 -k csm_af_alg_socket
`
// renderRules drops each watch whose parent directory is missing, such as
// cPanel paths on a host without cPanel: auditctl rejects that watch and
// stops loading, which would leave every later rule inactive.
func renderRules(dirExists func(string) bool) string {
var out strings.Builder
for _, line := range strings.SplitAfter(rules, "\n") {
fields := strings.Fields(line)
if len(fields) >= 2 && fields[0] == "-w" &&
!dirExists(filepath.Dir(strings.TrimSuffix(fields[1], "/"))) {
continue
}
out.WriteString(line)
}
return out.String()
}
func hostDirExists(path string) bool {
info, err := os.Stat(path)
return err == nil && info.IsDir()
}
func hostRules() string {
return renderRules(hostDirExists)
}
func Deploy() error {
return deployRules(rulesPath, hostRules(), runCommand)
}
func deployRules(path, content string, run commandRunner) error {
// #nosec G306 -- /etc/audit/rules.d/csm.rules is read by the auditd
// tooling (augenrules) on reload; 0640 keeps world-read off.
if err := os.WriteFile(path, []byte(content), 0640); err != nil {
return err
}
return run("augenrules", "--load")
}
// EnsureDeployed compares the on-disk rules file to the rules for this host
// and re-runs Deploy if they differ. Used by the daemon at
// startup so a CSM upgrade that ships new auditd rules does not silently
// remain inactive when the package postinstall did not invoke Deploy.
//
// Returns (redeployed, err): redeployed=true when the file was updated,
// false when it already matched. err is non-nil only when an unexpected
// I/O failure occurred; a missing rules file is treated as "drift" and
// triggers Deploy.
func EnsureDeployed() (bool, error) {
current, err := os.ReadFile(rulesPath)
if err == nil && string(current) == hostRules() {
return false, nil
}
if err != nil && !os.IsNotExist(err) {
return false, err
}
if err := Deploy(); err != nil {
return false, err
}
return true, nil
}
func Remove() error {
return removeRules(rulesPath, exec.LookPath, runCommand)
}
func removeRules(path string, lookPath func(string) (string, error), run commandRunner) error {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("removing audit rules: %w", err)
}
command, err := lookPath("augenrules")
if errors.Is(err, exec.ErrNotFound) {
return nil
}
if err != nil {
return fmt.Errorf("locating augenrules: %w", err)
}
if err := run(command, "--load"); err != nil {
return fmt.Errorf("reloading audit rules: %w", err)
}
return nil
}
// Package blockdigest sends the daily alert summary of attacks CSM handled,
// and batches auto-block events into a per-country roll-up for it so
// operators learn when IPs from their customers' countries get blocked. It is
// deliberately free of config/geoip/alert/daemon imports: the daemon injects
// a country lookup, persistence and email/webhook sinks, which keeps the
// logic unit-testable without the linux build tag.
package blockdigest
import (
"strings"
"sync"
"time"
)
type Bucket string
const (
BucketCustomer Bucket = "customer"
BucketAttacker Bucket = "attacker"
)
// Record is one deduplicated auto-block observation in the current window.
// Domains and URIs are populated only for modsec-category records, from the
// daemon-injected EnrichModSec lookup, so the digest can name which customer
// sites and request paths the blocked IP was hitting.
type Record struct {
TS time.Time
IP string
Country string
Reason string
Bucket Bucket
Category string
Domains []string
URIs []string
}
// Digest is the rolled-up view drained at each interval.
type Digest struct {
Window time.Duration
Countries []string
Total int
CustomerCount int
AttackerCount int
ByCountry map[string]int
ByReason map[string]int
ByCategory map[string]int
Records []Record
}
// Options is the fully-resolved collector configuration. The daemon resolves
// countries, interval, channel sinks, and lookups before constructing.
type Options struct {
// Watch turns on the per-country block roll-up and live alerts. Without
// it Observe is a no-op and only the daily summary is sent.
Watch bool
Countries []string // effective watch set, upper-cased; empty means all
SendOn string // any | customer; live alerts only
Interval time.Duration
Live bool
LiveChannel string // empty follows enabled alert channels; summary always does
MinBlock int // watched blocks needed to show the summary section
Host string
Version string
CountriesOf func() []string
CountryOf func(ip string) string
// LoadSummary and SaveSummary persist the daily summary window so a
// restart keeps the day's counts. Either may be nil.
LoadSummary func() ([]byte, error)
SaveSummary func([]byte) error
// Location is the zone of the daily send time. Nil means time.Local.
Location *time.Location
// EnrichModSec, when set, returns the customer domains and top request
// URIs a modsec-category blocked IP was hitting. The daemon injects a
// lookup over already-parsed findings so the package stays store-free.
EnrichModSec func(ip string) (domains, uris []string)
Now func() time.Time
EmailSink func(subject, body string) error
WebhookSink func(p WebhookPayload) error
OnError func(channel string, err error)
// DeliveryEnabled only reads current policy for one event (block_live or
// daily_summary) and channel; unfinished delivery also consults it during
// cleanup. Nil means every configured sink is eligible.
DeliveryEnabled func(event, channel string) bool
}
// Collector accumulates observations and drains them into digests.
type Collector struct {
opts Options
queues *digestQueues
mu sync.Mutex
records []Record
lastLive map[string]time.Time
lastLivePruned time.Time
sum Summary
// summaryMu serializes rollover, persistence and delivery. NoteHandled
// only takes mu, so dispatch never waits for a store or notification sink.
summaryMu sync.Mutex
pending *pendingSummary
lastSent time.Time
retries map[string]summaryRetry // by channel, for pending only
saveRetry summaryRetry
}
const maxBuffered = 5000
func New(opts Options) *Collector {
if opts.Now == nil {
opts.Now = time.Now
}
if opts.CountryOf == nil {
opts.CountryOf = func(string) string { return "" }
}
if opts.Location == nil {
opts.Location = time.Local
}
opts.Countries = append([]string(nil), opts.Countries...)
now := opts.Now()
return &Collector{
opts: opts, lastLive: make(map[string]time.Time), queues: newDigestQueues(opts),
sum: newSummary(now, nextSummaryDue(now, opts.Location)),
}
}
func (c *Collector) countriesSnapshot() []string {
if c.opts.CountriesOf != nil {
return normalizeCountries(c.opts.CountriesOf())
}
return append([]string(nil), c.opts.Countries...)
}
// ResolveCountries returns the effective upper-cased watch set: configured
// wins, else trusted_countries, else empty (meaning all countries).
func ResolveCountries(configured, trusted []string) []string {
if out := normalizeCountries(configured); len(out) > 0 {
return out
}
return normalizeCountries(trusted)
}
func normalizeCountries(src []string) []string {
out := make([]string, 0, len(src))
for _, c := range src {
c = strings.ToUpper(strings.TrimSpace(c))
if c != "" {
out = append(out, c)
}
}
return out
}
// classifyBucket maps an auto-block reason to a bucket. Attacker keywords are
// checked first; anything else (including unrecognized reasons) is treated as
// likely-customer so a possible false positive is never hidden from review.
func classifyBucket(reason string) Bucket {
r := strings.ToLower(reason)
attacker := []string{
"rule escalation", "modsecurity", "brute", "mail auth", "web_attack",
"web attack", "account compromise", "command-and-control",
"user-agent spoof", "ua spoof", "bad asn", "credential stuffing",
"credential-stuffing", "credential abuse", "credential-abuse",
"credentials compromised", "waf blocking high-volume attacker",
}
for _, k := range attacker {
if strings.Contains(r, k) {
return BucketAttacker
}
}
if containsWord(r, "c2") {
return BucketAttacker
}
return BucketCustomer
}
// categoryOf groups an auto-block reason into a stable signal class so the
// digest can break blocks down by what tripped them and call out WAF pressure.
// CSM custom rules (900xxx) and OWASP/Comodo CRS escalations collapse to one
// "modsec" class. Reason strings are CSM-internal and stable.
func categoryOf(reason string) string {
r := strings.ToLower(reason)
switch {
// Netblock and permanent-block notices have no reason; the daily summary
// passes their message.
case strings.HasPrefix(r, "auto-netblock:"):
return "netblock"
case strings.HasPrefix(r, "auto-permblock:"):
return "permanent"
case strings.Contains(r, "modsecurity escalation"),
strings.Contains(r, "rule escalation"),
strings.Contains(r, "waf blocking high-volume attacker"):
return "modsec"
case strings.Contains(r, "xml-rpc"):
return "xmlrpc"
case strings.Contains(r, "wordpress login brute"):
return "wp-bruteforce"
case strings.Contains(r, "admin panel brute"):
return "admin-bruteforce"
case strings.Contains(r, "mail account compromise"),
strings.Contains(r, "compromised email account"),
strings.Contains(r, "credentials compromised"),
strings.Contains(r, "outgoing mail hold"):
return "mail-compromise"
case strings.Contains(r, "mail auth"), strings.Contains(r, "smtp brute"):
return "mail-bruteforce"
case strings.Contains(r, "smtp probe"):
return "smtp-probe"
case strings.Contains(r, "ftp brute"):
return "ftp-bruteforce"
case strings.Contains(r, "url scanner profile"):
return "http-scanner"
case strings.Contains(r, "http request flood"):
return "http-flood"
case strings.Contains(r, "user-agent spoof"),
strings.Contains(r, "ua spoof"),
strings.Contains(r, "unverified claimed bot"):
return "ua-spoof"
case strings.Contains(r, "high local threat score"):
return "local-threat"
case strings.Contains(r, "known malicious ip"),
strings.Contains(r, "threat intelligence"),
strings.Contains(r, "command-and-control"),
strings.Contains(r, "bad asn"),
containsWord(r, "c2"):
return "threat-intel"
default:
return "other"
}
}
func containsWord(s, word string) bool {
for start := 0; start < len(s); {
idx := strings.Index(s[start:], word)
if idx < 0 {
return false
}
idx += start
after := idx + len(word)
if (idx == 0 || !isWordByte(s[idx-1])) && (after == len(s) || !isWordByte(s[after])) {
return true
}
start = after
}
return false
}
func isWordByte(b byte) bool {
return b >= 'a' && b <= 'z' || b >= '0' && b <= '9' || b == '_'
}
func (c *Collector) watched(country string) bool {
countries := c.countriesSnapshot()
if len(countries) == 0 {
return true
}
if country == "" {
return false
}
for _, w := range countries {
if strings.EqualFold(country, w) {
return true
}
}
return false
}
// Observe records one auto-block. It geo-classifies, filters to the watch set,
// buckets the reason, and (when Live) may fire an immediate alert.
func (c *Collector) Observe(ip, reason string, ts time.Time) {
c.observe(ip, reason, ts, true)
}
// ObserveKnownCustomer retains a watched block for the summary. Its
// customer alert replaces the live digest notification.
func (c *Collector) ObserveKnownCustomer(ip, reason string, ts time.Time) {
c.observe(ip, reason, ts, false)
}
func (c *Collector) observe(ip, reason string, ts time.Time, notify bool) {
if !c.opts.Watch {
return
}
country := c.opts.CountryOf(ip)
if !c.watched(country) {
return
}
bucket := classifyBucket(reason)
rec := Record{TS: ts, IP: ip, Country: country, Reason: reason, Bucket: bucket, Category: categoryOf(reason)}
if rec.Category == "modsec" && c.opts.EnrichModSec != nil {
rec.Domains, rec.URIs = c.opts.EnrichModSec(ip)
}
c.mu.Lock()
if len(c.records) < maxBuffered {
c.records = append(c.records, rec)
} else {
// drop-oldest: the digest only needs the interval's worth.
c.records = append(c.records[1:], rec)
}
c.queues.admit()
c.mu.Unlock()
if notify && c.opts.Live {
c.maybeLive(rec)
}
}
// Drain pulls the current window into a Digest (deduped by IP, customer first)
// and clears the buffer.
func (c *Collector) Drain() Digest {
d, batch := c.drain()
batch.finish(true)
return d
}
func (c *Collector) drain() (Digest, *digestBatch) {
c.mu.Lock()
recs := c.records
c.records = nil
batch := c.queues.detach()
c.mu.Unlock()
handedOff := false
defer func() {
if !handedOff {
batch.finish(false)
}
}()
d := Digest{
Window: c.opts.Interval,
Countries: c.countriesSnapshot(),
ByCountry: map[string]int{},
ByReason: map[string]int{},
ByCategory: map[string]int{},
}
selected := make(map[string]Record, len(recs))
order := make([]string, 0, len(recs))
for _, r := range recs {
existing, ok := selected[r.IP]
if !ok {
selected[r.IP] = r
order = append(order, r.IP)
continue
}
if existing.Bucket != BucketCustomer && r.Bucket == BucketCustomer {
selected[r.IP] = r
}
}
var customer, attacker []Record
for _, ip := range order {
r := selected[ip]
d.Total++
d.ByCountry[r.Country]++
d.ByReason[reasonKey(r.Reason)]++
d.ByCategory[r.Category]++
if r.Bucket == BucketCustomer {
d.CustomerCount++
customer = append(customer, r)
} else {
d.AttackerCount++
attacker = append(attacker, r)
}
}
d.Records = make([]Record, 0, len(customer)+len(attacker))
d.Records = append(d.Records, customer...)
d.Records = append(d.Records, attacker...)
batch.coalesce(d.Total)
handedOff = true
return d, batch
}
// reasonKey collapses a reason to its leading phrase (before the first ':')
// so per-reason counts group "ModSecurity escalation: 5+ ..." together.
func reasonKey(reason string) string {
if i := strings.IndexByte(reason, ':'); i > 0 {
return strings.TrimSpace(reason[:i])
}
return strings.TrimSpace(reason)
}
// maybeLive fires an immediate single-record alert for a qualifying block.
// It honors send_on (customer mode only alerts on customer-risk blocks) and
// dedups per IP within one Interval so a re-blocked IP cannot spam.
func (c *Collector) maybeLive(rec Record) {
if c.opts.SendOn == "customer" && rec.Bucket != BucketCustomer {
return
}
now := c.opts.Now()
c.mu.Lock()
c.pruneLastLiveLocked(now)
if last, ok := c.lastLive[rec.IP]; ok && c.opts.Interval > 0 && now.Sub(last) < c.opts.Interval {
c.mu.Unlock()
return
}
c.lastLive[rec.IP] = now
c.mu.Unlock()
// The delivery policy is injected by the caller, so it is consulted
// outside this lock. The dedup decision above already stands.
plan := c.beginDelivery(c.liveDeliveryEnabled, nil)
defer plan.finish()
d := Digest{
Window: c.opts.Interval, Countries: c.countriesSnapshot(),
Total: 1, ByCountry: map[string]int{rec.Country: 1},
ByReason: map[string]int{reasonKey(rec.Reason): 1},
ByCategory: map[string]int{rec.Category: 1},
Records: []Record{rec},
}
if rec.Bucket == BucketCustomer {
d.CustomerCount = 1
} else {
d.AttackerCount = 1
}
c.deliver(plan, c.renderSubject(d), c.renderBody(d), c.buildPayload("block_live", d), nil)
}
func (c *Collector) liveDeliveryEnabled(channel string) bool {
if c.opts.LiveChannel != "" {
return channel == c.opts.LiveChannel
}
return c.opts.DeliveryEnabled == nil || c.opts.DeliveryEnabled("block_live", channel)
}
func (c *Collector) pruneLastLiveLocked(now time.Time) {
interval := c.opts.Interval
if interval <= 0 {
clear(c.lastLive)
c.lastLivePruned = now
return
}
if !c.lastLivePruned.IsZero() && now.Sub(c.lastLivePruned) < interval {
return
}
cutoff := now.Add(-interval)
for ip, last := range c.lastLive {
if !last.After(cutoff) {
delete(c.lastLive, ip)
}
}
c.lastLivePruned = now
}
// Run loads the saved summary window, sends it if its time has passed, then
// folds watched blocks into it on each tick and sends it at each daily send
// time. On stop it folds and saves without sending.
func (c *Collector) Run(stop <-chan struct{}, tick <-chan time.Time) {
c.queues.setStopped(false)
defer c.queues.setStopped(true)
c.load()
c.SendDue(c.opts.Now())
timer := time.NewTimer(c.untilDue())
defer timer.Stop()
for {
select {
case <-stop:
c.Flush()
return
case _, ok := <-tick:
c.SendDue(c.opts.Now())
c.Flush()
if !ok {
return
}
timer.Reset(c.untilDue())
case <-timer.C:
c.SendDue(c.opts.Now())
timer.Reset(c.untilDue())
}
}
}
// deliveryOutcome hears each destination of a delivery: attempted reports
// whether its sink ran, err how it failed. A caller with an outcome reports
// failures itself.
type deliveryOutcome func(channel string, attempted bool, err error)
// deliver sends one notification through whichever sinks are configured.
// Alert delivery is best-effort and must never block the collector or the
// auto-block path beyond the sink's own timeout; failures are reported
// through OnError.
func (c *Collector) deliver(plan *digestDeliveryPlan, subject, body string, payload WebhookPayload, outcome deliveryOutcome) {
c.deliverTo(plan.email, "email", func() error { return c.opts.EmailSink(subject, body) }, outcome)
c.deliverTo(plan.webhook, "webhook", func() error { return c.opts.WebhookSink(payload) }, outcome)
}
func (c *Collector) deliverTo(job *digestDelivery, channel string, send func() error, outcome deliveryOutcome) {
if job == nil || !job.start() {
if outcome != nil {
outcome(channel, false, nil)
}
job.finish()
return
}
job.offered = true
err := send()
job.result(err)
if outcome != nil {
outcome(channel, true, err)
} else if err != nil {
c.reportSinkError(channel, err)
}
job.finish()
}
func (c *Collector) reportSinkError(channel string, err error) {
if c.opts.OnError != nil {
c.opts.OnError(channel, err)
}
}
package blockdigest
import (
"fmt"
"net/netip"
"strings"
"time"
)
// CustomerHit is an address that logged in successfully as a customer: the
// kind of login (cpanel, webmail, smtp, imap, pop, ftp), the account or mailbox
// when the log named one, and when it was seen. IP is the address, or for
// IPv6 its /64.
type CustomerHit struct {
IP string `json:"ip"`
Kind string `json:"kind"`
Account string `json:"account,omitempty"`
At time.Time `json:"at"`
}
// CustomerBlock is one automatic block, applied or a verdict only, that hit
// known customers. Target is the blocked address or network. Hits are the
// customers it covers, the most recent first; More counts those left out.
type CustomerBlock struct {
Target string `json:"target"`
Reason string `json:"reason"`
Check string `json:"check,omitempty"`
Hits []CustomerHit `json:"customers"`
More int `json:"more,omitempty"`
}
// maxWouldBlock caps each customer block list of one daily summary: the
// would-block verdicts and the applied blocks.
const maxWouldBlock = 50
// AlertCustomerBlocks sends one email and one webhook event naming every
// block in blocks. It is the false-positive signal of automatic blocking, so
// it goes out without Watch, ignores LiveChannel, and has no cap of its own:
// a cap an attacker could fill would hide it. The daily summary lists the
// blocks too, so a failed send still reaches the operator.
func (c *Collector) AlertCustomerBlocks(blocks []CustomerBlock) {
if len(blocks) == 0 {
return
}
c.mu.Lock()
for _, b := range blocks {
c.sum.noteCustomerBlock(b)
}
c.mu.Unlock()
plan := c.beginDelivery(func(channel string) bool {
return c.opts.DeliveryEnabled == nil || c.opts.DeliveryEnabled("customer_block", channel)
}, nil)
defer plan.finish()
subject := fmt.Sprintf("[CSM] %s - %d automatic blocks hit known customers", c.opts.Host, len(blocks))
if len(blocks) == 1 {
subject = fmt.Sprintf("[CSM] %s - automatic block hit a known customer: %s", c.opts.Host, blocks[0].Target)
}
var b strings.Builder
fmt.Fprintln(&b, subject)
fmt.Fprintln(&b, "These blocks hit addresses that recently logged in successfully as a customer. The blocks stand: a customer address is never exempt. Check each for a false positive.")
for _, blk := range blocks {
fmt.Fprintln(&b)
c.writeCustomerBlock(&b, "", blk)
writeUnblockHint(&b, "", blk.Target)
}
body := b.String()
payload := WebhookPayload{Text: body, CSM: WebhookCSM{Event: "customer_block", Host: c.opts.Host, Version: c.opts.Version, CustomerBlocks: blocks}}
c.deliver(plan, subject, body, payload, nil)
}
// noteWouldBlock lists each target once. Past maxWouldBlock targets it only
// counts verdicts, so a repeat of an unlisted target counts again.
func (s *Summary) noteWouldBlock(b CustomerBlock) {
for _, w := range s.WouldBlock {
if w.Target == b.Target {
return
}
}
if len(s.WouldBlock) < maxWouldBlock {
s.WouldBlock = append(s.WouldBlock, b)
return
}
s.WouldBlockMore++
}
// noteCustomerBlock lists each applied block, since each is a block of its
// own, up to maxWouldBlock, then counts the rest.
func (s *Summary) noteCustomerBlock(b CustomerBlock) {
if len(s.CustomerBlocks) < maxWouldBlock {
s.CustomerBlocks = append(s.CustomerBlocks, b)
return
}
s.CustomerBlocksMore++
}
func (c *Collector) writeCustomerBlock(b *strings.Builder, indent string, blk CustomerBlock) {
fmt.Fprintf(b, "%sBlock: %s\n", indent, blk.Target)
fmt.Fprintf(b, "%s Reason: %s\n", indent, blk.Reason)
if blk.Check != "" {
fmt.Fprintf(b, "%s Check: %s\n", indent, blk.Check)
}
if len(blk.Hits) == 1 && blk.More == 0 {
fmt.Fprintf(b, "%s Known customer: %s\n", indent, c.customerLogin(blk.Hits[0]))
return
}
fmt.Fprintf(b, "%s %d known customers, the latest %d:\n", indent, len(blk.Hits)+blk.More, len(blk.Hits))
for _, h := range blk.Hits {
fmt.Fprintf(b, "%s %s\n", indent, c.customerLogin(h))
}
if blk.More > 0 {
fmt.Fprintf(b, "%s and %d more\n", indent, blk.More)
}
}
// writeUnblockHint names the command that lifts a block of target. An allow
// rule does not pass a blocked network, whose drop comes first, but CSM never
// blocks a range over an allowed address, so it is offered for later blocks,
// with what it grants: the address on every port.
func writeUnblockHint(b *strings.Builder, indent, target string) {
if _, err := netip.ParsePrefix(target); err != nil {
fmt.Fprintf(b, "%s Unblock: csm firewall remove %s\n", indent, target)
return
}
fmt.Fprintf(b, "%s Unblock: csm firewall remove-subnet %s\n", indent, target)
fmt.Fprintf(b, "%s Allow a customer address on every port, which keeps later range blocks off it: csm firewall allow <ip>\n", indent)
}
func (c *Collector) customerLogin(h CustomerHit) string {
login := h.Kind + " login"
if h.Account != "" {
login += " as " + h.Account
}
return fmt.Sprintf("%s %s at %s", h.IP, login, h.At.In(c.opts.Location).Format("2006-01-02 15:04 MST"))
}
package blockdigest
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
type digestQueues struct {
mu sync.Mutex
arrivals []time.Time
batches map[*digestBatch]struct{}
losses *queuehealth.Tracker
maxLag time.Duration
stopped bool
sinks map[string]*digestSinkQueue
}
type digestBatch struct {
queue *digestQueues
count int
progress time.Time
finished bool
}
type digestSinkQueue struct {
mu sync.Mutex
tracker *queuehealth.Tracker
uncertain bool
uncertainAt time.Time
}
type digestDelivery struct {
queue *digestSinkQueue
ticket queuehealth.Ticket
offered, returned, finished bool
tracked bool
// repeat marks a retry of a destination whose loss is already counted.
repeat bool
channel string
deliveryEnabled func(string) bool
}
type digestDeliveryPlan struct{ email, webhook *digestDelivery }
func (p *digestDeliveryPlan) job(channel string) *digestDelivery {
if channel == "email" {
return p.email
}
return p.webhook
}
func newDigestQueues(opts Options) *digestQueues {
q := &digestQueues{batches: make(map[*digestBatch]struct{}), losses: queuehealth.New(maxBuffered, time.Minute), maxLag: max(time.Minute, opts.Interval+time.Minute), sinks: make(map[string]*digestSinkQueue)}
if opts.EmailSink != nil {
q.sinks["email"] = &digestSinkQueue{tracker: queuehealth.New(0, time.Minute)}
}
if opts.WebhookSink != nil {
q.sinks["webhook"] = &digestSinkQueue{tracker: queuehealth.New(0, time.Minute)}
}
return q
}
// Admission and detach run under the collector lock alongside the actual
// buffer. Health never takes that lock or calls injected lookups or sinks.
func (q *digestQueues) admit() {
q.mu.Lock()
defer q.mu.Unlock()
if len(q.arrivals) == maxBuffered {
q.arrivals = q.arrivals[1:]
q.losses.Lose(time.Now(), 1)
}
q.arrivals = append(q.arrivals, time.Now())
}
func (q *digestQueues) detach() *digestBatch {
q.mu.Lock()
defer q.mu.Unlock()
b := &digestBatch{queue: q, count: len(q.arrivals), progress: time.Now()}
q.arrivals = nil
q.batches[b] = struct{}{}
return b
}
func (b *digestBatch) coalesce(count int) {
b.queue.mu.Lock()
b.count = count
b.progress = time.Now()
b.queue.mu.Unlock()
}
func (b *digestBatch) finish(completed bool) {
if b == nil {
return
}
q := b.queue
q.mu.Lock()
defer q.mu.Unlock()
if b.finished {
return
}
if !completed {
q.losses.Lose(time.Now(), uint64(b.count)) // #nosec G115 -- batch count is a slice length bounded by maxBuffered; coalescing can only reduce it.
}
b.finished = true
delete(q.batches, b)
}
func (q *digestQueues) setStopped(stopped bool) {
q.mu.Lock()
q.stopped = stopped
q.mu.Unlock()
}
// begin starts one destination's delivery. A held ticket is a notification
// already waiting from an earlier failed attempt; it is reused, not queued
// again.
func (q *digestSinkQueue) begin(channel string, enabled func(string) bool, held queuehealth.Ticket) *digestDelivery {
if q == nil {
return nil
}
d := &digestDelivery{queue: q, channel: channel, deliveryEnabled: enabled}
if held != (queuehealth.Ticket{}) {
d.ticket, d.tracked = held, true
} else if d.enabled() {
d.enqueue()
}
return d
}
func (c *Collector) beginDelivery(enabled func(string) bool, held map[string]queuehealth.Ticket) *digestDeliveryPlan {
return &digestDeliveryPlan{
email: c.queues.sinks["email"].begin("email", enabled, held["email"]),
webhook: c.queues.sinks["webhook"].begin("webhook", enabled, held["webhook"]),
}
}
func (d *digestDelivery) enabled() bool {
return d.deliveryEnabled == nil || d.deliveryEnabled(d.channel)
}
func (d *digestDelivery) enqueue() {
d.ticket = d.queue.tracker.Begin(time.Now())
d.tracked = true
}
func (d *digestDelivery) start() bool {
if !d.enabled() {
d.returned = true
return false
}
// A reload may enable the second destination while the first is running.
if !d.tracked {
d.enqueue()
}
d.ticket.Start(time.Now())
return true
}
func (d *digestDelivery) result(err error) {
d.returned = true
if err != nil && !d.repeat {
d.queue.tracker.Lose(time.Now(), 1)
}
}
func (d *digestDelivery) finish() {
if d == nil || d.finished {
return
}
if !d.returned {
if d.offered {
d.queue.mu.Lock()
d.queue.uncertain = true
d.queue.uncertainAt = time.Now()
d.queue.mu.Unlock()
} else if d.enabled() && !d.repeat {
d.queue.tracker.Lose(time.Now(), 1)
}
}
if d.tracked {
d.ticket.Finish(time.Now())
}
d.finished = true
}
func (p *digestDeliveryPlan) finish() { p.email.finish(); p.webhook.finish() }
func (q *digestQueues) recordStatus(now time.Time) queuehealth.Status {
q.mu.Lock()
defer q.mu.Unlock()
row := q.losses.Snapshot(now)
row.DepthUnit = "records"
row.LagBasis = "operation_progress"
row.Depth = len(q.arrivals)
if row.Depth > 0 {
row.LagSeconds = max(0, now.Sub(q.arrivals[0]).Seconds())
}
for b := range q.batches {
row.InFlight += b.count
row.ProcessingSeconds = max(row.ProcessingSeconds, now.Sub(b.progress).Seconds())
}
switch {
case q.stopped && row.Depth > 0:
row.Status, row.Reason = "degraded", "consumer_stopped"
case row.LagSeconds >= q.maxLag.Seconds():
row.Status, row.Reason = "degraded", "backlog_lag"
case row.ProcessingSeconds >= time.Minute.Seconds():
row.Status, row.Reason = "degraded", "processing_lag"
}
return row
}
func (q *digestSinkQueue) snapshot(now time.Time) queuehealth.Status {
q.mu.Lock()
defer q.mu.Unlock()
row := q.tracker.Snapshot(now)
row.CapacityUnavailable = true
row.DepthUnit = "notifications"
row.DroppedLowerBound = q.uncertain
if !q.uncertainAt.IsZero() && now.Sub(q.uncertainAt) < time.Minute {
row.Status, row.Reason = "degraded", "delivery_uncertain"
}
return row
}
func (c *Collector) QueueStatuses(now time.Time) map[string]queuehealth.Status {
rows := map[string]queuehealth.Status{"records": c.queues.recordStatus(now)}
for name, q := range c.queues.sinks {
rows[name] = q.snapshot(now)
}
return rows
}
package blockdigest
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
)
// WebhookPayload renders for Slack/Mattermost (Text) and programmatic
// receivers (CSM) at once.
type WebhookPayload struct {
Text string `json:"text"`
CSM WebhookCSM `json:"csm"`
}
type WebhookCSM struct {
Event string `json:"event"`
Host string `json:"host"`
Version string `json:"version"`
Window string `json:"window"`
Countries []string `json:"countries"`
Counts WebhookCounts `json:"counts"`
Blocks []WebhookBlock `json:"blocks"`
// Summary carries the counts of a daily_summary event, which is written
// with summaryCSM instead of the block digest fields.
Summary *Summary `json:"summary,omitempty"`
// CustomerBlocks carries a customer_block event, written with
// customerCSM.
CustomerBlocks []CustomerBlock `json:"-"`
}
type summaryCSM struct {
Event string `json:"event"`
Host string `json:"host"`
Version string `json:"version"`
Summary *Summary `json:"summary"`
}
// customerCSM keeps customer blocks off csm.blocks, which block_live uses
// for blocked addresses.
type customerCSM struct {
Event string `json:"event"`
Host string `json:"host"`
Version string `json:"version"`
CustomerBlocks []CustomerBlock `json:"customer_blocks"`
}
// MarshalJSON writes a daily summary or a customer block alert with only its
// own csm fields. Block events keep the block digest shape.
func (p WebhookPayload) MarshalJSON() ([]byte, error) {
switch {
case p.CSM.Summary != nil:
return json.Marshal(struct {
Text string `json:"text"`
CSM summaryCSM `json:"csm"`
}{p.Text, summaryCSM{Event: p.CSM.Event, Host: p.CSM.Host, Version: p.CSM.Version, Summary: p.CSM.Summary}})
case p.CSM.CustomerBlocks != nil:
return json.Marshal(struct {
Text string `json:"text"`
CSM customerCSM `json:"csm"`
}{p.Text, customerCSM{Event: p.CSM.Event, Host: p.CSM.Host, Version: p.CSM.Version, CustomerBlocks: p.CSM.CustomerBlocks}})
}
type blockPayload WebhookPayload
return json.Marshal(blockPayload(p))
}
type WebhookCounts struct {
Total int `json:"total"`
Customer int `json:"customer"`
Attacker int `json:"attacker"`
ByCountry map[string]int `json:"by_country"`
ByReason map[string]int `json:"by_reason"`
ByCategory map[string]int `json:"by_category"`
}
type WebhookBlock struct {
IP string `json:"ip"`
Country string `json:"country"`
Reason string `json:"reason"`
Bucket string `json:"bucket"`
Category string `json:"category"`
Domains []string `json:"domains,omitempty"`
URIs []string `json:"uris,omitempty"`
TS string `json:"ts"`
}
const maxAttackerListed = 10
func countriesLabel(countries []string) string {
if len(countries) == 0 {
return "all"
}
return strings.Join(countries, ",")
}
func (c *Collector) renderSubject(d Digest) string {
return fmt.Sprintf("[%s] %d watched-country IPs blocked (%d customer-risk) last %s",
c.opts.Host, d.Total, d.CustomerCount, d.Window)
}
func (c *Collector) renderBody(d Digest) string {
var b strings.Builder
fmt.Fprintln(&b, c.renderSubject(d))
fmt.Fprintf(&b, "Countries: %s\n", countriesLabel(d.Countries))
fmt.Fprintf(&b, "By country: %s\n", sortedCounts(d.ByCountry))
fmt.Fprintf(&b, "By category: %s\n", sortedCounts(d.ByCategory))
fmt.Fprintf(&b, "By reason: %s\n", sortedCounts(d.ByReason))
fmt.Fprintln(&b)
fmt.Fprintln(&b, "LIKELY CUSTOMER (false-positive risk -- review/unblock with: csm firewall remove <ip>):")
wrote := false
for _, r := range d.Records {
if r.Bucket != BucketCustomer {
continue
}
fmt.Fprintf(&b, " %s | %s | %s\n", r.IP, r.Country, r.Reason)
wrote = true
}
if !wrote {
fmt.Fprintln(&b, " (none)")
}
fmt.Fprintln(&b)
fmt.Fprintf(&b, "Attacker blocks (correctly blocked): %d\n", d.AttackerCount)
listed := 0
for _, r := range d.Records {
if r.Bucket != BucketAttacker {
continue
}
if listed >= maxAttackerListed {
fmt.Fprintf(&b, " ... and %d more\n", d.AttackerCount-listed)
break
}
fmt.Fprintf(&b, " %s | %s | %s\n", r.IP, r.Country, r.Reason)
listed++
}
fmt.Fprintln(&b)
modsec := d.ByCategory["modsec"]
fmt.Fprintf(&b, "ModSecurity blocks (WAF escalations): %d\n", modsec)
listed = 0
for _, r := range d.Records {
if r.Category != "modsec" {
continue
}
if listed >= maxAttackerListed {
fmt.Fprintf(&b, " ... and %d more\n", modsec-listed)
break
}
fmt.Fprintf(&b, " %s | %s | %s\n", r.IP, r.Country, r.Reason)
if len(r.Domains) > 0 {
fmt.Fprintf(&b, " targets: %s\n", strings.Join(r.Domains, ", "))
} else {
fmt.Fprintln(&b, " targets: no customer domain recorded")
}
if len(r.URIs) > 0 {
fmt.Fprintf(&b, " top URIs: %s\n", strings.Join(r.URIs, " | "))
}
listed++
}
fmt.Fprintln(&b)
fmt.Fprintln(&b, "Deep per-IP report (successful-hit counts): run the on-host CSM block report.")
return b.String()
}
func sortedCounts(m map[string]int) string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, k := range keys {
parts = append(parts, fmt.Sprintf("%s=%d", k, m[k]))
}
if len(parts) == 0 {
return "(none)"
}
return strings.Join(parts, " ")
}
func (c *Collector) buildPayload(event string, d Digest) WebhookPayload {
blocks := make([]WebhookBlock, 0, len(d.Records))
for _, r := range d.Records {
blocks = append(blocks, WebhookBlock{
IP: r.IP, Country: r.Country, Reason: r.Reason,
Bucket: string(r.Bucket), Category: r.Category,
Domains: r.Domains, URIs: r.URIs,
TS: r.TS.UTC().Format(time.RFC3339),
})
}
return WebhookPayload{
Text: c.renderBody(d),
CSM: WebhookCSM{
Event: event, Host: c.opts.Host, Version: c.opts.Version,
Window: d.Window.String(), Countries: d.Countries,
Counts: WebhookCounts{
Total: d.Total, Customer: d.CustomerCount, Attacker: d.AttackerCount,
ByCountry: d.ByCountry, ByReason: d.ByReason, ByCategory: d.ByCategory,
},
Blocks: blocks,
},
}
}
// shouldSend gates the watched-country section of the daily summary: it is
// shown when watching and the folded blocks reach MinBlock.
func (c *Collector) shouldSend(w WatchedBlocks) bool {
return c.opts.Watch && w.Total >= c.opts.MinBlock
}
func (c *Collector) summarySubject(s Summary) string {
return fmt.Sprintf("[CSM] %s - daily summary %s", c.opts.Host, s.Since.In(c.opts.Location).Format("2006-01-02"))
}
func (c *Collector) summaryBody(s Summary, until time.Time) string {
const stamp = "2006-01-02 15:04 MST"
var b strings.Builder
fmt.Fprintln(&b, c.summarySubject(s))
fmt.Fprintf(&b, "Window: %s to %s\n", s.Since.In(c.opts.Location).Format(stamp), until.In(c.opts.Location).Format(stamp))
fmt.Fprintln(&b, "Attacks CSM handled on its own. They are not alerted one by one; the findings page and history keep every one.")
fmt.Fprintln(&b)
fmt.Fprintf(&b, "Handled findings: %d\n", s.total())
fmt.Fprintf(&b, "Muted in alert email: %d\n", s.Muted)
if s.Blocks > 0 {
fmt.Fprintf(&b, "Blocks applied: %d (%s)\n", s.Blocks, sortedCounts(s.BlockKinds))
} else {
fmt.Fprintln(&b, "Blocks applied: 0")
}
fmt.Fprintf(&b, "Challenges: %d\n", s.checkTotal("challenge_route"))
fmt.Fprintf(&b, "Mail login advisories, observe only: %d\n", s.checkTotal("mail_bruteforce_suspected"))
fmt.Fprintln(&b)
fmt.Fprintln(&b, "By check and severity:")
checks := make([]string, 0, len(s.Checks))
for check := range s.Checks {
checks = append(checks, check)
}
sort.Strings(checks)
for _, check := range checks {
fmt.Fprintf(&b, " %s: %s\n", check, sortedCounts(s.Checks[check]))
}
if len(checks) == 0 {
fmt.Fprintln(&b, " (none)")
}
fmt.Fprintln(&b)
fmt.Fprintln(&b, "Top sources:")
for _, src := range s.Sources {
if src.Country != "" {
fmt.Fprintf(&b, " %s (%s): %d\n", src.IP, src.Country, src.Count)
} else {
fmt.Fprintf(&b, " %s: %d\n", src.IP, src.Count)
}
}
if len(s.Sources) == 0 {
fmt.Fprintln(&b, " (none)")
}
fmt.Fprintln(&b)
fmt.Fprintln(&b, "Automatic blocks that hit known customers:")
for _, blk := range s.CustomerBlocks {
c.writeCustomerBlock(&b, " ", blk)
writeUnblockHint(&b, " ", blk.Target)
}
switch {
case len(s.CustomerBlocks) == 0:
fmt.Fprintln(&b, " (none)")
case s.CustomerBlocksMore > 0:
fmt.Fprintf(&b, " and %d more blocks\n", s.CustomerBlocksMore)
}
fmt.Fprintln(&b)
fmt.Fprintln(&b, "Known customers a dry-run or observe-mode verdict would have blocked:")
for _, w := range s.WouldBlock {
c.writeCustomerBlock(&b, " ", w)
}
switch {
case len(s.WouldBlock) == 0:
fmt.Fprintln(&b, " (none)")
case s.WouldBlockMore > 0:
fmt.Fprintf(&b, " and %d more verdicts\n", s.WouldBlockMore)
}
if c.shouldSend(s.Watched) {
fmt.Fprintln(&b)
fmt.Fprintf(&b, "Watched-country blocks: %d (countries: %s)\n", s.Watched.Total, countriesLabel(c.countriesSnapshot()))
fmt.Fprintf(&b, " By country: %s\n", sortedCounts(s.Watched.ByCountry))
fmt.Fprintf(&b, " By category: %s\n", sortedCounts(s.Watched.ByCategory))
}
return b.String()
}
func (c *Collector) summaryPayload(s Summary, body string) WebhookPayload {
return WebhookPayload{
Text: body,
CSM: WebhookCSM{Event: "daily_summary", Host: c.opts.Host, Version: c.opts.Version, Summary: &s},
}
}
package blockdigest
import (
"cmp"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
const (
// summaryHour is the local hour the daily summary is sent at.
summaryHour = 8
// maxSummarySources caps the tracked source addresses. The top list is
// a space-saving count, so a flood of one-off addresses cannot push a
// heavy source out.
maxSummarySources = 1000
summaryTopSources = 10
// maxSummaryKeys caps each count map; later keys count under
// overflowKey. Check names, kinds and countries come from CSM itself,
// so the cap is a backstop, not a limit real traffic reaches.
maxSummaryKeys = 512
overflowKey = "other"
)
// Handled is one summary-class finding as the daemon maps it. BlockReason is
// set for an applied auto_block only. WouldBlock keeps a customer verdict
// in the same window as its finding count.
type Handled struct {
Check, Severity, IP, BlockReason string
Muted bool
WouldBlock *CustomerBlock
}
// SourceCount is one source address and its summary findings. Country is
// filled only in a sent summary's top list.
type SourceCount struct {
IP string `json:"ip"`
Count int `json:"count"`
Country string `json:"country,omitempty"`
}
// WatchedBlocks are the watched-country blocks folded into the summary.
type WatchedBlocks struct {
Total int `json:"total"`
ByCountry map[string]int `json:"by_country"`
ByCategory map[string]int `json:"by_category"`
}
// Summary is the persisted daily summary window.
type Summary struct {
Since time.Time `json:"since"`
Due time.Time `json:"due"`
Checks map[string]map[string]int `json:"checks"` // check, then severity
Blocks int `json:"blocks"`
BlockKinds map[string]int `json:"block_kinds"` // by categoryOf
Muted int `json:"muted"`
Sources []SourceCount `json:"sources"`
Watched WatchedBlocks `json:"watched"`
// WouldBlock lists dry-run and observe-mode verdicts on known
// customers, at most maxWouldBlock; WouldBlockMore counts the rest.
WouldBlock []CustomerBlock `json:"would_block,omitempty"`
WouldBlockMore int `json:"would_block_more,omitempty"`
// CustomerBlocks lists the applied blocks that hit known customers, at
// most maxWouldBlock; CustomerBlocksMore counts the rest. Each was
// alerted at once; the summary still names it when that send failed.
CustomerBlocks []CustomerBlock `json:"customer_blocks,omitempty"`
CustomerBlocksMore int `json:"customer_blocks_more,omitempty"`
}
// One pending window bounds retry state during a prolonged outage. Findings
// arriving during delivery belong to the active window, not its retries.
type pendingSummary struct {
Summary
Until time.Time `json:"until"`
EmailDone bool `json:"email_done"`
WebhookDone bool `json:"webhook_done"`
}
func (p *pendingSummary) done(channel string) bool {
if channel == "email" {
return p.EmailDone
}
return p.WebhookDone
}
func (p *pendingSummary) markDone(channel string) {
if channel == "email" {
p.EmailDone = true
} else {
p.WebhookDone = true
}
}
// summaryRetry is the backoff of one failing destination of the pending
// window, or of failing saves. It lives in memory: a restart retries at once.
// ticket keeps a failed destination one waiting notification between
// attempts, so health shows the outage.
type summaryRetry struct {
delay time.Duration
next time.Time
ticket queuehealth.Ticket
// tried means an attempt in this process already accounted for the
// destination, as a loss or an uncertain delivery, so later failures and
// a drop of the window do not count it again.
tried bool
}
// fail doubles the wait from one minute up to one hour and reports whether
// to warn: on the first failure and when the wait reaches the hour.
func (r *summaryRetry) fail(now time.Time) bool {
previous := r.delay
r.delay = min(max(summaryRetryFirst, 2*previous), summaryRetryMax)
r.next = now.Add(r.delay)
return previous == 0 || previous < summaryRetryMax && r.delay == summaryRetryMax
}
var summaryChannels = [...]string{"email", "webhook"}
const (
summaryRetryFirst = time.Minute
summaryRetryMax = time.Hour
)
type summaryState struct {
Summary
Pending *pendingSummary `json:"pending,omitempty"`
LastSent time.Time `json:"last_sent,omitempty"`
}
func newSummary(since, due time.Time) Summary {
s := Summary{Since: since, Due: due}
s.init()
return s
}
func (s *Summary) init() {
if s.Checks == nil {
s.Checks = make(map[string]map[string]int)
}
if s.BlockKinds == nil {
s.BlockKinds = make(map[string]int)
}
if s.Watched.ByCountry == nil {
s.Watched.ByCountry = make(map[string]int)
}
if s.Watched.ByCategory == nil {
s.Watched.ByCategory = make(map[string]int)
}
}
// nextSummaryDue returns the first summaryHour:00 in loc strictly after now.
// It builds wall-clock dates instead of adding 24h, so the send time stays
// at 08:00 on daylight saving days.
func nextSummaryDue(now time.Time, loc *time.Location) time.Time {
t := now.In(loc)
due := time.Date(t.Year(), t.Month(), t.Day(), summaryHour, 0, 0, 0, loc)
if !due.After(now) {
due = time.Date(t.Year(), t.Month(), t.Day()+1, summaryHour, 0, 0, 0, loc)
}
return due
}
func boundedKey[V any](m map[string]V, key string) string {
if _, ok := m[key]; ok || len(m) < maxSummaryKeys {
return key
}
return overflowKey
}
func addCounts(dst, src map[string]int) {
for k, n := range src {
dst[boundedKey(dst, k)] += n
}
}
func (s *Summary) note(h Handled) {
check := boundedKey(s.Checks, h.Check)
if s.Checks[check] == nil {
s.Checks[check] = make(map[string]int)
}
s.Checks[check][h.Severity]++
if h.Muted {
s.Muted++
}
if h.BlockReason != "" {
s.Blocks++
s.BlockKinds[boundedKey(s.BlockKinds, categoryOf(h.BlockReason))]++
}
if h.IP != "" {
s.addSource(h.IP, 1)
}
if h.WouldBlock != nil {
s.noteWouldBlock(*h.WouldBlock)
}
}
// addSource counts n findings for ip. When the list is full, the address
// with the lowest count is replaced and the newcomer inherits that count, so
// a count can overstate a newcomer but never understates a heavy source.
// ponytail: linear scan of the capped list; a heap if inserts get hot.
func (s *Summary) addSource(ip string, n int) {
lowest := 0
for i := range s.Sources {
if s.Sources[i].IP == ip {
s.Sources[i].Count += n
return
}
if s.Sources[i].Count < s.Sources[lowest].Count {
lowest = i
}
}
if len(s.Sources) < maxSummarySources {
s.Sources = append(s.Sources, SourceCount{IP: ip, Count: n})
return
}
s.Sources[lowest] = SourceCount{IP: ip, Count: s.Sources[lowest].Count + n}
}
func (w *WatchedBlocks) add(total int, byCountry, byCategory map[string]int) {
w.Total += total
addCounts(w.ByCountry, byCountry)
addCounts(w.ByCategory, byCategory)
}
// merge adds the counts of o, noted before the saved window was loaded.
func (s *Summary) merge(o Summary) {
for check, bySeverity := range o.Checks {
key := boundedKey(s.Checks, check)
if s.Checks[key] == nil {
s.Checks[key] = make(map[string]int)
}
addCounts(s.Checks[key], bySeverity)
}
s.Blocks += o.Blocks
addCounts(s.BlockKinds, o.BlockKinds)
s.Muted += o.Muted
for _, src := range o.Sources {
s.addSource(src.IP, src.Count)
}
s.Watched.add(o.Watched.Total, o.Watched.ByCountry, o.Watched.ByCategory)
for _, b := range o.WouldBlock {
s.noteWouldBlock(b)
}
s.WouldBlockMore += o.WouldBlockMore
for _, b := range o.CustomerBlocks {
s.noteCustomerBlock(b)
}
s.CustomerBlocksMore += o.CustomerBlocksMore
}
func (s *Summary) total() int {
n := 0
for _, bySeverity := range s.Checks {
for _, c := range bySeverity {
n += c
}
}
return n
}
func (s *Summary) checkTotal(check string) int {
n := 0
for _, c := range s.Checks[check] {
n += c
}
return n
}
func sortSources(src []SourceCount) {
slices.SortFunc(src, func(a, b SourceCount) int {
return cmp.Or(cmp.Compare(b.Count, a.Count), cmp.Compare(a.IP, b.IP))
})
}
// NoteHandled counts summary-class findings into the current window.
func (c *Collector) NoteHandled(batch []Handled) {
if len(batch) == 0 {
return
}
c.mu.Lock()
defer c.mu.Unlock()
for _, h := range batch {
c.sum.note(h)
}
}
// fold moves the watched-country buffer into the summary window.
func (c *Collector) fold() {
d, batch := c.drain()
c.mu.Lock()
c.sum.Watched.add(d.Total, d.ByCountry, d.ByCategory)
c.mu.Unlock()
batch.finish(true)
}
// Flush checkpoints the active and pending windows without delivering. The
// daemon calls it again after its final alert drain, once all producers stop.
func (c *Collector) Flush() {
c.summaryMu.Lock()
defer c.summaryMu.Unlock()
c.fold()
c.save()
}
// save is called with summaryMu held; no injected callback runs under mu.
// A failure warns like a failing destination: on the first one, then when
// the wait between warnings reaches an hour.
func (c *Collector) save() {
if c.opts.SaveSummary == nil {
return
}
c.mu.Lock()
b, err := json.Marshal(summaryState{Summary: c.sum, Pending: c.pending, LastSent: c.lastSent})
c.mu.Unlock()
if err == nil {
err = c.opts.SaveSummary(b)
}
if err == nil {
c.saveRetry = summaryRetry{}
return
}
if now := c.opts.Now(); !now.Before(c.saveRetry.next) && c.saveRetry.fail(now) {
c.reportSinkError("summary", err)
}
}
// clamp pulls a send time past the next one, or a start in the future, back
// to now: either means the clock went back, and the summary must not be held
// for days.
func (s *Summary) clamp(now time.Time, loc *time.Location) {
if next := nextSummaryDue(now, loc); s.Due.After(next) {
s.Due = next
}
if s.Since.After(now) {
s.Since = now
}
}
// load replaces the fresh window with the saved one and adds what was noted
// since New. Run's first SendDue then clamps a send time the clock left behind.
func (c *Collector) load() {
c.summaryMu.Lock()
defer c.summaryMu.Unlock()
if c.opts.LoadSummary == nil {
return
}
b, err := c.opts.LoadSummary()
if err == nil && len(b) == 0 {
return
}
var s summaryState
if err == nil {
err = json.Unmarshal(b, &s)
}
if err == nil && (s.Since.IsZero() || s.Due.IsZero() || s.Pending != nil && (s.Pending.Since.IsZero() || s.Pending.Until.IsZero())) {
err = errors.New("saved daily summary has no window start or send time")
}
if err != nil {
c.reportSinkError("summary", err)
return
}
s.init()
if len(s.Sources) > maxSummarySources {
sortSources(s.Sources)
s.Sources = s.Sources[:maxSummarySources]
}
c.mu.Lock()
s.merge(c.sum)
c.sum = s.Summary
c.pending = s.Pending
c.lastSent = s.LastSent
c.mu.Unlock()
}
func (c *Collector) untilDue() time.Duration {
c.summaryMu.Lock()
defer c.summaryMu.Unlock()
if c.pending != nil {
return time.Minute
}
c.mu.Lock()
due := c.sum.Due
c.mu.Unlock()
// Go timers use elapsed time. Recheck the wall clock and any pending
// delivery every minute, even when the configured fold interval is long.
return min(time.Minute, max(time.Second, due.Sub(c.opts.Now())))
}
// maxSummaryLastSentLead is how far ahead of the clock a recorded send may
// be and still hold back the dates it covers.
const maxSummaryLastSentLead = 48 * time.Hour
// SendDue checkpoints a due window before offering it to either sink. Failed
// destinations retain that window and retry with backoff; completed
// destinations are checkpointed separately. A crash after remote acceptance
// but before its checkpoint can still repeat that destination, since
// notification sinks are not transactional. The return value reports a
// delivery attempt, not whether its sinks succeeded.
func (c *Collector) SendDue(now time.Time) bool {
c.summaryMu.Lock()
defer c.summaryMu.Unlock()
c.mu.Lock()
if now.Before(c.sum.Due) {
c.sum.clamp(now, c.opts.Location)
// A send stamped more than maxSummaryLastSentLead ahead came from a
// clock fault; honouring it would hold back every morning up to it.
if c.lastSent.After(now.Add(maxSummaryLastSentLead)) {
faulty := c.lastSent
c.lastSent = time.Time{}
c.mu.Unlock()
c.reportSinkError("summary", fmt.Errorf("last summary send at %s is ahead of the clock; resuming daily summaries", faulty.Format(time.RFC3339)))
c.mu.Lock()
}
// A correction back across this morning's send must not send the
// same local date twice, including after a restart.
if !c.lastSent.IsZero() && c.sum.Due.In(c.opts.Location).Format(time.DateOnly) <= c.lastSent.In(c.opts.Location).Format(time.DateOnly) {
c.sum.Due = nextSummaryDue(c.lastSent, c.opts.Location)
}
c.mu.Unlock()
if c.pending == nil {
return false
}
c.settleDisabled()
if !c.retryDue(now) {
return false
}
} else {
c.mu.Unlock()
// One pending window bounds retries: the next one replaces it, so a
// failing destination never holds back the others.
if c.pending != nil {
c.settleDisabled()
}
c.dropPending()
c.fold()
c.mu.Lock()
c.pending = &pendingSummary{Summary: c.sum, Until: now}
c.sum = newSummary(now, nextSummaryDue(now, c.opts.Location))
c.mu.Unlock()
// Sent for this date from now on, even while a destination still
// retries, so a clock stepped back cannot send the date again.
c.lastSent = now
}
// A failed save does not hold the summary back; a crash repeats it.
c.save()
pending := c.pending
waiting := func(channel string) bool { return now.Before(c.retries[channel].next) }
// A destination that failed before keeps its waiting notification.
held := make(map[string]queuehealth.Ticket, len(summaryChannels))
for channel, r := range c.retries {
if r.ticket != (queuehealth.Ticket{}) && !waiting(channel) {
held[channel] = r.ticket
r.ticket = queuehealth.Ticket{}
c.retries[channel] = r
}
}
plan := c.beginDelivery(func(channel string) bool {
if pending.done(channel) || waiting(channel) {
return false
}
return c.opts.DeliveryEnabled == nil || c.opts.DeliveryEnabled("daily_summary", channel)
}, held)
defer plan.finish()
if c.retries == nil {
c.retries = make(map[string]summaryRetry, len(summaryChannels))
}
for _, channel := range summaryChannels {
if job := plan.job(channel); job != nil && job.tracked {
r := c.retries[channel]
job.repeat = r.tried
r.tried = true
c.retries[channel] = r
}
}
c.deliverSummary(plan, pending, func(channel string, attempted bool, err error) {
switch {
case pending.done(channel), !attempted && waiting(channel):
case err == nil:
pending.markDone(channel)
c.save()
default:
c.retryLater(channel, now, err)
// Hold the failed notification as waiting for the next attempt.
// A logger that does not return leaves it to plan.finish.
job := plan.job(channel)
job.ticket.Requeue(time.Now())
job.tracked = false
r := c.retries[channel]
r.ticket = job.ticket
// A destination enabled after the attempt began was not marked
// tried with the others; its loss is counted now.
r.tried = true
c.retries[channel] = r
}
})
if !pending.EmailDone || !pending.WebhookDone {
return true
}
c.pending = nil
c.retries = nil
c.save()
return true
}
// deliverSummary offers p to each destination of plan. A renderer, an
// injected lookup or a sink that panics fails its destinations as a returned
// error would, with the same backoff and loss accounting: a panic that
// repeats at every start must not take the daemon down each time.
func (c *Collector) deliverSummary(plan *digestDeliveryPlan, p *pendingSummary, outcome deliveryOutcome) {
subject, body, payload, err := c.renderSummary(p)
sinks := map[string]func() error{
"email": func() error { return c.opts.EmailSink(subject, body) },
"webhook": func() error { return c.opts.WebhookSink(payload) },
}
for _, channel := range summaryChannels {
c.deliverTo(plan.job(channel), channel, func() (sendErr error) {
if err != nil {
return err
}
defer func() {
if r := recover(); r != nil {
sendErr = panicError(channel+" sink", r)
}
}()
return sinks[channel]()
}, outcome)
}
}
func (c *Collector) renderSummary(p *pendingSummary) (subject, body string, payload WebhookPayload, err error) {
defer func() {
if r := recover(); r != nil {
err = panicError("daily summary render", r)
}
}()
view := c.summaryView(p.Summary)
body = c.summaryBody(view, p.Until)
return c.summarySubject(view), body, c.summaryPayload(view, body), nil
}
// panicError keeps an error panic value matchable with errors.Is.
func panicError(what string, r any) error {
if e, ok := r.(error); ok {
return fmt.Errorf("%s panicked: %w", what, e)
}
return fmt.Errorf("%s panicked: %v", what, r)
}
// retryDue reports whether the pending window has a destination to try now,
// or none left, as after a crash between the last success and the rollover.
func (c *Collector) retryDue(now time.Time) bool {
for _, channel := range summaryChannels {
if !c.pending.done(channel) && !now.Before(c.retries[channel].next) {
return true
}
}
return c.pending.EmailDone && c.pending.WebhookDone
}
// retryLater schedules a failing destination's next attempt. Only the first
// failure and reaching the hourly cap are reported, so an outage of hours
// logs twice, not every minute.
func (c *Collector) retryLater(channel string, now time.Time, err error) {
r := c.retries[channel]
warn := r.fail(now)
c.retries[channel] = r
if warn {
c.reportSinkError(channel, err)
}
}
// settleDisabled marks done the unfinished destinations the delivery policy
// now switches off, as an attempt would, without waiting out their backoff,
// and releases their waiting notification.
func (c *Collector) settleDisabled() {
settled := false
for _, channel := range summaryChannels {
if c.pending.done(channel) {
continue
}
if c.queues.sinks[channel] != nil && (c.opts.DeliveryEnabled == nil || c.opts.DeliveryEnabled("daily_summary", channel)) {
continue
}
c.retries[channel].ticket.Finish(time.Now())
delete(c.retries, channel)
c.pending.markDone(channel)
settled = true
}
if settled {
c.save()
}
}
// dropPending gives up the pending window when the next one is due. Its
// unfinished destinations were accounted for on their first attempt; one not
// attempted in this process, as after a restart, counts its loss here.
func (c *Collector) dropPending() {
p := c.pending
if p == nil {
return
}
var unfinished []string
for _, channel := range summaryChannels {
if p.done(channel) {
continue
}
unfinished = append(unfinished, channel)
r := c.retries[channel]
r.ticket.Finish(time.Now())
if q := c.queues.sinks[channel]; q != nil && !r.tried && (c.opts.DeliveryEnabled == nil || c.opts.DeliveryEnabled("daily_summary", channel)) {
q.tracker.Lose(time.Now(), 1)
}
}
c.pending = nil
c.retries = nil
if len(unfinished) > 0 {
c.reportSinkError(strings.Join(unfinished, ","), fmt.Errorf("daily summary from %s not delivered before the next one was due; dropped", p.Since.In(c.opts.Location).Format(time.DateOnly)))
}
}
// summaryView is the sent form of s: its top sources, with country.
func (c *Collector) summaryView(s Summary) Summary {
top := append([]SourceCount(nil), s.Sources...)
sortSources(top)
if len(top) > summaryTopSources {
top = top[:summaryTopSources]
}
for i := range top {
top[i].Country = c.opts.CountryOf(top[i].IP)
}
s.Sources = top
return s
}
// Package bpf provides the shared scaffolding that BPF-backed live monitors
// across the daemon use: a common Backend interface, backend-kind constants
// for operator config, sentinel errors that distinguish "not built" from
// "kernel unsupported", and a per-feature backend metric.
//
// Real BPF code (program loading, ringbuf consumption, capability probing)
// lives behind the linux && bpf build tag in sibling files. Default builds
// compile stubs that report all capabilities as unavailable.
package bpf
import (
"context"
"errors"
"sync"
"github.com/pidginhost/csm/internal/metrics"
)
// Backend is the shape every BPF-backed live monitor implements. The legacy
// fallback for each feature implements the same interface so the coordinator
// hands the daemon a uniform handle.
type Backend interface {
Mode() string
EventCount() uint64
Run(ctx context.Context)
}
// Backend kind constants are the shared internal values used after each
// feature validates its operator-facing config setting. Individual features
// may keep older public values, such as AF_ALG's "auditd", and map them to
// BackendLegacy internally.
const (
BackendAuto = "auto"
BackendBPF = "bpf"
BackendLegacy = "legacy"
BackendNone = "none"
)
// ErrNotBuilt is returned by feature loaders when CSM was built without the
// bpf build tag. The coordinator treats this identically to a kernel that
// lacks the required BPF program type: log it and fall back to legacy.
var ErrNotBuilt = errors.New("BPF support not compiled in (rebuild with -tags bpf)")
// ErrUnsupported is returned when CSM was built with the bpf tag but the
// running kernel does not accept the requested BPF program type. Distinct
// from ErrNotBuilt so operator logs explain whether the fix is "rebuild" or
// "newer kernel".
var ErrUnsupported = errors.New("kernel does not support requested BPF program type")
var (
backendMetric *metrics.GaugeVec
backendMetricOnce sync.Once
activeMu sync.RWMutex
activeKinds map[string]string
)
// MetricFor returns the shared csm_bpf_backend gauge. The feature argument
// is accepted for call-site readability; all features share a single
// GaugeVec distinguished by the label value, not by separate vec instances.
// Registered exactly once across the process. Pair with SetActive instead
// of calling With directly.
func MetricFor(_ string) *metrics.GaugeVec {
backendMetricOnce.Do(func() {
backendMetric = metrics.NewGaugeVec(
"csm_bpf_backend",
"Active backend for each BPF-backed live monitor; 1 for the selected kind, 0 otherwise.",
[]string{"feature", "kind"},
)
metrics.MustRegister("csm_bpf_backend", backendMetric)
})
return backendMetric
}
// SetActive sets the metric series so that exactly one of {bpf, legacy, none}
// is at 1 and the others at 0 for the given feature. Call from the coordinator
// after backend selection. Also remembers the active kind in-process so
// internal/health can render the matching capability string without
// reaching into the metric registry.
func SetActive(feature, active string) {
g := MetricFor(feature)
for _, k := range []string{BackendBPF, BackendLegacy, BackendNone} {
v := 0.0
if k == active {
v = 1.0
}
g.With(feature, k).Set(v)
}
activeMu.Lock()
if activeKinds == nil {
activeKinds = make(map[string]string)
}
activeKinds[feature] = active
activeMu.Unlock()
}
// ActiveKinds returns a copy of every selected feature's kind, for status.
// It is never nil, so status shows an empty object rather than null.
func ActiveKinds() map[string]string {
activeMu.RLock()
defer activeMu.RUnlock()
out := make(map[string]string, len(activeKinds))
for feature, kind := range activeKinds {
if kind != "" {
out[feature] = kind
}
}
return out
}
// ActiveKind returns the kind currently selected for the given feature, or
// "" if SetActive was never called for it. internal/health uses this to
// render per-feature capability strings without importing the metrics
// registry.
func ActiveKind(feature string) string {
activeMu.RLock()
defer activeMu.RUnlock()
return activeKinds[feature]
}
package bpf
import "sync"
// Capabilities reports which BPF program types the running kernel can
// actually load and attach. Populated once at daemon startup by Probe.
//
// Each field maps to a kernel feature, not a CSM feature: a single CSM
// feature (e.g. AF_ALG kernel-side blocking) may need multiple capability bits
// (LSMAttach + Ringbuf).
type Capabilities struct {
LSMAttach bool // BPF LSM programs can attach (kernel >= 5.7 with BPF LSM trampoline)
CgroupSock bool // BPF_PROG_TYPE_CGROUP_SOCK_ADDR can attach to cgroup/connect4 (>= 4.10)
Tracepoint bool // BPF_PROG_TYPE_TRACEPOINT can attach to sched/sched_process_exec (>= 4.7)
Ringbuf bool // BPF_MAP_TYPE_RINGBUF available (>= 5.8)
}
// Any reports whether at least one capability is true. Used by callers that
// only need to know "any BPF surface is usable" before deciding on legacy
// vs auto.
func (c Capabilities) Any() bool {
return c.LSMAttach || c.CgroupSock || c.Tracepoint || c.Ringbuf
}
var (
probeOnce sync.Once
probeResult Capabilities
)
// Probe returns the cached BPF capability result for this process. The first
// call performs the privileged load/attach probes; later calls return the same
// value without touching the kernel again.
func Probe() Capabilities {
probeOnce.Do(func() {
probeResult = probeKernel()
})
return probeResult
}
//go:build !(linux && bpf)
package bpf
// probeKernel is the no-tag stub. It returns zero-value Capabilities so that
// all coordinators on this build see "no BPF surface available" and pick the
// legacy backend.
func probeKernel() Capabilities { return Capabilities{} }
package bpf
// dropEventLogStride is how often the drop path logs after the first drop.
// 256 keeps sustained back-pressure visible without flooding the daemon log.
const dropEventLogStride uint64 = 256
func shouldLogDroppedEvent(dropped uint64) bool {
return dropped == 1 || (dropped > 0 && dropped%dropEventLogStride == 0)
}
// Package broadcast provides a one-to-many publish bus for alert.Finding
// events. Subscribers each get a buffered channel; if a subscriber's
// buffer fills, that subscriber drops the message rather than blocking
// the publisher. Used by the SSE event stream and any other in-process
// passive consumer.
//
// This is intentionally separate from the daemon's primary alert pipeline
// (the unbuffered or large-buffered alertCh that feeds Dispatch). The bus
// is a side-channel for observers that should not influence dispatch.
package broadcast
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/queuehealth"
)
// defaultMaxSubscribers caps concurrent subscribers so a flood of event-stream
// connections (each a goroutine plus a buffered channel) cannot exhaust the
// daemon's memory. Generous for an operator dashboard.
const defaultMaxSubscribers = 256
// Bus fans out published findings to every subscriber.
type Bus struct {
mu sync.RWMutex
subscribers map[*Subscription]struct{}
buffer int
maxSubs int
closed bool
stats *queuehealth.Tracker
}
// NewBus constructs a Bus with the given per-subscriber buffer.
// A buffer < 1 falls back to 16.
func NewBus(buffer int) *Bus {
if buffer < 1 {
buffer = 16
}
return &Bus{
subscribers: make(map[*Subscription]struct{}),
buffer: buffer,
maxSubs: defaultMaxSubscribers,
stats: queuehealth.New(0, time.Minute),
}
}
// SetMaxSubscribers overrides the concurrent-subscriber cap. A value < 1 is
// ignored. Safe to call before the bus is in use.
func (b *Bus) SetMaxSubscribers(n int) {
if n < 1 {
return
}
b.mu.Lock()
b.maxSubs = n
b.mu.Unlock()
}
// TrySubscribe is Subscribe with the concurrent-subscriber cap enforced. It
// returns ok=false when the cap is reached so an untrusted caller (the SSE
// endpoint, reachable with a low-trust read token) cannot open unbounded
// long-lived streams. Use this for externally-driven subscriptions; Subscribe
// remains for trusted in-process consumers.
func (b *Bus) TrySubscribe() (*Subscription, bool) {
b.mu.Lock()
defer b.mu.Unlock()
if !b.closed && len(b.subscribers) >= b.maxSubs {
return nil, false
}
return b.subscribe(), true
}
// Subscribe is for trusted in-process consumers. Every received delivery must
// be processed once, and the consumer must Unsubscribe when it stops reading.
func (b *Bus) Subscribe() *Subscription {
b.mu.Lock()
defer b.mu.Unlock()
return b.subscribe()
}
func (b *Bus) subscribe() *Subscription {
sub := &Subscription{events: make(chan Delivery, b.buffer), stats: queuehealth.New(b.buffer, time.Minute)}
if b.closed {
close(sub.events)
} else {
b.subscribers[sub] = struct{}{}
}
return sub
}
// Unsubscribe withdraws demand for unread events, for example when a browser
// tab closes. A delivery already received remains the consumer's responsibility.
func (b *Bus) Unsubscribe(sub *Subscription) {
b.unsubscribe(sub, false)
}
// Abort removes a failed consumer and counts its unread deliveries as lost.
func (b *Bus) Abort(sub *Subscription) {
b.unsubscribe(sub, true)
}
func (b *Bus) unsubscribe(sub *Subscription, failed bool) {
b.mu.Lock()
defer b.mu.Unlock()
if _, ok := b.subscribers[sub]; !ok {
return
}
delete(b.subscribers, sub)
if !b.closed {
close(sub.events)
}
for delivery := range sub.events {
delivery.finish(failed)
}
}
// Publish sends f to every current subscriber. Non-blocking: if a
// subscriber's buffer is full, that delivery is skipped.
func (b *Bus) Publish(f alert.Finding) {
b.mu.RLock()
defer b.mu.RUnlock()
if b.closed {
return
}
for sub := range b.subscribers {
now := time.Now()
delivery := Delivery{finding: f, total: b.stats.Begin(now), local: sub.stats.Begin(now)}
select {
case sub.events <- delivery:
default:
delivery.finish(true)
}
}
}
// Close shuts the bus down and closes every outstanding subscriber channel.
// Idempotent.
func (b *Bus) Close() {
b.mu.Lock()
defer b.mu.Unlock()
if b.closed {
return
}
b.closed = true
// Consumers may still drain buffered deliveries after close. Retain their
// capacity and ownership until they unsubscribe.
for sub := range b.subscribers {
close(sub.events)
}
}
package broadcast
import (
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/queuehealth"
)
// Subscription owns one bounded delivery buffer. Events must have one consumer.
type Subscription struct {
events chan Delivery
stats *queuehealth.Tracker
}
func (s *Subscription) Events() <-chan Delivery { return s.events }
// Delivery retains queue ownership through encoding and writing to the client.
// Call Process exactly once for every received delivery, including failures.
type Delivery struct {
finding alert.Finding
total queuehealth.Ticket
local queuehealth.Ticket
}
func (d Delivery) Process(fn func(alert.Finding) error) error {
now := time.Now()
d.total.Start(now)
d.local.Start(now)
failed := true
defer func() { d.finish(failed) }()
err := fn(d.finding)
failed = err != nil
return err
}
func (d Delivery) finish(failed bool) {
now := time.Now()
if failed {
d.total.Reject(now)
d.local.Reject(now)
} else {
d.total.Finish(now)
d.local.Finish(now)
}
}
// QueueStatuses aggregates subscriber work without retaining departed clients
// or exposing client identities. The total tracker keeps in-flight work and
// cumulative loss after removal; local trackers prevent an empty peer from
// hiding a full subscriber buffer. The row is advisory: a client that stops
// reading loses its own copy of findings that are already stored.
func (b *Bus) QueueStatuses(now time.Time) map[string]queuehealth.Status {
b.mu.RLock()
defer b.mu.RUnlock()
status := b.stats.Snapshot(now)
status.Advisory = true
for sub := range b.subscribers {
local := sub.stats.Snapshot(now)
status.Capacity += local.Capacity
if local.Reason == "queue_full" && (status.Reason == "" || status.Reason == "dropped_work") {
status.Status, status.Reason = "degraded", "queue_full"
}
}
return map[string]queuehealth.Status{"deliveries": status}
}
package challenge
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
)
// CaptchaProvider verifies a third-party CAPTCHA token. Implementations
// post the operator's secret + the visitor's response token to the
// provider's siteverify endpoint and return a single bool: did the
// provider accept this submission?
type CaptchaProvider interface {
Name() string
Verify(ctx context.Context, token, remoteIP string) (bool, error)
}
// providerEndpoint is exposed as a package var so tests can point a
// provider at httptest.Server rather than the live siteverify URL.
var providerEndpoint = map[string]string{
"turnstile": "https://challenges.cloudflare.com/turnstile/v0/siteverify",
"hcaptcha": "https://hcaptcha.com/siteverify",
}
// captchaProvider implements both Cloudflare Turnstile and hCaptcha;
// they accept identical request/response shapes (POST form, JSON
// {"success":bool} reply) so a single struct covers both.
type captchaProvider struct {
name string
endpoint string
secret string
client *http.Client
}
// NewCaptchaProvider returns the right provider for the configured
// name. Returns nil + nil when the operator has not enabled CAPTCHA;
// the server treats nil as "feature off".
func NewCaptchaProvider(name, secret string, timeout time.Duration) (CaptchaProvider, error) {
name = strings.ToLower(strings.TrimSpace(name))
if name == "" {
return nil, nil
}
endpoint, ok := providerEndpoint[name]
if !ok {
return nil, fmt.Errorf("unknown captcha provider %q (want turnstile or hcaptcha)", name)
}
if secret == "" {
return nil, fmt.Errorf("captcha provider %q requires secret_key", name)
}
if timeout <= 0 {
timeout = 10 * time.Second
}
return &captchaProvider{
name: name,
endpoint: endpoint,
secret: secret,
client: &http.Client{Timeout: timeout},
}, nil
}
func (p *captchaProvider) Name() string { return p.name }
// Verify posts the operator's secret + the visitor's token to the
// provider. The remoteIP is optional but recommended; both Turnstile
// and hCaptcha accept it for binding the verification to a single
// client. Network errors propagate; a 200 with success=false returns
// (false, nil).
func (p *captchaProvider) Verify(ctx context.Context, token, remoteIP string) (bool, error) {
if token == "" {
return false, errors.New("empty captcha token")
}
form := url.Values{}
form.Set("secret", p.secret)
form.Set("response", token)
if remoteIP != "" {
form.Set("remoteip", remoteIP)
}
// #nosec G704 -- p.endpoint is set from the providerEndpoint package-level map (turnstile / hcaptcha) by NewCaptchaProvider, which rejects unknown names. Not attacker-controlled; SSRF is not possible.
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.endpoint, strings.NewReader(form.Encode()))
if err != nil {
return false, fmt.Errorf("building siteverify request: %w", err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
// #nosec G704 -- same as above: the request URL is hardcoded to a known siteverify endpoint, never operator or attacker input.
resp, err := p.client.Do(req)
if err != nil {
return false, fmt.Errorf("siteverify call: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false, fmt.Errorf("siteverify status %d", resp.StatusCode)
}
var body struct {
Success bool `json:"success"`
}
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
return false, fmt.Errorf("decoding siteverify response: %w", err)
}
return body.Success, nil
}
package challenge
import (
"context"
"net"
"strings"
"sync"
"time"
)
// crawlerSuffix names a verified-crawler family along with the PTR
// record suffixes that legitimate hosts in that family use.
type crawlerSuffix struct {
name string
domains []string
}
// builtinCrawlers lists the known canonical reverse-DNS suffixes for
// the supported crawler families. Adding a new family means appending
// to this list and documenting the name in csm.yaml's
// challenge.verified_crawlers.providers.
var builtinCrawlers = map[string]crawlerSuffix{
"googlebot": {name: "googlebot", domains: []string{".googlebot.com.", ".google.com."}},
"bingbot": {name: "bingbot", domains: []string{".search.msn.com."}},
}
// Resolver matches the subset of net.Resolver that CrawlerVerifier
// uses, so tests can swap in a fake without spinning up a real DNS
// server.
type Resolver interface {
LookupAddr(ctx context.Context, addr string) (names []string, err error)
LookupHost(ctx context.Context, host string) (addrs []string, err error)
}
// CrawlerVerifier classifies an IP as a verified search crawler iff
// the IP's reverse-DNS PTR matches one of the configured suffixes AND
// the PTR forward-resolves back to the same IP. The verifier caches
// both positive and negative results; positive cache TTL is the
// configured cacheTTL, negative is one-fifth of that to keep a
// transiently-broken resolver from locking out a legitimate crawler.
type CrawlerVerifier struct {
suffixes []crawlerSuffix
resolver Resolver
posTTL time.Duration
negTTL time.Duration
mu sync.Mutex
cache map[string]cacheEntry
maxSize int
}
// crawlerCacheMaxEntries bounds the verifier cache between the daemon's
// 60-second prune ticks. Without a cap a scan from many unique source IPs
// (every miss inserts a negative entry) grows the map to request-rate x 60s,
// an external memory-pressure lever. At the cap an insert first drops expired
// entries, then evicts the soonest-to-expire entry to make room.
const crawlerCacheMaxEntries = 50000
type cacheEntry struct {
verified bool
expires time.Time
}
// NewCrawlerVerifier builds a verifier with the named crawler families
// enabled. Unknown names are ignored (operators may have configured a
// family this binary does not know about; that is harmless).
func NewCrawlerVerifier(providers []string, cacheTTL time.Duration, resolver Resolver) *CrawlerVerifier {
if cacheTTL <= 0 {
cacheTTL = 15 * time.Minute
}
if resolver == nil {
resolver = net.DefaultResolver
}
enabled := make([]crawlerSuffix, 0, len(providers))
for _, name := range providers {
if c, ok := builtinCrawlers[strings.ToLower(strings.TrimSpace(name))]; ok {
enabled = append(enabled, c)
}
}
return &CrawlerVerifier{
suffixes: enabled,
resolver: resolver,
posTTL: cacheTTL,
negTTL: cacheTTL / 5,
cache: make(map[string]cacheEntry),
maxSize: crawlerCacheMaxEntries,
}
}
// Enabled reports whether at least one crawler family is configured;
// the server uses this to skip the verifier entirely (no DNS round
// trip) when the operator has not opted in.
func (v *CrawlerVerifier) Enabled() bool {
return v != nil && len(v.suffixes) > 0
}
// Verified does the reverse-DNS + forward-confirm dance for ip and
// caches the result. Returns true only when the PTR ends in one of the
// allowed suffixes AND a forward lookup of the PTR includes ip in the
// result set.
func (v *CrawlerVerifier) Verified(ctx context.Context, ip string) bool {
if !v.Enabled() {
return false
}
if hit, ok := v.cacheGet(ip); ok {
return hit
}
verified := v.probe(ctx, ip)
v.cachePut(ip, verified)
return verified
}
func (v *CrawlerVerifier) probe(ctx context.Context, ip string) bool {
names, err := v.resolver.LookupAddr(ctx, ip)
if err != nil || len(names) == 0 {
return false
}
for _, name := range names {
// LookupAddr returns FQDNs with a trailing dot. The suffix
// list also has trailing dots so HasSuffix is unambiguous.
lower := strings.ToLower(name)
if !v.suffixMatches(lower) {
continue
}
addrs, err := v.resolver.LookupHost(ctx, strings.TrimSuffix(lower, "."))
if err != nil {
continue
}
for _, addr := range addrs {
if addr == ip {
return true
}
}
}
return false
}
func (v *CrawlerVerifier) suffixMatches(name string) bool {
for _, s := range v.suffixes {
for _, d := range s.domains {
if strings.HasSuffix(name, d) {
return true
}
}
}
return false
}
func (v *CrawlerVerifier) cacheGet(ip string) (bool, bool) {
v.mu.Lock()
defer v.mu.Unlock()
e, ok := v.cache[ip]
if !ok {
return false, false
}
if time.Now().After(e.expires) {
delete(v.cache, ip)
return false, false
}
return e.verified, true
}
func (v *CrawlerVerifier) cachePut(ip string, verified bool) {
ttl := v.negTTL
if verified {
ttl = v.posTTL
}
v.mu.Lock()
defer v.mu.Unlock()
if _, exists := v.cache[ip]; !exists && v.maxSize > 0 && len(v.cache) >= v.maxSize {
v.evictForInsertLocked()
}
v.cache[ip] = cacheEntry{verified: verified, expires: time.Now().Add(ttl)}
}
// evictForInsertLocked frees a slot when the cache is at capacity. It first
// drops every expired entry; if that reclaimed nothing (all entries still
// live), it evicts the single soonest-to-expire entry. Caller holds v.mu.
func (v *CrawlerVerifier) evictForInsertLocked() {
now := time.Now()
freed := false
for ip, e := range v.cache {
if now.After(e.expires) {
delete(v.cache, ip)
freed = true
}
}
if freed {
return
}
var soonestIP string
var soonest time.Time
for ip, e := range v.cache {
if soonestIP == "" || e.expires.Before(soonest) {
soonestIP = ip
soonest = e.expires
}
}
if soonestIP != "" {
delete(v.cache, soonestIP)
}
}
// cleanExpired drops every entry whose TTL has lapsed. Without this,
// a scan from many IPs leaves the cache full of stale entries until
// each individual IP is queried again. Called from Server.CleanExpired
// on the daemon's 60-second ticker. now is passed in so the caller can
// share a single timestamp across multiple cleanup paths.
func (v *CrawlerVerifier) cleanExpired(now time.Time) {
if v == nil {
return
}
v.mu.Lock()
defer v.mu.Unlock()
for ip, e := range v.cache {
if now.After(e.expires) {
delete(v.cache, ip)
}
}
}
package challenge
import (
"bytes"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/atomicio"
)
// DefaultMapPath is the webserver-readable Apache / LSWS RewriteMap.
// It lives in the service's cache directory rather than state_path
// (mode 0700, private to CSM's bbolt database) or the runtime directory:
// Apache and LSWS validate a txt: RewriteMap at config-parse time and Nginx
// fails on a missing include, so the maps are part of the webserver's
// configuration and must exist while CSM is stopped -- package upgrades,
// restores, reboots. systemd deletes the runtime directory on every stop
// and keeps the cache directory.
const DefaultMapPath = "/var/cache/csm/challenge_ips.txt"
// DefaultNginxMapPath is the webserver-readable Nginx map include.
const DefaultNginxMapPath = "/var/cache/csm/challenge_ips.nginx.map"
// challengeEntry stores the challenge metadata for a single IP.
type challengeEntry struct {
FindingID string
// Check routed the address here; a timeout block names it when there
// is no root.
Check string
ExpiresAt time.Time
Reason string
NonEscalating bool
// Root is the admission evidence the challenge answered; the zero
// value when it had none.
Root admission.Evidence
}
// ExpiredEntry is returned by ExpiredEntries for escalation. Root is the
// admission evidence its challenge answered, for the timeout to answer
// again.
type ExpiredEntry struct {
FindingID string
Check string
IP string
Reason string
Root admission.Evidence
}
// PassStore keeps challenge passes across restarts.
type PassStore interface {
PutChallengePass(ip string, at time.Time) error
// ChallengePasses returns passes after cutoff and deletes older ones.
ChallengePasses(cutoff time.Time) (map[string]time.Time, error)
}
const (
// passEscalationGrace is how long a pass spares an address the timeout
// block. The bypass cookie waves a returning browser through for as long,
// so a timeout inside it means the visitor left, not that it failed.
passEscalationGrace = verifyCookieTTL
// PassMemory is how long a pass is remembered. Login, XML-RPC and
// enumeration counts read each site's last log lines, which on a quiet
// site can be days old; an address's lines from before its pass stop
// counting while the pass is remembered. Those counters must also
// discard older lines so forgetting a pass cannot revive its evidence.
PassMemory = 7 * 24 * time.Hour
)
// IPList manages the set of IPs that should see challenge pages.
// Webserver integrations read its maps to redirect IPs to the challenge server.
type IPList struct {
path string
nginxPath string
nginxReload func() error
ips map[string]challengeEntry
// passes holds when each address last passed a challenge.
passes map[string]time.Time
passStore PassStore
lastPassPrune time.Time
mu sync.Mutex
gate PortGate
}
// NewIPList creates an IP list writer for the webserver-facing map at
// mapPath and clears any entries a previous run left in it.
func NewIPList(mapPath string) *IPList {
l := &IPList{
path: mapPath,
ips: make(map[string]challengeEntry),
passes: make(map[string]time.Time),
}
_ = l.flush()
return l
}
// SetPassStore persists future passes to ps and restores the ones it still
// remembers. On a read error the earlier passes stay unknown; the list and
// future passes still work.
func (l *IPList) SetPassStore(ps PassStore) error {
cutoff := time.Now().Add(-PassMemory)
known, err := ps.ChallengePasses(cutoff)
l.mu.Lock()
defer l.mu.Unlock()
l.passStore = ps
for ip, at := range known {
if at.After(cutoff) && at.After(l.passes[ip]) {
l.passes[ip] = at
}
}
return err
}
// Passes returns remembered passes and their retention cutoff together,
// so a scan cannot retain evidence longer than this snapshot retains passes.
func (l *IPList) Passes() (map[string]time.Time, time.Time) {
l.mu.Lock()
defer l.mu.Unlock()
cutoff := time.Now().Add(-PassMemory)
out := make(map[string]time.Time, len(l.passes))
for ip, at := range l.passes {
if at.After(cutoff) {
out[ip] = at
}
}
return out, cutoff
}
// SetPortGate attaches a PortGate so every Add/Remove also opens or
// closes the kernel-level allow. Nil is a no-op (callers don't have to
// branch on whether the gate is configured). Safe to call before any
// Add/Remove; not safe to swap a non-nil gate for another at runtime.
func (l *IPList) SetPortGate(g PortGate) {
l.mu.Lock()
defer l.mu.Unlock()
l.gate = g
}
// SetNginxMap attaches a second map writer for Nginx stacks. The
// callback runs only when the rendered include content changes.
func (l *IPList) SetNginxMap(path string, reload func() error) {
if strings.TrimSpace(path) == "" {
path = DefaultNginxMapPath
}
l.mu.Lock()
l.nginxPath = path
l.nginxReload = reload
changed := l.flush()
l.mu.Unlock()
l.reloadNginx(changed, reload)
}
// Add marks an IP for challenge with the given reason.
func (l *IPList) Add(ip string, reason string, duration time.Duration) {
l.add(ip, reason, duration, false, "", "", admission.Evidence{})
}
// AddNonEscalating marks an IP for challenge without timeout-to-block escalation.
func (l *IPList) AddNonEscalating(ip string, reason string, duration time.Duration) {
l.add(ip, reason, duration, true, "", "", admission.Evidence{})
}
// AddWithFindingID preserves the originating audit identity for timeout
// escalation. It does not change challenge duration or escalation policy.
func (l *IPList) AddWithFindingID(ip, reason string, duration time.Duration, findingID string) {
l.add(ip, reason, duration, false, findingID, "", admission.Evidence{})
}
// AddWithRoot also keeps the check that routed the address and the
// admission evidence the challenge answered, so a timeout escalation answers
// the same observation again rather than a new one. root may be zero.
func (l *IPList) AddWithRoot(ip, reason string, duration time.Duration, findingID, check string, root admission.Evidence) {
l.add(ip, reason, duration, false, findingID, check, root)
}
func (l *IPList) add(ip string, reason string, duration time.Duration, nonEscalating bool, findingID, check string, root admission.Evidence) {
l.mu.Lock()
l.ips[ip] = challengeEntry{
ExpiresAt: time.Now().Add(duration),
Reason: reason,
NonEscalating: nonEscalating,
FindingID: findingID,
Check: check,
Root: root,
}
changed := l.flush()
gate := l.gate
reload := l.nginxReload
l.mu.Unlock()
if gate != nil {
if err := gate.Allow(ip, duration); err != nil {
fmt.Fprintf(os.Stderr, "challenge: port-gate allow %s: %v\n", ip, err)
}
}
l.reloadNginx(changed, reload)
}
// Remove stops challenging an IP without recording a pass.
func (l *IPList) Remove(ip string) { l.remove(ip, time.Time{}) }
// MarkPassed takes a challenged address off the list and remembers the pass.
// It reports false when the address was not listed, so concurrent completions
// of one challenge count once.
func (l *IPList) MarkPassed(ip string) bool { return l.remove(ip, time.Now()) }
func (l *IPList) remove(ip string, passedAt time.Time) bool {
l.mu.Lock()
if _, listed := l.ips[ip]; !listed {
l.mu.Unlock()
return false
}
delete(l.ips, ip)
if !passedAt.IsZero() && passedAt.After(l.passes[ip]) {
l.passes[ip] = passedAt
}
changed := l.flush()
gate := l.gate
reload := l.nginxReload
store := l.passStore
l.mu.Unlock()
// Only revoke the kernel gate for an IP that was actually on the list: the
// gate element is created alongside the entry (Add -> Allow), so an IP that
// was never listed has no element to delete. Without this guard, verified
// crawlers -- which bypass the gate and call Remove on every request -- make
// the gate delete a nonexistent element and log an ENOENT error each time.
if gate != nil {
if err := gate.Revoke(ip); err != nil {
fmt.Fprintf(os.Stderr, "challenge: port-gate revoke %s: %v\n", ip, err)
}
}
l.reloadNginx(changed, reload)
if !passedAt.IsZero() && store != nil {
if err := store.PutChallengePass(ip, passedAt); err != nil {
fmt.Fprintf(os.Stderr, "challenge: keep pass of %s: %v\n", ip, err)
}
}
return true
}
// Contains returns true if the IP is currently on the challenge list.
func (l *IPList) Contains(ip string) bool {
l.mu.Lock()
defer l.mu.Unlock()
_, ok := l.ips[ip]
return ok
}
// Count returns the number of IPs currently waiting on a challenge.
func (l *IPList) Count() int {
l.mu.Lock()
defer l.mu.Unlock()
return len(l.ips)
}
// ExpiredEntries removes expired entries and returns those eligible for escalation.
// The caller is expected to hard-block returned IPs. An address that passed
// within passEscalationGrace is not returned: it proved a browser and left.
func (l *IPList) ExpiredEntries() []ExpiredEntry {
l.mu.Lock()
now := time.Now()
graceStart := now.Add(-passEscalationGrace)
forgetBefore := now.Add(-PassMemory)
for ip, at := range l.passes {
if !at.After(forgetBefore) {
delete(l.passes, ip)
}
}
var expired []ExpiredEntry
removed := false
for ip, entry := range l.ips {
if now.After(entry.ExpiresAt) {
if !entry.NonEscalating && !l.passes[ip].After(graceStart) {
expired = append(expired, ExpiredEntry{IP: ip, Reason: entry.Reason, FindingID: entry.FindingID, Check: entry.Check, Root: entry.Root})
}
delete(l.ips, ip)
removed = true
}
}
var changed bool
if removed {
changed = l.flush()
}
reload := l.nginxReload
ps := l.passStore
prune := ps != nil && now.Sub(l.lastPassPrune) >= time.Hour
if prune {
l.lastPassPrune = now
}
l.mu.Unlock()
l.reloadNginx(changed, reload)
// Reuse the store's time-based cleanup during normal maintenance, not
// only at startup. Never evict a live pass to make room for another IP.
if prune {
if _, err := ps.ChallengePasses(forgetBefore); err != nil {
fmt.Fprintf(os.Stderr, "challenge: expire passes: %v\n", err)
l.mu.Lock()
l.lastPassPrune = time.Time{}
l.mu.Unlock()
}
}
return expired
}
// CleanExpired removes expired entries without returning them.
// Use ExpiredEntries() instead when escalation is needed.
func (l *IPList) CleanExpired() {
_ = l.ExpiredEntries()
}
// flush writes the IP list to disk in each configured webserver format.
// The caller must hold l.mu. It returns true when the Nginx include
// changed and needs a reload.
func (l *IPList) flush() bool {
ips := sortedIPKeys(l.ips)
var sb strings.Builder
sb.WriteString("# CSM Challenge IP list - auto-generated, do not edit\n")
sb.WriteString("# Format: IP challenge (for Apache RewriteMap)\n")
for _, ip := range ips {
fmt.Fprintf(&sb, "%s challenge\n", ip)
}
if err := writeMapFile(l.path, []byte(sb.String())); err != nil {
return false
}
if strings.TrimSpace(l.nginxPath) == "" {
return false
}
var nginx strings.Builder
nginx.WriteString("# CSM Challenge IP list - auto-generated, do not edit\n")
nginx.WriteString("# Format: IP 1; (for Nginx map include)\n")
for _, ip := range ips {
fmt.Fprintf(&nginx, "%s 1;\n", ip)
}
changed, err := writeMapFileIfChanged(l.nginxPath, []byte(nginx.String()))
return err == nil && changed
}
func sortedIPKeys(ips map[string]challengeEntry) []string {
keys := make([]string, 0, len(ips))
for ip := range ips {
keys = append(keys, ip)
}
sort.Strings(keys)
return keys
}
func writeMapFileIfChanged(path string, data []byte) (bool, error) {
// #nosec G304 -- path is the daemon-owned challenge map under /var/cache/csm
// (DefaultNginxMapPath or operator-set via SetNginxMap), never
// attacker-controlled. Read only to diff the rendered map and skip
// rewrite when content is unchanged.
if current, err := os.ReadFile(path); err == nil && bytes.Equal(current, data) {
return false, nil
}
return true, writeMapFile(path, data)
}
func ensureMapDir(path string) error {
// /var/cache/csm must be world-readable so the webserver user
// (www-data / nobody / lsws) can stat + read the map underneath.
// The directory holds no sensitive data; only CSM-owned IP files
// live inside.
//
// MkdirAll respects the process umask, so on cPanel/CloudLinux
// hosts where csm.service inherits umask 027 the directory ends
// up at 0o750 and the webserver gets EACCES on the RewriteMap.
// Explicit Chmod after creation forces the mode the integration
// requires regardless of umask.
mapDir := filepath.Dir(path)
// #nosec G301 -- world-readable rationale above.
if err := os.MkdirAll(mapDir, 0o755); err != nil {
return err
}
// #nosec G302 -- same world-readable rationale; needed for the
// webserver user to stat into the directory and read the map.
if err := os.Chmod(mapDir, 0o755); err != nil {
return err
}
return nil
}
// EnsureMapFile creates a readable empty map when path is absent and leaves
// existing map contents intact. Apache validates txt: RewriteMap sources even
// when challenge mode is disabled, before an IPList would otherwise create the
// file.
func EnsureMapFile(path string) error {
if err := ensureMapDir(path); err != nil {
return err
}
// #nosec G302 G304 G703 -- path is the daemon-owned, webserver-readable challenge map.
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
info, statErr := os.Lstat(path)
if statErr != nil {
return statErr
}
if !info.Mode().IsRegular() {
return fmt.Errorf("challenge map %s is not a regular file", path)
}
// #nosec G302 -- Apache/LSWS must be able to read the daemon-owned map.
return os.Chmod(path, 0o644)
}
cleanup := func() {
_ = f.Close()
_ = os.Remove(path)
}
// OpenFile respects umask, so force the public read bits Apache needs.
// #nosec G302 -- Apache/LSWS must be able to read the daemon-owned map.
if err := f.Chmod(0o644); err != nil {
cleanup()
return err
}
if err := f.Close(); err != nil {
_ = os.Remove(path)
return err
}
return nil
}
func writeMapFile(path string, data []byte) error {
if err := ensureMapDir(path); err != nil {
return err
}
// #nosec G306 -- webservers read this daemon-owned IP map directly.
return atomicio.AtomicWrite(path, 0o644, data)
}
func (l *IPList) reloadNginx(changed bool, reload func() error) {
if !changed || reload == nil {
return
}
if err := reload(); err != nil {
fmt.Fprintf(os.Stderr, "challenge: nginx reload after map update: %v\n", err)
}
}
package challenge
import (
"net/http"
"sync"
"github.com/pidginhost/csm/internal/metrics"
)
// How a visitor passed a challenge, the method label of
// csm_challenge_passed_total.
const (
passMethodPoW = "pow"
passMethodCaptcha = "captcha"
passMethodVerifyCookie = "verify_cookie"
passMethodAdminCookie = "admin_cookie"
passMethodCrawler = "crawler"
)
var (
challengePassedMetric *metrics.CounterVec
challengePassedMetricOnce sync.Once
)
func challengePassedCounter() *metrics.CounterVec {
challengePassedMetricOnce.Do(func() {
challengePassedMetric = metrics.NewCounterVec(
"csm_challenge_passed_total",
"Challenged addresses that passed, by method (pow, captcha, verify_cookie, admin_cookie, crawler).",
[]string{"method"},
)
metrics.MustRegister("csm_challenge_passed_total", challengePassedMetric)
})
return challengePassedMetric
}
func observeChallengePassed(method string) {
challengePassedCounter().With(method).Inc()
}
// PassedCount returns how many challenged addresses passed since start.
func PassedCount() int {
c := challengePassedCounter()
total := 0.0
for _, m := range []string{passMethodPoW, passMethodCaptcha, passMethodVerifyCookie, passMethodAdminCookie, passMethodCrawler} {
total += c.With(m).Value()
}
return int(total)
}
// bypassMethod names the shortcut that lets a pending visitor through
// without the puzzle, or "" when none applies. The order is the order the
// shortcuts are tried in.
func (s *Server) bypassMethod(r *http.Request, ip string) string {
switch {
case s.bypassByAdminCookie(r, ip):
return passMethodAdminCookie
case s.bypassByVerifiedCrawler(r.Context(), ip):
return passMethodCrawler
case s.bypassByVerifyCookie(r, ip):
return passMethodVerifyCookie
}
return ""
}
package challenge
import (
"net"
"strings"
"time"
)
// PortGate locks the challenge listener TCP port to specific source IPs
// via the host firewall. An IP is allowed only while it is on the
// challenge IPList (plus operator infra IPs and loopback). Everything
// else gets dropped at the kernel before the listener sees the SYN, so
// the listener is invisible to port scanners and stays reachable only
// for the visitors the daemon has actually redirected.
//
// Implementations are pluggable so the netlink-backed Linux variant
// can be swapped for a stub on platforms that do not have nftables.
// All methods are safe to call on a nil PortGate (no-op), so callers
// do not need to nil-check at every IPList Add/Remove site.
type PortGate interface {
// Allow opens the gate for the source IP for at most ttl. The
// underlying firewall enforces the TTL via the set's own timeout
// so the entry expires even if Revoke is never called (daemon
// crash, missed expiry). Returns nil on success or when the IP
// cannot be parsed (best-effort; the IPList accepts only validated
// IPs upstream, so a parse miss here is a bug to log, not block).
Allow(ip string, ttl time.Duration) error
// Revoke closes the gate for ip immediately. Safe to call for IPs
// that were never on the gate (no-op).
Revoke(ip string) error
// Close tears down the gate's nftables footprint (chain, sets,
// table). The port reverts to whatever the rest of the host
// firewall would do with it.
Close() error
}
// PortGateConfig wraps the inputs the gate needs to install rules.
type PortGateConfig struct {
ListenAddr string
ListenPort int
InfraCIDRs []string
}
// NewPortGate returns the platform-appropriate gate. On Linux it
// installs a dedicated `csm_chal` inet table; on non-Linux it returns
// nil so callers naturally no-op via the nil PortGate handling on the
// IPList side.
//
// Returns nil + nil when the listen address is loopback because no
// gate is needed (loopback traffic cannot originate from off-host).
// Caller treats nil as "gate not active" and proceeds without it.
func NewPortGate(cfg PortGateConfig) (PortGate, error) {
if localOnlyListener(cfg.ListenAddr) {
return nil, nil
}
return newPortGate(cfg)
}
// portGateFamily picks which address families the gate should bind to
// based on the listen address. 0.0.0.0 / blank -> v4 only; :: -> dual
// stack; a specific literal IP gates only that family.
type portGateFamily struct {
v4 bool
v6 bool
}
func familyForListenAddr(addr string) portGateFamily {
addr = strings.Trim(strings.TrimSpace(addr), "[]")
switch addr {
case "", "0.0.0.0":
return portGateFamily{v4: true}
case "::":
return portGateFamily{v4: true, v6: true}
}
ip := net.ParseIP(addr)
if ip == nil {
return portGateFamily{v4: true}
}
if ip.To4() != nil {
return portGateFamily{v4: true}
}
return portGateFamily{v6: true}
}
func portGateFamilyAcceptsIP(fam portGateFamily, ip net.IP) bool {
if ip == nil {
return false
}
if ip.To4() != nil {
return fam.v4
}
return fam.v6
}
//go:build linux
package challenge
import (
"errors"
"fmt"
"net"
"sync"
"syscall"
"time"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
"golang.org/x/sys/unix"
)
// linuxPortGate owns the nftables `csm_chal` table. The table is kept
// separate from the firewall package's `csm` table so the two can be
// installed and torn down independently (challenge can run with or
// without csm.firewall enabled).
type linuxPortGate struct {
mu sync.Mutex
conn *nftables.Conn
cfg PortGateConfig
family portGateFamily
table *nftables.Table
setChalIPs *nftables.Set
setChalIPs6 *nftables.Set
setInfra *nftables.Set
setInfra6 *nftables.Set
}
func newPortGate(cfg PortGateConfig) (PortGate, error) {
if cfg.ListenPort <= 0 || cfg.ListenPort > 65535 {
return nil, fmt.Errorf("port-gate: invalid listen port %d", cfg.ListenPort)
}
fam := familyForListenAddr(cfg.ListenAddr)
conn, err := nftables.New()
if err != nil {
return nil, fmt.Errorf("port-gate: nftables open: %w", err)
}
g := &linuxPortGate{conn: conn, cfg: cfg, family: fam}
if err := g.install(); err != nil {
return nil, err
}
return g, nil
}
// install lays down the table, sets, chain, and per-port rules. Any
// pre-existing `csm_chal` table is deleted first so a daemon restart
// always converges on a clean rule shape (and stale rules from a
// crashed previous run do not linger).
func (g *linuxPortGate) install() error {
g.mu.Lock()
defer g.mu.Unlock()
g.dropExistingTableLocked()
g.table = g.conn.AddTable(&nftables.Table{
Family: nftables.TableFamilyINet,
Name: "csm_chal",
})
if g.family.v4 {
g.setChalIPs = &nftables.Set{
Table: g.table,
Name: "chal_ips",
KeyType: nftables.TypeIPAddr,
HasTimeout: true,
}
if err := g.conn.AddSet(g.setChalIPs, nil); err != nil {
return fmt.Errorf("port-gate: add chal_ips: %w", err)
}
g.setInfra = &nftables.Set{
Table: g.table,
Name: "chal_infra",
KeyType: nftables.TypeIPAddr,
Interval: true,
}
if err := g.conn.AddSet(g.setInfra, infraElementsV4(g.cfg.InfraCIDRs)); err != nil {
return fmt.Errorf("port-gate: add chal_infra: %w", err)
}
}
if g.family.v6 {
g.setChalIPs6 = &nftables.Set{
Table: g.table,
Name: "chal_ips6",
KeyType: nftables.TypeIP6Addr,
HasTimeout: true,
}
if err := g.conn.AddSet(g.setChalIPs6, nil); err != nil {
return fmt.Errorf("port-gate: add chal_ips6: %w", err)
}
g.setInfra6 = &nftables.Set{
Table: g.table,
Name: "chal_infra6",
KeyType: nftables.TypeIP6Addr,
Interval: true,
}
if err := g.conn.AddSet(g.setInfra6, infraElementsV6(g.cfg.InfraCIDRs)); err != nil {
return fmt.Errorf("port-gate: add chal_infra6: %w", err)
}
}
prio := nftables.ChainPriority(-200)
policy := nftables.ChainPolicyAccept
chain := g.conn.AddChain(&nftables.Chain{
Name: "challenge_gate",
Table: g.table,
Type: nftables.ChainTypeFilter,
Hooknum: nftables.ChainHookInput,
Priority: &prio,
Policy: &policy,
})
g.addAcceptRules(chain)
g.addDropRule(chain)
if err := g.conn.Flush(); err != nil {
return fmt.Errorf("port-gate: install flush: %w", err)
}
return nil
}
// dropExistingTableLocked is idempotent: ListTables + DelTable so a
// re-install does not stack rules on top of a stale chain.
func (g *linuxPortGate) dropExistingTableLocked() {
tables, err := g.conn.ListTables()
if err != nil {
return
}
for _, t := range tables {
if t.Family == nftables.TableFamilyINet && t.Name == "csm_chal" {
g.conn.DelTable(t)
_ = g.conn.Flush()
return
}
}
}
func (g *linuxPortGate) addAcceptRules(chain *nftables.Chain) {
port := portU16(g.cfg.ListenPort)
if g.family.v4 {
// loopback bypass
g.addRule(chain, exprsTCPDportFromV4(port, net.IPv4(127, 0, 0, 0).To4(), net.IPv4Mask(255, 0, 0, 0), expr.VerdictAccept))
g.addRule(chain, exprsTCPDportSetMatchV4(port, g.setInfra, expr.VerdictAccept))
g.addRule(chain, exprsTCPDportSetMatchV4(port, g.setChalIPs, expr.VerdictAccept))
}
if g.family.v6 {
loop6 := net.ParseIP("::1").To16()
mask128 := net.CIDRMask(128, 128)
g.addRule(chain, exprsTCPDportFromV6(port, loop6, mask128, expr.VerdictAccept))
g.addRule(chain, exprsTCPDportSetMatchV6(port, g.setInfra6, expr.VerdictAccept))
g.addRule(chain, exprsTCPDportSetMatchV6(port, g.setChalIPs6, expr.VerdictAccept))
}
}
func (g *linuxPortGate) addDropRule(chain *nftables.Chain) {
port := portU16(g.cfg.ListenPort)
// Any packet that reached this rule with dport == challenge port
// did not match an accept above; drop it.
g.conn.AddRule(&nftables.Rule{
Table: g.table,
Chain: chain,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(port)},
&expr.Verdict{Kind: expr.VerdictDrop},
},
})
}
func (g *linuxPortGate) addRule(chain *nftables.Chain, exprs []expr.Any) {
g.conn.AddRule(&nftables.Rule{Table: g.table, Chain: chain, Exprs: exprs})
}
func (g *linuxPortGate) Allow(ip string, ttl time.Duration) error {
parsed := net.ParseIP(ip)
if parsed == nil {
return fmt.Errorf("port-gate: invalid ip %q", ip)
}
if ttl <= 0 {
ttl = 30 * time.Minute
}
if !portGateFamilyAcceptsIP(g.family, parsed) {
return nil
}
g.mu.Lock()
defer g.mu.Unlock()
if ip4 := parsed.To4(); ip4 != nil && g.setChalIPs != nil {
if err := g.conn.SetAddElements(g.setChalIPs, []nftables.SetElement{
{Key: ip4, Timeout: ttl},
}); err != nil {
return fmt.Errorf("port-gate: add v4 %s: %w", ip, err)
}
return g.conn.Flush()
}
if g.setChalIPs6 != nil {
if err := g.conn.SetAddElements(g.setChalIPs6, []nftables.SetElement{
{Key: parsed.To16(), Timeout: ttl},
}); err != nil {
return fmt.Errorf("port-gate: add v6 %s: %w", ip, err)
}
return g.conn.Flush()
}
// IP family is not gated (e.g., v6 IP on a v4-only listener); silently
// no-op so the IPList Add path does not surface an unactionable error.
return nil
}
func (g *linuxPortGate) Revoke(ip string) error {
parsed := net.ParseIP(ip)
if parsed == nil {
return fmt.Errorf("port-gate: invalid ip %q", ip)
}
if !portGateFamilyAcceptsIP(g.family, parsed) {
return nil
}
g.mu.Lock()
defer g.mu.Unlock()
if ip4 := parsed.To4(); ip4 != nil && g.setChalIPs != nil {
if err := g.conn.SetDeleteElements(g.setChalIPs, []nftables.SetElement{{Key: ip4}}); err != nil {
return fmt.Errorf("port-gate: del v4 %s: %w", ip, err)
}
return ignoreNftNotFound(g.conn.Flush())
}
if g.setChalIPs6 != nil {
if err := g.conn.SetDeleteElements(g.setChalIPs6, []nftables.SetElement{{Key: parsed.To16()}}); err != nil {
return fmt.Errorf("port-gate: del v6 %s: %w", ip, err)
}
return ignoreNftNotFound(g.conn.Flush())
}
return nil
}
// ignoreNftNotFound treats a "no such file or directory" netlink error from a
// set-element delete as success. Gate elements carry a TTL, so the kernel may
// have already expired the element by the time Revoke runs; deleting an absent
// element is a benign no-op, not a failure worth surfacing.
func ignoreNftNotFound(err error) error {
if errors.Is(err, syscall.ENOENT) {
return nil
}
return err
}
func (g *linuxPortGate) Close() error {
g.mu.Lock()
defer g.mu.Unlock()
if g.table == nil {
return nil
}
g.conn.DelTable(g.table)
if err := g.conn.Flush(); err != nil {
return fmt.Errorf("port-gate: close flush: %w", err)
}
g.table = nil
g.setChalIPs = nil
g.setChalIPs6 = nil
g.setInfra = nil
g.setInfra6 = nil
return nil
}
func portU16(p int) uint16 {
if p < 0 || p > 65535 {
return 0
}
// #nosec G115 -- bounds-checked above.
return uint16(p)
}
// infraElementsV4 builds nftables interval-set elements from the
// operator's infra_ips list, keeping only IPv4 entries. The interval
// set wants [start, end) pairs; net.ParseCIDR + binaryutil pack them
// the same way the firewall engine's infra set does.
func infraElementsV4(cidrs []string) []nftables.SetElement {
var out []nftables.SetElement
for _, raw := range cidrs {
ipnet := parseCIDROrIP(raw)
if ipnet == nil {
continue
}
start := ipnet.IP.To4()
if start == nil {
continue
}
end := lastIPv4(ipnet)
out = append(out,
nftables.SetElement{Key: start},
nftables.SetElement{Key: ipv4Inc(end), IntervalEnd: true},
)
}
return out
}
func infraElementsV6(cidrs []string) []nftables.SetElement {
var out []nftables.SetElement
for _, raw := range cidrs {
ipnet := parseCIDROrIP(raw)
if ipnet == nil {
continue
}
if ipnet.IP.To4() != nil {
continue
}
start := ipnet.IP.To16()
end := lastIPv6(ipnet)
out = append(out,
nftables.SetElement{Key: start},
nftables.SetElement{Key: ipv6Inc(end), IntervalEnd: true},
)
}
return out
}
// parseCIDROrIP accepts both "1.2.3.4" and "1.2.3.0/24". A bare IP is
// treated as a /32 (or /128 for v6).
func parseCIDROrIP(raw string) *net.IPNet {
if _, ipnet, err := net.ParseCIDR(raw); err == nil {
return ipnet
}
ip := net.ParseIP(raw)
if ip == nil {
return nil
}
if v4 := ip.To4(); v4 != nil {
return &net.IPNet{IP: v4, Mask: net.CIDRMask(32, 32)}
}
return &net.IPNet{IP: ip.To16(), Mask: net.CIDRMask(128, 128)}
}
func lastIPv4(n *net.IPNet) net.IP {
ip := n.IP.To4()
out := make(net.IP, 4)
for i := 0; i < 4; i++ {
out[i] = ip[i] | ^n.Mask[i]
}
return out
}
func lastIPv6(n *net.IPNet) net.IP {
ip := n.IP.To16()
out := make(net.IP, 16)
for i := 0; i < 16; i++ {
out[i] = ip[i] | ^n.Mask[i]
}
return out
}
func ipv4Inc(ip net.IP) net.IP {
out := make(net.IP, 4)
copy(out, ip.To4())
for i := 3; i >= 0; i-- {
out[i]++
if out[i] != 0 {
return out
}
}
return out
}
func ipv6Inc(ip net.IP) net.IP {
out := make(net.IP, 16)
copy(out, ip.To16())
for i := 15; i >= 0; i-- {
out[i]++
if out[i] != 0 {
return out
}
}
return out
}
// exprsTCPDportFromV4 produces "L4=TCP, src in CIDR, dport=port -> verdict".
// Mask is the 4-byte IPv4 subnet mask.
func exprsTCPDportFromV4(port uint16, network net.IP, mask net.IPMask, verdict expr.VerdictKind) []expr.Any {
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}},
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.NFPROTO_IPV4}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4},
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, Mask: mask, Xor: []byte{0, 0, 0, 0}},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: network},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(port)},
&expr.Verdict{Kind: verdict},
}
}
func exprsTCPDportFromV6(port uint16, network net.IP, mask net.IPMask, verdict expr.VerdictKind) []expr.Any {
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}},
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.NFPROTO_IPV6}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 8, Len: 16},
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 16, Mask: mask, Xor: make([]byte, 16)},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: network},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(port)},
&expr.Verdict{Kind: verdict},
}
}
func exprsTCPDportSetMatchV4(port uint16, set *nftables.Set, verdict expr.VerdictKind) []expr.Any {
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}},
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.NFPROTO_IPV4}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4},
&expr.Lookup{SourceRegister: 1, SetName: set.Name, SetID: set.ID},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(port)},
&expr.Verdict{Kind: verdict},
}
}
func exprsTCPDportSetMatchV6(port uint16, set *nftables.Set, verdict expr.VerdictKind) []expr.Any {
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}},
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.NFPROTO_IPV6}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 8, Len: 16},
&expr.Lookup{SourceRegister: 1, SetName: set.Name, SetID: set.ID},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(port)},
&expr.Verdict{Kind: verdict},
}
}
package challenge
import (
"context"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"fmt"
"html"
"net"
"net/http"
"net/netip"
"net/url"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/clientaddr"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/netutil"
)
// Server serves challenge pages to gray-listed IPs. Passing the challenge
// only takes the IP off the challenge list and hands the browser a bypass
// cookie; it grants nothing in the firewall. The list is the sole source of
// truth for who has a challenge pending, so every handler answers only for
// IPs on it -- serving the puzzle to anyone else would let an arbitrary
// client turn one cheap hash search into a verified session.
type Server struct {
cfg *config.Config
secret []byte
ipList *IPList
srv *http.Server
// trusted is the one trusted proxy list: web_server.trusted_proxies and
// challenge.trusted_proxies together.
trusted clientaddr.Trusted
// socket mode: listen_addr is unix:<path>. The only peer that can open
// the socket is the proxy, so its X-Forwarded-For names the visitor.
socket bool
socketPath string
socketMu sync.Mutex
socketListener net.Listener
socketClosed bool
// Track recently verified IPs to prevent replay
verified map[string]time.Time
verifiedMu sync.Mutex
// Optional subsystems wired in by configuration. Any of these may
// be nil; the handlers branch accordingly so a fresh deployment
// without the new blocks behaves exactly as before.
captcha CaptchaProvider
sessionSigner *AdminSessionSigner
crawlers *CrawlerVerifier
// verifySigner signs the csm_verified cookie handed to every visitor
// who passes the PoW/CAPTCHA. It binds the cookie to one IP and an
// expiry so a returning visitor skips the gate for the allow window
// without it being replayable elsewhere. Always constructed; the key
// rotates on restart like sessionSigner.
verifySigner *AdminSessionSigner
// adminFailures tracks failed admin-token submissions per source
// IP for rate-limiting brute-force probes. Sliding window: an IP
// that hits adminMaxFailuresInWindow within adminFailureWindow
// gets locked out (subsequent submissions return 429) until the
// oldest failure ages out.
adminFailures map[string][]time.Time
adminFailuresMu sync.Mutex
}
const (
adminFailureWindow = 5 * time.Minute
adminMaxFailuresInWindow = 5
// verifyCookieTTL is the lifetime of the csm_verified bypass cookie: how
// long a visitor who passed once is waved through if listed again.
verifyCookieTTL = 4 * time.Hour
)
// New creates a challenge server.
func New(cfg *config.Config, ipList *IPList) *Server {
secret := []byte(cfg.Challenge.Secret)
if len(secret) == 0 {
secret = make([]byte, 32)
_, _ = rand.Read(secret)
}
s := &Server{
cfg: cfg,
secret: secret,
ipList: ipList,
trusted: config.TrustedProxies(cfg),
verified: make(map[string]time.Time),
adminFailures: make(map[string][]time.Time),
}
// The verify-cookie signer is always on (independent of the optional
// admin-session feature).
if vs, err := NewAdminSessionSigner(verifyCookieTTL); err != nil {
fmt.Fprintf(os.Stderr, "[challenge] verify-cookie signing disabled: %v\n", err)
} else {
s.verifySigner = vs
}
// Optional sub-features. Each is opt-in via its own config block;
// initialization errors degrade to "feature off" with a stderr
// message rather than refusing to start the challenge server.
if name := cfg.Challenge.CaptchaFallback.Provider; name != "" {
p, err := NewCaptchaProvider(name, cfg.Challenge.CaptchaFallback.SecretKey, cfg.Challenge.CaptchaFallback.Timeout)
if err != nil {
fmt.Fprintf(os.Stderr, "[challenge] captcha disabled: %v\n", err)
}
s.captcha = p
}
if cfg.Challenge.VerifiedSession.Enabled {
signer, err := NewAdminSessionSigner(cfg.Challenge.VerifiedSession.TTL)
if err != nil {
fmt.Fprintf(os.Stderr, "[challenge] verified-session disabled: %v\n", err)
}
s.sessionSigner = signer
}
if cfg.Challenge.VerifiedCrawlers.Enabled {
s.crawlers = NewCrawlerVerifier(
cfg.Challenge.VerifiedCrawlers.Providers,
cfg.Challenge.VerifiedCrawlers.CacheTTL,
nil, // net.DefaultResolver
)
}
s.socketPath, s.socket = config.ChallengeSocketPath(cfg.Challenge.ListenAddr)
mux := http.NewServeMux()
mux.HandleFunc("/challenge", s.handleChallenge)
mux.HandleFunc("/challenge/gate", s.handleGate)
mux.HandleFunc("/challenge/verify", s.handleVerify)
mux.HandleFunc("/challenge/captcha-verify", s.handleCaptchaVerify)
mux.HandleFunc("/challenge/admin-token", s.handleAdminToken)
addr := ""
if !s.socket {
addr = net.JoinHostPort(ListenHost(cfg.Challenge.ListenAddr), strconv.Itoa(cfg.Challenge.ListenPort))
}
s.srv = &http.Server{
Addr: addr,
Handler: mux,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
ReadHeaderTimeout: 5 * time.Second,
}
return s
}
// Listen validates configured TLS material and binds the challenge address.
// Daemon startup calls this synchronously before publishing the challenge IP
// list to response routing, so a bind or certificate error fails open to
// alerting instead of treating an unavailable challenge as a handled finding.
func (s *Server) Listen() (net.Listener, error) {
cert, key := s.resolveTLSMaterial()
if cert != "" && key != "" {
if _, err := tls.LoadX509KeyPair(cert, key); err != nil {
return nil, fmt.Errorf("challenge TLS material: %w", err)
}
}
if s.socket {
s.socketMu.Lock()
defer s.socketMu.Unlock()
if s.socketClosed {
return nil, http.ErrServerClosed
}
if s.socketListener != nil {
return nil, fmt.Errorf("challenge socket %s is already listening", s.socketPath)
}
listener, err := listenUnixSocket(s.socketPath, s.cfg.Challenge.ListenGroup)
if err != nil {
return nil, err
}
s.socketListener = listener
return listener, nil
}
listener, err := net.Listen("tcp", s.srv.Addr)
if err != nil {
return nil, fmt.Errorf("challenge listen %s: %w", s.srv.Addr, err)
}
return listener, nil
}
// Addr is the address the server listens on: the socket path in socket
// mode, otherwise host:port.
func (s *Server) Addr() string {
if s.socket {
return s.socketPath
}
return s.srv.Addr
}
// Serve begins serving challenge pages on an already-bound listener. Explicit
// challenge TLS makes the listener HTTPS. Direct/public listeners can reuse
// the WebUI TLS pair. Loopback listeners stay plain HTTP by default.
//
// Resolution order:
// 1. challenge.tls_cert + challenge.tls_key (explicit per-service)
// 2. webui.tls_cert + webui.tls_key (direct/public binds only)
// 3. plain HTTP (loopback-only default)
func (s *Server) Serve(listener net.Listener) error {
// ServeTLS can fail before http.Server takes ownership of the listener.
defer func() { _ = listener.Close() }()
cert, key := s.resolveTLSMaterial()
if cert == "" || key == "" {
if !localOnlyListener(s.cfg.Challenge.ListenAddr) {
fmt.Fprintf(os.Stderr,
"[%s] WARNING: public challenge listener has no complete TLS cert/key configured; HSTS-pinned domains will fail with ERR_SSL_PROTOCOL_ERROR\n",
time.Now().Format("2006-01-02 15:04:05"))
}
return s.srv.Serve(listener)
}
return s.srv.ServeTLS(listener, cert, key)
}
// Start binds and serves in one call for standalone users. The daemon uses
// Listen followed by Serve so it can publish challenge routing only after the
// listener is known to be available.
func (s *Server) Start() error {
listener, err := s.Listen()
if err != nil {
return err
}
return s.Serve(listener)
}
// resolveTLSMaterial picks the cert / key pair the challenge listener
// should present. The WebUI fallback is only safe for direct/public binds;
// loopback and socket listeners stay plain HTTP unless challenge TLS is
// explicitly configured.
func (s *Server) resolveTLSMaterial() (cert, key string) {
if c, k := s.cfg.Challenge.TLSCert, s.cfg.Challenge.TLSKey; c != "" && k != "" {
return c, k
}
if localOnlyListener(s.cfg.Challenge.ListenAddr) {
return "", ""
}
if c, k := s.cfg.WebUI.TLSCert, s.cfg.WebUI.TLSKey; c != "" && k != "" {
return c, k
}
return "", ""
}
func isLoopbackListenAddr(addr string) bool {
ip := net.ParseIP(ListenHost(addr))
return ip != nil && ip.IsLoopback()
}
// localOnlyListener reports a listener only this host can reach: loopback
// or a Unix socket. Such listeners get no port gate and no Web UI TLS.
func localOnlyListener(addr string) bool {
if _, socket := config.ChallengeSocketPath(addr); socket {
return true
}
return isLoopbackListenAddr(addr)
}
// ListenHost gives the server, installer and diagnostics the same bind host.
// Localhost is pinned to IPv4 so proxy trust does not depend on DNS ordering.
func ListenHost(addr string) string {
host := strings.TrimSpace(addr)
host = strings.Trim(host, "[]")
if host == "" || strings.EqualFold(host, "localhost") {
return "127.0.0.1"
}
return canonicalIP(host)
}
// Shutdown gracefully stops the server and removes a socket it bound.
func (s *Server) Shutdown() {
_ = s.srv.Close()
s.socketMu.Lock()
defer s.socketMu.Unlock()
s.socketClosed = true
if s.socketListener != nil {
_ = s.socketListener.Close()
}
}
func (s *Server) handleChallenge(w http.ResponseWriter, r *http.Request) {
ip, ok := s.clientIP(w, r)
if !ok {
return
}
if !s.challengePending(ip) {
http.Error(w, "No challenge pending for this address", http.StatusNotFound)
return
}
// Bypass paths run before the PoW page is generated. Each
// short-circuits to the same markVerified flow, so a passing
// visitor never sees the challenge UI even once.
if method := s.bypassMethod(r, ip); method != "" {
s.markVerified(w, r, ip, "", method)
return
}
nonce := generateNonce()
difficulty := s.cfg.Challenge.Difficulty
token := s.makeToken(ip, nonce)
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
captchaBlock := s.captchaNoscriptHTML(token, nonce)
// #nosec G705 -- All interpolated values are constrained to non-HTML
// character sets: ip is validated via net.ParseIP / net.SplitHostPort
// in extractIP (never attacker-controlled string); nonce and token
// are hex.EncodeToString output (0-9, a-f only); difficulty is an int.
// captchaBlock is constructed from configured site keys (operator
// supplied) plus hex token/nonce; it never includes attacker input.
// html.EscapeString on ip is defence-in-depth so static analysers can
// see the request-derived value is rendered safe even if a future
// change to extractIP loosened validation.
fmt.Fprintf(w, challengePageHTML, html.EscapeString(ip), captchaBlock, nonce, token, difficulty, difficulty)
}
func (s *Server) handleGate(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
ip, ok := s.clientIP(w, r)
if !ok {
return
}
if s.bypassByAdminCookie(r, ip) || s.bypassByVerifiedCrawler(r.Context(), ip) || s.bypassByVerifyCookie(r, ip) {
w.WriteHeader(http.StatusNoContent)
return
}
if s.ipList == nil || !s.ipList.Contains(ip) {
w.WriteHeader(http.StatusNoContent)
return
}
w.WriteHeader(http.StatusUnauthorized)
}
// bypassByAdminCookie returns true when the visitor presents a valid
// signed-session cookie issued by THIS daemon for THIS IP. Daemon
// restart rotates the signing key, so old cookies fall back to the
// normal PoW flow automatically.
func (s *Server) bypassByAdminCookie(r *http.Request, ip string) bool {
if s.sessionSigner == nil {
return false
}
cookieName := s.cfg.Challenge.VerifiedSession.CookieName
if cookieName == "" {
cookieName = "csm_admin_session"
}
c, err := r.Cookie(cookieName)
if err != nil || c.Value == "" {
return false
}
return s.sessionSigner.Verify(c.Value, ip) == nil
}
// bypassByVerifiedCrawler resolves the visitor's reverse DNS and
// confirms the forward lookup -- skipping the PoW only for traffic
// from the configured crawler families. A spoofed UA from a residential
// IP fails forward-confirm and falls through to PoW.
func (s *Server) bypassByVerifiedCrawler(ctx context.Context, ip string) bool {
if s.crawlers == nil {
return false
}
return s.crawlers.Verified(ctx, ip)
}
func (s *Server) handleVerify(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
ip, ok := s.clientIP(w, r)
if !ok {
return
}
if !s.challengePending(ip) {
http.Error(w, "No challenge pending for this address", http.StatusNotFound)
return
}
nonce := r.FormValue("nonce")
token := r.FormValue("token")
solution := r.FormValue("solution")
// Verify HMAC token
expected := s.makeToken(ip, nonce)
if token != expected {
http.Error(w, "Invalid token", http.StatusForbidden)
return
}
// Verify proof-of-work solution
if !verifyPoW(nonce, solution, s.cfg.Challenge.Difficulty) {
http.Error(w, "Invalid solution", http.StatusForbidden)
return
}
// Prevent replay -- one nonce, one verification.
s.verifiedMu.Lock()
if _, seen := s.verified[nonce]; seen {
s.verifiedMu.Unlock()
http.Error(w, "Token already used", http.StatusForbidden)
return
}
s.verified[nonce] = time.Now()
s.verifiedMu.Unlock()
s.markVerified(w, r, ip, r.FormValue("dest"), passMethodPoW)
}
// handleCaptchaVerify accepts a provider token, validates it
// server-side, and (on success) puts the visitor through the same
// markVerified flow PoW uses. Available only when the operator has
// configured a CAPTCHA provider.
func (s *Server) handleCaptchaVerify(w http.ResponseWriter, r *http.Request) {
if s.captcha == nil {
http.NotFound(w, r)
return
}
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
ip, ok := s.clientIP(w, r)
if !ok {
return
}
if !s.challengePending(ip) {
http.Error(w, "No challenge pending for this address", http.StatusNotFound)
return
}
nonce := r.FormValue("nonce")
token := r.FormValue("token")
captchaToken := r.FormValue("captcha-token")
// Bind the CAPTCHA submission to the page token before asking the
// provider. The nonce is spent only after the provider accepts, so a
// rejected widget token can be retried from the same page.
if expected := s.makeToken(ip, nonce); token != expected {
http.Error(w, "Invalid token", http.StatusForbidden)
return
}
s.verifiedMu.Lock()
_, seen := s.verified[nonce]
s.verifiedMu.Unlock()
if seen {
http.Error(w, "Token already used", http.StatusForbidden)
return
}
ok, err := s.captcha.Verify(r.Context(), captchaToken, ip)
if err != nil || !ok {
http.Error(w, "CAPTCHA verification failed", http.StatusForbidden)
return
}
s.verifiedMu.Lock()
if _, seen := s.verified[nonce]; seen {
s.verifiedMu.Unlock()
http.Error(w, "Token already used", http.StatusForbidden)
return
}
s.verified[nonce] = time.Now()
s.verifiedMu.Unlock()
s.markVerified(w, r, ip, r.FormValue("dest"), passMethodCaptcha)
}
// handleAdminToken issues a signed-session cookie when the operator
// presents the configured admin_secret. Returns 204 with Set-Cookie on
// success, 403 on bad/missing secret, 429 once an IP has burned
// through adminMaxFailuresInWindow failures inside adminFailureWindow.
// The 429 is returned BEFORE the constant-time compare so an attacker
// cannot keep probing once they are throttled. Requests from this host's
// own addresses get 403 without a compare, and 503 while those addresses
// have never been read.
func (s *Server) handleAdminToken(w http.ResponseWriter, r *http.Request) {
if s.sessionSigner == nil {
http.NotFound(w, r)
return
}
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
ip, ok := s.clientIP(w, r)
if !ok {
return
}
// Every local account can send from loopback and this host's addresses,
// so a per-address failure budget there would not limit guessing.
switch local, known := netutil.IsHostLocal(net.ParseIP(ip)); {
case !known:
http.Error(w, "Cannot check this host's addresses; try again later.", http.StatusServiceUnavailable)
return
case local:
http.Error(w, "Admin sessions are not issued to this host's addresses.", http.StatusForbidden)
return
}
if s.adminRateLimited(ip) {
http.Error(w, "Too many failed attempts; try again later.", http.StatusTooManyRequests)
return
}
presented := r.FormValue("secret")
if !CompareAdminSecret(s.cfg.Challenge.VerifiedSession.AdminSecret, presented) {
s.recordAdminFailure(ip)
http.Error(w, "Forbidden", http.StatusForbidden)
return
}
// Successful auth clears the failure log so a legitimate operator
// who fat-fingered the secret a few times can keep using the
// endpoint after they get it right.
s.clearAdminFailures(ip)
cookieName := s.cfg.Challenge.VerifiedSession.CookieName
if cookieName == "" {
cookieName = "csm_admin_session"
}
cookie := &http.Cookie{
Name: cookieName,
Value: s.sessionSigner.Issue(ip),
Path: "/",
MaxAge: int(s.sessionSigner.TTL().Seconds()),
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteLaxMode,
}
http.SetCookie(w, cookie)
w.WriteHeader(http.StatusNoContent)
}
// adminRateLimited returns true when ip has at least
// adminMaxFailuresInWindow failures in the last adminFailureWindow.
// Also opportunistically prunes aged-out entries so the per-IP slice
// stays bounded.
func (s *Server) adminRateLimited(ip string) bool {
cutoff := time.Now().Add(-adminFailureWindow)
s.adminFailuresMu.Lock()
defer s.adminFailuresMu.Unlock()
pruned := s.adminFailures[ip][:0]
for _, t := range s.adminFailures[ip] {
if t.After(cutoff) {
pruned = append(pruned, t)
}
}
s.adminFailures[ip] = pruned
return len(pruned) >= adminMaxFailuresInWindow
}
func (s *Server) recordAdminFailure(ip string) {
s.adminFailuresMu.Lock()
defer s.adminFailuresMu.Unlock()
s.adminFailures[ip] = append(s.adminFailures[ip], time.Now())
}
func (s *Server) clearAdminFailures(ip string) {
s.adminFailuresMu.Lock()
defer s.adminFailuresMu.Unlock()
delete(s.adminFailures, ip)
}
// challengePending reports whether ip currently has a challenge to answer.
// Without a list nobody does.
func (s *Server) challengePending(ip string) bool {
return s.ipList != nil && s.ipList.Contains(ip)
}
// markVerified is the shared post-success path used by handleVerify,
// handleCaptchaVerify, and the bypass shortcuts in handleChallenge.
// Centralising the side effects (ipList removal, verification cookie,
// redirect render) keeps all four paths in sync; the alternative --
// copy-pasting four times -- is the easiest way to drift the behaviour of
// one path away from the others over time. Nothing here touches the
// firewall: a passed challenge proves a browser, not a trustworthy client,
// so the visitor goes back to the ordinary rules like everyone else. The
// list remembers the pass, so evidence the visitor already answered does
// not turn into a timeout block.
func (s *Server) markVerified(w http.ResponseWriter, r *http.Request, ip, destOverride, method string) {
if s.ipList.MarkPassed(ip) {
observeChallengePassed(method)
fmt.Fprintf(os.Stderr, "[%s] CHALLENGE-PASS: %s passed (%s)\n", time.Now().Format("2006-01-02 15:04:05"), ip, method)
}
// Set verification cookie so a visitor who is listed again inside the
// window skips the puzzle. Secure is always on: CSM is designed to run
// behind HTTPS and the cookie grants a multi-hour bypass of the PoW
// gate, so leaking it over plaintext is never acceptable.
if s.verifySigner != nil {
http.SetCookie(w, &http.Cookie{
Name: "csm_verified",
Value: s.verifySigner.Issue(ip),
Path: "/",
MaxAge: int(verifyCookieTTL.Seconds()),
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteLaxMode,
})
}
dest := sanitizeRedirectDest(destOverride, r.Host)
w.Header().Set("Content-Type", "text/html; charset=utf-8")
fmt.Fprintf(w, `<!DOCTYPE html><html><head><meta http-equiv="refresh" content="2;url=%s">
<style>body{font-family:system-ui;display:flex;justify-content:center;align-items:center;min-height:100vh;margin:0;background:#1a2234;color:#c8d3e0}
.ok{text-align:center}.ok h1{color:#2fb344;font-size:3em}p{font-size:1.2em}</style>
</head><body><div class="ok"><h1>✓</h1><p>Verified - redirecting...</p></div></body></html>`, html.EscapeString(dest))
}
// captchaNoscriptHTML renders the CAPTCHA fallback widget. Returns the
// empty string when no provider is configured, in which case the
// challenge page's <noscript> block falls back to the existing
// "JavaScript is required" message.
//
// token and nonce are guaranteed hex by upstream constructors; siteKey
// is operator-supplied so we escape it to keep a typo from breaking
// the page (an attacker would need write access to csm.yaml to inject
// HTML here, which is already game-over, but defence-in-depth is
// cheap).
func (s *Server) captchaNoscriptHTML(token, nonce string) string {
if s.captcha == nil {
return ""
}
siteKey := html.EscapeString(s.cfg.Challenge.CaptchaFallback.SiteKey)
switch s.captcha.Name() {
case "turnstile":
return fmt.Sprintf(`<script src="https://challenges.cloudflare.com/turnstile/v0/api.js" async defer></script>
<form method="POST" action="/challenge/captcha-verify">
<input type="hidden" name="token" value="%s">
<input type="hidden" name="nonce" value="%s">
<div class="cf-turnstile" data-sitekey="%s" data-callback="csmCaptchaCallback"></div>
<input type="hidden" name="captcha-token" id="captchaToken">
</form>
<script>function csmCaptchaCallback(t){document.getElementById('captchaToken').value=t;document.forms[0].submit();}</script>`,
token, nonce, siteKey)
case "hcaptcha":
return fmt.Sprintf(`<script src="https://js.hcaptcha.com/1/api.js" async defer></script>
<form method="POST" action="/challenge/captcha-verify">
<input type="hidden" name="token" value="%s">
<input type="hidden" name="nonce" value="%s">
<div class="h-captcha" data-sitekey="%s" data-callback="csmCaptchaCallback"></div>
<input type="hidden" name="captcha-token" id="captchaToken">
</form>
<script>function csmCaptchaCallback(t){document.getElementById('captchaToken').value=t;document.forms[0].submit();}</script>`,
token, nonce, siteKey)
default:
return ""
}
}
func (s *Server) makeToken(ip, nonce string) string {
mac := hmac.New(sha256.New, s.secret)
mac.Write([]byte(ip + ":" + nonce))
return hex.EncodeToString(mac.Sum(nil))
}
// bypassByVerifyCookie returns true when the visitor presents a valid
// csm_verified cookie issued by this daemon for this IP. The cookie binds
// to one IP and an expiry, so it cannot be replayed from another network or
// after the allow window; a daemon restart rotates the signing key and
// invalidates every outstanding cookie.
func (s *Server) bypassByVerifyCookie(r *http.Request, ip string) bool {
if s.verifySigner == nil {
return false
}
c, err := r.Cookie("csm_verified")
if err != nil || c.Value == "" {
return false
}
return s.verifySigner.Verify(c.Value, ip) == nil
}
// CleanExpired removes old verification records, prunes the
// admin-failure log, and evicts stale crawler-cache entries. Called
// from the daemon's challengeEscalator ticker every 60 seconds; under
// a sustained scan from many source IPs, this is the only thing
// keeping per-IP map entries from accumulating until restart.
func (s *Server) CleanExpired() {
now := time.Now()
s.verifiedMu.Lock()
verifiedCutoff := now.Add(-4 * time.Hour)
for k, t := range s.verified {
if t.Before(verifiedCutoff) {
delete(s.verified, k)
}
}
s.verifiedMu.Unlock()
// Drop admin-failure entries whose latest failure has aged out of
// the rate-limit window. An IP that hammered the endpoint once
// and stopped will otherwise sit in the map forever.
s.adminFailuresMu.Lock()
failureCutoff := now.Add(-adminFailureWindow)
for ip, times := range s.adminFailures {
kept := times[:0]
for _, t := range times {
if t.After(failureCutoff) {
kept = append(kept, t)
}
}
if len(kept) == 0 {
delete(s.adminFailures, ip)
} else {
s.adminFailures[ip] = kept
}
}
s.adminFailuresMu.Unlock()
if s.crawlers != nil {
s.crawlers.cleanExpired(now)
}
}
// clientIP rejects a request that names no client before any challenge or
// bypass decision. A trusted proxy's own address identifies a shared
// transport, not a visitor, so a pass or cookie for it could bypass
// challenges for unrelated visitors.
func (s *Server) clientIP(w http.ResponseWriter, r *http.Request) (string, bool) {
ip := s.extractIP(r)
// Test the list itself, not trust: a listed address the host-local guard
// refused to trust still names a shared transport, not a visitor.
if ip == "" || s.trusted.Covers(netip.MustParseAddr(ip)) {
http.Error(w, "Cannot identify the client behind the trusted proxy", http.StatusBadRequest)
return "", false
}
return ip, true
}
// extractIP returns the client of the request, or "" when there is none.
// X-Forwarded-For is read only behind a trusted proxy, right to
// left past further trusted proxies; proxies can append a field line instead
// of extending the first, so all lines are read. An unreadable, overlong or
// all-trusted chain names nobody: none falls back to the proxy. A listed
// address of this host is not trusted and is returned as itself; clientIP
// refuses it. On a socket the peer is the proxy and has no address.
func (s *Server) extractIP(r *http.Request) string {
xff := strings.Join(r.Header.Values("X-Forwarded-For"), ",")
var res clientaddr.Result
var err error
if s.socket {
res, err = clientaddr.ResolveBehindProxy(xff, s.trusted)
} else {
res, err = clientaddr.Resolve(r.RemoteAddr, xff, true, s.trusted)
}
if err != nil {
return ""
}
return res.Client.String()
}
// canonicalIP normalizes an IP string so downstream exact-match lookups (the
// firewall's blocked-IP index) agree on one form. A dual-stack listener
// reports an IPv4 peer as "::ffff:1.2.3.4"; net.IP.String collapses that to
// the plain IPv4 form the firewall stores. Non-parseable input is returned
// unchanged.
func canonicalIP(ip string) string {
if parsed := net.ParseIP(ip); parsed != nil {
return parsed.String()
}
return ip
}
func generateNonce() string {
b := make([]byte, 16)
_, _ = rand.Read(b)
return hex.EncodeToString(b)
}
// sanitizeRedirectDest validates that a redirect destination is a safe same-origin
// relative path or absolute URL matching the request host. Rejects cross-origin
// redirects, javascript: URIs, backslash-based bypasses, and other open-redirect payloads.
// Returns a reconstructed URL from parsed components to prevent any raw-string injection.
func sanitizeRedirectDest(dest, requestHost string) string {
if dest == "" {
return "/"
}
// Parse through url.Parse to normalize and detect scheme/host
parsed, err := url.Parse(dest)
if err != nil {
return "/"
}
// Scheme whitelist — applies even for opaque URLs with empty Host.
// Without this, `javascript:alert(1)` produces an opaque URL with
// Host="" and Scheme="javascript", which would slip past the
// host-equality check below and end up reconstructed as
// `"javascript:"`. The only acceptable schemes are the empty
// string (pure-path relatives) and the two HTTP variants.
scheme := strings.ToLower(parsed.Scheme)
if scheme != "" && scheme != "http" && scheme != "https" {
return "/"
}
// Reject anything with a host component that doesn't match the request host.
// This catches protocol-relative (//evil.com), backslash tricks (/\evil.com
// which some browsers normalize to //evil.com), and explicit cross-origin URLs.
if parsed.Host != "" {
destHost := parsed.Hostname()
reqHost := requestHost
if h, _, err := net.SplitHostPort(requestHost); err == nil {
reqHost = h
}
if destHost != reqHost {
return "/"
}
}
// For relative paths: reject anything that doesn't start with a clean /
if parsed.Host == "" && parsed.Scheme == "" {
if !strings.HasPrefix(parsed.Path, "/") {
return "/"
}
// Reject backslash in path (browser normalization attack)
if strings.ContainsRune(parsed.Path, '\\') {
return "/"
}
}
// Reconstruct from parsed components to prevent raw-string injection
safe := &url.URL{
Scheme: parsed.Scheme,
Host: parsed.Host,
Path: parsed.Path,
RawQuery: parsed.RawQuery,
Fragment: parsed.Fragment,
}
return safe.String()
}
// verifyPoW checks that SHA256(nonce + solution) starts with `difficulty` zero nibbles.
func verifyPoW(nonce, solution string, difficulty int) bool {
h := sha256.Sum256([]byte(nonce + solution))
hexHash := hex.EncodeToString(h[:])
for i := 0; i < difficulty; i++ {
if i >= len(hexHash) || hexHash[i] != '0' {
return false
}
}
return true
}
package challenge
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"encoding/binary"
"errors"
"strings"
"time"
)
// AdminSessionSigner mints and verifies the signed cookies that let
// authenticated operators bypass the PoW. The signing key is generated
// on construction; rebuilding the signer (i.e., daemon restart)
// invalidates every previously-issued cookie -- the rotation contract.
type AdminSessionSigner struct {
key []byte
ttl time.Duration
}
// ErrSessionExpired is returned by Verify for cookies whose embedded
// expiry has passed.
var ErrSessionExpired = errors.New("session expired")
// ErrSessionBadSignature is returned by Verify for cookies whose HMAC
// does not match. Includes tampered payloads and cookies signed by a
// previous AdminSessionSigner instance (post-rotation).
var ErrSessionBadSignature = errors.New("session signature invalid")
// ErrSessionIPMismatch is returned when the cookie was issued for a
// different IP than the one presenting it. Stops cookie theft from a
// different network.
var ErrSessionIPMismatch = errors.New("session IP mismatch")
// ErrSessionMalformed wraps decoding errors so a corrupt cookie has a
// distinct sentinel from a tampered one.
var ErrSessionMalformed = errors.New("session payload malformed")
// NewAdminSessionSigner generates a fresh 32-byte signing key. The
// caller must keep the returned pointer for the lifetime of the
// challenge server; never construct a second signer for the same
// server, or already-issued cookies will be invalidated mid-session.
func NewAdminSessionSigner(ttl time.Duration) (*AdminSessionSigner, error) {
if ttl <= 0 {
ttl = 4 * time.Hour
}
key := make([]byte, 32)
if _, err := rand.Read(key); err != nil {
return nil, err
}
return &AdminSessionSigner{key: key, ttl: ttl}, nil
}
// TTL exposes the configured cookie lifetime so the server can set the
// matching Max-Age on the Set-Cookie header.
func (s *AdminSessionSigner) TTL() time.Duration { return s.ttl }
// Issue returns a cookie value of the form "<base64(payload)>.<base64 hmac>".
// The payload binds the cookie to a single IP and a single expiry so a
// stolen cookie does not work elsewhere or after the TTL.
func (s *AdminSessionSigner) Issue(ip string) string {
return s.issueAt(ip, time.Now().Add(s.ttl))
}
// issueAt is the test seam for issuing a cookie with an explicit
// expiry. Lets expiry tests construct already-expired cookies without
// reaching into encodeSessionPayload directly.
func (s *AdminSessionSigner) issueAt(ip string, exp time.Time) string {
payload := encodeSessionPayload(ip, exp)
mac := hmac.New(sha256.New, s.key)
mac.Write(payload)
sig := mac.Sum(nil)
return base64.RawURLEncoding.EncodeToString(payload) + "." + base64.RawURLEncoding.EncodeToString(sig)
}
// Verify checks the HMAC, payload format, expiry, and IP binding. Use
// errors.Is to branch on the failure mode.
func (s *AdminSessionSigner) Verify(cookieValue, ip string) error {
dot := strings.LastIndexByte(cookieValue, '.')
if dot <= 0 || dot == len(cookieValue)-1 {
return ErrSessionMalformed
}
payloadEnc := cookieValue[:dot]
sigEnc := cookieValue[dot+1:]
payload, err := base64.RawURLEncoding.DecodeString(payloadEnc)
if err != nil {
return ErrSessionMalformed
}
sig, err := base64.RawURLEncoding.DecodeString(sigEnc)
if err != nil {
return ErrSessionMalformed
}
mac := hmac.New(sha256.New, s.key)
mac.Write(payload)
expected := mac.Sum(nil)
if subtle.ConstantTimeCompare(sig, expected) != 1 {
return ErrSessionBadSignature
}
cookieIP, exp, err := decodeSessionPayload(payload)
if err != nil {
return ErrSessionMalformed
}
if time.Now().After(exp) {
return ErrSessionExpired
}
if cookieIP != ip {
return ErrSessionIPMismatch
}
return nil
}
// CompareAdminSecret returns true when stored and presented secrets
// match in constant time. Empty stored secret always returns false so a
// misconfigured admin_secret cannot accidentally accept any caller.
func CompareAdminSecret(stored, presented string) bool {
if stored == "" {
return false
}
return subtle.ConstantTimeCompare([]byte(stored), []byte(presented)) == 1
}
// encodeSessionPayload formats: 1 byte version | 8 byte unix expiry BE
// | n byte IP. Variable-length tail keeps it simple; the IP read stops
// at end-of-buffer.
func encodeSessionPayload(ip string, exp time.Time) []byte {
out := make([]byte, 0, 9+len(ip))
out = append(out, 1)
var ts [8]byte
// #nosec G115 -- exp is always future-dated (now + positive TTL via NewAdminSessionSigner / Issue), so Unix() is positive and the int64->uint64 cast cannot overflow.
binary.BigEndian.PutUint64(ts[:], uint64(exp.Unix()))
out = append(out, ts[:]...)
out = append(out, []byte(ip)...)
return out
}
func decodeSessionPayload(p []byte) (ip string, exp time.Time, err error) {
if len(p) < 9 || p[0] != 1 {
return "", time.Time{}, ErrSessionMalformed
}
tsRaw := binary.BigEndian.Uint64(p[1:9])
if tsRaw > uint64(1<<62) {
return "", time.Time{}, ErrSessionMalformed
}
return string(p[9:]), time.Unix(int64(tsRaw), 0), nil
}
package challenge
import (
"errors"
"fmt"
"net"
"os"
"os/user"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/pidginhost/csm/internal/config"
)
const maxSocketPath = config.ChallengeSocketMaxPath
// CheckSocketPlacement checks where the challenge socket would live: an
// absolute path a client can connect to, a group that exists, a parent owned
// by the daemon's user that nobody else can write (so nobody can plant or
// swap the path), and nothing but a socket at the path. The server and
// doctor share it so their rules cannot drift.
func CheckSocketPlacement(path, group string) (gid int, err error) {
if pathErr := config.ChallengeSocketPathError(path); pathErr != nil {
return 0, pathErr
}
g, err := user.LookupGroup(group)
if err != nil {
return 0, fmt.Errorf("challenge.listen_group %q: %v", group, err)
}
gid, err = strconv.Atoi(g.Gid)
if err != nil {
return 0, fmt.Errorf("challenge.listen_group %q: gid %q: %v", group, g.Gid, err)
}
parent := filepath.Dir(path)
fi, err := os.Lstat(parent)
if err != nil {
return 0, fmt.Errorf("challenge socket directory %s: %w", parent, err)
}
st, ok := fi.Sys().(*syscall.Stat_t)
if !fi.IsDir() || !ok || int64(st.Uid) != int64(os.Geteuid()) || fi.Mode().Perm()&0o022 != 0 {
return 0, fmt.Errorf("challenge socket directory %s must be a directory owned by uid %d that group and others cannot write", parent, os.Geteuid())
}
if err := checkSocketAncestors(parent); err != nil {
return 0, err
}
switch fi, err := os.Lstat(path); {
case err == nil && fi.Mode()&os.ModeSocket == 0:
return 0, fmt.Errorf("challenge socket path %s exists and is not a socket; move it away", path)
case err != nil && !errors.Is(err, os.ErrNotExist):
return 0, fmt.Errorf("challenge socket path %s: %w", path, err)
}
return gid, nil
}
// A protected parent is insufficient if a tenant can replace it through a
// writable ancestor. Follow only trusted links and directories, including
// the directories used to resolve symlink targets and dot components.
func checkSocketAncestors(parent string) error {
current, rest, links := "/", strings.Split(strings.TrimPrefix(parent, "/"), "/"), 0
for {
fi, err := os.Lstat(current)
if err != nil {
return fmt.Errorf("challenge socket directory %s: %w", current, err)
}
st, ok := fi.Sys().(*syscall.Stat_t)
if !ok || !fi.IsDir() || (st.Uid != 0 && int64(st.Uid) != int64(os.Geteuid())) ||
(fi.Mode().Perm()&0o022 != 0 && fi.Mode()&os.ModeSticky == 0) {
return fmt.Errorf("challenge socket directory %s can be replaced by another user; use a protected directory", current)
}
if len(rest) == 0 {
return nil
}
name := rest[0]
rest = rest[1:]
switch name {
case "", ".":
continue
case "..":
current = filepath.Dir(current)
continue
}
next := filepath.Join(current, name)
fi, err = os.Lstat(next)
if err != nil {
return fmt.Errorf("challenge socket directory %s: %w", next, err)
}
st, ok = fi.Sys().(*syscall.Stat_t)
if !ok || (st.Uid != 0 && int64(st.Uid) != int64(os.Geteuid())) {
return fmt.Errorf("challenge socket directory %s must be owned by root or uid %d", next, os.Geteuid())
}
if fi.Mode()&os.ModeSymlink == 0 {
current = next
continue
}
links++
if links > 40 {
return fmt.Errorf("challenge socket directory %s: too many symbolic links", next)
}
target, err := os.Readlink(next)
if err != nil {
return fmt.Errorf("challenge socket directory %s: %w", next, err)
}
if filepath.IsAbs(target) {
current = "/"
}
rest = append(strings.Split(target, "/"), rest...)
}
}
type socketListener struct {
*net.UnixListener
path string
info os.FileInfo
once sync.Once
err error
}
func (l *socketListener) Addr() net.Addr {
return &net.UnixAddr{Name: l.path, Net: "unix"}
}
func (l *socketListener) Close() error {
l.once.Do(func() {
// Another server or an operator may have replaced the path. A
// repeated Close must not remove that replacement either. Keep the
// bound inode alive until after the check to prevent inode reuse.
if fi, err := os.Lstat(l.path); err == nil && os.SameFile(l.info, fi) {
if err := os.Remove(l.path); err != nil && !errors.Is(err, os.ErrNotExist) {
l.err = err
}
}
l.err = errors.Join(l.err, l.UnixListener.Close())
})
return l.err
}
func checkStaleSocket(path string) error {
fi, err := os.Lstat(path)
if errors.Is(err, os.ErrNotExist) {
return nil
}
if err != nil {
return fmt.Errorf("challenge socket path %s: %w", path, err)
}
if fi.Mode()&os.ModeSocket == 0 {
return fmt.Errorf("challenge socket path %s exists and is not a socket; move it away", path)
}
conn, err := net.DialTimeout("unix", path, time.Second)
if err == nil {
_ = conn.Close()
return fmt.Errorf("challenge socket %s is already listening", path)
}
if !errors.Is(err, syscall.ECONNREFUSED) {
return fmt.Errorf("challenge socket %s cannot be confirmed stale: %w", path, err)
}
return nil
}
// listenUnixSocket binds the challenge socket so that no local user outside
// group can connect at any moment: it binds inside a fresh 0700 directory,
// sets group and mode there, then renames the socket into place, which also
// replaces a stale socket atomically.
func listenUnixSocket(path, group string) (net.Listener, error) {
gid, err := CheckSocketPlacement(path, group)
if err != nil {
return nil, err
}
tmpDir, err := os.MkdirTemp(filepath.Dir(path), config.ChallengeSocketBindPrefix)
if err != nil {
return nil, fmt.Errorf("challenge socket: %w", err)
}
defer func() { _ = os.RemoveAll(tmpDir) }()
tmp := filepath.Join(tmpDir, "s")
if len(tmp) > maxSocketPath {
return nil, fmt.Errorf("challenge.listen_addr: socket path too long for the %d-byte limit: %s leaves no room to create it safely", maxSocketPath, filepath.Dir(path))
}
ln, err := net.ListenUnix("unix", &net.UnixAddr{Name: tmp, Net: "unix"})
if err != nil {
return nil, fmt.Errorf("challenge socket bind: %w", err)
}
fail := func(err error) (net.Listener, error) {
_ = ln.Close()
return nil, err
}
if chownErr := os.Lchown(tmp, -1, gid); chownErr != nil {
return fail(fmt.Errorf("challenge socket group: %w", chownErr))
}
if aclErr := clearSocketACL(tmp); aclErr != nil {
return fail(fmt.Errorf("challenge socket permissions: %w", aclErr))
}
// #nosec G302 -- the proxy's group must be able to connect; others cannot.
if chmodErr := os.Chmod(tmp, 0o660); chmodErr != nil {
return fail(fmt.Errorf("challenge socket mode: %w", chmodErr))
}
info, err := os.Lstat(tmp)
if err != nil {
return fail(fmt.Errorf("challenge socket identity: %w", err))
}
if err := checkStaleSocket(path); err != nil {
return fail(err)
}
if err := os.Rename(tmp, path); err != nil {
return fail(fmt.Errorf("challenge socket: %w", err))
}
// The wrapper owns cleanup of the published name on every close path.
ln.SetUnlinkOnClose(false)
return &socketListener{UnixListener: ln, path: path, info: info}, nil
}
//go:build linux
package challenge
import (
"errors"
"golang.org/x/sys/unix"
)
// chmod changes an ACL's mask, not its named grants. Clear inherited grants
// while the socket is still hidden so only its owner and listen_group get
// access when it is published.
func clearSocketACL(path string) error {
err := unix.Lremovexattr(path, "system.posix_acl_access")
if errors.Is(err, unix.ENODATA) || errors.Is(err, unix.EOPNOTSUPP) {
return nil
}
return err
}
package checks
import (
"errors"
"os"
"path/filepath"
"sort"
"strings"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/control"
)
// accountSuspended reports whether cPanel has suspended an account. cPanel
// locks the account's database users at suspension, so every query against its
// installs fails with an access error; scanning one produces no coverage, only
// a gap that no operator action can clear. A name that is not a single path
// element is rejected rather than resolved, so it cannot escape the directory.
func accountSuspended(account string) bool {
if account == "" || account == "." || account == ".." || strings.ContainsAny(account, `/\`) {
return false
}
_, err := osFS.Stat(filepath.Join("/var/cpanel/suspended", account))
return err == nil
}
// EnumerateScanAccounts returns the sorted list of cPanel account usernames
// eligible for a server-wide scan. Source of truth: the cPanel user registry
// (/var/cpanel/users) intersected with present home directories (/home/<user>).
// All filesystem access goes through the package osFS hook so it is fakeable.
//
// Fallback: when /var/cpanel/users is absent (non-cPanel platform), the
// function falls back to enumerating /home subdirectories whose names pass
// name validation. This makes the function usable on generic Linux hosts.
//
// A hard FS error reading the registry (not os.ErrNotExist) is propagated as
// an error. An empty registry returns ([]string{}, nil).
func EnumerateScanAccounts(_ *config.Config) ([]string, error) {
registryEntries, err := osFS.ReadDir("/var/cpanel/users")
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
if errors.Is(err, os.ErrNotExist) {
// Non-cPanel platform: fall back to /home subdirectories.
return enumerateFromHome()
}
// Build a set of names present under the account roots for the
// intersection step.
homes, _ := listAccountHomes()
homeSet := make(map[string]struct{}, len(homes))
for _, h := range homes {
homeSet[h.Name()] = struct{}{}
}
seen := make(map[string]struct{}, len(registryEntries))
var accounts []string
for _, e := range registryEntries {
name := e.Name()
if !control.ValidScanAccountTarget(name) {
continue
}
if _, inHome := homeSet[name]; !inHome {
continue
}
if _, dup := seen[name]; dup {
continue
}
seen[name] = struct{}{}
accounts = append(accounts, name)
}
sort.Strings(accounts)
if accounts == nil {
accounts = []string{}
}
return accounts, nil
}
// enumerateFromHome lists /home subdirectory names that pass name validation.
// Used when the cPanel registry is absent. Because /home is the sole source of
// truth in this path, a hard read error (anything other than os.ErrNotExist) is
// propagated so callers can distinguish a broken enumeration from an empty host.
func enumerateFromHome() ([]string, error) {
homes, err := listAccountHomes()
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
homeEntries := make([]os.DirEntry, 0, len(homes))
for _, h := range homes {
homeEntries = append(homeEntries, h.Entry)
}
seen := make(map[string]struct{}, len(homeEntries))
var accounts []string
for _, e := range homeEntries {
name := e.Name()
if !control.ValidScanAccountTarget(name) {
continue
}
if _, dup := seen[name]; dup {
continue
}
seen[name] = struct{}{}
accounts = append(accounts, name)
}
sort.Strings(accounts)
if accounts == nil {
accounts = []string{}
}
return accounts, nil
}
package checks
import (
"strings"
"sync"
"time"
)
// Domain->owner is read from cPanel's /etc/userdomains ("domain: user").
// Cached for ownerCacheTTL so a busy host does not re-read it per scan.
const ownerCacheTTL = 60 * time.Second
var (
ownerMu sync.Mutex
ownerMap map[string]string
ownerLoadedAt time.Time
)
// resetDomainOwnerCache clears the cache. Test-only seam.
func resetDomainOwnerCache() {
ownerMu.Lock()
ownerMap = nil
ownerLoadedAt = time.Time{}
ownerMu.Unlock()
}
// domainAccountOwner returns the cPanel account that owns domain, or "" when
// the map is unavailable (non-cPanel) or the domain is not present. The
// wildcard "*: nobody" line and any non "domain: user" line are ignored.
func domainAccountOwner(domain string) string {
domain = strings.ToLower(strings.TrimSpace(domain))
if domain == "" {
return ""
}
ownerMu.Lock()
defer ownerMu.Unlock()
if ownerMap == nil || time.Since(ownerLoadedAt) > ownerCacheTTL {
ownerMap = loadDomainOwners()
ownerLoadedAt = time.Now()
}
return ownerMap[domain]
}
func loadDomainOwners() map[string]string {
data, err := osFS.ReadFile("/etc/userdomains")
if err != nil {
return make(map[string]string)
}
return parseUserDomains(data)
}
// parseUserDomains preserves the legacy last-row-wins domain lookup.
func parseUserDomains(data []byte) map[string]string {
out := make(map[string]string)
for _, line := range strings.Split(string(data), "\n") {
domain, owner := parseUserDomain(line)
if domain != "" && owner != "" && owner != "nobody" {
out[domain] = owner
}
}
return out
}
// parseUserDomain preserves empty and unlisted owners so strict readers can
// distinguish an unresolved domain from an absent one. Empty and wildcard rows
// are ignored.
func parseUserDomain(line string) (string, string) {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
return "", ""
}
idx := strings.LastIndexByte(line, ':')
if idx <= 0 {
return "", ""
}
domain := strings.ToLower(strings.TrimSpace(line[:idx]))
owner := strings.TrimSpace(line[idx+1:])
if domain == "" || domain == "*" {
return "", ""
}
return domain, owner
}
package checks
import (
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/platform"
)
// accountOwnerForDomain is the production mapping: cPanel's /etc/userdomains
// file, which misses on every other panel. Tests in other packages inject a
// table through SetAccountOwnerLookupForTest.
var accountOwnerForDomain = func(domain string) (string, bool) {
if !platform.Detect().IsCPanel() {
return "", false
}
owner := domainAccountOwner(domain)
return owner, owner != ""
}
// AccountOwnerForDomain resolves the hosting account that owns a mail
// domain, for producers that know a verified local mailbox or domain.
// Correlation never calls it; owners are resolved where the mailbox is known.
func AccountOwnerForDomain(domain string) (string, bool) {
return accountOwnerForDomain(domain)
}
// SetAccountOwnerLookupForTest replaces the domain-to-owner mapping and
// returns a function that restores it. Test-only seam for packages that
// cannot reach this package's filesystem fakes.
func SetAccountOwnerLookupForTest(fn func(domain string) (string, bool)) func() {
prev := accountOwnerForDomain
accountOwnerForDomain = fn
return func() { accountOwnerForDomain = prev }
}
// MailOwner returns the hosting account for a mailbox or bare domain, or ""
// when the mapping is unavailable. It never returns the mailbox or domain
// itself as an owner.
func MailOwner(mailboxOrDomain string) string {
domain := strings.TrimSpace(mailboxOrDomain)
if at := strings.LastIndexByte(domain, '@'); at >= 0 {
domain = domain[at+1:]
}
if domain == "" {
return ""
}
owner, _ := AccountOwnerForDomain(domain)
return owner
}
// hostingAccountForUser is the production mapping from a system user name
// to a hosting account. Tests in other packages inject a table through
// SetHostingAccountLookupForTest.
var hostingAccountForUser = func(name string) string {
if name == "" || name == "root" || name == "unknown" || strings.ContainsAny(name, "/\\") {
return ""
}
home := defaultUIDCache.HomeDir(name)
if !filepath.IsAbs(home) {
return ""
}
if root, account, ok := accountRootOf(filepath.Join(home, "probe")); ok && account == name && filepath.Clean(home) == filepath.Join(root, account) {
return name
}
return ""
}
// HostingAccountForUser returns name when it is a hosting account: its
// passwd home directory sits directly under a configured account root.
// Root, system and service users, unknown names and the lookup sentinel
// resolve to "". The passwd cache is the same one process findings use for
// uid resolution, so tests point both at one fixture file.
func HostingAccountForUser(name string) string {
return hostingAccountForUser(name)
}
// SetHostingAccountLookupForTest replaces the user-to-account mapping and
// returns a function that restores it. Test-only seam for packages that
// cannot reach this package's passwd and account-root fakes.
func SetHostingAccountLookupForTest(fn func(name string) string) func() {
prev := hostingAccountForUser
hostingAccountForUser = fn
return func() { hostingAccountForUser = prev }
}
// ftpAccountOwner resolves an FTP login name: a virtual account is
// user@domain and belongs to the domain's owner; anything else is a system
// account name that must itself be a hosting account.
func ftpAccountOwner(account string) string {
if strings.Contains(account, "@") {
return MailOwner(account)
}
return HostingAccountForUser(account)
}
// accountHomeExists reports whether a configured account root contains a
// directory named user and passwd identifies it as the user's home. A
// leftover directory must not promote a service user to a hosting owner.
func accountHomeExists(user string) bool {
if HostingAccountForUser(user) == "" {
return false
}
for _, root := range accountHomeRoots() {
if info, err := osFS.Stat(filepath.Join(root, user)); err == nil && info.IsDir() {
return true
}
}
return false
}
// installOwner resolves the hosting account that owns a CMS install from
// its configuration path. ok is false outside every account root; callers
// keep their display label but must not stamp a tenant.
func installOwner(configPath string) (string, bool) {
_, account, ok := accountRootOf(configPath)
return account, ok
}
package checks
import (
"os"
"path/filepath"
"sort"
"strings"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
)
// accountHomeRoots answers "where do hosting accounts live" for every
// account-scoped check, remediation and re-check. A seam so tests can point
// the package at a temp tree; production follows platform detection.
var accountHomeRoots = func() []string { return platform.Detect().AccountHomeRoots() }
// accountHome is one account directory under one of the roots.
type accountHome struct {
Root string
Entry os.DirEntry
}
// Path is the account's home directory.
func (h accountHome) Path() string { return filepath.Join(h.Root, h.Entry.Name()) }
// Name is the account name.
func (h accountHome) Name() string { return h.Entry.Name() }
// listAccountHomes provides a best-effort account inventory. Stateful scanners
// use readAccountHomes so a partial inventory cannot retire unseen findings.
func listAccountHomes() ([]accountHome, error) {
homes, err := readAccountHomes()
if len(homes) > 0 {
return homes, nil
}
return homes, err
}
// readAccountHomes skips absent roots, but reports other read failures even
// when another root supplies accounts.
func readAccountHomes() ([]accountHome, error) {
var homes []accountHome
var firstErr error
for _, root := range accountHomeRoots() {
entries, err := osFS.ReadDir(root)
if err != nil {
if !os.IsNotExist(err) && firstErr == nil {
firstErr = err
}
continue
}
for _, e := range entries {
homes = append(homes, accountHome{Root: root, Entry: e})
}
}
return homes, firstErr
}
// AccountHomeRoots returns the directories hosting accounts live under on
// this platform: /home, or /var/www/vhosts on Plesk.
func AccountHomeRoots() []string {
return accountHomeRoots()
}
// protectHomeDirs are the trees ProtectHome=yes already hides from a unit.
var protectHomeDirs = []string{"/home", "/root", "/run/user"}
// AccountHomeVolumes returns the directories holding account homes that
// ProtectHome does not hide from a unit: the account roots, and the sibling
// home volumes registered cPanel accounts live on.
func AccountHomeVolumes() []string {
roots := accountHomeRoots()
dirs := make(map[string]bool)
for _, root := range roots {
dirs[filepath.Clean(root)] = true
}
if entries, err := osFS.ReadDir("/var/cpanel/users"); err == nil {
for _, e := range entries {
if home, ok := registeredAccountHome(roots, e.Name()); ok {
dirs[filepath.Dir(home)] = true
}
}
}
var out []string
for dir := range dirs {
hidden := false
for _, p := range protectHomeDirs {
if dir == p || strings.HasPrefix(dir, p+"/") {
hidden = true
break
}
}
if !hidden {
out = append(out, dir)
}
}
sort.Strings(out)
return out
}
func accountContentDir(name string) bool {
return !strings.HasPrefix(name, ".") && name != "mail" && name != "etc" &&
name != "logs" && name != "ssl" && name != "tmp"
}
// accountHomeDir resolves an account's home directory: the first root that
// holds it, then a registered cPanel account's home on another partition,
// or the first root when it exists nowhere (callers that need existence
// check it themselves).
func accountHomeDir(account string) string {
roots := accountHomeRoots()
if home, ok := accountHomeUnder(roots, account); ok {
return home
}
if home, ok := registeredAccountHome(roots, account); ok {
return home
}
return defaultAccountHome(roots, account)
}
// AccountHomeDirIn is accountHomeDir over the given roots, without the
// registry lookup.
func AccountHomeDirIn(roots []string, account string) string {
if home, ok := accountHomeUnder(roots, account); ok {
return home
}
return defaultAccountHome(roots, account)
}
func accountHomeUnder(roots []string, account string) (string, bool) {
for _, root := range roots {
candidate := filepath.Join(root, account)
if _, err := osFS.Stat(candidate); err == nil {
return candidate, true
}
}
return "", false
}
func defaultAccountHome(roots []string, account string) string {
if len(roots) == 0 {
return filepath.Join("/home", account)
}
return filepath.Join(roots[0], account)
}
// registeredAccountHome resolves a cPanel account that lives outside every
// account root: HOMEMATCH places new accounts on any matching partition
// (/home2, /home3, ...). The registry decides the name is an account and
// passwd names its home; the home must be a real directory named after the
// account, so a redirected or symlinked home is never scanned in its name.
func registeredAccountHome(roots []string, account string) (string, bool) {
if !validAccountName.MatchString(account) {
return "", false
}
if _, err := osFS.Stat(filepath.Join("/var/cpanel/users", account)); err != nil {
return "", false
}
home := defaultUIDCache.HomeDir(account)
if !filepath.IsAbs(home) || filepath.Clean(home) != home || filepath.Base(home) != account {
return "", false
}
// Only extend the configured home-volume boundary to a sibling volume.
// A passwd entry alone must not authorize another account or system tree.
volume := filepath.Dir(home)
allowed := false
for _, root := range roots {
if isHomeVolumeName(filepath.Base(root)) && isHomeVolumeName(filepath.Base(volume)) &&
filepath.Dir(root) == filepath.Dir(volume) {
allowed = true
break
}
}
if !allowed {
return "", false
}
volumeInfo, err := osFS.Lstat(volume)
if err != nil || !volumeInfo.IsDir() || volumeInfo.Mode()&os.ModeSymlink != 0 {
return "", false
}
info, err := osFS.Lstat(home)
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return "", false
}
return filepath.Clean(home), true
}
// accountHomeGlob globs "<root>/<pattern>" under every root and returns the
// concatenated matches. pattern typically starts with "*" (any account).
func accountHomeGlob(pattern string) ([]string, error) {
var out []string
var firstErr error
for _, root := range accountHomeRoots() {
matches, err := osFS.Glob(filepath.Join(root, pattern))
if err != nil && firstErr == nil {
firstErr = err
}
out = append(out, matches...)
}
if out == nil {
return nil, firstErr
}
return out, nil
}
// accountHomePatterns returns "<root>/*" for every account root: the glob
// form of "every account home".
func accountHomePatterns() []string {
roots := accountHomeRoots()
out := make([]string, 0, len(roots))
for _, root := range roots {
out = append(out, filepath.Join(root, "*"))
}
return out
}
// accountHomeSubPatterns returns "<root>/*/<sub>" for every account root.
func accountHomeSubPatterns(sub string) []string {
roots := accountHomeRoots()
out := make([]string, 0, len(roots))
for _, root := range roots {
out = append(out, filepath.Join(root, "*", sub))
}
return out
}
// accountRootOf reports the root and account a path belongs to. The path
// must lie strictly inside an account directory: a root or an account home
// itself is not "inside an account".
func accountRootOf(path string) (root, account string, ok bool) {
return accountRootOfAt(path, accountHomeRoots())
}
func accountRootOfAt(path string, roots []string) (root, account string, ok bool) {
clean := filepath.Clean(path)
for _, r := range roots {
r = filepath.Clean(r)
rest, found := strings.CutPrefix(clean, r+string(filepath.Separator))
if !found {
continue
}
account, tail, hasTail := strings.Cut(rest, string(filepath.Separator))
if account == "" || !hasTail || tail == "" {
continue
}
return r, account, true
}
return "", "", false
}
// isAccountRoot reports whether dir is one of the account roots.
func isAccountRoot(dir string) bool {
clean := filepath.Clean(dir)
for _, r := range accountHomeRoots() {
if filepath.Clean(r) == clean {
return true
}
}
return false
}
// underAccountRoot reports whether path is at or below any account root.
func underAccountRoot(path string) bool {
clean := filepath.Clean(path)
for _, r := range accountHomeRoots() {
r = filepath.Clean(r)
if clean == r || strings.HasPrefix(clean, r+string(filepath.Separator)) {
return true
}
}
return false
}
// accountRootPrefixes returns every account root with a trailing separator
// (the prefix form used to test "is this path inside an account") followed
// by extra prefixes verbatim.
func accountRootPrefixes(extra ...string) []string {
roots := accountHomeRoots()
out := make([]string, 0, len(roots)+len(extra))
for _, root := range roots {
out = append(out, filepath.Clean(root)+string(filepath.Separator))
}
return append(out, extra...)
}
// accountNameInTextAt returns the account named by the first
// "<root>/<account>/" reference in free text (a finding message or
// details), or "" when none is present.
func accountNameInTextAt(text string, roots []string) string {
for _, root := range roots {
prefix := filepath.Clean(root) + string(filepath.Separator)
idx := strings.Index(text, prefix)
if idx < 0 {
continue
}
rest := text[idx+len(prefix):]
if slash := strings.IndexByte(rest, '/'); slash > 0 {
return rest[:slash]
}
}
return ""
}
// effectiveFixRoots returns the allowed roots for a remediation: an
// explicit override (tests redirect under t.TempDir()) wins; otherwise the
// account roots plus any extra system directories the action may touch.
func effectiveFixRoots(override []string, extra ...string) []string {
if override != nil {
return override
}
roots := append([]string(nil), accountHomeRoots()...)
if cfg := config.Active(); cfg != nil {
// Failed roots are excluded; doctor reports the resolution errors.
// One tenant's symlink must not disable fixes for other accounts.
configured, _ := platform.ResolveAccountRoots(cfg.AccountRoots)
roots = append(roots, configured...)
}
return append(roots, extra...)
}
// quarantineExtraRoots are the non-account directories a quarantine may
// take a file from: the world-writable temp trees droppers land in.
var quarantineExtraRoots = []string{"/tmp", "/dev/shm", "/var/tmp"}
package checks
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
)
const accountScanMaxFilesDefault = 10000
func effectiveAccountScanMaxFiles(cfg *config.Config) int {
if cfg == nil || cfg.Thresholds.AccountScanMaxFiles <= 0 {
return accountScanMaxFilesDefault
}
return cfg.Thresholds.AccountScanMaxFiles
}
// rankPathsByMtimeDesc orders paths most-recent-first and optionally caps
// the result at maxFiles. Lexical glob order plus a downstream check timeout
// would otherwise keep cutting iteration off at the same prefix every cycle,
// hiding indicators on late-alphabet accounts.
//
// Stat failures are tolerated: the path is kept with a zero mtime so it
// sorts to the end, letting the cap chop it first when present.
// Best-effort ranking is the goal -- dropping silently here would
// reintroduce the same hidden-input bug class the helper exists to close.
// Downstream readers handle the missing-file case on their own.
//
// A canceled ctx returns nil; nil ctx is treated as Background.
// maxFiles <= 0 disables the cap; the sort still runs.
func rankPathsByMtimeDesc(ctx context.Context, paths []string, maxFiles int) []string {
if ctx == nil {
ctx = context.Background()
}
if err := ctx.Err(); err != nil {
return nil
}
type entry struct {
path string
mtime time.Time
}
ranked := make([]entry, 0, len(paths))
for _, p := range paths {
if err := ctx.Err(); err != nil {
return nil
}
var mt time.Time
if info, err := osFS.Stat(p); err == nil {
mt = info.ModTime()
}
ranked = append(ranked, entry{path: p, mtime: mt})
}
if err := ctx.Err(); err != nil {
return nil
}
sort.Slice(ranked, func(i, j int) bool {
if ranked[i].mtime.Equal(ranked[j].mtime) {
return ranked[i].path < ranked[j].path
}
return ranked[i].mtime.After(ranked[j].mtime)
})
if err := ctx.Err(); err != nil {
return nil
}
var droppedPaths []string
if maxFiles > 0 && len(ranked) > maxFiles {
dropped := len(ranked) - maxFiles
droppedPaths = make([]string, dropped)
for i, e := range ranked[maxFiles:] {
droppedPaths[i] = e.path
}
ranked = ranked[:maxFiles]
}
if len(droppedPaths) > 0 {
recordAccountScanTruncatedPaths(ctx, droppedPaths, maxFiles)
}
out := make([]string, len(ranked))
for i, e := range ranked {
out[i] = e.path
}
return out
}
// RunAccountScan runs all applicable checks scoped to a single cPanel account.
// Returns findings for that account only. Does NOT trigger auto-response actions.
//
// This is a thin wrapper around RunAccountScanWithOptions using
// DefaultAccountScanOptions so all existing callers retain their current behaviour.
func RunAccountScan(cfg *config.Config, store *state.Store, account string) []alert.Finding {
return RunAccountScanWithOptions(context.Background(), cfg, store, account, DefaultAccountScanOptions(cfg))
}
// RunAccountScanWithOptions is the options-aware entry point for per-account scans.
// Scope is propagated through ctx via ContextWithAccountScope, so parallel
// scans of different accounts no longer block on a single process-wide
// mutex and never bleed scope into each other.
func RunAccountScanWithOptions(ctx context.Context, cfg *config.Config, store *state.Store, account string, opts AccountScanOptions) []alert.Finding {
// Verify account exists
homeDir := accountHomeDir(account)
if _, err := osFS.Stat(homeDir); os.IsNotExist(err) {
return []alert.Finding{{
Severity: alert.Warning,
Check: "account_scan",
Message: fmt.Sprintf("Account '%s' not found (no %s directory)", account, homeDir),
Timestamp: time.Now(),
}}
}
// Account-scoped checks (filesystem + account-specific)
accountChecks := []namedCheck{
{"webshells", CheckWebshells},
{"htaccess", CheckHtaccess},
{"wp_core", CheckWPCore},
{"php_content", CheckPHPContent},
{"phishing", CheckPhishing},
{"filesystem", CheckFilesystem},
{"group_writable_php", CheckGroupWritablePHP},
{"nulled_plugins", CheckNulledPlugins},
{"open_basedir", CheckOpenBasedir},
{"symlink_attacks", CheckSymlinkAttacks},
{"db_content", CheckDatabaseContent},
{"php_config_changes", CheckPHPConfigChanges},
}
// Account-specific checks that need the account name
accountChecks = append(accountChecks,
namedCheck{"ssh_keys_account", makeAccountSSHKeyCheck(account)},
namedCheck{"crontab_account", makeAccountCrontabCheck(account)},
namedCheck{"backdoor_binaries", makeAccountBackdoorCheck(account)},
)
// File-index audit: only included for full scans (ForceFileIndex=true).
// In default scans CheckFileIndex writes live state (fileindex.current,
// fileindex.previous, dircache.json) that the host-wide incremental
// baseline relies on. Adding it unconditionally would corrupt that state
// once per triggered account scan. The audit branch (ForceFileIndex=true)
// is read-only and account-scoped so it is safe to include here.
if opts.ForceFileIndex {
accountChecks = append(accountChecks, namedCheck{"file_index", CheckFileIndex})
}
// The YARA content pass belongs to the explicit full-scan audit only: it
// reads every file in the account's document roots, and the default scan
// keeps its existing cost.
if opts.ForceContent {
accountChecks = append(accountChecks, namedCheck{"yara_deep", checkAccountYARA})
}
// Run under the host-wide scan budget - filesystem checks all walk the
// same directory tree, so too many concurrent checks starve each other on
// loaded servers with slow I/O, and a periodic tier may be running too.
scanCtx, truncations := withAccountScanTruncationCollector(ctx)
scanCtx = ContextWithAccountScope(scanCtx, account)
scanCtx = ContextWithScanOptions(scanCtx, opts)
scanCtx = withWPInstallCache(scanCtx)
findings := runAccountChecksBounded(scanCtx, cfg, store, accountChecks)
now := time.Now()
findings = append(findings, truncations.findings(now)...)
for i := range findings {
if findings[i].Timestamp.IsZero() {
findings[i].Timestamp = now
}
}
// Filter findings to only include this account's paths
var filtered []alert.Finding
for _, f := range findings {
if accountScanFindingInScope(f, account) {
filtered = append(filtered, f)
}
}
// The report never reaches the dispatcher, so the companion sweep runs
// here, after the checks have released their scan slots.
filtered = append(filtered, companionSweepFindings(scanCtx, cfg, filtered)...)
filtered = stampTenantIDIfEmpty(filtered, account)
annotateModsecDisabledMalware(platform.Detect(), filtered)
return filtered
}
// runAccountChecksBounded runs checks under the host-wide scan budget.
// A check still waiting for a slot when ctx is cancelled never starts: the
// slot wait used to ignore the context, so an operator's cancel left every
// queued check running to its (immediate) end and reporting a timeout.
func runAccountChecksBounded(ctx context.Context, cfg *config.Config, store *state.Store, checks []namedCheck) []alert.Finding {
scansInFlight.Add(1)
defer scansInFlight.Add(-1)
var mu sync.Mutex
var findings []alert.Finding
var wg sync.WaitGroup
budget := scanBudgetFrom(ctx)
dispatches := checkDispatches.begin(len(checks), budget)
checkDispatches.observe(ctx, dispatches)
for i, nc := range checks {
wg.Add(1)
c := nc
task := dispatches[i]
// Account checks run against user filesystem content (unparsed PHP,
// crafted archives, foreign encodings) so a panic is plausible.
// runAccountScanCheck surfaces it as check_panic, keeping the scan and
// daemon alive.
obs.SafeGo("account-scan-runner", task.wrap(func() {
defer wg.Done()
if !task.admit(ctx) {
task.withdraw(ctx)
return
}
if ctx.Err() != nil {
task.withdraw(ctx)
return
}
results := runAccountScanCheck(withCheckDispatch(ctx, task), c, cfg, store, timeoutFor(c.name))
if len(results) > 0 {
mu.Lock()
findings = append(findings, results...)
mu.Unlock()
}
}))
}
wg.Wait()
return findings
}
// runAccountScanCheck runs one check under a timeout, recovering any panic.
// A check cut short because the scan itself was cancelled reports nothing:
// that is not a timeout, and the warning would be persisted with the partial
// results the cancel keeps.
func runAccountScanCheck(ctx context.Context, c namedCheck, cfg *config.Config, store *state.Store, timeout time.Duration) []alert.Finding {
if ctx.Err() != nil {
checkDispatchFrom(ctx).withdraw(ctx)
return nil
}
cctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
partial := &accountScanPartialResults{}
cctx = context.WithValue(cctx, accountScanPartialKey{}, partial)
panicIdentity := alert.Finding{Check: "check_panic", DedupKey: fmt.Sprintf("account-scan:%q:%s", AccountFromContext(ctx), c.name)}
execution := executeCheckAsync(cctx, "account-scan-exec", func() []alert.Finding {
return c.fn(cctx, cfg, store)
})
defer execution.finishCaller()
select {
case outcome := <-execution.done:
execution.received()
if outcome.panicErr != "" {
// Keep stack churn out of identity without merging failures from
// different accounts or the scheduled runner.
return []alert.Finding{{
Severity: alert.High,
Check: "check_panic",
TenantID: AccountFromContext(ctx),
Message: fmt.Sprintf("Account scan check '%s' stopped after an internal panic", c.name),
Details: outcome.panicErr,
DedupKey: panicIdentity.DedupKey,
Timestamp: time.Now(),
}}
}
if store != nil && cctx.Err() == nil {
store.RearmFindings([]string{panicIdentity.Key()})
}
return outcome.findings
case <-cctx.Done():
execution.withdraw(cctx.Err())
if ctx.Err() != nil {
return nil
}
return append(partial.snapshot(), alert.Finding{
Severity: alert.Warning,
Check: "check_timeout",
Message: fmt.Sprintf("Account scan check '%s' timed out", c.name),
Timestamp: time.Now(),
})
}
}
type accountScanPartialKey struct{}
// A blocked filesystem or backend call can outlast the runner. Checkpoints
// retain completed detections while that execution still owns its scan slot.
type accountScanPartialResults struct {
mu sync.Mutex
findings []alert.Finding
}
func publishAccountScanFindings(ctx context.Context, findings []alert.Finding) {
if partial, ok := ctx.Value(accountScanPartialKey{}).(*accountScanPartialResults); ok {
partial.mu.Lock()
partial.findings = append(partial.findings, findings...)
partial.mu.Unlock()
}
}
func (p *accountScanPartialResults) snapshot() []alert.Finding {
p.mu.Lock()
defer p.mu.Unlock()
return append([]alert.Finding(nil), p.findings...)
}
func accountScanFindingInScope(f alert.Finding, account string) bool {
if account == "" {
return true
}
if f.FilePath != "" {
fileAccount := accountFromHomePath(f.FilePath)
if fileAccount != "" {
return fileAccount == account
}
if containsHomeReference(f.FilePath) {
return false
}
}
hasHomeRef, hasAccountRef := textHomeScope(f.Message, account)
detailsHasHomeRef, detailsHasAccountRef := textHomeScope(f.Details, account)
if hasAccountRef || detailsHasAccountRef {
return true
}
if hasHomeRef || detailsHasHomeRef {
return false
}
return true
}
// accountRootPrefixLen reports whether text starts with an account root
// and how long that prefix is. "/home" keeps cPanel's multi-home tolerance
// (/home2, /home3) so findings from those trees still resolve.
func accountRootPrefixLen(text string) (int, bool) {
for _, root := range accountHomeRoots() {
root = filepath.ToSlash(filepath.Clean(root))
if !strings.HasPrefix(text, root) {
continue
}
i := len(root)
if root == "/home" {
for i < len(text) && text[i] >= '0' && text[i] <= '9' {
i++
}
}
if i == len(text) || text[i] == '/' {
return i, true
}
}
return 0, false
}
func containsHomeReference(path string) bool {
_, ok := accountRootPrefixLen(filepath.ToSlash(filepath.Clean(path)))
return ok
}
func textHomeScope(text, account string) (hasHomeRef, hasAccountRef bool) {
for _, root := range accountHomeRoots() {
root = filepath.ToSlash(filepath.Clean(root))
for i := 0; i < len(text); {
idx := strings.Index(text[i:], root)
if idx < 0 {
break
}
start := i + idx
if homeAccount, ok := homeAccountAt(text[start:]); ok {
hasHomeRef = true
if homeAccount == account {
hasAccountRef = true
}
}
i = start + len(root)
}
}
return hasHomeRef, hasAccountRef
}
func homeAccountAt(text string) (string, bool) {
i, ok := accountRootPrefixLen(text)
if !ok {
return "", false
}
if i == len(text) {
return "", true
}
i++
start := i
for i < len(text) && isHomeAccountByte(text[i]) {
i++
}
if i == start {
return "", true
}
return text[start:i], true
}
func isHomeAccountByte(b byte) bool {
return b >= 'a' && b <= 'z' ||
b >= 'A' && b <= 'Z' ||
b >= '0' && b <= '9' ||
b == '_' || b == '-' || b == '.'
}
// stampTenantIDIfEmpty fills in Finding.TenantID with account when the
// detector emitted the finding without explicit tenant attribution.
// Account-scope detectors otherwise leave TenantID empty and the
// correlator falls back to weaker identities (UID, PID, file hash),
// fragmenting one account's incidents across multiple keys. Findings
// the detector did stamp keep their value; an empty account is a no-op.
func stampTenantIDIfEmpty(findings []alert.Finding, account string) []alert.Finding {
if account == "" {
return findings
}
for i := range findings {
if findings[i].TenantID == "" {
findings[i].TenantID = account
}
}
return findings
}
// GetScanHomeDirs returns the list of home directories to scan.
// When ctx carries an account scope (via ContextWithAccountScope), only
// that account is returned. Otherwise every entry under every account root
// is read. Nil ctx is tolerated for legacy callers and treated as host-wide.
// Callers that need the directory path use scanHomeDirPath on each entry.
func GetScanHomeDirs(ctx context.Context) ([]os.DirEntry, error) {
if account := AccountFromContext(ctx); account != "" {
info, err := osFS.Stat(accountHomeDir(account))
if err != nil {
return nil, err
}
return []os.DirEntry{fakeDirEntry{info}}, nil
}
homes, err := listAccountHomes()
if err != nil {
return nil, err
}
entries := make([]os.DirEntry, 0, len(homes))
for _, h := range homes {
entries = append(entries, rootedDirEntry{DirEntry: h.Entry, root: h.Root})
}
return entries, nil
}
// rootedDirEntry remembers which account root an entry came from.
type rootedDirEntry struct {
os.DirEntry
root string
}
// scanHomeDirPath returns the home directory for an entry returned by
// GetScanHomeDirs (or any account enumeration): the entry's own root when it
// carries one, otherwise the root that holds the account.
func scanHomeDirPath(entry os.DirEntry) string {
if r, ok := entry.(rootedDirEntry); ok {
return filepath.Join(r.root, r.Name())
}
return accountHomeDir(entry.Name())
}
// WebRootPatterns returns the configured web-root globs, including the
// platform default when the operator did not set account_roots.
func WebRootPatterns(cfg *config.Config) []string {
switch {
case cfg != nil && len(cfg.AccountRoots) > 0:
return append([]string(nil), cfg.AccountRoots...)
case platform.Detect().IsCPanel():
return accountHomeSubPatterns("public_html")
default:
return nil
}
}
// ResolveWebRoots returns the list of directory paths CSM should scan for
// web-facing content (wp-config.php, .htaccess, public_html trees, etc.).
//
// Resolution order:
// 1. If cfg.AccountRoots is set, expand each glob and return the result.
// Explicit config always wins.
// 2. On cPanel hosts (detected via platform.Detect), fall back to
// /home/*/public_html for backward compatibility.
// 3. On non-cPanel hosts with no config, return an empty list. Callers
// should treat this as "no scanning" and skip cleanly.
//
// Each returned path is an absolute directory that exists on disk.
func ResolveWebRoots(cfg *config.Config) []string {
roots, _ := resolveWebRootsWithGlob(cfg, func(pattern string) ([]string, bool) {
matches, err := osFS.Glob(pattern)
return matches, err == nil
})
return roots
}
// Stateful scans must distinguish an unreadable root from a removed one.
func resolveWebRootsChecked(cfg *config.Config) ([]string, bool) {
return resolveWebRootsWithGlob(cfg, checkedWebRootGlob)
}
func resolveWebRootsWithGlob(cfg *config.Config, glob func(string) ([]string, bool)) ([]string, bool) {
patterns := WebRootPatterns(cfg)
var roots []string
complete := true
seen := make(map[string]struct{})
for _, pattern := range patterns {
matches, patternComplete := glob(pattern)
if !patternComplete {
complete = false
}
for _, m := range matches {
info, err := osFS.Stat(m)
if err != nil {
if !os.IsNotExist(err) {
complete = false
}
continue
}
if !info.IsDir() {
continue
}
if _, ok := seen[m]; ok {
continue
}
seen[m] = struct{}{}
roots = append(roots, m)
}
}
return roots, complete
}
// fakeDirEntry wraps os.FileInfo to implement os.DirEntry.
type fakeDirEntry struct {
fi os.FileInfo
}
func (f fakeDirEntry) Name() string { return f.fi.Name() }
func (f fakeDirEntry) IsDir() bool { return f.fi.IsDir() }
func (f fakeDirEntry) Type() os.FileMode { return f.fi.Mode().Type() }
func (f fakeDirEntry) Info() (os.FileInfo, error) { return f.fi, nil }
// makeAccountSSHKeyCheck creates a check for SSH keys of a specific account.
func makeAccountSSHKeyCheck(account string) CheckFunc {
return func(_ context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
var findings []alert.Finding
keyFile := filepath.Join(accountHomeDir(account), ".ssh", "authorized_keys")
hash, err := hashFileContent(keyFile)
if err != nil {
return nil
}
key := fmt.Sprintf("_ssh_user_keys:%s", keyFile)
prev, exists := store.GetRaw(key)
if exists && prev != hash {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "ssh_keys",
Message: fmt.Sprintf("User authorized_keys modified: %s", keyFile),
})
}
return findings
}
}
// makeAccountCrontabCheck creates a check for a specific account's crontab.
func makeAccountCrontabCheck(account string) CheckFunc {
return func(_ context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
crontabFile := filepath.Join(cronSpoolDir(), account)
data, err := osFS.ReadFile(crontabFile)
if err != nil {
return nil
}
content := string(data)
for _, pattern := range MatchCrontabPatternsDeep(content, cfg) {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "suspicious_crontab",
Message: fmt.Sprintf("Suspicious pattern in crontab for %s: %s", account, pattern),
Details: fmt.Sprintf("File: %s\nContent:\n%s", crontabFile, content),
FilePath: crontabFile,
})
}
return findings
}
}
// makeAccountBackdoorCheck creates a check for backdoor binaries in account's .config.
func makeAccountBackdoorCheck(account string) CheckFunc {
return func(_ context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
backdoorNames := map[string]bool{
"defunct": true, "defunct.dat": true, "gs-netcat": true,
"gs-sftp": true, "gs-mount": true, "gsocket": true,
}
patterns := []string{
filepath.Join(accountHomeDir(account), ".config", "htop", "*"),
filepath.Join(accountHomeDir(account), ".config", "*", "*"),
}
for _, pattern := range patterns {
matches, _ := osFS.Glob(pattern)
for _, path := range matches {
if backdoorNames[filepath.Base(path)] {
info, _ := osFS.Stat(path)
var details string
if info != nil {
details = fmt.Sprintf("Size: %d bytes, Mtime: %s", info.Size(), info.ModTime().Format("2006-01-02 15:04:05"))
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "backdoor_binary",
Message: fmt.Sprintf("Backdoor binary found: %s", path),
Details: details,
FilePath: path,
})
}
}
}
return findings
}
}
// AccountHomePatterns returns the glob for every account home ("<root>/*") on
// this platform. The realtime scanner needs it to recognise an account tree
// without hardcoding /home.
func AccountHomePatterns() []string {
return accountHomePatterns()
}
package checks
import (
"context"
"fmt"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
)
type accountScanTruncationContextKey struct{}
type accountScanTruncationCollector struct {
mu sync.Mutex
// droppedBy is keyed by account so operators can see which tenant hit
// which cap. The empty-string key keeps dropped paths that are not
// attributable to /home/<account>/.
droppedBy map[string]map[int]int
}
func withAccountScanTruncationCollector(ctx context.Context) (context.Context, *accountScanTruncationCollector) {
if ctx == nil {
ctx = context.Background()
}
collector := &accountScanTruncationCollector{droppedBy: map[string]map[int]int{}}
return context.WithValue(ctx, accountScanTruncationContextKey{}, collector), collector
}
func recordAccountScanTruncated(ctx context.Context, dropped, cap int) {
if dropped <= 0 || cap <= 0 || ctx == nil {
return
}
collector, ok := ctx.Value(accountScanTruncationContextKey{}).(*accountScanTruncationCollector)
if !ok || collector == nil {
return
}
collector.record(AccountFromContext(ctx), dropped, cap)
}
func recordAccountScanTruncatedPaths(ctx context.Context, droppedPaths []string, cap int) {
if len(droppedPaths) == 0 || cap <= 0 || ctx == nil {
return
}
collector, ok := ctx.Value(accountScanTruncationContextKey{}).(*accountScanTruncationCollector)
if !ok || collector == nil {
return
}
for account, dropped := range accountScanTruncationAccounts(ctx, droppedPaths) {
collector.record(account, dropped, cap)
}
}
func accountScanTruncationAccounts(ctx context.Context, paths []string) map[string]int {
if account := AccountFromContext(ctx); account != "" {
return map[string]int{account: len(paths)}
}
counts := make(map[string]int)
for _, path := range paths {
counts[accountFromHomePath(path)]++
}
return counts
}
func accountFromHomePath(path string) string {
cleaned := filepath.ToSlash(filepath.Clean(path))
i, ok := accountRootPrefixLen(cleaned)
if !ok || i == len(cleaned) || cleaned[i] != '/' {
return ""
}
account := cleaned[i+1:]
if slash := strings.IndexByte(account, '/'); slash >= 0 {
account = account[:slash]
}
if account == "." || account == ".." {
return ""
}
return account
}
func (c *accountScanTruncationCollector) record(account string, dropped, cap int) {
c.mu.Lock()
defer c.mu.Unlock()
caps, ok := c.droppedBy[account]
if !ok {
caps = map[int]int{}
c.droppedBy[account] = caps
}
caps[cap] += dropped
}
func (c *accountScanTruncationCollector) findings(now time.Time) []alert.Finding {
c.mu.Lock()
defer c.mu.Unlock()
if len(c.droppedBy) == 0 {
return nil
}
accounts := make([]string, 0, len(c.droppedBy))
for a := range c.droppedBy {
accounts = append(accounts, a)
}
sort.Strings(accounts) // stable finding order across runs
var findings []alert.Finding
for _, account := range accounts {
caps := c.droppedBy[account]
capValues := make([]int, 0, len(caps))
for cap := range caps {
capValues = append(capValues, cap)
}
sort.Ints(capValues)
for _, cap := range capValues {
dropped := caps[cap]
scope := "host scan"
tenantID := ""
if account != "" {
scope = fmt.Sprintf("account %s", account)
tenantID = account
}
// The skipped count moves with every file added or removed; the
// condition is this scope hitting this cap.
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "account_scan_truncated",
TenantID: tenantID,
Message: fmt.Sprintf("Account scan truncated for %s: %d file(s) skipped past cap of %d", scope, dropped, cap),
Details: "Raise thresholds.account_scan_max_files if recent detection coverage matters more than scan duration.",
DedupKey: fmt.Sprintf("scope=%q cap=%d", account, cap),
Timestamp: now,
})
}
}
return findings
}
package checks
import (
"context"
"crypto/sha256"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/contenttype"
"github.com/pidginhost/csm/internal/safepath"
"github.com/pidginhost/csm/internal/signatures"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/yara"
)
// checkAccountYARA is the YARA content pass of a full account scan. It walks
// the scoped account's document roots, the same roots the other content
// checks use, and sends one file at a time through the active backend: the
// isolated worker in production, which serves one request at a time. It
// keeps no cursor, cache or carry-forward state; an account scan only
// reports.
func checkAccountYARA(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
backend := activeYARABackend()
if backend == nil || backend.RuleCount() == 0 {
if !yaraAvailable() {
return nil
}
return []alert.Finding{{
Severity: alert.High,
Check: "yara_scan_incomplete",
Message: "YARA account scan could not start because no compiled rules are available",
}}
}
maxBytes := scanMaxFileBytes(ctx)
if maxBytes <= 0 {
maxBytes = FullScanMaxFileBytes(cfg)
}
respectIgnores := scanRespectsIgnores(ctx, cfg)
// Leave time for the last worker call. Completed detections are also
// checkpointed in case that call or filesystem I/O exceeds the margin.
var softDeadline time.Time
if deadline, ok := ctx.Deadline(); ok {
softDeadline = deadline.Add(-yaraDeepDeadlineMargin)
}
gaps := newYARAGapCollector()
// The report only names gaps; nothing is carried forward by alias.
gaps.resolveAliases = func(string) ([]string, bool) { return nil, true }
var findings []alert.Finding
// Raster images go in a second pass: a shop's product photos can sort
// before its code folders and use up the whole time budget.
imagesPass := false
// Retain deferred entries, not just their directories: a fresh listing
// would silently lose images removed or replaced between passes.
imageEntries := make(map[string][]os.DirEntry)
codeGaps := false
stopped := false
stop := func(path string) bool {
if stopped {
return true
}
if ctx.Err() != nil || (!softDeadline.IsZero() && !yaraDeepNow().Before(softDeadline)) {
stopped = true
detail := fmt.Sprintf("stopped at %s: scan time budget reached", path)
if err := ctx.Err(); err != nil {
detail = fmt.Sprintf("stopped at %s: %v", path, err)
}
if imagesPass && !codeGaps {
detail += "; only image files were left"
} else if imagesPass {
detail += "; image pass"
}
gaps.recordUnknownRange(detail)
}
return stopped
}
// Image folders are revisited; report each unreadable location once.
reported := make(map[string]bool)
locationGap := func(path, detail string) {
if !reported[path] {
reported[path] = true
gaps.recordUnknownRange(detail)
}
}
var walk func(dir string, pinned *safepath.Dir)
walk = func(dir string, pinned *safepath.Dir) {
defer func() { _ = pinned.Close() }()
entries := imageEntries[dir]
if !imagesPass {
var err error
entries, err = pinned.ReadDir()
if err != nil {
locationGap(dir, fmt.Sprintf("reading %s: %v", dir, err))
}
sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() })
}
for _, entry := range entries {
path := filepath.Join(dir, entry.Name())
if respectIgnores && ignoredPath(cfg, path) {
continue
}
if stop(path) {
return
}
// A symlink is neither entered nor scanned: its target is not
// necessarily the account's and may be anywhere on the host.
switch {
case entry.IsDir():
// Replaced image directories must fail the pinned walk
// instead of disappearing from the coverage report.
child, err := pinned.Walk(entry.Name())
if err != nil {
locationGap(path, fmt.Sprintf("opening %s: %v", path, err))
continue
}
walk(path, child)
if !imagesPass && len(imageEntries[path]) > 0 {
imageEntries[dir] = append(imageEntries[dir], entry)
}
case entry.Type().IsRegular():
isImage := contenttype.IsImageExt(filepath.Ext(entry.Name()))
if isImage && !imagesPass {
imageEntries[dir] = append(imageEntries[dir], entry)
}
if isImage != imagesPass {
continue
}
matches := scanAccountYARAFile(backend, gaps, pinned, path, maxBytes)
findings = append(findings, matches...)
publishAccountScanFindings(ctx, matches)
}
}
}
scanHome := func(homePath string) {
if imagesPass && len(imageEntries[homePath]) == 0 {
return
}
if stop(homePath) {
return
}
pinned, err := openAccountYARAHome(homePath)
if err != nil {
locationGap(homePath, fmt.Sprintf("opening %s: %v", homePath, err))
return
}
defer func() { _ = pinned.Close() }()
entries := imageEntries[homePath]
if !imagesPass {
entries, err = pinned.ReadDir()
if err != nil {
locationGap(homePath, fmt.Sprintf("reading %s: %v", homePath, err))
}
sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() })
}
for _, entry := range entries {
if !accountContentDir(entry.Name()) {
continue
}
root := filepath.Join(homePath, entry.Name())
if stop(root) {
break
}
if respectIgnores && ignoredPath(cfg, root) {
continue
}
switch {
case entry.Type()&os.ModeSymlink != 0:
if entry.Name() == "public_html" {
locationGap(root, fmt.Sprintf("%s is a symbolic link and was not followed", root))
}
continue
case !entry.IsDir():
continue
}
child, err := pinned.Walk(entry.Name())
if err != nil {
locationGap(root, fmt.Sprintf("opening %s: %v", root, err))
continue
}
walk(root, child)
if !imagesPass && len(imageEntries[root]) > 0 {
imageEntries[homePath] = append(imageEntries[homePath], entry)
}
}
}
homes, err := GetScanHomeDirs(ctx)
if err != nil {
gaps.recordUnknownRange(fmt.Sprintf("resolving the account home: %v", err))
}
for _, imagesPass = range []bool{false, true} {
if imagesPass {
if len(imageEntries) == 0 {
break
}
codeGaps = !gaps.empty()
}
for _, home := range homes {
scanHome(scanHomeDirPath(home))
}
}
if !gaps.empty() {
findings = append(findings, gaps.finding())
}
return findings
}
func openAccountYARAHome(home string) (*safepath.Dir, error) {
// The platform's account-root parent is operator controlled. Pin the
// volume and account components below it so neither can redirect reads.
volume := filepath.Dir(home)
anchor, err := safepath.OpenDir(filepath.Dir(volume))
if err != nil {
return nil, err
}
defer func() { _ = anchor.Close() }()
return anchor.Walk(filepath.Join(filepath.Base(volume), filepath.Base(home)))
}
// scanAccountYARAFile sends one regular file through the backend. The size
// check comes first so a large media file or dump is never read at all.
func scanAccountYARAFile(backend yara.Backend, gaps *yaraGapCollector, dir *safepath.Dir, path string, maxBytes int64) []alert.Finding {
file, err := dir.OpenFile(filepath.Base(path), os.O_RDONLY, 0)
if err != nil {
gaps.record(path, "read_error")
return nil
}
defer func() { _ = file.Close() }()
info, err := file.Stat()
if err != nil || !info.Mode().IsRegular() {
gaps.record(path, "stat_error")
return nil
}
if info.Size() > maxBytes {
gaps.record(path, "oversize")
return nil
}
data, err := io.ReadAll(io.LimitReader(file, maxBytes))
if err != nil {
gaps.record(path, "read_error")
return nil
}
after, err := file.Stat()
if err != nil || !sameFileSnapshot(info, after) || int64(len(data)) != info.Size() {
gaps.record(path, "changed")
return nil
}
if len(data) == 0 {
return nil
}
matches, err := yara.ScanBytesChecked(backend, path, data)
if err != nil {
gaps.record(path, "scan_error")
return nil
}
if len(matches) == 0 {
return nil
}
contentSHA256 := fmt.Sprintf("%x", sha256.Sum256(data))
findings := make([]alert.Finding, 0, len(matches))
for _, match := range matches {
findings = append(findings, alert.Finding{
Severity: yaraMatchSeverity(match.Meta["severity"]),
Check: "yara_match_scheduled",
Message: fmt.Sprintf("YARA rule match [%s]: %s", match.RuleName, path),
Details: fmt.Sprintf("Full account scan matched YARA rule %s", match.RuleName) + signatures.ReferencedPayloadDetail(data),
FilePath: path,
ContentSHA256: contentSHA256,
DetectLogic: ContentDetectionVersion(),
})
}
return findings
}
// ignoredPath reports whether path matches the operator's ignore_paths.
func ignoredPath(cfg *config.Config, path string) bool {
for _, ignore := range cfg.Suppressions.IgnorePaths {
if matchGlob(path, ignore) {
return true
}
}
return false
}
package checks
import (
"context"
"path/filepath"
)
type accountScopeKey struct{}
// ContextWithAccountScope returns a derived context that restricts
// filesystem-based checks to a single cPanel/Linux account. Callers
// pass the resulting context into every check; helpers like
// GetScanHomeDirs read the scope back out. Empty account is a no-op
// (returns ctx unchanged, equivalent to a full host scan).
func ContextWithAccountScope(ctx context.Context, account string) context.Context {
if ctx == nil {
ctx = context.Background()
}
if account == "" {
return ctx
}
return context.WithValue(ctx, accountScopeKey{}, account)
}
// AccountFromContext returns the account scope previously attached by
// ContextWithAccountScope, or "" when no scope is set.
func AccountFromContext(ctx context.Context) string {
if ctx == nil {
return ""
}
v, _ := ctx.Value(accountScopeKey{}).(string)
return v
}
// homeGlob globs elem under every account (or the scoped account) across
// every account root.
func homeGlob(ctx context.Context, elem ...string) ([]string, error) {
account := AccountFromContext(ctx)
if account == "" {
account = "*"
}
parts := append([]string{account}, elem...)
return accountHomeGlob(filepath.Join(parts...))
}
package checks
import (
"strings"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/reporting"
)
// AutomaticResponseAllowed reports whether cfg still allows an automatic
// response of kind for check, arriving on entry. These are the switches
// the legacy funnels test before they act; admission serves queued work
// later and tests them again. The check registry's own response policy
// was enforced when the work was minted. An unknown entry is not allowed.
func AutomaticResponseAllowed(cfg *config.Config, kind admission.Kind, entry admission.Entry, check string) bool {
if cfg == nil || !CheckResponseEnabled(cfg, check) {
return false
}
central := cfg.Reputation.Central.Enabled
action := reporting.ParseAction(cfg.Reputation.Central.Action)
if kind == admission.KindChallenge {
// Challenges follow challenge routing, not the block switches.
switch entry {
case admission.EntryScan:
return responseActionForCheck(cfg, check) == responseChallenge && !isHardBlockCheck(check) && (ResponsePolicyFor(check).Block != BlockWithCpanelLogins || cfg.AutoResponse.BlockCpanelLogins)
case admission.EntryCentral:
return cfg.Challenge.Enabled && central && action != reporting.ActionOff
}
return false
}
if !cfg.AutoResponse.Enabled || !cfg.AutoResponse.BlockIPs {
return false
}
switch entry {
case admission.EntryScan:
return kind == admission.KindBlockIP && blockableCheck(check, cfg.AutoResponse.BlockCpanelLogins)
case admission.EntryChallengeTimeout:
return cfg.Challenge.Enabled
case admission.EntryIncident:
return cfg.Incidents.AutoBlock.Enabled && blocksAtSeverity(cfg.Incidents.AutoBlock.BlockAtSeverity)
case admission.EntryIncidentSpray:
s := cfg.Incidents.SpraySuppression
return s.Enabled && !s.DryRun && blocksAtSeverity(s.BlockAtSeverity)
case admission.EntryCentral:
return central && action == reporting.ActionBlockIfLocalCorroborated
case admission.EntryNetblock:
return cfg.AutoResponse.NetBlock
case admission.EntryPermblock:
return cfg.AutoResponse.PermBlock
case admission.EntryASNCrawl, admission.EntryMailSubnet:
return true
}
return false
}
// BlocksAProxy reports whether queued block c would block a trusted proxy
// under cfg, or a known CDN or website-firewall edge that one of its roots
// may have seen relaying other people's requests, as the mint refuses it.
// roots loads c's root evidence and is read only for an edge.
func BlocksAProxy(cfg *config.Config, c admission.Candidate, roots func() []admission.Evidence) bool {
addr, ok := c.Key.Target.Addr()
if c.Key.Kind != admission.KindBlockIP || !ok {
return false
}
ip := addr.String()
if IsTrustedProxy(ip, cfg) {
return true
}
if !KnownProxyPeer(ip, cfg) {
return false
}
for _, e := range roots() {
if producerMayRelay(e.Producer()) {
return true
}
}
return false
}
// producerMayRelay reports whether an address producer p reads can be an
// edge relaying other people's requests, as the mint judges it. The queue
// holds no proxied client, so a producer's trusted-chain reading never
// applies.
func producerMayRelay(p admission.ProducerID) bool {
for _, entry := range producerTable {
if entry.Spec.ID != p {
continue
}
for _, kind := range []admission.Attribution{admission.AttributionPeer, admission.AttributionHTTPPeer, admission.AttributionDerived} {
if entry.Spec.Attribution.Has(kind) && mayBeProxyEdge(kind, p) {
return true
}
}
}
return false
}
// blocksAtSeverity reports whether an incident block_at_severity setting
// blocks at all; empty or unknown means detection only.
func blocksAtSeverity(s string) bool {
switch strings.ToLower(s) {
case "high", "critical":
return true
}
return false
}
package checks
import (
"strings"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
)
// AdmissionEvidence is what a finding asks admission to mint: the producer
// that stamped its observation and the evidence input that producer attests,
// aimed at the address the finding's parser read. A finding
// without an observation, naming a producer the table does not list, or
// whose parser vouched for no address asks for nothing: provenance is never
// reconstructed from the finding's other fields, and message text never
// becomes a target. The caller supplies the inventory the owner resolves
// claims against.
func AdmissionEvidence(f alert.Finding) (admission.ProducerID, admission.EvidenceInput, error) {
if f.Observation.Producer == "" {
return "", admission.EvidenceInput{}, &admission.Error{Reason: admission.ReasonAttribution, Cause: admission.CauseNoObservation, Detail: "finding has no observation"}
}
producer := admission.ProducerID(f.Observation.Producer)
parser, ok := ProducerParser(producer)
if !ok {
return "", admission.EvidenceInput{}, &admission.Error{Reason: admission.ReasonPolicy, Detail: "finding names no registered producer"}
}
if !f.Attribution.Valid() {
return "", admission.EvidenceInput{}, &admission.Error{Reason: admission.ReasonAttribution, Cause: admission.CauseKindNotAllowed, Detail: "no parser vouched for the finding's address"}
}
// This build reads no separate peer field when the web server replaces
// the logged address, so neither the peer nor a chain can be verified.
// An unknown state is refused the same way.
if (f.Attribution == admission.AttributionHTTPPeer || f.Attribution == admission.AttributionTrustedHTTPChain) &&
CurrentHTTPRewrite().State != RewriteNone {
return "", admission.EvidenceInput{}, &admission.Error{Reason: admission.ReasonAttribution, Cause: admission.CauseRewrittenLog, Detail: "the web server may replace the client address it logs"}
}
// A client behind a trusted proxy sends no packets to this host, so no
// response this build applies can contain it.
if f.Attribution == admission.AttributionTrustedHTTPChain {
return "", admission.EvidenceInput{}, &admission.Error{Reason: admission.ReasonAttribution, Cause: admission.CauseUnreachable, Detail: "the address is a client behind a trusted proxy"}
}
// A known CDN or website-firewall edge carries every visitor behind it:
// blocking it as one of them is never an automatic response.
if mayBeProxyEdge(f.Attribution, producer) && KnownProxyPeer(f.SourceIP, config.Active()) {
return "", admission.EvidenceInput{}, &admission.Error{Reason: admission.ReasonAttribution, Cause: admission.CauseProxyPeer, Detail: "the address is a known CDN or proxy edge"}
}
target, err := admission.CanonicalAddress(f.SourceIP, admission.Caps{IPv6: true})
if err != nil {
return "", admission.EvidenceInput{}, err
}
return producer, admission.EvidenceInput{
Check: f.Check, FindingID: alert.FindingID(f), Severity: admissionSeverity(f.Severity),
Observation: admission.ObservationRef{Stream: f.Observation.Stream, Cursor: f.Observation.Cursor, Version: 1},
ObservedAt: f.Observation.ObservedAt, Parser: parser, Target: target, Attribution: f.Attribution, Claims: f.Claims, Intel: f.Intel,
}, nil
}
// mayBeProxyEdge reports whether a finding's address can be an edge that
// relays other people's requests: an HTTP peer, a cpsrvd peer (CDNs proxy
// the cPanel ports), a Stalwart peer (CDNs can proxy mail ports), or
// reputation intel resting on such a sighting.
func mayBeProxyEdge(kind admission.Attribution, producer admission.ProducerID) bool {
switch kind {
case admission.AttributionHTTPPeer, admission.AttributionDerived:
return true
case admission.AttributionPeer:
return producer == ProducerCpanelAccessLog || producer == ProducerCpanelScan || producer == ProducerStalwartLog
}
return false
}
// AdmissionTarget is the canonical target of a response: an address, or a
// prefix when the funnel acts on a subnet.
func AdmissionTarget(raw string, caps admission.Caps) (admission.Target, error) {
if strings.Contains(raw, "/") {
return admission.CanonicalPrefix(raw, caps)
}
return admission.CanonicalAddress(raw, caps)
}
// AdmissionSeverity is the admission severity of an alert severity.
func AdmissionSeverity(s alert.Severity) admission.Severity { return admissionSeverity(s) }
package checks
import (
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/metrics"
)
// ResponseAdmission is the admission owner as the response funnels see it.
// Every automatic response the legacy funnels select is also handed to it,
// where it is previewed, forecast in dry run, or, for a block admission
// applies, executed instead of the legacy block. The funnels never wait for
// admission.
type ResponseAdmission interface {
// Mint mints the evidence f's own observation supports, aimed at the
// address its parser read. It counts nothing.
Mint(f alert.Finding) (admission.Evidence, error)
// Refuse counts a response that could not be answered because f could
// not be minted.
Refuse(kind admission.Kind, f alert.Finding, via admission.Entry, err error)
// Respond hands the ingress a response of kind to target, the address
// or prefix the legacy path acts on, with e as its root, through the
// derived entry via when it is set. Admission refuses a target that is
// not e's address and does not contain it.
Respond(kind admission.Kind, target string, e admission.Evidence, via admission.Entry, ttl ...time.Duration) error
// ExecutesBlocks reports whether admission applies the block picks it
// takes, so legacy must leave them alone.
ExecutesBlocks() bool
}
type responseAdmissionSlot struct{ a ResponseAdmission }
var responseAdmissionHolder atomic.Pointer[responseAdmissionSlot]
// SetResponseAdmission wires the admission owner; nil unwires it.
func SetResponseAdmission(a ResponseAdmission) {
responseAdmissionHolder.Store(&responseAdmissionSlot{a: a})
}
func getResponseAdmission() ResponseAdmission {
if slot := responseAdmissionHolder.Load(); slot != nil {
return slot.a
}
return nil
}
var (
handoffSeconds *metrics.Histogram
handoffSecondsOnce sync.Once
)
// timeHandoff records how long a funnel waited on admission since start.
// The comparison reads the 10 ms bucket for its p99 criterion.
func timeHandoff(start time.Time) {
handoffSecondsOnce.Do(func() {
handoffSeconds = metrics.NewHistogram("csm_admission_handoff_seconds",
"Time a response funnel waits to hand an automatic response to admission.",
[]float64{.001, .0025, .005, .01, .025, .1, 1})
metrics.MustRegister("csm_admission_handoff_seconds", handoffSeconds)
})
handoffSeconds.Observe(time.Since(start).Seconds())
}
// respond asks admission for a response of kind to the address or prefix
// f names, through via when it is set, and returns the minted evidence for
// a derived response to answer later, whether legacy must leave the block
// to admission, and whether admission accepted it to apply.
func respond(kind admission.Kind, f alert.Finding, target string, via admission.Entry) (admission.Evidence, bool, bool) {
a := getResponseAdmission()
if a == nil {
return admission.Evidence{}, false, false
}
defer timeHandoff(time.Now())
executes := a.ExecutesBlocks()
// Spray lifetime and budget stay with legacy. Use the handoff's owner
// and execution snapshot so a concurrent publication cannot take it.
if kind == admission.KindBlockIP && f.BlockTTL > 0 && executes {
return admission.Evidence{}, false, false
}
e, err := a.Mint(f)
if err != nil {
a.Refuse(kind, f, via, err)
return admission.Evidence{}, taken(executes, kind, err, nil), false
}
t := taken(executes, kind, nil, a.Respond(kind, target, e, via))
return e, t, t
}
// taken reports whether legacy must leave a response of kind to admission:
// admission applies block picks and accepted this one, or refused it as a
// client behind a trusted proxy or a CDN edge, which no block contains.
// Any other refusal leaves the legacy block in place.
func taken(executes bool, kind admission.Kind, mintErr, respondErr error) bool {
if !executes || kind != admission.KindBlockIP {
return false
}
if mintErr != nil {
c := admission.CauseOf(mintErr)
return c == admission.CauseUnreachable || c == admission.CauseProxyPeer
}
return respondErr == nil
}
// respondSpray asks admission to block a spray's prefix through the mail
// subnet entry, with each counted address as a root, read as its tracker
// recorded it, newest first, at most as many as one candidate holds. A
// spray recorded without its constituents names no address of its own.
func respondSpray(f alert.Finding, cidr string) {
if len(f.SprayConstituents) == 0 {
respond(admission.KindBlockSubnet, f, cidr, admission.EntryMailSubnet)
return
}
constituents := slices.Clone(f.SprayConstituents)
slices.SortFunc(constituents, func(a, b alert.SprayConstituent) int {
if c := b.LastSeen.Compare(a.LastSeen); c != 0 {
return c
}
return strings.Compare(a.Address, b.Address)
})
for _, c := range constituents[:min(len(constituents), admission.MaxRoots)] {
root := f
root.SourceIP = c.Address
root.Observation, root.Attribution = c.Observation, c.Attribution
respond(admission.KindBlockSubnet, root, cidr, admission.EntryMailSubnet)
}
}
// respondDerived hands admission the block a derived path applies, with the
// root it carries, through its entry, and reports whether legacy must leave
// it to admission. A permanent block is never asked for implicitly: it is
// handed over without a root, as a path that kept none is, and admission
// refuses it. A block asked to last longer than lease, which admission would
// cut to lease, keeps its legacy block while admission executes; a
// previewing admission still sees it. A root refusal is answered first, so
// an edge or a proxied client is still blocked by neither path.
func respondDerived(req ApplyBlockRequest, lease time.Duration) bool {
a := getResponseAdmission()
if a == nil {
return false
}
defer timeHandoff(time.Now())
executes := a.ExecutesBlocks()
if req.TTL == 0 {
return answerRoot(a, executes, admission.KindBlockIP, req.IP, admission.Evidence{}, req.Entry)
}
if executes && lease > 0 && req.TTL > lease && req.RootErr == nil {
return false
}
return answerPreparedRoot(a, executes, admission.KindBlockIP, req.IP, req.Root, req.Entry, req.RootFinding, req.RootErr, req.TTL)
}
// AdmissionRoot keeps a root for a derived path whose retained finding
// may no longer be available when it responds. Failure leaves no root.
func AdmissionRoot(f alert.Finding) admission.Evidence {
e, _ := PrepareAdmissionRoot(f)
return e
}
// PrepareAdmissionRoot preserves a local finding's mint refusal so a
// selected response can count it once with its original check and reason.
func PrepareAdmissionRoot(f alert.Finding) (admission.Evidence, error) {
a := getResponseAdmission()
if a == nil {
return admission.Evidence{}, nil
}
return a.Mint(f)
}
// AnswerPreparedRoot counts a failed mint once, without replacing its
// provenance refusal with a second rootless-response refusal. It reports
// whether legacy must leave the response to admission.
func AnswerPreparedRoot(kind admission.Kind, target string, root admission.Evidence, via admission.Entry, f alert.Finding, err error, ttl time.Duration) bool {
a := getResponseAdmission()
if a == nil {
return false
}
defer timeHandoff(time.Now())
return answerPreparedRoot(a, a.ExecutesBlocks(), kind, target, root, via, f, err, ttl)
}
// AnswerRoot hands admission a response of kind to target a derived path
// decided, with the root it kept, through its entry, and reports whether
// legacy must leave it to admission.
func AnswerRoot(kind admission.Kind, target string, root admission.Evidence, via admission.Entry, ttl ...time.Duration) bool {
a := getResponseAdmission()
if a == nil {
return false
}
defer timeHandoff(time.Now())
return answerRoot(a, a.ExecutesBlocks(), kind, target, root, via, ttl...)
}
// answerPreparedRoot and answerRoot answer with the caller's reading of
// whether a executes blocks, so one hand-off decides on one reading.
func answerPreparedRoot(a ResponseAdmission, executes bool, kind admission.Kind, target string, root admission.Evidence, via admission.Entry, f alert.Finding, err error, ttl time.Duration) bool {
if err != nil {
a.Refuse(kind, f, via, err)
return taken(executes, kind, err, nil)
}
return answerRoot(a, executes, kind, target, root, via, ttl)
}
func answerRoot(a ResponseAdmission, executes bool, kind admission.Kind, target string, root admission.Evidence, via admission.Entry, ttl ...time.Duration) bool {
return taken(executes, kind, nil, a.Respond(kind, target, root, via, ttl...))
}
// respondNetblock hands admission a netblock escalation. It rests on past
// blocks rather than a root admission could answer, so admission refuses
// it until range corroboration exists.
func respondNetblock() {
if a := getResponseAdmission(); a != nil {
defer timeHandoff(time.Now())
_ = a.Respond(admission.KindBlockSubnet, "", admission.Evidence{}, admission.EntryNetblock)
}
}
package checks
import (
"context"
"fmt"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// afAlgEvent is the parsed view of a single SYSCALL record tagged with the
// csm_af_alg_socket auditd key. Fields are kept as strings because the
// downstream consumer is a human-readable Finding message.
type afAlgEvent struct {
Timestamp string // e.g. "1761826283.452"
Serial string // e.g. "91234"
UID string
AUID string
PID string // process id from the SYSCALL record; needed for live kill reaction
Comm string
Exe string
}
// AFAlgEvent is the package-public view of afAlgEvent used by callers
// outside internal/checks (the daemon's live audit-log listener emits
// findings derived from this shape).
type AFAlgEvent = afAlgEvent
// ParseAFAlgEventLine is the exported alias of parseAFAlgEvent for the
// daemon's live listener. The unexported form stays internal so the
// rest of this package can refer to the type by its short name.
func ParseAFAlgEventLine(line string) (AFAlgEvent, bool) {
return parseAFAlgEvent(line)
}
// AFAlgOwner resolves the hosting account whose process opened the socket.
// The audit uid is resolved through the shared passwd cache; root, service
// users and unknown uids yield "" so the finding stays unattributed.
func AFAlgOwner(ev AFAlgEvent) string {
uid, err := strconv.ParseUint(ev.UID, 10, 32)
if err != nil {
return ""
}
return HostingAccountForUser(LookupUser(uint32(uid)))
}
// after reports whether e is strictly newer than other. Comparison is
// (Timestamp, Serial) lexicographic with numeric semantics.
func (e afAlgEvent) after(other afAlgEvent) bool {
if e.Timestamp != other.Timestamp {
eFloat, _ := strconv.ParseFloat(e.Timestamp, 64)
otherFloat, _ := strconv.ParseFloat(other.Timestamp, 64)
return eFloat > otherFloat
}
// Avoid the local name `os` here — it shadows the stdlib package
// of the same name and would silently break a future edit that
// adds an `os` import to this file.
eSerial, _ := strconv.Atoi(e.Serial)
otherSerial, _ := strconv.Atoi(other.Serial)
return eSerial > otherSerial
}
// parseAFAlgEvent extracts the relevant fields from a single audit log line.
// It returns (event, true) only when the line is a SYSCALL record carrying
// the csm_af_alg_socket key. Anything else returns ok=false.
func parseAFAlgEvent(line string) (afAlgEvent, bool) {
// Require type=SYSCALL: auditd also writes the rule's key into
// CONFIG_CHANGE records on every add_rule / remove_rule (i.e. every
// CSM restart), and those are not exploit signatures.
if !strings.HasPrefix(line, "type=SYSCALL ") {
return afAlgEvent{}, false
}
if !strings.Contains(line, `key="csm_af_alg_socket"`) {
return afAlgEvent{}, false
}
ts, serial, ok := parseAuditMsgID(line)
if !ok {
return afAlgEvent{}, false
}
ev := afAlgEvent{Timestamp: ts, Serial: serial}
ev.UID = extractAuditField(line, "uid")
ev.AUID = extractAuditField(line, "auid")
ev.PID = extractAuditField(line, "pid")
ev.Comm = extractAuditField(line, "comm")
ev.Exe = extractAuditField(line, "exe")
return ev, true
}
// parseAuditMsgID extracts (timestamp, serial) from `msg=audit(TS:SERIAL):`.
func parseAuditMsgID(line string) (string, string, bool) {
const marker = "msg=audit("
i := strings.Index(line, marker)
if i < 0 {
return "", "", false
}
rest := line[i+len(marker):]
end := strings.Index(rest, ")")
if end < 0 {
return "", "", false
}
inside := rest[:end]
colon := strings.Index(inside, ":")
if colon < 0 {
return "", "", false
}
ts := inside[:colon]
serial := inside[colon+1:]
if _, err := strconv.ParseFloat(ts, 64); err != nil {
return "", "", false
}
if _, err := strconv.Atoi(serial); err != nil {
return "", "", false
}
return ts, serial, true
}
// extractAuditField returns the value of `key=...` from an audit log line.
// Quoted values may contain spaces; bare values are whitespace-delimited.
func extractAuditField(line, key string) string {
prefix := key + "="
idx := 0
for {
i := strings.Index(line[idx:], prefix)
if i < 0 {
return ""
}
i += idx
// Require start-of-line or preceding whitespace so "auid=" doesn't
// match when we asked for "uid=".
if i > 0 && line[i-1] != ' ' && line[i-1] != '\t' {
idx = i + 1
continue
}
rest := line[i+len(prefix):]
if strings.HasPrefix(rest, `"`) {
rest = rest[1:]
end := strings.Index(rest, `"`)
if end < 0 {
return ""
}
return rest[:end]
}
end := strings.IndexAny(rest, " \t")
if end < 0 {
return rest
}
return rest[:end]
}
}
const (
afAlgLogPath = "/var/log/audit/audit.log"
afAlgCursorKey = "_af_alg_last_seen"
)
// CheckAFAlgSocketUsage scans the audit log for csm_af_alg_socket events
// and emits one Critical finding per strictly-newer event. The first run
// alerts on every event found — AF_ALG-from-userland is an exploit signature
// for CVE-2026-31431 ("Copy Fail"), not a baseline metric, so silent seeding
// would hide pre-existing compromise. The cursor in state.Store prevents
// duplicates on subsequent sweeps and survives daemon restarts.
//
// Filtering is delegated to grep so we don't load the whole multi-hundred-MB
// audit log into memory each tick (same precedent as getAuditShadowInfo in
// auth.go). RunAllowNonZero is required because grep returns exit 1 on
// "no match" — the healthy default — and that must not surface as an error.
func CheckAFAlgSocketUsage(_ context.Context, _ *config.Config, st *state.Store) []alert.Finding {
// RunAllowNonZero swallows every non-zero exit (see runCmdAllowNonZeroReal
// in helpers.go) — including exit 1 (no match) and exit 2 (audit log not
// installed). The non-error path is therefore the only one we need to
// reason about here: empty output means "nothing to do".
out, err := cmdExec.RunAllowNonZero("grep", "-a", "csm_af_alg_socket", afAlgLogPath)
if err != nil {
return nil
}
if len(out) == 0 {
return nil
}
cursorRaw, hasCursor := st.GetRaw(afAlgCursorKey)
cursor := decodeCursor(cursorRaw)
var findings []alert.Finding
highest := cursor
highestSet := hasCursor
for _, line := range strings.Split(string(out), "\n") {
ev, ok := parseAFAlgEvent(line)
if !ok {
continue
}
if hasCursor && !ev.after(cursor) {
continue
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "af_alg_socket_use",
Message: fmt.Sprintf("AF_ALG socket opened by uid=%s exe=%s", ev.UID, ev.Exe),
TenantID: AFAlgOwner(ev),
Details: fmt.Sprintf(
"Audit event: timestamp=%s serial=%s\nauid=%s uid=%s comm=%q exe=%q\n"+
"AF_ALG is essentially never used by cPanel/PHP workloads. This is\n"+
"the kernel-level exploit signature for CVE-2026-31431 (\"Copy Fail\").\n"+
"Investigate this process immediately and consider unloading algif_aead\n"+
"(modprobe -r algif_aead af_alg) and adding a modprobe.d blacklist.",
ev.Timestamp, ev.Serial, ev.AUID, ev.UID, ev.Comm, ev.Exe,
),
})
if !highestSet || ev.after(highest) {
highest = ev
highestSet = true
}
}
if highestSet {
st.SetRaw(afAlgCursorKey, encodeCursor(highest))
}
return findings
}
func encodeCursor(ev afAlgEvent) string { return ev.Timestamp + ":" + ev.Serial }
func decodeCursor(s string) afAlgEvent {
if s == "" {
return afAlgEvent{}
}
colon := strings.Index(s, ":")
if colon < 0 {
return afAlgEvent{}
}
return afAlgEvent{Timestamp: s[:colon], Serial: s[colon+1:]}
}
package checks
import (
"context"
"errors"
"fmt"
"os"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/systemdrun"
)
// EnforceAction is the discrete outcome of the pure enforcement decision.
// Each value corresponds to one operational step the impure wrapper takes.
type EnforceAction int
const (
EnforceActionNoop EnforceAction = iota
EnforceActionRestoreMarker
EnforceActionUnloadModules
EnforceActionRestoreAndUnload
)
// decideAFAlgEnforcement is the pure, deterministic core of the enforcement
// check. Given the observed state of the marker file and the kernel module
// table, it returns exactly one action.
//
// Inputs:
// - markerPresent: /etc/modprobe.d/csm-copy-fail-mitigation.conf exists.
// - markerContentValid: that file's contents match the canonical CSM-managed
// content (so a hand-edited version still triggers a rewrite).
// - loaded: algif_aead OR af_alg is currently in /proc/modules.
//
// The "marker absent + modules loaded" combination intentionally returns
// Noop. The operator has not opted in to enforcement (no marker), so we will
// not unilaterally unload kernel modules they may legitimately be using —
// the existing hardening audit and auditd tripwire still surface the gap.
func decideAFAlgEnforcement(markerPresent, markerContentValid, loaded bool) EnforceAction {
if !markerPresent {
return EnforceActionNoop
}
switch {
case markerContentValid && !loaded:
return EnforceActionNoop
case markerContentValid && loaded:
return EnforceActionUnloadModules
case !markerContentValid && !loaded:
return EnforceActionRestoreMarker
default: // !markerContentValid && loaded
return EnforceActionRestoreAndUnload
}
}
// afAlgMarkerPath is the canonical location of the CSM-managed mitigation
// marker. Its presence is the signal that operator-driven enforcement is
// active for this host.
const afAlgMarkerPath = "/etc/modprobe.d/csm-copy-fail-mitigation.conf"
// canonicalAFAlgMarker is the byte-exact content the enforcer writes and
// re-asserts on drift. Hand-written variants (`blacklist algif_aead`, etc.)
// still satisfy the hardening audit, but the enforcer rewrites them to this
// canonical form so the file's content can be trivially validated.
const canonicalAFAlgMarker = `# CSM Copy Fail (CVE-2026-31431) mitigation — managed by CSM.
# Restored automatically by the af_alg_enforce critical-tier check.
# Remove this file (and run ` + "`csm harden --copy-fail`" + ` again) if you
# need to re-enable AF_ALG.
install algif_aead /bin/false
install af_alg /bin/false
`
// EnforceResult describes what enforceAFAlgBlocked observed and did, in a
// shape both the CLI subcommand and the periodic Check function can format
// for the operator without re-deriving the same conclusions.
//
// ModuleUnloaded reports the OBSERVED post-call state, not the syscall
// attempt: it is true only when /proc/modules no longer contains the
// targeted modules after `modprobe -r` ran. Use this field to distinguish
// "unload succeeded" from "unload attempted but module is in use".
type EnforceResult struct {
Action EnforceAction
MarkerPresent bool
MarkerValid bool
ModulesLoaded []string // names of currently-loaded targeted modules at start of call
MarkerWritten bool // wrapper wrote/restored the marker file this call
ModuleUnloaded bool // post-call /proc/modules shows targeted modules gone
Notes []string // operator-readable lines (warnings, stuck-module names)
}
func validateMarkerContent(data []byte) bool {
return string(data) == canonicalAFAlgMarker
}
// loadedTargetedModules returns the subset of {algif_aead, af_alg} currently
// present in /proc/modules. Used both before unload (to decide what to do)
// and after unload (to verify it actually took effect).
func loadedTargetedModules() []string {
var loaded []string
for _, mod := range loadModuleList() {
if mod == "algif_aead" || mod == "af_alg" {
loaded = append(loaded, mod)
}
}
return loaded
}
// enforceAFAlgBlocked inspects the marker file and /proc/modules, calls the
// pure decideAFAlgEnforcement, and applies the resulting action via osFS
// and cmdExec. Errors from osFS.WriteFile or unexpected osFS.Stat failures
// are returned; modprobe outcomes are observed via a post-call /proc/modules
// re-read, so an attempted unload is never reported as an observed success.
func enforceAFAlgBlocked() (EnforceResult, error) {
res := EnforceResult{}
// Marker presence + content check. ErrNotExist is the expected "advisory
// mode" path; any other Stat failure (e.g. EACCES on a hardened
// /etc/modprobe.d/) is surfaced as an error rather than silently
// classifying the host as advisory mode.
switch _, err := osFS.Stat(afAlgMarkerPath); {
case err == nil:
res.MarkerPresent = true
if data, readErr := osFS.ReadFile(afAlgMarkerPath); readErr == nil {
res.MarkerValid = validateMarkerContent(data)
}
case errors.Is(err, os.ErrNotExist):
// Advisory mode — operator has not opted in.
default:
return res, fmt.Errorf("stat %s: %w", afAlgMarkerPath, err)
}
res.ModulesLoaded = loadedTargetedModules()
res.Action = decideAFAlgEnforcement(res.MarkerPresent, res.MarkerValid, len(res.ModulesLoaded) > 0)
switch res.Action {
case EnforceActionRestoreMarker, EnforceActionRestoreAndUnload:
if err := osFS.WriteFile(afAlgMarkerPath, []byte(canonicalAFAlgMarker), 0o644); err != nil {
return res, err
}
res.MarkerWritten = true
}
switch res.Action {
case EnforceActionUnloadModules, EnforceActionRestoreAndUnload:
if err := unloadAFAlgModules(); err != nil {
res.Notes = append(res.Notes, fmt.Sprintf("module unload command failed: %v", err))
}
// Observe the kernel state even when the command fails or times out.
stillLoaded := loadedTargetedModules()
if len(stillLoaded) == 0 {
res.ModuleUnloaded = true
} else {
res.Notes = append(res.Notes, fmt.Sprintf(
"modprobe -r attempted but %v still loaded — module is in use; will retry next tick",
stillLoaded,
))
}
}
return res, nil
}
// Module removal is denied by the daemon's seccomp filter and kernel module
// protection. Keep those restrictions and delegate only the fixed opt-in action.
func unloadAFAlgModules() error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, err := systemdrun.Run(ctx, cmdExec.LookPath, cmdExec.RunContext, systemdrun.Options{Pipe: true, RuntimeMax: 30 * time.Second}, "modprobe", "-r", "algif_aead", "af_alg")
return err
}
// AFAlgMarkerPath returns the canonical marker file location. Exposed for
// the cmd/csm CLI which prints it to operators; production code in this
// package should reference the unexported constant directly.
func AFAlgMarkerPath() string { return afAlgMarkerPath }
// WriteAFAlgMarker forces the canonical marker content to disk regardless
// of current state. Used by `csm harden --copy-fail` to ensure subsequent
// EnforceAFAlgBlocked() runs see a valid marker even on first install.
func WriteAFAlgMarker() error {
return osFS.WriteFile(afAlgMarkerPath, []byte(canonicalAFAlgMarker), 0o644)
}
// EnforceAFAlgBlocked is the exported alias of enforceAFAlgBlocked for use
// by cmd/csm. The unexported form stays internal to the package so the
// periodic Check can call it without going through the export.
func EnforceAFAlgBlocked() (EnforceResult, error) { return enforceAFAlgBlocked() }
// enforceAFAlgBlockedFn is indirected so the observe-mode gate can be tested
// without a kernel to unload modules from.
var enforceAFAlgBlockedFn = enforceAFAlgBlocked
// CheckAFAlgEnforcement is the periodic critical-tier check that enforces
// the AF_ALG mitigation policy. When the operator has opted in (via
// `csm harden --copy-fail`, which writes the marker file), this check
// reverts any drift on each tick. It is a no-op in advisory mode.
//
// Emits a Warning finding (one per tick that took action) so the operator
// has an alert-pipeline record that system state was modified. Steady-state
// ticks emit no findings. Warning is the lowest severity available in the
// alert.Severity enum (Warning < High < Critical, no Info level).
func CheckAFAlgEnforcement(_ context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if cfg != nil && (cfg.AutoResponse.DisableEnforceAFAlg || cfg.ObserveMode()) {
return nil
}
res, err := enforceAFAlgBlockedFn()
if err != nil {
return []alert.Finding{{
Severity: alert.Warning,
Check: "af_alg_enforcement_corrected",
Message: "AF_ALG enforcement encountered an error",
Details: fmt.Sprintf("error: %v\nresult: %+v", err, res),
}}
}
if res.Action == EnforceActionNoop {
return nil
}
return []alert.Finding{{
Severity: alert.Warning,
Check: "af_alg_enforcement_corrected",
Message: fmt.Sprintf("AF_ALG enforcement re-applied (%s)", actionName(res.Action)),
Details: fmt.Sprintf(
"Action: %s\nMarker present: %v\nMarker valid: %v\nModules loaded: %v\nMarker written: %v\nModule unload succeeded: %v\nNotes: %v",
actionName(res.Action), res.MarkerPresent, res.MarkerValid, res.ModulesLoaded,
res.MarkerWritten, res.ModuleUnloaded, res.Notes,
),
}}
}
func actionName(a EnforceAction) string {
switch a {
case EnforceActionNoop:
return "Noop"
case EnforceActionRestoreMarker:
return "RestoreMarker"
case EnforceActionUnloadModules:
return "UnloadModules"
case EnforceActionRestoreAndUnload:
return "RestoreAndUnload"
}
return fmt.Sprintf("Unknown(%d)", int(a))
}
package checks
import (
"bytes"
"compress/gzip"
"os"
"strings"
)
// copyFailCVE is the CVE identifier used to match KernelCare/kpatch entries.
const copyFailCVE = "CVE-2026-31431"
// configHasBuiltInAFAlgAEAD reports whether the supplied kernel config text
// declares CRYPTO_USER_API_AEAD as built into the kernel image (=y) rather
// than as a loadable module (=m). On =y kernels, the modprobe blacklist
// mitigation is ineffective because there is no module to block from
// loading — the code is always present.
//
// The function tolerates the "is not set" comment form that kconfig uses
// for explicitly-disabled options. An unset CRYPTO_USER_API_AEAD returns
// false (not built-in; modular or absent).
func configHasBuiltInAFAlgAEAD(configText string) bool {
for _, line := range strings.Split(configText, "\n") {
line = strings.TrimSpace(line)
if line == "CONFIG_CRYPTO_USER_API_AEAD=y" {
return true
}
}
return false
}
// configHasModularAFAlgAEAD reports whether CRYPTO_USER_API_AEAD is set
// to =m (loadable module) in the supplied kernel config. Used by the
// "is this host actually exploitable?" policy decision: a kernel built
// without =y AND without =m has no AF_ALG aead interface at all, so
// Copy Fail is not reachable on it.
func configHasModularAFAlgAEAD(configText string) bool {
for _, line := range strings.Split(configText, "\n") {
line = strings.TrimSpace(line)
if line == "CONFIG_CRYPTO_USER_API_AEAD=m" {
return true
}
}
return false
}
// kcareReportsCopyFailPatched reports whether the supplied `kcarectl
// --patch-info` output advertises a patch covering the Copy Fail CVE.
// The kcarectl format emits per-patch records like:
//
// kpatch-name: rhel8/.../CVE-2026-NNNNN-foo.patch
// kpatch-cve: CVE-2026-NNNNN
// kpatch-cve-url: ...
//
// We match on a substring of the literal CVE id rather than the URL or
// filename so a future format change to either does not silently break
// detection.
func kcareReportsCopyFailPatched(out []byte) bool {
return bytes.Contains(out, []byte(copyFailCVE))
}
// kernelHasBuiltInAFAlgAEAD is the impure wrapper: it reads the kernel
// config from /boot/config-$(uname -r), falling back to /proc/config.gz,
// and asks configHasBuiltInAFAlgAEAD whether the AEAD interface is
// statically linked.
//
// Returns false when no config file is readable — the caller
// treats "config unknown" the same as "modular" so an inability to read
// the config never silently downgrades a real protection state.
func kernelHasBuiltInAFAlgAEAD() bool {
cfg, _ := readKernelConfigText()
if cfg == "" {
return false
}
return configHasBuiltInAFAlgAEAD(cfg)
}
// readKernelConfigText returns the running kernel's .config text from
// /boot/config-$(uname -r), falling back to /proc/config.gz (gunzipped).
// Returns ("", false) when neither is readable so the caller can decide
// whether to treat "unknown" as conservatively-vulnerable.
func readKernelConfigText() (string, bool) {
if uname, err := readKernelRelease(); err == nil {
if data, err := osFS.ReadFile("/boot/config-" + uname); err == nil {
return string(data), true
}
}
if data, err := osFS.ReadFile("/proc/config.gz"); err == nil {
zr, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
return "", false
}
defer func() { _ = zr.Close() }()
var buf bytes.Buffer
if _, err := buf.ReadFrom(zr); err != nil {
return "", false
}
return buf.String(), true
}
return "", false
}
// readKernelRelease returns the running kernel release string (the same
// value `uname -r` would print). Used to find the matching config-* file
// under /boot.
func readKernelRelease() (string, error) {
data, err := osFS.ReadFile("/proc/sys/kernel/osrelease")
if err != nil {
return "", err
}
return strings.TrimSpace(string(data)), nil
}
// kcareHasCopyFailPatch runs `kcarectl --patch-info` and reports whether
// KernelCare has applied a livepatch covering Copy Fail. Returns false
// (with a nil error) when kcarectl is absent or fails — KernelCare is
// optional and its absence is not an error condition.
func kcareHasCopyFailPatch() bool {
out, err := cmdExec.RunAllowNonZero("kcarectl", "--patch-info")
if err != nil {
return false
}
if len(out) == 0 {
return false
}
return kcareReportsCopyFailPatched(out)
}
// AFAlgKernelState is the assembled view of how the running kernel
// exposes AF_ALG. Used by the hardening audit, by csm harden, and by
// the live-monitor coordinator to decide whether protection is needed.
type AFAlgKernelState struct {
BuiltIn bool // CONFIG_CRYPTO_USER_API_AEAD=y in the running kernel
Modular bool // CONFIG_CRYPTO_USER_API_AEAD=m (loadable module exists)
ConfigReadable bool // /boot/config-$(uname -r) or /proc/config.gz was parseable
LivepatchActive bool // KernelCare/kpatch has applied a CVE-2026-31431 patch
}
// observeAFAlgKernelState assembles a kernel-state snapshot via the impure
// helpers above. The struct fields document precisely what we know vs
// what we couldn't determine — callers can apply policy without
// re-deriving the same probes.
func observeAFAlgKernelState() AFAlgKernelState {
state := AFAlgKernelState{}
if cfg, ok := readKernelConfigText(); ok {
state.ConfigReadable = true
state.BuiltIn = configHasBuiltInAFAlgAEAD(cfg)
state.Modular = configHasModularAFAlgAEAD(cfg)
}
state.LivepatchActive = kcareHasCopyFailPatch()
return state
}
// IsCopyFailExploitable reports whether this kernel is currently
// vulnerable to Copy Fail (CVE-2026-31431). Used by the daemon's
// live-monitor coordinator to skip starting the listener entirely on
// hosts that don't need protection — saving the inotify watch + tick
// loop for hosts that actually face the threat.
//
// Conservative defaults: when the kernel config is unreadable, treat
// the host as exploitable (better to over-monitor than miss). When a
// KernelCare livepatch is in place, treat as patched regardless of the
// underlying config — the syscall path itself is fixed.
func (s AFAlgKernelState) IsCopyFailExploitable() bool {
if s.LivepatchActive {
return false
}
if s.ConfigReadable && !s.BuiltIn && !s.Modular {
// Kernel was definitively built without the AF_ALG aead interface.
// Nothing to exploit, no listener needed.
return false
}
// Either: confirmed-vulnerable (=y or =m without livepatch), or
// unknown (config unreadable). Both go to "exploitable" so we err
// on the side of monitoring.
return true
}
// String renders the kernel state for inclusion in operator-visible
// messages. The format is stable and short enough to embed in a single
// AuditResult.Message line.
func (s AFAlgKernelState) String() string {
switch {
case s.LivepatchActive:
return "kernel patched by KernelCare (livepatch active for " + copyFailCVE + ")"
case s.BuiltIn:
return "AF_ALG is built into the kernel (CONFIG_CRYPTO_USER_API_AEAD=y) and no livepatch is active"
case s.Modular:
return "AF_ALG aead is a loadable module on this kernel"
case s.ConfigReadable:
return "AF_ALG aead is not present in this kernel build"
default:
return "kernel config unreadable; treating as potentially vulnerable"
}
}
// ObserveAFAlgKernelState is the exported alias for cmd/csm and the
// daemon. Production code inside this package uses the unexported form
// directly.
func ObserveAFAlgKernelState() AFAlgKernelState { return observeAFAlgKernelState() }
// EnsureFile is a sentinel return value: callers can use os.IsNotExist
// to test for the "kernel-config file absent" case explicitly.
var ErrKernelConfigUnreadable = os.ErrNotExist
package checks
import (
"fmt"
"os"
"path/filepath"
"strings"
)
// SeccompDropInBaseName is the file name CSM writes inside each unit's
// /etc/systemd/system/<unit>.d/ override directory. Stable so the
// hardening audit can scan for it and the remove path can clean up
// without guessing.
const SeccompDropInBaseName = "csm-copy-fail-seccomp.conf"
// seccompDropInContent is the byte-exact body of every drop-in CSM
// writes. The marker comment lets a curious sysadmin understand who
// manages the file without consulting external docs.
const seccompDropInContent = `# CSM Copy Fail (CVE-2026-31431) seccomp mitigation - managed by CSM.
# Blocks socket(AF_ALG, ...) for processes spawned by this unit.
# Remove this file to disable.
[Service]
RestrictAddressFamilies=~AF_ALG
`
// afAlgSeccompCandidateUnits is the catalog of systemd units that, on
// shared-hosting servers, regularly spawn untrusted user-level code
// and therefore need the AF_ALG block. Units that do not exist on the
// running host are filtered out at apply time.
//
// The list is intentionally inclusive: a unit that does not exist
// adds zero overhead because we filter via systemctl list-unit-files
// before writing anything.
var afAlgSeccompCandidateUnits = []string{
// Web servers
"lshttpd.service", // LiteSpeed (cPanel default)
"httpd.service", // Apache (RHEL family, cPanel EA4 fallback)
"apache2.service", // Apache (Debian/Ubuntu)
"nginx.service", // Nginx
// PHP-FPM, cPanel EA4
"ea-php72-php-fpm.service",
"ea-php73-php-fpm.service",
"ea-php74-php-fpm.service",
"ea-php80-php-fpm.service",
"ea-php81-php-fpm.service",
"ea-php82-php-fpm.service",
"ea-php83-php-fpm.service",
"ea-php84-php-fpm.service",
"cpanel_php_fpm.service",
// PHP-FPM, generic distro versions
"php-fpm.service",
"php7.4-fpm.service",
"php8.0-fpm.service",
"php8.1-fpm.service",
"php8.2-fpm.service",
"php8.3-fpm.service",
"php8.4-fpm.service",
// Cron and mail (drop privileges to user before running content
// filters or scheduled scripts)
"crond.service",
"cron.service",
"exim.service",
"dovecot.service",
}
// SeccompUnitState describes one unit's mitigation status in operator
// terms. Returned by ScanAFAlgSeccompState so both the CLI and the
// hardening audit can render the same view without re-deriving it.
type SeccompUnitState struct {
Unit string // e.g. "lshttpd.service"
Exists bool // unit is registered with systemd on this host
HasFile bool // CSM-managed drop-in is present on disk
}
// ScanAFAlgSeccompState walks the candidate unit list and returns one
// SeccompUnitState per candidate. Units missing from systemd are still
// reported (Exists=false) so the operator can confirm CSM did not
// silently skip something they expected.
func ScanAFAlgSeccompState() []SeccompUnitState {
var out []SeccompUnitState
for _, u := range afAlgSeccompCandidateUnits {
s := SeccompUnitState{Unit: u}
s.Exists = systemdUnitExists(u)
s.HasFile = seccompDropInPresent(u)
out = append(out, s)
}
return out
}
// SeccompCoverageSummary collapses the per-unit scan into the two
// numbers an operator cares about: how many existing units have the
// CSM drop-in, and how many do not.
type SeccompCoverageSummary struct {
Covered []string // existing units with the CSM drop-in
Uncovered []string // existing units without the drop-in
NotInstalled []string // candidate units not registered with systemd
}
// SummarizeAFAlgSeccompCoverage rolls up ScanAFAlgSeccompState into
// the three-way summary above. Used by the hardening audit and the
// CLI status output.
func SummarizeAFAlgSeccompCoverage() SeccompCoverageSummary {
var s SeccompCoverageSummary
for _, u := range ScanAFAlgSeccompState() {
switch {
case !u.Exists:
s.NotInstalled = append(s.NotInstalled, u.Unit)
case u.HasFile:
s.Covered = append(s.Covered, u.Unit)
default:
s.Uncovered = append(s.Uncovered, u.Unit)
}
}
return s
}
// ApplyAFAlgSeccompDropIns writes the canonical drop-in file for every
// candidate unit that exists on this host AND does not already have
// the file. After all writes, runs systemctl daemon-reload and a
// reload-or-restart per touched unit so the seccomp filter takes
// effect immediately.
//
// Returns the list of units that received a new drop-in this call. An
// empty list with a nil error means everything was already covered
// (idempotent re-run).
func ApplyAFAlgSeccompDropIns() ([]string, error) {
var written []string
for _, u := range afAlgSeccompCandidateUnits {
if !systemdUnitExists(u) {
continue
}
if seccompDropInPresent(u) {
continue
}
if err := writeSeccompDropIn(u); err != nil {
return written, fmt.Errorf("write drop-in for %s: %w", u, err)
}
written = append(written, u)
}
if len(written) == 0 {
return nil, nil
}
if _, err := cmdExec.Run("systemctl", "daemon-reload"); err != nil {
return written, fmt.Errorf("systemctl daemon-reload: %w", err)
}
for _, u := range written {
// try-restart performs a full restart (re-exec the master process)
// only if the unit is currently active. A reload (e.g., SIGUSR2 to
// PHP-FPM) is NOT enough: systemd attaches seccomp filters at
// process spawn time, so the existing master has to be replaced
// for RestrictAddressFamilies to take effect on it and its
// workers. try-restart skips units that were intentionally
// stopped, so we don't surprise-start anything.
if _, err := cmdExec.RunAllowNonZero("systemctl", "try-restart", u); err != nil {
return written, fmt.Errorf("systemctl try-restart %s: %w", u, err)
}
}
return written, nil
}
// RemoveAFAlgSeccompDropIns deletes every CSM-managed seccomp drop-in
// found on disk and runs systemctl daemon-reload + reload-or-restart
// per touched unit so the seccomp filter is dropped from running
// processes. Idempotent: a unit without our drop-in is skipped.
//
// Returns the list of units whose drop-in was removed.
func RemoveAFAlgSeccompDropIns() ([]string, error) {
var removed []string
for _, u := range afAlgSeccompCandidateUnits {
if !seccompDropInPresent(u) {
continue
}
if err := osFS.Remove(seccompDropInPath(u)); err != nil && !os.IsNotExist(err) {
return removed, fmt.Errorf("remove drop-in for %s: %w", u, err)
}
// Best-effort: clean up the now-empty .d directory if we created it.
dir := seccompDropInDir(u)
_ = osFS.Remove(dir) // succeeds only if empty; harmless otherwise
removed = append(removed, u)
}
if len(removed) == 0 {
return nil, nil
}
if _, err := cmdExec.Run("systemctl", "daemon-reload"); err != nil {
return removed, fmt.Errorf("systemctl daemon-reload: %w", err)
}
for _, u := range removed {
if !systemdUnitExists(u) {
continue
}
// Same reasoning as Apply: full re-exec is required so the
// seccomp filter is dropped from the running master.
if _, err := cmdExec.RunAllowNonZero("systemctl", "try-restart", u); err != nil {
return removed, fmt.Errorf("systemctl try-restart %s: %w", u, err)
}
}
return removed, nil
}
// seccompDropInDir returns the override directory for the given unit:
// /etc/systemd/system/<unit>.d
func seccompDropInDir(unit string) string {
return filepath.Join("/etc/systemd/system", unit+".d")
}
// seccompDropInPath returns the full path to the CSM-managed drop-in
// file for the given unit.
func seccompDropInPath(unit string) string {
return filepath.Join(seccompDropInDir(unit), SeccompDropInBaseName)
}
// seccompDropInPresent reports whether the CSM-managed drop-in exists
// for the given unit. Content is not inspected here; the file's
// presence at the canonical path is the policy signal.
func seccompDropInPresent(unit string) bool {
_, err := osFS.Stat(seccompDropInPath(unit))
return err == nil
}
// writeSeccompDropIn creates the override directory and writes the
// canonical drop-in content. Errors propagate to the caller.
func writeSeccompDropIn(unit string) error {
dir := seccompDropInDir(unit)
if err := osFS.MkdirAll(dir, 0o755); err != nil {
return err
}
return osFS.WriteFile(seccompDropInPath(unit), []byte(seccompDropInContent), 0o644)
}
// systemdUnitExists asks systemctl whether the given unit is known to
// the running systemd. Returns false on any error, including missing
// systemctl binary, so a non-systemd host (rare on RHEL/Ubuntu) is
// treated as "no units to mitigate."
func systemdUnitExists(unit string) bool {
out, err := cmdExec.RunAllowNonZero(
"systemctl", "list-unit-files", unit, "--no-legend", "--no-pager",
)
if err != nil {
return false
}
// list-unit-files prints a row per match. An empty stdout means
// the unit name is not registered with this systemd instance.
return len(strings.TrimSpace(string(out))) > 0
}
package checks
import (
"errors"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
)
// A running Apache rejects recursive includes. The audit still bounds descent
// so a changed-on-disk configuration cannot recurse forever before the next
// config test or reload notices the error.
const apacheIncludeMaxDepth = 32
// apacheConfigLine is one line of an assembled Apache configuration, tagged
// with its source so findings can name the file that set a directive.
type apacheConfigLine struct {
File string
Text string
}
var apacheConditionalContainers = map[string]bool{
"if": true,
"else": true,
"elseif": true,
"ifmodule": true,
"ifdefine": true,
"ifversion": true,
"iffile": true,
"ifsection": true,
"ifdirective": true,
}
// assembleApacheConfig reads path and splices Include and IncludeOptional
// targets at the directive position. Apache applies later directives over
// earlier ones, so appending snippets would invert precedence.
func assembleApacheConfig(path string) []apacheConfigLine {
lines, _ := assembleApacheConfigWithStatus(path)
return lines
}
func assembleApacheConfigWithStatus(path string) ([]apacheConfigLine, bool) {
assembler := apacheConfigAssembler{
serverRoot: defaultApacheServerRoot(path),
activePaths: make(map[string]bool),
complete: true,
}
lines := assembler.appendPath(nil, path, false, 0)
if len(assembler.containers) != 0 {
assembler.complete = false
}
return lines, assembler.complete
}
// defaultApacheServerRoot covers the compiled-in layouts used by supported
// distributions when the main file does not set ServerRoot.
func defaultApacheServerRoot(path string) string {
dir := filepath.Dir(path)
if strings.EqualFold(filepath.Base(path), "httpd.conf") &&
strings.EqualFold(filepath.Base(dir), "conf") {
return filepath.Dir(dir)
}
return dir
}
type apacheConfigAssembler struct {
serverRoot string
activePaths map[string]bool
containers []string
complete bool
}
func (a *apacheConfigAssembler) appendPath(dst []apacheConfigLine, path string, optional bool, depth int) []apacheConfigLine {
if depth > apacheIncludeMaxDepth {
a.complete = false
return dst
}
abs, err := filepath.Abs(path)
if err != nil {
abs = filepath.Clean(path)
}
if a.activePaths[abs] {
a.complete = false
return dst
}
a.activePaths[abs] = true
defer delete(a.activePaths, abs)
if info, statErr := osFS.Stat(path); statErr == nil && info.IsDir() {
// An unreadable directory is not an empty one.
entries, readErr := osFS.ReadDir(path)
if readErr != nil {
a.complete = false
}
sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() })
for _, entry := range entries {
dst = a.appendPath(dst, filepath.Join(path, entry.Name()), false, depth+1)
}
return dst
}
data, err := osFS.ReadFile(path)
if err != nil {
if !optional || !errors.Is(err, os.ErrNotExist) {
a.complete = false
}
return dst
}
lines := strings.SplitAfter(string(data), "\n")
if n := len(lines); n > 0 && lines[n-1] == "" {
lines = lines[:n-1]
}
continued := ""
for i, physical := range lines {
line, continues := apacheContinuationPrefix(continued + physical)
if continues && i < len(lines)-1 {
continued = line
continue
}
continued = ""
// Section handlers discard their suffix before reading arguments;
// quotes in that ignored text must not invalidate the whole tree.
if tag, isContainer, tagValid := parseApacheContainerTag(line); isContainer {
switch {
case !tagValid || apacheArgumentHasVariable(tag.name):
a.complete = false
case tag.closing:
last := len(a.containers) - 1
if last < 0 || !strings.EqualFold(a.containers[last], tag.name) {
a.complete = false
} else {
a.containers = a.containers[:last]
}
default:
a.containers = append(a.containers, tag.name)
}
dst = append(dst, apacheConfigLine{File: path, Text: line})
continue
}
fields, valid := parseApacheDirectiveFields(stripApacheComment(line))
if !valid {
a.complete = false
dst = append(dst, apacheConfigLine{File: path, Text: line})
continue
}
if len(fields) == 0 {
dst = append(dst, apacheConfigLine{File: path, Text: line})
continue
}
// Apache expands Define and process environment variables before it
// reads the directive name. An unresolved name could be an Include.
if apacheArgumentHasVariable(fields[0]) {
a.complete = false
}
switch {
case strings.EqualFold(fields[0], "ServerRoot"):
if len(a.containers) != 0 || len(fields) != 2 || fields[1] == "" || apacheArgumentHasVariable(fields[1]) || !filepath.IsAbs(fields[1]) {
a.complete = false
} else {
a.serverRoot = filepath.Clean(fields[1])
}
dst = append(dst, apacheConfigLine{File: path, Text: line})
case strings.EqualFold(fields[0], "Include"), strings.EqualFold(fields[0], "IncludeOptional"):
optionalInclude := strings.EqualFold(fields[0], "IncludeOptional")
if len(fields) != 2 || fields[1] == "" || apacheArgumentHasVariable(fields[1]) {
a.complete = false
continue
}
paths, globbed, expanded := expandApacheInclude(fields[1], a.serverRoot)
if !expanded || (!optionalInclude && len(paths) == 0) {
a.complete = false
}
// A file a glob found must be read; only an absent literal
// IncludeOptional target may be missing.
for _, included := range paths {
dst = a.appendPath(dst, included, optionalInclude && !globbed, depth+1)
}
default:
dst = append(dst, apacheConfigLine{File: path, Text: line})
}
}
return dst
}
func apacheArgumentHasVariable(arg string) bool {
return strings.Contains(arg, "${")
}
// expandApacheInclude resolves an Include target. A glob reports whether
// every directory it walked could be read: filepath.Glob drops those
// errors, so an unreadable directory would look empty.
func expandApacheInclude(target, serverRoot string) (paths []string, globbed, complete bool) {
if !filepath.IsAbs(target) {
target = filepath.Join(serverRoot, target)
}
// APR normalizes absolute as well as relative Include arguments.
target = filepath.Clean(target)
if !apacheIncludeHasGlob(target) {
return []string{target}, false, true
}
matches, complete := checkedApacheConfigGlob(target)
sort.Strings(matches)
return matches, true, complete
}
func apacheContinuationPrefix(line string) (string, bool) {
// Only a backslash immediately before LF or CRLF continues a line.
// Spaces after a comment's backslash must not swallow the next setting.
if !strings.HasSuffix(line, "\n") {
return line, false
}
line = strings.TrimSuffix(line, "\n")
line = strings.TrimSuffix(line, "\r")
if !strings.HasSuffix(line, "\\") {
return line, false
}
return line[:len(line)-1], true
}
func stripApacheComment(line string) string {
// httpd only treats a leading hash as a comment. A hash in an Include
// path is literal; truncating it can silently select a different tree.
if strings.HasPrefix(strings.TrimLeft(line, " \t\r\n\v\f"), "#") {
return ""
}
return line
}
// parseApacheDirectiveFields handles the quoting and escapes accepted in
// include paths and container arguments.
func parseApacheDirectiveFields(line string) ([]string, bool) {
var fields []string
for {
line = strings.TrimLeft(line, " \t\r\n\v\f")
if line == "" {
return fields, true
}
var field strings.Builder
var quote byte
if line[0] == '\'' || line[0] == '"' {
quote, line = line[0], line[1:]
}
i := 0
for i < len(line) {
c := line[i]
if quote != 0 && c == quote || quote == 0 && strings.ContainsRune(" \t\r\n\v\f", rune(c)) {
break
}
if c == '\r' || c == '\n' {
return fields, false
}
// ap_getword_conf only decodes a doubled backslash or the
// matching quote inside a quoted argument.
if c == '\\' && i+1 < len(line) && (line[i+1] == '\\' || quote != 0 && line[i+1] == quote) {
i++
c = line[i]
}
field.WriteByte(c)
i++
}
if quote != 0 {
if i == len(line) {
return fields, false
}
i++
}
fields = append(fields, field.String())
line = line[i:]
}
}
type apacheParsedContainerTag struct {
name string
label string
closing bool
}
func parseApacheContainerTag(line string) (apacheParsedContainerTag, bool, bool) {
text := strings.TrimSpace(stripApacheComment(line))
if !strings.HasPrefix(text, "<") {
return apacheParsedContainerTag{}, false, true
}
wordEnd := strings.IndexAny(text, " \t\r\n\v\f")
if strings.HasPrefix(text, "</") {
word := text
if wordEnd >= 0 {
word = text[:wordEnd]
}
// Closing tags are directive names, not section arguments. httpd
// checks the whole first word, including its terminating bracket.
if !strings.HasSuffix(word, ">") {
return apacheParsedContainerTag{}, true, false
}
name := word[2 : len(word)-1]
if name == "" || strings.ContainsAny(name, "<>") {
return apacheParsedContainerTag{}, true, false
}
if wordEnd >= 0 {
switch strings.ToLower(name) {
case "ifmodule", "ifdefine", "ifversion", "iffile", "ifsection", "ifdirective":
// Active read-time sections require an exact closing line.
// Their runtime condition is unavailable to this parser.
return apacheParsedContainerTag{}, true, false
}
}
return apacheParsedContainerTag{name: name, label: name, closing: true}, true, true
}
// Core section handlers end their arguments at the last '>'. A bracket
// inside the directive name cannot terminate a nonempty argument list.
end := strings.LastIndexByte(text, '>')
if end < 0 || (wordEnd >= 0 && end < wordEnd) || (wordEnd < 0 && end != len(text)-1) {
return apacheParsedContainerTag{}, true, false
}
label := strings.TrimSpace(text[1:end])
fields, valid := parseApacheDirectiveFields(label)
if !valid || len(fields) == 0 {
return apacheParsedContainerTag{}, true, false
}
// mod_version parses a fixed argument list instead of raw section text.
if strings.EqualFold(fields[0], "IfVersion") && end != len(text)-1 {
return apacheParsedContainerTag{}, true, false
}
return apacheParsedContainerTag{name: fields[0], label: label}, true, true
}
type apacheLineContext struct {
Scope []string
ScopeKey []string
Condition string
LateCondition bool
}
// walkApacheLines calls fn for directive lines and validates the container
// stack. A mismatched close never pops an unrelated scope.
func walkApacheLines(lines []apacheConfigLine, fn func(apacheConfigLine, []string, apacheLineContext)) bool {
type container struct {
name string
label string
key string
scoped bool
conditional bool
lateConditional bool
}
var stack []container
valid := true
serial := 0
context := func() apacheLineContext {
var ctx apacheLineContext
var conditions []string
for _, item := range stack {
if item.scoped {
ctx.Scope = append(ctx.Scope, item.label)
ctx.ScopeKey = append(ctx.ScopeKey, item.key)
}
if item.conditional {
conditions = append(conditions, item.key)
}
if item.lateConditional {
ctx.LateCondition = true
}
}
ctx.Condition = strings.Join(conditions, "\x1e")
return ctx
}
for _, line := range lines {
text := strings.TrimSpace(stripApacheComment(line.Text))
if text == "" {
continue
}
tag, isContainer, tagValid := parseApacheContainerTag(text)
if isContainer {
if !tagValid {
valid = false
continue
}
if tag.closing {
if len(stack) == 0 || !strings.EqualFold(stack[len(stack)-1].name, tag.name) {
valid = false
continue
}
stack = stack[:len(stack)-1]
continue
}
serial++
lowerName := strings.ToLower(tag.name)
conditional := apacheConditionalContainers[lowerName]
key := canonicalApacheContainerKey(tag.label)
if conditional || lowerName == "virtualhost" {
key += "\x00" + strconv.Itoa(serial)
}
stack = append(stack, container{
name: tag.name,
label: normalizeApacheContainerLabel(tag.label),
key: key,
scoped: !conditional,
conditional: conditional,
lateConditional: lowerName == "if" || lowerName == "else" || lowerName == "elseif",
})
continue
}
fields, fieldsValid := parseApacheDirectiveFields(text)
if !fieldsValid {
valid = false
continue
}
if len(fields) > 0 {
fn(line, fields, context())
}
}
return valid && len(stack) == 0
}
func canonicalApacheContainerKey(label string) string {
fields, valid := parseApacheDirectiveFields(label)
if !valid || len(fields) == 0 {
return strings.ToLower(strings.TrimSpace(label))
}
fields[0] = strings.ToLower(fields[0])
return strings.Join(fields, "\x1f")
}
func normalizeApacheContainerLabel(label string) string {
fields, valid := parseApacheDirectiveFields(label)
if !valid {
return strings.TrimSpace(label)
}
return strings.Join(fields, " ")
}
type apacheDirectiveValue struct {
File string
Scope string
Value string
Conditional bool
lateConditional bool
scopeKey string
}
// apacheDirectiveValues returns the last explicit value in each scope and
// keeps conditional branches separate from the unconditional default.
func apacheDirectiveValues(lines []apacheConfigLine, directive string) ([]apacheDirectiveValue, bool) {
const serverScope = "server config"
values := make(map[string]apacheDirectiveValue)
var order []string
conditionalValues := make(map[string]apacheDirectiveValue)
var conditionalOrder []string
directiveValid := true
containersValid := walkApacheLines(lines, func(line apacheConfigLine, fields []string, ctx apacheLineContext) {
if !strings.EqualFold(fields[0], directive) {
return
}
if len(fields) < 2 {
directiveValid = false
return
}
key := serverScope
displayScope := serverScope
if len(ctx.Scope) > 0 {
key = strings.Join(ctx.ScopeKey, "\x1d")
displayScope = strings.Join(ctx.Scope, " > ")
}
value := apacheDirectiveValue{
File: line.File,
Scope: displayScope,
Value: strings.Join(fields[1:], " "),
Conditional: ctx.Condition != "",
lateConditional: ctx.LateCondition,
scopeKey: key,
}
if value.Conditional {
conditionalKey := key + "\x00" + ctx.Condition
if _, seen := conditionalValues[conditionalKey]; !seen {
conditionalOrder = append(conditionalOrder, conditionalKey)
}
conditionalValues[conditionalKey] = value
return
}
if _, seen := values[key]; !seen {
order = append(order, key)
}
values[key] = value
for conditionalKey, conditionalValue := range conditionalValues {
if conditionalValue.scopeKey == key && !conditionalValue.lateConditional {
conditionalValue.File = value.File
conditionalValue.Value = value.Value
conditionalValues[conditionalKey] = conditionalValue
}
}
})
out := make([]apacheDirectiveValue, 0, len(values)+len(conditionalValues))
for _, key := range order {
out = append(out, values[key])
}
for _, key := range conditionalOrder {
out = append(out, conditionalValues[key])
}
return out, containersValid && directiveValid
}
func apacheIndexesScopes(lines []apacheConfigLine) []string {
scopes, _ := apacheIndexesScopesWithStatus(lines)
return scopes
}
func apacheIndexesScopesWithStatus(lines []apacheConfigLine) ([]string, bool) {
const serverScope = "server config"
effective := make(map[string]bool)
conditionalEffective := make(map[string]bool)
conditionalScopes := make(map[string]string)
lateConditional := make(map[string]bool)
displayScopes := make(map[string]string)
seen := make(map[string]bool)
var order []string
optionsValid := true
containersValid := walkApacheLines(lines, func(_ apacheConfigLine, fields []string, ctx apacheLineContext) {
if !strings.EqualFold(fields[0], "Options") {
return
}
if len(fields) < 2 {
optionsValid = false
return
}
key := serverScope
displayScope := serverScope
if len(ctx.Scope) > 0 {
key = strings.Join(ctx.ScopeKey, "\x1d")
displayScope = strings.Join(ctx.Scope, " > ")
}
if !seen[key] {
seen[key] = true
order = append(order, key)
displayScopes[key] = displayScope
}
if ctx.Condition != "" {
conditionalKey := key + "\x00" + ctx.Condition
current, exists := conditionalEffective[conditionalKey]
if !exists {
current = effective[key]
}
conditionalEffective[conditionalKey] = optionsEnableIndexes(fields[1:], current)
conditionalScopes[conditionalKey] = key
lateConditional[conditionalKey] = ctx.LateCondition
return
}
effective[key] = optionsEnableIndexes(fields[1:], effective[key])
for conditionalKey, conditionalValue := range conditionalEffective {
if conditionalScopes[conditionalKey] == key && !lateConditional[conditionalKey] {
conditionalEffective[conditionalKey] = optionsEnableIndexes(fields[1:], conditionalValue)
}
}
})
var out []string
for _, key := range order {
enabled := effective[key]
for conditionalKey, conditionalValue := range conditionalEffective {
if conditionalScopes[conditionalKey] == key && conditionalValue {
enabled = true
break
}
}
if enabled {
out = append(out, displayScopes[key])
}
}
return out, containersValid && optionsValid
}
// optionsEnableIndexes applies one Options directive to the scope's current
// state. Signed tokens merge; bare tokens replace the option set.
func optionsEnableIndexes(tokens []string, current bool) bool {
merge := false
for _, token := range tokens {
if strings.HasPrefix(token, "+") || strings.HasPrefix(token, "-") {
merge = true
break
}
}
if !merge {
for _, token := range tokens {
if strings.EqualFold(token, "Indexes") || strings.EqualFold(token, "All") {
return true
}
}
return false
}
for _, token := range tokens {
name := strings.TrimLeft(token, "+-")
if !strings.EqualFold(name, "Indexes") && !strings.EqualFold(name, "All") {
continue
}
current = strings.HasPrefix(token, "+")
}
return current
}
package checks
import (
"net"
"strings"
"time"
)
// apiAuthSessionLogTailLines is how much of session_log the counted check
// reads for token denials. session_log is far quieter than access_log, so
// this covers the span of the access_log tail with a wide margin. A shorter
// read only leaves more 401s counted, never fewer.
const apiAuthSessionLogTailLines = 1000
// recentSessionTokenDenials returns the token denials in the session_log tail.
func recentSessionTokenDenials() []SessionTokenDenial {
var denials []SessionTokenDenial
for _, line := range tailFile(sessionLogPath, apiAuthSessionLogTailLines) {
if d, ok := ParseSessionTokenDenial(line); ok {
denials = append(denials, d)
}
}
return denials
}
// StaleSessionMatchWindow bounds how far apart, in log time, a 401 and the
// token denial that explains it may be. cpsrvd writes both during one page
// load: the parallel requests of a stale tab fail, the third failure purges
// the session, and the requests still in flight fail with a dead cookie.
// Measured bursts spread at most 4 s from the first failure to the purge.
const StaleSessionMatchWindow = 5 * time.Second
// SessionTokenDenial is cpsrvd's session_log record that a client presented
// a live session cookie with the wrong URL security token often enough that
// cPanel ended the session ("PURGE <account>:<session> tokendenied").
type SessionTokenDenial struct {
IP string
Account string
At time.Time
}
// tokenDenialServices are the session daemons that enforce the URL security
// token for a browser session. Admin API lines ([xml-api], [security]) carry
// the caller's address, not the browser's.
var tokenDenialServices = map[string]bool{
"cpaneld": true,
"webmaild": true,
"whostmgrd": true,
}
// ParseSessionTokenDenial reads a session_log line of the form
// "[<time>] info [cpaneld] <ip> PURGE <account>:<session> tokendenied ...".
// The fields are matched by position so a reason text that mentions
// tokendenied inside another purge line is not accepted.
// cpsrvd records whole seconds; realtime correlation indexes that precision.
func ParseSessionTokenDenial(line string) (SessionTokenDenial, bool) {
r, ok := ParseSessionRecord(line)
if !ok || r.At.IsZero() || r.At.Nanosecond() != 0 || r.Level != "info" ||
!tokenDenialServices[r.Service] || r.Event != "PURGE" || r.Reason() != "tokendenied" {
return SessionTokenDenial{}, false
}
ip, ok := r.ClientAddr()
if !ok {
return SessionTokenDenial{}, false
}
return SessionTokenDenial{IP: ip.String(), Account: r.Account, At: r.At}, true
}
// StaleSessionRequest is a 401 on a cPanel session URL
// (/cpsess<token>/...) as cpsrvd logged it in access_log.
type StaleSessionRequest struct {
IP string
User string
Token string
At time.Time
}
// ParseStaleSessionRequest reads a cpsrvd access_log line of the form
// `<ip> - <user> [01/02/2006:15:04:05 -0700] "<method> /cpsess<token>/... <proto>" 401`.
// A line of any other shape is rejected, which keeps it counted as an
// authentication failure. Token and password API calls use no session URL,
// so they never parse here.
func ParseStaleSessionRequest(line string) (StaleSessionRequest, bool) {
rec, ok := ParseCpanelAccessRecord(line)
// Evidence that excuses a failure needs a dated line with one request
// and one status.
if !ok || rec.Proxy || rec.Status != "401" || rec.At.IsZero() {
return StaleSessionRequest{}, false
}
q, _ := parseCpanelRequest(rec.first[1:])
token, ok := cpsessToken(q.Target)
if !ok {
return StaleSessionRequest{}, false
}
return StaleSessionRequest{IP: net.ParseIP(rec.IP).String(), User: rec.User, Token: token, At: rec.At}, true
}
// cpsessToken returns the digits of a leading "/cpsess<digits>/" segment.
func cpsessToken(path string) (string, bool) {
rest, ok := strings.CutPrefix(path, "/cpsess")
if !ok {
return "", false
}
digits := 0
for digits < len(rest) && rest[digits] >= '0' && rest[digits] <= '9' {
digits++
}
if digits == 0 || digits >= len(rest) || rest[digits] != '/' {
return "", false
}
return rest[:digits], true
}
// StaleSessionEvidence is what cPanel logged about stale browser sessions:
// its token denials and the session-URL 401s that named a user.
type StaleSessionEvidence struct {
Denials []SessionTokenDenial
Rejected []StaleSessionRequest
}
// Explains reports whether r is a 401 from a stale browser tab rather than
// a failed credential.
//
// A request that names a user is explained by a token denial for that
// account from the same address within StaleSessionMatchWindow: cPanel only
// writes one after that address presented the account's live session cookie
// with the wrong URL token.
//
// A request logged as "-" carried no usable session; once cPanel purges the
// session, the tab's remaining requests look like that. It is explained
// only when it reuses the URL token of a request the denial explains. The
// token is private to that browser tab, so another client behind the same
// address cannot borrow the explanation. The account holder can, for their
// own purged session, which gives them nothing they did not already have
// unless the session daemons check other credentials on session URLs.
func (e StaleSessionEvidence) Explains(r StaleSessionRequest) bool {
for _, d := range e.Denials {
if d.IP != r.IP || !withinStaleSessionWindow(r.At, d.At) {
continue
}
if r.User != "-" {
if r.User == d.Account {
return true
}
continue
}
for _, rej := range e.Rejected {
if rej.IP == r.IP && rej.Token == r.Token && rej.User == d.Account && withinStaleSessionWindow(rej.At, d.At) {
return true
}
}
}
return false
}
func withinStaleSessionWindow(a, b time.Time) bool {
gap := a.Sub(b)
return gap >= -StaleSessionMatchWindow && gap <= StaleSessionMatchWindow
}
package checks
import (
"errors"
"fmt"
"os"
"slices"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
)
// Block sources label who asked for an auto-response block. They feed the
// outcome metric and let evidence rows say which pipeline produced them.
const (
BlockSourceScan = "scan"
BlockSourceChallenge = "challenge"
BlockSourceIncident = "incident"
BlockSourceCentral = "central_intel"
)
// ApplyBlockRequest describes one auto-response IP block. EngineReason is
// handed to the firewall engine (its provenance inference keys on it);
// Reason is the human evidence recorded in the threat DB, the tracker, and
// findings.
type ApplyBlockRequest struct {
// ActionID preserves one admission identity across retries.
ActionID string
// FindingID is the original audit identity, captured before display truncation.
// Empty means this decision has no originating finding.
FindingID string
IP string
EngineReason string
Reason string
TTL time.Duration
Source string
// LoginSpray marks a mail spray scan block. It does not charge the
// durable scan budget; the scan loop counts it on its own budget.
LoginSpray bool
// Root is the admission evidence the block answers again, carried by
// the derived path that applies it; Entry is that path's entry. The
// zero root means the path kept none.
Root admission.Evidence
Entry admission.Entry
// RootFinding and RootErr preserve a planned local mint refusal.
RootFinding alert.Finding
RootErr error
// Check names the check behind the block for a path that kept neither
// a root nor its finding, as a challenge routed without admission.
Check string
}
// ApplyBlockResult carries the engine outcome plus the auto_block findings
// the caller must route into its alert pipeline (scan batch or the daemon's
// async block recorder) so digests and alerting see every block source.
type ApplyBlockResult struct {
Outcome firewall.BlockOutcome
Findings []alert.Finding
}
// ErrNoIPBlocker is returned when no firewall engine is wired. Callers must
// treat it as "the block did not happen", never as success.
var ErrNoIPBlocker = errors.New("firewall engine not available")
// ErrAutoBlockDisabled is returned when auto_response.enabled or block_ips is
// off, or the originating detector is in observe mode.
// Like ErrNoIPBlocker it means the block did not happen.
var ErrAutoBlockDisabled = errors.New("automatic IP blocking is disabled")
// ErrBlockTakenByAdmission reports that legacy must not apply this block.
// It does not promise that admission applied or will apply it.
var ErrBlockTakenByAdmission = errors.New("block left to admission")
// ApplyBlock is the single chokepoint for auto-response IP blocks issued
// outside the scan loop (challenge escalation, central intel, incident
// spray). It performs the block and the same evidence bookkeeping a scan
// auto-block gets: threat-DB row, blocked-IPs tracker entry, auto_block
// finding with the Cloudflare coverage warning, and permanent-block
// escalation counting. The scan loop shares the inner implementation and
// keeps its own batch semantics (rate limit, pending queue) around it.
//
// auto_response.enabled and block_ips gate every source; with either off
// ApplyBlock refuses with ErrAutoBlockDisabled before reaching the engine.
//
// Non-scan sources deliberately neither consume nor enforce
// auto_response.max_blocks_per_hour: challenge escalation and central intel
// are already gated upstream, and letting them starve or be starved by the
// scan budget would change containment behavior.
func ApplyBlock(cfg *config.Config, req ApplyBlockRequest) (ApplyBlockResult, error) {
check := req.Root.Check()
if check == "" {
check = req.RootFinding.Check
}
if check == "" {
check = req.Check
}
// The switches gate every automatic block, not only scan blocks. The
// engine's dry-run gate still applies below them.
if !cfg.AutoResponse.Enabled || !cfg.AutoResponse.BlockIPs || !CheckResponseEnabled(cfg, check) {
return ApplyBlockResult{Outcome: firewall.BlockOutcomeNoop}, ErrAutoBlockDisabled
}
// A trusted proxy carries every visitor behind it: refuse it as the
// engine refuses infra, before admission is asked.
if IsTrustedProxy(req.IP, cfg) {
return ApplyBlockResult{Outcome: firewall.BlockOutcomeNoop}, fmt.Errorf("%w: %s is a trusted proxy", firewall.ErrIPProtected, req.IP)
}
if respondDerived(req, parseExpiry(cfg.AutoResponse.BlockExpiry)) {
return ApplyBlockResult{Outcome: firewall.BlockOutcomeNoop}, ErrBlockTakenByAdmission
}
blocker := getIPBlocker()
if blocker == nil {
res := ApplyBlockResult{Outcome: firewall.BlockOutcomeNoop}
observeBlockOutcome(res.Outcome, ErrNoIPBlocker, req.Source)
return res, ErrNoIPBlocker
}
work := autoBlockQueues.acquire()
defer work.finish()
state := work.loadState(cfg.StatePath)
attemptAt := autoBlockNow()
res, err := applyBlockLocked(cfg, blocker, state, req, work.progress, func(err error) { work.directOutcome(req.IP, attemptAt, err) })
// Derived paths retain the admitted root, the source finding or at least
// its check. Keep that attribution on their notices as the scan path does.
cause := alert.Cause{FindingID: req.Root.FindingID(), Check: req.Root.Check()}
if cause.Check == "" && req.RootFinding.Check != "" {
cause = alert.CauseOf(req.RootFinding)
}
if cause.Check == "" {
cause.Check = req.Check
}
if req.FindingID != "" {
cause.FindingID = req.FindingID
}
if cause != (alert.Cause{}) {
for i := range res.Findings {
res.Findings[i].Cause = &cause
}
}
if !errors.Is(err, firewall.ErrIPProtected) {
work.observe(err)
}
work.progress()
work.saveState(cfg.StatePath, state)
work.complete()
return res, err
}
// AdmissionBlock is a block the admission owner applied, as its verified
// firewall row records it.
type AdmissionBlock struct {
IP, Reason, Check, FindingID string
BlockedAt, ExpiresAt time.Time
}
// RecordAdmissionBlocks records blocks the admission owner applied where a
// legacy block is recorded: a tracker row, which netblock escalation
// counts, a permanent-block strike, and the Critical auto_block finding it
// returns for the caller's alert pipeline. A block already tracked at its
// expiry is skipped, so a repeated delivery changes nothing. It promotes
// nothing: a permanent block is not an admission response.
func RecordAdmissionBlocks(cfg *config.Config, blocks []AdmissionBlock) []alert.Finding {
work := autoBlockQueues.acquire()
defer work.finish()
state, err := work.readState(cfg.StatePath)
if err != nil {
fmt.Fprintf(os.Stderr, "autoblock: %v; skipping admission bookkeeping\n", err)
return nil
}
var fresh []AdmissionBlock
for _, b := range blocks {
work.progress()
if slices.ContainsFunc(state.IPs, func(r blockedIP) bool { return r.IP == b.IP && r.ExpiresAt.Equal(b.ExpiresAt) }) {
continue
}
state.IPs = append(state.IPs, blockedIP{IP: b.IP, FindingID: b.FindingID, Reason: b.Reason, BlockedAt: b.BlockedAt, ExpiresAt: b.ExpiresAt})
fresh = append(fresh, b)
}
if len(fresh) == 0 {
work.complete()
return nil
}
// Commit duplicate markers before any best-effort side effects. A crash
// may lose a strike or notice, but replay must never count it twice.
work.progress()
if err := work.writeState(cfg.StatePath, state); err != nil {
logBlockStateFailure(cfg.StatePath, err)
return nil
}
blocker := getIPBlocker()
var findings []alert.Finding
for _, b := range fresh {
work.progress()
details := fmt.Sprintf("Reason: %s", b.Reason)
if cc, ok := blocker.(cloudflareCoverChecker); ok && cc.CloudflareCovers(b.IP) {
details += " (warning: " + firewall.CloudflareCoverageWarning + ")"
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "auto_block",
Message: fmt.Sprintf("AUTO-BLOCK: %s blocked (expires in %s)", b.IP, b.ExpiresAt.Sub(b.BlockedAt)),
Details: details,
Timestamp: time.Now(),
SourceIP: b.IP,
Cause: &alert.Cause{FindingID: b.FindingID, Check: b.Check},
})
permBlockStrike(cfg, b.IP, work.progress)
}
work.complete()
return findings
}
// permBlockStrike counts a temporary block of ip toward permanent-block
// escalation and reports whether ip reached the configured count. A listed
// C2 server is no offender.
func permBlockStrike(cfg *config.Config, ip string, progress func()) bool {
if !cfg.AutoResponse.PermBlock || c2Listed(cfg, ip) {
return false
}
count, interval := permBlockLimits(cfg)
progress()
return checkPermBlockEscalation(cfg.StatePath, ip, count, interval)
}
// permBlockLimits is the configured permanent-block count and interval.
func permBlockLimits(cfg *config.Config) (int, time.Duration) {
// Load fills the default and validation rejects anything lower, so
// this only catches a Config assembled in code.
count := cfg.AutoResponse.PermBlockCount
if count < config.MinBlockEscalationCount {
count = config.DefaultPermBlockCount
}
return count, parseExpiryWithDefault(cfg.AutoResponse.PermBlockInterval, config.DefaultPermBlockInterval)
}
// requestBlocker takes a full firewall request.
type requestBlocker interface {
BlockIPRequest(firewall.ActionRequest, *firewall.ScanAdmission) (firewall.BlockOutcome, error)
}
// durableActionBlocker supplies atomic admission when the engine owns durable
// action state. The existing scan policy is passed to that transaction.
type durableActionBlocker interface {
DurableActionsEnabled() bool
requestBlocker
}
// applyBlockLocked performs one block attempt plus the evidence bookkeeping
// a live outcome requires. The caller holds blockStateMu and owns loading
// and saving state. It writes no stderr lines for live blocks - callers
// keep their own operational logging - and emits findings instead of
// dispatching them.
func applyBlockLocked(cfg *config.Config, blocker IPBlocker, state *blockState, req ApplyBlockRequest, progress func(), observe func(error)) (ApplyBlockResult, error) {
progress()
var outcome firewall.BlockOutcome
var err error
durableOutcome := false
if durable, ok := blocker.(durableActionBlocker); ok && durable.DurableActionsEnabled() {
durableOutcome = true
var admission *firewall.ScanAdmission
if req.Source == BlockSourceScan && !req.LoginSpray {
limit := cfg.AutoResponse.MaxBlocksPerHour
if limit <= 0 {
limit = config.DefaultMaxBlocksPerHour
}
admission = &firewall.ScanAdmission{Window: autoBlockNow().Format("2006-01-02T15"), Limit: limit}
}
outcome, err = durable.BlockIPRequest(firewall.ActionRequest{
ID: req.ActionID, Operation: "block", Target: req.IP, Reason: req.EngineReason,
Source: req.Source, FindingID: req.FindingID, TTL: req.TTL, Actor: "daemon", Automatic: true,
LoginSpray: req.LoginSpray,
}, admission)
} else if rb, ok := blocker.(requestBlocker); ok && req.LoginSpray {
// The request BlockIPOutcomeWithFindingID sends, plus the flag the
// engine needs to tag the row.
outcome, err = rb.BlockIPRequest(firewall.ActionRequest{
Operation: "block", Target: req.IP, Reason: req.EngineReason, TTL: req.TTL,
FindingID: req.FindingID, Automatic: true, Actor: "daemon", LoginSpray: true,
}, nil)
} else {
outcome, err = callBlockIP(blocker, req.IP, req.EngineReason, req.TTL, req.FindingID)
}
observe(err)
observeBlockOutcome(outcome, err, req.Source)
res := ApplyBlockResult{Outcome: outcome}
verifiedAuditPending := durableOutcome && outcome == firewall.BlockOutcomeLive && errors.Is(err, firewall.ErrActionAuditPending)
if err != nil && !verifiedAuditPending {
return res, err
}
switch outcome {
case firewall.BlockOutcomeLive:
// nft was mutated; record the block below.
case firewall.BlockOutcomeDryRun:
// dry-run intercepted: nft was NOT mutated. Do not record a real
// block locally or in the permanent threat DB; emit a Warning
// notice instead so operators see what would have been blocked.
res.Findings = append(res.Findings, alert.Finding{
Severity: alert.Warning,
Check: "auto_block",
Message: fmt.Sprintf("AUTO-BLOCK [dry-run]: %s would be blocked (expires in %s)", req.IP, req.TTL),
Details: fmt.Sprintf("Reason: %s", req.Reason),
Timestamp: time.Now(),
SourceIP: req.IP,
})
return res, nil
case firewall.BlockOutcomeAllowed, firewall.BlockOutcomeAllowlisted, firewall.BlockOutcomeNoop:
// Allowed: the verdict callback deliberately declined. Allowlisted:
// operator allow or verified bot. Noop: already blocked or guard
// rejected. Nothing to record for any of them.
return res, nil
default:
fmt.Fprintf(os.Stderr, "auto-block: unknown block outcome %q for %s, skipping local state\n", outcome, req.IP)
return res, nil
}
// Record in the local threat DB with the same lifetime as the firewall
// block. A permanent record here would re-flag the IP via ip_reputation
// after the temp block lapses and re-block it forever (permablock loop).
progress()
if db := GetThreatDB(); db != nil {
db.AddTemporary(req.IP, req.Reason, req.TTL)
}
state.IPs = append(state.IPs, blockedIP{
IP: req.IP,
FindingID: req.FindingID,
Reason: req.Reason,
BlockedAt: time.Now(),
ExpiresAt: time.Now().Add(req.TTL),
LoginSpray: req.LoginSpray,
})
details := fmt.Sprintf("Reason: %s", req.Reason)
progress()
if cc, ok := blocker.(cloudflareCoverChecker); ok && cc.CloudflareCovers(req.IP) {
details += " (warning: " + firewall.CloudflareCoverageWarning + ")"
}
res.Findings = append(res.Findings, alert.Finding{
Severity: alert.Critical,
Check: "auto_block",
Message: fmt.Sprintf("AUTO-BLOCK: %s blocked (expires in %s)", req.IP, req.TTL),
Details: details,
Timestamp: time.Now(),
SourceIP: req.IP,
})
// Permanent block escalation: promote after N temp blocks within the
// interval. Every source counts - a challenge-timeout or central-intel
// block is the same repeat-offender evidence as a scan block.
if permBlockStrike(cfg, req.IP, progress) {
count, interval := permBlockLimits(cfg)
permReason := fmt.Sprintf("PERMBLOCK: %d temp blocks within %s", count, interval)
progress()
if promoteToPermanentBlock(blocker, req.IP, permReason, req.FindingID) {
res.Findings = append(res.Findings, alert.Finding{
Severity: alert.Critical,
Check: "auto_block",
Message: fmt.Sprintf("AUTO-PERMBLOCK: %s promoted to permanent block (%d temp blocks)", req.IP, count),
Timestamp: time.Now(),
SourceIP: req.IP,
})
}
}
return res, err
}
package checks
import (
"context"
"encoding/json"
"fmt"
"net"
"path/filepath"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"gopkg.in/yaml.v3"
)
func CheckShadowChanges(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
var findings []alert.Finding
info, err := osFS.Stat("/etc/shadow")
if err != nil {
return nil
}
mtime := info.ModTime()
mtimeKey := "_shadow_mtime"
hashKey := "_shadow_hash"
// Load current shadow entries (user:hash pairs, no sensitive data stored)
currentEntries := parseShadowUsers()
currentHash := hashBytes([]byte(fmt.Sprintf("%v", currentEntries)))
prevMtimeRaw, mtimeExists := store.GetRaw(mtimeKey)
prevHash, hashExists := store.GetRaw(hashKey)
if mtimeExists {
var lastMtime time.Time
if err := json.Unmarshal([]byte(prevMtimeRaw), &lastMtime); err == nil {
if mtime.After(lastMtime) {
// Shadow file was modified - find what changed
var details string
if hashExists && prevHash != currentHash {
changed := diffShadowChanges(store, currentEntries)
if len(changed) > 0 {
details = fmt.Sprintf("Previous: %s\nCurrent: %s\nAccounts changed: %s",
lastMtime.Format("2006-01-02 15:04:05"),
mtime.Format("2006-01-02 15:04:05"),
strings.Join(changed, ", "))
}
}
if details == "" {
details = fmt.Sprintf("Previous: %s\nCurrent: %s",
lastMtime.Format("2006-01-02 15:04:05"),
mtime.Format("2006-01-02 15:04:05"))
}
// Check if within upcp window
sev := alert.Critical
if cfg.Suppressions.UPCPWindowStart != "" {
now := time.Now()
h, m := now.Hour(), now.Minute()
nowMin := h*60 + m
start := parseTimeMin(cfg.Suppressions.UPCPWindowStart)
end := parseTimeMin(cfg.Suppressions.UPCPWindowEnd)
if nowMin >= start && nowMin <= end {
sev = alert.Warning
}
}
// Check auditd for who made the change
auditInfo := getAuditShadowInfo()
if auditInfo != "" {
details += "\n" + auditInfo
}
// Suppress alerts for password changes made by infra IPs
// (admin-initiated password resets via WHM/xml-api), but only
// when the explaining log event is newer than the shadow file's
// last-seen mtime. A stale infra event from a prior change must
// not mask a fresh, unexplained /etc/shadow edit.
if isInfraShadowChange(cfg, lastMtime) {
// Still update state, but don't alert
goto storeState
}
// Separate root password change (higher severity)
changed := diffShadowChanges(store, currentEntries)
rootChanged := false
userCount := 0
for _, c := range changed {
if c == "root" {
rootChanged = true
} else {
userCount++
}
}
if rootChanged {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "root_password_change",
Message: "Root password changed",
Details: details,
})
}
// Bulk password changes (5+ accounts at once)
if userCount >= 5 {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "bulk_password_change",
Message: fmt.Sprintf("Bulk password change: %d accounts modified", userCount),
Details: details,
})
} else {
findings = append(findings, alert.Finding{
Severity: sev,
Check: "shadow_change",
Message: "/etc/shadow modified",
Details: details,
})
}
}
}
}
storeState:
// Store current state
mtimeData, _ := json.Marshal(mtime)
store.SetRaw(mtimeKey, string(mtimeData))
store.SetRaw(hashKey, currentHash)
// Store per-user hashes for diff next time
for user, hash := range currentEntries {
store.SetRaw("_shadow_user:"+user, hash)
}
return findings
}
// parseShadowUsers reads /etc/shadow and returns a map of user -> password hash.
// Only stores a hash of the hash, not the actual password hash.
func parseShadowUsers() map[string]string {
data, err := osFS.ReadFile("/etc/shadow")
if err != nil {
return nil
}
entries := make(map[string]string)
for _, line := range strings.Split(string(data), "\n") {
parts := strings.SplitN(line, ":", 3)
if len(parts) < 2 || parts[0] == "" {
continue
}
// Store a hash of the password field, not the field itself
entries[parts[0]] = hashBytes([]byte(parts[1]))
}
return entries
}
// diffShadowChanges compares current entries against stored per-user hashes.
func diffShadowChanges(store *state.Store, current map[string]string) []string {
var changed []string
for user, hash := range current {
prev, exists := store.GetRaw("_shadow_user:" + user)
if exists && prev != hash {
changed = append(changed, user)
} else if !exists {
changed = append(changed, user+" (new)")
}
}
return changed
}
// getAuditShadowInfo checks auditd for recent shadow change events.
func getAuditShadowInfo() string {
out, err := runCmd("grep", "csm_shadow_change", "/var/log/audit/audit.log")
if err != nil || len(out) == 0 {
return ""
}
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
if len(lines) == 0 {
return ""
}
// Get the last event
last := lines[len(lines)-1]
// Extract exe= field
exe := ""
for _, part := range strings.Fields(last) {
if strings.HasPrefix(part, "exe=") {
exe = strings.Trim(strings.TrimPrefix(part, "exe="), "\"")
break
}
}
// Decode hex comm if present
comm := ""
for _, part := range strings.Fields(last) {
if strings.HasPrefix(part, "comm=") {
raw := strings.Trim(strings.TrimPrefix(part, "comm="), "\"")
decoded := decodeHexString(raw)
if decoded != "" {
comm = decoded
} else {
comm = raw
}
break
}
}
if exe != "" || comm != "" {
return fmt.Sprintf("Changed by: %s (command: %s)", exe, comm)
}
return ""
}
// decodeHexString tries to decode a hex-encoded string (auditd encodes some comm fields).
func decodeHexString(s string) string {
if len(s)%2 != 0 || len(s) < 4 {
return ""
}
// Check if it looks like hex (all hex chars)
for _, c := range s {
isHex := (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
if !isHex {
return ""
}
}
var result []byte
for i := 0; i < len(s); i += 2 {
// #nosec G115 -- hexVal returns 0..15; (h<<4)|h fits in a byte.
b := byte(hexVal(s[i])<<4 | hexVal(s[i+1]))
if b == 0 {
break
}
result = append(result, b)
}
if len(result) == 0 {
return ""
}
return string(result)
}
func CheckUID0Accounts(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
data, err := osFS.ReadFile("/etc/passwd")
if err != nil {
return nil
}
for _, line := range strings.Split(string(data), "\n") {
user, unauthorized := classifyUID0Line(line)
if unauthorized {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "uid0_account",
Message: fmt.Sprintf("Unauthorized UID 0 account: %s", user),
Details: line,
})
}
}
return findings
}
func CheckSSHKeys(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
var findings []alert.Finding
// Check root authorized_keys
rootKeys := "/root/.ssh/authorized_keys"
if hash, err := hashFileContent(rootKeys); err == nil {
key := "_ssh_root_keys_hash"
prev, exists := store.GetRaw(key)
if exists && prev != hash {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "ssh_keys",
Message: "Root authorized_keys modified",
Details: fmt.Sprintf("File: %s", rootKeys),
})
}
store.SetRaw(key, hash)
}
// Check for new authorized_keys in /home. Rank by mtime desc so
// recently-touched accounts are processed first when the check timeout
// cuts iteration short.
homes, _ := accountHomeGlob("*/.ssh/authorized_keys")
for _, keyFile := range rankPathsByMtimeDesc(ctx, homes, accountScanMaxFiles(ctx, cfg)) {
if ctx.Err() != nil {
break
}
hash, err := hashFileContent(keyFile)
if err != nil {
continue
}
key := fmt.Sprintf("_ssh_user_keys:%s", keyFile)
prev, exists := store.GetRaw(key)
if exists && prev != hash {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "ssh_keys",
Message: fmt.Sprintf("User authorized_keys modified: %s", keyFile),
})
}
store.SetRaw(key, hash)
}
return findings
}
func CheckAPITokens(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
var findings []alert.Finding
// WHM root API tokens
if f, fire := checkWHMRootAPITokens(store); fire {
findings = append(findings, f)
}
// User API tokens - read directly from disk instead of spawning uapi per user.
// Token files are JSON at /home/<user>/.cpanel/api_tokens/<token_name>.
// Rank account dirs by mtime desc so recently touched accounts are processed
// first when the check timeout cuts iteration short.
tokenDirs, _ := accountHomeGlob("*/.cpanel/api_tokens")
for _, tokenDir := range rankPathsByMtimeDesc(ctx, tokenDirs, accountScanMaxFiles(ctx, cfg)) {
if ctx.Err() != nil {
break
}
user := filepath.Base(filepath.Dir(filepath.Dir(tokenDir)))
tokenFiles, _ := osFS.Glob(filepath.Join(tokenDir, "*"))
for _, tokenFile := range tokenFiles {
if ctx.Err() != nil {
return findings
}
tokenName := filepath.Base(tokenFile)
data, err := osFS.ReadFile(tokenFile)
if err != nil {
continue
}
content := string(data)
// Check for full access with no IP whitelist
hasFullAccess := strings.Contains(content, `"has_full_access":1`) ||
strings.Contains(content, `"has_full_access": 1`)
noWhitelist := strings.Contains(content, `"whitelist_ips":null`) ||
strings.Contains(content, `"whitelist_ips": null`) ||
strings.Contains(content, `"whitelist_ips":[]`) ||
strings.Contains(content, `"whitelist_ips": []`) ||
!strings.Contains(content, "whitelist_ips")
if hasFullAccess && noWhitelist {
known := false
for _, t := range cfg.Suppressions.KnownAPITokens {
if tokenName == t {
known = true
break
}
}
if !known {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "api_tokens",
Message: fmt.Sprintf("User %s has full-access API token '%s' with no IP whitelist", user, tokenName),
Details: fmt.Sprintf("File: %s", tokenFile),
})
}
}
}
}
return findings
}
const (
whmAPITokensStateKey = "_whm_api_tokens_state" // #nosec G101 -- bbolt state-store key name, not a credential
whmAPITokensHashKey = "_whm_api_tokens_hash" // #nosec G101 -- bbolt state-store key name, not a credential
)
func checkWHMRootAPITokens(store *state.Store) (alert.Finding, bool) {
out, err := runCmd("whmapi1", "api_token_list", "--output=json")
if err == nil {
if cur, ok := parseWHMTokenSig(out); ok {
return checkStructuredWHMRootAPITokens(store, cur, nil)
}
}
return checkWHMRootAPITokensLegacyHash(store)
}
func checkWHMRootAPITokensLegacyHash(store *state.Store) (alert.Finding, bool) {
out, err := runCmd("whmapi1", "api_token_list")
if err != nil {
return alert.Finding{}, false
}
if cur, ok := parseWHMTokenSigYAML(out); ok {
return checkStructuredWHMRootAPITokens(store, cur, out)
}
return checkWHMRootAPITokensLegacyHashOutput(store, out)
}
func checkWHMRootAPITokensLegacyHashOnly(store *state.Store) (alert.Finding, bool) {
out, err := runCmd("whmapi1", "api_token_list")
if err != nil {
return alert.Finding{}, false
}
return checkWHMRootAPITokensLegacyHashOutput(store, out)
}
func checkStructuredWHMRootAPITokens(store *state.Store, cur tokenSig, legacyOut []byte) (alert.Finding, bool) {
_, hadStructuredState := store.GetRaw(whmAPITokensStateKey)
_, hadLegacyState := store.GetRaw(whmAPITokensHashKey)
finding, fire := diffWHMTokens(store, cur)
switch {
case !hadStructuredState && hadLegacyState:
// Migrating from the legacy hash: the tokens were already vetted under
// the old scheme, so diff against the legacy hash rather than
// re-flagging the whole set as new.
var legacyFinding alert.Finding
var legacyFire bool
if legacyOut != nil {
legacyFinding, legacyFire = checkWHMRootAPITokensLegacyHashOutput(store, legacyOut)
} else {
legacyFinding, legacyFire = checkWHMRootAPITokensLegacyHashOnly(store)
}
if legacyFire {
finding, fire = legacyFinding, true
}
case !hadStructuredState && !hadLegacyState:
// True first run with no prior state of any kind. A root API token
// created by an attacker before CSM was installed would otherwise be
// baselined as "known" and never alert. Surface the pre-existing
// operator tokens for review.
finding, fire = baselineWHMTokens(cur)
}
store.SetRaw(whmAPITokensStateKey, marshalTokenSig(cur))
if legacyOut != nil {
store.SetRaw(whmAPITokensHashKey, hashBytes(legacyOut))
}
return finding, fire
}
func checkWHMRootAPITokensLegacyHashOutput(store *state.Store, out []byte) (alert.Finding, bool) {
hash := hashBytes(out)
prev, exists := store.GetRaw(whmAPITokensHashKey)
store.SetRaw(whmAPITokensHashKey, hash)
if exists && prev != hash {
return alert.Finding{
Severity: alert.Critical,
Check: "api_tokens",
Message: "WHM root API tokens changed",
Details: "Run 'whmapi1 api_token_list' to review",
}, true
}
return alert.Finding{}, false
}
// baselineWHMTokens reports pre-existing root API tokens on the very first
// scan. Routine cluster-managed trust tokens churn on their own and are
// expected, so a baseline made up only of those stays silent; any
// operator/full-access token present at baseline is surfaced for review so a
// token planted before CSM was installed cannot pass as "known".
func baselineWHMTokens(cur tokenSig) (alert.Finding, bool) {
var operator []string
for name, info := range cur {
if isClusterManagedToken(info) || isClusterManagedTokenAddition(name, info) {
continue
}
operator = append(operator, name)
}
if len(operator) == 0 {
return alert.Finding{}, false
}
sort.Strings(operator)
return alert.Finding{
Severity: alert.High,
Check: "api_tokens",
Message: "Pre-existing WHM root API tokens present at baseline",
Details: "Review with 'whmapi1 api_token_list': " + strings.Join(operator, "; "),
}, true
}
// tokenSig maps each WHM root API token name to the security traits compared
// between scans.
type tokenSig map[string]tokenInfo
type tokenInfo struct {
FullAccess bool `json:"full_access"`
ClusterManaged bool `json:"cluster_managed"`
}
// isClusterManagedToken reports whether cPanel owns the token's lifecycle.
// DNS clustering creates, rotates, and deletes these on its own:
// - reverse_trust_<uuid>: trust granted to a remote WHM peer
// - <host>-trust (e.g. ns2-trust): local end of a trust relationship
//
// Their churn is routine and must not page like an attacker-created token.
func isClusterManagedToken(info tokenInfo) bool {
return info.ClusterManaged && !info.FullAccess
}
func isClusterManagedTokenAddition(name string, info tokenInfo) bool {
return strings.HasPrefix(name, "reverse_trust_") && isClusterManagedToken(info)
}
func clusterManagedFromACLs(name string, acls map[string]bool) bool {
if strings.HasPrefix(name, "reverse_trust_") {
return true
}
return strings.HasSuffix(name, "-trust") && acls["clustering"]
}
// parseWHMTokenSig decodes `whmapi1 api_token_list --output=json` into a
// tokenSig. A token whose value is not a JSON object or whose ACLs cannot be
// read is kept with FullAccess=false, so an unparsable entry never hides a
// token's presence. ok=false means the caller should use the legacy hash path.
func parseWHMTokenSig(out []byte) (tokenSig, bool) {
var env struct {
Data struct {
Tokens map[string]json.RawMessage `json:"tokens"`
} `json:"data"`
}
if err := json.Unmarshal(out, &env); err != nil || env.Data.Tokens == nil {
return nil, false
}
sig := make(tokenSig, len(env.Data.Tokens))
for name, raw := range env.Data.Tokens {
var t struct {
ACLs map[string]json.RawMessage `json:"acls"`
}
_ = json.Unmarshal(raw, &t)
acls := decodeWHMTokenACLs(t.ACLs)
sig[name] = tokenInfo{
FullAccess: acls["all"],
ClusterManaged: clusterManagedFromACLs(name, acls),
}
}
return sig, true
}
func parseWHMTokenSigYAML(out []byte) (tokenSig, bool) {
type yamlToken struct {
ACLs map[string]any `yaml:"acls"`
}
type yamlTokenData struct {
Tokens map[string]yamlToken `yaml:"tokens"`
}
var env struct {
Data yamlTokenData `yaml:"data"`
Result struct {
Data yamlTokenData `yaml:"data"`
} `yaml:"result"`
}
if err := yaml.Unmarshal(out, &env); err != nil {
return nil, false
}
tokens := env.Data.Tokens
if tokens == nil {
tokens = env.Result.Data.Tokens
}
if tokens == nil {
return nil, false
}
sig := make(tokenSig, len(tokens))
for name, raw := range tokens {
acls := decodeWHMTokenYAMLACLs(raw.ACLs)
sig[name] = tokenInfo{
FullAccess: acls["all"],
ClusterManaged: clusterManagedFromACLs(name, acls),
}
}
return sig, true
}
func decodeWHMTokenACLs(raw map[string]json.RawMessage) map[string]bool {
acls := make(map[string]bool, len(raw))
for name, value := range raw {
acls[name] = decodeWHMTokenACL(value)
}
return acls
}
func decodeWHMTokenYAMLACLs(raw map[string]any) map[string]bool {
acls := make(map[string]bool, len(raw))
for name, value := range raw {
acls[name] = decodeWHMTokenYAMLACL(value)
}
return acls
}
func decodeWHMTokenACL(raw json.RawMessage) bool {
switch strings.TrimSpace(string(raw)) {
case "1", "true", `"1"`, `"true"`:
return true
case "0", "false", `"0"`, `"false"`, "null", "":
return false
}
var n json.Number
if err := json.Unmarshal(raw, &n); err == nil {
return n.String() == "1"
}
var s string
if err := json.Unmarshal(raw, &s); err == nil {
return s == "1" || strings.EqualFold(s, "true")
}
var b bool
return json.Unmarshal(raw, &b) == nil && b
}
func decodeWHMTokenYAMLACL(raw any) bool {
switch v := raw.(type) {
case bool:
return v
case int:
return v == 1
case int64:
return v == 1
case uint64:
return v == 1
case float64:
return v == 1
case string:
s := strings.TrimSpace(v)
return s == "1" || strings.EqualFold(s, "true")
default:
return false
}
}
// marshalTokenSig serializes a tokenSig deterministically (encoding/json sorts
// map keys), so an unchanged set always produces an identical stored string.
func marshalTokenSig(sig tokenSig) string {
b, _ := json.Marshal(sig)
return string(b)
}
func unmarshalTokenSig(raw string) (tokenSig, bool) {
var sig tokenSig
if err := json.Unmarshal([]byte(raw), &sig); err == nil {
return sig, true
}
var legacy map[string]bool
if err := json.Unmarshal([]byte(raw), &legacy); err != nil {
return nil, false
}
sig = make(tokenSig, len(legacy))
for name, all := range legacy {
sig[name] = tokenInfo{
FullAccess: all,
ClusterManaged: !all && (strings.HasPrefix(name, "reverse_trust_") || strings.HasSuffix(name, "-trust")),
}
}
return sig, true
}
// diffWHMTokens compares the current token set against the previously stored
// one and returns a single finding when something changed. Severity splits on
// intent:
// - Critical: any token added outside generated reverse_trust churn, any
// non-cluster token removed, or ANY token gaining the full-access "all" ACL.
// - Warning: only generated reverse_trust additions/removals or recorded
// cluster trust removals changed. cPanel does this during normal DNS
// clustering and it must not page.
//
// The first scan after the key is introduced just records a baseline.
func diffWHMTokens(store *state.Store, cur tokenSig) (alert.Finding, bool) {
raw, exists := store.GetRaw(whmAPITokensStateKey)
if !exists {
return alert.Finding{}, false
}
prev, ok := unmarshalTokenSig(raw)
if !ok {
return alert.Finding{}, false
}
var critical, cluster []string
for name, info := range cur {
prevInfo, had := prev[name]
if !had {
if isClusterManagedTokenAddition(name, info) {
cluster = append(cluster, "added "+name)
} else {
critical = append(critical, "added "+name)
}
continue
}
if info.FullAccess && !prevInfo.FullAccess {
critical = append(critical, "escalated "+name+" to full access")
}
}
for name, info := range prev {
if _, still := cur[name]; still {
continue
}
if isClusterManagedToken(info) {
cluster = append(cluster, "removed "+name)
} else {
critical = append(critical, "removed "+name)
}
}
switch {
case len(critical) > 0:
sort.Strings(critical)
return alert.Finding{
Severity: alert.Critical,
Check: "api_tokens",
Message: "WHM root API tokens changed",
Details: "Review with 'whmapi1 api_token_list': " + strings.Join(critical, "; "),
}, true
case len(cluster) > 0:
sort.Strings(cluster)
return alert.Finding{
Severity: alert.Warning,
Check: "api_tokens",
Message: "WHM root cluster trust tokens changed",
Details: "cPanel DNS clustering churn (expected): " + strings.Join(cluster, "; "),
}, true
}
return alert.Finding{}, false
}
// shadowMutatingWHMEndpoints lists WHM JSON-API endpoints whose handlers
// rewrite /etc/shadow as a side effect:
// - suspendacct/unsuspendacct: lock/unlock password field, swap login shell
// - passwd/forcepasswordchange: set or expire user password
// - createacct/removeacct/killacct: add or remove the shadow entry entirely
//
// Hits on these endpoints from infra IPs explain shadow mtime changes
// without involving an attacker; hits from non-infra IPs do not.
var shadowMutatingWHMEndpoints = []string{
"/json-api/suspendacct",
"/json-api/unsuspendacct",
"/json-api/passwd",
"/json-api/forcepasswordchange",
"/json-api/createacct",
"/json-api/removeacct",
"/json-api/killacct",
}
// isInfraShadowChange reports whether every recent log signal that could
// explain a /etc/shadow modification was originated by an infra IP. It
// fuses two sources:
//
// 1. session_log PURGE password_change events (WHM/cPanel sets a new
// password, which goes through the session machinery).
// 2. successful api_tokens_log entries for shadow-mutating WHM JSON-API endpoints
// (suspendacct, passwd, createacct, ...). The cPanel session log does
// NOT record these because they are not session events, so the older
// session-only check fired on every internal `suspendacct` call.
//
// Returns true only if at least one such event was seen AND every event in
// both sources came from an infra IP (or loopback / "internal"). Any
// successful external API call or unparseable source short-circuits to false
// so a stolen token or compromised neighbour does not get a free suppression.
// since is the shadow file's previously recorded mtime. Only log events newer
// than since can explain the modification under investigation; stale tail lines
// (a legit infra password change from days ago) must not suppress a fresh,
// unrelated /etc/shadow edit.
func isInfraShadowChange(cfg *config.Config, since time.Time) bool {
sessFound, sessAllInfra := scanSessionLogShadow(cfg, since)
tokFound, tokAllInfra := scanAPITokensLogShadow(cfg, since)
return (sessFound || tokFound) && sessAllInfra && tokAllInfra
}
func cpanelShadowLogAfterSince(ts, since time.Time) bool {
if since.IsZero() {
return true
}
// cPanel logs only whole seconds, while shadow mtimes can carry
// subsecond precision. Treat the stored mtime's logged second as
// in-window so an infra event at 10:00:00.900 is not rejected just
// because the prior mtime was 10:00:00.500.
return !ts.Before(since.Truncate(time.Second))
}
// scanSessionLogShadow walks the cPanel session log for PURGE password_change
// events and reports whether any were seen and whether every non-loopback,
// non-"internal" source IP belonged to the infra allowlist.
func scanSessionLogShadow(cfg *config.Config, since time.Time) (foundAny, allInfra bool) {
allInfra = true
lines := tailFile(sessionLogPath, 100)
for i := len(lines) - 1; i >= 0; i-- {
line := lines[i]
r, ok := ParseSessionRecord(line)
if !ok {
if strings.Contains(line, "PURGE") && strings.Contains(line, "password_change") {
// An unreadable line naming a password change cannot prove
// it came from infra. Fail toward alerting.
foundAny = true
allInfra = false
return
}
continue
}
if r.Event != "PURGE" || r.Reason() != "password_change" {
continue
}
if r.At.IsZero() {
// A shadow-mutating line without a usable timestamp cannot prove
// it is stale. Fail toward alerting instead of suppressing.
foundAny = true
allInfra = false
return
}
if !cpanelShadowLogAfterSince(r.At, since) {
// Stale line cannot explain this modification.
continue
}
foundAny = true
// Only the admin services record who asked for the change.
var ip string
switch r.Service {
case "xml-api", "whostmgr", "security":
ip = r.Origin
}
if ip == "internal" {
continue
}
if !isTrustedShadowSource(ip, cfg) {
allInfra = false
return
}
}
return
}
// scanAPITokensLogShadow walks the WHM api_tokens_log for recent calls to
// JSON-API endpoints that rewrite /etc/shadow. Returns (foundAny, allInfra)
// with the same semantics as scanSessionLogShadow.
func scanAPITokensLogShadow(cfg *config.Config, since time.Time) (foundAny, allInfra bool) {
allInfra = true
lines := tailFile("/usr/local/cpanel/logs/api_tokens_log", 200)
for i := len(lines) - 1; i >= 0; i-- {
line := lines[i]
if !lineHitsShadowEndpoint(line) {
continue
}
if !apiTokensHTTPStatusOK(line) {
continue
}
ts, ok := CPanelLogTime(line)
if !ok {
// A shadow-mutating line without a usable timestamp cannot prove
// it is stale. Fail toward alerting instead of suppressing.
foundAny = true
allInfra = false
return
}
if !cpanelShadowLogAfterSince(ts, since) {
// Stale line cannot explain this modification.
continue
}
foundAny = true
ip := extractAPITokensHost(line)
if ip == "internal" {
continue
}
if !isTrustedShadowSource(ip, cfg) {
allInfra = false
return
}
}
return
}
func lineHitsShadowEndpoint(line string) bool {
path := extractAPITokensRequestPath(line)
if path == "" {
return false
}
for _, ep := range shadowMutatingWHMEndpoints {
if path == ep {
return true
}
}
return false
}
func apiTokensHTTPStatusOK(line string) bool {
status := extractAPITokensField(line, "HTTP Status: ['")
return strings.HasPrefix(status, "2")
}
// extractAPITokensHost pulls the source IP out of the api_tokens_log line
// shape used by whostmgrd:
//
// [ts] info [whostmgrd] Host: ['<ip>'] HTTP Status: [...], ...
func extractAPITokensHost(line string) string {
return extractAPITokensField(line, "Host: ['")
}
func extractAPITokensRequestPath(line string) string {
request := extractAPITokensField(line, "Request: ['")
fields := strings.Fields(request)
if len(fields) < 2 {
return ""
}
path := fields[1]
if idx := strings.IndexByte(path, '?'); idx >= 0 {
path = path[:idx]
}
return path
}
func extractAPITokensField(line, marker string) string {
idx := strings.Index(line, marker)
if idx < 0 {
return ""
}
rest := line[idx+len(marker):]
end := strings.Index(rest, "']")
if end < 0 {
return ""
}
return rest[:end]
}
func isTrustedShadowSource(ip string, cfg *config.Config) bool {
parsed := net.ParseIP(ip)
if parsed != nil && parsed.IsLoopback() {
return true
}
return isInfraIP(ip, cfg.InfraIPs)
}
func parseTimeMin(s string) int {
parts := strings.Split(s, ":")
if len(parts) != 2 {
return 0
}
h := 0
m := 0
fmt.Sscanf(parts[0], "%d", &h)
fmt.Sscanf(parts[1], "%d", &m)
return h*60 + m
}
package checks
import (
"crypto/rand"
"encoding/json"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/atomicio"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/mailranges"
"github.com/pidginhost/csm/internal/netutil"
"github.com/pidginhost/csm/internal/store"
)
const (
blockStateFile = "blocked_ips.json"
// maxPendingBlocks bounds the retry queue. Under a sustained flood or
// firewall outage the daemon can accumulate more distinct attacker IPs
// than it can block; without a bound the queue grows without limit and
// bloats blocked_ips.json. Overflow is named in stderr and surfaced as
// a warning so the resulting loss is visible to operators.
maxPendingBlocks = 1000
// maxPendingAge drops queued entries whose evidence has gone stale: a
// pending IP survives requeue cycles (rate limit, engine down, block
// errors), and blocking hours after the triggering findings is worse
// than not blocking. Two hours covers one full rate-limit window plus
// slack for the queue to drain.
maxPendingAge = 2 * time.Hour
)
// PendingRetryAge is how long a queued scan block may still be retried;
// the comparison tool matches such a retry to its earlier preview.
const PendingRetryAge = maxPendingAge
// IPBlocker abstracts the firewall engine for auto-blocking.
// When set, blocks go through nftables firewall engine.
type IPBlocker interface {
BlockIP(ip string, reason string, timeout time.Duration) error
UnblockIP(ip string) error
IsBlocked(ip string) bool
}
// outcomeBlocker is satisfied by engines that report what they actually
// did. When the wired IPBlocker supports it, the auto-block path uses the
// outcome to decide whether to apply local side effects: a dry-run or
// verdict-allowed call must not mutate blocked_ips.json, must not bump the
// hourly counter, and must not emit the operator-facing "AUTO-BLOCK"
// finding (which would falsely claim a real block landed). The plain
// IPBlocker interface stays as a back-compat fallback for tests and any
// legacy implementation.
type outcomeBlocker interface {
BlockIPOutcome(ip, reason string, timeout time.Duration) (firewall.BlockOutcome, error)
}
// liveBlocker is satisfied by engines that can query the live kernel firewall
// state, not just an in-memory cache built from state.json. The tracker
// reconcile loop prefers this because the cache can drift when nft
// auto-expires entries faster than CSM rewrites state.json, or when an
// out-of-band flush dropped entries the cache still claims are live. Falls
// back to IPBlocker.IsBlocked when the live query is unavailable.
type liveBlocker interface {
IsBlockedLive(ip string) (bool, error)
}
// liveBlockedLister is satisfied by engines that can dump their live blocked
// sets once per cycle. The reconcile pass prefers it over liveBlocker: the
// per-IP query dumps an entire family set on every call, so pruning N tracked
// IPs cost N full dumps of the same data.
type liveBlockedLister interface {
LiveBlockedSet() (firewall.LiveBlockedSnapshot, error)
}
type subnetBlocker interface {
BlockSubnet(cidr string, reason string, timeout time.Duration) error
}
// subnetBlockValidator is satisfied by firewall engines that can run their
// subnet safety and capability checks without changing firewall state. Dry-run
// notices use it so they only describe blocks the live path could attempt.
type subnetBlockValidator interface {
ValidateSubnetBlock(cidr string) error
}
// cloudflareCoverChecker is satisfied by engines that can report whether an
// IP falls inside the Cloudflare allow ranges. The input chain accepts
// Cloudflare edges on TCP 80/443 before the blocked drop, so a block of a
// covered IP does not stop its web traffic; findings carry that caveat.
type cloudflareCoverChecker interface {
CloudflareCovers(ip string) bool
}
// allowChecker is satisfied by engines that can report whether an IP is
// firewall-allowed (whitelisted). http_asn_crawl uses it at emit time to drop
// any candidate subnet that contains an observed source IP which is already
// allowed, so a surgical subnet tempban can never include a whitelisted host.
// Optional: blockers that do not implement it leave all candidate CIDRs intact.
type allowChecker interface {
IsAllowed(ip string) bool
}
// blockProtector is satisfied by engines that report whether no block may
// name an address. Challenge routing skips those addresses, since a challenge
// can end in a block. Optional: without it only the configured lists apply.
type blockProtector interface {
BlockProtected(ip string) bool
}
// permanentPromoter is satisfied by engines that can upgrade an existing
// temporary block to a permanent one. PermBlock escalation runs in the same
// scan cycle as the temp block that triggered it, so the ordinary block path
// (which skips an already-blocked IP and returns BlockOutcomeNoop) would never
// clear the kernel timeout and the "permanent" block would silently expire.
type permanentPromoter interface {
PromoteToPermanentBlock(ip, reason string) error
}
type subnetBlockStatus interface {
IsSubnetBlocked(cidr string) bool
}
// subnetManager is satisfied by firewall engines that expose their blocked
// subnet state and support targeted unblock calls. Used by
// PruneExemptAutoSubnets to enumerate and remove stale subnet blocks whose
// CIDR has become DoS-exempt.
type subnetManager interface {
BlockedSubnets() []firewall.SubnetEntry
UnblockSubnet(cidr string) error
}
// fwBlockerSlot wraps an IPBlocker so atomic.Pointer can store it. The
// extra struct layer is required because atomic.Pointer needs a
// concrete type and interfaces cannot be stored directly.
type fwBlockerSlot struct{ b IPBlocker }
var fwBlockerHolder atomic.Pointer[fwBlockerSlot]
var blockStateMu sync.Mutex
var autoBlockNow = time.Now
// SetIPBlocker installs the firewall engine for auto-blocking. Safe to
// call concurrently with AutoBlockIPs: each call publishes the new
// blocker atomically and any in-flight scan keeps the snapshot it
// already loaded.
func SetIPBlocker(b IPBlocker) {
fwBlockerHolder.Store(&fwBlockerSlot{b: b})
}
// getIPBlocker returns the current blocker via a single atomic load.
// Callers should capture the result into a local variable and reuse it
// for the duration of one operation so a concurrent SetIPBlocker
// cannot split a single scan across two different engines.
func getIPBlocker() IPBlocker {
slot := fwBlockerHolder.Load()
if slot == nil {
return nil
}
return slot.b
}
type blockedIP struct {
FindingID string `json:"finding_id,omitempty"`
IP string `json:"ip"`
Reason string `json:"reason"`
BlockedAt time.Time `json:"blocked_at"`
ExpiresAt time.Time `json:"expires_at"`
// LoginSpray marks a mail spray block; IPv4 netblock escalation skips it.
LoginSpray bool `json:"login_spray,omitempty"`
}
type pendingIP struct {
ActionID string `json:"action_id,omitempty"`
ActionTTL time.Duration `json:"action_ttl,omitempty"`
FindingID string `json:"finding_id,omitempty"`
Cause *alert.Cause `json:"cause,omitempty"`
IP string `json:"ip"`
Reason string `json:"reason"`
Check string `json:"check,omitempty"`
Severity alert.Severity `json:"severity,omitempty"`
// TTL is the longest block length the findings asked for. Only a mail
// spray block uses it; any other block gets block_expiry. LoginSpray
// marks a mail spray block, which draws on its own hourly budget.
TTL time.Duration `json:"ttl,omitempty"`
LoginSpray bool `json:"login_spray,omitempty"`
// QueuedAt is when the IP first entered the queue; it survives
// requeue cycles so age accumulates instead of resetting. Stamped on
// the first requeue for eligible entries with no timestamp.
QueuedAt time.Time `json:"queued_at,omitempty"`
queueRecord *autoBlockPendingRecord
queueCandidate *autoBlockCandidate
}
type blockState struct {
IPs []blockedIP `json:"ips"`
Pending []pendingIP `json:"pending,omitempty"` // IPs waiting for another block attempt
CleanupPending []string `json:"cleanup_pending,omitempty"`
BlocksThisHour int `json:"blocks_this_hour"`
// LoginSprayBlocksThisHour counts mail spray blocks against a budget of
// the same size, so a spray wave cannot crowd out other blocks.
LoginSprayBlocksThisHour int `json:"login_spray_blocks_this_hour,omitempty"`
HourKey string `json:"hour_key"`
// RateLimitWarnedHour is the HourKey for which the rate-limit warning
// was already emitted. The warning reflects a steady-state condition,
// not a per-IP event, so it fires once per hour window instead of on
// every scan tick -- the per-tick emission flooded the audit log with
// one identical finding every few seconds during a sustained attack.
RateLimitWarnedHour string `json:"rate_limit_warned_hour,omitempty"`
// PendingDropWarnedHour throttles queue-overflow findings independently
// from rate-limit warnings. Engine-down and block-error retries can fill
// the queue without reaching the hourly block limit.
PendingDropWarnedHour string `json:"pending_drop_warned_hour,omitempty"`
}
// blockableCheck reports whether a finding's check may drive a firewall block.
// The policy is carried by the check registry; see response_policy.go.
func blockableCheck(check string, blockCpanelLogins bool) bool {
switch ResponsePolicyFor(check).Block {
case BlockAlways:
return true
case BlockWithCpanelLogins:
return blockCpanelLogins
default:
return false
}
}
func blockableFinding(f alert.Finding, blockCpanelLogins bool) bool {
return blockableCheck(f.Check, blockCpanelLogins) &&
(!ResponsePolicyFor(f.Check).CriticalOnly || f.Severity == alert.Critical)
}
// AutoBlockIPs processes all findings, including repeats, for IP blocking.
func AutoBlockIPs(cfg *config.Config, findings []alert.Finding) []alert.Finding {
if !cfg.AutoResponse.Enabled || !cfg.AutoResponse.BlockIPs {
return nil
}
work := autoBlockQueues.acquire()
defer work.finish()
// Snapshot the wired firewall engine ONCE per call. A concurrent
// SetIPBlocker (SIGHUP re-wire, test cleanup) can swap the global
// mid-scan; reading the atomic pointer once and reusing the
// returned value keeps every block decision in this batch routed
// to the same engine. The previous unsynchronized read of the
// global also tripped the race detector.
blocker := getIPBlocker()
// One bulk dump of the kernel's blocked sets answers every membership
// question this cycle asks. The per-IP live query dumps the whole set, so
// the reconcile pass below cost one full dump per tracked IP.
var liveBlocked firewall.LiveBlockedSnapshot
var useLiveBlocked bool
if blocker != nil {
work.progress()
liveBlocked, useLiveBlocked = liveBlockedSnapshot(blocker)
}
// exemptLogged deduplicates per-cycle log lines for DoS-exempt CIDR skips
// so each suppressed subnet is logged once per AutoBlockIPs call, not once
// per finding or per IP in the netblock counting sweep.
exemptLogged := make(map[string]struct{})
var actions []alert.Finding
answeredSubnets := make(map[string]bool)
// Load block state
work.progress()
state := work.loadState(cfg.StatePath)
// Reservations retain the on-disk backlog until the cycle finishes.
// Saving the drained queue here would lose unprocessed retries on a crash.
sprayReservation := *state
// Prune IPs that the firewall engine no longer has blocked.
// The engine handles expiry natively via nftables timeouts -
// we just sync our state to match. Use the live nftables query
// when the engine supports it so the tracker stays in lock-step
// with the kernel; the in-memory cache (IsBlocked) can lag when
// the kernel expires entries before state.json is rewritten.
var stillBlocked []blockedIP
for _, b := range state.IPs {
work.progress()
if blocker != nil {
if !blockedLiveOrCached(blocker, liveBlocked, useLiveBlocked, b.IP) {
// Engine expired this block - clean up our state
fmt.Fprintf(os.Stderr, "[%s] AUTO-UNBLOCK: %s removed (engine expired)\n", time.Now().Format("2006-01-02 15:04:05"), b.IP)
continue
}
}
stillBlocked = append(stillBlocked, b)
}
state.IPs = stillBlocked
// Prune auto-response subnet blocks that now intersect the DoS-exempt set
// before making new subnet decisions this cycle.
if blocker != nil {
pruneExemptAutoSubnets(cfg, blocker, work.progress)
}
// Check rate limit
currentHour := autoBlockNow().Format("2006-01-02T15")
if state.HourKey != currentHour {
state.HourKey = currentHour
state.BlocksThisHour = 0
state.LoginSprayBlocksThisHour = 0
}
// Collect IPs to block from findings
ipsToBlock := make(map[string]pendingIP)
type evaluatedBlock struct{ findingID, ip string }
evaluated := make(map[evaluatedBlock]bool)
freshIPs := make(map[string]bool)
for _, f := range findings {
if f.AutoIPResponseEvaluated {
evaluated[evaluatedBlock{alert.FindingID(f), extractIPFromFinding(f)}] = true
} else if BlockableFinding(cfg, f) {
freshIPs[extractIPFromFinding(f)] = true
}
}
// Drain pending queue first (IPs from prior rate-limited or failed
// cycles). Stale entries are dropped by name so the audit trail shows
// exactly which attackers aged out instead of being blocked.
var deferredPending []pendingIP
for _, p := range state.Pending {
work.beginPending(p)
ip := normalizeBlockIP(p.IP)
if ip == "" {
fmt.Fprintf(os.Stderr, "auto-block: dropping invalid pending IP %q\n", p.IP)
continue
}
p.IP = ip
// A newly trusted proxy is no longer a retry candidate, even when
// the budget or engine would defer its block before the final guard.
if IsTrustedProxy(ip, cfg) {
work.completePending(p)
continue
}
// Retry only evidence still eligible under the current policy. Older
// queues lack check identity; a free-text reason cannot establish it.
if !BlockableFinding(cfg, alert.Finding{Check: p.Check, Severity: p.Severity, Cause: p.Cause}) {
work.completePending(p)
fmt.Fprintf(os.Stderr, "auto-block: dropping ineligible pending %s (check %q)\n", p.IP, p.Check)
continue
}
if !p.QueuedAt.IsZero() && autoBlockNow().Sub(p.QueuedAt) > maxPendingAge {
fmt.Fprintf(os.Stderr, "auto-block: dropping stale pending %s (queued %s)\n",
p.IP, p.QueuedAt.Format(time.RFC3339))
continue
}
// Redispatch of the same observation must not retry a failed scan
// block through the pending queue. A later cycle can still drain it.
if evaluated[evaluatedBlock{p.FindingID, p.IP}] && !freshIPs[p.IP] && !isAlreadyBlocked(state, p.IP) {
deferredPending = append(deferredPending, p)
continue
}
if !isAlreadyBlocked(state, p.IP) {
p.queueCandidate = work.candidate(p, ipsToBlock[p.IP].queueCandidate)
ipsToBlock[p.IP] = p
} else {
work.completePending(p)
}
}
state.Pending = nil
// Subnet fast-path: checks that represent a subnet directly.
// Independent of the per-IP rate limit, because a single subnet block
// replaces what would otherwise be hundreds of per-IP blocks.
for _, f := range findings {
work.progress()
if f.AutoIPResponseEvaluated || ResponsePolicyFor(f.Check).Subnet != admission.EntryMailSubnet {
continue
}
cidr := extractCIDRFromFinding(f)
if cidr == "" {
continue
}
if cidrIntersectsInfra(cfg, cidr) {
continue
}
if shouldSkipAutoSubnet(cfg, cidr, exemptLogged) {
continue
}
respondSpray(f, cidr)
if isSubnetAlreadyBlocked(blocker, cidr) {
continue
}
if !isAutoResponseActive(cfg) {
if !canDryRunBlockSubnet(blocker, cidr) {
continue
}
// Dry-run: same visibility contract as per-IP blocks - emit a
// Warning notice instead of skipping silently.
actions = append(actions, dryRunSubnetNotice(cidr, "", f))
continue
}
if blocker == nil {
fmt.Fprintf(os.Stderr, "auto-block: firewall engine not available, skipping subnet %s\n", cidr)
continue
}
sb, ok := blocker.(subnetBlocker)
if !ok {
fmt.Fprintf(os.Stderr, "auto-block: firewall engine does not support subnet blocking, skipping %s\n", cidr)
continue
}
reason := fmt.Sprintf("CSM auto-block (subnet): %s", truncate(f.Message, 100))
if !autoFirewallActionApplied(cidr, callBlockSubnet(sb, cidr, reason, parseExpiry(cfg.AutoResponse.BlockExpiry), alert.FindingID(f))) {
continue
}
answeredSubnets[cidr] = true
fmt.Fprintf(os.Stderr, "[%s] AUTO-BLOCK-SUBNET: %s blocked\n", time.Now().Format("2006-01-02 15:04:05"), cidr)
cause := alert.CauseOf(f)
actions = append(actions, alert.Finding{
Severity: alert.Critical,
Check: "auto_block",
Message: fmt.Sprintf("AUTO-BLOCK-SUBNET: %s blocked", cidr),
Details: fmt.Sprintf("Reason: %s", f.Message),
Timestamp: time.Now(),
Cause: &cause,
})
}
// A durable retry must replay its stored request unchanged. Without
// durable actions an action ID names no stored request.
durableActions := false
if durable, ok := blocker.(durableActionBlocker); ok {
durableActions = durable.DurableActionsEnabled()
}
// An address admission accepted a block for has one enforcer: legacy
// drops its queued retry and every other finding for that address this
// scan. An address admission refused as an edge or a proxied client has
// no admission block, so its other findings keep theirs.
takenIPs := make(map[string]bool)
for _, f := range findings {
work.progress()
if f.AutoIPResponseEvaluated || !BlockableFinding(cfg, f) {
continue
}
ip := extractIPFromFinding(f)
if ip == "" {
continue
}
// Never block infra IPs or a trusted proxy, which carries every
// visitor behind it.
if isInfraIP(ip, cfg.InfraIPs) || ip == "127.0.0.1" || IsTrustedProxy(ip, cfg) {
continue
}
// Admission is asked for the block the policy selects; the tracker,
// the kernel and the hourly budget below are the legacy path's, and
// a block admission takes leaves them alone. Mail spray keeps its
// legacy block, whose lifetime and budget admission does not model,
// so an executing admission is not handed it.
if !challengeSelected(cfg, f) {
if _, taken, accepted := respond(admission.KindBlockIP, f, ip, 0); taken {
if !accepted {
continue
}
takenIPs[ip] = true
if p, ok := ipsToBlock[ip]; ok {
work.candidateOutcome(p.queueCandidate, nil)
work.finishCandidate(p.queueCandidate)
delete(ipsToBlock, ip)
}
continue
}
}
if takenIPs[ip] {
continue
}
// Don't re-block already blocked IPs.
if isAlreadyBlocked(state, ip) || (blocker != nil && blockedLiveOrCached(blocker, liveBlocked, useLiveBlocked, ip)) {
continue
}
// Skip IPs that are already being challenged, but do not let a
// prior challenge suppress a later hard-block-only finding.
if cl := GetChallengeIPList(); cl != nil && cl.Contains(ip) && shouldSkipAutoBlockForChallenge(cfg, f) {
continue
}
// A drained pending entry keeps its QueuedAt when the same IP
// recurs in fresh findings; the check, severity and reason are refreshed.
// The longest asked length survives the refresh. The block stays a
// spray block only while every finding for the IP asks for one: any
// other finding gives it block_expiry, the scan budget and a place
// in netblock counts.
findingID := alert.FindingID(f)
if existing, ok := ipsToBlock[ip]; ok {
if existing.ActionID == "" || !durableActions {
existing.Reason = f.Message
existing.Check = f.Check
existing.Severity = f.Severity
existing.FindingID = findingID
existing.Cause = f.Cause
existing.TTL = max(existing.TTL, f.BlockTTL)
existing.LoginSpray = existing.LoginSpray && f.BlockTTL > 0
}
ipsToBlock[ip] = existing
} else {
p := pendingIP{IP: ip, Reason: f.Message, Check: f.Check, Severity: f.Severity, FindingID: findingID, Cause: f.Cause, TTL: f.BlockTTL, LoginSpray: f.BlockTTL > 0}
p.queueCandidate = work.candidate(p, nil)
ipsToBlock[ip] = p
}
}
// Block IPs and queue candidates that cannot be attempted or completed.
expiry := parseExpiry(cfg.AutoResponse.BlockExpiry)
maxPerHour := cfg.AutoResponse.MaxBlocksPerHour
if maxPerHour <= 0 {
maxPerHour = config.DefaultMaxBlocksPerHour
}
budgetUnavailable := false
if durable, ok := blocker.(interface {
DurableActionsEnabled() bool
FirewallScanBudget(string) (int, error)
}); ok && durable.DurableActionsEnabled() {
used, budgetErr := durable.FirewallScanBudget(currentHour)
if budgetErr != nil {
budgetUnavailable = true
actions = append(actions, alert.Finding{Severity: alert.Warning, Check: "auto_block", Message: "Firewall action accounting unavailable; scan blocks deferred", Details: budgetErr.Error(), DedupKey: "accounting", Timestamp: time.Now()})
} else {
state.BlocksThisHour = used
}
}
// http_asn_crawl: surgical subnet tempban for confirmed Critical findings.
// Each CIDR consumes one MaxBlocksPerHour slot. Independent of the per-IP
// list but shares its hourly budget. Skips infra intersections and
// already-blocked subnets; dry-run emits notices instead of blocking and
// consumes no budget.
if sb, ok := blocker.(subnetBlocker); ok {
tempban := parseExpiryWithDefault(cfg.AutoResponse.HTTPASNCrawlTempban, config.DefaultHTTPASNCrawlTempban)
for _, f := range findings {
work.progress()
if f.AutoIPResponseEvaluated || ResponsePolicyFor(f.Check).Subnet != admission.EntryASNCrawl || f.Severity != alert.Critical || len(f.CIDRs) == 0 {
continue
}
for _, cidr := range f.CIDRs {
work.progress()
if cidrIntersectsInfra(cfg, cidr) {
continue
}
if shouldSkipAutoSubnet(cfg, cidr, exemptLogged) {
continue
}
respond(admission.KindBlockSubnet, f, cidr, admission.EntryASNCrawl)
if isSubnetAlreadyBlocked(blocker, cidr) {
continue
}
if !isAutoResponseActive(cfg) {
if !canDryRunBlockSubnet(blocker, cidr) {
continue
}
actions = append(actions, dryRunSubnetNotice(cidr, " (asn-crawl)", f))
continue
}
if budgetUnavailable || state.BlocksThisHour >= maxPerHour {
continue
}
reason := fmt.Sprintf("CSM auto-block (asn-crawl): %s", truncate(f.Message, 100))
var subnetErr error
if durable, ok := blocker.(interface {
DurableActionsEnabled() bool
BlockSubnetRequest(firewall.ActionRequest, *firewall.ScanAdmission) error
}); ok && durable.DurableActionsEnabled() {
subnetErr = durable.BlockSubnetRequest(firewall.ActionRequest{ID: rand.Text(), Operation: "block_subnet", Target: cidr, Reason: reason, TTL: tempban, FindingID: alert.FindingID(f), Actor: "daemon", Source: BlockSourceScan, Automatic: true}, &firewall.ScanAdmission{Window: currentHour, Limit: maxPerHour})
} else {
subnetErr = callBlockSubnet(sb, cidr, reason, tempban, alert.FindingID(f))
}
if !autoFirewallActionApplied(cidr, subnetErr) {
continue
}
state.BlocksThisHour++
cause := alert.CauseOf(f)
actions = append(actions, alert.Finding{
Severity: alert.Critical,
Check: "auto_block",
Message: fmt.Sprintf("AUTO-BLOCK-SUBNET: %s blocked (asn-crawl)", cidr),
Details: fmt.Sprintf("Reason: %s", f.Message),
Timestamp: time.Now(),
Cause: &cause,
})
}
}
}
scanLimited, sprayLimited := false, false
sprayBudgetUnavailable := false
droppedPending := 0
engineUnavailableRequeued := 0
// requeue preserves an IP that could not be blocked this cycle (rate
// limit, engine unavailable, transient block error). QueuedAt is
// stamped on first entry so the drain's age check can retire it; the
// bound keeps a sustained flood from growing the queue without limit,
// and overflow drops are named so they remain identifiable in logs.
// Spray retries hold at most half the queue, so a spray backlog over its
// own budget cannot push other retries out.
sprayPending, sprayShareDropped := 0, 0
requeue := func(p pendingIP) bool {
if p.QueuedAt.IsZero() {
p.QueuedAt = autoBlockNow()
}
if len(state.Pending) < maxPendingBlocks && (!p.LoginSpray || sprayPending < maxPendingBlocks/2) {
state.Pending = append(state.Pending, work.requeueCandidate(p))
if p.LoginSpray {
sprayPending++
}
return true
}
work.rejectCandidate(p.queueCandidate)
full := "pending queue"
if len(state.Pending) < maxPendingBlocks {
full = "mail spray share of the pending queue"
sprayShareDropped++
}
fmt.Fprintf(os.Stderr, "auto-block: %s full, dropping %s\n", full, p.IP)
droppedPending++
return false
}
// Admit all non-spray retries before spray retries, including deferred
// redispatches. Otherwise map order can discard non-spray evidence even
// while spray occupies less than half the queue.
type queuedCandidate struct {
pending pendingIP
deferred bool
}
candidates := make([]queuedCandidate, 0, len(ipsToBlock)+len(deferredPending))
for _, spray := range []bool{false, true} {
for _, p := range deferredPending {
if p.LoginSpray == spray {
p.queueCandidate = work.candidate(p, nil)
candidates = append(candidates, queuedCandidate{pending: p, deferred: true})
}
}
for _, p := range ipsToBlock {
if p.LoginSpray == spray {
candidates = append(candidates, queuedCandidate{pending: p})
}
}
}
for _, candidate := range candidates {
cand := candidate.pending
ip := cand.IP
work.progress()
work.startCandidate(cand.queueCandidate)
if candidate.deferred {
requeue(cand)
continue
}
durableRetry := false
if durable, ok := blocker.(durableActionBlocker); ok {
durableRetry = cand.ActionID != "" && durable.DurableActionsEnabled()
}
// Spray blocks have their own budget of the same size. The durable
// store counts only the scan budget.
used := &state.BlocksThisHour
if cand.LoginSpray {
used = &state.LoginSprayBlocksThisHour
// Nothing in the store caps a spray block: its retries wait
// for the spray budget like a first attempt.
durableRetry = false
}
// Existing requests need recovery even when their admission used the
// last slot. The durable store still caps any identity not yet admitted.
accountingUnavailable := budgetUnavailable || (cand.LoginSpray && sprayBudgetUnavailable)
if accountingUnavailable || (*used >= maxPerHour && !durableRetry) {
requeue(cand)
if !accountingUnavailable {
sprayLimited = sprayLimited || cand.LoginSpray
scanLimited = scanLimited || !cand.LoginSpray
}
continue
}
// Block via firewall engine (nftables)
blockReason := fmt.Sprintf("CSM auto-block: %s", truncate(cand.Reason, 100))
if blocker == nil {
work.candidateOutcome(cand.queueCandidate, ErrNoIPBlocker)
observeBlockOutcome(firewall.BlockOutcomeNoop, ErrNoIPBlocker, BlockSourceScan)
if requeue(cand) {
engineUnavailableRequeued++
}
continue
}
if cand.ActionID == "" {
cand.ActionID = rand.Text()
}
// A block that mixes in any other finding is a normal block. The
// spray verdict's marker still gives the address a spray block if it
// fails a login again after this one ends.
requestTTL := expiry
if cand.LoginSpray && cand.TTL > 0 {
requestTTL = cand.TTL
}
if durable, ok := blocker.(durableActionBlocker); ok && durable.DurableActionsEnabled() {
if cand.ActionTTL == 0 {
cand.ActionTTL = requestTTL
}
requestTTL = cand.ActionTTL
}
if cand.LoginSpray {
// Charge before the kernel call: an uncertain result or a crash
// after mutation must not leave the next cycle with fresh credit.
sprayReservation.HourKey = state.HourKey
sprayReservation.BlocksThisHour = state.BlocksThisHour
sprayReservation.LoginSprayBlocksThisHour = *used + 1
if err := persistAutoBlockState(cfg.StatePath, &sprayReservation); err != nil {
work.observe(err)
work.candidateOutcome(cand.queueCandidate, err)
sprayBudgetUnavailable = true
requeue(cand)
actions = append(actions, alert.Finding{Severity: alert.Warning, Check: "auto_block", Message: "Block accounting unavailable; mail spray blocks deferred", Details: err.Error(), DedupKey: "spray_accounting", Timestamp: time.Now()})
continue
}
*used++
}
res, err := applyBlockLocked(cfg, blocker, state, ApplyBlockRequest{
ActionID: cand.ActionID,
IP: ip,
EngineReason: blockReason,
Reason: cand.Reason,
TTL: requestTTL,
Source: BlockSourceScan,
FindingID: cand.FindingID,
LoginSpray: cand.LoginSpray,
}, work.progress, func(err error) { work.candidateOutcome(cand.queueCandidate, err) })
// Release only proven non-blocks. Unknown effects retain their slot
// even when later recovery returns a successful no-op for that ID.
if cand.LoginSpray && ((err == nil && res.Outcome != firewall.BlockOutcomeLive) ||
errors.Is(err, firewall.ErrActionFailed) || errors.Is(err, firewall.ErrIPProtected) || errors.Is(err, firewall.ErrActionDryRun)) {
*used--
}
verifiedAuditPending := res.Outcome == firewall.BlockOutcomeLive && errors.Is(err, firewall.ErrActionAuditPending)
if verifiedAuditPending {
fmt.Fprintf(os.Stderr, "auto-block: %s verified with audit delivery pending: %v\n", ip, err)
}
if err != nil && !verifiedAuditPending {
if errors.Is(err, firewall.ErrActionFailed) {
// A proven rejection is terminal for this request ID. A later
// attempt gets independent admission under the current policy.
cand.ActionID = ""
cand.ActionTTL = 0
}
// Protected IPs (the server's own interface or infra_ips) are
// intentionally never blocked -- an expected no-op, not a failure.
// The triggering finding still stands, so suspicious activity from a
// protected address is still surfaced. Any other error is treated
// as transient and the IP is requeued when capacity permits; the
// age cap retires it if the failure persists.
if !errors.Is(err, firewall.ErrIPProtected) {
if requeue(cand) {
fmt.Fprintf(os.Stderr, "auto-block: error blocking %s: %v (requeued)\n", ip, err)
} else {
fmt.Fprintf(os.Stderr, "auto-block: error blocking %s: %v (retry dropped)\n", ip, err)
}
} else {
work.finishCandidate(cand.queueCandidate)
}
continue
}
// Each notice names the finding behind the block, so the known-
// customer alert and the summary can say which check blocked.
for i := range res.Findings {
res.Findings[i].Cause = &alert.Cause{FindingID: cand.FindingID, Check: cand.Check}
}
actions = append(actions, res.Findings...)
if res.Outcome != firewall.BlockOutcomeLive {
work.finishCandidate(cand.queueCandidate)
continue
}
if blocker.IsBlocked(ip) {
fmt.Fprintf(os.Stderr, "[%s] AUTO-BLOCK: %s blocked (expires in %s)\n", time.Now().Format("2006-01-02 15:04:05"), ip, requestTTL)
}
if !cand.LoginSpray {
*used++
}
work.finishCandidate(cand.queueCandidate)
}
if engineUnavailableRequeued > 0 {
fmt.Fprintf(os.Stderr, "auto-block: firewall engine not available, requeued %d IPs\n", engineUnavailableRequeued)
}
// One warning an hour covers both budgets; a budget that fills after it
// is named in the next hour's warning.
rateLimited := scanLimited || sprayLimited
warnRateLimit := rateLimited && state.RateLimitWarnedHour != currentHour
warnPendingDrop := droppedPending > 0 && state.PendingDropWarnedHour != currentHour
if warnRateLimit || warnPendingDrop {
dropped := "queue full"
switch {
case sprayShareDropped == droppedPending:
dropped = "mail spray share of the queue full"
case sprayShareDropped > 0:
dropped = "queue or mail spray share full"
}
var msg string
kind := "pending_dropped"
if rateLimited {
kind = "rate_limit"
limit := fmt.Sprintf("rate limit reached (%d/hour)", maxPerHour)
switch {
case scanLimited && sprayLimited:
limit = fmt.Sprintf("rate limit and mail login spray budget reached (%d/hour each)", maxPerHour)
case sprayLimited:
limit = fmt.Sprintf("mail login spray budget reached (%d/hour)", maxPerHour)
}
msg = fmt.Sprintf("Auto-block %s, %d IPs queued for next cycle", limit, len(state.Pending))
if droppedPending > 0 {
// Drops are IPs that will never be blocked. They must not
// hide behind a rate limit that was already reported.
kind = "rate_limit_dropped"
msg += fmt.Sprintf(", %d dropped (%s)", droppedPending, dropped)
}
} else {
full := "pending " + dropped
if sprayShareDropped > 0 {
full = dropped
}
msg = fmt.Sprintf("Auto-block %s, %d IPs queued for retry, %d dropped", full, len(state.Pending), droppedPending)
}
// The identity is the kind: the counts change every hour it lasts.
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "auto_block",
Message: msg,
DedupKey: kind,
Timestamp: time.Now(),
})
if rateLimited {
state.RateLimitWarnedHour = currentHour
}
if droppedPending > 0 {
state.PendingDropWarnedHour = currentHour
}
}
// Subnet auto-blocking: detect per-family subnet patterns. Runs in
// dry-run too so escalations that would fire (from live blocks recorded
// before dry-run was enabled) surface as notices.
var netblockRows []blockedIP
var netblockRowsErr error
if cfg.AutoResponse.NetBlock && blocker != nil {
work.progress()
netblockRows, netblockRowsErr = netblockTrackedRows(blocker, state.IPs, liveBlocked, useLiveBlocked)
if netblockRowsErr != nil {
work.observe(netblockRowsErr)
fmt.Fprintf(os.Stderr, "autoblock: block provenance unavailable, deferring netblock escalation: %v\n", netblockRowsErr)
}
}
if cfg.AutoResponse.NetBlock && blocker != nil && netblockRowsErr == nil {
// Load fills the default and validation rejects anything lower, so
// this only catches a Config assembled in code.
threshold := cfg.AutoResponse.NetBlockThreshold
if threshold < config.MinBlockEscalationCount {
threshold = config.DefaultNetBlockThreshold
}
subnetExpiry := parseExpiry(cfg.AutoResponse.BlockExpiry)
// Count addresses blocked inside the window per subnet (IPv4 /24,
// IPv6 /64), not only those blocked right now: a subnet that rotates
// addresses keeps one block live at a time and would never escalate.
now := autoBlockNow()
window := netblockWindow(cfg)
work.progress()
// An unreadable history file is kept for inspection and never
// overwritten. Escalation falls back to the addresses blocked right
// now, which is what it counted before history existed.
history, historyErr := loadNetblockHistory(cfg.StatePath)
if historyErr != nil {
work.observe(historyErr)
fmt.Fprintf(os.Stderr, "autoblock: netblock history unavailable, counting current blocks only: %v\n", historyErr)
history = &netblockHistory{IPs: map[string]time.Time{}, Subnets: map[string]time.Time{}}
}
current := currentlyBlocked(netblockRows, liveBlocked, useLiveBlocked, history, blocker)
historyChanged := recordNetblockHistory(history, netblockRows, current, blocker, now, window)
// Direct mail-spray blocks answer the same prior offenders as a
// threshold-based block, even when the IP blocks have already ended.
for cidr := range answeredSubnets {
history.Subnets[cidr] = now
historyChanged = true
}
subnetCounts := netblockCounts(cfg, history, current, blocker, now, window)
// The cause is the newest block that counted; IPv4 spray blocks and
// listed C2 servers never do.
subnetCauses := make(map[string]blockedIP)
subnetBlocked := make(map[string]bool)
for _, b := range netblockRows {
cidr := subnetEscalationCIDR(b.IP)
prior := subnetCauses[cidr]
if cidr != "" && b.FindingID != "" && countsTowardNetblock(b) && !c2Listed(cfg, b.IP) && (prior.FindingID == "" || b.BlockedAt.After(prior.BlockedAt) || (b.BlockedAt.Equal(prior.BlockedAt) && b.FindingID > prior.FindingID)) {
subnetCauses[cidr] = b
}
}
for cidr, count := range subnetCounts {
work.progress()
if count >= threshold && !subnetBlocked[cidr] {
if sb, ok := blocker.(subnetBlocker); ok {
if isSubnetAlreadyBlocked(blocker, cidr) {
continue
}
if cidrIntersectsInfra(cfg, cidr) {
continue
}
if shouldSkipAutoSubnet(cfg, cidr, exemptLogged) {
continue
}
respondNetblock()
if !isAutoResponseActive(cfg) {
if !canDryRunBlockSubnet(blocker, cidr) {
continue
}
subnetBlocked[cidr] = true
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "auto_block",
Message: fmt.Sprintf("AUTO-NETBLOCK [dry-run]: %s would be blocked (%d IPs from same subnet)", cidr, count),
Timestamp: time.Now(),
Cause: netblockCause(subnetCauses[cidr]),
})
continue
}
reason := fmt.Sprintf("Auto-netblock: %d IPs from %s within %s", count, cidr, window)
if autoFirewallActionApplied(cidr, callBlockSubnet(sb, cidr, reason, subnetExpiry, subnetCauses[cidr].FindingID)) {
subnetBlocked[cidr] = true
history.Subnets[cidr] = now
historyChanged = true
fmt.Fprintf(os.Stderr, "[%s] AUTO-NETBLOCK: %s blocked (%d IPs from same subnet)\n", time.Now().Format("2006-01-02 15:04:05"), cidr, count)
actions = append(actions, alert.Finding{
Severity: alert.Critical,
Check: "auto_block",
Message: fmt.Sprintf("AUTO-NETBLOCK: %s blocked (%d IPs from same subnet)", cidr, count),
Timestamp: time.Now(),
Cause: netblockCause(subnetCauses[cidr]),
})
}
}
}
}
if historyChanged && historyErr == nil {
work.progress()
if err := saveNetblockHistory(cfg.StatePath, history); err != nil {
work.observe(err)
fmt.Fprintf(os.Stderr, "autoblock: %v\n", err)
}
}
}
// Save state (expired IPs were already pruned at the top of this function)
work.progress()
work.saveState(cfg.StatePath, state)
work.complete()
return actions
}
// isBlockedLiveOrCached returns the live nftables status when the
// blocker supports it, otherwise falls back to the cached IsBlocked
// view. The reconcile loop relies on this to prune blocked_ips.json
// entries the kernel has already expired even when state.json has not
// caught up yet. Live lookup errors keep the cached answer so transient
// netlink failures do not erase the local tracker.
func isBlockedLiveOrCached(b IPBlocker, ip string) bool {
if lb, ok := b.(liveBlocker); ok {
blocked, err := lb.IsBlockedLive(ip)
if err == nil {
return blocked
}
}
return b.IsBlocked(ip)
}
// liveBlockedSnapshot takes one bulk membership snapshot for the cycle. A
// blocker that cannot produce one leaves callers on the legacy per-IP path.
// A failed bulk dump returns an uncovered snapshot so valid IPs use cached
// status without issuing the same failing dump once per tracked address.
func liveBlockedSnapshot(b IPBlocker) (firewall.LiveBlockedSnapshot, bool) {
lister, ok := b.(liveBlockedLister)
if !ok {
return firewall.LiveBlockedSnapshot{}, false
}
snap, err := lister.LiveBlockedSet()
if err != nil {
// The snapshot still describes whichever families answered, so keep
// it: discarding it would drop a healthy family to cached status
// because the other one failed.
fmt.Fprintf(os.Stderr, "auto-block: live blocked-set dump incomplete, using cached status for uncovered families: %v\n", err)
}
return snap, true
}
// blockedLiveOrCached answers from the cycle's bulk snapshot when it covers
// the IP's address family. An uncovered family uses cached status; retrying a
// live query would repeat the same set dump and could turn an unknown result
// into a false absence. Blockers without snapshot support retain the legacy
// per-IP live lookup.
func blockedLiveOrCached(b IPBlocker, snap firewall.LiveBlockedSnapshot, useSnap bool, ip string) bool {
if useSnap {
if blocked, known := snap.Contains(ip); known {
return blocked
}
return b.IsBlocked(ip)
}
return isBlockedLiveOrCached(b, ip)
}
// callBlockIP dispatches to the outcome-reporting interface when the
// underlying blocker implements it, otherwise falls back to the legacy
// IPBlocker interface and assumes the call landed live (the behaviour
// every IPBlocker had before BlockIPOutcome existed). This keeps tests
// and any third-party implementations of IPBlocker working unchanged.
func callBlockIP(b IPBlocker, ip, reason string, timeout time.Duration, findingID string) (firewall.BlockOutcome, error) {
if ob, ok := b.(interface {
BlockIPOutcomeWithFindingID(string, string, time.Duration, string) (firewall.BlockOutcome, error)
}); ok {
return ob.BlockIPOutcomeWithFindingID(ip, reason, timeout, findingID)
}
if ob, ok := b.(outcomeBlocker); ok {
return ob.BlockIPOutcome(ip, reason, timeout)
}
if err := b.BlockIP(ip, reason, timeout); err != nil {
return firewall.BlockOutcomeNoop, err
}
return firewall.BlockOutcomeLive, nil
}
// shouldSkipAutoBlockForChallenge reports whether an IP carrying this finding
// should be left for the challenge gate instead of hard-blocked. It is the
// exact inverse of responseActionForFinding resolving to a block, so the two
// auto-response paths share one decision.
func shouldSkipAutoBlockForChallenge(cfg *config.Config, f alert.Finding) bool {
return responseActionForFinding(cfg, f) == responseChallenge
}
// promoteToPermanentBlock upgrades an existing temp block to permanent. The
// real engine implements permanentPromoter and clears the kernel timeout in
// place. Legacy blockers that only implement BlockIP have not marked the IP
// blocked in a way that trips skipExisting, so a fresh zero-timeout block on
// them lands live; that fallback preserves pre-existing behaviour for tests
// and third-party implementations.
func promoteToPermanentBlock(b IPBlocker, ip, reason, findingID string) bool {
respondDerived(ApplyBlockRequest{Entry: admission.EntryPermblock}, 0)
if pp, ok := b.(interface {
PromoteToPermanentBlockWithFindingID(string, string, string) error
}); ok {
return autoFirewallActionApplied(ip, pp.PromoteToPermanentBlockWithFindingID(ip, reason, findingID))
}
if pp, ok := b.(permanentPromoter); ok {
return autoFirewallActionApplied(ip, pp.PromoteToPermanentBlock(ip, reason))
}
outcome, err := callBlockIP(b, ip, reason, 0, findingID)
return outcome == firewall.BlockOutcomeLive && autoFirewallActionApplied(ip, err)
}
func autoFirewallActionApplied(target string, err error) bool {
if err == nil {
return true
}
// A verified mutation still needs its normal response evidence when the
// separate audit delivery is pending. Keep the degradation visible.
fmt.Fprintf(os.Stderr, "auto-block: firewall action for %s: %v\n", target, err)
return errors.Is(err, firewall.ErrActionAuditPending)
}
func isSubnetAlreadyBlocked(b IPBlocker, cidr string) bool {
sb, ok := b.(subnetBlockStatus)
return ok && sb.IsSubnetBlocked(cidr)
}
// ExtractIPFromFinding extracts an IP address from a finding.
func ExtractIPFromFinding(f alert.Finding) string {
return extractIPFromFinding(f)
}
// ManualBlockIP returns the address an operator may block from a finding: the
// one auto-block would act on, for any check that reports an attacker,
// including the login-failure checks that block only when configured. It is
// empty for every other check, whose messages can quote a victim or customer
// address from a log line.
func ManualBlockIP(f alert.Finding) string {
if !blockableCheck(f.Check, true) {
return ""
}
return extractIPFromFinding(f)
}
func extractIPFromFinding(f alert.Finding) string {
if strings.TrimSpace(f.SourceIP) != "" {
return normalizeBlockIP(f.SourceIP)
}
msg := f.Message
// Fallback for detectors that have not yet adopted the structured SourceIP
// field. Only findings whose Check is auto-block-eligible reach this path,
// and those detectors format their own messages with a CSM-parsed IP at the
// tail. Use LastIndex so the rightmost (CSM-appended) IP wins over any
// log-injected content earlier in the message.
for _, sep := range []string{" from ", ": "} {
if idx := strings.LastIndex(msg, sep); idx >= 0 {
rest := msg[idx+len(sep):]
fields := strings.Fields(rest)
if len(fields) > 0 {
if candidate, ok := netutil.ParseIPToken(fields[0]); ok {
if ip := normalizeBlockIP(candidate); ip != "" {
return ip
}
}
}
}
}
return ""
}
func normalizeBlockIP(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
if host, _, err := net.SplitHostPort(raw); err == nil {
raw = host
}
raw = strings.Trim(raw, "[]")
ip := net.ParseIP(raw)
if ip == nil || ip.IsLoopback() || ip.IsUnspecified() {
return ""
}
return ip.String()
}
func isAlreadyBlocked(state *blockState, ip string) bool {
for _, b := range state.IPs {
if b.IP == ip {
return true
}
}
return false
}
func parseExpiry(s string) time.Duration {
return parseExpiryWithDefault(s, config.DefaultBlockExpiry)
}
func parseExpiryWithDefault(s, fallback string) time.Duration {
d, err := time.ParseDuration(s)
if s != "" && err == nil && d > 0 {
return d
}
d, _ = time.ParseDuration(fallback)
return d
}
func loadBlockState(statePath string) *blockState {
state, err := readBlockState(statePath)
if err != nil {
fmt.Fprintf(os.Stderr, "autoblock: %v; ignoring queued blocks\n", err)
return &blockState{}
}
return state
}
func readBlockState(statePath string) (*blockState, error) {
path := filepath.Join(statePath, blockStateFile)
data, err := osFS.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return &blockState{}, nil
}
return nil, fmt.Errorf("reading %s: %w", path, err)
}
state := &blockState{}
if err := json.Unmarshal(data, state); err != nil {
return nil, fmt.Errorf("reading %s: %w", path, err)
}
return state, nil
}
func saveBlockState(statePath string, s *blockState) {
if err := writeBlockState(statePath, s); err != nil {
logBlockStateFailure(statePath, err)
}
}
func writeBlockState(statePath string, s *blockState) error {
return atomicio.AtomicWriteJSON(filepath.Join(statePath, blockStateFile), 0o600, s)
}
func logBlockStateFailure(statePath string, err error) {
fmt.Fprintf(os.Stderr, "autoblock: persist %s failed: %v\n", filepath.Join(statePath, blockStateFile), err)
}
// subnetEscalationCIDR returns the canonical CIDR used by the
// auto-netblock escalation path for the given IP. IPv4 collapses to
// /24 (the historical block size); IPv6 collapses to /64 because most
// providers hand out /64 prefixes to end users -- /128 would let
// attackers rotate addresses inside the same /64 and never escalate,
// while a wider prefix would risk taking down legitimate neighbours.
// Returns "" for unparseable input.
func subnetEscalationCIDR(ip string) string {
parsed := net.ParseIP(ip)
if parsed == nil {
return ""
}
if ip4 := parsed.To4(); ip4 != nil {
return fmt.Sprintf("%d.%d.%d.0/24", ip4[0], ip4[1], ip4[2])
}
ip16 := parsed.To16()
if ip16 == nil {
return ""
}
mask := net.CIDRMask(64, 128)
network := ip16.Mask(mask)
return (&net.IPNet{IP: network, Mask: mask}).String()
}
// --- Permanent block escalation (LF_PERMBLOCK) ---
type permBlockTracker struct {
IPs map[string][]time.Time `json:"ips"` // IP -> list of block timestamps
}
// checkPermBlockEscalation records a new block and returns true if the IP
// has been temp-blocked count times within interval.
func checkPermBlockEscalation(statePath, ip string, count int, interval time.Duration) bool {
tracker := loadPermBlockTracker(statePath)
now := time.Now()
cutoff := now.Add(-interval)
// Add current block timestamp
tracker.IPs[ip] = append(tracker.IPs[ip], now)
// Clean old entries for this IP
var recent []time.Time
for _, t := range tracker.IPs[ip] {
if t.After(cutoff) {
recent = append(recent, t)
}
}
tracker.IPs[ip] = recent
// Clean old IPs entirely (haven't been seen in 2x the interval)
for k, times := range tracker.IPs {
if len(times) == 0 {
delete(tracker.IPs, k)
continue
}
latest := times[len(times)-1]
if now.Sub(latest) > interval*2 {
delete(tracker.IPs, k)
}
}
savePermBlockTracker(statePath, tracker)
return len(recent) >= count
}
func loadPermBlockTracker(statePath string) *permBlockTracker {
tracker := &permBlockTracker{IPs: make(map[string][]time.Time)}
path := filepath.Join(statePath, "permblock_tracker.json")
data, err := osFS.ReadFile(path)
if err == nil {
if uerr := json.Unmarshal(data, tracker); uerr != nil {
fmt.Fprintf(os.Stderr, "autoblock: %s is corrupt, ignoring escalation history: %v\n", path, uerr)
}
if tracker.IPs == nil {
tracker.IPs = make(map[string][]time.Time)
}
}
return tracker
}
func savePermBlockTracker(statePath string, tracker *permBlockTracker) {
path := filepath.Join(statePath, "permblock_tracker.json")
if err := atomicio.AtomicWriteJSON(path, 0o600, tracker); err != nil {
fmt.Fprintf(os.Stderr, "autoblock: persist %s failed: %v\n", path, err)
}
}
// AutoBlockFlushResult reports which phases of a coordinated firewall flush
// completed and whether its best-effort persisted-state snapshot was readable.
type AutoBlockFlushResult struct {
Flushed bool
BlockedCount int
SnapshotErr error
}
// FlushAutoBlockState snapshots the engine's pre-flush IPs, runs an operator
// firewall flush, and clears the auto-block bookkeeping in one critical
// section. Without this the flush was self-reverting: surviving ThreatDB temp
// rows re-flagged every flushed IP through ip_reputation on the next scan and
// re-blocked it, and stale tracker entries suppressed re-block accounting.
// Tracker entries the engine never held are cleaned up too. Pending entries
// are kept - they are queued candidates, not blocks.
//
// The firewall mutation and cleanup stay serialized with AutoBlockIPs so a
// concurrent scan either finishes before the snapshot or starts after cleanup
// with fresh evidence. Result.Flushed is true with a non-nil error when the
// firewall was flushed but bookkeeping cleanup was only partial. SnapshotErr
// is advisory because the tracker-side union still covers tracked auto-blocks.
func FlushAutoBlockState(statePath string, flush func() error) (AutoBlockFlushResult, error) {
work := autoBlockQueues.acquire()
defer work.finish()
var result AutoBlockFlushResult
work.progress()
engineState, snapshotErr := firewall.LoadState(statePath)
result.SnapshotErr = snapshotErr
var ips []string
if snapshotErr == nil {
ips = make([]string, 0, len(engineState.Blocked))
for _, b := range engineState.Blocked {
if !b.Unverified {
ips = append(ips, b.IP)
}
}
result.BlockedCount = len(ips)
}
work.progress()
if err := flush(); err != nil {
work.observe(err)
work.complete()
return result, fmt.Errorf("flushing blocked IPs: %w", err)
}
result.Flushed = true
work.beginCleanup(statePath, ips, snapshotErr)
var cleanupErr error
work.progress()
state, err := work.readState(statePath)
seenCapacity := len(ips)
if state != nil {
seenCapacity += len(state.IPs) + len(state.CleanupPending)
}
seen := make(map[string]bool, seenCapacity)
cleanupIPs := make([]string, 0, seenCapacity)
addCleanupIP := func(ip string) {
if !seen[ip] {
seen[ip] = true
cleanupIPs = append(cleanupIPs, ip)
work.admitCleanup(ip)
}
}
for _, ip := range ips {
addCleanupIP(ip)
}
if err != nil {
work.observe(err)
cleanupErr = errors.Join(cleanupErr, fmt.Errorf("reading auto-block state: %w", err))
} else {
for _, b := range state.IPs {
addCleanupIP(b.IP)
}
for _, ip := range state.CleanupPending {
addCleanupIP(ip)
}
}
sdb := store.Global()
tdb := GetThreatDB()
failed := make([]string, 0)
for _, ip := range cleanupIPs {
work.startCleanup(ip)
cleanupFailed := false
if sdb != nil {
if err := RemoveStoredTemporaryBlock(sdb, ip); err != nil {
cleanupFailed = true
work.cleanupOutcome(ip, true)
work.observe(err)
cleanupErr = errors.Join(cleanupErr, fmt.Errorf("removing auto-block store row for %s: %w", ip, err))
failed = append(failed, ip)
}
}
if tdb != nil {
tdb.RemoveTemporary(ip)
}
work.cleanupOutcome(ip, cleanupFailed)
}
work.progress()
if err := saveNetblockHistory(statePath, &netblockHistory{}); err != nil {
work.observe(err)
cleanupErr = errors.Join(cleanupErr, fmt.Errorf("clearing netblock history: %w", err))
}
if state != nil {
state.IPs = nil
// A failed bbolt cleanup must survive in the tracker after the
// firewall state is empty, or a retry has no way to identify the
// stale row that can recreate the block after restart.
state.CleanupPending = failed
work.progress()
if err := work.writeState(statePath, state); err != nil {
work.observe(err)
logBlockStateFailure(statePath, err)
cleanupErr = errors.Join(cleanupErr, fmt.Errorf("clearing auto-block state: %w", err))
}
}
work.complete()
return result, cleanupErr
}
// netblockCause names the newest counted block behind a netblock. Tracker
// rows keep the finding, not its check.
func netblockCause(b blockedIP) *alert.Cause {
if b.FindingID == "" {
return nil
}
return &alert.Cause{FindingID: b.FindingID}
}
// dryRunSubnetNotice mirrors the per-IP dry-run notice for the subnet block
// paths so operators evaluating dry-run see subnet decisions instead of
// silence. The message deliberately differs from the live
// "AUTO-BLOCK-SUBNET:" token so alert-filter suppression never treats a
// notice as a real block.
func dryRunSubnetNotice(cidr, kind string, source alert.Finding) alert.Finding {
cause := alert.CauseOf(source)
return alert.Finding{
Severity: alert.Warning,
Check: "auto_block",
Message: fmt.Sprintf("AUTO-BLOCK-SUBNET [dry-run]: %s would be blocked%s", cidr, kind),
Details: fmt.Sprintf("Reason: %s", source.Message),
Timestamp: time.Now(),
Cause: &cause,
}
}
// canDryRunBlockSubnet applies the firewall engine's read-only preflight when
// available. A missing or incapable engine cannot make the claimed live block,
// so it must not produce a "would be blocked" notice.
func canDryRunBlockSubnet(blocker IPBlocker, cidr string) bool {
if blocker == nil {
fmt.Fprintf(os.Stderr, "auto-block: firewall engine not available, skipping dry-run subnet %s\n", cidr)
return false
}
if _, ok := blocker.(subnetBlocker); !ok {
fmt.Fprintf(os.Stderr, "auto-block: firewall engine does not support subnet blocking, skipping dry-run subnet %s\n", cidr)
return false
}
if validator, ok := blocker.(subnetBlockValidator); ok {
if err := validator.ValidateSubnetBlock(cidr); err != nil {
fmt.Fprintf(os.Stderr, "auto-block: dry-run subnet %s rejected: %v\n", cidr, err)
return false
}
}
return true
}
// isAutoResponseActive reports whether real blocking should happen now:
// auto-response enabled, IP blocking on, and not in dry-run.
// DryRun defaults to true (safe) when nil — operators must explicitly set
// dry_run: false to enable live nftables blocking.
func isAutoResponseActive(cfg *config.Config) bool {
return cfg.AutoResponse.Enabled && cfg.AutoResponse.BlockIPs && !cfg.AutoResponseDryRunEnabled()
}
// cidrIntersectsInfra reports whether the CIDR contains an operator infra
// IP/range (or loopback), so the subnet tempban never blackholes protected
// addresses. An unparseable CIDR fails safe (treated as intersecting and
// skipped). The firewall engine's dynamic per-IP allowlist is not
// enumerable across a subnet, so infra_ips is the operator's mechanism to
// exempt a specific address from subnet tempban.
func cidrIntersectsInfra(cfg *config.Config, cidr string) bool {
_, ipnet, err := net.ParseCIDR(cidr)
if err != nil {
return true
}
if ipnet.IP.IsLoopback() {
return true
}
candidates := append([]string{"127.0.0.1", "::1"}, cfg.InfraIPs...)
for _, raw := range candidates {
raw = strings.TrimSpace(raw)
if raw == "" {
continue
}
if p := net.ParseIP(raw); p != nil && ipnet.Contains(p) {
return true
}
if _, infraNet, err := net.ParseCIDR(raw); err == nil &&
(ipnet.Contains(infraNet.IP) || infraNet.Contains(ipnet.IP)) {
return true
}
}
return false
}
// dosExemptNets returns the effective DoS-exempt networks (operator ranges
// union-ed with the current mail-provider overlay) split into IPv4 and IPv6
// slices. Delegates to firewall.EffectiveDOSExemptNets so the same logic
// governs both nftables sets and auto-block guards.
func dosExemptNets(cfg *config.Config) (v4, v6 []*net.IPNet) {
var fc *firewall.FirewallConfig
if cfg != nil {
fc = cfg.Firewall
}
return firewall.EffectiveDOSExemptNets(fc, mailranges.ProviderNets())
}
// cidrIntersectsDOSExempt reports whether the CIDR overlaps any DoS-exempt
// network, so the subnet tempban never blocks exempt sources (e.g., operator-
// designated ranges or known mail-provider egress). An unparseable CIDR fails
// safe (treated as intersecting, block skipped).
func cidrIntersectsDOSExempt(cfg *config.Config, cidr string) bool {
_, ipnet, err := net.ParseCIDR(cidr)
if err != nil {
return true // fail-safe: skip block when CIDR is unreadable
}
v4, v6 := dosExemptNets(cfg)
for _, exempt := range append(v4, v6...) { //nolint:gocritic // intentional inline join
if ipnet.Contains(exempt.IP) || exempt.Contains(ipnet.IP) {
return true
}
}
return false
}
// shouldSkipAutoSubnet reports whether the auto-block path must suppress a
// BlockSubnet call for cidr because it intersects a DoS-exempt operator range.
// Logs once per CIDR per cycle (via the logged dedupe map) at stderr info level
// with reason dos_exempt_range. Returns true when the block must be suppressed.
// Manual operator subnet denies bypass this function entirely.
func shouldSkipAutoSubnet(cfg *config.Config, cidr string, logged map[string]struct{}) bool {
if !cidrIntersectsDOSExempt(cfg, cidr) {
return false
}
if _, already := logged[cidr]; !already {
logged[cidr] = struct{}{}
fmt.Fprintf(os.Stderr, "auto-block: skipping subnet %s (dos_exempt_range)\n", cidr)
}
return true
}
// PruneExemptAutoSubnets removes auto-response subnet blocks whose CIDR now
// intersects the DoS-exempt set (operator ranges or mail-provider overlay).
// Only entries with Source == firewall.SourceAutoResponse are touched; manual,
// CLI, web-UI, challenge, whitelist, dyndns, system, and unknown-source blocks
// are left untouched. If b does not implement subnetManager, returns 0.
// UnblockSubnet errors are logged and the entry is not counted as pruned.
// Nothing is pruned unless auto response is live: dry run promises a
// read-only firewall and pruning is a kernel mutation.
func PruneExemptAutoSubnets(cfg *config.Config, b IPBlocker) int {
return pruneExemptAutoSubnets(cfg, b, func() {})
}
func pruneExemptAutoSubnets(cfg *config.Config, b IPBlocker, progress func()) int {
if !isAutoResponseActive(cfg) {
return 0
}
sm, ok := b.(subnetManager)
if !ok {
return 0
}
pruned := 0
progress()
for _, entry := range sm.BlockedSubnets() {
progress()
if entry.Source != firewall.SourceAutoResponse {
continue
}
if !cidrIntersectsDOSExempt(cfg, entry.CIDR) {
continue
}
if err := sm.UnblockSubnet(entry.CIDR); err != nil {
fmt.Fprintf(os.Stderr, "auto-block: prune exempt subnet %s: %v\n", entry.CIDR, err)
continue
}
pruned++
}
return pruned
}
// extractCIDRFromFinding returns the CIDR appearing in the message after
// the canonical " from " separator. Returns "" if the value does not parse
// as a CIDR.
func extractCIDRFromFinding(f alert.Finding) string {
if cidr := alert.SubnetSprayCIDR(f); cidr != "" {
if _, ipnet, err := net.ParseCIDR(cidr); err == nil {
return ipnet.String()
}
return ""
}
msg := f.Message
idx := strings.LastIndex(msg, " from ")
if idx < 0 {
return ""
}
rest := msg[idx+len(" from "):]
fields := strings.Fields(rest)
if len(fields) == 0 {
return ""
}
// A CIDR never ends in ':' (the prefix length is last), so a trailing
// separator can be dropped here; IP tokens go through netutil instead.
candidate := strings.TrimRight(strings.TrimRight(fields[0], ",;)([]"), ":")
_, ipnet, err := net.ParseCIDR(candidate)
if err != nil {
return ""
}
return ipnet.String()
}
// callBlockSubnet uses causal metadata when the engine supports it, preserving
// the legacy interface for other blocker implementations.
func callBlockSubnet(b subnetBlocker, cidr, reason string, timeout time.Duration, findingID string) error {
if sb, ok := b.(interface {
BlockSubnetWithFindingID(string, string, time.Duration, string) error
}); ok {
return sb.BlockSubnetWithFindingID(cidr, reason, timeout, findingID)
}
return b.BlockSubnet(cidr, reason, timeout)
}
package checks
import (
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
// The state-call gate serializes real cleanup. Its separate metadata mutex
// keeps health available while filesystem and database operations are blocked.
type autoBlockCleanupQueue struct {
path string
records map[string]*autoBlockCleanupRecord
sources map[string]bool
blockSources map[string]map[autoBlockCleanupBlock]bool
depthKnown, historyUnknown, lowerBound bool
readFailed, writeFailed, snapshotFailed bool
loss *queuehealth.Tracker
}
type autoBlockCleanupRecord struct {
at time.Time
inFlight, completed, failed, lossKnown bool
blocks map[autoBlockCleanupBlock]bool
blocksKnown bool
}
type autoBlockCleanupBlock struct {
blockedAt, expiresAt time.Time
}
type autoBlockCleanupCycle struct {
entries map[string]*autoBlockCleanupRecord
settled bool
}
func newAutoBlockCleanupQueue() *autoBlockCleanupQueue {
return &autoBlockCleanupQueue{
records: make(map[string]*autoBlockCleanupRecord),
loss: queuehealth.New(0, time.Minute),
}
}
func (c *autoBlockCleanupQueue) setPath(path string) {
if c.path == path {
return
}
c.path = path
clear(c.records)
c.sources = nil
c.blockSources = nil
c.depthKnown, c.historyUnknown = false, false
c.readFailed, c.writeFailed, c.snapshotFailed = false, false, false
}
func (c *autoBlockCleanupQueue) record(ip string, known bool) *autoBlockCleanupRecord {
r := c.records[ip]
if r == nil {
r = &autoBlockCleanupRecord{at: time.Now(), lossKnown: known}
c.records[ip] = r
}
return r
}
func (w *autoBlockStateWork) beginCleanup(path string, ips []string, snapshotErr error) {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
c := q.cleanup
c.setPath(path)
c.snapshotFailed = snapshotErr != nil
c.lowerBound = c.lowerBound || snapshotErr != nil
w.cleanupCycle = &autoBlockCleanupCycle{entries: make(map[string]*autoBlockCleanupRecord)}
for _, ip := range ips {
r := c.record(ip, true)
// A live engine entry is new cleanup demand even if an earlier
// completed cleanup left its retry marker after a failed save.
r.completed, r.lossKnown = false, true
w.cleanupCycle.entries[ip] = r
}
}
func (w *autoBlockStateWork) admitCleanup(ip string) {
w.queue.mu.Lock()
c := w.queue.cleanup
r := c.record(ip, !c.historyUnknown)
blocks := c.blockSources[ip]
if c.depthKnown && r.blocksKnown {
for version := range blocks {
if !r.blocks[version] {
r.completed, r.lossKnown = false, true
break
}
}
}
// Each snapshot owns its immutable source set. Retain only the latest
// cleanup's generations, rather than accumulating every earlier block.
r.blocks = blocks
r.blocksKnown = c.depthKnown
w.cleanupCycle.entries[ip] = r
w.queue.mu.Unlock()
}
func (w *autoBlockStateWork) startCleanup(ip string) {
w.queue.mu.Lock()
w.cleanupCycle.entries[ip].inFlight = true
w.at = time.Now()
w.queue.mu.Unlock()
}
func (w *autoBlockStateWork) cleanupOutcome(ip string, failed bool) {
w.queue.mu.Lock()
r := w.cleanupCycle.entries[ip]
r.completed = r.completed || !failed
r.failed = failed
w.queue.mu.Unlock()
}
func (w *autoBlockStateWork) observeCleanupState(state *blockState, settle bool) {
c := w.queue.cleanup
sources := make(map[string]bool, len(state.IPs)+len(state.CleanupPending))
blocks := make(map[string]map[autoBlockCleanupBlock]bool, len(state.IPs))
for _, b := range state.IPs {
sources[b.IP] = true
if blocks[b.IP] == nil {
blocks[b.IP] = make(map[autoBlockCleanupBlock]bool)
}
blocks[b.IP][autoBlockCleanupBlock{b.BlockedAt.UTC(), b.ExpiresAt.UTC()}] = true
}
for _, ip := range state.CleanupPending {
sources[ip] = true
c.record(ip, !c.historyUnknown)
}
c.sources = sources
c.blockSources = blocks
c.depthKnown = true
for ip, r := range c.records {
if !r.blocksKnown {
// First rediscovery establishes a baseline, not fresh demand.
// Keep it until the next cleanup can distinguish newer blocks.
r.blocks, r.blocksKnown = blocks[ip], true
}
}
w.settleCleanupRecords(settle)
}
func (w *autoBlockStateWork) settleCleanupRecords(settle bool) {
c := w.queue.cleanup
for ip, r := range c.records {
active := w.cleanupCycle != nil && w.cleanupCycle.entries[ip] == r
if !c.sources[ip] && (settle || !active) {
if !r.completed && r.lossKnown {
c.loss.Lose(time.Now(), 1)
}
delete(c.records, ip)
} else if settle {
r.inFlight = false
}
}
}
func (w *autoBlockStateWork) forgetCleanupState() {
c := w.queue.cleanup
c.sources = nil
c.blockSources = nil
c.depthKnown, c.historyUnknown, c.lowerBound = false, true, true
for ip, r := range c.records {
if w.cleanupCycle != nil && w.cleanupCycle.entries[ip] != r {
delete(c.records, ip)
}
}
}
func (w *autoBlockStateWork) finishCleanupLocked() {
c := w.queue.cleanup
switch {
case w.readingState:
c.readFailed = true
w.forgetCleanupState()
case w.retryCycle != nil && w.retryCycle.saving && !w.retryCycle.settled:
c.writeFailed = true
w.forgetCleanupState()
}
if w.cleanupCycle == nil {
return
}
if !w.completed {
for _, r := range w.cleanupCycle.entries {
if r.inFlight && !r.completed {
r.failed = true
}
}
}
if w.cleanupCycle.settled {
return
}
if c.depthKnown {
w.settleCleanupRecords(true)
} else {
// Keep only the last observed batch across uncertainty so a later
// tracker read can recover its retry sources and acknowledgments.
for _, r := range c.records {
r.inFlight = false
}
}
}
func (c *autoBlockCleanupQueue) status(now time.Time, active *autoBlockStateWork) queuehealth.Status {
row := c.loss.Snapshot(now)
row.CapacityUnavailable = true
row.DepthUnavailable = !c.depthKnown
row.DroppedLowerBound = c.lowerBound
row.LagBasis = "deferred_checkpoint"
if active != nil && active.cleanupCycle != nil && len(c.records) > 0 {
row.ProcessingSeconds = max(0, now.Sub(active.at).Seconds())
}
failed := false
for ip, r := range c.records {
if r.inFlight {
row.InFlight++
} else if c.depthKnown || active != nil && active.cleanupCycle != nil && active.cleanupCycle.entries[ip] == r {
row.Depth++
row.LagSeconds = max(row.LagSeconds, now.Sub(r.at).Seconds())
}
failed = failed || r.failed
}
switch {
case c.readFailed || c.writeFailed || c.snapshotFailed:
row.Status, row.Reason = "degraded", "state_io"
case failed:
row.Status, row.Reason = "degraded", "retry_failed"
case row.ProcessingSeconds >= time.Minute.Seconds():
row.Status, row.Reason = "degraded", "processing_lag"
}
return row
}
package checks
import (
"encoding/json"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"time"
"github.com/pidginhost/csm/internal/atomicio"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
)
const netblockHistoryFile = "netblock_history.json"
// netblockHistoryPruneEvery bounds how often aged entries are rewritten out of
// the history file. Counting ignores them either way, and the file would
// otherwise be rewritten on nearly every cycle as entries cross the window.
const netblockHistoryPruneEvery = time.Hour
// netblockHistory remembers when each address was last seen blocked, so a
// subnet that rotates through addresses one block at a time still reaches
// the netblock threshold. Subnets records when each subnet was last blocked:
// only offenders seen after that count toward the next subnet block.
type netblockHistory struct {
IPs map[string]time.Time `json:"ips"`
Active map[string]bool `json:"active"`
Subnets map[string]time.Time `json:"subnets,omitempty"`
PrunedAt time.Time `json:"pruned_at,omitempty"`
}
func loadNetblockHistory(statePath string) (*netblockHistory, error) {
h := &netblockHistory{}
path := filepath.Join(statePath, netblockHistoryFile)
data, err := osFS.ReadFile(path)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("read %s: %w", path, err)
}
if err == nil {
if err := json.Unmarshal(data, h); err != nil {
return nil, fmt.Errorf("decode %s: %w", path, err)
}
}
if h.IPs == nil {
h.IPs = make(map[string]time.Time)
}
if h.Subnets == nil {
h.Subnets = make(map[string]time.Time)
}
return h, nil
}
func saveNetblockHistory(statePath string, h *netblockHistory) error {
path := filepath.Join(statePath, netblockHistoryFile)
if err := atomicio.AtomicWriteJSON(path, 0o600, h); err != nil {
return fmt.Errorf("persist %s: %w", path, err)
}
return nil
}
// ForgetNetblockHistory serializes the operator's firewall/database mutation
// and history removal with auto-block cycles. clear may be nil for history-only
// cleanup; it must not call another auto-block state operation.
func ForgetNetblockHistory(statePath, ip string, clear func()) error {
work := autoBlockQueues.acquire()
defer work.finish()
work.progress()
if clear != nil {
clear()
}
h, err := loadNetblockHistory(statePath)
if err == nil {
if _, ok := h.IPs[ip]; ok {
delete(h.IPs, ip)
delete(h.Active, ip)
err = saveNetblockHistory(statePath, h)
}
}
work.observe(err)
work.complete()
return err
}
// netblockWindow resolves the counting window. Load fills the default and
// validation rejects anything unparseable, so the fallback only covers a
// Config assembled in code.
func netblockWindow(cfg *config.Config) time.Duration {
return parseExpiryWithDefault(cfg.AutoResponse.NetBlockWindow, config.DefaultNetBlockWindow)
}
// netblockTrackedRows uses committed firewall tags even if the scan tracker
// was lost or an operator replaced a block between scans. It does not rebuild
// the tracker or its counters: these rows are only escalation evidence.
func netblockTrackedRows(blocker IPBlocker, tracked []blockedIP, live firewall.LiveBlockedSnapshot, useLive bool) ([]blockedIP, error) {
lister, ok := blocker.(interface {
BlockedEntries() ([]firewall.BlockedEntry, error)
})
if !ok {
return tracked, nil
}
entries, err := lister.BlockedEntries()
if err != nil {
return nil, err
}
byIP := make(map[string]firewall.BlockedEntry, len(entries))
for _, entry := range entries {
byIP[entry.IP] = entry
}
rows := make([]blockedIP, 0, len(tracked)+len(entries))
seen := make(map[string]bool, len(tracked))
for _, row := range tracked {
row.IP = normalizeBlockIP(row.IP)
if entry, found := byIP[row.IP]; found {
if row.LoginSpray != entry.LoginSpray {
// A replacement block cannot inherit the old spray finding.
row.FindingID = ""
row.BlockedAt, row.ExpiresAt = entry.BlockedAt, entry.ExpiresAt
}
row.LoginSpray = entry.LoginSpray
}
rows = append(rows, row)
seen[row.IP] = true
}
for _, entry := range entries {
// Untagged, untracked blocks retain their first-observed history.
// Saved metadata cannot revive a block absent from the kernel.
if entry.LoginSpray && !seen[entry.IP] && blockedLiveOrCached(blocker, live, useLive, entry.IP) {
rows = append(rows, blockedIP{IP: entry.IP, BlockedAt: entry.BlockedAt, ExpiresAt: entry.ExpiresAt, LoginSpray: entry.LoginSpray})
}
}
return rows, nil
}
// recordNetblockHistory notes every address blocked right now: tracker
// entries with their block times, and addresses only the live kernel set
// knows (operator and permanent blocks), stamped when first seen. It reports
// whether the history changed.
func recordNetblockHistory(h *netblockHistory, tracked []blockedIP, current map[string]bool, blocker IPBlocker, now time.Time, window time.Duration) bool {
changed := false
// Persist membership transitions, not per-cycle timestamps: an operator
// re-block is fresh evidence, but a long-lived block must not churn the file.
if h.Active == nil {
h.Active = make(map[string]bool)
for ip := range h.IPs {
h.Active[ip] = true
}
changed = true
}
if allow, ok := blocker.(allowChecker); ok {
for ip := range current {
if allow.IsAllowed(ip) {
delete(current, ip)
}
}
for ip := range h.IPs {
if allow.IsAllowed(ip) {
delete(h.IPs, ip)
delete(h.Active, ip)
changed = true
}
}
}
note := func(ip string, at time.Time) {
if prev, ok := h.IPs[ip]; !ok || at.After(prev) {
h.IPs[ip] = at
changed = true
}
}
inTracker := make(map[string]bool, len(tracked))
for _, b := range tracked {
inTracker[b.IP] = true
if current[b.IP] {
note(b.IP, b.BlockedAt)
}
}
for ip := range current {
if !inTracker[ip] {
if _, ok := h.IPs[ip]; !ok || !h.Active[ip] {
note(ip, now)
}
}
}
for ip := range h.Active {
if !current[ip] {
delete(h.Active, ip)
changed = true
}
}
for ip := range current {
if !h.Active[ip] {
h.Active[ip] = true
changed = true
}
}
if now.Sub(h.PrunedAt) >= netblockHistoryPruneEvery {
for ip, at := range h.IPs {
if !current[ip] && now.Sub(at) > window {
delete(h.IPs, ip)
}
}
for cidr, at := range h.Subnets {
if now.Sub(at) > window {
delete(h.Subnets, cidr)
}
}
h.PrunedAt = now
changed = true
}
return changed
}
// countsTowardNetblock reports whether a block is subnet escalation evidence.
// IPv4 mail spray blocks stay out of /24 escalation. An IPv6 /64 is one
// actor, so its spray blocks count like any other block.
func countsTowardNetblock(b blockedIP) bool {
return !b.LoginSpray || net.ParseIP(b.IP).To4() == nil
}
// currentlyBlocked is every address blocked right now, from the tracker and
// from the live kernel set when the engine can list it. An address only an
// IPv4 mail spray block holds is left out: such blocks neither enter the
// history nor count, while history the address earned before still counts.
func currentlyBlocked(tracked []blockedIP, live firewall.LiveBlockedSnapshot, useLive bool, h *netblockHistory, blocker IPBlocker) map[string]bool {
current := make(map[string]bool, len(tracked)+len(live.V4)+len(live.V6))
sprayed := make(map[string]bool)
for _, b := range tracked {
if !countsTowardNetblock(b) {
sprayed[b.IP] = true
} else {
current[b.IP] = true
}
}
if useLive {
for _, set := range []map[string]struct{}{live.V4, live.V6} {
for ip := range set {
if !sprayed[ip] {
current[ip] = true
}
}
}
}
// A missing family snapshot is unknown, not evidence that an operator's
// permanent block ended. Use the same cached fallback as tracker reconciliation.
for ip := range h.IPs {
if current[ip] || sprayed[ip] {
continue
}
if _, known := live.Contains(ip); useLive && known {
continue
}
if blocker.IsBlocked(ip) {
current[ip] = true
}
}
return current
}
// netblockCounts groups addresses by subnet: every address blocked now, and
// every address whose block ended inside the window. A block that is still in
// place counts however long ago it began. Addresses the firewall now allows
// are no longer evidence.
func netblockCounts(cfg *config.Config, h *netblockHistory, current map[string]bool, blocker IPBlocker, now time.Time, window time.Duration) map[string]int {
allow, _ := blocker.(allowChecker)
counts := make(map[string]int)
for ip, at := range h.IPs {
if !current[ip] && now.Sub(at) > window {
continue
}
cidr := subnetEscalationCIDR(ip)
// Exempt IPs do not contribute toward the netblock threshold so that
// a cluster of blocked addresses inside an operator-declared DoS-exempt
// range cannot inadvertently auto-block that range as a subnet.
if cidr == "" || cidrIntersectsDOSExempt(cfg, cidr) || c2Listed(cfg, ip) {
continue
}
// Ended blocks that an earlier subnet block already answered must
// not re-block the subnet for the rest of the window. Blocks still
// in place count as before.
if last, ok := h.Subnets[cidr]; ok && !current[ip] && !at.After(last) {
continue
}
if allow != nil && allow.IsAllowed(ip) {
continue
}
counts[cidr]++
}
return counts
}
package checks
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
var autoBlockQueues = newAutoBlockQueue()
type autoBlockQueueMonitor struct {
mu sync.Mutex
waiting map[*autoBlockStateWork]struct{}
active *autoBlockStateWork
retries *autoBlockRetryQueue
cleanup *autoBlockCleanupQueue
idleSince time.Time
waitingLoss, activeLoss *queuehealth.Tracker
}
type autoBlockStateWork struct {
queue *autoBlockQueueMonitor
at time.Time
failed, completed bool
readingState bool
retryCycle *autoBlockRetryCycle
cleanupCycle *autoBlockCleanupCycle
}
func newAutoBlockQueue() *autoBlockQueueMonitor {
return &autoBlockQueueMonitor{cleanup: newAutoBlockCleanupQueue(), retries: newAutoBlockRetryQueue(), waiting: make(map[*autoBlockStateWork]struct{}), waitingLoss: queuehealth.New(0, time.Minute), activeLoss: queuehealth.New(1, time.Minute)}
}
func (q *autoBlockQueueMonitor) acquire() *autoBlockStateWork {
w := &autoBlockStateWork{queue: q, at: time.Now()}
q.mu.Lock()
if q.active == nil && len(q.waiting) == 0 {
q.idleSince = w.at
}
q.waiting[w] = struct{}{}
q.mu.Unlock()
blockStateMu.Lock()
q.mu.Lock()
delete(q.waiting, w)
q.active = w
w.at = time.Now()
q.mu.Unlock()
return w
}
func (w *autoBlockStateWork) progress() {
w.queue.mu.Lock()
w.at = time.Now()
w.queue.mu.Unlock()
}
func (w *autoBlockStateWork) failLocked() {
if !w.failed {
w.failed = true
w.queue.activeLoss.Lose(time.Now(), 1)
}
}
func (w *autoBlockStateWork) observe(err error) {
if err == nil {
return
}
w.queue.mu.Lock()
w.failLocked()
w.queue.mu.Unlock()
}
func (w *autoBlockStateWork) complete() {
w.queue.mu.Lock()
w.completed = true
w.queue.mu.Unlock()
}
func (w *autoBlockStateWork) finish() {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
// Release the real slot with its owner, including when the accounting
// below fails. A successor must not publish over a callback whose cleanup
// is still running, and a failure must not strand every later block.
defer blockStateMu.Unlock()
if !w.completed {
w.failLocked()
}
w.finishRetriesLocked()
w.finishCleanupLocked()
q.active = nil
q.idleSince = time.Now()
}
// AutoBlockQueueStatuses reads queue memory without waiting for the state mutex,
// filesystem, firewall or database. A batch is timed by operation progress.
func AutoBlockQueueStatuses(now time.Time) map[string]queuehealth.Status {
q := autoBlockQueues
q.mu.Lock()
defer q.mu.Unlock()
waiting := q.waitingLoss.Snapshot(now)
waiting.CapacityUnavailable = true
active := q.activeLoss.Snapshot(now)
active.LagBasis = "operation_progress"
stalled := q.active == nil && !q.idleSince.IsZero() && now.Sub(q.idleSince) >= time.Minute
if w := q.active; w != nil {
active.InFlight = 1
active.ProcessingSeconds = max(0, now.Sub(w.at).Seconds())
if active.ProcessingSeconds >= time.Minute.Seconds() {
active.Status, active.Reason = "degraded", "processing_lag"
stalled = true
}
}
for w := range q.waiting {
waiting.Depth++
waiting.LagSeconds = max(waiting.LagSeconds, now.Sub(w.at).Seconds())
if stalled {
waiting.Status, waiting.Reason = "degraded", "backlog_lag"
}
}
pending, candidates := q.retries.statuses(now, q.active)
return map[string]queuehealth.Status{"waiting": waiting, "active": active, "pending": pending, "candidates": candidates, "cleanup": q.cleanup.status(now, q.active)}
}
package checks
import (
"errors"
"fmt"
"os"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/queuehealth"
)
var persistAutoBlockState = writeBlockState
// All fields use autoBlockQueueMonitor.mu. Disk identities live only as long
// as their records; an unreadable write outcome never accumulates old versions.
type autoBlockRetryQueue struct {
path string
disk []pendingIP
records []*autoBlockPendingRecord
candidates map[*autoBlockCandidate]struct{}
pendingLoss, candidateLoss *queuehealth.Tracker
depthKnown, historyUnknown, lowerBound bool
readFailed, writeFailed bool
}
type autoBlockPendingRecord struct {
at time.Time
inFlight, completed, lossKnown, failed bool
}
type autoBlockCandidate struct {
at time.Time
origins []*autoBlockPendingRecord
queued *autoBlockPendingRecord
running, completed, finished, failed, lost bool
}
type autoBlockRetryCycle struct {
saving, settled bool
}
func newAutoBlockRetryQueue() *autoBlockRetryQueue {
return &autoBlockRetryQueue{
candidates: make(map[*autoBlockCandidate]struct{}),
pendingLoss: queuehealth.New(maxPendingBlocks, maxPendingAge),
candidateLoss: queuehealth.New(0, time.Minute),
}
}
// InitAutoBlockQueueHealth observes existing retries before daemon consumers
// start, including when automatic blocking is disabled. It never changes state.
func InitAutoBlockQueueHealth(statePath string) error {
work := autoBlockQueues.acquire()
defer work.finish()
_, err := work.readState(statePath)
work.complete()
return err
}
func (w *autoBlockStateWork) readState(path string) (*blockState, error) {
w.progress()
q := w.queue
q.mu.Lock()
w.readingState = true
q.cleanup.setPath(path)
q.mu.Unlock()
state, err := readBlockState(path)
q.mu.Lock()
defer q.mu.Unlock()
w.readingState = false
r := q.retries
if r.path != path {
r.path = path
r.disk, r.records = nil, nil
r.depthKnown, r.historyUnknown = false, false
r.readFailed, r.writeFailed = false, false
}
w.retryCycle = &autoBlockRetryCycle{}
q.cleanup.readFailed = err != nil
r.readFailed = err != nil
if err != nil {
w.failLocked()
r.forgetUncertain()
w.forgetCleanupState()
} else {
r.reconcile(state.Pending, nil, time.Now())
w.observeCleanupState(state, false)
}
return state, err
}
func (w *autoBlockStateWork) loadState(path string) *blockState {
state, err := w.readState(path)
if err != nil {
fmt.Fprintf(os.Stderr, "autoblock: %v; ignoring queued blocks\n", err)
return &blockState{}
}
return state
}
type autoBlockPendingKey struct {
ip, check string
queuedAt time.Time
severity alert.Severity
}
func pendingRecordKey(p pendingIP) autoBlockPendingKey {
return autoBlockPendingKey{ip: p.IP, check: p.Check, queuedAt: p.QueuedAt.UTC(), severity: p.Severity}
}
func (r *autoBlockRetryQueue) reconcile(actual, proposed []pendingIP, now time.Time) {
pool := make(map[autoBlockPendingKey][]*autoBlockPendingRecord, len(r.disk)+len(proposed))
for _, version := range [][]pendingIP{r.disk, proposed} {
for _, p := range version {
if p.queueRecord != nil {
key := pendingRecordKey(p)
pool[key] = append(pool[key], p.queueRecord)
}
}
}
kept := make(map[*autoBlockPendingRecord]bool, len(actual))
records := make([]*autoBlockPendingRecord, 0, len(actual))
for i := range actual {
p := &actual[i]
record := p.queueRecord
if record == nil {
key := pendingRecordKey(*p)
available := pool[key]
for len(available) > 0 {
known := available[0]
available = available[1:]
if !kept[known] {
record = known
break
}
}
pool[key] = available
}
if record == nil {
at := p.QueuedAt
if at.IsZero() {
at = now
}
record = &autoBlockPendingRecord{at: at, lossKnown: !r.historyUnknown}
}
p.queueRecord = record
if !p.QueuedAt.IsZero() {
record.at = p.QueuedAt
}
record.inFlight = false
kept[record] = true
records = append(records, record)
}
// A duplicate coalesced into a retained retry still has an owner. A
// successful block remains completed even when its record survives rollback.
for c := range r.candidates {
survives := kept[c.queued]
for _, origin := range c.origins {
survives = survives || kept[origin]
}
if c.completed || survives {
for _, origin := range c.origins {
if c.completed || !kept[origin] {
origin.completed = true
}
}
} else if len(c.origins) == 0 && !c.lost {
r.candidateLoss.Lose(now, 1)
c.lost = true
}
c.finished = true
}
for _, old := range r.disk {
record := old.queueRecord
if !kept[record] && !record.completed && record.lossKnown {
r.pendingLoss.Lose(now, 1)
}
}
r.records = records
r.disk = append([]pendingIP(nil), actual...)
for i := range r.disk {
r.disk[i].queueCandidate = nil
}
r.depthKnown = true
}
func (r *autoBlockRetryQueue) forgetUncertain() {
r.disk, r.records = nil, nil
r.depthKnown, r.historyUnknown, r.lowerBound = false, true, true
for c := range r.candidates {
c.finished = true
}
}
func (w *autoBlockStateWork) beginPending(p pendingIP) {
w.queue.mu.Lock()
p.queueRecord.inFlight = true
w.queue.mu.Unlock()
}
func (w *autoBlockStateWork) completePending(p pendingIP) {
w.queue.mu.Lock()
p.queueRecord.completed = true
w.queue.mu.Unlock()
}
func (w *autoBlockStateWork) candidate(p pendingIP, existing *autoBlockCandidate) *autoBlockCandidate {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
c := existing
if c == nil {
at := p.QueuedAt
if at.IsZero() {
at = time.Now()
}
c = &autoBlockCandidate{at: at}
q.retries.candidates[c] = struct{}{}
}
if p.queueRecord != nil {
c.origins = append(c.origins, p.queueRecord)
if p.queueRecord.at.Before(c.at) {
c.at = p.queueRecord.at
}
}
return c
}
func (w *autoBlockStateWork) startCandidate(c *autoBlockCandidate) {
w.queue.mu.Lock()
c.running = true
w.queue.mu.Unlock()
}
func (w *autoBlockStateWork) candidateOutcome(c *autoBlockCandidate, err error) {
success := err == nil || errors.Is(err, firewall.ErrIPProtected)
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
c.completed = c.completed || success
c.failed = !success
for _, origin := range c.origins {
origin.completed = origin.completed || success
origin.failed = !success
}
}
// A direct source can complete an eligible retry without consuming it from
// the scan queue. Preserve that acknowledgment before secondary bookkeeping.
func (w *autoBlockStateWork) directOutcome(ip string, attemptAt time.Time, err error) {
if err != nil && !errors.Is(err, firewall.ErrIPProtected) {
return
}
ip = normalizeBlockIP(ip)
if ip == "" {
return
}
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
for _, p := range q.retries.disk {
if normalizeBlockIP(p.IP) == ip && (p.QueuedAt.IsZero() || attemptAt.Sub(p.QueuedAt) <= maxPendingAge) {
p.queueRecord.completed = true
p.queueRecord.failed = false
}
}
}
func (w *autoBlockStateWork) finishCandidate(c *autoBlockCandidate) {
w.queue.mu.Lock()
c.finished = true
w.queue.mu.Unlock()
}
func (w *autoBlockStateWork) rejectCandidate(c *autoBlockCandidate) {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
if len(c.origins) == 0 && !c.completed && !c.lost {
q.retries.candidateLoss.Lose(time.Now(), 1)
c.lost = true
}
c.finished = true
}
func (w *autoBlockStateWork) requeueCandidate(p pendingIP) pendingIP {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
c := p.queueCandidate
record := p.queueRecord
if record == nil {
record = &autoBlockPendingRecord{at: p.QueuedAt, lossKnown: true}
q.retries.records = append(q.retries.records, record)
}
record.inFlight = true
record.failed = c.failed || record.failed
p.queueRecord = record
c.queued = record
return p
}
func (w *autoBlockStateWork) writeState(path string, state *blockState) error {
q := w.queue
q.mu.Lock()
w.retryCycle.saving = true
q.mu.Unlock()
err := persistAutoBlockState(path, state)
q.mu.Lock()
q.retries.writeFailed = err != nil
q.cleanup.writeFailed = err != nil
if err != nil {
w.failLocked()
q.retries.depthKnown = false
q.cleanup.depthKnown = false
}
q.mu.Unlock()
// Rename may have succeeded before directory fsync returned an error.
// Only readback can distinguish a retained old record from a new one.
actual := state
var readErr error
if err != nil {
w.progress()
actual, readErr = readBlockState(path)
}
q.mu.Lock()
defer q.mu.Unlock()
if readErr != nil {
q.retries.readFailed = true
q.retries.forgetUncertain()
q.cleanup.readFailed = true
w.forgetCleanupState()
} else {
q.retries.reconcile(actual.Pending, state.Pending, time.Now())
w.observeCleanupState(actual, true)
q.retries.readFailed = false
q.cleanup.readFailed = false
if err == nil {
q.retries.historyUnknown = false
q.cleanup.historyUnknown = false
}
}
w.retryCycle.settled = true
if w.cleanupCycle != nil && readErr == nil {
w.cleanupCycle.settled = true
}
return err
}
func (w *autoBlockStateWork) saveState(path string, state *blockState) {
if err := w.writeState(path, state); err != nil {
logBlockStateFailure(path, err)
}
}
func (w *autoBlockStateWork) finishRetriesLocked() {
if w.readingState {
w.queue.retries.readFailed = true
w.queue.retries.forgetUncertain()
}
if w.retryCycle == nil {
return
}
r := w.queue.retries
if !w.completed {
for c := range r.candidates {
if c.running && !c.completed {
for _, origin := range c.origins {
origin.failed = true
}
}
}
}
if !w.retryCycle.settled {
switch {
case w.retryCycle.saving:
r.writeFailed = true
r.forgetUncertain()
case r.depthKnown:
r.reconcile(r.disk, nil, time.Now())
default:
// The original state was unreadable, but no write started. Fresh work
// accepted since that read cannot have reached the durable file.
for c := range r.candidates {
if !c.completed && !c.lost && len(c.origins) == 0 {
r.candidateLoss.Lose(time.Now(), 1)
c.lost = true
}
}
}
}
clear(r.candidates)
}
func (r *autoBlockRetryQueue) statuses(now time.Time, active *autoBlockStateWork) (queuehealth.Status, queuehealth.Status) {
pending := r.pendingLoss.Snapshot(now)
pending.DroppedLowerBound = r.lowerBound
pending.DepthUnavailable = !r.depthKnown
pending.LagBasis = "queued_or_observed_age"
if !r.depthKnown {
pending.LagBasis = "unavailable"
}
candidates := r.candidateLoss.Snapshot(now)
candidates.CapacityUnavailable = true
candidates.DroppedLowerBound = r.lowerBound
candidates.LagBasis = "operation_progress"
processing := 0.0
if active != nil {
processing = max(0, now.Sub(active.at).Seconds())
}
failed := false
for _, record := range r.records {
if record.inFlight {
pending.InFlight++
pending.ProcessingSeconds = processing
} else if r.depthKnown {
pending.Depth++
pending.LagSeconds = max(pending.LagSeconds, now.Sub(record.at).Seconds())
}
failed = failed || record.failed
}
for c := range r.candidates {
if c.finished {
continue
}
if c.running {
candidates.InFlight++
candidates.ProcessingSeconds = processing
} else {
candidates.Depth++
candidates.LagSeconds = max(candidates.LagSeconds, now.Sub(c.at).Seconds())
}
failed = failed || c.failed
}
switch {
case r.readFailed || r.writeFailed:
pending.Status, pending.Reason = "degraded", "state_io"
case failed:
pending.Status, pending.Reason = "degraded", "retry_failed"
case pending.InFlight > 0 && processing >= time.Minute.Seconds():
pending.Status, pending.Reason = "degraded", "processing_lag"
case pending.LagSeconds > maxPendingAge.Seconds():
pending.Status, pending.Reason = "degraded", "backlog_lag"
}
if candidates.InFlight+candidates.Depth > 0 && processing >= time.Minute.Seconds() {
candidates.Status, candidates.Reason = "degraded", "processing_lag"
}
return pending, candidates
}
package checks
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"syscall"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/processhandle"
)
// var (not const) so tests can redirect to t.TempDir().
var quarantineDir = "/opt/csm/quarantine"
// InQuarantine reports whether path lies inside CSM's quarantine directory.
func InQuarantine(path string) bool {
path = filepath.Clean(path)
return path == quarantineDir || strings.HasPrefix(path, quarantineDir+string(filepath.Separator))
}
// RemovedFileRestoredCheck reports a process writing back, byte for byte,
// content CSM quarantined or cleaned at the same path.
const RemovedFileRestoredCheck = "removed_file_restored_realtime"
// autoQuarantineChecks are the findings the scheduled auto-responder may
// quarantine on its own: the manual move set plus the kill-and-quarantine
// family, and the realtime signature match, which must
// additionally pass isHighConfidenceRealtimeMatch. Membership is pinned by
// test against the check registry.
var autoQuarantineChecks = map[string]bool{
"webshell": true,
"backdoor_binary": true,
"new_webshell_file": true,
"new_executable_in_config": true,
"obfuscated_php": true,
"suspicious_php_content": true,
"new_php_in_languages": true,
"new_php_in_upgrade": true,
"phishing_page": true,
"phishing_directory": true,
"signature_match_realtime": true,
RemovedFileRestoredCheck: true,
}
var signalProcess = processhandle.Signal
var errProcessNotEligible = errors.New("process is no longer eligible for termination")
// AutoKillProcesses kills processes that match critical findings.
// Only targets: fake kernel threads, reverse shells, GSocket processes, and
// detached web shells and the processes that wrote back a file CSM removed.
// Never kills root system services or cPanel processes.
// Marks evaluated input findings so alert delivery cannot repeat a response.
func AutoKillProcesses(ctx context.Context, cfg *config.Config, findings []alert.Finding) []alert.Finding {
if !cfg.AutoResponse.Enabled || !cfg.AutoResponse.KillProcesses {
return nil
}
var actions []alert.Finding
// Sibling chains and independent checks can name the same process.
killed := make(map[int]bool)
for i, f := range findings {
// Only act on specific high-confidence critical checks
switch f.Check {
case "fake_kernel_thread", "suspicious_process", "php_suspicious_execution", RemovedFileRestoredCheck, DetachedWebProcessCheck:
default:
continue
}
if f.Severity != alert.Critical || f.AutoProcessResponseEvaluated {
continue
}
// A scan hands its findings to alert delivery, which must not
// signal the same process again.
findings[i].AutoProcessResponseEvaluated = true
if f.Check == RemovedFileRestoredCheck || f.Check == DetachedWebProcessCheck {
// The parents go first so a loop cannot start another writer.
// Recheck origin and terminal inside the process handle too:
// a captured process can exec or acquire a terminal before this.
for j := len(f.WriterChain) - 1; j >= 0; j-- {
pid := f.WriterChain[j]
if killed[pid] {
continue
}
detached := f.Check == DetachedWebProcessCheck
allow := func(exe string) bool {
chain := restoreWriterChain(pid, f.WriterObservedAt, !detached && j == 0)
if len(chain) == 0 || chain[0] != pid {
return false
}
// Every target must still be a shell, including ancestors.
// Read again after the ancestry walk, which can span an exec.
return !detached || isShellProcess(strconv.Itoa(pid), exe)
}
if action, ok := killFindingProcess(ctx, f, pid, allow); ok {
killed[pid] = true
actions = append(actions, action)
}
}
continue
}
// Use structured PID field when available, fall back to text extraction
pid := fmt.Sprintf("%d", f.PID)
if f.PID == 0 {
pid = extractPID(f.Details)
if pid == "" {
continue
}
}
pidInt, validPID := parseProcessPID(pid)
if !validPID || killed[pidInt] {
continue
}
if action, ok := killFindingProcess(ctx, f, pidInt, func(exe string) bool { return !isSafeProcess(exe) }); ok {
killed[pidInt] = true
actions = append(actions, action)
}
}
return actions
}
// killFindingProcess signals one process for f after verifying, through the
// captured handle, that it is still a non-root process with an executable
// allowExe accepts that started before the finding.
func killFindingProcess(ctx context.Context, f alert.Finding, pidInt int, allowExe func(string) bool) (alert.Finding, bool) {
pid := strconv.Itoa(pidInt)
var uid, exe string
err := signalProcess(ctx, pidInt, syscall.SIGKILL, func() error {
uid, exe = getProcessUID(pid), getProcessExe(pid)
if uid == "0" || uid == "" || exe == "" || !processStartedBefore(pid, f.Timestamp) || !allowExe(exe) {
return errProcessNotEligible
}
return nil
})
recordKillAction(&f, pid, exe, err)
if err != nil {
if !errors.Is(err, errProcessNotEligible) && !errors.Is(err, os.ErrProcessDone) && ctx.Err() == nil {
csmlog.Warn("auto-kill: safe process signaling failed", "pid", pidInt, "err", err)
}
return alert.Finding{}, false
}
return alert.Finding{
Severity: alert.Critical,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-KILL: Process %s killed (was: %s)", pid, f.Check),
Timestamp: time.Now(),
Details: fmt.Sprintf("Original finding: %s\nProcess: %s (UID: %s)", f.Message, exe, uid),
}, true
}
// recordKillAction writes the unified action record for one termination
// attempt. A refusal is recorded as well as a kill: "the safety rules stopped
// this" is the answer to a question an operator will ask about a process that
// is still running.
func recordKillAction(f *alert.Finding, pid, exe string, err error) {
rec := actionlog.Record{
Op: "respond.kill_process",
Actor: actionlog.DefaultActor(),
Target: "pid " + pid,
ActorDetail: exe,
Reason: "manual process termination",
FindingID: "",
Result: actionlog.Applied,
}
if f != nil {
rec.Reason = f.Check
rec.FindingID = alert.FindingID(*f)
}
switch {
case errors.Is(err, errProcessNotEligible):
rec.Result = actionlog.Refused
rec.Error = "process is not eligible for automatic termination"
case errors.Is(err, os.ErrProcessDone):
rec.Result = actionlog.Refused
rec.Error = "process had already exited"
case err != nil:
rec.Result = actionlog.Failed
rec.Error = err.Error()
}
actionlog.Write(rec)
}
// AutoQuarantineFiles moves malicious files to quarantine directory.
// Preserves original path and metadata in a sidecar .meta file.
// Marks evaluated input findings so alert delivery cannot repeat a response.
func AutoQuarantineFiles(cfg *config.Config, findings []alert.Finding) []alert.Finding {
if cfg == nil || !cfg.AutoResponse.Enabled || !cfg.AutoResponse.QuarantineFiles || cfg.ObserveMode() {
return nil
}
var actions []alert.Finding
seen := make(map[string]bool)
for i, f := range findings {
if f.AutoFileResponseEvaluated || !autoQuarantineChecks[f.Check] || f.Severity != alert.Critical {
continue
}
path := f.FilePath
if path == "" {
path = extractFilePath(f.Message)
}
if path == "" {
continue
}
findings[i].AutoFileResponseEvaluated = true
key := filepath.Clean(path)
if seen[key] {
continue
}
realtime := f.Check == "signature_match_realtime"
if realtime && !isHighConfidenceRealtimeMatch(f, path, nil) {
continue
}
info, err := osFS.Lstat(path)
if err != nil || info.Mode()&os.ModeSymlink != 0 {
continue
}
if f.FileResponseIdentity != "" && f.FileResponseIdentity != fileResponseIdentity(info) {
continue
}
clean := !realtime && ShouldCleanInsteadOfQuarantine(path)
restored := f.Check == RemovedFileRestoredCheck
var removed RemovedFile
if restored {
// Repeat what CSM did to these bytes before: the path alone does
// not say whether the whole file was malicious.
removed, info = restoredFileResponseInfo(path, f.DedupKey, info)
if info == nil {
continue
}
clean = removed.Action == "clean"
if clean && filepath.Base(path) == ".htaccess" && !cfg.AutoResponse.CleanHtaccess {
continue
}
}
// Multiple checks can report one file. Do not re-clean a repaired
// target or charge repeated failures for the same batch of evidence.
seen[key] = true
paused := runAutoFileResponse(cfg, path, info, func() error {
if restored {
latest, ok := RemovedFileAt(path)
current, statErr := osFS.Lstat(path)
if !ok || latest != removed || statErr != nil || !SameFileVersion(info, current) {
return refuseFileResponse(errors.New("restored file evidence changed before response"))
}
}
// Cleaning is one response attempt. A failed cleaner leaves the file
// and any backup for review; it must not escalate to removing the file.
var wholeFile CleanResult
if clean {
if restored && filepath.Base(path) == ".htaccess" {
result := cleanHtaccessFileIdentified(path, info)
if result.Success {
actions = append(actions, alert.Finding{Severity: alert.Critical, Check: "auto_response", Message: fmt.Sprintf("AUTO-CLEAN: %s hardened directives removed", path), Details: result.Description, Timestamp: time.Now()})
}
if result.Error != "" && !result.Refused {
return errors.New(result.Error)
}
return nil
}
result := cleanInfectedFileIdentified(path, info)
// Malware that is the whole file is quarantined below: a
// clean would leave an empty stub in its place.
if !result.WholeFile {
if result.Error != "" {
outcome := "failed"
if result.Refused {
outcome = "refused"
}
actions = append(actions, alert.Finding{Severity: alert.Warning, Check: "auto_response", Message: fmt.Sprintf("AUTO-CLEAN %s for %s; manual review required", outcome, path), Details: result.Error, Timestamp: time.Now()})
// Safety refusals consume capacity without charging a failure.
if result.Refused {
return nil
}
return errors.New(result.Error)
}
if result.Cleaned {
actions = append(actions, alert.Finding{Severity: alert.Critical, Check: "auto_response", Message: fmt.Sprintf("AUTO-CLEAN: %s surgically cleaned", path), Details: fmt.Sprintf("Backup: %s\n%s", result.BackupPath, strings.Join(result.Removals, "\n")), Timestamp: time.Now()})
}
return nil
}
wholeFile = result
}
qPath := newQuarantinePath(quarantineDir, path)
meta := quarantineMetadata(path, info, f.Message)
meta.FindingID = alert.FindingID(f)
var err error
switch {
case restored:
err = quarantineRestoredTarget(path, qPath, info, meta, removed)
case wholeFile.WholeFile:
err = quarantineVersionedTargetFD(path, qPath, wholeFile.wholeFileInfo, meta, -1, wholeFile.wholeFileContent)
default:
err = quarantineTarget(path, qPath, info, meta)
}
warning := ""
if err != nil {
var completed bool
warning, completed = completedQuarantineWarning(err)
if !completed {
return err
}
}
details := fmt.Sprintf("Quarantined to: %s\nOriginal finding: %s", qPath, f.Message)
if warning != "" {
details += "\nWarning: " + warning
}
actions = append(actions, alert.Finding{Severity: alert.Critical, Check: "auto_response", Message: fmt.Sprintf("AUTO-QUARANTINE: %s moved to quarantine", path), Timestamp: time.Now(), Details: details})
return nil
})
if paused != nil {
actions = append(actions, *paused)
}
}
return actions
}
// AutoFixPermissions sets world/group-writable PHP files to 0644.
// Returns the auto-response action findings and the keys of original findings
// that were successfully fixed (so the caller can dismiss them from the UI).
func AutoFixPermissions(cfg *config.Config, findings []alert.Finding) (actions []alert.Finding, fixedKeys []string) {
if !cfg.AutoResponse.Enabled || !cfg.AutoResponse.EnforcePermissions {
return nil, nil
}
for _, f := range findings {
switch f.Check {
case "world_writable_php", "group_writable_php":
default:
continue
}
path := selectFindingPath(f.Message, f.FilePath)
if path == "" {
continue
}
result, oldMode := applyPermissionFix(path, f.Check)
if !result.Success {
continue
}
if oldMode != 0 {
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-FIX: %s permissions set to 644 (was %o)", path, oldMode),
Timestamp: time.Now(),
})
}
fixedKeys = append(fixedKeys, f.Key())
}
return actions, fixedKeys
}
// AutoFixWPCron disables WP-Cron and installs a per-user system cron for every
// perf_wp_cron finding. Returns the auto-response action findings and the keys
// of the originals so the caller can dismiss them. Gated behind an explicit
// opt-in because it edits customer wp-config.php and crontabs.
func AutoFixWPCron(cfg *config.Config, findings []alert.Finding) (actions []alert.Finding, fixedKeys []string) {
if !cfg.AutoResponse.Enabled || !cfg.AutoResponse.FixWPCron {
return nil, nil
}
opts := WPCronFixOptions{
IntervalMinutes: cfg.Performance.WPCronFix.IntervalMinutes,
PHPBin: cfg.Performance.WPCronFix.PHPBin,
}
allowedRoots := ResolveValidatedDocroots(cfg)
for _, f := range findings {
if f.Check != "perf_wp_cron" {
continue
}
path := extractWPConfigPath(f.Details)
if path == "" {
continue
}
res := FixDisableWPCronInRoots(path, allowedRoots, opts)
if !res.Success {
continue
}
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-FIX: %s", res.Description),
Timestamp: time.Now(),
})
fixedKeys = append(fixedKeys, f.Key())
}
return actions, fixedKeys
}
// extractWPConfigPath pulls the wp-config.php path out of a perf_wp_cron
// finding's Details, formatted as "File: <path> - add define(...)".
func extractWPConfigPath(details string) string {
const prefix = "File: "
idx := strings.Index(details, prefix)
if idx < 0 {
return ""
}
rest := details[idx+len(prefix):]
if j := strings.Index(rest, " - "); j >= 0 {
rest = rest[:j]
}
return strings.TrimSpace(rest)
}
func extractPID(details string) string {
// Look for "PID: 12345" pattern. Stop at the first whitespace, comma,
// or newline so a trailing word ("PID: 42 exe=/bin/ls") doesn't get
// returned as part of the PID string.
idx := strings.Index(details, "PID: ")
if idx < 0 {
return ""
}
rest := details[idx+5:]
for i, c := range rest {
if c == ' ' || c == ',' || c == '\n' || c == '\t' {
return strings.TrimSpace(rest[:i])
}
}
return strings.TrimSpace(rest)
}
func extractFilePath(message string) string {
return findingPathInMessage(message, " ,\n")
}
func getProcessUID(pid string) string {
data, err := osFS.ReadFile(filepath.Join("/proc", pid, "status"))
if err != nil {
return ""
}
for _, line := range strings.Split(string(data), "\n") {
if rest, found := strings.CutPrefix(line, "Uid:"); found {
fields := strings.Fields(rest)
if len(fields) != 4 {
return ""
}
var owner uint64
for index, field := range fields {
uid, err := strconv.ParseUint(field, 10, 32)
if err != nil {
return ""
}
// Effective, saved, and filesystem root credentials are also
// privileged even when the real UID still names a tenant.
if uid == 0 {
return "0"
}
if index == 0 {
owner = uid
}
}
return strconv.FormatUint(owner, 10)
}
}
return ""
}
func getProcessExe(pid string) string {
exe, err := osFS.Readlink(filepath.Join("/proc", pid, "exe"))
if err != nil {
return ""
}
return exe
}
func isSafeProcess(exe string) bool {
safePrefixes := []string{
"/usr/local/cpanel/",
"/usr/sbin/",
"/usr/bin/",
"/usr/libexec/",
"/opt/cpanel/",
"/opt/cloudlinux/",
"/opt/imunify360/",
}
for _, prefix := range safePrefixes {
if strings.HasPrefix(exe, prefix) {
return true
}
}
return false
}
// isHighConfidenceRealtimeMatch validates whether a realtime signature match
// is truly malicious and safe to auto-quarantine. Prevents false positives
// on legitimate libraries (PHPMailer, zip) and theme code.
//
// The data parameter should be the file content already read by the caller
// (fanotify fd or scanner) to avoid TOCTOU re-reads. Pass nil to read from path.
//
// Criteria:
// 1. Category must be "dropper" or "webshell"
// 2. File must be >= 512 bytes (entropy unreliable below this)
// 3. Content must show obfuscation indicators:
// Shannon entropy >= 5.5 OR hex density > 20% plus an execution signal.
// This applies to BOTH dropper and webshell categories to avoid
// false-positive quarantine of legitimate plugins that happen to
// match a dropper rule (e.g. curl_exec + eval on distant lines).
func isHighConfidenceRealtimeMatch(f alert.Finding, path string, data []byte) bool {
cat := extractCategory(f.Details)
switch cat {
case "dropper", "webshell":
default:
return false
}
if data == nil {
var err error
data, err = osFS.ReadFile(path)
if err != nil {
return false
}
}
if len(data) < 512 {
return false
}
// Both dropper and webshell categories go through the same entropy/encoding
// checks to avoid false positives. Normal PHP with heavy class constants
// (binary literals, many named constants) lands in the 4.5-5.3 range --
// measured: WPML wpml_zip.php = 5.25, Breakdance google-fonts.php = 4.90.
// The 5.5 floor leaves that headroom. Obfuscated packers land at 5.8+.
// Files that use long hex-string payloads (LEVIATHAN signature) are
// caught by the hex-density arm instead, which stays at 20%.
content := string(data)
// High Shannon entropy is a strong standalone signal: packed/encrypted
// payloads land at 5.8+, while ordinary library code (even with a handful
// of binary constants) stays below 5.5 -- measured WPML wpml_zip.php =
// 5.25, Breakdance google-fonts.php = 4.90.
if shannonEntropy(content) >= 5.5 {
return true
}
// High hex density alone is NOT enough: a ZIP/PDF library's magic-byte
// constant tables ("\x50\x4b\x03\x04" ...) saturate that metric while being
// inert data. Require a structural obfuscated-execution signal too. This
// replaces a hardcoded library-path allowlist (vendor/, node_modules/,
// named plugin slugs) that an attacker could defeat by planting a webshell
// under any "trusted" directory -- the file is now judged by content, so a
// hex-encoded packer still quarantines wherever it hides and a benign
// data-heavy library file is spared on any path.
if hexEncodingDensity(content) > 0.20 {
return hasObfuscatedExecutionSignal(content)
}
return false
}
var (
reVariableFunctionCall = regexp.MustCompile(`\$[A-Za-z_]\w*\s*\(`)
reHexEscapedStringConcat = regexp.MustCompile(`(?i)"(?:\\x[0-9a-f]{2})+"\s*\.\s*"(?:\\x[0-9a-f]{2})+"`)
)
// hasObfuscatedExecutionSignal reports whether content carries a structural
// sign of obfuscated code execution, distinguishing a packed webshell from
// inert binary data such as a ZIP library's magic-byte constants. Any signal
// is sufficient:
// - LEVIATHAN-style control-flow obfuscation (goto spaghetti).
// - Function names built from concatenated hex escapes ("\x65"."\x76"... to
// dodge literal-name detection).
// - A variable bound to a decoder/exec primitive and later invoked.
// - A decoder (base64/gz/rot13/openssl/hex2bin) paired with an executor
// (eval/assert/create_function, a literal dangerous callback, or a
// request-scoped variable-function call).
func hasObfuscatedExecutionSignal(content string) bool {
code := stripPHPCommentsFromCode(content)
codeNoStrings := strings.ToLower(stripPHPStringsFromCode(code))
if countOccurrences(codeNoStrings, "goto ") > 10 {
return true
}
// Function-name obfuscation: many double-quoted hex string literals joined
// by the concatenation operator. Standalone hex constant tables (no concat)
// are inert data and do not match.
if countHexEscapedStringConcats(code) > 10 {
return true
}
if detectVarFuncDangerousAssignment(code) {
return true
}
if !containsDirectPHPFunctionCall(codeNoStrings, []string{
"base64_decode", "gzinflate", "gzuncompress", "gzdecode",
"str_rot13", "openssl_decrypt", "hex2bin", "convert_uudecode",
}) {
return false
}
if containsDirectPHPFunctionCall(codeNoStrings, []string{
"eval", "assert", "create_function",
}) {
return true
}
if hasLiteralCallbackExecutor(code) {
return true
}
return hasRequestScopedVariableFunctionCall(codeNoStrings)
}
func countHexEscapedStringConcats(code string) int {
return len(reHexEscapedStringConcat.FindAllStringIndex(code, -1))
}
func containsDirectPHPFunctionCall(codeNoStrings string, names []string) bool {
for _, name := range names {
if containsStandaloneFunc(codeNoStrings, name+"(") {
return true
}
}
return false
}
func hasRequestScopedVariableFunctionCall(codeNoStrings string) bool {
for _, line := range strings.Split(codeNoStrings, "\n") {
if containsRequestSuperglobal(line) && reVariableFunctionCall.MatchString(line) {
return true
}
}
return false
}
func hasLiteralCallbackExecutor(code string) bool {
for i := 0; i < len(code); i++ {
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
nameStart := i
if code[i] == '\\' {
if i+1 >= len(code) || !isPHPIdentifierStart(code[i+1]) || !canStartGlobalPHPFunction(code, i) {
continue
}
nameStart = i + 1
} else if !isPHPIdentifierStart(code[i]) || !canStartPHPFunctionName(code, i) {
continue
}
nameEnd := nameStart + 1
for nameEnd < len(code) && isPHPIdentifierPart(code[nameEnd]) {
nameEnd++
}
name := strings.ToLower(code[nameStart:nameEnd])
if _, ok := callbackFirstArgFuncs[name]; !ok {
i = nameEnd - 1
continue
}
openParen := skipPHPWhitespace(code, nameEnd)
if openParen >= len(code) || code[openParen] != '(' {
i = nameEnd - 1
continue
}
firstArg := skipPHPWhitespace(code, openParen+1)
if firstArg >= len(code) || !isPHPQuote(code[firstArg]) {
i = nameEnd - 1
continue
}
callbackName, _, ok := readPHPFunctionString(code, firstArg)
if !ok {
i = nameEnd - 1
continue
}
if _, dangerous := callbackExecNames[callbackName]; dangerous {
return true
}
i = nameEnd - 1
}
return false
}
// hexEncodingDensity returns the fraction of a string's bytes that are part
// of PHP hex escape sequences (\xNN). LEVIATHAN AES-encrypted webshells
// encode their payload as long hex strings - the \x prefix repeats so
// frequently that Shannon entropy drops to ~3.5 (below normal PHP), but
// the hex density reaches 40-60%.
func hexEncodingDensity(s string) float64 {
if len(s) == 0 {
return 0
}
hexBytes := 0
for i := 0; i < len(s)-3; i++ {
if s[i] == '\\' && s[i+1] == 'x' &&
isHexDigit(s[i+2]) && isHexDigit(s[i+3]) {
hexBytes += 4
i += 3 // skip past this sequence
}
}
return float64(hexBytes) / float64(len(s))
}
func isHexDigit(b byte) bool {
return (b >= '0' && b <= '9') || (b >= 'a' && b <= 'f') || (b >= 'A' && b <= 'F')
}
// InlineQuarantineGated applies the operator's quarantine policy before
// InlineQuarantine moves anything. The realtime fanotify path detects malware
// continuously, but moving a file is a customer-impacting auto-response
// action: it must honor the same master switch and quarantine opt-in as the
// batch AutoQuarantineFiles dispatcher, never act on detection alone. An
// operator in monitor mode (auto-response off, or quarantine_files off) gets
// the alert without having files moved out from under them.
func InlineQuarantineGated(cfg *config.Config, f alert.Finding, path string, data []byte) (string, bool) {
path, ok, _ := InlineQuarantineGatedIdentified(cfg, &f, path, data, nil)
return path, ok
}
// InlineQuarantineGatedIdentified applies the auto-response policy gate and
// then quarantines the exact file the caller scanned. See
// InlineQuarantineIdentified for why the identity matters.
// Marks the finding evaluated only once it reaches the shared budget gate.
func InlineQuarantineGatedIdentified(cfg *config.Config, f *alert.Finding, path string, data []byte, scanned os.FileInfo) (string, bool, *alert.Finding) {
if f == nil || cfg == nil || !cfg.AutoResponse.Enabled || !cfg.AutoResponse.QuarantineFiles || cfg.ObserveMode() {
return "", false, nil
}
info, ok := inlineQuarantineInfo(*f, path, data, scanned)
if !ok {
return "", false, nil
}
return inlineQuarantineGatedWithInfo(cfg, f, path, info)
}
// InlineQuarantineConfirmedPHPGatedIdentified applies automatic quarantine to
// a Critical PHP execution finding that the realtime detector confirmed from
// an open file snapshot. It accepts only the two high-confidence content
// families and requires both the inspected bytes and identity. Scanner rules,
// URL-only droppers and path-based findings retain their existing gates.
func InlineQuarantineConfirmedPHPGatedIdentified(cfg *config.Config, f *alert.Finding, path string, data []byte, scanned os.FileInfo) (string, bool, *alert.Finding) {
return InlineQuarantineConfirmedPHPGatedIdentifiedFD(cfg, f, path, data, scanned, -1)
}
// InlineQuarantineConfirmedPHPGatedIdentifiedFD keeps the detector's open
// description through quarantine so Linux can exclude other writers without
// treating the fanotify descriptor itself as a conflict.
func InlineQuarantineConfirmedPHPGatedIdentifiedFD(cfg *config.Config, f *alert.Finding, path string, data []byte, scanned os.FileInfo, sourceFD int) (string, bool, *alert.Finding) {
if f == nil || cfg == nil || !cfg.AutoResponse.Enabled || !cfg.AutoResponse.QuarantineFiles || cfg.ObserveMode() {
return "", false, nil
}
if !isPinnedCriticalPHPContent(*f, data, scanned) {
return "", false, nil
}
info, err := osFS.Lstat(path)
if err != nil || info.Mode()&os.ModeSymlink != 0 || !SameFileVersion(info, scanned) {
return "", false, nil
}
return inlineQuarantineGatedWithInfoFD(cfg, f, path, info, sourceFD, data)
}
func inlineQuarantineGatedWithInfo(cfg *config.Config, f *alert.Finding, path string, info os.FileInfo) (string, bool, *alert.Finding) {
return inlineQuarantineGatedWithInfoFD(cfg, f, path, info, -1, nil)
}
func inlineQuarantineGatedWithInfoFD(cfg *config.Config, f *alert.Finding, path string, info os.FileInfo, sourceFD int, scannedHead []byte) (string, bool, *alert.Finding) {
var qPath string
f.AutoFileResponseEvaluated = true
paused := runAutoFileResponse(cfg, path, info, func() error {
var err error
qPath, err = quarantineInlineTargetFD(*f, path, info, sourceFD, scannedHead)
return err
})
return qPath, qPath != "", paused
}
// InlineQuarantine moves a file to quarantine immediately if it passes the
// high-confidence validation gates. Called from fanotify's analyzeFile to
// quarantine malware without waiting for the 5-second batch dispatcher.
// The data parameter is the file content already read by the caller (avoids
// TOCTOU re-read). Pass nil to read from path.
// Returns the quarantine path and true if the file was quarantined.
func InlineQuarantine(f alert.Finding, path string, data []byte) (string, bool) {
return InlineQuarantineIdentified(f, path, data, nil)
}
// InlineQuarantineIdentified is InlineQuarantine with the identity of the file
// the caller actually scanned. The realtime scanner reads content from the
// fanotify event descriptor, so passing that descriptor's stat pins the move to
// the object that was examined: a file replaced between detection and
// quarantine fails the identity check instead of being moved in place of the
// malware. A nil identity keeps the older path-based behaviour for callers that
// began from a path in the first place, such as the batch dispatcher.
func InlineQuarantineIdentified(f alert.Finding, path string, data []byte, scanned os.FileInfo) (string, bool) {
info, ok := inlineQuarantineInfo(f, path, data, scanned)
if !ok {
return "", false
}
qPath, err := quarantineInlineTarget(f, path, info)
if err != nil {
csmlog.Warn("inline quarantine refused", "path", path, "err", err)
}
return qPath, err == nil
}
func inlineQuarantineInfo(f alert.Finding, path string, data []byte, scanned os.FileInfo) (os.FileInfo, bool) {
if !isHighConfidenceRealtimeMatch(f, path, data) {
return nil, false
}
info, err := osFS.Lstat(path)
if err != nil || info.Mode()&os.ModeSymlink != 0 {
return nil, false
}
if scanned != nil && (!sameFileIdentity(info, scanned) || !sameContentShape(info, scanned)) {
return nil, false
}
return info, true
}
// isPinnedCriticalPHPContent accepts only the strongest realtime PHP findings
// when the caller supplies the exact bytes and file identity it inspected.
// The detector has already proved the execution dataflow or packed execution;
// this gate proves that quarantine still names that same object. URL-only
// droppers, filename checks and scanner matches keep their existing review or
// scanner-specific confidence paths.
func isPinnedCriticalPHPContent(f alert.Finding, data []byte, scanned os.FileInfo) bool {
if f.Severity != alert.Critical || len(data) == 0 || scanned == nil {
return false
}
return isConfirmedPHPResponseCheck(f.Check)
}
func isConfirmedPHPResponseCheck(check string) bool {
switch check {
case "webshell_content_realtime", "obfuscated_php_realtime", "webshell_realtime", "php_in_uploads_realtime":
return true
default:
return false
}
}
func quarantineInlineTarget(f alert.Finding, path string, info os.FileInfo) (string, error) {
return quarantineInlineTargetFD(f, path, info, -1, nil)
}
func quarantineInlineTargetFD(f alert.Finding, path string, info os.FileInfo, sourceFD int, scannedHead []byte) (string, error) {
qPath := newQuarantinePath(quarantineDir, path)
reason := "Inline quarantine: high-confidence realtime signature match"
move := quarantineTarget
if isConfirmedPHPResponseCheck(f.Check) {
reason = "Inline quarantine: confirmed realtime PHP content"
move = func(path, qPath string, info os.FileInfo, metadata QuarantineMeta) error {
return quarantineVersionedTargetFD(path, qPath, info, metadata, sourceFD, scannedHead)
}
}
meta := quarantineMetadata(path, info, reason)
meta.FindingID = alert.FindingID(f)
if err := move(path, qPath, info, meta); err != nil {
if warning, completed := completedQuarantineWarning(err); completed {
csmlog.Warn("inline quarantine completed with warning", "warning", warning)
} else {
return "", err
}
}
return qPath, nil
}
// extractCategory parses "Category: <value>" from a finding's Details field.
func extractCategory(details string) string {
for _, line := range strings.Split(details, "\n") {
if strings.HasPrefix(line, "Category: ") {
return strings.TrimPrefix(line, "Category: ")
}
}
return ""
}
// AutoCleanHtaccess runs the hardened .htaccess cleaner against
// every finding emitted by the new detector registry, gated by
// AutoResponse.CleanHtaccess. Skipped when the daemon's auto-response
// pipeline is disabled overall.
//
// Unlike AutoQuarantineFiles, this routes around the
// quarantine/clean fork (.htaccess files are infrastructure -- moving
// them to /opt/csm/quarantine breaks the site). Each invocation
// backs up the original to /opt/csm/quarantine/pre_clean/<ts>_*
// inside CleanHtaccessFile before atomic-replacing.
// Marks evaluated input findings so alert delivery cannot repeat a response.
func AutoCleanHtaccess(cfg *config.Config, findings []alert.Finding) []alert.Finding {
if cfg == nil || !cfg.AutoResponse.Enabled || !cfg.AutoResponse.CleanHtaccess || cfg.ObserveMode() {
return nil
}
var actions []alert.Finding
seen := make(map[string]bool)
for i, f := range findings {
if f.AutoFileResponseEvaluated || !isHtaccessHardenedFinding(f.Check) {
continue
}
path := f.FilePath
if path == "" {
path = extractFilePath(f.Message)
}
if path == "" {
continue
}
// One Clean per file per autoresponse pass: multiple
// detector findings on the same file converge on a single
// cleaning call (CleanHtaccessFile re-runs every detector).
key := filepath.Clean(path)
if removable, ok := seen[key]; ok {
findings[i].AutoFileResponseEvaluated = removable
continue
}
seen[key] = false
info, err := osFS.Lstat(path)
if err != nil || info.Mode()&os.ModeSymlink != 0 {
continue
}
// Retained preludes and stale findings have no removal spans. They
// must not spend response capacity on a permanent cleaner refusal.
content, ok, err := readHtaccessBounded(path)
if err != nil || !ok {
continue
}
_, ranges := auditHtaccessPatterns(path, content)
if len(ranges) == 0 {
continue
}
seen[key] = true
findings[i].AutoFileResponseEvaluated = true
paused := runAutoFileResponse(cfg, path, info, func() error {
result := cleanHtaccessFileIdentified(path, info)
if result.Success {
actions = append(actions, alert.Finding{
Severity: alert.Critical,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-CLEAN: %s hardened directives removed", path),
Details: result.Description,
Timestamp: time.Now(),
})
} else if result.Error != "" && !result.Refused {
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-CLEAN failed: %s", path),
Details: result.Error,
Timestamp: time.Now(),
})
}
if result.Error != "" && !result.Refused {
return errors.New(result.Error)
}
return nil
})
if paused != nil {
actions = append(actions, *paused)
}
}
return actions
}
func isHtaccessHardenedFinding(check string) bool {
if check == "htaccess_handler_abuse" {
return true
}
for _, detector := range htaccessDetectors {
if check == detector.Name {
return true
}
}
return false
}
package checks
import (
"fmt"
"net"
"sync/atomic"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
)
type asnLookupFunc func(ip string) (asn uint, org string)
type asnLookupHolder struct {
fn asnLookupFunc
}
// asnLookup resolves an IP to its autonomous system number and organization
// via the GeoLite2-ASN database. The daemon injects it at startup/reload;
// nil (no ASN database, or unit tests that do not exercise the live path)
// disables bad-ASN classification on the outbound connection scan.
var asnLookup atomic.Pointer[asnLookupHolder]
// SetASNLookup wires the GeoLite2-ASN resolver used by the outbound
// connection scan. Passing nil clears it.
func SetASNLookup(fn func(ip string) (asn uint, org string)) {
if fn == nil {
asnLookup.Store(nil)
return
}
asnLookup.Store(&asnLookupHolder{fn: fn})
}
// CurrentASNLookup returns the wired ASN resolver, or nil when none is set.
// Both the polling connection scan and the live BPF connection evaluator
// use it so bad-ASN classification behaves identically on either path.
func CurrentASNLookup() func(ip string) (asn uint, org string) {
h := asnLookup.Load()
if h == nil {
return nil
}
return h.fn
}
// EvaluateBadASNOutbound classifies one outbound connection's destination by
// autonomous system and returns a finding when the ASN is bad. It is a pure
// function: the caller supplies the destination IP and the ASN/org already
// resolved from the GeoLite2-ASN database, so the classifier has no IO and
// is the third leg of the host-takeover chain correlator.
//
// Classification:
// - blocked_asns always flags (e.g. known bulletproof hosters);
// - when allowed_asns is non-empty, any ASN outside it flags (allowlist
// mode for hosts whose legitimate egress is confined to a few providers).
//
// An ASN of 0 (no AS found for the IP) is skipped: classifying it would flag
// every destination missing from the ASN database. Private, loopback,
// link-local, and unspecified destinations are skipped because ASN lookup is
// meaningless for them.
func EvaluateBadASNOutbound(cfg *config.Config, dstIP net.IP, asn uint, asOrg string) (alert.Finding, bool) {
if cfg == nil || !cfg.Detection.BadASNOutbound.Enabled {
return alert.Finding{}, false
}
if dstIP == nil || dstIP.IsLoopback() || dstIP.IsUnspecified() ||
dstIP.IsPrivate() || dstIP.IsLinkLocalUnicast() || dstIP.IsLinkLocalMulticast() {
return alert.Finding{}, false
}
if asn == 0 {
return alert.Finding{}, false
}
if !asnIsBad(cfg, asn) {
return alert.Finding{}, false
}
org := asOrg
if org == "" {
org = "unknown organization"
}
dst := dstIP.String()
if dstIP.To4() == nil {
dst = "[" + dst + "]"
}
return alert.Finding{
Severity: alert.High,
Check: "bad_asn_outbound",
Message: fmt.Sprintf("Outbound connection to bad ASN %d (%s): %s", asn, org, dst),
Details: fmt.Sprintf("Destination: %s\nASN: %d (%s)\n"+
"Combined with a new uid-0 account or a planted suid binary this escalates to a host takeover.",
dst, asn, org),
SourceIP: dstIP.String(),
}, true
}
// asnIsBad applies the blocklist-then-allowlist policy to a single ASN.
func asnIsBad(cfg *config.Config, asn uint) bool {
for _, b := range cfg.Detection.BadASNOutbound.BlockedASNs {
if b == asn {
return true
}
}
allowed := cfg.Detection.BadASNOutbound.AllowedASNs
if len(allowed) == 0 {
return false
}
for _, a := range allowed {
if a == asn {
return false
}
}
return true
}
package checks
import (
"errors"
"sync"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/metrics"
)
// Operator block sources, alongside the auto-response sources in
// applyblock.go. Operator paths bypass the chokepoint (they force-block and
// audit separately) but report into the same outcome metric.
const (
BlockSourceCLI = "cli"
BlockSourceWebUI = "web_ui"
blockSourceUnknown = "unknown"
blockOutcomeError = "error"
blockOutcomeProtected = "protected"
)
var (
blockOutcomeMetric *metrics.CounterVec
blockOutcomeMetricOnce sync.Once
blockOutcomeLabels = [...]string{
string(firewall.BlockOutcomeLive),
string(firewall.BlockOutcomeDryRun),
string(firewall.BlockOutcomeAllowed),
string(firewall.BlockOutcomeAllowlisted),
string(firewall.BlockOutcomeNoop),
blockOutcomeProtected,
blockOutcomeError,
}
blockSourceLabels = [...]string{
BlockSourceScan,
BlockSourceChallenge,
BlockSourceIncident,
BlockSourceCentral,
BlockSourceCLI,
BlockSourceWebUI,
blockSourceUnknown,
}
)
func init() {
// An aliveness alert needs zero-value series before the first attempt;
// otherwise Prometheus treats a dead block path as missing data.
blockOutcomeCounter()
}
// blockOutcomeCounter registers the outcome metric once and creates every
// closed label combination so zero-attempt paths remain visible to scrapes.
func blockOutcomeCounter() *metrics.CounterVec {
blockOutcomeMetricOnce.Do(func() {
blockOutcomeMetric = metrics.NewCounterVec(
"csm_firewall_block_outcome_total",
"Firewall IP block attempts by outcome (live, dry_run, allowed, allowlisted, noop, protected, error) and source (scan, challenge, incident, central_intel, cli, web_ui, unknown).",
[]string{"outcome", "source"},
)
// CounterVec reserves one child for overflow, so allow the complete
// closed set plus that defensive sentinel and no further growth.
blockOutcomeMetric.SetMaxChildren(len(blockOutcomeLabels)*len(blockSourceLabels) + 1)
for _, outcome := range blockOutcomeLabels {
for _, source := range blockSourceLabels {
blockOutcomeMetric.With(outcome, source)
}
}
metrics.MustRegister("csm_firewall_block_outcome_total", blockOutcomeMetric)
})
return blockOutcomeMetric
}
// observeBlockOutcome counts one block attempt. Protected-IP refusals get
// their own label so expected no-ops do not read as failures on dashboards;
// every other error is outcome=error.
func observeBlockOutcome(outcome firewall.BlockOutcome, err error, source string) {
blockOutcomeCounter().With(blockOutcomeLabel(outcome, err), blockSourceLabel(source)).Inc()
}
func blockOutcomeLabel(outcome firewall.BlockOutcome, err error) string {
if errors.Is(err, firewall.ErrIPProtected) {
return blockOutcomeProtected
}
if err != nil {
return blockOutcomeError
}
switch outcome {
case firewall.BlockOutcomeLive,
firewall.BlockOutcomeDryRun,
firewall.BlockOutcomeAllowed,
firewall.BlockOutcomeAllowlisted,
firewall.BlockOutcomeNoop:
return string(outcome)
default:
// An implementation that returns an undocumented outcome violated the
// engine contract; report it as an error without creating a new label.
return blockOutcomeError
}
}
func blockSourceLabel(source string) string {
switch source {
case BlockSourceScan,
BlockSourceChallenge,
BlockSourceIncident,
BlockSourceCentral,
BlockSourceCLI,
BlockSourceWebUI:
return source
default:
return blockSourceUnknown
}
}
// ObserveOperatorBlock reports an operator-initiated force block (CLI or
// web UI) into the outcome metric. Force blocks bypass the dry-run gate, so
// a nil error means the block landed live.
func ObserveOperatorBlock(err error, source string) {
outcome := firewall.BlockOutcomeLive
observeBlockOutcome(outcome, err, source)
}
package checks
import (
"context"
"fmt"
"net"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
)
// domlogDiscoveryDropped counts per-vhost log paths the discovery helper
// dropped silently (broken symlink, Stat failure). Same family of
// hidden-input bug as the lex-order issue scanDomlogs already fixed:
// without telemetry, operators have no way to notice when discovery
// loses a sizable fraction of the vhosts they expected to scan.
var (
domlogDiscoveryDropped *metrics.CounterVec
domlogDiscoveryDroppedOnce sync.Once
)
func observeDomlogDrop(reason string) {
domlogDiscoveryDroppedOnce.Do(func() {
domlogDiscoveryDropped = metrics.NewCounterVec(
"csm_checks_domlog_discovery_dropped_total",
"Per-vhost access-log paths the WP brute-force domlog discovery helper dropped before scanning. Labels: reason (evalsymlinks_error|stat_error). Steady growth means a chunk of vhosts is being silently skipped each cycle -- usually a broken symlink farm or a permissions regression on the log directory. Stale-mtime drops are intentional filtering, not counted here.",
[]string{"reason"},
)
metrics.MustRegister("csm_checks_domlog_discovery_dropped_total", domlogDiscoveryDropped)
})
domlogDiscoveryDropped.With(reason).Inc()
}
const (
wpLoginThreshold = 20 // attempts per IP across all logs
// xmlrpcThreshold is the fallback used only when emitLegacy is called with a
// nil config (tests). The live default comes from config
// (thresholds.xmlrpc_threshold, default DefaultXMLRPCThreshold); keep this in
// sync with that default.
xmlrpcThreshold = 100
ftpFailThreshold = 10
webmailThreshold = 10
apiFailThreshold = 10
// domlogTailLines is the built-in default for how many trailing lines
// to read from each domlog. Operators can override via
// cfg.Thresholds.DomlogTailLines. 500 covers ~10 minutes of traffic
// on a busy site.
domlogTailLines = 500
// domlogMaxAge skips domlogs not modified recently (inactive sites).
domlogMaxAge = 30 * time.Minute
// domlogMaxFiles caps the number of domlogs scanned per cycle
// to prevent unbounded I/O on servers with thousands of domains.
domlogMaxFiles = 500
)
// CheckWPBruteForce detects brute force attacks against wp-login.php and
// xmlrpc.php by scanning access logs. Always scans per-domain domlogs
// because on LiteSpeed+cPanel, virtual host traffic only appears there.
// The central access log is scanned as a supplement.
//
// Aggregates per-IP counts across ALL domains -- catches attackers who
// distribute requests across many sites to stay under per-site thresholds.
func CheckWPBruteForce(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
window := cfg.Thresholds.BruteForceWindow
if window <= 0 {
window = 5000
}
stats := newDomlogStats()
stats.callers = loadWebshellCallers(store)
hostInfo := platform.Detect()
var centralPath string
var centralRecords []syslogRecord
for _, p := range hostInfo.AccessLogPaths {
if records := tailFileRecords(p, window); len(records) > 0 {
centralPath, centralRecords = p, records
break
}
}
// 1. Per-domain domlogs -- primary source on LiteSpeed.
// Glob both SSL and non-SSL logs: attackers may use HTTP.
scanned := scanDomlogsStats(ctx, cfg, stats, centralPath)
// 2. Central access log -- supplement for non-vhost traffic.
// On LiteSpeed this mostly has WHM/server-level requests.
// On Apache it duplicates domlog data; minor double-counting is
// acceptable since thresholds are high enough.
central := hostInfo.IsCPanelCentralAccessLog(centralPath)
for _, r := range centralRecords {
rec, ok := parseAccessLogRecord(r.text)
if !ok {
continue
}
rec.ref = r
rec.Central = central
stats.scan(rec, cfg, currentBotClassifier(cfg))
}
findings := stats.emit(cfg)
// Replace the generic legacy Details with the actual scanned-file count.
for i := range findings {
if findings[i].Details == "Aggregated across per-vhost access logs" {
findings[i].Details = "Aggregated across " + itoa(scanned) + " per-vhost access logs"
}
}
return findings
}
// knownCentralAccessLogPaths lists central web-server log paths that may
// appear in broad per-vhost glob patterns. CheckWPBruteForce tails these on
// its own pass, so per-vhost discovery filters them to avoid counting the
// same lines twice.
var knownCentralAccessLogPaths = []string{
"/var/log/apache2/access.log",
"/var/log/apache2/access_log",
"/var/log/httpd/access.log",
"/var/log/httpd/access_log",
"/var/log/nginx/access.log",
"/usr/local/lsws/logs/access.log",
}
// discoverFreshDomlogs returns per-vhost access-log paths ready to tail.
// It globs platform.DomlogGlobs, dedupes by resolved-symlink real path,
// excludes the well-known central logs (so they are not double-counted),
// drops files untouched in the last maxAge, ranks survivors
// most-recent-first, and caps the result at maxFiles.
//
// Mtime-desc + cap is the fairness invariant: lexical glob order plus a
// hard cap would systematically hide brute force on late-alphabet
// domains. maxFiles <= 0 falls back to the built-in domlogMaxFiles
// default; maxAge <= 0 falls back to the built-in domlogMaxAge default.
// A canceled ctx returns nil and stops before any further work.
//
// Shared by scanDomlogs and scanDomlogsStats so the discovery semantics
// stay locked together; each caller layers its own per-line aggregator
// on top.
func discoverFreshDomlogs(ctx context.Context, maxFiles int, maxAge time.Duration) []string {
return discoverFreshDomlogsWithErrors(ctx, maxFiles, maxAge, nil)
}
// Stateful consumers must preserve findings when discovery could not enumerate
// their logs. Best-effort traffic summaries can leave onError nil.
func discoverFreshDomlogsWithErrors(ctx context.Context, maxFiles int, maxAge time.Duration, onError func(error)) []string {
return discoverFreshDomlogsExcluding(ctx, maxFiles, maxAge, onError, "")
}
func discoverFreshDomlogsExcluding(ctx context.Context, maxFiles int, maxAge time.Duration, onError func(error), centralPath string) []string {
if maxFiles <= 0 {
maxFiles = domlogMaxFiles
}
if maxAge <= 0 {
maxAge = domlogMaxAge
}
if ctx == nil {
ctx = context.Background()
}
if err := ctx.Err(); err != nil {
return nil
}
platformInfo := platform.Detect()
globs := platformInfo.DomlogGlobs
centralLogs := centralAccessLogSet(platformInfo.AccessLogPaths)
// Only the selected candidate is scanned centrally. Exclude its aliases,
// while keeping unselected candidates eligible for per-vhost discovery.
addCentralAccessLog(centralLogs, centralPath)
var domlogs []string
for _, pattern := range globs {
if err := ctx.Err(); err != nil {
return nil
}
matches, err := osFS.Glob(pattern)
if err != nil && onError != nil {
onError(err)
}
domlogs = append(domlogs, matches...)
}
type domlogEntry struct {
path string
mtime time.Time
}
fresh := make([]domlogEntry, 0, len(domlogs))
seen := make(map[string]bool)
cutoff := time.Now().Add(-maxAge)
for _, dl := range domlogs {
if err := ctx.Err(); err != nil {
return nil
}
// Resolve symlinks first -- cPanel symlinks SSL and non-SSL
// logs to the same backing file; dedupe on the real path.
real, err := filepath.EvalSymlinks(dl)
if err != nil {
if onError != nil {
onError(err)
}
observeDomlogDrop("evalsymlinks_error")
continue
}
if seen[real] || centralLogs[real] {
continue
}
seen[real] = true
// Inactive sites add no signal; filter before the cap so they
// cannot crowd active sites out of the budget.
info, err := osFS.Stat(real)
if err != nil {
if onError != nil {
onError(err)
}
observeDomlogDrop("stat_error")
continue
}
if info.ModTime().Before(cutoff) {
continue
}
fresh = append(fresh, domlogEntry{path: real, mtime: info.ModTime()})
}
if err := ctx.Err(); err != nil {
return nil
}
sort.Slice(fresh, func(i, j int) bool {
if fresh[i].mtime.Equal(fresh[j].mtime) {
return fresh[i].path < fresh[j].path
}
return fresh[i].mtime.After(fresh[j].mtime)
})
if len(fresh) > maxFiles {
fresh = fresh[:maxFiles]
}
out := make([]string, len(fresh))
for i, e := range fresh {
out[i] = e.path
}
return out
}
func centralAccessLogSet(configured []string) map[string]bool {
out := make(map[string]bool, len(knownCentralAccessLogPaths)+len(configured))
for _, p := range knownCentralAccessLogPaths {
addCentralAccessLog(out, p)
}
// Multiple AccessLogPaths are fallback candidates; CheckWPBruteForce
// tails only the first one with data, so excluding every candidate here
// would drop later logs that were never scanned centrally.
if len(configured) == 1 {
addCentralAccessLog(out, configured[0])
}
return out
}
func addCentralAccessLog(out map[string]bool, path string) {
if path == "" {
return
}
out[path] = true
if real, err := filepath.EvalSymlinks(path); err == nil {
out[real] = true
}
}
// effectiveDomlogTailLines returns the operator-configured
// thresholds.domlog_tail_lines value or the built-in default when unset.
func effectiveDomlogTailLines(cfg *config.Config) int {
if cfg == nil || cfg.Thresholds.DomlogTailLines <= 0 {
return domlogTailLines
}
return cfg.Thresholds.DomlogTailLines
}
// effectiveDomlogMaxAge returns the operator-configured
// thresholds.domlog_max_age_min as a Duration, or the built-in default
// when unset.
func effectiveDomlogMaxAge(cfg *config.Config) time.Duration {
if cfg == nil || cfg.Thresholds.DomlogMaxAgeMin <= 0 {
return domlogMaxAge
}
return time.Duration(cfg.Thresholds.DomlogMaxAgeMin) * time.Minute
}
// tailDomlogsInto tails each discovered path and feeds every parsed
// access-log record into stats. Returns the number of files actually
// tailed (loop exits early if ctx is cancelled mid-pass).
//
// Single tail-and-aggregate loop shared by every domlog scanner so the
// per-file ctx gate, the parse-or-skip behaviour, and the scanned
// counter cannot drift between callers.
func tailDomlogsInto(ctx context.Context, paths []string, cfg *config.Config, stats *domlogStats, classifier botClassifier, tailLines int) int {
parse := parseAccessLogRecord
if stats.callers != nil {
parse = parseWebshellCallerRecord
}
scanned := 0
for _, p := range paths {
if ctx != nil {
if err := ctx.Err(); err != nil {
break
}
}
domain := domainFromDomlogPath(p)
account := domainAccountOwner(domain)
for _, r := range tailFileRecords(p, tailLines) {
rec, ok := parse(r.text)
if !ok {
continue
}
rec.ref = r
rec.Domain = domain
rec.Account = account
stats.scan(rec, cfg, classifier)
}
scanned++
}
return scanned
}
// domainFromDomlogPath derives the vhost from a per-domain domlog file
// path. Returns "" for paths that do not look like a domain log so the
// central access log and odd filenames do not pollute the per-IP vhost set.
func domainFromDomlogPath(p string) string {
base := filepath.Base(p)
if domain, ok := pleskDomlogDomain(p, base); ok {
return cleanDomlogDomain(domain)
}
if domain, ok := trimDomlogSuffix(base); ok {
return cleanDomlogDomain(domain)
}
return cleanDomlogDomain(base)
}
func pleskDomlogDomain(p, base string) (string, bool) {
switch strings.ToLower(strings.TrimSpace(base)) {
case "access_log", "access_ssl_log", "proxy_access_ssl_log":
return filepath.Base(filepath.Dir(filepath.Dir(p))), true
default:
return "", false
}
}
func trimDomlogSuffix(base string) (string, bool) {
trimmed := strings.TrimSpace(base)
low := strings.ToLower(trimmed)
for _, suffix := range []string{
".access.log",
"-access.log",
"_access.log",
"-access_log",
"_access_log",
"-ssl_log",
"_log",
".log",
} {
if strings.HasSuffix(low, suffix) {
return trimmed[:len(trimmed)-len(suffix)], true
}
}
return trimmed, false
}
func cleanDomlogDomain(domain string) string {
domain = strings.ToLower(strings.TrimSpace(domain))
if domain == "" || len(domain) > 253 || !strings.Contains(domain, ".") {
return ""
}
if strings.HasPrefix(domain, ".") || strings.HasSuffix(domain, ".") ||
strings.Contains(domain, "..") {
return ""
}
if net.ParseIP(domain) != nil {
return ""
}
labels := strings.Split(domain, ".")
for _, label := range labels {
if label == "" || len(label) > 63 ||
strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") {
return ""
}
for _, c := range label {
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' {
continue
}
return ""
}
}
return domain
}
// scanDomlogsStats discovers per-vhost logs honouring the operator's
// thresholds and feeds each parsed record into stats. Returns the number
// of files actually tailed.
func scanDomlogsStats(ctx context.Context, cfg *config.Config, stats *domlogStats, centralPath string) int {
if cfg == nil {
cfg = &config.Config{}
}
paths := discoverFreshDomlogsExcluding(ctx, cfg.Thresholds.DomlogMaxFiles, effectiveDomlogMaxAge(cfg), nil, centralPath)
return tailDomlogsInto(ctx, paths, cfg, stats, currentBotClassifier(cfg), effectiveDomlogTailLines(cfg))
}
// scanDomlogs is the legacy infra-IPs-only entry kept for test fixtures
// that drive the brute-force counters directly. Production code calls
// scanDomlogsStats. Both share discoverFreshDomlogs + tailDomlogsInto so
// path selection and per-file ctx semantics cannot diverge.
//
// maxFiles <= 0 falls back to the built-in domlogMaxFiles default.
func scanDomlogs(ctx context.Context, infraIPs []string, maxFiles int, wpLogin, xmlrpc, userEnum map[string]int) int {
cfg := &config.Config{InfraIPs: infraIPs}
stats := newDomlogStats()
stats.wpLogin = wpLogin
stats.xmlrpc = xmlrpc
stats.userEnum = userEnum
paths := discoverFreshDomlogs(ctx, maxFiles, 0)
return tailDomlogsInto(ctx, paths, cfg, stats, nopBotClassifier{}, domlogTailLines)
}
// countBruteForce parses Combined Log Format lines and increments per-IP
// counters via the shared domlogStats aggregator. Kept as a thin shim
// for tests that feed lines directly (no file discovery / tail step).
func countBruteForce(lines []string, infraIPs []string, wpLogin, xmlrpc, userEnum map[string]int) {
cfg := &config.Config{InfraIPs: infraIPs}
stats := newDomlogStats()
stats.wpLogin = wpLogin
stats.xmlrpc = xmlrpc
stats.userEnum = userEnum
for _, line := range lines {
rec, ok := parseAccessLogRecord(line)
if !ok {
continue
}
stats.scan(rec, cfg, nopBotClassifier{})
}
}
// syslogMessagesTailLinesDefault is the built-in fallback for the legacy
// direct CheckFTPLogins path used when no state store is available.
// Operator override: cfg.Thresholds.SyslogMessagesTailLines.
const syslogMessagesTailLinesDefault = 200
// CheckFTPLogins detects pure-ftpd brute force. With a state store it reads
// /var/log/messages forward-only and accumulates per-IP failures over a sliding
// window; without a store it falls back to the legacy per-cycle tail.
func CheckFTPLogins(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
if cfg == nil {
cfg = &config.Config{}
}
if store == nil {
return checkFTPLoginsLegacy(cfg)
}
ftpTrackerMu.Lock()
defer ftpTrackerMu.Unlock()
if ctx != nil && ctx.Err() != nil {
return nil
}
now := time.Now()
initialRaw, _ := store.GetRaw(ftpTrackerKey)
tracker := loadFTPFailTracker(store)
records, next, skipped, err := readNewSyslogRecords(ftpSyslogPath, tracker.Follow)
if err != nil {
return nil // leave stored state untouched
}
if skipped > 0 {
observeFTPSkippedBytes(skipped)
}
windowMin := effectiveFTPFailWindowMin(cfg)
tracker.evict(now, windowMin)
cutoff := now.Add(-time.Duration(windowMin) * time.Minute)
var findings []alert.Finding
// An address is reported when this pass read a new failure for it. The
// window still sums earlier failures, but a burst already reported is not
// reported again on every scan while it ages out: the repeat would count
// it again in the attack database and re-block an address the operator
// unblocked.
freshFailures := make(map[string]alert.Observation)
for _, rec := range records {
line := rec.text
if !isPureFTPDLogFields(strings.Fields(line)) {
continue
}
ip, account := ftpLoginFields(line)
if account == "" {
ip = PureFTPDClientIP(line)
}
if ignoredFTPClientIP(ip, cfg.InfraIPs) {
continue
}
switch {
case strings.Contains(line, "Authentication failed"), strings.Contains(line, "auth failed"):
// Count the failure when the log recorded it, not when it was
// read: a first run or a large gap catches up on hours or days
// of history, and stamping that with now would turn scattered
// failures into one burst and auto-block the address.
at, ok := syslogLineTime(line, now)
if !ok || at.After(now) {
// A missing or future timestamp cannot safely seed a future
// minute bucket that survives eviction indefinitely. Treat the
// record as current, matching the timestamp-free fallback.
at = now
}
if at.Before(cutoff) {
continue
}
tracker.record(ip, at)
// The newest new failure is the observation the report names.
freshFailures[ip] = rec.observation(ProducerFTPScan, at)
case account != "":
findings = append(findings, ftpLoginFinding(ip, account, line, tracker.count(ip)))
}
}
tracker.capIPs(maxTrackedIPs)
for _, off := range tracker.offenders(ftpFailThreshold) {
observation, fresh := freshFailures[off.IP]
if !fresh {
continue
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "ftp_bruteforce",
SourceIP: off.IP,
Message: fmt.Sprintf("FTP brute force from %s: %d failed attempts in %dm", off.IP, off.Count, windowMin),
Observation: observation,
Attribution: admission.AttributionPeer,
})
}
// Diagnostic scans suppress auto-response, so leave their evidence for
// the next live scan rather than consuming its only reporting opportunity.
// The runner also discards results from cancelled checks.
if ctx != nil && ctx.Err() != nil {
return nil
}
dryRun := false
if ctx != nil {
dryRun, _ = ctx.Value(scanDryRunKey{}).(bool)
}
if !dryRun {
tracker.Follow = next
if ctx != nil {
if observation, _ := ctx.Value(ftpScanObservationKey{}).(*ftpScanObservation); observation != nil {
observation.prepare(initialRaw, tracker, findings)
return nil
}
}
tracker.save(store)
}
return findings
}
// checkFTPLoginsLegacy is the pre-store tail-based detector, used only when no
// state store is available (direct callers / tests). The daemon always supplies
// a store and uses the forward-only store-backed path above.
func checkFTPLoginsLegacy(cfg *config.Config) []alert.Finding {
var findings []alert.Finding
tailLines := syslogMessagesTailLinesDefault
if cfg != nil && cfg.Thresholds.SyslogMessagesTailLines > 0 {
tailLines = cfg.Thresholds.SyslogMessagesTailLines
}
lines := tailFile("/var/log/messages", tailLines)
if len(lines) == 0 {
return nil
}
failedFTP := make(map[string]int)
for _, line := range lines {
fields := strings.Fields(line)
// pure-ftpd logs: "pure-ftpd: ... [WARNING] Authentication failed for user"
if !isPureFTPDLogFields(fields) {
continue
}
ip, account := ftpLoginFields(line)
if account == "" {
ip = PureFTPDClientIP(line)
}
if ignoredFTPClientIP(ip, cfg.InfraIPs) {
continue
}
switch {
case strings.Contains(line, "Authentication failed"), strings.Contains(line, "auth failed"):
failedFTP[ip]++
case account != "":
// failedFTP holds failures seen earlier in this batch; pure-ftpd logs
// chronologically so a brute-then-success appears as fails before the
// "logged in" line, letting the shared builder escalate.
findings = append(findings, ftpLoginFinding(ip, account, line, failedFTP[ip]))
}
}
for ip, count := range failedFTP {
if count >= ftpFailThreshold {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "ftp_bruteforce",
SourceIP: ip,
Message: fmt.Sprintf("FTP brute force from %s: %d failed attempts", ip, count),
})
}
}
return findings
}
// CheckWebmailLogins parses cPanel access log for webmail logins from non-infra IPs.
func CheckWebmailLogins(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if cfg.Suppressions.SuppressWebmail {
return nil
}
// Webmail ports and this log are cPanel-specific; on other panels the
// path does not exist and the check would silently return nothing.
if !platform.Detect().IsCPanel() {
return nil
}
var findings []alert.Finding
records := tailFileRecords("/usr/local/cpanel/logs/access_log", 300)
loginAttempts := make(map[string]int)
newest := make(map[string]scanRecord)
now := time.Now()
for _, rec := range records {
// Count login attempts per IP on the webmail ports.
access, ok := ParseCpanelAccessRecord(rec.text)
if !ok || !access.WebmailLogin() {
continue
}
ip := access.IP
if !access.Proxy && (isInfraIP(ip, cfg.InfraIPs) || ip == "127.0.0.1") {
continue
}
key := ip
if access.Proxy {
key = "proxy:" + ip
}
loginAttempts[key]++
keepNewest(newest, key, scanned(rec, access.At), now)
}
for key, count := range loginAttempts {
if count >= webmailThreshold {
ip, proxied := strings.CutPrefix(key, "proxy:")
sourceIP, source, details, dedup := ip, ip, "", ""
if proxied {
sourceIP = ""
source = "a service subdomain (client address unverified)"
details = "Unverified logged address: " + ip
dedup = "proxy:" + ip
}
f := alert.Finding{
Severity: alert.High,
Check: "webmail_bruteforce",
Message: fmt.Sprintf("Webmail brute force from %s: %d attempts", source, count),
SourceIP: sourceIP,
Details: details,
DedupKey: dedup,
}
stampCpanelScan(&f, newest[key], cfg, proxied)
findings = append(findings, f)
}
}
return findings
}
// CheckAPIAuthFailures parses cPanel access log for failed API authentication.
func CheckAPIAuthFailures(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
// The cPanel/WHM API access log is cPanel-specific.
if !platform.Detect().IsCPanel() {
return nil
}
var findings []alert.Finding
records := tailFileRecords("/usr/local/cpanel/logs/access_log", 300)
evidence := StaleSessionEvidence{Denials: recentSessionTokenDenials()}
if len(evidence.Denials) > 0 {
for _, rec := range records {
if r, ok := ParseStaleSessionRequest(rec.text); ok && r.User != "-" {
evidence.Rejected = append(evidence.Rejected, r)
}
}
}
failedAPI := make(map[string]int)
newest := make(map[string]scanRecord)
now := time.Now()
for _, rec := range records {
access, ok := ParseCpanelAccessRecord(rec.text)
if !ok || !access.APIAuthFailure() {
continue
}
ip := access.IP
if !access.Proxy && (isInfraIP(ip, cfg.InfraIPs) || ip == "127.0.0.1") {
continue
}
if r, ok := ParseStaleSessionRequest(rec.text); ok && evidence.Explains(r) {
continue
}
key := ip
if access.Proxy {
key = "proxy:" + ip
}
failedAPI[key]++
keepNewest(newest, key, scanned(rec, access.At), now)
}
for key, count := range failedAPI {
if count >= apiFailThreshold {
ip, proxied := strings.CutPrefix(key, "proxy:")
sourceIP, source, dedup := ip, ip, ""
details := "Possible API token brute force or unauthorized API access"
if proxied {
sourceIP = ""
source = "a service subdomain (client address unverified)"
details = "Unverified logged address: " + ip + "\n" + details
dedup = "proxy:" + ip
}
f := alert.Finding{
Severity: alert.High,
Check: "api_auth_failure",
Message: fmt.Sprintf("cPanel API auth failures from %s: %d attempts", source, count),
SourceIP: sourceIP,
Details: details,
DedupKey: dedup,
}
stampCpanelScan(&f, newest[key], cfg, proxied)
findings = append(findings, f)
}
}
return findings
}
// A service-subdomain address lacks peer provenance even when the scan
// retained an observation timestamp. Its counts stay separate from peers.
func stampCpanelScan(f *alert.Finding, r scanRecord, cfg *config.Config, proxied bool) {
if !proxied && !IsTrustedProxy(f.SourceIP, cfg) {
f.Attribution = admission.AttributionPeer
}
r.stamp(f, ProducerCpanelScan)
}
// FTPLoginFinding builds the finding for a successful pure-ftpd login line,
// reporting false when the line is not a login or the client address is
// loopback, infrastructure, or unusable. The daemon's realtime log watcher
// calls it so a login seen live and the same line re-read by CheckFTPLogins
// carry one identity; without that the state store sees two findings and the
// operator gets one login reported twice. It has no brute-force history, so
// the escalation to ftp_login_after_bruteforce stays with the scheduled check.
func FTPLoginFinding(line string, cfg *config.Config) (alert.Finding, bool) {
if cfg == nil {
cfg = &config.Config{}
}
ip, account := ftpLoginFields(line)
if account == "" {
return alert.Finding{}, false
}
if ignoredFTPClientIP(ip, cfg.InfraIPs) {
return alert.Finding{}, false
}
return ftpLoginFinding(ip, account, line, 0), true
}
// ftpLoginFinding builds the finding for a successful FTP login from a
// non-infra, non-loopback IP. A login from a source that has already crossed
// the brute-force threshold is a likely cracked credential and pages as
// Critical; any other login is audit-level (Warning), matching the cPanel-login
// detector ("logins are audit trail, not paging-level").
// account is the user token ftpLoginFields validated.
func ftpLoginFinding(ip, account, line string, recentFails int) alert.Finding {
now := time.Now()
at, _ := syslogLineTime(line, now)
if at.After(now) {
at = now
}
owner := ""
if account != "" {
owner = ftpAccountOwner(account)
}
details := truncate(line, ftpLoginDetailsBytes)
loginAccount := strings.Clone(truncate(account, ftpLoginDetailsBytes))
if recentFails >= ftpFailThreshold {
msg := fmt.Sprintf("FTP login succeeded from brute-force source %s after %d failed attempts", ip, recentFails)
if account != "" {
msg = fmt.Sprintf("FTP login succeeded for account %s from brute-force source %s after %d failed attempts", account, ip, recentFails)
}
return alert.Finding{
Severity: alert.Critical,
Check: "ftp_login_after_bruteforce",
DedupKey: loginRecordKey(line),
SourceIP: ip,
Message: msg,
Details: details,
TenantID: owner,
LoginAccount: loginAccount,
Timestamp: at,
}
}
return alert.Finding{
Severity: alert.Warning,
Check: "ftp_login",
DedupKey: loginRecordKey(line),
SourceIP: ip,
Message: fmt.Sprintf("FTP login from non-infra IP: %s", ip),
Details: details,
TenantID: owner,
LoginAccount: loginAccount,
Timestamp: at,
}
}
// ftpLoginDetailsBytes bounds the displayed record and login name. Customer
// tracking reads the authenticated name separately from the cut record.
const ftpLoginDetailsBytes = 512
// FTPLoginAccount returns the authenticated account only from a complete
// pure-ftpd success record. Findings keep the record, so customer tracking
// can fall back to this login name when no hosting account owns it.
func FTPLoginAccount(line string) string {
_, account := ftpLoginFields(line)
return account
}
func ftpLoginFields(line string) (ip, account string) {
msg := pureFTPDMessageFields(strings.Fields(line))
// Match server-owned structure before interpreting any client text.
if len(msg) != 7 || msg[1] != "[INFO]" || msg[3] != "is" || msg[4] != "now" || msg[5] != "logged" || msg[6] != "in" {
return "", ""
}
ip = ftpPeerIP(msg[0])
if ip == "" {
return "", ""
}
return ip, msg[2]
}
func ignoredFTPClientIP(ip string, infraIPs []string) bool {
if ip == "" || isInfraIP(ip, infraIPs) {
return true
}
parsed := net.ParseIP(ip)
return parsed == nil || parsed.IsLoopback()
}
func isPureFTPDLogFields(fields []string) bool {
return pureFTPDMessageFields(fields) != nil
}
// PureFTPDClientIP returns only the peer immediately after a validated
// pure-ftpd program tag. Missing, malformed and hostname peers must not
// fall through to client-controlled text later in the message.
func PureFTPDClientIP(line string) string {
msg := pureFTPDMessageFields(strings.Fields(line))
if len(msg) == 0 {
return ""
}
return ftpPeerIP(msg[0])
}
func ftpPeerIP(peer string) string {
if ip := ipFromParenPeer(peer); ip != "" {
return ip
}
// Retain the bare-address format only in the same server-owned slot.
if net.ParseIP(peer) != nil {
return peer
}
return ""
}
func pureFTPDMessageFields(fields []string) []string {
if len(fields) == 0 {
return nil
}
if isPureFTPDProgramToken(fields[0]) {
return fields[1:]
}
if len(fields) >= 5 && isSyslogTimestampPrefix(fields) && isPureFTPDProgramToken(fields[4]) {
return fields[5:]
}
if len(fields) >= 3 && isPureFTPDProgramToken(fields[2]) {
if _, err := time.Parse(time.RFC3339Nano, fields[0]); err == nil {
return fields[3:]
}
}
return nil
}
func isPureFTPDProgramToken(field string) bool {
if field == "pure-ftpd:" {
return true
}
if !strings.HasPrefix(field, "pure-ftpd[") || !strings.HasSuffix(field, "]:") {
return false
}
pid := field[len("pure-ftpd[") : len(field)-len("]:")]
if pid == "" {
return false
}
for _, r := range pid {
if r < '0' || r > '9' {
return false
}
}
return true
}
func isSyslogTimestampPrefix(fields []string) bool {
return isSyslogMonth(fields[0]) && isSyslogDay(fields[1]) && isSyslogClock(fields[2])
}
func isSyslogMonth(s string) bool {
switch s {
case "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec":
return true
default:
return false
}
}
func isSyslogDay(s string) bool {
if len(s) < 1 || len(s) > 2 {
return false
}
for _, r := range s {
if r < '0' || r > '9' {
return false
}
}
return true
}
func isSyslogClock(s string) bool {
if len(s) != len("00:00:00") || s[2] != ':' || s[5] != ':' {
return false
}
for i, r := range s {
if i == 2 || i == 5 {
continue
}
if r < '0' || r > '9' {
return false
}
}
return true
}
// ipFromParenPeer extracts an IP from pure-ftpd's "(user@ip)" / "(?@ip)" peer
// token, supporting both IPv4 and IPv6 addresses. Returns "" when f is not
// such a token or the candidate after the last '@' does not parse as an IP.
func ipFromParenPeer(f string) string {
if len(f) < 4 || f[0] != '(' || f[len(f)-1] != ')' {
return ""
}
inner := f[1 : len(f)-1]
at := strings.LastIndexByte(inner, '@')
if at < 0 || at+1 >= len(inner) {
return ""
}
cand := inner[at+1:]
if net.ParseIP(cand) == nil {
return ""
}
return cand
}
package checks
import (
"fmt"
"os"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/metrics"
)
var (
challengeRoutedMetric *metrics.CounterVec
challengeRoutedMetricOnce sync.Once
)
// observeChallengeRouted counts one IP routed to the proof-of-work challenge,
// labelled by the source check that flagged it, so operators can graph
// challenge volume per detector (e.g. http_scanner_profile). Registered lazily
// on first use, mirroring the auto-response metric in the runner.
func observeChallengeRouted(check string) {
challengeRoutedMetricOnce.Do(func() {
challengeRoutedMetric = metrics.NewCounterVec(
"csm_challenge_routed_total",
"IPs routed to the proof-of-work challenge, by the source check that flagged them.",
[]string{"check"},
)
metrics.MustRegister("csm_challenge_routed_total", challengeRoutedMetric)
})
challengeRoutedMetric.With(check).Inc()
}
// ChallengeIPList abstracts the challenge IP list for routing.
type ChallengeIPList interface {
Add(ip string, reason string, duration time.Duration)
AddNonEscalating(ip string, reason string, duration time.Duration)
Remove(ip string)
Contains(ip string) bool
}
var challengeIPList ChallengeIPList
// SetChallengeIPList sets the challenge IP list for routing.
func SetChallengeIPList(list ChallengeIPList) {
challengeIPList = list
}
// GetChallengeIPList returns the current challenge IP list (for AutoBlockIPs skip check).
func GetChallengeIPList() ChallengeIPList {
return challengeIPList
}
// challengePasses returns remembered passes and their retention cutoff,
// or nil and zero when the list keeps no passes.
func challengePasses() (map[string]time.Time, time.Time) {
if list, ok := challengeIPList.(interface {
Passes() (map[string]time.Time, time.Time)
}); ok {
return list.Passes()
}
return nil, time.Time{}
}
func isChallengeableCheck(check string) bool {
return ResponsePolicyFor(check).ChallengeFirst
}
// Auto-response actions a challengeable check can resolve to.
const (
responseChallenge = "challenge"
responseBlock = "block"
)
// responseActionForCheck returns the effective auto-response for a check:
// "challenge" to route the IP to the PoW gate, or "block" to hard-block it.
// Challengeable checks default to "challenge" only while challenge routing is
// enabled; otherwise they fall through to "block". An operator-selectable
// override (currently only http_scanner_profile via
// auto_response.http_scanner_action) forces "block". Non-challengeable checks
// always resolve to "block". This is the single source of truth for the
// challenge-vs-block decision, shared by ChallengeRouteIPs and AutoBlockIPs so
// the two cannot diverge.
func responseActionForCheck(cfg *config.Config, check string) string {
if !cfg.Challenge.Enabled || !challengeRoutesCheck(cfg, check) {
return responseBlock
}
return responseChallenge
}
// challengeRoutesCheck is the challenge-vs-block policy for a check with
// challenge routing assumed on.
func challengeRoutesCheck(cfg *config.Config, check string) bool {
if !isChallengeableCheck(check) {
return false
}
return check != "http_scanner_profile" || cfg.AutoResponse.HTTPScannerAction != responseBlock
}
// responseActionForFinding narrows responseActionForCheck for one finding.
// ip_reputation grades its sighting severity by detection vector
// (reputationSightingSeverity: HTTP and cPanel access are High, every
// other vector Critical), so a Critical reputation sighting came from a
// browserless channel (SMTP, IMAP, FTP, SSH) where nothing can ever
// answer the PoW page -- challenge-routing it just leaves the attacker
// unblocked, retrying daily. Those resolve to a hard block.
func responseActionForFinding(cfg *config.Config, f alert.Finding) string {
if !cfg.Challenge.Enabled || !challengeRoutesFinding(cfg, f) {
return responseBlock
}
return responseChallenge
}
// challengeRoutesFinding narrows challengeRoutesCheck for one finding, with
// challenge routing assumed on.
func challengeRoutesFinding(cfg *config.Config, f alert.Finding) bool {
if f.Check == "ip_reputation" && f.Severity == alert.Critical {
return false
}
return challengeRoutesCheck(cfg, f.Check)
}
// challengeSelected reports whether challenge routing answers f instead of
// a block: routing serves, and the policy routes the finding.
func challengeSelected(cfg *config.Config, f alert.Finding) bool {
return cfg.Challenge.Enabled && challengeIPList != nil && !isHardBlockCheck(f.Check) && responseActionForFinding(cfg, f) == responseChallenge
}
// isHardBlockCheck reports whether a check must never be routed to the
// challenge: its registry policy says so, or it is a runtime-built name the
// prefix contract covers.
func isHardBlockCheck(check string) bool {
return ResponsePolicyFor(check).NeverChallenge || neverChallengeDynamicName(check)
}
// ChallengeDuration is how long a routed address stays challenged.
const ChallengeDuration = 30 * time.Minute
// ChallengeThenBlock runs the two IP-disposition stages in their required
// order -- challenge routing first so an eligible IP is on the challenge list
// before AutoBlockIPs checks membership, then hard-blocking -- and returns both
// action sets. Auto-response call sites use this single helper instead of
// hand-ordering the two calls, so the "challenge before block" invariant cannot
// be silently broken by reordering in one path. Both stages run on the same
// finding set (the full/repeat-offender set); callers append the returned
// actions wherever their pipeline expects them.
//
// Both stages skip findings an earlier pass evaluated, and this pass marks
// every finding it was given, so a scan finding the runner evaluated is not
// acted on or handed to admission again when it is dispatched.
func ChallengeThenBlock(cfg *config.Config, findings []alert.Finding) (challengeActions, blockActions []alert.Finding) {
challengeActions = ChallengeRouteIPs(cfg, findings)
blockActions = AutoBlockIPs(cfg, findings)
for i := range findings {
findings[i].AutoIPResponseEvaluated = true
}
return challengeActions, blockActions
}
// ChallengeRouteIPs processes findings and routes eligible IPs to the challenge
// list instead of hard-blocking them. Must be called BEFORE AutoBlockIPs so
// that challenged IPs are on the list when AutoBlockIPs checks Contains().
func ChallengeRouteIPs(cfg *config.Config, findings []alert.Finding) []alert.Finding {
if !cfg.Challenge.Enabled || challengeIPList == nil {
return nil
}
var actions []alert.Finding
routed := make(map[string]bool)
// A challenge can end in a block, so skip what the firewall refuses to
// block: both infra lists, and from a running engine also resolved infra
// hostnames and the host's own addresses.
infra := config.EffectiveFirewallConfig(cfg).InfraIPs
protector, _ := getIPBlocker().(blockProtector)
// A direct finding anywhere in the batch keeps its address escalating,
// whichever of its findings is listed first.
escalates := make(map[string]bool)
for _, f := range findings {
if !f.ProxiedClient && isChallengeableCheck(f.Check) && f.Check != "http_claimed_bot_unverified" {
if ip := extractIPFromFinding(f); ip != "" {
escalates[ip] = true
}
}
}
for _, f := range findings {
if f.AutoIPResponseEvaluated || !CheckResponseEnabled(cfg, f.Check) {
continue
}
// Challenge timeouts can hard-block too, so gated authentication
// checks must honor the same opt-in as direct firewall responses.
if ResponsePolicyFor(f.Check).Block == BlockWithCpanelLogins && !cfg.AutoResponse.BlockCpanelLogins {
continue
}
if isHardBlockCheck(f.Check) {
continue
}
// Only route checks that are known to contain attacker IPs.
// This is an allowlist: a new IP-bearing check must be given a
// ChallengeFirst Response in the check registry. Defaulting to skip
// prevents version numbers, sizes, and other numeric finding fields
// from being blocked as IPs.
if !isChallengeableCheck(f.Check) {
continue
}
// The scanner-profile response is operator-selectable: "block"
// skips routing here so AutoBlockIPs hard-blocks the IP instead.
if responseActionForFinding(cfg, f) == responseBlock {
continue
}
ip := extractIPFromFinding(f)
if ip == "" {
continue
}
if isInfraIP(ip, infra) || ip == "127.0.0.1" || (protector != nil && protector.BlockProtected(ip)) || IsTrustedProxy(ip, cfg) {
continue
}
// Admission judges existing effects itself; the list's state is
// the legacy path's.
root, _, _ := respond(admission.KindChallenge, f, ip, 0)
if routed[ip] || challengeIPList.Contains(ip) {
continue
}
// A client only a trusted proxy forwarded sends no packets here and
// the challenge map matches the connecting address, so its challenge
// can only time out: that timeout is no reason to block it.
nonEscalating := f.Check == "http_claimed_bot_unverified" || f.ProxiedClient && !escalates[ip]
addChallengeIP(f.Check, ip, f.Message, ChallengeDuration, alert.FindingID(f), root, nonEscalating)
routed[ip] = true
observeChallengeRouted(f.Check)
recordChallengeRouteStat(ip, f.Check, time.Now())
fmt.Fprintf(os.Stderr, "[%s] CHALLENGE: %s routed to challenge (check: %s)\n",
time.Now().Format("2006-01-02 15:04:05"), ip, f.Check)
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "challenge_route",
Message: fmt.Sprintf("CHALLENGE: %s sent to PoW challenge (expires in %s)", ip, ChallengeDuration),
Details: fmt.Sprintf("Reason: %s", f.Message),
Timestamp: time.Now(),
})
}
return actions
}
func addChallengeIP(check, ip, reason string, duration time.Duration, findingID string, root admission.Evidence, nonEscalating bool) {
if nonEscalating {
challengeIPList.AddNonEscalating(ip, reason, duration)
return
}
if list, ok := challengeIPList.(interface {
AddWithRoot(string, string, time.Duration, string, string, admission.Evidence)
}); ok {
list.AddWithRoot(ip, reason, duration, findingID, check, root)
return
}
if list, ok := challengeIPList.(interface {
AddWithFindingID(string, string, time.Duration, string)
}); ok {
list.AddWithFindingID(ip, reason, duration, findingID)
return
}
challengeIPList.Add(ip, reason, duration)
}
func removeChallengeIP(ip string) {
if challengeIPList == nil {
return
}
challengeIPList.Remove(ip)
}
package checks
import (
"sync"
"time"
)
// challengeRecentMax bounds the recent-routes ring buffer surfaced to the web
// UI. Small: the panel shows the latest handful, history holds the rest.
const challengeRecentMax = 20
// ChallengeRouteRecord is one IP routed to the challenge, for the web UI's
// recent-activity list.
type ChallengeRouteRecord struct {
IP string `json:"ip"`
Check string `json:"check"`
At time.Time `json:"at"`
}
// ChallengeUIStatsSnapshot is the web-UI view of challenge routing: cumulative
// per-check counts since daemon start plus the most recent routes. It is a
// copy, safe for the caller to read without locking.
type ChallengeUIStatsSnapshot struct {
RoutedByCheck map[string]int `json:"routed_by_check"`
Recent []ChallengeRouteRecord `json:"recent"`
}
var (
challengeStatsMu sync.Mutex
challengeRoutedByCheck = map[string]int{}
challengeRecentRoutes []ChallengeRouteRecord
)
// recordChallengeRouteStat records one route for the web-UI stats. Kept
// separate from the Prometheus counter (observeChallengeRouted) so the UI does
// not depend on scraping /metrics.
func recordChallengeRouteStat(ip, check string, at time.Time) {
challengeStatsMu.Lock()
defer challengeStatsMu.Unlock()
challengeRoutedByCheck[check]++
challengeRecentRoutes = append(challengeRecentRoutes, ChallengeRouteRecord{IP: ip, Check: check, At: at})
if len(challengeRecentRoutes) > challengeRecentMax {
challengeRecentRoutes = challengeRecentRoutes[len(challengeRecentRoutes)-challengeRecentMax:]
}
}
// ChallengeUIStats returns a copy of the current challenge routing stats for
// the web UI. Most recent route is last in Recent.
func ChallengeUIStats() ChallengeUIStatsSnapshot {
challengeStatsMu.Lock()
defer challengeStatsMu.Unlock()
byCheck := make(map[string]int, len(challengeRoutedByCheck))
for k, v := range challengeRoutedByCheck {
byCheck[k] = v
}
recent := make([]ChallengeRouteRecord, len(challengeRecentRoutes))
copy(recent, challengeRecentRoutes)
return ChallengeUIStatsSnapshot{RoutedByCheck: byCheck, Recent: recent}
}
package checks
import (
"context"
"fmt"
"maps"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/state"
)
// incompleteCheckCollector records owners whose coverage this run could not
// complete. Known file gaps can be preserved in the eventual store transaction;
// an unknown range prevents the owner from retiring anything.
type incompleteCheckCollector struct {
mu sync.Mutex
names map[string]struct{}
skipped map[string]bool
}
type incompleteCheckContextKey struct{}
type coverageGapsContextKey struct{}
type coveragePathCollectorContextKey struct{}
type coveragePathCollector struct {
mu sync.Mutex
pathsByOwner map[string]map[string]bool
scopesByOwner map[string]map[string]bool
}
// CoverageGaps holds file gaps and completed database scopes for a scan.
// The caller passes it to the atomic purge-and-merge operation so a concurrent
// update made after the scanner read LatestFindings cannot be retired by a
// stale carry-forward snapshot.
type CoverageGaps struct {
mu sync.Mutex
pathsByCheck map[string]map[string]bool
completedScopes map[string]map[string]bool
incompleteChecks map[string]bool
}
// Paths returns an isolated snapshot of the completed run's path gaps. Each
// inner key is a stable lexical or resolved alias captured during the scan.
func (g *CoverageGaps) Paths() map[string]map[string]bool {
if g == nil {
return nil
}
g.mu.Lock()
defer g.mu.Unlock()
return cloneCoverageGapPaths(g.pathsByCheck)
}
// WithCoverageGaps requests an atomic coverage-aware store operation from the
// caller. The runner publishes only its completed snapshot into the handle.
func WithCoverageGaps(ctx context.Context) (context.Context, *CoverageGaps) {
if ctx == nil {
ctx = context.Background()
}
gaps := &CoverageGaps{}
return context.WithValue(ctx, coverageGapsContextKey{}, gaps), gaps
}
func withIncompleteCheckCollector(ctx context.Context) (context.Context, *incompleteCheckCollector) {
if ctx == nil {
ctx = context.Background()
}
collector := &incompleteCheckCollector{names: make(map[string]struct{})}
return context.WithValue(ctx, incompleteCheckContextKey{}, collector), collector
}
func withCoveragePathCollector(ctx context.Context) (context.Context, *coveragePathCollector) {
collector := &coveragePathCollector{pathsByOwner: make(map[string]map[string]bool)}
return context.WithValue(ctx, coveragePathCollectorContextKey{}, collector), collector
}
func coverageGapsFrom(ctx context.Context) *CoverageGaps {
if ctx == nil {
return nil
}
gaps, _ := ctx.Value(coverageGapsContextKey{}).(*CoverageGaps)
return gaps
}
func (g *CoverageGaps) replace(pathsByCheck map[string]map[string]bool, completedScopes map[string]map[string]bool, incompleteChecks map[string]bool) {
if g == nil {
return
}
g.mu.Lock()
g.pathsByCheck = cloneCoverageGapPaths(pathsByCheck)
g.completedScopes = cloneCoverageGapPaths(completedScopes)
g.incompleteChecks = maps.Clone(incompleteChecks)
g.mu.Unlock()
}
func cloneCoverageGapPaths(pathsByCheck map[string]map[string]bool) map[string]map[string]bool {
if len(pathsByCheck) == 0 {
return nil
}
out := make(map[string]map[string]bool, len(pathsByCheck))
for check, paths := range pathsByCheck {
if len(paths) == 0 {
continue
}
cloned := make(map[string]bool, len(paths))
for path := range paths {
cloned[path] = true
}
out[check] = cloned
}
if len(out) == 0 {
return nil
}
return out
}
// markCheckIncomplete records a coverage gap that cannot be attributed to
// particular files, so the owner keeps every finding it has until it completes.
func markCheckIncomplete(ctx context.Context, name string) {
collector := incompleteCollectorFrom(ctx)
if collector == nil {
return
}
collector.mu.Lock()
collector.names[name] = struct{}{}
collector.mu.Unlock()
}
// markCheckSkipped preserves both discovered state and the last run's coverage
// summary when an internal refresh interval prevents any new audit work.
func markCheckSkipped(ctx context.Context, name string) {
collector := incompleteCollectorFrom(ctx)
if collector == nil {
return
}
collector.mu.Lock()
defer collector.mu.Unlock()
collector.names[name] = struct{}{}
if collector.skipped == nil {
collector.skipped = make(map[string]bool)
}
collector.skipped[name] = true
}
func (c *incompleteCheckCollector) wasSkipped(name string) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.skipped[name]
}
// A removed path is covered absence; a failed read is not evidence of cleanup.
func markScanReadError(ctx context.Context, owner string, err error) {
if err != nil && !os.IsNotExist(err) {
markCheckIncomplete(ctx, owner)
}
}
// Unlike the best-effort inventory, partial discovery cannot authorize a
// stateful scanner to retire findings from accounts it never enumerated.
func scanHomeDirsWithCoverage(ctx context.Context, owner string) []os.DirEntry {
if AccountFromContext(ctx) != "" {
entries, err := GetScanHomeDirs(ctx)
markScanReadError(ctx, owner, err)
return entries
}
homes, err := readAccountHomes()
markScanReadError(ctx, owner, err)
entries := make([]os.DirEntry, 0, len(homes))
for _, home := range homes {
entries = append(entries, rootedDirEntry{DirEntry: home.Entry, root: home.Root})
}
return entries
}
// recordCoverageGapPaths records the stable aliases captured when a known file
// gap was observed. The store must consume these aliases as identities, without
// resolving them again after a symlink may have changed targets.
func recordCoverageGapPaths(ctx context.Context, owner string, paths []string) {
if ctx == nil || len(paths) == 0 {
return
}
collector, _ := ctx.Value(coveragePathCollectorContextKey{}).(*coveragePathCollector)
if collector == nil {
return
}
collector.mu.Lock()
if collector.pathsByOwner[owner] == nil {
collector.pathsByOwner[owner] = make(map[string]bool)
}
for _, path := range paths {
if path != "" {
collector.pathsByOwner[owner][path] = true
}
}
collector.mu.Unlock()
}
// coveragePathAliases captures every path spelling a Finding.FilePath emitted
// by this walk can use: its absolute lexical form and, while the observed path
// still resolves, its symlink-resolved form. Callers retain the returned set so
// a later symlink retarget cannot change the preservation identity.
func coveragePathAliases(path string) []string {
lexical := coverageLexicalPath(path)
if lexical == "" {
return nil
}
aliases := []string{lexical}
if real, err := filepath.EvalSymlinks(lexical); err == nil {
real = filepath.Clean(real)
if real != lexical {
aliases = append(aliases, real)
}
}
return aliases
}
func coverageLexicalPath(path string) string {
if path == "" {
return ""
}
lexical := filepath.Clean(path)
if absolute, err := filepath.Abs(lexical); err == nil {
lexical = filepath.Clean(absolute)
}
return lexical
}
// stableCoveragePathAliases binds aliases to the file metadata that caused the
// scanner's decision. Re-checking both the lexical and resolved paths prevents
// a symlink retarget during alias construction from preserving a different file
// while retiring the one that actually went unexamined.
func stableCoveragePathAliases(path string, expected os.FileInfo) ([]string, bool) {
aliases := coveragePathAliases(path)
if expected == nil || len(aliases) == 0 {
return aliases, false
}
lexicalInfo, err := osFS.Lstat(aliases[0])
if err != nil || lexicalInfo.Mode()&os.ModeSymlink != 0 || !os.SameFile(expected, lexicalInfo) {
return aliases, false
}
for _, alias := range aliases[1:] {
resolvedInfo, statErr := osFS.Stat(alias)
if statErr != nil || !os.SameFile(expected, resolvedInfo) {
return aliases, false
}
}
lexicalInfo, err = osFS.Lstat(aliases[0])
if err != nil || lexicalInfo.Mode()&os.ModeSymlink != 0 || !os.SameFile(expected, lexicalInfo) {
return aliases, false
}
return aliases, true
}
// gapPathSet is the bookkeeping the deep-scan gap collectors share: the exact
// paths a run walked but could not examine, a count and one example per
// status, and the ranges the walk could not enumerate at all. Carry-forward
// needs exact paths, so the set is bounded and reports itself incomplete past
// the bound instead of carrying forward an arbitrary prefix.
type gapPathSet struct {
paths map[string]struct{}
pathAliases map[string]struct{}
aliasesByPath map[string][]string
byStatus map[string]int
example map[string]string
// resolveAliases captures every spelling a finding for the path can use.
// It reports false when the path changed while its identity was captured.
resolveAliases func(string) ([]string, bool)
// unknown counts walk failures whose affected paths cannot be enumerated
// (an unreadable directory, a failed Lstat that may hide one). They are
// kept apart from paths because carry-forward needs exact paths, but they
// must still reach the operator.
unknown int
unknownExample string
// pathsTruncated records that the exact-path set hit maxPaths, so the
// carry-forward can no longer be trusted to cover every gapped path.
pathsTruncated bool
maxPaths int
exampleMaxBytes int
}
func newGapPathSet(maxPaths, exampleMaxBytes int, resolveAliases func(string) ([]string, bool)) gapPathSet {
return gapPathSet{
paths: map[string]struct{}{},
pathAliases: map[string]struct{}{},
aliasesByPath: map[string][]string{},
byStatus: map[string]int{},
example: map[string]string{},
resolveAliases: resolveAliases,
maxPaths: maxPaths,
exampleMaxBytes: exampleMaxBytes,
}
}
// lexicalGapAliases names a path by its absolute lexical spelling only. It is
// the resolver of a collector whose walk supplies no observed aliases.
func lexicalGapAliases(path string) ([]string, bool) {
return []string{coverageLexicalPath(path)}, true
}
// retain counts path under status and returns the aliases of an
// authoritatively retained path. It returns nil once the set is full, or when
// the path changed while its identity was captured.
func (g *gapPathSet) retain(path, status string) []string {
aliases, retained := g.aliasesByPath[path]
if !retained && len(g.paths) >= g.maxPaths {
// Stop retaining paths, but never stop counting: the count is what
// tells an operator how much of the host went unexamined.
g.pathsTruncated = true
} else if !retained {
var stable bool
aliases, stable = g.resolveAliases(path)
if stable {
g.paths[path] = struct{}{}
g.aliasesByPath[path] = aliases
for _, alias := range aliases {
g.pathAliases[alias] = struct{}{}
}
} else {
g.recordUnknownRange(fmt.Sprintf("%s changed while its path identity was captured", path))
aliases = nil
}
}
g.byStatus[status]++
if _, ok := g.example[status]; !ok {
g.example[status] = sanitizeJSTaintDisplay(path, g.exampleMaxBytes)
}
return aliases
}
// recordUnknownRange notes coverage lost over a range this walk cannot
// enumerate. It deliberately does not add to paths: claiming specific paths
// would be false, and the unknown range already forces a partial run, which
// suppresses the purge for every prior finding.
func (g *gapPathSet) recordUnknownRange(detail string) {
g.unknown++
if g.unknownExample == "" {
g.unknownExample = sanitizeJSTaintDisplay(detail, g.exampleMaxBytes)
}
}
// pathsIncomplete reports that this run could not enumerate every gapped path,
// either because retention hit its bound or because the walk lost an unknown
// range. Its carry-forward set is therefore not authoritative and the purge
// must be suppressed for the whole owner.
func (g *gapPathSet) pathsIncomplete() bool { return g.pathsTruncated || g.unknown > 0 }
func (g *gapPathSet) empty() bool { return len(g.byStatus) == 0 && g.unknown == 0 }
func (g *gapPathSet) hasPath(path string) bool {
if _, ok := g.paths[path]; ok {
return true
}
for _, alias := range coveragePathAliases(path) {
if _, ok := g.pathAliases[alias]; ok {
return true
}
}
return false
}
// statusParts renders byStatus as one "status=count (example)" part per
// status, in status order, and returns the summed count.
func (g *gapPathSet) statusParts(byStatus map[string]int) (int, []string) {
total := 0
statuses := make([]string, 0, len(byStatus))
for status, n := range byStatus {
total += n
statuses = append(statuses, status)
}
sort.Strings(statuses)
parts := make([]string, 0, len(statuses)+2)
for _, status := range statuses {
parts = append(parts, fmt.Sprintf("%s=%d (example: %s)", status, byStatus[status], g.example[status]))
}
return total, parts
}
// rangeLossParts renders the facts that make the exact-path set incomplete.
func (g *gapPathSet) rangeLossParts() []string {
var parts []string
if g.unknown > 0 {
parts = append(parts, fmt.Sprintf("unreadable-range=%d (example: %s)", g.unknown, g.unknownExample))
}
if g.pathsTruncated {
parts = append(parts, fmt.Sprintf("exact paths retained for only the first %d", g.maxPaths))
}
return parts
}
// incompleteFinding reports every gap in one alert. filesFormat takes the
// file count; rangesFormat takes the lost-range count and is used when no
// file was counted.
func (g *gapPathSet) incompleteFinding(severity alert.Severity, check, filesFormat, rangesFormat string) alert.Finding {
total, parts := g.statusParts(g.byStatus)
parts = append(parts, g.rangeLossParts()...)
message := fmt.Sprintf(filesFormat, total)
if total == 0 {
message = fmt.Sprintf(rangesFormat, g.unknown)
}
return alert.Finding{
Severity: severity,
Check: check,
Message: message,
Details: strings.Join(parts, "; "),
// One host-wide condition: counts and examples vary per cycle and
// must not re-alert while coverage stays degraded.
DedupKey: "coverage_gap",
}
}
// carryForwardLatestPerPath keeps at most one prior finding of check for each
// path the current full cycle could not analyze. Partial rolling windows can
// leave more than one historical variant at a path because their owner is not
// purgeable; the most recently observed variant is the one that represents
// current state when a later full cycle hits a path-specific gap.
func carryForwardLatestPerPath(prior []alert.Finding, check string, gaps *gapPathSet) []alert.Finding {
byPath := make(map[string]alert.Finding)
for _, finding := range prior {
if finding.Check != check || !gaps.hasPath(finding.FilePath) {
continue
}
current, exists := byPath[finding.FilePath]
if !exists || finding.Timestamp.After(current.Timestamp) ||
(finding.Timestamp.Equal(current.Timestamp) && finding.Key() < current.Key()) {
byPath[finding.FilePath] = finding
}
}
paths := make([]string, 0, len(byPath))
for path := range byPath {
paths = append(paths, path)
}
sort.Strings(paths)
carried := make([]alert.Finding, 0, len(paths))
for _, path := range paths {
finding := byPath[path]
finding.ScanCarryForward = true
carried = append(carried, finding)
}
return carried
}
func incompleteCollectorFrom(ctx context.Context) *incompleteCheckCollector {
if ctx == nil {
return nil
}
collector, _ := ctx.Value(incompleteCheckContextKey{}).(*incompleteCheckCollector)
return collector
}
func checkMarkedIncomplete(ctx context.Context, name string) bool {
collector := incompleteCollectorFrom(ctx)
return collector != nil && collector.contains(name)
}
func (c *coveragePathCollector) gapPaths(owner string) map[string]bool {
c.mu.Lock()
defer c.mu.Unlock()
out := make(map[string]bool, len(c.pathsByOwner[owner]))
for path := range c.pathsByOwner[owner] {
out[path] = true
}
return out
}
func (c *incompleteCheckCollector) contains(name string) bool {
c.mu.Lock()
defer c.mu.Unlock()
_, ok := c.names[name]
return ok
}
// Snapshot includes both file gaps and completed database scopes from the same
// runner result. Callers must pass this snapshot to the atomic store operation.
func (g *CoverageGaps) Snapshot() *state.ScanCoverage {
if g == nil {
return nil
}
g.mu.Lock()
defer g.mu.Unlock()
return &state.ScanCoverage{
PreservePaths: cloneCoverageGapPaths(g.pathsByCheck),
CompletedScopes: cloneCoverageGapPaths(g.completedScopes),
IncompleteChecks: maps.Clone(g.incompleteChecks),
}
}
func recordCompletedCoverageScopes(ctx context.Context, owner string, scopes map[string]bool) {
if ctx == nil {
return
}
collector, _ := ctx.Value(coveragePathCollectorContextKey{}).(*coveragePathCollector)
if collector == nil {
return
}
collector.mu.Lock()
defer collector.mu.Unlock()
if collector.scopesByOwner == nil {
collector.scopesByOwner = make(map[string]map[string]bool)
}
complete := make(map[string]bool)
for scope, covered := range scopes {
if covered && scope != "" {
complete[scope] = true
}
}
collector.scopesByOwner[owner] = complete
}
func (c *coveragePathCollector) completedScopes(owner string) map[string]bool {
c.mu.Lock()
defer c.mu.Unlock()
out := make(map[string]bool, len(c.scopesByOwner[owner]))
for scope := range c.scopesByOwner[owner] {
out[scope] = true
}
return out
}
package checks
import (
"context"
"errors"
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
const checkDispatchControlBudget = time.Minute
var checkDispatches = newCheckDispatchMonitor()
type checkDispatchMonitor struct {
mu sync.Mutex
batches map[*checkDispatchBatch]struct{}
losses *queuehealth.Tracker
}
type checkDispatchBatch struct {
monitor *checkDispatchMonitor
tasks map[*checkDispatch]struct{}
budget *scanBudget
progress time.Time
observer *CheckDispatchProgress
}
// Timing, failure and batch membership fields are guarded by the monitor mutex.
type checkDispatch struct {
batch *checkDispatchBatch
slot *scanSlot // set by the runner before starting the execution
started time.Time
deadline time.Time
failed bool
}
func newCheckDispatchMonitor() *checkDispatchMonitor {
return &checkDispatchMonitor{
batches: make(map[*checkDispatchBatch]struct{}),
losses: queuehealth.New(0, time.Minute),
}
}
func (m *checkDispatchMonitor) begin(count int, budget *scanBudget) []*checkDispatch {
now := time.Now()
batch := &checkDispatchBatch{
monitor: m, tasks: make(map[*checkDispatch]struct{}, count),
budget: budget, progress: now,
}
tasks := make([]*checkDispatch, count)
for i := range tasks {
tasks[i] = &checkDispatch{batch: batch}
batch.tasks[tasks[i]] = struct{}{}
}
if count > 0 {
m.mu.Lock()
m.batches[batch] = struct{}{}
m.mu.Unlock()
}
return tasks
}
func (t *checkDispatch) admit(ctx context.Context) bool {
t.slot = t.batch.budget.acquire(ctx)
if t.slot == nil {
return false
}
m := t.batch.monitor
m.mu.Lock()
defer m.mu.Unlock()
now := time.Now()
t.started = now
t.deadline = now.Add(checkDispatchControlBudget)
t.batch.progressed(now)
return true
}
func (t *checkDispatch) executing(ctx context.Context) {
if t == nil {
return
}
// A caller without a deadline is unbounded rather than already overdue.
deadline, _ := ctx.Deadline()
m := t.batch.monitor
m.mu.Lock()
defer m.mu.Unlock()
t.deadline = deadline
t.batch.progressed(time.Now())
}
func (t *checkDispatch) returned() {
if t == nil {
return
}
m := t.batch.monitor
m.mu.Lock()
defer m.mu.Unlock()
now := time.Now()
t.deadline = now.Add(checkDispatchControlBudget)
t.batch.progressed(now)
}
func (t *checkDispatch) withdraw(ctx context.Context) {
if t == nil || !errors.Is(ctx.Err(), context.DeadlineExceeded) {
return
}
m := t.batch.monitor
m.mu.Lock()
defer m.mu.Unlock()
t.failLocked(time.Now())
}
func (t *checkDispatch) failLocked(now time.Time) {
if !t.failed {
t.failed = true
t.batch.monitor.losses.Lose(now, 1)
}
}
func (t *checkDispatch) wrap(fn func()) func() {
return func() {
completed := false
defer func() {
t.slot.release()
m := t.batch.monitor
m.mu.Lock()
defer m.mu.Unlock()
now := time.Now()
if !completed {
t.failLocked(now)
}
delete(t.batch.tasks, t)
t.batch.progressed(now)
if len(t.batch.tasks) == 0 {
delete(m.batches, t.batch)
if t.batch.observer != nil {
delete(t.batch.observer.batches, t.batch)
}
}
}()
fn()
completed = true
}
}
func (m *checkDispatchMonitor) QueueStatus(now time.Time) queuehealth.Status {
m.mu.Lock()
defer m.mu.Unlock()
status := m.losses.Snapshot(now)
status.CapacityUnavailable = true
status.LagBasis = "consumer_progress"
var dispatchLate, runnerLate bool
for batch := range m.batches {
waiting, running := 0, 0
for task := range batch.tasks {
if task.started.IsZero() {
waiting++
} else {
running++
status.ProcessingSeconds = max(status.ProcessingSeconds, now.Sub(task.started).Seconds())
runnerLate = runnerLate || overdue(now, task.deadline)
}
}
status.Depth += waiting
status.InFlight += running
if waiting > 0 {
lag := now.Sub(batch.progress)
status.LagSeconds = max(status.LagSeconds, lag.Seconds())
// Other batches and withdrawn executions can still own slots.
// Only unused shared capacity indicates stalled dispatch.
dispatchLate = dispatchLate || (batch.budget.hasCapacity() && lag >= checkDispatchControlBudget)
}
}
switch {
case runnerLate:
status.Reason = "processing_lag"
case dispatchLate:
status.Reason = "backlog_lag"
}
if status.Reason != "" {
status.Status = "degraded"
}
return status
}
type checkDispatchContextKey struct{}
func withCheckDispatch(ctx context.Context, task *checkDispatch) context.Context {
return context.WithValue(ctx, checkDispatchContextKey{}, task)
}
func checkDispatchFrom(ctx context.Context) *checkDispatch {
task, _ := ctx.Value(checkDispatchContextKey{}).(*checkDispatch)
return task
}
// CheckDispatchQueueStatus measures pending checks and their runner wrappers.
func CheckDispatchQueueStatus(now time.Time) queuehealth.Status {
return checkDispatches.QueueStatus(now)
}
package checks
import (
"context"
"time"
)
// DispatchProgressSnapshot measures only the check batches owned by one caller.
// LastProgress remains available after the last wrapper exits, so its caller
// can time result handling without borrowing the completed check's budget.
type DispatchProgressSnapshot struct {
Active bool
Overdue bool
LastProgress time.Time
}
// CheckDispatchProgress is memory-only evidence for a scan's orchestration.
// All fields are guarded by the owning dispatch monitor's mutex.
type CheckDispatchProgress struct {
monitor *checkDispatchMonitor
batches map[*checkDispatchBatch]struct{}
last time.Time
}
type dispatchProgressContextKey struct{}
// WithCheckDispatchProgress binds subsequent check batches to this operation.
// A new binding isolates late callbacks from a previously canceled operation.
func WithCheckDispatchProgress(ctx context.Context) (context.Context, *CheckDispatchProgress) {
p := &CheckDispatchProgress{monitor: checkDispatches, batches: make(map[*checkDispatchBatch]struct{}), last: time.Now()}
return context.WithValue(ctx, dispatchProgressContextKey{}, p), p
}
func (m *checkDispatchMonitor) observe(ctx context.Context, tasks []*checkDispatch) {
p, _ := ctx.Value(dispatchProgressContextKey{}).(*CheckDispatchProgress)
if p == nil || len(tasks) == 0 {
return
}
m.mu.Lock()
defer m.mu.Unlock()
batch := tasks[0].batch
batch.observer = p
p.batches[batch] = struct{}{}
p.last = time.Now()
}
func (b *checkDispatchBatch) progressed(now time.Time) {
b.progress = now
if b.observer != nil {
b.observer.last = now
}
}
// Snapshot does not query contexts, the filesystem or the job store.
func (p *CheckDispatchProgress) Snapshot(now time.Time) DispatchProgressSnapshot {
p.monitor.mu.Lock()
defer p.monitor.mu.Unlock()
s := DispatchProgressSnapshot{Active: len(p.batches) != 0, LastProgress: p.last}
for batch := range p.batches {
waiting := 0
for task := range batch.tasks {
if task.started.IsZero() {
waiting++
} else {
s.Overdue = s.Overdue || !now.Before(task.deadline)
}
}
if waiting > 0 && batch.budget.hasCapacity() && now.Sub(batch.progress) >= checkDispatchControlBudget {
s.Overdue = true
}
}
return s
}
package checks
import (
"context"
"fmt"
"runtime/debug"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/obs"
)
type checkExecutionOutcome struct {
findings []alert.Finding
panicErr string
}
func executeCheckAsync(ctx context.Context, component string, fn func() []alert.Finding) *checkExecution {
return checkExecutions.execute(ctx, component, fn)
}
func (e *checkExecution) run(component string, fn func() []alert.Finding) {
defer e.slot.release()
e.monitor.mu.Lock()
e.started = time.Now()
e.monitor.mu.Unlock()
defer e.release()
outcome := checkExecutionOutcome{}
completed := false
defer func() {
if !completed {
e.fail()
}
if recovered := recover(); recovered != nil {
panicValue := fmt.Sprint(recovered)
outcome.panicErr = fmt.Sprintf("%s\n%s", panicValue, debug.Stack())
obs.CaptureMsg(component, "security check panic: "+panicValue)
}
e.done <- outcome
}()
outcome.findings = fn()
completed = true
}
package checks
import (
"context"
"errors"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/queuehealth"
)
var checkExecutions = newCheckExecutionMonitor()
type checkExecutionMonitor struct {
mu sync.Mutex
pending map[*checkExecution]struct{}
losses *queuehealth.Tracker
}
func newCheckExecutionMonitor() *checkExecutionMonitor {
return &checkExecutionMonitor{
pending: make(map[*checkExecution]struct{}),
losses: queuehealth.New(0, time.Minute),
}
}
type checkExecution struct {
monitor *checkExecutionMonitor
dispatch *checkDispatch
slot *scanSlot
queued time.Time
started time.Time // guarded by monitor.mu
deadline time.Time
remaining atomic.Int32
failOnce sync.Once
done chan checkExecutionOutcome
// Only the caller changes this; the function owns its separate release.
callerSettled bool
}
func (m *checkExecutionMonitor) begin(deadline time.Time) *checkExecution {
execution := &checkExecution{
monitor: m, queued: time.Now(), deadline: deadline,
done: make(chan checkExecutionOutcome, 1),
}
execution.remaining.Store(2)
m.mu.Lock()
m.pending[execution] = struct{}{}
m.mu.Unlock()
return execution
}
func (m *checkExecutionMonitor) execute(ctx context.Context, component string, fn func() []alert.Finding) *checkExecution {
// Both runners construct a bounded per-check context before dispatch. A
// caller without one is unbounded rather than already overdue.
deadline, _ := ctx.Deadline()
execution := m.begin(deadline)
execution.dispatch = checkDispatchFrom(ctx)
execution.dispatch.executing(ctx)
if execution.dispatch != nil {
execution.slot = execution.dispatch.slot
execution.slot.retain()
}
go execution.run(component, fn)
return execution
}
func (e *checkExecution) fail() {
e.failOnce.Do(func() { e.monitor.losses.Lose(time.Now(), 1) })
}
func (e *checkExecution) received() {
if !e.callerSettled {
e.dispatch.returned()
e.callerSettled = true
}
}
func (e *checkExecution) withdraw(err error) {
e.received()
if errors.Is(err, context.DeadlineExceeded) {
e.fail()
}
}
func (e *checkExecution) finishCaller() {
if !e.callerSettled {
e.fail()
e.dispatch.returned()
}
e.release()
}
func (e *checkExecution) release() {
// A deadline can release the caller while the function still runs. A
// finished function likewise retains its buffered result for the caller.
if e.remaining.Add(-1) == 0 {
e.monitor.mu.Lock()
delete(e.monitor.pending, e)
e.monitor.mu.Unlock()
}
}
func (m *checkExecutionMonitor) QueueStatus(now time.Time) queuehealth.Status {
m.mu.Lock()
defer m.mu.Unlock()
status := m.losses.Snapshot(now)
status.CapacityUnavailable = true
var waitingLate, runningLate bool
for execution := range m.pending {
if execution.started.IsZero() {
status.Depth++
status.LagSeconds = max(status.LagSeconds, now.Sub(execution.queued).Seconds())
waitingLate = waitingLate || overdue(now, execution.deadline)
} else {
status.InFlight++
status.ProcessingSeconds = max(status.ProcessingSeconds, now.Sub(execution.started).Seconds())
runningLate = runningLate || overdue(now, execution.deadline)
}
}
// Each call has its own deadline. A heavy check must not lend its longer
// budget to an overdue short check, or get a fresh budget after dispatch.
switch {
case waitingLate:
status.Reason = "backlog_lag"
case runningLate:
status.Reason = "processing_lag"
}
if status.Reason != "" {
status.Status = "degraded"
}
return status
}
// CheckExecutionQueueStatus reads memory only, including after a runner exits.
func CheckExecutionQueueStatus(now time.Time) queuehealth.Status {
return checkExecutions.QueueStatus(now)
}
// overdue reports whether a bounded deadline has passed. Work with no deadline
// has no bound to miss.
func overdue(now, deadline time.Time) bool {
return !deadline.IsZero() && !now.Before(deadline)
}
package checks
import (
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"io/fs"
"math"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync/atomic"
"syscall"
"time"
"unicode"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/safepath"
)
// CleanResult describes the outcome of a cleaning attempt.
type CleanResult struct {
Path string
Cleaned bool
BackupPath string
Removals []string // descriptions of what was removed
Error string
// Refused marks a safety refusal without changing the customer file.
// It consumes attempt capacity but does not indicate a broken action.
Refused bool
// WholeFile marks a refusal because cleaning would leave nothing of a file
// that sends no page output: it is quarantined whole instead.
WholeFile bool
wholeFileInfo os.FileInfo
wholeFileContent []byte
}
var (
// Leading class is [\s\x00\x0b] rather than \s: an injection can pad
// the line start with NUL or vertical-tab bytes that Go's \s does not
// cover. Detection still flags the file (analyzePHPContent is not
// anchored); widening the class here lets the surgical cleaner strip
// the injection instead of falling back to whole-file quarantine.
cleanRegexpMaliciousInclude = regexp.MustCompile(
`(?i)^[\s\x00\x0b]*@include\s*\(\s*(?:` +
`['"](?:/tmp/|/dev/shm/|/var/tmp/)` +
`|base64_decode\s*\(` +
`|str_rot13\s*\(` +
`|gzinflate\s*\(` +
`)`)
cleanRegexpVarInclude = regexp.MustCompile(`(?i)^[\s\x00\x0b]*@include\s*\(\s*\$[a-zA-Z_]+\s*\)`)
cleanRegexpCloseOpen = regexp.MustCompile(`(?i)\?>\s*<\?php`)
cleanRegexpInlineEval = regexp.MustCompile(
`(?i)^[\s\x00\x0b]*(?:@?)eval\s*\(\s*(?:base64_decode|gzinflate|gzuncompress|str_rot13)\s*\(`)
cleanRegexpMultiB64 = regexp.MustCompile(`(?i)(?:base64_decode\s*\(\s*){2,}`)
cleanRegexpChainedB64 = regexp.MustCompile(`(?i)\$\w+\s*=\s*base64_decode\s*\(\s*base64_decode`)
cleanRegexpChrChain = regexp.MustCompile(`(?i)\bchr\s*\(\s*\d+\s*\)(?:\s*\.?\s*\bchr\s*\(\s*\d+\s*\)){4,}`)
cleanRegexpPackHex = regexp.MustCompile(`(?i)pack\s*\(\s*["']H\*["']\s*,`)
cleanRegexpHexVar = regexp.MustCompile(`(?:"\x5c\x78[0-9a-fA-F]{2}){3,}|(?:\\x[0-9a-fA-F]{2}){3,}`)
)
// cleanMaxFileSize bounds how large a file surgical cleaning will read
// into memory. The detector that routes a file here (analyzePHPContent)
// only inspects bounded head and tail windows, so an attacker can match a
// signature inside those windows and pad the rest to many gigabytes. Reading
// that whole file with io.ReadAll plus the strings.Split and regex passes
// below would OOM the root daemon. Above this ceiling we refuse; the caller
// decides whether whole-file quarantine is safe for that remediation path.
// Legitimate plugin/theme PHP files are far smaller than this. Var, not
// const, so tests can lower it. 8 MiB.
var cleanMaxFileSize int64 = 8 << 20
// CleanInfectedFile attempts to surgically remove malicious code from a PHP file
// while preserving the legitimate content. Always creates a backup first.
//
// Cleaning strategies (tried in order):
// 1. @include injection - remove @include lines pointing to /tmp, eval, base64, or via variables
// 2. Prepend injection - remove malicious code blocks at start of file (entropy-validated)
// 3. Append injection - remove malicious code after closing ?> or end of PSR-12 file
// 4. Inline eval injection - remove eval(base64_decode(...)) single-line injections
func CleanInfectedFile(path string) CleanResult {
return cleanInfectedFileIdentified(path, nil)
}
func cleanInfectedFileIdentified(path string, expected os.FileInfo) (result CleanResult) {
result = CleanResult{Path: path}
audit := newCleanAction(path)
defer func() { audit.finish(result.Error) }()
target, err := openCleanTarget(path)
if err != nil {
result.Refused = errors.Is(fileResponseSourceError(err), errFileResponseRefused)
result.Error = fmt.Sprintf("cannot read file: %v", err)
return result
}
defer target.Close()
if expected != nil && (!sameFileIdentity(expected, target.Info) || !sameContentShape(expected, target.Info)) {
result.Refused = true
result.Error = "file changed before automatic cleaning"
return result
}
if sz := target.Info.Size(); sz > cleanMaxFileSize {
result.Refused = true
result.Error = fmt.Sprintf("file too large to clean (%d bytes > %d)", sz, cleanMaxFileSize)
return result
}
audit.rec.Result = actionlog.Failed
data, err := io.ReadAll(target.File)
if err != nil {
result.Error = fmt.Sprintf("cannot read file: %v", err)
return result
}
audit.capture(target, data)
// Create backup before any modification
backupDir := filepath.Join(quarantineDir, "pre_clean")
backupPath := newQuarantinePath(backupDir, path)
// Metadata sidecar derived from the same fd we read, so a directory
// race after open cannot change the metadata we record.
meta := quarantineMetadata(path, target.Info, "Pre-clean backup (surgical cleaning)")
if err := storeQuarantineBackup(backupPath, data, meta, 0600); err != nil {
result.Error = fmt.Sprintf("cannot create durable backup: %v", err)
return result
}
result.BackupPath = backupPath
content := string(data)
originalLen := len(content)
var removals []string
// Strategy 1: Remove @include injections (including variable-based)
content, removed := removeIncludeInjections(content)
removals = append(removals, removed...)
// Strategy 2: Remove prepend injections (with entropy validation)
content, removed = removePrependInjection(content)
removals = append(removals, removed...)
// Strategy 3: Remove append injections (handles files with and without closing ?>)
content, removed = removeAppendInjection(content)
removals = append(removals, removed...)
// Strategy 4: Remove inline eval(base64_decode(...)) injections
content, removed = removeInlineEvalInjections(content)
removals = append(removals, removed...)
// Strategy 5: Remove multi-layer base64 decode chains
content, removed = removeMultiLayerBase64(content)
removals = append(removals, removed...)
// Strategy 6: Remove chr()/pack() constructed code
content, removed = removeChrPackInjections(content)
removals = append(removals, removed...)
// Strategy 7: Remove hex-encoded variable injections
content, removed = removeHexVarInjections(content)
removals = append(removals, removed...)
// If nothing was removed, file couldn't be cleaned
if len(removals) == 0 || len(content) == originalLen {
audit.rec.Result = actionlog.Refused
result.Refused = true
result.Error = "no known injection patterns found - file may need manual review"
return result
}
if cleanLeavesNothing(content) {
audit.rec.Result = actionlog.Refused
result.Refused = true
// Line and block removals can take site code with the injection; a
// stub would lose it as well, so the file goes to quarantine whole,
// where a restore can recover it. Page output stays for an operator.
if !phpWithoutOutput(string(data)) {
result.Error = "cleaning would erase the file's page output; manual review required"
return result
}
result.WholeFile = true
result.wholeFileInfo = target.Info
result.wholeFileContent = data
result.Error = "cleaning would leave nothing of the file - quarantine it whole instead"
return result
}
audit.rec.Reason = strings.Join(removals, "; ")
if err := audit.replace(target, []byte(content), backupPath); err != nil {
result.Refused = errors.Is(err, errFileResponseRefused)
result.Error = fmt.Sprintf("cannot write cleaned file: %v", err)
return result
}
result.Cleaned = true
result.Removals = removals
// Keep the pre-rewrite change time, but only a successful cleanup proves
// the file contained an injection.
companionSweeps.add(path, target.Info, time.Now())
return result
}
type cleanTarget struct {
Path string
Dir *safepath.Dir
Name string
File *os.File
Info os.FileInfo
UID int
GID int
replacementInfo os.FileInfo
installed bool
OwnerKnown bool
}
func (t *cleanTarget) Close() {
if t.File != nil {
_ = t.File.Close()
}
if t.Dir != nil {
_ = t.Dir.Close()
}
}
// openCleanTarget opens path without following a symlink in any component.
// The account owner controls every directory below its home, so a symlink
// planted anywhere on the path after the caller's checks must not steer the
// read or the replacement into another tree.
func openCleanTarget(path string) (*cleanTarget, error) {
return openCleanTargetFlags(path, os.O_RDONLY)
}
func openCleanTargetFlags(path string, flags int) (*cleanTarget, error) {
parent, name := filepath.Split(path)
if name == "" || name == "." || name == ".." {
return nil, fmt.Errorf("invalid target path %q", path)
}
parent, err := filepath.Abs(parent)
if err != nil {
return nil, fmt.Errorf("resolve parent directory: %w", err)
}
dir, err := safepath.OpenDirNoFollow(parent)
if err != nil {
return nil, fmt.Errorf("open parent directory: %w", fileResponseSourceError(err))
}
closeDir := true
defer func() {
if closeDir {
_ = dir.Close()
}
}()
file, err := dir.OpenFile(name, flags, 0)
if err != nil {
return nil, err
}
closeFile := true
defer func() {
if closeFile {
_ = file.Close()
}
}()
info, err := file.Stat()
if err != nil {
return nil, fmt.Errorf("stat file: %w", err)
}
if !info.Mode().IsRegular() {
return nil, refuseFileResponse(fmt.Errorf("refusing non-regular file (mode=%v)", info.Mode()))
}
target := &cleanTarget{
Path: path,
Dir: dir,
Name: name,
File: file,
Info: info,
}
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
target.UID = int(stat.Uid)
target.GID = int(stat.Gid)
target.OwnerKnown = true
}
closeDir = false
closeFile = false
return target, nil
}
var closeCleanTemp = (*os.File).Close
// cleanAfterTempCreated lets a test act as the account owner once the staged
// replacement exists in the directory the owner controls.
var cleanAfterTempCreated = func(string) {}
var syncCleanParent = (*safepath.Dir).Sync
// writeCleanedFileAtomic stages content through a hidden sibling name under
// the pinned parent directory and renames it over the original only after the
// name still holds the inode we read. Owner and mode are set on the staged
// descriptor, never by path.
func writeCleanedFileAtomic(target *cleanTarget, content []byte) error {
tmp, tmpName, err := createCleanTempFile(target.Dir)
if err != nil {
return err
}
cleanAfterTempCreated(filepath.Join(filepath.Dir(target.Path), tmpName))
removeTmp := true
defer func() {
if removeTmp {
_ = target.Dir.Remove(tmpName)
}
_ = tmp.Close()
}()
if _, err := tmp.Write(content); err != nil {
return err
}
if target.OwnerKnown {
if err := tmp.Chown(target.UID, target.GID); err != nil {
return err
}
}
if err := tmp.Chmod(cleanReplacementMode(target.Info.Mode())); err != nil {
return err
}
if err := tmp.Sync(); err != nil {
return err
}
replacementInfo, statErr := tmp.Stat()
if statErr != nil {
return statErr
}
if err := closeCleanTemp(tmp); err != nil {
return err
}
if err := verifyCleanTargetUnchanged(target); err != nil {
return err
}
// The owner can replace the staged name too. Install only the file
// written above, so a refusal is reported instead of a planted entry.
staged, stagedErr := target.Dir.Stat(tmpName)
if stagedErr != nil || !sameFileIdentity(staged, replacementInfo) {
return refuseFileResponse(errors.New("staged replacement changed before install"))
}
if err := target.Dir.ReplaceTo(tmpName, target.Dir, target.Name); err != nil {
return err
}
removeTmp = false
target.installed = true
target.replacementInfo = replacementInfo
if err := syncCleanParent(target.Dir); err != nil {
return fmt.Errorf("replacement installed but directory sync failed: %w", err)
}
return nil
}
func createCleanTempFile(dir *safepath.Dir) (*os.File, string, error) {
for i := 0; i < 100; i++ {
var raw [8]byte
if _, err := rand.Read(raw[:]); err != nil {
return nil, "", fmt.Errorf("random temp name: %w", err)
}
name := ".csm-clean-" + hex.EncodeToString(raw[:])
file, err := dir.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if errors.Is(err, fs.ErrExist) {
continue
}
if err != nil {
return nil, "", err
}
return file, name, nil
}
return nil, "", fmt.Errorf("could not allocate temp file name")
}
func verifyCleanTargetUnchanged(target *cleanTarget) error {
file, err := target.Dir.OpenFile(target.Name, os.O_RDONLY, 0)
if err != nil {
return fmt.Errorf("open target before rename: %w", fileResponseSourceError(err))
}
defer func() { _ = file.Close() }()
info, err := file.Stat()
if err != nil {
return fmt.Errorf("stat target before rename: %w", err)
}
if !info.Mode().IsRegular() {
return refuseFileResponse(fmt.Errorf("refusing non-regular target before rename (mode=%v)", info.Mode()))
}
if !sameCleanFileState(info, target.Info) {
return refuseFileResponse(errors.New("file changed during cleaning"))
}
return nil
}
// sameCleanFileState also compares mode and owner, so a permission change
// made while the replacement was staged is not overwritten.
func sameCleanFileState(a, b os.FileInfo) bool {
if !sameFileIdentity(a, b) || !sameCleanContentShape(a, b) || a.Mode() != b.Mode() {
return false
}
as, aok := a.Sys().(*syscall.Stat_t)
bs, bok := b.Sys().(*syscall.Stat_t)
return aok && bok && as.Uid == bs.Uid && as.Gid == bs.Gid
}
func sameCleanContentShape(a, b os.FileInfo) bool {
if a == nil || b == nil {
return false
}
return a.Size() == b.Size() && a.ModTime().Equal(b.ModTime())
}
func cleanReplacementMode(mode os.FileMode) os.FileMode {
return mode & (os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky)
}
// packageListingFunc reports whether the official WordPress core or plugin
// release that path belongs to ships it. A known absence requires trusted
// installation provenance; a tenant-selected release cannot supply it.
type packageListingFunc func(path string) (listed, known bool)
// packageListing is unset until the daemon provides its checksum cache.
var packageListing atomic.Pointer[packageListingFunc]
// SetPackageListing supplies the official file lists of WordPress releases.
func SetPackageListing(fn func(path string) (listed, known bool)) {
if fn == nil {
packageListing.Store(nil)
return
}
listing := packageListingFunc(fn)
packageListing.Store(&listing)
}
// notShippedByRelease accepts only a definitive absence from the provider.
// A manifest miss is known only with an authenticated release header.
func notShippedByRelease(path string) bool {
listing := packageListing.Load()
if listing == nil {
return false
}
listed, known := (*listing)(path)
return known && !listed
}
// shippedByRelease accepts only a definitive listing from the provider.
func shippedByRelease(path string) bool {
listing := packageListing.Load()
if listing == nil {
return false
}
listed, known := (*listing)(path)
return known && listed
}
// ShouldCleanInsteadOfQuarantine returns true if the file should be cleaned
// (surgical removal) instead of quarantined (full removal).
// WP core files and plugin files are better cleaned - removing them breaks the site.
// Unknown standalone files (droppers, webshells) should be quarantined.
// Package paths stay with the cleaner unless the provider can prove the
// file does not belong to the installed site; manifest misses alone cannot.
func ShouldCleanInsteadOfQuarantine(path string) bool {
// Keep core paths with the cleaner whenever membership is uncertain.
if strings.Contains(path, "/wp-includes/") || strings.Contains(path, "/wp-admin/") {
return !notShippedByRelease(path)
}
// Plugin/theme main files - clean to preserve functionality
if strings.Contains(path, "/wp-content/plugins/") || strings.Contains(path, "/wp-content/themes/") {
// But not if the file itself is the malware (h4x0r.php inside a theme)
name := strings.ToLower(filepath.Base(path))
if isWebshellName(name) {
return false // quarantine this - it's a standalone webshell
}
return !notShippedByRelease(path)
}
// Everything else - quarantine
return false
}
// cleanLeavesNothing reports whether cleaned content keeps nothing of the
// file: no PHP code and no page output, only PHP tags, comments, statement
// separators and whitespace.
func cleanLeavesNothing(content string) bool {
inPHP := false
for i := 0; i < len(content); {
if isPHPSpace(content[i]) {
i++
continue
}
if !inPHP {
if i+5 > len(content) || !strings.EqualFold(content[i:i+5], "<?php") ||
i+5 < len(content) && !isPHPOpenTagSpace(content[i+5]) {
return false
}
inPHP = true
i += 5
continue
}
switch {
case strings.HasPrefix(content[i:], "?>"):
inPHP = false
i += 2
case strings.HasPrefix(content[i:], "/*"):
end := strings.Index(content[i+2:], "*/")
if end < 0 {
return false
}
i += end + 4
case isPHPLineCommentStart(content, i):
i = skipPHPLineComment(content, i)
case content[i] == ';':
i++
default:
return false
}
}
return true
}
// phpWithoutOutput reports whether src has only whitespace outside long-tag
// PHP regions. It does not evaluate output from PHP statements. Short and echo
// tags stay for review. Use the lexer boundary, not its mask: the mask also
// removes line comments ending at a closing tag and loses tag positions.
func phpWithoutOutput(src string) bool {
var code strings.Builder
for i := 0; i < len(src); {
if isPHPSpace(src[i]) {
i++
continue
}
if i+5 > len(src) || !strings.EqualFold(src[i:i+5], "<?php") ||
i+5 < len(src) && !isPHPOpenTagSpace(src[i+5]) {
return false
}
i = copyPHPModeRegion(&code, src, i+5)
}
return true
}
// removeIncludeInjections removes @include lines that load malicious files.
// Catches:
// - @include("/tmp/...") - literal paths to temp dirs
// - @include(base64_decode("...")) - encoded includes
// - @include($var) where $var is built from obfuscated strings nearby
func removeIncludeInjections(content string) (string, []string) {
var removals []string
lines := strings.Split(content, "\n")
var clean []string
for i, line := range lines {
if cleanRegexpMaliciousInclude.MatchString(line) {
removals = append(removals, fmt.Sprintf("removed @include injection: %s", trimCleanRemovalLine(line)))
continue
}
// Variable-based @include - check surrounding context for obfuscation
if cleanRegexpVarInclude.MatchString(line) {
context := getLineContext(lines, i, 3)
contextLower := strings.ToLower(context)
isObfuscated := strings.Contains(contextLower, "base64_decode") ||
strings.Contains(contextLower, "str_rot13") ||
strings.Contains(contextLower, "chr(") ||
strings.Contains(contextLower, `"\x`) ||
strings.Count(contextLower, ". ") > 5 // heavy string concatenation
if isObfuscated {
removals = append(removals, fmt.Sprintf("removed obfuscated @include: %s", trimCleanRemovalLine(line)))
continue
}
}
clean = append(clean, line)
}
return strings.Join(clean, "\n"), removals
}
func trimCleanRemovalLine(line string) string {
return strings.TrimFunc(line, func(r rune) bool {
return r == '\x00' || unicode.IsSpace(r)
})
}
// removePrependInjection removes malicious PHP code injected before the
// legitimate file content. Uses entropy analysis to verify the prefix
// is actually obfuscated (not legitimate minified code).
func removePrependInjection(content string) (string, []string) {
var removals []string
trimmed := strings.TrimSpace(content)
// PHP open tags are case-insensitive (<?php, <?PHP, <?Php all execute), so
// match without lowercasing the whole file.
if len(trimmed) < 5 || !strings.EqualFold(trimmed[:5], "<?php") {
return content, nil
}
// Find if there's a malicious block at the start followed by ?><?php
loc := cleanRegexpCloseOpen.FindStringIndex(content)
if loc == nil {
return content, nil
}
prefix := content[:loc[0]]
prefixLower := strings.ToLower(prefix)
// Check if the prefix contains malicious patterns
hasMaliciousPatterns := strings.Contains(prefixLower, "eval(") ||
strings.Contains(prefixLower, "base64_decode") ||
strings.Contains(prefixLower, "gzinflate") ||
strings.Contains(prefixLower, "str_rot13") ||
strings.Contains(prefixLower, "@include")
if !hasMaliciousPatterns {
return content, nil
}
// Additional safety: verify the prefix has high entropy (obfuscated code)
// or contains long encoded strings. This prevents false positives on
// legitimate minified PHP that happens to use ?><?php patterns.
entropy := shannonEntropy(prefix)
hasLongStrings := containsLongEncodedString(prefix, 100)
if entropy < 4.5 && !hasLongStrings {
// Low entropy and no long encoded strings - likely legitimate code
return content, nil
}
// Remove everything before the second <?php
cleaned := "<?php" + content[loc[1]:]
removals = append(removals, fmt.Sprintf("removed %d-byte prepend injection (entropy: %.2f)", loc[1], entropy))
return cleaned, removals
}
// removeAppendInjection removes malicious code appended after the end of
// legitimate PHP content. Handles both files with closing ?> and files
// without (PSR-12 style).
func removeAppendInjection(content string) (string, []string) {
var removals []string
// Case 1: File has closing ?> with malicious code after it
lastClose := strings.LastIndex(content, "?>")
if lastClose >= 0 {
after := content[lastClose+2:]
afterTrimmed := strings.TrimSpace(after)
if afterTrimmed != "" {
afterLower := strings.ToLower(afterTrimmed)
isMalicious := strings.Contains(afterLower, "eval(") ||
strings.Contains(afterLower, "base64_decode") ||
strings.Contains(afterLower, "gzinflate") ||
strings.Contains(afterLower, "system(") ||
strings.Contains(afterLower, "exec(") ||
strings.Contains(afterLower, "@include") ||
strings.Contains(afterLower, "<?php") // second PHP block appended
if isMalicious {
cleaned := content[:lastClose+2] + "\n"
removals = append(removals, fmt.Sprintf("removed %d-byte append injection (after ?>)", len(after)))
return cleaned, removals
}
}
}
// Case 2: PSR-12 style file (no closing ?>) - check if there's a malicious
// block appended at the very end, separated by multiple newlines
lines := strings.Split(content, "\n")
if len(lines) < 5 {
return content, nil
}
// Check last 10 lines for injected code block
startCheck := len(lines) - 10
if startCheck < 0 {
startCheck = 0
}
blankLineIdx := -1
for i := startCheck; i < len(lines); i++ {
if strings.TrimSpace(lines[i]) == "" && blankLineIdx < 0 {
blankLineIdx = i
}
}
if blankLineIdx >= 0 {
tailBlock := strings.Join(lines[blankLineIdx:], "\n")
tailLower := strings.ToLower(tailBlock)
if strings.Contains(tailLower, "eval(") && strings.Contains(tailLower, "base64_decode") {
cleaned := strings.Join(lines[:blankLineIdx], "\n") + "\n"
removals = append(removals, fmt.Sprintf("removed %d-byte PSR-12 append injection", len(tailBlock)))
return cleaned, removals
}
}
return content, nil
}
// removeInlineEvalInjections removes single-line eval(base64_decode("..."));
// injections that are inserted as standalone lines in PHP files.
func removeInlineEvalInjections(content string) (string, []string) {
var removals []string
lines := strings.Split(content, "\n")
var clean []string
for _, line := range lines {
trimmedLine := strings.TrimSpace(line)
// Strip /* ... */ block comments and trailing // / # comments
// before matching. Attackers wedge comments between the keyword
// and the open paren ("@eval/*x*/(base64_decode(...))") to slip
// past a strict regex; the cleaner has to see the line the way
// the PHP tokenizer does, not byte-for-byte.
normalized := stripPHPComments(trimmedLine)
if cleanRegexpInlineEval.MatchString(normalized) {
// Length gate stays on the ORIGINAL line so a short legitimate
// eval() does not get sucked into the removal path.
if len(trimmedLine) > 50 {
removals = append(removals, fmt.Sprintf("removed inline eval injection (%d chars)", len(trimmedLine)))
continue
}
}
clean = append(clean, line)
}
return strings.Join(clean, "\n"), removals
}
// stripPHPComments removes PHP comments while leaving quoted strings
// intact, so comment-looking payload data does not change the code the
// cleaner evaluates.
func stripPHPComments(line string) string {
return strings.TrimSpace(stripPHPCommentsFromCode(line))
}
// --- Helper functions ---
// shannonEntropy calculates the Shannon entropy of a string.
// Obfuscated/encoded code typically has entropy > 5.0.
// Normal PHP code typically has entropy 4.0-4.5.
func shannonEntropy(s string) float64 {
if len(s) == 0 {
return 0
}
freq := make(map[byte]float64)
for i := 0; i < len(s); i++ {
freq[s[i]]++
}
length := float64(len(s))
entropy := 0.0
for _, count := range freq {
p := count / length
if p > 0 {
entropy -= p * math.Log2(p)
}
}
return entropy
}
// containsLongEncodedString checks if the text contains a long base64-like
// string (alphanumeric + /+ without spaces).
func containsLongEncodedString(s string, minLength int) bool {
if minLength <= 0 {
return true
}
run := 0
for i := 0; i < len(s); i++ {
if isEncodedStringByte(s[i]) {
run++
if run >= minLength {
return true
}
continue
}
run = 0
}
return false
}
func isEncodedStringByte(b byte) bool {
return b >= 'A' && b <= 'Z' ||
b >= 'a' && b <= 'z' ||
b >= '0' && b <= '9' ||
b == '+' || b == '/' || b == '='
}
// getLineContext returns N lines before and after the given line index.
func getLineContext(lines []string, idx, window int) string {
start := idx - window
if start < 0 {
start = 0
}
end := idx + window + 1
if end > len(lines) {
end = len(lines)
}
return strings.Join(lines[start:end], "\n")
}
// FormatCleanResult returns a human-readable summary of a clean operation.
func FormatCleanResult(r CleanResult) string {
if r.Refused {
return fmt.Sprintf("REFUSED to clean %s: %s", r.Path, r.Error)
}
if r.Error != "" {
return fmt.Sprintf("FAILED to clean %s: %s", r.Path, r.Error)
}
if !r.Cleaned {
return fmt.Sprintf("No changes made to %s", r.Path)
}
var b strings.Builder
fmt.Fprintf(&b, "CLEANED %s\n", r.Path)
fmt.Fprintf(&b, " Backup: %s\n", r.BackupPath)
for _, removal := range r.Removals {
fmt.Fprintf(&b, " - %s\n", removal)
}
return b.String()
}
// --- Strategy 5: Multi-layer base64 decode chains ---
// Catches: eval(base64_decode(base64_decode("...")))
// Catches: $x=base64_decode("...");$y=base64_decode($x);eval($y);
func removeMultiLayerBase64(content string) (string, []string) {
var removals []string
lines := strings.Split(content, "\n")
var clean []string
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if len(trimmed) < 80 {
clean = append(clean, line)
continue
}
if cleanRegexpMultiB64.MatchString(trimmed) || cleanRegexpChainedB64.MatchString(trimmed) {
removals = append(removals, fmt.Sprintf("removed multi-layer base64 chain (%d chars)", len(trimmed)))
continue
}
clean = append(clean, line)
}
return strings.Join(clean, "\n"), removals
}
// --- Strategy 6: chr()/pack() constructed code ---
// Catches: eval(chr(115).chr(121).chr(115)...);
// Catches: $f = pack("H*", "73797374656d"); $f($_POST['cmd']);
func removeChrPackInjections(content string) (string, []string) {
var removals []string
lines := strings.Split(content, "\n")
// Find chr-chain spans across the full content first so multi-line
// chains (chr(115)\n.chr(121)\n...) are recognized as one chain
// instead of slipping past the per-line 5+ count.
dropLine := make(map[int]bool)
offsets := make([]int, len(lines))
off := 0
for i, line := range lines {
offsets[i] = off
off += len(line) + 1 // +1 for the newline strings.Split consumed
}
for _, span := range cleanRegexpChrChain.FindAllStringIndex(content, -1) {
startLine, endLine := chrChainStatementLineRange(lines, offsets, span)
for i := startLine; i <= endLine; i++ {
dropLine[i] = true
}
}
var clean []string
for i, line := range lines {
if dropLine[i] {
removals = append(removals, fmt.Sprintf("removed chr() chain injection (line %d, %d chars)", i+1, len(strings.TrimSpace(line))))
continue
}
trimmed := strings.TrimSpace(line)
if cleanRegexpPackHex.MatchString(trimmed) {
lower := strings.ToLower(trimmed)
// Only remove if combined with execution
if strings.Contains(lower, "eval") || strings.Contains(lower, "$_") ||
strings.Contains(lower, "system") || strings.Contains(lower, "exec") {
removals = append(removals, fmt.Sprintf("removed pack() code construction (line %d)", i+1))
continue
}
}
clean = append(clean, line)
}
return strings.Join(clean, "\n"), removals
}
func chrChainStatementLineRange(lines []string, offsets []int, span []int) (int, int) {
startLine := lineIndexForOffset(offsets, span[0])
endLine := lineIndexForOffset(offsets, span[1]-1)
for startLine > 0 && chrChainPrefixContinues(lines[startLine-1]) {
startLine--
}
for endLine+1 < len(lines) && !strings.Contains(lines[endLine], ";") && chrChainSuffixContinues(lines[endLine+1]) {
endLine++
if strings.Contains(lines[endLine], ";") {
break
}
}
return startLine, endLine
}
func chrChainPrefixContinues(line string) bool {
trimmed := strings.TrimSpace(line)
if trimmed == "" || trimmed == "<?php" {
return false
}
return strings.HasSuffix(trimmed, "=") ||
strings.HasSuffix(trimmed, ".") ||
strings.HasSuffix(trimmed, "(") ||
strings.HasSuffix(trimmed, ",") ||
strings.HasSuffix(trimmed, "[")
}
func chrChainSuffixContinues(line string) bool {
trimmed := strings.TrimSpace(line)
if trimmed == "" {
return true
}
return strings.HasPrefix(trimmed, ".") ||
strings.HasPrefix(trimmed, ")") ||
strings.HasPrefix(trimmed, ",") ||
strings.HasPrefix(trimmed, ";")
}
// lineIndexForOffset returns the index of the line containing the byte
// at offset within the original content.
func lineIndexForOffset(offsets []int, byteOffset int) int {
if len(offsets) == 0 || byteOffset < 0 {
return 0
}
idx := sort.Search(len(offsets), func(i int) bool {
return offsets[i] > byteOffset
}) - 1
if idx < 0 {
return 0
}
if idx >= len(offsets) {
return len(offsets) - 1
}
return idx
}
// --- Strategy 7: Hex-encoded variable injections ---
// Catches: $GLOBALS["\x61\x64\x6d\x69\x6e"] = eval(...)
// Catches: ${"\x47\x4c\x4f\x42\x41\x4c\x53"}[...] = ...
func removeHexVarInjections(content string) (string, []string) {
var removals []string
lines := strings.Split(content, "\n")
var clean []string
for i, line := range lines {
trimmed := strings.TrimSpace(line)
if len(trimmed) < 30 {
clean = append(clean, line)
continue
}
if cleanRegexpHexVar.MatchString(trimmed) {
lower := strings.ToLower(trimmed)
// Only remove if combined with dangerous operations
if strings.Contains(lower, "eval") || strings.Contains(lower, "system(") ||
strings.Contains(lower, "exec(") || strings.Contains(lower, "base64_decode") ||
strings.Contains(lower, "assert(") || strings.Contains(lower, "$_post") ||
strings.Contains(lower, "$_request") || strings.Contains(lower, "$_get") {
removals = append(removals, fmt.Sprintf("removed hex-encoded variable injection (line %d, %d chars)", i+1, len(trimmed)))
continue
}
}
clean = append(clean, line)
}
return strings.Join(clean, "\n"), removals
}
package checks
import (
"errors"
"fmt"
"github.com/pidginhost/csm/internal/actionlog"
)
type cleanAction struct{ rec actionlog.Record }
func newCleanAction(path string) *cleanAction {
return &cleanAction{rec: actionlog.Record{Op: "respond.clean_file", Target: path, Before: actionlog.Metadata(path), Result: actionlog.Refused}}
}
func (a *cleanAction) capture(target *cleanTarget, data []byte) {
a.rec.Target = target.Path
a.rec.Before = actionlog.ContentState(target.Info, data)
}
func (a *cleanAction) replace(target *cleanTarget, content []byte, backupPath string) error {
a.rec.Result = actionlog.Failed
a.rec.RecoveryPath = backupPath
err := writeCleanedFileAtomic(target, content)
// Rename can succeed even when the following directory sync fails. The
// record must still describe the installed bytes and retained recovery copy.
if target.installed {
a.rec.Result = actionlog.Applied
a.rec.After = actionlog.ContentState(target.replacementInfo, content)
if a.rec.Before != nil {
rememberRemovedFile(target.Path, a.rec.Before.Digest, a.rec.Before.Size, "clean")
}
if err != nil {
err = fmt.Errorf("%w; recovery backup retained", err)
}
} else if errors.Is(err, errFileResponseRefused) {
a.rec.Result = actionlog.Refused
}
return err
}
func (a *cleanAction) finish(message string) {
a.rec.Error = message
if a.rec.After == nil {
a.rec.After = actionlog.Metadata(a.rec.Target)
}
actionlog.Write(a.rec)
}
package checks
import (
"context"
"errors"
"io"
"os"
)
const maxCMSConfigBytes = 1 << 20
var errCMSConfigTooLarge = errors.New("CMS configuration exceeds the read limit")
func readCMSConfig(ctx context.Context, path string) ([]byte, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
file, err := openTenantRegularFile(path)
if err != nil {
return nil, err
}
defer file.Close()
before, err := file.Stat()
if err != nil {
return nil, err
}
if before.Size() > maxCMSConfigBytes {
return nil, errCMSConfigTooLarge
}
data, err := io.ReadAll(io.LimitReader(cmsConfigReader{ctx: ctx, file: file}, maxCMSConfigBytes+1))
if err != nil {
return nil, err
}
if err = ctx.Err(); err != nil {
return nil, err
}
if len(data) > maxCMSConfigBytes {
return nil, errCMSConfigTooLarge
}
after, err := file.Stat()
if err != nil {
return nil, err
}
if !sameFileSnapshot(before, after) || int64(len(data)) != before.Size() {
return nil, errFileChanged
}
return data, nil
}
type cmsConfigReader struct {
ctx context.Context
file *os.File
}
func (r cmsConfigReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.file.Read(p)
}
package checks
import (
"crypto/sha256"
"encoding/hex"
"io"
"sync"
)
// CMSHashCache stores SHA256 hashes of verified-clean CMS core files.
// After wp core verify-checksums confirms an installation is clean,
// all its core files are hashed and cached. The real-time scanner
// checks this cache before reporting signature matches - if a file's
// hash is in the cache, it's a known-clean CMS file and signature
// matches on it are false positives.
type CMSHashCache struct {
mu sync.RWMutex
hashes map[string]bool // SHA256 hex -> true
sizes map[int64]bool // file sizes that can possibly match a cached hash
}
var (
globalCache *CMSHashCache
globalCacheOnce sync.Once
)
// GlobalCMSCache returns the singleton cache, creating it on first call.
func GlobalCMSCache() *CMSHashCache {
globalCacheOnce.Do(func() {
globalCache = &CMSHashCache{
hashes: make(map[string]bool),
sizes: make(map[int64]bool),
}
})
return globalCache
}
// Add inserts a file hash and its content size into the cache.
func (c *CMSHashCache) Add(hash string, size int64) {
c.mu.Lock()
c.hashes[hash] = true
c.sizes[size] = true
c.mu.Unlock()
}
// Contains checks if a file hash is in the cache.
func (c *CMSHashCache) Contains(hash string) bool {
c.mu.RLock()
ok := c.hashes[hash]
c.mu.RUnlock()
return ok
}
// Size returns the number of cached hashes.
func (c *CMSHashCache) Size() int {
c.mu.RLock()
n := len(c.hashes)
c.mu.RUnlock()
return n
}
// MayContainSize reports whether a verified file of size bytes was cached.
// It lets realtime scanning reject attacker-sized files before hashing them.
func (c *CMSHashCache) MayContainSize(size int64) bool {
c.mu.RLock()
ok := c.sizes[size]
c.mu.RUnlock()
return ok
}
// Clear removes all cached hashes (used before rebuilding).
func (c *CMSHashCache) Clear() {
c.mu.Lock()
c.hashes = make(map[string]bool)
c.sizes = make(map[int64]bool)
c.mu.Unlock()
}
// HashFile computes the SHA256 hash of a file. Returns empty string on error.
func HashFile(path string) string {
f, err := osFS.Open(path)
if err != nil {
return ""
}
defer func() { _ = f.Close() }()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return ""
}
return hex.EncodeToString(h.Sum(nil))
}
// IsVerifiedCMSFile checks if a file at the given path matches a
// known-clean CMS core file by comparing its SHA256 hash against the cache.
//
// The cache is keyed by SHA256 hash alone (not path+hash) - this is correct:
// - SHA256 preimage resistance makes it computationally infeasible for an
// attacker to craft a file that produces the same hash as a legitimate
// WP core file. Birthday attacks do not apply here because the attacker
// must hit a specific pre-existing hash, not merely find any collision.
// - If file content matches a known WP core file byte-for-byte, it IS that
// file regardless of where it is located on disk. The path is irrelevant
// to whether the content is clean.
func IsVerifiedCMSFile(path string) bool {
cache := GlobalCMSCache()
if cache.Size() == 0 {
return false
}
hash := HashFile(path)
if hash == "" {
return false
}
return cache.Contains(hash)
}
// IsVerifiedCMSHash reports whether a content hash belongs to a verified CMS
// core file. Callers that already hold the content -- the realtime scanner
// hashes the event descriptor -- use this instead of IsVerifiedCMSFile, whose
// re-read resolves the path again and can hash something other than what was
// examined.
func IsVerifiedCMSHash(hash string) bool {
if hash == "" {
return false
}
cache := GlobalCMSCache()
if cache.Size() == 0 {
return false
}
return cache.Contains(hash)
}
// CMSCacheEmpty reports whether any verified core files are cached, so a caller
// can skip hashing entirely when the answer cannot be yes.
func CMSCacheEmpty() bool { return GlobalCMSCache().Size() == 0 }
// CMSCacheMayContainSize reports whether hashing a file of size bytes can
// possibly produce a cached CMS hash.
func CMSCacheMayContainSize(size int64) bool { return GlobalCMSCache().MayContainSize(size) }
package checks
import (
"bytes"
"container/heap"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/contenttype"
"github.com/pidginhost/csm/internal/phptaint"
"github.com/pidginhost/csm/internal/yara"
)
// companionPlantWindow is how far another file's change time may lie from the
// detected file's and still count as planted in the same session. A dropper or
// a scripted upload writes a whole kit within seconds; a person driving a
// webshell's file manager uploads a handful of files within a few minutes. Ten
// minutes either side covers both with margin and stays short enough that
// unrelated writes to the directory, hours or days apart, fall outside it.
// Change time is used because an unprivileged writer cannot set it; kits
// backdate modification times.
const companionPlantWindow = 10 * time.Minute
// companionClaimTTL is how long a swept plant window stays claimed. A finding
// that persists is alerted again after a day, and the sweep repeats with it.
const companionClaimTTL = 24 * time.Hour
// companionSampleLimit bounds how many files one report names. The report
// always states the full count.
const companionSampleLimit = 25
// companionBackdateGap is how much older than its change time a modification
// time must be before the report marks it. Unpacking an archive keeps old
// modification times too, so the mark is a hint, not proof.
const companionBackdateGap = 24 * time.Hour
// companionReadDirBatch is how many entries one directory read returns, so a
// huge directory is streamed rather than held in memory.
const companionReadDirBatch = 512
// companionSweepTriggers are the findings that prove one file malicious by its
// content, a scanner rule or a known backdoor name. Only a Critical one starts
// a sweep. Findings that rest on location alone are left out: their file may
// be legitimate, and so would its neighbours be.
var companionSweepTriggers = map[string]bool{
"backdoor_binary": true,
"cgi_backdoor_realtime": true,
"js_keylogger_dataflow": true,
"new_webshell_file": true,
"obfuscated_php": true,
"obfuscated_php_realtime": true,
"php_dropper_realtime": true,
"php_in_image_realtime": true,
"php_in_uploads_realtime": true,
"php_remote_taint": true,
"signature_match_realtime": true,
"suspicious_php_content": true,
"webshell": true,
"webshell_content_realtime": true,
"webshell_realtime": true,
"yara_match_realtime": true,
"yara_match_scheduled": true,
}
// companionSweeps is the daemon's queue. Detections and quarantines feed it;
// RunCompanionSweeps drains it.
var companionSweeps = newCompanionSweepQueue()
// QueueCompanionSweeps queues a sweep for every confirmed malicious file in
// findings that is still present. A file already moved to quarantine was
// queued by the quarantine itself, with the stat taken before the move.
func QueueCompanionSweeps(findings []alert.Finding) {
companionSweeps.observe(findings, time.Now())
}
// RunCompanionSweeps sweeps queued directories until ctx ends, one at a time,
// and hands each result to emit.
func RunCompanionSweeps(ctx context.Context, cfg func() *config.Config, emit func([]alert.Finding)) {
companionSweeps.run(ctx, cfg, emit)
}
// companionSweepFindings sweeps at once for every confirmed detection in
// findings, for a scan that returns a report instead of alerting.
func companionSweepFindings(ctx context.Context, cfg *config.Config, findings []alert.Finding) []alert.Finding {
q := newCompanionSweepQueue()
now := time.Now()
for i := range findings {
if ctx.Err() != nil {
return nil
}
q.observe(findings[i:i+1], now)
}
var out []alert.Finding
for _, a := range q.pending {
if ctx.Err() != nil {
break
}
out = append(out, sweepCompanions(ctx, cfg, *a)...)
}
return out
}
// companionAnchor is one detected file and the plant window around it.
type companionAnchor struct {
path string
info os.FileInfo
changed time.Time
due time.Time
// detected holds other detections in the same window; they have their own
// findings and are not reported again.
detected map[[2]uint64]companionDetection
detectedMu *sync.Mutex
claim *companionClaim
}
type companionDetection struct {
info os.FileInfo
at time.Time
}
func newCompanionAnchor(path string, info os.FileInfo, now time.Time) (companionAnchor, bool) {
changed, ok := changeTimeOf(info)
if !ok || !filepath.IsAbs(path) || !info.Mode().IsRegular() {
return companionAnchor{}, false
}
// A file caught as it is written may be the first of its kit to land, so
// its sweep waits for the window to close. A change time in the future
// (clock step) cannot postpone it past one window from now.
due := changed.Add(companionPlantWindow)
if limit := now.Add(companionPlantWindow); due.After(limit) {
due = limit
}
a := companionAnchor{path: filepath.Clean(path), info: info, changed: changed, due: due,
detected: make(map[[2]uint64]companionDetection), detectedMu: new(sync.Mutex)}
a.rememberAt(info, now)
return a, true
}
func (a companionAnchor) isDetected(info os.FileInfo) bool {
id, ok := selfWriteIdentityFromFileInfo(info)
if !ok {
return false
}
a.detectedMu.Lock()
defer a.detectedMu.Unlock()
return sameDetectedFile(a.info, info) || sameDetectedFile(a.detected[[2]uint64{id.Device, id.Inode}].info, info)
}
func (a companionAnchor) rememberAt(info os.FileInfo, now time.Time) {
id, ok := selfWriteIdentityFromFileInfo(info)
if !ok {
return
}
a.detectedMu.Lock()
defer a.detectedMu.Unlock()
a.detected[[2]uint64{id.Device, id.Inode}] = companionDetection{info: info, at: now}
}
// sameDetectedFile also compares size and modification time: the detected file
// may be gone, and a new file can reuse its inode number.
func sameDetectedFile(a, b os.FileInfo) bool {
if !sameFileIdentity(a, b) || !sameContentShape(a, b) || a.Mode() != b.Mode() || !sameFileOwner(a, b) {
return false
}
ac, aok := changeTimeOf(a)
bc, bok := changeTimeOf(b)
return aok && bok && ac.Equal(bc)
}
type companionClaim struct {
changed time.Time
at time.Time
// pending is the queued sweep for this window, nil once it has started.
pending *companionAnchor
anchor companionAnchor
}
type companionDirectoryKey struct {
dir string
uid uint32
}
// companionSweepQueue holds sweeps until their window closes and claims each
// plant window of a directory once, so the other members of a kit, repeated
// alerts and the quarantine of a detected file do not sweep it again.
type companionSweepQueue struct {
mu sync.Mutex
pending companionHeap
claims map[companionDirectoryKey][]*companionClaim
pruned time.Time
wake chan struct{}
closed bool
}
func newCompanionSweepQueue() *companionSweepQueue {
return &companionSweepQueue{claims: map[companionDirectoryKey][]*companionClaim{}, wake: make(chan struct{}, 1)}
}
func (q *companionSweepQueue) observe(findings []alert.Finding, now time.Time) {
for _, f := range findings {
if f.Severity != alert.Critical || !companionSweepTriggers[f.Check] || f.FilePath == "" {
continue
}
info, err := osFS.Lstat(f.FilePath)
if err != nil || !info.Mode().IsRegular() {
continue
}
q.add(f.FilePath, info, now)
}
}
// add queues a sweep for path unless its directory already has one claimed
// for an overlapping window. It reports whether a sweep was queued.
func (q *companionSweepQueue) add(path string, info os.FileInfo, now time.Time) bool {
a, ok := newCompanionAnchor(path, info, now)
if !ok {
return false
}
dir := companionDirectoryKey{dir: filepath.Dir(a.path), uid: info.Sys().(*syscall.Stat_t).Uid}
q.mu.Lock()
defer q.mu.Unlock()
if q.closed {
return false
}
q.pruneLocked(now)
claims := q.claims[dir]
live := claims[:0]
claimed := false
for _, c := range claims {
if c.pending == nil && now.Sub(c.at) > companionClaimTTL {
continue
}
live = append(live, c)
if c.anchor.isDetected(info) {
claimed = true
}
if gap := a.changed.Sub(c.changed); !claimed && gap <= companionPlantWindow && gap >= -companionPlantWindow {
claimed = true
c.anchor.rememberAt(info, now)
}
}
clear(claims[len(live):])
if claimed {
q.claims[dir] = live
return false
}
if len(live) > 0 {
// Overlapping windows share exact file versions. A result emitted by
// one sweep must neither be emitted again nor enqueue another sweep.
a.detected, a.detectedMu = live[0].anchor.detected, live[0].anchor.detectedMu
a.rememberAt(info, now)
}
queued := &a
queued.claim = &companionClaim{changed: a.changed, at: now, pending: queued, anchor: a}
q.claims[dir] = append(live, queued.claim)
heap.Push(&q.pending, queued)
select {
case q.wake <- struct{}{}:
default:
}
return true
}
// next returns the earliest sweep that is due. Otherwise it returns how long
// until one is, or zero when nothing is queued.
func (q *companionSweepQueue) next(now time.Time) (companionAnchor, time.Duration, bool) {
q.mu.Lock()
defer q.mu.Unlock()
q.pruneLocked(now)
if len(q.pending) == 0 {
return companionAnchor{}, 0, false
}
top := q.pending[0]
if top.due.After(now) {
return companionAnchor{}, top.due.Sub(now), false
}
heap.Pop(&q.pending)
top.claim.pending = nil
top.claim.at = now
return *top, 0, true
}
// pruneLocked drops expired claims of directories that saw no new detection.
// It runs at most once per window so a burst of detections stays linear.
func (q *companionSweepQueue) pruneLocked(now time.Time) {
if now.Sub(q.pruned) < companionPlantWindow {
return
}
q.pruned = now
prunedDetections := make(map[*sync.Mutex]bool)
for dir, claims := range q.claims {
live := claims[:0]
for _, c := range claims {
if !prunedDetections[c.anchor.detectedMu] {
prunedDetections[c.anchor.detectedMu] = true
c.anchor.detectedMu.Lock()
for key, detection := range c.anchor.detected {
if now.Sub(detection.at) > companionClaimTTL {
delete(c.anchor.detected, key)
}
}
c.anchor.detectedMu.Unlock()
}
if c.pending != nil || now.Sub(c.at) <= companionClaimTTL {
live = append(live, c)
}
}
clear(claims[len(live):])
if len(live) == 0 {
delete(q.claims, dir)
} else {
q.claims[dir] = live
}
}
}
func (q *companionSweepQueue) run(ctx context.Context, cfg func() *config.Config, emit func([]alert.Finding)) {
defer func() {
q.mu.Lock()
defer q.mu.Unlock()
q.closed = true
q.pending = nil
clear(q.claims)
}()
timer := time.NewTimer(time.Hour)
timer.Stop()
defer timer.Stop()
for ctx.Err() == nil {
a, wait, ok := q.next(time.Now())
if ok {
sweepCompanionsTo(ctx, cfg(), a, emit)
continue
}
if wait <= 0 || wait > companionPlantWindow {
wait = companionPlantWindow
}
timer.Reset(wait)
select {
case <-ctx.Done():
case <-q.wake:
case <-timer.C:
}
}
}
type companionHeap []*companionAnchor
func (h companionHeap) Len() int { return len(h) }
func (h companionHeap) Less(i, j int) bool { return h[i].due.Before(h[j].due) }
func (h companionHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *companionHeap) Push(x any) { *h = append(*h, x.(*companionAnchor)) }
func (h *companionHeap) Pop() any {
old := *h
last := old[len(old)-1]
old[len(old)-1] = nil
*h = old[:len(old)-1]
return last
}
// plantedFile is one unmatched file of a plant window.
type plantedFile struct {
name string
kind string
note string
size int64
changed time.Time
modified time.Time
}
// sweepCompanions examines the detected file's own directory, without
// descending. The first pass only stats entries: the content scan is bounded
// to files written in the plant window, which an attacker had to write and the
// normal scans read anyway, plus the directory's .htaccess. Matches are
// ordinary findings; files that match nothing but are of a kind a kit carries
// are named together in one report. Nothing is quarantined here.
func sweepCompanions(ctx context.Context, cfg *config.Config, a companionAnchor) []alert.Finding {
var findings []alert.Finding
sweepCompanionsTo(ctx, cfg, a, func(batch []alert.Finding) { findings = append(findings, batch...) })
return findings
}
func sweepCompanionsTo(parent context.Context, cfg *config.Config, a companionAnchor, emit func([]alert.Finding)) {
if parent.Err() != nil {
return
}
ctx, cancel := context.WithTimeout(parent, timeoutFor("malware_companion_files"))
defer cancel()
dir := filepath.Dir(a.path)
defer func() {
if ctx.Err() == context.DeadlineExceeded && parent.Err() == nil {
emit([]alert.Finding{{Severity: alert.Warning, Check: "check_timeout", FilePath: dir,
Message: "Companion scan did not finish: " + dir, Timestamp: time.Now()}})
}
}()
slot := scanBudgetFrom(ctx).acquire(ctx)
if slot == nil {
return
}
defer func() { slot.release() }()
dirInfo, err := osFS.Lstat(dir)
if err != nil || !dirInfo.IsDir() {
return
}
d, err := openTenantDirectory(dir)
if err != nil {
return
}
defer func() { _ = d.Close() }()
if opened, statErr := d.Stat(); statErr != nil || !opened.IsDir() || !sameFileIdentity(dirInfo, opened) {
return
}
from, to := a.changed.Add(-companionPlantWindow), a.changed.Add(companionPlantWindow)
maxBytes := FullScanMaxFileBytes(cfg)
if limit := scanMaxFileBytes(ctx); limit > 0 {
maxBytes = limit
}
respectIgnores := scanRespectsIgnores(ctx, cfg)
var findings []alert.Finding
var planted []plantedFile
total := 0
failed := 0
failureExample := ""
flush := func() {
// Alert delivery can wait on a busy dispatcher. It must not retain a
// scan slot or all matches from a tenant's directory while waiting.
slot.release()
slot = nil
if len(findings) > 0 && parent.Err() == nil {
now := time.Now()
for i := range findings {
if findings[i].Timestamp.IsZero() {
findings[i].Timestamp = now
}
}
emit(findings)
}
findings = nil
}
defer flush()
defer func() {
if failed > 0 {
findings = append(findings, alert.Finding{Severity: alert.Warning, Check: "malware_companion_scan_incomplete", FilePath: dir,
Message: "Companion content scan incomplete: " + dir,
Details: fmt.Sprintf("Could not fully scan %d file(s). Example: %s", failed, failureExample)})
}
}()
for {
if ctx.Err() != nil {
return
}
if slot == nil {
slot = scanBudgetFrom(ctx).acquire(ctx)
if slot == nil {
return
}
}
entries, readErr := d.ReadDir(companionReadDirBatch)
for _, entry := range entries {
if ctx.Err() != nil {
return
}
if !entry.Type().IsRegular() {
continue
}
path := filepath.Join(dir, entry.Name())
if respectIgnores && PathMatchesIgnore(path, cfg.Suppressions.IgnorePaths) {
continue
}
info, err := osFS.Lstat(path)
// An unprivileged writer can only create files it owns, so a file
// of another owner cannot be part of the same drop.
if err != nil || !info.Mode().IsRegular() || !sameFileOwner(info, a.info) || a.isDetected(info) {
continue
}
changed, _ := changeTimeOf(info)
inWindow := !changed.Before(from) && !changed.After(to)
if !inWindow && entry.Name() != ".htaccess" {
continue
}
matches, file := scanCompanionFile(ctx, cfg, d, path, info, maxBytes)
findings = append(findings, matches...)
if file.note != "" {
failed++
if failureExample == "" {
failureExample = strconv.QuoteToASCII(path) + ": " + file.note
}
}
if len(matches) > 0 {
a.rememberAt(info, time.Now())
}
matched := false
for _, f := range matches {
if f.Check != "php_taint_scan_incomplete" {
matched = true
}
}
if inWindow && !matched && file.kind != "" {
total++
if len(planted) < companionSampleLimit {
planted = append(planted, file)
}
}
}
if readErr != nil {
if !errors.Is(readErr, io.EOF) {
findings = append(findings, alert.Finding{Severity: alert.Warning, Check: "malware_companion_scan_incomplete", FilePath: dir,
Message: "Companion scan could not finish reading directory: " + dir, Details: readErr.Error()})
}
break
}
flush()
}
if total > 0 && ctx.Err() == nil {
findings = append(findings, plantedFilesFinding(a, planted, total))
}
}
// scanCompanionFile runs the content engines over one file regardless of any
// scan cache or modification-time skip.
func scanCompanionFile(ctx context.Context, cfg *config.Config, dir *tenantDirectory, path string, info os.FileInfo, maxBytes int64) ([]alert.Finding, plantedFile) {
name := filepath.Base(path)
changed, _ := changeTimeOf(info)
file := plantedFile{name: name, size: info.Size(), changed: changed, modified: info.ModTime()}
data, note := readCompanionFileFrom(ctx, dir, path, info, maxBytes)
file.kind = companionKind(name, info, data)
if note != "" {
file.note = note
return nil, file
}
digest := sha256.Sum256(data)
contentSHA256 := hex.EncodeToString(digest[:])
var findings []alert.Finding
if !yaraDeepConsumerDisabled(cfg) {
backend := activeYARABackend()
switch {
case backend != nil && backend.RuleCount() > 0:
matches, err := yara.ScanBytesChecked(backend, path, data)
if err != nil {
file.note = "YARA could not scan it"
}
for _, m := range matches {
findings = append(findings, scheduledYARAFinding(m, path, data, contentSHA256))
}
case yaraAvailable():
// YARA should have checked this file and could not: say so
// rather than list it as if it had been scanned.
file.note = "YARA could not scan it"
}
}
if contenttype.IsPHPSourceName(strings.ToLower(name)) && !runnerDisabled(cfg, "php_content") {
if result := analyzePHPContentReaderAt(path, bytes.NewReader(data), int64(len(data))); result.severity >= 0 {
findings = append(findings, phpContentFinding(path, result, info, contentSHA256))
}
}
if len(data) <= phptaint.MaxSourceBytes && !phpTaintDeepConsumerDisabled(cfg) && phpTaintAnalyzerReady() {
gaps := newPHPTaintGapCollector()
findings = append(findings, analyzePHPTaintSnapshot(ctx, path, contentSHA256, data, gaps)...)
findings = append(findings, gaps.findings()...)
}
if name == ".htaccess" && !runnerDisabled(cfg, "htaccess") {
if len(data) > htaccessMaxFileBytes {
file.note = "too large to audit"
} else {
audit, _ := auditHtaccessContent(ctx, path, data)
findings = append(findings, audit...)
}
}
return findings, file
}
// readCompanionFile reads the file the stat pass selected, refusing a symlink,
// a special file or a different object swapped into its place.
func readCompanionFile(path string, info os.FileInfo, maxBytes int64) ([]byte, string) {
dir, err := openTenantDirectory(filepath.Dir(path))
if err != nil {
return nil, "could not be opened"
}
defer func() { _ = dir.Close() }()
return readCompanionFileFrom(context.Background(), dir, path, info, maxBytes)
}
// companionFileStat lets filesystem providers supply matching metadata for
// both the selection pass and the opened descriptor in tests.
var companionFileStat = (*os.File).Stat
func readCompanionFileFrom(ctx context.Context, dir *tenantDirectory, path string, info os.FileInfo, maxBytes int64) ([]byte, string) {
f, err := dir.open(filepath.Base(path))
if err != nil {
return nil, "could not be opened"
}
defer func() { _ = f.Close() }()
opened, err := companionFileStat(f)
if err != nil || !opened.Mode().IsRegular() || !sameDetectedFile(info, opened) {
return nil, "changed while it was read"
}
if opened.Size() > maxBytes {
return nil, "larger than the scan limit, not scanned"
}
data, err := io.ReadAll(io.LimitReader(companionContextReader{ctx: ctx, reader: f}, opened.Size()+1))
if err != nil || int64(len(data)) != opened.Size() {
return nil, "could not be read whole"
}
after, err := companionFileStat(f)
if err != nil || !sameDetectedFile(opened, after) {
return nil, "changed while it was read"
}
return data, ""
}
type companionContextReader struct {
ctx context.Context
reader io.Reader
}
func (r companionContextReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.reader.Read(p)
}
// companionKind names the kinds of file a kit carries. Anything else in the
// window, such as images or stylesheets, is not reported. Content decides
// where a name would mislead. A binary is named before PHP: droppers carry the
// PHP they write.
func companionKind(name string, info os.FileInfo, data []byte) string {
lower := strings.ToLower(name)
ext := filepath.Ext(lower)
switch {
case name == ".htaccess":
return "htaccess"
case bytes.HasPrefix(data, []byte("\x7fELF")):
return "executable"
case contenttype.IsPHPSourceName(lower) || contenttype.HasPHPOpenTag(data):
return "php"
case info.Mode().Perm()&0o111 != 0:
return "executable"
case contenttype.IsArchiveExt(ext) || contenttype.IsCompressedArchive(data):
return "archive"
case ext == "":
return "no extension"
}
return ""
}
func plantedFilesFinding(a companionAnchor, planted []plantedFile, total int) alert.Finding {
sort.Slice(planted, func(i, j int) bool { return planted[i].name < planted[j].name })
var b strings.Builder
fmt.Fprintf(&b, "Detected file: %s\n", a.path)
fmt.Fprintf(&b, "Detected file changed: %s\n", a.changed.UTC().Format(time.RFC3339))
fmt.Fprintf(&b, "Window: %s either side\n", companionPlantWindow)
fmt.Fprintf(&b, "Planted files: %d\n", total)
for _, p := range planted {
fmt.Fprintf(&b, "- %s: %s, %d bytes, changed %s, modified %s",
strconv.QuoteToASCII(p.name), p.kind, p.size,
p.changed.UTC().Format(time.RFC3339), p.modified.UTC().Format(time.RFC3339))
if gap := p.changed.Sub(p.modified); gap > companionBackdateGap {
fmt.Fprintf(&b, ", modified %d days before its change, possible backdating", int(gap/(24*time.Hour)))
}
if p.note != "" {
fmt.Fprintf(&b, ", %s", p.note)
}
b.WriteString("\n")
}
if extra := total - len(planted); extra > 0 {
fmt.Fprintf(&b, "... and %d more\n", extra)
}
b.WriteString("No scanner matched these files. Review them before removal; CSM never quarantines them automatically.")
return alert.Finding{
Severity: alert.High,
Check: "malware_companion_files",
Message: "Files planted alongside detected malware: " + a.path,
Details: b.String(),
FilePath: filepath.Dir(a.path),
}
}
// changeTimeOf returns the inode change time recorded in info.
func changeTimeOf(info os.FileInfo) (time.Time, bool) {
id, ok := selfWriteIdentityFromFileInfo(info)
if !ok {
return time.Time{}, false
}
return time.Unix(id.ChangeSec, id.ChangeNsec), true
}
func sameFileOwner(a, b os.FileInfo) bool {
as, aok := a.Sys().(*syscall.Stat_t)
bs, bok := b.Sys().(*syscall.Stat_t)
return aok && bok && as.Uid == bs.Uid
}
// runnerDisabled reports whether the operator disabled a check runner, by its
// own name or one of its finding names.
func runnerDisabled(cfg *config.Config, runner string) bool {
_, disabled := splitDisabledChecks(cfg, []namedCheck{{name: runner}})
return len(disabled) > 0
}
package checks
import (
"context"
"encoding/hex"
"fmt"
"net"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/netutil"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
)
// safeRemotePorts and safeUsers are package-level so both legacy polling and
// the BPF live coordinator share one source of truth.
var safeRemotePorts = map[uint16]bool{
53: true, 80: true, 443: true, 25: true, 587: true, 465: true,
993: true, 995: true, 110: true, 143: true,
}
var safeUsers = map[string]bool{
"imunify360-webshield": true,
"named": true,
"mysql": true,
"memcached": true,
"icinga": true,
"dovecot": true,
"mailman": true,
}
var serverLocalPorts = map[uint16]bool{
21: true, 25: true, 26: true, 53: true, 80: true, 110: true,
143: true, 443: true, 465: true, 587: true, 993: true, 995: true,
2082: true, 2083: true, 2086: true, 2087: true, 2095: true, 2096: true,
3306: true, 4190: true,
52223: true, 52224: true, 52227: true, 52228: true,
52229: true, 52230: true, 52231: true, 52232: true,
}
// EvaluateConnection returns a populated alert.Finding and true when the
// connection should be reported, or a zero finding and false when it should
// be ignored. Host-interface lookups are cached. Used by the BPF live backend
// (per-event) and the polling backend (per row of /proc/net/tcp[6]).
func EvaluateConnection(
cfg *config.Config,
uid uint32,
dstIP net.IP,
dstPort uint16,
localPort uint16,
proto string,
user string,
) (alert.Finding, bool) {
if uid == 0 {
return alert.Finding{}, false
}
if dstIP == nil || dstIP.IsLoopback() || dstIP.IsUnspecified() {
return alert.Finding{}, false
}
// Panel front ends proxy to their own backend over the machine's public
// address rather than loopback, so the packet never leaves the host and is
// no more an outbound connection than the loopback case above. Left
// reported, the proxy hop is classed as C2 traffic and drives the host's
// own address to a critical local threat score.
if netutil.IsHostAddress(dstIP.String()) {
return alert.Finding{}, false
}
if serverLocalPorts[localPort] {
return alert.Finding{}, false
}
if safeRemotePorts[dstPort] {
return alert.Finding{}, false
}
if isInfraIP(dstIP.String(), cfg.InfraIPs) {
return alert.Finding{}, false
}
if safeUsers[user] {
return alert.Finding{}, false
}
dst := dstIP.String()
if dstIP.To4() == nil {
dst = "[" + dst + "]"
}
return alert.Finding{
Severity: alert.High,
Check: "user_outbound_connection",
Message: fmt.Sprintf("Non-root user connecting to unusual destination: %s:%d", dst, dstPort),
Details: fmt.Sprintf("UID: %d (%s), Local port: %d, Proto: %s", uid, user, localPort, proto),
}, true
}
// CheckOutboundUserConnections looks for non-root user processes making
// outbound connections to IPs that aren't infra or well-known services.
// Catches compromised accounts phoning home.
func CheckOutboundUserConnections(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
if data, err := osFS.ReadFile("/proc/net/tcp"); err == nil {
findings = append(findings, scanProcNetTCP(cfg, data, false)...)
} else {
// Preserve historical behaviour: if /proc/net/tcp is unreadable,
// return nil rather than continuing to tcp6.
return nil
}
if tcp6Data, err := osFS.ReadFile("/proc/net/tcp6"); err == nil {
findings = append(findings, scanProcNetTCP(cfg, tcp6Data, true)...)
}
return findings
}
// scanProcNetTCP parses one /proc/net/tcp[6] dump and returns findings for
// every ESTABLISHED row that EvaluateConnection flags.
//
// A first pass collects local sockets in LISTEN state so that an ESTABLISHED
// row whose local address and port have a listener is recognised as the
// accept side of an inbound connection (e.g. pure-ftpd PASV data channels,
// user-owned daemons listening on high ports) and not an outbound connect().
// Wildcard listeners match every local address for that port.
func scanProcNetTCP(cfg *config.Config, data []byte, ipv6 bool) []alert.Finding {
var findings []alert.Finding
proto := "tcp"
if ipv6 {
proto = "tcp6"
}
lines := strings.Split(string(data), "\n")
listeners := collectListenSockets(lines, ipv6)
directSMTPEnabled := DirectSMTPEgressBackendEnabled(cfg, "legacy")
var mta platform.MTAIdents
if directSMTPEnabled {
// Resolve MTA identities once per scan; legacy poller has no per-PID context,
// so the EvaluateDirectSMTPEgress UID/user gate carries the load.
mta = platform.LocalMTAIdentities(platform.Detect())
}
for _, line := range lines {
fields := strings.Fields(line)
if len(fields) < 8 || fields[0] == "sl" {
continue
}
// State 01 = ESTABLISHED
if fields[3] != "01" {
continue
}
uidStr := fields[7]
uidU64, err := strconv.ParseUint(uidStr, 10, 32)
if err != nil {
continue
}
uidU32 := uint32(uidU64)
var (
localIP net.IP
dstIP net.IP
dstPort int
localPort int
)
if ipv6 {
localIP, localPort = parseHex6Addr(fields[1])
dstIP, dstPort = parseHex6Addr(fields[2])
} else {
localAddr, parsedLocalPort := parseHexAddr(fields[1])
localIP = net.ParseIP(localAddr)
localPort = parsedLocalPort
remoteIP, remotePort := parseHexAddr(fields[2])
dstIP = net.ParseIP(remoteIP)
dstPort = remotePort
}
if localIP == nil || dstIP == nil || localPort <= 0 || dstPort <= 0 {
continue
}
if listeners.has(localIP, localPort) {
continue
}
// Bad-ASN egress is classified for every UID, root included: a
// post-exploit root process exfiltrating to a bad ASN is the
// host-takeover signal. The live BPF tracker drops root events for
// flood control, so this periodic scan is where root egress is seen.
// The non-root detectors below intentionally skip root.
if lookup := CurrentASNLookup(); lookup != nil && cfg.Detection.BadASNOutbound.Enabled {
asn, org := lookup(dstIP.String())
if f, ok := EvaluateBadASNOutbound(cfg, dstIP, asn, org); ok {
AttributeSocketOwner(&f, uidU32)
f.Timestamp = time.Now()
findings = append(findings, f)
}
}
if uidU64 == 0 {
continue
}
user := LookupUser(uidU32)
// #nosec G115 -- ports parsed from /proc/net/tcp[6] are bounded by uint16.
if f, ok := EvaluateConnection(cfg, uidU32, dstIP, uint16(dstPort), uint16(localPort), proto, user); ok {
f.Timestamp = time.Now()
findings = append(findings, f)
}
if directSMTPEnabled {
// #nosec G115 -- ports parsed from /proc/net/tcp[6] are bounded by uint16.
if f, ok := EvaluateDirectSMTPEgress(cfg, DirectSMTPEgressInput{
UID: uidU32,
User: user,
DstIP: dstIP,
DstPort: uint16(dstPort),
MTA: mta,
}); ok {
f.Timestamp = time.Now()
findings = append(findings, f)
}
}
}
return findings
}
type listenSocket struct {
address string
port int
}
type listenSocketSet struct {
wildcardPorts map[int]bool
sockets map[listenSocket]bool
}
func (s listenSocketSet) has(ip net.IP, port int) bool {
if port <= 0 {
return false
}
if s.wildcardPorts[port] {
return true
}
if ip == nil {
return false
}
return s.sockets[listenSocket{address: normalizeListenIP(ip), port: port}]
}
// collectListenSockets scans /proc/net/tcp[6] rows for state 0A (LISTEN) and
// returns the set of local sockets a process is bound to.
func collectListenSockets(lines []string, ipv6 bool) listenSocketSet {
listeners := listenSocketSet{
wildcardPorts: make(map[int]bool),
sockets: make(map[listenSocket]bool),
}
for _, line := range lines {
fields := strings.Fields(line)
if len(fields) < 8 || fields[0] == "sl" {
continue
}
if fields[3] != "0A" {
continue
}
var (
localIP net.IP
localPort int
)
if ipv6 {
localIP, localPort = parseHex6Addr(fields[1])
} else {
localAddr, parsedLocalPort := parseHexAddr(fields[1])
localIP = net.ParseIP(localAddr)
localPort = parsedLocalPort
}
if localIP == nil || localPort <= 0 {
continue
}
if localIP.IsUnspecified() {
listeners.wildcardPorts[localPort] = true
continue
}
listeners.sockets[listenSocket{address: normalizeListenIP(localIP), port: localPort}] = true
}
return listeners
}
func normalizeListenIP(ip net.IP) string {
if v4 := ip.To4(); v4 != nil {
return net.IP(v4).String()
}
return ip.String()
}
// parseHex6Addr parses an IPv6 address:port from /proc/net/tcp6 format.
// IPv6 addresses are 32 hex chars (128 bits) in little-endian 4-byte groups.
func parseHex6Addr(s string) (net.IP, int) {
parts := strings.SplitN(s, ":", 2)
if len(parts) != 2 {
return nil, 0
}
hexIP := parts[0]
hexPort := parts[1]
if len(hexIP) != 32 {
return nil, 0
}
port, ok := parseProcNetHexPort(hexPort)
if !ok {
return nil, 0
}
// Parse as 4 little-endian 32-bit words
ip := make(net.IP, 16)
for i := 0; i < 4; i++ {
word := hexIP[i*8 : (i+1)*8]
b, _ := hex.DecodeString(word)
if len(b) != 4 {
return nil, 0
}
// Reverse bytes within each 32-bit word (little-endian to big-endian)
ip[i*4+0] = b[3]
ip[i*4+1] = b[2]
ip[i*4+2] = b[1]
ip[i*4+3] = b[0]
}
return ip, port
}
// CheckSSHDConfig monitors sshd_config for dangerous changes.
func CheckSSHDConfig(ctx context.Context, _ *config.Config, store *state.Store) []alert.Finding {
var findings []alert.Finding
parsed := parseSSHDConfig()
if !parsed.Present() {
return nil
}
// Hash every file sshd reads, not only the root: a drop-in under an
// Include can flip PermitRootLogin while the root file stays identical.
hash := parsed.Digest()
current := settingsFromSSHDConfig(parsed)
hashKey := "_sshd_config_hash"
passKey := "_sshd_passwordauthentication"
rootKey := "_sshd_permitrootlogin"
prevHash, exists := store.GetRaw(hashKey)
prevPass, _ := store.GetRaw(passKey)
prevRoot, _ := store.GetRaw(rootKey)
if exists && prevHash != hash {
// Only alert when the effective setting changed into a dangerous value.
// This avoids false positives from commented defaults or Match blocks.
if current.PasswordAuthentication == "yes" && prevPass != "yes" {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "sshd_config_change",
Message: "PasswordAuthentication changed to 'yes' in sshd_config",
Details: "This allows password-based SSH login - high risk if passwords are compromised",
})
}
if current.PermitRootLogin == "yes" && prevRoot != "yes" {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "sshd_config_change",
Message: "PermitRootLogin changed to 'yes' in sshd_config",
})
}
// Generic change alert if no specific dangerous setting found
if len(findings) == 0 {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "sshd_config_change",
Message: "sshd_config modified",
})
}
}
store.SetRaw(hashKey, hash)
store.SetRaw(passKey, current.PasswordAuthentication)
store.SetRaw(rootKey, current.PermitRootLogin)
return findings
}
// CheckNulledPlugins scans WordPress plugin directories for signs of
// nulled/pirated plugins: missing licenses, known crack patterns, GPL
// bypass code, and plugins not found on wordpress.org.
func CheckNulledPlugins(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
// Known crack/null signatures in PHP files
crackSignatures := []string{
"nulled by", "cracked by", "gpl-club", "gpldl.com",
"developer license", "remove license check",
"license_key_bypass", "activation_bypass",
"@remove_license", "null_license",
}
homeDirs, _ := GetScanHomeDirs(ctx)
for _, homeEntry := range homeDirs {
if !homeEntry.IsDir() {
continue
}
pluginsDir := filepath.Join(scanHomeDirPath(homeEntry), "public_html", "wp-content", "plugins")
plugins, err := osFS.ReadDir(pluginsDir)
if err != nil {
continue
}
for _, plugin := range plugins {
if !plugin.IsDir() {
continue
}
pluginDir := filepath.Join(pluginsDir, plugin.Name())
// Check main plugin PHP file for crack signatures
mainFiles, _ := osFS.Glob(filepath.Join(pluginDir, "*.php"))
for _, mainFile := range mainFiles {
// Only read the first 10KB of each file
data := readFileHead(mainFile, 10*1024)
if data == nil {
continue
}
contentLower := strings.ToLower(string(data))
for _, sig := range crackSignatures {
if strings.Contains(contentLower, sig) {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "nulled_plugin",
Message: fmt.Sprintf("Possible nulled plugin: %s/%s", homeEntry.Name(), plugin.Name()),
Details: fmt.Sprintf("File: %s\nSignature: %s", mainFile, sig),
})
break
}
}
}
}
}
return findings
}
// readFileHead reads the first N bytes of a file.
func readFileHead(path string, maxBytes int) []byte {
f, err := osFS.Open(path)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
buf := make([]byte, maxBytes)
n, _ := f.Read(buf)
if n == 0 {
return nil
}
return buf[:n]
}
package checks
import (
"context"
"github.com/pidginhost/csm/internal/alert"
)
// LatestFindingStore is the subset of the state store the sweep needs.
type LatestFindingStore interface {
LatestFindings() []alert.Finding
// Each mutation takes the snapshot verification actually looked at, so a
// scan or realtime alert that refreshed the same key while the re-check was
// in flight is never overwritten by the older verdict.
PrepareFindingResolution(expected alert.Finding) func() bool
DemoteLatestFinding(expected alert.Finding, severity alert.Severity) bool
RestoreLatestFindingSeverity(expected alert.Finding) bool
}
// ContentReverifyDismissal records one finding the sweep cleared, for the
// caller to audit-log (the checks package has no logger of its own).
type ContentReverifyDismissal struct {
Check string
Path string
Detail string
// Demoted and Promoted distinguish the outcomes for the audit log: a
// cleared finding is gone, a demoted one is still listed at a lower
// severity, a promoted one had an earlier demotion reversed.
Demoted bool
Promoted bool
}
// isAutomaticallyDemoted reports whether this finding is sitting at Warning
// because a previous sweep lowered it, rather than because it was raised there.
func isAutomaticallyDemoted(f alert.Finding) bool {
return f.Severity == alert.Warning &&
f.DemotedFrom >= alert.High && f.DemotedFrom <= alert.Critical
}
// ShouldRestoreSeverity reports whether an automatic demotion must be reversed.
// A demotion holds only while the replacement keeps satisfying the inert-content
// gate. Restore on a positive match and on every uncertain or newly-active shape
// alike; otherwise a second edit into a detection gap would leave live malware
// at Warning.
func ShouldRestoreSeverity(f alert.Finding, res VerifyResult) bool {
return isAutomaticallyDemoted(f) && !res.Demote
}
// ShouldDemoteSeverity reports whether a verdict retires a remediated but
// unproven finding from the live queue. It is never a clear: an attacker must
// not retire a finding by editing the file. Demoting an already-Warning finding
// would be churn.
//
// The unattended sweep and the operator's Re-check both ask this, so the two
// cannot disagree about what a verdict means.
func ShouldDemoteSeverity(f alert.Finding, res VerifyResult) bool {
return res.Checked && res.Demote && f.Severity > alert.Warning
}
// autoReverifiable reports whether the sweep may re-check and dismiss a finding
// of this type on its own. Membership is deliberately narrow: only families
// whose verifier re-runs the same test that raised the finding and fails closed
// on any uncertainty. Every other registered verifier stays operator-driven
// through the web UI.
func autoReverifiable(check string) bool {
return IsContentReverifiable(check) || isExposedVerifiable(check)
}
// ReverifyStaleFindings re-checks every auto-reverifiable finding in the store
// against current detection logic and dismisses those that are now confirmed
// stale: for content findings a file that is gone, or identical bytes the
// current classifier no longer flags; for web_exposed_* findings an exposure
// that a complete pinned probe no longer confirms. Dispatch goes through the
// verifier registry, so each family keeps its own safety invariant -- a
// still-present file is cleared only when its bytes are unchanged since
// detection, and an exposure only when a complete probe says the server no
// longer serves it. Returns the dismissed findings for the caller to log.
// Read-only except for dismissing confirmed-stale findings.
func ReverifyStaleFindings(store LatestFindingStore) []ContentReverifyDismissal {
dismissed, _ := ReverifyStaleFindingsContext(context.Background(), store)
return dismissed
}
// ReverifyStaleFindingsContext is the cancellable form used by the daemon so a
// large exposure queue cannot delay shutdown for every remaining probe. The
// bool is false after cancellation so the daemon leaves the sweep version
// uncommitted and retries it on the next start.
// ReverifySweepStats describes what a sweep actually did. A sweep that changed
// nothing is otherwise silent, which makes "ran and found nothing"
// indistinguishable from "never ran" and from "could not check a single
// finding" -- the difference an operator needs when findings are not draining.
type ReverifySweepStats struct {
Considered int
Cleared int
Demoted int
Promoted int
Unchecked int
// TopUncheckedReason is the most common reason a finding could not be
// re-checked at all, which is where a silent sweep usually goes wrong.
TopUncheckedReason string
}
func ReverifyStaleFindingsContext(ctx context.Context, store LatestFindingStore) ([]ContentReverifyDismissal, bool) {
out, _, complete := ReverifyStaleFindingsStats(ctx, store)
return out, complete
}
// ReverifyStaleFindingsStats is ReverifyStaleFindingsContext with a summary of
// everything the sweep looked at, including the findings it could not check.
func ReverifyStaleFindingsStats(ctx context.Context, store LatestFindingStore) ([]ContentReverifyDismissal, ReverifySweepStats, bool) {
if ctx == nil {
ctx = context.Background()
}
var dismissed []ContentReverifyDismissal
var stats ReverifySweepStats
if ctx.Err() != nil {
return nil, stats, false
}
uncheckedReasons := map[string]int{}
var exposureVhosts *exposureVhostIndex
for _, f := range store.LatestFindings() {
if ctx.Err() != nil {
return dismissed, stats, false
}
if !autoReverifiable(f.Check) {
continue
}
stats.Considered++
in := VerifyInput{
Check: f.Check, Message: f.Message, Details: f.Details, Path: f.FilePath,
ContentSHA256: f.ContentSHA256, DetectLogic: f.DetectLogic,
Context: ctx,
}
if isExposedVerifiable(f.Check) {
if exposureVhosts == nil {
loaded := loadExposureVhostIndex()
exposureVhosts = &loaded
}
in.exposureVhosts = exposureVhosts
}
resolve := store.PrepareFindingResolution(f)
res := VerifyFindingInput(in)
if ctx.Err() != nil {
return dismissed, stats, false
}
switch {
case res.Checked && res.Resolved:
if resolve() {
stats.Cleared++
dismissed = append(dismissed, ContentReverifyDismissal{Check: f.Check, Path: f.FilePath, Detail: res.Detail})
}
case ShouldRestoreSeverity(f, res):
if store.RestoreLatestFindingSeverity(f) {
stats.Promoted++
dismissed = append(dismissed, ContentReverifyDismissal{
Check: f.Check, Path: f.FilePath, Detail: res.Detail, Promoted: true})
}
case ShouldDemoteSeverity(f, res):
if store.DemoteLatestFinding(f, alert.Warning) {
stats.Demoted++
dismissed = append(dismissed, ContentReverifyDismissal{
Check: f.Check, Path: f.FilePath, Detail: res.Detail, Demoted: true})
}
case !res.Checked:
// The verifier could not form an opinion at all -- a scanner that
// was unavailable, a file that changed under it. This is the
// bucket that looks identical to a sweep that never ran.
stats.Unchecked++
uncheckedReasons[res.Detail]++
}
}
top, topN := "", 0
for reason, n := range uncheckedReasons {
if n > topN || (n == topN && reason < top) {
top, topN = reason, n
}
}
stats.TopUncheckedReason = top
return dismissed, stats, true
}
package checks
import (
"crypto/sha256"
"fmt"
"io"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/signatures"
"github.com/pidginhost/csm/internal/yara"
)
// ContentLogicVersion identifies the current shape of the PHP content-analysis
// heuristic set (analyzePHPContent and its helpers). BUMP IT in the same commit
// as any change to those heuristics, so findings produced by the previous logic
// are re-verified and cleared by the daemon sweep.
const ContentLogicVersion = 2
// ContentScannerVersion identifies scanner behavior that is not represented by
// the loaded YAML signature version or YARA rule count. Bump it when shared
// content classification changes so the daemon re-checks stale findings.
const ContentScannerVersion = 4
// JSTaintLogicVersion identifies the current semantics of the JavaScript
// keystroke taint analyzer (internal/jstaint). BUMP IT in the same commit as
// any change to its sources, propagation, sinks, resource limits, parser
// version, or content pre-filter so findings produced by the previous logic
// are re-verified under the new one.
const JSTaintLogicVersion = 2
// PHPTaintLogicVersion identifies the current semantics of the PHP remote-
// source taint analyzer (internal/phptaint). BUMP IT in the same commit as any
// change to its parser, pre-filter, propagation, sinks, resource limits, or
// evidence semantics so existing findings are re-verified by the isolated
// worker under the new logic.
const PHPTaintLogicVersion = 4
// contentReverifiableChecks are content findings whose condition can be
// re-evaluated here by re-running the classifier that produced them on the
// file's current bytes. Unlike presenceVerifiableChecks, a still-present file
// may be resolved -- but ONLY when its bytes are byte-for-byte identical to
// detection time (ContentSHA256 match) and the current classifier no longer
// flags them. A file modified since detection is never auto-cleared (it could
// be a partial clean or an evasion edit), preserving the guarantee behind the
// presence-only design.
var contentReverifiableChecks = []string{
"suspicious_php_content",
"obfuscated_php",
"signature_match_realtime", "yara_match_realtime", "yara_match_scheduled",
"js_keylogger_dataflow",
"php_remote_taint",
}
var contentReverifiableSet = func() map[string]struct{} {
m := make(map[string]struct{}, len(contentReverifiableChecks))
for _, c := range contentReverifiableChecks {
m[c] = struct{}{}
}
return m
}()
// IsContentReverifiable reports whether a check type is re-evaluated by
// re-running the content classifier (vs presence-only).
func IsContentReverifiable(check string) bool {
_, ok := contentReverifiableSet[check]
return ok
}
// ContentDetectionVersion returns a token identifying the full content-detection
// logic in effect: the heuristic and scanner versions, the loaded signature-set
// version, the loaded YARA rule count, and both taint analyzer versions. The
// re-verifier always re-runs the real classifier, so this token only gates the
// daemon sweep and enriches audit detail; its precision is not security-critical.
func ContentDetectionVersion() string {
sigVer := 0
if s := signatures.Global(); s != nil {
sigVer = s.Version()
}
yaraRules := 0
if y := yara.Active(); y != nil {
yaraRules = y.RuleCount()
}
return contentDetectionVersionToken(
ContentLogicVersion,
ContentScannerVersion,
sigVer,
yaraRules,
JSTaintLogicVersion,
PHPTaintLogicVersion,
)
}
// reverifySweepLogicVersion identifies the sweep's own semantics, as opposed to
// the detection logic it re-runs: which outcomes it may apply and which paths
// its verifiers can reach. BUMP IT in the same commit as any such change, so an
// upgraded host sweeps at startup instead of carrying the old behaviour until
// its next deep-scan cycle.
const reverifySweepLogicVersion = 2
// FindingReverifyVersion identifies every verifier family the startup sweep
// runs unattended, plus the sweep's own semantics. Including the exposure
// verifier forces one sweep when that family is first deployed or its
// fail-closed semantics change.
func FindingReverifyVersion() string {
return fmt.Sprintf("%s;exposed=%d;reverify=%d",
ContentDetectionVersion(), exposedReverifyLogicVersion, reverifySweepLogicVersion)
}
// contentDetectionVersionToken renders the version components. Pure so a test
// can pass two different component values and prove the tokens differ.
func contentDetectionVersionToken(phpVer, scanVer, sigVer, yaraRules, jsTaintVer, phpTaintVer int) string {
return fmt.Sprintf(
"php=%d;scan=%d;sig=%d;yara=%d;jstaint=%d;phptaint=%d",
phpVer, scanVer, sigVer, yaraRules, jsTaintVer, phpTaintVer,
)
}
// contentFingerprintMaxBytes caps the file size hashed for a finding's
// fingerprint. Larger files get an empty fingerprint, so the re-verifier treats
// them as un-fingerprinted (never auto-cleared while present).
const contentFingerprintMaxBytes = 16 << 20 // 16 MiB
// FileContentSHA256 returns the hex SHA-256 of the whole file, or "" if the
// file cannot be read, is not a regular file, or exceeds the size cap.
func FileContentSHA256(path string) string {
info, err := osFS.Stat(path)
if err != nil || !info.Mode().IsRegular() || info.Size() > contentFingerprintMaxBytes {
return ""
}
f, err := osFS.Open(path)
if err != nil {
return ""
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return ""
}
return fmt.Sprintf("%x", h.Sum(nil))
}
// StampContentFingerprint records the detection-time content fingerprint on a
// content-reverifiable finding so the Re-check / sweep can later distinguish a
// superseded-heuristic false positive from a file edited after detection.
// A producer that analyzed an already-open snapshot may supply its exact hash;
// retain that fingerprint instead of reopening a path that may now name
// different content. No-op for non-content findings or findings without a path.
func StampContentFingerprint(f *alert.Finding) {
if f == nil || f.FilePath == "" || !IsContentReverifiable(f.Check) {
return
}
if f.ContentSHA256 != "" {
if f.DetectLogic == "" {
f.DetectLogic = ContentDetectionVersion()
}
return
}
f.ContentSHA256 = FileContentSHA256(f.FilePath)
f.DetectLogic = ContentDetectionVersion()
}
package checks
import (
"fmt"
"path/filepath"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
)
// correlationWindow bounds how far apart two findings may be and still count
// as one cross-account event in the persisted active set. A dispatch batch
// supplies its own grouping and can include carried-forward timestamps, so
// the batch callers do not apply this bound.
//
// Without it the aggregate is a latch rather than an alert: replaying a
// 100-day recording of one production host left coordinated_attack raised for
// 76% of the recording, and a two-day recording of a second host for 99%,
// because the first three accounts that ever carried a critical finding never
// left the set. One hour was chosen against those recordings: it holds the
// aggregate raised for 2.4% of the first recording where six hours leaves
// 19.6% and a day leaves 46.1%, and it still spans a full scan sweep, whose
// findings land together. Re-derive it with scripts/correlation-calibrate.
const correlationWindow = time.Hour
// CorrelationResult is the output of one CorrelateFindings call.
type CorrelationResult struct {
// Derived findings: the coordinated_attack result first, then one
// cross_account_malware result per qualifying check, sorted by check.
// Timestamps are left unset for the caller to stamp.
Derived []alert.Finding
// Unattributed counts qualifying input rows per check that carried no
// account identity. It is a snapshot for this call, not a running total,
// using the same window as the aggregates. Batch correlation has no age
// filter; persisted correlation also counts unstamped legacy rows.
Unattributed map[string]int
// CriticalAccounts is the distinct attributed account count after the
// same eligibility and time filters used to derive coordinated_attack.
CriticalAccounts int
}
// Correlator shares production rules with offline replay. Construct one with
// NewCorrelator to supply recording roots without triggering host discovery.
type Correlator struct {
window time.Duration
accountOf func(alert.Finding) string
}
// NewCorrelator uses only the supplied account roots, with no host lookups.
// Window must be nonnegative; zero reproduces unbounded correlation.
func NewCorrelator(window time.Duration, accountRoots []string) Correlator {
roots := append([]string(nil), accountRoots...)
return Correlator{window: window, accountOf: func(f alert.Finding) string {
return extractAccountFromFindingAt(f, func() []string { return roots })
}}
}
var defaultCorrelator = Correlator{window: correlationWindow, accountOf: extractAccountFromFinding}
// CorrelateFindings raises cross-account findings. Eligibility comes from
// the registry classification and identity from extractAccountFromFinding.
// Callers initialize platform.Detect before correlation so account roots
// come from its cache. It does not log or mutate its input.
func CorrelateFindings(findings []alert.Finding) CorrelationResult {
return defaultCorrelator.Correlate(findings, time.Time{})
}
// CorrelateBatchFindings preserves dispatch grouping even when a scan carries
// forward a prior finding whose original timestamp lies outside the window.
func CorrelateBatchFindings(findings []alert.Finding) CorrelationResult {
batch := defaultCorrelator
batch.window = 0
return batch.Correlate(findings, time.Time{})
}
// Correlate derives aggregates at the supplied observation time. A zero time
// uses the newest non-derived input as its reference. Persisted state supplies
// the merge time so even an empty scan can expire old evidence.
func (c Correlator) Correlate(findings []alert.Finding, at time.Time) CorrelationResult {
res := CorrelationResult{Unattributed: make(map[string]int)}
accounts := make(map[string]bool)
malwareByCheck := make(map[string]map[string]bool)
cutoff := c.cutoff(findings, at)
for _, f := range findings {
class := correlationClassOf(f.Check)
if class != CorrelationSecurityEvent && class != CorrelationMalwareArtifact {
continue
}
if observed := observedAt(f); c.window > 0 && !observed.IsZero() && observed.Before(cutoff) {
continue
}
countsForAttack := f.Severity == alert.Critical
countsForMalware := class == CorrelationMalwareArtifact
if !countsForAttack && !countsForMalware {
continue
}
account := c.accountOf(f)
if account == "" {
res.Unattributed[f.Check]++
continue
}
if countsForAttack {
accounts[account] = true
}
if countsForMalware {
if malwareByCheck[f.Check] == nil {
malwareByCheck[f.Check] = make(map[string]bool)
}
malwareByCheck[f.Check][account] = true
}
}
res.CriticalAccounts = len(accounts)
// The sorted account sets are part of the condition: acknowledging one
// group must not hide spread to another account. Source text is omitted.
if len(accounts) >= 3 {
names := sortedKeys(accounts)
res.Derived = append(res.Derived, alert.Finding{
Severity: alert.Critical,
Check: "coordinated_attack",
Message: fmt.Sprintf("Possible coordinated attack: %d accounts have critical security events", len(names)),
Details: fmt.Sprintf("Affected accounts: %s", strings.Join(names, ", ")),
})
}
for _, check := range sortedKeys(malwareByCheck) {
names := sortedKeys(malwareByCheck[check])
if len(names) < 2 {
continue
}
res.Derived = append(res.Derived, alert.Finding{
Severity: alert.Critical,
Check: "cross_account_malware",
Message: fmt.Sprintf("Same malware type (%s) found in %d accounts", check, len(names)),
Details: fmt.Sprintf("Accounts: %s", strings.Join(names, ", ")),
})
}
return res
}
// CorrelationInputOf reports how one finding enters cross-account
// correlation: the hosting account it resolves to, empty when none could be
// determined, and whether its check is an eligible input at all. It applies
// the same registry classification and identity rules CorrelateFindings uses,
// so a caller can explain or calibrate a correlation result without
// re-implementing them. Eligibility here is the check's class only; whether a
// given finding then counts also depends on its severity.
func CorrelationInputOf(f alert.Finding) (account string, eligible bool) {
return defaultCorrelator.InputOf(f)
}
// InputOf uses the same classification and identity rules as Correlate.
// Eligibility is the check class only, before severity and window filtering.
func (c Correlator) InputOf(f alert.Finding) (account string, eligible bool) {
class := correlationClassOf(f.Check)
eligible = class == CorrelationSecurityEvent || class == CorrelationMalwareArtifact
return c.accountOf(f), eligible
}
// observedAt is when a finding's condition started: its first observation,
// falling back to its report time. A scan re-emits every finding it still
// sees with a fresh report time, so judging membership by Timestamp let a
// months-old condition re-enter the window on every cycle.
func observedAt(f alert.Finding) time.Time {
if !f.FirstSeen.IsZero() {
return f.FirstSeen
}
return f.Timestamp
}
// A missing timestamp must not silently discard legacy stored evidence.
func (c Correlator) cutoff(findings []alert.Finding, at time.Time) time.Time {
if c.window == 0 {
return time.Time{}
}
if at.IsZero() {
for _, f := range findings {
// Synthesized findings must not feed back into either aggregate
// membership or attribution-health accounting. The reference is
// the newest report, not the newest first observation: it stands
// for "now" when the caller supplied no observation time.
if !IsDerivedCorrelationCheck(f.Check) && f.Timestamp.After(at) {
at = f.Timestamp
}
}
}
if at.IsZero() {
return time.Time{}
}
return at.Add(-c.window)
}
func uniqueStrings(input []string) []string {
seen := make(map[string]bool)
var result []string
for _, s := range input {
if !seen[s] {
seen[s] = true
result = append(result, s)
}
}
return result
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// extractAccountFromFinding resolves the hosting account a finding belongs
// to: the producer's TenantID verbatim, then the account home that contains
// an absolute FilePath, then the legacy free-text scan of Message and
// Details. The structured sources win so a path mentioned in free text
// cannot re-attribute a finding whose producer knew its owner.
func extractAccountFromFinding(f alert.Finding) string {
return extractAccountFromFindingAt(f, accountHomeRoots)
}
func extractAccountFromFindingAt(f alert.Finding, accountRoots func() []string) string {
if f.TenantID != "" {
return f.TenantID
}
roots := accountRoots()
if filepath.IsAbs(f.FilePath) {
if _, account, ok := accountRootOfAt(f.FilePath, roots); ok {
return account
}
}
for _, s := range []string{f.Message, f.Details} {
if account := accountNameInTextAt(s, roots); account != "" {
return account
}
}
return ""
}
package checks
import (
"fmt"
"sort"
"sync"
)
// CorrelationClass is a check's role in cross-account correlation. Every
// registry entry sets one; the zero value fails the completeness test.
type CorrelationClass uint8
const (
// CorrelationUnclassified is the zero value and never valid.
CorrelationUnclassified CorrelationClass = iota
// CorrelationIgnored is never an input to correlation; a reason is required.
CorrelationIgnored
// CorrelationSecurityEvent: an attributed Critical counts toward coordinated_attack.
CorrelationSecurityEvent
// CorrelationMalwareArtifact is a SecurityEvent that also raises
// cross_account_malware when the same check appears on two accounts.
CorrelationMalwareArtifact
// CorrelationDerived is an output of correlation and never an input.
CorrelationDerived
)
// Ignore reasons. The value is a short token; the sentence is what the
// generated policy table prints and what a reviewer reads.
const (
reasonPosture = "posture"
reasonAttackerSide = "attacker-side"
reasonInformational = "informational"
reasonSelfHealth = "self-health"
reasonResponse = "response"
reasonHostScope = "host-scope"
reasonAccountAggregate = "account-aggregate"
reasonPerformance = "performance"
)
var correlationReasonSentences = map[string]string{
reasonPosture: "static configuration, hardening or hygiene state; a Critical means a misconfiguration, not an attack on the account",
reasonAttackerSide: "attacker activity or attempted access, not evidence of compromise of the named victim",
reasonInformational: "audit trail or inventory event with no compromise claim",
reasonSelfHealth: "CSM's own health, capacity or coverage state",
reasonResponse: "record of an automatic action already taken; feeding it back would double count",
reasonHostScope: "host-wide condition with no account to attribute; a cross-account count cannot use it even when it is a real compromise",
reasonAccountAggregate: "already summarizes several accounts without a single victim identity",
reasonPerformance: "resource usage",
}
// Attribution gaps documented on eligible checks. A gap never changes
// eligibility: an attributed Critical still counts.
const gapEnvelopeSender = "envelope-sender"
var correlationGapSentences = map[string]string{
gapEnvelopeSender: "sender-domain volume aggregate is unattributed when contributing submissions are unverified or belong to different accounts",
}
// validateCorrelationPolicy returns the first policy violation in entries,
// naming the offending check.
func validateCorrelationPolicy(entries []CheckInfo) error {
for _, c := range entries {
switch c.Correlation {
case CorrelationIgnored:
if _, ok := correlationReasonSentences[c.CorrelationReason]; !ok {
return fmt.Errorf("check %q is ignored without a known reason (%q)", c.Name, c.CorrelationReason)
}
if c.CorrelationGap != "" {
return fmt.Errorf("check %q is ignored but carries gap %q", c.Name, c.CorrelationGap)
}
case CorrelationSecurityEvent, CorrelationMalwareArtifact:
if c.CorrelationReason != "" {
return fmt.Errorf("check %q is eligible but carries reason %q", c.Name, c.CorrelationReason)
}
if c.CorrelationGap != "" {
if _, ok := correlationGapSentences[c.CorrelationGap]; !ok {
return fmt.Errorf("check %q carries unknown gap %q", c.Name, c.CorrelationGap)
}
}
case CorrelationDerived:
if c.CorrelationReason != "" || c.CorrelationGap != "" {
return fmt.Errorf("check %q is derived but carries a reason or gap", c.Name)
}
default:
return fmt.Errorf("check %q has no correlation classification (class %d)", c.Name, c.Correlation)
}
}
return nil
}
// correlationIndex is built once from the registry. It never reads the
// filesystem, the store or the network.
type correlationIndex struct {
classes map[string]CorrelationClass
reasons map[string]string
derived []string
}
var (
correlationOnce sync.Once
correlationTable *correlationIndex
)
func loadCorrelationIndex() *correlationIndex {
correlationOnce.Do(func() {
idx := &correlationIndex{
classes: make(map[string]CorrelationClass, len(checkRegistry)),
reasons: make(map[string]string, len(checkRegistry)),
}
for _, c := range checkRegistry {
idx.classes[c.Name] = c.Correlation
idx.reasons[c.Name] = c.CorrelationReason
if c.Correlation == CorrelationDerived {
idx.derived = append(idx.derived, c.Name)
}
}
sort.Strings(idx.derived)
correlationTable = idx
})
return correlationTable
}
func correlationClassOf(name string) CorrelationClass {
return loadCorrelationIndex().classes[name]
}
// correlationReasonOf returns the ignore reason a check is registered with,
// or "" for an eligible or unknown check.
func correlationReasonOf(name string) string {
return loadCorrelationIndex().reasons[name]
}
// securityEventEligible reports whether an attributed Critical finding of
// this check counts toward coordinated_attack.
func securityEventEligible(name string) bool {
switch correlationClassOf(name) {
case CorrelationSecurityEvent, CorrelationMalwareArtifact:
return true
}
return false
}
// DerivedCorrelationChecks returns the names correlation itself emits,
// sorted. The caller owns the slice.
func DerivedCorrelationChecks() []string {
return append([]string(nil), loadCorrelationIndex().derived...)
}
// IsDerivedCorrelationCheck reports whether name is an output of correlation.
func IsDerivedCorrelationCheck(name string) bool {
return correlationClassOf(name) == CorrelationDerived
}
package checks
import (
"sync"
"time"
csmlog "github.com/pidginhost/csm/internal/log"
)
// AttributionReport is the operator-facing view of correlation attribution.
// Current is what the latest-state active set looks like right now;
// Cumulative is the history since the daemon started. The two answer
// different questions: a producer that lost attribution for weeks and one
// that missed once look identical in a log line, and neither is visible in
// a health endpoint without this.
type AttributionReport struct {
// Current holds, per check, the qualifying rows in the latest-state
// active set that carry no hosting owner and fall inside the correlation
// window at its most recent merge. Unstamped legacy rows also count.
// A later merge clears rows that gain an owner or age out of the window.
Current map[string]int
// Cumulative sums every unattributed row reported since start, by
// check, across active-set merges and per-batch derivations.
Cumulative map[string]int
// ActiveSetUpdates counts active-set merges since start.
ActiveSetUpdates int
// Since is when the first active set was recorded; zero before then.
Since time.Time
}
// unattributedReporter logs once per check name per process when
// cross-account correlation could not attribute a qualifying finding to a
// hosting account, and keeps the counts behind AttributionHealth. Per-call
// counts are never summed into the active-set snapshot; only the cumulative
// history adds them up.
type unattributedReporter struct {
mu sync.Mutex
seen map[string]struct{}
current map[string]int
cumulative map[string]int
activeSetUpdates int
since time.Time
warn func(msg string, args ...any)
}
func newUnattributedReporter(warn func(string, ...any)) *unattributedReporter {
return &unattributedReporter{
seen: make(map[string]struct{}),
current: make(map[string]int),
cumulative: make(map[string]int),
warn: warn,
}
}
// eligibleCounts keeps the entries that describe a real attribution loss:
// positive counts for checks that correlation would otherwise count.
func eligibleCounts(counts map[string]int) map[string]int {
out := make(map[string]int, len(counts))
for check, n := range counts {
if n > 0 && securityEventEligible(check) {
out[check] = n
}
}
return out
}
type unattributedWarning struct {
check string
n int
}
// record publishes all counters together and returns pending warnings so
// callers can release their merge locks before invoking the logger.
func (r *unattributedReporter) record(counts map[string]int, activeSet bool) []unattributedWarning {
filtered := eligibleCounts(counts)
var fresh []unattributedWarning
r.mu.Lock()
defer r.mu.Unlock()
if activeSet {
r.current = filtered
r.activeSetUpdates++
if r.since.IsZero() {
r.since = time.Now()
}
}
for check, n := range filtered {
r.cumulative[check] += n
if _, dup := r.seen[check]; !dup {
r.seen[check] = struct{}{}
fresh = append(fresh, unattributedWarning{check, n})
}
}
return fresh
}
func (r *unattributedReporter) warnCounts(fresh []unattributedWarning) {
for _, f := range fresh {
r.warn("cross-account correlation could not attribute findings to an account", "check", f.check, "rows", f.n)
}
}
// Report records a per-batch derivation: the rows count toward the
// cumulative history and warn once per check, but the active-set snapshot
// is untouched because a batch is not the persisted state.
func (r *unattributedReporter) Report(counts map[string]int) {
r.warnCounts(r.record(counts, false))
}
// RecordActiveSet records the latest-state merge: it replaces the
// active-set snapshot, so a merge whose rows all carry owners clears it,
// and adds to the history like a batch report.
func (r *unattributedReporter) RecordActiveSet(counts map[string]int) {
r.warnCounts(r.record(counts, true))
}
// Health returns a copy of the current state.
func (r *unattributedReporter) Health() AttributionReport {
r.mu.Lock()
defer r.mu.Unlock()
return AttributionReport{
Current: copyCounts(r.current),
Cumulative: copyCounts(r.cumulative),
ActiveSetUpdates: r.activeSetUpdates,
Since: r.since,
}
}
func copyCounts(m map[string]int) map[string]int {
out := make(map[string]int, len(m))
for k, v := range m {
out[k] = v
}
return out
}
var defaultUnattributedReporter = newUnattributedReporter(csmlog.Warn)
// ReportUnattributedCorrelation records unattributed rows from a per-batch
// derivation through the process-wide reporter. Callers invoke it after
// releasing any state store lock; it never re-enters the store.
func ReportUnattributedCorrelation(counts map[string]int) {
defaultUnattributedReporter.Report(counts)
}
// RecordUnattributedActiveSet records the unattributed rows of the
// latest-state active set after a merge. Same locking contract as
// ReportUnattributedCorrelation.
func RecordUnattributedActiveSet(counts map[string]int) {
defaultUnattributedReporter.RecordActiveSet(counts)
}
// AttributionHealth reports the process-wide attribution state for the
// health snapshot and doctor.
func AttributionHealth() AttributionReport {
return defaultUnattributedReporter.Health()
}
// ResetAttributionHealthForTest replaces the process-wide reporter with a
// fresh one. Test-only.
func ResetAttributionHealthForTest() {
defaultUnattributedReporter = newUnattributedReporter(csmlog.Warn)
}
package checks
import (
"iter"
"net"
"strings"
"time"
)
// CpanelAccessRecord is one line of cpsrvd's access_log:
// `<ip> <ident> <user> [01/02/2006:15:04:05 -0700] "<method> <target> <proto>" <status> <size>`,
// then quoted fields and the listener port, where ident is "-" or "proxy".
// cpsrvd escapes no quotes, so the user, request, referer and user agent
// are client text that may hold raw quotes and whole forged records. Client
// text can add a possible request or status to a record, never hide the one
// cpsrvd wrote; it never supplies the peer, the port or the time.
type CpanelAccessRecord struct {
IP string // the leading address: the connecting peer unless Proxy
User string // text before the first request; "-" when none was named
// Status is the response code when the line holds one request and one
// status, "" when client text makes it ambiguous.
Status string
// Port served the request: the bare last field after a quoted field, or
// the host:port of the last quoted field when the first candidate's
// suffix parses strictly through the end of the line.
Port string
// At is the time cpsrvd stamped, zero unless cpanelAccessLogTime dates
// the line.
At time.Time
// Proxy marks the "proxy" ident cpsrvd writes on service-subdomain
// requests. Its address is unverified and must never act as a peer.
Proxy bool
// rest is the line after the ident, read again for each request so a
// line full of forged requests costs no memory.
rest string
// first is the text from the first request's opening quote.
first string
}
// CpanelRequest is one request a cpsrvd access_log line may hold. The
// method and the start of the target are read from just after the opening
// quote; raw quotes later on cannot move them.
type CpanelRequest struct {
Method string // raw log escapes kept; compare with MethodIs
Target string // raw, up to the first blank; may hold raw quotes
}
// ParseCpanelAccessRecord reads line as cpsrvd writes it and rejects lines
// with no request after a bracketed field.
func ParseCpanelAccessRecord(line string) (CpanelAccessRecord, bool) {
peer, rest, ok := strings.Cut(line, " ")
if !ok || net.ParseIP(peer) == nil {
return CpanelAccessRecord{}, false
}
// cpsrvd writes "proxy" as the ident on service-subdomain requests.
ident, rest, ok := strings.Cut(rest, " ")
if !ok || ident != "-" && ident != "proxy" {
return CpanelAccessRecord{}, false
}
rec := CpanelAccessRecord{IP: peer, Proxy: ident == "proxy", rest: rest}
requests := 0
for open, after := range cpanelRequestStarts(rest) {
if _, ok := parseCpanelRequest(after[1:]); !ok {
continue
}
if requests++; requests > 1 {
break
}
if open >= 2 && (rest[open-1] == ' ' || rest[open-1] == '\t') {
rec.User = rest[:open-1]
}
rec.first = after
}
if requests == 0 {
return CpanelAccessRecord{}, false
}
if requests == 1 {
for code := range cpanelStatuses(rec.first) {
if rec.Status != "" {
rec.Status = ""
break
}
rec.Status = code
}
}
rec.Port = cpanelServedPort(line)
if rec.Port == "" {
// A later candidate could be inside a user agent. Its strict suffix
// cannot establish that the final field is a server-supplied host.
for open, after := range cpanelRequestStarts(rest) {
if open < 0 {
continue
}
rec.Port, _ = strictCpanelSuffix(after)
break
}
}
rec.At, _ = cpanelAccessLogTime(line)
return rec, true
}
// cpanelRequestStarts yields, for each bracketed field followed by blanks
// and a quote, the index of the last opening bracket before it (-1 when
// none) and the text from the quote on. Client text adds starts; the
// request cpsrvd wrote is always one of them.
func cpanelRequestStarts(rest string) iter.Seq2[int, string] {
return func(yield func(int, string) bool) {
open := -1
for i := 0; i < len(rest); i++ {
if rest[i] == '[' {
open = i
continue
}
if rest[i] != ']' {
continue
}
after := strings.TrimLeft(rest[i+1:], " \t")
if len(after) == len(rest)-i-1 || after == "" || after[0] != '"' {
continue
}
if !yield(open, after) {
return
}
}
}
}
// Requests yields every request the line may hold, in line order.
func (r CpanelAccessRecord) Requests() iter.Seq[CpanelRequest] {
return func(yield func(CpanelRequest) bool) {
for _, after := range cpanelRequestStarts(r.rest) {
if q, ok := parseCpanelRequest(after[1:]); ok && !yield(q) {
return
}
}
}
}
// Any reports whether any request the line may hold matches.
func (r CpanelAccessRecord) Any(match func(CpanelRequest) bool) bool {
for q := range r.Requests() {
if match(q) {
return true
}
}
return false
}
// TargetContains reports whether any request target on the line contains
// one of words, ignoring ASCII case.
func (r CpanelAccessRecord) TargetContains(words ...string) bool {
return r.Any(func(q CpanelRequest) bool {
for _, w := range words {
if q.TargetContains(w) {
return true
}
}
return false
})
}
// MayHaveStatus reports whether any status-shaped field after the first
// request is one of codes. Only use it where a forged status can count only
// against the client that sent it.
func (r CpanelAccessRecord) MayHaveStatus(codes ...string) bool {
for s := range cpanelStatuses(r.first) {
for _, c := range codes {
if s == c {
return true
}
}
}
return false
}
// APIAuthFailure reports whether the record may be a failed cPanel or WHM
// API authentication. A session prefix alone does not identify an API call.
// Any target or status on the line counts, since a forged one can only count
// against the client that wrote it. Suppression needs separate session evidence.
func (r CpanelAccessRecord) APIAuthFailure() bool {
return r.MayHaveStatus("401", "403") && r.Any(cpanelAPIRequest)
}
func cpanelAPIRequest(q CpanelRequest) bool {
// cpsrvd also accepts API tokens on WHM DNS calls, transfer streams and
// plugin endpoints. Looking only for the three main API names lets a
// client choose another authenticated route to hide its failures.
dnsPath, dnsCall := false, false
for i := 0; i < len(q.Target); {
for _, endpoint := range [...]string{
"json-api", "xml-api", "/execute/", "/websocket/", "/cpxfer/",
"/wpt/", "/nova/",
} {
if cpanelTargetHasPrefix(q.Target, i, endpoint) {
return true
}
}
if !dnsPath {
dnsPath = cpanelTargetHasPrefix(q.Target, i, "/scripts2/")
}
if !dnsCall {
dnsCall = cpanelTargetHasPrefix(q.Target, i, "_local")
}
if dnsPath && dnsCall {
return true
}
_, i = cpanelTargetByte(q.Target, i)
}
return false
}
// Compare decoded bytes in place: a line can hold arbitrarily many candidate
// requests, so decoding each target into a string would amplify allocations.
func cpanelTargetHasPrefix(raw string, at int, prefix string) bool {
for i := range len(prefix) {
if at == len(raw) {
return false
}
var c byte
c, at = cpanelTargetByte(raw, at)
if cpanelASCIILower(c) != prefix[i] {
return false
}
}
return true
}
// cpsrvd unescapes the URL once before routing. Decode only the target,
// after record framing, so escaped delimiters cannot supply log metadata.
// A malformed escape elsewhere must not erase an API endpoint.
func cpanelTargetByte(raw string, at int) (byte, int) {
c, next := patternLogByte(raw, at)
if c == '%' && next < len(raw) {
hi, afterHi := patternLogByte(raw, next)
if patternIsHex(hi) && afterHi < len(raw) {
lo, afterLo := patternLogByte(raw, afterHi)
if patternIsHex(lo) {
return hexToByte(string([]byte{hi, lo})), afterLo
}
}
}
return c, next
}
// WebmailLogin reports whether the record is a login POST to webmail.
func (r CpanelAccessRecord) WebmailLogin() bool {
if r.Port != "2095" && r.Port != "2096" {
return false
}
return r.Any(func(q CpanelRequest) bool {
return q.MethodIs("POST") && (q.TargetContains("login") || q.TargetContains("auth"))
})
}
// MethodIs compares an HTTP method ignoring case without copying log escapes.
func (q CpanelRequest) MethodIs(method string) bool {
j := 0
for i := 0; i < len(q.Method); {
var c byte
c, i = patternLogByte(q.Method, i)
if j == len(method) || cpanelASCIILower(c) != cpanelASCIILower(method[j]) {
return false
}
j++
}
return j == len(method)
}
// TargetContains compares ASCII endpoint names without copying the target.
func (q CpanelRequest) TargetContains(word string) bool {
for i := 0; i <= len(q.Target)-len(word); i++ {
if cpanelEqualFold(q.Target[i:i+len(word)], word) {
return true
}
}
return false
}
// TargetHasPrefix compares an ASCII endpoint prefix without copying the target.
func (q CpanelRequest) TargetHasPrefix(prefix string) bool {
return len(q.Target) >= len(prefix) && cpanelEqualFold(q.Target[:len(prefix)], prefix)
}
func cpanelEqualFold(a, b string) bool {
for i := range len(b) {
if cpanelASCIILower(a[i]) != cpanelASCIILower(b[i]) {
return false
}
}
return true
}
func cpanelASCIILower(c byte) byte {
if c >= 'A' && c <= 'Z' {
return c + ('a' - 'A')
}
return c
}
// parseCpanelRequest reads the method and target from just after a
// request's opening quote.
func parseCpanelRequest(s string) (CpanelRequest, bool) {
method, rest, ok := strings.Cut(s, " ")
if !ok {
return CpanelRequest{}, false
}
// Method tokens have no fixed length. Rejecting a long one would hide
// the whole request.
target, _, _ := strings.Cut(rest, " ")
if !validCpanelMethod(method) || !patternValidRawTarget(target) {
return CpanelRequest{}, false
}
return CpanelRequest{Method: method, Target: target}, true
}
// Validate decoded bytes in place: client text can contain any number
// of escaped method candidates, including arbitrarily long ones.
func validCpanelMethod(raw string) bool {
if raw == "" {
return false
}
for i := 0; i < len(raw); {
var c byte
c, i = patternLogByte(raw, i)
valid := ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') ||
('0' <= c && c <= '9') || strings.ContainsRune("!#$%&'*+-.^_`|~", rune(c))
if !valid {
return false
}
}
return true
}
// cpanelStatuses yields `" <code> ` fields without collecting client text.
func cpanelStatuses(s string) iter.Seq[string] {
return func(yield func(string) bool) {
for i := 0; i < len(s); i++ {
j := strings.IndexByte(s[i:], '"')
if j < 0 {
break
}
i += j
rest := s[i+1:]
if rest == "" || (rest[0] != ' ' && rest[0] != '\t') {
continue
}
code, tail := patternToken(rest)
if len(code) == 3 && patternDigitsOnly(code) && code[0] >= '1' && code[0] <= '5' &&
tail != "" && (tail[0] == ' ' || tail[0] == '\t') && !yield(code) {
return
}
}
}
}
// cpanelServedPort reads the port cpsrvd ends its line with, after its
// last quoted field. Client text sits inside quotes before it.
func cpanelServedPort(line string) string {
line = strings.TrimRight(line, " \t\r\n")
i := strings.LastIndexAny(line, " \t")
if i < 0 || !strings.HasSuffix(strings.TrimRight(line[:i], " \t"), `"`) || !validPort(line[i+1:]) {
return ""
}
return line[i+1:]
}
// cpanelAccessLogTime reads the time cpsrvd stamped on an access_log line
// with ident "-". The client chooses the user, which may hold spaces,
// brackets and quotes, so only one complete parse dates the line: a user
// holding its own timestamp followed by a quote leaves the line with no
// time, never the time the client wrote. An earlier bracketed candidate
// or a suffix that cannot be parsed strictly leaves the line undated.
func cpanelAccessLogTime(line string) (time.Time, bool) {
peer, rest, ok := strings.Cut(line, " ")
if !ok || net.ParseIP(peer) == nil {
return time.Time{}, false
}
rest, ok = strings.CutPrefix(rest, "- ")
if !ok {
return time.Time{}, false
}
stamp, suffix, ok := cpanelUniqueStamp(rest)
if !ok {
return time.Time{}, false
}
// With an earlier candidate, this timestamp might be in a header on
// a line whose actual timestamp has another shape.
for open, after := range cpanelRequestStarts(rest) {
if open < 0 {
continue
}
if len(after) != len(suffix) {
return time.Time{}, false
}
break
}
if _, ok := strictCpanelSuffix(suffix); !ok {
return time.Time{}, false
}
at, err := time.Parse("01/02/2006:15:04:05 -0700", stamp)
return at, err == nil
}
// cpanelStampLen is the length of cpsrvd's `01/02/2006:15:04:05 -0700`.
const cpanelStampLen = 25
// cpanelUniqueStamp finds the bracketed timestamp, followed by blanks and a
// quote, that ends a nonempty user field. Any second such timestamp anywhere
// in the line, valid date or not, makes the split ambiguous. Candidates never
// overlap, so one pass bounds the work by the line.
func cpanelUniqueStamp(s string) (stamp, rest string, ok bool) {
found, at := 0, -1
for i := 0; i < len(s); i++ {
j := strings.IndexByte(s[i:], '[')
if j < 0 {
break
}
i += j
end := i + 1 + cpanelStampLen
if end >= len(s) || s[end] != ']' || !cpanelStampShape(s[i+1:end]) {
continue
}
after := strings.TrimLeft(s[end+1:], " \t")
if len(after) == len(s)-end-1 || after == "" || after[0] != '"' {
continue
}
found++
at, stamp, rest = i, s[i+1:end], after
i = end
}
if found != 1 || at < 2 || (s[at-1] != ' ' && s[at-1] != '\t') {
return "", "", false
}
return stamp, rest, true
}
// cpanelStampShape reports whether s has the shape of cpsrvd's timestamp.
func cpanelStampShape(s string) bool {
const shape = "00/00/0000:00:00:00 +0000"
if len(s) != len(shape) {
return false
}
for i := range len(shape) {
switch c := s[i]; shape[i] {
case '0':
if c < '0' || c > '9' {
return false
}
case '+':
if c != '+' && c != '-' {
return false
}
default:
if c != shape[i] {
return false
}
}
}
return true
}
// strictCpanelSuffix parses what follows a timestamp through the end of the
// line, reading quoted fields with escapes. It returns the port the line
// names: its bare last field, or the host:port of a third or later quoted
// field. Use it only at the first bracketed candidate: a later strict
// suffix may belong to client text.
func strictCpanelSuffix(s string) (port string, ok bool) {
request, s, ok := patternQuotedField(s)
if !ok {
return "", false
}
method, afterMethod, hasMethod := strings.Cut(request, " ")
target, proto, hasProto := strings.Cut(afterMethod, " ")
if !hasMethod || !hasProto || !validCpanelMethod(method) ||
!patternValidRawTarget(target) || !patternHTTPVersion(proto) {
return "", false
}
status, s := patternToken(s)
if len(status) != 3 || !patternDigitsOnly(status) || status[0] < '1' || status[0] > '5' {
return "", false
}
size, s := patternToken(s)
if size != "-" && !patternDigitsOnly(size) {
return "", false
}
var last string
quoted := 0
for ; ; quoted++ {
s = strings.TrimLeft(s, " \t")
if s == "" || s[0] != '"' {
break
}
if quoted == 4 {
return "", false
}
if last, s, ok = patternQuotedField(s); !ok {
return "", false
}
}
if s == "" {
// Referer and user agent come first; only a third field names the host.
if quoted >= 3 {
return hostPort(last), true
}
return "", true
}
port, tail := patternToken(s)
if strings.TrimLeft(tail, " \t") != "" || !validPort(port) {
return "", false
}
return port, true
}
// hostPort returns the valid port ending "host:port", or "".
func hostPort(s string) string {
i := strings.LastIndexByte(s, ':')
if i < 0 || !validPort(s[i+1:]) {
return ""
}
return s[i+1:]
}
func validPort(s string) bool {
if s == "" || len(s) > 5 || !patternDigitsOnly(s) {
return false
}
n := atoiSafe(s)
return n > 0 && n <= 65535
}
package checks
import (
"context"
"fmt"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
const (
sessionLogPath = "/usr/local/cpanel/logs/session_log"
sessionLogTailLines = 1000
defaultMultiIPThreshold = 3
defaultMultiIPWindowMin = 60
)
// CheckCpanelLogins parses the cPanel session log for suspicious login activity:
// - cPanel (cpaneld) logins from non-infra IPs
// - Same account logged in from multiple distinct IPs (credential compromise indicator)
// - Password change purge events (attacker or auto-response password resets)
func CheckCpanelLogins(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
var findings []alert.Finding
lines := tailFile(sessionLogPath, sessionLogTailLines)
if len(lines) == 0 {
return nil
}
// Track logins per account for multi-IP correlation
accountIPs := make(map[string]map[string]bool)
var passwordChanges []string
// Determine cutoff - only alert on events within the scan window
cutoff := time.Now().Add(-time.Duration(multiIPWindowMin(cfg)) * time.Minute)
for _, line := range lines {
r, ok := ParseSessionRecord(line)
if !ok || r.At.IsZero() || r.At.Before(cutoff) {
continue
}
// Detect cPanel logins from non-infra IPs
// Skip API/portal sessions (create_user_session) - only alert on direct form login
if r.Service == "cpaneld" && r.Event == "NEW" {
if r.CreatedByAPI() {
continue
}
addr, ok := r.ClientAddr()
if !ok || addr.IsLoopback() || isInfraIP(addr.String(), cfg.InfraIPs) {
continue
}
ip, account := addr.String(), r.Account
// Always track for multi-IP correlation (even when login alerts suppressed)
if accountIPs[account] == nil {
accountIPs[account] = make(map[string]bool)
}
accountIPs[account][ip] = true
// WARNING severity - logins are audit trail, not paging-level.
// Multi-IP correlation stays CRITICAL via its own check below.
if !cfg.Suppressions.SuppressCpanelLogin {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "cpanel_login",
Message: fmt.Sprintf("cPanel direct login from non-infra IP: %s (account: %s)", ip, account),
Details: truncateString(line, 300),
SourceIP: ip,
})
}
}
// Detect password change purge events
if r.Event == "PURGE" && r.Reason() == "password_change" {
passwordChanges = append(passwordChanges, r.Account)
}
}
// Multi-IP correlation: same account from 3+ distinct non-infra IPs
threshold := multiIPThreshold(cfg)
for account, ips := range accountIPs {
if len(ips) >= threshold {
ipList := make([]string, 0, len(ips))
for ip := range ips {
ipList = append(ipList, ip)
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "cpanel_multi_ip_login",
Message: fmt.Sprintf("Account '%s' logged in from %d distinct IPs (credential compromise likely)", account, len(ips)),
Details: fmt.Sprintf("IPs: %s\nThreshold: %d IPs within %d minutes", strings.Join(ipList, ", "), threshold, multiIPWindowMin(cfg)),
TenantID: HostingAccountForUser(account),
})
}
}
// Password change events - deduplicate by account
seen := make(map[string]bool)
for _, account := range passwordChanges {
if seen[account] {
continue
}
seen[account] = true
// Check if triggered by security module (Imunify auto-response) vs user action
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "cpanel_password_purge",
Message: fmt.Sprintf("cPanel sessions purged via password change for account: %s", account),
Details: "This may indicate an automated security response or attacker-initiated password change",
})
}
return findings
}
// CheckCpanelFileManager parses the cPanel access log for file management
// operations from non-infra IPs (file uploads, edits via cPanel File Manager).
func CheckCpanelFileManager(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
lines := tailFile("/usr/local/cpanel/logs/access_log", 300)
// Only match actual write actions - not read-only calls like get_homedir.
// Skip 401/403 responses - the server rejected the request, no write occurred.
// Match against request URI only, not the full line (referer can contain "upload").
filemanWriteActions := []string{
"fileman/save_file_content",
"fileman/upload_files",
"fileman/save_file",
"fileman/paste",
"fileman/rename",
"fileman/delete",
}
for _, line := range lines {
rec, ok := ParseCpanelAccessRecord(line)
// Only check cPanel (port 2083) entries. Skip rejected requests - no
// write occurred - when the line names its status unambiguously.
if !ok || rec.Port != "2083" || rec.Status == "401" || rec.Status == "403" {
continue
}
ip := rec.IP
if !rec.Proxy && (isInfraIP(ip, cfg.InfraIPs) || ip == "127.0.0.1") {
continue
}
source, dedup := "non-infra IP: "+ip, ""
if rec.Proxy {
source, dedup = "a service subdomain (client address unverified)", "proxy:"+ip
}
// Match the request target only: the referer can name upload pages
// like upload-ajax.html. The user is "-" when unauthenticated.
owner := ""
if rec.User != "-" && rec.User != "" {
owner = HostingAccountForUser(rec.User)
}
if rec.TargetContains(filemanWriteActions...) {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "cpanel_file_upload",
Message: fmt.Sprintf("cPanel File Manager write operation from %s", source),
Details: truncateString(line, 300),
DedupKey: dedup,
TenantID: owner,
})
}
}
return findings
}
func multiIPThreshold(cfg *config.Config) int {
if cfg.Thresholds.MultiIPLoginThreshold > 0 {
return cfg.Thresholds.MultiIPLoginThreshold
}
return defaultMultiIPThreshold
}
func multiIPWindowMin(cfg *config.Config) int {
if cfg.Thresholds.MultiIPLoginWindowMin > 0 {
return cfg.Thresholds.MultiIPLoginWindowMin
}
return defaultMultiIPWindowMin
}
package checks
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"path/filepath"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// credentialReuseMinAccounts is the threshold for emitting the
// password-reuse finding: the same WordPress admin password hash present
// on this many or more distinct accounts. Two is the common shared-hosting
// pattern (an agency reusing one admin password across client sites) where
// a single credential disclosure compromises every site at once.
const credentialReuseMinAccounts = 2
// CheckCredentialReuse flags WordPress administrator accounts that share
// an identical password hash across two or more distinct hosting accounts.
// Password hashes are salted on modern WordPress installs, so this is an
// exact at-rest hash reuse signal, not a weak-password detector.
//
// Privacy: the raw password hash is never stored, logged, or emitted. Only
// a truncated one-way fingerprint is used to group identical hashes, and
// findings report the affected accounts and a count -- not the hash.
func CheckCredentialReuse(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
wpConfigs := credentialReuseWPConfigs(ctx)
// fingerprint -> set of distinct accounts carrying that admin hash.
byFingerprint := map[string]map[string]struct{}{}
for _, wpConfig := range wpConfigs {
if ctx.Err() != nil {
return nil
}
account := wpConfigUser(filepath.Dir(wpConfig))
if account == "" {
continue
}
creds, complete := parseWPConfigChecked(wpConfig)
if !complete {
markCheckIncomplete(ctx, "credential_reuse")
continue
}
if creds.dbName == "" {
markCheckIncomplete(ctx, "credential_reuse")
continue
}
prefix, ok := resolveTablePrefix(creds)
if !ok {
markCheckIncomplete(ctx, "credential_reuse")
continue
}
creds.tablePrefix = prefix
fingerprints, err := adminPasswordFingerprintsForSite(creds, prefix)
if err != nil {
markCheckIncomplete(ctx, "credential_reuse")
continue
}
for _, fp := range fingerprints {
if fp == "" {
continue
}
if byFingerprint[fp] == nil {
byFingerprint[fp] = map[string]struct{}{}
}
byFingerprint[fp][account] = struct{}{}
}
}
return buildCredentialReuseFindings(byFingerprint, credentialReuseMinAccounts)
}
// adminPasswordFingerprintsForSite returns fingerprints for the admin
// user_pass hashes currently stored on the WordPress site. Uses root MySQL
// because wp-config passwords drift on cPanel hosts (same rationale as
// adminEmailsForSite).
func adminPasswordFingerprintsForSite(creds wpDBCreds, prefix string) ([]string, error) {
query := fmt.Sprintf(
"SELECT DISTINCT u.user_pass FROM `%susers` u "+
"JOIN `%susermeta` um ON u.ID = um.user_id "+
"WHERE um.meta_key = '%scapabilities' AND um.meta_value LIKE '%%administrator%%'",
prefix, prefix, prefix,
)
rows, err := runMySQLQueryRootWithError(creds.dbName, query)
if err != nil {
return nil, err
}
var out []string
for _, row := range rows {
fp := credentialHashFingerprint(strings.TrimSpace(row))
if fp != "" {
out = append(out, fp)
}
}
return out, nil
}
// credentialHashFingerprint maps a raw password hash to a short,
// non-reversible grouping key. Two identical hashes map to the same
// fingerprint without returning the raw hash. Empty input yields "".
func credentialHashFingerprint(rawHash string) string {
if rawHash == "" {
return ""
}
sum := sha256.Sum256([]byte(rawHash))
return "fp:" + hex.EncodeToString(sum[:])[:16]
}
// buildCredentialReuseFindings emits one Warning per fingerprint shared by
// at least minAccounts distinct accounts. The finding never includes the
// hash or fingerprint-as-secret -- only the affected account list and a
// count, so an operator can rotate the shared credential.
func buildCredentialReuseFindings(byFingerprint map[string]map[string]struct{}, minAccounts int) []alert.Finding {
if minAccounts < 2 {
minAccounts = 2
}
type credentialReuseGroup struct {
accounts []string
}
groups := make([]credentialReuseGroup, 0, len(byFingerprint))
for _, accountSet := range byFingerprint {
if len(accountSet) < minAccounts {
continue
}
accounts := make([]string, 0, len(accountSet))
for a := range accountSet {
accounts = append(accounts, a)
}
sort.Strings(accounts)
groups = append(groups, credentialReuseGroup{accounts: accounts})
}
sort.Slice(groups, func(i, j int) bool {
return strings.Join(groups[i].accounts, "\x00") < strings.Join(groups[j].accounts, "\x00")
})
var out []alert.Finding
for _, group := range groups {
accounts := group.accounts
out = append(out, alert.Finding{
Severity: alert.Warning,
Check: "credential_reuse",
Message: fmt.Sprintf("Identical WordPress admin password hash reused across %d accounts: %s",
len(accounts), strings.Join(accounts, ", ")),
Details: fmt.Sprintf("Accounts sharing one admin password hash: %s\n"+
"Rotate the shared credential: a single disclosure compromises every listed site.",
strings.Join(accounts, ", ")),
Timestamp: time.Now(),
})
}
return out
}
// credentialReuseWPConfigs lists the WordPress installs this check fingerprints.
// A hash reused between a primary site and an addon install is still reuse.
func credentialReuseWPConfigs(ctx context.Context) []string {
installs := wpInstalls(ctx, "credential_reuse")
out := make([]string, 0, len(installs))
for _, in := range installs {
out = append(out, in.ConfigPath)
}
return out
}
package checks
import (
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"hash"
"path/filepath"
"regexp"
"strconv"
"strings"
)
// cronScheduleOnlyReason names the content evidence in a demoted finding.
const cronScheduleOnlyReason = "only cron job run times changed"
// cronDailyTimeLine matches a system crontab line whose minute and hour are
// single numbers, capturing everything around them verbatim. Such a line is
// never an environment assignment: cron reads a leading name, and a name
// followed by blanks and then anything but "=" makes cron parse a job.
var cronDailyTimeLine = regexp.MustCompile(`^([ \t]*)([0-9]{1,2})([ \t]+)([0-9]{1,2})([ \t].*)$`)
// Cronie chooses randomized calendar fields again on every file reload,
// including entries whose bytes did not change. They cannot prove a pure
// time-of-day move, even if another entry caused the reload. The minute and
// hour admit ranges and the leading "-" that turns off logging, so any job
// line reaches the calendar test.
var cronCalendarLine = regexp.MustCompile(`^[ \t]*[0-9*,/~-]+[ \t]+[0-9*,/~-]+[ \t]+([^ \t]+)[ \t]+([^ \t]+)[ \t]+([^ \t]+)[ \t]+`)
// cronScheduleFingerprint digests a cron.d file with the minute and hour of
// each once-a-day job left out, and returns "" for any other path or a file
// with randomized calendar fields. Equal fingerprints differ at most in
// the time of day their daily jobs run: every command, user, environment line,
// comment and their order is compared byte for byte, so no new program, job,
// frequency or environment can hide behind an equal fingerprint.
//
// Only a cron.d drop-in is read this way. A user crontab has no user field,
// and in a run-parts script the leading numbers are a command and its
// argument.
func cronScheduleFingerprint(path string, content []byte) string {
if filepath.Base(filepath.Dir(path)) != "cron.d" {
return ""
}
h := sha256.New()
for _, line := range strings.Split(string(content), "\n") {
if m := cronCalendarLine.FindStringSubmatch(line); m != nil && strings.ContainsAny(m[1]+m[2]+m[3], "~") {
return ""
}
// Times outside cron's range make the job invalid. Leaving them in
// the fingerprint keeps a dormant job from starting as a mere move.
if m := cronDailyTimeLine.FindStringSubmatch(line); m != nil && cronTimeWithin(m[2], 59) && cronTimeWithin(m[4], 23) {
writeFingerprintLine(h, 'J', m[1], m[3], m[5])
continue
}
writeFingerprintLine(h, 'L', line)
}
return hex.EncodeToString(h.Sum(nil))
}
// cronScheduleUnchanged reports whether two fingerprints show a change that
// only moved daily jobs. An empty fingerprint -- a path outside cron.d, or no
// earlier version on record -- proves nothing.
func cronScheduleUnchanged(prev, cur string) bool {
return cur != "" && prev == cur
}
// The polling checks need the same metadata evidence as the watchset refresh.
// A permission, ownership or symlink change can activate a previously ignored
// cron file even when its commands are unchanged.
func cronScheduleSnapshotFingerprint(path string, content []byte) string {
schedule := cronScheduleFingerprint(path, content)
if schedule == "" {
return ""
}
identity, regular, known := sensitivePathIdentity(path)
if !known || !regular {
return ""
}
h := sha256.New()
writeFingerprintLine(h, 'P', identity, schedule)
return hex.EncodeToString(h.Sum(nil))
}
// Separate raw-key writes can be saved or observed between updates. Bind the
// fingerprint to its content hash so mixed baselines never justify a demotion.
func cronScheduleBaselineUnchanged(prevHash, prevRecord, curSchedule string) bool {
return curSchedule != "" && prevRecord == prevHash+":"+curSchedule
}
// writeFingerprintLine adds one tagged, length-prefixed line to h, so a
// masked job line can never collide with a literal line or a line split.
func writeFingerprintLine(h hash.Hash, tag byte, parts ...string) {
_, _ = h.Write([]byte{tag})
var n [8]byte
for _, p := range parts {
binary.BigEndian.PutUint64(n[:], uint64(len(p)))
_, _ = h.Write(n[:])
_, _ = h.Write([]byte(p))
}
}
func cronTimeWithin(field string, maxValue int) bool {
v, err := strconv.Atoi(field)
return err == nil && v <= maxValue
}
package checks
import "github.com/pidginhost/csm/internal/platform"
// cronSpoolDir returns the directory holding per-user crontabs for this
// platform (cronie: /var/spool/cron; Debian cron: /var/spool/cron/crontabs).
// Var so tests can pin a layout without touching the host.
var cronSpoolDir = func() string { return platform.Detect().CronSpoolDir() }
// webServerUsers returns the accounts the web server runs as on this
// platform; a seam so tests can stand in for platform detection.
var webServerUsers = func() []string { return platform.Detect().WebServerUsers() }
package checks
import (
"fmt"
"os"
"path/filepath"
"github.com/pidginhost/csm/internal/quarantinefs"
)
// fixCrontabAllowedRoots limits suspicious_crontab remediation to the cron
// spool. Declared as a var so tests can redirect it under t.TempDir()
// without touching the real /var/spool/cron.
var fixCrontabAllowedRoots = []string{"/var/spool/cron"}
// fixSuspiciousCrontab copies a user crontab matching known-bad persistence
// markers into quarantine, writes a restore-ready metadata sidecar, and then
// truncates the live file to empty. Truncation (not deletion) keeps cron(8)
// from re-reading stale content and preserves the caller's ability to
// inspect file perms while the malware is gone.
func fixSuspiciousCrontab(path string) RemediationResult {
if path == "" {
return RemediationResult{Error: "could not extract file path from finding"}
}
path, info, err := resolveExistingFixPath(path, fixCrontabAllowedRoots)
if err != nil {
return RemediationResult{Error: err.Error()}
}
data, err := osFS.ReadFile(path)
if err != nil {
return RemediationResult{Error: fmt.Sprintf("cannot read: %v", err)}
}
user := filepath.Base(path)
qPath := newQuarantinePath(quarantineDir, "crontab_"+user)
meta := quarantineMetadata(path, info, "suspicious_crontab remediation")
if err := storeQuarantineBackup(qPath, data, meta, 0600); err != nil {
return RemediationResult{Error: fmt.Sprintf("cannot create durable crontab backup: %v", err)}
}
// Truncate live crontab. 0600 is the mode cron(8) enforces for user
// spool files; any other mode makes cron skip the file with a warning.
// #nosec G306 -- cron(8) rejects world-readable user crontabs, so 0600
// is the only safe mode for /var/spool/cron/<user>.
if err := os.WriteFile(path, []byte{}, 0600); err != nil {
return RemediationResult{Error: fmt.Sprintf("cannot truncate crontab: %v", err)}
}
if err := quarantinefs.SyncFilePath(path); err != nil {
return RemediationResult{Error: fmt.Sprintf("crontab truncated but not synced; recovery copy retained at %s: %v", qPath, err)}
}
return RemediationResult{
Success: true,
Action: fmt.Sprintf("quarantined crontab %s -> %s and truncated", path, qPath),
Description: fmt.Sprintf("Truncated %d-byte crontab; copy saved to quarantine", len(data)),
}
}
package checks
import (
"context"
"encoding/base64"
"fmt"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/state"
)
// crontabBase64Truncated counts base64 candidates that hit
// crontabBase64BlobMaxBytes. Sustained growth means an attacker is
// padding outer blobs to push the real payload past the decode window;
// raise the cap or split the scanner.
var (
crontabBase64Truncated *metrics.Counter
crontabBase64TruncatedOnce sync.Once
)
func observeCrontabBase64Truncation() {
crontabBase64TruncatedOnce.Do(func() {
crontabBase64Truncated = metrics.NewCounter(
"csm_checks_crontab_base64_truncated_total",
"Crontab base64 candidates that exceeded the per-blob decode cap before decoded-content pattern matching ran. Sustained growth means the scanner inspected only the leading decoded window of large encoded cron content.",
)
metrics.MustRegister("csm_checks_crontab_base64_truncated_total", crontabBase64Truncated)
})
crontabBase64Truncated.Inc()
}
func CheckCrontabs(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
var findings []alert.Finding
// Rank account crontabs by mtime desc so recently-touched users
// process first when the check timeout cuts iteration short. Keep
// root outside the account cap; it is a system baseline, not an
// account-scoped path.
if ctx.Err() != nil {
return findings
}
crontabs, _ := osFS.Glob(filepath.Join(cronSpoolDir(), "*"))
var rootCrontabs []string
accountCrontabs := make([]string, 0, len(crontabs))
for _, path := range crontabs {
if filepath.Base(path) == "root" {
rootCrontabs = append(rootCrontabs, path)
continue
}
accountCrontabs = append(accountCrontabs, path)
}
rankedRootCrontabs := rankPathsByMtimeDesc(ctx, rootCrontabs, 0)
if ctx.Err() != nil {
return findings
}
for _, path := range rankedRootCrontabs {
if ctx.Err() != nil {
return findings
}
hash, err := hashFileContent(path)
if err != nil {
continue
}
if ctx.Err() != nil {
return findings
}
key := "_crontab_root_hash"
prev, exists := store.GetRaw(key)
if exists && prev != hash {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "crontab_change",
Message: "Root crontab modified",
Details: "Review with: crontab -l",
})
}
store.SetRaw(key, hash)
}
rankedCrontabs := rankPathsByMtimeDesc(ctx, accountCrontabs, accountScanMaxFiles(ctx, cfg))
if ctx.Err() != nil {
return findings
}
for _, path := range rankedCrontabs {
if ctx.Err() != nil {
return findings
}
user := filepath.Base(path)
data, err := osFS.ReadFile(path)
if err != nil {
continue
}
if ctx.Err() != nil {
return findings
}
// The spool basename is a hosting owner only when an account home
// of that name exists; service users keep the finding unattributed.
owner := ""
if accountHomeExists(user) {
owner = user
}
content := string(data)
for _, pattern := range MatchCrontabPatternsDeep(content, cfg) {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "suspicious_crontab",
Message: fmt.Sprintf("Suspicious pattern in crontab for user %s: %s", user, pattern),
Details: fmt.Sprintf("File: %s\nContent:\n%s", path, truncate(content, 500)),
FilePath: path,
TenantID: owner,
})
}
}
// Check /etc/cron.d for new files. This is a system directory, so
// account_scan_max_files must not hide older cron.d baselines.
if ctx.Err() != nil {
return findings
}
cronDFiles, globErr := osFS.Glob("/etc/cron.d/*")
rankedCronDFiles := rankPathsByMtimeDesc(ctx, cronDFiles, 0)
if ctx.Err() != nil {
return findings
}
// Before the first complete pass every file is install backlog and
// only gets baselined; afterwards a file with no stored hash appeared
// since the last run and is reported, not silently absorbed.
_, cronDBaselined := store.GetRaw(cronDBaselineKey)
cronDBaselineComplete := globErr == nil
for _, path := range rankedCronDFiles {
if ctx.Err() != nil {
return findings
}
data, err := osFS.ReadFile(path)
if err != nil {
cronDBaselineComplete = false
continue
}
if ctx.Err() != nil {
return findings
}
hash := hashBytes(data)
key := fmt.Sprintf("_crond:%s", filepath.Base(path))
prev, exists := store.GetRaw(key)
// Bind the fingerprint to its hash: a partial baseline update must
// not make a changed command look like a time-of-day move.
schedule := cronScheduleSnapshotFingerprint(path, data)
scheduleKey := cronDScheduleKeyPrefix + filepath.Base(path)
prevSchedule, _ := store.GetRaw(scheduleKey)
// The same file reaches the realtime write detector, which rescores
// a vendor-driven change instead of paging. Scoring the scheduled
// diff on its own left an upgrade or a panel maintenance run
// reporting High through whichever detector saw it first.
switch {
case exists && prev != hash:
scheduleOnly := cronScheduleBaselineUnchanged(prev, prevSchedule, schedule)
findings = append(findings, rescoreSensitive(alert.Finding{
Severity: alert.High,
Check: "crond_change",
Message: fmt.Sprintf("Cron.d file modified: %s", path),
}, "cron", data, 0, scheduleOnly, time.Now()))
case !exists && cronDBaselined:
findings = append(findings, rescoreSensitive(alert.Finding{
Severity: alert.High,
Check: "crond_change",
Message: fmt.Sprintf("Cron.d file added: %s", path),
Details: fmt.Sprintf("File: %s\nContent: %s", path, alert.RedactCommandLine(truncate(strings.TrimSpace(string(data)), cronDExcerptLen))),
}, "cron", data, 0, false, time.Now()))
}
store.SetRaw(key, hash)
if schedule != "" {
store.SetRaw(scheduleKey, hash+":"+schedule)
} else {
store.DeleteRaw(scheduleKey)
}
}
if cronDBaselineComplete && ctx.Err() == nil {
store.SetRaw(cronDBaselineKey, "1")
}
return findings
}
// cronDBaselineKey marks that /etc/cron.d was fully enumerated once; from
// then on an unknown file is new rather than backlog.
const cronDBaselineKey = "_crond:_baseline_complete"
// cronDScheduleKeyPrefix keys each cron.d fingerprint bound to its content hash.
const cronDScheduleKeyPrefix = "_crond_schedule:"
// cronDExcerptLen bounds the content quoted for a new cron.d file.
const cronDExcerptLen = 300
func truncate(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
}
// crontabSuspiciousPatterns is the shared allowlist of case-insensitive
// substrings that mark a crontab line as likely malicious. Single source of
// truth for CheckCrontabs (system scan) and makeAccountCrontabCheck
// (per-account scan) so the two cannot drift apart.
var crontabSuspiciousPatterns = []string{
"defunct-kernel",
"SEED PRNG",
"base64_decode",
"base64 -d|bash",
"base64 -d | bash",
"base64 --decode|bash",
"eval(",
"/dev/tcp/",
"gsocket",
"gs-netcat",
"reverse",
"bash -i",
"/bin/sh -i",
"nc -e",
"ncat -e",
"python -c",
"perl -e",
}
// matchCrontabPatterns returns patterns from crontabSuspiciousPatterns that
// appear as case-insensitive substrings of content, preserving list order.
func matchCrontabPatterns(content string) []string {
lower := strings.ToLower(content)
var matched []string
for _, pattern := range crontabSuspiciousPatterns {
if strings.Contains(lower, strings.ToLower(pattern)) {
matched = append(matched, pattern)
}
}
return matched
}
// crontabBase64BlobMaxBytesDefault is the built-in fallback cap for a
// single base64 candidate before decoding. 16384 encoded bytes
// (~12 KiB decoded) comfortably fits any realistic gsocket /
// `base64 -d|bash` payload while bounding work on adversarial input.
// Operator override: cfg.Thresholds.CrontabBase64BlobMaxBytes.
//
// Must stay a multiple of 4 -- standard base64 needs aligned input or
// DecodeString errors and the candidate is silently skipped. The
// validator rejects non-aligned operator values.
const crontabBase64BlobMaxBytesDefault = 16384
// effectiveCrontabBase64BlobMaxBytes returns the operator-configured cap
// or the built-in default when unset. The validator enforces multiple-of-4
// alignment so this returns a safe value without further checks.
func effectiveCrontabBase64BlobMaxBytes(cfg *config.Config) int {
if cfg == nil || cfg.Thresholds.CrontabBase64BlobMaxBytes <= 0 {
return crontabBase64BlobMaxBytesDefault
}
return cfg.Thresholds.CrontabBase64BlobMaxBytes
}
// crontabBase64BlobMaxCount caps the number of base64 candidates examined
// per crontab. A realistic gsocket cron entry has one outer blob; we
// allow enough headroom for a handful without doing unbounded work.
const crontabBase64BlobMaxCount = 16
// crontabBase64BlobRE matches contiguous standard-alphabet base64 of
// length >= 40 (with optional padding). The 40-char floor avoids matching
// short config IDs and noise like Wordfence cookie names.
var crontabBase64BlobRE = regexp.MustCompile(`[A-Za-z0-9+/]{40,}={0,2}`)
// MatchCrontabPatternsDeep is matchCrontabPatterns plus a single base64
// decode pass: it pulls out base64 candidates from content and re-runs
// pattern matching on the decoded bytes. Catches attackers who wrap the
// `base64 -d|bash` pipe chain in an outer base64 layer so the literal
// markers never appear in the cron file as written. Single decode depth;
// no recursion. cfg nil uses the built-in defaults; pass the live
// operator config to honour `thresholds.crontab_base64_blob_max_bytes`.
func MatchCrontabPatternsDeep(content string, cfg *config.Config) []string {
maxBytes := effectiveCrontabBase64BlobMaxBytes(cfg)
matched := matchCrontabPatterns(content)
seen := make(map[string]bool, len(matched))
for _, m := range matched {
seen[m] = true
}
candidates := crontabBase64BlobRE.FindAllString(content, crontabBase64BlobMaxCount)
for _, blob := range candidates {
if len(blob) > maxBytes {
observeCrontabBase64Truncation()
blob = blob[:maxBytes]
}
decoded, err := base64.StdEncoding.DecodeString(blob)
if err != nil {
continue
}
for _, m := range matchCrontabPatterns(string(decoded)) {
if !seen[m] {
matched = append(matched, m)
seen[m] = true
}
}
}
return matched
}
package checks
import (
"context"
"crypto/sha256"
"fmt"
"path/filepath"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
// adminEmailRetention bounds how far back an admin observation stays
// relevant for overlap detection. A contractor email seen six months
// ago on one account and never since is not actionable signal -- the
// access likely lapsed. The window is a deliberate balance against the
// alternative of evicting on every scan, which would lose overlaps when
// scans run asynchronously across customer accounts.
const adminEmailRetention = 90 * 24 * time.Hour
// adminEmailDefaultMinAccounts is the default threshold for emitting
// the cross-account overlap finding. Matches the most common
// compromise pattern on shared hosting: a contractor administering two
// or more customer cPanels.
const adminEmailDefaultMinAccounts = 2
// CheckAdminEmailOverlap records every WordPress administrator email
// encountered during an account scan into a server-wide bbolt bucket,
// then emits a Warning finding for each email whose owner list now
// spans the configured minimum number of distinct accounts. The
// detection surface is shared-hosting credential leakage: a single
// compromised contractor account is one credential disclosure away
// from administrator access on every site they touch.
//
// The check is silent when the bbolt store is unavailable (early
// daemon startup, test harness without state injection) -- it can't
// observe overlap without persistence between scans, and falling
// silent is better than a misleading partial result.
func CheckAdminEmailOverlap(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
db := store.Global()
if db == nil {
return nil
}
now := time.Now()
wpConfigs := adminOverlapWPConfigs(ctx)
for _, wpConfig := range wpConfigs {
if ctx.Err() != nil {
return nil
}
account := wpConfigUser(filepath.Dir(wpConfig))
creds, complete := parseWPConfigChecked(wpConfig)
if !complete {
markCheckIncomplete(ctx, "admin_overlap")
continue
}
if creds.dbName == "" {
markCheckIncomplete(ctx, "admin_overlap")
continue
}
prefix, ok := resolveTablePrefix(creds)
if !ok {
markCheckIncomplete(ctx, "admin_overlap")
continue
}
creds.tablePrefix = prefix
emails, err := adminEmailsForSite(creds, prefix)
if err != nil {
markCheckIncomplete(ctx, "admin_overlap")
continue
}
for _, email := range emails {
if err := db.RecordAdminEmail(email, account, creds.dbName, now); err != nil {
markCheckIncomplete(ctx, "admin_overlap")
}
}
}
min := adminEmailDefaultMinAccounts
if cfg != nil && cfg.Detection.AdminOverlapMinAccounts > 0 {
min = cfg.Detection.AdminOverlapMinAccounts
}
overlaps, err := db.OverlappingAdminEmails(min, adminEmailRetention)
if err != nil {
markCheckIncomplete(ctx, "admin_overlap")
return nil
}
if len(overlaps) == 0 {
return nil
}
overlaps = filterTrustedAdminOverlaps(overlaps, cfg)
return buildAdminOverlapFindings(overlaps)
}
// adminEmailsForSite returns the lowercase admin emails currently
// configured on the WordPress site. Uses the existing root-MySQL
// helper so it works on cPanel hosts where wp-config passwords drift.
func adminEmailsForSite(creds wpDBCreds, prefix string) ([]string, error) {
query := fmt.Sprintf(
"SELECT DISTINCT LOWER(u.user_email) FROM `%susers` u "+
"JOIN `%susermeta` um ON u.ID = um.user_id "+
"WHERE um.meta_key = '%scapabilities' AND um.meta_value LIKE '%%administrator%%'",
prefix, prefix, prefix,
)
rows, err := runMySQLQueryRootWithError(creds.dbName, query)
if err != nil {
return nil, err
}
var out []string
for _, row := range rows {
row = strings.TrimSpace(row)
if row != "" {
out = append(out, row)
}
}
return out, nil
}
// buildAdminOverlapFindings collapses each overlap entry into a single Warning
// finding. The sorted, de-duplicated account set feeds both operator-facing
// text and the explicit identity, so input order and multiple schemas owned by
// one account cannot change the finding key.
func buildAdminOverlapFindings(overlaps map[string][]store.AdminEmailEntry) []alert.Finding {
emails := make([]string, 0, len(overlaps))
for email := range overlaps {
emails = append(emails, email)
}
sort.Strings(emails)
out := make([]alert.Finding, 0, len(emails))
for _, email := range emails {
owners := overlaps[email]
accountSet := make(map[string]struct{}, len(owners))
for _, o := range owners {
accountSet[o.Account] = struct{}{}
}
accounts := make([]string, 0, len(accountSet))
for a := range accountSet {
accounts = append(accounts, a)
}
sort.Strings(accounts)
details := strings.Builder{}
fmt.Fprintf(&details, "Email: %s\nAccounts: %s\n", email, strings.Join(accounts, ", "))
for _, o := range owners {
fmt.Fprintf(&details, "- %s (schema %s, last seen %s)\n", o.Account, o.Schema, o.LastSeen.Format(time.RFC3339))
}
out = append(out, alert.Finding{
Severity: alert.Warning,
Check: "admin_cross_account_overlap",
// The overlap itself is the identity: this email on this set of
// accounts. Details carry each account's last-seen time, and
// Finding.Key() hashes Details, so without an explicit key every
// scan minted a new finding for an unchanged overlap.
DedupKey: adminOverlapDedupKey(email, accounts),
Message: fmt.Sprintf("Admin email %s appears on %d accounts: %s", email, len(accounts), strings.Join(accounts, ", ")),
Details: details.String(),
Timestamp: time.Now(),
})
}
return out
}
// adminOverlapDedupKey identifies one overlap by its substance: the shared
// email and the set of accounts carrying it. An email that spreads to another
// account is a new situation and gets its own key; the same overlap re-observed
// on the next scan keeps this one. accounts is already sorted and de-duplicated
// by the caller.
func adminOverlapDedupKey(email string, accounts []string) string {
identity := strings.Join(append([]string{email}, accounts...), "\x00")
digest := sha256.Sum256([]byte(identity))
return fmt.Sprintf("admin-overlap:%x", digest[:12])
}
func filterTrustedAdminOverlaps(overlaps map[string][]store.AdminEmailEntry, cfg *config.Config) map[string][]store.AdminEmailEntry {
if cfg == nil || (len(cfg.Detection.AdminOverlapTrustedEmails) == 0 && len(cfg.Detection.AdminOverlapTrustedDomains) == 0) {
return overlaps
}
out := make(map[string][]store.AdminEmailEntry, len(overlaps))
for email, owners := range overlaps {
if trustedAdminOverlapEmail(email, cfg) {
continue
}
out[email] = owners
}
return out
}
func trustedAdminOverlapEmail(email string, cfg *config.Config) bool {
email = strings.ToLower(strings.TrimSpace(email))
if email == "" {
return false
}
for _, trusted := range cfg.Detection.AdminOverlapTrustedEmails {
if email == strings.ToLower(strings.TrimSpace(trusted)) {
return true
}
}
domain := adminEmailDomain(email)
if domain == "" {
return false
}
for _, trusted := range cfg.Detection.AdminOverlapTrustedDomains {
if domain == strings.ToLower(strings.TrimSpace(trusted)) {
return true
}
}
return false
}
func adminEmailDomain(email string) string {
at := strings.LastIndexByte(email, '@')
if at < 0 || at == len(email)-1 {
return ""
}
return email[at+1:]
}
// adminOverlapWPConfigs lists the WordPress installs this check compares.
// Overlap between a primary site and a subdomain install is the shape this
// check exists to catch, so both must be discovered.
func adminOverlapWPConfigs(ctx context.Context) []string {
installs := wpInstalls(ctx, "admin_overlap")
out := make([]string, 0, len(installs))
for _, in := range installs {
out = append(out, in.ConfigPath)
}
return out
}
package checks
import (
"context"
"fmt"
"net"
"regexp"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mysqlclient"
)
// AutoRespondDBMalware processes database injection findings and takes
// automated action: reports the addresses in WordPress session tokens
// without blocking them, revokes compromised user sessions, and cleans
// confirmed malicious content from wp_options or stored database objects.
//
// Only acts on high-confidence findings:
// - db_options_injection with confirmed malicious external script URLs
// - db_siteurl_hijack (siteurl/home pointing to malicious content)
// - db_malicious_trigger/event/procedure/function with structured metadata
//
// Does NOT act on:
// - db_spam_injection (spam posts — needs manual review)
// - db_post_injection (script in posts — too many FPs from page builders)
// - db_options_injection without confirmed malicious URLs
func AutoRespondDBMalware(cfg *config.Config, findings []alert.Finding) []alert.Finding {
return AutoRespondDBMalwareWithPolicy(cfg, findings, nil)
}
// AutoRespondDBMalwareWithPolicy reports session addresses whether or not it
// may edit a database or revoke sessions. A nil policy permits both; callers
// with suppressions supply a per-finding remediation decision.
func AutoRespondDBMalwareWithPolicy(cfg *config.Config, findings []alert.Finding, canRemediate func(alert.Finding) bool) []alert.Finding {
if !cfg.AutoResponse.Enabled || !cfg.AutoResponse.CleanDatabase {
return nil
}
var actions []alert.Finding
for _, f := range findings {
remediate := canRemediate == nil || canRemediate(f)
switch f.Check {
case "db_options_injection":
acts := handleMaliciousOption(cfg, f, remediate)
actions = append(actions, acts...)
case "db_siteurl_hijack":
acts := handleSiteurlHijack(cfg, f, remediate)
actions = append(actions, acts...)
case "db_malicious_trigger", "db_malicious_event",
"db_malicious_procedure", "db_malicious_function":
if !remediate {
continue
}
acts := handleMaliciousDBObject(f)
actions = append(actions, acts...)
}
}
return actions
}
// dbDropObjectFn is the seam through which handleMaliciousDBObject performs
// the backup-then-DROP. Overridden in tests so the routing and action
// emission can be exercised without a live MySQL server.
var dbDropObjectFn = DBDropObject
// handleMaliciousDBObject auto-cleans a confirmed malicious stored database
// object (trigger/event/procedure/function). Detection always fires; the
// DROP only runs when the operator has enabled auto_response.clean_database
// (checked by the caller). The object kind comes from the check name
// (db_malicious_<kind>); account/schema/name come from the finding details.
// DBDropObject records a SHOW CREATE backup in bbolt before dropping, so the
// action is reversible.
func handleMaliciousDBObject(f alert.Finding) []alert.Finding {
kind := maliciousDBObjectKind(f.Check)
if kind == "" {
return nil
}
account, schema, detailKind, name := parseDBObjectFindingDetails(f.Details)
if account == "" || schema == "" || detailKind == "" || name == "" {
return nil
}
if detailKind != kind {
return nil
}
res := dbDropObjectFn(account, schema, kind, name, false)
if !res.Success {
return []alert.Finding{{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-DB-CLEAN failed to drop %s %s.%s: %s", kind, schema, name, res.Message),
Timestamp: time.Now(),
}}
}
return []alert.Finding{{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-DB-CLEAN: Dropped malicious %s %s.%s (backup retained for restore)", kind, schema, name),
Timestamp: time.Now(),
}}
}
func maliciousDBObjectKind(check string) string {
const prefix = "db_malicious_"
if !strings.HasPrefix(check, prefix) {
return ""
}
kind := strings.TrimPrefix(check, prefix)
if !IsDBObjectKind(kind) {
return ""
}
return kind
}
// parseDBObjectFindingDetails extracts the structured header fields a
// db_malicious_<kind> finding carries in its Details block. The SQL body is
// attacker-controlled and may contain lines that look like metadata, so parsing
// stops at Body and keeps the first value for each header key.
func parseDBObjectFindingDetails(details string) (account, schema, kind, name string) {
for _, line := range strings.Split(details, "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "Body:") {
break
}
switch {
case strings.HasPrefix(line, "Account: "):
if account == "" {
account = strings.TrimSpace(strings.TrimPrefix(line, "Account: "))
}
case strings.HasPrefix(line, "Schema: "):
if schema == "" {
schema = strings.TrimSpace(strings.TrimPrefix(line, "Schema: "))
}
case strings.HasPrefix(line, "Kind: "):
if kind == "" {
kind = strings.TrimSpace(strings.TrimPrefix(line, "Kind: "))
}
case strings.HasPrefix(line, "Name: "):
if name == "" {
name = strings.TrimSpace(strings.TrimPrefix(line, "Name: "))
}
}
}
return
}
// handleMaliciousOption checks if a db_options_injection finding contains
// a confirmed malicious external script URL, and if so:
// 1. Reports the addresses of active WP sessions without blocking them
// 2. Revokes sessions for users with non-infra, non-private IPs only
// 3. Backs up and cleans the malicious content from the option
func handleMaliciousOption(cfg *config.Config, f alert.Finding, remediate bool) []alert.Finding {
var actions []alert.Finding
dbName, optionName := parseDBFindingDetails(f.Details)
if dbName == "" || optionName == "" {
return nil
}
// Validate option name — must be a plausible WP option name.
if !isValidOptionName(optionName) {
return nil
}
// Never act on CSM backup options — they preserve original malicious
// content for recovery. Acting on them causes cascading backup loops.
if strings.HasPrefix(optionName, "csm_backup_") {
return nil
}
creds := findCredsForDB(dbName)
if creds.dbName == "" {
return nil
}
prefix := creds.tablePrefix
if prefix == "" {
prefix = "wp_"
}
// Re-read the FULL option value from the database — the finding's
// Details field only has a truncated 200-char preview.
fullValue := readOptionValue(creds, prefix, optionName)
if fullValue == "" {
return nil
}
// Only act on options with confirmed malicious external script URLs.
maliciousURL := extractMaliciousScriptURL(fullValue)
if maliciousURL == "" {
return nil
}
// 1. Report the addresses of active WP sessions. They are not blocked:
// the site owner's sessions are among them.
suspiciousIPs := extractSuspiciousSessionIPs(creds, prefix, cfg.InfraIPs)
if len(suspiciousIPs) > 0 {
actions = append(actions, sessionIPsNotice(suspiciousIPs,
fmt.Sprintf("active WP session on compromised site, DB: %s", dbName), alert.CauseOf(f)))
}
if !remediate {
return actions
}
// 2. Revoke sessions only for users with suspicious IPs.
// This preserves the site admin's session if they're on an infra IP.
revoked := revokeCompromisedSessions(creds, prefix, cfg.InfraIPs)
if revoked > 0 {
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-DB-CLEAN: Revoked %d compromised WordPress sessions (DB: %s)", revoked, dbName),
Timestamp: time.Now(),
})
}
// 3. Back up the original value, then clean the malicious content.
cleaned := backupAndCleanOption(creds, prefix, optionName, fullValue, maliciousURL)
if cleaned {
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-DB-CLEAN: Removed malicious script from wp_options '%s' (DB: %s, URL: %s)", optionName, dbName, maliciousURL),
Timestamp: time.Now(),
})
}
return actions
}
// handleSiteurlHijack handles siteurl/home hijacking by revoking sessions
// and reporting their addresses. Does NOT modify siteurl/home values.
func handleSiteurlHijack(cfg *config.Config, f alert.Finding, remediate bool) []alert.Finding {
var actions []alert.Finding
dbName, _ := parseDBFindingDetails(f.Details)
if dbName == "" {
return nil
}
creds := findCredsForDB(dbName)
if creds.dbName == "" {
return nil
}
prefix := creds.tablePrefix
if prefix == "" {
prefix = "wp_"
}
suspiciousIPs := extractSuspiciousSessionIPs(creds, prefix, cfg.InfraIPs)
if len(suspiciousIPs) > 0 {
actions = append(actions, sessionIPsNotice(suspiciousIPs,
fmt.Sprintf("active session on hijacked site, DB: %s", dbName), alert.CauseOf(f)))
}
if !remediate {
return actions
}
revoked := revokeCompromisedSessions(creds, prefix, cfg.InfraIPs)
if revoked > 0 {
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-DB-CLEAN: Revoked %d sessions on hijacked site (DB: %s)", revoked, dbName),
Timestamp: time.Now(),
})
}
return actions
}
// sessionIPsNotice reports the addresses of WordPress sessions active on a
// compromised site. A session address cannot establish attacker attribution
// and may belong to the site owner. Keep the originating finding identity
// without setting address fields another responder could act on.
func sessionIPsNotice(ips []string, siteContext string, cause alert.Cause) alert.Finding {
return alert.Finding{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-DB-CLEAN: %d addresses had sessions on a compromised site (%s); not blocked", len(ips), siteContext),
Details: "Session addresses: " + strings.Join(ips, ", "),
Timestamp: time.Now(),
Cause: &cause,
}
}
// --- URL analysis ---
// scriptSrcRe matches <script src="..."> or <script src=...> patterns.
// Accepts https://, http://, and protocol-relative // URLs, since real
// attackers use all three forms to load external payloads.
var scriptSrcRe = regexp.MustCompile(`(?i)<script[^>]+src\s*=\s*["']?((?:https?:)?//[^"'\s>]+)`)
// escapedSlashRe matches a forward slash behind one or more backslashes, the
// form every JSON-encoded and re-serialised option value takes.
var escapedSlashRe = regexp.MustCompile(`\\+/`)
// unescapeStoredSlashes restores the slashes of a URL stored inside a JSON or
// serialised option value. WordPress writes json_encode output straight into
// wp_options, so a stored payload reads "https:\/\/host\/payload.js"; without
// this the script matcher sees no "//" after the scheme and extracts nothing.
func unescapeStoredSlashes(value string) string {
if !strings.Contains(value, `\/`) {
return value
}
return escapedSlashRe.ReplaceAllString(value, "/")
}
// knownSafeDomains are legitimate services that embed scripts in wp_options.
var knownSafeDomains = []string{
"googletagmanager.com",
"google-analytics.com",
"googleapis.com",
"gstatic.com",
"google.com",
"facebook.net",
"facebook.com",
"fbcdn.net",
"connect.facebook.net",
"chimpstatic.com",
"mailchimp.com",
"hotjar.com",
"clarity.ms",
"cloudflare.com",
"cdnjs.cloudflare.com",
"jquery.com",
"jsdelivr.net",
"unpkg.com",
"wp.com",
"wordpress.com",
"gravatar.com",
"tawk.to",
"crisp.chat",
"tidio.co",
"intercom.io",
"zendesk.com",
"hubspot.com",
"hubspot.net",
"hs-scripts.com",
"hs-analytics.net",
"hsforms.com",
"mautic.net",
"pinterest.com",
"twitter.com",
"linkedin.com",
"addthis.com",
"sharethis.com",
"recaptcha.net",
"stripe.com",
"paypal.com",
"brevo-mail.com",
}
// extractMaliciousScriptURL finds a <script src="..."> URL in the content
// that is classified as an attacker script by isAttackerScriptURL.
//
// The classification uses an attack-indicator model (see url_reputation.go):
// a URL flags only when it shows attacker-characteristic markers (raw IP
// host, abused TLD, plaintext HTTP, known-bad exfil host, or no valid
// TLD). The previous allowlist-only model produced HIGH-severity findings
// for legitimate third-party widgets (OneTrust, Issuu, regional video
// embeds, regional tax-form widgets) whose domains were not on the
// allowlist; the attack-indicator model eliminates those false positives
// while still catching the injection patterns attackers actually use.
//
// knownSafeDomains is retained as a fast-path optimisation and operator-
// pre-approved list — see isAttackerScriptURL for the composition order.
func extractMaliciousScriptURL(content string) string {
// WordPress stores json_encode output verbatim, so a stored loader reads
// "https:\/\/host\/payload.js" and the src grammar never matches it.
matches := scriptSrcRe.FindAllStringSubmatch(unescapeStoredSlashes(content), -1)
for _, match := range matches {
if len(match) < 2 {
continue
}
url := match[1]
if isAttackerScriptURL(url) {
return url
}
}
return ""
}
// isSafeScriptDomain checks if a script URL is from a known safe domain.
// Handles https://host, http://host, //host (protocol-relative), and
// host-with-port forms.
func isSafeScriptDomain(url string) bool {
urlLower := strings.ToLower(url)
urlLower = strings.TrimPrefix(urlLower, "https://")
urlLower = strings.TrimPrefix(urlLower, "http://")
urlLower = strings.TrimPrefix(urlLower, "//")
host := urlLower
if idx := strings.IndexByte(host, '/'); idx >= 0 {
host = host[:idx]
}
if idx := strings.IndexByte(host, ':'); idx >= 0 {
host = host[:idx]
}
for _, safe := range knownSafeDomains {
if host == safe || strings.HasSuffix(host, "."+safe) {
return true
}
}
return false
}
// --- Validation ---
// validOptionNameRe allows alphanumeric, underscores, hyphens, colons, and dots.
// Rejects anything that could be SQL injection.
var validOptionNameRe = regexp.MustCompile(`^[a-zA-Z0-9_\-:.]+$`)
// isValidOptionName validates that an option name is safe for SQL interpolation.
func isValidOptionName(name string) bool {
return len(name) > 0 && len(name) <= 191 && validOptionNameRe.MatchString(name)
}
// --- DB helpers ---
// parseDBFindingDetails extracts the database name and option name from
// a finding's Details field.
func parseDBFindingDetails(details string) (dbName, optionName string) {
for _, line := range strings.Split(details, "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "Database: ") {
dbName = strings.TrimPrefix(line, "Database: ")
}
if strings.HasPrefix(line, "Option: ") {
optionName = strings.TrimPrefix(line, "Option: ")
}
}
return
}
// findCredsForDB finds wp-config.php credentials that match a database name.
// Skips wp-configs whose $table_prefix fails the safety check -- those
// values come straight from a cPanel-user-writable file and end up in
// root-credentialled SQL via handleMaliciousOption / handleSiteurlHijack.
func findCredsForDB(dbName string) wpDBCreds {
wpConfigs := wpInstallConfigPaths(wpInstalls(context.Background(), "db_content"))
for _, path := range wpConfigs {
creds := parseWPConfig(path)
if creds.dbName != dbName {
continue
}
prefix, ok := resolveTablePrefix(creds)
if !ok {
continue
}
creds.tablePrefix = prefix
return creds
}
return wpDBCreds{}
}
// readOptionValue reads the full value of a wp_option from the database.
//
// mysqlclient returns mysql batch-mode output, where control bytes are rendered
// as escape sequences (a real newline becomes the two bytes "\n"). The value is
// unescaped back to its true bytes before returning so callers that write it
// back (the backup copy and the cleaned value) persist the original content
// rather than the escaped text, keeping PHP-serialized length prefixes valid.
//
// This path intentionally bypasses runMySQLQuery: that legacy scan helper trims
// each returned row before handing it to callers, but wp_options values may
// contain significant leading/trailing whitespace that must survive byte-for-
// byte when CSM writes the backup and cleaned option value.
func readOptionValue(creds wpDBCreds, prefix, optionName string) string {
if !isValidOptionName(optionName) {
return ""
}
query := fmt.Sprintf(
"SELECT option_value FROM %soptions WHERE option_name='%s' LIMIT 1",
prefix, escapeSQLString(optionName))
lines, err := mysqlclient.PerAccountQuery(context.Background(), mysqlclient.Creds{
User: creds.dbUser,
Password: creds.dbPass,
Host: creds.dbHost,
DBName: creds.dbName,
}, query)
if err != nil {
return ""
}
if len(lines) == 0 {
return ""
}
return mysqlclient.BatchUnescape(lines[0])
}
// extractSuspiciousSessionIPs reads WP session tokens and returns IPs that
// are NOT infra IPs, not private, and not loopback.
func extractSuspiciousSessionIPs(creds wpDBCreds, prefix string, infraIPs []string) []string {
query := fmt.Sprintf(
"SELECT meta_value FROM %susermeta WHERE meta_key='session_tokens' AND meta_value != ''",
prefix)
lines := runMySQLQuery(creds, query)
seen := make(map[string]bool)
var ips []string
ipRe := regexp.MustCompile(`"ip";s:\d+:"([^"]+)"`)
for _, line := range lines {
matches := ipRe.FindAllStringSubmatch(line, -1)
for _, m := range matches {
if len(m) < 2 {
continue
}
ip := m[1]
parsed := net.ParseIP(ip)
if parsed == nil || parsed.IsLoopback() || parsed.IsPrivate() {
continue
}
if isInfraIP(ip, infraIPs) {
continue
}
if !seen[ip] {
seen[ip] = true
ips = append(ips, ip)
}
}
}
return ips
}
// revokeCompromisedSessions clears session_tokens only for WP users whose
// sessions contain non-infra, non-private IPs. Returns count of users revoked.
func revokeCompromisedSessions(creds wpDBCreds, prefix string, infraIPs []string) int {
// Get user IDs with active sessions.
query := fmt.Sprintf(
"SELECT user_id, meta_value FROM %susermeta WHERE meta_key='session_tokens' AND meta_value != ''",
prefix)
lines := runMySQLQuery(creds, query)
ipRe := regexp.MustCompile(`"ip";s:\d+:"([^"]+)"`)
revoked := 0
for _, line := range lines {
parts := strings.SplitN(line, "\t", 2)
if len(parts) != 2 {
continue
}
userID := strings.TrimSpace(parts[0])
sessionData := parts[1]
// Check if this user has any suspicious (non-infra, non-private) IPs.
hasSuspicious := false
matches := ipRe.FindAllStringSubmatch(sessionData, -1)
for _, m := range matches {
if len(m) < 2 {
continue
}
ip := m[1]
parsed := net.ParseIP(ip)
if parsed == nil || parsed.IsLoopback() || parsed.IsPrivate() {
continue
}
if isInfraIP(ip, infraIPs) {
continue
}
hasSuspicious = true
break
}
if hasSuspicious {
revokeQuery := fmt.Sprintf(
"UPDATE %susermeta SET meta_value='' WHERE user_id=%s AND meta_key='session_tokens'",
prefix, escapeSQLString(userID))
runMySQLQuery(creds, revokeQuery)
revoked++
}
}
return revoked
}
// backupAndCleanOption saves the original value to a backup option, then
// removes malicious script injections from the option value.
func backupAndCleanOption(creds wpDBCreds, prefix, optionName, originalValue, maliciousURL string) bool {
if maliciousURL == "" {
return false
}
cleaned := removeMaliciousScripts(originalValue)
// Never claim a clean (nor write back) unless the confirmed attacker
// script is actually gone. This keeps removal locked to detection: if
// a script form is flagged but the remover cannot strip it, report the
// finding but never persist a value that still carries a live payload.
// Plain text references to the same URL are inert option data and must
// not block a valid script cleanup.
if optionInjectionRemains(optionName, cleaned) {
return false
}
if cleaned == originalValue {
return false
}
// Save original value as a backup option (csm_backup_<name>_<timestamp>).
backupName := fmt.Sprintf("csm_backup_%s_%d", optionName, time.Now().Unix())
if len(backupName) > 191 {
backupName = backupName[:191]
}
backupQuery := fmt.Sprintf(
"INSERT INTO %soptions (option_name, option_value, autoload) VALUES ('%s', '%s', 'no')",
prefix, escapeSQLString(backupName), escapeSQLString(originalValue))
runMySQLQuery(creds, backupQuery)
// Write the cleaned value.
updateQuery := fmt.Sprintf(
"UPDATE %soptions SET option_value='%s' WHERE option_name='%s'",
prefix, escapeSQLString(cleaned), escapeSQLString(optionName))
runMySQLQuery(creds, updateQuery)
return true
}
// A notice sink makes executable markup malicious regardless of URL
// reputation. Removing one known attacker URL must not permit a partial write
// while an ordinary HTTPS loader or inline script survives in the same row.
func optionInjectionRemains(option, value string) bool {
if extractMaliciousScriptURL(value) != "" {
return true
}
_, sink := pluginNoticeSinkOptions[strings.ToLower(strings.TrimSpace(option))]
return sink && executableMarkupRe.MatchString(unescapeStoredSlashes(value))
}
// --- Script removal ---
// maliciousScriptRe matches the style-break injection pattern:
// </style><script src=...></script><style>
var maliciousScriptRe = regexp.MustCompile(
`(?i)</style>\s*<script[^>]*src\s*=\s*[^>]+>\s*</script>\s*<style>`)
// simpleScriptRe matches standalone <script src="..."></script> tags.
// The src grammar mirrors scriptSrcRe (https://, http://, and
// protocol-relative //) so removal stays paired with detection: a URL
// form the detector flags must be one the remover can strip.
var simpleScriptRe = regexp.MustCompile(
`(?i)<script[^>]*src\s*=\s*["']?(?:https?:)?//[^"'\s>]+["']?[^>]*>\s*</script>`)
// removeMaliciousScripts strips malicious <script> injections from content,
// preserving scripts that are not classified as attacker scripts.
//
// Uses the same isAttackerScriptURL predicate as extractMaliciousScriptURL
// so detection and removal stay semantically paired. If the detector
// would not flag a given URL as malicious, the remover must not strip
// it — otherwise an operator running DBCleanOption on an option that
// contains a real injection alongside a legitimate third-party embed
// (OneTrust, Issuu, regional widget) would silently lose the legitimate
// embed along with the attacker's script.
func removeMaliciousScripts(content string) string {
// First pass: remove style-break patterns only when the embedded URL has
// attacker indicators. The wrapper is suspicious, but the cleaner must
// not remove a legitimate embed just because it appears next to a real
// attacker script in the same option.
content = maliciousScriptRe.ReplaceAllStringFunc(content, func(match string) string {
urls := scriptSrcRe.FindStringSubmatch(match)
if len(urls) >= 2 && isAttackerScriptURL(urls[1]) {
return ""
}
return match
})
// Second pass: remove standalone script tags only when the URL
// shows attacker indicators.
content = simpleScriptRe.ReplaceAllStringFunc(content, func(match string) string {
urls := scriptSrcRe.FindStringSubmatch(match)
if len(urls) >= 2 && isAttackerScriptURL(urls[1]) {
return ""
}
return match
})
return strings.TrimSpace(content)
}
// escapeSQLString escapes special characters for MySQL string interpolation.
func escapeSQLString(s string) string {
s = strings.ReplaceAll(s, `\`, `\\`)
s = strings.ReplaceAll(s, `'`, `\'`)
s = strings.ReplaceAll(s, "\x00", `\0`)
s = strings.ReplaceAll(s, "\n", `\n`)
s = strings.ReplaceAll(s, "\r", `\r`)
s = strings.ReplaceAll(s, "\x1a", `\Z`)
return s
}
package checks
import (
"context"
"fmt"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/mysqlclient"
)
// spamPostIDRe guards the ID list interpolated into root DELETE statements.
var spamPostIDRe = regexp.MustCompile(`^[0-9]+$`)
// DBCleanResult describes the outcome of a database cleanup operation.
type DBCleanResult struct {
Account string
Database string
Action string // "clean-option", "revoke-user", "delete-spam"
Success bool
Message string
Details []string // individual actions taken
BackupNames []string // names of backup options created
}
// DBCleanOption removes malicious script injections from a wp_option value.
// Creates a backup option before modifying. Returns a result describing
// what was done. If preview is true, reports what would be done without
// modifying the database.
func DBCleanOption(account, optionName string, preview bool) DBCleanResult {
result := DBCleanResult{
Account: account,
Action: "clean-option",
}
if !isValidOptionName(optionName) {
result.Message = fmt.Sprintf("Invalid option name: %q", optionName)
return result
}
creds, prefix := findCredsForAccount(account)
if creds.dbName == "" {
result.Message = fmt.Sprintf("No WordPress database found for account %q", account)
return result
}
result.Database = creds.dbName
// Read current value.
value := readOptionValue(creds, prefix, optionName)
if value == "" {
result.Message = fmt.Sprintf("Option %q not found or empty in %s", optionName, creds.dbName)
return result
}
// Check for malicious content.
maliciousURL := extractMaliciousScriptURL(value)
if maliciousURL == "" {
result.Message = fmt.Sprintf("No malicious external script found in %q", optionName)
result.Details = append(result.Details, "Option exists but contains no confirmed malicious URLs")
return result
}
cleaned := removeMaliciousScripts(value)
if cleaned == value {
result.Message = "Content unchanged after cleaning"
return result
}
if optionInjectionRemains(optionName, cleaned) {
result.Message = "Failed to remove all malicious scripts"
return result
}
result.Details = append(result.Details, fmt.Sprintf("Malicious URL: %s", maliciousURL))
result.Details = append(result.Details, fmt.Sprintf("Original length: %d, Cleaned length: %d", len(value), len(cleaned)))
if preview {
result.Message = fmt.Sprintf("PREVIEW: Would clean malicious script from %q", optionName)
result.Success = true
return result
}
// Backup and clean.
if backupAndCleanOption(creds, prefix, optionName, value, maliciousURL) {
backupName := fmt.Sprintf("csm_backup_%s_%d", optionName, time.Now().Unix())
if len(backupName) > 191 {
backupName = backupName[:191]
}
result.BackupNames = append(result.BackupNames, backupName)
result.Details = append(result.Details, fmt.Sprintf("Backup saved as: %s", backupName))
result.Message = fmt.Sprintf("Cleaned malicious script from %q", optionName)
result.Success = true
} else {
result.Message = "Failed to clean option"
}
return result
}
// DBRevokeUser revokes WordPress sessions for a specific user and optionally
// demotes them to subscriber role. If preview is true, reports what would be
// done without modifying the database.
func DBRevokeUser(account string, userID int, demote, preview bool) DBCleanResult {
result := DBCleanResult{
Account: account,
Action: "revoke-user",
}
creds, prefix := findCredsForAccount(account)
if creds.dbName == "" {
result.Message = fmt.Sprintf("No WordPress database found for account %q", account)
return result
}
result.Database = creds.dbName
// Verify user exists.
query := fmt.Sprintf(
"SELECT user_login, user_email FROM %susers WHERE ID=%d LIMIT 1",
prefix, userID)
lines := runMySQLQueryRoot(creds.dbName, query)
if len(lines) == 0 {
result.Message = fmt.Sprintf("User ID %d not found in %s", userID, creds.dbName)
return result
}
parts := strings.SplitN(lines[0], "\t", 2)
login := parts[0]
email := ""
if len(parts) > 1 {
email = parts[1]
}
result.Details = append(result.Details, fmt.Sprintf("User: %s (email: %s)", login, email))
// Check current sessions.
sessQuery := fmt.Sprintf(
"SELECT LEFT(meta_value, 200) FROM %susermeta WHERE user_id=%d AND meta_key='session_tokens'",
prefix, userID)
sessLines := runMySQLQueryRoot(creds.dbName, sessQuery)
sessionCount := 0
if len(sessLines) > 0 && sessLines[0] != "" {
sessionCount = strings.Count(sessLines[0], `"expiration"`)
}
result.Details = append(result.Details, fmt.Sprintf("Active sessions: %d", sessionCount))
if preview {
msg := fmt.Sprintf("PREVIEW: Would revoke %d sessions for user %s (ID %d)", sessionCount, login, userID)
if demote {
msg += " and demote to subscriber"
}
result.Message = msg
result.Success = true
return result
}
// Revoke sessions.
revokeQuery := fmt.Sprintf(
"UPDATE %susermeta SET meta_value='' WHERE user_id=%d AND meta_key='session_tokens'",
prefix, userID)
runMySQLQueryRoot(creds.dbName, revokeQuery)
result.Details = append(result.Details, "Sessions revoked")
// Demote to subscriber.
if demote {
// Read current capabilities to find the meta_key (varies by prefix).
capQuery := fmt.Sprintf(
"SELECT meta_key FROM %susermeta WHERE user_id=%d AND meta_key LIKE '%%capabilities'",
prefix, userID)
capLines := runMySQLQueryRoot(creds.dbName, capQuery)
if len(capLines) > 0 {
capKey := capLines[0]
demoteQuery := fmt.Sprintf(
"UPDATE %susermeta SET meta_value='a:1:{s:10:\"subscriber\";b:1;}' WHERE user_id=%d AND meta_key='%s'",
prefix, userID, escapeSQLString(capKey))
runMySQLQueryRoot(creds.dbName, demoteQuery)
result.Details = append(result.Details, "Demoted to subscriber role")
}
}
result.Message = fmt.Sprintf("Revoked sessions for user %s (ID %d)", login, userID)
result.Success = true
return result
}
// DBDeleteSpam deletes published posts matching spam patterns from a WordPress
// database. Only deletes posts of type 'post' with status 'publish' to avoid
// touching pages, attachments, or plugin data. If preview is true, reports
// counts without deleting.
//
// Two restrictions keep a keyword match from destroying real content: the
// SQL LIKE result is re-tested Go-side on a word boundary, and only the
// patterns marked deletable are considered. Injections that live inside an
// otherwise legitimate page are out of scope here -- deleting the page
// would destroy the customer's own content, so those need the injected
// markup stripped instead.
func DBDeleteSpam(account string, preview bool) DBCleanResult {
result := DBCleanResult{
Account: account,
Action: "delete-spam",
}
creds, prefix := findCredsForAccount(account)
if creds.dbName == "" {
result.Message = fmt.Sprintf("No WordPress database found for account %q", account)
return result
}
result.Database = creds.dbName
// SQL LIKE narrows candidates; the Go word-boundary check decides. LIKE
// alone matches "cialis" inside "specialist" and "pharma" inside a
// conference name, so a substring-only hit must never reach a DELETE.
// dbSpamPatterns is shared with the scanner so detection and deletion
// can never drift apart again.
spamIDs := make(map[string]struct{})
var order []string
var keywordCounts []string
for _, sp := range dbSpamPatterns {
if !sp.deletable {
continue
}
query := "SELECT ID, CONCAT_WS(' ', post_title, post_content) FROM " + prefix +
"posts WHERE post_type='post' AND post_status='publish' AND (post_content LIKE '" +
sp.likeFragment + "' OR post_title LIKE '" + sp.likeFragment + "')"
matched := 0
rows, err := runMySQLQueryRootWithError(creds.dbName, query)
if err != nil {
result.Message = fmt.Sprintf("Failed to query spam posts for %q: %v", sp.keyword, err)
return result
}
for _, line := range rows {
id, text, ok := splitSpamCandidate(line)
if !ok || len(spamKeywordMatchIndexes(sp, text)) == 0 {
continue
}
matched++
if _, seen := spamIDs[id]; !seen {
spamIDs[id] = struct{}{}
order = append(order, id)
}
}
if matched > 0 {
keywordCounts = append(keywordCounts,
fmt.Sprintf("%s=%s", sp.keyword, formatSpamPostCount(matched)))
}
}
if len(order) == 0 {
result.Message = "No spam posts found"
result.Success = true
return result
}
result.Details = append(result.Details,
"Matched "+formatCountedNoun(len(order), "unique post", "unique posts"),
"Keyword match counts overlap when a post contains several keywords: "+
strings.Join(keywordCounts, ", "))
if preview {
result.Message = "PREVIEW: Would delete " + formatSpamPostCount(len(order))
result.Success = true
return result
}
// Delete spam posts (and their revisions/meta) in batches.
deleted := 0
for i := 0; i < len(order); i += 100 {
end := i + 100
if end > len(order) {
end = len(order)
}
idList := strings.Join(order[i:end], ",")
affected, err := deleteSpamBatch(creds.dbName, prefix, idList)
if err != nil {
result.Message = fmt.Sprintf("Spam cleanup failed after deleting %s: %v",
formatSpamPostCount(deleted), err)
result.Details = append(result.Details,
"Cleanup stopped at the first failed database statement; rerun it after fixing the database error.")
return result
}
deleted += affected
}
result.Message = fmt.Sprintf("Deleted %s and %s metadata",
formatSpamPostCount(deleted), spamPostPossessive(deleted))
if notDeleted := len(order) - deleted; notDeleted > 0 {
verb := "were"
if notDeleted == 1 {
verb = "was"
}
result.Message += fmt.Sprintf("; %s %s not deleted",
formatCountedNoun(notDeleted, "candidate", "candidates"), verb)
}
result.Success = true
return result
}
func deleteSpamBatch(dbName, prefix, idList string) (int, error) {
statements := []struct {
action string
query string
}{
{
action: "delete post metadata",
query: fmt.Sprintf("DELETE FROM %spostmeta WHERE post_id IN (%s)", prefix, idList),
},
{
action: "delete post revisions",
query: fmt.Sprintf(
"DELETE FROM %sposts WHERE post_parent IN (%s) AND post_type='revision'",
prefix, idList),
},
}
for _, statement := range statements {
if _, err := runMySQLExecRootAffected(dbName, statement.query); err != nil {
return 0, fmt.Errorf("%s: %w", statement.action, err)
}
}
affected, err := runMySQLExecRootAffected(dbName, fmt.Sprintf(
"DELETE FROM %sposts WHERE ID IN (%s) AND post_type='post' AND post_status='publish'",
prefix, idList))
if err != nil {
return 0, fmt.Errorf("delete posts: %w", err)
}
return int(affected), nil
}
func formatSpamPostCount(count int) string {
return formatCountedNoun(count, "spam post", "spam posts")
}
func formatCountedNoun(count int, singular, plural string) string {
noun := plural
if count == 1 {
noun = singular
}
return fmt.Sprintf("%d %s", count, noun)
}
func spamPostPossessive(count int) string {
if count == 1 {
return "its"
}
return "their"
}
// FormatDBCleanResult formats a DBCleanResult for terminal output.
func FormatDBCleanResult(r DBCleanResult) string {
var sb strings.Builder
status := "FAILED"
if r.Success {
status = "OK"
}
fmt.Fprintf(&sb, "[%s] %s — %s\n", status, r.Action, r.Message)
if r.Database != "" {
fmt.Fprintf(&sb, " Database: %s\n", r.Database)
}
for _, d := range r.Details {
fmt.Fprintf(&sb, " %s\n", d)
}
return sb.String()
}
// --- helpers ---
// splitSpamCandidate splits an "ID\ttext" row from the spam candidate
// query. mysqlclient keeps each database row in one string and batch-escapes
// embedded controls; the bounded split also keeps literal tabs in test or
// alternate input from becoming extra columns. A nonnumeric ID is dropped
// rather than interpolated into a DELETE ... IN (...) list.
func splitSpamCandidate(line string) (id, text string, ok bool) {
parts := strings.SplitN(line, "\t", 2)
if len(parts) != 2 {
return "", "", false
}
id = strings.TrimSpace(parts[0])
if !spamPostIDRe.MatchString(id) {
return "", "", false
}
return id, mysqlclient.BatchUnescape(parts[1]), true
}
// findCredsForAccount finds WP database credentials for a cPanel account.
// Returns root-authenticated credentials that use /root/.my.cnf instead of
// wp-config.php passwords (which are often stale on cPanel servers).
func findCredsForAccount(account string) (wpDBCreds, string) {
patterns := wpInstallConfigPaths(wpInstallsForAccount(context.Background(), "db_content", account))
if len(patterns) == 0 {
return wpDBCreds{}, ""
}
sort.Strings(patterns)
primary := filepath.Join(accountHomeDir(account), "public_html", "wp-config.php")
if canonical, err := canonicalWPInstallPath(primary); err == nil {
primary = canonical
}
for i, path := range patterns {
if path == primary {
patterns[0], patterns[i] = patterns[i], patterns[0]
break
}
}
for _, path := range patterns {
creds := parseWPConfig(path)
if creds.dbName == "" {
continue
}
prefix, ok := resolveTablePrefix(creds)
if !ok {
continue
}
creds.tablePrefix = prefix
// Use root auth — CSM runs as root with /root/.my.cnf.
// wp-config.php passwords are unreliable (cPanel password
// rotations don't always update the file).
creds.dbUser = ""
creds.dbPass = ""
creds.dbHost = "localhost"
return creds, prefix
}
return wpDBCreds{}, ""
}
// resolveTablePrefix returns the safe `wp_options`-style prefix for the
// parsed wp-config. Empty input defaults to "wp_". Anything outside
// [A-Za-z0-9_]+ returns ("", false) so callers refuse to interpolate
// attacker-controlled $table_prefix into root-credentialled MySQL.
func resolveTablePrefix(creds wpDBCreds) (string, bool) {
prefix := creds.tablePrefix
if prefix == "" {
prefix = "wp_"
}
if !validTablePrefix.MatchString(prefix) {
return "", false
}
return prefix, true
}
// runMySQLExecRoot runs a non-SELECT MySQL statement under root
// credentials and returns the underlying exec error verbatim. Used
// by the persistence-mechanism cleaner where success is signalled by
// a zero exit + empty stdout, which runMySQLQueryRoot misclassifies
// as "no output, treat as failure".
func runMySQLExecRoot(dbName, stmt string) error {
_, err := runMySQLExecRootAffected(dbName, stmt)
return err
}
func runMySQLExecRootAffected(dbName, stmt string) (int64, error) {
return mysqlclient.RootExecSchema(context.Background(), dbName, stmt)
}
// runMySQLQueryRoot runs a MySQL query using root credentials from
// /root/.my.cnf (no explicit user/password args).
func runMySQLQueryRoot(dbName, query string) []string {
rows, _ := runMySQLQueryRootWithError(dbName, query)
return rows
}
func runMySQLQueryRootWithError(dbName, query string) ([]string, error) {
rows, err := mysqlclient.RootQuerySchema(context.Background(), dbName, query)
if err != nil {
return nil, err
}
if len(rows) == 0 {
return nil, nil
}
var lines []string
for _, line := range rows {
line = strings.TrimSpace(line)
if line != "" {
lines = append(lines, line)
}
}
return lines, nil
}
package checks
import (
"context"
"fmt"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// DB persistence-mechanism scanner.
//
// Vanilla CMS installs (WordPress, Joomla, Drupal, Magento, OpenCart)
// ship zero triggers / events / stored procedures / stored functions.
// Any presence is operator-review territory at minimum, and a body
// matching known-malware patterns is critical -- attacker persistence
// often re-injects on the next INSERT after a file-level cleanup, so
// detection here closes a real gap.
//
// The scanner reuses the existing parseWPConfig + runMySQLQuery
// infrastructure from dbscan.go. When the multi-CMS adapter layer
// lands later, this file's helpers become reusable across CMSes by
// swapping the credential discovery, not the queries.
//
// Detection only, no auto-drop. Operators drop manually
// via `csm db-clean drop-object`.
// dbPersistenceMalwarePatterns supplements dbMalwarePatterns from
// dbscan.go with MySQL-specific persistence-attack signals. Lowercase
// for case-insensitive matching against SQL bodies.
var dbPersistenceMalwarePatterns = []string{
"sys_exec",
"lib_mysqludf_sys",
"into outfile",
"into dumpfile",
"load_file(",
"load data infile",
}
// magicTokenRegex extracts high-entropy activation tokens from trigger
// bodies that gate privileged actions on `display_name LIKE
// '%<token>%'`. Common display-name filters such as "%administrator%"
// must not escalate a merely unexpected trigger into Critical.
var magicTokenRegex = regexp.MustCompile(`(?i)display_name\s+like\s+['"]%([A-Za-z0-9_-]{10,32})%['"]`)
// validTablePrefix matches the character class WordPress accepts for
// $table_prefix (alphanumerics and underscore). Untrusted prefixes
// from a malformed wp-config.php fail this check, which keeps
// scanMagicTokenUsers from concatenating attacker-controlled data into
// its SQL literal.
var validTablePrefix = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
// dbPersistenceMalwareRegexes catches multi-token shapes that no
// substring set can match cleanly: role-escalation writes and
// password-hash exfiltration reads. Pre-compiled at package init time
// -- a regex parse error here is a build-time bug, not a runtime one.
//
// Patterns intentionally case-insensitive ((?i) prefix) and tolerant of
// whitespace / line breaks across MySQL trigger bodies. The role-write
// pattern requires the literal string "administrator" inside the
// serialized capabilities payload -- promotion to subscriber/customer
// is the legitimate WP-signup shape and must not match.
var dbPersistenceMalwareRegexes = []*regexp.Regexp{
// Role escalation: UPDATE on *_usermeta writing administrator caps.
// The (?s) flag lets `.` match newlines so multi-line trigger
// bodies with the UPDATE split across lines still hit.
regexp.MustCompile(`(?is)update\s+` + "`?" + `\w*usermeta` + "`?" + `\s+set\s+meta_value\s*=.*?(?:s:13:["\x60]administrator["\x60]|["\x60]administrator["\x60])`),
// Password-hash exfil read: SELECT user_pass FROM <users-like>
// table. Real WP code goes through wp_check_password() in PHP, never
// raw SELECT user_pass from SQL.
regexp.MustCompile(`(?i)select\s+user_pass\s+from\s+` + "`?" + `\w*users`),
}
// dbObjectKind names the four MySQL object types this scanner
// inspects. Used in finding categories and CLI subcommands.
type dbObjectKind string
const (
dbObjectTrigger dbObjectKind = "trigger"
dbObjectEvent dbObjectKind = "event"
dbObjectProcedure dbObjectKind = "procedure"
dbObjectFunction dbObjectKind = "function"
)
// dbObjectAllKinds lists all valid kinds for the CLI's type validator.
var dbObjectAllKinds = []dbObjectKind{
dbObjectTrigger, dbObjectEvent, dbObjectProcedure, dbObjectFunction,
}
// dbObjectFinding describes one detector hit before it is converted
// to an alert.Finding -- carrying the structured fields the CLI
// drop-object subcommand needs to look up the same row.
type dbObjectFinding struct {
Account string
Schema string
Kind dbObjectKind
Name string
Body string
IsMalw bool // true: malware pattern hit; false: unexpected presence
}
// CheckDatabaseObjects scans every WordPress installation's database
// for triggers, events, procedures, and functions. Critical findings
// fire when the body matches a known-malware pattern; Warning
// findings fire when an object exists at all (vanilla CMSes ship
// none). The Detection.DBObjectScanning kill-switch silences both
// emit paths without disabling the manual drop-object CLI.
func CheckDatabaseObjects(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if !dbObjectScanningEnabled(cfg) {
return nil
}
if ctx == nil {
ctx = context.Background()
}
var findings []alert.Finding
wpConfigs := dbObjectWPConfigs(ctx)
if len(wpConfigs) == 0 {
return nil
}
allowlist := dbObjectAllowlistMap(cfg)
// Rank by mtime desc so recently touched WP installs are processed
// first when the check timeout cuts iteration short.
for _, wpConfig := range rankPathsByMtimeDesc(ctx, wpConfigs, accountScanMaxFiles(ctx, cfg)) {
if ctx.Err() != nil {
return findings
}
account := wpConfigUser(filepath.Dir(wpConfig))
creds, complete := parseWPConfigChecked(wpConfig)
if !complete {
markCheckIncomplete(ctx, "db_objects")
continue
}
if creds.dbName == "" || creds.dbUser == "" {
markCheckIncomplete(ctx, "db_objects")
continue
}
var installFindings []alert.Finding
hits, err := dbObjectScanner(account, creds)
if err != nil {
markCheckIncomplete(ctx, "db_objects")
}
for _, h := range hits {
if !h.IsMalw && allowlist[allowlistKey(h)] {
continue
}
installFindings = append(installFindings, h.toFinding())
}
// Retro-scan: when a trigger gates a privileged action on a
// secret token in display_name, find users whose display_name
// still carries the token. Zero matches is itself useful for
// the incident report ("no evidence the backdoor fired").
for _, h := range hits {
if h.Kind != dbObjectTrigger {
continue
}
tokens := magicTokensOf(h.Body)
if len(tokens) == 0 {
continue
}
tokenFindings, err := magicTokenScanner(account, creds.dbName, creds.tablePrefix, tokens)
if err != nil {
markCheckIncomplete(ctx, "db_objects")
}
installFindings = append(installFindings, tokenFindings...)
}
// The display label may be a lookup sentinel; only a resolved
// account root owner is stamped, per install, before merging.
owner, ok := installOwner(wpConfig)
if ok {
installFindings = stampTenantIDIfEmpty(installFindings, owner)
}
findings = append(findings, installFindings...)
}
return findings
}
// Per-install scan boundaries. Tests replace them with inert scanners to
// prove ownership stamping for every finding name the check owns.
var (
dbObjectScanner = scanDBObjects
magicTokenScanner = scanMagicTokenUsers
magicTokensOf = extractMagicTokens
)
// scanDBObjects runs the three INFORMATION_SCHEMA queries and
// classifies every row. Pure function over the cmdExec injector --
// tests provide canned MySQL CLI output and assert on the structured
// findings without touching a real database.
//
// Connections use root credentials via /root/.my.cnf. WP-config
// passwords are unreliable on cPanel hosts (password rotations
// don't update the file), so a WP-creds path here would silently
// miss persistence objects on the very platform we care most about.
// The existing db-clean code (db_clean.go: findCredsForAccount)
// hits the same constraint and reaches the same conclusion.
func scanDBObjects(account string, creds wpDBCreds) ([]dbObjectFinding, error) {
if creds.dbName == "" {
return nil, nil
}
schema := creds.dbName
schemaLit := mysqlSchemaLiteral(schema)
var hits []dbObjectFinding
// TRIGGERS
rows, err := runMySQLQueryRootWithError(schema, fmt.Sprintf(
`SELECT TRIGGER_NAME, ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE TRIGGER_SCHEMA = %s`,
schemaLit))
if err != nil {
return hits, err
}
for _, row := range rows {
name, body := splitTabRow(row)
if name == "" {
continue
}
hits = append(hits, classifyDBObject(account, schema, dbObjectTrigger, name, body))
}
// EVENTS
rows, err = runMySQLQueryRootWithError(schema, fmt.Sprintf(
`SELECT EVENT_NAME, EVENT_DEFINITION FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_SCHEMA = %s`,
schemaLit))
if err != nil {
return hits, err
}
for _, row := range rows {
name, body := splitTabRow(row)
if name == "" {
continue
}
hits = append(hits, classifyDBObject(account, schema, dbObjectEvent, name, body))
}
// ROUTINES (procedures + functions)
rows, err = runMySQLQueryRootWithError(schema, fmt.Sprintf(
`SELECT ROUTINE_NAME, ROUTINE_TYPE, ROUTINE_DEFINITION FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA = %s`,
schemaLit))
if err != nil {
return hits, err
}
for _, row := range rows {
name, rtype, body := splitTabRow3(row)
if name == "" {
continue
}
kind := dbObjectProcedure
if strings.EqualFold(rtype, "FUNCTION") {
kind = dbObjectFunction
}
hits = append(hits, classifyDBObject(account, schema, kind, name, body))
}
return hits, nil
}
// classifyDBObject decides whether a row matches the malware
// patterns (Critical) or merely exists (Warning).
func classifyDBObject(account, schema string, kind dbObjectKind, name, body string) dbObjectFinding {
return dbObjectFinding{
Account: account,
Schema: schema,
Kind: kind,
Name: name,
Body: body,
IsMalw: bodyHasMalwarePattern(body),
}
}
// bodyHasMalwarePattern returns true when the SQL body matches any of
// the three classifier tiers:
//
// 1. dbMalwarePatterns / dbPersistenceMalwarePatterns: substring tokens
// for OS-exec UDFs and file-IO sinks (sys_exec, INTO OUTFILE, etc.).
// 2. extractMagicTokens: high-entropy display_name activation gates.
// 3. dbPersistenceMalwareRegexes: multi-token shapes for role-escalation
// writes and password-hash exfiltration reads.
//
// Substring matching stays case-insensitive via ToLower; the regex tier
// keeps its own `(?i)` flags so its semantics travel with the pattern.
func bodyHasMalwarePattern(body string) bool {
body = normalizeDBPatternBody(body)
lower := strings.ToLower(body)
for _, p := range dbMalwarePatterns {
if strings.Contains(lower, strings.ToLower(p.pattern)) {
return true
}
}
for _, p := range dbPersistenceMalwarePatterns {
if strings.Contains(lower, p) {
return true
}
}
if len(extractMagicTokens(body)) > 0 {
return true
}
for _, re := range dbPersistenceMalwareRegexes {
if re.MatchString(body) {
return true
}
}
return false
}
func normalizeDBPatternBody(body string) string {
repl := strings.NewReplacer(`\n`, "\n", `\r`, "\r", `\t`, "\t")
return repl.Replace(body)
}
// toFinding renders the structured hit into the alert.Finding shape
// the rest of the pipeline expects. Finding category encodes both
// kind and severity tier so operators can suppress per attack type.
func (h dbObjectFinding) toFinding() alert.Finding {
check := fmt.Sprintf("db_unexpected_%s", h.Kind)
severity := alert.Warning
intro := "Unexpected"
if h.IsMalw {
check = fmt.Sprintf("db_malicious_%s", h.Kind)
severity = alert.Critical
intro = "Malicious"
}
excerpt := h.Body
if len(excerpt) > 240 {
excerpt = excerpt[:240] + "..."
}
return alert.Finding{
Severity: severity,
Check: check,
Message: fmt.Sprintf("%s %s %s in %s.%s", intro, h.Kind, h.Name, h.Account, h.Schema),
Details: fmt.Sprintf("Account: %s\nSchema: %s\nKind: %s\nName: %s\nBody: %s", h.Account, h.Schema, h.Kind, h.Name, excerpt),
Timestamp: time.Now(),
}
}
// allowlistKey shapes the suppression key:
// `<account>:<schema>:<type>:<name>`. Used for the Warning tier
// only -- Critical malware-pattern hits always fire.
func allowlistKey(h dbObjectFinding) string {
return fmt.Sprintf("%s:%s:%s:%s", h.Account, h.Schema, h.Kind, h.Name)
}
func dbObjectAllowlistMap(cfg *config.Config) map[string]bool {
out := map[string]bool{}
if cfg == nil {
return out
}
for _, e := range cfg.Detection.DBObjectAllowlist {
out[strings.TrimSpace(e)] = true
}
return out
}
// splitTabRow returns the first two tab-separated fields from a
// MySQL `-B -N` row. Empty strings if the row has fewer than two
// fields.
func splitTabRow(row string) (string, string) {
parts := strings.SplitN(row, "\t", 2)
if len(parts) < 2 {
return "", ""
}
return parts[0], parts[1]
}
// splitTabRow3 returns the first three tab-separated fields. Used
// for ROUTINES which carries (name, type, body).
func splitTabRow3(row string) (string, string, string) {
parts := strings.SplitN(row, "\t", 3)
if len(parts) < 3 {
return "", "", ""
}
return parts[0], parts[1], parts[2]
}
// mysqlSchemaLiteral wraps the schema name as a single-quoted
// string literal with backslash escaping. The DB name comes from
// wp-config.php (operator-controlled) so the risk is low, but the
// string-literal route is consistent with how the existing dbscan
// queries handle string args.
func mysqlSchemaLiteral(name string) string {
escaped := strings.ReplaceAll(name, `\`, `\\`)
escaped = strings.ReplaceAll(escaped, `'`, `\'`)
return "'" + escaped + "'"
}
// dbObjectScanningEnabled resolves the tri-state cfg flag: nil and
// missing-config both mean default-on; an explicit *false means off.
func dbObjectScanningEnabled(cfg *config.Config) bool {
if cfg == nil {
return true
}
if cfg.Detection.DBObjectScanning == nil {
return true
}
return *cfg.Detection.DBObjectScanning
}
// IsDBObjectKind reports whether s is one of the four valid kinds.
// Used by the CLI subcommand to validate user input before opening
// a connection.
func IsDBObjectKind(s string) bool {
for _, k := range dbObjectAllKinds {
if string(k) == s {
return true
}
}
return false
}
// extractMagicTokens returns the secret activation tokens referenced in
// a trigger body's `display_name LIKE '%<token>%'` clauses. The body
// classifier uses the same helper, and the user retro scan reuses the
// returned tokens to search the *_users table for matches. Tokens are
// deduplicated to keep query count bounded when a trigger references
// the same token across multiple branches.
//
// Returns nil for benign bodies so callers can skip MySQL entirely.
func extractMagicTokens(body string) []string {
body = normalizeDBPatternBody(body)
matches := magicTokenRegex.FindAllStringSubmatch(body, -1)
if len(matches) == 0 {
return nil
}
seen := make(map[string]struct{}, len(matches))
var out []string
for _, m := range matches {
if len(m) < 2 {
continue
}
tok := m[1]
if !validMagicToken(tok) {
continue
}
if _, ok := seen[tok]; ok {
continue
}
seen[tok] = struct{}{}
out = append(out, tok)
}
return out
}
func validMagicToken(tok string) bool {
if len(tok) < 10 || len(tok) > 32 {
return false
}
hasUpper, hasLower, hasDigit := false, false, false
for _, r := range tok {
switch {
case r >= 'A' && r <= 'Z':
hasUpper = true
case r >= 'a' && r <= 'z':
hasLower = true
case r >= '0' && r <= '9':
hasDigit = true
case r == '_' || r == '-':
default:
return false
}
}
return hasUpper && hasLower && hasDigit
}
// scanMagicTokenUsers searches the WordPress users table for accounts
// whose display_name carries a backdoor activation token. A match is
// forensic evidence that the trigger fired against that user -- they
// may still be administrator, or the attacker may have demoted them
// after promotion. Either way the user requires manual review and is
// surfaced as Critical.
//
// The function is conservative about query construction. Tokens are
// guaranteed to be high-entropy [A-Za-z0-9_-]{10,32} strings by
// extractMagicTokens, and the table prefix is validated against
// [A-Za-z0-9_]+ before concatenation. Anything outside those character
// classes causes the scan to skip the query entirely rather than emit a
// half-built SQL statement against an untrusted prefix.
func scanMagicTokenUsers(account, schema, tablePrefix string, tokens []string) ([]alert.Finding, error) {
if len(tokens) == 0 || tablePrefix == "" || !validTablePrefix.MatchString(tablePrefix) {
return nil, nil
}
var findings []alert.Finding
for _, tok := range tokens {
if !validMagicToken(tok) {
continue
}
query := fmt.Sprintf(
"SELECT ID, user_login, user_email, display_name FROM `%susers` WHERE display_name LIKE '%%%s%%'",
tablePrefix, tok,
)
rows, err := runMySQLQueryRootWithError(schema, query)
if err != nil {
return findings, err
}
for _, row := range rows {
parts := strings.SplitN(row, "\t", 4)
if len(parts) < 4 {
continue
}
userID, userLogin, userEmail, displayName := parts[0], parts[1], parts[2], parts[3]
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "db_magic_token_user",
Message: fmt.Sprintf("User %s (ID %s) carries backdoor activation token in %s.%susers", userLogin, userID, account, tablePrefix),
Details: fmt.Sprintf("Account: %s\nSchema: %s\nTable prefix: %s\nToken: %s\nUser ID: %s\nUser login: %s\nUser email: %s\nDisplay name: %s", account, schema, tablePrefix, tok, userID, userLogin, userEmail, displayName),
Timestamp: time.Now(),
})
}
}
return findings, nil
}
// dbObjectWPConfigs lists the WordPress installs this check scans. Discovery is
// shared (wpinstalls.go): a subdomain or nested install carries the same
// injected objects as a primary one.
func dbObjectWPConfigs(ctx context.Context) []string {
installs := wpInstalls(ctx, "db_objects")
out := make([]string, 0, len(installs))
for _, in := range installs {
out = append(out, in.ConfigPath)
}
return out
}
package checks
import (
"context"
"errors"
"fmt"
"regexp"
"strings"
"time"
"github.com/pidginhost/csm/internal/mysqlclient"
"github.com/pidginhost/csm/internal/store"
)
// showCreateStatement extracts the CREATE statement from one batch-mode SHOW
// CREATE row. The client returns a row as tab-joined columns with newlines
// escaped, so the statement is one column, not the row: TRIGGER, PROCEDURE and
// FUNCTION put it third after the name and sql_mode, EVENT inserts time_zone
// before it. Replaying the whole row would fail with a syntax error every
// time, after the object is already gone, so anything else is rejected.
func showCreateStatement(kind, row string) (string, error) {
cols := strings.Split(row, "\t")
idx := 2
if strings.EqualFold(kind, "event") {
idx = 3
}
if len(cols) <= idx {
return "", fmt.Errorf("expected at least %d columns, got %d", idx+1, len(cols))
}
stmt := mysqlclient.BatchUnescape(cols[idx])
fields := strings.Fields(stmt)
if len(fields) < 2 || !strings.EqualFold(fields[0], "CREATE") {
return "", errors.New("statement column does not hold a CREATE statement")
}
objectIdx := 1
if len(fields) > objectIdx+1 && strings.EqualFold(fields[objectIdx], "OR") && strings.EqualFold(fields[objectIdx+1], "REPLACE") {
objectIdx += 2
}
if len(fields) > objectIdx && strings.HasPrefix(strings.ToUpper(fields[objectIdx]), "DEFINER=") {
objectIdx++
}
if len(fields) <= objectIdx || !strings.EqualFold(fields[objectIdx], kind) {
return "", fmt.Errorf("CREATE statement is not for a %s", kind)
}
return stmt, nil
}
// reAccountName matches the cPanel-username shape we accept from
// operator CLI input. Constrained on purpose: anything outside this
// charset would either fail later validation (QuoteIdent on schema)
// or escape /home via the path interpolation in findAccountSchemas
// when an unwary glob expanded a `*`.
var reAccountName = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_-]{0,31}$`)
// errInvalidAccountName flags an account string that fails the
// allowed-charset check. Surfaced through the CLI so the operator
// sees a clear error before any filesystem or SQL lookup.
var errInvalidAccountName = errors.New("invalid account name (want [a-zA-Z][a-zA-Z0-9_-]{0,31})")
// DBDropObject drops a single trigger / event / stored procedure /
// stored function from the operator-supplied account+schema, after:
//
// 1. Validating the kind ("trigger" | "event" | "procedure" | "function").
// 2. Validating that <schema> is one of the databases this account
// hosts. The account is taken from /home/<account>/* wp-config.php
// files; an attacker who can pass an arbitrary <schema> here gets
// no further than DROP'ping their own database.
// 3. QuoteIdent on both <schema> and <name>, so identifier strings
// never participate in SQL string concatenation.
// 4. SHOW CREATE the object and persist the result to the
// db_object_backups bbolt bucket as the backup -- replaying the
// CREATE SQL restores the object byte-for-byte.
// 5. DROP the object.
//
// preview=true short-circuits before step 4: the function reports
// what it would do (kind, schema, name, captured CREATE SQL) without
// touching the database.
//
// Detection is enabled by default; dropping objects is operator-driven.
func DBDropObject(account, schema, kind, name string, preview bool) DBCleanResult {
result := DBCleanResult{
Account: account,
Action: "drop-object",
}
if !reAccountName.MatchString(account) {
result.Message = fmt.Sprintf("%v: %q", errInvalidAccountName, account)
return result
}
if !IsDBObjectKind(kind) {
result.Message = fmt.Sprintf("Invalid object kind %q (want trigger|event|procedure|function)", kind)
return result
}
quotedSchema, err := QuoteIdent(schema)
if err != nil {
result.Message = fmt.Sprintf("Invalid schema name: %v", err)
return result
}
quotedName, err := QuoteIdent(name)
if err != nil {
result.Message = fmt.Sprintf("Invalid object name: %v", err)
return result
}
knownSchemas := findAccountSchemas(account)
if !containsString(knownSchemas, schema) {
result.Message = fmt.Sprintf("Schema %q is not one of the databases discovered for account %q (known: %v)",
schema, account, knownSchemas)
return result
}
result.Database = schema
// SHOW CREATE captures the backup. Different MySQL grammars per
// kind: TRIGGER and EVENT use the schema-qualified name in
// `<schema>.<name>` form; PROCEDURE and FUNCTION accept the same
// shape under modern MySQL. Use the unified form for consistency.
showCreateSQL := fmt.Sprintf("SHOW CREATE %s %s.%s",
strings.ToUpper(kind), quotedSchema, quotedName)
createOutput := runMySQLQueryRoot(schema, showCreateSQL)
if len(createOutput) == 0 {
result.Message = fmt.Sprintf("SHOW CREATE returned no rows for %s %s.%s -- object missing or permission denied",
kind, schema, name)
return result
}
createSQL, err := showCreateStatement(kind, createOutput[0])
if err != nil {
result.Message = fmt.Sprintf("SHOW CREATE for %s %s.%s did not yield a restorable statement (refusing to drop): %v",
kind, schema, name, err)
return result
}
if preview {
result.Message = fmt.Sprintf("PREVIEW: would drop %s %s.%s", kind, schema, name)
result.Details = []string{
fmt.Sprintf("Captured CREATE SQL (%d bytes)", len(createSQL)),
"No backup written and no DROP executed in preview mode.",
}
result.Success = true
return result
}
// Persist backup BEFORE the drop so a SQL failure on DROP still
// leaves the operator with a record of what existed.
sdb := store.Global()
if sdb == nil {
result.Message = "bbolt store not available; refusing to drop without a recorded backup"
return result
}
if err := sdb.PutDBObjectBackup(store.DBObjectBackup{
Account: account,
Schema: schema,
Kind: kind,
Name: name,
CreateSQL: createSQL,
DroppedAt: time.Now().UTC(),
DroppedBy: "csm-cli",
}); err != nil {
result.Message = fmt.Sprintf("recording backup failed (refusing to drop): %v", err)
return result
}
dropSQL := fmt.Sprintf("DROP %s IF EXISTS %s.%s",
strings.ToUpper(kind), quotedSchema, quotedName)
// runMySQLExecRoot reports the mysql client's exec error
// directly. The previous use of runMySQLQueryRoot misread a
// zero-exit + empty-stdout (the success signature for DROP) as
// failure.
if err := runMySQLExecRoot(schema, dropSQL); err != nil {
result.Message = fmt.Sprintf("DROP %s %s.%s failed: %v", kind, schema, name, err)
return result
}
result.Details = []string{
fmt.Sprintf("Dropped %s %s.%s", kind, schema, name),
fmt.Sprintf("Backup recorded in bbolt: %d bytes", len(createSQL)),
}
result.Message = fmt.Sprintf("Dropped %s %s.%s (backup retained)", kind, schema, name)
result.Success = true
return result
}
// findAccountSchemas returns every distinct database name discovered
// across the account's wp-config.php files. Multiple WordPress
// installations under the same account commonly reuse one database
// but can use several; the CLI relies on this list to validate
// operator input before opening any connection.
func findAccountSchemas(account string) []string {
patterns := wpInstallConfigPaths(wpInstallsForAccount(context.Background(), "db_objects", account))
seen := map[string]struct{}{}
var out []string
for _, path := range patterns {
// parseWPConfig handles missing files silently, so the bare
// non-glob first-entry path is harmless when the account has
// no public_html/wp-config.php.
creds := parseWPConfig(path)
if creds.dbName == "" {
continue
}
if _, ok := seen[creds.dbName]; ok {
continue
}
seen[creds.dbName] = struct{}{}
out = append(out, creds.dbName)
}
return out
}
// containsString reports whether haystack contains needle. Local
// because the package's other helper of the same name lives in a
// _test.go file (waf_test.go) and is not visible to production
// builds.
func containsString(haystack []string, needle string) bool {
for _, h := range haystack {
if h == needle {
return true
}
}
return false
}
// RestoreDBObjectBackup re-executes the captured CREATE SQL for a
// previously-dropped MySQL trigger / event / procedure / function.
// Looks up the row in the db_object_backups bbolt bucket by exact
// key; the caller (typically the web UI's cleanup-history page)
// supplies the key it got from the listing endpoint.
//
// The operation is operator-driven: there is no auto-
// restore. The webui handler enforces the same.
func RestoreDBObjectBackup(backupKey string) DBCleanResult {
result := DBCleanResult{Action: "restore-object"}
sdb := store.Global()
if sdb == nil {
result.Message = "bbolt store not available"
return result
}
rec, ok, err := sdb.GetDBObjectBackupByKey(backupKey)
if err != nil {
result.Message = fmt.Sprintf("looking up backup: %v", err)
return result
}
if !ok {
result.Message = "backup not found (may have been pruned)"
return result
}
result.Account = rec.Account
result.Database = rec.Schema
if rec.CreateSQL == "" {
result.Message = "backup record has no CREATE SQL"
return result
}
if err := runMySQLExecRoot(rec.Schema, rec.CreateSQL); err != nil {
result.Message = fmt.Sprintf("re-executing CREATE failed: %v", err)
return result
}
result.Details = []string{
fmt.Sprintf("Restored %s %s.%s", rec.Kind, rec.Schema, rec.Name),
fmt.Sprintf("Original drop: %s by %s", rec.DroppedAt.Format(time.RFC3339), rec.DroppedBy),
}
if err := sdb.MarkDBObjectBackupRestored(backupKey, time.Now().UTC()); err != nil {
result.Details = append(result.Details, fmt.Sprintf("Restore state was not recorded: %v", err))
}
result.Message = fmt.Sprintf("Restored %s %s.%s from backup", rec.Kind, rec.Schema, rec.Name)
result.Success = true
return result
}
package checks
import (
"fmt"
"net/url"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/store"
)
// externalScriptHosts returns the hosts of every <script src> in content
// that is neither a known-safe service nor structurally malicious. The
// structural classifier already reports the latter as Critical; the former
// are pre-approved. What remains is an unremarkable external host, which is
// exactly the shape a careful injection takes and which no other signal
// covers.
func externalScriptHosts(content string) []string {
var hosts []string
seen := map[string]bool{}
// A payload stored as JSON carries escaped slashes, which the src
// grammar does not match; normalise before extracting.
content = unescapeStoredSlashes(content)
for _, match := range scriptSrcRe.FindAllStringSubmatch(content, -1) {
if len(match) < 2 {
continue
}
raw := match[1]
if isSafeScriptDomain(raw) {
continue
}
if bad, _ := scriptSrcMaliciousReason(raw); bad {
continue
}
normalised := raw
if strings.HasPrefix(normalised, "//") {
normalised = "https:" + normalised
}
u, err := url.Parse(normalised)
if err != nil || u == nil {
continue
}
host := strings.ToLower(u.Hostname())
if host == "" || seen[host] {
continue
}
seen[host] = true
hosts = append(hosts, host)
}
return hosts
}
// newExternalScriptFindings reports, as a Warning, each external script host
// in an option value that firstSeen has not recorded before. Severity stays
// below the auto-response threshold: this is a "look at this" signal, not
// proof of injection.
func newExternalScriptFindings(user string, creds wpDBCreds, prefix, option, value string, firstSeen func(option, host string) bool) []alert.Finding {
var findings []alert.Finding
for _, host := range externalScriptHosts(value) {
if !firstSeen(option, host) {
continue
}
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "db_options_new_external_script",
Message: fmt.Sprintf("New external script host in wp_options '%s' (account: %s): %s", option, user, host),
Details: dbContentFindingDetails(creds, prefix,
fmt.Sprintf("Option: %s", option),
fmt.Sprintf("Script host: %s", host),
fmt.Sprintf("Content preview: %s", truncateDB(value, 200)),
"First appearance of this host since the site's baseline; verify it is a service the site owner added."),
DedupKey: dbContentDedupKey(user, creds, prefix,
fmt.Sprintf("Option: %s", option),
fmt.Sprintf("Script host: %s", host),
fmt.Sprintf("Content preview: %s", truncateDB(value, 200)),
"First appearance of this host since the site's baseline; verify it is a service the site owner added."),
})
}
return findings
}
// externalScriptSiteKey identifies one WordPress install's options table in
// the first-seen store.
func externalScriptSiteKey(dbName, prefix string) string {
return dbName + "|" + prefix
}
// storeFirstSeen adapts the bbolt store to the firstSeen callback; without a
// store nothing is ever new, so the scan stays silent rather than reporting
// every host on every cycle.
func storeFirstSeen(site string) func(option, host string) bool {
db := store.Global()
if db == nil {
return func(string, string) bool { return false }
}
return func(option, host string) bool {
isNew, err := db.MarkExternalScriptSeen(site, option, host, time.Now())
return err == nil && isNew
}
}
package checks
import (
"encoding/hex"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/alert"
)
// pluginNoticeSinkOptions are plugin status options whose contents WordPress
// renders into the admin dashboard as a notice. A stored cross-site scripting
// bug in the plugin turns one of these rows into a loader that runs with the
// privileges of whichever administrator opens the dashboard next.
//
// LiteSpeed Cache before 5.7.0.1 (CVE-2023-40000) lets an unauthenticated
// request write its CDN setup status, and the observed campaign puts a
// <script src> into cdn_setup_err, then into the rendered message list. Both
// rows hold plugin state, never site content, so markup that executes is
// proof of injection on its own.
var pluginNoticeSinkOptions = map[string]string{
"litespeed.cdn_setup._summary": "LiteSpeed Cache CDN setup status",
"litespeed.admin_display.messages": "LiteSpeed Cache admin notices",
"litespeed.admin_display.msg_pin": "LiteSpeed Cache pinned admin notice",
}
// executableMarkupRe matches markup that runs code when a notice is rendered.
// Notice sinks legitimately carry layout markup -- LiteSpeed writes its own
// errors as a styled div -- so only the executing constructs count.
var executableMarkupRe = regexp.MustCompile(`(?i)<script[\s>]|<iframe[\s>]|javascript:|\bon(?:error|load|click|mouseover)\s*=|eval\s*\(\s*atob`)
const maxPluginNoticeBytes = 65536
// Notice lists can grow beyond a preview-sized read. Bound each value in bytes
// and carry its stored length so omitted content never looks like a clean scan.
// Hex preserves whitespace and stored escapes through the batch-row transport.
func checkWPPluginNotices(user string, creds wpDBCreds, prefix string) []alert.Finding {
query := fmt.Sprintf(
"SELECT option_name, OCTET_LENGTH(option_value), "+
"CONCAT('x', HEX(LEFT(CAST(option_value AS BINARY), %d))) FROM %soptions "+
"WHERE option_name IN (%s) LIMIT %d",
maxPluginNoticeBytes, prefix, pluginNoticeSinkNameList(), len(pluginNoticeSinkOptions))
var findings []alert.Finding
for _, line := range runMySQLQuery(creds, query) {
option, value, complete := parsePluginNoticeRow(line)
if !complete {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
if finding := pluginNoticeInjectionFinding(user, creds, prefix, option, value); finding != nil {
findings = append(findings, *finding)
}
}
return findings
}
func parsePluginNoticeRow(line string) (option, value string, complete bool) {
parts := strings.SplitN(line, "\t", 3)
if len(parts) != 3 {
return "", "", false
}
size, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil || size < 0 || size > maxPluginNoticeBytes {
return "", "", false
}
encoded := parts[2]
if !strings.HasPrefix(encoded, "x") || int64(len(encoded)-1) != 2*size {
return "", "", false
}
decoded, err := hex.DecodeString(encoded[1:])
if err != nil {
return "", "", false
}
return parts[0], string(decoded), true
}
// pluginNoticeInjectionFinding reports executable markup stored in a plugin
// status option that WordPress renders as an admin notice. The option's
// identity carries the verdict: neither host reputation nor a first-seen
// baseline applies, so a payload that predates CSM's baseline and a loader on
// an ordinary HTTPS host are both reported.
func pluginNoticeInjectionFinding(user string, creds wpDBCreds, prefix, option, value string) *alert.Finding {
name := strings.ToLower(strings.TrimSpace(option))
sink, ok := pluginNoticeSinkOptions[name]
if !ok {
return nil
}
decoded := unescapeStoredSlashes(value)
marker := executableMarkupRe.FindString(decoded)
if marker == "" {
return nil
}
detail := []string{
fmt.Sprintf("Option: %s (%s)", option, sink),
fmt.Sprintf("Executable markup: %s", strings.TrimSpace(marker)),
}
if hosts := externalScriptHosts(value); len(hosts) > 0 {
detail = append(detail, fmt.Sprintf("Script host: %s", strings.Join(hosts, ", ")))
}
detail = append(detail,
fmt.Sprintf("Content preview: %s", truncateDB(decoded, 200)),
"This option holds plugin state, not site content. WordPress prints it in the dashboard, so the code runs for the next administrator who opens it.")
return &alert.Finding{
Severity: alert.Critical,
Check: "db_options_plugin_notice_injection",
Message: fmt.Sprintf("Executable markup stored in plugin notice option '%s' (account: %s)", option, user),
Details: dbContentFindingDetails(creds, prefix, detail...),
DedupKey: dbContentDedupKey(user, creds, prefix, detail...),
}
}
// pluginNoticeSinkNameList renders the sink names for a SQL IN clause.
func pluginNoticeSinkNameList() string {
names := make([]string, 0, len(pluginNoticeSinkOptions))
for name := range pluginNoticeSinkOptions {
names = append(names, "'"+name+"'")
}
sort.Strings(names)
return strings.Join(names, ", ")
}
package checks
import (
"bufio"
"context"
"crypto/sha256"
"encoding/binary"
"fmt"
"io"
"net/netip"
"net/url"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mysqlclient"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
// Malicious patterns in WordPress database content.
//
// requiresExternalScript: when true, a matching row is only reported if
// its content also contains a <script src=...> pointing at a domain NOT
// on the known-safe list. This filters out the legitimate analytics and
// widget embeds that site owners place in page content (Google Tag
// Manager, Google merchant badge, HubSpot, Mailchimp, etc.) without
// weakening detection of attacker-injected external loaders.
var dbMalwarePatterns = []struct {
pattern string
severity alert.Severity
desc string
requiresExternalScript bool
}{
// The script-tag entry catches BOTH inline <script> blocks and
// <script src=...> loaders as a fast LIKE pre-filter; the Go post-
// filter (hasMaliciousExternalScript) verifies the presence of a
// non-safe-domain external src before raising a finding. Inline
// obfuscation without an external src is caught by the subsequent
// code-pattern entries below.
{"<script", alert.High, "injected <script> tag with non-safe external src", true},
{"eval(", alert.High, "eval() in database content", false},
{"base64_decode", alert.High, "base64_decode in database content", false},
{"document.write(", alert.High, "document.write injection", false},
{"String.fromCharCode", alert.High, "JavaScript obfuscation (fromCharCode)", false},
{".workers.dev", alert.Critical, "Cloudflare Workers exfiltration URL", false},
{"gist.githubusercontent.com", alert.Critical, "GitHub Gist payload URL", false},
{"pastebin.com/raw", alert.Critical, "Pastebin payload URL", false},
}
// nonDocRootDirs are common account-data directories that are never candidate
// document roots during the home-directory walk. "www" is deliberately absent:
// where it is cPanel's alias for public_html the discovery walk collapses the
// symlink (canonicalWPInstallPath), and where it is a real directory it is a
// document root serving a real site.
var nonDocRootDirs = map[string]bool{
"mail": true, "etc": true, "logs": true, "ssl": true, "tmp": true,
"public_ftp": true, "cache": true, ".cagefs": true,
"access-logs": true, "access_logs": true, "backups": true,
"cgi-bin": true, "perl5": true, "spamassassin": true, "var": true,
}
// servedState records whether the panel currently serves a document root.
// A dormant install is not harmless -- it holds a live database and becomes
// public again the moment the domain is re-pointed -- but it is not being
// served to anyone today, and triage that cannot tell the two apart orders its
// queue wrongly in both directions.
type servedState int
const (
// servedUnknown is the honest answer when the panel's domain map could not
// be read. It is not "not served".
servedUnknown servedState = iota
servedByPanel
notServed
)
// wpConfigPaths returns direct wp-config.php files at account document roots,
// each with whether the panel currently serves that root.
func wpConfigPaths(ctx context.Context) ([]string, map[string]servedState) {
paths, served, _ := wpConfigPathsWithDomains(ctx)
return paths, served
}
// wpConfigPathsWithDomains projects the shared install seam into the shapes
// CheckDatabaseContent works in. Discovery itself lives in wpinstalls.go, so
// this check, the object and overlap scanners, the core verifier and every
// fixer see the same installs.
func wpConfigPathsWithDomains(ctx context.Context) ([]string, map[string]servedState, map[string][]string) {
installs, panelDomains := wpInstallsWithDomains(ctx, "db_content")
paths := make([]string, 0, len(installs))
served := make(map[string]servedState, len(installs))
for _, in := range installs {
paths = append(paths, in.ConfigPath)
served[in.ConfigPath] = in.Served
}
return paths, served, panelDomains
}
// wpConfigOwners maps each discovered wp-config.php to the hosting account
// discovery attributed it to; unattributable installs are absent so their
// findings stay unstamped.
func wpConfigOwners(installs []wpInstall) map[string]string {
owners := make(map[string]string, len(installs))
for _, in := range installs {
if in.Account != "" {
owners[in.ConfigPath] = in.Account
}
}
return owners
}
// The shared vhost parser omits wildcard names because they cannot be used as
// an HTTP Host for exposure probes. They still declare a served document root
// and tenant ownership, so the database scan parses those rows separately.
func parseWildcardUserdataDomainRootsChecked(content string) ([]vhost, bool) {
var out []vhost
complete := true
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, "*.") {
continue
}
parsed, lineComplete := parseUserdataDomainRootsChecked(strings.TrimPrefix(line, "*.") + "\n")
if !lineComplete || len(parsed) != 1 {
complete = false
continue
}
parsed[0].domain = "*." + parsed[0].domain
out = append(out, parsed[0])
}
return out, complete
}
func docrootBelongsToCPanelUser(root, user string) bool {
parts := strings.Split(filepath.Clean(root), string(filepath.Separator))
for i, part := range parts {
if !isCPanelHomeBase(part) || i+2 >= len(parts) || parts[i+1] != user {
continue
}
return true
}
return false
}
func isCPanelHomeBase(name string) bool {
if name == "home" {
return true
}
if !strings.HasPrefix(name, "home") || len(name) == len("home") {
return false
}
for _, r := range name[len("home"):] {
if r < '0' || r > '9' {
return false
}
}
return true
}
const maxWPSecondaryBlogs = 100
// dbSpamSampleLimit bounds the rows pulled back per spam pattern. When a
// pattern fills it, the reported count is a floor rather than a total.
const dbSpamSampleLimit = 200
func spamCountLabel(n int, truncated bool) string {
if truncated {
return fmt.Sprintf("at least %d", n)
}
return strconv.Itoa(n)
}
// dbScanCoverage counts why discovered installs could not be inspected and
// keeps one example path per reason. Multisite limits have their own detailed
// findings and are not counted again here.
//
// The owner remains incomplete when any install fails. Independently
// completed database scopes can still retire their own previous findings.
type dbScanCoverage struct {
discovered int
discoveryIncomplete bool
counts map[string]int
examples map[string]string
queryFailures map[string]int
queryFailureOverflow int
}
func (c *dbScanCoverage) record(reason, configPath string) {
if c == nil {
return
}
if c.counts == nil {
c.counts = make(map[string]int, 5)
c.examples = make(map[string]string, 5)
}
c.counts[reason]++
if c.examples[reason] == "" {
c.examples[reason] = configPath
}
}
func (c *dbScanCoverage) skipped() int {
if c == nil {
return 0
}
var n int
for _, v := range c.counts {
n += v
}
return n
}
// summary renders the reason breakdown, or the empty string when nothing was
// attributed, including when discovery stops before reaching any install.
func (c *dbScanCoverage) summary() string {
if c.skipped() == 0 {
return ""
}
reasons := make([]string, 0, len(c.counts))
for reason := range c.counts {
reasons = append(reasons, reason)
}
sort.Strings(reasons)
var b strings.Builder
fmt.Fprintf(&b, "%d of %d discovered installs could not be fully inspected.\n", c.skipped(), c.discovered)
for _, reason := range reasons {
// Account-controlled names must not forge reason lines or terminal
// commands. Bound the escaped display so expansion cannot grow it.
example := strconv.QuoteToASCII(c.examples[reason])
example = truncateDB(example[1:len(example)-1], 200)
fmt.Fprintf(&b, "%s=%d (example: %s)\n", reason, c.counts[reason], example)
}
b.WriteString(c.queryFailureSummary())
if c.discoveryIncomplete {
b.WriteString("Document-root discovery was incomplete; additional installs may be missing.\n")
}
return b.String()
}
// CheckDatabaseContent scans WordPress databases for injected malware,
// spam content, siteurl hijacking, and rogue admin accounts.
func CheckDatabaseContent(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
coverage := &dbScanCoverage{}
installs, panelDomains := wpInstallsWithDomains(ctx, "db_content")
coverage.discoveryIncomplete = checkMarkedIncomplete(ctx, "db_content")
if len(installs) == 0 {
return appendDatabaseScanIncompleteFinding(ctx, nil, coverage)
}
// cPanel locks an account's database users when it suspends the account,
// so every query against its installs fails with an access error. Such an
// install is dropped before coverage is counted: leaving it in made the
// scan permanently short of full coverage, with a warning no operator
// action could clear while the account stayed suspended.
wpConfigs := make([]string, 0, len(installs))
servedRoots := make(map[string]servedState, len(installs))
scannable := make([]wpInstall, 0, len(installs))
for _, in := range installs {
if accountSuspended(wpConfigUser(filepath.Dir(in.ConfigPath))) {
continue
}
scannable = append(scannable, in)
wpConfigs = append(wpConfigs, in.ConfigPath)
servedRoots[in.ConfigPath] = in.Served
}
installs = scannable
if len(installs) == 0 {
return appendDatabaseScanIncompleteFinding(ctx, nil, coverage)
}
coverage.discovered = len(installs)
owners := wpConfigOwners(installs)
domainOwnership := newPanelDomainOwnership(panelDomains)
// Cache the coverage outcome as well as the scan: aliases of an unreadable
// database are affected installs too, but must not repeat its queries.
seenDatabases := make(map[string]string, len(wpConfigs))
completedScopes := make(map[string]bool)
for _, wpConfig := range wpConfigs {
if ctx.Err() != nil {
return findings
}
user := wpConfigUser(filepath.Dir(wpConfig))
creds, complete := parseWPConfigChecked(wpConfig)
if !complete {
coverage.record("unreadable_config", wpConfig)
markCheckIncomplete(ctx, "db_content")
continue
}
if creds.dbName == "" || creds.dbUser == "" {
// A missing login does not erase an otherwise known scope. Its
// healthy alias must not retire findings this install could not
// examine, regardless of which config discovery returned first.
if prefix, ok := resolveTablePrefix(creds); creds.dbName != "" && ok {
completedScopes[dbContentDedupKey(user, creds, prefix)] = false
}
coverage.record("missing_credentials", wpConfig)
markCheckIncomplete(ctx, "db_content")
continue
}
prefix, ok := resolveTablePrefix(creds)
if !ok {
coverage.record("unresolved_table_prefix", wpConfig)
markCheckIncomplete(ctx, "db_content")
continue
}
creds.tablePrefix = prefix
creds.docrootServed = servedRoots[wpConfig]
creds.panelDomains = domainOwnership
databaseKey := strings.Join([]string{
user, creds.dbHost, creds.dbName, creds.dbUser, creds.dbPass, prefix,
strconv.FormatBool(creds.multisite),
}, "\x00")
if reason, duplicate := seenDatabases[databaseKey]; duplicate {
if reason != "" {
coverage.record(reason, wpConfig)
}
continue
}
// Isolate content-read gaps so an earlier install's failure cannot
// mask this one's. The scanner keeps the outer context for multisite
// limits, which already emit a separate account-specific finding.
queryCtx, contentIncomplete := withIncompleteCheckCollector(ctx)
creds.queryCtx = queryCtx
creds.queryState = new(dbQueryState)
// Stamp each install's own slice before merging: the host-wide
// summary appended below must never inherit an owner.
installFindings := capPhantomAuthorFindings(wpInstallScanner(ctx, user, creds, prefix), maxPhantomAuthorsReported)
scope := dbContentDedupKey(user, creds, prefix)
multisiteLimited := false
for i := range installFindings {
installFindings[i].CoverageScope = scope
if installFindings[i].Check == "db_content_scan_incomplete" {
multisiteLimited = true
}
}
findings = append(findings, stampTenantIDIfEmpty(installFindings, owners[wpConfig])...)
var reason string
if creds.queryState.failed {
reason = "query_failed"
} else if contentIncomplete.contains("db_content") {
reason = "incomplete_content"
}
scopeComplete := reason == "" && !multisiteLimited
if prior, seen := completedScopes[scope]; seen {
scopeComplete = scopeComplete && prior
}
completedScopes[scope] = scopeComplete
coverage.recordQueryFailures(creds.queryState)
seenDatabases[databaseKey] = reason
if reason != "" {
coverage.record(reason, wpConfig)
markCheckIncomplete(ctx, "db_content")
}
}
recordCompletedCoverageScopes(ctx, "db_content", completedScopes)
return appendDatabaseScanIncompleteFinding(ctx, findings, coverage)
}
// wpInstallScanner is the per-install scan boundary. Tests replace it with
// an inert scanner to prove ownership stamping for every finding name
// without driving each SQL scanner.
var wpInstallScanner = scanWPInstall
// scanWPInstall runs every content, user and multisite scanner for one
// discovered install and returns the unstamped findings.
func scanWPInstall(ctx context.Context, user string, creds wpDBCreds, prefix string) []alert.Finding {
var installFindings []alert.Finding
// Always scan the main-site (or single-site) tables. In
// multisite, blog ID 1 keeps the unprefixed names; in a
// single-site install these are the only tables.
installFindings = append(installFindings, scanWPBlog(user, creds, prefix, prefix)...)
// wp_users / wp_usermeta are network-wide in multisite, so
// the user-table scan runs once regardless of the layout.
installFindings = append(installFindings, checkWPUsers(user, creds.withQueryStage("users"), prefix)...)
// Multisite: enumerate active secondary blog IDs and scan
// each one's wp_<N>_options / wp_<N>_posts. Spam, archived,
// and deleted blogs are excluded -- their content is
// already operator-suppressed at the WP level, and most
// hosts have stale ones we'd otherwise alert on
// indefinitely.
if creds.multisite {
installFindings = append(installFindings, scanMultisiteSecondaryBlogs(ctx, user, creds, prefix)...)
}
return installFindings
}
// scanWPBlog runs checks whose tables belong to one blog. usersPrefix stays
// separate because multisite blogs share the network-wide users table.
func scanWPBlog(user string, creds wpDBCreds, sitePrefix, usersPrefix string) []alert.Finding {
var findings []alert.Finding
findings = append(findings, checkWPOptions(user, creds.withQueryStage("options"), sitePrefix)...)
findings = append(findings, checkWPPosts(user, creds.withQueryStage("posts"), sitePrefix)...)
findings = append(findings, checkWPStoredCode(user, creds.withQueryStage("stored_code"), sitePrefix)...)
findings = append(findings, checkWPSpamTaxonomy(user, creds.withQueryStage("taxonomy"), sitePrefix)...)
findings = append(findings, checkWPHiddenLinks(user, creds.withQueryStage("hidden_links"), sitePrefix)...)
findings = append(findings, checkWPCloakConfig(user, creds.withQueryStage("cloak_config"), sitePrefix)...)
// Rate change rather than vocabulary: the next kit will use different
// words, but it will still publish a flood onto a long-quiet site.
findings = append(findings, checkWPPostVolumeBurst(user, creds.withQueryStage("post_burst"), sitePrefix)...)
findings = append(findings,
checkWPPhantomAuthors(user, creds.withQueryStage("phantom_authors"), sitePrefix, usersPrefix, maxPhantomAuthorsReported)...)
return findings
}
func wpConfigUser(path string) string {
parts := strings.Split(filepath.Clean(path), string(filepath.Separator))
for i, part := range parts {
if isCPanelHomeBase(part) && i+1 < len(parts) {
return parts[i+1]
}
}
return extractUser(path)
}
// dbContentHostCoverageDedupKey is the identity of the host-wide coverage
// summary. Its Details carry per-reason counts and examples that shift between
// cycles while the same degradation persists. The multisite limit warning keeps
// its own per-install identity.
const dbContentHostCoverageDedupKey = "install_coverage"
func appendDatabaseScanIncompleteFinding(ctx context.Context, findings []alert.Finding, coverage *dbScanCoverage) []alert.Finding {
if !checkMarkedIncomplete(ctx, "db_content") {
return findings
}
// A multisite-limit warning covers only its own network. Suppress the
// generic fallback only when no other coverage gaps need reporting.
if coverage.skipped() == 0 && !coverage.discoveryIncomplete {
for _, finding := range findings {
if finding.Check == "db_content_scan_incomplete" {
return findings
}
}
}
return append(findings, alert.Finding{
Severity: alert.Warning,
Check: "db_content_scan_incomplete",
Message: "WordPress database scan could not inspect every discovered install",
Details: databaseScanIncompleteDetails(coverage),
DedupKey: dbContentHostCoverageDedupKey,
})
}
// databaseScanIncompleteDetails names what was skipped and why when the scan
// got far enough to attribute a cause, and falls back to the generic sentence
// when no install-specific cause was recorded.
func databaseScanIncompleteDetails(coverage *dbScanCoverage) string {
const retained = "Findings without complete database coverage are retained."
if summary := coverage.summary(); summary != "" {
return summary + retained
}
return "A document-root record, wp-config.php file, or database query could not be read safely. " + retained
}
// scanMultisiteSecondaryBlogs queries wp_blogs for active blog IDs other than 1
// and runs the per-blog scans against each. The user-table scan does not
// iterate because WP shares wp_users / wp_usermeta across the entire network
// by default. A site-specific user table only exists on configurations that
// override that, which we ignore here for v1. The phantom-author scan does
// iterate each posts table, but joins it to that shared users table.
//
// blog_id=1 is excluded because its tables are unprefixed and were
// already scanned by the caller.
func scanMultisiteSecondaryBlogs(ctx context.Context, user string, creds wpDBCreds, prefix string) []alert.Finding {
query := fmt.Sprintf(
"SELECT blog_id FROM %sblogs WHERE archived = 0 AND deleted = 0 AND spam = 0 AND blog_id != 1 ORDER BY blog_id LIMIT %d",
prefix, maxWPSecondaryBlogs+1,
)
rows := runMySQLQuery(creds.withQueryStage("multisite_discovery"), query)
var findings []alert.Finding
truncated := len(rows) > maxWPSecondaryBlogs
if truncated {
rows = rows[:maxWPSecondaryBlogs]
}
for _, row := range rows {
if ctx.Err() != nil {
return findings
}
blogID := strings.TrimSpace(row)
if blogID == "" || blogID == "1" {
continue
}
// Guard against any garbage in the row -- only digits.
if !isAllDigits(blogID) {
markCheckIncomplete(creds.queryCtx, "db_content")
markCheckIncomplete(ctx, "db_content")
continue
}
sitePrefix := fmt.Sprintf("%s%s_", prefix, blogID)
findings = append(findings, scanWPBlog(user, creds, sitePrefix, prefix)...)
}
if truncated {
markCheckIncomplete(ctx, "db_content")
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "db_content_scan_incomplete",
Message: fmt.Sprintf("WordPress multisite database scan reached its %d-site safety limit (account: %s)", maxWPSecondaryBlogs, user),
Details: dbContentFindingDetails(creds, prefix,
"The network has more active secondary sites than one scheduled scan can safely inspect."),
DedupKey: dbContentDedupKey(user, creds, prefix,
"The network has more active secondary sites than one scheduled scan can safely inspect."),
})
}
return findings
}
func isAllDigits(s string) bool {
if s == "" {
return false
}
for _, r := range s {
if r < '0' || r > '9' {
return false
}
}
return true
}
type wpDBCreds struct {
dbName string
dbUser string
dbPass string
dbHost string
tablePrefix string
// docrootServed records whether the panel serves this install's document
// root, so a finding says whether it is reachable today.
docrootServed servedState
// panelDomains is the complete panel domain ownership map. The foreign-host
// check needs every account, not just this one, so a more-specific domain
// delegated to another tenant wins over this account's parent domain.
panelDomains *panelDomainOwnership
// queryCtx ties scheduled database work to the runner's deadline. Command
// paths leave it nil and retain the per-query timeout below.
queryCtx context.Context
// queryOwner identifies the CMS check whose coverage depends on a query.
// WordPress callers use the default owner when this is empty.
queryOwner string
// queryState is shared by the sequential queries for one install.
// Coverage failures and an unusable connection are tracked separately.
queryState *dbQueryState
queryStage string
// multisite is set when wp-config.php declares
// `define('MULTISITE', true)`. In multisite, the main blog
// (ID 1) keeps the unprefixed table names and secondary blogs
// live under `wp_<N>_options` / `wp_<N>_posts`. CheckDatabaseContent
// scans both layouts when this is set; a single-site install
// (multisite=false) skips the wp_blogs lookup and per-site
// iteration entirely.
multisite bool
}
// parseWPConfig extracts database credentials from wp-config.php.
func parseWPConfig(path string) wpDBCreds {
creds, complete := parseWPConfigChecked(path)
if !complete {
return wpDBCreds{}
}
return creds
}
// parseWPConfigChecked bounds account-controlled input so a special or very
// large wp-config.php cannot strand the scheduled database scan.
func parseWPConfigChecked(path string) (wpDBCreds, bool) {
f, err := openTenantRegularFile(path)
if err != nil {
return wpDBCreds{}, false
}
defer func() { _ = f.Close() }()
var creds wpDBCreds
limited := &io.LimitedReader{R: f, N: maxCMSConfigBytes + 1}
scanner := bufio.NewScanner(limited)
scanner.Buffer(make([]byte, 64*1024), maxCMSConfigBytes+1)
for scanner.Scan() {
line := scanner.Text()
// Match: define( 'DB_NAME', 'value' );
if val := extractDefine(line, "DB_NAME"); val != "" {
creds.dbName = val
}
if val := extractDefine(line, "DB_USER"); val != "" {
creds.dbUser = val
}
if val := extractDefine(line, "DB_PASSWORD"); val != "" {
creds.dbPass = val
}
if val := extractDefine(line, "DB_HOST"); val != "" {
creds.dbHost = val
}
// Match: $table_prefix = 'wp_';
if strings.Contains(line, "$table_prefix") {
if val := extractPHPString(line); val != "" {
creds.tablePrefix = val
}
}
// Match: define( 'MULTISITE', true );
if extractDefineBool(line, "MULTISITE") {
creds.multisite = true
}
}
if creds.dbHost == "" {
creds.dbHost = "localhost"
}
return creds, scanner.Err() == nil && limited.N > 0
}
// extractDefine extracts the value from: define( 'KEY', 'value' );
func extractDefine(line, key string) string {
if !strings.Contains(line, key) {
return ""
}
// Skip comments
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "//") || strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, "/*") {
return ""
}
// After the literal key, step past the first comma so
// extractPHPString picks up the VALUE's opening quote rather than
// the KEY's trailing closing quote. Without this, on input
// define( 'DB_NAME', 'wordpress_db' );
// extractPHPString would see `', 'wordpress_db' );` and return
// `, ` — the substring between the closing quote of 'DB_NAME' and
// the opening quote of 'wordpress_db'. Every real WordPress
// install's wp-config.php triggered this, which silently broke
// the entire WP database scan check.
rest := line[strings.Index(line, key)+len(key):]
if commaIdx := strings.Index(rest, ","); commaIdx >= 0 {
rest = rest[commaIdx+1:]
}
return extractPHPString(rest)
}
// extractDefineBool returns true when line is a non-comment
// define('<key>', true) -- i.e., a bare boolean value rather than a
// quoted string. Used for `MULTISITE` and any future bool defines
// CSM cares about. Whitespace is permissive, case-insensitive on
// the literal `true`, trailing PHP/inline comments tolerated.
//
// The key must appear inside its enclosing PHP quotes (single or
// double). This avoids a substring-match collision: a WordPress
// wp-config.php commonly carries `define('WP_ALLOW_MULTISITE',
// true)` to enable the admin network creator on single-site
// installs; matching MULTISITE as a bare substring would falsely
// detect those as multisite hosts.
//
// Operators using anything other than the canonical `true` literal
// (e.g., `!false`, `1`, `defined('FOO')`) won't get multisite
// scanning. That's preferable to running an arbitrary PHP expression
// evaluator over wp-config.php.
func extractDefineBool(line, key string) bool {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "//") || strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, "/*") {
return false
}
if !strings.Contains(trimmed, "define") {
return false
}
// Find the key's quoted form and seek past the closing quote.
// Two acceptable openings: 'KEY' and "KEY". The key never
// appears unquoted inside a define() literal in valid PHP.
var keyEnd int
switch {
case strings.Contains(trimmed, "'"+key+"'"):
keyEnd = strings.Index(trimmed, "'"+key+"'") + len(key) + 2
case strings.Contains(trimmed, `"`+key+`"`):
keyEnd = strings.Index(trimmed, `"`+key+`"`) + len(key) + 2
default:
return false
}
rest := trimmed[keyEnd:]
commaIdx := strings.Index(rest, ",")
if commaIdx < 0 {
return false
}
value := rest[commaIdx+1:]
// The value runs until the closing paren; everything after it
// is the statement terminator and any trailing comment.
if closeIdx := strings.Index(value, ")"); closeIdx >= 0 {
value = value[:closeIdx]
}
return strings.EqualFold(strings.TrimSpace(value), "true")
}
// extractPHPString extracts the first quoted string value from a line.
func extractPHPString(s string) string {
// Find opening quote
for _, quote := range []byte{'\'', '"'} {
start := strings.IndexByte(s, quote)
if start < 0 {
continue
}
rest := s[start+1:]
end := strings.IndexByte(rest, quote)
if end < 0 {
continue
}
return rest[:end]
}
return ""
}
// runMySQLQuery executes a MySQL query via the in-process database/sql
// driver and returns each row tab-joined, matching the legacy
// `mysql -N -B -e <query>` output shape so existing tab-split callers
// keep working unchanged. Returns nil on any open / query / scan
// error (the legacy implementation swallowed errors the same way).
// Var so tests can serve canned rows without a live database.
var runMySQLQuery = func(creds wpDBCreds, query string) []string {
if creds.queryState != nil && creds.queryState.halted {
return nil
}
parent := creds.queryCtx
if parent == nil {
parent = context.Background()
}
ctx, cancel := context.WithTimeout(parent, 2*time.Minute)
defer cancel()
rows, err := mysqlclient.PerAccountQuery(ctx, mysqlclient.Creds{
User: creds.dbUser,
Password: creds.dbPass,
Host: creds.dbHost,
DBName: creds.dbName,
}, query)
if err != nil {
creds.queryState.record(creds.queryStage, err)
markCheckIncomplete(creds.queryCtx, creds.queryCheck())
return nil
}
out := make([]string, 0, len(rows))
for _, line := range rows {
line = strings.TrimSpace(line)
if line != "" {
out = append(out, line)
}
}
if len(out) == 0 {
return nil
}
return out
}
func (c wpDBCreds) queryCheck() string {
if c.queryOwner != "" {
return c.queryOwner
}
return "db_content"
}
// siteURLPoisonReason reports why a siteurl/home value cannot be a real site
// address, and whether it is one at all. WordPress concatenates this value to
// build every asset URL it emits, so an attacker who rewrites it makes the
// address it names load on every page without touching a single file.
//
// The test is the URL's shape, not its host. Hosting a site under a domain
// that is not served locally is ordinary -- sites move, staging lives
// elsewhere, and a theme demo keeps its vendor address -- so a host check
// would report dozens of healthy installs. Shape does not have that problem:
// a site address is an origin plus an optional subdirectory. A backslash, a
// query string, a fragment, a script for a path, or a non-web scheme cannot
// appear in one.
func siteURLPoisonReason(value string) (string, bool) {
// MySQL query rows preserve batch-mode escaping. Parse the stored bytes,
// not the escaped transport form, or control characters can look like an
// ordinary path and evade the shape checks below.
value = strings.TrimSpace(mysqlclient.BatchUnescape(value))
if value == "" {
return "", false
}
if strings.ContainsRune(value, '\\') {
return "address carries a backslash", true
}
u, err := url.Parse(value)
if err != nil {
return "value is not a URL", true
}
if scheme := strings.ToLower(u.Scheme); scheme != "http" && scheme != "https" {
return "scheme is not http or https", true
}
if u.Hostname() == "" {
return "no host", true
}
if port := u.Port(); port != "" {
n, err := strconv.Atoi(port)
if err != nil || n < 1 || n > 65535 {
return "port is outside the valid range", true
}
}
if u.RawQuery != "" || strings.Contains(value, "?") {
return "address carries a query string", true
}
if u.Fragment != "" || strings.Contains(value, "#") {
return "address carries a fragment", true
}
if isScriptPath(u.Path) {
return "address resolves to a script", true
}
return "", false
}
// isScriptPath reports whether a URL path's final segment names a client- or
// server-side script. A site address always ends at a directory.
func isScriptPath(path string) bool {
segment := path
if i := strings.LastIndex(segment, "/"); i >= 0 {
segment = segment[i+1:]
}
segment = strings.ToLower(segment)
if isExecutablePHPName(segment) {
return true
}
switch filepath.Ext(segment) {
case ".js", ".mjs", ".cjs",
".asp", ".aspx", ".ashx", ".asmx",
".jsp", ".jspx", ".cfm",
".cgi", ".pl", ".py", ".rb":
return true
default:
return false
}
}
// checkWPOptions checks for siteurl/home hijacking and injected JavaScript.
func checkWPOptions(user string, creds wpDBCreds, prefix string) []alert.Finding {
var findings []alert.Finding
foreignByOption := make(map[string]*alert.Finding, 2)
// Check siteurl and home for hijacking
query := fmt.Sprintf(
"SELECT option_name, option_value FROM %soptions WHERE option_name IN ('siteurl', 'home', 'admin_email') LIMIT 10",
prefix)
lines := runMySQLQuery(creds, query)
for _, line := range lines {
parts := strings.SplitN(line, "\t", 2)
if len(parts) != 2 {
continue
}
// WordPress's default option_name collation is case-insensitive, so a
// differently-cased row can satisfy get_option("siteurl") and this SQL
// query. Keep the Go-side security check consistent with that lookup.
optName := strings.ToLower(strings.TrimSpace(parts[0]))
optValue := strings.ToLower(parts[1])
if optName == "siteurl" || optName == "home" {
if strings.Contains(optValue, "eval(") || strings.Contains(optValue, "<script") {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "db_siteurl_hijack",
Message: fmt.Sprintf("WordPress %s contains malicious code (account: %s)", optName, user),
Details: dbContentFindingDetails(creds, prefix,
fmt.Sprintf("%s = %s", optName, truncateDB(parts[1], 200))),
DedupKey: dbContentDedupKey(user, creds, prefix,
fmt.Sprintf("%s = %s", optName, truncateDB(parts[1], 200))),
})
} else if reason, bad := siteURLPoisonReason(parts[1]); bad {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "db_siteurl_invalid",
Message: fmt.Sprintf("WordPress %s is not a site address (account: %s): %s", optName, user, reason),
Details: dbContentFindingDetails(creds, prefix,
fmt.Sprintf("%s = %s\nWordPress builds every asset URL from this value, so the address it names is loaded on every page.",
optName, truncateDB(parts[1], 200))),
DedupKey: dbContentDedupKey(user, creds, prefix,
"reason="+reason,
fmt.Sprintf("%s = %s\nWordPress builds every asset URL from this value, so the address it names is loaded on every page.",
optName, truncateDB(parts[1], 200))),
})
} else if foreign := foreignSiteURLFinding(user, creds, prefix, optName, parts[1]); foreign != nil {
// siteurl and home commonly hold the same address. Emit one stable
// condition per blog, preferring siteurl regardless of row order.
if current := foreignByOption[optName]; current == nil || foreign.Details < current.Details {
foreignByOption[optName] = foreign
}
}
}
}
for _, option := range []string{"siteurl", "home"} {
if foreign := foreignByOption[option]; foreign != nil {
findings = append(findings, *foreign)
break
}
}
// Path 1: External script URLs in any option — only flag non-safe domains.
query = fmt.Sprintf(
"SELECT option_name, option_value FROM %soptions WHERE option_value LIKE '%%<script%%src%%' LIMIT 20",
prefix)
lines = runMySQLQuery(creds, query)
firstSeen := storeFirstSeen(externalScriptSiteKey(creds.dbName, prefix))
for _, line := range lines {
parts := strings.SplitN(line, "\t", 2)
if len(parts) != 2 {
continue
}
optName := parts[0]
optValue := parts[1]
// Skip CSM backup options — they preserve the original malicious
// content for recovery and should not be re-detected/re-cleaned.
if strings.HasPrefix(optName, "csm_backup_") {
continue
}
maliciousURL := extractMaliciousScriptURL(optValue)
if maliciousURL == "" {
// No attacker marker. A loader on an unremarkable HTTPS host is
// still reported once, the first time it appears after the
// site's baseline.
findings = append(findings, newExternalScriptFindings(user, creds, prefix, optName, optValue, firstSeen)...)
continue
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "db_options_injection",
Message: fmt.Sprintf("Malicious script injection in wp_options '%s' (account: %s)", optName, user),
Details: dbContentFindingDetails(creds, prefix,
fmt.Sprintf("Option: %s", optName),
fmt.Sprintf("Malicious URL: %s", maliciousURL),
fmt.Sprintf("Content preview: %s", truncateDB(optValue, 200))),
DedupKey: dbContentDedupKey(user, creds, prefix,
fmt.Sprintf("Option: %s", optName),
fmt.Sprintf("Malicious URL: %s", maliciousURL),
fmt.Sprintf("Content preview: %s", truncateDB(optValue, 200))),
})
}
queryComplete := creds.queryState == nil || !creds.queryState.failed
if queryComplete {
if sdb := store.Global(); sdb != nil {
_ = sdb.FinishExternalScriptBaseline(externalScriptSiteKey(creds.dbName, prefix), time.Now())
}
}
// Path 1b: Plugin status options that WordPress renders as admin
// notices. These are queried by name because the generic script lookup
// above caps its result set and requires a src attribute, while an
// injection here may be inline. The option's identity is the verdict,
// so neither host reputation nor the first-seen baseline applies.
findings = append(findings, checkWPPluginNotices(user, creds, prefix)...)
// Path 2: Inline script/code injection in core WP options that should
// NEVER contain JavaScript (siteurl, home, blogname, blogdescription).
coreOpts := "siteurl', 'home', 'blogname', 'blogdescription', 'admin_email"
codePatterns := "<script"
query = fmt.Sprintf(
"SELECT option_name, LEFT(option_value, 500) FROM %soptions WHERE option_name IN ('%s') AND option_value LIKE '%%%s%%'",
prefix, coreOpts, codePatterns)
lines = runMySQLQuery(creds, query)
for _, line := range lines {
parts := strings.SplitN(line, "\t", 2)
if len(parts) != 2 {
continue
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "db_options_injection",
Message: fmt.Sprintf("Malicious content in core wp_option '%s' (account: %s)", parts[0], user),
Details: dbContentFindingDetails(creds, prefix,
fmt.Sprintf("Option: %s", parts[0]),
fmt.Sprintf("Content preview: %s", truncateDB(parts[1], 200))),
DedupKey: dbContentDedupKey(user, creds, prefix,
fmt.Sprintf("Option: %s", parts[0]),
fmt.Sprintf("Content preview: %s", truncateDB(parts[1], 200))),
})
}
return findings
}
// checkWPPosts checks post content for injected scripts and malware.
//
// Two classes of false positive are suppressed compared to a naive LIKE-
// based scan:
//
// - post_types used for plugin-managed storage (form submissions,
// revisions, templates, minified bundles) are excluded via the
// shared nonScannablePostTypes denylist. See dbscan_filters.go for
// the rationale and the full list.
//
// - Patterns that match too broadly at the SQL layer (the bare
// <script substring, and bare-word spam keywords like "cialis")
// are post-filtered in Go against word-boundary regexes and the
// known-safe-domain list. Legitimate analytics embeds and
// substring coincidences ("specialist" containing "cialis") no
// longer produce findings.
//
// The denylist is defense-in-depth: custom post_types created by a
// theme or plugin remain in scope, so attackers cannot evade by
// inventing a new post_type value.
func checkWPPosts(user string, creds wpDBCreds, prefix string) []alert.Finding {
var findings []alert.Finding
postTypeExcl := nonScannablePostTypesSQLList()
// Keep each pattern's independent LIMIT, but send the bounded selects as
// one UNION. On a host with hundreds of installs this avoids one database
// connection and round trip per signature without letting a noisy pattern
// consume every candidate slot for the others.
malwareSelects := make([]string, 0, len(dbMalwarePatterns))
for i, mp := range dbMalwarePatterns {
pattern := mysqlEscapeForLike(mp.pattern)
selectedContent := "'_content_not_required'"
if mp.requiresExternalScript {
selectedContent = "CONCAT_WS(CHAR(10), post_content, post_content_filtered)"
}
malwareSelects = append(malwareSelects, fmt.Sprintf(
"(SELECT %d AS pattern_index, ID, %s FROM %sposts WHERE post_status='publish' AND post_type NOT IN (%s) AND (post_content LIKE '%%%s%%' OR post_content_filtered LIKE '%%%s%%') LIMIT 20)",
i, selectedContent, prefix, postTypeExcl, pattern, pattern))
}
malwareRows := runMySQLQuery(creds, strings.Join(malwareSelects, " UNION ALL "))
confirmedByPattern := make([][]string, len(dbMalwarePatterns))
seenByPattern := make([]map[string]struct{}, len(dbMalwarePatterns))
for _, row := range malwareRows {
parts := strings.SplitN(row, "\t", 3)
if len(parts) != 3 {
continue
}
patternIndex, err := strconv.Atoi(parts[0])
if err != nil || patternIndex < 0 || patternIndex >= len(dbMalwarePatterns) {
continue
}
mp := dbMalwarePatterns[patternIndex]
content := mysqlclient.BatchUnescape(parts[2])
if mp.requiresExternalScript && !hasMaliciousExternalScriptInPost(content) {
continue
}
if seenByPattern[patternIndex] == nil {
seenByPattern[patternIndex] = make(map[string]struct{})
}
postID := strings.TrimSpace(parts[1])
if postID == "" {
continue
}
if _, duplicate := seenByPattern[patternIndex][postID]; duplicate {
continue
}
seenByPattern[patternIndex][postID] = struct{}{}
if len(confirmedByPattern[patternIndex]) < 5 {
confirmedByPattern[patternIndex] = append(confirmedByPattern[patternIndex], postID)
}
}
for i, confirmedIDs := range confirmedByPattern {
if len(confirmedIDs) == 0 {
continue
}
mp := dbMalwarePatterns[i]
findings = append(findings, alert.Finding{
Severity: mp.severity,
Check: "db_post_injection",
Message: fmt.Sprintf("WordPress posts contain %s (account: %s, %d posts)", mp.desc, user, len(confirmedIDs)),
Details: dbContentFindingDetails(creds, prefix,
fmt.Sprintf("Affected post IDs: %s", strings.Join(confirmedIDs, ", ")),
fmt.Sprintf("Pattern: %s", mp.pattern)),
DedupKey: dbContentDedupKey(user, creds, prefix,
fmt.Sprintf("Affected post IDs: %s", strings.Join(confirmedIDs, ", ")),
fmt.Sprintf("Pattern: %s", mp.pattern)),
})
}
// Spam keyword scan. Three-layer filter:
//
// 1. SQL LIKE as a fast server-side pre-filter (reduces rows).
// 2. Word-boundary regex in countCloakedSpamMatches (rejects
// substring false positives like "specialist" / "cialis").
// 3. SEO-context requirement in contentHasSpamContext: a keyword
// hit only counts when accompanied by CSS cloaking, an
// injection fingerprint, or an external anchor whose URL
// path contains the keyword. Bare prose mentions (industry
// verticals, advisor bios, product catalogs listing a
// pharmaceutical supply chain) do not fire.
//
// The context requirement catches the real attack pattern — hidden
// off-screen div with external commercial link — while leaving
// legitimate content silent. See spam_context.go for the full
// signal catalog.
spamSelects := make([]string, 0, len(dbSpamPatterns))
for i, sp := range dbSpamPatterns {
spamSelects = append(spamSelects, fmt.Sprintf(
"(SELECT %d AS pattern_index, ID, post_content FROM %sposts WHERE post_status='publish' AND post_type NOT IN (%s) AND post_content LIKE '%s' LIMIT %d)",
i, prefix, postTypeExcl, mysqlEscapeForLike(sp.likeFragment), dbSpamSampleLimit))
}
spamRows := runMySQLQuery(creds, strings.Join(spamSelects, " UNION ALL "))
spamContents := make([][]string, len(dbSpamPatterns))
spamSampled := make([]int, len(dbSpamPatterns))
for _, row := range spamRows {
parts := strings.SplitN(row, "\t", 3)
patternIndex, err := strconv.Atoi(parts[0])
if err != nil || patternIndex < 0 || patternIndex >= len(dbSpamPatterns) {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
spamSampled[patternIndex]++
if len(parts) != 3 {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
spamContents[patternIndex] = append(spamContents[patternIndex], mysqlclient.BatchUnescape(parts[2]))
}
for i, sp := range dbSpamPatterns {
n := countCloakedSpamMatches(sp, spamContents[i])
if n == 0 {
continue
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "db_spam_injection",
// The per-pattern LIMIT bounds the sample, so a full sample means
// the real figure may be larger. Reporting it as exact understates the
// scale, and scale is what decides whether an operator looks.
Message: fmt.Sprintf("WordPress posts contain cloaked spam keyword '%s' (%s posts, account: %s)",
sp.keyword, spamCountLabel(n, spamSampled[i] >= dbSpamSampleLimit), user),
Details: dbContentFindingDetails(creds, prefix),
// The pattern is the identity; the count is not. Spam grows between
// scans, and that is the same finding, not a new one.
DedupKey: dbContentDedupKey(user, creds, prefix, "keyword="+sp.keyword),
})
}
return findings
}
// dbContentDedupKey pins a database-content finding's identity to the database
// it was found in, the account using it, and what was found there. Callers
// include stable distinctions from Message as well as Details, excluding
// observation-only changes such as site age and document-root served state.
//
// The document-root note is deliberately excluded. It reports what the panel's
// domain map said during this scan, not anything the scan found in the
// database, and that map read fails transiently -- when it does the note
// disappears, the default Message+Details identity changes with it, and the
// store keeps a second copy of a finding that never changed.
func dbContentDedupKey(user string, creds wpDBCreds, prefix string, lines ...string) string {
identity := make([]byte, 0, 128)
appendField := func(value string) {
identity = binary.BigEndian.AppendUint64(identity, uint64(len(value)))
identity = append(identity, value...)
}
appendField(user)
appendField(creds.dbHost)
appendField(creds.dbName)
appendField(prefix)
for _, line := range lines {
appendField(line)
}
digest := sha256.Sum256(identity)
return fmt.Sprintf("db-content:%x", digest[:12])
}
// dbContentFindingDetails renders a database-content finding's details. The
// document-root note it adds is scan-time context, not part of what was found,
// so every caller supplies a DedupKey that excludes this note.
func dbContentFindingDetails(creds wpDBCreds, prefix string, lines ...string) string {
out := []string{
fmt.Sprintf("Database: %s", creds.dbName),
fmt.Sprintf("Table prefix: %s", prefix),
}
if note := docrootServedNote(creds.docrootServed); note != "" {
out = append(out, note)
}
out = append(out, lines...)
return strings.Join(out, "\n")
}
// wpInstallAdminGrace is how far after the first user registration an
// admin account still counts as created by the site install itself.
// Softaculous and similar installers register the initial admin (and any
// setup-wizard co-admins) within moments of creating the users table, so
// those accounts are the install, not a takeover. Anything later is a
// change to an existing site and stays alert-worthy.
const wpInstallAdminGrace = 15 * time.Minute
const wpRegisteredLayout = "2006-01-02 15:04:05"
func parseWPRegistered(value string) (time.Time, error) {
return time.Parse(wpRegisteredLayout, strings.TrimSpace(value))
}
// wpInstallEraAdmin reports whether an admin registration timestamp falls
// within the install grace of the site's first user registration. Fails
// open: unparseable timestamps never suppress.
func wpInstallEraAdmin(registered, firstRegistered string) bool {
reg, err := parseWPRegistered(registered)
if err != nil {
return false
}
first, err := parseWPRegistered(firstRegistered)
if err != nil {
return false
}
diff := reg.Sub(first)
return diff >= 0 && diff <= wpInstallAdminGrace
}
// checkWPUsers checks for rogue admin accounts created recently.
func checkWPUsers(user string, creds wpDBCreds, prefix string) []alert.Finding {
var findings []alert.Finding
// Find admin users created in the last 7 days. Missing registration
// timestamps stay in scope so the suppression fails open.
// MIN(user_registered) rides along as the install marker, but any NULL
// invalidates that marker. EXISTS keeps duplicate capability metadata
// from consuming the bounded result.
query := fmt.Sprintf(
"SELECT u.ID, u.user_login, u.user_email, u.user_registered, "+
"(SELECT CASE WHEN COUNT(*) <> COUNT(user_registered) "+
"THEN NULL ELSE MIN(user_registered) END FROM %susers) FROM %susers u "+
"WHERE EXISTS (SELECT 1 FROM %susermeta m "+
"WHERE m.user_id = u.ID AND m.meta_key = '%scapabilities' "+
"AND m.meta_value LIKE '%%administrator%%') "+
"AND (u.user_registered >= DATE_SUB(NOW(), INTERVAL 7 DAY) "+
"OR u.user_registered IS NULL "+
"OR CAST(u.user_registered AS CHAR) = '0000-00-00 00:00:00') "+
"ORDER BY (u.user_registered IS NULL OR "+
"CAST(u.user_registered AS CHAR) = '0000-00-00 00:00:00') DESC, "+
"u.user_registered DESC "+
"LIMIT 10",
prefix, prefix, prefix, prefix)
lines := runMySQLQuery(creds, query)
for _, line := range lines {
parts := strings.SplitN(line, "\t", 5)
if len(parts) < 3 {
continue
}
if wpInstallEraAdmin(safeGet(parts, 3), safeGet(parts, 4)) {
continue
}
registered := safeGet(parts, 3)
message := fmt.Sprintf("New WordPress admin account created in last 7 days: %s (account: %s)", parts[1], user)
if _, err := parseWPRegistered(registered); err != nil {
message = fmt.Sprintf("WordPress admin account has a missing or invalid registration timestamp: %s (account: %s)", parts[1], user)
}
severity := alert.Critical
details := fmt.Sprintf("Database: %s\nTable prefix: %s\nUser ID: %s\nLogin: %s\nEmail: %s\nRegistered: %s",
creds.dbName, prefix, parts[0], parts[1], parts[2], registered)
// A stable session pattern can support the legitimate developer or agency
// case, but session metadata is not authoritative. Downgrade -- never
// suppress -- so a forged session record cannot hide the account.
loginIPs := wpAdminLoginIPs(creds, prefix, parts[0])
distinct := uniqueStrings(loginIPs)
switch {
case len(loginIPs) == 0:
details += "\nSession-token IP evidence: none recorded."
case len(loginIPs) >= wpEstablishedLoginSessions && len(distinct) == 1:
severity = alert.Warning
details += fmt.Sprintf("\nSession-token IP evidence: %d stored sessions from a single IP (%s) -- consistent with a stable operator login pattern, but not proof the account is legitimate. Verify with the account owner before acting.",
len(loginIPs), distinct[0])
default:
details += fmt.Sprintf("\nSession-token IP evidence: %d stored sessions from %d IP(s): %s",
len(loginIPs), len(distinct), strings.Join(firstN(distinct, 5), ", "))
}
findings = append(findings, alert.Finding{
Severity: severity,
Check: "db_rogue_admin",
Message: message,
Details: details,
})
}
// Check for admin users with suspicious email patterns
query = fmt.Sprintf(
"SELECT u.user_login, u.user_email FROM %susers u "+
"INNER JOIN %susermeta m ON u.ID = m.user_id "+
"WHERE m.meta_key = '%scapabilities' AND m.meta_value LIKE '%%administrator%%' "+
"LIMIT 50",
prefix, prefix, prefix)
lines = runMySQLQuery(creds, query)
for _, line := range lines {
parts := strings.SplitN(line, "\t", 2)
if len(parts) != 2 {
continue
}
email := strings.ToLower(parts[1])
// Flag suspicious admin emails (disposable/temporary email domains)
suspiciousDomains := []string{
"tempmail", "guerrillamail", "mailinator", "throwaway",
"yopmail", "sharklasers", "trashmail", "maildrop",
}
for _, sd := range suspiciousDomains {
if strings.Contains(email, sd) {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "db_suspicious_admin_email",
Message: fmt.Sprintf("WordPress admin '%s' has disposable email (account: %s)", parts[0], user),
Details: fmt.Sprintf("Database: %s\nTable prefix: %s\nEmail: %s", creds.dbName, prefix, email),
})
break
}
}
}
return findings
}
func safeGet(parts []string, idx int) string {
if idx < len(parts) {
return parts[idx]
}
return ""
}
func truncateDB(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
}
// wpEstablishedLoginSessions is the number of stored login sessions from a
// single stable IP that is strong enough to downgrade the finding for review.
const wpEstablishedLoginSessions = 5
// parseSessionTokenIPs extracts structurally valid login source IPs from a
// WordPress session_tokens usermeta blob in session order.
func parseSessionTokenIPs(serialized string) []string {
var ips []string
for i := 0; i < len(serialized); {
key, next, ok := parsePHPSerializedStringAt(serialized, i)
if !ok {
if serialized[i] == 's' && i+1 < len(serialized) && serialized[i+1] == ':' {
return nil
}
i++
continue
}
if key != "ip" {
i = next
continue
}
value, afterValue, ok := parsePHPSerializedStringAt(serialized, next)
if !ok {
return nil
}
addr, err := netip.ParseAddr(value)
if err == nil {
ips = append(ips, addr.Unmap().String())
}
i = afterValue
}
return ips
}
// parsePHPSerializedStringAt reads one PHP s:<length>:"<value>"; token. Using
// the declared byte length prevents a forged token-looking fragment inside a
// session's attacker-controlled user-agent string from being treated as a key.
func parsePHPSerializedStringAt(serialized string, start int) (string, int, bool) {
if start < 0 || start+2 > len(serialized) ||
serialized[start] != 's' || serialized[start+1] != ':' {
return "", start, false
}
lengthStart := start + 2
lengthEnd := lengthStart
for lengthEnd < len(serialized) &&
serialized[lengthEnd] >= '0' && serialized[lengthEnd] <= '9' {
lengthEnd++
}
if lengthEnd == lengthStart || lengthEnd+2 > len(serialized) ||
serialized[lengthEnd] != ':' || serialized[lengthEnd+1] != '"' {
return "", start, false
}
valueLen, err := strconv.Atoi(serialized[lengthStart:lengthEnd])
if err != nil {
return "", start, false
}
valueStart := lengthEnd + 2
if valueLen > len(serialized)-valueStart {
return "", start, false
}
valueEnd := valueStart + valueLen
if valueEnd+2 > len(serialized) ||
serialized[valueEnd] != '"' || serialized[valueEnd+1] != ';' {
return "", start, false
}
return serialized[valueStart:valueEnd], valueEnd + 2, true
}
// wpAdminLoginIPs returns source IPs encoded in an admin's stored login
// sessions. The caller treats this forgeable metadata only as downgrade
// evidence and never uses it to suppress the finding.
func wpAdminLoginIPs(creds wpDBCreds, prefix, userID string) []string {
if !wpNumericID(userID) {
return nil
}
q := fmt.Sprintf(
"SELECT meta_value FROM %susermeta WHERE user_id = %s AND meta_key = 'session_tokens' LIMIT 1",
prefix, userID)
serialized := mysqlclient.BatchUnescape(strings.Join(runMySQLQuery(creds, q), ""))
return parseSessionTokenIPs(serialized)
}
// wpNumericID guards the user id (a prior query row value) before it is
// interpolated into the session lookup.
func wpNumericID(s string) bool {
if s == "" {
return false
}
for _, r := range s {
if r < '0' || r > '9' {
return false
}
}
return true
}
// firstN caps a slice for display in finding details.
func firstN(in []string, n int) []string {
if len(in) <= n {
return in
}
return in[:n]
}
// docrootServedNote states whether this install is reachable today. Both
// answers change how a finding should be queued: a dormant install is not
// serving anyone right now, and a served one is. Silence when the panel's map
// could not be read -- claiming either would be a guess.
func docrootServedNote(state servedState) string {
switch state {
case servedByPanel:
return "Document root: served by the panel, so this is live now."
case notServed:
return "Document root: not currently served. The database is still live " +
"and the content publishes again the moment a domain is pointed here."
default:
return ""
}
}
package checks
import (
"encoding/base64"
"encoding/hex"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/alert"
)
// Cloaking infrastructure kept in the options table.
//
// Two shapes, both from doorway kits found on one host:
//
// - Configuration stored under a digest of the site's own hostname, so the
// row is unfindable without knowing which site you are looking at, with a
// base64 layer over the serialized array so its contents do not appear in
// any search of the table.
// - One rewrite rule per doorway cluster, routing a numbered sitemap
// straight into a matching numbered feed, to hand crawlers the generated
// pages without them appearing in the site's real sitemap.
//
// Neither is code execution. Both are the scaffolding a doorway network needs,
// and both survive the deletion of every spam post.
const (
// maxCloakOptionBytes bounds one option value. Configuration blobs are
// normally small, but rewrite_rules can exceed this on a large site. The
// query also returns the original byte length so a bounded read can never
// be mistaken for a complete value.
maxCloakOptionBytes = 256 * 1024
// maxCloakOptionRows bounds the digest-named candidates read.
maxCloakOptionRows = 50
// maxCloakSamplesShown bounds what the finding names.
maxCloakSamplesShown = 10
)
// digestOptionName matches an option named by a 32-character hex digest and
// nothing else. WordPress core and plugins name options after what they hold.
var digestOptionName = regexp.MustCompile(`^[0-9a-f]{32}$`)
// numberedSitemapRoute and numberedSitemapFeed are the two halves of the
// doorway routing. Reporting needs both with the same number: real sitemap
// plugins add rewrite rules too, but none of them route sitemap<N>.xml into a
// feed named xmlsitemap<N>.
var (
numberedSitemapRoute = regexp.MustCompile(`(?i)(?:^|\^|/)sitemap([0-9]+)\\?\.xml(?:\$|$)`)
numberedSitemapFeed = regexp.MustCompile(`(?i)(?:^|[?&])feed=xmlsitemap([0-9]+)(?:&|$)`)
)
// hostnameKeyedOption reports whether an option is cloak configuration keyed
// by a digest, returning the decoded size. Both halves are required: a plugin
// may hash a cache key, and base64 alone is ordinary.
func hostnameKeyedOption(name, value string) (int, bool) {
if !digestOptionName.MatchString(strings.ToLower(name)) {
return 0, false
}
decoded, ok := decodeBase64Payload(value)
if !ok || !isPHPSerializedArray(decoded) {
return 0, false
}
return len(decoded), true
}
// decodeBase64Payload decodes a stored base64 value. Kits wrap the stored text
// at arbitrary widths, so whitespace is removed before decoding, and padding is
// not always present.
func decodeBase64Payload(value string) ([]byte, bool) {
if value == "" || len(value) > maxCloakOptionBytes {
return nil, false
}
compactLen := 0
for i := 0; i < len(value); i++ {
c := value[i]
switch c {
case ' ', '\t', '\n', '\r', '\f', '\v':
continue
}
if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
(c >= '0' && c <= '9') || c == '+' || c == '/' || c == '=' {
compactLen++
continue
}
// Reject base64url, NUL, non-ASCII, and other invalid bytes
// before allocating a decoder-sized output buffer.
return nil, false
}
if compactLen == 0 {
return nil, false
}
compact := make([]byte, 0, compactLen)
for i := 0; i < len(value); i++ {
switch value[i] {
case ' ', '\t', '\n', '\r', '\f', '\v':
continue
default:
compact = append(compact, value[i])
}
}
encoding := base64.StdEncoding
switch len(compact) % 4 {
case 0:
// Padded standard base64 and unpadded complete quanta both land
// here and are accepted by StdEncoding.
case 2, 3:
encoding = base64.RawStdEncoding
default:
return nil, false
}
decoded := make([]byte, encoding.DecodedLen(len(compact)))
n, err := encoding.Decode(decoded, compact)
if err != nil {
// Decode may have written a valid prefix. Never inspect it after an
// error or trailing garbage could fabricate the serialized marker.
return nil, false
}
return decoded[:n], true
}
type phpSerializedFrame struct {
remaining int
closing byte
hasKeys bool
}
// isPHPSerializedArray validates one complete PHP serialization without
// materializing it. Checking the full grammar prevents a decoded prefix such
// as "a:1:{" (including one followed by NUL or garbage) from becoming a
// finding. The explicit frame stack also keeps hostile nesting off Go's call
// stack.
func isPHPSerializedArray(data []byte) bool {
if len(data) == 0 || data[0] != 'a' {
return false
}
frames := []phpSerializedFrame{{remaining: 1}}
pos := 0
for len(frames) > 0 {
frameIndex := len(frames) - 1
frame := frames[frameIndex]
if frame.remaining == 0 {
if frame.closing == 0 {
return len(frames) == 1 && pos == len(data)
}
if pos >= len(data) || data[pos] != frame.closing {
return false
}
pos++
frames = frames[:frameIndex]
continue
}
if pos >= len(data) {
return false
}
expectingKey := frame.hasKeys && frame.remaining%2 == 0
if expectingKey &&
data[pos] != 'i' && data[pos] != 's' {
return false
}
frames[frameIndex].remaining--
next, child, hasChild, ok := consumePHPSerializedValue(data, pos)
if !ok {
return false
}
pos = next
if hasChild {
frames = append(frames, child)
}
}
return false
}
func consumePHPSerializedValue(data []byte, pos int) (int, phpSerializedFrame, bool, bool) {
var noChild phpSerializedFrame
if pos >= len(data) {
return pos, noChild, false, false
}
switch data[pos] {
case 'N':
if pos+2 <= len(data) && string(data[pos:pos+2]) == "N;" {
return pos + 2, noChild, false, true
}
case 'b':
if pos+4 <= len(data) && data[pos+1] == ':' &&
(data[pos+2] == '0' || data[pos+2] == '1') && data[pos+3] == ';' {
return pos + 4, noChild, false, true
}
case 'i':
if pos+2 > len(data) || data[pos+1] != ':' {
break
}
next, ok := consumePHPSerializedInteger(data, pos+2)
return next, noChild, false, ok
case 'd':
if pos+2 > len(data) || data[pos+1] != ':' {
break
}
end := pos + 2
for end < len(data) && data[end] != ';' {
end++
}
if end == len(data) || end == pos+2 || end-(pos+2) > 64 {
break
}
number := string(data[pos+2 : end])
if number == "INF" || number == "-INF" || number == "NAN" {
return end + 1, noChild, false, true
}
if _, err := strconv.ParseFloat(number, 64); err == nil {
return end + 1, noChild, false, true
}
case 's', 'E':
if pos+2 > len(data) || data[pos+1] != ':' {
break
}
next, ok := consumePHPSerializedBytes(data, pos+2, ';')
return next, noChild, false, ok
case 'R', 'r':
if pos+2 > len(data) || data[pos+1] != ':' {
break
}
ref, next, ok := parsePHPSerializedUint(data, pos+2, ';')
return next, noChild, false, ok && ref > 0
case 'a':
if pos+2 > len(data) || data[pos+1] != ':' {
break
}
count, next, ok := parsePHPSerializedUint(data, pos+2, ':')
if !ok || count > len(data) || count > int(^uint(0)>>1)/2 ||
next >= len(data) || data[next] != '{' {
break
}
return next + 1, phpSerializedFrame{
remaining: count * 2,
closing: '}',
hasKeys: true,
}, true, true
case 'O':
if pos+2 > len(data) || data[pos+1] != ':' {
break
}
next, ok := consumePHPSerializedBytes(data, pos+2, ':')
if !ok {
break
}
count, next, ok := parsePHPSerializedUint(data, next, ':')
if !ok || count > len(data) || count > int(^uint(0)>>1)/2 ||
next >= len(data) || data[next] != '{' {
break
}
return next + 1, phpSerializedFrame{
remaining: count * 2,
closing: '}',
// __serialize() may return integer as well as string keys.
hasKeys: true,
}, true, true
case 'C':
if pos+2 > len(data) || data[pos+1] != ':' {
break
}
next, ok := consumePHPSerializedBytes(data, pos+2, ':')
if !ok {
break
}
payloadLen, next, ok := parsePHPSerializedUint(data, next, ':')
if !ok || next >= len(data) || data[next] != '{' ||
payloadLen > len(data)-(next+1) {
break
}
next += 1 + payloadLen
if next < len(data) && data[next] == '}' {
return next + 1, noChild, false, true
}
}
return pos, noChild, false, false
}
func consumePHPSerializedInteger(data []byte, pos int) (int, bool) {
start := pos
if pos < len(data) && data[pos] == '-' {
pos++
}
digitStart := pos
for pos < len(data) && data[pos] >= '0' && data[pos] <= '9' {
pos++
}
if pos == digitStart || pos >= len(data) || data[pos] != ';' || pos-start > 20 {
return start, false
}
if _, err := strconv.ParseInt(string(data[start:pos]), 10, 64); err != nil {
return start, false
}
return pos + 1, true
}
// consumePHPSerializedBytes consumes <length>:"<raw bytes>"<terminator>.
func consumePHPSerializedBytes(data []byte, pos int, terminator byte) (int, bool) {
length, next, ok := parsePHPSerializedUint(data, pos, ':')
if !ok || next >= len(data) || data[next] != '"' || length > len(data)-(next+1) {
return pos, false
}
next += 1 + length
if next+1 >= len(data) || data[next] != '"' || data[next+1] != terminator {
return pos, false
}
return next + 2, true
}
func parsePHPSerializedUint(data []byte, pos int, delimiter byte) (int, int, bool) {
start := pos
n := 0
maxInt := int(^uint(0) >> 1)
for pos < len(data) && data[pos] >= '0' && data[pos] <= '9' {
digit := int(data[pos] - '0')
if n > (maxInt-digit)/10 {
return 0, start, false
}
n = n*10 + digit
pos++
}
if pos == start || pos >= len(data) || data[pos] != delimiter {
return 0, start, false
}
return n, pos + 1, true
}
// doorwaySitemapRoutes returns the canonical cluster numbers routed from a
// numbered sitemap key into the matching numbered feed value. Requiring both
// halves in the same serialized rewrite-rule pair avoids correlating unrelated
// rules that merely occur in the same option.
func doorwaySitemapRoutes(rewriteRules string) []string {
routes, _ := doorwaySitemapRoutesChecked(rewriteRules)
return routes
}
func doorwaySitemapRoutesChecked(rewriteRules string) ([]string, bool) {
data := []byte(rewriteRules)
if len(data) < 6 || data[0] != 'a' || data[1] != ':' {
return nil, false
}
count, pos, ok := parsePHPSerializedUint(data, 2, ':')
if !ok || count > len(data) || pos >= len(data) || data[pos] != '{' {
return nil, false
}
pos++
routes := make(map[string]struct{})
for i := 0; i < count; i++ {
key, next, ok := parsePHPSerializedStringAt(rewriteRules, pos)
if !ok {
return nil, false
}
value, nextValue, ok := parsePHPSerializedStringAt(rewriteRules, next)
if !ok {
return nil, false
}
pos = nextValue
keyNumbers := cloakClusterNumbers(numberedSitemapRoute, key)
valueNumbers := cloakClusterNumbers(numberedSitemapFeed, value)
for number := range keyNumbers {
if _, paired := valueNumbers[number]; paired {
routes[number] = struct{}{}
}
}
}
if pos >= len(data) || data[pos] != '}' || pos+1 != len(data) {
return nil, false
}
out := make([]string, 0, len(routes))
for number := range routes {
out = append(out, number)
}
sort.Slice(out, func(i, j int) bool {
if len(out[i]) != len(out[j]) {
return len(out[i]) < len(out[j])
}
return out[i] < out[j]
})
return out, true
}
func cloakClusterNumbers(pattern *regexp.Regexp, value string) map[string]struct{} {
numbers := make(map[string]struct{})
for _, match := range pattern.FindAllStringSubmatch(value, -1) {
if len(match) < 2 || len(match[1]) > 10 {
continue
}
number := strings.TrimLeft(match[1], "0")
if number == "" {
number = "0"
}
numbers[number] = struct{}{}
}
return numbers
}
type cloakOptionRow struct {
kind string
name string
value []byte
}
func parseCloakOptionRow(line string) (cloakOptionRow, bool, bool) {
var row cloakOptionRow
parts := strings.SplitN(strings.TrimRight(line, "\r\n"), "\t", 4)
if len(parts) != 4 {
return row, false, false
}
valueBytes, err := strconv.ParseInt(strings.TrimSpace(parts[2]), 10, 64)
if err != nil || valueBytes < 0 {
return row, false, false
}
encoded := strings.TrimSpace(parts[3])
if !strings.HasPrefix(encoded, "x") {
return row, false, false
}
value, err := hex.DecodeString(encoded[1:])
if err != nil {
return row, false, false
}
expected := valueBytes
if expected > maxCloakOptionBytes {
expected = maxCloakOptionBytes
}
if int64(len(value)) != expected {
return row, false, false
}
return cloakOptionRow{
kind: strings.TrimSpace(parts[0]),
name: parts[1],
value: value,
}, true, valueBytes <= maxCloakOptionBytes
}
// checkWPCloakConfig reports doorway scaffolding kept in the options table.
func checkWPCloakConfig(user string, creds wpDBCreds, prefix string) []alert.Finding {
query := fmt.Sprintf(
"(SELECT 'opt' AS kind, option_name, OCTET_LENGTH(option_value), "+
"CONCAT('x', HEX(LEFT(CAST(option_value AS BINARY), %d))) FROM %soptions "+
"WHERE autoload IN ('yes', 'on', 'auto-on', 'auto') "+
"AND OCTET_LENGTH(option_name) = 32 AND UNHEX(option_name) IS NOT NULL "+
"ORDER BY option_name LIMIT %d) UNION ALL "+
"(SELECT 'rules', option_name, OCTET_LENGTH(option_value), "+
"CONCAT('x', HEX(LEFT(CAST(option_value AS BINARY), %d))) FROM %soptions "+
"WHERE option_name = 'rewrite_rules' LIMIT 1)",
maxCloakOptionBytes, prefix, maxCloakOptionRows+1, maxCloakOptionBytes, prefix)
var keyed []string
routeSet := make(map[string]struct{})
candidates := 0
candidatesTruncated := false
for _, line := range runMySQLQuery(creds, query) {
row, ok, complete := parseCloakOptionRow(line)
if !ok {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
if !complete {
markCheckIncomplete(creds.queryCtx, "db_content")
}
switch row.kind {
case "opt":
candidates++
if candidates > maxCloakOptionRows {
candidatesTruncated = true
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
if complete {
if size, matched := hostnameKeyedOption(row.name, string(row.value)); matched {
keyed = append(keyed, fmt.Sprintf("%s (%d bytes decoded)", row.name, size))
}
}
case "rules":
if complete {
routes, parsed := doorwaySitemapRoutesChecked(string(row.value))
if !parsed {
continue
}
for _, route := range routes {
routeSet[route] = struct{}{}
}
}
default:
markCheckIncomplete(creds.queryCtx, "db_content")
}
}
sort.Strings(keyed)
routes := make([]string, 0, len(routeSet))
for route := range routeSet {
routes = append(routes, route)
}
sort.Slice(routes, func(i, j int) bool {
if len(routes[i]) != len(routes[j]) {
return len(routes[i]) < len(routes[j])
}
return routes[i] < routes[j]
})
var findings []alert.Finding
if len(keyed) > 0 {
count := strconv.Itoa(len(keyed))
if candidatesTruncated {
count = "at least " + count
}
noun, verb, hold := "options", "are", "hold"
if len(keyed) == 1 {
noun, verb, hold = "option", "is", "holds"
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "db_hostname_keyed_option",
Message: fmt.Sprintf("%s autoloaded WordPress %s %s named by digest and %s encoded data (account: %s)",
count, noun, verb, hold, user),
Details: dbContentFindingDetails(creds, prefix,
"An option named after a digest cannot be found without already knowing "+
"the key, and the base64 layer keeps its contents out of any search of "+
"the table. Cloak kits key that digest to the site's own hostname so one "+
"payload serves many sites. The row is autoloaded, so it is read on every request.",
cloakSample("Options", keyed)),
DedupKey: dbContentDedupKey(user, creds, prefix,
"An option named after a digest cannot be found without already knowing "+
"the key, and the base64 layer keeps its contents out of any search of "+
"the table. Cloak kits key that digest to the site's own hostname so one "+
"payload serves many sites. The row is autoloaded, so it is read on every request.",
cloakSample("Options", keyed)),
})
}
if len(routes) > 0 {
noun, verb := "routes", "feed"
if len(routes) == 1 {
noun, verb = "route", "feeds"
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "db_doorway_sitemap_routes",
Message: fmt.Sprintf("%d numbered sitemap %s %s generated pages to crawlers (account: %s)",
len(routes), noun, verb, user),
Details: dbContentFindingDetails(creds, prefix,
"Each rule routes sitemap<N>.xml straight into a matching feed, one per "+
"doorway cluster, so crawlers are handed the generated pages without them "+
"appearing in the site's real sitemap. Sitemap plugins add rewrite rules "+
"too, but none of them pair a numbered sitemap with a feed of the same number.",
cloakSample("Clusters", routes)),
DedupKey: dbContentDedupKey(user, creds, prefix,
"Each rule routes sitemap<N>.xml straight into a matching feed, one per "+
"doorway cluster, so crawlers are handed the generated pages without them "+
"appearing in the site's real sitemap. Sitemap plugins add rewrite rules "+
"too, but none of them pair a numbered sitemap with a feed of the same number.",
cloakSample("Clusters", routes)),
})
}
return findings
}
func cloakSample(label string, values []string) string {
shown := values
if len(shown) > maxCloakSamplesShown {
shown = shown[:maxCloakSamplesShown]
label += fmt.Sprintf(" (showing %d of %d)", len(shown), len(values))
}
return label + ": " + strings.Join(shown, ", ")
}
package checks
import (
"context"
"fmt"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// cmsScanRowLimit bounds every non-WordPress CMS content, settings and
// admin query. The WordPress scanner caps each of its selects; these
// adapters used to pull whole tables through the mysql client.
const cmsScanRowLimit = 200
// One extra row distinguishes a complete result at the cap from a truncated
// query. Failed or partial queries must not establish an administrator baseline.
func runCMSQuery(creds wpDBCreds, query string) ([]string, bool) {
if creds.queryState == nil {
creds.queryState = new(dbQueryState)
}
markIncomplete := func(halt bool) {
creds.queryState.failed = true
creds.queryState.halted = creds.queryState.halted || halt
markCheckIncomplete(creds.queryCtx, creds.queryCheck())
}
if creds.queryCtx != nil && creds.queryCtx.Err() != nil {
markIncomplete(true)
return nil, false
}
rows := runMySQLQuery(creds, fmt.Sprintf("%s LIMIT %d", query, cmsScanRowLimit+1))
if creds.queryCtx != nil && creds.queryCtx.Err() != nil {
markIncomplete(true)
return nil, false
}
if len(rows) > cmsScanRowLimit {
rows = rows[:cmsScanRowLimit]
// Overflow leaves the connection usable. Do not let one large
// result hide threats in the installation's other tables.
markIncomplete(false)
}
return rows, !creds.queryState.failed
}
// cmsDiscover globs every pattern under every account root and returns the
// unique matches. Installs live under public_html and under addon-domain
// document roots (<home>/<domain>/...), so callers pass both shapes.
func cmsDiscover(ctx context.Context, owner string, patterns ...string) []string {
var out []string
for _, p := range patterns {
for _, root := range accountHomeRoots() {
if ctx.Err() != nil {
markCheckIncomplete(ctx, owner)
return uniqueStrings(out)
}
// One root's matches cannot cover another root's discovery error.
matches, err := osFS.Glob(filepath.Join(root, p))
if err != nil {
markCheckIncomplete(ctx, owner)
}
out = append(out, matches...)
}
}
return uniqueStrings(out)
}
// cmsDropSuspended removes installs owned by a suspended account before their
// configuration is read or counted against the scan file limit. The account's
// database user is locked, so scanning it only leaves the check incomplete.
func cmsDropSuspended(paths []string) []string {
kept := paths[:0:0]
for _, path := range paths {
if owner, owned := installOwner(path); owned && accountSuspended(owner) {
continue
}
kept = append(kept, path)
}
return kept
}
func rankCMSConfigs(ctx context.Context, owner string, paths []string, maxFiles int) []string {
ranked := rankPathsByMtimeDesc(ctx, paths, maxFiles)
if len(ranked) < len(paths) || ctx.Err() != nil {
markCheckIncomplete(ctx, owner)
}
return ranked
}
// scanCMSInstalls runs scan on every install of one CMS that patterns
// discover, skipping suspended accounts. Installs are ranked by mtime desc so
// recently touched ones are processed first when the check timeout cuts
// iteration short.
func scanCMSInstalls(ctx context.Context, cfg *config.Config, store *state.Store, check string,
scan func(context.Context, string, *state.Store) []alert.Finding, patterns ...string) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
var findings []alert.Finding
configs := cmsDropSuspended(cmsDiscover(ctx, check, patterns...))
if len(configs) == 0 {
return nil
}
for _, path := range rankCMSConfigs(ctx, check, configs, accountScanMaxFiles(ctx, cfg)) {
if ctx.Err() != nil {
return findings
}
findings = append(findings, scan(ctx, path, store)...)
}
return findings
}
// cmsAdminFindings reports CMS administrator rows. With a store, the first
// complete pass records every admin id for the install and stays quiet;
// from then on only an id not seen before is reported, once, as a High
// finding. Without a store (ad-hoc runs, tests) it keeps the historical
// per-row visibility Warning. describe renders the message tail and the
// details for one row's tab-separated fields; fields[0] is the id.
func cmsAdminFindings(store *state.Store, cms, check, account string, creds wpDBCreds, rows []string, complete bool, describe func(fields []string) (message, details string)) []alert.Finding {
return cmsAdminFindingsEscalated(store, cms, check, account, creds, rows, complete, describe, nil)
}
// cmsAdminEscalation may raise a newly seen administrator's finding above
// High. event names why, in the message and details, and gives the finding
// its own identity so it is not filtered as a repeat of an earlier one.
type cmsAdminEscalation func(fields []string) (sev alert.Severity, event string, ok bool)
const cmsAdminSeenPrefix = "_cmsadmin:v2:"
// cmsAdminSeenKey is the store key recording that an administrator id has
// been seen in this installation.
func cmsAdminSeenKey(account string, creds wpDBCreds, cms, id string) string {
return cmsAdminSeenPrefix + dbContentDedupKey(account, creds, creds.tablePrefix, "cms-admin="+cms, "id="+id)
}
// cmsAdminFindingsEscalated is cmsAdminFindings with an optional escalation
// applied to new administrators on the store path. Without a store, rows stay
// one Warning each.
func cmsAdminFindingsEscalated(store *state.Store, cms, check, account string, creds wpDBCreds, rows []string, complete bool,
describe func(fields []string) (message, details string), escalate cmsAdminEscalation) []alert.Finding {
for _, row := range rows {
id, _, _ := strings.Cut(row, "\t")
if !isAllDigits(id) {
complete = false
markCheckIncomplete(creds.queryCtx, creds.queryCheck())
}
}
var findings []alert.Finding
// Account-wide keys cannot establish which installation supplied an id.
// Each database and prefix gets a fresh baseline after the key migration.
siteKey := dbContentDedupKey(account, creds, creds.tablePrefix, "cms-admin="+cms)
baselineKey := "_cmsadmin_baseline:v2:" + siteKey
baselined := false
if store != nil {
_, baselined = store.GetRaw(baselineKey)
}
seenIDs := make(map[string]bool, len(rows))
for _, row := range rows {
fields := strings.Split(row, "\t")
if !isAllDigits(fields[0]) || seenIDs[fields[0]] {
continue
}
seenIDs[fields[0]] = true
message, details := describe(fields)
details = dbContentFindingDetails(creds, creds.tablePrefix,
"Database host: "+creds.dbHost, details)
dedupKey := dbContentDedupKey(account, creds, creds.tablePrefix, "cms-admin="+cms, "id="+fields[0])
if store == nil {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: check,
Message: message,
Details: details,
DedupKey: dedupKey,
})
continue
}
key := cmsAdminSeenPrefix + dedupKey
if _, seen := store.GetRaw(key); seen {
continue
}
if complete {
store.SetRaw(key, "seen")
}
if !baselined {
continue
}
finding := alert.Finding{
Severity: alert.High,
Check: check,
Message: "New " + message,
Details: details + "\nThis administrator was not present when the install was baselined.",
DedupKey: dedupKey,
}
if escalate != nil {
if sev, event, ok := escalate(fields); ok {
finding.Severity = sev
finding.Message += " (" + event + ")"
finding.Details += "\nEvent: " + event
finding.DedupKey = dbContentDedupKey(account, creds, creds.tablePrefix, "cms-admin="+cms, "id="+fields[0], "event="+event)
}
}
findings = append(findings, finding)
}
if store != nil && !baselined && complete {
store.SetRaw(baselineKey, "1")
}
return findings
}
package checks
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mysqlclient"
"github.com/pidginhost/csm/internal/state"
)
// Drupal database content scanner.
//
// v1 covers Drupal 8 and later (the unified-table layout: config /
// node_field_data / users_field_data). Drupal 7 uses a different
// schema (variable / node / users) and reached EOL in January 2025;
// scanning it lands as a follow-up if any operator reports D7 sites
// still in production.
//
// Discovery: glob /home/*/public_html/sites/default/settings.php
// and confirm core/lib/Drupal.php exists in the site root --
// that file is the canonical D8+ marker (D7 has bootstrap.inc /
// modules/ but no core/ directory).
//
// Credentials: parsed by regex over the canonical $databases
// array literal. Drupal allows both array() and short [] syntax;
// the regex accepts either.
//
// Scanned tables (all unprefixed -- D8+ does not use a per-site
// table prefix in standard installs):
//
// config name + data; data is a
// serialized PHP array carrying
// site name, slogan, theme, etc.
// Common hijack target.
// node_revision__body entity_id + body_value; the
// actual article body. Scanned for
// any pattern in dbMalwarePatterns
// with the same external-script
// post-filter the WP and Joomla
// scanners use.
// users_field_data user table; joined with
// user__roles on entity_id = uid. Rows where
// roles_target_id = 'administrator'
// surface as drupal_admin_injection.
//
// Three new finding categories with CMS-explicit names:
// drupal_settings_injection, drupal_content_injection,
// drupal_admin_injection.
// drupalAdminRoleID is the canonical role identifier for Drupal
// site administrators in vanilla D8+. Operators on hardened
// installs may have renumbered or renamed; v1 narrows to this.
const drupalAdminRoleID = "administrator"
// drupalSettingsRe pulls credentials out of the $databases array
// literal. Each field is matched independently rather than
// trying to parse the array structure -- attackers occasionally
// reorder keys, and a key-only regex ignores layout differences.
var (
drupalDBNameRe = regexp.MustCompile(`'database'\s*=>\s*['"]([^'"]+)['"]`)
drupalDBUserRe = regexp.MustCompile(`'username'\s*=>\s*['"]([^'"]+)['"]`)
drupalDBPassRe = regexp.MustCompile(`'password'\s*=>\s*['"]([^'"]+)['"]`)
drupalDBHostRe = regexp.MustCompile(`'host'\s*=>\s*['"]([^'"]+)['"]`)
)
// drupalCreds carries the parsed connection details. Mirrors the
// jConfigCreds shape so existing helpers (runMySQLQuery,
// asWPDBCreds) work uniformly.
type drupalCreds struct {
// ctx ties every query for this install to the runner's deadline.
ctx context.Context
dbName string
dbUser string
dbPass string
dbHost string
path string
queryState *dbQueryState
}
func (c drupalCreds) asWPDBCreds() wpDBCreds {
return wpDBCreds{
dbName: c.dbName,
dbUser: c.dbUser,
dbPass: c.dbPass,
dbHost: c.dbHost,
queryCtx: c.ctx,
queryOwner: "db_content_drupal",
queryState: c.queryState,
}
}
// CheckDrupalContent discovers Drupal 8+ sites and scans the three
// canonical attacker-touched tables. Mirrors CheckJoomlaContent
// without sharing code -- the credential layout and table set are
// distinct enough that a generic dispatcher would be more
// abstraction than a 4-CMS pipeline calls for.
// scanDrupalInstall scans one discovered install and stamps its findings
// with the owner resolved from the settings path. The display label stays
// as before; an install outside every account root is not stamped.
func scanDrupalInstall(ctx context.Context, path string, store *state.Store) []alert.Finding {
// public_html is three dirs up from sites/default/settings.php.
publicHTML := filepath.Dir(filepath.Dir(filepath.Dir(path)))
matched, err := looksLikeDrupal8Plus(publicHTML)
if err != nil {
markCheckIncomplete(ctx, "db_content_drupal")
return nil
}
if !matched {
return nil
}
// /home/<account> is one level above public_html.
account := extractUser(filepath.Dir(publicHTML))
creds, err := parseDrupalSettings(ctx, path)
if err != nil || creds.dbName == "" || creds.dbUser == "" {
markCheckIncomplete(ctx, "db_content_drupal")
return nil
}
creds.ctx = ctx
creds.queryState = new(dbQueryState)
var findings []alert.Finding
findings = append(findings, scanDrupalConfig(account, creds)...)
findings = append(findings, scanDrupalContent(account, creds)...)
findings = append(findings, scanDrupalAdmins(store, account, creds)...)
if owner, ok := installOwner(path); ok {
findings = stampTenantIDIfEmpty(findings, owner)
}
return findings
}
func CheckDrupalContent(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
return scanCMSInstalls(ctx, cfg, store, "db_content_drupal", scanDrupalInstall,
"*/public_html/sites/default/settings.php", "*/*/sites/default/settings.php")
}
// The marker distinguishes D8+ from D7 without reading its contents. A
// symlink or special file cannot establish the installation's version.
func looksLikeDrupal8Plus(publicHTML string) (bool, error) {
marker := filepath.Join(publicHTML, "core", "lib", "Drupal.php")
info, err := osFS.Lstat(marker)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
if err != nil {
return false, err
}
if !info.Mode().IsRegular() {
return false, errNonRegularFile
}
return true, nil
}
// parseDrupalSettings reads settings.php and returns the database
// credentials from the $databases['default']['default'] entry. If
// settings.php uses split-DB or per-environment overrides, only
// the first 'default' connection is reported -- the rest are
// followed by the same regex on subsequent calls.
func parseDrupalSettings(ctx context.Context, path string) (drupalCreds, error) {
creds := drupalCreds{path: path}
data, err := readCMSConfig(ctx, path)
if err != nil {
return creds, err
}
body := string(data)
if m := drupalDBNameRe.FindStringSubmatch(body); m != nil {
creds.dbName = m[1]
}
if m := drupalDBUserRe.FindStringSubmatch(body); m != nil {
creds.dbUser = m[1]
}
if m := drupalDBPassRe.FindStringSubmatch(body); m != nil {
creds.dbPass = m[1]
}
if m := drupalDBHostRe.FindStringSubmatch(body); m != nil {
creds.dbHost = m[1]
}
if creds.dbHost == "" {
creds.dbHost = "localhost"
}
return creds, nil
}
// scanDrupalConfig pulls rows from the config table whose data
// blob matches any malware pattern, then refines via
// classifyMalwareRow (strict / config-storage variant).
func scanDrupalConfig(account string, creds drupalCreds) []alert.Finding {
query := fmt.Sprintf(
"SELECT name, data FROM config WHERE %s",
paramsLikeClause("data"))
rows, _ := runCMSQuery(creds.asWPDBCreds(), query)
var findings []alert.Finding
for _, row := range rows {
name, body := splitTabRow(row)
if name == "" {
continue
}
sev, desc, ok := classifyMalwareRow(mysqlclient.BatchUnescape(body), false)
if !ok {
continue
}
findings = append(findings, alert.Finding{
Severity: sev,
Check: "drupal_settings_injection",
Message: fmt.Sprintf("Drupal config injection on %s: %s (%s)", account, name, desc),
Details: fmt.Sprintf("Account: %s\nConfig name: %s\nMatch: %s", account, name, desc),
})
}
return findings
}
// scanDrupalContent walks node_revision__body for malware-pattern
// matches in article bodies. The looser post-filter
// (hasMaliciousExternalScriptInPost) applies because article
// content is author-written and may carry pre-TLS-era embeds.
func scanDrupalContent(account string, creds drupalCreds) []alert.Finding {
query := fmt.Sprintf(
"SELECT entity_id, body_value FROM node_revision__body WHERE %s",
paramsLikeClause("body_value"))
rows, _ := runCMSQuery(creds.asWPDBCreds(), query)
var findings []alert.Finding
for _, row := range rows {
entityID, body := splitTabRow(row)
if entityID == "" {
continue
}
sev, desc, ok := classifyMalwareRow(mysqlclient.BatchUnescape(body), true)
if !ok {
continue
}
findings = append(findings, alert.Finding{
Severity: sev,
Check: "drupal_content_injection",
Message: fmt.Sprintf("Drupal article content injection on %s: node %s (%s)", account, entityID, desc),
Details: fmt.Sprintf("Account: %s\nNode entity_id: %s\nMatch: %s", account, entityID, desc),
})
}
return findings
}
// scanDrupalAdmins joins users_field_data with user__roles to
// surface every account in the administrator role. Same Warning
// severity / per-row emission as the Joomla equivalent --
// legitimate site admin shows up here too, so this is operator
// review territory rather than auto-actionable.
//
// users_field_data is multilingual in D8+: a single uid can appear
// once per language code on translated sites. The default_langcode
// = 1 filter keeps each admin to exactly one finding regardless of
// how many translations the site has.
func scanDrupalAdmins(store *state.Store, account string, creds drupalCreds) []alert.Finding {
query := fmt.Sprintf(
"SELECT u.uid, u.name, u.mail FROM users_field_data u JOIN user__roles r ON u.uid = r.entity_id WHERE r.roles_target_id = '%s' AND u.default_langcode = 1",
drupalAdminRoleID)
rows, complete := runCMSQuery(creds.asWPDBCreds(), query)
return cmsAdminFindings(store, "drupal", "drupal_admin_injection", account, creds.asWPDBCreds(), rows, complete, func(fields []string) (string, string) {
return fmt.Sprintf("Drupal administrator account on %s: %s", account, fields[0]),
fmt.Sprintf("Account: %s\nRow: %s\nReview: confirm this is the legitimate site administrator.", account, strings.Join(fields, "\t"))
})
}
package checks
import (
"regexp"
"strings"
"unicode"
"unicode/utf8"
)
// This file contains pure-function helpers used by the database content
// scanner (checkWPPosts in dbscan.go). Keeping them pure (no MySQL, no
// filesystem) makes them deterministically testable and independently
// reusable.
//
// Two classes of false positive were historically observed on real
// production traffic:
//
// 1. db_post_injection fired on every post containing a script tag,
// including site-owner-added analytics and widget embeds (Google Tag
// Manager, Google merchant rating badge, etc.).
//
// 2. db_spam_injection used substring LIKE matching, so "specialist"
// triggered on "cialis", "pharmaceutical" triggered on "pharma",
// "casino resort" triggered on "casino", etc. It also scanned all
// post_types including Contact Form 7 / WPForms / Jetpack stored
// submissions, which routinely contain spambot form fills the site
// owner never displays.
//
// The helpers below encode the decisions needed to eliminate those FPs
// without opening detection holes: word-boundary keyword matching,
// post_type filtering against a denylist (not an allowlist, so attackers
// cannot hide a post by renaming post_type to one we didn't anticipate),
// and safe-domain filtering for external script-tag sources.
// nonScannablePostTypes are post_type values that legitimately store
// non-site-content data (form submissions, revisions, templates, feeds).
// These are excluded from malware and spam scans because their content
// is operator-invisible storage, not material rendered to site visitors.
//
// This is a DENYLIST, not an allowlist. A custom post_type created by a
// theme or plugin (for example WooCommerce `product`, events, portfolios)
// is still scanned. Adding a new value here is safe; an attacker cannot
// hide a post by choosing a new post_type, because we default to
// scanning anything not on this list.
var nonScannablePostTypes = []string{
// WordPress internals / templates / navigation
"revision",
"customize_changeset",
"oembed_cache",
"nav_menu_item",
"wp_template",
"wp_template_part",
"wp_global_styles",
"wp_navigation",
// Minification plugins (store compiled bundles that legitimately
// contain JavaScript and obfuscated character sequences).
"wphb_minify_group",
// Form builders (store plugin configuration and visitor submissions.
// Contact-form spam landing here is noise, not site compromise.)
"wpforms",
"wpforms_entries",
"wpforms-log",
"wpcf7_contact_form",
"flamingo_inbound",
"flamingo_outbound",
"cf7_message",
"feedback",
"jetpack_feedback",
}
// isScannablePostType returns true if the given post_type should be
// included in malware and spam scans. The decision mirrors the SQL
// `post_type NOT IN (...)` clause used in checkWPPosts, so Go-side
// callers and test assertions stay consistent with the live SQL.
func isScannablePostType(postType string) bool {
for _, t := range nonScannablePostTypes {
if t == postType {
return false
}
}
return true
}
// nonScannablePostTypesSQLList returns the denylist as a comma-separated
// SQL literal list (for example "'revision','wp_template',...") suitable
// for use inside a `post_type NOT IN (...)` clause. The values are
// hardcoded and contain only [a-z_-] characters, so SQL injection is not
// a risk here; nonetheless the function escapes defensively.
func nonScannablePostTypesSQLList() string {
parts := make([]string, 0, len(nonScannablePostTypes))
for _, t := range nonScannablePostTypes {
// Hardcoded list contains only [a-z_-], but defensive escape in
// case future maintainers add a value with a quote or backslash.
escaped := strings.ReplaceAll(t, `\`, `\\`)
escaped = strings.ReplaceAll(escaped, `'`, `\'`)
parts = append(parts, "'"+escaped+"'")
}
return strings.Join(parts, ",")
}
// dbSpamPattern is a single spam-keyword detector. The LIKE fragment
// is a MySQL server-side pre-filter that quickly narrows the set of
// candidate posts; the Go regex finds case-insensitive occurrences and
// spamKeywordMatchIndexes applies strict word boundaries so that
// "specialist" does not match "cialis", "pharmacy" does not match
// "pharma", and so on.
//
// Patterns that end with a non-word character (dash) already have an
// implicit right boundary from that character and only need a left
// word boundary. Patterns ending in a word character need boundaries
// on both sides, even when they contain an internal dash.
type dbSpamPattern struct {
keyword string // human-readable keyword used in finding messages
regex *regexp.Regexp // applied Go-side to candidate rows
likeFragment string // SQL LIKE fragment, always bracketed with '%'
// deletable gates the DESTRUCTIVE path only. Scanning uses every
// pattern; db-clean --delete-spam uses this subset. "pharma" and
// "betting" are genuine spam keywords that are also ordinary words in
// legitimate publishing ("Health and Pharma Summit", coverage of a
// licensed bookmaker), and a word boundary cannot tell the two apart.
// Flagging those for review is useful; deleting on them is not.
deletable bool
}
// dbSpamPatterns enumerates the keywords we flag as SEO/pharma/gambling
// spam in WordPress post content. Each entry pairs a fast SQL LIKE with
// a case-insensitive Go regex; spamKeywordMatchIndexes applies the strict
// Unicode word-boundary check.
//
// The regexes are case-insensitive to catch CIALIS / Cialis / cialis.
// The LIKE fragments are lowercase because MySQL LIKE is case-insensitive
// under the default _ci collation used by cPanel MariaDB.
var dbSpamPatterns = []dbSpamPattern{
newDBSpamPattern("viagra", "%viagra%", true),
newDBSpamPattern("cialis", "%cialis%", true),
newDBSpamPattern("pharma", "%pharma%", false),
newDBSpamPattern("betting", "%betting%", false),
// Dashed variants: the trailing dash is itself a non-word char and
// serves as the right boundary. Only a left word-boundary is needed.
// The dash makes these URL-slug shaped, which prose does not produce,
// so they carry enough signal to delete on.
newDBSpamPattern("casino-", "%casino-%", true),
newDBSpamPattern("buy-cheap-", "%buy-cheap-%", true),
// These slug forms end in a letter, so the matcher requires both
// boundaries and rejects longer words such as "free-downloadable".
newDBSpamPattern("free-download", "%free-download%", true),
newDBSpamPattern("crack-serial", "%crack-serial%", true),
}
func newDBSpamPattern(keyword, likeFragment string, deletable bool) dbSpamPattern {
return dbSpamPattern{
keyword: keyword,
regex: regexp.MustCompile(`(?i)` + regexp.QuoteMeta(keyword)),
likeFragment: likeFragment,
deletable: deletable,
}
}
// spamKeywordMatchIndexes returns only whole-word keyword matches. Go's \b
// is ASCII-only, so checking adjacent runes explicitly prevents a keyword
// surrounded by non-ASCII letters from being treated as a standalone word.
// A keyword ending in punctuation, such as "casino-", needs no right check:
// the punctuation is already the delimiter that makes the pattern specific.
func spamKeywordMatchIndexes(pattern dbSpamPattern, content string) [][]int {
matches := pattern.regex.FindAllStringIndex(content, -1)
confirmed := matches[:0]
for _, match := range matches {
if hasSpamWordBoundaries(content, match[0], match[1]) {
confirmed = append(confirmed, match)
}
}
return confirmed
}
func hasSpamWordBoundaries(content string, start, end int) bool {
first, _ := utf8.DecodeRuneInString(content[start:end])
if start > 0 && isSpamWordRune(first) {
previous, _ := utf8.DecodeLastRuneInString(content[:start])
if isSpamWordRune(previous) {
return false
}
}
last, _ := utf8.DecodeLastRuneInString(content[start:end])
if end < len(content) && isSpamWordRune(last) {
next, _ := utf8.DecodeRuneInString(content[end:])
if isSpamWordRune(next) {
return false
}
}
return true
}
func isSpamWordRune(r rune) bool {
return r == '_' || r == '\u200c' || r == '\u200d' ||
r == unicode.ReplacementChar || unicode.IsLetter(r) ||
unicode.IsNumber(r) || unicode.IsMark(r) || unicode.Is(unicode.Pc, r)
}
// countSpamMatches returns the number of candidate rows with a bounded
// keyword match. The caller is responsible for passing only rows that were
// already narrowed by the pattern.likeFragment SQL pre-filter.
func countSpamMatches(pattern dbSpamPattern, contents []string) int {
n := 0
for _, c := range contents {
if len(spamKeywordMatchIndexes(pattern, c)) > 0 {
n++
}
}
return n
}
// hasMaliciousExternalScript reports whether the content contains a
// script-tag with a src attribute pointing at a domain NOT on the known-
// safe list (see knownSafeDomains in db_autoresponse.go).
//
// This predicate uses the STRICT classifier (isAttackerScriptURL): it
// flags raw-IP hosts, abused TLDs, known-bad exfil hosts, empty hosts,
// AND plaintext HTTP external scripts. It is the right predicate for
// wp_options and similar configuration storage where fresh-today
// content is expected; see hasMaliciousExternalScriptInPost for the
// looser post_content predicate.
//
// Inline script blocks without a src attribute are not classified by
// this function; those are covered by the separate code-pattern entries
// in dbMalwarePatterns which catch common inline obfuscation techniques.
//
// Rationale: a bare script-tag match was the primary source of false
// positives on real traffic. Legitimate analytics embeds (Google Tag
// Manager, Google Analytics, Google merchant rating badge, Mailchimp,
// HubSpot, etc.) install both an external loader tag AND an inline
// initialization block. Flagging the inline block alone produced many
// HIGH severity noise findings on customer sites. Requiring a
// non-safe-domain external src reduces this to zero FPs in practice
// while still catching attackers who inject a tag pointing at an
// untrusted domain.
func hasMaliciousExternalScript(content string) bool {
return extractMaliciousScriptURL(content) != ""
}
// hasMaliciousExternalScriptInPost is the post_content variant of
// hasMaliciousExternalScript. It applies the same regex-based script-tag
// extraction but classifies each src URL with isAttackerScriptURLInPost,
// which drops the plaintext-HTTP indicator.
//
// Rationale: post_content carries author-written text that can contain
// pre-TLS-era embeds (e.g. a 2013-era video embed on a Romanian video
// site). Those embeds use http:// and, under the strict classifier,
// would flag on scheme alone even though the post has not been modified
// in a decade. Attackers in 2026 almost never land on plaintext-HTTP
// mainstream-TLD URLs — they use raw IPs, abused TLDs, or cheap exfil
// hosts — so dropping the HTTP signal for this context eliminates the
// legacy-embed false positives without giving up meaningful detection.
func hasMaliciousExternalScriptInPost(content string) bool {
// Page builders keep their markup as JSON inside the post row, which
// escapes every slash of an injected loader URL.
matches := scriptSrcRe.FindAllStringSubmatch(unescapeStoredSlashes(content), -1)
for _, match := range matches {
if len(match) < 2 {
continue
}
if isAttackerScriptURLInPost(match[1]) {
return true
}
}
return false
}
package checks
import (
"fmt"
"math"
"net"
"net/url"
"sort"
"strconv"
"strings"
"unicode/utf8"
"github.com/tdewolff/parse/v2"
cssparser "github.com/tdewolff/parse/v2/css"
"golang.org/x/net/html"
"golang.org/x/net/idna"
"golang.org/x/net/publicsuffix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/mysqlclient"
)
// Hidden-container link injection stored in the database.
//
// The kit wraps outbound links in a container the reader never sees -- an
// absolutely positioned block pushed thousands of pixels off the canvas, a
// text-indent far outside the viewport, or a display:none wrapper -- so a
// visitor sees an ordinary page while crawlers follow the links. CSM already
// ships file-side rules for this shape, but these injections live in
// post_content and option rows, where no file scanner ever looks.
//
// A hidden container on its own is ordinary: themes hide screen-reader text,
// mobile menus and collapsed panels. What has no benign reading is a hidden
// container wrapping links to somebody else's domain.
const (
// maxHiddenLinkRows bounds the rows pulled per table.
maxHiddenLinkRows = 200
// maxHiddenLinkValueBytes bounds one row's markup. Injected blocks sit at
// the top or bottom of the content, so half is read from each end rather
// than letting padding at the front hide a trailing block.
maxHiddenLinkValueBytes = 128 * 1024
maxHiddenLinkSampleBytes = maxHiddenLinkValueBytes / 2
// maxHiddenLinkSiteURLBytes prevents a poisoned site address from making
// the otherwise bounded candidate query return an attacker-sized value.
maxHiddenLinkSiteURLBytes = 4 * 1024
// maxHiddenLinkNodes bounds one row's parsed markup. The walk is iterative
// so nesting cannot exhaust the stack, but a hostile row must not be able
// to spend unbounded time either.
maxHiddenLinkNodes = 200000
// maxHiddenLinkHostsShown bounds the hosts named in the finding.
maxHiddenLinkHostsShown = 12
// maxHiddenLinkRowsShown bounds the rows named in the finding.
maxHiddenLinkRowsShown = 10
// offScreenPixels is how far outside the canvas an offset must sit before
// it is cloaking rather than layout. Real layouts nudge elements by a few
// pixels; the kits observed used four- and five-digit offsets.
offScreenPixels = 1000
// Font-relative offsets reach the same distance with smaller numbers. The
// threshold remains high enough to exclude ordinary indentation.
offScreenFontUnits = 100
)
// hiddenLinkCandidateCondition selects rows whose sampled markup may hide a
// link. The Go parser joins the samples when they cover the whole value, so
// SQL must examine that value together too. Larger values use separate end
// windows. Character windows cover every sampled byte, with bounded work.
//
// commented adds matching for declarations split by CSS comments. That can
// exhaust the server's regex work limit on dense content. The selection
// without it stays well inside that limit for a sampled window, so callers
// retry without it after a regex timeout.
func hiddenLinkCandidateCondition(column string, commented bool) string {
head := fmt.Sprintf("LEFT(%s, CASE WHEN OCTET_LENGTH(%s) <= %d THEN %d ELSE %d END)",
column, column, maxHiddenLinkValueBytes, maxHiddenLinkValueBytes, maxHiddenLinkSampleBytes)
return fmt.Sprintf("(%s OR (OCTET_LENGTH(%s) > %d AND %s))",
hiddenLinkWindowCondition(head, commented), column, maxHiddenLinkValueBytes,
hiddenLinkWindowCondition(fmt.Sprintf("RIGHT(%s, %d)", column, maxHiddenLinkSampleBytes), commented))
}
// hiddenLinkPlainDeclarations are the declaration prefixes the parser can
// hide or move off-screen, after whitespace and calc( are removed. Longer
// property names such as margin-left end in one of these.
var hiddenLinkPlainDeclarations = []string{
"display:none", "visibility:hidden", "visibility:collapse",
"opacity:0", "opacity:+0", "opacity:.0", "opacity:+.0", "opacity:-",
"text-indent:-", "left:-", "top:-", "right:-", "bottom:-",
}
// hiddenLinkRegexPattern is matched against the reversed, normalized window.
// Encodings there are backslashes, and each plain declaration has become
// style= followed by a backslash. From each backslash the pattern searches
// back to style=, stopping at the preceding tag delimiter or backslash, so no
// stretch of text is traversed twice and encoded page text outside a style
// attribute is not selected. A single alternative keeps the cost of each
// backslash low enough for dense content.
func hiddenLinkRegexPattern() string {
return `\\[^>\\]*=elyts`
}
// hiddenLinkCommentPattern reads the unmodified ASCII-converted window.
// Removing whitespace or calc( inside a comment can manufacture a closing
// delimiter and lose a declaration that the CSS parser recognizes as hidden.
func hiddenLinkCommentPattern() string {
// Reverse the whole comment grammar: a comment can contain /*, so its
// reversed body can contain */. A comment fallback must keep the
// declaration's tokens adjacent, or a visible declaration could join
// unrelated page text and exhaust the candidate limit.
// ASCII conversion maps Unicode whitespace to ?, without joining comment
// body characters. Keep gaps outside the comment repetition unambiguous.
space := `[[:space:]?]*`
gap := space + `(/[*]+([^*]*[^*/][*]+)*[^*]*[*]/` + space + `)*`
return strings.Join([]string{
"enon" + gap + ":" + gap + "yalpsid",
"(neddih|espalloc)" + gap + ":" + gap + "ytilibisiv",
// Positive values with negative exponents can underflow to zero.
"(-|0[.]?[+]?|-e[0-9_.]+[+]?)" + gap + ":" + gap + "yticapo",
"-" + gap + "([(]" + gap + "clac" + gap + ")?:" + gap + "(tnedni-txet|tfel|pot|thgir|mottob)",
}, "|")
}
// hiddenLinkWindowCondition selects one sample window. Literal replacements
// do the bulk of the work because they carry no regex budget. The chain is
// written in both branches of one CASE, so the server evaluates it once.
func hiddenLinkWindowCondition(window string, commented bool) string {
ascii := func(code int) string { return fmt.Sprintf("CHAR(%d USING ascii)", code) }
lower := "CAST(LOWER(" + window + ") AS BINARY)"
// Conversion turns every other character into a question mark. Removing
// those and ASCII whitespace joins each declaration's tokens; the value
// parser trims Unicode whitespace, so every parsed form stays adjacent.
normalized := "CONVERT(LOWER(" + window + ") USING ascii)"
for _, code := range []int{'\t', '\n', '\v', '\f', '\r', ' ', '?'} {
normalized = fmt.Sprintf("REPLACE(%s, %s, '')", normalized, ascii(code))
}
normalized = fmt.Sprintf("REPLACE(%s, 'calc(', '')", normalized)
// A plain declaration becomes a match the regex finds without searching.
matched := "CONCAT('style=', " + ascii(92) + ")"
for _, declaration := range hiddenLinkPlainDeclarations {
normalized = fmt.Sprintf("REPLACE(%s, '%s', %s)", normalized, declaration, matched)
}
// Numeric entities only need their first digit, as in the old prefix match.
normalized = fmt.Sprintf("REPLACE(REPLACE(%s, '&#x', '&#'), '&colon', %s)", normalized, ascii(92))
for _, digit := range "0123456789abcdef" {
normalized = fmt.Sprintf("REPLACE(%s, '&#%c', %s)", normalized, digit, ascii(92))
}
// Only rows carrying an encoding or comment need a regex. Hex literals
// and CHAR keep backslashes independent of SQL escape modes.
encoded := fmt.Sprintf("LOCATE('&#', %[1]s) > 0 OR LOCATE('&colon', %[1]s) > 0 OR LOCATE(CHAR(92), %[1]s) > 0", lower)
pattern := fmt.Sprintf("CONVERT(0x%x USING ascii)", hiddenLinkRegexPattern())
selection := fmt.Sprintf("(CASE WHEN %s THEN REVERSE(%s) REGEXP %s ELSE LOCATE(%s, %s) > 0 END)",
encoded, normalized, pattern, ascii(92), normalized)
if commented {
selection = fmt.Sprintf("(CASE WHEN %s THEN 1 WHEN LOCATE('/*', %s) > 0 "+
"THEN REVERSE(CONVERT(LOWER(%s) USING ascii)) REGEXP CONVERT(0x%x USING ascii) ELSE 0 END)",
selection, lower, window, hiddenLinkCommentPattern())
}
return fmt.Sprintf("(LOCATE('style', %s) > 0 AND %s)", lower, selection)
}
// hiddenLinkHit is what one row's markup revealed.
type hiddenLinkHit struct {
// offScreen records the strong signal: a container moved outside the
// canvas. Nothing legitimate positions readable content there.
offScreen bool
// hosts are the distinct off-site hosts linked from inside hidden
// containers, sorted.
hosts []string
// domains are the distinct registrable domains behind hosts. Severity is
// based on these so subdomains of one target do not look like a link farm.
domains []string
// multiDomain records that one hidden container, rather than merely one
// database row, links to at least two registrable domains.
multiDomain bool
// spammy records gambling or pharmacy vocabulary in the hidden anchors.
spammy bool
}
// hiddenOffsiteLinks reports the off-site hosts linked from inside a container
// the page hides from readers. siteHost is the site's own address; links back
// to it are ordinary navigation whatever their styling.
//
// This tokenizes rather than building a tree. x/net/html refuses markup nested
// deeper than 512 elements, and returning nothing for those rows would hand
// every kit a one-line evasion. The tokenizer has no depth limit, and tracking
// open elements on an explicit heap stack keeps attacker-controlled nesting off
// the goroutine stack, where an overflow is fatal and unrecoverable.
func hiddenOffsiteLinks(markup, siteHost string) hiddenLinkHit {
return hiddenOffsiteLinksForSites(markup, []string{siteHost})
}
func hiddenOffsiteLinksForSites(markup string, siteHosts []string) hiddenLinkHit {
siteDomains := make(map[string]bool, len(siteHosts))
for _, host := range siteHosts {
if domain := registrableDomain(host); domain != "" {
siteDomains[domain] = true
}
}
if len(siteDomains) == 0 {
return hiddenLinkHit{}
}
type openElement struct {
name string
hidden bool
hiddenGroup int
visibilityHidden bool
visibilityGroup int
offScreen bool
}
var stack []openElement
inherited := func() openElement {
if len(stack) == 0 {
return openElement{}
}
return stack[len(stack)-1]
}
var hit hiddenLinkHit
seenHosts := make(map[string]bool)
seenDomains := make(map[string]bool)
nextHiddenGroup := 1
groupDomains := make(map[int]map[string]bool)
newHiddenGroup := func() int {
group := nextHiddenGroup
nextHiddenGroup++
return group
}
recordGroupDomain := func(group int, domain string) {
if group == 0 {
return
}
domains := groupDomains[group]
if domains == nil {
domains = make(map[string]bool)
groupDomains[group] = domains
}
domains[domain] = true
if len(domains) >= 2 {
hit.multiDomain = true
}
}
// anchorHost is the host of the hidden anchor currently open, so its link
// text can be graded when the anchor closes.
anchorHost, anchorText := "", strings.Builder{}
anchorDepth := -1
closeAnchor := func() {
if anchorHost == "" {
return
}
if termNameSpamVocabulary.MatchString(anchorText.String()) || termNameSpamVocabulary.MatchString(anchorHost) {
hit.spammy = true
}
anchorHost, anchorText = "", strings.Builder{}
anchorDepth = -1
}
popOpenElement := func(name string) {
for i := len(stack) - 1; i >= 0; i-- {
if stack[i].name != name {
continue
}
if anchorDepth >= i {
closeAnchor()
}
stack = stack[:i]
return
}
}
z := html.NewTokenizer(strings.NewReader(markup))
for tokens := 0; tokens < maxHiddenLinkNodes; tokens++ {
switch z.Next() {
case html.ErrorToken:
closeAnchor()
sort.Strings(hit.hosts)
sort.Strings(hit.domains)
return hit
case html.TextToken:
if anchorHost != "" && anchorText.Len() < 512 &&
(len(stack) == 0 || !rawTextHTMLElements[stack[len(stack)-1].name]) {
text := z.Text()
remaining := 512 - anchorText.Len()
if len(text) > remaining {
text = text[:remaining]
}
anchorText.Write(text)
}
case html.StartTagToken, html.SelfClosingTagToken:
name, style, href := tokenAttrs(z)
if name == "a" {
popOpenElement("a")
closeAnchor()
}
state := inherited()
if style != "" {
styleState := parseHiddenCSSState(style)
if styleState.hidden {
if !state.hidden {
state.hiddenGroup = newHiddenGroup()
}
state.hidden = true
}
if styleState.visibilitySet {
if styleState.visibilityHidden {
if !state.visibilityHidden {
state.visibilityGroup = newHiddenGroup()
}
} else {
state.visibilityGroup = 0
}
state.visibilityHidden = styleState.visibilityHidden
}
if styleState.offScreen {
state.offScreen = true
}
}
if name == "a" && (state.hidden || state.visibilityHidden) && href != "" {
if host, ok := absoluteLinkHost(href); ok {
domain := registrableDomain(host)
if domain != "" && !siteDomains[domain] {
if !seenHosts[host] {
seenHosts[host] = true
hit.hosts = append(hit.hosts, host)
}
if !seenDomains[domain] {
seenDomains[domain] = true
hit.domains = append(hit.domains, domain)
}
recordGroupDomain(state.hiddenGroup, domain)
recordGroupDomain(state.visibilityGroup, domain)
if state.offScreen {
hit.offScreen = true
}
anchorHost = host
anchorDepth = len(stack)
}
}
}
if !voidHTMLElements[name] {
state.name = name
stack = append(stack, state)
}
case html.EndTagToken:
name, _, _ := tokenAttrs(z)
// Unclosed tags are ordinary in real content, so pop back to the
// nearest matching element rather than assuming balance.
popOpenElement(name)
if name == "a" {
closeAnchor()
}
}
}
closeAnchor()
sort.Strings(hit.hosts)
sort.Strings(hit.domains)
return hit
}
// voidHTMLElements never have an end tag, so they must not be pushed onto the
// open-element stack.
var voidHTMLElements = map[string]bool{
"area": true, "base": true, "br": true, "col": true, "embed": true,
"hr": true, "img": true, "input": true, "link": true, "meta": true,
"param": true, "source": true, "track": true, "wbr": true,
}
var rawTextHTMLElements = map[string]bool{
"script": true, "style": true, "textarea": true, "title": true,
}
// tokenAttrs returns the current token's lowercased tag name plus the two
// attributes this check reads.
func tokenAttrs(z *html.Tokenizer) (name, style, href string) {
raw, hasAttr := z.TagName()
name = strings.ToLower(string(raw))
for hasAttr {
var key, val []byte
key, val, hasAttr = z.TagAttr()
switch strings.ToLower(string(key)) {
case "style":
style = string(val)
case "href":
href = strings.TrimSpace(string(val))
}
}
return name, style, href
}
// absoluteLinkHost returns the host of a link that leaves the current page. A
// relative link cannot leave the site and is not one.
func absoluteLinkHost(href string) (string, bool) {
parsed, err := url.Parse(href)
if err != nil || parsed.Host == "" {
return "", false
}
switch strings.ToLower(parsed.Scheme) {
case "http", "https", "":
return normalizeHost(parsed.Hostname()), true
default:
return "", false
}
}
// registrableDomain canonicalizes a host to the public suffix plus one. IP and
// single-label hosts remain their own identity so sites served on either can
// still distinguish their own links from external ones.
func registrableDomain(host string) string {
host = normalizeHost(host)
if host == "" {
return ""
}
if ip := net.ParseIP(host); ip != nil {
return ip.String()
}
domain, err := publicsuffix.EffectiveTLDPlusOne(host)
if err != nil {
return host
}
return domain
}
func normalizeHost(host string) string {
host = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(host)), ".")
if host == "" {
return ""
}
if ip := net.ParseIP(host); ip != nil {
return ip.String()
}
ascii, err := idna.Lookup.ToASCII(host)
if err != nil {
return ""
}
return strings.ToLower(ascii)
}
type cssDeclaration struct {
value string
important bool
}
type hiddenCSSState struct {
hidden bool
visibilitySet bool
visibilityHidden bool
offScreen bool
}
func parseHiddenCSSState(declarations string) hiddenCSSState {
effective := make(map[string]cssDeclaration)
parser := cssparser.NewParser(parse.NewInputString(declarations), true)
for {
grammar, _, propertyBytes := parser.Next()
if grammar == cssparser.ErrorGrammar {
break
}
if grammar != cssparser.DeclarationGrammar {
continue
}
property := strings.ToLower(cssUnescape(string(propertyBytes)))
var rawValue strings.Builder
for _, token := range parser.Values() {
rawValue.Write(token.Data)
}
value, important := cssDeclarationValue(strings.ToLower(cssUnescape(rawValue.String())))
previous, exists := effective[property]
if exists && previous.important && !important {
continue
}
effective[property] = cssDeclaration{value: value, important: important}
}
var state hiddenCSSState
if declaration, ok := effective["display"]; ok && declaration.value == "none" {
state.hidden = true
}
if declaration, ok := effective["visibility"]; ok {
switch declaration.value {
case "hidden", "collapse":
state.visibilitySet = true
state.visibilityHidden = true
case "visible", "initial":
state.visibilitySet = true
}
}
if declaration, ok := effective["opacity"]; ok && cssOpacityIsHidden(declaration.value) {
state.hidden = true
}
for _, property := range []string{
"text-indent", "left", "top", "right", "bottom", "margin-left", "margin-top",
} {
if declaration, ok := effective[property]; ok && cssLengthIsOffScreen(declaration.value) {
state.hidden = true
state.offScreen = true
break
}
}
return state
}
func cssUnescape(value string) string {
if !strings.ContainsRune(value, '\\') {
return value
}
var out strings.Builder
out.Grow(len(value))
for i := 0; i < len(value); i++ {
if value[i] != '\\' {
out.WriteByte(value[i])
continue
}
i++
if i >= len(value) {
break
}
if isCSSHex(value[i]) {
codePoint := uint32(0)
digits := 0
for i < len(value) && digits < 6 && isCSSHex(value[i]) {
codePoint = codePoint*16 + uint32(cssHexValue(value[i]))
i++
digits++
}
if codePoint == 0 || codePoint > utf8.MaxRune || 0xD800 <= codePoint && codePoint <= 0xDFFF {
out.WriteRune(utf8.RuneError)
} else {
out.WriteRune(rune(codePoint))
}
if i < len(value) && isCSSWhitespace(value[i]) {
if value[i] == '\r' && i+1 < len(value) && value[i+1] == '\n' {
i++
}
} else {
i--
}
continue
}
if value[i] == '\r' && i+1 < len(value) && value[i+1] == '\n' {
i++
continue
}
if value[i] == '\n' || value[i] == '\r' || value[i] == '\f' {
continue
}
r, size := utf8.DecodeRuneInString(value[i:])
out.WriteRune(r)
i += size - 1
}
return out.String()
}
func isCSSHex(value byte) bool {
return value >= '0' && value <= '9' || value >= 'a' && value <= 'f' || value >= 'A' && value <= 'F'
}
func cssHexValue(value byte) byte {
switch {
case value >= '0' && value <= '9':
return value - '0'
case value >= 'a' && value <= 'f':
return value - 'a' + 10
default:
return value - 'A' + 10
}
}
func isCSSWhitespace(value byte) bool {
return value == ' ' || value == '\t' || value == '\n' || value == '\r' || value == '\f'
}
func cssOpacityIsHidden(value string) bool {
value = strings.TrimSpace(value)
value = strings.TrimSpace(strings.TrimSuffix(value, "%"))
number, err := strconv.ParseFloat(value, 64)
return err == nil && number <= 0 || math.IsInf(number, -1)
}
func cssDeclarationValue(value string) (string, bool) {
value = strings.TrimSpace(value)
marker := strings.LastIndexByte(value, '!')
if marker < 0 || strings.TrimSpace(value[marker+1:]) != "important" {
return value, false
}
return strings.TrimSpace(value[:marker]), true
}
func cssLengthIsOffScreen(value string) bool {
value = strings.TrimSpace(value)
if strings.HasPrefix(value, "calc(") && strings.HasSuffix(value, ")") {
value = strings.TrimSpace(value[len("calc(") : len(value)-1])
}
unit := ""
for _, candidate := range []string{"vmin", "vmax", "rem", "px", "em", "vw", "vh", "%"} {
if strings.HasSuffix(value, candidate) {
unit = candidate
value = strings.TrimSpace(strings.TrimSuffix(value, candidate))
break
}
}
number, err := strconv.ParseFloat(value, 64)
if err != nil {
return math.IsInf(number, -1)
}
threshold := float64(offScreenPixels)
if unit == "em" || unit == "rem" {
threshold = offScreenFontUnits
}
return number <= -threshold
}
// hiddenLinkRow is one database row that carried a hidden link block.
type hiddenLinkRow struct {
label string
hit hiddenLinkHit
}
// checkWPHiddenLinks reports link blocks the page hides from its readers.
func checkWPHiddenLinks(user string, creds wpDBCreds, prefix string) []alert.Finding {
siteHosts, optionRows := hiddenLinkOptionRows(creds, prefix)
if len(siteHosts) == 0 {
// Every absolute link would look external. Report nothing rather than
// flood, and let the incomplete marker say why.
markCheckIncomplete(creds.queryCtx, "db_content")
return nil
}
rows := make([]hiddenLinkRow, 0, len(optionRows))
for _, row := range optionRows {
if hit := hiddenOffsiteLinkSamples(row, siteHosts); len(hit.hosts) > 0 {
rows = append(rows, hiddenLinkRow{label: "option " + strconv.Quote(row.label), hit: hit})
}
}
for _, row := range hiddenLinkPostRows(creds, prefix) {
if hit := hiddenOffsiteLinkSamples(row, siteHosts); len(hit.hosts) > 0 {
rows = append(rows, hiddenLinkRow{label: "post " + strconv.Quote(row.label), hit: hit})
}
}
return buildHiddenLinkFindings(user, creds, prefix, rows)
}
type hiddenLinkSource struct {
label string
markup string
tailMarkup string
valueBytes int
}
func hiddenOffsiteLinkSamples(source hiddenLinkSource, siteHosts []string) hiddenLinkHit {
if source.valueBytes > 0 && source.valueBytes <= maxHiddenLinkValueBytes && source.tailMarkup != "" {
overlap := len(source.markup) + len(source.tailMarkup) - source.valueBytes
if overlap >= 0 && overlap <= len(source.markup) && overlap <= len(source.tailMarkup) &&
source.markup[len(source.markup)-overlap:] == source.tailMarkup[:overlap] {
source.markup += source.tailMarkup[overlap:]
source.tailMarkup = ""
}
}
// Treat the two samples as separate fragments. Concatenating them could
// carry an unclosed hidden container across the omitted middle and turn a
// visible trailing link into a false positive.
hit := hiddenOffsiteLinksForSites(source.markup, siteHosts)
if source.tailMarkup == "" || source.tailMarkup == source.markup {
return hit
}
return mergeHiddenLinkHits(hit, hiddenOffsiteLinksForSites(source.tailMarkup, siteHosts))
}
func mergeHiddenLinkHits(left, right hiddenLinkHit) hiddenLinkHit {
merged := hiddenLinkHit{
offScreen: left.offScreen || right.offScreen,
multiDomain: left.multiDomain || right.multiDomain,
spammy: left.spammy || right.spammy,
}
for _, values := range [][]string{left.hosts, right.hosts} {
for _, value := range values {
if len(merged.hosts) == 0 || merged.hosts[len(merged.hosts)-1] != value {
merged.hosts = append(merged.hosts, value)
}
}
}
for _, values := range [][]string{left.domains, right.domains} {
for _, value := range values {
if len(merged.domains) == 0 || merged.domains[len(merged.domains)-1] != value {
merged.domains = append(merged.domains, value)
}
}
}
sort.Strings(merged.hosts)
merged.hosts = compactSortedStrings(merged.hosts)
sort.Strings(merged.domains)
merged.domains = compactSortedStrings(merged.domains)
return merged
}
func compactSortedStrings(values []string) []string {
if len(values) < 2 {
return values
}
out := values[:1]
for _, value := range values[1:] {
if value != out[len(out)-1] {
out = append(out, value)
}
}
return out
}
// runHiddenLinkCandidateQuery runs the statement built around a candidate
// condition. A regex timeout stops only commented-style matching: the failure
// stays recorded, so coverage remains incomplete, and the retry keeps plain
// and encoded styles covered.
func runHiddenLinkCandidateQuery(creds wpDBCreds, query func(commented bool) string) []string {
timeouts := creds.queryState.regexTimeoutCount()
rows := runMySQLQuery(creds, query(true))
if creds.queryState.regexTimeoutCount() == timeouts {
return rows
}
return runMySQLQuery(creds, query(false))
}
// hiddenLinkOptionRows reads the site address and the option rows worth
// parsing in one round trip.
func hiddenLinkOptionRows(creds wpDBCreds, prefix string) ([]string, []hiddenLinkSource) {
query := func(commented bool) string {
return fmt.Sprintf(
"(SELECT 'site' AS kind, option_name, LEFT(CAST(option_value AS BINARY), %d), "+
"'', OCTET_LENGTH(option_value), 'site' FROM %soptions "+
"WHERE option_name IN ('siteurl', 'home') LIMIT 4) UNION ALL "+
"(SELECT 'opt', option_name, LEFT(CAST(option_value AS BINARY), %d), "+
"RIGHT(CAST(option_value AS BINARY), %d), OCTET_LENGTH(option_value), 'opt' FROM %soptions "+
"WHERE %s LIMIT %d)",
maxHiddenLinkSiteURLBytes, prefix, maxHiddenLinkSampleBytes, maxHiddenLinkSampleBytes, prefix,
hiddenLinkCandidateCondition("option_value", commented), maxHiddenLinkRows+1)
}
var siteHosts []string
seenSiteHosts := make(map[string]bool)
var out []hiddenLinkSource
rows := runHiddenLinkCandidateQuery(creds, query)
optionRowsSeen := 0
truncated := false
for _, line := range rows {
// Column separators are literal tabs while tabs and newlines inside a
// value remain batch escapes. Split that transport form before decoding
// individual columns or an embedded tab becomes indistinguishable from
// a separator.
parts := strings.SplitN(strings.TrimRight(line, "\r\n"), "\t", 6)
if len(parts) != 6 {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
kind := strings.TrimSpace(parts[0])
if kind != "site" {
optionRowsSeen++
if optionRowsSeen > maxHiddenLinkRows {
if !truncated {
markCheckIncomplete(creds.queryCtx, "db_content")
truncated = true
}
continue
}
}
name := strings.TrimSpace(mysqlclient.BatchUnescape(parts[1]))
encodedValue := parts[2]
value := mysqlclient.BatchUnescape(encodedValue)
valueBytes, err := strconv.Atoi(strings.TrimSpace(parts[4]))
if err != nil || valueBytes < 0 {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
if kind == "site" {
if valueBytes > maxHiddenLinkSiteURLBytes || len(value) != valueBytes {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
if reason, _ := siteURLPoisonReason(encodedValue); reason != "" {
continue
}
if parsed, err := url.Parse(strings.TrimSpace(value)); err == nil {
host := normalizeHost(parsed.Hostname())
if registrableDomain(host) != "" && !seenSiteHosts[host] {
seenSiteHosts[host] = true
siteHosts = append(siteHosts, host)
}
}
continue
}
tailMarkup := mysqlclient.BatchUnescape(parts[3])
expectedSampleBytes := valueBytes
if expectedSampleBytes > maxHiddenLinkSampleBytes {
expectedSampleBytes = maxHiddenLinkSampleBytes
}
if len(value) != expectedSampleBytes || len(tailMarkup) != expectedSampleBytes {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
if valueBytes > maxHiddenLinkValueBytes {
markCheckIncomplete(creds.queryCtx, "db_content")
}
out = append(out, hiddenLinkSource{
label: name, markup: value, tailMarkup: tailMarkup, valueBytes: valueBytes,
})
}
return siteHosts, out
}
func hiddenLinkPostRows(creds wpDBCreds, prefix string) []hiddenLinkSource {
query := func(commented bool) string {
return fmt.Sprintf(
"SELECT ID, LEFT(CAST(post_content AS BINARY), %d), "+
"RIGHT(CAST(post_content AS BINARY), %d), OCTET_LENGTH(post_content), 'post' "+
"FROM %sposts WHERE post_status = 'publish' "+
"AND post_type NOT IN (%s) AND %s LIMIT %d",
maxHiddenLinkSampleBytes, maxHiddenLinkSampleBytes, prefix,
nonScannablePostTypesSQLList(), hiddenLinkCandidateCondition("post_content", commented), maxHiddenLinkRows+1)
}
rows := runHiddenLinkCandidateQuery(creds, query)
if len(rows) > maxHiddenLinkRows {
markCheckIncomplete(creds.queryCtx, "db_content")
rows = rows[:maxHiddenLinkRows]
}
out := make([]hiddenLinkSource, 0, len(rows))
for _, line := range rows {
parts := strings.SplitN(strings.TrimRight(line, "\r\n"), "\t", 5)
if len(parts) != 5 {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
markup := mysqlclient.BatchUnescape(parts[1])
tailMarkup := mysqlclient.BatchUnescape(parts[2])
valueBytes, err := strconv.Atoi(strings.TrimSpace(parts[3]))
if err != nil || valueBytes < 0 {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
expectedSampleBytes := valueBytes
if expectedSampleBytes > maxHiddenLinkSampleBytes {
expectedSampleBytes = maxHiddenLinkSampleBytes
}
if len(markup) != expectedSampleBytes || len(tailMarkup) != expectedSampleBytes {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
if valueBytes > maxHiddenLinkValueBytes {
markCheckIncomplete(creds.queryCtx, "db_content")
}
out = append(out, hiddenLinkSource{
label: strings.TrimSpace(mysqlclient.BatchUnescape(parts[0])),
markup: markup,
tailMarkup: tailMarkup,
valueBytes: valueBytes,
})
}
return out
}
// buildHiddenLinkFindings grades the rows. An off-canvas container has no
// benign reading and is reported on its own. display:none does have one --
// themes hide panels that link out -- so it is reported only once the block
// points at several unrelated domains or carries spam vocabulary.
func buildHiddenLinkFindings(user string, creds wpDBCreds, prefix string, rows []hiddenLinkRow) []alert.Finding {
var reported []hiddenLinkRow
offScreen := false
hosts := make(map[string]bool)
domains := make(map[string]bool)
for _, row := range rows {
if !row.hit.offScreen && !row.hit.multiDomain && !row.hit.spammy {
continue
}
reported = append(reported, row)
if row.hit.offScreen {
offScreen = true
}
for _, host := range row.hit.hosts {
hosts[host] = true
}
for _, domain := range row.hit.domains {
domains[domain] = true
}
}
if len(reported) == 0 {
return nil
}
named := make([]string, 0, len(hosts))
for host := range hosts {
named = append(named, host)
}
sort.Strings(named)
shownHosts := named
if len(shownHosts) > maxHiddenLinkHostsShown {
shownHosts = shownHosts[:maxHiddenLinkHostsShown]
}
labels := make([]string, 0, len(reported))
for _, row := range reported {
if len(labels) >= maxHiddenLinkRowsShown {
break
}
labels = append(labels, row.label)
}
details := []string{
"A container the page hides from readers wraps links to other domains. " +
"Crawlers still follow them, which is the point: the site's ranking " +
"is lent to the linked domains without a visitor ever seeing it.",
hiddenLinkSample("Linked hosts", shownHosts, len(named)),
hiddenLinkSample("Rows", labels, len(reported)),
}
// An off-canvas container is cloaking by construction; a merely hidden one
// needed corroboration to be reported at all, which is a weaker claim.
severity := alert.Warning
if offScreen {
severity = alert.High
}
// Concealment strength and the complete destination set identify this
// site's finding. Row growth and display samples must not re-alert, while
// stronger concealment must survive a prior baseline or dismissal.
identity := append([]string{fmt.Sprintf("off-screen=%t", offScreen)}, named...)
return []alert.Finding{{
Severity: severity,
Check: "db_hidden_link_injection",
Message: fmt.Sprintf("%d WordPress rows hide outbound links to %d hosts across %d domains (account: %s)",
len(reported), len(named), len(domains), user),
Details: dbContentFindingDetails(creds, prefix, details...),
DedupKey: dbContentDedupKey(user, creds, prefix, identity...),
}}
}
func hiddenLinkSample(label string, shown []string, total int) string {
if total > len(shown) {
label += fmt.Sprintf(" (showing %d of %d)", len(shown), total)
}
return label + ": " + strings.Join(shown, ", ")
}
package checks
import (
"context"
"fmt"
"path/filepath"
"regexp"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mysqlclient"
"github.com/pidginhost/csm/internal/state"
)
// Joomla database content scanner.
//
// Discovery: glob /home/*/public_html/configuration.php and
// verify the file contains `class JConfig` -- the canonical marker
// for a Joomla site, distinguishing it from PHP files that happen
// to share the configuration.php filename. Credentials are read
// via regex over public-property assignments (`public $host = ...;`)
// rather than PHP eval; the parser ignores anything outside the
// JConfig class body.
//
// Scanned tables (all prefixed; the prefix is operator-controlled
// via configuration.php's $dbprefix and defaults to `jos_` /
// `<random>_` on fresh installs):
//
// <prefix>extensions params blob -- live_site, sitename,
// offline_message are common hijack targets
// <prefix>content article body for malware patterns
// <prefix>users user table; joined with
// <prefix>user_usergroup_map to find rogue Super Users
// (group_id = 8 in vanilla Joomla)
//
// Three new finding categories. CMS-explicit names so operators
// running mixed-CMS hosts can suppress per-CMS:
//
// joomla_extensions_injection (Critical) -- malware pattern in
// an extension's params
// joomla_content_injection (Critical) -- malware pattern in
// an article body
// joomla_admin_injection (Critical) -- rogue Super User
// account
// jConfigCredsPattern parses a `public $foo = 'value';` line
// (single OR double quotes). Anchored to the start of the line so
// arbitrary text inside string literals further along can't be
// misread as a credential.
var jConfigCredsPattern = regexp.MustCompile(`^\s*public\s+\$(\w+)\s*=\s*['"]([^'"]*)['"]\s*;`)
// joomlaSuperUserGroupID is the canonical group id for "Super Users"
// in vanilla Joomla 3+. Operators on hardened installs may have
// renumbered; only group 8 is checked.
const joomlaSuperUserGroupID = 8
// jConfigCreds carries the credentials extracted from a Joomla
// configuration.php. Mirrors the wpDBCreds shape so the existing
// runMySQLQuery / mysqlSchemaLiteral helpers can be reused, but
// kept distinct because Joomla configuration.php and WordPress
// wp-config.php are not interchangeable.
type jConfigCreds struct {
// ctx ties every query for this install to the runner's deadline.
ctx context.Context
dbName string
dbUser string
dbPass string
dbHost string
dbPrefix string
path string
queryState *dbQueryState
}
// asWPDBCreds returns the equivalent wpDBCreds for runMySQLQuery
// reuse. The `multisite` field is irrelevant here; the existing
// mysql client wrapper does not look at it.
func (c jConfigCreds) asWPDBCreds() wpDBCreds {
return wpDBCreds{
dbName: c.dbName,
dbUser: c.dbUser,
dbPass: c.dbPass,
dbHost: c.dbHost,
tablePrefix: c.dbPrefix,
queryCtx: c.ctx,
queryOwner: "db_content_joomla",
queryState: c.queryState,
}
}
// CheckJoomlaContent scans every Joomla installation under
// /home/*/public_html for malware-pattern matches in the three
// canonical attacker-touched tables. Mirrors the structure of
// CheckDatabaseContent without sharing code -- the credentials and
// table layout differ enough that a generic dispatcher is more
// abstraction than this point in the codebase needs.
func CheckJoomlaContent(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
return scanCMSInstalls(ctx, cfg, store, "db_content_joomla", scanJoomlaInstall,
"*/public_html/configuration.php", "*/*/configuration.php")
}
// scanJoomlaInstall scans one discovered install and stamps its findings
// with the owner resolved from the configuration path. The display label
// stays as before; an install outside every account root is not stamped.
func scanJoomlaInstall(ctx context.Context, path string, store *state.Store) []alert.Finding {
matched, err := looksLikeJoomlaConfig(ctx, path)
if err != nil {
markCheckIncomplete(ctx, "db_content_joomla")
return nil
}
if !matched {
return nil
}
account := extractUser(filepath.Dir(path))
creds, err := parseJConfig(ctx, path)
if err != nil || creds.dbName == "" || creds.dbUser == "" {
markCheckIncomplete(ctx, "db_content_joomla")
return nil
}
creds.ctx = ctx
creds.queryState = new(dbQueryState)
prefix := creds.dbPrefix
if prefix == "" {
prefix = "jos_"
}
var findings []alert.Finding
findings = append(findings, scanJoomlaExtensions(account, creds, prefix)...)
findings = append(findings, scanJoomlaContent(account, creds, prefix)...)
findings = append(findings, scanJoomlaSuperUsers(store, account, creds, prefix)...)
if owner, ok := installOwner(path); ok {
findings = stampTenantIDIfEmpty(findings, owner)
}
return findings
}
// A defaced configuration can still identify a Joomla installation. Its
// credentials are parsed without executing any PHP.
func looksLikeJoomlaConfig(ctx context.Context, path string) (bool, error) {
data, err := readCMSConfig(ctx, path)
if err != nil {
return false, err
}
return strings.Contains(strings.ToLower(string(data)), "class jconfig"), nil
}
// parseJConfig reads configuration.php and pulls credentials out of
// the public-property assignments. Lines outside the class body
// (PHP comments, namespaced statements, etc.) are tolerated
// silently because the regex is line-anchored to "public $foo = ...".
func parseJConfig(ctx context.Context, path string) (jConfigCreds, error) {
creds := jConfigCreds{path: path}
data, err := readCMSConfig(ctx, path)
if err != nil {
return creds, err
}
for _, line := range strings.Split(string(data), "\n") {
m := jConfigCredsPattern.FindStringSubmatch(line)
if m == nil {
continue
}
switch strings.ToLower(m[1]) {
case "host":
creds.dbHost = m[2]
case "user":
creds.dbUser = m[2]
case "password":
creds.dbPass = m[2]
case "db":
creds.dbName = m[2]
case "dbprefix":
creds.dbPrefix = m[2]
}
}
if creds.dbHost == "" {
creds.dbHost = "localhost"
}
return creds, nil
}
// scanJoomlaExtensions queries the extensions table for params
// blobs that match the malware-pattern pre-filter, then applies a
// Go-side post-filter to drop rows whose only match was <script>
// LIKE noise (legitimate analytics embeds in extension params).
//
// Two-phase classifier:
//
// 1. SQL pre-filter via LIKE keeps the result set bounded -- a
// vanilla Joomla #__extensions table has ~50 rows, but a
// plugin-heavy install can exceed 200, and we don't want to
// pull every params blob into the daemon.
//
// 2. Go classifyMalwareRow re-checks each pattern individually
// against the full body, applying the same requiresExternalScript
// filter the WP scanner uses for wp_options. Strict predicate
// here (hasMaliciousExternalScript) because params is config
// storage.
func scanJoomlaExtensions(account string, creds jConfigCreds, prefix string) []alert.Finding {
query := fmt.Sprintf(
"SELECT name, params FROM %sextensions WHERE %s",
prefix, paramsLikeClause("params"))
rows, _ := runCMSQuery(creds.asWPDBCreds(), query)
var findings []alert.Finding
for _, row := range rows {
name, body := splitTabRow(row)
if name == "" {
continue
}
sev, desc, ok := classifyMalwareRow(mysqlclient.BatchUnescape(body), false)
if !ok {
continue
}
findings = append(findings, alert.Finding{
Severity: sev,
Check: "joomla_extensions_injection",
Message: fmt.Sprintf("Joomla extension params injection on %s: %s (%s)", account, name, desc),
Details: fmt.Sprintf("Account: %s\nExtension: %s\nMatch: %s", account, name, desc),
})
}
return findings
}
// scanJoomlaContent queries article bodies (introtext) for malware
// patterns. Same two-phase classifier as scanJoomlaExtensions but
// uses the looser post-filter (hasMaliciousExternalScriptInPost)
// because articles are author-written and may carry pre-TLS-era
// embeds the strict predicate would flag on scheme alone.
//
// fulltext_ is not scanned in v1: it's almost never populated on
// modern Joomla installs (the read-more split is a layout choice
// most templates don't bother with), and adding it doubles the
// query cost for marginal coverage. Follow-up if operators see
// missed detections.
func scanJoomlaContent(account string, creds jConfigCreds, prefix string) []alert.Finding {
query := fmt.Sprintf(
"SELECT id, title, introtext FROM %scontent WHERE %s",
prefix, paramsLikeClause("introtext"))
rows, _ := runCMSQuery(creds.asWPDBCreds(), query)
var findings []alert.Finding
for _, row := range rows {
fields := strings.SplitN(row, "\t", 3)
if len(fields) < 3 {
continue
}
id, title, body := fields[0], fields[1], fields[2]
sev, desc, ok := classifyMalwareRow(mysqlclient.BatchUnescape(body), true)
if !ok {
continue
}
findings = append(findings, alert.Finding{
Severity: sev,
Check: "joomla_content_injection",
Message: fmt.Sprintf("Joomla article content injection on %s: id=%s title=%q (%s)", account, id, title, desc),
Details: fmt.Sprintf("Account: %s\nArticle ID: %s\nTitle: %s\nMatch: %s", account, id, title, desc),
})
}
return findings
}
// classifyMalwareRow walks dbMalwarePatterns against body and
// returns the strongest pattern match that survives the
// requiresExternalScript filter. Returns ok=false when nothing
// genuine matched -- the caller skips that row entirely.
//
// inPostContext switches between the strict
// hasMaliciousExternalScript (for config-storage rows like Joomla
// extension params or Drupal config) and the looser
// hasMaliciousExternalScriptInPost (for author-written article
// content). Mirrors how the WP scanner picks its predicate per
// table. Shared by the Joomla and Drupal scanners; the function
// lives in dbscan_joomla.go for historical reasons (added with
// the Joomla scanner; renamed when Drupal also needed it).
func classifyMalwareRow(body string, inPostContext bool) (alert.Severity, string, bool) {
if body == "" {
return 0, "", false
}
lower := strings.ToLower(body)
var bestSev alert.Severity
var bestDesc string
matched := false
for _, p := range dbMalwarePatterns {
if !strings.Contains(lower, strings.ToLower(p.pattern)) {
continue
}
if p.requiresExternalScript {
ok := hasMaliciousExternalScript(body)
if inPostContext {
ok = hasMaliciousExternalScriptInPost(body)
}
if !ok {
continue
}
}
if !matched || p.severity > bestSev {
bestSev = p.severity
bestDesc = p.desc
}
matched = true
}
return bestSev, bestDesc, matched
}
// scanJoomlaSuperUsers detects rogue accounts in the Super Users
// group (group_id = 8 by default). The two-table join is necessary
// because Joomla stores group membership separately from the user
// row; a single rogue admin shows up only when the join fires.
func scanJoomlaSuperUsers(store *state.Store, account string, creds jConfigCreds, prefix string) []alert.Finding {
query := fmt.Sprintf(
"SELECT u.id, u.username, u.email FROM %susers u JOIN %suser_usergroup_map m ON u.id = m.user_id WHERE m.group_id = %d",
prefix, prefix, joomlaSuperUserGroupID)
rows, complete := runCMSQuery(creds.asWPDBCreds(), query)
// The legitimate site admin is in this set too: the store baseline
// keeps known Super Users quiet and reports only a newcomer.
dbCreds := creds.asWPDBCreds()
dbCreds.tablePrefix = prefix
return cmsAdminFindings(store, "joomla", "joomla_admin_injection", account, dbCreds, rows, complete, func(fields []string) (string, string) {
return fmt.Sprintf("Joomla Super User account on %s: %s", account, fields[0]),
fmt.Sprintf("Account: %s\nRow: %s\nReview: confirm this is the legitimate site administrator.", account, strings.Join(fields, "\t"))
})
}
// paramsLikeClause builds an OR'd LIKE clause over the supplied
// columns and the existing dbMalwarePatterns list. The patterns
// are escaped for MySQL string literal syntax (single quotes
// doubled, backslashes doubled).
//
// We use plain LIKE rather than full-text search so the query
// runs against any MySQL configuration -- some shared-hosting
// instances have ngram FTS off, and we don't want to depend on
// it for a security scan.
func paramsLikeClause(columns ...string) string {
if len(columns) == 0 {
return "1=0"
}
var clauses []string
for _, col := range columns {
for _, p := range dbMalwarePatterns {
lit := mysqlEscapeForLike(p.pattern)
clauses = append(clauses, fmt.Sprintf("%s LIKE '%%%s%%'", col, lit))
}
}
return strings.Join(clauses, " OR ")
}
// mysqlEscapeForLike escapes a literal for use inside a single-quoted
// MySQL LIKE pattern. Only `'` and `\` need escaping; LIKE's `%` and
// `_` are intentionally left alone because the malware-pattern list
// uses literal substrings and never SQL wildcards.
func mysqlEscapeForLike(s string) string {
s = strings.ReplaceAll(s, `\`, `\\`)
s = strings.ReplaceAll(s, `'`, `\'`)
return s
}
package checks
import (
"context"
"encoding/xml"
"fmt"
"path/filepath"
"regexp"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mysqlclient"
"github.com/pidginhost/csm/internal/state"
)
// Magento database content scanner.
//
// Single file covering both major versions: M1 (Magento 1.x, EOL'd
// June 2020 but still found on legacy hosts) and M2 (Magento 2.x).
// The two versions share table names and content shape but disagree
// on configuration file format -- M1 stores credentials in
// app/etc/local.xml as XML, M2 stores them in app/etc/env.php as
// PHP arrays.
//
// Discovery is mutually exclusive: a host either has app/etc/env.php
// (M2) or app/etc/local.xml (M1), never both. We probe in that order
// because M2 is the actively maintained version and we want fresh
// installs to be picked up first.
//
// Scanned tables (identical between M1 and M2):
//
// core_config_data (path, value) -- settings.
// web/unsecure/base_url is the
// canonical hijack target;
// attackers redirect the storefront
// by overwriting it.
// catalog_product_entity_text product description text;
// spam-injection vector for SEO
// cms_block + cms_page CMS-managed content; same
// pattern as the product text scan
// admin_user backend administrator accounts.
// One Warning per row -- legitimate
// admin shows up too.
//
// Three new finding categories with CMS-explicit names:
// magento_settings_injection, magento_content_injection,
// magento_admin_injection.
// magentoCreds carries the parsed connection details. Mirrors
// jConfigCreds / drupalCreds; the version field tells the scanner
// which discovery path produced the creds (useful for messages).
type magentoCreds struct {
// ctx ties every query for this install to the runner's deadline.
ctx context.Context
dbName string
dbUser string
dbPass string
dbHost string
dbPrefix string
version string // "M1" | "M2"
path string
queryState *dbQueryState
}
func (c magentoCreds) asWPDBCreds() wpDBCreds {
return wpDBCreds{
dbName: c.dbName,
dbUser: c.dbUser,
dbPass: c.dbPass,
dbHost: c.dbHost,
tablePrefix: c.dbPrefix,
queryCtx: c.ctx,
queryOwner: "db_content_magento",
queryState: c.queryState,
}
}
// magentoM1XMLRoot is the minimum struct surface encoding/xml needs
// to extract the connection block out of a Magento 1.x local.xml.
// CDATA wrapping is transparent to the decoder; both the bare and
// CDATA-wrapped forms produce the same string value.
type magentoM1XMLRoot struct {
XMLName xml.Name `xml:"config"`
Connection magentoM1XMLConnBlock `xml:"global>resources>default_setup>connection"`
Resources magentoM1XMLResources `xml:"global>resources>db"`
}
type magentoM1XMLConnBlock struct {
Host string `xml:"host"`
Username string `xml:"username"`
Password string `xml:"password"`
DBName string `xml:"dbname"`
}
type magentoM1XMLResources struct {
TablePrefix string `xml:"table_prefix"`
}
// M2 env.php is a PHP file returning an array; we extract by regex
// rather than wiring a PHP parser. The patterns match the canonical
// nested-array layout that vendor/magento installers produce; hand-
// rolled env.php files with reordered keys still parse because
// each pattern matches independently.
var (
magentoM2HostRe = regexp.MustCompile(`['"]host['"]\s*=>\s*['"]([^'"]+)['"]`)
magentoM2UserRe = regexp.MustCompile(`['"]username['"]\s*=>\s*['"]([^'"]+)['"]`)
magentoM2PassRe = regexp.MustCompile(`['"]password['"]\s*=>\s*['"]([^'"]+)['"]`)
magentoM2DBRe = regexp.MustCompile(`['"]dbname['"]\s*=>\s*['"]([^'"]+)['"]`)
magentoM2PrefixRe = regexp.MustCompile(`['"]table_prefix['"]\s*=>\s*['"]([^'"]*)['"]`)
)
// CheckMagentoContent discovers Magento installs (M1 + M2) and
// scans the four canonical tables. Mirrors CheckJoomlaContent and
// CheckDrupalContent without sharing code -- the version-branching
// is local to this scanner.
//
// Accounts that produced creds via the M2 (env.php) path are
// tracked in seenAccounts so the M1 fallback doesn't re-scan a
// host that's already been processed -- including the common case
// where M2 found zero malware findings (a clean install). Without
// this, a half-migrated host with both env.php and stale local.xml
// would scan the database twice with different credential sets.
// scanMagentoInstall scans one discovered install (either configuration
// layout) and stamps its findings with the owner resolved from the
// configuration path. The display label stays as before; an install
// outside every account root is not stamped.
func scanMagentoInstall(ctx context.Context, path, account string, creds magentoCreds, store *state.Store) []alert.Finding {
creds.ctx = ctx
creds.queryState = new(dbQueryState)
findings := scanMagentoAll(store, account, creds)
if owner, ok := installOwner(path); ok {
findings = stampTenantIDIfEmpty(findings, owner)
}
return findings
}
func CheckMagentoContent(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
var findings []alert.Finding
seenAccounts := map[string]bool{}
// M2 discovery first (active version). Rank by mtime desc so recently
// touched installs are processed first when the check timeout cuts
// iteration short.
m2Files := cmsDropSuspended(cmsDiscover(ctx, "db_content_magento", "*/public_html/app/etc/env.php", "*/*/app/etc/env.php"))
for _, path := range rankCMSConfigs(ctx, "db_content_magento", m2Files, accountScanMaxFiles(ctx, cfg)) {
if ctx.Err() != nil {
return findings
}
account := magentoAccountFromPath(path)
creds, err := parseMagentoM2(ctx, path)
if err != nil || creds.dbName == "" || creds.dbUser == "" {
markCheckIncomplete(ctx, "db_content_magento")
continue
}
seenAccounts[account] = true
findings = append(findings, scanMagentoInstall(ctx, path, account, creds, store)...)
}
// M1 fallback for hosts where env.php is absent or unparseable.
m1Files := cmsDropSuspended(cmsDiscover(ctx, "db_content_magento", "*/public_html/app/etc/local.xml", "*/*/app/etc/local.xml"))
for _, path := range rankCMSConfigs(ctx, "db_content_magento", m1Files, accountScanMaxFiles(ctx, cfg)) {
if ctx.Err() != nil {
return findings
}
account := magentoAccountFromPath(path)
if seenAccounts[account] {
continue
}
creds, err := parseMagentoM1(ctx, path)
if err != nil || creds.dbName == "" || creds.dbUser == "" {
markCheckIncomplete(ctx, "db_content_magento")
continue
}
findings = append(findings, scanMagentoInstall(ctx, path, account, creds, store)...)
}
return findings
}
// magentoAccountFromPath strips the conventional cPanel prefix
// (/home/<account>/public_html/app/etc/...) down to the account
// component.
func magentoAccountFromPath(path string) string {
// /home/<account>/public_html/app/etc/<file> -- four Dirs up.
cur := path
for i := 0; i < 4; i++ {
cur = filepath.Dir(cur)
}
return extractUser(cur)
}
// parseMagentoM1 reads local.xml and extracts the connection block.
// A read or XML error keeps the installation out of the completed scan.
func parseMagentoM1(ctx context.Context, path string) (magentoCreds, error) {
creds := magentoCreds{path: path, version: "M1"}
data, err := readCMSConfig(ctx, path)
if err != nil {
return creds, err
}
var root magentoM1XMLRoot
if err := xml.Unmarshal(data, &root); err != nil {
return creds, err
}
creds.dbHost = strings.TrimSpace(root.Connection.Host)
creds.dbUser = strings.TrimSpace(root.Connection.Username)
creds.dbPass = strings.TrimSpace(root.Connection.Password)
creds.dbName = strings.TrimSpace(root.Connection.DBName)
creds.dbPrefix = strings.TrimSpace(root.Resources.TablePrefix)
if creds.dbHost == "" {
creds.dbHost = "localhost"
}
return creds, nil
}
// parseMagentoM2 reads env.php and pulls credentials out via the
// field-level regexes. Unlike Drupal we have a stable nested-array
// layout to match against (the one Magento Setup writes), but to
// stay robust against operator-edited env.php files we match each
// key independently.
func parseMagentoM2(ctx context.Context, path string) (magentoCreds, error) {
creds := magentoCreds{path: path, version: "M2"}
data, err := readCMSConfig(ctx, path)
if err != nil {
return creds, err
}
body := string(data)
if m := magentoM2HostRe.FindStringSubmatch(body); m != nil {
creds.dbHost = m[1]
}
if m := magentoM2UserRe.FindStringSubmatch(body); m != nil {
creds.dbUser = m[1]
}
if m := magentoM2PassRe.FindStringSubmatch(body); m != nil {
creds.dbPass = m[1]
}
if m := magentoM2DBRe.FindStringSubmatch(body); m != nil {
creds.dbName = m[1]
}
if m := magentoM2PrefixRe.FindStringSubmatch(body); m != nil {
creds.dbPrefix = m[1]
}
if creds.dbHost == "" {
creds.dbHost = "localhost"
}
return creds, nil
}
// scanMagentoAll runs the four scan paths against one Magento
// install. Helper exists so M1 and M2 dispatch through the same
// post-creds code path.
func scanMagentoAll(store *state.Store, account string, creds magentoCreds) []alert.Finding {
var findings []alert.Finding
findings = append(findings, scanMagentoSettings(account, creds)...)
findings = append(findings, scanMagentoContent(account, creds, "catalog_product_entity_text", "value")...)
findings = append(findings, scanMagentoContent(account, creds, "cms_block", "content")...)
findings = append(findings, scanMagentoContent(account, creds, "cms_page", "content")...)
findings = append(findings, scanMagentoAdmins(store, account, creds)...)
return findings
}
// scanMagentoSettings looks for malware patterns in core_config_data
// values. The path column carries dotted-namespace identifiers
// (web/unsecure/base_url, design/header/welcome, etc.) so we keep
// it in the finding details for triage.
func scanMagentoSettings(account string, creds magentoCreds) []alert.Finding {
query := fmt.Sprintf(
"SELECT path, value FROM %score_config_data WHERE %s",
creds.dbPrefix, paramsLikeClause("value"))
rows, _ := runCMSQuery(creds.asWPDBCreds(), query)
var findings []alert.Finding
for _, row := range rows {
cfgPath, body := splitTabRow(row)
if cfgPath == "" {
continue
}
sev, desc, ok := classifyMalwareRow(mysqlclient.BatchUnescape(body), false)
if !ok {
continue
}
findings = append(findings, alert.Finding{
Severity: sev,
Check: "magento_settings_injection",
Message: fmt.Sprintf("Magento %s settings injection on %s: %s (%s)", creds.version, account, cfgPath, desc),
Details: fmt.Sprintf("Account: %s\nConfig path: %s\nMatch: %s", account, cfgPath, desc),
})
}
return findings
}
// scanMagentoContent walks one CMS table (catalog_product_entity_text,
// cms_block, cms_page) for malware patterns. The looser
// hasMaliciousExternalScriptInPost predicate applies because the
// tables carry author-written content.
func scanMagentoContent(account string, creds magentoCreds, table, valueCol string) []alert.Finding {
idCol := "row_id"
switch table {
case "catalog_product_entity_text":
idCol = "entity_id"
case "cms_block":
idCol = "block_id"
case "cms_page":
idCol = "page_id"
}
query := fmt.Sprintf(
"SELECT %s, %s FROM %s%s WHERE %s",
idCol, valueCol, creds.dbPrefix, table, paramsLikeClause(valueCol))
rows, _ := runCMSQuery(creds.asWPDBCreds(), query)
var findings []alert.Finding
for _, row := range rows {
id, body := splitTabRow(row)
if id == "" {
continue
}
sev, desc, ok := classifyMalwareRow(mysqlclient.BatchUnescape(body), true)
if !ok {
continue
}
findings = append(findings, alert.Finding{
Severity: sev,
Check: "magento_content_injection",
Message: fmt.Sprintf("Magento %s content injection on %s: %s id=%s (%s)", creds.version, account, table, id, desc),
Details: fmt.Sprintf("Account: %s\nTable: %s\nRow id: %s\nMatch: %s", account, table, id, desc),
})
}
return findings
}
// scanMagentoAdmins enumerates the admin_user table. Rows include
// the legitimate site admin -- one Warning per row, operator
// review territory.
func scanMagentoAdmins(store *state.Store, account string, creds magentoCreds) []alert.Finding {
query := fmt.Sprintf(
"SELECT user_id, username, email FROM %sadmin_user",
creds.dbPrefix)
rows, complete := runCMSQuery(creds.asWPDBCreds(), query)
return cmsAdminFindings(store, "magento", "magento_admin_injection", account, creds.asWPDBCreds(), rows, complete, func(fields []string) (string, string) {
return fmt.Sprintf("Magento %s admin account on %s: user_id=%s", creds.version, account, fields[0]),
fmt.Sprintf("Account: %s\nRow: %s\nReview: confirm this is the legitimate site administrator.", account, strings.Join(fields, "\t"))
})
}
package checks
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mysqlclient"
"github.com/pidginhost/csm/internal/state"
)
// OpenCart database content scanner.
//
// Discovery: glob /home/*/public_html/config.php and confirm both
// it AND /home/*/public_html/admin/config.php contain
// `define('DB_DRIVER'`. The admin-side config.php pair is the
// canonical OpenCart marker -- plain PHP sites carry a config.php
// in the document root that's nothing to do with OpenCart.
//
// Credentials use PHP define() constants with OC-specific names:
//
// DB_HOSTNAME DB_USERNAME DB_PASSWORD DB_DATABASE DB_PREFIX
//
// Reuses the existing extractDefine helper from dbscan.go (the WP
// scanner already understands this shape). DB_PREFIX defaults to
// "oc_" on vanilla installs.
//
// Scanned tables (all prefixed):
//
// <prefix>setting k/v pairs; values are JSON
// blobs. config_url / config_ssl
// are the canonical hijack
// targets for storefront redirect.
// <prefix>product_description product description text
// <prefix>information_description CMS-managed information pages
// <prefix>user admin/staff accounts.
// Customer accounts live in the
// oc_customer table, not here;
// every oc_user row is admin-shaped.
//
// Three new finding categories:
// opencart_settings_injection, opencart_content_injection,
// opencart_admin_injection.
type opencartCreds struct {
// ctx ties every query for this install to the runner's deadline.
ctx context.Context
dbName string
dbUser string
dbPass string
dbHost string
dbPrefix string
path string
queryState *dbQueryState
}
func (c opencartCreds) asWPDBCreds() wpDBCreds {
return wpDBCreds{
dbName: c.dbName,
dbUser: c.dbUser,
dbPass: c.dbPass,
dbHost: c.dbHost,
tablePrefix: c.dbPrefix,
queryCtx: c.ctx,
queryOwner: "db_content_opencart",
queryState: c.queryState,
}
}
// CheckOpenCartContent discovers OpenCart installs and scans the
// four canonical attacker-touched tables. Mirrors the other CMS
// scanners; the discovery and credentials parsing are the only
// OC-specific bits.
// scanOpenCartInstall scans one discovered install and stamps its findings
// with the owner resolved from the configuration path. The display label
// stays as before; an install outside every account root is not stamped.
func scanOpenCartInstall(ctx context.Context, path string, store *state.Store) []alert.Finding {
matched, err := looksLikeOpenCart(ctx, path)
if err != nil {
markCheckIncomplete(ctx, "db_content_opencart")
return nil
}
if !matched {
return nil
}
account := extractUser(filepath.Dir(path))
creds, err := parseOpenCartConfig(ctx, path)
if err != nil || creds.dbName == "" || creds.dbUser == "" {
markCheckIncomplete(ctx, "db_content_opencart")
return nil
}
creds.ctx = ctx
creds.queryState = new(dbQueryState)
prefix := creds.dbPrefix
if prefix == "" {
prefix = "oc_"
}
creds.dbPrefix = prefix
var findings []alert.Finding
findings = append(findings, scanOpenCartSettings(account, creds)...)
findings = append(findings, scanOpenCartContentTable(account, creds, "product_description", "description")...)
findings = append(findings, scanOpenCartContentTable(account, creds, "information_description", "description")...)
findings = append(findings, scanOpenCartAdmins(store, account, creds)...)
if owner, ok := installOwner(path); ok {
findings = stampTenantIDIfEmpty(findings, owner)
}
return findings
}
func CheckOpenCartContent(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
return scanCMSInstalls(ctx, cfg, store, "db_content_opencart", scanOpenCartInstall,
"*/public_html/config.php", "*/*/config.php")
}
// looksLikeOpenCart confirms both config.php files exist and both
// reference DB_DRIVER. The admin-side file is what distinguishes
// OpenCart from arbitrary PHP sites that happen to ship a
// config.php at the document root.
func looksLikeOpenCart(ctx context.Context, rootConfig string) (bool, error) {
matched, err := configContainsDBDriver(ctx, rootConfig)
if err != nil || !matched {
return false, err
}
adminConfig := filepath.Join(filepath.Dir(rootConfig), "admin", "config.php")
matched, err = configContainsDBDriver(ctx, adminConfig)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
return matched, err
}
func configContainsDBDriver(ctx context.Context, path string) (bool, error) {
data, err := readCMSConfig(ctx, path)
if err != nil {
return false, err
}
return strings.Contains(string(data), "DB_DRIVER"), nil
}
// parseOpenCartConfig extracts the DB_* defines from a config.php.
// Reuses the WP scanner's extractDefine helper -- the OC defines
// have the same `define('KEY', 'value')` shape WP uses, and the
// helper already strips comments and walks past the key's closing
// quote correctly.
func parseOpenCartConfig(ctx context.Context, path string) (opencartCreds, error) {
creds := opencartCreds{path: path}
data, err := readCMSConfig(ctx, path)
if err != nil {
return creds, err
}
for _, line := range strings.Split(string(data), "\n") {
if v := extractDefine(line, "DB_HOSTNAME"); v != "" {
creds.dbHost = v
}
if v := extractDefine(line, "DB_USERNAME"); v != "" {
creds.dbUser = v
}
if v := extractDefine(line, "DB_PASSWORD"); v != "" {
creds.dbPass = v
}
if v := extractDefine(line, "DB_DATABASE"); v != "" {
creds.dbName = v
}
if v := extractDefine(line, "DB_PREFIX"); v != "" {
creds.dbPrefix = v
}
}
if creds.dbHost == "" {
creds.dbHost = "localhost"
}
return creds, nil
}
// scanOpenCartSettings walks oc_setting k/v rows. The value column
// is a JSON-serialized blob; same external-script post-filter as
// the other CMS settings scanners (strict variant -- this is
// config storage, not author-written content).
func scanOpenCartSettings(account string, creds opencartCreds) []alert.Finding {
query := fmt.Sprintf(
"SELECT `key`, value FROM %ssetting WHERE %s",
creds.dbPrefix, paramsLikeClause("value"))
rows, _ := runCMSQuery(creds.asWPDBCreds(), query)
var findings []alert.Finding
for _, row := range rows {
key, body := splitTabRow(row)
if key == "" {
continue
}
sev, desc, ok := classifyMalwareRow(mysqlclient.BatchUnescape(body), false)
if !ok {
continue
}
findings = append(findings, alert.Finding{
Severity: sev,
Check: "opencart_settings_injection",
Message: fmt.Sprintf("OpenCart settings injection on %s: %s (%s)", account, key, desc),
Details: fmt.Sprintf("Account: %s\nSetting key: %s\nMatch: %s", account, key, desc),
})
}
return findings
}
// scanOpenCartContentTable walks one of the description tables
// (product_description, information_description). Both have an id
// column and a description column; the id-column name varies but
// the schema is consistent enough that we accept it as a parameter.
//
// Looser post-filter (hasMaliciousExternalScriptInPost) because
// these tables carry author-written content.
//
// Both description tables carry one row per language per product
// or page. Without filtering, a multilingual storefront emits N
// findings per malware-injected row (one per installed language).
// language_id = 1 is English / the vanilla default; non-English
// monolingual sites and genuine multilingual coverage need a
// follow-up that reads config_language_id from oc_setting first.
func scanOpenCartContentTable(account string, creds opencartCreds, table, valueCol string) []alert.Finding {
idCol := "product_id"
if table == "information_description" {
idCol = "information_id"
}
query := fmt.Sprintf(
"SELECT %s, %s FROM %s%s WHERE language_id = 1 AND %s",
idCol, valueCol, creds.dbPrefix, table, paramsLikeClause(valueCol))
rows, _ := runCMSQuery(creds.asWPDBCreds(), query)
var findings []alert.Finding
for _, row := range rows {
id, body := splitTabRow(row)
if id == "" {
continue
}
sev, desc, ok := classifyMalwareRow(mysqlclient.BatchUnescape(body), true)
if !ok {
continue
}
findings = append(findings, alert.Finding{
Severity: sev,
Check: "opencart_content_injection",
Message: fmt.Sprintf("OpenCart content injection on %s: %s id=%s (%s)", account, table, id, desc),
Details: fmt.Sprintf("Account: %s\nTable: %s\nRow id: %s\nMatch: %s", account, table, id, desc),
})
}
return findings
}
// scanOpenCartAdmins enumerates the oc_user table (admins/staff,
// not customers -- customers live in oc_customer). Same Warning
// per row as the other CMS adapters.
func scanOpenCartAdmins(store *state.Store, account string, creds opencartCreds) []alert.Finding {
query := fmt.Sprintf(
"SELECT user_id, username, email FROM %suser",
creds.dbPrefix)
rows, complete := runCMSQuery(creds.asWPDBCreds(), query)
return cmsAdminFindings(store, "opencart", "opencart_admin_injection", account, creds.asWPDBCreds(), rows, complete, func(fields []string) (string, string) {
return fmt.Sprintf("OpenCart admin account on %s: user_id=%s", account, fields[0]),
fmt.Sprintf("Account: %s\nRow: %s\nReview: confirm this is the legitimate site administrator.", account, strings.Join(fields, "\t"))
})
}
package checks
import (
"fmt"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/alert"
)
// phantomAuthorFarmSize is the published-post count at which a phantom author
// is treated as a content farm rather than a likely orphan. A direct SQL user
// deletion can leave a small number of posts behind; a large group warrants a
// Critical alert.
const phantomAuthorFarmSize = 100
// maxPhantomAuthorsReported bounds the finding count across one installation,
// including all blogs in a multisite network.
const maxPhantomAuthorsReported = 25
// checkWPPhantomAuthors finds published posts whose post_author has no row in
// the network users table. postsPrefix and usersPrefix are kept separate so a
// multisite blog can join its own posts table to the shared users table.
// Callers derive both from resolveTablePrefix, appending only a numeric blog ID
// to postsPrefix, before either value reaches SQL construction.
//
// Doorway kits attribute their pages to invented author IDs and then install a
// filter that removes those IDs from admin queries, patching the post counts to
// match. The dashboard therefore shows a clean site while the posts are served
// to visitors and listed in crawler-facing sitemaps. The orphaned author ID is
// what the cloak cannot hide: it is a property of the data, not of the code
// doing the hiding, so it survives every renaming and re-obfuscation of the
// filter itself.
func checkWPPhantomAuthors(user string, creds wpDBCreds, postsPrefix, usersPrefix string, limit int) []alert.Finding {
var findings []alert.Finding
if limit <= 0 {
return findings
}
query := fmt.Sprintf(
"SELECT p.post_author, COUNT(*) AS c FROM %sposts p "+
"LEFT JOIN %susers u ON u.ID = p.post_author "+
"WHERE u.ID IS NULL AND p.post_author <> 0 "+
"AND p.post_type = 'post' AND p.post_status = 'publish' "+
"GROUP BY p.post_author ORDER BY c DESC LIMIT %d",
postsPrefix, usersPrefix, limit)
for _, line := range runMySQLQuery(creds, query) {
parts := strings.SplitN(strings.TrimSpace(line), "\t", 2)
if len(parts) != 2 {
continue
}
authorID, err := strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 64)
if err != nil || authorID == 0 {
continue
}
count, err := strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 64)
if err != nil || count == 0 {
continue
}
severity := alert.Warning
if count >= phantomAuthorFarmSize {
severity = alert.Critical
}
postNoun := "posts"
if count == 1 {
postNoun = "post"
}
findings = append(findings, alert.Finding{
Severity: severity,
Check: "db_phantom_post_author",
Message: fmt.Sprintf("%d published %s are attributed to a non-existent user (account: %s, author ID %d)",
count, postNoun, user, authorID),
Details: dbContentFindingDetails(creds, postsPrefix,
fmt.Sprintf("post_author = %d has no row in %susers.\n"+
"A small number can remain after an administrator deletes a user directly in SQL. "+
"Large groups can indicate hidden or injected content.",
authorID, usersPrefix)),
// Crossing into a content farm must alert even if the orphan group
// was baselined or dismissed. Counts within either tier stay stable.
DedupKey: dbContentDedupKey(user, creds, postsPrefix,
"farm="+strconv.FormatBool(count >= phantomAuthorFarmSize),
fmt.Sprintf("post_author = %d has no row in %susers.\n"+
"A small number can remain after an administrator deletes a user directly in SQL. "+
"Large groups can indicate hidden or injected content.",
authorID, usersPrefix)),
})
if len(findings) >= limit {
break
}
}
return findings
}
// capPhantomAuthorFindings bounds one installation's alert volume after every
// multisite posts table has been checked. Critical farms take precedence over
// likely orphan warnings, so a noisy primary blog cannot hide a compromised
// secondary blog by consuming the cap first.
func capPhantomAuthorFindings(findings []alert.Finding, limit int) []alert.Finding {
selected := make([]bool, len(findings))
remaining := limit
for i, finding := range findings {
if remaining == 0 {
break
}
if finding.Check == "db_phantom_post_author" && finding.Severity == alert.Critical {
selected[i] = true
remaining--
}
}
for i, finding := range findings {
if remaining == 0 {
break
}
if finding.Check == "db_phantom_post_author" && !selected[i] {
selected[i] = true
remaining--
}
}
out := make([]alert.Finding, 0, len(findings))
for i, finding := range findings {
if finding.Check != "db_phantom_post_author" || selected[i] {
out = append(out, finding)
}
}
return out
}
package checks
import (
"fmt"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/alert"
)
const (
// postBurstWindowDays is the recent period compared against everything the
// site published before it.
postBurstWindowDays = 365
// postBurstMinSiteAgeDays keeps a young site quiet. A launch legitimately
// produces its whole archive at once, and there is no history to judge it
// against.
postBurstMinSiteAgeDays = 400
// postBurstMinRecent avoids reporting ordinary editorial activity on a very
// quiet site, where any uptick is a large multiple of almost nothing.
postBurstMinRecent = 50
// postBurstRatio is how many times the site's entire prior output the recent
// window must exceed. A steady publisher never reaches it; a doorway kit
// clears it by an order of magnitude.
postBurstRatio = 5
)
// checkWPPostVolumeBurst reports a site that suddenly publishes far more than
// it ever did before.
//
// This is the keyword-free half of spam detection. A live compromise published
// 508 posts across seven languages in under a year on a site that had managed
// 17 in the preceding seven; a gambling word list matched fewer than half of
// them, while the change in publishing rate separated spam from real content
// exactly. It says nothing about what the posts contain, which is the point --
// the next kit will use a different vocabulary.
func checkWPPostVolumeBurst(user string, creds wpDBCreds, prefix string) []alert.Finding {
query := fmt.Sprintf(
"SELECT DATEDIFF(NOW(), MIN(post_date)) AS site_age_days, "+
"SUM(post_date < DATE_SUB(NOW(), INTERVAL %d DAY)) AS prior_posts, "+
"SUM(post_date >= DATE_SUB(NOW(), INTERVAL %d DAY)) AS recent_posts "+
"FROM %sposts WHERE post_type = 'post' AND post_status = 'publish'",
postBurstWindowDays, postBurstWindowDays, prefix)
rows := runMySQLQuery(creds, query)
if len(rows) == 0 {
return nil
}
parts := strings.SplitN(strings.TrimSpace(rows[0]), "\t", 3)
if len(parts) != 3 {
return nil
}
ageDays, err1 := strconv.Atoi(strings.TrimSpace(parts[0]))
prior, err2 := strconv.Atoi(strings.TrimSpace(parts[1]))
recent, err3 := strconv.Atoi(strings.TrimSpace(parts[2]))
if err1 != nil || err2 != nil || err3 != nil {
return nil
}
if ageDays < postBurstMinSiteAgeDays || recent < postBurstMinRecent {
return nil
}
// prior == 0 on an established site means every post it has was published
// in the recent window, which is the strongest form of this signal rather
// than a division-by-zero edge case.
if prior > 0 && recent < prior*postBurstRatio {
return nil
}
return []alert.Finding{{
Severity: alert.High,
Check: "db_post_volume_burst",
Message: fmt.Sprintf("WordPress published %d posts in the last year against %d in the %d years before (account: %s)",
recent, prior, ageDays/365, user),
Details: dbContentFindingDetails(creds, prefix,
fmt.Sprintf("Site has been publishing for %d days. A sudden flood on a long-quiet site is how "+
"doorway spam arrives, and it is visible without knowing what language or vocabulary the "+
"spam uses.\nReview the recent posts before acting: a genuine content migration looks the same.",
ageDays)),
// One burst per site. The details count days and posts, all of which
// move on their own between scans without the burst being a new one.
DedupKey: dbContentDedupKey(user, creds, prefix),
}}
}
package checks
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mysqlclient"
"github.com/pidginhost/csm/internal/state"
)
// PrestaShop database content scanner.
//
// Discovery: <root>/config/defines.inc.php containing _PS_ROOT_DIR_ marks a
// PrestaShop tree (1.6 through 9.x). Credentials come from
// app/config/parameters.php (1.7.4 and later). A root with only
// app/config/parameters.yml is 1.7.0 to 1.7.3 and is not read; a root with
// neither falls back to the 1.6 config/settings.inc.php defines.
//
// parameters.php also holds the mailer password, the application secret,
// cookie keys and an API private key. Only the six database_* entries are
// ever looked up.
//
// Scanned tables (all prefixed): configuration and configuration_lang
// (settings), cms_lang, product_lang and category_lang (content), and
// employee (back-office accounts). Findings:
// prestashop_settings_injection, prestashop_content_injection,
// prestashop_admin_injection.
const prestashopOwner = "db_content_prestashop"
var (
errPrestaShopYAMLConfig = errors.New("PrestaShop 1.7.0-1.7.3 configuration is not read")
errPrestaShopCreds = errors.New("PrestaShop database credentials are incomplete or invalid")
)
type prestashopCreds struct {
// ctx ties every query for this install to the runner's deadline.
ctx context.Context
dbName string
dbUser string
dbPass string
dbHost string
dbPrefix string
// path is the configuration file the values came from.
path string
queryState *dbQueryState
}
// CheckPrestaShopContent discovers PrestaShop installs under every account
// root and scans their settings, content and back-office employees.
func CheckPrestaShopContent(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
markers := cmsDiscover(ctx, prestashopOwner, "*/public_html/config/defines.inc.php", "*/*/config/defines.inc.php")
var kept []string
for _, marker := range markers {
// Staging and backup copies query a database the site does not serve.
if wpSkipDirNames[strings.ToLower(filepath.Base(prestashopRoot(marker)))] {
continue
}
kept = append(kept, marker)
}
kept = cmsDropSuspended(kept)
if len(kept) == 0 {
return nil
}
var findings []alert.Finding
// Rank by mtime desc so recently touched installs are scanned first when
// the check timeout cuts iteration short.
for _, marker := range rankCMSConfigs(ctx, prestashopOwner, kept, accountScanMaxFiles(ctx, cfg)) {
if ctx.Err() != nil {
return findings
}
findings = append(findings, scanPrestaShopInstall(ctx, prestashopRoot(marker), store)...)
}
return findings
}
func prestashopRoot(marker string) string {
return filepath.Dir(filepath.Dir(marker))
}
// scanPrestaShopInstall scans one install and stamps its findings with the
// owner resolved from the configuration path.
func scanPrestaShopInstall(ctx context.Context, root string, store *state.Store) []alert.Finding {
creds, isShop, err := readPrestaShopCreds(ctx, root)
if err != nil {
markCheckIncomplete(ctx, prestashopOwner)
return nil
}
if !isShop {
return nil
}
owner, owned := installOwner(creds.path)
account := extractUser(root)
creds.ctx = ctx
creds.queryState = new(dbQueryState)
// Establish employee state before unrelated settings or content limits
// mark the installation incomplete and prevent baseline updates.
findings := scanPrestaShopEmployees(store, account, creds)
findings = append(findings, scanPrestaShopSettings(account, creds)...)
findings = append(findings, scanPrestaShopContent(account, creds)...)
if owned {
findings = stampTenantIDIfEmpty(findings, owner)
}
return findings
}
func (c prestashopCreds) asWPDBCreds() wpDBCreds {
return wpDBCreds{
dbName: c.dbName,
dbUser: c.dbUser,
dbPass: c.dbPass,
dbHost: c.dbHost,
tablePrefix: c.dbPrefix,
queryCtx: c.ctx,
queryOwner: prestashopOwner,
queryState: c.queryState,
}
}
// prestashopMarker reports whether defines.inc.php is PrestaShop's: without
// it, app/config/parameters.php alone could belong to any Symfony app.
func prestashopMarker(ctx context.Context, definesPath string) (bool, error) {
data, err := readCMSConfig(ctx, definesPath)
if err != nil {
return false, err
}
return strings.Contains(string(data), "_PS_ROOT_DIR_"), nil
}
// readPrestaShopCreds reads the live database credentials of the install at
// root. isShop is false when root is not a PrestaShop install, or is one with
// no configuration yet; an error with isShop true leaves the scan incomplete.
func readPrestaShopCreds(ctx context.Context, root string) (prestashopCreds, bool, error) {
matched, err := prestashopMarker(ctx, filepath.Join(root, "config", "defines.inc.php"))
if errors.Is(err, os.ErrNotExist) {
return prestashopCreds{}, false, nil
}
if err != nil {
return prestashopCreds{}, false, err
}
if !matched {
return prestashopCreds{}, false, nil
}
creds, err := parsePrestaShopParameters(ctx, filepath.Join(root, "app", "config", "parameters.php"))
if errors.Is(err, os.ErrNotExist) {
// parameters.yml is the live config on 1.7.0 to 1.7.3; a settings.inc.php
// beside it is a 1.6 leftover that may name a stale database.
_, yamlErr := readCMSConfig(ctx, filepath.Join(root, "app", "config", "parameters.yml"))
if yamlErr == nil {
return prestashopCreds{}, true, errPrestaShopYAMLConfig
}
if !errors.Is(yamlErr, os.ErrNotExist) {
return prestashopCreds{}, true, yamlErr
}
creds, err = parsePrestaShopSettingsInc(ctx, filepath.Join(root, "config", "settings.inc.php"))
if errors.Is(err, os.ErrNotExist) {
return prestashopCreds{}, false, nil
}
}
if err != nil {
return prestashopCreds{}, true, err
}
if creds.dbName == "" || creds.dbUser == "" || (creds.dbPrefix != "" && !validTablePrefix.MatchString(creds.dbPrefix)) {
return prestashopCreds{}, true, errPrestaShopCreds
}
return creds, true, nil
}
// prestashopParamKeys are the only parameters.php entries ever looked up.
var prestashopParamKeys = []string{
"database_host", "database_port", "database_name",
"database_user", "database_password", "database_prefix",
}
type prestashopParamValue struct {
value string
quoted bool
found bool
}
// parsePrestaShopParameters reads the database entries of parameters.php.
// PrestaShop writes the file with var_export after doubling every %, and its
// bootstrap halves them again, so the same is done here.
func parsePrestaShopParameters(ctx context.Context, path string) (prestashopCreds, error) {
data, err := readCMSConfig(ctx, path)
if err != nil {
return prestashopCreds{}, err
}
vals, err := parsePrestaShopParameterValues(string(data))
if err != nil {
return prestashopCreds{}, err
}
for _, key := range []string{"database_name", "database_user", "database_prefix"} {
if !vals[key].found || !vals[key].quoted {
return prestashopCreds{}, fmt.Errorf("%w: %s", errPrestaShopCreds, key)
}
}
for _, key := range []string{"database_host", "database_password"} {
if vals[key].found && !vals[key].quoted {
return prestashopCreds{}, fmt.Errorf("%w: %s", errPrestaShopCreds, key)
}
}
value := func(key string) string { return strings.ReplaceAll(vals[key].value, "%%", "%") }
host := value("database_host")
if host == "" {
host = "localhost"
}
// PrestaShop appends any port PHP does not treat as empty; the installer
// stores a socket path here when the host was given as localhost:/path.
if port := value("database_port"); port != "" && port != "0" {
if !isAllDigits(port) && !strings.HasPrefix(port, "/") {
return prestashopCreds{}, fmt.Errorf("%w: database_port", errPrestaShopCreds)
}
host += ":" + port
}
return prestashopCreds{
dbName: value("database_name"),
dbUser: value("database_user"),
dbPass: value("database_password"),
dbHost: host,
dbPrefix: value("database_prefix"),
path: path,
}, nil
}
// parsePrestaShopParameterValues accepts the returned parameters array only.
// Unrelated values stay opaque, including strings that resemble database keys.
func parsePrestaShopParameterValues(body string) (map[string]prestashopParamValue, error) {
p := prestashopPHPParser{rest: body}
if !p.take("<?php") || len(p.rest) == 0 || !strings.ContainsRune(" \t\r\n", rune(p.rest[0])) || !p.keyword("return") {
return nil, errPrestaShopCreds
}
vals, ok := p.array(false)
if !ok || !p.take(";") {
return nil, errPrestaShopCreds
}
p.take("?>")
if !p.space() || p.rest != "" {
return nil, errPrestaShopCreds
}
return vals, nil
}
type prestashopPHPParser struct {
rest string
}
func (p *prestashopPHPParser) space() bool {
for {
p.rest = strings.TrimLeft(p.rest, " \t\r\n\v\f")
switch {
case strings.HasPrefix(p.rest, "/*"):
end := strings.Index(p.rest[2:], "*/")
if end < 0 {
return false
}
p.rest = p.rest[end+4:]
case strings.HasPrefix(p.rest, "//"), strings.HasPrefix(p.rest, "#"):
end := strings.IndexAny(p.rest, "\r\n")
if end < 0 {
p.rest = ""
} else {
p.rest = p.rest[end:]
}
default:
return true
}
}
}
func (p *prestashopPHPParser) take(token string) bool {
if !p.space() || !strings.HasPrefix(p.rest, token) {
return false
}
p.rest = p.rest[len(token):]
return true
}
func (p *prestashopPHPParser) keyword(word string) bool {
if !p.space() || len(p.rest) < len(word) || !strings.EqualFold(p.rest[:len(word)], word) {
return false
}
if len(p.rest) > len(word) {
c := p.rest[len(word)]
if c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c >= 0x80 {
return false
}
}
p.rest = p.rest[len(word):]
return true
}
func (p *prestashopPHPParser) arrayEnd() (string, bool) {
if p.take("[") {
return "]", true
}
if p.keyword("array") && p.take("(") {
return ")", true
}
return "", false
}
func (p *prestashopPHPParser) array(parameters bool) (map[string]prestashopParamValue, bool) {
end, ok := p.arrayEnd()
if !ok {
return nil, false
}
vals := make(map[string]prestashopParamValue)
for !p.take(end) {
key, quoted, rest, valid := parsePHPScalarLiteral(p.rest)
if !valid || !quoted {
return nil, false
}
p.rest = rest
if !p.take("=>") || !p.space() {
return nil, false
}
switch {
case !parameters && key == "parameters":
// A later parameters entry replaces the whole earlier array.
vals, ok = p.array(true)
case parameters && slices.Contains(prestashopParamKeys, key):
var v prestashopParamValue
v, ok = p.databaseValue(key)
vals[key] = v
default:
ok = p.skipValue()
}
if !ok {
return nil, false
}
if p.take(end) {
return vals, true
}
if !p.take(",") {
return nil, false
}
}
return vals, true
}
func (p *prestashopPHPParser) databaseValue(key string) (prestashopParamValue, bool) {
// PHP empty() also accepts an empty array for the port. No array values
// are decoded or retained.
if key == "database_port" && (strings.HasPrefix(p.rest, "[") || len(p.rest) >= 5 && strings.EqualFold(p.rest[:5], "array")) {
end, ok := p.arrayEnd()
return prestashopParamValue{found: true}, ok && p.take(end)
}
value, quoted, rest, ok := parsePHPScalarLiteral(p.rest)
if !ok {
return prestashopParamValue{}, false
}
p.rest = rest
return prestashopParamValue{value: value, quoted: quoted, found: true}, true
}
// skipString walks a quoted token without decoding or retaining its value.
func (p *prestashopPHPParser) skipString() bool {
if len(p.rest) == 0 || (p.rest[0] != '\'' && p.rest[0] != '"') {
return false
}
quote := p.rest[0]
for i := 1; i < len(p.rest); i++ {
switch p.rest[i] {
case '\\':
i++
case quote:
p.rest = p.rest[i+1:]
return true
}
}
return false
}
// skipValue tracks delimiters iteratively so deeply nested unrelated values
// cannot exhaust the stack. Only the returned array and parameters recurse.
func (p *prestashopPHPParser) skipValue() bool {
var ends []byte
consumed := false
for p.space() && len(p.rest) > 0 {
c := p.rest[0]
switch c {
case '\'', '"':
if !p.skipString() {
return false
}
consumed = true
continue
case '(':
ends = append(ends, ')')
case '[':
ends = append(ends, ']')
case ')', ']':
if len(ends) == 0 {
return consumed
}
if ends[len(ends)-1] != c {
return false
}
ends = ends[:len(ends)-1]
case ',':
if len(ends) == 0 {
return consumed
}
case '=':
if strings.HasPrefix(p.rest, "=>") {
p.rest = p.rest[2:]
consumed = true
continue
}
case ';', '{', '}', '`', '<', '>':
return false
}
consumed = true
p.rest = p.rest[1:]
}
return false
}
// parsePHPScalarLiteral reads a string, a nonnegative PHP integer,
// or a PHP-empty scalar. It never evaluates an expression or interpolation.
func parsePHPScalarLiteral(s string) (value string, quoted bool, rest string, ok bool) {
if s == "" {
return "", false, "", false
}
switch quote := s[0]; quote {
case '\'', '"':
var b strings.Builder
for i := 1; i < len(s); i++ {
c := s[i]
switch {
case c == quote:
return b.String(), true, s[i+1:], true
case c == '\\' && i+1 < len(s) && (s[i+1] == quote || s[i+1] == '\\'):
i++
b.WriteByte(s[i])
case quote == '"' && (c == '\\' || c == '$'):
return "", false, "", false
default:
b.WriteByte(c)
}
}
return "", false, "", false
}
end := strings.IndexAny(s, " \t\r\n\v\f,)]/#")
if end < 0 {
end = len(s)
}
token := s[:end]
switch {
case strings.EqualFold(token, "null"), len(token) == len("false") && strings.EqualFold(token, "false"):
return "", false, s[end:], true
case isAllDigits(token):
// PHP interprets a leading zero as octal and overflowing integers as
// floats. Do not silently turn either into a different decimal port.
base := 10
if len(token) > 1 && token[0] == '0' {
base = 8
}
if v, err := strconv.ParseUint(token, base, 63); err == nil {
return strconv.FormatUint(v, 10), false, s[end:], true
}
case strings.Trim(token, "+-0123456789.eE") == "":
if v, err := strconv.ParseFloat(token, 64); err == nil && v == 0 {
return "", false, s[end:], true
}
}
return "", false, "", false
}
// parsePrestaShopSettingsInc reads the 1.6 settings.inc.php defines. An empty
// prefix cannot be told apart from a missing one here, so it becomes ps_.
func parsePrestaShopSettingsInc(ctx context.Context, path string) (prestashopCreds, error) {
creds := prestashopCreds{path: path}
data, err := readCMSConfig(ctx, path)
if err != nil {
return prestashopCreds{}, err
}
for _, line := range strings.Split(string(data), "\n") {
if v := extractDefine(line, "_DB_SERVER_"); v != "" {
creds.dbHost = v
}
if v := extractDefine(line, "_DB_NAME_"); v != "" {
creds.dbName = v
}
if v := extractDefine(line, "_DB_USER_"); v != "" {
creds.dbUser = v
}
if v := extractDefine(line, "_DB_PASSWD_"); v != "" {
creds.dbPass = v
}
if v := extractDefine(line, "_DB_PREFIX_"); v != "" {
creds.dbPrefix = v
}
}
if creds.dbHost == "" {
creds.dbHost = "localhost"
}
if creds.dbPrefix == "" {
creds.dbPrefix = "ps_"
}
return creds, nil
}
// prestashopContentTable is one author-content table: one row per language
// and shop, keyed by idCol.
type prestashopContentTable struct {
table, idCol string
cols []string
}
var prestashopContentTables = []prestashopContentTable{
{"cms_lang", "id_cms", []string{"content"}},
{"product_lang", "id_product", []string{"description", "description_short"}},
// additional_description exists only from 8.x on, so it is not queried.
{"category_lang", "id_category", []string{"description"}},
}
// prestashopFindingSet keeps one finding per identity, the most severe match
// winning, in first-seen order: a product injected in every language and shop
// is one finding.
type prestashopFindingSet struct {
order []string
byKey map[string]alert.Finding
}
func (s *prestashopFindingSet) add(key string, f alert.Finding) {
if s.byKey == nil {
s.byKey = map[string]alert.Finding{}
}
prev, seen := s.byKey[key]
if !seen {
s.order = append(s.order, key)
}
if !seen || f.Severity > prev.Severity {
s.byKey[key] = f
}
}
func (s *prestashopFindingSet) findings() []alert.Finding {
out := make([]alert.Finding, 0, len(s.order))
for _, key := range s.order {
out = append(out, s.byKey[key])
}
return out
}
func prestashopFindingDetails(creds wpDBCreds, lines ...string) string {
return dbContentFindingDetails(creds, creds.tablePrefix, append([]string{"Database host: " + creds.dbHost}, lines...)...)
}
// scanPrestaShopSettings scans configuration and configuration_lang values
// with the strict config-storage rules. Values are classified, never output:
// PS_MAIL_PASSWD is stored here in plain text.
func scanPrestaShopSettings(account string, creds prestashopCreds) []alert.Finding {
w := creds.asWPDBCreds()
p := creds.dbPrefix
queries := []string{
fmt.Sprintf("SELECT id_configuration, name, value FROM %sconfiguration WHERE %s", p, paramsLikeClause("value")),
fmt.Sprintf("SELECT c.id_configuration, c.name, l.value FROM %sconfiguration_lang l JOIN %sconfiguration c ON c.id_configuration = l.id_configuration WHERE %s",
p, p, paramsLikeClause("l.value")),
}
var set prestashopFindingSet
for _, query := range queries {
rows, _ := runCMSQuery(w, query)
for _, row := range rows {
fields := strings.SplitN(row, "\t", 3)
if len(fields) < 3 || !isAllDigits(fields[0]) {
continue
}
id, name := fields[0], fields[1]
// Rows arrive batch-escaped; classify the stored value, or a tab
// or newline inside a script tag hides it.
sev, desc, ok := classifyMalwareRow(mysqlclient.BatchUnescape(fields[2]), false)
if !ok {
continue
}
set.add(id, alert.Finding{
Severity: sev,
Check: "prestashop_settings_injection",
Message: fmt.Sprintf("PrestaShop settings injection on %s: %s (%s)", account, name, desc),
Details: prestashopFindingDetails(w, "Account: "+account, "Setting key: "+name,
"Setting id: "+id, "Match: "+desc),
DedupKey: dbContentDedupKey(account, w, p, "prestashop-setting", "id="+id),
})
}
}
return set.findings()
}
// scanPrestaShopContent scans page, product and category text in every
// language and shop with the looser author-content rules.
func scanPrestaShopContent(account string, creds prestashopCreds) []alert.Finding {
w := creds.asWPDBCreds()
p := creds.dbPrefix
var set prestashopFindingSet
for _, t := range prestashopContentTables {
for _, col := range t.cols {
rows, _ := runCMSQuery(w, fmt.Sprintf("SELECT %s, %s FROM %s%s WHERE %s",
t.idCol, col, p, t.table, paramsLikeClause(col)))
for _, row := range rows {
id, body := splitTabRow(row)
if !isAllDigits(id) {
continue
}
sev, desc, ok := classifyMalwareRow(mysqlclient.BatchUnescape(body), true)
if !ok {
continue
}
set.add(t.table+"\x00"+id, alert.Finding{
Severity: sev,
Check: "prestashop_content_injection",
Message: fmt.Sprintf("PrestaShop content injection on %s: %s id=%s (%s)", account, t.table, id, desc),
Details: prestashopFindingDetails(w, "Account: "+account, "Table: "+t.table,
"Row id: "+id, "Match: "+desc),
DedupKey: dbContentDedupKey(account, w, p, "prestashop-content", "table="+t.table, "id="+id),
})
}
}
}
return set.findings()
}
const (
// prestashopSuperAdminProfile is _PS_ADMIN_PROFILE_, unchanged from 1.6 to
// 9.2. Profile names are localised, so the id is what identifies it.
prestashopSuperAdminProfile = "1"
prestashopEventNewSuperAdmin = "new SuperAdmin"
prestashopEventPromoted = "promoted to SuperAdmin"
)
// scanPrestaShopEmployees baselines back-office employees like the other shop
// scanners and reports a new or promoted SuperAdmin as Critical. Password
// hashes are never selected.
func scanPrestaShopEmployees(store *state.Store, account string, creds prestashopCreds) []alert.Finding {
w := creds.asWPDBCreds()
rows, complete := runCMSQuery(w, fmt.Sprintf("SELECT id_employee, id_profile, email FROM %semployee", creds.dbPrefix))
describe := func(fields []string) (string, string) {
var profile, email string
if len(fields) > 1 {
profile = fields[1]
}
// The query helper trims trailing whitespace, including an empty email column.
if len(fields) > 2 {
email = fields[2]
}
return fmt.Sprintf("PrestaShop employee on %s: id_employee=%s", account, fields[0]),
fmt.Sprintf("Account: %s\nRow: %s\nEmployee id: %s\nProfile id: %s\nEmail: %s\nReview: confirm this is a legitimate back-office employee.",
account, strings.Join(fields, "\t"), fields[0], profile, email)
}
// Promotions are judged before the shared baseline records this pass's
// new ids, so a new SuperAdmin is reported once, by the new-employee path.
findings := prestashopPromotions(store, account, w, rows, complete, describe)
return append(findings, cmsAdminFindingsEscalated(store, "prestashop", "prestashop_admin_injection", account, w, rows, complete, describe,
func(fields []string) (alert.Severity, string, bool) {
if len(fields) > 1 && fields[1] == prestashopSuperAdminProfile {
return alert.Critical, prestashopEventNewSuperAdmin, true
}
return 0, "", false
})...)
}
// prestashopPromotions reports known employees that moved to the SuperAdmin
// profile since the recorded set. The set changes only after a complete
// result; a demotion drops the id, so a later re-promotion alerts again.
func prestashopPromotions(store *state.Store, account string, w wpDBCreds, rows []string, complete bool,
describe func(fields []string) (string, string)) []alert.Finding {
if store == nil {
return nil
}
setKey := "_pssuperadmin:v1:" + dbContentDedupKey(account, w, w.tablePrefix, "cms-admin=prestashop", "superadmins")
stored, recorded := store.GetRaw(setKey)
known := map[string]bool{}
for _, id := range strings.Split(stored, ",") {
known[id] = id != ""
}
var current []string
seen := map[string]bool{}
var findings []alert.Finding
for _, row := range rows {
fields := strings.Split(row, "\t")
id := fields[0]
if !isAllDigits(id) {
complete = false
continue
}
if len(fields) < 2 || fields[1] != prestashopSuperAdminProfile || seen[id] {
continue
}
seen[id] = true
current = append(current, id)
if !recorded || known[id] {
continue
}
if _, existing := store.GetRaw(cmsAdminSeenKey(account, w, "prestashop", id)); !existing {
continue
}
message, details := describe(fields)
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "prestashop_admin_injection",
Message: message + " (" + prestashopEventPromoted + ")",
Details: prestashopFindingDetails(w, details) + "\nEvent: " + prestashopEventPromoted,
DedupKey: dbContentDedupKey(account, w, w.tablePrefix, "cms-admin=prestashop", "id="+id, "event="+prestashopEventPromoted),
})
}
if complete {
sort.Strings(current)
store.SetRaw(setKey, strings.Join(current, ","))
}
return findings
}
package checks
import (
"context"
"database/sql/driver"
"errors"
"fmt"
"io"
"net"
"sort"
"strings"
"github.com/go-sql-driver/mysql"
)
const maxDatabaseQueryDiagnostics = 16
// mysqlRegexTimeout is the server error for a regular expression that
// exhausted its work limit.
const mysqlRegexTimeout = 3699
// dbQueryState separates incomplete coverage from an unusable connection.
// Both prevent a clean baseline, but a statement-local failure must not stop
// independent detectors from reading other tables in the same installation.
type dbQueryState struct {
failed bool
halted bool
regexTimeouts int
failures map[string]int
}
func (c wpDBCreds) withQueryStage(stage string) wpDBCreds {
c.queryStage = stage
return c
}
func (s *dbQueryState) record(stage string, err error) {
if s == nil {
return
}
class, code, halt := databaseQueryErrorClass(err)
s.failed = true
s.halted = s.halted || halt
if code == mysqlRegexTimeout {
s.regexTimeouts++
}
if stage == "" {
stage = "query"
}
if s.failures == nil {
s.failures = make(map[string]int)
}
// Only a code-defined stage, class and numeric error code leave the query
// boundary. Server messages and SQL can contain account data or values.
key := fmt.Sprintf("stage=%s class=%s code=%d", stage, class, code)
s.failures[key]++
}
// regexTimeoutCount lets a caller tell whether its statement stopped at the
// regex work limit. Paths without query state never retry.
func (s *dbQueryState) regexTimeoutCount() int {
if s == nil {
return 0
}
return s.regexTimeouts
}
func databaseQueryErrorClass(err error) (class string, code uint16, halt bool) {
var sqlErr *mysql.MySQLError
if errors.As(err, &sqlErr) {
code = sqlErr.Number
switch code {
case 1054, 1146:
return "schema", code, false
case 1064:
return "syntax", code, false
case 1142, 1143:
return "permission", code, false
case 1139, 1267, 1271:
return "expression", code, false
case mysqlRegexTimeout:
// ICU stops this expression when its work budget is exhausted;
// the connection and independent statements remain usable.
return "timeout", code, false
case 1044, 1045:
return "authentication", code, true
case 1049:
return "database_missing", code, true
case 1040, 1203, 1226:
return "resource", code, true
}
}
// Unknown failures keep the existing stop behavior. Continuing is safe
// only when the server identified a failure confined to this statement.
if errors.Is(err, context.DeadlineExceeded) {
return "timeout", code, true
}
if errors.Is(err, context.Canceled) {
return "canceled", code, true
}
var netErr net.Error
if errors.As(err, &netErr) {
if netErr.Timeout() {
return "timeout", code, true
}
return "connection", code, true
}
if errors.Is(err, driver.ErrBadConn) || errors.Is(err, mysql.ErrInvalidConn) ||
errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return "connection", code, true
}
return "unknown", code, true
}
func (c *dbScanCoverage) recordQueryFailures(state *dbQueryState) {
if state == nil || len(state.failures) == 0 {
return
}
if c.queryFailures == nil {
c.queryFailures = make(map[string]int)
}
keys := make([]string, 0, len(state.failures))
for key := range state.failures {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
count := state.failures[key]
if c.queryFailures[key] == 0 && len(c.queryFailures) >= maxDatabaseQueryDiagnostics {
c.queryFailureOverflow += count
continue
}
c.queryFailures[key] += count
}
}
func (c *dbScanCoverage) queryFailureSummary() string {
keys := make([]string, 0, len(c.queryFailures))
for key := range c.queryFailures {
keys = append(keys, key)
}
sort.Strings(keys)
var b strings.Builder
for _, key := range keys {
fmt.Fprintf(&b, "Query failures: %s queries=%d\n", key, c.queryFailures[key])
}
if c.queryFailureOverflow > 0 {
fmt.Fprintf(&b, "Other query failures: queries=%d\n", c.queryFailureOverflow)
}
return b.String()
}
package checks
import (
"crypto/sha256"
"fmt"
"net"
"net/url"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/mysqlclient"
"golang.org/x/net/publicsuffix"
)
// A WordPress address pointing off the account.
//
// siteURLPoisonReason deliberately tests the shape of siteurl and not its
// host, because hosting a site under a domain that is not served locally is
// ordinary: sites move, staging lives elsewhere, a theme demo keeps its vendor
// address. Testing the host alone would report all of them.
//
// What is not ordinary is a document root the panel is serving right now,
// under domains the account owns, whose WordPress address points at an
// unrelated domain. A site that moved is no longer served here; a site that is
// served here should address itself by a name the account holds. That
// combination is either a hijack or a migration left half-finished, and both
// want an operator.
//
// It matters because the shape check only caught the live case by luck: the
// poisoned value carried a query string. A hijack to a plausible-looking
// address passes every shape test there is.
// foreignSiteURLFinding reports a served install addressed at a domain the
// account does not own, or nil when the value is fine, unreadable, or the
// question cannot be answered.
func foreignSiteURLFinding(user string, creds wpDBCreds, prefix, option, value string) *alert.Finding {
// Only a served root rules out the migration explanation.
if creds.docrootServed != servedByPanel || !creds.panelDomains.hasAccount(user) {
return nil
}
// A value whose shape is wrong is siteURLPoisonReason's finding, not this
// one; reporting it twice says nothing new.
if _, bad := siteURLPoisonReason(value); bad {
return nil
}
parsed, err := url.Parse(strings.TrimSpace(mysqlclient.BatchUnescape(value)))
if err != nil {
return nil
}
host := normalizeHost(parsed.Hostname())
if host == "" {
return nil
}
if panelHostOwnedByAccount(creds.panelDomains, user, host) {
return nil
}
return &alert.Finding{
Severity: alert.High,
Check: "db_siteurl_foreign_host",
DedupKey: foreignSiteURLDedupKey(user, creds, prefix),
Message: fmt.Sprintf("WordPress %s addresses a domain this account does not own (account: %s): %s",
option, user, host),
Details: dbContentFindingDetails(creds, prefix,
fmt.Sprintf("%s = %s", option, truncateDB(value, 200)),
"WordPress builds every asset URL from this value, so the address it "+
"names is loaded on every page of a site the panel is serving right now.",
"This is reported only for a served document root. A site that moved "+
"away is no longer served here, which is why an unowned address on a "+
"dormant root is left alone."),
}
}
func foreignSiteURLDedupKey(user string, creds wpDBCreds, prefix string) string {
identity := strings.Join([]string{user, creds.dbHost, creds.dbName, prefix}, "\x00")
digest := sha256.Sum256([]byte(identity))
return fmt.Sprintf("wp-site:%x", digest[:12])
}
// panelDomainOwnership indexes the complete panel map in both directions.
type panelDomainOwnership struct {
owners map[string]string
wildcards map[string]string
accounts map[string]struct{}
}
func newPanelDomainOwnership(panelDomains map[string][]string) *panelDomainOwnership {
if len(panelDomains) == 0 {
return nil
}
ownership := &panelDomainOwnership{
owners: make(map[string]string),
wildcards: make(map[string]string),
accounts: make(map[string]struct{}, len(panelDomains)),
}
for account, domains := range panelDomains {
for _, rawDomain := range domains {
wildcard := strings.HasPrefix(rawDomain, "*.")
domain := normalizeHost(strings.TrimPrefix(rawDomain, "*."))
if domain == "" {
return nil
}
owners := ownership.owners
if wildcard {
owners = ownership.wildcards
}
if owner, exists := owners[domain]; exists && owner != account {
return nil
}
owners[domain] = account
ownership.accounts[account] = struct{}{}
}
}
if len(ownership.owners) == 0 && len(ownership.wildcards) == 0 {
return nil
}
return ownership
}
func (ownership *panelDomainOwnership) hasAccount(account string) bool {
if ownership == nil {
return false
}
_, ok := ownership.accounts[account]
return ok
}
// panelHostOwnedByAccount resolves ownership using the most-specific panel
// domain that covers host. This preserves delegated subdomains: when alice
// owns example.com but bob owns shop.example.com, bob owns both
// shop.example.com and www.shop.example.com.
func panelHostOwnedByAccount(ownership *panelDomainOwnership, account, host string) bool {
host = normalizeHost(host)
if ownership == nil || host == "" {
return false
}
if net.ParseIP(host) != nil {
owner, exists := ownership.owners[host]
return exists && owner == account
}
for domain, exact := host, true; ; exact = false {
// A wildcard delegation is more specific than an exact mapping of its
// parent, but it never owns the parent name itself. Like an exact
// ancestor, its base must be an ownable domain: a malformed wildcard
// such as *.com must not claim every registrable name below it.
if !exact {
if owner, exists := ownership.wildcards[domain]; exists {
if panelDomainOwnsDescendants(domain) {
return owner == account
}
}
}
if owner, exists := ownership.owners[domain]; exists {
// Exact names remain authoritative. An ancestor must itself be an
// ownable domain; a malformed "com" row cannot claim google.com.
if exact {
return owner == account
}
if panelDomainOwnsDescendants(domain) {
return owner == account
}
}
dot := strings.IndexByte(domain, '.')
if dot < 0 {
break
}
domain = domain[dot+1:]
}
return false
}
func panelDomainOwnsDescendants(domain string) bool {
if net.ParseIP(domain) != nil {
return false
}
_, err := publicsuffix.EffectiveTLDPlusOne(domain)
return err == nil
}
package checks
import (
"fmt"
"regexp"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/mysqlclient"
)
const (
maxTaxonomyRowsScanned = 500
maxTaxonomySamplesPerKind = 10
)
const (
termNameURLPattern = `^[[:space:]]*(https?://|www[.])[^[:space:]]+`
termNameSpamPattern = `(^|[^a-z])(casino|casinos|kasino|gambling|betting|bookmaker|mostbet|viagra|cialis|pharmacy|pokies|slots|bahis)([^a-z]|$)`
)
// termNameIsURL matches a term whose name is a URL. Categories and tags are
// human labels; a URL in that field is put there by a link-farm kit, never by
// an editor.
var termNameIsURL = regexp.MustCompile(`(?i)` + termNameURLPattern)
// termNameSpamVocabulary covers gambling and pharmacy vocabulary in term
// names. It is bounded on word edges so "specialist" does not match "cialis".
var termNameSpamVocabulary = regexp.MustCompile(`(?i)` + termNameSpamPattern)
type spamTaxonomyRow struct {
taxonomy string
name string
}
func parseSpamTaxonomyRow(line string) (spamTaxonomyRow, bool) {
parts := strings.SplitN(strings.TrimRight(line, "\r\n"), "\t", 4)
if len(parts) != 4 {
return spamTaxonomyRow{}, false
}
name := strings.TrimSpace(mysqlclient.BatchUnescape(parts[3]))
if name == "" {
return spamTaxonomyRow{}, false
}
return spamTaxonomyRow{
taxonomy: strings.TrimSpace(mysqlclient.BatchUnescape(parts[1])),
name: name,
}, true
}
// checkWPSpamTaxonomy reports attacker-created taxonomy terms.
//
// Removing spam posts leaves their taxonomy behind, and a category archive is a
// public page: a spam taxonomy is a doorway network even with no posts attached.
// This was missed during a live cleanup -- the homepage still served gambling
// links after every spam post had been deleted.
func checkWPSpamTaxonomy(user string, creds wpDBCreds, prefix string) []alert.Finding {
urlCandidate := fmt.Sprintf("LOWER(t.name) REGEXP '%s'", termNameURLPattern)
spamCandidate := fmt.Sprintf("LOWER(t.name) REGEXP '%s'", termNameSpamPattern)
query := fmt.Sprintf(
"SELECT t.term_id, tt.taxonomy, tt.count, t.name FROM %sterms t "+
"JOIN %sterm_taxonomy tt ON tt.term_id = t.term_id "+
"WHERE %s OR %s ORDER BY CASE WHEN %s THEN 0 ELSE 1 END, t.term_id LIMIT %d",
prefix, prefix, urlCandidate, spamCandidate, urlCandidate, maxTaxonomyRowsScanned+1)
var urlNamed, keywordNamed []string
var urlCount, keywordCount int
rows := runMySQLQuery(creds, query)
truncated := len(rows) > maxTaxonomyRowsScanned
if truncated {
markCheckIncomplete(creds.queryCtx, "db_content")
rows = rows[:maxTaxonomyRowsScanned]
}
for _, line := range rows {
row, ok := parseSpamTaxonomyRow(line)
if !ok {
markCheckIncomplete(creds.queryCtx, "db_content")
continue
}
label := fmt.Sprintf("%q (%q)", row.name, row.taxonomy)
switch {
case termNameIsURL.MatchString(row.name):
urlCount++
if len(urlNamed) < maxTaxonomySamplesPerKind {
urlNamed = append(urlNamed, label)
}
case termNameSpamVocabulary.MatchString(row.name):
keywordCount++
if len(keywordNamed) < maxTaxonomySamplesPerKind {
keywordNamed = append(keywordNamed, label)
}
}
}
total := urlCount + keywordCount
if total == 0 {
return nil
}
details := []string{
"Taxonomy terms can feed public archives, navigation and other rendered " +
"content. Deleting spam posts does not remove them.",
}
if urlCount > 0 {
details = append(details, taxonomySampleDetails("Named after a URL", urlNamed, urlCount))
}
if keywordCount > 0 {
details = append(details, taxonomySampleDetails("Spam vocabulary", keywordNamed, keywordCount))
}
// A URL as a term name has no benign reading; vocabulary alone can be a
// legitimate site writing about an industry.
severity := alert.High
if urlCount == 0 {
severity = alert.Warning
}
return []alert.Finding{{
Severity: severity,
Check: "db_spam_taxonomy",
Message: fmt.Sprintf("%s spam taxonomy terms found in WordPress (account: %s)",
spamCountLabel(total, truncated), user),
Details: dbContentFindingDetails(creds, prefix, details...),
DedupKey: dbContentDedupKey(user, creds, prefix, details...),
}}
}
func taxonomySampleDetails(label string, samples []string, total int) string {
if total > len(samples) {
label += fmt.Sprintf(" (showing %d of %d)", len(samples), total)
}
return label + ": " + strings.Join(samples, ", ")
}
package checks
import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/alert"
)
// Cloak logic stored in the database.
//
// A cloak has to decide what to serve on every request, which means the page
// must not be cached. Stored code that disables caching and, in the same
// snippet, looks for a search-engine crawler is serving one page to the index
// and another to visitors.
//
// Neither half is evidence on its own, which is why both are required: caching
// plugins invoke cache helpers from their own files as a matter of course, and
// reading the user agent is ordinary. It is a stored snippet doing both that
// has no innocent reading -- the snippet is not the caching plugin, and it has
// no reason to care whether the visitor is Googlebot.
// crawlerUserAgent matches the crawlers a doorway kit cares about. The list is
// the set worth cloaking for: the engines that index and rank, plus the SEO
// crawlers kits hide from to stay out of backlink reports.
var crawlerUserAgent = regexp.MustCompile(
`(?i)\b(googlebot|bingbot|msnbot|yandex(?:bot)?|baiduspider|duckduckbot|slurp|` +
`applebot|sogou|exabot|facebot|ia_archiver|ahrefsbot|semrushbot|mj12bot|dotbot|petalbot|` +
`oai-searchbot|claude-searchbot)\b`)
const (
// The detector only reports known signals, but explicit caps keep finding
// text and derived-literal work bounded if those vocabularies grow.
maxStoredCloakMatches = 64
maxStoredCloakDetailNames = 8
maxStoredCloakDerivedBytes = maxStoredCodeBytes
maxStoredScalarParens = 32
)
// storedCloakComponents returns the cache-defeat and crawler-detection markers
// found in one stored snippet.
func storedCloakComponents(code []byte) (cacheDefeat, crawler []string) {
php := stripPHPCommentsFromCode(string(code))
// Heredoc and nowdoc bodies are string data. Keep them available to the
// crawler-literal scan, but do not parse helper calls or $_SERVER accesses
// written inside them as executable PHP.
executablePHP := blankStoredPHPHeredocs(php)
cacheDefeat = storedCacheDefeatSignals(executablePHP)
if len(cacheDefeat) == 0 {
return nil, nil
}
// A crawler name in documentation or output is not a visitor test. Require
// the snippet to inspect the HTTP user agent, including a name assembled
// from constant string operations.
if !storedHasUserAgentInspection(executablePHP) {
return cacheDefeat, nil
}
derived := storedCloakDerivedStrings(executablePHP)
searchable := php + "\n" + derived
crawler = appendCapturedNames(nil, make(map[string]bool), crawlerUserAgent,
[]byte(searchable), strings.ToLower)
return limitStoredCloakNames(cacheDefeat), limitStoredCloakNames(crawler)
}
// appendCapturedNames collects the first capture group of each match, so the
// finding names the constant or crawler rather than the surrounding syntax.
func appendCapturedNames(out []string, seen map[string]bool, re *regexp.Regexp, code []byte, canonical func(string) string) []string {
for _, m := range re.FindAllSubmatch(code, maxStoredCloakMatches) {
if len(m) < 2 {
continue
}
name := canonical(strings.TrimSpace(string(m[1])))
if name == "" || seen[name] {
continue
}
seen[name] = true
out = append(out, name)
}
sort.Strings(out)
return out
}
func limitStoredCloakNames(names []string) []string {
if len(names) > maxStoredCloakDetailNames {
return names[:maxStoredCloakDetailNames]
}
return names
}
func blankStoredPHPHeredocs(code string) string {
var out strings.Builder
out.Grow(len(code))
for i := 0; i < len(code); i++ {
// A quoted string can contain text that looks like an unterminated
// heredoc opener. Skip it whole so that text cannot blank executable
// code after the closing quote.
if isPHPQuote(code[i]) {
end := skipPHPString(code, i)
out.WriteString(code[i : end+1])
i = end
continue
}
if label, bodyStart, ok := phpHeredocOpen(code, i); ok {
end := phpHeredocEnd(code, bodyStart, label)
blankInlineHTML(&out, code[i:end])
i = end - 1
continue
}
out.WriteByte(code[i])
}
return out.String()
}
type storedPHPBool uint8
const (
storedPHPBoolUnknown storedPHPBool = iota
storedPHPBoolFalse
storedPHPBoolTrue
)
type storedPHPBoolAssignment struct {
pos int
value storedPHPBool
}
// storedCacheDefeatSignals recognises operations that stop the current request
// from being cached. Parsing real calls and assignments avoids treating a
// comment, string example, false DONOTCACHE value, or true WP_CACHE value as
// executable cache control.
func storedCacheDefeatSignals(code string) []string {
seen := make(map[string]bool)
var signals []string
var assignments map[string][]storedPHPBoolAssignment
boolValue := func(expr string, before int) storedPHPBool {
value := storedPHPBoolExpression(expr, nil, before)
if value != storedPHPBoolUnknown {
return value
}
if _, ok := singlePHPVariableExpr(trimStoredPHPParens(strings.TrimSpace(expr))); !ok {
return storedPHPBoolUnknown
}
if assignments == nil {
assignments = storedPHPBoolAssignments(code)
}
return storedPHPBoolExpression(expr, assignments, before)
}
callNames := map[string]struct{}{
"define": {},
"header": {},
"nocache_headers": {},
}
for searchFrom := 0; searchFrom < len(code); {
callStart, openParen, closeParen, ok := nextStandalonePHPCall(code, searchFrom, callNames)
if !ok {
break
}
searchFrom = nextSearchOffset(closeParen, len(code))
if closeParen >= len(code) || storedPHPCallIsNonGlobal(code, callStart) {
continue
}
name := storedPHPCallName(code[callStart:openParen])
args := phpCallArguments(code, openParen+1, closeParen)
switch name {
case "define":
if len(args) < 2 {
continue
}
constantName, ok := storedConstantStringExpression(args[0])
if !ok {
continue
}
constantName = strings.ToUpper(strings.TrimSpace(constantName))
value := boolValue(args[1], callStart)
switch constantName {
case "DONOTCACHEPAGE":
if value == storedPHPBoolTrue {
signals = appendStoredCloakName(signals, seen, constantName)
}
case "WP_CACHE":
if value == storedPHPBoolFalse {
signals = appendStoredCloakName(signals, seen, constantName)
}
}
case "header":
if len(args) == 0 {
continue
}
headerValue, ok := storedConstantStringExpression(args[0])
if ok && storedLiteSpeedNoCacheHeader(headerValue) {
signals = appendStoredCloakName(signals, seen, "X-LiteSpeed-Cache-Control")
}
case "nocache_headers":
if len(args) == 0 {
signals = appendStoredCloakName(signals, seen, "nocache_headers")
}
}
}
sort.Strings(signals)
return limitStoredCloakNames(signals)
}
func appendStoredCloakName(out []string, seen map[string]bool, name string) []string {
if name == "" || seen[name] {
return out
}
seen[name] = true
return append(out, name)
}
func storedPHPCallName(callPrefix string) string {
return strings.ToLower(strings.TrimPrefix(strings.TrimSpace(callPrefix), `\`))
}
func storedPHPCallIsNonGlobal(code string, callStart int) bool {
i := callStart - 1
for i >= 0 && isPHPSpace(code[i]) {
i--
}
// PHP permits whitespace around member, static, and namespace operators.
// Those calls do not invoke the WordPress/PHP global cache helpers.
if i >= 0 && (code[i] == '>' || code[i] == ':' || code[i] == '\\') {
return true
}
if i >= 0 && code[i] == '&' {
i--
for i >= 0 && isPHPSpace(code[i]) {
i--
}
}
end := i + 1
for i >= 0 && isPHPIdentifierPart(code[i]) {
i--
}
return strings.EqualFold(code[i+1:end], "function")
}
func storedLiteSpeedNoCacheHeader(value string) bool {
name, directive, ok := strings.Cut(value, ":")
if !ok || !strings.EqualFold(strings.TrimSpace(name), "X-LiteSpeed-Cache-Control") {
return false
}
for _, item := range strings.Split(directive, ",") {
if strings.EqualFold(strings.TrimSpace(item), "no-cache") {
return true
}
}
return false
}
func storedPHPBoolAssignments(code string) map[string][]storedPHPBoolAssignment {
assignments := make(map[string][]storedPHPBoolAssignment)
for i := 0; i < len(code); i++ {
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
if code[i] != '$' {
continue
}
variable, next, ok := readPHPVariableName(code, i)
if !ok {
continue
}
operator := skipPHPWhitespace(code, next)
opLen, direct, _, ok := phpAssignmentOperator(code, operator)
if !ok {
i = next - 1
continue
}
exprStart := skipPHPWhitespace(code, operator+opLen)
exprEnd := phpExpressionEnd(code, exprStart)
value := storedPHPBoolUnknown
if direct {
value = storedPHPBoolExpression(code[exprStart:exprEnd], assignments, exprStart)
}
assignments[variable] = append(assignments[variable], storedPHPBoolAssignment{
pos: exprEnd, value: value,
})
i = exprEnd - 1
}
return assignments
}
func storedPHPBoolExpression(expr string, assignments map[string][]storedPHPBoolAssignment, before int) storedPHPBool {
expr = trimStoredPHPParens(strings.TrimSpace(expr))
switch strings.ToLower(expr) {
case "true":
return storedPHPBoolTrue
case "false", "null":
return storedPHPBoolFalse
}
if value, ok := storedConstantStringExpression(expr); ok {
if value == "" || value == "0" {
return storedPHPBoolFalse
}
return storedPHPBoolTrue
}
if number, err := strconv.ParseFloat(expr, 64); err == nil {
if number == 0 {
return storedPHPBoolFalse
}
return storedPHPBoolTrue
}
variable, ok := singlePHPVariableExpr(expr)
if !ok {
return storedPHPBoolUnknown
}
values := assignments[variable]
index := sort.Search(len(values), func(i int) bool {
return values[i].pos > before
})
if index > 0 {
return values[index-1].value
}
return storedPHPBoolUnknown
}
func trimStoredPHPParens(expr string) string {
// Scalar expressions are attacker-controlled. Cap redundant-wrapper work so
// deeply nested input cannot turn repeated matching scans quadratic.
for depth := 0; depth < maxStoredScalarParens && len(expr) >= 2 && expr[0] == '(' && matchingParen(expr, 0) == len(expr)-1; depth++ {
expr = strings.TrimSpace(expr[1 : len(expr)-1])
}
return expr
}
// storedCloakDerivedStrings evaluates only bounded, constant string operations
// commonly used to hide crawler literals. General PHP evaluation is outside
// this detector; literal XOR remains covered by the backdoor signature.
func storedCloakDerivedStrings(code string) string {
var derived strings.Builder
for i := 0; i < len(code) && derived.Len() < maxStoredCloakDerivedBytes; i++ {
if !isPHPQuote(code[i]) {
continue
}
closeQuote, closed := storedPHPStringClose(code, i)
if !closed {
break
}
dot := skipPHPWhitespace(code, closeQuote+1)
if dot >= len(code) || code[dot] != '.' {
i = closeQuote
continue
}
nextOperand := skipPHPWhitespace(code, dot+1)
if nextOperand >= len(code) || !isPHPQuote(code[nextOperand]) {
i = closeQuote
continue
}
value, end, parts, ok := storedConstantStringAt(code, i)
if !ok {
continue
}
if parts > 1 {
appendStoredDerivedString(&derived, value)
}
i = end - 1
}
rot13 := map[string]struct{}{"str_rot13": {}}
for searchFrom := 0; searchFrom < len(code) && derived.Len() < maxStoredCloakDerivedBytes; {
callStart, openParen, closeParen, ok := nextStandalonePHPCall(code, searchFrom, rot13)
if !ok {
break
}
searchFrom = nextSearchOffset(closeParen, len(code))
if closeParen >= len(code) || storedPHPCallIsNonGlobal(code, callStart) {
continue
}
args := phpCallArguments(code, openParen+1, closeParen)
if len(args) != 1 {
continue
}
value, ok := storedConstantStringExpression(args[0])
if ok {
appendStoredDerivedString(&derived, storedROT13(value))
}
}
return derived.String()
}
func appendStoredDerivedString(out *strings.Builder, value string) {
remaining := maxStoredCloakDerivedBytes - out.Len()
if remaining <= 1 {
return
}
out.WriteByte('\n')
remaining--
if len(value) > remaining {
value = value[:remaining]
}
out.WriteString(value)
}
func storedConstantStringExpression(expr string) (string, bool) {
expr = strings.TrimSpace(expr)
value, end, _, ok := storedConstantStringAt(expr, 0)
return value, ok && skipPHPWhitespace(expr, end) == len(expr)
}
func storedConstantStringAt(code string, start int) (string, int, int, bool) {
if start >= len(code) || !isPHPQuote(code[start]) {
return "", start, 0, false
}
var value strings.Builder
parts := 0
pos := start
for {
if pos >= len(code) || !isPHPQuote(code[pos]) {
return "", start, 0, false
}
closeQuote, closed := storedPHPStringClose(code, pos)
if !closed {
return "", start, 0, false
}
if storedPHPStringInterpolates(code, pos, closeQuote) {
return "", start, 0, false
}
value.WriteString(phpStringLiteralValue(code, pos, closeQuote))
parts++
next := skipPHPWhitespace(code, closeQuote+1)
if next >= len(code) || code[next] != '.' {
return value.String(), closeQuote + 1, parts, true
}
pos = skipPHPWhitespace(code, next+1)
if pos >= len(code) || !isPHPQuote(code[pos]) {
return value.String(), closeQuote + 1, parts, true
}
}
}
func storedPHPStringInterpolates(code string, start, end int) bool {
if code[start] != '"' {
return false
}
for i := start + 1; i < end; i++ {
if code[i] == '\\' && i+1 < end {
i++
continue
}
if code[i] == '$' && i+1 < end && (code[i+1] == '{' || isPHPIdentifierStart(code[i+1])) {
return true
}
}
return false
}
func storedPHPStringClose(code string, start int) (int, bool) {
quote := code[start]
for i := start + 1; i < len(code); i++ {
if code[i] == '\\' && i+1 < len(code) {
i++
continue
}
if code[i] == quote {
return i, true
}
}
return len(code), false
}
// storedHasUserAgentInspection requires an executable $_SERVER access instead
// of accepting HTTP_USER_AGENT in prose or another string literal. Constant
// string concatenation is enough to cover the common lightly-obfuscated key
// without following arbitrary attacker-controlled variables. Parsing just the
// supported scalar key avoids repeatedly searching nested, hostile brackets.
func storedHasUserAgentInspection(code string) bool {
for i := 0; i < len(code); i++ {
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
if code[i] != '$' {
continue
}
variable, next, ok := readPHPVariableName(code, i)
if !ok || !strings.EqualFold(variable, "_SERVER") {
continue
}
openBracket := skipPHPWhitespace(code, next)
if openBracket >= len(code) || code[openBracket] != '[' {
continue
}
keyStart := skipPHPWhitespace(code, openBracket+1)
key, keyEnd, _, ok := storedConstantStringAt(code, keyStart)
if !ok {
continue
}
closeBracket := skipPHPWhitespace(code, keyEnd)
if closeBracket < len(code) && code[closeBracket] == ']' &&
strings.EqualFold(key, "HTTP_USER_AGENT") {
return true
}
}
return false
}
func storedROT13(value string) string {
return strings.Map(func(r rune) rune {
switch {
case r >= 'a' && r <= 'z':
return 'a' + (r-'a'+13)%26
case r >= 'A' && r <= 'Z':
return 'A' + (r-'A'+13)%26
default:
return r
}
}, value)
}
// storedCloakFindingWithComponents reports a stored snippet that both defeats
// caching and looks for a crawler, or nil when only one half is present.
func storedCloakFindingWithComponents(user string, creds wpDBCreds, prefix string, row storedCodeRow, cacheDefeat, crawler []string) *alert.Finding {
if len(cacheDefeat) == 0 || len(crawler) == 0 {
return nil
}
// Only a published snippet runs. A draft still documents the intent.
severity := alert.Warning
if row.status == "publish" {
severity = alert.High
}
return &alert.Finding{
Severity: severity,
Check: "db_stored_cloak_logic",
Message: fmt.Sprintf("Stored PHP snippet %s (%s) serves crawlers differently from visitors (account: %s)",
row.id, row.status, user),
Details: dbContentFindingDetails(creds, prefix,
fmt.Sprintf("Snippet %s is stored in %sposts, so no filesystem scan reads it.", row.id, prefix),
"It disables caching for the request and, in the same snippet, tests the "+
"visitor against a search or SEO crawler. Cloaks need both: the decision "+
"is per request, so the page must not be served from cache. Cache helpers "+
"and user-agent checks are ordinary separately, but not together here.",
"Cache defeat: "+strings.Join(cacheDefeat, ", "),
"Crawlers named: "+strings.Join(crawler, ", ")),
DedupKey: dbContentDedupKey(user, creds, prefix,
"status="+row.status,
fmt.Sprintf("Snippet %s is stored in %sposts, so no filesystem scan reads it.", row.id, prefix),
"It disables caching for the request and, in the same snippet, tests the "+
"visitor against a search or SEO crawler. Cloaks need both: the decision "+
"is per request, so the page must not be served from cache. Cache helpers "+
"and user-agent checks are ordinary separately, but not together here.",
"Cache defeat: "+strings.Join(cacheDefeat, ", "),
"Crawlers named: "+strings.Join(crawler, ", ")),
}
}
// storedCloakNote adds the cloak components to a snippet that already matched a
// signature, rather than raising a second finding about the same row.
func storedCloakNote(cacheDefeat, crawler []string) string {
if len(cacheDefeat) == 0 || len(crawler) == 0 {
return ""
}
return fmt.Sprintf("\nIt also cloaks: caching is disabled for the request (%s) "+
"while the visitor is tested against %s, so what a crawler is served is "+
"not what a visitor sees.",
strings.Join(cacheDefeat, ", "), strings.Join(crawler, ", "))
}
package checks
import (
"encoding/hex"
"fmt"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/alert"
)
const (
// A snippet is code a human wrote into a form; the ones that matter are far
// smaller than this. The bound keeps one pathological row from dominating a
// scan, and the hex encoding below doubles whatever is transferred.
maxStoredCodeBytes = 65536
maxStoredCodeRows = 200
)
type storedCodeRow struct {
id string
status string
contentSize int64
code []byte
}
// parseStoredCodeRow decodes one tab-separated query row. ok remains true for
// a decoded prefix when the query or transport truncates it; complete tells the
// caller not to present that partial row as a complete scan.
func parseStoredCodeRow(line string) (row storedCodeRow, ok, complete bool) {
parts := strings.SplitN(strings.TrimSpace(line), "\t", 4)
if len(parts) != 4 {
return row, false, false
}
contentSize, err := strconv.ParseInt(strings.TrimSpace(parts[2]), 10, 64)
if err != nil || contentSize < 0 {
return row, false, false
}
encodedCode := strings.TrimSpace(parts[3])
code, decodeErr := hex.DecodeString(encodedCode)
if len(code) == 0 {
return row, false, false
}
return storedCodeRow{
id: strings.TrimSpace(parts[0]),
status: strings.TrimSpace(parts[1]),
contentSize: contentSize,
code: code,
}, true, decodeErr == nil && int64(len(code)) == contentSize
}
// checkWPStoredCode scans WPCode PHP that lives in the database rather than in
// a file.
//
// WPCode executes stored code by design, which makes the posts table an
// executable surface that no filesystem scan covers. On a live
// compromise a 17KB obfuscated backdoor ran on every request from a WPCode row
// while a full file sweep of the same account -- core checksums, eval chains,
// upload shells -- came back clean.
//
// Content is hex-encoded by the query because snippets contain newlines and
// tabs, which would otherwise break row parsing.
func checkWPStoredCode(user string, creds wpDBCreds, prefix string) []alert.Finding {
scanner := contentSignatureScanner()
if scanner == nil {
return nil
}
// WPCode / Insert Headers and Footers. Code Snippets keeps its code in its
// own table and is the next surface worth adding here.
query := fmt.Sprintf(
"SELECT p.ID, p.post_status, OCTET_LENGTH(p.post_content), "+
"HEX(LEFT(CAST(p.post_content AS BINARY), %d)) FROM %sposts p "+
"WHERE p.post_type = 'wpcode' AND p.post_content <> '' "+
"AND EXISTS (SELECT 1 FROM %sterm_relationships tr "+
"JOIN %sterm_taxonomy tt ON tt.term_taxonomy_id = tr.term_taxonomy_id "+
"JOIN %sterms t ON t.term_id = tt.term_id "+
"WHERE tr.object_id = p.ID AND tt.taxonomy = 'wpcode_type' "+
"AND t.slug IN ('php', 'universal')) "+
"ORDER BY CASE p.post_status WHEN 'publish' THEN 0 WHEN 'draft' THEN 1 "+
"WHEN 'trash' THEN 2 ELSE 3 END, p.ID LIMIT %d",
maxStoredCodeBytes, prefix, prefix, prefix, prefix, maxStoredCodeRows+1)
var findings []alert.Finding
rows := runMySQLQuery(creds, query)
if len(rows) > maxStoredCodeRows {
markCheckIncomplete(creds.queryCtx, "db_content")
rows = rows[:maxStoredCodeRows]
}
for _, line := range rows {
row, ok, complete := parseStoredCodeRow(line)
if !complete {
markCheckIncomplete(creds.queryCtx, "db_content")
}
if !ok {
continue
}
// MySQL HEX() always emits valid hex, so a decode error means the row
// was truncated in transport. Scan whatever decoded rather than
// dropping the row: a partially recovered payload still identifies a
// backdoor, and silence here would read as "clean".
cacheDefeat, crawler := storedCloakComponents(row.code)
hits := scanner.ScanContentWithSize(row.code, ".php", row.contentSize)
if len(hits) == 0 {
// No signature matched, but stored code that both defeats caching
// and looks for a crawler is a cloak on its own terms.
if cloak := storedCloakFindingWithComponents(user, creds, prefix, row, cacheDefeat, crawler); cloak != nil {
findings = append(findings, *cloak)
}
continue
}
// Only a published snippet runs. A draft is one click from running; a
// trashed one is evidence of what was run before.
severity := alert.Warning
switch row.status {
case "publish":
severity = alert.Critical
case "draft":
severity = alert.High
}
names := make([]string, 0, len(hits))
for _, h := range hits {
names = append(names, h.RuleName)
}
findings = append(findings, alert.Finding{
Severity: severity,
Check: "db_stored_code_execution",
Message: fmt.Sprintf("Stored PHP snippet %s (%s) matches %s (account: %s)",
row.id, row.status, strings.Join(names, ", "), user),
Details: dbContentFindingDetails(creds, prefix,
fmt.Sprintf("Snippet %s is stored in %sposts for WPCode, "+
"so it is not visible to any filesystem scan.\nMatched: %s%s",
row.id, prefix, strings.Join(names, ", "),
storedCloakNote(cacheDefeat, crawler))),
DedupKey: dbContentDedupKey(user, creds, prefix,
"status="+row.status,
fmt.Sprintf("Snippet %s is stored in %sposts for WPCode, "+
"so it is not visible to any filesystem scan.\nMatched: %s%s",
row.id, prefix, strings.Join(names, ", "),
storedCloakNote(cacheDefeat, crawler))),
})
}
return findings
}
package checks
import (
"context"
"fmt"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// DetachedWebProcessCheck reports a process holding a PHP listener without a
// live web worker in its ancestry. Legitimate background jobs also do this,
// so only a shell, the loop shape a webshell leaves behind, is Critical and
// may be stopped; anything else is a Warning to review.
const DetachedWebProcessCheck = "detached_web_process"
// detachedWebProcessMinAge keeps short jobs a request starts in the
// background, such as an image conversion, out of the report.
const detachedWebProcessMinAge = 5 * time.Minute
type webSocketHolder struct {
pid int
exe string
uid string
start uint64
sockets []string
}
// CheckDetachedWebProcesses reports non-root processes that hold a PHP
// handler's listening socket without a web runtime in their ancestry.
func CheckDetachedWebProcesses(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
listeners := unixListeners()
if len(listeners) == 0 || ctx.Err() != nil {
return nil
}
procs, _ := osFS.Glob("/proc/[0-9]*")
heldByPHP := make(map[string]bool)
var holders []webSocketHolder
now := time.Now()
for _, dir := range procs {
if ctx.Err() != nil {
return nil
}
id := filepath.Base(dir)
pid, ok := parseProcessPID(id)
if !ok {
continue
}
start, ok := procStartTicks(id)
if !ok {
continue
}
exe := getProcessExe(id)
php := isServerPHPExe(exe)
uid := getProcessUID(id)
// Descriptor tables can be large. Only PHP workers and eligible detached
// processes can contribute evidence, so reject other processes first.
if !php && (exe == "" || uid == "" || uid == "0" || isWebServerExe(exe) ||
!processStartedBefore(id, now.Add(-detachedWebProcessMinAge))) {
continue
}
if !php {
if _, ok := detachedWebParent(id); !ok {
continue
}
}
held := heldListeners(ctx, id, listeners)
if len(held) == 0 {
continue
}
h := webSocketHolder{pid: pid, exe: exe, uid: uid, start: start, sockets: held}
if !h.current() {
continue
}
if php {
for _, inode := range held {
heldByPHP[inode] = true
}
continue
}
holders = append(holders, h)
}
var findings []alert.Finding
for _, h := range holders {
if ctx.Err() != nil {
return nil
}
var sockets []string
for _, inode := range h.sockets {
if heldByPHP[inode] || phpHandlerSocketPath(listeners[inode]) {
sockets = append(sockets, listeners[inode])
}
}
if len(sockets) == 0 {
continue
}
id := strconv.Itoa(h.pid)
parent, ok := detachedWebParent(id)
if !ok {
continue
}
cmdline, _ := osFS.ReadFile(filepath.Join("/proc", id, "cmdline"))
// A scan spans many processes. Do not attach an old process's socket
// evidence to a replacement PID, executable or account.
if !h.current() {
continue
}
f := alert.Finding{
Severity: alert.Warning,
Check: DetachedWebProcessCheck,
Message: fmt.Sprintf("Detached process holds a PHP handler socket: %s (PID %d)", h.exe, h.pid),
Details: fmt.Sprintf("PID: %d, UID: %s, parent PID: %d, exe: %s, cmdline: %s\nHolds PHP handler socket: %s\nSocket inheritance also occurs in legitimate background jobs; review before taking action.", h.pid, h.uid, parent, h.exe, redactProcCommandLine(cmdline), strings.Join(sockets, ", ")),
PID: h.pid,
TenantID: HostingAccountForUser(uidStringToUser(h.uid)),
Timestamp: now,
}
if isShellCommand(h.exe, cmdline) {
f.Severity = alert.Critical
f.Message = fmt.Sprintf("Shell left running by a web request: %s (PID %d)", h.exe, h.pid)
f.Details = fmt.Sprintf("PID: %d, UID: %s, parent PID: %d, exe: %s, cmdline: %s\nHolds PHP handler socket: %s", h.pid, h.uid, parent, h.exe, redactProcCommandLine(cmdline), strings.Join(sockets, ", "))
// The rules for a process that writes back a removed file decide
// what may be stopped: nil with a terminal or an SSH, cron or
// cPanel session in the ancestry.
for _, pid := range restoreWriterChain(h.pid, now, false) {
id := strconv.Itoa(pid)
if isShellProcess(id, getProcessExe(id)) {
f.WriterChain = append(f.WriterChain, pid)
}
}
f.WriterObservedAt = now
}
findings = append(findings, f)
}
return dropCoveredDetachedProcesses(findings)
}
// procfs resolves executable symlinks. argv[0] is only used for BusyBox,
// whose executable is shared by shells and non-shell applets.
func isShellCommand(exe string, cmdline []byte) bool {
switch filepath.Base(strings.TrimSuffix(exe, " (deleted)")) {
case "sh", "bash", "dash", "ash", "zsh", "ksh", "mksh", "csh", "tcsh":
return true
case "busybox":
arg0, rest, ok := strings.Cut(string(cmdline), "\x00")
if !ok {
return false
}
applet := filepath.Base(strings.TrimPrefix(arg0, "-"))
for strings.HasPrefix(applet, "busybox") {
applet, rest, ok = strings.Cut(rest, "\x00")
if !ok {
return false
}
applet = filepath.Base(applet)
}
return applet == "sh" || applet == "ash" || applet == "hush" || applet == "bash"
}
return false
}
func isShellProcess(pid, exe string) bool {
cmdline, _ := osFS.ReadFile(filepath.Join("/proc", pid, "cmdline"))
return isShellCommand(exe, cmdline) && getProcessExe(pid) == exe
}
// dropCoveredDetachedProcesses keeps one finding per process tree: a process
// already in another finding's stop chain is stopped with it.
func dropCoveredDetachedProcesses(findings []alert.Finding) []alert.Finding {
covered := make(map[int]bool)
for _, f := range findings {
for _, pid := range f.WriterChain {
if pid != f.PID {
covered[pid] = true
}
}
}
out := findings[:0]
for _, f := range findings {
if !covered[f.PID] {
out = append(out, f)
}
}
return out
}
func (h webSocketHolder) current() bool {
id := strconv.Itoa(h.pid)
exe, uid := getProcessExe(id), getProcessUID(id)
start, ok := procStartTicks(id)
return ok && start == h.start && exe == h.exe && uid == h.uid
}
// detachedWebParent requires a complete ancestry to init with no live web
// runtime. Checking only the immediate parent mistakes a request's shell
// grandchildren for detached jobs. Procfs exposes parents in its own PID
// namespace; missing ancestors or namespace boundaries are inconclusive.
func detachedWebParent(id string) (int, bool) {
parent, _, ok := procParentAndTTY(id)
if !ok {
return 0, false
}
for p, depth := parent, 0; depth < maxRestoreChainDepth; depth++ {
if p == 1 {
return parent, true
}
if p < 1 {
return 0, false
}
ancestor := strconv.Itoa(p)
exe := getProcessExe(ancestor)
if exe == "" || isWebServerExe(exe) {
return 0, false
}
p, _, ok = procParentAndTTY(ancestor)
if !ok {
return 0, false
}
}
return 0, false
}
// unixListeners maps the inode of each listening unix socket to its path.
func unixListeners() map[string]string {
data, err := osFS.ReadFile("/proc/net/unix")
if err != nil {
return nil
}
out := make(map[string]string)
for _, line := range strings.Split(string(data), "\n") {
inode, path, ok := parseUnixListener(line)
if ok {
out[inode] = path
}
}
return out
}
func parseUnixListener(line string) (inode, path string, ok bool) {
// Only the seven fixed columns are whitespace-delimited. Everything after
// the inode's separator is the socket name, including spaces and tabs.
var fields [7]string
for i := range fields {
line = strings.TrimLeft(line, " \t")
end := strings.IndexAny(line, " \t")
if end < 0 {
return "", "", false
}
fields[i], line = line[:end], line[end+1:]
}
flags, err := strconv.ParseUint(fields[3], 16, 32)
if err != nil || flags&0x10000 == 0 || fields[4] != "0001" || line == "" {
return "", "", false
}
number, err := strconv.ParseUint(fields[6], 10, 64)
if err != nil || number == 0 {
return "", "", false
}
return fields[6], line, true
}
// heldListeners returns the inodes of listening sockets in listeners that the
// process holds open.
func heldListeners(ctx context.Context, pid string, listeners map[string]string) []string {
fds, _ := osFS.Glob(filepath.Join("/proc", pid, "fd", "*"))
var held []string
seen := make(map[string]bool)
for _, fd := range fds {
if ctx.Err() != nil {
return nil
}
target, err := osFS.Readlink(fd)
if err != nil {
continue
}
inode, ok := strings.CutPrefix(target, "socket:[")
if !ok {
continue
}
inode, ok = strings.CutSuffix(inode, "]")
if !ok {
continue
}
if _, listening := listeners[inode]; listening && !seen[inode] {
held = append(held, inode)
seen[inode] = true
}
}
return held
}
// phpHandlerSocketPath reports a listener a web server hands PHP requests to:
// LiteSpeed's APVH_<user>_<handler>.sock for a PHP handler, or a PHP-FPM
// pool. The account name in a LiteSpeed socket is never taken as the handler.
func phpHandlerSocketPath(path string) bool {
base := filepath.Base(path)
if rest, ok := strings.CutPrefix(base, "APVH_"); ok {
_, handler, ok := strings.Cut(rest, "_")
return ok && strings.Contains(strings.ToLower(handler), "php")
}
return strings.Contains(path, "/php-fpm/") || strings.HasPrefix(base, "php") && strings.Contains(base, "fpm")
}
package checks
import (
"fmt"
"net"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/processctx"
)
// DirectSMTPEgressInput is the input to the evaluator. The caller
// (BPF connection consumer or legacy poller) builds it from the live
// event and passes the platform-resolved MTA allowlist as MTA.
//
// Process is optional; when present the resulting finding includes the
// full process-ancestry tree. UID/User/PID/Comm/Exe are the live event
// fields used in finding details and account attribution.
type DirectSMTPEgressInput struct {
UID uint32
User string
PID uint32
Comm string
Exe string
DstIP net.IP
DstPort uint16
MTA platform.MTAIdents
Process *processctx.ProcessContext
// Domain is an optional rDNS-resolved name for DstIP. When set, it
// is included in the finding details. Populating it is the caller's
// responsibility.
Domain string
}
// EvaluateDirectSMTPEgress returns a populated finding when the input
// represents a non-MTA local process opening an outbound SMTP connection.
// Owner resolution uses the shared passwd cache. Detector-disabled config
// returns (zero, false) without inspecting the input.
func EvaluateDirectSMTPEgress(cfg *config.Config, in DirectSMTPEgressInput) (alert.Finding, bool) {
if cfg == nil || !cfg.Detection.DirectSMTPEgress.Enabled || directSMTPEgressBackend(cfg) == "none" {
return alert.Finding{}, false
}
if in.UID == 0 {
return alert.Finding{}, false
}
if in.DstIP == nil || in.DstIP.IsLoopback() || in.DstIP.IsUnspecified() {
return alert.Finding{}, false
}
if !portInList(in.DstPort, cfg.Detection.DirectSMTPEgress.Ports) {
return alert.Finding{}, false
}
if isInfraIP(in.DstIP.String(), cfg.InfraIPs) {
return alert.Finding{}, false
}
if in.MTA.IsMTAUser(in.User, in.UID) || SMTPAllowedUser(cfg, in.User) {
return alert.Finding{}, false
}
dst := in.DstIP.String()
if in.DstIP.To4() == nil {
dst = "[" + dst + "]"
}
details := fmt.Sprintf("UID: %d (%s), Process: %s, PID: %d, Destination: %s:%d",
in.UID, in.User, in.Comm, in.PID, dst, in.DstPort)
if in.Domain != "" {
details += ", Domain: " + in.Domain
}
return alert.Finding{
Severity: alert.High,
Check: "direct_smtp_egress",
Message: fmt.Sprintf("Non-MTA process opened outbound SMTP connection to %s:%d", dst, in.DstPort),
Details: details,
TenantID: directSMTPTenant(in),
Process: in.Process,
}, true
}
// SMTPAllowedUser reports whether the operator lets user send through the
// firewall's smtp_block, so its connections are neither direct egress nor
// blocked by BPF enforcement.
func SMTPAllowedUser(cfg *config.Config, user string) bool {
if user == "" || cfg == nil || cfg.Firewall == nil {
return false
}
for _, allowed := range cfg.Firewall.SMTPAllowUsers {
if allowed == user {
return true
}
}
return false
}
func DirectSMTPEgressBackendEnabled(cfg *config.Config, backend string) bool {
if cfg == nil || !cfg.Detection.DirectSMTPEgress.Enabled {
return false
}
choice := directSMTPEgressBackend(cfg)
switch choice {
case "auto":
return true
case "bpf", "legacy":
return backend == choice
default:
return false
}
}
func directSMTPEgressBackend(cfg *config.Config) string {
backend := strings.ToLower(strings.TrimSpace(cfg.Detection.DirectSMTPEgress.Backend))
if backend == "" {
return "auto"
}
return backend
}
func directSMTPTenant(in DirectSMTPEgressInput) string {
if in.Process != nil && in.Process.Account != "" {
return HostingAccountForUser(in.Process.Account)
}
return HostingAccountForUser(in.User)
}
func portInList(p uint16, list []int) bool {
for _, q := range list {
if q <= 0 || q > 65535 {
continue
}
if q == int(p) {
return true
}
}
return false
}
package checks
import (
"sync/atomic"
"github.com/pidginhost/csm/internal/metrics"
)
var directSMTPEgressFindingsTotal atomic.Uint64
// RegisterDirectSMTPEgressMetrics binds the per-finding counter to reg.
// Production callers should pass metrics.Default(); tests pass
// metrics.NewRegistry() to keep registration isolated.
func RegisterDirectSMTPEgressMetrics(reg *metrics.Registry) {
reg.RegisterCounterFunc(
"csm_direct_smtp_egress_findings_total",
"Direct SMTP egress findings emitted by the connection consumer.",
func() float64 { return float64(directSMTPEgressFindingsTotal.Load()) },
)
}
// BumpDirectSMTPEgressFindings increments the per-finding counter.
// Called by the connection consumer when EvaluateDirectSMTPEgress
// returns a finding.
func BumpDirectSMTPEgressFindings() {
directSMTPEgressFindingsTotal.Add(1)
}
// resetDirectSMTPEgressMetricsForTest is a test seam.
func resetDirectSMTPEgressMetricsForTest() {
directSMTPEgressFindingsTotal.Store(0)
}
package checks
import (
"bufio"
"context"
"fmt"
"net"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// DNS server processes that legitimately connect to many resolvers
// (e.g. BIND doing recursive resolution on a cPanel server).
var dnsServerUsers = map[string]bool{
"named": true, // BIND
"unbound": true, // Unbound
"pdns": true, // PowerDNS
"systemd-resolve": true, // systemd-resolved forwarding to its upstreams
"dnsmasq": true, // dnsmasq forwarding to its upstreams
}
// resolvedUpstreamsPath lists the real upstreams when /etc/resolv.conf only
// names the systemd-resolved stub.
const resolvedUpstreamsPath = "/run/systemd/resolve/resolv.conf"
const (
dnsmasqConfigPath = "/etc/dnsmasq.conf"
dnsmasqConfigGlob = "/etc/dnsmasq.d/*.conf"
)
// CheckDNSConnections looks for established connections to port 53 on
// DNS servers that are NOT in /etc/resolv.conf. This catches DNS
// tunneling, GSocket relay discovery, and malware using hardcoded resolvers.
// Connections owned by known DNS server processes (e.g. named) are skipped.
func CheckDNSConnections(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
// Parse configured resolvers
resolvers := parseResolvers()
if len(resolvers) == 0 {
return nil
}
// Also allow infra IPs and localhost
allowed := make(map[string]bool)
allowed["127.0.0.1"] = true
allowed["0.0.0.0"] = true
for _, r := range resolvers {
allowed[r] = true
}
// Build a set of UIDs belonging to DNS server processes (named, unbound,
// etc.) so we can skip their connections without reading /etc/passwd
// on every loop iteration.
dnsServerUIDs := resolveDNSServerUIDs()
// Parse /proc/net/tcp for connections to port 53
data, err := osFS.ReadFile("/proc/net/tcp")
if err != nil {
return nil
}
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) < 8 || fields[0] == "sl" {
continue
}
// State 01 = ESTABLISHED
if fields[3] != "01" {
continue
}
remoteIP, remotePort := parseHexAddr(fields[2])
if remotePort != 53 {
continue
}
if allowed[remoteIP] {
continue
}
// Check if it's an infra IP
if isInfraIP(remoteIP, cfg.InfraIPs) {
continue
}
// Skip connections owned by DNS server processes (e.g. named
// doing recursive resolution talks to many different servers)
if dnsServerUIDs[fields[7]] {
continue
}
_, localPort := parseHexAddr(fields[1])
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "dns_connection",
Message: fmt.Sprintf("DNS connection to non-configured resolver: %s", remoteIP),
Details: fmt.Sprintf("Local port: %d, Remote: %s:53\nConfigured resolvers: %s", localPort, remoteIP, strings.Join(resolvers, ", ")),
})
}
return findings
}
// resolveDNSServerUIDs returns a set of UIDs that belong to known DNS
// server users (named, unbound, pdns) by reading /etc/passwd once.
func resolveDNSServerUIDs() map[string]bool {
uids := make(map[string]bool)
data, err := osFS.ReadFile("/etc/passwd")
if err != nil {
return uids
}
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Split(line, ":")
if len(fields) >= 3 && dnsServerUsers[fields[0]] {
uids[fields[2]] = true
}
}
return uids
}
// parseResolvers returns the configured nameservers. When /etc/resolv.conf
// points at a loopback stub (systemd-resolved's 127.0.0.53, a local
// dnsmasq) the upstreams behind it are configured resolvers too: clients
// never reach them directly, but the stub does, and nothing else on the
// host is expected to.
func parseResolvers() []string {
resolvers := parseResolverFile("/etc/resolv.conf")
loopback := false
for _, r := range resolvers {
if ip := net.ParseIP(r); ip != nil && ip.IsLoopback() {
loopback = true
break
}
}
if loopback {
resolvers = append(resolvers, parseResolverFile(resolvedUpstreamsPath)...)
resolvers = append(resolvers, parseDNSMasqServerFile(dnsmasqConfigPath)...)
if paths, err := osFS.Glob(dnsmasqConfigGlob); err == nil {
for _, path := range paths {
resolvers = append(resolvers, parseDNSMasqServerFile(path)...)
}
}
}
seen := make(map[string]bool, len(resolvers))
unique := resolvers[:0]
for _, resolver := range resolvers {
if !seen[resolver] {
seen[resolver] = true
unique = append(unique, resolver)
}
}
return unique
}
func parseResolverFile(path string) []string {
f, err := osFS.Open(path)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
var resolvers []string
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if strings.HasPrefix(line, "nameserver ") {
ip := strings.TrimSpace(strings.TrimPrefix(line, "nameserver"))
if ip != "" {
resolvers = append(resolvers, ip)
}
}
}
return resolvers
}
func parseDNSMasqServerFile(path string) []string {
f, err := osFS.Open(path)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
var resolvers []string
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
key, value, ok := strings.Cut(line, "=")
if !ok || strings.TrimSpace(key) != "server" {
continue
}
if fields := strings.Fields(value); len(fields) > 0 {
if ip := dnsmasqServerIP(fields[0]); ip != "" {
resolvers = append(resolvers, ip)
}
}
}
return resolvers
}
func dnsmasqServerIP(value string) string {
value = strings.TrimSpace(value)
if strings.HasPrefix(value, "/") {
if slash := strings.LastIndexByte(value, '/'); slash >= 0 {
value = value[slash+1:]
}
}
if at := strings.IndexByte(value, '@'); at >= 0 {
value = value[:at]
}
if strings.HasPrefix(value, "[") {
if close := strings.IndexByte(value, ']'); close > 1 {
value = value[1:close]
}
} else if hash := strings.LastIndexByte(value, '#'); hash > 0 {
value = value[:hash]
}
if ip := net.ParseIP(value); ip != nil {
return ip.String()
}
return ""
}
package checks
import (
"context"
"encoding/json"
"fmt"
"net/netip"
"path/filepath"
"sort"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// CheckDNSZoneChanges monitors named zone files for tampering.
//
// A raw file-hash watch over /var/named is too coarse: every cPanel serial
// bump, AutoSSL DCV TXT record, DKIM rotation, and customer Zone Editor edit
// rewrites the file, so hashing the whole zone alerts on routine activity and
// buries real hijacks. This check instead compares only the security-relevant
// records and weighs the change against cPanel provenance:
//
// - The "security fingerprint" covers delegation (NS), mail (MX), and apex/
// wildcard address records -- the records an attacker rewrites to take over
// a domain. Serial, TXT/DKIM/SPF/DCV, and ordinary subdomain A records are
// ignored, so legitimate churn stays quiet.
// - cPanel stamps each zone it writes with an "(update_time):" header. A
// security change with no advance of that stamp means the file was edited
// out of band (direct file write, or a non-cPanel path) -- the signature of
// a hijack -- and is reported High. A security change that did go through
// cPanel is trusted more: an NS/MX move still surfaces as a Warning (could
// be a compromised account), while an apex/wildcard address repoint by the
// authenticated owner is routine and stays quiet.
//
// There is deliberately no bulk-suppression gate: a mass out-of-band NS rewrite
// across every hosted domain is exactly the incident operators must see, and
// the per-record/provenance model already keeps benign mass operations quiet.
func CheckDNSZoneChanges(_ context.Context, _ *config.Config, store *state.Store) []alert.Finding {
// cPanel stores zone files in /var/named/
zoneDir := "/var/named"
zones, err := osFS.ReadDir(zoneDir)
if err != nil {
return nil
}
var findings []alert.Finding
for _, zone := range zones {
if zone.IsDir() {
continue
}
name := zone.Name()
if !strings.HasSuffix(name, ".db") {
continue
}
fullPath := filepath.Join(zoneDir, name)
data, err := osFS.ReadFile(fullPath)
if err != nil {
continue
}
key := "_dns_zone:" + name
rawPrev, exists := store.GetRaw(key)
var prev dnsZoneState
prevOK := false
if exists {
prev, prevOK = decodeDNSZoneState(rawPrev)
}
secHash, delegHash := parseZoneSecurity(data, zoneOrigin(name))
cur := dnsZoneState{
File: hashBytes(data),
Sec: secHash,
Deleg: delegHash,
Prov: zoneUpdateTime(data),
}
cur.Panel = cur.Prov > 0
if exists && prevOK && !prev.Panel {
cur.Panel = false
cur.Prov = 0
}
if !cur.Panel {
cur.Prov = 0
} else if exists && prevOK && prev.Panel && cur.Prov < prev.Prov {
cur.Prov = prev.Prov
}
store.SetRaw(key, cur.encode())
if !exists {
continue // first sight: baseline only
}
if !prevOK {
continue // legacy bare-hash state: re-baseline silently, no alert
}
if prev.File == cur.File {
continue // file unchanged
}
if prev.Sec == cur.Sec {
continue // only non-security content changed (serial, TXT, subdomain A)
}
// A security-relevant record changed. Weigh it against cPanel provenance.
if !prev.Panel || !cur.Panel || cur.Prov <= prev.Prov {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "dns_zone_change",
Message: fmt.Sprintf("DNS records changed outside cPanel: %s", name),
Details: fmt.Sprintf("File: %s\nDelegation, mail, or apex address records changed with no matching cPanel zone edit. A direct edit to a zone file that bypasses cPanel is the signature of DNS hijacking.", fullPath),
})
continue
}
if prev.Deleg == cur.Deleg {
continue // cPanel-applied apex/wildcard address repoint: routine owner action
}
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "dns_zone_change",
Message: fmt.Sprintf("DNS delegation or mail records changed: %s", name),
Details: fmt.Sprintf("File: %s\nNS or MX records were changed through cPanel. Confirm this was an authorized account action and not a compromised login redirecting the domain or its mail.", fullPath),
})
}
return findings
}
// dnsZoneState is the per-zone watch state persisted under "_dns_zone:<name>".
// File is the whole-file hash (cheap "did anything change" gate); Sec is the
// hash of the security-relevant record set (NS, MX, apex/wildcard A/AAAA);
// Deleg is the hash of the delegation/mail subset (NS, MX). Panel records
// whether the zone was already known as cPanel-managed; manually managed zones
// cannot become trusted just by adding a cPanel-looking file header later. Prov
// is the cPanel "(update_time):" stamp, 0 when Panel is false.
type dnsZoneState struct {
File string `json:"f"`
Sec string `json:"s"`
Deleg string `json:"d"`
Prov int64 `json:"p"`
Panel bool `json:"c,omitempty"`
}
func (z dnsZoneState) encode() string {
b, _ := json.Marshal(z)
return string(b)
}
// decodeDNSZoneState parses persisted state. It returns ok=false for the legacy
// bare-hash format (a 64-char hex string written by earlier versions) so the
// caller re-baselines instead of treating the upgrade as a zone change.
func decodeDNSZoneState(s string) (dnsZoneState, bool) {
if !strings.HasPrefix(s, "{") {
return dnsZoneState{}, false
}
var raw struct {
File string `json:"f"`
Sec string `json:"s"`
Deleg string `json:"d"`
Prov int64 `json:"p"`
Panel *bool `json:"c"`
}
if err := json.Unmarshal([]byte(s), &raw); err != nil || raw.File == "" {
return dnsZoneState{}, false
}
z := dnsZoneState{
File: raw.File,
Sec: raw.Sec,
Deleg: raw.Deleg,
Prov: raw.Prov,
Panel: raw.Prov > 0,
}
if raw.Panel != nil {
z.Panel = *raw.Panel
}
if !z.Panel {
z.Prov = 0
}
return z, true
}
// zoneOrigin derives the zone apex (FQDN, trailing dot) from a zone file name,
// e.g. "example.com.db" -> "example.com.".
func zoneOrigin(filename string) string {
return canonicalZoneName(strings.TrimSuffix(filename, ".db"))
}
// zoneUpdateTime extracts the epoch from cPanel's "(update_time):<n>" zone
// header. Returns 0 when absent (manually managed or non-cPanel zone).
func zoneUpdateTime(data []byte) int64 {
const marker = "(update_time):"
for _, raw := range strings.Split(string(data), "\n") {
line := strings.TrimSpace(raw)
if line == "" {
continue
}
if !strings.HasPrefix(line, ";") {
return 0
}
if !strings.HasPrefix(line, "; cPanel ") ||
!strings.Contains(line, "Cpanel::ZoneFile::VERSION:") ||
!strings.Contains(line, marker) {
continue
}
rest := line[strings.Index(line, marker)+len(marker):]
j := 0
for j < len(rest) && rest[j] >= '0' && rest[j] <= '9' {
j++
}
if j == 0 {
return 0
}
n, err := strconv.ParseInt(rest[:j], 10, 64)
if err != nil {
return 0
}
return n
}
return 0
}
// parseZoneSecurity parses a BIND zone file and returns two hashes: the
// security fingerprint (NS, MX, apex/wildcard A and AAAA) and the delegation
// subset (NS, MX only). Records are canonicalized and sorted so reordering or
// whitespace changes do not register as a difference. SOA, ordinary subdomain
// addresses, and all TXT/DKIM/SPF/DCV records are excluded by design.
func parseZoneSecurity(data []byte, origin string) (secHash, delegHash string) {
var secRecs, delegRecs []string
lastOwnerFQDN := ""
zoneOrigin := canonicalZoneName(origin)
curOrigin := zoneOrigin
parenDepth := 0
pendingRaw := ""
pendingLine := ""
processRecord := func(raw, line string) {
fields := zoneRecordFields(line)
if len(fields) == 0 {
return
}
// A leading blank means "same owner as the previous record".
fqdn := lastOwnerFQDN
rest := fields
if r := []rune(raw); len(r) > 0 && r[0] != ' ' && r[0] != '\t' {
fqdn = toFQDN(fields[0], curOrigin)
lastOwnerFQDN = fqdn
rest = fields[1:]
}
typ, rdata := zoneRecordType(rest)
if typ == "" || fqdn == "" {
return
}
malformedParenSuffix := zoneMalformedParenSuffix(line)
switch typ {
case "NS":
if len(rdata) >= 1 {
rec := "NS " + fqdn + " " + toFQDN(rdata[0], curOrigin) + malformedParenSuffix
delegRecs = append(delegRecs, rec)
secRecs = append(secRecs, rec)
}
case "MX":
if len(rdata) >= 2 {
rec := "MX " + fqdn + " " + canonicalMXPreference(rdata[0]) + " " + toFQDN(rdata[1], curOrigin) + malformedParenSuffix
delegRecs = append(delegRecs, rec)
secRecs = append(secRecs, rec)
}
case "A", "AAAA":
if len(rdata) >= 1 && isApexOrWildcard(fqdn, zoneOrigin) {
rec := typ + " " + fqdn + " " + canonicalIPLiteral(typ, rdata[0]) + malformedParenSuffix
secRecs = append(secRecs, rec)
}
}
}
lines := strings.Split(string(data), "\n")
pendingCloses := false
for i, raw := range lines {
line := stripZoneComment(raw)
trimmed := strings.TrimSpace(line)
if parenDepth > 0 {
if !pendingCloses && continuationStartsZoneEntry(raw, line) {
processRecord(pendingRaw, pendingLine)
pendingRaw = ""
pendingLine = ""
parenDepth = 0
} else {
if trimmed != "" {
pendingLine += " " + trimmed
}
parenDepth += zoneParenDelta(line)
if parenDepth > 0 {
continue
}
parenDepth = 0
processRecord(pendingRaw, pendingLine)
pendingRaw = ""
pendingLine = ""
pendingCloses = false
continue
}
}
if trimmed == "" {
continue
}
if strings.HasPrefix(trimmed, "$") {
f := strings.Fields(trimmed)
if len(f) == 0 {
continue
}
if len(f) >= 2 && strings.EqualFold(f[0], "$ORIGIN") {
curOrigin = toFQDN(f[1], curOrigin)
continue
}
if rec, ok := zoneDirectiveFingerprint(f); ok {
delegRecs = append(delegRecs, rec)
secRecs = append(secRecs, rec)
}
continue
}
parenDepth = zoneParenDelta(line)
if parenDepth > 0 {
pendingRaw = raw
pendingLine = trimmed
pendingCloses = zoneContinuationCloses(parenDepth, lines[i+1:])
continue
}
if parenDepth < 0 {
parenDepth = 0
}
processRecord(raw, line)
}
if pendingLine != "" {
processRecord(pendingRaw, pendingLine)
}
return hashSortedRecords(secRecs), hashSortedRecords(delegRecs)
}
func zoneContinuationCloses(depth int, lines []string) bool {
for _, raw := range lines {
depth += zoneParenDelta(stripZoneComment(raw))
if depth <= 0 {
return true
}
}
return false
}
// zoneRecordType skips leading TTL and class tokens and returns the record type
// (upper-cased) plus its rdata tokens. Returns "" when no type is present.
func zoneRecordType(tokens []string) (string, []string) {
i := 0
for i < len(tokens) && (isZoneTTL(tokens[i]) || isZoneClass(tokens[i])) {
i++
}
if i >= len(tokens) {
return "", nil
}
return strings.ToUpper(tokens[i]), tokens[i+1:]
}
func isZoneClass(t string) bool {
switch strings.ToUpper(t) {
case "IN", "CH", "HS", "CS":
return true
}
return false
}
// isZoneTTL reports whether a token is a BIND TTL. BIND accepts plain seconds
// and compact unit sequences such as 1h30m.
func isZoneTTL(t string) bool {
if t == "" {
return false
}
i := 0
for i < len(t) {
start := i
for i < len(t) && t[i] >= '0' && t[i] <= '9' {
i++
}
if i == start {
return false
}
if i == len(t) {
return true
}
switch t[i] {
case 's', 'S', 'm', 'M', 'h', 'H', 'd', 'D', 'w', 'W':
i++
default:
return false
}
}
return true
}
func zoneRecordFields(line string) []string {
var b strings.Builder
b.Grow(len(line))
inQuote := false
escaped := false
for i := 0; i < len(line); i++ {
if escaped {
escaped = false
b.WriteByte(line[i])
continue
}
switch line[i] {
case '\\':
if inQuote {
escaped = true
}
b.WriteByte(line[i])
case '"':
inQuote = !inQuote
b.WriteByte(line[i])
case '(', ')':
if inQuote {
b.WriteByte(line[i])
} else {
b.WriteByte(' ')
}
default:
b.WriteByte(line[i])
}
}
return strings.Fields(b.String())
}
func zoneDirectiveFingerprint(fields []string) (string, bool) {
switch strings.ToUpper(fields[0]) {
case "$INCLUDE", "$GENERATE":
return "DIRECTIVE " + strings.ToUpper(fields[0]) + " " + strings.Join(fields[1:], " "), true
}
return "", false
}
func continuationStartsZoneEntry(raw, line string) bool {
if raw == "" || raw[0] == ' ' || raw[0] == '\t' {
return false
}
trimmed := strings.TrimSpace(line)
if trimmed == "" {
return false
}
if strings.HasPrefix(trimmed, "$") {
return true
}
fields := zoneRecordFields(line)
if len(fields) < 2 {
return false
}
typ, _ := zoneRecordType(fields[1:])
return isZoneRecordType(typ)
}
func isZoneRecordType(typ string) bool {
switch typ {
case "A", "AAAA", "CAA", "CNAME", "DNAME", "DNSKEY", "DS", "HTTPS",
"LOC", "MX", "NAPTR", "NS", "PTR", "SOA", "SPF", "SRV", "SSHFP",
"SVCB", "TLSA", "TXT":
return true
}
if !strings.HasPrefix(typ, "TYPE") {
return false
}
_, err := strconv.ParseUint(strings.TrimPrefix(typ, "TYPE"), 10, 16)
return err == nil
}
func canonicalMXPreference(s string) string {
if !isDecimalUint16(s) {
return s
}
n, err := strconv.ParseUint(s, 10, 16)
if err != nil {
return s
}
return strconv.FormatUint(n, 10)
}
func isDecimalUint16(s string) bool {
if s == "" {
return false
}
for i := 0; i < len(s); i++ {
if s[i] < '0' || s[i] > '9' {
return false
}
}
_, err := strconv.ParseUint(s, 10, 16)
return err == nil
}
func canonicalIPLiteral(typ, s string) string {
if strings.Contains(s, "%") {
return s
}
addr, err := netip.ParseAddr(s)
if err != nil {
return s
}
switch typ {
case "A":
if !addr.Is4() {
return s
}
case "AAAA":
if !addr.Is6() {
return s
}
default:
return s
}
return addr.String()
}
func isApexOrWildcard(fqdn, origin string) bool {
return fqdn == origin || strings.HasPrefix(fqdn, "*.")
}
func toFQDN(name, origin string) string {
name = strings.TrimSpace(name)
origin = canonicalZoneName(origin)
if name == "" || name == "@" {
return origin
}
if strings.HasSuffix(name, ".") {
return canonicalZoneName(name)
}
if origin == "." {
return canonicalZoneName(name)
}
return canonicalZoneName(name + "." + origin)
}
func canonicalZoneName(s string) string {
s = strings.TrimSpace(s)
if s == "." {
return "."
}
if strings.HasSuffix(s, ".") {
s = strings.TrimSuffix(s, ".")
if s == "" {
return "."
}
return lowerASCII(s) + "."
}
if s == "" {
return "."
}
return lowerASCII(s) + "."
}
func lowerASCII(s string) string {
var b strings.Builder
changed := false
for i := 0; i < len(s); i++ {
c := s[i]
if c >= 'A' && c <= 'Z' {
if !changed {
b.Grow(len(s))
b.WriteString(s[:i])
changed = true
}
c += 'a' - 'A'
}
if changed {
b.WriteByte(c)
}
}
if !changed {
return s
}
return b.String()
}
func stripZoneComment(line string) string {
inQuote := false
escaped := false
for i := 0; i < len(line); i++ {
if escaped {
escaped = false
continue
}
switch line[i] {
case '\\':
if inQuote {
escaped = true
}
case '"':
inQuote = !inQuote
case ';':
if !inQuote {
return line[:i]
}
}
}
return line
}
func zoneParenDelta(line string) int {
inQuote := false
escaped := false
delta := 0
for i := 0; i < len(line); i++ {
if escaped {
escaped = false
continue
}
switch line[i] {
case '\\':
if inQuote {
escaped = true
}
case '"':
inQuote = !inQuote
case '(':
if !inQuote {
delta++
}
case ')':
if !inQuote {
delta--
}
}
}
return delta
}
func zoneMalformedParenSuffix(line string) string {
if !zoneParensMalformed(line) {
return ""
}
return " MALFORMED_PARENS " + strings.TrimSpace(line)
}
func zoneParensMalformed(line string) bool {
inQuote := false
escaped := false
depth := 0
for i := 0; i < len(line); i++ {
if escaped {
escaped = false
continue
}
switch line[i] {
case '\\':
if inQuote {
escaped = true
}
case '"':
inQuote = !inQuote
case '(':
if !inQuote {
if zoneParenAttachedToToken(line, i) {
return true
}
depth++
}
case ')':
if !inQuote {
if zoneParenAttachedToToken(line, i) {
return true
}
depth--
if depth < 0 {
return true
}
}
}
}
return depth != 0
}
func zoneParenAttachedToToken(line string, i int) bool {
return (i > 0 && !isZoneSpace(line[i-1])) || (i+1 < len(line) && !isZoneSpace(line[i+1]))
}
func isZoneSpace(b byte) bool {
switch b {
case ' ', '\t', '\r', '\n':
return true
}
return false
}
func hashSortedRecords(recs []string) string {
sort.Strings(recs)
n := 0
for _, rec := range recs {
if n == 0 || rec != recs[n-1] {
recs[n] = rec
n++
}
}
return hashBytes([]byte(strings.Join(recs[:n], "\n")))
}
// CheckSSLCertIssuance monitors AutoSSL logs for new certificate issuance.
// Attackers may issue certificates for phishing domains using compromised accounts.
func CheckSSLCertIssuance(ctx context.Context, _ *config.Config, store *state.Store) []alert.Finding {
var findings []alert.Finding
// Check AutoSSL log
logPath := "/var/cpanel/logs/autossl"
entries, err := osFS.ReadDir(logPath)
if err != nil {
return nil
}
// Track the count of log files as a simple change indicator
currentCount := len(entries)
key := "_ssl_autossl_count"
prev, exists := store.GetRaw(key)
store.SetRaw(key, fmt.Sprintf("%d", currentCount))
if !exists {
return nil
}
prevCount := 0
fmt.Sscanf(prev, "%d", &prevCount)
if currentCount > prevCount {
// New AutoSSL activity - check the latest log
var latestLog string
var latestTime int64
for _, entry := range entries {
if entry.IsDir() {
continue
}
info, err := entry.Info()
if err != nil {
continue
}
if info.ModTime().Unix() > latestTime {
latestTime = info.ModTime().Unix()
latestLog = filepath.Join(logPath, entry.Name())
}
}
if latestLog != "" {
// Read the tail of the latest log for certificate issuance
lines := tailFile(latestLog, 50)
for _, line := range lines {
lineLower := strings.ToLower(line)
if strings.Contains(lineLower, "installed") || strings.Contains(lineLower, "issued") {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "ssl_cert_issued",
Message: "New SSL certificate issued via AutoSSL",
Details: truncate(line, 300),
})
}
}
}
}
return findings
}
package checks
import (
"bytes"
"crypto/sha256"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/processctx"
"github.com/pidginhost/csm/internal/safepath"
"github.com/pidginhost/csm/internal/yara"
)
var elfMagic = []byte("\x7fELF")
// docrootBinaryVerdict is what reading one program file established.
type docrootBinaryVerdict struct {
program bool // the file is an ELF image, not a script
matches []yara.Match
sha256 string
gap string // why the content was not scanned; empty when it was
}
type docrootBinaryKey struct {
id selfWriteFileIdentity
size int64
version string
maxBytes int64
yaraGeneration uint64
}
// docrootBinaryCacheMax bounds the verdict cache. Reaching it drops the whole
// cache, which costs only a rescan of the programs that start next.
const docrootBinaryCacheMax = 4096
// docrootBinaryCache keeps verdicts per file version so a program a site runs
// on every request is read once. The key includes the inode change time,
// which a writer cannot set back, so changed content is always read again.
var docrootBinaryCache = struct {
sync.Mutex
verdicts map[docrootBinaryKey]docrootBinaryVerdict
}{}
func resetDocrootBinaryCache() {
docrootBinaryCache.Lock()
docrootBinaryCache.verdicts = nil
docrootBinaryCache.Unlock()
}
func procExePath(pid int) string {
return "/proc/" + strconv.Itoa(pid) + "/exe"
}
// InspectDocrootExec judges a non-root process whose program file sits in a
// document root. Web servers run CGI programs from there and build tools run
// from node_modules, so the location alone is not a finding: a program that
// matches a YARA rule is reported at the rule's severity whoever started it,
// and one started directly by a PHP interpreter is reported as a Warning,
// because that is how a webshell launches a dropper or miner.
//
// filename is the name the process was started with and may be relative.
// /proc decides what actually runs: a script names its interpreter there,
// which lies outside the document root. Only when the process has already
// exited is the started name read instead, and then a file without the ELF
// magic is taken to be a script.
func InspectDocrootExec(cfg *config.Config, pid, uid int, filename, parentComm string, roots *DocrootIndex) *alert.Finding {
if uid == 0 || pid <= 0 {
return nil
}
exe, source := "", procExePath(pid)
if target, err := osFS.Readlink(source); err == nil {
// The kernel keeps a deleted program running and marks its path.
exe = strings.TrimSuffix(target, " (deleted)")
} else {
if !errors.Is(err, os.ErrNotExist) && !errors.Is(err, unix.ESRCH) {
return nil
}
// Cleaning '..' before refusing symlinks can name a different file
// than execve did. An exited process cannot disambiguate that path.
if !filepath.IsAbs(filename) || filepath.Clean(filename) != filename {
return nil
}
exe, source = filename, filename
}
docroot := roots.Lookup(exe)
if docroot == "" {
return nil
}
file, err := openDocrootBinary(source, source == procExePath(pid))
if err != nil {
return nil
}
defer file.Close()
if _, production := osFS.(realOS); production && source == procExePath(pid) {
// A PID can exec again between Readlink and Open. Attribute the
// pinned image, never the path observed before the open.
target, err := osFS.Readlink(fmt.Sprintf("/proc/self/fd/%d", file.Fd()))
if err != nil {
return nil
}
exe = strings.TrimSuffix(target, " (deleted)")
docroot = roots.Lookup(exe)
if docroot == "" {
return nil
}
}
v, ok := inspectDocrootBinary(cfg, file)
if !ok || !v.program {
return nil
}
details := fmt.Sprintf("UID: %d\nParent process: %s\nDocument root: %s", uid, parentComm, docroot)
finding := alert.Finding{
Check: "docroot_binary_exec",
Details: details,
FilePath: exe,
PID: pid,
ContentSHA256: v.sha256,
Timestamp: time.Now(),
}
if userName, account := processIdentityForUID(uid); userName != "" {
finding.Process = &processctx.ProcessContext{PID: pid, UID: uid, User: userName, Account: account}
}
if len(v.matches) > 0 {
names := make([]string, 0, len(v.matches))
finding.Severity = alert.Warning
for _, m := range v.matches {
names = append(names, m.RuleName)
if sev := yaraMatchSeverity(m.Meta["severity"]); sev > finding.Severity {
finding.Severity = sev
}
}
finding.Message = fmt.Sprintf("Program matching YARA rule %s started from document root: %s", strings.Join(names, ", "), exe)
return &finding
}
if !isPHPInterpreterComm(parentComm) {
return nil
}
note := "No YARA rule matched the program."
if v.gap != "" {
note = "The program was not scanned: " + v.gap + "."
}
finding.Severity = alert.Warning
finding.Message = fmt.Sprintf("PHP started a program from document root: %s", exe)
finding.Details = note + "\n" + details
return &finding
}
// openDocrootBinary follows only kernel-owned /proc executable links. An
// exited process's start name is tenant-controlled, including every parent.
func openDocrootBinary(path string, proc bool) (*os.File, error) {
if fs, production := osFS.(realOS); production {
if proc {
return fs.openRegularFile(path, 0)
}
dir, err := safepath.OpenDirNoFollow(filepath.Dir(path))
if err != nil {
return nil, err
}
defer func() { _ = dir.Close() }()
return dir.OpenFile(filepath.Base(path), os.O_RDONLY, 0)
}
return osFS.Open(path)
}
// inspectDocrootBinary returns the verdict for the opened image, reading it
// only when this version of the file has no verdict yet. Non-regular files,
// failed reads and files changed during the read have no usable verdict.
func inspectDocrootBinary(cfg *config.Config, file *os.File) (docrootBinaryVerdict, bool) {
info, err := file.Stat()
if err != nil || !info.Mode().IsRegular() {
return docrootBinaryVerdict{}, false
}
id, keyed := selfWriteIdentityFromFileInfo(info)
maxBytes := FullScanMaxFileBytes(cfg)
key := docrootBinaryKey{id: id, size: info.Size(), version: ContentDetectionVersion(), maxBytes: maxBytes, yaraGeneration: yara.CacheGeneration()}
if keyed {
docrootBinaryCache.Lock()
v, hit := docrootBinaryCache.verdicts[key]
docrootBinaryCache.Unlock()
if hit {
return v, true
}
}
var header [4]byte
_, err = file.ReadAt(header[:], 0)
if err != nil && err != io.EOF {
return docrootBinaryVerdict{}, false
}
v := docrootBinaryVerdict{program: bytes.Equal(header[:], elfMagic)}
final := true
switch {
case !v.program:
case info.Size() > maxBytes:
v.gap = fmt.Sprintf("it is larger than the %d-byte scan limit", maxBytes)
default:
data, readErr := io.ReadAll(io.LimitReader(file, maxBytes+1))
if readErr != nil || int64(len(data)) != info.Size() {
return docrootBinaryVerdict{}, false
}
afterRead, statErr := file.Stat()
if statErr != nil || !sameFileSnapshot(info, afterRead) {
return docrootBinaryVerdict{}, false
}
if readID, ok := selfWriteIdentityFromFileInfo(afterRead); keyed && (!ok || id != readID) {
return docrootBinaryVerdict{}, false
}
v.sha256 = fmt.Sprintf("%x", sha256.Sum256(data))
backend := activeYARABackend()
if backend == nil || backend.RuleCount() == 0 {
v.gap, final = "no YARA rules are loaded", false
break
}
// No name: a program is scanned whatever it is called.
matches, scanErr := yara.ScanBytesChecked(backend, "", data)
if scanErr != nil {
v.gap, final = "YARA scan failed: "+scanErr.Error(), false
break
}
v.matches = matches
}
after, err := file.Stat()
if err != nil || !sameFileSnapshot(info, after) {
final = false
}
if err == nil {
if afterID, ok := selfWriteIdentityFromFileInfo(after); keyed && (!ok || id != afterID) {
final = false
}
}
if keyed && final {
docrootBinaryCache.Lock()
if len(docrootBinaryCache.verdicts) >= docrootBinaryCacheMax || docrootBinaryCache.verdicts == nil {
docrootBinaryCache.verdicts = make(map[docrootBinaryKey]docrootBinaryVerdict)
}
docrootBinaryCache.verdicts[key] = v
docrootBinaryCache.Unlock()
}
return v, true
}
// isPHPInterpreterComm reports whether a process name is a PHP interpreter:
// lsphp, php-fpm, php-cgi or the php CLI, with an optional version suffix as
// distributions install them (php8.2, php-fpm8.1). A process started through
// a symlink is named after the link, which covers cPanel's ea-php81 commands.
func isPHPInterpreterComm(comm string) bool {
for _, name := range []string{"lsphp", "php-fpm", "php-cgi", "ea-php", "php"} {
if rest, ok := strings.CutPrefix(comm, name); ok && strings.Trim(rest, "0123456789.") == "" {
return true
}
}
return false
}
package checks
import "path/filepath"
// DocrootIndex holds the web document roots, normalized once per refresh.
// Lookups run on every file event and every process start, so they walk the
// path's ancestors instead of the root list: the cost follows path depth, not
// the number of docroots on the host.
type DocrootIndex struct {
roots map[string]struct{}
}
// NewDocrootIndex indexes docroots. Relative entries are ignored.
func NewDocrootIndex(docroots []string) *DocrootIndex {
idx := &DocrootIndex{roots: make(map[string]struct{}, len(docroots))}
for _, configuredRoot := range docroots {
root := filepath.Clean(configuredRoot)
if filepath.IsAbs(root) {
idx.roots[root] = struct{}{}
}
}
return idx
}
// Lookup returns the longest docroot that contains path, or "" when the path
// is not inside any docroot. Matching is component-safe:
// /home/a/public_html_old is not inside /home/a/public_html.
func (idx *DocrootIndex) Lookup(path string) string {
if idx == nil || len(idx.roots) == 0 {
return ""
}
path = filepath.Clean(path)
if !filepath.IsAbs(path) {
return ""
}
// The first ancestor found is the deepest, so the longest root wins. A
// path equal to a root is not inside it, so the walk starts at the parent.
for dir := path; ; {
parent := filepath.Dir(dir)
if parent == dir {
return ""
}
if _, ok := idx.roots[parent]; ok {
return parent
}
dir = parent
}
}
package checks
import (
"strings"
"github.com/pidginhost/csm/internal/netutil"
)
// DovecotRemoteIP returns the connection's address from a Dovecot login
// line, or "" when the line has none. Only fields outside user=<...> can
// name the connection; even the last rip= may be client text without them.
func DovecotRemoteIP(line string) string {
_, fields := DovecotLoginFields(line)
var ip string
for field := range strings.FieldsFuncSeq(fields, dovecotFieldSeparator) {
if value, ok := strings.CutPrefix(field, "rip="); ok {
ip, _ = netutil.ParseIPToken(value)
}
}
return ip
}
// DovecotLoginFields separates the raw login name from the fields that
// follow it. Dovecot does not escape angle brackets inside the name. Its
// closing bracket is therefore read from the right, before the optional
// server session, rather than at the first client-supplied '>'. A missing
// closing bracket withholds all fields. No-auth summaries may omit user=.
func DovecotLoginFields(line string) (user, fields string) {
start := strings.Index(line, "user=<")
if start < 0 {
return "", line
}
start += len("user=<")
end := len(line)
if session := dovecotFieldIndex(line, "session=<"); session >= start {
// A session follows the connection fields. A marker before them
// is still part of the name, including one with a forged rip=.
lastField := max(dovecotFieldIndex(line, "method="), dovecotFieldIndex(line, "rip="), dovecotFieldIndex(line, "lip="))
if session > lastField {
value := line[session+len("session=<"):]
close := strings.IndexByte(value, '>')
if close < 0 || ((close+1 == len(value) || dovecotFieldSeparator(rune(value[close+1]))) && !strings.ContainsAny(value[:close], "<")) {
end = session
}
}
}
close := strings.LastIndexByte(line[start:end], '>')
if close < 0 {
return "", ""
}
close += start
if close+1 < len(line) && !dovecotFieldSeparator(rune(line[close+1])) {
return "", ""
}
return line[start:close], line[close+1:]
}
func dovecotFieldSeparator(r rune) bool {
return r == ',' || r == ' ' || r == '\t' || r == '\r' || r == '\n'
}
func dovecotFieldIndex(line, key string) int {
for {
i := strings.LastIndex(line, key)
if i <= 0 || dovecotFieldSeparator(rune(line[i-1])) {
return i
}
line = line[:i]
}
}
// DovecotServiceMessage returns the message only for a Dovecot syslog tag.
func DovecotServiceMessage(line string) string {
tagEnd := strings.Index(line, ": ")
if tagEnd < 0 {
return ""
}
prefix := line[:tagEnd]
tagStart := strings.LastIndexAny(prefix, " \t")
if tagStart >= 0 {
prefix = prefix[tagStart+1:]
}
if !isDovecotProgramTag(prefix) {
return ""
}
return strings.TrimSpace(line[tagEnd+2:])
}
func isDovecotProgramTag(tag string) bool {
if tag == "dovecot" {
return true
}
const prefix = "dovecot["
if !strings.HasPrefix(tag, prefix) || !strings.HasSuffix(tag, "]") {
return false
}
pid := tag[len(prefix) : len(tag)-1]
if pid == "" {
return false
}
for i := 0; i < len(pid); i++ {
if pid[i] < '0' || pid[i] > '9' {
return false
}
}
return true
}
// DovecotLoginSucceeded matches the server status before any client text.
func DovecotLoginSucceeded(line string) bool {
msg := DovecotServiceMessage(line)
return strings.HasPrefix(msg, "imap-login: Logged in: user=<") ||
strings.HasPrefix(msg, "imap-login: Login: user=<") ||
strings.HasPrefix(msg, "pop3-login: Logged in: user=<") ||
strings.HasPrefix(msg, "pop3-login: Login: user=<") ||
strings.HasPrefix(msg, "managesieve-login: Logged in: user=<") ||
strings.HasPrefix(msg, "managesieve-login: Login: user=<")
}
// DovecotLoginMessage excludes auth database messages and other services
// whose client text can imitate a login summary.
func DovecotLoginMessage(line string) string {
msg := DovecotServiceMessage(line)
if strings.HasPrefix(msg, "imap-login:") || strings.HasPrefix(msg, "pop3-login:") || strings.HasPrefix(msg, "managesieve-login:") {
return msg
}
return ""
}
package checks
import (
"context"
"errors"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
var emailHashes = newEmailHashPool(3)
var emailMailboxAudits = newScanBatchMonitor()
type emailHashPool struct {
slots chan struct{}
waiting *queuehealth.Tracker
hashes *queuehealth.Tracker
}
func newEmailHashPool(capacity int) *emailHashPool {
return &emailHashPool{
slots: make(chan struct{}, capacity),
waiting: queuehealth.New(0, checkTimeout),
hashes: queuehealth.NewSharedCapacity(capacity, checkTimeout),
}
}
type emailHashWork struct {
pool *emailHashPool
ticket queuehealth.Ticket
remaining atomic.Int32
failOnce sync.Once
}
func (p *emailHashPool) acquire(ctx context.Context) (_ *emailHashWork, err error) {
waiting := p.waiting.Begin(time.Now())
defer func() {
if errors.Is(err, context.DeadlineExceeded) {
waiting.Reject(time.Now())
} else {
// Explicit cancellation withdraws demand without losing work.
waiting.Finish(time.Now())
}
}()
select {
case p.slots <- struct{}{}:
case <-ctx.Done():
return nil, ctx.Err()
}
if ctxErr := ctx.Err(); ctxErr != nil {
<-p.slots
return nil, ctxErr
}
work := &emailHashWork{pool: p, ticket: p.hashes.Begin(time.Now())}
work.remaining.Store(2)
return work, nil
}
func (w *emailHashWork) fail() {
w.failOnce.Do(func() { w.pool.hashes.Lose(time.Now(), 1) })
}
func (w *emailHashWork) release() {
// KDFs cannot be interrupted. The worker and caller share the slot so
// cancellation cannot hide a running KDF or admit unbounded late results.
if w.remaining.Add(-1) == 0 {
w.ticket.Finish(time.Now())
<-w.pool.slots
}
}
type emailHashResult struct {
match bool
err error
}
func (p *emailHashPool) execute(work *emailHashWork, match func(string) (bool, error), candidate string, done chan<- emailHashResult) {
work.ticket.Start(time.Now())
completed := false
defer func() {
if !completed {
work.fail()
}
work.release()
}()
matched, err := match(candidate)
if err != nil {
work.fail()
// Decoder/KDF errors may embed secret input.
err = errEmailPasswordVerify
}
done <- emailHashResult{match: matched, err: err}
completed = true
}
func (p *emailHashPool) matches(ctx context.Context, match func(string) (bool, error), candidate string) (_ bool, err error) {
if ctxErr := ctx.Err(); ctxErr != nil {
return false, ctxErr
}
if len(candidate) > maxEmailCandidateBytes || strings.ContainsRune(candidate, 0) {
return false, errEmailCandidate
}
work, err := p.acquire(ctx)
if err != nil {
return false, err
}
defer func() {
if errors.Is(err, context.DeadlineExceeded) {
work.fail()
}
work.release()
}()
done := make(chan emailHashResult, 1)
go p.execute(work, match, candidate, done)
select {
case got := <-done:
if ctxErr := ctx.Err(); ctxErr != nil {
return false, ctxErr
}
return got.match, got.err
case <-ctx.Done():
return false, ctx.Err()
}
}
func (p *emailHashPool) QueueStatuses(now time.Time) map[string]queuehealth.Status {
waiting := p.waiting.Snapshot(now)
// Scan invocations share the hash slots; their waiting callers have no
// fixed global cap. Do not label that backlog with the KDF concurrency.
waiting.CapacityUnavailable = true
return map[string]queuehealth.Status{
"waiting": waiting,
"hashes": p.hashes.Snapshot(now),
}
}
// EmailPasswordQueueStatuses reads admission and KDF ownership from memory.
func EmailPasswordQueueStatuses(now time.Time) map[string]queuehealth.Status {
rows := emailHashes.QueueStatuses(now)
rows["mailboxes"] = emailMailboxAudits.snapshot(now)
return rows
}
package checks
import (
"context"
"crypto/md5" // #nosec G501 -- Read-only verification of existing Dovecot MD5 hashes, never password creation.
"crypto/sha1" // #nosec G505 -- Read-only verification of existing Dovecot SHA1 hashes, never password creation.
"crypto/sha256"
"crypto/sha512"
"crypto/subtle"
"encoding/base64"
"encoding/hex"
"errors"
"hash"
"strconv"
"strings"
"github.com/go-crypt/crypt/algorithm"
"github.com/go-crypt/crypt/algorithm/argon2"
"github.com/go-crypt/crypt/algorithm/bcrypt"
"github.com/go-crypt/crypt/algorithm/md5crypt"
"github.com/go-crypt/crypt/algorithm/shacrypt"
)
const (
maxEmailHashBytes = 4096
maxEmailCandidateBytes = 256
)
var (
errEmailHashUnsupported = errors.New("unsupported password hash scheme")
errEmailHashInvalid = errors.New("malformed password hash")
errEmailHashCost = errors.New("password hash exceeds audit cost limits")
errEmailPasswordVerify = errors.New("password verification failed")
errEmailCandidate = errors.New("password candidate exceeds audit limits")
)
type emailPasswordVerifier struct {
match func(string) (bool, error)
}
func (v *emailPasswordVerifier) matches(ctx context.Context, candidate string) (bool, error) {
return emailHashes.matches(ctx, v.match, candidate)
}
func (v *emailPasswordVerifier) firstMatch(ctx context.Context, candidates []string) (string, error) {
for _, candidate := range candidates {
matched, err := v.matches(ctx, candidate)
if err != nil {
return "", err
}
if matched {
return candidate, nil
}
}
return "", ctx.Err()
}
func parseEmailPasswordHash(stored string) (*emailPasswordVerifier, error) {
if len(stored) == 0 || len(stored) > maxEmailHashBytes || strings.ContainsRune(stored, 0) {
return nil, errEmailHashInvalid
}
scheme, encoded := "CRYPT", stored
if strings.HasPrefix(stored, "{") {
end := strings.IndexByte(stored, '}')
if end < 2 {
return nil, errEmailHashInvalid
}
scheme, encoded = strings.ToUpper(stored[1:end]), stored[end+1:]
}
base, encoding, _ := strings.Cut(scheme, ".")
if encoding != "" && (base == "PLAIN" || strings.HasSuffix(base, "CRYPT") || strings.HasPrefix(base, "ARGON2")) {
decoded, err := decodeEmailDigest(encoding, encoded)
if err != nil {
return nil, err
}
encoded, encoding = string(decoded), ""
if strings.ContainsRune(encoded, 0) {
return nil, errEmailHashInvalid
}
}
switch base {
case "PLAIN":
return &emailPasswordVerifier{match: func(candidate string) (bool, error) {
return subtle.ConstantTimeCompare([]byte(encoded), []byte(candidate)) == 1, nil
}}, nil
case "PLAIN-MD5", "LDAP-MD5", "SMD5", "SHA", "SHA1", "SSHA", "SHA256", "SSHA256", "SHA512", "SSHA512":
return parseEmailDigest(base, encoding, encoded)
case "CRYPT", "SHA512-CRYPT", "SHA256-CRYPT", "MD5-CRYPT", "BLF-CRYPT", "ARGON2I", "ARGON2ID":
if encoding != "" {
return nil, errEmailHashUnsupported
}
default:
return nil, errEmailHashUnsupported
}
prefixes := map[string]string{
"SHA512-CRYPT": "$6$", "SHA256-CRYPT": "$5$", "MD5-CRYPT": "$1$",
"BLF-CRYPT": "$2", "ARGON2I": "$argon2i$", "ARGON2ID": "$argon2id$",
}
if prefix := prefixes[base]; prefix != "" && !strings.HasPrefix(encoded, prefix) {
return nil, errEmailHashInvalid
}
decode, err := boundedEmailCryptDecoder(encoded)
if err != nil {
return nil, err
}
digest, err := decode(encoded)
if err != nil {
return nil, errEmailHashInvalid
}
return &emailPasswordVerifier{match: digest.MatchAdvanced}, nil
}
// Validate costs before a library sees the digest. Strict fields also prevent
// duplicate parameters or library defaults from changing the checked cost.
func boundedEmailCryptDecoder(encoded string) (func(string) (algorithm.Digest, error), error) {
p := strings.Split(encoded, "$")
if len(p) < 2 || p[0] != "" {
return nil, errEmailHashUnsupported
}
switch p[1] {
case "1", "5", "6":
keyLen, saltMax := 22, 8
if p[1] != "1" {
saltMax, keyLen = 16, 43
if p[1] == "6" {
keyLen = 86
}
if len(p) == 5 {
rounds, ok := strings.CutPrefix(p[2], "rounds=")
if !ok {
return nil, errEmailHashInvalid
}
if _, err := emailHashNumber(rounds, 1000, 1000000); err != nil {
return nil, err
}
p = append(p[:2:2], p[3:]...)
}
}
if len(p) != 4 || len(p[2]) > saltMax || !emailCryptChars(p[2]) || len(p[3]) != keyLen || !emailCryptChars(p[3]) {
return nil, errEmailHashInvalid
}
if p[1] == "1" {
return md5crypt.Decode, nil
}
return shacrypt.Decode, nil
case "2a", "2b", "2y":
if len(p) != 4 || len(p[2]) != 2 || len(p[3]) != 53 || !emailCryptChars(p[3]) {
return nil, errEmailHashInvalid
}
if _, err := emailHashNumber(p[2], 4, 14); err != nil {
return nil, err
}
return bcrypt.Decode, nil
case "argon2i", "argon2id":
if err := validateEmailArgon2(p); err != nil {
return nil, err
}
return argon2.Decode, nil
default:
return nil, errEmailHashUnsupported
}
}
func emailHashNumber(s string, min, max uint64) (uint64, error) {
if s == "" || strings.IndexFunc(s, func(r rune) bool { return r < '0' || r > '9' }) >= 0 {
return 0, errEmailHashInvalid
}
n, err := strconv.ParseUint(s, 10, 32)
if err != nil || n > max {
return 0, errEmailHashCost
}
if n < min {
return 0, errEmailHashInvalid
}
return n, nil
}
func emailCryptChars(s string) bool {
return s != "" && strings.IndexFunc(s, func(r rune) bool {
valid := r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '.' || r == '/'
return !valid
}) < 0
}
func validateEmailArgon2(p []string) error {
if len(p) != 6 || p[2] != "v=19" {
return errEmailHashInvalid
}
params := strings.Split(p[3], ",")
if len(params) != 3 {
return errEmailHashInvalid
}
values := make(map[string]uint64, 3)
limits := map[string]uint64{"m": 65536, "t": 4, "p": 4}
for _, param := range params {
key, value, ok := strings.Cut(param, "=")
if !ok || limits[key] == 0 || values[key] != 0 {
return errEmailHashInvalid
}
n, err := emailHashNumber(value, 1, limits[key])
if err != nil {
return err
}
values[key] = n
}
if values["m"] < 8*values["p"] {
return errEmailHashInvalid
}
for i, field := range p[4:] {
decoded, err := base64.RawStdEncoding.Strict().DecodeString(field)
minLen := 8
if i == 1 {
minLen = 16
}
if err != nil || len(decoded) < minLen || len(decoded) > 64 {
return errEmailHashInvalid
}
}
return nil
}
func parseEmailDigest(scheme, encoding, encoded string) (*emailPasswordVerifier, error) {
var newHash func() hash.Hash
switch scheme {
case "PLAIN-MD5", "LDAP-MD5", "SMD5":
newHash = md5.New // #nosec G401 -- Verify legacy Dovecot hashes; never create stored credentials.
case "SHA", "SHA1", "SSHA":
newHash = sha1.New // #nosec G401 -- Verify legacy Dovecot hashes; never create stored credentials.
case "SHA256", "SSHA256":
newHash = sha256.New
case "SHA512", "SSHA512":
newHash = sha512.New
}
autoEncoding := encoding == ""
if autoEncoding {
encoding = "BASE64"
if scheme == "PLAIN-MD5" {
encoding = "HEX"
}
}
decoded, err := decodeEmailDigest(encoding, encoded)
size := newHash().Size()
salted := strings.HasPrefix(scheme, "SSHA") || scheme == "SMD5"
if autoEncoding && !salted && (err != nil || len(decoded) != size) {
other := "HEX"
if encoding == "HEX" {
other = "BASE64"
}
decoded, err = decodeEmailDigest(other, encoded)
}
if err != nil || len(decoded) < size || !salted && len(decoded) != size || salted && (len(decoded) == size || len(decoded) > size+64) {
return nil, errEmailHashInvalid
}
return &emailPasswordVerifier{match: func(candidate string) (bool, error) {
h := newHash()
_, _ = h.Write([]byte(candidate))
_, _ = h.Write(decoded[size:])
return subtle.ConstantTimeCompare(h.Sum(nil), decoded[:size]) == 1, nil
}}, nil
}
func decodeEmailDigest(encoding, encoded string) ([]byte, error) {
var decoded []byte
var err error
switch encoding {
case "HEX":
decoded, err = hex.DecodeString(encoded)
case "B64", "BASE64":
decoded, err = base64.StdEncoding.Strict().DecodeString(encoded)
default:
return nil, errEmailHashUnsupported
}
if err != nil {
return nil, errEmailHashInvalid
}
return decoded, nil
}
package checks
import (
"bufio"
"context"
"crypto/sha1" // #nosec G505 -- SHA1 is the digest format required by the Have I Been Pwned range API (https://haveibeenpwned.com/API/v3#PwnedPasswords). We send the first 5 chars of the digest and compare remaining chars against the returned list — HIBP does not offer a stronger-hash endpoint.
"crypto/sha256"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"sync"
"time"
"unicode"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
// hibpClient is used for HIBP API requests.
var hibpClient = &http.Client{Timeout: 10 * time.Second}
// hibpEndpoint is the base URL queried for HIBP password-range lookups.
// Declared as a var (not const) so tests can swap in an httptest server.
// Production callers must not modify this.
var hibpEndpoint = "https://api.pwnedpasswords.com/range/"
// currentYear returns the current year. Called each audit cycle so
// long-running daemons don't use a stale year after Jan 1.
func currentYear() int { return time.Now().Year() }
// weakPasswordCache caches the bundled wordlist (loaded once).
var (
weakPasswordOnce sync.Once
weakPasswords []string
)
// parseShadowLine parses a Dovecot shadow line "mailbox:{scheme}hash".
// cPanel writes the shadow(5) layout "mailbox:hash:lastchg:::::", so the
// hash is the second field only. Returns empty strings when either the
// mailbox or the hash is missing.
func parseShadowLine(line string) (mailbox, hash string) {
mailbox, rest, ok := strings.Cut(line, ":")
if !ok || mailbox == "" {
return "", ""
}
hash, _, _ = strings.Cut(rest, ":")
if hash == "" {
return "", ""
}
return mailbox, hash
}
// isLockedHash returns true if the hash indicates a locked/disabled account.
func isLockedHash(hash string) bool {
if hash == "" {
return true
}
return hash[0] == '!' || hash[0] == '*'
}
// generateCandidates creates password candidates from username and domain.
// All candidates are >= 6 characters. No duplicates.
func generateCandidates(username, domain string) []string {
seen := make(map[string]bool)
var candidates []string
add := func(s string) {
if len(s) >= 6 && !seen[s] {
seen[s] = true
candidates = append(candidates, s)
}
}
domainLabel := domain
if idx := strings.IndexByte(domain, '.'); idx > 0 {
domainLabel = domain[:idx]
}
bases := []string{username, domainLabel}
for _, base := range bases {
add(base)
// Capitalize first letter variant
upper := capitalizeFirst(base)
add(upper)
// Year variants: current year +/- 2
year := currentYear()
for y := year - 2; y <= year+2; y++ {
ys := strconv.Itoa(y)
add(base + ys)
add(upper + ys)
}
// Two-digit suffix variants: 00-99
for n := 0; n <= 99; n++ {
suffix := fmt.Sprintf("%02d", n)
add(base + suffix)
add(upper + suffix)
}
}
return candidates
}
// capitalizeFirst returns the string with its first rune upper-cased.
func capitalizeFirst(s string) string {
if len(s) == 0 {
return s
}
runes := []rune(s)
runes[0] = unicode.ToUpper(runes[0])
return string(runes)
}
// hashFingerprint returns a SHA256 hex fingerprint of a password hash
// (used for change detection -- re-audit only when hash changes).
func hashFingerprint(hash string) string {
h := sha256.Sum256([]byte(hash))
return fmt.Sprintf("%x", h[:])
}
// parseHIBPCount searches a HIBP range response body for a hash suffix
// and returns the breach count. Returns 0 if not found.
func parseHIBPCount(body, suffix string) int {
upperSuffix := strings.ToUpper(suffix)
for _, line := range strings.Split(body, "\n") {
line = strings.TrimSpace(line)
parts := strings.SplitN(line, ":", 2)
if len(parts) != 2 {
continue
}
if strings.ToUpper(strings.TrimSpace(parts[0])) == upperSuffix {
count, err := strconv.Atoi(strings.TrimSpace(parts[1]))
if err != nil {
return 0
}
return count
}
}
return 0
}
// checkHIBP queries the HIBP Pwned Passwords API for a plaintext password.
// Returns the breach count (0 if not found or on error).
func checkHIBP(plaintext string) int {
return checkHIBPWithContext(context.Background(), plaintext)
}
func checkHIBPWithContext(ctx context.Context, plaintext string) int {
if ctx == nil {
ctx = context.Background()
}
if ctx.Err() != nil {
return 0
}
// #nosec G401 -- SHA1 is mandated by the HIBP Pwned Passwords range API; see import comment.
h := sha1.Sum([]byte(plaintext))
hex := fmt.Sprintf("%X", h[:])
prefix := hex[:5]
suffix := hex[5:]
req, err := http.NewRequestWithContext(ctx, http.MethodGet, hibpEndpoint+prefix, nil)
if err != nil {
return 0
}
resp, err := hibpClient.Do(req)
if err != nil {
return 0
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return 0
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return 0
}
return parseHIBPCount(string(body), suffix)
}
type shadowFile struct {
path string
account string
domain string
}
type mailboxEntry struct {
account string
domain string
mailbox string
hash string
}
// discoverShadowFiles finds all Dovecot shadow files under /home/*/etc/*/shadow.
// Results are ranked by mtime desc so recently touched mailbox password files
// are inspected first when the check timeout cuts work short. maxFiles caps
// iteration; 0 disables the cap.
func discoverShadowFiles(ctx context.Context, maxFiles int) []shadowFile {
matches, _ := accountHomeGlob("*/etc/*/shadow")
ranked := rankPathsByMtimeDesc(ctx, matches, maxFiles)
results := make([]shadowFile, 0, len(ranked))
for _, m := range ranked {
parts := strings.Split(m, "/")
// /home/{account}/etc/{domain}/shadow
if len(parts) >= 5 {
results = append(results, shadowFile{
path: m,
account: parts[2],
domain: parts[4],
})
}
}
return results
}
// readShadowFile reads all mailbox entries from a Dovecot shadow file.
func readShadowFile(sf shadowFile) []mailboxEntry {
f, err := osFS.Open(sf.path)
if err != nil {
return nil
}
defer f.Close()
var entries []mailboxEntry
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
mailbox, hash := parseShadowLine(line)
if mailbox == "" || hash == "" {
continue
}
if isLockedHash(hash) {
continue
}
entries = append(entries, mailboxEntry{
account: sf.account,
domain: sf.domain,
mailbox: mailbox,
hash: hash,
})
}
return entries
}
// loadWeakPasswords reads the bundled wordlist once and caches it.
func loadWeakPasswords() []string {
weakPasswordOnce.Do(func() {
f, err := osFS.Open("/opt/csm/configs/weak_passwords.txt")
if err != nil {
return
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
word := strings.TrimSpace(scanner.Text())
if len(word) < 6 || strings.HasPrefix(word, "#") {
continue
}
weakPasswords = append(weakPasswords, word)
}
})
return weakPasswords
}
// CheckEmailPasswords audits Dovecot email account passwords for weak/predictable
// patterns. Uses internal throttle: skips if last refresh was less than
// password_check_interval_min ago.
func CheckEmailPasswords(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
db := store.Global()
if db == nil {
return nil
}
// Internal throttle. Unlike CheckOutdatedPlugins, which throttles only its
// cache refresh and still evaluates that cache every cycle, this check has
// nothing to report from until it runs again. Returning nothing silently
// would read as "ran, found nothing" and let the runner retire the weak
// passwords the last run found, so the skipped scope is declared.
if !ForceAll {
lastRefresh := db.GetEmailPWLastRefresh()
interval := time.Duration(cfg.EmailProtection.PasswordCheckIntervalMin) * time.Minute
if time.Since(lastRefresh) < interval {
markCheckSkipped(ctx, "email_weak_password")
return nil
}
}
shadowFiles := discoverShadowFiles(ctx, accountScanMaxFiles(ctx, cfg))
if len(shadowFiles) == 0 {
return nil
}
// Collect all mailbox entries
var allEntries []mailboxEntry
for _, sf := range shadowFiles {
if ctx.Err() != nil {
return nil
}
allEntries = append(allEntries, readShadowFile(sf)...)
}
if ctx.Err() != nil {
return nil
}
if len(allEntries) == 0 {
_ = db.SetEmailPWLastRefresh(time.Now())
return nil
}
var mu sync.Mutex
var findings []alert.Finding
var incomplete int
var unauditable int
var wg sync.WaitGroup
sem := make(chan struct{}, 5)
batch := emailMailboxAudits.begin(len(allEntries), cap(sem))
defer batch.abandon(ctx)
mailboxes:
for i, entry := range allEntries {
select {
case sem <- struct{}{}:
case <-ctx.Done():
break mailboxes
}
work := batch.tasks[i]
work.admit()
wg.Go(func() {
defer func() { <-sem }()
work.run(ctx, checkTimeout, func() {
if err := ctx.Err(); err != nil {
work.withdraw(err)
return
}
fullMailbox := entry.mailbox + "@" + entry.domain
storeKey := fmt.Sprintf("email:pwaudit:%s:%s", entry.account, fullMailbox)
// Older versions also cached failed verifications as clean.
fp := "v2:" + hashFingerprint(entry.hash)
if db.GetMetaString(storeKey) == fp {
return
}
finding, err := auditEmailPassword(ctx, entry)
work.progress()
if err != nil && !errors.Is(err, context.Canceled) &&
!errors.Is(err, errEmailHashUnsupported) && !errors.Is(err, errEmailHashInvalid) &&
!errors.Is(err, errEmailHashCost) && !errors.Is(err, errEmailCandidate) {
work.fail()
}
mu.Lock()
if err != nil {
if emailHashPermanentlyUnauditable(err) {
unauditable++
} else {
incomplete++
}
} else if finding != nil {
findings = append(findings, *finding)
}
mu.Unlock()
if err != nil {
return
}
if err := ctx.Err(); err != nil {
work.withdraw(err)
return
}
if err := db.SetMetaString(storeKey, fp); err != nil {
work.fail()
}
})
})
}
batch.abandon(ctx)
wg.Wait()
if incomplete > 0 || unauditable > 0 || ctx.Err() != nil {
markCheckIncomplete(ctx, "email_weak_password")
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "email_password_audit_incomplete",
Message: "Email password audit did not complete",
Details: fmt.Sprintf("Mailboxes to retry: %d. Mailboxes whose stored hash this build cannot audit at all: %d. See the email password audit documentation for supported formats and limits.", incomplete, unauditable),
// The mailbox counts in Details are not the condition.
DedupKey: "unfinished_verification",
})
}
// A hash this build cannot audit never becomes auditable by rerunning, so
// it must not hold the stamp back: that turned the interval into "every
// scan" and re-verified the whole mailbox set each cycle. A cycle skipped
// by the stamp now declares its skipped scope, so the findings survive it.
if incomplete == 0 && ctx.Err() == nil {
_ = db.SetEmailPWLastRefresh(time.Now())
}
return findings
}
// emailHashPermanentlyUnauditable reports whether a verification failure is a
// property of the stored hash rather than a condition that may clear.
func emailHashPermanentlyUnauditable(err error) bool {
// The parser returns these sentinels directly. A wrapper can describe a
// transient verification failure, so unwrapping is not proof that only
// the stored hash prevented this audit from completing.
return err == errEmailHashUnsupported || err == errEmailHashInvalid || err == errEmailHashCost
}
func auditEmailPassword(ctx context.Context, entry mailboxEntry) (*alert.Finding, error) {
verifier, err := parseEmailPasswordHash(entry.hash)
if err != nil {
return nil, err
}
matched, err := verifier.firstMatch(ctx, generateCandidates(entry.mailbox, entry.domain))
if err != nil {
return nil, err
}
matchType := "heuristic"
if matched == "" {
matched, err = verifier.firstMatch(ctx, loadWeakPasswords())
if err != nil {
return nil, err
}
matchType = "wordlist"
}
if matched == "" {
return nil, nil
}
fullMailbox := entry.mailbox + "@" + entry.domain
details := fmt.Sprintf("Account: %s\nMailbox: %s\nMatch type: %s", entry.account, fullMailbox, matchType)
if breachCount := checkHIBPWithContext(ctx, matched); breachCount > 0 {
details += fmt.Sprintf("\nHIBP: password found in %d data breaches", breachCount)
}
return &alert.Finding{
Severity: alert.Critical,
Check: "email_weak_password",
Message: fmt.Sprintf("Weak email password for %s (account: %s)", fullMailbox, entry.account),
Details: details,
Domain: entry.domain,
Mailbox: fullMailbox,
}, nil
}
package checks
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"mime"
"net/textproto"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/emailspool"
"github.com/pidginhost/csm/internal/eximlog"
emime "github.com/pidginhost/csm/internal/mime"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
)
const emailBodySampleSize = 65536 // Analyze the first 64KB of email body
// maxSpoolHeaderBytes bounds the -H file read into memory. Exim's own header
// size limit keeps real mail far below it.
const maxSpoolHeaderBytes = 16 << 20
// Suspicious mailer headers that indicate mass mailing scripts. PHPMailer is
// deliberately absent: it is the default WordPress transport and appears on
// essentially all legitimate WordPress mail, so it carries no signal. The
// "phpmail" substring is excluded for the same reason (it matches PHPMailer).
var suspiciousMailers = []string{
"swiftmailer", "mass mailer", "bulk mailer",
"leaf phpmailer", "mail.php",
}
// Known safe mailers that should not be flagged.
var safeMailers = []string{
"wordpress", "woocommerce", "roundcube", "squirrelmail",
"thunderbird", "outlook", "apple mail", "cpanel",
"postfix", "exim", "dovecot",
}
// Phishing URL patterns in email body.
var emailPhishPatterns = []string{
".workers.dev",
"//bit.ly/", "//tinyurl.com/", "//is.gd/", "//rb.gy/",
"//t.co/",
"/redir?url=", "/redirect?url=", "link?url=",
"effi.redir",
}
// Phishing language in email body.
var emailPhishLanguage = []string{
"verify your account",
"confirm your identity",
"unusual activity",
"your account will be",
"suspended unless",
"click here to verify",
"update your payment",
"confirm your email address",
"security alert",
"unauthorized access",
}
// Brand impersonation in email body (when sender doesn't match).
var emailBrandNames = []string{
"paypal", "microsoft", "apple", "google", "amazon",
"netflix", "facebook", "instagram", "bank of",
"wells fargo", "chase", "citibank",
}
// CheckOutboundEmailContent samples outbound email content from Exim spool
// for phishing URLs, credential harvesting language, suspicious mailers,
// and Reply-To mismatches.
func CheckOutboundEmailContent(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
// Read recent outbound messages from the Exim main log
lines := tailFile(platform.EximMainLog, 200)
if len(lines) == 0 {
return nil
}
scanned := make(map[string]bool) // avoid scanning same message twice
for _, line := range lines {
msgID, sender, ok := eximlog.Arrival(line)
if !ok || sender == "<>" {
continue // not an arrival, or a bounce
}
// Outbound means submitted here: an authenticated mailbox or a local
// caller. Inbound mail from other servers is not this check's job.
if !eximlog.IsSubmission(line) {
continue
}
if scanned[msgID] {
continue
}
scanned[msgID] = true
// Scan the message
result := scanEximMessage(msgID, sender)
if result != nil {
findings = append(findings, *result)
}
}
return findings
}
// scanEximMessage reads an Exim spool message and checks for suspicious content.
func scanEximMessage(msgID, sender string) *alert.Finding {
var headerPath, bodyPath string
for _, dir := range eximSpoolMessageDirs(msgID) {
h := filepath.Join(dir, msgID+"-H")
b := filepath.Join(dir, msgID+"-D")
if _, err := osFS.Stat(h); err == nil {
headerPath = h
bodyPath = b
break
}
}
if headerPath == "" {
return nil // message already delivered/removed from spool
}
var indicators []string
// Keep checks within the bounded prefix, even when padding exceeds the
// field parser's per-line limit. The MIME parser already handles long
// lines within this buffer and retains headers preceding a malformed tail.
headerData, _, err := readRegularFileHead(headerPath, maxSpoolHeaderBytes)
if err != nil {
return nil
}
mimeHeaders := emime.ParseSpoolMIMEHeaders(headerData)
parsed, err := emailspool.ParseHeadersReader(bytes.NewReader(headerData))
if err != nil {
parsed = emailspool.Headers{
From: mimeHeaders.Get("From"),
ReplyTo: mimeHeaders.Get("Reply-To"),
XMailer: mimeHeaders.Get("X-Mailer"),
UserAgent: mimeHeaders.Get("User-Agent"),
}
}
// Check 1: Reply-To mismatch
if parsed.From != "" && parsed.ReplyTo != "" {
fromDomain := emailspool.ExtractDomain(parsed.From)
replyDomain := emailspool.ExtractDomain(parsed.ReplyTo)
if fromDomain != "" && replyDomain != "" && fromDomain != replyDomain {
indicators = append(indicators, fmt.Sprintf("Reply-To mismatch: From=%s, Reply-To=%s", fromDomain, replyDomain))
}
}
// Check 2: Suspicious X-Mailer (fall back to User-Agent, which the
// emailspool parser surfaces alongside the other RFC 5322 fields).
mailer := parsed.XMailer
if mailer == "" {
mailer = parsed.UserAgent
}
if mailer != "" {
mailerLower := strings.ToLower(mailer)
isSafe := false
for _, safe := range safeMailers {
if strings.Contains(mailerLower, safe) {
isSafe = true
break
}
}
if !isSafe {
for _, suspicious := range suspiciousMailers {
if strings.Contains(mailerLower, suspicious) {
indicators = append(indicators, fmt.Sprintf("suspicious mailer: %s", strings.TrimSpace(mailer)))
break
}
}
}
}
// Check 3: Spoofed display name (brand name in From: but sender is not that brand)
if parsed.From != "" {
fromLower := strings.ToLower(parsed.From)
senderDomain := emailspool.ExtractDomain(sender)
for _, brand := range emailBrandNames {
if strings.Contains(fromLower, brand) && !strings.Contains(strings.ToLower(senderDomain), brand) {
indicators = append(indicators, fmt.Sprintf("spoofed brand in From: '%s' (actual sender: %s)", strings.TrimSpace(parsed.From), sender))
break
}
}
}
// Read and analyze a bounded body sample; the rest of the body is never
// read into memory.
bodyData, _, _ := readRegularFileHead(bodyPath, emailBodySampleSize)
if len(bodyData) > 0 {
bodyLower := strings.ToLower(string(bodyData))
indicators = append(indicators, bodyContentIndicators(bodyLower)...)
// A base64 Content-Transfer-Encoding is standard MIME framing, not an
// indicator by itself. Decode the body and run the same content checks
// on the plaintext so a payload hidden behind base64 is caught -- the
// raw base64 blob would otherwise sail past every text pattern.
// Only the exact Exim marker is framing; arbitrary body text ending
// in -D must remain available to the transfer decoder.
if marker, body, ok := bytes.Cut(bodyData, []byte("\n")); ok && string(bytes.TrimSuffix(marker, []byte("\r"))) == msgID+"-D" {
bodyData = body
}
if decoded := decodeBase64Body(bodyData, mimeHeaders.Get("Content-Type"), mimeHeaders.Get("Content-Transfer-Encoding")); decoded != "" {
indicators = append(indicators, bodyContentIndicators(strings.ToLower(decoded))...)
}
}
indicators = uniqueStrings(indicators)
// Require at least two independent indicators before alerting. A lone weak
// signal (Reply-To mismatch, one brand word, one suspicious header) fires
// on ordinary legitimate mail, so a single indicator is not enough.
if len(indicators) < 2 {
return nil
}
severity := alert.High
if len(indicators) >= 3 {
severity = alert.Critical
}
_, senderDomain := alert.SplitEmail(sender)
// Action targets must survive redaction of attacker-controlled sender
// text at persistence boundaries. Keep that text out of the ID message.
return &alert.Finding{
Severity: severity,
Check: "email_phishing_content",
Message: fmt.Sprintf("Suspicious outbound email (message: %s)", msgID),
Details: fmt.Sprintf("Envelope sender: %s\nIndicators:\n- %s", sender, strings.Join(indicators, "\n- ")),
Domain: senderDomain,
Mailbox: sender,
}
}
// bodyContentIndicators runs the phishing-URL and credential-harvesting content
// checks over an already-lowercased body text. Shared by the raw-body pass and
// the decoded-base64 pass so both apply identical logic.
func bodyContentIndicators(bodyLower string) []string {
var indicators []string
for _, pattern := range emailPhishPatterns {
if strings.Contains(bodyLower, pattern) {
indicators = append(indicators, fmt.Sprintf("phishing URL pattern: %s", pattern))
break
}
}
harvestCount := 0
for _, phrase := range emailPhishLanguage {
if strings.Contains(bodyLower, phrase) {
harvestCount++
}
}
if harvestCount >= 2 {
indicators = append(indicators, fmt.Sprintf("credential harvesting language (%d phrases)", harvestCount))
}
return indicators
}
// decodeBase64Body walks actual MIME parts instead of searching body text for
// header names. Boundaries are case-sensitive and only delimit their enclosing
// multipart; other dash lines remain part of the lenient base64 input.
func decodeBase64Body(raw []byte, contentType, transferEncoding string) string {
var decodedParts []string
var walk func([]byte, string, string, int)
walk = func(body []byte, ct, cte string, depth int) {
// Match attachment extraction's nesting bound for attacker-written MIME.
if depth >= 16 {
return
}
visit := func(part []byte) {
reader := bufio.NewReader(bytes.NewReader(part))
headers, err := textproto.NewReader(reader).ReadMIMEHeader()
if err != nil {
// A damaged part must not hide its siblings.
return
}
data, _ := io.ReadAll(reader)
walk(data, headers.Get("Content-Type"), headers.Get("Content-Transfer-Encoding"), depth+1)
}
encoding := strings.ToLower(strings.TrimSpace(withoutMIMEComments(cte)))
mediaType, params, _ := mime.ParseMediaType(withoutMIMEComments(ct))
if strings.HasPrefix(mediaType, "multipart/") {
boundary := params["boundary"]
if boundary == "" {
return
}
visitBase64MIMEParts(body, boundary, visit)
return
}
if (mediaType == "message/rfc822" || mediaType == "message/global") && encoding != "base64" {
visit(body)
return
}
if encoding == "base64" {
for _, variant := range emime.DecodeTransferVariants("base64", body) {
if len(variant) > 0 {
decodedParts = append(decodedParts, string(variant))
}
}
}
}
walk(raw, contentType, transferEncoding, 0)
return strings.Join(decodedParts, "\n")
}
// Frame parts before parsing their headers so malformed headers cannot stop
// iteration at the next sibling. A clipped final part is still worth scanning.
func visitBase64MIMEParts(body []byte, boundary string, visit func([]byte)) {
delimiter := "--" + boundary
start := -1
for offset := 0; offset < len(body); {
end := bytes.IndexByte(body[offset:], '\n')
next := len(body)
if end >= 0 {
next = offset + end + 1
}
line := strings.TrimRight(string(body[offset:next]), "\r\n \t")
if line == delimiter || line == delimiter+"--" {
if start >= 0 {
visit(body[start:offset])
}
if line == delimiter+"--" {
return
}
start = next
}
offset = next
}
if start >= 0 {
visit(body[start:])
}
}
// MIME comments are allowed around tokens, but parentheses inside quoted
// parameters (notably boundary values) are literal. Keep a space where a
// comment was removed so separate tokens cannot be joined accidentally.
func withoutMIMEComments(value string) string {
var out strings.Builder
depth := 0
quoted := false
for i := 0; i < len(value); i++ {
b := value[i]
if depth > 0 {
switch b {
case '\\':
if i+1 < len(value) {
i++
}
case '(':
depth++
case ')':
depth--
}
continue
}
if quoted {
out.WriteByte(b)
if b == '\\' && i+1 < len(value) {
i++
out.WriteByte(value[i])
} else if b == '"' {
quoted = false
}
continue
}
switch b {
case '(':
depth = 1
out.WriteByte(' ')
case '"':
quoted = true
out.WriteByte(b)
default:
out.WriteByte(b)
}
}
return out.String()
}
package checks
import (
"bytes"
"crypto/sha256"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/contenttype"
"github.com/pidginhost/csm/internal/safepath"
)
// realtimePHPChecks are the realtime PHP content checks whose files get the
// scheduled analysis when realtime reports them.
var realtimePHPChecks = map[string]bool{
"obfuscated_php_realtime": true,
"webshell_content_realtime": true,
"php_dropper_realtime": true,
}
// RealtimePHPEscalationCandidates selects files not already covered by a file
// response in this batch. Another detector may have evaluated the same file,
// including a refused attempt, so inspect the whole batch first.
func RealtimePHPEscalationCandidates(cfg *config.Config, findings []alert.Finding) []alert.Finding {
if cfg == nil || !cfg.AutoResponse.Enabled || !cfg.AutoResponse.QuarantineFiles || cfg.ObserveMode() {
return nil
}
seen := make(map[string]bool)
for _, f := range findings {
if f.FilePath != "" && (f.AutoFileResponseEvaluated ||
(autoQuarantineChecks[f.Check] && f.Severity == alert.Critical)) {
seen[filepath.Clean(f.FilePath)] = true
}
}
var out []alert.Finding
for _, f := range findings {
if !realtimePHPChecks[f.Check] || f.Severity != alert.Critical || f.FilePath == "" {
continue
}
path := filepath.Clean(f.FilePath)
if seen[path] {
continue
}
seen[path] = true
f.FilePath = path
out = append(out, f)
}
return out
}
// EscalateRealtimePHP runs scheduled analysis outside the alert dispatcher.
// Results must re-enter normal dispatch for state and suppression checks.
func EscalateRealtimePHP(cfg *config.Config, findings []alert.Finding) []alert.Finding {
var out []alert.Finding
for _, f := range RealtimePHPEscalationCandidates(cfg, findings) {
if finding, ok := scheduledPHPFinding(f.FilePath); ok {
finding.TenantID = f.TenantID
finding.Domain = f.Domain
finding.ProcessInfo = f.ProcessInfo
finding.PID = f.PID
out = append(out, finding)
}
}
return out
}
// scheduledPHPFinding analyses path as the scheduled content scan does. The
// open rejects links in every component and does not wait on a FIFO.
// Regular-file I/O can still stall, so callers must run outside dispatch.
func scheduledPHPFinding(path string) (alert.Finding, bool) {
if !contenttype.IsPHPSourceName(strings.ToLower(filepath.Base(path))) {
return alert.Finding{}, false
}
if !filepath.IsAbs(path) || filepath.Clean(path) != path {
return alert.Finding{}, false
}
target, err := safepath.OpenTarget("/", strings.TrimPrefix(path, "/"), false)
if err != nil {
return alert.Finding{}, false
}
defer target.Close()
f, err := target.Parent.OpenFile(target.Name, os.O_RDONLY, 0)
if err != nil {
return alert.Finding{}, false
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil || !info.Mode().IsRegular() {
return alert.Finding{}, false
}
var result phpAnalysisResult
digest := ""
if info.Size() <= contentFingerprintMaxBytes {
data, readErr := io.ReadAll(io.LimitReader(f, contentFingerprintMaxBytes+1))
if readErr != nil || int64(len(data)) != info.Size() {
return alert.Finding{}, false
}
result = analyzePHPContentReaderAt(path, bytes.NewReader(data), int64(len(data)))
digest = fmt.Sprintf("%x", sha256.Sum256(data))
} else {
result = analyzePHPContentReaderAt(path, f, info.Size())
}
after, err := f.Stat()
if err != nil || !SameFileVersion(info, after) || target.Check() != nil {
return alert.Finding{}, false
}
current, err := target.Parent.Stat(target.Name)
if err != nil || !SameFileVersion(info, current) {
return alert.Finding{}, false
}
if !result.readOK || result.severity != alert.Critical {
return alert.Finding{}, false
}
finding := phpContentFinding(path, result, info, digest)
finding.FileResponseIdentity = fileResponseIdentity(info)
if finding.FileResponseIdentity == "" {
return alert.Finding{}, false
}
return finding, true
}
// The token includes ctime so restoring size and mtime cannot disguise a
// same-inode rewrite. It survives pending replay without publishing inode data.
func fileResponseIdentity(info os.FileInfo) string {
identity, ok := selfWriteIdentityFromFileInfo(info)
if !ok {
return ""
}
return fmt.Sprintf("%d:%d:%d:%d:%d:%d:%d", identity.Device, identity.Inode,
identity.ChangeSec, identity.ChangeNsec, info.Size(), info.ModTime().UnixNano(), info.Mode())
}
package checks
import (
"context"
"fmt"
"path/filepath"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// uidStringToUser converts a string-form uid (as read from /proc/<pid>/status)
// to a username via the cached LookupUser. Falls back to the raw uid string
// when it is not parseable as a uint32.
func uidStringToUser(uid string) string {
u64, err := strconv.ParseUint(uid, 10, 32)
if err != nil {
return uid
}
return LookupUser(uint32(u64))
}
// CheckDatabaseDumps detects mysqldump/pg_dump processes running under
// non-root users - potential data exfiltration.
func CheckDatabaseDumps(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
dumpTools := []string{"mysqldump", "pg_dump", "mongodump"}
procs, _ := osFS.Glob("/proc/[0-9]*/cmdline")
for _, cmdPath := range procs {
pid := filepath.Base(filepath.Dir(cmdPath))
// Read UID
statusData, _ := osFS.ReadFile(filepath.Join("/proc", pid, "status"))
var uid string
for _, line := range strings.Split(string(statusData), "\n") {
if strings.HasPrefix(line, "Uid:\t") {
fields := strings.Fields(strings.TrimPrefix(line, "Uid:\t"))
if len(fields) > 0 {
uid = fields[0]
}
}
}
// Skip root - root may run legitimate backups
if uid == "0" || uid == "" {
continue
}
cmdline, err := osFS.ReadFile(cmdPath)
if err != nil {
continue
}
cmdStr := strings.ReplaceAll(string(cmdline), "\x00", " ")
safeCmdStr := redactProcCommandLine(cmdline)
for _, tool := range dumpTools {
if strings.Contains(cmdStr, tool) {
user := uidStringToUser(uid)
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "database_dump",
Message: fmt.Sprintf("Database dump by non-root user: %s (%s)", user, tool),
Details: fmt.Sprintf("PID: %s, UID: %s, cmdline: %s", pid, uid, safeCmdStr),
})
break
}
}
}
return findings
}
// CheckOutboundPasteSites detects connections to known paste/exfiltration sites.
func CheckOutboundPasteSites(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
// Check running processes for connections to paste sites
pasteSites := []string{
"pastebin.com", "hastebin.com", "ghostbin.co",
"paste.ee", "dpaste.org", "gist.githubusercontent.com",
"raw.githubusercontent.com", "transfer.sh",
"file.io", "0x0.st", "ix.io",
}
procs, _ := osFS.Glob("/proc/[0-9]*/cmdline")
for _, cmdPath := range procs {
pid := filepath.Base(filepath.Dir(cmdPath))
statusData, _ := osFS.ReadFile(filepath.Join("/proc", pid, "status"))
var uid string
for _, line := range strings.Split(string(statusData), "\n") {
if strings.HasPrefix(line, "Uid:\t") {
fields := strings.Fields(strings.TrimPrefix(line, "Uid:\t"))
if len(fields) > 0 {
uid = fields[0]
}
}
}
if uid == "0" {
continue
}
cmdline, err := osFS.ReadFile(cmdPath)
if err != nil {
continue
}
cmdStr := strings.ToLower(strings.ReplaceAll(string(cmdline), "\x00", " "))
safeCmdStr := redactProcCommandLine(cmdline)
// Check if process is connecting to paste sites
for _, site := range pasteSites {
if strings.Contains(cmdStr, site) {
user := uidStringToUser(uid)
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "exfiltration_paste_site",
Message: fmt.Sprintf("Process connecting to paste/exfiltration site: %s (user: %s)", site, user),
Details: fmt.Sprintf("PID: %s, cmdline: %s", pid, safeCmdStr),
TenantID: HostingAccountForUser(user),
})
break
}
}
}
return findings
}
package checks
import (
"bytes"
"compress/gzip"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"io/fs"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"golang.org/x/sys/unix"
)
// userdataDomainsPath is cPanel's authoritative domain->docroot map. It covers
// addon and subdomain docroots that /home/*/public_html misses.
var userdataDomainsPath = "/etc/userdatadomains"
const cpanelInstallPath = "/usr/local/cpanel"
// Bounds so the deep scan stays cheap on a host with hundreds of docroots.
const (
exposureMaxFilesPerRoot = 4000
exposureMaxDirsPerRoot = 4000
exposureProbeConnTimeout = 4 * time.Second
exposureProbeTotalTimeout = 6 * time.Second
)
// walkSkipDirs are directory names never worth descending for a leaked dump or
// backup and expensive to traverse.
var walkSkipDirs = map[string]bool{
"node_modules": true, ".git": true, ".svn": true,
}
// CheckExposedFiles scans every cPanel docroot for sensitive files (database
// dumps, backup archives, config/source backups, phpinfo) that the web server
// actually serves, and reports each confirmed exposure. It reads only response
// headers except for a bounded phpinfo body used to reject empty stubs.
func CheckExposedFiles(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
content, err := osFS.ReadFile(userdataDomainsPath)
if err != nil {
// Absence is an expected no-op only on a non-cPanel host. Other failures
// make the run incomplete: retaining prior findings is safer than
// clearing them because the map was temporarily unreadable.
if vhostMapFailureIsIncomplete(err) {
markCheckIncomplete(ctx, "exposed_files")
}
return nil
}
vhosts, complete := parseUserdataDomainsChecked(string(content))
if !complete {
markCheckIncomplete(ctx, "exposed_files")
}
return scanVhostsForExposure(ctx, dedupVhostsByDocroot(vhosts), cfg)
}
func vhostMapFailureIsIncomplete(readErr error) bool {
if !errors.Is(readErr, fs.ErrNotExist) {
return true
}
_, statErr := osFS.Stat(cpanelInstallPath)
// A missing map is an expected no-op only when cPanel itself is absent.
// If the install path exists (or cannot be checked), preserve prior
// findings until the authoritative map is readable again.
return statErr == nil || !errors.Is(statErr, fs.ErrNotExist)
}
// dedupVhostsByDocroot keeps one vhost per docroot (multiple domains -- www,
// parked -- can share a docroot; one probe host is enough). Prefer a vhost with
// a usable serving IP, then a real main/addon/subdomain over a parked alias,
// since parked domains commonly redirect and cannot confirm an exposure.
func dedupVhostsByDocroot(vhosts []vhost) []vhost {
chosen := make(map[string]int, len(vhosts))
out := vhosts[:0:0]
for _, vh := range vhosts {
key := filepath.Clean(vh.docroot)
if i, ok := chosen[key]; ok {
if betterProbeVhost(vh, out[i]) {
out[i] = vh
}
continue
}
chosen[key] = len(out)
out = append(out, vh)
}
return out
}
func betterProbeVhost(candidate, current vhost) bool {
candidateUsable := probeHost(candidate) != ""
currentUsable := probeHost(current) != ""
if candidateUsable != currentUsable {
return candidateUsable
}
return preferredProbeVhost(candidate) && !preferredProbeVhost(current)
}
func preferredProbeVhost(vh vhost) bool {
switch strings.ToLower(strings.TrimSpace(vh.typ)) {
case "main", "addon", "sub":
return true
default:
return false
}
}
// scanVhostsForExposure walks each vhost's docroot, classifies candidate files,
// confirms the server serves them, and emits a finding per confirmed exposure.
func scanVhostsForExposure(ctx context.Context, vhosts []vhost, cfg *config.Config) []alert.Finding {
depth := exposureScanDepth(cfg)
var findings []alert.Finding
for _, vh := range vhosts {
if ctx.Err() != nil {
return findings
}
host := probeHost(vh)
if host == "" {
// A loopback fallback is not a meaningful reachability check:
// LiteSpeed rejects loopback-originated requests even when the same
// vhost serves the file on its configured address. Skip this vhost
// and preserve findings from the last complete scan instead.
markCheckIncomplete(ctx, "exposed_files")
continue
}
paths, complete := walkExposureCandidates(ctx, vh.docroot, depth)
if !complete {
markCheckIncomplete(ctx, "exposed_files")
}
for _, path := range paths {
if ctx.Err() != nil {
return findings
}
class := classifyExposedPath(path)
if class == classNone {
// An archive named after the site it holds carries no backup
// token, so the name tells us nothing. Its entry list does.
holdsSite, complete := archiveSiteBackupStatus(ctx, path)
if !complete {
markCheckIncomplete(ctx, "exposed_files")
}
if holdsSite {
class = classBackupArchive
}
}
if class == classNone {
continue
}
rel := relURLPath(vh.docroot, path)
if rel == "" {
continue
}
class = demoteSampleSQL(class, rel)
pr := webProber.probe(ctx, vh.domain, host, rel)
confirmed := confirmExposure(class, pr)
if !confirmed && (!pr.reachable || pr.partial) {
// A transport failure cannot prove that a previous exposure was
// fixed. Keep prior findings until a later probe gets a response.
markCheckIncomplete(ctx, "exposed_files")
}
if !confirmed {
continue
}
if class == classPHPInfo {
// Headers cannot distinguish a real dump from a stub whose
// phpinfo() call is commented out or guarded: both answer 200
// text/html. Only a body carrying actual phpinfo output is an
// information disclosure.
scheme, exposed, complete := confirmPHPInfoBody(ctx, pr.scheme, vh.domain, host, rel)
if !complete {
markCheckIncomplete(ctx, "exposed_files")
}
if !exposed {
continue
}
pr.scheme = scheme
}
findings = append(findings, buildExposedFinding(vh, path, rel, class, pr))
}
}
return findings
}
func exposureScanDepth(cfg *config.Config) int {
depth := config.DefaultExposedFileScanDepth
if cfg != nil && cfg.Thresholds.ExposedFileScanDepth > 0 {
depth = cfg.Thresholds.ExposedFileScanDepth
}
if depth > config.MaxExposedFileScanDepth {
return config.MaxExposedFileScanDepth
}
return depth
}
// walkExposureCandidates returns files under docroot down to maxDepth directory
// levels, capped at exposureMaxFilesPerRoot to bound I/O on large trees. It
// visits shallower directories first so a large cache subtree cannot consume
// the cap before root-level leaks are considered.
func walkExposureCandidates(ctx context.Context, docroot string, maxDepth int) ([]string, bool) {
return walkExposureCandidatesLimit(ctx, docroot, maxDepth, exposureMaxFilesPerRoot)
}
func walkExposureCandidatesLimit(ctx context.Context, docroot string, maxDepth, maxFiles int) ([]string, bool) {
if maxFiles <= 0 {
return nil, false
}
type pendingDir struct {
path string
depth int
}
queue := []pendingDir{{path: docroot}}
queuedDirs := 1
var out []string
complete := true
for len(queue) > 0 && len(out) < maxFiles {
if ctx.Err() != nil {
return out, false
}
dir := queue[0]
queue = queue[1:]
entries, err := osFS.ReadDir(dir.path)
if err != nil {
// Only a proven absent root is a complete negative. A subdirectory
// that vanished after its listing may have moved, so it stays a gap.
if dir.depth == 0 && errors.Is(err, fs.ErrNotExist) {
return nil, exposureDocrootAbsent(ctx, docroot)
}
complete = false
continue
}
for _, e := range entries {
if ctx.Err() != nil {
return out, false
}
if len(out) >= maxFiles {
complete = false
break
}
if e.IsDir() {
// A checked-out repository is never descended (thousands of
// objects), but its marker file is a candidate in its own
// right: a web-served .git/ or .svn/ hands out the site's
// source and, routinely, its credentials.
for _, marker := range repoMetadataMarkers(e.Name()) {
markerPath := filepath.Join(dir.path, e.Name(), marker)
if info, statErr := osFS.Stat(markerPath); statErr == nil && info.Mode().IsRegular() {
out = append(out, markerPath)
}
}
if dir.depth < maxDepth && !walkSkipDirs[e.Name()] {
if queuedDirs >= exposureMaxDirsPerRoot {
complete = false
continue
}
queue = append(queue, pendingDir{
path: filepath.Join(dir.path, e.Name()),
depth: dir.depth + 1,
})
queuedDirs++
}
continue
}
out = append(out, filepath.Join(dir.path, e.Name()))
}
}
if len(queue) > 0 {
complete = false
}
return out, complete
}
// ENOENT through a symlink is not proof of absence: process-relative links can
// resolve differently for the web server. Walk from / with pinned directories
// and no-follow opens so a swapped ancestor cannot redirect the absence check.
// Readable symlinked roots still take the normal exposure walk above.
func exposureDocrootAbsent(ctx context.Context, docroot string) bool {
if !filepath.IsAbs(docroot) || filepath.Clean(docroot) != docroot || docroot == string(filepath.Separator) {
return false
}
dir, err := osFS.Open(string(filepath.Separator))
if err != nil {
return false
}
defer func() { _ = dir.Close() }()
for _, part := range strings.Split(strings.TrimPrefix(docroot, string(filepath.Separator)), string(filepath.Separator)) {
if ctx.Err() != nil {
return false
}
fd, err := unix.Openat(int(dir.Fd()), part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC|unix.O_NONBLOCK, 0)
if err != nil {
return errors.Is(err, fs.ErrNotExist) && ctx.Err() == nil
}
_ = dir.Close()
// #nosec G115 -- Openat returned a non-negative descriptor because err is nil.
dir = os.NewFile(uintptr(fd), part)
}
return false
}
func relURLPath(docroot, path string) string {
r, err := filepath.Rel(docroot, path)
if err != nil || r == ".." || strings.HasPrefix(r, ".."+string(filepath.Separator)) {
return ""
}
return "/" + filepath.ToSlash(r)
}
func buildExposedFinding(vh vhost, path, rel string, class exposedClass, pr probeResult) alert.Finding {
size := int64(-1)
if fi, err := osFS.Stat(path); err == nil {
size = fi.Size()
}
scheme := pr.scheme
if scheme == "" {
scheme = "https"
}
exposureURL := (&url.URL{Scheme: scheme, Host: vh.domain, Path: rel}).String()
return alert.Finding{
Severity: class.severity(),
Check: class.findingName(),
Message: fmt.Sprintf("Web-exposed %s reachable at %s", exposureLabel(class), exposureURL),
Details: fmt.Sprintf("File: %s (%d bytes), served as %q. Remove it from the web root or deny HTTP access.",
path, size, pr.contentType),
FilePath: path,
Domain: vh.domain,
TenantID: vh.user,
Timestamp: time.Now(),
}
}
func exposureLabel(class exposedClass) string {
switch class {
case classRepoMetadata:
return "version-control repository (source and history)"
case classConfigLeak:
return "configuration/credentials file"
case classDBDump:
return "database dump"
case classBackupArchive:
return "site backup archive"
case classSourceBackup:
return "source-code backup"
case classPHPInfo:
return "phpinfo diagnostic"
case classSampleSQL:
return "framework/sample SQL file"
default:
return "sensitive file"
}
}
// ---------------------------------------------------------------------------
// Reachability probe seam
// ---------------------------------------------------------------------------
// webProbe abstracts a headers-only reachability check against the local web
// server. Tests inject a fake; production pins the connection to the vhost's
// serving IP while using its domain as SNI/Host, and reads only the status line
// and Content-Type.
type webProbe interface {
probe(ctx context.Context, domain, host, urlPath string) probeResult
probeComplete(ctx context.Context, domain, host, urlPath string) probeResult
}
var webProber webProbe = realWebProbe{}
// SetWebProbe replaces the reachability prober. Test-only seam.
func SetWebProbe(p webProbe) { webProber = p }
type realWebProbe struct{}
func (realWebProbe) probe(ctx context.Context, domain, host, urlPath string) probeResult {
return probeLocalSchemes(ctx, domain, host, urlPath, true, doLocalProbe)
}
// probeComplete always attempts both origin protocols. Detection may stop once
// one protocol proves a raw exposure, but verification needs every protocol to
// answer before a negative result can safely clear an earlier finding.
func (realWebProbe) probeComplete(ctx context.Context, domain, host, urlPath string) probeResult {
return probeLocalSchemes(ctx, domain, host, urlPath, false, doLocalProbe)
}
type localProbeFunc func(context.Context, string, string, string, string) (probeResult, bool)
func probeLocalSchemes(
ctx context.Context,
domain, host, urlPath string,
stopOnRawExposure bool,
probeOne localProbeFunc,
) probeResult {
results := make([]probeResult, 0, 2)
partial := false
for _, scheme := range []string{"https", "http"} {
if pr, ok := probeOne(ctx, scheme, domain, host, urlPath); ok {
results = append(results, pr)
// A successful non-HTML response is sufficient for every raw
// leak class; avoid an unnecessary second request.
if stopOnRawExposure && successfulProbe(pr) && !isHTMLContentType(pr.contentType) {
return pr
}
} else {
partial = true
}
}
pr := bestProbeResult(results)
pr.partial = partial
return pr
}
func successfulProbe(pr probeResult) bool {
return pr.status == http.StatusOK || pr.status == http.StatusPartialContent
}
// bestProbeResult prefers a successful response over an HTTP error and a raw
// response over HTML. This lets an HTTP-only exposure win when HTTPS redirects,
// blocks the path, or executes a different handler.
func bestProbeResult(results []probeResult) probeResult {
var firstReachable probeResult
var firstSuccess probeResult
for _, pr := range results {
if !pr.reachable {
continue
}
if !firstReachable.reachable {
firstReachable = pr
}
if !successfulProbe(pr) {
continue
}
if !isHTMLContentType(pr.contentType) {
return pr
}
if !firstSuccess.reachable {
firstSuccess = pr
}
}
if firstSuccess.reachable {
return firstSuccess
}
return firstReachable
}
// doLocalProbe issues one HEAD (falling back to a ranged GET when HEAD is
// disallowed) to the vhost's own serving IP, presenting domain as the vhost. It
// never reads the response body. The dial is pinned to the host's configured
// serving address (never DNS-resolved) so the request reaches this origin, not
// a CDN, and stays on-box. Loopback is not used: LiteSpeed answers 403 to
// loopback-originated requests even for files it serves on the public IP.
func doLocalProbe(ctx context.Context, scheme, domain, host, urlPath string) (probeResult, bool) {
host = normalizeServingIP(host)
if host == "" {
return probeResult{}, false
}
probeCtx, cancel := context.WithTimeout(ctx, exposureProbeTotalTimeout)
defer cancel()
client, closeIdle := exposureProbeHTTPClient(domain, host)
defer closeIdle()
return doLocalProbeWithClient(probeCtx, client, scheme, domain, urlPath)
}
func doLocalProbeWithClient(ctx context.Context, client *http.Client, scheme, domain, urlPath string) (probeResult, bool) {
u := url.URL{Scheme: scheme, Host: domain, Path: urlPath}
resp, err := doProbeRequest(ctx, client, http.MethodHead, u.String(), "")
if err != nil {
return probeResult{}, false
}
if resp.StatusCode == http.StatusMethodNotAllowed || resp.StatusCode == http.StatusNotImplemented {
_ = resp.Body.Close()
resp, err = doProbeRequest(ctx, client, http.MethodGet, u.String(), "bytes=0-0")
if err != nil {
return probeResult{}, false
}
}
pr := probeResult{scheme: scheme, status: resp.StatusCode, contentType: resp.Header.Get("Content-Type"), reachable: true}
_ = resp.Body.Close()
return pr, true
}
// exposureProbeHTTPClient builds the pinned-dial client shared by the
// headers-only probe and the phpinfo body confirmation: connections dial the
// vhost's configured serving address (never DNS), SNI/Host carry the domain,
// and redirects are never followed. The returned func releases idle
// connections; a fresh transport per probe is needed because SNI
// (ServerName) is per-domain.
func exposureProbeHTTPClient(domain, host string) (*http.Client, func()) {
transport := &http.Transport{
MaxResponseHeaderBytes: 64 << 10,
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
_, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
d := &net.Dialer{Timeout: exposureProbeConnTimeout}
return d.DialContext(ctx, network, net.JoinHostPort(host, port))
},
// #nosec G402 -- reachability probe to this host's own serving IP (never
// an external endpoint); no trust decision rides on the certificate, and
// SNI must match the requested domain to select the right vhost.
TLSClientConfig: &tls.Config{InsecureSkipVerify: true, ServerName: domain},
}
client := &http.Client{
Transport: transport,
Timeout: exposureProbeTotalTimeout,
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
}
return client, transport.CloseIdleConnections
}
// phpinfoBodyReadMax bounds how much of the phpinfo.php response body the
// confirmation stage reads. Real dumps put "PHP Version" in the first couple
// of KB; 64 KiB leaves generous room for custom headers while keeping the
// probe cheap.
const phpinfoBodyReadMax = 64 << 10
// phpinfoMinBodyBytes is the minimum confirmed-dump size. Even a minimal PHP
// build renders tens of KB of phpinfo output; the observed false-positive
// stubs answer with empty or sub-100-byte bodies.
const phpinfoMinBodyBytes = 4096
// phpinfoBodyFetcher retrieves up to phpinfoBodyReadMax bytes of the response
// body for a phpinfo candidate. The bool reports whether the result can clear
// an earlier finding: non-success HTTP statuses are complete negatives, while
// transport, body-read, and inconclusive partial-response failures are not.
// This separate seam keeps every other exposure class on its headers-only
// contract.
type phpinfoBodyFetcher func(ctx context.Context, scheme, domain, host, urlPath string) ([]byte, bool)
var fetchPHPInfoBody phpinfoBodyFetcher = realFetchPHPInfoBody
func realFetchPHPInfoBody(ctx context.Context, scheme, domain, host, urlPath string) ([]byte, bool) {
host = normalizeServingIP(host)
if host == "" {
return nil, false
}
fetchCtx, cancel := context.WithTimeout(ctx, exposureProbeTotalTimeout)
defer cancel()
client, closeIdle := exposureProbeHTTPClient(domain, host)
defer closeIdle()
u := url.URL{Scheme: scheme, Host: domain, Path: urlPath}
return fetchPHPInfoBodyWithClient(fetchCtx, client, u.String())
}
func fetchPHPInfoBodyWithClient(ctx context.Context, client *http.Client, u string) ([]byte, bool) {
resp, err := doProbeRequest(ctx, client, http.MethodGet, u, "")
if err != nil {
return nil, false
}
defer func() { _ = resp.Body.Close() }()
partial := resp.StatusCode == http.StatusPartialContent
if resp.StatusCode != http.StatusOK && !partial {
return nil, true
}
reader := io.Reader(resp.Body)
switch strings.ToLower(strings.TrimSpace(resp.Header.Get("Content-Encoding"))) {
case "":
case "gzip":
compressed, gzErr := gzip.NewReader(resp.Body)
if gzErr != nil {
return nil, false
}
defer func() { _ = compressed.Close() }()
reader = compressed
default:
return nil, false
}
body, err := io.ReadAll(io.LimitReader(reader, phpinfoBodyReadMax))
if err != nil {
return nil, false
}
if partial && !isRealPHPInfoBody(body) {
return nil, false
}
return body, true
}
// confirmPHPInfoBody checks both origin protocols because HEAD and GET can be
// routed differently, and HTTPS and HTTP can execute different handlers. A
// dump observed on either protocol is conclusive; if neither exposes a dump,
// every GET must complete before an earlier finding can be cleared.
func confirmPHPInfoBody(ctx context.Context, preferredScheme, domain, host, urlPath string) (string, bool, bool) {
schemes := []string{"https", "http"}
if preferredScheme == "http" {
schemes[0], schemes[1] = schemes[1], schemes[0]
}
complete := true
for _, scheme := range schemes {
body, ok := fetchPHPInfoBody(ctx, scheme, domain, host, urlPath)
if !ok {
complete = false
continue
}
if isRealPHPInfoBody(body) {
return scheme, true, true
}
}
return "", false, complete
}
// isRealPHPInfoBody reports whether a response body is genuine phpinfo()
// output: the version banner every phpinfo variant emits (HTML "PHP Version
// x.y" heading or CLI "PHP Version => x.y") plus a dump-sized body. UTF-16
// variants cover output handlers that transcode the otherwise ASCII banner.
func isRealPHPInfoBody(body []byte) bool {
if len(body) < phpinfoMinBodyBytes {
return false
}
marker := []byte("PHP Version")
if bytes.Contains(body, marker) {
return true
}
le := make([]byte, len(marker)*2)
be := make([]byte, len(marker)*2)
for i, b := range marker {
le[i*2] = b
be[i*2+1] = b
}
return bytes.Contains(body, le) || bytes.Contains(body, be)
}
func doProbeRequest(ctx context.Context, client *http.Client, method, u, rangeHdr string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, method, u, nil)
if err != nil {
return nil, err
}
if rangeHdr != "" {
req.Header.Set("Range", rangeHdr)
}
req.Header.Set("User-Agent", "csm-exposure-probe")
return client.Do(req)
}
// exposedClass labels a docroot file by the kind of exposure it represents
// when the web server serves it as a raw download. The zero value classNone
// means "not a sensitive file" and is never emitted.
type exposedClass int
const (
classNone exposedClass = iota
// classRepoMetadata is a web-served version-control directory (.git,
// .svn): the site's source, its history and, routinely, credentials.
classRepoMetadata
classConfigLeak
classDBDump
classBackupArchive
classSourceBackup
classPHPInfo
// classSampleSQL is a plain SQL file with a sample-specific name under a
// framework/vendor/downloaded-project directory. It is still served, but is
// a Warning rather than a Critical database dump.
classSampleSQL
)
// severity maps a class to its finding severity. Credential- and
// database-bearing exposures are Critical; a leaked source backup is High;
// a phpinfo dump is Warning (information disclosure only).
func (c exposedClass) severity() alert.Severity {
switch c {
case classConfigLeak, classDBDump, classBackupArchive, classRepoMetadata:
return alert.Critical
case classSourceBackup:
return alert.High
default:
return alert.Warning
}
}
// findingName is the stable registry key / audit-log check name per class.
func (c exposedClass) findingName() string {
switch c {
case classRepoMetadata:
return "web_exposed_repo_metadata"
case classConfigLeak:
return "web_exposed_config_leak"
case classDBDump:
return "web_exposed_db_dump"
case classBackupArchive:
return "web_exposed_backup_archive"
case classSourceBackup:
return "web_exposed_source_backup"
case classPHPInfo:
return "web_exposed_phpinfo"
case classSampleSQL:
return "web_exposed_sample_sql"
default:
return ""
}
}
// vhost is one cPanel virtual host: the domain the web server answers for and
// the docroot it serves. Parsed from /etc/userdatadomains, which -- unlike the
// /home/*/public_html glob -- covers addon and subdomain docroots too.
type vhost struct {
domain string
user string
typ string
mainDomain string
docroot string
// ip is the vhost's serving address (from the ip:443/ip:80 columns). The
// reachability probe dials this rather than 127.0.0.1: LiteSpeed returns
// 403 to loopback-originated requests even for files it serves (HTTP 200)
// to a real request on the public IP, so a loopback probe confirms nothing.
ip string
// phpVersion is the vhost's MultiPHP selection ("ea-php83", "alt-php81")
// from the PHP-version column, empty when the row omits it. Callers that run
// code against a docroot need this: the system default interpreter is not
// necessarily the one the site is pinned to.
phpVersion string
// phpInherit marks a well-formed row whose vhost has no MultiPHP selection
// of its own ("inherit" or an empty column).
phpInherit bool
}
// parseUserdataDomains parses the /etc/userdatadomains map. Each line is
// "<domain>: <user>==<reseller>==<type>==<maindomain>==<docroot>==...".
// Wildcard, comment, and short/malformed lines are skipped.
func parseUserdataDomains(content string) []vhost {
vhosts, _ := parseUserdataDomainsChecked(content)
return vhosts
}
func parseUserdataDomainsChecked(content string) ([]vhost, bool) {
return parseUserdataDomainsForUse(content, true)
}
// parseUserdataDomainRootsChecked validates the fields needed by filesystem
// scanners. Unlike exposure probing, a local docroot walk does not need a
// serving IP, so a row without one is still complete for this use.
func parseUserdataDomainRootsChecked(content string) ([]vhost, bool) {
return parseUserdataDomainsForUse(content, false)
}
func parseUserdataDomainsForUse(content string, requireServingIP bool) ([]vhost, bool) {
var out []vhost
complete := true
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "*:") || strings.HasPrefix(line, "*.") {
continue
}
colon := strings.Index(line, ": ")
if colon <= 0 {
complete = false
continue
}
domain := strings.ToLower(strings.TrimSpace(line[:colon]))
fields := strings.Split(line[colon+2:], "==")
if !validProbeDomain(domain) || len(fields) < 5 {
complete = false
continue
}
docroot := filepath.Clean(strings.TrimSpace(fields[4]))
user := strings.TrimSpace(fields[0])
if !filepath.IsAbs(docroot) || docroot == string(filepath.Separator) ||
!validAccountName.MatchString(user) {
complete = false
continue
}
servingIP := parseServingIP(fields)
if requireServingIP && servingIP == "" {
// Without a literal serving address the vhost cannot be probed
// reliably. Keep the row so callers can preserve partial-scan state,
// but do not treat this map as a complete scan input.
complete = false
}
out = append(out, vhost{
domain: domain,
user: user,
typ: strings.TrimSpace(fields[2]),
mainDomain: strings.ToLower(strings.TrimSpace(fields[3])),
docroot: docroot,
ip: servingIP,
phpVersion: parseVhostPHPVersion(fields),
phpInherit: vhostPHPInherits(fields),
})
}
return out, complete
}
// parseServingIP extracts the vhost's serving address from the ip:443 column
// (preferred) or the ip:80 column. Only literal unicast IP addresses are
// accepted so a malformed map can never turn the pinned origin probe into a
// DNS lookup or a connection to a non-serving address.
func parseServingIP(fields []string) string {
bindings := []struct {
field int
port string
}{
{field: 6, port: "443"},
{field: 5, port: "80"},
}
for _, binding := range bindings {
if binding.field >= len(fields) {
continue
}
hp := strings.TrimSpace(fields[binding.field])
if hp == "" {
continue
}
if host, port, err := net.SplitHostPort(hp); err == nil {
if port == binding.port {
if ip := normalizeServingIP(host); ip != "" {
return ip
}
}
continue
}
// cPanel may render IPv6 bindings without brackets. SplitHostPort
// rejects those, so split at the last colon and validate both parts.
if idx := strings.LastIndexByte(hp, ':'); idx > 0 {
if strings.TrimSpace(hp[idx+1:]) == binding.port {
if ip := normalizeServingIP(hp[:idx]); ip != "" {
return ip
}
}
}
}
return ""
}
// normalizeServingIP returns a canonical literal unicast address. In
// particular, it rejects hostnames, unspecified/listener addresses, loopback,
// multicast, and link-local addresses, none of which identify the vhost's
// externally serving origin.
func normalizeServingIP(host string) string {
ip := net.ParseIP(strings.TrimSpace(host))
if ip == nil || !ip.IsGlobalUnicast() {
return ""
}
return ip.String()
}
// probeHost is the literal serving address the reachability probe dials. An
// empty result means the vhost must be skipped and the scan marked incomplete;
// loopback is deliberately not a fallback because it produces false 403s on
// LiteSpeed.
func probeHost(vh vhost) string {
return normalizeServingIP(vh.ip)
}
// validProbeDomain rejects URL authority syntax. Without this guard a corrupt
// map entry containing a port could make the privileged daemon probe an
// arbitrary service on the serving IP instead of the vhost's port 80/443.
func validProbeDomain(domain string) bool {
if domain == "" || len(domain) > 253 || strings.ContainsAny(domain, ":/\\?#@[]%") {
return false
}
for _, r := range domain {
if r <= ' ' || r == 0x7f {
return false
}
}
return true
}
// probeResult is the local reachability check outcome for one candidate file.
// It carries the requested scheme and only headers-level response facts
// (status, Content-Type); the body is never read, so a leaked file's secret
// contents never enter CSM.
type probeResult struct {
scheme string
status int
contentType string
reachable bool
// partial is true when one protocol could not be reached. Unless the other
// protocol confirms an exposure, that uncertainty must prevent purging a
// finding observed during an earlier complete scan.
partial bool
}
// confirmExposure reports whether a classified candidate is a confirmed,
// downloadable exposure. It fails closed: anything the server blocks
// (403/404), redirects (3xx), or that comes back as executed HTML on a
// non-executing class is not a finding.
func confirmExposure(class exposedClass, pr probeResult) bool {
if !pr.reachable {
return false
}
if pr.status != 200 && pr.status != 206 {
return false
}
if class == classPHPInfo {
return true
}
// Non-executing sensitive files: a raw (non-HTML) body means the server
// served the file itself. An HTML body means it executed or returned an
// error/challenge page -- not a confirmed source leak.
return !isHTMLContentType(pr.contentType)
}
func isHTMLContentType(ct string) bool {
ct = strings.ToLower(strings.TrimSpace(ct))
return strings.HasPrefix(ct, "text/html") || strings.HasPrefix(ct, "application/xhtml")
}
// phpExecExts are the extensions a cPanel PHP handler executes rather than
// serving as text. A backup whose FINAL extension is one of these is run by
// the interpreter, emits no source, and is therefore not a leak.
var phpExecExts = map[string]bool{
"php": true, "php3": true, "php4": true, "php5": true, "php7": true,
"php8": true, "phtml": true, "pht": true, "phar": true,
}
// backupSuffixSet are terminal dot-segments that mark a renamed backup a web
// server serves as text (not executed). "bak-<timestamp>" style segments are
// matched by prefix in isBackupSuffix.
var backupSuffixSet = map[string]bool{
"old": true, "bak": true, "save": true, "orig": true, "broken": true,
"swp": true, "swo": true, "tmp": true, "copy": true, "backup": true,
}
// dbDumpSuffixes are raw database-dump endings. Each may optionally be wrapped
// in one of dbDumpArchiveSuffixes.
var dbDumpSuffixes = []string{
".sql", ".dump", ".mysql",
}
var dbDumpArchiveSuffixes = []string{
"", ".gz", ".zip", ".bz2", ".xz", ".zst", ".7z", ".rar",
".tar", ".tar.gz", ".tar.bz2", ".tar.xz", ".tar.zst", ".tgz", ".tbz",
}
// archiveExts are archive endings that, combined with a backup token, denote a
// full-site backup a visitor could download whole.
var archiveExts = []string{
".tar.gz", ".tgz", ".tar.bz2", ".tbz", ".tar.xz", ".tar.zst", ".tar",
".zip", ".rar", ".7z", ".gz", ".bz2", ".xz", ".zst",
}
// backupTokens are name fragments that mark an archive as a site/db backup
// rather than a legitimately-offered download. Kept strong to avoid flagging
// ordinary user zips.
var backupTokens = []string{
"public_html", "wp-content",
}
// These generic words are backup signals only as complete filename tokens;
// substring matching would turn names such as dumpster.zip or immigration.zip
// into Critical false positives.
var delimitedBackupTokens = []string{
"backup", "dump", "full",
}
// sampleSQLDirMarkers are directory-name segments that provide supporting
// context for a sample-specific SQL file name. Directory context alone is not
// enough to demote a dump because these paths are customer-controlled.
var sampleSQLDirMarkers = map[string]bool{
"example": true, "examples": true,
"sample": true, "samples": true,
"demo": true, "demos": true,
"doc": true, "docs": true,
"fixture": true, "fixtures": true, "testdata": true,
"vendor": true, "node_modules": true, "bower_components": true,
}
var sampleSQLNameTokens = map[string]bool{
"demo": true, "example": true, "fixture": true, "install": true,
"migration": true, "sample": true, "schema": true, "seed": true,
"setup": true, "structure": true, "update": true, "upgrade": true,
}
// isSampleSQLPath reports whether rel (a leading-slash URL path) names a plain
// SQL file with a sample/schema-specific name under framework scaffolding.
// Both the file name and directory context must support that classification.
// Archived, renamed, and customer-named dumps remain Critical even under a
// generic docs/vendor path.
func isSampleSQLPath(rel string) bool {
segs := strings.Split(rel, "/")
if len(segs) < 2 {
return false
}
name := strings.ToLower(segs[len(segs)-1])
if !strings.HasSuffix(name, ".sql") {
return false
}
stem := strings.TrimSuffix(name, ".sql")
sampleName := false
for _, token := range strings.FieldsFunc(stem, func(r rune) bool {
return r == '-' || r == '_' || r == '.' || r == ' '
}) {
if sampleSQLNameTokens[token] {
sampleName = true
break
}
}
archiveProject := false
for _, seg := range segs[:len(segs)-1] { // dir segments only, skip file name
s := strings.ToLower(seg)
if s == "" {
continue
}
if sampleName && sampleSQLDirMarkers[s] {
return true
}
// Directories unpacked from a GitHub archive keep a "-master"/"-main"
// suffix, a strong signal the tree is a downloaded sample project.
if (strings.HasSuffix(s, "-master") && len(s) > len("-master")) ||
(strings.HasSuffix(s, "-main") && len(s) > len("-main")) {
archiveProject = true
}
}
return archiveProject && (sampleName || stem == "database")
}
// demoteSampleSQL lowers a database-dump candidate to classSampleSQL when its
// path has specific sample-file and framework-scaffolding signals. Only
// classDBDump is eligible; credential, archive, and source classes are never
// demoted.
func demoteSampleSQL(class exposedClass, rel string) exposedClass {
if class == classDBDump && isSampleSQLPath(rel) {
return classSampleSQL
}
return class
}
// classifyExposedFile classifies a base file name. It is deliberately
// conservative: the benign long tail (samples, examples, live scripts, and
// backups that still execute) returns classNone so the detector does not
// drown operators in false positives.
func classifyExposedFile(name string) exposedClass {
lower := strings.ToLower(strings.TrimSpace(name))
if lower == "" {
return classNone
}
// Diagnostic by its unambiguous conventional name. A generic info.php can
// be any application endpoint, and headers alone cannot prove it calls
// phpinfo(), so classifying that name would create false positives.
if lower == "phpinfo.php" {
return classPHPInfo
}
// Benign long tail excluded before any leak matching.
if isBenignExposedName(lower) {
return classNone
}
// Live scripts are executed by the PHP handler rather than served as
// source. Keep the phpinfo diagnostic exceptions above, but reject every
// other candidate whose final extension still executes -- including names
// such as .env.php that would otherwise match the dotenv family.
if hasPHPExecExtension(lower) {
return classNone
}
// Dotenv family (secrets), including editor backups such as .env~.
if isDotenvName(lower) {
return classConfigLeak
}
if hasDBDumpSuffix(lower) {
return classDBDump
}
if isBackupArchive(lower) {
return classBackupArchive
}
// A sensitive file can itself have been renamed with one or more backup
// suffixes (for example wp-config.php.bak.old).
if stripped, ok := stripBackupSuffixes(lower); ok {
switch {
case isDotenvName(stripped):
return classConfigLeak
case hasDBDumpSuffix(stripped):
return classDBDump
case isBackupArchive(stripped):
return classBackupArchive
case looksLikePHPSource(stripped):
if looksLikeConfig(stripped) {
return classConfigLeak
}
return classSourceBackup
}
}
return classNone
}
func isDotenvName(lower string) bool {
return lower == ".env" || strings.HasPrefix(lower, ".env.")
}
// isBenignExposedName matches shipped samples and templates that carry no
// secret and must never be flagged.
func isBenignExposedName(lower string) bool {
if lower == "wp-config-sample.php" {
return true
}
if stripped, ok := stripBackupSuffixes(lower); ok && stripped == "wp-config-sample.php" {
return true
}
for _, template := range []string{".env.example", ".env.sample", ".env.dist", ".env.default"} {
if lower == template || strings.HasPrefix(lower, template+".") || strings.HasPrefix(lower, template+"~") {
return true
}
}
return strings.HasSuffix(lower, ".dist") || strings.HasSuffix(lower, ".default")
}
func hasDBDumpSuffix(lower string) bool {
for _, base := range dbDumpSuffixes {
for _, archive := range dbDumpArchiveSuffixes {
if strings.HasSuffix(lower, base+archive) {
return true
}
}
}
return false
}
func isBackupArchive(lower string) bool {
if strings.HasSuffix(lower, ".wpress") {
return true
}
hasArchiveExt := false
for _, e := range archiveExts {
if strings.HasSuffix(lower, e) {
hasArchiveExt = true
break
}
}
if !hasArchiveExt {
return false
}
for _, tok := range backupTokens {
if strings.Contains(lower, tok) {
return true
}
}
for _, tok := range delimitedBackupTokens {
if hasDelimitedFilenameToken(lower, tok) {
return true
}
}
return false
}
func hasDelimitedFilenameToken(name, token string) bool {
for start := 0; start <= len(name)-len(token); {
i := strings.Index(name[start:], token)
if i < 0 {
return false
}
i += start
beforeOK := i == 0 || !isASCIIAlphaNumeric(name[i-1])
after := i + len(token)
afterOK := after == len(name) || !isASCIIAlphaNumeric(name[after])
if beforeOK && afterOK {
return true
}
start = i + 1
}
return false
}
func isASCIIAlphaNumeric(b byte) bool {
return b >= 'a' && b <= 'z' || b >= '0' && b <= '9'
}
// stripBackupSuffix removes a single trailing backup marker (a "~" or a
// terminal dot-segment such as ".old" / ".bak-20260515") and reports whether
// one was found.
func stripBackupSuffix(lower string) (string, bool) {
if strings.HasSuffix(lower, "~") {
return strings.TrimSuffix(lower, "~"), true
}
idx := strings.LastIndexByte(lower, '.')
if idx <= 0 {
return lower, false
}
seg := lower[idx+1:]
if isBackupSuffix(seg) {
return lower[:idx], true
}
return lower, false
}
func stripBackupSuffixes(lower string) (string, bool) {
stripped := lower
found := false
for {
next, ok := stripBackupSuffix(stripped)
if !ok {
return stripped, found
}
stripped = next
found = true
}
}
func isBackupSuffix(seg string) bool {
if backupSuffixSet[seg] || isASCIIDigits(seg) {
return true
}
// "bak-20260515-124446", "bak_1", "save1" style timestamped variants.
for _, p := range []string{"bak-", "bak_", "old-", "old_", "save-", "save_", "backup-", "backup_", "orig-", "orig_"} {
if strings.HasPrefix(seg, p) {
return true
}
}
for _, p := range []string{"bak", "old", "save", "backup", "orig"} {
if strings.HasPrefix(seg, p) && isASCIIDigits(strings.TrimPrefix(seg, p)) {
return true
}
}
return false
}
func isASCIIDigits(s string) bool {
if s == "" {
return false
}
for i := 0; i < len(s); i++ {
if s[i] < '0' || s[i] > '9' {
return false
}
}
return true
}
// looksLikePHPSource reports whether the name (with its backup suffix already
// removed) is a PHP source file the server would serve as text.
func looksLikePHPSource(stripped string) bool {
return hasPHPExecExtension(stripped)
}
func hasPHPExecExtension(name string) bool {
idx := strings.LastIndexByte(name, '.')
if idx < 0 {
return false
}
ext := name[idx+1:]
return phpExecExts[ext]
}
// looksLikeConfig reports whether a source backup carries configuration /
// credentials, warranting Critical instead of High.
func looksLikeConfig(stripped string) bool {
for _, tok := range []string{"config", "wp-config", "settings", "database", "db-config", "credentials", "secret"} {
if strings.Contains(stripped, tok) {
return true
}
}
return false
}
package checks
import (
"context"
"errors"
"io/fs"
"os"
"path"
"strings"
"syscall"
"github.com/pidginhost/csm/internal/zipdir"
)
const (
// The central directory is the only part of a zip this check reads. A byte
// bound covers entry count, names, comments, and extra fields together while
// allowing substantially more than the old 4096-entry cutoff.
archiveDirectoryScanByteLimit = 16 << 20
zipDirectoryHeaderLen = zipdir.HeaderLen
zipDirectoryEndLen = zipdir.EndLen
zipDirectory64EndLen = zipdir.End64Len
zipDirectory64LocLen = zipdir.End64LocatorLen
// A site backup names its document root at or just below the archive
// root. Past that a configuration file needs corroboration.
shallowSiteConfigDepth = 2
zipDirectoryHeaderSignature = zipdir.HeaderSignature
zipDirectoryEndSignature = zipdir.EndSignature
zipDirectory64EndSignature = zipdir.End64Signature
zipDirectory64LocSignature = zipdir.End64LocatorSign
)
var errArchiveFormat = zipdir.ErrFormat
// docrootDirNames are hosting conventions for the web root. An archive whose
// entries sit under one is a copy of a served directory tree.
var docrootDirNames = map[string]bool{
"wwwroot": true,
"public_html": true,
"htdocs": true,
"httpdocs": true,
}
// archiveHoldsSiteBackup reports whether a web-reachable archive contains a
// copy of a site, judged by its entry list rather than its file name.
func archiveHoldsSiteBackup(p string) bool {
holds, _ := archiveSiteBackupStatus(context.Background(), p)
return holds
}
// archiveSiteBackupStatus also reports whether the result is complete. A
// resource-limited or interrupted inspection must retain an earlier finding.
// Malformed zip data without readable evidence is a complete negative result:
// a file merely ending in .zip must not hold the whole check incomplete forever.
func archiveSiteBackupStatus(ctx context.Context, p string) (holds, complete bool) {
if !strings.HasSuffix(strings.ToLower(p), ".zip") {
return false, true
}
info, err := osFS.Lstat(p)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return false, true
}
return false, false
}
if !info.Mode().IsRegular() && info.Mode()&os.ModeSymlink == 0 {
return false, true
}
var f *os.File
if _, productionFS := osFS.(realOS); productionFS {
// The account controls this path. A nonblocking open plus the descriptor
// type check allows web-served regular symlinks without letting a FIFO
// swapped in after the directory walk strand the scan.
// #nosec G304 -- read-only document-root candidate; flags reject unsafe types.
f, err = os.OpenFile(p, os.O_RDONLY|syscall.O_NONBLOCK, 0)
} else {
f, err = osFS.Open(p)
}
if err != nil {
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ELOOP) {
return false, true
}
return false, false
}
defer func() { _ = f.Close() }()
openedInfo, err := f.Stat()
if err != nil {
return false, false
}
if !openedInfo.Mode().IsRegular() {
return false, true
}
dir, err := zipdir.Read(f, openedInfo.Size(), archiveDirectoryScanByteLimit)
if err != nil {
if errors.Is(err, errArchiveFormat) {
return false, true
}
return false, false
}
holds, err = scanZipDirectory(ctx, f, dir)
if err != nil {
if errors.Is(err, errArchiveFormat) {
// A damaged tail cannot erase site markers already read.
return holds, true
}
return holds, false
}
return holds, true
}
func scanZipDirectory(ctx context.Context, f *os.File, dir zipdir.Directory) (bool, error) {
holdsSite := false
// A wp-config.php nested past the shallow bound only counts alongside a
// WordPress runtime file, so both are accumulated across the whole
// directory rather than decided per entry.
deepConfig := false
wpRuntime := false
// A nested Joomla configuration.php counts only when its own directory
// also holds a Joomla entry point, so both are keyed by directory.
joomlaConfigDirs := map[string]bool{}
joomlaRuntimeDirs := map[string]bool{}
err := zipdir.Walk(ctx, f, dir, func(e zipdir.Entry) error {
entry := e.Name
if archiveEntrySignalsSiteBackup(entry) {
holdsSite = true
return nil
}
if archiveEntryIsDeepWPConfig(entry) {
deepConfig = true
}
if archiveEntryIsWPRuntime(entry) {
wpRuntime = true
}
// Names alone cannot distinguish PHP files from directories or
// symlinks. Interpret central-directory attributes without opening
// or decompressing any entry payload.
if e.Mode().IsRegular() {
if d, ok := archiveNestedJoomlaConfigDir(entry); ok {
joomlaConfigDirs[d] = true
holdsSite = holdsSite || joomlaRuntimeDirs[d]
}
if d, ok := archiveJoomlaRuntimeDir(entry); ok {
joomlaRuntimeDirs[d] = true
holdsSite = holdsSite || joomlaConfigDirs[d]
}
}
return nil
})
return holdsSite || (deepConfig && wpRuntime), err
}
// archiveEntryPath normalises a raw zip entry name and rejects the shapes that
// must never be read as a marker: absolute, drive-qualified, and traversal
// names. It returns nil for anything unusable.
func archiveEntryPath(rawName string) []string {
return archiveEntryPathPreservingCase(strings.ToLower(strings.TrimSpace(rawName)))
}
// Directory identity must retain case and whitespace when pairing entries:
// archives from Linux can contain distinct directories differing only by either.
func archiveEntryPathPreservingCase(rawName string) []string {
name := strings.ReplaceAll(rawName, `\`, "/")
if name == "" || strings.HasPrefix(name, "/") ||
(len(name) >= 3 && ((name[0] >= 'a' && name[0] <= 'z') || (name[0] >= 'A' && name[0] <= 'Z')) && name[1] == ':' && name[2] == '/') {
return nil
}
name = path.Clean(name)
if name == "." || name == ".." || strings.HasPrefix(name, "../") {
return nil
}
parts := strings.Split(strings.Trim(name, "/"), "/")
if len(parts) == 0 || parts[0] == "" {
return nil
}
return parts
}
func archiveEntrySignalsSiteBackup(rawName string) bool {
parts := archiveEntryPath(rawName)
if len(parts) == 0 {
return false
}
for _, part := range parts[:len(parts)-1] {
if docrootDirNames[part] {
return true
}
}
if len(parts) == 1 && hasDBDumpSuffix(parts[0]) {
return true
}
switch parts[len(parts)-1] {
case "wp-config.php":
// A configuration file this close to the archive root is the archive's
// own subject. Deeper ones are ambiguous -- plugins ship fixtures at
// arbitrary depth -- so those are paired with a runtime marker instead,
// via archiveEntryIsDeepWPConfig.
return len(parts) <= shallowSiteConfigDepth
case "configuration.php":
return len(parts) == 1
case "settings.php":
return hasArchivePathSuffix(parts, "sites", "default", "settings.php")
default:
return false
}
}
// archiveEntryIsDeepWPConfig reports a wp-config.php nested past the depth the
// shallow rule accepts on its own.
func archiveEntryIsDeepWPConfig(rawName string) bool {
parts := archiveEntryPath(rawName)
return len(parts) > shallowSiteConfigDepth && parts[len(parts)-1] == "wp-config.php"
}
// archiveEntryIsWPRuntime reports a file only a WordPress installation carries.
// A plugin or theme bundle shipping a configuration fixture has none of these,
// which is what separates a nested backup from a nested fixture.
func archiveEntryIsWPRuntime(rawName string) bool {
parts := archiveEntryPath(rawName)
if len(parts) == 0 {
return false
}
switch parts[len(parts)-1] {
case "wp-load.php", "wp-settings.php", "wp-blog-header.php":
return true
}
for _, part := range parts[:len(parts)-1] {
if part == "wp-includes" {
return true
}
}
return false
}
// archiveNestedJoomlaConfigDir returns the directory of a configuration.php
// below the archive root. A root-level one already counts on its own.
func archiveNestedJoomlaConfigDir(rawName string) (string, bool) {
parts := archiveEntryPathPreservingCase(rawName)
if len(parts) < 2 || !strings.EqualFold(parts[len(parts)-1], "configuration.php") {
return "", false
}
return strings.Join(parts[:len(parts)-1], "/"), true
}
// archiveJoomlaRuntimeDir returns the site directory of a Joomla core entry
// point. Every Joomla release carries these and no extension package does.
func archiveJoomlaRuntimeDir(rawName string) (string, bool) {
parts := archiveEntryPathPreservingCase(rawName)
if len(parts) < 2 {
return "", false
}
parent, file := strings.ToLower(parts[len(parts)-2]), strings.ToLower(parts[len(parts)-1])
if (parent == "includes" && file == "defines.php") ||
(parent == "administrator" && file == "index.php") {
return strings.Join(parts[:len(parts)-2], "/"), true
}
return "", false
}
func hasArchivePathSuffix(parts []string, suffix ...string) bool {
if len(parts) < len(suffix) {
return false
}
start := len(parts) - len(suffix)
for i := range suffix {
if parts[start+i] != suffix[i] {
return false
}
}
return true
}
package checks
import (
"context"
"fmt"
"net/url"
"strings"
)
// Exposure verification shares the detection-time confirmation rule and only
// clears a finding after a complete probe pinned to the current local vhost.
// exposedVerifiableChecks is the web_exposed_* family, keyed the same way the
// findings are.
var exposedVerifiableChecks = []string{
"web_exposed_repo_metadata",
"web_exposed_config_leak",
"web_exposed_db_dump",
"web_exposed_backup_archive",
"web_exposed_source_backup",
"web_exposed_phpinfo",
"web_exposed_sample_sql",
}
// exposedReverifyLogicVersion is part of the daemon sweep token. Bump it when
// unattended exposure verification semantics change so existing findings are
// revisited once after upgrade.
const exposedReverifyLogicVersion = 1
// isExposedVerifiable reports whether check belongs to the web_exposed_* family,
// whose findings the automatic sweep may re-probe and dismiss.
func isExposedVerifiable(check string) bool {
for _, c := range exposedVerifiableChecks {
if c == check {
return true
}
}
return false
}
// exposedClassForCheck is the inverse of exposedClass.findingName. The class
// decides what counts as a confirmed exposure, so verification has to recover
// it or it would judge a phpinfo page by the raw-leak rule.
func exposedClassForCheck(check string) (exposedClass, bool) {
for _, c := range []exposedClass{
classRepoMetadata, classConfigLeak, classDBDump, classBackupArchive,
classSourceBackup, classPHPInfo, classSampleSQL,
} {
if c.findingName() == check {
return c, true
}
}
return classNone, false
}
// exposureURLFromMessage pulls the probed URL back out of the finding message
// ("Web-exposed <label> reachable at <url>").
func exposureURLFromMessage(message string) string {
const marker = " reachable at "
i := strings.Index(message, marker)
if i < 0 {
return ""
}
return strings.TrimSpace(message[i+len(marker):])
}
type exposureVhostIndex struct {
servingIPs map[string]string
complete bool
}
func loadExposureVhostIndex() exposureVhostIndex {
index := exposureVhostIndex{servingIPs: map[string]string{}}
content, err := osFS.ReadFile(userdataDomainsPath)
if err != nil {
return index
}
vhosts, complete := parseUserdataDomainsChecked(string(content))
index.complete = complete
for _, vh := range vhosts {
host := probeHost(vh)
if previous, exists := index.servingIPs[vh.domain]; exists {
if previous != host {
index.complete = false
}
continue
}
index.servingIPs[vh.domain] = host
}
return index
}
func (index exposureVhostIndex) servingIPForDomain(domain string) string {
domain = strings.ToLower(strings.TrimSpace(domain))
return index.servingIPs[domain]
}
// verifyExposedFile re-probes a web_exposed_* finding and resolves it only when
// a complete pinned probe says the server no longer serves the exposure.
func verifyExposedFile(in VerifyInput) VerifyResult {
vhosts := in.exposureVhosts
if vhosts == nil {
loaded := loadExposureVhostIndex()
vhosts = &loaded
}
return verifyExposedFileWithVhosts(in, vhosts)
}
func verifyExposedFileWithVhosts(in VerifyInput, vhosts *exposureVhostIndex) VerifyResult {
ctx := in.Context
if ctx == nil {
ctx = context.Background()
}
class, ok := exposedClassForCheck(in.Check)
if !ok {
return VerifyResult{Checked: false, Detail: fmt.Sprintf("unknown exposure class for '%s'", in.Check)}
}
raw := exposureURLFromMessage(in.Message)
if raw == "" {
return VerifyResult{Checked: false, Detail: "could not extract the exposure URL from the finding"}
}
u, err := url.Parse(raw)
if err != nil || u.Host == "" || u.Path == "" {
return VerifyResult{Checked: false, Detail: fmt.Sprintf("unusable exposure URL %q", raw)}
}
domain := u.Hostname()
if !validProbeDomain(domain) {
return VerifyResult{Checked: false, Detail: fmt.Sprintf("unusable exposure domain %q", domain)}
}
if !vhosts.complete {
return VerifyResult{Checked: false,
Detail: "local vhost routing is incomplete; re-check cannot select a trusted serving address"}
}
// Pin to this host's own serving address. Resolving the domain through
// public DNS would ask whoever owns it now: a domain that has migrated away
// answers from its new provider, and that answer -- 200 with the new site's
// HTML, or a clean 404 -- says nothing about what this server exposes.
host := vhosts.servingIPForDomain(domain)
if host == "" {
return VerifyResult{Checked: false,
Detail: fmt.Sprintf("%s is no longer served by this host; re-check cannot reach the original vhost", domain)}
}
pr := webProber.probeComplete(ctx, domain, host, u.Path)
if !pr.reachable {
return VerifyResult{Checked: false,
Detail: fmt.Sprintf("could not reach %s to re-check; leaving the finding open", domain)}
}
if pr.partial {
return VerifyResult{Checked: false,
Detail: "only one protocol answered; leaving the finding open until a complete probe"}
}
if confirmExposure(class, pr) {
if class == classPHPInfo {
_, exposed, complete := confirmPHPInfoBody(ctx, pr.scheme, domain, host, u.Path)
switch {
case exposed:
return VerifyResult{Checked: true, Resolved: false,
Detail: "still downloadable: confirmed phpinfo output"}
case !complete:
return VerifyResult{Checked: false,
Detail: "could not complete phpinfo body confirmation; leaving the finding open"}
}
return VerifyResult{Checked: true, Resolved: true,
Detail: "no longer served as confirmed phpinfo output"}
}
return VerifyResult{Checked: true, Resolved: false,
Detail: fmt.Sprintf("still downloadable: HTTP %d %s", pr.status, pr.contentType)}
}
return VerifyResult{Checked: true, Resolved: true,
Detail: fmt.Sprintf("no longer served as an exposure (HTTP %d %s)", pr.status, pr.contentType)}
}
package checks
import (
"path/filepath"
"strings"
)
// repoMetadataMarkers lists the files whose presence proves that a directory
// named like a version-control store is one, so the walker can surface the
// exposure without descending into thousands of objects.
func repoMetadataMarkers(dirName string) []string {
switch strings.ToLower(dirName) {
case ".git":
return []string{"HEAD"}
case ".svn":
return []string{"wc.db", "entries"}
default:
return nil
}
}
// isRepoMetadataDir reports whether dir is a version-control store.
func isRepoMetadataDir(dir string) bool {
return len(repoMetadataMarkers(filepath.Base(dir))) > 0
}
// classifyExposedPath classifies a candidate by its full path: a marker file
// inside a repository directory is the repository's exposure, whatever the
// file is called; every other candidate is classified by name.
func classifyExposedPath(path string) exposedClass {
dir := filepath.Dir(path)
base := filepath.Base(path)
for _, marker := range repoMetadataMarkers(filepath.Base(dir)) {
if strings.EqualFold(base, marker) {
return classRepoMetadata
}
}
return classifyExposedFile(base)
}
package checks
import (
"bytes"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/quarantinefs"
)
// Virtual patching for web-exposed files.
//
// A confirmed web_exposed_* finding means a visitor can download a sensitive
// file. Rather than delete the customer's file, CSM can write an .htaccess
// "Require all denied" rule that blocks HTTP access while leaving the file on
// disk (so the application, which reads it from the filesystem, keeps working).
//
// Every write records rollback metadata under the quarantine pre_clean dir so
// the existing /api/v1/quarantine-restore path can restore an existing file or
// remove one CSM created. Gating (off/manual/auto + dry_run) lives in callers.
// chownFunc is overridable in tests. It receives the already-open temporary
// file so ownership cannot be redirected through a path swap.
var chownFunc = func(file *os.File, uid, gid int) error {
return file.Chown(uid, gid)
}
// virtualPatchBeforeCommitForTest simulates a customer or deploy process
// changing .htaccess between the initial read and the atomic commit.
var virtualPatchBeforeCommitForTest func(string, string)
var syncVirtualPatchDirectory = quarantinefs.SyncDir
const maxVirtualPatchHtaccessSize = 4 << 20
const (
QuarantineRestoreReplaceIfUnchanged = "replace_if_unchanged"
QuarantineRestoreRemoveIfUnchanged = "remove_if_unchanged"
)
const (
vpBeginFile = "# BEGIN CSM exposed-file virtual-patch"
vpEndFile = "# END CSM exposed-file virtual-patch"
vpBeginDir = "# BEGIN CSM exposed-file virtual-patch: deny directory"
vpEndDir = "# END CSM exposed-file virtual-patch: deny directory"
// The parent block lives one directory up, outside anything a backup
// plugin owns, so it survives the plugin rewriting its own .htaccess.
vpBeginParent = "# BEGIN CSM exposed-file virtual-patch: deny archive extension"
vpEndParent = "# END CSM exposed-file virtual-patch: deny archive extension"
)
const virtualPatchAlreadyApplied = "already virtual-patched"
var errVirtualPatchAlreadyApplied = errors.New(virtualPatchAlreadyApplied)
// virtualPatchRevertedPrefix marks the finding raised when CSM has to write a
// deny block it already wrote once. Backup plugins own the .htaccess inside
// their own directory and rewrite it on every run, so the deny disappears and
// the archives are downloadable again until the next scan.
const virtualPatchRevertedPrefix = "VIRTUAL-PATCH re-applied after revert:"
var ErrVirtualPatchRestoreConflict = errors.New("virtual-patch restore conflicts with current .htaccess")
var errVirtualPatchRollbackIncomplete = errors.New("virtual-patch rollback incomplete")
type htaccessState struct {
content []byte
info os.FileInfo
existed bool
uid int
gid int
mode os.FileMode
}
type virtualPatchBackup struct {
itemPath string
metaPath string
}
// vpExposedChecks are the web_exposed_* finding names eligible for virtual
// patching -- every web-reachable file class the detector reports.
var vpExposedChecks = map[string]bool{
"web_exposed_repo_metadata": true,
"web_exposed_config_leak": true,
"web_exposed_db_dump": true,
"web_exposed_sample_sql": true,
"web_exposed_backup_archive": true,
"web_exposed_source_backup": true,
"web_exposed_phpinfo": true,
}
func isVirtualPatchableExposedCheck(check string) bool { return vpExposedChecks[check] }
// VirtualPatchExposedFile writes an .htaccess "Require all denied" rule that
// blocks HTTP download of a confirmed web-exposed file without modifying the
// file itself. For an archive inside a known backup-plugin directory the whole
// directory is denied. Returns Success=false with an "already" error when all
// applicable rules are already present (idempotent no-op).
func VirtualPatchExposedFile(filePath string) RemediationResult {
resolved, targetInfo, err := resolveExistingFixPath(filePath, effectiveFixRoots(fixHtaccessAllowedRoots))
if err != nil {
return RemediationResult{Error: err.Error()}
}
if !targetInfo.Mode().IsRegular() {
return RemediationResult{Error: "virtual-patch target is not a regular file"}
}
name := filepath.Base(resolved)
dir := filepath.Dir(resolved)
// A repository directory is denied whole: denying only the marker file
// would leave objects, refs and config downloadable.
dirDeny := isKnownBackupPluginDir(dir) || isRepoMetadataDir(dir)
if !dirDeny {
if nameErr := validDenyName(name); nameErr != nil {
return RemediationResult{Error: nameErr.Error()}
}
}
block := buildDenyBlock(name, dirDeny)
reverted, err := applyHtaccessDeny(dir, block)
primaryChanged := err == nil
if err != nil && !errors.Is(err, errVirtualPatchAlreadyApplied) {
return RemediationResult{Error: err.Error()}
}
target := name
if dirDeny {
target = filepath.Base(dir) + "/ (whole directory)"
}
parentChanged := false
parentNote := ""
if dirDeny && isKnownBackupPluginDir(dir) {
parentChanged, parentNote = denyArchiveExtensionInParent(dir)
}
if !primaryChanged && !parentChanged && parentNote == "" {
return RemediationResult{Error: virtualPatchAlreadyApplied}
}
verb := "Wrote"
if !primaryChanged {
verb = "Kept existing"
}
description := fmt.Sprintf("%s Require all denied for %s in %s", verb, target, filepath.Join(dir, ".htaccess"))
if parentNote != "" {
description += "; " + parentNote
}
return RemediationResult{
Success: true,
Action: fmt.Sprintf("denied HTTP access to %s", target),
Description: description,
Reverted: reverted,
}
}
// applyHtaccessDeny appends block to the .htaccess in dir unless the complete
// block is already present, keeping a restorable pre-patch copy. It reports
// whether an earlier CSM block had been removed or damaged in that file.
func applyHtaccessDeny(dir string, block []byte) (bool, error) {
htaccess := filepath.Join(dir, ".htaccess")
if _, err := sanitizeFixPath(htaccess, effectiveFixRoots(fixHtaccessAllowedRoots)); err != nil {
return false, err
}
state, err := readHtaccessState(htaccess, dir)
if err != nil {
return false, err
}
if state.existed && containsHtaccessBlock(state.content, block) {
return false, errVirtualPatchAlreadyApplied
}
reverted := priorPrePatchBackupExists(htaccess, block)
newContent := patchedHtaccessContent(state.content, state.existed, block)
if len(newContent) > maxVirtualPatchHtaccessSize {
return false, fmt.Errorf("refusing patched .htaccess larger than %d bytes", maxVirtualPatchHtaccessSize)
}
backup, created, err := backupHtaccessBeforePatch(htaccess, state, newContent)
if err != nil {
return false, err
}
keepBackup := !created
defer func() {
if !keepBackup {
backup.remove()
}
}()
tmp, tempState, err := writeVirtualPatchTemp(dir, newContent, state)
if err != nil {
return false, err
}
if virtualPatchBeforeCommitForTest != nil {
virtualPatchBeforeCommitForTest(htaccess, tmp)
}
if err := commitVirtualPatchTemp(tmp, htaccess, state, tempState); err != nil {
removeVirtualPatchTemp(tmp, tempState)
if errors.Is(err, errVirtualPatchRollbackIncomplete) {
keepBackup = true
}
return false, err
}
keepBackup = true
if err := syncVirtualPatchDirectory(dir); err != nil {
return false, fmt.Errorf("virtual-patch installed but directory sync failed; backup retained at %s: %w", backup.itemPath, err)
}
return reverted, nil
}
// backupPluginArchiveExt maps a backup-plugin directory to the archive
// extension that can safely be denied from the parent directory. Plugins
// writing .zip or .gz are absent on purpose: wp-content serves those
// legitimately, so a parent-level deny would break working downloads. The
// plugin-directory deny still covers them.
var backupPluginArchiveExt = map[string]string{
"ai1wm-backups": ".wpress",
}
// denyArchiveExtensionInParent writes the durable half of the protection into
// the parent directory, which the plugin does not own. It never fails the
// caller: the plugin-directory deny is the immediate protection and must not
// be given up because the parent could not be written.
func denyArchiveExtensionInParent(dir string) (bool, string) {
ext, ok := backupPluginArchiveExt[strings.ToLower(filepath.Base(dir))]
if !ok {
return false, ""
}
parent := filepath.Dir(dir)
reverted, err := applyHtaccessDeny(parent, buildParentExtensionDenyBlock(ext))
switch {
case err == nil:
if reverted {
return true, fmt.Sprintf("re-applied the durable %s deny under %s", ext, parent)
}
return true, fmt.Sprintf("also denied %s under %s", ext, parent)
case errors.Is(err, errVirtualPatchAlreadyApplied):
return false, ""
default:
return false, fmt.Sprintf("durable %s deny in %s could not be written: %v", ext, filepath.Join(parent, ".htaccess"), err)
}
}
func buildParentExtensionDenyBlock(ext string) []byte {
return []byte(fmt.Sprintf("%s %s\n<FilesMatch \"\\%s$\">\nRequire all denied\n</FilesMatch>\n%s %s\n",
vpBeginParent, ext, ext, vpEndParent, ext))
}
func patchedHtaccessContent(content []byte, existed bool, block []byte) []byte {
if !existed {
return append([]byte(nil), block...)
}
base := ensureTrailingNewline(append([]byte(nil), content...))
return append(base, block...)
}
// priorPrePatchBackupExists reports whether CSM previously wrote this exact
// block to this .htaccess. Reconstructing the recorded post-patch hash avoids
// treating a first patch for another file in the same directory as a revert.
func priorPrePatchBackupExists(htaccess string, block []byte) bool {
found := false
eachPrePatchBackup(func(meta QuarantineMeta, name string, read func(string) ([]byte, error)) bool {
if meta.OriginalPath != htaccess {
return false
}
archived, err := read(strings.TrimSuffix(name, ".meta"))
if err != nil {
return false
}
var existed bool
switch meta.RestoreAction {
case QuarantineRestoreReplaceIfUnchanged:
existed = true
case QuarantineRestoreRemoveIfUnchanged:
if len(archived) != 0 {
return false
}
default:
return false
}
if meta.ExpectedCurrentSHA256 == virtualPatchSHA256(patchedHtaccessContent(archived, existed, block)) {
found = true
return true
}
return false
})
return found
}
// validDenyName rejects file names that could break out of the quoted
// <Files "..."> argument and inject arbitrary .htaccess directives.
func validDenyName(name string) error {
if strings.TrimSpace(name) == "" {
return fmt.Errorf("empty file name")
}
if strings.ContainsAny(name, "\"\n\r<>\\*?[]${}") || strings.HasPrefix(strings.TrimLeft(name, " \t"), "#") {
return fmt.Errorf("file name %q contains characters unsafe for an .htaccess directive", name)
}
for _, r := range name {
if r < 0x20 || r == 0x7f {
return fmt.Errorf("file name %q contains control characters unsafe for an .htaccess directive", name)
}
}
return nil
}
func buildDenyBlock(name string, dirDeny bool) []byte {
if dirDeny {
return []byte(fmt.Sprintf("%s\n<FilesMatch \"^\">\nRequire all denied\n</FilesMatch>\n%s\n", vpBeginDir, vpEndDir))
}
return []byte(fmt.Sprintf("%s %s\n<Files \"%s\">\nRequire all denied\n</Files>\n%s %s\n",
vpBeginFile, name, name, vpEndFile, name))
}
func isKnownBackupPluginDir(dir string) bool {
base := strings.ToLower(filepath.Base(dir))
switch base {
case "ai1wm-backups", "wpvividbackups", "updraft":
return strings.EqualFold(filepath.Base(filepath.Dir(dir)), "wp-content")
default:
return false
}
}
func containsHtaccessBlock(content, block []byte) bool {
normalized := bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n"))
return bytes.Contains(normalized, block)
}
func ensureTrailingNewline(b []byte) []byte {
if len(b) > 0 && b[len(b)-1] != '\n' {
return append(b, '\n')
}
return b
}
func readHtaccessState(htaccess, dir string) (htaccessState, error) {
info, err := os.Lstat(htaccess)
if err != nil {
if !os.IsNotExist(err) {
return htaccessState{}, fmt.Errorf("inspecting .htaccess: %v", err)
}
dirInfo, statErr := os.Stat(dir)
if statErr != nil {
return htaccessState{}, fmt.Errorf("inspecting target directory: %v", statErr)
}
uid, gid, ownerErr := ownerFromInfo(dirInfo)
if ownerErr != nil {
return htaccessState{}, ownerErr
}
return htaccessState{uid: uid, gid: gid, mode: 0644}, nil
}
if info.Mode()&os.ModeSymlink != 0 {
return htaccessState{}, fmt.Errorf("refusing symlinked .htaccess: %s", htaccess)
}
if !info.Mode().IsRegular() {
return htaccessState{}, fmt.Errorf("refusing non-regular .htaccess: %s", htaccess)
}
if info.Size() > maxVirtualPatchHtaccessSize {
return htaccessState{}, fmt.Errorf("refusing .htaccess larger than %d bytes", maxVirtualPatchHtaccessSize)
}
// #nosec G304 -- htaccess is under the resolved and validated target dir;
// O_NOFOLLOW rejects a path swap to a symlink.
file, err := os.OpenFile(htaccess, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
if err != nil {
return htaccessState{}, fmt.Errorf("opening .htaccess: %v", err)
}
defer file.Close()
openedInfo, err := file.Stat()
if err != nil {
return htaccessState{}, fmt.Errorf("stating .htaccess: %v", err)
}
if !os.SameFile(info, openedInfo) {
return htaccessState{}, fmt.Errorf(".htaccess changed while preparing virtual-patch")
}
content, err := io.ReadAll(io.LimitReader(file, maxVirtualPatchHtaccessSize+1))
if err != nil {
return htaccessState{}, fmt.Errorf("reading .htaccess: %v", err)
}
if len(content) > maxVirtualPatchHtaccessSize {
return htaccessState{}, fmt.Errorf("refusing .htaccess larger than %d bytes", maxVirtualPatchHtaccessSize)
}
uid, gid, err := ownerFromInfo(openedInfo)
if err != nil {
return htaccessState{}, err
}
return htaccessState{
content: content,
info: openedInfo,
existed: true,
uid: uid,
gid: gid,
mode: openedInfo.Mode().Perm(),
}, nil
}
func ownerFromInfo(info os.FileInfo) (int, int, error) {
st, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return 0, 0, fmt.Errorf("cannot determine filesystem owner")
}
return int(st.Uid), int(st.Gid), nil
}
func writeVirtualPatchTemp(dir string, content []byte, state htaccessState) (string, htaccessState, error) {
tmp, err := os.CreateTemp(dir, ".htaccess.csm-vpatch-*")
if err != nil {
return "", htaccessState{}, fmt.Errorf("creating temporary .htaccess: %v", err)
}
tmpPath := tmp.Name()
remove := true
defer func() {
_ = tmp.Close()
if remove {
_ = os.Remove(tmpPath)
}
}()
if _, writeErr := tmp.Write(content); writeErr != nil {
return "", htaccessState{}, fmt.Errorf("writing temporary .htaccess: %v", writeErr)
}
if syncErr := tmp.Sync(); syncErr != nil {
return "", htaccessState{}, fmt.Errorf("syncing temporary .htaccess: %v", syncErr)
}
if chownErr := chownFunc(tmp, state.uid, state.gid); chownErr != nil {
return "", htaccessState{}, fmt.Errorf("setting owner on temporary .htaccess: %v", chownErr)
}
if chmodErr := tmp.Chmod(state.mode); chmodErr != nil {
return "", htaccessState{}, fmt.Errorf("setting mode on temporary .htaccess: %v", chmodErr)
}
if syncErr := tmp.Sync(); syncErr != nil {
return "", htaccessState{}, fmt.Errorf("syncing temporary .htaccess metadata: %v", syncErr)
}
tmpInfo, err := tmp.Stat()
if err != nil {
return "", htaccessState{}, fmt.Errorf("stating temporary .htaccess: %v", err)
}
if err := tmp.Close(); err != nil {
return "", htaccessState{}, fmt.Errorf("closing temporary .htaccess: %v", err)
}
remove = false
return tmpPath, htaccessState{
content: append([]byte(nil), content...),
info: tmpInfo,
existed: true,
uid: state.uid,
gid: state.gid,
mode: state.mode,
}, nil
}
func htaccessStateMatchesPath(path string, state htaccessState) error {
info, err := os.Lstat(path)
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return fmt.Errorf(".htaccess path is no longer a regular file")
}
if state.info == nil || !os.SameFile(state.info, info) {
return fmt.Errorf(".htaccess inode changed")
}
// #nosec G304 -- path is either the validated .htaccess or its randomized
// same-directory staging name; O_NOFOLLOW and the repeated inode check keep
// a path swap from redirecting the read.
file, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
if err != nil {
return err
}
defer file.Close()
openedInfo, err := file.Stat()
if err != nil {
return err
}
if !os.SameFile(info, openedInfo) || !os.SameFile(state.info, openedInfo) {
return fmt.Errorf(".htaccess inode changed")
}
uid, gid, err := ownerFromInfo(openedInfo)
if err != nil {
return err
}
if uid != state.uid || gid != state.gid || openedInfo.Mode().Perm() != state.mode.Perm() {
return fmt.Errorf(".htaccess ownership or mode changed")
}
content, err := io.ReadAll(io.LimitReader(file, maxVirtualPatchHtaccessSize+1))
if err != nil {
return err
}
if len(content) > maxVirtualPatchHtaccessSize {
return fmt.Errorf(".htaccess content exceeds validation limit")
}
if !bytes.Equal(content, state.content) {
return fmt.Errorf(".htaccess content changed")
}
return nil
}
func removeVirtualPatchTemp(path string, state htaccessState) {
if htaccessStateMatchesPath(path, state) == nil {
_ = os.Remove(path)
}
}
// eachPrePatchBackup calls fn for every archived pre-patch backup, stopping
// when fn returns true. Reads go through an os.Root scoped to the backup
// directory, so a swapped entry cannot redirect them outside it.
func eachPrePatchBackup(fn func(meta QuarantineMeta, name string, read func(string) ([]byte, error)) bool) {
root, err := os.OpenRoot(htaccessBackupDirRoot)
if err != nil {
return
}
defer func() { _ = root.Close() }()
entries, err := fs.ReadDir(root.FS(), ".")
if err != nil {
return
}
read := func(name string) ([]byte, error) {
pathInfo, lstatErr := root.Lstat(name)
if lstatErr != nil {
return nil, lstatErr
}
if !pathInfo.Mode().IsRegular() {
return nil, fmt.Errorf("pre-patch backup entry is not a regular file")
}
file, openErr := root.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK, 0)
if openErr != nil {
return nil, openErr
}
defer func() { _ = file.Close() }()
info, statErr := file.Stat()
if statErr != nil {
return nil, statErr
}
if !info.Mode().IsRegular() || !os.SameFile(pathInfo, info) {
return nil, fmt.Errorf("pre-patch backup entry changed while opening")
}
data, readErr := io.ReadAll(io.LimitReader(file, maxVirtualPatchHtaccessSize+1))
if readErr != nil {
return nil, readErr
}
if len(data) > maxVirtualPatchHtaccessSize {
return nil, fmt.Errorf("pre-patch backup entry exceeds %d bytes", maxVirtualPatchHtaccessSize)
}
currentInfo, lstatErr := root.Lstat(name)
if lstatErr != nil || !currentInfo.Mode().IsRegular() || !os.SameFile(pathInfo, currentInfo) {
return nil, fmt.Errorf("pre-patch backup entry changed while reading")
}
return data, nil
}
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".meta") {
continue
}
metaData, readErr := read(entry.Name())
if readErr != nil {
continue
}
var meta QuarantineMeta
if json.Unmarshal(metaData, &meta) != nil {
continue
}
if fn(meta, entry.Name(), read) {
return
}
}
}
// findExistingPrePatchBackup returns an archived pre-patch backup for this
// .htaccess whose stored content and expected post-patch hash both match what
// this patch would produce. Both must match, so a backup recorded for a
// different patch is never reused as this one's rollback point.
func findExistingPrePatchBackup(htaccess string, state htaccessState, patched []byte) (virtualPatchBackup, bool) {
wantPatched := virtualPatchSHA256(patched)
// Rolling back a file CSM created means deleting it; rolling back one it
// appended to means restoring the original bytes. An empty .htaccess and
// no .htaccess hold the same content, so without this the reused backup
// could delete a file the customer owns.
wantAction := QuarantineRestoreReplaceIfUnchanged
if !state.existed {
wantAction = QuarantineRestoreRemoveIfUnchanged
}
var found virtualPatchBackup
ok := false
eachPrePatchBackup(func(meta QuarantineMeta, name string, read func(string) ([]byte, error)) bool {
if meta.OriginalPath != htaccess ||
meta.ExpectedCurrentSHA256 != wantPatched ||
meta.RestoreAction != wantAction ||
meta.Owner != state.uid ||
meta.Group != state.gid ||
meta.Mode != state.mode.String() ||
(state.existed && !meta.OriginalModTime.Equal(state.info.ModTime())) ||
meta.Size != int64(len(state.content)) {
return false
}
itemName := strings.TrimSuffix(name, ".meta")
archived, readErr := read(itemName)
if readErr != nil || !bytes.Equal(archived, state.content) {
return false
}
found = virtualPatchBackup{
itemPath: filepath.Join(htaccessBackupDirRoot, itemName),
metaPath: filepath.Join(htaccessBackupDirRoot, name),
}
ok = true
return true
})
return found, ok
}
// backupHtaccessBeforePatch records both the pre-patch content and the exact
// expected post-patch hash. Restore can then replace or remove .htaccess only
// while it still matches the version CSM wrote, preserving later user edits.
// The bool reports whether this call created the backup, so a failed patch
// never removes an archived copy shared with an earlier successful patch.
func backupHtaccessBeforePatch(htaccess string, state htaccessState, patched []byte) (virtualPatchBackup, bool, error) {
if err := quarantinefs.EnsureDir(htaccessBackupDirRoot, 0750); err != nil {
return virtualPatchBackup{}, false, fmt.Errorf("creating backup dir: %v", err)
}
// Reuse only an identical recovery state. Equal bytes with different
// attributes need a new backup so restore keeps the captured metadata.
if existing, found := findExistingPrePatchBackup(htaccess, state, patched); found {
for _, path := range []string{existing.itemPath, existing.metaPath} {
if err := quarantinefs.SyncFilePath(path); err != nil {
return virtualPatchBackup{}, false, fmt.Errorf("syncing existing backup: %w", err)
}
}
if err := quarantinefs.SyncDir(htaccessBackupDirRoot); err != nil {
return virtualPatchBackup{}, false, err
}
return existing, false, nil
}
stamp := time.Now().UTC().Format("20060102T150405Z")
pathSum := sha256.Sum256([]byte(htaccess))
backupFile, err := os.CreateTemp(htaccessBackupDirRoot, fmt.Sprintf("%s_vpatch_%x_", stamp, pathSum[:6]))
if err != nil {
return virtualPatchBackup{}, false, fmt.Errorf("creating backup: %v", err)
}
backup := virtualPatchBackup{itemPath: backupFile.Name(), metaPath: backupFile.Name() + ".meta"}
keep := false
defer func() {
_ = backupFile.Close()
if !keep {
backup.remove()
}
}()
if state.existed {
if _, writeErr := backupFile.Write(state.content); writeErr != nil {
return virtualPatchBackup{}, false, fmt.Errorf("writing backup: %v", writeErr)
}
}
if chmodErr := backupFile.Chmod(0640); chmodErr != nil { // #nosec G302 -- backups permit administrative group reads inside the private quarantine directory.
return virtualPatchBackup{}, false, fmt.Errorf("setting backup mode: %v", chmodErr)
}
if syncErr := backupFile.Sync(); syncErr != nil {
return virtualPatchBackup{}, false, fmt.Errorf("syncing backup: %v", syncErr)
}
if closeErr := backupFile.Close(); closeErr != nil {
return virtualPatchBackup{}, false, fmt.Errorf("closing backup: %v", closeErr)
}
restoreAction := QuarantineRestoreReplaceIfUnchanged
if !state.existed {
restoreAction = QuarantineRestoreRemoveIfUnchanged
}
meta := QuarantineMeta{
OriginalPath: htaccess,
Owner: state.uid,
Group: state.gid,
Mode: state.mode.String(),
Size: int64(len(state.content)),
QuarantineAt: time.Now().UTC(),
Reason: "exposed-file virtual-patch: pre-patch .htaccess backup",
RestoreAction: restoreAction,
ExpectedCurrentSHA256: virtualPatchSHA256(patched),
}
if state.existed {
meta.OriginalModTime = state.info.ModTime().UTC()
}
metaJSON, err := json.Marshal(meta)
if err != nil {
return virtualPatchBackup{}, false, fmt.Errorf("encoding backup meta: %v", err)
}
metaFile, err := os.OpenFile(backup.metaPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
if err != nil {
return virtualPatchBackup{}, false, fmt.Errorf("creating backup meta: %v", err)
}
if _, err := metaFile.Write(metaJSON); err != nil {
_ = metaFile.Close()
return virtualPatchBackup{}, false, fmt.Errorf("writing backup meta: %v", err)
}
if err := metaFile.Sync(); err != nil {
_ = metaFile.Close()
return virtualPatchBackup{}, false, fmt.Errorf("syncing backup meta: %v", err)
}
if err := metaFile.Close(); err != nil {
return virtualPatchBackup{}, false, fmt.Errorf("closing backup meta: %v", err)
}
if err := quarantinefs.SyncDir(htaccessBackupDirRoot); err != nil {
return virtualPatchBackup{}, false, fmt.Errorf("syncing backup directory: %w", err)
}
keep = true
return backup, true, nil
}
func (backup virtualPatchBackup) remove() {
if backup.metaPath != "" {
_ = os.Remove(backup.metaPath)
}
if backup.itemPath != "" {
_ = os.Remove(backup.itemPath)
}
}
func virtualPatchSHA256(content []byte) string {
sum := sha256.Sum256(content)
return fmt.Sprintf("sha256:%x", sum[:])
}
func parseVirtualPatchMode(value string) (os.FileMode, error) {
if len(value) != 10 || value[0] != '-' {
return 0, fmt.Errorf("invalid virtual-patch mode %q", value)
}
perms := value[1:]
wantChars := "rwxrwxrwx"
bits := []os.FileMode{0400, 0200, 0100, 0040, 0020, 0010, 0004, 0002, 0001}
var mode os.FileMode
for i, char := range perms {
if char == '-' {
continue
}
if char != rune(wantChars[i]) {
return 0, fmt.Errorf("invalid virtual-patch mode %q", value)
}
mode |= bits[i]
}
return mode, nil
}
// VirtualPatchExposedFindings applies (apply=true) or previews (apply=false) a
// deny rule for each virtual-patchable web_exposed_* finding, deduplicated by
// path. It returns one auto_response action finding per file.
func VirtualPatchExposedFindings(_ *config.Config, findings []alert.Finding, apply bool) []alert.Finding {
var actions []alert.Finding
seen := make(map[string]struct{})
for _, f := range findings {
if !isVirtualPatchableExposedCheck(f.Check) {
continue
}
path := f.FilePath
if path == "" {
path = extractFilePath(f.Message)
}
if path == "" {
continue
}
if _, ok := seen[path]; ok {
continue
}
seen[path] = struct{}{}
if !apply {
details := "Enable auto_response.virtual_patch_exposed_files=auto with dry_run:false, or run `csm virtual-patch --apply`, to write the deny rule."
if f.Check == "web_exposed_sample_sql" {
details = "Warning-only sample SQL is never enforced automatically; run `csm virtual-patch --apply` to write the deny rule."
}
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("VIRTUAL-PATCH (preview): would deny HTTP access to %s", path),
Details: details,
Timestamp: time.Now(),
})
continue
}
res := VirtualPatchExposedFile(path)
switch {
case res.Success && res.Reverted:
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("%s %s", virtualPatchRevertedPrefix, path),
Details: fmt.Sprintf("%s. CSM had already denied this path; something rewrote the .htaccess and the file was downloadable until this scan. "+
"Backup plugins own the .htaccess in their own directory and restore it on every run, so the deny belongs somewhere the plugin does not overwrite.",
res.Description),
Timestamp: time.Now(),
})
case res.Success:
actions = append(actions, alert.Finding{
Severity: alert.Critical,
Check: "auto_response",
Message: fmt.Sprintf("VIRTUAL-PATCH: denied HTTP access to %s", path),
Details: res.Description,
Timestamp: time.Now(),
})
case res.Error != "" && res.Error != virtualPatchAlreadyApplied:
actions = append(actions, alert.Finding{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("VIRTUAL-PATCH failed: %s", path),
Details: res.Error,
Timestamp: time.Now(),
})
}
}
return actions
}
// AutoVirtualPatchExposedFiles is the scan-time entry point. It acts only when
// auto_response is enabled and the mode is "auto"; the write is gated by the
// shared auto_response dry_run flag (dry_run reports the intended denials).
func AutoVirtualPatchExposedFiles(cfg *config.Config, findings []alert.Finding) []alert.Finding {
if cfg == nil || !cfg.AutoResponse.Enabled || cfg.VirtualPatchMode() != config.VirtualPatchAuto {
return nil
}
autoEligible := make([]alert.Finding, 0, len(findings))
for _, finding := range findings {
if finding.Check != "web_exposed_sample_sql" {
autoEligible = append(autoEligible, finding)
}
}
return VirtualPatchExposedFindings(cfg, autoEligible, !cfg.AutoResponseDryRunEnabled())
}
//go:build linux
package checks
import (
"errors"
"fmt"
"os"
"golang.org/x/sys/unix"
)
func commitVirtualPatchTemp(tmp, htaccess string, state, tempState htaccessState) error {
if !state.existed {
if err := unix.Renameat2(unix.AT_FDCWD, tmp, unix.AT_FDCWD, htaccess, unix.RENAME_NOREPLACE); err != nil {
if errors.Is(err, unix.EEXIST) {
return fmt.Errorf(".htaccess changed while preparing virtual-patch")
}
return fmt.Errorf("committing new .htaccess: %v", err)
}
if err := htaccessStateMatchesPath(htaccess, tempState); err != nil {
if rollbackErr := unix.Renameat2(unix.AT_FDCWD, htaccess, unix.AT_FDCWD, tmp, unix.RENAME_NOREPLACE); rollbackErr != nil {
return fmt.Errorf("%w: temporary .htaccess changed and rollback failed: %v", errVirtualPatchRollbackIncomplete, rollbackErr)
}
return fmt.Errorf("temporary .htaccess changed before commit: %v", err)
}
return nil
}
if err := unix.Renameat2(unix.AT_FDCWD, tmp, unix.AT_FDCWD, htaccess, unix.RENAME_EXCHANGE); err != nil {
return fmt.Errorf("atomically exchanging .htaccess: %v", err)
}
oldErr := htaccessStateMatchesPath(tmp, state)
newErr := htaccessStateMatchesPath(htaccess, tempState)
if oldErr != nil || newErr != nil {
if rollbackErr := unix.Renameat2(unix.AT_FDCWD, tmp, unix.AT_FDCWD, htaccess, unix.RENAME_EXCHANGE); rollbackErr != nil {
return fmt.Errorf("%w: .htaccess changed while preparing virtual-patch and rollback failed: %v", errVirtualPatchRollbackIncomplete, rollbackErr)
}
if oldErr != nil {
return fmt.Errorf(".htaccess changed while preparing virtual-patch: %v", oldErr)
}
return fmt.Errorf("temporary .htaccess changed before commit: %v", newErr)
}
if err := os.Remove(tmp); err != nil {
if rollbackErr := unix.Renameat2(unix.AT_FDCWD, tmp, unix.AT_FDCWD, htaccess, unix.RENAME_EXCHANGE); rollbackErr != nil {
return fmt.Errorf("%w: removing replaced .htaccess failed (%v) and rollback failed: %v", errVirtualPatchRollbackIncomplete, err, rollbackErr)
}
_ = os.Remove(tmp)
return fmt.Errorf("removing replaced .htaccess: %v", err)
}
return nil
}
package checks
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"syscall"
"github.com/pidginhost/csm/internal/safepath"
)
// RestoreVirtualPatchBackup reverts only the captured patch, using the pinned
// destination directory so tenant renames cannot redirect reads or changes.
func RestoreVirtualPatchBackup(backupPath string, target *safepath.Target, meta QuarantineMeta) error {
if target.Name != ".htaccess" {
return fmt.Errorf("virtual-patch restore applies only to .htaccess")
}
if meta.RestoreAction != QuarantineRestoreReplaceIfUnchanged &&
meta.RestoreAction != QuarantineRestoreRemoveIfUnchanged {
return fmt.Errorf("unsupported virtual-patch restore action %q", meta.RestoreAction)
}
mode, err := parseVirtualPatchMode(meta.Mode)
if err != nil {
return err
}
state, err := readRestoreHtaccess(target.Parent, target.Name)
if err != nil {
return fmt.Errorf("%w: %v", ErrVirtualPatchRestoreConflict, err)
}
if virtualPatchSHA256(state.content) != meta.ExpectedCurrentSHA256 ||
state.uid != meta.Owner || state.gid != meta.Group || state.mode.Perm() != mode.Perm() {
return fmt.Errorf("%w: live file was modified after enforcement", ErrVirtualPatchRestoreConflict)
}
var content []byte
if meta.RestoreAction == QuarantineRestoreReplaceIfUnchanged {
// Quarantine is daemon-owned; the final backup name is still opened
// without following a symlink and validated before reading.
backup, openErr := os.OpenFile(backupPath, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0) // #nosec G304 -- caller supplies a daemon-owned quarantine entry; no-follow and regular-file checks protect the read
if openErr != nil {
return fmt.Errorf("opening virtual-patch backup: %w", openErr)
}
content, err = readRestoreContent(backup)
closeErr := backup.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
}
if opErr := target.Check(); opErr != nil {
return fmt.Errorf("%w: %v", ErrVirtualPatchRestoreConflict, opErr)
}
stage, stageName, err := target.Parent.CreatePrivateTemp()
if err != nil {
return err
}
defer func() { _ = stage.Close() }()
keep := false
defer func() {
if !keep {
_ = target.Parent.RemoveDir(stageName)
}
}()
temp, err := stage.CreateTemp()
if err != nil {
return err
}
name := filepath.Base(temp.Name())
defer func() {
_ = temp.Close()
if !keep {
_ = stage.Remove(name)
}
}()
if _, opErr := temp.Write(content); opErr != nil {
return opErr
}
if opErr := temp.Chmod(mode); opErr != nil {
return opErr
}
if opErr := temp.Chown(meta.Owner, meta.Group); opErr != nil {
return opErr
}
if !meta.OriginalModTime.IsZero() {
if opErr := safepath.SetModTime(temp, meta.OriginalModTime); opErr != nil {
return opErr
}
}
if opErr := temp.Sync(); opErr != nil {
return opErr
}
prepared, err := temp.Stat()
if err != nil {
return err
}
if opErr := temp.Close(); opErr != nil {
return opErr
}
if opErr := target.Check(); opErr != nil {
return fmt.Errorf("%w: %v", ErrVirtualPatchRestoreConflict, opErr)
}
remove := meta.RestoreAction == QuarantineRestoreRemoveIfUnchanged
if remove {
if opErr := stage.Remove(name); opErr != nil {
return opErr
}
// Isolate the old file without leaving a placeholder whose later
// unlink could delete a concurrent replacement at the live name.
err = target.Parent.RenameTo(target.Name, stage, name)
} else {
err = stage.ExchangeTo(name, target.Parent, target.Name)
}
if err != nil {
return fmt.Errorf("%w: %v", ErrVirtualPatchRestoreConflict, err)
}
if virtualPatchRestoreAfterMoveForTest != nil {
virtualPatchRestoreAfterMoveForTest()
}
conflict := func(cause error) error {
keep = true
return fmt.Errorf("%w: %v; recovery files retained in %s", ErrVirtualPatchRestoreConflict, cause, stageName)
}
rollback := func(cause error) error {
keep = true
if remove {
if opErr := stage.RenameTo(name, target.Parent, target.Name); opErr != nil {
return conflict(fmt.Errorf("%v; rollback failed: %w", cause, opErr))
}
keep = false
return fmt.Errorf("%w: %v", ErrVirtualPatchRestoreConflict, cause)
}
// Capture the live name before deciding what to put back. A second
// exchange after a check could overwrite another intervening edit.
captured, opErr := stage.CreateTemp()
if opErr != nil {
return conflict(opErr)
}
captureName := filepath.Base(captured.Name())
if opErr := captured.Close(); opErr != nil {
return conflict(opErr)
}
if opErr := stage.Remove(captureName); opErr != nil {
return conflict(opErr)
}
captureErr := target.Parent.RenameTo(target.Name, stage, captureName)
if captureErr != nil && !os.IsNotExist(captureErr) {
return conflict(fmt.Errorf("%v; rollback failed: %w", cause, captureErr))
}
restoreName := name
if captureErr == nil {
current, readErr := readRestoreHtaccess(stage, captureName)
if readErr != nil || !os.SameFile(prepared, current.info) ||
!bytes.Equal(content, current.content) || current.uid != meta.Owner ||
current.gid != meta.Group || current.mode != mode.Perm() {
restoreName = captureName
}
}
if opErr := stage.RenameTo(restoreName, target.Parent, target.Name); opErr != nil {
return conflict(fmt.Errorf("%v; rollback failed: %w", cause, opErr))
}
if captureErr != nil {
keep = false
return fmt.Errorf("%w: %v", ErrVirtualPatchRestoreConflict, cause)
}
// Keep every captured inode on conflict, including one that looked
// unchanged: a writer may still hold an open descriptor to it.
return conflict(cause)
}
oldState, err := readRestoreHtaccess(stage, name)
if err != nil {
return rollback(err)
}
if !os.SameFile(state.info, oldState.info) || !bytes.Equal(state.content, oldState.content) ||
state.uid != oldState.uid || state.gid != oldState.gid || state.mode != oldState.mode {
return rollback(fmt.Errorf("live file changed during restore"))
}
if remove {
if _, statErr := target.Parent.Stat(target.Name); !os.IsNotExist(statErr) {
return conflict(fmt.Errorf("live file was recreated during restore"))
}
} else {
current, readErr := readRestoreHtaccess(target.Parent, target.Name)
if readErr != nil {
return rollback(readErr)
}
if !os.SameFile(prepared, current.info) || !bytes.Equal(content, current.content) ||
current.uid != meta.Owner || current.gid != meta.Group || current.mode != mode.Perm() {
return conflict(fmt.Errorf("prepared restore changed"))
}
}
if opErr := target.Check(); opErr != nil {
return rollback(opErr)
}
if opErr := target.Parent.Sync(); opErr != nil {
keep = true
return fmt.Errorf("restore applied but destination sync failed; recovery files retained in %s: %w", stageName, opErr)
}
if opErr := stage.Remove(name); opErr != nil {
keep = true
return fmt.Errorf("restore applied but replaced file could not be removed from %s: %w", stageName, opErr)
}
return nil
}
var virtualPatchRestoreAfterMoveForTest func()
func readRestoreContent(file *os.File) ([]byte, error) {
info, err := file.Stat()
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() || info.Size() > maxVirtualPatchHtaccessSize {
return nil, fmt.Errorf("restore input must be a bounded regular file")
}
content, err := io.ReadAll(io.LimitReader(file, maxVirtualPatchHtaccessSize+1))
if err != nil {
return nil, err
}
if len(content) > maxVirtualPatchHtaccessSize {
return nil, fmt.Errorf("restore input exceeds size limit")
}
return content, nil
}
func readRestoreHtaccess(dir *safepath.Dir, name string) (htaccessState, error) {
file, err := dir.OpenFile(name, os.O_RDONLY, 0)
if err != nil {
return htaccessState{}, err
}
defer file.Close()
content, err := readRestoreContent(file)
if err != nil {
return htaccessState{}, err
}
info, err := file.Stat()
if err != nil {
return htaccessState{}, err
}
uid, gid, err := ownerFromInfo(info)
if err != nil {
return htaccessState{}, err
}
return htaccessState{content: content, info: info, existed: true, uid: uid, gid: gid, mode: info.Mode().Perm()}, nil
}
package checks
import (
"crypto/rand"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"sync"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/atomicio"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
)
const fileResponseStateName = "file-response.json"
// An attempt is reserved durably as failed before touching a customer file.
// Interrupted operations therefore keep both their budget and failure charge.
type fileResponseAttempt struct {
At time.Time `json:"at"`
Account string `json:"account"`
Failed bool `json:"failed"`
ID string `json:"id,omitempty"`
Pending bool `json:"pending,omitempty"`
}
type fileResponseState struct {
Version int `json:"version"`
Attempts []fileResponseAttempt `json:"attempts"`
}
var fileResponseNow = time.Now
var writeFileResponseState = atomicio.AtomicWriteJSON
var activeFileResponses sync.Map
// A refusal leaves the customer file untouched and consumes attempt capacity,
// but does not indicate a failed response mechanism.
var errFileResponseRefused = errors.New("file response refused")
// fileResponseRefusal classifies an error as a refusal for errors.Is while
// keeping the refusing check's message. Manual remediation shares these
// checks, so the breaker's accounting must not show up in operator text.
type fileResponseRefusal struct{ err error }
func (r fileResponseRefusal) Error() string { return r.err.Error() }
func (r fileResponseRefusal) Unwrap() error { return r.err }
func (fileResponseRefusal) Is(target error) bool { return target == errFileResponseRefused }
func refuseFileResponse(err error) error { return fileResponseRefusal{err: err} }
func fileResponseSourceError(err error) error {
// A socket replacement cannot be opened as a file and returns ENXIO
// before the descriptor-based regular-file check can refuse it.
if errors.Is(err, os.ErrNotExist) || errors.Is(err, unix.ELOOP) || errors.Is(err, unix.ENOTDIR) || errors.Is(err, unix.ENXIO) {
return refuseFileResponse(err)
}
return err
}
// runAutoFileResponse is shared by the automatic PHP/access-file cleaners and
// both quarantine entry points. Manual remediation does not enter this gate.
// A bounded process-shared lock protects reservation and outcome updates. The
// file action runs outside the lock, so a slow copy does not stall other slots.
func runAutoFileResponse(cfg *config.Config, path string, info os.FileInfo, apply func() error) *alert.Finding {
now := fileResponseNow()
notice := func(reason, account, detail string) *alert.Finding {
return fileResponseNotice(cfg.StatePath, now, reason, account, detail)
}
if !info.Mode().IsRegular() {
return notice("file_type", "", "Automatic file response refused a directory or special file. Review the detection and use manual remediation for this target.")
}
hostLimit, accountLimit, failureLimit := fileResponseLimits(cfg)
if hostLimit < 1 || accountLimit < 1 || failureLimit < 1 || hostLimit > config.MaxFileResponseLimit || accountLimit > config.MaxFileResponseLimit || failureLimit > config.MaxFileResponseLimit {
return notice("state", "", "Automatic file response is paused because its safety limits are invalid. Correct the configuration; detection continues.")
}
if cfg.StatePath == "" || !filepath.IsAbs(cfg.StatePath) {
return notice("state", "", "Automatic file response is paused because its safety state directory is unavailable. Detection continues.")
}
// The operator-owned state directory is provisioned at daemon startup.
// Never create a new directory here and silently reset a missing state root.
lockPath := filepath.Join(cfg.StatePath, "file-response.lock")
statePath := filepath.Join(cfg.StatePath, fileResponseStateName)
_, account, _ := accountRootOf(path)
attemptID := rand.Text()
admitted := false
reserved := func() *alert.Finding {
lock, err := acquireFileResponseLock(lockPath)
if err != nil {
return notice("busy", "", "Automatic file response refused an attempt while its safety state was busy. Detection continues; review the original finding before retrying manually.")
}
defer lock.Close()
state, err := readFileResponseState(statePath, now)
if err != nil {
csmlog.Warn("automatic file response state unreadable", "err", err)
return notice("state", "", "Automatic file response is paused because its safety state cannot be read. Repair the state storage; detection continues.")
}
// Unknown paths share one budget. Finding text/TenantID must not select a
// fresh account bucket, and UID alone conflates root-owned tenant files.
used, failures, pending := fileResponseUsage(state, account)
if failures >= failureLimit {
return notice("failures", "", "Automatic file response is paused after repeated action failures in the rolling hour. Detection continues; inspect the action log and recovery copies before manual remediation.")
}
if failures+pending >= failureLimit {
return notice("pending", "", "Automatic file response is paused while earlier file actions use the available failure capacity. Detection continues; review the original finding before manual remediation.")
}
if len(state.Attempts) >= hostLimit {
return notice("host_limit", "", "Automatic file response reached its host limit for the rolling hour. Detection continues; review outstanding findings for manual remediation.")
}
if used >= accountLimit {
return notice("account_limit", account, "Automatic file response reached its account limit for the rolling hour. Other accounts remain eligible; detection continues.")
}
state.Attempts = append(state.Attempts, fileResponseAttempt{At: now, Account: account, Failed: true, ID: attemptID, Pending: true})
activeFileResponses.Store(attemptID, struct{}{})
if err = writeFileResponseState(statePath, 0600, state); err != nil {
activeFileResponses.Delete(attemptID)
csmlog.Warn("automatic file response reservation failed", "err", err)
return notice("state", "", "Automatic file response is paused because its safety reservation could not be saved. Detection continues.")
}
admitted = true
return nil
}()
if !admitted {
return reserved
}
defer activeFileResponses.Delete(attemptID)
// The caller's snapshot predates budget persistence. Recheck it after that
// I/O; the descriptor-based quarantine/cleaner checks it again when opening.
current, err := os.Lstat(path)
err = fileResponseSourceError(err)
if err == nil && (!sameFileIdentity(info, current) || !sameContentShape(info, current)) {
err = refuseFileResponse(errors.New("file changed before automatic response"))
}
if err == nil {
err = apply()
}
failed := err != nil && !errors.Is(err, errFileResponseRefused)
refused := errors.Is(err, errFileResponseRefused)
if failed {
csmlog.Warn("automatic file response failed", "path", path, "err", err)
} else if refused {
csmlog.Warn("automatic file response refused", "path", path, "err", err)
}
failures, err := finishFileResponseAttempt(lockPath, statePath, attemptID, now, failed)
if err != nil {
// The durable reservation remains charged even when outcome persistence
// fails. Never retry the action or refund its slot based on this error.
csmlog.Warn("automatic file response outcome save failed", "err", err)
return notice("state", "", "An automatic file response completed but its safety outcome could not be saved. Its reservation remains charged; inspect storage and recovery evidence.")
}
if failed && failures >= failureLimit {
return notice("failures", "", "Automatic file response is paused after repeated action failures in the rolling hour. Detection continues; inspect the action log and recovery copies before manual remediation.")
}
return nil
}
func acquireFileResponseLock(path string) (*os.File, error) {
// #nosec G304 -- fixed filename inside the operator-configured state directory.
lock, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0600)
if err != nil {
return nil, err
}
info, err := lock.Stat()
if err != nil || !info.Mode().IsRegular() {
_ = lock.Close()
return nil, errors.New("safety lock is not a regular file")
}
// #nosec G115 -- an open POSIX file descriptor fits in int.
if err := waitForFileResponseLock(int(lock.Fd()), 5*time.Second); err != nil {
_ = lock.Close()
return nil, err
}
return lock, nil
}
func fileResponseUsage(state *fileResponseState, account string) (used, failures, pending int) {
for _, attempt := range state.Attempts {
if attempt.Account == account {
used++
}
if !attempt.Failed {
continue
}
if attempt.Pending {
if _, active := activeFileResponses.Load(attempt.ID); active {
pending++
continue
}
}
failures++
}
return used, failures, pending
}
func finishFileResponseAttempt(lockPath, statePath, attemptID string, now time.Time, failed bool) (int, error) {
lock, err := acquireFileResponseLock(lockPath)
if err != nil {
return 0, err
}
defer lock.Close()
state, err := readFileResponseState(statePath, now)
if err != nil {
return 0, err
}
found := false
for i := range state.Attempts {
if state.Attempts[i].ID != attemptID {
continue
}
if found {
return 0, errors.New("duplicate safety reservation")
}
found = true
state.Attempts[i].Pending = false
state.Attempts[i].Failed = failed
}
if !found {
return 0, errors.New("safety reservation disappeared")
}
if err := writeFileResponseState(statePath, 0600, state); err != nil {
return 0, err
}
_, failures, _ := fileResponseUsage(state, "")
return failures, nil
}
func waitForFileResponseLock(fd int, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for {
err := unix.Flock(fd, unix.LOCK_EX|unix.LOCK_NB)
if err == nil {
return nil
}
if !errors.Is(err, unix.EINTR) && !errors.Is(err, unix.EWOULDBLOCK) {
return err
}
if !time.Now().Before(deadline) {
return err
}
time.Sleep(10 * time.Millisecond)
}
}
func fileResponseLimits(cfg *config.Config) (host, account, failures int) {
host, account, failures = cfg.AutoResponse.MaxFileActionsPerHour, cfg.AutoResponse.MaxFileActionsPerAccountPerHour, cfg.AutoResponse.MaxFileActionFailuresPerHour
if host == 0 {
host = config.DefaultMaxFileActionsPerHour
}
if account == 0 {
account = config.DefaultMaxFileActionsPerAccountPerHour
}
if failures == 0 {
failures = config.DefaultMaxFileActionFailuresPerHour
}
return
}
func readFileResponseState(path string, now time.Time) (*fileResponseState, error) {
// #nosec G304 -- fixed filename inside the operator-owned state directory.
file, err := os.OpenFile(path, os.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0)
if errors.Is(err, os.ErrNotExist) {
return &fileResponseState{Version: 1}, nil
}
if err != nil {
return nil, err
}
defer file.Close()
info, err := file.Stat()
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() {
return nil, errors.New("safety state is not a regular file")
}
// Bound corrupted or externally replaced state before decoding.
const maxSize = 4 << 20
data, err := io.ReadAll(io.LimitReader(file, maxSize+1))
if err != nil {
return nil, err
}
if len(data) > maxSize {
return nil, errors.New("safety state is too large")
}
var stored struct {
Version int `json:"version"`
Attempts json.RawMessage `json:"attempts"`
}
if err := json.Unmarshal(data, &stored); err != nil {
return nil, err
}
if stored.Version != 1 {
return nil, errors.New("unknown safety state format")
}
if len(stored.Attempts) == 0 {
return nil, errors.New("safety state is missing reservations")
}
// Account and Failed have meaningful zero values. Missing or null fields
// must not silently turn a charged reservation into an unknown account
// or a successful action and reopen capacity after state corruption.
var attempts []struct {
At time.Time `json:"at"`
Account *string `json:"account"`
Failed *bool `json:"failed"`
ID string `json:"id"`
Pending bool `json:"pending"`
}
if err := json.Unmarshal(stored.Attempts, &attempts); err != nil {
return nil, err
}
if attempts == nil {
return nil, errors.New("safety state has null reservations")
}
state := &fileResponseState{Version: stored.Version}
for _, attempt := range attempts {
if attempt.At.IsZero() {
return nil, errors.New("safety state has an undated reservation")
}
if attempt.Account == nil || attempt.Failed == nil || attempt.Pending && attempt.ID == "" {
return nil, errors.New("safety state has an incomplete reservation")
}
// Future entries remain charged when the clock moves backwards.
_, active := activeFileResponses.Load(attempt.ID)
if attempt.At.After(now.Add(-time.Hour)) || attempt.Pending && active {
state.Attempts = append(state.Attempts, fileResponseAttempt{At: attempt.At, Account: *attempt.Account, Failed: *attempt.Failed, ID: attempt.ID, Pending: attempt.Pending})
}
}
return state, nil
}
var fileResponseNotices = struct {
sync.Mutex
last map[string]time.Time
}{last: make(map[string]time.Time)}
// Pause notices must not become a second alert flood during a detector or
// storage failure. Dedup is local (restart reports the pause again), bounded,
// and independent of the safety state so storage failures are reportable.
func fileResponseNotice(statePath string, now time.Time, reason, account, detail string) *alert.Finding {
// An account-wide detector fault can exhaust many tenant budgets. Emit
// one host notice per cause, so those warnings cannot spend the alert
// budget that other non-critical detections need.
key := statePath + "\x00" + reason
fileResponseNotices.Lock()
defer fileResponseNotices.Unlock()
if last, ok := fileResponseNotices.last[key]; ok && now.Sub(last) < time.Hour {
return nil
}
for k, last := range fileResponseNotices.last {
if now.Sub(last) >= time.Hour {
delete(fileResponseNotices.last, k)
}
}
if len(fileResponseNotices.last) >= 1024 {
// Do not let arbitrary state paths grow daemon memory without a bound.
// Keep the existing notices suppressed until their window expires.
return nil
}
fileResponseNotices.last[key] = now
message := "Automatic file response paused"
if account != "" {
message = "Automatic file response paused for accounts at their limit"
}
return &alert.Finding{Check: "auto_response_paused", Severity: alert.Warning, Message: message, Details: detail, DedupKey: reason, Timestamp: now}
}
//go:build linux
package checks
import (
"os"
"syscall"
)
// SameFileVersion requires the same observed Linux file version. Ctime closes
// the same-inode, same-size rewrite case where an attacker restores mtime after
// changing content.
func SameFileVersion(a, b os.FileInfo) bool {
if !sameFileIdentity(a, b) || !sameContentShape(a, b) {
return false
}
as, aok := a.Sys().(*syscall.Stat_t)
bs, bok := b.Sys().(*syscall.Stat_t)
return aok && bok && as.Ctim == bs.Ctim
}
package checks
import (
"bufio"
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync/atomic"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// fileIndexScanCount schedules a full walk at startup and every sixth scan.
// Failed or interrupted walks reset it so retries cannot trust cached mtimes
// for directories that have become unreadable.
var fileIndexScanCount int32
// fileIndexShrinkSkips counts consecutive scans whose current index shrank far
// enough to trip the mass-deletion guard. It self-heals the guard: after
// fileIndexShrinkPromoteThreshold consecutive shrinking scans the smaller index
// is promoted as the new baseline, so new-file detection resumes instead of
// forever diffing against a stale, permanently-larger previous index.
// The live CheckFileIndex path is serialized by fileIndexLiveScanGate, so this
// counter advances once per completed stateful index build rather than racing
// between a periodic deep scan and an operator-triggered tier run.
var fileIndexShrinkSkips int32
// fileIndexLiveScanGate serializes the stateful CheckFileIndex path. The
// scanner reads and writes fileindex.current, fileindex.previous, dircache.json,
// and the shrink streak as one logical baseline update; overlapping live runs
// can otherwise double-count one shrink window or compare against a half-written
// peer scan. ForceFileIndex audit scans return before this gate because they do
// not touch live baseline state.
var fileIndexLiveScanGate = make(chan struct{}, 1)
// fileIndexShrinkPromoteThreshold is how many consecutive shrinking scans must
// occur before the smaller index becomes the baseline. Three deep cycles is
// long enough to rule out a one-off read glitch or a transient mass rename,
// while still healing the wedge automatically within a few scan intervals so no
// operator has to delete fileindex.previous by hand.
const fileIndexShrinkPromoteThreshold = 3
// evaluateFileIndexShrink classifies the current-vs-previous index sizes and
// advances the consecutive-shrink counter. isShrink reports whether this scan
// tripped the mass-deletion guard (an empty current against a populated
// previous, or a non-empty current below half the previous). promote reports
// whether the shrink has now persisted for fileIndexShrinkPromoteThreshold
// consecutive scans, in which case the caller must adopt the smaller index as
// the baseline. Empty current indexes are never promoted: a persistent read
// failure has the same shape, and adopting it would make recovery alert on
// every indexed file as if the whole host were new.
// A non-shrink scan resets the streak so only *consecutive* shrinks promote.
func evaluateFileIndexShrink(prevLen, curLen int) (isShrink, promote bool) {
if prevLen > 10 && curLen == 0 {
atomic.StoreInt32(&fileIndexShrinkSkips, 0)
return true, false
}
isShrink = prevLen > 0 && curLen > 0 && curLen*2 < prevLen
if !isShrink {
atomic.StoreInt32(&fileIndexShrinkSkips, 0)
return false, false
}
if atomic.AddInt32(&fileIndexShrinkSkips, 1) >= fileIndexShrinkPromoteThreshold {
atomic.StoreInt32(&fileIndexShrinkSkips, 0)
return true, true
}
return true, false
}
// suspiciousExtensions are file extensions worth reading in a web root. Being
// listed here only routes a file into content analysis; the verdict is still
// the content scanner's. ".phps" earns its place despite a stock handler
// rendering it as source rather than executing it: staging a dropper under that
// extension leaves it unread until a rename makes it live.
var suspiciousExtensions = map[string]bool{
".phtml": true, ".pht": true, ".php5": true, ".phps": true,
".haxor": true, ".cgix": true,
}
// dirMtimeCache maps directory paths to their last-known mtime (unix seconds).
// Directories with unchanged mtime are skipped during scanning.
type dirMtimeCache map[string]int64
func loadDirCache(stateDir string) (dirMtimeCache, error) {
cache := make(dirMtimeCache)
data, err := osFS.ReadFile(filepath.Join(stateDir, "dircache.json"))
if os.IsNotExist(err) {
return cache, nil
}
if err != nil {
return cache, err
}
err = json.Unmarshal(data, &cache)
return cache, err
}
func saveDirCache(stateDir string, cache dirMtimeCache) error {
data, _ := json.Marshal(cache)
tmpPath := filepath.Join(stateDir, "dircache.json.tmp")
if err := os.WriteFile(tmpPath, data, 0600); err != nil {
return err
}
return os.Rename(tmpPath, filepath.Join(stateDir, "dircache.json"))
}
// dirChanged returns true if the directory mtime has changed since last scan.
// Updates the cache with the new mtime. If forceFullScan is true, always
// returns true to force a ReadDir regardless of mtime (catches writes that
// bypass parent mtime updates, e.g. hard links or mount tricks).
func dirChanged(dir string, cache dirMtimeCache, forceFullScan bool) bool {
info, err := osFS.Stat(dir)
if err != nil {
return true // can't stat, scan it to be safe
}
mtime := info.ModTime().Unix()
prev, exists := cache[dir]
cache[dir] = mtime
if forceFullScan {
return true
}
if !exists {
return true // first time seeing this dir
}
return mtime != prev
}
type subtreeChangeTracker struct {
cache dirMtimeCache
changedBelow map[string]struct{}
built bool
}
func newSubtreeChangeTracker(cache dirMtimeCache) *subtreeChangeTracker {
snapshot := make(dirMtimeCache, len(cache))
for dir, mtime := range cache {
snapshot[dir] = mtime
}
return &subtreeChangeTracker{cache: snapshot}
}
func (t *subtreeChangeTracker) hasChangedDir(ctx context.Context, dir string) bool {
if t == nil {
return false
}
if !t.built {
t.build(ctx)
}
_, ok := t.changedBelow[dir]
return ok
}
func (t *subtreeChangeTracker) build(ctx context.Context) {
t.built = true
t.changedBelow = make(map[string]struct{})
for cached, mtime := range t.cache {
if ctx != nil && ctx.Err() != nil {
return
}
info, err := osFS.Stat(cached)
if err == nil && info.ModTime().Unix() == mtime {
continue
}
for parent := filepath.Dir(cached); parent != "." && parent != string(filepath.Separator); parent = filepath.Dir(parent) {
if _, ok := t.cache[parent]; ok {
t.changedBelow[parent] = struct{}{}
}
}
}
}
// CheckFileIndex builds an index of suspicious files using pure Go directory
// reads, diffs against the previous index, and alerts on new files.
// Uses directory mtime caching: unchanged dirs carry forward previous entries
// without calling ReadDir, while changed dirs are re-scanned.
//
// When ctx carries AccountScanOptions with ForceFileIndex=true the function
// runs in audit mode: it enumerates only the in-scope account, bypasses the
// directory mtime cache entirely, and writes none of the live state files
// (fileindex.current, fileindex.previous, dircache.json). The normal incremental
// baseline is left byte-for-byte intact. All indexed files are treated as new
// so the caller receives findings for the full current state of the account.
func CheckFileIndex(ctx context.Context, cfg *config.Config, st *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
// Audit mode: enumerate only the in-scope account, bypass all state I/O.
// CRITICAL: none of fileindex.current, fileindex.previous, or dircache.json
// may be written -- corrupting them would silently break future incremental
// scans by shifting their baseline.
if scanForceFileIndex(ctx) {
// Fresh empty cache so every directory is treated as changed.
auditCache := make(dirMtimeCache)
// Empty prevByDir so every indexed file is diff'd as new.
currentEntries := buildFileIndex(ctx, auditCache, nil, true)
if ctx.Err() != nil {
return nil
}
// All entries are "new" -- diff against empty previous set.
return checkFileIndexAnalyzeNewFiles(ctx, cfg, currentEntries)
}
return fileIndexQueues.run(ctx, func(work *fileIndexWork) []alert.Finding {
return checkFileIndexLive(ctx, cfg, st, work)
})
}
func checkFileIndexLive(ctx context.Context, cfg *config.Config, st *state.Store, work *fileIndexWork) []alert.Finding {
ctx = context.WithValue(ctx, fileIndexWorkKey{}, work)
scanNum := atomic.AddInt32(&fileIndexScanCount, 1)
forceFullScan := scanNum == 1 || scanNum%6 == 0
defer func() {
work.local()
if ctx.Err() != nil {
atomic.StoreInt32(&fileIndexScanCount, 0)
}
}()
indexDir := cfg.StatePath
currentPath := filepath.Join(indexDir, "fileindex.current")
previousPath := filepath.Join(indexDir, "fileindex.previous")
// Load caches
dirCache, err := loadDirCache(indexDir)
work.observe(err)
// Build a set of previous entries grouped by directory ancestry, so
// unchanged dirs can carry forward their whole subtree without ReadDir.
previousEntries, err := loadIndex(previousPath)
work.observe(err)
prevByDir := groupEntriesByUploadDir(previousEntries)
// Track completeness locally as well as in the runner: direct calls must
// not replace the baseline with a partial walk either.
indexCtx, completion := withIncompleteCheckCollector(ctx)
work.execution()
currentEntries := buildFileIndex(indexCtx, dirCache, prevByDir, forceFullScan)
incomplete := completion.contains("file_index")
if incomplete {
work.fail()
atomic.StoreInt32(&fileIndexScanCount, 0)
markCheckIncomplete(ctx, "file_index")
}
// A cancelled scan produced a partial index. Do not write or promote it:
// the partial set or its mtimes would make the next scan compare against
// stale cache state instead of the last complete baseline.
if err := ctx.Err(); err != nil {
work.withdraw(err)
return nil
}
prevSet := make(map[string]bool, len(previousEntries))
for _, e := range previousEntries {
prevSet[e] = true
}
// The baseline only tracks new paths. Active findings also need a current
// verdict before this scan can retire them, even on a cached walk. Two of
// the owned names are shared with the content scan, so a re-verified path
// may carry a finding this check never raised.
activeChecks := make(map[string]map[string]bool)
if st != nil {
for _, f := range st.LatestFindings() {
for _, name := range runnerFindingNames["file_index"] {
if f.Check == name && f.FilePath != "" {
if activeChecks[f.FilePath] == nil {
activeChecks[f.FilePath] = make(map[string]bool)
}
activeChecks[f.FilePath][f.Check] = true
break
}
}
}
}
newSet := make(map[string]bool)
var newFiles, activeFiles, scanFiles []string
for _, e := range currentEntries {
if !prevSet[e] {
newSet[e] = true
newFiles = append(newFiles, e)
}
if activeChecks[e] != nil {
activeFiles = append(activeFiles, e)
}
if !prevSet[e] || activeChecks[e] != nil {
scanFiles = append(scanFiles, e)
}
}
// Every finding this check raises asserts the file is new. A path is only
// re-verified because it already carries one, so the re-verification may
// renew that finding but must never raise a different one about a file
// that has been in the baseline all along -- which would also keep itself
// alive, by putting the path back into this set on the next cycle.
keepReverified := func(findings []alert.Finding) []alert.Finding {
out := findings[:0]
for _, f := range findings {
if !newSet[f.FilePath] && activeChecks[f.FilePath] != nil && !activeChecks[f.FilePath][f.Check] {
continue
}
out = append(out, f)
}
return out
}
if incomplete {
// Publishing partial entries or mtimes would let the next cached
// walk retire findings for directories we still have not read.
return keepReverified(checkFileIndexAnalyzeNewFiles(ctx, cfg, scanFiles))
}
work.local()
// Write current index (atomic)
work.observe(writeIndex(currentPath, currentEntries))
// First run - save baseline
if _, err := osFS.Stat(previousPath); os.IsNotExist(err) {
work.observe(copyFile(currentPath, previousPath))
work.observe(saveDirCache(indexDir, dirCache))
work.execution()
return keepReverified(checkFileIndexAnalyzeNewFiles(ctx, cfg, activeFiles))
}
isShrink, promote := evaluateFileIndexShrink(len(previousEntries), len(currentEntries))
if isShrink && !promote {
// The preserved baseline still contains removed paths. Its entries
// cannot be carried forward using mtimes from the smaller current
// walk, or the next cached scan resurrects those paths and resets
// the shrink streak. Rewalk until the smaller baseline is adopted.
dirCache = make(dirMtimeCache)
}
work.observe(saveDirCache(indexDir, dirCache))
// A large shrink (mass deletion, a WP install removed, or a transient read
// failure) must not instantly flush the baseline: promoting an empty or
// much-smaller index would make every surviving file look "new" next cycle
// and flood alerts. Still, entries that are present in the shrunken current
// index but absent from the old baseline are genuinely new and must be
// classified now. Non-promoting shrink cycles merge those paths into the old
// baseline so they do not alert repeatedly while the deletion guard is still
// preserving the removed paths against recovery floods.
if isShrink {
work.execution()
findings := keepReverified(checkFileIndexAnalyzeNewFiles(ctx, cfg, scanFiles))
work.local()
if promote {
fmt.Fprintf(os.Stderr, "file_index: shrink persisted %d scans; adopting smaller index (%d entries, was %d) as new baseline\n",
fileIndexShrinkPromoteThreshold, len(currentEntries), len(previousEntries))
work.observe(copyFile(currentPath, previousPath))
} else {
if len(newFiles) > 0 {
work.observe(writeIndex(previousPath, mergeIndexEntries(previousEntries, newFiles)))
}
fmt.Fprintf(os.Stderr, "file_index: current index (%d) shrank vs previous (%d); preserving prior baseline this cycle\n",
len(currentEntries), len(previousEntries))
}
return findings
}
work.execution()
findings := keepReverified(checkFileIndexAnalyzeNewFiles(ctx, cfg, scanFiles))
work.local()
work.observe(copyFile(currentPath, previousPath))
return findings
}
func mergeIndexEntries(base, extra []string) []string {
seen := make(map[string]bool, len(base)+len(extra))
merged := make([]string, 0, len(base)+len(extra))
for _, e := range base {
if seen[e] {
continue
}
seen[e] = true
merged = append(merged, e)
}
for _, e := range extra {
if seen[e] {
continue
}
seen[e] = true
merged = append(merged, e)
}
sort.Strings(merged)
return merged
}
// checkFileIndexAnalyzeNewFiles classifies a list of newly-detected file paths
// and returns alert.Finding values for those that match known threat patterns.
// Called by both the normal incremental path and the audit (ForceFileIndex) path.
func checkFileIndexAnalyzeNewFiles(ctx context.Context, cfg *config.Config, newFiles []string) []alert.Finding {
var findings []alert.Finding
for _, path := range newFiles {
name := filepath.Base(path)
nameLower := strings.ToLower(name)
// Bypassed for explicit full-scan / audit requests.
suppressed := false
if scanRespectsIgnores(ctx, cfg) {
for _, ignore := range cfg.Suppressions.IgnorePaths {
if matchGlob(path, ignore) {
suppressed = true
break
}
}
}
if suppressed {
continue
}
severity := alert.Severity(-1)
check := ""
message := ""
contentSHA256 := ""
if strings.Contains(path, "/wp-content/uploads/") && phpPathExecutes(path, nameLower) {
// Content decides, never the path or name. A negative
// severity is a content-verified inert stub (e.g. the
// WordPress "silence is golden" index.php, or BackWPup's
// "<?php //<json>" working files) and is suppressed; any
// real code surfaces, malicious or merely present.
sev, ck, msg, hash, readOK := classifyUploadPHPWithFingerprint(path)
if !readOK {
recordCoverageGapPaths(ctx, "file_index", coveragePathAliases(path))
reportFileIndexFailure(ctx)
}
if sev >= 0 {
severity = sev
check = ck
message = msg
contentSHA256 = hash
} else {
continue
}
}
// PHP files in wp-content/languages and wp-content/upgrade: content-first.
// Path-only Critical buried real alerts under location noise (WPML
// translation queues, WP auto-update staging). See classifySensitiveDirPHP.
sev, ck, msg, hash, readOK := classifySensitiveDirPHPWithFingerprint(path, name)
// There the content was read and judged; a word in the name adds nothing.
contentJudged := readOK && isSensitiveDirPHPPath(path)
if !readOK {
recordCoverageGapPaths(ctx, "file_index", coveragePathAliases(path))
reportFileIndexFailure(ctx)
}
if sev >= 0 {
severity = sev
check = ck
message = msg
contentSHA256 = hash
}
if strings.Contains(path, "/.config/") {
severity = alert.Critical
check = "new_executable_in_config"
message = fmt.Sprintf("New executable in .config: %s", path)
}
if isWebshellName(nameLower) {
severity = alert.Critical
check = "new_webshell_file"
message = fmt.Sprintf("New file with webshell name: %s", path)
}
if isExecutablePHPName(nameLower) && isSuspiciousPHPName(nameLower) && !contentJudged {
if severity < 0 {
severity = alert.High
check = "new_suspicious_php"
message = fmt.Sprintf("New suspicious PHP file: %s", path)
}
}
if severity >= 0 {
details := ""
if info, err := osFS.Stat(path); err == nil {
details = fmt.Sprintf("Size: %d, Mtime: %s", info.Size(), info.ModTime().Format("2006-01-02 15:04:05"))
}
f := alert.Finding{
Severity: severity,
Check: check,
Message: message,
Details: details,
FilePath: path,
}
if IsContentReverifiable(f.Check) {
f.ContentSHA256 = contentSHA256
f.DetectLogic = ContentDetectionVersion()
}
findings = append(findings, f)
}
}
return findings
}
// classifySensitiveDirPHP returns (severity, check, message) for a PHP file
// in /wp-content/languages/ or /wp-content/upgrade/. Returns a negative
// severity when the path is not in a sensitive dir.
//
// Every PHP file in a sensitive dir is content-analysed -- there is no
// filename allowlist, so an attacker cannot hide a backdoor by naming it like
// a translation or index file. A real indicator keeps Critical severity and
// the content-based check name (obfuscated_php / suspicious_php_content, both
// already wired into autoresponse, remediate, correlation, and attackdb). A
// clean file is demoted to Warning with check new_php_in_sensitive_dir_clean,
// which is intentionally NOT in any of those maps -- a clean file is a
// visibility signal, not an attack. Mirrors the realtime path at fanotify.go.
func classifySensitiveDirPHP(path, name string) (alert.Severity, string, string) {
sev, check, message, _, _ := classifySensitiveDirPHPWithFingerprint(path, name)
return sev, check, message
}
// isSensitiveDirPHPPath reports whether the content check judges PHP at path.
func isSensitiveDirPHPPath(path string) bool {
return strings.Contains(path, "/wp-content/languages/") || strings.Contains(path, "/wp-content/upgrade/")
}
func classifySensitiveDirPHPWithFingerprint(path, name string) (alert.Severity, string, string, string, bool) {
nameLower := strings.ToLower(name)
if !phpPathExecutes(path, nameLower) {
return -1, "", "", "", true
}
if !isSensitiveDirPHPPath(path) {
return -1, "", "", "", true
}
locLabel := "wp-content/languages"
if strings.Contains(path, "/wp-content/upgrade/") {
locLabel = "wp-content/upgrade"
}
result, contentSHA256 := analyzePHPContentWithFingerprint(path)
if result.severity >= 0 {
return result.severity, result.check, fmt.Sprintf("%s: %s", result.message, path), contentSHA256, result.readOK
}
// Fail closed: an unreadable body (attacker racing the scanner with rm or
// chmod 000) must not be demoted to a clean Warning. Mirrors classifyUploadPHP.
if !result.readOK {
return alert.High, "new_php_in_sensitive_dir",
fmt.Sprintf("New unreadable PHP file in %s: %s", locLabel, path), "", false
}
// A zero-byte body reaching here was verified stable across the read (a
// truncation under the scanner fails the post-read stat and is handled
// above), so it holds no code and is visibility, not an attack.
if result.empty {
return alert.Warning, "new_php_in_sensitive_dir_clean",
fmt.Sprintf("New empty PHP file in %s (no content): %s", locLabel, path), "", true
}
// Content-verified inert stub (e.g. the "silence is golden" index.php) is
// suppressed; any real code surfaces as a non-actionable visibility Warning.
if IsBenignPHPStub(path) {
return -1, "", "", "", true
}
// WordPress 6.5+ auto-generates *.l10n.php translation caches as pure data
// return arrays. Recognized by content structure (not filename), they carry
// no executable construct, so suppress rather than warn on every locale file.
if isWPTranslationCache(path) {
return -1, "", "", "", true
}
return alert.Warning, "new_php_in_sensitive_dir_clean",
fmt.Sprintf("New PHP file in %s (content clean): %s", locLabel, path), "", true
}
// groupEntriesByUploadDir groups index entries by each ancestor directory.
// Used to carry forward the whole cached subtree when an unchanged directory
// is skipped before walking into its children.
func groupEntriesByUploadDir(entries []string) map[string][]string {
grouped := make(map[string][]string)
for _, path := range entries {
for dir := filepath.Dir(path); dir != "." && dir != string(filepath.Separator); dir = filepath.Dir(dir) {
grouped[dir] = append(grouped[dir], path)
}
}
return grouped
}
// buildFileIndex scans targeted directory subtrees using ReadDir.
// Skips directories whose mtime hasn't changed - carries forward
// their entries from the previous index instead.
// If forceFullScan is true, all directories are re-scanned regardless of mtime.
func buildFileIndex(ctx context.Context, dirCache dirMtimeCache, prevByDir map[string][]string, forceFullScan bool) []string {
var entries []string
if ctx == nil {
ctx = context.Background()
}
if ctx.Err() != nil {
return nil
}
homeDirs, err := fileIndexHomeDirs(ctx)
if err != nil {
markCheckIncomplete(ctx, "file_index")
}
var subtreeChanges *subtreeChangeTracker
if !forceFullScan {
subtreeChanges = newSubtreeChangeTracker(dirCache)
}
for _, homeEntry := range homeDirs {
// A cancelled scan must stop walking and let the caller discard the
// partial index rather than promote it as the new baseline.
if ctx.Err() != nil {
return entries
}
if !homeEntry.IsDir() {
continue
}
homeDir := scanHomeDirPath(homeEntry)
// Scan wp-content/uploads for PHP files
uploadDirs := []string{
filepath.Join(homeDir, "public_html", "wp-content", "uploads"),
}
// Scan directories that shouldn't normally contain user PHP:
// languages (translation files only), upgrade (temp dir), mu-plugins
sensitiveWPDirs := []string{
filepath.Join(homeDir, "public_html", "wp-content", "languages"),
filepath.Join(homeDir, "public_html", "wp-content", "upgrade"),
filepath.Join(homeDir, "public_html", "wp-content", "mu-plugins"),
}
subDirs, err := osFS.ReadDir(homeDir)
if err != nil && !os.IsNotExist(err) {
markCheckIncomplete(ctx, "file_index")
}
for _, sd := range subDirs {
if ctx.Err() != nil {
return entries
}
if sd.IsDir() && sd.Name() != "public_html" && sd.Name() != "mail" &&
!strings.HasPrefix(sd.Name(), ".") && sd.Name() != "etc" &&
sd.Name() != "logs" && sd.Name() != "ssl" && sd.Name() != "tmp" {
uploadsPath := filepath.Join(homeDir, sd.Name(), "wp-content", "uploads")
if info, err := osFS.Stat(uploadsPath); err == nil && info.IsDir() {
uploadDirs = append(uploadDirs, uploadsPath)
} else if err != nil && !os.IsNotExist(err) {
markCheckIncomplete(ctx, "file_index")
}
// Also track sensitive dirs for addon domains
for _, subDir := range []string{"languages", "upgrade", "mu-plugins"} {
if ctx.Err() != nil {
return entries
}
sensitiveDir := filepath.Join(homeDir, sd.Name(), "wp-content", subDir)
if info, err := osFS.Stat(sensitiveDir); err == nil && info.IsDir() {
sensitiveWPDirs = append(sensitiveWPDirs, sensitiveDir)
} else if err != nil && !os.IsNotExist(err) {
markCheckIncomplete(ctx, "file_index")
}
}
}
}
for _, uploadsDir := range uploadDirs {
if ctx.Err() != nil {
return entries
}
scanDirForPHPContextWithTracker(ctx, uploadsDir, 6, dirCache, prevByDir, forceFullScan, phpHandlerOverlay{}, subtreeChanges, &entries)
}
// Scan sensitive WP directories for any PHP files
for _, sensitiveDir := range sensitiveWPDirs {
if ctx.Err() != nil {
return entries
}
scanDirForPHPContextWithTracker(ctx, sensitiveDir, 4, dirCache, prevByDir, forceFullScan, phpHandlerOverlay{}, subtreeChanges, &entries)
}
// Scan .config for executables
configDir := filepath.Join(homeDir, ".config")
if ctx.Err() != nil {
return entries
}
scanDirForExecutablesContextWithTracker(ctx, configDir, 3, dirCache, prevByDir, forceFullScan, subtreeChanges, &entries)
}
if AccountFromContext(ctx) == "" {
// Scan tmp dirs
for _, tmpDir := range []string{"/tmp", "/dev/shm", "/var/tmp"} {
if ctx.Err() != nil {
return entries
}
scanDirForSuspiciousExtContextWithTracker(ctx, tmpDir, 2, dirCache, prevByDir, forceFullScan, subtreeChanges, &entries)
}
}
sort.Strings(entries)
return entries
}
func fileIndexHomeDirs(ctx context.Context) ([]os.DirEntry, error) {
if AccountFromContext(ctx) != "" {
return GetScanHomeDirs(ctx)
}
homes, err := readAccountHomes()
entries := make([]os.DirEntry, 0, len(homes))
for _, home := range homes {
entries = append(entries, rootedDirEntry{DirEntry: home.Entry, root: home.Root})
}
return entries, err
}
// scanDirForPHP recursively reads directories for PHP-executable files.
// If directory mtime is unchanged, carries forward previous entries.
func scanDirForPHP(dir string, maxDepth int, cache dirMtimeCache, prev map[string][]string, forceFullScan bool, overlay phpHandlerOverlay, entries *[]string) {
scanDirForPHPContext(context.Background(), dir, maxDepth, cache, prev, forceFullScan, overlay, entries)
}
func scanDirForPHPContext(ctx context.Context, dir string, maxDepth int, cache dirMtimeCache, prev map[string][]string, forceFullScan bool, overlay phpHandlerOverlay, entries *[]string) {
var subtreeChanges *subtreeChangeTracker
if !forceFullScan {
subtreeChanges = newSubtreeChangeTracker(cache)
}
scanDirForPHPContextWithTracker(ctx, dir, maxDepth, cache, prev, forceFullScan, overlay, subtreeChanges, entries)
}
func scanDirForPHPContextWithTracker(ctx context.Context, dir string, maxDepth int, cache dirMtimeCache, prev map[string][]string, forceFullScan bool, overlay phpHandlerOverlay, subtreeChanges *subtreeChangeTracker, entries *[]string) {
if maxDepth <= 0 || ctx.Err() != nil {
return
}
if htaccess, err := osFS.ReadFile(filepath.Join(dir, ".htaccess")); err == nil {
overlay = overlay.mergeHtaccess(htaccess)
} else if !os.IsNotExist(err) {
markCheckIncomplete(ctx, "file_index")
}
if ctx.Err() != nil {
return
}
changed := dirChanged(dir, cache, forceFullScan || overlay.active())
if ctx.Err() != nil {
return
}
subtreeChanged := false
if !changed {
subtreeChanged = subtreeChanges.hasChangedDir(ctx, dir)
}
if ctx.Err() != nil {
return
}
if !changed && !subtreeChanged {
// dir's mtime is unchanged and no cached subdirectory changed, so the
// whole subtree is stable: carry the previous entries forward without
// re-reading disk.
*entries = append(*entries, prev[dir]...)
return
}
// Either dir changed, or a file was dropped deep in the subtree (bumping
// only a subdirectory's mtime). Walk it so the new file is indexed now
// instead of hiding until the periodic forced full scan.
dirEntries, err := osFS.ReadDir(dir)
if err != nil {
if !os.IsNotExist(err) {
markCheckIncomplete(ctx, "file_index")
}
return
}
for _, entry := range dirEntries {
if ctx.Err() != nil {
return
}
name := entry.Name()
fullPath := filepath.Join(dir, name)
if entry.IsDir() {
scanDirForPHPContextWithTracker(ctx, fullPath, maxDepth-1, cache, prev, forceFullScan, overlay, subtreeChanges, entries)
continue
}
nameLower := strings.ToLower(name)
// index.php is indexed too: a webshell named index.php must not hide
// behind the WordPress silence-stub convention. The inert stub itself
// is suppressed later by content analysis. All PHP-executable
// extensions are indexed, not just .php, so a .phtml/.php7 backdoor
// cannot dodge the index by extension. The two predicates overlap
// (the PHP family is also "suspicious"), so index each file once.
if overlay.executes(nameLower) || suspiciousExtensions[filepath.Ext(nameLower)] {
*entries = append(*entries, fullPath)
}
}
}
// scanDirForExecutables reads .config dirs for executable files.
func scanDirForExecutables(dir string, maxDepth int, cache dirMtimeCache, prev map[string][]string, forceFullScan bool, entries *[]string) {
scanDirForExecutablesContext(context.Background(), dir, maxDepth, cache, prev, forceFullScan, entries)
}
func scanDirForExecutablesContext(ctx context.Context, dir string, maxDepth int, cache dirMtimeCache, prev map[string][]string, forceFullScan bool, entries *[]string) {
var subtreeChanges *subtreeChangeTracker
if !forceFullScan {
subtreeChanges = newSubtreeChangeTracker(cache)
}
scanDirForExecutablesContextWithTracker(ctx, dir, maxDepth, cache, prev, forceFullScan, subtreeChanges, entries)
}
func scanDirForExecutablesContextWithTracker(ctx context.Context, dir string, maxDepth int, cache dirMtimeCache, prev map[string][]string, forceFullScan bool, subtreeChanges *subtreeChangeTracker, entries *[]string) {
if maxDepth <= 0 || ctx.Err() != nil {
return
}
changed := dirChanged(dir, cache, forceFullScan)
if ctx.Err() != nil {
return
}
subtreeChanged := false
if !changed {
subtreeChanged = subtreeChanges.hasChangedDir(ctx, dir)
}
if ctx.Err() != nil {
return
}
if !changed && !subtreeChanged {
*entries = append(*entries, prev[dir]...)
return
}
dirEntries, err := osFS.ReadDir(dir)
if err != nil {
if !os.IsNotExist(err) {
markCheckIncomplete(ctx, "file_index")
}
return
}
for _, entry := range dirEntries {
if ctx.Err() != nil {
return
}
fullPath := filepath.Join(dir, entry.Name())
if entry.IsDir() {
scanDirForExecutablesContextWithTracker(ctx, fullPath, maxDepth-1, cache, prev, forceFullScan, subtreeChanges, entries)
continue
}
info, err := entry.Info()
if err != nil {
if !os.IsNotExist(err) {
reportFileIndexFailure(ctx)
}
continue
}
if info.Mode()&0111 != 0 {
*entries = append(*entries, fullPath)
}
}
}
// scanDirForSuspiciousExt reads tmp dirs for files with suspicious extensions.
func scanDirForSuspiciousExt(dir string, maxDepth int, cache dirMtimeCache, prev map[string][]string, forceFullScan bool, entries *[]string) {
scanDirForSuspiciousExtContext(context.Background(), dir, maxDepth, cache, prev, forceFullScan, entries)
}
func scanDirForSuspiciousExtContext(ctx context.Context, dir string, maxDepth int, cache dirMtimeCache, prev map[string][]string, forceFullScan bool, entries *[]string) {
var subtreeChanges *subtreeChangeTracker
if !forceFullScan {
subtreeChanges = newSubtreeChangeTracker(cache)
}
scanDirForSuspiciousExtContextWithTracker(ctx, dir, maxDepth, cache, prev, forceFullScan, subtreeChanges, entries)
}
func scanDirForSuspiciousExtContextWithTracker(ctx context.Context, dir string, maxDepth int, cache dirMtimeCache, prev map[string][]string, forceFullScan bool, subtreeChanges *subtreeChangeTracker, entries *[]string) {
if maxDepth <= 0 || ctx.Err() != nil {
return
}
changed := dirChanged(dir, cache, forceFullScan)
if ctx.Err() != nil {
return
}
subtreeChanged := false
if !changed {
subtreeChanged = subtreeChanges.hasChangedDir(ctx, dir)
}
if ctx.Err() != nil {
return
}
if !changed && !subtreeChanged {
*entries = append(*entries, prev[dir]...)
return
}
dirEntries, err := osFS.ReadDir(dir)
if err != nil {
if !os.IsNotExist(err) {
markCheckIncomplete(ctx, "file_index")
}
return
}
for _, entry := range dirEntries {
if ctx.Err() != nil {
return
}
name := entry.Name()
fullPath := filepath.Join(dir, name)
if entry.IsDir() {
scanDirForSuspiciousExtContextWithTracker(ctx, fullPath, maxDepth-1, cache, prev, forceFullScan, subtreeChanges, entries)
continue
}
ext := filepath.Ext(strings.ToLower(name))
if suspiciousExtensions[ext] {
*entries = append(*entries, fullPath)
}
}
}
func isWebshellName(name string) bool {
webshells := map[string]bool{
"h4x0r.php": true, "c99.php": true, "r57.php": true,
"wso.php": true, "alfa.php": true, "b374k.php": true,
"shell.php": true, "cmd.php": true, "backdoor.php": true,
"webshell.php": true, "hack.php": true, "0x.php": true,
"up.php": true, "uploader.php": true, "filemanager.php": true,
}
return webshells[name]
}
func isSuspiciousPHPName(name string) bool {
suspicious := []string{
"shell", "cmd", "hack", "backdoor", "upload",
"exploit", "reverse", "connect", "proxy", "tunnel",
"0x", "x0", "passthru",
}
for _, s := range suspicious {
if strings.Contains(name, s) {
return true
}
}
// These also sit inside ordinary words (retrieval, execute, assertions),
// so they count only where no letter adjoins them.
for _, w := range []string{"eval", "exec", "assert"} {
if containsStandaloneWord(name, w) {
return true
}
}
nameNoExt := strings.TrimSuffix(name, ".php")
if len(nameNoExt) <= 5 && strings.ContainsAny(nameNoExt, "0123456789") {
return true
}
return false
}
// containsStandaloneWord reports whether w occurs in name with no letter
// directly before or after it.
func containsStandaloneWord(name, w string) bool {
isLetter := func(c byte) bool { return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' }
for start := 0; ; {
i := strings.Index(name[start:], w)
if i < 0 {
return false
}
i += start
end := i + len(w)
if (i == 0 || !isLetter(name[i-1])) && (end == len(name) || !isLetter(name[end])) {
return true
}
start = i + 1
}
}
func writeIndex(path string, entries []string) error {
tmpPath := path + ".tmp"
// #nosec G304 -- path is filepath.Join under operator-configured StatePath.
f, err := os.Create(tmpPath)
if err != nil {
return err
}
defer func() { _ = f.Close() }()
w := bufio.NewWriter(f)
for _, e := range entries {
if _, err := w.WriteString(e + "\n"); err != nil {
return err
}
}
if err := w.Flush(); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
return os.Rename(tmpPath, path)
}
func loadIndex(path string) ([]string, error) {
f, err := osFS.Open(path)
if os.IsNotExist(err) {
return nil, nil
}
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
var entries []string
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 0, 1024*1024), 1024*1024)
for scanner.Scan() {
line := scanner.Text()
if line != "" {
entries = append(entries, line)
}
}
return entries, scanner.Err()
}
func copyFile(src, dst string) error {
data, err := osFS.ReadFile(src)
if err != nil {
return err
}
return os.WriteFile(dst, data, 0600)
}
package checks
import (
"context"
"errors"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/queuehealth"
)
var fileIndexQueues = newFileIndexQueue()
type fileIndexWorkKey struct{}
// Content and metadata failures retain existing findings and baseline policy.
// They still belong to the live scan, even when a walker can continue.
func reportFileIndexFailure(ctx context.Context) {
if work, ok := ctx.Value(fileIndexWorkKey{}).(*fileIndexWork); ok {
work.fail()
}
}
type fileIndexQueueMonitor struct {
mu sync.Mutex
waiting map[*fileIndexWork]struct{}
active *fileIndexWork
idleSince time.Time
waitingLoss, activeLoss *queuehealth.Tracker
}
type fileIndexWork struct {
queue *fileIndexQueueMonitor
at, deadline, executionDeadline time.Time
running, failed bool
}
func newFileIndexQueue() *fileIndexQueueMonitor {
return &fileIndexQueueMonitor{
waiting: make(map[*fileIndexWork]struct{}),
waitingLoss: queuehealth.New(0, time.Minute),
activeLoss: queuehealth.New(1, time.Minute),
}
}
func (q *fileIndexQueueMonitor) run(ctx context.Context, fn func(*fileIndexWork) []alert.Finding) []alert.Finding {
if ctx.Err() != nil {
return nil
}
now := time.Now()
budget := timeoutFor("file_index")
parent, hasParent := ctx.Deadline()
deadline := now.Add(budget)
if hasParent && parent.Before(deadline) {
deadline = parent
}
w := &fileIndexWork{queue: q, at: now, deadline: deadline}
q.mu.Lock()
if q.active == nil && len(q.waiting) == 0 {
q.idleSince = now
}
q.waiting[w] = struct{}{}
q.mu.Unlock()
completed := false
defer func() { w.finish(completed) }()
select {
case fileIndexLiveScanGate <- struct{}{}:
now = time.Now()
executionDeadline := now.Add(budget)
if hasParent && parent.Before(executionDeadline) {
executionDeadline = parent
}
q.mu.Lock()
delete(q.waiting, w)
q.active = w
q.idleSince = time.Time{}
w.running = true
w.at = now
w.deadline = now.Add(time.Minute)
w.executionDeadline = executionDeadline
q.mu.Unlock()
case <-ctx.Done():
w.withdraw(ctx.Err())
completed = true
return nil
}
findings := fn(w)
completed = true
return findings
}
func (w *fileIndexWork) execution() {
w.queue.mu.Lock()
w.at = time.Now()
w.deadline = w.executionDeadline
w.queue.mu.Unlock()
}
func (w *fileIndexWork) local() {
w.queue.mu.Lock()
w.at = time.Now()
w.deadline = w.at.Add(time.Minute)
w.queue.mu.Unlock()
}
func (w *fileIndexWork) failLocked() {
if w.failed {
return
}
w.failed = true
losses := w.queue.waitingLoss
if w.running {
losses = w.queue.activeLoss
}
losses.Lose(time.Now(), 1)
}
func (w *fileIndexWork) fail() {
w.queue.mu.Lock()
w.failLocked()
w.queue.mu.Unlock()
}
func (w *fileIndexWork) observe(err error) {
if err != nil {
w.fail()
}
}
func (w *fileIndexWork) withdraw(err error) {
if errors.Is(err, context.Canceled) {
return
}
w.queue.mu.Lock()
w.failLocked()
w.queue.mu.Unlock()
}
func (w *fileIndexWork) finish(completed bool) {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
if !completed {
w.failLocked()
}
if w.running {
q.active = nil
q.idleSince = time.Now()
// The scan owns this token. Release its health owner and token under
// one lock so its successor cannot publish before this owner leaves.
<-fileIndexLiveScanGate
} else {
delete(q.waiting, w)
}
}
// FileIndexQueueStatuses reads memory without waiting for a scan or filesystem.
func FileIndexQueueStatuses(now time.Time) map[string]queuehealth.Status {
q := fileIndexQueues
q.mu.Lock()
defer q.mu.Unlock()
waiting := q.waitingLoss.Snapshot(now)
waiting.CapacityUnavailable = true
active := q.activeLoss.Snapshot(now)
for w := range q.waiting {
waiting.Depth++
waiting.LagSeconds = max(waiting.LagSeconds, now.Sub(w.at).Seconds())
if !now.Before(w.deadline) || (q.active == nil && now.Sub(q.idleSince) >= time.Minute) {
waiting.Status, waiting.Reason = "degraded", "backlog_lag"
}
}
if w := q.active; w != nil {
active.InFlight = 1
active.ProcessingSeconds = max(0, now.Sub(w.at).Seconds())
if !now.Before(w.deadline) {
active.Status, active.Reason = "degraded", "processing_lag"
}
}
return map[string]queuehealth.Status{"waiting": waiting, "active": active}
}
package checks
import (
"fmt"
"github.com/pidginhost/csm/internal/alert"
)
// classifyUploadPHP decides severity, check name, and message for a fresh PHP
// file under wp-content/uploads using its CONTENT, never its path or name.
// Uploads should hold media, not PHP, so any new PHP is at least a visibility
// signal; the body decides whether it is an attack.
//
// A negative severity means "suppress" (a content-verified inert stub). It
// mirrors classifySensitiveDirPHP so the two anomalous-PHP-location detectors
// behave identically. Path/name allowlists are intentionally absent: skipping
// a file because it sits under /cache/ or is named index.php is exactly how an
// attacker hides a webshell in a "safe" location.
//
// An unreadable body fails closed at High: an attacker who races the scanner
// with `rm` or chmod 000 must not earn a demote. A body truncated mid-read
// fails the post-read stat comparison and lands in that same branch, so a
// zero-byte result carries proof the file really was empty for the whole read.
// Empty means no code, which cannot execute, so it joins content-clean
// real-code files as a non-actionable Warning under a check name that is
// intentionally absent from the correlation and auto-response maps -- neither
// is an attack, and rating them High buries the findings that are.
func classifyUploadPHP(path string) (alert.Severity, string, string) {
sev, check, message, _, _ := classifyUploadPHPWithFingerprint(path)
return sev, check, message
}
func classifyUploadPHPWithFingerprint(path string) (alert.Severity, string, string, string, bool) {
r, contentSHA256 := analyzePHPContentWithFingerprint(path)
if r.severity >= 0 {
return r.severity, r.check, fmt.Sprintf("%s: %s", r.message, path), contentSHA256, r.readOK
}
if !r.readOK {
return alert.High, "new_php_in_uploads", fmt.Sprintf("New unreadable PHP file in uploads: %s", path), "", false
}
if r.empty {
return alert.Warning, "new_php_in_uploads_clean", fmt.Sprintf("New empty PHP file in uploads (no content): %s", path), "", true
}
if IsBenignPHPStub(path) {
return -1, "", "", "", true
}
return alert.Warning, "new_php_in_uploads_clean", fmt.Sprintf("New PHP file in uploads (content clean): %s", path), "", true
}
package checks
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// CheckFilesystem uses globs and targeted ReadDir to check for backdoors,
// hidden files, and SUID binaries. No `find` command needed.
func CheckFilesystem(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
var findings []alert.Finding
// GSocket / backdoor binaries in .config dirs - glob (instant).
// Rank by mtime desc so recently-touched accounts process first
// when the check timeout cuts iteration short.
backdoorNames := map[string]bool{
"defunct": true, "defunct.dat": true, "gs-netcat": true,
"gs-sftp": true, "gs-mount": true, "gsocket": true,
}
configGlobs := [][]string{
{".config", "htop", "*"},
{".config", "*", "*"},
}
// The htop glob is a subset of the wider .config glob. Deduplicate
// before ranking so the per-account cap applies to this scanner once.
configCandidates := make([]string, 0)
seenConfigCandidate := make(map[string]struct{})
for _, pattern := range configGlobs {
if ctx.Err() != nil {
return findings
}
matches, err := homeGlob(ctx, pattern...)
markScanReadError(ctx, "filesystem", err)
for _, path := range matches {
if ctx.Err() != nil {
return findings
}
if backdoorNames[filepath.Base(path)] {
if _, seen := seenConfigCandidate[path]; seen {
continue
}
seenConfigCandidate[path] = struct{}{}
configCandidates = append(configCandidates, path)
}
}
}
rankedConfigCandidates := rankPathsByMtimeDesc(ctx, configCandidates, accountScanMaxFiles(ctx, cfg))
if len(rankedConfigCandidates) < len(configCandidates) {
markCheckIncomplete(ctx, "filesystem")
}
if ctx.Err() != nil {
return findings
}
for _, path := range rankedConfigCandidates {
if ctx.Err() != nil {
return findings
}
info, _ := osFS.Stat(path)
var details string
if info != nil {
details = fmt.Sprintf("Size: %d bytes, Mtime: %s", info.Size(), info.ModTime().Format("2006-01-02 15:04:05"))
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "backdoor_binary",
Message: fmt.Sprintf("Backdoor binary found: %s", path),
Details: details,
FilePath: path,
})
}
if AccountFromContext(ctx) == "" {
// Hidden files in /tmp, /dev/shm, /var/tmp - glob (instant)
safeHiddenPrefixes := []string{
".s.PGSQL", ".font-unix", ".ICE-unix", ".X11-unix",
".XIM-unix", ".crontab.", ".Test-unix",
}
// One candidate set across all three roots: on CloudLinux /var/tmp is
// the same filesystem as /tmp, so the same physical file is reachable
// through two of these patterns and was reported once per pattern.
var candidates []string
for _, pattern := range []string{"/tmp/.*", "/dev/shm/.*", "/var/tmp/.*"} {
if ctx.Err() != nil {
return findings
}
matches, err := osFS.Glob(pattern)
markScanReadError(ctx, "filesystem", err)
for _, match := range matches {
if ctx.Err() != nil {
return findings
}
base := filepath.Base(match)
safe := false
for _, prefix := range safeHiddenPrefixes {
if strings.HasPrefix(base, prefix) {
safe = true
break
}
}
if safe {
continue
}
candidates = append(candidates, match)
}
}
// These are global temp locations, not account paths; do not let
// account_scan_max_files hide older suspicious files here.
ranked := rankPathsByMtimeDesc(ctx, candidates, 0)
if ctx.Err() != nil {
return findings
}
var reported []os.FileInfo
for _, match := range ranked {
if ctx.Err() != nil {
return findings
}
info, err := osFS.Stat(match)
markScanReadError(ctx, "filesystem", err)
if err != nil || info.IsDir() {
continue
}
// A leading dot is not by itself a signal: these directories are
// full of root-owned infrastructure state. What matters is whether
// the file could execute.
canExecute, err := hiddenTempFileCanExecute(match, info)
markScanReadError(ctx, "filesystem", err)
if !canExecute {
continue
}
seen := false
for _, prev := range reported {
if os.SameFile(prev, info) {
seen = true
break
}
}
if seen {
continue
}
reported = append(reported, info)
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "suspicious_file",
Message: fmt.Sprintf("Suspicious hidden file: %s", match),
Details: fmt.Sprintf("Size: %d, Mtime: %s", info.Size(), info.ModTime()),
FilePath: match,
})
}
// SUID binaries in tmp dirs - ReadDir + stat (small dirs, fast)
for _, dir := range []string{"/tmp", "/var/tmp", "/dev/shm"} {
if ctx.Err() != nil {
return findings
}
scanForSUID(ctx, dir, 3, &findings)
}
}
// SUID in /home - shallow scan only
if ctx.Err() != nil {
return findings
}
homeDirs := scanHomeDirsWithCoverage(ctx, "filesystem")
for _, entry := range homeDirs {
if ctx.Err() != nil {
return findings
}
if !entry.IsDir() {
continue
}
scanForSUID(ctx, scanHomeDirPath(entry), 3, &findings)
}
return findings
}
// hiddenTempFileCanExecute reports whether a hidden file in a world-writable
// temp directory could run: an executable bit or ELF magic, or a script marker
// that an interpreter would honour. Inert data written there by system
// components is not a finding.
func hiddenTempFileCanExecute(path string, info os.FileInfo) (bool, error) {
if !info.Mode().IsRegular() {
return false, nil
}
if info.Mode()&0o111 != 0 {
return true, nil
}
// Stat alone cannot protect a world-writable path: it can be replaced with
// a FIFO before open. Use the provider's nonblocking, fd-verified opener
// and bind the content decision to the same inode used for deduplication.
opener, ok := osFS.(interface {
openRegularFile(string, int) (*os.File, error)
})
if !ok {
return false, fmt.Errorf("filesystem provider cannot safely open %s", path)
}
f, err := opener.openRegularFile(path, 0)
if err != nil {
return false, err
}
defer func() { _ = f.Close() }()
opened, err := f.Stat()
if err != nil {
return false, err
}
if !sameFileSnapshot(info, opened) {
return false, errFileChanged
}
var head [8]byte
n, err := io.ReadFull(f, head[:])
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
return false, err
}
after, err := f.Stat()
if err != nil {
return false, err
}
if !sameFileSnapshot(opened, after) {
return false, errFileChanged
}
prefix := strings.ToLower(string(head[:n]))
return strings.HasPrefix(string(head[:n]), "\x7fELF") ||
strings.HasPrefix(prefix, "#!") ||
strings.HasPrefix(prefix, "<?php") ||
strings.HasPrefix(prefix, "<?="), nil
}
// scanForSUID checks for SUID binaries using ReadDir.
func scanForSUID(ctx context.Context, dir string, maxDepth int, findings *[]alert.Finding) {
if ctx.Err() != nil {
return
}
if maxDepth <= 0 {
return
}
entries, err := osFS.ReadDir(dir)
if err != nil {
markScanReadError(ctx, "filesystem", err)
return
}
for _, entry := range entries {
if ctx.Err() != nil {
return
}
fullPath := filepath.Join(dir, entry.Name())
if entry.IsDir() {
// Skip virtfs and known large dirs
if entry.Name() == "virtfs" || entry.Name() == "mail" || entry.Name() == "public_html" {
continue
}
scanForSUID(ctx, fullPath, maxDepth-1, findings)
continue
}
info, err := entry.Info()
if err != nil {
markScanReadError(ctx, "filesystem", err)
continue
}
if info.Mode()&os.ModeSetuid != 0 {
*findings = append(*findings, alert.Finding{
Severity: alert.Critical,
Check: "suid_binary",
Message: fmt.Sprintf("SUID binary in unusual location: %s", fullPath),
Details: fmt.Sprintf("Mode: %s, Size: %d", info.Mode(), info.Size()),
FilePath: fullPath,
})
}
}
}
// CheckWebshells uses pure Go ReadDir to scan for known webshell files
// and directories. No `find` command needed.
func CheckWebshells(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
webshellNames := map[string]bool{
"h4x0r.php": true, "c99.php": true, "r57.php": true,
"wso.php": true, "alfa.php": true, "b374k.php": true,
"mini.php": true, "adminer.php": true,
}
webshellDirs := map[string]bool{
"LEVIATHAN": true, "haxorcgiapi": true,
}
// Scan each user's public_html and addon domains
homeDirs := scanHomeDirsWithCoverage(ctx, "webshells")
for _, homeEntry := range homeDirs {
if ctx.Err() != nil {
return findings
}
if !homeEntry.IsDir() {
continue
}
homeDir := scanHomeDirPath(homeEntry)
// Get all potential document roots
docRoots := []string{filepath.Join(homeDir, "public_html")}
subDirs, err := osFS.ReadDir(homeDir)
markScanReadError(ctx, "webshells", err)
for _, sd := range subDirs {
if sd.IsDir() && sd.Name() != "public_html" && sd.Name() != "mail" &&
!strings.HasPrefix(sd.Name(), ".") && sd.Name() != "etc" &&
sd.Name() != "logs" && sd.Name() != "ssl" && sd.Name() != "tmp" {
docRoots = append(docRoots, filepath.Join(homeDir, sd.Name()))
}
}
for _, docRoot := range docRoots {
scanForWebshells(ctx, docRoot, 8, webshellNames, webshellDirs, cfg, &findings)
if ctx.Err() != nil {
return findings
}
}
}
return findings
}
// scanForWebshells recursively reads directories looking for known webshell
// files and directories. Uses ReadDir (getdents) - no stat unless matched.
func scanForWebshells(ctx context.Context, dir string, maxDepth int, names map[string]bool, dirs map[string]bool, cfg *config.Config, findings *[]alert.Finding) {
if ctx.Err() != nil {
return
}
if maxDepth <= 0 {
return
}
entries, err := osFS.ReadDir(dir)
if err != nil {
markScanReadError(ctx, "webshells", err)
return
}
for _, entry := range entries {
if ctx.Err() != nil {
return
}
name := entry.Name()
fullPath := filepath.Join(dir, name)
// Check suppressed paths (bypassed for explicit full-scan / audit requests).
suppressed := false
if scanRespectsIgnores(ctx, cfg) {
for _, ignore := range cfg.Suppressions.IgnorePaths {
if matchGlob(fullPath, ignore) {
suppressed = true
break
}
}
}
if suppressed {
continue
}
if entry.IsDir() {
if dirs[name] {
*findings = append(*findings, alert.Finding{
Severity: alert.Critical,
Check: "webshell",
Message: fmt.Sprintf("Webshell directory found: %s", fullPath),
FilePath: fullPath,
})
}
scanForWebshells(ctx, fullPath, maxDepth-1, names, dirs, cfg, findings)
continue
}
nameLower := strings.ToLower(name)
if names[nameLower] {
info, _ := osFS.Stat(fullPath)
var details string
if info != nil {
details = fmt.Sprintf("Size: %d, Mtime: %s", info.Size(), info.ModTime())
}
*findings = append(*findings, alert.Finding{
Severity: alert.Critical,
Check: "webshell",
Message: fmt.Sprintf("Known webshell found: %s", fullPath),
Details: details,
FilePath: fullPath,
})
}
// .haxor extension
if strings.HasSuffix(nameLower, ".haxor") || strings.HasSuffix(nameLower, ".cgix") {
*findings = append(*findings, alert.Finding{
Severity: alert.Critical,
Check: "webshell",
Message: fmt.Sprintf("Suspicious CGI file: %s", fullPath),
FilePath: fullPath,
})
}
// File permission anomalies - only check PHP-executable files to keep it fast
if isExecutablePHPName(nameLower) {
info, err := entry.Info()
markScanReadError(ctx, "webshells", err)
if err == nil {
mode := info.Mode()
// World-writable PHP
if mode&0002 != 0 {
*findings = append(*findings, alert.Finding{
Severity: alert.High,
Check: "world_writable_php",
Message: fmt.Sprintf("World-writable PHP file: %s", fullPath),
Details: fmt.Sprintf("Mode: %s", mode),
FilePath: fullPath,
})
}
// Note: executable PHP check removed - most PHP files on cPanel
// have +x due to suPHP/lsapi, making this too noisy.
}
}
}
}
// matchGlob reports whether path is covered by an operator suppression pattern.
//
// Matching is tried in order:
// 1. filepath.Match against the basename ("*.php", "*.log") and the full path.
// 2. For a leading-any-depth glob pattern, a substring match of the
// wildcard-stripped residue -- but ONLY when that residue still contains a
// path separator with literal content (e.g. "*/node_modules/*" ->
// "/node_modules/"). This preserves the "directory anywhere in the path, at
// any depth" intent without broadening anchored full-path globs like
// "/tmp/safe/*" into recursive subtree suppressions.
// 3. For a pattern with no wildcards, a literal substring match, so an operator
// can suppress a directory ("/uploads/") or a filename ("adminer.php").
//
// The separator requirement in step 2 is the fix for an over-suppression
// footgun: the previous code stripped every "*" and substring-matched the
// remainder, so "*.php" became the bare token ".php" and silenced every file
// whose path merely contained ".php" -- turning a narrow pattern into a
// whole-subtree allowlist an attacker could hide a webshell in.
// PathMatchesIgnore reports whether path is covered by any of the operator's
// suppressions.ignore_paths patterns, using the same glob semantics the
// content checks apply. Exported so the real-time watchers honour the same
// suppression list instead of maintaining a second interpretation of it.
func PathMatchesIgnore(path string, ignores []string) bool {
for _, ignore := range ignores {
if matchGlob(path, ignore) {
return true
}
}
return false
}
func matchGlob(path, pattern string) bool {
if pattern == "" {
return false
}
if strings.ContainsAny(pattern, "*?[") {
if matched, _ := filepath.Match(pattern, filepath.Base(path)); matched {
return true
}
if matched, _ := filepath.Match(pattern, path); matched {
return true
}
if strings.ContainsAny(pattern, "?[") || !hasLeadingAnyDepthGlob(pattern) {
return false
}
residue := strings.ReplaceAll(pattern, "*", "")
if strings.Contains(residue, "/") && strings.Trim(residue, "/") != "" {
return strings.Contains(path, residue)
}
return false
}
return strings.Contains(path, pattern)
}
func hasLeadingAnyDepthGlob(pattern string) bool {
firstSlash := strings.Index(pattern, "/")
if firstSlash <= 0 {
return false
}
return strings.Trim(pattern[:firstSlash], "*") == ""
}
package checks
import (
"context"
"fmt"
"net"
"net/netip"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/state"
)
// firewallHostIPv6Addrs is the host's global-unicast IPv6 address source;
// swapped in tests.
var firewallHostIPv6Addrs = hostGlobalIPv6Addrs
// hostGlobalIPv6Addrs returns the host's global-unicast IPv6 addresses.
// Loopback, link-local, ULA, and IPv4 do not count: only globally routable
// IPv6 makes the unmanaged-family bypass reachable from the internet.
func hostGlobalIPv6Addrs() ([]string, error) {
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil, fmt.Errorf("enumerate interface addresses: %w", err)
}
var out []string
for _, a := range addrs {
if ip := globalUnicastIPv6FromCIDR(a.String()); ip != "" {
out = append(out, ip)
}
}
return out, nil
}
func globalUnicastIPv6FromCIDR(cidr string) string {
addrText, prefixText, hasPrefix := strings.Cut(cidr, "/")
ip, err := netip.ParseAddr(addrText)
if err != nil {
return ""
}
if hasPrefix {
bits, err := strconv.Atoi(prefixText)
if err != nil || bits < 0 || bits > ip.BitLen() {
return ""
}
}
if ip.Is4() || ip.Is4In6() {
return ""
}
ip = ip.WithZone("")
if !ip.IsGlobalUnicast() || ip.IsPrivate() {
return ""
}
return ip.String()
}
func CheckFirewall(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
var findings []alert.Finding
if !cfg.Firewall.Enabled {
// Firewall not managed by CSM - skip nftables checks
return findings
}
// When firewall.ipv6 is false the engine inserts a blanket NFPROTO
// ipv6 accept ahead of the blocked sets and every port rule, so on a
// dual-stack host the entire IPv6 attack surface bypasses the
// DROP-policy firewall. That trade-off must never be silent.
if !cfg.Firewall.IPv6 {
addrs, err := firewallHostIPv6Addrs()
if err != nil {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "firewall_ipv6_unmanaged",
Message: "Unable to inspect host IPv6 addresses while firewall.ipv6 is disabled; CSM cannot determine whether the unmanaged IPv6 path exposes this host",
Timestamp: time.Now(),
})
} else if len(addrs) > 0 {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "firewall_ipv6_unmanaged",
Message: "IPv6 traffic bypasses the CSM firewall: firewall.ipv6 is disabled but this host has a global IPv6 address; all IPv6 traffic is accepted unfiltered - set firewall.ipv6: true",
Timestamp: time.Now(),
})
}
}
// The running engine pairs live rules with its applied baseline. Standalone
// checks have no engine and inspect the existing table through cmdExec.
var out []byte
var applied string
var err error
monitor, managed := getIPBlocker().(interface {
RulesetSnapshot() (current, applied string, err error)
})
if managed {
var current string
current, applied, err = monitor.RulesetSnapshot()
out = []byte(current)
} else {
// The listing the engine baselines: counters are traffic, not structure.
out, err = cmdExec.RunAllowNonZero("nft", "-s", "-t", "list", "table", "inet", "csm")
}
if err != nil {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "firewall",
Message: "CSM firewall table not found in nftables - rules may not be active",
Timestamp: time.Now(),
})
return findings
}
output := string(out)
required := []string{"chain input", "chain output", "set blocked_ips", "set allowed_ips", "set infra_ips"}
dropIPv4Options := cfg.Firewall.DropIPv4Options
if applied != "" {
// Restart and rollback can change the engine without replacing the
// daemon's startup config. Use the same applied snapshot as the hash
// check so a concurrent re-apply cannot split the expected policy.
dropIPv4Options = strings.Contains(applied, "chain prerouting")
}
if dropIPv4Options {
required = append(required, "chain prerouting")
}
for _, component := range required {
if !strings.Contains(output, component) {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "firewall",
Message: fmt.Sprintf("Firewall missing expected component: %s", component),
Timestamp: time.Now(),
})
}
}
// Hash the rule structure, excluding dynamic set members which change on
// every block/unblock.
hash := nftRulesetStructureHash(out)
// Only a successful engine Apply establishes a ruleset baseline. The
// unkeyed digest in csm.yaml can change without any firewall
// transaction, including when SIGHUP defers restart-required settings.
prev, exists := store.GetRaw("_nftables_rules_hash")
if applied != "" {
prev, exists = nftRulesetStructureHash([]byte(applied)), true
} else if managed {
findings = append(findings, alert.Finding{
Severity: alert.Warning, Check: "firewall", Timestamp: time.Now(),
Message: "Firewall integrity baseline unavailable after applying rules; re-apply the firewall to restore monitoring",
})
}
if exists && prev != hash {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "firewall",
Message: "nftables ruleset modified outside of CSM",
Timestamp: time.Now(),
})
}
// Retain the trusted baseline while a mismatch persists so acknowledging
// one finding cannot make the next check treat the modified rules as clean.
if applied != "" {
store.SetRaw("_nftables_rules_hash", prev)
} else if !exists {
store.SetRaw("_nftables_rules_hash", hash)
}
// Check for dangerous ports in config
findings = append(findings, checkDangerousPorts(cfg)...)
return findings
}
func nftRulesetStructureHash(out []byte) string {
out = []byte(firewall.StableRuleset(string(out)))
var stableLines []byte
inElements := false
elementBraceDepth := 0
for _, line := range strings.Split(string(out), "\n") {
trimmed := strings.TrimSpace(line)
if inElements {
elementBraceDepth += nftBraceDelta(trimmed)
if elementBraceDepth <= 0 {
inElements = false
elementBraceDepth = 0
}
continue
}
if depth, ok := nftElementsBlockDepth(trimmed); ok {
if depth > 0 {
inElements = true
elementBraceDepth = depth
}
continue
}
stableLines = append(stableLines, line...)
stableLines = append(stableLines, '\n')
}
return hashBytes(stableLines)
}
func nftElementsBlockDepth(line string) (int, bool) {
const key = "elements"
if !strings.HasPrefix(line, key) {
return 0, false
}
rest := line[len(key):]
if rest != "" && rest[0] != '=' && rest[0] != ' ' && rest[0] != '\t' {
return 0, false
}
rest = strings.TrimSpace(rest)
if !strings.HasPrefix(rest, "=") {
return 0, false
}
rest = strings.TrimSpace(rest[1:])
if !strings.HasPrefix(rest, "{") {
return 0, false
}
return nftBraceDelta(rest), true
}
func nftBraceDelta(line string) int {
depth := 0
inQuote := false
escaped := false
for _, r := range line {
if inQuote {
if escaped {
escaped = false
continue
}
switch r {
case '\\':
escaped = true
case '"':
inQuote = false
}
continue
}
switch r {
case '"':
inQuote = true
case '#':
return depth
case '{':
depth++
case '}':
depth--
}
}
return depth
}
func checkDangerousPorts(cfg *config.Config) []alert.Finding {
var findings []alert.Finding
dangerousPorts := make(map[int]bool)
for _, p := range cfg.BackdoorPorts {
dangerousPorts[p] = true
}
restricted := make(map[int]bool)
for _, p := range cfg.Firewall.RestrictedTCP {
restricted[p] = true
}
for _, port := range cfg.Firewall.TCPIn {
if dangerousPorts[port] {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "firewall_ports",
Message: fmt.Sprintf("Known backdoor port %d is open in firewall TCP_IN", port),
Timestamp: time.Now(),
})
}
if restricted[port] {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "firewall_ports",
Message: fmt.Sprintf("Restricted port %d found in public TCP_IN - should be infra-only", port),
Timestamp: time.Now(),
})
}
}
return findings
}
package checks
import (
"bufio"
"context"
"crypto/sha256"
"fmt"
"io"
"path/filepath"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
// splitValiasLine splits a valiases line "key: destinations" into its key
// and raw, still-quoted destination list. Returns empty strings for
// comments, blank lines, or malformed lines.
func splitValiasLine(line string) (key, rawDest string) {
line = strings.Trim(line, " \t\r\n\v\f")
if line == "" || strings.HasPrefix(line, "#") {
return "", ""
}
idx := strings.IndexByte(line, ':')
if idx < 0 {
return "", ""
}
return strings.TrimSpace(line[:idx]), strings.Trim(line[idx+1:], " \t\r\n\v\f")
}
// ValiasEntry is one destination of one valiases alias.
type ValiasEntry struct {
LocalPart string
Domain string
Dest string
}
// ParseValiasEntries reads a valiases file for fileDomain and returns one
// entry per destination. cPanel keys aliases by full address
// ("bob@example.com"); a bare key ("bob", "*") belongs to fileDomain.
func ParseValiasEntries(r io.Reader, fileDomain string) ([]ValiasEntry, error) {
var entries []ValiasEntry
scanner := bufio.NewScanner(r)
for scanner.Scan() {
key, rawDest := splitValiasLine(scanner.Text())
if key == "" || rawDest == "" {
continue
}
localPart, domain := key, fileDomain
if at := strings.LastIndexByte(key, '@'); at > 0 && at < len(key)-1 {
localPart, domain = key[:at], key[at+1:]
}
for _, d := range splitValiasDests(rawDest) {
if d == "" {
continue
}
entries = append(entries, ValiasEntry{LocalPart: localPart, Domain: domain, Dest: d})
}
}
return entries, scanner.Err()
}
// unquoteValiasDest strips one layer of matching double or single quotes.
// cPanel writes pipe and command destinations quoted ("|/path args"), and
// the pipe detector keys on the leading "|".
func unquoteValiasDest(s string) string {
s = strings.Trim(s, " \t\r\n\v\f")
if len(s) >= 2 && (s[0] == '"' || s[0] == '\'') && s[len(s)-1] == s[0] {
quote := s[0]
s = strings.Trim(s[1:len(s)-1], " \t\r\n\v\f")
// The redirect router removes one backslash layer in double-quoted
// pipes/files before the pipe transport splits command arguments.
if quote == '"' && (strings.HasPrefix(s, "|") || strings.HasPrefix(s, "/")) {
var b strings.Builder
for i := 0; i < len(s); i++ {
if s[i] == '\\' && i+1 < len(s) {
i++
}
b.WriteByte(s[i])
}
return b.String()
}
}
return s
}
// splitValiasDests splits a comma-separated destination list, keeping
// commas inside quotes with their destination, and unquotes each item.
func splitValiasDests(dest string) []string {
var out []string
var cur strings.Builder
var quote byte
for i := 0; i < len(dest); i++ {
c := dest[i]
switch {
case quote != 0:
if quote == '"' && c == '\\' && i+1 < len(dest) {
cur.WriteByte(c)
i++
cur.WriteByte(dest[i])
continue
}
if c == quote {
quote = 0
}
cur.WriteByte(c)
case c == '"' || c == '\'':
quote = c
cur.WriteByte(c)
case c == ',':
out = append(out, unquoteValiasDest(cur.String()))
cur.Reset()
default:
cur.WriteByte(c)
}
}
if quote != 0 {
// A malformed unmatched quote must not turn every later comma into
// quoted data and hide a pipe or external forwarder. Fall back to a
// conservative raw split; false positives are preferable to dropping
// the rest of an attacker-controlled valias line.
parts := strings.Split(dest, ",")
out = out[:0]
for _, part := range parts {
part = strings.Trim(part, " \t\r\n\v\f")
part = strings.Trim(strings.Trim(part, "\"'"), " \t\r\n\v\f")
out = append(out, part)
}
return out
}
out = append(out, unquoteValiasDest(cur.String()))
return out
}
// IsPipeForwarder returns true if the destination is a pipe forwarder,
// excluding pipes whose executed command is a cPanel built-in.
func IsPipeForwarder(dest string) bool {
return strings.HasPrefix(dest, "|") && !isSafePipe(dest)
}
// isDevNullForwarder returns true if the destination is /dev/null.
func isDevNullForwarder(dest string) bool {
return dest == "/dev/null"
}
// IsExternalDest returns true if the destination is an email address
// with a domain not in the local domains set. A pipe is a command, even when
// its arguments contain an address.
func IsExternalDest(dest string, localDomains map[string]bool) bool {
if strings.HasPrefix(dest, "|") {
return false
}
atIdx := strings.LastIndexByte(dest, '@')
if atIdx < 0 || atIdx >= len(dest)-1 {
return false
}
domain := strings.ToLower(dest[atIdx+1:])
return !localDomains[domain]
}
// parseVfilterExternalDests extracts external email destinations from vfilter content.
// Looks for `to "dest@domain"` directives.
func parseVfilterExternalDests(content string, localDomains map[string]bool) []string {
var external []string
lines := strings.Split(content, "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
// Look for: to "email@domain"
if !strings.HasPrefix(line, "to ") && !strings.HasPrefix(line, "to\t") {
continue
}
// Extract the quoted destination
quoteStart := strings.IndexByte(line, '"')
if quoteStart < 0 {
continue
}
rest := line[quoteStart+1:]
quoteEnd := strings.IndexByte(rest, '"')
if quoteEnd < 0 {
continue
}
dest := rest[:quoteEnd]
if IsExternalDest(dest, localDomains) {
external = append(external, dest)
}
}
return external
}
// parseLocalDomainsContent parses the content of /etc/localdomains or /etc/virtualdomains.
func parseLocalDomainsContent(content string) map[string]bool {
domains := make(map[string]bool)
lines := strings.Split(content, "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
// virtualdomains format: "domain: user" - take domain part
if idx := strings.IndexByte(line, ':'); idx > 0 {
line = strings.TrimSpace(line[:idx])
}
domains[strings.ToLower(line)] = true
}
return domains
}
// loadLocalDomains reads /etc/localdomains and /etc/virtualdomains.
func loadLocalDomains() map[string]bool {
domains := make(map[string]bool)
for _, path := range []string{"/etc/localdomains", "/etc/virtualdomains"} {
data, err := osFS.ReadFile(path)
if err != nil {
continue
}
for k, v := range parseLocalDomainsContent(string(data)) {
domains[k] = v
}
}
return domains
}
// IsKnownForwarder checks if a forwarder rule matches the known forwarders suppression list.
func IsKnownForwarder(localPart, domain, dest string, knownForwarders []string) bool {
entry := fmt.Sprintf("%s@%s: %s", localPart, domain, dest)
for _, known := range knownForwarders {
if strings.EqualFold(strings.TrimSpace(known), entry) {
return true
}
}
return false
}
// fileContentHash returns the SHA256 hex hash of a file's content.
func fileContentHash(path string) (string, error) {
data, err := osFS.ReadFile(path)
if err != nil {
return "", err
}
h := sha256.Sum256(data)
return fmt.Sprintf("%x", h[:]), nil
}
// CheckForwarders audits all valiases and vfilters files for dangerous forwarder
// patterns. Uses internal throttle: skips if last refresh was less than
// password_check_interval_min ago (reuses the same interval).
func CheckForwarders(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
db := store.Global()
if db == nil {
return nil
}
// Internal throttle (24h) - reuse PasswordCheckIntervalMin
if !ForceAll {
lastRefreshStr := db.GetMetaString("email:fwd_last_refresh")
if lastRefreshStr != "" {
if lastRefresh, err := time.Parse(time.RFC3339, lastRefreshStr); err == nil {
interval := time.Duration(cfg.EmailProtection.PasswordCheckIntervalMin) * time.Minute
if time.Since(lastRefresh) < interval {
// A skipped cycle examined nothing, so it must not let the
// runner retire what the last run found.
markCheckSkipped(ctx, "email_forwarder_audit")
return nil
}
}
}
}
if ctx.Err() != nil {
return nil
}
localDomains := loadLocalDomains()
var findings []alert.Finding
// Audit valiases. Rank by mtime desc so recently-changed mail
// domains process first when the check timeout cuts iteration short.
if ctx.Err() != nil {
return findings
}
maxFiles := accountScanMaxFiles(ctx, cfg)
baselineComplete := true
valiasFiles, err := osFS.Glob("/etc/valiases/*")
if err != nil {
baselineComplete = false
}
baselineComplete = baselineComplete && scanCoversAllFiles(valiasFiles, maxFiles)
rankedValiasFiles := rankPathsByMtimeDesc(ctx, valiasFiles, maxFiles)
if ctx.Err() != nil {
return findings
}
for _, path := range rankedValiasFiles {
if ctx.Err() != nil {
return findings
}
domain := filepath.Base(path)
entries, ok := auditValiasFileWithStatus(path, domain, localDomains, cfg)
if !ok {
baselineComplete = false
}
findings = append(findings, entries...)
}
// Audit vfilters
if ctx.Err() != nil {
return findings
}
vfilterFiles, err := osFS.Glob("/etc/vfilters/*")
if err != nil {
baselineComplete = false
}
baselineComplete = baselineComplete && scanCoversAllFiles(vfilterFiles, maxFiles)
rankedVfilterFiles := rankPathsByMtimeDesc(ctx, vfilterFiles, maxFiles)
if ctx.Err() != nil {
return findings
}
for _, path := range rankedVfilterFiles {
if ctx.Err() != nil {
return findings
}
domain := filepath.Base(path)
entries, ok := auditVfilterFileWithStatus(path, domain, localDomains, cfg)
if !ok {
baselineComplete = false
}
findings = append(findings, entries...)
}
if ctx.Err() != nil {
return findings
}
baselineExists := db.GetMetaString("email:fwd_last_refresh") != ""
if baselineComplete || baselineExists {
_ = db.SetMetaString("email:fwd_last_refresh", time.Now().Format(time.RFC3339))
}
return findings
}
// forwarderFileIsNew reports whether a forwarder/filter file should be
// treated as newly added. A stored hash that differs means the file changed.
// No stored hash means one of two things: before the first complete audit
// (no baseline marker) it is pre-existing install backlog and stays quiet;
// after the baseline it genuinely appeared post-audit -- the classic BEC
// drop the first-sight suppression used to silence forever, because the
// next scan saw an unchanged hash.
func forwarderFileIsNew(db *store.DB, baselineKey, hashKey, currentHash string) bool {
old, found := db.GetForwarderHash(hashKey)
if found {
return old != currentHash
}
return db.GetMetaString(baselineKey) != ""
}
func scanCoversAllFiles(paths []string, maxFiles int) bool {
return maxFiles <= 0 || len(paths) <= maxFiles
}
// auditValiasFileWithStatus parses a valiases file and returns findings for
// dangerous entries, and whether the audit completed.
func auditValiasFileWithStatus(path, domain string, localDomains map[string]bool, cfg *config.Config) ([]alert.Finding, bool) {
f, err := osFS.Open(path)
if err != nil {
return nil, false
}
defer f.Close()
db := store.Global()
var findings []alert.Finding
complete := true
isNew := false
hashKey := "valiases:" + domain
var currentHash string
if db != nil {
var hashErr error
currentHash, hashErr = fileContentHash(path)
if hashErr == nil {
isNew = forwarderFileIsNew(db, "email:fwd_last_refresh", hashKey, currentHash)
} else {
complete = false
}
}
entries, err := ParseValiasEntries(f, domain)
if err != nil {
complete = false
}
for _, e := range entries {
localPart, mailDomain, d := e.LocalPart, e.Domain, e.Dest
if IsKnownForwarder(localPart, mailDomain, d, cfg.EmailProtection.KnownForwarders) {
continue
}
newContext := ""
if isNew {
newContext = " (newly added)"
}
if IsPipeForwarder(d) {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "email_pipe_forwarder",
Message: fmt.Sprintf("Pipe forwarder detected: %s@%s -> %s%s", localPart, mailDomain, d, newContext),
Details: fmt.Sprintf("Domain: %s\nLocal part: %s\nDestination: %s\nFile: %s\nPipe forwarders execute arbitrary commands on incoming mail.", mailDomain, localPart, d, path),
Domain: domain,
TenantID: MailOwner(domain),
})
continue
}
if isDevNullForwarder(d) {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "email_suspicious_forwarder",
Message: fmt.Sprintf("Mail blackhole: %s@%s -> /dev/null%s", localPart, mailDomain, newContext),
Details: fmt.Sprintf("Domain: %s\nLocal part: %s\nDestination: /dev/null\nFile: %s\nAll mail to this address is silently discarded.", mailDomain, localPart, path),
Domain: domain,
TenantID: MailOwner(domain),
})
continue
}
if IsExternalDest(d, localDomains) && isNew {
severity := alert.High
msg := fmt.Sprintf("External forwarder: %s@%s -> %s%s", localPart, mailDomain, d, newContext)
if localPart == "*" {
msg = fmt.Sprintf("Wildcard catch-all to external: *@%s -> %s%s", mailDomain, d, newContext)
}
findings = append(findings, alert.Finding{
Severity: severity,
Check: "email_suspicious_forwarder",
Message: msg,
Details: fmt.Sprintf("Domain: %s\nLocal part: %s\nDestination: %s\nFile: %s", mailDomain, localPart, d, path),
Domain: domain,
TenantID: MailOwner(domain),
})
}
}
if complete && db != nil {
if err := db.SetForwarderHash(hashKey, currentHash); err != nil {
complete = false
}
}
return findings, complete
}
// auditVfilterFileWithStatus parses a vfilters file and returns findings for
// external destinations, and whether the audit completed.
func auditVfilterFileWithStatus(path, domain string, localDomains map[string]bool, cfg *config.Config) ([]alert.Finding, bool) {
data, err := osFS.ReadFile(path)
if err != nil {
return nil, false
}
db := store.Global()
content := string(data)
complete := true
// Same newness logic as valiases above.
isNew := false
if db != nil {
currentHash := fmt.Sprintf("%x", sha256.Sum256(data))
isNew = forwarderFileIsNew(db, "email:fwd_last_refresh", "vfilters:"+domain, currentHash)
if err := db.SetForwarderHash("vfilters:"+domain, currentHash); err != nil {
complete = false
}
}
externalDests := parseVfilterExternalDests(content, localDomains)
var findings []alert.Finding
for _, dest := range externalDests {
// Suppression check - use "*" as localPart for vfilter entries
if IsKnownForwarder("*", domain, dest, cfg.EmailProtection.KnownForwarders) {
continue
}
// Only alert when the vfilter file actually changed; existing forwarders
// are normal customer configuration, not an attack indicator.
if !isNew {
continue
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "email_suspicious_forwarder",
Message: fmt.Sprintf("External destination in vfilter: %s -> %s (newly added)", domain, dest),
Details: fmt.Sprintf("Domain: %s\nDestination: %s\nFile: %s\nA mail filter rule forwards messages to an external address.", domain, dest, path),
})
}
return findings, complete
}
package checks
import (
"bytes"
"encoding/json"
"fmt"
"hash/fnv"
"io"
"os"
"sort"
"strconv"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/state"
)
const (
maxCatchUpBytes = 8 << 20
maxTrackedIPs = 4096
fingerprintBytes = 512
)
// followState is the persisted position+identity of the syslog follower.
type followState struct {
Offset int64 `json:"offset"`
HeadLen int `json:"head_len"`
HeadFP string `json:"head_fp"`
AnchorLen int `json:"anchor_len"`
AnchorFP string `json:"anchor_fp"`
}
// ftpFailTracker is the persisted detector state: where we last read to, and a
// per-IP sliding window of pure-ftpd auth-failure counts bucketed by minute.
type ftpFailTracker struct {
Follow followState `json:"follow"`
Buckets map[string]map[int64]int `json:"buckets"`
}
func newFTPFailTracker() *ftpFailTracker {
return &ftpFailTracker{Buckets: map[string]map[int64]int{}}
}
func (t *ftpFailTracker) record(ip string, at time.Time) {
if t.Buckets == nil {
t.Buckets = map[string]map[int64]int{}
}
m := t.Buckets[ip]
if m == nil {
m = map[int64]int{}
t.Buckets[ip] = m
}
m[at.Unix()/60]++
}
// evict drops minute buckets older than windowMin minutes and any IP left empty.
func (t *ftpFailTracker) evict(now time.Time, windowMin int) {
cutoff := now.Unix()/60 - int64(windowMin)
for ip, m := range t.Buckets {
for minute := range m {
if minute < cutoff {
delete(m, minute)
}
}
if len(m) == 0 {
delete(t.Buckets, ip)
}
}
}
// capIPs bounds the tracked-IP set, evicting the weakest candidates first.
// Count wins over recency so a flood of one-off sources cannot push an IP that
// is close to the brute-force threshold out of the retained window.
func (t *ftpFailTracker) capIPs(max int) {
if len(t.Buckets) <= max {
return
}
type ipScore struct {
ip string
recent int64
count int
}
scores := make([]ipScore, 0, len(t.Buckets))
for ip, m := range t.Buckets {
var recent int64
var count int
for minute := range m {
if minute > recent {
recent = minute
}
count += m[minute]
}
scores = append(scores, ipScore{ip: ip, recent: recent, count: count})
}
sort.Slice(scores, func(i, j int) bool {
if scores[i].count != scores[j].count {
return scores[i].count < scores[j].count
}
if scores[i].recent != scores[j].recent {
return scores[i].recent < scores[j].recent
}
return scores[i].ip < scores[j].ip
})
for i := 0; i < len(scores)-max; i++ {
delete(t.Buckets, scores[i].ip)
}
}
// count returns the summed failure count for ip across the retained buckets.
func (t *ftpFailTracker) count(ip string) int {
sum := 0
for _, c := range t.Buckets[ip] {
sum += c
}
return sum
}
type ftpOffender struct {
IP string
Count int
}
// offenders returns IPs whose summed failure count over the retained buckets is
// at least threshold, sorted by IP for stable output.
func (t *ftpFailTracker) offenders(threshold int) []ftpOffender {
var out []ftpOffender
for ip, m := range t.Buckets {
sum := 0
for _, c := range m {
sum += c
}
if sum >= threshold {
out = append(out, ftpOffender{ip, sum})
}
}
sort.Slice(out, func(i, j int) bool { return out[i].IP < out[j].IP })
return out
}
// fpAt returns the fnv64a fingerprint (hex) of n bytes at offset off.
// n <= 0 returns "".
func fpAt(f *os.File, off, n int64) (string, error) {
if n <= 0 {
return "", nil
}
buf := make([]byte, n)
read, err := f.ReadAt(buf, off)
if err != nil {
return "", err
}
if read != len(buf) {
return "", io.ErrUnexpectedEOF
}
h := fnv.New64a()
_, _ = h.Write(buf)
return strconv.FormatUint(h.Sum64(), 16), nil
}
// nextNewline returns the byte offset just after the first '\n' at or after
// from, or size if none is found before size.
func nextNewline(f *os.File, from, size int64) (int64, error) {
buf := make([]byte, 1)
for pos := from; pos < size; pos++ {
read, err := f.ReadAt(buf, pos)
if err != nil {
return 0, err
}
if read != len(buf) {
return 0, io.ErrUnexpectedEOF
}
if buf[0] == '\n' {
return pos + 1, nil
}
}
return size, nil
}
// syslogRecord is one line, the offset where it starts and the stream of the
// file it was read from (empty when the file has no identity, or when the line
// is unterminated: a writer can still append to it).
type syslogRecord struct {
text string
offset int64
stream string
unterminated bool
}
// observation names the record as every reader of its file names it; zero
// when the file has no identity.
func (r syslogRecord) observation(producer admission.ProducerID, at time.Time) alert.Observation {
if r.stream == "" {
return alert.Observation{}
}
return alert.Observation{Producer: string(producer), Stream: r.stream, Cursor: alert.RecordCursor(r.offset, r.text), ObservedAt: at.UTC()}
}
// completeRecords splits data, which starts at offset start, into complete
// newline-terminated lines with their offsets, skipping blank lines, and
// returns the bytes consumed through the last '\n'. Trailing partial bytes
// are neither returned nor consumed.
func completeRecords(data []byte, start int64) ([]syslogRecord, int64) {
lastNL := bytes.LastIndexByte(data, '\n')
if lastNL < 0 {
return nil, 0
}
var records []syslogRecord
pos := 0
for pos <= lastNL {
end := pos + bytes.IndexByte(data[pos:], '\n')
if end > pos {
records = append(records, syslogRecord{text: string(data[pos:end]), offset: start + int64(pos)})
}
pos = end + 1
}
return records, int64(lastNL + 1)
}
// chooseStart returns the byte offset to begin reading from, plus skipped bytes
// for first-run catch-up. It restarts at 0 on truncate / rotate / anchor
// mismatch, and follows from st.Offset on a verified append.
func chooseStart(f *os.File, st followState, curSize int64) (int64, int64, error) {
zero := st.Offset == 0 && st.HeadLen == 0 && st.HeadFP == "" && st.AnchorLen == 0 && st.AnchorFP == ""
if zero {
start := curSize - maxCatchUpBytes
if start <= 0 {
return 0, 0, nil
}
aligned, err := nextNewline(f, start, curSize)
if err != nil {
return 0, 0, err
}
return aligned, aligned, nil // skipped == bytes before aligned
}
if curSize < st.Offset || curSize < int64(st.HeadLen) {
return 0, 0, nil // truncation / copytruncate
}
if st.HeadLen > 0 {
fp, err := fpAt(f, 0, int64(st.HeadLen))
if err != nil {
return 0, 0, err
}
if fp != st.HeadFP {
return 0, 0, nil // different-head rotate
}
}
if st.Offset > 0 && (st.AnchorLen == 0 || st.AnchorFP == "") {
return 0, 0, nil // incomplete stored anchor
}
if st.Offset > 0 {
fp, err := fpAt(f, st.Offset-int64(st.AnchorLen), int64(st.AnchorLen))
if err != nil {
return 0, 0, err
}
if fp != st.AnchorFP {
return 0, 0, nil // same-head replacement
}
}
return st.Offset, 0, nil
}
// fillIdentity sets next's head and anchor fingerprints from the file content.
func fillIdentity(f *os.File, st *followState, curSize int64) error {
headLen := int64(fingerprintBytes)
if curSize < headLen {
headLen = curSize
}
hfp, err := fpAt(f, 0, headLen)
if err != nil {
return err
}
st.HeadLen = int(headLen)
st.HeadFP = hfp
anchorLen := int64(fingerprintBytes)
if st.Offset < anchorLen {
anchorLen = st.Offset
}
afp, err := fpAt(f, st.Offset-anchorLen, anchorLen)
if err != nil {
return err
}
st.AnchorLen = int(anchorLen)
st.AnchorFP = afp
return nil
}
// readNewSyslogLines reads complete lines appended to path since st, returning
// the new lines, the next follow state, bytes skipped by the catch-up cap, and
// any I/O error. On error, next == st so the caller leaves stored state intact.
func readNewSyslogLines(path string, st followState) ([]string, followState, int64, error) {
records, next, skipped, err := readNewSyslogRecords(path, st)
lines := make([]string, len(records))
for i, r := range records {
lines[i] = r.text
}
return lines, next, skipped, err
}
// readNewSyslogRecords is readNewSyslogLines with each line's start offset.
func readNewSyslogRecords(path string, st followState) ([]syslogRecord, followState, int64, error) {
f, err := osFS.Open(path)
if err != nil {
return nil, st, 0, err
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil {
return nil, st, 0, err
}
curSize := info.Size()
start, skipped, err := chooseStart(f, st, curSize)
if err != nil {
return nil, st, 0, err
}
if curSize-start > maxCatchUpBytes {
capped := curSize - maxCatchUpBytes
aligned, aerr := nextNewline(f, capped, curSize)
if aerr != nil {
return nil, st, 0, aerr
}
skipped += aligned - start
start = aligned
}
framed := true
if start > 0 {
// Catch-up may have reached EOF inside an unfinished line. Its
// eventual suffix still feeds legacy checks but names no record.
var before [1]byte
if _, err := f.ReadAt(before[:], start-1); err != nil {
return nil, st, 0, err
}
framed = before[0] == '\n'
}
var records []syslogRecord
var consumed int64
var data []byte
if curSize-start > 0 {
data = make([]byte, curSize-start)
read, rerr := f.ReadAt(data, start)
if rerr != nil {
return nil, st, 0, rerr
}
if read != len(data) {
return nil, st, 0, io.ErrUnexpectedEOF
}
records, consumed = completeRecords(data, start)
}
next := followState{Offset: start + consumed}
if err := fillIdentity(f, &next, curSize); err != nil {
return nil, st, 0, err
}
if id, ok := selfWriteIdentityFromFileInfo(info); ok {
stream := alert.FileStream(id.Device, id.Inode)
for i := range records {
if framed || records[i].offset > start {
records[i].stream = stream
}
}
}
return records, next, skipped, nil
}
// ftpTrackerKey is underscore-prefixed so state.Store.Update does not prune it
// as a stale non-finding key after 24h.
const ftpTrackerKey = "_ftp_fail_tracker"
func loadFTPFailTracker(store *state.Store) *ftpFailTracker {
raw, ok := store.GetRaw(ftpTrackerKey)
if !ok || raw == "" {
return newFTPFailTracker()
}
var decoded ftpFailTracker
if err := json.Unmarshal([]byte(raw), &decoded); err != nil {
return newFTPFailTracker()
}
if invalidFollowState(decoded.Follow) {
return newFTPFailTracker()
}
if decoded.Buckets == nil {
decoded.Buckets = map[string]map[int64]int{}
}
return &decoded
}
// invalidFollowState reports stored follow state that is impossible or lacks
// the identity needed to verify a claimed offset; such state is dropped so the
// reader falls back to a bounded first-run catch-up.
func invalidFollowState(st followState) bool {
if st.Offset < 0 || st.HeadLen < 0 || st.HeadLen > fingerprintBytes ||
st.AnchorLen < 0 || st.AnchorLen > fingerprintBytes ||
int64(st.AnchorLen) > st.Offset {
return true
}
if st.Offset <= 0 {
return false
}
return st.HeadLen <= 0 || st.HeadFP == "" || st.AnchorLen <= 0 || st.AnchorFP == ""
}
func (t *ftpFailTracker) save(store *state.Store) {
b, err := json.Marshal(t)
if err != nil {
return
}
if err := store.SetRawAndSave(ftpTrackerKey, string(b)); err != nil {
fmt.Fprintf(os.Stderr, "state: error saving FTP fail tracker: %v\n", err)
}
}
const ftpSyslogPath = "/var/log/messages"
// effectiveFTPFailWindowMin returns the operator-configured sliding-window
// length in minutes, or the built-in default (30) when unset.
func effectiveFTPFailWindowMin(cfg *config.Config) int {
if cfg == nil || cfg.Thresholds.FTPFailWindowMin <= 0 {
return 30
}
return cfg.Thresholds.FTPFailWindowMin
}
var (
ftpTrackerMu sync.Mutex
ftpSkippedBytes *metrics.Counter
ftpSkippedBytesOnce sync.Once
)
func observeFTPSkippedBytes(n int64) {
ftpSkippedBytesOnce.Do(func() {
ftpSkippedBytes = metrics.NewCounter(
"csm_checks_ftp_syslog_skipped_bytes_total",
"Bytes of /var/log/messages skipped by the FTP detector catch-up cap (8 MiB). Steady growth means the detector is falling behind the syslog write rate or the log has large bursts between cycles.",
)
metrics.MustRegister("csm_checks_ftp_syslog_skipped_bytes_total", ftpSkippedBytes)
})
ftpSkippedBytes.Add(float64(n))
}
package checks
import (
"context"
"sync"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
type ftpScanObservationKey struct{}
// A check can finish before the tier is cancelled by another check. Keep its
// cursor and findings together until the runner accepts the whole scan.
type ftpScanObservation struct {
mu sync.Mutex
initialRaw string
tracker *ftpFailTracker
findings []alert.Finding
}
func (o *ftpScanObservation) prepare(initialRaw string, tracker *ftpFailTracker, findings []alert.Finding) {
o.mu.Lock()
defer o.mu.Unlock()
o.initialRaw, o.tracker, o.findings = initialRaw, tracker, findings
}
func (o *ftpScanObservation) complete(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
o.mu.Lock()
initialRaw, tracker, findings := o.initialRaw, o.tracker, o.findings
o.tracker, o.findings = nil, nil
o.mu.Unlock()
if tracker == nil {
return nil
}
ftpTrackerMu.Lock()
currentRaw, _ := store.GetRaw(ftpTrackerKey)
if currentRaw == initialRaw {
defer ftpTrackerMu.Unlock()
if ctx.Err() != nil {
return nil
}
tracker.save(store)
return findings
}
ftpTrackerMu.Unlock()
// Another accepted scan advanced the cursor while this tier was running.
// Re-read from its position so we neither restore stale counters nor
// dispatch an already consumed burst. The retry commits directly.
ctx = context.WithValue(ctx, ftpScanObservationKey{}, (*ftpScanObservation)(nil))
return CheckFTPLogins(ctx, cfg, store)
}
package checks
import (
"fmt"
"os"
"strings"
"github.com/pidginhost/csm/internal/alert"
)
// eligibleFullScanChecks is the set of check types that map to a pure file
// quarantine (fixQuarantine) in ApplyFix. These are the only checks eligible
// for full-scan --quarantine remediation.
//
// Explicitly excluded:
// - backdoor_binary, new_executable_in_config → fixKillAndQuarantine (process kill forbidden)
// - htaccess_* → file edit, not a move
// - email_phishing_content → Exim spool, not a regular file
// - suspicious_crontab → crontab truncate, not a pure file move
// - world_writable_php, group_writable_php → chmod, not a move
var eligibleFullScanChecks = map[string]bool{
"webshell": true,
"new_webshell_file": true,
"obfuscated_php": true,
"suspicious_php_content": true,
"new_php_in_languages": true,
"new_php_in_upgrade": true,
"phishing_page": true,
"phishing_directory": true,
}
// QuarantineFindingFile quarantines the file a malware/webshell finding points
// at, for the full-scan --quarantine path. It reuses fixQuarantine (move to the
// quarantine dir + .meta sidecar) and deliberately covers ONLY the pure
// file-quarantine check set — it never kills processes, cleans databases, or
// touches the firewall. Returns eligible=false for any finding that is not a
// quarantinable malware/webshell FILE finding (caller marks those
// "left_for_review").
//
// The job runs unattended, so it gets the same bar the scheduled auto-response
// applies: only a Critical finding (two converging indicators) may act, only on
// a regular file that is not a symlink, never on a whole directory, and a
// WordPress core, plugin or theme file is cleaned in place rather than moved,
// because moving it takes the site down while only the injected code had to go.
func QuarantineFindingFile(f alert.Finding) (RemediationResult, bool) {
if !eligibleFullScanChecks[f.Check] || f.FilePath == "" || f.Severity != alert.Critical {
return RemediationResult{}, false
}
info, err := osFS.Lstat(f.FilePath)
if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return RemediationResult{}, false
}
if ShouldCleanInsteadOfQuarantine(f.FilePath) {
clean := cleanInfectedFileIdentified(f.FilePath, info)
switch {
case clean.WholeFile:
// Nothing of the file is the site's; a clean would leave a stub.
return quarantineWholeFile(f.FilePath, clean), true
case clean.Cleaned:
return RemediationResult{
Success: true,
Action: fmt.Sprintf("cleaned %s in place", f.FilePath),
Description: fmt.Sprintf("Removed: %s (backup: %s)", strings.Join(clean.Removals, "; "), clean.BackupPath),
RemediationStatus: "cleaned",
}, true
case clean.Refused || clean.Error == "":
// Nothing the cleaner recognises: a core file with no removable
// injection is an operator decision, not a move.
return RemediationResult{}, false
default:
// Moving a WordPress core, plugin or theme file after the safer
// clean failed defeats this branch's purpose and can take the site
// down. Leave the file in place and report the failed remediation.
return RemediationResult{Error: fmt.Sprintf("cleaning %s in place: %s", f.FilePath, clean.Error)}, true
}
}
return fixQuarantine(f.FilePath), true
}
// The cleaner's refusal describes bytes it read earlier, not whatever now
// occupies the pathname. Retain that evidence through the quarantine lease.
func quarantineWholeFile(path string, clean CleanResult) RemediationResult {
path, _, err := resolveExistingFixPath(path, effectiveFixRoots(fixQuarantineAllowedRoots, quarantineExtraRoots...))
if err != nil {
return RemediationResult{Error: err.Error()}
}
qPath := newQuarantinePath(quarantineDir, path)
meta := quarantineMetadata(path, clean.wholeFileInfo, "Whole-file malware")
err = quarantineVersionedTargetFD(path, qPath, clean.wholeFileInfo, meta, -1, clean.wholeFileContent)
warning := ""
if err != nil {
var completed bool
warning, completed = completedQuarantineWarning(err)
if !completed {
return RemediationResult{Error: err.Error()}
}
}
description := fmt.Sprintf("Moved to quarantine: %s", qPath)
if warning != "" {
description += ". Warning: " + warning
}
return RemediationResult{Success: true, Action: fmt.Sprintf("quarantined %s", path), Description: description}
}
package checks
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// CheckOpenBasedir verifies that each cPanel account has proper PHP
// isolation via CageFS and/or open_basedir.
// Flags accounts where CageFS is disabled AND open_basedir is not set.
func CheckOpenBasedir(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
// Check global CageFS mode
cageFSMode := getCageFSMode()
// Get list of disabled CageFS users (if mode is "Enable All", check exceptions)
disabledUsers := getCageFSDisabledUsers(cageFSMode)
users := cPanelUsersForOpenBasedirScan(ctx)
if cageFSMode == "unknown" && len(disabledUsers) == 0 {
for _, user := range users {
disabledUsers[user] = true
}
}
// For users without CageFS, check if open_basedir is set
for _, user := range users {
// Skip if CageFS is active for this user
if !disabledUsers[user] {
continue
}
// CageFS is disabled for this user - check open_basedir
if !hasOpenBasedir(user) {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "open_basedir",
Message: fmt.Sprintf("Account %s has no PHP isolation: CageFS disabled and no open_basedir", user),
Details: "This account's PHP scripts can read any file on the server including other accounts' wp-config.php and /etc/shadow",
})
}
}
return findings
}
func cPanelUsersForOpenBasedirScan(ctx context.Context) []string {
if account := AccountFromContext(ctx); account != "" {
if _, err := osFS.Stat(filepath.Join("/var/cpanel/users", account)); err != nil {
return nil
}
return []string{account}
}
userDirs, _ := osFS.ReadDir("/var/cpanel/users")
users := make([]string, 0, len(userDirs))
for _, userEntry := range userDirs {
users = append(users, userEntry.Name())
}
return users
}
func getCageFSMode() string {
// CageFS mode file
data, err := osFS.ReadFile("/etc/cagefs/cagefs.mp")
if err != nil {
return "unknown"
}
content := strings.TrimSpace(string(data))
if content != "" {
return "enabled"
}
return "unknown"
}
func getCageFSDisabledUsers(mode string) map[string]bool {
disabled := make(map[string]bool)
// Check cagefsctl --list-disabled
out, _ := runCmd("cagefsctl", "--list-disabled")
if out != nil {
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
user := strings.TrimSpace(line)
if user != "" {
disabled[user] = true
}
}
}
// If mode is unknown (no CageFS), all users are "disabled"
if mode == "unknown" && len(disabled) == 0 {
userDirs, _ := osFS.ReadDir("/var/cpanel/users")
for _, u := range userDirs {
disabled[u.Name()] = true
}
}
return disabled
}
func hasOpenBasedir(user string) bool {
// Check .user.ini in public_html
userIni := filepath.Join(accountHomeDir(user), "public_html", ".user.ini")
if data, err := osFS.ReadFile(userIni); err == nil {
if strings.Contains(strings.ToLower(string(data)), "open_basedir") {
return true
}
}
// Check .htaccess for php_value open_basedir
htaccess := filepath.Join(accountHomeDir(user), "public_html", ".htaccess")
if data, err := osFS.ReadFile(htaccess); err == nil {
if strings.Contains(strings.ToLower(string(data)), "open_basedir") {
return true
}
}
// Check per-user PHP config set via cPanel MultiPHP
phpConfDirs, _ := osFS.Glob("/opt/cpanel/ea-php*/root/etc/php.d/")
for _, confDir := range phpConfDirs {
userConf := filepath.Join(confDir, "local.ini")
if data, err := osFS.ReadFile(userConf); err == nil {
if strings.Contains(string(data), "open_basedir") {
return true
}
}
}
return false
}
// CheckSymlinkAttacks detects symbolic links inside user public_html
// directories that point outside the account's own directory.
// This is a classic shared hosting attack to read other users' files.
func CheckSymlinkAttacks(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
homeDirs, _ := GetScanHomeDirs(ctx)
for _, homeEntry := range homeDirs {
if !homeEntry.IsDir() {
continue
}
user := homeEntry.Name()
homeDir := scanHomeDirPath(homeEntry)
docRoot := filepath.Join(homeDir, "public_html")
scanForMaliciousSymlinks(docRoot, user, homeDir, 4, &findings)
}
return findings
}
func scanForMaliciousSymlinks(dir, user, homeDir string, maxDepth int, findings *[]alert.Finding) {
if maxDepth <= 0 {
return
}
entries, err := osFS.ReadDir(dir)
if err != nil {
return
}
for _, entry := range entries {
fullPath := filepath.Join(dir, entry.Name())
// Check if it's a symlink
if entry.Type()&os.ModeSymlink == 0 {
if entry.IsDir() {
scanForMaliciousSymlinks(fullPath, user, homeDir, maxDepth-1, findings)
}
continue
}
// It's a symlink - read the target
target, err := osFS.Readlink(fullPath)
if err != nil {
continue
}
// Resolve relative targets
if !filepath.IsAbs(target) {
target = filepath.Join(filepath.Dir(fullPath), target)
}
target = filepath.Clean(target)
// Check if target is outside the user's home
if isSymlinkSafe(target, homeDir) {
continue
}
// Check if target points to another user's home
if _, targetUser, ok := accountRootOf(target); ok {
if targetUser != user {
*findings = append(*findings, alert.Finding{
Severity: alert.Critical,
Check: "symlink_attack",
Message: fmt.Sprintf("Symlink to another user's directory: %s -> %s", fullPath, target),
Details: fmt.Sprintf("User: %s, target user: %s\nThis could be used to read other accounts' files", user, targetUser),
FilePath: fullPath,
TenantID: user,
})
continue
}
}
// Check if target points to sensitive system files
sensitiveTargets := []string{"/etc/shadow", "/etc/passwd", "/root/"}
for _, sens := range sensitiveTargets {
if strings.HasPrefix(target, sens) {
*findings = append(*findings, alert.Finding{
Severity: alert.Critical,
Check: "symlink_attack",
Message: fmt.Sprintf("Symlink to sensitive system file: %s -> %s", fullPath, target),
Details: fmt.Sprintf("User: %s", user),
FilePath: fullPath,
TenantID: user,
})
break
}
}
}
}
func isSymlinkSafe(target, homeDir string) bool {
// Inside own home directory
if strings.HasPrefix(target, homeDir+"/") || target == homeDir {
return true
}
// Standard cPanel-created symlinks
safeTargets := []string{
"/etc/apache2/logs/",
"/usr/local/apache/logs/",
"/var/cpanel/",
"/opt/cpanel/",
"/usr/",
"/var/lib/mysql/",
"/var/run/",
}
for _, safe := range safeTargets {
if strings.HasPrefix(target, safe) {
return true
}
}
return false
}
package checks
import (
"context"
"encoding/hex"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"syscall"
"time"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/sshdconf"
"github.com/pidginhost/csm/internal/store"
)
// auditCmdTimeout is the per-subprocess timeout for audit checks.
// Audit checks are fast config reads, not heavy scans.
const auditCmdTimeout = 10 * time.Second
// RunHardeningAudit runs all hardening checks and returns a report.
// Pure function — reads system state only, no store access.
func RunHardeningAudit(cfg *config.Config) *store.AuditReport {
serverType := detectServerType()
var results []store.AuditResult
results = append(results, auditSSH()...)
results = append(results, auditPHP(serverType)...)
results = append(results, auditWebServer()...)
results = append(results, auditMail()...)
if serverType != "bare" {
results = append(results, auditCPanel(serverType)...)
}
results = append(results, auditOS()...)
results = append(results, auditFirewall()...)
score := 0
for _, r := range results {
if r.Status == "pass" {
score++
}
}
return &store.AuditReport{
Timestamp: time.Now(),
ServerType: serverType,
Results: results,
Score: score,
Total: len(results),
}
}
func detectServerType() string {
info := platform.Detect()
if info.IsCPanel() {
if info.OS == platform.OSCloudLinux {
return "cloudlinux"
}
return "cpanel"
}
return "bare"
}
// auditRunCmd executes a command with the audit-specific timeout via the
// cmdExec injector so tests can mock systemctl/cagefsctl/etc. without
// invoking real binaries on the host.
func auditRunCmd(name string, args ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), auditCmdTimeout)
defer cancel()
out, err := cmdExec.RunContext(ctx, name, args...)
if ctx.Err() == context.DeadlineExceeded {
return nil, fmt.Errorf("command timed out: %s", name)
}
return out, err
}
// --- SSH checks ---
var sshdConfigPath = sshdconf.DefaultPath
type sshdSettings struct {
PasswordAuthentication string
PermitRootLogin string
X11Forwarding string
}
func parseSSHDConfig() *sshdconf.Config {
return sshdconf.Parse(osFS, sshdConfigPath)
}
func settingsFromSSHDConfig(parsed *sshdconf.Config) sshdSettings {
return sshdSettings{
PasswordAuthentication: parsed.Value("passwordauthentication"),
PermitRootLogin: parsed.Value("permitrootlogin"),
X11Forwarding: parsed.Value("x11forwarding"),
}
}
// portsLabel renders a port list for operator-facing audit messages.
func portsLabel(ports []int) string {
parts := make([]string, 0, len(ports))
for _, p := range ports {
parts = append(parts, strconv.Itoa(p))
}
if len(parts) == 1 {
return "port " + parts[0]
}
return "ports " + strings.Join(parts, ", ")
}
func auditSSH() []store.AuditResult {
parsed := parseSSHDConfig()
var results []store.AuditResult
// ssh_port. sshd binds every Port directive, so exposure on 22 counts
// even when the config also names an alternate port.
ports := parsed.ListenPorts()
if slices.Contains(ports, 22) {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_port", Title: "SSH Port",
Status: "warn", Message: "SSH is running on default port 22",
Fix: "Change to a non-standard port in /etc/ssh/sshd_config to reduce automated scan noise. Update firewall rules before changing.",
})
} else {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_port", Title: "SSH Port",
Status: "pass", Message: fmt.Sprintf("SSH on non-standard %s", portsLabel(ports)),
})
}
// ssh_protocol
proto := parsed.Value("protocol")
if strings.Contains(proto, "1") {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_protocol", Title: "SSH Protocol",
Status: "fail", Message: "SSHv1 protocol is enabled",
Fix: "Set 'Protocol 2' in /etc/ssh/sshd_config. SSHv1 has known cryptographic weaknesses.",
})
} else {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_protocol", Title: "SSH Protocol",
Status: "pass", Message: "SSHv1 disabled",
})
}
// ssh_password_auth
passAuth := parsed.Value("passwordauthentication")
if passAuth != "no" {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_password_auth", Title: "SSH PasswordAuthentication",
Status: "fail", Message: "Password authentication is enabled",
Fix: "Set 'PasswordAuthentication no' in /etc/ssh/sshd_config and use SSH key authentication only.",
})
} else {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_password_auth", Title: "SSH PasswordAuthentication",
Status: "pass", Message: "Password authentication disabled",
})
}
// ssh_root_login
rootLogin := parsed.Value("permitrootlogin")
if rootLogin == "yes" {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_root_login", Title: "SSH PermitRootLogin",
Status: "fail", Message: "Direct root login is permitted",
Fix: "Set 'PermitRootLogin no' or 'PermitRootLogin prohibit-password' in /etc/ssh/sshd_config.",
})
} else {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_root_login", Title: "SSH PermitRootLogin",
Status: "pass", Message: fmt.Sprintf("PermitRootLogin set to %s", rootLogin),
})
}
// ssh_max_auth_tries
maxTries := parsed.Value("maxauthtries")
n, _ := strconv.Atoi(maxTries)
if n == 0 {
n = 6 // default
}
if n > 4 {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_max_auth_tries", Title: "SSH MaxAuthTries",
Status: "warn", Message: fmt.Sprintf("MaxAuthTries is %d (recommended: 4 or less)", n),
Fix: "Set 'MaxAuthTries 4' in /etc/ssh/sshd_config to limit brute-force attempts per connection.",
})
} else {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_max_auth_tries", Title: "SSH MaxAuthTries",
Status: "pass", Message: fmt.Sprintf("MaxAuthTries set to %d", n),
})
}
// ssh_x11_forwarding
x11 := parsed.Value("x11forwarding")
if x11 != "no" {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_x11_forwarding", Title: "SSH X11Forwarding",
Status: "warn", Message: "X11 forwarding is enabled",
Fix: "Set 'X11Forwarding no' in /etc/ssh/sshd_config unless X11 forwarding is actively needed.",
})
} else {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_x11_forwarding", Title: "SSH X11Forwarding",
Status: "pass", Message: "X11 forwarding disabled",
})
}
// ssh_use_dns
useDNS := parsed.Value("usedns")
if useDNS != "no" {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_use_dns", Title: "SSH UseDNS",
Status: "warn", Message: "UseDNS is enabled",
Fix: "Set 'UseDNS no' in /etc/ssh/sshd_config. Otherwise lfd may not track SSH login failures by IP.",
})
} else {
results = append(results, store.AuditResult{
Category: "ssh", Name: "ssh_use_dns", Title: "SSH UseDNS",
Status: "pass", Message: "UseDNS disabled",
})
}
return results
}
// --- OS hardening checks ---
func auditOS() []store.AuditResult {
var results []store.AuditResult
// /tmp and /var/tmp permissions
for _, dir := range []struct {
path, id, title string
}{
{"/tmp", "os_tmp_permissions", "/tmp Permissions"},
{"/var/tmp", "os_var_tmp_permissions", "/var/tmp Permissions"},
} {
info, err := osFS.Stat(dir.path)
if err != nil {
results = append(results, store.AuditResult{
Category: "os", Name: dir.id, Title: dir.title,
Status: "warn", Message: fmt.Sprintf("Cannot stat %s: %v", dir.path, err),
})
continue
}
// Use the raw Unix mode bits from syscall to get the traditional
// permission representation (sticky=01000, setuid=04000, etc.).
// Go's os.ModeSticky uses high bits that don't map to Unix octal,
// so os.FileMode math produces wrong values for comparison.
// Only check the lower 12 bits (sticky + rwx) — ignore setuid/setgid
// which CloudLinux/CageFS may set on virtmp mounts.
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
results = append(results, store.AuditResult{
Category: "os", Name: dir.id, Title: dir.title,
Status: "warn", Message: fmt.Sprintf("Cannot read ownership of %s", dir.path),
})
continue
}
mode := stat.Mode & 01777 // sticky + rwxrwxrwx, ignore setuid/setgid
if mode != 01777 || stat.Uid != 0 || stat.Gid != 0 {
results = append(results, store.AuditResult{
Category: "os", Name: dir.id, Title: dir.title,
Status: "fail",
Message: fmt.Sprintf("%s has mode %04o uid=%d gid=%d (expected 1777 root:root)", dir.path, mode, stat.Uid, stat.Gid),
Fix: fmt.Sprintf("chmod 1777 %s && chown root:root %s", dir.path, dir.path),
})
} else {
results = append(results, store.AuditResult{
Category: "os", Name: dir.id, Title: dir.title,
Status: "pass", Message: fmt.Sprintf("%s is 1777 root:root", dir.path),
})
}
}
// /etc/shadow permissions
// Accept 0000, 0600 (RHEL/CentOS default), and 0640 (Debian default).
// All three restrict access to root only. 0600 is the standard on
// CentOS/CloudLinux — changing it can break passwd/chage.
if info, err := osFS.Stat("/etc/shadow"); err == nil {
perm := info.Mode().Perm()
if perm == 0 || perm == 0o600 || perm == 0o640 {
results = append(results, store.AuditResult{
Category: "os", Name: "os_shadow_permissions", Title: "/etc/shadow Permissions",
Status: "pass", Message: fmt.Sprintf("/etc/shadow has mode %04o", perm),
})
} else {
results = append(results, store.AuditResult{
Category: "os", Name: "os_shadow_permissions", Title: "/etc/shadow Permissions",
Status: "fail", Message: fmt.Sprintf("/etc/shadow has mode %04o (expected 0000, 0600, or 0640)", perm),
Fix: "chmod 0600 /etc/shadow",
})
}
} else {
results = append(results, store.AuditResult{
Category: "os", Name: "os_shadow_permissions", Title: "/etc/shadow Permissions",
Status: "warn", Message: "Cannot stat /etc/shadow",
})
}
// Swap
if data, err := osFS.ReadFile("/proc/swaps"); err == nil {
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
if len(lines) > 1 {
results = append(results, store.AuditResult{
Category: "os", Name: "os_swap", Title: "Swap Configured",
Status: "pass", Message: fmt.Sprintf("%d swap device(s) active", len(lines)-1),
})
} else {
results = append(results, store.AuditResult{
Category: "os", Name: "os_swap", Title: "Swap Configured",
Status: "warn", Message: "No swap configured",
Fix: "Configure swap space to prevent OOM kills: fallocate -l 2G /swapfile && chmod 600 /swapfile && mkswap /swapfile && swapon /swapfile",
})
}
}
// Distro EOL
results = append(results, checkDistroEOL()...)
// Web-server user crontab: the web server's account (cPanel nobody,
// Debian www-data, RHEL apache/nginx) never needs a crontab; one with
// content is a persistence mechanism planted through the web tier.
// nobody is always included: it is the suEXEC/CGI fallback everywhere.
results = append(results, auditWebUserCrontab(cronSpoolDir(), webUsersWithNobody(webServerUsers())))
// Unnecessary services
results = append(results, checkUnnecessaryServices()...)
// CVE-2026-31431 "Copy Fail" — algif_aead is the AF_ALG submodule the
// exploit chains through. Blacklisting it neutralises the attack on
// unpatched kernels.
results = append(results, auditAlgifAEAD())
// Sysctl checks (table-driven)
sysctlChecks := []struct {
id, title, path, expected string
}{
{"os_sysctl_syncookies", "TCP SYN Cookies", "/proc/sys/net/ipv4/tcp_syncookies", "1"},
{"os_sysctl_aslr", "Address Space Layout Randomization", "/proc/sys/kernel/randomize_va_space", "2"},
{"os_sysctl_rp_filter", "Reverse Path Filtering", "/proc/sys/net/ipv4/conf/all/rp_filter", "1"},
{"os_sysctl_icmp_broadcast", "ICMP Broadcast Ignore", "/proc/sys/net/ipv4/icmp_echo_ignore_broadcasts", "1"},
{"os_sysctl_symlinks", "Protected Symlinks", "/proc/sys/fs/protected_symlinks", "1"},
{"os_sysctl_hardlinks", "Protected Hardlinks", "/proc/sys/fs/protected_hardlinks", "1"},
}
for _, sc := range sysctlChecks {
data, err := osFS.ReadFile(sc.path)
if err != nil {
results = append(results, store.AuditResult{
Category: "os", Name: sc.id, Title: sc.title,
Status: "warn", Message: fmt.Sprintf("Cannot read %s", sc.path),
})
continue
}
val := strings.TrimSpace(string(data))
// Convert /proc/sys path to sysctl dotted notation for fix command
sysctlKey := strings.TrimPrefix(sc.path, "/proc/sys/")
sysctlKey = strings.ReplaceAll(sysctlKey, "/", ".")
if val == sc.expected {
results = append(results, store.AuditResult{
Category: "os", Name: sc.id, Title: sc.title,
Status: "pass", Message: fmt.Sprintf("%s = %s", sysctlKey, val),
})
} else {
results = append(results, store.AuditResult{
Category: "os", Name: sc.id, Title: sc.title,
Status: "fail", Message: fmt.Sprintf("%s = %s (expected %s)", sysctlKey, val, sc.expected),
Fix: fmt.Sprintf("sysctl -w %s=%s && echo '%s = %s' >> /etc/sysctl.d/99-csm-hardening.conf", sysctlKey, sc.expected, sysctlKey, sc.expected),
})
}
}
return results
}
// algifAEADBlacklisted reports whether any of the supplied modprobe.d files
// contain a non-comment directive that prevents algif_aead from loading.
// Either of the following blocks is sufficient:
//
// blacklist algif_aead
// install algif_aead /bin/false
// blacklist af_alg (parent — algif_aead depends on it)
// install af_alg /bin/false (parent — same reason)
//
// The dependency relationship matters for hand-hardened images: a sysadmin
// who blocked only the parent `af_alg` has correctly mitigated Copy Fail
// without needing to also block the AEAD submodule. Reporting "no blacklist
// exists" on such hosts would be a false-fail alert.
//
// `install <module> /sbin/modprobe --ignore-install <module>` is the
// idiomatic re-load form and explicitly does NOT block the module — we
// detect that by skipping any install replacement whose first token's
// basename is "modprobe".
func algifAEADBlacklisted(confs map[string]string) bool {
for _, body := range confs {
for _, line := range strings.Split(body, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
if fields[1] != "algif_aead" && fields[1] != "af_alg" {
continue
}
switch fields[0] {
case "blacklist":
return true
case "install":
if len(fields) < 3 {
// Malformed (no replacement command). Don't claim a
// pass on a half-written directive.
continue
}
// Match on the basename of the first token, not a substring
// anywhere in the line. That correctly classifies
// /sbin/modprobe --ignore-install <module> → re-load
// /bin/false → block
// /usr/local/bin/my-modprobe-wrapper → block (wrapper, not modprobe itself)
// Substring matching would have lumped the wrapper case in
// with the re-load case, producing a false-fail alert.
if filepath.Base(fields[2]) == "modprobe" {
continue
}
return true
}
}
}
return false
}
// algifAEAD identifiers shared by the pure helper and the impure wrapper —
// hoisted to package scope so a future edit cannot drift the two copies out
// of sync (which would silently mismatch Name fields between the pass/fail
// path and the warn path).
const (
algifAEADAuditID = "os_algif_aead_blocked"
algifAEADAuditTitle = "AF_ALG (algif_aead) Blocked — CVE-2026-31431"
)
// evaluateAlgifAEAD is the pure, testable core of the algif_aead hardening
// check. `loaded` reports whether algif_aead currently shows up in
// /proc/modules; `confs` is a map of modprobe.d file path → contents.
func evaluateAlgifAEAD(loaded bool, confs map[string]string) store.AuditResult {
blocked := algifAEADBlacklisted(confs)
switch {
case !loaded && blocked:
msg := "algif_aead is blacklisted and not loaded"
if content, ok := confs[afAlgMarkerPath]; ok && validateMarkerContent([]byte(content)) {
msg = "algif_aead is blocked by CSM-managed enforcement (csm harden --copy-fail)"
}
return store.AuditResult{
Category: "os", Name: algifAEADAuditID, Title: algifAEADAuditTitle,
Status: "pass", Message: msg,
}
case loaded:
return store.AuditResult{
Category: "os", Name: algifAEADAuditID, Title: algifAEADAuditTitle,
Status: "fail",
Message: "algif_aead is currently loaded — Copy Fail (CVE-2026-31431) exploitable",
Fix: "echo 'install algif_aead /bin/false' > /etc/modprobe.d/csm-disable-algif.conf && modprobe -r algif_aead af_alg",
}
default:
return store.AuditResult{
Category: "os", Name: algifAEADAuditID, Title: algifAEADAuditTitle,
Status: "fail",
Message: "algif_aead is not loaded but no modprobe.d blacklist exists — module can be loaded on demand",
Fix: "echo 'install algif_aead /bin/false' > /etc/modprobe.d/csm-disable-algif.conf",
}
}
}
// auditAlgifAEAD is the impure wrapper: it reads the running kernel's
// build configuration, KernelCare/livepatch state, /proc/modules, and
// /etc/modprobe.d/*.conf via osFS/cmdExec, then produces an AuditResult.
//
// Decision order:
// 1. KernelCare has applied a Copy Fail livepatch -> pass.
// 2. Kernel has CONFIG_CRYPTO_USER_API_AEAD=y -> fail with a
// truthful message; the modprobe blacklist is ineffective on this
// kernel because the AEAD code is statically linked.
// 3. Otherwise, fall through to the modprobe-state evaluator (the
// existing logic for hosts where AF_ALG is a loadable module).
//
// If any modprobe.d file is unreadable, return a "warn" AuditResult
// naming the offending file rather than silently misreporting.
func auditAlgifAEAD() store.AuditResult {
kernelState := observeAFAlgKernelState()
if kernelState.LivepatchActive {
return store.AuditResult{
Category: "os", Name: algifAEADAuditID, Title: algifAEADAuditTitle,
Status: "pass",
Message: kernelState.String(),
}
}
if kernelState.BuiltIn {
// On built-in kernels the modprobe blacklist is ineffective. Two
// interim options exist: KernelCare livepatch (handled above) or
// per-service seccomp drop-ins. Recognize the seccomp coverage
// here so an operator who ran `csm harden --copy-fail-seccomp`
// gets a truthful pass.
seccomp := SummarizeAFAlgSeccompCoverage()
if len(seccomp.Covered) > 0 && len(seccomp.Uncovered) == 0 {
return store.AuditResult{
Category: "os", Name: algifAEADAuditID, Title: algifAEADAuditTitle,
Status: "pass",
Message: fmt.Sprintf(
"kernel built-in but Copy Fail blocked by seccomp drop-ins on %d units (%s)",
len(seccomp.Covered), strings.Join(seccomp.Covered, ", "),
),
}
}
fixMsg := "Apply KernelCare/kpatch when the CVE-2026-31431 patch ships (kcarectl --update); " +
"or run `csm harden --copy-fail-seccomp` to apply per-service seccomp drop-ins now. " +
"The modprobe blacklist file is harmless but does not protect this kernel."
messageDetail := "modprobe blacklist is ineffective on this kernel and Copy Fail (CVE-2026-31431) is exploitable"
if len(seccomp.Covered) > 0 {
messageDetail = fmt.Sprintf(
"seccomp drop-ins present on %d units but %d candidate units still uncovered (%s)",
len(seccomp.Covered), len(seccomp.Uncovered), strings.Join(seccomp.Uncovered, ", "),
)
}
return store.AuditResult{
Category: "os", Name: algifAEADAuditID, Title: algifAEADAuditTitle,
Status: "fail",
Message: "AF_ALG aead is built into the kernel (CONFIG_CRYPTO_USER_API_AEAD=y); " +
messageDetail,
Fix: fixMsg,
}
}
loaded := false
for _, mod := range loadModuleList() {
if mod == "algif_aead" {
loaded = true
break
}
}
confs := make(map[string]string)
matches, err := osFS.Glob("/etc/modprobe.d/*.conf")
if err == nil {
for _, p := range matches {
data, err := osFS.ReadFile(p)
if err != nil {
return store.AuditResult{
Category: "os", Name: algifAEADAuditID, Title: algifAEADAuditTitle,
Status: "warn",
Message: fmt.Sprintf("Cannot read %s: %v — blacklist state undetermined", p, err),
}
}
confs[p] = string(data)
}
}
return evaluateAlgifAEAD(loaded, confs)
}
// distroEOLPolicy encodes the oldest supported major version per known OS.
// Anything below the minimum is considered EOL by this check.
var distroEOLPolicy = map[platform.OSFamily]int{
platform.OSAlma: 8,
platform.OSRocky: 8,
platform.OSRHEL: 8,
platform.OSCloudLinux: 7,
platform.OSUbuntu: 20, // 20.04 is the oldest non-EOL LTS
platform.OSDebian: 11, // Debian 11 "bullseye"
}
func checkDistroEOL() []store.AuditResult {
return evaluateDistroEOL(platform.Detect(), readOSReleasePretty())
}
// evaluateDistroEOL is the pure, testable core of checkDistroEOL. It returns
// an AuditResult based purely on the supplied platform info and
// PRETTY_NAME string (either may be empty).
func evaluateDistroEOL(info platform.Info, prettyName string) []store.AuditResult {
if prettyName == "" && info.OSVersion != "" {
prettyName = fmt.Sprintf("%s %s", info.OS, info.OSVersion)
}
if info.OS == platform.OSUnknown || info.OSVersion == "" {
return []store.AuditResult{{
Category: "os", Name: "os_distro_eol", Title: "Distribution End of Life",
Status: "warn", Message: "Unable to determine distribution version",
}}
}
if info.OS == platform.OSCentOS {
return []store.AuditResult{{
Category: "os", Name: "os_distro_eol", Title: "Distribution End of Life",
Status: "fail",
Message: fmt.Sprintf("%s — CentOS is end-of-life", prettyName),
Fix: "Migrate to a supported replacement such as AlmaLinux, Rocky Linux, or RHEL. CentOS no longer receives security patches.",
}}
}
// Extract the major version. Ubuntu/Debian use "24.04" / "12", RHEL
// family uses "8.6" / "10", etc. Taking the integer prefix handles both.
majorStr, _, _ := strings.Cut(info.OSVersion, ".")
major, err := strconv.Atoi(majorStr)
if err != nil {
return []store.AuditResult{{
Category: "os", Name: "os_distro_eol", Title: "Distribution End of Life",
Status: "warn", Message: fmt.Sprintf("%s — unable to parse version %q", prettyName, info.OSVersion),
}}
}
minVersion, known := distroEOLPolicy[info.OS]
if !known {
return []store.AuditResult{{
Category: "os", Name: "os_distro_eol", Title: "Distribution End of Life",
Status: "warn", Message: fmt.Sprintf("%s — no EOL policy configured for this distro", prettyName),
}}
}
if major < minVersion {
fix := "Upgrade to a supported release. EOL distributions receive no security patches."
if info.IsRHELFamily() {
fix = fmt.Sprintf("Upgrade to %s %d+ or newer. EOL distributions receive no security patches.", info.OS, minVersion)
}
if info.IsDebianFamily() {
fix = fmt.Sprintf("Upgrade to %s %d+ or newer LTS. EOL distributions receive no security patches.", info.OS, minVersion)
}
return []store.AuditResult{{
Category: "os", Name: "os_distro_eol", Title: "Distribution End of Life",
Status: "fail",
Message: fmt.Sprintf("%s — major version %d is EOL", prettyName, major),
Fix: fix,
}}
}
return []store.AuditResult{{
Category: "os", Name: "os_distro_eol", Title: "Distribution End of Life",
Status: "pass", Message: prettyName,
}}
}
// readOSReleasePretty returns the PRETTY_NAME from /etc/os-release or "".
func readOSReleasePretty() string {
data, err := osFS.ReadFile("/etc/os-release")
if err != nil {
return ""
}
for _, line := range strings.Split(string(data), "\n") {
if strings.HasPrefix(line, "PRETTY_NAME=") {
return strings.Trim(strings.TrimPrefix(line, "PRETTY_NAME="), `"'`)
}
}
return ""
}
func checkUnnecessaryServices() []store.AuditResult {
badServices := []string{
"avahi-daemon", "bluetooth", "cups", "cupsd", "gdm",
"ModemManager", "packagekit", "rpcbind", "wpa_supplicant", "firewalld",
}
out, err := auditRunCmd("systemctl", "list-unit-files", "--state=enabled", "--no-pager", "--no-legend")
if err != nil {
return []store.AuditResult{{
Category: "os", Name: "os_services", Title: "Unnecessary Services",
Status: "warn", Message: "Cannot query systemd unit files",
}}
}
var found []string
lines := strings.Split(string(out), "\n")
for _, line := range lines {
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
unit := strings.TrimSuffix(fields[0], ".service")
for _, bad := range badServices {
if unit == bad {
found = append(found, bad)
}
}
}
if len(found) == 0 {
return []store.AuditResult{{
Category: "os", Name: "os_services", Title: "Unnecessary Services",
Status: "pass", Message: "No unnecessary services enabled",
}}
}
return []store.AuditResult{{
Category: "os", Name: "os_services", Title: "Unnecessary Services",
Status: "warn",
Message: fmt.Sprintf("Unnecessary services enabled: %s", strings.Join(found, ", ")),
Fix: fmt.Sprintf("systemctl disable --now %s", strings.Join(found, " ")),
}}
}
// --- Firewall checks ---
func auditFirewall() []store.AuditResult {
var results []store.AuditResult
// Gather nft and iptables state
nftOut, nftErr := auditRunCmd("nft", "list", "ruleset")
nftRules := string(nftOut)
hasNft := nftErr == nil && strings.TrimSpace(nftRules) != ""
iptOut, iptErr := auditRunCmd("iptables", "-L", "INPUT", "-n")
iptRules := string(iptOut)
hasIpt := iptErr == nil && strings.TrimSpace(iptRules) != ""
// fw_active
if hasNft || hasIpt {
results = append(results, store.AuditResult{
Category: "firewall", Name: "fw_active", Title: "Firewall Active",
Status: "pass", Message: "Firewall has active rules",
})
} else {
results = append(results, store.AuditResult{
Category: "firewall", Name: "fw_active", Title: "Firewall Active",
Status: "fail", Message: "No active firewall rules detected",
Fix: "Install and configure nftables or iptables with a default-deny policy.",
})
}
results = append(results, checkFirewallDefaultPolicy(hasNft, nftRules, hasIpt, iptRules))
// fw_mysql_exposed
results = append(results, checkMySQLExposed(hasNft, nftRules, hasIpt, iptRules)...)
// fw_telnet
if isPortListening(23) {
results = append(results, store.AuditResult{
Category: "firewall", Name: "fw_telnet", Title: "Telnet Service",
Status: "fail", Message: "Something is listening on port 23 (telnet)",
Fix: "Disable telnet: systemctl disable --now telnet.socket xinetd; use SSH instead.",
})
} else {
results = append(results, store.AuditResult{
Category: "firewall", Name: "fw_telnet", Title: "Telnet Service",
Status: "pass", Message: "Nothing listening on port 23",
})
}
// fw_ipv6
results = append(results, checkIPv6Firewall()...)
return results
}
// getListeningAddr reads /proc/net/tcp for a port in LISTEN state (0A)
// and returns the hex-encoded local IP, or "" if not found.
func getListeningAddr(port int) string {
hexPort := fmt.Sprintf("%04X", port)
for _, path := range []string{"/proc/net/tcp", "/proc/net/tcp6"} {
data, err := osFS.ReadFile(path)
if err != nil {
continue
}
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) < 4 {
continue
}
// fields[1] = local_address (hex_ip:hex_port), fields[3] = state
if fields[3] != "0A" { // 0A = LISTEN
continue
}
parts := strings.SplitN(fields[1], ":", 2)
if len(parts) != 2 {
continue
}
if parts[1] == hexPort {
return parts[0]
}
}
}
return ""
}
// hexToIPv4 converts a /proc/net/tcp hex IP (little-endian 32-bit) to dotted notation.
func hexToIPv4(h string) string {
if len(h) != 8 {
return h
}
b, err := hex.DecodeString(h)
if err != nil || len(b) != 4 {
return h
}
// /proc/net/tcp stores IPs in little-endian byte order
return fmt.Sprintf("%d.%d.%d.%d", b[3], b[2], b[1], b[0])
}
// isPrivateOrLoopback returns true for loopback, RFC1918, and RFC4193 addresses.
func isPrivateOrLoopback(ipStr string) bool {
ip := net.ParseIP(ipStr)
if ip == nil {
return false
}
if ip.IsLoopback() {
return true
}
// Check private ranges
privateRanges := []string{
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16",
"fc00::/7",
}
for _, cidr := range privateRanges {
_, network, err := net.ParseCIDR(cidr)
if err != nil {
continue
}
if network.Contains(ip) {
return true
}
}
return false
}
// isPortListening checks /proc/net/tcp and /proc/net/tcp6 for a port in LISTEN state.
func isPortListening(port int) bool {
hexPort := fmt.Sprintf("%04X", port)
for _, path := range []string{"/proc/net/tcp", "/proc/net/tcp6"} {
data, err := osFS.ReadFile(path)
if err != nil {
continue
}
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) < 4 {
continue
}
if fields[3] != "0A" {
continue
}
parts := strings.SplitN(fields[1], ":", 2)
if len(parts) == 2 && parts[1] == hexPort {
return true
}
}
}
return false
}
func checkMySQLExposed(hasNft bool, nftRules string, hasIpt bool, iptRules string) []store.AuditResult {
hexAddr := getListeningAddr(3306)
if hexAddr == "" {
return []store.AuditResult{{
Category: "firewall", Name: "fw_mysql_exposed", Title: "MySQL Exposure",
Status: "pass", Message: "MySQL is not listening on any port",
}}
}
// Convert hex addr to IP and check if private/loopback
var ip string
if len(hexAddr) == 8 {
ip = hexToIPv4(hexAddr)
} else {
// IPv6: 32 hex chars, little-endian 4-byte groups
if b, err := hex.DecodeString(hexAddr); err == nil {
ipBytes := make(net.IP, len(b))
// Reverse each 4-byte group for /proc/net/tcp6 little-endian encoding
for i := 0; i+4 <= len(b); i += 4 {
ipBytes[i] = b[i+3]
ipBytes[i+1] = b[i+2]
ipBytes[i+2] = b[i+1]
ipBytes[i+3] = b[i]
}
ip = ipBytes.String()
}
}
// All zeros = wildcard bind
allZero := true
for _, c := range hexAddr {
if c != '0' {
allZero = false
break
}
}
if !allZero && ip != "" && isPrivateOrLoopback(ip) {
return []store.AuditResult{{
Category: "firewall", Name: "fw_mysql_exposed", Title: "MySQL Exposure",
Status: "pass", Message: fmt.Sprintf("MySQL bound to private/loopback address %s", ip),
}}
}
// Wildcard or public bind — check if firewall blocks 3306
// nft has rules, none mention 3306, and the input hook denies by
// default: the port is blocked.
fwBlocks3306 := hasNft && !strings.Contains(nftRules, "3306") && nftInputDefaultDeny(nftRules)
if !fwBlocks3306 && hasIpt && !strings.Contains(iptRules, "3306") {
for _, line := range strings.Split(iptRules, "\n") {
if strings.HasPrefix(line, "Chain INPUT") {
upper := strings.ToUpper(line)
if strings.Contains(upper, "POLICY DROP") || strings.Contains(upper, "POLICY REJECT") {
fwBlocks3306 = true
}
break
}
}
}
bindDesc := "wildcard (0.0.0.0)"
if !allZero && ip != "" {
bindDesc = ip
}
if fwBlocks3306 {
return []store.AuditResult{{
Category: "firewall", Name: "fw_mysql_exposed", Title: "MySQL Exposure",
Status: "warn",
Message: fmt.Sprintf("MySQL bound to %s but firewall blocks port 3306", bindDesc),
Fix: "Bind MySQL to 127.0.0.1 in /etc/my.cnf: bind-address = 127.0.0.1",
}}
}
return []store.AuditResult{{
Category: "firewall", Name: "fw_mysql_exposed", Title: "MySQL Exposure",
Status: "fail",
Message: fmt.Sprintf("MySQL bound to %s and port 3306 appears accessible", bindDesc),
Fix: "Bind MySQL to 127.0.0.1 in /etc/my.cnf and/or block port 3306 in firewall.",
}}
}
func checkIPv6Firewall() []store.AuditResult {
// Check if any non-loopback, non-link-local IPv6 addresses exist
data, err := osFS.ReadFile("/proc/net/if_inet6")
if err != nil {
return []store.AuditResult{{
Category: "firewall", Name: "fw_ipv6", Title: "IPv6 Firewall",
Status: "pass", Message: "IPv6 not active (cannot read /proc/net/if_inet6)",
}}
}
hasIPv6 := false
for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
fields := strings.Fields(line)
if len(fields) < 6 {
continue
}
addr := fields[0]
iface := fields[5]
// Skip loopback
if iface == "lo" {
continue
}
// Skip link-local (fe80::/10)
if strings.HasPrefix(strings.ToLower(addr), "fe80") {
continue
}
hasIPv6 = true
break
}
if !hasIPv6 {
return []store.AuditResult{{
Category: "firewall", Name: "fw_ipv6", Title: "IPv6 Firewall",
Status: "pass", Message: "No non-link-local IPv6 addresses found",
}}
}
// Check nftables for inet/ip6 family chain with input hook and default deny
nftChains, err := auditRunCmd("nft", "list", "chains")
if err == nil {
chainsStr := strings.ToLower(string(nftChains))
// Look for chains in inet or ip6 family with filter hook input
// nft list chains output looks like:
// table inet filter {
// chain input {
// type filter hook input priority filter; policy drop;
// }
// }
var currentFamily string
for _, line := range strings.Split(chainsStr, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "table ") {
parts := strings.Fields(trimmed)
if len(parts) >= 3 {
currentFamily = parts[1]
}
}
if (currentFamily == "inet" || currentFamily == "ip6") &&
strings.Contains(trimmed, "hook input") {
if strings.Contains(trimmed, "policy drop") || strings.Contains(trimmed, "policy reject") {
return []store.AuditResult{{
Category: "firewall", Name: "fw_ipv6", Title: "IPv6 Firewall",
Status: "pass", Message: fmt.Sprintf("IPv6 active; nftables %s family has default-deny input chain", currentFamily),
}}
}
}
}
}
// Fallback: check ip6tables
ip6Out, err := auditRunCmd("ip6tables", "-L", "INPUT", "-n")
if err == nil {
for _, line := range strings.Split(string(ip6Out), "\n") {
if strings.HasPrefix(line, "Chain INPUT") {
upper := strings.ToUpper(line)
if strings.Contains(upper, "POLICY DROP") || strings.Contains(upper, "POLICY REJECT") {
return []store.AuditResult{{
Category: "firewall", Name: "fw_ipv6", Title: "IPv6 Firewall",
Status: "pass", Message: "IPv6 active; ip6tables INPUT chain has default-deny policy",
}}
}
break
}
}
}
return []store.AuditResult{{
Category: "firewall", Name: "fw_ipv6", Title: "IPv6 Firewall",
Status: "fail", Message: "IPv6 is active but no default-deny input policy found",
Fix: "Configure ip6tables or nftables inet family with a default DROP policy for INPUT.",
}}
}
// --- cPanel/WHM and CloudLinux checks ---
func auditCPanel(serverType string) []store.AuditResult {
var results []store.AuditResult
cpConf := parseCpanelConfig("/var/cpanel/cpanel.config")
// Table-driven boolean checks on cpanel.config.
// fix is the human-readable remediation shown in the UI.
type cpCheck struct {
id, title, key, wantVal string
invert bool // true = fail when value matches wantVal
fix string
}
checks := []cpCheck{
{"cp_ssl_only", "Always Redirect to SSL", "alwaysredirecttossl", "1", false,
"In WHM > Tweak Settings > Redirection, set 'Always redirect to SSL/TLS' to On."},
{"cp_boxtrapper", "BoxTrapper Disabled", "skipboxtrapper", "1", false,
"In WHM > Tweak Settings > Mail, set 'Enable BoxTrapper spam trap' to Off."},
{"cp_password_reset", "Password Reset Disabled", "resetpass", "1", true,
"In WHM > Tweak Settings > System, set 'Reset Password for cPanel accounts' to Off."},
{"cp_password_reset_sub", "Subaccount Password Reset Disabled", "resetpass_sub", "1", true,
"In WHM > Tweak Settings > System, set 'Reset Password for Subaccounts' to Off."},
{"cp_email_passwords", "Email Passwords Disabled", "emailpasswords", "1", true,
"In WHM > Tweak Settings > Security, set 'Send passwords when creating a new account' to Off."},
{"cp_cookie_validation", "Cookie IP Validation", "cookieipvalidation", "strict", false,
"In WHM > Tweak Settings > Security, set 'Cookie IP validation' to strict."},
{"cp_remote_domains", "Remote Domains Disabled", "allowremotedomains", "1", true,
"In WHM > Tweak Settings > Domains, set 'Allow Remote Domains' to Off."},
{"cp_core_dumps", "Core Dumps Disabled", "coredump", "1", true,
"In WHM > Tweak Settings > Security, set 'Generate core dumps' to Off."},
{"cp_nobodyspam", "Nobody Spam Prevention", "nobodyspam", "1", false,
"In WHM > Tweak Settings > Mail, set 'Prevent nobody from sending mail' to On."},
}
for _, c := range checks {
val := cpConf[c.key]
var pass bool
if c.invert {
pass = val != c.wantVal
} else {
pass = val == c.wantVal
}
if pass {
results = append(results, store.AuditResult{
Category: "cpanel", Name: c.id, Title: c.title,
Status: "pass", Message: fmt.Sprintf("%s = %s", c.key, val),
})
} else {
results = append(results, store.AuditResult{
Category: "cpanel", Name: c.id, Title: c.title,
Status: "fail", Message: fmt.Sprintf("%s = %s", c.key, val),
Fix: c.fix,
})
}
}
// cp_max_emails_hour
maxEmail := cpConf["maxemailsperhour"]
if maxEmail != "" && maxEmail != "0" {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cp_max_emails_hour", Title: "Max Emails Per Hour",
Status: "pass", Message: fmt.Sprintf("maxemailsperhour = %s", maxEmail),
})
} else {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cp_max_emails_hour", Title: "Max Emails Per Hour",
Status: "fail", Message: "maxemailsperhour is not set or is 0",
Fix: "In WHM > Tweak Settings, set 'Max emails per hour per domain' to a reasonable limit (e.g., 200).",
})
}
// cp_compilers: check /usr/bin/cc permissions
if info, err := osFS.Stat("/usr/bin/cc"); err == nil {
perm := info.Mode().Perm()
if perm <= 0o750 {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cp_compilers", Title: "Compiler Access Restricted",
Status: "pass", Message: fmt.Sprintf("/usr/bin/cc has mode %04o", perm),
})
} else {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cp_compilers", Title: "Compiler Access Restricted",
Status: "fail", Message: fmt.Sprintf("/usr/bin/cc has mode %04o (should be <= 0750)", perm),
Fix: "WHM > Security Center > Compiler Access, or: chmod 750 /usr/bin/cc",
})
}
} else {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cp_compilers", Title: "Compiler Access Restricted",
Status: "pass", Message: "No compiler found at /usr/bin/cc",
})
}
// cp_ftp_anonymous: parse /etc/pure-ftpd.conf
if data, err := osFS.ReadFile("/etc/pure-ftpd.conf"); err == nil {
noAnon := false
for _, line := range strings.Split(string(data), "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "#") {
continue
}
if strings.HasPrefix(trimmed, "NoAnonymous") {
parts := strings.Fields(trimmed)
if len(parts) >= 2 && strings.EqualFold(parts[1], "yes") {
noAnon = true
}
}
}
if noAnon {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cp_ftp_anonymous", Title: "Anonymous FTP Disabled",
Status: "pass", Message: "NoAnonymous is enabled in pure-ftpd",
})
} else {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cp_ftp_anonymous", Title: "Anonymous FTP Disabled",
Status: "fail", Message: "Anonymous FTP may be enabled",
Fix: "Set 'NoAnonymous yes' in /etc/pure-ftpd.conf and restart pure-ftpd.",
})
}
}
// cp_updates: parse /etc/cpupdate.conf
if data, err := osFS.ReadFile("/etc/cpupdate.conf"); err == nil {
updatesDaily := false
for _, line := range strings.Split(string(data), "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "#") {
continue
}
if strings.HasPrefix(strings.ToUpper(trimmed), "UPDATES=") {
val := strings.TrimPrefix(trimmed, trimmed[:strings.Index(trimmed, "=")+1])
if strings.EqualFold(strings.TrimSpace(val), "daily") {
updatesDaily = true
}
}
}
if updatesDaily {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cp_updates", Title: "cPanel Auto-Updates",
Status: "pass", Message: "UPDATES=daily in cpupdate.conf",
})
} else {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cp_updates", Title: "cPanel Auto-Updates",
Status: "warn", Message: "cPanel auto-updates not set to daily",
Fix: "Set UPDATES=daily in /etc/cpupdate.conf or WHM > Update Preferences.",
})
}
}
// CloudLinux-specific checks
if serverType == "cloudlinux" {
results = append(results, auditCloudLinux()...)
}
return results
}
func parseCpanelConfig(path string) map[string]string {
conf := make(map[string]string)
data, err := osFS.ReadFile(path)
if err != nil {
return conf
}
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if idx := strings.Index(line, "="); idx > 0 {
key := strings.TrimSpace(line[:idx])
val := strings.TrimSpace(line[idx+1:])
conf[key] = val
}
}
return conf
}
func auditCloudLinux() []store.AuditResult {
var results []store.AuditResult
// cl_cagefs
out, err := auditRunCmd("cagefsctl", "--cagefs-status")
switch {
case err != nil:
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cl_cagefs", Title: "CageFS Enabled",
Status: "warn", Message: "Cannot check CageFS status",
})
case strings.Contains(string(out), "Enabled"):
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cl_cagefs", Title: "CageFS Enabled",
Status: "pass", Message: "CageFS is enabled",
})
default:
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cl_cagefs", Title: "CageFS Enabled",
Status: "fail", Message: "CageFS is not enabled",
Fix: "Enable CageFS: cagefsctl --enable-all",
})
}
// cl_symlink_protection
if data, err := osFS.ReadFile("/proc/sys/fs/enforce_symlinksifowner"); err == nil {
val := strings.TrimSpace(string(data))
n, _ := strconv.Atoi(val)
if n >= 1 {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cl_symlink_protection", Title: "CloudLinux Symlink Protection",
Status: "pass", Message: fmt.Sprintf("enforce_symlinksifowner = %s", val),
})
} else {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cl_symlink_protection", Title: "CloudLinux Symlink Protection",
Status: "fail", Message: fmt.Sprintf("enforce_symlinksifowner = %s (expected >= 1)", val),
Fix: "sysctl -w fs.enforce_symlinksifowner=1",
})
}
}
// cl_proc_virtualization
if data, err := osFS.ReadFile("/proc/sys/fs/proc_can_see_other_uid"); err == nil {
val := strings.TrimSpace(string(data))
if val == "0" {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cl_proc_virtualization", Title: "CloudLinux /proc Virtualization",
Status: "pass", Message: "proc_can_see_other_uid = 0",
})
} else {
results = append(results, store.AuditResult{
Category: "cpanel", Name: "cl_proc_virtualization", Title: "CloudLinux /proc Virtualization",
Status: "fail", Message: fmt.Sprintf("proc_can_see_other_uid = %s (expected 0)", val),
Fix: "sysctl -w fs.proc_can_see_other_uid=0",
})
}
}
return results
}
// --- PHP checks ---
var phpAuditNow = time.Now
// phpSecuritySupportEnds is the last day each PHP branch gets upstream
// security fixes (php.net/supported-versions.php). Branches older than the
// oldest entry are long past end of life; newer ones are still supported.
var phpSecuritySupportEnds = map[string]time.Time{
"8.1": time.Date(2025, 12, 31, 0, 0, 0, 0, time.UTC),
"8.2": time.Date(2026, 12, 31, 0, 0, 0, 0, time.UTC),
"8.3": time.Date(2027, 12, 31, 0, 0, 0, 0, time.UTC),
"8.4": time.Date(2028, 12, 31, 0, 0, 0, 0, time.UTC),
"8.5": time.Date(2029, 12, 31, 0, 0, 0, 0, time.UTC),
}
func phpBranchEOL(major, minor int, now time.Time) bool {
end, ok := phpSecuritySupportEnds[fmt.Sprintf("%d.%d", major, minor)]
if !ok {
return major < 8 || (major == 8 && minor < 1)
}
return !now.Before(end.AddDate(0, 0, 1))
}
func auditPHP(serverType string) []store.AuditResult {
var results []store.AuditResult
type phpInstall struct {
version string // e.g. "8.1"
shortID string // e.g. "81"
iniPath string
scanDir string // additional .ini files PHP parses after iniPath
scanFiles []string
scanFilesKnown bool
fpmDir string // for FPM pool override merging
}
var installs []phpInstall
// cPanel EA4 PHP installs
eaInis, _ := osFS.Glob("/opt/cpanel/ea-php*/root/etc/php.ini")
for _, ini := range eaInis {
// Extract version from path: /opt/cpanel/ea-php81/root/etc/php.ini -> "81"
dir := filepath.Dir(filepath.Dir(filepath.Dir(ini))) // /opt/cpanel/ea-php81/root -> /opt/cpanel/ea-php81
base := filepath.Base(dir) // ea-php81
shortID := strings.TrimPrefix(base, "ea-php")
if len(shortID) >= 2 {
ver := shortID[:len(shortID)-1] + "." + shortID[len(shortID)-1:]
fpmDir := filepath.Join(dir, "root", "etc", "php-fpm.d")
installs = append(installs, phpInstall{
version: ver,
shortID: shortID,
iniPath: ini,
scanDir: filepath.Join(dir, "root", "etc", "php.d"),
fpmDir: fpmDir,
})
}
}
// CloudLinux alt-php installs (skip Imunify360's internal PHP builds)
if serverType == "cloudlinux" {
altInis, _ := osFS.Glob("/opt/alt/php*/etc/php.ini")
for _, ini := range altInis {
dir := filepath.Dir(filepath.Dir(ini)) // /opt/alt/php81
base := filepath.Base(dir) // php81
if strings.Contains(base, "-") {
continue // skip php74-imunify, php81-hardened, etc.
}
shortID := strings.TrimPrefix(base, "php")
if len(shortID) >= 2 {
ver := shortID[:len(shortID)-1] + "." + shortID[len(shortID)-1:]
// No scanDir: alt-php scans a per-user CageFS link, and the
// PHP Selector writes global settings into php.ini itself.
installs = append(installs, phpInstall{
version: ver,
shortID: shortID,
iniPath: ini,
})
}
}
}
// Bare server fallback
if len(installs) == 0 {
out, err := auditRunCmd("php", "-i")
if err == nil {
scanDir := ""
for _, line := range strings.Split(string(out), "\n") {
if strings.HasPrefix(line, "Scan this dir for additional .ini files") {
if parts := strings.SplitN(line, "=>", 2); len(parts) == 2 {
if dir := strings.TrimSpace(parts[1]); dir != "(none)" {
scanDir = dir
}
}
}
if strings.HasPrefix(line, "Loaded Configuration File") {
parts := strings.SplitN(line, "=>", 2)
if len(parts) == 2 {
iniPath := strings.TrimSpace(parts[1])
if iniPath != "(none)" && iniPath != "" {
installs = append(installs, phpInstall{
version: "unknown",
shortID: "system",
iniPath: iniPath,
})
}
}
}
}
if len(installs) > 0 {
installs[0].scanDir = scanDir
installs[0].scanFiles, installs[0].scanFilesKnown = phpAuditScannedFiles(string(out))
}
}
// Try to get version for bare
if len(installs) > 0 && installs[0].version == "unknown" {
vout, verr := auditRunCmd("php", "-v")
if verr == nil {
first := strings.SplitN(string(vout), "\n", 2)[0]
// "PHP 8.2.15 (cli) ..."
fields := strings.Fields(first)
if len(fields) >= 2 {
verParts := strings.SplitN(fields[1], ".", 3)
if len(verParts) >= 2 {
installs[0].version = verParts[0] + "." + verParts[1]
installs[0].shortID = verParts[0] + verParts[1]
}
}
}
}
}
for _, inst := range installs {
data, err := osFS.ReadFile(inst.iniPath)
if err != nil {
continue
}
settings := phpAuditINI{"": {}}
settings.merge(string(data), false)
complete := true
// PHP reads the scan dir in name order after php.ini, so a later
// file overrides an earlier one.
extra := inst.scanFiles
if !inst.scanFilesKnown && inst.scanDir != "" {
for _, dir := range filepath.SplitList(inst.scanDir) {
if dir == "" {
complete = false
continue
}
files, err := osFS.Glob(filepath.Join(dir, "*.ini"))
if err != nil {
complete = false
}
slices.Sort(files)
extra = append(extra, files...)
}
}
for _, f := range extra {
edata, eerr := osFS.ReadFile(f)
if eerr != nil {
complete = false
continue
}
settings.merge(string(edata), false)
}
// Existing checks merge pool values below. Preserve the global and
// conditional settings for the upload-progress audit.
ini := make(map[string]string, len(settings[""]))
for k, v := range settings[""] {
ini[k] = v
}
pools := phpAuditINI{}
// Merge FPM pool overrides if available
if inst.fpmDir != "" {
poolConfs, err := osFS.Glob(filepath.Join(inst.fpmDir, "*.conf"))
if err != nil {
complete = false
}
slices.Sort(poolConfs)
for _, pc := range poolConfs {
pdata, perr := osFS.ReadFile(pc)
if perr != nil {
complete = false
continue
}
pools.merge(string(pdata), true)
for _, line := range strings.Split(string(pdata), "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, ";") {
continue
}
// php_admin_value[key] = val or php_value[key] = val
for _, prefix := range []string{"php_admin_value[", "php_value["} {
if strings.HasPrefix(line, prefix) {
rest := strings.TrimPrefix(line, prefix)
if idx := strings.Index(rest, "]"); idx > 0 {
key := rest[:idx]
valPart := rest[idx+1:]
if eqIdx := strings.Index(valPart, "="); eqIdx >= 0 {
val := strings.TrimSpace(valPart[eqIdx+1:])
ini[key] = val
}
}
}
}
}
}
}
suffix := inst.shortID
// php_version check
major, minor := parsePHPVersion(inst.version)
if major > 0 {
if phpBranchEOL(major, minor, phpAuditNow()) {
results = append(results, store.AuditResult{
Category: "php", Name: "php_version_" + suffix, Title: fmt.Sprintf("PHP %s Version", inst.version),
Status: "fail", Message: fmt.Sprintf("PHP %s is end-of-life", inst.version),
Fix: fmt.Sprintf("Upgrade PHP %s to a supported branch. Upstream no longer releases fixes for it; confirm any vendor backport in the package changelog.", inst.version),
})
} else {
results = append(results, store.AuditResult{
Category: "php", Name: "php_version_" + suffix, Title: fmt.Sprintf("PHP %s Version", inst.version),
Status: "pass", Message: fmt.Sprintf("PHP %s is supported", inst.version),
})
}
}
// php_disable_functions
df := strings.TrimSpace(ini["disable_functions"])
if df == "" || strings.EqualFold(df, "none") {
results = append(results, store.AuditResult{
Category: "php", Name: "php_disable_functions_" + suffix, Title: fmt.Sprintf("PHP %s disable_functions", inst.version),
Status: "fail", Message: "disable_functions is empty",
Fix: fmt.Sprintf("Set disable_functions in %s to include dangerous functions like exec, system, passthru, shell_exec, popen, proc_open.", inst.iniPath),
})
} else {
results = append(results, store.AuditResult{
Category: "php", Name: "php_disable_functions_" + suffix, Title: fmt.Sprintf("PHP %s disable_functions", inst.version),
Status: "pass", Message: "disable_functions is configured",
})
}
// php_expose
expose := strings.TrimSpace(strings.ToLower(ini["expose_php"]))
if expose == "off" || expose == "0" {
results = append(results, store.AuditResult{
Category: "php", Name: "php_expose_" + suffix, Title: fmt.Sprintf("PHP %s expose_php", inst.version),
Status: "pass", Message: "expose_php is off",
})
} else {
results = append(results, store.AuditResult{
Category: "php", Name: "php_expose_" + suffix, Title: fmt.Sprintf("PHP %s expose_php", inst.version),
Status: "warn", Message: "expose_php is on — PHP version disclosed in headers",
Fix: fmt.Sprintf("Set expose_php = Off in %s", inst.iniPath),
})
}
// php_allow_url_fopen
auf := strings.TrimSpace(strings.ToLower(ini["allow_url_fopen"]))
if auf == "off" || auf == "0" {
results = append(results, store.AuditResult{
Category: "php", Name: "php_allow_url_fopen_" + suffix, Title: fmt.Sprintf("PHP %s allow_url_fopen", inst.version),
Status: "pass", Message: "allow_url_fopen is off",
})
} else {
results = append(results, store.AuditResult{
Category: "php", Name: "php_allow_url_fopen_" + suffix, Title: fmt.Sprintf("PHP %s allow_url_fopen", inst.version),
Status: "warn", Message: "allow_url_fopen is on — remote file inclusion risk",
Fix: fmt.Sprintf("Set allow_url_fopen = Off in %s", inst.iniPath),
})
}
// php_enable_dl
edl := strings.TrimSpace(strings.ToLower(ini["enable_dl"]))
if edl == "on" || edl == "1" {
results = append(results, store.AuditResult{
Category: "php", Name: "php_enable_dl_" + suffix, Title: fmt.Sprintf("PHP %s enable_dl", inst.version),
Status: "fail", Message: "enable_dl is on — allows loading arbitrary shared objects",
Fix: fmt.Sprintf("Set enable_dl = Off in %s", inst.iniPath),
})
} else {
results = append(results, store.AuditResult{
Category: "php", Name: "php_enable_dl_" + suffix, Title: fmt.Sprintf("PHP %s enable_dl", inst.version),
Status: "pass", Message: "enable_dl is off",
})
}
offFix := func(setting string) string {
if inst.scanDir != "" && !strings.Contains(inst.scanDir, string(filepath.ListSeparator)) {
return fmt.Sprintf("Add %s = Off to an .ini file in %s", setting, inst.scanDir)
}
return fmt.Sprintf("Set %s = Off in %s", setting, inst.iniPath)
}
// php_upload_progress: on by default, and its handler runs on any
// multipart POST during request startup, before any script or
// auto_prepend file, so only this switch removes that code path.
if !complete || !settings.settingDisabled(phpUploadProgressKey, pools) {
fix := offFix(phpUploadProgressKey)
results = append(results, store.AuditResult{
Category: "php", Name: "php_upload_progress_" + suffix, Title: fmt.Sprintf("PHP %s session upload progress", inst.version),
Status: "warn", Message: "session.upload_progress.enabled is on or could not be confirmed off in all inspected settings",
Fix: fix + ", check site and FPM pool overrides, then restart PHP and verify the web configuration.",
})
} else {
results = append(results, store.AuditResult{
Category: "php", Name: "php_upload_progress_" + suffix, Title: fmt.Sprintf("PHP %s session upload progress", inst.version),
Status: "pass", Message: "session.upload_progress.enabled is off in all inspected settings",
})
}
// php_fatal_error_backtraces: PHP 8.5 added the setting, on by
// default. The CVE-2026-109619 advisory names turning it off as the
// mitigation for unpatched builds. Scripts can still enable it at
// runtime, so a pass means the configured default is off.
if major > 8 || (major == 8 && minor >= 5) {
name := "php_fatal_error_backtraces_" + suffix
title := fmt.Sprintf("PHP %s fatal error backtraces", inst.version)
if !complete || !settings.settingDisabled(phpFatalErrorBacktracesKey, pools) {
results = append(results, store.AuditResult{
Category: "php", Name: name, Title: title,
Status: "warn", Message: "fatal_error_backtraces is on or could not be confirmed off in all inspected settings",
Fix: offFix(phpFatalErrorBacktracesKey) + ", check site and FPM pool overrides, then restart PHP.",
})
} else {
results = append(results, store.AuditResult{
Category: "php", Name: name, Title: title,
Status: "pass", Message: "fatal_error_backtraces is off in all inspected settings; scripts can still enable it at runtime",
})
}
}
}
return results
}
func parsePHPIni(content string) map[string]string {
ini := phpAuditINI{"": {}}
ini.merge(content, false)
return ini[""]
}
func parsePHPVersion(ver string) (int, int) {
parts := strings.SplitN(ver, ".", 3)
if len(parts) < 2 {
return 0, 0
}
major, _ := strconv.Atoi(parts[0])
minor, _ := strconv.Atoi(parts[1])
return major, minor
}
// --- Web server checks ---
func auditWebServer() []store.AuditResult {
var results []store.AuditResult
if configPath := findWebServerConfigPath(platform.Detect()); configPath != "" {
results = append(results, auditApacheDirectives(configPath)...)
}
return append(results, auditWebTLS()...)
}
// findWebServerConfigPath returns the active Apache-compatible main file.
// Native LiteSpeed XML uses different syntax; cPanel LiteSpeed consumes the
// EA4 Apache tree and is the only LiteSpeed case this audit can inspect.
func findWebServerConfigPath(info platform.Info) string {
if info.WebServer == platform.WSLiteSpeed && !info.IsCPanel() {
return ""
}
// A selected root is authoritative even when its main file is missing.
// Falling back could report another server's direct logs as this one's.
if root := info.ApacheConfigDir; root != "" {
var main string
switch {
case info.IsCPanel() && filepath.Base(filepath.Clean(root)) == "conf":
// Older cPanel detection supplies the conf directory itself.
main = filepath.Join(root, "httpd.conf")
case info.IsCPanel(), info.IsRHELFamily():
main = filepath.Join(root, "conf", "httpd.conf")
case info.IsDebianFamily():
main = filepath.Join(root, "apache2.conf")
default:
main = filepath.Join(root, "httpd.conf")
}
// Packages such as php-fpm leave Apache snippet directories on
// hosts another web server runs; no main file there means no Apache.
if info.WebServer != platform.WSApache && !info.IsCPanel() {
if _, err := osFS.Stat(main); errors.Is(err, os.ErrNotExist) {
return ""
}
}
return main
}
if info.IsCPanel() {
return "/etc/apache2/conf/httpd.conf"
}
var candidates []string
switch {
case info.IsDebianFamily():
candidates = append(candidates, "/etc/apache2/apache2.conf")
case info.IsRHELFamily():
candidates = append(candidates, "/etc/httpd/conf/httpd.conf")
case len(candidates) == 0:
candidates = append(candidates, "/etc/httpd/conf/httpd.conf", "/etc/apache2/apache2.conf")
}
seen := make(map[string]bool)
for _, p := range candidates {
if seen[p] {
continue
}
seen[p] = true
if _, err := osFS.Stat(p); err == nil {
return p
} else if !errors.Is(err, os.ErrNotExist) {
return p
}
}
return ""
}
// auditApacheDirectivesMaxScopes caps how many indexing scopes a single
// finding names, so a host with hundreds of vhosts still gets a readable
// message.
const auditApacheDirectivesMaxScopes = 5
// auditApacheDirectives checks the effective Apache configuration after
// Include and IncludeOptional targets have been spliced into place.
func auditApacheDirectives(configPath string) []store.AuditResult {
var results []store.AuditResult
lines, assemblyComplete := assembleApacheConfigWithStatus(configPath)
type directiveCheck struct {
id, title, directive string
goodValues []string
allowScoped bool
}
dirChecks := []directiveCheck{
{"web_server_tokens", "ServerTokens", "ServerTokens", []string{"prod", "productonly"}, false},
{"web_server_signature", "ServerSignature", "ServerSignature", []string{"off"}, true},
{"web_trace_enable", "TraceEnable", "TraceEnable", []string{"off"}, false},
{"web_file_etag", "FileETag", "FileETag", []string{"none"}, true},
}
for _, dc := range dirChecks {
values, configValid := apacheDirectiveValues(lines, dc.directive)
var serverValue *apacheDirectiveValue
var unsafeValue *apacheDirectiveValue
var unsafeConditional *apacheDirectiveValue
for i := range values {
value := &values[i]
if value.Scope == "server config" && !value.Conditional {
serverValue = value
}
if value.Scope != "server config" && !dc.allowScoped {
configValid = false
continue
}
if apacheDirectiveValueIsGood(value.Value, dc.goodValues) {
continue
}
if value.Conditional {
unsafeConditional = value
} else {
unsafeValue = value
break
}
}
if unsafeValue != nil {
results = append(results, store.AuditResult{
Category: "webserver", Name: dc.id, Title: dc.title,
Status: "fail", Message: fmt.Sprintf("%s = %s in %s (%s)", dc.directive, unsafeValue.Value, unsafeValue.Scope, unsafeValue.File),
Fix: fmt.Sprintf("Set '%s %s' in %s", dc.directive, dc.goodValues[0], unsafeValue.File),
})
continue
}
if unsafeConditional != nil {
results = append(results, store.AuditResult{
Category: "webserver", Name: dc.id, Title: dc.title,
Status: "warn", Message: fmt.Sprintf("%s may be %s in conditional %s scope (%s)", dc.directive, unsafeConditional.Value, unsafeConditional.Scope, unsafeConditional.File),
Fix: fmt.Sprintf("Set '%s %s' in %s", dc.directive, dc.goodValues[0], unsafeConditional.File),
})
continue
}
if serverValue == nil {
results = append(results, store.AuditResult{
Category: "webserver", Name: dc.id, Title: dc.title,
Status: "warn", Message: fmt.Sprintf("%s not set in %s or its included snippets", dc.directive, configPath),
Fix: fmt.Sprintf("Add '%s %s' to %s", dc.directive, dc.goodValues[0], configPath),
})
continue
}
if assemblyComplete && configValid {
results = append(results, store.AuditResult{
Category: "webserver", Name: dc.id, Title: dc.title,
Status: "pass", Message: fmt.Sprintf("%s = %s (%s)", dc.directive, serverValue.Value, serverValue.File),
})
continue
}
results = append(results, store.AuditResult{
Category: "webserver", Name: dc.id, Title: dc.title,
Status: "warn", Message: fmt.Sprintf("%s appears secure, but the Apache configuration could not be fully evaluated", dc.directive),
})
}
scopes, optionsValid := apacheIndexesScopesWithStatus(lines)
if len(scopes) == 0 {
if !assemblyComplete || !optionsValid {
results = append(results, store.AuditResult{
Category: "webserver", Name: "web_directory_listing", Title: "Directory Listing",
Status: "warn", Message: "Apache Options could not be fully evaluated",
})
return results
}
results = append(results, store.AuditResult{
Category: "webserver", Name: "web_directory_listing", Title: "Directory Listing",
Status: "pass", Message: "No configured scope enables directory listing",
})
return results
}
shown := scopes
suffix := ""
if len(shown) > auditApacheDirectivesMaxScopes {
suffix = fmt.Sprintf(" (+%d more)", len(shown)-auditApacheDirectivesMaxScopes)
shown = shown[:auditApacheDirectivesMaxScopes]
}
results = append(results, store.AuditResult{
Category: "webserver", Name: "web_directory_listing", Title: "Directory Listing",
Status: "warn",
Message: fmt.Sprintf("Directory listing enabled in: %s%s", strings.Join(shown, ", "), suffix),
Fix: "Replace 'Indexes' with '-Indexes' in the Options directive of each scope listed.",
})
return results
}
func apacheDirectiveValueIsGood(value string, goodValues []string) bool {
for _, good := range goodValues {
if strings.EqualFold(value, good) {
return true
}
}
return false
}
// auditWebTLS probes the local HTTPS listener for legacy TLS versions.
func auditWebTLS() []store.AuditResult {
var results []store.AuditResult
for _, tc := range []struct {
id, title, flag, version string
}{
{"web_tls_version", "Legacy TLS Disabled", "-tls1", "TLSv1.0"},
{"web_tls11_version", "TLS 1.1 Disabled", "-tls1_1", "TLSv1.1"},
} {
out, err := auditRunCmd("openssl", "s_client", "-connect", "localhost:443", tc.flag)
output := string(out)
// If the handshake succeeds, output contains "SSL-Session:" without ":error:" on the same handshake
succeeded := err == nil && strings.Contains(output, "SSL-Session:") && !strings.Contains(output, ":error:")
if succeeded {
results = append(results, store.AuditResult{
Category: "webserver", Name: tc.id, Title: tc.title,
Status: "fail", Message: fmt.Sprintf("Server accepts %s connections", tc.version),
Fix: fmt.Sprintf("Disable %s in your web server's SSL configuration. Minimum should be TLSv1.2.", tc.version),
})
} else {
results = append(results, store.AuditResult{
Category: "webserver", Name: tc.id, Title: tc.title,
Status: "pass", Message: fmt.Sprintf("%s is rejected", tc.version),
})
}
}
return results
}
// --- Mail checks ---
// detectMTA is a seam so tests can pin the host's mail stack.
var detectMTA = platform.DetectMTA
var isCPanelHost = func() bool { return platform.Detect().IsCPanel() }
func auditMail() []store.AuditResult {
var results []store.AuditResult
// mail_root_forwarder
if info, err := osFS.Stat("/root/.forward"); err != nil {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_root_forwarder", Title: "Root Mail Forwarder",
Status: "warn", Message: "/root/.forward does not exist — root mail may go unread",
Fix: "Create /root/.forward with an email address to receive root's mail.",
})
} else if info.Size() == 0 {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_root_forwarder", Title: "Root Mail Forwarder",
Status: "warn", Message: "/root/.forward is empty — root mail may go unread",
Fix: "Add an email address to /root/.forward to receive root's mail.",
})
} else {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_root_forwarder", Title: "Root Mail Forwarder",
Status: "pass", Message: "Root mail forwarding is configured",
})
}
// Exim-specific checks only run where exim is the delivery agent.
// Reporting them on a postfix host produced both phantom warnings
// and a fabricated pass for a cPanel-only override file.
if detectMTA() == platform.MTAExim {
// Get exim config for multiple checks
eximOut, eximErr := auditRunCmd("exim", "-bP")
eximConfig := ""
if eximErr == nil {
eximConfig = string(eximOut)
}
// mail_exim_logging
if eximConfig != "" {
lower := strings.ToLower(eximConfig)
if strings.Contains(lower, "+arguments") || strings.Contains(lower, "+all") {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_exim_logging", Title: "Exim Argument Logging",
Status: "pass", Message: "Exim logs include +arguments",
})
} else {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_exim_logging", Title: "Exim Argument Logging",
Status: "warn", Message: "Exim log_selector does not include +arguments",
Fix: "Add '+arguments' to log_selector in exim configuration for better forensics.",
})
}
} else {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_exim_logging", Title: "Exim Argument Logging",
Status: "warn", Message: "Cannot query exim configuration",
})
}
// mail_exim_tls: check for SSLv2 in tls_require_ciphers
// +no_sslv2 in openssl_options means SSLv2 is DISABLED (good).
// Only flag if SSLv2 is referenced WITHOUT +no_ prefix.
if eximConfig != "" {
lower := strings.ToLower(eximConfig)
hasSslv2 := strings.Contains(lower, "sslv2")
isDisabled := strings.Contains(lower, "+no_sslv2") || strings.Contains(lower, "no_sslv2")
if hasSslv2 && !isDisabled {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_exim_tls", Title: "Exim TLS Ciphers",
Status: "fail", Message: "Exim allows SSLv2 connections",
Fix: "Add '+no_sslv2' to openssl_options in exim configuration.",
})
} else {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_exim_tls", Title: "Exim TLS Ciphers",
Status: "pass", Message: "SSLv2 is disabled in exim TLS configuration",
})
}
} else {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_exim_tls", Title: "Exim TLS Ciphers",
Status: "warn", Message: "Cannot query exim TLS configuration",
})
}
// require_secure_auth is a cPanel-managed Exim setting. Other Exim
// packages do not use this file or option.
if isCPanelHost() {
if data, err := osFS.ReadFile("/etc/exim.conf.localopts"); err == nil {
disabled, valid := cpanelSecureAuthDisabled(data)
switch {
case !valid:
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_secure_auth", Title: "Exim Secure Authentication",
Status: "warn", Message: "Cannot evaluate require_secure_auth in /etc/exim.conf.localopts",
})
case disabled:
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_secure_auth", Title: "Exim Secure Authentication",
Status: "fail", Message: "require_secure_auth is disabled in /etc/exim.conf.localopts",
Fix: "Remove or set require_secure_auth=1 in /etc/exim.conf.localopts.",
})
default:
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_secure_auth", Title: "Exim Secure Authentication",
Status: "pass", Message: "Secure authentication is not disabled",
})
}
} else if errors.Is(err, os.ErrNotExist) {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_secure_auth", Title: "Exim Secure Authentication",
Status: "pass", Message: "No local exim overrides file found (default is secure)",
})
} else {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_secure_auth", Title: "Exim Secure Authentication",
Status: "warn", Message: "Cannot read /etc/exim.conf.localopts",
})
}
}
}
if detectMTA() == platform.MTAPostfix {
results = append(results, auditPostfix()...)
}
// mail_dovecot_tls: check ssl_min_protocol
// Use 'doveconf -a' for the effective config — cPanel manages Dovecot
// settings outside the standard config files, so file parsing misses
// them. Routed through cmdExec so tests can mock the doveconf output.
dovecotTLS := false
if out, err := cmdExec.Run("doveconf", "-a"); err == nil {
for _, line := range strings.Split(string(out), "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "ssl_min_protocol") {
val := strings.TrimSpace(strings.TrimPrefix(trimmed, "ssl_min_protocol"))
val = strings.TrimLeft(val, "= ")
if strings.Contains(val, "TLSv1.2") || strings.Contains(val, "TLSv1.3") {
dovecotTLS = true
}
}
}
} else {
// Fallback: try config files
for _, path := range []string{"/etc/dovecot/conf.d/10-ssl.conf", "/etc/dovecot/dovecot.conf"} {
data, readErr := osFS.ReadFile(path)
if readErr != nil {
continue
}
for _, line := range strings.Split(string(data), "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "#") {
continue
}
if strings.HasPrefix(trimmed, "ssl_min_protocol") {
val := strings.TrimSpace(strings.TrimPrefix(trimmed, "ssl_min_protocol"))
val = strings.TrimLeft(val, "= ")
if strings.Contains(val, "TLSv1.2") || strings.Contains(val, "TLSv1.3") {
dovecotTLS = true
}
}
}
}
}
if dovecotTLS {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_dovecot_tls", Title: "Dovecot TLS Minimum",
Status: "pass", Message: "Dovecot ssl_min_protocol is TLSv1.2 or higher",
})
} else {
if _, err := osFS.Stat("/etc/dovecot/dovecot.conf"); err != nil {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_dovecot_tls", Title: "Dovecot TLS Minimum",
Status: "warn", Message: "Dovecot configuration not found",
})
} else {
results = append(results, store.AuditResult{
Category: "mail", Name: "mail_dovecot_tls", Title: "Dovecot TLS Minimum",
Status: "fail", Message: "Dovecot ssl_min_protocol not set to TLSv1.2 or higher",
Fix: "Set 'ssl_min_protocol = TLSv1.2' in /etc/dovecot/conf.d/10-ssl.conf.",
})
}
}
return results
}
func cpanelSecureAuthDisabled(data []byte) (disabled, valid bool) {
valid = true
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
assignment, _, _ := strings.Cut(line, "#")
key, value, found := strings.Cut(assignment, "=")
if !found || !strings.EqualFold(strings.TrimSpace(key), "require_secure_auth") {
continue
}
switch strings.TrimSpace(value) {
case "0":
disabled, valid = true, true
case "1":
disabled, valid = false, true
default:
disabled, valid = false, false
}
}
return disabled, valid
}
func webUsersWithNobody(users []string) []string {
if slices.Contains(users, "nobody") {
return users
}
return append(append([]string(nil), users...), "nobody")
}
// auditWebUserCrontab reports a crontab with content for any web-server
// user. The result keeps the historical os_nobody_cron name so stored
// audits and dashboards stay comparable across platforms.
func auditWebUserCrontab(spoolDir string, users []string) store.AuditResult {
for _, user := range users {
info, err := osFS.Stat(filepath.Join(spoolDir, user))
if err != nil || info.Size() == 0 {
continue
}
return store.AuditResult{
Category: "os", Name: "os_nobody_cron", Title: "Web User Crontab",
Status: "fail", Message: fmt.Sprintf("web server user %s has a crontab with content", user),
Fix: fmt.Sprintf("Review and remove: crontab -u %s -r", user),
}
}
return store.AuditResult{
Category: "os", Name: "os_nobody_cron", Title: "Web User Crontab",
Status: "pass", Message: fmt.Sprintf("No crontab with content for web server user(s) %s", strings.Join(users, ", ")),
}
}
// checkFirewallDefaultPolicy audits the default policy of the chain that
// filters inbound traffic. Only an nft chain on the input hook counts: a
// Docker host carries "policy drop" on its FORWARD chain while INPUT stays
// at accept, and that used to pass this audit.
func checkFirewallDefaultPolicy(hasNft bool, nftRules string, hasIpt bool, iptRules string) store.AuditResult {
defaultDeny := hasNft && nftInputDefaultDeny(nftRules)
if !defaultDeny && hasIpt {
for _, line := range strings.Split(iptRules, "\n") {
if strings.HasPrefix(line, "Chain INPUT") {
upper := strings.ToUpper(line)
if strings.Contains(upper, "POLICY DROP") || strings.Contains(upper, "POLICY REJECT") {
defaultDeny = true
}
break
}
}
}
if defaultDeny {
return store.AuditResult{
Category: "firewall", Name: "fw_default_policy", Title: "Default INPUT Policy",
Status: "pass", Message: "INPUT chain has default-deny policy",
}
}
return store.AuditResult{
Category: "firewall", Name: "fw_default_policy", Title: "Default INPUT Policy",
Status: "fail", Message: "INPUT chain does not have a DROP/REJECT policy",
Fix: "Set the default INPUT policy to DROP: iptables -P INPUT DROP (or nft equivalent).",
}
}
// nftInputDefaultDeny reports whether an nft ruleset has a drop or reject
// policy on a chain hooked at input. nft prints the hook and the policy on
// one line; a policy on a following line inside the same chain also counts.
func nftInputDefaultDeny(nftRules string) bool {
inChain := false
chainDepth := 0
inputHook := false
denyPolicy := false
for _, raw := range strings.Split(nftRules, "\n") {
line := strings.ToLower(strings.TrimSpace(nftCodeOnly(raw)))
opens := strings.Count(line, "{")
closes := strings.Count(line, "}")
if strings.HasPrefix(line, "chain ") {
inChain = true
chainDepth = 0
inputHook = false
denyPolicy = false
}
if !inChain {
continue
}
inputHook = inputHook || nftHasWordPair(line, "hook", "input")
denyPolicy = denyPolicy || nftHasWordPair(line, "policy", "drop") || nftHasWordPair(line, "policy", "reject")
if inputHook && denyPolicy {
return true
}
chainDepth += opens - closes
if chainDepth <= 0 && closes > 0 {
inChain = false
}
}
return false
}
func nftCodeOnly(line string) string {
var b strings.Builder
var quote byte
escaped := false
for i := 0; i < len(line); i++ {
c := line[i]
if quote != 0 {
if escaped {
escaped = false
continue
}
if c == '\\' {
escaped = true
continue
}
if c == quote {
quote = 0
}
continue
}
if c == '#' {
break
}
if c == '\'' || c == '"' {
quote = c
b.WriteByte(' ')
continue
}
b.WriteByte(c)
}
return b.String()
}
func nftHasWordPair(line, first, second string) bool {
words := strings.FieldsFunc(line, func(r rune) bool {
return (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '_'
})
for i := 1; i < len(words); i++ {
if words[i-1] == first && words[i] == second {
return true
}
}
return false
}
package checks
import "strings"
const (
phpUploadProgressKey = "session.upload_progress.enabled"
phpFatalErrorBacktracesKey = "fatal_error_backtraces"
)
// FPM pool overrides are tracked only for the settings the audit evaluates.
var phpAuditPoolSettings = []string{phpUploadProgressKey, phpFatalErrorBacktracesKey}
// Keep conditional settings separate: disabling one site does not disable
// other sites that inherit PHP's global value.
type phpAuditINI map[string]map[string]string
func (ini phpAuditINI) merge(content string, pools bool) {
scope := ""
lines := strings.Split(content, "\n")
for i := 0; i < len(lines); i++ {
line := strings.TrimSpace(lines[i])
if line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") {
continue
}
if strings.HasPrefix(line, "[") {
if end := strings.IndexByte(line, ']'); end > 0 {
section := strings.TrimSpace(line[1:end])
scope = ""
if pools {
scope = section
} else if upper := strings.ToUpper(section); strings.HasPrefix(upper, "PATH") || strings.HasPrefix(upper, "HOST") {
name := strings.TrimRight(strings.TrimLeft(section[4:], "= \t"), "/\\")
if strings.HasPrefix(upper, "HOST") {
name = strings.ToLower(name)
}
scope = upper[:4] + "=" + name
}
}
continue
}
key, value, ok := strings.Cut(line, "=")
if !ok {
continue
}
key = strings.TrimSpace(key)
// Quoted INI values can span lines. A directive-looking line inside
// such a string must not override a real setting.
start := i
quote := phpAuditQuoteState(value, 0)
for quote != 0 && i+1 < len(lines) {
i++
quote = phpAuditQuoteState(lines[i], quote)
}
if i > start {
value += "\n" + strings.Join(lines[start+1:i+1], "\n")
}
if pools {
if key = phpAuditPoolKey(key); key == "" {
continue
}
}
if ini[scope] == nil {
ini[scope] = make(map[string]string)
}
if _, set := ini[scope][key]; pools && set {
// FPM prepends pool values to a list and applies it in reverse
// configuration order, so the first duplicate wins.
continue
}
ini[scope][key] = strings.TrimSpace(value)
}
}
func phpAuditQuotesClosed(value string) bool {
return phpAuditQuoteState(value, 0) == 0
}
func phpAuditQuoteState(value string, quote byte) byte {
for i := 0; i < len(value); i++ {
ch := value[i]
switch {
case quote == 0:
if ch == ';' {
return 0
}
if ch == '\'' || ch == '"' {
quote = ch
}
case ch == quote:
quote = 0
case quote == '"' && ch == '\\' && i+1 < len(value):
i++
}
}
return quote
}
// phpAuditPoolKey maps a tracked pool directive to "admin:<setting>" or
// "value:<setting>", and anything else to "".
func phpAuditPoolKey(directive string) string {
prefix, offset, ok := strings.Cut(directive, "[")
if !ok || !strings.HasSuffix(offset, "]") {
return ""
}
var kind string
switch strings.TrimSpace(prefix) {
case "php_value", "php_flag":
kind = "value:"
case "php_admin_value", "php_admin_flag":
kind = "admin:"
default:
return ""
}
offset = strings.TrimLeft(strings.TrimSuffix(offset, "]"), " \t")
if !phpAuditQuotesClosed(offset) {
return ""
}
// PHP ignores whitespace after a closing quote, but keeps it as part
// of an unquoted offset. Trimming both would alias distinct settings.
trimmed := strings.TrimRight(offset, " \t")
if strings.HasSuffix(trimmed, "\"") || strings.HasSuffix(trimmed, "'") {
offset = trimmed
}
offset, _ = joinPHPIniQuotedFragments(offset)
for _, setting := range phpAuditPoolSettings {
if offset == setting {
return kind + setting
}
}
return ""
}
// settingDisabled reports whether a boolean setting that defaults to on is
// explicitly off globally and not turned back on by any section or pool.
func (ini phpAuditINI) settingDisabled(key string, pools phpAuditINI) bool {
value, set := ini[""][key]
if !set || !phpAuditBoolDisabled(value) {
return false
}
for _, settings := range ini {
if value, set := settings[key]; set && !phpAuditBoolDisabled(value) {
return false
}
}
for _, settings := range pools {
value, set := settings["admin:"+key]
if !set {
value, set = settings["value:"+key]
}
if set && !phpAuditBoolDisabled(value) {
return false
}
}
return true
}
func phpAuditBoolDisabled(raw string) bool {
raw = stripPHPIniInlineComment(raw)
if !phpAuditQuotesClosed(raw) || strings.Contains(raw, "${") {
// PHP workers can have a different environment from the daemon.
return false
}
value, expression := normalizePHPIniValueWithExpression(raw)
if expression {
n, ok := parsePHPIniIntExpression(value)
return ok && n == 0
}
switch strings.ToLower(value) {
case "", "off", "no", "false", "none":
return true
}
// Unresolved constants are not proof of a disabled setting. Numeric
// strings use PHP's leading-integer conversion, shared with the scanner.
number := strings.TrimLeft(value, "+-")
if number == "" || number[0] < '0' || number[0] > '9' {
return false
}
return !phpIniBoolEnabled(raw)
}
// phpinfo lists the files actually parsed, including the expansion of empty
// PHP_INI_SCAN_DIR components to the compiled-in directory. Keep that order.
func phpAuditScannedFiles(info string) ([]string, bool) {
var files []string
found := false
for _, line := range strings.Split(info, "\n") {
switch {
case strings.HasPrefix(line, "Additional .ini files parsed =>"):
_, line, _ = strings.Cut(line, "=>")
found = true
case !found:
continue
case strings.TrimSpace(line) == "" || strings.Contains(line, "=>"):
return files, true
}
for _, name := range strings.Split(line, ",") {
if name = strings.TrimSpace(name); name != "" && name != "(none)" {
files = append(files, name)
}
}
}
return files, found
}
package checks
import (
"context"
"fmt"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
)
// CheckHealth verifies that CSM's dependencies are working.
// Reports on missing external commands, broken auditd, etc.
func CheckHealth(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
info := platform.Detect()
// Required commands depend on the platform. On plain Linux hosts we
// don't need Exim/cPanel-specific tooling.
requiredCmds := platformRequiredCommands(info)
for _, cmd := range requiredCmds {
if _, err := cmdExec.LookPath(cmd); err != nil {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "csm_health",
Message: fmt.Sprintf("Required command not found: %s", cmd),
Details: "Some checks will be skipped",
})
}
}
// Optional commands. Only complain about cPanel tools on cPanel hosts.
optionalCmds := map[string]string{
"wp": "WordPress core integrity check will be skipped",
}
if info.IsCPanel() {
optionalCmds["whmapi1"] = "WHM API token check will be skipped"
}
for cmd, impact := range optionalCmds {
if _, err := cmdExec.LookPath(cmd); err != nil {
if cmd == "wp" {
// Failed discovery cannot establish that wp-cli is unnecessary.
discovery := lookupWPInstalls(ctx, "")
if len(discovery.installs) == 0 && !discovery.incomplete {
continue
}
}
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "csm_health",
Message: fmt.Sprintf("Optional command not found: %s", cmd),
Details: impact,
})
}
}
// Check auditd is running and has CSM rules
out, _ := runCmd("auditctl", "-l")
if out != nil {
rules := string(out)
if !strings.Contains(rules, "csm_shadow_change") {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "csm_health",
Message: "auditd CSM rules not loaded",
Details: "Run 'csm install' to deploy auditd rules, then 'service auditd restart'",
})
}
}
if cfg != nil && cfg.BPFEnforcement.Enabled && cfg.BPFEnforcement.DirectSMTPEgress {
switch active := bpf.ActiveKind("connection_tracker"); active {
case bpf.BackendLegacy, bpf.BackendNone:
message := "BPF enforcement enabled but connection tracker is running on legacy backend"
if active == bpf.BackendNone {
message = "BPF enforcement enabled but connection tracker has no active backend"
}
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "csm_health",
Message: message,
Details: "bpf_enforcement.direct_smtp_egress requires the connection tracker BPF backend. Check kernel version, LSM availability, or CAP_BPF.",
})
}
}
// Check state directory is writable
stateDir := "/var/lib/csm/state"
if cfg != nil && cfg.StatePath != "" {
stateDir = cfg.StatePath
}
testFile := filepath.Join(stateDir, ".health_check")
if err := osFS.WriteFile(testFile, []byte("ok"), 0600); err != nil {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "csm_health",
Message: fmt.Sprintf("State directory not writable: %s", stateDir),
Details: err.Error(),
})
} else {
_ = osFS.Remove(testFile)
}
return findings
}
// platformRequiredCommands returns the external commands CSM needs on the
// detected platform. On plain Linux hosts Exim is not required.
func platformRequiredCommands(info platform.Info) []string {
cmds := []string{"find", "auditctl"}
if info.IsCPanel() {
cmds = append(cmds, "exim")
}
return cmds
}
package checks
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// cmdTimeout bounds one external command. A variable so a test can exercise
// the real deadline without waiting two minutes for it.
var cmdTimeout = 2 * time.Minute
var systemCommandSearchDirs = []string{
"/usr/local/sbin",
"/usr/sbin",
"/sbin",
"/usr/local/bin",
"/usr/bin",
"/bin",
}
func hashFileContent(path string) (string, error) {
data, err := osFS.ReadFile(path)
if err != nil {
return "", err
}
h := sha256.Sum256(data)
return fmt.Sprintf("%x", h[:]), nil
}
func hashBytes(data []byte) string {
h := sha256.Sum256(data)
return fmt.Sprintf("%x", h[:])
}
// runCmd delegates to the package-level cmdExec provider.
// Check functions call runCmd; tests swap cmdExec via SetCmdRunner.
func runCmd(name string, args ...string) ([]byte, error) {
return cmdExec.Run(name, args...)
}
func runCmdAllowNonZero(name string, args ...string) ([]byte, error) {
return cmdExec.RunAllowNonZero(name, args...)
}
func runCmdCombinedContext(parent context.Context, name string, args ...string) ([]byte, error) {
return cmdExec.RunContext(parent, name, args...)
}
func lookupSystemCommand(name string) (string, error) {
if strings.ContainsRune(name, os.PathSeparator) {
return exec.LookPath(name)
}
path, err := exec.LookPath(name)
if err == nil {
return path, nil
}
for _, dir := range systemCommandSearchDirs {
candidate := filepath.Join(dir, name)
info, statErr := os.Stat(candidate)
if statErr == nil && !info.IsDir() && info.Mode()&0111 != 0 {
return candidate, nil
}
}
return "", err
}
func resolveSystemCommand(name string) string {
path, err := lookupSystemCommand(name)
if err != nil {
return name
}
return path
}
// ---------------------------------------------------------------------------
// Real implementations — used by realCmd in provider.go
//
// Every caller of these helpers passes a constant system command name
// (nft, rpm, wp-cli, doveadm, systemctl, etc.) with arguments built from
// either config, filesystem state, or CSM-generated data. Nothing here
// reaches out to HTTP request bodies or webui form inputs. The gosec
// G204 suppressions below are in that trust model.
// ---------------------------------------------------------------------------
func runCmdReal(name string, args ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), cmdTimeout)
defer cancel()
// #nosec G204 -- see package-level trust note above.
out, err := exec.CommandContext(ctx, resolveSystemCommand(name), args...).Output()
if ctx.Err() == context.DeadlineExceeded {
fmt.Fprintf(os.Stderr, "Command timed out: %s %v\n", name, args)
return nil, nil
}
return out, err
}
func runCmdAllowNonZeroReal(name string, args ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), cmdTimeout)
defer cancel()
// #nosec G204 -- see package-level trust note above.
out, err := exec.CommandContext(ctx, resolveSystemCommand(name), args...).Output()
if ctx.Err() == context.DeadlineExceeded {
fmt.Fprintf(os.Stderr, "Command timed out: %s %v\n", name, args)
return nil, nil
}
var exitErr *exec.ExitError
if err != nil && errors.As(err, &exitErr) {
return out, nil
}
return out, err
}
// commandRefused reports whether a command ran to completion and answered with
// a failure, such as wp-cli on a tree that is not a WordPress installation or
// one whose wp-config.php fatals. The check has its answer, so the queue lost
// no work. A command that never started, was killed by a signal or ran out of
// time is still lost work.
func commandRefused(err error) bool {
var exit *exec.ExitError
return errors.As(err, &exit) && exit.ExitCode() > 0
}
func runCmdCombinedContextReal(parent context.Context, name string, args ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(parent, cmdTimeout)
defer cancel()
// #nosec G204 -- see package-level trust note above.
out, err := exec.CommandContext(ctx, resolveSystemCommand(name), args...).CombinedOutput()
if ctx.Err() == context.DeadlineExceeded {
fmt.Fprintf(os.Stderr, "Command timed out: %s %v\n", name, args)
return nil, context.DeadlineExceeded
}
if parent.Err() != nil {
return nil, parent.Err()
}
return out, err
}
// runCmdStdoutContextReal runs a command with a per-call timeout and returns
// stdout only. Stderr is discarded so chatter from the child process (PHP
// warnings, MySQL deprecation notices, wp-cli plugin backtraces, ...) cannot
// poison parsers that expect JSON/URL bytes on stdout. On timeout the caller
// receives context.DeadlineExceeded rather than a silent (nil, nil), so it
// can distinguish a hung command from a legitimately empty result.
func runCmdStdoutContextReal(parent context.Context, name string, args ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(parent, cmdTimeout)
defer cancel()
// #nosec G204 -- see package-level trust note above.
out, err := exec.CommandContext(ctx, resolveSystemCommand(name), args...).Output()
if ctx.Err() == context.DeadlineExceeded {
return nil, context.DeadlineExceeded
}
if parent.Err() != nil {
return nil, parent.Err()
}
return out, err
}
func runCmdWithEnvReal(name string, args []string, extraEnv ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), cmdTimeout)
defer cancel()
// #nosec G204 -- see package-level trust note above.
cmd := exec.CommandContext(ctx, resolveSystemCommand(name), args...)
cmd.Env = append(os.Environ(), extraEnv...)
out, err := cmd.Output()
if ctx.Err() == context.DeadlineExceeded {
fmt.Fprintf(os.Stderr, "Command timed out: %s\n", name)
return nil, nil
}
return out, err
}
package checks
import (
"fmt"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/platform"
)
// HostingSnapshot contains all required inventory sources read without error.
// It does not establish an atomic read.
type HostingSnapshot struct {
// Accounts is sorted and contains no duplicates.
Accounts []string
// Domains maps canonical domains to listed accounts. Ambiguous and
// malformed rows are excluded.
Domains map[string]string
// Unresolved holds every domain /etc/userdomains lists but Domains leaves
// out: its rows conflict, or its owner is not an account. Such a domain is
// hosted here even though no account can be named for it.
Unresolved map[string]bool
// AmbiguousDomains counts domains listed for more than one owner. They
// are left out of Domains, so evidence about them stays host-scoped.
AmbiguousDomains int
// Incarnations maps every account to a server-owned lifetime token.
// cPanel's zero-dated system entries keep it across recreation.
Incarnations map[string]string
}
// HostingInventory reads the hosting accounts and the domains they own. On
// cPanel the account registry is the account list; homes may sit on any
// home partition, so they are not consulted. Elsewhere every account root
// must be readable and its directories are the accounts. A missing or
// unreadable required source fails the whole read: a partial snapshot would
// retire accounts and hand them new generations when they reappear. Off
// cPanel, mount state is not checked: a readable empty root contributes no
// accounts even if its expected filesystem is unmounted. Other readable
// roots still contribute their accounts.
func HostingInventory() (HostingSnapshot, error) {
if !platform.Detect().IsCPanel() {
return homeRootInventory()
}
registry, err := osFS.ReadDir("/var/cpanel/users")
if err != nil {
return HostingSnapshot{}, err
}
snap := HostingSnapshot{Accounts: []string{}, Domains: map[string]string{}, Unresolved: map[string]bool{}, Incarnations: map[string]string{}}
known := map[string]bool{}
for _, e := range registry {
name := e.Name()
if e.IsDir() || !admission.ValidAccountName(name) || known[name] {
continue
}
known[name] = true
snap.Accounts = append(snap.Accounts, name)
if snap.Incarnations[name], err = cpanelIncarnation(name); err != nil {
return HostingSnapshot{}, err
}
}
sort.Strings(snap.Accounts)
data, err := osFS.ReadFile("/etc/userdomains")
if err != nil {
return HostingSnapshot{}, err
}
// Check every row before collapsing duplicates or filtering owners; the
// legacy lookup keeps the last row and would hide a conflict.
owners := map[string]string{}
ambiguous := map[string]bool{}
for _, line := range strings.Split(string(data), "\n") {
domain, owner := parseUserDomain(line)
domain = strings.TrimSuffix(domain, ".")
// More than one trailing dot cannot produce a stable inventory key.
if domain == "" || strings.HasSuffix(domain, ".") {
continue
}
if old, exists := owners[domain]; exists && old != owner {
ambiguous[domain] = true
}
owners[domain] = owner
}
for domain, owner := range owners {
if !ambiguous[domain] && known[owner] {
snap.Domains[domain] = owner
} else {
snap.Unresolved[domain] = true
}
}
snap.AmbiguousDomains = len(ambiguous)
return snap, nil
}
// homeRootInventory lists the account directories under every account root.
// An account listed under several roots is the one in the first.
func homeRootInventory() (HostingSnapshot, error) {
snap := HostingSnapshot{Accounts: []string{}, Domains: map[string]string{}, Unresolved: map[string]bool{}, Incarnations: map[string]string{}}
for _, root := range accountHomeRoots() {
entries, err := osFS.ReadDir(root)
if err != nil {
return HostingSnapshot{}, err
}
for _, e := range entries {
name := e.Name()
if _, seen := snap.Incarnations[name]; !admission.ValidAccountName(name) || seen {
continue
}
if e.Type()&os.ModeSymlink != 0 {
return HostingSnapshot{}, fmt.Errorf("account home for %s is a symlink", name)
}
if !e.IsDir() {
continue
}
snap.Accounts = append(snap.Accounts, name)
if snap.Incarnations[name], err = accountIncarnation(filepath.Join(root, name)); err != nil {
return HostingSnapshot{}, err
}
}
}
sort.Strings(snap.Accounts)
return snap, nil
}
// cpanelIncarnation is the creation time cPanel records in an account's
// user file. Edits keep it. cPanel's own system entries record zero, which
// stays valid even when recreation leaves it unchanged.
func cpanelIncarnation(name string) (string, error) {
data, err := osFS.ReadFile(filepath.Join("/var/cpanel/users", name))
if err != nil {
return "", err
}
var token string
for _, line := range strings.Split(string(data), "\n") {
v, ok := strings.CutPrefix(strings.TrimSuffix(line, "\r"), "STARTDATE=")
if !ok {
continue
}
_, parseErr := strconv.ParseInt(v, 10, 64)
if token != "" || len(v) > 19 || parseErr != nil || strings.Trim(v, "0123456789") != "" {
return "", fmt.Errorf("cPanel user file for %s has an invalid creation date", name)
}
token = "startdate:" + v
}
if token != "" {
return token, nil
}
return "", fmt.Errorf("cPanel user file for %s names no creation date", name)
}
// accountIncarnation names one incarnation of the account home at path.
var accountIncarnation = accountDirIncarnation
//go:build linux
package checks
import (
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"golang.org/x/sys/unix"
)
// accountDirIncarnation names the home directory at path by its device,
// inode and birth time. Its owner can change neither, while a directory
// created in its place has another inode or birth time. The mount ID stays
// out: it names the mount instance, which the service's own mount namespace
// renews at each start.
func accountDirIncarnation(path string) (string, error) {
var st unix.Statx_t
if err := unix.Statx(unix.AT_FDCWD, path, unix.AT_SYMLINK_NOFOLLOW, unix.STATX_TYPE|unix.STATX_INO|unix.STATX_BTIME, &st); err != nil {
return "", err
}
return accountStatxIncarnation(st)
}
func accountStatxIncarnation(st unix.Statx_t) (string, error) {
if st.Mask&unix.STATX_INO == 0 {
return "", errors.New("account home has no inode number")
}
if st.Mask&unix.STATX_TYPE == 0 || st.Mode&unix.S_IFMT != unix.S_IFDIR {
return "", errors.New("account home is not a directory")
}
if st.Mask&unix.STATX_BTIME == 0 {
return "", errors.New("account home has no birth time")
}
return statxIncarnation(st), nil
}
// statxIncarnation digests the identity fields of st, so the token stays
// short.
func statxIncarnation(st unix.Statx_t) string {
b := binary.BigEndian.AppendUint32(nil, st.Dev_major)
b = binary.BigEndian.AppendUint32(b, st.Dev_minor)
b = binary.BigEndian.AppendUint64(b, st.Ino)
if st.Mask&unix.STATX_BTIME != 0 {
b = binary.BigEndian.AppendUint64(b, uint64(st.Btime.Sec)) // #nosec G115 -- bit pattern only.
b = binary.BigEndian.AppendUint32(b, st.Btime.Nsec)
}
sum := sha256.Sum256(b)
return "fs:" + hex.EncodeToString(sum[:16])
}
package checks
import (
"io"
"os"
"path/filepath"
"strings"
"golang.org/x/sys/unix"
)
// htaccessMaxFileBytes bounds every scheduled .htaccess read. A real
// .htaccess is a few kilobytes; the ceiling matches the per-line limit the
// directive scanner already enforces. Reading the whole file with no bound
// let a tenant park a multi-gigabyte .htaccess and turn the deep scan into
// an out-of-memory crash loop.
const htaccessMaxFileBytes = htaccessMaxLineBytes
// readHtaccessBounded returns the file's content when it is at most
// htaccessMaxFileBytes. An oversized file yields ok=false with no error: the
// caller knows the file exists but cannot be judged, which differs from a
// missing file. Any other failure is returned as err.
func readHtaccessBounded(path string) (data []byte, ok bool, err error) {
f, err := osFS.Open(path)
if err != nil {
return nil, false, err
}
defer func() { _ = f.Close() }()
return readHtaccessFileBounded(f)
}
// readTenantHtaccessBounded is readHtaccessBounded for a caller acting on the
// tenant's behalf as root. Every tenant-controlled directory is opened without
// following symlinks, so a path replacement cannot redirect a privileged read
// outside the passwd home. Special files cannot block the read.
func readTenantHtaccessBounded(home, dir string) (data []byte, ok bool, err error) {
var f *os.File
if _, production := osFS.(realOS); production {
f, err = openTenantHtaccess(home, dir)
} else {
f, err = openTenantRegularFile(filepath.Join(dir, ".htaccess"))
}
if err != nil {
return nil, false, err
}
defer func() { _ = f.Close() }()
return readHtaccessFileBounded(f)
}
func openTenantHtaccess(home, dir string) (*os.File, error) {
rel, err := filepath.Rel(home, dir)
if err != nil || !filepath.IsAbs(home) || !filepath.IsLocal(rel) {
return nil, os.ErrPermission
}
const dirFlags = unix.O_RDONLY | unix.O_DIRECTORY | unix.O_NOFOLLOW | unix.O_CLOEXEC | unix.O_NONBLOCK
fd, err := unix.Open(home, dirFlags, 0)
if err != nil {
// A missing directory is not an absent .htaccess. The caller must
// stop instead of falling back to a handler in a surviving ancestor.
return nil, errNonRegularFile
}
defer func() { _ = unix.Close(fd) }()
if rel != "." {
for _, part := range strings.Split(rel, string(filepath.Separator)) {
next, openErr := unix.Openat(fd, part, dirFlags, 0)
if openErr != nil {
return nil, errNonRegularFile
}
_ = unix.Close(fd)
fd = next
}
}
fileFD, err := unix.Openat(fd, ".htaccess", unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_CLOEXEC|unix.O_NONBLOCK, 0)
if err != nil {
return nil, err
}
// #nosec G115 -- unix.Openat returned a non-negative descriptor because err is nil.
return os.NewFile(uintptr(fileFD), filepath.Join(dir, ".htaccess")), nil
}
func readHtaccessFileBounded(f *os.File) (data []byte, ok bool, err error) {
info, err := f.Stat()
if err != nil {
return nil, false, err
}
if !info.Mode().IsRegular() || info.Size() > htaccessMaxFileBytes {
return nil, false, nil
}
// Read one byte past the limit: a file that grows between Stat and
// Read must not slip through as complete.
data, err = io.ReadAll(io.LimitReader(f, htaccessMaxFileBytes+1))
if err != nil {
return nil, false, err
}
if int64(len(data)) > htaccessMaxFileBytes {
return nil, false, nil
}
return data, true, nil
}
// htaccessOversized reports whether err/ok from readHtaccessBounded mean
// "present but too large".
func htaccessOversized(ok bool, err error) bool {
return !ok && err == nil
}
package checks
import (
"context"
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/alert"
)
// .htaccess hardened detection / cleaning.
//
// Detection emits a specific finding name for each pattern so operators
// can suppress, route, or auto-respond per attack pattern instead of
// relying on the generic htaccess_injection / htaccess_handler_abuse
// categories. Cleaning is gated by AutoResponse.CleanHtaccess and is
// always backed up under /opt/csm/quarantine/pre_clean/.
//
// Each detector returns matches as byte ranges into the file
// content; the cleaner merges all ranges (deduplicating overlaps),
// removes them, and writes the result atomically. If post-clean
// content is identical to pre-clean (no detector matched anything
// new), no write happens and no backup is created.
// htaccessBackupDirRoot is the parent directory under which
// CleanHtaccessFile writes unique recovery backups. Exposed
// as a package var so tests can redirect it to a t.TempDir().
var htaccessBackupDirRoot = "/opt/csm/quarantine/pre_clean"
// htaccessByteRange is a half-open [start, end) byte slice into the
// file content. Cleaning removes the bytes; the line ending after
// `end` is included when `end` falls just before a `\n` so we do
// not leave a blank line behind.
type htaccessByteRange struct {
Start int
End int
}
// htaccessMatch is one finding-worthy hit returned by a detector.
type htaccessMatch struct {
Range htaccessByteRange
Excerpt string // the offending line(s), trimmed for the finding details
// Retain keeps the match out of the cleaner's removal set: the line is
// reported for review, but removing it automatically could break the site.
Retain bool
}
// htaccessDetector pairs a finding category with a function that
// scans the content and reports every range the detector wants
// removed. Adding a pattern is one entry in this slice.
type htaccessDetector struct {
Name string
Severity alert.Severity
Detect func(content []byte, path string) []htaccessMatch
DetectContext func(context.Context, []byte, string) []htaccessMatch
}
// htaccessSpamTLDs lists TLDs commonly abused for spam-redirect
// .htaccess injections. Operators on legitimate hosts in these TLDs
// can suppress the finding by file path.
var htaccessSpamTLDs = []string{
".xyz", ".tk", ".ml", ".ga", ".cf", ".gq", ".click",
".country", ".loan", ".work", ".top",
}
// htaccessNonScriptDirHints names directory components where PHP
// execution is rarely legitimate. .htaccess files inside one of
// these get the htaccess_php_in_uploads finding when they map
// non-PHP extensions to a PHP handler.
//
// /tmp/ is intentionally NOT in this list even though attackers
// drop payloads there: a real-world .htaccess inside /tmp/ would
// only be reached if the webserver served /tmp/, which is rare
// outside misconfigurations -- and including it caused
// false-positive matches against Linux t.TempDir() paths under
// /tmp/ in the test suite. The auto_prepend detector covers the
// /tmp/ payload-target angle separately.
var htaccessNonScriptDirHints = []string{
"/uploads/", "/images/", "/cache/",
"/wp-content/uploads/", "/wp-content/cache/",
"/files/", "/media/",
}
// htaccessSuspiciousAutoPrependPaths are scratch locations that a prelude
// script is never legitimately served from.
var htaccessSuspiciousAutoPrependPaths = []string{
"/tmp/", "/dev/shm/", "/var/tmp/",
}
// Apache accepts quoted directive arguments, including paths with spaces.
// Keep the quotes in the capture; autoPrependTargetSuspicious removes them
// after the parser has found the complete target.
const htaccessPreludeTargetPattern = `("[^"\r\n]*"|'[^'\r\n]*'|\S+)`
// A raw newline ends the directive. Apache continuation is explicit, so only
// horizontal whitespace or a backslash-newline may separate its arguments.
const htaccessPreludeSeparatorPattern = `(?:[\t ]|\\\r?\n)+`
// reAutoPrependTarget captures the argument of either prelude directive in
// any of the forms .htaccess and php.ini fragments use.
var reAutoPrependTarget = regexp.MustCompile(`(?i)auto_(?:prepend|append)_file(?:[\t ]*=[\t ]*|[\t ]+)` + htaccessPreludeTargetPattern)
// autoPrependTargetIsKnownPrelude reports whether target names a prelude
// script shipped by a security plugin. Only the basename is consulted: every
// other byte of the directive is text the account owner types, so a
// substring test anywhere else on the line is an exemption the attacker
// controls.
func autoPrependTargetIsKnownPrelude(target string) bool {
base := preludeBase(target)
switch base {
case "wordfence-waf.php", "malcare-waf.php":
return true
}
return strings.HasPrefix(base, "sucuri") && strings.HasSuffix(base, ".php")
}
func preludeBase(target string) string {
return strings.ToLower(filepath.Base(strings.ReplaceAll(target, `\`, "/")))
}
// rssslPreludeName is the prelude Really Simple Security installs.
const rssslPreludeName = "advanced-headers.php"
// rssslPreludeBound reports whether an unquoted target is the Really Simple
// Security prelude of the site whose .htaccess carries the directive. The
// plugin writes WP_CONTENT_DIR/advanced-headers.php, absolute and verbatim,
// into the .htaccess of the site root, so its target is the wp-content
// directory beside that file. The file name alone is no evidence: the
// plugin's marker block is text anyone who can write the .htaccess can copy
// and point anywhere.
func rssslPreludeBound(target, htaccessPath string) bool {
if !filepath.IsAbs(target) || target != filepath.Clean(target) || !strings.HasSuffix(target, "/wp-content/"+rssslPreludeName) {
return false
}
site := filepath.Dir(htaccessPath)
if target == filepath.Join(site, "wp-content", rssslPreludeName) {
return true
}
// Fanotify resolves a symlinked docroot, while the plugin or the
// scheduled scan can name its alias. Resolve only the site directories;
// the raw target must still have the exact suffix and no dot segments.
resolvedSite, err := filepath.EvalSymlinks(site)
if err != nil {
return false
}
targetSite, err := filepath.EvalSymlinks(filepath.Dir(filepath.Dir(target)))
return err == nil && targetSite == resolvedSite
}
// autoPrependTargetSuspicious reports whether an auto_prepend_file or
// auto_append_file target can point at code the account owner controls.
// htaccessPath is the file carrying the directive. Unless the target is a
// known plugin prelude, it is suspicious when it sits in a scratch location,
// is not a PHP file at all, is relative (it resolves inside the docroot),
// lives under any home directory, or shares the .htaccess file's own account
// tree. An unbound Really Simple Security target is always reported. Other
// root-owned paths elsewhere (/etc, /opt, /usr) need root to write and are
// left alone; "none" merely disables an inherited prelude.
func autoPrependTargetSuspicious(target, htaccessPath string) bool {
target = strings.Trim(strings.TrimSpace(target), `"'`)
lower := strings.ToLower(target)
if lower == "" || lower == "none" || autoPrependTargetIsKnownPrelude(lower) {
return false
}
if preludeBase(target) == rssslPreludeName {
return !rssslPreludeBound(target, htaccessPath)
}
// PHP resolves lexical dot segments before opening the file. Classify the
// same normalized path so an account-controlled target cannot hide behind
// an apparently root-owned prefix such as /etc/../home/user/prelude.php.
lower = strings.ToLower(filepath.Clean(target))
for _, p := range htaccessSuspiciousAutoPrependPaths {
if strings.HasPrefix(lower, p) {
return true
}
}
if !strings.HasSuffix(lower, ".php") || !strings.HasPrefix(lower, "/") || underAccountRoot(lower) {
return true
}
if tree := htaccessAccountTree(htaccessPath); tree != "" && strings.HasPrefix(lower, strings.ToLower(tree)+"/") {
return true
}
return false
}
// htaccessAccountTree returns the first two path components of the .htaccess
// location (/home/<user>, /var/www/<vhosts>), which is the tree the account
// can write to under every supported panel layout.
func htaccessAccountTree(htaccessPath string) string {
parts := strings.Split(filepath.Clean(htaccessPath), "/")
if len(parts) < 4 || parts[0] != "" {
return ""
}
return "/" + parts[1] + "/" + parts[2]
}
// htaccessTrackingHeaders is a small allowlist of header *names*
// known to be used in injection campaigns. Scoped intentionally;
// false positives on legitimate analytics/CDN headers are worse
// than missed detections here.
var htaccessTrackingHeaders = []string{
"X-Track-", "X-Affiliate-", "X-Promo-", "X-Click-ID",
}
var (
// SetHandler / ForceType take effect with a single argument: they map
// EVERY file in the directory to the PHP interpreter, so an uploaded
// image runs as PHP. That directory-wide form is the worst case and has
// no extension list, so the second argument is optional for them.
// AddHandler is inert without an extension list, so it still requires the
// trailing token to be a genuine remap.
rePHPHandlerMap = regexp.MustCompile(`(?im)^\s*(?:(?:SetHandler|ForceType)\s+\S*php\S*(?:\s+\S[^\n]*)?|AddHandler\s+\S*php\S*\s+\S[^\n]*)\s*$`)
// Match both forms because mod_php and some LSAPI builds honor either
// directive in .htaccess.
reAutoPrepend = regexp.MustCompile(`(?im)^[\t ]*php(?:_admin)?_value[\t ]+(auto_(?:prepend|append)_file)` + htaccessPreludeSeparatorPattern + htaccessPreludeTargetPattern)
reUACloakCond = regexp.MustCompile(`(?im)^\s*RewriteCond\s+%\{HTTP_USER_AGENT\}\s+([^\n]+)`)
reSpamRedirect = regexp.MustCompile(`(?im)^\s*RewriteRule\s+\S+\s+(https?://[^\s\[]+)`)
reFilesMatchOpen = regexp.MustCompile(`(?im)^\s*<FilesMatch\s+["']?[^"'>]*\\\.(php|phtml|ph[2-7])[^"'>]*["']?\s*>`)
reFilesMatchClose = regexp.MustCompile(`(?im)^\s*</FilesMatch>`)
reHeaderSetAdd = regexp.MustCompile(`(?im)^\s*Header\s+(set|add)\s+([A-Za-z0-9_-]+)`)
reErrorDocument = regexp.MustCompile(`(?im)^\s*ErrorDocument\s+\d+\s+(https?://[^\s]+)`)
// crawlerUARegex matches the UA strings frequently used in cloak
// conditions: search-engine bots and social-share scrapers. Used
// as a positive filter on the htaccess_user_agent_cloak finding;
// matching one of these names is what makes a UA-keyed redirect
// suspicious.
crawlerUARegex = regexp.MustCompile(`(?i)(googlebot|bingbot|baiduspider|yandex|facebookexternalhit|slurp|duckduckbot)`)
// searchCrawlerUARegex names the indexers a cloak targets. Operator
// blocklists are made of scrapers and SEO tools; a list whose members
// are mostly search engines is a cloak whatever its length, because no
// site blocks Googlebot, Bingbot, Yandex and Baidu together.
searchCrawlerUARegex = regexp.MustCompile(`(?i)googlebot|bingbot|baiduspider|yandex|slurp|duckduckbot|applebot|sogou|seznambot|petalbot`)
)
// htaccessDetectors is the registry. Detectors run in slice order for
// deterministic finding emission. Overlapping removal ranges are merged later.
var htaccessDetectors = []htaccessDetector{
{
Name: "htaccess_php_in_uploads",
Severity: alert.Critical,
Detect: detectPHPInUploads,
},
{
Name: "htaccess_auto_prepend",
Severity: alert.Critical,
Detect: detectAutoPrepend,
},
{
Name: "htaccess_user_agent_cloak",
Severity: alert.High,
Detect: detectUserAgentCloak,
},
{
Name: "htaccess_spam_redirect",
Severity: alert.High,
Detect: detectSpamRedirect,
},
{
Name: "htaccess_filesmatch_shield",
Severity: alert.Critical,
Detect: detectFilesMatchShield,
DetectContext: detectFilesMatchShieldContext,
},
{
Name: "htaccess_rewrite_off_shield",
Severity: alert.High,
Detect: detectRewriteOffShield,
},
{
Name: "htaccess_header_injection",
Severity: alert.High,
Detect: detectHeaderInjection,
},
{
Name: "htaccess_errordocument_hijack",
Severity: alert.High,
Detect: detectErrorDocumentHijack,
},
{
Name: "htaccess_cgi_handler_abuse",
Severity: alert.Critical,
Detect: detectCGIHandlerAbuse,
},
{
Name: "htaccess_security_disabled",
Severity: alert.High,
Detect: detectSecurityDisabled,
},
}
func htaccessDetectorNames() []string {
names := make([]string, 0, len(htaccessDetectors))
for _, detector := range htaccessDetectors {
names = append(names, detector.Name)
}
return names
}
// handlerIsCGI reports whether an Apache handler/MIME token routes matching
// files to a CGI interpreter (as opposed to serving them statically). This is
// the CGI counterpart of handlerIsPHP.
func handlerIsCGI(token string) bool {
token = strings.ToLower(strings.Trim(strings.TrimSpace(token), `"'`))
switch token {
case "cgi-script",
"fcgid-script",
"fastcgi-script",
"application/x-httpd-cgi",
"application/x-httpd-fcgi":
return true
}
return false
}
// conventionalCGIExt reports whether ext (leading dot, lowercase) is one a
// legitimate cgi-bin routes to a CGI handler. A CGI handler mapped onto any
// other extension is the webshell-arming remap.
func conventionalCGIExt(ext string) bool {
switch ext {
case ".cgi", ".fcg", ".fcgi", ".pl", ".py", ".rb":
return true
}
return false
}
type cgiHandlerContext struct {
onlyConventional bool
}
func openCGIHandlerContext(line string) (cgiHandlerContext, bool) {
lower := strings.ToLower(strings.TrimSpace(line))
switch {
case strings.HasPrefix(lower, "<filesmatch"):
return cgiHandlerContext{
onlyConventional: filesMatchTargetsOnlyConventionalCGI(apacheContainerArgument(line)),
}, true
case strings.HasPrefix(lower, "<files"):
name := strings.ToLower(strings.TrimSpace(apacheContainerArgument(line)))
if name == "" || strings.Contains(name, "/") {
return cgiHandlerContext{}, true
}
if strings.HasPrefix(name, "~") {
return cgiHandlerContext{
onlyConventional: filesMatchTargetsOnlyConventionalCGI(name),
}, true
}
ext := normalizeExt(filepath.Ext(name))
if ext == "" {
return cgiHandlerContext{}, true
}
return cgiHandlerContext{
onlyConventional: conventionalCGIExt(ext),
}, true
default:
return cgiHandlerContext{}, false
}
}
func filesMatchTargetsOnlyConventionalCGI(pattern string) bool {
pattern = strings.TrimSpace(pattern)
if strings.HasPrefix(pattern, "~") {
pattern = strings.TrimSpace(strings.TrimPrefix(pattern, "~"))
}
if len(pattern) >= 2 {
quote := pattern[0]
if (quote == '"' || quote == '\'') && pattern[len(pattern)-1] == quote {
pattern = pattern[1 : len(pattern)-1]
}
}
pattern = strings.ToLower(strings.TrimSpace(pattern))
switch {
case strings.HasSuffix(pattern, `\z`):
pattern = pattern[:len(pattern)-2]
case strings.HasSuffix(pattern, "$"):
pattern = pattern[:len(pattern)-1]
default:
// An unanchored "\.cgi" also matches names such as shell.cgi.jpg.
return false
}
dot := lastRegexLiteralDot(pattern)
if dot < 0 || strings.Contains(pattern[:dot], "|") {
return false
}
extPattern := pattern[dot+2:]
switch {
case strings.HasPrefix(extPattern, "(?:") && strings.HasSuffix(extPattern, ")"):
extPattern = extPattern[3 : len(extPattern)-1]
case strings.HasPrefix(extPattern, "(") && strings.HasSuffix(extPattern, ")"):
extPattern = extPattern[1 : len(extPattern)-1]
}
parts := strings.Split(extPattern, "|")
exts := make([]string, 0, len(parts))
for _, part := range parts {
ext := normalizeExt(part)
if ext == "" {
return false
}
exts = append(exts, ext)
}
return allConventionalCGIExts(exts)
}
// lastRegexLiteralDot returns the escape backslash before the final literal
// dot. An odd backslash run escapes the dot; an even run leaves it as a regex
// wildcard and cannot prove an extension-only match.
func lastRegexLiteralDot(pattern string) int {
for dot := len(pattern) - 1; dot >= 0; dot-- {
if pattern[dot] != '.' {
continue
}
backslashes := 0
for i := dot - 1; i >= 0 && pattern[i] == '\\'; i-- {
backslashes++
}
if backslashes%2 == 1 {
return dot - 1
}
}
return -1
}
func allConventionalCGIExts(exts []string) bool {
if len(exts) == 0 {
return false
}
for _, ext := range exts {
if !conventionalCGIExt(ext) {
return false
}
}
return true
}
func cgiHandlerTargetsOnlyConventional(contexts []cgiHandlerContext) bool {
for _, context := range contexts {
// Nested file containers intersect. Any conventional-only container
// therefore prevents the handler from reaching a custom extension even
// when another container is broader.
if context.onlyConventional {
return true
}
}
return false
}
func cgiExtensionListIsSuspicious(tokens []string) bool {
for _, token := range tokens {
ext := normalizeExt(token)
// AddHandler and AddType treat every remaining token as an
// extension. If the shared plain-extension parser cannot normalize
// one, it is still a custom target and must fail closed.
if ext == "" || !conventionalCGIExt(ext) {
return true
}
}
return false
}
type cgiOptionsScope struct {
name string
neutralized bool
childMayEnable bool
}
func apacheContainerTag(line string) (name string, closing bool, ok bool) {
line = strings.TrimSpace(line)
if len(line) < 3 || line[0] != '<' {
return "", false, false
}
end := strings.IndexByte(line, '>')
if end < 0 {
return "", false, false
}
tag := strings.TrimSpace(line[1:end])
if tag == "" || strings.HasPrefix(tag, "!") {
return "", false, false
}
if strings.HasPrefix(tag, "/") {
closing = true
tag = strings.TrimSpace(tag[1:])
}
if fields := strings.Fields(tag); len(fields) > 0 {
return strings.ToLower(fields[0]), closing, true
}
return "", false, false
}
func unquoteApacheOption(token string) (string, bool) {
token = strings.TrimSpace(token)
if token == "" {
return "", false
}
if token[0] == '"' || token[0] == '\'' {
if len(token) < 2 || token[len(token)-1] != token[0] {
return "", false
}
token = token[1 : len(token)-1]
}
if token == "" || strings.ContainsAny(token, `"'`) {
return "", false
}
return strings.ToLower(token), true
}
// applyCGIOptions follows Apache's Options parser for the ExecCGI bit. Bare
// option lists replace the inherited set, while lists made only of +/- tokens
// merge with it. All and None may start a list before relative adjustments.
func applyCGIOptions(fields []string, neutralized bool) (bool, bool) {
if len(fields) < 2 {
return false, false
}
first := true
merge := false
allOrNone := false
for _, field := range fields[1:] {
token, ok := unquoteApacheOption(field)
if !ok {
return false, false
}
var action byte
switch {
case token[0] == '+' || token[0] == '-':
action = token[0]
token = token[1:]
if token == "" || (!merge && !first && !allOrNone) {
return false, false
}
merge = true
case first:
neutralized = true
case merge:
return false, false
}
switch token {
case "none":
if !first || merge {
return false, false
}
neutralized = true
allOrNone = true
case "all":
if !first || merge {
return false, false
}
neutralized = false
allOrNone = true
case "execcgi", "runscripts":
neutralized = action == '-'
case "indexes", "includes", "includesnoexec", "followsymlinks",
"symlinksifownermatch", "multiviews":
default:
return false, false
}
first = false
}
return neutralized, true
}
// cgiExecutionNeutralized reports whether CGI execution is disabled for every
// request in this directory. Conditional or scoped Options blocks cannot prove
// a global disable and may re-enable ExecCGI after directory options merge.
func cgiExecutionNeutralized(content []byte) bool {
neutralized := false
nestedMayEnable := false
var scopes []cgiOptionsScope
for _, logical := range htaccessLogicalByteLines(content) {
line := strings.TrimSpace(logical.text)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if name, closing, ok := apacheContainerTag(line); ok {
if !closing {
scopes = append(scopes, cgiOptionsScope{
name: name,
neutralized: true,
})
continue
}
if len(scopes) == 0 || scopes[len(scopes)-1].name != name {
return false
}
scope := scopes[len(scopes)-1]
scopes = scopes[:len(scopes)-1]
mayEnable := !scope.neutralized || scope.childMayEnable
if len(scopes) == 0 {
nestedMayEnable = nestedMayEnable || mayEnable
} else {
scopes[len(scopes)-1].childMayEnable =
scopes[len(scopes)-1].childMayEnable || mayEnable
}
continue
}
fields := apacheDirectiveFields(line)
if len(fields) == 0 || !strings.EqualFold(fields[0], "Options") {
continue
}
var valid bool
if len(scopes) == 0 {
neutralized, valid = applyCGIOptions(fields, neutralized)
} else {
scope := &scopes[len(scopes)-1]
scope.neutralized, valid = applyCGIOptions(fields, scope.neutralized)
}
if !valid {
return false
}
}
return len(scopes) == 0 && neutralized && !nestedMayEnable
}
// detectCGIHandlerAbuse flags an .htaccess that maps a non-conventional
// extension (or the whole directory) to a CGI interpreter. An attacker uses
// this to make an uploaded Perl/binary file execute: e.g.
// `AddHandler cgi-script .alfa` / `AddType application/x-httpd-cgi .alfa`.
func detectCGIHandlerAbuse(content []byte, _ string) []htaccessMatch {
// A cgi-script handler mapping is inert when the directory disables CGI
// execution, so a hardening block that maps .php to cgi-script alongside
// Options -ExecCGI is not webshell arming and must not be flagged or
// auto-cleaned out of a legitimate security plugin's .htaccess.
if cgiExecutionNeutralized(content) {
return nil
}
var out []htaccessMatch
var contexts []cgiHandlerContext
for _, logical := range htaccessLogicalByteLines(content) {
line := strings.TrimSpace(logical.text)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if context, ok := openCGIHandlerContext(line); ok {
contexts = append(contexts, context)
continue
}
if closesPHPHandlerContext(line) {
if len(contexts) > 0 {
contexts = contexts[:len(contexts)-1]
}
continue
}
fields := apacheDirectiveFields(line)
if len(fields) < 2 || !handlerIsCGI(fields[1]) {
continue
}
var flagged bool
switch strings.ToLower(fields[0]) {
case "addhandler", "addtype":
flagged = cgiExtensionListIsSuspicious(fields[2:])
case "sethandler", "forcetype":
flagged = !cgiHandlerTargetsOnlyConventional(contexts)
}
if flagged {
out = append(out, htaccessMatch{
Range: logical.span,
Excerpt: trimExcerpt(content, logical.span.Start, logical.span.End),
})
}
}
return out
}
func normalizeModSecurityDirective(name string) string {
return strings.Map(func(r rune) rune {
switch {
case r >= 'A' && r <= 'Z':
return r + ('a' - 'A')
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
return r
default:
return -1
}
}, name)
}
func modSecurityDirectiveDisablesWAF(name, value string) bool {
name = normalizeModSecurityDirective(name)
value = strings.ToLower(strings.Trim(strings.TrimSpace(value), `"'`))
switch value {
case "off":
switch name {
case "secruleengine",
"secfilterengine",
"secfilterscanpost",
"secruleinheritance",
"secrequestbodyaccess",
"secengine",
"secscanpost":
return true
}
case "detectiononly":
switch name {
case "secruleengine", "secfilterengine", "secengine":
return true
}
}
return false
}
// detectSecurityDisabled flags a per-account .htaccess that disables
// ModSecurity enforcement, request inspection, or inherited rules. A customer
// .htaccess never legitimately weakens those controls; attackers do it to mask
// an intrusion. Punctuation is removed from directive names so known
// obfuscated spellings are classified without matching unrelated settings such
// as SecAuditEngine or SecStatusEngine. The mod_security 1.x spellings
// (SecFilterEngine, SecFilterScanPOST) are live too: LiteSpeed accepts them
// inside <IfModule mod_security.c> and by default lets .htaccess switch its WAF
// off with them, which is how Magento's stock block disables it.
func detectSecurityDisabled(content []byte, _ string) []htaccessMatch {
var out []htaccessMatch
for _, logical := range htaccessLogicalByteLines(content) {
fields := apacheDirectiveFields(strings.TrimSpace(logical.text))
if len(fields) < 2 || !modSecurityDirectiveDisablesWAF(fields[0], fields[1]) {
continue
}
out = append(out, htaccessMatch{
Range: logical.span,
Excerpt: trimExcerpt(content, logical.span.Start, logical.span.End),
})
}
return out
}
// AuditHtaccessFile runs AuditHtaccessContent against the file at path:
// every finding, and the merged byte ranges a generic finding's manual fix
// removes.
func AuditHtaccessFile(path string) ([]alert.Finding, []htaccessByteRange) {
findings, ranges, _ := auditHtaccessFile(path)
return findings, ranges
}
func auditHtaccessFile(path string) ([]alert.Finding, []htaccessByteRange, bool) {
if filepath.Base(path) != ".htaccess" {
return nil, nil, true
}
content, ok, err := readHtaccessBounded(path)
if htaccessOversized(ok, err) {
return []alert.Finding{{
Severity: alert.High,
Check: "htaccess_injection",
Message: fmt.Sprintf(".htaccess too large to audit: %s", path),
Details: fmt.Sprintf("File exceeds %d bytes; a real .htaccess is a few kilobytes. Inspect it by hand.", htaccessMaxFileBytes),
FilePath: path,
Timestamp: time.Now(),
}}, nil, false
}
if err != nil || !ok {
return nil, nil, os.IsNotExist(err)
}
findings, ranges := AuditHtaccessContent(path, content)
return findings, ranges, true
}
// AuditHtaccessContent judges content with the generic scanner and every
// per-pattern detector: what realtime reports and what a generic finding's
// manual fix removes.
func AuditHtaccessContent(path string, content []byte) ([]alert.Finding, []htaccessByteRange) {
return auditHtaccessContent(context.Background(), path, content)
}
func auditHtaccessContent(ctx context.Context, path string, content []byte) ([]alert.Finding, []htaccessByteRange) {
if ctx.Err() != nil {
return nil, nil
}
legacy, matches := auditHtaccessLegacyContent(path, content, htaccessSuspiciousPatterns, htaccessSafePatterns)
var findings []alert.Finding
var ranges []htaccessByteRange
for i, finding := range legacy {
// The specific prelude detector reports retained matches. Do not add a
// generic finding that would advertise automatic removal for them.
// Handler abuse arrives with the per-pattern set below.
if !matches[i].Retain && finding.Check != "htaccess_handler_abuse" {
findings = append(findings, finding)
ranges = append(ranges, matches[i].Range)
}
}
patternFindings, patternRanges := auditHtaccessPatternsContext(ctx, path, content)
if ctx.Err() != nil {
return nil, nil
}
return append(findings, patternFindings...), mergeRanges(append(ranges, patternRanges...))
}
// auditHtaccessPatterns runs the per-pattern detectors plus handler abuse.
// Automatic cleaning and their manual fix act on this set: the rest of the
// generic scanner also flags directives a site may rely on, such as a PHP
// handler for .html pages.
func auditHtaccessPatterns(path string, content []byte) ([]alert.Finding, []htaccessByteRange) {
return auditHtaccessPatternsContext(context.Background(), path, content)
}
func auditHtaccessPatternsContext(ctx context.Context, path string, content []byte) ([]alert.Finding, []htaccessByteRange) {
findings, abuse := htaccessHandlerAbuse(path, content)
var ranges []htaccessByteRange
for _, m := range abuse {
ranges = append(ranges, m.Range)
}
for _, d := range htaccessDetectors {
if ctx.Err() != nil {
return nil, nil
}
var matches []htaccessMatch
if d.DetectContext != nil {
matches = d.DetectContext(ctx, content, path)
} else {
matches = d.Detect(content, path)
}
for _, m := range matches {
findings = append(findings, alert.Finding{
Severity: d.Severity,
Check: d.Name,
Message: fmt.Sprintf("%s in %s", d.Name, path),
Details: fmt.Sprintf("File: %s\nMatch: %s", path, m.Excerpt),
FilePath: path,
Timestamp: time.Now(),
})
if !m.Retain {
ranges = append(ranges, m.Range)
}
}
}
return findings, mergeRanges(ranges)
}
// CleanHtaccessFile audits the file, computes the removal range
// set, backs up the original, and writes the trimmed content.
// Returns success=false with no Action when no detector matched
// (i.e., nothing to clean).
//
// Caller is responsible for gating on cfg.AutoResponse.CleanHtaccess
// before invoking; this function will clean unconditionally if
// detectors find anything.
func CleanHtaccessFile(path string) RemediationResult {
return cleanHtaccessFileIdentified(path, nil)
}
func cleanHtaccessFileIdentified(path string, expected os.FileInfo) (result RemediationResult) {
audit := newCleanAction(path)
defer func() { audit.finish(result.Error) }()
if filepath.Base(path) != ".htaccess" {
return RemediationResult{Refused: true, Error: "automated .htaccess remediation only applies to .htaccess files"}
}
resolved, info, err := resolveExistingFixPath(path, effectiveFixRoots(fixHtaccessAllowedRoots))
if err != nil {
return RemediationResult{Refused: errors.Is(fileResponseSourceError(err), errFileResponseRefused), Error: err.Error()}
}
// The account owner controls this directory and we run as root, so the
// file is pinned by inode and replaced through a random O_EXCL name under
// the pinned parent fd. A guessable staging name would let the owner
// plant a symlink there and have the cleaned bytes written anywhere.
target, err := openFixTarget(resolved, info)
if err != nil {
return RemediationResult{Refused: errors.Is(fileResponseSourceError(err), errFileResponseRefused), Error: fmt.Sprintf("cannot open: %v", err)}
}
defer target.Close()
if expected != nil && (!sameFileIdentity(expected, target.Info) || !sameContentShape(expected, target.Info)) {
result.Refused = true
result.Error = "file changed before automatic cleaning"
return result
}
audit.rec.Result = actionlog.Failed
original, ok, err := readHtaccessFileBounded(target.File)
if err != nil {
return RemediationResult{Error: fmt.Sprintf("cannot read: %v", err)}
}
if !ok {
audit.rec.Result = actionlog.Refused
return RemediationResult{Refused: true, Error: ".htaccess too large to clean automatically"}
}
audit.capture(target, original)
audit.rec.Result = actionlog.Refused
_, ranges := auditHtaccessPatterns(resolved, original)
if len(ranges) == 0 {
return RemediationResult{Refused: true, Error: "no malicious directives found to remove"}
}
cleaned := applyRangeRemoval(original, ranges)
if len(cleaned) == len(original) {
return RemediationResult{Refused: true, Error: "no bytes removed (range computation produced empty diff)"}
}
backupDir := htaccessBackupDirRoot
backupPath := newQuarantinePath(backupDir, resolved)
meta := quarantineMetadata(resolved, target.Info, fmt.Sprintf("htaccess clean: %d ranges removed (%d -> %d bytes)", len(ranges), len(original), len(cleaned)))
audit.rec.Result = actionlog.Failed
audit.rec.Reason = meta.Reason
if err := storeQuarantineBackup(backupPath, original, meta, 0640); err != nil {
return RemediationResult{Error: fmt.Sprintf("writing durable backup: %v", err)}
}
if err := audit.replace(target, cleaned, backupPath); err != nil {
return RemediationResult{Refused: errors.Is(err, errFileResponseRefused), Error: fmt.Sprintf("atomic replace: %v", err)}
}
bytesRemoved := len(original) - len(cleaned)
companionSweeps.add(resolved, target.Info, time.Now())
return RemediationResult{
Success: true,
Action: fmt.Sprintf("removed %d malicious byte(s) from %s", bytesRemoved, resolved),
Description: fmt.Sprintf("Cleaned .htaccess: %d ranges, %d bytes removed (backup: %s)", len(ranges), bytesRemoved, backupPath),
}
}
// mergeRanges normalises the input slice: sort by start, then merge
// overlapping or adjacent ranges so cleaning produces deterministic
// output regardless of detector order.
func mergeRanges(in []htaccessByteRange) []htaccessByteRange {
if len(in) == 0 {
return nil
}
cp := make([]htaccessByteRange, len(in))
copy(cp, in)
sort.Slice(cp, func(i, j int) bool { return cp[i].Start < cp[j].Start })
out := []htaccessByteRange{cp[0]}
for _, r := range cp[1:] {
last := &out[len(out)-1]
if r.Start <= last.End {
if r.End > last.End {
last.End = r.End
}
continue
}
out = append(out, r)
}
return out
}
// applyRangeRemoval slices `content` minus every range in `ranges`
// (which mergeRanges has already sorted/merged). Each removal also
// includes the trailing newline if `end` lands on one, so we do not
// leave a blank line behind.
func applyRangeRemoval(content []byte, ranges []htaccessByteRange) []byte {
out := make([]byte, 0, len(content))
cursor := 0
for _, r := range ranges {
if r.Start > cursor {
out = append(out, content[cursor:r.Start]...)
}
end := r.End
if end < len(content) && content[end] == '\n' {
end++
}
cursor = end
}
if cursor < len(content) {
out = append(out, content[cursor:]...)
}
return out
}
type htaccessPhysicalByteLine struct {
text string
start int
end int
}
type htaccessLogicalByteLine struct {
text string
span htaccessByteRange
}
func splitHtaccessPhysicalByteLines(content []byte) []htaccessPhysicalByteLine {
var lines []htaccessPhysicalByteLine
for start := 0; start <= len(content); {
if start == len(content) {
lines = append(lines, htaccessPhysicalByteLine{
text: "",
start: start,
end: start,
})
break
}
end := start
for end < len(content) && content[end] != '\n' {
end++
}
lines = append(lines, htaccessPhysicalByteLine{
text: string(content[start:end]),
start: start,
end: end,
})
if end == len(content) {
break
}
start = end + 1
}
return lines
}
func htaccessLogicalByteLines(content []byte) []htaccessLogicalByteLine {
physical := splitHtaccessPhysicalByteLines(content)
var out []htaccessLogicalByteLine
for i := 0; i < len(physical); {
start := physical[i].start
var end int
var sb strings.Builder
for {
body, continues := htaccessContinuationBody(physical[i].text, i < len(physical)-1)
end = physical[i].end
if continues {
sb.WriteString(body)
i++
continue
}
sb.WriteString(body)
break
}
out = append(out, htaccessLogicalByteLine{
text: sb.String(),
span: htaccessByteRange{Start: start, End: end},
})
i++
}
return out
}
func matchesFromLogicalLineRegex(content []byte, re *regexp.Regexp) []htaccessMatch {
var out []htaccessMatch
for _, logical := range htaccessLogicalByteLines(content) {
if !re.MatchString(logical.text) {
continue
}
out = append(out, htaccessMatch{
Range: logical.span,
Excerpt: trimExcerpt(content, logical.span.Start, logical.span.End),
})
}
return out
}
// detectPHPInUploads flags AddHandler/SetHandler/ForceType lines
// that map to PHP when the .htaccess lives inside a directory where
// PHP execution is rarely legitimate.
func detectPHPInUploads(content []byte, path string) []htaccessMatch {
if !pathInNonScriptDir(path) {
return nil
}
return matchesFromLogicalLineRegex(content, rePHPHandlerMap)
}
func pathInNonScriptDir(path string) bool {
lower := strings.ToLower(path)
for _, dir := range htaccessNonScriptDirHints {
if strings.Contains(lower, dir) {
return true
}
}
return false
}
// detectAutoPrepend flags PHP auto_prepend_file and auto_append_file
// directives whose target the account owner can write to (see
// autoPrependTargetSuspicious).
func detectAutoPrepend(content []byte, path string) []htaccessMatch {
idxs := reAutoPrepend.FindAllSubmatchIndex(content, -1)
var out []htaccessMatch
for _, idx := range idxs {
if len(idx) < 6 {
continue
}
target := string(content[idx[4]:idx[5]])
if !autoPrependTargetSuspicious(target, path) {
continue
}
out = append(out, htaccessMatch{
Range: lineRange(content, idx[0], idx[1]),
Excerpt: trimExcerpt(content, idx[0], idx[1]),
// A WordPress installed below the site root puts the plugin's
// real prelude outside the bound location, so a mismatch is
// reported for review but never removed automatically.
Retain: strings.EqualFold(string(content[idx[2]:idx[3]]), "auto_prepend_file") && preludeBase(strings.Trim(target, `"'`)) == rssslPreludeName,
})
}
return out
}
// reUARewriteRuleLine captures the substitution and flag list of a single
// RewriteRule directive. Pairing is handled by uaCloakPairedRule so a later
// unrelated rule is not mistaken for the rule fed by a RewriteCond chain.
var reUARewriteRuleLine = regexp.MustCompile(`(?i)^\s*RewriteRule\s+\S+\s+(\S+)(?:\s+\[([^\]]+)\])?\s*$`)
type uaRewriteRulePair struct {
start int
end int
substitution string
flags string
parsed bool
}
// uaCloakDefensiveFlags lists RewriteRule flags that, when set on the
// rule paired with a UA cond, indicate defensive blocking rather than
// content cloaking. Apache combines these flags with a comma so each
// flag is matched as a substring of the bracketed flag list.
var uaCloakDefensiveFlags = []string{"f", "g"}
// uaCloakBlocklistThreshold is the number of OR-list entries in the
// UA cond's regex that converts the cond from "potential cloak" to
// "operator-installed bot blocklist". A cond with 4+ alternation
// entries is overwhelmingly a defensive block (the canonical
// SoftAculous / Apache Bad Bots list ships ~20+ entries).
const uaCloakBlocklistThreshold = 4
// uaCloakAlternationCount counts the top-level "|" alternation
// branches in a UA cond regex pattern, ignoring "|" characters inside
// nested parentheses. Used to identify long bot blocklists.
func uaCloakAlternationCount(pattern string) int {
return len(uaCloakAlternationBranches(pattern))
}
// uaCloakAlternationBranches splits a UA cond regex pattern on its
// top-level "|" alternations, ignoring "|" inside nested parentheses.
func uaCloakAlternationBranches(pattern string) []string {
depth := 0
inClass := false
prevEscape := false
start := 0
var branches []string
for i := 0; i < len(pattern); i++ {
c := pattern[i]
if prevEscape {
prevEscape = false
continue
}
switch c {
case '\\':
prevEscape = true
case '[':
if !inClass {
inClass = true
}
case ']':
if inClass {
inClass = false
}
case '(':
if !inClass {
depth++
}
case ')':
if !inClass && depth > 0 {
depth--
}
case '|':
if !inClass && depth <= 1 {
branches = append(branches, pattern[start:i])
start = i + 1
}
}
}
return append(branches, pattern[start:])
}
// uaCloakSearchCrawlerMajority reports whether more than half of the UA
// alternatives across the given cond patterns name search-engine crawlers.
// Such a chain is a cloak target list, not a scraper blocklist, so the
// long-list gates must not silence it.
func uaCloakSearchCrawlerMajority(patterns []string) bool {
total, search := 0, 0
for _, p := range patterns {
for _, branch := range uaCloakAlternationBranches(p) {
total++
if searchCrawlerUARegex.MatchString(branch) {
search++
}
}
}
return total > 0 && search*2 > total
}
// uaCloakPairedRuleIsDefensive scans forward from condEnd for the
// RewriteRule directive that Apache will pair with the cond. Returns
// true when that rule is a no-op ("-" substitution), a forbid
// ([F]/[G] flag), or absent. Both shapes are defensive, not cloaking.
//
// Apache's chaining model: a chain of RewriteCond lines applies to
// the FIRST RewriteRule that follows them in the file. We walk
// line-by-line and stop at the first RewriteRule. A blank line, a
// non-RewriteCond directive, or end-of-file means there is no paired
// rule - the cond is dead text and not actively cloaking anything.
func uaCloakPairedRuleIsDefensive(content []byte, condEnd int) bool {
rule, ok := uaCloakPairedRule(content, condEnd)
if !ok {
return true
}
if !rule.parsed {
return false
}
if rule.substitution == "-" {
return true
}
for _, f := range uaCloakDefensiveFlags {
// Match flag as a comma-bounded token: "F" matches "[F]",
// "[F,L]", "[L,F]"; does NOT match "[NC]" or "[QSA]".
for _, token := range strings.Split(rule.flags, ",") {
if strings.TrimSpace(token) == f {
return true
}
}
}
return false
}
// uaCloakPairedRule returns the first RewriteRule in the contiguous
// RewriteCond chain that starts after condEnd. Blank lines and comments are
// ignored by Apache, so they do not break the chain; any other directive does.
// Otherwise a stale UA condition can be paired with an unrelated later
// RewriteRule and cleaning would remove too much.
func uaCloakPairedRule(content []byte, condEnd int) (uaRewriteRulePair, bool) {
pos := condEnd
for {
lineStart, lineEnd, rawEnd, ok := htaccessLineAfter(content, pos)
if !ok {
return uaRewriteRulePair{}, false
}
trimmed := strings.TrimSpace(string(content[lineStart:lineEnd]))
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
pos = rawEnd
continue
}
switch htaccessDirectiveName(trimmed) {
case "rewritecond":
pos = rawEnd
continue
case "rewriterule":
rule := uaRewriteRulePair{start: lineStart, end: lineEnd}
if idx := reUARewriteRuleLine.FindStringSubmatchIndex(string(content[lineStart:lineEnd])); idx != nil {
line := string(content[lineStart:lineEnd])
rule.substitution = line[idx[2]:idx[3]]
if idx[4] != -1 {
rule.flags = strings.ToLower(line[idx[4]:idx[5]])
}
rule.parsed = true
}
return rule, true
default:
return uaRewriteRulePair{}, false
}
}
}
func htaccessLineAfter(content []byte, pos int) (start, lineEnd, rawEnd int, ok bool) {
for pos < len(content) && content[pos] != '\n' {
pos++
}
if pos >= len(content) {
return 0, 0, 0, false
}
pos++
if pos >= len(content) {
return 0, 0, 0, false
}
start = pos
rawEnd = pos
for rawEnd < len(content) && content[rawEnd] != '\n' {
rawEnd++
}
lineEnd = rawEnd
if lineEnd > start && content[lineEnd-1] == '\r' {
lineEnd--
}
return start, lineEnd, rawEnd, true
}
func htaccessLineBefore(content []byte, pos int) (start, lineEnd int, ok bool) {
if pos <= 0 {
return 0, 0, false
}
lineEnd = pos
for lineEnd > 0 && (content[lineEnd-1] == '\n' || content[lineEnd-1] == '\r') {
lineEnd--
}
if lineEnd <= 0 {
return 0, 0, false
}
start = lineEnd
for start > 0 && content[start-1] != '\n' {
start--
}
return start, lineEnd, true
}
func htaccessDirectiveName(line string) string {
fields := strings.Fields(line)
if len(fields) == 0 {
return ""
}
return strings.ToLower(fields[0])
}
// detectUserAgentCloak flags RewriteCond %{HTTP_USER_AGENT}
// directives that match a known crawler UA AND are part of an active
// content-cloaking rule. Four suppression gates filter legitimate
// shapes before emitting the High alert:
//
// 1. Negated cond ("RewriteCond %{HTTP_USER_AGENT} !..."): the rule
// applies only when the UA is NOT this crawler. Cache plugins
// (WP Fastest Cache, WP Super Cache) ship long negated lists to
// exclude social-share scrapers from the cached-content rewrite.
//
// 2. Long alternation (>= uaCloakBlocklistThreshold OR-branches):
// operator-installed defensive blocklists ship many bot UAs in
// a single OR chain paired with a [F] forbid or sinkhole rewrite.
// Cloakers use one or two crawler names.
//
// 3. Long multi-line chain: the canonical Apache "Bad Bots" snippet
// puts each scraper UA on its own RewriteCond line, terminated
// by one RewriteRule. The chain length is the blocklist signal;
// per-cond alternation is always 1.
//
// 4. Paired RewriteRule is defensive ("-" substitution, [F]/[G]
// flag, or absent): the cond is part of a forbid / env-var-set
// block, not a content swap.
//
// All gates fail-closed: any uncertainty (no paired rule found,
// parse failure) keeps the original alert firing.
func detectUserAgentCloak(content []byte, _ string) []htaccessMatch {
idxs := reUACloakCond.FindAllSubmatchIndex(content, -1)
chainSize := uaCloakChainSizes(content, idxs)
// A chain whose UA alternatives are mostly search-engine crawlers is a
// cloak target list, not a scraper blocklist: gates 2 and 3 do not
// apply to it and only the paired rule (gate 4) can clear it.
searchList := make([]bool, len(idxs))
for _, group := range uaCloakChainGroups(content, idxs) {
patterns := make([]string, 0, len(group))
for _, i := range group {
if len(idxs[i]) >= 4 {
patterns = append(patterns, string(content[idxs[i][2]:idxs[i][3]]))
}
}
majority := uaCloakSearchCrawlerMajority(patterns)
for _, i := range group {
searchList[i] = majority
}
}
var out []htaccessMatch
for i, idx := range idxs {
if len(idx) < 4 {
continue
}
uaPattern := string(content[idx[2]:idx[3]])
if !crawlerUARegex.MatchString(uaPattern) {
continue
}
// Gate 1: negated cond. Strip leading whitespace and look
// for "!" before the rest of the pattern.
if strings.HasPrefix(strings.TrimSpace(uaPattern), "!") {
continue
}
// Gate 2: long alternation list = bot blocklist.
if !searchList[i] && uaCloakAlternationCount(uaPattern) >= uaCloakBlocklistThreshold {
continue
}
// Gate 3: long multi-line chain = bot blocklist.
if !searchList[i] && chainSize[i] >= uaCloakBlocklistThreshold {
continue
}
// Gate 4: paired RewriteRule is defensive.
if uaCloakPairedRuleIsDefensive(content, idx[1]) {
continue
}
// Removal must take the whole cloak unit: the full contiguous
// RewriteCond chain plus the RewriteRule it feeds. Removing only the
// cond would leave the rule unconditional, turning a crawler-only
// cloak into a redirect of every visitor to the attacker URL.
block := uaCloakBlockRange(content, idx[0], idx[1])
out = append(out, htaccessMatch{
Range: block,
Excerpt: trimExcerpt(content, block.Start, block.End),
})
}
return out
}
// uaCloakBlockRange returns the byte range covering the entire cloak unit for
// the UA cond at [condStart, condEnd): the full contiguous RewriteCond chain it
// belongs to (Apache applies a run of conds to the first following
// RewriteRule) plus that paired RewriteRule. Falls back to the cond's own line
// when no paired rule follows - a dead cond redirects nobody, so there is
// nothing extra to strip.
func uaCloakBlockRange(content []byte, condStart, condEnd int) htaccessByteRange {
chainStart := uaCondChainStart(content, condStart)
rule, ok := uaCloakPairedRule(content, condEnd)
if !ok {
return lineRange(content, chainStart, condEnd)
}
return lineRange(content, chainStart, rule.end)
}
// uaCondChainStart walks backward from the start of a RewriteCond line over
// immediately preceding RewriteCond lines and returns the byte offset where the
// chain begins. Comments and blank lines between conds do not break Apache's
// chain, but they also are not swallowed when they merely precede the first
// cond. A non-RewriteCond directive terminates the walk.
func uaCondChainStart(content []byte, condStart int) int {
start := condStart
scan := condStart
for {
prevStart, prevEnd, ok := htaccessLineBefore(content, scan)
if !ok {
return start
}
trimmed := strings.TrimSpace(string(content[prevStart:prevEnd]))
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
scan = prevStart
continue
}
if htaccessDirectiveName(trimmed) != "rewritecond" {
return start
}
start = prevStart
scan = prevStart
}
}
// uaCloakChainSizes returns, for each UA cond match, the number of
// UA conds in its chain. Two UA conds belong to the same chain when
// only whitespace, comments, or blank lines sit between them; any
// other directive breaks the chain. That mirrors how Apache groups
// conds onto the next RewriteRule.
func uaCloakChainSizes(content []byte, idxs [][]int) []int {
sizes := make([]int, len(idxs))
for _, group := range uaCloakChainGroups(content, idxs) {
for _, i := range group {
sizes[i] = len(group)
}
}
return sizes
}
// uaCloakChainGroups partitions the UA cond matches into chains (see
// uaCloakChainSizes) and returns each chain as the match indexes it holds.
func uaCloakChainGroups(content []byte, idxs [][]int) [][]int {
var groups [][]int
var current []int
for i := range idxs {
if i > 0 && !uaCondsAreAdjacent(content, idxs[i-1][1], idxs[i][0]) {
groups = append(groups, current)
current = nil
}
current = append(current, i)
}
if len(current) > 0 {
groups = append(groups, current)
}
return groups
}
// uaCondsAreAdjacent reports whether the gap between two UA cond matches
// contains only line breaks, whitespace, blank lines, or comments.
func uaCondsAreAdjacent(content []byte, prevEnd, nextStart int) bool {
if prevEnd > nextStart || nextStart > len(content) {
return false
}
seenNewline := false
for pos := prevEnd; pos < nextStart; {
lineEnd := pos
for lineEnd < nextStart && content[lineEnd] != '\n' {
lineEnd++
}
trimmed := strings.TrimSpace(string(content[pos:lineEnd]))
if trimmed != "" && !strings.HasPrefix(trimmed, "#") {
return false
}
if lineEnd < nextStart {
seenNewline = true
lineEnd++
}
pos = lineEnd
}
return seenNewline
}
// detectSpamRedirect flags RewriteRule directives whose target host
// is on a known spam TLD. Operator-supplied legitimate hosts in
// these TLDs need a per-path suppression.
func detectSpamRedirect(content []byte, _ string) []htaccessMatch {
idxs := reSpamRedirect.FindAllSubmatchIndex(content, -1)
var out []htaccessMatch
for _, idx := range idxs {
if len(idx) < 4 {
continue
}
target := string(content[idx[2]:idx[3]])
host := extractHost(target)
if !hostOnSpamTLD(host) {
continue
}
out = append(out, htaccessMatch{
Range: lineRange(content, idx[0], idx[1]),
Excerpt: trimExcerpt(content, idx[0], idx[1]),
})
}
return out
}
func extractHost(rawURL string) string {
u, err := url.Parse(rawURL)
if err != nil {
return ""
}
return strings.ToLower(u.Hostname())
}
func hostOnSpamTLD(host string) bool {
for _, tld := range htaccessSpamTLDs {
if strings.HasSuffix(host, tld) {
return true
}
}
return false
}
// reFilesMatchPattern captures the regex pattern inside the FilesMatch
// quotes. The capture group is everything between the optional quote
// characters, which is the Apache-side regex applied to filenames.
var reFilesMatchPattern = regexp.MustCompile(`(?im)^\s*<FilesMatch\s+["']?([^"'>]+?)["']?\s*>`)
// reFilesMatchExtensionTail strips the canonical PHP extension suffix
// from a FilesMatch pattern so the remaining literal can be examined.
// Anchors and end-of-string markers are left to the caller.
var reFilesMatchExtensionTail = regexp.MustCompile(`(?i)\\\.(?:php|phtml|ph[2-7])\$?\)?$`)
// filesMatchPatternIsTargeted reports whether the FilesMatch regex
// names at least one specific PHP filename rather than granting access
// to every .php file in the directory. Stock plugins ship targeted
// patterns ("wpc\.php$", "ps_facetedsearch-.+\.php$",
// "(webp-on-demand\.php|webp-realizer\.php)$"); the malicious shape
// is a bare wildcard ("\.php$", ".*\.php$", "[^/]+\.php$").
//
// The check is character-class based: any literal alphanumeric, dash,
// or underscore in the pattern (after stripping the trailing
// "\.php$" / "\.phtml$" extension) means the pattern names something
// specific. A pattern composed only of regex meta-characters (".",
// "*", "^", "$", "[", "]", "(", ")", "|", "+", "?", "\\") is treated
// as a wildcard and continues to the wildcard-context check.
//
// The test is applied per alternative: `^(a|.*)\.php$` carries a literal
// yet still matches every .php through its second branch, and treating it
// as targeted let an attacker write a shield the detector skipped. Every
// top-level alternative (after unwrapping one outer group) must name
// something; escapes such as `\w` and character classes do not count.
func filesMatchPatternIsTargeted(pattern string) bool {
stripped := reFilesMatchExtensionTail.ReplaceAllString(pattern, "")
stripped = strings.TrimSuffix(strings.TrimPrefix(stripped, "^"), "$")
if inner, ok := unwrapOuterGroup(stripped); ok {
stripped = inner
}
alternatives := topLevelAlternatives(stripped)
if len(alternatives) == 0 {
return false
}
for _, alt := range alternatives {
if !regexAlternativeNamesSomething(alt) {
return false
}
}
return true
}
// htaccessParentPHPFileCount counts sibling handler files up to the gate's
// threshold. Used to differentiate a plugin directory full of legitimate
// PHP dispatchers from a freshly-attacker-written upload directory
// containing one or zero PHP files.
//
// Errors (parent missing, permission denied, race) return 0 - the
// caller treats that as "not enough sibling PHP" and keeps firing.
func htaccessParentPHPFileCount(ctx context.Context, htaccessPath string) int {
parent := filepath.Dir(htaccessPath)
dir, err := openTenantDirectory(parent)
if err != nil {
return 0
}
defer func() { _ = dir.Close() }()
n := 0
for ctx.Err() == nil {
entries, readErr := dir.ReadDir(companionReadDirBatch)
for _, e := range entries {
if ctx.Err() != nil {
return 0
}
if e.IsDir() {
continue
}
ext := strings.ToLower(filepath.Ext(e.Name()))
switch ext {
case ".php", ".phtml", ".ph2", ".ph3", ".ph4", ".ph5", ".ph6", ".ph7":
n++
if n >= filesMatchShieldSiblingThreshold {
return n
}
}
}
if readErr != nil {
return n
}
}
return n
}
// filesMatchShieldSiblingThreshold is the number of sibling PHP files
// that converts a bare-wildcard FilesMatch shield from "fire" to
// "treat as legitimate plugin allowlist". A directory with 3+ stock
// PHP dispatchers existing alongside the shield is overwhelmingly
// likely to be a legitimate webapp module (KCFinder ships ~5,
// PrestaShop modules ship ~10+, vendor dispatchers ship a handful).
// The attacker drop pattern is .htaccess + one or zero PHP files.
const filesMatchShieldSiblingThreshold = 3
// detectFilesMatchShield finds <FilesMatch ...\.php(tml)?> blocks
// that grant Allow from all or Require all granted -- the canonical
// "let everyone execute everything we just dropped" pattern. The
// returned range covers the full block, opening tag through closing
// tag inclusive.
//
// Two suppression gates run before emitting the finding:
//
// 1. Targeted pattern: if the FilesMatch regex names a specific
// filename ("wpc\.php$", named allowlist, or prefix pattern), it
// is a legitimate plugin allowlist and is skipped.
//
// 2. Bare wildcard with sibling PHP context: if the FilesMatch is a
// bare wildcard but the .htaccess parent directory contains
// multiple sibling PHP dispatchers, the shield is protecting an
// existing legitimate plugin layout, not a freshly-dropped
// dropper. Threshold is filesMatchShieldSiblingThreshold.
//
// Both gates fail-open: any uncertainty (parse failure, IO error)
// keeps the original Critical alert firing. The sibling-PHP gate
// requires the htaccess path so the detector now reads it from the
// caller (passed as the second argument to all htaccessDetector.Detect
// implementations).
func detectFilesMatchShield(content []byte, path string) []htaccessMatch {
return detectFilesMatchShieldContext(context.Background(), content, path)
}
func detectFilesMatchShieldContext(ctx context.Context, content []byte, path string) []htaccessMatch {
openIdxs := reFilesMatchOpen.FindAllIndex(content, -1)
closeIdxs := reFilesMatchClose.FindAllIndex(content, -1)
if len(openIdxs) == 0 || len(closeIdxs) == 0 {
return nil
}
patternIdxs := reFilesMatchPattern.FindAllSubmatchIndex(content, -1)
var out []htaccessMatch
siblingCount := -1
for _, open := range openIdxs {
if ctx.Err() != nil {
return nil
}
// pair this opening tag with the next closing tag after it
var paired []int
for _, c := range closeIdxs {
if c[0] >= open[1] {
paired = c
break
}
}
if paired == nil {
continue
}
if !htaccessBodyGrantsAccess(content[open[1]:paired[0]]) {
continue
}
// Look up the FilesMatch pattern that opened at this position
// so we can apply the targeted-vs-wildcard discriminator.
var openPattern string
for _, pIdx := range patternIdxs {
if len(pIdx) < 4 {
continue
}
if pIdx[0] == open[0] {
openPattern = string(content[pIdx[2]:pIdx[3]])
break
}
}
if openPattern != "" && filesMatchPatternIsTargeted(openPattern) {
continue
}
// Bare wildcard: check sibling PHP count. A directory with
// multiple stock PHP dispatchers is a legitimate plugin layout.
// Not inside an upload-style tree, though: there the siblings
// are whatever the uploader chose to drop, and three dummy .php
// files are the cheapest way to silence this finding.
if path != "" && !htaccessInUploadTree(path) {
if siblingCount < 0 {
siblingCount = htaccessParentPHPFileCount(ctx, path)
}
if ctx.Err() != nil {
return nil
}
if siblingCount >= filesMatchShieldSiblingThreshold {
continue
}
}
out = append(out, htaccessMatch{
Range: blockRange(content, open[0], paired[1]),
Excerpt: trimExcerpt(content, open[0], paired[1]),
})
}
return out
}
// detectHeaderInjection flags Header set / Header add directives
// whose name is on the small tracking-header allowlist. Generic
// CSP / HSTS / X-Frame-Options headers do not match.
func detectHeaderInjection(content []byte, _ string) []htaccessMatch {
idxs := reHeaderSetAdd.FindAllSubmatchIndex(content, -1)
var out []htaccessMatch
for _, idx := range idxs {
if len(idx) < 6 {
continue
}
name := string(content[idx[4]:idx[5]])
if !headerNameSuspicious(name) {
continue
}
out = append(out, htaccessMatch{
Range: lineRange(content, idx[0], idx[1]),
Excerpt: trimExcerpt(content, idx[0], idx[1]),
})
}
return out
}
func headerNameSuspicious(name string) bool {
lower := strings.ToLower(name)
for _, h := range htaccessTrackingHeaders {
if strings.HasPrefix(lower, strings.ToLower(h)) {
return true
}
}
return false
}
// errorDocumentHostShareThreshold bounds how short a host-vs-path
// substring match can be while still treating the redirect as
// "same-brand". Three characters is the floor: anything shorter would
// match incidental segments ("us" inside "user", "co" inside "co.uk")
// and let an attacker tunnel through with a name like "co.evil.com".
const errorDocumentHostShareThreshold = 4
// errorDocumentHostIsSameBrand reports whether the URL host's
// "registrable label" (the leftmost segment of the public suffix +
// 1) shares an alphanumeric stem of >= errorDocumentHostShareThreshold
// chars with any path component of the .htaccess file. This catches
// the dominant legitimate shape: a custom 404 redirect to the site's
// own homepage on the same brand domain.
//
// Examples:
//
// /home/acme/public_html/.htaccess + https://acmegroup.example
// -> account "acme" is a substring of label "acmegroup" -> same-brand
//
// /home/shop/example-shop.com/.htaccess + https://www.example-shop.com/404
// -> domain dir "example-shop.com" contains label "example-shop" -> same-brand
//
// /home/victim/public_html/.htaccess + https://attacker.com/landing
// -> "attacker" shares no >=4-char stem with any path component -> different-brand
func errorDocumentHostIsSameBrand(htaccessPath, urlHost string) bool {
label := registrableLabel(urlHost)
if len(label) < errorDocumentHostShareThreshold {
return false
}
labelLower := strings.ToLower(label)
for _, component := range strings.Split(htaccessPath, string(filepath.Separator)) {
if component == "" || component == "public_html" || component == "home" {
continue
}
comp := strings.ToLower(component)
if longestCommonAlnumRun(comp, labelLower) >= errorDocumentHostShareThreshold {
return true
}
}
return false
}
// registrableLabel extracts the leftmost segment of the public
// suffix + 1: for "www.example-shop.com" returns "example-shop", for
// "acmegroup.example" returns "acmegroup". This is heuristic - we treat
// the last dot-separated segment as the TLD - but it is robust to
// the common cases (single-segment TLD, two-segment country code TLD
// like "co.uk" handled by stripping known double-segment suffixes).
//
// Returns "" for inputs that look like an IPv4 dotted quad: numeric
// targets are flagged separately and never qualify as same-brand.
func registrableLabel(host string) string {
host = strings.ToLower(strings.TrimSpace(host))
if host == "" {
return ""
}
// IPv4 dotted quad: each segment must be 1-3 digits.
if isIPv4(host) {
return ""
}
// Strip a leading "www." for canonical comparison.
host = strings.TrimPrefix(host, "www.")
parts := strings.Split(host, ".")
if len(parts) < 2 {
return host
}
// Two-segment public suffix heuristic: "co.uk", "co.za",
// "com.au" etc. If the second-to-last segment is one of these
// short country-code prefixes, the registrable label is the
// THIRD-from-last segment.
if len(parts) >= 3 {
penultimate := parts[len(parts)-2]
twoSegmentCC := map[string]bool{
"co": true, "com": true, "net": true, "org": true,
"ac": true, "gov": true, "edu": true,
}
if twoSegmentCC[penultimate] {
return parts[len(parts)-3]
}
}
return parts[len(parts)-2]
}
// isIPv4 reports whether s parses as a dotted-quad IPv4 address.
// IPv6 / hex / mixed forms are caught separately by IP-target
// signalling above the same-brand check.
func isIPv4(s string) bool {
parts := strings.Split(s, ".")
if len(parts) != 4 {
return false
}
for _, p := range parts {
if p == "" || len(p) > 3 {
return false
}
for _, c := range p {
if c < '0' || c > '9' {
return false
}
}
}
return true
}
// longestCommonAlnumRun returns the length of the longest common
// substring of a and b that consists entirely of alphanumeric
// characters. Symbols (".", "-", "_") segment the run so an
// "example-shop" path component can match a "example-shop.com" URL
// host on the "example" stem without crediting the dash.
func longestCommonAlnumRun(a, b string) int {
best := 0
// Walk a, accumulate alnum tokens, check each token against b.
tokens := splitAlnumTokens(a)
bLower := b
for _, tok := range tokens {
if len(tok) < best {
continue
}
// Find tok inside b: contiguous alnum substring match.
if strings.Contains(bLower, tok) {
if len(tok) > best {
best = len(tok)
}
}
}
return best
}
// splitAlnumTokens splits s on any non-alphanumeric character, returning
// the maximal alphanumeric runs.
func splitAlnumTokens(s string) []string {
var out []string
start := -1
for i := 0; i < len(s); i++ {
c := s[i]
isAlnum := (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')
if isAlnum && start == -1 {
start = i
} else if !isAlnum && start != -1 {
out = append(out, s[start:i])
start = -1
}
}
if start != -1 {
out = append(out, s[start:])
}
return out
}
// detectErrorDocumentHijack flags ErrorDocument directives whose
// target is an external http(s) URL pointing at a host that does NOT
// share a brand stem with the .htaccess file's path. Same-brand
// redirects (custom 404 -> site homepage) are extremely common and
// must not be flagged. Spam-TLD targets and IP-address targets are
// always flagged regardless of any brand match.
func detectErrorDocumentHijack(content []byte, path string) []htaccessMatch {
idxs := reErrorDocument.FindAllSubmatchIndex(content, -1)
var out []htaccessMatch
for _, idx := range idxs {
if len(idx) < 4 {
continue
}
target := string(content[idx[2]:idx[3]])
host := extractHost(target)
// Spam TLDs and IP-address targets always fire. Both are
// signals of compromise even when the path-share heuristic
// would otherwise consider the host same-brand.
if hostOnSpamTLD(host) || isIPv4(host) {
out = append(out, htaccessMatch{
Range: lineRange(content, idx[0], idx[1]),
Excerpt: trimExcerpt(content, idx[0], idx[1]),
})
continue
}
// Same-brand check only when we have a path to compare
// against. Detector is sometimes called with an empty path
// (unit tests of the regex alone); fail-closed.
if path != "" && errorDocumentHostIsSameBrand(path, host) {
continue
}
out = append(out, htaccessMatch{
Range: lineRange(content, idx[0], idx[1]),
Excerpt: trimExcerpt(content, idx[0], idx[1]),
})
}
return out
}
// lineRange computes a range covering the full line(s) that contain
// [start, end]. Line boundaries are at '\n'; the returned end is
// the position of the trailing '\n' (exclusive) or len(content) if
// the match is at EOF without a trailing newline.
func lineRange(content []byte, start, end int) htaccessByteRange {
if start < 0 {
start = 0
}
if end > len(content) {
end = len(content)
}
for start > 0 && content[start-1] != '\n' {
start--
}
for end < len(content) && content[end] != '\n' {
end++
}
return htaccessByteRange{Start: start, End: end}
}
// blockRange covers the whole block including the opening and
// closing directive lines. Same line-snapping rules as lineRange.
func blockRange(content []byte, start, end int) htaccessByteRange {
r := lineRange(content, start, end)
return r
}
// trimExcerpt returns up to ~200 bytes of the matched content,
// trimmed and with newlines collapsed, for finding details.
func trimExcerpt(content []byte, start, end int) string {
if start < 0 {
start = 0
}
if end > len(content) {
end = len(content)
}
s := strings.TrimSpace(string(content[start:end]))
s = strings.ReplaceAll(s, "\n", " | ")
if len(s) > 200 {
s = s[:200] + "..."
}
return s
}
package checks
import (
"fmt"
"regexp"
"strings"
"github.com/pidginhost/csm/internal/alert"
)
// Rewrite flags end at a comma or closing bracket. Those delimiters belong
// to Apache, not the PHP target; SetEnv values can contain either character.
var reRewritePreludeTarget = regexp.MustCompile(`(?i)auto_(?:prepend|append)_file(?:[\t ]*=[\t ]*|[\t ]+)("[^"\r\n]*"|'[^'\r\n]*'|[^\s,\]]+)`)
var htaccessSuspiciousPatterns = []string{
"auto_prepend_file",
"auto_append_file",
"eval(",
"base64_decode",
"gzinflate",
"str_rot13",
"php_value disable_functions",
"addhandler",
"addtype",
"sethandler",
}
var htaccessSafePatterns = []string{
"wordfence-waf.php",
"litespeed",
"advanced-headers.php",
"rsssl",
// Standard handler directives for PHP/static files are safe
"application/x-httpd-php",
"application/x-httpd-php5",
"application/x-httpd-ea-php",
"application/x-httpd-alt-php",
"text/html",
"text/css",
"text/javascript",
"application/javascript",
"image/",
"font/",
"proxy:unix",
// Security plugins that use handler directives to BLOCK execution
"-execcgi", // Options -ExecCGI disables CGI (Wordfence pattern)
"sethandler none", // Disables all handlers (security measure)
"sethandler default-handler", // Resets to default (security measure)
// Legitimate MIME type additions
"application/font",
"application/vnd",
".woff",
".woff2",
".ttf",
".eot",
".svg",
// Wordfence code execution protection
"wordfence",
}
// auditHtaccessLegacyContent keeps generic findings and their removal spans together.
// Manual fixes, automatic cleaning and verification must judge the same bytes.
func auditHtaccessLegacyContent(path string, content []byte, suspicious, safe []string) ([]alert.Finding, []htaccessMatch) {
var findings []alert.Finding
var matches []htaccessMatch
// Build full file content for context checks
fullContentLower := strings.ToLower(string(content))
// If file contains handler directives paired with -ExecCGI, the whole
// block is a security measure (e.g., Wordfence execution protection)
hasExecCGIBlock := strings.Contains(fullContentLower, "-execcgi")
var phpHandlerContexts []phpHandlerOverlay
nextLine := 0
logicalLines := htaccessLogicalByteLines(content)
for lineIndex, logical := range logicalLines {
lineNum := nextLine
end := logical.span.End
if end < len(content) {
end++
}
nextLine += strings.Count(string(content[logical.span.Start:end]), "\n")
trimmed := strings.TrimSpace(logical.text)
lineLower := strings.ToLower(trimmed)
// Skip comments entirely - commented-out directives are not active
if strings.HasPrefix(trimmed, "#") {
continue
}
if ctx, ok := openPHPHandlerContext(trimmed); ok {
phpHandlerContexts = append(phpHandlerContexts, ctx)
continue
}
if closesPHPHandlerContext(trimmed) {
if len(phpHandlerContexts) > 0 {
phpHandlerContexts = phpHandlerContexts[:len(phpHandlerContexts)-1]
}
continue
}
// A PHP execution handler mapped onto a non-PHP extension is the
// handler-remap webshell technique: an uploaded .jpg then runs as
// PHP. The safe-pattern and AddType skips below would otherwise
// suppress it because the handler name itself is a normal PHP
// handler, so this override fires first and unconditionally.
remapsNonPHP := phpHandlerRemapsNonPHP(lineLower)
if !remapsNonPHP && len(phpHandlerContexts) > 0 {
remapsNonPHP = phpHandlerRemapsNonPHPInContext(lineLower, phpHandlerContexts)
}
if remapsNonPHP {
matches = append(matches, htaccessMatch{Range: logical.span})
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "htaccess_injection",
Message: "PHP handler mapped to non-PHP extension (handler remap)",
Details: fmt.Sprintf("File: %s (line %d)\nContent: %s", path, lineNum+1, trimmed),
FilePath: path,
})
continue
}
// The prelude is judged wherever the line carries one: PHP-FPM also
// takes it from a PHP_VALUE environment variable, which SetEnv or a
// RewriteRule E= flag can set.
fields := apacheDirectiveFields(trimmed)
preludeText := trimmed
preludePattern := reAutoPrependTarget
if len(fields) > 0 {
if strings.EqualFold(fields[0], "SetEnv") || strings.EqualFold(fields[0], "RewriteRule") {
// PHP parses the value after Apache removes its outer quotes
// and escapes, including escaped quotes around a PHP path.
if decoded, ok := parseApacheDirectiveFields(trimmed); ok {
preludeText = strings.Join(decoded, " ")
}
}
if strings.EqualFold(fields[0], "RewriteRule") {
preludePattern = reRewritePreludeTarget
}
}
preludes := preludePattern.FindAllStringSubmatchIndex(preludeText, -1)
var suspiciousPrelude, pluginPrelude bool
for _, m := range preludes {
target := preludeText[m[2]:m[3]]
if !autoPrependTargetSuspicious(target, path) {
continue
}
suspiciousPrelude = true
// Really Simple Security writes its prelude only as a php_value
// prepend, so only that form is kept out of cleaning.
prefix := strings.ToLower(strings.TrimSpace(preludeText[:m[0]]))
pluginPrelude = (prefix == "" || prefix == "php_value" || prefix == "php_admin_value") &&
strings.HasPrefix(strings.ToLower(preludeText[m[0]:]), "auto_prepend_file") &&
preludeBase(strings.Trim(target, `"'`)) == rssslPreludeName
// The retained php_value form has one executable target. Later
// text must not turn that plugin directive into a removable hit.
break
}
if len(preludes) > 0 && !suspiciousPrelude {
continue
}
// Other tokens inside a rewrite condition or rule are regex patterns
// in an attack-query blocklist, not code.
if len(preludes) == 0 && len(fields) > 0 &&
(strings.EqualFold(fields[0], "RewriteCond") || strings.EqualFold(fields[0], "RewriteRule")) {
continue
}
for _, pattern := range suspicious {
patternLower := strings.ToLower(pattern)
if !strings.Contains(lineLower, patternLower) {
continue
}
if strings.Contains(patternLower, "auto_") && len(preludes) == 0 {
continue
}
// A prelude directive is judged by its target file alone. The
// line-wide safe list below would let a target such as
// ".../uploads/fonts/x.ttf" or ".../litespeed/x.php" exempt itself
// with a word the attacker chose.
if len(preludes) == 0 && (patternLower == "addhandler" || patternLower == "sethandler" || patternLower == "addtype") {
// Check per-line safe patterns
isSafe := false
for _, sp := range safe {
if strings.Contains(lineLower, strings.ToLower(sp)) {
isSafe = true
break
}
}
if isSafe {
continue
}
}
// For handler directives, apply context-aware checks
if patternLower == "addhandler" || patternLower == "sethandler" {
// Skip if paired with -ExecCGI (Wordfence protection)
if hasExecCGIBlock {
continue
}
// Skip Drupal security handlers
if strings.Contains(lineLower, "drupal_security") {
continue
}
// Skip SetHandler none/default (disabling handlers = security measure)
if strings.Contains(lineLower, "sethandler none") ||
strings.Contains(lineLower, "sethandler default") {
continue
}
// Skip AddHandler for standard CGI extensions only (.cgi, .pl)
if strings.Contains(lineLower, "addhandler") {
// Only flag if mapping non-standard extensions
standardCGI := true
hasNonStandard := false
// Check each extension on the line
for _, ext := range []string{".haxor", ".cgix", ".phtml", ".php3",
".php5", ".suspected", ".bak.php", ".shtml", ".sh"} {
if strings.Contains(lineLower, ext) {
hasNonStandard = true
break
}
}
// If line only has .cgi and/or .pl, it's standard
if !hasNonStandard && standardCGI {
onlyStandard := true
parts := strings.Fields(lineLower)
for _, p := range parts {
if strings.HasPrefix(p, ".") && p != ".cgi" && p != ".pl" && p != ".py" &&
p != ".php" && p != ".jsp" && p != ".asp" {
// Has non-standard extension
onlyStandard = false
break
}
}
if onlyStandard {
continue
}
}
}
}
// Skip AddType for any MIME type (application/*, text/*, x-mapp-*, etc.)
if patternLower == "addtype" {
// AddType is only dangerous if it maps to a PHP/CGI handler
// Standard MIME type declarations are safe
if strings.Contains(lineLower, "application/") ||
strings.Contains(lineLower, "text/") ||
strings.Contains(lineLower, "image/") ||
strings.Contains(lineLower, "font/") ||
strings.Contains(lineLower, "x-mapp-") ||
strings.Contains(lineLower, "audio/") ||
strings.Contains(lineLower, "video/") {
continue
}
}
span := logical.span
if len(fields) > 0 && strings.EqualFold(fields[0], "RewriteRule") {
// Orphaned conditions would apply to the next surviving rule,
// changing unrelated site routing after the malicious rule goes.
for previous := lineIndex - 1; previous >= 0; previous-- {
line := strings.TrimSpace(logicalLines[previous].text)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if htaccessDirectiveName(line) != "rewritecond" {
break
}
span.Start = logicalLines[previous].span.Start
}
}
matches = append(matches, htaccessMatch{Range: span, Retain: pluginPrelude})
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "htaccess_injection",
Message: fmt.Sprintf("Suspicious .htaccess directive: %s", pattern),
Details: fmt.Sprintf("File: %s (line %d)\nContent: %s", path, lineNum+1, trimmed),
FilePath: path,
})
}
}
abuse, abuseMatches := htaccessHandlerAbuse(path, content)
return append(findings, abuse...), append(matches, abuseMatches...)
}
// htaccessHandlerAbuse flags an AddHandler mapping for an extension only an
// attacker uses. No site needs these mappings, so unlike the rest of the
// generic scanner its matches are cleaned automatically.
func htaccessHandlerAbuse(path string, content []byte) ([]alert.Finding, []htaccessMatch) {
var findings []alert.Finding
var matches []htaccessMatch
if strings.Contains(strings.ToLower(string(content)), "addhandler") {
cgiDisabled := cgiExecutionNeutralized(content)
nextLine := 0
for _, logical := range htaccessLogicalByteLines(content) {
lineNum := nextLine
end := logical.span.End
if end < len(content) {
end++
}
nextLine += strings.Count(string(content[logical.span.Start:end]), "\n")
line := logical.text
if strings.HasPrefix(strings.TrimSpace(line), "#") {
continue
}
fields, valid := parseApacheDirectiveFields(stripApacheComment(line))
if !valid || len(fields) < 3 || !strings.EqualFold(fields[0], "AddHandler") {
continue
}
// Disabling CGI does not disable PHP or unrelated custom handlers.
if cgiDisabled && handlerIsCGI(fields[1]) {
continue
}
extensions:
for _, token := range fields[2:] {
ext := "." + strings.TrimPrefix(strings.ToLower(token), ".")
switch ext {
case ".haxor", ".cgix", ".suspected", ".bak.php":
matches = append(matches, htaccessMatch{Range: logical.span})
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "htaccess_handler_abuse",
Message: fmt.Sprintf("Malicious handler mapping for %s extension", ext),
Details: fmt.Sprintf("File: %s (line %d)\nContent: %s", path, lineNum+1, strings.TrimSpace(line)),
FilePath: path,
})
break extensions
}
}
}
}
return findings, matches
}
package checks
import "strings"
func htaccessLineGrantsAccess(line string) bool {
fields, valid := parseApacheDirectiveFields(line)
if !valid || len(fields) < 3 {
return false
}
if strings.EqualFold(fields[0], "Allow") && strings.EqualFold(fields[1], "from") {
for _, host := range fields[2:] {
if strings.EqualFold(host, "all") {
return true
}
}
}
return len(fields) == 3 && strings.EqualFold(fields[0], "Require") && strings.EqualFold(fields[1], "all") && strings.EqualFold(fields[2], "granted")
}
func htaccessBodyGrantsAccess(body []byte) bool {
for _, line := range htaccessLogicalByteLines(body) {
if htaccessLineGrantsAccess(line.text) {
return true
}
}
return false
}
// detectRewriteOffShield recognizes the kit's empty grant together with its
// rewrite switch. Logical lines keep comments and continuations from becoming
// directives, and retain the original byte spans for surgical removal.
func detectRewriteOffShield(content []byte, _ string) []htaccessMatch {
type container struct {
name string
argument string
span htaccessByteRange
switches int
}
var stack []container
var grants, switches []htaccessMatch
for _, line := range htaccessLogicalByteLines(content) {
text := strings.TrimSpace(line.text)
if text == "" || strings.HasPrefix(text, "#") {
continue
}
tag, isTag, valid := parseApacheContainerTag(text)
if isTag {
if !valid {
return nil
}
name := strings.ToLower(tag.name)
if !tag.closing {
stack = append(stack, container{name, apacheContainerArgument(text), line.span, len(switches)})
continue
}
if len(stack) == 0 || stack[len(stack)-1].name != name {
return nil
}
open := stack[len(stack)-1]
stack = stack[:len(stack)-1]
span := htaccessByteRange{Start: open.span.Start, End: line.span.End}
if name == "filesmatch" && filesMatchNamesNoFile(open.argument) &&
htaccessBodyGrantsAccess(content[open.span.End:line.span.Start]) {
grants = append(grants, htaccessMatch{Range: span, Excerpt: trimExcerpt(content, span.Start, span.End)})
}
// Remove a wrapper only when the switch is its entire body. Other
// settings and comments inside a shared wrapper must survive.
if name == "ifmodule" && (strings.EqualFold(open.argument, "mod_rewrite.c") || strings.EqualFold(open.argument, "rewrite_module")) &&
len(switches) == open.switches+1 {
m := &switches[open.switches]
if strings.TrimSpace(string(content[open.span.End:m.Range.Start])) == "" &&
strings.TrimSpace(string(content[m.Range.End:line.span.Start])) == "" {
m.Range = span
m.Excerpt = trimExcerpt(content, span.Start, span.End)
}
}
continue
}
fields, valid := parseApacheDirectiveFields(text)
if valid && len(fields) == 2 && strings.EqualFold(fields[0], "RewriteEngine") && strings.EqualFold(fields[1], "off") {
switches = append(switches, htaccessMatch{Range: line.span, Excerpt: trimExcerpt(content, line.span.Start, line.span.End)})
}
}
if len(stack) != 0 || len(grants) == 0 || len(switches) == 0 {
return nil
}
return append(switches, grants...)
}
package checks
import (
"path/filepath"
"regexp"
"strings"
)
// unwrapOuterGroup returns the inside of pattern when the whole pattern is
// one parenthesised group, so its alternatives can be judged one by one.
func unwrapOuterGroup(pattern string) (string, bool) {
if len(pattern) < 2 || pattern[0] != '(' || pattern[len(pattern)-1] != ')' {
return pattern, false
}
depth := 0
for i := 0; i < len(pattern); i++ {
switch pattern[i] {
case '\\':
i++
case '(':
depth++
case ')':
depth--
if depth == 0 && i != len(pattern)-1 {
// The first group closes before the end: not one outer group.
return pattern, false
}
}
}
if depth != 0 {
return pattern, false
}
return pattern[1 : len(pattern)-1], true
}
// regexAlternativeNamesSomething reports whether one regex alternative
// carries a literal name character. Escaped sequences (`\w`, `\.`) and
// character classes are skipped: they describe shapes, not names.
func regexAlternativeNamesSomething(alt string) bool {
inClass := false
for i := 0; i < len(alt); i++ {
c := alt[i]
switch {
case c == '\\':
i++
case inClass:
if c == ']' {
inClass = false
}
case c == '[':
inClass = true
case (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-':
return true
}
}
return false
}
// filesMatchNamesNoFile reports whether a FilesMatch pattern can match only
// the empty name: anchored at both ends with nothing but grouping,
// alternation and repetition between the anchors. No request names such a
// file, so a grant under it grants nothing.
func filesMatchNamesNoFile(pattern string) bool {
if len(pattern) < 2 || pattern[0] != '^' || pattern[len(pattern)-1] != '$' {
return false
}
// Only grouping, alternation and repetition may appear between the
// anchors. A colon outside a non-capturing group names a real file.
body := strings.ReplaceAll(pattern[1:len(pattern)-1], "(?:", "(")
if strings.Trim(body, "()|?*+") != "" {
return false
}
// Alternation can detach an anchor: ^()|()$ also matches nonempty
// names. With consuming atoms excluded above, these two probes prove
// that the valid expression accepts only the empty string.
re, err := regexp.Compile(pattern)
return err == nil && re.MatchString("") && !re.MatchString("x")
}
// uploadTreeDirNames are directory names under which web-writable content
// lands. Findings there are judged without the sibling-PHP gate.
var uploadTreeDirNames = map[string]bool{
"uploads": true, "upload": true, "cache": true, "tmp": true, "temp": true,
"files": true, "media": true, "attachments": true, "images": true,
}
// htaccessInUploadTree reports whether the .htaccess at path sits under an
// upload-style directory. The top-level directory (/home, /var, /tmp) is
// never a docroot-relative upload dir and is skipped, so a system temp root
// does not make every path below it an upload tree.
func htaccessInUploadTree(path string) bool {
for _, part := range webTreeComponents(filepath.Dir(path)) {
if uploadTreeDirNames[strings.ToLower(part)] {
return true
}
}
return false
}
// docrootMarkerNames are directory names that begin a web tree on hosts
// whose layout is not one of the configured account roots.
var docrootMarkerNames = map[string]bool{
"public_html": true, "www": true, "htdocs": true, "httpdocs": true,
"web": true, "html": true, "public": true,
}
// webTreeComponents returns the directory components that lie inside the
// account's web tree: everything below <root>/<account> when dir is under
// an account root, otherwise everything below the first document-root
// marker. The prefix above the web tree (a temp root, /var/www, an account
// literally named tmp) never counts: it is not attacker-writable content.
func webTreeComponents(dir string) []string {
clean := filepath.ToSlash(filepath.Clean(dir))
if root, account, ok := accountRootOf(clean); ok {
rel := strings.TrimPrefix(clean, filepath.ToSlash(filepath.Join(root, account))+"/")
return strings.Split(rel, "/")
}
parts := strings.Split(strings.Trim(clean, "/"), "/")
for i, part := range parts {
if docrootMarkerNames[strings.ToLower(part)] {
return parts[i+1:]
}
}
return nil
}
// HTTP abuse detection.
//
// This file holds the access-log line parser, the per-scan aggregator
// struct (domlogStats), the UA classifier, the bot-classifier interface,
// and the static allowlist classifier that consults embedded bot IP ranges.
package checks
import (
"errors"
"fmt"
"net"
"net/netip"
"net/url"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/clientaddr"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/threatintel"
)
// accessLogRecord is the parsed shape of one access-log line. Combined
// Log Format plus the cPanel final-vhost extension:
//
// IP - - [time] "METHOD URI PROTO" status bytes "referer" "ua" "vhost"
//
// The parser tolerates either the 9-field plain CLF or the 10-field
// cPanel variant. Bad lines return ok=false; callers must skip them.
type accessLogRecord struct {
RemoteIP string
Time time.Time
Method string
URI string
// fullURI preserves the request target for file attribution before display truncation.
fullURI string
Status int
UserAgent string
XFF string // optional; only trusted when RemoteIP is a trusted proxy
// XFFPartial reports that the log kept only the right end of XFF.
XFFPartial bool
Domain string // vhost the line came from (per-domain domlog); empty for the central log
Account string // cPanel account owning Domain; empty when unknown/non-cPanel
// Central is set only for a recognized central access log on cPanel.
Central bool
// proxyPath is classified before the URI is truncated for aggregation.
proxyPath bool
// ref is the log record the line was read from; zero when the caller
// read no file.
ref syslogRecord
}
// uaKind is the User-Agent classification produced by classifyUA and
// consumed by domlogStats.scan and the http_ua_spoof emit logic.
type uaKind int
const (
uaKindBrowser uaKind = iota
uaKindClaimedBot
uaKindClaimedBotNegative
// uaKindClaimedBotPending is a claimed-bot UA whose rDNS verification has
// not resolved yet (cache miss, async job in flight). It is neither trusted
// (would have returned early via IsVerifiedBot) nor a confirmed spoof. Such
// an IP's flood/scanner abuse is routed to the reversible
// http_claimed_bot_unverified check instead of a hard block, so a real
// crawler is not blocked during the verification window.
uaKindClaimedBotPending
uaKindKnownScanner
uaKindWPSpoofPingback
uaKindScriptingLang
uaKindHeadless
uaKindEmpty
)
// botClassifier decides whether the source IP is a known verified bot
// the detector should NOT count or flag. Returns true to skip. The
// real implementation lives in internal/threatintel; tests use the
// nopBotClassifier below.
type botClassifier interface {
IsVerifiedBot(ip string, ua string) bool
}
type confirmedNegativeBotClassifier interface {
ConfirmedNegative(ip, ua string) bool
}
type pendingBotVerificationClassifier interface {
VerificationPending(ip, ua string) bool
}
type nopBotClassifier struct{}
func (nopBotClassifier) IsVerifiedBot(string, string) bool { return false }
// httpSample is one representative request kept per IP for forensic
// display in the finding Details field. First-seen wins; subsequent
// requests increment counters only.
type httpSample struct {
Method string
URI string
UA string
}
// domlogStats is the per-scan aggregator. One instance per CheckWPBruteForce
// invocation. Every counter is map[ip] -> int so emit can produce findings
// per source IP without a second pass.
type domlogStats struct {
wpLogin map[string]int
xmlrpc map[string]int
userEnum map[string]int
httpReqs map[string]int
uaCat map[string]map[uaKind]int
samples map[string]httpSample
// domains tracks the set of distinct vhosts each IP touched, so the
// per-IP aggregate findings can report cross-site spread (one IP
// scanning many vhosts on a shared host). Empty-domain records (the
// central access log) do not contribute.
domains map[string]map[string]struct{}
// abuseDomains tracks the in-window vhosts where each source made the
// request shape for a specific HTTP-abuse finding. The distributed
// rollup uses this instead of all in-window touches so an IP that was
// abusive elsewhere cannot make a normal hit count against a vhost.
abuseDomains map[string]map[string]map[string]struct{}
// scannerErr counts in-window non-asset requests per IP whose status
// matched the configured probe-error set. scannerReqs is the matching
// denominator, so the two counters must be gated by the same window
// check and the same display-asset exclusion.
scannerErr map[string]int
// scannerReqs is the per-IP scanner-profile denominator: in-window
// requests excluding static display assets (see isDisplayAssetProbe),
// so broken images, styles, scripts, and fonts neither dilute the
// error rate nor pad the volume gate. Separate from httpReqs so the
// flood detector's denominator is unchanged.
scannerReqs map[string]int
// scannerPaths is the distinct error-status path set per IP, query
// strings stripped, capped at httpScannerMaxTrackedPaths to bound
// memory under a flood of unique probe URLs.
scannerPaths map[string]map[string]struct{}
// scannerSamples keeps the first error-status request per IP for the
// finding Details field; the generic samples map can hold a 200 hit.
scannerSamples map[string]httpSample
// scannerDomainReqs and friends track the scanner profile per vhost so
// the distributed rollup only attributes a source to domains where the
// scanner shape actually crossed the detector gates. scannerDomainReqs is
// flat-keyed by ip+"\x00"+domain so a busy host's tens of thousands of
// one-off client IPs do not each allocate a nested map that the gates only
// read for the few IPs with errors.
scannerDomainReqs map[string]int
scannerDomainErr map[string]map[string]int
scanTime time.Time
// asnCrawl accumulates per-(scope, ASN) crawl fingerprints. Populated by
// observeASNCrawl from scan(); read by emitASNCrawl. Lazily allocated.
asnCrawl map[string]*asnCrawlScope
// newest is each address's newest record that counted toward a finding,
// by finding check (or User-Agent kind), so the finding names it.
newest map[string]map[string]scanRecord
// forwarded marks, by the same key, the addresses a forwarded header
// supplied for any counted record: such a count is not the HTTP peer's
// and reads as a trusted chain.
forwarded map[string]map[string]bool
// seenDirect marks the addresses any record named as its own connecting
// peer, whatever it counted toward: such an address is a packet source.
seenDirect map[string]bool
// callers collects requests for flagged files; nil when none is flagged.
callers *webshellCallers
// trusted is the trust list scan resolves clients against, parsed from
// cfg once per scan like the thresholds below.
trusted clientaddr.Trusted
trustedSet bool
// Scanner thresholds are derived from cfg once per scan -- cfg is stable
// across a single domlogStats lifetime -- instead of on every parsed record.
scannerThreshComputed bool
scannerMinReq int
scannerPct int
scannerMinPaths int
scannerEnabled bool
// passes holds when each address last passed a challenge, read once per
// scan.
passes map[string]time.Time
passesSet bool
passCutoff time.Time
}
func newDomlogStats() *domlogStats {
return newDomlogStatsAt(time.Now())
}
func newDomlogStatsAt(t time.Time) *domlogStats {
return &domlogStats{
wpLogin: make(map[string]int),
xmlrpc: make(map[string]int),
userEnum: make(map[string]int),
httpReqs: make(map[string]int),
uaCat: make(map[string]map[uaKind]int),
samples: make(map[string]httpSample),
domains: make(map[string]map[string]struct{}),
abuseDomains: make(map[string]map[string]map[string]struct{}),
scannerErr: make(map[string]int),
scannerReqs: make(map[string]int),
scannerPaths: make(map[string]map[string]struct{}),
scannerSamples: make(map[string]httpSample),
scannerDomainReqs: make(map[string]int),
scannerDomainErr: make(map[string]map[string]int),
newest: make(map[string]map[string]scanRecord),
scanTime: t,
}
}
// uaObservationKey names the newest-record slot of a User-Agent kind.
func uaObservationKey(kind uaKind) string { return "ua:" + itoa(int(kind)) }
// markForwarded notes, for key, a counted record whose address a forwarded
// header supplied. Every record that decides a finding is marked, whether
// or not it can be the finding's observation.
func (s *domlogStats) markForwarded(key, ip string, direct bool) {
if direct {
return
}
if s.forwarded == nil {
s.forwarded = make(map[string]map[string]bool)
}
if s.forwarded[key] == nil {
s.forwarded[key] = make(map[string]bool)
}
s.forwarded[key][ip] = true
}
// observe keeps r as ip's newest record for key, and notes a record whose
// address a forwarded header supplied.
func (s *domlogStats) observe(key, ip string, r scanRecord, direct bool) {
s.markForwarded(key, ip, direct)
if r.at.IsZero() {
return
}
if s.newest == nil {
s.newest = make(map[string]map[string]scanRecord)
}
m := s.newest[key]
if m == nil {
m = make(map[string]scanRecord)
s.newest[key] = m
}
keepNewest(m, ip, r, s.scanTime)
}
// stamp names the newest record of f's address kept under key as f's
// observation, read as the HTTP peer only when every counted record was and
// as a trusted chain otherwise.
func (s *domlogStats) stamp(f *alert.Finding, key string) {
r := s.newest[key][f.SourceIP]
r.stamp(f, ProducerDomlogScan)
// A forwarded client keeps its kind without a record. It is a proxied
// client only when no record in the scan came from the address itself.
if s.forwarded[key][f.SourceIP] {
f.Attribution = admission.AttributionTrustedHTTPChain
f.ProxiedClient = !s.seenDirect[f.SourceIP]
}
}
// trustedFor returns cfg's trusted proxies, parsed on the first record.
func (s *domlogStats) trustedFor(cfg *config.Config) clientaddr.Trusted {
if !s.trustedSet {
s.trusted, s.trustedSet = config.TrustedProxies(cfg), true
}
return s.trusted
}
// answeredBefore reports whether a line was answered by a pass or is older
// than the retained pass history. Unknown timestamps still count.
func (s *domlogStats) answeredBefore(ip string, at time.Time) bool {
if !s.passesSet {
s.passes, s.passCutoff = challengePasses()
s.passesSet = true
}
if at.IsZero() {
return false
}
// While passes are tracked, evidence cannot outlive them: otherwise a
// quiet site's next request revives the lines an expired pass answered.
if !s.passCutoff.IsZero() && !at.After(s.passCutoff) {
return true
}
passed, ok := s.passes[ip]
return ok && !at.After(passed)
}
// scan updates counters for one parsed record. cfg may be nil
// (parity tests pass nil); otherwise thresholds come from it. bot is
// consulted before any count so a verified Googlebot does not contribute
// to either legacy or new metrics.
func (s *domlogStats) scan(rec accessLogRecord, cfg *config.Config, bot botClassifier) {
// A per-vhost domlog never names a proxied request by path.
if rec.Central && rec.proxyPath {
return
}
// A record no client can be resolved for counts against nobody: a
// trusted proxy's own address names a shared transport (L6).
client, err := recordClient(rec, s.trustedFor(cfg))
if err != nil || client.Client.IsLoopback() || client.Client.IsUnspecified() {
return
}
ip := client.Client.String()
infra := cfg != nil && isInfraIP(ip, cfg.InfraIPs)
verifiedBot := !infra && bot != nil && bot.IsVerifiedBot(ip, rec.UserAgent)
// Exempt clients still show that a URL is public, but never get caller findings.
s.callers.record(rec, ip, s.scanTime, !infra && !verifiedBot)
if infra {
return
}
if verifiedBot {
removeChallengeIP(ip)
return
}
// The record the counters below name. Its address is the HTTP peer only
// when no trusted proxy chain supplied it; stamp reads a count with any
// chained record as a trusted chain.
direct := !client.Chained
if direct {
if s.seenDirect == nil {
s.seenDirect = make(map[string]bool)
}
s.seenDirect[ip] = true
}
var r scanRecord
if rec.ref.stream != "" {
r = scanned(rec.ref, rec.Time)
if direct {
r.kind = admission.AttributionHTTPPeer
}
}
// These counts read each site's last log lines, which can be days old;
// lines from before an address passed a challenge are what it answered.
answered := s.answeredBefore(ip, rec.Time)
wpLoginHit := rec.Method == "POST" && strings.Contains(rec.URI, "wp-login.php")
xmlrpcHit := rec.Method == "POST" && strings.Contains(rec.URI, "xmlrpc.php")
userEnumHit := false
if rec.Method == "POST" && !answered {
if wpLoginHit {
s.wpLogin[ip]++
s.observe("wp_login_bruteforce", ip, r, direct)
}
if xmlrpcHit {
s.xmlrpc[ip]++
s.observe("xmlrpc_abuse", ip, r, direct)
}
}
if !answered && strings.Contains(rec.URI, "?author=") {
s.userEnum[ip]++
userEnumHit = true
} else if !answered && strings.Contains(rec.URI, "/wp-json/wp/v2/users") &&
!strings.Contains(rec.URI, "/users/me") {
s.userEnum[ip]++
userEnumHit = true
}
if userEnumHit {
s.observe("wp_user_enumeration", ip, r, direct)
}
// Rate and UA counters only fire for requests that fall inside the
// flood window. Malformed timestamp lines still feed the legacy POST
// counters above but not rate or UA findings.
if withinHTTPFloodWindow(rec.Time, cfg, s.scanTime) {
if rec.Domain != "" {
set := s.domains[ip]
if set == nil {
set = make(map[string]struct{})
s.domains[ip] = set
}
set[rec.Domain] = struct{}{}
if wpLoginHit && !answered {
s.recordAbuseDomain("wp_login_bruteforce", ip, rec.Domain)
}
if xmlrpcHit && !answered {
s.recordAbuseDomain("xmlrpc_abuse", ip, rec.Domain)
}
if userEnumHit {
s.recordAbuseDomain("wp_user_enumeration", ip, rec.Domain)
}
s.recordAbuseDomain("http_request_flood", ip, rec.Domain)
}
if _, ok := s.samples[ip]; !ok {
s.samples[ip] = httpSample{Method: rec.Method, URI: rec.URI, UA: rec.UserAgent}
}
s.httpReqs[ip]++
s.observe("http_request_flood", ip, r, direct)
// Every in-window request decides a pending claimed bot: its
// majority and its flood or scanner gate, whatever its User-Agent.
s.markForwarded(uaObservationKey(uaKindClaimedBotPending), ip, direct)
_, _, _, scannerEnabled := s.scannerThresholds(cfg)
// Static display assets (images, styles, scripts, fonts, media)
// are excluded from the scanner profile entirely: a 404 on a
// browser sub-resource is a broken-asset signal, not URL
// enumeration. Dropping them from both the numerator (scannerErr)
// and the denominator (scannerReqs) stops a site with a missing
// CDN from making ordinary visitors look like scanners, without
// blinding the profile to real probes in the same window.
if scannerEnabled && !isDisplayAssetProbe(rec.URI) {
s.scannerReqs[ip]++
// The whole denominator decides the profile, not only errors.
s.markForwarded("http_scanner_profile", ip, direct)
if rec.Domain != "" {
s.recordScannerDomainRequest(ip, rec.Domain)
}
if isScannerErrorStatus(rec.Status, cfg.Thresholds.HTTPScannerStatusCodes) {
path := probePath(rec.URI)
s.scannerErr[ip]++
s.observe("http_scanner_profile", ip, r, direct)
if _, ok := s.scannerSamples[ip]; !ok {
s.scannerSamples[ip] = httpSample{Method: rec.Method, URI: rec.URI, UA: rec.UserAgent}
}
paths := s.scannerPaths[ip]
if paths == nil {
paths = make(map[string]struct{})
s.scannerPaths[ip] = paths
}
if len(paths) < httpScannerMaxTrackedPaths {
paths[path] = struct{}{}
}
if rec.Domain != "" {
s.recordScannerDomainError(ip, rec.Domain)
}
}
}
kind := classifyUA(rec.UserAgent, rec.Method)
if kind == uaKindClaimedBot {
// Static allowlist hits returned early through IsVerifiedBot
// above. For IPs outside the static range, check whether the
// async rDNS verifier has confirmed a negative result. Only
// promote to uaKindClaimedBotNegative when the cache has a
// definitive negative. The reversible pending-bot route is only
// safe when an active verifier can resolve the cache miss later;
// static-only/no-verifier paths keep the old flood/scanner
// behavior instead of softening spoofed bot UAs forever.
if cv, ok := bot.(confirmedNegativeBotClassifier); ok && cv.ConfirmedNegative(ip, rec.UserAgent) {
kind = uaKindClaimedBotNegative
} else if pv, ok := bot.(pendingBotVerificationClassifier); ok && pv.VerificationPending(ip, rec.UserAgent) {
// Verification still pending: not a confirmed spoof, so do not
// flag ua_spoof. Tracked separately so a flood/scan crawl from
// this IP routes to the reversible challenge, not a hard block.
kind = uaKindClaimedBotPending
} else {
kind = uaKindBrowser
}
}
if _, ok := s.uaCat[ip]; !ok {
s.uaCat[ip] = make(map[uaKind]int)
}
s.uaCat[ip][kind]++
s.observe(uaObservationKey(kind), ip, r, direct)
if rec.Domain != "" && kind != uaKindBrowser && kind != uaKindClaimedBotPending {
s.recordAbuseDomain("http_ua_spoof", ip, rec.Domain)
}
}
// http_asn_crawl uses its OWN lookback window (default 60 min), which is
// wider than the flood window (default 5 min), so it must gate on its own
// window OUTSIDE the flood block; nesting it would cap the detector at the
// flood window. ip is already infra/verified-bot
// filtered and proxy-resolved above, so it is valid here.
if asnCrawlWithinWindow(rec.Time, cfg, s.scanTime) {
s.observeASNCrawl(ip, rec, cfg)
}
}
func (s *domlogStats) recordAbuseDomain(check, ip, domain string) {
byIP := s.abuseDomains[check]
if byIP == nil {
byIP = make(map[string]map[string]struct{})
s.abuseDomains[check] = byIP
}
set := byIP[ip]
if set == nil {
set = make(map[string]struct{})
byIP[ip] = set
}
set[domain] = struct{}{}
}
// scannerDomainKey joins an IP and a domain into the flat scannerDomainReqs
// key. The NUL separator cannot appear in either part, so distinct (ip,domain)
// pairs never collide.
func scannerDomainKey(ip, domain string) string {
return ip + "\x00" + domain
}
// scannerThresholds returns the scanner-profile gates derived from cfg, caching
// them on first use. cfg does not change across a single scan, so this avoids
// re-deriving the thresholds on every parsed access-log record.
func (s *domlogStats) scannerThresholds(cfg *config.Config) (minReq, pct, minPaths int, ok bool) {
if !s.scannerThreshComputed {
s.scannerMinReq, s.scannerPct, s.scannerMinPaths, s.scannerEnabled = scannerProfileThresholds(cfg)
s.scannerThreshComputed = true
}
return s.scannerMinReq, s.scannerPct, s.scannerMinPaths, s.scannerEnabled
}
func (s *domlogStats) recordScannerDomainRequest(ip, domain string) {
s.scannerDomainReqs[scannerDomainKey(ip, domain)]++
}
func (s *domlogStats) recordScannerDomainError(ip, domain string) {
byDomain := s.scannerDomainErr[ip]
if byDomain == nil {
byDomain = make(map[string]int)
s.scannerDomainErr[ip] = byDomain
}
byDomain[domain]++
}
// emitLegacy returns the three pre-existing finding kinds. Kept
// separate from the new emit() (Tasks 3/4) so the parity test can
// assert "no new findings yet".
func (s *domlogStats) emitLegacy(cfg *config.Config) []alert.Finding {
var out []alert.Finding
// xmlrpc_threshold is operator-tunable; <= 0 disables the check entirely.
// Fall back to the package default only when called without a config (tests).
xmlrpcThr := xmlrpcThreshold
if cfg != nil {
xmlrpcThr = cfg.Thresholds.XMLRPCThreshold
}
for ip, count := range s.wpLogin {
if count >= wpLoginThreshold {
f := alert.Finding{
Severity: alert.Critical,
Check: "wp_login_bruteforce",
SourceIP: ip,
Message: formatLegacyMessage("WordPress login brute force", ip, count, "attempts"),
Details: "Aggregated across per-vhost access logs",
}
s.stamp(&f, f.Check)
out = append(out, f)
}
}
for ip, count := range s.xmlrpc {
if xmlrpcThr > 0 && count >= xmlrpcThr {
f := alert.Finding{
Severity: alert.Critical,
Check: "xmlrpc_abuse",
SourceIP: ip,
Message: formatLegacyMessage("XML-RPC abuse", ip, count, "requests"),
Details: "Aggregated across per-vhost access logs",
}
s.stamp(&f, f.Check)
out = append(out, f)
}
}
for ip, count := range s.userEnum {
if count >= 5 {
f := alert.Finding{
Severity: alert.High,
Check: "wp_user_enumeration",
SourceIP: ip,
Message: formatLegacyMessage("WordPress user enumeration", ip, count, "requests"),
Details: "Requests to /wp-json/wp/v2/users or ?author=",
}
s.stamp(&f, f.Check)
out = append(out, f)
}
}
return out
}
// emit produces all finding kinds from a single populated domlogStats.
// Legacy three kinds come first (so existing callers still get them when
// running through emit), then http_request_flood, then http_ua_spoof.
func (s *domlogStats) emit(cfg *config.Config) []alert.Finding {
out := s.emitLegacy(cfg)
out = append(out, s.callers.emit()...)
if cfg != nil && cfg.Thresholds.HTTPFloodThreshold > 0 {
for ip, count := range s.httpReqs {
if count < cfg.Thresholds.HTTPFloodThreshold {
continue
}
if s.isPendingClaimedBot(ip) {
// Routed to http_claimed_bot_unverified (challenge) below so a
// real crawler mid-verification is not hard-blocked.
continue
}
sample := s.samples[ip]
f := alert.Finding{
Severity: alert.High,
Check: "http_request_flood",
SourceIP: ip,
Domain: s.singleDomain(ip),
Message: "HTTP request flood from " + ip + ": " + itoa(count) + " requests" + s.vhostSuffix(ip),
Details: "Sample: " + sample.Method + " " + sample.URI + " UA=" + truncate(sample.UA, 120),
}
s.stamp(&f, f.Check)
out = append(out, f)
}
}
if cfg != nil {
threshold := cfg.Thresholds.HTTPUASpoofThreshold
if threshold <= 0 {
threshold = 30
}
for ip, byKind := range s.uaCat {
// Require sustained spoofing before a hard block. A single
// rDNS-failed bot-UA request is too often a legitimate residential
// or mobile client that happens to send a bot-like User-Agent;
// gating on the same threshold as the other spoof kinds keeps real
// spoof-crawlers (which make many requests) blocked.
if hits, ok := byKind[uaKindClaimedBotNegative]; ok && hits >= threshold {
out = append(out, s.makeUASpoofFinding(ip, uaKindClaimedBotNegative,
"claimed search-engine bot failed rDNS verification",
s.samples[ip], hits))
continue
}
if hits, ok := byKind[uaKindKnownScanner]; ok && hits > 0 {
out = append(out, s.makeUASpoofFinding(ip, uaKindKnownScanner, "known scanner UA",
s.samples[ip], hits))
continue
}
if hits := byKind[uaKindWPSpoofPingback]; hits >= threshold {
out = append(out, s.makeUASpoofFinding(ip, uaKindWPSpoofPingback,
"WordPress/<ver> UA on GET (pingback spoof)",
s.samples[ip], hits))
continue
}
if cfg.Thresholds.HTTPUAScriptingEnabled {
if hits := byKind[uaKindScriptingLang]; hits >= threshold {
out = append(out, s.makeUASpoofFinding(ip, uaKindScriptingLang,
"scripting-language UA (curl/python/etc.)",
s.samples[ip], hits))
continue
}
}
if cfg.Thresholds.HTTPUAHeadlessEnabled {
if hits := byKind[uaKindHeadless]; hits >= threshold {
out = append(out, s.makeUASpoofFinding(ip, uaKindHeadless,
"headless browser UA", s.samples[ip], hits))
continue
}
}
if cfg.Thresholds.HTTPUAEmptyEnabled {
if hits := byKind[uaKindEmpty]; hits >= threshold {
out = append(out, s.makeUASpoofFinding(ip, uaKindEmpty,
"empty/dash User-Agent", s.samples[ip], hits))
continue
}
}
}
}
out = append(out, s.emitScannerProfile(cfg)...)
out = append(out, s.emitClaimedBotUnverified(cfg)...)
// Distributed attack: many distinct abusive IPs hitting one vhost.
// Built from the per-IP findings already emitted above, so only IPs
// that crossed an abuse threshold count -- a popular site's normal
// visitor spread never trips it.
out = append(out, s.emitDistributedFlood(cfg, out)...)
// Domain is set only from a per-vhost log path, which server
// configuration names, so it is a domain claim. A stamped finding
// claims none: the vhosts an address touched change with the window and
// would change the evidence of an unchanged record, and the newest
// record's vhost is the client's choice. Its evidence stays host-owned.
for i := range out {
if out[i].Domain != "" && out[i].Observation == (alert.Observation{}) {
out[i].Claims = []admission.Claim{{Kind: admission.ClaimDomain, Value: out[i].Domain}}
}
}
out = append(out, s.emitASNCrawl(cfg)...)
return out
}
// httpScannerMaxTrackedPaths bounds the per-IP distinct probe-path set.
// A scanner cycling unique URLs past the cap keeps incrementing the
// error counter, and any sane min_distinct_paths threshold sits far
// below the cap, so detection quality does not depend on growth beyond it.
const httpScannerMaxTrackedPaths = config.HTTPScannerMaxDistinctPaths
// emitScannerProfile produces http_scanner_profile findings: source IPs
// whose in-window traffic is almost entirely probe-error responses spread
// across many distinct paths -- the shape of URL enumeration for
// downloadable files, exposed backups, and dormant shells. Volume,
// error-rate, and path-breadth gates must all pass so that dead
// bookmarks, broken assets, and site migrations stay out of scope.
func (s *domlogStats) emitScannerProfile(cfg *config.Config) []alert.Finding {
minReq, pct, minPaths, ok := s.scannerThresholds(cfg)
if !ok {
return nil
}
var out []alert.Finding
for ip, errs := range s.scannerErr {
total := s.scannerReqs[ip]
paths := len(s.scannerPaths[ip])
if !scannerProfilePasses(total, errs, paths, minReq, pct, minPaths) {
continue
}
if s.isPendingClaimedBot(ip) {
// Routed to http_claimed_bot_unverified (challenge) instead of a
// hard scanner-profile block while verification is pending.
continue
}
for _, domain := range s.scannerProfileDomains(ip, minReq, pct) {
s.recordAbuseDomain("http_scanner_profile", ip, domain)
}
sample := s.scannerSamples[ip]
f := alert.Finding{
Severity: alert.High,
Check: "http_scanner_profile",
SourceIP: ip,
Domain: s.singleDomain(ip),
Message: "URL scanner profile from " + ip + ": " + itoa(errs) + " of " + itoa(total) +
" requests hit probe-error statuses across " + itoa(paths) + " distinct paths" + s.vhostSuffix(ip),
Details: "Sample: " + sample.Method + " " + sample.URI + " UA=" + truncate(sample.UA, 120),
}
s.stamp(&f, f.Check)
// Display assets can defeat a pending-bot majority even though
// they do not count toward the scanner profile itself.
if s.uaCat[ip][uaKindClaimedBotPending] > 0 && s.forwarded[uaObservationKey(uaKindClaimedBotPending)][ip] {
f.Attribution = admission.AttributionTrustedHTTPChain
f.ProxiedClient = !s.seenDirect[ip]
}
out = append(out, f)
}
return out
}
// isPendingClaimedBot reports whether ip's in-window traffic is dominated by a
// claimed-bot UA whose rDNS verification has not resolved. The claimed-bot
// requests must be a strict majority so an attacker cannot downgrade a hard
// block to the softer challenge route by mixing in bot-UA requests.
func (s *domlogStats) isPendingClaimedBot(ip string) bool {
pending := s.uaCat[ip][uaKindClaimedBotPending]
if pending == 0 {
return false
}
return pending*2 > s.httpReqs[ip]
}
// emitClaimedBotUnverified emits one http_claimed_bot_unverified finding per
// pending-claimed-bot IP whose flood or scanner-profile volume crossed a hard
// threshold. The check is challengeable (routes to the PoW gate when challenge
// is enabled) and blockable (hard-blocked when it is not), so a real crawler
// mid-verification clears itself while a spoofer that cannot solve the
// challenge stays blocked.
func (s *domlogStats) emitClaimedBotUnverified(cfg *config.Config) []alert.Finding {
if cfg == nil {
return nil
}
minReq, pct, minPaths, scannerOK := s.scannerThresholds(cfg)
floodThreshold := cfg.Thresholds.HTTPFloodThreshold
var out []alert.Finding
for ip := range s.uaCat {
if !s.isPendingClaimedBot(ip) {
continue
}
total := s.httpReqs[ip]
flood := floodThreshold > 0 && total >= floodThreshold
scanner := scannerOK && scannerProfilePasses(s.scannerReqs[ip], s.scannerErr[ip], len(s.scannerPaths[ip]), minReq, pct, minPaths)
if !flood && !scanner {
continue
}
reason := "request flood"
sample := s.samples[ip]
if scanner && !flood {
reason = "scanner-profile crawl"
sample = s.scannerSamples[ip]
}
s.recordClaimedBotUnverifiedDomains(ip, flood, scanner, minReq, pct)
f := alert.Finding{
Severity: alert.High,
Check: "http_claimed_bot_unverified",
SourceIP: ip,
Domain: s.singleDomain(ip),
Message: "Unverified claimed bot from " + ip + ": " + reason + " (" + itoa(total) +
" requests, rDNS not yet confirmed)" + s.vhostSuffix(ip),
Details: "Sample: " + sample.Method + " " + sample.URI + " UA=" + truncate(sample.UA, 120),
}
s.stamp(&f, uaObservationKey(uaKindClaimedBotPending))
out = append(out, f)
}
return out
}
func (s *domlogStats) recordClaimedBotUnverifiedDomains(ip string, flood, scanner bool, minReq, pct int) {
if flood {
for domain := range s.abuseDomains["http_request_flood"][ip] {
s.recordAbuseDomain("http_claimed_bot_unverified", ip, domain)
}
}
if scanner {
for _, domain := range s.scannerProfileDomains(ip, minReq, pct) {
s.recordAbuseDomain("http_claimed_bot_unverified", ip, domain)
}
}
}
func scannerProfileThresholds(cfg *config.Config) (minReq, pct, minPaths int, ok bool) {
if cfg == nil {
return 0, 0, 0, false
}
minReq = cfg.Thresholds.HTTPScannerMinRequests
if minReq <= 0 {
return 0, 0, 0, false
}
pct = cfg.Thresholds.HTTPScannerErrorPct
if pct <= 0 {
pct = config.DefaultHTTPScannerErrorPct
}
if pct > 100 {
pct = 100
}
minPaths = cfg.Thresholds.HTTPScannerMinDistinctPaths
if minPaths <= 0 {
minPaths = config.DefaultHTTPScannerMinDistinctPaths
}
if minPaths > httpScannerMaxTrackedPaths {
minPaths = httpScannerMaxTrackedPaths
}
return minReq, pct, minPaths, true
}
func scannerProfilePasses(total, errs, paths, minReq, pct, minPaths int) bool {
if total < minReq {
return false
}
if errs*100 < total*pct {
return false
}
return paths >= minPaths
}
// scannerDomainDefaultMinErrors is the small absolute floor of probe-error
// hits a single vhost must receive before a confirmed per-IP scanner is
// attributed to it for the distributed rollup. The per-IP gates (minReq,
// minPaths) already proved the source is a scanner; per vhost we only require
// a few errors plus the same error-rate gate, so a scanner spread thin across
// many vhosts still feeds the rollup while the default scanner thresholds do
// not attribute a vhost that caught one incidental 404.
const scannerDomainDefaultMinErrors = 3
func scannerDomainErrorFloor(minReq int) int {
if minReq <= 0 {
return scannerDomainDefaultMinErrors
}
if minReq < scannerDomainDefaultMinErrors {
return minReq
}
return scannerDomainDefaultMinErrors
}
// scannerProfileDomains returns the vhosts a confirmed scanner IP should be
// attributed to in the distributed rollup. The full per-IP minimum-request and
// min-path gates are deliberately NOT reused per vhost: a scanner that sprays a
// few probes across many vhosts trips the per-IP profile in aggregate but lands
// only a handful of hits on each vhost, so requiring the per-IP minimums per
// vhost would drop it from the rollup. Instead each vhost needs a small
// absolute error floor and the same error-rate gate.
func (s *domlogStats) scannerProfileDomains(ip string, minReq, pct int) []string {
errsByDomain := s.scannerDomainErr[ip]
if len(errsByDomain) == 0 {
return nil
}
var domains []string
errorFloor := scannerDomainErrorFloor(minReq)
for domain, errs := range errsByDomain {
total := s.scannerDomainReqs[scannerDomainKey(ip, domain)]
if scannerDomainAttributes(total, errs, pct, errorFloor) {
domains = append(domains, domain)
}
}
sort.Strings(domains)
return domains
}
func scannerDomainAttributes(total, errs, pct, minErrors int) bool {
if errs < minErrors {
return false
}
return errs*100 >= total*pct
}
// isScannerErrorStatus reports whether status belongs to the configured
// probe-error set. Linear scan: the set is a handful of entries, so this
// beats building a lookup map on a per-record hot path.
func isScannerErrorStatus(status int, codes []int) bool {
if status == 0 {
return false
}
if len(codes) == 0 {
return status == 404 || status == 403
}
for _, c := range codes {
if status == c {
return true
}
}
return false
}
// probePath strips the query string and fragment so cache-buster style
// queries on one missing endpoint count as a single probe path.
func probePath(uri string) string {
if i := strings.IndexAny(uri, "?#"); i >= 0 {
uri = uri[:i]
}
if uri == "" {
return "/"
}
return uri
}
// displayAssetExts are static, browser-rendered sub-resource extensions.
// A 404 on one of these is a broken-asset signal -- the static-file
// handler looked for a file and did not find it, executing no code and
// disclosing nothing -- not URL enumeration. Excluding them keeps a site
// whose CDN is missing its images, styles, or scripts from making every
// ordinary visitor look like a scanner. Archives, code, configs, dumps,
// and extensionless paths -- the targets a real scanner enumerates -- are
// deliberately absent and keep counting toward the profile.
var displayAssetExts = map[string]struct{}{
".gif": {}, ".jpg": {}, ".jpeg": {}, ".png": {}, ".webp": {}, ".bmp": {},
".svg": {}, ".ico": {}, ".avif": {}, ".tif": {}, ".tiff": {},
".css": {}, ".js": {}, ".mjs": {}, ".cjs": {},
".woff": {}, ".woff2": {}, ".ttf": {}, ".eot": {}, ".otf": {},
".mp4": {}, ".webm": {}, ".ogg": {}, ".mp3": {}, ".wav": {},
".m4a": {}, ".mov": {}, ".avi": {}, ".flac": {},
}
// scannerProbeExts are extensions commonly used when probing for code,
// configs, source maps, dumps, archives, and backups. If one appears before a
// final display-asset suffix (for example shell.php.gif), the request still
// has scanner shape and must not be hidden by the asset exclusion.
var scannerProbeExts = map[string]struct{}{
".php": {}, ".phtml": {}, ".phps": {}, ".php3": {}, ".php4": {}, ".php5": {}, ".php7": {},
".asp": {}, ".aspx": {}, ".ashx": {}, ".asmx": {}, ".jsp": {}, ".jspx": {}, ".cfm": {},
".cgi": {}, ".pl": {}, ".py": {}, ".rb": {}, ".sh": {}, ".bash": {}, ".zsh": {},
".env": {}, ".conf": {}, ".config": {}, ".ini": {}, ".yaml": {}, ".yml": {}, ".json": {}, ".xml": {},
".sql": {}, ".dump": {}, ".db": {}, ".sqlite": {}, ".sqlite3": {}, ".log": {},
".zip": {}, ".rar": {}, ".7z": {}, ".tar": {}, ".tgz": {}, ".gz": {}, ".bz2": {}, ".xz": {},
".bak": {}, ".backup": {}, ".old": {}, ".orig": {}, ".save": {}, ".swp": {}, ".tmp": {},
".map": {},
}
// isDisplayAssetProbe reports whether uri's final path segment ends in one
// of displayAssetExts. The query string and fragment are stripped first so
// a cache-buster suffix cannot hide the extension.
func isDisplayAssetProbe(uri string) bool {
path := probePath(uri)
if i := strings.LastIndexByte(path, '/'); i >= 0 {
path = path[i+1:]
}
if unescaped, err := url.PathUnescape(path); err == nil {
path = unescaped
}
dot := strings.LastIndexByte(path, '.')
if dot <= 0 {
return false
}
if _, ok := displayAssetExts[strings.ToLower(path[dot:])]; !ok {
return false
}
return !hasEmbeddedScannerProbeExt(path[:dot])
}
func hasEmbeddedScannerProbeExt(pathPrefix string) bool {
parts := strings.Split(strings.ToLower(pathPrefix), ".")
for _, part := range parts[1:] {
if part == "" {
continue
}
if _, ok := scannerProbeExts["."+part]; ok {
return true
}
}
return false
}
// httpAbuseChecks are the per-IP finding kinds that mark a source IP as
// abusive for the distributed-attack rollup.
var httpAbuseChecks = map[string]struct{}{
"wp_login_bruteforce": {},
"xmlrpc_abuse": {},
"wp_user_enumeration": {},
"http_request_flood": {},
"http_claimed_bot_unverified": {},
"http_ua_spoof": {},
"http_scanner_profile": {},
}
// emitDistributedFlood rolls the per-IP HTTP-abuse findings up per vhost:
// when at least HTTPDistributedMinIPs distinct abusive IPs hit one vhost
// in this window, it emits a single http_distributed_flood finding for
// that vhost. The per-IP findings still stand; this adds the
// many-IPs-one-target view (botnet / distributed brute-force) that the
// per-IP and per-source-IP-spray paths cannot see. Disabled when the
// threshold is <= 0.
func (s *domlogStats) emitDistributedFlood(cfg *config.Config, prior []alert.Finding) []alert.Finding {
if cfg == nil {
return nil
}
minIPs := cfg.Thresholds.HTTPDistributedMinIPs
if minIPs <= 0 {
return nil
}
domainIPs := map[string]map[string]struct{}{}
for _, f := range prior {
if _, ok := httpAbuseChecks[f.Check]; !ok || f.SourceIP == "" {
continue
}
for dom := range s.abuseDomains[f.Check][f.SourceIP] {
if domainIPs[dom] == nil {
domainIPs[dom] = map[string]struct{}{}
}
domainIPs[dom][f.SourceIP] = struct{}{}
}
}
domains := make([]string, 0, len(domainIPs))
for dom := range domainIPs {
domains = append(domains, dom)
}
sort.Strings(domains)
var out []alert.Finding
for _, dom := range domains {
n := len(domainIPs[dom])
if n < minIPs {
continue
}
out = append(out, alert.Finding{
Severity: alert.High,
Check: "http_distributed_flood",
Domain: dom,
Message: fmt.Sprintf("Distributed HTTP attack on %s: %d distinct abusive source IPs", dom, n),
Details: fmt.Sprintf("%d source IPs each tripped an HTTP-abuse threshold against %s in this window. "+
"Likely a botnet or distributed brute-force; consider edge rate-limiting or a challenge.", n, dom),
Timestamp: time.Now(),
})
}
return out
}
func (s *domlogStats) makeUASpoofFinding(ip string, kind uaKind, reason string, sample httpSample, hits int) alert.Finding {
f := alert.Finding{
Severity: alert.Critical,
Check: "http_ua_spoof",
SourceIP: ip,
Domain: s.singleDomain(ip),
Message: "User-Agent spoof from " + ip + ": " + reason + s.vhostSuffix(ip),
Details: "Hits: " + itoa(hits) + ", Sample: " + sample.Method + " " +
sample.URI + " UA=" + truncate(sample.UA, 200),
}
s.stamp(&f, uaObservationKey(kind))
return f
}
// singleDomain returns the one vhost ip touched, or "" when zero or more
// than one (the per-IP aggregate spans several vhosts, so no single domain
// attributes the finding). vhostSuffix renders the cross-site spread for
// the operator-facing message.
func (s *domlogStats) singleDomain(ip string) string {
set := s.domains[ip]
if len(set) != 1 {
return ""
}
for d := range set {
return d
}
return ""
}
func (s *domlogStats) vhostSuffix(ip string) string {
if n := len(s.domains[ip]); n > 1 {
return " across " + itoa(n) + " vhosts"
}
return ""
}
// withinHTTPFloodWindow reports whether a log timestamp falls inside the
// configured flood rate window relative to the scan start time. Timestamps
// in the future (up to one clock-skew minute) are accepted. Zero timestamps
// from malformed lines return false so they do not contribute to rate counts.
func withinHTTPFloodWindow(ts time.Time, cfg *config.Config, now time.Time) bool {
if ts.IsZero() || cfg == nil {
return false
}
windowMin := cfg.Thresholds.HTTPFloodWindowMin
if windowMin <= 0 {
windowMin = 5
}
cutoff := now.Add(-time.Duration(windowMin) * time.Minute)
return !ts.Before(cutoff) && !ts.After(now.Add(time.Minute))
}
func formatLegacyMessage(kind, ip string, n int, unit string) string {
return kind + " from " + ip + ": " + itoa(n) + " " + unit
}
func itoa(n int) string {
if n == 0 {
return "0"
}
var buf [20]byte
pos := len(buf)
neg := n < 0
if neg {
n = -n
}
for n > 0 {
pos--
buf[pos] = byte('0' + n%10)
n /= 10
}
if neg {
pos--
buf[pos] = '-'
}
return string(buf[pos:])
}
const accessLogURILimit = 4096
// parseAccessLogRecord parses one Combined Log Format line into an
// accessLogRecord. It does NOT use strings.Fields because quoted
// request/referer/user-agent fields can contain spaces.
//
// Format:
//
// <ip> <ident> <user> [<time>] "<method> <uri> <proto>" <status> <bytes> "<ref>" "<ua>" ["<vhost>"]
//
// Returns ok=false for any line that cannot be parsed. Never panics.
func parseAccessLogRecord(line string) (accessLogRecord, bool) {
return parseAccessLogRecordWithURILimit(line, accessLogURILimit)
}
// Callers grouping by request target must retain its full identity rather than
// grouping by the truncated URI used by the general traffic summaries.
func parseAccessLogRecordWithURILimit(line string, maxURILen int) (accessLogRecord, bool) {
const maxUALen = 512
var rec accessLogRecord
// IP is everything up to the first space.
sp := strings.IndexByte(line, ' ')
if sp <= 0 {
return rec, false
}
rec.RemoteIP = line[:sp]
timeStr, rest, ok := patternSplitHeader(line[sp+1:])
if !ok {
return rec, false
}
// time format: 02/Jan/2006:15:04:05 -0700
t, err := time.Parse("02/Jan/2006:15:04:05 -0700", timeStr)
if err == nil {
rec.Time = t
}
// Request quoted field.
request, rest, ok := patternQuotedField(rest)
if !ok {
return rec, false
}
parts := strings.SplitN(request, " ", 3)
if len(parts) >= 1 {
rec.Method = parts[0]
}
if len(parts) >= 2 {
uri, _ := patternDecodeField(parts[1], len(parts[1]))
rec.proxyPath = IsProxiedPanelRequest(uri, true)
if len(uri) > maxURILen {
uri = uri[:maxURILen]
}
rec.URI = uri
}
// status (skip leading spaces)
rest = strings.TrimLeft(rest, " ")
end := strings.IndexByte(rest, ' ')
if end > 0 {
rec.Status = atoiSafe(rest[:end])
rest = rest[end+1:]
}
// bytes field -- skip leading spaces then advance past the token
rest = strings.TrimLeft(rest, " ")
end = strings.IndexByte(rest, ' ')
if end > 0 {
rest = rest[end+1:]
} else {
// no more fields after bytes
return rec, true
}
// referer quoted field (skipped).
_, rest, ok = patternQuotedField(rest)
if !ok {
// Optional client fields cannot invalidate a server-written request.
return rec, true
}
// UA quoted field.
ua, rest, ok := patternQuotedField(rest)
if !ok {
return rec, true
}
rec.UserAgent, _ = patternDecodeField(ua, maxUALen)
// Supported proxy formats put XFF after UA or after a cPanel host:port.
// Later extensions are client text, not a second source of attribution.
extra, rest, ok := patternQuotedField(rest)
if !ok {
return rec, true
}
extra, _ = patternDecodeField(extra, len(extra))
if patternVhostExtension(extra) {
extra, _, ok = patternQuotedField(rest)
if !ok {
return rec, true
}
extra, _ = patternDecodeField(extra, len(extra))
}
rec.XFF, rec.XFFPartial, _ = patternXFFSuffix(extra)
return rec, true
}
func atoiSafe(s string) int {
n := 0
for i := 0; i < len(s); i++ {
c := s[i]
if c < '0' || c > '9' {
break
}
n = n*10 + int(c-'0')
}
return n
}
// errListedClient refuses a resolved address that is itself a listed proxy.
var errListedClient = errors.New("the resolved address is a listed proxy")
// recordClient resolves rec's client: the direct peer, or the client a
// trusted proxy chain names. A trusted peer whose chain resolves
// to nobody is refused, never read as the client, and so is a listed address
// whose header is not trusted (an address of this host, or any listed address
// while the interface set cannot be read).
func recordClient(rec accessLogRecord, trusted clientaddr.Trusted) (clientaddr.Result, error) {
res, err := clientaddr.Resolve(rec.RemoteIP, rec.XFF, !rec.XFFPartial, trusted)
if err != nil {
return clientaddr.Result{}, err
}
if trusted.Covers(res.Client) {
return clientaddr.Result{}, errListedClient
}
return res, nil
}
// IsTrustedProxy reports whether addr, a bare address as a log writes a
// peer, is listed in cfg's one trusted proxy list: web_server.trusted_proxies
// and challenge.trusted_proxies together. A listed address stands for every
// visitor behind it even where its header is not trusted (an address of this
// host, or any while the interface set cannot be read), so callers never
// count, escalate or sight it. A spelling with a port, brackets or a zone is
// not an address.
func IsTrustedProxy(addr string, cfg *config.Config) bool {
a, err := netip.ParseAddr(strings.TrimSpace(addr))
return err == nil && a.Zone() == "" && config.TrustedProxies(cfg).Covers(a)
}
// classifyUA maps a User-Agent string to a uaKind. Matching is performed
// on a lower-cased copy of the UA because scanner and impersonation tools
// routinely vary case. Precedence order: scanner
// signatures win over claimed-bot, claimed-bot wins over headless, etc.
func classifyUA(ua, method string) uaKind {
const maxUALen = 512
if len(ua) > maxUALen {
ua = ua[:maxUALen]
}
if ua == "" || ua == "-" {
return uaKindEmpty
}
low := strings.ToLower(ua)
for _, s := range knownScannerSubstrings {
if strings.Contains(low, s) {
return uaKindKnownScanner
}
}
// WordPress pingback UA on a GET request is illegal: legitimate
// pingback clients always POST. A GET with this UA is a content
// scraper or probe spoofing the pingback agent.
if method == "GET" && strings.HasPrefix(low, "wordpress/") {
return uaKindWPSpoofPingback
}
for _, s := range claimedBotSubstrings {
if strings.Contains(low, s) {
return uaKindClaimedBot
}
}
// Operator-configured bots (reputation.verified_bots) classify as
// claimed bots too, so an impostor reusing the UA is caught as a spoof.
if threatintel.OperatorBotFromUA(low) != "" {
return uaKindClaimedBot
}
for _, s := range headlessSubstrings {
if strings.Contains(low, s) {
return uaKindHeadless
}
}
for _, s := range scriptingSubstrings {
if strings.Contains(low, s) {
return uaKindScriptingLang
}
}
return uaKindBrowser
}
var (
knownScannerSubstrings = []string{
"nikto", "sqlmap", "acunetix", "nmap ", "masscan", "wpscan",
"nuclei", "dirbuster", "gobuster", "feroxbuster",
}
claimedBotSubstrings = []string{
"googlebot", "bingbot", "applebot", "duckduckbot", "yandexbot",
"baiduspider", "facebookexternalhit", "twitterbot",
// Appendix A bots plus AI crawlers verified by published ranges.
"amazonbot", "gptbot", "chatgpt-user", "oai-searchbot",
"claudebot", "claude-user", "claude-searchbot",
"perplexitybot", "meta-externalagent", "meta-webindexer", "bravebot",
"seranking",
}
headlessSubstrings = []string{
"headlesschrome", "phantomjs", "puppeteer", "playwright",
}
scriptingSubstrings = []string{
"python-requests/", "curl/", "go-http-client/", "java/", "wget/",
"libwww-perl/", "node-fetch/",
}
)
// staticAllowlistClassifier consults only DNS-free bot verification: embedded
// static IP ranges and operator-configured IP ranges. rDNS verification for
// IPs outside those ranges is handled by verifyingClassifier.
type staticAllowlistClassifier struct{}
func (staticAllowlistClassifier) IsVerifiedBot(ipStr, ua string) bool {
bot := threatintel.ClaimedBotFromUA(ua)
if bot == "" {
return false
}
ip := net.ParseIP(ipStr)
if ip == nil {
return false
}
if threatintel.DefaultRanges().IPInBot(ip, bot) {
return true
}
return threatintel.OperatorBotIPVerified(bot, ip)
}
// verifyingClassifier consults the static allowlist first, then the
// rDNS verify cache. Cache misses enqueue an async job and return
// false (treat as unverified for this scan cycle).
type verifyingClassifier struct {
async *threatintel.AsyncBotVerifier
cacheGet func(net.IP, string) (bool, bool)
}
func newVerifyingClassifier(async *threatintel.AsyncBotVerifier,
cacheGet func(net.IP, string) (bool, bool)) verifyingClassifier {
return verifyingClassifier{async: async, cacheGet: cacheGet}
}
func (c verifyingClassifier) IsVerifiedBot(ipStr, ua string) bool {
bot := threatintel.ClaimedBotFromUA(ua)
if bot == "" {
return false
}
ip := net.ParseIP(ipStr)
if ip == nil {
return false
}
// Static range is the fast positive path.
if threatintel.DefaultRanges().IPInBot(ip, bot) {
return true
}
// Operator IP-range bots (AI agents) verify synchronously, no rDNS.
if threatintel.OperatorBotIPVerified(bot, ip) {
return true
}
// Cache lookup: valid positive -> verified, valid negative -> false
// (ConfirmedNegative handles it), no entry -> enqueue and fail open.
if c.cacheGet != nil {
if verified, valid := c.cacheGet(ip, bot); valid {
return verified
}
}
// Enqueue async verification only when a later scan can read the result;
// otherwise there is no bounded pending window to route through challenge.
if c.async != nil && c.cacheGet != nil {
c.async.Enqueue(ip, bot)
}
return false
}
// ConfirmedNegative reports whether the rDNS cache has a definitive
// negative result for this IP+UA pair. Called from scan() to decide
// whether to promote uaKindClaimedBot to uaKindClaimedBotNegative.
func (c verifyingClassifier) ConfirmedNegative(ipStr, ua string) bool {
bot := threatintel.ClaimedBotFromUA(ua)
if bot == "" {
return false
}
ip := net.ParseIP(ipStr)
if ip == nil || c.cacheGet == nil {
return false
}
verified, valid := c.cacheGet(ip, bot)
return valid && !verified
}
func (c verifyingClassifier) VerificationPending(ipStr, ua string) bool {
bot := threatintel.ClaimedBotFromUA(ua)
if bot == "" {
return false
}
ip := net.ParseIP(ipStr)
if ip == nil || c.async == nil || c.cacheGet == nil {
return false
}
if threatintel.DefaultRanges().IPInBot(ip, bot) || threatintel.OperatorBotIPVerified(bot, ip) {
return false
}
if _, valid := c.cacheGet(ip, bot); valid {
return false
}
return c.async.Pending(ip, bot)
}
var (
globalBotVerifier *threatintel.AsyncBotVerifier
globalBotGet func(net.IP, string) (bool, bool)
botMu sync.RWMutex
)
// SetBotVerifier installs the daemon-lifetime async verifier and cache
// reader. Called from daemon.go after the store and goroutine are ready.
func SetBotVerifier(v *threatintel.AsyncBotVerifier, get func(net.IP, string) (bool, bool)) {
botMu.Lock()
defer botMu.Unlock()
globalBotVerifier = v
globalBotGet = get
}
// currentBotClassifier returns the appropriate botClassifier based on
// config. When bot_verify_enabled is false, falls back to the
// static-only classifier so DNS calls are never made.
func currentBotClassifier(cfg *config.Config) botClassifier {
if cfg == nil || !cfg.BotVerifyEnabled() {
return staticAllowlistClassifier{}
}
botMu.RLock()
defer botMu.RUnlock()
return newVerifyingClassifier(globalBotVerifier, globalBotGet)
}
// Package checks: http_asn_crawl detector — single-ASN distributed crawl of
// uncacheable URLs saturating one account's PHP pool.
package checks
import (
"fmt"
"net"
"net/url"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/crawlid"
)
// Keep one extension heuristic for both crawl classifiers.
var httpASNCrawlStaticExts = crawlid.StaticExtensions
// httpASNCrawlExpensive selects GET/HEAD requests with nonempty queries and
// non-static raw extensions. This heuristic does not prove backend cost.
func httpASNCrawlExpensive(rec accessLogRecord) bool {
if rec.Method != "GET" && rec.Method != "HEAD" {
return false
}
p, q, ok := crawlid.SplitTarget(rec.URI)
if !ok || q == "" {
return false
}
_, isStatic := httpASNCrawlStaticExts[crawlid.PathExtension(p)]
return !isStatic
}
// httpASNCrawlAmplifyKeys are query parameter names that signal an
// expensive layered-nav/search request; their presence raises severity.
var httpASNCrawlAmplifyKeys = map[string]struct{}{
"orderby": {}, "add-to-cart": {}, "s": {}, "paged": {}, "product-page": {},
}
// httpASNCrawlAmplified reports whether the URI's query carries a known
// expensive layered-nav/search parameter. Key names are matched
// case-insensitively; values alone never match.
func httpASNCrawlAmplified(uri string) bool {
q := strings.IndexByte(uri, '?')
if q < 0 {
return false
}
for raw := range strings.SplitSeq(uri[q+1:], "&") {
key, _, _ := strings.Cut(raw, "=")
key, err := url.QueryUnescape(key)
if err != nil {
continue
}
lk := strings.ToLower(key)
if strings.HasPrefix(lk, "filter_") || strings.HasPrefix(lk, "query_type_") {
return true
}
if _, ok := httpASNCrawlAmplifyKeys[lk]; ok {
return true
}
}
return false
}
// asnCrawlASN accumulates one ASN's footprint within one scope.
type asnCrawlASN struct {
org string
expensive int
total int
amplified int
domains map[string]struct{}
samples []string // up to 5 distinct expensive URIs (<=200 bytes)
ips map[string]struct{} // distinct source IPs, capped
ipsCapped bool
cidr24 map[string]int // observed /24 (v4) or /64 (v6) -> count
}
func (a *asnCrawlASN) distinctIPs() int {
// saturated at cap; callers may render as ">=cap" at emit
return len(a.ips)
}
// asnCrawlScope is one account-or-domain scope's per-ASN map plus the
// scope-wide expensive denominator (excludes reverse-proxy traffic).
type asnCrawlScope struct {
byASN map[uint]*asnCrawlASN
scopeExpensive int
}
func (s *domlogStats) observeASNCrawl(ip string, rec accessLogRecord, cfg *config.Config) {
if cfg == nil || cfg.Thresholds.HTTPASNCrawlMinIPs <= 0 {
return // detector disabled
}
if rec.Domain == "" {
// Central access logs (and malformed paths) carry no domain/account,
// so they cannot be PHP-pool correlated and must not feed this
// detector. Per-domain domlogs always set Domain;
// without this guard they key a degenerate catch-all "domain:" scope
// and double-count the share denominator.
return
}
if !httpASNCrawlExpensive(rec) {
return
}
lookup := CurrentASNLookup()
if lookup == nil {
return
}
asn, org := lookup(ip)
if asn == 0 {
// Unknown ASN: still counts in the scope denominator so known-ASN
// share is not inflated, but cannot form its own fingerprint.
s.asnCrawlScopeFor(rec).scopeExpensive++
return
}
if uintInSlice(asn, cfg.Thresholds.HTTPASNCrawlReverseProxyASNs) {
return // reverse-proxy: dropped from numerator AND denominator
}
// v1 limitation: real-client attribution behind a
// reverse proxy/CDN depends on the domlog resolver restoring the client
// from XFF, which only happens for a peer listed in
// web_server.trusted_proxies or challenge.trusted_proxies. Operators
// fronting hosts with a CDN MUST list it so the real client IP (and its
// ASN) is attributed here; without it a CDN-fronted
// distributed crawl is seen as proxy-ASN traffic and not flagged (a false
// negative). The catastrophic false positive is still prevented because the
// proxy ASN is never emitted or tempbanned. A general CDN-aware fix spans
// all http-abuse detectors and is out of scope for this detector.
scope := s.asnCrawlScopeFor(rec)
scope.scopeExpensive++
a := scope.byASN[asn]
if a == nil {
a = &asnCrawlASN{
org: org,
domains: map[string]struct{}{},
ips: map[string]struct{}{},
cidr24: map[string]int{},
}
scope.byASN[asn] = a
}
a.total++
a.expensive++
if httpASNCrawlAmplified(rec.URI) {
a.amplified++
}
if rec.Domain != "" {
a.domains[rec.Domain] = struct{}{}
}
if len(a.samples) < 5 {
a.samples = append(a.samples, truncate(rec.URI, 200))
}
if _, ok := a.ips[ip]; !ok {
maxIPs := cfg.Thresholds.HTTPASNCrawlMaxTrackedIPs
if maxIPs <= 0 {
maxIPs = config.DefaultHTTPASNCrawlMaxTrackedIPs
}
if len(a.ips) < maxIPs {
a.ips[ip] = struct{}{}
if g := asnCrawlGroupCIDR(ip); g != "" {
a.cidr24[g]++
}
} else {
a.ipsCapped = true
}
}
}
func (s *domlogStats) asnCrawlScopeFor(rec accessLogRecord) *asnCrawlScope {
if s.asnCrawl == nil {
s.asnCrawl = map[string]*asnCrawlScope{}
}
key := rec.Account
if key == "" {
key = "domain:" + rec.Domain
}
sc := s.asnCrawl[key]
if sc == nil {
sc = &asnCrawlScope{byASN: map[uint]*asnCrawlASN{}}
s.asnCrawl[key] = sc
}
return sc
}
func uintInSlice(v uint, list []uint) bool {
for _, x := range list {
if x == v {
return true
}
}
return false
}
// asnCrawlGroupCIDR returns the /24 (IPv4) or /64 (IPv6) the ip belongs to.
func asnCrawlGroupCIDR(ip string) string {
p := net.ParseIP(ip)
if p == nil {
return ""
}
if v4 := p.To4(); v4 != nil {
return net.IP(v4.Mask(net.CIDRMask(24, 32))).String() + "/24"
}
return p.Mask(net.CIDRMask(64, 128)).String() + "/64"
}
// asnCrawlWithinWindow gates a record to the detector's own lookback window
// (thresholds.http_asn_crawl_window_min). A zero/absent timestamp is excluded,
// and future timestamps beyond a small clock-skew allowance are ignored.
func asnCrawlWithinWindow(ts time.Time, cfg *config.Config, now time.Time) bool {
if ts.IsZero() || cfg == nil {
return false
}
win := cfg.Thresholds.HTTPASNCrawlWindowMin
if win <= 0 {
win = config.DefaultHTTPASNCrawlWindowMin
}
cutoff := now.Add(-time.Duration(win) * time.Minute)
return !ts.Before(cutoff) && !ts.After(now.Add(time.Minute))
}
// phpWorkersByUserFn is a seam for testing; production code uses phpWorkersByUser.
var phpWorkersByUserFn = phpWorkersByUser
// asnCrawlSaturated reports whether account's live lsphp worker count meets or
// exceeds the saturation threshold. Returns false for empty account (domain-scoped
// findings never escalate) or when no positive threshold is configured.
func asnCrawlSaturated(account string, workers map[string][]string, cfg *config.Config) bool {
if account == "" {
return false
}
threshold := cfg.Thresholds.HTTPASNCrawlSaturation
if threshold <= 0 {
threshold = cfg.Performance.PHPProcessWarnPerUser
}
if threshold <= 0 {
return false
}
return len(workers[account]) >= threshold
}
// emitASNCrawl produces Warning/High findings for each (scope, ASN) pair that
// passes all stage-1 gates. When an accounted scope is PHP-pool saturated the
// severity is escalated to Critical (stage 2).
func (s *domlogStats) emitASNCrawl(cfg *config.Config) []alert.Finding {
if cfg == nil || cfg.Thresholds.HTTPASNCrawlMinIPs <= 0 || s.asnCrawl == nil {
return nil
}
th := cfg.Thresholds
minIPs := th.HTTPASNCrawlMinIPs
minExpensive := th.HTTPASNCrawlMinExpensive
if minExpensive <= 0 {
minExpensive = config.DefaultHTTPASNCrawlMinExpensive
}
minSharePct := th.HTTPASNCrawlMinSharePct
if minSharePct <= 0 {
minSharePct = config.DefaultHTTPASNCrawlMinSharePct
}
highAmpPct := th.HTTPASNCrawlHighAmpPct
if highAmpPct <= 0 {
highAmpPct = config.DefaultHTTPASNCrawlHighAmpPct
}
highVolMult := th.HTTPASNCrawlHighVolumeMult
if highVolMult <= 0 {
highVolMult = config.DefaultHTTPASNCrawlHighVolMult
}
var out []alert.Finding
var workers map[string][]string
workersLoaded := false
for scopeKey, scope := range s.asnCrawl {
for asn, a := range scope.byASN {
if uintInSlice(asn, th.HTTPASNCrawlAllowlistASNs) ||
uintInSlice(asn, th.HTTPASNCrawlReverseProxyASNs) {
continue
}
if a.distinctIPs() < minIPs {
continue
}
if a.expensive < minExpensive {
continue
}
if scope.scopeExpensive == 0 ||
a.expensive*100 < minSharePct*scope.scopeExpensive {
continue
}
sev := alert.Warning
ampHigh := a.expensive > 0 && a.amplified*100 >= highAmpPct*a.expensive
volHigh := a.expensive >= highVolMult*minExpensive
if ampHigh || volHigh {
sev = alert.High
}
account, domain := asnCrawlScopeParts(scopeKey, a)
if account != "" {
if !workersLoaded {
workers = phpWorkersByUserFn()
workersLoaded = true
}
if asnCrawlSaturated(account, workers, cfg) {
sev = alert.Critical
}
}
cidrs := dropFirewallAllowedCIDRs(collapseASNCrawlCIDRs(a, cfg), a)
out = append(out, alert.Finding{
Severity: sev,
Check: "http_asn_crawl",
TenantID: account,
Domain: domain,
Message: fmt.Sprintf("Distributed crawl from AS%d (%s) against %s", asn, a.org, asnCrawlScopeLabel(scopeKey)),
Details: asnCrawlDetails(asn, a, scope, cfg, cidrs),
CIDRs: cidrs,
Claims: asnCrawlClaims(a.domains),
Timestamp: time.Now(),
})
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Message < out[j].Message })
return out
}
// asnCrawlScopeParts returns (account, domain) for a finding. account is the
// scope key unless it is a domain-scoped fallback ("domain:<d>"); domain is set
// only when exactly one domain was observed.
func asnCrawlScopeParts(scopeKey string, a *asnCrawlASN) (account, domain string) {
if !strings.HasPrefix(scopeKey, "domain:") {
account = scopeKey
}
if len(a.domains) == 1 {
for d := range a.domains {
domain = d
}
}
return account, domain
}
func asnCrawlScopeLabel(scopeKey string) string {
return strings.TrimPrefix(scopeKey, "domain:")
}
func asnCrawlDetails(asn uint, a *asnCrawlASN, scope *asnCrawlScope, cfg *config.Config, cidrs []string) string {
maxTracked := cfg.Thresholds.HTTPASNCrawlMaxTrackedIPs
if maxTracked <= 0 {
maxTracked = config.DefaultHTTPASNCrawlMaxTrackedIPs
}
ipCount := fmt.Sprintf("%d", a.distinctIPs())
if a.ipsCapped {
ipCount = fmt.Sprintf(">=%d", maxTracked)
}
share := 0
if scope.scopeExpensive > 0 {
share = a.expensive * 100 / scope.scopeExpensive
}
var b strings.Builder
fmt.Fprintf(&b, "ASN: AS%d (%s)\n", asn, a.org)
fmt.Fprintf(&b, "Distinct source IPs: %s\n", ipCount)
fmt.Fprintf(&b, "Expensive/total reqs: %d/%d (%d%% of scope expensive)\n", a.expensive, a.total, share)
fmt.Fprintf(&b, "Amplified (layered-nav/search) reqs: %d\n", a.amplified)
if len(a.samples) > 0 {
fmt.Fprintf(&b, "Sample URIs: %s\n", strings.Join(a.samples, " | "))
}
if len(cidrs) > 0 {
fmt.Fprintf(&b, "Suggested subnets: %s\n", strings.Join(cidrs, ", "))
}
return b.String()
}
// collapseASNCrawlCIDRs reduces an ASN's observed IPs to a sorted, capped set
// of /24 (IPv4) / /64 (IPv6) subnets. When a single /16 covers >=
// http_asn_crawl_16_pref_pct of the IPv4 IPs across >= 4 distinct /24s, that
// /16 replaces its member /24s. Remaining /24s and any /64s are kept as-is.
// The result is sorted and capped to http_asn_crawl_max_prefix entries.
func collapseASNCrawlCIDRs(a *asnCrawlASN, cfg *config.Config) []string {
if len(a.cidr24) == 0 {
return nil
}
th := cfg.Thresholds
pct16 := th.HTTPASNCrawl16PrefPct
if pct16 <= 0 {
pct16 = config.DefaultHTTPASNCrawl16PrefPct
}
// Count total IPv4 IPs and group /24s by their /16 parent.
totalV4 := 0
bySixteen := map[string][]string{} // "x.y.0.0/16" -> member /24 CIDRs
for cidr, n := range a.cidr24 {
if strings.HasSuffix(cidr, "/24") {
totalV4 += n
sl := strings.SplitN(cidr, ".", 4)
s16 := sl[0] + "." + sl[1] + ".0.0/16"
bySixteen[s16] = append(bySixteen[s16], cidr)
}
}
chosen := map[string]struct{}{}
covered := map[string]struct{}{} // /24s replaced by their /16
// Promote a /16 when >= 4 member /24s cover >= pct16% of all IPv4 IPs.
for s16, members := range bySixteen {
if len(members) < 4 {
continue
}
count := 0
for _, m := range members {
count += a.cidr24[m]
}
if totalV4 > 0 && count*100 >= pct16*totalV4 {
chosen[s16] = struct{}{}
for _, m := range members {
covered[m] = struct{}{}
}
}
}
// Add uncovered /24s and all /64s.
for cidr := range a.cidr24 {
if _, done := covered[cidr]; done {
continue
}
chosen[cidr] = struct{}{}
}
out := make([]string, 0, len(chosen))
for c := range chosen {
out = append(out, c)
}
sort.Strings(out)
maxPrefix := th.HTTPASNCrawlMaxPrefix
if maxPrefix <= 0 {
maxPrefix = config.DefaultHTTPASNCrawlMaxPrefix
}
if len(out) > maxPrefix {
out = out[:maxPrefix]
}
return out
}
// dropFirewallAllowedCIDRs removes any candidate CIDR that contains an observed
// source IP already firewall-allowed (whitelisted). This
// only ever REMOVES subnets, so it is strictly more conservative and can never
// cause over-blocking. When no blocker is wired, or it does not implement
// allowChecker, all CIDRs are kept. Each allowed observed IP is tested against
// the (<=8) collapsed CIDRs only.
func dropFirewallAllowedCIDRs(cidrs []string, a *asnCrawlASN) []string {
if len(cidrs) == 0 {
return cidrs
}
ac, ok := getIPBlocker().(allowChecker)
if !ok {
return cidrs
}
drop := map[string]struct{}{}
for ipStr := range a.ips {
if !ac.IsAllowed(ipStr) {
continue
}
parsed := net.ParseIP(ipStr)
if parsed == nil {
continue
}
for _, cidr := range cidrs {
if _, dropped := drop[cidr]; dropped {
continue
}
_, ipnet, err := net.ParseCIDR(cidr)
if err != nil {
continue
}
if ipnet.Contains(parsed) {
drop[cidr] = struct{}{}
}
}
}
if len(drop) == 0 {
return cidrs
}
kept := make([]string, 0, len(cidrs))
for _, cidr := range cidrs {
if _, dropped := drop[cidr]; !dropped {
kept = append(kept, cidr)
}
}
return kept
}
// Preserve the vhost identities instead of promoting their inferred owner
// to an authenticated account. Admission verifies all domains together.
func asnCrawlClaims(observedDomains map[string]struct{}) []admission.Claim {
var domains []string
for domain := range observedDomains {
domains = append(domains, domain)
}
sort.Strings(domains)
var claims []admission.Claim
for _, domain := range domains {
claims = append(claims, admission.Claim{Kind: admission.ClaimDomain, Value: domain})
}
return claims
}
package checks
import (
"encoding/xml"
"errors"
"fmt"
"io"
"net/netip"
"strconv"
"strings"
"sync/atomic"
"github.com/pidginhost/csm/internal/platform"
)
// RewriteState says whether the web server may replace the connecting
// address it logs.
type RewriteState uint8
const (
// RewriteUnknown: not detected yet, or the configuration could not be
// read completely. Admission treats it as rewritten.
RewriteUnknown RewriteState = iota
// RewriteNone: the web logs record the connecting address.
RewriteNone
// RewriteActive: the web logs may record an address from a request
// header or PROXY protocol; this build reads no separate peer field.
RewriteActive
)
// HTTPRewrite is what the host's web server does to the address it logs.
// Directive and File name the first setting that rewrites; Reason says why
// the state is unknown.
type HTTPRewrite struct {
State RewriteState
Server platform.WebServer
Directive string
File string
Reason string
}
func (s RewriteState) String() string {
switch s {
case RewriteNone:
return "none"
case RewriteActive:
return "active"
}
return "unknown"
}
// httpRewrite is the state the daemon last read; nil until it reads one.
var httpRewrite atomic.Pointer[HTTPRewrite]
// SetHTTPRewrite publishes the web server state admission reads.
func SetHTTPRewrite(r HTTPRewrite) { httpRewrite.Store(&r) }
// CurrentHTTPRewrite is the published state, unknown before the first.
func CurrentHTTPRewrite() HTTPRewrite {
if r := httpRewrite.Load(); r != nil {
return *r
}
return HTTPRewrite{}
}
// lswsConfigPath is LiteSpeed's own server configuration.
const lswsConfigPath = platform.LiteSpeedConfigPath
// rewriteSources names the files a web server reads. apacheRequired marks
// a server that always loads an Apache tree, so a missing one is unknown.
type rewriteSources struct {
server platform.WebServer
apacheConfig string
apacheRequired bool
nginxConfig string
lswsConfig string
}
// DetectHTTPRewrite reads the host web server's configuration from disk: an
// Apache RemoteIPHeader or RemoteIPProxyProtocol, mod_cloudflare or
// mod_rpaf, an nginx set_real_ip_from, or LiteSpeed's client IP from
// header, in any scope; failing those, a log format that writes a request
// value where CSM reads the client.
func DetectHTTPRewrite(info platform.Info) HTTPRewrite {
src := rewriteSources{server: info.WebServer}
switch info.WebServer {
case platform.WSApache:
src.apacheConfig, src.apacheRequired = findWebServerConfigPath(info), true
case platform.WSLiteSpeed:
// cPanel LiteSpeed also applies the Apache tree it loads.
src.lswsConfig = lswsConfigPath
if info.IsCPanel() {
src.apacheConfig, src.apacheRequired = findWebServerConfigPath(info), true
}
case platform.WSNginx:
src.nginxConfig = joinWebConfigPath(info.NginxConfigRoot(), "nginx.conf")
}
return detectHTTPRewrite(src)
}
func detectHTTPRewrite(src rewriteSources) HTTPRewrite {
r := HTTPRewrite{Server: src.server}
switch src.server {
case platform.WSNone:
r.State = RewriteNone
return r
case platform.WSApache, platform.WSLiteSpeed, platform.WSNginx:
default:
r.Reason = fmt.Sprintf("web server %q is not one CSM reads", src.server)
return r
}
unknown := ""
if src.lswsConfig != "" {
directive, err := lswsClientAddressRewrite(src.lswsConfig)
switch {
case directive != "":
r.State, r.Directive, r.File = RewriteActive, directive, src.lswsConfig
return r
case err != nil:
unknown = err.Error()
}
}
if src.apacheConfig != "" {
directive, file, complete := apacheClientAddressRewrite(src.apacheConfig, src.server == platform.WSLiteSpeed)
if directive != "" {
r.State, r.Directive, r.File = RewriteActive, directive, file
return r
}
if !complete && unknown == "" {
unknown = "the Apache configuration " + src.apacheConfig + " could not be read completely"
}
} else if src.apacheRequired && unknown == "" {
unknown = "no Apache configuration file was found"
}
if src.nginxConfig != "" {
directive, file, complete := nginxClientAddressRewrite(src.nginxConfig)
if directive != "" {
r.State, r.Directive, r.File = RewriteActive, directive, file
return r
}
if !complete {
unknown = "the nginx configuration " + src.nginxConfig + " could not be read completely"
}
}
if unknown != "" {
r.Reason = unknown
return r
}
r.State = RewriteNone
return r
}
// apacheClientAddressRewrite returns the first directive that replaces the
// client address, its file, and whether the whole tree was read: a
// mod_remoteip header or PROXY protocol, a loaded or configured
// mod_cloudflare, or an enabled mod_rpaf (0.6 or 0.8 names); failing
// those, a log format that writes a request value where CSM reads the
// client. A rewrite or format directive Apache would refuse leaves the
// tree unknown.
func apacheClientAddressRewrite(path string, liteSpeed bool) (directive, file string, complete bool) {
lines, complete := assembleApacheConfigWithStatus(path)
malformed := false
logs := apacheLogFormats{liteSpeed: liteSpeed}
valid := walkApacheLines(lines, func(line apacheConfigLine, fields []string, ctx apacheLineContext) {
if directive != "" {
return
}
fields, resolved := apacheMacroFields(fields, ctx)
if !resolved {
// A macro parameter can name any directive.
malformed = true
return
}
logs.add(line, fields, ctx)
is := func(names ...string) bool {
for _, n := range names {
if strings.EqualFold(fields[0], n) {
return true
}
}
return false
}
switch {
case is("RemoteIPHeader", "CloudFlareRemoteIPHeader"):
if len(fields) != 2 || is("CloudFlareRemoteIPHeader") && fields[1] == "" {
malformed = true
return
}
directive, file = strings.Join(fields, " "), line.File
case is("CloudFlareRemoteIPTrustedProxy"):
if len(fields) < 2 || fields[1] == "" {
malformed = true
return
}
for _, proxy := range fields[1:] {
// Apache's ITERATE handler stops at an empty quoted argument.
if proxy == "" {
break
}
if !apacheCloudflareProxyKnown(proxy) {
malformed = true
return
}
}
directive, file = strings.Join(fields, " "), line.File
case is("DenyAllButCloudFlare"):
if len(fields) != 1 {
malformed = true
return
}
// This also implies mod_cloudflare's default address rewrite.
directive, file = strings.Join(fields, " "), line.File
case is("LoadModule"):
// Only Apache can resolve its Define and environment values.
if len(fields) != 3 || fields[1] == "" || fields[2] == "" || apacheArgumentHasVariable(fields[1]) {
malformed = true
return
}
if fields[1] == "cloudflare_module" {
// mod_cloudflare trusts its built-in Cloudflare ranges once loaded.
directive, file = strings.Join(fields, " "), line.File
}
case is("RemoteIPProxyProtocol", "RPAF_Enable", "RPAFenable"):
switch {
case len(fields) == 2 && strings.EqualFold(fields[1], "On"):
directive, file = strings.Join(fields, " "), line.File
case len(fields) != 2 || !strings.EqualFold(fields[1], "Off"):
malformed = true
}
}
})
if directive == "" {
if f := logs.clientValue(); f != nil {
directive, file = f.directive, f.file
}
}
return directive, file, complete && valid && !malformed && !logs.malformed
}
// apacheCloudflareProxyKnown follows APR's numeric subnet syntax. Hostnames
// and variables need Apache's own resolution, so cannot prove a valid setting.
func apacheCloudflareProxyKnown(value string) bool {
ip, mask, hasMask := strings.Cut(value, "/")
addr, err := netip.ParseAddr(ip)
if err != nil {
// APR also accepts one to four decimal octets, with an optional final
// dot, as an implicit IPv4 network, but only without an explicit mask.
if hasMask || ip == "" || len(ip) > len("255.255.255.255") || strings.Trim(ip, "0123456789.") != "" {
return false
}
octets := strings.Split(strings.TrimSuffix(ip, "."), ".")
if len(octets) > 4 {
return false
}
for _, octet := range octets {
n, parseErr := strconv.Atoi(octet)
if parseErr != nil || n > 255 {
return false
}
}
return true
}
if addr.Is4In6() || addr.Zone() != "" {
return false
}
if !hasMask {
return true
}
// APR uses strtol for prefix lengths and accepts a dotted IPv4 mask,
// including non-contiguous masks. A zero prefix length is rejected.
bits, err := strconv.Atoi(strings.TrimLeft(mask, " \t\r\n\v\f"))
if err == nil && bits > 0 && bits <= addr.BitLen() {
return true
}
netmask, err := netip.ParseAddr(mask)
return err == nil && addr.Is4() && netmask.Is4()
}
// nginxClientAddressRewrite returns the first set_real_ip_from, which
// turns on nginx's realip rewrite from real_ip_header or its X-Real-IP
// default; failing that, the first log_format an access_log writes that
// logs a request value where CSM reads the client. It also returns the
// file and whether the tree and its format framing could be read. HTTP
// and stream have separate namespaces; only HTTP has built-in combined.
func nginxClientAddressRewrite(path string) (directive, file string, complete bool) {
directives, complete := nginxDirectives(path)
type format struct {
directive nginxDirective
value bool
}
type key struct{ module, name string }
formats := map[key]format{{"http", "combined"}: {}}
seen := make(map[key]bool)
var used []format
// Optional modules can leave their variable names available for user
// definitions. A familiar spelling alone does not prove a peer value.
custom := make(map[string]map[string]bool)
for _, d := range directives {
if nginxDataContext(d.Context) {
continue
}
var target string
switch d.Name {
case "map", "split_clients":
if len(d.Args) != 2 {
complete = false
continue
}
target = d.Args[1]
case "set", "auth_request_set", "perl_set":
if len(d.Args) != 2 {
complete = false
continue
}
target = d.Args[0]
case "geo":
if len(d.Args) > 2 {
complete = false
continue
}
target = "$geo"
if len(d.Args) > 0 {
target = d.Args[len(d.Args)-1]
}
default:
continue
}
if !strings.HasPrefix(target, "$") || len(target) == 1 {
complete = false
continue
}
if custom[d.Module] == nil {
custom[d.Module] = make(map[string]bool)
}
custom[d.Module][strings.ToLower(target[1:])] = true
}
for _, d := range directives {
if d.Name == "set_real_ip_from" {
return strings.Join(append([]string{d.Name}, d.Args...), " "), d.File, complete
}
if nginxDataContext(d.Context) {
continue
}
if d.Name != "log_format" && d.Name != "access_log" {
continue
}
if d.Module != "http" && d.Module != "stream" || d.Block {
complete = false
continue
}
switch d.Name {
case "log_format":
if len(d.Args) < 2 || d.Context != d.Module {
complete = false
continue
}
exact := key{d.Module, d.Args[0]}
if seen[exact] || d.Module == "http" && d.Args[0] == "combined" {
complete = false
continue
}
seen[exact] = true
args, escape := d.Args[1:], "default"
if strings.HasPrefix(args[0], "escape=") {
escape = strings.TrimPrefix(args[0], "escape=")
args = args[1:]
}
value, ok := nginxFormatLogsClientValue(d.Module, escape, custom[d.Module], args...)
if !ok {
complete = false
continue
}
name := key{d.Module, lowerASCII(d.Args[0])}
// nginx permits case variants but binds to the first ASCII
// case-insensitive match already declared at the use site.
if _, exists := formats[name]; !exists {
formats[name] = format{d, value}
}
case "access_log":
contextOK := d.Context == d.Module || d.Context == "server"
if d.Module == "http" {
contextOK = contextOK || d.Context == "location" || d.Context == "if" || d.Context == "limit_except"
}
if len(d.Args) == 0 || !contextOK {
complete = false
continue
}
if d.Args[0] == "off" {
if len(d.Args) != 1 {
complete = false
}
continue
}
if len(d.Args) == 1 {
// Only the HTTP module supplies an implicit combined format.
if d.Module != "http" {
complete = false
}
continue
}
name := key{d.Module, lowerASCII(d.Args[1])}
if f, exists := formats[name]; exists {
used = append(used, f)
} else {
complete = false
}
}
}
for _, f := range used {
if f.value {
return strings.Join(append([]string{f.directive.Name}, f.directive.Args...), " "), f.directive.File, complete
}
}
return "", "", complete
}
// These blocks use their own entry handlers, not nginx directive lookup.
func nginxDataContext(context string) bool {
switch context {
case "map", "geo", "split_clients", "types":
return true
}
return false
}
// lswsClientAddressRewrite returns the first LiteSpeed setting that logs a
// replaced or request-chosen client address: useIpInProxyHeader when it is
// on, or a logFormat that writes a request value where CSM reads the
// client; "" when there is none. Anything but one well-formed
// httpServerConfig document with at most one useIpInProxyHeader and
// readable formats is an error. A completed active setting is returned
// even if later XML is unreadable.
func lswsClientAddressRewrite(path string) (string, error) {
f, err := osFS.Open(path)
if err != nil {
return "", fmt.Errorf("the LiteSpeed configuration %s could not be read: %w", path, err)
}
defer f.Close()
active := ""
invalid := func(why string) (string, error) {
return active, fmt.Errorf("the LiteSpeed configuration %s %s", path, why)
}
dec := xml.NewDecoder(f)
var value, format strings.Builder
depth, roots, settings := 0, 0, 0
inSetting, inFormat := false, false
for {
tok, err := dec.Token()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return active, fmt.Errorf("the LiteSpeed configuration %s could not be parsed: %w", path, err)
}
switch t := tok.(type) {
case xml.StartElement:
if inSetting {
return invalid("nests an element in useIpInProxyHeader")
}
if inFormat {
return invalid("nests an element in logFormat")
}
if depth == 0 {
if roots++; roots > 1 || t.Name.Local != "httpServerConfig" {
return invalid("is not one httpServerConfig document")
}
}
switch t.Name.Local {
case "useIpInProxyHeader":
if settings++; settings > 1 {
return invalid("sets useIpInProxyHeader more than once")
}
inSetting = true
case "logFormat":
format.Reset()
inFormat = true
}
depth++
case xml.CharData:
if depth == 0 && strings.Trim(string(t), " \t\r\n") != "" {
return invalid("has text outside its document element")
}
if inSetting {
value.Write(t)
}
if inFormat {
format.Write(t)
}
case xml.EndElement:
depth--
if inSetting {
if v := strings.TrimSpace(value.String()); v != "" && v != "0" && active == "" {
active = "useIpInProxyHeader " + v
}
}
if inFormat {
v := strings.TrimSpace(format.String())
logged, ok := liteSpeedFormatLogsClientValue(v, false)
if !ok {
return invalid("has a logFormat it cannot read")
}
if logged && active == "" {
active = "logFormat " + v
}
}
inSetting, inFormat = false, false
}
}
if roots != 1 {
return invalid("is not one httpServerConfig document")
}
if settings == 1 && active == "" {
if v := strings.TrimSpace(value.String()); v == "" {
return invalid("sets an empty useIpInProxyHeader")
}
}
return active, nil
}
package checks
import (
"sort"
"strings"
)
// CSM access readers use the first space-delimited field or skip leading
// ASCII whitespace and read the first token. splitlogs strips one field
// before either reader runs. Both interpretations must be safe. The
// ModSecurity reader takes its client from the error-log prefix. A format
// that writes request data at either read position logs a chosen address.
// apacheLogItem is one placeholder of an Apache log format: its handler
// ("h", "i", "^ti") and its {argument}.
type apacheLogItem struct {
handler string
arg string
required bool
level int
}
// apacheLogItemAt parses the placeholder whose '%' is at format[i], as
// mod_log_config does, or core's ErrorLogFormat when errorLog is set:
// modifiers and {arguments}, then a known handler. ok is false where the
// format is malformed or uses an unsupported handler.
func apacheLogItemAt(format string, i int, errorLog bool) (item apacheLogItem, next int, ok bool) {
for i++; i < len(format); {
switch c := format[i]; {
case c >= '0' && c <= '9':
level := 0
for i < len(format) && format[i] >= '0' && format[i] <= '9' {
// Only error-log levels up to trace8 are meaningful.
if level <= 15 {
level = level*10 + int(format[i]-'0')
}
i++
}
if errorLog {
item.level = level
}
case !errorLog && (c == '!' || c == '<' || c == '>' || c == ','):
i++
case errorLog && (c == '+' || c == '-'):
item.required = item.required || c == '+'
i++
case c == '{':
end := strings.IndexByte(format[i+1:], '}')
if end < 0 {
return item, 0, false
}
item.arg = format[i+1 : i+1+end]
i += end + 2
case c == '^' && !errorLog:
if i+3 > len(format) {
return item, 0, false
}
item.handler = format[i : i+3]
return item, i + 3, item.handler == "^ti" || item.handler == "^to"
default:
item.handler = format[i : i+1]
return item, i + 1, apacheLogHandlerKnown(item.handler, errorLog)
}
}
return item, 0, false
}
// Module-defined handlers outside this grammar are ambiguous, even after
// the normal peer field: their output framing is not known here.
func apacheLogHandlerKnown(handler string, errorLog bool) bool {
known := "aAhbBCDefHiIklLmnoOpPqrRsStTuUvVxX"
if errorLog {
known = "aAeEFiklLmnMPtTvV"
}
return len(handler) == 1 && strings.IndexByte(known, handler[0]) >= 0
}
// Custom strftime formats may emit newlines, tabs or an empty value. Only
// a nonempty, whitespace-free expansion can be treated as an opaque token.
// Other time formats are ambiguous; do not guess their output boundaries.
func apacheTimeTokenStable(arg string) bool {
if arg == "" || arg == "begin" || arg == "end" {
return true
}
arg = strings.TrimPrefix(strings.TrimPrefix(arg, "begin:"), "end:")
if arg == "" {
return true
}
for i := 0; i < len(arg); i++ {
if arg[i] == ' ' || arg[i] == '\n' || logWhitespace(arg[i]) {
return false
}
if arg[i] == '%' {
i++
if i == len(arg) || !strings.ContainsRune("%CdDFgGHIjmMSsRTuUVwWyYz", rune(arg[i])) {
return false
}
}
}
return true
}
// apacheTimeNewline reports whether a time argument can write a line
// break: a raw newline, or strftime's %n with glibc flags, width and E/O
// in that order. A flag after the width is an unknown literal conversion.
func apacheTimeNewline(arg string) bool {
if strings.IndexByte(arg, '\n') >= 0 {
return true
}
for i := 0; i < len(arg); i++ {
if arg[i] != '%' {
continue
}
j := i + 1
for j < len(arg) && strings.IndexByte("_-0^#", arg[j]) >= 0 {
j++
}
for j < len(arg) && arg[j] >= '0' && arg[j] <= '9' {
j++
}
if j < len(arg) && (arg[j] == 'E' || arg[j] == 'O') {
j++
}
if j < len(arg) && arg[j] == 'n' {
return true
}
i = j
}
return false
}
// These handlers write an unrecognized argument verbatim. Restrict them
// to their documented forms rather than assuming a nonempty server token.
func apacheAccessArgumentKnown(item apacheLogItem) bool {
arg := strings.ToLower(item.arg)
switch item.handler {
case "p":
return arg == "" || arg == "canonical" || arg == "local" || arg == "remote"
case "P":
return arg == "" || arg == "pid" || arg == "tid" || arg == "hextid"
case "T":
return arg == "" || arg == "s" || arg == "ms" || arg == "us"
}
return true
}
// apacheAccessLogSafe reports whether an access-log placeholder writes the
// connection's peer or a value the server sets.
func apacheAccessLogSafe(item apacheLogItem, hostnames bool) bool {
switch item.handler {
case "a":
return item.arg == "" || item.arg == "c"
case "h":
// HostnameLookups On logs the client's reverse DNS name unverified.
return !hostnames && (item.arg == "" || item.arg == "c")
case "A", "b", "B", "D", "I", "k", "L", "O", "p", "P", "s", "S", "t", "T", "v", "X":
return true
}
return false
}
// addressStartByte reports whether c can begin an address CSM parses: a
// token that starts with any other byte is never read as a client.
func addressStartByte(c byte) bool {
return c == '[' || c == ':' || c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F'
}
// logWhitespace reports whether c is whitespace other than the space that
// ends a field. A reader may trim it, so it never guards a field.
func logWhitespace(c byte) bool {
return c == '\t' || c == '\r' || c == '\v' || c == '\f'
}
// apacheFormatLogsClientValue reports whether an Apache access format
// writes a value the request chooses where CSM reads the client: the first
// field of each line, or the second when splitlogs strips the first. A
// read field that starts with a literal no address starts with is safe; a
// stripped field is not, since a value with spaces shifts the next field.
// Leading whitespace cannot finish the first token; a newline resets the
// read position. Unsupported output boundaries leave ok false.
func apacheFormatLogsClientValue(format string, splitlogs bool) (value, ok bool) {
return apacheFormatClientValue(format, splitlogs, false)
}
// apacheFormatClientValue is apacheFormatLogsClientValue on a server whose
// %h may log a name the client's reverse DNS chose.
func apacheFormatClientValue(format string, splitlogs, hostnames bool) (value, ok bool) {
read := 0
if splitlogs {
read = 1
}
field, start, guarded := 0, true, false
literal := func(c byte) {
if start && !logWhitespace(c) {
guarded, start = !addressStartByte(c), false
}
}
for i := 0; i < len(format); {
c := format[i]
switch {
case c == '%' && i+1 < len(format) && format[i+1] == '%':
literal('%')
i += 2
case c == '%':
item, next, parsed := apacheLogItemAt(format, i, false)
if !parsed {
return false, false
}
// A line break anywhere starts a record whose first field is
// read; other framing matters only up to the read field.
if item.handler == "t" && apacheTimeNewline(item.arg) {
return false, false
}
if field <= read && (item.handler == "t" && !apacheTimeTokenStable(item.arg) || !apacheAccessArgumentKnown(item)) {
return false, false
}
start = false
if !apacheAccessLogSafe(item, hostnames) && (field < read || field == read && !guarded) {
value = true
}
i = next
case c == '\n':
field, start, guarded = 0, true, false
i++
case c == '\\' && i+1 < len(format) && format[i+1] == 'n':
field, start, guarded = 0, true, false
i += 2
case c == '\\' && i+1 < len(format) && strings.IndexByte(`\rt`, format[i+1]) >= 0:
// \r and \t write whitespace, \\ a backslash.
if format[i+1] == '\\' {
literal('\\')
} else {
literal('\t')
}
i += 2
case c == ' ':
// splitlogs strips one field; the realtime reader then skips
// leading whitespace before choosing its first token.
if !start {
field++
}
start, guarded = true, false
i++
default:
literal(c)
i++
}
}
return value, true
}
// LiteSpeed shares the common Apache format, but not its whole grammar.
// Unknown escapes lose their backslash, unsupported items can disappear,
// and brace arguments can select request variables even on server items.
// Only reuse Apache's field analysis where both dialects agree.
func liteSpeedFormatLogsClientValue(format string, splitlogs bool) (value, ok bool) {
for i := 0; i < len(format); {
switch format[i] {
case '\\':
if i+1 == len(format) || !strings.ContainsRune(`\rnt"'`, rune(format[i+1])) {
return false, false
}
i += 2
case '%':
if i+1 < len(format) && format[i+1] == '%' {
i += 2
continue
}
item, next, parsed := apacheLogItemAt(format, i, false)
if !parsed || len(item.handler) != 1 || strings.IndexByte("aABbCDefhHilmnopPqrstTuUvVXIOSx", item.handler[0]) < 0 {
return false, false
}
modifiers := format[i+1 : next-1]
if modifiers != "" && modifiers != "<" && modifiers != ">" && (item.arg == "" || modifiers != "{"+item.arg+"}") {
return false, false
}
if !apacheAccessArgumentKnown(item) {
return false, false
}
if item.arg != "" {
switch item.handler {
case "t":
if !strings.Contains(item.arg, "%") && item.arg != "sec" && item.arg != "msec" && item.arg != "usec" && item.arg != "msec_frac" && item.arg != "usec_frac" {
return false, false
}
case "a", "h":
if item.arg != "c" {
return false, false
}
case "p", "P", "T", "C", "e", "i", "n", "o", "x":
default:
return false, false
}
}
i = next
default:
i++
}
}
return apacheFormatLogsClientValue(format, splitlogs)
}
// apacheErrorLogSafe reports whether an ErrorLogFormat placeholder writes
// the connection's peer or a value the server sets.
func apacheErrorLogSafe(item apacheLogItem) bool {
switch item.handler {
case "a":
return item.arg == "" || item.arg == "c"
case "A", "E", "F", "k", "l", "L", "m", "P", "t", "T", "v":
return true
}
return false
}
// A missing or level-filtered error item erases its whole field, including
// an earlier %M. A message emitted while a field is skipping can also be
// erased in the next field. Trust messages only when their fields contain
// no other placeholders. Unusual shared fields stay conservative.
func apacheErrorMessagesStable(format string) bool {
message, other := false, false
for i := 0; i < len(format); {
if format[i] == ' ' {
message, other = false, false
}
if format[i] == '%' {
if i+1 < len(format) && format[i+1] == ' ' {
message, other = false, false
i += 2
continue
}
if i+1 < len(format) && format[i+1] == '%' {
i += 2
continue
}
item, next, parsed := apacheLogItemAt(format, i, true)
if !parsed {
return false
}
if item.handler == "M" {
message = true
} else {
other = true
}
if message && other {
return false
}
i = next
continue
}
// Core keeps escaped spaces in a literal item. Only a raw space
// starting the next item, or % , begins a new field.
atStart := true
for i < len(format) && format[i] != '%' && (format[i] != ' ' || atStart) {
if format[i] == '\\' && i+1 < len(format) {
i += 2
continue
}
if format[i] != ' ' {
atStart = false
}
i++
}
}
return true
}
// apacheErrorFormatLogsClientValue reports whether an ErrorLogFormat
// writes a value the request chooses before the message in any record, where the
// ModSecurity reader looks for "[client ". ok is false where Apache
// refuses the format.
func apacheErrorFormatLogsClientValue(format string, main bool) (value, ok bool) {
stableMessage := apacheErrorMessagesStable(format)
message, seenMessage := false, false
for i := 0; i < len(format); {
switch {
case format[i] == '\n':
message = false
i++
case format[i] == '\\' && i+1 < len(format):
// A new record has its own prefix before a message marker.
if format[i+1] == 'n' {
message = false
}
// Core consumes the escaped byte as a literal, including '%'.
i += 2
case format[i] == '%' && i+1 < len(format) && (format[i+1] == '%' || format[i+1] == ' '):
i += 2
case format[i] == '%':
item, next, parsed := apacheLogItemAt(format, i, true)
if !parsed {
return false, false
}
// Only a line break can move the prefix the reader searches.
if item.handler == "t" && apacheTimeNewline(item.arg) {
return false, false
}
if item.level > 15 || main && item.required || !main && (item.level != 0 || item.handler == "M") {
return false, false
}
if item.handler == "M" {
seenMessage = true
// A shared field may erase a message already written.
if item.level == 0 && stableMessage {
message = true
}
} else if !message && !apacheErrorLogSafe(item) {
value = true
}
i = next
default:
i++
}
}
return value, !main || seenMessage
}
// apacheSplitlogsPipe reports whether a log target pipes to cPanel's
// splitlogs, which strips the first field (the virtual host) of each line.
// A pipe that names splitlogs anywhere counts, wrappers included.
func apacheSplitlogsPipe(target string) bool {
return strings.HasPrefix(target, "|") && strings.Contains(target, "splitlogs")
}
// apacheLogTargetUnresolved reports whether expansion can change whether a
// log target is a splitlogs pipe. Only a leading "|" pipes, so a variable
// later in a file path is harmless. A leading variable could hold a pipe,
// except Debian's APACHE_LOG_DIR when configuration has not overridden it.
func apacheLogTargetUnresolved(target string, logDirOverridden bool) bool {
switch {
case !apacheArgumentHasVariable(target):
return false
case strings.HasPrefix(target, "|"):
return !strings.Contains(target, "splitlogs")
case !strings.HasPrefix(target, "${"):
return false
}
return logDirOverridden || !strings.HasPrefix(target, "${APACHE_LOG_DIR}/")
}
// apacheMacroFields reads a line inside mod_macro's <Macro> as it may look
// after Use substitutes the macro's parameters: each one becomes an
// unresolved ${...} value, so the rules for Define values apply. ok is false
// when a parameter can supply the directive name. ServerName keeps its text,
// since a parameter there names a host.
func apacheMacroFields(fields []string, ctx apacheLineContext) (out []string, ok bool) {
var params []string
// Scope labels flatten quoted arguments; keys keep their boundaries.
for _, key := range ctx.ScopeKey {
if words := strings.Split(key, "\x1f"); len(words) > 2 && words[0] == "macro" {
params = append(params, words[2:]...)
}
}
if len(params) == 0 {
return fields, true
}
// The replacer tries names in order: longer first, so a name never
// leaves part of another.
sort.SliceStable(params, func(i, j int) bool { return len(params[i]) > len(params[j]) })
oldnew := make([]string, 0, 2*len(params))
for _, param := range params {
oldnew = append(oldnew, param, "${macro}")
}
replace := strings.NewReplacer(oldnew...)
if replace.Replace(fields[0]) != fields[0] {
return fields, false
}
if strings.EqualFold(fields[0], "ServerName") {
return fields, true
}
out = make([]string, len(fields))
for i, field := range fields {
out[i] = replace.Replace(field)
}
return out, true
}
// apacheLogFormat is a format a directive defines or uses inline.
type apacheLogFormat struct {
format string
directive string
file string
scope string
condition string
}
func (f apacheLogFormat) availableIn(scope string, conditions ...string) bool {
if f.scope != "" && f.scope != scope {
return false
}
if f.condition == "" {
return true
}
for _, condition := range conditions {
// A log inside the same condition or a nested one cannot run
// without the definition in the enclosing condition.
if condition == f.condition || strings.HasPrefix(condition, f.condition+"\x1e") {
return true
}
}
return false
}
// apacheLogUse is a log that writes a named, inline or default format.
type apacheLogUse struct {
transfer bool
target string
inline apacheLogFormat
}
// apacheLogFormats collects the access formats an Apache tree defines and
// the logs that use them, and checks error-log formats as it reads them.
type apacheLogFormats struct {
named map[string][]apacheLogFormat
defaults []apacheLogFormat
uses []apacheLogUse
rewrite *apacheLogFormat
malformed bool
liteSpeed bool
// hostnames is set by HostnameLookups On in any scope.
hostnames bool
// Define values are global, including those in conditional and macro
// bodies. A macro may use a value declared after its body was read.
logDirOverridden bool
}
func (l *apacheLogFormats) parse(format string, splitlogs bool) (bool, bool) {
if l.liteSpeed {
if _, ok := liteSpeedFormatLogsClientValue(format, splitlogs); !ok {
return false, false
}
}
return apacheFormatClientValue(format, splitlogs, l.hostnames)
}
// add reads one directive. Apache refuses a format it cannot parse even
// when no log uses it, so such a format leaves the tree unknown.
func (l *apacheLogFormats) add(line apacheConfigLine, fields []string, ctx apacheLineContext) {
switch name := strings.ToLower(fields[0]); name {
case "logformat", "customlog", "globallog", "transferlog", "errorlogformat", "servername":
for i, arg := range fields[1:] {
if i == 0 && (name == "customlog" || name == "globallog" || name == "transferlog") {
// Check targets after all possible Define values are known.
continue
}
if apacheArgumentHasVariable(arg) {
// Expansion can change the format, nickname or server name.
l.malformed = true
return
}
}
}
def := apacheLogFormat{
directive: strings.Join(fields, " "), file: line.File,
scope: strings.Join(ctx.ScopeKey, "\x1e"), condition: ctx.Condition,
}
check := func(format string) bool {
if _, ok := l.parse(format, false); !ok {
l.malformed = true
return false
}
return true
}
switch {
case strings.EqualFold(fields[0], "Define") || strings.EqualFold(fields[0], "UnDefine"):
// Apache looks up defined names ignoring case, unlike the environment.
if len(fields) > 1 && (strings.EqualFold(fields[1], "APACHE_LOG_DIR") || apacheArgumentHasVariable(fields[1])) {
l.logDirOverridden = true
}
case strings.EqualFold(fields[0], "LogFormat"):
switch len(fields) {
case 2:
def.format = fields[1]
if check(def.format) {
l.defaults = append(l.defaults, def)
}
case 3:
def.format = fields[1]
if check(def.format) {
if l.named == nil {
l.named = make(map[string][]apacheLogFormat)
}
l.named[strings.ToLower(fields[2])] = append(l.named[strings.ToLower(fields[2])], def)
}
default:
l.malformed = true
}
case strings.EqualFold(fields[0], "CustomLog") || strings.EqualFold(fields[0], "GlobalLog"):
if len(fields) != 3 && len(fields) != 4 {
l.malformed = true
return
}
def.format = fields[2]
// Apache parses this as a format before resolving its nickname.
if !check(def.format) {
return
}
if len(fields) == 4 {
condition := fields[3]
// A condition only suppresses records, so the format is read
// either way. Apache refuses an empty variable or expression.
switch {
case len(condition) > 4 && strings.EqualFold(condition[:4], "env=") && condition[4:] != "!":
case len(condition) > 5 && strings.EqualFold(condition[:5], "expr=") && strings.TrimSpace(condition[5:]) != "":
default:
l.malformed = true
}
}
l.uses = append(l.uses, apacheLogUse{target: fields[1], inline: def})
case strings.EqualFold(fields[0], "TransferLog"):
if len(fields) != 2 {
l.malformed = true
return
}
l.uses = append(l.uses, apacheLogUse{transfer: true, target: fields[1], inline: def})
case strings.EqualFold(fields[0], "HostnameLookups"):
switch {
case len(fields) != 2:
l.malformed = true
case strings.EqualFold(fields[1], "On"):
l.hostnames = true
case !strings.EqualFold(fields[1], "Off") && !strings.EqualFold(fields[1], "Double"):
l.malformed = true
}
case strings.EqualFold(fields[0], "ServerName"):
if len(fields) != 2 {
l.malformed = true
return
}
host := fields[1]
if _, after, found := strings.Cut(host, "://"); found {
host = after
}
host, _, _ = strings.Cut(host, ":")
// Core accepts a blank host before a port. Its %v then writes no
// token, leaving the following request field first.
if strings.TrimSpace(host) == "" {
l.malformed = true
}
case strings.EqualFold(fields[0], "ErrorLogFormat"):
format := ""
switch {
case len(fields) == 2:
format = fields[1]
case len(fields) == 3 && (strings.EqualFold(fields[1], "connection") || strings.EqualFold(fields[1], "request")):
format = fields[2]
default:
l.malformed = true
return
}
value, ok := apacheErrorFormatLogsClientValue(format, len(fields) == 2)
switch {
case !ok:
l.malformed = true
case value && l.rewrite == nil:
l.rewrite = &def
}
}
}
const apacheCommonLogFormat = `%h %l %u %t "%r" %>s %b`
// Every server starts with CLF. An override in another scope cannot prove
// that a log stopped using that built-in definition.
func (l *apacheLogFormats) withBuiltin(formats []apacheLogFormat, name string, source apacheLogFormat) []apacheLogFormat {
if !strings.EqualFold(name, "CLF") {
return formats
}
source.format = apacheCommonLogFormat
return append(append([]apacheLogFormat(nil), formats...), source)
}
func (l *apacheLogFormats) resolve(source, use apacheLogFormat) []apacheLogFormat {
named := l.named[strings.ToLower(source.format)]
formats := l.withBuiltin(named, source.format, source)
for _, def := range formats {
if def.availableIn(use.scope, use.condition, source.condition) {
return formats
}
}
// A nickname in another scope may not exist for this log. Apache then
// uses the argument itself as an inline format.
return append(append([]apacheLogFormat(nil), formats...), source)
}
// clientValue returns the first format a log uses that writes a value the
// request chooses where CSM reads the client. A nickname no LogFormat
// defines is an inline format, as Apache reads it; TransferLog writes the
// default formats (or the formats they name), or the common log format
// when none is set.
func (l *apacheLogFormats) clientValue() *apacheLogFormat {
if l.rewrite != nil {
return l.rewrite
}
for _, use := range l.uses {
if apacheLogTargetUnresolved(use.target, l.logDirOverridden) {
l.malformed = true
continue
}
var formats []apacheLogFormat
switch {
case use.transfer:
defaultKnown := false
for _, def := range l.defaults {
defaultKnown = defaultKnown || def.availableIn(use.inline.scope, use.inline.condition)
}
// A default in another scope cannot replace this server's CLF.
if !defaultKnown {
def := use.inline
def.format = apacheCommonLogFormat
formats = append(formats, def)
}
for _, def := range l.defaults {
// A one-argument LogFormat may name a format instead.
formats = append(formats, l.resolve(def, use.inline)...)
}
default:
formats = l.resolve(use.inline, use.inline)
}
for i := range formats {
value, ok := l.parse(formats[i].format, apacheSplitlogsPipe(use.target))
if !ok {
l.malformed = true
continue
}
if value {
return &formats[i]
}
}
}
return nil
}
// nginxSafeLogVariables are HTTP core values with server-owned output.
// Optional realip names are not proof of a peer: without that module,
// regex captures and extension modules can define them from request data.
var nginxSafeLogVariables = map[string]bool{
"remote_addr": true, "remote_port": true, "server_addr": true,
"server_port": true, "server_name": true, "hostname": true,
"time_local": true, "time_iso8601": true, "msec": true, "pid": true,
"connection": true, "connection_requests": true, "status": true,
"body_bytes_sent": true, "bytes_sent": true, "request_length": true,
"request_time": true, "pipe": true, "nginx_version": true,
}
// A name that is built in to HTTP may be a user variable in stream.
// Stream server_name also remains user-definable before nginx 1.25.5.
func nginxSafeLogVariable(module, name string) bool {
if module == "http" {
return nginxSafeLogVariables[name]
}
switch name {
case "remote_addr", "remote_port",
"server_addr", "server_port", "hostname", "time_local",
"time_iso8601", "msec", "pid", "connection", "status", "bytes_sent", "nginx_version",
"bytes_received", "session_time", "protocol":
return true
}
return false
}
// nginxFormatLogsClientValue keeps output-field state across arguments,
// but parses each argument's variables separately, as nginx does. Without
// escaping, a later request value can inject a fresh log line.
func nginxFormatLogsClientValue(module, escape string, custom map[string]bool, formats ...string) (value, ok bool) {
if escape != "default" && escape != "json" && escape != "none" {
return false, false
}
field, start, guarded := 0, true, false
for _, format := range formats {
for i := 0; i < len(format); {
c := format[i]
switch c {
case '$':
j := i + 1
braced := j < len(format) && format[j] == '{'
if braced {
j++
}
k := j
for k < len(format) && (format[k] == '_' || format[k] >= '0' && format[k] <= '9' ||
format[k] >= 'a' && format[k] <= 'z' || format[k] >= 'A' && format[k] <= 'Z') {
k++
}
if k == j || braced && (k == len(format) || format[k] != '}') {
return false, false
}
if strings.EqualFold(format[j:k], "binary_remote_addr") {
return false, false
}
name := strings.ToLower(format[j:k])
if (custom[name] || !nginxSafeLogVariable(module, name)) && (escape == "none" || field == 0 && !guarded) {
value = true
}
switch name {
case "server_name", "hostname", "remote_port", "realip_remote_port", "server_port":
// These can be empty; the next token may still be first.
default:
start = false
}
if braced {
k++
}
i = k
case '\n':
field, start, guarded = 0, true, false
i++
case ' ':
if !start {
field++
}
start, guarded = true, false
i++
default:
if start && !logWhitespace(c) {
guarded, start = !addressStartByte(c), false
}
i++
}
}
}
return value, true
}
package checks
import (
"errors"
"fmt"
)
// MySQL identifier limit per the manual is 64 bytes. Anything beyond
// is silently truncated by the server, but for the cleaner we want
// the failure to be loud at the validation step.
const mysqlIdentMaxLen = 64
// errEmptyIdent and errInvalidIdent are surfaced to the operator so
// the CLI prints a clear "the name you typed is not a valid MySQL
// identifier" instead of a SQL syntax error from the server.
var (
errEmptyIdent = errors.New("identifier is empty")
errInvalidIdent = errors.New("identifier contains characters outside [A-Za-z0-9_$]")
errLongIdent = fmt.Errorf("identifier exceeds %d bytes (MySQL limit)", mysqlIdentMaxLen)
)
// QuoteIdent returns a backtick-quoted MySQL identifier, or an error
// if the input is empty, longer than 64 bytes, or contains characters
// outside the safe class. Used at every site where an attacker-
// controlled object name (trigger / event / routine / schema) would
// otherwise reach a SQL string concatenation.
//
// The safe class is intentionally narrow: standard MySQL allows more
// (digits-only names, dotted names, $-prefixed) but the cleaner only
// needs to handle CMS-shaped identifiers and operator-typed schema
// names. Rejecting anything weirder is cheaper than reasoning about
// edge cases in dynamic SQL.
func QuoteIdent(name string) (string, error) {
if name == "" {
return "", errEmptyIdent
}
if len(name) > mysqlIdentMaxLen {
return "", errLongIdent
}
for _, r := range name {
switch {
case r >= 'A' && r <= 'Z':
case r >= 'a' && r <= 'z':
case r >= '0' && r <= '9':
case r == '_' || r == '$':
default:
return "", fmt.Errorf("%w: %q", errInvalidIdent, name)
}
}
return "`" + name + "`", nil
}
package checks
import (
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
)
func init() {
// CLI scans and standalone web UI dispatches also need these policies,
// before any daemon is constructed. They depend only on check metadata
// and the config passed to each call, not on a daemon's lifetime.
alert.SetIPResponsePolicy(IPResponseAnswersFinding)
alert.SetSummaryNoticePolicy(IsSummaryNotice)
alert.SetWebTrafficPolicy(webTrafficFinding)
}
// IPResponseAnswersFinding is the alert.IPResponsePolicy that decides which
// findings about a handled source go to the summary. A block or challenge on
// the source address answers only attacker-side findings: attacker activity
// or attempted access that is not evidence of compromise. Once the source is
// stopped the operator has nothing left to do. Compromise evidence,
// successful logins and audit events stay immediate even when their source
// address is already blocked.
//
// A challenge answers only the findings challenge routing would send to the
// gate. The address being on the challenge list is itself proof the gate is
// in use, so the policy does not consult challenge.enabled. A block of a
// Cloudflare-covered address does not answer web traffic; the class split
// applies that with the firewall's ranges, read once per batch.
func IPResponseAnswersFinding(cfg *config.Config, f alert.Finding, blocked bool) bool {
if correlationReasonOf(f.Check) != reasonAttackerSide {
return false
}
switch f.Check {
case "whm_unauth_scripts_realtime":
// An exploit indicator: a later block cannot tell whether the
// tokenless request already succeeded, so a person must check.
return false
case "email_phishing_content", "email_malware", "email_av_parse_worker_source":
// Mail content may be evidence of a compromised local sender, and
// an IP in the message may come from a mail header or an account
// name. A parse source can also represent a whole IPv6 subnet.
return false
case "mail_account_spray", "smtp_account_spray", "mail_subnet_spray", "smtp_subnet_spray",
"http_distributed_flood", "http_asn_crawl":
// These summarize many sources. SourceIP can be the latest sender
// or a subnet; blocking one address does not answer the finding.
return false
}
return blocked || challengeRoutesFinding(cfg, f)
}
// webTrafficFinding reports whether f is about requests on the web ports: an
// HTTP-family check, or a reputation sighting below Critical, which is
// browser-facing traffic.
func webTrafficFinding(f alert.Finding) bool {
// Raw WAF observations never authorize a response, so their response
// policy has no evidence family. They still describe HTTP traffic.
switch f.Check {
case "modsec_block_realtime", "modsec_warning_realtime", "modsec_low_confidence_burst":
return true
}
return ResponsePolicyFor(f.Check).Evidence == admission.FamilyHTTP ||
(f.Check == "ip_reputation" && f.Severity != alert.Critical)
}
package checks
import (
"context"
"time"
"github.com/pidginhost/csm/internal/jstaint"
)
// jsTaintAnalyze is indirected so tests can substitute the analyzer at the
// adapter boundary: forcing a panic or a specific status proves each owner's
// containment without crafting pathological JavaScript.
var jsTaintAnalyze = jstaint.Analyze
// jsTaintReverifyTimeout is the per-file safety net for a re-verification
// analysis. The analyzer checks its context between AST nodes, and its node,
// depth, and fact caps bound normal work far below this; the deadline only
// stops a defect from stalling an operator-triggered re-check.
const jsTaintReverifyTimeout = 15 * time.Second
// runJSTaintAnalysis is the adapter recovery boundary around the analyzer
// call. The analyzer converts its own internal panics to StatusPanic, but a
// panic in adapter-side code around the call must have the same contained
// outcome instead of escaping through the owning check.
func runJSTaintAnalysis(ctx context.Context, data []byte) (report jstaint.Report) {
defer func() {
if r := recover(); r != nil {
report = jstaint.Report{Status: jstaint.StatusPanic, Reason: "panic"}
}
}()
return jsTaintAnalyze(ctx, data)
}
package checks
import (
"context"
"fmt"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/jstaint"
)
// jsTaintDeepCursorCheck is the host-scope scan-cursor key for the scheduled
// JS taint consumer of the shared deep-content walk.
const jsTaintDeepCursorCheck = logicalOwnerJSTaintDeep
// jsTaintDeepPerFileTimeout is the per-file safety net for one deep-scan
// analysis. The engine's node, depth, and fact caps bound normal work far
// below this; the deadline only stops an analyzer defect from eating the
// walk's soft-deadline margin on a single file.
const jsTaintDeepPerFileTimeout = 20 * time.Second
// maxJSTaintGapPaths bounds the exact paths one run retains for carry-forward,
// as maxPHPTaintGapPaths does for the PHP consumer. Past it the run keeps
// counting but reports its path set as incomplete, which suppresses the purge
// rather than carrying forward an arbitrary prefix.
const maxJSTaintGapPaths = 50_000
// Display bounds: the sanitized path in the finding message, the rendered
// evidence in its details, and diagnostic example paths (512, 2048
// and 256 bytes, markers included).
const (
jsTaintMessageMaxBytes = 512
jsTaintDetailsMaxBytes = 2048
jsTaintExampleMaxBytes = 256
)
// sanitizeJSTaintDisplay renders untrusted path or evidence text for
// operator-facing fields: valid UTF-8, control and invalid bytes become '?',
// truncation happens at a rune boundary and the marker counts into the cap.
func sanitizeJSTaintDisplay(s string, maxBytes int) string {
s = strings.ToValidUTF8(s, "?")
s = strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return '?'
}
return r
}, s)
if len(s) <= maxBytes {
return s
}
cut := maxBytes - 3
for cut > 0 && !utf8.RuneStart(s[cut]) {
cut--
}
return s[:cut] + "..."
}
// jsTaintGapCollector aggregates per-path JS coverage gaps for one deep run:
// exact paths feed the carry-forward, counts and one example per status feed
// the js_taint_scan_incomplete diagnostic. Non-analyzed statuses are never
// counted as clean files.
type jsTaintGapCollector struct {
gapPathSet
recordCoverage func([]string)
}
func newJSTaintGapCollector() *jsTaintGapCollector {
return &jsTaintGapCollector{gapPathSet: newGapPathSet(maxJSTaintGapPaths, jsTaintExampleMaxBytes, lexicalGapAliases)}
}
func (g *jsTaintGapCollector) record(path, status string) {
if aliases := g.retain(path, status); len(aliases) > 0 && g.recordCoverage != nil {
g.recordCoverage(aliases)
}
}
func (g *jsTaintGapCollector) finding() alert.Finding {
return g.incompleteFinding(alert.Warning, "js_taint_scan_incomplete",
"JavaScript taint deep scan could not analyze %d file(s)",
"JavaScript taint deep scan could not cover %d location(s)")
}
// carryForwardJSTaintFindings keeps at most one prior state finding for each
// path the current full cycle could not analyze.
func carryForwardJSTaintFindings(prior []alert.Finding, gaps *jsTaintGapCollector) []alert.Finding {
return carryForwardLatestPerPath(prior, "js_keylogger_dataflow", &gaps.gapPathSet)
}
// analyzeJSTaintSnapshot runs the JS consumer on one complete in-memory
// snapshot and converts the result into at most one finding. A non-completed
// status is recorded as a known-path coverage gap, never as a clean file.
func analyzeJSTaintSnapshot(ctx context.Context, path, contentSHA256 string, data []byte, gaps *jsTaintGapCollector) []alert.Finding {
fileCtx, cancel := context.WithTimeout(ctx, jsTaintDeepPerFileTimeout)
report := runJSTaintAnalysis(fileCtx, data)
cancel()
switch report.Status {
case jstaint.StatusAnalyzed:
if len(report.Results) == 0 {
return nil
}
return []alert.Finding{jsTaintDeepFinding(path, contentSHA256, report)}
case jstaint.StatusNotCandidate:
return nil
default:
gaps.record(path, report.Status.String())
return nil
}
}
// jsTaintDeepFinding renders the single finding for one analyzed file: the
// evidence flows the engine returned plus the count of endpoint flows beyond
// them, with every display field sanitized and bounded. The fingerprint is
// the hash of the exact analyzed bytes; FilePath keeps the exact live path
// for remediation while only its display copy is sanitized.
func jsTaintDeepFinding(path, contentSHA256 string, report jstaint.Report) alert.Finding {
flows := make([]string, 0, len(report.Results))
for _, res := range report.Results {
segs := make([]string, 0, len(res.Via)+2)
segs = append(segs, res.Source)
segs = append(segs, res.Via...)
segs = append(segs, res.Sink)
flows = append(flows, strings.Join(segs, " -> "))
}
details := "Keystroke data reaches a network sink. Evidence: " + strings.Join(flows, "; ")
if extra := report.TotalResults - len(report.Results); extra > 0 {
details += fmt.Sprintf("; %d additional flow(s) beyond returned evidence", extra)
}
if report.EvidenceTruncated {
details += " [evidence truncated]"
}
return alert.Finding{
Severity: alert.Critical,
Check: "js_keylogger_dataflow",
Message: "JavaScript keystroke exfiltration data flow: " + sanitizeJSTaintDisplay(path, jsTaintMessageMaxBytes),
Details: sanitizeJSTaintDisplay(details, jsTaintDetailsMaxBytes),
FilePath: path,
ContentSHA256: contentSHA256,
DetectLogic: ContentDetectionVersion(),
CoverageScope: deepFileCoverageScope(path),
}
}
package checks
import (
"math"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
// procClockTicks is USER_HZ, 100 on the platforms CSM runs on.
const procClockTicks = 100.0
// processStartedBefore reports whether pid's process began before t.
//
// A PID is recycled freely on a busy host, and a finding can be acted on long
// after it was raised -- immediately by auto-response, or whenever an operator
// clicks Fix. Killing by number alone therefore risks destroying a process that
// merely inherited the PID. A process that started after the finding cannot be
// the one the finding describes.
//
// Unverifiable input fails closed: no uptime, no stat, or an unparsable
// field all report false, and the caller does not kill.
//
// (internal/daemon carries its own copy of this parse for the af_alg reaction:
// it reads procfs directly rather than through this package's filesystem seam,
// and sharing one implementation would mean mixing two abstractions.)
func processStartedBefore(pid string, t time.Time) bool {
pidInt, ok := parseProcessPID(pid)
if !ok || t.IsZero() {
return false
}
ticks, ok := procStartTicks(strconv.Itoa(pidInt))
if !ok {
return false
}
uptime, ok := procUptime()
if !ok {
return false
}
// Read wall time after uptime so elapsed is conservatively rounded up. A
// process close enough to the boundary to be ambiguous is not killed.
elapsed := time.Since(t).Seconds()
if elapsed < 0 {
return false
}
eventUptime := uptime - elapsed
return eventUptime >= 0 && float64(ticks)/procClockTicks <= eventUptime
}
func parseProcessPID(pid string) (int, bool) {
n, err := strconv.Atoi(pid)
return n, err == nil && n > 1 && n <= math.MaxInt32
}
func procUptime() (float64, bool) {
data, err := osFS.ReadFile("/proc/uptime")
if err != nil {
return 0, false
}
fields := strings.Fields(string(data))
if len(fields) == 0 {
return 0, false
}
uptime, err := strconv.ParseFloat(fields[0], 64)
return uptime, err == nil && !math.IsNaN(uptime) && !math.IsInf(uptime, 0) && uptime >= 0
}
func procStartTicks(pid string) (uint64, bool) {
fields, ok := procStatFields(pid)
// starttime is field 22 overall, index 19 once pid and comm are behind us.
if !ok || len(fields) < 20 {
return 0, false
}
ticks, err := strconv.ParseUint(fields[19], 10, 64)
if err != nil {
return 0, false
}
return ticks, true
}
// procStatFields returns the /proc/<pid>/stat fields after the command name.
func procStatFields(pid string) ([]string, bool) {
data, err := osFS.ReadFile(filepath.Join("/proc", pid, "stat"))
if err != nil {
return nil, false
}
// The comm field is parenthesised and may contain spaces, so fields are
// counted from after its closing parenthesis.
closing := strings.LastIndex(string(data), ")")
if closing < 0 {
return nil, false
}
return strings.Fields(string(data)[closing+1:]), true
}
// processUsesFileIdentity reports whether pid still references target, as its
// executable or through an open descriptor. The kill in fixKillAndQuarantine
// exists to release the file being quarantined; a process that no longer
// references that file is not the process the finding meant, so it is not killed.
func processUsesFileIdentity(pid int, target os.FileInfo) bool {
if pid <= 1 || target == nil || target.Mode()&os.ModeSymlink != 0 {
return false
}
procDir := filepath.Join("/proc", strconv.Itoa(pid))
if info, statErr := osFS.Stat(filepath.Join(procDir, "exe")); statErr == nil && sameObject(info, target) {
return true
}
fdDir := filepath.Join(procDir, "fd")
entries, err := osFS.ReadDir(fdDir)
if err != nil {
return false
}
for _, entry := range entries {
info, err := osFS.Stat(filepath.Join(fdDir, entry.Name()))
if err == nil && sameObject(info, target) {
return true
}
}
return false
}
// sameObject reports whether two stats describe the same file, requiring the
// content shape to agree as well as device and inode. A deleted inode is handed
// straight back to the next file created in the same directory, so dev+ino
// alone would call a descriptor for the removed file the same object as its
// replacement -- the same inode-reuse hole the quarantine move already guards
// against.
func sameObject(a, b os.FileInfo) bool {
return sameFileIdentity(a, b) && sameContentShape(a, b)
}
package checks
import (
"net/netip"
"slices"
"strings"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/emailspool"
)
// knownProxyProviders names the built-in blocks of CDN and website-firewall
// edges. The same list's payment webhook senders are callers, not proxies.
var knownProxyProviders = []string{"Cloudflare (IPv4)", "Cloudflare (IPv6)", "QUIC.cloud", "Sucuri firewall"}
// knownProxyPrefixes is the edge ranges of knownProxyProviders, read once.
var knownProxyPrefixes = func() []netip.Prefix {
var out []netip.Prefix
for _, p := range emailspool.BuiltInProxyProviders() {
if slices.Contains(knownProxyProviders, p.Name) {
out = append(out, p.Prefixes...)
}
}
return out
}()
// KnownProxyPeer reports whether addr, a bare address, is a known CDN or
// website-firewall edge: inside a built-in edge range, or, when the ASN
// database names it, in a network cfg lists under
// thresholds.http_asn_crawl_reverse_proxy_asns. Such an address carries many
// visitors, so it is never the attacker an HTTP, cPanel or reputation finding
// names, whether or not cloudflare.enabled is set.
func KnownProxyPeer(addr string, cfg *config.Config) bool {
// ASN readers use address bytes and can discard a zone.
a, err := netip.ParseAddr(strings.TrimSpace(addr))
if err != nil || a.Zone() != "" {
return false
}
a = a.Unmap()
for _, p := range knownProxyPrefixes {
if p.Contains(a) {
return true
}
}
lookup := CurrentASNLookup()
if lookup == nil || cfg == nil {
return false
}
asn, _ := lookup(a.String())
return asn != 0 && slices.Contains(cfg.Thresholds.HTTPASNCrawlReverseProxyASNs, asn)
}
package checks
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"net/netip"
"os"
"path/filepath"
"strings"
"time"
"unicode"
"github.com/pidginhost/csm/internal/admission"
)
// legacyHourLayout is the local-time hour the legacy counter is keyed by.
const legacyHourLayout = "2006-01-02T15"
// maxLegacyHourLead bounds how far ahead of now a legacy hour may end. A
// clock stepped back leaves a later hour behind, and its count still holds
// until its own window ends; a key further ahead is damage, which would
// otherwise hold the ceiling for as long as it claims.
const maxLegacyHourLead = 24 * time.Hour
// LegacyBlockSpend reads the legacy hourly block counter for the admission
// ledger's first limit. The whole state file must
// validate: a damaged or unreadable file is an error, never zero spend. No
// file means no legacy blocks. The count is dated at the end of its hour in
// now's zone, the later instance of an hour the clocks repeat, and an hour
// whose window ended by now counts nothing.
func LegacyBlockSpend(statePath string, now time.Time) (admission.LegacySpend, error) {
state, path, err := readLegacyBlockState(statePath, now.Location())
if err != nil {
return admission.LegacySpend{}, err
}
return legacySpendOf(state, path, now)
}
// LegacyHandover reads legacy blocking's state for the admission owner's
// live start: the hourly count as LegacyBlockSpend reads it. Pending entries
// stay legacy's to retry, since a finding admission takes never enters the
// queue. The whole file must validate, as for LegacyBlockSpend.
func LegacyHandover(statePath string, now time.Time) (admission.LegacyHandover, error) {
state, path, err := readLegacyBlockState(statePath, now.Location())
if err != nil {
return admission.LegacyHandover{}, err
}
spend, err := legacySpendOf(state, path, now)
if err != nil {
return admission.LegacyHandover{}, err
}
return admission.LegacyHandover{Spend: spend}, nil
}
// readLegacyBlockState reads and validates the whole legacy state file; no
// file is the empty state.
func readLegacyBlockState(statePath string, loc *time.Location) (blockState, string, error) {
path := filepath.Join(statePath, blockStateFile)
data, err := osFS.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
return blockState{}, path, nil
}
if err != nil {
return blockState{}, path, fmt.Errorf("reading %s: %w", path, err)
}
if err = validateLegacyJSON(data); err != nil {
return blockState{}, path, fmt.Errorf("reading %s: %w", path, err)
}
var state blockState
dec := json.NewDecoder(bytes.NewReader(data))
dec.DisallowUnknownFields()
if err = dec.Decode(&state); err != nil {
return blockState{}, path, fmt.Errorf("reading %s: %w", path, err)
}
if _, err = dec.Token(); err != io.EOF {
return blockState{}, path, fmt.Errorf("reading %s: data after the state", path)
}
if err = validateLegacyState(state, loc); err != nil {
return blockState{}, path, fmt.Errorf("reading %s: %w", path, err)
}
return state, path, nil
}
// legacySpendOf is the hourly count of a validated state, dated at the end
// of its hour.
func legacySpendOf(state blockState, path string, now time.Time) (admission.LegacySpend, error) {
end, _ := legacyHourEnd(state.HourKey, now.Location())
switch {
case end.After(now.Add(maxLegacyHourLead)):
return admission.LegacySpend{}, fmt.Errorf("reading %s: hour %s ends more than a day ahead", path, state.HourKey)
case !end.Add(admission.CeilingWindow).After(now):
return admission.LegacySpend{}, nil
}
if state.BlocksThisHour == 0 {
return admission.LegacySpend{}, nil
}
return admission.LegacySpend{Units: uint32(min(state.BlocksThisHour, math.MaxUint32)), At: end}, nil // #nosec G115 -- bounded by the min.
}
func validateLegacyState(s blockState, loc *time.Location) error {
switch {
case s.BlocksThisHour < 0:
return errors.New("negative hourly count")
case s.BlocksThisHour > 0 && s.HourKey == "":
return errors.New("hourly count without its hour")
}
for _, key := range []string{s.HourKey, s.RateLimitWarnedHour, s.PendingDropWarnedHour} {
if _, ok := legacyHourEnd(key, loc); key != "" && !ok {
return fmt.Errorf("malformed hour %q", key)
}
}
addrs := make([]string, 0, len(s.IPs)+len(s.Pending)+len(s.CleanupPending))
for _, b := range s.IPs {
addrs = append(addrs, b.IP)
}
for _, p := range s.Pending {
addrs = append(addrs, p.IP)
}
addrs = append(addrs, s.CleanupPending...)
for _, a := range addrs {
if _, err := netip.ParseAddr(a); err != nil {
return fmt.Errorf("malformed address %q", a)
}
}
return nil
}
// legacyHourEnd is the end of the local hour key names in loc. An hour the
// clocks repeat ends with its later instance.
func legacyHourEnd(key string, loc *time.Location) (time.Time, bool) {
civil, err := time.Parse(legacyHourLayout, key)
if err != nil || civil.Format(legacyHourLayout) != key {
return time.Time{}, false
}
start, err := time.ParseInLocation(legacyHourLayout, key, loc)
if err != nil {
return time.Time{}, false
}
// A backward step can separate two instances of an hour by other
// hours, and can repeat a whole civil day. Intersect the civil hour
// with every zone interval around the parser's chosen instance.
stop := start.Add(25 * time.Hour)
var end time.Time
for cursor := start.Add(-24 * time.Hour); cursor.Before(stop); {
_, offset := cursor.Zone()
_, zoneEnd := cursor.ZoneBounds()
until := stop
if !zoneEnd.IsZero() && zoneEnd.Before(until) {
until = zoneEnd
}
from := civil.Add(-time.Duration(offset) * time.Second)
to := from.Add(time.Hour)
if from.Before(cursor) {
from = cursor
}
if to.After(until) {
to = until
}
if from.Before(to) && to.After(end) {
end = to
}
cursor = until
}
return end.In(loc), !end.IsZero()
}
// The typed JSON decoder accepts null scalars and repeated keys. Neither
// can prove an hourly count; validate the shape before using zero values.
func validateLegacyJSON(data []byte) error {
dec := json.NewDecoder(bytes.NewReader(data))
dec.UseNumber()
if err := uniqueLegacyValue(dec, false); err != nil {
return err
}
var fields map[string]json.RawMessage
if err := json.NewDecoder(bytes.NewReader(data)).Decode(&fields); err != nil {
return err
}
for _, key := range []string{"blocks_this_hour", "hour_key"} {
value, ok := fields[key]
if !ok || bytes.Equal(bytes.TrimSpace(value), []byte("null")) {
return fmt.Errorf("missing legacy field %s", key)
}
}
return nil
}
func uniqueLegacyValue(dec *json.Decoder, nullOK bool) error {
tok, tokenErr := dec.Token()
if tokenErr != nil {
return tokenErr
}
if tok == nil && !nullOK {
return errors.New("null legacy scalar or array entry")
}
delim, ok := tok.(json.Delim)
if !ok {
return nil
}
switch delim {
case '{':
seen := map[string]bool{}
for dec.More() {
key, keyErr := dec.Token()
if keyErr != nil {
return keyErr
}
name := legacyJSONFieldName(key.(string))
if seen[name] {
return errors.New("repeated legacy field")
}
seen[name] = true
// The writer may encode nil collections and an absent cause.
allowsNull := name == legacyJSONFieldName("ips") || name == legacyJSONFieldName("pending") ||
name == legacyJSONFieldName("cleanup_pending") || name == legacyJSONFieldName("cause")
if valueErr := uniqueLegacyValue(dec, allowsNull); valueErr != nil {
return valueErr
}
}
case '[':
for dec.More() {
if valueErr := uniqueLegacyValue(dec, false); valueErr != nil {
return valueErr
}
}
}
_, closeErr := dec.Token()
return closeErr
}
// Match the typed JSON decoder's case folding when comparing field names.
func legacyJSONFieldName(key string) string {
return strings.Map(func(r rune) rune {
folded := r
for next := unicode.SimpleFold(r); next != r; next = unicode.SimpleFold(next) {
if next < folded {
folded = next
}
}
return folded
}, key)
}
package checks
import (
"crypto/sha256"
"fmt"
)
// loginRecordKey retains session identity even when the syslog prefix is long
// enough to push the PID or client port past the display-details truncation.
func loginRecordKey(line string) string {
return fmt.Sprintf("%x", sha256.Sum256([]byte(line)))
}
package checks
import (
"context"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/systemdrun"
)
// eximQueueTimeout bounds the queue probe. `exim -bpc` walks the spool, so it
// is slower on a deep queue but never long-running.
const eximQueueTimeout = 30 * time.Second
var errEximNotInstalled = errors.New("exim is not installed")
func CheckMailQueue(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
count, err := eximQueueCount(ctx)
if err != nil {
if errors.Is(err, errEximNotInstalled) {
return nil
}
// A queue depth CSM cannot read is a monitoring blind spot, not a
// clean bill of health. Reporting nothing here is what let a fully
// broken probe run unnoticed on a production host for months.
return []alert.Finding{{
Severity: alert.Warning,
Check: "mail_queue_unavailable",
Message: "Exim mail queue depth could not be read",
Details: fmt.Sprintf("Spam-outbreak detection via queue depth is inactive until this succeeds. %v",
err),
}}
}
if count >= cfg.Thresholds.MailQueueCrit {
return []alert.Finding{{
Severity: alert.Critical,
Check: "mail_queue",
Message: fmt.Sprintf("Exim mail queue critical: %d messages", count),
Details: "Possible spam outbreak from compromised account",
}}
}
if count >= cfg.Thresholds.MailQueueWarn {
return []alert.Finding{{
Severity: alert.Warning,
Check: "mail_queue",
Message: fmt.Sprintf("Exim mail queue elevated: %d messages", count),
}}
}
return nil
}
// eximQueueCount returns the number of messages in the Exim queue.
//
// exim opens its own main log for append on every invocation and aborts when it
// cannot. Under the daemon's ProtectSystem=strict sandbox /var/log is read-only,
// so a direct call fails with "Cannot open main log file" and yields no count at
// all. Running the probe as a transient unit hands it to PID 1, outside the
// sandbox. Hosts without systemd-run have no sandbox to escape and call exim
// directly.
func eximQueueCount(parent context.Context) (int, error) {
eximPath, err := cmdExec.LookPath("exim")
if err != nil {
return 0, fmt.Errorf("%w: %v", errEximNotInstalled, err)
}
ctx, cancel := context.WithTimeout(parent, eximQueueTimeout)
defer cancel()
out, err := systemdrun.Run(ctx, cmdExec.LookPath, cmdExec.RunContextStdout, systemdrun.Options{
Pipe: true,
RuntimeMax: eximQueueTimeout,
}, eximPath, "-bpc")
if err != nil {
return 0, fmt.Errorf("exim -bpc: %w", err)
}
return parseEximQueueCount(out)
}
func parseEximQueueCount(out []byte) (int, error) {
trimmed := strings.TrimSpace(string(out))
count, err := strconv.Atoi(trimmed)
if err != nil {
return 0, fmt.Errorf("exim -bpc returned %q, not a queue count", truncateForDetail(trimmed))
}
return count, nil
}
// truncateForDetail keeps an unexpected command output short enough to sit in a
// finding's Details without pasting a whole error page into the alert.
func truncateForDetail(s string) string {
const max = 120
if len(s) <= max {
return s
}
return s[:max] + "..."
}
package checks
import (
"context"
"crypto/sha256"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
// Mail-filter exfiltration detector.
//
// cPanel stores per-mailbox Exim filters at
// /home/<user>/etc/<domain>/<localpart>/filter and domain-wide defaults at
// /etc/vfilters/<domain>. A compromised webmail account is commonly weaponised
// by writing a filter that copies every inbound message to an external dropbox
// while keeping a local copy, so the victim never notices the interception
// (business email compromise). This check parses those Exim filters and scores
// the deliver/save actions for that stealth pattern.
//
// Unlike CheckForwarders (valiases redirects), interception-shaped rules are
// reported even when the filter predates CSM. Newness gating only applies to
// plain external forwards that are frequently legitimate customer
// configuration; severity for copy-forwards is assigned after corroboration.
// filterAction is a single Exim filter action (deliver/save/pipe/finish/...).
type filterAction struct {
verb string
arg string
unseen bool
knownSuppressible bool
matchesAll bool
}
// filterRule is one branch of an Exim filter: the condition that guards it and
// the actions it performs. The unconditional top level is represented as a rule
// with an empty condition.
type filterRule struct {
condition string
matchesAll bool
actions []filterAction
}
// filterMailbox identifies the mailbox a filter file belongs to. localPart is
// "*" for a domain-wide /etc/vfilters file.
type filterMailbox struct {
localPart string
domain string
}
func (m filterMailbox) String() string {
return m.localPart + "@" + m.domain
}
// filterFinding is the scorer's intermediate result before it is turned into an
// alert.Finding (which needs file path and newness context).
type filterFinding struct {
severity alert.Severity
check string
kind string // "exfil" | "forwarder" | "pipe" | "blackhole"
dest string // external destination, when applicable, for correlation
reason string
onlyIfNew bool
// retainsLocalCopy marks an external delivery the mailbox still receives a
// copy of. Alone that is indistinguishable from a user-created forward.
retainsLocalCopy bool
}
// safePipeCommands are cPanel built-in pipe targets that are not attacker code.
// cPanel writes Mailman list aliases as pipes to the 3rdparty mail binaries.
var safePipeCommands = []string{
"/usr/local/cpanel/bin/autorespond",
"/usr/local/cpanel/bin/boxtrapper",
"/usr/local/cpanel/bin/mailman",
"/usr/local/cpanel/3rdparty/mailman/mail/mailman",
"/usr/local/cpanel/3rdparty/mailman/mail/wrapper",
}
// ---------------------------------------------------------------------------
// Parser
// ---------------------------------------------------------------------------
type eximToken struct {
text string
str bool
}
// tokenizeExim splits Exim filter source into tokens. Quoted strings (with \"
// and \\ escapes) become a single string token with the quotes removed;
// parentheses are standalone tokens; everything else is a bareword. Comments
// (# to end of line, outside strings) are dropped.
func tokenizeExim(s string) []eximToken {
var toks []eximToken
runes := []rune(s)
i := 0
for i < len(runes) {
c := runes[i]
switch c {
case ' ', '\t', '\n', '\r':
i++
case '#':
for i < len(runes) && runes[i] != '\n' {
i++
}
case '(', ')':
toks = append(toks, eximToken{text: string(c)})
i++
case '"':
i++
var b strings.Builder
for i < len(runes) && runes[i] != '"' {
if runes[i] == '\\' && i+1 < len(runes) {
i++
}
b.WriteRune(runes[i])
i++
}
if i < len(runes) {
i++ // closing quote
}
toks = append(toks, eximToken{text: b.String(), str: true})
default:
start := i
for i < len(runes) {
r := runes[i]
if r == ' ' || r == '\t' || r == '\n' || r == '\r' || r == '(' || r == ')' || r == '"' || r == '#' {
break
}
i++
}
toks = append(toks, eximToken{text: string(runes[start:i])})
}
}
return toks
}
// renderCondition reconstructs a condition string from its tokens so the
// match-all heuristic can run against text that matches the source form
// (string tokens are re-quoted).
func renderCondition(toks []eximToken) string {
var parts []string
for _, t := range toks {
if t.str {
parts = append(parts, `"`+t.text+`"`)
} else {
parts = append(parts, t.text)
}
}
return strings.Join(parts, " ")
}
var actionVerbs = map[string]bool{
"deliver": true,
"save": true,
"pipe": true,
"finish": true,
"mail": true,
"vacation": true,
}
var actionArgs = map[string]bool{
"deliver": true,
"save": true,
"pipe": true,
"mail": true,
"vacation": true,
}
var controlWords = map[string]bool{
"if": true,
"elif": true,
"else": true,
"endif": true,
"then": true,
"unseen": true,
}
type filterRuleNode struct {
rule filterRule
parent *filterRuleNode
}
// parseEximFilter parses Exim filter source into a flat list of rules, one per
// if/elif/else branch plus one for any unconditional top-level actions. Nested
// branches include ancestor actions so split deliver/save patterns still score
// as one executed branch.
func parseEximFilter(content string) []filterRule {
toks := tokenizeExim(content)
top := &filterRuleNode{rule: filterRule{matchesAll: true}}
stack := []*filterRuleNode{top}
rules := []*filterRuleNode{top}
pendingUnseen := false
i := 0
for i < len(toks) {
t := toks[i]
if t.str {
i++
continue
}
kw := strings.ToLower(t.text)
switch kw {
case "if", "elif":
if kw == "elif" && len(stack) > 1 {
stack = stack[:len(stack)-1]
}
parent := stack[len(stack)-1]
i++
condStart := i
for i < len(toks) && !tokenIs(toks[i], "then") {
i++
}
cond := renderCondition(toks[condStart:i])
if i < len(toks) {
i++ // consume "then"
}
r := &filterRuleNode{
rule: filterRule{condition: cond, matchesAll: conditionMatchesAll(cond)},
parent: parent,
}
rules = append(rules, r)
stack = append(stack, r)
pendingUnseen = false
case "else":
if len(stack) > 1 {
stack = stack[:len(stack)-1]
}
parent := stack[len(stack)-1]
i++
r := &filterRuleNode{
rule: filterRule{condition: "else"},
parent: parent,
}
rules = append(rules, r)
stack = append(stack, r)
pendingUnseen = false
case "endif":
if len(stack) > 1 {
stack = stack[:len(stack)-1]
}
i++
pendingUnseen = false
case "unseen":
pendingUnseen = true
i++
default:
if !actionVerbs[kw] {
i++
continue
}
verb := kw
i++
arg := ""
if actionArgs[verb] && i < len(toks) {
if toks[i].str || isBareActionArg(toks[i]) {
arg = toks[i].text
i++
}
}
cur := stack[len(stack)-1]
cur.rule.actions = append(cur.rule.actions, filterAction{verb: verb, arg: arg, unseen: pendingUnseen})
pendingUnseen = false
}
}
out := make([]filterRule, 0, len(rules))
for _, r := range rules {
if len(r.rule.actions) > 0 {
out = append(out, flattenRuleNode(r))
}
}
return out
}
func tokenIs(t eximToken, word string) bool {
return !t.str && strings.EqualFold(t.text, word)
}
func isBareActionArg(t eximToken) bool {
if t.str {
return true
}
lower := strings.ToLower(t.text)
return !controlWords[lower] && !actionVerbs[lower]
}
func flattenRuleNode(node *filterRuleNode) filterRule {
var chain []*filterRuleNode
for n := node; n != nil; n = n.parent {
chain = append(chain, n)
}
out := filterRule{matchesAll: true}
var conditions []string
for i := len(chain) - 1; i >= 0; i-- {
r := chain[i].rule
if r.condition != "" {
conditions = append(conditions, r.condition)
}
if !r.matchesAll {
out.matchesAll = false
}
out.actions = append(out.actions, r.actions...)
}
out.condition = strings.Join(conditions, " && ")
return out
}
// conditionMatchesAll reports whether a filter condition fires on effectively
// all mail: an unconditional rule, or one that only tests that an address or
// header comparison is true for every normal email address.
func conditionMatchesAll(cond string) bool {
c := strings.TrimSpace(cond)
if c == "" {
return true
}
return tokenExpressionMatchesAll(tokenizeExim(c))
}
func tokenExpressionMatchesAll(toks []eximToken) bool {
toks = trimOuterParens(toks)
if len(toks) == 0 {
return false
}
for _, term := range splitTopLevel(toks, "or") {
if tokenConjunctionMatchesAll(term) {
return true
}
}
return false
}
func tokenConjunctionMatchesAll(toks []eximToken) bool {
toks = trimOuterParens(toks)
if len(toks) == 0 {
return false
}
parts := splitTopLevel(toks, "and")
for _, part := range parts {
if !tokenTermMatchesAll(part) {
return false
}
}
return len(parts) > 0
}
func tokenTermMatchesAll(toks []eximToken) bool {
toks = trimOuterParens(toks)
if len(toks) == 0 {
return false
}
if parts := splitTopLevel(toks, "or"); len(parts) > 1 {
return tokenExpressionMatchesAll(toks)
}
if parts := splitTopLevel(toks, "and"); len(parts) > 1 {
return tokenConjunctionMatchesAll(toks)
}
hasMatchAllComparison := false
for i := 0; i < len(toks); i++ {
if toks[i].str {
continue
}
if strings.EqualFold(toks[i].text, "not") {
return false
}
}
for i := 0; i+1 < len(toks); i++ {
if toks[i].str {
continue
}
op := strings.ToLower(toks[i].text)
if !isAddressComparisonOperator(op) {
continue
}
if !comparisonMatchesAllAddress(op, toks[i+1].text) || !comparisonHasAddressOperand(toks, i) {
return false
}
hasMatchAllComparison = true
}
return hasMatchAllComparison
}
func splitTopLevel(toks []eximToken, word string) [][]eximToken {
var parts [][]eximToken
start := 0
depth := 0
for i, t := range toks {
if t.str {
continue
}
switch t.text {
case "(":
depth++
case ")":
if depth > 0 {
depth--
}
default:
if depth == 0 && strings.EqualFold(t.text, word) {
if start < i {
parts = append(parts, toks[start:i])
}
start = i + 1
}
}
}
if start < len(toks) {
parts = append(parts, toks[start:])
}
return parts
}
func trimOuterParens(toks []eximToken) []eximToken {
for len(toks) >= 2 && tokenIs(toks[0], "(") && tokenIs(toks[len(toks)-1], ")") && outerParensEncloseAll(toks) {
toks = toks[1 : len(toks)-1]
}
return toks
}
func outerParensEncloseAll(toks []eximToken) bool {
depth := 0
for i, t := range toks {
if t.str {
continue
}
switch t.text {
case "(":
depth++
case ")":
depth--
if depth == 0 && i != len(toks)-1 {
return false
}
if depth < 0 {
return false
}
}
}
return depth == 0
}
func isAddressComparisonOperator(op string) bool {
switch op {
case "contains", "matches", "is":
return true
}
return false
}
func comparisonMatchesAllAddress(op, value string) bool {
v := strings.ToLower(strings.TrimSpace(value))
switch op {
case "contains":
return v == "@"
case "matches":
switch v {
case "@", ".*@.*", ".+@.+", "^.*@.*$", "^.+@.+$":
return true
}
case "is":
return v == "*@*"
}
return false
}
func comparisonHasAddressOperand(toks []eximToken, opIndex int) bool {
for i := opIndex - 1; i >= 0; i-- {
if toks[i].str {
continue
}
word := strings.ToLower(toks[i].text)
switch word {
case "and", "or", "then", "else":
return false
case "(", ")":
return false
case "not":
continue
}
if tokenLooksAddressOperand(word) {
return true
}
}
return false
}
func tokenLooksAddressOperand(token string) bool {
addressOperands := []string{
"$thisaddress",
"foranyaddress",
"$sender_address",
"$return_path",
"$header_from",
"$h_from",
"$header_to",
"$h_to",
"$header_cc",
"$h_cc",
"$header_bcc",
"$h_bcc",
"$header_reply-to",
"$h_reply-to",
"$header_sender",
"$h_sender",
"$header_return-path",
"$h_return-path",
}
for _, operand := range addressOperands {
if strings.Contains(token, operand) {
return true
}
}
return false
}
// ---------------------------------------------------------------------------
// Scorer
// ---------------------------------------------------------------------------
// destIsExternal reports whether an Exim deliver destination leaves the local
// mail system. Exim variables ($domain etc.) and same-domain/local-domain
// addresses are not external.
func destIsExternal(dest string, mb filterMailbox, localDomains map[string]bool) bool {
_, dom, ok := splitDeliverDest(dest)
if !ok {
return false
}
if deliverDomainIsLocal(dom, mb, localDomains) {
return false
}
return true
}
// destIsLocalSelf reports whether a deliver destination routes back into the
// local mail system (a self re-delivery that keeps a copy for the victim).
func destIsLocalSelf(dest string, mb filterMailbox, localDomains map[string]bool) bool {
_, dom, ok := splitDeliverDest(dest)
if !ok {
return false
}
return deliverDomainIsLocal(dom, mb, localDomains)
}
func splitDeliverDest(dest string) (string, string, bool) {
clean := strings.Trim(strings.TrimSpace(dest), `"`)
at := strings.LastIndexByte(clean, '@')
if at < 0 || at == len(clean)-1 {
return "", "", false
}
local := strings.Trim(strings.TrimSpace(clean[:at]), `"`)
dom := strings.ToLower(strings.Trim(strings.TrimSpace(clean[at+1:]), `"`))
if local == "" || dom == "" {
return "", "", false
}
return local, dom, true
}
func deliverDomainIsLocal(dom string, mb filterMailbox, localDomains map[string]bool) bool {
d := strings.ToLower(strings.TrimSpace(dom))
if d == "$domain" || d == "${domain}" {
return true
}
if d == strings.ToLower(mb.domain) {
return true
}
return localDomains[d]
}
func isSafePipe(cmd string) bool {
first := firstPipeCommandWord(cmd)
for _, s := range safePipeCommands {
if first == s {
return true
}
}
return false
}
func firstPipeCommandWord(cmd string) string {
// Exim's transport_set_up_command uses byte whitespace and only treats
// a quote at the start of an argument specially. Shell-style quote
// concatenation or Unicode trimming can turn a different path into a
// trusted executable here.
const whitespace = " \t\r\n\v\f"
s := strings.TrimLeft(strings.TrimPrefix(strings.TrimLeft(cmd, whitespace), "|"), whitespace)
if s == "" || strings.IndexByte(s, 0) >= 0 {
return ""
}
quote := s[0]
if quote != '\'' && quote != '"' {
if end := strings.IndexAny(s, whitespace); end >= 0 {
return s[:end]
}
return s
}
var b strings.Builder
for i := 1; i < len(s); i++ {
c := s[i]
if c == quote {
return b.String()
}
if quote == '"' && c == '\\' {
if i+1 == len(s) {
return ""
}
var consumed int
c, consumed = pipeCommandEscape(s[i+1:])
i += consumed
if c == 0 {
return ""
}
}
b.WriteByte(c)
}
// Incomplete quoting is not evidence of a trusted command.
return ""
}
// pipeCommandEscape follows Exim's string_interpret_escape: up to three
// octal digits, up to two hex digits, C control escapes, or a literal byte.
// s begins immediately after the backslash and is nonempty.
func pipeCommandEscape(s string) (byte, int) {
switch s[0] {
case 'b':
return '\b', 1
case 'f':
return '\f', 1
case 'n':
return '\n', 1
case 'r':
return '\r', 1
case 't':
return '\t', 1
case 'v':
return '\v', 1
}
base := byte(8)
start := 0
if s[0] == 'x' {
base, start = 16, 1
} else if s[0] < '0' || s[0] > '7' {
return s[0], 1
}
var value byte
i := start
for ; i < len(s) && i < 3; i++ {
c := s[i] | 0x20
var digit byte
switch {
case c >= '0' && c <= '9':
digit = c - '0'
case c >= 'a' && c <= 'f':
digit = c - 'a' + 10
default:
return value, i
}
if digit >= base {
break
}
value = value*base + digit
}
return value, i
}
// scoreFilterRules evaluates one mailbox's parsed filter rules and returns the
// dangerous patterns found. Suppression entries in known (format
// "local@domain: dest") drop matching expected plain destinations and Sieve
// :copy redirects. Other stealth patterns remain non-suppressible.
func scoreFilterRules(rules []filterRule, mb filterMailbox, localDomains map[string]bool, known []string) []filterFinding {
var out []filterFinding
seen := map[string]bool{}
add := func(f filterFinding) {
key := f.kind + "|" + f.dest
if seen[key] {
// Multiple rules in one file can target the same destination. If
// any occurrence prevents local delivery, keep that stronger
// evidence instead of letting an earlier copy-forward hide it.
if f.kind == "exfil" && !f.retainsLocalCopy {
for i := range out {
if out[i].kind == f.kind && out[i].dest == f.dest {
out[i] = f
break
}
}
}
return
}
if f.kind == "forwarder" && seen["exfil|"+f.dest] {
return
}
if f.kind == "exfil" && seen["forwarder|"+f.dest] {
delete(seen, "forwarder|"+f.dest)
for i := range out {
if out[i].kind == "forwarder" && out[i].dest == f.dest {
out = append(out[:i], out[i+1:]...)
break
}
}
}
seen[key] = true
out = append(out, f)
}
type externalDelivery struct {
dest string
unseen bool
knownSuppressible bool
matchesAll bool
}
for _, r := range rules {
var external []externalDelivery
hasLocalCopy := false
hasDevNull := false
hasMatchAllDevNull := false
for _, a := range r.actions {
switch a.verb {
case "deliver":
switch {
case destIsExternal(a.arg, mb, localDomains):
external = append(external, externalDelivery{
dest: a.arg,
unseen: a.unseen,
knownSuppressible: a.knownSuppressible,
matchesAll: a.matchesAll,
})
case destIsLocalSelf(a.arg, mb, localDomains):
hasLocalCopy = true
}
case "save":
if strings.TrimSpace(a.arg) == "/dev/null" {
hasDevNull = true
hasMatchAllDevNull = hasMatchAllDevNull || a.matchesAll
} else {
hasLocalCopy = true
}
case "pipe":
if !isSafePipe(a.arg) {
add(filterFinding{
severity: alert.Critical,
check: "email_filter_pipe",
kind: "pipe",
dest: a.arg,
reason: fmt.Sprintf("filter pipes mail to a command: %s", a.arg),
})
}
}
}
if len(external) > 0 {
for _, delivery := range external {
matchesAll := r.matchesAll || delivery.matchesAll
stealth := hasLocalCopy || hasDevNull || matchesAll || delivery.unseen
// An explicit local delivery still happens alongside /dev/null.
// Only an implicit keep created by unseen/:copy is canceled by a
// destructive action in the same rule.
retainsLocalCopy := hasLocalCopy || (delivery.unseen && !hasDevNull)
knownAllowed := !stealth || (delivery.knownSuppressible && !hasDevNull)
if knownAllowed && IsKnownForwarder(mb.localPart, mb.domain, delivery.dest, known) {
continue
}
if stealth {
add(filterFinding{
severity: alert.Critical,
check: "email_filter_exfil",
kind: "exfil",
dest: delivery.dest,
reason: stealthReason(retainsLocalCopy, hasDevNull, matchesAll),
retainsLocalCopy: retainsLocalCopy,
})
} else {
add(filterFinding{
severity: alert.High,
check: "email_filter_forwarder",
kind: "forwarder",
dest: delivery.dest,
reason: "filter forwards mail to an external address",
onlyIfNew: true,
})
}
}
continue
}
if hasDevNull && (r.matchesAll || hasMatchAllDevNull) {
add(filterFinding{
severity: alert.High,
check: "email_filter_blackhole",
kind: "blackhole",
reason: "filter discards all mail to /dev/null",
})
}
}
return out
}
func stealthReason(retainsLocalCopy, devNull, matchAll bool) string {
switch {
case retainsLocalCopy:
if !matchAll {
return "filter sends matching mail to an external address while keeping a local copy"
}
return "filter copies every message to an external address while keeping a local copy"
case devNull:
if matchAll {
return "filter forwards all mail externally and discards the local copy to hide it"
}
return "filter forwards mail externally and discards the local copy to hide it"
case matchAll:
return "filter forwards all mail to an external address"
}
return "filter forwards mail to an external address"
}
// ---------------------------------------------------------------------------
// Check
// ---------------------------------------------------------------------------
// mailboxFromFilterPath derives the mailbox from a filter file path. Per-mailbox
// filters live at /home/<user>/etc/<domain>/<localpart>/filter; domain-wide
// filters at /etc/vfilters/<domain>.
func mailboxFromFilterPath(path string) filterMailbox {
if dir := filepath.Dir(path); dir == "/etc/vfilters" {
return filterMailbox{localPart: "*", domain: filepath.Base(path)}
}
parts := strings.Split(path, "/")
for i := 0; i+2 < len(parts); i++ {
if parts[i] == "etc" && parts[i+1] != "" && parts[i+2] != "" {
return filterMailbox{localPart: parts[i+2], domain: parts[i+1]}
}
}
return filterMailbox{localPart: "*", domain: filepath.Base(filepath.Dir(path))}
}
// CheckMailFilters scans per-mailbox and domain-wide Exim filters for BEC-style
// exfiltration rules. Throttled to PasswordCheckIntervalMin, like the forwarder
// audit it complements.
func CheckMailFilters(ctx context.Context, cfg *config.Config, st *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
db := store.Global()
if db == nil {
if !shouldReportMailFilterStoreUnavailable(st, cfg) {
return nil
}
// Without the store the whole check is inoperative (hashes and the
// throttle live there). Say so instead of looking like a clean host.
return []alert.Finding{{
Severity: alert.Warning,
Check: "email_mail_filters",
Message: "Mail filter audit skipped: state store unavailable",
Timestamp: time.Now(),
}}
}
// A host upgraded from an Exim-only release already has the shared
// mail-filter throttle marker but no Sieve baseline. Run once immediately
// so the new source is inventoried instead of waiting a full audit interval.
// Explicit account scans also bypass the host-wide throttle; they write only
// their per-file hashes and never establish either shared baseline below.
if !ForceAll && AccountFromContext(ctx) == "" && db.GetMetaString("email:mailsieve_last_refresh") != "" {
if last := db.GetMetaString("email:mailfilter_last_refresh"); last != "" {
if ts, err := time.Parse(time.RFC3339, last); err == nil {
interval := time.Duration(cfg.EmailProtection.PasswordCheckIntervalMin) * time.Minute
if time.Since(ts) < interval {
return nil
}
}
}
}
if ctx.Err() != nil {
return nil
}
localDomains := loadLocalDomains()
var files []string
baselineComplete := true
if perMailbox, err := homeGlob(ctx, "etc", "*", "*", "filter"); err == nil {
files = append(files, perMailbox...)
} else {
baselineComplete = false
}
if AccountFromContext(ctx) == "" {
if vfilters, err := osFS.Glob("/etc/vfilters/*"); err == nil {
files = append(files, vfilters...)
} else {
baselineComplete = false
}
}
maxFiles := accountScanMaxFiles(ctx, cfg)
baselineComplete = baselineComplete && scanCoversAllFiles(files, maxFiles)
ranked := rankPathsByMtimeDesc(ctx, files, maxFiles)
if ctx.Err() != nil {
return nil
}
var collected []mailFilterPending
for _, path := range ranked {
if ctx.Err() != nil {
applyMailFilterCorroboration(collected)
return findingsFromPending(collected)
}
data, err := osFS.ReadFile(path)
if err != nil {
baselineComplete = false
continue
}
currentHash := sha256Hex(data)
isNew := forwarderFileIsNew(db, "email:mailfilter_last_refresh", "mailfilter:"+path, currentHash)
if err := db.SetForwarderHash("mailfilter:"+path, currentHash); err != nil {
baselineComplete = false
}
mb := mailboxFromFilterPath(path)
rules := parseEximFilter(string(data))
collected = append(collected, mailFilterPendings(path, mb, rules, localDomains, cfg.EmailProtection.KnownForwarders, isNew, mailFilterMechanismExim)...)
}
sievePending, sieveComplete := scanSieveMailFilters(ctx, db, cfg, localDomains, maxFiles)
collected = append(collected, sievePending...)
baselineComplete = baselineComplete && sieveComplete
applyMailFilterCorroboration(collected)
if ctx.Err() != nil {
return nil
}
// Only a full scan establishes the baseline / refreshes the throttle:
// an account-scoped scan hashes one account's files, and marking it
// complete would make the next full scan treat every other account's
// existing filters as newly created.
baselineExists := db.GetMetaString("email:mailfilter_last_refresh") != ""
if AccountFromContext(ctx) == "" && (sieveComplete || db.GetMetaString("email:mailsieve_last_refresh") != "") {
_ = db.SetMetaString("email:mailsieve_last_refresh", time.Now().Format(time.RFC3339))
}
if AccountFromContext(ctx) == "" && (baselineComplete || baselineExists) {
_ = db.SetMetaString("email:mailfilter_last_refresh", time.Now().Format(time.RFC3339))
}
return findingsFromPending(collected)
}
// mailFilterPendings scores one parsed filter/sieve file and turns each
// dangerous pattern into a pending finding. Shared by the Exim filter and Sieve
// scan loops so both mechanisms report identically.
func mailFilterPendings(path string, mb filterMailbox, rules []filterRule, localDomains map[string]bool, known []string, isNew bool, mechanism mailFilterMechanism) []mailFilterPending {
var out []mailFilterPending
for _, ff := range scoreFilterRules(rules, mb, localDomains, known) {
if ff.onlyIfNew && !isNew {
continue
}
out = append(out, mailFilterPending{
finding: alert.Finding{
Severity: ff.severity,
Check: ff.check,
Message: fmt.Sprintf("%s: %s", mb.String(), ff.reason),
Details: mailFilterDetails(mb, path, ff),
FilePath: path,
Domain: mb.domain,
Mailbox: mailboxField(mb),
},
dest: ff.dest,
mailbox: mb.String(),
mechanism: mechanism,
retainsLocalCopy: ff.retainsLocalCopy,
})
}
return out
}
// scanSieveMailFilters scans per-mailbox Sieve scripts (the rules webmail
// actually executes) for the same BEC exfil patterns the Exim filter audit
// covers. Returns the pending findings plus whether the scan saw every file it
// meant to (a partial scan must not establish the baseline).
func scanSieveMailFilters(ctx context.Context, db *store.DB, cfg *config.Config, localDomains map[string]bool, maxFiles int) ([]mailFilterPending, bool) {
type activePointer struct {
path string
hash string
}
complete := true
var files []string
seen := make(map[string]bool)
activePaths := make(map[string]bool)
activePointers := make(map[string][]activePointer)
addFile := func(path string) {
if !seen[path] {
seen[path] = true
files = append(files, path)
}
}
if scripts, err := homeGlob(ctx, "mail", "*", "*", "sieve", "*.sieve"); err == nil {
for _, path := range scripts {
addFile(path)
}
} else {
complete = false
}
if active, err := homeGlob(ctx, "mail", "*", "*", ".dovecot.sieve"); err == nil {
for _, path := range active {
// The active file normally links to a source already included by the
// sieve/*.sieve glob. Remove that alias before quota accounting so it
// cannot crowd the real source out of every capped scan. A link to an
// unusual target must itself be scanned; on any metadata error, retain
// the path and let ReadFile make the final decision.
if info, statErr := osFS.Lstat(path); statErr == nil && info.Mode()&os.ModeSymlink != 0 {
if target, linkErr := osFS.Readlink(path); linkErr == nil {
if !filepath.IsAbs(target) {
target = filepath.Join(filepath.Dir(path), target)
}
target = filepath.Clean(target)
if seen[target] && mailboxFromSievePath(path) == mailboxFromSievePath(target) {
activePaths[target] = true
activePointers[target] = append(activePointers[target], activePointer{
path: path,
hash: sha256Hex([]byte(target)),
})
continue
}
}
}
addFile(path)
activePaths[path] = true
}
} else {
complete = false
}
complete = complete && scanCoversAllFiles(files, maxFiles)
rankedByMtime := rankPathsByMtimeDesc(ctx, files, 0)
ranked := make([]string, 0, len(rankedByMtime))
for _, path := range rankedByMtime {
if activePaths[path] {
ranked = append(ranked, path)
}
}
for _, path := range rankedByMtime {
if !activePaths[path] {
ranked = append(ranked, path)
}
}
if maxFiles > 0 && len(ranked) > maxFiles {
recordAccountScanTruncatedPaths(ctx, ranked[maxFiles:], maxFiles)
ranked = ranked[:maxFiles]
}
var out []mailFilterPending
reportedContent := make(map[string]bool)
for _, path := range ranked {
if ctx.Err() != nil {
return out, false
}
data, err := osFS.ReadFile(path)
if err != nil {
complete = false
continue
}
currentHash := sha256Hex(data)
isNew := forwarderFileIsNew(db, "email:mailsieve_last_refresh", "mailsieve:"+path, currentHash)
for _, pointer := range activePointers[path] {
if forwarderFileIsNew(db, "email:mailsieve_last_refresh", "mailsieve-active:"+pointer.path, pointer.hash) {
isNew = true
}
if err := db.SetForwarderHash("mailsieve-active:"+pointer.path, pointer.hash); err != nil {
complete = false
}
}
if err := db.SetForwarderHash("mailsieve:"+path, currentHash); err != nil {
complete = false
}
mb := mailboxFromSievePath(path)
contentKey := mb.String() + "\x00" + currentHash
rules := parseSieveFilter(string(data))
pending := mailFilterPendings(path, mb, rules, localDomains, cfg.EmailProtection.KnownForwarders, isNew, mailFilterMechanismSieve)
if len(pending) > 0 && !reportedContent[contentKey] {
reportedContent[contentKey] = true
out = append(out, pending...)
}
}
return out, complete
}
func shouldReportMailFilterStoreUnavailable(st *state.Store, cfg *config.Config) bool {
if st == nil {
return true
}
return st.ShouldRunThrottled("email_mail_filters_store_unavailable", mailFilterAuditIntervalMin(cfg))
}
func mailFilterAuditIntervalMin(cfg *config.Config) int {
if cfg == nil || cfg.EmailProtection.PasswordCheckIntervalMin <= 0 {
return 1440
}
return cfg.EmailProtection.PasswordCheckIntervalMin
}
func mailFilterDetails(mb filterMailbox, path string, ff filterFinding) string {
var b strings.Builder
fmt.Fprintf(&b, "Mailbox: %s\nDomain: %s\nFile: %s\n", mb.String(), mb.domain, path)
if ff.dest != "" {
fmt.Fprintf(&b, "Destination: %s\n", ff.dest)
}
b.WriteString(ff.reason)
return b.String()
}
func mailboxField(mb filterMailbox) string {
if mb.localPart == "*" {
return ""
}
return mb.String()
}
func sha256Hex(data []byte) string {
h := sha256.Sum256(data)
return fmt.Sprintf("%x", h[:])
}
type mailFilterMechanism string
const (
mailFilterMechanismExim mailFilterMechanism = "exim"
mailFilterMechanismSieve mailFilterMechanism = "sieve"
)
// mailFilterPending is an in-flight finding plus the fields needed for the
// corroboration passes before findings are emitted.
type mailFilterPending struct {
finding alert.Finding
dest string
mailbox string
mechanism mailFilterMechanism
// retainsLocalCopy marks an external delivery the mailbox still receives a
// copy of, which is indistinguishable from a user-created forward.
retainsLocalCopy bool
uncorroboratedCopyForward bool
}
// downgradeUncorroboratedCopyExfil lowers an exfil finding to Warning when its
// only stealth evidence is that the forward keeps a local copy. That is exactly
// what a webmail "forward and keep a copy" rule produces, so by itself it cannot
// tell an interception apart from the owner forwarding their own mail. A second
// forwarding mechanism on the same mailbox keeps it Critical, as does mail the
// mailbox never receives; cross-account reuse re-escalates separately.
func downgradeUncorroboratedCopyExfil(collected []mailFilterPending) {
// Count only interception-shaped rules: a plain selective forwarder is
// ordinary mail routing and cannot corroborate anything. Destinations are
// deliberately not part of the key, because planting redundant persistence
// is the signature even when each mechanism aims at a different drop.
mechanisms := map[string]map[mailFilterMechanism]bool{}
for _, p := range collected {
if p.finding.Check != "email_filter_exfil" {
continue
}
if mechanisms[p.mailbox] == nil {
mechanisms[p.mailbox] = map[mailFilterMechanism]bool{}
}
mechanisms[p.mailbox][p.mechanism] = true
}
for i := range collected {
p := &collected[i]
if !p.retainsLocalCopy {
continue
}
if len(mechanisms[p.mailbox]) > 1 {
continue
}
p.finding.Severity = alert.Warning
p.uncorroboratedCopyForward = true
}
}
func applyMailFilterCorroboration(collected []mailFilterPending) {
downgradeUncorroboratedCopyExfil(collected)
// Campaign evidence must run last so it can re-escalate copy-forwards.
annotateCrossAccount(collected)
for i := range collected {
p := &collected[i]
if !p.uncorroboratedCopyForward || p.finding.Severity != alert.Warning {
continue
}
p.finding.Details += "\nThe mailbox still receives this mail and nothing else corroborates the forward, so it is reported for review rather than as a confirmed interception. Confirm with the account owner that the forward is theirs."
}
}
// annotateCrossAccount marks exfil findings whose external destination appears
// across two or more distinct mailboxes -- a strong campaign signal.
func annotateCrossAccount(collected []mailFilterPending) {
byDest := map[string]map[string]bool{}
for _, p := range collected {
if p.finding.Check != "email_filter_exfil" || p.dest == "" {
continue
}
destKey := mailFilterDestinationKey(p.dest)
if byDest[destKey] == nil {
byDest[destKey] = map[string]bool{}
}
byDest[destKey][p.mailbox] = true
}
for i := range collected {
if collected[i].finding.Check != "email_filter_exfil" {
continue
}
dest := collected[i].dest
boxes := byDest[mailFilterDestinationKey(dest)]
if len(boxes) < 2 {
continue
}
others := make([]string, 0, len(boxes))
for b := range boxes {
if b != collected[i].mailbox {
others = append(others, b)
}
}
sort.Strings(others)
collected[i].finding.Severity = alert.Critical
collected[i].finding.Details += fmt.Sprintf(
"\nCross-account: the same destination %s is used by %d mailboxes (also %s). This indicates a coordinated campaign.",
dest, len(boxes), strings.Join(others, ", "))
}
}
func mailFilterDestinationKey(dest string) string {
localPart, domain, ok := splitDeliverDest(dest)
if !ok {
return dest
}
return localPart + "@" + domain
}
func findingsFromPending(collected []mailFilterPending) []alert.Finding {
out := make([]alert.Finding, 0, len(collected))
for _, p := range collected {
out = append(out, p.finding)
}
return out
}
package checks
import (
"context"
"fmt"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/eximlog"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
)
const perAccountMailThreshold = 100 // emails per recent log window
// mailLogTailLinesDefault is the built-in fallback for how many trailing
// lines of /var/log/exim_mainlog CheckMailPerAccount tails per cycle.
// Operator override: cfg.Thresholds.MailLogTailLines.
const mailLogTailLinesDefault = 500
// CheckMailPerAccount counts recent Exim arrivals per envelope-sender domain.
// Ownership requires the same verified submitter across the entire count.
func CheckMailPerAccount(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
tailLines := mailLogTailLinesDefault
if cfg != nil && cfg.Thresholds.MailLogTailLines > 0 {
tailLines = cfg.Thresholds.MailLogTailLines
}
lines := tailFile(platform.EximMainLog, tailLines)
// Keep the existing sender-domain volume calculation.
type volume struct {
count int
owner string
}
counts := make(map[string]volume)
for _, line := range lines {
domain := ArrivalSenderDomain(line)
if domain == "" {
continue
}
identity := eximlog.Submitter(line)
owner := ""
if strings.Contains(identity, "@") {
owner = MailOwner(identity)
} else {
owner = HostingAccountForUser(identity)
}
v := counts[domain]
if v.count == 0 {
v.owner = owner
} else if v.owner != owner {
v.owner = ""
}
v.count++
counts[domain] = v
}
// Keep the sender-domain volume signal, including unverified messages.
// Only a unanimous verified submitter establishes ownership of its count.
for domain, v := range counts {
if v.count >= perAccountMailThreshold {
message := fmt.Sprintf("High email volume from %s: %d messages in recent log", domain, v.count)
if v.owner != "" {
message += fmt.Sprintf(" (account %s)", v.owner)
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "mail_per_account",
Message: message,
Details: "Possible spam outbreak or compromised email account",
Domain: domain,
TenantID: v.owner,
})
}
}
return findings
}
// ArrivalSenderDomain returns the envelope-sender domain of an Exim arrival
// record, or "" for other records, bounces, cPanel system mail and senders
// without a domain.
func ArrivalSenderDomain(line string) string {
_, sender, ok := eximlog.Arrival(line)
if !ok || sender == "<>" || strings.HasPrefix(sender, "cPanel") {
return ""
}
at := strings.LastIndex(sender, "@")
if at < 0 {
return ""
}
return sender[at+1:]
}
package checks
import (
"net/mail"
"sort"
"strconv"
"strings"
)
// Dovecot/Sieve exfiltration detector.
//
// cPanel webmail (Roundcube) stores per-mailbox Sieve scripts at
// /home/<user>/mail/<domain>/<localpart>/sieve/*.sieve, with the active script
// pointed to by .dovecot.sieve in the mailbox root. A compromised webmail
// account is commonly weaponised with a Sieve rule that redirects every message
// to an external dropbox while keeping a local copy, so the victim never
// notices the interception (business email compromise). Unlike the Exim filter
// path, Sieve is what actually executes for webmail-managed rules, so an
// exfil-only Exim audit misses it entirely.
//
// Sieve is a different grammar from Exim's filter language, so it has its own
// tokenizer and parser here, but it lowers into the same filterRule model the
// Exim scorer already understands (scoreFilterRules): a `redirect` becomes a
// deliver, `:copy` maps to Exim's `unseen` (deliver externally while the
// implicit keep preserves a local copy), `keep`/`fileinto` become a local
// save, and `discard` becomes a /dev/null save. Stealth scoring and reasons
// therefore stay aligned across both mechanisms; an exact KnownForwarders
// entry may additionally acknowledge Roundcube's ordinary :copy forwards.
type sieveTokenKind int
const (
sieveWord sieveTokenKind = iota
sieveString
sieveTag // :copy, :contains, ...
sievePunct
)
type sieveToken struct {
text string
kind sieveTokenKind
}
// tokenizeSieve splits Sieve source into tokens. Quoted strings (with \" and \\
// escapes) become a single string token; tags (:copy) become tag tokens;
// braces, brackets, parens, commas and semicolons are punctuation; line (#) and
// block (/* */) comments are dropped.
func tokenizeSieve(s string) []sieveToken {
var toks []sieveToken
runes := []rune(s)
i := 0
for i < len(runes) {
c := runes[i]
switch {
case c == ' ' || c == '\t' || c == '\n' || c == '\r':
i++
case c == '#':
for i < len(runes) && runes[i] != '\n' {
i++
}
case c == '/' && i+1 < len(runes) && runes[i+1] == '*':
i += 2
for i+1 < len(runes) && (runes[i] != '*' || runes[i+1] != '/') {
i++
}
i += 2
if i > len(runes) {
i = len(runes)
}
case isSieveTextLiteralStart(runes, i):
var text string
text, i = consumeSieveTextLiteral(runes, i)
toks = append(toks, sieveToken{text: decodeSieveEncodedCharacters(text), kind: sieveString})
case c == '"':
i++
var b strings.Builder
for i < len(runes) && runes[i] != '"' {
if runes[i] == '\\' && i+1 < len(runes) {
i++
}
b.WriteRune(runes[i])
i++
}
if i < len(runes) {
i++ // closing quote
}
toks = append(toks, sieveToken{text: decodeSieveEncodedCharacters(b.String()), kind: sieveString})
case c == '{' || c == '}' || c == '[' || c == ']' || c == '(' || c == ')' || c == ',' || c == ';':
toks = append(toks, sieveToken{text: string(c), kind: sievePunct})
i++
case c == ':':
start := i
i++
for i < len(runes) && isSieveIdentRune(runes[i]) {
i++
}
toks = append(toks, sieveToken{text: string(runes[start:i]), kind: sieveTag})
default:
start := i
for i < len(runes) && isSieveIdentRune(runes[i]) {
i++
}
if i == start {
// Unknown punctuation (e.g. a stray operator); skip one rune so
// the tokenizer always makes progress on hostile input.
i++
continue
}
toks = append(toks, sieveToken{text: string(runes[start:i]), kind: sieveWord})
}
}
return toks
}
func isSieveTextLiteralStart(runes []rune, start int) bool {
const prefix = "text:"
if len(runes)-start < len(prefix) {
return false
}
for i, want := range prefix {
got := runes[start+i]
if got >= 'A' && got <= 'Z' {
got += 'a' - 'A'
}
if got != want {
return false
}
}
next := start + len(prefix)
return next == len(runes) || runes[next] == ' ' || runes[next] == '\t' ||
runes[next] == '\r' || runes[next] == '\n' || runes[next] == '#'
}
// consumeSieveTextLiteral consumes Sieve's text: multi-line string form. Its
// contents must stay opaque to the parser: prose in a vacation response can
// legitimately contain text such as `redirect "address";` without being an
// executable action. Unterminated literals consume the rest of the input.
func consumeSieveTextLiteral(runes []rune, start int) (string, int) {
i := start + len("text:")
for i < len(runes) && runes[i] != '\n' {
i++
}
if i == len(runes) {
return "", i
}
i++
var b strings.Builder
for i < len(runes) {
lineStart := i
for i < len(runes) && runes[i] != '\n' {
i++
}
lineEnd := i
if lineEnd > lineStart && runes[lineEnd-1] == '\r' {
lineEnd--
}
line := runes[lineStart:lineEnd]
if len(line) == 1 && line[0] == '.' {
if i < len(runes) {
i++
}
return b.String(), i
}
if len(line) >= 2 && line[0] == '.' && line[1] == '.' {
line = line[1:]
}
b.WriteString(string(line))
if i < len(runes) {
b.WriteByte('\n')
i++
}
}
return b.String(), i
}
// decodeSieveEncodedCharacters handles the standard encoded-character
// extension. Without this normalization, `${hex:40}` can hide the @ in a
// redirect destination from the external-address scorer.
func decodeSieveEncodedCharacters(s string) string {
var out strings.Builder
for i := 0; i < len(s); {
if s[i] != '$' || i+2 >= len(s) || s[i+1] != '{' {
out.WriteByte(s[i])
i++
continue
}
endRel := strings.IndexByte(s[i+2:], '}')
if endRel < 0 {
out.WriteString(s[i:])
break
}
end := i + 2 + endRel
body := s[i+2 : end]
colon := strings.IndexByte(body, ':')
if colon < 0 {
out.WriteByte(s[i])
i++
continue
}
decoded, ok := decodeSieveCharacterSequence(strings.ToLower(strings.TrimSpace(body[:colon])), body[colon+1:])
if !ok {
out.WriteString(s[i : end+1])
} else {
out.WriteString(decoded)
}
i = end + 1
}
return out.String()
}
func decodeSieveCharacterSequence(kind, payload string) (string, bool) {
fields := strings.Fields(payload)
if len(fields) == 0 {
return "", false
}
var out strings.Builder
for _, field := range fields {
if !isSieveHex(field) {
return "", false
}
switch kind {
case "hex":
if len(field) > 2 {
return "", false
}
value, err := strconv.ParseUint(field, 16, 8)
if err != nil {
return "", false
}
out.WriteByte(byte(value))
case "unicode":
if len(field) > 6 {
return "", false
}
value, err := strconv.ParseUint(field, 16, 32)
if err != nil || value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff) {
return "", false
}
out.WriteRune(rune(value))
default:
return "", false
}
}
return out.String(), true
}
func isSieveHex(s string) bool {
if s == "" {
return false
}
for i := range len(s) {
c := s[i]
switch {
case c >= '0' && c <= '9':
case c >= 'a' && c <= 'f':
case c >= 'A' && c <= 'F':
default:
return false
}
}
return true
}
func isSieveIdentRune(r rune) bool {
return r == '_' || r == '.' || r == '-' ||
(r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')
}
func isSievePunct(t sieveToken, text string) bool {
return t.kind == sievePunct && t.text == text
}
var sieveActions = map[string]bool{
"redirect": true,
"keep": true,
"fileinto": true,
"discard": true,
"stop": true,
"return": true,
}
type sieveOrderedAction struct {
position int
action filterAction
}
type sieveBranchChain struct {
parent *sieveNode
allPreviousNever bool
previousAlways bool
continuePossible bool
stopPossible bool
}
type sieveNode struct {
rule filterRule
parent *sieveNode
branch *sieveBranchChain
actions []sieveOrderedAction
reachable bool
flow sieveTruth
}
func (c *sieveBranchChain) recordBranch(flow sieveTruth) {
c.continuePossible = c.continuePossible || flow != sieveNever
c.stopPossible = c.stopPossible || flow != sieveAlways
}
func finishSieveBranchChain(c *sieveBranchChain) {
if c == nil || c.parent == nil {
return
}
// With no final else, messages that match none of the branch tests flow
// through the chain unchanged.
if !c.previousAlways {
c.continuePossible = true
}
continuation := sieveConditional
switch {
case c.continuePossible && !c.stopPossible:
continuation = sieveAlways
case !c.continuePossible && c.stopPossible:
continuation = sieveNever
}
c.parent.flow = composeSieveFlow(c.parent.flow, continuation)
}
func composeSieveFlow(first, second sieveTruth) sieveTruth {
if first == sieveNever || second == sieveNever {
return sieveNever
}
if first == sieveAlways {
return second
}
return sieveConditional
}
// parseSieveFilter parses Sieve source into the flat filterRule list the Exim
// scorer consumes: one rule per if/elsif/else branch plus the unconditional top
// level, with ancestor actions folded into each branch so a keep in an outer
// block still pairs with a redirect in an inner branch.
func parseSieveFilter(content string) []filterRule {
toks := tokenizeSieve(content)
top := &sieveNode{rule: filterRule{matchesAll: true}, reachable: true, flow: sieveAlways}
stack := []*sieveNode{top}
nodes := []*sieveNode{top}
var pendingChain *sieveBranchChain
i := 0
for i < len(toks) {
t := toks[i]
continuesBranch := pendingChain != nil && pendingChain.parent == stack[len(stack)-1] &&
t.kind == sieveWord && (strings.EqualFold(t.text, "elsif") || strings.EqualFold(t.text, "else"))
if pendingChain != nil && !continuesBranch {
finishSieveBranchChain(pendingChain)
pendingChain = nil
}
if isSievePunct(t, "}") {
if len(stack) > 1 {
closed := stack[len(stack)-1]
if closed.reachable {
closed.branch.recordBranch(closed.flow)
}
pendingChain = closed.branch
stack = stack[:len(stack)-1]
} else {
pendingChain = nil
}
i++
continue
}
if t.kind == sieveWord && (strings.EqualFold(t.text, "if") || strings.EqualFold(t.text, "elsif") || strings.EqualFold(t.text, "else")) {
keyword := strings.ToLower(t.text)
isElse := keyword == "else"
parent := stack[len(stack)-1]
chain := pendingChain
if keyword == "if" || chain == nil || chain.parent != parent {
chain = &sieveBranchChain{parent: parent, allPreviousNever: keyword == "if"}
}
pendingChain = nil
i++
condStart := i
for i < len(toks) && !isSievePunct(toks[i], "{") {
i++
}
branchReachable := !chain.previousAlways
branchMatchesAll := chain.allPreviousNever
if isElse {
chain.previousAlways = true
chain.allPreviousNever = false
} else {
truth := sieveTestTruthValue(toks[condStart:i])
branchReachable = branchReachable && truth != sieveNever
branchMatchesAll = branchMatchesAll && truth == sieveAlways
chain.previousAlways = chain.previousAlways || truth == sieveAlways
chain.allPreviousNever = chain.allPreviousNever && truth == sieveNever
}
if i < len(toks) {
i++ // consume "{"
}
n := &sieveNode{
rule: filterRule{matchesAll: parent.rule.matchesAll && parent.flow == sieveAlways && branchMatchesAll},
parent: parent,
branch: chain,
reachable: parent.reachable && parent.flow != sieveNever && branchReachable,
flow: sieveAlways,
}
nodes = append(nodes, n)
stack = append(stack, n)
continue
}
pendingChain = nil
if t.kind == sieveWord && sieveActions[strings.ToLower(t.text)] {
verb := strings.ToLower(t.text)
position := i
i++
var args []sieveToken
for i < len(toks) && !isSievePunct(toks[i], ";") {
if isSievePunct(toks[i], "{") || isSievePunct(toks[i], "}") {
break
}
args = append(args, toks[i])
i++
}
if i >= len(toks) || !isSievePunct(toks[i], ";") {
continue
}
i++
cur := stack[len(stack)-1]
if cur.flow == sieveNever {
continue
}
if act, ok := sieveActionToFilter(verb, args); ok {
act.matchesAll = cur.rule.matchesAll && cur.flow == sieveAlways
cur.actions = append(cur.actions, sieveOrderedAction{position: position, action: act})
if act.verb == "finish" {
cur.flow = sieveNever
}
}
continue
}
i++
}
finishSieveBranchChain(pendingChain)
out := make([]filterRule, 0, len(nodes))
for _, n := range nodes {
if n.reachable && len(n.actions) > 0 {
out = append(out, flattenSieveNode(n))
}
}
return out
}
// sieveActionToFilter lowers one Sieve action into the Exim filterAction the
// scorer understands. Returns ok=false for actions that carry no exfil signal.
func sieveActionToFilter(verb string, args []sieveToken) (filterAction, bool) {
switch verb {
case "redirect":
dest := lastSieveString(args)
if dest == "" {
return filterAction{}, false
}
if address, err := mail.ParseAddress(dest); err == nil {
dest = address.Address
}
// :copy keeps the implicit local copy alongside the forward -- the same
// stealth signal as Exim's `unseen`.
copyRedirect := sieveHasTag(args, ":copy")
return filterAction{
verb: "deliver",
arg: dest,
unseen: copyRedirect,
knownSuppressible: copyRedirect,
}, true
case "fileinto":
dest := lastSieveString(args)
if dest == "" {
return filterAction{}, false
}
return filterAction{verb: "save", arg: dest}, true
case "keep":
return filterAction{verb: "save", arg: "$home/mail/INBOX"}, true
case "discard":
return filterAction{verb: "save", arg: "/dev/null"}, true
case "stop", "return":
return filterAction{verb: "finish"}, true
}
return filterAction{}, false
}
func lastSieveString(args []sieveToken) string {
for i := len(args) - 1; i >= 0; i-- {
if args[i].kind == sieveString {
return args[i].text
}
}
return ""
}
func sieveHasTag(args []sieveToken, tag string) bool {
for _, a := range args {
if a.kind == sieveTag && strings.EqualFold(a.text, tag) {
return true
}
}
return false
}
func flattenSieveNode(node *sieveNode) filterRule {
var chain []*sieveNode
for n := node; n != nil; n = n.parent {
chain = append(chain, n)
}
var actions []sieveOrderedAction
for _, n := range chain {
actions = append(actions, n.actions...)
}
sort.SliceStable(actions, func(i, j int) bool {
return actions[i].position < actions[j].position
})
// Sieve actions carry their own match-all reachability because a prior
// conditional stop can make later actions selective within the same scope.
// Exim rules retain their rule-wide matchesAll representation.
out := filterRule{}
for _, ordered := range actions {
out.actions = append(out.actions, ordered.action)
if ordered.action.verb == "finish" {
break
}
}
return out
}
type sieveTruth int
const (
sieveNever sieveTruth = iota
sieveConditional
sieveAlways
)
// sieveTestMatchesAll reports whether a Sieve test fires on effectively all
// mail. It evaluates anyof/allof structure instead of combining unrelated
// operands from different subtests, and stays conservative on malformed or
// excessively nested input.
func sieveTestMatchesAll(test []sieveToken) bool {
return sieveTestTruthValue(test) == sieveAlways
}
func sieveTestTruthValue(test []sieveToken) sieveTruth {
return sieveExpressionTruth(test, 0)
}
func sieveExpressionTruth(test []sieveToken, depth int) sieveTruth {
const maxSieveTestDepth = 256
if depth >= maxSieveTestDepth {
return sieveConditional
}
test, ok := trimSieveOuterParens(test)
if !ok || len(test) == 0 || test[0].kind != sieveWord {
return sieveConditional
}
keyword := strings.ToLower(test[0].text)
switch keyword {
case "true":
if len(test) == 1 {
return sieveAlways
}
case "false":
if len(test) == 1 {
return sieveNever
}
case "not":
inner := sieveExpressionTruth(test[1:], depth+1)
switch inner {
case sieveAlways:
return sieveNever
case sieveNever:
return sieveAlways
default:
return sieveConditional
}
case "anyof", "allof":
parts, ok := sieveCallArguments(test[1:])
if !ok || len(parts) == 0 {
return sieveConditional
}
if keyword == "anyof" {
allNever := true
for _, part := range parts {
truth := sieveExpressionTruth(part, depth+1)
if truth == sieveAlways {
return sieveAlways
}
allNever = allNever && truth == sieveNever
}
if allNever {
return sieveNever
}
return sieveConditional
}
allAlways := true
for _, part := range parts {
truth := sieveExpressionTruth(part, depth+1)
if truth == sieveNever {
return sieveNever
}
allAlways = allAlways && truth == sieveAlways
}
if allAlways {
return sieveAlways
}
return sieveConditional
case "address", "header":
if sieveAddressTestMatchesAll(test) {
return sieveAlways
}
}
return sieveConditional
}
func trimSieveOuterParens(test []sieveToken) ([]sieveToken, bool) {
matchingClose := make([]int, len(test))
for i := range matchingClose {
matchingClose[i] = -1
}
var stack []int
for i, tok := range test {
switch {
case isSievePunct(tok, "("):
stack = append(stack, i)
case isSievePunct(tok, ")"):
if len(stack) == 0 {
return nil, false
}
open := stack[len(stack)-1]
stack = stack[:len(stack)-1]
matchingClose[open] = i
}
}
if len(stack) != 0 {
return nil, false
}
left, right := 0, len(test)-1
for left < right && isSievePunct(test[left], "(") && matchingClose[left] == right {
left++
right--
}
return test[left : right+1], true
}
func sieveCallArguments(test []sieveToken) ([][]sieveToken, bool) {
if len(test) < 2 || !isSievePunct(test[0], "(") || !isSievePunct(test[len(test)-1], ")") {
return nil, false
}
return splitSieveTopLevel(test[1:len(test)-1], ",")
}
func splitSieveTopLevel(test []sieveToken, separator string) ([][]sieveToken, bool) {
var parts [][]sieveToken
start := 0
parenDepth := 0
listDepth := 0
for i, tok := range test {
switch {
case isSievePunct(tok, "("):
parenDepth++
case isSievePunct(tok, ")"):
parenDepth--
case isSievePunct(tok, "["):
listDepth++
case isSievePunct(tok, "]"):
listDepth--
case parenDepth == 0 && listDepth == 0 && isSievePunct(tok, separator):
if start == i {
return nil, false
}
parts = append(parts, test[start:i])
start = i + 1
}
if parenDepth < 0 || listDepth < 0 {
return nil, false
}
}
if parenDepth != 0 || listDepth != 0 || start >= len(test) {
return nil, false
}
parts = append(parts, test[start:])
return parts, true
}
func sieveAddressTestMatchesAll(test []sieveToken) bool {
testKind := strings.ToLower(test[0].text)
matchType := ""
addressPart := ":all"
addressPartSeen := false
comparatorSeen := false
var operands [][]string
for i := 1; i < len(test); {
tok := test[i]
switch {
case tok.kind == sieveTag:
tag := strings.ToLower(tok.text)
switch tag {
case ":contains", ":matches", ":is":
if matchType != "" {
return false
}
matchType = tag
case ":all", ":localpart", ":domain", ":detail":
if testKind != "address" {
return false
}
if addressPartSeen {
return false
}
addressPartSeen = true
addressPart = tag
case ":comparator":
if comparatorSeen {
return false
}
comparatorSeen = true
i++
if i >= len(test) || test[i].kind != sieveString {
return false
}
default:
// Extensions such as :index/:last and :mime/:anychild alter
// which header occurrence or MIME part is examined. Even From
// is not guaranteed to exist in that narrowed scope, so it is
// unsafe to classify the test as match-all. Unknown modifiers
// stay conservative for the same reason.
return false
}
i++
case tok.kind == sieveString:
operands = append(operands, []string{tok.text})
i++
case isSievePunct(tok, "["):
var values []string
i++
for i < len(test) && !isSievePunct(test[i], "]") {
if test[i].kind == sieveString {
values = append(values, test[i].text)
} else if !isSievePunct(test[i], ",") {
return false
}
i++
}
if i >= len(test) || len(values) == 0 {
return false
}
i++
operands = append(operands, values)
default:
return false
}
}
if len(operands) != 2 {
return false
}
// The first string-list is the header name and the final string-list is
// the match key. Only From is reliably present and address-bearing on every
// normal message; Subject/To and arbitrary headers are not match-all.
hasFrom := false
for _, header := range operands[0] {
if strings.EqualFold(strings.TrimSpace(header), "from") {
hasFrom = true
break
}
}
if !hasFrom {
return false
}
for _, key := range operands[1] {
switch matchType {
case ":contains":
if key == "" || (addressPart == ":all" && key == "@") {
return true
}
case ":matches":
if key == "*" || (addressPart == ":all" && key == "*@*") {
return true
}
}
}
return false
}
// mailboxFromSievePath derives the mailbox from a sieve script path of the form
// /home/<user>/mail/<domain>/<localpart>/... .
func mailboxFromSievePath(path string) filterMailbox {
parts := strings.Split(path, "/")
for i := 0; i+4 < len(parts); i++ {
if parts[i] == "home" && parts[i+1] != "" && parts[i+2] == "mail" && parts[i+3] != "" && parts[i+4] != "" {
return filterMailbox{localPart: parts[i+4], domain: parts[i+3]}
}
}
return filterMailbox{localPart: "*", domain: ""}
}
package checks
import (
"path/filepath"
"sort"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/platform"
)
// modsecOffLinePrefix starts every line this annotation writes, which keeps a
// second pass over the same finding from repeating it.
const modsecOffLinePrefix = "ModSecurity is disabled for "
// annotateModsecDisabledMalware adds a context line to each malware finding
// whose account or serving site has ModSecurity switched off. The host-wide
// modsec_disabled_vhost finding lists such scopes, but on a host with many of
// them the compromised account is easy to miss; here the fact sits on the
// finding about that account. Only Context changes: it is outside the
// finding's identity, so turning ModSecurity off or on neither re-alerts nor
// undoes a dismissal.
//
// Reads are targeted: nothing at all without a malware finding, then the
// vhost map once and the ModSecurity switches of only the accounts and sites
// that carry one.
func annotateModsecDisabledMalware(info platform.Info, findings []alert.Finding) {
if !info.IsCPanel() {
return
}
type candidate struct {
index int
account string
}
var candidates []candidate
for i, f := range findings {
if !modsecCorrelatedMalware(f) {
continue
}
if account := extractAccountFromFinding(f); validAccountName.MatchString(account) {
candidates = append(candidates, candidate{index: i, account: account})
}
}
if len(candidates) == 0 {
return
}
vhostsOf := make(map[string][]vhost)
var aliases map[string][]string
if data, err := osFS.ReadFile(userdataDomainsPath); err == nil {
vhosts, _ := parseUserdataDomainRootsChecked(string(data))
for _, vh := range vhosts {
vhostsOf[vh.user] = append(vhostsOf[vh.user], vh)
}
aliases = vhostAliasSets(string(data))
}
domainsOf := make([][]string, len(candidates))
// The targeted lookup takes each site as an account and domain pair.
var sites []alert.Finding
for i, c := range candidates {
domainsOf[i] = servingDomains(findings[c.index], vhostsOf[c.account])
sites = append(sites, alert.Finding{TenantID: c.account})
for _, domain := range domainsOf[i] {
sites = append(sites, alert.Finding{TenantID: c.account, Domain: domain})
}
}
cov := vpCoverage{aliases: aliases, disabled: modsecDisabledScopesForFindings(info, sites, aliases)}
if len(cov.disabled) == 0 {
return
}
for i, c := range candidates {
var lines []string
seen := make(map[string]bool)
// The empty domain asks for the account-wide switch alone.
for _, domain := range append([]string{""}, domainsOf[i]...) {
reason, source := cov.inertReason(c.account, domain)
if reason == "" {
continue
}
line := reason + " (set in " + source + "): requests there are not filtered and leave no modsec audit record."
if !seen[line] {
seen[line] = true
lines = append(lines, line)
}
}
if len(lines) == 0 {
continue
}
f := &findings[c.index]
if f.Context != "" {
f.Context += "\n"
}
f.Context += strings.Join(lines, "\n")
}
}
// modsecCorrelatedMalware selects malware findings that may describe a
// compromise. A carried-forward finding is a stored snapshot the latest-state
// merge keeps as it is, so reading for it would be wasted; one already
// annotated is left as it is.
func modsecCorrelatedMalware(f alert.Finding) bool {
if f.ScanCarryForward || strings.Contains("\n"+f.Context, "\n"+modsecOffLinePrefix) {
return false
}
return malwareFileCheck(f.Check)
}
// malwareFileCheck reports whether check is a malware check whose findings
// may describe a compromise.
func malwareFileCheck(check string) bool {
switch correlationClassOf(check) {
case CorrelationSecurityEvent, CorrelationMalwareArtifact:
default:
return false
}
info, ok := LookupCheck(check)
return ok && info.Category == CategoryMalware
}
// servingDomains returns the domains that can serve the finding: the one it
// names, plus every one of the account's vhosts whose document root holds its
// file. Nested roots serve a file under each of them.
func servingDomains(f alert.Finding, accountVhosts []vhost) []string {
set := make(map[string]bool)
if domain := cleanDomlogDomain(f.Domain); domain != "" {
set[domain] = true
}
if filepath.IsAbs(f.FilePath) {
path := filepath.Clean(f.FilePath)
for _, vh := range accountVhosts {
if strings.HasPrefix(path, vh.docroot+string(filepath.Separator)) {
set[vh.domain] = true
}
}
}
domains := make([]string, 0, len(set))
for domain := range set {
domains = append(domains, domain)
}
sort.Strings(domains)
return domains
}
package checks
import (
"context"
"encoding/json"
"fmt"
"net"
"strconv"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
func CheckOutboundConnections(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
findings, sockets := scanOutboundConnections(cfg, store)
for inode, process := range socketProcesses(ctx, sockets) {
for _, i := range sockets[inode] {
findings[i].Details += ", " + process
}
}
return findings
}
// A socket snapshot and its first-seen update must stay ordered across scans.
// Process lookup happens after saving so it cannot delay another snapshot.
var c2ScanMu sync.Mutex
func scanOutboundConnections(cfg *config.Config, store *state.Store) ([]alert.Finding, map[string][]int) {
c2ScanMu.Lock()
defer c2ScanMu.Unlock()
var findings []alert.Finding
// Parse /proc/net/tcp for established connections
// Format: sl local_address rem_address st ...
// local_address = IP:port we are listening/connecting from
// rem_address = IP:port of the remote end
data, err := osFS.ReadFile("/proc/net/tcp")
if err != nil {
return nil, nil
}
lines := strings.Split(string(data), "\n")
listeners := collectListenSockets(lines, false)
// Separate boots; sampled inode reuse keeps its earlier sighting.
bootID, _ := osFS.ReadFile("/proc/sys/kernel/random/boot_id")
boot := strings.TrimSpace(string(bootID))
stream := ""
if boot != "" {
stream = alert.SocketStream(boot)
}
now := time.Now()
firstSeen := loadC2SocketsSeen(store, boot)
seen := make(map[string]int64)
sockets := make(map[string][]int)
for _, line := range lines {
fields := strings.Fields(line)
if len(fields) < 4 {
continue
}
if fields[0] == "sl" {
continue
}
// State 01 = ESTABLISHED
if fields[3] != "01" {
continue
}
firstFinding := len(findings)
// Ownership belongs to this kernel row, not to a later PID lookup.
// Keep detections visible when an incomplete row has no usable UID.
var uid uint64
var uidKnown bool
if len(fields) >= 8 {
var err error
uid, err = strconv.ParseUint(fields[7], 10, 32)
uidKnown = err == nil
}
localAddr := fields[1]
remoteAddr := fields[2]
localIP, localPort := parseHexAddr(localAddr)
remoteIP, remotePort := parseHexAddr(remoteAddr)
if remoteIP == "" || remoteIP == "127.0.0.1" || remoteIP == "0.0.0.0" {
continue
}
var inode string
if len(fields) >= 10 {
if n, err := strconv.ParseUint(fields[9], 10, 64); err == nil && n > 0 {
inode = fields[9]
}
}
// Check remote IP against C2 blocklist. A local port with a listener
// may be the accept side, or a process listening on its own source
// port to look like one: it still reports, but is no outbound evidence.
for _, blocked := range cfg.C2Blocklist {
if remoteIP == blocked {
outbound := !listeners.has(net.ParseIP(localIP), localPort)
details := fmt.Sprintf("Local socket: %s:%d", localIP, localPort)
identity := fmt.Sprintf("%s:%d>%s:%d", localIP, localPort, remoteIP, remotePort)
if inode != "" {
details += ", socket inode " + inode
identity += " inode " + inode
}
if uidKnown {
details += fmt.Sprintf(", uid %d", uid)
identity += fmt.Sprintf(" uid %d", uid)
}
if !outbound {
details += ", local port also listening"
}
f := alert.Finding{
Severity: alert.Critical,
Check: "c2_connection",
Message: fmt.Sprintf("Connection to known C2 IP: %s:%d", remoteIP, remotePort),
Details: details,
SourceIP: remoteIP,
// Details carry process and listener text that can change
// while the same connection stays open.
DedupKey: identity,
}
if stream != "" && inode != "" {
// A later scan of the same socket rereads the same
// observation: it keeps the time first seen.
at, known := firstSeen[inode]
if !known {
at = now.UnixNano()
}
seen[inode] = at
if outbound {
f.Observation = alert.Observation{Producer: string(ProducerConnectionScan), Stream: stream, Cursor: inode, ObservedAt: time.Unix(0, at).UTC()}
f.Attribution = admission.AttributionSocketDestination
}
}
if inode != "" {
sockets[inode] = append(sockets[inode], len(findings))
}
findings = append(findings, f)
}
}
// Check if OUR LOCAL port is a backdoor port (we're listening on it)
// This catches backdoor listeners, not clients connecting from high ports
for _, bp := range cfg.BackdoorPorts {
if localPort == bp {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "backdoor_port",
Message: fmt.Sprintf("Listening on known backdoor port %d, connected from %s:%d", localPort, remoteIP, remotePort),
SourceIP: remoteIP,
})
}
}
// Also check if we're connecting OUT to a backdoor port on a remote host
// (e.g. reverse shell calling back to attacker's listener)
// Skip if our local port is a known service (the remote port is just
// the client's ephemeral port, not a backdoor listener)
knownServicePorts := map[int]bool{
21: true, 25: true, 26: true, 53: true, 80: true, 110: true,
143: true, 443: true, 465: true, 587: true, 993: true, 995: true,
2082: true, 2083: true, 2086: true, 2087: true, 2095: true, 2096: true,
3306: true, 4190: true,
}
if !knownServicePorts[localPort] {
for _, bp := range cfg.BackdoorPorts {
if remotePort == bp {
if isInfraIP(remoteIP, cfg.InfraIPs) {
continue
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "backdoor_port_outbound",
Message: fmt.Sprintf("Outbound connection to backdoor port: %s:%d", remoteIP, remotePort),
Details: fmt.Sprintf("Local port: %d", localPort),
SourceIP: remoteIP,
})
}
}
}
if uidKnown {
for i := firstFinding; i < len(findings); i++ {
AttributeSocketOwner(&findings[i], uint32(uid))
}
}
}
saveC2SocketsSeen(store, boot, seen)
return findings, sockets
}
// c2SocketsSeenKey keeps when the C2 scan first saw each matched socket of
// the current boot, so a scan after a restart names the same observation.
const c2SocketsSeenKey = "_c2_sockets_seen"
type c2SocketsSeen struct {
Boot string `json:"boot"`
Seen map[string]int64 `json:"seen"`
}
func loadC2SocketsSeen(store *state.Store, boot string) map[string]int64 {
if store == nil || boot == "" {
return nil
}
raw, ok := store.GetRaw(c2SocketsSeenKey)
if !ok {
return nil
}
var saved c2SocketsSeen
if json.Unmarshal([]byte(raw), &saved) != nil || saved.Boot != boot {
return nil
}
return saved.Seen
}
// saveC2SocketsSeen keeps only this scan's matched inodes. Reuse after an
// absent scan gets a new time, which cannot replace retained evidence.
func saveC2SocketsSeen(store *state.Store, boot string, seen map[string]int64) {
if store == nil {
return
}
if len(seen) == 0 {
if _, ok := store.GetRaw(c2SocketsSeenKey); ok {
store.DeleteRaw(c2SocketsSeenKey)
}
return
}
data, err := json.Marshal(c2SocketsSeen{Boot: boot, Seen: seen})
if err != nil {
return
}
store.SetRaw(c2SocketsSeenKey, string(data))
}
// socketProcesses names a process holding each socket inode in want, best
// effort: the first process found, with its executable when readable.
func socketProcesses(ctx context.Context, want map[string][]int) map[string]string {
found := make(map[string]string)
if len(want) == 0 {
return found
}
fds, _ := osFS.Glob("/proc/*/fd/*")
for _, fd := range fds {
if ctx.Err() != nil || len(found) == len(want) {
break
}
pid := strings.Split(strings.TrimPrefix(fd, "/proc/"), "/")[0]
if _, err := strconv.ParseUint(pid, 10, 32); err != nil {
continue
}
target, err := osFS.Readlink(fd)
if err != nil {
continue
}
inode, ok := strings.CutPrefix(target, "socket:[")
if !ok {
continue
}
inode, ok = strings.CutSuffix(inode, "]")
if _, wanted := want[inode]; !ok || !wanted || found[inode] != "" {
continue
}
process := "process " + pid
if exe, err := osFS.Readlink("/proc/" + pid + "/exe"); err == nil {
process += " " + strconv.Quote(exe)
}
found[inode] = process
}
return found
}
// IsInfraIP reports whether ip is infrastructure the daemon must never act
// against: an operator infra entry (CIDR or address) or a Cloudflare edge.
// One implementation serves scans and the realtime path alike.
func IsInfraIP(ip string, infraNets []string) bool { return isInfraIP(ip, infraNets) }
// IsCloudflareIP reports whether ip is inside the published Cloudflare
// ranges the daemon last refreshed.
func IsCloudflareIP(ip net.IP) bool { return isCloudflareIP(ip) }
func isInfraIP(ip string, infraNets []string) bool {
parsed := net.ParseIP(ip)
if parsed == nil {
return false
}
for _, entry := range infraNets {
entry = strings.TrimSpace(entry)
if entry == "" {
continue
}
// Try CIDR first (e.g. "10.0.0.0/8")
_, network, err := net.ParseCIDR(entry)
if err == nil {
if network.Contains(parsed) {
return true
}
continue
}
// Fall back to plain IP match (e.g. "1.2.3.4")
if entryIP := net.ParseIP(entry); entryIP != nil && entryIP.Equal(parsed) {
return true
}
}
// Also check Cloudflare IPs - these must never be blocked/challenged
// because blocking a CF edge IP blocks thousands of legitimate users.
// The detection/alert still fires; only the nftables action is skipped.
if isCloudflareIP(parsed) {
return true
}
return false
}
var (
cfNets []*net.IPNet
cfNetsMu sync.RWMutex
)
// SetCloudflareNets updates the cached Cloudflare IP ranges.
// Called by the daemon after fetching CF IPs.
func SetCloudflareNets(cidrs []string) {
var nets []*net.IPNet
for _, cidr := range cidrs {
_, network, err := net.ParseCIDR(cidr)
if err == nil {
nets = append(nets, network)
}
}
cfNetsMu.Lock()
cfNets = nets
cfNetsMu.Unlock()
}
func isCloudflareIP(ip net.IP) bool {
cfNetsMu.RLock()
defer cfNetsMu.RUnlock()
for _, network := range cfNets {
if network.Contains(ip) {
return true
}
}
return false
}
func parseHexAddr(hexAddr string) (string, int) {
parts := strings.Split(hexAddr, ":")
if len(parts) != 2 {
return "", 0
}
hexIP := parts[0]
hexPort := parts[1]
if len(hexIP) != 8 {
return "", 0
}
// Parse little-endian hex IP
var octets [4]byte
for i := 0; i < 4; i++ {
val := hexToByte(hexIP[6-2*i : 8-2*i])
octets[i] = val
}
ip := net.IPv4(octets[0], octets[1], octets[2], octets[3]).String()
port, ok := parseProcNetHexPort(hexPort)
if !ok {
return ip, 0
}
return ip, port
}
func parseProcNetHexPort(hexPort string) (int, bool) {
if len(hexPort) != 4 {
return 0, false
}
port := 0
for i := 0; i < len(hexPort); i++ {
c := hexPort[i]
if !isHexDigit(c) {
return 0, false
}
port = port*16 + hexVal(c)
}
return port, true
}
func hexToByte(s string) byte {
if len(s) != 2 {
return 0
}
// #nosec G115 -- hexVal returns 0..15; (h<<4)|h fits in a byte (0..255).
return byte(hexVal(s[0])<<4 | hexVal(s[1]))
}
func hexVal(c byte) int {
switch {
case c >= '0' && c <= '9':
return int(c - '0')
case c >= 'a' && c <= 'f':
return int(c-'a') + 10
case c >= 'A' && c <= 'F':
return int(c-'A') + 10
}
return 0
}
package checks
import (
"path/filepath"
"sort"
"strings"
)
// nginxDirective is one simple or block directive of an nginx
// configuration, tagged with the file that set it.
type nginxDirective struct {
Name string
Args []string
File string
Module string
Context string
Block bool
}
// nginxDirectives returns every directive of the nginx configuration at
// path, with include targets expanded in place, and whether every file was
// read and parsed. Relative includes resolve against the main file's
// directory, nginx's configuration prefix.
func nginxDirectives(path string) ([]nginxDirective, bool) {
prefix, _ := filepath.Split(path)
if prefix == "" {
prefix = "."
}
r := nginxReader{prefix: strings.TrimSuffix(prefix, string(filepath.Separator)), active: make(map[string]bool), complete: true}
out := r.read(nil, path, 0, "", "")
return out, r.complete
}
type nginxReader struct {
prefix string
active map[string]bool
complete bool
}
func (r *nginxReader) read(dst []nginxDirective, path string, depth int, module, context string) []nginxDirective {
if depth > apacheIncludeMaxDepth || r.active[path] {
r.complete = false
return dst
}
r.active[path] = true
defer delete(r.active, path)
data, err := osFS.ReadFile(path)
if err != nil {
r.complete = false
return dst
}
directives, ok := parseNginxDirectives(string(data), path)
if !ok {
r.complete = false
}
for _, d := range directives {
if d.Module == "" {
d.Module = module
}
if d.Context == "" {
d.Context = context
}
if d.Name != "include" {
dst = append(dst, d)
continue
}
if len(d.Args) != 1 || d.Block {
r.complete = false
continue
}
target := d.Args[0]
if !filepath.IsAbs(target) {
target = joinWebConfigPath(r.prefix, target)
}
if !strings.ContainsAny(target, "*?[") {
dst = r.read(dst, target, depth+1, d.Module, d.Context)
continue
}
// An unreadable directory is not an empty one.
matches, listed := checkedWebConfigGlob(target)
if !listed {
r.complete = false
}
sort.Strings(matches)
for _, m := range matches {
// nginx cannot read a directory as a configuration file.
if info, statErr := osFS.Stat(m); statErr != nil || info.IsDir() {
r.complete = false
continue
}
dst = r.read(dst, m, depth+1, d.Module, d.Context)
}
}
return dst
}
// parseNginxDirectives splits one file into directives with nginx's token
// rules (ngx_conf_read_token): a comment, a quote or a block brace starts
// only at a token boundary, a backslash escapes the next character, and
// "${" stays inside a variable. ok is false wherever nginx would refuse the
// file: an unterminated quote, statement or block, a stray brace or
// semicolon, text glued to a closing quote, or an empty directive name.
func parseNginxDirectives(text, file string) ([]nginxDirective, bool) {
var out []nginxDirective
var words []string
modules := []string{""}
contexts := []string{""}
depth, start := 0, 0
lastSpace, needSpace, comment, escaped, variable := true, false, false, false, false
var quote byte
ok := true
statement := func(block bool) {
if len(words) == 0 || words[0] == "" {
ok = false
} else {
out = append(out, nginxDirective{
Name: words[0], Args: words[1:], File: file,
Module: modules[depth], Context: contexts[depth], Block: block,
})
}
if block {
module := modules[depth]
if len(words) > 0 && (words[0] == "http" || words[0] == "stream") {
module = words[0]
}
modules = append(modules, module)
context := ""
if len(words) > 0 {
context = words[0]
}
contexts = append(contexts, context)
depth++
}
words = nil
}
for i := 0; i < len(text); i++ {
c := text[i]
space := c == ' ' || c == '\t' || c == '\r' || c == '\n'
switch {
case comment:
comment = c != '\n'
case escaped:
escaped = false
case needSpace:
needSpace = false
switch {
case space:
lastSpace = true
case c == ';' || c == '{':
statement(c == '{')
lastSpace = true
case c == ')':
// ngx_conf_read_token starts a new token with the closing
// parenthesis after a quoted condition operand.
start, lastSpace = i, false
default:
return out, false
}
case lastSpace:
if space {
continue
}
start = i
switch c {
case ';', '{':
statement(c == '{')
continue
case '}':
if len(words) != 0 || depth == 0 {
ok = false
} else {
modules = modules[:depth]
contexts = contexts[:depth]
depth--
}
continue
case '#':
comment = true
continue
case '\\':
escaped = true
case '"', '\'':
quote, start = c, i+1
case '$':
variable = true
}
lastSpace = false
default:
if c == '{' && variable {
continue
}
variable = false
switch {
case c == '\\':
escaped = true
continue
case c == '$':
variable = true
continue
case quote != 0:
if c != quote {
continue
}
quote, needSpace = 0, true
case space || c == ';' || c == '{':
lastSpace = true
default:
continue
}
words = append(words, nginxUnescape(text[start:i]))
if c == ';' || c == '{' {
statement(c == '{')
}
}
}
return out, ok && depth == 0 && len(words) == 0 && lastSpace && !needSpace && quote == 0 && !escaped
}
// nginxUnescape decodes a token as nginx copies it: \", \', \\, \t, \r
// and \n are decoded; any other backslash is kept.
func nginxUnescape(token string) string {
if !strings.Contains(token, "\\") {
return token
}
var b strings.Builder
for i := 0; i < len(token); i++ {
if token[i] == '\\' && i+1 < len(token) {
switch token[i+1] {
case '"', '\'', '\\':
i++
case 't':
b.WriteByte('\t')
i++
continue
case 'r':
b.WriteByte('\r')
i++
continue
case 'n':
b.WriteByte('\n')
i++
continue
}
}
b.WriteByte(token[i])
}
return b.String()
}
package checks
import (
"net/netip"
"net/url"
"strings"
"time"
"unicode"
"github.com/pidginhost/csm/internal/crawlid"
)
// These are parser resource bounds, not qualified server acceptance limits.
// A caller must bound the raw line, including log-escape expansion, separately.
const (
patternMaxTarget = 8192
patternMaxUA = 512
patternMaxReferer = 2048
patternMaxHost = 253
patternMaxExtension = 256
patternMaxMethod = 32
)
type refererState uint8
const (
refererMissing refererState = iota
refererDash
refererEmpty
refererMalformed
refererValid
)
// Target and RefererHost are transient: derive identity/evidence and discard
// them before persistence. Query values and raw Referer must never be stored.
type patternRecord struct {
RemoteIP string
Time time.Time
TimeOK bool
Method string
Target string
TargetOverflow bool
TargetInvalid bool
Status int
Referer refererState
RefererHost string
UserAgent string
UAOverflow bool
XFF string
XFFUnusable bool
// XFFPartial reports that client-side entries left of XFF were dropped.
XFFPartial bool
}
// parsePatternRecord accepts common/combined logs with optional quoted
// extensions. Unsupported structure is a coverage gap, not a quiet record.
func parsePatternRecord(line string) (patternRecord, bool) {
var rec patternRecord
sp := strings.IndexByte(line, ' ')
if sp <= 0 {
return rec, false
}
rec.RemoteIP = line[:sp]
stampText, rest, ok := patternSplitHeader(line[sp+1:])
if !ok {
return rec, false
}
if stamp, err := time.Parse("02/Jan/2006:15:04:05 -0700", stampText); err == nil {
rec.Time, rec.TimeOK = stamp, true
}
request, rest, ok := patternQuotedField(rest)
if !ok {
return rec, false
}
// Split log-level separators first: a decoded \x20 belongs to the target,
// not to request-line framing. Never reinterpret a target prefix as whole.
method, afterMethod, hasMethod := strings.Cut(request, " ")
target, proto, hasProto := strings.Cut(afterMethod, " ")
var methodOver bool
rec.Method, methodOver = patternDecodeField(method, patternMaxMethod)
if !hasMethod || !hasProto || methodOver || !patternValidMethod(rec.Method) ||
!patternValidRawTarget(target) || !patternHTTPVersion(proto) {
rec.TargetInvalid = true
} else {
rec.Target, rec.TargetOverflow = patternDecodeField(target, patternMaxTarget)
if rec.TargetOverflow {
rec.Target = ""
} else if _, _, supported := crawlid.SplitTarget(rec.Target); !supported {
rec.Target, rec.TargetInvalid = "", true
}
}
statusTok, rest := patternToken(rest)
if len(statusTok) != 3 || !patternDigitsOnly(statusTok) || statusTok[0] < '1' || statusTok[0] > '5' {
return rec, false
}
rec.Status = atoiSafe(statusTok)
bytesTok, rest := patternToken(rest)
if bytesTok != "-" && !patternDigitsOnly(bytesTok) {
return rec, false
}
if strings.TrimSpace(rest) == "" {
return rec, true
}
refRaw, rest, ok := patternQuotedField(rest)
if !ok {
return rec, false
}
ref, refOver := patternDecodeField(refRaw, patternMaxReferer)
if !patternRefererSyntax(refRaw) {
rec.Referer = refererMalformed
} else {
rec.Referer, rec.RefererHost = patternRefererFromValue(ref, refOver)
}
if strings.TrimSpace(rest) == "" {
return rec, true
}
ua, uaOver, rest, ok := scanPatternQuoted(rest, patternMaxUA)
if !ok {
return rec, false
}
rec.UserAgent, rec.UAOverflow = ua, uaOver
// Consume leading separators only. Re-scanning a trailing whitespace run
// for every extension would make a long line quadratic to parse.
sawXFF := false
for rest = strings.TrimLeft(rest, " \t"); rest != ""; rest = strings.TrimLeft(rest, " \t") {
raw, tail, fieldOK := patternQuotedField(rest)
if !fieldOK {
return rec, false
}
// Decode the whole field so a bound drops client entries from the
// left of a forwarded-for list, never the proxy's own on the right.
value, _ := patternDecodeField(raw, len(raw))
suffix, partial, isList := patternXFFSuffix(value)
switch {
case value == "" || value == "-":
case isList:
// A second list or a proxy entry beyond the bound leaves no
// unique authority.
if sawXFF || suffix == "" {
rec.XFFUnusable = true
}
sawXFF = true
rec.XFF, rec.XFFPartial = suffix, partial
case len(value) > patternMaxExtension || !patternVhostExtension(value):
// A field that does not end in an address can still be malformed
// proxy evidence. Only a recognized vhost field is safe to ignore.
rec.XFFUnusable = true
}
rest = tail
}
if rec.XFFUnusable {
rec.XFF, rec.XFFPartial = "", false
} else {
// Detach the bounded suffix from any discarded client-controlled
// prefix in the decoded extension's backing allocation.
rec.XFF = strings.Clone(rec.XFF)
}
return rec, true
}
// patternXFFSuffix reads a forwarded-for list from the right. A trusted proxy
// appends the peer it saw, so the right end is its evidence and everything to
// the left is client-supplied. It keeps the longest run of valid addresses
// that ends the list and fits the bound; partial reports dropped entries.
// isList is false when the last entry is not an address.
func patternXFFSuffix(v string) (suffix string, partial, isList bool) {
start := -1
for end := len(v); ; {
comma := strings.LastIndexByte(v[:end], ',')
entry := v[comma+1 : end]
addr, err := netip.ParseAddr(strings.TrimSpace(entry))
if err != nil || addr.Zone() != "" {
break
}
isList = true
at := end - len(strings.TrimLeftFunc(entry, unicode.IsSpace))
if len(v)-at > patternMaxExtension {
break
}
start = at
if comma < 0 {
return v[start:], false, true
}
end = comma
}
if start < 0 {
return "", false, isList
}
return v[start:], true, true
}
func patternHTTPVersion(s string) bool {
v, over := patternDecodeField(s, len("HTTP/1.1"))
if over || !strings.HasPrefix(v, "HTTP/") {
return false
}
switch len(v) {
case len("HTTP/1.1"):
return v[5] >= '0' && v[5] <= '9' && v[6] == '.' && v[7] >= '0' && v[7] <= '9'
case len("HTTP/2"):
// HTTP/2 and HTTP/3 have no request line, so the server writes the
// version itself and may omit the minor digit. HTTP/1.x never does.
return v[5] >= '2' && v[5] <= '9'
}
return false
}
// patternSplitHeader separates "<ident> <user> [<time>]" from the request.
// The remote user is logged unescaped apart from quotes, backslashes and
// non-printable bytes, and a client chooses it even when authentication
// fails, so it may hold spaces and brackets. The empty user is logged as "",
// so exclude ident before looking for ']' followed by blanks and a quote.
func patternSplitHeader(s string) (stamp, rest string, ok bool) {
ident, s := patternToken(s)
if ident == "" {
return "", "", false
}
for i := 1; i < len(s); i++ {
if s[i] != ']' {
continue
}
j := i + 1
for j < len(s) && (s[j] == ' ' || s[j] == '\t') {
j++
}
if j == i+1 || j == len(s) || s[j] != '"' {
continue
}
open := strings.LastIndexByte(s[:i], '[')
// Keep one separator on each side of the user. Trimming would
// discard a nonempty username made entirely of spaces.
if open < 3 || (s[open-1] != ' ' && s[open-1] != '\t') {
return "", "", false
}
return s[open+1 : i], s[i+1:], true
}
return "", "", false
}
func patternValidMethod(s string) bool {
if s == "" {
return false
}
for i := range len(s) {
c := s[i]
valid := ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') ||
('0' <= c && c <= '9') || strings.ContainsRune("!#$%&'*+-.^_`|~", rune(c))
if !valid {
return false
}
}
return true
}
func patternValidRawTarget(s string) bool {
if s == "" {
return false
}
for i := range len(s) {
// Log-escaped controls remain data; literal controls cannot frame a
// request target. Validate before decoding to preserve that distinction.
if s[i] <= ' ' || s[i] == 0x7f {
return false
}
}
return true
}
func patternToken(s string) (string, string) {
s = strings.TrimLeft(s, " \t")
at := strings.IndexAny(s, " \t")
if at < 0 {
return s, ""
}
return s[:at], s[at:]
}
// patternQuotedField retains encoded bytes so decoded delimiters cannot
// change request framing. It never searches past unexpected unquoted text.
func patternQuotedField(s string) (raw, rest string, ok bool) {
s = strings.TrimLeft(s, " \t")
if len(s) == 0 || s[0] != '"' {
return "", s, false
}
for i := 1; i < len(s); {
// Find either delimiter in one pass. Looking for the closing quote
// again after each escape would repeatedly scan the same suffix.
next := strings.IndexAny(s[i:], "\"\\")
if next < 0 {
break
}
i += next
if s[i] == '\\' {
_, i = patternLogByte(s, i)
continue
}
if i+1 < len(s) && s[i+1] != ' ' && s[i+1] != '\t' {
return "", s, false
}
return s[1:i], s[i+1:], true
}
return "", s, false
}
// patternLogByte decodes one log escape; percent escapes are untouched.
// Unknown/incomplete log escapes remain literal bytes.
func patternLogByte(s string, i int) (byte, int) {
if s[i] == '\\' && i+1 < len(s) {
switch s[i+1] {
case '"', '\\':
return s[i+1], i + 2
case 'b':
return '\b', i + 2
case 'f':
return '\f', i + 2
case 'n':
return '\n', i + 2
case 'r':
return '\r', i + 2
case 't':
return '\t', i + 2
case 'v':
return '\v', i + 2
case 'x':
if i+3 < len(s) && patternIsHex(s[i+2]) && patternIsHex(s[i+3]) {
return hexToByte(s[i+2 : i+4]), i + 4
}
}
}
return s[i], i + 1
}
func patternDecodeField(raw string, limit int) (string, bool) {
// Every escape starts with a backslash; a field without one decodes to
// itself, so most lines need no copy.
if strings.IndexByte(raw, '\\') < 0 {
if len(raw) <= limit {
return raw, false
}
return raw[:max(limit, 0)], len(raw) > 0
}
var out strings.Builder
over := false
for i := 0; i < len(raw); {
var c byte
c, i = patternLogByte(raw, i)
if out.Len() < limit {
out.WriteByte(c)
} else {
over = true
}
}
return out.String(), over
}
func scanPatternQuoted(s string, limit int) (val string, overflow bool, rest string, ok bool) {
raw, rest, ok := patternQuotedField(s)
if !ok {
return "", false, rest, false
}
val, overflow = patternDecodeField(raw, limit)
return val, overflow, rest, true
}
// Validate all decoded Referer bytes even beyond the retained host prefix.
// An overlong path is fine; an illegal URL byte or escape is not evidence.
func patternRefererSyntax(raw string) bool {
hexLeft := 0
for i := 0; i < len(raw); {
var c byte
c, i = patternLogByte(raw, i)
if c <= ' ' || c == 0x7f || c == '"' || c == '\\' {
return false
}
if hexLeft > 0 {
if !patternIsHex(c) {
return false
}
hexLeft--
} else if c == '%' {
hexLeft = 2
}
}
return hexLeft == 0
}
func patternRefererFromValue(v string, overflow bool) (refererState, string) {
switch {
case v == "-" && !overflow:
return refererDash, ""
case v == "" && !overflow:
return refererEmpty, ""
}
host, ok := patternRefererHost(v, overflow)
if !ok {
return refererMalformed, ""
}
return refererValid, host
}
func patternRefererHost(v string, overflow bool) (string, bool) {
var after string
switch {
case len(v) >= 8 && strings.EqualFold(v[:8], "https://"):
after = v[8:]
case len(v) >= 7 && strings.EqualFold(v[:7], "http://"):
after = v[7:]
default:
return "", false
}
end := strings.IndexAny(after, "/?#")
if end < 0 {
if overflow {
return "", false
}
end = len(after)
}
authority := after[:end]
// Parse only the complete authority: a truncated path may end mid-escape.
parsed, err := url.Parse("http://" + authority)
if err != nil {
return "", false
}
authority = parsed.Host
if strings.HasPrefix(authority, "[") {
rb := strings.IndexByte(authority, ']')
if rb < 0 {
return "", false
}
addr, addrErr := netip.ParseAddr(authority[1:rb])
if addrErr != nil || !addr.Is6() || addr.Zone() != "" || !patternValidPortSuffix(authority[rb+1:]) {
return "", false
}
return addr.Unmap().String(), true
}
host, port, hasPort := strings.Cut(authority, ":")
if hasPort && !patternValidPortSuffix(":"+port) {
return "", false
}
host = strings.TrimSuffix(patternASCIILower(host), ".")
if host == "" || len(host) > patternMaxHost {
return "", false
}
for _, label := range strings.Split(host, ".") {
if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' {
return "", false
}
for i := range len(label) {
c := label[i]
valid := ('a' <= c && c <= 'z') || ('0' <= c && c <= '9') || c == '-'
if !valid {
return "", false
}
}
}
return host, true
}
func patternValidPortSuffix(s string) bool {
if s == "" {
return true
}
if s[0] != ':' || len(s) < 2 || len(s) > 6 {
return false
}
n := 0
for i := 1; i < len(s); i++ {
if s[i] < '0' || s[i] > '9' {
return false
}
n = n*10 + int(s[i]-'0')
}
return n <= 65535
}
func patternVhostExtension(s string) bool {
colon := strings.LastIndexByte(s, ':')
if colon < 0 || !patternValidPortSuffix(s[colon:]) || strings.ContainsAny(s, "/?#@") {
return false
}
_, ok := patternRefererHost("http://"+s, false)
return ok
}
func patternASCIILower(s string) string {
b := []byte(s)
for i, c := range b {
if 'A' <= c && c <= 'Z' {
b[i] = c + ('a' - 'A')
}
}
return string(b)
}
func patternIsHex(c byte) bool {
return ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')
}
func patternDigitsOnly(s string) bool {
if s == "" {
return false
}
for i := range len(s) {
if s[i] < '0' || s[i] > '9' {
return false
}
}
return true
}
type refererClass uint8
const (
refClassNone refererClass = iota
refClassMalformed
refClassCrossSite
refClassSameSite
)
// isSiteHost comes from verified inventory, never from request headers.
// Consumers discard RefererHost after this call; same-site is not a bypass.
func classifyReferer(r patternRecord, isSiteHost func(string) bool) refererClass {
switch r.Referer {
case refererValid:
if isSiteHost(r.RefererHost) {
return refClassSameSite
}
return refClassCrossSite
case refererMalformed:
return refClassMalformed
default:
return refClassNone
}
}
package checks
import (
"slices"
"time"
)
// Referer classes of a parsed crawl log record, for offline calibration.
const (
CrawlRefererNone = "none"
CrawlRefererMalformed = "malformed"
CrawlRefererCrossSite = "cross-site"
CrawlRefererSameSite = "same-site"
)
// CrawlLogRecord is one domlog line parsed by the crawl detector's record
// parser, for the offline calibration tools. Target and UserAgent carry
// request data: callers derive identities and labels from them and must
// neither store nor print them. The Referer host never leaves this package.
type CrawlLogRecord struct {
RemoteIP string
Time time.Time
TimeOK bool
Method string
Target string
TargetOverflow bool
TargetInvalid bool
Status int
RefererClass string
UserAgent string
UAOverflow bool
XFF string
XFFUnusable bool
XFFPartial bool
}
// ParseCrawlLogLine parses one complete domlog line, its terminal LF or CRLF
// removed, exactly as the crawl detector will. Callers must bound the raw line
// length before parsing. siteHosts contains verified aliases of the line's site:
// lowercase DNS names without a terminal dot, or canonical unmapped IP addresses,
// without ports. An empty list grants no same-site classification.
func ParseCrawlLogLine(line string, siteHosts []string) (CrawlLogRecord, bool) {
r, ok := parsePatternRecord(line)
if !ok {
return CrawlLogRecord{}, false
}
out := CrawlLogRecord{
RemoteIP: r.RemoteIP, Time: r.Time, TimeOK: r.TimeOK, Method: r.Method, Target: r.Target,
TargetOverflow: r.TargetOverflow, TargetInvalid: r.TargetInvalid, Status: r.Status,
UserAgent: r.UserAgent, UAOverflow: r.UAOverflow, XFF: r.XFF, XFFUnusable: r.XFFUnusable, XFFPartial: r.XFFPartial,
}
// Keep the transient Referer host inside this package, including callbacks.
switch classifyReferer(r, func(host string) bool { return slices.Contains(siteHosts, host) }) {
case refClassMalformed:
out.RefererClass = CrawlRefererMalformed
case refClassCrossSite:
out.RefererClass = CrawlRefererCrossSite
case refClassSameSite:
out.RefererClass = CrawlRefererSameSite
default:
out.RefererClass = CrawlRefererNone
}
return out, true
}
// CrawlTargetLimit bounds target bytes after log-escape decoding; a longer
// decoded target is an explicit overflow.
const CrawlTargetLimit = patternMaxTarget
package checks
import (
"errors"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"github.com/pidginhost/csm/internal/config"
)
type docrootScanRoot struct {
path string
account string
}
// SetUserdataDomainsPathForTest points the cPanel domain map at path and
// returns a function that restores the previous location. For tests only:
// packages outside checks cannot reach the injectable filesystem, so this is
// how they exercise addon-domain docroots.
func SetUserdataDomainsPathForTest(path string) func() {
prev := userdataDomainsPath
userdataDomainsPath = path
return func() { userdataDomainsPath = prev }
}
// validatedDocroots lists the document roots the per-site performance checks
// scan.
//
// ResolveWebRoots alone resolves to /home/*/public_html on cPanel, which misses
// every addon domain: those are served from /home/<user>/<domain>/ and never
// appear beneath public_html. On a live host that hid 177 of 277 WordPress
// installs from the WP-Cron check, so the fix could never reach them. cPanel's own domain map is the
// authoritative list, and is already how the exposed-files and php-config
// checks enumerate document roots.
//
// Roots are de-duplicated and path-sorted. Nested roots stay in the list so
// each gets its own full depth allowance; the scanner assigns the nested
// subtree to the more-specific root to avoid duplicate work and findings.
func validatedDocroots(cfg *config.Config) []string {
rootSet, _ := validatedDocrootSet(cfg)
roots := make([]string, 0, len(rootSet))
for _, root := range rootSet {
roots = append(roots, root.path)
}
return roots
}
// ResolveValidatedDocroots returns the same validated roots the scans use.
// Remediation callers use it so a finding from an addon domain remains inside
// the exact root authorized by cPanel's map.
func ResolveValidatedDocroots(cfg *config.Config) []string {
return validatedDocroots(cfg)
}
func validatedDocrootSet(cfg *config.Config) ([]docrootScanRoot, bool) {
configured, complete := resolveWebRootsChecked(cfg)
byPath := make(map[string]docrootScanRoot, len(configured))
for _, root := range configured {
clean := filepath.Clean(root)
byPath[clean] = docrootScanRoot{path: clean, account: accountFromPath(clean)}
}
data, err := osFS.ReadFile(userdataDomainsPath)
if err != nil {
if vhostMapFailureIsIncomplete(err) {
complete = false
}
return sortedDocrootScanRoots(byPath), complete
}
vhosts, parsedComplete := parseUserdataDomainRootsChecked(string(data))
complete = complete && parsedComplete
attempted := make(map[string]struct{}, len(vhosts))
for _, vhost := range vhosts {
clean := filepath.Clean(vhost.docroot)
key := vhost.user + "\x00" + clean
if _, duplicate := attempted[key]; duplicate {
continue
}
attempted[key] = struct{}{}
homeBase, accountHome, safe := docrootMapAccountHome(vhost.user, clean, configured)
if !safe {
complete = false
continue
}
exists, pathComplete := docrootMapRootIsSafe(homeBase, accountHome, clean)
if !pathComplete {
complete = false
}
if !exists {
continue
}
byPath[clean] = docrootScanRoot{path: clean, account: vhost.user}
}
return sortedDocrootScanRoots(byPath), complete
}
func sortedDocrootScanRoots(byPath map[string]docrootScanRoot) []docrootScanRoot {
roots := make([]docrootScanRoot, 0, len(byPath))
for _, root := range byPath {
roots = append(roots, root)
}
sort.Slice(roots, func(i, j int) bool { return roots[i].path < roots[j].path })
return roots
}
func docrootMapAccountHome(user, docroot string, configured []string) (homeBase, accountHome string, ok bool) {
clean := filepath.Clean(docroot)
if !filepath.IsAbs(clean) {
return "", "", false
}
parts := strings.Split(strings.TrimPrefix(clean, string(filepath.Separator)), string(filepath.Separator))
if len(parts) >= 3 && isHomeVolumeName(parts[0]) && parts[1] == user {
homeBase = filepath.Join(string(filepath.Separator), parts[0])
accountHome = filepath.Join(homeBase, user)
return homeBase, accountHome, clean != accountHome
}
// Explicit account_roots may point into a chroot-style test or custom
// mount. Only extend a map root to a sibling when the configured root
// already anchors that same /home/USER boundary.
for _, configuredRoot := range configured {
rootParts := strings.Split(strings.TrimPrefix(filepath.Clean(configuredRoot), string(filepath.Separator)), string(filepath.Separator))
for i := 0; i+1 < len(rootParts); i++ {
if rootParts[i] != "home" || rootParts[i+1] != user {
continue
}
homeBaseParts := append([]string{string(filepath.Separator)}, rootParts[:i+1]...)
homeBase = filepath.Join(homeBaseParts...)
accountHome = filepath.Join(homeBase, user)
if filepath.Clean(configuredRoot) != accountHome &&
isPathWithinOrEqual(configuredRoot, accountHome) &&
clean != accountHome && isPathWithinOrEqual(clean, accountHome) {
return homeBase, accountHome, true
}
}
}
return "", "", false
}
func isHomeVolumeName(name string) bool {
if name == "home" {
return true
}
if !strings.HasPrefix(name, "home") || len(name) == len("home") {
return false
}
for _, r := range name[len("home"):] {
if r < '0' || r > '9' {
return false
}
}
return true
}
// docrootMapRootIsSafe rejects symlinks at every component from the home mount
// through the docroot. Stat on only the final path would follow an intermediate
// symlink and let a malformed map redirect the scan outside the account.
func docrootMapRootIsSafe(homeBase, accountHome, root string) (exists, complete bool) {
if !isPathWithinOrEqual(accountHome, homeBase) ||
!isPathWithinOrEqual(root, accountHome) || root == accountHome {
return false, false
}
rel, err := filepath.Rel(homeBase, root)
if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return false, false
}
paths := []string{homeBase}
current := homeBase
for _, part := range strings.Split(rel, string(filepath.Separator)) {
current = filepath.Join(current, part)
paths = append(paths, current)
}
for _, path := range paths {
info, statErr := osFS.Lstat(path)
if statErr != nil {
if errors.Is(statErr, fs.ErrNotExist) {
return false, true
}
return false, false
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return false, false
}
}
return true, true
}
package checks
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"syscall"
)
// fixPerfAllowedRoots scopes performance remediations to per-account web
// content. Same shape as the other fix*AllowedRoots so tests can swap in
// a t.TempDir(); nil means the platform's account roots.
var fixPerfAllowedRoots []string
// FixErrorLogBloat truncates an account-owned error_log file in place.
// Truncating preserves the inode and file ownership so any PHP process
// holding the descriptor keeps appending to the same file without an
// open/reopen race; this is also the safest action because nothing in
// the host needs the historical lines to keep serving traffic.
func FixErrorLogBloat(path string) RemediationResult {
return FixErrorLogBloatInRoots(path, effectiveFixRoots(fixPerfAllowedRoots))
}
// FixErrorLogBloatInRoots is FixErrorLogBloat with caller-supplied roots.
// The Web UI uses this to include configured account_roots while tests can
// keep writes under t.TempDir().
func FixErrorLogBloatInRoots(path string, allowedRoots []string) RemediationResult {
if path == "" {
return RemediationResult{Error: "could not extract file path from finding"}
}
resolved, info, err := resolveExistingFixPath(path, allowedRoots)
if err != nil {
return RemediationResult{Error: err.Error()}
}
if info.IsDir() {
return RemediationResult{Error: "refusing to truncate a directory"}
}
if filepath.Base(resolved) != "error_log" {
return RemediationResult{Error: fmt.Sprintf("refusing to truncate non error_log file: %s", resolved)}
}
oldSize := info.Size()
if err := truncateFilePreservingIdentity(resolved, info); err != nil {
return RemediationResult{Error: fmt.Sprintf("truncate failed: %v", err)}
}
return RemediationResult{
Success: true,
Action: fmt.Sprintf("truncate %s", resolved),
Description: fmt.Sprintf("Emptied error_log (was %s)", humanBytes(oldSize)),
}
}
// FixDisplayErrorsOn rewrites an INI / .htaccess / .user.ini file so the
// display_errors directive is set to Off. The original line is preserved
// commented out for operator review; an override line is appended at the
// end of the file so the last-write-wins semantics of every supported
// config format land on Off regardless of earlier statements.
//
// Only .user.ini, php.ini, and .htaccess are accepted. wp-config.php and
// other PHP source files require code-level edits this routine does not
// attempt. The caller (web UI) should not advertise the fix for those.
func FixDisplayErrorsOn(path string) RemediationResult {
return FixDisplayErrorsOnInRoots(path, effectiveFixRoots(fixPerfAllowedRoots))
}
// FixDisplayErrorsOnInRoots is FixDisplayErrorsOn with caller-supplied
// roots. The Web UI uses this to honor account_roots outside /home.
func FixDisplayErrorsOnInRoots(path string, allowedRoots []string) RemediationResult {
if path == "" {
return RemediationResult{Error: "could not extract file path from finding"}
}
resolved, info, err := resolveExistingFixPath(path, allowedRoots)
if err != nil {
return RemediationResult{Error: err.Error()}
}
if info.IsDir() {
return RemediationResult{Error: "refusing to edit a directory"}
}
base := filepath.Base(resolved)
var (
isHtaccess bool
supported bool
)
switch {
case base == ".user.ini" || base == "php.ini" || strings.HasSuffix(base, ".ini"):
supported = true
case base == ".htaccess":
supported = true
isHtaccess = true
}
if !supported {
return RemediationResult{Error: fmt.Sprintf("automated display_errors fix only supports .user.ini, php.ini, and .htaccess (got %s)", base)}
}
target, err := openFixTarget(resolved, info)
if err != nil {
return RemediationResult{Error: fmt.Sprintf("read failed: %v", err)}
}
defer target.Close()
data, err := io.ReadAll(target.File)
if err != nil {
return RemediationResult{Error: fmt.Sprintf("read failed: %v", err)}
}
rewritten, changedLines := commentDisplayErrorsLines(data)
if changedLines == 0 {
return RemediationResult{Error: "no display_errors directive found in file"}
}
if isHtaccess {
rewritten = appendHtaccessOverride(rewritten)
} else {
rewritten = appendIniOverride(rewritten)
}
// Preserve ownership + mode. Write atomically via a sibling temp file +
// rename so a partial write does not leave the operator with a broken
// config.
if err := writeCleanedFileAtomic(target, rewritten); err != nil {
return RemediationResult{Error: err.Error()}
}
return RemediationResult{
Success: true,
Action: fmt.Sprintf("disable display_errors in %s", resolved),
Description: fmt.Sprintf(
"Commented %d display_errors line(s) and appended an Off override at end of file",
changedLines,
),
}
}
// commentDisplayErrorsLines walks the file line-by-line, comments out any
// non-comment line whose directive is display_errors (matching the
// detector's logic). Returns the rewritten bytes and the count of lines
// changed.
func commentDisplayErrorsLines(data []byte) ([]byte, int) {
var out bytes.Buffer
changed := 0
for _, raw := range bytes.SplitAfter(data, []byte("\n")) {
if len(raw) == 0 {
continue
}
line := raw
newline := []byte(nil)
if bytes.HasSuffix(raw, []byte("\n")) {
line = raw[:len(raw)-1]
newline = []byte("\n")
}
lineText := string(line)
trimmed := strings.TrimSpace(lineText)
if trimmed == "" || strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, ";") {
out.Write(raw)
continue
}
if !strings.Contains(strings.ToLower(trimmed), "display_errors") {
out.Write(raw)
continue
}
out.WriteString("# csm: disabled by remediation -- ")
out.WriteString(lineText)
out.Write(newline)
changed++
}
return out.Bytes(), changed
}
func appendIniOverride(data []byte) []byte {
override := "display_errors = Off"
return appendOverrideLine(data, override, "; csm: appended by remediation")
}
func appendHtaccessOverride(data []byte) []byte {
override := "php_flag display_errors Off"
return appendOverrideLine(data, override, "# csm: appended by remediation")
}
func appendOverrideLine(data []byte, directive, marker string) []byte {
var out bytes.Buffer
out.Write(data)
if len(data) > 0 && data[len(data)-1] != '\n' {
out.WriteByte('\n')
}
out.WriteString(marker)
out.WriteByte('\n')
out.WriteString(directive)
out.WriteByte('\n')
return out.Bytes()
}
// truncateFilePreservingIdentity re-checks the target inode after the initial
// path validation so a file swap between validation and mutation fails closed.
func truncateFilePreservingIdentity(path string, expected os.FileInfo) error {
target, err := openFixTargetFlags(path, expected, os.O_WRONLY)
if err != nil {
return err
}
defer target.Close()
return target.File.Truncate(0)
}
func sameFileIdentity(a, b os.FileInfo) bool {
if a == nil || b == nil {
return false
}
if os.SameFile(a, b) {
return true
}
as, aok := a.Sys().(*syscall.Stat_t)
bs, bok := b.Sys().(*syscall.Stat_t)
return aok && bok && as.Dev == bs.Dev && as.Ino == bs.Ino
}
package checks
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"math"
"net"
"os"
"sort"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/netutil"
"github.com/pidginhost/csm/internal/state"
)
const (
// wpLoopbackMaxPerHour is the most self-requests one job makes in an hour
// under WordPress's own schedulers: the cron spawn and Action Scheduler's
// admin dispatch each hold a 60-second lock. A job over it every hour is
// either re-triggered by page views or re-dispatching a queue that does
// not drain.
wpLoopbackMaxPerHour = 60
// wpLoopbackHours is how many consecutive complete hours must each be over
// the rate. A bulk import or a one-off backlog stops well before this.
wpLoopbackHours = 3
// wpLoopbackMaxLineBytes bounds one log line; longer lines are skipped.
wpLoopbackMaxLineBytes = 64 * 1024
wpLoopbackMaxTargetLen = 256
// Job names are tenant-controlled. Bound aggregate memory independently
// of log size and preserve prior findings when coverage exceeds this cap.
wpLoopbackMaxSeries = 4096
)
var wpLoopbackNow = time.Now
// wpLoopbackFromHost reports whether a request came from this server: its
// loopback interface or one of its own addresses, which is how cPanel sites
// usually reach themselves.
var wpLoopbackFromHost = func(ip string) bool {
parsed := net.ParseIP(ip)
return parsed != nil && (parsed.IsLoopback() || netutil.IsHostAddress(ip))
}
// wpLoopbackStateKey is underscore-prefixed so state.Store.Update does not
// prune it as a stale non-finding key.
const wpLoopbackStateKey = "_perf_wp_loopback"
// wpLoopbackJob counts one job's self-requests by local clock hour. Hours maps
// the hour's start (Unix seconds) to its request and 5xx counts. Target and UA
// are display forms; the job's identity is the hash of its full target.
type wpLoopbackJob struct {
Target string `json:"target"`
UA string `json:"ua"`
Hours map[int64][2]int `json:"hours"`
}
// wpLoopbackLog is one vhost log's read position and the jobs seen in it
// within the window.
type wpLoopbackLog struct {
Follow followState `json:"follow"`
Jobs map[string]*wpLoopbackJob `json:"jobs,omitempty"`
}
// wpLoopbackState is keyed by log path.
type wpLoopbackState map[string]*wpLoopbackLog
// CheckWPLoopbackRequests follows every active vhost log for WordPress sites
// calling themselves faster than WordPress's own schedulers ever do, hour
// after hour. Each run reads only what the logs gained since the last one and
// keeps hourly counts in the scan state. The runner enforces a 60-minute
// throttle via checkThrottleMin.
func CheckWPLoopbackRequests(ctx context.Context, cfg *config.Config, scanState *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
now := wpLoopbackNow()
windowStart := wpLoopbackWindowEnd(now).Add(-wpLoopbackHours * time.Hour)
discoveryFailed := false
paths := discoverFreshDomlogsWithErrors(ctx, math.MaxInt, now.Sub(windowStart), func(err error) {
discoveryFailed = true
markScanReadError(ctx, "perf_wp_loopback", err)
})
previous := loadWPLoopbackState(scanState)
findings, next := scanWPLoopbackLogsState(ctx, paths, now, previous)
if ctx.Err() != nil {
return nil
}
if discoveryFailed {
// Logs discovery could not list were not read, not gone.
for path, log := range previous {
if _, ok := next[path]; !ok {
next[path] = log
}
}
}
storeWPLoopbackState(scanState, next)
return findings
}
// scanWPLoopbackLogs evaluates logs with no saved state, reading each whole
// (up to the first-run catch-up limit).
func scanWPLoopbackLogs(ctx context.Context, paths []string, now time.Time) []alert.Finding {
findings, _ := scanWPLoopbackLogsState(ctx, paths, now, nil)
return findings
}
// scanWPLoopbackLogsState follows each log from its saved position and judges
// the last wpLoopbackHours complete hours before now; the hour in progress is
// counted for later runs but not judged. It returns the findings and the
// state to save. A log that fails to read keeps its previous state.
func scanWPLoopbackLogsState(ctx context.Context, paths []string, now time.Time, previous wpLoopbackState) ([]alert.Finding, wpLoopbackState) {
end := wpLoopbackWindowEnd(now)
start := end.Add(-wpLoopbackHours * time.Hour)
next := make(wpLoopbackState, len(paths))
var findings []alert.Finding
for _, path := range paths {
if ctx.Err() != nil {
break
}
domain := domainFromDomlogPath(path)
if domain == "" {
continue
}
log, ok := followWPLoopbackLog(ctx, path, previous[path], start)
if !ok {
markCheckIncomplete(ctx, "perf_wp_loopback")
if old := previous[path]; old != nil {
next[path] = old
}
continue
}
next[path] = log
for key, job := range log.Jobs {
if counts, serverKO, sustained := wpLoopbackWindowCounts(job, start); sustained {
findings = append(findings, newWPLoopbackFinding(domain, key, job, counts, serverKO, now))
}
}
}
if ctx.Err() != nil {
markCheckIncomplete(ctx, "perf_wp_loopback")
return nil, previous
}
sort.Slice(findings, func(i, j int) bool {
if findings[i].Message == findings[j].Message {
return findings[i].DedupKey < findings[j].DedupKey
}
return findings[i].Message < findings[j].Message
})
return findings, next
}
func wpLoopbackWindowEnd(now time.Time) time.Time {
// Truncate rounds absolute time, which splits local clock hours in zones
// whose UTC offset includes a half or quarter hour.
return now.Add(-time.Duration(now.Minute())*time.Minute -
time.Duration(now.Second())*time.Second - time.Duration(now.Nanosecond()))
}
// wpLoopbackWindowCounts returns the job's requests in each window hour, its
// 5xx answers across them, and whether every hour is over the rate.
func wpLoopbackWindowCounts(job *wpLoopbackJob, start time.Time) (counts [wpLoopbackHours]int, serverKO int, sustained bool) {
sustained = true
for i := range counts {
hour := job.Hours[start.Add(time.Duration(i)*time.Hour).Unix()]
counts[i], serverKO = hour[0], serverKO+hour[1]
if counts[i] <= wpLoopbackMaxPerHour {
sustained = false
}
}
return counts, serverKO, sustained
}
func newWPLoopbackFinding(domain, key string, job *wpLoopbackJob, counts [wpLoopbackHours]int, serverKO int, now time.Time) alert.Finding {
shown := make([]string, len(counts))
for i, n := range counts {
shown[i] = strconv.Itoa(n)
}
details := fmt.Sprintf("From this server, %s per hour over the last %d hours", strings.Join(shown, ", "), wpLoopbackHours)
if serverKO > 0 {
details += fmt.Sprintf("; %d answered with a 5xx error", serverKO)
}
details += ". User-Agent: " + job.UA
return alert.Finding{
Severity: alert.Warning,
Check: "perf_wp_loopback",
Message: fmt.Sprintf("Sustained WordPress loopback requests on %s: POST %s", domain, job.Target),
Details: details,
DedupKey: domain + " " + key,
Timestamp: now,
}
}
// followWPLoopbackLog reads the complete lines path gained since old and
// returns the updated log state. Requests are stamped when they start but
// logged when they finish, so no part of a log can be skipped by its
// timestamps; following it instead reads every byte once across runs. A
// rotated, truncated or replaced file is read from its start, and a first
// run catches up on at most the tail the shared follower allows. Hours before
// start are dropped. old is never modified.
func followWPLoopbackLog(ctx context.Context, path string, old *wpLoopbackLog, start time.Time) (*wpLoopbackLog, bool) {
f, err := osFS.Open(path)
if err != nil {
return nil, false
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil {
return nil, false
}
size := info.Size()
log := &wpLoopbackLog{Jobs: make(map[string]*wpLoopbackJob)}
var follow followState
if old != nil {
follow = old.Follow
for key, job := range old.Jobs {
kept := &wpLoopbackJob{Target: job.Target, UA: job.UA, Hours: make(map[int64][2]int, len(job.Hours))}
for hour, c := range job.Hours {
if hour >= start.Unix() {
kept.Hours[hour] = c
}
}
if len(kept.Hours) > 0 {
log.Jobs[key] = kept
}
}
}
offset, _, err := chooseStart(f, follow, size)
if err != nil {
return nil, false
}
// The size snapshot bounds the read against concurrent appends; a file
// that shrinks while being read ends early and fails the run.
limited := &io.LimitedReader{R: io.NewSectionReader(f, offset, size-offset), N: size - offset}
reader := bufio.NewReaderSize(limited, wpLoopbackMaxLineBytes)
pos := offset
for {
line, n, err := readDomlogLine(ctx, reader)
if err == io.EOF && limited.N == 0 {
// Anything left is a line still being written; the next run
// starts at its beginning.
break
}
if err != nil {
return nil, false
}
pos += int64(n)
if line != nil && bytes.Contains(line, []byte(`"WordPress/`)) && bytes.Contains(line, []byte(`"POST `)) {
if !countWPLoopback(log.Jobs, string(line), start) {
return nil, false
}
}
}
log.Follow = followState{Offset: pos}
if err := fillIdentity(f, &log.Follow, size); err != nil {
return nil, false
}
return log, true
}
func countWPLoopback(jobs map[string]*wpLoopbackJob, line string, start time.Time) bool {
rec, ok := parseAccessLogRecordWithURILimit(line, wpLoopbackMaxLineBytes)
if !ok || rec.Method != "POST" || !strings.HasPrefix(rec.UserAgent, "WordPress/") {
return true
}
if rec.Time.Before(start) || !wpLoopbackFromHost(rec.RemoteIP) {
return true
}
target := wpLoopbackTarget(rec.URI)
sum := sha256.Sum256([]byte(target))
key := hex.EncodeToString(sum[:])
job := jobs[key]
if job == nil {
if len(jobs) >= wpLoopbackMaxSeries {
return false
}
// Copy bounded display fields so short substrings do not retain the
// entire input line, including discarded query values and headers.
job = &wpLoopbackJob{
Target: strings.Clone(sanitizeJSTaintDisplay(target, wpLoopbackMaxTargetLen)),
UA: strings.Clone(sanitizeJSTaintDisplay(rec.UserAgent, 512)),
Hours: make(map[int64][2]int),
}
jobs[key] = job
}
hour := wpLoopbackWindowEnd(rec.Time).Unix()
c := job.Hours[hour]
c[0]++
if rec.Status >= 500 && rec.Status < 600 {
c[1]++
}
job.Hours[hour] = c
return true
}
func loadWPLoopbackState(scanState *state.Store) wpLoopbackState {
if scanState == nil {
return nil
}
raw, ok := scanState.GetRaw(wpLoopbackStateKey)
if !ok {
return nil
}
var st wpLoopbackState
if json.Unmarshal([]byte(raw), &st) != nil {
return nil
}
return st
}
func storeWPLoopbackState(scanState *state.Store, st wpLoopbackState) {
if scanState == nil {
return
}
if len(st) == 0 {
if err := scanState.DeleteRawAndSave(wpLoopbackStateKey); err != nil {
fmt.Fprintf(os.Stderr, "perf_wp_loopback: state clear: %v\n", err)
}
return
}
raw, err := json.Marshal(st)
if err != nil {
return
}
if err := scanState.SetRawAndSave(wpLoopbackStateKey, string(raw)); err != nil {
fmt.Fprintf(os.Stderr, "perf_wp_loopback: state write: %v\n", err)
}
}
// wpLoopbackTarget names the job a self-request runs: the path, plus the
// admin-ajax action when there is one. Nonces and other per-request values are
// dropped so every run of one job counts together; the path is kept as logged
// because security plugins rename admin-ajax.php. Keep the logged bytes for
// identity; sanitizing or truncating them would merge different jobs.
func wpLoopbackTarget(uri string) string {
path, query, _ := strings.Cut(uri, "?")
target := path
for _, param := range strings.Split(query, "&") {
if action, ok := strings.CutPrefix(param, "action="); ok && action != "" {
target = path + "?action=" + action
break
}
}
return target
}
// readDomlogLine returns the next line without its newline and the bytes it
// consumed, newline included. A line over wpLoopbackMaxLineBytes is consumed
// and returned as nil. At the end of input, a final line with no newline is
// returned with io.EOF.
func readDomlogLine(ctx context.Context, r *bufio.Reader) ([]byte, int, error) {
var line []byte
consumed := 0
tooLong := false
for {
if err := ctx.Err(); err != nil {
return nil, consumed, err
}
chunk, err := r.ReadSlice('\n')
consumed += len(chunk)
if line == nil && !tooLong && err != bufio.ErrBufferFull && len(chunk) <= wpLoopbackMaxLineBytes {
return bytes.TrimRight(chunk, "\r\n"), consumed, err
}
if !tooLong {
if len(line)+len(chunk) > wpLoopbackMaxLineBytes {
tooLong, line = true, nil
} else {
line = append(line, chunk...)
}
}
if err == bufio.ErrBufferFull {
continue
}
if tooLong {
return nil, consumed, err
}
return bytes.TrimRight(line, "\r\n"), consumed, err
}
}
package checks
import (
"context"
"fmt"
"net"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mysqlclient"
"github.com/pidginhost/csm/internal/netutil"
"github.com/pidginhost/csm/internal/state"
)
// wpMyISAMQuery lists every MyISAM table on the server in one catalogue read.
// It carries no tenant-supplied names; installs are matched in Go.
const wpMyISAMQuery = "SELECT TABLE_SCHEMA, TABLE_NAME, COALESCE(DATA_LENGTH, 0) + COALESCE(INDEX_LENGTH, 0), @@port, @@socket " +
"FROM information_schema.TABLES WHERE ENGINE = 'MyISAM' AND TABLE_TYPE = 'BASE TABLE'"
const wpMyISAMListedTables = 10
type wpMyISAMTable struct {
name string
bytes int64
}
// wpMyISAMScope is one set of WordPress tables: a database and table prefix,
// with every install configured to use it.
type wpMyISAMScope struct {
schema string
prefix string
installs []wpMyISAMInstall
tables []wpMyISAMTable
}
type wpMyISAMInstall struct {
wpInstall
host string
}
// CheckWPMyISAM reports WordPress installs whose tables still use MyISAM.
// MyISAM locks the whole table for every write, so a burst of uncached
// requests queues behind those locks until it holds every database connection
// the account may open; InnoDB locks rows instead.
// The runner enforces a 60-minute throttle via checkThrottleMin.
func CheckWPMyISAM(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
scopes := make(map[string]*wpMyISAMScope)
prefixesBySchema := make(map[string][]string)
unresolvedAccounts := make(map[string]bool)
unresolvedSchemas := make(map[string]bool)
for _, in := range wpInstalls(ctx, "perf_wp_myisam") {
if ctx.Err() != nil {
return nil
}
creds, complete := readWPMyISAMConfig(ctx, in.ConfigPath)
if !complete {
markCheckIncomplete(ctx, "perf_wp_myisam")
if creds.dbName != "" {
unresolvedSchemas[creds.dbName] = true
} else {
unresolvedAccounts[in.Account] = true
}
continue
}
prefix := creds.tablePrefix
if !wpDBHostIsLocal(creds.dbHost) {
continue
}
// Every readable install claims its prefix, reported or not, so a
// table is never attributed to a sibling with a shorter prefix.
key := creds.dbName + "\x00" + prefix
scope, seen := scopes[key]
if !seen {
scope = &wpMyISAMScope{schema: creds.dbName, prefix: prefix}
scopes[key] = scope
}
scope.installs = append(scope.installs, wpMyISAMInstall{wpInstall: in, host: creds.dbHost})
}
if len(scopes) == 0 {
return nil
}
rows, err := mysqlclient.RootQuery(ctx, wpMyISAMQuery)
if err != nil {
markCheckIncomplete(ctx, "perf_wp_myisam")
return nil
}
if len(rows) == 0 {
return nil
}
metadata := strings.Split(rows[0], "\t")
if len(metadata) != 5 {
markCheckIncomplete(ctx, "perf_wp_myisam")
return nil
}
port, err := strconv.ParseUint(metadata[3], 10, 16)
if err != nil {
markCheckIncomplete(ctx, "perf_wp_myisam")
return nil
}
socket := mysqlclient.BatchUnescape(metadata[4])
for _, scope := range scopes {
matched := scope.installs[:0]
for _, in := range scope.installs {
if wpMyISAMEndpointMatches(in.host, port, socket) {
matched = append(matched, in)
}
}
scope.installs = matched
if len(matched) > 0 {
prefixesBySchema[scope.schema] = append(prefixesBySchema[scope.schema], scope.prefix)
}
}
for _, line := range rows {
fields := strings.Split(line, "\t")
if len(fields) != 5 {
markCheckIncomplete(ctx, "perf_wp_myisam")
continue
}
schema := mysqlclient.BatchUnescape(fields[0])
table := mysqlclient.BatchUnescape(fields[1])
owner := ""
var scope *wpMyISAMScope
for _, prefix := range prefixesBySchema[schema] {
if strings.HasPrefix(table, prefix) && (scope == nil || len(prefix) > len(owner)) {
owner = prefix
scope = scopes[schema+"\x00"+owner]
}
}
if scope == nil {
continue
}
size, _ := strconv.ParseInt(fields[2], 10, 64)
scope.tables = append(scope.tables, wpMyISAMTable{name: table, bytes: size})
}
var findings []alert.Finding
for _, scope := range scopes {
if len(scope.tables) == 0 || unresolvedSchemas[scope.schema] {
continue
}
// An unreadable sibling may own a longer prefix. Its account's
// readable configs cannot establish ownership until it can be read.
ambiguous := false
for _, in := range scope.installs {
ambiguous = ambiguous || unresolvedAccounts[in.Account]
}
if ambiguous {
continue
}
if in, ok := wpMyISAMReportedInstall(scope.installs); ok {
findings = append(findings, newWPMyISAMFinding(scope, in))
}
}
sort.Slice(findings, func(i, j int) bool { return findings[i].Details < findings[j].Details })
return findings
}
// wpMyISAMReportedInstall picks the install a finding names. A dormant root
// and a suspended account serve no traffic, so their tables cannot queue
// requests; the same tables used by a live root are still reported.
func wpMyISAMReportedInstall(installs []wpMyISAMInstall) (wpInstall, bool) {
var live []wpInstall
for _, in := range installs {
if in.Served == notServed || accountSuspended(in.Account) {
continue
}
live = append(live, in.wpInstall)
}
if len(live) == 0 {
return wpInstall{}, false
}
sort.Slice(live, func(i, j int) bool { return live[i].ConfigPath < live[j].ConfigPath })
return live[0], true
}
func newWPMyISAMFinding(scope *wpMyISAMScope, in wpInstall) alert.Finding {
tables := scope.tables
sort.Slice(tables, func(i, j int) bool {
if tables[i].bytes != tables[j].bytes {
return tables[i].bytes > tables[j].bytes
}
return tables[i].name < tables[j].name
})
var total int64
names := make([]string, 0, wpMyISAMListedTables)
for i, t := range tables {
total += t.bytes
if i < wpMyISAMListedTables {
names = append(names, t.name)
}
}
list := strings.Join(names, ", ")
if more := len(tables) - len(names); more > 0 {
list += fmt.Sprintf(" and %d more", more)
}
owner := in.Account
if owner == "" {
owner = in.DocRoot
}
return alert.Finding{
Severity: alert.Warning,
Check: "perf_wp_myisam",
Message: fmt.Sprintf("WordPress tables use MyISAM for %s", owner),
Details: fmt.Sprintf(
"Database: %s, prefix: %s, MyISAM tables: %d (%s): %s. File: %s - "+
"convert them to InnoDB after a backup; MyISAM locks the whole table on every write",
scope.schema, scope.prefix, len(tables), humanBytes(total), list, in.ConfigPath,
),
// Sizes and table membership change; identity is the database and prefix.
DedupKey: fmt.Sprintf("db=%q prefix=%q", scope.schema, scope.prefix),
Timestamp: time.Now(),
}
}
// wpDBHostIsLocal reports whether a wp-config.php DB_HOST names this server.
// WordPress accepts host, host:port and host:/socket. The catalogue read only
// sees the local server, where a schema with the same name as a remote
// database is a different database, often a copy left behind by a migration.
func wpDBHostIsLocal(host string) bool {
h, _, _, ok := wpMyISAMDBHost(host)
if !ok {
return false
}
if h == "localhost" {
return true
}
ip := net.ParseIP(h)
return ip != nil && (ip.IsLoopback() || netutil.IsHostAddress(h))
}
func wpMyISAMEndpointMatches(host string, serverPort uint64, serverSocket string) bool {
h, port, socket, ok := wpMyISAMDBHost(host)
if !ok {
return false
}
// mysqli uses a Unix socket for localhost, even if a port is given.
// A socket argument with a TCP hostname does not change the transport.
if h == "localhost" {
return socket == "" || filepath.Clean(socket) == filepath.Clean(serverSocket)
}
return port == serverPort
}
func wpMyISAMDBHost(host string) (name string, port uint64, socket string, ok bool) {
h := strings.TrimSpace(host)
// WordPress peels off a socket before parsing the host and optional port.
if pos := strings.Index(h, ":/"); pos >= 0 {
socket, h = h[pos+1:], h[:pos]
}
h = strings.ToLower(h)
port = 3306
var suffix string
if strings.HasPrefix(h, "[") {
end := strings.IndexByte(h, ']')
if end < 0 {
return "", 0, "", false
}
suffix, h = h[end+1:], h[1:end]
if !strings.Contains(h, ":") || net.ParseIP(h) == nil {
return "", 0, "", false
}
} else if strings.Count(h, ":") == 1 {
pos := strings.IndexByte(h, ':')
suffix, h = h[pos:], h[:pos]
}
if suffix != "" {
if !strings.HasPrefix(suffix, ":") {
return "", 0, "", false
}
var err error
port, err = strconv.ParseUint(suffix[1:], 10, 16)
if err != nil || port == 0 {
return "", 0, "", false
}
}
return h, port, socket, true
}
package checks
import (
"context"
"regexp"
"strings"
)
var (
wpMyISAMDefineStart = regexp.MustCompile(`(?i)\bdefine\s*\(`)
wpMyISAMPrefixUse = regexp.MustCompile(`\$table_prefix\b`)
)
func readWPMyISAMConfig(ctx context.Context, path string) (wpDBCreds, bool) {
data, err := readCMSConfig(ctx, path)
if err != nil {
return wpDBCreds{}, false
}
return parseWPMyISAMConfig(data)
}
// Root's catalogue cannot verify a guessed scope by authenticating as the
// site. Require explicit literal settings instead of the shared credential
// parser's defaults, and never execute tenant PHP to resolve expressions.
func parseWPMyISAMConfig(data []byte) (wpDBCreds, bool) {
code := stripPHPCommentsFromCode(phpCodeOnly(string(data)))
masked := stripPHPStringsFromCode(code)
topLevel := wpMyISAMStatementStarts(masked)
values := make(map[string]string)
seen := make(map[string]bool)
valid := true
for _, loc := range wpMyISAMDefineStart.FindAllStringIndex(masked, -1) {
key, pos, ok := wpMyISAMLiteral(code, loc[1])
if !ok || (key != "DB_NAME" && key != "DB_HOST") {
continue
}
start := loc[0]
if start > 0 && code[start-1] == '\\' {
start--
}
pos, comma := wpMyISAMTake(code, pos, ',')
value, pos, literal := wpMyISAMLiteral(code, pos)
pos, closeParen := wpMyISAMTake(code, pos, ')')
_, semicolon := wpMyISAMTake(code, pos, ';')
if seen[key] || !topLevel[start] || !comma || !literal || !closeParen || !semicolon {
valid = false
values[key] = ""
} else {
values[key] = value
}
seen[key] = true
}
creds := wpDBCreds{dbName: values["DB_NAME"], dbHost: values["DB_HOST"]}
// A second prefix reference may mutate it conditionally or pass it by
// reference. Only a sole literal assignment establishes table ownership.
uses := wpMyISAMPrefixUse.FindAllStringIndex(masked, -1)
if len(uses) != 1 || !topLevel[uses[0][0]] {
return creds, false
}
pos, equal := wpMyISAMTake(code, uses[0][1], '=')
prefix, pos, literal := wpMyISAMLiteral(code, pos)
_, semicolon := wpMyISAMTake(code, pos, ';')
creds.tablePrefix = prefix
prefixOK := prefix == "" || validTablePrefix.MatchString(prefix)
return creds, valid && creds.dbName != "" && creds.dbHost != "" && equal && literal && semicolon && prefixOK
}
// Work on the string-masked code so punctuation in literals cannot turn a
// conditional assignment into an unconditional one. Record every position in
// one pass rather than rescanning the prefix for each candidate statement.
func wpMyISAMStatementStarts(masked string) []bool {
starts := make([]bool, len(masked))
depth := 0
boundary := true
for i := 0; i < len(masked); i++ {
starts[i] = depth == 0 && boundary
switch masked[i] {
case ' ', '\t', '\r', '\n', '\v', '\f':
continue
case '(', '[', '{':
depth++
boundary = false
case ')', ']', '}':
depth--
boundary = masked[i] == '}' && depth == 0
case ';':
boundary = depth == 0
case ':':
if depth == 0 {
// Alternative control syntax has no braces. Subsequent
// statements cannot be proved unconditional by this scanner.
return starts
}
boundary = false
default:
boundary = false
}
}
return starts
}
func wpMyISAMTake(code string, pos int, want byte) (int, bool) {
pos = skipPHPWhitespace(code, pos)
if pos == len(code) || code[pos] != want {
return pos, false
}
return pos + 1, true
}
func wpMyISAMLiteral(code string, pos int) (string, int, bool) {
pos = skipPHPWhitespace(code, pos)
if pos == len(code) || (code[pos] != '\'' && code[pos] != '"') {
return "", pos, false
}
end, closed := storedPHPStringClose(code, pos)
if !closed || storedPHPStringInterpolates(code, pos, end) {
return "", end, false
}
value := phpStringLiteralValue(code, pos, end)
// Database settings never need embedded NULs, which cannot name a schema,
// table or endpoint and would make scope keys ambiguous.
return value, end + 1, !strings.ContainsRune(value, 0)
}
package checks
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mysqlclient"
"github.com/pidginhost/csm/internal/redisinfo"
"github.com/pidginhost/csm/internal/state"
)
// perfEnabled returns false only if Performance.Enabled is explicitly set to false.
// nil (unset) is treated as enabled.
func perfEnabled(cfg *config.Config) bool {
if cfg.Performance.Enabled == nil {
return true
}
return *cfg.Performance.Enabled
}
// cpuCoresOnce guards the cached CPU core count.
var (
cpuCoresOnce sync.Once
cpuCoresCache int
)
// getCPUCores reads /proc/cpuinfo and counts "processor\t" lines.
// The result is cached after the first call. Returns 1 on error.
func getCPUCores() int {
cpuCoresOnce.Do(func() {
f, err := osFS.Open("/proc/cpuinfo")
if err != nil {
cpuCoresCache = 1
return
}
defer func() { _ = f.Close() }()
count := 0
scanner := bufio.NewScanner(f)
for scanner.Scan() {
if strings.HasPrefix(scanner.Text(), "processor\t") {
count++
}
}
if count == 0 {
count = 1
}
cpuCoresCache = count
})
return cpuCoresCache
}
// parseLoadAvg reads /proc/loadavg and returns the first three load average
// values (1m, 5m, 15m).
func parseLoadAvg() ([3]float64, error) {
var result [3]float64
data, err := osFS.ReadFile("/proc/loadavg")
if err != nil {
return result, fmt.Errorf("reading /proc/loadavg: %w", err)
}
fields := strings.Fields(string(data))
if len(fields) < 3 {
return result, fmt.Errorf("unexpected /proc/loadavg format: %q", string(data))
}
for i := 0; i < 3; i++ {
v, err := strconv.ParseFloat(fields[i], 64)
if err != nil {
return result, fmt.Errorf("parsing load avg field %d: %w", i, err)
}
result[i] = v
}
return result, nil
}
// parseMemInfo reads /proc/meminfo and returns total memory, available memory,
// swap total, and swap free - all in kilobytes.
func parseMemInfo() (total, available, swapTotal, swapFree uint64) {
f, err := osFS.Open("/proc/meminfo")
if err != nil {
return
}
defer func() { _ = f.Close() }()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
key := fields[0]
val, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
continue
}
switch key {
case "MemTotal:":
total = val
case "MemAvailable:":
available = val
case "SwapTotal:":
swapTotal = val
case "SwapFree:":
swapFree = val
}
}
return
}
const (
maxInt64Value = int64(1<<63 - 1)
maxUint64Value = ^uint64(0)
)
// uint64ToInt64Clamped narrows a uint64 to int64 for byte display.
func uint64ToInt64Clamped(v uint64) int64 {
if v > uint64(maxInt64Value) {
return maxInt64Value
}
return int64(v)
}
func redisLargeDatasetThresholdBytes(gb int) uint64 {
if gb <= 0 {
return 0
}
const gbBytes uint64 = 1024 * 1024 * 1024
thresholdGB := uint64(gb)
if thresholdGB > maxUint64Value/gbBytes {
return maxUint64Value
}
return thresholdGB * gbBytes
}
// humanBytes formats a byte count as a human-readable string.
// Thresholds: >=1G → "1.0G", >=1M → "1M", >=1K → "1K", else "0B".
func humanBytes(b int64) string {
const (
KB = 1024
MB = 1024 * KB
GB = 1024 * MB
)
switch {
case b >= GB:
return fmt.Sprintf("%.1fG", float64(b)/float64(GB))
case b >= MB:
return fmt.Sprintf("%dM", b/MB)
case b >= KB:
return fmt.Sprintf("%dK", b/KB)
default:
return "0B"
}
}
func kibToDisplayBytes(kib uint64) int64 {
const maxKiBForInt64Bytes = uint64(maxInt64Value / 1024)
if kib > maxKiBForInt64Bytes {
return maxInt64Value
}
return int64(kib) * 1024
}
// CheckLoadAverage compares load averages against per-core thresholds
// from config. The 1-minute load drives the Critical / High findings;
// when 1-minute is below the High threshold we additionally check the
// 5- and 15-minute averages for sustained pressure (>= 0.7 * High
// threshold on both) and emit a Warning. The sustained variant catches
// "constant 22%-of-cores busy for 15 minutes" which is invisible to a
// 1-minute spike check but is what operators actually want to see.
func CheckLoadAverage(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
loads, err := parseLoadAvg()
if err != nil {
return nil
}
cores := getCPUCores()
load1 := loads[0]
critThreshold := float64(cores) * cfg.Performance.LoadCriticalMultiplier
highThreshold := float64(cores) * cfg.Performance.LoadHighMultiplier
switch {
case load1 > critThreshold:
return []alert.Finding{{
Severity: alert.Critical,
Check: "perf_load",
Message: "High load average exceeds critical threshold",
Details: fmt.Sprintf("Load: %.1f/%.1f/%.1f, Cores: %d, Threshold: %.1f",
loads[0], loads[1], loads[2], cores, critThreshold),
Timestamp: time.Now(),
}}
case load1 > highThreshold:
return []alert.Finding{{
Severity: alert.High,
Check: "perf_load",
Message: "High load average exceeds high threshold",
Details: fmt.Sprintf("Load: %.1f/%.1f/%.1f, Cores: %d, Threshold: %.1f",
loads[0], loads[1], loads[2], cores, highThreshold),
Timestamp: time.Now(),
}}
}
// Sustained pressure: 1-minute is calm but 5- and 15-minute
// averages are both above 70% of the High threshold. This is the
// "load 9 on 40 cores for 15 minutes" shape -- below the spike
// threshold but a real operator concern.
sustainedThreshold := highThreshold * 0.7
if loads[1] > sustainedThreshold && loads[2] > sustainedThreshold {
return []alert.Finding{{
Severity: alert.Warning,
Check: "perf_load",
Message: "Sustained load (5m + 15m) above 70% of high threshold",
Details: fmt.Sprintf("Load: %.1f/%.1f/%.1f, Cores: %d, Sustained threshold: %.1f",
loads[0], loads[1], loads[2], cores, sustainedThreshold),
Timestamp: time.Now(),
}}
}
return nil
}
// phpWorkersByUser walks /proc and returns, per username, the cmdline samples
// of that user's live PHP web-worker processes. Instantaneous snapshot; callers that
// need a count use len(result[user]).
func phpWorkersByUser() map[string][]string {
cmdlinePaths, _ := osFS.Glob("/proc/[0-9]*/cmdline")
userProcs := make(map[string][]string)
for _, cmdPath := range cmdlinePaths {
pid := filepath.Base(filepath.Dir(cmdPath))
data, err := osFS.ReadFile(cmdPath)
if err != nil {
continue
}
cmdStr := strings.ReplaceAll(string(data), "\x00", " ")
cmdStr = strings.TrimSpace(cmdStr)
safeCmdStr := redactProcCommandLine(data)
if !isPHPWorkerCommand(cmdStr) {
continue
}
// Read UID from status
statusData, _ := osFS.ReadFile(filepath.Join("/proc", pid, "status"))
var uid string
for _, line := range strings.Split(string(statusData), "\n") {
if strings.HasPrefix(line, "Uid:\t") {
fields := strings.Fields(strings.TrimPrefix(line, "Uid:\t"))
if len(fields) > 0 {
uid = fields[0]
}
break
}
}
if uid == "" {
uid = "unknown"
}
username := uidStringToUser(uid)
userProcs[username] = append(userProcs[username], safeCmdStr)
}
return userProcs
}
func isPHPWorkerCommand(command string) bool {
fields := strings.Fields(command)
if len(fields) == 0 {
return false
}
name := strings.TrimSuffix(strings.ToLower(filepath.Base(fields[0])), ":")
return isVersionedPHPWorkerBinary(name, "lsphp") ||
isVersionedPHPWorkerBinary(name, "php-cgi") ||
isVersionedPHPWorkerBinary(name, "php-fpm")
}
func isVersionedPHPWorkerBinary(name, base string) bool {
if name == base {
return true
}
suffix := strings.TrimPrefix(name, base)
if suffix == name || suffix == "" || suffix[0] < '0' || suffix[0] > '9' {
return false
}
for _, char := range suffix {
if (char < '0' || char > '9') && char != '.' {
return false
}
}
return suffix[len(suffix)-1] != '.'
}
// CheckPHPProcessLoad scans /proc for PHP web workers, groups them by user,
// and fires Critical if total exceeds cores*multiplier, High per user if
// individual count exceeds threshold.
func CheckPHPProcessLoad(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
cores := getCPUCores()
userProcs := phpWorkersByUser()
total := 0
for _, procs := range userProcs {
total += len(procs)
}
if total == 0 {
return nil
}
var findings []alert.Finding
// Critical: total PHP worker count exceeds cores * multiplier
critTotalThreshold := cores * cfg.Performance.PHPProcessCriticalTotalMult
if total > critTotalThreshold {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "perf_php_processes",
Message: "Total PHP worker process count exceeds critical threshold",
Details: fmt.Sprintf("Count: %d, Threshold: %d (cores: %d × %d)", total, critTotalThreshold, cores, cfg.Performance.PHPProcessCriticalTotalMult),
Timestamp: time.Now(),
})
}
// High: per-user count exceeds threshold
for username, procs := range userProcs {
if len(procs) > cfg.Performance.PHPProcessWarnPerUser {
// Collect up to 3 sample cmdlines
samples := procs
if len(samples) > 3 {
samples = samples[:3]
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "perf_php_processes",
Message: fmt.Sprintf("Excessive PHP worker processes for user %s", username),
Details: fmt.Sprintf("Count: %d, Threshold: %d, Sample cmdlines: %s", len(procs), cfg.Performance.PHPProcessWarnPerUser, alert.RedactCommandLine(strings.Join(samples, " | "))),
Timestamp: time.Now(),
})
}
}
return findings
}
// CheckSwapAndOOM checks for OOM killer events in dmesg and elevated swap
// usage from /proc/meminfo. Host OOM is Critical, cgroup OOM Warning, and
// swap usage above 50% High.
func CheckSwapAndOOM(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
var findings []alert.Finding
// Check dmesg for OOM events
// Prefer ISO timestamps so we can filter to the last hour.
// Fall back to -T (human-readable) on older kernels that don't support --time-format.
dmesgOut, isoErr := runCmd("dmesg", "--time-format", "iso", "--level=err")
useISO := isoErr == nil && dmesgOut != nil
if !useISO {
dmesgOut, _ = runCmd("dmesg", "--level=err", "-T")
}
if dmesgOut != nil {
cutoff := time.Now().Add(-1 * time.Hour)
seen := make(map[string]bool)
for _, line := range strings.Split(string(dmesgOut), "\n") {
lower := strings.ToLower(line)
if !strings.Contains(lower, "out of memory") && !strings.Contains(lower, "oom_reaper") {
continue
}
// Both the ISO and the -T fallback are filtered to the last
// hour. A line whose timestamp cannot be parsed is skipped, not
// reported: an OOM event we cannot date is exactly the stale
// finding that previously fired a Critical on every scan.
when, ok := parseDmesgOOMTime(line, useISO)
if !ok || when.Before(cutoff) {
continue
}
key := oomDedupKey(line)
if seen[key] {
continue
}
seen[key] = true
severity, accountScoped := classifyOOMLine(line)
message := "OOM killer invoked in the last hour"
if accountScoped {
message = "Account memory limit reached in the last hour"
}
findings = append(findings, alert.Finding{
Severity: severity,
Check: "perf_memory",
Message: message,
Details: strings.TrimSpace(line),
// Every kill logs a fresh pid and byte counts; keying dedup on
// the victim process name keeps an ongoing OOM loop to one
// finding per state-expiry window instead of one per scan.
DedupKey: key,
Timestamp: time.Now(),
})
}
}
// Check swap usage
_, _, swapTotal, swapFree := parseMemInfo()
if swapTotal > 0 {
swapUsed := swapTotal - swapFree
usagePct := float64(swapUsed) / float64(swapTotal) * 100
if usagePct > 50 {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "perf_memory",
Message: "High swap usage",
Details: fmt.Sprintf("Swap used: %s / %s (%.0f%%)", humanBytes(kibToDisplayBytes(swapUsed)), humanBytes(kibToDisplayBytes(swapTotal)), usagePct),
// The percentage moves every scan; without a pinned identity
// each drift re-alerts the same sustained condition.
DedupKey: "swap_high",
Timestamp: time.Now(),
})
}
}
return findings
}
// oomVictimProcess returns the killed process name from a dmesg OOM line
// ("... Killed process 2845662 (lsphp) ..."), or "host" when the line names
// no victim, so the dedup identity always has a stable value. Parenthesized
// OOM context before the process marker must not be mistaken for the victim.
func oomVictimProcess(line string) string {
for _, marker := range []string{"Killed process ", "reaped process "} {
markerIdx := strings.Index(line, marker)
if markerIdx < 0 {
continue
}
rest := line[markerIdx+len(marker):]
pidEnd := strings.IndexAny(rest, " \t")
if pidEnd <= 0 {
continue
}
if _, err := strconv.ParseUint(rest[:pidEnd], 10, 64); err != nil {
continue
}
rest = strings.TrimLeft(rest[pidEnd:], " \t")
if len(rest) < 3 || rest[0] != '(' {
continue
}
closeIdx := strings.IndexByte(rest[1:], ')')
if closeIdx <= 0 {
continue
}
if process := strings.TrimSpace(rest[1 : closeIdx+1]); process != "" {
return process
}
}
return "host"
}
// classifyOOMLine separates a host-wide OOM from a cgroup one. On a shared
// host a cgroup kill is an account reaching the memory limit its plan sets:
// routine, and not evidence about host health. Only real memory exhaustion is
// Critical, or the two become indistinguishable in the alert stream.
func classifyOOMLine(line string) (alert.Severity, bool) {
if strings.Contains(strings.ToLower(line), "memory cgroup out of memory") {
return alert.Warning, true
}
return alert.Critical, false
}
// oomDedupKey keeps the account-scoped and host-wide cases on separate dedup
// identities, so one account repeatedly hitting its limit cannot suppress the
// host-wide alert that follows it.
func oomDedupKey(line string) string {
if _, accountScoped := classifyOOMLine(line); accountScoped {
return "oom:cgroup:" + oomVictimProcess(line)
}
return "oom:host:" + oomVictimProcess(line)
}
// parseDmesgOOMTime extracts the event time from a dmesg line. ISO lines
// (--time-format iso) carry an absolute timestamp with a timezone offset as
// the first field. The -T fallback carries a bracketed ctime in local time
// ("[Mon Jan _2 15:04:05 2006] ..."). Returns ok=false when no timestamp can
// be parsed, so the caller drops the line rather than reporting an undatable
// (and therefore possibly stale) OOM event.
func parseDmesgOOMTime(line string, useISO bool) (time.Time, bool) {
if useISO {
// 2006-01-02T15:04:05,000000+0300 -- comma decimal, first field.
ts := strings.Replace(strings.SplitN(line, " ", 2)[0], ",", ".", 1)
for _, layout := range []string{"2006-01-02T15:04:05.000000-0700", "2006-01-02T15:04:05.000000-07:00"} {
if parsed, err := time.Parse(layout, ts); err == nil {
return parsed, true
}
}
return time.Time{}, false
}
open := strings.IndexByte(line, '[')
closeIdx := strings.IndexByte(line, ']')
if open != 0 || closeIdx <= open {
return time.Time{}, false
}
// dmesg -T prints local wall-clock time with no zone, so parse in Local.
parsed, err := time.ParseInLocation("Mon Jan _2 15:04:05 2006", strings.TrimSpace(line[open+1:closeIdx]), time.Local)
if err != nil {
return time.Time{}, false
}
return parsed, true
}
// CheckPHPHandler detects PHP CGI handler usage on LiteSpeed servers.
// On LiteSpeed, CGI is significantly slower than LSAPI; this check fires
// a Critical finding for each PHP version using the CGI handler.
func CheckPHPHandler(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
// Only relevant on LiteSpeed
if _, err := osFS.Stat("/usr/local/lsws/bin/litespeed"); err != nil {
return nil
}
var cgiVersions []string
// Try whmapi1 first
out, err := runCmd("whmapi1", "php_get_handlers", "--output=json")
if err == nil && len(out) > 0 {
// Parse JSON: look for handler entries with type "cgi"
var result struct {
Data struct {
Handlers []struct {
Version string `json:"version"`
Handler string `json:"handler"`
Type string `json:"type"`
} `json:"handlers"`
} `json:"data"`
}
if jsonErr := json.Unmarshal(out, &result); jsonErr == nil {
for _, h := range result.Data.Handlers {
t := strings.ToLower(h.Handler + " " + h.Type)
if strings.Contains(t, "cgi") && !strings.Contains(t, "lsapi") && !strings.Contains(t, "fpm") {
cgiVersions = append(cgiVersions, h.Version)
}
}
}
} else {
// Fallback: read /etc/cpanel/ea4/ea4.conf
data, readErr := osFS.ReadFile("/etc/cpanel/ea4/ea4.conf")
if readErr == nil {
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
// Lines like: ea-php74.handler = cgi
if !strings.Contains(line, ".handler") {
continue
}
parts := strings.SplitN(line, "=", 2)
if len(parts) != 2 {
continue
}
val := strings.TrimSpace(parts[1])
if val == "cgi" {
versionPart := strings.TrimSpace(parts[0])
cgiVersions = append(cgiVersions, versionPart)
}
}
}
}
if len(cgiVersions) == 0 {
return nil
}
return []alert.Finding{{
Severity: alert.Critical,
Check: "perf_php_handler",
Message: "PHP handler set to CGI instead of LSAPI on LiteSpeed",
Details: fmt.Sprintf("Affected PHP versions: %s", strings.Join(cgiVersions, ", ")),
Timestamp: time.Now(),
}}
}
// CheckMySQLConfig inspects MySQL global variables and runtime status for
// performance-impacting misconfigurations. Each issue emits its own finding
// with a stable message so deduplication works correctly.
func CheckMySQLConfig(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
var findings []alert.Finding
// --- Global variables ---
varRows, err := mysqlclient.RootQuery(ctx,
"SHOW GLOBAL VARIABLES WHERE Variable_name IN ('join_buffer_size','wait_timeout','interactive_timeout','max_user_connections','slow_query_log')")
varOut := []byte(strings.Join(varRows, "\n"))
if err == nil && len(varOut) > 0 {
joinBufThresholdBytes := int64(cfg.Performance.MySQLJoinBufferMaxMB) * 1024 * 1024
waitTimeoutMax := cfg.Performance.MySQLWaitTimeoutMax
for _, line := range strings.Split(string(varOut), "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
name := fields[0]
val := fields[1]
switch name {
case "join_buffer_size":
v, convErr := strconv.ParseInt(val, 10, 64)
if convErr == nil && v > joinBufThresholdBytes {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "perf_mysql_config",
Message: "MySQL join_buffer_size exceeds safe maximum",
Details: fmt.Sprintf("Current: %s, Max: %s", humanBytes(v), humanBytes(joinBufThresholdBytes)),
Timestamp: time.Now(),
})
}
case "wait_timeout":
v, convErr := strconv.Atoi(val)
if convErr == nil && v > waitTimeoutMax {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "perf_mysql_config",
Message: "MySQL wait_timeout is too high",
Details: fmt.Sprintf("Current: %ds, Max: %ds", v, waitTimeoutMax),
Timestamp: time.Now(),
})
}
case "interactive_timeout":
v, convErr := strconv.Atoi(val)
if convErr == nil && v > waitTimeoutMax {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "perf_mysql_config",
Message: "MySQL interactive_timeout is too high",
Details: fmt.Sprintf("Current: %ds, Max: %ds", v, waitTimeoutMax),
Timestamp: time.Now(),
})
}
case "max_user_connections":
if val == "0" {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "perf_mysql_config",
Message: "MySQL max_user_connections is unlimited",
Details: fmt.Sprintf("Current: 0 (unlimited), Recommended: %d", cfg.Performance.MySQLMaxConnectionsPerUser),
Timestamp: time.Now(),
})
}
case "slow_query_log":
if strings.ToUpper(val) == "OFF" {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "perf_mysql_config",
Message: "MySQL slow query log is disabled",
Details: "Set slow_query_log=ON to help diagnose performance issues",
Timestamp: time.Now(),
})
}
}
}
}
// --- InnoDB buffer pool hit ratio + temporary disk tables ---
statusRows, err := mysqlclient.RootQuery(ctx,
"SHOW GLOBAL STATUS WHERE Variable_name IN ('Innodb_buffer_pool_read_requests','Innodb_buffer_pool_reads','Created_tmp_disk_tables','Created_tmp_tables')")
statusOut := []byte(strings.Join(statusRows, "\n"))
if err == nil && len(statusOut) > 0 {
var readRequests, reads, tmpDiskTables, tmpTables int64
for _, line := range strings.Split(string(statusOut), "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
v, convErr := strconv.ParseInt(fields[1], 10, 64)
if convErr != nil {
continue
}
switch fields[0] {
case "Innodb_buffer_pool_read_requests":
readRequests = v
case "Innodb_buffer_pool_reads":
reads = v
case "Created_tmp_disk_tables":
tmpDiskTables = v
case "Created_tmp_tables":
tmpTables = v
}
}
if tmpTables > 0 && tmpDiskTables > 0 {
diskRatio := float64(tmpDiskTables) / float64(tmpTables) * 100
if diskRatio > 25.0 {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "perf_mysql_config",
Message: "MySQL creating excessive temporary tables on disk",
Details: fmt.Sprintf("Disk ratio: %.1f%% (%d disk tables / %d total tables)", diskRatio, tmpDiskTables, tmpTables),
Timestamp: time.Now(),
})
}
}
if readRequests > 0 {
hitRatio := float64(readRequests-reads) / float64(readRequests) * 100
if hitRatio < 95.0 {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "perf_mysql_config",
Message: "InnoDB buffer pool hit ratio is low",
Details: fmt.Sprintf("Hit ratio: %.1f%% (threshold: 95%%), disk reads: %d", hitRatio, reads),
Timestamp: time.Now(),
})
}
}
}
// --- Per-user connection counts ---
plRows, err := mysqlclient.RootQuery(ctx, "SHOW PROCESSLIST")
if err == nil && len(plRows) > 0 {
userCounts := make(map[string]int)
for _, line := range plRows {
fields := strings.Fields(line)
// SHOW PROCESSLIST columns: Id, User, Host, db, Command, Time, State, Info
if len(fields) < 2 {
continue
}
user := fields[1]
if user == "" || user == "User" {
continue
}
userCounts[user]++
}
maxConn := cfg.Performance.MySQLMaxConnectionsPerUser
for dbUser, count := range userCounts {
if count > maxConn {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "perf_mysql_config",
Message: fmt.Sprintf("MySQL user %s holding excessive connections", dbUser),
Details: fmt.Sprintf("Connections: %d, Threshold: %d", count, maxConn),
Timestamp: time.Now(),
})
}
}
}
return findings
}
// CheckRedisConfig inspects a local Redis instance for performance-impacting
// misconfigurations: unset maxmemory, noeviction policy, non-expiring keys,
// and an overly aggressive bgsave schedule for the dataset size.
func CheckRedisConfig(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
// Skip on hosts without a local redis: peek at the redisinfo
// client by trying a fast INFO server. Any error short-circuits
// the whole check (matches the historical behaviour where a
// missing redis-cli binary made every redis check a no-op).
if _, _, err := redisinfo.MemoryUsage(ctx); err != nil {
return nil
}
var findings []alert.Finding
// --- maxmemory ---
if maxMem, err := redisinfo.ConfigGet(ctx, "maxmemory"); err == nil && maxMem == "0" {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "perf_redis_config",
Message: "Redis maxmemory is not set",
Details: "maxmemory=0 means Redis will use all available system memory without bound",
Timestamp: time.Now(),
})
}
// --- maxmemory-policy ---
policy, policyErr := redisinfo.ConfigGet(ctx, "maxmemory-policy")
policyLower := ""
if policyErr == nil {
policyLower = strings.ToLower(strings.TrimSpace(policy))
}
if policyLower == "noeviction" {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "perf_redis_config",
Message: "Redis maxmemory-policy is noeviction",
Details: "noeviction causes Redis to return errors when memory is full instead of evicting keys",
Timestamp: time.Now(),
})
}
// --- Non-expiring keys ratio via keyspace ---
// A high non-expiring ratio only breaks eviction under volatile-* policies
// (which evict keys carrying a TTL) or noeviction. Under allkeys-* Redis
// evicts any key, so non-expiring keys are reclaimable and the ratio is
// benign.
if stats, err := redisinfo.KeyspaceStats(ctx); err == nil && stats.TotalKeys > 0 && !strings.HasPrefix(policyLower, "allkeys-") {
nonExpiring := stats.TotalKeys - stats.TotalExpires
ratio := float64(nonExpiring) / float64(stats.TotalKeys) * 100
if ratio > 95.0 {
details := fmt.Sprintf("Non-expiring: %d / %d total keys (%.1f%%); %s",
nonExpiring, stats.TotalKeys, ratio, redisNonExpiringPolicyDetail(policyLower))
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "perf_redis_config",
Message: "Redis has excessive non-expiring keys",
Details: details,
Timestamp: time.Now(),
})
}
}
// --- bgsave interval vs dataset size ---
saveSpec, _ := redisinfo.ConfigGet(ctx, "save")
usedBytes, _, _ := redisinfo.MemoryUsage(ctx)
largeDatasetBytes := redisLargeDatasetThresholdBytes(cfg.Performance.RedisLargeDatasetGB)
bgsaveMinInterval := cfg.Performance.RedisBgsaveMinInterval
if usedBytes > largeDatasetBytes && saveSpec != "" {
// `CONFIG GET save` returns the spec as a single space-separated
// string of alternating "<seconds> <changes>" pairs, e.g.
// "900 1 300 10 60 10000". Walk the seconds tokens (every
// other field) and flag any below the configured floor.
fields := strings.Fields(saveSpec)
aggressiveSave := false
for i := 0; i < len(fields); i += 2 {
seconds, convErr := strconv.Atoi(fields[i])
if convErr == nil && seconds < bgsaveMinInterval {
aggressiveSave = true
break
}
}
if aggressiveSave {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "perf_redis_config",
Message: "Redis bgsave interval too aggressive for dataset size",
Details: fmt.Sprintf(
"Used memory: %s, Threshold: %s, Minimum safe bgsave interval: %ds",
humanBytes(uint64ToInt64Clamped(usedBytes)),
humanBytes(uint64ToInt64Clamped(largeDatasetBytes)),
bgsaveMinInterval,
),
Timestamp: time.Now(),
})
}
}
// --- used_memory vs maxmemory headroom ---
// The maxmemory==0 branch above flags the unset case. When maxmemory
// IS set, used/max ratio is the operator-meaningful signal: at 80%
// the eviction policy is about to start churning hot keys; at 90%
// noeviction-policy instances start returning OOM errors.
_, maxBytes, _ := redisinfo.MemoryUsage(ctx)
if maxBytes > 0 && usedBytes > 0 {
pct := float64(usedBytes) / float64(maxBytes) * 100
switch {
case pct >= 90:
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "perf_redis_config",
Message: "Redis used memory >= 90% of maxmemory",
Details: fmt.Sprintf(
"Used: %s / Max: %s (%.1f%%)",
humanBytes(uint64ToInt64Clamped(usedBytes)),
humanBytes(uint64ToInt64Clamped(maxBytes)),
pct,
),
Timestamp: time.Now(),
})
case pct >= 80:
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "perf_redis_config",
Message: "Redis used memory >= 80% of maxmemory",
Details: fmt.Sprintf(
"Used: %s / Max: %s (%.1f%%)",
humanBytes(uint64ToInt64Clamped(usedBytes)),
humanBytes(uint64ToInt64Clamped(maxBytes)),
pct,
),
Timestamp: time.Now(),
})
}
}
return findings
}
func redisNonExpiringPolicyDetail(policyLower string) string {
switch {
case policyLower == "":
return "maxmemory-policy is unavailable, so non-expiring keys may be unsafe under memory pressure"
case policyLower == "noeviction":
return "maxmemory-policy noeviction does not evict keys under memory pressure"
case strings.HasPrefix(policyLower, "volatile-"):
return fmt.Sprintf("maxmemory-policy %q only evicts keys with a TTL under memory pressure", policyLower)
default:
return fmt.Sprintf("maxmemory-policy %q may leave non-expiring keys unreclaimable under memory pressure", policyLower)
}
}
// ---------------------------------------------------------------------------
// Performance check helpers (WP-specific)
// ---------------------------------------------------------------------------
// safeIdentifier returns true if s matches ^[a-zA-Z0-9_]+$ (non-empty).
// Used to reject values with shell metacharacters before use in commands/SQL.
var safeIdentRe = regexp.MustCompile(`^[a-zA-Z0-9_]+$`)
func safeIdentifier(s string) bool {
return s != "" && safeIdentRe.MatchString(s)
}
// extractPHPDefine extracts the value argument from a PHP define() line:
//
// define('KEY', 'value'); or define("KEY", "value");
//
// It is distinct from extractDefine (dbscan.go) which requires a key parameter.
// Returns the empty string if no value can be extracted.
func extractPHPDefine(line string) string {
// Trim whitespace and trailing semicolons/comments.
line = strings.TrimSpace(line)
// Find the opening parenthesis.
parenIdx := strings.Index(line, "(")
if parenIdx < 0 {
return ""
}
inner := line[parenIdx+1:]
// Strip closing paren and anything after.
if closeIdx := strings.LastIndex(inner, ")"); closeIdx >= 0 {
inner = inner[:closeIdx]
}
// inner is now like: 'KEY', 'value' or "KEY", "value"
// Split on the first comma, ignoring the key part.
commaIdx := strings.Index(inner, ",")
if commaIdx < 0 {
return ""
}
valuePart := strings.TrimSpace(inner[commaIdx+1:])
if valuePart == "" {
return ""
}
// Strip surrounding quotes (single or double) when present.
q := valuePart[0]
if q == '\'' || q == '"' {
if len(valuePart) < 2 {
return ""
}
end := strings.LastIndexByte(valuePart, q)
if end <= 0 {
return ""
}
return valuePart[1:end]
}
// Unquoted literal (boolean/number constant). Strip a trailing ); or
// whitespace and return the bare token. Examples wp-config.php uses:
// define('DISABLE_WP_CRON', true);
// define('WP_DEBUG', false);
// define('WP_MEMORY_LIMIT', 256);
for i, c := range valuePart {
if c == ' ' || c == '\t' || c == ';' || c == ')' || c == ',' {
return strings.TrimSpace(valuePart[:i])
}
}
return strings.TrimSpace(valuePart)
}
// ---------------------------------------------------------------------------
// Subdirs to skip in recursive helpers.
// ---------------------------------------------------------------------------
var skipDirs = map[string]bool{
"wp-admin": true,
"wp-content": true,
"wp-includes": true,
"cache": true,
"node_modules": true,
"vendor": true,
}
// ---------------------------------------------------------------------------
// CheckErrorLogBloat
// ---------------------------------------------------------------------------
const (
errorLogFindingLimit = 20
errorLogSizesKey = "_perf_error_log_sizes"
// errorLogMinRateSpan is the shortest span a growth rate is computed over.
// A restart or manual rescan minutes after the last run would otherwise
// extrapolate a few seconds of writes into a daily rate.
errorLogMinRateSpan = time.Hour
)
var errorLogNow = time.Now
type bloatedErrorLog struct {
path string
size int64
device uint64
inode uint64
}
// Size and Seen anchor the rate window; LastSize detects a shrink even when
// frequent scans have not yet advanced that window. Identity detects rotation.
type errorLogObservation struct {
Size int64 `json:"size"`
Seen int64 `json:"seen"`
LastSize int64 `json:"last_size"`
Device uint64 `json:"device"`
Inode uint64 `json:"inode"`
}
// errorLogScan collects bloated logs and the paths the walk proved. Only a log
// with a saved baseline or one bloated now can carry a finding to retire, so
// coverage is recorded for those alone rather than for every directory walked.
type errorLogScan struct {
logs []bloatedErrorLog
covered map[string]bool
tracked map[string]bool
}
func (s *errorLogScan) observe(path string, info os.FileInfo, thresholdBytes int64) {
bloated := info != nil && info.Mode().IsRegular() && info.Size() > thresholdBytes
if !bloated && !s.tracked[path] {
return
}
if s.covered == nil {
s.covered = make(map[string]bool)
}
s.covered[path] = true
if !bloated {
return
}
identity, _ := selfWriteIdentityFromFileInfo(info)
s.logs = append(s.logs, bloatedErrorLog{
path: path, size: info.Size(), device: identity.Device, inode: identity.Inode,
})
}
// Heavy trees are not descended, but their immediate error_log is checked.
// A tracked log found small or missing is recorded as covered, so a partial
// scan can retire its finding without discarding evidence from unreadable
// paths.
func scanErrorLogs(ctx context.Context, dir string, thresholdBytes int64, depth int, roots map[string]struct{}, scan *errorLogScan) bool {
if ctx.Err() != nil {
return false
}
if depth < 0 {
return true
}
entries, err := osFS.ReadDir(dir)
if err != nil {
return errors.Is(err, fs.ErrNotExist)
}
complete, sawLog := true, false
for _, e := range entries {
if ctx.Err() != nil {
return false
}
name := e.Name()
fullPath := filepath.Join(dir, name)
if name == "error_log" {
sawLog = true
info, infoErr := e.Info()
switch {
case infoErr == nil:
scan.observe(fullPath, info, thresholdBytes)
case errors.Is(infoErr, fs.ErrNotExist):
scan.observe(fullPath, nil, thresholdBytes)
default:
complete = false
}
}
if !e.IsDir() || depth == 0 {
continue
}
if _, ownRoot := roots[fullPath]; ownRoot {
continue
}
if skipDirs[name] {
path := filepath.Join(fullPath, "error_log")
info, statErr := osFS.Lstat(path)
switch {
case statErr == nil:
scan.observe(path, info, thresholdBytes)
case errors.Is(statErr, fs.ErrNotExist):
scan.observe(path, nil, thresholdBytes)
default:
complete = false
}
continue
}
if !scanErrorLogs(ctx, fullPath, thresholdBytes, depth-1, roots, scan) {
complete = false
}
}
if !sawLog {
scan.observe(filepath.Join(dir, "error_log"), nil, thresholdBytes)
}
return complete
}
// CheckErrorLogBloat walks the configured web roots and every docroot in
// cPanel's domain map looking for error_log files over the warning size. Logs
// over the critical size are High so the operator is told; the rest only show
// on the Performance page. The largest logs are kept when the finding cap is
// reached. The runner enforces a 60-minute throttle via checkThrottleMin.
func CheckErrorLogBloat(ctx context.Context, cfg *config.Config, scanState *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
warnBytes := int64(cfg.Performance.ErrorLogWarnSizeMB) * 1024 * 1024
critBytes := int64(cfg.Performance.ErrorLogCriticalSizeMB) * 1024 * 1024
roots, complete := validatedDocrootSet(cfg)
rootSet := make(map[string]struct{}, len(roots))
for _, root := range roots {
rootSet[root.path] = struct{}{}
}
previous := loadErrorLogSizes(scanState)
scan := errorLogScan{tracked: make(map[string]bool, len(previous))}
for path := range previous {
scan.tracked[path] = true
}
for _, root := range roots {
if !scanErrorLogs(ctx, root.path, warnBytes, 3, rootSet, &scan) {
complete = false
}
}
if !complete {
markCheckIncomplete(ctx, "perf_error_logs")
}
if ctx.Err() != nil {
markCheckIncomplete(ctx, "perf_error_logs")
return nil
}
sort.Slice(scan.logs, func(i, j int) bool {
if scan.logs[i].size != scan.logs[j].size {
return scan.logs[i].size > scan.logs[j].size
}
return scan.logs[i].path < scan.logs[j].path
})
now := errorLogNow()
current := make(map[string]errorLogObservation, len(scan.logs))
var findings []alert.Finding
for i, log := range scan.logs {
details, baseline := errorLogGrowth(log, previous[log.path], now)
current[log.path] = baseline
if i < errorLogFindingLimit {
findings = append(findings, newErrorLogFinding(log, details, critBytes, now))
}
}
for path, obs := range previous {
if _, seen := current[path]; seen {
continue
}
// Without the map, a nested root may be hidden behind an ancestor's
// depth or directory exclusions. Only direct observations retire it.
if complete || scan.covered[path] {
scan.observe(path, nil, warnBytes)
} else {
current[path] = obs
}
}
if ctx.Err() != nil {
markCheckIncomplete(ctx, "perf_error_logs")
return nil
}
scopes := make(map[string]bool, len(scan.covered))
for path := range scan.covered {
scopes[errorLogCoverageScope(path)] = true
}
recordCompletedCoverageScopes(ctx, "perf_error_logs", scopes)
storeErrorLogSizes(scanState, current)
return findings
}
// errorLogGrowth renders the finding details and returns the baseline to keep.
// A baseline younger than errorLogMinRateSpan keeps its rate anchor while
// tracking the latest size; a shrink or file replacement resets the anchor.
func errorLogGrowth(log bloatedErrorLog, prev errorLogObservation, now time.Time) (string, errorLogObservation) {
details := fmt.Sprintf("Size: %s", humanBytes(log.size))
fresh := errorLogObservation{
Size: log.size, Seen: now.Unix(), LastSize: log.size, Device: log.device, Inode: log.inode,
}
if prev.Seen == 0 || prev.Inode != log.inode || prev.Device != log.device ||
log.size < prev.Size || log.size < prev.LastSize || now.Unix() < prev.Seen {
return details, fresh
}
span := now.Sub(time.Unix(prev.Seen, 0))
if span < errorLogMinRateSpan {
prev.LastSize = log.size
return details, prev
}
perDay := int64(float64(log.size-prev.Size) * float64(24*time.Hour) / float64(span))
if perDay > 0 {
details += fmt.Sprintf(", growing %s/day", humanBytes(perDay))
}
return details, fresh
}
// newErrorLogFinding keys the finding on its path and tier, not its size, so
// a growing log stays one finding (one alert per day, a dismissal that
// sticks) while crossing into the critical tier is a new event.
func newErrorLogFinding(log bloatedErrorLog, details string, critBytes int64, now time.Time) alert.Finding {
severity, tier := alert.Warning, "warning"
if log.size > critBytes {
severity, tier = alert.High, "critical"
}
return alert.Finding{
Severity: severity,
Check: "perf_error_logs",
Message: fmt.Sprintf("Bloated error_log: %s", log.path),
Details: details,
DedupKey: tier + ":" + log.path,
CoverageScope: errorLogCoverageScope(log.path),
Timestamp: now,
}
}
func errorLogCoverageScope(path string) string {
return "error_log:" + path
}
func loadErrorLogSizes(scanState *state.Store) map[string]errorLogObservation {
if scanState == nil {
return nil
}
raw, ok := scanState.GetRaw(errorLogSizesKey)
if !ok {
return nil
}
var sizes map[string]errorLogObservation
if json.Unmarshal([]byte(raw), &sizes) != nil {
return nil
}
return sizes
}
// storeErrorLogSizes replaces the saved baselines, which drops any log that is
// gone or back under the warning size.
func storeErrorLogSizes(scanState *state.Store, sizes map[string]errorLogObservation) {
if scanState == nil {
return
}
if len(sizes) == 0 {
if err := scanState.DeleteRawAndSave(errorLogSizesKey); err != nil {
fmt.Fprintf(os.Stderr, "perf_error_logs: baseline clear: %v\n", err)
}
return
}
raw, err := json.Marshal(sizes)
if err != nil {
return
}
if err := scanState.SetRawAndSave(errorLogSizesKey, string(raw)); err != nil {
fmt.Fprintf(os.Stderr, "perf_error_logs: baseline write: %v\n", err)
}
}
// ---------------------------------------------------------------------------
// CheckWPConfig
// ---------------------------------------------------------------------------
// parseMemoryLimit converts a PHP memory_limit string (e.g. "256M", "1G")
// to megabytes. Returns 0 if the value cannot be parsed.
func parseMemoryLimit(s string) int {
s = strings.TrimSpace(strings.ToUpper(s))
if s == "" || s == "-1" {
return 0
}
suffix := s[len(s)-1]
numStr := s
mult := 1
switch suffix {
case 'K':
numStr = s[:len(s)-1]
v, err := strconv.Atoi(numStr)
if err != nil {
return 0
}
return v / 1024
case 'M':
numStr = s[:len(s)-1]
mult = 1
case 'G':
numStr = s[:len(s)-1]
mult = 1024
}
v, err := strconv.Atoi(numStr)
if err != nil {
return 0
}
return v * mult
}
// scanWPConfigs recursively searches dir (max depth) for wp-config.php files
// and checks WP_MEMORY_LIMIT and co-located config files for issues.
func scanWPConfigs(dir, account string, cfg *config.Config, depth int, findings *[]alert.Finding) {
if depth < 0 {
return
}
entries, err := osFS.ReadDir(dir)
if err != nil {
return
}
for _, e := range entries {
name := e.Name()
fullPath := filepath.Join(dir, name)
if e.IsDir() {
if skipDirs[name] {
continue
}
scanWPConfigs(fullPath, account, cfg, depth-1, findings)
continue
}
if name != "wp-config.php" {
continue
}
// --- WP_MEMORY_LIMIT ---
wpData, readErr := osFS.ReadFile(fullPath)
if readErr == nil {
for _, line := range strings.Split(string(wpData), "\n") {
if strings.Contains(line, "WP_MEMORY_LIMIT") {
val := extractPHPDefine(strings.TrimSpace(line))
if mb := parseMemoryLimit(val); mb > cfg.Performance.WPMemoryLimitMaxMB {
*findings = append(*findings, alert.Finding{
Severity: alert.Warning,
Check: "perf_wp_config",
Message: fmt.Sprintf("Excessive WP_MEMORY_LIMIT for %s", account),
Details: fmt.Sprintf("File: %s, Value: %s", fullPath, val),
Timestamp: time.Now(),
})
}
break
}
}
}
// --- Co-located PHP config files ---
wpDir := filepath.Dir(fullPath)
for _, cfgFile := range []string{".htaccess", "php.ini", ".user.ini"} {
cfgPath := filepath.Join(wpDir, cfgFile)
data, readErr2 := osFS.ReadFile(cfgPath)
if readErr2 != nil {
continue
}
// cPanel MultiPHP INI Editor writes .user.ini with a fixed
// header and owns the file's content. Values inside a
// cPanel-managed .user.ini (max_execution_time=0 for a
// backup importer, display_errors=On for a staging account)
// reflect operator choices made through the cPanel UI and
// are not attacker actions. Suppress findings for this
// file in that case — operators do not need alerts for
// their own configuration. The suppression is scoped
// strictly to .user.ini: the same signature in php.ini or
// .htaccess is not authoritative (cPanel does not write
// those files) and the scanner treats it normally.
if cfgFile == ".user.ini" && isCpanelManagedUserIni(data) {
continue
}
for _, line := range strings.Split(string(data), "\n") {
trimmed := strings.TrimSpace(line)
// Skip comment lines
if strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, ";") {
continue
}
lc := strings.ToLower(trimmed)
switch {
case strings.Contains(lc, "max_execution_time"):
// max_execution_time = 0 (or php_value max_execution_time 0)
parts := strings.FieldsFunc(trimmed, func(r rune) bool { return r == '=' || r == ' ' || r == '\t' })
if len(parts) >= 2 && parts[len(parts)-1] == "0" {
*findings = append(*findings, alert.Finding{
Severity: alert.High,
Check: "perf_wp_config",
Message: fmt.Sprintf("Unlimited max_execution_time for %s", account),
Details: fmt.Sprintf("File: %s, Value: 0", cfgPath),
Timestamp: time.Now(),
})
}
case strings.Contains(lc, "display_errors"):
parts := strings.FieldsFunc(trimmed, func(r rune) bool { return r == '=' || r == ' ' || r == '\t' })
if len(parts) >= 2 && strings.ToLower(parts[len(parts)-1]) == "on" {
*findings = append(*findings, alert.Finding{
Severity: alert.Warning,
Check: "perf_wp_config",
Message: fmt.Sprintf("display_errors enabled in production for %s", account),
Details: fmt.Sprintf("File: %s, Value: On", cfgPath),
Timestamp: time.Now(),
})
}
}
}
}
}
}
// CheckWPConfig scans /home/*/public_html (max depth 2) for wp-config.php
// files and reports excessive WP_MEMORY_LIMIT values, unlimited
// max_execution_time, and display_errors enabled in production.
// The runner enforces a 60-minute throttle via checkThrottleMin.
func CheckWPConfig(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
homeDirs := ResolveWebRoots(cfg)
var findings []alert.Finding
for _, dir := range homeDirs {
scanWPConfigs(dir, accountFromPath(dir), cfg, 2, &findings)
}
return findings
}
// accountFromPath extracts a best-effort account name from a web root path.
// On cPanel (/home/USER/public_html) it returns USER. On other layouts it
// returns the parent directory name, or the final path component if there
// is no parent. Used for reporting only — never for authorization.
func accountFromPath(dir string) string {
parts := strings.Split(dir, string(filepath.Separator))
// cPanel shape: /home/<account>/public_html
for i, p := range parts {
if p == "home" && i+1 < len(parts) {
return parts[i+1]
}
}
// Generic shape: /var/www/<site>, /srv/http/<site>, etc.
if len(parts) >= 2 && parts[len(parts)-1] != "" {
return parts[len(parts)-2]
}
return filepath.Base(dir)
}
// ---------------------------------------------------------------------------
// CheckWPTransientBloat
// ---------------------------------------------------------------------------
// findWPTransients recursively searches dir for wp-config.php files and
// queries the WordPress database for bloated transients.
func findWPTransients(dir string, warnBytes, critBytes int64, depth int, findings *[]alert.Finding) {
if depth < 0 {
return
}
entries, err := osFS.ReadDir(dir)
if err != nil {
return
}
for _, e := range entries {
name := e.Name()
fullPath := filepath.Join(dir, name)
if e.IsDir() {
if skipDirs[name] {
continue
}
findWPTransients(fullPath, warnBytes, critBytes, depth-1, findings)
continue
}
if name != "wp-config.php" {
continue
}
info := parseWPConfig(fullPath)
if info.dbName == "" || info.dbUser == "" {
continue
}
// Apply default table prefix when not set.
if info.tablePrefix == "" {
info.tablePrefix = "wp_"
}
// Security: validate identifiers before use in SQL.
if !safeIdentifier(info.dbName) || !safeIdentifier(info.dbUser) || !safeIdentifier(info.tablePrefix) {
continue
}
query := fmt.Sprintf(
"SELECT option_name, LENGTH(option_value) as size FROM %soptions WHERE option_name LIKE '_transient_%%' AND LENGTH(option_value) > %d ORDER BY size DESC LIMIT 5",
info.tablePrefix,
warnBytes,
)
rows, runErr := mysqlclient.PerAccountQuery(context.Background(), mysqlclient.Creds{
User: info.dbUser,
Password: info.dbPass,
Host: info.dbHost,
DBName: info.dbName,
}, query)
if runErr != nil || len(rows) == 0 {
continue
}
for _, line := range rows {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
optionName := fields[0]
sizeBytes, convErr := strconv.ParseInt(fields[1], 10, 64)
if convErr != nil {
continue
}
var sev alert.Severity
switch {
case sizeBytes > critBytes:
sev = alert.High
case sizeBytes > warnBytes:
sev = alert.Warning
default:
continue
}
*findings = append(*findings, alert.Finding{
Severity: sev,
Check: "perf_wp_transients",
Message: fmt.Sprintf("Bloated transient %s in %s", optionName, info.dbName),
Details: fmt.Sprintf("Size: %s", humanBytes(sizeBytes)),
Timestamp: time.Now(),
})
}
}
}
// CheckWPTransientBloat scans configured web roots (default /home/*/public_html
// on cPanel) for WordPress installs and queries each database for oversized
// transients. DB credentials are read from wp-config.php; the password is
// passed via MYSQL_PWD environment variable (never on the command line).
// The runner enforces a 60-minute throttle via checkThrottleMin.
func CheckWPTransientBloat(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
warnBytes := int64(cfg.Performance.WPTransientWarnMB) * 1024 * 1024
critBytes := int64(cfg.Performance.WPTransientCriticalMB) * 1024 * 1024
homeDirs := ResolveWebRoots(cfg)
var findings []alert.Finding
for _, dir := range homeDirs {
findWPTransients(dir, warnBytes, critBytes, 2, &findings)
}
return findings
}
// ---------------------------------------------------------------------------
// CheckWPCron
// ---------------------------------------------------------------------------
const (
wpCronFindingLimit = 30
wpCronCursorKey = "_wpcron_scan_cursor"
)
var (
wpCronTablePrefixAssignRe = regexp.MustCompile(`(?im)^[\t ]*\$table_prefix[\t ]*=`)
wpCronRequireRe = regexp.MustCompile(`(?im)^[\t ]*require(?:_once)?\b`)
)
type wpCronCandidate struct {
path string
account string
}
type wpCronScanCursor struct {
Root string `json:"root"`
Path string `json:"path,omitempty"`
}
// scanWPCronCandidates recursively finds real WordPress installs. When a
// nested vhost is also a scan root, the broader root leaves that subtree to
// the more-specific root. This preserves the full depth allowance for both
// roots without reading or reporting the same install twice.
func scanWPCronCandidates(
ctx context.Context,
scanRoot, dir, account string,
depth int,
allRoots map[string]struct{},
candidates map[string]wpCronCandidate,
) bool {
if depth < 0 {
return true
}
if ctx.Err() != nil {
return false
}
cleanDir := filepath.Clean(dir)
if cleanDir != filepath.Clean(scanRoot) {
if _, nestedRoot := allRoots[cleanDir]; nestedRoot {
return true
}
}
entries, err := osFS.ReadDir(cleanDir)
if err != nil {
return errors.Is(err, fs.ErrNotExist)
}
complete := true
for _, entry := range entries {
if ctx.Err() != nil {
return false
}
name := entry.Name()
fullPath := filepath.Join(cleanDir, name)
if entry.IsDir() {
if skipDirs[name] {
continue
}
if !scanWPCronCandidates(ctx, scanRoot, fullPath, account, depth-1, allRoots, candidates) {
complete = false
}
continue
}
if name != "wp-config.php" || entry.Type()&os.ModeSymlink != 0 {
continue
}
info, infoErr := entry.Info()
if infoErr != nil {
complete = false
continue
}
if !info.Mode().IsRegular() {
continue
}
data, readErr := osFS.ReadFile(fullPath)
if readErr != nil {
if !errors.Is(readErr, fs.ErrNotExist) {
complete = false
}
continue
}
if wpCronHasActiveDisableDefine(data) {
continue
}
isWordPress, validateErr := wpCronInstallIsValid(fullPath, data)
if validateErr != nil {
complete = false
continue
}
if !isWordPress {
continue
}
candidates[fullPath] = wpCronCandidate{path: fullPath, account: account}
}
return complete
}
// wpCronInstallIsValid requires both the WordPress bootstrap shape and the
// core files the remediation will invoke. A stray or backup wp-config.php is
// not enough evidence to edit customer data or install a crontab entry.
func wpCronInstallIsValid(configPath string, data []byte) (bool, error) {
code := stripPHPCommentsFromCode(phpCodeOnly(string(data)))
codeWithoutStrings := stripPHPStringsFromCode(code)
if !wpCronTablePrefixAssignRe.MatchString(codeWithoutStrings) ||
!wpCronHasSettingsRequire(code, codeWithoutStrings) {
return false, nil
}
docroot := filepath.Dir(configPath)
for _, name := range []string{"wp-settings.php", "wp-cron.php", "wp-load.php"} {
info, err := osFS.Lstat(filepath.Join(docroot, name))
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return false, nil
}
return false, err
}
if !info.Mode().IsRegular() {
return false, nil
}
}
for _, name := range []string{"wp-admin", "wp-includes"} {
info, err := osFS.Lstat(filepath.Join(docroot, name))
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return false, nil
}
return false, err
}
if !info.IsDir() {
return false, nil
}
}
return true, nil
}
// wpCronHasSettingsRequire ties the wp-settings.php literal to an active
// require statement. Looking for both tokens independently lets quoted sample
// text masquerade as a WordPress bootstrap and can trigger destructive fixes.
func wpCronHasSettingsRequire(code, codeWithoutStrings string) bool {
for _, loc := range wpCronRequireRe.FindAllStringIndex(codeWithoutStrings, -1) {
statementEnd := strings.IndexByte(codeWithoutStrings[loc[1]:], ';')
if statementEnd < 0 {
continue
}
statementEnd += loc[1]
if strings.Contains(strings.ToLower(code[loc[0]:statementEnd]), "wp-settings.php") {
return true
}
}
return false
}
func sortedWPCronCandidates(candidates map[string]wpCronCandidate) []wpCronCandidate {
out := make([]wpCronCandidate, 0, len(candidates))
for _, candidate := range candidates {
out = append(out, candidate)
}
sort.Slice(out, func(i, j int) bool { return out[i].path < out[j].path })
return out
}
func newWPCronFinding(candidate wpCronCandidate) alert.Finding {
return alert.Finding{
Severity: alert.Warning,
Check: "perf_wp_cron",
Message: fmt.Sprintf("WP-Cron not disabled for %s", candidate.account),
Details: fmt.Sprintf(
"File: %s - add define('DISABLE_WP_CRON', true); and use a real cron job instead",
candidate.path,
),
Timestamp: time.Now(),
}
}
// CheckWPCron scans configured web roots plus validated cPanel document roots
// for WordPress installs that have not disabled the built-in WP-Cron
// mechanism. Running WP-Cron via HTTP is a common cause of high load on busy
// sites.
// The runner enforces a 60-minute throttle via checkThrottleMin.
func CheckWPCron(ctx context.Context, cfg *config.Config, scanState *state.Store) []alert.Finding {
if !perfEnabled(cfg) {
return nil
}
roots, rootsComplete := validatedDocrootSet(cfg)
if !rootsComplete {
markCheckIncomplete(ctx, "perf_wp_cron")
}
if len(roots) == 0 {
clearWPCronCursor(scanState)
return nil
}
allRoots := make(map[string]struct{}, len(roots))
for _, root := range roots {
allRoots[root.path] = struct{}{}
}
cursor := loadWPCronCursor(scanState)
start, resumeCurrent := wpCronStartRoot(roots, cursor)
var findings []alert.Finding
var nextCursor wpCronScanCursor
capped := false
for visited := 0; visited < len(roots); visited++ {
if ctx.Err() != nil {
return findings
}
root := roots[(start+visited)%len(roots)]
candidates := make(map[string]wpCronCandidate)
if !scanWPCronCandidates(ctx, root.path, root.path, root.account, 2, allRoots, candidates) {
markCheckIncomplete(ctx, "perf_wp_cron")
}
sorted := sortedWPCronCandidates(candidates)
lastPath := ""
if visited == 0 && resumeCurrent && root.path == cursor.Root {
lastPath = cursor.Path
}
first := sort.Search(len(sorted), func(i int) bool { return sorted[i].path > lastPath })
eligible := sorted[first:]
remaining := wpCronFindingLimit - len(findings)
selected := len(eligible)
if selected > remaining {
selected = remaining
}
for _, candidate := range eligible[:selected] {
findings = append(findings, newWPCronFinding(candidate))
}
if len(findings) == wpCronFindingLimit {
nextCursor.Root = root.path
if selected < len(eligible) {
nextCursor.Path = eligible[selected-1].path
}
capped = true
break
}
}
if ctx.Err() != nil {
return findings
}
if capped {
storeWPCronCursor(scanState, nextCursor)
} else {
clearWPCronCursor(scanState)
}
sort.Slice(findings, func(i, j int) bool { return findings[i].Details < findings[j].Details })
return findings
}
func loadWPCronCursor(scanState *state.Store) wpCronScanCursor {
if scanState == nil {
return wpCronScanCursor{}
}
raw, ok := scanState.GetRaw(wpCronCursorKey)
if !ok {
return wpCronScanCursor{}
}
var cursor wpCronScanCursor
if json.Unmarshal([]byte(raw), &cursor) != nil {
return wpCronScanCursor{}
}
return cursor
}
func wpCronStartRoot(roots []docrootScanRoot, cursor wpCronScanCursor) (start int, resume bool) {
if cursor.Root == "" {
return 0, false
}
if cursor.Path != "" {
i := sort.Search(len(roots), func(i int) bool { return roots[i].path >= cursor.Root })
if i < len(roots) && roots[i].path == cursor.Root {
return i, true
}
}
i := sort.Search(len(roots), func(i int) bool { return roots[i].path > cursor.Root })
if i == len(roots) {
i = 0
}
return i, false
}
func storeWPCronCursor(scanState *state.Store, cursor wpCronScanCursor) {
if scanState == nil {
return
}
raw, err := json.Marshal(cursor)
if err != nil {
return
}
if err := scanState.SetRawAndSave(wpCronCursorKey, string(raw)); err != nil {
fmt.Fprintf(os.Stderr, "wpcron: cursor write: %v\n", err)
}
}
func clearWPCronCursor(scanState *state.Store) {
if scanState == nil {
return
}
if err := scanState.DeleteRawAndSave(wpCronCursorKey); err != nil {
fmt.Fprintf(os.Stderr, "wpcron: cursor clear: %v\n", err)
}
}
package checks
import (
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"slices"
"strings"
"sync"
"unicode"
"unicode/utf16"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/workerproc"
"golang.org/x/net/html"
)
// phishingReadSize is how much of an accepted HTML page is analysed. It
// matches the 100 KB acceptance ceiling: reading only the first 16 KB let a
// kit that opens with a large inline stylesheet keep its form past the read
// window and pass as clean.
const phishingReadSize = 100000
// phishingReadBuffers recycles the read windows. Most pages the walk reads
// are a few KB, and allocating and zeroing a fresh window for each one was a
// measurable share of the check's CPU. Only buf[:n] of a reused window holds
// the current file.
var phishingReadBuffers = sync.Pool{New: func() any {
buf := make([]byte, phishingReadSize)
return &buf
}}
// phishingScanMaxDepth bounds how deep CheckPhishing recurses below each doc
// root. Real kits land in date-nested WordPress upload folders
// (wp-content/uploads/YYYY/MM/<kit>/), six directory levels below the root, so
// the budget must clear that. No directory is pruned by name: anything the web
// server serves can hold a kit.
const phishingScanMaxDepth = 8
// ---------------------------------------------------------------------------
// Brand impersonation patterns
// ---------------------------------------------------------------------------
// phishingGenericBrandScoreFloor is the score a page matching only the
// "Generic Login" pseudo-brand must reach before it is flagged. The generic
// title patterns ("Sign In", "Secure Access") are ubiquitous on legitimate
// login pages and, unlike a real brand, are not themselves evidence of
// impersonation. A single equally-ubiquitous JS token (window.location,
// fetch) would otherwise clear the normal brand floor and mislabel a
// customer's own login page. Requiring this higher floor forces multiple
// independent signals (external exfil, trust badge, urgency, server-side
// credential capture) that a plain login page does not carry.
const phishingGenericBrandScoreFloor = 7
var phishingBrands = []struct {
name string
titlePatterns []string
bodyPatterns []string
generic bool
}{
{
name: "Microsoft/SharePoint",
titlePatterns: []string{"sharepoint", "onedrive", "microsoft 365", "outlook web", "office 365", "ms online"},
bodyPatterns: []string{"sharepoint", "onedrive", "secured by microsoft", "microsoft corporation"},
},
{
name: "Google",
titlePatterns: []string{"google drive", "google docs", "google sign", "gmail", "google workspace"},
bodyPatterns: []string{"google drive", "google docs", "accounts.google", "secured by google"},
},
{
name: "Dropbox",
titlePatterns: []string{"dropbox", "shared file", "shared folder"},
bodyPatterns: []string{"dropbox", "dropbox.com", "secured by dropbox"},
},
{
name: "DocuSign",
titlePatterns: []string{"docusign", "document signing", "e-signature"},
bodyPatterns: []string{"docusign", "please review and sign", "e-signature"},
},
{
name: "Adobe",
titlePatterns: []string{"adobe sign", "adobe document", "adobe acrobat"},
bodyPatterns: []string{"adobe sign", "adobe document cloud", "secured by adobe"},
},
{
name: "WeTransfer",
titlePatterns: []string{"wetransfer", "file transfer"},
bodyPatterns: []string{"wetransfer", "download your files"},
},
{
name: "Apple/iCloud",
titlePatterns: []string{"icloud", "apple id", "find my"},
bodyPatterns: []string{"icloud.com", "apple id", "secured by apple"},
},
{
name: "PayPal",
titlePatterns: []string{"paypal", "pay pal"},
bodyPatterns: []string{"paypal.com", "secured by paypal"},
},
{
name: "Webmail/Roundcube",
titlePatterns: []string{"roundcube", "horde", "webmail login", "webmail ::", "squirrelmail", "zimbra"},
bodyPatterns: []string{"roundcube webmail", "horde login", "zimbra web client", "webmail login"},
// Note: bare "squirrelmail"/"roundcube" removed from body - sites legitimately
// link to their server's webmail (e.g. href="squirrelmail/index.php").
},
{
name: "cPanel/WHM",
titlePatterns: []string{"cpanel", "whm login", "webhost manager"},
bodyPatterns: []string{"cpanel login", "whm login", "webhost manager"},
},
{
name: "Banking/Financial",
titlePatterns: []string{"online banking", "bank login", "secure banking", "account login"},
bodyPatterns: []string{"online banking", "bank account", "transaction verification"},
},
{
name: "Generic Login",
titlePatterns: []string{"secure access", "verify your", "confirm your identity", "account verification", "email verification", "sign in"},
bodyPatterns: []string{"verify your identity", "confirm your account", "unusual activity"},
generic: true,
},
}
// ---------------------------------------------------------------------------
// Content-based indicators
// ---------------------------------------------------------------------------
// Credential harvesting patterns in page body.
var harvestIndicators = []string{
"window.location.href",
"window.location.replace",
"window.location =",
"document.location.href",
"form.submit()",
".workers.dev",
"confirm access",
"verify your email",
"confirm your email",
"verify identity",
"continue to document",
"access confirmed, redirecting",
"secured by microsoft",
"secured by google",
"secured by apple",
"256-bit encrypted",
"256‑bit encrypted",
// fetch/XHR exfiltration - silent credential POST without redirect
"fetch(",
"xmlhttprequest",
"$.ajax(",
"$.post(",
"navigator.sendbeacon(",
}
// Redirect/exfiltration URL patterns.
var exfilPatterns = []string{
".workers.dev",
"//t.co/",
"/redir?",
"/redirect?",
"effi.redir",
"link?url=",
"goto_url=",
"//bit.ly/",
"//tinyurl.com/",
"//rb.gy/",
"//is.gd/",
"/servlet/effi.redir",
}
// Fake trust badge patterns - security claims in pages not on the brand's domain.
var trustBadgePatterns = []string{
"secured by microsoft",
"secured by google",
"secured by apple",
"secured by dropbox",
"secured by adobe",
"verified by microsoft",
"protected by microsoft",
"256-bit encrypted",
"256‑bit encrypted",
"ssl secured",
"bank-level encryption",
"enterprise security",
}
// Urgency language used to pressure victims.
var urgencyPatterns = []string{
"expires in",
"temporary hold",
"limited time",
"unusual activity detected",
"suspicious activity",
"your account will be",
"verify within",
"action required",
"immediate action",
"account suspended",
"access will be revoked",
}
// Embedded asset indicators - phishing kits embed logos to avoid external loading.
var embeddedAssetPatterns = []string{
"data:image/png;base64,",
"data:image/svg+xml;base64,",
"data:image/jpeg;base64,",
}
// ---------------------------------------------------------------------------
// Main check entry point
// ---------------------------------------------------------------------------
// CheckPhishing scans HTML files in user document roots for phishing pages.
// Uses three detection layers:
// 1. Content analysis - brand impersonation + credential harvesting patterns
// 2. Structural analysis - self-contained HTML with embedded assets
// 3. Directory anomaly - lone HTML files in otherwise empty directories
func CheckPhishing(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
homeDirs := scanHomeDirsWithCoverage(ctx, "phishing")
for _, homeEntry := range homeDirs {
if ctx.Err() != nil {
return findings
}
if !homeEntry.IsDir() {
continue
}
user := homeEntry.Name()
if strings.HasPrefix(user, ".") || user == "virtfs" {
continue
}
homeDir := scanHomeDirPath(homeEntry)
docRoots := []string{filepath.Join(homeDir, "public_html")}
subDirs, err := osFS.ReadDir(homeDir)
markScanReadError(ctx, "phishing", err)
for _, sd := range subDirs {
if sd.IsDir() && sd.Name() != "public_html" && sd.Name() != "mail" &&
!strings.HasPrefix(sd.Name(), ".") && sd.Name() != "etc" &&
sd.Name() != "logs" && sd.Name() != "ssl" && sd.Name() != "tmp" {
docRoots = append(docRoots, filepath.Join(homeDir, sd.Name()))
}
}
for _, docRoot := range docRoots {
scanForPhishing(ctx, docRoot, phishingScanMaxDepth, user, cfg, &findings)
if ctx.Err() != nil {
return findings
}
}
}
return findings
}
// ---------------------------------------------------------------------------
// Directory scanner
// ---------------------------------------------------------------------------
// phishingCandidateName reports whether any file check in scanForPhishing can
// judge a file of this name. The walk covers millions of files, so the stat
// is paid only for these names.
func phishingCandidateName(nameLower string) bool {
switch {
case strings.HasSuffix(nameLower, ".html"), strings.HasSuffix(nameLower, ".htm"):
return true
case isExecutablePHPName(nameLower):
return !isKnownCMSFile(nameLower)
case strings.HasSuffix(nameLower, ".zip"):
return isPhishingKitZipName(nameLower)
}
return isCredentialLogName(nameLower)
}
func scanForPhishing(ctx context.Context, dir string, maxDepth int, user string, cfg *config.Config, findings *[]alert.Finding) {
if ctx.Err() != nil {
return
}
if maxDepth <= 0 {
return
}
entries, err := osFS.ReadDir(dir)
if err != nil {
markScanReadError(ctx, "phishing", err)
return
}
for _, entry := range entries {
if ctx.Err() != nil {
return
}
name := entry.Name()
fullPath := filepath.Join(dir, name)
if scanRespectsIgnores(ctx, cfg) && PathMatchesIgnore(fullPath, cfg.Suppressions.IgnorePaths) {
continue
}
if entry.IsDir() {
// --- Directory anomaly detection ---
dirResult := analyzeDirectoryStructure(ctx, fullPath, user, cfg)
if dirResult != nil {
*findings = append(*findings, *dirResult)
}
scanForPhishing(ctx, fullPath, maxDepth-1, user, cfg, findings)
continue
}
nameLower := strings.ToLower(name)
if !phishingCandidateName(nameLower) {
continue
}
info, err := entry.Info()
if err != nil {
markScanReadError(ctx, "phishing", err)
continue
}
if !info.Mode().IsRegular() {
continue
}
size := info.Size()
// --- HTML/HTM phishing pages ---
if strings.HasSuffix(nameLower, ".html") || strings.HasSuffix(nameLower, ".htm") {
// Standard phishing page check (3KB-100KB)
if size >= 3000 && size <= 100000 {
result := analyzeHTMLForPhishing(ctx, fullPath)
if result != nil {
*findings = append(*findings, alert.Finding{
Severity: alert.Critical,
Check: "phishing_page",
Message: fmt.Sprintf("Phishing page detected (%s impersonation): %s", result.brand, fullPath),
Details: fmt.Sprintf("Account: %s\nBrand: %s\nScore: %d/10\nIndicators:\n- %s\nSize: %d bytes",
user, result.brand, result.score, strings.Join(result.indicators, "\n- "), size),
FilePath: fullPath,
})
}
}
// --- iframe phishing (tiny HTML files that embed external phishing) ---
if size > 0 && size < 3000 {
if result := checkIframePhishing(ctx, fullPath); result != "" {
*findings = append(*findings, alert.Finding{
Severity: alert.Critical,
Check: "phishing_iframe",
Message: fmt.Sprintf("Iframe phishing page detected: %s", fullPath),
Details: fmt.Sprintf("Account: %s\n%s", user, result),
FilePath: fullPath,
})
}
}
continue
}
// --- PHP phishing pages and open redirectors ---
if isExecutablePHPName(nameLower) {
// PHP phishing (3KB-100KB) - same brand/content analysis as HTML
if size >= 3000 && size <= 100000 {
result := analyzePHPForPhishing(ctx, fullPath)
if result != nil {
*findings = append(*findings, alert.Finding{
Severity: alert.Critical,
Check: "phishing_php",
Message: fmt.Sprintf("PHP phishing page detected (%s): %s", result.brand, fullPath),
Details: fmt.Sprintf("Account: %s\nBrand: %s\nScore: %d/10\nIndicators:\n- %s\nSize: %d bytes",
user, result.brand, result.score, strings.Join(result.indicators, "\n- "), size),
FilePath: fullPath,
})
}
}
// PHP open redirector (tiny PHP files under 1KB)
if size > 0 && size < 1024 {
if result := checkPHPRedirector(ctx, fullPath); result != "" {
*findings = append(*findings, alert.Finding{
Severity: alert.High,
Check: "phishing_redirector",
Message: fmt.Sprintf("PHP open redirector detected: %s", fullPath),
Details: fmt.Sprintf("Account: %s\n%s", user, result),
FilePath: fullPath,
})
}
}
continue
}
// --- Credential log files ---
if !strings.HasSuffix(nameLower, ".zip") && isCredentialLogName(nameLower) &&
size > 0 && size < 10*1024*1024 {
if result := checkCredentialLog(ctx, fullPath); result != "" {
*findings = append(*findings, alert.Finding{
Severity: alert.Critical,
Check: "phishing_credential_log",
Message: fmt.Sprintf("Harvested credential log file detected: %s", fullPath),
Details: fmt.Sprintf("Account: %s\n%s", user, result),
FilePath: fullPath,
})
}
continue
}
// --- Phishing kit ZIP archives ---
if strings.HasSuffix(nameLower, ".zip") && size > 1000 && size < 50*1024*1024 && isPhishingKitZipName(nameLower) {
switch verdict, sum := classifyKitZip(ctx, fullPath); verdict {
case zipKit:
*findings = append(*findings, alert.Finding{
Severity: alert.High,
Check: "phishing_kit_archive",
Message: fmt.Sprintf("Suspected phishing kit archive: %s", fullPath),
Details: fmt.Sprintf("Account: %s\nFilename: %s\nSize: %d bytes",
user, name, size),
FilePath: fullPath,
})
case zipSkipped:
*findings = append(*findings, alert.Finding{
Severity: alert.Warning,
Check: "phishing_kit_archive_unscannable",
Message: "Phishing kit check skipped an archive that made the parse worker fail twice",
Details: fmt.Sprintf("Account: %s\nPath: %s\nSize: %d bytes\nSHA-256: %s",
user, fullPath, size, sum),
FilePath: fullPath,
})
}
}
}
}
// ---------------------------------------------------------------------------
// Layer 1: Content analysis
// ---------------------------------------------------------------------------
type phishingResult struct {
brand string
score int
indicators []string
}
func analyzeHTMLForPhishing(ctx context.Context, path string) *phishingResult {
f, err := openTenantRegularFile(path)
if err != nil {
markScanReadError(ctx, "phishing", err)
return nil
}
defer func() { _ = f.Close() }()
bufp := phishingReadBuffers.Get().(*[]byte)
defer phishingReadBuffers.Put(bufp)
buf := *bufp
n, err := io.ReadFull(f, buf)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
markCheckIncomplete(ctx, "phishing")
}
if n == 0 {
return nil
}
content := string(buf[:n])
contentLower := strings.ToLower(content)
// Must contain a form or input
hasForm := strings.Contains(contentLower, "<form") ||
strings.Contains(contentLower, "<input")
if !hasForm {
return nil
}
// Must contain email/password input
hasCredentialInput := hasHTMLCredentialInput(contentLower)
if !hasCredentialInput {
return nil
}
var indicators []string
score := 0
// --- Brand impersonation ---
brandMatch := ""
matchedGeneric := false
titleContent := extractTitle(contentLower)
passwordInput := hasHTMLPasswordInput(contentLower)
for _, brand := range phishingBrands {
titleHit := false
bodyHit := false
for _, tp := range brand.titlePatterns {
if titleNamesBrand(titleContent, tp, passwordInput) {
titleHit = true
indicators = append(indicators, fmt.Sprintf("title impersonates '%s'", tp))
score += 3
break
}
}
for _, bp := range brand.bodyPatterns {
if strings.Contains(contentLower, bp) {
bodyHit = true
if !titleHit {
indicators = append(indicators, fmt.Sprintf("body impersonates '%s'", bp))
score += 2
}
break
}
}
if titleHit || bodyHit {
brandMatch = brand.name
matchedGeneric = brand.generic
break
}
}
// --- Credential harvesting / redirect indicators ---
for _, pattern := range harvestIndicators {
if strings.Contains(contentLower, pattern) {
score++
indicators = append(indicators, fmt.Sprintf("harvest: '%s'", pattern))
}
}
// --- Exfiltration URL patterns ---
for _, pattern := range exfilPatterns {
if strings.Contains(contentLower, pattern) {
score += 2
indicators = append(indicators, fmt.Sprintf("exfiltration: '%s'", pattern))
}
}
// --- Fake trust badges ---
for _, pattern := range trustBadgePatterns {
if strings.Contains(contentLower, pattern) {
score++
indicators = append(indicators, fmt.Sprintf("fake trust badge: '%s'", pattern))
}
}
// --- Urgency language ---
urgencyCount := 0
for _, pattern := range urgencyPatterns {
if strings.Contains(contentLower, pattern) {
urgencyCount++
}
}
if urgencyCount > 0 {
score++
indicators = append(indicators, fmt.Sprintf("urgency language (%d patterns)", urgencyCount))
}
// --- Embedded Base64 assets (logos embedded to avoid external loading) ---
embeddedCount := 0
for _, pattern := range embeddedAssetPatterns {
embeddedCount += countOccurrences(contentLower, pattern)
}
if embeddedCount > 0 {
score++
indicators = append(indicators, fmt.Sprintf("embedded base64 assets (%d)", embeddedCount))
}
// --- Form action pointing to external domain ---
if hasExternalFormAction(content) {
score += 2
indicators = append(indicators, "form action points to external domain")
}
// --- Self-contained page (all CSS inline, no external stylesheets except CDN) ---
if isSelfContainedHTML(contentLower) {
score++
indicators = append(indicators, "self-contained HTML (all styles inline)")
}
// --- Person name as filename ---
baseName := strings.TrimSuffix(strings.TrimSuffix(filepath.Base(path), ".html"), ".htm")
if looksLikePersonName(baseName) {
score += 2
indicators = append(indicators, fmt.Sprintf("filename looks like person name: '%s'", baseName))
}
// --- Decision ---
// Real brand match: need score >= 4 (brand gives 2-3 + at least 1 other
// signal). Generic pseudo-brand: need a higher floor since its title
// patterns are ubiquitous on legitimate login pages. No brand: need
// score >= 6 (multiple strong signals).
if brandMatch != "" {
floor := 4
if matchedGeneric {
floor = phishingGenericBrandScoreFloor
}
if score >= floor {
return &phishingResult{brand: brandMatch, score: score, indicators: indicators}
}
}
if brandMatch == "" && score >= 6 {
return &phishingResult{brand: "Unknown", score: score, indicators: indicators}
}
return nil
}
// ---------------------------------------------------------------------------
// Layer 2: Structural analysis helpers
// ---------------------------------------------------------------------------
// Title evidence needs a stricter address match than harvested credential
// logs: empty dot segments and invalid DNS labels can be title separators,
// and must not hide a separate brand name.
var titleEmailPattern = regexp.MustCompile(`[A-Za-z0-9_%+\-]+(?:\.[A-Za-z0-9_%+\-]+)*@(?:[A-Za-z0-9](?:[A-Za-z0-9\-]*[A-Za-z0-9])?\.)+[A-Za-z]{2,}`)
// titleNamesBrand reports whether a title names a brand. A brand inside an
// email address is not impersonation, even when the page has a login widget.
// Password capture or a login title supplies the context for concatenated
// brand names; digits and a standalone @ are decoration, not part of another
// word.
func titleNamesBrand(title, pattern string, passwordInput bool) bool {
loginTitle := strings.Contains(title, "login") || strings.Contains(title, "log in") ||
strings.Contains(title, "log-in") || strings.Contains(title, "signin") ||
strings.Contains(title, "sign in") || strings.Contains(title, "sign-in")
isLetter := func(b byte) bool { return b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' }
addresses := titleEmailPattern.FindAllStringIndex(title, -1)
address := 0
for offset := 0; offset < len(title); {
idx := strings.Index(title[offset:], pattern)
if idx < 0 {
return false
}
start := offset + idx
end := start + len(pattern)
for address < len(addresses) && addresses[address][1] <= start {
address++
}
mailbox := address < len(addresses) && addresses[address][0] <= start && end <= addresses[address][1]
standsAlone := (start == 0 || !isLetter(title[start-1])) &&
(end == len(title) || !isLetter(title[end]))
if !mailbox && (standsAlone || loginTitle || passwordInput) {
return true
}
offset = start + 1
}
return false
}
// extractTitle pulls the <title> content from HTML.
func extractTitle(contentLower string) string {
for offset := 0; offset < len(contentLower); {
idx := strings.Index(contentLower[offset:], "<title")
if idx < 0 {
return ""
}
start := offset + idx
afterName := start + len("<title")
if afterName < len(contentLower) && !isTagBoundary(contentLower[afterName]) {
offset = afterName
continue
}
openEnd := findTagEnd(contentLower, afterName)
if openEnd < 0 {
return ""
}
closeOffset := strings.Index(contentLower[openEnd+1:], "</title>")
if closeOffset < 0 {
return ""
}
return strings.TrimSpace(contentLower[openEnd+1 : openEnd+1+closeOffset])
}
return ""
}
// hasExternalFormAction checks if a <form> action points to a different domain.
func hasExternalFormAction(content string) bool {
for _, url := range htmlAttrValues(strings.ToLower(content), "form", "action", false) {
// External if it starts with http:// or https:// (not relative)
if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
return true
}
}
return false
}
func hasHTMLCredentialInput(contentLower string) bool {
for _, value := range htmlAttrValues(contentLower, "input", "type", true) {
switch strings.TrimSpace(value) {
case "email", "password":
return true
}
}
for _, value := range htmlAttrValues(contentLower, "input", "name", true) {
switch strings.TrimSpace(value) {
case "email", "pass", "password", "login":
return true
}
}
for _, value := range htmlAttrValues(contentLower, "input", "placeholder", true) {
value = strings.TrimSpace(value)
if strings.HasPrefix(value, "email") ||
strings.HasPrefix(value, "you@") ||
strings.HasPrefix(value, "your email") {
return true
}
}
return strings.Contains(contentLower, "work or school email") ||
strings.Contains(contentLower, "corporate email")
}
func hasHTMLPasswordInput(contentLower string) bool {
for _, value := range htmlAttrValues(contentLower, "input", "type", true) {
if strings.TrimSpace(value) == "password" {
return true
}
}
for _, value := range htmlAttrValues(contentLower, "input", "name", true) {
switch strings.TrimSpace(value) {
case "pass", "password", "passwd", "pwd", "passcode":
return true
}
}
return false
}
func htmlAttrValues(contentLower, tagName, attrName string, allowUnquoted bool) []string {
var values []string
needle := "<" + tagName
for offset := 0; offset < len(contentLower); {
idx := strings.Index(contentLower[offset:], needle)
if idx < 0 {
break
}
start := offset + idx
afterName := start + len(needle)
if afterName < len(contentLower) && !isTagBoundary(contentLower[afterName]) {
offset = afterName
continue
}
end := findTagEnd(contentLower, afterName)
if end < 0 {
break
}
if value, ok := tagAttrValue(contentLower[afterName:end], attrName, allowUnquoted); ok {
values = append(values, value)
}
offset = end + 1
}
return values
}
func isTagBoundary(c byte) bool {
return c == '>' || c == '/' || unicode.IsSpace(rune(c))
}
func findTagEnd(content string, start int) int {
var quote byte
for i := start; i < len(content); i++ {
c := content[i]
if quote != 0 {
if c == quote {
quote = 0
}
continue
}
if c == '"' || c == '\'' {
quote = c
continue
}
if c == '>' {
return i
}
}
return -1
}
func tagAttrValue(attrs, attrName string, allowUnquoted bool) (string, bool) {
for i := 0; i < len(attrs); {
for i < len(attrs) && (unicode.IsSpace(rune(attrs[i])) || attrs[i] == '/') {
i++
}
nameStart := i
for i < len(attrs) && attrs[i] != '=' && attrs[i] != '>' &&
attrs[i] != '/' && !unicode.IsSpace(rune(attrs[i])) {
i++
}
if nameStart == i {
i++
continue
}
name := attrs[nameStart:i]
for i < len(attrs) && unicode.IsSpace(rune(attrs[i])) {
i++
}
if i >= len(attrs) || attrs[i] != '=' {
continue
}
i++
for i < len(attrs) && unicode.IsSpace(rune(attrs[i])) {
i++
}
if i >= len(attrs) {
return "", false
}
value := ""
if attrs[i] == '"' || attrs[i] == '\'' {
quote := attrs[i]
i++
valueStart := i
for i < len(attrs) && attrs[i] != quote {
i++
}
value = attrs[valueStart:i]
if i < len(attrs) {
i++
}
} else {
valueStart := i
for i < len(attrs) && attrs[i] != '>' && !unicode.IsSpace(rune(attrs[i])) {
i++
}
if !allowUnquoted {
continue
}
value = attrs[valueStart:i]
}
if name == attrName {
return strings.TrimSpace(value), true
}
}
return "", false
}
// isSelfContainedHTML checks if a page has all its CSS inline (embedded <style> tags)
// with no or minimal external stylesheet references - typical of phishing kits.
func isSelfContainedHTML(contentLower string) bool {
hasInlineStyle := strings.Contains(contentLower, "<style")
externalCSS := countOccurrences(contentLower, "rel=\"stylesheet\"") +
countOccurrences(contentLower, "rel='stylesheet'")
// Allow 1 external CSS (e.g., Font Awesome CDN) - phishing kits often use one
return hasInlineStyle && externalCSS <= 1
}
// looksLikePersonName checks if a filename looks like a person name (CamelCase
// with 2+ capitalized words, e.g., "PalmerHamilton", "MarilynEsguerra").
func looksLikePersonName(name string) bool {
if len(name) < 6 {
return false
}
// Count uppercase transitions (start of words in CamelCase)
upperCount := 0
for i, c := range name {
if unicode.IsUpper(c) {
if i == 0 || unicode.IsLower(rune(name[i-1])) {
upperCount++
}
}
}
// 2+ capitalized words, all letters, no common web words
if upperCount < 2 {
return false
}
allLetters := true
for _, c := range name {
if !unicode.IsLetter(c) {
allLetters = false
break
}
}
if !allLetters {
return false
}
// Exclude common web filenames
nameLower := strings.ToLower(name)
webNames := []string{"index", "default", "portal", "login", "home", "main",
"readme", "changelog", "license", "manifest", "service"}
for _, w := range webNames {
if nameLower == w {
return false
}
}
return true
}
// ---------------------------------------------------------------------------
// Layer 3: Directory anomaly detection
// ---------------------------------------------------------------------------
// analyzeDirectoryStructure checks if a directory looks like a phishing drop:
// - Contains only 1-3 HTML files and nothing else significant
// - Directory name looks like a business/organization name
// - No CMS markers (wp-config, index.php, etc.)
func analyzeDirectoryStructure(ctx context.Context, dir string, user string, cfg *config.Config) *alert.Finding {
entries, err := osFS.ReadDir(dir)
if err != nil {
markScanReadError(ctx, "phishing", err)
return nil
}
var htmlFiles []string
otherFiles := 0
totalFiles := 0
for _, entry := range entries {
if entry.IsDir() {
return nil // Has subdirectories - likely not a simple phishing drop
}
name := entry.Name()
if strings.HasPrefix(name, ".") {
continue // Skip dotfiles (.htaccess etc.)
}
totalFiles++
nameLower := strings.ToLower(name)
if entry.Type().IsRegular() && (strings.HasSuffix(nameLower, ".html") || strings.HasSuffix(nameLower, ".htm")) {
htmlFiles = append(htmlFiles, name)
} else {
otherFiles++
}
}
// Must have exactly 1-3 HTML files and at most 1 other file
if len(htmlFiles) == 0 || len(htmlFiles) > 3 || otherFiles > 1 {
return nil
}
// Directory name should look like a business/organization (CamelCase or multi-word)
dirName := filepath.Base(dir)
if !looksLikeBusinessName(dirName) {
return nil
}
// Verify at least one HTML file has credential inputs (quick check)
hasPhishingContent := false
for _, htmlFile := range htmlFiles {
fullPath := filepath.Join(dir, htmlFile)
if scanRespectsIgnores(ctx, cfg) && PathMatchesIgnore(fullPath, cfg.Suppressions.IgnorePaths) {
continue
}
if quickPhishingCheck(ctx, fullPath) {
hasPhishingContent = true
break
}
}
if !hasPhishingContent {
return nil
}
// Build indicators
indicators := []string{
fmt.Sprintf("directory '%s' contains only %d HTML file(s)", dirName, len(htmlFiles)),
fmt.Sprintf("directory name resembles business/organization: '%s'", dirName),
}
for _, h := range htmlFiles {
baseName := strings.TrimSuffix(strings.TrimSuffix(h, ".html"), ".htm")
if looksLikePersonName(baseName) {
indicators = append(indicators, fmt.Sprintf("HTML filename looks like person name: '%s'", baseName))
}
}
return &alert.Finding{
Severity: alert.High,
Check: "phishing_directory",
Message: fmt.Sprintf("Suspected phishing directory (lone HTML in business-named folder): %s", dir),
Details: fmt.Sprintf("Account: %s\nDirectory: %s\nHTML files: %s\nIndicators:\n- %s",
user, dirName, strings.Join(htmlFiles, ", "), strings.Join(indicators, "\n- ")),
FilePath: dir,
}
}
// looksLikeBusinessName checks if a directory name looks like a business or
// organization name rather than a standard web directory.
func looksLikeBusinessName(name string) bool {
if len(name) < 5 {
return false
}
nameLower := strings.ToLower(name)
// Skip names that start with tech/dev terms - these are tutorial
// or test directories, not business names (e.g. "php-email-form",
// "PHP-Login", "JavaScript Login")
techPrefixes := []string{
"php", "javascript", "js-", "css", "html", "python",
"java", "node", "react", "vue", "angular", "jquery",
"bootstrap", "wordpress", "wp-", "laravel",
}
for _, prefix := range techPrefixes {
if strings.HasPrefix(nameLower, prefix) {
return false
}
}
// Skip standard web directories
standardDirs := []string{
"images", "img", "css", "js", "fonts", "assets", "static",
"media", "uploads", "downloads", "files", "docs", "data",
"api", "admin", "config", "templates", "scripts", "lib",
"src", "dist", "build", "public", "private", "backup",
"old", "new", "test", "dev", "staging", "demo",
}
for _, sd := range standardDirs {
if nameLower == sd {
return false
}
}
// CamelCase detection (e.g., WashingtonGolf, XRFScientificAmericasInc)
upperTransitions := 0
for i, c := range name {
if unicode.IsUpper(c) && i > 0 && unicode.IsLower(rune(name[i-1])) {
upperTransitions++
}
}
if upperTransitions >= 1 && unicode.IsUpper(rune(name[0])) {
return true
}
// Multi-word with separators (e.g., federated-lighting, northwest_crawlspace)
if strings.ContainsAny(name, "-_") {
parts := strings.FieldsFunc(name, func(r rune) bool { return r == '-' || r == '_' })
if len(parts) >= 2 {
return true
}
}
// Long lowercase name that doesn't match standard dirs (e.g., "healthcornerpediattrics")
allLower := true
for _, c := range name {
if !unicode.IsLetter(c) {
allLower = false
break
}
}
if allLower && len(name) >= 12 {
return true
}
return false
}
// quickPhishingCheck does a fast read of an HTML file and confirms phishing
// shape: a credential-collection form AND at least one phishing-kit signal
// in the page body itself. Used by the directory-anomaly heuristic to
// avoid flagging benign HTML that happens to ship an <input> tag.
//
// A bare "<form> + email/password keyword" gate matches developer demo
// pages, JavaScript login tutorials, contact forms, and password-reset
// stubs. Phishing kits add at least one of:
// - a form action that posts to an external host (exfiltration target),
// - a fully self-contained inline-styled HTML body (kits ship one file),
// - real brand impersonation in the title or visible body (Office/PayPal/etc).
//
// Requiring credential intake plus one of those signals keeps real
// phishing kits in scope while letting tutorials and trivial forms drop
// out without consulting any path-name allowlist.
func quickPhishingCheck(ctx context.Context, path string) bool {
f, err := openTenantRegularFile(path)
if err != nil {
markScanReadError(ctx, "phishing", err)
return false
}
defer func() { _ = f.Close() }()
buf := make([]byte, 4096) // first 4KB is enough for the head, form attrs, brand strings
n, err := io.ReadFull(f, buf)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
markCheckIncomplete(ctx, "phishing")
}
if n == 0 {
return false
}
content := string(buf[:n])
contentLower := strings.ToLower(content)
if !hasHTMLCredentialInput(contentLower) {
return false
}
if hasExternalFormAction(content) {
return true
}
if isSelfContainedHTML(contentLower) {
return true
}
titleContent := extractTitle(contentLower)
passwordInput := hasHTMLPasswordInput(contentLower)
bodyContent := visiblePageContent(contentLower, true)
for _, brand := range phishingBrands {
if brand.generic {
continue
}
for _, tp := range brand.titlePatterns {
if titleNamesBrand(titleContent, tp, passwordInput) {
return true
}
}
for _, bp := range brand.bodyPatterns {
if strings.Contains(bodyContent, bp) {
return true
}
}
}
return false
}
// ---------------------------------------------------------------------------
// Layer 4: PHP phishing pages
// ---------------------------------------------------------------------------
// analyzePHPForPhishing reads a PHP file and checks for embedded HTML with
// brand impersonation. PHP phishing kits often have PHP code at the top
// (credential handling, emailing) and HTML output below.
func analyzePHPForPhishing(ctx context.Context, path string) *phishingResult {
f, err := openTenantRegularFile(path)
if err != nil {
markScanReadError(ctx, "phishing", err)
return nil
}
defer func() { _ = f.Close() }()
bufp := phishingReadBuffers.Get().(*[]byte)
defer phishingReadBuffers.Put(bufp)
buf := *bufp
n, err := io.ReadFull(f, buf)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
markCheckIncomplete(ctx, "phishing")
}
if n == 0 || !mayBePHPPhishing(buf[:n]) {
return nil
}
return analyzePHPPhishingContent(string(buf[:n]))
}
// phpPhishingPatterns are PHP credential handling indicators. Only truly
// specific patterns belong here - generic functions like mail() and fwrite()
// are handled separately with context checks.
var phpPhishingPatterns = []string{
"$_post['email']",
"$_post['password']",
"$_post['pass']",
"$_post[\"email\"]",
"$_post[\"password\"]",
"$_post[\"pass\"]",
"$_request['email']",
"$_request['password']",
}
// mayBePHPPhishing reports whether raw can reach a result in
// analyzePHPPhishingContent, which needs a credential pattern (including
// literal superglobal aliases) or "<form" or "<input" in the content.
// Most PHP files the walk reads hold none of them, and this byte scan rules
// them out without lowercasing a copy of the whole file.
func mayBePHPPhishing(raw []byte) bool {
// strings.ToLower maps U+0130 to "i" and U+212A to "k", the only
// non-ASCII runes it turns into ASCII. Content holding either takes the
// full analysis instead of the ASCII scan below.
if bytes.Contains(raw, []byte("\u0130")) || bytes.Contains(raw, []byte("\u212a")) {
return true
}
return anchorFollowedByFold(raw, '$', "_post", "_request") ||
hasRequestAliasBytes(raw) ||
anchorFollowedByFold(raw, '<', "form", "input")
}
// hasRequestAliasBytes reports whether raw may hold ${'_POST'} or
// $GLOBALS['_POST'] (or _REQUEST). Superglobal names are case-sensitive.
func hasRequestAliasBytes(raw []byte) bool {
return (bytes.Contains(raw, []byte("${")) || bytes.Contains(raw, []byte("$GLOBALS"))) &&
(bytes.Contains(raw, []byte("_POST")) || bytes.Contains(raw, []byte("_REQUEST")))
}
// anchorFollowedByFold reports whether some anchor byte in raw is followed by
// one of the lower-case ASCII suffixes, matched under ASCII case folding.
func anchorFollowedByFold(raw []byte, anchor byte, suffixes ...string) bool {
for i := bytes.IndexByte(raw, anchor); i >= 0; {
rest := raw[i+1:]
for _, suffix := range suffixes {
if hasPrefixFoldASCII(rest, suffix) {
return true
}
}
next := bytes.IndexByte(rest, anchor)
if next < 0 {
return false
}
i += 1 + next
}
return false
}
func hasPrefixFoldASCII(b []byte, lower string) bool {
if len(b) < len(lower) {
return false
}
for i := 0; i < len(lower); i++ {
c := b[i]
if 'A' <= c && c <= 'Z' {
c += 'a' - 'A'
}
if c != lower[i] {
return false
}
}
return true
}
// analyzePHPPhishingContent judges the head of a PHP file for credential
// capture behind an impersonated brand.
func analyzePHPPhishingContent(content string) *phishingResult {
contentLower := strings.ToLower(content)
// Superglobal names are case-sensitive. Resolve aliases before folding
// and share that view across credential capture and exfiltration checks.
phpLower := strings.ToLower(stripPHPCommentsFromCode(phpCodeOnly(content)))
var indicators []string
score := 0
// Check for PHP credential handling
phpCredHandling := false
for _, pattern := range phpPhishingPatterns {
if strings.Contains(phpLower, pattern) {
phpCredHandling = true
indicators = append(indicators, fmt.Sprintf("PHP credential handling: '%s'", pattern))
score += 2
break
}
}
// Must have either PHP credential handling OR HTML form output
hasForm := strings.Contains(contentLower, "<form") || strings.Contains(contentLower, "<input")
hasCredentialInput := hasHTMLCredentialInput(contentLower)
if !phpCredHandling && (!hasForm || !hasCredentialInput) {
return nil
}
// A title brand is strong evidence on its own. A visible body or logo brand
// is only accepted when the PHP reads a submitted password. Provider names
// in backend backup, payment, and AJAX integration code are not page
// impersonation and must not establish a brand.
brandMatch := ""
matchedGeneric := false
pageMarkup := stripPHPBlocks(contentLower)
titleContent := extractTitle(pageMarkup)
passwordInput := hasHTMLPasswordInput(pageMarkup)
if titleContent != "" {
for _, brand := range phishingBrands {
for _, tp := range brand.titlePatterns {
if titleNamesBrand(titleContent, tp, passwordInput) {
brandMatch = brand.name
matchedGeneric = brand.generic
indicators = append(indicators, fmt.Sprintf("title impersonates '%s'", tp))
score += 3
break
}
}
if brandMatch != "" {
break
}
}
}
if brandMatch == "" && hasPHPSubmittedPassword(phpLower) {
bodyContent := visiblePageContent(pageMarkup, true)
for _, brand := range phishingBrands {
if brand.generic {
continue
}
for _, bp := range brand.bodyPatterns {
if strings.Contains(bodyContent, bp) {
brandMatch = brand.name
indicators = append(indicators, fmt.Sprintf("body impersonates '%s'", bp))
score += 2
break
}
}
if brandMatch != "" {
break
}
}
}
// Check harvest/exfil patterns
for _, pattern := range harvestIndicators {
if strings.Contains(contentLower, pattern) {
score++
indicators = append(indicators, fmt.Sprintf("harvest: '%s'", pattern))
}
}
for _, pattern := range exfilPatterns {
if strings.Contains(contentLower, pattern) {
score += 2
indicators = append(indicators, fmt.Sprintf("exfiltration: '%s'", pattern))
}
}
// PHP-specific exfil: emailing or writing harvested credentials.
// These only fire when the file also reads from $_POST/$_REQUEST,
// because a phishing kit must capture form data before exfiltrating it.
// Without this gate, any PHP file using fwrite()+config keywords triggers.
hasPostData := strings.Contains(phpLower, "$_post") || strings.Contains(phpLower, "$_request")
if hasPostData && strings.Contains(contentLower, "mail(") &&
(strings.Contains(contentLower, "password") || strings.Contains(contentLower, "email")) {
score += 2
indicators = append(indicators, "PHP mail() with credential data")
}
if hasPostData &&
(strings.Contains(contentLower, "fwrite(") || strings.Contains(contentLower, "file_put_contents(")) {
if strings.Contains(contentLower, "password") || strings.Contains(contentLower, "email") ||
strings.Contains(contentLower, "result") || strings.Contains(contentLower, "log") {
score += 2
indicators = append(indicators, "PHP writes credential data to file")
}
}
// Require brand impersonation to flag as phishing.
// PHP files with $_POST['email'] + mail() are normal (contact forms, CMS user
// admin, gallery software). Without brand impersonation, these are almost
// always legitimate applications.
if brandMatch == "" {
return nil
}
// The generic pseudo-brand ("Sign In" titles) matches a customer's own
// login.php, which legitimately reads $_POST credentials. Require the
// higher floor so a real brand or genuine exfil behaviour is needed.
floor := 4
if matchedGeneric {
floor = phishingGenericBrandScoreFloor
}
if score >= floor {
return &phishingResult{brand: brandMatch, score: score, indicators: indicators}
}
return nil
}
func hasPHPSubmittedPassword(code string) bool {
passwordKeys := map[string]bool{
"password": true,
"pass": true,
"passwd": true,
"pwd": true,
"passcode": true,
}
for i := 0; i < len(code); {
if label, bodyStart, ok := phpHeredocOpen(code, i); ok {
i = phpHeredocEnd(code, bodyStart, label)
continue
}
if isPHPQuote(code[i]) {
i = skipPHPString(code, i) + 1
continue
}
nameLen := 0
switch {
case strings.HasPrefix(code[i:], "$_post"):
nameLen = len("$_post")
case strings.HasPrefix(code[i:], "$_request"):
nameLen = len("$_request")
}
if nameLen == 0 || (i+nameLen < len(code) && isPHPIdentifierPart(code[i+nameLen])) {
i++
continue
}
j := skipPHPWhitespace(code, i+nameLen)
if j >= len(code) || code[j] != '[' {
i += nameLen
continue
}
j = skipPHPWhitespace(code, j+1)
if j >= len(code) || !isPHPQuote(code[j]) {
i += nameLen
continue
}
keyEnd := skipPHPString(code, j)
if keyEnd <= j || keyEnd >= len(code) || code[keyEnd] != code[j] {
return false
}
key := code[j+1 : keyEnd]
j = skipPHPWhitespace(code, keyEnd+1)
if j < len(code) && code[j] == ']' && passwordKeys[key] {
return true
}
i += nameLen
}
return false
}
func stripPHPBlocks(content string) string {
codeOnly := phpCodeRegions(content)
visible := []byte(content)
for i := range visible {
switch codeOnly[i] {
case ' ', '\t', '\n', '\r':
default:
visible[i] = ' '
}
}
return string(visible)
}
// isKnownCMSFile returns true for PHP files that are standard CMS files
// and should not be scanned for phishing (too many false positives).
func isKnownCMSFile(nameLower string) bool {
cmsFiles := map[string]bool{
"index.php": true, "wp-config.php": true, "wp-login.php": true,
"wp-cron.php": true, "wp-settings.php": true, "wp-load.php": true,
"wp-blog-header.php": true, "wp-links-opml.php": true,
"xmlrpc.php": true, "wp-signup.php": true, "wp-activate.php": true,
"wp-trackback.php": true, "wp-comments-post.php": true,
"wp-mail.php": true, "configuration.php": true,
"config.php": true, "settings.php": true,
}
return cmsFiles[nameLower]
}
// ---------------------------------------------------------------------------
// Layer 5: PHP open redirectors
// ---------------------------------------------------------------------------
// checkPHPRedirector reads a small PHP file and checks if it's an open
// redirector - a file that redirects the visitor to a URL from a parameter.
func checkPHPRedirector(ctx context.Context, path string) string {
f, err := openTenantRegularFile(path)
if err != nil {
markScanReadError(ctx, "phishing", err)
return ""
}
defer func() { _ = f.Close() }()
buf := make([]byte, 1024)
n, err := io.ReadFull(f, buf)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
markCheckIncomplete(ctx, "phishing")
}
if n == 0 {
return ""
}
content := strings.ToLower(string(buf[:n]))
hasHeader := strings.Contains(content, "header(") &&
(strings.Contains(content, "location:") || strings.Contains(content, "location :"))
if !hasHeader {
return ""
}
// Pattern 1: user-controlled redirect target - the URL in header() must
// come from user input. Just having $_GET anywhere + header() is too broad;
// normal form handlers use $_POST for data then header() for redirect.
// Only flag when the redirect URL itself is parameterized.
userControlledRedirect := false
redirectPatterns := []string{
"$_get['url']", "$_get[\"url\"]",
"$_get['redirect']", "$_get[\"redirect\"]",
"$_get['r']", "$_get[\"r\"]",
"$_get['return']", "$_get[\"return\"]",
"$_get['next']", "$_get[\"next\"]",
"$_get['goto']", "$_get[\"goto\"]",
"$_get['link']", "$_get[\"link\"]",
"$_request['url']", "$_request[\"url\"]",
"$_request['redirect']", "$_request[\"redirect\"]",
"header(\"location: \".$_get", "header(\"location: \".$_request",
"header('location: '.$_get", "header('location: '.$_request",
"header(\"location:\".$_get", "header('location:'.$_get",
}
for _, p := range redirectPatterns {
if strings.Contains(content, p) {
userControlledRedirect = true
break
}
}
if userControlledRedirect {
return "PHP open redirector: header(Location) with user-supplied URL"
}
// Pattern 2: Hardcoded redirect to suspicious domain
for _, pattern := range exfilPatterns {
if strings.Contains(content, pattern) {
return fmt.Sprintf("PHP redirect to suspicious destination matching '%s'", pattern)
}
}
return ""
}
// ---------------------------------------------------------------------------
// Layer 6: Credential log files
// ---------------------------------------------------------------------------
// isCredentialLogName checks if a filename matches patterns used by phishing
// kits to store harvested credentials.
func isCredentialLogName(nameLower string) bool {
// Exact names commonly used by phishing kits
exactNames := map[string]bool{
"results.txt": true, "result.txt": true, "log.txt": true,
"logs.txt": true, "emails.txt": true, "data.txt": true,
"passwords.txt": true, "creds.txt": true, "credentials.txt": true,
"victims.txt": true, "output.txt": true, "harvested.txt": true,
"results.log": true, "emails.log": true, "data.log": true,
"results.csv": true, "emails.csv": true, "data.csv": true,
"results.html": true,
}
if exactNames[nameLower] {
return true
}
// Pattern: contains "result", "victim", "harvested", "credential" in name
suspiciousWords := []string{"result", "victim", "harvest", "credential", "creds", "stolen"}
for _, word := range suspiciousWords {
if strings.Contains(nameLower, word) {
return true
}
}
return false
}
// emailPattern matches a plausibly-real email address. The old heuristic
// counted any line containing '@', which random binary bytes and stray code
// tokens trip constantly. Requiring a local part, host and TLD keeps the count
// tied to actual addresses.
var (
emailPattern = regexp.MustCompile(`[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}`)
emailAddressPattern = regexp.MustCompile(`^[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}$`)
)
// credentialLogReadLimit bounds how much of a candidate file is read for
// credential analysis.
const credentialLogReadLimit = 10 * 1024 * 1024
// looksBinary reports whether normalized text bytes still look binary. Images,
// video, fonts, and other files can carry result/harvest in their names and
// incidental email delimiters in their bytes. A NUL or a high share of control
// bytes keeps those files out of the credential parser.
func looksBinary(data []byte) bool {
control := 0
for _, b := range data {
if b == 0x00 {
return true
}
// Bytes >= 0x80 are kept as text so UTF-8 (accented names in an email
// list) is not misread as binary.
if b < 0x09 || (b > 0x0d && b < 0x20) || b == 0x7f {
control++
}
}
return control*10 > len(data)
}
func normalizeCredentialLogText(data []byte) []byte {
if len(data) >= 3 && data[0] == 0xef && data[1] == 0xbb && data[2] == 0xbf {
return data[3:]
}
isLittleEndian := len(data) >= 2 && data[0] == 0xff && data[1] == 0xfe
isBigEndian := len(data) >= 2 && data[0] == 0xfe && data[1] == 0xff
body := data
var order binary.ByteOrder
switch {
case isLittleEndian:
body = data[2:]
order = binary.LittleEndian
case isBigEndian:
body = data[2:]
order = binary.BigEndian
default:
pairs := len(data) / 2
if pairs < 4 {
return data
}
evenNUL, oddNUL := 0, 0
for i := 0; i+1 < len(data); i += 2 {
if data[i] == 0 {
evenNUL++
}
if data[i+1] == 0 {
oddNUL++
}
}
switch {
case oddNUL >= 4 && oddNUL*2 >= pairs && evenNUL*20 <= pairs:
order = binary.LittleEndian
case evenNUL >= 4 && evenNUL*2 >= pairs && oddNUL*20 <= pairs:
order = binary.BigEndian
default:
return data
}
}
units := make([]uint16, len(body)/2)
for i := range units {
units[i] = order.Uint16(body[i*2:])
}
return []byte(string(utf16.Decode(units)))
}
// checkCredentialLog reads a text file and checks if it contains harvested
// credentials (email:password pairs, one per line) or a harvested address list.
func checkCredentialLog(ctx context.Context, path string) string {
f, err := openTenantRegularFile(path)
if err != nil {
markScanReadError(ctx, "phishing", err)
return ""
}
defer func() { _ = f.Close() }()
// Reject binaries from the head before reading the whole file into memory:
// images, video, and fonts that carry a "result"/"harvest" keyword in their
// name would otherwise be slurped in full only to be discarded. The head is
// decoded first so a UTF-16 dump, whose raw bytes look binary, survives the
// peek.
head := make([]byte, 8192)
hn, err := io.ReadFull(f, head)
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
markCheckIncomplete(ctx, "phishing")
return ""
}
head = head[:hn]
if hn == 0 || looksBinary(normalizeCredentialLogText(head)) {
return ""
}
rest, err := io.ReadAll(io.LimitReader(f, credentialLogReadLimit+1-int64(hn)))
if err != nil {
markScanReadError(ctx, "phishing", err)
return ""
}
return analyzeCredentialLog(append(head, rest...), path)
}
// analyzeCredentialLog classifies bounded, already-read file content as a
// harvested credential dump (email:password pairs) or address list, returning
// the finding detail or an empty string.
func analyzeCredentialLog(data []byte, path string) string {
if len(data) == 0 || len(data) > credentialLogReadLimit {
return ""
}
data = normalizeCredentialLogText(data)
if looksBinary(data) {
return ""
}
lines := strings.Split(string(data), "\n")
credentialLines := 0
emailLines := 0
nonEmpty := 0
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
nonEmpty++
if !emailPattern.MatchString(line) {
continue
}
emailLines++
// Pattern: email:password or email|password or email,password, where the
// field before the delimiter holds the address and the field after is a
// non-empty secret token with no whitespace.
for _, delim := range []string{":", "|", "\t", ","} {
parts := strings.SplitN(line, delim, 3)
if len(parts) >= 2 {
secret := strings.TrimSpace(parts[1])
address := strings.TrimSpace(parts[0])
if emailAddressPattern.MatchString(address) && secret != "" && !strings.ContainsAny(secret, " \t") {
credentialLines++
break
}
}
}
}
// 3+ lines that look like email:password pairs = credential log
if credentialLines >= 3 {
return fmt.Sprintf("File contains %d credential-like lines (email:password format) out of %d email lines",
credentialLines, emailLines)
}
// A harvested address list is dominated by addresses (about one per line).
// Source files (JavaScript modules, Drupal handlers) legitimately embed a
// handful of contributor/support addresses among mostly-code lines; those
// are not harvested lists, so require the addresses to be the majority of
// non-empty lines. .csv exports of a contact list are excluded outright.
if emailLines >= 10 && emailLines*2 >= nonEmpty && !strings.HasSuffix(strings.ToLower(path), ".csv") {
return fmt.Sprintf("File contains %d email addresses - possible harvested email list", emailLines)
}
return ""
}
// ---------------------------------------------------------------------------
// Layer 7: Iframe phishing
// ---------------------------------------------------------------------------
// checkIframePhishing checks small HTML files for iframe-based phishing -
// a minimal HTML page that just loads an external phishing page in a full-screen iframe.
func checkIframePhishing(ctx context.Context, path string) string {
f, err := openTenantRegularFile(path)
if err != nil {
markScanReadError(ctx, "phishing", err)
return ""
}
defer func() { _ = f.Close() }()
buf := make([]byte, 3000)
n, err := io.ReadFull(f, buf)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
markCheckIncomplete(ctx, "phishing")
}
if n == 0 {
return ""
}
contentLower := strings.ToLower(string(buf[:n]))
// Must contain an iframe
if !strings.Contains(contentLower, "<iframe") {
return ""
}
// Check if iframe src points to an external URL
idx := strings.Index(contentLower, "<iframe")
if idx < 0 {
return ""
}
rest := contentLower[idx:]
endTag := strings.Index(rest, ">")
if endTag < 0 {
return ""
}
iframeTag := rest[:endTag+1]
// Extract src
srcIdx := strings.Index(iframeTag, "src=")
if srcIdx < 0 {
return ""
}
srcRest := iframeTag[srcIdx+4:]
if len(srcRest) == 0 {
return ""
}
quote := srcRest[0]
if quote != '"' && quote != '\'' {
return ""
}
srcEnd := strings.IndexByte(srcRest[1:], quote)
if srcEnd < 0 {
return ""
}
src := srcRest[1 : srcEnd+1]
// Must be external (http:// or https://)
if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
return ""
}
// Check if iframe is fullscreen (width/height 100% or style covers viewport)
isFullscreen := strings.Contains(iframeTag, "100%") ||
strings.Contains(contentLower, "width:100%") ||
strings.Contains(contentLower, "width: 100%") ||
strings.Contains(contentLower, "position:fixed") ||
strings.Contains(contentLower, "position: fixed")
// A phishing wrapper is essentially just the iframe - its only purpose is to
// fill the screen with the external page. A documented embed or demo (e.g.
// software shipping an "iframe-example.html" with an explanatory paragraph)
// carries prose around the iframe, so real visible text means this is not a
// bare redirect wrapper.
if isFullscreen && visibleTextLen(contentLower) <= 40 {
return fmt.Sprintf("Full-screen iframe loading external URL: %s", src)
}
// Even non-fullscreen, check if URL matches known phishing/exfil patterns
for _, pattern := range exfilPatterns {
if strings.Contains(src, pattern) {
return fmt.Sprintf("Iframe loading suspicious external URL matching '%s': %s", pattern, src)
}
}
return ""
}
// ---------------------------------------------------------------------------
// Layer 8: Phishing kit ZIP archives
// ---------------------------------------------------------------------------
// isPhishingKitZipName checks if a ZIP filename matches common phishing kit names.
// Requires 2+ keyword matches to reduce false positives (e.g. "CssCheckboxKit"
// matched "kit" alone, but legitimate UI kits, CSS kits, etc. are common).
func isPhishingKitZipName(nameLower string) bool {
// High-confidence single-match keywords (brand impersonation in filename)
singleMatch := []string{
"office365", "office 365", "sharepoint", "onedrive",
"microsoft", "outlook", "gmail",
"dropbox", "docusign", "wetransfer",
"paypal", "icloud", "netflix",
"facebook", "instagram", "linkedin",
"roundcube", "cpanel",
"phish", "scam",
}
for _, kw := range singleMatch {
if strings.Contains(nameLower, kw) {
return true
}
}
// Lower-confidence keywords - require 2+ matches to flag.
// Words like "login", "verify", "secure", "google", "apple", "bank"
// appear in legitimate archives too.
multiMatch := []string{
"login", "verify", "secure", "bank",
"google", "apple", "adobe", "webmail",
}
matches := 0
for _, kw := range multiMatch {
if strings.Contains(nameLower, kw) {
matches++
}
}
return matches >= 2
}
// kitCaptureScripts are filenames phishing kits use for the server-side
// credential-capture step. These names are kit idiom, not generic app files.
var kitCaptureScripts = map[string]bool{
"next.php": true, "post.php": true, "send.php": true, "grab.php": true,
"result.php": true, "results.php": true,
}
var kitAntibotScripts = map[string]bool{
"antibots": true, "blocker": true,
"antibot.php": true, "blocker.php": true, "bots.php": true, "killbot.php": true,
}
// kitLoginPageWords identify login or verification pages inside an archive.
// The archive prefilter already supplies the brand signal, so repeating brand
// names here would misclassify ordinary provider plugins as login pages.
var kitLoginPageWords = []string{
"signin", "sign-in", "verify", "login", "log-in", "secure",
}
func isKitCredentialSinkName(base string) bool {
switch strings.ToLower(filepath.Ext(base)) {
case ".txt", ".log", ".csv":
return isCredentialLogName(base)
default:
return false
}
}
// maxPhishingZipBytes is the largest zip the check inspects. Its central
// directory is streamed one record at a time, so padding a kit with filler
// entries costs reading time, never memory, and cannot hide it.
const maxPhishingZipBytes = 50 * 1024 * 1024
// zipLooksLikeKit inspects a ZIP's central directory (entry names only, no
// decompression) and reports whether the contents match a phishing kit rather
// than a legitimate plugin/theme distribution archive. Filename brand keywords
// alone flagged legitimate plugin zips (Instagram Feed, Facebook articles,
// cPanel eCRM); the archive body is what actually distinguishes a kit: a
// credential sink file, a capture script, a brand-login page, an anti-bot
// blocker. Two signal categories from at least two distinct entries are
// required so one generic result filename cannot decide the archive alone.
func zipLooksLikeKit(ctx context.Context, path string) bool {
verdict, _ := classifyKitZip(ctx, path)
return verdict == zipKit
}
// zipVerdict is what the kit check concluded about one zip.
type zipVerdict int
const (
zipNotKit zipVerdict = iota
zipKit
// zipSkipped: the zip made the parse worker fail StrikeLimit times, so
// it is not sent again. Retrying would cost a worker on every scan and
// keep the check incomplete forever.
zipSkipped
)
// classifyKitZip opens path and judges it. A skipped zip comes with its
// SHA-256.
func classifyKitZip(ctx context.Context, path string) (zipVerdict, string) {
f, err := openTenantRegularFile(path)
if err != nil {
if err != errNonRegularFile {
markScanReadError(ctx, "phishing", err)
}
return zipNotKit, ""
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil {
markScanReadError(ctx, "phishing", err)
return zipNotKit, ""
}
// Out of bounds or not a regular file is a scope decision, not a coverage
// failure: the caller already gates on the same range, and marking the
// check incomplete here would stop it ever retiring a stale finding on a
// host that simply keeps a large archive under a docroot.
if !info.Mode().IsRegular() || info.Size() <= 1000 || info.Size() >= maxPhishingZipBytes {
return zipNotKit, ""
}
lister := currentZipLister()
if lister == nil || ctx.Err() != nil {
markCheckIncomplete(ctx, "phishing")
return zipNotKit, ""
}
if zipStrikes.Len() > 0 {
sum, hashErr := zipSHA256(f, info)
if hashErr != nil || ctx.Err() != nil {
markCheckIncomplete(ctx, "phishing")
return zipNotKit, ""
}
if zipStrikes.Blocked(sum) {
return zipSkipped, sum
}
}
kit, err := phishingKitZipHasEvidence(ctx, lister, f, info.Size())
if err != nil {
markCheckIncomplete(ctx, "phishing")
if errors.Is(err, workerproc.ErrWorkerFailed) {
if sum, hashErr := zipSHA256(f, info); hashErr == nil {
zipStrikes.Record(sum)
saveZipCrashSample(sum, f, info.Size())
}
}
}
if kit {
return zipKit, ""
}
return zipNotKit, ""
}
func zipSHA256(f *os.File, before os.FileInfo) (string, error) {
h := sha256.New()
if _, err := io.CopyN(h, io.NewSectionReader(f, 0, before.Size()), before.Size()); err != nil {
return "", err
}
after, err := f.Stat()
if err != nil {
return "", err
}
// The tenant can rewrite the open inode while the worker runs or while
// we hash it. Such a failure cannot be charged to the replacement bytes.
if !SameFileVersion(before, after) {
return "", errors.New("zip changed while being checked")
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// phishingKitZipHasEvidence confirms a bounded ZIP using independent entry-name
// signals. The names come from the parse worker; nothing of the archive is
// parsed or decompressed in this process.
func phishingKitZipHasEvidence(ctx context.Context, lister ZipLister, f *os.File, size int64) (bool, error) {
if size <= 1000 || size >= maxPhishingZipBytes {
return false, nil
}
capture, credSink, antibot := false, false, false
brandPages := 0
// Two distinct evidence entries are all the verdict needs, so at most two
// names are kept however many entries carry evidence.
var evidenceEntries []string
errKit := errors.New("kit evidence complete")
signals := func() int {
n := 0
for _, on := range []bool{capture, credSink, antibot, brandPages >= 1} {
if on {
n++
}
}
return n
}
err := lister.ListZip(ctx, f, size, maxPhishingZipBytes, func(name string, dir bool) error {
if dir {
return nil
}
base := name
if i := strings.LastIndexAny(base, "/\\"); i >= 0 {
base = base[i+1:]
}
base = strings.ToLower(base)
hasEvidence := false
if kitCaptureScripts[base] {
capture = true
hasEvidence = true
}
if isKitCredentialSinkName(base) {
credSink = true
hasEvidence = true
}
if kitAntibotScripts[base] || strings.Contains(base, "antibot") {
antibot = true
hasEvidence = true
}
if strings.HasSuffix(base, ".html") || strings.HasSuffix(base, ".htm") || isExecutablePHPName(base) {
for _, w := range kitLoginPageWords {
if strings.Contains(base, w) {
brandPages++
hasEvidence = true
break
}
}
}
if hasEvidence && len(evidenceEntries) < 2 && !slices.Contains(evidenceEntries, name) {
evidenceEntries = append(evidenceEntries, name)
}
if signals() >= 2 && len(evidenceEntries) >= 2 {
return errKit
}
return nil
})
if errors.Is(err, errKit) {
return true, nil
}
return false, err
}
func visiblePageContent(s string, includeImageAttrs bool) string {
doc, err := html.Parse(strings.NewReader(s))
if err != nil {
return ""
}
hiddenClasses, hiddenIDs := stylesheetHiddenSelectors(doc)
var content strings.Builder
var walk func(*html.Node, bool)
walk = func(node *html.Node, hidden bool) {
if node.Type == html.ElementNode {
switch node.Data {
case "head", "script", "style", "template", "noscript", "iframe":
hidden = true
}
if htmlElementHidden(node, hiddenClasses, hiddenIDs) {
hidden = true
}
if includeImageAttrs && !hidden && node.Data == "img" {
for _, attr := range node.Attr {
switch attr.Key {
case "alt", "title", "aria-label", "src":
content.WriteByte(' ')
content.WriteString(attr.Val)
}
}
}
}
if node.Type == html.TextNode && !hidden {
content.WriteByte(' ')
content.WriteString(node.Data)
}
for child := node.FirstChild; child != nil; child = child.NextSibling {
walk(child, hidden)
}
}
walk(doc, false)
return content.String()
}
func htmlElementHidden(node *html.Node, hiddenClasses, hiddenIDs map[string]bool) bool {
for _, attr := range node.Attr {
switch attr.Key {
case "hidden":
return true
case "class":
for _, class := range strings.Fields(attr.Val) {
if hiddenClasses[class] {
return true
}
}
case "id":
if hiddenIDs[attr.Val] {
return true
}
case "aria-hidden":
if strings.EqualFold(strings.TrimSpace(attr.Val), "true") {
return true
}
case "style":
if cssDeclarationsHide(attr.Val) {
return true
}
}
}
return false
}
func stylesheetHiddenSelectors(doc *html.Node) (map[string]bool, map[string]bool) {
hiddenClasses := make(map[string]bool)
hiddenIDs := make(map[string]bool)
var walk func(*html.Node)
walk = func(node *html.Node) {
if node.Type == html.ElementNode && node.Data == "style" {
var css strings.Builder
for child := node.FirstChild; child != nil; child = child.NextSibling {
if child.Type == html.TextNode {
css.WriteString(child.Data)
}
}
collectHiddenStylesheetSelectors(css.String(), hiddenClasses, hiddenIDs)
}
for child := node.FirstChild; child != nil; child = child.NextSibling {
walk(child)
}
}
walk(doc)
return hiddenClasses, hiddenIDs
}
func collectHiddenStylesheetSelectors(css string, hiddenClasses, hiddenIDs map[string]bool) {
css = stripCSSComments(css)
for {
open := strings.IndexByte(css, '{')
if open < 0 {
return
}
closeOffset := strings.IndexByte(css[open+1:], '}')
if closeOffset < 0 {
return
}
closeIndex := open + 1 + closeOffset
if cssDeclarationsHide(css[open+1 : closeIndex]) {
for _, selector := range strings.Split(css[:open], ",") {
selector = strings.TrimSpace(selector)
if len(selector) < 2 || strings.ContainsAny(selector, " >+~:[]*") {
continue
}
marker := strings.LastIndexAny(selector, ".#")
if marker < 0 || !isSimpleCSSIdentifier(selector[marker+1:]) {
continue
}
prefix := selector[:marker]
if prefix != "" && !isSimpleCSSIdentifier(prefix) {
continue
}
switch selector[marker] {
case '.':
hiddenClasses[selector[marker+1:]] = true
case '#':
hiddenIDs[selector[marker+1:]] = true
}
}
}
css = css[closeIndex+1:]
}
}
func stripCSSComments(css string) string {
var stripped strings.Builder
for len(css) > 0 {
start := strings.Index(css, "/*")
if start < 0 {
stripped.WriteString(css)
break
}
stripped.WriteString(css[:start])
end := strings.Index(css[start+2:], "*/")
if end < 0 {
break
}
css = css[start+2+end+2:]
}
return stripped.String()
}
func cssDeclarationsHide(declarations string) bool {
for _, declaration := range strings.Split(strings.ToLower(declarations), ";") {
parts := strings.SplitN(declaration, ":", 2)
if len(parts) != 2 {
continue
}
property := strings.TrimSpace(parts[0])
value := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(parts[1]), "!important"))
if (property == "display" && value == "none") ||
(property == "visibility" && (value == "hidden" || value == "collapse")) ||
(property == "opacity" && value == "0") {
return true
}
}
return false
}
func isSimpleCSSIdentifier(s string) bool {
if s == "" {
return false
}
for _, r := range s {
if !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '-' && r != '_' {
return false
}
}
return true
}
// visibleTextLen counts rendered non-whitespace characters. Script, style,
// template, noscript, iframe fallback, comment, and head contents do not
// document an embed.
func visibleTextLen(s string) int {
n := 0
for _, r := range visiblePageContent(s, false) {
if !unicode.IsSpace(r) {
n++
}
}
return n
}
package checks
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/contenttype"
"github.com/pidginhost/csm/internal/state"
)
// nestedEvalDecodeRe matches the PHP token sequence
// `<eval|assert> [ws] ( [ws] [@] [\] <ident> [ws] (`, with DOTALL so the
// source can have line breaks inside the whitespace gaps. Attackers wedge
// comments or common call modifiers between the sink and decoder; comments and
// strings are stripped before matching so only executable token structure is
// evaluated.
var nestedEvalDecodeRe = regexp.MustCompile(`(?is)\b(eval|assert)\s*\(\s*(?:(?:"\s*"|'\s*')?\s*\.\s*)*@?\s*\\?\s*(\w+)\s*\(`)
// reEvalRequestInput matches eval/assert applied to request input with no
// decoder at all (`eval($_POST['c'])`), the simplest possible shell. Strings
// are stripped before matching, so the superglobal name is what remains.
var reEvalRequestInput = regexp.MustCompile(`(?is)\b(eval|assert)\s*\(\s*(?:(?:"\s*"|'\s*')?\s*\.\s*)*@?\s*\$_(?:post|get|request|cookie)\b`)
// reEvalVarCallee matches eval wrapping a variable function call,
// e.g. `eval($f(...))`. The literal-callee form above cannot capture a
// `$var` callee, yet eval'ing the result of a dynamic function call is a
// near-certain dropper signal in user web directories.
var reEvalVarCallee = regexp.MustCompile(`(?is)\beval\s*\(\s*@?\s*\$\w+\s*\(`)
// evalExecWrapInner lists code-construction primitives that, when wrapped
// directly by eval, indicate dynamic code execution rather than the
// decoder/decompressor chain nestedEvalDecodeRe already covers.
var evalExecWrapInner = map[string]struct{}{
"create_function": {},
"call_user_func": {},
"call_user_func_array": {},
}
var callbackFirstArgFuncs = map[string]struct{}{
"array_map": {},
"call_user_func": {},
"call_user_func_array": {},
"register_shutdown_function": {},
"register_tick_function": {},
}
// callbackExecNames are names that execute code when used as a callback. They
// are RCE regardless of where the call's arguments come from, so they flag
// unconditionally.
var callbackExecNames = map[string]struct{}{
"assert": {},
"call_user_func": {},
"create_function": {},
"eval": {},
"exec": {},
"passthru": {},
"popen": {},
"proc_open": {},
"shell_exec": {},
"system": {},
}
// callbackDecoderNames are decode/decompress primitives. As a callback they
// only transform data (array_map('base64_decode', $data) returns decoded
// bytes, it executes nothing), so legitimate plugins use them constantly. They
// only signal a dropper when the same call is fed request input; the
// decode-then-eval shape is covered separately by the eval-chain detectors.
var callbackDecoderNames = map[string]struct{}{
"base64_decode": {},
"gzinflate": {},
"gzuncompress": {},
"str_rot13": {},
}
// reVarVarCall matches a variable-variable or dynamic-expression function
// invocation, e.g. `$$h(...)` or `${$x}(...)`. On its own this shows up in
// some dispatcher code, so it is only treated as an indicator when the same
// line also carries a request superglobal (the RCE shape).
var reVarVarCall = regexp.MustCompile(`(?:\$\$\w+|\$\{[^}]{1,64}\})\s*\(`)
// Goto-obfuscation discrimination. A label an obfuscator generates carries no
// meaning: it is short, or it is a stem plus a counter. A hand-written state
// machine names its labels after what they mean, which is why WordPress core's
// HTML5 insertion-mode labels must not count.
var reGotoLabel = regexp.MustCompile(`(?i)\bgoto\s+([A-Za-z_][A-Za-z0-9_]{0,63})\s*;`)
// reGotoExecSink is the evidence half of the goto heuristic, kept in step with
// the php_goto_obfuscation signature in configs/. call_user_func is absent on
// purpose: plugin loaders dispatch their own callables through it.
var reGotoExecSink = regexp.MustCompile(`(?i)\b(?:eval|assert|create_function|system|exec|passthru|shell_exec|proc_open|popen|pcntl_exec|base64_decode|gzinflate|gzuncompress|gzdecode|str_rot13|hex2bin|convert_uudecode)(?:\s|/\*[^*]*\*+(?:[^/*][^*]*\*+)*/|//[^\r\n]*[\r\n]|#[^\r\n]*[\r\n])*\(` +
`|(?-i:\$_(?:GET|POST|REQUEST|COOKIE|FILES)\b)` +
`|\$[A-Za-z_][A-Za-z0-9_]*(?:\s*\[[^\]\r\n]+\])*(?:\s|/\*[^*]*\*+(?:[^/*][^*]*\*+)*/|//[^\r\n]*[\r\n]|#[^\r\n]*[\r\n])*(?:\)(?:\s|/\*[^*]*\*+(?:[^/*][^*]*\*+)*/|//[^\r\n]*[\r\n]|#[^\r\n]*[\r\n])*)?\(` +
`|\b(?:include|require)(?:_once)?\b(?:\s|/\*[^*]*\*+(?:[^/*][^*]*\*+)*/|//[^\r\n]*[\r\n]|#[^\r\n]*[\r\n])*(?:\(\s*)?\$`)
// gotoLabelIsGenerated reports whether a goto label looks machine-generated.
// Digits are the strongest tell (lbl0, x9k, a1); anything shorter than four
// characters cannot carry meaning either.
func gotoLabelIsGenerated(label string) bool {
if len(label) < 4 {
return true
}
return strings.ContainsAny(label, "0123456789")
}
// includeDangerWrappers are stream wrappers / remote schemes that, as an
// include/require target, mean remote-file inclusion or php://input code
// execution. Matched on the comment-stripped (strings preserved) source.
var includeDangerWrappers = []string{"data://", "php://", "phar://", "http://", "https://", "ftp://"}
var includeKeywords = []string{"include_once", "require_once", "include", "require"}
var (
pregReplaceCallName = map[string]struct{}{"preg_replace": {}}
codeEvalPrimitiveCallNames = map[string]struct{}{"assert": {}, "create_function": {}}
callUserFuncCallNames = map[string]struct{}{"call_user_func": {}, "call_user_func_array": {}}
)
// phpContentReadSize bounds the head window read from a PHP file for content
// analysis. Most real PHP is far smaller; the window is large enough that an
// attacker cannot cheaply hide a payload by prepending benign padding.
const phpContentReadSize = 1 << 20 // 1 MiB head window
// phpContentTailSize is also scanned for files larger than the head window so
// a payload appended after >1 MiB of padding is still seen. The two windows
// are scanned together; bounded total memory is head+tail.
const phpContentTailSize = 64 << 10 // 64 KiB tail window
// readPHPContentWindows returns the bytes to analyse for a PHP file: the
// first phpContentReadSize bytes, plus the last phpContentTailSize bytes when
// the file is larger than the head window. tailOffset is -1 when tail is empty.
// readOK is false only on a real read error.
func readPHPContentWindows(f io.ReaderAt, size int64) (head, tail []byte, tailOffset int64, readOK bool) {
tailOffset = -1
headLen := int64(phpContentReadSize)
if size >= 0 && size < headLen {
headLen = size
}
headBuf := make([]byte, headLen)
n, err := f.ReadAt(headBuf, 0)
if err != nil && !errors.Is(err, io.EOF) {
return nil, nil, -1, false
}
head = headBuf[:n]
if size > int64(phpContentReadSize) {
tailLen := int64(phpContentTailSize)
off := size - tailLen
if off < int64(phpContentReadSize) {
// Overlap into the head window is fine; it only re-scans bytes.
off = int64(phpContentReadSize)
tailLen = size - off
}
if tailLen > 0 {
tailBuf := make([]byte, tailLen)
tn, terr := f.ReadAt(tailBuf, off)
if terr != nil && !errors.Is(terr, io.EOF) {
return nil, nil, -1, false
}
tail = tailBuf[:tn]
tailOffset = off
}
}
return head, tail, tailOffset, true
}
func combinePHPContentWindows(head, tail []byte) []byte {
if len(tail) == 0 {
return head
}
capacity := len(head) + len(tail)
raw := make([]byte, 0, capacity)
raw = append(raw, head...)
raw = append(raw, tail...)
return raw
}
func phpCodeOnlyWindows(head, tail []byte) string {
return phpCodeOnly(string(combinePHPContentWindows(head, tail)))
}
func hasBacktickSuperglobal(code string) bool {
for i := 0; i < len(code); i++ {
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
if code[i] != '`' {
continue
}
end := i + 1
for end < len(code) {
if code[end] == '\\' && end+1 < len(code) {
end += 2
continue
}
if code[end] == '`' {
break
}
end++
}
if end >= len(code) {
break
}
if containsRequestSuperglobal(code[i+1 : end]) {
return true
}
i = end
}
return false
}
func hasCallbackExecName(code string) bool {
for i := 0; i < len(code); i++ {
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
nameStart := i
if code[i] == '\\' {
if i+1 >= len(code) || !isPHPIdentifierStart(code[i+1]) || !canStartGlobalPHPFunction(code, i) {
continue
}
nameStart = i + 1
} else if !isPHPIdentifierStart(code[i]) || !canStartPHPFunctionName(code, i) {
continue
}
nameEnd := nameStart + 1
for nameEnd < len(code) && isPHPIdentifierPart(code[nameEnd]) {
nameEnd++
}
name := strings.ToLower(code[nameStart:nameEnd])
if _, ok := callbackFirstArgFuncs[name]; !ok {
i = nameEnd - 1
continue
}
openParen := skipPHPWhitespace(code, nameEnd)
if openParen >= len(code) || code[openParen] != '(' {
i = nameEnd - 1
continue
}
firstArg := skipPHPWhitespace(code, openParen+1)
if firstArg >= len(code) || !isPHPQuote(code[firstArg]) {
i = nameEnd - 1
continue
}
callbackName, _, ok := readPHPFunctionString(code, firstArg)
if !ok {
i = nameEnd - 1
continue
}
if _, dangerous := callbackExecNames[callbackName]; dangerous {
return true
}
if _, decoder := callbackDecoderNames[callbackName]; decoder {
closeParen := matchingParen(code, openParen)
if containsRequestSuperglobalExpression(code[openParen:closeParen]) {
return true
}
}
i = nameEnd - 1
}
return false
}
// matchingParen returns the index of the close paren matching the open paren
// at openParen, or len(code) if unbalanced. Quoted strings are skipped so
// parens inside string literals do not throw off the depth count.
func matchingParen(code string, openParen int) int {
depth := 0
for i := openParen; i < len(code); i++ {
if label, bodyStart, ok := phpHeredocOpen(code, i); ok {
i = phpHeredocEnd(code, bodyStart, label) - 1
continue
}
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
switch code[i] {
case '(':
depth++
case ')':
depth--
if depth == 0 {
return i
}
}
}
return len(code)
}
// hasDangerousInclude reports an include/require whose target is request
// input or a remote/stream wrapper -- the LFI/RFI and php://input code-exec
// shapes. Scans the include expression, not the whole line, so unrelated
// request reads or URLs after a static include do not trip the detector.
func hasDangerousInclude(code string) bool {
searchFrom := 0
for {
exprStart, exprEnd, ok := nextIncludeExpression(code, searchFrom)
if !ok {
break
}
expr := code[exprStart:exprEnd]
if containsIncludeTargetExpression(expr) {
return true
}
exprLower := strings.ToLower(expr)
for _, w := range includeDangerWrappers {
if strings.Contains(exprLower, w) {
return true
}
}
searchFrom = exprEnd
if searchFrom >= len(code) {
break
}
}
return false
}
// hasCodeEvalPrimitiveWithRequest reports assert()/create_function() fed
// request input. create_function() evals its body argument, so any request
// superglobal inside the call is a code-eval sink. assert() (pre-8.0) only
// evaluates its first argument as PHP when that argument is a string, so the
// argument expression is classified first: expressions that provably yield a
// non-string value (boolean/comparison operators, negation, builtins that
// never return strings) are not code-eval sinks. Anything not provably
// non-string stays flagged, fail closed.
func hasCodeEvalPrimitiveWithRequest(code string) bool {
// Whether an unqualified builtin name can be trusted is only consulted on
// the assert()-with-request-argument path, which almost no file hits.
// phpMayRedirectUnqualifiedBuiltin tokenizes the whole scan window, so
// defer that pass until the first time it is actually needed, then cache.
trustComputed := false
trustUnqualified := false
resolveTrust := func() bool {
if !trustComputed {
trustUnqualified = !phpMayRedirectUnqualifiedBuiltin(code)
trustComputed = true
}
return trustUnqualified
}
searchFrom := 0
for {
callStart, openParen, closeParen, ok := nextStandalonePHPCall(code, searchFrom, codeEvalPrimitiveCallNames)
if !ok {
break
}
name := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(code[callStart:openParen]), `\`))
exprEnd := closeParen
if exprEnd < len(code) {
exprEnd++
}
if containsRequestSuperglobal(code[openParen:exprEnd]) {
if name != "assert" {
return true
}
if closeParen >= len(code) {
// The call is unbalanced in the scan window, so the
// argument cannot be classified. Fail closed.
return true
}
args := phpCallArguments(code, openParen+1, closeParen)
if len(args) > 0 && containsRequestSuperglobal(args[0]) &&
!phpExpressionYieldsNonString(args[0], resolveTrust()) {
return true
}
}
searchFrom = nextSearchOffset(closeParen, len(code))
if searchFrom >= len(code) {
break
}
}
return false
}
// Precedence ranks for depth-0 binary operators inside an assert() argument,
// lowest precedence first. The lowest-precedence operator present produces
// the expression's final value, so it alone decides whether the result can
// be a string.
const (
phpRankWordLogical = 1 // and / or / xor -- boolean result
phpRankAssign = 2 // = and compound assignments -- value passes through
phpRankTernary = 3 // ?: / ?? / ?-> -- string-capable value
phpRankLogicalOr = 4 // || -- boolean result
phpRankLogicalAnd = 5 // && -- boolean result
phpRankBitwiseOr = 6 // | -- string-capable when both operands are strings
phpRankBitwiseXor = 7 // ^ -- string-capable when both operands are strings
phpRankBitwiseAnd = 8 // & -- string-capable when both operands are strings
phpRankComparison = 9 // == != === !== <> <=> < > <= >= instanceof -- non-string
phpRankShift = 10 // << >> -- integer result; tied with . for PHP 7/8 precedence drift
phpRankConcat = 10 // . -- string result
phpRankAdditive = 12 // + - -- numeric/array result, never string
phpRankMultiplicative = 13 // * / % -- numeric result
phpRankPower = 14 // ** -- numeric result
phpRankNone = 100 // no depth-0 binary operator: single atom
)
type phpOperatorValue int
const (
phpOperatorNone phpOperatorValue = iota
phpOperatorNonString
phpOperatorStringCapable
)
// phpExpressionYieldsNonString reports whether a PHP expression provably
// evaluates to a non-string value, so assert() cannot treat it as code.
// trustUnqualified is false when the file declares a namespace or imports a
// function alias, because then an unqualified builtin name can resolve to
// attacker-defined code. Anything unrecognized counts as string-capable.
//
// Wrapping parentheses are peeled in one parser pass so a payload with tens of
// thousands of nested parens cannot exhaust the stack or trigger quadratic
// rescans.
func phpExpressionYieldsNonString(expr string, trustUnqualified bool) bool {
expr = peelPHPAssertionExpression(expr)
if expr == "" {
return false
}
switch phpTopLevelOperatorValue(expr) {
case phpOperatorNonString:
return true
case phpOperatorStringCapable:
return false
}
if expr[0] == '!' {
// Negation of a whole atom is always boolean. Operators inside the
// negated expression would have been classified above, so this point
// is only reached for a single negated atom.
return true
}
return phpIsNonStringBuiltinCall(expr, trustUnqualified)
}
func peelPHPAssertionExpression(expr string) string {
start, end := trimPHPExpressionBounds(expr, 0, len(expr))
if start >= end || expr[start] != '(' {
return expr[start:end]
}
prefixOpens := phpLeadingParenPositions(expr, start, end)
if len(prefixOpens) == 0 {
return expr[start:end]
}
matches, ok := phpMatchPrefixParens(expr, start, end, prefixOpens)
if !ok {
return expr[start:end]
}
innerStart, innerEnd := start, end
for idx, open := range prefixOpens {
if open != skipPHPWhitespaceUntil(expr, innerStart, innerEnd) {
break
}
close := matches[idx]
if close != skipPHPWhitespaceBack(expr, innerStart, innerEnd)-1 {
break
}
innerStart = open + 1
innerEnd = close
}
start, end = trimPHPExpressionBounds(expr, innerStart, innerEnd)
return expr[start:end]
}
func phpLeadingParenPositions(expr string, start, end int) []int {
var positions []int
for {
start = skipPHPWhitespaceUntil(expr, start, end)
if start >= end || expr[start] != '(' {
return positions
}
positions = append(positions, start)
start++
}
}
func phpMatchPrefixParens(expr string, start, end int, prefixOpens []int) ([]int, bool) {
matches := make([]int, len(prefixOpens))
for i := range matches {
matches[i] = -1
}
depth := 0
for i := start; i < end; i++ {
if label, bodyStart, ok := phpHeredocOpen(expr, i); ok {
i = phpHeredocEnd(expr, bodyStart, label) - 1
continue
}
if isPHPQuote(expr[i]) {
i = skipPHPString(expr, i)
continue
}
switch expr[i] {
case '(':
depth++
case ')':
if depth == 0 {
return nil, false
}
depth--
if depth < len(matches) && matches[depth] == -1 {
matches[depth] = i
}
}
}
if depth != 0 {
return nil, false
}
for _, match := range matches {
if match == -1 {
return nil, false
}
}
return matches, true
}
func trimPHPExpressionBounds(expr string, start, end int) (int, int) {
if start < 0 {
start = 0
}
if end > len(expr) {
end = len(expr)
}
start = skipPHPWhitespaceUntil(expr, start, end)
end = skipPHPWhitespaceBack(expr, start, end)
return start, end
}
func skipPHPWhitespaceUntil(expr string, start, end int) int {
for start < end && isPHPSpace(expr[start]) {
start++
}
return start
}
func skipPHPWhitespaceBack(expr string, start, end int) int {
for end > start && isPHPSpace(expr[end-1]) {
end--
}
return end
}
// phpTopLevelOperatorValue scans expr outside string literals at paren depth 0
// and returns whether the lowest-precedence operator found has a provably
// non-string result. String-capable operators win ties to keep the classifier
// fail-closed across PHP version precedence differences.
func phpTopLevelOperatorValue(expr string) phpOperatorValue {
rank := phpRankNone
value := phpOperatorNone
observe := func(opRank int, opValue phpOperatorValue) {
if opRank < rank || (opRank == rank && opValue == phpOperatorStringCapable) {
rank = opRank
value = opValue
}
}
depth := 0
prev := byte(0) // last significant byte
i := 0
for i < len(expr) {
c := expr[i]
if label, bodyStart, ok := phpHeredocOpen(expr, i); ok {
observe(phpRankNone, phpOperatorStringCapable)
i = phpHeredocEnd(expr, bodyStart, label)
prev = '"'
continue
}
if isPHPQuote(c) {
i = skipPHPString(expr, i) + 1
prev = '"'
continue
}
if isPHPSpace(c) {
i++
continue
}
switch c {
case '(', '[', '{':
depth++
prev = c
i++
continue
case ')', ']', '}':
if depth > 0 {
depth--
}
prev = c
i++
continue
}
if depth > 0 {
prev = c
i++
continue
}
if isPHPIdentifierStart(c) {
start := i
for i < len(expr) && isPHPIdentifierPart(expr[i]) {
i++
}
// A word preceded by $, ->, ::, or \ is a variable, member, or
// namespaced name, never the operator keyword.
if !isPHPIdentifierPart(prev) && prev != '$' && prev != '>' && prev != ':' && prev != '\\' {
switch strings.ToLower(expr[start:i]) {
case "and", "or", "xor":
observe(phpRankWordLogical, phpOperatorNonString)
case "instanceof":
observe(phpRankComparison, phpOperatorNonString)
}
}
prev = expr[i-1]
continue
}
next := byte(0)
if i+1 < len(expr) {
next = expr[i+1]
}
consumed := 1
switch c {
case '=':
switch next {
case '=':
observe(phpRankComparison, phpOperatorNonString)
consumed = 2
if i+2 < len(expr) && expr[i+2] == '=' {
consumed = 3
}
case '>':
// A stray => at depth 0 is not valid expression syntax;
// treat it like an assignment and fail closed.
observe(phpRankAssign, phpOperatorStringCapable)
consumed = 2
default:
observe(phpRankAssign, phpOperatorStringCapable)
}
case '!':
if next == '=' {
observe(phpRankComparison, phpOperatorNonString)
consumed = 2
if i+2 < len(expr) && expr[i+2] == '=' {
consumed = 3
}
}
case '?':
observe(phpRankTernary, phpOperatorStringCapable)
if next == '?' {
consumed = 2
} else if next == '-' && i+2 < len(expr) && expr[i+2] == '>' {
consumed = 3
}
case '<':
switch {
case next == '<':
observe(phpRankShift, phpOperatorNonString)
consumed = 2
case next == '=' && i+2 < len(expr) && expr[i+2] == '>':
observe(phpRankComparison, phpOperatorNonString)
consumed = 3
default:
observe(phpRankComparison, phpOperatorNonString)
if next == '=' || next == '>' {
consumed = 2
}
}
case '>':
switch next {
case '>':
observe(phpRankShift, phpOperatorNonString)
consumed = 2
case '=':
observe(phpRankComparison, phpOperatorNonString)
consumed = 2
default:
observe(phpRankComparison, phpOperatorNonString)
}
case '-':
switch next {
case '>':
consumed = 2 // property or method access, not an operator
case '-':
consumed = 2 // string increment/decrement is string-capable; fail closed as an atom
default:
observe(phpRankAdditive, phpOperatorNonString)
}
case '+':
if next == '+' {
consumed = 2 // string increment is string-capable; fail closed as an atom
} else {
observe(phpRankAdditive, phpOperatorNonString)
}
case '|':
if next == '|' {
observe(phpRankLogicalOr, phpOperatorNonString)
consumed = 2
} else {
observe(phpRankBitwiseOr, phpOperatorStringCapable)
}
case '&':
if next == '&' {
observe(phpRankLogicalAnd, phpOperatorNonString)
consumed = 2
} else {
observe(phpRankBitwiseAnd, phpOperatorStringCapable)
}
case '^':
observe(phpRankBitwiseXor, phpOperatorStringCapable)
case '.':
if !phpDotBelongsToNumberLiteral(expr, i) {
observe(phpRankConcat, phpOperatorStringCapable)
}
case '*':
if next == '*' {
observe(phpRankPower, phpOperatorNonString)
consumed = 2
} else {
observe(phpRankMultiplicative, phpOperatorNonString)
}
case '/', '%':
observe(phpRankMultiplicative, phpOperatorNonString)
}
prev = expr[i+consumed-1]
i += consumed
}
return value
}
func phpDotBelongsToNumberLiteral(expr string, dot int) bool {
if dot > 0 && isPHPDecimalDigit(expr[dot-1]) {
return phpDotContinuesDecimalLiteral(expr, dot)
}
if dot+1 >= len(expr) || !isPHPDecimalDigit(expr[dot+1]) {
return false
}
for i := dot - 1; i >= 0; i-- {
if isPHPSpace(expr[i]) {
continue
}
switch expr[i] {
case '(', '[', '{', ',', '?', ':', '=', '!', '<', '>', '+', '-', '*', '/', '%', '&', '|', '^', '~':
return true
default:
return false
}
}
return true
}
func phpDotContinuesDecimalLiteral(expr string, dot int) bool {
for i := dot - 1; i >= 0; i-- {
if isPHPDecimalDigit(expr[i]) || expr[i] == '_' {
continue
}
switch expr[i] {
case '.', 'e', 'E', 'x', 'X', 'b', 'B', 'o', 'O':
return false
case '+', '-':
return i == 0 || (expr[i-1] != 'e' && expr[i-1] != 'E')
default:
return !isPHPIdentifierPart(expr[i]) && expr[i] != '$' && expr[i] != '\\'
}
}
return true
}
func isPHPDecimalDigit(c byte) bool {
return c >= '0' && c <= '9'
}
// phpNonStringReturnBuiltins lists PHP builtins (plus the isset/empty
// language constructs) whose return value can never be a string, so feeding
// their result to assert() cannot evaluate attacker input as code. Functions
// that can return a string (filter_var, base64_decode, ...) must never be
// added here.
var phpNonStringReturnBuiltins = map[string]struct{}{
"isset": {}, "empty": {}, "defined": {},
"file_exists": {}, "is_file": {}, "is_dir": {}, "is_link": {},
"is_readable": {}, "is_writable": {}, "is_writeable": {}, "is_executable": {},
"is_uploaded_file": {},
"is_numeric": {}, "is_string": {}, "is_array": {}, "is_int": {},
"is_integer": {}, "is_long": {}, "is_bool": {}, "is_float": {},
"is_double": {}, "is_null": {}, "is_object": {}, "is_callable": {},
"is_iterable": {}, "is_countable": {}, "is_resource": {}, "is_a": {},
"is_subclass_of": {},
"in_array": {}, "array_key_exists": {}, "key_exists": {},
"property_exists": {}, "method_exists": {}, "function_exists": {},
"class_exists": {}, "interface_exists": {}, "trait_exists": {}, "enum_exists": {},
"str_contains": {}, "str_starts_with": {}, "str_ends_with": {},
"preg_match": {}, "preg_match_all": {},
"count": {}, "sizeof": {}, "strlen": {}, "mb_strlen": {},
"strcmp": {}, "strcasecmp": {}, "strncmp": {}, "strncasecmp": {},
"substr_compare": {}, "version_compare": {}, "hash_equals": {},
"password_verify": {}, "checkdate": {},
"ctype_alnum": {}, "ctype_alpha": {}, "ctype_cntrl": {}, "ctype_digit": {},
"ctype_graph": {}, "ctype_lower": {}, "ctype_print": {}, "ctype_punct": {},
"ctype_space": {}, "ctype_upper": {}, "ctype_xdigit": {},
}
// phpIsNonStringBuiltinCall reports whether expr is exactly one call to a
// builtin that never returns a string. isset/empty are language constructs
// and cannot be shadowed; other names are trusted only when fully qualified
// (\file_exists) or when the file cannot redirect unqualified names.
func phpIsNonStringBuiltinCall(expr string, trustUnqualified bool) bool {
i := 0
qualified := false
if i < len(expr) && expr[i] == '\\' {
qualified = true
i++
}
if i >= len(expr) || !isPHPIdentifierStart(expr[i]) {
return false
}
nameStart := i
for i < len(expr) && isPHPIdentifierPart(expr[i]) {
i++
}
name := strings.ToLower(expr[nameStart:i])
if _, ok := phpNonStringReturnBuiltins[name]; !ok {
return false
}
open := skipPHPWhitespace(expr, i)
if open >= len(expr) || expr[open] != '(' {
return false
}
if matchingParen(expr, open) != len(expr)-1 {
return false
}
if name == "isset" || name == "empty" {
return true
}
return qualified || trustUnqualified
}
// phpMayRedirectUnqualifiedBuiltin reports whether the source can make an
// unqualified trusted builtin name resolve to attacker-defined code: namespace
// fallback/shadowing, use-function imports, or same-file global polyfills for
// names that are builtins only on newer PHP versions. Closure captures
// (function () use ($x)), class methods, and trait use inside classes do not
// count.
func phpMayRedirectUnqualifiedBuiltin(code string) bool {
prev := byte(0) // last significant byte
classDepth := 0
var braceStack []bool
pendingClassScope := false
i := 0
for i < len(code) {
c := code[i]
if label, bodyStart, ok := phpHeredocOpen(code, i); ok {
i = phpHeredocEnd(code, bodyStart, label)
prev = '"'
continue
}
if isPHPQuote(c) {
i = skipPHPString(code, i) + 1
prev = '"'
continue
}
if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
i++
continue
}
switch c {
case '{':
braceStack = append(braceStack, pendingClassScope)
if pendingClassScope {
classDepth++
}
pendingClassScope = false
prev = c
i++
continue
case '}':
if len(braceStack) > 0 {
last := len(braceStack) - 1
if braceStack[last] && classDepth > 0 {
classDepth--
}
braceStack = braceStack[:last]
}
pendingClassScope = false
prev = c
i++
continue
case ';':
pendingClassScope = false
}
if !isPHPIdentifierStart(c) {
prev = c
i++
continue
}
start := i
for i < len(code) && isPHPIdentifierPart(code[i]) {
i++
}
word := strings.ToLower(code[start:i])
guarded := isPHPIdentifierPart(prev) || prev == '$' || prev == '>' || prev == ':' || prev == '\\'
prev = code[i-1]
if guarded {
continue
}
switch word {
case "class", "interface", "trait", "enum":
pendingClassScope = true
case "namespace":
return true
case "function":
if classDepth == 0 && phpFunctionDeclarationShadowsNonStringBuiltin(code, i) {
return true
}
case "use":
if classDepth > 0 {
continue
}
j := skipPHPWhitespace(code, i)
if j < len(code) && code[j] == '(' {
continue // closure capture
}
if phpUseStatementImportsFunction(code, j) {
return true
}
}
}
return false
}
func phpFunctionDeclarationShadowsNonStringBuiltin(code string, from int) bool {
i := skipPHPWhitespace(code, from)
if i < len(code) && code[i] == '&' {
i = skipPHPWhitespace(code, i+1)
}
if i >= len(code) || !isPHPIdentifierStart(code[i]) {
return false
}
start := i
for i < len(code) && isPHPIdentifierPart(code[i]) {
i++
}
_, shadows := phpNonStringReturnBuiltins[strings.ToLower(code[start:i])]
return shadows
}
// phpUseStatementImportsFunction scans one use statement (until the first
// semicolon) for the function keyword, covering both use function a\b and
// use a\{function b} forms.
func phpUseStatementImportsFunction(code string, from int) bool {
i := from
for i < len(code) && code[i] != ';' {
if isPHPQuote(code[i]) {
i = skipPHPString(code, i) + 1
continue
}
if !isPHPIdentifierStart(code[i]) {
i++
continue
}
start := i
for i < len(code) && isPHPIdentifierPart(code[i]) {
i++
}
if strings.ToLower(code[start:i]) == "function" {
return true
}
}
return false
}
// hasPregReplaceEvalWithRequest reports a preg_replace() call whose pattern
// carries the /e modifier and whose evaluated replacement or subject reads
// request input. The /e modifier (removed in PHP 7.0) evaluates the replacement
// as PHP after backreferences from the subject are interpolated, so pattern-only
// request input is not enough to call this a code-execution sink. Bare /e with
// static arguments is the legacy WordPress serialize-fix / autolink idiom --
// real /e usage, but not a dropper -- so it is gated on request-input
// correlation, the same way the shell, include, and assert detectors here gate
// their sinks.
// Walks the source skipping string literals so a documentation example in a
// string does not trip, then inspects the literal first argument of each call.
func hasPregReplaceEvalWithRequest(code string) bool {
searchFrom := 0
for {
callStart, openParen, closeParen, ok := nextStandalonePHPCall(code, searchFrom, pregReplaceCallName)
if !ok {
break
}
args := phpCallArguments(code, openParen+1, closeParen)
if len(args) < 3 || !pregPatternArgumentHasEvalModifier(args[0]) {
searchFrom = nextSearchOffset(closeParen, len(code))
continue
}
tainted := requestTaintedVariablesBefore(code, callStart)
if pregReplacementReadsRequest(args[1], tainted) || phpExpressionReadsRequest(args[2], tainted) {
return true
}
searchFrom = nextSearchOffset(closeParen, len(code))
}
return false
}
func pregPatternArgumentHasEvalModifier(expr string) bool {
expr = strings.TrimSpace(expr)
if expr == "" || !isPHPQuote(expr[0]) {
return false
}
end := skipPHPString(expr, 0)
// skipPHPString returns the closing-quote index, or the last index for an
// unterminated literal. Guard the slice: a quote in the final position
// leaves no string body to inspect.
if end <= 0 {
return false
}
return pregPatternHasEvalModifier(expr[1:end])
}
func pregReplacementReadsRequest(expr string, tainted map[string]struct{}) bool {
if phpExpressionReadsRequest(expr, tainted) {
return true
}
body, _, ok := phpStringLiteralExpression(expr)
if !ok {
return false
}
return phpExpressionReadsRequest(body, tainted)
}
func phpExpressionReadsRequest(expr string, tainted map[string]struct{}) bool {
return containsRequestSuperglobalExpression(expr) || phpExpressionReferencesTaintedVariable(expr, tainted)
}
func phpExpressionReferencesTaintedVariable(expr string, tainted map[string]struct{}) bool {
if len(tainted) == 0 {
return false
}
for i := 0; i < len(expr); i++ {
if isPHPQuote(expr[i]) {
i = skipPHPString(expr, i)
continue
}
if expr[i] != '$' {
continue
}
variable, next, ok := readPHPVariableName(expr, i)
if !ok {
continue
}
if _, found := tainted[variable]; found {
return true
}
i = next - 1
}
return false
}
func requestTaintedVariablesBefore(code string, limit int) map[string]struct{} {
if limit > len(code) {
limit = len(code)
}
if limit <= 0 {
return nil
}
scan := code[:limit]
taintStack := []map[string]struct{}{{}}
functionBraceStack := []bool{}
pendingFunctionScope := false
for i := 0; i < len(scan); i++ {
if isPHPQuote(scan[i]) {
i = skipPHPString(scan, i)
continue
}
if end, ok := phpKeywordAt(scan, i, "function"); ok {
pendingFunctionScope = true
i = end - 1
continue
}
switch scan[i] {
case '{':
functionBraceStack = append(functionBraceStack, pendingFunctionScope)
if pendingFunctionScope {
taintStack = append(taintStack, map[string]struct{}{})
}
pendingFunctionScope = false
continue
case '}':
if len(functionBraceStack) > 0 {
last := len(functionBraceStack) - 1
if functionBraceStack[last] && len(taintStack) > 1 {
taintStack = taintStack[:len(taintStack)-1]
}
functionBraceStack = functionBraceStack[:last]
}
pendingFunctionScope = false
continue
case ';':
pendingFunctionScope = false
}
if scan[i] != '$' {
continue
}
variable, next, ok := readPHPVariableName(scan, i)
if !ok || isRequestSuperglobalVariable(variable) {
continue
}
j := skipPHPWhitespace(scan, next)
opLen, directAssign, appendAssign, ok := phpAssignmentOperator(scan, j)
if !ok {
i = next - 1
continue
}
exprStart := skipPHPWhitespace(scan, j+opLen)
exprEnd := phpExpressionEnd(scan, exprStart)
expr := scan[exprStart:exprEnd]
tainted := taintStack[len(taintStack)-1]
exprTainted := phpExpressionReadsRequest(expr, tainted)
switch {
case directAssign:
if exprTainted {
tainted[variable] = struct{}{}
} else {
delete(tainted, variable)
}
case appendAssign:
if exprTainted {
tainted[variable] = struct{}{}
}
default:
if exprTainted {
tainted[variable] = struct{}{}
} else {
delete(tainted, variable)
}
}
i = exprEnd - 1
}
return taintStack[len(taintStack)-1]
}
func phpKeywordAt(code string, start int, keyword string) (int, bool) {
end := start + len(keyword)
if end > len(code) || !strings.EqualFold(code[start:end], keyword) {
return 0, false
}
if start > 0 {
prev := code[start-1]
if isPHPIdentifierPart(prev) || prev == '$' || prev == '>' || prev == ':' || prev == '\\' {
return 0, false
}
}
if end < len(code) && isPHPIdentifierPart(code[end]) {
return 0, false
}
return end, true
}
func isRequestSuperglobalVariable(variable string) bool {
switch strings.ToLower(variable) {
case "_request", "_post", "_get", "_cookie", "_server":
return true
default:
return false
}
}
func phpStringLiteralExpression(expr string) (string, byte, bool) {
expr = strings.TrimSpace(expr)
if expr == "" || !isPHPQuote(expr[0]) {
return "", 0, false
}
end := skipPHPString(expr, 0)
if end <= 0 || end >= len(expr) || expr[end] != expr[0] {
return "", 0, false
}
if skipPHPWhitespace(expr, end+1) != len(expr) {
return "", 0, false
}
return expr[1:end], expr[0], true
}
func phpCallArguments(code string, start, end int) []string {
if start < 0 {
start = 0
}
if end > len(code) {
end = len(code)
}
if start > end {
return nil
}
var args []string
argStart := skipPHPWhitespace(code, start)
depth := 0
for i := start; i < end; i++ {
if label, bodyStart, ok := phpHeredocOpen(code, i); ok {
i = phpHeredocEnd(code, bodyStart, label) - 1
continue
}
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
switch code[i] {
case '(', '[', '{':
depth++
case ')', ']', '}':
if depth > 0 {
depth--
}
case ',':
if depth == 0 {
args = append(args, strings.TrimSpace(code[argStart:i]))
argStart = skipPHPWhitespace(code, i+1)
}
}
}
if tail := strings.TrimSpace(code[argStart:end]); tail != "" || len(args) > 0 {
args = append(args, tail)
}
return args
}
// pregPatternHasEvalModifier returns true when the PCRE pattern string carries
// an "e" modifier after its closing delimiter.
func pregPatternHasEvalModifier(pat string) bool {
if len(pat) < 2 {
return false
}
open := pat[0]
// PHP forbids alphanumeric, backslash, and whitespace delimiters.
if isPHPIdentifierPart(open) || open == '\\' || open == ' ' {
return false
}
closeDelim := open
switch open {
case '(':
closeDelim = ')'
case '[':
closeDelim = ']'
case '{':
closeDelim = '}'
case '<':
closeDelim = '>'
}
idx := strings.LastIndexByte(pat, closeDelim)
if idx <= 0 {
return false
}
for _, c := range pat[idx+1:] {
if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') {
return false
}
if c == 'e' {
return true
}
}
return false
}
func nextIncludeExpression(line string, searchFrom int) (int, int, bool) {
for i := searchFrom; i < len(line); i++ {
if isPHPQuote(line[i]) {
i = skipPHPString(line, i)
continue
}
keywordEnd, ok := includeKeywordEnd(line, i)
if !ok {
continue
}
exprStart := skipPHPWhitespace(line, keywordEnd)
return exprStart, phpExpressionEnd(line, exprStart), true
}
return 0, 0, false
}
func includeKeywordEnd(line string, start int) (int, bool) {
if !canStartIncludeKeyword(line, start) {
return 0, false
}
for _, keyword := range includeKeywords {
end := start + len(keyword)
if end > len(line) || !strings.EqualFold(line[start:end], keyword) {
continue
}
if end < len(line) && isPHPIdentifierPart(line[end]) {
continue
}
if precededByFunctionKeyword(line, start) {
continue
}
return end, true
}
return 0, false
}
func canStartIncludeKeyword(line string, start int) bool {
if start == 0 {
return true
}
prev := line[start-1]
return !isPHPIdentifierPart(prev) && prev != '$' && prev != '>' && prev != ':' && prev != '\\'
}
// precededByFunctionKeyword reports whether the token before start is the
// `function` keyword, i.e. start begins a declared function or method name
// rather than a language construct. PHP 7 allows a reserved word as a method
// name, and a bootstrap method called include() is a common plugin idiom;
// treating that declaration as an include statement makes the method body its
// target expression, so any request input inside the method reads as an
// include of request input.
func precededByFunctionKeyword(line string, start int) bool {
const kw = "function"
// Only inspect the separator after a keyword match, never at each byte
// offset. Separators have no length limit in PHP; each whitespace run is
// visited at most once per matched keyword, keeping the full scan linear.
end := skipPHPWhitespaceBack(line, 0, start)
if end > 0 && line[end-1] == '&' {
end = skipPHPWhitespaceBack(line, 0, end-1)
}
begin := end - len(kw)
return begin >= 0 && strings.EqualFold(line[begin:end], kw) && canStartIncludeKeyword(line, begin)
}
func phpExpressionEnd(code string, start int) int {
depth := 0
for i := start; i < len(code); i++ {
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
switch code[i] {
case '(', '[', '{':
depth++
case ')', ']', '}':
if depth == 0 {
return i
}
depth--
case ';':
if depth == 0 {
return i
}
case ',':
if depth == 0 {
return i
}
}
}
return len(code)
}
// PHPExecutableCode returns src with inline HTML and PHP comments blanked,
// keeping byte offsets and line breaks, so line-oriented detectors see only
// code PHP would run. src must begin at the start of a file: a window cut from
// the middle can open inside a string or comment and shift which bytes count
// as code.
func PHPExecutableCode(src string) string {
return stripPHPCommentsFromCode(phpCodeOnly(src))
}
// phpCodeOnly blanks the inline-HTML regions of a PHP source so only the code
// inside <?php ... ?> (and <?= ... ?>) spans is analysed for execution sinks.
// Inline HTML is literal output and cannot execute PHP, so scanning it as code
// only yields false positives: apostrophes in prose ("don't", "you're") desync
// the string scanner, and href URLs, JS backtick template literals, and English
// words like "include"/"require" in markup then read as PHP execution sinks.
//
// HTML bytes become spaces (newlines kept so line-oriented detectors keep their
// line structure); a closing "?>" becomes "; " so it still bounds the preceding
// statement and consecutive <?php?> blocks do not run their expressions
// together. The "?>" scan skips PHP strings, heredoc/nowdoc bodies, and block
// comments so a "?>" inside them does not end PHP mode and blank real code. A
// file with no PHP open tag yields all blanks -- it executes nothing.
func phpCodeOnly(src string) string {
return canonicalSuperglobalAliases(phpCodeRegions(src))
}
// phpCodeRegions preserves PHP bytes for callers that use the result as a mask.
func phpCodeRegions(src string) string {
var b strings.Builder
b.Grow(len(src))
n := len(src)
i := 0
for i < n {
// HTML mode: blank up to the next "<?" open tag.
htmlStart := i
for i < n && (src[i] != '<' || i+1 >= n || src[i+1] != '?') {
i++
}
blankInlineHTML(&b, src[htmlStart:i])
if i >= n {
break
}
// Blank the opening tag: "<?php" (needs trailing whitespace/EOF), "<?=",
// or a bare "<?" short tag.
i += 2
b.WriteString(" ")
if i+3 <= n && strings.EqualFold(src[i:i+3], "php") && (i+3 == n || isPHPSpace(src[i+3])) {
b.WriteString(" ")
i += 3
} else if i < n && src[i] == '=' {
b.WriteByte(' ')
i++
}
// PHP mode: copy verbatim until a top-level "?>".
i = copyPHPModeRegion(&b, src, i)
}
return b.String()
}
// A block comment must end at its first terminator; letting it cross that
// terminator can turn a concatenated variable name into a superglobal.
const phpGapRe = `(?:[ \t\r\n\v\f]|/\*[^*]*\*+(?:[^/*][^*]*\*+)*/|(?://|#)[^\r\n]*(?:\r\n|[\r\n]))*`
const superglobalNameRe = `('_(?:GET|POST|REQUEST|COOKIE|FILES|SERVER|ENV|SESSION)'|"_(?:GET|POST|REQUEST|COOKIE|FILES|SERVER|ENV|SESSION)")`
var reSuperglobalAlias = regexp.MustCompile(
`^(?:\$\{` + phpGapRe + superglobalNameRe + phpGapRe + `\}` +
`|\$GLOBALS` + phpGapRe + `\[` + phpGapRe + superglobalNameRe + phpGapRe + `\])`)
// canonicalSuperglobalAliases exposes literal superglobal aliases to the
// request-input heuristics. Literal data and comments stay unchanged, including
// escaped dollars and nowdocs. Replacements retain byte and line-break offsets.
func canonicalSuperglobalAliases(code string) string {
if !strings.Contains(code, "${") && !strings.Contains(code, "$GLOBALS") {
return code
}
// A brace frame is executable interpolation inside a quoted string. A
// heredoc frame ends at a known offset, not at a quote in its body.
type mode struct {
quote byte
end int
}
modes := []mode{{}}
var out []byte
for i := 0; i < len(code); {
current := modes[len(modes)-1]
if current.end != 0 && i >= current.end {
modes = modes[:len(modes)-1]
continue
}
inString := current.quote != 0 && current.quote != '{'
if inString {
switch {
case code[i] == '\\':
i += 2
continue
case current.end == 0 && code[i] == current.quote:
modes = modes[:len(modes)-1]
i++
continue
case strings.HasPrefix(code[i:], "{$"):
modes = append(modes, mode{quote: '{'})
i++
continue
case !strings.HasPrefix(code[i:], "${"):
i++
continue
}
} else {
if label, body, ok := phpHeredocOpen(code, i); ok {
end := phpHeredocEnd(code, body, label)
if strings.Contains(code[i:body], "'") {
i = end
} else {
modes = append(modes, mode{quote: '\n', end: end})
i = body
}
continue
}
if code[i] == '\'' {
i = skipPHPString(code, i) + 1
continue
}
if code[i] == '"' || code[i] == '`' {
modes = append(modes, mode{quote: code[i]})
i++
continue
}
if strings.HasPrefix(code[i:], "/*") {
end := strings.Index(code[i+2:], "*/")
if end < 0 {
break
}
i += end + 4
continue
}
if isPHPLineCommentStart(code, i) {
i = skipPHPLineComment(code, i)
continue
}
if current.quote == '{' {
switch code[i] {
case '{':
modes = append(modes, mode{quote: '{'})
case '}':
modes = modes[:len(modes)-1]
}
}
}
if code[i] == '$' && (strings.HasPrefix(code[i:], "${") || strings.HasPrefix(code[i:], "$GLOBALS")) {
if m := reSuperglobalAlias.FindStringSubmatchIndex(code[i:]); m != nil {
nameStart, nameEnd := m[2], m[3]
if nameStart < 0 {
nameStart, nameEnd = m[4], m[5]
}
name := "$" + code[i+nameStart+1:i+nameEnd-1]
if out == nil {
out = []byte(code)
}
start, end := i, i+m[1]
at := end - len(name)
if len(modes) > 1 {
// Retain interpolation delimiters so later string masking
// cannot mistake nested quotes for the outer closing quote.
start, end = i+nameStart, i+nameEnd
at = start
} else {
for strings.ContainsAny(code[at:at+len(name)], "\r\n") {
at--
}
}
for j := start; j < end; j++ {
if out[j] != '\n' && out[j] != '\r' {
out[j] = ' '
}
}
copy(out[at:], name)
i += m[1]
continue
}
if inString {
modes = append(modes, mode{quote: '{'})
i += 2
continue
}
}
i++
}
if out == nil {
return code
}
return string(out)
}
// copyPHPModeRegion copies src[start:] into b verbatim until a top-level "?>"
// (which it replaces with "; ") or EOF, and returns the resume index. Strings,
// heredoc/nowdoc bodies, and block comments are copied whole so a "?>" inside
// them is not mistaken for a closing tag. A "?>" inside a // or # line comment
// does end PHP mode, matching PHP's own tokeniser.
func copyPHPModeRegion(b *strings.Builder, src string, start int) int {
n := len(src)
i := start
for i < n {
if label, bodyStart, ok := phpHeredocOpen(src, i); ok {
end := phpHeredocEnd(src, bodyStart, label)
b.WriteString(src[i:end])
i = end
continue
}
if isPHPQuote(src[i]) || src[i] == '`' {
i = copyPHPString(b, src, i) + 1
continue
}
if src[i] == '/' && i+1 < n && src[i+1] == '*' {
b.WriteString("/*")
i += 2
for i < n {
if src[i] == '*' && i+1 < n && src[i+1] == '/' {
b.WriteString("*/")
i += 2
break
}
b.WriteByte(src[i])
i++
}
continue
}
if isPHPLineCommentStart(src, i) {
end := skipPHPLineComment(src, i)
if end+1 < n && src[end] == '?' && src[end+1] == '>' {
// Code after the tag lands on this same output line, so the
// comment text must go: a later comment strip would run past
// the replaced tag and blank that code.
b.WriteString(strings.Repeat(" ", end-i))
b.WriteString("; ")
return end + 2
}
b.WriteString(src[i:end])
i = end
continue
}
if src[i] == '?' && i+1 < n && src[i+1] == '>' {
b.WriteString("; ")
return i + 2
}
b.WriteByte(src[i])
i++
}
return i
}
// blankInlineHTML writes s as spaces, preserving newlines so line-based
// detectors keep their line boundaries across blanked template text.
func blankInlineHTML(b *strings.Builder, s string) {
for i := 0; i < len(s); i++ {
if s[i] == '\n' || s[i] == '\r' {
b.WriteByte(s[i])
} else {
b.WriteByte(' ')
}
}
}
func nextStandalonePHPCall(code string, searchFrom int, names map[string]struct{}) (int, int, int, bool) {
for i := searchFrom; i < len(code); i++ {
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
nameStart := i
if code[i] == '\\' {
if i+1 >= len(code) || !isPHPIdentifierStart(code[i+1]) || !canStartGlobalPHPFunction(code, i) {
continue
}
nameStart = i + 1
} else if !isPHPIdentifierStart(code[i]) || !canStartPHPFunctionName(code, i) {
continue
}
nameEnd := nameStart + 1
for nameEnd < len(code) && isPHPIdentifierPart(code[nameEnd]) {
nameEnd++
}
if _, ok := names[strings.ToLower(code[nameStart:nameEnd])]; !ok {
i = nameEnd - 1
continue
}
openParen := skipPHPWhitespace(code, nameEnd)
if openParen >= len(code) || code[openParen] != '(' {
i = nameEnd - 1
continue
}
return i, openParen, matchingParen(code, openParen), true
}
return 0, 0, 0, false
}
func nextSearchOffset(pos, codeLen int) int {
if pos >= codeLen {
return codeLen
}
return pos + 1
}
var requestSuperglobalNames = []string{"$_request", "$_post", "$_get", "$_cookie", "$_server"}
// includeTargetSuperglobalNames omits $_SERVER path keys: including a path
// built from the server document root or script filename is the standard
// WordPress bootstrap idiom, not an LFI/RFI primitive. Header-derived
// $_SERVER keys are handled separately below.
var includeTargetSuperglobalNames = []string{"$_request", "$_post", "$_get", "$_cookie"}
func containsRequestSuperglobal(code string) bool {
return containsAnySuperglobal(code, requestSuperglobalNames)
}
func containsAnySuperglobal(code string, names []string) bool {
code = strings.ToLower(code)
for _, requestVar := range names {
searchFrom := 0
for {
pos := strings.Index(code[searchFrom:], requestVar)
if pos < 0 {
break
}
end := searchFrom + pos + len(requestVar)
if end >= len(code) || !isPHPIdentifierPart(code[end]) {
return true
}
searchFrom = end
}
}
return false
}
// PHP single-quoted strings do not interpolate variables, but double-quoted
// strings do. The decoder callback gate needs that distinction so a literal
// '$_POST' data value does not look like request input.
func containsRequestSuperglobalExpression(code string) bool {
return containsSuperglobalExpression(code, requestSuperglobalNames)
}
// containsIncludeTargetExpression reports whether an include/require target
// expression reads an attacker-controlled superglobal. Non-header $_SERVER
// path keys are left alone (see includeTargetSuperglobalNames).
func containsIncludeTargetExpression(code string) bool {
return containsSuperglobalExpression(code, includeTargetSuperglobalNames) ||
containsServerHeaderIncludeExpression(code)
}
func containsSuperglobalExpression(code string, names []string) bool {
start := 0
for i := 0; i < len(code); i++ {
if !isPHPQuote(code[i]) {
continue
}
if containsAnySuperglobal(code[start:i], names) {
return true
}
end := skipPHPString(code, i)
if code[i] == '"' && containsAnySuperglobal(code[i:end+1], names) {
return true
}
i = end
start = end + 1
}
return containsAnySuperglobal(code[start:], names)
}
func containsServerHeaderIncludeExpression(code string) bool {
for i := 0; i < len(code); i++ {
if isPHPQuote(code[i]) {
end := skipPHPString(code, i)
if code[i] == '"' && end > i && containsServerHeaderReference(code[i+1:end]) {
return true
}
i = end
continue
}
dangerous, next, ok := serverIncludeReferenceAt(code, i)
if !ok {
continue
}
if dangerous {
return true
}
i = next - 1
}
return false
}
func containsServerHeaderReference(code string) bool {
for i := 0; i < len(code); i++ {
dangerous, next, ok := serverIncludeReferenceAt(code, i)
if !ok {
continue
}
if dangerous {
return true
}
i = next - 1
}
return false
}
func serverIncludeReferenceAt(code string, start int) (bool, int, bool) {
const serverName = "$_server"
end := start + len(serverName)
if end > len(code) || !strings.EqualFold(code[start:end], serverName) {
return false, start, false
}
if end < len(code) && isPHPIdentifierPart(code[end]) {
return false, start, false
}
bracket := skipPHPWhitespace(code, end)
if bracket >= len(code) || code[bracket] != '[' {
return true, end, true
}
keyStart := skipPHPWhitespace(code, bracket+1)
if keyStart >= len(code) {
return true, len(code), true
}
if isPHPQuote(code[keyStart]) {
keyEnd := skipPHPString(code, keyStart)
key := phpStringLiteralValue(code, keyStart, keyEnd)
next := skipPHPWhitespace(code, keyEnd+1)
if next < len(code) && code[next] == ']' {
return isAttackerControlledServerKey(key), next + 1, true
}
return true, next, true
}
if isPHPIdentifierStart(code[keyStart]) {
keyEnd := keyStart + 1
for keyEnd < len(code) && isPHPIdentifierPart(code[keyEnd]) {
keyEnd++
}
key := code[keyStart:keyEnd]
next := skipPHPWhitespace(code, keyEnd)
if next < len(code) && code[next] == ']' {
return isAttackerControlledServerKey(key), next + 1, true
}
return true, next, true
}
return true, keyStart + 1, true
}
func phpStringLiteralValue(code string, start, end int) string {
var b strings.Builder
quote := code[start]
for i := start + 1; i < end; i++ {
if code[i] != '\\' || i+1 >= end {
b.WriteByte(code[i])
continue
}
i++
esc := code[i]
if quote == '\'' {
if esc == '\'' || esc == '\\' {
b.WriteByte(esc)
} else {
b.WriteByte('\\')
b.WriteByte(esc)
}
continue
}
switch esc {
case 'x':
if i+1 >= end || !isHexDigit(code[i+1]) {
b.WriteByte('\\')
b.WriteByte(esc)
continue
}
value := hexVal(code[i+1])
i++
if i+1 < end && isHexDigit(code[i+1]) {
value = value*16 + hexVal(code[i+1])
i++
}
// #nosec G115 -- PHP hex string escapes are at most one byte.
b.WriteByte(byte(value))
case 'u':
if i+2 >= end || code[i+1] != '{' || !isHexDigit(code[i+2]) {
b.WriteByte('\\')
b.WriteByte(esc)
continue
}
value := 0
j := i + 2
for ; j < end && isHexDigit(code[j]); j++ {
value = value*16 + hexVal(code[j])
}
if j >= end || code[j] != '}' {
b.WriteByte('\\')
b.WriteByte(esc)
continue
}
if value > 0x10ffff {
b.WriteByte('\\')
b.WriteByte(esc)
continue
}
// #nosec G115 -- value is capped at the largest valid Unicode code point.
b.WriteRune(rune(value))
i = j
case '0', '1', '2', '3', '4', '5', '6', '7':
value := int(esc - '0')
digits := 1
for i+1 < end && digits < 3 && code[i+1] >= '0' && code[i+1] <= '7' {
value = value*8 + int(code[i+1]-'0')
i++
digits++
}
// #nosec G115 -- PHP octal string escapes are byte escapes.
b.WriteByte(byte(value))
case 'n':
b.WriteByte('\n')
case 'r':
b.WriteByte('\r')
case 't':
b.WriteByte('\t')
case 'v':
b.WriteByte('\v')
case 'e':
b.WriteByte(0x1b)
case 'f':
b.WriteByte('\f')
case '\\', '$', '"':
b.WriteByte(esc)
default:
b.WriteByte('\\')
b.WriteByte(esc)
}
}
return b.String()
}
func isAttackerControlledServerKey(key string) bool {
key = strings.ToUpper(strings.TrimSpace(key))
if strings.HasPrefix(key, "HTTP_") {
return true
}
switch key {
case "CONTENT_LENGTH", "CONTENT_TYPE", "PHP_AUTH_DIGEST", "PHP_AUTH_PW", "PHP_AUTH_USER":
return true
default:
return false
}
}
func canStartGlobalPHPFunction(code string, slash int) bool {
if slash == 0 {
return true
}
prev := code[slash-1]
return !isPHPIdentifierPart(prev) && prev != '$' && prev != '>' && prev != ':' && prev != '\\'
}
func canStartPHPFunctionName(code string, start int) bool {
if start == 0 {
return true
}
prev := code[start-1]
return !isPHPIdentifierPart(prev) && prev != '$' && prev != '>' && prev != ':' && prev != '\\'
}
// CheckPHPContent scans new/suspicious PHP files for obfuscation patterns,
// remote payload fetching, and eval chains. This is designed to catch droppers
// like the LEVIATHAN attack's file.php and files.php that use goto spaghetti,
// hex-encoded strings, and call_user_func with string-built function names.
//
// This check scans PHP files in directories that shouldn't normally contain
// user-authored PHP: wp-content/languages, wp-content/upgrade, wp-content/mu-plugins,
// and also checks any PHP files flagged by the file index as new.
// phpFileStamp is the cheap content-version key for a scanned PHP file. A file
// whose stamp matches the previous cycle is treated as unchanged.
type phpFileStamp struct {
Mtime int64 `json:"m"`
Size int64 `json:"s"`
// Unlike mtime, change time cannot be restored by a file's owner.
Dev uint64 `json:"d,omitempty"`
Inode uint64 `json:"i,omitempty"`
Ctime int64 `json:"c,omitempty"`
}
func phpFileStampOf(info os.FileInfo) phpFileStamp {
stamp := phpFileStamp{Mtime: info.ModTime().Unix(), Size: info.Size()}
if id, ok := selfWriteIdentityFromFileInfo(info); ok {
stamp.Dev, stamp.Inode = id.Device, id.Inode
stamp.Ctime = id.ChangeSec*1_000_000_000 + id.ChangeNsec
}
return stamp
}
// phpContentNow is indirected so tests can treat fixtures they just wrote as
// settled without sleeping out the change-time window.
var phpContentNow = time.Now
func (stamp phpFileStamp) cacheableAt(start time.Time) bool {
// Missing identity must fail closed, including legacy mtime+size entries.
// A recent read cannot authorize future skips: Linux can give a later
// write the same coarse ctime. Wait a full second before the clean read,
// also covering filesystems with whole-second timestamps. Do not sleep in
// the scan; a later visit will read the file again and can then cache it.
return stamp.Inode != 0 && stamp.Ctime != 0 &&
time.Unix(0, stamp.Ctime).Before(start.Add(-time.Second))
}
// phpContentCache maps a file path to the stamp it carried when last confirmed
// clean. Only clean files are stored, so a present, matching entry means
// "unchanged and previously produced no finding."
type phpContentCache map[string]phpFileStamp
func loadPHPContentCache(stateDir string) phpContentCache {
cache := phpContentCache{}
if stateDir == "" {
return cache
}
data, err := osFS.ReadFile(filepath.Join(stateDir, "phpcontentcache.json"))
if err == nil {
_ = json.Unmarshal(data, &cache)
}
return cache
}
func savePHPContentCache(stateDir string, cache phpContentCache) {
if stateDir == "" {
return
}
data, _ := json.Marshal(cache)
tmpPath := filepath.Join(stateDir, "phpcontentcache.json.tmp")
_ = os.WriteFile(tmpPath, data, 0600)
_ = os.Rename(tmpPath, filepath.Join(stateDir, "phpcontentcache.json"))
}
// A timed-out check can outlive the runner's drain grace. Only the latest
// host scan may publish its snapshot; an older scan must not restore clean
// stamps that its successor invalidated. Account and audit scans do not write
// this cache and therefore do not take ownership away from a host scan.
var phpContentCacheWriter struct {
sync.Mutex
generation uint64
}
func beginPHPContentCacheRun() uint64 {
phpContentCacheWriter.Lock()
defer phpContentCacheWriter.Unlock()
phpContentCacheWriter.generation++
return phpContentCacheWriter.generation
}
func savePHPContentCacheRun(stateDir string, scan *phpContentScan, generation uint64) {
phpContentCacheWriter.Lock()
defer phpContentCacheWriter.Unlock()
if generation == phpContentCacheWriter.generation {
savePHPContentCache(stateDir, scan.merged())
}
}
// phpContentHostScanCount drives a periodic forced full rescan that bypasses
// the content cache, mirroring the file-index cadence. This remains a backstop
// for filesystems whose metadata does not reliably distinguish content changes.
var phpContentHostScanCount int32
// phpContentAccountScanCount keeps account-scoped scans from consuming the
// host-wide full-rescan cadence. Account scans are subsets and never save the
// shared cache, so mixing the counters can make the host-wide backstop miss its
// intended cycle.
var phpContentAccountScanCount int32
func phpContentForceFull(ctx context.Context) bool {
if AccountFromContext(ctx) != "" {
return atomic.AddInt32(&phpContentAccountScanCount, 1)%6 == 0
}
return atomic.AddInt32(&phpContentHostScanCount, 1)%6 == 0
}
// phpContentScan carries the per-cycle cache state through the recursive walk.
// prev holds prior clean stamps not yet invalidated by this run; next holds
// files confirmed clean during this run.
type phpContentScan struct {
cfg *config.Config
prev phpContentCache
next phpContentCache
forceFull bool
// visited holds directories whose full entry list this run read. Their
// prior stamps are authoritative-by-absence: a cached file the walk did
// not see again is gone. Everything else is simply unreached.
visited map[string]bool
// attempted records whether each attempted content read succeeded.
attempted map[string]bool
}
func newPHPContentScan(cfg *config.Config, prev phpContentCache, forceFull bool) *phpContentScan {
if prev == nil {
prev = phpContentCache{}
}
return &phpContentScan{
cfg: cfg,
prev: prev,
next: phpContentCache{},
forceFull: forceFull,
visited: map[string]bool{},
attempted: map[string]bool{},
}
}
// merged is the cache to persist after a run that may have been cut short. It
// keeps every stamp confirmed this cycle and carries forward prior stamps from
// directories the run never finished reading. A run that walked a directory
// prunes what vanished from it; a run that never reached one leaves its files
// cached so the next cycle does not re-read the whole host from scratch.
func (s *phpContentScan) merged() phpContentCache {
out := make(phpContentCache, len(s.next)+len(s.prev))
for path, stamp := range s.prev {
// The periodic forced pass must also expire stamps outside its fixed
// directories, so rolling coverage re-reads them on its next visit.
if s.forceFull || s.visited[filepath.Dir(path)] {
continue
}
out[path] = stamp
}
for path, stamp := range s.next {
out[path] = stamp
}
return out
}
// pruneMissing drops cached stamps under roots for paths that are no longer on
// disk. Enumeration can omit unreadable directories and remapped extensions,
// so absence from its list alone is not proof that a cached path is gone.
func (s *phpContentScan) pruneMissing(roots []string, present []string) {
live := make(map[string]struct{}, len(present))
for _, path := range present {
live[path] = struct{}{}
}
for path := range s.prev {
if _, ok := live[path]; ok {
continue
}
for _, root := range roots {
if strings.HasPrefix(path, root+string(filepath.Separator)) {
if _, err := osFS.Stat(path); os.IsNotExist(err) {
delete(s.prev, path)
}
break
}
}
}
}
// accountDocRoots lists an account's document roots: public_html plus the
// addon-domain directories beside it, skipping the mail, control and temp
// directories that never serve web content.
func accountDocRoots(homeDir string) []string {
roots, _ := accountDocRootsComplete(homeDir)
return roots
}
func accountDocRootsComplete(homeDir string) ([]string, bool) {
docRoots := []string{filepath.Join(homeDir, "public_html")}
subDirs, err := osFS.ReadDir(homeDir)
for _, sd := range subDirs {
if sd.IsDir() && sd.Name() != "public_html" && accountContentDir(sd.Name()) {
docRoots = append(docRoots, filepath.Join(homeDir, sd.Name()))
}
}
return docRoots, err == nil || os.IsNotExist(err)
}
func CheckPHPContent(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
homeDirs, err := GetScanHomeDirs(ctx)
if err != nil {
return nil
}
forcedFull := phpContentForceFull(ctx) || scanForceContent(ctx)
persistCache := AccountFromContext(ctx) == "" && !scanForceContent(ctx)
var cacheGeneration uint64
if persistCache {
cacheGeneration = beginPHPContentCacheRun()
}
scan := newPHPContentScan(cfg, loadPHPContentCache(cfg.StatePath), forcedFull)
// A run cut short still falls through to the cache write below, so the
// stamps it confirmed are not thrown away with the rest of the cycle.
accounts:
for _, homeEntry := range homeDirs {
if ctx.Err() != nil {
break
}
if !homeEntry.IsDir() {
continue
}
homeDir := scanHomeDirPath(homeEntry)
docRoots := accountDocRoots(homeDir)
for _, docRoot := range docRoots {
// Scan directories that shouldn't contain user PHP
suspiciousDirs := []string{
filepath.Join(docRoot, "wp-content", "languages"),
filepath.Join(docRoot, "wp-content", "upgrade"),
filepath.Join(docRoot, "wp-content", "mu-plugins"),
filepath.Join(docRoot, "wp-content", "plugins"),
filepath.Join(docRoot, "wp-content", "themes"),
}
for _, dir := range suspiciousDirs {
scan.scanDir(ctx, dir, 4, phpHandlerOverlay{}, &findings)
if ctx.Err() != nil {
break accounts
}
}
}
}
if rollingContentEnabled(ctx, cfg, forcedFull) {
rollingContentPass(ctx, cfg, scan, homeDirs, &findings)
}
// Persist on every host-wide run, including one cut short by the check
// budget: scan.merged() carries forward the stamps for directories this run
// never read, so a host too large to finish in one cycle still makes
// progress instead of re-reading everything next cycle. An account-scoped
// run (account_scan) only walks one account, so its stamps must not
// overwrite the host-wide cache. A forced-content scan (ForceContent=true)
// re-reads every file regardless of the cache, so scan.next reflects only
// the files visited this run and must not overwrite the host-wide live cache.
if persistCache {
savePHPContentCacheRun(cfg.StatePath, scan, cacheGeneration)
}
return findings
}
// scanDirForObfuscatedPHP scans dir without the content cache: every PHP file
// is read and analysed. Used where caching does not apply (no prior cycle to
// compare against).
func scanDirForObfuscatedPHP(ctx context.Context, dir string, maxDepth int, cfg *config.Config, findings *[]alert.Finding) {
newPHPContentScan(cfg, nil, true).scanDir(ctx, dir, maxDepth, phpHandlerOverlay{}, findings)
}
// scanDir recursively scans dir for PHP files with malicious content patterns.
// A file that was clean last cycle and has an unchanged stable stamp skips the
// read+parse, unless this is a forced full rescan. Files that produce a finding
// are never cached, so they re-surface on every cycle for the alert pipeline.
func (s *phpContentScan) scanDir(ctx context.Context, dir string, maxDepth int, overlay phpHandlerOverlay, findings *[]alert.Finding) {
if ctx.Err() != nil {
return
}
if maxDepth <= 0 {
return
}
entries, err := osFS.ReadDir(dir)
if err != nil {
// A directory that is gone has no files left to cache. One we cannot
// read may still hold them, so its stamps stay.
if os.IsNotExist(err) {
s.visited[dir] = true
}
return
}
// Layer this directory's .htaccess PHP handler remappings onto the set
// inherited from parent directories. An attacker who maps a non-PHP
// extension to the PHP interpreter (the LEVIATHAN .htaccess trick) or who
// SetHandlers the whole directory must not be able to hide executable PHP
// behind a name the stock handler would not run. Read once per directory.
if htaccess, herr := osFS.ReadFile(filepath.Join(dir, ".htaccess")); herr == nil {
overlay = overlay.mergeHtaccess(htaccess)
}
for _, entry := range entries {
if ctx.Err() != nil {
return
}
name := entry.Name()
fullPath := filepath.Join(dir, name)
// Check suppressed paths (bypassed for explicit full-scan / audit requests).
suppressed := false
if scanRespectsIgnores(ctx, s.cfg) {
for _, ignore := range s.cfg.Suppressions.IgnorePaths {
if matchGlob(fullPath, ignore) {
suppressed = true
break
}
}
}
if suppressed {
continue
}
if entry.IsDir() {
s.scanDir(ctx, fullPath, maxDepth-1, overlay, findings)
continue
}
s.scanFile(ctx, fullPath, overlay, findings)
}
s.visited[dir] = true
}
// scanFile content-analyses one PHP file using the same cache, size-guard, and
// finding logic as scanDir's per-entry loop. It re-stats fullPath (it no longer
// has the DirEntry) and decides whether it is PHP source or executes under the
// overlay. Files that produce a finding are never cached, so they re-surface
// each cycle for the alert pipeline. scanDir calls this for every non-directory
// entry; the rolling driver calls it for each path in its bounded slice.
func (s *phpContentScan) scanFile(ctx context.Context, fullPath string, overlay phpHandlerOverlay, findings *[]alert.Finding) (readOK bool) {
// Once attempted, only a fresh clean result may retain a stamp. Rolling
// and interrupted directory walks cannot rely on directory completion to
// invalidate a finding, read failure, or an earlier read in the same run.
s.attempted[fullPath] = false
previous, cached := s.prev[fullPath]
delete(s.next, fullPath)
defer func() {
s.attempted[fullPath] = readOK
if _, clean := s.next[fullPath]; !clean {
delete(s.prev, fullPath)
}
}()
nameLower := strings.ToLower(filepath.Base(fullPath))
if !contenttype.IsPHPSourceName(nameLower) && !overlay.executes(nameLower) {
return
}
started := phpContentNow()
info, statErr := osFS.Stat(fullPath)
var stamp phpFileStamp
canCache := statErr == nil
if canCache {
stamp = phpFileStampOf(info)
canCache = stamp.cacheableAt(started)
}
// Full-scan file-size guard: when the caller sets a per-file byte cap
// (MaxFileBytes > 0), skip content analysis for oversized files and
// record a job-scoped warning instead. The warning is persisted in
// scan_job_findings by the scan-job manager and never dispatched to the
// live alert pipeline. Normal scheduled scans set MaxFileBytes=0 and
// reach this block unchanged.
if limit := scanMaxFileBytes(ctx); limit > 0 && info != nil && info.Size() > limit {
*findings = append(*findings, alert.Finding{
Severity: alert.Warning,
Check: "full_scan_file_too_large",
Message: fmt.Sprintf("Full scan skipped oversized file: %s", fullPath),
Details: fmt.Sprintf("Size: %d, Limit: %d", info.Size(), limit),
FilePath: fullPath,
})
return
}
f, err := osFS.Open(fullPath)
if err != nil {
return
}
defer func() { _ = f.Close() }()
opened, openStatErr := f.Stat()
canCache = canCache && openStatErr == nil && opened.Mode().IsRegular() && phpFileStampOf(opened) == stamp
// Opening alone proves readability, not identity: a path can be swapped
// between Stat and Open. Only reuse the clean result for the same inode.
if canCache && !s.forceFull && cached && previous == stamp {
s.next[fullPath] = stamp
return true
}
var size int64 = -1
if openStatErr == nil {
size = opened.Size()
}
// Every PHP source file is content-analysed. No filename/path allowlist:
// clean files produce no finding, so there is no benefit to skipping
// them, and any skip is a place an attacker can hide a backdoor.
result := analyzePHPContentReaderAt(fullPath, f, size)
readOK = result.readOK
if result.severity >= 0 {
*findings = append(*findings, phpContentFinding(fullPath, result, info, FileContentSHA256(fullPath)))
return
}
// A clean read only validates the stamp if both the path and the opened
// inode still match. Never attach one file's result to another's stamp or
// retain a clean result for an inode changed during the read.
if canCache && result.readOK {
afterPath, pathErr := osFS.Stat(fullPath)
afterFile, fileErr := f.Stat()
if pathErr == nil && fileErr == nil &&
phpFileStampOf(afterPath) == stamp && phpFileStampOf(afterFile) == stamp {
s.next[fullPath] = stamp
}
}
return readOK
}
// phpContentFinding is the finding for one flagged PHP file. info may be nil
// when the file could not be statted.
func phpContentFinding(path string, result phpAnalysisResult, info os.FileInfo, contentSHA256 string) alert.Finding {
details := result.details
if info != nil {
details += fmt.Sprintf("\nSize: %d, Mtime: %s", info.Size(), info.ModTime().Format("2006-01-02 15:04:05"))
}
return alert.Finding{
Severity: result.severity,
Check: result.check,
Message: fmt.Sprintf("%s: %s", result.message, path),
Details: details,
FilePath: path,
ContentSHA256: contentSHA256,
DetectLogic: ContentDetectionVersion(),
}
}
type phpAnalysisResult struct {
severity alert.Severity
check string
message string
details string
indicators []string
readOK bool
empty bool
}
// analyzePHPContent reads a PHP file's head window (and tail window for large
// files) and checks for obfuscation and malicious patterns.
func analyzePHPContent(path string) phpAnalysisResult {
f, err := osFS.Open(path)
if err != nil {
return phpAnalysisResult{severity: -1}
}
defer func() { _ = f.Close() }()
var size int64 = -1
if info, statErr := f.Stat(); statErr == nil {
size = info.Size()
}
return analyzePHPContentReaderAt(path, f, size)
}
func analyzePHPContentWithFingerprint(path string) (phpAnalysisResult, string) {
f, err := osFS.Open(path)
if err != nil {
return phpAnalysisResult{severity: -1}, ""
}
defer func() { _ = f.Close() }()
var size int64 = -1
info, statErr := f.Stat()
if statErr == nil {
size = info.Size()
}
if statErr == nil && info.Mode().IsRegular() && size <= contentFingerprintMaxBytes {
data, err := io.ReadAll(io.LimitReader(f, contentFingerprintMaxBytes+1))
if err != nil {
return phpAnalysisResult{severity: -1, readOK: false}, ""
}
after, err := f.Stat()
if err != nil || !sameCleanContentShape(after, info) || int64(len(data)) != size {
return phpAnalysisResult{severity: -1, readOK: false}, ""
}
result := analyzePHPContentReaderAt(path, bytes.NewReader(data), int64(len(data)))
if !result.readOK {
return result, ""
}
sum := sha256.Sum256(data)
return result, fmt.Sprintf("%x", sum)
}
return analyzePHPContentReaderAt(path, f, size), ""
}
func analyzePHPContentReaderAt(path string, f io.ReaderAt, size int64) phpAnalysisResult {
head, tail, tailOffset, readOK := readPHPContentWindows(f, size)
if !readOK {
return phpAnalysisResult{severity: -1, readOK: false}
}
if len(head) == 0 && len(tail) == 0 {
return phpAnalysisResult{severity: -1, readOK: true, empty: true}
}
if len(tail) > 0 && tailOffset > int64(len(head)) {
// The skipped bytes may close a quote or comment that starts in the
// head. Score windows independently so head parser state cannot hide
// executable tail code we did read.
return mergePHPAnalysisResults(
analyzePHPCode(path, phpCodeOnly(string(head)), readOK),
analyzePHPCode(path, phpCodeOnly("<?php\n"+string(tail)), readOK),
)
}
return analyzePHPCode(path, phpCodeOnlyWindows(head, tail), readOK)
}
func mergePHPAnalysisResults(results ...phpAnalysisResult) phpAnalysisResult {
readOK := true
var indicators []string
for _, result := range results {
readOK = readOK && result.readOK
if result.severity >= 0 {
indicators = append(indicators, result.indicators...)
}
}
return phpAnalysisFromIndicators(indicators, readOK)
}
func analyzePHPCode(path, content string, readOK bool) phpAnalysisResult {
contentLower := strings.ToLower(content)
var indicators []string
// --- Critical: Remote payload fetching ---
// Paste sites are always suspicious in PHP files.
// GitHub raw URLs are common in legitimate plugin update checkers,
// so only count them as indicators when they appear on the same line
// as a dangerous PHP function call.
pasteHosts := []string{
"pastebin.com/raw",
"paste.ee/r/",
"ghostbin.co/paste/",
"hastebin.com/raw/",
}
for _, host := range pasteHosts {
if strings.Contains(contentLower, host) {
indicators = append(indicators, fmt.Sprintf("remote payload URL: %s", host))
}
}
githubHosts := []string{"gist.githubusercontent.com", "raw.githubusercontent.com"}
dangerousCalls := []string{"file_put_contents(", "fwrite(", "shell_", "passthru(", "popen("}
for _, host := range githubHosts {
if !strings.Contains(contentLower, host) {
continue
}
// Same-line = strong signal (critical)
sameLine := false
for _, line := range strings.Split(contentLower, "\n") {
if !strings.Contains(line, host) {
continue
}
for _, fn := range dangerousCalls {
if strings.Contains(line, fn) {
indicators = append(indicators, fmt.Sprintf("remote payload URL with dangerous call: %s", host))
sameLine = true
break
}
}
if sameLine {
break
}
}
// Co-presence (different lines, same 32 KB window) was previously
// emitted as a weaker indicator. It generated standing FPs on
// legit plugins that fetch upstream resources from github mirrors
// (wp-statistics GeoLite2 updates, unyson font fetcher, polylang
// language packs). Same-line is the strong signal kept above; the
// co-presence path is removed entirely.
}
// --- Critical: eval() chains with decoding ---
// Only flag when eval directly wraps a decoder (structural nesting),
// not when they merely co-exist in the same file (which causes false
// positives on legitimate plugins that use eval for templates and
// base64_decode for unrelated data processing).
decoders := []string{
"base64_decode", "gzinflate", "gzuncompress", "str_rot13",
"rawurldecode", "gzdecode", "bzdecompress",
// Droppers rotate through every reversible transform PHP ships;
// the four below were the unlisted ones seen wrapping eval in the
// wild (hex payloads, reversed strings, URL-encoded blobs, uuencode).
"hex2bin", "strrev", "urldecode", "convert_uudecode",
}
hasDecoder := false
hasNestedEvalDecode := false
for _, d := range decoders {
if strings.Contains(contentLower, d) {
hasDecoder = true
}
}
// Check for structural nesting: eval(base64_decode(...)), eval(gzinflate(...)), etc.
// PHP tolerates inline comments and arbitrary whitespace (including line
// breaks) between the keyword and its open paren, so a naive line-by-line
// `eval(` substring scan misses `eval /*x*/ ( base64_decode(...))` and
// `eval // bypass\n( base64_decode(...))`. Strip PHP comments and
// strings first, then match the structural pattern across whitespace,
// and require the inner callee to be one of the known decoders /
// decompressors.
commentStripped := stripPHPCommentsFromCode(content)
codeLower := strings.ToLower(stripPHPStringsFromCode(commentStripped))
for _, m := range nestedEvalDecodeRe.FindAllStringSubmatch(codeLower, -1) {
if len(m) < 3 {
continue
}
inner := m[2]
for _, d := range decoders {
if inner == d {
hasNestedEvalDecode = true
break
}
}
if hasNestedEvalDecode {
break
}
}
if hasNestedEvalDecode {
indicators = append(indicators, "eval() directly wrapping encoding/compression function")
}
if reEvalRequestInput.MatchString(codeLower) {
indicators = append(indicators, "eval() of request input")
}
// eval wrapping dynamic code construction the decoder loop above
// ignores: a variable callee (eval($f(...))) or a code-building
// primitive (eval(create_function(...)), eval(call_user_func(...))).
// These never appear in legitimate user-directory PHP; a single hit is
// surfaced as a High signal (the >=2 gate still governs quarantine).
hasEvalExecWrap := reEvalVarCallee.MatchString(codeLower)
if !hasEvalExecWrap {
for _, m := range nestedEvalDecodeRe.FindAllStringSubmatch(codeLower, -1) {
if len(m) < 3 {
continue
}
if m[1] != "eval" {
continue
}
if _, ok := evalExecWrapInner[m[2]]; ok {
hasEvalExecWrap = true
break
}
}
}
if hasEvalExecWrap {
indicators = append(indicators, "eval() wrapping a dynamic code-execution primitive")
}
// Backtick shell execution with request input -- `...$_GET...`.
// Match only executable backtick spans, not quoted examples.
if hasBacktickSuperglobal(commentStripped) {
indicators = append(indicators, "backtick shell execution with request input")
}
// Callback-position exec: an exec/decoder function name passed as a
// string callback (array_map("system", ...), register_shutdown_function(
// "passthru", ...)). Runs on the comment-stripped source so the literal
// callback name is preserved.
if hasCallbackExecName(commentStripped) {
indicators = append(indicators, "exec/decoder function name passed as a callback")
}
// Variable-variable / dynamic-expression function call co-located with
// request input on the same line -- $$h($_GET[...]) -- a dynamic-dispatch
// RCE shape. The same-line request-var gate keeps benign dispatcher code
// (which uses $$var without attacker input) from tripping.
for _, line := range strings.Split(codeLower, "\n") {
if reVarVarCall.MatchString(line) && lineContainsRequestVar(line) {
indicators = append(indicators, "variable-variable function call with request input")
break
}
}
// preg_replace() with the /e modifier evaluates its replacement as PHP.
// Removed in PHP 7.0; flag it when request input reaches the evaluated
// replacement or its subject backreferences.
if hasPregReplaceEvalWithRequest(commentStripped) {
indicators = append(indicators, "preg_replace with /e modifier (code execution)")
}
// include/require of request input or a remote/stream wrapper -- LFI,
// RFI, and php://input code execution.
if hasDangerousInclude(commentStripped) {
indicators = append(indicators, "include/require of request input or remote/data wrapper")
}
// assert()/create_function() driven by request input -- both evaluate a
// string argument as PHP.
if hasCodeEvalPrimitiveWithRequest(commentStripped) {
indicators = append(indicators, "code-eval primitive (assert/create_function) with request input")
}
// --- Critical: call_user_func with string-built function names ---
// LEVIATHAN droppers build the target function name on the call itself:
// call_user_func("\x63"."\x75"."\x72"."\x6c", $payload) == call_user_func("curl", ...)
// File-wide hex/concat counts are unsafe here: WPML bundles PHPZip
// (inc/wpml_zip.php) which declares 20+ ZIP-format signature constants
// as hex literals ("\x50\x4b\x03\x04" etc.) and makes a single benign
// call_user_func(self::$temp) call to invoke a temp-file factory.
// Match the obfuscation on the callable target argument itself.
if strings.Contains(contentLower, "call_user_func") {
foundCallUserFuncObfuscation := false
for _, line := range strings.Split(content, "\n") {
if !strings.Contains(strings.ToLower(line), "call_user_func") {
continue
}
for _, targetArg := range phpCallUserFuncTargetArgs(line) {
// PHP 7+ accepts both "\xNN" hex and "\u{NN}" unicode-codepoint
// escapes inside double-quoted strings. Treat them as
// equivalent obfuscation forms so an attacker cannot bypass
// the detector by swapping syntax.
lineHex := countOccurrences(targetArg, `"\x`) + countOccurrences(targetArg, `"\u{`)
lineConcat := countOccurrences(targetArg, `" . "`) + countOccurrences(targetArg, `"."`)
// Typical shortest obfuscated name is 3-4 bytes ("exec", "curl",
// "eval"); require >=3 escapes AND >=2 concatenations on the
// call target argument.
if lineHex >= 3 && lineConcat >= 2 {
indicators = append(indicators, "call_user_func with obfuscated function names")
foundCallUserFuncObfuscation = true
break
}
}
if foundCallUserFuncObfuscation {
break
}
}
}
// --- High: Goto obfuscation (LEVIATHAN signature) ---
// Counting goto statements alone measures the wrong thing. WordPress
// core's HTML API drives the HTML5 insertion-mode state machine with goto
// and names every label after its spec section, so a plain count reports
// authentic core on every site of every account. A descriptive label is
// evidence against obfuscation: an obfuscator emits generated labels
// precisely because they carry no meaning.
//
// Label shape is not enough on its own either. Commercial obfuscators
// sold to plugin vendors emit the same generated labels, and their
// output carries no payload: a paid-for plugin looked exactly like a
// dropper. Malware still has to decode, execute, or read request input
// somewhere, so both branches want a sink. This mirrors the evidence
// the php_goto_obfuscation signature requires.
var generatedGotos, alphaGotos int
for _, m := range reGotoLabel.FindAllStringSubmatch(content, -1) {
if gotoLabelIsGenerated(m[1]) {
generatedGotos++
} else {
alphaGotos++
}
}
switch {
case generatedGotos > 8 && reGotoExecSink.MatchString(content):
indicators = append(indicators, fmt.Sprintf("goto obfuscation (%d generated labels)", generatedGotos))
case alphaGotos > 10 && reGotoExecSink.MatchString(content):
indicators = append(indicators, fmt.Sprintf("goto obfuscation (%d labels reaching an execution sink)", alphaGotos))
}
// --- High: Hex-encoded string construction ---
// Only flag hex strings when accompanied by concatenation - real obfuscation
// builds function names like "\x63" . "\x75" . "\x72" . "\x6c" (= "curl").
// Standalone hex arrays (Wordfence IPv6 subnet masks, binary data) are benign.
hexStringCount := countOccurrences(content, `"\x`)
dotConcatCount := countOccurrences(content, `" . "`)
if hexStringCount > 20 && dotConcatCount > 10 {
indicators = append(indicators, fmt.Sprintf("heavy hex-encoded strings with concatenation (%d hex, %d concat - obfuscation pattern)", hexStringCount, dotConcatCount))
}
// The standalone "concat>30 alone" branch was removed: WordPress
// themes and page builders concatenate literal CSS/HTML tokens
// dozens of times in dynamic style/markup builders (sydney theme,
// elementor, beaver builder), producing FPs on every install. Real
// function-name obfuscation always pairs concat with hex escapes
// and is still caught by the combined branch above.
// --- Critical: variable-function indirection that resolves to a
// decoder. Attackers slip past the literal "eval(base64_decode("
// detector by binding the dangerous name to a variable on one line
// and calling it on another:
// $d = "base64_decode";
// $r = "eval";
// $r($d("AAAA"));
// The heuristic looks for an assignment $var = "decoder_or_exec"
// followed by a $var( invocation in the same file. Hits must
// reference at least one decoder OR one shell-exec primitive,
// since plain variable function calls show up in legitimate
// metaprogramming.
if detectVarFuncDangerousAssignment(content) {
indicators = append(indicators, "variable function name resolves to decoder or exec primitive")
}
// --- High: Variable function calls with obfuscated names ---
// call_user_func + decoder alone is too broad - Elementor, WooCommerce, and
// dozens of plugins use call_user_func_array with base64_decode legitimately.
// Only flag when combined with hex-built callable target obfuscation.
if strings.Contains(contentLower, "call_user_func") && hasDecoder {
if hasCallUserFuncHexNameBuild(content) {
indicators = append(indicators, "variable function call with decoder and obfuscation")
}
}
// --- High: Shell execution functions combined with request input ---
// Uses containsStandaloneFunc to avoid substring false positives
// (e.g. "WP_Filesystem(" matching "exec(", "preg_match(" matching "exec(")
shellFuncs := []string{"system(", "passthru(", "exec(", "shell_exec(", "popen(", "proc_open(", "pcntl_exec("}
// Two-tier detection:
// Same line = CRITICAL signal (auto-quarantine eligible)
// Co-presence = HIGH signal (alert only, not quarantined alone)
// This prevents bypass by splitting across lines while avoiding
// false-positive quarantine of legitimate plugins.
hasShellFunc := false
sameLineShellRequest := false
for _, sf := range shellFuncs {
if containsStandaloneFunc(contentLower, sf) {
hasShellFunc = true
break
}
}
hasRequestVar := containsRequestSuperglobal(contentLower)
// Same-line is a strong signal on its own: "$ret = system($_POST['cmd']);"
// almost never occurs in legitimate code. Co-presence is weaker: elFinder
// and other media-processing libraries legitimately call exec() for
// ImageMagick and also consume $_POST for AJAX routing, placing both
// tokens in the same 32 KB window. The co-presence finding is therefore
// only emitted as CORROBORATION after all the stronger indicators have
// been collected -- see the deferred append further below.
coPresenceCandidate := false
if hasShellFunc && hasRequestVar {
for _, line := range strings.Split(contentLower, "\n") {
lineHasShell := false
for _, sf := range shellFuncs {
if containsStandaloneFunc(line, sf) {
lineHasShell = true
break
}
}
if !lineHasShell {
continue
}
if containsRequestSuperglobal(line) {
sameLineShellRequest = true
break
}
}
if sameLineShellRequest {
indicators = append(indicators, "shell function with request input on same line")
} else if !IsVerifiedCMSFile(path) {
coPresenceCandidate = true
}
}
// --- High: base64 encoding/decoding with execution on same line ---
if strings.Contains(contentLower, "base64_decode") && strings.Contains(contentLower, "base64_encode") {
for _, line := range strings.Split(contentLower, "\n") {
hasBoth := strings.Contains(line, "base64_decode") && strings.Contains(line, "base64_encode")
hasExec := false
for _, sf := range shellFuncs {
if containsStandaloneFunc(line, sf) {
hasExec = true
break
}
}
if hasBoth && hasExec {
indicators = append(indicators, "base64 encode+decode with execution on same line (command relay)")
break
}
}
}
// Deferred corroboration: a lone co-presence is not enough. If a
// stronger indicator was produced above, the co-presence is appended
// both as extra context for the operator and to nudge the severity
// into the >=2 Critical band for obfuscated droppers.
if coPresenceCandidate && len(indicators) > 0 {
indicators = append(indicators, "shell function co-present with request input")
}
// --- Determine severity based on indicators ---
return phpAnalysisFromIndicators(indicators, readOK)
}
func phpAnalysisFromIndicators(indicators []string, readOK bool) phpAnalysisResult {
indicators = dedupePHPIndicators(indicators)
if len(indicators) == 0 {
return phpAnalysisResult{severity: -1, readOK: readOK}
}
// Auto-quarantine in autoresponse.AutoQuarantineFiles acts only on
// Critical findings. A single heuristic indicator has false-positive
// classes severe enough to rm live production files (WPML's PHPZip
// tripped the former "call_user_func with obfuscated" bypass on hex
// constants that build ZIP magic bytes; legitimate plugins embed
// pastebin URLs in support docstrings and release notes). Require
// two converging indicators before the severity crosses the
// destructive-action threshold; single hits surface as High and stay
// in the operator queue.
if len(indicators) >= 2 {
return phpAnalysisResult{
severity: alert.Critical,
check: "obfuscated_php",
message: "Obfuscated/malicious PHP detected",
details: fmt.Sprintf("Indicators found:\n- %s", strings.Join(indicators, "\n- ")),
readOK: readOK,
indicators: indicators,
}
}
return phpAnalysisResult{
severity: alert.High,
check: "suspicious_php_content",
message: "Suspicious PHP content detected",
details: fmt.Sprintf("Indicators found:\n- %s", strings.Join(indicators, "\n- ")),
readOK: readOK,
indicators: indicators,
}
}
func dedupePHPIndicators(indicators []string) []string {
seen := make(map[string]struct{}, len(indicators))
out := indicators[:0]
for _, indicator := range indicators {
if _, ok := seen[indicator]; ok {
continue
}
seen[indicator] = struct{}{}
out = append(out, indicator)
}
return out
}
func countOccurrences(s, substr string) int {
count := 0
offset := 0
for {
idx := strings.Index(s[offset:], substr)
if idx < 0 {
break
}
count++
offset += idx + len(substr)
}
return count
}
type hexNameAssignment struct {
obfuscated bool
hexEscapes int
concatOps int
pos int
}
func hasCallUserFuncHexNameBuild(content string) bool {
code := stripPHPCommentsFromCode(content)
assignments := findHexNameAssignments(code)
searchFrom := 0
for {
callStart, openParen, closeParen, ok := nextStandalonePHPCall(code, searchFrom, callUserFuncCallNames)
if !ok {
return false
}
arg, argOK := firstPHPCallArgument(code, openParen+1)
if argOK && phpExprHasHexNameBuild(arg) {
return true
}
if argOK {
if variable, varOK := singlePHPVariableExpr(arg); varOK && hexNameBuildAt(assignments[variable], callStart) {
return true
}
}
searchFrom = nextSearchOffset(closeParen, len(code))
}
}
func findHexNameAssignments(code string) map[string][]hexNameAssignment {
assignments := map[string][]hexNameAssignment{}
for i := 0; i < len(code); i++ {
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
if code[i] != '$' {
continue
}
variable, next, ok := readPHPVariableName(code, i)
if !ok {
continue
}
j := skipPHPWhitespace(code, next)
opLen, directAssign, appendAssign, ok := phpAssignmentOperator(code, j)
if !ok {
i = next - 1
continue
}
exprStart := skipPHPWhitespace(code, j+opLen)
exprEnd := phpExpressionEnd(code, exprStart)
hexEscapes := 0
concatOps := 0
if directAssign || appendAssign {
hexEscapes = countPHPStringHexEscapes(code[exprStart:exprEnd])
concatOps = countPHPConcatOperators(code[exprStart:exprEnd])
}
if appendAssign {
prev := lastHexNameAssignment(assignments[variable])
hexEscapes += prev.hexEscapes
concatOps += prev.concatOps + 1
}
assignments[variable] = append(assignments[variable], hexNameAssignment{
obfuscated: phpExprCountsHaveHexNameBuild(hexEscapes, concatOps),
hexEscapes: hexEscapes,
concatOps: concatOps,
pos: exprEnd,
})
i = next - 1
}
return assignments
}
func phpAssignmentOperator(code string, pos int) (int, bool, bool, bool) {
if pos >= len(code) {
return 0, false, false, false
}
if code[pos] == '=' && (pos+1 >= len(code) || (code[pos+1] != '=' && code[pos+1] != '>')) {
return 1, true, false, true
}
if pos+1 < len(code) && code[pos] == '.' && code[pos+1] == '=' {
return 2, false, true, true
}
if pos+1 < len(code) && strings.ContainsRune("+-*/%&|^", rune(code[pos])) && code[pos+1] == '=' {
return 2, false, false, true
}
if pos+2 < len(code) {
op := code[pos : pos+3]
if op == "??=" || op == "<<=" || op == ">>=" {
return 3, false, false, true
}
}
return 0, false, false, false
}
func lastHexNameAssignment(assignments []hexNameAssignment) hexNameAssignment {
if len(assignments) == 0 {
return hexNameAssignment{}
}
return assignments[len(assignments)-1]
}
func hexNameBuildAt(assignments []hexNameAssignment, callPos int) bool {
var last hexNameAssignment
found := false
for _, assignment := range assignments {
if assignment.pos > callPos {
break
}
last = assignment
found = true
}
return found && last.obfuscated
}
func singlePHPVariableExpr(expr string) (string, bool) {
expr = strings.TrimSpace(expr)
if expr == "" || expr[0] != '$' {
return "", false
}
variable, next, ok := readPHPVariableName(expr, 0)
if !ok || skipPHPWhitespace(expr, next) != len(expr) {
return "", false
}
return variable, true
}
func phpExprHasHexNameBuild(expr string) bool {
return phpExprCountsHaveHexNameBuild(countPHPStringHexEscapes(expr), countPHPConcatOperators(expr))
}
func phpExprCountsHaveHexNameBuild(hexEscapes, concatOps int) bool {
return hexEscapes >= 3 && concatOps >= 2
}
func countPHPStringHexEscapes(expr string) int {
count := 0
for i := 0; i < len(expr); i++ {
if expr[i] != '"' {
if isPHPQuote(expr[i]) {
i = skipPHPString(expr, i)
}
continue
}
end := skipPHPString(expr, i)
if end > i {
literal := strings.ToLower(expr[i+1 : end])
count += countOccurrences(literal, `\x`)
count += countOccurrences(literal, `\u{`)
}
i = end
}
return count
}
func countPHPConcatOperators(expr string) int {
count := 0
for i := 0; i < len(expr); i++ {
if isPHPQuote(expr[i]) {
i = skipPHPString(expr, i)
continue
}
if expr[i] == '.' {
count++
}
}
return count
}
func phpCallUserFuncTargetArgs(line string) []string {
lower := strings.ToLower(line)
var args []string
for _, name := range []string{"call_user_func_array", "call_user_func"} {
searchFrom := 0
for {
pos := strings.Index(lower[searchFrom:], name)
if pos < 0 {
break
}
pos += searchFrom
next := pos + len(name)
searchFrom = next
if !isPHPFuncNameBoundary(line, pos, next) {
continue
}
i := skipPHPSpaceString(line, next)
if i >= len(line) || line[i] != '(' {
continue
}
if arg, ok := firstPHPCallArgument(line, i+1); ok {
args = append(args, arg)
}
}
}
return args
}
func isPHPFuncNameBoundary(s string, start, end int) bool {
if start > 0 {
prev := s[start-1]
if isIdentCont(prev) || prev == '>' || prev == ':' {
return false
}
}
return end >= len(s) || !isIdentCont(s[end])
}
func skipPHPSpaceString(s string, i int) int {
for i < len(s) && isPHPSpace(s[i]) {
i++
}
return i
}
func firstPHPCallArgument(s string, start int) (string, bool) {
start = skipPHPSpaceString(s, start)
i := start
depth := 0
var quote byte
escaped := false
for i < len(s) {
c := s[i]
if quote != 0 {
if escaped {
escaped = false
i++
continue
}
if c == '\\' {
escaped = true
i++
continue
}
if c == quote {
quote = 0
}
i++
continue
}
switch c {
case '"', '\'':
quote = c
case '(', '[', '{':
depth++
case ')':
if depth == 0 {
return s[start:i], true
}
depth--
case ',':
if depth == 0 {
return s[start:i], true
}
}
i++
}
return "", false
}
// containsStandaloneFunc reports whether content contains an occurrence of
// funcCall (e.g. "exec(") that is a real call to the named PHP function
// rather than something that shares the same suffix.
//
// Four shapes must be rejected:
//
// - embedded identifiers: "doubleval(" must not match "eval("; the
// preceding character is a letter/digit/underscore;
// - method invocations: "$this->DB->exec(" must not match "exec(" even
// though the preceding ">" is non-alphanumeric;
// - static invocations: "Foo::exec(" must not match for the same reason;
// - function declarations: "function exec(" names a local function of
// the same name and must not be counted as a call site.
//
// The earlier implementation only guarded against the first case and was
// the source of false positives on elFinder volume drivers that call
// "$this->DB->exec(...)" (SQLite) alongside $_SERVER references on the
// same line.
func containsStandaloneFunc(content, funcCall string) bool {
idx := 0
for {
pos := strings.Index(content[idx:], funcCall)
if pos < 0 {
return false
}
absPos := idx + pos
nextIdx := absPos + len(funcCall)
advance := func() bool {
if nextIdx >= len(content) {
return false
}
idx = nextIdx
return true
}
if absPos == 0 {
return true
}
prev := content[absPos-1]
isAlnum := (prev >= 'a' && prev <= 'z') || (prev >= 'A' && prev <= 'Z') ||
(prev >= '0' && prev <= '9') || prev == '_'
if isAlnum {
if !advance() {
return false
}
continue
}
if absPos >= 2 {
op := content[absPos-2 : absPos]
if op == "->" || op == "::" {
if !advance() {
return false
}
continue
}
} else if absPos == 1 && (prev == '>' || prev == ':') {
// Degenerate position: only one preceding byte, and it is
// the tail char of a possible method ("->") or static
// ("::") operator. We cannot confirm the second char
// because there is no second char. The conservative choice
// is to skip, so a truncated "->exec(" or "::exec(" at the
// very start of a buffer does not get flagged as a real
// shell-function call.
if !advance() {
return false
}
continue
}
const decl = "function "
if absPos >= len(decl) && content[absPos-len(decl):absPos] == decl {
if !advance() {
return false
}
continue
}
return true
}
}
func containsAny(strs []string, substrs ...string) bool {
for _, s := range strs {
for _, sub := range substrs {
if strings.Contains(s, sub) {
return true
}
}
}
return false
}
// benignPHPStubMaxScan caps how many bytes of a candidate stub the
// recogniser will read. Stub files in the wild (BackWPup folder.php
// caches at ~160 KB, WP "silence is golden" index.php at ~30 B, plugin
// 404-stub headers under 1 KB) fit comfortably under this bound;
// anything larger must surface for normal alerting rather than be
// accepted on faith.
const benignPHPStubMaxScan = 4 * 1024 * 1024
// MaxInertPHPScanBytes is the largest file the inert-content recognizers read
// in full. Translation caches and comment-only stubs need every byte to prove
// that no code follows; a proven PHP terminator can still be accepted from an
// incomplete prefix because its tail is unreachable.
const MaxInertPHPScanBytes = benignPHPStubMaxScan
// IsBenignPHPStub reports whether the reachable code region of a PHP
// file consists only of whitespace and comments, or terminates with a
// literal-argument die / exit, or __halt_compiler before any other statement.
// Files matching either shape cannot execute attacker-controlled code
// via a web request: PHP either runs to EOF emitting nothing, or hits
// the terminator and stops with the remaining bytes unreachable.
//
// The recogniser is content-shape only -- it does not look at the path,
// filename, parent directory, or whether a plugin is installed. An
// attacker cannot bypass it by naming a payload to mimic a known-plugin
// file because the gate fails the moment any executable statement
// appears before a terminator. Conversely a legitimate plugin that
// writes a stub-shaped working file (BackWPup writes
// "<?php //<json>" for job state and "<?php\n//path1\n//path2..." for
// folder caches) is recognised regardless of where it puts the file.
//
// Other detectors -- signature scans, YARA, suspicious filename, the
// webshell name list -- still run on the file in their own pipelines.
// Only the path-only "anomalous PHP location" warning is suppressed
// for files that this recogniser accepts.
func IsBenignPHPStub(path string) bool {
f, err := osFS.Open(path)
if err != nil {
return false
}
defer func() { _ = f.Close() }()
buf := make([]byte, benignPHPStubMaxScan)
n, _ := f.Read(buf)
if n == 0 {
return false
}
info, err := f.Stat()
complete := err == nil && info.Size() <= int64(n)
return IsBenignPHPStubBytesComplete(buf[:n], complete)
}
// IsBenignPHPStubBytes is the buffer-only variant. The realtime fanotify
// path uses it on the bytes it already read from the file descriptor;
// IsBenignPHPStub provides the path-based entry point for the polled
// fileindex scan. Both rely on the same parser so realtime and scheduled
// scans agree on which files are stubs.
//
// The parser tokenises the leading region of the buffer:
//
// - Optional UTF-8 BOM and whitespace, then the literal "<?php" opener.
// The short-echo opener "<?=" is rejected because it emits output.
// A "<?phpfoo" run-together opener is rejected because PHP requires
// whitespace (or EOF) after the tag.
// - Repeatedly accept whitespace, line comments ("//..." or "#..." up to
// newline or "?>"), and balanced block comments ("/* ... */"). A "/*"
// without a matching "*/" inside the scanned window is rejected -- we
// cannot prove the rest of the file is comment.
// - Accept die, exit, and __halt_compiler as terminators. die and exit may
// carry a single literal argument (a non-interpolating string or a
// decimal integer); __halt_compiler takes none. Once seen, the rest of
// the buffer is treated as unreachable.
// - Reject any closing "?>" tag (would allow HTML escape and a later
// "<?php" re-entry that this gate does not analyse).
// - Reject any other identifier (return, if, system, eval, function,
// class, ...) and any stray punctuation ("$", "(", "=", ";", ...).
// Those are statements we cannot prove benign.
// - If the loop reaches EOF in a complete buffer having only seen
// whitespace and comments, accept: PHP outputs nothing and executes
// nothing.
func IsBenignPHPStubBytes(buf []byte) bool {
return IsBenignPHPStubBytesComplete(buf, true)
}
// IsBenignPHPStubBytesComplete is like IsBenignPHPStubBytes, but complete
// tells the parser whether buf contains the entire file. Comment-only stubs
// require a complete buffer; terminators do not, because bytes after them are
// unreachable to PHP.
func IsBenignPHPStubBytesComplete(buf []byte, complete bool) bool {
if len(buf) >= 3 && buf[0] == 0xEF && buf[1] == 0xBB && buf[2] == 0xBF {
buf = buf[3:]
}
i := 0
for i < len(buf) && isPHPSpace(buf[i]) {
i++
}
const opener = "<?php"
if !bytes.HasPrefix(buf[i:], []byte(opener)) {
return false
}
i += len(opener)
if i < len(buf) && !isPHPOpenTagSpace(buf[i]) {
return false
}
for i < len(buf) {
c := buf[i]
if isPHPSpace(c) {
i++
continue
}
if isPHPLineCommentStart(buf, i) {
i = skipPHPLineComment(buf, i)
continue
}
if c == '/' && i+1 < len(buf) && buf[i+1] == '*' {
i += 2
end := bytes.Index(buf[i:], []byte("*/"))
if end < 0 {
return false
}
i += end + 2
continue
}
if c == '?' && i+1 < len(buf) && buf[i+1] == '>' {
return false
}
if isIdentStart(c) {
start := i
for i < len(buf) && isIdentCont(buf[i]) {
i++
}
word := strings.ToLower(string(buf[start:i]))
return isPHPTerminatorStatement(buf, i, word, complete)
}
return false
}
return complete
}
// PHPTerminatesImmediately recognizes a leading exit, die, or __halt_compiler
// in unencoded PHP source, with at most one plain literal argument. Callers
// must establish that PHP source conversion is disabled before using this
// to suppress findings: conversion can remove even a raw terminator keyword.
// Only the opening tag and whitespace may precede it. Unlike the broader
// stub parser, it never accepts comments before the terminator. A completed
// terminator can be recognized from a partial head; EOF alone is not proof.
func PHPTerminatesImmediately(buf []byte) bool {
_, ok := PHPTerminatesImmediatelyAt(buf)
return ok
}
// PHPTerminatesImmediatelyAt is PHPTerminatesImmediately plus the offset just
// past the terminator statement, so a caller can judge the unreachable tail.
func PHPTerminatesImmediatelyAt(buf []byte) (int, bool) {
buf = bytes.TrimPrefix(buf, []byte{0xEF, 0xBB, 0xBF})
i := skipPHPSpace(buf, 0)
const opener = "<?php"
if !bytes.HasPrefix(buf[i:], []byte(opener)) {
return 0, false
}
i += len(opener)
// PHP needs whitespace after the opening tag. Without it the tag is
// literal text, the file never enters code mode here, and a later
// `<?php` block is what actually runs.
if i >= len(buf) || !isPHPOpenTagSpace(buf[i]) {
return 0, false
}
i = skipPHPSpace(buf, i)
start := i
for i < len(buf) && isIdentCont(buf[i]) {
i++
}
if i == start || !isIdentStart(buf[start]) {
return 0, false
}
word := strings.ToLower(string(buf[start:i]))
if !isPHPTerminatorStatement(buf, i, word, false) {
return 0, false
}
return phpTerminatorStatementEnd(buf, i), true
}
// phpTerminatorStatementEnd returns the offset where the file's unreachable
// bytes begin. Past the validated terminator it also consumes any further
// terminator statements and the closing tag, because a state file commonly
// opens `exit('...'); __halt_compiler(); ?>` before its data.
func phpTerminatorStatementEnd(buf []byte, i int) int {
for {
i = skipPHPSpace(buf, i)
if i < len(buf) && buf[i] == '(' {
if next, ok := consumeEmptyPHPParens(buf, i); ok {
i = next
} else if next, ok := consumeLiteralPHPParens(buf, i); ok {
i = next
}
}
i = skipPHPSpace(buf, i)
if i < len(buf) && buf[i] == ';' {
i++
}
i = skipPHPSpace(buf, i)
if i+1 < len(buf) && buf[i] == '?' && buf[i+1] == '>' {
i += 2
// PHP swallows one newline directly after the closing tag.
if i < len(buf) && buf[i] == '\n' {
i++
} else if i+1 < len(buf) && buf[i] == '\r' && buf[i+1] == '\n' {
i += 2
}
return i
}
start := i
for i < len(buf) && isIdentCont(buf[i]) {
i++
}
if i == start || !isIdentStart(buf[start]) {
return start
}
next := strings.ToLower(string(buf[start:i]))
if next != "die" && next != "exit" && next != "__halt_compiler" {
return start
}
}
}
func isPHPSpace(c byte) bool {
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f'
}
// PHP's long opening tag excludes the form feed and vertical tab accepted
// by generic whitespace scanners. Accepting either would hide later PHP blocks.
func isPHPOpenTagSpace(c byte) bool {
return c == ' ' || c == '\t' || c == '\n' || c == '\r'
}
func isIdentStart(c byte) bool {
return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}
func isIdentCont(c byte) bool {
return isIdentStart(c) || (c >= '0' && c <= '9')
}
func skipPHPSpace(buf []byte, i int) int {
for i < len(buf) && isPHPSpace(buf[i]) {
i++
}
return i
}
// PHP 8 attributes begin with #[ and can precede executable declarations and
// statements on the same line. Every inert-content scanner must preserve them.
func isPHPLineCommentStart[T string | []byte](buf T, i int) bool {
if buf[i] == '#' {
return i+1 == len(buf) || buf[i+1] != '['
}
return buf[i] == '/' && i+1 < len(buf) && buf[i+1] == '/'
}
// skipPHPLineComment returns the index that ends the "//" or "#" comment at
// i. PHP ends one at a bare CR as well as at LF, so a CR must not be read as
// comment text: the statement after it runs.
func skipPHPLineComment[T string | []byte](buf T, i int) int {
for i < len(buf) {
if buf[i] == '\n' || buf[i] == '\r' {
return i
}
if buf[i] == '?' && i+1 < len(buf) && buf[i+1] == '>' {
return i
}
i++
}
return i
}
// isPHPTerminatorStatement reports whether the identifier at word, which
// starts the first statement of the buffer, ends execution. __halt_compiler
// takes no argument; die and exit may print one literal before stopping.
func isPHPTerminatorStatement(buf []byte, i int, word string, complete bool) bool {
if word != "die" && word != "exit" && word != "__halt_compiler" {
return false
}
i = skipPHPSpace(buf, i)
if word == "__halt_compiler" {
next, ok := consumeEmptyPHPParens(buf, i)
if !ok {
return false
}
return phpTerminatorStatementEnds(buf, next, complete)
}
if i >= len(buf) {
return complete
}
if buf[i] == ';' {
return true
}
if buf[i] == '?' && i+1 < len(buf) && buf[i+1] == '>' {
return true
}
if buf[i] != '(' {
return false
}
next, ok := consumeEmptyPHPParens(buf, i)
if !ok {
if next, ok = consumeLiteralPHPParens(buf, i); !ok {
return false
}
}
return phpTerminatorStatementEnds(buf, next, complete)
}
// consumeLiteralPHPParens accepts `( <literal> )` where the literal is a
// single-quoted string, a double-quoted string that interpolates nothing, or
// a decimal integer. exit and die evaluate their argument before stopping, so
// a literal is the only shape that proves no other code runs.
func consumeLiteralPHPParens(buf []byte, i int) (int, bool) {
if i >= len(buf) || buf[i] != '(' {
return i, false
}
i = skipPHPSpace(buf, i+1)
if i >= len(buf) {
return i, false
}
if isPHPQuote(buf[i]) {
end, ok := endOfPHPLiteralString(buf, i)
if !ok {
return i, false
}
i = end
} else {
start := i
for i < len(buf) && buf[i] >= '0' && buf[i] <= '9' {
i++
}
if i == start {
return i, false
}
}
i = skipPHPSpace(buf, i)
if i >= len(buf) || buf[i] != ')' {
return i, false
}
return skipPHPSpace(buf, i+1), true
}
// endOfPHPLiteralString returns the index just past the string literal that
// starts at i. It fails on a string the buffer does not terminate and on a
// double-quoted string carrying a variable, because PHP evaluates `$x` and
// `{$x}` inside double quotes. Keep literals plain ASCII without escape or
// encoding-shift bytes: the shared stub parser also runs without an encoding
// policy, and source conversion can expose expressions inside such strings.
func endOfPHPLiteralString(buf []byte, i int) (int, bool) {
quote := buf[i]
for j := i + 1; j < len(buf); j++ {
if buf[j] < ' ' || buf[j] > '~' || strings.ContainsRune("\\+=&~", rune(buf[j])) {
return j, false
}
switch buf[j] {
case '$':
if quote == '"' {
return j, false
}
case quote:
return j + 1, true
}
}
return len(buf), false
}
func consumeEmptyPHPParens(buf []byte, i int) (int, bool) {
if i >= len(buf) || buf[i] != '(' {
return i, false
}
i = skipPHPSpace(buf, i+1)
if i >= len(buf) || buf[i] != ')' {
return i, false
}
return skipPHPSpace(buf, i+1), true
}
func phpTerminatorStatementEnds(buf []byte, i int, complete bool) bool {
i = skipPHPSpace(buf, i)
if i >= len(buf) {
return complete
}
if buf[i] == ';' {
return true
}
return buf[i] == '?' && i+1 < len(buf) && buf[i+1] == '>'
}
package checks
import (
"os"
"sync"
"github.com/pidginhost/csm/internal/store"
)
// Keep unpersisted progress while the daemon is alive. Otherwise one failed
// cursor write repeatedly spends the host's entire budget on the same account.
// Successful writes evict the fallback; account removal and database changes
// discard it too, so historical accounts cannot accumulate in memory.
var rollingContentCursors = phpContentCursors{}
type phpContentCursors struct {
mu sync.Mutex
db *store.DB
pending map[string]store.ScanCursorRecord
}
func (c *phpContentCursors) resetLocked(db *store.DB) {
if c.db != db {
c.db = db
c.pending = make(map[string]store.ScanCursorRecord)
}
}
func (c *phpContentCursors) retain(db *store.DB, entries []os.DirEntry) {
c.mu.Lock()
defer c.mu.Unlock()
c.resetLocked(db)
live := make(map[string]bool, len(entries))
for _, entry := range entries {
if entry.IsDir() {
live[entry.Name()] = true
}
}
for account := range c.pending {
if !live[account] {
delete(c.pending, account)
}
}
}
func (c *phpContentCursors) load(db *store.DB, account string) (store.ScanCursorRecord, error) {
c.mu.Lock()
defer c.mu.Unlock()
c.resetLocked(db)
cur, _, err := db.GetScanCursor(account, rollingScanCheck)
if pending, ok := c.pending[account]; ok {
return pending, err
}
return cur, err
}
func (c *phpContentCursors) save(db *store.DB, cur store.ScanCursorRecord) error {
c.mu.Lock()
defer c.mu.Unlock()
c.resetLocked(db)
if err := db.PutScanCursor(cur); err != nil {
c.pending[cur.Account] = cur
return err
}
delete(c.pending, cur.Account)
return nil
}
package checks
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/contenttype"
"github.com/pidginhost/csm/internal/store"
)
// rollingScanCheck is the cursor key under which rolling content coverage
// records its per-account progress. It reuses the existing content findings, so
// no new check name is registered.
const rollingScanCheck = "php_content"
// rollingWalkMaxDepth bounds how deep enumeratePHPFiles descends under each
// docroot, so a deeply nested or symlink-looped tree cannot make enumeration
// unbounded. The fixed suspicious-dir scan uses a shallow depth; rolling needs
// to reach app code nested a few levels down (e.g. wp-content/plugins/x/inc/y)
// but does not need to chase arbitrarily deep trees.
const rollingWalkMaxDepth = 12
// rollingContentEnabled gates rolling coverage to normal periodic host-wide
// runs: the knob is on, this is not an account-scoped run, and it is not a
// forced/audit full content scan (those already read every file). forcedFull is
// the per-cycle decision already computed by CheckPHPContent
// (phpContentForceFull || scanForceContent); it is threaded in rather than
// recomputed because phpContentForceFull advances a cadence counter on each
// call, so calling it twice per cycle would skew the forced-rescan cadence.
func rollingContentEnabled(ctx context.Context, cfg *config.Config, forcedFull bool) bool {
return cfg.Thresholds.RollingCoverage &&
AccountFromContext(ctx) == "" &&
!forcedFull
}
// rollingContentPass spends one cycle's file budget on rolling coverage across
// the host. The budget is the operator's per-scan file cap: sizing a window
// that large for every account asked for hundreds of windows inside one check
// budget, so the check never finished and its findings were never reported.
// Accounts are taken least-recently-covered first, so a host too large for one
// cycle keeps moving instead of resweeping the same alphabetical prefix.
func rollingContentPass(ctx context.Context, cfg *config.Config, scan *phpContentScan, homeDirs []os.DirEntry, findings *[]alert.Finding) {
db := store.Global()
if db == nil {
// Cannot persist a cursor, so rolling would scan from the start every
// cycle without making progress. Skip rather than spin in place.
return
}
rollingContentCursors.retain(db, homeDirs)
accounts := rollingAccountOrder(db, homeDirs)
budget := accountScanMaxFiles(ctx, cfg)
budget -= rollingChangedPass(ctx, cfg, scan, db, accounts, budget, findings)
wrapped := 0
for _, account := range accounts {
if ctx.Err() != nil || budget <= 0 {
break
}
whole, used := rollingContentCoverage(ctx, cfg, scan, account.name, accountDocRoots(account.home), budget, findings)
budget -= used
if whole {
wrapped++
}
}
if wrapped != len(accounts) {
// The accounts this cycle did not finish are not re-emitting their
// earlier findings, and completing the check would purge them
// (mirrors yara_deep).
markCheckIncomplete(ctx, "php_content")
}
}
// rollingAccount pairs an account name with its home directory.
type rollingAccount struct {
name string
home string
}
// rollingAccountOrder sorts accounts by the time each last completed a full
// traversal, oldest first, so an account that has never been covered goes
// first and no account can be starved by the ones before it in the alphabet.
func rollingAccountOrder(db *store.DB, homeDirs []os.DirEntry) []rollingAccount {
type ordered struct {
rollingAccount
last time.Time
}
all := make([]ordered, 0, len(homeDirs))
for _, entry := range homeDirs {
if !entry.IsDir() {
continue
}
cur, _ := rollingContentCursors.load(db, entry.Name())
all = append(all, ordered{
rollingAccount: rollingAccount{name: entry.Name(), home: scanHomeDirPath(entry)},
last: cur.LastFullCycleTS,
})
}
sort.Slice(all, func(i, j int) bool {
if !all[i].last.Equal(all[j].last) {
return all[i].last.Before(all[j].last)
}
return all[i].name < all[j].name
})
out := make([]rollingAccount, 0, len(all))
for _, a := range all {
out = append(out, a.rollingAccount)
}
return out
}
// rollingContentCoverage sweeps a bounded path-sorted slice of the account's
// full docroot PHP-source set, advancing the per-account cursor so every stock
// PHP source or source-view file is eventually content-scanned over cycles.
// The caller guarantees the gate (rolling on, host-scope periodic, not a
// forced/audit run) and hands it what is left of the cycle's file budget.
// Findings append to the live findings slice (rolling is part of the periodic
// scan, not a report-only full-scan job). A canceled run leaves the prior
// cursor untouched. It returns how many files it read so the caller can charge
// them against the budget.
//
// Limitation: rolling enumerates only stock-PHP-executable filenames across the
// whole docroot. A file whose non-stock extension is remapped to PHP by an
// .htaccess handler (the LEVIATHAN trick) is NOT enumerated here; the fixed
// suspicious-dir scan (which layers per-directory overlays as it descends) and
// realtime fanotify still cover those.
// It reports whether this cycle covered the account's whole file list. A
// window that did not wrap leaves files from earlier windows unvisited, and
// their findings are not re-emitted this cycle, so the caller must mark the
// check incomplete or the runner purges them from the latest set.
func rollingContentCoverage(ctx context.Context, cfg *config.Config, scan *phpContentScan, account string, docRoots []string, limit int, findings *[]alert.Finding) (bool, int) {
db := store.Global()
files := enumeratePHPFiles(ctx, cfg, docRoots)
if ctx.Err() != nil {
return false, 0
}
// Enumeration already covers all roots independently of the content
// window. Prune even on partial or empty windows, including lists that
// keep growing before the cursor can ever wrap.
scan.pruneMissing(docRoots, files)
if len(files) == 0 {
return true, 0
}
cur, curErr := rollingContentCursors.load(db, account)
if curErr != nil {
// Keep storage failures visible even when in-memory progress lets
// this daemon continue covering the account.
fmt.Fprintf(os.Stderr, "php_content rolling: cursor read for %s: %v\n", account, curErr)
}
selected, newLast, wrapped := rollingCandidatesAfter(files, cur.LastPath, limit)
if len(selected) == 0 {
return true, 0
}
// Crossing the end of the list completes a traversal across several
// windows, but this run still did not re-emit findings from the earlier
// windows. Only a window containing the whole list is safe to report as a
// completed check to the runner.
windowComplete := len(selected) == len(files)
fullTraversal := wrapped || windowComplete
// Reconstruct the .htaccess handler overlay once per directory: every file
// in the slice that shares a directory shares the same overlay, and reading
// the ancestor .htaccess chain per file would multiply the read cost.
overlayCache := make(map[string]phpHandlerOverlay)
read := 0
for _, file := range selected {
if ctx.Err() != nil {
break
}
// The fixed walk or the changed pass already read it this cycle.
if ok, done := scan.attempted[file]; done {
if !ok {
windowComplete = false
}
continue
}
// Opening FIFOs or device nodes can block the scan; rolling only needs
// regular PHP files (including symlinks that resolve to regular files).
if !rollingRegularCandidate(file) {
// A failed stat or changed file type invalidates any earlier
// clean result just like an unsuccessful content read.
delete(scan.prev, file)
delete(scan.next, file)
continue
}
read++
dir := filepath.Dir(file)
overlay, ok := overlayCache[dir]
if !ok {
overlay = reconstructOverlay(rollingDocRootFor(file, docRoots), dir)
overlayCache[dir] = overlay
}
if !scan.scanFile(ctx, file, overlay, findings) {
windowComplete = false
}
}
// Advance the cursor only on a complete, uncanceled run. A run cut short by
// ctx cancellation leaves the prior cursor so the next cycle resumes where
// this one stopped instead of skipping the unscanned tail.
if ctx.Err() != nil {
return false, read
}
cur.Account = account
cur.Check = rollingScanCheck
cur.LastPath = newLast
if fullTraversal {
now := time.Now().UTC()
cur.LastFullCycleTS = now
if wrapped {
cur.WrappedAt = now
}
}
if err := rollingContentCursors.save(db, cur); err != nil {
fmt.Fprintf(os.Stderr, "php_content rolling: cursor write for %s: %v\n", account, err)
}
return windowComplete, read
}
// The overlap covers coarse ctime ticks and small clock corrections. Larger
// backward steps rewind the bound using the account's wall/boot clock pair.
// Tests zero this only for fixtures written just before their first scan.
var rollingChangedMargin = 2 * time.Minute
// rollingChangedPass uses the existing host-wide file budget. Only one
// account's file list is retained at a time, and no content-cache sweep is
// needed to discover changes: forced cache expiry must not create a backlog.
func rollingChangedPass(ctx context.Context, cfg *config.Config, scan *phpContentScan, db *store.DB, accounts []rollingAccount, budget int, findings *[]alert.Finding) int {
start := phpContentNow()
uptime, clockOK := procUptime()
boot, _ := osFS.ReadFile("/proc/sys/kernel/random/boot_id")
clock := rollingChangeClock{wall: start, uptime: uptime}
if clockOK {
clock.boot = strings.TrimSpace(string(boot))
}
ordered := rollingChangedOrder(db, accounts)
// Logical service time keeps rotation fair through backward clock steps.
served := start
for _, account := range ordered {
cur, _ := rollingContentCursors.load(db, account.name)
if !cur.ChangedServedAt.Before(served) {
served = cur.ChangedServedAt.Add(time.Nanosecond)
}
}
read := 0
for _, account := range ordered {
if ctx.Err() != nil || read >= budget {
break
}
read += rollingChangedAccount(ctx, cfg, scan, db, account, clock, served, budget-read, findings)
}
return read
}
func rollingChangedOrder(db *store.DB, accounts []rollingAccount) []rollingAccount {
type ordered struct {
rollingAccount
served time.Time
}
all := make([]ordered, 0, len(accounts))
for _, account := range accounts {
cur, _ := rollingContentCursors.load(db, account.name)
all = append(all, ordered{rollingAccount: account, served: cur.ChangedServedAt})
}
sort.SliceStable(all, func(i, j int) bool {
if !all[i].served.Equal(all[j].served) {
return all[i].served.Before(all[j].served)
}
return all[i].name < all[j].name
})
out := make([]rollingAccount, 0, len(all))
for _, a := range all {
out = append(out, a.rollingAccount)
}
return out
}
type rollingChangeClock struct {
wall time.Time
uptime float64
boot string
}
func (clock rollingChangeClock) update(cur *store.ScanCursorRecord) {
if clock.boot == "" && cur.ChangedBootID != "" {
// Keep the last verifiable reference and bound until procfs recovers.
cur.ChangedLapFailed = true
return
}
through := cur.ChangedThrough
if !cur.ChangedClock.IsZero() {
delta := clock.wall.Sub(cur.ChangedClock)
if clock.boot != "" && clock.boot == cur.ChangedBootID && clock.uptime >= cur.ChangedUptime {
delta -= time.Duration((clock.uptime - cur.ChangedUptime) * float64(time.Second))
}
// After a reboot only the wall clocks compare: one now behind the last
// pass rewinds the bound below. Rereading every file instead would
// leave new files waiting behind that backlog for days.
// procfs uptime is rounded; the overlap absorbs subsecond jitter.
if delta < -time.Second {
through = through.Add(delta)
}
}
if clock.wall.Before(through) {
through = clock.wall.Add(-rollingChangedMargin)
}
if through.Before(cur.ChangedThrough) {
cur.ChangedThrough = through
clearChangedLap(cur)
}
cur.ChangedClock, cur.ChangedUptime, cur.ChangedBootID = clock.wall, clock.uptime, clock.boot
}
func clearChangedLap(cur *store.ScanCursorRecord) {
cur.ChangedFrom, cur.ChangedEnd = "", ""
cur.ChangedLapStart = time.Time{}
cur.ChangedLapFailed = false
}
func rollingChangedAccount(ctx context.Context, cfg *config.Config, scan *phpContentScan, db *store.DB, account rollingAccount, clock rollingChangeClock, served time.Time, limit int, findings *[]alert.Finding) int {
cur, curErr := rollingContentCursors.load(db, account.name)
if curErr != nil {
fmt.Fprintf(os.Stderr, "php_content changed pass: cursor read for %s: %v\n", account.name, curErr)
}
cur.Account, cur.Check = account.name, rollingScanCheck
clock.update(&cur)
cur.ChangedServedAt = served
// Publish service before I/O, so a timed-out account yields to its peers.
// A canceled scan must not later overwrite its successor's progress.
saveRollingCursor(db, cur)
defer func() {
if cur.ChangedLapFailed {
markCheckIncomplete(ctx, "php_content")
}
if ctx.Err() == nil {
saveRollingCursor(db, cur)
}
}()
if cur.ChangedThrough.IsZero() {
cur.ChangedThrough = clock.wall.Add(-rollingChangedMargin)
return 0
}
roots, rootsComplete := accountDocRootsComplete(account.home)
files, complete := enumeratePHPFilesComplete(ctx, cfg, roots)
if ctx.Err() != nil {
return 0
}
if cur.ChangedLapStart.IsZero() {
cur.ChangedLapStart = clock.wall
if len(files) > 0 {
cur.ChangedEnd = files[len(files)-1]
}
}
cur.ChangedLapFailed = cur.ChangedLapFailed || !complete || !rootsComplete
overlays := make(map[string]phpHandlerOverlay)
read := 0
for _, file := range files {
if ctx.Err() != nil {
return read
}
if file <= cur.ChangedFrom || (cur.ChangedEnd != "" && file > cur.ChangedEnd) {
continue
}
if read >= limit {
return read
}
info, ctime, err := rollingChangedInfo(file)
if err != nil || !info.Mode().IsRegular() {
delete(scan.prev, file)
delete(scan.next, file)
cur.ChangedLapFailed = cur.ChangedLapFailed || (err != nil && !os.IsNotExist(err))
} else {
changed := ctime == 0 || !time.Unix(0, ctime).Before(cur.ChangedThrough)
// Changes made after this lap started belong to the next lap. This
// bounds a burst even if the account keeps creating files. Future
// timestamps after a clock step are still read, never silently skipped.
later := ctime != 0 && time.Unix(0, ctime).After(cur.ChangedLapStart) && !time.Unix(0, ctime).After(clock.wall)
if changed && !later {
if ok, attempted := scan.attempted[file]; attempted {
cur.ChangedLapFailed = cur.ChangedLapFailed || !ok
} else {
read++
dir := filepath.Dir(file)
overlay, ok := overlays[dir]
if !ok {
overlay = reconstructOverlay(rollingDocRootFor(file, roots), dir)
overlays[dir] = overlay
}
if !scan.scanFile(ctx, file, overlay, findings) {
cur.ChangedLapFailed = true
}
}
}
}
cur.ChangedFrom = file
}
if ctx.Err() == nil {
if !cur.ChangedLapFailed {
cur.ChangedThrough = cur.ChangedLapStart.Add(-rollingChangedMargin)
} else {
markCheckIncomplete(ctx, "php_content")
}
clearChangedLap(&cur)
}
return read
}
// A new symlink can point at an old inode without changing the target's
// ctime. Either the link or its target changing must queue a priority read.
func rollingChangedInfo(file string) (os.FileInfo, int64, error) {
info, err := osFS.Lstat(file)
if err != nil {
return nil, 0, err
}
ctime := phpFileStampOf(info).Ctime
if info.Mode()&os.ModeSymlink == 0 {
return info, ctime, nil
}
target, err := osFS.Stat(file)
if err != nil {
return nil, 0, err
}
targetTime := phpFileStampOf(target).Ctime
if ctime == 0 || targetTime == 0 {
return target, 0, nil
}
return target, max(ctime, targetTime), nil
}
func saveRollingCursor(db *store.DB, cur store.ScanCursorRecord) {
if err := rollingContentCursors.save(db, cur); err != nil {
fmt.Fprintf(os.Stderr, "php_content changed pass: cursor write for %s: %v\n", cur.Account, err)
}
}
func rollingRegularCandidate(file string) bool {
info, err := osFS.Stat(file)
return err == nil && info.Mode().IsRegular()
}
// rollingDocRootFor returns the docRoot that contains file. file always sits
// under exactly one of docRoots (enumeratePHPFiles built it by descending from
// them); the longest matching prefix wins so nested account roots resolve to
// the most specific one.
func rollingDocRootFor(file string, docRoots []string) string {
best := ""
for _, root := range docRoots {
if root == file || strings.HasPrefix(file, root+string(filepath.Separator)) {
if len(root) > len(best) {
best = root
}
}
}
return best
}
// enumeratePHPFiles recursively collects, under each docRoot, candidate paths
// whose name contains stock PHP source. This includes source-view .phps files
// without classifying them as executable. The walk is bounded to
// rollingWalkMaxDepth, honours ctx cancellation, and
// respects suppressions.ignore_paths exactly like scanDir when the scan is not
// an explicit full-scan/audit. The result is ascending-sorted and de-duplicated
// so rollingCandidatesAfter can cursor through it stably.
func enumeratePHPFiles(ctx context.Context, cfg *config.Config, docRoots []string) []string {
files, _ := enumeratePHPFilesComplete(ctx, cfg, docRoots)
return files
}
func enumeratePHPFilesComplete(ctx context.Context, cfg *config.Config, docRoots []string) ([]string, bool) {
seen := make(map[string]struct{})
complete := true
respectIgnores := scanRespectsIgnores(ctx, cfg)
for _, root := range docRoots {
if !walkPHPFiles(ctx, cfg, root, rollingWalkMaxDepth, respectIgnores, seen) {
complete = false
}
if ctx.Err() != nil {
break
}
}
if len(seen) == 0 {
return nil, complete
}
files := make([]string, 0, len(seen))
for f := range seen {
files = append(files, f)
}
sort.Strings(files)
return files, complete
}
func walkPHPFiles(ctx context.Context, cfg *config.Config, dir string, maxDepth int, respectIgnores bool, seen map[string]struct{}) bool {
if ctx.Err() != nil {
return false
}
if maxDepth <= 0 {
return true
}
entries, err := osFS.ReadDir(dir)
if err != nil {
return os.IsNotExist(err)
}
complete := true
for _, entry := range entries {
if ctx.Err() != nil {
return false
}
fullPath := filepath.Join(dir, entry.Name())
// Same suppression gate as scanDir: this is not a path allowlist for
// "safe" files but an operator-configured ignore that the periodic scan
// already honours. It is bypassed for explicit full-scan/audit runs,
// which never reach rolling anyway (the gate excludes forced runs).
if respectIgnores && pathIsIgnored(cfg, fullPath) {
continue
}
if entry.IsDir() {
if !walkPHPFiles(ctx, cfg, fullPath, maxDepth-1, respectIgnores, seen) {
complete = false
}
continue
}
if contenttype.IsPHPSourceName(strings.ToLower(entry.Name())) {
seen[fullPath] = struct{}{}
}
}
return complete
}
func pathIsIgnored(cfg *config.Config, fullPath string) bool {
for _, ignore := range cfg.Suppressions.IgnorePaths {
if matchGlob(fullPath, ignore) {
return true
}
}
return false
}
// reconstructOverlay builds the handler overlay for fileDir by merging the
// .htaccess files from rootDir down through each ancestor to fileDir, starting
// from an empty overlay at rootDir. This matches how scanDir accumulates
// overlays as it descends. fileDir must be rootDir or a descendant; if rootDir
// is empty (file resolved to no docRoot, which should not happen) the overlay
// is built from fileDir alone.
func reconstructOverlay(rootDir, fileDir string) phpHandlerOverlay {
overlay := phpHandlerOverlay{}
if rootDir == "" {
if data, ok, err := readHtaccessBounded(filepath.Join(fileDir, ".htaccess")); err == nil && ok {
overlay = overlay.mergeHtaccess(data)
} else if htaccessOversized(ok, err) {
overlay.unrestricted = true
}
return overlay
}
// Build the ordered list of directories from rootDir down to fileDir
// inclusive by stripping the shared prefix and walking the relative
// components back on.
dirs := []string{rootDir}
rel, err := filepath.Rel(rootDir, fileDir)
if err == nil && rel != "." && rel != "" && !strings.HasPrefix(rel, "..") {
cur := rootDir
for _, part := range strings.Split(rel, string(filepath.Separator)) {
cur = filepath.Join(cur, part)
dirs = append(dirs, cur)
}
}
for _, d := range dirs {
if data, ok, err := readHtaccessBounded(filepath.Join(d, ".htaccess")); err == nil && ok {
overlay = overlay.mergeHtaccess(data)
} else if htaccessOversized(ok, err) {
// Unreadable handler configuration: scan every name rather
// than assume the default extension set.
overlay.unrestricted = true
}
}
return overlay
}
package checks
import "strings"
// PHPRequestFlowStructure returns executable PHP with comments, inline HTML,
// and string bodies blanked while preserving byte offsets and line breaks.
// It is suitable for confirming that a dangerous token is PHP syntax rather
// than documentation or sample text stored in a literal. A literal whose
// whole value is php://input stays visible: the request body is the one
// request source PHP code names with a string.
func PHPRequestFlowStructure(src string) string {
return stripPHPStringsKeeping(PHPExecutableCode(src), isPHPRequestBodyLiteral)
}
func isPHPRequestBodyLiteral(token string) bool {
if len(token) < 2 || (token[0] != '\'' && token[0] != '"') || token[len(token)-1] != token[0] {
return false
}
return strings.EqualFold(token[1:len(token)-1], "php://input")
}
package checks
import (
"bytes"
"io"
"strings"
)
// wpTranslationMaxDepth bounds array nesting so a crafted file cannot drive
// unbounded recursion in the recognizer. Real translation caches nest one
// level deep (the "messages" map), so this leaves ample headroom.
const wpTranslationMaxDepth = 16
// IsWPTranslationCacheBytesComplete reports whether buf is exactly a WordPress
// PHP translation cache: the "<?php" opener, the keyword "return", a single PHP
// array literal whose elements are only string/integer scalars (optionally
// concatenated string literals, as GlotPress joins plural forms with a "\0"
// separator) or nested arrays of the same, then a ";" and nothing else.
// WordPress 6.5+ auto-generates these as pure data return maps (*.l10n.php);
// each one previously opened a sensitive-dir Warning incident.
//
// This is a content-structure recognizer, not a path or filename allowlist. A
// variable, a function call, string interpolation, a concatenation operand that
// is not a literal, a closing "?>" tag, or any statement after the array makes
// it return false, so an attacker cannot smuggle code into a file shaped like a
// translation cache. complete must be true: a truncated buffer cannot prove the
// unseen tail carries no code, so it is never suppressed.
func IsWPTranslationCacheBytesComplete(buf []byte, complete bool) bool {
if !complete || len(buf) == 0 {
return false
}
if len(buf) >= 3 && buf[0] == 0xEF && buf[1] == 0xBB && buf[2] == 0xBF {
buf = buf[3:]
}
s := &phpLiteralScanner{buf: buf}
s.skipSpace()
if !s.consumeOpener() {
return false
}
s.skipTrivia()
if id, ok := s.readIdent(); !ok || id != "return" {
return false
}
s.skipTrivia()
if !s.parseTopArray() {
return false
}
s.skipTrivia()
if s.i >= len(s.buf) || s.buf[s.i] != ';' {
return false
}
s.i++
s.skipTrivia()
return s.i == len(s.buf)
}
// phpLiteralScanner walks a byte buffer that is expected to be a constant PHP
// data literal. It never evaluates anything; it only proves the bytes contain
// no executable construct.
type phpLiteralScanner struct {
buf []byte
i int
}
// skipSpace advances past whitespace only. Used before the open tag, where any
// non-whitespace would be raw output (HTML) rather than a PHP comment.
func (s *phpLiteralScanner) skipSpace() {
for s.i < len(s.buf) && isPHPSpace(s.buf[s.i]) {
s.i++
}
}
// skipTrivia advances past whitespace and PHP comments. A "/*" without a
// matching "*/" consumes to EOF, leaving the scanner at the end so callers
// expecting a token fail closed.
func (s *phpLiteralScanner) skipTrivia() {
for s.i < len(s.buf) {
c := s.buf[s.i]
if isPHPSpace(c) {
s.i++
continue
}
if isPHPLineCommentStart(s.buf, s.i) {
s.i = skipPHPLineComment(s.buf, s.i)
continue
}
if c == '/' && s.i+1 < len(s.buf) && s.buf[s.i+1] == '*' {
end := bytes.Index(s.buf[s.i+2:], []byte("*/"))
if end < 0 {
s.i = len(s.buf)
return
}
s.i += 2 + end + 2
continue
}
return
}
}
func (s *phpLiteralScanner) consumeOpener() bool {
const opener = "<?php"
if !bytes.HasPrefix(s.buf[s.i:], []byte(opener)) {
return false
}
s.i += len(opener)
// PHP requires a space, tab, CR or LF (or EOF) after the tag; "<?phpreturn"
// is invalid, "<?=" is the short-echo opener, and after a vertical tab or
// form feed PHP prints the whole file as text.
if s.i < len(s.buf) && !isPHPOpenTagSpace(s.buf[s.i]) {
return false
}
return true
}
func (s *phpLiteralScanner) readIdent() (string, bool) {
if s.i >= len(s.buf) || !isIdentStart(s.buf[s.i]) {
return "", false
}
start := s.i
for s.i < len(s.buf) && isIdentCont(s.buf[s.i]) {
s.i++
}
return strings.ToLower(string(s.buf[start:s.i])), true
}
// parseTopArray requires the value at the cursor to be an array literal. A bare
// scalar return (return 'x';) is not a translation cache.
func (s *phpLiteralScanner) parseTopArray() bool {
if s.i >= len(s.buf) {
return false
}
if s.buf[s.i] == '[' {
return s.parseArray(0, '[', ']')
}
if isIdentStart(s.buf[s.i]) {
save := s.i
if id, _ := s.readIdent(); id == "array" {
s.skipTrivia()
return s.parseArray(0, '(', ')')
}
s.i = save
}
return false
}
// parseValue accepts one array element value: a nested array, or a chain of
// constant scalars joined by the "." concatenation operator.
func (s *phpLiteralScanner) parseValue(depth int) bool {
s.skipTrivia()
if s.i >= len(s.buf) {
return false
}
if s.buf[s.i] == '[' {
return s.parseArray(depth, '[', ']')
}
if isIdentStart(s.buf[s.i]) {
save := s.i
if id, _ := s.readIdent(); id == "array" {
s.skipTrivia()
return s.parseArray(depth, '(', ')')
}
s.i = save
}
return s.parseConcatChain()
}
// parseArray consumes an array literal delimited by open/close. Elements are
// "value" or "value => value"; an empty array and a trailing comma are allowed.
func (s *phpLiteralScanner) parseArray(depth int, open, close byte) bool {
if depth >= wpTranslationMaxDepth {
return false
}
if s.i >= len(s.buf) || s.buf[s.i] != open {
return false
}
s.i++
for {
s.skipTrivia()
if s.i >= len(s.buf) {
return false
}
if s.buf[s.i] == close {
s.i++
return true
}
if !s.parseValue(depth + 1) {
return false
}
s.skipTrivia()
if s.i+1 < len(s.buf) && s.buf[s.i] == '=' && s.buf[s.i+1] == '>' {
s.i += 2
if !s.parseValue(depth + 1) {
return false
}
s.skipTrivia()
}
if s.i >= len(s.buf) {
return false
}
switch s.buf[s.i] {
case ',':
s.i++
case close:
s.i++
return true
default:
return false
}
}
}
func (s *phpLiteralScanner) parseConcatChain() bool {
if !s.parseScalar() {
return false
}
for {
s.skipTrivia()
if s.i < len(s.buf) && s.buf[s.i] == '.' {
s.i++
if !s.parseScalar() {
return false
}
continue
}
return true
}
}
func (s *phpLiteralScanner) parseScalar() bool {
s.skipTrivia()
if s.i >= len(s.buf) {
return false
}
c := s.buf[s.i]
switch {
case c == '\'':
return s.parseSingleQuoted()
case c == '"':
return s.parseDoubleQuoted()
case c == '-' || (c >= '0' && c <= '9'):
return s.parseInt()
case isIdentStart(c):
id, ok := s.readIdent()
return ok && (id == "true" || id == "false" || id == "null")
default:
return false
}
}
// parseSingleQuoted consumes a PHP single-quoted string. Inside one, only "\\"
// and "\'" are escapes and "$" never interpolates, so the contents are inert.
func (s *phpLiteralScanner) parseSingleQuoted() bool {
s.i++ // opening quote
for s.i < len(s.buf) {
switch s.buf[s.i] {
case '\\':
s.i += 2
case '\'':
s.i++
return true
default:
s.i++
}
}
return false // unterminated
}
// parseDoubleQuoted consumes a PHP double-quoted string. A backslash escapes
// the next byte (so "\0", "\n", "\$" are literal). An unescaped "$" introduces
// variable interpolation, which is an execution vector, so it is rejected.
func (s *phpLiteralScanner) parseDoubleQuoted() bool {
s.i++ // opening quote
for s.i < len(s.buf) {
switch s.buf[s.i] {
case '\\':
s.i += 2
case '"':
s.i++
return true
case '$':
return false
default:
s.i++
}
}
return false // unterminated
}
func (s *phpLiteralScanner) parseInt() bool {
if s.buf[s.i] == '-' {
s.i++
}
start := s.i
for s.i < len(s.buf) && s.buf[s.i] >= '0' && s.buf[s.i] <= '9' {
s.i++
}
return s.i > start
}
// isWPTranslationCache is the path-based entry point for the polled scan. It
// reads up to the same window as IsBenignPHPStub and requires the whole file to
// fit, since the recognizer must see the entire body to prove it is inert.
func isWPTranslationCache(path string) bool {
f, err := osFS.Open(path)
if err != nil {
return false
}
defer func() { _ = f.Close() }()
buf, err := io.ReadAll(io.LimitReader(f, benignPHPStubMaxScan+1))
if err != nil || len(buf) == 0 || len(buf) > benignPHPStubMaxScan {
return false
}
info, err := f.Stat()
if err != nil || info.Size() != int64(len(buf)) {
return false
}
return IsWPTranslationCacheBytesComplete(buf, true)
}
package checks
// IsWPVersionDataBytesComplete recognizes only literal assignments to the
// variables WordPress reads from its version file. Matching an installed copy
// alone cannot establish that a short-lived version probe carried no payload.
func IsWPVersionDataBytesComplete(buf []byte, complete bool) bool {
if !complete || len(buf) == 0 {
return false
}
s := &phpLiteralScanner{buf: buf}
s.skipSpace()
if !s.consumeOpener() {
return false
}
version := false
for {
s.skipTrivia()
if s.i == len(s.buf) {
return version
}
if s.buf[s.i] != '$' {
return false
}
s.i++
name, ok := s.readIdent()
if !ok {
return false
}
switch name {
case "wp_version", "wp_db_version", "tinymce_version", "required_php_version",
"required_php_extensions", "required_mysql_version", "wp_local_package":
default:
return false
}
s.skipTrivia()
if s.i == len(s.buf) || s.buf[s.i] != '=' {
return false
}
s.i++
s.skipTrivia()
if name == "required_php_extensions" {
if !s.parseTopArray() {
return false
}
} else if !s.parseScalar() {
return false
}
s.skipTrivia()
if s.i == len(s.buf) || s.buf[s.i] != ';' {
return false
}
s.i++
version = version || name == "wp_version"
}
}
package checks
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"syscall"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
)
const (
phpIniWalkMaxDepth = -1
phpIniMaxRootsPerUser = 1024
phpIniReadDirBatch = 256
phpIniStateFormat = 1
phpIniFindingCheck = "php_config_change"
phpIniIncompleteCheck = "php_config_scan_incomplete"
phpIniIncompleteDetails = "The file exceeds the PHP configuration scan limit and was not parsed."
phpIniSpecialDetails = "The PHP configuration path is not a regular file and was not parsed."
phpIniStateLockShards = 64
// PHPConfigMaxBytes is the shared scheduled and realtime read ceiling.
PHPConfigMaxBytes = 1 << 20
)
// Walk limits. Per-root limits bound one document root; the per-account
// limits bound the whole account. They are vars so tests can shrink them.
//
// The per-root allowance is what the account limits used to be, because a
// single shared budget meant a large first root (a vendor or node_modules tree
// is thousands of directories on its own) consumed everything and every later
// addon domain was reported unscanned without being walked.
var (
phpIniWalkMaxDirs = 10000
phpIniWalkMaxEntries = 250000
phpIniWalkAccountMaxDirs = 150000
phpIniWalkAccountMaxEntries = 3000000
)
var (
errPHPIniNonRegular = errors.New("PHP configuration is not a regular file")
errPHPIniTooLarge = errors.New("PHP configuration exceeds scan limit")
phpIniStateLocks [phpIniStateLockShards]sync.Mutex
)
type phpIniFileState struct {
Version int `json:"version"`
Hash string `json:"hash"`
Assessed bool `json:"assessed"`
FullAnalysis bool `json:"full_analysis,omitempty"`
}
// phpIniWalkBudget carries both the allowance for the root currently being
// walked and the running total for the account. collectPHPIniFilesWithBudget
// resets the per-root counters on entry, so roots do not starve each other.
type phpIniWalkBudget struct {
dirs int
entries int
accountDirs int
accountEntries int
// maxDirs is the operator-set per-root ceiling. It rides on the budget
// rather than on the package var so concurrent per-account scans cannot
// overwrite each other's limit. Zero falls back to the package default.
maxDirs int
maxEntries int
// limitHit distinguishes a walk stopped by one of the configured ceilings
// from one stopped by an unreadable directory. Both leave the scan
// incomplete, but only the first means coverage can be bought back by
// raising a setting, so the operator has to be told which happened.
limitKind walkLimit
}
// walkLimit identifies the bound that stopped a walk. The two ceilings are
// raised by two different settings, so naming the wrong one sends the
// operator to a knob that cannot recover the lost coverage.
type walkLimit int
const (
walkLimitNone walkLimit = iota
walkLimitDirs
walkLimitEntries
)
// phpIniIncompleteReason explains why a walk below root did not finish. A
// ceiling that stopped it is reported with the distance covered and the setting
// that raises it; anything else is left as an unreadable-entry report.
func phpIniIncompleteReason(root string, b *phpIniWalkBudget) string {
if b == nil {
return fmt.Sprintf("Could not finish scanning PHP configuration files below %s.", root)
}
switch b.limitKind {
case walkLimitDirs:
return fmt.Sprintf(
"Scanning below %s stopped after %d directories, the per-root limit, so the rest was not examined for PHP configuration files. Raise thresholds.php_config_walk_max_dirs to cover the whole account.",
root, b.dirs,
)
case walkLimitEntries:
return fmt.Sprintf(
"Scanning below %s stopped after %d entries, the per-root limit, so the rest was not examined for PHP configuration files. Raise thresholds.php_config_walk_max_entries to cover the whole account.",
root, b.entries,
)
}
return fmt.Sprintf("Could not finish scanning PHP configuration files below %s.", root)
}
// dirLimit is the per-root directory ceiling in force for this walk.
// phpIniConfiguredMaxDirs reads the operator's per-root ceiling, tolerating a
// nil config so callers that scan without one keep the built-in default.
func phpIniConfiguredMaxDirs(cfg *config.Config) int {
if cfg == nil {
return 0
}
return cfg.Thresholds.PHPConfigWalkMaxDirs
}
// phpIniConfiguredMaxEntries reads the operator's per-root entry ceiling.
func phpIniConfiguredMaxEntries(cfg *config.Config) int {
if cfg == nil {
return 0
}
return cfg.Thresholds.PHPConfigWalkMaxEntries
}
func (b *phpIniWalkBudget) dirLimit() int {
if b.maxDirs > 0 {
return b.maxDirs
}
return phpIniWalkMaxDirs
}
// entryLimit is the per-root entry ceiling in force for this walk.
func (b *phpIniWalkBudget) entryLimit() int {
if b.maxEntries > 0 {
return b.maxEntries
}
return phpIniWalkMaxEntries
}
func (b *phpIniWalkBudget) startRoot() {
b.dirs = 0
b.entries = 0
}
func (b *phpIniWalkBudget) accountExhausted() bool {
return b.accountDirs >= phpIniWalkAccountMaxDirs || b.accountEntries >= phpIniWalkAccountMaxEntries
}
func (b *phpIniWalkBudget) addDir() {
b.dirs++
b.accountDirs++
}
func (b *phpIniWalkBudget) addEntry() {
b.entries++
b.accountEntries++
}
// CheckPHPConfigChanges monitors .user.ini and php.ini files anywhere under an
// account's document roots for settings that weaken PHP security (disable_functions
// cleared or neutralized, allow_url_include enabled, open_basedir removed). It runs
// as a deep check; the fanotify watcher also catches these writes in real-time.
func CheckPHPConfigChanges(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
var findings []alert.Finding
accountScope := AccountFromContext(ctx)
var cpanelVhosts []vhost
vhostRootsComplete := true
vhostMapExpected := false
vhostData, vhostErr := osFS.ReadFile(userdataDomainsPath)
if vhostErr == nil {
vhostMapExpected = true
var complete bool
cpanelVhosts, complete = parseUserdataDomainRootsChecked(string(vhostData))
if !complete {
vhostRootsComplete = false
markCheckIncomplete(ctx, "php_config_changes")
if accountScope == "" {
findings = append(findings, phpIniScanIncompleteFinding(
"PHP configuration document-root map is incomplete",
"Some cPanel domain records were malformed, so their document roots could not be scanned.",
))
}
}
} else {
vhostRootsComplete = false
vhostMapExpected = vhostMapFailureIsIncomplete(vhostErr)
if vhostMapExpected {
markCheckIncomplete(ctx, "php_config_changes")
if accountScope == "" {
findings = append(findings, phpIniScanIncompleteFinding(
"PHP configuration document-root map is unavailable",
fmt.Sprintf("Could not read %s: %v", userdataDomainsPath, vhostErr),
))
}
}
}
homeDirs, homeErr := GetScanHomeDirs(ctx)
var users []string
userSet := make(map[string]struct{})
addUser := func(user string) {
if _, exists := userSet[user]; exists {
return
}
userSet[user] = struct{}{}
users = append(users, user)
}
if homeErr == nil {
for _, homeEntry := range homeDirs {
if homeEntry.IsDir() {
addUser(homeEntry.Name())
}
}
}
for _, vh := range cpanelVhosts {
if accountScope == "" || vh.user == accountScope {
addUser(vh.user)
}
}
if homeErr != nil &&
(!errors.Is(homeErr, os.ErrNotExist) || len(users) == 0 && vhostMapExpected) {
markCheckIncomplete(ctx, "php_config_changes")
if accountScope == "" {
findings = append(findings, phpIniScanIncompleteFinding(
"PHP configuration account discovery is incomplete",
fmt.Sprintf("Could not enumerate account home directories: %v", homeErr),
))
}
}
for _, user := range users {
if ctx.Err() != nil {
markCheckIncomplete(ctx, "php_config_changes")
return findings
}
// Collect every .user.ini and php.ini under the account's document
// roots. An attacker plants a php.ini (or a nested .user.ini) deep in
// the tree -- e.g. wp-includes/assets/php.ini -- to weaken PHP, so the
// scan cannot stop at the docroot root or watch .user.ini alone.
var incompleteReason string
recordIncomplete := func(reason string) {
markCheckIncomplete(ctx, "php_config_changes")
if incompleteReason == "" {
incompleteReason = reason
}
}
roots := []string{filepath.Join(accountHomeDir(user), "public_html")}
rootSet := map[string]struct{}{roots[0]: {}}
addRoot := func(root string) bool {
root = filepath.Clean(root)
if _, duplicate := rootSet[root]; duplicate {
return true
}
if len(roots) >= phpIniMaxRootsPerUser {
recordIncomplete(fmt.Sprintf(
"Account %s has more than %d candidate document roots.",
user,
phpIniMaxRootsPerUser,
))
return false
}
rootSet[root] = struct{}{}
roots = append(roots, root)
return true
}
hasAuthoritativeRoot := false
for _, vh := range cpanelVhosts {
if vh.user != user {
continue
}
hasAuthoritativeRoot = true
if !addRoot(vh.docroot) {
break
}
}
// Non-cPanel panels commonly keep domains below an account-owned
// top-level directory (for example domains/<name>/public_html).
// Preserve that layout as a fallback when no complete cPanel map is
// available instead of narrowing the scan to public_html.
if !hasAuthoritativeRoot || !vhostRootsComplete {
fallbackRoots, complete, err := phpIniFallbackRoots(
ctx,
user,
phpIniMaxRootsPerUser-len(roots),
phpIniWalkMaxEntries,
)
if err != nil {
recordIncomplete(fmt.Sprintf(
"Could not enumerate fallback document roots for account %s: %v",
user,
err,
))
} else {
if !complete {
recordIncomplete(fmt.Sprintf(
"Could not finish enumerating fallback document roots for account %s.",
user,
))
}
for _, root := range fallbackRoots {
if !addRoot(root) {
break
}
}
}
}
var iniPaths []string
walkBudget := &phpIniWalkBudget{
maxDirs: phpIniConfiguredMaxDirs(cfg),
maxEntries: phpIniConfiguredMaxEntries(cfg),
}
for _, root := range roots {
paths, complete := collectPHPIniFilesWithBudget(ctx, root, phpIniWalkMaxDepth, walkBudget)
if !complete {
recordIncomplete(phpIniIncompleteReason(root, walkBudget))
}
iniPaths = append(iniPaths, paths...)
}
seen := make(map[string]struct{}, len(iniPaths))
for _, iniPath := range iniPaths {
if ctx.Err() != nil {
markCheckIncomplete(ctx, "php_config_changes")
return findings
}
if _, duplicate := seen[iniPath]; duplicate {
continue
}
seen[iniPath] = struct{}{}
// Bound reads and reject special files before opening them. A user
// can create a FIFO named php.ini; reading it as an ordinary file
// would strand a scan worker after its context timed out.
dangerous, err := readAndAssessPHPIniFile(store, iniPath)
if err != nil {
switch {
case errors.Is(err, os.ErrNotExist):
// The candidate vanished after directory enumeration. A
// later stable scan can confirm removal; this run cannot
// safely clear an earlier finding.
markCheckIncomplete(ctx, "php_config_changes")
case errors.Is(err, errPHPIniNonRegular):
markCheckIncomplete(ctx, "php_config_changes")
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: phpIniFindingCheck,
Message: fmt.Sprintf("Special file used as PHP configuration: %s (user: %s)", iniPath, user),
Details: phpIniSpecialDetails,
FilePath: iniPath,
})
case errors.Is(err, errPHPIniTooLarge):
markCheckIncomplete(ctx, "php_config_changes")
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: phpIniFindingCheck,
Message: fmt.Sprintf("PHP configuration too large to inspect: %s (user: %s)", iniPath, user),
Details: phpIniIncompleteDetails,
FilePath: iniPath,
})
default:
recordIncomplete(fmt.Sprintf(
"Could not inspect PHP configuration %s: %v",
iniPath,
err,
))
}
continue
}
if len(dangerous) > 0 {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: phpIniFindingCheck,
Message: fmt.Sprintf("Dangerous PHP configuration: %s (user: %s)", iniPath, user),
Details: fmt.Sprintf("Dangerous settings:\n- %s", strings.Join(dangerous, "\n- ")),
FilePath: iniPath,
})
}
}
if incompleteReason != "" && ctx.Err() == nil {
findings = append(findings, phpIniScanIncompleteFinding(
fmt.Sprintf("PHP configuration scan incomplete for user: %s", user),
incompleteReason,
))
}
}
return findings
}
func phpIniScanIncompleteFinding(message, details string) alert.Finding {
return alert.Finding{
Severity: alert.Warning,
Check: phpIniIncompleteCheck,
Message: message,
Details: details,
}
}
func phpIniFallbackRoots(
ctx context.Context,
user string,
maxRoots int,
maxEntries int,
) ([]string, bool, error) {
if maxRoots <= 0 || maxEntries <= 0 {
return nil, false, nil
}
home := accountHomeDir(user)
var roots []string
entriesSeen := 0
complete, err := forEachPHPIniDirEntry(ctx, home, func(entry os.DirEntry) bool {
entriesSeen++
if entriesSeen > maxEntries {
return false
}
name := entry.Name()
if !entry.IsDir() ||
name == "public_html" ||
name == "mail" ||
name == "etc" ||
name == "logs" ||
name == "ssl" ||
name == "tmp" ||
strings.HasPrefix(name, ".") {
return true
}
if len(roots) >= maxRoots {
return false
}
roots = append(roots, filepath.Join(home, name))
return true
})
if err != nil {
return roots, false, err
}
return roots, complete, nil
}
func readPHPIniFile(path string) ([]byte, error) {
info, err := osFS.Stat(path)
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() {
return nil, errPHPIniNonRegular
}
if info.Size() > PHPConfigMaxBytes {
return nil, errPHPIniTooLarge
}
var f *os.File
var openErr error
_, productionFS := osFS.(realOS)
if productionFS {
// A non-blocking open prevents a regular-file-to-FIFO swap from
// stranding the worker between the Stat above and this open.
// #nosec G304 -- read-only candidate discovered below an account web root.
f, openErr = os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0)
} else {
f, openErr = osFS.Open(path)
}
if openErr == nil {
defer func() { _ = f.Close() }()
openedInfo, statErr := f.Stat()
if statErr != nil {
return nil, statErr
}
if !openedInfo.Mode().IsRegular() {
return nil, errPHPIniNonRegular
}
data, readErr := io.ReadAll(io.LimitReader(f, PHPConfigMaxBytes+1))
if readErr != nil {
return nil, readErr
}
if len(data) > PHPConfigMaxBytes {
return nil, errPHPIniTooLarge
}
return data, nil
}
if productionFS {
return nil, openErr
}
// Map-backed providers cannot return an *os.File, so their bounded test
// data uses the ReadFile hook.
data, readErr := osFS.ReadFile(path)
if readErr != nil {
return nil, readErr
}
if len(data) > PHPConfigMaxBytes {
return nil, errPHPIniTooLarge
}
return data, nil
}
func readAndAssessPHPIniFile(store *state.Store, path string) ([]string, error) {
// The read and state update are one critical section. Concurrent host and
// account scans must not let an older file snapshot overwrite the state
// recorded for a newer snapshot. Locks are sharded by path so unrelated
// account scans can still progress in parallel.
lock := phpIniStateLock(path)
lock.Lock()
defer lock.Unlock()
data, err := readPHPIniFile(path)
if err != nil {
return nil, err
}
return assessPHPIniFile(store, path, hashBytes(data), string(data)), nil
}
func phpIniStateLock(path string) *sync.Mutex {
var hash uint32 = 2166136261
for i := 0; i < len(path); i++ {
hash ^= uint32(path[i])
hash *= 16777619
}
return &phpIniStateLocks[hash%phpIniStateLockShards]
}
// assessPHPIniFile stores the content hash and whether that content needs the
// full change analysis. Re-running the selected analysis for unchanged files
// keeps active findings visible and automatically applies future parser fixes.
// The caller holds the path's state lock so the state update stays ordered
// with its file snapshot.
func assessPHPIniFile(store *state.Store, path, hash, content string) []string {
key := "_phpini:" + path
raw, exists := store.GetRaw(key)
previous := decodePHPIniFileState(raw)
if exists && previous.Assessed && previous.Hash == hash {
if previous.FullAnalysis {
return analyzePHPINI(content)
}
return PHPConfigSecurityBypasses(content)
}
fullAnalysis := exists && previous.Hash != "" && previous.Hash != hash
var dangerous []string
if fullAnalysis {
dangerous = analyzePHPINI(content)
} else {
// New files and legacy hash-only state are judged on the strong,
// low-false-positive bypass signals.
dangerous = PHPConfigSecurityBypasses(content)
}
next := phpIniFileState{
Version: phpIniStateFormat,
Hash: hash,
Assessed: true,
FullAnalysis: fullAnalysis,
}
encoded, err := json.Marshal(next)
if err != nil {
panic(err)
}
store.SetRaw(key, string(encoded))
return append([]string(nil), dangerous...)
}
func decodePHPIniFileState(raw string) phpIniFileState {
var decoded phpIniFileState
if err := json.Unmarshal([]byte(raw), &decoded); err == nil &&
decoded.Version == phpIniStateFormat && decoded.Hash != "" {
return decoded
}
return phpIniFileState{Hash: raw}
}
func analyzePHPINI(content string) []string {
var dangerous []string
seen := make(map[string]bool)
appendFinding := func(message string) {
if !seen[message] {
seen[message] = true
dangerous = append(dangerous, message)
}
}
for _, directives := range parsePHPIniDirectiveSets(content) {
if val, ok := directives["disable_functions"]; ok {
disabled := disabledPHPFunctions(val)
if len(disabled) == 0 {
appendFinding("disable_functions cleared or neutralized (all PHP functions enabled)")
}
for _, fn := range dangerousPHPFunctions {
if !disabled[fn] {
appendFinding(fmt.Sprintf("%s not in disable_functions", fn))
}
}
}
if val, ok := directives["allow_url_include"]; ok && phpIniBoolEnabled(val) {
appendFinding("allow_url_include enabled (remote code inclusion)")
}
if val, ok := directives["open_basedir"]; ok && openBasedirUnrestricted(val) {
appendFinding("open_basedir cleared or set to / (no restriction)")
}
}
return dangerous
}
// collectPHPIniFilesContext walks root up to maxDepth and returns every
// .user.ini and php.ini path found. A negative maxDepth walks all depths. No
// directory is skipped by name -- attackers hide these under node_modules and
// vendor trees. Total directory and entry limits keep a user-controlled tree
// from occupying a scan worker indefinitely.
func collectPHPIniFilesContext(ctx context.Context, root string, maxDepth int) ([]string, bool) {
return collectPHPIniFilesWithBudget(ctx, root, maxDepth, &phpIniWalkBudget{})
}
func collectPHPIniFilesWithBudget(
ctx context.Context,
root string,
maxDepth int,
budget *phpIniWalkBudget,
) ([]string, bool) {
type pendingDir struct {
path string
depth int
observed bool
}
if budget.accountExhausted() {
budget.limitKind = walkLimitDirs
return nil, false
}
budget.startRoot()
var out []string
queue := []pendingDir{{path: root}}
budget.addDir()
complete := true
for len(queue) > 0 {
if ctx.Err() != nil {
return out, false
}
dir := queue[0]
queue = queue[1:]
visitComplete, err := forEachPHPIniDirEntry(ctx, dir.path, func(e os.DirEntry) bool {
budget.addEntry()
if budget.entries > budget.entryLimit() || budget.accountEntries > phpIniWalkAccountMaxEntries {
budget.limitKind = walkLimitEntries
return false
}
name := e.Name()
full := filepath.Join(dir.path, name)
if e.IsDir() {
if maxDepth < 0 || dir.depth < maxDepth {
if budget.dirs >= budget.dirLimit() || budget.accountDirs >= phpIniWalkAccountMaxDirs {
complete = false
budget.limitKind = walkLimitDirs
return true
}
queue = append(queue, pendingDir{path: full, depth: dir.depth + 1, observed: true})
budget.addDir()
}
return true
}
if name == ".user.ini" || name == "php.ini" {
out = append(out, full)
}
return true
})
if err != nil {
if !errors.Is(err, os.ErrNotExist) || dir.observed {
complete = false
}
continue
}
if !visitComplete {
return out, false
}
}
return out, complete
}
func forEachPHPIniDirEntry(
ctx context.Context,
path string,
visit func(os.DirEntry) bool,
) (bool, error) {
if ctx.Err() != nil {
return false, ctx.Err()
}
if _, productionFS := osFS.(realOS); !productionFS {
entries, err := osFS.ReadDir(path)
if err != nil {
return false, err
}
for _, entry := range entries {
if ctx.Err() != nil {
return false, ctx.Err()
}
if !visit(entry) {
return false, nil
}
}
return true, nil
}
// Stream directory entries in bounded batches. os.ReadDir loads the whole
// directory, which lets one account force a large allocation before the
// entry budget can stop the scan.
// #nosec G304 -- directory discovered below an account web root.
dir, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0)
if err != nil {
return false, err
}
defer func() { _ = dir.Close() }()
info, err := dir.Stat()
if err != nil {
return false, err
}
if !info.IsDir() {
return false, syscall.ENOTDIR
}
for {
entries, readErr := dir.ReadDir(phpIniReadDirBatch)
for _, entry := range entries {
if ctx.Err() != nil {
return false, ctx.Err()
}
if !visit(entry) {
return false, nil
}
}
switch {
case errors.Is(readErr, io.EOF):
return true, nil
case readErr != nil:
return false, readErr
}
}
}
// PHPConfigSecurityBypasses returns the strong, low-false-positive signals that a
// PHP ini file weakens security: disable_functions cleared or neutralized,
// allow_url_include enabled, or open_basedir removed. Used for newly-seen
// files, where the noisier per-function diffing of analyzePHPINI would flag
// benign partial disable lists.
func PHPConfigSecurityBypasses(content string) []string {
var out []string
var disableFunctions, allowURLInclude, openBasedir bool
for _, directives := range parsePHPIniDirectiveSets(content) {
if val, ok := directives["disable_functions"]; ok && DisableFunctionsNeutralized(val) {
disableFunctions = true
}
if val, ok := directives["allow_url_include"]; ok && phpIniBoolEnabled(val) {
allowURLInclude = true
}
if val, ok := directives["open_basedir"]; ok && openBasedirUnrestricted(val) {
openBasedir = true
}
}
if disableFunctions {
out = append(out, "disable_functions cleared or neutralized (dangerous PHP functions enabled)")
}
if allowURLInclude {
out = append(out, "allow_url_include enabled (remote code inclusion)")
}
if openBasedir {
out = append(out, "open_basedir cleared (filesystem sandbox removed)")
}
return out
}
// PHPConfigRealtimeRootPatterns returns the startup-time path patterns used to
// admit php.ini writes into the fanotify analyzer. Explicit account_roots are
// authoritative. On cPanel, account homes are intentionally broader than the
// primary public_html root because addon domains can live anywhere below an
// account and on alternate home mounts.
func PHPConfigRealtimeRootPatterns(cfg *config.Config) []string {
if cfg != nil && len(cfg.AccountRoots) > 0 {
return WebRootPatterns(cfg)
}
if len(WebRootPatterns(cfg)) == 0 {
return nil
}
patterns := accountHomePatterns()
seen := make(map[string]struct{}, len(patterns))
for _, p := range patterns {
seen[p] = struct{}{}
}
data, err := osFS.ReadFile(userdataDomainsPath)
if err != nil {
return patterns
}
vhosts, _ := parseUserdataDomainRootsChecked(string(data))
for _, vhost := range vhosts {
pattern := phpConfigRealtimeRootPattern(vhost.user, vhost.docroot)
if pattern == "" {
continue
}
if _, exists := seen[pattern]; exists {
continue
}
seen[pattern] = struct{}{}
patterns = append(patterns, pattern)
}
return patterns
}
// RealtimeDocumentRootPatterns returns the served roots used by realtime
// content detectors. cPanel's domain map is authoritative for addon domains
// and accounts on alternate home mounts; the conventional public_html glob
// remains the fallback when that map is unavailable.
func RealtimeDocumentRootPatterns(cfg *config.Config) []string {
patterns := WebRootPatterns(cfg)
if (cfg != nil && len(cfg.AccountRoots) > 0) || !platform.Detect().IsCPanel() {
return patterns
}
data, err := osFS.ReadFile(userdataDomainsPath)
if err != nil {
return patterns
}
vhosts, _ := parseUserdataDomainRootsChecked(string(data))
seen := make(map[string]struct{}, len(patterns)+len(vhosts))
for _, pattern := range patterns {
seen[filepath.Clean(pattern)] = struct{}{}
}
for _, vhost := range vhosts {
root := filepath.Clean(vhost.docroot)
if _, exists := seen[root]; exists {
continue
}
seen[root] = struct{}{}
patterns = append(patterns, root)
}
return patterns
}
func phpConfigRealtimeRootPattern(user, docroot string) string {
clean := filepath.Clean(docroot)
if !filepath.IsAbs(clean) {
return ""
}
parts := strings.Split(strings.TrimPrefix(clean, string(filepath.Separator)), string(filepath.Separator))
if len(parts) >= 2 && strings.HasPrefix(parts[0], "home") && parts[1] == user {
return filepath.Join(string(filepath.Separator), parts[0], "*")
}
return clean
}
var dangerousPHPFunctions = []string{
"exec",
"system",
"passthru",
"shell_exec",
"popen",
"proc_open",
"pcntl_exec",
}
// DisableFunctionsNeutralized reports whether a disable_functions value fails
// to actually disable any dangerous function -- empty, "none", or set to junk
// (the `disable_functions=ByPassed By 0xNix` camouflage). A genuine hardening
// list names at least one exact dangerous function; substrings such as
// "systemd" do not disable system().
func DisableFunctionsNeutralized(val string) bool {
return len(disabledPHPFunctions(val)) == 0
}
func disabledPHPFunctions(val string) map[string]bool {
disabled := make(map[string]bool)
val = strings.ToLower(normalizePHPIniValue(val))
items := strings.FieldsFunc(val, func(r rune) bool {
return r == ',' || r == ' '
})
for _, name := range items {
for _, dangerous := range dangerousPHPFunctions {
if name == dangerous {
disabled[dangerous] = true
break
}
}
}
return disabled
}
func parsePHPIniDirectiveSets(content string) []map[string]string {
sets := []map[string]string{{}}
sections := make(map[string]int)
current := 0
for _, raw := range strings.Split(content, "\n") {
line := strings.TrimSpace(raw)
line = strings.TrimPrefix(line, "\ufeff")
if line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") {
continue
}
if strings.HasPrefix(line, "[") {
end := strings.IndexByte(line, ']')
if end <= 1 {
continue
}
section := strings.TrimSpace(line[1:end])
upperSection := strings.ToUpper(section)
var sectionKey string
switch {
case strings.HasPrefix(upperSection, "PATH="):
sectionKey = "PATH=" + strings.TrimSpace(section[len("PATH="):])
case strings.HasPrefix(upperSection, "HOST="):
sectionKey = "HOST=" + strings.ToLower(strings.TrimSpace(section[len("HOST="):]))
default:
// Ordinary php.ini section headings are ignored by PHP and
// return parsing to the global directive set. Only PATH and
// HOST headings create conditional settings.
current = 0
continue
}
if index, exists := sections[sectionKey]; exists {
current = index
} else {
current = len(sets)
sections[sectionKey] = current
sets = append(sets, make(map[string]string))
}
continue
}
parts := strings.SplitN(line, "=", 2)
if len(parts) != 2 {
continue
}
key := strings.ToLower(strings.TrimSpace(parts[0]))
switch key {
case "disable_functions", "allow_url_include", "open_basedir":
sets[current][key] = parts[1]
}
}
return sets
}
func normalizePHPIniValue(value string) string {
value, _ = normalizePHPIniValueWithExpression(value)
return value
}
func normalizePHPIniValueWithExpression(value string) (string, bool) {
value = strings.TrimSpace(stripPHPIniInlineComment(value))
joined, expression := joinPHPIniQuotedFragments(value)
return strings.TrimSpace(joined), expression
}
// joinPHPIniQuotedFragments mirrors the INI scanner's concatenation of quoted
// and unquoted fragments. For example, ex"ec",sys"tem" is the effective value
// exec,system. An unclosed quote makes PHP reject the value; return empty so
// security restrictions fail closed instead of treating the malformed token
// as an effective hardening value.
func joinPHPIniQuotedFragments(value string) (string, bool) {
var out strings.Builder
out.Grow(len(value))
var quote byte
expression := false
for i := 0; i < len(value); i++ {
ch := value[i]
if quote == 0 {
if ch == '"' || ch == '\'' {
quote = ch
continue
}
if strings.ContainsRune("|&^~!()", rune(ch)) {
expression = true
}
out.WriteByte(ch)
continue
}
if ch == quote {
quote = 0
continue
}
if ch == '\\' && i+1 < len(value) {
next := value[i+1]
if next == quote || next == '\\' {
out.WriteByte(next)
i++
continue
}
}
out.WriteByte(ch)
}
if quote != 0 {
return "", false
}
return out.String(), expression
}
func stripPHPIniInlineComment(value string) string {
var quote byte
escaped := false
for i := 0; i < len(value); i++ {
ch := value[i]
if quote != 0 {
if quote == '"' && ch == '\\' && !escaped {
escaped = true
continue
}
if ch == quote && !escaped {
quote = 0
}
escaped = false
continue
}
if ch == '"' || ch == '\'' {
quote = ch
continue
}
if ch == ';' {
return value[:i]
}
}
return value
}
func phpIniBoolEnabled(value string) bool {
value, expression := normalizePHPIniValueWithExpression(value)
value = strings.ToLower(value)
switch value {
case "1", "on", "true", "yes":
return true
}
if expression {
evaluated, ok := parsePHPIniIntExpression(value)
// A syntactically unusual expression is not a trustworthy disabled
// value. Treat it as enabled rather than let unsupported constants or
// excessive nesting bypass the security check.
return !ok || evaluated != 0
}
// PHP converts the leading signed integer portion to bool: "1foo" and
// "1e-2" are enabled, while "0.5" is not.
i := 0
if i < len(value) && (value[i] == '+' || value[i] == '-') {
i++
}
start := i
nonZero := false
for i < len(value) && value[i] >= '0' && value[i] <= '9' {
nonZero = nonZero || value[i] != '0'
i++
}
return i > start && nonZero
}
const phpIniExpressionMaxDepth = 64
type phpIniIntExprParser struct {
input string
pos int
limitExceeded bool
}
func parsePHPIniIntExpression(value string) (int64, bool) {
p := phpIniIntExprParser{input: value}
result, ok := p.parseBinary(0)
if p.limitExceeded {
return 1, true
}
p.skipSpace()
return result, ok && p.pos == len(p.input)
}
func (p *phpIniIntExprParser) parseBinary(depth int) (int64, bool) {
left, ok := p.parseUnary(depth)
if !ok {
return 0, false
}
for {
p.skipSpace()
if p.pos >= len(p.input) {
return left, true
}
op := p.input[p.pos]
if op != '|' && op != '&' && op != '^' {
return left, true
}
p.pos++
right, valid := p.parseUnary(depth)
if !valid {
return 0, false
}
switch op {
case '|':
left |= right
case '&':
left &= right
case '^':
left ^= right
}
}
}
func (p *phpIniIntExprParser) parseUnary(depth int) (int64, bool) {
if depth >= phpIniExpressionMaxDepth {
p.limitExceeded = true
return 0, false
}
p.skipSpace()
switch {
case p.take('!'):
value, ok := p.parseUnary(depth + 1)
if !ok {
return 0, false
}
if value == 0 {
return 1, true
}
return 0, true
case p.take('~'):
value, ok := p.parseUnary(depth + 1)
return ^value, ok
case p.take('+'):
return p.parseUnary(depth + 1)
case p.take('-'):
value, ok := p.parseUnary(depth + 1)
return -value, ok
case p.take('('):
value, ok := p.parseBinary(depth + 1)
p.skipSpace()
if !ok || !p.take(')') {
return 0, false
}
return value, true
default:
return p.parseDecimal()
}
}
func (p *phpIniIntExprParser) parseDecimal() (int64, bool) {
p.skipSpace()
start := p.pos
var value uint64
const maxInt64 = uint64(^uint64(0) >> 1)
for p.pos < len(p.input) {
ch := p.input[p.pos]
if ch < '0' || ch > '9' {
break
}
digit := uint64(ch - '0')
if value > (maxInt64-digit)/10 {
value = maxInt64
} else {
value = value*10 + digit
}
p.pos++
}
return int64(value), p.pos > start
}
func (p *phpIniIntExprParser) skipSpace() {
for p.pos < len(p.input) {
switch p.input[p.pos] {
case ' ', '\t', '\r', '\n', '\v', '\f':
p.pos++
default:
return
}
}
}
func (p *phpIniIntExprParser) take(ch byte) bool {
if p.pos >= len(p.input) || p.input[p.pos] != ch {
return false
}
p.pos++
return true
}
func openBasedirUnrestricted(value string) bool {
value = normalizePHPIniValue(value)
if value == "" {
return true
}
for _, dir := range strings.Split(value, string(os.PathListSeparator)) {
if filepath.Clean(strings.TrimSpace(dir)) == string(filepath.Separator) {
return true
}
}
return false
}
package checks
import (
"path/filepath"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/contenttype"
)
// PHPExecutionOverlay is an immutable snapshot of inherited .htaccess PHP
// handler mappings for one directory. Realtime monitors can cache it and test
// arbitrary filenames without duplicating the periodic scanner's parser.
type PHPExecutionOverlay struct {
overlay phpHandlerOverlay
}
// ResolvePHPExecutionOverlay reconstructs the PHP handler mappings inherited
// by fileDir from docroot through the directory's own .htaccess file.
func ResolvePHPExecutionOverlay(docroot, fileDir string) PHPExecutionOverlay {
return PHPExecutionOverlay{overlay: reconstructOverlay(docroot, fileDir)}
}
// Executes reports whether nameLower is handled as PHP by this overlay.
func (o PHPExecutionOverlay) Executes(nameLower string) bool {
return o.overlay.executes(nameLower)
}
// isExecutablePHPName is the stock-handler extension gate; per-directory
// .htaccess handler remappings are layered on top via phpHandlerOverlay.
func isExecutablePHPName(nameLower string) bool {
return contenttype.IsExecutablePHPName(nameLower)
}
// phpHandlerOverlay carries the extra PHP execution mappings discovered from
// .htaccess files while walking a directory tree. Apache merges a parent
// directory's directives into its children, so the overlay accumulates down
// the recursion: a mapping declared in a parent applies to every descendant.
type phpHandlerOverlay struct {
// exts holds extra ".ext" entries (lowercase, leading dot) that a local
// AddHandler/AddType maps to a PHP handler, e.g. "AddHandler
// application/x-httpd-php .inc".
exts map[string]struct{}
// names holds exact lowercase basenames matched by a <Files> container.
names map[string]struct{}
// scanAll is set when a SetHandler/ForceType routes the PHP interpreter
// for the whole directory with no extension filter. Every file in the
// subtree then executes as PHP and must be content-analysed.
scanAll bool
// unrestricted marks a <FilesMatch> context whose pattern selects files
// by name rather than by extension ("logo", "^(config|data)$", or any
// alternative without a "\." marker). A PHP handler inside it executes
// files no extension list describes, so it is treated like a
// directory-wide handler.
unrestricted bool
}
func (o phpHandlerOverlay) active() bool {
return o.scanAll || len(o.exts) > 0 || len(o.names) > 0
}
// executes reports whether a file named nameLower (lowercased) runs as PHP
// under this overlay, either by a stock extension, a directory-wide handler,
// or an .htaccess-mapped extension.
func (o phpHandlerOverlay) executes(nameLower string) bool {
if o.scanAll {
return true
}
if isExecutablePHPName(nameLower) {
return true
}
if _, ok := o.names[nameLower]; ok {
return true
}
for ext := range o.exts {
if strings.HasSuffix(nameLower, ext) {
return true
}
}
return false
}
// mergeHtaccess returns a new overlay combining the receiver (inherited from
// the parent directory) with any PHP handler directives found in the .htaccess
// at dirHtaccessContent. The receiver is never mutated, so sibling directories
// do not see each other's mappings.
func (o phpHandlerOverlay) mergeHtaccess(content []byte) phpHandlerOverlay {
if len(content) == 0 {
return o
}
parsed := parsePHPHandlerDirectives(content)
if !parsed.active() {
return o
}
merged := phpHandlerOverlay{scanAll: o.scanAll || parsed.scanAll}
if len(o.exts) > 0 || len(parsed.exts) > 0 {
merged.exts = make(map[string]struct{}, len(o.exts)+len(parsed.exts))
for e := range o.exts {
merged.exts[e] = struct{}{}
}
for e := range parsed.exts {
merged.exts[e] = struct{}{}
}
}
if len(o.names) > 0 || len(parsed.names) > 0 {
merged.names = make(map[string]struct{}, len(o.names)+len(parsed.names))
for name := range o.names {
merged.names[name] = struct{}{}
}
for name := range parsed.names {
merged.names[name] = struct{}{}
}
}
return merged
}
// parsePHPHandlerDirectives extracts extension-to-PHP mappings from .htaccess
// content. It recognises:
//
// AddHandler <php-handler> .ext [.ext...]
// AddType <php-mime> .ext [.ext...]
// SetHandler <php-handler> (no extension -> whole directory)
// ForceType <php-mime> (no extension -> whole directory)
//
// A directive counts as PHP when its handler/MIME token names PHP directly or
// routes through proxy-fcgi. x-httpd-php-source is excluded because it renders
// highlighted source instead of executing. Matching is case-insensitive.
func parsePHPHandlerDirectives(content []byte) phpHandlerOverlay {
var overlay phpHandlerOverlay
var contexts []phpHandlerOverlay
for _, logical := range joinHtaccessContinuations(strings.Split(string(content), "\n")) {
line := strings.TrimSpace(logical.text)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if ctx, ok := openPHPHandlerContext(line); ok {
contexts = append(contexts, ctx)
continue
}
if closesPHPHandlerContext(line) {
if len(contexts) > 0 {
contexts = contexts[:len(contexts)-1]
}
continue
}
fields := apacheDirectiveFields(line)
if len(fields) < 2 {
continue
}
directive := strings.ToLower(fields[0])
handler := fields[1]
switch directive {
case "addhandler", "addtype":
if !handlerIsPHP(handler) {
continue
}
// Remaining fields are extensions.
addExtensions(&overlay, normalizedExts(fields[2:]))
if len(fields) == 2 {
mergeContext(&overlay, contexts)
}
case "sethandler", "forcetype":
if !handlerIsPHP(handler) {
continue
}
exts := normalizedExts(fields[2:])
if len(exts) > 0 {
addExtensions(&overlay, exts)
continue
}
if mergeContext(&overlay, contexts) {
continue
}
if len(contexts) > 0 {
continue
}
// No extension or file filter: the handler applies to every
// file in this directory.
overlay.scanAll = true
}
}
return overlay
}
func handlerIsPHP(token string) bool {
token = strings.ToLower(strings.Trim(strings.TrimSpace(token), `"'`))
// The source viewer renders highlighted source instead of executing.
if strings.Contains(token, "php-source") {
return false
}
if strings.Contains(token, "php") {
return true
}
// cPanel/PHP-FPM .htaccess wiring can use a custom socket alias whose
// path does not include the literal "php". A proxy-fcgi handler still
// routes matching files to an executable backend, so remapped extensions
// must be treated as PHP-executed for scanning and .htaccess alerts.
return strings.HasPrefix(token, "proxy:") && strings.Contains(token, "fcgi://")
}
// normalizeExt turns an .htaccess extension token into a lowercase
// leading-dot extension, rejecting anything that is not a plain extension
// token (e.g. a stray flag or MIME fragment).
func normalizeExt(token string) string {
t := strings.ToLower(strings.Trim(strings.TrimSpace(token), `"'`))
t = strings.TrimPrefix(t, ".")
if t == "" {
return ""
}
for _, r := range t {
alnum := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9')
if !alnum && r != '_' && r != '-' {
return ""
}
}
return "." + t
}
// htaccessLogicalLine is one Apache directive after joining physical
// continuation lines. text is the joined directive (continuation backslashes
// removed); lines are the original physical lines it spans, so a per-line
// rewrite can drop or keep them together. start is the 0-based index of the
// first physical line.
type htaccessLogicalLine struct {
text string
lines []string
start int
}
// joinHtaccessContinuations groups physical .htaccess lines into logical
// directives, honoring Apache's trailing-backslash line continuation: a line
// ending in "\" is joined with the next. Without this, a directive split as
// "AddHandler ...php \" + ".jpg" reads as two harmless physical lines and
// every per-line scanner misses the remap.
func joinHtaccessContinuations(physical []string) []htaccessLogicalLine {
var out []htaccessLogicalLine
for i := 0; i < len(physical); {
start := i
var sb strings.Builder
var span []string
for {
cur := physical[i]
span = append(span, cur)
body, continues := htaccessContinuationBody(cur, i < len(physical)-1)
if continues {
sb.WriteString(body)
i++
continue
}
sb.WriteString(body)
break
}
out = append(out, htaccessLogicalLine{text: sb.String(), lines: span, start: start})
i++
}
return out
}
func htaccessContinuationBody(line string, hasNext bool) (string, bool) {
// On CRLF input (strings.Split keeps the trailing "\r") the continuation
// backslash sits before the carriage return, so strip it before the suffix
// test and from the joined text.
body := strings.TrimRight(line, "\r")
// A trailing backslash continues only when a next line exists; a backslash on
// the final line is left literal, matching Apache.
if !hasNext || !strings.HasSuffix(body, `\`) {
return body, false
}
if htaccessContainerLineKeepsTrailingBackslash(body) {
return body, false
}
return body[:len(body)-1], true
}
func htaccessContainerLineKeepsTrailingBackslash(body string) bool {
candidate := strings.TrimSpace(strings.TrimSuffix(body, `\`))
if !strings.Contains(candidate, ">") {
return false
}
if _, ok := openPHPHandlerContext(candidate); ok {
return true
}
return closesPHPHandlerContext(candidate)
}
func apacheDirectiveFields(line string) []string {
fields := strings.Fields(line)
for i, field := range fields {
if strings.HasPrefix(field, "#") {
return fields[:i]
}
}
return fields
}
func normalizedExts(tokens []string) []string {
var exts []string
for _, token := range tokens {
if ext := normalizeExt(token); ext != "" {
exts = append(exts, ext)
}
}
return exts
}
func addExtensions(overlay *phpHandlerOverlay, exts []string) {
if len(exts) == 0 {
return
}
for _, ext := range exts {
if isExecutablePHPName("x" + ext) {
continue
}
if overlay.exts == nil {
overlay.exts = make(map[string]struct{}, len(exts))
}
overlay.exts[ext] = struct{}{}
}
}
func mergeContext(overlay *phpHandlerOverlay, contexts []phpHandlerOverlay) bool {
merged := false
for _, ctx := range contexts {
if ctx.unrestricted {
// The container selects files by name: the handler can reach
// any file in the directory, so every file must be analysed.
overlay.scanAll = true
merged = true
}
if len(ctx.exts) > 0 {
if overlay.exts == nil {
overlay.exts = make(map[string]struct{}, len(ctx.exts))
}
for ext := range ctx.exts {
overlay.exts[ext] = struct{}{}
merged = true
}
}
if len(ctx.names) > 0 {
if overlay.names == nil {
overlay.names = make(map[string]struct{}, len(ctx.names))
}
for name := range ctx.names {
overlay.names[name] = struct{}{}
merged = true
}
}
}
return merged
}
func openPHPHandlerContext(line string) (phpHandlerOverlay, bool) {
lower := strings.ToLower(line)
switch {
case strings.HasPrefix(lower, "<filesmatch"):
pattern := apacheContainerArgument(line)
return overlayForFilesMatch(pattern), true
case strings.HasPrefix(lower, "<files"):
name := apacheContainerArgument(line)
name = strings.ToLower(strings.TrimSpace(name))
if name == "" || strings.Contains(name, "/") {
return phpHandlerOverlay{}, true
}
if strings.ContainsAny(name, "*?[") {
overlay := phpHandlerOverlay{}
addExtensions(&overlay, []string{filepath.Ext(name)})
return overlay, true
}
overlay := phpHandlerOverlay{names: map[string]struct{}{name: {}}}
return overlay, true
default:
return phpHandlerOverlay{}, false
}
}
func closesPHPHandlerContext(line string) bool {
lower := strings.ToLower(line)
return strings.HasPrefix(lower, "</filesmatch") || strings.HasPrefix(lower, "</files")
}
func apacheContainerArgument(line string) string {
start := strings.IndexAny(line, " \t")
end := strings.LastIndexByte(line, '>')
if start < 0 || end <= start {
return ""
}
arg := strings.TrimSpace(line[start:end])
if len(arg) >= 2 {
quote := arg[0]
if (quote == '"' || quote == '\'') && arg[len(arg)-1] == quote {
arg = arg[1 : len(arg)-1]
}
}
return arg
}
func overlayForFilesMatch(pattern string) phpHandlerOverlay {
overlay := phpHandlerOverlay{}
addExtensions(&overlay, extensionsFromFilesMatchPattern(pattern))
overlay.unrestricted = filesMatchSelectsByName(pattern)
return overlay
}
// filesMatchSelectsByName reports whether a FilesMatch pattern can match a
// filename without requiring a literal dot. An empty or unsupported pattern
// is treated as unrestricted so the content scanner fails toward coverage.
func filesMatchSelectsByName(pattern string) bool {
pattern = strings.TrimSpace(pattern)
if pattern == "" {
return true
}
p := filesMatchRegexParser{pattern: pattern}
requiresDot, valid := p.alternation(0)
return !valid || p.pos != len(pattern) || !requiresDot
}
type filesMatchRegexParser struct {
pattern string
pos int
}
// alternation returns true only when every branch requires a literal dot.
// That conservative proof is enough to keep extension-only handlers narrow;
// regex constructs outside the small parser fall back to scanning all names.
func (p *filesMatchRegexParser) alternation(stop byte) (bool, bool) {
allRequireDot := true
for {
requiresDot, valid := p.sequence(stop)
if !valid {
return false, false
}
allRequireDot = allRequireDot && requiresDot
if p.pos < len(p.pattern) && p.pattern[p.pos] == '|' {
p.pos++
continue
}
if stop != 0 {
if p.pos >= len(p.pattern) || p.pattern[p.pos] != stop {
return false, false
}
p.pos++
}
return allRequireDot, true
}
}
func (p *filesMatchRegexParser) sequence(stop byte) (bool, bool) {
requiresDot := false
for p.pos < len(p.pattern) {
if p.pattern[p.pos] == '|' || (stop != 0 && p.pattern[p.pos] == stop) {
break
}
atomRequiresDot, valid := p.atom()
if !valid {
return false, false
}
if p.pos < len(p.pattern) {
switch p.pattern[p.pos] {
case '*', '?':
atomRequiresDot = false
p.pos++
case '+':
p.pos++
case '{':
minimum, next, ok := regexRepeatMinimum(p.pattern, p.pos)
if !ok {
return false, false
}
p.pos = next
if minimum == 0 {
atomRequiresDot = false
}
}
}
requiresDot = requiresDot || atomRequiresDot
}
return requiresDot, true
}
func (p *filesMatchRegexParser) atom() (bool, bool) {
if p.pos >= len(p.pattern) {
return false, false
}
c := p.pattern[p.pos]
p.pos++
switch c {
case '\\':
if p.pos >= len(p.pattern) {
return false, false
}
escaped := p.pattern[p.pos]
p.pos++
return escaped == '.', true
case '[':
return p.characterClass()
case '(':
parseRequirement := true
if p.pos < len(p.pattern) && p.pattern[p.pos] == '?' {
p.pos++
switch {
case p.pos < len(p.pattern) && p.pattern[p.pos] == ':':
p.pos++
case p.regexFlagGroup():
default:
// Lookarounds, named groups and other PCRE extensions are
// parsed for balance but not used as a coverage proof.
parseRequirement = false
}
}
requiresDot, valid := p.alternation(')')
return parseRequirement && requiresDot, valid
case ')':
return false, false
default:
return false, true
}
}
func (p *filesMatchRegexParser) regexFlagGroup() bool {
start := p.pos
for p.pos < len(p.pattern) {
c := p.pattern[p.pos]
if c == ':' {
if p.pos == start {
return false
}
p.pos++
return true
}
if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && c != '-' {
p.pos = start
return false
}
p.pos++
}
p.pos = start
return false
}
func (p *filesMatchRegexParser) characterClass() (bool, bool) {
for p.pos < len(p.pattern) {
c := p.pattern[p.pos]
p.pos++
if c == ']' {
// Even a class that currently contains only a dot is not used as
// an extension proof: the extension extractor deliberately handles
// only escaped-dot forms. Treating it as unrestricted keeps the two
// decisions aligned and cannot lose content coverage.
return false, true
}
if c == '\\' {
if p.pos >= len(p.pattern) {
return false, false
}
p.pos++
}
}
return false, false
}
func regexRepeatMinimum(pattern string, start int) (int, int, bool) {
end := strings.IndexByte(pattern[start+1:], '}')
if end < 0 {
return 0, start, false
}
end += start + 1
fields := strings.Split(pattern[start+1:end], ",")
if len(fields) > 2 || fields[0] == "" {
return 0, start, false
}
minimum, err := strconv.Atoi(fields[0])
if err != nil || minimum < 0 {
return 0, start, false
}
if len(fields) == 2 && fields[1] != "" {
maximum, err := strconv.Atoi(fields[1])
if err != nil || maximum < minimum {
return 0, start, false
}
}
return minimum, end + 1, true
}
// topLevelAlternatives splits a regex on "|" at nesting depth zero only,
// ignoring "|" inside groups, character classes and after a backslash.
func topLevelAlternatives(pattern string) []string {
var out []string
depth := 0
inClass := false
start := 0
for i := 0; i < len(pattern); i++ {
switch c := pattern[i]; {
case c == '\\':
i++
case inClass:
if c == ']' {
inClass = false
}
case c == '[':
inClass = true
case c == '(':
depth++
case c == ')':
if depth > 0 {
depth--
}
case c == '|' && depth == 0:
out = append(out, pattern[start:i])
start = i + 1
}
}
return append(out, pattern[start:])
}
func extensionsFromFilesMatchPattern(pattern string) []string {
pattern = strings.ToLower(pattern)
seen := make(map[string]struct{})
var exts []string
add := func(ext string) {
ext = normalizeExt(ext)
if ext == "" {
return
}
if _, ok := seen[ext]; ok {
return
}
seen[ext] = struct{}{}
exts = append(exts, ext)
}
for i := 0; i+1 < len(pattern); i++ {
if pattern[i] != '\\' || pattern[i+1] != '.' {
continue
}
j := i + 2
if j < len(pattern) && pattern[j] == '(' {
end := strings.IndexByte(pattern[j+1:], ')')
if end >= 0 {
group := pattern[j+1 : j+1+end]
group = strings.TrimPrefix(group, "?:")
for _, part := range strings.Split(group, "|") {
add(part)
}
}
continue
}
start := j
for j < len(pattern) {
c := pattern[j]
alnum := (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')
if !alnum && c != '_' && c != '-' {
break
}
j++
}
add(pattern[start:j])
i = j
}
return exts
}
func phpPathExecutes(path, nameLower string) bool {
if isExecutablePHPName(nameLower) {
return true
}
overlay := phpHandlerOverlay{}
for _, dir := range htaccessAncestorDirs(path) {
htaccess, ok, err := readHtaccessBounded(filepath.Join(dir, ".htaccess"))
if htaccessOversized(ok, err) {
// A .htaccess too large to read may carry any handler: fail
// toward coverage and treat the name as executable.
return true
}
if err == nil && ok {
overlay = overlay.mergeHtaccess(htaccess)
}
}
return overlay.executes(nameLower)
}
func htaccessAncestorDirs(path string) []string {
dir := filepath.Clean(filepath.Dir(path))
if dir == "." {
return nil
}
var dirs []string
for {
dirs = append(dirs, dir)
if stopHtaccessAncestorWalk(dir) {
break
}
parent := filepath.Dir(dir)
if parent == dir {
break
}
dir = parent
}
for i, j := 0, len(dirs)-1; i < j; i, j = i+1, j-1 {
dirs[i], dirs[j] = dirs[j], dirs[i]
}
return dirs
}
func stopHtaccessAncestorWalk(dir string) bool {
// Stop at an account home: "<root>/<account>" is not inside an account,
// while "<root>/<account>/x" is.
if _, _, inside := accountRootOf(dir); inside {
return false
}
parent := filepath.Dir(dir)
return isAccountRoot(parent) && filepath.Base(dir) != "" && dir != parent
}
package checks
import (
"context"
"sync"
"github.com/pidginhost/csm/internal/phptaint"
)
// phpTaintWorker is the isolated analyzer this adapter routes through. It is
// nil until the daemon supervises one.
var (
phpTaintWorkerMu sync.RWMutex
phpTaintWorker PHPTaintAnalyzer
)
// PHPTaintAnalyzer is what the daemon supplies: an analyzer that runs in a
// process the caller can kill.
type PHPTaintAnalyzer interface {
Analyze(ctx context.Context, src []byte) phptaint.Report
}
// SetPHPTaintAnalyzer installs the supervised worker. Passing nil removes it,
// after which every candidate is reported as an unexamined coverage gap rather
// than analyzed in this process.
func SetPHPTaintAnalyzer(a PHPTaintAnalyzer) {
phpTaintWorkerMu.Lock()
defer phpTaintWorkerMu.Unlock()
phpTaintWorker = a
}
// phpTaintAnalyzerReady reports whether an isolated analyzer is available.
//
// The consumer does not dispatch without one. Analysis cannot safely happen in
// this process, so the alternative would be recording every candidate file as
// an unexamined gap -- turning "this feature is not active here" into a
// per-file coverage report on every scan, which tells an operator nothing they
// can act on. A gap is for content that should have been examined and was not.
func phpTaintAnalyzerReady() bool {
phpTaintWorkerMu.RLock()
defer phpTaintWorkerMu.RUnlock()
return phpTaintWorker != nil
}
// defaultPHPTaintAnalyze routes to the supervised worker, and reports a
// coverage gap when there is none.
//
// It deliberately does NOT fall back to calling phptaint.Analyze here. The
// parser can enter an infinite loop on attacker-controlled input; a loop is not
// a panic, so recover() cannot catch it, and the parser never checks context,
// so no deadline in this process can stop it. An in-process fallback would take
// a core from the daemon permanently the first time a crafted file was scanned
// -- which is the entire failure this indirection exists to prevent. Reporting
// a gap loses coverage on that file; the fallback would lose the daemon.
func defaultPHPTaintAnalyze(ctx context.Context, src []byte) phptaint.Report {
phpTaintWorkerMu.RLock()
worker := phpTaintWorker
phpTaintWorkerMu.RUnlock()
if worker == nil {
return phptaint.Report{
Status: phptaint.StatusWorkerFailure,
Reason: "no isolated php analyzer configured; content not examined",
}
}
return worker.Analyze(ctx, src)
}
// phpTaintAnalyze is indirected so tests can substitute the analyzer at the
// adapter boundary: forcing a panic or a specific status proves containment
// without needing a live worker or pathological PHP.
var phpTaintAnalyze = defaultPHPTaintAnalyze
// runPHPTaintAnalysis is the adapter recovery boundary around the analyzer
// call. The analyzer converts its own internal panics to StatusPanic, but a
// panic in adapter-side code around the call must have the same contained
// outcome instead of escaping through the owning check.
func runPHPTaintAnalysis(ctx context.Context, data []byte) (report phptaint.Report) {
defer func() {
if r := recover(); r != nil {
report = phptaint.Report{Status: phptaint.StatusPanic, Reason: "panic"}
}
}()
return phpTaintAnalyze(ctx, data)
}
package checks
import (
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"fmt"
"hash"
"os"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/phptaint"
)
// phpTaintDeepCursorCheck is the host-scope scan-cursor key for the scheduled
// PHP taint consumer of the shared deep-content walk. It is distinct from the
// JS consumer's key so the two advance independently: they admit different
// files, so a shared cursor would let one consumer's progress hide the other's
// unscanned remainder.
const phpTaintDeepCursorCheck = logicalOwnerPHPTaintDeep
// phpTaintDeepPerFileTimeout bounds one file's analysis as seen by this
// consumer. The supervised worker applies its own, shorter deadline and kills
// the process when it expires; this outer bound only covers the case where the
// worker layer itself becomes unresponsive.
const phpTaintDeepPerFileTimeout = 30 * time.Second
// Display bounds mirror the JS consumer's: message, details, and diagnostic
// example paths.
// maxPHPTaintGapPaths bounds the exact paths one run retains for carry-forward.
//
// The supervisor rejects non-candidates in-process, but an unavailable worker
// can still leave every candidate unexamined. Bound retained path strings for
// large scans. Past the bound the run stops enumerating and reports itself as
// unable to enumerate, which suppresses the purge wholesale rather than
// carrying forward an arbitrary prefix.
const maxPHPTaintGapPaths = 50_000
const (
phpTaintMessageMaxBytes = 512
phpTaintDetailsMaxBytes = 2048
phpTaintExampleMaxBytes = 256
)
// phpTaintGapCollector aggregates per-path PHP coverage gaps for one deep run:
// exact paths feed the carry-forward, counts and one example per status feed
// the php_taint_scan_incomplete diagnostic. A non-completed status is never
// counted as a clean file.
type phpTaintGapCollector struct {
gapPathSet
recordCoverage func([]string)
defeatInputs [][sha256.Size]byte
defeatOverflow hash.Hash
}
func newPHPTaintGapCollector() *phpTaintGapCollector {
return &phpTaintGapCollector{gapPathSet: newGapPathSet(maxPHPTaintGapPaths, phpTaintExampleMaxBytes, lexicalGapAliases)}
}
func (g *phpTaintGapCollector) record(path, status string) {
g.recordSnapshot(path, status, "")
}
func (g *phpTaintGapCollector) recordSnapshot(path, status, contentSHA256 string) {
if isPHPTaintAnalyzerDefeatStatus(status) {
identity := sha256.Sum256([]byte(path + "\x00" + status + "\x00" + contentSHA256))
if len(g.defeatInputs) < maxPHPTaintGapPaths {
g.defeatInputs = append(g.defeatInputs, identity)
} else {
// Beyond the memory bound, retain all evidence in a streaming
// digest. Order changes may re-alert in this extreme case, but
// a dismissal must never hide failures beyond the retained set.
if g.defeatOverflow == nil {
g.defeatOverflow = sha256.New()
}
_, _ = g.defeatOverflow.Write(identity[:])
}
}
if aliases := g.retain(path, status); len(aliases) > 0 && g.recordCoverage != nil {
g.recordCoverage(aliases)
}
}
// isPHPTaintAnalyzerDefeatStatus identifies per-file hard failures.
// StatusPanic covers defects anywhere in the analyzer stack, including the
// known upstream lexer defect. StatusTimeout means the isolated worker had to
// be killed after analysis stopped making progress. Neither may be attributed
// solely to the parser, but both need to remain visible instead of being buried
// under routine coverage gaps.
func isPHPTaintAnalyzerDefeatStatus(status string) bool {
return status == phptaint.StatusPanic.String() || status == phptaint.StatusTimeout.String()
}
// findings splits routine coverage gaps from content that crashed or stalled
// the analyzer. Collector-wide range-loss facts are attached exactly once.
func (g *phpTaintGapCollector) findings() []alert.Finding {
var out []alert.Finding
routine := make(map[string]int, len(g.byStatus))
defeats := make(map[string]int, 2)
for status, n := range g.byStatus {
if isPHPTaintAnalyzerDefeatStatus(status) {
defeats[status] = n
} else {
routine[status] = n
}
}
hasRoutineFinding := len(routine) > 0 || g.unknown > 0
if hasRoutineFinding {
out = append(out, g.buildFinding(routine, true, false))
}
if len(defeats) > 0 {
out = append(out, g.buildFinding(defeats, !hasRoutineFinding, true))
}
return out
}
// finding reports every gap in one alert. Retained for callers that do not
// need the split.
func (g *phpTaintGapCollector) finding() alert.Finding {
return g.buildFinding(g.byStatus, true, false)
}
func (g *phpTaintGapCollector) buildFinding(byStatus map[string]int, includeRangeLoss, analyzerDefeat bool) alert.Finding {
total, parts := g.statusParts(byStatus)
if includeRangeLoss {
parts = append(parts, g.rangeLossParts()...)
}
message := fmt.Sprintf("PHP taint deep scan could not analyze %d file(s)", total)
// Routine coverage loss is one host condition. Analyzer defeats are
// input-specific so dismissing one cannot hide different failing files.
dedupKey := "coverage_gap"
if analyzerDefeat {
message = fmt.Sprintf("PHP taint deep scan was defeated by %d file(s) that crashed or stalled the analyzer", total)
dedupKey = g.analyzerDefeatDedupKey()
} else if total == 0 {
message = fmt.Sprintf("PHP taint deep scan could not cover %d location(s)", g.unknown)
}
return alert.Finding{
Severity: alert.Warning,
Check: "php_taint_scan_incomplete",
Message: message,
Details: strings.Join(parts, "; "),
DedupKey: dedupKey,
}
}
func (g *phpTaintGapCollector) analyzerDefeatDedupKey() string {
// Hash every full input identity, not just sanitized display examples.
// Sorting keeps ordinary traversal-order changes out of the alert key.
sort.Slice(g.defeatInputs, func(i, j int) bool {
return bytes.Compare(g.defeatInputs[i][:], g.defeatInputs[j][:]) < 0
})
digest := sha256.New()
for _, identity := range g.defeatInputs {
_, _ = digest.Write(identity[:])
}
if g.defeatOverflow != nil {
_, _ = digest.Write([]byte("overflow:"))
_, _ = digest.Write(g.defeatOverflow.Sum(nil))
}
return fmt.Sprintf("analyzer_defeat:%x", digest.Sum(nil))
}
// carryForwardPHPTaintFindings keeps at most one prior state finding for each
// path the current full cycle could not analyze, so a file that goes from
// analyzed to unexaminable does not silently lose its existing finding.
func carryForwardPHPTaintFindings(prior []alert.Finding, gaps *phpTaintGapCollector) []alert.Finding {
return carryForwardLatestPerPath(prior, "php_remote_taint", &gaps.gapPathSet)
}
// analyzePHPTaintSnapshot runs the PHP consumer on one complete in-memory
// snapshot and converts the result into at most one finding. Only StatusAnalyzed
// and StatusNotCandidate mean the file was examined; every other status is
// recorded as a known-path coverage gap.
func analyzePHPTaintSnapshot(ctx context.Context, path, contentSHA256 string, data []byte, gaps *phpTaintGapCollector) []alert.Finding {
fileCtx, cancel := context.WithTimeout(ctx, phpTaintDeepPerFileTimeout)
report := runPHPTaintAnalysis(fileCtx, data)
cancel()
switch report.Status {
case phptaint.StatusAnalyzed:
if len(report.Results) == 0 {
return nil
}
return []alert.Finding{phpTaintDeepFinding(path, contentSHA256, report)}
case phptaint.StatusNotCandidate:
return nil
default:
gaps.recordSnapshot(path, report.Status.String(), contentSHA256)
return nil
}
}
// phpTaintDeepFinding renders the single finding for one analyzed file. Every
// display field is sanitized and bounded; FilePath keeps the exact live path
// for remediation while only its display copy is sanitized.
func phpTaintDeepFinding(path, contentSHA256 string, report phptaint.Report) alert.Finding {
flows := make([]string, 0, len(report.Results))
for _, res := range report.Results {
flows = append(flows, fmt.Sprintf("%s -> %s (%s, %s)", res.Source, res.Sink, phpTaintConfidence(res.Confidence), res.Basis))
}
details := "Remotely fetched content reaches a code-execution construct. Evidence: " + strings.Join(flows, "; ")
if extra := report.TotalResults - len(report.Results); extra > 0 {
details += fmt.Sprintf("; %d additional flow(s) beyond returned evidence", extra)
}
if report.EvidenceTruncated {
details += " [evidence truncated]"
}
if len(report.PrecisionLoss) > 0 {
details += "; reduced precision: " + strings.Join(report.PrecisionLoss, ", ")
}
severity := phpTaintSeverity(report.Results)
return alert.Finding{
Severity: severity,
Check: "php_remote_taint",
Message: "PHP remote-source code execution data flow: " + sanitizeJSTaintDisplay(path, phpTaintMessageMaxBytes),
Details: sanitizeJSTaintDisplay(details, phpTaintDetailsMaxBytes),
DedupKey: phpTaintDedupKey(path, severity, report.Results),
FilePath: path,
ContentSHA256: contentSHA256,
DetectLogic: ContentDetectionVersion(),
CoverageScope: deepFileCoverageScope(path),
}
}
// phpTaintDedupKey pins a finding's identity to the file, its severity and
// the distinct source and sink endpoints of its flows. Details carry the
// evidence wording, basis and context, which change between releases; if
// they fed the key, each such change would re-key every stored finding, drop
// its dismissal and alert again. The content hash is left out too: a library
// file still flagged after an update is the same finding. Any change to the
// reported flows, file or severity still makes a new one.
func phpTaintDedupKey(path string, severity alert.Severity, results []phptaint.Result) string {
type endpoint struct{ source, sink string }
seen := make(map[endpoint]bool, len(results))
pairs := make([]endpoint, 0, len(results))
for _, res := range results {
e := endpoint{res.Source, res.Sink}
if !seen[e] {
seen[e] = true
pairs = append(pairs, e)
}
}
sort.Slice(pairs, func(i, j int) bool {
if pairs[i].source != pairs[j].source {
return pairs[i].source < pairs[j].source
}
return pairs[i].sink < pairs[j].sink
})
identity := make([]byte, 0, 128)
appendField := func(value string) {
identity = binary.BigEndian.AppendUint64(identity, uint64(len(value)))
identity = append(identity, value...)
}
appendField(path)
appendField(severity.String())
for _, e := range pairs {
appendField(e.source)
appendField(e.sink)
}
digest := sha256.Sum256(identity)
return fmt.Sprintf("php-taint:%x", digest[:12])
}
// phpTaintSeverity grades a finding by the strongest flow it contains.
//
// Confidence is what separates a fetch this analyzer PROVED was remote from
// one it merely could not rule out, and on real hosts that distinction is the
// whole signal. Measured over 1,104,790 files on a production cPanel server:
// 30,276 analyzed, 33 findings, every one of them ConfidenceLow and every one
// a third-party library that legitimately reads a file and evaluates it --
// template compilers, cache layers, SDK bootstrap code. Nothing graded High or
// Certain. Reporting all of them at one severity would bury a genuine remote
// code-execution flow among library noise on its first scan.
//
// Low is downgraded, not dropped: a real cross-function flow whose URL sits at
// the caller grades Low too, because the acquiring call cannot see it. The
// finding stays for review; it just does not page anyone.
func phpTaintSeverity(results []phptaint.Result) alert.Severity {
strongest := phptaint.ConfidenceLow
for _, res := range results {
if res.Confidence > strongest {
strongest = res.Confidence
}
}
switch strongest {
case phptaint.ConfidenceCertain:
return alert.Critical
case phptaint.ConfidenceHigh:
return alert.High
default:
return alert.Warning
}
}
func phpTaintConfidence(c phptaint.Confidence) string {
switch c {
case phptaint.ConfidenceCertain:
return "certain"
case phptaint.ConfidenceHigh:
return "high"
default:
return "low"
}
}
// phpTaintOversizePeekBytes bounds the prefix read to decide whether an
// oversize file could be PHP. An open tag in a PHP file is at the top; a
// larger peek would only buy false positives from binary content that happens
// to contain the byte sequence.
const phpTaintOversizePeekBytes = 64 << 10
type phpRegularFilePrefixReader interface {
ReadRegularFilePrefix(string, os.FileInfo, int64) ([]byte, error)
}
package checks
import (
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
var pluginInventoryBatches = newScanBatchMonitor()
// PluginInventoryQueueStatus includes sites waiting for a worker and inventories
// still executing or committing their result. Concurrent refreshes share no cap.
func PluginInventoryQueueStatus(now time.Time) queuehealth.Status {
return pluginInventoryBatches.snapshot(now)
}
package checks
import (
"context"
"sync"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/store"
)
type wpInventoryCycle struct {
mu sync.Mutex
calls map[*store.DB]*wpInventoryCycleCall
}
type wpInventoryCycleCall struct {
done chan struct{}
fresh bool
}
// ensurePluginCacheFresh reuses one refresh result for all consumers in a
// scan, even if the first consumer has finished before the next one starts.
func ensurePluginCacheFresh(ctx context.Context, cfg *config.Config, db *store.DB) bool {
cycle := wpInstallCacheFrom(ctx)
if cycle == nil {
return ensurePluginCacheFreshShared(ctx, cfg, db)
}
inventory := &cycle.inventory
inventory.mu.Lock()
if call := inventory.calls[db]; call != nil {
inventory.mu.Unlock()
select {
case <-call.done:
return call.fresh
case <-ctx.Done():
return false
}
}
call := &wpInventoryCycleCall{done: make(chan struct{})}
if inventory.calls == nil {
inventory.calls = make(map[*store.DB]*wpInventoryCycleCall)
}
inventory.calls[db] = call
inventory.mu.Unlock()
defer close(call.done)
call.fresh = ensurePluginCacheFreshShared(ctx, cfg, db)
return call.fresh
}
package checks
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
neturl "net/url"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
// wpCLIFlags are the extra flags and env CSM adds to every wp-cli invocation.
//
// WP_CLI_PHP_ARGS disables PHP's display_errors/error_reporting for the
// bootstrap so stray Notices/Warnings/Deprecated messages from the site
// don't get emitted (they'd only be a problem on stderr, which we already
// discard, but suppressing them also avoids exit-255 on strict hosts that
// promote warnings to errors).
//
// --skip-plugins and --skip-themes make wp-cli enumerate plugins from the
// filesystem without loading them. That removes the biggest source of log
// noise: one broken plugin (e.g. a PHP Parse error in litespeed-cache on a
// site nobody updated for years) would otherwise crash the whole `wp plugin
// list` call with exit 255, or spew backtraces from plugins that call
// wp_redirect() during admin bootstrap. Skipping loads gives us the list
// plus update_version unchanged.
const wpCLIFlags = `WP_CLI_PHP_ARGS='-d display_errors=0 -d error_reporting=0' wp --skip-plugins --skip-themes `
var wpOrgHTTPClient = &http.Client{Timeout: 10 * time.Second}
type wpOrgResponse struct {
Slug string `json:"slug"`
Version string `json:"version"`
Tested string `json:"tested"`
Error string `json:"error"`
}
// parseWPOrgPluginResponse parses a JSON body from the WordPress.org plugin
// information API into a store.PluginInfo. It returns an error if the response
// contains an error field (e.g. "Plugin not found.") or if the JSON is invalid.
func parseWPOrgPluginResponse(body []byte) (store.PluginInfo, error) {
var resp wpOrgResponse
if err := json.Unmarshal(body, &resp); err != nil {
return store.PluginInfo{}, fmt.Errorf("wporg: invalid JSON: %w", err)
}
if resp.Error != "" {
return store.PluginInfo{}, fmt.Errorf("wporg: %s", resp.Error)
}
return store.PluginInfo{
LatestVersion: resp.Version,
TestedUpTo: resp.Tested,
LastChecked: time.Now().Unix(),
}, nil
}
// fetchWPOrgPluginInfo queries the WordPress.org plugin information API for the
// given slug and returns the parsed PluginInfo.
func fetchWPOrgPluginInfo(ctx context.Context, slug string) (store.PluginInfo, error) {
url := "https://api.wordpress.org/plugins/info/1.2/?action=plugin_information" +
"&request[slug]=" + neturl.QueryEscape(slug) +
"&request[fields][version]=1&request[fields][tested]=1"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return store.PluginInfo{}, fmt.Errorf("wporg: building request: %w", err)
}
resp, err := wpOrgHTTPClient.Do(req)
if err != nil {
return store.PluginInfo{}, fmt.Errorf("wporg: HTTP request failed: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return store.PluginInfo{}, fmt.Errorf("wporg: reading response body: %w", err)
}
if resp.StatusCode != http.StatusOK {
return store.PluginInfo{}, fmt.Errorf("wporg: unexpected status %d", resp.StatusCode)
}
return parseWPOrgPluginResponse(body)
}
// parseVersion splits a dotted version string like "6.4.2" into []int{6, 4, 2}.
// Non-numeric segments are treated as 0.
func parseVersion(v string) []int {
if v == "" {
return nil
}
parts := strings.Split(v, ".")
out := make([]int, len(parts))
for i, p := range parts {
n, err := strconv.Atoi(p)
if err != nil {
n = 0
}
out[i] = n
}
return out
}
// compareVersions returns whether there is a major version gap and
// how many minor versions behind the installed version is.
func compareVersions(installed, available string) (majorGap bool, minorBehind int) {
iv := parseVersion(installed)
av := parseVersion(available)
if len(iv) < 2 || len(av) < 2 {
return false, 0
}
if av[0] > iv[0] {
return true, 0
}
if av[0] == iv[0] && av[1] > iv[1] {
return false, av[1] - iv[1]
}
return false, 0
}
// pluginAlertSeverity returns "critical", "high", "warning", or "" for a version gap.
func pluginAlertSeverity(installed, available string) string {
majorGap, minorBehind := compareVersions(installed, available)
if majorGap {
return "critical"
}
if minorBehind >= 3 {
return "high"
}
// Check if there is any difference at all.
iv := parseVersion(installed)
av := parseVersion(available)
if len(iv) < 2 || len(av) < 2 {
return ""
}
// Compare all parsed components to detect if available is actually newer.
// If installed >= available at every component, the site is up to date
// (or ahead, e.g. custom/premium builds). Only warn when behind.
maxLen := len(iv)
if len(av) > maxLen {
maxLen = len(av)
}
for i := 0; i < maxLen; i++ {
var a, b int
if i < len(iv) {
a = iv[i]
}
if i < len(av) {
b = av[i]
}
if b > a {
return "warning" // available is newer at this component
}
if a > b {
return "" // installed is ahead - not outdated
}
}
return "" // identical
}
const pluginCheckWorkers = 5
const defaultPluginCheckIntervalMin = 1440
type pluginCacheRefreshCall struct {
done chan struct{}
}
var (
pluginCacheRefreshMu sync.Mutex
pluginCacheRefreshes = make(map[*store.DB]*pluginCacheRefreshCall)
)
func pluginCacheFresh(db *store.DB, cfg *config.Config) bool {
lastRefresh := db.GetPluginRefreshTime()
if lastRefresh.IsZero() {
return false
}
intervalMin := defaultPluginCheckIntervalMin
if cfg != nil && cfg.Thresholds.PluginCheckIntervalMin > 0 {
intervalMin = cfg.Thresholds.PluginCheckIntervalMin
}
elapsed := time.Since(lastRefresh)
return elapsed >= 0 && elapsed <= time.Duration(intervalMin)*time.Minute
}
// ensurePluginCacheFresh coalesces refreshes shared by the outdated and
// known-vulnerable detectors. The checks run concurrently, so list order in
// the runner cannot establish the inventory dependency. Waiters reuse the
// leader's failed result as well, avoiding a duplicate walk in the same scan.
func ensurePluginCacheFreshShared(ctx context.Context, cfg *config.Config, db *store.DB) bool {
if pluginCacheFresh(db, cfg) {
return true
}
pluginCacheRefreshMu.Lock()
if call := pluginCacheRefreshes[db]; call != nil {
pluginCacheRefreshMu.Unlock()
select {
case <-call.done:
return pluginCacheFresh(db, cfg)
case <-ctx.Done():
return false
}
}
if pluginCacheFresh(db, cfg) {
pluginCacheRefreshMu.Unlock()
return true
}
call := &pluginCacheRefreshCall{done: make(chan struct{})}
pluginCacheRefreshes[db] = call
pluginCacheRefreshMu.Unlock()
func() {
defer func() {
pluginCacheRefreshMu.Lock()
delete(pluginCacheRefreshes, db)
close(call.done)
pluginCacheRefreshMu.Unlock()
}()
refreshPluginCache(ctx, db)
}()
return pluginCacheFresh(db, cfg)
}
// CheckOutdatedPlugins scans all WordPress installations for plugins with
// available updates and emits findings based on severity of the version gap.
// Results are cached in bbolt with a configurable refresh interval (default 24h).
func CheckOutdatedPlugins(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
db := store.Global()
if db == nil {
return nil
}
ensurePluginCacheFresh(ctx, cfg, db)
return evaluatePluginCache(db)
}
// findAllWPInstalls lists the WordPress installs to inventory plugins for.
// Discovery is shared (wpinstalls.go), which adds the panel's document-root
// map: a root outside the home layout used to have no plugin coverage, so its
// vulnerable plugins were never found and never virtually patched.
func findAllWPInstalls(ctx context.Context) []string {
seen := make(map[string]bool)
var results []string
installs := wpInstalls(ctx, "vulnerable_plugins")
if checkMarkedIncomplete(ctx, "vulnerable_plugins") {
// The same inventory backs both checks, regardless of which one won
// the concurrent refresh. Neither may purge after a partial walk.
markCheckIncomplete(ctx, "outdated_plugins")
}
for _, install := range installs {
if seen[install.ConfigPath] {
continue
}
seen[install.ConfigPath] = true
results = append(results, install.ConfigPath)
}
return results
}
// Only an absent directory component qualifies; a failed Readlink can mean
// the account replaced a symlink while discovery was resolving it.
var errWPDocrootMissing = errors.New("WordPress document-root component is missing")
func canonicalWPInstallPath(wpConfig string) (string, error) {
clean := filepath.Clean(wpConfig)
dir := filepath.Dir(clean)
home := wpInstallAccountRoot(clean)
if home == "" {
return "", fmt.Errorf("WordPress document root is outside an account home: %s", dir)
}
rel, err := filepath.Rel(home, dir)
if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("WordPress document root is outside an account home: %s", dir)
}
parts := strings.Split(rel, string(filepath.Separator))
resolved := home
const maxSymlinks = 40
symlinks := 0
for len(parts) > 0 {
part := parts[0]
parts = parts[1:]
if part == "" || part == "." || part == ".." {
return "", fmt.Errorf("invalid WordPress document-root component in %s", dir)
}
candidate := filepath.Join(resolved, part)
info, err := osFS.Lstat(candidate)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return "", fmt.Errorf("%w: %s: %w", errWPDocrootMissing, candidate, err)
}
return "", fmt.Errorf("inspect WordPress document root %s: %w", candidate, err)
}
if info.Mode()&os.ModeSymlink == 0 {
if !info.IsDir() {
return "", fmt.Errorf("WordPress document root is not a directory: %s", candidate)
}
resolved = candidate
continue
}
symlinks++
if symlinks > maxSymlinks {
return "", fmt.Errorf("too many symlinks in WordPress document root: %s", dir)
}
target, err := osFS.Readlink(candidate)
if err != nil {
return "", fmt.Errorf("resolve WordPress document root %s: %w", candidate, err)
}
if target == "" {
return "", fmt.Errorf("resolve WordPress document root %s: empty symlink target", candidate)
}
if !filepath.IsAbs(target) {
target = filepath.Join(resolved, target)
}
target = filepath.Clean(target)
if !isPathWithinOrEqual(target, home) {
return "", fmt.Errorf("WordPress document root %s resolves outside its account", candidate)
}
targetRel, err := filepath.Rel(home, target)
if err != nil {
return "", fmt.Errorf("resolve WordPress document root %s: %w", candidate, err)
}
var targetParts []string
if targetRel != "." {
targetParts = strings.Split(targetRel, string(filepath.Separator))
}
parts = append(targetParts, parts...)
resolved = home
}
return filepath.Join(resolved, filepath.Base(clean)), nil
}
func wpInstallAccountRoot(path string) string {
if home := homeAccountRoot(path); home != "" {
return home
}
clean := filepath.Clean(path)
if !filepath.IsAbs(clean) {
return ""
}
parts := strings.Split(clean, string(filepath.Separator))
for i, part := range parts {
if !isCPanelHomeBase(part) || i+2 >= len(parts) || !validAccountName.MatchString(parts[i+1]) {
continue
}
return filepath.Join(string(filepath.Separator), filepath.Join(parts[1:i+2]...))
}
return ""
}
// wpCLIPluginEntry mirrors the JSON output of `wp plugin list --format=json`.
type wpCLIPluginEntry struct {
Name string `json:"name"`
Status string `json:"status"`
Version string `json:"version"`
UpdateVersion string `json:"update_version"`
}
// refreshPluginCache discovers all WP installs, runs wp-cli to inventory
// plugins for each site, enriches free plugins via the WordPress.org API,
// and stores everything in bbolt.
func refreshPluginCache(ctx context.Context, db *store.DB) {
if incompleteCollectorFrom(ctx) == nil {
ctx, _ = withIncompleteCheckCollector(ctx)
}
wpConfigs := findAllWPInstalls(ctx)
discoveryIncomplete := checkMarkedIncomplete(ctx, "vulnerable_plugins")
coverage := newWPVerificationBatch(ctx, db, "plugins", "wp_plugin_inventory", wpConfigs)
defer func() {
if err := coverage.finish(ctx, !discoveryIncomplete); err != nil {
fmt.Fprintln(os.Stderr, "plugincheck: could not save verification history")
}
}()
if ctx.Err() != nil {
return
}
if len(wpConfigs) == 0 {
if discoveryIncomplete {
return
}
for path := range db.AllSitePlugins() {
if err := db.DeleteSitePlugins(path); err != nil {
fmt.Fprintf(os.Stderr, "plugincheck: prune failed for %s: %v\n", path, err)
return
}
}
return
}
var mu sync.Mutex
var wg sync.WaitGroup
successCount := 0
var timeoutCount, execFailCount, parseFailCount, cleanupFailCount int
slugsSeen := make(map[string]bool)
discoveredPaths := make(map[string]bool)
batch := pluginInventoryBatches.begin(len(wpConfigs), pluginCheckWorkers)
defer batch.abandon(ctx)
jobs := make(chan int, len(wpConfigs))
for i := 0; i < pluginCheckWorkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for index := range jobs {
wpConfig, work := wpConfigs[index], batch.tasks[index]
work.admit()
stop := false
// Site URL discovery and plugin enumeration each have a command timeout.
work.run(ctx, 2*cmdTimeout, func() {
if err := ctx.Err(); err != nil {
work.withdraw(err)
stop = true
return
}
wpPath := filepath.Dir(wpConfig)
if err := ctx.Err(); err != nil {
work.withdraw(err)
stop = true
return
}
mu.Lock()
discoveredPaths[wpPath] = true
mu.Unlock()
sitePlugins, err := inventoryWPSite(ctx, wpConfig)
work.progress()
if err != nil {
parentErr := ctx.Err()
if (!errors.Is(err, context.Canceled) || parentErr == nil) && (!commandRefused(err) || errors.Is(err, errWPInventoryNoOutput)) {
work.fail()
}
if parentErr != nil {
work.withdraw(parentErr)
stop = true
return
}
coverage.record(wpPath, wpVerificationFailure(err, nil))
mu.Lock()
switch {
case errors.Is(err, context.DeadlineExceeded):
timeoutCount++
case errors.Is(err, errWPInventoryParse):
parseFailCount++
default:
execFailCount++
}
mu.Unlock()
if err := db.DeleteSitePlugins(wpPath); err != nil {
fmt.Fprintf(os.Stderr, "plugincheck: stale cache cleanup failed for %s: %v\n", wpPath, err)
mu.Lock()
cleanupFailCount++
mu.Unlock()
}
return
}
mu.Lock()
for _, p := range sitePlugins.Plugins {
slugsSeen[p.Slug] = true
}
mu.Unlock()
coverage.record(wpPath, store.WPVerificationResult{State: "verified"})
if err := db.SetSitePlugins(wpPath, sitePlugins); err != nil {
coverage.record(wpPath, store.WPVerificationResult{State: "unverified", Reason: "CSM could not save the plugin inventory"})
work.fail()
fmt.Fprintf(os.Stderr, "plugincheck: store failed for %s: %v\n", wpPath, err)
if cleanupErr := db.DeleteSitePlugins(wpPath); cleanupErr != nil {
fmt.Fprintf(os.Stderr, "plugincheck: stale cache cleanup failed for %s: %v\n", wpPath, cleanupErr)
}
mu.Lock()
cleanupFailCount++
mu.Unlock()
return
}
mu.Lock()
successCount++
mu.Unlock()
})
if stop {
return
}
}
}()
}
for index := range wpConfigs {
if ctx.Err() != nil {
break
}
jobs <- index
}
close(jobs)
wg.Wait()
batch.abandon(ctx)
if ctx.Err() != nil {
return
}
// Enrich free plugins via WordPress.org API (one lookup per unique slug).
mu.Lock()
slugList := make([]string, 0, len(slugsSeen))
for slug := range slugsSeen {
slugList = append(slugList, slug)
}
mu.Unlock()
for _, slug := range slugList {
if ctx.Err() != nil {
return
}
// Skip if we have a recent cached entry (< 24h).
if cached, ok := db.GetPluginInfo(slug); ok {
if time.Since(time.Unix(cached.LastChecked, 0)) < 24*time.Hour {
continue
}
}
info, err := fetchWPOrgPluginInfo(ctx, slug)
if err != nil {
// Not found on .org = premium/custom plugin, skip silently.
continue
}
_ = db.SetPluginInfo(slug, info)
}
// A partial walk cannot prove that an absent path was removed.
if !discoveryIncomplete {
allCached := db.AllSitePlugins()
for path := range allCached {
if !discoveredPaths[path] {
if err := db.DeleteSitePlugins(path); err != nil {
fmt.Fprintf(os.Stderr, "plugincheck: prune failed for %s: %v\n", path, err)
cleanupFailCount++
}
}
}
}
// Only mark refresh as complete if the majority of sites refreshed
// successfully. A partial failure (e.g. one wp-cli timeout on a 100-site
// server) should not freeze ALL stale data for 24 hours. But if most
// sites failed (e.g. transient PHP issue), don't mark as fresh - allow
// retry next cycle.
mu.Lock()
sc := successCount
to, exf, pf, cf := timeoutCount, execFailCount, parseFailCount, cleanupFailCount
mu.Unlock()
failCount := len(wpConfigs) - sc
ts := time.Now().Format("2006-01-02 15:04:05")
if sc == 0 {
fmt.Fprintf(os.Stderr, "[%s] plugincheck: refresh failed, 0/%d sites succeeded%s, not updating timestamp\n",
ts, len(wpConfigs), failureBreakdown(to, exf, pf))
return
}
if failCount > sc {
fmt.Fprintf(os.Stderr, "[%s] plugincheck: refresh partial, %d/%d sites failed%s, not updating timestamp\n",
ts, failCount, len(wpConfigs), failureBreakdown(to, exf, pf))
return
}
if cf > 0 {
fmt.Fprintf(os.Stderr, "[%s] plugincheck: refresh incomplete, %d stale cache cleanup(s) failed, not updating timestamp\n",
ts, cf)
return
}
if discoveryIncomplete {
fmt.Fprintf(os.Stderr, "[%s] plugincheck: discovery incomplete, not updating timestamp\n", ts)
return
}
if failCount > 0 {
fmt.Fprintf(os.Stderr, "[%s] plugincheck: refreshed %d/%d sites%s\n",
ts, sc, len(wpConfigs), failureBreakdown(to, exf, pf))
}
if err := db.SetPluginRefreshTime(time.Now()); err != nil {
fmt.Fprintf(os.Stderr, "plugincheck: refresh timestamp failed: %v\n", err)
}
}
// failureBreakdown formats " (timeout=N exec_fail=N json_fail=N)" when any
// category is non-zero, or "" otherwise. Keeps the refresh log to one line
// instead of one line per broken site.
func failureBreakdown(timeout, execFail, parseFail int) string {
if timeout == 0 && execFail == 0 && parseFail == 0 {
return ""
}
return fmt.Sprintf(" (timeout=%d exec_fail=%d json_fail=%d)", timeout, execFail, parseFail)
}
// evaluatePluginCache reads the cached plugin inventory and emits one
// aggregated finding per site listing every outdated active plugin. The
// per-site rollup keeps the alert channel under control during a deep
// scan tier on hosts with many sites: the previous one-finding-per-
// outdated-plugin shape produced ~1000 findings on a 200-account host
// and saturated the 500-deep alert channel buffer, dropping real
// signal under "alert channel full, dropping deep finding:
// outdated_plugins".
//
// Aggregation rules:
// - Severity = max of constituents (critical > high > warning).
// - Message = "<count> outdated plugins on <domain> (<account>):
// <severity-label>" - searchable and self-describing.
// - Details lists each plugin slug, installed version, available
// version, and per-plugin severity, one per line, so an operator
// triaging the alert sees the same per-plugin breakdown as before.
func evaluatePluginCache(db *store.DB) []alert.Finding {
var findings []alert.Finding
allSites := db.AllSitePlugins()
for wpPath, site := range allSites {
var (
detailLines []string
worstSeverity alert.Severity
worstSevLabel string
outdatedTotal int
)
// Track whether worstSeverity has been set at all: alert.Severity's
// zero value is Warning, so a strict "newer rank > current rank"
// comparison would never overwrite the initial state on a site
// whose constituents are all Warning, leaving worstSevLabel empty.
worstSet := false
for _, p := range site.Plugins {
severity, sevLabel, available, ok := pluginOutdatedSeverity(p, db)
if !ok {
continue
}
outdatedTotal++
detailLines = append(detailLines, fmt.Sprintf("- %s (%s): %s -> %s [%s]",
p.Slug, p.Name, p.InstalledVersion, available, sevLabel))
if !worstSet || severityRank(severity) > severityRank(worstSeverity) {
worstSeverity = severity
worstSevLabel = sevLabel
worstSet = true
}
}
if outdatedTotal == 0 {
continue
}
findings = append(findings, alert.Finding{
Severity: worstSeverity,
Check: "outdated_plugins",
Message: fmt.Sprintf("%d outdated plugin%s on %s (%s): worst severity %s",
outdatedTotal, plural(outdatedTotal), site.Domain, site.Account, worstSevLabel),
Details: fmt.Sprintf("Path: %s\nOutdated plugins (%d):\n%s",
wpPath, outdatedTotal, strings.Join(detailLines, "\n")),
})
}
return findings
}
// errWPInventoryParse marks a wp-cli plugin-list response that ran but produced
// unparseable JSON, so callers can tell a parse failure from an exec failure.
var errWPInventoryParse = errors.New("wp-cli plugin list: invalid JSON")
// A refusal without command output cannot account for the inventory work.
var errWPInventoryNoOutput = errors.New("wp-cli plugin list: no output")
// inventoryWPSite runs wp-cli for a single site (as the site owner) and returns
// its current plugin inventory. Shared by the periodic cache refresh and the
// per-finding re-check so both see identical results. Read-only: it inventories,
// it does not change anything.
func inventoryWPSite(ctx context.Context, wpConfig string) (store.SitePlugins, error) {
return inventoryWPSiteWithDomain(ctx, wpConfig, true)
}
func inventoryWPSiteForVerify(ctx context.Context, wpConfig string) (store.SitePlugins, error) {
return inventoryWPSiteWithDomain(ctx, wpConfig, false)
}
func inventoryWPSiteWithDomain(ctx context.Context, wpConfig string, includeDomain bool) (store.SitePlugins, error) {
wpPath := filepath.Dir(wpConfig)
user := wpConfigUser(wpPath)
if !validWPCLIUser(user) {
return store.SitePlugins{}, fmt.Errorf("invalid WordPress site owner: %q", user)
}
domain := user
if includeDomain {
domain = extractWPDomain(ctx, wpPath, user)
}
// Run wp plugin list as the site owner on stdout-only so PHP
// notices/warnings on stderr can't corrupt the JSON we parse. Use --path
// instead of a shell cd to avoid shell injection via crafted directory
// names on shared hosting.
out, err := runWPCLIStdout(ctx, user,
wpCLIFlags+"plugin list --fields=name,status,version,update_version --format=json --path="+shellQuote(wpPath),
)
if out == nil {
// Output retains a failed command's stderr on ExitError. A refusal
// there still answers the check without contaminating the JSON input.
var exit *exec.ExitError
if !errors.As(err, &exit) || len(exit.Stderr) == 0 {
return store.SitePlugins{}, errors.Join(errWPInventoryNoOutput, err)
}
}
if err != nil {
return store.SitePlugins{}, &wpInventoryError{err: err, result: wpVerificationFailure(err, out)}
}
var entries []wpCLIPluginEntry
if err := json.Unmarshal(out, &entries); err != nil {
return store.SitePlugins{}, fmt.Errorf("%w: %v", errWPInventoryParse, err)
}
site := store.SitePlugins{Account: user, Domain: domain}
for _, e := range entries {
site.Plugins = append(site.Plugins, store.SitePluginEntry{
Slug: e.Name,
Name: e.Name,
Status: e.Status,
InstalledVersion: e.Version,
UpdateVersion: e.UpdateVersion,
})
}
return site, nil
}
func validWPCLIUser(user string) bool {
if user == "" || user == "unknown" || strings.HasPrefix(user, "-") {
return false
}
for _, r := range user {
if r >= 'a' && r <= 'z' {
continue
}
if r >= 'A' && r <= 'Z' {
continue
}
if r >= '0' && r <= '9' {
continue
}
switch r {
case '_', '-', '.', '@', '$':
continue
default:
return false
}
}
return true
}
func runWPCLIStdout(ctx context.Context, user, command string) ([]byte, error) {
if !validWPCLIUser(user) {
return nil, fmt.Errorf("invalid WordPress site owner: %q", user)
}
// runuser, not su: under the hardened systemd unit /var/log is read-only, and
// su's PAM stack runs pam_lastlog on every call (floods the journal with
// "/var/log/lastlog: Read-only file system" and would record CSM's internal
// scans as user logins). runuser's PAM session has no pam_lastlog. Same
// login-shell semantics otherwise.
return cmdExec.RunContextStdout(ctx, "runuser", "-l", "-s", "/bin/bash", "-c", command, "--", user)
}
// pluginOutdatedSeverity classifies one plugin entry. It returns ok=false for
// inactive plugins, plugins with no known available version (custom/premium),
// and plugins already current. The "available" version prefers wp-cli's
// update_version and falls back to the cached WordPress.org latest version.
func pluginOutdatedSeverity(p store.SitePluginEntry, db *store.DB) (severity alert.Severity, sevLabel, available string, ok bool) {
if p.Status != "active" && p.Status != "active-network" {
return 0, "", "", false
}
available = p.UpdateVersion
if available == "" && db != nil {
if info, found := db.GetPluginInfo(p.Slug); found {
available = info.LatestVersion
}
}
if available == "" {
return 0, "", "", false
}
sevLabel = pluginAlertSeverity(p.InstalledVersion, available)
if sevLabel == "" {
return 0, "", "", false
}
switch sevLabel {
case "critical":
severity = alert.Critical
case "high":
severity = alert.High
default:
severity = alert.Warning
}
return severity, sevLabel, available, true
}
// countOutdatedActivePlugins reports how many active plugins on a site still
// have an available update, using the same classification as the alert path.
func countOutdatedActivePlugins(site store.SitePlugins, db *store.DB) int {
n := 0
for _, p := range site.Plugins {
if _, _, _, ok := pluginOutdatedSeverity(p, db); ok {
n++
}
}
return n
}
// severityRank orders severities so an aggregate can pick the worst.
// Higher returned value means more severe.
func severityRank(s alert.Severity) int {
switch s {
case alert.Critical:
return 3
case alert.High:
return 2
case alert.Warning:
return 1
default:
return 0
}
}
func plural(n int) string {
if n == 1 {
return ""
}
return "s"
}
// extractWPDomain runs `wp option get siteurl` to discover the site's domain.
// Falls back to directory name heuristics if wp-cli fails.
func extractWPDomain(ctx context.Context, wpPath, user string) string {
// Stdout-only: some sites print "WARNING: MYSQL_OPT_RECONNECT deprecated"
// or similar on stderr during wp-cli boot. Mixing that into the value
// would produce a poisoned domain like "Warning: ... https://site.com".
out, err := runWPCLIStdout(ctx, user,
wpCLIFlags+"option get siteurl --path="+shellQuote(wpPath),
)
if err == nil {
url := strings.TrimSpace(string(out))
if url != "" {
// Strip protocol prefix for display.
url = strings.TrimPrefix(url, "https://")
url = strings.TrimPrefix(url, "http://")
return url
}
}
// Fallback: use directory name after public_html (addon domain)
// or account name (main domain).
parts := strings.Split(wpPath, "/")
for i, p := range parts {
if p == "public_html" && i+1 < len(parts) {
return parts[i+1]
}
}
return user
}
// shellQuote wraps a string in single quotes for safe shell argument passing.
// Any embedded single quotes are escaped as '\” (end quote, literal quote, start quote).
func shellQuote(s string) string {
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}
package checks
import (
"strings"
"github.com/pidginhost/csm/internal/store"
)
// postfixAuditParams are the parameters auditPostfix reads. postconf reports
// the effective value, so a parameter left at its built-in default still
// comes back with one.
var postfixAuditParams = []string{
"smtpd_relay_restrictions",
"smtpd_recipient_restrictions",
"smtpd_tls_security_level",
"smtpd_use_tls",
"smtpd_tls_protocols",
"smtpd_tls_mandatory_protocols",
"smtpd_sasl_auth_enable",
"smtpd_tls_auth_only",
"disable_vrfy_command",
}
// postconfSettings returns the effective value of each requested Postfix
// parameter. Only requested keys are kept, so warnings postconf prints
// alongside the values are never mistaken for settings.
func postconfSettings(params ...string) (map[string]string, bool) {
out, err := auditRunCmd("postconf", params...)
if err != nil {
return nil, false
}
wanted := make(map[string]bool, len(params))
for _, p := range params {
wanted[p] = true
}
settings := make(map[string]string, len(params))
for _, line := range strings.Split(string(out), "\n") {
key, value, found := strings.Cut(line, "=")
if !found {
continue
}
key = strings.TrimSpace(key)
if !wanted[key] {
continue
}
settings[key] = strings.TrimSpace(value)
}
return settings, len(settings) > 0
}
// auditPostfix checks the settings that decide whether a Postfix host relays
// for strangers or hands out credentials in the clear.
func auditPostfix() []store.AuditResult {
settings, ok := postconfSettings(postfixAuditParams...)
if !ok {
return []store.AuditResult{{
Category: "mail", Name: "mail_postfix_config", Title: "Postfix Configuration",
Status: "warn", Message: "Cannot query postfix configuration",
Fix: "Make 'postconf' runnable so postfix mail hardening can be audited.",
}}
}
return []store.AuditResult{
auditPostfixRelay(settings),
auditPostfixTLS(settings),
auditPostfixTLSProtocols(settings),
auditPostfixAuthOnly(settings),
auditPostfixVrfy(settings),
}
}
func auditPostfixRelay(settings map[string]string) store.AuditResult {
result := store.AuditResult{Category: "mail", Name: "mail_postfix_relay", Title: "Postfix Relay Control"}
// smtpd_relay_restrictions is the modern home for this rule; releases
// before 2.10 enforced it from smtpd_recipient_restrictions instead.
for _, key := range []string{"smtpd_relay_restrictions", "smtpd_recipient_restrictions"} {
if postfixRejectsUnauthDestination(settings[key]) {
result.Status = "pass"
result.Message = "Relaying to unauthorised destinations is rejected"
return result
}
}
result.Status = "fail"
result.Message = "No restriction list rejects relaying to unauthorised destinations"
result.Fix = "Add 'reject_unauth_destination' to smtpd_relay_restrictions in main.cf."
return result
}
func postfixRejectsUnauthDestination(value string) bool {
for _, token := range splitPostfixList(value) {
if strings.EqualFold(token, "reject_unauth_destination") ||
strings.EqualFold(token, "defer_unauth_destination") {
return true
}
}
return false
}
func auditPostfixTLS(settings map[string]string) store.AuditResult {
result := store.AuditResult{Category: "mail", Name: "mail_postfix_tls", Title: "Postfix Inbound TLS"}
level := strings.ToLower(settings["smtpd_tls_security_level"])
if level == "may" || level == "encrypt" {
result.Status = "pass"
result.Message = "Inbound TLS is offered (smtpd_tls_security_level = " + level + ")"
return result
}
// smtpd_use_tls predates smtpd_tls_security_level and still enables TLS
// on configurations that were never migrated.
if isPostfixYes(settings["smtpd_use_tls"]) {
result.Status = "pass"
result.Message = "Inbound TLS is offered via the legacy smtpd_use_tls setting"
return result
}
result.Status = "warn"
result.Message = "Inbound TLS is not offered, so mail and credentials cross the network in the clear"
result.Fix = "Set 'smtpd_tls_security_level = may' in main.cf."
return result
}
func auditPostfixTLSProtocols(settings map[string]string) store.AuditResult {
result := store.AuditResult{Category: "mail", Name: "mail_postfix_tls_protocols", Title: "Postfix TLS Protocols"}
var weak []string
for _, key := range []string{"smtpd_tls_protocols", "smtpd_tls_mandatory_protocols"} {
if !postfixProtocolsExcludeSSL(settings[key]) {
weak = append(weak, key)
}
}
if len(weak) == 0 {
result.Status = "pass"
result.Message = "SSLv2 and SSLv3 are excluded from both protocol lists"
return result
}
result.Status = "fail"
result.Message = "SSLv2 or SSLv3 is still permitted by " + strings.Join(weak, " and ")
result.Fix = "Set '>=TLSv1.2' for smtpd_tls_protocols and smtpd_tls_mandatory_protocols in main.cf."
return result
}
// postfixProtocolsExcludeSSL reports whether a protocol list keeps SSLv2 and
// SSLv3 out. Postfix accepts either an exclusion list (!SSLv2, !SSLv3) or a
// minimum-version floor (>=TLSv1.2), which rules the SSL versions out on its
// own.
func postfixProtocolsExcludeSSL(value string) bool {
excluded := make(map[string]bool)
for _, token := range splitPostfixList(value) {
if floor, ok := strings.CutPrefix(token, ">="); ok {
if strings.HasPrefix(strings.ToLower(floor), "tlsv") {
return true
}
continue
}
if name, ok := strings.CutPrefix(token, "!"); ok {
excluded[strings.ToLower(name)] = true
}
}
return excluded["sslv2"] && excluded["sslv3"]
}
func auditPostfixAuthOnly(settings map[string]string) store.AuditResult {
result := store.AuditResult{Category: "mail", Name: "mail_postfix_auth_only", Title: "Postfix Authentication Over TLS"}
if !isPostfixYes(settings["smtpd_sasl_auth_enable"]) {
result.Status = "pass"
result.Message = "SMTP authentication is not offered, so no credentials can be exposed"
return result
}
if isPostfixYes(settings["smtpd_tls_auth_only"]) {
result.Status = "pass"
result.Message = "SMTP authentication is offered only over TLS"
return result
}
result.Status = "fail"
result.Message = "SMTP authentication is offered on unencrypted connections"
result.Fix = "Set 'smtpd_tls_auth_only = yes' in main.cf."
return result
}
func auditPostfixVrfy(settings map[string]string) store.AuditResult {
result := store.AuditResult{Category: "mail", Name: "mail_postfix_vrfy", Title: "Postfix VRFY Command"}
if isPostfixYes(settings["disable_vrfy_command"]) {
result.Status = "pass"
result.Message = "The VRFY command is disabled"
return result
}
result.Status = "warn"
result.Message = "The VRFY command is enabled, letting senders confirm which mailboxes exist"
result.Fix = "Set 'disable_vrfy_command = yes' in main.cf."
return result
}
func splitPostfixList(value string) []string {
return strings.FieldsFunc(value, func(r rune) bool {
return r == ',' || r == ' ' || r == '\t' || r == '\r' || r == '\n'
})
}
func isPostfixYes(value string) bool {
switch strings.ToLower(strings.TrimSpace(value)) {
case "yes", "true", "1":
return true
}
return false
}
package checks
import (
"context"
"fmt"
"math"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/processctx"
"github.com/pidginhost/csm/internal/state"
)
func processStatusUID(data []byte) (int, bool) {
for _, line := range strings.Split(string(data), "\n") {
if strings.HasPrefix(line, "Uid:\t") {
fields := strings.Fields(strings.TrimPrefix(line, "Uid:\t"))
if len(fields) == 0 {
return 0, false
}
uid, err := strconv.Atoi(fields[0])
return uid, err == nil && uid >= 0
}
}
return 0, false
}
func processIdentityForUID(uid int) (string, string) {
if uid < 0 || uid > math.MaxUint32 {
return "", ""
}
// #nosec G115 -- uid is range-checked against math.MaxUint32 above.
user := LookupUser(uint32(uid))
if uid >= 1000 && user != "" && !strings.HasPrefix(user, "uid:") {
return user, user
}
return user, ""
}
// suspiciousExeNames flags processes whose exe basename contains any of
// these substrings. Shared between the periodic CheckSuspiciousProcesses
// and the live BPF exec backend (which cannot see cmdline patterns and
// relies on exe-name + exe-path matching).
var suspiciousExeNames = []string{"defunct", "gsocket", "gs-netcat", "gs-sftp"}
// suspiciousExePaths flags processes whose exe path contains any of these
// directory prefixes. Shared with the live BPF exec backend.
var suspiciousExePaths = []string{"/tmp/", "/dev/shm/", "/.config/"}
// suspiciousCmdlinePatterns is checked only by the periodic
// CheckSuspiciousProcesses; the BPF exec backend cannot read cmdline at
// the moment of exec.
var suspiciousCmdlinePatterns = []string{
"/bin/sh -i", "/bin/bash -i", "bash -i",
"/dev/tcp/", "semutmerah", "gsocket",
"reverse", "nc -e", "ncat -e",
}
// EvaluateExec returns findings for a single execve event observed by the
// BPF live backend. Inputs are the (UID, PID, comm, exe, parentComm)
// tuple the kernel hook collects. It stamps the detection time because these
// findings enter the realtime bus directly. The legacy periodic checks
// (CheckSuspiciousProcesses, CheckFakeKernelThreads) keep using cmdline-aware
// detection that this function cannot replicate.
func EvaluateExec(uid uint32, pid uint32, comm, exe, parentComm string) []alert.Finding {
var out []alert.Finding
pidInt := int(pid)
detectedAt := time.Now()
if uid != 0 && len(comm) >= 2 && comm[0] == '[' && comm[len(comm)-1] == ']' {
out = append(out, alert.Finding{
Severity: alert.Critical,
Check: "fake_kernel_thread",
Message: fmt.Sprintf("Non-root process masquerading as kernel thread: %s", comm),
Details: fmt.Sprintf("PID: %d, UID: %d, exe: %s, parent: %s", pid, uid, exe, parentComm),
PID: pidInt,
Timestamp: detectedAt,
})
}
if uid == 0 {
return out
}
exeName := filepath.Base(exe)
exeNameLower := strings.ToLower(exeName)
for _, s := range suspiciousExeNames {
if strings.Contains(exeNameLower, s) {
out = append(out, alert.Finding{
Severity: alert.Critical,
Check: "suspicious_process",
Message: fmt.Sprintf("Suspicious process name: %s", exeName),
Details: fmt.Sprintf("PID: %d, UID: %d, exe: %s, comm: %s, parent: %s", pid, uid, exe, comm, parentComm),
PID: pidInt,
Timestamp: detectedAt,
})
break
}
}
for _, p := range suspiciousExePaths {
if strings.Contains(exe, p) {
out = append(out, alert.Finding{
Severity: alert.High,
Check: "suspicious_process",
Message: fmt.Sprintf("Process running from suspicious path: %s", exe),
Details: fmt.Sprintf("PID: %d, UID: %d, comm: %s, parent: %s", pid, uid, comm, parentComm),
PID: pidInt,
Timestamp: detectedAt,
})
break
}
}
return out
}
func CheckFakeKernelThreads(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
procs, _ := osFS.Glob("/proc/[0-9]*/status")
for _, statusPath := range procs {
pid := filepath.Base(filepath.Dir(statusPath))
data, err := osFS.ReadFile(statusPath)
if err != nil {
continue
}
var name, uid string
for _, line := range strings.Split(string(data), "\n") {
if strings.HasPrefix(line, "Name:\t") {
name = strings.TrimPrefix(line, "Name:\t")
}
if strings.HasPrefix(line, "Uid:\t") {
fields := strings.Fields(strings.TrimPrefix(line, "Uid:\t"))
if len(fields) > 0 {
uid = fields[0]
}
}
}
// Kernel threads run as root (uid 0). Non-root process with
// a name that looks like a kernel thread is suspicious.
if uid == "0" || uid == "" {
continue
}
// Read cmdline - real kernel threads have empty cmdline
cmdline, _ := osFS.ReadFile(filepath.Join("/proc", pid, "cmdline"))
cmdStr := strings.TrimRight(strings.ReplaceAll(string(cmdline), "\x00", " "), " ")
safeCmdStr := redactProcCommandLine(cmdline)
// Check if the process name contains brackets (faking kernel thread)
// or if cmdline starts with [
if strings.HasPrefix(cmdStr, "[") || strings.HasPrefix(name, "[") {
// This is a non-root process masquerading as a kernel thread
exe, _ := osFS.Readlink(filepath.Join("/proc", pid, "exe"))
uidInt, _ := strconv.Atoi(uid)
pidInt, _ := strconv.Atoi(pid)
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "fake_kernel_thread",
Timestamp: time.Now(),
Message: fmt.Sprintf("Non-root process masquerading as kernel thread: [%s]", name),
Details: fmt.Sprintf("PID: %s, UID: %d, exe: %s, cmdline: %s", pid, uidInt, exe, safeCmdStr),
PID: pidInt,
})
}
}
return findings
}
func CheckSuspiciousProcesses(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
if ctx.Err() != nil {
return nil
}
suspiciousNames := suspiciousExeNames
suspiciousCmdline := suspiciousCmdlinePatterns
suspiciousPaths := suspiciousExePaths
docroots := NewDocrootIndex(ResolveWebRoots(cfg))
procs, _ := osFS.Glob("/proc/[0-9]*/exe")
for _, exePath := range procs {
if ctx.Err() != nil {
break
}
pid := filepath.Base(filepath.Dir(exePath))
pidInt, _ := strconv.Atoi(pid)
statusData, _ := osFS.ReadFile(filepath.Join("/proc", pid, "status"))
var uid string
for _, line := range strings.Split(string(statusData), "\n") {
if strings.HasPrefix(line, "Uid:\t") {
fields := strings.Fields(strings.TrimPrefix(line, "Uid:\t"))
if len(fields) > 0 {
uid = fields[0]
}
}
}
if uid == "0" {
continue // Skip root processes for this check
}
exe, _ := osFS.Readlink(exePath)
cmdline, _ := osFS.ReadFile(filepath.Join("/proc", pid, "cmdline"))
cmdStr := strings.TrimRight(strings.ReplaceAll(string(cmdline), "\x00", " "), " ")
safeCmdStr := redactProcCommandLine(cmdline)
// Check executable name
exeName := filepath.Base(exe)
for _, s := range suspiciousNames {
if strings.Contains(strings.ToLower(exeName), s) {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "suspicious_process",
Message: fmt.Sprintf("Suspicious process name: %s", exeName),
Timestamp: time.Now(),
Details: fmt.Sprintf("PID: %s, UID: %s, exe: %s, cmdline: %s", pid, uid, exe, safeCmdStr),
PID: pidInt,
})
}
}
// Check cmdline for suspicious patterns
cmdLower := strings.ToLower(cmdStr)
for _, s := range suspiciousCmdline {
if strings.Contains(cmdLower, strings.ToLower(s)) {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "suspicious_process",
Message: fmt.Sprintf("Suspicious cmdline pattern: %s", s),
Timestamp: time.Now(),
Details: fmt.Sprintf("PID: %s, UID: %s, exe: %s, cmdline: %s", pid, uid, exe, safeCmdStr),
PID: pidInt,
})
break
}
}
// Check executable path
for _, s := range suspiciousPaths {
if strings.Contains(exe, s) {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "suspicious_process",
Message: fmt.Sprintf("Process running from suspicious path: %s", exe),
Timestamp: time.Now(),
Details: fmt.Sprintf("PID: %s, UID: %s, cmdline: %s", pid, uid, safeCmdStr),
PID: pidInt,
})
break
}
}
if uidInt, err := strconv.Atoi(uid); err == nil && docroots.Lookup(strings.TrimSuffix(exe, " (deleted)")) != "" {
if f := InspectDocrootExec(cfg, pidInt, uidInt, exe, procParentComm(statusData), docroots); f != nil {
findings = append(findings, *f)
}
}
}
return findings
}
// procParentComm returns the name of the parent of the process whose
// /proc/<pid>/status is given, or "" when it cannot be read.
func procParentComm(status []byte) string {
for _, line := range strings.Split(string(status), "\n") {
if ppid, ok := strings.CutPrefix(line, "PPid:"); ok {
comm, err := osFS.ReadFile(filepath.Join("/proc", strings.TrimSpace(ppid), "comm"))
if err != nil {
return ""
}
return strings.TrimSpace(string(comm))
}
}
return ""
}
// CheckPHPProcesses inspects running lsphp processes to detect active
// webshell execution. Only reads /proc cmdline - zero disk I/O.
func CheckPHPProcesses(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
suspiciousPHPPaths := []string{
"/tmp/",
"/dev/shm/",
"/wp-content/uploads/",
"/.config/",
}
procs, _ := osFS.Glob("/proc/[0-9]*/cmdline")
for _, cmdPath := range procs {
pid := filepath.Base(filepath.Dir(cmdPath))
pidInt, _ := strconv.Atoi(pid)
cmdline, err := osFS.ReadFile(cmdPath)
if err != nil {
continue
}
cmdStr := strings.ReplaceAll(string(cmdline), "\x00", " ")
safeCmdStr := redactProcCommandLine(cmdline)
// Only check lsphp processes
if !strings.Contains(cmdStr, "lsphp") {
continue
}
for _, sus := range suspiciousPHPPaths {
if strings.Contains(cmdStr, sus) {
statusData, _ := osFS.ReadFile(filepath.Join("/proc", pid, "status"))
uid, uidOK := processStatusUID(statusData)
uidText := ""
var proc *processctx.ProcessContext
if uidOK {
uidText = strconv.Itoa(uid)
userName, account := processIdentityForUID(uid)
proc = &processctx.ProcessContext{
PID: pidInt,
UID: uid,
User: userName,
Account: account,
}
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "php_suspicious_execution",
Message: fmt.Sprintf("PHP executing from suspicious path: %s", sus),
Details: fmt.Sprintf("PID: %s, UID: %s, cmdline: %s", pid, uidText, safeCmdStr),
PID: pidInt,
Process: proc,
})
break
}
}
}
return findings
}
func redactProcCommandLine(raw []byte) string {
return strings.TrimSpace(alert.RedactCommandLine(string(raw)))
}
package checks
import (
"slices"
"github.com/pidginhost/csm/internal/admission"
)
// ProducerEntry is one producer of admission evidence: what it registers,
// and the parser its observations name.
type ProducerEntry struct {
Spec admission.ProducerSpec
Parser admission.ParserRef
}
// Producer IDs the readers name when they stamp an observation.
const (
ProducerSSHLog admission.ProducerID = "sshd_log"
ProducerSSHLoginScan admission.ProducerID = "ssh_login_scan"
ProducerPAMSocket admission.ProducerID = "pam_socket"
ProducerFTPLog admission.ProducerID = "ftp_log"
ProducerFTPScan admission.ProducerID = "ftp_scan"
ProducerEximLog admission.ProducerID = "exim_log"
ProducerEximHistory admission.ProducerID = "exim_history"
ProducerMailLog admission.ProducerID = "mail_log"
ProducerStalwartLog admission.ProducerID = "stalwart_log"
ProducerAccessLog admission.ProducerID = "access_log"
ProducerDomlogScan admission.ProducerID = "domlog_scan"
ProducerModSecLog admission.ProducerID = "modsec_log"
ProducerModSecAuditScan admission.ProducerID = "modsec_audit_scan"
ProducerCpanelAccessLog admission.ProducerID = "cpanel_access_log"
ProducerCpanelScan admission.ProducerID = "cpanel_access_scan"
ProducerReputationScan admission.ProducerID = "reputation_scan"
ProducerConnectionScan admission.ProducerID = "connection_scan"
ProducerPHPRelaySpool admission.ProducerID = "php_relay_spool"
)
// The attribution kinds each producer's parsers read addresses by.
var (
mintsPeer = admission.Attributions(admission.AttributionPeer)
mintsHTTPPeer = admission.Attributions(admission.AttributionHTTPPeer)
mintsHTTPClient = admission.Attributions(admission.AttributionHTTPPeer, admission.AttributionTrustedHTTPChain)
mintsDerived = admission.Attributions(admission.AttributionDerived)
mintsDestination = admission.Attributions(admission.AttributionSocketDestination)
)
// producerTable lists every producer of address evidence. Realtime and
// scheduled readers are separate producers because each has its own
// observation stream. Admission has no realtime entry: both reach the
// single-address scan admission. Claims are the kinds a producer's findings
// may carry: a vhost log names a domain, a login names an authenticated
// account or mailbox, a request names a host it chose itself.
var producerTable = []ProducerEntry{
{admission.ProducerSpec{ID: ProducerSSHLog, Entry: admission.EntryScan, Observation: admission.ObservationLogCursor,
Checks: []string{"ssh_login_unknown_ip"}, Claims: []admission.ClaimKind{admission.ClaimAccount}, Attribution: mintsPeer},
admission.ParserRef{Name: "sshd", Version: 1}},
{admission.ProducerSpec{ID: ProducerSSHLoginScan, Entry: admission.EntryScan, Observation: admission.ObservationLogCursor,
Checks: []string{"ssh_login_unknown_ip"}, Claims: []admission.ClaimKind{admission.ClaimAccount}, Attribution: mintsPeer},
admission.ParserRef{Name: "sshd", Version: 1}},
{admission.ProducerSpec{ID: ProducerPAMSocket, Entry: admission.EntryScan, Observation: admission.ObservationEventSeq,
Checks: []string{"pam_bruteforce", "credential_stuffing"}, Attribution: mintsPeer},
admission.ParserRef{Name: "pam_csm", Version: 1}},
{admission.ProducerSpec{ID: ProducerFTPLog, Entry: admission.EntryScan, Observation: admission.ObservationLogCursor,
Checks: []string{"ftp_auth_failure_realtime"}, Attribution: mintsPeer},
admission.ParserRef{Name: "pureftpd", Version: 1}},
{admission.ProducerSpec{ID: ProducerFTPScan, Entry: admission.EntryScan, Observation: admission.ObservationLogCursor,
Checks: []string{"ftp_bruteforce"}, Attribution: mintsPeer},
admission.ParserRef{Name: "pureftpd", Version: 1}},
{admission.ProducerSpec{ID: ProducerEximLog, Entry: admission.EntryScan, Observation: admission.ObservationLogCursor,
Checks: []string{"smtp_bruteforce", "smtp_subnet_spray", "smtp_probe_abuse", "email_cloud_relay_abuse", "mail_account_compromised"},
Claims: []admission.ClaimKind{admission.ClaimAccount, admission.ClaimMailbox}, Attribution: mintsPeer},
admission.ParserRef{Name: "exim", Version: 1}},
{admission.ProducerSpec{ID: ProducerEximHistory, Entry: admission.EntryScan, Observation: admission.ObservationScanPass,
Checks: []string{"email_cloud_relay_abuse"}, Claims: []admission.ClaimKind{admission.ClaimAccount, admission.ClaimMailbox}, Attribution: mintsPeer},
admission.ParserRef{Name: "exim", Version: 1}},
{admission.ProducerSpec{ID: ProducerMailLog, Entry: admission.EntryScan, Observation: admission.ObservationLogCursor,
// Postfix hosts track SMTP auth failures from the mail log too.
Checks: []string{"mail_bruteforce", "mail_subnet_spray", "mail_account_compromised", "smtp_bruteforce", "smtp_subnet_spray"},
Claims: []admission.ClaimKind{admission.ClaimAccount, admission.ClaimMailbox}, Attribution: mintsPeer},
admission.ParserRef{Name: "dovecot", Version: 1}},
{admission.ProducerSpec{ID: ProducerStalwartLog, Entry: admission.EntryScan, Observation: admission.ObservationLogCursor,
// SMTP, IMAP, POP3 and ManageSieve listeners log the TCP peer.
// HTTP listener events never carry an attribution.
Checks: []string{"mail_bruteforce", "mail_subnet_spray", "mail_account_compromised", "smtp_bruteforce", "smtp_subnet_spray", "email_cloud_relay_abuse", "stalwart_ban"},
Claims: []admission.ClaimKind{admission.ClaimAccount, admission.ClaimMailbox}, Attribution: mintsPeer},
admission.ParserRef{Name: "stalwart", Version: 1}},
{admission.ProducerSpec{ID: ProducerAccessLog, Entry: admission.EntryScan, Observation: admission.ObservationLogCursor,
Checks: []string{"wp_login_bruteforce", "xmlrpc_abuse", "admin_panel_bruteforce"}, Attribution: mintsHTTPPeer},
admission.ParserRef{Name: "access_log", Version: 1}},
{admission.ProducerSpec{ID: ProducerDomlogScan, Entry: admission.EntryScan, Observation: admission.ObservationScanPass,
Checks: []string{"wp_login_bruteforce", "xmlrpc_abuse", "wp_user_enumeration", "http_request_flood", "http_ua_spoof",
"http_scanner_profile", "http_claimed_bot_unverified", "http_asn_crawl"},
Claims: []admission.ClaimKind{admission.ClaimDomain}, Attribution: mintsHTTPClient},
admission.ParserRef{Name: "access_log", Version: 1}},
{admission.ProducerSpec{ID: ProducerModSecLog, Entry: admission.EntryScan, Observation: admission.ObservationLogCursor,
Checks: []string{"modsec_block_escalation", "modsec_csm_block_escalation"}, Claims: []admission.ClaimKind{admission.ClaimRequestName}, Attribution: mintsHTTPPeer},
admission.ParserRef{Name: "modsec", Version: 1}},
{admission.ProducerSpec{ID: ProducerModSecAuditScan, Entry: admission.EntryScan, Observation: admission.ObservationScanPass,
Checks: []string{"waf_attack_blocked"}, Attribution: mintsHTTPPeer},
admission.ParserRef{Name: "modsec_audit", Version: 1}},
{admission.ProducerSpec{ID: ProducerCpanelAccessLog, Entry: admission.EntryScan, Observation: admission.ObservationLogCursor,
Checks: []string{"api_auth_failure_realtime"}, Attribution: mintsPeer},
admission.ParserRef{Name: "cpsrvd", Version: 1}},
{admission.ProducerSpec{ID: ProducerCpanelScan, Entry: admission.EntryScan, Observation: admission.ObservationScanPass,
Checks: []string{"api_auth_failure", "webmail_bruteforce"}, Attribution: mintsPeer},
admission.ParserRef{Name: "cpsrvd", Version: 1}},
{admission.ProducerSpec{ID: ProducerReputationScan, Entry: admission.EntryScan, Observation: admission.ObservationScanPass,
Checks: []string{"ip_reputation"}, Attribution: mintsDerived},
admission.ParserRef{Name: "reputation", Version: 1}},
{admission.ProducerSpec{ID: ProducerConnectionScan, Entry: admission.EntryScan, Observation: admission.ObservationScanPass,
Checks: []string{"c2_connection"}, Attribution: mintsDestination},
admission.ParserRef{Name: "proc_net_tcp", Version: 1}},
{admission.ProducerSpec{ID: ProducerPHPRelaySpool, Entry: admission.EntryScan, Observation: admission.ObservationEventSeq,
// The client address is PHP's REMOTE_ADDR as written into a mail
// header any local account can forge: no socket CSM observed, so no
// attribution kind, and its findings mint nothing.
Checks: []string{"email_php_form_relay"}},
admission.ParserRef{Name: "exim_spool", Version: 1}},
}
// ProducerTable returns a copy of the producer table.
func ProducerTable() []ProducerEntry {
out := make([]ProducerEntry, len(producerTable))
for i, p := range producerTable {
out[i] = p
out[i].Spec.Checks = slices.Clone(p.Spec.Checks)
out[i].Spec.Claims = slices.Clone(p.Spec.Claims)
}
return out
}
// derivedEntries bind each derived entry to the root checks it may answer
// again. A derived response never mints evidence of its own: it carries a
// root a producer of the table minted.
var derivedEntries = []admission.ProducerSpec{
{ID: "mail_subnet", Entry: admission.EntryMailSubnet, Observation: admission.ObservationLogCursor,
Checks: []string{"mail_subnet_spray", "smtp_subnet_spray"}},
{ID: "asn_crawl", Entry: admission.EntryASNCrawl, Observation: admission.ObservationScanPass,
Checks: []string{"http_asn_crawl"}},
}
// DerivedEntries returns the derived entries' producer specs. A challenge
// timeout answers the checks routed to the challenge; central intel and
// incidents answer the published checks that observe a local attack.
func DerivedEntries() []admission.ProducerSpec {
var published, challenged []string
seen := map[string]bool{}
for _, p := range producerTable {
for _, check := range p.Spec.Checks {
if seen[check] {
continue
}
seen[check] = true
if ResponsePolicyFor(check).ChallengeFirst {
challenged = append(challenged, check)
}
if _, pol, ok := AdmissionPolicy(check); ok && pol.Family.LocalAttack() {
published = append(published, check)
}
}
}
out := []admission.ProducerSpec{
{ID: "challenge_timeout", Entry: admission.EntryChallengeTimeout, Observation: admission.ObservationEventSeq, Checks: challenged},
{ID: "central", Entry: admission.EntryCentral, Observation: admission.ObservationEventSeq, Checks: slices.Clone(published)},
{ID: "incident", Entry: admission.EntryIncident, Observation: admission.ObservationEventSeq, Checks: slices.Clone(published)},
{ID: "incident_spray", Entry: admission.EntryIncidentSpray, Observation: admission.ObservationEventSeq, Checks: slices.Clone(published)},
}
for _, spec := range derivedEntries {
spec.Checks = slices.Clone(spec.Checks)
out = append(out, spec)
}
return out
}
// ProducerParser returns the parser a producer's observations name.
func ProducerParser(id admission.ProducerID) (admission.ParserRef, bool) {
for _, p := range producerTable {
if p.Spec.ID == id {
return p.Parser, true
}
}
return admission.ParserRef{}, false
}
package checks
import (
"context"
"errors"
"io"
"os"
"path/filepath"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/safepath"
)
// ---------------------------------------------------------------------------
// OS abstraction — filesystem read operations
// ---------------------------------------------------------------------------
// OS abstracts filesystem operations (read and write) used by check functions.
// Production code uses realOS{}; tests swap in a mockOS via SetOS().
type OS interface {
ReadFile(name string) ([]byte, error)
ReadDir(name string) ([]os.DirEntry, error)
Stat(name string) (os.FileInfo, error)
Lstat(name string) (os.FileInfo, error)
Readlink(name string) (string, error)
Open(name string) (*os.File, error)
WriteFile(name string, data []byte, perm os.FileMode) error
MkdirAll(path string, perm os.FileMode) error
Remove(name string) error
Glob(pattern string) ([]string, error)
}
type realOS struct{}
var errNonRegularFile = errors.New("not a regular file")
var errFileChanged = errors.New("file changed while reading")
// #nosec G304 -- filesystem abstraction; check functions pass trusted paths.
func (realOS) ReadFile(name string) ([]byte, error) { return os.ReadFile(name) }
func (realOS) ReadDir(name string) ([]os.DirEntry, error) { return os.ReadDir(name) }
func (realOS) Stat(name string) (os.FileInfo, error) { return os.Stat(name) }
func (realOS) Lstat(name string) (os.FileInfo, error) { return os.Lstat(name) }
func (realOS) Readlink(name string) (string, error) { return os.Readlink(name) }
// #nosec G304 -- filesystem abstraction; check functions pass trusted paths.
func (realOS) openRegularFile(name string, extraFlags int) (*os.File, error) {
fd, err := unix.Open(name, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NONBLOCK|extraFlags, 0)
if err != nil {
return nil, err
}
// #nosec G115 -- unix.Open returns a non-negative descriptor whenever err is nil.
file := os.NewFile(uintptr(fd), name)
info, err := file.Stat()
if err != nil {
_ = file.Close()
return nil, err
}
if !info.Mode().IsRegular() {
_ = file.Close()
return nil, errNonRegularFile
}
return file, nil
}
// ReadRegularFile opens without blocking on a path raced to a FIFO, then
// verifies the opened object before reading from the inode bound to the fd.
func (r realOS) ReadRegularFile(name string) ([]byte, error) {
file, err := r.openRegularFile(name, 0)
if err != nil {
return nil, err
}
defer file.Close()
return io.ReadAll(file)
}
// ReadRegularFilePrefix bounds consumers that need only an admission prefix.
// Refusing symlinks and matching the opened inode to the caller's snapshot
// stop a path replacement from redirecting a content-based admission decision.
func (r realOS) ReadRegularFilePrefix(name string, expected os.FileInfo, limit int64) ([]byte, error) {
file, err := r.openRegularFile(name, unix.O_NOFOLLOW)
if err != nil {
return nil, err
}
defer file.Close()
opened, err := file.Stat()
if err != nil {
return nil, err
}
if !sameFileSnapshot(expected, opened) {
return nil, errFileChanged
}
prefix, err := io.ReadAll(io.LimitReader(file, limit))
if err != nil {
return prefix, err
}
after, err := file.Stat()
if err != nil {
return prefix, err
}
if !sameFileSnapshot(opened, after) {
return prefix, errFileChanged
}
return prefix, nil
}
func sameFileSnapshot(expected, actual os.FileInfo) bool {
return expected != nil && actual != nil &&
os.SameFile(expected, actual) &&
expected.Mode() == actual.Mode() &&
expected.Size() == actual.Size() &&
expected.ModTime().Equal(actual.ModTime())
}
// openTenantRegularFile opens a file a tenant controls. It does not follow the
// final symlink or wait for a FIFO writer, and it validates the opened object.
func openTenantRegularFile(path string) (*os.File, error) {
if fs, production := osFS.(realOS); production {
return fs.openRegularFile(path, unix.O_NOFOLLOW)
}
file, err := osFS.Open(path)
if err != nil {
return nil, err
}
info, err := file.Stat()
if err != nil {
_ = file.Close()
return nil, err
}
if !info.Mode().IsRegular() {
_ = file.Close()
return nil, errNonRegularFile
}
return file, nil
}
// tenantDirectory keeps content opens bound to the directory being enumerated.
// Test providers retain their injected Open implementation.
type tenantDirectory struct {
file *os.File
pinned *safepath.Dir
path string
}
func openTenantDirectory(path string) (*tenantDirectory, error) {
if _, production := osFS.(realOS); production {
dir, err := safepath.OpenDirNoFollow(path)
if err != nil {
return nil, err
}
return &tenantDirectory{pinned: dir, path: path}, nil
}
file, err := osFS.Open(path)
if err != nil {
return nil, err
}
return &tenantDirectory{file: file, path: path}, nil
}
func (d *tenantDirectory) Close() error {
if d.pinned != nil {
return d.pinned.Close()
}
return d.file.Close()
}
func (d *tenantDirectory) Stat() (os.FileInfo, error) {
if d.pinned != nil {
return d.pinned.Info()
}
return d.file.Stat()
}
func (d *tenantDirectory) ReadDir(n int) ([]os.DirEntry, error) {
if d.pinned != nil {
return d.pinned.ReadDirN(n)
}
return d.file.ReadDir(n)
}
func (d *tenantDirectory) open(name string) (*os.File, error) {
if d.pinned != nil {
return d.pinned.OpenFile(name, os.O_RDONLY, 0)
}
return openTenantRegularFile(filepath.Join(d.path, name))
}
// readRegularFileHead returns at most limit bytes of a regular file and whether the
// file held more. Production refuses a final symlink and never reads past
// limit+1 bytes; test filesystems fall back to ReadFile and truncate.
func readRegularFileHead(path string, limit int64) (data []byte, truncated bool, err error) {
if fs, production := osFS.(realOS); production {
var file *os.File
if file, err = fs.openRegularFile(path, unix.O_NOFOLLOW); err != nil {
return nil, false, err
}
defer file.Close()
data, err = io.ReadAll(io.LimitReader(file, limit+1))
if err != nil {
return nil, false, err
}
} else if data, err = osFS.ReadFile(path); err != nil {
return nil, false, err
}
if int64(len(data)) > limit {
return data[:limit], true, nil
}
return data, false, nil
}
// #nosec G304 -- filesystem abstraction; check functions pass trusted paths.
func (realOS) Open(name string) (*os.File, error) { return os.Open(name) }
// #nosec G306 -- callers pass explicit perm; intent is operator-readable mode.
func (realOS) WriteFile(name string, data []byte, perm os.FileMode) error {
return os.WriteFile(name, data, perm)
}
func (realOS) MkdirAll(path string, perm os.FileMode) error { return os.MkdirAll(path, perm) }
func (realOS) Remove(name string) error { return os.Remove(name) }
func (realOS) Glob(pattern string) ([]string, error) { return filepath.Glob(pattern) }
// osFS is the package-level filesystem provider. All check functions use
// this instead of calling os.ReadFile / os.ReadDir / etc. directly.
var osFS OS = realOS{}
// SetOS replaces the filesystem provider. Used by tests to inject mocks.
func SetOS(o OS) { osFS = o }
// ---------------------------------------------------------------------------
// CmdRunner abstraction — external command execution
// ---------------------------------------------------------------------------
// CmdRunner abstracts external command execution used by check functions.
// Production code uses realCmd{}; tests swap in a mockCmdRunner via SetCmdRunner().
//
// RunContext returns stdout+stderr merged (CombinedOutput) and is fine for
// tools that only write to stdout. RunContextStdout returns stdout only and
// should be used when the command prints structured output (JSON, a URL, ...)
// on stdout and chatter (warnings, PHP notices, MySQL deprecations) on stderr
// -- mixing them there would corrupt the parse. RunContextStdout also surfaces
// context.DeadlineExceeded on timeout so callers can distinguish "no output"
// from "empty output".
type CmdRunner interface {
Run(name string, args ...string) ([]byte, error)
RunAllowNonZero(name string, args ...string) ([]byte, error)
RunContext(parent context.Context, name string, args ...string) ([]byte, error)
RunContextStdout(parent context.Context, name string, args ...string) ([]byte, error)
RunWithEnv(name string, args []string, extraEnv ...string) ([]byte, error)
LookPath(file string) (string, error)
}
type realCmd struct{}
func (realCmd) Run(name string, args ...string) ([]byte, error) {
return runCmdReal(name, args...)
}
func (realCmd) RunAllowNonZero(name string, args ...string) ([]byte, error) {
return runCmdAllowNonZeroReal(name, args...)
}
func (realCmd) RunContext(parent context.Context, name string, args ...string) ([]byte, error) {
return runCmdCombinedContextReal(parent, name, args...)
}
func (realCmd) RunContextStdout(parent context.Context, name string, args ...string) ([]byte, error) {
return runCmdStdoutContextReal(parent, name, args...)
}
func (realCmd) RunWithEnv(name string, args []string, extraEnv ...string) ([]byte, error) {
return runCmdWithEnvReal(name, args, extraEnv...)
}
func (realCmd) LookPath(file string) (string, error) {
return lookupSystemCommand(file)
}
// cmdExec is the package-level command runner. All check functions use
// this instead of calling runCmd / exec.Command directly.
var cmdExec CmdRunner = realCmd{}
// SetCmdRunner replaces the command runner. Used by tests to inject mocks.
func SetCmdRunner(r CmdRunner) { cmdExec = r }
package checks
import (
"net/url"
"path"
"strings"
)
// proxySubdomainPath prefixes the requests LiteSpeed passes to the panel.
const proxySubdomainPath = "/___proxy_subdomain_"
// IsProxiedPanelRequest reports a web-server log record of a request that a
// cPanel proxy subdomain passed to the panel: the proxy path LiteSpeed logs
// in its central log. A client chooses the logged path, so the path counts
// only in a recognized central log (central), where proxied requests land,
// and only when it still starts with the prefix after the server's own
// decoding and dot-segment removal: a request a site actually served is never
// skipped. No log field names the proxy vhost reliably: a trailing quoted
// field can be a client header, and Apache keeps proxied requests out of its
// traffic log. The panel's own log carries these requests, so one request
// never counts as two families of evidence.
func IsProxiedPanelRequest(uri string, central bool) bool {
if !central || !strings.HasPrefix(uri, proxySubdomainPath) {
return false
}
raw, _, _ := strings.Cut(uri, "?")
decoded, err := url.PathUnescape(raw)
if err != nil {
return false
}
return strings.HasPrefix(path.Clean(decoded), proxySubdomainPath)
}
package checks
import (
"encoding/json"
"os"
"syscall"
"time"
)
// QuarantineMeta stores original file metadata alongside quarantined files.
type QuarantineMeta struct {
OriginalPath string `json:"original_path"`
Owner int `json:"owner_uid"`
Group int `json:"group_gid"`
Mode string `json:"mode"`
Size int64 `json:"size"`
QuarantineAt time.Time `json:"quarantined_at"`
OriginalModTime time.Time `json:"original_mtime,omitzero"`
Reason string `json:"reason"`
// FindingID ties the quarantine to the finding that caused it, using the
// same identifier the audit log and the action log emit. Empty when the
// quarantine came from an operator command rather than a detection.
FindingID string `json:"finding_id,omitempty"`
MessageID string `json:"message_id,omitempty"`
SpoolDir string `json:"spool_dir,omitempty"`
RestoreAction string `json:"restore_action,omitempty"`
ExpectedCurrentSHA256 string `json:"expected_current_sha256,omitempty"`
}
// UnmarshalJSON accepts the timestamp spelling used by historical manual
// fixes. Missing original mtimes remain unknown; archive mtimes are not evidence
// of when the original was modified.
func (m *QuarantineMeta) UnmarshalJSON(data []byte) error {
type wire QuarantineMeta
var decoded struct {
wire
LegacyQuarantineAt time.Time `json:"quarantine_at"`
}
if err := json.Unmarshal(data, &decoded); err != nil {
return err
}
if decoded.QuarantineAt.IsZero() {
decoded.QuarantineAt = decoded.LegacyQuarantineAt
}
*m = QuarantineMeta(decoded.wire)
return nil
}
func quarantineMetadata(path string, info os.FileInfo, reason string) QuarantineMeta {
meta := QuarantineMeta{
OriginalPath: path,
Mode: info.Mode().String(),
Size: info.Size(),
QuarantineAt: time.Now().UTC(),
OriginalModTime: info.ModTime().UTC(),
Reason: reason,
}
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
meta.Owner, meta.Group = int(stat.Uid), int(stat.Gid)
}
return meta
}
package checks
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"path/filepath"
"strings"
"time"
)
// quarantineSafeNameMax keeps the generated name, plus the timestamp prefix
// callers add, under the 255-byte filename limit.
const quarantineSafeNameMax = 180
func newQuarantinePath(dir, original string) string {
// Repeated cleanups can occur within one second. Independent names keep
// a new recovery point from replacing the only copy of an earlier state.
return filepath.Join(dir, time.Now().UTC().Format("20060102-150405")+"_"+rand.Text()+"_"+quarantineSafeName(original))
}
// quarantineSafeName turns a source path into one flat quarantine filename.
// Short paths keep the familiar slash-to-underscore form. A path that would
// exceed the filename limit is shortened to a hash of the whole path plus
// its tail, so the file name survives, two long paths cannot collide, and
// the move no longer fails with ENAMETOOLONG (which left the malware in
// place while the finding reported a quarantine attempt).
func quarantineSafeName(path string) string {
flat := strings.ReplaceAll(path, "/", "_")
if len(flat) <= quarantineSafeNameMax {
return flat
}
sum := sha256.Sum256([]byte(path))
prefix := hex.EncodeToString(sum[:6])
tail := flat[len(flat)-(quarantineSafeNameMax-len(prefix)-1):]
return prefix + "_" + tail
}
//go:build linux
package checks
import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"syscall"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/quarantinefs"
"github.com/pidginhost/csm/internal/safepath"
)
// var so Linux tests can interleave a source-path mutation after the verified
// fd is open without racing the test process itself.
var quarantineCopyByFD = func(fd *os.File, qPath string, metadata []byte, size int64) error {
// Generic quarantine permits open writers. Bound the read to the verified
// size so a growing file cannot keep capture copying indefinitely.
return quarantinefs.Store(qPath, io.NewSectionReader(fd, 0, size), metadata, 0600)
}
// quarantineFileTOCTOUSafe moves a single regular file into quarantine in
// a way that defends against the classic detect-then-quarantine race: an
// attacker who controls the directory can swap a legitimate file in
// between Lstat and Rename, tricking CSM into moving the wrong file out
// of the user's home. The defence:
//
// 1. Open the path with O_RDONLY|O_NOFOLLOW. Symlinks are refused at
// the kernel level; the fd is bound to the inode that existed at
// open time.
// 2. Fstat the fd and verify it still matches the inode we detected
// earlier (sameFileIdentity). A late swap loses here.
// 3. Realtime PHP response takes an exclusive Linux file lease, so another
// open descriptor refuses the action and a later open breaks the lease
// before removal. Under the lease it checks the file still starts with
// the scanned bytes, because a coarse-clock kernel can rewrite it within
// one timestamp tick. Other callers skip both: Linux never grants the
// lease while any other process has the file open or is executing it.
// 4. Copy from that verified fd into a private, independent quarantine
// inode. A hardlink is never used: the account could add another name
// after the initial fstat and keep the quarantine inode writable.
// Persist the copy, metadata, and quarantine directory before unlinking.
// 5. Atomically move the source name into a private directory on the same
// filesystem, validate the captured inode, and unlink only that private
// name. A replacement created at the original name is left alone.
//
// Returns nil on success. Errors describe what failed; callers should
// not retry blindly because a failure usually means the file moved.
func quarantineFileTOCTOUSafe(path, qPath string, originalInfo os.FileInfo, metadata []byte) error {
return quarantineFileTOCTOUSafeMode(path, qPath, originalInfo, metadata, false, -1, nil)
}
func quarantineFileVersionedTOCTOUSafe(path, qPath string, originalInfo os.FileInfo, metadata, scannedHead []byte) error {
return quarantineFileVersionedTOCTOUSafeFD(path, qPath, originalInfo, metadata, -1, scannedHead)
}
func quarantineFileVersionedTOCTOUSafeFD(path, qPath string, originalInfo os.FileInfo, metadata []byte, sourceFD int, scannedHead []byte) error {
return quarantineFileTOCTOUSafeMode(path, qPath, originalInfo, metadata, true, sourceFD, func(fd *os.File) error {
return requireScannedHead(fd, scannedHead)
})
}
func quarantineFileTOCTOUSafeMode(path, qPath string, originalInfo os.FileInfo, metadata []byte, requireVersion bool, sourceFD int, verifyContent func(*os.File) error) error {
if originalInfo == nil {
return fmt.Errorf("quarantine: missing original stat")
}
if originalInfo.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("quarantine: refused symlink at %s", path)
}
// Pin the source directory before opening the file. Later capture uses the
// same descriptor, so renaming a tenant-controlled parent cannot redirect
// the unlink transaction to a different directory.
source, err := safepath.OpenDirNoFollow(filepath.Dir(path))
if err != nil {
return fmt.Errorf("quarantine: open source directory for %s: %w", path, err)
}
defer func() { _ = source.Close() }()
fd, err := openQuarantineSource(source, filepath.Base(path), sourceFD)
if err != nil {
return fmt.Errorf("quarantine: open %s: %w", path, fileResponseSourceError(err))
}
defer fd.Close()
// Re-stat the open fd and confirm the inode matches what we
// detected. The race window between Lstat and OpenFile is narrow
// but real - an attacker who hits it gets caught here.
cur, err := fd.Stat()
if err != nil {
return fmt.Errorf("quarantine: fstat %s: %w", path, err)
}
if !sameFileIdentity(cur, originalInfo) {
return refuseFileResponse(fmt.Errorf("quarantine: file at %s changed between detection and quarantine (TOCTOU)", path))
}
// Defence against inode reuse: on busy tmpfs / ext4 mounts the kernel
// can hand out the freed inode to whatever the attacker wrote next.
// A matching inode is necessary but not sufficient; also require the
// content shape (size + mtime) to match what the detector recorded.
if !sameContentShape(cur, originalInfo) || requireVersion && !SameFileVersion(cur, originalInfo) {
return refuseFileResponse(fmt.Errorf("quarantine: file at %s changed between detection and quarantine (TOCTOU, inode reused)", path))
}
// Refuse to quarantine a non-regular file (block, char, socket,
// FIFO). The detector only flags regular files, so a non-regular
// shape at this point means someone is trying to move CSM at a
// device node or pipe.
if !cur.Mode().IsRegular() {
return fmt.Errorf("quarantine: refusing non-regular file at %s (mode=%v)", path, cur.Mode())
}
var leased *os.File
if requireVersion {
if prepareErr := quarantineBeforeLease(path); prepareErr != nil {
return fmt.Errorf("quarantine: prepare stable content for %s: %w", path, prepareErr)
}
if leaseErr := acquireQuarantineLease(fd); leaseErr != nil {
return refuseFileResponse(fmt.Errorf("quarantine: cannot hold stable content for %s while another descriptor is open: %w", path, leaseErr))
}
defer releaseQuarantineLease(fd)
leased = fd
if contentErr := verifyContent(fd); contentErr != nil {
return refuseFileResponse(fmt.Errorf("quarantine: file at %s changed before atomic capture: %w", path, contentErr))
}
}
stable, err := fd.Stat()
if err != nil {
return fmt.Errorf("quarantine: fstat stable source %s: %w", path, err)
}
versionChanged := !SameFileVersion(stable, originalInfo)
if !sameFileIdentity(stable, originalInfo) || !sameContentShape(stable, originalInfo) || versionChanged && (requireVersion || fileLinkCount(stable) <= 1) {
return refuseFileResponse(fmt.Errorf("quarantine: file at %s changed before atomic capture", path))
}
cur = stable
// Always create an independent root-owned copy. Checking st_nlink before a
// hardlink is not sufficient: the account can add another name after the
// check and retain write access to the inode placed in quarantine.
if err = quarantineCopyByFD(fd, qPath, metadata, cur.Size()); err != nil {
return fmt.Errorf("quarantine: copy %s -> %s: %w", path, qPath, err)
}
err = removeQuarantinedSourcePinned(source, filepath.Base(path), path, qPath, cur, requireVersion, leased)
if err != nil {
return err
}
remaining, err := fd.Stat()
if err != nil {
return &quarantineCompletedWarning{message: fmt.Sprintf("quarantine: copied %s to %s and removed that name, but could not count surviving hard links: %v", path, qPath, err)}
}
if links := fileLinkCount(remaining); links > 0 {
return &quarantineCompletedWarning{message: fmt.Sprintf("quarantine: copied %s to %s and removed that name, but at least %d other hard link(s) to the same content remain reachable elsewhere", path, qPath, links)}
}
return nil
}
func removeQuarantinedSourcePlatform(path, qPath string, original os.FileInfo, requireVersion bool) error {
source, err := safepath.OpenDirNoFollow(filepath.Dir(path))
if err != nil {
return fmt.Errorf("quarantine: open source directory for %s: %w", path, err)
}
defer func() { _ = source.Close() }()
fd, err := source.OpenFile(filepath.Base(path), os.O_RDONLY, 0)
if err != nil {
return fmt.Errorf("quarantine: open %s: %w", path, fileResponseSourceError(err))
}
defer fd.Close()
opened, err := fd.Stat()
if err != nil {
return fmt.Errorf("quarantine: fstat %s: %w", path, err)
}
if !sameFileIdentity(opened, original) {
return refuseFileResponse(fmt.Errorf("quarantine: source at %s was replaced before stable-content removal; the detected content is kept at %s and the replacement was left in place", path, qPath))
}
versionChanged := !SameFileVersion(opened, original)
if !sameContentShape(opened, original) || versionChanged && (requireVersion || fileLinkCount(opened) <= 1) {
return discardUntrustworthyQuarantineCopy(qPath, fmt.Errorf("quarantine: source changed before stable-content removal %s", path))
}
var leased *os.File
if requireVersion {
if err := acquireQuarantineLease(fd); err != nil {
return refuseFileResponse(fmt.Errorf("quarantine: cannot hold stable content for %s while another descriptor is open: %w", path, err))
}
defer releaseQuarantineLease(fd)
leased = fd
}
return removeQuarantinedSourcePinned(source, filepath.Base(path), path, qPath, original, requireVersion, leased)
}
// removeQuarantinedSourcePinned atomically captures the live directory entry
// into a private same-filesystem directory before checking and removing it.
// Validation and unlink therefore operate on one name that tenant writers
// cannot swap between syscalls. A nil leased file means the caller holds no
// lease to recheck.
func removeQuarantinedSourcePinned(source *safepath.Dir, sourceName, path, qPath string, original os.FileInfo, requireVersion bool, leased *os.File) error {
currentParent, err := safepath.OpenDirNoFollow(filepath.Dir(path))
if err != nil {
return refuseFileResponse(fmt.Errorf("quarantine: source parent changed before atomic capture %s; recovery copy retained at %s: %w", path, qPath, err))
}
sameParent, compareErr := source.SameFile(currentParent)
_ = currentParent.Close()
if compareErr != nil {
return refuseFileResponse(fmt.Errorf("quarantine: source parent changed before atomic capture %s; recovery copy retained at %s: %v", path, qPath, compareErr))
}
if !sameParent {
return refuseFileResponse(fmt.Errorf("quarantine: source parent changed before atomic capture %s; recovery copy retained at %s", path, qPath))
}
beforeCapture, err := source.Lstat(sourceName)
if err != nil {
if os.IsNotExist(err) {
if syncErr := source.Sync(); syncErr != nil {
return fmt.Errorf("quarantine: source vanished but removal is not durable; recovery copy retained at %s: %w", qPath, syncErr)
}
return nil
}
return fmt.Errorf("quarantine: stat source before atomic capture %s; recovery copy retained at %s: %w", path, qPath, fileResponseSourceError(err))
}
if beforeCapture.Mode()&os.ModeSymlink != 0 || !sameFileIdentity(beforeCapture, original) {
return refuseFileResponse(fmt.Errorf("quarantine: source at %s was replaced before atomic capture; the detected content is kept at %s and the replacement was left in place", path, qPath))
}
// A generic quarantine can accept a new hard link because the surviving
// name keeps the inode recoverable. Realtime PHP response requires the exact
// scanned version, including ctime, before any namespace mutation.
versionChanged := !SameFileVersion(beforeCapture, original)
if !sameContentShape(beforeCapture, original) || versionChanged && (requireVersion || fileLinkCount(beforeCapture) <= 1) {
return discardUntrustworthyQuarantineCopy(qPath, fmt.Errorf("quarantine: source changed before atomic capture %s", path))
}
if leaseErr := requireQuarantineLease(leased); leaseErr != nil {
return discardUntrustworthyQuarantineCopy(qPath, fmt.Errorf("quarantine: stable content lease broke before atomic capture %s: %w", path, leaseErr))
}
stage, stageName, err := source.CreatePrivateTemp()
if err != nil {
return fmt.Errorf("quarantine: create private source capture for %s: %w", path, err)
}
keepStage := false
defer func() {
_ = stage.Close()
if !keepStage {
_ = source.RemoveDir(stageName)
}
}()
const capturedName = "captured"
if err := source.RenameTo(sourceName, stage, capturedName); err != nil {
if os.IsNotExist(err) {
if syncErr := source.Sync(); syncErr != nil {
return fmt.Errorf("quarantine: source vanished but removal is not durable; recovery copy retained at %s: %w", qPath, syncErr)
}
return nil
}
return fmt.Errorf("quarantine: capture source before unlink %s; recovery copy retained at %s: %w", path, qPath, fileResponseSourceError(err))
}
captured, statErr := stage.Lstat(capturedName)
identityOK := statErr == nil && captured.Mode()&os.ModeSymlink == 0 && sameFileIdentity(captured, original)
shapeOK := identityOK && sameContentShape(captured, original)
if !shapeOK {
cause := fmt.Errorf("quarantine: source at %s was replaced before atomic capture", path)
if identityOK {
cause = fmt.Errorf("quarantine: source changed before atomic capture %s", path)
}
if err := restoreCapturedSource(stage, capturedName, source, sourceName); err != nil {
recovery, preserveErr := preserveDisplacedSource(stage, capturedName, qPath, path, captured)
if preserveErr != nil {
keepStage = true
return fmt.Errorf("%w; displaced entry retained in private source directory %s after rollback failed: %v", cause, stageName, errors.Join(err, preserveErr))
}
if syncErr := source.Sync(); syncErr != nil {
return fmt.Errorf("%w; displaced entry retained at %s but source directory sync failed: %v", cause, recovery, syncErr)
}
return refuseFileResponse(fmt.Errorf("%w; displaced entry retained at %s because the source name was recreated", cause, recovery))
}
if syncErr := errors.Join(stage.Sync(), source.Sync()); syncErr != nil {
return fmt.Errorf("%w; restored source sync failed: %v", cause, syncErr)
}
if identityOK {
return discardUntrustworthyQuarantineCopy(qPath, cause)
}
return refuseFileResponse(fmt.Errorf("%w; the detected content is kept at %s and the replacement was restored", cause, qPath))
}
if err := quarantineBeforeCapturedUnlink(path); err != nil {
if rollbackErr := restoreCapturedSource(stage, capturedName, source, sourceName); rollbackErr != nil {
keepStage = true
return fmt.Errorf("quarantine: unlink source %s failed; captured source retained in private directory %s and recovery copy retained at %s: %w", path, stageName, qPath, errors.Join(err, rollbackErr))
}
if syncErr := errors.Join(stage.Sync(), source.Sync()); syncErr != nil {
return fmt.Errorf("quarantine: unlink source %s failed and restored source sync failed; recovery copy retained at %s: %w", path, qPath, errors.Join(err, syncErr))
}
return fmt.Errorf("quarantine: unlink source %s; recovery copy retained at %s: %w", path, qPath, err)
}
if err := validateCapturedQuarantineSource(stage, capturedName, captured, leased); err != nil {
cause := fmt.Errorf("quarantine: verify captured source before unlink %s: %w", path, err)
if rollbackErr := restoreCapturedSource(stage, capturedName, source, sourceName); rollbackErr != nil {
keepStage = true
return fmt.Errorf("%w; captured source retained in private directory %s and recovery copy retained at %s: %v", cause, stageName, qPath, rollbackErr)
}
if syncErr := errors.Join(stage.Sync(), source.Sync()); syncErr != nil {
return fmt.Errorf("%w; restored source sync failed and recovery copy retained at %s: %v", cause, qPath, syncErr)
}
return discardUntrustworthyQuarantineCopy(qPath, cause)
}
if err := stage.Remove(capturedName); err != nil {
keepStage = true
return fmt.Errorf("quarantine: unlink captured source %s; private source retained in %s and recovery copy retained at %s: %w", path, stageName, qPath, err)
}
if err := stage.Sync(); err != nil {
return fmt.Errorf("quarantine: source removal is not durable inside private capture; recovery copy retained at %s: %w", qPath, err)
}
if err := quarantineBeforeSourceSync(filepath.Dir(path)); err != nil {
return fmt.Errorf("quarantine: source removal is not durable; recovery copy retained at %s, inspect original path %s before retrying: %w", qPath, path, err)
}
if err := source.Sync(); err != nil {
return fmt.Errorf("quarantine: source removal is not durable; recovery copy retained at %s, inspect original path %s before retrying: %w", qPath, path, err)
}
if _, err := source.Lstat(sourceName); err == nil || !os.IsNotExist(err) {
return &quarantineCompletedWarning{message: fmt.Sprintf("quarantine: removed the captured file at %s, but another entry now occupies that name; recovery copy retained at %s", path, qPath)}
}
return nil
}
func validateCapturedQuarantineSource(stage *safepath.Dir, name string, captured os.FileInfo, leased *os.File) error {
if leased != nil {
return requireQuarantineLease(leased)
}
// Moving a name into a private directory does not revoke existing writers.
// Compare against the post-rename stat because rename itself changes ctime.
current, err := stage.Lstat(name)
if err != nil {
return err
}
if !SameFileVersion(current, captured) {
return errors.New("source changed after atomic capture")
}
return nil
}
func openQuarantineSource(source *safepath.Dir, name string, sourceFD int) (*os.File, error) {
if sourceFD < 0 {
return source.OpenFile(name, os.O_RDONLY, 0)
}
dup, err := unix.FcntlInt(uintptr(sourceFD), unix.F_DUPFD_CLOEXEC, 0)
if err != nil {
return nil, err
}
return os.NewFile(uintptr(dup), name), nil
}
func acquireQuarantineLease(file *os.File) error {
_, err := unix.FcntlInt(file.Fd(), unix.F_SETLEASE, unix.F_WRLCK)
return err
}
func requireQuarantineLease(file *os.File) error {
if file == nil {
return nil
}
lease, err := unix.FcntlInt(file.Fd(), unix.F_GETLEASE, 0)
if err != nil {
return err
}
if lease != unix.F_WRLCK {
return errors.New("exclusive lease is no longer held")
}
return nil
}
func releaseQuarantineLease(file *os.File) {
_, _ = unix.FcntlInt(file.Fd(), unix.F_SETLEASE, unix.F_UNLCK)
}
func restoreCapturedSource(stage *safepath.Dir, capturedName string, source *safepath.Dir, sourceName string) error {
return stage.RenameTo(capturedName, source, sourceName)
}
func preserveDisplacedSource(stage *safepath.Dir, capturedName, qPath, path string, info os.FileInfo) (string, error) {
if info == nil || !info.Mode().IsRegular() {
return "", errors.New("displaced entry is not a regular file")
}
file, err := stage.OpenFile(capturedName, os.O_RDONLY, 0)
if err != nil {
return "", err
}
defer file.Close()
meta := quarantineMetadata(path, info, "Source replacement preserved during quarantine race")
metadata, err := json.MarshalIndent(meta, "", " ")
if err != nil {
return "", err
}
recovery := newQuarantinePath(filepath.Dir(qPath), path+".displaced")
if err := copyQuarantineFileByFD(file, recovery, metadata); err != nil {
return "", err
}
if err := stage.Remove(capturedName); err != nil {
return recovery, err
}
if err := stage.Sync(); err != nil {
return recovery, err
}
return recovery, nil
}
// fileLinkCount returns the inode's link count, or 1 when the stat carries
// no platform data.
func fileLinkCount(info os.FileInfo) uint64 {
if st, ok := info.Sys().(*syscall.Stat_t); ok {
return uint64(st.Nlink) // #nosec G115 -- link count, never negative.
}
return 1
}
package checks
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"github.com/pidginhost/csm/internal/quarantinefs"
)
type quarantineCompletedWarning struct {
message string
}
func (w *quarantineCompletedWarning) Error() string {
return w.message
}
func completedQuarantineWarning(err error) (string, bool) {
var warning *quarantineCompletedWarning
if !errors.As(err, &warning) {
return "", false
}
return warning.Error(), true
}
// sameContentShape verifies that two stats describe a file with the same
// size and modification time. Used as a defence-in-depth check after
// sameFileIdentity passes, because inode reuse on tmpfs / ext4 lets an
// attacker recreate a file under the same path with a fresh ino that
// happens to match the freed slot.
func sameContentShape(a, b os.FileInfo) bool {
if a == nil || b == nil {
return false
}
if a.Size() != b.Size() {
return false
}
return a.ModTime().Equal(b.ModTime())
}
// copyQuarantineFileByFD copies the already-open source into qPath. The copy
// is created by the daemon (root-owned, 0600), so unlike a hard link it is
// never writable through any name the account still holds.
func copyQuarantineFileByFD(src *os.File, qPath string, metadata []byte) error {
if _, err := src.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("seek source: %w", err)
}
return quarantinefs.Store(qPath, src, metadata, 0600)
}
// requireScannedHead checks that the file still starts with the bytes the
// detector scanned. Stat identity cannot see a rewrite within one timestamp
// tick, and the detector reads only a leading window of the file.
func requireScannedHead(fd *os.File, head []byte) error {
if len(head) == 0 {
return errors.New("no scanned content to compare")
}
current := make([]byte, len(head))
if _, err := fd.ReadAt(current, 0); err != nil {
return fmt.Errorf("read scanned window: %w", err)
}
if !bytes.Equal(current, head) {
return errors.New("content differs from the scanned bytes")
}
return nil
}
// Test hooks fail a transaction at the two durability boundaries without
// replacing the descriptor-relative operations used in production.
var quarantineBeforeLease = func(string) error { return nil }
var quarantineBeforeCapturedUnlink = func(string) error { return nil }
var quarantineBeforeSourceSync = func(string) error { return nil }
// removeQuarantinedSource removes the detected name once the content sits in
// quarantine. Production Linux capture binds the final identity check and
// unlink to a private descriptor-relative name. A replacement at the original
// name is left alone and the caller is told what completed.
func removeQuarantinedSource(path, qPath string, original os.FileInfo) error {
return removeQuarantinedSourceMode(path, qPath, original, false)
}
func removeQuarantinedSourceMode(path, qPath string, original os.FileInfo, requireVersion bool) error {
return removeQuarantinedSourcePlatform(path, qPath, original, requireVersion)
}
func discardUntrustworthyQuarantineCopy(qPath string, cause error) error {
if err := os.Remove(qPath); err != nil && !os.IsNotExist(err) {
return errors.Join(cause, err)
}
if err := os.Remove(qPath + ".meta"); err != nil && !os.IsNotExist(err) {
return errors.Join(cause, err)
}
if err := quarantinefs.SyncDir(filepath.Dir(qPath)); err != nil {
return errors.Join(cause, err)
}
return refuseFileResponse(cause)
}
package checks
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/quarantinefs"
"github.com/pidginhost/csm/internal/safepath"
)
// quarantineTarget moves a file or directory into quarantine and records the
// action. It is the single point every quarantine passes through, which is why
// the action record is taken here rather than at each caller.
func quarantineTarget(path, qPath string, info os.FileInfo, metadata QuarantineMeta) (err error) {
return quarantineTargetWith(path, qPath, info, metadata, quarantineTargetFn)
}
func quarantineVersionedTargetFD(path, qPath string, info os.FileInfo, metadata QuarantineMeta, sourceFD int, scannedHead []byte) (err error) {
return quarantineTargetWith(path, qPath, info, metadata, func(path, qPath string, info os.FileInfo, data []byte) error {
return quarantineFileVersionedTOCTOUSafeFD(path, qPath, info, data, sourceFD, scannedHead)
})
}
func quarantineTargetWith(path, qPath string, info os.FileInfo, metadata QuarantineMeta, move func(string, string, os.FileInfo, []byte) error) (err error) {
existed := actionlog.Metadata(qPath).Exists
defer func() { recordQuarantineAction(path, qPath, metadata, info, existed, err) }()
data, err := json.MarshalIndent(metadata, "", " ")
if err != nil {
return fmt.Errorf("encoding quarantine metadata: %w", err)
}
if err = quarantinefs.EnsureDir(filepath.Dir(qPath), 0700); err != nil {
return err
}
err = move(path, qPath, info, data)
_, completed := completedQuarantineWarning(err)
// A refused manual action is not evidence of malware. Successful moves
// retain the stat from before capture; mail spool entries are excluded.
if (err == nil || completed) && info.Mode().IsRegular() && metadata.SpoolDir == "" {
companionSweeps.add(path, info, time.Now())
}
return err
}
// quarantineTargetFn performs the move. It is indirected so the action record
// can be tested against a failing move without a read-only filesystem.
var quarantineTargetFn = func(path, qPath string, info os.FileInfo, data []byte) error {
if info.IsDir() {
return quarantineDirectory(path, qPath, info, data)
}
return quarantineFileTOCTOUSafe(path, qPath, info, data)
}
// recordQuarantineAction writes the unified action record for one quarantine.
// The digest of the removed content is the evidence a reviewer needs: it
// distinguishes "this exact file left the account" from "something was moved".
func recordQuarantineAction(path, qPath string, metadata QuarantineMeta, info os.FileInfo, existed bool, err error) {
rec := actionlog.Record{
Op: "respond.quarantine_file",
Actor: actionlog.DefaultActor(),
Target: path,
Reason: metadata.Reason,
FindingID: metadata.FindingID,
Before: actionlog.FromInfo(info),
After: actionlog.Metadata(path),
Result: actionlog.Applied,
}
// Hash the retained copy only after capture, never a tenant-controlled
// source before the security action. Oversized copies remain unhashed.
if !existed {
retained := actionlog.Stat(qPath)
if retained.Exists {
rec.Before.Digest = retained.Digest
rec.RecoveryPath = qPath
}
}
if err != nil {
rec.Result = actionlog.Failed
rec.Error = err.Error()
if _, completed := completedQuarantineWarning(err); completed {
rec.Result = actionlog.Applied
} else if errors.Is(err, errFileResponseRefused) {
rec.Result = actionlog.Refused
}
}
// Audit hashing can omit a digest; a successful removal still rearms
// alerts. Mail spool entries are not files a tenant can write back.
if rec.Result == actionlog.Applied && info.Mode().IsRegular() && metadata.SpoolDir == "" {
rememberRemovedFile(path, rec.Before.Digest, rec.Before.Size, "quarantine")
}
actionlog.Write(rec)
}
var storeQuarantineBackup = func(path string, content []byte, metadata QuarantineMeta, mode os.FileMode) error {
data, err := json.MarshalIndent(metadata, "", " ")
if err != nil {
return fmt.Errorf("encoding quarantine metadata: %w", err)
}
return quarantinefs.Store(path, bytes.NewReader(content), data, mode)
}
var syncQuarantineTree = quarantinefs.SyncTree
var syncQuarantineDirectory = (*safepath.Dir).Sync
var renameQuarantineDirectory = (*safepath.Dir).RenameTo
func quarantineDirectory(path, qPath string, expected os.FileInfo, metadata []byte) error {
source, err := safepath.OpenDir(filepath.Dir(path))
if err != nil {
return err
}
defer func() { _ = source.Close() }()
quarantine, err := safepath.OpenDir(filepath.Dir(qPath))
if err != nil {
return err
}
defer func() { _ = quarantine.Close() }()
sourceName, name := filepath.Base(path), filepath.Base(qPath)
dir, err := source.OpenFile(sourceName, os.O_RDONLY|unix.O_DIRECTORY, 0)
if err != nil {
return err
}
defer dir.Close()
info, err := dir.Stat()
if err != nil {
return err
}
if !os.SameFile(info, expected) {
return errors.New("quarantine directory changed before capture")
}
if err := syncQuarantineTree(path, info); err != nil {
return fmt.Errorf("syncing quarantine directory: %w", err)
}
if err := dir.Close(); err != nil {
return fmt.Errorf("closing quarantine directory before move: %w", err)
}
if err := quarantinefs.WriteExclusive(qPath+".meta", bytes.NewReader(metadata), 0600); err != nil {
return fmt.Errorf("writing quarantine directory metadata: %w", err)
}
if err := renameQuarantineDirectory(source, sourceName, quarantine, name); err != nil {
return errors.Join(fmt.Errorf("moving quarantine directory: %w", err), os.Remove(qPath+".meta"))
}
got, statErr := quarantine.Stat(name)
if statErr != nil || !os.SameFile(got, info) {
if rollbackErr := quarantine.RenameTo(name, source, sourceName); rollbackErr != nil {
return fmt.Errorf("quarantine directory changed during capture; displaced directory retained at %s: %w", qPath, rollbackErr)
}
if syncErr := errors.Join(source.Sync(), quarantine.Sync()); syncErr != nil {
return fmt.Errorf("quarantine directory changed; rollback sync failed, inspect %s and %s: %w", path, qPath, syncErr)
}
if removeErr := os.Remove(qPath + ".meta"); removeErr != nil {
return fmt.Errorf("quarantine directory changed; original name restored but metadata retained at %s: %w", qPath+".meta", removeErr)
}
return errors.New("quarantine directory changed during capture; original name restored")
}
if err := syncQuarantineDirectory(quarantine); err != nil {
return fmt.Errorf("quarantine directory moved to %s, but destination sync failed; inspect both %s and %s before retrying: %w", qPath, path, qPath, err)
}
if err := syncQuarantineDirectory(source); err != nil {
return fmt.Errorf("quarantine directory retained at %s, but source removal is not durable; inspect %s before retrying: %w", qPath, path, err)
}
return nil
}
package checks
import (
"container/list"
"net"
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
// RDNSCacheConfig is the config block for NewRDNSCache. Resolve is the
// function used to perform the actual reverse lookup; production
// callers wrap net.LookupAddr. ResolveDeadline bounds each lookup;
// 0 disables the deadline. MaxSize caps the number of cached entries
// to keep memory bounded on hosts that see a wide spread of remote
// IPs (BPF SMTP-egress is the motivating case); the oldest entry by
// cachedAt is evicted before a new one is inserted past the cap.
// 0 falls back to rdnsCacheDefaultMaxSize.
type RDNSCacheConfig struct {
TTL time.Duration
Resolve func(ip net.IP) (string, error)
ResolveDeadline time.Duration
MaxSize int
// MaxConcurrent caps deadline-bound lookups, including returned results
// still owned by their callers. A goroutine blocked on a wedged resolver cannot be
// cancelled in Go, so without a cap a burst of distinct IPs under deadline
// saturation spawns one abandonable goroutine per IP. 0 falls back to
// rdnsCacheDefaultMaxConcurrent.
MaxConcurrent int
}
const (
rdnsCacheDefaultMaxSize = 10000
rdnsCacheDefaultMaxConcurrent = 64
)
// RDNSCache is a small TTL cache around reverse DNS lookups. Cached
// negative results (resolver error / NXDOMAIN) are kept until TTL too,
// so the detector does not hammer a slow resolver on a known-bad IP.
// Entries are capped at maxSize; the oldest-by-cachedAt entry is
// evicted on insert once the cap is reached.
type RDNSCache struct {
mu sync.Mutex
ttl time.Duration
deadln time.Duration
maxSize int
resolve func(ip net.IP) (string, error)
now func() time.Time
order *list.List
entries map[string]*list.Element
sem chan struct{}
stats *queuehealth.Tracker
}
type rdnsEntry struct {
key string
host string
cachedAt time.Time
}
// NewRDNSCache returns a ready cache.
func NewRDNSCache(cfg RDNSCacheConfig) *RDNSCache {
maxSize := cfg.MaxSize
if maxSize <= 0 {
maxSize = rdnsCacheDefaultMaxSize
}
maxConcurrent := cfg.MaxConcurrent
if maxConcurrent <= 0 {
maxConcurrent = rdnsCacheDefaultMaxConcurrent
}
c := &RDNSCache{
ttl: cfg.TTL,
deadln: cfg.ResolveDeadline,
maxSize: maxSize,
resolve: cfg.Resolve,
now: time.Now,
order: list.New(),
entries: map[string]*list.Element{},
sem: make(chan struct{}, maxConcurrent),
}
if cfg.ResolveDeadline > 0 {
c.stats = queuehealth.NewSharedCapacity(maxConcurrent, cfg.ResolveDeadline)
}
return c
}
// evictOldestLocked drops the oldest cached entry. Caller holds c.mu.
func (c *RDNSCache) evictOldestLocked() {
el := c.order.Front()
if el == nil {
return
}
entry := el.Value.(*rdnsEntry)
delete(c.entries, entry.key)
c.order.Remove(el)
}
// Lookup returns the cached hostname for ip, or "" on miss/error/deadline.
// Lookup blocks the caller for at most cfg.ResolveDeadline; cache hits
// return immediately.
func (c *RDNSCache) Lookup(ip net.IP) string {
if ip == nil {
return ""
}
key := ip.String()
c.mu.Lock()
now := c.now()
if el, ok := c.entries[key]; ok {
e := el.Value.(*rdnsEntry)
if now.Sub(e.cachedAt) <= c.ttl {
c.mu.Unlock()
return e.host
}
}
c.mu.Unlock()
host := c.runWithDeadline(ip)
c.mu.Lock()
now = c.now()
if el, present := c.entries[key]; present {
e := el.Value.(*rdnsEntry)
e.host = host
e.cachedAt = now
c.order.MoveToBack(el)
c.mu.Unlock()
return host
}
if c.order.Len() >= c.maxSize {
c.evictOldestLocked()
}
c.entries[key] = c.order.PushBack(&rdnsEntry{key: key, host: host, cachedAt: now})
c.mu.Unlock()
return host
}
func (c *RDNSCache) runWithDeadline(ip net.IP) string {
if c.deadln <= 0 {
host, err := c.resolve(ip)
if err != nil {
return ""
}
return host
}
// Cap in-flight resolve goroutines. A goroutine blocked on a wedged
// resolver keeps its slot until the syscall finally returns, so under
// deadline saturation further lookups fail fast (return "" like a
// deadline miss) instead of spawning more abandonable goroutines.
work := c.acquireResolve()
if work == nil {
return ""
}
defer work.release()
ch := make(chan rdnsResult, 1)
go c.resolveTracked(work, ip, ch)
timer := time.NewTimer(c.deadln)
defer timer.Stop()
select {
case r := <-ch:
if r.err != nil {
return ""
}
return r.host
case <-timer.C:
work.fail()
return ""
}
}
package checks
import (
"errors"
"net"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
type rdnsWork struct {
cache *RDNSCache
ticket queuehealth.Ticket
remaining atomic.Int32
failOnce sync.Once
}
type rdnsResult struct {
host string
err error
}
func (c *RDNSCache) acquireResolve() *rdnsWork {
ticket := c.stats.Begin(time.Now())
select {
case c.sem <- struct{}{}:
work := &rdnsWork{cache: c, ticket: ticket}
work.remaining.Store(2)
return work
default:
ticket.Reject(time.Now())
return nil
}
}
func (w *rdnsWork) fail() {
w.failOnce.Do(func() { w.cache.stats.Lose(time.Now(), 1) })
}
func (w *rdnsWork) release() {
// The caller can leave while DNS still runs, or DNS can return before
// its caller receives the buffered result. Both own the admitted slot.
if w.remaining.Add(-1) == 0 {
w.ticket.Finish(time.Now())
<-w.cache.sem
}
}
func (c *RDNSCache) resolveTracked(work *rdnsWork, ip net.IP, done chan<- rdnsResult) {
work.ticket.Start(time.Now())
completed := false
defer func() {
if !completed {
work.fail()
}
work.release()
}()
host, err := c.resolve(ip)
var dnsErr *net.DNSError
if err != nil && (!errors.As(err, &dnsErr) || !dnsErr.IsNotFound) {
work.fail()
}
done <- rdnsResult{host: host, err: err}
completed = true
}
// QueueStatuses reads memory only. Without a deadline, Lookup is synchronous
// and does not use this bounded resolver pool.
func (c *RDNSCache) QueueStatuses(now time.Time) map[string]queuehealth.Status {
if c.stats == nil {
return nil
}
return map[string]queuehealth.Status{"resolves": c.stats.Snapshot(now)}
}
package checks
import (
"sort"
"github.com/pidginhost/csm/internal/admission"
)
// CheckInfo describes a single named check emitted as an alert.Finding.Check.
// Category groups related checks for display in the settings UI. Internal is
// true for checks that exist for plumbing (self-tests, plumbing findings) and
// should not appear in user-facing dropdowns like alerts.email.disabled_checks.
type CheckInfo struct {
Name string
Category string
Internal bool
// Correlation says how cross-account correlation treats this check.
// Every entry must set it; the zero value fails TestEveryCheckIsClassified.
Correlation CorrelationClass
// CorrelationReason names the policy that excludes an Ignored check. One
// of the reason constants in correlation_policy.go.
CorrelationReason string
// CorrelationGap documents a known missing producer identity path for an
// eligible check. It never changes eligibility.
CorrelationGap string
// Response is the check's automatic IP response policy. The zero value
// neither blocks nor challenges; see response_policy.go.
Response ResponsePolicy
// Summary marks a check whose findings report an applied automatic
// response or an observation that never acts, so they go to the daily
// summary instead of an immediate alert. See IsSummaryNotice.
Summary bool
}
// Category labels are the groupings shown in the multi-select UI. Keep the
// order below in sync with checkCategoryOrder so categories render in a sane
// order rather than alphabetically (Auth first, Internal last).
const (
CategoryAuth = "Authentication & Login"
CategoryBruteForce = "Brute Force"
CategoryMalware = "Malware & Webshells"
CategoryWeb = "Web & Application"
CategoryDatabase = "Database Content"
CategoryEmail = "Email & Phishing"
CategoryPerformance = "Performance"
CategoryNetwork = "Network & Firewall"
CategorySystem = "System Integrity"
CategoryWAF = "WAF & ModSecurity"
CategoryCorrelation = "Correlation & Health"
CategoryInternal = "Internal"
)
var checkCategoryOrder = []string{
CategoryAuth,
CategoryBruteForce,
CategoryMalware,
CategoryWeb,
CategoryDatabase,
CategoryEmail,
CategoryPerformance,
CategoryNetwork,
CategorySystem,
CategoryWAF,
CategoryCorrelation,
CategoryInternal,
}
// checkRegistry is the authoritative list of every Check string the daemon
// may emit. Adding a new alert.Finding Check name anywhere in internal/checks,
// internal/daemon, or internal/webui without also adding it here will fail
// TestCheckRegistryCoversProductionCode.
// The attribution kinds an evidence row accepts.
var (
attributedByPeer = admission.Attributions(admission.AttributionPeer)
attributedByHTTP = admission.Attributions(admission.AttributionHTTPPeer, admission.AttributionTrustedHTTPChain)
attributedByDestination = admission.Attributions(admission.AttributionSocketDestination)
attributedByDerived = admission.Attributions(admission.AttributionDerived)
)
var checkRegistry = []CheckInfo{
// --- Authentication & Login ------------------------------------------
// The admin-panel detector's tight path set makes false positives unlikely;
// a challenge would only delay containment of repeated attacks.
{Name: "admin_panel_bruteforce", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, NeverChallenge: true, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
{Name: "api_auth_failure", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockWithCpanelLogins, Evidence: admission.FamilyPanel, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
// API clients have no browser to answer the challenge.
{Name: "api_auth_failure_realtime", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockWithCpanelLogins, NeverChallenge: true, Evidence: admission.FamilyPanel, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "api_tokens", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "bulk_password_change", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonAccountAggregate},
{Name: "cpanel_file_upload", Category: CategoryAuth, Correlation: CorrelationSecurityEvent},
{Name: "cpanel_file_upload_realtime", Category: CategoryAuth, Correlation: CorrelationSecurityEvent},
{Name: "cpanel_login", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "cpanel_login_realtime", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "cpanel_multi_ip_login", Category: CategoryAuth, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{Block: BlockWithCpanelLogins, Evidence: admission.FamilyPanel, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "cpanel_password_purge", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "cpanel_password_purge_realtime", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
// FTP authentication cannot be gated by an HTTP challenge.
{Name: "ftp_auth_failure_realtime", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockWithCpanelLogins, NeverChallenge: true, Evidence: admission.FamilyFTP, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "ftp_bruteforce", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, Evidence: admission.FamilyFTP, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "ftp_login", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "ftp_login_after_bruteforce", Category: CategoryAuth, Correlation: CorrelationSecurityEvent},
// PAM breadth and brute-force signals come from non-browser clients.
{Name: "credential_stuffing", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, NeverChallenge: true, Evidence: admission.FamilySSH, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "pam_bruteforce", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, NeverChallenge: true, Evidence: admission.FamilySSH, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "pam_login", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "pam_auth_failures", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "password_hijack_confirmed", Category: CategoryAuth, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "root_password_change", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "shadow_change", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "ssh_keys", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "ssh_login_unknown_ip", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational, Response: ResponsePolicy{Block: BlockAlways, Evidence: admission.FamilySSH, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "sshd_config_change", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "uid0_account", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope, Response: ResponsePolicy{NeverChallenge: true}},
// Pre-auth attacks on a public webmail login page can meet the gate on
// their next request, unlike successful post-auth webmail audit events.
{Name: "webmail_bruteforce", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockWithCpanelLogins, ChallengeFirst: true, Evidence: admission.FamilyPanel, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "webmail_login_realtime", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "whm_account_action", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "whm_login_realtime", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "whm_password_change", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "whm_password_change_noninfra", Category: CategoryAuth, Correlation: CorrelationSecurityEvent},
{Name: "whm_unauth_scripts_realtime", Category: CategoryAuth, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
// --- Brute Force -----------------------------------------------------
{Name: "http_request_flood", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
// URL enumeration is browser-visible; http_scanner_action can still opt
// this check out of the challenge in favor of a direct block.
{Name: "http_scanner_profile", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, ChallengeFirst: true, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
// A claimed crawler may still pass reverse-DNS verification next cycle.
// Challenge it without timeout escalation while that verdict is pending;
// with challenge routing disabled, the existing block policy still applies.
{Name: "http_claimed_bot_unverified", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, ChallengeFirst: true, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
{Name: "http_ua_spoof", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
{Name: "http_distributed_flood", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
{Name: "http_asn_crawl", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP, Subnet: admission.EntryASNCrawl}},
// Mail protocols cannot answer an HTTP challenge. Compromise severity
// decides blocking.
{Name: "mail_account_compromised", Category: CategoryBruteForce, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{Block: BlockAlways, CriticalOnly: true, NeverChallenge: true, Evidence: admission.FamilyMail, Basis: admission.BasisCompromise, Attribution: attributedByPeer}},
{Name: "mail_account_spray", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
{Name: "mail_bruteforce", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, NeverChallenge: true, Evidence: admission.FamilyMail, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "mail_bruteforce_suspected", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Summary: true},
// A subnet summary does not authorize a single-IP block; the subnet-spray
// path blocks the subnet itself.
{Name: "mail_subnet_spray", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{NeverChallenge: true, Evidence: admission.FamilyMail, Basis: admission.BasisLocal, Attribution: attributedByPeer, Subnet: admission.EntryMailSubnet}},
{Name: "smtp_account_spray", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
// SMTP authentication, connection probes and subnet sprays have no browser
// at the other end, so a challenge cannot stop their traffic.
{Name: "smtp_bruteforce", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, NeverChallenge: true, Evidence: admission.FamilyMail, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "smtp_probe_abuse", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, NeverChallenge: true, Evidence: admission.FamilyMail, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "smtp_subnet_spray", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{NeverChallenge: true, Evidence: admission.FamilyMail, Basis: admission.BasisLocal, Attribution: attributedByPeer, Subnet: admission.EntryMailSubnet}},
// Stalwart's verdict on a peer it served on a mail listener of this host.
// A ban of a known customer's address is High and is not blocked.
{Name: "stalwart_ban", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, CriticalOnly: true, NeverChallenge: true, Evidence: admission.FamilyMail, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
// Stalwart's HTTP listener can log a forwarded address: alert only.
{Name: "stalwart_webmail_bruteforce", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
// These WordPress probes reach public HTTP endpoints before authentication;
// the next request from the same source can be sent to the browser gate.
{Name: "wp_login_bruteforce", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, ChallengeFirst: true, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
{Name: "wp_user_enumeration", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{ChallengeFirst: true, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
{Name: "xmlrpc_abuse", Category: CategoryBruteForce, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, ChallengeFirst: true, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
// --- Malware & Webshells --------------------------------------------
{Name: "backdoor_binary", Category: CategoryMalware, Correlation: CorrelationMalwareArtifact, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "cgi_backdoor_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "cgi_suspicious_location_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "cross_account_malware", Category: CategoryMalware, Correlation: CorrelationDerived, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "docroot_binary_exec", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "executable_in_config_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "executable_in_tmp_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "fake_kernel_thread", Category: CategoryMalware, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "group_writable_php", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "js_keylogger_dataflow", Category: CategoryMalware, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "js_taint_scan_incomplete", Category: CategoryMalware, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "php_remote_taint", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "php_taint_scan_incomplete", Category: CategoryMalware, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
// Reports files planted with a detection. The detection itself counts
// toward correlation; this report is never Critical, so it adds nothing.
{Name: "malware_companion_files", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "malware_companion_scan_incomplete", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "placed_dir_incomplete_realtime", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "placement_unresolved_realtime", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "new_executable_in_config", Category: CategoryMalware, Correlation: CorrelationMalwareArtifact},
{Name: "new_php_in_sensitive_dir", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "new_php_in_sensitive_dir_clean", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "new_php_in_uploads", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "new_php_in_uploads_clean", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
// Retired: emitted by the file index until a20c6f76 and never registered
// afterwards. Kept registered so the file_index runner can purge findings
// written by older versions; nothing emits them today.
{Name: "new_php_in_languages", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "new_php_in_upgrade", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "new_suspicious_php", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "new_webshell_file", Category: CategoryMalware, Correlation: CorrelationMalwareArtifact},
{Name: "nulled_plugin", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "obfuscated_php", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "obfuscated_php_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "php_dropper_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "php_in_image_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "php_in_sensitive_dir_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "php_in_uploads_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "removed_file_restored_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "self_deleting_dropper_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "self_deleting_dropper_overflow", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "php_shield_block", Category: CategoryMalware, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "php_shield_eval", Category: CategoryMalware, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "php_shield_webshell", Category: CategoryMalware, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "php_suspicious_execution", Category: CategoryMalware, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "signature_match_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "suid_binary", Category: CategoryMalware, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "suspicious_file", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "suspicious_php_content", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "suspicious_process", Category: CategoryMalware, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: DetachedWebProcessCheck, Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "webshell", Category: CategoryMalware, Correlation: CorrelationMalwareArtifact, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "webshell_caller", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
{Name: "webshell_content_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "webshell_path_public", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "webshell_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent},
{Name: "world_writable_php", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "yara_match_realtime", Category: CategoryMalware, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "yara_match_scheduled", Category: CategoryMalware, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "yara_scan_incomplete", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "yara_realtime_scan_error", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "yara_backend_unavailable", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "yara_worker_crashed", Category: CategoryMalware, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
// --- Web & Application ----------------------------------------------
{Name: "htaccess_auto_prepend", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "htaccess_cgi_handler_abuse", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "htaccess_errordocument_hijack", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "htaccess_filesmatch_shield", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "htaccess_handler_abuse", Category: CategoryWeb, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "htaccess_header_injection", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "htaccess_injection", Category: CategoryWeb, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "htaccess_injection_realtime", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "htaccess_php_in_uploads", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "htaccess_rewrite_off_shield", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "htaccess_security_disabled", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "web_exposed_backup_archive", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "web_exposed_config_leak", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "web_exposed_repo_metadata", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "web_exposed_db_dump", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "web_exposed_phpinfo", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "web_exposed_sample_sql", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "web_exposed_source_backup", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "htaccess_spam_redirect", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "htaccess_user_agent_cloak", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "open_basedir", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "outdated_plugins", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "vulnerable_plugins", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "vulnerable_timthumb", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "php_config_change", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "php_config_scan_incomplete", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "php_config_realtime", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "symlink_attack", Category: CategoryWeb, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "wp_core_integrity", Category: CategoryWeb, Correlation: CorrelationSecurityEvent},
{Name: "wp_core_unverified", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "wp_plugin_inventory_unverified", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
// --- Database Content -----------------------------------------------
{Name: "database_dump", Category: CategoryDatabase, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "db_malicious_event", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_malicious_function", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_malicious_procedure", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "admin_cross_account_overlap", Category: CategoryDatabase, Correlation: CorrelationIgnored, CorrelationReason: reasonAccountAggregate},
{Name: "credential_reuse", Category: CategoryDatabase, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "supply_chain_vuln", Category: CategoryWeb, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "db_magic_token_user", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_malicious_trigger", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_options_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "db_options_new_external_script", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_options_plugin_notice_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_post_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "db_content_scan_incomplete", Category: CategoryDatabase, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "db_unexpected_event", Category: CategoryDatabase, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "db_unexpected_function", Category: CategoryDatabase, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "db_unexpected_procedure", Category: CategoryDatabase, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "db_unexpected_trigger", Category: CategoryDatabase, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "drupal_admin_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "drupal_content_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "drupal_settings_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "joomla_admin_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "joomla_content_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "joomla_extensions_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "magento_admin_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "magento_content_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "magento_settings_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "opencart_admin_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "opencart_content_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "opencart_settings_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "prestashop_admin_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "prestashop_content_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "prestashop_settings_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_phantom_post_author", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_post_volume_burst", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_hidden_link_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_hostname_keyed_option", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_doorway_sitemap_routes", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_spam_taxonomy", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_stored_code_execution", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_stored_cloak_logic", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_rogue_admin", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "db_siteurl_hijack", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "db_siteurl_foreign_host", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
{Name: "db_siteurl_invalid", Category: CategoryDatabase, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "db_spam_injection", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "db_suspicious_admin_email", Category: CategoryDatabase, Correlation: CorrelationSecurityEvent},
// --- Email & Phishing -----------------------------------------------
{Name: "credential_log_realtime", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "antispam_apply_failed", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "antispam_feed", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "antispam_ip_map", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "antispam_controller_open", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "antispam_file_replaced", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "antispam_learning", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "antispam_precondition", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "antispam_rspamd_down", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "antispam_rspamd_recovered", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "antispam_scan_error", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "stalwart_address_list", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "stalwart_spam_filter_off", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_auth_failure_realtime", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
{Name: "email_cloud_relay_abuse", Category: CategoryEmail, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{Block: BlockAlways, Evidence: admission.FamilyMail, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "email_av_degraded", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_encrypted_archive", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_scanner_panic", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_source_backlog", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "realtime_scanner_panic", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_hold_bypass", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_isolation_unavailable", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_late_verdict", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_parse_error", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_parse_worker_failed", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_parse_worker_rejected", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_parse_worker_source", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
{Name: "email_av_queue_overflow", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_quarantine_error", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_scan_error", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_av_timeout", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_compromised_account", Category: CategoryEmail, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{Block: BlockAlways, Evidence: admission.FamilyMail, Basis: admission.BasisLocal, Attribution: attributedByPeer}},
{Name: "email_credential_leak", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_dkim_failure", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "email_filter_blackhole", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_filter_exfil", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_filter_forwarder", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_filter_pipe", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_mail_filters", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_malware", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
{Name: "email_phishing_content", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
// A client that submitted web forms; a browser can answer the challenge.
{Name: "email_php_form_relay", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, ChallengeFirst: true, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
// Observe mode reports the same client and never aims a response.
{Name: "email_php_form_relay_observed", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Summary: true},
// Names the abused script, not a client. Its message carries the
// client-controlled request path, so it must never aim a response.
{Name: "email_php_form_relay_script", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_php_form_relay_table_full", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_abuse", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_php_relay_account_volume_capped", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_action_dry_run", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonResponse, Summary: true},
{Name: "email_php_relay_action_failed", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonResponse},
{Name: "email_mailbox_response", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonResponse},
{Name: "email_php_relay_action_skipped", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonResponse},
{Name: "email_php_relay_cpanel_limit_unreadable", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_disabled", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_inotify_overflow", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_inotify_overflow_recovered", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_msgindex_persist_failed", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_no_exim", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_overflow_scan_truncated", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_path2b_disabled", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_policies_reload", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_rate_limit_hit", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonResponse, Summary: true},
{Name: "email_php_relay_sweep_failed", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_php_relay_watcher_failed", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "email_defer_fail_governor", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "email_pipe_forwarder", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_rate_critical", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_rate_warning", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_spam_outbreak", Category: CategoryEmail, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "email_spf_rejection", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "email_suspicious_forwarder", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_suspicious_geo", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "email_weak_password", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "email_password_audit_incomplete", Category: CategoryEmail, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "exim_frozen_realtime", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "mail_per_account", Category: CategoryEmail, Correlation: CorrelationSecurityEvent, CorrelationGap: gapEnvelopeSender},
{Name: "mail_queue", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "mail_queue_unavailable", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "mail_spam_source", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
{Name: "phishing_credential_log", Category: CategoryEmail, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "phishing_directory", Category: CategoryEmail, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "phishing_iframe", Category: CategoryEmail, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "phishing_kit_archive", Category: CategoryEmail, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "phishing_kit_archive_unscannable", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "phishing_kit_realtime", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "phishing_page", Category: CategoryEmail, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "phishing_php", Category: CategoryEmail, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "phishing_realtime", Category: CategoryEmail, Correlation: CorrelationSecurityEvent},
{Name: "phishing_redirector", Category: CategoryEmail, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "shared_senders_stale", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "spam_evidence_evicted", Category: CategoryEmail, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
// --- Performance -----------------------------------------------------
{Name: "perf_error_logs", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
{Name: "perf_load", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
{Name: "perf_memory", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
{Name: "perf_mysql_config", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
{Name: "perf_php_handler", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
{Name: "perf_php_processes", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
{Name: "perf_redis_config", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
{Name: "perf_wp_config", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
{Name: "perf_wp_cron", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
{Name: "perf_wp_loopback", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
{Name: "perf_wp_myisam", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
{Name: "perf_wp_transients", Category: CategoryPerformance, Correlation: CorrelationIgnored, CorrelationReason: reasonPerformance},
// --- Network & Firewall ---------------------------------------------
{Name: "backdoor_port", Category: CategoryNetwork, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "backdoor_port_outbound", Category: CategoryNetwork, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "c2_connection", Category: CategoryNetwork, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{Block: BlockAlways, NeverChallenge: true, Evidence: admission.FamilyNetwork, Basis: admission.BasisCompromise, Attribution: attributedByDestination}},
{Name: "direct_smtp_egress", Category: CategoryNetwork, Correlation: CorrelationSecurityEvent},
{Name: "dns_connection", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "dns_zone_change", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "exfiltration_paste_site", Category: CategoryNetwork, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "firewall", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "firewall_ports", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "firewall_ipv6_unmanaged", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "firewall_state_unreadable", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "bad_asn_outbound", Category: CategoryNetwork, Correlation: CorrelationSecurityEvent},
{Name: "infra_ips_unresolvable", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
// Reputation on the HTTP path gives a browser one verifier before blocking.
// Critical sightings come from browserless vectors and bypass the gate.
{Name: "ip_reputation", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, ChallengeFirst: true, Evidence: admission.FamilyReputation, Basis: admission.BasisIntel, Attribution: attributedByDerived}},
{Name: "reputation_quota_exhausted", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "threat_feed_stale", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "ssl_cert_issued", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "user_outbound_connection", Category: CategoryNetwork, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
// --- System Integrity ------------------------------------------------
{Name: "af_alg_enforcement_corrected", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "af_alg_socket_use", Category: CategorySystem, Correlation: CorrelationSecurityEvent},
{Name: "account_scan_error", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "account_scan_truncated", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "bpf_unavailable", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "bpf_ringbuf_error", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "check_panic", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "full_scan_file_too_large", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "crond_change", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "crontab_change", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
{Name: "dpkg_integrity", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "kernel_module", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "mysql_superuser", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "rpm_integrity", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "sensitive_file_modified", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
{Name: "signature_update_rollback", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "signature_update_rescan_queued", Category: CategorySystem, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "suspicious_crontab", Category: CategorySystem, Correlation: CorrelationSecurityEvent, Response: ResponsePolicy{NeverChallenge: true}},
// --- WAF & ModSecurity ----------------------------------------------
{Name: "modsec_block_escalation", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
{Name: "modsec_block_realtime", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
{Name: "modsec_classifier_gap", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "modsec_csm_block_escalation", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, NeverChallenge: true, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
{Name: "modsec_low_confidence_burst", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
{Name: "modsec_warning_realtime", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
// The WAF already denied repeated attacks; keep containment direct.
{Name: "waf_attack_blocked", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide, Response: ResponsePolicy{Block: BlockAlways, NeverChallenge: true, Evidence: admission.FamilyHTTP, Basis: admission.BasisLocal, Attribution: attributedByHTTP}},
{Name: "modsec_disabled_vhost", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
// Retired in favour of modsec_disabled_vhost. Kept registered so the
// waf_status runner can still purge findings written by older versions.
{Name: "waf_bypass", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "waf_detection_only", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "waf_rules", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "waf_rules_stale", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
{Name: "waf_status", Category: CategoryWAF, Correlation: CorrelationIgnored, CorrelationReason: reasonPosture},
// --- Correlation & Health -------------------------------------------
{Name: "account_scan", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "auto_block", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonResponse, Summary: true},
{Name: "auto_response", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonResponse},
{Name: "auto_response_paused", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonResponse},
{Name: "challenge_route", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonResponse, Summary: true},
{Name: "check_timeout", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "config_reload_error", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "config_reload_restart_required", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "yara_forge_rollback", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "coordinated_attack", Category: CategoryCorrelation, Correlation: CorrelationDerived, Response: ResponsePolicy{NeverChallenge: true}},
{Name: "csm_health", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "fanotify_kernel_overflow", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "fanotify_overflow", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "integrity", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonHostScope},
// Retired: no build emits it and it asks for no response; it stays so
// a completed scan clears the findings older versions stored.
{Name: "local_threat_score", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonAttackerSide},
{Name: "mail_auth_backend_degraded", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "mail_log_source_unavailable", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "stalwart_log_degraded", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "protection_queue_degraded", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
// Admission notices report the response engine's own gaps; the
// finding they concern keeps its own correlation.
{Name: "auto_response_withheld", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "response_capacity_exhausted", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "protection_queue_recovered", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
{Name: "trusted_proxy_unusable", Category: CategoryCorrelation, Correlation: CorrelationIgnored, CorrelationReason: reasonSelfHealth},
// --- Internal (not shown in user-facing dropdowns) -------------------
{Name: "test_alert", Category: CategoryInternal, Internal: true, Correlation: CorrelationIgnored, CorrelationReason: reasonInformational},
}
// AllCheckNames returns every registered Check name, sorted alphabetically.
// Includes internal names; callers that render user-facing UI should use
// PublicCheckInfos instead.
func AllCheckNames() []string {
out := make([]string, 0, len(checkRegistry))
for _, c := range checkRegistry {
out = append(out, c.Name)
}
sort.Strings(out)
return out
}
// PublicCheckInfos returns all non-Internal checks grouped by category in
// the canonical category order (see checkCategoryOrder). Within a category
// names are sorted alphabetically. This is the list the settings UI shows
// for alerts.email.disabled_checks.
func PublicCheckInfos() []CheckInfo {
byCategory := make(map[string][]CheckInfo, len(checkCategoryOrder))
for _, c := range checkRegistry {
if c.Internal {
continue
}
byCategory[c.Category] = append(byCategory[c.Category], c)
}
var out []CheckInfo
for _, cat := range checkCategoryOrder {
items := byCategory[cat]
sort.Slice(items, func(i, j int) bool { return items[i].Name < items[j].Name })
out = append(out, items...)
}
return out
}
// LookupCheck returns the registry entry for name, if any.
func LookupCheck(name string) (CheckInfo, bool) {
for _, c := range checkRegistry {
if c.Name == name {
return c, true
}
}
return CheckInfo{}, false
}
package checks
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"syscall"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/safepath"
)
// quarantineMoveChecks are the findings whose manual fix, and whose
// full-scan quarantine, is a plain move of the named file into quarantine.
// The automatic responder's broader set lives in autoQuarantineChecks; the
// full-scan set must stay equal to this one (pinned by test).
var quarantineMoveChecks = map[string]bool{
"webshell": true,
"new_webshell_file": true,
"obfuscated_php": true,
"suspicious_php_content": true,
"new_php_in_languages": true,
"new_php_in_upgrade": true,
"phishing_page": true,
"phishing_directory": true,
}
// eximMsgIDRegex validates Exim message ID format. Exim 4.96 and older use
// 6-6-2 ids; Exim 4.97 and newer use 6-11-4 ids.
var eximMsgIDRegex = regexp.MustCompile(`^[0-9A-Za-z]{6}-(?:[0-9A-Za-z]{6}-[0-9A-Za-z]{2}|[0-9A-Za-z]{11}-[0-9A-Za-z]{4})$`)
// Allowed roots for each fix action. Declared as vars (not consts) so tests
// can redirect remediation under t.TempDir() without writing to real /home,
// /tmp, or /var/spool. Production must not mutate these at runtime.
// A nil list means "the platform's account roots" (plus, for quarantine,
// the temp trees in quarantineExtraRoots); see effectiveFixRoots.
var (
fixPermissionsAllowedRoots []string
fixQuarantineAllowedRoots []string
fixHtaccessAllowedRoots []string
eximSpoolDirs = []string{"/var/spool/exim/input", "/var/spool/exim4/input"}
)
// eximSpoolMessageDirs lists where Exim may keep a message's -H and -D
// files: with split_spool_directory (the cPanel default) the input/
// subdirectory named after the sixth character of the message ID, otherwise
// input/ itself.
func eximSpoolMessageDirs(msgID string) []string {
var dirs []string
for _, root := range eximSpoolDirs {
if len(msgID) > 5 {
dirs = append(dirs, filepath.Join(root, msgID[5:6]))
}
dirs = append(dirs, root)
}
return dirs
}
// chmodFunc performs the permission change for fixPermissions on the pinned
// descriptor. It is a var so tests can simulate failures (e.g. a read-only
// mount returning EROFS) without an actual read-only filesystem, and assert an
// already-compliant file is never chmodded.
var chmodFunc = (*os.File).Chmod
// RemediationResult describes the outcome of a fix action.
type RemediationResult struct {
Success bool `json:"success"`
Action string `json:"action"` // human-readable description of what was done
Description string `json:"description"` // what fix was applied
Error string `json:"error,omitempty"`
// Refused distinguishes an unchanged, ineligible target from an I/O failure.
Refused bool `json:"-"`
// RemediationStatus lets a caller that supports more than one successful
// disposition distinguish an in-place clean from whole-file quarantine.
// It is transport metadata, not part of the generic remediation API.
RemediationStatus string `json:"-"`
// Reverted marks a virtual patch that had to be written again because
// something removed or damaged CSM's earlier block -- typically a backup
// plugin rewriting the .htaccess it owns.
Reverted bool `json:"reverted,omitempty"`
}
// FixDescription returns a human-readable description of what the fix will do
// for a given check type and file path. Returns empty string if no fix is available.
func FixDescription(checkType, message string, filePath ...string) string {
path := selectFindingPath(message, filePath...)
if isHtaccessHardenedFinding(checkType) {
if path != "" {
return fmt.Sprintf("Remove malicious directives from %s", path)
}
return ""
}
if quarantineMoveChecks[checkType] {
if path != "" {
return fmt.Sprintf("Quarantine %s to /opt/csm/quarantine/", path)
}
return ""
}
switch checkType {
case "world_writable_php", "group_writable_php":
if path != "" {
return fmt.Sprintf("Set permissions to 644 on %s", path)
}
case "backdoor_binary", "new_executable_in_config":
if path != "" {
return fmt.Sprintf("Kill process and quarantine %s", path)
}
case "suspicious_crontab":
if path != "" {
return fmt.Sprintf("Quarantine and truncate crontab %s", path)
}
return "Quarantine and truncate crontab"
case "htaccess_injection", "htaccess_injection_realtime", "htaccess_handler_abuse":
if path != "" {
return fmt.Sprintf("Remove malicious directives from %s", path)
}
case "email_phishing_content":
msgID := extractEximMsgID(message)
if msgID != "" {
return fmt.Sprintf("Quarantine Exim spool message %s", msgID)
}
}
return ""
}
// HasFix returns true if the check type has a known automated fix.
func HasFix(checkType string) bool {
if isHtaccessHardenedFinding(checkType) || quarantineMoveChecks[checkType] {
return true
}
fixableChecks := map[string]bool{
"world_writable_php": true,
"group_writable_php": true,
"backdoor_binary": true,
"new_executable_in_config": true,
"htaccess_injection": true,
"htaccess_injection_realtime": true,
"htaccess_handler_abuse": true,
"email_phishing_content": true,
"suspicious_crontab": true,
}
return fixableChecks[checkType]
}
// ApplyFix executes the remediation action for a finding.
func ApplyFix(ctx context.Context, checkType, message, details string, filePath ...string) RemediationResult {
if err := ctx.Err(); err != nil {
return RemediationResult{Error: err.Error()}
}
path := selectFindingPath(message, filePath...)
if isHtaccessHardenedFinding(checkType) {
// CleanHtaccessFile re-runs the full detector registry, so a single
// action removes every malicious directive the audit found.
return CleanHtaccessFile(path)
}
if quarantineMoveChecks[checkType] {
return fixQuarantine(path)
}
switch checkType {
case "world_writable_php", "group_writable_php":
return fixPermissions(path, checkType)
case "backdoor_binary", "new_executable_in_config":
return fixKillAndQuarantine(ctx, path, details)
case "htaccess_injection", "htaccess_injection_realtime":
return fixHtaccess(path)
case "email_phishing_content":
return fixQuarantineSpoolMessage(message)
case "suspicious_crontab":
return fixSuspiciousCrontab(path)
default:
return RemediationResult{Error: fmt.Sprintf("no automated fix available for check type '%s'", checkType)}
}
}
// fixPermissions sets file permissions to 0644. checkType selects which write
// bit is the dangerous one: world-writable (0002) for world_writable_php,
// group-writable (0020) for group_writable_php.
//
// If the file no longer carries that bit -- because an operator already fixed
// it by hand, or it changed since the scan -- the finding is treated as
// already resolved and no chmod is attempted. This is the path an operator
// hits when they manually correct perms and then click "Apply automated fix":
// rather than erroring, the finding clears.
func fixPermissions(path, checkType string) RemediationResult {
result, _ := applyPermissionFix(path, checkType)
return result
}
// applyPermissionFix is fixPermissions that also returns the mode it
// replaced, or 0 when it changed nothing.
func applyPermissionFix(path, checkType string) (RemediationResult, os.FileMode) {
if path == "" {
return RemediationResult{Error: "could not extract file path from finding"}, 0
}
path, info, err := resolveExistingFixPath(path, effectiveFixRoots(fixPermissionsAllowedRoots))
if err != nil {
return RemediationResult{Error: err.Error()}, 0
}
target, err := openFixTarget(path, info)
if err != nil {
return RemediationResult{
Refused: errors.Is(fileResponseSourceError(err), errFileResponseRefused),
Error: fmt.Sprintf("cannot open %s: %v", path, err),
}, 0
}
defer target.Close()
oldMode := target.Info.Mode().Perm()
dangerBit, label := os.FileMode(0002), "world-writable"
if checkType == "group_writable_php" {
dangerBit, label = 0020, "group-writable"
}
if oldMode&dangerBit == 0 {
return RemediationResult{
Success: true,
Action: fmt.Sprintf("verified %s: no longer %s (mode %o)", path, label, oldMode),
Description: fmt.Sprintf("File is already not %s; no change needed", label),
}, 0
}
// #nosec G302 -- Intentional: this is the remediation that sets the
// canonical "safe web content" mode on a user file after we flagged
// the file as having dangerous perms (e.g. 0777). 0644 is what the
// webserver needs to serve static content as the file owner.
if err := chmodFunc(target.File, 0644); err != nil {
if errors.Is(err, syscall.EROFS) {
return RemediationResult{Error: fmt.Sprintf(
"cannot fix %s: the file is on a read-only mount (e.g. a backup snapshot or bind mount), not the live site. Dismiss or suppress this finding instead.",
path)}, 0
}
return RemediationResult{Error: fmt.Sprintf("chmod failed: %v", err)}, 0
}
return RemediationResult{
Success: true,
Action: fmt.Sprintf("chmod 644 %s", path),
Description: fmt.Sprintf("Changed permissions from %o to 644", oldMode),
}, oldMode
}
// fixQuarantine moves a file or directory to quarantine.
func fixQuarantine(path string) RemediationResult {
if path == "" {
return RemediationResult{Error: "could not extract file path from finding"}
}
path, info, err := resolveExistingFixPath(path, effectiveFixRoots(fixQuarantineAllowedRoots, quarantineExtraRoots...))
if err != nil {
return RemediationResult{Error: err.Error()}
}
return quarantineResolvedTarget(path, info)
}
// quarantineResolvedTarget quarantines the exact object admitted by the
// caller's boundary check. Regular-file quarantine reopens the path and
// verifies this identity before copying or unlinking it.
func quarantineResolvedTarget(path string, info os.FileInfo) RemediationResult {
qPath := newQuarantinePath(quarantineDir, path)
var quarantineWarning string
meta := quarantineMetadata(path, info, "Fixed via CSM Web UI")
if err := quarantineTarget(path, qPath, info, meta); err != nil {
var completed bool
quarantineWarning, completed = completedQuarantineWarning(err)
if !completed {
return RemediationResult{Error: err.Error()}
}
}
description := fmt.Sprintf("Moved to quarantine: %s", qPath)
if quarantineWarning != "" {
description += ". Warning: " + quarantineWarning
}
return RemediationResult{
Success: true,
Action: fmt.Sprintf("quarantined %s -> %s", path, qPath),
Description: description,
}
}
// fixKillAndQuarantine kills any process using the file, then quarantines it.
func fixKillAndQuarantine(ctx context.Context, path, details string) RemediationResult {
if path == "" {
return RemediationResult{Error: "could not extract file path from finding"}
}
resolvedPath, target, err := resolveExistingFixPath(path, effectiveFixRoots(fixQuarantineAllowedRoots, quarantineExtraRoots...))
if err != nil {
return RemediationResult{Error: err.Error()}
}
path = resolvedPath
// Try to extract and kill PID from details
pid := extractPID(details)
killed := false
var signalErr error
if pidInt, ok := parseProcessPID(pid); ok {
pid = strconv.Itoa(pidInt)
signalErr = signalProcess(ctx, pidInt, syscall.SIGKILL, func() error {
uid := getProcessUID(pid)
if uid == "0" || uid == "" || !processUsesFileIdentity(pidInt, target) {
return errProcessNotEligible
}
return nil
})
recordKillAction(nil, pid, path, signalErr)
killed = signalErr == nil
if errors.Is(signalErr, errProcessNotEligible) || errors.Is(signalErr, os.ErrProcessDone) {
signalErr = nil
}
}
if err := ctx.Err(); err != nil && !killed {
return RemediationResult{Error: err.Error()}
}
// Quarantine the same object used for the process decision. If the path was
// replaced after validation, the pinned-identity quarantine refuses it.
result := quarantineResolvedTarget(path, target)
if signalErr != nil {
result.Success = false
if result.Error != "" {
result.Error += "; "
}
result.Error += "process was not stopped: " + signalErr.Error()
}
if killed {
if result.Success {
result.Action = fmt.Sprintf("killed PID %s and %s", pid, result.Action)
result.Description = "Process killed and file quarantined"
} else {
result.Action = fmt.Sprintf("killed PID %s; quarantine failed", pid)
result.Description = "Process killed, but the file was not quarantined"
}
}
return result
}
// fixHtaccess removes malicious directives from an .htaccess file while
// preserving comments and known-safe directives (e.g., Wordfence, LiteSpeed).
func fixHtaccess(path string) (result RemediationResult) {
audit := newCleanAction(path)
defer func() { audit.finish(result.Error) }()
if path == "" {
return RemediationResult{Error: "could not extract file path"}
}
if filepath.Base(path) != ".htaccess" {
return RemediationResult{Error: "automated .htaccess remediation only applies to .htaccess files"}
}
path, info, err := resolveExistingFixPath(path, effectiveFixRoots(fixHtaccessAllowedRoots))
if err != nil {
return RemediationResult{Error: err.Error()}
}
// Same pinned-inode read and atomic replace as CleanHtaccessFile: the
// directory belongs to the account, so nothing here may follow a path
// the owner can redirect between the check and the write.
target, err := openFixTarget(path, info)
if err != nil {
return RemediationResult{Refused: errors.Is(fileResponseSourceError(err), errFileResponseRefused), Error: fmt.Sprintf("cannot open: %v", err)}
}
defer target.Close()
audit.rec.Result = actionlog.Failed
data, ok, err := readHtaccessFileBounded(target.File)
if err != nil {
return RemediationResult{Error: fmt.Sprintf("cannot read: %v", err)}
}
if !ok {
audit.rec.Result = actionlog.Refused
return RemediationResult{Refused: true, Error: ".htaccess too large to clean automatically"}
}
audit.capture(target, data)
audit.rec.Result = actionlog.Refused
_, ranges := AuditHtaccessContent(path, data)
removed := len(ranges)
cleaned := applyRangeRemoval(data, ranges)
if removed == 0 {
return RemediationResult{Error: "no malicious directives found to remove"}
}
backupPath := newQuarantinePath(htaccessBackupDirRoot, path)
meta := quarantineMetadata(path, target.Info, "Pre-clean .htaccess backup")
audit.rec.Result = actionlog.Failed
audit.rec.Reason = meta.Reason
if err := storeQuarantineBackup(backupPath, data, meta, 0600); err != nil {
return RemediationResult{Error: fmt.Sprintf("cannot create durable backup: %v", err)}
}
if err := audit.replace(target, cleaned, backupPath); err != nil {
return RemediationResult{Error: fmt.Sprintf("write failed; backup retained at %s: %v", backupPath, err)}
}
return RemediationResult{
Success: true,
Action: fmt.Sprintf("removed %d malicious directive(s) from %s", removed, path),
Description: fmt.Sprintf("Cleaned .htaccess: removed %d line(s) (backup: %s)", removed, backupPath),
}
}
// extractFilePathFromMessage extracts a file path from a finding message.
// Handles patterns like "World-writable PHP file: /path/to/file"
// and "Webshell found: /path/to/file"
func extractFilePathFromMessage(message string) string {
return findingPathInMessage(message, " ,\n)")
}
// findingPathInMessage returns the first absolute path in a finding message
// when it lies under an account or temp root. A root that only appears inside
// a longer path names a different file, so that gives "". The path ends at
// the first terminator byte; a name holding one is cut short, which is why
// callers prefer Finding.FilePath.
func findingPathInMessage(message, terminators string) string {
start := -1
for i := 0; i < len(message); i++ {
if message[i] == '/' && (i == 0 || strings.IndexByte(" \t\n(=\"'", message[i-1]) >= 0) {
start = i
break
}
}
if start < 0 {
return ""
}
rest := message[start:]
for _, prefix := range accountRootPrefixes("/var/tmp/", "/dev/shm/", "/tmp/") {
if strings.HasPrefix(rest, prefix) {
if end := strings.IndexAny(rest, terminators); end >= 0 {
return rest[:end]
}
return rest
}
}
return ""
}
func selectFindingPath(message string, filePath ...string) string {
if len(filePath) > 0 {
path := filePath[0]
if strings.TrimSpace(path) != "" {
return path
}
}
return extractFilePathFromMessage(message)
}
func resolveExistingFixPath(path string, allowedRoots []string) (string, os.FileInfo, error) {
cleanPath, err := sanitizeFixPath(path, allowedRoots)
if err != nil {
return "", nil, err
}
info, err := osFS.Lstat(cleanPath)
if err != nil {
return "", nil, fmt.Errorf("file not found: %w", err)
}
if info.Mode()&os.ModeSymlink != 0 {
return "", nil, refuseFileResponse(fmt.Errorf("symlinked paths are not eligible for automated remediation: %s", cleanPath))
}
resolved, err := filepath.EvalSymlinks(cleanPath)
if err != nil {
return "", nil, fmt.Errorf("cannot resolve path: %w", err)
}
resolved, err = sanitizeFixPath(resolved, allowedRoots)
if err != nil {
return "", nil, err
}
if accountRoot := homeAccountRoot(cleanPath); accountRoot != "" && !isPathWithinOrEqual(resolved, accountRoot) {
return "", nil, refuseFileResponse(fmt.Errorf("resolved path escapes account boundary: %s", resolved))
}
resolvedInfo, err := osFS.Lstat(resolved)
if err != nil {
return "", nil, fmt.Errorf("file not found: %w", err)
}
if resolvedInfo.Mode()&os.ModeSymlink != 0 {
return "", nil, refuseFileResponse(fmt.Errorf("symlinked paths are not eligible for automated remediation: %s", resolved))
}
fixAfterPathResolved(resolved)
return resolved, resolvedInfo, nil
}
// fixAfterPathResolved lets a test act as the account owner right after the
// path check, where a real swap would have to win a race.
var fixAfterPathResolved = func(string) {}
// openFixTarget pins the file resolveExistingFixPath admitted. Another inode
// at that name means the owner swapped it after the check.
func openFixTarget(path string, expected os.FileInfo) (*cleanTarget, error) {
return openFixTargetFlags(path, expected, os.O_RDONLY)
}
func openFixTargetFlags(path string, expected os.FileInfo, flags int) (*cleanTarget, error) {
target, err := openCleanTargetFlags(path, flags)
if err != nil {
return nil, err
}
if !sameFileIdentity(target.Info, expected) {
target.Close()
return nil, refuseFileResponse(errors.New("file changed during remediation"))
}
return target, nil
}
func sanitizeFixPath(path string, allowedRoots []string) (string, error) {
if strings.TrimSpace(path) == "" {
return "", refuseFileResponse(errors.New("file path is required"))
}
path = filepath.Clean(path)
if !filepath.IsAbs(path) {
return "", refuseFileResponse(errors.New("file path must be absolute"))
}
for _, root := range allowedRoots {
if fixTargetDepthBelow(path, root) >= fixTargetMinDepth(root) {
return path, nil
}
}
return "", refuseFileResponse(fmt.Errorf("file path is outside the allowed remediation roots: %s", path))
}
// fixTargetDepthBelow returns how many path components path lies below
// root, or 0 when path is root itself or not under it.
func fixTargetDepthBelow(path, root string) int {
cleanRoot := filepath.Clean(root)
if !strings.HasPrefix(path, cleanRoot+string(filepath.Separator)) {
return 0
}
rel := strings.TrimPrefix(path, cleanRoot+string(filepath.Separator))
return strings.Count(rel, string(filepath.Separator)) + 1
}
// fixTargetMinDepth is how far below a remediation root a target must lie.
// A root itself is never a target, and under /home neither is an account's
// home directory: quarantining or chmod-ing either takes a whole tree away.
func fixTargetMinDepth(root string) int {
if isAccountRoot(root) {
return 2
}
return 1
}
func isPathWithinOrEqual(path, base string) bool {
return safepath.Within(path, base)
}
func homeAccountRoot(path string) string {
root, account, ok := accountRootOf(path)
if !ok {
return ""
}
return filepath.Join(root, account)
}
// extractEximMsgID extracts an Exim message ID from a finding message.
// Matches the pattern "(message: XXXXXX-XXXXXX-XX)" used by emailscan.go.
func extractEximMsgID(message string) string {
prefix := "(message: "
// Sender text can contain the same marker. The target is always the
// final suffix of the finding, never an earlier occurrence.
idx := strings.LastIndex(message, prefix)
if idx < 0 {
return ""
}
// Redaction may have consumed the real suffix of a stored finding.
// Such a message cannot prove a target, even if an earlier marker remains.
rest, ok := strings.CutSuffix(message[idx+len(prefix):], ")")
if !ok {
return ""
}
return strings.TrimSpace(rest)
}
// QuarantinedSpoolFile returns the Exim spool directory and file suffix (-H
// or -D) of a quarantined spool file. It accepts only what the spool Fix
// records: a valid message ID, one of that ID's spool directories, and the
// message's header or body file in it.
func QuarantinedSpoolFile(meta QuarantineMeta) (dir, suffix string, err error) {
if !eximMsgIDRegex.MatchString(meta.MessageID) {
return "", "", fmt.Errorf("quarantined spool file has an invalid Exim message ID")
}
dir = filepath.Clean(meta.SpoolDir)
if !slices.Contains(eximSpoolMessageDirs(meta.MessageID), dir) {
return "", "", fmt.Errorf("quarantined spool file is not from an Exim spool directory")
}
for _, s := range []string{"-D", "-H"} {
if filepath.Clean(meta.OriginalPath) == filepath.Join(dir, meta.MessageID+s) {
return dir, s, nil
}
}
return "", "", fmt.Errorf("quarantined spool file is not the message's header or body")
}
// fixQuarantineSpoolMessage moves Exim spool files (-H header and -D body)
// for a message ID into quarantine.
func fixQuarantineSpoolMessage(message string) RemediationResult {
msgID := extractEximMsgID(message)
if msgID == "" {
return RemediationResult{Error: "could not extract Exim message ID from finding"}
}
// Validate Exim message ID format to prevent path traversal
if !eximMsgIDRegex.MatchString(msgID) {
return RemediationResult{Error: fmt.Sprintf("invalid Exim message ID format: %s", msgID)}
}
var spoolDir string
for _, dir := range eximSpoolMessageDirs(msgID) {
if _, err := osFS.Stat(filepath.Join(dir, msgID+"-H")); err == nil {
spoolDir = dir
break
}
}
if spoolDir == "" {
return RemediationResult{Error: fmt.Sprintf("spool message %s not found (already delivered or removed)", msgID)}
}
base := newQuarantinePath(quarantineDir, "exim_"+msgID)
moved := 0
for _, suffix := range []string{"-H", "-D"} {
src := filepath.Join(spoolDir, msgID+suffix)
info, err := os.Lstat(src)
if os.IsNotExist(err) {
continue
}
if err != nil {
return RemediationResult{Error: fmt.Sprintf("cannot inspect spool file after quarantining %d files: %v", moved, err)}
}
meta := quarantineMetadata(src, info, "Phishing email quarantined via CSM Web UI")
meta.MessageID, meta.SpoolDir = msgID, spoolDir
dst := base + suffix
if err := quarantineTarget(src, dst, info, meta); err != nil {
return RemediationResult{Error: fmt.Sprintf("spool quarantine stopped after %d files; inspect recovery copies under %s: %v", moved, quarantineDir, err)}
}
moved++
}
if moved == 0 {
return RemediationResult{Error: fmt.Sprintf("no spool files found for message %s", msgID)}
}
return RemediationResult{
Success: true,
Action: fmt.Sprintf("quarantined spool message %s (%d files)", msgID, moved),
Description: fmt.Sprintf("Exim spool files moved to quarantine for message %s", msgID),
}
}
package checks
import (
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"os"
)
// A queued finding can outlive its removal record or the bytes it reported.
// Only a current removal and a stable read authorize repeating the response.
func restoredFileResponseInfo(path, key string, expected os.FileInfo) (RemovedFile, os.FileInfo) {
removed, ok := RemovedFileAt(path)
if !ok || key != removed.RestoreKey() {
return RemovedFile{}, nil
}
target, err := openCleanTarget(path)
if err != nil {
return RemovedFile{}, nil
}
defer target.Close()
if target.Info.Size() != removed.Size || !SameFileVersion(expected, target.Info) {
return RemovedFile{}, nil
}
if contentErr := requireRemovedContent(target.File, removed); contentErr != nil {
return RemovedFile{}, nil
}
after, err := target.File.Stat()
if err != nil || !SameFileVersion(target.Info, after) {
return RemovedFile{}, nil
}
return removed, after
}
func requireRemovedContent(file *os.File, removed RemovedFile) error {
current, ok := RemovedFileAt(removed.Path)
if !ok || current != removed {
return errors.New("removal evidence changed")
}
h := sha256.New()
if _, err := io.Copy(h, io.NewSectionReader(file, 0, removed.Size)); err != nil {
return err
}
if hex.EncodeToString(h.Sum(nil)) != removed.SHA256 {
return errors.New("content differs from the removed bytes")
}
return nil
}
func quarantineRestoredTarget(path, qPath string, info os.FileInfo, metadata QuarantineMeta, removed RemovedFile) error {
return quarantineTargetWith(path, qPath, info, metadata, func(path, qPath string, info os.FileInfo, data []byte) error {
// Validate the whole digest under the existing exclusive lease. A
// same-tick rewrite must not substitute content after the first hash.
return quarantineFileTOCTOUSafeMode(path, qPath, info, data, true, -1, func(file *os.File) error {
return requireRemovedContent(file, removed)
})
})
}
package checks
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
"unicode/utf8"
"github.com/pidginhost/csm/internal/atomicio"
)
// RemovedFile is content a file response took away from a path: the bytes a
// quarantine moved out or a clean replaced.
type RemovedFile struct {
Path string `json:"path"`
SHA256 string `json:"sha256,omitempty"`
Size int64 `json:"size"`
At time.Time `json:"at"`
// Action is "quarantine" or "clean".
Action string `json:"action"`
// Seq orders removals within this process. Loaded removals have none.
Seq uint64 `json:"-"`
}
// JSON replaces invalid UTF-8 in strings. Keep those Linux path bytes separate
// so a restart cannot bind the removal to a different file.
type savedRemovedFile struct {
RemovedFile
PathBytes []byte `json:"path_bytes,omitempty"`
}
func savedRemoval(r RemovedFile) savedRemovedFile {
saved := savedRemovedFile{RemovedFile: r}
if !utf8.ValidString(r.Path) {
saved.PathBytes = []byte(r.Path)
saved.Path = ""
}
return saved
}
func (s savedRemovedFile) removal(now time.Time) (RemovedFile, bool) {
r := s.RemovedFile
if s.PathBytes != nil {
if r.Path != "" {
return RemovedFile{}, false
}
r.Path = string(s.PathBytes)
}
r.Seq = 0
return r, validRemovedFile(r, now)
}
// removedFilesStateName keeps the ledger across daemon restarts.
const removedFilesStateName = "removed-files.json"
// RestoreKey binds a finding to the removal it observed, including across
// restarts, when the in-memory sequence starts over.
func (r RemovedFile) RestoreKey() string {
return fmt.Sprintf("%s:removed-%d", r.Path, r.At.UnixNano())
}
// removedFileMemory is how long a removal stays recognisable. A restore loop
// can outwait an operator for days; entries are small and their number is
// bounded by the file response budget and by operator actions.
const removedFileMemory = 7 * 24 * time.Hour
var removedFilesNow = time.Now
// removedFiles is saved to path, once LoadRemovedFiles names it, on every
// change. Only the daemon loads it: other processes keep their removals in
// memory.
var removedFiles struct {
sync.RWMutex
byPath map[string]RemovedFile
seq uint64
path string
}
// removedFilesWrite orders saves; each one writes the ledger as it stands.
var removedFilesWrite sync.Mutex
// LoadRemovedFiles restores the removals saved under statePath and saves the
// ledger there from now on. Loaded removals are not reported again by
// RemovedFilesSince: their alert state was cleared when they happened.
func LoadRemovedFiles(statePath string) {
if statePath == "" {
return
}
path := filepath.Join(statePath, removedFilesStateName)
var saved []savedRemovedFile
data, err := readRemovedFilesState(path)
switch {
case err == nil:
if uerr := json.Unmarshal(data, &saved); uerr != nil {
fmt.Fprintf(os.Stderr, "removed files: %s is damaged, starting empty: %v\n", path, uerr)
saved = nil
}
case !os.IsNotExist(err):
fmt.Fprintf(os.Stderr, "removed files: reading %s: %v\n", path, err)
}
now := removedFilesNow()
removedFiles.Lock()
defer removedFiles.Unlock()
removedFiles.path = path
if removedFiles.byPath == nil {
removedFiles.byPath = make(map[string]RemovedFile)
}
for _, entry := range saved {
r, valid := entry.removal(now)
if !valid {
continue
}
if current, ok := removedFiles.byPath[r.Path]; ok && !current.At.Before(r.At) {
continue
}
removedFiles.byPath[r.Path] = r
}
}
// The file-action budget bounds normal ledger growth. This separate read bound
// prevents damaged state from exhausting memory before those records are checked.
const maxRemovedFilesStateBytes = 64 << 20
func readRemovedFilesState(path string) ([]byte, error) {
file, err := openTenantRegularFile(path)
if err != nil {
return nil, err
}
defer file.Close()
info, err := file.Stat()
if err != nil {
return nil, err
}
if !privateRemovedFilesState(info) {
return nil, errors.New("removal state must be private and owned by the daemon user")
}
if info.Size() > maxRemovedFilesStateBytes {
return nil, errors.New("removal state is too large")
}
data, err := io.ReadAll(io.LimitReader(file, maxRemovedFilesStateBytes+1))
if err != nil {
return nil, err
}
if len(data) > maxRemovedFilesStateBytes {
return nil, errors.New("removal state is too large")
}
if !utf8.Valid(data) {
return nil, errors.New("removal state contains invalid UTF-8")
}
after, err := file.Stat()
if err != nil {
return nil, err
}
if !SameFileVersion(info, after) || !privateRemovedFilesState(after) {
return nil, errFileChanged
}
return data, nil
}
func privateRemovedFilesState(info os.FileInfo) bool {
stat, ok := info.Sys().(*syscall.Stat_t)
return ok && int(stat.Uid) == os.Geteuid() && info.Mode().IsRegular() && info.Mode().Perm() == 0o600
}
func validRemovedFile(r RemovedFile, now time.Time) bool {
if !filepath.IsAbs(r.Path) || filepath.Clean(r.Path) != r.Path || strings.ContainsRune(r.Path, 0) ||
(r.Action != "quarantine" && r.Action != "clean") || r.Size < 0 ||
r.At.IsZero() || r.At.After(now) || now.Sub(r.At) >= removedFileMemory {
return false
}
// Large removals can legitimately lack an audit digest. They still clear
// alert state, but cannot authorize a repeat response without a content hash.
if r.SHA256 == "" {
return true
}
digest, err := hex.DecodeString(r.SHA256)
return err == nil && len(digest) == sha256.Size && hex.EncodeToString(digest) == r.SHA256
}
// saveRemovedFiles writes the ledger to its state file, if it has one.
func saveRemovedFiles() {
removedFilesWrite.Lock()
defer removedFilesWrite.Unlock()
removedFiles.RLock()
path := removedFiles.path
list := make([]savedRemovedFile, 0, len(removedFiles.byPath))
for _, r := range removedFiles.byPath {
list = append(list, savedRemoval(r))
}
removedFiles.RUnlock()
if path == "" {
return
}
if err := atomicio.AtomicWriteJSON(path, 0o600, list); err != nil {
fmt.Fprintf(os.Stderr, "removed files: saving %s: %v\n", path, err)
}
}
func rememberRemovedFile(path, digest string, size int64, action string) {
if path == "" {
return
}
now := removedFilesNow()
path = filepath.Clean(path)
removedFiles.Lock()
if removedFiles.byPath == nil {
removedFiles.byPath = make(map[string]RemovedFile)
}
for p, r := range removedFiles.byPath {
if now.Sub(r.At) >= removedFileMemory {
delete(removedFiles.byPath, p)
}
}
removedFiles.seq++
removedFiles.byPath[path] = RemovedFile{Path: path, SHA256: digest, Size: size, At: now, Action: action, Seq: removedFiles.seq}
removedFiles.Unlock()
saveRemovedFiles()
}
// RemovedFileAt reports the content CSM last removed from path, while that
// removal is recent enough to recognise.
func RemovedFileAt(path string) (RemovedFile, bool) {
removedFiles.RLock()
r, ok := removedFiles.byPath[filepath.Clean(path)]
removedFiles.RUnlock()
if !ok || removedFilesNow().Sub(r.At) >= removedFileMemory {
return RemovedFile{}, false
}
return r, true
}
// RemovedFilesSince returns the removals recorded after seq and the sequence
// to pass on the next call.
func RemovedFilesSince(seq uint64) ([]RemovedFile, uint64) {
removedFiles.RLock()
defer removedFiles.RUnlock()
var out []RemovedFile
for _, r := range removedFiles.byPath {
if r.Seq > seq {
out = append(out, r)
}
}
return out, removedFiles.seq
}
// RemovedFiles returns every removal still recent enough to recognise,
// including those loaded after a restart.
func RemovedFiles() []RemovedFile {
now := removedFilesNow()
removedFiles.RLock()
defer removedFiles.RUnlock()
var out []RemovedFile
for _, r := range removedFiles.byPath {
if now.Sub(r.At) < removedFileMemory {
out = append(out, r)
}
}
return out
}
// ForgetRemovedFile drops path before an operator restores it, so the restore
// is not taken for the removed file coming back.
func ForgetRemovedFile(path string) {
removedFiles.Lock()
delete(removedFiles.byPath, filepath.Clean(path))
removedFiles.Unlock()
saveRemovedFiles()
}
package checks
import (
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
)
// BlockableFinding reports whether the scan admission path may block the
// address in f. The offline replay tool builds its model from this, so the
// model applies the registry and detector modes exactly as the live path does.
func BlockableFinding(cfg *config.Config, f alert.Finding) bool {
return CheckResponseEnabled(cfg, f.Check) && blockableFinding(f, cfg.AutoResponse.BlockCpanelLogins)
}
// ChallengeRoutesFinding reports whether the live path sends f to the
// challenge instead of blocking it.
func ChallengeRoutesFinding(cfg *config.Config, f alert.Finding) bool {
return CheckResponseEnabled(cfg, f.Check) && responseActionForFinding(cfg, f) == responseChallenge
}
// ReputationMessageSourceIP recovers the address an ip_reputation finding
// names in its message. The audit log does not record the structured source
// address, so a replay of a recording can reach it only this way. It reads
// nothing but the producers' own form.
func ReputationMessageSourceIP(f alert.Finding) string {
if f.Check != "ip_reputation" {
return ""
}
rest, ok := strings.CutPrefix(f.Message, reputationMessagePrefix)
if !ok {
return ""
}
ip, _, ok := strings.Cut(rest, " (")
if !ok {
return ""
}
return normalizeBlockIP(ip)
}
package checks
import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/eximlog"
"github.com/pidginhost/csm/internal/mailstats"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/netutil"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
"github.com/pidginhost/csm/internal/threatintel"
)
const (
reputationCacheFile = "reputation_cache.json"
reputationEximMainlog = platform.EximMainLog
reputationWHMAccessLog = "/usr/local/cpanel/logs/access_log"
cacheExpiry = 6 * time.Hour
errorCacheExpiry = 1 * time.Hour // cache transient API errors to avoid retrying same IP
abuseConfidenceThreshold = 50
maxQueriesPerCycle = 5 // max AbuseIPDB API calls per 10-min cycle (~720/day, fits free tier)
maxCacheEntries = 5000 // cap cache size
reputationHealthReminder = 24 * time.Hour
reputationQuotaStateKey = "_reputation_health:quota"
reputationFeedStateKey = "_reputation_health:feed_stale"
sharedSendersStateKey = "_reputation_health:shared_senders"
)
var reputationHealthStateKeys = map[string]string{
logicalOwnerReputationQuota: reputationQuotaStateKey,
logicalOwnerReputationFeedStale: reputationFeedStateKey,
logicalOwnerSharedSendersStale: sharedSendersStateKey,
}
// SharedSendersHealth is the spam source detector's answer about its list of
// shared mail services, which spam source reports wait for.
type SharedSendersHealth int
const (
// SharedSendersUnknown means no answer yet: the detector is starting and
// has not finished a refresh. A warning already shown stays as it is.
SharedSendersUnknown SharedSendersHealth = iota
// SharedSendersOK means the list is current or the detector is off.
SharedSendersOK
// SharedSendersStale means the list is not current after a refresh.
SharedSendersStale
)
var sharedSendersHealth atomic.Pointer[func(time.Time) SharedSendersHealth]
// SetSharedSendersHealth wires the spam source detector's answer. Passing nil
// clears it, which reads as SharedSendersUnknown.
func SetSharedSendersHealth(fn func(time.Time) SharedSendersHealth) {
if fn == nil {
sharedSendersHealth.Store(nil)
return
}
sharedSendersHealth.Store(&fn)
}
// SharedSendersHealthAt asks the wired detector, if any.
func SharedSendersHealthAt(now time.Time) SharedSendersHealth {
if fn := sharedSendersHealth.Load(); fn != nil {
health := (*fn)(now)
// Shutdown can retire this callback and remove its detector while
// it runs. That late answer cannot establish recovery or staleness.
if sharedSendersHealth.Load() == fn {
return health
}
}
return SharedSendersUnknown
}
// maxDailyAbuseQueries is the store-backed daily circuit-breaker below
// the 1000/day free-tier ceiling. The 100-slot cushion below 1000 leaves
// room for API-side accounting differences and fallback paths that cannot
// share the store counter. Declared as a var (not const) so tests can lower it
// without burning seconds on 900 bbolt transactions. Production callers
// must not modify this.
var maxDailyAbuseQueries = 900
// abuseIPDBEndpoint is the URL queried for IP reputation. Declared as a
// var (not const) so tests can point it at an httptest server. Production
// callers must not modify this.
var abuseIPDBEndpoint = "https://api.abuseipdb.com/api/v2/check"
// abuseIPDBClient is the HTTP client used for AbuseIPDB queries. Declared
// at package scope so tests can swap in a mock client (e.g., one whose
// transport routes all traffic to an httptest server).
var abuseIPDBClient = &http.Client{Timeout: 10 * time.Second}
type reputationCache struct {
Entries map[string]*reputationEntry `json:"entries"`
// dirty tracks entries touched since load. nil means the cache was
// assembled directly rather than hydrated by loadReputationCache, in
// which case a save persists every entry.
dirty map[string]bool
// removed tracks evictions since load so bbolt saves can delete entries
// pruned from the in-memory view.
removed map[string]bool
}
type reputationEntry struct {
Score int `json:"score"`
Category string `json:"category"`
CheckedAt time.Time `json:"checked_at"`
}
// set records an entry and marks it changed so a bbolt-backed save can
// persist just this cycle's writes instead of the whole map.
func (c *reputationCache) set(ip string, e *reputationEntry) {
c.Entries[ip] = e
if c.dirty != nil {
c.dirty[ip] = true
delete(c.removed, ip)
}
}
// remove evicts an entry. Deleting any pending dirty mark keeps a later
// save from writing what eviction just removed; recording the removal lets
// bbolt delete a prior stored value for the same IP.
func (c *reputationCache) remove(ip string) {
delete(c.Entries, ip)
if c.dirty != nil {
delete(c.dirty, ip)
if c.removed != nil {
c.removed[ip] = true
}
}
}
// changedEntries returns what a bbolt-backed save must persist. Without
// change tracking every entry counts as changed. With tracking, only
// entries touched since load and still present are returned: re-putting
// the full map here used to resurrect every entry the TTL/cap prune had
// just deleted, so the bucket grew without bound.
func (c *reputationCache) changedEntries() map[string]store.ReputationEntry {
out := make(map[string]store.ReputationEntry)
if c.dirty == nil {
for ip, e := range c.Entries {
out[ip] = store.ReputationEntry{Score: e.Score, Category: e.Category, CheckedAt: e.CheckedAt}
}
return out
}
for ip := range c.dirty {
e, ok := c.Entries[ip]
if !ok {
continue
}
out[ip] = store.ReputationEntry{Score: e.Score, Category: e.Category, CheckedAt: e.CheckedAt}
}
return out
}
// CheckIPReputation looks up non-infra IPs against threat intelligence.
// Four-tier approach:
// 1. Skip if already blocked
// 2. Check local threat DB (permanent blocklist + free feeds)
// 3. Check AbuseIPDB cache
// 4. Query AbuseIPDB for truly unknown IPs (max 5/cycle, ~720/day)
func CheckIPReputation(ctx context.Context, cfg *config.Config, scanState *state.Store) []alert.Finding {
sdb := store.Global()
now := time.Now()
quotaExhausted := !abuseQuotaReady(sdb, now)
var findings []alert.Finding
supplementalAgg := newSupplementalThreatAggregator(cfg)
ips, sightings := collectRecentSightings(cfg, now)
if len(ips) == 0 {
return reputationHealthFindings(ctx, cfg, sdb, scanState, now, quotaExhausted)
}
authenticated := collectAuthenticatedIPs(cfg)
alreadyBlocked := loadAllBlockedIPs(cfg.StatePath)
threatDB := GetThreatDB()
cache := loadReputationCache(cfg.StatePath)
client := abuseIPDBClient
utcDay := now.UTC().Format("2006-01-02")
// Two-pass design so the slow AbuseIPDB HTTP queries can run in
// parallel:
//
// Pass 1 (serial): walk every IP and resolve via tier 1/2/3 plus
// the supplemental aggregator. Collect the IPs that genuinely
// need a tier-4 HTTP lookup into pendingQueries.
//
// Pass 2 (parallel, up to maxQueriesPerCycle workers): fan out
// queryAbuseIPDB and collect results.
//
// Pass 3 (serial): apply results back into the cache and emit
// findings.
//
// Pre-cache, all five HTTP queries ran in a serial loop, so a
// cycle paid ~5x worst-case AbuseIPDB latency. A busy production
// host saw ip_reputation averaging ~3.6 s per run because of
// this; the fan-out brings that down to ~max(single-call latency).
type pendingQuery struct {
ip string
source string
}
var pendingQueries []pendingQuery
for ip, source := range ips {
// A source that authenticated successfully holds valid credentials and is
// a real customer on a recycled dynamic/CGNAT address, not a drive-by
// scanner. Reputation auto-block keys on mere passive access, so without
// this skip a legitimate customer whose ISP-recycled IP appears in a
// public feed gets a 24h block the instant they open webmail. Genuine
// attacks from the same IP are still caught by the brute-force,
// compromise, and takeover detectors, which do not honour this exemption.
if authenticated[ip] {
continue
}
// Tier 1: Skip if already blocked
if alreadyBlocked[ip] {
continue
}
// Tier 2: Check local threat DB
if threatDB != nil {
if match, found := threatDB.LookupMatch(ip); found {
seen := time.Now()
expires := seen.Add(cacheExpiry)
if !match.ExpiresAt.IsZero() && match.ExpiresAt.Before(expires) {
expires = match.ExpiresAt
}
findings = append(findings, alert.Finding{
Severity: reputationSightingSeverity(source),
Check: "ip_reputation",
Message: fmt.Sprintf(reputationMessagePrefix+"%s (source: %s)", ip, match.Source),
Details: fmt.Sprintf("Detected via: %s\nMatched in local threat intelligence database", source),
Timestamp: seen,
SourceIP: ip,
Intel: &admission.IntelRef{Source: intelSource("threatdb:", match.Source), Expires: expires},
MailLoginFailure: source == reputationViaMailLogin,
})
continue
}
}
// Tier 3: Check AbuseIPDB cache. Treat entries with CheckedAt in
// the future (legacy data written by a prior buggy error-caching
// formula) as expired so they get re-queried or aged out.
if entry, ok := cache.Entries[ip]; ok {
age := time.Since(entry.CheckedAt)
if age >= 0 && age < cacheExpiry {
if entry.Score >= abuseConfidenceThreshold {
appendReputationFinding(&findings, ip, source, "AbuseIPDB", entry.Score, entry.Category, entry.CheckedAt.Add(cacheExpiry))
} else if score, src, ok := supplementalThreatScore(ctx, supplementalAgg, ip); ok && score >= abuseConfidenceThreshold {
appendReputationFinding(&findings, ip, source, src, score, strings.ToLower(src)+" history", time.Now().Add(cacheExpiry))
}
continue
}
}
// Tier 4 candidate; defer the HTTP call to pass 2 unless the
// quota / config gates already preclude querying.
if cfg.Reputation.AbuseIPDBKey == "" || quotaExhausted || len(pendingQueries) >= maxQueriesPerCycle {
if score, src, ok := supplementalThreatScore(ctx, supplementalAgg, ip); ok && score >= abuseConfidenceThreshold {
appendReputationFinding(&findings, ip, source, src, score, strings.ToLower(src)+" history", time.Now().Add(cacheExpiry))
}
continue
}
pendingQueries = append(pendingQueries, pendingQuery{ip: ip, source: source})
}
// Pass 2: reserve daily quota slots up front and fan out the HTTP
// calls. The pre-reservation matches the prior "count the attempt
// before the call so a crash or network hang still consumes a slot"
// guarantee, while keeping near-cap cycles from spending more slots
// than the store can reserve.
type queryResult struct {
score int
category string
err error
}
var refusedQueries []pendingQuery
if len(pendingQueries) > 0 && sdb != nil {
reserved := sdb.ReserveAbuseQuerySlots(utcDay, len(pendingQueries), maxDailyAbuseQueries)
refusedQueries = pendingQueries[reserved:]
pendingQueries = pendingQueries[:reserved]
}
batch := &reputationQueryBatch{}
queryWork := make([]*reputationQueryWork, len(pendingQueries))
for i := range queryWork {
queryWork[i] = reputationQueries.begin(batch)
}
defer func() {
for _, work := range queryWork {
work.finish(false)
}
}()
// Reserved work already exists while the refused tail is scored. Publish
// it first so a stalled or abandoned fallback cannot hide those queries.
for _, q := range refusedQueries {
if supplemental, src, ok := supplementalThreatScore(ctx, supplementalAgg, q.ip); ok && supplemental >= abuseConfidenceThreshold {
appendReputationFinding(&findings, q.ip, q.source, src, supplemental, strings.ToLower(src)+" history", time.Now().Add(cacheExpiry))
}
}
results := make(map[string]queryResult, len(pendingQueries))
quotaErrorObserved := false
if len(pendingQueries) > 0 {
var mu sync.Mutex
var wg sync.WaitGroup
workers := len(pendingQueries)
if workers > maxQueriesPerCycle {
workers = maxQueriesPerCycle
}
jobs := make(chan int, len(pendingQueries))
for i := range pendingQueries {
jobs <- i
}
close(jobs)
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for index := range jobs {
q, work := pendingQueries[index], queryWork[index]
work.start(client.Timeout)
func() {
completed := false
defer func() { work.returned(completed) }()
score, category, err := queryAbuseIPDB(client, q.ip, cfg.Reputation.AbuseIPDBKey, work)
if err != nil && !abuseQuotaError(err) {
work.fail()
}
mu.Lock()
results[q.ip] = queryResult{score: score, category: category, err: err}
mu.Unlock()
completed = true
}()
}
}()
}
wg.Wait()
}
// Pass 3: apply each tier-4 result back into cache + findings.
// Serial so cache writes and quota-exhaustion handling stay
// consistent regardless of which worker observed which HTTP error.
var cacheWork []*reputationQueryWork
for index, q := range pendingQueries {
work := queryWork[index]
work.phase(true)
res, ok := results[q.ip]
if !ok {
work.finish(false)
continue
}
if res.err != nil {
if abuseQuotaError(res.err) {
quotaErrorObserved = true
resetAt := nextUTCMidnight(time.Now())
fmt.Fprintf(os.Stderr, "abuseipdb: quota exhausted (%v), pausing lookups until %s\n",
res.err, resetAt.Format(time.RFC3339))
// Persisted backoff is the load-bearing signal for the
// next cycle's classifier; the cycle-local quotaExhausted
// flag has no further reader past this loop.
if sdb != nil {
if err := sdb.SetAbuseQuotaExhaustedUntil(resetAt); err != nil {
work.fail()
fmt.Fprintf(os.Stderr, "abuseipdb: persisting quota backoff: %v\n", err)
}
}
if supplemental, src, ok := supplementalResultThreatScore(ctx, cfg, supplementalAgg, q.ip, work); ok && supplemental >= abuseConfidenceThreshold {
appendReputationFinding(&findings, q.ip, q.source, src, supplemental, strings.ToLower(src)+" history", time.Now().Add(cacheExpiry))
}
work.finish(true)
continue
}
cache.set(q.ip, &reputationEntry{
Score: -1,
Category: fmt.Sprintf("error: %v", res.err),
// CheckedAt is shifted into the past so time.Since returns
// ~(cacheExpiry-errorCacheExpiry) immediately; the Tier-3
// freshness check then flips false after a further
// errorCacheExpiry, giving a real ~1h TTL on error entries.
CheckedAt: time.Now().Add(-(cacheExpiry - errorCacheExpiry)),
})
if supplemental, src, ok := supplementalResultThreatScore(ctx, cfg, supplementalAgg, q.ip, work); ok && supplemental >= abuseConfidenceThreshold {
appendReputationFinding(&findings, q.ip, q.source, src, supplemental, strings.ToLower(src)+" history", time.Now().Add(cacheExpiry))
}
work.phase(false)
cacheWork = append(cacheWork, work)
continue
}
cache.set(q.ip, &reputationEntry{
Score: res.score,
Category: res.category,
CheckedAt: time.Now(),
})
score := res.score
category := res.category
provider := "AbuseIPDB"
if supplemental, src, ok := supplementalResultThreatScore(ctx, cfg, supplementalAgg, q.ip, work); ok && supplemental > score {
score = supplemental
category = strings.ToLower(src) + " history"
provider = src
}
if score >= abuseConfidenceThreshold {
expires := time.Now().Add(cacheExpiry)
if provider == "AbuseIPDB" {
expires = cache.Entries[q.ip].CheckedAt.Add(cacheExpiry)
}
appendReputationFinding(&findings, q.ip, q.source, provider, score, category, expires)
}
work.phase(false)
cacheWork = append(cacheWork, work)
}
// All results awaiting the shared cache commit retain their own owner.
for _, work := range cacheWork {
work.phase(true)
}
cleanCache(cache)
saved := saveReputationCache(cfg.StatePath, cache)
for _, work := range cacheWork {
work.finish(saved)
}
stampReputationSightings(findings, sightings)
healthNow := time.Now()
quotaExhausted = quotaErrorObserved || !abuseQuotaReady(sdb, healthNow)
findings = append(findings, reputationHealthFindings(ctx, cfg, sdb, scanState, healthNow, quotaExhausted)...)
return findings
}
func newSupplementalThreatAggregator(cfg *config.Config) *threatintel.Aggregator {
if !cfg.Reputation.Upstream.Enabled {
threatintel.ClearUpstreamMetricsSource()
}
if !cfg.Reputation.Rspamd.Enabled && !cfg.Reputation.Upstream.Enabled {
return nil
}
agg := threatintel.NewAggregator()
if cfg.Reputation.Rspamd.Enabled {
agg.Register(threatintel.NewRspamdSource(
cfg.Reputation.Rspamd.URL,
cfg.Reputation.Rspamd.Token,
cfg.Reputation.Rspamd.TokenEnv,
))
}
if cfg.Reputation.Upstream.Enabled {
upstream := threatintel.NewUpstreamSource(threatintel.UpstreamConfig{
URL: cfg.Reputation.Upstream.URL,
Token: cfg.Reputation.Upstream.Token,
TokenEnv: cfg.Reputation.Upstream.TokenEnv,
CacheTTL: time.Duration(cfg.Reputation.Upstream.CacheTTLMin) * time.Minute,
Timeout: time.Duration(cfg.Reputation.Upstream.TimeoutSec) * time.Second,
})
threatintel.RegisterUpstreamMetrics(metrics.Default(), upstream)
agg.Register(upstream)
}
return agg
}
func supplementalResultThreatScore(ctx context.Context, cfg *config.Config, agg *threatintel.Aggregator, ip string, work *reputationQueryWork) (int, string, bool) {
if agg == nil {
return 0, "", false
}
// The aggregator runs its sources serially. Keep their configured HTTP
// budget separate from cache writes and other local result handling.
var budget time.Duration
if cfg.Reputation.Rspamd.Enabled {
budget += 5 * time.Second
}
if cfg.Reputation.Upstream.Enabled {
timeout := cfg.Reputation.Upstream.TimeoutSec
if timeout == 0 {
timeout = 5
}
budget += time.Duration(timeout) * time.Second
}
work.consuming(ctx, budget)
defer work.phase(true)
return supplementalThreatScore(ctx, agg, ip)
}
// supplementalThreatScore queries the aggregator for ip and returns the
// aggregated score, the name of the highest-scoring individual source
// (capitalised for operator-facing messages), and whether a usable score
// was found. Returns ("", 0, false) when agg is nil or no source scored.
func supplementalThreatScore(ctx context.Context, agg *threatintel.Aggregator, ip string) (int, string, bool) {
if agg == nil {
return 0, "", false
}
res, err := agg.Score(ctx, ip)
if err != nil || res.AggregatedScore == 0 {
return 0, "", false
}
// Identify the source with the highest individual score so callers can
// label findings accurately (e.g. "Rspamd" vs "Upstream").
dominant := "supplemental"
max := 0
for name, s := range res.Sources {
if s > max {
max = s
dominant = name
}
}
return res.AggregatedScore, capitalizeProvider(dominant), true
}
// capitalizeProvider title-cases known source names for operator-facing
// messages ("rspamd" -> "Rspamd", "upstream" -> "Upstream").
func capitalizeProvider(name string) string {
if len(name) == 0 {
return name
}
return strings.ToUpper(name[:1]) + name[1:]
}
// reputationSightingSeverity grades a threat-intel sighting by what the IP
// was doing. Auth-surface contact (SSH, mail credential attacks) is an
// active threat and stays Critical; a passive web sighting of a listed IP
// is ambient scanner noise, downgraded to High so thousands of drive-by
// scanners do not drown compromise-class Criticals. Auto-block eligibility
// is keyed on the check name and is not affected by this severity. Unknown
// future surfaces fail closed to Critical.
func reputationSightingSeverity(detectedVia string) alert.Severity {
switch detectedVia {
case "HTTP request", "cPanel/WHM access":
return alert.High
default:
return alert.Critical
}
}
// reputationViaMailLogin names the sighting of an address in failed IMAP or
// POP3 logins. Its findings are mail attacks, not web attacks.
const reputationViaMailLogin = "Dovecot IMAP/POP3 auth failure"
// reputationMessagePrefix opens every ip_reputation message; the address
// follows it up to " (". ReputationMessageSourceIP depends on that form.
const reputationMessagePrefix = "Known malicious IP accessing server: "
// appendReputationFinding reports a listed address. expires is when the
// provider's claim lapses: the cache entry's end for a cached score, one
// cache period for a fresh one.
func appendReputationFinding(findings *[]alert.Finding, ip, detectedVia, provider string, score int, category string, expires time.Time) {
*findings = append(*findings, alert.Finding{
Severity: reputationSightingSeverity(detectedVia),
Check: "ip_reputation",
Message: fmt.Sprintf(reputationMessagePrefix+"%s (%s score: %d/100)", ip, provider, score),
Details: fmt.Sprintf("Detected via: %s\nCategory: %s\nThis IP is reported in threat intelligence databases", detectedVia, category),
Timestamp: time.Now(),
SourceIP: ip,
Intel: &admission.IntelRef{Source: intelSource("", strings.ToLower(provider)), Expires: expires},
MailLoginFailure: detectedVia == reputationViaMailLogin,
})
}
// intelSource names an intel provider as a bounded token: printable bytes
// only, at most 64 of them.
func intelSource(prefix, name string) string {
b := []byte(prefix + name)
for i, c := range b {
if c < 0x21 || c > 0x7e {
b[i] = '_'
}
}
if len(b) > 64 {
b = b[:64]
}
return string(b)
}
// reputationHealthFindings surfaces degraded reputation coverage: an
// exhausted AbuseIPDB quota (tier-4 lookups paused) and threat feeds that
// have not refreshed in over a week (stale tier-2 data). Feeds that never
// loaded stay silent - a fresh install's first download may still be
// pending, which is not operator-actionable. It also reports spam source
// reports waiting on a shared mail service list that is not current.
func reputationHealthFindings(ctx context.Context, cfg *config.Config, sdb *store.DB, scanState *state.Store, now time.Time, quotaExhausted bool) []alert.Finding {
var out []alert.Finding
disabled := disabledLogicalOwners(cfg)
quotaActive := cfg.Reputation.AbuseIPDBKey != "" && quotaExhausted
if _, off := disabled[logicalOwnerReputationQuota]; off {
clearReputationHealthState(scanState, reputationQuotaStateKey)
} else if quotaActive {
detail := "Daily AbuseIPDB query budget reached; uncached AbuseIPDB lookups resume at 00:00 UTC. Local threat DB, rspamd, upstream, and cached scores still apply when configured."
if sdb != nil {
if until := sdb.AbuseQuotaExhaustedUntil(); !until.IsZero() && now.Before(until) {
detail = fmt.Sprintf("API returned a quota error; uncached AbuseIPDB lookups are paused until %s. Local threat DB, rspamd, upstream, and cached scores still apply when configured.", until.UTC().Format(time.RFC3339))
}
}
finding := alert.Finding{
Severity: alert.Warning,
Check: "reputation_quota_exhausted",
Message: "AbuseIPDB quota exhausted: uncached AbuseIPDB lookups are paused",
Details: detail,
Timestamp: now,
}
appendReputationHealthFinding(ctx, &out, scanState, logicalOwnerReputationQuota, reputationQuotaStateKey, now, finding)
} else {
clearReputationHealthState(scanState, reputationQuotaStateKey)
}
lastRefresh := time.Time{}
if db := GetThreatDB(); db != nil {
lastRefresh = db.LastFeedRefresh()
}
feedActive := feedRefreshStale(lastRefresh, now)
if _, off := disabled[logicalOwnerReputationFeedStale]; off {
clearReputationHealthState(scanState, reputationFeedStateKey)
} else if feedActive {
finding := alert.Finding{
Severity: alert.Warning,
Check: "threat_feed_stale",
Message: "Threat intelligence feeds stale: reputation matching runs on old data",
Details: fmt.Sprintf("Last successful feed update: %s. Downloads retry on each deep-scan cycle; check outbound connectivity to the feed mirrors.", lastRefresh.UTC().Format(time.RFC3339)),
Timestamp: now,
}
appendReputationHealthFinding(ctx, &out, scanState, logicalOwnerReputationFeedStale, reputationFeedStateKey, now, finding)
} else {
clearReputationHealthState(scanState, reputationFeedStateKey)
}
if _, off := disabled[logicalOwnerSharedSendersStale]; off {
clearReputationHealthState(scanState, sharedSendersStateKey)
} else {
sharedSendersHealthFinding(ctx, &out, scanState, now)
}
return out
}
// sharedSendersHealthFinding warns while spam source reports wait on a shared
// mail service list that should be current by now.
func sharedSendersHealthFinding(ctx context.Context, out *[]alert.Finding, scanState *state.Store, now time.Time) {
switch SharedSendersHealthAt(now) {
case SharedSendersStale:
finding := alert.Finding{
Severity: alert.Warning,
Check: "shared_senders_stale",
Message: "Shared mail service list not current: spam source reports are paused",
Details: "Spam source reports wait until CSM can read the sending hosts that Google, Microsoft and the other shared mail services publish. Refreshes retry on their own; the daemon log names the services that failed. Check this host's DNS resolver.",
Timestamp: now,
}
appendReputationHealthFinding(ctx, out, scanState, logicalOwnerSharedSendersStale, sharedSendersStateKey, now, finding)
case SharedSendersUnknown:
// Before the detector's first refresh, keep a warning from before a
// restart, and its reminder, as they are.
markCheckIncomplete(ctx, logicalOwnerSharedSendersStale)
default:
clearReputationHealthState(scanState, sharedSendersStateKey)
}
}
func appendReputationHealthFinding(ctx context.Context, out *[]alert.Finding, scanState *state.Store, owner, stateKey string, now time.Time, finding alert.Finding) {
if scanState != nil {
emit, err := scanState.ClaimRawTimestamp(stateKey, now, reputationHealthReminder)
if err != nil {
fmt.Fprintf(os.Stderr, "reputation: persisting %s reminder: %v\n", finding.Check, err)
}
if !emit {
markCheckIncomplete(ctx, owner)
return
}
}
*out = append(*out, finding)
}
func clearReputationHealthState(scanState *state.Store, stateKey string) {
if scanState == nil {
return
}
if err := scanState.DeleteRawAndSave(stateKey); err != nil {
fmt.Fprintf(os.Stderr, "reputation: clearing health reminder state %s: %v\n", stateKey, err)
}
}
func feedRefreshStale(lastRefresh, now time.Time) bool {
if lastRefresh.IsZero() {
return false
}
lastRefresh = lastRefresh.Round(0)
now = now.Round(0)
return !lastRefresh.After(now) && now.Sub(lastRefresh) > 7*24*time.Hour
}
// nextUTCMidnight returns 00:00 UTC on the day after now — the point at
// which AbuseIPDB's daily quota resets.
func nextUTCMidnight(now time.Time) time.Time {
u := now.UTC()
return time.Date(u.Year(), u.Month(), u.Day()+1, 0, 0, 0, 0, time.UTC)
}
// abuseQuotaReady reports whether we may call AbuseIPDB right now. It
// combines the persisted backoff (set when the API returns 429/402) with
// the daily query counter (stops before we approach the free-tier cap).
// Returns true when no bbolt store is available (fallback mode).
func abuseQuotaReady(sdb *store.DB, now time.Time) bool {
if sdb == nil {
return true
}
if until := sdb.AbuseQuotaExhaustedUntil(); !until.IsZero() && now.Before(until) {
return false
}
if sdb.AbuseQueryCount(now.UTC().Format("2006-01-02")) >= maxDailyAbuseQueries {
return false
}
return true
}
// collectRecentIPs gathers non-infra IPs from multiple log sources.
// Returns map of IP → source description (e.g. "SSH login", "Dovecot IMAP auth failure").
func collectRecentIPs(cfg *config.Config) map[string]string {
ips, _ := collectRecentSightings(cfg, time.Now())
return ips
}
// reputationSightingWindow is how old a local record may be and still
// corroborate intel.
const reputationSightingWindow = 24 * time.Hour
// collectRecentSightings keeps the legacy surface choice and, separately,
// the strongest surface that can supply evidence. Rewritten HTTP logs can
// report an address without displacing a direct sighting from another log.
// The kept record is that surface's newest within the sighting window.
func collectRecentSightings(cfg *config.Config, now time.Time) (ips map[string]string, records map[string]scanRecord) {
ips, records = make(map[string]string), make(map[string]scanRecord)
surfaces := make(map[string]string)
add := func(ip, source string, r scanRecord) {
addIfNotInfra(ips, ip, source, cfg)
previous := surfaces[ip]
if !addIfNotInfra(surfaces, ip, source, cfg) {
return
}
if previous != source {
// A stronger surface replaced the earlier one and its record.
delete(records, ip)
}
// Dated within the window, and never after the scan itself: the
// shared skew for scan records does not apply to a sighting.
if !r.at.After(now) && !r.at.Before(now.Add(-reputationSightingWindow)) {
keepNewest(records, ip, r, now)
}
}
info := platform.Detect()
// SSH logins. Path differs by OS family (secure vs auth.log); the old
// hardcoded /var/log/secure made this loop dead on Debian/Ubuntu.
for _, rec := range tailFileRecords(info.AuthLogPath(), 50) {
if !strings.Contains(rec.text, "Accepted") {
continue
}
if _, ip, ok := sshAcceptedRecord(strings.Fields(rec.text)); ok {
at, _ := syslogLineTime(rec.text, now)
add(ip, "SSH login", scanned(rec, at))
}
}
// Web server access logs, platform-detected (Apache/Nginx/LiteSpeed).
for _, path := range info.AccessLogPaths {
if path == "" || isWHMAccessLog(path) {
continue
}
for _, rec := range tailFileRecords(path, 100) {
if ip := firstField(rec.text); ip != "" {
if CurrentHTTPRewrite().State != RewriteNone {
addIfNotInfra(ips, ip, "HTTP request", cfg)
continue
}
add(ip, "HTTP request", accessLogSighting(rec, ip, cfg))
}
}
}
// Dovecot - IMAP/POP3 auth failures.
if mailLog := reputationMailLogPath(cfg, info); mailLog != "" {
for _, rec := range tailFileRecords(mailLog, 50) {
msg, _, _ := strings.Cut(DovecotLoginMessage(rec.text), "user=<")
if strings.Contains(msg, "auth failed") || strings.Contains(msg, "Aborted login") {
if ip := DovecotRemoteIP(rec.text); ip != "" {
at, _ := syslogLineTime(rec.text, now)
add(ip, reputationViaMailLogin, scanned(rec, at))
}
}
}
}
if info.IsCPanel() {
// Service-subdomain records cannot nominate an address for an IP
// lookup, even when a malformed suffix prevents full parsing.
for _, rec := range tailFileRecords(reputationWHMAccessLog, 100) {
if fields := strings.Fields(rec.text); len(fields) > 1 && fields[1] == "proxy" {
continue
}
if ip := firstField(rec.text); ip != "" {
at, _ := cpanelAccessLogTime(rec.text)
// A trusted proxy in front of cpsrvd is not the visitor.
r := scanned(rec, at)
if IsTrustedProxy(ip, cfg) {
r = scanRecord{}
}
add(ip, "cPanel/WHM access", r)
}
}
}
if shouldCollectEximMainlog(info) {
for _, rec := range tailFileRecords(reputationEximMainlog, 50) {
// Only the failure or rejection record names the client: the
// same wording in a subject or greeting is client text.
line := rec.text
ip := eximlog.AuthFailure(line).ClientIP
if ip == "" && strings.Contains(line, " rejected RCPT ") {
if r, ok := mailstats.ParseReject(line, time.UTC, time.Time{}); ok && r.Command == "RCPT" {
ip = r.IP
}
}
if ip != "" {
// A stamp without a zone is in the scan clock's zone, as for syslog.
at, _ := eximlog.LineTime(line, now.Location())
add(ip, "SMTP auth failure", scanned(rec, at))
}
}
}
return ips, records
}
// accessLogSighting is the record of an access-log line its parser reads as
// a request from ip itself. A trusted proxy's own address is never a
// sighting: the requests it carries are its clients'. The caller excludes
// web servers that replace the peer address they log.
func accessLogSighting(rec syslogRecord, ip string, cfg *config.Config) scanRecord {
parsed, ok := parseAccessLogRecord(rec.text)
if !ok || parsed.RemoteIP != ip || IsTrustedProxy(parsed.RemoteIP, cfg) {
return scanRecord{}
}
return scanned(rec, parsed.Time)
}
// stampReputationSightings names each reputation finding's local sighting
// as its observation: intel plus a recent local record of the address make
// derived evidence. A finding without such a record names none and mints
// nothing.
func stampReputationSightings(findings []alert.Finding, records map[string]scanRecord) {
for i := range findings {
if findings[i].Check != "ip_reputation" {
continue
}
if r, ok := records[findings[i].SourceIP]; ok {
r.kind = admission.AttributionDerived
r.stamp(&findings[i], ProducerReputationScan)
}
}
}
// collectAuthenticatedIPs returns IPs that successfully authenticated to a
// mailbox in the recent log window. A source that holds valid mail credentials
// is a real customer, not a drive-by scanner: a threat-feed match on such an IP
// is almost always a recycled dynamic/CGNAT address rather than the attacker the
// feed once listed. Romanian and other residential ISPs rotate these pools
// aggressively, so an IP an attacker used last week routinely lands on a paying
// customer this week. Webmail authenticates through dovecot, so this also covers
// Horde/Roundcube users.
//
// Successful SSH logins are deliberately excluded: a clean SSH auth from a
// feed-listed IP is itself a red flag (a cracked or attacker-controlled host),
// and collectRecentIPs already surfaces those IPs precisely so their reputation
// is checked. Authenticated mail attackers (compromised accounts) remain covered
// by the brute-force, compromise, and account-takeover detectors, which do not
// honour this exemption. The mail window is wider than collectRecentIPs uses so
// a customer's success is not pushed out of view by a burst of attacker failures
// occupying the tail.
func collectAuthenticatedIPs(cfg *config.Config) map[string]bool {
authed := make(map[string]bool)
info := platform.Detect()
mailLog := reputationMailLogPath(cfg, info)
if mailLog == "" {
return authed
}
for _, line := range tailFile(mailLog, 200) {
if !DovecotLoginSucceeded(line) {
continue
}
if ip := DovecotRemoteIP(line); ip != "" {
authed[ip] = true
}
}
return authed
}
func reputationMailLogPath(cfg *config.Config, info platform.Info) string {
if cfg == nil {
return info.MailLogPath()
}
if cfg.MailLogs.Source == "journal" {
return ""
}
if cfg.MailLogs.File != "" {
return cfg.MailLogs.File
}
return info.MailLogPath()
}
func isWHMAccessLog(path string) bool {
return filepath.Clean(path) == reputationWHMAccessLog
}
func shouldCollectEximMainlog(info platform.Info) bool {
if info.IsCPanel() {
return true
}
if _, err := osFS.Stat(reputationEximMainlog); err != nil {
return !os.IsNotExist(err)
}
return true
}
// addIfNotInfra records source as ip's surface unless ip is local or
// infrastructure, and reports whether source is ip's surface afterwards.
func addIfNotInfra(ips map[string]string, ip, source string, cfg *config.Config) bool {
if ip == "127.0.0.1" || ip == "::1" || ip == "" {
return false
}
if isInfraIP(ip, cfg.InfraIPs) {
return false
}
// One address can appear on several surfaces in the same scan. Keep the
// strongest sighting so an earlier passive web request cannot hide a later
// authentication attack and downgrade the resulting finding.
current, exists := ips[ip]
if !exists || reputationSightingSeverity(source) > reputationSightingSeverity(current) {
ips[ip] = source
}
return ips[ip] == source
}
func firstField(line string) string {
fields := strings.Fields(line)
if len(fields) == 0 {
return ""
}
ip := fields[0]
// Validate it looks like an IP (v4 or v6)
if strings.Count(ip, ".") == 3 || strings.Contains(ip, ":") {
return ip
}
return ""
}
func extractIPAfterKeyword(line, keyword string) string {
idx := strings.Index(line, keyword)
if idx < 0 {
return ""
}
rest := line[idx+len(keyword):]
rest = strings.TrimLeft(rest, " =")
fields := strings.Fields(rest)
if len(fields) == 0 {
return ""
}
if ip, ok := netutil.ParseIPToken(fields[0]); ok {
return ip
}
return ""
}
// loadAllBlockedIPs returns all IPs currently blocked in CSM.
// It reads firewall state through osFS so tests can inject a filesystem,
// then merges the legacy blocked_ips.json file.
func loadAllBlockedIPs(statePath string) map[string]bool {
blocked := make(map[string]bool)
unverified := make(map[string]bool)
// Read the authoritative firewall engine state. The engine persists
// every block to firewall/state.json; the parallel bbolt fw:blocked
// bucket is written only at migration, so reading it would return a
// frozen snapshot that misses live blocks.
fwPath := filepath.Join(statePath, "firewall", "state.json")
if fwData, err := osFS.ReadFile(fwPath); err == nil {
var fwState struct {
Blocked []struct {
IP string `json:"ip"`
ExpiresAt time.Time `json:"expires_at"`
Unverified bool `json:"unverified"`
} `json:"blocked"`
}
if uerr := json.Unmarshal(fwData, &fwState); uerr != nil {
fmt.Fprintf(os.Stderr, "reputation: %s is corrupt, alert suppression degraded: %v\n", fwPath, uerr)
} else {
now := time.Now()
for _, entry := range fwState.Blocked {
// An unverified row's effect is unproven; it is not a block.
if entry.Unverified {
if ip := net.ParseIP(entry.IP); ip != nil {
unverified[ip.String()] = true
}
} else if entry.ExpiresAt.IsZero() || now.Before(entry.ExpiresAt) {
blocked[entry.IP] = true
}
}
}
}
// Also read from blocked_ips.json (legacy CSM auto-block)
type blockedEntry struct {
IP string `json:"ip"`
ExpiresAt time.Time `json:"expires_at"`
}
type blockFile struct {
IPs []blockedEntry `json:"ips"`
Pending []struct {
IP string `json:"ip"`
} `json:"pending,omitempty"`
}
legacyPath := filepath.Join(statePath, "blocked_ips.json")
data, err := osFS.ReadFile(legacyPath)
if err == nil {
var bf blockFile
if uerr := json.Unmarshal(data, &bf); uerr != nil {
fmt.Fprintf(os.Stderr, "reputation: %s is corrupt, alert suppression degraded: %v\n", legacyPath, uerr)
} else {
now := time.Now()
for _, entry := range bf.IPs {
if now.Before(entry.ExpiresAt) {
blocked[entry.IP] = true
}
}
for _, entry := range bf.Pending {
blocked[entry.IP] = true
}
}
}
// A stale legacy tracker cannot prove an unresolved firewall effect.
for ip := range blocked {
if parsed := net.ParseIP(ip); parsed != nil && unverified[parsed.String()] {
delete(blocked, ip)
}
}
return blocked
}
type abuseIPDBResponse struct {
Data struct {
AbuseConfidenceScore int `json:"abuseConfidenceScore"`
UsageType string `json:"usageType"`
ISP string `json:"isp"`
TotalReports int `json:"totalReports"`
} `json:"data"`
Errors []struct {
Detail string `json:"detail"`
Status int `json:"status"`
} `json:"errors"`
}
// queryAbuseIPDB returns (score, category, error).
// Returns specific errors for rate limiting (429) and quota exhaustion (402).
func queryAbuseIPDB(client *http.Client, ip, apiKey string, work *reputationQueryWork) (score int, category string, err error) {
req, err := http.NewRequest("GET", abuseIPDBEndpoint+"?ipAddress="+url.QueryEscape(ip)+"&maxAgeInDays=90", nil)
if err != nil {
return 0, "", err
}
req.Header.Set("Key", apiKey)
req.Header.Set("Accept", "application/json")
resp, err := client.Do(req)
if err != nil {
return 0, "", err
}
defer func() {
work.cleanup(err)
_ = resp.Body.Close()
}()
if resp.StatusCode == 429 {
return 0, "", fmt.Errorf("429 rate limited")
}
if resp.StatusCode == 402 {
return 0, "", fmt.Errorf("402 quota exceeded")
}
if resp.StatusCode != 200 {
return 0, "", fmt.Errorf("HTTP %d", resp.StatusCode)
}
var result abuseIPDBResponse
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return 0, "", err
}
if len(result.Errors) > 0 {
return 0, "", fmt.Errorf("API error: %s", result.Errors[0].Detail)
}
category = result.Data.UsageType
if result.Data.ISP != "" {
category += " (" + result.Data.ISP + ")"
}
if result.Data.TotalReports > 0 {
category += fmt.Sprintf(", %d reports", result.Data.TotalReports)
}
return result.Data.AbuseConfidenceScore, category, nil
}
func abuseQuotaError(err error) bool {
return strings.Contains(err.Error(), "429") || strings.Contains(err.Error(), "402")
}
// cleanCache removes expired entries and caps at maxCacheEntries.
func cleanCache(cache *reputationCache) {
// When using bbolt store, delegate cleanup to store methods, then
// mirror the prune on the in-memory view: entries deleted only in
// bbolt would linger in the map and could be pushed back by a save.
if sdb := store.Global(); sdb != nil {
sdb.CleanExpiredReputation(cacheExpiry)
sdb.EnforceReputationCap(maxCacheEntries)
}
pruneCacheEntries(cache)
}
// pruneCacheEntries drops expired entries from the in-memory map and
// enforces the size cap, evicting oldest first.
func pruneCacheEntries(cache *reputationCache) {
now := time.Now()
// Remove expired entries - use same expiry as cache freshness check
for ip, entry := range cache.Entries {
if now.Sub(entry.CheckedAt) > cacheExpiry {
cache.remove(ip)
}
}
// Cap at max entries - remove oldest if over limit
if len(cache.Entries) > maxCacheEntries {
type aged struct {
ip string
age time.Duration
}
entries := make([]aged, 0, len(cache.Entries))
for ip, entry := range cache.Entries {
entries = append(entries, aged{ip, now.Sub(entry.CheckedAt)})
}
// Sort by age descending (oldest first)
sort.Slice(entries, func(i, j int) bool {
return entries[i].age > entries[j].age
})
// Remove oldest until under limit
for i := 0; i < len(entries)-maxCacheEntries; i++ {
cache.remove(entries[i].ip)
}
}
}
func loadReputationCache(statePath string) *reputationCache {
cache := &reputationCache{
Entries: make(map[string]*reputationEntry),
dirty: make(map[string]bool),
removed: make(map[string]bool),
}
// Try bbolt store first - after migration the flat file is renamed to .bak.
if sdb := store.Global(); sdb != nil {
for ip, entry := range sdb.AllReputation() {
cache.Entries[ip] = &reputationEntry{
Score: entry.Score,
Category: entry.Category,
CheckedAt: entry.CheckedAt,
}
}
return cache
}
// Fallback: flat-file JSON (pre-migration).
data, err := osFS.ReadFile(filepath.Join(statePath, reputationCacheFile))
if err == nil {
_ = json.Unmarshal(data, cache)
if cache.Entries == nil {
cache.Entries = make(map[string]*reputationEntry)
}
}
return cache
}
func saveReputationCache(statePath string, cache *reputationCache) bool {
if sdb := store.Global(); sdb != nil {
changed := cache.changedEntries()
if len(changed) == 0 && len(cache.removed) == 0 {
return true
}
if err := sdb.ApplyReputationChanges(changed, cache.removed); err != nil {
// Keep the pending marks so a later save can retry the flush.
return false
}
clear(cache.dirty)
clear(cache.removed)
return true
}
// Fallback: flat-file JSON.
data, err := json.MarshalIndent(cache, "", " ")
if err != nil {
return false
}
tmpPath := filepath.Join(statePath, reputationCacheFile+".tmp")
if err := os.WriteFile(tmpPath, data, 0600); err != nil {
return false
}
return os.Rename(tmpPath, filepath.Join(statePath, reputationCacheFile)) == nil
}
package checks
import (
"context"
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
var reputationQueries = newReputationQueue()
type reputationQueueMonitor struct {
mu sync.Mutex
pending map[*reputationQueryWork]struct{}
losses *queuehealth.Tracker
}
type reputationQueryBatch struct {
consumer *reputationQueryWork // guarded by the queue mutex
}
type reputationQueryWork struct {
batch *reputationQueryBatch
queue *reputationQueueMonitor
at, deadline time.Time
running, queryStarted, queryDone, deliveryDone, failed bool
}
func newReputationQueue() *reputationQueueMonitor {
return &reputationQueueMonitor{pending: make(map[*reputationQueryWork]struct{}), losses: queuehealth.New(0, time.Minute)}
}
func (q *reputationQueueMonitor) begin(batch *reputationQueryBatch) *reputationQueryWork {
now := time.Now()
w := &reputationQueryWork{queue: q, batch: batch, at: now, deadline: now.Add(time.Minute)}
q.mu.Lock()
q.pending[w] = struct{}{}
q.mu.Unlock()
return w
}
func (w *reputationQueryWork) moveLocked(running bool, budget time.Duration) {
w.running = running
w.at = time.Now()
w.deadline = w.at.Add(budget)
}
func (w *reputationQueryWork) start(budget time.Duration) {
w.queue.mu.Lock()
w.queryStarted = true
w.moveLocked(true, budget)
w.queue.mu.Unlock()
}
func (w *reputationQueryWork) failLocked() {
if !w.failed {
w.failed = true
w.queue.losses.Lose(time.Now(), 1)
}
}
func (w *reputationQueryWork) fail() {
w.queue.mu.Lock()
w.failLocked()
w.queue.mu.Unlock()
}
func (w *reputationQueryWork) cleanup(err error) {
// Protocol calls outside the worker pool have no queued delivery owner.
if w == nil {
return
}
failed := err != nil && !abuseQuotaError(err)
w.queue.mu.Lock()
if failed {
w.failLocked()
}
w.moveLocked(true, time.Minute)
w.queue.mu.Unlock()
}
func (w *reputationQueryWork) returned(completed bool) {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
if !completed {
w.failLocked()
}
w.queryDone = true
w.moveLocked(false, time.Minute)
w.releaseLocked()
}
func (w *reputationQueryWork) phase(running bool) {
w.queue.mu.Lock()
w.moveLocked(running, time.Minute)
if running {
w.batch.consumer = w
} else if w.batch.consumer == w {
w.batch.consumer = nil
}
w.queue.mu.Unlock()
}
func (w *reputationQueryWork) consuming(ctx context.Context, budget time.Duration) {
deadline := time.Now().Add(budget)
if parent, ok := ctx.Deadline(); ok && parent.Before(deadline) {
deadline = parent
}
w.queue.mu.Lock()
w.moveLocked(true, budget)
w.deadline = deadline
w.batch.consumer = w
w.queue.mu.Unlock()
}
func (w *reputationQueryWork) finish(success bool) {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
if w.deliveryDone {
return
}
w.deliveryDone = true
if w.batch.consumer == w {
w.batch.consumer = nil
}
if !success {
w.failLocked()
}
// Dispatch can end before a worker starts. Already running HTTP keeps its
// separate owner until response cleanup and result publication finish.
if !w.queryStarted {
w.queryDone = true
}
w.releaseLocked()
}
func (w *reputationQueryWork) releaseLocked() {
if w.queryDone && w.deliveryDone {
delete(w.queue.pending, w)
}
}
// ReputationQueueStatus reads memory only, including queries outliving a check.
func ReputationQueueStatus(now time.Time) queuehealth.Status {
q := reputationQueries
q.mu.Lock()
defer q.mu.Unlock()
s := q.losses.Snapshot(now)
s.CapacityUnavailable = true
var waitingLate, runningLate bool
for w := range q.pending {
if w.running {
s.InFlight++
s.ProcessingSeconds = max(s.ProcessingSeconds, now.Sub(w.at).Seconds())
runningLate = runningLate || !now.Before(w.deadline)
} else {
s.Depth++
s.LagSeconds = max(s.LagSeconds, now.Sub(w.at).Seconds())
// A result waiting behind its own batch consumer follows that
// consumer's deadline. Another batch cannot lend it progress.
waitingLate = waitingLate || ((!w.queryDone || w.batch.consumer == nil) && !now.Before(w.deadline))
}
}
switch {
case waitingLate:
s.Reason = "backlog_lag"
case runningLate:
s.Reason = "processing_lag"
}
if s.Reason != "" {
s.Status = "degraded"
}
return s
}
package checks
import (
"fmt"
"net"
"strings"
"sync"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
)
// BlockEligibility says whether findings of a check may drive an automatic
// firewall block of their source address.
type BlockEligibility uint8
const (
// BlockNever is the zero value: the check never drives a single-IP scan
// block.
BlockNever BlockEligibility = iota
// BlockAlways: the finding carries a confirmed attacker IP: thresholded
// brute force, confirmed compromise, C2/reputation, or escalation. Raw
// mailbox auth failures and account-only mail findings feed incident
// grouping and thresholded trackers, but one row is not enough evidence
// for a block. Eligibility alone does not prove attribution or authorize
// the engine; callers retain their severity, mode and action gates.
BlockAlways
// BlockWithCpanelLogins: blockable only when block_cpanel_logins is
// enabled (disabled by default). Every such check reports a FAILED or
// thresholded authentication attempt, which is real evidence.
//
// Checks that report a SUCCESSFUL operation are deliberately never
// blockable, and must stay that way. cpanel_login and
// cpanel_login_realtime were excluded first: they fire on every direct
// form login from a non-infra IP, and blocking on one such Warning turns a
// legitimate customer logging in from a new country into a 24h lockout.
//
// cpanel_file_upload_realtime, ftp_login and webmail_login_realtime were
// missed at the time and caused exactly that. A customer was blocked one
// second after uploading a file in File Manager, and five addresses were
// blocked for logging in to FTP successfully. The handler skips 401 and
// 403, so these only fire once the user has authenticated; on shared
// hosting every customer is a non-infra IP, so they fire on ordinary use
// of core features. They remain findings, which is where their value is --
// correlated with other evidence on the same account -- but they never
// block on their own.
BlockWithCpanelLogins
)
// ResponsePolicy is a check's automatic IP response policy, carried by its
// registry entry. It governs single-IP scan admission (AutoBlockIPs) and
// challenge routing (ChallengeRouteIPs). The zero value neither blocks nor
// challenges there, so a check nobody classified cannot aim those responses
// at an address. The subnet-spray, ASN-crawl and netblock escalation paths,
// and the challenge-timeout, incident and central-intel callers, do not
// consult this policy yet.
type ResponsePolicy struct {
Block BlockEligibility
// CriticalOnly limits blocking to Critical findings of the check. An
// established multi-mailbox source of mail_account_compromised is advisory
// below Critical.
CriticalOnly bool
// ChallengeFirst routes the source to the proof-of-work challenge before
// any firewall block while challenge routing is enabled. Two rules for
// setting it: the address must be a client making an HTTP(S) request a
// browser could answer, and the finding must be attack signal, not an
// audit event. Background tasks (DNS, SSH or FTP clients, internal auth
// daemons) have no browser, so routing them only produces
// challenge-timeout blocks.
//
// Deliberately not challenge-first (do not change without revisiting the
// two rules above):
//
// - cpanel_login / cpanel_login_realtime: post-auth audit events; the
// user is already inside cPanel and never makes a fresh connection
// the gate could catch.
// - cpanel_file_upload / cpanel_file_upload_realtime: same; post-auth.
// - cpanel_multi_ip_login / whm_password_change: multi-vector audit.
// - ftp_login / ssh_login_unknown_ip: no browser at the other end of
// FTP or SSH.
// - webmail_login_realtime: same as cpanel_login_realtime; post-auth.
// - dns_connection / user_outbound_connection: recursive resolvers and
// egress targets have no client browser.
// - api_auth_failure: API clients, not browsers.
// - brute_force: legacy bucket; superseded by per-protocol entries.
ChallengeFirst bool
// NeverChallenge marks a check whose source must never be offered a
// challenge: there is no browser at the other end, or the evidence is
// strong enough that a challenge would only delay containment.
NeverChallenge bool
// Evidence is the observation family of the check's address evidence.
// Every check that can drive a response has one. FamilyNone means its
// address, if any, is never admissible evidence: a destination, an
// authenticated customer, an advisory or a record of a response.
Evidence admission.Family
// Basis is the priority class the check's own evidence supports.
// BasisCompromise is reserved for the reviewed compromise checks.
Basis admission.Basis
// Attribution is the set of kinds the check accepts its address from:
// only kinds its evidence family can be read by. Every check
// with an evidence family names at least one.
Attribution admission.AttributionSet
// Subnet names the subnet response the check feeds: the mail subnet
// spray block or the crawl tempban. A subnet summary never authorizes
// a single-address block.
Subnet admission.Entry
}
// neverChallengePrefixes is the contract for check names built at runtime,
// which the registry cannot list. A matching name is never challenged. The
// contract only ever restricts: it cannot make a name blockable or
// challengeable.
var neverChallengePrefixes = []string{
"outgoing_mail_",
"spam_",
"modsec_",
"email_auth_failure", // SMTP/IMAP authentication failures
"email_compromised", // confirmed compromised mail account
"email_credential", // credential leak
}
func neverChallengeDynamicName(check string) bool {
for _, prefix := range neverChallengePrefixes {
if strings.HasPrefix(check, prefix) {
return true
}
}
return false
}
// validateResponsePolicy returns the first contradictory policy in entries,
// naming the offending check.
func validateResponsePolicy(entries []CheckInfo) error {
for _, c := range entries {
p := c.Response
switch p.Block {
case BlockNever, BlockAlways, BlockWithCpanelLogins:
default:
return fmt.Errorf("check %q has unknown block eligibility %d", c.Name, p.Block)
}
if p.CriticalOnly && p.Block == BlockNever {
return fmt.Errorf("check %q is Critical-only but never blocks", c.Name)
}
if p.ChallengeFirst && (p.NeverChallenge || neverChallengeDynamicName(c.Name)) {
return fmt.Errorf("check %q is challenge-first and never-challenge at once", c.Name)
}
if err := admission.ValidPolicy(p.Evidence, p.Basis); err != nil {
return fmt.Errorf("check %q: %w", c.Name, err)
}
switch {
case !p.Attribution.Valid():
return fmt.Errorf("check %q accepts an unknown attribution kind", c.Name)
case (p.Evidence == admission.FamilyNone) != (p.Attribution == 0):
return fmt.Errorf("check %q must accept an attribution exactly when it has an evidence family", c.Name)
case !p.Attribution.Within(p.Evidence.Attributions()):
return fmt.Errorf("check %q accepts an attribution its evidence family cannot carry", c.Name)
}
if (p.Block != BlockNever || p.ChallengeFirst || p.Subnet != 0) && p.Evidence == admission.FamilyNone {
return fmt.Errorf("check %q can drive a response but has no evidence family", c.Name)
}
switch p.Subnet {
case 0:
case admission.EntryMailSubnet, admission.EntryASNCrawl:
if p.Block != BlockNever {
return fmt.Errorf("check %q is a subnet summary that blocks an address", c.Name)
}
default:
return fmt.Errorf("check %q names %s, which is no subnet response", c.Name, p.Subnet)
}
}
return nil
}
// responseOnce builds responseTable once from the registry; it is never read
// from disk.
var (
responseOnce sync.Once
responseTable map[string]ResponsePolicy
)
func loadResponseIndex() map[string]ResponsePolicy {
responseOnce.Do(func() {
idx := make(map[string]ResponsePolicy, len(checkRegistry))
for _, c := range checkRegistry {
if c.Response != (ResponsePolicy{}) {
idx[c.Name] = c.Response
}
}
responseTable = idx
})
return responseTable
}
// ResponsePolicyFor returns the registered response policy of a check after
// mapping a renamed producer to its current name. An unknown or unclassified
// check gets the zero policy, which never blocks and never challenges.
func ResponsePolicyFor(check string) ResponsePolicy {
return loadResponseIndex()[config.CanonicalCheckName(check)]
}
// CheckResponseEnabled applies detector response modes to retained findings
// as well as new ones. Passing this gate does not grant response authority;
// callers must still apply the registry policy and their action switches.
func CheckResponseEnabled(cfg *config.Config, check string) bool {
switch config.CanonicalCheckName(check) {
case "email_php_form_relay_observed":
return false
case "email_php_form_relay":
return cfg != nil && cfg.EmailProtection.PHPRelay.IPRecipientMode == "enforce"
default:
return true
}
}
// AdmissionPolicy is the registry projection automatic response admission
// reads. It maps a renamed producer to its current name first; ok is false
// for an unregistered check. A Critical-only check admits only Critical
// findings as evidence. Producers and the engine use this one lookup.
func AdmissionPolicy(check string) (string, admission.Policy, bool) {
name := config.CanonicalCheckName(check)
if _, registered := loadCorrelationIndex().classes[name]; !registered {
return "", admission.Policy{}, false
}
p := ResponsePolicyFor(name)
pol := admission.Policy{Family: p.Evidence, Basis: p.Basis, Attribution: p.Attribution}
if p.CriticalOnly {
pol.MinSeverity = admission.SeverityCritical
}
return name, pol, true
}
// AddressEvidence reports whether the structured address of a finding from
// check at sev is local attack evidence: the registry gives the check a local
// attack family and the finding meets its severity floor. Central intel and
// incidents ask this before acting on a finding's address, so a destination,
// a customer login, an advisory or an outside reputation match is never
// blocked for it.
func AddressEvidence(check string, sev alert.Severity) bool {
_, pol, ok := AdmissionPolicy(check)
if !ok || !pol.Family.LocalAttack() {
return false
}
return admissionSeverity(sev) >= pol.MinSeverity
}
// c2Listed reports whether ip is on c2_blocklist: where local processes
// connect, never a source of attacks, so its blocks are never escalation
// evidence, whatever finding asked for them.
func c2Listed(cfg *config.Config, ip string) bool {
addr := net.ParseIP(ip)
if cfg == nil || addr == nil {
return false
}
for _, listed := range cfg.C2Blocklist {
if l := net.ParseIP(listed); l != nil && l.Equal(addr) {
return true
}
}
return false
}
// IsRetiredThreatScoreFinding reports a finding of local_threat_score, which
// nothing produces any more: neither the retired score scan nor the
// database-session response, whose addresses are now reported, not blocked.
// Findings stored by older builds still carry it.
func IsRetiredThreatScoreFinding(f alert.Finding) bool {
return f.Check == "local_threat_score"
}
func admissionSeverity(s alert.Severity) admission.Severity {
switch s {
case alert.Critical:
return admission.SeverityCritical
case alert.High:
return admission.SeverityHigh
case alert.Warning:
return admission.SeverityWarning
}
return 0
}
package checks
import (
"path/filepath"
"strconv"
"strings"
"time"
)
// maxRestoreChainDepth bounds the full ancestry walk, including parents
// that are not termination targets. Incomplete ancestry refuses the response.
const maxRestoreChainDepth = 16
// RestoreWriterChain names the processes to stop after pid wrote back a file
// CSM had removed: the writer and its parents of the same account that keep it
// running, writer first. It returns nil unless the chain was started by the
// web server or runs detached under init. A process with a terminal, or one an
// SSH, cPanel or cron session started, belongs to a person or a scheduled job:
// CSM still removes the file but leaves the process to the operator.
func RestoreWriterChain(pid int, writtenAt time.Time) []int {
return restoreWriterChain(pid, writtenAt, true)
}
func restoreWriterChain(pid int, writtenAt time.Time, directWriter bool) []int {
uid := getProcessUID(strconv.Itoa(pid))
if uid == "" || uid == "0" {
return nil
}
var chain []int
collecting := true
for p, depth := pid, 0; depth < maxRestoreChainDepth; depth++ {
id := strconv.Itoa(p)
parent, tty, ok := procParentAndTTY(id)
exe := getProcessExe(id)
owner := getProcessUID(id)
if !ok || parent < 1 || tty != 0 || exe == "" || owner == "" || !processStartedBefore(id, writtenAt) {
return nil
}
if p == pid && owner != uid {
return nil
}
if isSessionExe(exe) {
return nil
}
if isWebServerExe(exe) {
// A web worker ends the set of targets, not the safety walk:
// PHP can itself have been launched from SSH or cron.
if p == pid && directWriter && isServerPHPExe(exe) {
chain = append(chain, p)
}
collecting = false
} else {
if isControlPlaneExe(exe) || collecting && owner != uid {
return nil
}
if collecting {
chain = append(chain, p)
}
}
if parent == 1 {
return chain
}
p = parent
}
return nil
}
// isSessionExe identifies administrative origins even when they run as the
// account or PHP appears between the session and the writer.
func isSessionExe(exe string) bool {
exe = strings.TrimSuffix(exe, " (deleted)")
if strings.HasPrefix(exe, "/usr/local/cpanel/") {
return true
}
switch filepath.Base(exe) {
case "sshd", "sshd-session", "sshd-auth", "cron", "crond", "anacron", "cpsrvd", "cpsrvd-dormant":
return true
}
return false
}
// isServerPHPExe matches the PHP binaries a web server runs requests in.
// The PHP command-line binary is not one: a script it runs is stoppable.
func isServerPHPExe(exe string) bool {
base := filepath.Base(strings.TrimSuffix(exe, " (deleted)"))
return strings.HasPrefix(base, "lsphp") || strings.HasPrefix(base, "php-fpm") || strings.HasPrefix(base, "php-cgi")
}
func isWebServerExe(exe string) bool {
if isServerPHPExe(exe) {
return true
}
switch filepath.Base(strings.TrimSuffix(exe, " (deleted)")) {
case "httpd", "apache2", "lshttpd", "litespeed", "openlitespeed", "lscgid", "nginx":
return true
}
return false
}
// isControlPlaneExe matches panel, web server and system service binaries,
// including the per-user processes cPanel and SSH run as the account.
func isControlPlaneExe(exe string) bool {
exe = strings.TrimSuffix(exe, " (deleted)")
if isSessionExe(exe) || isWebServerExe(exe) {
return true
}
switch filepath.Base(exe) {
case "systemd", "init", "dbus-daemon", "dbus-broker", "dbus-broker-launch",
"mysqld", "mariadbd", "postgres", "redis-server", "valkey-server", "memcached":
return true
}
// The PHP command-line binary ships under /opt/cpanel but runs account
// scripts; its ancestry decides, not its install path.
if isPHPCLIExe(exe) {
return false
}
for _, prefix := range []string{
"/usr/local/cpanel/", "/opt/cpanel/", "/usr/local/lsws/",
"/sbin/", "/usr/sbin/", "/usr/local/sbin/", "/usr/libexec/", "/usr/lib/", "/lib/",
"/opt/cloudlinux/", "/opt/imunify360/",
} {
if strings.HasPrefix(exe, prefix) {
return true
}
}
return false
}
// isPHPCLIExe matches php, php81 or php8.1, not the server binaries.
func isPHPCLIExe(exe string) bool {
base := filepath.Base(exe)
return strings.HasPrefix(base, "php") && strings.Trim(base[len("php"):], "0123456789.") == ""
}
// procParentAndTTY reads the parent PID and controlling terminal of pid.
func procParentAndTTY(pid string) (parent, tty int, ok bool) {
fields, ok := procStatFields(pid)
// After the command name: state, ppid, pgrp, session, tty_nr.
if !ok || len(fields) < 5 {
return 0, 0, false
}
parent, err := strconv.Atoi(fields[1])
if err != nil {
return 0, 0, false
}
tty, err = strconv.Atoi(fields[4])
if err != nil {
return 0, 0, false
}
return parent, tty, true
}
package checks
import "sort"
// rollingCandidatesAfter selects up to `limit` candidates from the path-sorted,
// de-duplicated `sorted` slice whose path sorts strictly AFTER `lastPath`,
// wrapping once to the start of the list if the end is reached. It returns the
// selected paths (in scan order), the new cursor value (the last selected path,
// to persist as last_path), and whether a wrap occurred during this selection.
//
// Contract:
// - `sorted` MUST be ascending sorted + de-duplicated (caller guarantees).
// - lastPath == "" starts from the beginning.
// - Never returns more than len(sorted) items (one full cycle max), even if
// limit exceeds len(sorted).
// - Wrap happens at most once; selection stops when it would revisit the
// first item of this selection (no infinite loop, no double-scan in a cycle).
// - limit <= 0 returns (nil, lastPath, false) -- no rolling this cycle.
// - Empty `sorted` returns (nil, lastPath, false).
// - newLast is the last path actually selected; if nothing was selected it is
// lastPath unchanged.
// - Robust to add/remove between cycles: selection is by VALUE comparison
// against lastPath (sort.SearchStrings for the first index strictly greater
// than lastPath), never an integer offset -- a deleted lastPath or inserted
// earlier file does not corrupt progress.
//
// Documented end-of-list rule for limit >= len(sorted):
//
// When starting from "" (or any cursor where the pre-wrap tail plus the
// wrapped head together would cover every element), at most len(sorted) items
// are returned. If the initial cursor is at the very beginning ("") and the
// list fits inside limit, the traversal reaches the end without issuing a
// wrap, so wrapped=false. If the cursor is mid-list and the remaining tail
// plus the needed head from the wrap together account for all elements,
// wrapped=true and the selection still caps at len(sorted) with no repeats.
func rollingCandidatesAfter(sorted []string, lastPath string, limit int) (selected []string, newLast string, wrapped bool) {
if limit <= 0 || len(sorted) == 0 {
return nil, lastPath, false
}
// Cap to one full cycle so we never return duplicates.
maxCount := limit
if maxCount > len(sorted) {
maxCount = len(sorted)
}
// Find the first index strictly greater than lastPath.
// sort.SearchStrings returns the smallest i where sorted[i] >= lastPath.
// If sorted[i] == lastPath we advance by 1 to get strictly greater.
start := sort.SearchStrings(sorted, lastPath)
if start < len(sorted) && sorted[start] == lastPath {
start++
}
selected = make([]string, 0, maxCount)
// Phase 1: collect from start to end of sorted.
i := start
for len(selected) < maxCount && i < len(sorted) {
selected = append(selected, sorted[i])
i++
}
// Phase 2: if we still need more and reached the end, wrap to the beginning.
// Stop before `start` so we never revisit a Phase-1 item (no repeats). This
// also covers a cursor at/after the last element (start >= len(sorted) with
// start > 0, e.g. lastPath == max or a single-element list): Phase 1 collects
// nothing and the wrap re-covers from the head.
if len(selected) < maxCount && i >= len(sorted) && start > 0 {
wrapped = true
for j := 0; len(selected) < maxCount && j < start; j++ {
selected = append(selected, sorted[j])
}
}
if len(selected) == 0 {
return nil, lastPath, false
}
return selected, selected[len(selected)-1], wrapped
}
package checks
import (
"context"
"fmt"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
)
// autoResponseActions counts every auto-response action fired, by
// action class. Registered lazily on first observation.
var (
autoResponseActions *metrics.CounterVec
autoResponseActionsOnce sync.Once
)
func observeAutoResponse(action string, n int) {
if n <= 0 {
return
}
autoResponseActionsOnce.Do(func() {
autoResponseActions = metrics.NewCounterVec(
"csm_auto_response_actions_total",
"Auto-response actions fired, by action. Labels: action (kill|quarantine|block). Incremented once per finding the auto-response subsystem produced in each tier run; a batch of four IPs blocked in one cycle contributes 4 to action=block.",
[]string{"action"},
)
metrics.MustRegister("csm_auto_response_actions_total", autoResponseActions)
})
autoResponseActions.With(action).Add(float64(n))
}
func observeFileResponseActions(action string, findings []alert.Finding) {
n := 0
for _, finding := range findings {
if finding.Check == "auto_response" {
n++
}
}
observeAutoResponse(action, n)
}
// checkDuration is the per-check latency histogram for /metrics.
// Labelled by check name and tier so scrapers can spot a single check
// regressing without scanning logs. Buckets span the observed range
// from ~millisecond process-list passes up to the heavy-check timeout
// ceiling.
var (
checkDuration *metrics.HistogramVec
checkDurationOnce sync.Once
)
var checkDurationBuckets = []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 180, 300, 600, 900}
func observeCheckDuration(name, tier string, d time.Duration) {
checkDurationOnce.Do(func() {
checkDuration = metrics.NewHistogramVec(
"csm_check_duration_seconds",
"Wall-clock time for each security check to complete. Label `name` is a check runner name; label `tier` is critical|deep|all. Use p95 across name to spot a single check regressing, and sum across name to track per-cycle pressure.",
[]string{"name", "tier"},
checkDurationBuckets,
)
metrics.MustRegister("csm_check_duration_seconds", checkDuration)
})
checkDuration.With(name, tier).Observe(d.Seconds())
}
// CheckFunc is the signature for all check functions.
// The context is cancelled when the check times out so goroutines can exit.
type CheckFunc func(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding
// splitDisabledChecks partitions checks by cfg.DisabledChecks. Finding names
// are the public vocabulary used by the settings UI and docs; runner names
// remain accepted for existing operator configs.
func splitDisabledChecks(cfg *config.Config, checks []namedCheck) (enabled, disabled []namedCheck) {
if cfg == nil || len(cfg.DisabledChecks) == 0 {
return checks, nil
}
disabledSet := make(map[string]struct{}, len(cfg.DisabledChecks))
knownRunners := make(map[string]struct{}, len(checks))
for _, nc := range checks {
knownRunners[nc.name] = struct{}{}
}
for _, name := range cfg.DisabledChecks {
name = strings.TrimSpace(name)
if name == "" {
continue
}
if _, ok := knownRunners[name]; ok {
disabledSet[name] = struct{}{}
continue
}
for _, runner := range runnerNamesForFinding(name) {
if _, ok := knownRunners[runner]; ok {
disabledSet[runner] = struct{}{}
}
}
}
ownerOff := disabledLogicalOwners(cfg)
for name := range disabledSet {
for _, owner := range physicalCheckLogicalOwners[name] {
if _, off := ownerOff[owner]; !off {
// An enabled hosted owner keeps the wrapper runnable; the
// check itself skips its disabled consumer.
delete(disabledSet, name)
break
}
}
}
if len(disabledSet) == 0 {
return checks, nil
}
enabled = make([]namedCheck, 0, len(checks))
disabledChecks := make([]namedCheck, 0, len(disabledSet))
for _, nc := range checks {
if _, skip := disabledSet[nc.name]; skip {
disabledChecks = append(disabledChecks, nc)
continue
}
enabled = append(enabled, nc)
}
return enabled, disabledChecks
}
func runnerNamesForFinding(finding string) []string {
return findingNameToRunnerNames[finding]
}
// DisabledCheckNames returns the sorted public finding-name vocabulary accepted
// by top-level disabled_checks for scheduled check execution. Runner IDs are
// also accepted by splitDisabledChecks for existing configs, but are not
// exposed in the UI.
func DisabledCheckNames() []string {
seen := make(map[string]struct{}, len(findingNameToRunnerNames))
for finding := range findingNameToRunnerNames {
seen[finding] = struct{}{}
}
for _, aliases := range logicalOwnerDisableAliases {
for _, alias := range aliases {
seen[alias] = struct{}{}
}
}
out := make([]string, 0, len(seen))
for finding := range seen {
info, ok := LookupCheck(finding)
if !ok || info.Internal {
continue
}
out = append(out, finding)
}
sort.Strings(out)
return out
}
// DisabledCheckConfigNames returns every value top-level disabled_checks
// accepts: every emitted finding name (including internal ones), compatibility
// runner ID and retired purge name. Retired values are accepted without
// disabling a live runner.
// It is broader than DisabledCheckNames (the UI vocabulary) so POST-side
// validation never rejects a value an existing operator config relies on.
func DisabledCheckConfigNames() []string {
seen := make(map[string]struct{}, len(findingNameToRunnerNames)+len(runnerFindingNames))
for finding := range findingNameToRunnerNames {
seen[finding] = struct{}{}
}
for runner, findings := range runnerFindingNames {
seen[runner] = struct{}{}
for _, finding := range findings {
seen[finding] = struct{}{}
}
}
// Logical-owner IDs and their public finding aliases are valid disable
// values; the coverage diagnostic stays rejected because it appears in no
// accepted vocabulary.
for _, aliases := range logicalOwnerDisableAliases {
for _, alias := range aliases {
seen[alias] = struct{}{}
}
}
out := make([]string, 0, len(seen))
for name := range seen {
out = append(out, name)
}
sort.Strings(out)
return out
}
var findingNameToRunnerNames = buildFindingNameToRunnerNames()
func buildFindingNameToRunnerNames() map[string][]string {
out := map[string][]string{}
for runner, findings := range runnerFindingNames {
for _, finding := range findings {
// This is purge ownership only. The retired score check's
// disable value must not disable external reputation or health.
if runner == "ip_reputation" && finding == "local_threat_score" {
continue
}
out[finding] = append(out[finding], runner)
}
}
return out
}
var runnerFindingNames = map[string][]string{
"admin_overlap": {"admin_cross_account_overlap"},
"credential_reuse": {"credential_reuse"},
"supply_chain": {"supply_chain_vuln"},
"af_alg_enforcement": {"af_alg_enforcement_corrected"},
"af_alg_socket_use": {"af_alg_socket_use"},
"api_auth_failures": {"api_auth_failure"},
"api_tokens": {"api_tokens"},
"cpanel_filemanager": {"cpanel_file_upload"},
"cpanel_logins": {"cpanel_login", "cpanel_multi_ip_login", "cpanel_password_purge"},
"crontabs": {"crond_change", "crontab_change", "suspicious_crontab"},
"database_dumps": {"database_dump"},
"db_content": {"db_content_scan_incomplete", "db_doorway_sitemap_routes", "db_hidden_link_injection", "db_hostname_keyed_option", "db_options_injection", "db_options_new_external_script", "db_options_plugin_notice_injection", "db_phantom_post_author", "db_post_injection", "db_post_volume_burst", "db_spam_taxonomy", "db_stored_cloak_logic", "db_stored_code_execution", "db_rogue_admin", "db_siteurl_foreign_host", "db_siteurl_hijack", "db_siteurl_invalid", "db_spam_injection", "db_suspicious_admin_email"},
"db_content_drupal": {"drupal_admin_injection", "drupal_content_injection", "drupal_settings_injection"},
"db_content_joomla": {"joomla_admin_injection", "joomla_content_injection", "joomla_extensions_injection"},
"db_content_magento": {"magento_admin_injection", "magento_content_injection", "magento_settings_injection"},
"db_content_opencart": {"opencart_admin_injection", "opencart_content_injection", "opencart_settings_injection"},
"db_content_prestashop": {"prestashop_admin_injection", "prestashop_content_injection", "prestashop_settings_injection"},
"db_objects": {"db_magic_token_user", "db_malicious_event", "db_malicious_function", "db_malicious_procedure", "db_malicious_trigger", "db_unexpected_event", "db_unexpected_function", "db_unexpected_procedure", "db_unexpected_trigger"},
"dns_connections": {"dns_connection"},
"dns_zones": {"dns_zone_change"},
"email_content": {"email_phishing_content"},
"email_forwarder_audit": {"email_pipe_forwarder", "email_suspicious_forwarder"},
"email_mail_filters": {"email_filter_blackhole", "email_filter_exfil", "email_filter_forwarder", "email_filter_pipe", "email_mail_filters"},
"email_weak_password": {"email_weak_password", "email_password_audit_incomplete"},
"exfiltration_paste": {"exfiltration_paste_site"},
"fake_kernel_threads": {"fake_kernel_thread"},
// new_php_in_languages and new_php_in_upgrade were emitted until a20c6f76;
// they stay here so a completed scan clears rows written by older versions.
"file_index": {"new_executable_in_config", "new_php_in_languages", "new_php_in_sensitive_dir", "new_php_in_sensitive_dir_clean", "new_php_in_upgrade", "new_php_in_uploads", "new_php_in_uploads_clean", "new_suspicious_php", "new_webshell_file", "obfuscated_php", "suspicious_php_content"},
"filesystem": {"backdoor_binary", "suid_binary", "suspicious_file"},
"firewall": {"firewall", "firewall_ports", "firewall_ipv6_unmanaged"},
"ftp_logins": {"ftp_bruteforce", "ftp_login", "ftp_login_after_bruteforce"},
"group_writable_php": {"group_writable_php"},
"health": {"csm_health"},
"htaccess": append([]string{"htaccess_handler_abuse", "htaccess_injection"}, htaccessDetectorNames()...),
"exposed_files": {"web_exposed_config_leak", "web_exposed_db_dump", "web_exposed_backup_archive", "web_exposed_source_backup", "web_exposed_phpinfo", "web_exposed_sample_sql"},
// local_threat_score came from the retired attack database score scan;
// it stays here so a completed critical scan clears rows older versions
// stored.
"ip_reputation": {"ip_reputation", "local_threat_score"},
"kernel_modules": {"kernel_module"},
"mail_per_account": {"mail_per_account"},
"mail_queue": {"mail_queue", "mail_queue_unavailable"},
"modsec_audit": {"waf_attack_blocked"},
"mysql_users": {"mysql_superuser"},
"nulled_plugins": {"nulled_plugin"},
"open_basedir": {"open_basedir"},
"outbound_connections": {"backdoor_port", "backdoor_port_outbound", "c2_connection"},
"outdated_plugins": {"outdated_plugins"},
"vulnerable_plugins": {"vulnerable_plugins"},
"vulnerable_timthumb": {"vulnerable_timthumb"},
"perf_error_logs": {"perf_error_logs"},
"perf_load": {"perf_load"},
"perf_memory": {"perf_memory"},
"perf_mysql_config": {"perf_mysql_config"},
"perf_php_handler": {"perf_php_handler"},
"perf_php_processes": {"perf_php_processes"},
"perf_redis_config": {"perf_redis_config"},
"perf_wp_config": {"perf_wp_config"},
"perf_wp_cron": {"perf_wp_cron"},
"perf_wp_loopback": {"perf_wp_loopback"},
"perf_wp_myisam": {"perf_wp_myisam"},
"perf_wp_transients": {"perf_wp_transients"},
"phishing": {"phishing_credential_log", "phishing_directory", "phishing_iframe", "phishing_kit_archive", "phishing_kit_archive_unscannable", "phishing_page", "phishing_php", "phishing_redirector"},
"yara_deep": {"yara_match_scheduled", "yara_scan_incomplete"},
"php_config_changes": {"php_config_change", "php_config_scan_incomplete"},
"php_content": {"obfuscated_php", "suspicious_php_content"},
"php_processes": {"php_suspicious_execution"},
"rpm_integrity": {"dpkg_integrity", "rpm_integrity"},
"shadow_changes": {"bulk_password_change", "root_password_change", "shadow_change"},
"ssh_keys": {"ssh_keys"},
"ssh_logins": {"ssh_login_unknown_ip"},
"sshd_config": {"sshd_config_change"},
"ssl_certs": {"ssl_cert_issued"},
"suspicious_processes": {"docroot_binary_exec", "suspicious_process"},
"detached_processes": {DetachedWebProcessCheck},
"symlink_attacks": {"symlink_attack"},
"uid0_accounts": {"uid0_account"},
"user_outbound": {"user_outbound_connection", "direct_smtp_egress", "bad_asn_outbound"},
"waf_status": {"modsec_disabled_vhost", "waf_bypass", "waf_detection_only", "waf_rules", "waf_rules_stale", "waf_status"},
"webmail_logins": {"webmail_bruteforce"},
"webshells": {"webshell", "world_writable_php"},
"whm_access": {"whm_account_action", "whm_password_change"},
"wp_bruteforce": {"wp_login_bruteforce", "wp_user_enumeration", "xmlrpc_abuse", "http_request_flood", "http_scanner_profile", "http_claimed_bot_unverified", "http_ua_spoof", "http_distributed_flood", "http_asn_crawl", "webshell_caller", "webshell_path_public"},
"wp_core": {"wp_core_integrity"},
"wp_plugin_inventory": {"wp_plugin_inventory_unverified"},
}
const (
logicalOwnerJSTaintDeep = "js_taint_deep"
logicalOwnerPHPTaintDeep = "php_taint_deep"
logicalOwnerReputationQuota = "reputation_quota_health"
logicalOwnerReputationFeedStale = "reputation_feed_health"
logicalOwnerSharedSendersStale = "shared_senders_health"
logicalOwnerWPCoreVerification = "wp_core_verification"
)
// logicalOwnerFindingNames maps a logical finding owner hosted inside another
// physical check to the finding names it owns. A logical owner is not a
// runnable check, so it must never be a runnerFindingNames key; the hosting
// check reports each owner's completion independently by calling
// markCheckIncomplete with the owner's exact name.
var logicalOwnerFindingNames = map[string][]string{
logicalOwnerJSTaintDeep: {"js_keylogger_dataflow", "js_taint_scan_incomplete"},
logicalOwnerPHPTaintDeep: {"php_remote_taint", "php_taint_scan_incomplete"},
logicalOwnerReputationQuota: {"reputation_quota_exhausted"},
logicalOwnerReputationFeedStale: {"threat_feed_stale"},
logicalOwnerSharedSendersStale: {"shared_senders_stale"},
logicalOwnerWPCoreVerification: {"wp_core_unverified"},
}
// logicalOwnerDisableAliases maps a logical owner to the disabled_checks
// values that disable it. A physical host alias is included only when
// disabling that host is also meant to disable the hosted consumer.
var logicalOwnerDisableAliases = map[string][]string{
logicalOwnerJSTaintDeep: {logicalOwnerJSTaintDeep, "js_keylogger_dataflow"},
logicalOwnerPHPTaintDeep: {logicalOwnerPHPTaintDeep, "php_remote_taint"},
logicalOwnerReputationQuota: {logicalOwnerReputationQuota, "reputation_quota_exhausted", "ip_reputation"},
logicalOwnerReputationFeedStale: {logicalOwnerReputationFeedStale, "threat_feed_stale", "ip_reputation"},
logicalOwnerSharedSendersStale: {logicalOwnerSharedSendersStale, "shared_senders_stale", "ip_reputation"},
logicalOwnerWPCoreVerification: {logicalOwnerWPCoreVerification, "wp_core_unverified", "wp_core"},
}
// physicalCheckLogicalOwners maps a runnable check to the logical owners it
// hosts. splitDisabledChecks keeps the physical wrapper runnable while any
// hosted owner is enabled, and the runner purges each owner by its own
// completion mark rather than the wrapper's.
var physicalCheckLogicalOwners = map[string][]string{
"yara_deep": {logicalOwnerJSTaintDeep, logicalOwnerPHPTaintDeep},
"ip_reputation": {logicalOwnerReputationQuota, logicalOwnerReputationFeedStale, logicalOwnerSharedSendersStale},
"wp_core": {logicalOwnerWPCoreVerification},
}
// disabledLogicalOwners returns the logical owners disabled by cfg.
func disabledLogicalOwners(cfg *config.Config) map[string]struct{} {
out := map[string]struct{}{}
if cfg == nil || len(cfg.DisabledChecks) == 0 {
return out
}
disabled := make(map[string]struct{}, len(cfg.DisabledChecks))
for _, name := range cfg.DisabledChecks {
if name = strings.TrimSpace(name); name != "" {
disabled[name] = struct{}{}
// A public finding alias also disables owners that explicitly
// inherit disablement from its physical check.
for _, runner := range runnerNamesForFinding(name) {
disabled[runner] = struct{}{}
}
}
}
for owner, aliases := range logicalOwnerDisableAliases {
for _, alias := range aliases {
if _, ok := disabled[alias]; ok {
out[owner] = struct{}{}
break
}
}
}
return out
}
// jsTaintDeepConsumerDisabled reports whether the scheduled JS taint consumer
// hosted by yara_deep is disabled.
func jsTaintDeepConsumerDisabled(cfg *config.Config) bool {
_, off := disabledLogicalOwners(cfg)[logicalOwnerJSTaintDeep]
return off
}
// phpTaintDeepConsumerDisabled reports whether the PHP taint consumer is
// disabled; the physical wrapper may still run for the other consumers.
func phpTaintDeepConsumerDisabled(cfg *config.Config) bool {
_, off := disabledLogicalOwners(cfg)[logicalOwnerPHPTaintDeep]
return off
}
// yaraDeepConsumerDisabled reports whether the YARA consumer itself is
// disabled; the physical wrapper may still run for the JS consumer.
func yaraDeepConsumerDisabled(cfg *config.Config) bool {
if cfg == nil {
return false
}
for _, name := range cfg.DisabledChecks {
name = strings.TrimSpace(name)
if name == "yara_deep" {
return true
}
for _, alias := range runnerFindingNames["yara_deep"] {
if name == alias {
return true
}
}
}
return false
}
// namedCheck pairs a check function with its name for timeout reporting.
type namedCheck struct {
name string
fn CheckFunc
}
// ForceAll forces all checks to run regardless of throttle (used by baseline).
var ForceAll bool
// Tier identifies which set of checks to run.
type Tier string
const (
TierCritical Tier = "critical" // Fast checks - processes, auth, network (~5 seconds)
TierDeep Tier = "deep" // Filesystem scans - webshells, htaccess, WP core (~90 seconds)
TierAll Tier = "all" // Both tiers
)
const checkTimeout = 5 * time.Minute
// heavyCheckTimeout applies to host-wide work over account web roots or their
// databases. On busy shared servers these legitimately run longer than the
// default 5-minute budget, so they get a wider window to avoid noisy
// check_timeout warnings while leaving fast checks aggressive.
const heavyCheckTimeout = 15 * time.Minute
// checkTimeoutDrainGrace bounds how long the runner waits for a check to hand
// back the findings it gathered before its deadline. A check that honors ctx
// returns within one file of the cancellation, and the slice is already paid
// for; a check wedged in a parser gets abandoned rather than holding the scan.
// A var so tests can shrink it without stalling on a deliberately stuck check.
var checkTimeoutDrainGrace = 5 * time.Second
// heavyChecks names the deep-tier checks that traverse every account's web
// roots or databases. Keep this list short and explicit; only checks that
// observably blow past 5 minutes on production hosts belong here.
var heavyChecks = map[string]bool{
"filesystem": true,
"webshells": true,
"htaccess": true,
"exposed_files": true,
"php_content": true,
"file_index": true,
"phishing": true,
"yara_deep": true,
"php_config_changes": true,
"db_content": true,
}
// timeoutFor returns the per-check execution budget. Heavy host-wide scans
// get heavyCheckTimeout, everything else gets checkTimeout.
// Indirected through timeoutForFunc so tests can shrink budgets
// without mutating the const.
func timeoutFor(name string) time.Duration {
return timeoutForFunc(name)
}
var timeoutForFunc = func(name string) time.Duration {
if heavyChecks[name] {
return heavyCheckTimeout
}
return checkTimeout
}
func criticalChecks() []namedCheck {
return []namedCheck{
{"fake_kernel_threads", CheckFakeKernelThreads},
{"suspicious_processes", CheckSuspiciousProcesses},
{"detached_processes", CheckDetachedWebProcesses},
{"php_processes", CheckPHPProcesses},
{"shadow_changes", CheckShadowChanges},
{"uid0_accounts", CheckUID0Accounts},
{"ssh_keys", CheckSSHKeys},
{"sshd_config", CheckSSHDConfig},
{"ssh_logins", CheckSSHLogins},
{"api_tokens", CheckAPITokens},
{"crontabs", CheckCrontabs},
{"outbound_connections", CheckOutboundConnections},
{"user_outbound", CheckOutboundUserConnections},
{"dns_connections", CheckDNSConnections},
{"whm_access", CheckWHMAccess},
{"cpanel_logins", CheckCpanelLogins},
{"cpanel_filemanager", CheckCpanelFileManager},
{"firewall", CheckFirewall},
{"mail_queue", CheckMailQueue},
{"mail_per_account", CheckMailPerAccount},
{"kernel_modules", CheckKernelModules},
{"af_alg_socket_use", CheckAFAlgSocketUsage},
{"af_alg_enforcement", CheckAFAlgEnforcement},
{"mysql_users", CheckMySQLUsers},
{"database_dumps", CheckDatabaseDumps},
{"exfiltration_paste", CheckOutboundPasteSites},
{"wp_bruteforce", CheckWPBruteForce},
{"ftp_logins", CheckFTPLogins},
{"webmail_logins", CheckWebmailLogins},
{"api_auth_failures", CheckAPIAuthFailures},
{"ip_reputation", CheckIPReputation},
{"modsec_audit", CheckModSecAuditLog},
{"health", CheckHealth},
{"perf_load", CheckLoadAverage},
{"perf_php_processes", CheckPHPProcessLoad},
{"perf_memory", CheckSwapAndOOM},
}
}
// deepChecks also runs in full while the realtime file monitor is attached.
// Close after write is the only fanotify event. A file renamed into place is
// reported only by the placement backend where one runs; a setuid bit set by
// chmod and a write outside the mark raise no event, and probe-based checks
// have no file event at all.
func deepChecks() []namedCheck {
return []namedCheck{
{"filesystem", CheckFilesystem},
{"webshells", CheckWebshells},
{"htaccess", CheckHtaccess},
{"exposed_files", CheckExposedFiles},
{"wp_core", CheckWPCore},
{"file_index", CheckFileIndex},
{"php_content", CheckPHPContent},
{"yara_deep", CheckYARADeep},
{"phishing", CheckPhishing},
{"nulled_plugins", CheckNulledPlugins},
{"rpm_integrity", CheckRPMIntegrity},
{"group_writable_php", CheckGroupWritablePHP},
{"open_basedir", CheckOpenBasedir},
{"symlink_attacks", CheckSymlinkAttacks},
{"php_config_changes", CheckPHPConfigChanges},
{"dns_zones", CheckDNSZoneChanges},
{"ssl_certs", CheckSSLCertIssuance},
{"waf_status", CheckWAFStatus},
{"db_content", CheckDatabaseContent},
{"db_content_drupal", CheckDrupalContent},
{"db_content_joomla", CheckJoomlaContent},
{"db_content_magento", CheckMagentoContent},
{"db_content_opencart", CheckOpenCartContent},
{"db_content_prestashop", CheckPrestaShopContent},
{"db_objects", CheckDatabaseObjects},
{"admin_overlap", CheckAdminEmailOverlap},
{"credential_reuse", CheckCredentialReuse},
{"email_content", CheckOutboundEmailContent},
{"outdated_plugins", CheckOutdatedPlugins},
{"wp_plugin_inventory", CheckWPPluginVerification},
{"vulnerable_plugins", CheckVulnerablePlugins},
{"vulnerable_timthumb", CheckVulnerableTimThumb},
{"supply_chain", CheckSupplyChain},
{"email_weak_password", CheckEmailPasswords},
{"email_forwarder_audit", CheckForwarders},
{"email_mail_filters", CheckMailFilters},
{"perf_php_handler", CheckPHPHandler},
{"perf_mysql_config", CheckMySQLConfig},
{"perf_redis_config", CheckRedisConfig},
{"perf_error_logs", CheckErrorLogBloat},
{"perf_wp_config", CheckWPConfig},
{"perf_wp_transients", CheckWPTransientBloat},
{"perf_wp_cron", CheckWPCron},
{"perf_wp_loopback", CheckWPLoopbackRequests},
{"perf_wp_myisam", CheckWPMyISAM},
}
}
// checkThrottleMin maps a check name to its minimum interval in minutes
// between executions. The runner reserves this before invoking the check
// function and stamps the slot only after the whole scan completes, so a
// timed-out or interrupted check keeps its slot and retries next cycle.
// Throttled checks that get skipped in a given cycle are NOT added to the
// per-scan purge list, so their previously-emitted findings stay in the
// latest set instead of being wiped every cycle. Without this gating in the
// runner, a deep scan that ran while the throttle window was still open
// would purge stale findings and merge nothing, hiding real issues until
// the next non-throttled cycle (or daemon restart).
var checkThrottleMin = map[string]int{
// Every candidate is confirmed with a live request to the customer's
// vhost, and the findings are posture that does not change between
// cycles, so this runs a few times a day rather than on each deep cycle.
"exposed_files": 360,
"perf_php_handler": 60,
"perf_mysql_config": 60,
"perf_redis_config": 60,
"perf_error_logs": 60,
"perf_wp_config": 60,
"perf_wp_transients": 60,
"perf_wp_cron": 60,
"perf_wp_loopback": 60,
"perf_wp_myisam": 60,
}
// LatestPurgeCheckNamesForTier returns every emitted finding name owned by a
// tier. The daemon uses this to replace a tier's current scan output without
// retaining stale findings from prior runs.
func LatestPurgeCheckNamesForTier(tier Tier) []string {
return withLogicalOwnerPurgeNames(checksForTier(tier))
}
// withLogicalOwnerPurgeNames extends the physical ownership expansion with the
// finding names of every logical owner hosted by a check in the set. Only the
// tier-level "everything this tier owns" views use it; the runner's per-cycle
// purge tracks each logical owner's completion individually instead.
func withLogicalOwnerPurgeNames(toScan []namedCheck) []string {
names := latestPurgeCheckNamesForChecks(toScan)
seen := make(map[string]struct{}, len(names))
for _, name := range names {
seen[name] = struct{}{}
}
for _, nc := range toScan {
for _, owner := range physicalCheckLogicalOwners[nc.name] {
for _, name := range logicalOwnerFindingNames[owner] {
if _, ok := seen[name]; !ok {
seen[name] = struct{}{}
names = append(names, name)
}
}
}
}
sort.Strings(names)
return names
}
// perRunFindingNames lists finding names that describe a single run's scan
// coverage rather than discovered state. They are purged whenever their
// owning check ran and returned, even when that run marked itself
// incomplete: a rolling scan on a large host never completes in one run, so
// gating these on completion would let every run's status finding accumulate
// forever. Discovered-state names (yara_match_scheduled) stay
// completion-gated so mid-cycle windows never wipe earlier windows' finds.
// An empty entry lets a stateful logical owner preserve its last finding
// without inventing a per-run status finding.
var perRunFindingNames = map[string][]string{
"email_weak_password": {"email_password_audit_incomplete"},
"yara_deep": {"yara_scan_incomplete"},
"db_content": {"db_content_scan_incomplete"},
logicalOwnerJSTaintDeep: {"js_taint_scan_incomplete"},
logicalOwnerPHPTaintDeep: {"php_taint_scan_incomplete"},
logicalOwnerReputationQuota: {},
logicalOwnerReputationFeedStale: {},
logicalOwnerSharedSendersStale: {},
logicalOwnerWPCoreVerification: {},
"php_config_changes": {"php_config_scan_incomplete"},
}
var latestVolatileCheckNames = []string{
"account_scan_truncated",
"auto_block",
"auto_response",
"auto_response_paused",
"challenge_route",
"check_panic",
"check_timeout",
}
// Keep merge completion and health publication in the same order. The
// store lock alone cannot prevent an older caller publishing after a newer
// merge once both have returned from the store.
var latestScanMergeMu sync.Mutex
// StoreLatestScanFindings replaces the latest findings owned by a scan, then
// rebuilds derived correlation findings from the merged current set. One-shot
// auto-response actions stay in history and alerts, not the active findings
// view.
func StoreLatestScanFindings(st *state.Store, purgeChecks []string, findings []alert.Finding) {
StoreLatestScanFindingsWithGaps(st, purgeChecks, findings, nil)
}
// StoreLatestScanFindingsWithGaps preserves the latest state for files a
// completed scan could not examine while replacing its covered state. gapPaths
// contains the lexical and resolved aliases captured when each gap occurred;
// the state store applies that frozen set under the same lock as the purge.
func StoreLatestScanFindingsWithGaps(st *state.Store, purgeChecks []string, findings []alert.Finding, gapPaths map[string]map[string]bool) {
StoreLatestScanFindingsWithCoverage(st, purgeChecks, findings, &state.ScanCoverage{PreservePaths: gapPaths})
}
// StoreLatestScanFindingsWithCoverage retires only completed checks or scopes
// and preserves current findings for file gaps in the same atomic operation.
func StoreLatestScanFindingsWithCoverage(st *state.Store, purgeChecks []string, findings []alert.Finding, coverage *state.ScanCoverage) {
if st == nil {
return
}
if len(purgeChecks) == 0 && len(findings) == 0 && (coverage == nil || len(coverage.CompletedScopes) == 0) {
return
}
// Cold detection runs commands; correlation under latestMu must only
// read cached platform roots.
platform.Detect()
latestScanMergeMu.Lock()
var healthChecks []string
for _, check := range purgeChecks {
switch check {
case "php_taint_scan_incomplete", "js_taint_scan_incomplete", "yara_scan_incomplete",
"email_password_audit_incomplete":
healthChecks = append(healthChecks, check)
case "db_content_scan_incomplete":
// Only the host summary is a per-run condition. A partial run
// cannot resolve the existing per-install multisite limit.
summary := alert.Finding{Check: check, DedupKey: dbContentHostCoverageDedupKey}
present := false
for _, f := range findings {
if f.Key() == summary.Key() {
present = true
break
}
}
if !present {
st.RearmDismissedFindings([]string{summary.Key()})
}
}
}
st.RearmAbsentDedupFindings(healthChecks, findings)
now := time.Now()
retired, kept := st.PurgeAndMergeFindingsDerivedWithCoverage(
latestPurgeWithVolatile(purgeChecks),
latestPersistentFindings(findings),
coverage,
DerivedCorrelationChecks(),
func(merged []alert.Finding) []alert.Finding {
res := defaultCorrelator.Correlate(merged, now)
for i := range res.Derived {
if res.Derived[i].Timestamp.IsZero() {
res.Derived[i].Timestamp = now
}
}
return res.Derived
},
)
st.ForgetCleared(clearedFindings(retired))
st.MarkKept(kept)
// Adding derived findings can evict source rows at the active-set cap.
// Count the final set, not the intermediate input to derivation.
unattributed := defaultCorrelator.Correlate(st.LatestFindings(), now).Unattributed
reporter := defaultUnattributedReporter
warnings := reporter.record(unattributed, true)
latestScanMergeMu.Unlock()
reporter.warnCounts(warnings)
}
// clearedFindings keeps the retired rows whose return should alert again.
// Scan coverage gaps flap between runs: one run that completes does not
// prove the next will, so they clear only after 24h unseen.
func clearedFindings(retired []alert.Finding) []alert.Finding {
var cleared []alert.Finding
for _, f := range retired {
if correlationReasonOf(f.Check) != reasonSelfHealth {
cleared = append(cleared, f)
}
}
return cleared
}
func latestPurgeWithVolatile(purgeChecks []string) []string {
out := make([]string, 0, len(purgeChecks)+len(latestVolatileCheckNames))
out = append(out, purgeChecks...)
out = append(out, latestVolatileCheckNames...)
return out
}
func latestPersistentFindings(findings []alert.Finding) []alert.Finding {
out := make([]alert.Finding, 0, len(findings))
for _, f := range findings {
if isLatestVolatileFinding(f.Check) || isLatestDerivedFinding(f.Check) {
continue
}
out = append(out, f)
}
return out
}
func isLatestVolatileFinding(check string) bool {
for _, name := range latestVolatileCheckNames {
if check == name {
return true
}
}
return false
}
func isLatestDerivedFinding(check string) bool {
return IsDerivedCorrelationCheck(check)
}
func checksForTier(tier Tier) []namedCheck {
switch tier {
case TierCritical:
return criticalChecks()
case TierDeep:
return deepChecks()
case TierAll:
return append(criticalChecks(), deepChecks()...)
default:
return nil
}
}
func latestPurgeCheckNamesForChecks(toScan []namedCheck) []string {
ran := make(map[string]struct{}, len(toScan))
for _, nc := range toScan {
ran[nc.name] = struct{}{}
}
seen := make(map[string]struct{})
for _, nc := range toScan {
seen[nc.name] = struct{}{}
for _, name := range runnerFindingNames[nc.name] {
// A finding name owned by several checks (php_content and
// file_index both emit obfuscated_php) is purged only once all
// its owners ran this cycle; otherwise one check completing
// wipes the other's live findings.
if !allFindingOwnersRan(name, ran) {
continue
}
seen[name] = struct{}{}
}
}
names := make([]string, 0, len(seen))
for name := range seen {
names = append(names, name)
}
sort.Strings(names)
return names
}
// allFindingOwnersRan reports whether every check that emits finding name
// is in ran.
func allFindingOwnersRan(name string, ran map[string]struct{}) bool {
for check, names := range runnerFindingNames {
if _, ok := ran[check]; ok {
continue
}
for _, n := range names {
if n == name {
return false
}
}
}
return true
}
// RunTier runs only the specified tier of checks. The second return value
// is the per-scan purge name list (emitted finding aliases owned by the
// checks that actually executed this cycle); pass it to
// StoreLatestScanFindings so throttled-out checks keep their prior
// findings.
//
// Passes the requested dry-run state into runParallel via a scoped
// parameter rather than the previous package-level toggle, so concurrent
// periodic scanners no longer race with a manual `csm check` invocation.
func RunTier(cfg *config.Config, store *state.Store, tier Tier) ([]alert.Finding, []string) {
return RunTierWithContext(context.Background(), cfg, store, tier)
}
// RunTierWithContext is RunTier with a caller-owned parent context. Daemon
// periodic scans pass their shutdown context here so an interrupted scan does
// not stall process exit.
func RunTierWithContext(ctx context.Context, cfg *config.Config, store *state.Store, tier Tier) ([]alert.Finding, []string) {
return runParallelWithContext(ctx, cfg, store, checksForTier(tier), string(tier), false)
}
// RunTierDryRunWithContext is RunTierWithContext with auto-response actions
// skipped. Control-socket scans use it to collect coverage gaps without
// enabling auto-response actions.
func RunTierDryRunWithContext(ctx context.Context, cfg *config.Config, store *state.Store, tier Tier) ([]alert.Finding, []string) {
return runParallelWithContext(ctx, cfg, store, checksForTier(tier), string(tier), true)
}
// RunAll runs critical checks always. Deep checks run if throttle allows or
// ForceAll is set. The second return value is the per-scan purge name list
// scoped to the checks that actually executed this cycle.
func RunAll(cfg *config.Config, store *state.Store) ([]alert.Finding, []string) {
return runAll(cfg, store, false)
}
// RunAllDryRun is the dry-run variant of RunAll for `csm baseline`.
func RunAllDryRun(cfg *config.Config, store *state.Store) ([]alert.Finding, []string) {
return runAll(cfg, store, true)
}
func runAll(cfg *config.Config, store *state.Store, dryRun bool) ([]alert.Finding, []string) {
toRun := criticalChecks()
if ForceAll || store.ShouldRunThrottled("deep_scan", cfg.Thresholds.DeepScanIntervalMin) {
toRun = append(toRun, deepChecks()...)
}
return runParallelWithContext(context.Background(), cfg, store, toRun, string(TierAll), dryRun)
}
// runParallel executes the supplied checks concurrently. It returns the
// emitted findings plus the per-scan purge name list. Throttled checks whose
// window has not elapsed stay out of the purge list so the previous cycle's
// findings persist; the same applies to checks that hit their per-check
// timeout, since a timed-out run produced no results to merge back.
// Disabled checks do not run, but their names stay in the purge list so
// disabling a check clears any findings it previously owned.
func runParallel(cfg *config.Config, store *state.Store, checks []namedCheck, tier string, dryRun bool) ([]alert.Finding, []string) {
return runParallelWithContext(context.Background(), cfg, store, checks, tier, dryRun)
}
// scansInFlight counts check runs in progress in this process: tier scans
// from any caller, account scans and scan jobs.
var scansInFlight atomic.Int64
type scanDryRunKey struct{}
// ScanInProgress reports whether any check run is in progress.
func ScanInProgress() bool {
return scansInFlight.Load() > 0
}
func runParallelWithContext(parent context.Context, cfg *config.Config, store *state.Store, checks []namedCheck, tier string, dryRun bool) ([]alert.Finding, []string) {
scansInFlight.Add(1)
defer scansInFlight.Add(-1)
if parent == nil {
parent = context.Background()
}
coverageGaps := coverageGapsFrom(parent)
// Clear a reused handle before work starts. An interrupted run must never
// expose the preceding run's path set as its own.
coverageGaps.replace(nil, nil, nil)
enabledChecks, disabledChecks := splitDisabledChecks(cfg, checks)
// Logical owners hosted by checks in this set: a disabled owner purges
// its names every cycle like a disabled check, while an enabled owner is
// purged only on its own completion mark, independent of its host's.
ownerOff := disabledLogicalOwners(cfg)
hostedOwners := make(map[string][]string)
disabledOwnerSet := make(map[string]struct{})
for _, nc := range checks {
for _, owner := range physicalCheckLogicalOwners[nc.name] {
if _, off := ownerOff[owner]; off {
disabledOwnerSet[owner] = struct{}{}
} else {
hostedOwners[nc.name] = append(hostedOwners[nc.name], owner)
}
}
}
for owner := range disabledOwnerSet {
if stateKey, ok := reputationHealthStateKeys[owner]; ok {
clearReputationHealthState(store, stateKey)
}
}
// Hide the caller's sink from nested runners and late check goroutines. The
// private path collector below is copied out only after this runner's workers
// completed inside their budgets.
scanCtx := context.WithValue(parent, coverageGapsContextKey{}, (*CoverageGaps)(nil))
scanCtx = context.WithValue(scanCtx, scanDryRunKey{}, dryRun)
ftpObservation := &ftpScanObservation{}
if !dryRun {
scanCtx = context.WithValue(scanCtx, ftpScanObservationKey{}, ftpObservation)
}
scanCtx, truncations := withAccountScanTruncationCollector(scanCtx)
scanCtx, coveragePaths := withCoveragePathCollector(scanCtx)
scanCtx, incompleteChecks := withIncompleteCheckCollector(scanCtx)
// One WordPress discovery per cycle, shared by every check that needs it.
scanCtx = withWPInstallCache(scanCtx)
var mu sync.Mutex
var findings []alert.Finding
var wg sync.WaitGroup
// completedChecks collects the checks whose function returned within
// budget (guarded by mu; workers run concurrently). Only completed
// checks join the purge list: a timed-out check produced no results,
// so purging its names would wipe every finding from earlier cycles
// while merging nothing back.
completedChecks := make([]namedCheck, 0, len(enabledChecks))
completedOwners := make([]string, 0)
completedThrottled := make([]string, 0)
recoveredPanicKeys := make([]string, 0)
// incompleteRan collects checks that returned within budget but marked
// themselves incomplete; their per-run status finding names still purge.
incompleteRan := make([]string, 0)
coverageGapPaths := make(map[string]map[string]bool)
completedScopeNames := make(map[string]map[string]bool)
addCoverageGapPaths := func(owner string) {
paths := coveragePaths.gapPaths(owner)
if len(paths) == 0 {
return
}
findingNames := []string{owner}
findingNames = append(findingNames, runnerFindingNames[owner]...)
findingNames = append(findingNames, logicalOwnerFindingNames[owner]...)
for _, findingName := range findingNames {
if coverageGapPaths[findingName] == nil {
coverageGapPaths[findingName] = make(map[string]bool, len(paths))
}
for path := range paths {
coverageGapPaths[findingName][path] = true
}
}
}
// Concurrency comes from the host-wide budget, so a periodic tier and an
// operator-triggered account scan cannot each run their own full set of
// checks on the same cores.
budget := scanBudgetFrom(scanCtx)
dispatches := checkDispatches.begin(len(enabledChecks), budget)
checkDispatches.observe(scanCtx, dispatches)
for i, nc := range enabledChecks {
wg.Add(1)
c := nc
task := dispatches[i]
// Check functions run against user filesystem content (unparsed PHP,
// crafted archives, foreign encodings), so contain both a panic in the
// runner and one inside the check execution. The inner recovery reports
// check_panic immediately with a stack trace.
obs.SafeGo("check-runner", task.wrap(func() {
defer wg.Done()
if !task.admit(scanCtx) {
task.withdraw(scanCtx)
return
}
if scanCtx.Err() != nil {
task.withdraw(scanCtx)
return
}
throttleReserved := false
if min, ok := checkThrottleMin[c.name]; ok && store != nil {
// Reserve atomically right before execution. The success
// stamp still happens only after the scan completes, so
// timeouts and interrupted scans can retry without allowing
// concurrent scans to start the same expensive check in the
// meantime.
if !store.ReserveThrottle(c.name, min) {
return
}
throttleReserved = true
}
// Run with cancellable context so timed-out checks stop
budget := timeoutFor(c.name)
ctx, cancel := context.WithTimeout(withCheckDispatch(scanCtx, task), budget)
start := time.Now()
execution := executeCheckAsync(ctx, "check-exec", func() []alert.Finding {
return c.fn(ctx, cfg, store)
})
defer execution.finishCaller()
select {
case outcome := <-execution.done:
execution.received()
if outcome.panicErr != "" {
cancel()
if throttleReserved {
store.ReleaseThrottle(c.name)
}
observeCheckDuration(c.name, tier, time.Since(start))
mu.Lock()
// The stack trace carries goroutine ids and addresses that
// differ on every run; a check that panics each cycle is
// one ongoing condition.
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "check_panic",
Message: fmt.Sprintf("Check '%s' stopped after an internal panic", c.name),
Details: outcome.panicErr,
DedupKey: "check:" + c.name,
Timestamp: time.Now(),
})
mu.Unlock()
return
}
results := outcome.findings
if ctx.Err() != nil {
execution.withdraw(ctx.Err())
cancel()
if throttleReserved {
store.ReleaseThrottle(c.name)
}
observeCheckDuration(c.name, tier, time.Since(start))
if scanCtx.Err() != nil {
return
}
mu.Lock()
// A heavy scan returns what it found before the budget ran
// out. The check stays out of completedChecks either way, so
// these findings merge into the latest set without
// authorizing a purge of anything the run never reached.
findings = append(findings, results...)
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "check_timeout",
Message: fmt.Sprintf("Check '%s' timed out after %s", c.name, budget),
Timestamp: time.Now(),
})
mu.Unlock()
return
}
cancel()
observeCheckDuration(c.name, tier, time.Since(start))
// An internal interval skip did not run an audit. Like a runner
// throttle skip, it cannot replace per-run coverage summaries
// or acknowledge recovery from an earlier failure.
if incompleteChecks.wasSkipped(c.name) {
if throttleReserved {
store.ReleaseThrottle(c.name)
}
return
}
mu.Lock()
recoveredPanicKeys = append(recoveredPanicKeys, (alert.Finding{Check: "check_panic", DedupKey: "check:" + c.name}).Key())
if scopes := coveragePaths.completedScopes(c.name); len(scopes) > 0 {
names := append([]string{c.name}, runnerFindingNames[c.name]...)
for _, name := range names {
completedScopeNames[name] = scopes
}
}
if !incompleteChecks.contains(c.name) {
completedChecks = append(completedChecks, c)
addCoverageGapPaths(c.name)
} else {
incompleteRan = append(incompleteRan, c.name)
}
// A hosted logical owner completes or stays partial on its own
// mark, honored only because the physical check returned in
// budget without a panic; a YARA-side failure must never purge
// JS findings the run did not re-emit, and vice versa.
for _, owner := range hostedOwners[c.name] {
if scopes := coveragePaths.completedScopes(owner); len(scopes) > 0 {
for _, name := range append([]string{owner}, logicalOwnerFindingNames[owner]...) {
completedScopeNames[name] = scopes
}
}
if !incompleteChecks.contains(owner) {
completedOwners = append(completedOwners, owner)
addCoverageGapPaths(owner)
} else {
incompleteRan = append(incompleteRan, owner)
}
}
if len(results) > 0 {
findings = append(findings, results...)
}
if throttleReserved && incompleteChecks.contains(c.name) {
store.ReleaseThrottle(c.name)
} else if throttleReserved {
completedThrottled = append(completedThrottled, c.name)
}
mu.Unlock()
case <-ctx.Done():
execution.withdraw(ctx.Err())
// The deadline and the check's own return race: a heavy scan
// that honors ctx hands back its partial findings just after
// the deadline fires, and select picks whichever is ready
// first. Wait a bounded moment for them instead of discarding
// detections the scan already paid for.
var drained []alert.Finding
grace := time.NewTimer(checkTimeoutDrainGrace)
select {
case outcome := <-execution.done:
// A panic carries no findings, and the deadline is what the
// operator needs to see either way.
if outcome.panicErr == "" {
drained = outcome.findings
}
case <-grace.C:
case <-scanCtx.Done():
// Shutdown has no partial findings to publish and must
// not wait for checks that ignore cancellation.
}
grace.Stop()
cancel()
if throttleReserved {
store.ReleaseThrottle(c.name)
}
observeCheckDuration(c.name, tier, time.Since(start))
// Distinguish a real per-check timeout from a daemon shutdown.
// On shutdown the parent scan context is cancelled, so abort
// quietly rather than flooding the scan with bogus check_timeout
// findings for work that was deliberately interrupted.
if scanCtx.Err() != nil {
return
}
mu.Lock()
// The check stays out of completedChecks, so these merge into
// the latest set without authorizing a purge of the range the
// run never reached.
findings = append(findings, drained...)
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "check_timeout",
Message: fmt.Sprintf("Check '%s' timed out after %s", c.name, budget),
Timestamp: time.Now(),
})
mu.Unlock()
}
}))
}
wg.Wait()
if scanCtx.Err() != nil {
for _, name := range completedThrottled {
store.ReleaseThrottle(name)
}
// The scan did not complete, so keep the caller from replacing the
// last completed scan state with partial findings or an empty purge.
return nil, nil
}
findings = append(findings, ftpObservation.complete(scanCtx, cfg, store)...)
for _, name := range completedThrottled {
store.MarkThrottledRan(name)
}
if store != nil {
store.RearmFindings(recoveredPanicKeys)
}
purgeChecks := make([]namedCheck, 0, len(disabledChecks)+len(completedChecks))
purgeChecks = append(purgeChecks, disabledChecks...)
purgeChecks = append(purgeChecks, completedChecks...)
now := time.Now()
findings = append(findings, truncations.findings(now)...)
for i := range findings {
if findings[i].Timestamp.IsZero() {
findings[i].Timestamp = now
}
}
// Cross-account correlation reads cached platform roots, so detection
// runs first; malware findings name a disabled ModSecurity scope.
if len(findings) > 0 {
annotateModsecDisabledMalware(platform.Detect(), findings)
}
correlated := CorrelateBatchFindings(findings)
for i := range correlated.Derived {
if correlated.Derived[i].Timestamp.IsZero() {
correlated.Derived[i].Timestamp = now
}
}
findings = append(findings, correlated.Derived...)
ReportUnattributedCorrelation(correlated.Unattributed)
// Auto-response: skip when the caller requested a dry run
// (check/baseline commands).
if !dryRun {
// Challenge routing and hard-blocking are resolved together, in order,
// so an eligible IP lands on the challenge list before the block stage
// checks membership. The block actions are appended in their own slot
// below (after the kill/quarantine/htaccess actions) to keep the
// emitted ordering stable; no intervening stage acts on action findings.
challengeActions, blockActions := ChallengeThenBlock(cfg, findings)
for i := range challengeActions {
if challengeActions[i].Timestamp.IsZero() {
challengeActions[i].Timestamp = now
}
}
findings = append(findings, challengeActions...)
killActions := AutoKillProcesses(parent, cfg, findings)
for i := range killActions {
if killActions[i].Timestamp.IsZero() {
killActions[i].Timestamp = now
}
}
observeAutoResponse("kill", len(killActions))
findings = append(findings, killActions...)
quarantineActions := AutoQuarantineFiles(cfg, findings)
for i := range quarantineActions {
if quarantineActions[i].Timestamp.IsZero() {
quarantineActions[i].Timestamp = now
}
}
observeFileResponseActions("quarantine", quarantineActions)
findings = append(findings, quarantineActions...)
htaccessActions := AutoCleanHtaccess(cfg, findings)
for i := range htaccessActions {
if htaccessActions[i].Timestamp.IsZero() {
htaccessActions[i].Timestamp = now
}
}
observeFileResponseActions("htaccess_clean", htaccessActions)
findings = append(findings, htaccessActions...)
vpatchActions := AutoVirtualPatchExposedFiles(cfg, findings)
for i := range vpatchActions {
if vpatchActions[i].Timestamp.IsZero() {
vpatchActions[i].Timestamp = now
}
}
observeAutoResponse("virtual_patch_exposed", len(vpatchActions))
findings = append(findings, vpatchActions...)
for i := range blockActions {
if blockActions[i].Timestamp.IsZero() {
blockActions[i].Timestamp = now
}
}
observeAutoResponse("block", len(blockActions))
findings = append(findings, blockActions...)
}
purgeNames := latestPurgeCheckNamesForChecks(purgeChecks)
for _, owner := range completedOwners {
purgeNames = append(purgeNames, logicalOwnerFindingNames[owner]...)
}
for owner := range disabledOwnerSet {
purgeNames = append(purgeNames, logicalOwnerFindingNames[owner]...)
}
// Selected checks that never completed (including timeouts, panics and
// throttle skips) did not examine their prior findings. Reuse the known
// completion sets to protect that state without authorizing any purge.
uncompletedOwners := make(map[string]bool)
for _, check := range enabledChecks {
uncompletedOwners[check.name] = true
for _, owner := range hostedOwners[check.name] {
uncompletedOwners[owner] = true
}
}
for _, check := range completedChecks {
delete(uncompletedOwners, check.name)
}
for _, owner := range completedOwners {
delete(uncompletedOwners, owner)
}
incompleteFindingNames := make(map[string]bool)
for owner := range uncompletedOwners {
names := append([]string{owner}, runnerFindingNames[owner]...)
names = append(names, logicalOwnerFindingNames[owner]...)
for _, name := range names {
incompleteFindingNames[name] = true
}
}
// Only checks that returned replace their per-run coverage summaries.
for _, owner := range incompleteRan {
for _, name := range perRunFindingNames[owner] {
delete(incompleteFindingNames, name)
}
}
coverageGaps.replace(coverageGapPaths, completedScopeNames, incompleteFindingNames)
return findings, mergePerRunPurgeNames(purgeNames, incompleteRan)
}
// mergePerRunPurgeNames preserves the purge list's sorted, duplicate-free
// contract. incompleteRan is populated by concurrent workers, so appending it
// directly made the returned order depend on goroutine completion order.
func mergePerRunPurgeNames(purgeNames, incompleteRan []string) []string {
seen := make(map[string]struct{}, len(purgeNames)+len(incompleteRan))
for _, name := range purgeNames {
seen[name] = struct{}{}
}
for _, owner := range incompleteRan {
for _, name := range perRunFindingNames[owner] {
seen[name] = struct{}{}
}
}
merged := make([]string, 0, len(seen))
for name := range seen {
merged = append(merged, name)
}
sort.Strings(merged)
return merged
}
package checks
import (
"context"
"errors"
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
type scanBatchMonitor struct {
mu sync.Mutex
batches map[*scanBatch]struct{}
losses *queuehealth.Tracker
}
type scanBatch struct {
monitor *scanBatchMonitor
tasks []*scanBatchTask
pending map[*scanBatchTask]struct{}
parallel int
progress time.Time
}
// Mutable batch and task state is guarded by the monitor mutex.
type scanBatchTask struct {
batch *scanBatch
started, deadline time.Time
failed bool
}
func newScanBatchMonitor() *scanBatchMonitor {
return &scanBatchMonitor{batches: make(map[*scanBatch]struct{}), losses: queuehealth.New(0, time.Minute)}
}
func (m *scanBatchMonitor) begin(count, parallel int) *scanBatch {
b := &scanBatch{monitor: m, tasks: make([]*scanBatchTask, count), pending: make(map[*scanBatchTask]struct{}, count), parallel: parallel, progress: time.Now()}
for i := range b.tasks {
b.tasks[i] = &scanBatchTask{batch: b}
b.pending[b.tasks[i]] = struct{}{}
}
if count > 0 {
m.mu.Lock()
m.batches[b] = struct{}{}
m.mu.Unlock()
}
return b
}
func (t *scanBatchTask) admit() {
m := t.batch.monitor
m.mu.Lock()
defer m.mu.Unlock()
t.started = time.Now()
t.deadline = t.started.Add(time.Minute)
t.batch.progress = t.started
}
func (t *scanBatchTask) executing(ctx context.Context, budget time.Duration) {
deadline := time.Now().Add(budget)
if parent, ok := ctx.Deadline(); ok && parent.Before(deadline) {
deadline = parent
}
m := t.batch.monitor
m.mu.Lock()
t.deadline = deadline
m.mu.Unlock()
}
func (t *scanBatchTask) progress() {
m := t.batch.monitor
m.mu.Lock()
t.deadline = time.Now().Add(time.Minute)
m.mu.Unlock()
}
func (t *scanBatchTask) failLocked() {
if !t.failed {
t.failed = true
t.batch.monitor.losses.Lose(time.Now(), 1)
}
}
func (t *scanBatchTask) fail() {
m := t.batch.monitor
m.mu.Lock()
t.failLocked()
m.mu.Unlock()
}
func (t *scanBatchTask) withdraw(err error) {
if errors.Is(err, context.DeadlineExceeded) {
t.fail()
}
}
func (t *scanBatchTask) finish() {
m := t.batch.monitor
m.mu.Lock()
defer m.mu.Unlock()
delete(t.batch.pending, t)
t.batch.progress = time.Now()
if len(t.batch.pending) == 0 {
delete(m.batches, t.batch)
}
}
// The callback reports failures and deadline withdrawal where work is
// abandoned. A later deadline cannot undo a successfully completed callback.
func (t *scanBatchTask) run(ctx context.Context, budget time.Duration, fn func()) {
completed := false
defer func() {
if !completed {
t.fail()
}
t.finish()
}()
t.executing(ctx, budget)
fn()
completed = true
}
// abandon runs after dispatch has stopped. Running workers retain their own
// tasks, including when a caller has canceled but an operation ignores it.
func (b *scanBatch) abandon(ctx context.Context) {
err := ctx.Err()
m := b.monitor
m.mu.Lock()
defer m.mu.Unlock()
for task := range b.pending {
if !task.started.IsZero() {
continue
}
if !errors.Is(err, context.Canceled) {
task.failLocked()
}
delete(b.pending, task)
}
if len(b.pending) == 0 {
delete(m.batches, b)
}
}
func (m *scanBatchMonitor) snapshot(now time.Time) queuehealth.Status {
m.mu.Lock()
defer m.mu.Unlock()
s := m.losses.Snapshot(now)
s.CapacityUnavailable = true
s.LagBasis = "consumer_progress"
var waitingLate, runningLate bool
for b := range m.batches {
waiting, running := 0, 0
for task := range b.pending {
if task.started.IsZero() {
waiting++
} else {
running++
s.ProcessingSeconds = max(s.ProcessingSeconds, now.Sub(task.started).Seconds())
runningLate = runningLate || !now.Before(task.deadline)
}
}
s.Depth += waiting
s.InFlight += running
if waiting > 0 {
lag := now.Sub(b.progress)
s.LagSeconds = max(s.LagSeconds, lag.Seconds())
// Filling a finite batch is expected while every worker is busy.
// Its own deadlines bound that work; a free slot needs progress.
waitingLate = waitingLate || (running < b.parallel && lag >= time.Minute)
}
}
switch {
case runningLate:
s.Reason = "processing_lag"
case waitingLate:
s.Reason = "backlog_lag"
}
if s.Reason != "" {
s.Status = "degraded"
}
return s
}
package checks
import (
"context"
"runtime"
"sync/atomic"
)
const (
// minScanParallelism keeps a single-core host making progress across the
// check set instead of serialising a scan behind one slow filesystem walk.
minScanParallelism = 2
// maxScanParallelism is the ceiling every scan path used to apply
// unconditionally. Checks walk account trees and hash file content, so
// past this point they starve each other on I/O rather than finishing
// sooner, and the Web UI stops answering.
maxScanParallelism = 5
)
// hostScanBudget bounds how many CPU-heavy checks run at once across the whole
// daemon. The tier runner and the per-account scan used to carry separate
// fixed limits, so an operator-triggered account scan during a periodic tier
// ran nine checks together on a four-core host.
var hostScanBudget = newScanBudget(scanParallelismFor(runtime.NumCPU()))
type scanBudgetKey struct{}
// withScanBudget scopes a scan to its own budget. Production scans carry none
// and share the host budget; tests use this so one test's checks cannot hold
// slots another test is waiting for.
func withScanBudget(ctx context.Context, slots int) context.Context {
return context.WithValue(ctx, scanBudgetKey{}, newScanBudget(slots))
}
// scanBudgetFrom returns the budget this scan draws from.
func scanBudgetFrom(ctx context.Context) *scanBudget {
if ctx != nil {
if budget, ok := ctx.Value(scanBudgetKey{}).(*scanBudget); ok && budget != nil {
return budget
}
}
return hostScanBudget
}
// scanParallelismFor sizes the budget from the host's core count, bounded at
// both ends.
func scanParallelismFor(cpus int) int {
if cpus < minScanParallelism {
return minScanParallelism
}
if cpus > maxScanParallelism {
return maxScanParallelism
}
return cpus
}
// scanBudget is a counting semaphore. Callers acquire one slot per check and
// must not hold one while acquiring another: nothing in the scan paths nests,
// and a nested acquisition would deadlock against a full budget.
type scanBudget struct {
slots chan struct{}
}
func newScanBudget(slots int) *scanBudget {
if slots < 1 {
slots = 1
}
return &scanBudget{slots: make(chan struct{}, slots)}
}
// acquire blocks until a slot is free or ctx is done. The runner and its
// asynchronous execution share ownership so cancellation cannot free a slot
// while a check that ignores its context is still consuming resources.
func (b *scanBudget) acquire(ctx context.Context) *scanSlot {
select {
case b.slots <- struct{}{}:
slot := &scanSlot{budget: b}
slot.owners.Store(1)
return slot
case <-ctx.Done():
return nil
}
}
func (b *scanBudget) size() int { return cap(b.slots) }
func (b *scanBudget) hasCapacity() bool { return len(b.slots) < b.size() }
type scanSlot struct {
budget *scanBudget
owners atomic.Int32
}
func (s *scanSlot) retain() {
if s != nil {
s.owners.Add(1)
}
}
func (s *scanSlot) release() {
if s != nil && s.owners.Add(-1) == 0 {
<-s.budget.slots
}
}
package checks
import (
"context"
"github.com/pidginhost/csm/internal/config"
)
const (
fullScanMaxFileMBDefault = 16
fullScanMaxFileMBMax = 4096
)
// scanForceContent reports whether the current scan should bypass the
// clean-file content cache (phpcontentcache.json). True only when ctx carries
// AccountScanOptions with ForceContent=true, i.e. an explicit full-scan audit.
// Normal scheduled scans and any context without options return false.
func scanForceContent(ctx context.Context) bool {
opts, ok := ScanOptionsFromContext(ctx)
return ok && opts.ForceContent
}
// scanForceFileIndex reports whether the current scan is a file-index audit
// run. True only when ctx carries AccountScanOptions with ForceFileIndex=true.
// In audit mode CheckFileIndex enumerates only the in-scope account, bypasses
// the directory mtime cache, and writes none of the three live state files
// (fileindex.current, fileindex.previous, dircache.json).
func scanForceFileIndex(ctx context.Context) bool {
opts, ok := ScanOptionsFromContext(ctx)
return ok && opts.ForceFileIndex
}
// scanRespectsIgnores reports whether the current scan should honour
// cfg.Suppressions.IgnorePaths. When ctx carries AccountScanOptions with
// RespectIgnores=false (i.e. an explicit full-scan / audit request), the caller
// wants full coverage and ignore_paths is bypassed. Normal scheduled scans and
// any call without options carry RespectIgnores=true (the safe default).
func scanRespectsIgnores(ctx context.Context, _ *config.Config) bool {
if opts, ok := ScanOptionsFromContext(ctx); ok {
return opts.RespectIgnores
}
return true
}
// scanMaxFileBytes returns the per-file byte limit for a full-scan context.
// When ctx carries AccountScanOptions, the options MaxFileBytes value is
// returned so an oversized file can be skipped with a warning. Returns 0 for
// all other contexts so normal scheduled scans are completely unaffected.
func scanMaxFileBytes(ctx context.Context) int64 {
opts, ok := ScanOptionsFromContext(ctx)
if !ok {
return 0
}
return opts.MaxFileBytes
}
// accountScanMaxFiles returns the effective file cap for the current scan. When
// ctx carries AccountScanOptions (i.e. the caller is RunAccountScanWithOptions),
// the options MaxFiles value wins so a full scan (MaxFiles=0) is uncapped.
// Otherwise it falls back to the config-derived value so normal scheduled scans
// are unaffected.
func accountScanMaxFiles(ctx context.Context, cfg *config.Config) int {
if opts, ok := ScanOptionsFromContext(ctx); ok {
return opts.MaxFiles
}
return effectiveAccountScanMaxFiles(cfg)
}
// AccountScanOptions controls how RunAccountScanWithOptions enumerates and
// content-scans an account. The zero value is NOT the default: MaxFiles 0 means
// uncapped. Callers use DefaultAccountScanOptions for normal behaviour.
type AccountScanOptions struct {
MaxFiles int // 0 = uncapped path ranking
ForceContent bool // true = bypass clean-file content caches
ForceFileIndex bool // true = bypass file-index dir mtime cache, do not write live index
RespectIgnores bool // false = also scan suppressions.ignore_paths
MaxFileBytes int64 // 0 = use each check's existing per-file limit
}
// DefaultAccountScanOptions returns the options that reproduce today's
// RunAccountScan behaviour. All callers that want the existing cap and cache
// semantics should use this rather than constructing AccountScanOptions directly.
func DefaultAccountScanOptions(cfg *config.Config) AccountScanOptions {
return AccountScanOptions{
MaxFiles: effectiveAccountScanMaxFiles(cfg),
ForceContent: false,
ForceFileIndex: false,
RespectIgnores: true,
MaxFileBytes: 0,
}
}
type scanOptionsKey struct{}
// ContextWithScanOptions attaches opts to ctx so that helpers called during a
// scan can read the active options without threading a parameter through every
// call site. Use ScanOptionsFromContext to retrieve them.
func ContextWithScanOptions(ctx context.Context, opts AccountScanOptions) context.Context {
if ctx == nil {
ctx = context.Background()
}
return context.WithValue(ctx, scanOptionsKey{}, opts)
}
// ScanOptionsFromContext retrieves the AccountScanOptions stored by
// ContextWithScanOptions. ok is false when the context carries no options,
// which callers should treat as "use defaults".
func ScanOptionsFromContext(ctx context.Context) (AccountScanOptions, bool) {
if ctx == nil {
return AccountScanOptions{}, false
}
opts, ok := ctx.Value(scanOptionsKey{}).(AccountScanOptions)
return opts, ok
}
// FullScanOptions builds the canonical option set for an uncapped full-scan
// audit job: no file cap, force content + file-index (bypass the clean-file and
// directory-mtime caches), and the configured per-file byte ceiling.
// respectIgnores is the only caller-chosen knob. Centralised here so the control
// handler and the WebUI enqueue path cannot drift on this security-relevant set.
func FullScanOptions(cfg *config.Config, respectIgnores bool) AccountScanOptions {
return AccountScanOptions{
MaxFiles: 0,
ForceContent: true,
ForceFileIndex: true,
RespectIgnores: respectIgnores,
MaxFileBytes: FullScanMaxFileBytes(cfg),
}
}
// FullScanMaxFileBytes converts cfg.Thresholds.FullScanMaxFileMB to a byte
// limit for per-file content reads during a full-scan job. A configured value
// of 0, negative, or above the validated maximum falls back to 16 MiB so the
// caller never gets an unconstrained or overflowed limit from a bad field.
func FullScanMaxFileBytes(cfg *config.Config) int64 {
mb := fullScanMaxFileMBDefault
if cfg != nil && cfg.Thresholds.FullScanMaxFileMB > 0 && cfg.Thresholds.FullScanMaxFileMB <= fullScanMaxFileMBMax {
mb = cfg.Thresholds.FullScanMaxFileMB
}
return int64(mb) * 1024 * 1024
}
package checks
import (
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
)
// scanRecord is a record a scan counted toward a finding: its file stream,
// cursor and offset, and the time its log wrote it. A scan that re-reads a
// window names a finding by its newest counted record, and by that record's
// own time: a re-read of unchanged input then names the same evidence,
// whatever the scan's clock says. It keeps the cursor, not the line, so a
// scan holding one record per address holds no request text.
type scanRecord struct {
stream, cursor string
offset int64
at time.Time
// kind is how the record's parser read the finding's address, when the
// scan knows it only per record; unknown leaves the finding's kind.
kind admission.Attribution
}
// scanned is the scan record of r written at at; zero without a log time.
func scanned(r syslogRecord, at time.Time) scanRecord {
if at.IsZero() {
return scanRecord{}
}
return scanRecord{stream: r.stream, cursor: alert.RecordCursor(r.offset, r.text), offset: r.offset, at: at}
}
// newer orders records by log time, then by file and offset, so the newest
// record is the same whichever order the scan read its files in.
func (r scanRecord) newer(o scanRecord) bool {
switch {
case !r.at.Equal(o.at):
return r.at.After(o.at)
case r.stream != o.stream:
return r.stream > o.stream
default:
return r.offset > o.offset
}
}
// scanRecordFutureSkew is how far past the scan's clock a record's log time
// may be. Admission refuses a root dated in the future.
const scanRecordFutureSkew = time.Second
// keepNewest keeps r under key when it is the newest record seen for it. A
// record without a file stream (one still being written) is never kept: its
// reference changes when its line ends. A record without a log time, or dated
// after the scan at now, is never kept either: a time a client wrote into the
// line cannot pin the observation.
func keepNewest(newest map[string]scanRecord, key string, r scanRecord, now time.Time) {
if r.stream == "" {
return
}
if r.at.IsZero() || r.at.After(now.Add(scanRecordFutureSkew)) {
return
}
if cur, ok := newest[key]; !ok || r.newer(cur) {
newest[key] = r
}
}
// observation names r as every reader of its file names it; zero when the
// file has no identity.
func (r scanRecord) observation(producer admission.ProducerID) alert.Observation {
if r.stream == "" || r.at.IsZero() {
return alert.Observation{}
}
return alert.Observation{Producer: string(producer), Stream: r.stream, Cursor: r.cursor, ObservedAt: r.at.UTC()}
}
// stamp names r as the observation of f, with r's kind when it has one; f
// keeps no observation when the scan kept no record for it.
func (r scanRecord) stamp(f *alert.Finding, producer admission.ProducerID) {
if o := r.observation(producer); o != (alert.Observation{}) {
f.Observation = o
if r.kind.Valid() {
f.Attribution = r.kind
}
}
}
package checks
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"sync"
"time"
"github.com/pidginhost/csm/internal/state"
)
// selfWriteTTL bounds how long a CSM-performed write to a sensitive path
// suppresses the sensitive-file detectors. Short enough that an independent
// tamper layered on later is still caught by the next scan.
const selfWriteTTL = 15 * time.Minute
const durableSelfWriteVersion = 1
var (
selfWriteMu sync.Mutex
selfWrites = map[string]selfWriteRecord{}
selfWriteNow = time.Now // overridable in tests
// selfWriteStore persists self-write content and file identity across
// restarts. Nil in unit tests and one-shot CLI runs, where the in-memory
// ledger is enough.
selfWriteStore *state.Store
)
// selfWriteKey namespaces the durable record. The leading underscore marks it
// as housekeeping so the state sweeper never evicts it; the record is cleared
// explicitly by forgetSelfWrites, or superseded by the next write to the path.
func selfWriteKey(path string) string { return "_selfwrite:" + path }
// SetSelfWriteStore gives the self-write ledger somewhere durable to record
// what CSM wrote. Without it, a daemon restart -- or a crontab the cPanel
// wrapper reformats after CSM hands it over -- makes CSM's own write look like
// a third-party change to the sensitive-file detectors.
func SetSelfWriteStore(st *state.Store) {
selfWriteMu.Lock()
defer selfWriteMu.Unlock()
selfWriteStore = st
}
type selfWriteRecord struct {
hash string
identity *selfWriteFileIdentity
expires time.Time
}
type selfWriteFileIdentity struct {
Device uint64 `json:"device"`
Inode uint64 `json:"inode"`
ChangeSec int64 `json:"change_sec"`
ChangeNsec int64 `json:"change_nsec"`
}
type durableSelfWriteRecord struct {
Version int `json:"version"`
Hash string `json:"hash"`
Identity selfWriteFileIdentity `json:"identity"`
}
// RecordSelfWrite registers that CSM remediation just wrote content to a
// sensitive watched file. A current file is also recorded durably with its
// identity; a provisional record made before the write stays memory-only.
func RecordSelfWrite(path string, content []byte) {
sum := sha256.Sum256(content)
hash := hex.EncodeToString(sum[:])
identity, hasIdentity := currentSelfWriteIdentity(path, content)
selfWriteMu.Lock()
defer selfWriteMu.Unlock()
now := selfWriteNow()
pruneExpiredSelfWritesLocked(now)
rec := selfWriteRecord{
hash: hash,
expires: now.Add(selfWriteTTL),
}
if hasIdentity {
rec.identity = &identity
}
selfWrites[path] = rec
if selfWriteStore != nil && hasIdentity {
// Marshal of a fixed struct of scalars cannot fail; if it somehow did,
// losing the durable record costs a redundant finding, not safety, so
// remediation must not die here.
raw, err := json.Marshal(durableSelfWriteRecord{
Version: durableSelfWriteVersion,
Hash: hash,
Identity: identity,
})
if err != nil {
fmt.Fprintf(os.Stderr, "self-write: encode %s: %v\n", path, err)
return
}
if err := selfWriteStore.SetRawAndSave(selfWriteKey(path), string(raw)); err != nil {
fmt.Fprintf(os.Stderr, "self-write: persist %s: %v\n", path, err)
}
}
}
func forgetSelfWrites(paths ...string) {
selfWriteMu.Lock()
defer selfWriteMu.Unlock()
for _, path := range paths {
delete(selfWrites, path)
if selfWriteStore != nil {
if err := selfWriteStore.DeleteRawAndSave(selfWriteKey(path)); err != nil {
fmt.Fprintf(os.Stderr, "self-write: forget %s: %v\n", path, err)
}
}
}
}
// isExpectedSelfWrite reports whether content and file identity still match a
// CSM self-write. Provisional in-memory records expire; identity-bound durable
// records do not.
func isExpectedSelfWrite(path string, content []byte) bool {
selfWriteMu.Lock()
defer selfWriteMu.Unlock()
now := selfWriteNow()
pruneExpiredSelfWritesLocked(now)
sum := sha256.Sum256(content)
got := hex.EncodeToString(sum[:])
if rec, ok := selfWrites[path]; ok {
if got != rec.hash {
return false
}
// A provisional record is installed immediately before CSM invokes
// crontab so a synchronous filesystem event can be suppressed. Once
// the write finishes, RecordSelfWrite replaces it with file identity.
if rec.identity == nil {
return len(content) > 0
}
identity, ok := currentSelfWriteIdentity(path, content)
return ok && identity == *rec.identity
}
if selfWriteStore == nil {
return false
}
raw, ok := selfWriteStore.GetRaw(selfWriteKey(path))
if !ok {
return false
}
var want durableSelfWriteRecord
if err := json.Unmarshal([]byte(raw), &want); err != nil ||
want.Version != durableSelfWriteVersion ||
want.Hash == "" ||
got != want.Hash {
return false
}
identity, ok := currentSelfWriteIdentity(path, content)
return ok && identity == want.Identity
}
func pruneExpiredSelfWritesLocked(now time.Time) {
for path, rec := range selfWrites {
if now.After(rec.expires) {
delete(selfWrites, path)
}
}
}
func currentSelfWriteIdentity(path string, content []byte) (selfWriteFileIdentity, bool) {
f, err := osFS.Open(path)
if err != nil {
return selfWriteFileIdentity{}, false
}
defer func() { _ = f.Close() }()
got, err := io.ReadAll(io.LimitReader(f, int64(len(content))+1))
if err != nil || !bytes.Equal(got, content) {
return selfWriteFileIdentity{}, false
}
info, err := f.Stat()
if err != nil || !info.Mode().IsRegular() {
return selfWriteFileIdentity{}, false
}
return selfWriteIdentityFromFileInfo(info)
}
//go:build linux
package checks
import (
"os"
"syscall"
)
func selfWriteIdentityFromFileInfo(info os.FileInfo) (selfWriteFileIdentity, bool) {
st, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return selfWriteFileIdentity{}, false
}
return selfWriteFileIdentity{
Device: uint64(st.Dev),
Inode: st.Ino,
ChangeSec: st.Ctim.Sec,
ChangeNsec: st.Ctim.Nsec,
}, true
}
//go:build linux
package checks
import (
"os"
"syscall"
)
func sensitiveFileOwnership(info os.FileInfo) (uid, gid uint32, ok bool) {
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return 0, 0, false
}
return stat.Uid, stat.Gid, true
}
package checks
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// sensitiveWatchset is the static set of system-configuration paths CSM
// raises a finding on when any of them is opened for write. The set is
// intentionally narrow and not operator-configurable: an attacker who
// learns that a path is excluded gets a free landing pad.
//
// Glob entries expand at runtime; non-glob entries appear once.
var sensitiveWatchset = []string{
"/etc/shadow",
"/etc/gshadow",
"/etc/passwd",
"/etc/group",
"/etc/sudoers",
"/etc/sudoers.d/*",
"/etc/ssh/sshd_config",
"/etc/ssh/sshd_config.d/*",
"/etc/cron.d/*",
"/etc/cron.hourly/*",
"/etc/cron.daily/*",
"/etc/cron.weekly/*",
"/etc/cron.monthly/*",
"/var/spool/cron/*",
// Debian cron keeps user crontabs one level deeper than cronie.
"/var/spool/cron/crontabs/*",
}
const sensitiveFileBaselineKey = "_sensitive_file_hash:__baseline_complete"
// sensitiveFileScheduleKeyPrefix keys each cron.d fingerprint bound to its
// content hash in the periodic check.
const sensitiveFileScheduleKeyPrefix = "_sensitive_file_schedule:"
// ExpandWatchset returns the absolute paths in the watchset, with globs
// expanded against the given filesystem root. Non-existent paths drop
// silently; the next refresh picks them up once they are created. root
// is "/" in production and a t.TempDir in tests.
func ExpandWatchset(root string) []string {
var out []string
for _, pat := range sensitiveWatchset {
full := filepath.Join(root, pat)
if strings.ContainsAny(pat, "*?[") {
matches, _ := filepath.Glob(full)
out = append(out, matches...)
continue
}
out = append(out, full)
}
return out
}
// classifySensitive returns a stable kind label for a watchset path so
// findings can vary their severity and message.
func classifySensitive(path string) string {
dir := filepath.Dir(path)
// Drop-in names are arbitrary; a cron file named after an auth file
// still needs cron-content scoring.
if strings.Contains(dir, "/cron") || strings.Contains(dir, "/spool/cron") {
return "cron"
}
switch filepath.Base(path) {
case "shadow", "gshadow", "passwd", "group":
return "auth"
case "sshd_config":
return "sshd"
case "sudoers":
return "sudo"
}
if strings.Contains(dir, "/sudoers.d") {
return "sudo"
}
if strings.Contains(dir, "/sshd_config.d") {
return "sshd"
}
return ""
}
// EvaluateSensitiveFileWrite returns a populated alert.Finding and true when
// the BPF live backend observed a write to a watchset path. It reads current
// content for content-bound self-write suppression.
// Returns false for paths classifySensitive does not recognise -- the BPF
// program already filters via its dev+inode map, but this guards against
// stale map entries pointing at unrelated files.
func EvaluateSensitiveFileWrite(path string, uid, pid uint32, comm string) (alert.Finding, bool) {
content, err := osFS.ReadFile(path)
return EvaluateSensitiveFileWriteSnapshot(path, uid, pid, comm, content, err == nil)
}
// EvaluateSensitiveFileWriteSnapshot evaluates a live write against bytes
// captured with the file state that will be recorded for refresh deduplication.
// Keeping those two observations together prevents a concurrent rename from
// making the finding describe or suppress different content.
func EvaluateSensitiveFileWriteSnapshot(path string, uid, pid uint32, comm string, content []byte, contentKnown bool) (alert.Finding, bool) {
kind := classifySensitive(path)
if kind == "" {
return alert.Finding{}, false
}
// Suppress writes CSM itself just performed (e.g. installing a per-user
// wp-cron). Content-bound: a tamper layered on top changes the hash and
// is still reported.
if contentKnown && isExpectedSelfWrite(path, content) {
return alert.Finding{}, false
}
// Durable complement to the TTL-bounded self-write ledger: a user crontab
// holding only CSM-installed WP-Cron jobs is CSM's own managed block, not
// attacker persistence, even after a restart cleared the ledger.
if suppressedAsManagedWPCron(path, content) {
return alert.Finding{}, false
}
sev := alert.High
if uid != 0 {
sev = alert.Critical
}
now := time.Now()
f := alert.Finding{
Severity: sev,
Check: "sensitive_file_modified",
Message: fmt.Sprintf("Write to sensitive system file: %s (uid=%d)", path, uid),
Details: fmt.Sprintf("Class: %s, PID: %d, Comm: %s, User: %s", kind, pid, comm, LookupUser(uid)),
FilePath: path,
Timestamp: now,
}
f = rescoreSensitive(f, kind, content, pid, false, now)
// PID and comm differ on every write; the file, its writer and the
// severity do not. Severity stays in the identity so a write demoted by
// provenance cannot hide a later undemoted one. The refresh diff skips
// content already reported live, so known content is part of it too: a
// second change to the same file is a new alert.
f.DedupKey = fmt.Sprintf("path=%q class=%s uid=%d severity=%s", path, kind, uid, f.Severity)
if contentKnown {
f.DedupKey += fmt.Sprintf(" content=%x", sha256.Sum256(content))
}
return f, true
}
// EvaluateSensitiveFileAppearance returns a finding when a path no previous
// watchset refresh had seen shows up -- a genuinely new cron drop-in, sudoers
// fragment, or user crontab.
func EvaluateSensitiveFileAppearance(path string) (alert.Finding, bool) {
content, err := osFS.ReadFile(path)
return evaluateSensitiveWatchsetChange(path, "New sensitive system file appeared", content, err == nil, false)
}
// evaluateSensitiveWatchsetChange builds the finding both refresh-diff outcomes
// share. Callers pass the bytes that produced the compared digest so a later
// rewrite cannot change self-write suppression or cron-content scoring.
// scheduleOnly reports that those bytes differ from the previous snapshot
// only in the time of day of cron.d daily jobs.
func evaluateSensitiveWatchsetChange(path, summary string, content []byte, contentKnown, scheduleOnly bool) (alert.Finding, bool) {
kind := classifySensitive(path)
if kind == "" {
return alert.Finding{}, false
}
if contentKnown {
if isExpectedSelfWrite(path, content) {
return alert.Finding{}, false
}
}
if suppressedAsManagedWPCron(path, content) {
return alert.Finding{}, false
}
now := time.Now()
f := alert.Finding{
Severity: alert.High,
Check: "sensitive_file_modified",
Message: fmt.Sprintf("%s: %s", summary, path),
Details: fmt.Sprintf("Class: %s", kind),
FilePath: path,
Timestamp: now,
}
var scoreContent []byte
if kind == "cron" {
scoreContent = content
}
return rescoreSensitive(f, kind, scoreContent, 0, scheduleOnly, now), true
}
// SensitiveFileState is the stable identity of one watchset path. Regular-file
// inode churn is deliberately excluded, while security metadata and symlink
// targets remain visible. CronSchedule is the schedule fingerprint of a cron.d
// file, taken from the same bytes as ContentDigest, and empty for every other
// path.
type SensitiveFileState struct {
ContentDigest string
PathIdentity string
CronSchedule string
}
// NextSensitiveDigests builds the state snapshot for a refresh cycle and
// returns the exact readable regular-file content behind each digest. A path
// whose content cannot be read keeps its previous digest so a transient read
// error does not surface as a content change. Non-regular objects retain type
// identity without being read. Paths absent from paths drop out. prev is only
// read and is never mutated.
func NextSensitiveDigests(prev map[string]SensitiveFileState, paths []string) (map[string]SensitiveFileState, map[string][]byte) {
next := make(map[string]SensitiveFileState, len(paths))
contents := make(map[string][]byte, len(paths))
for _, path := range paths {
state := prev[path]
identity, regularContent, identityKnown := sensitivePathIdentity(path)
if identityKnown {
state.PathIdentity = identity
}
if !regularContent {
next[path] = state
continue
}
data, err := readSensitiveRegularFile(path)
if err == nil {
sum := sha256.Sum256(data)
state.ContentDigest = hex.EncodeToString(sum[:])
state.CronSchedule = cronScheduleFingerprint(path, data)
contents[path] = data
}
next[path] = state
}
return next, contents
}
type sensitiveRegularFileReader interface {
ReadRegularFile(string) ([]byte, error)
}
// Production uses a nonblocking, fd-verified read. The fallback keeps custom
// test providers source-compatible after the preceding type check.
func readSensitiveRegularFile(path string) ([]byte, error) {
if reader, ok := osFS.(sensitiveRegularFileReader); ok {
return reader.ReadRegularFile(path)
}
return osFS.ReadFile(path)
}
func sensitivePathIdentity(path string) (identity string, regularContent, ok bool) {
info, err := osFS.Lstat(path)
if err != nil {
return "", false, false
}
if info.Mode()&os.ModeSymlink != 0 {
target, err := osFS.Readlink(path)
if err != nil {
return "", false, false
}
targetInfo, err := osFS.Stat(path)
if err != nil {
return "symlink\x00" + target + "\x00unresolved", false, true
}
resolvedIdentity, regular := sensitiveResolvedIdentity(targetInfo)
return "symlink\x00" + target + "\x00" + resolvedIdentity, regular, true
}
identity, regular := sensitiveResolvedIdentity(info)
return identity, regular, true
}
func sensitiveResolvedIdentity(info os.FileInfo) (string, bool) {
modeType := info.Mode() & os.ModeType
identity := fmt.Sprintf("type\x00%x\x00perm\x00%o", modeType, info.Mode().Perm())
if uid, gid, ok := sensitiveFileOwnership(info); ok {
identity += fmt.Sprintf("\x00owner\x00%d\x00%d", uid, gid)
}
if modeType == 0 {
return identity, true
}
if fileIdentity, ok := selfWriteIdentityFromFileInfo(info); ok {
identity += fmt.Sprintf("\x00%d\x00%d", fileIdentity.Device, fileIdentity.Inode)
}
return identity, false
}
// DiffSensitiveWatchset compares two refresh snapshots of the watchset and
// returns the findings the newer one warrants. Both maps are keyed by
// absolute path. contents holds the bytes used for cur's content digests.
//
// Identity is the path, never the inode. Keying on dev+inode reported every
// atomic rewrite (write temp, rename over) as a brand-new file, which is how
// /etc/passwd came to "appear" 16 times in a month on a live host. A path the
// previous snapshot knew about can only have changed, not appeared.
//
// Paths that vanished produce nothing: the caller unwatches the inode, and a
// deletion is not evidence of the tampering this watchset exists to catch.
//
// liveReported holds the exact states whose live-hook findings were delivered
// since the last refresh. A matching state is adopted without a duplicate.
// An appearance is still reported: the hook describes a write, not the fact
// that the path did not exist before.
func DiffSensitiveWatchset(prev, cur map[string]SensitiveFileState, contents map[string][]byte, liveReported map[string]SensitiveFileState) []alert.Finding {
paths := make([]string, 0, len(cur))
for path := range cur {
paths = append(paths, path)
}
sort.Strings(paths)
var findings []alert.Finding
for _, path := range paths {
curState := cur[path]
prevState, known := prev[path]
content, contentKnown := contents[path]
switch {
case !known:
if f, emit := evaluateSensitiveWatchsetChange(path, "New sensitive system file appeared", content, contentKnown, false); emit {
findings = append(findings, f)
}
case !sensitiveFileStateChanged(prevState, curState):
// Unknown evidence and equivalent regular-file rewrites do not
// establish a change.
case sensitiveLiveReportMatches(liveReported[path], curState):
// The live hook already delivered a finding for this exact state.
default:
summary := "Content changed on sensitive system file"
if prevState.PathIdentity != "" && curState.PathIdentity != "" && prevState.PathIdentity != curState.PathIdentity {
summary = "Path identity changed on sensitive system file"
}
// A metadata change is its own evidence; only a pure content
// change can be shown to have moved daily jobs and nothing else.
scheduleOnly := contentKnown && prevState.PathIdentity == curState.PathIdentity &&
cronScheduleUnchanged(prevState.CronSchedule, curState.CronSchedule)
if f, emit := evaluateSensitiveWatchsetChange(path, summary, content, contentKnown, scheduleOnly); emit {
findings = append(findings, f)
}
}
}
return findings
}
func sensitiveFileStateChanged(prev, cur SensitiveFileState) bool {
contentChanged := prev.ContentDigest != "" && cur.ContentDigest != "" && prev.ContentDigest != cur.ContentDigest
pathChanged := prev.PathIdentity != "" && cur.PathIdentity != "" && prev.PathIdentity != cur.PathIdentity
return contentChanged || pathChanged
}
func sensitiveLiveReportMatches(reported, cur SensitiveFileState) bool {
return reported.ContentDigest != "" && reported.PathIdentity != "" && reported == cur
}
// CheckSensitiveFiles is the periodic safety-net that runs when the BPF
// live monitor is unavailable or disabled. It content-hashes every watchset
// path and emits a finding when a hash differs from the previous run. The
// first run records baselines without emitting findings.
//
// CheckShadowChanges in auth.go does richer per-user diff and infra-IP
// suppression for /etc/shadow specifically; this catch-all complements
// that for sshd_config, sudoers, cron drop-ins, etc. Both run in parallel;
// audit-log dedup handles the (rare) overlap.
func CheckSensitiveFiles(_ context.Context, _ *config.Config, store *state.Store) []alert.Finding {
if store == nil {
return nil
}
var findings []alert.Finding
_, baselineComplete := store.GetRaw(sensitiveFileBaselineKey)
for _, path := range ExpandWatchset("/") {
data, err := osFS.ReadFile(path)
if err != nil {
continue
}
sum := sha256.Sum256(data)
hashHex := hex.EncodeToString(sum[:])
key := "_sensitive_file_hash:" + path
prev, ok := store.GetRaw(key)
schedule := cronScheduleSnapshotFingerprint(path, data)
scheduleKey := sensitiveFileScheduleKeyPrefix + path
prevSchedule, _ := store.GetRaw(scheduleKey)
if schedule != "" {
store.SetRaw(scheduleKey, hashHex+":"+schedule)
} else {
store.DeleteRaw(scheduleKey)
}
if !ok {
store.SetRaw(key, hashHex)
if baselineComplete {
if f, emit := EvaluateSensitiveFileAppearance(path); emit {
findings = append(findings, f)
}
}
continue
}
if prev == hashHex {
continue
}
store.SetRaw(key, hashHex)
// A content change CSM itself made (e.g. installing a wp-cron) updates
// the stored baseline above but raises no finding.
if isExpectedSelfWrite(path, data) {
continue
}
if suppressedAsManagedWPCron(path, data) {
continue
}
kind := classifySensitive(path)
var contentForScore []byte
if kind == "cron" {
contentForScore = data
}
hashChange := alert.Finding{
Severity: alert.High,
Check: "sensitive_file_modified",
Message: fmt.Sprintf("Periodic check: content hash changed for %s", path),
Details: fmt.Sprintf("Previous: %s, Current: %s", prev, hashHex),
FilePath: path,
Timestamp: time.Now(),
}
scheduleOnly := cronScheduleBaselineUnchanged(prev, prevSchedule, schedule)
findings = append(findings, rescoreSensitive(hashChange, kind, contentForScore, 0, scheduleOnly, time.Now()))
}
if !baselineComplete {
store.SetRaw(sensitiveFileBaselineKey, "1")
}
return findings
}
package checks
import (
"bytes"
"fmt"
"os"
"time"
"github.com/pidginhost/csm/internal/alert"
)
// pkgManagerLogs is the ordered set of package-manager log files whose
// recent mtime acts as evidence of a legitimate root-driven file system
// change. RPM-family hosts touch dnf.rpm.log / yum.log; Debian-family
// hosts touch dpkg.log; minimal installs add history.log for unattended-
// upgrades. The variable is package-private (no operator override) so an
// attacker who learns CSM is here cannot point the daemon at an empty
// path -- the trade-off is that mtime spoofing requires root, which is
// already game-over for this detector class.
var pkgManagerLogs = []string{
"/var/log/dnf.rpm.log",
"/var/log/dnf.log",
"/var/log/yum.log",
"/var/log/dpkg.log",
"/var/log/apt/history.log",
}
// AncestryProvenance names the trusted system component that owns the process
// tree rooted at pid -- a package transaction, or the control panel's own
// maintenance -- and returns "" when nothing in the chain proves one. The
// string is recorded in the finding, so an operator can tell which evidence
// produced a demotion. Nil on hosts where the daemon has not wired it, in
// which case the pkg-window and cron-content layers still apply.
var AncestryProvenance func(pid uint32) string
// pkgManagerWindow returns true when any pkgManagerLogs file was modified
// within window. Reads file mtime only; does not parse log contents.
func pkgManagerWindow(now time.Time, window time.Duration) bool {
cutoff := now.Add(-window)
for _, p := range pkgManagerLogs {
fi, err := os.Stat(p)
if err != nil {
continue
}
if fi.ModTime().After(cutoff) {
return true
}
}
return false
}
// cronDangerTokens are byte fragments whose presence in a cron drop-in is
// inconsistent with vendor-shipped automation and consistent with
// post-exploitation persistence. The list is intentionally narrow: false
// positives here cancel the demote, which is the safe failure mode.
var cronDangerTokens = [][]byte{
[]byte("| sh"),
[]byte("|sh "),
[]byte("| bash"),
[]byte("|bash "),
[]byte("; sh "),
[]byte(";sh "),
[]byte("; bash"),
[]byte(";bash "),
[]byte("base64 -d"),
[]byte("base64 --decode"),
[]byte("base64_decode"),
[]byte("eval("),
[]byte("eval $("),
[]byte("eval \""),
[]byte("/tmp/"),
[]byte("/var/tmp/"),
[]byte("/dev/shm/"),
[]byte("python -c"),
[]byte("python3 -c"),
[]byte("perl -e"),
[]byte("ruby -e"),
[]byte("nc -e"),
[]byte("ncat -e"),
[]byte("bash -i"),
[]byte("\\x"),
[]byte("curl "),
[]byte("wget "),
}
// cronHasDangerTokens also uses the crontab detector so known persistence,
// including encoded payloads, cannot be downgraded by a smaller token list.
// The extra shell fragments are case-sensitive. The curl/wget tokens are
// broad on purpose: a vendor cron that needs to fetch is rare enough that
// flagging is the right default.
func cronHasDangerTokens(content []byte) bool {
for _, tok := range cronDangerTokens {
if bytes.Contains(content, tok) {
return true
}
}
return len(MatchCrontabPatternsDeep(string(content), nil)) > 0
}
// rescoreSensitive returns f with severity adjusted per provenance signals:
// - package-manager activity inside pkgWindowDefault demotes High to Warning
// - AncestryProvenance(pid) naming a trusted component demotes High to Warning
// - scheduleOnly, a cron.d change that only moved the time of day of its
// daily jobs (cronScheduleFingerprint), demotes High to Warning
// - cron class with cronHasDangerTokens(content) vetoes any demote
//
// content and pid are optional (nil / 0). class is "" for non-classified
// findings. now is injected for deterministic testing.
func rescoreSensitive(f alert.Finding, class string, content []byte, pid uint32, scheduleOnly bool, now time.Time) alert.Finding {
if f.Severity != alert.High {
return f
}
veto := class == "cron" && len(content) > 0 && cronHasDangerTokens(content)
if veto {
return f
}
var reason string
if pkgManagerWindow(now, pkgWindowDefault) {
reason = "package manager active within window"
} else if pid != 0 && AncestryProvenance != nil {
reason = AncestryProvenance(pid)
}
// The writer evidence above needs a pid or a package transaction in the
// last two minutes. The scheduled cron.d diff and the watchset refresh
// have neither, so a panel rewrite they notice later paged High while
// the live write of the same bytes was Warning.
if reason == "" && scheduleOnly {
reason = cronScheduleOnlyReason
}
if reason == "" {
return f
}
f.Severity = alert.Warning
if f.Details == "" {
f.Details = fmt.Sprintf("Demoted: %s", reason)
} else {
f.Details = fmt.Sprintf("%s [demoted: %s]", f.Details, reason)
}
return f
}
// PkgManagerRecentlyActive reports whether any package-manager log was
// modified within the provenance demotion window. Exported for the fanotify
// /tmp-executable demotion, which gates on the same evidence as
// rescoreSensitive.
func PkgManagerRecentlyActive(now time.Time) bool {
return pkgManagerWindow(now, pkgWindowDefault)
}
// pkgWindowDefault is the slack we give for a legitimate root-driven file
// system change after a package transaction. dnf scriptlets observed up to
// a few seconds between the rpm log entry and post-install file drops; 2
// minutes covers slower scriptlets without inviting a multi-minute window
// for an attacker who happened to time a transaction.
const pkgWindowDefault = 2 * time.Minute
package checks
import (
"path/filepath"
"regexp"
"strconv"
"strings"
)
// csmManagedWPCronJobRe matches a single crontab job line exactly as
// wpCronJobLine emits it. The command shape is fixed -- run wp-cron.php under a
// per-docroot flock -- and carries no general-purpose payload, so a job line
// matching this pattern cannot be repurposed for attacker persistence. Any
// foreign command (reverse shell, curl|bash, miner) fails the match, which is
// what keeps crontabIsExclusivelyCSMWPCron from suppressing a tampered crontab.
var csmManagedWPCronJobRe = regexp.MustCompile(
`^([0-9*/,-]+) \* \* \* \* cd ('(?:[^']|'\\'')*') && flock -n "\$HOME/\.csm-wpcron-([0-9a-f]{8})\.lock" ('(?:[^']|'\\'')*') -d max_execution_time=300 wp-cron\.php >/dev/null 2>&1$`)
var (
cpanelPHPBinRe = regexp.MustCompile(`^/opt/cpanel/ea-php[0-9]{2}/root/usr/bin/php$`)
cloudLinuxPHPBinRe = regexp.MustCompile(`^/opt/alt/php[0-9]{2}/usr/bin/php$`)
)
// safeCrontabShells is the set of SHELL values a fully CSM-managed crontab may
// carry. SHELL is honored by crond to exec every job, so an arbitrary value is
// a code-execution vector; only known shells (cPanel's jailshell and the
// standard system shells) are accepted. cPanel prepends the jailshell line to
// every user crontab it touches.
var safeCrontabShells = map[string]bool{
"/usr/local/cpanel/bin/jailshell": true,
"/bin/bash": true,
"/bin/sh": true,
"/usr/bin/bash": true,
"/usr/bin/sh": true,
}
// crontabIsExclusivelyCSMWPCron reports whether every meaningful line in a user
// crontab is either an inert header (blank, comment, or a vetted environment
// assignment) or a CSM-installed WP-Cron job line. Such a crontab is fully
// CSM-managed; flagging it as a sensitive-file change is a false positive that
// the in-memory, TTL-bounded self-write ledger misses after a daemon restart or
// a crontab reformat by crond/cPanel.
//
// Safety: this is a content-structure recognizer, not a path allowlist. A
// single foreign cron entry, an unrecognized environment assignment (PATH,
// BASH_ENV, ...), or an unsafe SHELL value makes it return false, so attacker
// persistence layered into a crontab is still surfaced. At least one CSM job
// line is required so an all-headers crontab is not mistaken for ours.
func crontabIsExclusivelyCSMWPCron(owner string, content []byte) bool {
sawCSMJob := false
pendingMarker := ""
for _, raw := range strings.Split(string(content), "\n") {
line := strings.TrimSpace(raw)
if line == "" {
pendingMarker = ""
continue
}
if strings.HasPrefix(line, "#") {
// Comments are never executed by crond. The CSM marker is retained
// only to pin the following managed job to the docroot CSM wrote.
pendingMarker = ""
if strings.HasPrefix(line, wpCronJobMarker) {
pendingMarker = strings.TrimSpace(strings.TrimPrefix(line, wpCronJobMarker))
}
continue
}
if name, val, ok := splitCrontabEnv(line); ok {
if !safeCrontabEnvAssignment(name, val) {
return false
}
pendingMarker = ""
continue
}
if docroot, ok := csmManagedWPCronJob(line, owner); ok && pendingMarker == docroot {
sawCSMJob = true
pendingMarker = ""
continue
}
return false
}
return sawCSMJob
}
func csmManagedWPCronJob(line, owner string) (string, bool) {
m := csmManagedWPCronJobRe.FindStringSubmatch(line)
if m == nil {
return "", false
}
minute, lockHex := m[1], m[3]
docroot, ok := unquoteShellSingle(m[2])
if !ok || !safeManagedWPCronDocroot(owner, docroot) {
return "", false
}
phpBin, ok := unquoteShellSingle(m[4])
if !ok || !safeManagedWPCronPHPBin(phpBin) {
return "", false
}
lockID, err := strconv.ParseUint(lockHex, 16, 32)
if err != nil || uint32(lockID) != wpCronLockID(docroot) {
return "", false
}
if !wpCronMinuteMatchesOwnerDocroot(minute, owner, docroot) {
return "", false
}
return docroot, true
}
func unquoteShellSingle(q string) (string, bool) {
if len(q) < 2 || q[0] != '\'' || q[len(q)-1] != '\'' {
return "", false
}
body := q[1 : len(q)-1]
var out strings.Builder
for len(body) > 0 {
i := strings.IndexByte(body, '\'')
if i < 0 {
out.WriteString(body)
return out.String(), true
}
out.WriteString(body[:i])
if !strings.HasPrefix(body[i:], `'\''`) {
return "", false
}
out.WriteByte('\'')
body = body[i+4:]
}
return out.String(), true
}
func safeManagedWPCronDocroot(owner, docroot string) bool {
if owner == "" || !safeWPCronDocroot(docroot) {
return false
}
_, account, ok := accountRootOf(docroot)
return ok && account == owner
}
func safeManagedWPCronPHPBin(path string) bool {
if !safeCronCommandString(path) || !filepath.IsAbs(path) || filepath.Clean(path) != path {
return false
}
switch path {
case "/usr/local/bin/php", "/usr/bin/php", "/bin/php":
return true
default:
return cpanelPHPBinRe.MatchString(path) || cloudLinuxPHPBinRe.MatchString(path)
}
}
func wpCronMinuteMatchesOwnerDocroot(minute, owner, docroot string) bool {
for interval := 1; interval <= wpCronMaxIntervalMin; interval++ {
if wpCronMinuteField(wpCronStaggerOffset(owner, docroot, interval), interval) == minute {
return true
}
}
return false
}
// splitCrontabEnv parses a crontab environment line of the form NAME=value.
// crond treats a line as an assignment (not a command) when the text left of
// the first '=' is a single bare identifier; a job line such as
// "* * * * * FOO=bar cmd" has spaces before '=' and is not an assignment. The
// value's surrounding quotes are stripped to match how cPanel writes them.
func splitCrontabEnv(line string) (name, value string, ok bool) {
eq := strings.IndexByte(line, '=')
if eq <= 0 {
return "", "", false
}
name = strings.TrimSpace(line[:eq])
if strings.ContainsAny(name, " \t") {
return "", "", false
}
for i, r := range name {
switch {
case r == '_':
case r >= 'A' && r <= 'Z':
case r >= 'a' && r <= 'z':
case r >= '0' && r <= '9' && i > 0:
default:
return "", "", false
}
}
value = strings.TrimSpace(line[eq+1:])
if value == "" {
return name, value, true
}
switch value[0] {
case '\'', '"':
if len(value) < 2 || value[len(value)-1] != value[0] {
return "", "", false
}
value = value[1 : len(value)-1]
}
return name, value, true
}
// safeCrontabEnvAssignment reports whether a crontab environment assignment is
// inert enough to appear in a crontab still considered fully CSM-managed.
// MAILTO is stored verbatim by crond and never executed, so any value is safe.
// SHELL and HOME influence job execution, so their values are constrained.
// Every other name (PATH, BASH_ENV, LD_*, ...) is rejected.
func safeCrontabEnvAssignment(name, value string) bool {
switch strings.ToUpper(name) {
case "MAILTO":
return true
case "SHELL":
return safeCrontabShells[value]
case "HOME":
return safeCrontabHome(value)
default:
return false
}
}
func safeCrontabHome(value string) bool {
if !safeCronCommandString(value) || !filepath.IsAbs(value) || filepath.Clean(value) != value {
return false
}
return value == "/root" || underAccountRoot(value) || strings.HasPrefix(value, "/root/")
}
// suppressedAsManagedWPCron reports whether a sensitive-file finding for a user
// crontab should be suppressed because the crontab is exclusively a
// CSM-installed WP-Cron block. Scoped to the platform's user crontab spool,
// the only place CSM installs WP-Cron jobs; system drop-ins under /etc/cron.d
// are never suppressed here.
func suppressedAsManagedWPCron(path string, content []byte) bool {
if len(content) == 0 {
return false
}
owner, ok := cronSpoolOwner(path)
if !ok {
return false
}
return crontabIsExclusivelyCSMWPCron(owner, content)
}
func cronSpoolOwner(path string) (string, bool) {
clean := filepath.ToSlash(filepath.Clean(path))
if filepath.ToSlash(filepath.Dir(clean)) != filepath.ToSlash(filepath.Clean(cronSpoolDir())) {
return "", false
}
owner := filepath.Base(clean)
if owner == "." || owner == "/" || owner == "" {
return "", false
}
if owner == "root" || !validCPUser.MatchString(owner) {
return "", false
}
return owner, true
}
package checks
import (
"net/netip"
"strings"
"time"
)
// SessionRecord is one cpsrvd session_log record:
// "[<time>] info [<service>] <origin> <event> <account>:<session> <detail>...".
// Every field is matched by position, so text a client controls later in
// the line (a login name, the address= list) cannot supply the service,
// the origin, the event or the account.
type SessionRecord struct {
At time.Time // zero when the timestamp does not parse
Level string
Service string // without brackets: cpaneld, webmaild, whostmgrd, whostmgr, security, xml-api
Origin string // the client address, or "internal" for a server-side event
Event string // NEW, PURGE, ...
Account string
Session string
Detail []string // the fields after <account>:<session>
}
// ParseSessionRecord reads one info-level session_log line. Records without
// a bracketed service or an <account>:<session> pair are rejected.
func ParseSessionRecord(line string) (SessionRecord, bool) {
if !strings.HasPrefix(line, "[") || strings.ContainsAny(line, "\r\n") {
return SessionRecord{}, false
}
end := strings.IndexByte(line, ']')
if end < 0 || end+1 >= len(line) || (line[end+1] != ' ' && line[end+1] != '\t') {
return SessionRecord{}, false
}
fields := strings.Fields(line[end+1:])
if len(fields) < 5 || fields[0] != "info" {
return SessionRecord{}, false
}
service, opened := strings.CutPrefix(fields[1], "[")
service, closed := strings.CutSuffix(service, "]")
if !opened || !closed || service == "" || strings.ContainsAny(service, "[]") {
return SessionRecord{}, false
}
account, session, found := strings.Cut(fields[4], ":")
if !found || account == "" || session == "" {
return SessionRecord{}, false
}
at, _ := CPanelLogTime(line[:end+1])
return SessionRecord{
At: at,
Level: fields[0],
Service: service,
Origin: fields[2],
Event: fields[3],
Account: account,
Session: session,
Detail: fields[5:],
}, true
}
// ClientAddr is the record's origin as an address. "internal" and any other
// text that is not an address report false.
func (r SessionRecord) ClientAddr() (netip.Addr, bool) {
addr, err := netip.ParseAddr(r.Origin)
if err != nil || addr.Zone() != "" {
return netip.Addr{}, false
}
return addr.Unmap(), true
}
// Reason is the first detail field, which names why a PURGE ended the
// session (password_change, tokendenied, logout, ...).
func (r SessionRecord) Reason() string {
if len(r.Detail) == 0 {
return ""
}
return r.Detail[0]
}
// Attr is the value of key in the comma-separated key=value detail of a NEW
// record ("address=...,app=cpaneld,method=handle_form_login"). Duplicate keys
// yield no value: client-controlled detail must not override server metadata.
func (r SessionRecord) Attr(key string) string {
var value string
found := false
for _, field := range r.Detail {
for _, pair := range strings.Split(field, ",") {
if k, v, ok := strings.Cut(pair, "="); ok && k == key {
if found {
return ""
}
value, found = v, true
}
}
}
return value
}
// CreatedByAPI reports a session an API call opened for the account (the
// portal's create_user_session), not a login the user made.
func (r SessionRecord) CreatedByAPI() bool {
method := r.Attr("method")
return method == "create_user_session" || method == "create_session"
}
// CPanelLogTime reads the leading "[<timestamp>]" of a cPanel session_log
// or api_tokens_log line. Both the timezone-qualified form
// ("2006-01-02 15:04:05 -0700") and the older bare form ("2006-01-02 15:04:05",
// interpreted in the host's local zone) are accepted. Returns ok=false when no
// bracketed timestamp is present or it does not parse.
func CPanelLogTime(line string) (time.Time, bool) {
open := strings.IndexByte(line, '[')
if open < 0 {
return time.Time{}, false
}
end := strings.IndexByte(line[open:], ']')
if end < 0 {
return time.Time{}, false
}
stamp := strings.TrimSpace(line[open+1 : open+end])
for _, layout := range []string{"2006-01-02 15:04:05 -0700", "2006-01-02 15:04:05"} {
if t, err := time.ParseInLocation(layout, stamp, time.Local); err == nil {
return t, true
}
}
return time.Time{}, false
}
package checks
import (
"os"
"path/filepath"
"strings"
)
// Recognisers used by the realtime fanotify restore/probe dedup logic.
// The scheduled file index deliberately does not call these as scan skips:
// upload PHP is indexed first and then classified by content.
// LooksLikeCpanelRestoreStaging recognises files inside cPanel's
// pkgacct/restorepkg staging tree. cPanel extracts the user backup
// as root into /home/cpanelpkgrestore.TMP.work.<id>/ for inspection,
// then re-extracts it under the user identity into /home/<account>/.
// Both extractions raise events; the user-context one carries the
// real signal, so the staging-side alert is a duplicate.
//
// The recogniser requires the marker to sit directly under /home (the
// only place cPanel ever creates it) plus a non-empty alphanumeric id
// of >=2 chars. A user account at /home/<user>/ cannot create
// siblings of itself, so this gate cannot be spoofed by a non-root
// attacker.
func LooksLikeCpanelRestoreStaging(path string) bool {
const homeRoot = "/home"
const marker = "/cpanelpkgrestore.TMP.work."
idx := strings.Index(path, marker)
if idx < 0 {
return false
}
if idx != len(homeRoot) {
return false
}
if !strings.HasPrefix(path, homeRoot) {
return false
}
rest := path[idx+len(marker):]
if rest == "" {
return false
}
end := strings.IndexByte(rest, '/')
var token string
if end < 0 {
token = rest
} else {
token = rest[:end]
}
if len(token) < 2 {
return false
}
for i := 0; i < len(token); i++ {
c := token[i]
switch {
case c >= '0' && c <= '9':
case c >= 'a' && c <= 'z':
case c >= 'A' && c <= 'Z':
default:
return false
}
}
return true
}
// LooksLikeWPOptimizeProbeByPath recognises WP-Optimize's per-server
// probe files using path structure alone. WP-Optimize writes tiny
// <?php files to /wp-content/uploads/wpo/.../test.php to test whether
// the host honours certain Apache/Nginx directives.
//
// Path-only gates (no content read):
//
// 1. Path lies under /wp-content/uploads/wpo/.
// 2. The basename is exactly "test.php" (the literal filename
// WP-Optimize uses for these probes; an attacker dropping
// /uploads/wpo/webshell.php fails this gate and continues to
// the standard alert).
// 3. The wp-optimize plugin directory is actually present in this
// site's wp-content/plugins/ tree (filesystem stat).
//
// The realtime path additionally applies a content shape gate before
// suppressing the duplicate alert, so the path predicate here stays
// narrow to the literal probe filename and installed plugin directory.
func LooksLikeWPOptimizeProbeByPath(path string) bool {
const marker = "/wp-content/uploads/wpo/"
if !strings.Contains(path, marker) {
return false
}
if filepath.Base(path) != "test.php" {
return false
}
uploadsIdx := strings.Index(path, "/wp-content/uploads/")
if uploadsIdx < 0 {
return false
}
pluginDir := path[:uploadsIdx] + "/wp-content/plugins/wp-optimize"
st, err := os.Stat(pluginDir)
if err != nil || !st.IsDir() {
return false
}
return true
}
package checks
import (
"fmt"
"github.com/pidginhost/csm/internal/alert"
)
// AttributeSocketOwner attaches the hosting account identified by a kernel
// socket or connection-event UID. A missing process snapshot does not remove
// this evidence. Root, service and unresolved UIDs remain unattributed.
// The message carries the account too, so dispatch and audit identities keep
// different accounts contacting the same destination separate.
func AttributeSocketOwner(f *alert.Finding, uid uint32) {
if uid == 0 {
return
}
if owner := HostingAccountForUser(LookupUser(uid)); owner != "" {
f.TenantID = owner
f.Message += fmt.Sprintf(" (account %s)", owner)
}
}
package checks
import (
"regexp"
"strings"
)
// SEO-spam context analysis for WordPress post content.
//
// Word-boundary keyword matching (see countSpamMatches) eliminated the
// substring false positives where "specialist" triggered "cialis" and
// "pharmaceutical" triggered "pharma". It did not eliminate a second
// class of false positive: legitimate prose mentions of an industry or
// product category ("our advisor covers consumer goods, energy,
// pharma" or "Industria alimentara si Pharma"). Word-boundary matching
// cannot distinguish the prose mention from the cloaked black-hat SEO
// link the acmebakery attack injected on the same site.
//
// This file classifies a keyword HIT as SPAM only when the surrounding
// HTML shows an attacker signal: CSS cloaking (off-screen absolute
// positioning, display:none, visibility:hidden, text-indent, micro
// height, font-size:0), an injection fingerprint (short hex HTML
// comment bracketing content), or an external anchor whose URL path
// itself contains the keyword ("/buy/pharma/" style commercial paths).
//
// Bare keyword mentions with none of those signals do not fire, so
// legitimate industry-vertical prose is silent.
//
// The proximity window is bounded to ±spamContextWindow bytes around
// the keyword match. Cloaking at the top of a long post unrelated to
// a keyword mention at the bottom does not spuriously associate.
// spamContextWindow bounds how far (in bytes) around a keyword match
// the analyzer looks for cloaking signals. 400 covers a typical
// cloaked-div attack (small <div> with style + <a> + keyword) without
// reaching into unrelated content.
const spamContextWindow = 400
// cssCloakPatterns are regexes that each, when matched, identify a
// CSS property value indicative of content cloaking. The list is
// conservative: each entry corresponds to a technique widely used in
// real SEO spam campaigns and rarely to nothing else at production
// scale. `(?i)` makes matching case-insensitive; `\s*` around colons
// and values tolerates the whitespace variants attackers use to evade
// naive string scanners ("display : none" vs "display:none").
var cssCloakPatterns = []*regexp.Regexp{
// display:none and visibility:hidden — classic hide.
regexp.MustCompile(`(?i)\bdisplay\s*:\s*none\b`),
regexp.MustCompile(`(?i)\bvisibility\s*:\s*hidden\b`),
// text-indent with a negative value — pushes text off-screen.
regexp.MustCompile(`(?i)\btext-indent\s*:\s*-\s*\d+`),
// Micro height with a positive integer of 0 or 1, paired with
// overflow:hidden to suppress contents. We require the pair
// because height:1 alone is occasionally legitimate.
regexp.MustCompile(`(?i)\bheight\s*:\s*[01](\s*px)?\b[^"'}]*\boverflow\s*:\s*hidden\b`),
regexp.MustCompile(`(?i)\boverflow\s*:\s*hidden\b[^"'}]*\bheight\s*:\s*[01](\s*px)?\b`),
// font-size:0 — classic invisible-text technique.
regexp.MustCompile(`(?i)\bfont-size\s*:\s*0\b`),
// position:absolute paired with a negative coordinate in the same
// style attribute (non-greedy [^"'}]* stays inside one attribute).
// Legitimate uses of position:absolute (menus, tooltips) have
// non-negative coordinates; the pairing is the signal.
regexp.MustCompile(`(?i)\bposition\s*:\s*absolute\b[^"'}]*\b(left|top|right|bottom)\s*:\s*-\s*\d+`),
regexp.MustCompile(`(?i)\b(left|top|right|bottom)\s*:\s*-\s*\d+[^"'}]*\bposition\s*:\s*absolute\b`),
// Off-screen via large negative margin on a block element.
regexp.MustCompile(`(?i)\bmargin(-left|-top)?\s*:\s*-\s*\d{4,}`),
}
// injectionFingerprintRe matches the short hex HTML comment (4-8 hex
// chars) attackers use to tag their own injections across a campaign.
// The length bracket is deliberate: shorter matches are too ambiguous,
// longer matches overlap with legitimate WordPress UUIDs.
var injectionFingerprintRe = regexp.MustCompile(`<!--\s*[a-f0-9]{4,8}\s*-->`)
// anchorHrefRe extracts the href attribute value from each anchor tag
// in a fragment. Double- and single-quoted values are both accepted.
var anchorHrefRe = regexp.MustCompile(`(?i)<a\b[^>]+href\s*=\s*["']([^"']+)["']`)
// externalSchemeRe recognises absolute or protocol-relative URLs. A
// relative URL ("/services/pharma/") is same-origin navigation and not
// an external spam link.
var externalSchemeRe = regexp.MustCompile(`^(?i)(https?:)?//`)
// contentHasSpamContext reports whether any occurrence of the keyword
// in content is accompanied by an SEO-spam signal within
// spamContextWindow bytes. Returning false is a direct statement that
// every keyword hit is a bare mention without cloaking context — the
// caller should suppress the finding in that case.
func contentHasSpamContext(content string, pattern dbSpamPattern) bool {
matches := spamKeywordMatchIndexes(pattern, content)
for _, m := range matches {
if hitHasSpamContext(content, m[0], m[1], pattern.keyword) {
return true
}
}
return false
}
// countCloakedSpamMatches returns the number of rows in contents whose
// text contains the spam keyword AND shows an accompanying SEO-spam
// context signal. It is the aggregator used by checkWPPosts to decide
// whether to emit a db_spam_injection finding: bare prose mentions of
// a keyword do not count; only cloaked/SEO-style injections do.
//
// Each qualifying row is counted exactly once regardless of how many
// keyword hits it contains — the finding is per-post, not per-hit.
func countCloakedSpamMatches(pattern dbSpamPattern, contents []string) int {
n := 0
for _, c := range contents {
if contentHasSpamContext(c, pattern) {
n++
}
}
return n
}
// hitHasSpamContext examines the ±spamContextWindow byte region around
// a single keyword hit for cloaking, injection-fingerprint, or
// external-spam-link signals. Extracted as a helper so tests can target
// one hit at a time.
func hitHasSpamContext(content string, start, end int, keyword string) bool {
ws := start - spamContextWindow
if ws < 0 {
ws = 0
}
we := end + spamContextWindow
if we > len(content) {
we = len(content)
}
window := content[ws:we]
if windowHasCSSCloaking(window) {
return true
}
if injectionFingerprintRe.MatchString(window) {
return true
}
if windowHasExternalSpamAnchor(window, keyword) {
return true
}
return false
}
// positionAbsoluteRe and negativeCoordRe are evaluated together in
// windowHasCSSCloaking for the "off-screen absolute positioning"
// signal. Keeping them as two independent regexes (rather than one
// paired regex with [^"'}]* between them) closes an evasion where the
// attacker splits the cloak across two CSS rules — e.g.
// `<style>.a{position:absolute}.b{left:-9999px}</style>` — which the
// paired form stops matching at the first `}`. Both signals must
// appear somewhere in the proximity window; the window itself is
// bounded (see spamContextWindow) so the association is not unbounded.
var (
positionAbsoluteRe = regexp.MustCompile(`(?i)\bposition\s*:\s*absolute\b`)
negativeCoordRe = regexp.MustCompile(`(?i)\b(left|top|right|bottom|margin-left|margin-top)\s*:\s*-\s*\d{2,}`)
)
// windowHasCSSCloaking returns true if the window contains any CSS
// declaration from cssCloakPatterns, OR if it contains both
// position:absolute and a negative coordinate somewhere in the window
// (independent-signal form, for rule-split evasions).
func windowHasCSSCloaking(window string) bool {
for _, re := range cssCloakPatterns {
if re.MatchString(window) {
return true
}
}
if positionAbsoluteRe.MatchString(window) && negativeCoordRe.MatchString(window) {
return true
}
return false
}
// windowHasExternalSpamAnchor returns true if the window contains an
// <a href> whose destination is an external URL (absolute or
// protocol-relative) AND whose URL path contains the spam keyword as
// a bounded segment. Same-origin relative links ("/services/pharma/")
// are skipped — they are internal navigation, not spam.
func windowHasExternalSpamAnchor(window, keyword string) bool {
anchors := anchorHrefRe.FindAllStringSubmatch(window, -1)
if len(anchors) == 0 {
return false
}
keywordLower := strings.ToLower(keyword)
for _, a := range anchors {
href := a[1]
if !externalSchemeRe.MatchString(href) {
continue
}
if !hrefPathContainsKeyword(href, keywordLower) {
continue
}
return true
}
return false
}
// hrefPathContainsKeyword checks whether the URL path portion of href
// contains the keyword bounded by non-alphanumeric characters. The
// boundary guarantees "/buy/pharma/cheap" matches "pharma" but
// "/pharmaceutical/" does not.
func hrefPathContainsKeyword(href, keywordLower string) bool {
// Strip scheme://host prefix. externalSchemeRe already confirmed
// the URL starts with //host or scheme://host.
rest := href
if i := strings.Index(rest, "://"); i >= 0 {
rest = rest[i+3:]
} else {
rest = strings.TrimPrefix(rest, "//")
}
// Everything after the first '/' is the path+query.
var path string
if i := strings.IndexByte(rest, '/'); i >= 0 {
path = rest[i:]
} else {
path = ""
}
lower := strings.ToLower(path)
idx := strings.Index(lower, keywordLower)
if idx < 0 {
return false
}
// Check bounded: preceding char (if any) and following char (if
// any) must not be alphanumeric. This disambiguates
// "/buy/pharma/" (bounded by '/') from "/pharmacist/" (bounded by
// 'c' after "pharma").
if idx > 0 {
c := lower[idx-1]
if isURLWordChar(c) {
return false
}
}
after := idx + len(keywordLower)
if after < len(lower) {
c := lower[after]
if isURLWordChar(c) {
return false
}
}
return true
}
// isURLWordChar reports whether a byte is an ASCII alphanumeric used
// for URL-path word-boundary analysis. Underscore is treated as a word
// character by convention; hyphen is NOT (so "buy-cheap-pharma" still
// matches "pharma" bounded by the trailing hyphen/slash).
func isURLWordChar(b byte) bool {
switch {
case b >= 'a' && b <= 'z':
return true
case b >= 'A' && b <= 'Z':
return true
case b >= '0' && b <= '9':
return true
case b == '_':
return true
}
return false
}
package checks
import (
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
const sshLoginFollowKey = "_ssh_login_follow"
// sshLoginLookback bounds how far back a first run or a large read gap
// reports accepted logins. Anything older is history, not a new login, and
// ssh_login_unknown_ip is a critical, always-block finding.
const sshLoginLookback = time.Hour
var sshLoginMu sync.Mutex
type sshLoginFollow struct {
Follow followState `json:"follow"`
}
func loadSSHLoginFollow(store *state.Store) followState {
raw, ok := store.GetRaw(sshLoginFollowKey)
if !ok || raw == "" {
return followState{}
}
var decoded sshLoginFollow
if err := json.Unmarshal([]byte(raw), &decoded); err != nil || invalidFollowState(decoded.Follow) {
return followState{}
}
return decoded.Follow
}
func saveSSHLoginFollow(store *state.Store, st followState) {
b, err := json.Marshal(sshLoginFollow{Follow: st})
if err != nil {
return
}
if err := store.SetRawAndSave(sshLoginFollowKey, string(b)); err != nil {
fmt.Fprintf(os.Stderr, "state: error saving SSH login follow state: %v\n", err)
}
}
// checkSSHLoginsFollow reads the auth log forward from the stored offset, so
// a login is seen however much brute-force noise follows it before the next
// cycle, and skips logins older than sshLoginLookback (first-run catch-up).
func checkSSHLoginsFollow(cfg *config.Config, store *state.Store) []alert.Finding {
sshLoginMu.Lock()
defer sshLoginMu.Unlock()
st := loadSSHLoginFollow(store)
records, next, _, err := readNewSyslogRecords(authLogPath(), st)
if err != nil {
return nil // leave stored state untouched
}
now := time.Now()
cutoff := now.Add(-sshLoginLookback)
var findings []alert.Finding
for _, rec := range records {
if !strings.Contains(rec.text, "Accepted") {
continue
}
at, ok := syslogLineTime(rec.text, now)
if ok && at.Before(cutoff) {
continue
}
if !ok {
at = now
}
if f, ok := SSHAcceptedLoginFinding(rec.text, cfg); ok {
f.Observation = rec.observation(ProducerSSHLoginScan, at)
findings = append(findings, f)
}
}
saveSSHLoginFollow(store, next)
return findings
}
package checks
import (
"strings"
"sync"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/firewall"
)
var summaryChecks = sync.OnceValue(func() map[string]bool {
names := make(map[string]bool)
for _, c := range checkRegistry {
if c.Summary {
names[c.Name] = true
}
}
return names
})
// IsSummaryNotice is the alert summary-notice policy: the finding's check is
// registered Summary. auto_block also reports queue, accounting and other
// response gaps as Warnings, which need a person, so only an applied block
// (Critical), a dry-run verdict or the admission applied-responses summary
// counts. A block of an address in a Cloudflare allow range leaves its web
// ports open, so that notice needs a person too.
func IsSummaryNotice(f alert.Finding) bool {
if !summaryChecks()[f.Check] {
return false
}
if f.Check == "auto_block" {
if strings.Contains(f.Details, firewall.CloudflareCoverageWarning) {
return false
}
return f.Severity == alert.Critical || strings.Contains(f.Message, "[dry-run]") ||
f.Message == admission.AppliedSummaryMessage
}
return true
}
package checks
import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// Supply-chain dependency scanning.
//
// This is the scanner half of the supply-chain check: it parses
// composer.lock / package-lock.json dependency trees under customer
// document roots and matches the resolved versions against a local
// advisory database. The advisory database itself is operational data,
// not shipped in the binary -- an operator or a sync job writes
// <state>/advisories/supply-chain.json (format documented in
// docs/src/supply-chain-advisories.md). With no advisory file present the
// check is dormant: it parses nothing it cannot match and emits nothing.
// This mirrors the YARA-forge mirror posture (machinery in CSM, signed
// data delivered out of band).
// supplyChainAdvisoryRelPath is where CSM looks for the advisory DB,
// relative to the configured state directory.
const supplyChainAdvisoryRelPath = "advisories/supply-chain.json"
// supplyChainPkg is one resolved dependency from a lockfile.
type supplyChainPkg struct {
Ecosystem string // "composer" | "npm"
Name string
Version string
}
// supplyChainAdvisory is the OSV-subset advisory shape CSM matches
// against. A version is vulnerable when it falls inside any range:
// version >= introduced AND (fixed == "" OR version < fixed).
type supplyChainAdvisory struct {
Ecosystem string `json:"ecosystem"`
Package string `json:"package"`
Ranges []supplyChainAdvisoryRange `json:"ranges"`
ID string `json:"id"`
Severity string `json:"severity"`
Summary string `json:"summary"`
}
type supplyChainAdvisoryRange struct {
Introduced string `json:"introduced"`
Fixed string `json:"fixed"`
}
type supplyChainAdvisoryFile struct {
Advisories []supplyChainAdvisory `json:"advisories"`
}
// CheckSupplyChain scans customer dependency lockfiles for versions with
// known advisories. Dormant unless an advisory database is present at
// <state>/advisories/supply-chain.json.
func CheckSupplyChain(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if cfg == nil {
return nil
}
advisories := loadSupplyChainAdvisories(cfg.StatePath)
if len(advisories) == 0 {
return nil // no data -> nothing to match against
}
index := indexAdvisories(advisories)
var findings []alert.Finding
for _, lock := range discoverLockfiles(ctx) {
if ctx != nil && ctx.Err() != nil {
return findings
}
data, err := osFS.ReadFile(lock.path)
if err != nil {
continue
}
pkgs := lock.parse(data)
for _, p := range pkgs {
for _, adv := range index[advisoryKey(p.Ecosystem, p.Name)] {
if versionVulnerable(p.Version, adv.Ranges) {
findings = append(findings, supplyChainFinding(lock.account, lock.path, p, adv))
}
}
}
}
return findings
}
func loadSupplyChainAdvisories(statePath string) []supplyChainAdvisory {
if statePath == "" {
return nil
}
data, err := osFS.ReadFile(filepath.Join(statePath, supplyChainAdvisoryRelPath))
if err != nil {
return nil
}
var f supplyChainAdvisoryFile
if json.Unmarshal(data, &f) != nil {
return nil
}
return f.Advisories
}
func advisoryKey(ecosystem, pkg string) string {
return strings.ToLower(ecosystem) + "\x00" + strings.ToLower(pkg)
}
func indexAdvisories(advisories []supplyChainAdvisory) map[string][]supplyChainAdvisory {
out := map[string][]supplyChainAdvisory{}
for _, a := range advisories {
if a.Package == "" || a.Ecosystem == "" {
continue
}
k := advisoryKey(a.Ecosystem, a.Package)
out[k] = append(out[k], a)
}
return out
}
type lockfile struct {
path string
account string
parse func([]byte) []supplyChainPkg
}
type packageLockDependency struct {
Version string `json:"version"`
Dependencies map[string]packageLockDependency `json:"dependencies"`
}
// discoverLockfiles globs composer.lock and package-lock.json at the
// common project depths under customer home directories. Bounded by the
// glob shape (no recursive walk) so a deep node_modules tree cannot turn
// the scan into an unbounded crawl.
func discoverLockfiles(ctx context.Context) []lockfile {
patterns := []struct {
glob string
parse func([]byte) []supplyChainPkg
}{
// Relative to each account root; see accountHomeGlob.
{"*/public_html/composer.lock", parseComposerLock},
{"*/composer.lock", parseComposerLock},
{"*/public_html/package-lock.json", parsePackageLock},
{"*/package-lock.json", parsePackageLock},
}
var out []lockfile
seen := map[string]struct{}{}
for _, p := range patterns {
if ctx != nil && ctx.Err() != nil {
return out
}
matches, _ := accountHomeGlob(p.glob)
for _, m := range matches {
if _, dup := seen[m]; dup {
continue
}
seen[m] = struct{}{}
out = append(out, lockfile{path: m, account: extractUser(m), parse: p.parse})
}
}
return out
}
func parseComposerLock(data []byte) []supplyChainPkg {
var doc struct {
Packages []struct{ Name, Version string } `json:"packages"`
PackagesDev []struct{ Name, Version string } `json:"packages-dev"`
}
if json.Unmarshal(data, &doc) != nil {
return nil
}
var out []supplyChainPkg
for _, set := range [][]struct{ Name, Version string }{doc.Packages, doc.PackagesDev} {
for _, p := range set {
if p.Name == "" || p.Version == "" {
continue
}
out = append(out, supplyChainPkg{Ecosystem: "composer", Name: p.Name, Version: p.Version})
}
}
return out
}
func parsePackageLock(data []byte) []supplyChainPkg {
var doc struct {
Packages map[string]struct {
Version string `json:"version"`
} `json:"packages"`
Dependencies map[string]packageLockDependency `json:"dependencies"`
}
if json.Unmarshal(data, &doc) != nil {
return nil
}
var out []supplyChainPkg
// npm v2/v3: keyed by "node_modules/<name>" (the root "" entry is the
// project itself and has no node_modules prefix).
paths := make([]string, 0, len(doc.Packages))
for path := range doc.Packages {
paths = append(paths, path)
}
sort.Strings(paths)
for _, path := range paths {
v := doc.Packages[path]
name := npmNameFromPackagesKey(path)
if name == "" || v.Version == "" {
continue
}
out = append(out, supplyChainPkg{Ecosystem: "npm", Name: name, Version: v.Version})
}
// npm v1: dependency tree rooted at the top-level dependencies map.
if len(doc.Packages) == 0 {
return appendPackageLockV1Dependencies(out, doc.Dependencies)
}
return out
}
func appendPackageLockV1Dependencies(out []supplyChainPkg, deps map[string]packageLockDependency) []supplyChainPkg {
stack := []map[string]packageLockDependency{deps}
for len(stack) > 0 {
cur := stack[len(stack)-1]
stack = stack[:len(stack)-1]
names := make([]string, 0, len(cur))
for name := range cur {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
dep := cur[name]
if name != "" && dep.Version != "" {
out = append(out, supplyChainPkg{Ecosystem: "npm", Name: name, Version: dep.Version})
}
if len(dep.Dependencies) > 0 {
stack = append(stack, dep.Dependencies)
}
}
}
return out
}
// npmNameFromPackagesKey extracts the package name from a v2/v3
// package-lock "packages" key. The key is the path "node_modules/<name>"
// (or nested "node_modules/a/node_modules/b"); the name is whatever
// follows the last "node_modules/". The root project key "" yields "".
func npmNameFromPackagesKey(key string) string {
if key == "" {
return ""
}
parts := strings.Split(key, "/")
for i := len(parts) - 2; i >= 0; i-- {
if parts[i] != "node_modules" {
continue
}
if parts[i+1] == "" {
return ""
}
if strings.HasPrefix(parts[i+1], "@") {
if i+2 >= len(parts) || parts[i+2] == "" {
return ""
}
return parts[i+1] + "/" + parts[i+2]
}
return parts[i+1]
}
return ""
}
// versionVulnerable reports whether version falls inside any advisory
// range: version >= introduced AND (fixed == "" OR version < fixed).
// An empty/"0" introduced means "from the beginning".
func versionVulnerable(version string, ranges []supplyChainAdvisoryRange) bool {
for _, r := range ranges {
introOK := r.Introduced == "" || r.Introduced == "0" || semverCompare(version, r.Introduced) >= 0
fixedOK := r.Fixed == "" || semverCompare(version, r.Fixed) < 0
if introOK && fixedOK {
return true
}
}
return false
}
// semverCompare compares two dotted versions numerically segment by
// segment, tolerating a leading "v" and ignoring any pre-release/build
// suffix after the first "-" or "+". Returns -1, 0, or 1. Non-numeric
// segments compare as 0 so a malformed version never panics.
func semverCompare(a, b string) int {
as := semverSegments(a)
bs := semverSegments(b)
n := len(as)
if len(bs) > n {
n = len(bs)
}
for i := 0; i < n; i++ {
var av, bv int
if i < len(as) {
av = as[i]
}
if i < len(bs) {
bv = bs[i]
}
if av < bv {
return -1
}
if av > bv {
return 1
}
}
return 0
}
func semverSegments(v string) []int {
v = strings.TrimSpace(v)
v = strings.TrimPrefix(v, "v")
v = strings.TrimPrefix(v, "V")
if i := strings.IndexAny(v, "-+"); i >= 0 {
v = v[:i]
}
parts := strings.Split(v, ".")
out := make([]int, 0, len(parts))
for _, p := range parts {
n, err := strconv.Atoi(strings.TrimSpace(p))
if err != nil {
n = 0
}
out = append(out, n)
}
return out
}
func supplyChainFinding(account, lockPath string, p supplyChainPkg, adv supplyChainAdvisory) alert.Finding {
sev := alert.High
switch strings.ToLower(adv.Severity) {
case "critical":
sev = alert.Critical
case "low", "medium", "moderate", "":
sev = alert.Warning
}
id := adv.ID
if id == "" {
id = "advisory"
}
fixed := "no fixed version published"
for _, r := range adv.Ranges {
if r.Fixed != "" {
fixed = "fixed in " + r.Fixed
break
}
}
return alert.Finding{
Severity: sev,
Check: "supply_chain_vuln",
Message: fmt.Sprintf("Vulnerable %s dependency %s %s (%s) on account %s",
p.Ecosystem, p.Name, p.Version, id, account),
Details: fmt.Sprintf("Account: %s\nLockfile: %s\nEcosystem: %s\nPackage: %s\nVersion: %s\nAdvisory: %s\nSeverity: %s\nFix: %s\n%s",
account, lockPath, p.Ecosystem, p.Name, p.Version, id, adv.Severity, fixed, adv.Summary),
FilePath: lockPath,
Timestamp: time.Now(),
}
}
package checks
import (
"strconv"
"strings"
"time"
)
// syslogLineTime returns the time a syslog line was written, from either the
// traditional BSD prefix ("Sep 2 04:12:33", no year, local time) or an
// RFC 3339 prefix as rsyslog writes with high-precision timestamps. The BSD
// form uses the most recent plausible year; a result more than a day in the
// future belongs to an earlier year (a January read of December lines). ok is
// false when the line carries neither.
func syslogLineTime(line string, now time.Time) (time.Time, bool) {
fields := strings.Fields(line)
if len(fields) == 0 {
return time.Time{}, false
}
if t, err := time.Parse(time.RFC3339Nano, fields[0]); err == nil {
return t, true
}
if len(fields) < 3 || !isSyslogTimestampPrefix(fields) {
return time.Time{}, false
}
stamp := fields[0] + " " + fields[1] + " " + fields[2]
parseYear := func(year int) (time.Time, error) {
return time.ParseInLocation("2006 Jan _2 15:04:05", strconv.Itoa(year)+" "+stamp, now.Location())
}
// Eight years covers the largest gap between Gregorian leap years. This
// also handles a leap-day record read more than one year later without
// treating an unparseable timestamp as a current event.
for yearsAgo := 0; yearsAgo <= 8; yearsAgo++ {
t, err := parseYear(now.Year() - yearsAgo)
if err == nil && !t.After(now.Add(24*time.Hour)) {
return t, true
}
}
return time.Time{}, false
}
package checks
import (
"bufio"
"context"
"fmt"
"io"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
"syscall"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mysqlclient"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
)
// CheckKernelModules compares loaded kernel modules against baseline.
// All modules present at baseline time are considered known.
// Only modules loaded AFTER baseline trigger alerts.
func CheckKernelModules(ctx context.Context, _ *config.Config, store *state.Store) []alert.Finding {
var findings []alert.Finding
modules := loadModuleList()
if len(modules) == 0 {
return nil
}
// Check if baseline exists for kernel modules
_, baselineExists := store.GetRaw("_kmod_baseline_set")
if !baselineExists {
// First run - store all current modules as baseline
for _, mod := range modules {
store.SetRaw("_kmod:"+mod, "baseline")
}
store.SetRaw("_kmod_baseline_set", "true")
return nil
}
// Check for modules not seen at baseline
for _, mod := range modules {
key := "_kmod:" + mod
_, known := store.GetRaw(key)
if !known {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "kernel_module",
Message: fmt.Sprintf("New kernel module loaded after baseline: %s", mod),
Details: "This module was not present when CSM baseline was set. Verify it is legitimate.",
})
// Store it so we don't re-alert
store.SetRaw(key, "new")
}
}
return findings
}
func loadModuleList() []string {
f, err := osFS.Open("/proc/modules")
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
var modules []string
scanner := bufio.NewScanner(f)
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) >= 1 {
modules = append(modules, fields[0])
}
}
return modules
}
// CheckRPMIntegrity verifies critical system binaries haven't been modified.
// Only checks a small set of security-critical packages. Dispatches to
// rpm -V on RHEL-family systems and debsums/dpkg --verify on Debian family.
func CheckRPMIntegrity(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
info := platform.Detect()
switch {
case info.IsRHELFamily():
return checkRPMPackageIntegrity(rpmCriticalPackages)
case info.IsDebianFamily():
return checkDebianPackageIntegrity(debianCriticalPackages)
}
return nil
}
var rpmCriticalPackages = []string{
"openssh-server",
"shadow-utils",
"sudo",
"coreutils",
"util-linux",
"passwd",
}
var debianCriticalPackages = []string{
"openssh-server",
"passwd",
"sudo",
"coreutils",
"util-linux",
"login",
}
func checkRPMPackageIntegrity(packages []string) []alert.Finding {
var findings []alert.Finding
for _, pkg := range packages {
// rpm -V exits non-zero when it finds problems; treat that as
// "findings present" rather than command failure.
out, err := runCmdAllowNonZero("rpm", "-V", pkg)
if err != nil || out == nil {
continue
}
output := strings.TrimSpace(string(out))
if output == "" {
continue
}
// Parse rpm -V output: each line starts with flags
// S=size, 5=md5, T=mtime, etc. We care about S, 5, and M (mode)
for _, line := range strings.Split(output, "\n") {
if len(line) < 9 {
continue
}
flags := line[:9]
file := strings.TrimSpace(line[9:])
// Skip config files (c) and documentation (d)
if strings.Contains(line, " c ") || strings.Contains(line, " d ") {
continue
}
// Check for size (S) or checksum (5) changes. Config and doc files
// were already skipped above; report any tampered executable or
// shared library regardless of directory.
if strings.Contains(flags, "S") || strings.Contains(flags, "5") {
if looksExecutableOrLibrary(file) {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "rpm_integrity",
Message: fmt.Sprintf("Modified system binary or library: %s (package: %s)", file, pkg),
Details: fmt.Sprintf("RPM verification flags: %s", flags),
})
}
}
}
}
return findings
}
// checkDebianPackageIntegrity verifies Debian/Ubuntu packages using debsums
// (from the debsums package) when available, falling back to dpkg --verify
// (built into dpkg and always present).
func checkDebianPackageIntegrity(packages []string) []alert.Finding {
// Prefer debsums: it's the Debian equivalent of `rpm -V` and reports
// changed files vs. the md5sum shipped by the package.
if _, err := cmdExec.LookPath("debsums"); err == nil {
return checkDebsums(packages)
}
return checkDpkgVerify(packages)
}
func checkDebsums(packages []string) []alert.Finding {
var findings []alert.Finding
for _, pkg := range packages {
// debsums -c exits 2 when it finds mismatches; treat as findings.
out, err := runCmdAllowNonZero("debsums", "-c", pkg)
if err != nil || out == nil {
continue
}
for _, file := range strings.Split(strings.TrimSpace(string(out)), "\n") {
file = strings.TrimSpace(file)
if file == "" {
continue
}
if !looksExecutableOrLibrary(file) {
continue
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "dpkg_integrity",
Message: fmt.Sprintf("Modified system binary or library: %s (package: %s)", file, pkg),
Details: "debsums reported md5 mismatch against the package manifest.",
})
}
}
return findings
}
func checkDpkgVerify(packages []string) []alert.Finding {
var findings []alert.Finding
for _, pkg := range packages {
// dpkg --verify exits 1 when it finds mismatches; treat as findings.
out, err := runCmdAllowNonZero("dpkg", "--verify", pkg)
if err != nil || out == nil {
continue
}
// dpkg --verify prints lines like:
// ??5?????? /usr/bin/passwd
// where position 2 == '5' means md5 mismatch.
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
if len(line) < 10 {
continue
}
flags := line[:9]
file := strings.TrimSpace(line[9:])
// Skip config files (marked with 'c' after flags).
if strings.Contains(line, " c ") {
continue
}
if !strings.Contains(flags, "5") && !strings.Contains(flags, "S") {
continue
}
if !looksExecutableOrLibrary(file) {
continue
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "dpkg_integrity",
Message: fmt.Sprintf("Modified system binary or library: %s (package: %s)", file, pkg),
Details: fmt.Sprintf("dpkg --verify flags: %s", flags),
})
}
}
return findings
}
// looksExecutableOrLibrary reports whether the installed file at path is an
// executable or a shared library. A package-integrity mismatch on one of these
// is the threat we care about (a trojaned binary or .so), so it is reported
// wherever it lives. Judging by file type instead of a directory allowlist
// means an attacker cannot dodge the check by tampering a packaged binary that
// sits outside /usr/bin (e.g. under /usr/lib64, /usr/local, or /opt), while
// changed package-manager state files (manifests, caches, databases) -- which
// are not executable -- do not generate noise.
func looksExecutableOrLibrary(path string) bool {
info, err := osFS.Stat(path)
if err != nil || !info.Mode().IsRegular() {
return false
}
if info.Mode()&0o111 != 0 {
return true
}
// Shared libraries are commonly mode 0644; identify them by ELF magic so a
// trojaned .so is reported regardless of its permission bits.
f, err := osFS.Open(path)
if err != nil {
return false
}
defer func() { _ = f.Close() }()
var magic [4]byte
n, _ := io.ReadFull(f, magic[:])
return n == 4 && string(magic[:]) == "\x7fELF"
}
// CheckMySQLUsers queries for MySQL users with elevated privileges
// that aren't standard cPanel-managed users.
func CheckMySQLUsers(ctx context.Context, _ *config.Config, store *state.Store) []alert.Finding {
var findings []alert.Finding
rows, err := mysqlclient.RootQuery(ctx,
"SELECT user, host FROM mysql.user WHERE Super_priv='Y' AND user NOT IN ('root','mysql.session','mysql.sys','mysql.infoschema','debian-sys-maint')")
if err != nil {
return nil
}
if slices.Contains(rows, "mysql\tlocalhost") {
// Only MariaDB's stock socket account is exempt. mysql.user hides
// alternative auth plugins, so inspect the full definition without
// fetching password hashes. MySQL lacks global_priv; on any lookup
// failure the account remains in the audit for operator review.
stock, err := mysqlclient.RootQuery(ctx, stockMariaDBAccountQuery)
if err == nil && len(stock) == 1 && stock[0] == "1" {
rows = slices.DeleteFunc(rows, func(row string) bool { return row == "mysql\tlocalhost" })
}
}
sort.Strings(rows)
output := strings.TrimSpace(strings.Join(rows, "\n"))
out := []byte(output)
// Track known MySQL superusers
hash := hashBytes(out)
key := "_mysql_super_users"
prev, exists := store.GetRaw(key)
switch {
case !exists:
if output == "" {
break
}
// First run establishes the baseline. The query already excludes the
// standard cPanel/system superusers, so every account here is a
// non-standard privileged account. Baselining them silently would let
// a rogue superuser planted before CSM was installed (the
// CSM-installed-after-breach case) pass as "known" forever. Surface the
// pre-existing set for operator review instead.
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "mysql_superuser",
Message: "Non-standard MySQL superuser accounts present at baseline",
Details: fmt.Sprintf("Review these privileged accounts:\n%s", output),
})
case prev != hash:
details := "Current superusers: none"
if output != "" {
details = fmt.Sprintf("Current superusers:\n%s", output)
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "mysql_superuser",
Message: "MySQL superuser accounts changed",
Details: details,
})
}
store.SetRaw(key, hash)
return findings
}
// Match the authentication definition installed by mariadb-install-db:
// only the mysql OS user can log in, and password authentication is disabled.
// Extra alternatives or socket identity mappings must remain auditable.
const stockMariaDBAccountQuery = `SELECT 1 FROM mysql.global_priv
WHERE User='mysql' AND Host='localhost'
AND BINARY JSON_VALUE(Priv, '$.plugin') = 'mysql_native_password'
AND BINARY JSON_VALUE(Priv, '$.authentication_string') = 'invalid'
AND BINARY JSON_COMPACT(JSON_EXTRACT(Priv, '$.auth_or')) = '[{},{"plugin":"unix_socket"}]'`
// CheckGroupWritablePHP scans for PHP files that are group-writable
// where the group is the web server (nobody/www-data). This allows
// webshells to persist by the web server modifying PHP files.
func CheckGroupWritablePHP(ctx context.Context, _ *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
// Get web server group GIDs
webGroupGIDs := getWebServerGIDs()
if len(webGroupGIDs) == 0 {
return nil
}
homeDirs, _ := GetScanHomeDirs(ctx)
for _, homeEntry := range homeDirs {
if !homeEntry.IsDir() {
continue
}
docRoot := filepath.Join(scanHomeDirPath(homeEntry), "public_html")
scanGroupWritablePHP(docRoot, 4, webGroupGIDs, &findings)
}
return findings
}
func scanGroupWritablePHP(dir string, maxDepth int, webGIDs map[uint32]bool, findings *[]alert.Finding) {
if maxDepth <= 0 {
return
}
entries, err := osFS.ReadDir(dir)
if err != nil {
return
}
for _, entry := range entries {
name := entry.Name()
fullPath := dir + "/" + name
if entry.IsDir() {
scanGroupWritablePHP(fullPath, maxDepth-1, webGIDs, findings)
continue
}
if !isExecutablePHPName(strings.ToLower(name)) {
continue
}
info, err := entry.Info()
if err != nil {
continue
}
// Symlink mode bits describe the link, not whether PHP is writable.
if !info.Mode().IsRegular() {
continue
}
// Check group-write bit
if info.Mode()&0020 == 0 {
continue
}
// Check if group is web server
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
continue
}
if webGIDs[stat.Gid] {
*findings = append(*findings, alert.Finding{
Severity: alert.High,
Check: "group_writable_php",
Message: fmt.Sprintf("Web-server group-writable PHP: %s", fullPath),
Details: fmt.Sprintf("Mode: %s, GID: %d", info.Mode(), stat.Gid),
FilePath: fullPath,
})
}
}
}
// webServerGroupNames are group names that belong to a web server on any
// platform; the detected platform's own users are added at lookup time.
var webServerGroupNames = map[string]bool{
"nobody": true, "www-data": true, "apache": true, "www": true, "nginx": true,
}
func getWebServerGIDs() map[uint32]bool {
gids := make(map[uint32]bool)
data, err := osFS.ReadFile("/etc/group")
if err != nil {
return gids
}
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Split(line, ":")
if len(fields) < 3 {
continue
}
name := fields[0]
if webServerGroupNames[name] || slices.Contains(webServerUsers(), name) {
gid, parseErr := strconv.ParseUint(fields[2], 10, 32)
if parseErr != nil {
continue
}
gids[uint32(gid)] = true
}
}
return gids
}
package checks
import (
"io"
"os"
"github.com/pidginhost/csm/internal/jstaint"
"github.com/pidginhost/csm/internal/phptaint"
)
// jsTaintOversizePeekBytes bounds the prefix read that decides whether an
// oversize file could be JavaScript. It matches the PHP peek so both gates
// judge the same window of the same file.
const jsTaintOversizePeekBytes = 64 << 10
// taintSourcePrefixLooks reads a bounded prefix of a file too large to analyze
// and asks pred whether the content could be source of that language.
//
// Every failure answers yes. An unreadable file, a file swapped under us, a
// FIFO that would block: each is a file this scan could not examine, which is
// exactly what the coverage report exists to name. The failure direction that
// matters is the other one -- silently deciding a file was uninteresting and
// dropping it from the report.
func taintSourcePrefixLooks(path string, expected os.FileInfo, limit int64, pred func([]byte) bool) bool {
if reader, ok := osFS.(phpRegularFilePrefixReader); ok {
prefix, err := reader.ReadRegularFilePrefix(path, expected, limit)
if err != nil {
return true
}
return pred(prefix)
}
f, err := osFS.Open(path)
if err != nil {
return true
}
defer func() { _ = f.Close() }()
opened, err := f.Stat()
if err != nil || !opened.Mode().IsRegular() || !sameFileSnapshot(expected, opened) {
return true
}
prefix, err := io.ReadAll(io.LimitReader(f, limit))
if err != nil {
return true
}
after, err := f.Stat()
if err != nil || !sameFileSnapshot(opened, after) {
return true
}
return pred(prefix)
}
// jsFileMayBeJS reports whether a file too large to analyze nonetheless looks
// like JavaScript source, judged only by its leading bytes.
func jsFileMayBeJS(path string, expected os.FileInfo) bool {
return taintSourcePrefixLooks(path, expected, jsTaintOversizePeekBytes, jstaint.MayBeJSSource)
}
// phpFileMayBePHP reports whether a file too large to analyze nonetheless
// looks like PHP source, judged only by its leading bytes.
func phpFileMayBePHP(path string, expected os.FileInfo) bool {
return taintSourcePrefixLooks(path, expected, phpTaintOversizePeekBytes, phptaint.MayBePHPSource)
}
package checks
// SetAccountHomeRootsForTest points account-root discovery at roots and
// returns a function that restores the previous source. For tests only.
func SetAccountHomeRootsForTest(roots []string) (restore func()) {
prev := accountHomeRoots
accountHomeRoots = func() []string { return roots }
return func() { accountHomeRoots = prev }
}
// SetQuarantineDirForTest redirects quarantine moves to dir and returns a
// function that restores the previous directory. For tests only.
func SetQuarantineDirForTest(dir string) (restore func()) {
prev := quarantineDir
quarantineDir = dir
return func() { quarantineDir = prev }
}
// SetEximSpoolDirsForTest points Exim spool lookups at dirs and returns a
// function that restores the previous directories. For tests only.
func SetEximSpoolDirsForTest(dirs []string) (restore func()) {
prev := eximSpoolDirs
eximSpoolDirs = dirs
return func() { eximSpoolDirs = prev }
}
package checks
import (
"bufio"
"fmt"
"io"
"net"
"net/http"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/atomicio"
"github.com/pidginhost/csm/internal/store"
)
// ThreatDB is a local IP reputation database built from:
// 1. CSM's own block history (permanent)
// 2. Public threat intelligence feeds (updated daily)
// 3. AbuseIPDB as fallback for unknown IPs
type ThreatDB struct {
// writeMu orders local threat mutations across persistence and publication.
// Lookup and config reload use mu only; applied batches persist outside it.
writeMu sync.Mutex
mu sync.RWMutex
badIPs map[string]string // ip -> source/reason
badIPExpiry map[string]time.Time // ip -> temp-entry expiry; absent = permanent
badNets []*net.IPNet // flat CIDR list for Lookup, rebuilt from feedNets
feedIPs map[string]map[string]struct{} // feed name -> IPs, so overlapping feeds retain ownership
feedNets map[string][]*net.IPNet // feed name -> CIDRs, so a failed feed keeps coverage
whitelist map[string]bool // operator-managed (persisted) IPs to never flag
whitelistMeta map[string]*whitelistEntry // expiry metadata
// configWhitelist mirrors reputation.whitelist from csm.yaml. It is
// replaced wholesale on config reload and never persisted, so the
// file stays the source of truth for these entries.
configWhitelist map[string]bool
lastUpdate time.Time
dbPath string
// Stats for WebUI
PermanentCount int
FeedIPCount int
FeedNetCount int
LastFeedUpdate time.Time
LastUpdated time.Time // tracks when feeds were last successfully loaded
}
var (
globalThreatDB *ThreatDB
threatDBOnce sync.Once
)
// Minimum expected entries per feed - alerts if feed returns less (corrupted/down)
var feedMinEntries = map[string]int{
"spamhaus-drop": 50,
"blocklist-de": 1000,
"cins-army": 5000,
}
// Free public threat intelligence feeds
var threatFeeds = []struct {
name string
url string
}{
{"spamhaus-drop", "https://www.spamhaus.org/drop/drop.txt"},
{"blocklist-de", "https://lists.blocklist.de/lists/all.txt"},
{"cins-army", "https://cinsscore.com/list/ci-badguys.txt"},
}
// InitThreatDB initializes the global threat database.
func InitThreatDB(statePath string, whitelistIPs []string) *ThreatDB {
threatDBOnce.Do(func() {
db := &ThreatDB{
badIPs: make(map[string]string),
badIPExpiry: make(map[string]time.Time),
whitelist: make(map[string]bool),
configWhitelist: configWhitelistSet(whitelistIPs),
dbPath: filepath.Join(statePath, "threat_db"),
}
_ = os.MkdirAll(db.dbPath, 0700)
db.loadPermanentBlocklist()
db.loadPersistedWhitelist()
db.loadFeedCache()
globalThreatDB = db
})
return globalThreatDB
}
// GetThreatDB returns the global threat database.
func GetThreatDB() *ThreatDB {
return globalThreatDB
}
// SetGlobalThreatDBForTest installs a freshly-constructed threat DB rooted at
// statePath, bypassing the once-guard, and returns a function that restores the
// previous global. For tests only: lets a test exercise the threat-DB path
// without permanently polluting the global for order-dependent tests.
func SetGlobalThreatDBForTest(statePath string) func() {
prev := globalThreatDB
db := &ThreatDB{
badIPs: make(map[string]string),
badIPExpiry: make(map[string]time.Time),
whitelist: make(map[string]bool),
configWhitelist: make(map[string]bool),
dbPath: filepath.Join(statePath, "threat_db"),
}
_ = os.MkdirAll(db.dbPath, 0700)
globalThreatDB = db
return func() { globalThreatDB = prev }
}
// ClearGlobalThreatDBForTest unsets the global threat DB, as before the
// daemon initializes it, and returns a function that restores it.
func ClearGlobalThreatDBForTest() func() {
prev := globalThreatDB
globalThreatDB = nil
return func() { globalThreatDB = prev }
}
func configWhitelistSet(ips []string) map[string]bool {
set := make(map[string]bool, len(ips))
for _, raw := range ips {
if ip := net.ParseIP(strings.TrimSpace(raw)); ip != nil {
set[ip.String()] = true
}
}
return set
}
// SetConfigWhitelist replaces the entries that come from reputation.whitelist
// in csm.yaml. Called on config reload; the hot-reload path reported success
// for that field while lookups kept honouring the startup list. Operator-
// managed entries added at runtime are untouched.
func (db *ThreatDB) SetConfigWhitelist(ips []string) {
db.mu.Lock()
db.configWhitelist = configWhitelistSet(ips)
db.mu.Unlock()
}
// IsConfigWhitelisted reports whether ip is managed by reputation.whitelist.
// Runtime removal must not claim to remove these entries because the config
// remains authoritative and restores them on reload.
func (db *ThreatDB) IsConfigWhitelisted(ip string) bool {
db.mu.RLock()
defer db.mu.RUnlock()
return db.configWhitelist[ip]
}
// ThreatMatch describes why an IP is flagged: which source named it, and
// whether that evidence is permanent (re-flags the address on every future
// sighting) or lapses with the block that recorded it.
type ThreatMatch struct {
Source string
Permanent bool
ExpiresAt time.Time // zero unless the entry lapses
}
// Lookup checks if an IP is in the local threat database.
// Returns (source, true) if found, ("", false) if unknown.
// Whitelisted IPs always return false.
func (db *ThreatDB) Lookup(ip string) (string, bool) {
match, ok := db.LookupMatch(ip)
return match.Source, ok
}
// LookupMatch is Lookup plus the lifetime of the matched evidence, so the
// Web UI can explain why an unblocked IP still scores as malicious.
func (db *ThreatDB) LookupMatch(ip string) (ThreatMatch, bool) {
db.mu.RLock()
defer db.mu.RUnlock()
// Never flag whitelisted IPs
if db.whitelist[ip] || db.configWhitelist[ip] {
return ThreatMatch{}, false
}
// Check exact IP match. Lapsed temp entries no longer count as
// evidence: honouring them here is what turned every temporary
// auto-block into a forever re-flag loop. The leftover row is
// removed by the periodic prune; until then, fall through to the
// feed data and CIDR ranges.
if source, ok := db.badIPs[ip]; ok {
exp, hasExp := db.badIPExpiry[ip]
if !hasExp {
// Feed-owned entries are refreshed from upstream, so only local
// rows (operator or legacy) count as permanent local evidence.
return ThreatMatch{Source: source, Permanent: !isFeedSourceName(source)}, true
}
if time.Now().Before(exp) {
return ThreatMatch{Source: source, ExpiresAt: exp}, true
}
if feed, ok := db.feedSourceLocked(ip); ok {
return ThreatMatch{Source: feed}, true
}
}
// Check CIDR ranges (supports both IPv4 and IPv6)
parsed := net.ParseIP(ip)
if parsed != nil {
for _, cidr := range db.badNets {
if cidr.Contains(parsed) {
return ThreatMatch{Source: "threat-feed-cidr"}, true
}
}
}
return ThreatMatch{}, false
}
// AddPermanent adds an IP to the permanent local blocklist.
// Called for deliberate operator blocks - persists across restarts and
// never expires. Upgrades an existing temp entry to permanent.
func (db *ThreatDB) AddPermanent(ip, reason string) {
db.writeMu.Lock()
defer db.writeMu.Unlock()
db.mu.Lock()
_, exists := db.badIPs[ip]
_, wasTemp := db.badIPExpiry[ip]
_, wasFeed := db.feedSourceLocked(ip)
db.badIPs[ip] = reason
delete(db.badIPExpiry, ip)
db.mu.Unlock()
if sdb := store.Global(); sdb != nil {
_ = sdb.AddPermanentBlock(ip, reason)
return
}
// Only append to the flat-file fallback if this is a new local IP. A temp
// entry or feed-owned entry upgrading to operator evidence must still
// append, or the operator block would disappear after restart.
if exists && !wasTemp && !wasFeed {
return
}
// Fallback: flat-file permanent.txt.
f, err := os.OpenFile(filepath.Join(db.dbPath, "permanent.txt"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return
}
defer func() { _ = f.Close() }()
fmt.Fprintf(f, "%s # %s [%s]\n", ip, reason, time.Now().Format("2006-01-02"))
}
// AddTemporary records an auto-blocked IP for the lifetime of its firewall
// block. Unlike AddPermanent the entry lapses with the block: a permanent
// record turned every temporary auto-block into a forever "known malicious
// IP" that re-flagged (and re-blocked) the address on each later access.
// ttl <= 0 is ignored because auto-block evidence must never become a
// never-expiring threat row.
func (db *ThreatDB) AddTemporary(ip, reason string, ttl time.Duration) {
db.addExpiring(ip, reason, ttl, false, func(expiresAt time.Time) {
if sdb := store.Global(); sdb != nil {
_ = sdb.AddTempBlock(ip, reason, expiresAt)
}
})
}
// AddTemporaryUntil records one applied block through the batch consumer.
func (db *ThreatDB) AddTemporaryUntil(ip, reason string, expiresAt time.Time) error {
return db.AddTemporaryBatch([]store.PermanentBlockEntry{{IP: ip, Reason: reason, ExpiresAt: expiresAt}})
}
// persistAppliedThreats lets tests hold a durable write before publication.
var persistAppliedThreats = (*store.DB).AddAppliedTempBlocks
// AddTemporaryBatch publishes memory only after the batch's durable write.
// The outbox retries on error, including after a process restart.
func (db *ThreatDB) AddTemporaryBatch(entries []store.PermanentBlockEntry) error {
return db.AddTemporaryBatchFrom(func() []store.PermanentBlockEntry { return entries })
}
// AddTemporaryBatchFrom is AddTemporaryBatch over the entries choose returns
// under the writer lock. A caller that chooses from the firewall state reads
// it there: a removal after that read waits for the commit and then removes
// what the batch wrote.
func (db *ThreatDB) AddTemporaryBatchFrom(choose func() []store.PermanentBlockEntry) error {
db.writeMu.Lock()
defer db.writeMu.Unlock()
entries := choose()
if sdb := store.Global(); sdb != nil {
if err := persistAppliedThreats(sdb, entries); err != nil {
return err
}
}
db.mu.Lock()
defer db.mu.Unlock()
for _, entry := range entries {
ip, reason, expiresAt := entry.IP, entry.Reason, entry.ExpiresAt
if !time.Now().Before(expiresAt) {
continue
}
if source, exists := db.badIPs[ip]; exists {
cur, isTemp := db.badIPExpiry[ip]
// Keep local permanent evidence, but retain the applied row
// independently of feeds that may withdraw this address.
if (!isTemp && !isFeedSourceName(source)) || cur.After(expiresAt) {
continue
}
// Reloaded evidence uses this source label. Replaying its
// outcome must not change the in-memory view either.
if cur.Equal(expiresAt) && (source == reason || source == "permanent-blocklist") {
continue
}
}
db.badIPs[ip] = reason
db.badIPExpiry[ip] = expiresAt
}
return nil
}
// AddOperatorTemporary records a timed operator block (the Web UI 24h block)
// for the lifetime of its firewall block. The evidence is operator-sourced
// but lapses with the block, so a mistaken 24h block of a customer address
// does not leave it permanently malicious.
func (db *ThreatDB) AddOperatorTemporary(ip, reason string, ttl time.Duration) {
db.addExpiring(ip, reason, ttl, true, func(expiresAt time.Time) {
if sdb := store.Global(); sdb != nil {
_ = sdb.AddOperatorTempBlock(ip, reason, expiresAt)
}
})
}
func (db *ThreatDB) addExpiring(ip, reason string, ttl time.Duration, operator bool, persist func(time.Time)) {
db.writeMu.Lock()
defer db.writeMu.Unlock()
if ttl <= 0 {
return
}
expiresAt := time.Now().Add(ttl)
db.mu.Lock()
if source, exists := db.badIPs[ip]; exists {
cur, isTemp := db.badIPExpiry[ip]
if !isTemp && (!operator || !isFeedSourceName(source)) {
// Keep permanent local evidence. Operator decisions must also
// survive feed withdrawal; feedIPs retains independent coverage.
db.mu.Unlock()
return
}
if time.Now().Before(cur) && cur.After(expiresAt) {
// Keep the longer live window; re-blocks extend, never truncate.
db.mu.Unlock()
return
}
}
db.badIPs[ip] = reason
db.badIPExpiry[ip] = expiresAt
db.mu.Unlock()
persist(expiresAt)
// Flat-file fallback (pre-migration) deliberately does not persist:
// permanent.txt has no expiry column, so a line there would recreate
// the forever-row this method exists to avoid. The firewall keeps the
// block itself across restarts.
}
// RemoveTemporary removes evidence that only lives as long as a firewall
// block: auto-block rows and timed operator blocks. It leaves permanent
// evidence untouched and restores feed ownership when the IP is
// independently present in a threat feed.
func (db *ThreatDB) RemoveTemporary(ip string) {
db.writeMu.Lock()
defer db.writeMu.Unlock()
db.mu.Lock()
if _, isTemp := db.badIPExpiry[ip]; isTemp {
delete(db.badIPExpiry, ip)
if feed, ok := db.feedSourceLocked(ip); ok {
db.badIPs[ip] = feed
} else {
delete(db.badIPs, ip)
}
}
db.mu.Unlock()
}
// RemoveStoredTemporaryBlock removes ip's auto-block store row under the
// threat writer lock. An unblock removes the firewall row first, so an
// applied batch that read the row as standing commits before this removal
// and cannot write the threat row back after it.
func RemoveStoredTemporaryBlock(sdb *store.DB, ip string) error {
if tdb := GetThreatDB(); tdb != nil {
tdb.writeMu.Lock()
defer tdb.writeMu.Unlock()
}
_, err := sdb.RemoveTemporaryBlock(ip)
return err
}
// DropAutoBlockThreatRow removes the auto-block threat row for ip after an
// operator unblock. Operator permanent blocks are left in place: a
// firewall-only unblock must not silently clear a deliberate block. A stale
// auto-block row that outlived the firewall block would let ip_reputation
// re-flag the IP into a new block loop, so both the persisted row and the
// in-memory temporary copy go.
func DropAutoBlockThreatRow(ip string) {
if parsed := net.ParseIP(ip); parsed != nil {
ip = parsed.String()
}
if sdb := store.Global(); sdb != nil {
_ = RemoveStoredTemporaryBlock(sdb, ip)
}
if tdb := GetThreatDB(); tdb != nil {
tdb.RemoveTemporary(ip)
}
}
// RemovePermanent removes an IP from the permanent blocklist and in-memory DB.
func (db *ThreatDB) RemovePermanent(ip string) {
db.writeMu.Lock()
defer db.writeMu.Unlock()
db.mu.Lock()
delete(db.badIPExpiry, ip)
if feed, ok := db.feedSourceLocked(ip); ok {
db.badIPs[ip] = feed
} else {
delete(db.badIPs, ip)
}
db.mu.Unlock()
if sdb := store.Global(); sdb != nil {
_ = sdb.RemovePermanentBlock(ip)
return
}
// Fallback: rewrite permanent.txt without this IP.
path := filepath.Join(db.dbPath, "permanent.txt")
data, err := osFS.ReadFile(path)
if err != nil {
return
}
var kept []string
for _, line := range strings.Split(string(data), "\n") {
trimmed := strings.TrimSpace(line)
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
kept = append(kept, line)
continue
}
fields := strings.Fields(trimmed)
if len(fields) > 0 && fields[0] == ip {
continue // skip this IP
}
kept = append(kept, line)
}
tmpPath := path + ".tmp"
_ = os.WriteFile(tmpPath, []byte(strings.Join(kept, "\n")+"\n"), 0600)
_ = os.Rename(tmpPath, path)
}
// whitelistEntry tracks an IP with optional expiry.
type whitelistEntry struct {
ExpiresAt time.Time // zero = permanent
}
// AddWhitelist adds an IP to the permanent whitelist.
func (db *ThreatDB) AddWhitelist(ip string) {
db.addWhitelistEntry(ip, time.Time{})
}
// TempWhitelist adds an IP to the whitelist with a TTL.
func (db *ThreatDB) TempWhitelist(ip string, ttl time.Duration) {
db.addWhitelistEntry(ip, time.Now().Add(ttl))
}
func (db *ThreatDB) addWhitelistEntry(ip string, expiresAt time.Time) {
db.mu.Lock()
db.whitelist[ip] = true
if db.whitelistMeta == nil {
db.whitelistMeta = make(map[string]*whitelistEntry)
}
db.whitelistMeta[ip] = &whitelistEntry{ExpiresAt: expiresAt}
delete(db.badIPs, ip)
delete(db.badIPExpiry, ip)
db.mu.Unlock()
if sdb := store.Global(); sdb != nil {
permanent := expiresAt.IsZero()
_ = sdb.AddWhitelistEntry(ip, expiresAt, permanent)
return
}
db.saveWhitelistFile()
}
// RemoveWhitelist removes an operator-managed whitelist entry. Config-managed
// entries are replaced only by SetConfigWhitelist.
func (db *ThreatDB) RemoveWhitelist(ip string) {
db.mu.Lock()
delete(db.whitelist, ip)
delete(db.whitelistMeta, ip)
db.mu.Unlock()
if sdb := store.Global(); sdb != nil {
_ = sdb.RemoveWhitelistEntry(ip)
return
}
db.saveWhitelistFile()
}
// PruneExpiredWhitelist removes expired temporary whitelist entries.
// Called periodically from the daemon heartbeat.
func (db *ThreatDB) PruneExpiredWhitelist() int {
now := time.Now()
pruned := 0
db.mu.Lock()
for ip, entry := range db.whitelistMeta {
if !entry.ExpiresAt.IsZero() && now.After(entry.ExpiresAt) {
delete(db.whitelist, ip)
delete(db.whitelistMeta, ip)
pruned++
}
}
db.mu.Unlock()
if pruned > 0 {
if sdb := store.Global(); sdb != nil {
sdb.PruneExpiredWhitelist()
} else {
db.saveWhitelistFile()
}
fmt.Fprintf(os.Stderr, "[%s] Pruned %d expired whitelist entries\n",
time.Now().Format("2006-01-02 15:04:05"), pruned)
}
return pruned
}
// PruneExpiredThreats removes threat entries whose temp lifetime has
// lapsed, in memory and in the persistent store (which also drops legacy
// no-source auto-block rows written before expiry tagging existed).
// Called periodically from the daemon heartbeat; this is by-design
// lifecycle cleanup, not data retention, so it does not sit behind the
// opt-in retention sweeps.
func (db *ThreatDB) PruneExpiredThreats() int {
db.writeMu.Lock()
defer db.writeMu.Unlock()
now := time.Now()
pruned := 0
db.mu.Lock()
for ip, exp := range db.badIPExpiry {
if now.Before(exp) {
continue
}
delete(db.badIPExpiry, ip)
// A feed may list the IP independently; hand the entry back to
// the feed instead of dropping it until the next feed rebuild.
if feed, ok := db.feedSourceLocked(ip); ok {
db.badIPs[ip] = feed
} else {
delete(db.badIPs, ip)
}
pruned++
}
db.mu.Unlock()
removed := pruned
if sdb := store.Global(); sdb != nil {
// The store count is authoritative: it also covers rows never
// loaded into memory (expired rows skipped at startup and legacy
// auto-block rows).
removed = sdb.PruneExpiredThreats()
}
if removed > 0 {
fmt.Fprintf(os.Stderr, "[%s] Pruned %d expired threat entries\n",
time.Now().Format("2006-01-02 15:04:05"), removed)
}
return removed
}
// isFeedSourceName reports whether a match source names a threat feed
// rather than local evidence.
func isFeedSourceName(source string) bool {
if source == "threat-feed-cidr" {
return true
}
for _, feed := range threatFeeds {
if feed.name == source {
return true
}
}
return false
}
// SetThreatFeedForTest replaces one feed's addresses and networks in the
// global threat DB. For tests only, like SetGlobalThreatDBForTest.
func SetThreatFeedForTest(name string, ips, nets []string) {
db := globalThreatDB
db.mu.Lock()
defer db.mu.Unlock()
if db.feedIPs == nil {
db.feedIPs = make(map[string]map[string]struct{})
}
if db.feedNets == nil {
db.feedNets = make(map[string][]*net.IPNet)
}
db.feedIPs[name] = make(map[string]struct{}, len(ips))
for _, ip := range ips {
db.feedIPs[name][ip] = struct{}{}
}
db.feedNets[name] = nil
for _, raw := range nets {
if _, n, err := net.ParseCIDR(raw); err == nil {
db.feedNets[name] = append(db.feedNets[name], n)
}
}
}
// IsWhitelisted reports whether ip is on the operator, temporary or config
// whitelist. Expired temporary entries no longer count, as in MapSources.
func (db *ThreatDB) IsWhitelisted(ip string) bool {
db.mu.RLock()
defer db.mu.RUnlock()
if db.configWhitelist[ip] {
return true
}
if !db.whitelist[ip] {
return false
}
entry := db.whitelistMeta[ip]
return entry == nil || entry.ExpiresAt.IsZero() || time.Now().Before(entry.ExpiresAt)
}
// ThreatMapSources is what the CSM IP map takes from the threat DB:
// CSM's permanent block history, each feed's addresses and
// networks by feed name, and every currently whitelisted address. All sorted copies.
type ThreatMapSources struct {
Permanent []string
FeedIPs map[string][]string
FeedNets map[string][]string
Whitelist []string
}
// MapSources copies the entries the CSM IP map is built from.
func (db *ThreatDB) MapSources() ThreatMapSources {
db.mu.RLock()
defer db.mu.RUnlock()
out := ThreatMapSources{FeedIPs: map[string][]string{}, FeedNets: map[string][]string{}}
for ip, source := range db.badIPs {
if _, temporary := db.badIPExpiry[ip]; !temporary && !isFeedSourceName(source) {
out.Permanent = append(out.Permanent, ip)
}
}
for name, ips := range db.feedIPs {
for ip := range ips {
out.FeedIPs[name] = append(out.FeedIPs[name], ip)
}
slices.Sort(out.FeedIPs[name])
}
for name, nets := range db.feedNets {
for _, n := range nets {
out.FeedNets[name] = append(out.FeedNets[name], n.String())
}
slices.Sort(out.FeedNets[name])
}
// A snapshot may outlive the next cleanup, so expired temporary entries
// must not become exceptions in a newly built map.
now := time.Now()
for ip := range db.whitelist {
entry := db.whitelistMeta[ip]
if entry != nil && !entry.ExpiresAt.IsZero() && !now.Before(entry.ExpiresAt) {
continue
}
if !db.configWhitelist[ip] {
out.Whitelist = append(out.Whitelist, ip)
}
}
for ip := range db.configWhitelist {
out.Whitelist = append(out.Whitelist, ip)
}
slices.Sort(out.Permanent)
slices.Sort(out.Whitelist)
return out
}
// feedSourceLocked reports which feed lists ip, if any. Caller holds db.mu.
func (db *ThreatDB) feedSourceLocked(ip string) (string, bool) {
for _, feed := range threatFeeds {
if _, ok := db.feedIPs[feed.name][ip]; ok {
return feed.name, true
}
}
return "", false
}
// WhitelistInfo returns all whitelisted IPs with their expiry info.
type WhitelistIP struct {
IP string `json:"ip"`
ExpiresAt *time.Time `json:"expires_at,omitempty"` // nil = permanent
Permanent bool `json:"permanent"`
Configured bool `json:"configured,omitempty"`
}
func (db *ThreatDB) WhitelistedIPs() []WhitelistIP {
db.mu.RLock()
defer db.mu.RUnlock()
var ips []string
for ip := range db.whitelist {
ips = append(ips, ip)
}
for ip := range db.configWhitelist {
if !db.whitelist[ip] {
ips = append(ips, ip)
}
}
sort.Strings(ips)
result := make([]WhitelistIP, len(ips))
for i, ip := range ips {
w := WhitelistIP{IP: ip, Configured: db.configWhitelist[ip]}
if db.whitelist[ip] {
entry := db.whitelistMeta[ip]
w.Permanent = entry == nil || entry.ExpiresAt.IsZero()
if entry != nil && !entry.ExpiresAt.IsZero() {
t := entry.ExpiresAt
w.ExpiresAt = &t
}
}
result[i] = w
}
return result
}
func (db *ThreatDB) saveWhitelistFile() {
path := filepath.Join(db.dbPath, "whitelist.txt")
db.mu.RLock()
var lines []string
for ip := range db.whitelist {
entry := db.whitelistMeta[ip]
if entry != nil && !entry.ExpiresAt.IsZero() {
lines = append(lines, fmt.Sprintf("%s expires=%s", ip, entry.ExpiresAt.Format(time.RFC3339)))
} else {
lines = append(lines, fmt.Sprintf("%s permanent", ip))
}
}
db.mu.RUnlock()
sort.Strings(lines)
tmpPath := path + ".tmp"
_ = os.WriteFile(tmpPath, []byte(strings.Join(lines, "\n")+"\n"), 0600)
_ = os.Rename(tmpPath, path)
}
// loadPersistedWhitelist loads IPs from the bbolt store (if available)
// or from the flat-file whitelist.txt.
func (db *ThreatDB) loadPersistedWhitelist() {
if db.whitelistMeta == nil {
db.whitelistMeta = make(map[string]*whitelistEntry)
}
if sdb := store.Global(); sdb != nil {
entries := sdb.ListWhitelist()
now := time.Now()
for _, e := range entries {
// Skip expired entries
if !e.Permanent && !e.ExpiresAt.IsZero() && now.After(e.ExpiresAt) {
continue
}
db.whitelist[e.IP] = true
db.whitelistMeta[e.IP] = &whitelistEntry{ExpiresAt: e.ExpiresAt}
delete(db.badIPs, e.IP)
}
return
}
// Fallback: flat-file whitelist.txt.
path := filepath.Join(db.dbPath, "whitelist.txt")
f, err := osFS.Open(path)
if err != nil {
return
}
defer func() { _ = f.Close() }()
now := time.Now()
needsRewrite := false
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
ip := fields[0]
if net.ParseIP(ip) == nil {
continue
}
entry := &whitelistEntry{}
// Parse "expires=2026-03-28T19:00:00Z" if present
for _, f := range fields[1:] {
if strings.HasPrefix(f, "expires=") {
if t, err := time.Parse(time.RFC3339, f[8:]); err == nil {
entry.ExpiresAt = t
}
}
}
// Skip expired entries
if !entry.ExpiresAt.IsZero() && now.After(entry.ExpiresAt) {
needsRewrite = true
continue
}
db.whitelist[ip] = true
db.whitelistMeta[ip] = entry
delete(db.badIPs, ip)
}
if needsRewrite {
// Synchronous: the load path runs once at startup, so the cost
// is negligible, and a fire-and-forget goroutine would race the
// daemon's shutdown (potentially leaving a `.tmp` file behind or
// writing a half-serialized whitelist.txt if the process is
// killed before the rewrite lands).
db.saveWhitelistFile()
}
}
// Count returns the total number of entries in the database.
func (db *ThreatDB) Count() int {
db.mu.RLock()
defer db.mu.RUnlock()
return len(db.badIPs) + len(db.badNets)
}
// Stats returns statistics for the WebUI dashboard.
func (db *ThreatDB) Stats() map[string]interface{} {
db.mu.RLock()
defer db.mu.RUnlock()
stats := map[string]interface{}{
"permanent_ips": db.PermanentCount,
"feed_ips": db.FeedIPCount,
"feed_cidrs": db.FeedNetCount,
"total": len(db.badIPs) + len(db.badNets),
"whitelist": db.whitelistCountLocked(),
}
if !db.LastFeedUpdate.IsZero() {
stats["last_update"] = db.LastFeedUpdate
}
return stats
}
func (db *ThreatDB) whitelistCountLocked() int {
count := len(db.whitelist)
for ip := range db.configWhitelist {
if !db.whitelist[ip] {
count++
}
}
return count
}
// LastFeedRefresh returns when feeds last loaded successfully, preferring
// the in-memory timestamp over the persisted one. Zero means never.
func (db *ThreatDB) LastFeedRefresh() time.Time {
db.mu.RLock()
defer db.mu.RUnlock()
if !db.LastUpdated.IsZero() {
return db.LastUpdated
}
return db.lastUpdate
}
// FeedsStale returns true if threat feeds have not been updated in over 7 days.
func (db *ThreatDB) FeedsStale() bool {
db.mu.RLock()
defer db.mu.RUnlock()
lastRefresh := db.LastUpdated
if lastRefresh.IsZero() {
lastRefresh = db.lastUpdate
}
return lastRefresh.IsZero() || feedRefreshStale(lastRefresh, time.Now())
}
// UpdateFeeds downloads fresh threat intelligence feeds.
// Downloads outside the lock, then swaps data under lock to avoid blocking
// lookups. Each feed is swapped independently: a feed that fails to download
// (or fails validation) keeps its previously loaded IPs and CIDRs, so a
// transient outage never wipes that feed's coverage. lastUpdate only advances
// when at least one feed succeeded, otherwise the next cycle would skip the
// retry and serve zero feed data for the whole 20h window.
func (db *ThreatDB) UpdateFeeds() error {
db.mu.RLock()
lastUpdate := db.lastUpdate
db.mu.RUnlock()
// Only update once per day. Strip monotonic readings so suspend time is
// counted, and retry immediately if the wall clock moved behind the last
// refresh instead of suppressing downloads until it catches up.
now := time.Now().Round(0)
lastUpdate = lastUpdate.Round(0)
if !lastUpdate.IsZero() && !lastUpdate.After(now) && now.Sub(lastUpdate) < 20*time.Hour {
return nil
}
client := &http.Client{Timeout: 30 * time.Second}
// Download all feeds OUTSIDE the lock
type feedResult struct {
ips map[string]struct{}
nets []*net.IPNet
}
fresh := make(map[string]feedResult)
for _, feed := range threatFeeds {
ips, nets, err := downloadFeed(client, feed.url)
if err != nil {
fmt.Fprintf(os.Stderr, "threatdb: error downloading %s: %v\n", feed.name, err)
continue // keep previous data for this feed
}
// Validate feed - reject partial downloads to avoid losing good data
minExpected := feedMinEntries[feed.name]
if minExpected > 0 && len(ips)+len(nets) < minExpected {
fmt.Fprintf(os.Stderr, "threatdb: WARNING %s returned only %d entries (expected >%d), keeping cached version\n",
feed.name, len(ips)+len(nets), minExpected)
continue // keep previous cached data for this feed
}
ipSet := make(map[string]struct{}, len(ips))
for _, ip := range ips {
ipSet[ip] = struct{}{}
}
fresh[feed.name] = feedResult{ips: ipSet, nets: nets}
// Cache to disk. IPs and CIDRs share one file: CIDR lines are
// distinguishable by "/", so the nets survive a daemon restart
// and legacy IP-only cache files keep loading unchanged.
lines := make([]string, 0, len(ipSet)+len(nets))
for ip := range ipSet {
lines = append(lines, ip)
}
for _, n := range nets {
lines = append(lines, n.String())
}
saveLines(filepath.Join(db.dbPath, feed.name+".txt"), lines)
}
if len(fresh) == 0 {
// Leave lastUpdate untouched so the next cycle retries instead of
// sitting on stale (or empty) data until the skip window expires.
return fmt.Errorf("threatdb: all %d feeds failed, keeping previous data", len(threatFeeds))
}
feedNames := make(map[string]bool, len(threatFeeds))
for _, feed := range threatFeeds {
feedNames[feed.name] = true
}
// Swap data UNDER the lock - fast operation
db.mu.Lock()
if db.feedIPs == nil {
db.feedIPs = make(map[string]map[string]struct{})
for ip, source := range db.badIPs {
if feedNames[source] {
if db.feedIPs[source] == nil {
db.feedIPs[source] = make(map[string]struct{})
}
db.feedIPs[source][ip] = struct{}{}
}
}
}
if db.feedNets == nil {
db.feedNets = make(map[string][]*net.IPNet)
}
for name, result := range fresh {
// Replace only this feed's previous entries; failed feeds keep
// their per-feed ownership and are merged back into badIPs below.
db.feedIPs[name] = result.ips
db.feedNets[name] = result.nets
}
totalIPs, totalNets := db.rebuildFeedLookup(feedNames)
now = time.Now()
db.lastUpdate = now
db.FeedIPCount = totalIPs
db.FeedNetCount = totalNets
db.LastFeedUpdate = now
db.LastUpdated = now
db.mu.Unlock()
// Save timestamp
_ = os.WriteFile(filepath.Join(db.dbPath, "last_update"),
[]byte(now.Format(time.RFC3339)), 0600)
fmt.Fprintf(os.Stderr, "threatdb: updated %d IPs + %d CIDR ranges (%d/%d feeds succeeded)\n",
totalIPs, totalNets, len(fresh), len(threatFeeds))
return nil
}
// loadPermanentBlocklist loads the permanent blocklist from bbolt store
// (if available) or from the flat-file permanent.txt.
func (db *ThreatDB) loadPermanentBlocklist() {
if sdb := store.Global(); sdb != nil {
blocks := sdb.AllPermanentBlocks()
now := time.Now()
live := 0
for _, b := range blocks {
// Expired rows stay out of memory, including legacy no-source
// auto-block rows (store classifies them in Expired): loading
// them would resume the permablock loop on upgraded hosts. The
// periodic prune deletes them from disk.
if b.Expired(now) {
continue
}
db.badIPs[b.IP] = "permanent-blocklist"
if !b.ExpiresAt.IsZero() {
db.badIPExpiry[b.IP] = b.ExpiresAt
}
live++
}
db.PermanentCount = live
return
}
// Fallback: flat-file permanent.txt.
path := filepath.Join(db.dbPath, "permanent.txt")
f, err := osFS.Open(path)
if err != nil {
return
}
defer func() { _ = f.Close() }()
seen := make(map[string]bool)
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
ip := strings.Fields(line)[0]
// Support both IPv4 and IPv6
if net.ParseIP(ip) != nil && !seen[ip] {
db.badIPs[ip] = "permanent-blocklist"
seen[ip] = true
}
}
db.PermanentCount = len(seen)
// Compact the file if it has duplicates (rewrite with unique entries)
if db.PermanentCount > 0 {
compactPermanentFile(path, seen)
}
}
// compactPermanentFile rewrites the permanent blocklist with unique entries only.
func compactPermanentFile(path string, uniqueIPs map[string]bool) {
// Read all lines to preserve comments/reasons
data, err := osFS.ReadFile(path)
if err != nil {
return
}
lines := strings.Split(string(data), "\n")
if len(lines) <= len(uniqueIPs)+5 {
return // not worth compacting - minimal duplicates
}
// Rewrite with deduplication
seen := make(map[string]bool)
var unique []string
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
unique = append(unique, line)
continue
}
ip := strings.Fields(trimmed)[0]
if !seen[ip] {
seen[ip] = true
unique = append(unique, line)
}
}
tmpPath := path + ".tmp"
_ = os.WriteFile(tmpPath, []byte(strings.Join(unique, "\n")+"\n"), 0600)
_ = os.Rename(tmpPath, path)
}
// loadFeedCache loads cached feed data from disk.
func (db *ThreatDB) loadFeedCache() {
data, err := osFS.ReadFile(filepath.Join(db.dbPath, "last_update"))
if err == nil {
if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(data))); err == nil {
db.lastUpdate = t
db.LastFeedUpdate = t
db.LastUpdated = t
}
}
if db.feedNets == nil {
db.feedNets = make(map[string][]*net.IPNet)
}
if db.feedIPs == nil {
db.feedIPs = make(map[string]map[string]struct{})
}
feedNames := make(map[string]bool, len(threatFeeds))
incomplete := false
for _, feed := range threatFeeds {
feedNames[feed.name] = true
db.feedIPs[feed.name] = make(map[string]struct{})
db.feedNets[feed.name] = nil
cachePath := filepath.Join(db.dbPath, feed.name+".txt")
_, statErr := osFS.Stat(cachePath)
cacheExists := statErr == nil
for _, line := range loadLines(cachePath) {
// Cache files mix plain IPs and CIDR lines ("/" marks a
// CIDR); legacy IP-only files parse the same way.
if strings.Contains(line, "/") {
if _, cidr, err := net.ParseCIDR(line); err == nil {
db.feedNets[feed.name] = append(db.feedNets[feed.name], cidr)
}
continue
}
db.feedIPs[feed.name][line] = struct{}{}
}
// The download path refuses a feed below its floor; a cache below it
// is a truncated write, not a smaller feed. Serve nothing from it and
// drop the update marker so the next cycle downloads instead of
// honouring the 20-hour skip. A feed with no cache at all (never
// fetched, or every download refused) is not truncated and must not
// force a refresh of the others.
if minExpected := feedMinEntries[feed.name]; cacheExists && minExpected > 0 {
if n := len(db.feedIPs[feed.name]) + len(db.feedNets[feed.name]); n < minExpected {
if n > 0 {
fmt.Fprintf(os.Stderr, "threatdb: WARNING cached %s holds only %d entries (expected >%d); ignoring it until refreshed\n", feed.name, n, minExpected)
}
db.feedIPs[feed.name] = make(map[string]struct{})
db.feedNets[feed.name] = nil
incomplete = true
}
}
}
if incomplete {
db.lastUpdate = time.Time{}
}
db.FeedIPCount, db.FeedNetCount = db.rebuildFeedLookup(feedNames)
// Warn on startup if feeds are stale
if db.LastUpdated.IsZero() && db.FeedIPCount == 0 && db.FeedNetCount == 0 {
fmt.Fprintf(os.Stderr, "threatdb: WARNING no threat feed data loaded, feeds have never been fetched\n")
} else if !db.LastUpdated.IsZero() && time.Since(db.LastUpdated) > 7*24*time.Hour {
fmt.Fprintf(os.Stderr, "threatdb: WARNING threat feeds are stale (last updated %s, %d days ago)\n",
db.LastUpdated.Format("2006-01-02"), int(time.Since(db.LastUpdated).Hours()/24))
}
}
// rebuildFeedLookup rebuilds the flat lookup maps from per-feed data. The
// caller must hold db.mu, or be in the startup load path before publication.
func (db *ThreatDB) rebuildFeedLookup(feedNames map[string]bool) (int, int) {
preserved := make(map[string]string, len(db.badIPs))
for ip, source := range db.badIPs {
if feedNames[source] {
continue
}
preserved[ip] = source
}
db.badIPs = preserved
totalIPs := 0
for _, feed := range threatFeeds {
for ip := range db.feedIPs[feed.name] {
totalIPs++
if _, exists := db.badIPs[ip]; !exists {
db.badIPs[ip] = feed.name
}
}
}
var nets []*net.IPNet
for _, feed := range threatFeeds {
nets = append(nets, db.feedNets[feed.name]...)
}
db.badNets = nets
return totalIPs, len(nets)
}
func downloadFeed(client *http.Client, url string) ([]string, []*net.IPNet, error) {
resp, err := client.Get(url)
if err != nil {
return nil, nil, err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != 200 {
return nil, nil, fmt.Errorf("HTTP %d", resp.StatusCode)
}
limited := io.LimitReader(resp.Body, 10*1024*1024)
scanner := bufio.NewScanner(limited)
var ips []string
var nets []*net.IPNet
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") {
continue
}
if idx := strings.IndexAny(line, ";#"); idx > 0 {
line = strings.TrimSpace(line[:idx])
}
if strings.Contains(line, "/") {
_, cidr, err := net.ParseCIDR(line)
if err == nil {
nets = append(nets, cidr)
}
continue
}
ip := strings.Fields(line)[0]
if net.ParseIP(ip) != nil {
ips = append(ips, ip)
}
}
return ips, nets, nil
}
func saveLines(path string, lines []string) {
sort.Strings(lines) // sorted for diffing
// Written whole and renamed into place: an in-place truncate left a
// crash mid-write serving a partial feed for the next 20 hours.
data := strings.Join(lines, "\n")
if len(lines) > 0 {
data += "\n"
}
if err := atomicio.AtomicWrite(path, 0o600, []byte(data)); err != nil {
fmt.Fprintf(os.Stderr, "threatdb: error saving %s: %v\n", path, err)
}
}
func loadLines(path string) []string {
f, err := osFS.Open(path)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
var lines []string
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" && !strings.HasPrefix(line, "#") {
lines = append(lines, line)
}
}
return lines
}
package checks
import (
"bytes"
"context"
"fmt"
"path/filepath"
"regexp"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// timThumbFixedVersion is the last TimThumb release. The project was abandoned
// in 2014; anything below this carries the CVE-2011-4106 remote-code-execution
// bug that let attackers write PHP into the image cache -- the entry vector in
// the 2026-07-20 cross-account compromise.
const timThumbFixedVersion = "2.8.14"
// timThumbReadHead bounds how much of each candidate file is read: the version
// define and feature constants are always near the top.
const timThumbReadHead = 16 * 1024
// timThumbScanDepth bounds the per-docroot recursive descent. TimThumb ships
// bundled in themes, sometimes several directories deep (framework/scripts).
const timThumbScanDepth = 10
var timThumbVersionRE = timThumbDefineMatcher("VERSION", `['"]([0-9]+(?:\.[0-9]+)*)['"]`)
// timThumbFeatureRE holds the boolean feature constants that make a copy
// directly exploitable, compiled once rather than per scanned file.
var timThumbFeatureRE = map[string]*regexp.Regexp{
timThumbWebshot: timThumbFeatureMatcher(timThumbWebshot),
timThumbExternal: timThumbFeatureMatcher(timThumbExternal),
timThumbAllExternal: timThumbFeatureMatcher(timThumbAllExternal),
}
const (
timThumbWebshot = "WEBSHOT_ENABLED"
timThumbExternal = "ALLOW_EXTERNAL"
timThumbAllExternal = "ALLOW_ALL_EXTERNAL_SITES"
)
// timThumbFeatureMatcher builds the define('NAME', true) matcher for a feature
// constant.
func timThumbFeatureMatcher(name string) *regexp.Regexp {
return timThumbDefineMatcher(name, `(true)`)
}
// timThumbFeatureDisabledRE matches define('NAME', false). Absence of a define
// is deliberately NOT treated as disabled: TimThumb sets these defaults
// elsewhere, so a file that simply does not mention the constant tells us
// nothing about whether the feature is on.
var timThumbFeatureDisabledRE = map[string]*regexp.Regexp{
timThumbWebshot: timThumbDisabledMatcher(timThumbWebshot),
timThumbExternal: timThumbDisabledMatcher(timThumbExternal),
timThumbAllExternal: timThumbDisabledMatcher(timThumbAllExternal),
}
func timThumbDisabledMatcher(name string) *regexp.Regexp {
return timThumbDefineMatcher(name, `(false)`)
}
// timThumbFeatureExplicitlyDisabled reports a visible define(NAME, false).
// An unknown name compiles on demand rather than silently reporting false, so a
// new caller cannot get a wrong answer; the map itself is never mutated, which
// keeps concurrent scans race-free.
func timThumbFeatureExplicitlyDisabled(head []byte, name string) bool {
re, ok := timThumbFeatureDisabledRE[name]
if !ok {
re = timThumbDisabledMatcher(name)
}
if timThumbDefineValue(head, re) == "" {
return false
}
return !timThumbFeatureEnabled(head, name)
}
// timThumbCandidateName reports whether a filename is a TimThumb-style script by
// convention. Content confirmation happens in looksLikeTimThumb.
func timThumbCandidateName(nameLower string) bool {
return nameLower == "timthumb.php" || nameLower == "thumb.php"
}
// looksLikeTimThumb confirms a file is actually TimThumb rather than an
// unrelated theme thumbnail helper. It keys on constants unique to TimThumb so
// a generic thumb.php is never flagged.
func looksLikeTimThumb(head []byte) bool {
lower := bytes.ToLower(head)
if bytes.Contains(lower, []byte("timthumb")) {
return true
}
// Older or renamed copies without the header name still carry these
// TimThumb-specific configuration constants.
if bytes.Contains(lower, []byte("block_external_leechers")) &&
(bytes.Contains(lower, []byte("webshot")) || bytes.Contains(lower, []byte("allow_external"))) {
return true
}
return false
}
// parseTimThumbVersion extracts the value of TimThumb's VERSION define, or ""
// when it is absent.
func parseTimThumbVersion(head []byte) string {
return timThumbDefineValue(head, timThumbVersionRE)
}
func timThumbDefineMatcher(name, valuePattern string) *regexp.Regexp {
return regexp.MustCompile(`(?i)^define\s*\(\s*['"]` + regexp.QuoteMeta(name) +
`['"]\s*,\s*` + valuePattern + `\s*\)`)
}
// timThumbDefineValue returns a literal from an executable define() call.
// Comment and string text cannot prove a copy is patched or a feature is off,
// so walk only PHP identifiers outside those regions before applying the small
// call matcher.
func timThumbDefineValue(head []byte, matcher *regexp.Regexp) string {
code := stripPHPCommentsFromCode(string(head))
inPHP := false
for i := 0; i < len(code); i++ {
if !inPHP {
if i+1 < len(code) && code[i] == '<' && code[i+1] == '?' &&
(i+5 > len(code) || !strings.EqualFold(code[i:i+5], "<?xml")) {
inPHP = true
i++
}
continue
}
if label, bodyStart, ok := phpHeredocOpen(code, i); ok {
i = phpHeredocEnd(code, bodyStart, label) - 1
continue
}
if isPHPQuote(code[i]) || code[i] == '`' {
i = skipPHPString(code, i)
continue
}
if i+1 < len(code) && code[i] == '?' && code[i+1] == '>' {
inPHP = false
i++
continue
}
if !isPHPIdentifierStart(code[i]) {
continue
}
end := i + 1
for end < len(code) && isPHPIdentifierPart(code[end]) {
end++
}
if strings.EqualFold(code[i:end], "define") {
if match := matcher.FindStringSubmatch(code[i:]); len(match) == 2 {
return match[1]
}
}
i = end - 1
}
return ""
}
// timThumbVersionLess reports whether dotted numeric version a is older than b.
func timThumbVersionLess(a, b string) bool {
as, bs := strings.Split(a, "."), strings.Split(b, ".")
for i := 0; i < len(as) || i < len(bs); i++ {
var av, bv int
if i < len(as) {
av, _ = strconv.Atoi(as[i])
}
if i < len(bs) {
bv, _ = strconv.Atoi(bs[i])
}
if av != bv {
return av < bv
}
}
return false
}
// assessTimThumb grades a confirmed TimThumb file. A version below the last
// patched release (or an unparseable one) carries the known RCE and is High; a
// patched-but-abandoned copy is a Warning to remove.
func assessTimThumb(head []byte) (alert.Severity, []string) {
var reasons []string
version := parseTimThumbVersion(head)
exploitable := false
switch {
case version == "":
reasons = append(reasons, "version could not be determined")
exploitable = true
case timThumbVersionLess(version, timThumbFixedVersion):
reasons = append(reasons, fmt.Sprintf("version %s predates the last patch %s (CVE-2011-4106 remote code execution)", version, timThumbFixedVersion))
exploitable = true
default:
reasons = append(reasons, fmt.Sprintf("version %s is deprecated and unmaintained", version))
}
if timThumbFeatureEnabled(head, timThumbWebshot) {
reasons = append(reasons, "WebShot feature is enabled (remote command execution)")
exploitable = true
}
if timThumbFeatureEnabled(head, timThumbExternal) {
reasons = append(reasons, "external image fetching is enabled (SSRF and cache-poisoning surface)")
exploitable = true
}
if exploitable {
return alert.High, reasons
}
return alert.Warning, reasons
}
// timThumbMitigated reports whether a copy has nothing left to act on: the final
// release, with every remote-fetch and screenshot feature explicitly off. The
// remote-fetch path is what CVE-2011-4106 abused to write PHP into the image
// cache, so with it disabled on the last version there is no fix to apply and no
// residual exposure -- only the fact that the project is abandoned, which
// re-reporting every scan does not help anyone act on.
//
// Anything else (older release, unreadable version, any risky feature on) is
// still reported. This is a judgement about the file's own contents, never about
// where it lives.
func timThumbMitigated(head []byte) bool {
if parseTimThumbVersion(head) != timThumbFixedVersion {
return false
}
for _, feature := range []string{timThumbWebshot, timThumbExternal, timThumbAllExternal} {
if !timThumbFeatureExplicitlyDisabled(head, feature) {
return false
}
}
return true
}
// timThumbFeatureEnabled reports whether a TimThumb boolean constant is defined
// true. Matches define('NAME', true) with optional whitespace. Unknown names
// fall back to compiling on demand so a new caller cannot silently read false.
func timThumbFeatureEnabled(head []byte, name string) bool {
re, ok := timThumbFeatureRE[name]
if !ok {
re = timThumbFeatureMatcher(name)
}
return timThumbDefineValue(head, re) != ""
}
// CheckVulnerableTimThumb scans web document roots for bundled TimThumb scripts
// and reports each confirmed instance. Detection-only: TimThumb is legitimate
// (if abandoned) code, so it is never auto-quarantined -- removing it would
// break the theme.
func CheckVulnerableTimThumb(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
var findings []alert.Finding
homeDirs, _ := GetScanHomeDirs(ctx)
for _, homeEntry := range homeDirs {
if ctx.Err() != nil {
return findings
}
if !homeEntry.IsDir() {
continue
}
homeDir := scanHomeDirPath(homeEntry)
docRoots := []string{filepath.Join(homeDir, "public_html")}
subDirs, _ := osFS.ReadDir(homeDir)
for _, sd := range subDirs {
if sd.IsDir() && sd.Name() != "public_html" && sd.Name() != "mail" &&
!strings.HasPrefix(sd.Name(), ".") && sd.Name() != "etc" &&
sd.Name() != "logs" && sd.Name() != "ssl" && sd.Name() != "tmp" {
docRoots = append(docRoots, filepath.Join(homeDir, sd.Name()))
}
}
for _, docRoot := range docRoots {
scanForTimThumb(ctx, docRoot, timThumbScanDepth, &findings)
if ctx.Err() != nil {
return findings
}
}
}
return findings
}
// scanForTimThumb recursively walks dir for TimThumb scripts and appends a
// finding per confirmed instance.
func scanForTimThumb(ctx context.Context, dir string, maxDepth int, findings *[]alert.Finding) {
if ctx.Err() != nil || maxDepth <= 0 {
return
}
entries, err := osFS.ReadDir(dir)
if err != nil {
return
}
for _, entry := range entries {
if ctx.Err() != nil {
return
}
name := entry.Name()
fullPath := filepath.Join(dir, name)
if entry.IsDir() {
scanForTimThumb(ctx, fullPath, maxDepth-1, findings)
continue
}
if !timThumbCandidateName(strings.ToLower(name)) {
continue
}
head := readFileHead(fullPath, timThumbReadHead)
if head == nil || !looksLikeTimThumb(head) {
continue
}
if timThumbMitigated(head) {
continue
}
severity, reasons := assessTimThumb(head)
*findings = append(*findings, alert.Finding{
Severity: severity,
Check: "vulnerable_timthumb",
Message: fmt.Sprintf("Vulnerable TimThumb image resizer: %s", fullPath),
Details: "Remove or replace it. " + strings.Join(reasons, "; ") + ".",
FilePath: fullPath,
})
}
}
package checks
import (
"fmt"
"os"
"strconv"
"strings"
"sync"
"time"
)
const uidCacheMissTTL = time.Minute
// uidCache caches uid -> username from /etc/passwd. The first Lookup of an
// unknown uid reads and parses the file; subsequent lookups return from the
// in-memory map. Process-lifetime: callers that need fresh data after a
// useradd should call Refresh().
type uidCache struct {
lastHomeRead time.Time
path string
mu sync.RWMutex
m map[uint32]string
homes map[string]string
}
var defaultUIDCache = newUIDCache("/etc/passwd")
func newUIDCache(path string) *uidCache {
return &uidCache{path: path, m: map[uint32]string{}, homes: map[string]string{}}
}
// LookupUser returns the username for uid, or "uid:<n>" if not resolvable.
// Safe for concurrent use; the underlying cache is shared across the daemon.
func LookupUser(uid uint32) string { return defaultUIDCache.Lookup(uid) }
// swapDefaultUIDCacheForTest replaces defaultUIDCache with a cache pointed at
// path and returns a function that restores the original. Test-only helper:
// existing tests stub /etc/passwd via osFS, but the cache reads the real file
// directly (so the daemon never burns syscalls per-event). This shim lets the
// tests stage a fixture file and have LookupUser read from it.
func swapDefaultUIDCacheForTest(path string) func() {
prev := defaultUIDCache
defaultUIDCache = newUIDCache(path)
return func() { defaultUIDCache = prev }
}
// SwapUIDCacheForTest is the exported form of swapDefaultUIDCacheForTest for
// packages whose producers resolve uids through LookupUser.
func SwapUIDCacheForTest(path string) func() { return swapDefaultUIDCacheForTest(path) }
func (c *uidCache) Lookup(uid uint32) string {
c.mu.RLock()
if name, ok := c.m[uid]; ok {
c.mu.RUnlock()
return name
}
c.mu.RUnlock()
c.mu.Lock()
defer c.mu.Unlock()
if name, ok := c.m[uid]; ok {
return name
}
c.parseLocked()
if name, ok := c.m[uid]; ok {
return name
}
miss := fmt.Sprintf("uid:%d", uid)
c.m[uid] = miss
return miss
}
// Refresh drops the cache. The next Lookup re-reads /etc/passwd.
func (c *uidCache) Refresh() {
c.mu.Lock()
c.m = map[uint32]string{}
c.homes = map[string]string{}
c.lastHomeRead = time.Time{}
c.mu.Unlock()
}
// HomeDir returns the home directory recorded for name, or "" when the user
// is unknown. Producers use it to tell a hosting account from a system user.
func (c *uidCache) HomeDir(name string) string {
c.mu.RLock()
home, ok := c.homes[name]
fresh := time.Since(c.lastHomeRead) < uidCacheMissTTL
c.mu.RUnlock()
if ok || fresh {
return home
}
c.mu.Lock()
defer c.mu.Unlock()
if home, ok = c.homes[name]; ok || time.Since(c.lastHomeRead) < uidCacheMissTTL {
return home
}
// Throttle misses across all names without storing attacker-chosen keys.
// Expiry lets newly provisioned accounts become attributable after a miss.
c.parseLocked()
return c.homes[name]
}
// parseLocked replaces the cache contents with a fresh scan. Caller holds the
// write lock.
func (c *uidCache) parseLocked() {
c.lastHomeRead = time.Now()
data, err := os.ReadFile(c.path)
if err != nil {
return
}
for _, line := range strings.Split(string(data), "\n") {
fields := strings.SplitN(line, ":", 7)
if len(fields) < 3 {
continue
}
uid64, err := strconv.ParseUint(fields[2], 10, 32)
if err != nil {
continue
}
c.m[uint32(uid64)] = fields[0]
if len(fields) >= 6 {
c.homes[fields[0]] = fields[5]
}
}
}
package checks
import (
"net"
"net/url"
"strings"
)
// URL reputation — attack-indicator based classifier for external
// <script src="..."> URLs embedded in WordPress content.
//
// PHILOSOPHY
//
// Earlier versions of this package classified script sources against a
// hardcoded allowlist of "known safe" domains (Google Tag Manager,
// Cloudflare CDN, HubSpot, Stripe, etc.). In practice the allowlist is
// unmaintainable: every new widget service (OneTrust, Issuu, regional
// video embeds, tax-form widgets, etc.) adds an entry and operators
// still see HIGH-severity findings for legitimate third-party embeds.
//
// This file takes the inverse approach: rather than asking "is this
// domain on my list of safe services?", it asks "does this URL show
// attacker-characteristic markers?". A <script src> fires a finding
// only when at least one attack indicator is present:
//
// - the host is a raw IP address (attackers dodge domain reputation);
// - the host TLD is on a well-known abused-TLD list (.tk, .ml, .ga,
// .cf, .gq free-abuse; .top, .icu, .click, .pw and similar cheap
// gTLDs per Spamhaus recurring bad-TLD reports);
// - the host is on the existing short known-bad-exfil list
// (Cloudflare Workers free tier, Pastebin raw, GitHub Gist raw —
// legitimate content rarely loads from these hosts);
// - the scheme is plaintext HTTP (an external JS loader without TLS
// is a plaintext MITM target regardless of the destination);
// - the host is empty, contains no dot, or otherwise fails basic
// FQDN validation.
//
// The knownSafeDomains list is retained as a FAST-PATH tie-breaker: if
// the host matches a well-known service we return "not malicious"
// immediately, skipping further analysis. It is an optimization, not
// the primary filter. Unknown hosts on unremarkable TLDs (e.g.
// onetrust.com, issuu.com, trilulilu.ro, formular230.ro) pass because
// they have zero attack indicators — no allowlist growth needed.
//
// TRADE-OFF
//
// An attacker who hosts payload on a compromised mainstream domain
// (.com/.org/.net HTTPS, normal-looking path) is not caught. This gap
// existed under the prior allowlist too — it is a fundamental limit of
// URL-only classification. Closing it requires threat-intelligence
// correlation or content-based JS analysis, both of which are out of
// scope here. The defensive value is in raising the bar for casual
// injection, not in defeating sophisticated adversaries.
// abusedTLDs are top-level domains whose registrations are cheap,
// unverified, and overwhelmingly abused for phishing, malware, and SEO
// spam per recurring Spamhaus and KnowBe4 reports.
//
// The entry bar is high: a TLD is included only if (a) it has shown up
// in top-10 abuse rankings across multiple reporting years, AND
// (b) legitimate business usage is rare. Mixed-use TLDs with
// significant legitimate traffic (.xyz, .online, .site, .live, .space)
// are intentionally excluded to keep false-positive rates low.
//
// Entries are stored without the leading dot; comparison strips the
// leading dot from the observed TLD before lookup.
var abusedTLDs = map[string]bool{
// Former Freenom TLDs — free registration, no verification.
// Near-100% abuse rate; Freenom itself was shut down in 2023 but
// the TLDs remain in DNS.
"tk": true,
"ml": true,
"ga": true,
"cf": true,
"gq": true,
// Cheap new gTLDs consistently in the Spamhaus top-abused list.
"top": true,
"icu": true,
"click": true,
"pw": true,
"loan": true,
"work": true,
"download": true,
// Spamhaus badlist recurring entries — legitimate usage is
// essentially nonexistent at meaningful volume.
"kim": true,
"gdn": true,
"stream": true,
"bid": true,
"racing": true,
"win": true,
"party": true,
"science": true,
"trade": true,
}
// knownBadExfilHosts are hosts where legitimate WordPress content
// essentially never loads JavaScript from, but attackers routinely do.
// The list is intentionally short — see knownSafeDomains for the
// inverse fast-path.
var knownBadExfilHosts = []string{
// Cloudflare Workers free-tier subdomain (e.g. x.workers.dev).
// Legitimate apps host on custom domains; free .workers.dev is a
// common payload drop.
".workers.dev",
// Pastebin and GitHub Gist raw endpoints — legitimate sites do not
// load JS from these paths.
"pastebin.com",
"gist.githubusercontent.com",
// bit.ly and similar URL shorteners in a <script src> are a very
// strong attack signal — no legitimate embed uses a shortener for
// a JS asset.
"bit.ly",
"tinyurl.com",
"is.gd",
"cutt.ly",
}
// scriptSrcStrongReason classifies a <script src> URL by structural
// attack indicators that are context-independent: a raw IP host, an
// abused TLD, a known-bad exfil host, or an empty/unparseable/no-TLD
// host. These markers are rare-to-nonexistent in legitimate content of
// any age and remain valid signals whether the script appears in a
// freshly-written wp_options value or in decade-old post_content.
//
// Callers that operate on storage which is expected to hold current
// configuration (wp_options) should prefer scriptSrcMaliciousReason,
// which layers the plaintext-HTTP indicator on top. The HTTP signal
// catches attacker convenience ("don't bother with TLS") in fresh
// configuration but produces false positives on legacy author content
// where pre-TLS embeds are normal.
//
// The function does not consult knownSafeDomains — callers should do
// that first as a fast path. Separating the two concerns keeps this
// function single-purpose and trivially testable.
func scriptSrcStrongReason(rawURL string) (bool, string) {
normalised := rawURL
if strings.HasPrefix(normalised, "//") {
normalised = "https:" + normalised
}
u, err := url.Parse(normalised)
if err != nil || u == nil {
return true, "unparseable URL"
}
host := strings.ToLower(u.Hostname())
if host == "" {
return true, "empty host"
}
if ip := net.ParseIP(host); ip != nil {
return true, "raw IP address host"
}
for _, bad := range knownBadExfilHosts {
// Match the host itself or a subdomain of it, never a longer name
// that merely ends in the same characters (orbit.ly is not bit.ly).
bare := strings.TrimPrefix(bad, ".")
if host == bare || strings.HasSuffix(host, "."+bare) {
return true, "known-bad exfil host: " + bad
}
}
lastDot := strings.LastIndexByte(host, '.')
if lastDot < 0 || lastDot == len(host)-1 {
return true, "host without valid TLD"
}
tld := host[lastDot+1:]
if abusedTLDs[tld] {
return true, "abused TLD: ." + tld
}
return false, ""
}
// scriptSrcMaliciousReason classifies a <script src> URL with the full
// indicator set: everything scriptSrcStrongReason flags, plus plaintext
// HTTP. This is the classifier for wp_options and similar configuration
// storage, where a plaintext HTTP external script loader is a signal on
// its own (a site's analytics configuration should be HTTPS in 2026).
//
// The function does not consult knownSafeDomains — callers should do
// that first as a fast path.
func scriptSrcMaliciousReason(rawURL string) (bool, string) {
// Structural markers take precedence: a raw-IP host over HTTP should
// report the IP, not the scheme, because the scheme is merely the
// delivery method while the IP is the identity of the attacker
// infrastructure.
if bad, reason := scriptSrcStrongReason(rawURL); bad {
return true, reason
}
// Plaintext HTTP for an external script is a strong indicator in
// configuration storage. Protocol-relative URLs (//host/path)
// inherit the page's scheme and are not flagged here.
normalised := rawURL
if strings.HasPrefix(normalised, "//") {
normalised = "https:" + normalised
}
u, err := url.Parse(normalised)
if err != nil || u == nil {
// scriptSrcStrongReason would have caught this; defensive only.
return true, "unparseable URL"
}
if strings.EqualFold(u.Scheme, "http") && !strings.HasPrefix(rawURL, "//") {
return true, "plaintext HTTP external script"
}
return false, ""
}
// isAttackerScriptURL is the caller-facing predicate for contexts where
// a plaintext-HTTP external script is a signal on its own (wp_options
// and similar configuration storage). It combines the known-safe fast
// path with the strict attack-indicator classifier.
//
// The order matters: the fast path is checked first because it lets us
// short-circuit common legitimate widgets (Google Tag Manager,
// Cloudflare CDN, Stripe) without parsing the URL. Only unknown hosts
// are subjected to the attack-indicator analysis.
func isAttackerScriptURL(rawURL string) bool {
if isSafeScriptDomain(rawURL) {
return false
}
bad, _ := scriptSrcMaliciousReason(rawURL)
return bad
}
// isAttackerScriptURLInPost is the caller-facing predicate for
// post_content classification. It uses scriptSrcStrongReason, which
// omits the plaintext-HTTP indicator: legacy author embeds from the
// pre-TLS era are legitimate content, not injection, and must not
// produce db_post_injection findings.
//
// Fresh attacker injections still flag because they almost always point
// at structural markers (raw IP hosts, abused TLDs, cheap exfil hosts)
// rather than at an unremarkable mainstream-TLD host — and an attacker
// who did somehow land on a plaintext-HTTP mainstream host URL would
// still be caught by other checks (obfuscated_php_realtime scanning the
// attacker's dropper, remote payload URLs in the served page, etc.).
func isAttackerScriptURLInPost(rawURL string) bool {
if isSafeScriptDomain(rawURL) {
return false
}
bad, _ := scriptSrcStrongReason(rawURL)
return bad
}
package checks
import "bytes"
// .user.ini cPanel-managed signature detection.
//
// cPanel's MultiPHP INI Editor writes .user.ini files with a fixed
// four-line header that begins:
//
// ; cPanel-generated php ini directives, do not edit
// ; Manual editing of this file may result in unexpected behavior.
// ; To make changes to this file, use the cPanel MultiPHP INI Editor ...
// ; For more information, read our documentation ...
//
// When this header is present the values in the file reflect operator
// choices made through cPanel's UI (max_execution_time=0 for a backup
// importer, display_errors=On for a staging account, etc.). These are
// not attacker signals. The severity of findings on values in a
// cPanel-managed file should be reduced to informational.
//
// When the header is absent we cannot tell whether the site owner
// hand-edited the file or an attacker planted it. In that case we
// preserve the original severity.
//
// Detection rule (minimal and precise):
//
// The signature must appear on the FIRST non-blank line of the file.
//
// Reasoning: cPanel always writes the header at the top of the file
// and rewrites the whole file on every edit through the UI. An
// attacker appending content below keeps the header position intact.
// An attacker prepending content above pushes the header down; the
// file is then no longer a regular cPanel-managed file and the
// attacker's values take precedence anyway — we must NOT accept this
// as "managed" because doing so would let attackers suppress findings
// by inserting one of their own lines and then the real cPanel header.
// cpanelUserIniSignature is the exact string cPanel writes on the first
// comment line of a managed .user.ini. Case-sensitive: alternate
// capitalizations are rejected to avoid accepting forgeries.
const cpanelUserIniSignature = "cPanel-generated php ini directives"
// cpanelUserIniMaxLeadingBlanks caps how many blank lines may precede
// the signature. cPanel itself writes the signature as the very first
// line, so any tolerance here is purely for admins who may have
// round-tripped the file through a text editor that adds a trailing
// newline or through an FTP client that accumulates CRLFs. A small cap
// also closes a forgery route: without a bound, an attacker who wants
// to suppress severity on their injected values could prepend a large
// run of blank lines followed by the genuine cPanel header string, and
// a first-non-blank-line-only check would classify the file as
// managed despite the attacker owning all the content above the
// header.
const cpanelUserIniMaxLeadingBlanks = 5
// isCpanelManagedUserIni reports whether data begins with the cPanel
// MultiPHP-managed .user.ini header. Empty/whitespace-only input
// returns false. A small number (cpanelUserIniMaxLeadingBlanks) of
// leading blank lines is tolerated; beyond that, the file is not
// considered cPanel-managed even if the signature appears later.
func isCpanelManagedUserIni(data []byte) bool {
if len(data) == 0 {
return false
}
blanks := 0
start := 0
check := func(line []byte) (matched bool, terminate bool) {
line = bytes.TrimRight(line, "\r")
line = bytes.TrimSpace(line)
if len(line) == 0 {
blanks++
if blanks > cpanelUserIniMaxLeadingBlanks {
return false, true
}
return false, false
}
return bytes.Contains(line, []byte(cpanelUserIniSignature)), true
}
for i := 0; i < len(data); i++ {
if data[i] != '\n' {
continue
}
matched, terminate := check(data[start:i])
if terminate {
return matched
}
start = i + 1
}
if start < len(data) {
matched, _ := check(data[start:])
return matched
}
return false
}
package checks
import (
"strings"
)
type indirectFuncKind uint8
const (
indirectDecoder indirectFuncKind = 1 << iota
indirectEval
indirectShell
)
var indirectFuncKinds = map[string]indirectFuncKind{
"base64_decode": indirectDecoder,
"gzinflate": indirectDecoder,
"gzuncompress": indirectDecoder,
"gzdecode": indirectDecoder,
"bzdecompress": indirectDecoder,
"str_rot13": indirectDecoder,
"rawurldecode": indirectDecoder,
"eval": indirectEval,
"assert": indirectEval,
"create_function": indirectEval,
"system": indirectShell,
"passthru": indirectShell,
"exec": indirectShell,
"shell_exec": indirectShell,
"popen": indirectShell,
"proc_open": indirectShell,
"pcntl_exec": indirectShell,
}
type indirectAssignment struct {
kind indirectFuncKind
pos int
}
type indirectCall struct {
variable string
kind indirectFuncKind
pos int
lineStart int
lineEnd int
line string
}
// detectVarFuncDangerousAssignment returns true when content contains a
// `$var = "dangerous_name"` assignment followed by a corresponding
// `$var(` invocation that forms an execution sink. Decoder-only
// callbacks are not enough; direct base64_decode() calls are common in
// legitimate plugins, and indirect callbacks need the same restraint.
func detectVarFuncDangerousAssignment(content string) bool {
code := stripPHPCommentsFromCode(content)
assignments := findIndirectAssignments(code)
if len(assignments) == 0 {
return false
}
calls := findIndirectCalls(code, assignments)
if len(calls) == 0 {
return false
}
for _, call := range calls {
switch {
case call.kind&indirectShell != 0:
if lineContainsRequestVar(call.line) {
return true
}
case call.kind&indirectEval != 0:
if lineContainsRequestVar(call.line) ||
lineContainsDirectDecoderCall(call.line) ||
lineContainsIndirectCallKind(calls, call.lineStart, call.lineEnd, indirectDecoder) {
return true
}
case call.kind&indirectDecoder != 0:
if lineContainsDirectEvalCall(call.line) ||
lineContainsIndirectCallKind(calls, call.lineStart, call.lineEnd, indirectEval) {
return true
}
}
}
return false
}
func findIndirectAssignments(code string) map[string][]indirectAssignment {
assignments := map[string][]indirectAssignment{}
for i := 0; i < len(code); i++ {
// Literal examples must neither create nor overwrite a callable binding.
if label, bodyStart, ok := phpHeredocOpen(code, i); ok {
i = phpHeredocEnd(code, bodyStart, label) - 1
continue
}
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
if code[i] != '$' {
continue
}
variable, next, ok := readPHPVariableName(code, i)
if !ok {
continue
}
j := skipPHPWhitespace(code, next)
if j >= len(code) || code[j] != '=' || (j+1 < len(code) && (code[j+1] == '=' || code[j+1] == '>')) {
i = next - 1
continue
}
assignment := indirectAssignment{pos: j + 1}
valueStart := skipPHPWhitespace(code, j+1)
if valueStart < len(code) && isPHPQuote(code[valueStart]) {
value, valueEnd, valueOK := readPHPFunctionString(code, valueStart)
assignment.pos = valueEnd
if valueOK {
assignment.kind = indirectFuncKinds[value]
}
}
assignments[variable] = append(assignments[variable], assignment)
i = next - 1
}
return assignments
}
func findIndirectCalls(code string, assignments map[string][]indirectAssignment) []indirectCall {
var calls []indirectCall
// Strip strings before slicing lines: a call's line may start inside a
// multiline literal whose opening quote is on an earlier line.
codeNoStrings := stripPHPStringsFromCode(code)
for i := 0; i < len(code); i++ {
if label, bodyStart, ok := phpHeredocOpen(code, i); ok {
i = phpHeredocEnd(code, bodyStart, label) - 1
continue
}
if isPHPQuote(code[i]) {
i = skipPHPString(code, i)
continue
}
if code[i] != '$' {
continue
}
variable, next, ok := readPHPVariableName(code, i)
if !ok {
continue
}
j := skipPHPWhitespace(code, next)
if j >= len(code) || code[j] != '(' {
i = next - 1
continue
}
kind, ok := indirectKindAt(assignments[variable], i)
if !ok {
i = next - 1
continue
}
lineStart, lineEnd := phpLineBounds(code, i)
calls = append(calls, indirectCall{
variable: variable,
kind: kind,
pos: i,
lineStart: lineStart,
lineEnd: lineEnd,
line: codeNoStrings[lineStart:lineEnd],
})
i = next - 1
}
return calls
}
func indirectKindAt(assignments []indirectAssignment, callPos int) (indirectFuncKind, bool) {
var last indirectFuncKind
found := false
for _, assignment := range assignments {
if assignment.pos > callPos {
break
}
last = assignment.kind
found = true
}
return last, found && last != 0
}
func lineContainsDirectDecoderCall(line string) bool {
codeLine := strings.ToLower(stripPHPStringsFromCode(line))
for name, kind := range indirectFuncKinds {
if kind&indirectDecoder != 0 && containsStandaloneFunc(codeLine, name+"(") {
return true
}
}
return false
}
func lineContainsDirectEvalCall(line string) bool {
codeLine := strings.ToLower(stripPHPStringsFromCode(line))
for name, kind := range indirectFuncKinds {
if kind&indirectEval != 0 && containsStandaloneFunc(codeLine, name+"(") {
return true
}
}
return false
}
func lineContainsRequestVar(line string) bool {
return containsRequestSuperglobal(stripPHPStringsFromCode(line))
}
func lineContainsIndirectCallKind(calls []indirectCall, lineStart, lineEnd int, kind indirectFuncKind) bool {
for _, call := range calls {
if call.lineStart == lineStart && call.lineEnd == lineEnd && call.kind&kind != 0 {
return true
}
}
return false
}
func stripPHPCommentsFromCode(code string) string {
var b strings.Builder
b.Grow(len(code))
for i := 0; i < len(code); i++ {
// Heredoc/nowdoc bodies are string literals, not code. Copy them
// verbatim so a '#', '//', or '/*' inside the body is not mistaken
// for a comment (which would corrupt the surrounding code).
if label, bodyStart, ok := phpHeredocOpen(code, i); ok {
end := phpHeredocEnd(code, bodyStart, label)
b.WriteString(code[i:end])
i = end - 1
continue
}
if isPHPQuote(code[i]) || code[i] == '`' {
i = copyPHPString(&b, code, i)
continue
}
if code[i] == '/' && i+1 < len(code) && code[i+1] == '*' {
b.WriteString(" ")
i += 2
for i < len(code) {
if code[i] == '*' && i+1 < len(code) && code[i+1] == '/' {
b.WriteString(" ")
i++
break
}
writeCommentReplacementByte(&b, code[i])
i++
}
continue
}
// "#[" opens a PHP 8 attribute, which can precede a statement on the
// same line; only other "#" forms are comments. A line comment also
// ends at "?>", and the code after that tag runs.
if isPHPLineCommentStart(code, i) {
end := skipPHPLineComment(code, i)
for ; i < end; i++ {
b.WriteByte(' ')
}
i--
continue
}
b.WriteByte(code[i])
}
return b.String()
}
func stripPHPStringsFromCode(code string) string {
return stripPHPStringsKeeping(code, nil)
}
// stripPHPStringsKeeping blanks string literals like stripPHPStringsFromCode,
// except quoted literals for which keep reports true.
func stripPHPStringsKeeping(code string, keep func(token string) bool) string {
var b strings.Builder
b.Grow(len(code))
for i := 0; i < len(code); i++ {
// Blank heredoc/nowdoc bodies (and their opener/closing label) the same
// way single/double-quoted strings are blanked, so their contents are
// not analysed as code and a quote inside the body cannot desync the
// scanner and swallow real code that follows the heredoc.
if label, bodyStart, ok := phpHeredocOpen(code, i); ok {
end := phpHeredocEnd(code, bodyStart, label)
for k := i; k < end; k++ {
if code[k] == '\n' || code[k] == '\r' {
b.WriteByte(code[k])
} else {
b.WriteByte(' ')
}
}
i = end - 1
continue
}
// Backtick bodies use shell syntax. Their quotes must not expose
// literals to keep or consume the PHP code after the closing backtick.
if isPHPQuote(code[i]) || code[i] == '`' {
if end := skipPHPString(code, i); keep != nil && end < len(code) && keep(code[i:end+1]) {
b.WriteString(code[i : end+1])
i = end
continue
}
i = replacePHPString(&b, code, i)
continue
}
b.WriteByte(code[i])
}
return b.String()
}
// phpHeredocOpen reports whether code[i:] opens a heredoc or nowdoc. On success
// it returns the label and the byte index where the body begins (just past the
// opening line's newline). It recognises `<<<LABEL`, `<<<"LABEL"` (heredoc) and
// `<<<'LABEL'` (nowdoc), with optional spaces/tabs after `<<<`.
func phpHeredocOpen(code string, i int) (label string, bodyStart int, ok bool) {
if i+3 > len(code) || code[i] != '<' || code[i+1] != '<' || code[i+2] != '<' {
return "", 0, false
}
if i > 0 && code[i-1] == '<' {
return "", 0, false
}
j := i + 3
for j < len(code) && (code[j] == ' ' || code[j] == '\t') {
j++
}
var quote byte
if j < len(code) && (code[j] == '\'' || code[j] == '"') {
quote = code[j]
j++
}
if j >= len(code) || !isPHPIdentifierStart(code[j]) {
return "", 0, false
}
start := j
j++
for j < len(code) && isPHPIdentifierPart(code[j]) {
j++
}
label = code[start:j]
if quote != 0 {
if j >= len(code) || code[j] != quote {
return "", 0, false
}
j++
}
for j < len(code) && (code[j] == ' ' || code[j] == '\t') {
j++
}
if j >= len(code) || (code[j] != '\n' && code[j] != '\r') {
return "", 0, false
}
if code[j] == '\r' && j+1 < len(code) && code[j+1] == '\n' {
j++
}
return label, j + 1, true
}
// phpHeredocEnd returns the byte index just past the closing label of a heredoc
// whose body begins at bodyStart. PHP 7.3+ permits the closing label to be
// indented; the label must appear at the start of a line (after optional
// spaces/tabs) and be followed by a non-identifier byte. An unterminated
// heredoc consumes the rest of the input.
func phpHeredocEnd(code string, bodyStart int, label string) int {
i := bodyStart
for i < len(code) {
lineEnd := i
for lineEnd < len(code) && code[lineEnd] != '\n' && code[lineEnd] != '\r' {
lineEnd++
}
k := i
for k < lineEnd && (code[k] == ' ' || code[k] == '\t') {
k++
}
if k+len(label) <= lineEnd && code[k:k+len(label)] == label {
after := k + len(label)
if after >= len(code) || !isPHPIdentifierPart(code[after]) {
return after
}
}
if lineEnd >= len(code) {
break
}
i = lineEnd + 1
}
return len(code)
}
func copyPHPString(b *strings.Builder, code string, start int) int {
end := skipPHPString(code, start)
b.WriteString(code[start : end+1])
return end
}
func replacePHPString(b *strings.Builder, code string, start int) int {
end := skipPHPString(code, start)
for i := start; i <= end; i++ {
writeCommentReplacementByte(b, code[i])
}
return end
}
func writeCommentReplacementByte(b *strings.Builder, c byte) {
if c == '\n' || c == '\r' {
b.WriteByte(c)
return
}
b.WriteByte(' ')
}
func readPHPVariableName(code string, dollar int) (string, int, bool) {
if dollar+1 >= len(code) || !isPHPIdentifierStart(code[dollar+1]) {
return "", dollar + 1, false
}
i := dollar + 2
for i < len(code) && isPHPIdentifierPart(code[i]) {
i++
}
return code[dollar+1 : i], i, true
}
func readPHPFunctionString(code string, start int) (string, int, bool) {
quote := code[start]
var b strings.Builder
for i := start + 1; i < len(code); i++ {
if code[i] == '\\' && i+1 < len(code) {
if code[i+1] == quote || code[i+1] == '\\' {
i++
b.WriteByte(code[i])
continue
}
b.WriteByte(code[i])
continue
}
if code[i] == quote {
value, ok := normalizeIndirectFunctionName(b.String())
return value, i + 1, ok
}
b.WriteByte(code[i])
}
return "", len(code), false
}
func normalizeIndirectFunctionName(value string) (string, bool) {
value = strings.TrimLeft(value, "\\")
if !isPHPIdentifier(value) {
return "", false
}
return strings.ToLower(value), true
}
func skipPHPString(code string, start int) int {
// Complex interpolation enters PHP mode, where array keys and nested
// expressions can use the enclosing quote. Track those modes iteratively
// so their quotes cannot end the outer string or exhaust the call stack.
modes := []byte{code[start]}
for i := start + 1; i < len(code); i++ {
mode := modes[len(modes)-1]
if mode != '{' {
if code[i] == '\\' && i+1 < len(code) {
i++
continue
}
if code[i] == mode {
modes = modes[:len(modes)-1]
if len(modes) == 0 {
return i
}
continue
}
if mode != '\'' && i+1 < len(code) {
if code[i] == '{' && code[i+1] == '$' {
modes = append(modes, '{')
} else if code[i] == '$' && code[i+1] == '{' {
modes = append(modes, '{')
i++
}
}
continue
}
if label, bodyStart, ok := phpHeredocOpen(code, i); ok {
i = phpHeredocEnd(code, bodyStart, label) - 1
continue
}
if isPHPLineCommentStart(code, i) {
i = skipPHPLineComment(code, i) - 1
continue
}
if code[i] == '/' && i+1 < len(code) && code[i+1] == '*' {
end := strings.Index(code[i+2:], "*/")
if end < 0 {
return len(code) - 1
}
i += end + 3
continue
}
switch code[i] {
case '\'', '"', '`', '{':
modes = append(modes, code[i])
case '}':
modes = modes[:len(modes)-1]
}
}
return len(code) - 1
}
func skipPHPWhitespace(code string, start int) int {
for start < len(code) {
switch code[start] {
case ' ', '\t', '\n', '\r', '\f', '\v':
start++
default:
return start
}
}
return start
}
func phpLineBounds(code string, pos int) (int, int) {
start := pos
for start > 0 && code[start-1] != '\n' && code[start-1] != '\r' {
start--
}
end := pos
for end < len(code) && code[end] != '\n' && code[end] != '\r' {
end++
}
return start, end
}
func isPHPQuote(c byte) bool {
return c == '\'' || c == '"'
}
func isPHPIdentifier(value string) bool {
if value == "" || !isPHPIdentifierStart(value[0]) {
return false
}
for i := 1; i < len(value); i++ {
if !isPHPIdentifierPart(value[i]) {
return false
}
}
return true
}
func isPHPIdentifierStart(c byte) bool {
return c == '_' || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
}
func isPHPIdentifierPart(c byte) bool {
return isPHPIdentifierStart(c) || (c >= '0' && c <= '9')
}
package checks
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"syscall"
"unicode"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/jstaint"
"github.com/pidginhost/csm/internal/phptaint"
"github.com/pidginhost/csm/internal/signatures"
"github.com/pidginhost/csm/internal/yara"
)
// VerifyResult reports whether a finding's underlying condition still holds.
//
// Checked is false when the finding's check type has no cheap, reliable
// single-target re-check (the caller should tell the operator to dismiss after
// manual review or run a full account scan). When Checked is true, Resolved
// reports whether the condition is gone and the finding can be cleared.
type VerifyResult struct {
Checked bool `json:"checked"`
Resolved bool `json:"resolved"`
// Demote marks a finding whose flagged content is gone but whose
// remediation cannot be proven, because the file changed since detection.
// It is never cleared -- an attacker must not retire a finding by editing
// the file -- but it stops ranking beside live threats.
Demote bool `json:"demote"`
Detail string `json:"detail"`
}
// VerifyInput carries everything a finding verifier may need. ContentSHA256 and
// DetectLogic are populated only for content findings emitted with a fingerprint.
// Context is optional; long-running verifiers use Background when it is nil.
type VerifyInput struct {
Check, Message, Details, Path string
ContentSHA256, DetectLogic string
Context context.Context
exposureVhosts *exposureVhostIndex
}
// presenceVerifiableChecks are findings whose remediation removes or
// quarantines a single flagged file, so the honest, cheap re-check is "is the
// flagged path still there?". Content-family checks are handled by
// reverifyContentFinding only when this package can re-run the same classifier
// that produced them; realtime PHP heuristic findings stay presence-based.
var presenceVerifiableChecks = []string{
"webshell", "webshell_realtime",
"webshell_content_realtime", "obfuscated_php_realtime",
"php_dropper_realtime",
"new_webshell_file", "new_suspicious_php",
"new_php_in_sensitive_dir", "new_php_in_uploads",
"php_in_sensitive_dir_realtime", "php_in_uploads_realtime",
"suspicious_file",
"nulled_plugin", "symlink_attack",
"vulnerable_timthumb",
"backdoor_binary", "new_executable_in_config",
"executable_in_config_realtime", "executable_in_tmp_realtime",
"cgi_backdoor_realtime", "cgi_suspicious_location_realtime",
"phishing_page", "phishing_directory", "phishing_php",
"phishing_kit_archive", "phishing_kit_realtime", "phishing_iframe",
"phishing_redirector", "phishing_credential_log", "phishing_realtime",
"credential_log_realtime",
}
// htaccessGenericChecks come from the generic scanner. They re-audit with it
// and every per-pattern detector, since their manual fix removes both sets.
var htaccessGenericChecks = []string{"htaccess_injection", "htaccess_injection_realtime"}
// htaccessPatternChecks re-audit with the set automatic cleaning removes.
var htaccessPatternChecks = append([]string{"htaccess_handler_abuse"}, htaccessDetectorNames()...)
// htaccessVerifiableChecks re-audit the .htaccess and resolve when no malicious
// directive remains (or the file is gone).
var htaccessVerifiableChecks = append(append([]string(nil), htaccessGenericChecks...), htaccessPatternChecks...)
// findingVerifiers maps a finding's Check to a read-only re-check. A check not
// present here has no automated re-check -- either an event finding (a brute
// force, a past login: history cannot be re-evaluated by reading current state)
// or a condition we cannot yet cheaply and safely confirm. CanVerify reports
// membership so the Web UI shows the "Re-check" action only where it can act.
var findingVerifiers = buildFindingVerifiers()
var (
contentSignatureScanner = signatures.Global
contentYARAScanner = yara.Active
)
func buildFindingVerifiers() map[string]func(VerifyInput) VerifyResult {
m := map[string]func(VerifyInput) VerifyResult{}
register := func(fn func(VerifyInput) VerifyResult, names ...string) {
for _, n := range names {
m[n] = fn
}
}
register(func(in VerifyInput) VerifyResult { return verifyWriteBit(in.Path, 0002, "world-writable") },
"world_writable_php")
register(func(in VerifyInput) VerifyResult { return verifyWriteBit(in.Path, 0020, "group-writable") },
"group_writable_php")
register(func(in VerifyInput) VerifyResult {
return verifyPathAbsent(in.Path, effectiveFixRoots(fixQuarantineAllowedRoots, quarantineExtraRoots...))
},
presenceVerifiableChecks...)
register(reverifyContentFinding, contentReverifiableChecks...)
register(func(in VerifyInput) VerifyResult { return verifyHtaccessClean(in.Path, AuditHtaccessContent) },
htaccessGenericChecks...)
register(func(in VerifyInput) VerifyResult { return verifyHtaccessClean(in.Path, auditHtaccessPatterns) },
htaccessPatternChecks...)
register(func(in VerifyInput) VerifyResult { return verifyEximSpoolAbsent(in.Message) },
"email_phishing_content")
register(func(in VerifyInput) VerifyResult { return verifyCrontabClear(in.Path) },
"suspicious_crontab")
register(verifyExposedFile, exposedVerifiableChecks...)
register(func(in VerifyInput) VerifyResult { return verifyOutdatedPlugins(in.Details) },
"outdated_plugins")
register(func(in VerifyInput) VerifyResult { return verifyWPCoreIntegrity(in.Details) },
"wp_core_integrity")
register(func(in VerifyInput) VerifyResult { return verifyUID0Account(in.Message) },
"uid0_account")
register(func(in VerifyInput) VerifyResult { return verifySuidCleared(in.Path) },
"suid_binary")
register(func(in VerifyInput) VerifyResult { return verifyRPMIntegrity(in.Message) },
"rpm_integrity")
register(func(in VerifyInput) VerifyResult { return verifyDpkgIntegrity(in.Message) },
"dpkg_integrity")
register(func(in VerifyInput) VerifyResult { return verifyDBOptionsInjection(in.Message, in.Details) },
"db_options_injection")
register(func(in VerifyInput) VerifyResult { return verifyDBSiteurlHijack(in.Message, in.Details) },
"db_siteurl_hijack", "db_siteurl_invalid")
register(func(in VerifyInput) VerifyResult { return verifyDBPostInjection(in.Message, in.Details) },
"db_post_injection")
register(func(in VerifyInput) VerifyResult { return verifyDBSpamInjection(in.Message, in.Details) },
"db_spam_injection")
register(func(in VerifyInput) VerifyResult { return verifyDrupalSettingsInjection(in.Message, in.Details) },
"drupal_settings_injection")
register(func(in VerifyInput) VerifyResult { return verifyDrupalContentInjection(in.Message, in.Details) },
"drupal_content_injection")
register(func(in VerifyInput) VerifyResult { return verifyJoomlaExtensionsInjection(in.Message, in.Details) },
"joomla_extensions_injection")
register(func(in VerifyInput) VerifyResult { return verifyJoomlaContentInjection(in.Message, in.Details) },
"joomla_content_injection")
register(func(in VerifyInput) VerifyResult { return verifyMagentoSettingsInjection(in.Message, in.Details) },
"magento_settings_injection")
register(func(in VerifyInput) VerifyResult { return verifyMagentoContentInjection(in.Message, in.Details) },
"magento_content_injection")
register(func(in VerifyInput) VerifyResult { return verifyOpenCartSettingsInjection(in.Message, in.Details) },
"opencart_settings_injection")
register(func(in VerifyInput) VerifyResult { return verifyOpenCartContentInjection(in.Message, in.Details) },
"opencart_content_injection")
register(func(in VerifyInput) VerifyResult { return verifyPrestaShopSettingsInjection(in.Message, in.Details) },
"prestashop_settings_injection")
register(func(in VerifyInput) VerifyResult { return verifyPrestaShopContentInjection(in.Message, in.Details) },
"prestashop_content_injection")
register(func(in VerifyInput) VerifyResult { return verifyDBObject(in.Details, false) },
"db_unexpected_trigger", "db_unexpected_event",
"db_unexpected_procedure", "db_unexpected_function")
register(func(in VerifyInput) VerifyResult { return verifyDBObject(in.Details, true) },
"db_malicious_trigger", "db_malicious_event",
"db_malicious_procedure", "db_malicious_function")
register(func(in VerifyInput) VerifyResult { return verifyDBMagicTokenUser(in.Message, in.Details) },
"db_magic_token_user")
register(func(in VerifyInput) VerifyResult { return verifyDBRogueAdmin(in.Message, in.Details) },
"db_rogue_admin")
register(func(in VerifyInput) VerifyResult { return verifyDBSuspiciousAdminEmail(in.Message, in.Details) },
"db_suspicious_admin_email")
register(func(in VerifyInput) VerifyResult { return verifyDrupalAdminInjection(in.Message, in.Details) },
"drupal_admin_injection")
register(func(in VerifyInput) VerifyResult { return verifyJoomlaAdminInjection(in.Message, in.Details) },
"joomla_admin_injection")
register(func(in VerifyInput) VerifyResult { return verifyMagentoAdminInjection(in.Message, in.Details) },
"magento_admin_injection")
register(func(in VerifyInput) VerifyResult { return verifyOpenCartAdminInjection(in.Message, in.Details) },
"opencart_admin_injection")
register(func(in VerifyInput) VerifyResult { return verifyPrestaShopAdminInjection(in.Message, in.Details) },
"prestashop_admin_injection")
return m
}
// VerifyFinding re-evaluates a finding by check type + message/details/path.
// Preserved signature for CLI/legacy callers; carries no content fingerprint.
func VerifyFinding(checkType, message, details string, filePath ...string) VerifyResult {
return VerifyFindingInput(VerifyInput{
Check: checkType, Message: message, Details: details,
Path: selectFindingPath(message, filePath...),
})
}
// VerifyFindingInput re-evaluates a finding from a full VerifyInput. Callers
// that verify content findings must provide the stored detection fingerprint.
func VerifyFindingInput(in VerifyInput) VerifyResult {
if in.Path == "" {
in.Path = selectFindingPath(in.Message)
}
if fn, ok := findingVerifiers[in.Check]; ok {
return fn(in)
}
return VerifyResult{Checked: false, Detail: fmt.Sprintf("no automated re-check available for '%s'", in.Check)}
}
// reverifyContentFinding re-runs the content classifier that originally
// produced the finding. It resolves only when the file is gone OR the file's
// bytes are byte-for-byte identical to detection time AND the current
// classifier no longer flags them -- a superseded-heuristic false positive.
// A file modified since detection is never auto-cleared to prevent partial
// cleans or evasion edits from being mistaken for a fix.
func reverifyContentFinding(in VerifyInput) VerifyResult {
if in.Path == "" {
return VerifyResult{Checked: false, Detail: "could not extract file path from finding"}
}
clean, info, exists, err := readOnlyFixPath(in.Path, effectiveFixRoots(fixQuarantineAllowedRoots, quarantineExtraRoots...))
if err != nil {
return VerifyResult{Checked: false, Detail: err.Error()}
}
if !exists {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("file no longer present (removed or quarantined): %s", clean)}
}
if !info.Mode().IsRegular() {
return VerifyResult{Checked: false, Detail: "path is not a regular file; not auto-verifiable"}
}
matched, label, currentHash, err := contentStillMatches(in.Check, clean, info)
if err != nil {
return VerifyResult{Checked: false, Detail: err.Error()}
}
if matched {
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("still flagged by current detection logic: %s", label)}
}
switch {
case in.ContentSHA256 != "" && currentHash != "" && currentHash == in.ContentSHA256:
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf(
"identical content (sha256 unchanged) is no longer flagged by current detection logic (%s) -- superseded-heuristic false positive",
ContentDetectionVersion())}
case in.ContentSHA256 != "":
return demotionForChangedContent(in.Check, clean, info, currentHash,
"file modified since detection (sha256 mismatch); not auto-cleared")
default:
return demotionForChangedContent(in.Check, clean, info, currentHash,
"current detection logic no longer flags this file, but no detection-time fingerprint exists to rule out modification")
}
}
// demotionEntropyCeiling rejects a replacement that reads as packed or encoded.
// The inert-source gate below is stricter, but checking entropy first keeps a
// large encoded comment blob ineligible too.
const demotionEntropyCeiling = 5.0
// changedContentDemotionEligible reports whether a changed-content finding may
// be considered for demotion. Every check whose condition this package can
// re-run qualifies: what makes a demotion safe is the inert-replacement gate,
// which reads the bytes, not the name of the detector that flagged them.
func changedContentDemotionEligible(check string) bool {
return IsContentReverifiable(check)
}
// demotionForChangedContent decides whether a finding whose file changed since
// detection, and which current detection no longer flags, may drop out of the
// live queue. It never clears the finding.
func demotionForChangedContent(check, path string, info os.FileInfo, classifiedHash, reason string) VerifyResult {
if !changedContentDemotionEligible(check) {
return VerifyResult{Checked: true, Resolved: false,
Detail: reason + "; changed content is not eligible for automatic demotion -- review manually"}
}
// The classifier and the demotion gates must inspect the same bytes. A
// bounded second snapshot avoids unbounded allocation, preserves inode and
// metadata identity, and its digest closes the gap between the two reads.
snap, err := readContentSnapshotForReverifyBounded(path, info, contentFingerprintMaxBytes)
if err != nil {
return VerifyResult{Checked: true, Resolved: false,
Detail: reason + "; replacement exceeds the demotion read limit or changed during verification -- review manually"}
}
if classifiedHash == "" || snap.sha256 != classifiedHash {
return VerifyResult{Checked: true, Resolved: false,
Detail: reason + "; replacement changed during verification -- review manually"}
}
content := string(snap.data)
if hasObfuscatedExecutionSignal(content) {
return VerifyResult{Checked: true, Resolved: false,
Detail: reason + "; replacement content still carries an obfuscated-execution signal -- review manually"}
}
if shannonEntropy(content) >= demotionEntropyCeiling {
return VerifyResult{Checked: true, Resolved: false,
Detail: reason + "; replacement content reads as packed or encoded -- review manually"}
}
if !isInertPHPReplacement(content) {
return VerifyResult{Checked: true, Resolved: false,
Detail: reason + "; replacement still contains active PHP or web content -- review manually"}
}
return VerifyResult{Checked: true, Resolved: false, Demote: true,
Detail: reason + "; replacement is an inert PHP stub -- confirm remediation"}
}
// isInertPHPReplacement deliberately proves a very small safe shape instead of
// trying to blacklist every way PHP can execute. Current detection not matching
// is already a precondition here; another deny-list would make any omitted
// include, callback, side-effect function, or inline script a severity-demotion
// bypass. Only an empty file or a comment-only, non-closing PHP stub is inert.
// Rejecting closing tags also rejects inline HTML or JavaScript that would stay
// live after PHP execution stops.
func isInertPHPReplacement(content string) bool {
// Preserve the byte after the opener even when it is trailing whitespace:
// trimming it could turn a non-tag into the valid "<?php" at EOF.
trimmed := strings.TrimLeftFunc(content, unicode.IsSpace)
if trimmed == "" {
return true
}
if len(trimmed) < len("<?php") || !strings.EqualFold(trimmed[:len("<?php")], "<?php") {
return false
}
if len(trimmed) > len("<?php") && !isPHPOpenTagSpace(trimmed[len("<?php")]) {
return false
}
if strings.Contains(trimmed, "?>") {
return false
}
code := stripPHPCommentsFromCode(phpCodeOnly(trimmed))
return strings.TrimSpace(code) == ""
}
// contentStillMatches re-runs the appropriate classifier for the check type.
// For files small enough to fingerprint, the returned hash and classifier result
// come from the same opened content snapshot.
func contentStillMatches(check, path string, info os.FileInfo) (bool, string, string, error) {
switch check {
case "signature_match_realtime":
s := contentSignatureScanner()
if s == nil || s.RuleCount() == 0 {
return false, "", "", fmt.Errorf("signature scanner unavailable")
}
// Bounded like the deep scan: the sweep runs unattended over every
// stored content finding, and a flagged file an attacker has since
// grown must leave the finding unresolved, not be read whole.
snap, err := readContentSnapshotForReverifyBounded(path, info, FullScanMaxFileBytes(config.Active()))
if err != nil {
return false, "", "", fmt.Errorf("cannot read file: %w", err)
}
if hits := s.ScanContent(snap.data, strings.ToLower(filepath.Ext(path))); len(hits) > 0 {
return true, fmt.Sprintf("%d signature match(es)", len(hits)), snap.sha256, nil
}
return false, "", snap.sha256, nil
case "yara_match_realtime", "yara_match_scheduled":
y := contentYARAScanner()
if y == nil || y.RuleCount() == 0 {
return false, "", "", fmt.Errorf("YARA scanner unavailable")
}
snap, err := readContentSnapshotForReverifyBounded(path, info, FullScanMaxFileBytes(config.Active()))
if err != nil {
return false, "", "", fmt.Errorf("cannot read file: %w", err)
}
hits, err := yara.ScanBytesChecked(y, path, snap.data)
if err != nil {
// Fail closed: a scan that could not complete (worker down, a
// transport error) must not auto-clear a still-infected finding as
// a superseded false positive.
return false, "", "", fmt.Errorf("YARA scan error: %v", err)
}
// YARA findings carry the scanned digest even above the size cap
// used by path-based fingerprinting. Compare the same complete bytes.
if snap.sha256 == "" {
snap.sha256 = fmt.Sprintf("%x", sha256.Sum256(snap.data))
}
if len(hits) > 0 {
return true, fmt.Sprintf("%d YARA rule match(es)", len(hits)), snap.sha256, nil
}
return false, "", snap.sha256, nil
case "js_keylogger_dataflow":
if info.Size() > jstaint.MaxSourceBytes {
return false, "", "", fmt.Errorf("JS taint analysis did not complete: %s", jstaint.StatusOversize)
}
snap, err := readContentSnapshotForReverifyBounded(path, info, jstaint.MaxSourceBytes)
if err != nil {
if errors.Is(err, errContentSnapshotTooLarge) {
return false, "", "", fmt.Errorf("JS taint analysis did not complete: %s", jstaint.StatusOversize)
}
return false, "", "", fmt.Errorf("cannot read file: %v", err)
}
rctx, cancel := context.WithTimeout(context.Background(), jsTaintReverifyTimeout)
defer cancel()
report := runJSTaintAnalysis(rctx, snap.data)
switch report.Status {
case jstaint.StatusAnalyzed, jstaint.StatusNotCandidate:
if len(report.Results) > 0 {
return true, fmt.Sprintf("%d keystroke exfiltration flow(s)", report.TotalResults), snap.sha256, nil
}
return false, "", snap.sha256, nil
default:
// Fail closed: a coverage-gap status (oversize, parse error,
// resource limit, cancellation, panic) must not auto-clear a
// still-infected finding as a superseded false positive.
return false, "", "", fmt.Errorf("JS taint analysis did not complete: %s", report.Status)
}
case "php_remote_taint":
if info.Size() > phptaint.MaxSourceBytes {
return false, "", "", fmt.Errorf("PHP taint analysis did not complete: %s", phptaint.StatusOversize)
}
snap, err := readContentSnapshotForReverifyBounded(path, info, phptaint.MaxSourceBytes)
if err != nil {
if errors.Is(err, errContentSnapshotTooLarge) {
return false, "", "", fmt.Errorf("PHP taint analysis did not complete: %s", phptaint.StatusOversize)
}
return false, "", "", fmt.Errorf("cannot read file: %v", err)
}
rctx, cancel := context.WithTimeout(context.Background(), phpTaintDeepPerFileTimeout)
defer cancel()
report := runPHPTaintAnalysis(rctx, snap.data)
switch report.Status {
case phptaint.StatusAnalyzed, phptaint.StatusNotCandidate:
if len(report.Results) > 0 {
return true, fmt.Sprintf("%d remote-source code execution flow(s)", report.TotalResults), snap.sha256, nil
}
return false, "", snap.sha256, nil
default:
// The isolated worker is mandatory here too. Any incomplete status
// leaves the finding unresolved instead of falling through to the
// unrelated PHP heuristic classifier and clearing it.
return false, "", "", fmt.Errorf("PHP taint analysis did not complete: %s", report.Status)
}
default: // PHP heuristic content family
res, currentHash, err := analyzePHPContentForReverify(path, info)
if err != nil {
return false, "", "", fmt.Errorf("cannot read file: %v", err)
}
if res.severity >= 0 {
return true, strings.Join(res.indicators, ", "), currentHash, nil
}
return false, "", currentHash, nil
}
}
func analyzePHPContentForReverify(path string, expected os.FileInfo) (phpAnalysisResult, string, error) {
if expected != nil && expected.Size() <= contentFingerprintMaxBytes {
snap, err := readContentSnapshotForReverify(path, expected)
if err != nil {
return phpAnalysisResult{}, "", err
}
res := analyzePHPContentReaderAt(path, bytes.NewReader(snap.data), int64(len(snap.data)))
if !res.readOK {
return phpAnalysisResult{}, "", fmt.Errorf("content read failed")
}
return res, snap.sha256, nil
}
f, info, err := openReadOnlyPreservingIdentity(path, expected)
if err != nil {
return phpAnalysisResult{}, "", err
}
defer func() { _ = f.Close() }()
res := analyzePHPContentReaderAt(path, f, info.Size())
if !res.readOK {
return phpAnalysisResult{}, "", fmt.Errorf("content read failed")
}
return res, "", nil
}
type reverifyContentSnapshot struct {
data []byte
sha256 string
}
var errContentSnapshotTooLarge = errors.New("content snapshot exceeds read limit")
func readContentSnapshotForReverify(path string, expected os.FileInfo) (reverifyContentSnapshot, error) {
return readContentSnapshotForReverifyBounded(path, expected, 0)
}
func readContentSnapshotForReverifyBounded(path string, expected os.FileInfo, maxBytes int64) (reverifyContentSnapshot, error) {
f, info, err := openReadOnlyPreservingIdentity(path, expected)
if err != nil {
return reverifyContentSnapshot{}, err
}
defer func() { _ = f.Close() }()
if maxBytes > 0 && info.Size() > maxBytes {
return reverifyContentSnapshot{}, errContentSnapshotTooLarge
}
var reader io.Reader = f
if maxBytes > 0 {
reader = io.LimitReader(f, maxBytes+1)
}
data, err := io.ReadAll(reader)
if err != nil {
return reverifyContentSnapshot{}, err
}
after, err := f.Stat()
if err != nil {
return reverifyContentSnapshot{}, err
}
if !sameFileIdentity(after, info) || !sameCleanContentShape(after, info) {
return reverifyContentSnapshot{}, fmt.Errorf("file changed during verification")
}
if int64(len(data)) != info.Size() {
return reverifyContentSnapshot{}, fmt.Errorf("file changed during verification")
}
if maxBytes > 0 && int64(len(data)) > maxBytes {
return reverifyContentSnapshot{}, errContentSnapshotTooLarge
}
var digest string
if info.Size() <= contentFingerprintMaxBytes {
sum := sha256.Sum256(data)
digest = fmt.Sprintf("%x", sum)
}
return reverifyContentSnapshot{data: data, sha256: digest}, nil
}
func openReadOnlyPreservingIdentity(path string, expected os.FileInfo) (*os.File, os.FileInfo, error) {
// #nosec G304 -- path was validated by readOnlyFixPath against remediation
// roots; O_NOFOLLOW plus sameFileIdentity fails closed on inode swap.
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
if err != nil {
return nil, nil, err
}
info, err := f.Stat()
if err != nil {
_ = f.Close()
return nil, nil, err
}
if !sameFileIdentity(info, expected) || !sameCleanContentShape(info, expected) {
_ = f.Close()
return nil, nil, fmt.Errorf("file changed during verification")
}
return f, info, nil
}
// CanVerify reports whether VerifyFinding has an automated re-check for the
// given check type. The Web UI gates the per-finding "Re-check" action on this
// so it never shows a button that could only report "not auto-verifiable".
func CanVerify(checkType string) bool {
_, ok := findingVerifiers[checkType]
return ok
}
func verifyWriteBit(path string, bit os.FileMode, label string) VerifyResult {
if path == "" {
return VerifyResult{Checked: false, Detail: "could not extract file path from finding"}
}
clean, info, exists, err := readOnlyFixPath(path, effectiveFixRoots(fixPermissionsAllowedRoots))
if err != nil {
return VerifyResult{Checked: false, Detail: err.Error()}
}
if !exists {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("file no longer exists: %s", clean)}
}
if !info.Mode().IsRegular() {
return VerifyResult{Checked: false, Detail: "path is not a regular file; not auto-verifiable"}
}
if info.Mode().Perm()&bit == 0 {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("file is no longer %s (mode %o)", label, info.Mode().Perm())}
}
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("file is still %s (mode %o)", label, info.Mode().Perm())}
}
func verifyPathAbsent(path string, roots []string) VerifyResult {
if path == "" {
return VerifyResult{Checked: false, Detail: "could not extract file path from finding"}
}
clean, exists, err := readOnlyPathPresence(path, roots)
if err != nil {
return VerifyResult{Checked: false, Detail: err.Error()}
}
if !exists {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("file no longer present (removed or quarantined): %s", clean)}
}
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("file is still present: %s", clean)}
}
func verifyHtaccessClean(path string, audit func(string, []byte) ([]alert.Finding, []htaccessByteRange)) VerifyResult {
if path == "" {
return VerifyResult{Checked: false, Detail: "could not extract file path from finding"}
}
clean, info, exists, err := readOnlyFixPath(path, effectiveFixRoots(fixHtaccessAllowedRoots))
if err != nil {
return VerifyResult{Checked: false, Detail: err.Error()}
}
if filepath.Base(clean) != ".htaccess" {
return VerifyResult{Checked: false, Detail: "not a .htaccess file; not auto-verifiable"}
}
if !exists {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf(".htaccess no longer exists: %s", clean)}
}
if !info.Mode().IsRegular() {
return VerifyResult{Checked: false, Detail: ".htaccess path is not a regular file; not auto-verifiable"}
}
snap, err := readContentSnapshotForReverifyBounded(clean, info, htaccessMaxFileBytes)
if errors.Is(err, errContentSnapshotTooLarge) {
return VerifyResult{Checked: false, Detail: ".htaccess too large to verify automatically"}
}
if err != nil {
return VerifyResult{Checked: false, Detail: fmt.Sprintf("cannot read .htaccess: %v", err)}
}
findings, _ := audit(clean, snap.data)
if len(findings) == 0 {
return VerifyResult{Checked: true, Resolved: true, Detail: "no malicious directives remain in .htaccess"}
}
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("%d malicious directive(s) still present in .htaccess", len(findings))}
}
func verifyEximSpoolAbsent(message string) VerifyResult {
msgID := extractEximMsgID(message)
if msgID == "" {
return VerifyResult{Checked: false, Detail: "could not extract Exim message ID from finding"}
}
if !eximMsgIDRegex.MatchString(msgID) {
return VerifyResult{Checked: false, Detail: fmt.Sprintf("invalid Exim message ID format: %s", msgID)}
}
for _, dir := range eximSpoolMessageDirs(msgID) {
if _, err := osFS.Lstat(filepath.Join(dir, msgID+"-H")); err == nil {
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("spool message %s is still queued", msgID)}
} else if !os.IsNotExist(err) {
return VerifyResult{Checked: false, Detail: fmt.Sprintf("cannot stat spool message %s: %v", msgID, err)}
}
}
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("spool message %s no longer present (delivered or removed)", msgID)}
}
func verifyCrontabClear(path string) VerifyResult {
if path == "" {
return VerifyResult{Checked: false, Detail: "could not extract crontab path from finding"}
}
clean, info, exists, err := readOnlyFixPath(path, fixCrontabAllowedRoots)
if err != nil {
return VerifyResult{Checked: false, Detail: err.Error()}
}
if !exists {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("crontab no longer exists: %s", clean)}
}
if !info.Mode().IsRegular() {
return VerifyResult{Checked: false, Detail: "crontab path is not a regular file; not auto-verifiable"}
}
if info.Size() == 0 {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("crontab is empty: %s", clean)}
}
// A non-empty crontab may still be legitimate; re-scanning its content is
// out of scope for a single-finding re-check, so report it as still
// present rather than guessing.
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("crontab is still present and non-empty: %s", clean)}
}
func readOnlyFixPath(path string, allowedRoots []string) (string, os.FileInfo, bool, error) {
clean, err := sanitizeFixPath(path, allowedRoots)
if err != nil {
return "", nil, false, err
}
root := matchingAllowedRoot(clean, allowedRoots)
if root == "" {
return "", nil, false, fmt.Errorf("file path is outside the allowed remediation roots: %s", clean)
}
current := root
info, err := osFS.Lstat(current)
if err != nil {
if os.IsNotExist(err) {
return clean, nil, false, nil
}
return "", nil, false, fmt.Errorf("cannot stat: %v", err)
}
if info.Mode()&os.ModeSymlink != 0 {
return "", nil, false, fmt.Errorf("symlinked paths are not eligible for automated verification: %s", current)
}
if current == clean {
return clean, info, true, nil
}
rel, err := filepath.Rel(current, clean)
if err != nil {
return "", nil, false, fmt.Errorf("cannot resolve path under allowed root: %v", err)
}
parts := strings.Split(rel, string(filepath.Separator))
for i, part := range parts {
if part == "" || part == "." {
continue
}
if !info.IsDir() {
return "", nil, false, fmt.Errorf("path ancestor is not a directory; not auto-verifiable: %s", current)
}
current = filepath.Join(current, part)
info, err = osFS.Lstat(current)
if err != nil {
if os.IsNotExist(err) {
return clean, nil, false, nil
}
return "", nil, false, fmt.Errorf("cannot stat: %v", err)
}
if info.Mode()&os.ModeSymlink != 0 {
return "", nil, false, fmt.Errorf("symlinked paths are not eligible for automated verification: %s", current)
}
if i < len(parts)-1 && !info.IsDir() {
return "", nil, false, fmt.Errorf("path ancestor is not a directory; not auto-verifiable: %s", current)
}
}
return clean, info, true, nil
}
func readOnlyPathPresence(path string, allowedRoots []string) (string, bool, error) {
clean, err := sanitizeFixPath(path, allowedRoots)
if err != nil {
return "", false, err
}
root := matchingAllowedRoot(clean, allowedRoots)
if root == "" {
return "", false, fmt.Errorf("file path is outside the allowed remediation roots: %s", clean)
}
current := root
info, err := osFS.Lstat(current)
if err != nil {
if os.IsNotExist(err) {
return clean, false, nil
}
return "", false, fmt.Errorf("cannot stat: %v", err)
}
if info.Mode()&os.ModeSymlink != 0 {
return "", false, fmt.Errorf("symlinked paths are not eligible for automated verification: %s", current)
}
if current == clean {
return clean, true, nil
}
rel, err := filepath.Rel(current, clean)
if err != nil {
return "", false, fmt.Errorf("cannot resolve path under allowed root: %v", err)
}
parts := strings.Split(rel, string(filepath.Separator))
for i, part := range parts {
if part == "" || part == "." {
continue
}
if !info.IsDir() {
return "", false, fmt.Errorf("path ancestor is not a directory; not auto-verifiable: %s", current)
}
current = filepath.Join(current, part)
info, err = osFS.Lstat(current)
if err != nil {
if os.IsNotExist(err) {
return clean, false, nil
}
return "", false, fmt.Errorf("cannot stat: %v", err)
}
if i < len(parts)-1 {
if info.Mode()&os.ModeSymlink != 0 {
return "", false, fmt.Errorf("symlinked paths are not eligible for automated verification: %s", current)
}
if !info.IsDir() {
return "", false, fmt.Errorf("path ancestor is not a directory; not auto-verifiable: %s", current)
}
}
}
return clean, true, nil
}
func matchingAllowedRoot(path string, allowedRoots []string) string {
var best string
for _, root := range allowedRoots {
cleanRoot := filepath.Clean(strings.TrimSpace(root))
if isPathWithinOrEqual(path, cleanRoot) && len(cleanRoot) > len(best) {
best = cleanRoot
}
}
return best
}
package checks
import (
"context"
"fmt"
"regexp"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/mysqlclient"
)
// Per-finding Re-check for the database-content family.
//
// These verifiers re-evaluate a single database finding against the live
// database and clear it only on confirmed evidence -- the row is gone, or its
// current value no longer matches the detector that raised it. The cardinal
// rule is the same as the filesystem re-checks: NEVER false-resolve. Clearing a
// db_* finding auto-dismisses a live database compromise, so any ambiguity
// (connection failure, query error, a site we can no longer locate, an
// unparseable finding) returns Checked:false and leaves the finding in place
// for a full account scan.
//
// WordPress re-checks query as root against the finding's schema
// (mysqlclient.RootQuerySchema) rather than the per-account credentials in
// wp-config.php, which drift on cPanel password rotations. The non-WordPress
// CMS re-checks use each install's own login and endpoint instead
// (runDBVerifyQueryAs): the scanner already depends on them, a shop's database
// can live on another server or socket where root's same-named schema is not
// the shop's, and a failed login only leaves the finding unchecked.
// dbVerifyTimeout bounds a single synchronous database re-check.
var dbVerifyTimeout = 30 * time.Second
// validAccountName guards an account name parsed out of finding text before it
// is interpolated into a /home/<account>/... glob.
var validAccountName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,31}$`)
// messageAccountRe pulls account tokens out of WordPress finding messages.
var messageAccountRe = regexp.MustCompile(`(^|[,(])\s*account:\s*([A-Za-z][A-Za-z0-9_-]{0,31})([,)]|$)`)
// detailField returns the value of a "Key: value" line in a finding's Details
// block, or "" when the key is absent. The key match is case-sensitive and the
// value is whitespace-trimmed.
func detailField(details, key string) string {
value, ok := detailFieldPresent(details, key)
if !ok {
return ""
}
return value
}
func detailFieldPresent(details, key string) (string, bool) {
want := key + ":"
for _, line := range strings.Split(details, "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, want) {
return strings.TrimSpace(line[len(want):]), true
}
}
return "", false
}
// dbFindingAccount resolves the cPanel account a database finding belongs to.
// CMS, DB-object, and admin findings carry it as an "Account:" detail line; the
// WordPress content findings carry it only in the message "(account: <user>)".
func dbFindingAccount(message, details string) string {
if a, ok := detailFieldPresent(details, "Account"); ok {
return validFindingAccount(a)
}
matches := messageAccountRe.FindAllStringSubmatch(message, -1)
if len(matches) > 0 {
return matches[len(matches)-1][2]
}
return ""
}
func validFindingAccount(account string) string {
account = strings.TrimSpace(account)
if !validAccountName.MatchString(account) {
return ""
}
return account
}
// runDBVerifyQueryAs runs a re-check query with an install's own database
// login and endpoint, preserving the error like runDBVerifyQueryRoot.
func runDBVerifyQueryAs(creds wpDBCreds, query string, args ...any) ([]string, error) {
ctx, cancel := context.WithTimeout(context.Background(), dbVerifyTimeout)
defer cancel()
return mysqlclient.PerAccountQuery(ctx, mysqlclient.Creds{
Host: creds.dbHost, DBName: creds.dbName, User: creds.dbUser, Password: creds.dbPass,
}, query, args...)
}
// runDBVerifyQueryRoot runs a root-credential query against an explicit schema
// and returns the rows plus the error. Unlike runMySQLQueryRoot (which collapses
// any error into a nil slice), this preserves the error so a re-check can tell
// "query ran, zero rows" (the row is gone/clean -> safe to resolve) apart from
// "query failed" (DB down, connection refused -> must NOT resolve).
func runDBVerifyQueryRoot(schema, query string, args ...any) ([]string, error) {
if strings.TrimSpace(schema) == "" {
return nil, fmt.Errorf("empty schema")
}
ctx, cancel := context.WithTimeout(context.Background(), dbVerifyTimeout)
defer cancel()
return mysqlclient.RootQuerySchema(ctx, schema, query, args...)
}
// dbVerifyQueryError is the standard Checked:false result for a failed re-check
// query. The raw error is intentionally not surfaced to the operator (it can
// carry DSN/host internals); the generic guidance is enough to act on.
func dbVerifyQueryError() VerifyResult {
return VerifyResult{Checked: false, Detail: "could not query the database (try again, or run an account scan)"}
}
// dbVerifyNotLocatable is the standard Checked:false result when the re-check
// cannot re-discover the install (site removed, DB renamed, config unreadable).
func dbVerifyNotLocatable(kind string) VerifyResult {
return VerifyResult{Checked: false, Detail: fmt.Sprintf("could not locate the %s to re-check (run an account scan)", kind)}
}
// findWPVerifyPrefixes re-discovers the WordPress install for account whose
// database matches dbName, returning the table prefixes a verifier should query.
// New findings carry an exact Table prefix detail. Older findings do not, so the
// fallback enumerates every prefix in that WordPress install, including active
// multisite secondary blogs. ok=false means the caller refuses to guess.
func findWPVerifyPrefixes(account, dbName, details string) (prefixes []string, ok bool) {
if !validAccountName.MatchString(account) || dbName == "" {
return nil, false
}
// Shared discovery (wpinstalls.go): a re-check that cannot re-locate the
// install it flagged leaves the finding unresolvable for the life of the
// site, so it must see exactly what the detector saw.
patterns := wpInstallConfigPaths(wpInstallsForAccount(context.Background(), "db_content", account))
targetPrefix := detailField(details, "Table prefix")
if targetPrefix != "" && !validTablePrefix.MatchString(targetPrefix) {
return nil, false
}
seen := map[string]bool{}
for _, path := range patterns {
creds := parseWPConfig(path)
if creds.dbName != dbName {
continue
}
p, ok := resolveTablePrefix(creds)
if !ok {
return nil, false
}
if targetPrefix != "" {
if wpVerifyPrefixMatchesBase(targetPrefix, p) {
return []string{targetPrefix}, true
}
continue
}
addPrefix(&prefixes, seen, p)
if creds.multisite {
multisitePrefixes, err := wpVerifyMultisitePrefixes(dbName, p)
if err != nil {
return nil, false
}
for _, prefix := range multisitePrefixes {
addPrefix(&prefixes, seen, prefix)
}
}
}
if len(prefixes) == 0 {
return nil, false
}
return prefixes, true
}
func addPrefix(prefixes *[]string, seen map[string]bool, prefix string) {
if seen[prefix] {
return
}
seen[prefix] = true
*prefixes = append(*prefixes, prefix)
}
func wpVerifyPrefixMatchesBase(prefix, base string) bool {
if prefix == base {
return true
}
if !strings.HasPrefix(prefix, base) || !strings.HasSuffix(prefix, "_") {
return false
}
blogID := strings.TrimSuffix(strings.TrimPrefix(prefix, base), "_")
return isAllDigits(blogID)
}
func wpVerifyMultisitePrefixes(dbName, basePrefix string) ([]string, error) {
rows, err := runDBVerifyQueryRoot(dbName, fmt.Sprintf(
"SELECT blog_id FROM `%sblogs` WHERE archived = 0 AND deleted = 0 AND spam = 0 AND blog_id != 1",
basePrefix,
))
if err != nil {
return nil, err
}
var prefixes []string
for _, row := range rows {
blogID := strings.TrimSpace(row)
if blogID == "" || blogID == "1" || !isAllDigits(blogID) {
continue
}
prefixes = append(prefixes, fmt.Sprintf("%s%s_", basePrefix, blogID))
}
return prefixes, nil
}
// dbInjectionCoreOptions are the wp_options names that must never carry a
// <script> tag (mirrors the Path-2 core-option check in checkWPOptions).
var dbInjectionCoreOptions = map[string]bool{
"siteurl": true, "home": true, "blogname": true,
"blogdescription": true, "admin_email": true,
}
// verifyDBOptionsInjection re-reads the flagged wp_options row and resolves the
// finding when the option is gone or no longer carries an injected external
// script (mirrors checkWPOptions' two detection paths).
func verifyDBOptionsInjection(message, details string) VerifyResult {
dbName := detailField(details, "Database")
optName := detailField(details, "Option")
if optName == "" {
return VerifyResult{Checked: false, Detail: "could not parse the option name from the finding"}
}
prefixes, ok := findWPVerifyPrefixes(dbFindingAccount(message, details), dbName, details)
if !ok {
return dbVerifyNotLocatable("WordPress site")
}
present := false
for _, prefix := range prefixes {
rows, err := runDBVerifyQueryRoot(dbName,
fmt.Sprintf("SELECT option_value FROM `%soptions` WHERE option_name = ?", prefix), optName)
if err != nil {
return dbVerifyQueryError()
}
for _, row := range rows {
present = true
if optionValueStillMalicious(optName, row) {
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("option %q still contains injected content", optName)}
}
}
}
if !present {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("option %q is no longer present", optName)}
}
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("option %q no longer contains injected content", optName)}
}
func optionValueStillMalicious(optName, value string) bool {
if extractMaliciousScriptURL(value) != "" {
return true
}
if dbInjectionCoreOptions[optName] && strings.Contains(strings.ToLower(value), "<script") {
return true
}
return false
}
// verifyDBSiteurlHijack re-reads the flagged siteurl/home option and resolves
// when the option is gone or no longer matches either site-address check in
// checkWPOptions.
func verifyDBSiteurlHijack(message, details string) VerifyResult {
dbName := detailField(details, "Database")
optName := siteurlOptionFromDetails(details)
if optName == "" {
return VerifyResult{Checked: false, Detail: "could not parse the option name from the finding"}
}
prefixes, ok := findWPVerifyPrefixes(dbFindingAccount(message, details), dbName, details)
if !ok {
return dbVerifyNotLocatable("WordPress site")
}
present := false
for _, prefix := range prefixes {
rows, err := runDBVerifyQueryRoot(dbName,
fmt.Sprintf("SELECT option_value FROM `%soptions` WHERE option_name = ?", prefix), optName)
if err != nil {
return dbVerifyQueryError()
}
for _, row := range rows {
present = true
if siteurlValueStillMalicious(row) {
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("option %q is still a poisoned site address", optName)}
}
}
}
if !present {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("option %q is no longer present", optName)}
}
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("option %q is no longer a poisoned site address", optName)}
}
func siteurlValueStillMalicious(value string) bool {
lower := strings.ToLower(value)
if strings.Contains(lower, "eval(") || strings.Contains(lower, "<script") {
return true
}
_, poisoned := siteURLPoisonReason(value)
return poisoned
}
// siteurlOptionFromDetails extracts the option name from a site-address finding
// detail block whose body line is "<optName> = <value>". The detector only ever
// emits siteurl or home; any other shape returns "".
func siteurlOptionFromDetails(details string) string {
for _, line := range strings.Split(details, "\n") {
line = strings.TrimSpace(line)
for _, opt := range []string{"siteurl", "home"} {
if strings.HasPrefix(line, opt+" =") {
return opt
}
}
}
return ""
}
// verifyDBPostInjection searches the published posts for the injected pattern
// again, the way the detector found it, and resolves the finding only when no
// post still matches (mirrors checkWPPosts' post-status, post-type, and
// external-script filters). The finding lists at most five example post IDs;
// re-reading only those resolved a finding whose injection sat in every other
// post once the examples were cleaned.
func verifyDBPostInjection(message, details string) VerifyResult {
dbName := detailField(details, "Database")
pattern := detailField(details, "Pattern")
if pattern == "" {
return VerifyResult{Checked: false, Detail: "could not parse the injected pattern from the finding"}
}
requiresExternalScript, ok := lookupMalwarePattern(pattern)
if !ok {
return VerifyResult{Checked: false, Detail: "finding pattern is not auto-verifiable"}
}
prefixes, ok := findWPVerifyPrefixes(dbFindingAccount(message, details), dbName, details)
if !ok {
return dbVerifyNotLocatable("WordPress site")
}
like := likeContains(pattern)
for _, prefix := range prefixes {
var afterID uint64
for {
rows, err := runDBVerifyQueryRoot(dbName,
fmt.Sprintf("SELECT ID, post_content, post_content_filtered FROM `%sposts` WHERE ID > ? AND post_status='publish' AND post_type NOT IN (%s) AND (post_content LIKE ? OR post_content_filtered LIKE ?) ORDER BY ID LIMIT %d",
prefix, nonScannablePostTypesSQLList(), dbVerifyPostSearchLimit),
afterID, like, like)
if err != nil {
return dbVerifyQueryError()
}
for _, row := range rows {
parts := strings.SplitN(row, "\t", 3)
if len(parts) < 2 {
return VerifyResult{Checked: false, Detail: "database returned an unexpected post row; finding was not resolved"}
}
id, err := strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 64)
if err != nil || id <= afterID {
return VerifyResult{Checked: false, Detail: "database returned an invalid post ID; finding was not resolved"}
}
afterID = id
content := parts[1]
if len(parts) == 3 {
content += "\n" + parts[2]
}
if postContentMatchesPattern(pattern, requiresExternalScript, content) {
return VerifyResult{Checked: true, Resolved: false, Detail: "an affected post still contains the injected pattern"}
}
}
if len(rows) < dbVerifyPostSearchLimit {
break
}
}
}
return VerifyResult{Checked: true, Resolved: true, Detail: "no affected post still contains the injected pattern"}
}
func postContentMatchesPattern(pattern string, requiresExternalScript bool, content string) bool {
if !strings.Contains(strings.ToLower(content), strings.ToLower(pattern)) {
return false
}
if requiresExternalScript {
return hasMaliciousExternalScriptInPost(content)
}
return true
}
// lookupMalwarePattern reports whether pattern is a known dbMalwarePatterns
// entry and, if so, whether it requires the external-script post-filter.
func lookupMalwarePattern(pattern string) (requiresExternalScript bool, ok bool) {
for _, mp := range dbMalwarePatterns {
if mp.pattern == pattern {
return mp.requiresExternalScript, true
}
}
return false, false
}
// verifyDBSpamInjection re-runs the cloaked-spam scan for the finding's keyword
// against the site's published posts and resolves when no post still matches
// (mirrors checkWPPosts' three-layer spam filter).
func verifyDBSpamInjection(message, details string) VerifyResult {
dbName := detailField(details, "Database")
keyword := spamKeywordFromMessage(message)
if keyword == "" {
return VerifyResult{Checked: false, Detail: "could not parse the spam keyword from the finding"}
}
sp, ok := lookupSpamPattern(keyword)
if !ok {
return VerifyResult{Checked: false, Detail: "spam keyword is not auto-verifiable"}
}
prefixes, ok := findWPVerifyPrefixes(dbFindingAccount(message, details), dbName, details)
if !ok {
return dbVerifyNotLocatable("WordPress site")
}
var contents []string
for _, prefix := range prefixes {
rows, err := runDBVerifyQueryRoot(dbName,
fmt.Sprintf("SELECT ID, post_content FROM `%sposts` WHERE post_status='publish' AND post_type NOT IN (%s) AND post_content LIKE ? LIMIT 200",
prefix, nonScannablePostTypesSQLList()), sp.likeFragment)
if err != nil {
return dbVerifyQueryError()
}
for _, row := range rows {
parts := strings.SplitN(row, "\t", 2)
if len(parts) >= 2 {
contents = append(contents, parts[1])
}
}
}
if countCloakedSpamMatches(sp, contents) == 0 {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("no published post still contains cloaked spam keyword %q", keyword)}
}
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("published posts still contain cloaked spam keyword %q", keyword)}
}
var spamKeywordRe = regexp.MustCompile(`cloaked spam keyword '([^']+)'`)
func spamKeywordFromMessage(message string) string {
m := spamKeywordRe.FindStringSubmatch(message)
if m == nil {
return ""
}
return m[1]
}
func lookupSpamPattern(keyword string) (dbSpamPattern, bool) {
for _, sp := range dbSpamPatterns {
if sp.keyword == keyword {
return sp, true
}
}
return dbSpamPattern{}, false
}
// parsePostIDList parses a "1, 2, 3" comma list into a deduplicated slice of
// numeric IDs. Non-numeric tokens are dropped so a malformed finding can never
// inject into the IN clause.
func parsePostIDList(s string) []string {
if s == "" {
return nil
}
seen := map[string]bool{}
var out []string
for _, tok := range strings.Split(s, ",") {
tok = strings.TrimSpace(tok)
if tok == "" || !isAllDigits(tok) || seen[tok] {
continue
}
seen[tok] = true
out = append(out, tok)
}
return out
}
// dbVerifyPostSearchLimit bounds the rows a post re-check reads back; one
// surviving match is enough to keep the finding open.
const dbVerifyPostSearchLimit = 50
// likeContains builds a bound LIKE argument that matches rows containing s
// literally: LIKE wildcards and the escape character inside s are escaped so
// a pattern such as "base64_decode" cannot match "base64Xdecode".
func likeContains(s string) string {
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
return "%" + r.Replace(s) + "%"
}
package checks
import (
"context"
"fmt"
"strings"
)
// Per-finding Re-check for the administrator-account database findings: the
// WordPress rogue-admin and disposable-email-admin findings, and the five
// per-CMS administrator findings (Drupal, Joomla, Magento, OpenCart,
// PrestaShop). Each
// resolves only when the specific flagged account is no longer present (or, for
// the WordPress rogue admin, no longer holds administrator capability). A
// surviving account -- which for the per-CMS Warning findings includes the
// legitimate site administrator -- keeps the finding active for operator review.
// Any query failure or undiscoverable install returns Checked:false.
// findWPAdminVerifyPrefixes re-derives the base table prefixes for WordPress
// installs whose database matches dbName. The users/usermeta tables are
// network-wide in multisite, so this returns only each install's base prefix,
// never secondary blog prefixes.
func findWPAdminVerifyPrefixes(account, dbName, details string) ([]string, bool) {
if !validAccountName.MatchString(account) || dbName == "" {
return nil, false
}
targetPrefix := detailField(details, "Table prefix")
if targetPrefix != "" && !validTablePrefix.MatchString(targetPrefix) {
return nil, false
}
var prefixes []string
seen := map[string]bool{}
// Shared discovery (wpinstalls.go): an administrator finding raised on a
// nested or panel-mapped install must be re-checkable there too.
patterns := wpInstallConfigPaths(wpInstallsForAccount(context.Background(), "db_content", account))
for _, path := range patterns {
creds := parseWPConfig(path)
if creds.dbName != dbName {
continue
}
prefix, ok := resolveTablePrefix(creds)
if !ok {
return nil, false
}
if targetPrefix != "" {
if targetPrefix == prefix {
return []string{prefix}, true
}
continue
}
addPrefix(&prefixes, seen, prefix)
}
if len(prefixes) == 0 {
return nil, false
}
return prefixes, true
}
// wpAdminQueryTables backtick-quotes the users and usermeta table names for the
// given (already validTablePrefix-validated) prefix.
func wpAdminQueryTables(prefix string) (users, usermeta string, ok bool) {
u, err := QuoteIdent(prefix + "users")
if err != nil {
return "", "", false
}
m, err := QuoteIdent(prefix + "usermeta")
if err != nil {
return "", "", false
}
return u, m, true
}
// verifyDBRogueAdmin resolves when the flagged WordPress user no longer holds
// administrator capability (deleted or demoted). The detector's "created in the
// last 7 days" heuristic is intentionally NOT re-applied -- the account is older
// now; what matters for the re-check is whether that specific account is still
// an administrator.
func verifyDBRogueAdmin(message, details string) VerifyResult {
dbName := detailField(details, "Database")
userID := detailField(details, "User ID")
if userID == "" || !isAllDigits(userID) {
return VerifyResult{Checked: false, Detail: "could not parse the user ID from the finding"}
}
prefixes, ok := findWPAdminVerifyPrefixes(dbFindingAccount(message, details), dbName, details)
if !ok {
return dbVerifyNotLocatable("WordPress site")
}
for _, prefix := range prefixes {
users, usermeta, ok := wpAdminQueryTables(prefix)
if !ok {
return VerifyResult{Checked: false, Detail: "could not validate the WordPress tables from the finding"}
}
rows, err := runDBVerifyQueryRoot(dbName, fmt.Sprintf(
"SELECT u.ID FROM %s u JOIN %s m ON u.ID = m.user_id WHERE m.meta_key = ? AND m.meta_value LIKE '%%administrator%%' AND u.ID = ?",
users, usermeta), prefix+"capabilities", userID)
if err != nil {
return dbVerifyQueryError()
}
if len(rows) > 0 {
return VerifyResult{Checked: true, Resolved: false, Detail: "the flagged account is still a WordPress administrator"}
}
}
return VerifyResult{Checked: true, Resolved: true, Detail: "the flagged account is no longer a WordPress administrator"}
}
// verifyDBSuspiciousAdminEmail resolves when no administrator still uses the
// flagged disposable email address.
func verifyDBSuspiciousAdminEmail(message, details string) VerifyResult {
dbName := detailField(details, "Database")
email := strings.ToLower(strings.TrimSpace(detailField(details, "Email")))
if email == "" {
return VerifyResult{Checked: false, Detail: "could not parse the admin email from the finding"}
}
prefixes, ok := findWPAdminVerifyPrefixes(dbFindingAccount(message, details), dbName, details)
if !ok {
return dbVerifyNotLocatable("WordPress site")
}
for _, prefix := range prefixes {
users, usermeta, ok := wpAdminQueryTables(prefix)
if !ok {
return VerifyResult{Checked: false, Detail: "could not validate the WordPress tables from the finding"}
}
rows, err := runDBVerifyQueryRoot(dbName, fmt.Sprintf(
"SELECT u.ID FROM %s u JOIN %s m ON u.ID = m.user_id WHERE m.meta_key = ? AND m.meta_value LIKE '%%administrator%%' AND LOWER(u.user_email) = ?",
users, usermeta), prefix+"capabilities", email)
if err != nil {
return dbVerifyQueryError()
}
if len(rows) > 0 {
return VerifyResult{Checked: true, Resolved: false, Detail: "an administrator still uses the flagged email address"}
}
}
return VerifyResult{Checked: true, Resolved: true, Detail: "no administrator still uses the flagged email address"}
}
// dbAdminRowID extracts the numeric account id from a per-CMS admin finding's
// "Row:" detail line, whose first tab- (or space-) separated field is the id.
func dbAdminRowID(details string) (string, bool) {
row := detailField(details, "Row")
if row == "" {
return "", false
}
first := row
if i := strings.IndexAny(row, "\t "); i >= 0 {
first = row[:i]
}
first = strings.TrimSpace(first)
if first == "" || !isAllDigits(first) {
return "", false
}
return first, true
}
// dbAdminPresenceResult maps a presence query result to a VerifyResult: a query
// error is never resolved; zero rows means the flagged account is gone.
func dbAdminPresenceResult(err error, rows []string) VerifyResult {
if err != nil {
return dbVerifyQueryError()
}
if len(rows) == 0 {
return VerifyResult{Checked: true, Resolved: true, Detail: "the flagged administrator account is no longer present"}
}
return VerifyResult{Checked: true, Resolved: false, Detail: "the flagged administrator account is still present"}
}
// cmsAdminInstall finds the install whose database the administrator finding
// records, so a clean main site never clears an addon site's finding.
func cmsAdminInstall(installs []wpDBCreds, located bool, details string) (wpDBCreds, bool) {
if !located {
return wpDBCreds{}, false
}
return cmsVerifyRecordedInstall(installs, details)
}
func verifyDrupalAdminInjection(message, details string) VerifyResult {
uid, ok := dbAdminRowID(details)
if !ok {
return VerifyResult{Checked: false, Detail: "could not parse the account id from the finding"}
}
installs, located := drupalVerifyInstalls(dbFindingAccount(message, details))
creds, ok := cmsAdminInstall(installs, located, details)
if !ok {
return dbVerifyNotLocatable("Drupal site")
}
rows, err := runDBVerifyQueryAs(creds,
"SELECT u.uid FROM users_field_data u JOIN user__roles r ON u.uid = r.entity_id WHERE r.roles_target_id = ? AND u.default_langcode = 1 AND u.uid = ?",
drupalAdminRoleID, uid)
return dbAdminPresenceResult(err, rows)
}
func verifyJoomlaAdminInjection(message, details string) VerifyResult {
id, ok := dbAdminRowID(details)
if !ok {
return VerifyResult{Checked: false, Detail: "could not parse the account id from the finding"}
}
installs, located := joomlaVerifyInstalls(dbFindingAccount(message, details))
creds, ok := cmsAdminInstall(installs, located, details)
if !ok {
return dbVerifyNotLocatable("Joomla site")
}
users, err := QuoteIdent(creds.tablePrefix + "users")
if err != nil {
return VerifyResult{Checked: false, Detail: "could not validate the Joomla tables from the finding"}
}
mapTable, err := QuoteIdent(creds.tablePrefix + "user_usergroup_map")
if err != nil {
return VerifyResult{Checked: false, Detail: "could not validate the Joomla tables from the finding"}
}
rows, err := runDBVerifyQueryAs(creds, fmt.Sprintf(
"SELECT u.id FROM %s u JOIN %s m ON u.id = m.user_id WHERE m.group_id = ? AND u.id = ?",
users, mapTable), joomlaSuperUserGroupID, id)
return dbAdminPresenceResult(err, rows)
}
func verifyMagentoAdminInjection(message, details string) VerifyResult {
id, ok := dbAdminRowID(details)
if !ok {
return VerifyResult{Checked: false, Detail: "could not parse the account id from the finding"}
}
installs, located := magentoVerifyInstalls(dbFindingAccount(message, details))
creds, ok := cmsAdminInstall(installs, located, details)
if !ok {
return dbVerifyNotLocatable("Magento site")
}
table, err := QuoteIdent(creds.tablePrefix + "admin_user")
if err != nil {
return VerifyResult{Checked: false, Detail: "could not validate the Magento table from the finding"}
}
rows, err := runDBVerifyQueryAs(creds, fmt.Sprintf("SELECT user_id FROM %s WHERE user_id = ?", table), id)
return dbAdminPresenceResult(err, rows)
}
func verifyOpenCartAdminInjection(message, details string) VerifyResult {
id, ok := dbAdminRowID(details)
if !ok {
return VerifyResult{Checked: false, Detail: "could not parse the account id from the finding"}
}
installs, located := openCartVerifyInstalls(dbFindingAccount(message, details))
creds, ok := cmsAdminInstall(installs, located, details)
if !ok {
return dbVerifyNotLocatable("OpenCart site")
}
table, err := QuoteIdent(creds.tablePrefix + "user")
if err != nil {
return VerifyResult{Checked: false, Detail: "could not validate the OpenCart table from the finding"}
}
rows, err := runDBVerifyQueryAs(creds, fmt.Sprintf("SELECT user_id FROM %s WHERE user_id = ?", table), id)
return dbAdminPresenceResult(err, rows)
}
// verifyPrestaShopAdminInjection resolves a new-employee finding when the
// employee is gone, and a SuperAdmin finding when the employee is gone or no
// longer holds the SuperAdmin profile.
func verifyPrestaShopAdminInjection(message, details string) VerifyResult {
id, ok := dbAdminRowID(details)
if !ok {
return VerifyResult{Checked: false, Detail: "could not parse the account id from the finding"}
}
creds, ok := prestashopRecordedInstall(message, details)
if !ok {
return dbVerifyNotLocatable("PrestaShop site")
}
table, err := QuoteIdent(creds.tablePrefix + "employee")
if err != nil {
return VerifyResult{Checked: false, Detail: "could not validate the PrestaShop table from the finding"}
}
query := fmt.Sprintf("SELECT id_employee FROM %s WHERE id_employee = ?", table)
event := detailField(details, "Event")
superAdmin := event == prestashopEventNewSuperAdmin || event == prestashopEventPromoted
if superAdmin {
query += " AND id_profile = " + prestashopSuperAdminProfile
}
rows, err := runDBVerifyQueryAs(creds, query, id)
res := dbAdminPresenceResult(err, rows)
if superAdmin && res.Checked {
res.Detail = "the flagged employee still holds the SuperAdmin profile"
if res.Resolved {
res.Detail = "the flagged employee no longer holds the SuperAdmin profile"
}
}
return res
}
package checks
import (
"fmt"
"strings"
)
// Per-finding Re-check for the non-WordPress CMS database-content findings
// (Drupal, Joomla, Magento, OpenCart, PrestaShop). Each verifier re-discovers
// the account's installs (verify_db_cms_installs.go), re-reads the flagged row
// with each install's own database login, and re-runs the SAME malware
// classifier the detector used. It resolves only when the row is gone or no
// longer classifies as malware; any query failure or uninspectable install
// returns Checked:false so a live injection is never auto-cleared.
// --- Drupal ---------------------------------------------------------------
func verifyDrupalSettingsInjection(message, details string) VerifyResult {
name := detailField(details, "Config name")
if name == "" {
return VerifyResult{Checked: false, Detail: "could not parse the config name from the finding"}
}
installs, ok := drupalVerifyInstalls(dbFindingAccount(message, details))
if !ok {
return dbVerifyNotLocatable("Drupal site")
}
return cmsVerifyEveryInstall(installs, false, func(string) (string, error) {
return "SELECT data FROM config WHERE name = ?", nil
}, name)
}
func verifyDrupalContentInjection(message, details string) VerifyResult {
entityID := detailField(details, "Node entity_id")
if entityID == "" || !isAllDigits(entityID) {
return VerifyResult{Checked: false, Detail: "could not parse the node id from the finding"}
}
installs, ok := drupalVerifyInstalls(dbFindingAccount(message, details))
if !ok {
return dbVerifyNotLocatable("Drupal site")
}
return cmsVerifyEveryInstall(installs, true, func(string) (string, error) {
return "SELECT body_value FROM node_revision__body WHERE entity_id = ?", nil
}, entityID)
}
// --- Joomla ---------------------------------------------------------------
func verifyJoomlaExtensionsInjection(message, details string) VerifyResult {
name := detailField(details, "Extension")
if name == "" {
return VerifyResult{Checked: false, Detail: "could not parse the extension name from the finding"}
}
installs, ok := joomlaVerifyInstalls(dbFindingAccount(message, details))
if !ok {
return dbVerifyNotLocatable("Joomla site")
}
return cmsVerifyEveryInstall(installs, false, func(prefix string) (string, error) {
table, err := QuoteIdent(prefix + "extensions")
return fmt.Sprintf("SELECT params FROM %s WHERE name = ?", table), err
}, name)
}
func verifyJoomlaContentInjection(message, details string) VerifyResult {
id := detailField(details, "Article ID")
if id == "" || !isAllDigits(id) {
return VerifyResult{Checked: false, Detail: "could not parse the article id from the finding"}
}
installs, ok := joomlaVerifyInstalls(dbFindingAccount(message, details))
if !ok {
return dbVerifyNotLocatable("Joomla site")
}
return cmsVerifyEveryInstall(installs, true, func(prefix string) (string, error) {
table, err := QuoteIdent(prefix + "content")
return fmt.Sprintf("SELECT introtext FROM %s WHERE id = ?", table), err
}, id)
}
// --- Magento --------------------------------------------------------------
func verifyMagentoSettingsInjection(message, details string) VerifyResult {
cfgPath := detailField(details, "Config path")
if cfgPath == "" {
return VerifyResult{Checked: false, Detail: "could not parse the config path from the finding"}
}
installs, ok := magentoVerifyInstalls(dbFindingAccount(message, details))
if !ok {
return dbVerifyNotLocatable("Magento site")
}
return cmsVerifyEveryInstall(installs, false, func(prefix string) (string, error) {
table, err := QuoteIdent(prefix + "core_config_data")
return fmt.Sprintf("SELECT value FROM %s WHERE path = ?", table), err
}, cfgPath)
}
// magentoContentTableCols maps a Magento content table to its (idColumn,
// valueColumn). Only the three tables the detector scans are accepted; anything
// else returns ok=false so a malformed finding can never name an arbitrary
// table in the re-query.
func magentoContentTableCols(table string) (idCol, valueCol string, ok bool) {
switch table {
case "catalog_product_entity_text":
return "entity_id", "value", true
case "cms_block":
return "block_id", "content", true
case "cms_page":
return "page_id", "content", true
}
return "", "", false
}
func verifyMagentoContentInjection(message, details string) VerifyResult {
table := detailField(details, "Table")
rowID := detailField(details, "Row id")
idCol, valueCol, ok := magentoContentTableCols(table)
if !ok || rowID == "" || !isAllDigits(rowID) {
return VerifyResult{Checked: false, Detail: "could not parse the affected row from the finding"}
}
installs, ok := magentoVerifyInstalls(dbFindingAccount(message, details))
if !ok {
return dbVerifyNotLocatable("Magento site")
}
return cmsVerifyEveryInstall(installs, true, func(prefix string) (string, error) {
quoted, err := QuoteIdent(prefix + table)
return fmt.Sprintf("SELECT %s FROM %s WHERE %s = ?", valueCol, quoted, idCol), err
}, rowID)
}
// --- OpenCart -------------------------------------------------------------
func verifyOpenCartSettingsInjection(message, details string) VerifyResult {
key := detailField(details, "Setting key")
if key == "" {
return VerifyResult{Checked: false, Detail: "could not parse the setting key from the finding"}
}
installs, ok := openCartVerifyInstalls(dbFindingAccount(message, details))
if !ok {
return dbVerifyNotLocatable("OpenCart site")
}
return cmsVerifyEveryInstall(installs, false, func(prefix string) (string, error) {
table, err := QuoteIdent(prefix + "setting")
return fmt.Sprintf("SELECT value FROM %s WHERE `key` = ?", table), err
}, key)
}
// openCartContentTableCols maps an OpenCart content table to its id column. Both
// description tables share the "description" value column. Unknown tables are
// rejected.
func openCartContentTableCols(table string) (idCol string, ok bool) {
switch table {
case "product_description":
return "product_id", true
case "information_description":
return "information_id", true
}
return "", false
}
func verifyOpenCartContentInjection(message, details string) VerifyResult {
table := detailField(details, "Table")
rowID := detailField(details, "Row id")
idCol, ok := openCartContentTableCols(table)
if !ok || rowID == "" || !isAllDigits(rowID) {
return VerifyResult{Checked: false, Detail: "could not parse the affected row from the finding"}
}
installs, ok := openCartVerifyInstalls(dbFindingAccount(message, details))
if !ok {
return dbVerifyNotLocatable("OpenCart site")
}
return cmsVerifyEveryInstall(installs, true, func(prefix string) (string, error) {
quoted, err := QuoteIdent(prefix + table)
return fmt.Sprintf("SELECT description FROM %s WHERE %s = ? AND language_id = 1", quoted, idCol), err
}, rowID)
}
// --- PrestaShop -------------------------------------------------------------
// prestashopRecordedInstall finds the shop whose database the finding records.
// An addon shop or a changed configuration can name a different database with
// the same row ids; a finding is never cleared against another shop.
func prestashopRecordedInstall(message, details string) (wpDBCreds, bool) {
installs, ok := prestashopVerifyInstalls(dbFindingAccount(message, details))
if !ok {
return wpDBCreds{}, false
}
return cmsVerifyRecordedInstall(installs, details)
}
func verifyPrestaShopSettingsInjection(message, details string) VerifyResult {
id := detailField(details, "Setting id")
if !isAllDigits(id) {
return VerifyResult{Checked: false, Detail: "could not parse the setting id from the finding"}
}
creds, ok := prestashopRecordedInstall(message, details)
if !ok {
return dbVerifyNotLocatable("PrestaShop site")
}
return cmsVerifyEveryInstall([]wpDBCreds{creds}, false, func(prefix string) (string, error) {
base, err := QuoteIdent(prefix + "configuration")
if err != nil {
return "", err
}
lang, err := QuoteIdent(prefix + "configuration_lang")
return fmt.Sprintf("SELECT value FROM %s WHERE id_configuration = ? UNION ALL SELECT value FROM %s WHERE id_configuration = ?",
base, lang), err
}, id, id)
}
// verifyPrestaShopContentInjection re-reads every language, shop and column
// of the flagged row: the finding stands for all of them.
func verifyPrestaShopContentInjection(message, details string) VerifyResult {
tableName := detailField(details, "Table")
id := detailField(details, "Row id")
var spec *prestashopContentTable
for i := range prestashopContentTables {
if prestashopContentTables[i].table == tableName {
spec = &prestashopContentTables[i]
}
}
if spec == nil || !isAllDigits(id) {
return VerifyResult{Checked: false, Detail: "could not parse the table and row id from the finding"}
}
creds, ok := prestashopRecordedInstall(message, details)
if !ok {
return dbVerifyNotLocatable("PrestaShop site")
}
var args []any
for range spec.cols {
args = append(args, id)
}
return cmsVerifyEveryInstall([]wpDBCreds{creds}, true, func(prefix string) (string, error) {
table, err := QuoteIdent(prefix + spec.table)
var selects []string
for _, col := range spec.cols {
selects = append(selects, fmt.Sprintf("SELECT %s FROM %s WHERE %s = ?", col, table, spec.idCol))
}
return strings.Join(selects, " UNION ALL "), err
}, args...)
}
package checks
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/mysqlclient"
)
// Re-checks for the non-WordPress CMS findings look at every install of the
// account in the layouts the scanner discovers: the account's public_html and
// each folder directly under its home. Each install is queried with its own
// database login and endpoint, the ones the scanner used. Root on the local
// server could read a same-named schema that is not the install's database.
// cmsInstallLoader reads one discovered config path. isInstall is false when
// the path is not that CMS; an error means the install cannot be inspected.
type cmsInstallLoader func(ctx context.Context, path string) (creds wpDBCreds, isInstall bool, err error)
// cmsVerifyInstalls returns every install of the account, one per database.
// ok is false when the account has none or any of them cannot be read: the
// re-check then cannot vouch for the account and stays unchecked.
func cmsVerifyInstalls(account, rel string, load cmsInstallLoader) ([]wpDBCreds, bool) {
if !validAccountName.MatchString(account) {
return nil, false
}
home := accountHomeDir(account)
// Glob discards filesystem errors, so an inaccessible addon could vanish
// from the inventory and let a clean site clear its finding.
entries, err := osFS.ReadDir(home)
if err != nil {
return nil, false
}
var paths []string
for _, entry := range entries {
root := filepath.Join(home, entry.Name())
isDir := entry.IsDir()
if entry.Type()&os.ModeSymlink != 0 {
info, err := osFS.Stat(root)
// A link to nothing holds no install; any other error might hide one.
if errors.Is(err, os.ErrNotExist) {
continue
}
if err != nil {
return nil, false
}
isDir = info.IsDir()
}
if !isDir {
continue
}
path := filepath.Join(root, rel)
_, err := osFS.Lstat(path)
if errors.Is(err, os.ErrNotExist) {
continue
}
if err != nil {
return nil, false
}
paths = append(paths, path)
}
var installs []wpDBCreds
seen := map[string]bool{}
for _, path := range paths {
creds, isInstall, err := load(context.Background(), path)
if err != nil {
return nil, false
}
if !isInstall {
continue
}
if creds.dbName == "" || creds.dbUser == "" || (creds.tablePrefix != "" && !validTablePrefix.MatchString(creds.tablePrefix)) {
return nil, false
}
// A linked folder (www -> public_html) names the same database twice.
identity := creds.dbHost + "\x00" + creds.dbName + "\x00" + creds.tablePrefix
if !seen[identity] {
seen[identity] = true
installs = append(installs, creds)
}
}
return installs, len(installs) > 0
}
// cmsVerifyRecordedInstall returns the install whose database the finding
// recorded. Administrator findings and every PrestaShop finding carry that
// block, so they are re-checked there and nowhere else.
func cmsVerifyRecordedInstall(installs []wpDBCreds, details string) (wpDBCreds, bool) {
for _, creds := range installs {
matches := true
for key, want := range map[string]string{
"Database": creds.dbName, "Table prefix": creds.tablePrefix, "Database host": creds.dbHost,
} {
if got, present := detailFieldPresent(details, key); !present || got != want {
matches = false
}
}
if matches {
return creds, true
}
}
return wpDBCreds{}, false
}
// cmsVerifyEveryInstall re-reads the flagged row in each install and resolves
// only when none still classifies. Settings and content findings of Joomla,
// Drupal, Magento and OpenCart do not record their install, so a clean main
// site must not clear a finding an addon site may have raised. query builds
// the statement for one install's table prefix.
func cmsVerifyEveryInstall(installs []wpDBCreds, inPostContext bool, query func(prefix string) (string, error), args ...any) VerifyResult {
injected := false
for _, creds := range installs {
q, err := query(creds.tablePrefix)
if err != nil {
return VerifyResult{Checked: false, Detail: "could not validate the tables from the finding"}
}
rows, err := runDBVerifyQueryAs(creds, q, args...)
if err != nil {
return dbVerifyQueryError()
}
for _, row := range rows {
if _, _, ok := classifyMalwareRow(mysqlclient.BatchUnescape(row), inPostContext); ok {
injected = true
break
}
}
}
if injected {
return VerifyResult{Checked: true, Resolved: false, Detail: "the flagged row still contains injected content"}
}
return VerifyResult{Checked: true, Resolved: true, Detail: "the flagged row no longer contains injected content"}
}
func drupalVerifyInstall(ctx context.Context, path string) (wpDBCreds, bool, error) {
// public_html is three folders up from sites/default/settings.php.
matched, err := looksLikeDrupal8Plus(filepath.Dir(filepath.Dir(filepath.Dir(path))))
if err != nil || !matched {
return wpDBCreds{}, false, err
}
creds, err := parseDrupalSettings(ctx, path)
if err != nil {
return wpDBCreds{}, true, err
}
return creds.asWPDBCreds(), true, nil
}
func joomlaVerifyInstall(ctx context.Context, path string) (wpDBCreds, bool, error) {
matched, err := looksLikeJoomlaConfig(ctx, path)
if err != nil || !matched {
return wpDBCreds{}, false, err
}
creds, err := parseJConfig(ctx, path)
if err != nil {
return wpDBCreds{}, true, err
}
w := creds.asWPDBCreds()
if w.tablePrefix == "" {
w.tablePrefix = "jos_"
}
return w, true, nil
}
// magentoVerifyInstall reads one app/etc folder: env.php (Magento 2) first,
// local.xml (Magento 1) only when env.php does not exist. An unsafe or
// unreadable env.php must not select an older, unrelated database.
func magentoVerifyInstall(ctx context.Context, etcDir string) (wpDBCreds, bool, error) {
creds, err := parseMagentoM2(ctx, filepath.Join(etcDir, "env.php"))
if errors.Is(err, os.ErrNotExist) {
creds, err = parseMagentoM1(ctx, filepath.Join(etcDir, "local.xml"))
if errors.Is(err, os.ErrNotExist) {
return wpDBCreds{}, false, nil
}
}
if err != nil {
return wpDBCreds{}, true, err
}
return creds.asWPDBCreds(), true, nil
}
func openCartVerifyInstall(ctx context.Context, path string) (wpDBCreds, bool, error) {
matched, err := looksLikeOpenCart(ctx, path)
if err != nil || !matched {
return wpDBCreds{}, false, err
}
creds, err := parseOpenCartConfig(ctx, path)
if err != nil {
return wpDBCreds{}, true, err
}
w := creds.asWPDBCreds()
if w.tablePrefix == "" {
w.tablePrefix = "oc_"
}
return w, true, nil
}
// prestashopVerifyInstall reads one config/defines.inc.php marker. Staging and
// backup copies are skipped, as the scanner skips them.
func prestashopVerifyInstall(ctx context.Context, marker string) (wpDBCreds, bool, error) {
root := prestashopRoot(marker)
if wpSkipDirNames[strings.ToLower(filepath.Base(root))] {
return wpDBCreds{}, false, nil
}
creds, isShop, err := readPrestaShopCreds(ctx, root)
if err != nil || !isShop {
return wpDBCreds{}, isShop, err
}
return creds.asWPDBCreds(), true, nil
}
func drupalVerifyInstalls(account string) ([]wpDBCreds, bool) {
return cmsVerifyInstalls(account, filepath.Join("sites", "default", "settings.php"), drupalVerifyInstall)
}
func joomlaVerifyInstalls(account string) ([]wpDBCreds, bool) {
return cmsVerifyInstalls(account, "configuration.php", joomlaVerifyInstall)
}
func magentoVerifyInstalls(account string) ([]wpDBCreds, bool) {
return cmsVerifyInstalls(account, filepath.Join("app", "etc"), magentoVerifyInstall)
}
func openCartVerifyInstalls(account string) ([]wpDBCreds, bool) {
return cmsVerifyInstalls(account, "config.php", openCartVerifyInstall)
}
func prestashopVerifyInstalls(account string) ([]wpDBCreds, bool) {
return cmsVerifyInstalls(account, filepath.Join("config", "defines.inc.php"), prestashopVerifyInstall)
}
package checks
import (
"context"
"fmt"
"strings"
"github.com/pidginhost/csm/internal/mysqlclient"
)
// Per-finding Re-check for the database persistence-mechanism findings
// (triggers, events, procedures, functions) and the backdoor magic-token user
// finding. The object re-checks re-read the object's CURRENT body from
// INFORMATION_SCHEMA (not the truncated copy stored in the finding details) and
// re-run the same malware classifier the detector used, so a partially-cleaned
// object is never mistaken for a fixed one. All queries run as root and any
// failure returns Checked:false.
// runDBVerifyQueryRootGlobal runs a root query with no pinned default schema,
// for INFORMATION_SCHEMA lookups whose WHERE clause already scopes the schema.
// If the account database was dropped entirely, the object is gone with it and
// the query simply returns zero rows -- the correct "resolved" signal.
func runDBVerifyQueryRootGlobal(query string, args ...any) ([]string, error) {
ctx, cancel := context.WithTimeout(context.Background(), dbVerifyTimeout)
defer cancel()
return mysqlclient.RootQuery(ctx, query, args...)
}
// verifyDBObject re-checks an unexpected/malicious trigger, event, procedure, or
// function. For the unexpected tier any surviving object keeps the finding
// active. For the malicious tier the finding clears when the object is gone or
// its current body no longer matches a malware pattern.
func verifyDBObject(details string, malicious bool) VerifyResult {
schema := detailField(details, "Schema")
kind := detailField(details, "Kind")
name := detailField(details, "Name")
if schema == "" || name == "" || !IsDBObjectKind(kind) {
return VerifyResult{Checked: false, Detail: "could not parse the database object from the finding"}
}
body, present, err := fetchDBObjectBody(schema, dbObjectKind(kind), name)
if err != nil {
return dbVerifyQueryError()
}
if !present {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("%s %q no longer exists", kind, name)}
}
if !malicious {
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("%s %q is still present", kind, name)}
}
if bodyHasMalwarePattern(body) {
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("%s %q still matches a malware pattern", kind, name)}
}
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("%s %q no longer matches a malware pattern", kind, name)}
}
// fetchDBObjectBody returns the current definition body of one database object
// and whether it still exists. The object name and schema are passed as bound
// parameters so a malformed finding cannot inject into the lookup.
func fetchDBObjectBody(schema string, kind dbObjectKind, name string) (body string, present bool, err error) {
var query string
var args []any
switch kind {
case dbObjectTrigger:
query = "SELECT ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE TRIGGER_SCHEMA = ? AND TRIGGER_NAME = ?"
args = []any{schema, name}
case dbObjectEvent:
query = "SELECT EVENT_DEFINITION FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_SCHEMA = ? AND EVENT_NAME = ?"
args = []any{schema, name}
case dbObjectProcedure, dbObjectFunction:
routineType := "PROCEDURE"
if kind == dbObjectFunction {
routineType = "FUNCTION"
}
query = "SELECT ROUTINE_DEFINITION FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA = ? AND ROUTINE_NAME = ? AND ROUTINE_TYPE = ?"
args = []any{schema, name, routineType}
default:
return "", false, fmt.Errorf("unknown object kind")
}
rows, err := runDBVerifyQueryRootGlobal(query, args...)
if err != nil {
return "", false, err
}
if len(rows) == 0 {
return "", false, nil
}
return rows[0], true, nil
}
// verifyDBMagicTokenUser re-queries the WordPress users table for any account
// whose flagged user row still matches the trigger's backdoor activation
// pattern. The token is validated as the high-entropy [A-Za-z0-9_-]{10,32}
// shape the detector emits before it is used in the LIKE.
func verifyDBMagicTokenUser(message, details string) VerifyResult {
token := detailField(details, "Token")
if token == "" || !validMagicToken(token) {
return VerifyResult{Checked: false, Detail: "could not parse a valid activation token from the finding"}
}
schema := strings.TrimSpace(detailField(details, "Schema"))
if schema == "" {
return VerifyResult{Checked: false, Detail: "could not parse the database schema from the finding"}
}
userID, ok := dbMagicTokenUserID(details)
if !ok {
return VerifyResult{Checked: false, Detail: "could not parse the WordPress user ID from the finding"}
}
prefix, ok := dbMagicTokenUserTablePrefix(message, details)
if !ok {
return VerifyResult{Checked: false, Detail: "could not parse the WordPress table prefix from the finding"}
}
prefix, ok = locateDBMagicTokenUserPrefix(dbFindingAccount(message, details), schema, prefix)
if !ok {
return dbVerifyNotLocatable("WordPress site")
}
table, err := QuoteIdent(prefix + "users")
if err != nil {
return VerifyResult{Checked: false, Detail: "could not validate the WordPress users table from the finding"}
}
rows, err := runDBVerifyQueryRoot(schema,
fmt.Sprintf("SELECT ID FROM %s WHERE ID = ? AND display_name LIKE ?", table),
userID, "%"+token+"%")
if err != nil {
return dbVerifyQueryError()
}
if len(rows) == 0 {
return VerifyResult{Checked: true, Resolved: true, Detail: "the flagged user no longer carries the backdoor activation token"}
}
return VerifyResult{Checked: true, Resolved: false, Detail: "the flagged user still carries the backdoor activation token"}
}
func dbMagicTokenUserID(details string) (string, bool) {
userID := strings.TrimSpace(detailField(details, "User ID"))
if userID == "" || !isAllDigits(userID) {
return "", false
}
return userID, true
}
func dbMagicTokenUserTablePrefix(message, details string) (string, bool) {
if prefix, ok := validDBMagicTokenTablePrefix(detailField(details, "Table prefix")); ok {
return prefix, true
}
const marker = " in "
idx := strings.LastIndex(message, marker)
if idx < 0 {
return "", false
}
ref := strings.TrimSpace(message[idx+len(marker):])
if !strings.HasSuffix(ref, "users") {
return "", false
}
dot := strings.LastIndex(ref, ".")
if dot < 0 {
return "", false
}
return validDBMagicTokenTablePrefix(strings.TrimSuffix(ref[dot+1:], "users"))
}
func validDBMagicTokenTablePrefix(prefix string) (string, bool) {
prefix = strings.TrimSpace(prefix)
if prefix == "" || !validTablePrefix.MatchString(prefix) {
return "", false
}
return prefix, true
}
func locateDBMagicTokenUserPrefix(account, schema, prefix string) (string, bool) {
prefixes, ok := findWPVerifyPrefixes(account, schema, "Table prefix: "+prefix)
if !ok || len(prefixes) != 1 {
return "", false
}
return prefixes[0], true
}
package checks
import (
"context"
"fmt"
"regexp"
"strings"
"time"
)
// pkgVerifyTimeout bounds the synchronous package-manager re-check a Re-check
// click runs.
var pkgVerifyTimeout = 30 * time.Second
// pkgNameRe is the safe character set for a package name passed as an argument
// to rpm/dpkg/debsums. Findings carry CSM-generated package names, but the
// re-check still validates before exec so a malformed name can never inject.
var pkgNameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+:~-]*$`)
// parsePkgIntegrityFinding extracts the file and package from an
// rpm_integrity / dpkg_integrity message:
// "Modified system binary or library: <file> (package: <pkg>)".
func parsePkgIntegrityFinding(message string) (file, pkg string, ok bool) {
rest, ok := strings.CutPrefix(message, "Modified system binary or library: ")
if !ok {
return "", "", false
}
const sep = " (package: "
i := strings.LastIndex(rest, sep)
if i < 0 || !strings.HasSuffix(rest, ")") {
return "", "", false
}
file = strings.TrimSpace(rest[:i])
pkg = strings.TrimSpace(rest[i+len(sep) : len(rest)-1])
if file == "" || pkg == "" {
return "", "", false
}
return file, pkg, true
}
type packageVerifyOutputState struct {
targetFlagged bool
sawReport bool
sawUnknown bool
}
// verifyManifestLineFlagsFile reports whether an rpm -V / dpkg --verify output
// line marks the target file as size/checksum-modified (mirrors the detector:
// skip config/doc and require S or 5). The existing finding already classified
// the file; changing its mode or removing it must not clear a reported mismatch.
func verifyManifestLineFlagsFile(line, file string) (targetFlagged, recognized bool) {
flags, got, ok := parseManifestVerifyLine(line)
if !ok {
return false, false
}
if !manifestVerifyLineHasPackagePath(line, got) {
return false, false
}
if strings.Contains(line, " c ") || strings.Contains(line, " d ") {
return false, true
}
if !strings.Contains(flags, "S") && !strings.Contains(flags, "5") {
return false, true
}
return got == file, true
}
func manifestVerifyLineHasPackagePath(line, got string) bool {
if strings.HasPrefix(got, "/") {
return true
}
fields := strings.Fields(strings.TrimSpace(line[9:]))
if len(fields) != 2 || (fields[0] != "c" && fields[0] != "d") {
return false
}
return strings.HasPrefix(fields[1], "/")
}
func parseManifestVerifyLine(line string) (flags, file string, ok bool) {
if len(line) < 10 {
return "", "", false
}
flags = line[:9]
for _, ch := range flags {
if !strings.ContainsRune(".?SM5DLUGTP", ch) {
return "", "", false
}
}
file = strings.TrimSpace(line[9:])
if file == "" {
return "", "", false
}
return flags, file, true
}
func manifestOutputState(out []byte, file string) packageVerifyOutputState {
var state packageVerifyOutputState
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
if strings.TrimSpace(line) == "" {
continue
}
targetFlagged, recognized := verifyManifestLineFlagsFile(line, file)
if !recognized {
state.sawUnknown = true
continue
}
state.sawReport = true
if targetFlagged {
state.targetFlagged = true
}
}
return state
}
func debsumsOutputState(out []byte, file string) packageVerifyOutputState {
var state packageVerifyOutputState
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
got := strings.TrimSpace(line)
if got == "" {
continue
}
if !strings.HasPrefix(got, "/") {
state.sawUnknown = true
continue
}
state.sawReport = true
if got == file {
state.targetFlagged = true
}
}
return state
}
func resolvePkgVerifyOutput(state packageVerifyOutputState, file, verifier string) VerifyResult {
if state.targetFlagged {
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("%s is still reported as modified", file)}
}
if state.sawReport && !state.sawUnknown {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("%s is no longer reported as modified", file)}
}
return VerifyResult{Checked: false, Detail: fmt.Sprintf("could not parse %s output (try again, or run an account scan)", verifier)}
}
func pkgVerifyTimedOut(ctx context.Context, verifier string) (VerifyResult, bool) {
if ctx.Err() == nil {
return VerifyResult{}, false
}
return VerifyResult{Checked: false, Detail: fmt.Sprintf("%s timed out (try again, or run an account scan)", verifier)}, true
}
// verifyRPMIntegrity re-runs `rpm -V <pkg>` and resolves the finding when the
// flagged file is no longer reported as modified (or the whole package verifies
// clean). Read-only, bounded; any command failure returns Checked:false so a
// real tampered binary is never auto-cleared.
func verifyRPMIntegrity(message string) VerifyResult {
file, pkg, ok := parsePkgIntegrityFinding(message)
if !ok {
return VerifyResult{Checked: false, Detail: "could not parse the package finding"}
}
if !pkgNameRe.MatchString(pkg) {
return VerifyResult{Checked: false, Detail: "package name in finding is not auto-verifiable"}
}
ctx, cancel := context.WithTimeout(context.Background(), pkgVerifyTimeout)
defer cancel()
// rpm -V exits non-zero when files are modified; treat output, not exit
// code, as the signal.
out, err := cmdExec.RunContext(ctx, "rpm", "-V", pkg)
if res, timedOut := pkgVerifyTimedOut(ctx, "rpm -V"); timedOut {
return res
}
if strings.TrimSpace(string(out)) == "" {
if err == nil {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("package %s verifies clean", pkg)}
}
return VerifyResult{Checked: false, Detail: "could not run rpm -V (try again, or run an account scan)"}
}
return resolvePkgVerifyOutput(manifestOutputState(out, file), file, "rpm -V")
}
// verifyDpkgIntegrity re-runs debsums (preferred) or `dpkg --verify` for the
// package and resolves the finding when the flagged file is no longer reported
// as modified. Same safety contract as verifyRPMIntegrity.
func verifyDpkgIntegrity(message string) VerifyResult {
file, pkg, ok := parsePkgIntegrityFinding(message)
if !ok {
return VerifyResult{Checked: false, Detail: "could not parse the package finding"}
}
if !pkgNameRe.MatchString(pkg) {
return VerifyResult{Checked: false, Detail: "package name in finding is not auto-verifiable"}
}
ctx, cancel := context.WithTimeout(context.Background(), pkgVerifyTimeout)
defer cancel()
if _, err := cmdExec.LookPath("debsums"); err == nil {
// debsums -c prints one modified file path per line (exit 2 on mismatch).
out, err := cmdExec.RunContext(ctx, "debsums", "-c", pkg)
if res, timedOut := pkgVerifyTimedOut(ctx, "debsums"); timedOut {
return res
}
if strings.TrimSpace(string(out)) == "" {
if err == nil {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("package %s verifies clean", pkg)}
}
return VerifyResult{Checked: false, Detail: "could not run debsums (try again, or run an account scan)"}
}
return resolvePkgVerifyOutput(debsumsOutputState(out, file), file, "debsums")
}
out, err := cmdExec.RunContext(ctx, "dpkg", "--verify", pkg)
if res, timedOut := pkgVerifyTimedOut(ctx, "dpkg --verify"); timedOut {
return res
}
if strings.TrimSpace(string(out)) == "" {
if err == nil {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("package %s verifies clean", pkg)}
}
return VerifyResult{Checked: false, Detail: "could not run dpkg --verify (try again, or run an account scan)"}
}
return resolvePkgVerifyOutput(manifestOutputState(out, file), file, "dpkg --verify")
}
package checks
import (
"fmt"
"os"
"strings"
)
// allowedUID0 are the system accounts that legitimately carry UID 0; the
// detector and the re-check share this list so they agree on what counts as
// "unauthorized".
var allowedUID0 = map[string]bool{
"root": true, "sync": true, "shutdown": true,
"halt": true, "operator": true,
}
// classifyUID0Line parses one /etc/passwd line and reports the account name and
// whether it is an unauthorized UID 0 account.
func classifyUID0Line(line string) (user string, unauthorized bool) {
fields := strings.Split(line, ":")
if len(fields) < 4 {
return "", false
}
user = fields[0]
return user, fields[2] == "0" && !allowedUID0[user]
}
// verifyUID0Account re-reads /etc/passwd and resolves the finding when the
// flagged account is gone, no longer UID 0, or now an allowed system account.
// Read-only; an unreadable /etc/passwd returns Checked:false rather than a
// false clear.
func verifyUID0Account(message string) VerifyResult {
rest, ok := strings.CutPrefix(message, "Unauthorized UID 0 account: ")
if !ok {
return VerifyResult{Checked: false, Detail: "could not determine the account from the finding"}
}
user := rest
if user == "" {
return VerifyResult{Checked: false, Detail: "could not determine the account from the finding"}
}
if strings.ContainsAny(user, ":\r\n") {
return VerifyResult{Checked: false, Detail: "account name in finding is not auto-verifiable"}
}
data, err := osFS.ReadFile("/etc/passwd")
if err != nil {
return VerifyResult{Checked: false, Detail: fmt.Sprintf("cannot read /etc/passwd: %v", err)}
}
found := false
for _, line := range strings.Split(string(data), "\n") {
u, unauthorized := classifyUID0Line(line)
if u != user {
continue
}
found = true
if unauthorized {
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("%s is still an unauthorized UID 0 account", user)}
}
}
if found {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("%s is no longer an unauthorized UID 0 account", user)}
}
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("account %s no longer exists", user)}
}
// verifySuidCleared re-stats a flagged SUID binary and resolves the finding
// when the file is gone or no longer carries the setuid bit. Bounded to the
// same roots scanForSUID covers (/home, /tmp, /dev/shm, /var/tmp).
func verifySuidCleared(path string) VerifyResult {
if path == "" {
return VerifyResult{Checked: false, Detail: "could not extract file path from finding"}
}
clean, info, exists, err := readOnlyFixPath(path, effectiveFixRoots(fixQuarantineAllowedRoots, quarantineExtraRoots...))
if err != nil {
return VerifyResult{Checked: false, Detail: err.Error()}
}
if !exists {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("SUID binary no longer present: %s", clean)}
}
if !info.Mode().IsRegular() {
return VerifyResult{Checked: false, Detail: "path is not a regular file; not auto-verifiable"}
}
if info.Mode()&os.ModeSetuid != 0 {
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("file is still setuid (mode %s)", info.Mode())}
}
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("file is no longer setuid (mode %s)", info.Mode())}
}
package checks
import (
"context"
"fmt"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/contenttype"
"github.com/pidginhost/csm/internal/store"
)
// wpVerifyAllowedRoots bounds where a WordPress re-check may run wp-cli. It is a
// var so tests can redirect under t.TempDir(); nil means the platform's
// account roots.
var wpVerifyAllowedRoots []string
// wpVerifyTimeout bounds the synchronous wp-cli re-scan a Re-check click runs.
var wpVerifyTimeout = 30 * time.Second
// findingDetailPath extracts the "Path: <dir>" value emitted in a finding's
// Details (outdated_plugins and the WordPress checks record the install path
// there). Returns "" when no such line is present.
func findingDetailPath(details string) string {
for _, line := range strings.Split(details, "\n") {
line = strings.TrimSpace(line)
if rest, ok := strings.CutPrefix(line, "Path:"); ok {
return strings.TrimSpace(rest)
}
}
return ""
}
func wpChecksumLineHasExtraneousCoreFile(line string) bool {
return strings.Contains(line, "should not exist") && !strings.Contains(line, "error_log")
}
// The two per-file shapes wp-cli has used. The closing summary line
// ("WordPress installation doesn't verify against checksums.") matches
// neither: the current shape needs the colon and the legacy one the singular
// "checksum." directly after the file name.
const (
wpChecksumMismatchCurrent = "doesn't verify against checksum: "
wpChecksumMismatchLegacy = " doesn't verify against checksum."
)
// wpChecksumModifiedCoreFile returns the install-relative path of a core file
// that wp-cli reports as changed, or "" when the line is not such a report.
// Localised packages legitimately ship their own root readme and license,
// which carry no code, so those two are not reported.
func wpChecksumModifiedCoreFile(line string) string {
rel := wpChecksumModifiedFilePath(line)
if rel == "readme.html" || rel == "license.txt" {
return ""
}
return rel
}
// Keep recognized but intentionally unreported mismatches distinct from command
// failures when accounting for a completed installation scan.
func wpChecksumModifiedFilePath(line string) string {
var rel string
if idx := strings.Index(line, wpChecksumMismatchCurrent); idx >= 0 {
rel = strings.TrimSpace(line[idx+len(wpChecksumMismatchCurrent):])
} else if idx := strings.Index(line, wpChecksumMismatchLegacy); idx >= 0 {
if fields := strings.Fields(line[:idx]); len(fields) > 0 {
rel = fields[len(fields)-1]
}
}
return rel
}
// wpCoreModifiedSeverity grades a modified core file by what an attacker could
// do with it.
//
// Critical is what auto-response acts on, so it is reserved for files that can
// carry executable content to a visitor: anything a PHP handler runs, and the
// scripts and templates served into the browser. Everything else -- stylesheets,
// images, translations, fonts -- still gets a finding, but a mismatch there is
// far more often an asset optimiser or an install whose version.php no longer
// names the release its files came from than it is an appended backdoor.
func wpCoreModifiedSeverity(path, rel string) alert.Severity {
if contenttype.IsExecutablePHPName(strings.ToLower(rel)) {
return alert.Critical
}
switch strings.ToLower(filepath.Ext(rel)) {
case ".js", ".mjs", ".html", ".htm", ".htaccess":
return alert.Critical
case ".svg", ".xml", ".xhtml":
// Markup, not an image format: SVG carries <script> and event
// handlers and the browser runs them. Judge it by what it holds,
// because most core SVG mismatches are an optimiser's whitespace.
if wpCoreMarkupIsActive(path) {
return alert.Critical
}
}
return alert.High
}
// wpCoreMarkupActiveMarkers are the element-level constructs that make markup
// executable. Event attributes are matched separately: SVG defines dozens of
// them, so any list of names would really be a list of the ones an attacker
// has to avoid.
var wpCoreMarkupActiveMarkers = []string{
"<script", "javascript:", "<foreignobject", "<!entity", "<handler",
"<animate", "<set ", "<use ", "data:text/html",
}
// wpCoreMarkupEventAttr matches any on<name>= handler attribute.
var wpCoreMarkupEventAttr = regexp.MustCompile(`(?i)\bon[a-z]+\s*=`)
// wpCoreMarkupPeekBytes bounds the read. A shipped core asset is far smaller;
// anything larger is judged active without reading further, because the part
// that was not read is exactly where content would be hidden.
const wpCoreMarkupPeekBytes = 256 << 10
// wpCoreMarkupIsActive reports whether a markup file carries anything the
// browser would execute.
//
// Every uncertain answer is "active": an unreadable file, one larger than the
// peek, a path raced to something that is not a regular file. None of those is
// evidence of innocence, and the point of the grade is that a core asset which
// cannot be shown inert keeps the higher severity.
func wpCoreMarkupIsActive(path string) bool {
if path == "" {
return true
}
info, err := osFS.Lstat(path)
if err != nil || !info.Mode().IsRegular() || info.Size() > wpCoreMarkupPeekBytes {
return true
}
// The path is in a tenant-writable tree, so it can be raced to a FIFO
// between the report and this read. Use the non-blocking, O_NOFOLLOW,
// regular-file-verified reader rather than a plain open.
reader, ok := osFS.(phpRegularFilePrefixReader)
if !ok {
return true
}
body, err := reader.ReadRegularFilePrefix(path, info, wpCoreMarkupPeekBytes)
if err != nil {
return true
}
lower := strings.ToLower(string(body))
for _, marker := range wpCoreMarkupActiveMarkers {
if strings.Contains(lower, marker) {
return true
}
}
return wpCoreMarkupEventAttr.MatchString(lower)
}
// wpCoreFilePathWithin joins a wp-cli reported relative path onto the install
// only when it stays inside it; wp-cli output is not a path oracle for the
// remediation and Re-check code that consumes FilePath.
func wpCoreFilePathWithin(wpPath, rel string) string {
clean := filepath.Clean(rel)
if filepath.IsAbs(clean) || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") {
return ""
}
return filepath.Join(wpPath, clean)
}
// verifyOutdatedPlugins re-inventories a single WordPress site with wp-cli (run
// as the site owner) and resolves the finding when no active plugin still has
// an available update. It is heavier than the file re-checks but read-only and
// bounded by wpVerifyTimeout. Any failure to re-scan returns Checked:false so a
// finding is never falsely cleared on a transient wp-cli error.
func verifyOutdatedPlugins(details string) VerifyResult {
wpPath := findingDetailPath(details)
if wpPath == "" {
return VerifyResult{Checked: false, Detail: "could not determine the WordPress path from the finding"}
}
clean, _, exists, err := readOnlyFixPath(wpPath, effectiveFixRoots(wpVerifyAllowedRoots))
if err != nil {
return VerifyResult{Checked: false, Detail: err.Error()}
}
if !exists {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("WordPress install no longer exists: %s", clean)}
}
wpConfig, info, exists, err := readOnlyFixPath(filepath.Join(clean, "wp-config.php"), effectiveFixRoots(wpVerifyAllowedRoots))
if err != nil {
return VerifyResult{Checked: false, Detail: err.Error()}
}
if !exists {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("WordPress install no longer present: %s", clean)}
}
if !info.Mode().IsRegular() {
return VerifyResult{Checked: false, Detail: "wp-config.php path is not a regular file; not auto-verifiable"}
}
ctx, cancel := context.WithTimeout(context.Background(), wpVerifyTimeout)
defer cancel()
site, err := inventoryWPSiteForVerify(ctx, wpConfig)
if err != nil {
return VerifyResult{Checked: false, Detail: fmt.Sprintf("could not re-scan plugins (try again, or run an account scan): %v", err)}
}
if n := countOutdatedActivePlugins(site, store.Global()); n > 0 {
return VerifyResult{Checked: true, Resolved: false, Detail: fmt.Sprintf("%d active plugin(s) still outdated", n)}
}
return VerifyResult{Checked: true, Resolved: true, Detail: "no active plugins are outdated anymore"}
}
// verifyWPCoreIntegrity re-runs `wp core verify-checksums` for one install and
// resolves the finding when no extraneous core file ("should not exist")
// remains -- mirroring CheckWPCore, which only flags those lines. It is
// read-only and bounded by wpVerifyTimeout. To avoid ever clearing a real
// compromise it resolves only when verification is clean, every remaining
// difference is line endings, or the install is gone; any wp-cli error
// (including modified files with no remaining extra-file line) returns
// Checked:false.
func verifyWPCoreIntegrity(details string) VerifyResult {
wpPath := findingDetailPath(details)
if wpPath == "" {
return VerifyResult{Checked: false, Detail: "could not determine the WordPress path from the finding"}
}
clean, _, exists, err := readOnlyFixPath(wpPath, effectiveFixRoots(wpVerifyAllowedRoots))
if err != nil {
return VerifyResult{Checked: false, Detail: err.Error()}
}
if !exists {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("WordPress install no longer exists: %s", clean)}
}
_, info, exists, err := readOnlyFixPath(filepath.Join(clean, "wp-config.php"), effectiveFixRoots(wpVerifyAllowedRoots))
if err != nil {
return VerifyResult{Checked: false, Detail: err.Error()}
}
if !exists {
return VerifyResult{Checked: true, Resolved: true, Detail: fmt.Sprintf("WordPress install no longer present: %s", clean)}
}
if !info.Mode().IsRegular() {
return VerifyResult{Checked: false, Detail: "wp-config.php path is not a regular file; not auto-verifiable"}
}
ctx, cancel := context.WithTimeout(context.Background(), wpVerifyTimeout)
defer cancel()
// Mirrors CheckWPCore: run as root with --allow-root (not su as the user).
// Args are passed directly (no shell), so the sanitized path cannot inject.
out, err := cmdExec.RunContext(ctx, "wp", "core", "verify-checksums", "--path="+clean, "--allow-root")
if err == nil {
return VerifyResult{Checked: true, Resolved: true, Detail: "WordPress core checksums verify clean"}
}
if len(out) == 0 {
return VerifyResult{Checked: false, Detail: "could not run wp core verify-checksums (try again, or run an account scan)"}
}
var releases wpCoreReleaseChecksums
lineEndingOnly, onlyLineEndings := releases.lineEndingOnly(clean, out)
if onlyLineEndings && wpCoreVerificationCompleted(err, out) {
return VerifyResult{Checked: true, Resolved: true, Detail: "remaining WordPress core differences are line endings only"}
}
for _, line := range strings.Split(string(out), "\n") {
if wpChecksumLineHasExtraneousCoreFile(line) {
return VerifyResult{Checked: true, Resolved: false, Detail: "WordPress core still has extraneous files"}
}
if rel := wpChecksumModifiedCoreFile(line); rel != "" && !lineEndingOnly[rel] {
return VerifyResult{Checked: true, Resolved: false, Detail: "WordPress core still has modified files"}
}
}
// Non-zero exit with neither an extraneous nor a modified file named: a
// wp-cli error we cannot interpret. Do not resolve.
return VerifyResult{Checked: false, Detail: "could not confirm core integrity (verify-checksums reported other issues); re-run or use an account scan"}
}
package checks
import (
"context"
_ "embed"
"fmt"
"strings"
"time"
"gopkg.in/yaml.v3"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
// Known-vulnerable WordPress plugin detector.
//
// CheckOutdatedPlugins grades severity by how far a plugin is behind the latest
// release, which buries an actively-exploited version that is only a few
// releases stale. This detector instead matches the installed version against a
// curated feed of known-vulnerable ranges and elevates confirmed hits (CISA-KEV
// / unauthenticated RCE, privesc, SQLi, secret disclosure) to High or Critical
// regardless of version gap. It shares the cached inventory refresh with
// outdated_plugins and only alerts (v1 never disables a plugin, so a real
// customer site is never taken down).
//go:embed embed/plugin_vulns.yaml
var pluginVulnFeedData []byte
// pluginVuln is one known-vulnerable version range for a plugin slug.
type pluginVuln struct {
Slug string `yaml:"slug"`
CVE string `yaml:"cve"`
Title string `yaml:"title"`
FixedIn string `yaml:"fixed_in"`
MinAffected string `yaml:"min_affected"` // optional lower bound
VirtualPatch bool `yaml:"virtual_patch"` // CSM ships a ModSecurity rule for this CVE
KEV bool `yaml:"kev"`
Severity string `yaml:"severity"`
Reference string `yaml:"reference"`
}
type pluginVulnFeed struct {
Plugins []pluginVuln `yaml:"plugins"`
}
// loadPluginVulnFeed parses the curated feed and fails closed on individual
// malformed entries: an entry with missing or invalid range fields is dropped
// (rather than matching everything) so a bad line cannot create false positives.
func loadPluginVulnFeed(data []byte) ([]pluginVuln, error) {
var feed pluginVulnFeed
if err := yaml.Unmarshal(data, &feed); err != nil {
return nil, err
}
out := make([]pluginVuln, 0, len(feed.Plugins))
for _, v := range feed.Plugins {
if strings.TrimSpace(v.Slug) == "" || strings.TrimSpace(v.CVE) == "" || strings.TrimSpace(v.FixedIn) == "" {
continue
}
if _, ok := parsePluginVersion(v.FixedIn); !ok {
continue
}
if strings.TrimSpace(v.MinAffected) != "" {
cmp, ok := comparePluginVersions(v.MinAffected, v.FixedIn)
if !ok || cmp >= 0 {
continue
}
}
out = append(out, v)
}
return out, nil
}
// parsePluginVersion returns normalized decimal components without converting
// them to machine integers. WordPress versions are not strict semver, so a
// suffix on a numeric component is tolerated and ignored. A component that
// does not start with a digit makes the version unusable; an unknown version
// must never underflow into a false vulnerability match.
func parsePluginVersion(v string) ([]string, bool) {
v = strings.TrimSpace(v)
if len(v) > 1 && (v[0] == 'v' || v[0] == 'V') && v[1] >= '0' && v[1] <= '9' {
v = v[1:]
}
if v == "" {
return nil, false
}
parts := strings.Split(v, ".")
out := make([]string, 0, len(parts))
for _, part := range parts {
i := 0
for i < len(part) && part[i] >= '0' && part[i] <= '9' {
i++
}
if i == 0 {
return nil, false
}
digits := strings.TrimLeft(part[:i], "0")
if digits == "" {
digits = "0"
}
out = append(out, digits)
if i != len(part) {
if part[i] != '-' && part[i] != '+' {
return nil, false
}
break
}
}
return out, true
}
func comparePluginVersions(a, b string) (int, bool) {
av, aok := parsePluginVersion(a)
bv, bok := parsePluginVersion(b)
if !aok || !bok {
return 0, false
}
n := len(av)
if len(bv) > n {
n = len(bv)
}
for i := 0; i < n; i++ {
x, y := "0", "0"
if i < len(av) {
x = av[i]
}
if i < len(bv) {
y = bv[i]
}
if len(x) < len(y) {
return -1, true
}
if len(x) > len(y) {
return 1, true
}
if x < y {
return -1, true
}
if x > y {
return 1, true
}
}
return 0, true
}
// versionLess reports whether version a is strictly older than b. Invalid
// versions are incomparable and therefore never considered older.
func versionLess(a, b string) bool {
cmp, ok := comparePluginVersions(a, b)
return ok && cmp < 0
}
// matchPluginVuln reports whether an installed version falls inside a
// vulnerable range: strictly below fixed_in and, when a lower bound is given,
// at or above min_affected. An install at or above fixed_in is patched and
// never matches.
func matchPluginVuln(installed string, v pluginVuln) bool {
if strings.TrimSpace(installed) == "" {
return false
}
if !versionLess(installed, v.FixedIn) {
return false
}
if strings.TrimSpace(v.MinAffected) != "" && versionLess(installed, v.MinAffected) {
return false
}
return true
}
// vulnPluginSeverity maps a matched vulnerability to a finding severity. A
// version-matched known CVE is actionable by definition, so it is never a
// Warning: KEV/actively-exploited or an explicit "critical" is Critical, an
// explicit "high" is High, and anything else defaults to Critical.
func vulnPluginSeverity(v pluginVuln) alert.Severity {
if v.KEV {
return alert.Critical
}
if strings.EqualFold(strings.TrimSpace(v.Severity), "high") {
return alert.High
}
return alert.Critical
}
func vulnAllowKey(slug, version string) string {
return strings.ToLower(strings.TrimSpace(slug) + "@" + strings.TrimSpace(version))
}
// evaluatePluginVulns matches the cached per-site plugin inventory against the
// feed and returns one finding per confirmed vulnerable install, skipping any
// slug@version the operator has explicitly accepted via the allowlist. Each
// match carries the facts the WAF-coverage correlation needs and the finding
// itself does not record.
func evaluatePluginVulns(sites map[string]store.SitePlugins, feed []pluginVuln, allow map[string]bool) []vulnMatch {
bySlug := make(map[string][]pluginVuln, len(feed))
for _, v := range feed {
key := strings.ToLower(strings.TrimSpace(v.Slug))
bySlug[key] = append(bySlug[key], v)
}
var matches []vulnMatch
for wpPath, site := range sites {
for _, p := range site.Plugins {
for _, v := range bySlug[strings.ToLower(strings.TrimSpace(p.Slug))] {
if !matchPluginVuln(p.InstalledVersion, v) {
continue
}
if allow[vulnAllowKey(p.Slug, p.InstalledVersion)] {
continue
}
matches = append(matches, vulnMatch{
finding: buildVulnPluginFinding(wpPath, site, p, v),
active: vulnPluginActive(p.Status),
vpCovered: v.VirtualPatch,
})
}
}
}
return matches
}
// vulnMatch is one confirmed vulnerable install: the finding the detector
// built, plus whether WordPress actually loads the plugin and whether CSM
// ships a ModSecurity virtual patch for the CVE.
//
// An inactive plugin still earns an inventory finding because some plugins
// expose directly callable files, but that alone does not establish that this
// CVE is reachable without WordPress loading it, so it is never described as
// left open by a missing request filter.
type vulnMatch struct {
finding alert.Finding
active bool
vpCovered bool
}
func buildVulnPluginFinding(wpPath string, site store.SitePlugins, p store.SitePluginEntry, v pluginVuln) alert.Finding {
activeNote := "inactive"
if vulnPluginActive(p.Status) {
activeNote = "active"
}
kevNote := ""
if v.KEV {
kevNote = " [CISA-KEV: actively exploited]"
}
details := fmt.Sprintf("%s (%s)%s. Installed %s %s is %s; fixed in %s. Remediate: update to %s or newer, or remove the plugin.",
v.Title, v.CVE, kevNote, p.Slug, p.InstalledVersion, activeNote, v.FixedIn, v.FixedIn)
if v.Reference != "" {
details += "\nReference: " + v.Reference
}
details += "\nPath: " + wpPath
return alert.Finding{
Severity: vulnPluginSeverity(v),
Check: "vulnerable_plugins",
Message: fmt.Sprintf("Known-vulnerable plugin %s %s (%s) on %s", p.Slug, p.InstalledVersion, v.CVE, site.Domain),
Details: details,
Domain: site.Domain,
TenantID: site.Account,
Timestamp: time.Now(),
}
}
func vulnPluginActive(status string) bool {
switch strings.ToLower(strings.TrimSpace(status)) {
case "active", "active-network", "must-use":
return true
default:
return false
}
}
// CheckVulnerablePlugins matches the shared WordPress plugin inventory against
// the curated known-vulnerable feed. It participates in the same serialized
// refresh as CheckOutdatedPlugins and only reports -- it never disables a
// plugin.
func CheckVulnerablePlugins(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
if cfg != nil && !cfg.VulnerablePluginScanningEnabled() {
return nil
}
if incompleteCollectorFrom(ctx) == nil {
ctx, _ = withIncompleteCheckCollector(ctx)
}
db := store.Global()
if db == nil {
return nil
}
fresh := ensurePluginCacheFresh(ctx, cfg, db)
if ctx.Err() != nil {
return nil
}
// Discovery gaps retain the last usable inventory; evaluating it keeps a
// broken account map from hiding known vulnerabilities until discovery recovers.
if !fresh && !checkMarkedIncomplete(ctx, "vulnerable_plugins") {
return nil
}
feed, err := loadPluginVulnFeed(pluginVulnFeedData)
if err != nil || len(feed) == 0 {
return nil
}
matches := evaluatePluginVulns(db.AllSitePlugins(), feed, vulnPluginAllowSet(cfg))
if len(matches) == 0 {
return nil
}
var candidates []alert.Finding
for _, m := range matches {
if m.active {
candidates = append(candidates, m.finding)
}
}
if len(candidates) > 0 {
annotateUnprotected(matches, vpCoverageForHost(candidates))
}
findings := make([]alert.Finding, 0, len(matches))
for _, m := range matches {
findings = append(findings, m.finding)
}
return findings
}
func vulnPluginAllowSet(cfg *config.Config) map[string]bool {
if cfg == nil || len(cfg.Detection.VulnerablePluginAllow) == 0 {
return nil
}
set := make(map[string]bool, len(cfg.Detection.VulnerablePluginAllow))
for _, e := range cfg.Detection.VulnerablePluginAllow {
e = strings.TrimSpace(e)
if e == "" {
continue
}
set[strings.ToLower(e)] = true
}
return set
}
package checks
import (
"fmt"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/platform"
)
// Correlation between a known-vulnerable plugin and the compensating control
// that is supposed to hold the line until it is patched.
//
// CSM ships ModSecurity virtual patches for several of the CVEs in the plugin
// feed, so a vulnerable install is normally filtered while the operator
// schedules an update. When ModSecurity is off for that traffic the patch never
// executes, and the same finding now describes a directly reachable
// vulnerability. Reported separately, the two facts read as routine
// housekeeping; joined, they are an emergency.
//
// This adds no new detection: both inputs are findings CSM already produces.
// vpCoverage is the state that decides whether CSM's shipped virtual patches
// filter a given site.
type vpCoverage struct {
// engineMode is the host-wide SecRuleEngine value ("on", "detectiononly",
// "off"), empty when it could not be read.
engineMode string
// disabled lists the accounts and vhosts with ModSecurity switched off.
disabled []modsecDisabledScope
// aliases maps an account's addon domain to its unambiguous cPanel-associated
// subdomain (and vice versa). A per-vhost disabled flag can be recorded
// against that servername while the plugin inventory knows the public name.
aliases map[string][]string
}
// aliasKey scopes an alias set to one account: unrelated accounts may name the
// same docroot path, and one account's disabled flag says nothing about
// another's traffic.
func aliasKey(user, domain string) string {
return strings.ToLower(strings.TrimSpace(user)) + "\x00" + strings.ToLower(strings.TrimSpace(domain))
}
// vhostAliasSets links only an unambiguous addon/subdomain pair: the docroot
// group must contain exactly those two records and the subdomain must be the
// exact cPanel association <addon-domain>.<main-domain>. A shared docroot is
// not itself proof that two hostnames are aliases: parked domains, a main
// domain, and an addon can all intentionally route different sites from
// /home/<user>/public_html.
func vhostAliasSets(userdataDomains string) map[string][]string {
vhosts, complete := parseUserdataDomainRootsChecked(userdataDomains)
if !complete {
return nil
}
byRoot := make(map[string][]vhost, len(vhosts))
for _, vh := range vhosts {
root := aliasKey(vh.user, vh.docroot)
byRoot[root] = append(byRoot[root], vh)
}
sets := make(map[string][]string, len(vhosts))
for _, group := range byRoot {
if len(group) != 2 {
continue
}
firstType := strings.ToLower(strings.TrimSpace(group[0].typ))
secondType := strings.ToLower(strings.TrimSpace(group[1].typ))
var addon, sub vhost
switch {
case firstType == "addon" && secondType == "sub":
addon, sub = group[0], group[1]
case firstType == "sub" && secondType == "addon":
addon, sub = group[1], group[0]
default:
continue
}
mainDomain := cleanDomlogDomain(addon.mainDomain)
if mainDomain == "" ||
!strings.EqualFold(cleanDomlogDomain(sub.mainDomain), mainDomain) ||
!strings.EqualFold(sub.domain, addon.domain+"."+mainDomain) {
continue
}
for _, vh := range group {
sets[aliasKey(vh.user, vh.domain)] = []string{group[0].domain, group[1].domain}
}
}
return sets
}
// siteNames is every domain the site answers to: the one the inventory
// recorded plus its docroot peers.
func (c vpCoverage) siteNames(account, domain string) map[string]bool {
names := map[string]bool{strings.ToLower(strings.TrimSpace(domain)): true}
for _, peer := range c.aliases[aliasKey(account, domain)] {
names[strings.ToLower(strings.TrimSpace(peer))] = true
}
delete(names, "")
return names
}
// inertReason explains why CSM's virtual patches do not filter this site, or
// returns an empty reason when they do.
func (c vpCoverage) inertReason(account, domain string) (reason, source string) {
switch strings.ToLower(strings.TrimSpace(c.engineMode)) {
case "off":
return "the ModSecurity engine is off host-wide", ""
case "detectiononly":
return "the ModSecurity engine runs in DetectionOnly mode host-wide", ""
}
if strings.TrimSpace(account) == "" {
return "", ""
}
names := c.siteNames(account, domain)
var accountWide, exact, alias *modsecDisabledScope
for _, s := range c.disabled {
if !strings.EqualFold(strings.TrimSpace(s.User), strings.TrimSpace(account)) {
continue
}
if s.Domain == "" {
if accountWide == nil {
scope := s
accountWide = &scope
}
continue
}
if !names[strings.ToLower(strings.TrimSpace(s.Domain))] {
continue
}
if strings.EqualFold(s.Domain, domain) {
if exact == nil {
scope := s
exact = &scope
}
continue
}
if alias == nil {
scope := s
alias = &scope
}
}
if accountWide != nil {
return "ModSecurity is disabled for account " + accountWide.User + " (all domains)", accountWide.Source
}
if exact != nil {
return "ModSecurity is disabled for " + exact.Domain, exact.Source
}
if alias != nil {
return fmt.Sprintf("ModSecurity is disabled for %s, the cPanel-associated subdomain of %s", alias.Domain, domain), alias.Source
}
return "", ""
}
// annotateUnprotected rewrites vulnerable-plugin findings whose traffic no
// longer passes through ModSecurity, so the alert itself carries the fact that
// nothing stands between the vulnerability and the internet.
//
// Only an active install is rewritten: an inactive plugin is a finding because
// its files sit in the docroot, not because WordPress will run the vulnerable
// code path, so a missing request filter says nothing about its reachability.
// What the annotation claims depends on whether CSM ships a virtual patch for
// the CVE -- naming a patch that was never written would misdescribe the gap.
func annotateUnprotected(matches []vulnMatch, cov vpCoverage) {
for i := range matches {
if !matches[i].active {
continue
}
f := &matches[i].finding
// The production caller passes freshly built findings, but keeping this
// helper idempotent prevents a retrying caller from changing the alert
// identity and appending the same operator guidance repeatedly.
if strings.Contains(f.Message, " -- unprotected: ") ||
strings.Contains(f.Details, "\n\nUnprotected: ") {
continue
}
reason, source := cov.inertReason(f.TenantID, f.Domain)
if reason == "" {
continue
}
f.Severity = alert.Critical
f.Message += " -- unprotected: " + reason
located := reason
if source != "" {
located += " (" + source + ")"
}
gap := "No ModSecurity rule filters this traffic and no modsec audit record\n" +
"is written for it, so an attempt to exploit this leaves no WAF evidence.\n"
if matches[i].vpCovered {
gap = "CSM ships a virtual patch for this CVE and it cannot run here, and no\n" +
"modsec audit record is written for this traffic either, so the\n" +
"vulnerability is directly reachable and an attempt to exploit it\n" +
"leaves no WAF evidence.\n"
}
f.Details += "\n\nUnprotected: " + located + ".\n" + gap +
"Patch the plugin now, or restore filtering for this scope first."
}
}
// vpCoverageForHost is the seam the wiring is tested through: the snapshot it
// returns is assembled from platform detection and host config that a unit
// test cannot stage.
var vpCoverageForHost = currentVPCoverage
// currentVPCoverage reads the live ModSecurity state. Only cPanel receives
// CSM's virtual patches and only cPanel expresses per-vhost ModSecurity
// state, so on every other platform the question has no answer and the
// findings are left untouched.
func currentVPCoverage(findings []alert.Finding) vpCoverage {
info := platform.Detect()
if !info.IsCPanel() {
return vpCoverage{}
}
cov := vpCoverage{engineMode: checkEngineMode(info)}
if data, err := osFS.ReadFile(userdataDomainsPath); err == nil {
cov.aliases = vhostAliasSets(string(data))
}
// CheckWAFStatus already walks every userdata record in this scan tier.
// Correlation needs only the vulnerable sites, so read their account,
// domain, and unambiguous associated-subdomain paths instead of repeating the
// host-wide O(number of vhosts) traversal.
cov.disabled = modsecDisabledScopesForFindings(info, findings, cov.aliases)
return cov
}
package checks
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/modsec"
"github.com/pidginhost/csm/internal/netutil"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
)
// wafRulesAssembleRetryDelay is the wait between the first negative
// probe and the re-probe on cPanel+LiteSpeed hosts where cPanel's
// nightly modsec_assemble briefly leaves both `whmapi1
// modsec_get_configs` and the vendor dir empty while it rewrites the
// tree in place. Observed windows are <10s; 30s gives margin without
// meaningfully delaying the surrounding deep-scan tier. Tests override
// this to keep the suite fast.
var wafRulesAssembleRetryDelay = 30 * time.Second
// CheckWAFStatus verifies that ModSecurity is loaded, the engine is in
// enforcement mode (not DetectionOnly), OWASP/Comodo rules are active,
// and rules are up to date.
func CheckWAFStatus(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
manageHost := cfg == nil || !cfg.ObserveMode()
info := platform.Detect()
// If there is no web server at all, WAF concerns don't apply to this host.
if info.WebServer == platform.WSNone {
return findings
}
modsecActive := modsecDetected(info)
if !modsecActive {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "waf_status",
Message: "ModSecurity WAF is not active",
Details: wafInstallHint(info),
})
return findings // no point checking further
}
// --- Engine mode check ---
engineMode := checkEngineMode(info)
if engineMode == "detectiononly" {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "waf_detection_only",
Message: "ModSecurity is in DetectionOnly mode - attacks are logged but NOT blocked",
Details: "SecRuleEngine is set to DetectionOnly. Change to 'On' for enforcement:\nWHM > Security Center > ModSecurity > Edit Global Directive",
})
}
// --- Rule vendor check ---
ruleDirs := modsecRuleDirs(info)
hasRules := probeWAFRules(info, ruleDirs)
// cPanel+LiteSpeed: cPanel's nightly modsec_assemble rewrites the
// vendor tree in place, so for ~6-10s both `whmapi1
// modsec_get_configs` and the vendor dir return empty. A production
// false positive at 01:10:27 fired 6s after the rewrite. Re-probe
// once after a short delay before alerting; on a host that really
// has no rules, the re-probe is still negative and we alert in the
// same scan, so this doesn't shift detection to the next deep tier.
if !hasRules && info.IsCPanel() && info.WebServer == platform.WSLiteSpeed {
select {
case <-time.After(wafRulesAssembleRetryDelay):
case <-ctx.Done():
return findings
}
hasRules = probeWAFRules(info, ruleDirs)
}
if !hasRules {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "waf_rules",
Message: "ModSecurity has no WAF rules loaded",
Details: wafRulesHint(info),
})
}
// --- Rule age check + auto-update ---
if hasRules {
staleAge := checkRuleAge(info, ruleDirs)
if staleAge > 0 {
// Attempt auto-update before alerting
updated := false
if info.IsCPanel() && manageHost {
updated = autoUpdateWAFRules()
}
if updated {
// Re-check age after update
staleAge = checkRuleAge(info, ruleDirs)
}
if staleAge > 0 {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "waf_rules_stale",
Message: fmt.Sprintf("ModSecurity rules last updated %d days ago - update recommended", staleAge),
Details: wafRulesStaleHint(info),
})
}
}
}
// --- Virtual patch deployment ---
// Only cPanel has the modsec user config dirs we write into.
if info.IsCPanel() && manageHost {
reloadCommand := ""
if cfg != nil {
reloadCommand = cfg.ModSec.ReloadCommand
}
// Hosts without a reload command are warned at daemon startup; a
// finding here would repeat every scan with nothing CSM can verify.
if err := deployAndReconcileModSec(ctx, reloadCommand); err != nil {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "waf_status",
Message: "CSM ModSecurity rule activation could not be confirmed",
Details: err.Error(),
})
}
}
// --- Disabled ModSecurity scopes ---
// whmapi1 reports disabled *rules*, never a disabled engine, so this
// reads the userdata and conf.d state that actually gates filtering.
findings = append(findings, modsecDisabledFindings(info)...)
return findings
}
// probeWAFRules checks whether any WAF rule source -- cPanel's whmapi1
// active-config list or the on-disk vendor/CRS directories -- currently
// reports rules. Used by CheckWAFStatus directly and again on retry
// for the cPanel+LiteSpeed modsec_assemble race.
func probeWAFRules(info platform.Info, ruleDirs []string) bool {
if info.IsCPanel() {
if activeRules, ok := cPanelActiveRules(); ok {
if activeRules.present {
return true
}
// During LiteSpeed's modsec_assemble window, the WHM config
// list and filesystem can become visible in either order. Keep
// the existing filesystem backstop so either source can end the
// retry without letting retired trees affect other cPanel hosts.
if info.WebServer != platform.WSLiteSpeed {
return false
}
return hasRuleArtifacts(ruleDirs)
}
if out, _ := runCmd("whmapi1", "modsec_get_vendors"); out != nil {
outStr := string(out)
if strings.Contains(outStr, "comodo") || strings.Contains(outStr, "owasp") ||
strings.Contains(outStr, "OWASP") || strings.Contains(outStr, "Comodo") {
return true
}
}
}
return hasRuleArtifacts(ruleDirs)
}
// modsecDetected returns true if a ModSecurity module is loaded for the
// detected web server. It first consults the platform layer, then falls
// back to scanning config files.
func modsecDetected(info platform.Info) bool {
// cPanel fast path
if info.IsCPanel() {
if out, _ := runCmd("whmapi1", "modsec_is_installed"); out != nil &&
strings.Contains(string(out), "installed: 1") {
return true
}
}
// Generic file-based probes per web server
for _, conf := range expandPathGlobs(modsecActivationCandidates(info)) {
data, err := osFS.ReadFile(conf)
if err != nil {
continue
}
if modsecEnabledInConfig(info, string(data)) {
return true
}
}
return false
}
// modsecActivationCandidates returns the config files that can enable the
// ModSecurity module for the detected web server.
func modsecActivationCandidates(info platform.Info) []string {
var paths []string
switch info.WebServer {
case platform.WSApache:
if info.ApacheConfigDir != "" {
paths = append(paths,
filepath.Join(info.ApacheConfigDir, "httpd.conf"),
filepath.Join(info.ApacheConfigDir, "apache2.conf"),
filepath.Join(info.ApacheConfigDir, "modsec2.conf"),
filepath.Join(info.ApacheConfigDir, "conf.d", "modsec2.conf"),
filepath.Join(info.ApacheConfigDir, "mods-enabled", "security2.conf"),
filepath.Join(info.ApacheConfigDir, "conf-enabled", "security2.conf"),
filepath.Join(info.ApacheConfigDir, "conf.d", "mod_security.conf"),
filepath.Join(info.ApacheConfigDir, "conf.modules.d", "10-mod_security.conf"),
filepath.Join(info.ApacheConfigDir, "conf.d", "*.conf"),
filepath.Join(info.ApacheConfigDir, "mods-enabled", "*.conf"),
filepath.Join(info.ApacheConfigDir, "conf-enabled", "*.conf"),
)
}
case platform.WSNginx:
if info.NginxConfigDir != "" {
paths = append(paths,
filepath.Join(info.NginxConfigDir, "nginx.conf"),
filepath.Join(info.NginxConfigDir, "conf.d", "*.conf"),
filepath.Join(info.NginxConfigDir, "sites-enabled", "*"),
)
}
case platform.WSLiteSpeed:
paths = append(paths,
"/usr/local/lsws/conf/httpd_config.xml",
)
}
return paths
}
// modsecConfigCandidates returns the set of config files worth scanning
// for ModSecurity directives on the detected web server.
func modsecConfigCandidates(info platform.Info) []string {
paths := append([]string(nil), modsecActivationCandidates(info)...)
switch info.WebServer {
case platform.WSNginx:
if info.NginxConfigDir != "" {
paths = append(paths,
filepath.Join(info.NginxConfigDir, "modules-enabled", "*.conf"),
filepath.Join(info.NginxConfigDir, "modsec", "main.conf"),
)
}
case platform.WSLiteSpeed:
paths = append(paths, "/usr/local/lsws/conf/modsec2.conf")
}
return paths
}
func modsecEnabledInConfig(info platform.Info, contents string) bool {
scanner := bufio.NewScanner(strings.NewReader(contents))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
lineLower := strings.ToLower(line)
switch info.WebServer {
case platform.WSNginx:
if strings.HasPrefix(lineLower, "#") {
continue
}
if strings.HasPrefix(lineLower, "modsecurity on") ||
strings.HasPrefix(lineLower, "modsecurity_rules ") ||
strings.HasPrefix(lineLower, "modsecurity_rules_file ") {
return true
}
case platform.WSLiteSpeed:
if strings.Contains(lineLower, "mod_security") || strings.Contains(lineLower, "modsecurity") {
return true
}
default:
if strings.HasPrefix(lineLower, "#") {
continue
}
if strings.Contains(lineLower, "security2_module") ||
strings.HasPrefix(lineLower, "secruleengine ") ||
strings.Contains(lineLower, "mod_security2") {
return true
}
}
}
return false
}
func expandPathGlobs(paths []string) []string {
var expanded []string
seen := make(map[string]struct{})
for _, candidate := range paths {
matches := []string{candidate}
if strings.ContainsAny(candidate, "*?[") {
if globbed, err := osFS.Glob(candidate); err == nil && len(globbed) > 0 {
matches = globbed
}
}
for _, match := range matches {
if _, ok := seen[match]; ok {
continue
}
seen[match] = struct{}{}
expanded = append(expanded, match)
}
}
return expanded
}
// modsecRuleDirs delegates to the canonical helper in internal/modsec.
// Kept as a package-local thin wrapper because the existing waf check tests
// reference this name directly.
func modsecRuleDirs(info platform.Info) []string {
return modsec.RuleDirs(info)
}
// wafInstallHint returns platform-specific install instructions.
func wafInstallHint(info platform.Info) string {
switch {
case info.IsCPanel():
return "No ModSecurity module detected. Install: WHM > Security Center > ModSecurity"
case info.WebServer == platform.WSNginx && info.IsDebianFamily():
return "No ModSecurity module detected for Nginx.\nInstall: apt install libnginx-mod-http-modsecurity modsecurity-crs"
case info.WebServer == platform.WSApache && info.IsDebianFamily():
return "No ModSecurity module detected for Apache.\nInstall: apt install libapache2-mod-security2 modsecurity-crs && a2enmod security2"
case info.WebServer == platform.WSApache && info.IsRHELFamily():
return "No ModSecurity module detected for Apache.\nInstall (requires EPEL): dnf install -y epel-release && dnf install -y mod_security mod_security_crs && systemctl restart httpd"
case info.WebServer == platform.WSNginx && info.IsRHELFamily():
return "No ModSecurity module detected for Nginx.\nInstall (requires EPEL): dnf install -y epel-release && dnf install -y nginx-mod-http-modsecurity && systemctl restart nginx"
}
return "No ModSecurity module detected. The server has no web application firewall protecting against SQL injection, XSS, and other web attacks."
}
// wafRulesHint returns platform-specific rules-install instructions.
func wafRulesHint(info platform.Info) string {
if info.IsCPanel() {
return "ModSecurity is installed but has no OWASP or Comodo rules. Add rules: WHM > Security Center > ModSecurity Vendors"
}
if info.IsDebianFamily() {
return "ModSecurity is installed but has no rules loaded. Install OWASP CRS: apt install modsecurity-crs"
}
if info.IsRHELFamily() {
return "ModSecurity is installed but has no rules loaded. Install OWASP CRS: dnf install --enablerepo=epel modsecurity-crs"
}
return "ModSecurity is installed but has no rules loaded."
}
// wafRulesStaleHint returns platform-specific advice for updating stale
// ModSecurity vendor rules.
func wafRulesStaleHint(info platform.Info) string {
if info.IsCPanel() {
return "At least one vendor with active configuration files has not refreshed its rules in over a month. Inactive vendor trees are not counted. Check: WHM > Security Center > ModSecurity Vendors > Update"
}
if info.IsDebianFamily() {
return "Vendor rules should be updated at least monthly. Update with: apt update && apt upgrade modsecurity-crs"
}
if info.IsRHELFamily() {
return "Vendor rules should be updated at least monthly. Update with: dnf upgrade modsecurity-crs"
}
return "Vendor rules should be updated at least monthly."
}
// checkEngineMode determines the host-wide SecRuleEngine setting. On cPanel
// the generated configuration is authoritative: modsec2.conf turns the
// engine on and then includes modsec2.cpanel.conf (WHM "Edit Global
// Directive") and modsec2.user.conf, and Apache applies the last directive
// it parses, so the mode is whatever the last file in that chain says. A
// distro package's file is never consulted on cPanel: an inactive file
// reported as the host setting would manufacture a false unprotected
// Critical. Returns "on", "detectiononly", "off", or "" if unknown.
func checkEngineMode(info platform.Info) string {
if info.IsCPanel() {
return cPanelEngineMode(info)
}
configPaths := modsecConfigCandidates(info)
// Also include the top-level modsecurity.conf installed by distro packages.
configPaths = append(configPaths,
"/etc/modsecurity/modsecurity.conf",
"/etc/nginx/modsec/modsecurity.conf",
)
for _, path := range configPaths {
if mode := engineModeInFile(path); mode != "" {
return mode
}
}
return ""
}
// cPanelModsecEngineChain lists, in include order, the cPanel files that can
// set SecRuleEngine at server scope under the Apache-compatible config
// directory (cPanel + LiteSpeed reads the same tree through loadApacheConf).
// The modsec include directory is spelled both ways across EA4 and the
// older /usr/local/apache layout, and CSM already writes virtual patches to
// both, so each link tries every known spelling and uses the first that
// exists.
func cPanelModsecEngineChain(configDir string) [][]string {
return [][]string{
{filepath.Join(configDir, "conf.d", "modsec2.conf")},
{
filepath.Join(configDir, "conf.d", "modsec", "modsec2.cpanel.conf"),
filepath.Join(configDir, "conf.d", "modsec2.cpanel.conf"),
filepath.Join(configDir, "modsec2.cpanel.conf"),
},
{
filepath.Join(configDir, "conf.d", "modsec", "modsec2.user.conf"),
filepath.Join(configDir, "conf.d", "modsec2.user.conf"),
},
}
}
// cPanelEngineMode walks the include chain and returns the last engine
// directive. A chain file that exists but cannot be read, or that this
// parser cannot interpret authoritatively, makes the effective mode
// unknown: guessing from the files around it could hide a DetectionOnly
// set through WHM or report a stale On.
func cPanelEngineMode(info platform.Info) string {
configDir := filepath.Clean(info.ApacheCompatibleConfigDir())
if configDir == "." || configDir == string(filepath.Separator) {
return ""
}
mode := ""
for _, spellings := range cPanelModsecEngineChain(configDir) {
for _, path := range spellings {
m, known, err := engineDirectiveInFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
continue
}
return ""
}
if !known {
return ""
}
if m != "" {
mode = m
}
// The first spelling that exists is the file Apache reads.
break
}
}
return mode
}
// engineModeInFile is the distro-package form of engineDirectiveInFile: a
// missing, unreadable or ambiguous file simply yields unknown.
func engineModeInFile(path string) string {
mode, known, err := engineDirectiveInFile(path)
if err != nil || !known {
return ""
}
return mode
}
// lastEngineDirective returns the last server-scope SecRuleEngine value
// Apache would apply from path, or "" when the file is missing or cannot be
// interpreted authoritatively.
func lastEngineDirective(path string) string {
mode, known := readLastEngineDirective(path)
if !known {
return ""
}
return mode
}
// readLastEngineDirective also reports whether the file was interpreted
// authoritatively. A missing optional include has no effect, but an
// unreadable or ambiguous later include must invalidate an earlier mode
// instead of silently preserving it.
func readLastEngineDirective(path string) (string, bool) {
mode, known, err := engineDirectiveInFile(path)
if err != nil {
return "", errors.Is(err, os.ErrNotExist)
}
return mode, known
}
// engineDirectiveInFile opens path and returns the last server-scope
// SecRuleEngine value Apache would apply from it, lower-cased. known is
// false when the file could not be interpreted authoritatively; err carries
// the open or read failure so callers can tell a missing file from an
// unreadable one.
func engineDirectiveInFile(path string) (mode string, known bool, err error) {
f, err := osFS.Open(path)
if err != nil {
return "", false, err
}
defer func() { _ = f.Close() }()
mode, known, err = parseEngineDirectives(f)
return mode, known, err
}
// parseEngineDirectives scans Apache configuration text for the last
// server-scope SecRuleEngine directive. The cPanel wrapper commonly places
// the directive inside a positive mod_security2 IfModule block. A directive
// in an unrelated IfModule, in a request or virtual-host container, or in a
// runtime-conditional container makes the effective mode unknown because
// this parser cannot know which other Apache modules or defines are live.
func parseEngineDirectives(r io.Reader) (mode string, known bool, err error) {
type container struct {
name string
active bool
activeKnown bool
serverScope bool
}
var stack []container
active, activeKnown, serverScope := true, true, true
valid := true
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := strings.TrimSpace(stripApacheComment(scanner.Text()))
if line == "" {
continue
}
tag, isContainer, tagValid := parseApacheContainerTag(line)
if isContainer {
if !tagValid {
valid = false
continue
}
if tag.closing {
if len(stack) == 0 || !strings.EqualFold(stack[len(stack)-1].name, tag.name) {
valid = false
continue
}
stack = stack[:len(stack)-1]
active, activeKnown, serverScope = true, true, true
if len(stack) > 0 {
active = stack[len(stack)-1].active
activeKnown = stack[len(stack)-1].activeKnown
serverScope = stack[len(stack)-1].serverScope
}
continue
}
next := container{name: tag.name, active: active, activeKnown: activeKnown, serverScope: serverScope}
switch {
case strings.EqualFold(tag.name, "IfModule"):
condition, condKnown := activeModSecurityIfModule(tag.label)
switch {
case activeKnown && !active:
// An inactive parent keeps every nested directive inactive.
case condKnown && !condition:
next.active = false
next.activeKnown = true
case condKnown:
next.activeKnown = activeKnown
case activeKnown:
next.active = false
next.activeKnown = false
default:
next.active = false
}
case apacheEngineScopedContainer(tag.name):
// A directive in a request or virtual-host context does not
// define the server-wide engine mode.
next.serverScope = false
case activeKnown && active:
// IfDefine, IfVersion, IfFile and expression containers keep
// server scope but depend on runtime state unavailable here.
next.active = false
next.activeKnown = false
}
stack = append(stack, next)
active, activeKnown, serverScope = next.active, next.activeKnown, next.serverScope
continue
}
fields, fieldsValid := parseApacheDirectiveFields(line)
if !fieldsValid {
valid = false
continue
}
if !serverScope || len(fields) == 0 || !strings.EqualFold(fields[0], "SecRuleEngine") {
continue
}
if !activeKnown {
valid = false
continue
}
if !active {
continue
}
if len(fields) != 2 {
valid = false
continue
}
switch value := strings.ToLower(fields[1]); value {
case "on", "off", "detectiononly":
mode = value
default:
valid = false
}
}
if scanErr := scanner.Err(); scanErr != nil {
return "", false, scanErr
}
if len(stack) != 0 || !valid {
return "", false, nil
}
return mode, true, nil
}
func apacheEngineScopedContainer(name string) bool {
switch strings.ToLower(name) {
case "virtualhost", "directory", "directorymatch", "files", "filesmatch",
"location", "locationmatch", "proxy", "limit", "limitexcept":
return true
default:
return false
}
}
func activeModSecurityIfModule(label string) (active, known bool) {
fields, valid := parseApacheDirectiveFields(label)
if !valid || len(fields) != 2 {
return false, false
}
module := strings.ToLower(fields[1])
negated := strings.HasPrefix(module, "!")
module = strings.TrimPrefix(module, "!")
switch module {
case "mod_security2.c", "security2_module":
return !negated, true
default:
return false, false
}
}
// checkRuleAge returns the age of the rules that protect the host, or 0 when
// they were refreshed within the last 30 days.
//
// cPanel retains inactive vendor trees indefinitely, so its active-config API
// narrows the scan to loaded vendor paths. The newest artifact in each loaded
// vendor tree is its refresh time; the least recently refreshed loaded vendor
// decides the reported age. On other platforms, and when the API is
// unavailable, retaining the former oldest-file behavior is conservative: a
// fresh candidate tree that is not loaded must not hide stale rules in use.
func checkRuleAge(info platform.Info, ruleDirs []string) int {
artifactMtime := oldestRuleArtifact
if info.IsCPanel() {
activeRules, ok := cPanelActiveRules()
if ok {
if !activeRules.present || len(activeRules.vendorDirs) == 0 {
return 0
}
ruleDirs = activeRules.vendorDirs
artifactMtime = oldestRulesetRefresh
}
}
mtime, found := artifactMtime(ruleDirs)
if !found {
return 0
}
age := int(time.Since(mtime).Hours() / 24)
if age > 30 {
return age
}
return 0
}
func hasRuleArtifacts(ruleDirs []string) bool {
_, found := newestRuleArtifact(ruleDirs)
return found
}
type cPanelModSecVendor struct {
Path string `json:"path"`
VendorID string `json:"vendor_id"`
}
type cPanelModSecVendorsResponse struct {
Data struct {
Vendors []cPanelModSecVendor `json:"vendors"`
} `json:"data"`
Metadata struct {
Result int `json:"result"`
} `json:"metadata"`
}
type cPanelModSecConfig struct {
Active int `json:"active"`
VendorID string `json:"vendor_id"`
}
type cPanelModSecConfigsResponse struct {
Data struct {
Configs []cPanelModSecConfig `json:"configs"`
} `json:"data"`
Metadata struct {
Result int `json:"result"`
} `json:"metadata"`
}
type cPanelActiveRuleState struct {
present bool
vendorDirs []string
}
// cPanelActiveRules maps configuration files that WHM reports as active back
// to their vendor trees. Vendor-wide enabled state is insufficient because
// cPanel permits individual files to override it in either direction. The
// boolean is false when WHM cannot provide an authoritative mapping, in which
// case callers retain conservative filesystem behavior.
func cPanelActiveRules() (cPanelActiveRuleState, bool) {
out, err := runCmd("whmapi1", "modsec_get_configs", "--output=json")
if err != nil || len(out) == 0 {
return cPanelActiveRuleState{}, false
}
var configsResponse cPanelModSecConfigsResponse
if unmarshalErr := json.Unmarshal(out, &configsResponse); unmarshalErr != nil || configsResponse.Metadata.Result != 1 {
return cPanelActiveRuleState{}, false
}
activeVendorIDs := make(map[string]struct{})
activeRules := cPanelActiveRuleState{}
for _, config := range configsResponse.Data.Configs {
if config.Active != 1 {
continue
}
activeRules.present = true
if config.VendorID != "" {
activeVendorIDs[config.VendorID] = struct{}{}
}
}
if len(activeVendorIDs) == 0 {
return activeRules, true
}
out, err = runCmd("whmapi1", "modsec_get_vendors", "--output=json")
if err != nil || len(out) == 0 {
return cPanelActiveRuleState{}, false
}
var vendorsResponse cPanelModSecVendorsResponse
if unmarshalErr := json.Unmarshal(out, &vendorsResponse); unmarshalErr != nil || vendorsResponse.Metadata.Result != 1 {
return cPanelActiveRuleState{}, false
}
seen := make(map[string]struct{})
for _, vendor := range vendorsResponse.Data.Vendors {
if _, active := activeVendorIDs[vendor.VendorID]; !active {
continue
}
if vendor.Path == "" || !filepath.IsAbs(vendor.Path) {
return cPanelActiveRuleState{}, false
}
path := filepath.Clean(vendor.Path)
if path == string(filepath.Separator) {
return cPanelActiveRuleState{}, false
}
if _, exists := seen[path]; exists {
delete(activeVendorIDs, vendor.VendorID)
continue
}
seen[path] = struct{}{}
delete(activeVendorIDs, vendor.VendorID)
activeRules.vendorDirs = append(activeRules.vendorDirs, path)
}
if len(activeVendorIDs) != 0 {
return cPanelActiveRuleState{}, false
}
return activeRules, true
}
// oldestRulesetRefresh treats each directory as one ruleset. Files within a
// ruleset are not all rewritten by every update, so its newest artifact is the
// useful refresh signal. Every loaded ruleset must stay current, making the
// oldest of those per-directory refresh times the value to check.
func oldestRulesetRefresh(ruleDirs []string) (time.Time, bool) {
var oldest time.Time
found := false
for _, dir := range ruleDirs {
newest, ok := newestRuleArtifact([]string{dir})
if !ok {
continue
}
if !found || newest.Before(oldest) {
oldest = newest
found = true
}
}
return oldest, found
}
func newestRuleArtifact(ruleDirs []string) (time.Time, bool) {
return ruleArtifactMtime(ruleDirs, func(candidate, current time.Time) bool {
return candidate.After(current)
})
}
func oldestRuleArtifact(ruleDirs []string) (time.Time, bool) {
return ruleArtifactMtime(ruleDirs, func(candidate, current time.Time) bool {
return candidate.Before(current)
})
}
func ruleArtifactMtime(ruleDirs []string, replace func(time.Time, time.Time) bool) (time.Time, bool) {
var selected time.Time
found := false
pending := append([]string(nil), ruleDirs...)
visited := make(map[string]struct{})
for len(pending) > 0 {
dir := pending[len(pending)-1]
pending = pending[:len(pending)-1]
dir = filepath.Clean(dir)
if _, seen := visited[dir]; seen {
continue
}
visited[dir] = struct{}{}
entries, err := osFS.ReadDir(dir)
if err != nil {
continue
}
for _, entry := range entries {
path := filepath.Join(dir, entry.Name())
if entry.IsDir() {
pending = append(pending, path)
continue
}
if !isRuleArtifact(entry.Name()) {
continue
}
info, err := entry.Info()
if err != nil {
continue
}
if !found || replace(info.ModTime(), selected) {
selected = info.ModTime()
found = true
}
}
}
return selected, found
}
// isRuleArtifact reports whether a filename looks like a ModSecurity rule
// or data artifact (.conf, .data, .rules) so unrelated files like README
// or LICENSE don't dominate the age calculation.
func isRuleArtifact(name string) bool {
name = strings.ToLower(name)
return strings.HasSuffix(name, ".conf") ||
strings.HasSuffix(name, ".data") ||
strings.HasSuffix(name, ".rules")
}
// Markers delimiting the CSM-managed section inside modsec2.user.conf.
// That file is shared with operator-maintained rules (Host-scoped
// ctl:ruleRemoveById exclusions and the like), so CSM may only ever
// rewrite the bytes between these two lines. vpLegacyMarker is the header
// comment of the rules file itself, which is all that pre-delimiter CSM
// versions wrote; it locates those deployments for upgrade.
const (
vpBeginMarker = "# BEGIN CSM Custom ModSecurity Rules (managed by CSM - do not edit inside this block)"
vpEndMarker = "# END CSM Custom ModSecurity Rules"
vpLegacyMarker = "# CSM Custom ModSecurity Rules"
vpOverridesIncludeMarker = "# CSM overrides - managed by CSM rule management"
)
// deployVirtualPatches ensures CSM's custom ModSec rules are installed.
// These provide virtual patches for known WordPress CVEs.
//
// The destination is shared with operator rules, so CSM only ever creates
// or rewrites its own marker-delimited section; every byte outside the
// section is preserved verbatim.
func deployVirtualPatches() {
srcPath := "/opt/csm/configs/csm_modsec_custom.conf"
srcData, err := osFS.ReadFile(srcPath)
if err != nil {
return // no custom rules to deploy
}
section := buildVPSection(srcData)
for _, dest := range vpDestPaths {
dir := filepath.Dir(dest)
if _, err := osFS.Stat(dir); os.IsNotExist(err) {
continue
}
existing, err := osFS.ReadFile(dest)
if err != nil && !os.IsNotExist(err) {
// Present but unreadable: rewriting blind could destroy
// operator rules, so leave this candidate alone.
continue
}
merged, upToDate := mergeVPSection(existing, section)
if upToDate {
return
}
// #nosec G306 -- WAF rule file read by Apache/nginx as a different user.
if err := osFS.WriteFile(dest, merged, 0644); err != nil {
continue
}
fmt.Fprintf(os.Stderr, "[%s] Virtual patches deployed to %s\n",
time.Now().Format("2006-01-02 15:04:05"), dest)
return
}
}
// MergeModSecUserConfSection merges CSM's ModSecurity rules payload into
// the current contents of a modsec2.user.conf, confining CSM to its
// marker-delimited section so operator-maintained rules outside the
// section survive every deploy. It is exported because three call sites
// write this file (the WAF check cycle here, `csm install`, and the
// daemon startup config deploy); routing them all through one merge
// guarantees no caller ever whole-file-overwrites operator rules.
//
// existing is the current file contents (nil for a missing file). merged
// is only meaningful when changed is true; changed=false means the file
// already carries the wanted section and must not be rewritten.
func MergeModSecUserConfSection(existing, srcData []byte) (merged []byte, changed bool) {
merged, upToDate := mergeVPSection(existing, buildVPSection(srcData))
return merged, !upToDate
}
// RemoveModSecUserConfSections removes only content owned by CSM from the
// shared ModSecurity user configuration. Operator bytes outside the exact CSM
// marker lines are preserved.
func RemoveModSecUserConfSections(existing []byte) (cleaned []byte, changed bool) {
cleaned = existing
for {
if begin, _, ok := markerLineBounds(cleaned, vpBeginMarker); ok {
end := vpSectionEnd(cleaned[begin:])
if end < 0 {
end = vpBlockEndBeforePreservedTail(cleaned, begin) - begin
}
cleaned = removeByteRange(cleaned, begin, begin+end)
changed = true
continue
}
if legacy, _, ok := markerLineBounds(cleaned, vpLegacyMarker); ok {
end := vpBlockEndBeforePreservedTail(cleaned, legacy)
cleaned = removeByteRange(cleaned, legacy, end)
changed = true
continue
}
break
}
for {
markerStart, markerEnd, ok := markerLineBounds(cleaned, vpOverridesIncludeMarker)
if !ok {
break
}
removeEnd := markerEnd
if includeEnd := nextLineEnd(cleaned, markerEnd); includeEnd > markerEnd {
line := bytes.TrimSpace(cleaned[markerEnd:includeEnd])
if bytes.HasPrefix(line, []byte("Include ")) && bytes.Contains(line, []byte("modsec2.csm-overrides.conf")) {
removeEnd = includeEnd
}
}
cleaned = removeByteRange(cleaned, markerStart, removeEnd)
changed = true
}
return cleaned, changed
}
func removeByteRange(data []byte, start, end int) []byte {
out := make([]byte, 0, len(data)-(end-start))
out = append(out, data[:start]...)
return append(out, data[end:]...)
}
func nextLineEnd(data []byte, start int) int {
if start >= len(data) {
return start
}
if offset := bytes.IndexByte(data[start:], '\n'); offset >= 0 {
return start + offset + 1
}
return len(data)
}
// buildVPSection wraps the rules payload in the begin/end marker lines.
// The result is deterministic for a given payload so later cycles can
// recognize an up-to-date section by byte comparison.
func buildVPSection(srcData []byte) []byte {
section := make([]byte, 0, len(vpBeginMarker)+len(srcData)+len(vpEndMarker)+3)
section = append(section, vpBeginMarker...)
section = append(section, '\n')
section = append(section, srcData...)
if len(srcData) > 0 && srcData[len(srcData)-1] != '\n' {
section = append(section, '\n')
}
section = append(section, vpEndMarker...)
section = append(section, '\n')
return section
}
// mergeVPSection computes the new content for a modsec user conf so that
// it carries exactly one copy of the CSM section while every byte outside
// the section stays untouched. upToDate reports that the file already
// holds the wanted section and no write is needed.
func mergeVPSection(existing, section []byte) (merged []byte, upToDate bool) {
if len(existing) == 0 {
return section, false
}
if begin, _, ok := markerLineBounds(existing, vpBeginMarker); ok {
if end := vpSectionEnd(existing[begin:]); end >= 0 {
if bytes.Equal(existing[begin:begin+end], section) {
return nil, true
}
merged = append(merged, existing[:begin]...)
merged = append(merged, section...)
merged = append(merged, existing[begin+end:]...)
return merged, false
}
// A begin marker without an end marker is a malformed CSM block.
// Replace from the exact begin line so the next cycle is delimited
// again instead of falling through to the legacy header inside it.
blockEnd := vpBlockEndBeforePreservedTail(existing, begin)
merged = append(merged, existing[:begin]...)
merged = append(merged, section...)
merged = append(merged, existing[blockEnd:]...)
return merged, false
}
if legacy, _, ok := markerLineBounds(existing, vpLegacyMarker); ok {
// Pre-delimiter CSM versions appended the raw rules file, so the
// CSM content starts at this exact header line. Installer and
// daemon deploys appended the overrides Include after the raw
// rules, so preserve that tail when it is present.
legacyEnd := vpBlockEndBeforePreservedTail(existing, legacy)
merged = append(merged, existing[:legacy]...)
merged = append(merged, section...)
merged = append(merged, existing[legacyEnd:]...)
return merged, false
}
// Operator-only file: append the section, separated by one blank
// line, keeping the existing bytes exactly as they are.
merged = append(merged, existing...)
if existing[len(existing)-1] != '\n' {
merged = append(merged, '\n')
}
merged = append(merged, '\n')
merged = append(merged, section...)
return merged, false
}
func vpBlockEndBeforePreservedTail(existing []byte, blockStart int) int {
blockEnd := len(existing)
if tail, _, ok := markerLineBounds(existing[blockStart:], vpOverridesIncludeMarker); ok && tail > 0 {
blockEnd = blockStart + tail
if existing[blockEnd-1] == '\n' {
blockEnd--
}
}
return blockEnd
}
// markerLineBounds returns the start offset and end offset (including the
// trailing newline when present) for an exact marker line. Matching the
// whole line keeps operator comments that merely mention marker text from
// being treated as CSM-owned content.
func markerLineBounds(data []byte, marker string) (start, end int, ok bool) {
markerBytes := []byte(marker)
for start < len(data) {
lineEnd := bytes.IndexByte(data[start:], '\n')
end = len(data)
next := len(data)
if lineEnd >= 0 {
end = start + lineEnd
next = end + 1
}
line := data[start:end]
if len(line) > 0 && line[len(line)-1] == '\r' {
line = line[:len(line)-1]
}
if bytes.Equal(line, markerBytes) {
return start, next, true
}
start = next
}
return 0, 0, false
}
// vpSectionEnd returns the offset just past the end-marker line (its
// trailing newline included when present), or -1 when there is no end
// marker. data must start at the section's begin line.
func vpSectionEnd(data []byte) int {
if _, end, ok := markerLineBounds(data, vpEndMarker); ok {
return end
}
return -1
}
// autoUpdateWAFRules triggers ModSecurity vendor rule updates via whmapi1.
// Returns true if an update was successfully triggered.
func autoUpdateWAFRules() bool {
// Get installed vendors
out, err := runCmd("whmapi1", "modsec_get_vendors")
if err != nil || out == nil {
return false
}
// Parse vendor IDs from output (look for "vendor_id:" lines)
var vendors []string
for _, line := range strings.Split(string(out), "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "vendor_id:") || strings.HasPrefix(line, "id:") {
parts := strings.SplitN(line, ":", 2)
if len(parts) == 2 {
vid := strings.TrimSpace(parts[1])
if vid != "" {
vendors = append(vendors, vid)
}
}
}
}
if len(vendors) == 0 {
return false
}
// Update each vendor
updated := false
for _, vid := range vendors {
out, err := runCmd("whmapi1", "modsec_update_vendor", fmt.Sprintf("vendor_id=%s", vid))
if err == nil && out != nil && strings.Contains(string(out), "result: 1") {
fmt.Fprintf(os.Stderr, "[%s] WAF auto-update: vendor %s updated successfully\n",
time.Now().Format("2006-01-02 15:04:05"), vid)
updated = true
}
}
return updated
}
// CheckModSecAuditLog parses the ModSecurity audit log for blocked attacks.
// High-volume attackers are reported for potential auto-blocking.
func CheckModSecAuditLog(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
var findings []alert.Finding
logPaths := modsecAuditLogPaths()
if len(logPaths) == 0 {
return nil
}
var blocked map[string]int
var newest map[string]scanRecord
for _, path := range logPaths {
var nonempty bool
blocked, newest, nonempty = readModSecDenials(ctx, path, modsecAuditIsErrorLog(path, cfg))
if nonempty {
break
}
}
if len(blocked) == 0 {
return nil
}
// A trusted proxy's address stands for every visitor behind it.
for ip := range blocked {
if isInfraIP(ip, cfg.InfraIPs) || !wafAttackerIsReportable(ip) || IsTrustedProxy(ip, cfg) {
delete(blocked, ip)
}
}
// Alert on high-volume attackers (auto-block integration via check name)
for ip, count := range blocked {
if count >= 20 {
f := alert.Finding{
Severity: alert.High,
Check: "waf_attack_blocked",
SourceIP: ip,
Message: fmt.Sprintf("WAF blocking high-volume attacker: %s (%d blocked requests)", ip, count),
Details: wafBlockAdvice(ip, count),
// The audit record names the connecting client.
Attribution: admission.AttributionHTTPPeer,
}
newest[ip].stamp(&f, ProducerModSecAuditScan)
findings = append(findings, f)
}
}
return findings
}
// wafAttackerIsReportable reports whether a ModSecurity denial count belongs
// to an address worth telling the operator about.
//
// The control panel proxies its own traffic over loopback and, on cPanel,
// through the machine's public address rather than 127.0.0.1, so denials
// attributed to either accumulate on any busy host. Reporting those as a
// high-volume attacker and advising a permanent block points the operator at
// their own machine -- and the firewall's local-address guard excludes
// loopback, so the block is accepted rather than refused.
//
// A lookup failure fails open: a real attacker must never be suppressed by a
// transient syscall error. Loopback is decided without the lookup.
func wafAttackerIsReportable(ip string) bool {
parsed := net.ParseIP(ip)
if parsed == nil {
return false
}
if parsed.IsLoopback() || parsed.IsUnspecified() {
return false
}
return !netutil.IsHostAddress(ip)
}
// wafBlockAdvice is the operator guidance attached to a WAF attacker finding.
func wafBlockAdvice(ip string, count int) string {
details := fmt.Sprintf("IP %s has been blocked %d times by ModSecurity.", ip, count)
// The firewall refuses link-local blocks, so advising one sends the
// operator to an action that cannot succeed.
if parsed := net.ParseIP(ip); parsed.IsLinkLocalUnicast() || parsed.IsLinkLocalMulticast() {
return details + " Review the source of this link-local traffic; CSM does not block link-local addresses."
}
return details + " Consider permanent block via CSM."
}
// modsecAuditLogPaths yields the audit log candidates; a seam for tests.
var modsecAuditLogPaths = func() []string { return platform.Detect().ModSecAuditLogPaths }
// modsecAuditTailLines bounds the recent window used for denial counts.
// The serial format spends ten or more lines per transaction, so 200 lines
// could never hold the 20 denials the threshold asks for.
const modsecAuditTailLines = 4000
// modsecSectionBoundary parses a serial-format "--<id>-<letter>--" line.
func modsecSectionBoundary(line string) (id string, section byte, ok bool) {
line = strings.TrimSpace(line)
if len(line) < 7 || !strings.HasPrefix(line, "--") || !strings.HasSuffix(line, "--") {
return "", 0, false
}
body := line[2 : len(line)-2]
dash := strings.LastIndexByte(body, '-')
if dash <= 0 || dash != len(body)-2 {
return "", 0, false
}
section = body[dash+1]
if section < 'A' || section > 'Z' {
return "", 0, false
}
return body[:dash], section, true
}
// modsecAuditHeaderTime reads the time the server wrote at the start of an
// A-section header; zero when it has none.
func modsecAuditHeaderTime(line string) time.Time {
fields := strings.Fields(line)
if len(fields) != 7 || !strings.HasPrefix(fields[0], "[") || !strings.HasSuffix(fields[1], "]") {
return time.Time{}
}
for _, i := range []int{3, 5} {
address := fields[i]
if strings.HasPrefix(address, "[") && strings.HasSuffix(address, "]") {
address = address[1 : len(address)-1]
}
if net.ParseIP(address) == nil {
return time.Time{}
}
}
for _, i := range []int{4, 6} {
if _, err := strconv.ParseUint(fields[i], 10, 16); err != nil {
return time.Time{}
}
}
return modsecAuditTime(fields[0][1:] + " " + strings.TrimSuffix(fields[1], "]"))
}
// modsecAuditClientIP reads the client address from an A-section header:
// "[timestamp] uniqueid client-ip client-port server-ip server-port".
func modsecAuditClientIP(line string) string {
fields := strings.Fields(line)
if len(fields) < 5 || !strings.HasPrefix(fields[0], "[") {
return ""
}
clientIP := strings.Trim(fields[3], "[]")
ip := net.ParseIP(clientIP)
if ip == nil {
return ""
}
return ip.String()
}
package checks
import (
"bufio"
"context"
"encoding/json"
"io"
"maps"
"net"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/modsec"
"github.com/pidginhost/csm/internal/platform"
)
// A tail can consist entirely of client-controlled body text, including valid
// audit records. Only offset zero or a previously framed cursor is a safe start.
// Keep one reader for the selected platform path; rotation replays from zero.
var modsecAuditReader struct {
sync.Mutex
path string
info os.FileInfo
follow followState
parser modsecAuditParser
}
// The same text can be an error-log prefix or a client-controlled audit Host.
// Select the interpretation from the configured source, never from that text.
func modsecAuditIsErrorLog(path string, cfg *config.Config) bool {
path = filepath.Clean(path)
if cfg.ModSecErrorLog != "" && filepath.Clean(cfg.ModSecErrorLog) == path {
return true
}
for _, candidate := range platform.Detect().ErrorLogPaths {
if filepath.Clean(candidate) == path {
return true
}
}
return false
}
// readModSecDenials returns the recent denials of each address and the
// newest of them with a recorded time, named in the file's stream.
func readModSecDenials(ctx context.Context, path string, errorLog bool) (map[string]int, map[string]scanRecord, bool) {
f, err := osFS.Open(path)
if err != nil {
return nil, nil, false
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil || !info.Mode().IsRegular() || info.Size() == 0 {
return nil, nil, false
}
modsecAuditReader.Lock()
defer modsecAuditReader.Unlock()
now := time.Now()
r := &modsecAuditReader
start := int64(0)
if r.path == path && r.parser.errorLog == errorLog && r.info != nil && os.SameFile(r.info, info) && r.follow.Offset > 0 {
start, _, err = chooseStart(f, r.follow, info.Size())
if err != nil {
r.info = nil
return nil, nil, false
}
}
if start == 0 {
r.parser = modsecAuditParser{errorLog: errorLog}
}
// A partially advanced parser must never be paired with the old cursor.
r.info = nil
snapshot := followState{Offset: start}
if err = fillIdentity(f, &snapshot, info.Size()); err != nil {
return nil, nil, false
}
src := io.NewSectionReader(f, start, info.Size()-start)
br := bufio.NewReaderSize(src, 64*1024)
consumed := start
cancelled := false
for {
if ctx.Err() != nil {
// Large initial logs may need several scan budgets. Retain the
// framed prefix, but never report its stale counts as current.
cancelled = true
break
}
lineStart := consumed
line, truncated, readErr := readBoundedLineLog(br, maxLogLineBytes)
if readErr == io.EOF {
// Do not interpret an unfinished write. Resume before this line.
break
}
if readErr != nil {
return nil, nil, false
}
pos, seekErr := src.Seek(0, io.SeekCurrent)
if seekErr != nil {
return nil, nil, false
}
consumed = start + pos - int64(br.Buffered())
if truncated {
line = ""
}
r.parser.feed(syslogRecord{text: trimLogLineEnding(line), offset: lineStart}, truncated)
}
// A copytruncate during the read must not attach old framing to new bytes.
head, err := fpAt(f, 0, int64(snapshot.HeadLen))
if err != nil || head != snapshot.HeadFP {
return nil, nil, false
}
next := followState{Offset: consumed}
if err = fillIdentity(f, &next, info.Size()); err != nil {
return nil, nil, false
}
r.path, r.info, r.follow = path, info, next
if cancelled {
return nil, nil, info.Size() > 0
}
var stream string
if id, ok := selfWriteIdentityFromFileInfo(info); ok {
stream = alert.FileStream(id.Device, id.Inode)
}
eligible := r.parser.newestAt(now)
newest := make(map[string]scanRecord, len(eligible))
for ip, n := range eligible {
n.stream = stream
newest[ip] = n.scanRecord
}
return maps.Clone(r.parser.blocked), newest, info.Size() > 0
}
type modsecAuditParser struct {
errorLog bool
line, start int64
txID, txIP string
section byte
wantFirst bool
denied bool
// txRec is the open transaction's A-section header: the record its
// client was read from, with the time the server wrote there.
txRec scanRecord
// Each slot belongs to a physical line, including oversized body lines.
// Attribute a completed transaction to its A line so the recent window
// never counts a transaction whose start has already aged out.
recent [modsecAuditTailLines]modsecDenial
blocked map[string]int
}
type modsecDenial struct {
ip string
modsecNewest
}
type modsecNewest struct {
scanRecord
line int64
}
func (p *modsecAuditParser) feed(rec syslogRecord, truncated bool) {
if p.blocked == nil {
p.blocked = make(map[string]int)
}
line := rec.text
p.line++
slot := p.line % modsecAuditTailLines
if ip := p.recent[slot].ip; ip != "" {
p.blocked[ip]--
if p.blocked[ip] == 0 {
delete(p.blocked, ip)
}
p.recent[slot] = modsecDenial{}
}
if truncated {
p.wantFirst = false
return
}
if p.errorLog {
if ip, denied := modsecErrorLogDenial(line); denied {
at, _ := modsec.ErrorLogTime(line)
p.record(ip, p.line, scanned(rec, at))
}
return
}
id, section, boundary := modsecSectionBoundary(line)
if p.txID == "" {
if boundary {
if section == 'A' {
p.txID, p.txIP, p.txRec, p.section = id, "", scanRecord{}, 'A'
p.start, p.wantFirst, p.denied = p.line, true, false
}
return
}
if ip, at, denied := modsecLineDenial(line); denied {
p.record(ip, p.line, scanned(rec, at))
}
return
}
if boundary {
// Other IDs are body content. They cannot end this transaction,
// suppress its denial, or erase another client's counts.
if id != p.txID {
return
}
if section == 'Z' {
if p.denied && p.txIP != "" {
p.record(p.txIP, p.start, p.txRec)
}
p.txID = ""
return
}
p.section, p.wantFirst = section, true
return
}
if p.wantFirst && strings.TrimSpace(line) != "" {
p.wantFirst = false
switch p.section {
case 'A':
p.txIP = modsecAuditClientIP(line)
p.txRec = scanned(rec, modsecAuditHeaderTime(line))
case 'F':
fields := strings.Fields(line)
if len(fields) >= 2 && strings.HasPrefix(fields[0], "HTTP/") && fields[1] == "403" {
p.denied = true
}
}
}
if p.section == 'H' && (strings.HasPrefix(line, "Message: Access denied with code ") ||
strings.HasPrefix(line, "ModSecurity: Access denied with code ")) {
p.denied = true
}
}
func (p *modsecAuditParser) record(ip string, line int64, r scanRecord) {
if p.line-line < modsecAuditTailLines {
p.recent[line%modsecAuditTailLines] = modsecDenial{ip, modsecNewest{r, line}}
p.blocked[ip]++
}
}
// Eligibility is relative to this scan, not the first read. Keeping the
// record with its counted line makes cached reads agree with a full replay
// as the clock advances or moves back, without retaining request text.
func (p *modsecAuditParser) newestAt(now time.Time) map[string]modsecNewest {
newest := make(map[string]modsecNewest)
for i := range p.recent {
d := &p.recent[i]
if d.at.IsZero() || d.at.After(now) {
continue
}
if cur, ok := newest[d.ip]; !ok || d.line > cur.line {
newest[d.ip] = d.modsecNewest
}
}
return newest
}
const (
modsecIndexWord = `\S+`
modsecIndexQuoted = `"(?:\\.|[^"\\\r\n])*"`
modsecIndexDate = `[0-9]{2}/[A-Za-z]{3}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2}`
modsecIndexEnd = ` ` + modsecIndexWord + ` [0-9]+ [0-9]+(?:\.[0-9]+)? md5:[0-9a-fA-F]{32}$`
)
var (
// v2 writes a microsecond timestamp and two user columns; v3 writes
// whole seconds and one. Validate the entire grammar so a malformed Host
// cannot change dialect or fall through to an embedded [client] field.
modsecIndexV2 = regexp.MustCompile(`^` + modsecIndexWord + ` (` + modsecIndexWord + `) ` + modsecIndexWord + ` ` + modsecIndexWord +
` \[(` + modsecIndexDate + `\.[0-9]{6} [+-][0-9]{4})\] ` + modsecIndexQuoted + ` ([0-9]{3}) [0-9-]+ ` +
modsecIndexQuoted + ` ` + modsecIndexQuoted + ` ` + modsecIndexWord + ` ` + modsecIndexQuoted + modsecIndexEnd)
modsecIndexV3 = regexp.MustCompile(`^` + modsecIndexWord + ` (` + modsecIndexWord + `) ` + modsecIndexWord +
` \[(` + modsecIndexDate + ` [+-][0-9]{4})\] ` + modsecIndexQuoted + ` ([0-9]{3}) [0-9-]+ ` +
modsecIndexWord + ` ` + modsecIndexQuoted + ` ` + modsecIndexWord + ` ` + modsecIndexWord + modsecIndexEnd)
)
// modsecLineDenial reads a one-line audit record: the client, the time the
// server recorded and whether the request was denied. The time comes from
// its fixed place in the validated grammar, never from client text.
func modsecLineDenial(line string) (string, time.Time, bool) {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "{") {
var record struct {
Transaction struct {
ClientIP string `json:"client_ip"`
TimeStamp json.RawMessage `json:"time_stamp"`
Interrupted bool `json:"is_interrupted"`
Response struct {
Status int `json:"http_code"`
} `json:"response"`
} `json:"transaction"`
}
if json.Unmarshal([]byte(line), &record) != nil {
return "", time.Time{}, false
}
ip := net.ParseIP(record.Transaction.ClientIP)
if ip == nil {
return "", time.Time{}, false
}
// Unknown timestamp metadata must not discard a legacy denial.
var stamp string
var at time.Time
if json.Unmarshal(record.Transaction.TimeStamp, &stamp) == nil {
// libmodsecurity writes ctime in the server's local time.
at, _ = time.ParseInLocation("Mon Jan 2 15:04:05 2006", stamp, time.Local)
}
return ip.String(), at, record.Transaction.Response.Status == 403 || record.Transaction.Interrupted
}
for _, re := range []*regexp.Regexp{modsecIndexV2, modsecIndexV3} {
if fields := re.FindStringSubmatch(line); fields != nil {
ip := net.ParseIP(fields[1])
if ip == nil {
return "", time.Time{}, false
}
at := modsecAuditTime(fields[2])
return ip.String(), at, fields[3] == "403"
}
}
return "", time.Time{}, false
}
func modsecAuditTime(stamp string) time.Time {
at, err := time.Parse("02/Jan/2006:15:04:05 -0700", stamp)
if err != nil {
return time.Time{}
}
// Go accepts overflowing zone hours and minutes and normalizes them.
// Those are not server timestamps and cannot date admission evidence.
zone := stamp[len(stamp)-5:]
if zone[1:3] > "23" || zone[3:] > "59" {
return time.Time{}
}
return at
}
func modsecErrorLogDenial(line string) (string, bool) {
// An error-log line: the action opens the message after the module
// marker, and the client comes only from the server-written prefix.
if msg, _ := modsec.ErrorLogMessage(line); strings.HasPrefix(msg, "Access denied with code ") {
ip := modsec.ErrorLogClientIP(line)
return ip, ip != ""
}
return "", false
}
package checks
import (
"fmt"
"path/filepath"
"sort"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/platform"
)
// modsecDisabledScope is one place where ModSecurity is switched off for
// customer traffic. Domain is empty when the scope covers every vhost of
// the account.
type modsecDisabledScope struct {
User string
Domain string
Source string
}
// modsecDisabledScopes reports every vhost or account with ModSecurity
// turned off. A disabled scope voids every CSM virtual patch for that
// traffic and writes nothing to the modsec audit log, so an attack there
// is both unblocked and invisible.
//
// cPanel expresses "off" through several independent mechanisms and an
// audit of any one of them badly understates the exposure, so all of them
// are walked:
//
// - secruleengineoff in /var/cpanel/userdata (per domain)
// - modsec.conf under conf.d/userdata (per account, and per domain)
//
// The conf.d tree is mirrored into std and ssl. Because virtually all
// traffic is HTTPS, a scope present only under ssl still leaves the site
// unfiltered, so both trees carry equal weight here.
func modsecDisabledScopes(info platform.Info) []modsecDisabledScope {
if !info.IsCPanel() {
return nil
}
var scopes []modsecDisabledScope
scopes = append(scopes, userdataDisabledScopes()...)
scopes = append(scopes, confTreeDisabledScopes(info)...)
return dedupeScopes(scopes)
}
// modsecDisabledScopesForFindings reads only the cPanel paths that can affect
// the supplied vulnerable-plugin findings. The normal WAF audit already does
// the exhaustive /var/cpanel/userdata/*/* walk; repeating it for correlation
// makes a deep scan do thousands of redundant reads on a large shared host.
func modsecDisabledScopesForFindings(info platform.Info, findings []alert.Finding, aliases map[string][]string) []modsecDisabledScope {
if !info.IsCPanel() {
return nil
}
domainsByAccount := make(map[string]map[string]bool)
for _, finding := range findings {
account := strings.TrimSpace(finding.TenantID)
if !validAccountName.MatchString(account) {
continue
}
if domainsByAccount[account] == nil {
domainsByAccount[account] = make(map[string]bool)
}
domain := cleanDomlogDomain(finding.Domain)
if domain == "" {
continue
}
domainsByAccount[account][domain] = true
for _, peer := range aliases[aliasKey(account, domain)] {
if peer = cleanDomlogDomain(peer); peer != "" {
domainsByAccount[account][peer] = true
}
}
}
var scopes []modsecDisabledScope
for account, domains := range domainsByAccount {
scopes = append(scopes, targetedUserdataDisabledScopes(account, domains)...)
scopes = append(scopes, targetedConfTreeDisabledScopes(info, account, domains)...)
}
return dedupeScopes(scopes)
}
func targetedUserdataDisabledScopes(account string, domains map[string]bool) []modsecDisabledScope {
var scopes []modsecDisabledScope
for domain := range domains {
for _, name := range []string{domain, domain + "_SSL"} {
path := filepath.Join("/var/cpanel/userdata", account, name)
data, err := osFS.ReadFile(path)
if err != nil || !userdataSecRuleEngineOff(string(data)) {
continue
}
scopes = append(scopes, modsecDisabledScope{User: account, Domain: domain, Source: path})
}
}
return scopes
}
func targetedConfTreeDisabledScopes(info platform.Info, account string, domains map[string]bool) []modsecDisabledScope {
configDir := info.ApacheCompatibleConfigDir()
if configDir == "" {
return nil
}
var scopes []modsecDisabledScope
for _, base := range userdataTreeBases(configDir) {
accountPath := filepath.Join(base, account, "modsec.conf")
if confSecRuleEngineOff(accountPath) {
scopes = append(scopes, modsecDisabledScope{User: account, Source: accountPath})
}
for domain := range domains {
path := filepath.Join(base, account, domain, "modsec.conf")
if confSecRuleEngineOff(path) {
scopes = append(scopes, modsecDisabledScope{User: account, Domain: domain, Source: path})
}
}
}
return scopes
}
// userdataDisabledScopes walks the per-domain userdata flag. cPanel keeps
// <domain>, <domain>_SSL and <domain>.cache copies of the same record, so
// the domain name is normalised and duplicates collapse later.
func userdataDisabledScopes() []modsecDisabledScope {
var scopes []modsecDisabledScope
for _, path := range globPaths("/var/cpanel/userdata/*/*") {
domain, ok := cpanelUserdataDomain(filepath.Base(path))
if !ok {
continue
}
data, err := osFS.ReadFile(path)
if err != nil || !userdataSecRuleEngineOff(string(data)) {
continue
}
scopes = append(scopes, modsecDisabledScope{
User: filepath.Base(filepath.Dir(path)),
Domain: domain,
Source: path,
})
}
return scopes
}
// cpanelUserdataDomain accepts only per-vhost records. The same directory
// contains main/scope metadata and generated JSON/cache files; rejecting them
// by shape before ReadFile avoids bogus scopes and large cache reads.
func cpanelUserdataDomain(name string) (string, bool) {
if strings.HasSuffix(name, ".cache") || strings.HasSuffix(name, ".json") {
return "", false
}
domain := cleanDomlogDomain(strings.TrimSuffix(name, "_SSL"))
return domain, domain != ""
}
// confTreeDisabledScopes walks the Apache userdata include tree, where a
// modsec.conf directly under the account directory disables every vhost
// the account owns and one a level deeper disables a single domain.
func confTreeDisabledScopes(info platform.Info) []modsecDisabledScope {
configDir := info.ApacheCompatibleConfigDir()
if configDir == "" {
return nil
}
var scopes []modsecDisabledScope
for _, base := range userdataTreeBases(configDir) {
for _, path := range globPaths(filepath.Join(base, "*", "modsec.conf")) {
if !confSecRuleEngineOff(path) {
continue
}
scopes = append(scopes, modsecDisabledScope{
User: filepath.Base(filepath.Dir(path)),
Source: path,
})
}
for _, path := range globPaths(filepath.Join(base, "*", "*", "modsec.conf")) {
if !confSecRuleEngineOff(path) {
continue
}
domainDir := filepath.Dir(path)
scopes = append(scopes, modsecDisabledScope{
User: filepath.Base(filepath.Dir(domainDir)),
Domain: filepath.Base(domainDir),
Source: path,
})
}
}
return scopes
}
// userdataTreeBases returns the per-vhost include roots to walk.
//
// cPanel reports its config dir as /usr/local/apache/conf, and the
// userdata tree hangs directly off it. The distro-style path for the same
// directory is /etc/apache2/conf.d/userdata, so "conf.d" belongs to that
// spelling of the path, not under the cPanel config dir. Both layouts are
// walked because either spelling can be the one the platform layer
// reports; duplicate hits collapse during deduplication.
func userdataTreeBases(configDir string) []string {
var bases []string
for _, prefix := range [][]string{{"userdata"}, {"conf.d", "userdata"}} {
for _, tree := range []string{"std", "ssl"} {
parts := append(append([]string{configDir}, prefix...), tree, "2_4")
bases = append(bases, filepath.Join(parts...))
}
}
return bases
}
func confSecRuleEngineOff(path string) bool {
data, err := osFS.ReadFile(path)
if err != nil {
return false
}
return secRuleEngineOffDirective(string(data))
}
// dedupeScopes collapses the several files that can describe one scope
// into a single entry, keeping the lowest-sorting source so the reported
// path is stable across runs.
func dedupeScopes(scopes []modsecDisabledScope) []modsecDisabledScope {
sort.Slice(scopes, func(i, j int) bool {
if scopes[i].User != scopes[j].User {
return scopes[i].User < scopes[j].User
}
if scopes[i].Domain != scopes[j].Domain {
return scopes[i].Domain < scopes[j].Domain
}
return scopes[i].Source < scopes[j].Source
})
var out []modsecDisabledScope
seen := make(map[string]bool, len(scopes))
for _, s := range scopes {
key := s.User + "\x00" + s.Domain
if seen[key] {
continue
}
seen[key] = true
out = append(out, s)
}
return out
}
// globPaths expands a pattern, discarding the error: a malformed pattern
// is a programming bug and an unreadable directory simply yields nothing.
func globPaths(pattern string) []string {
matches, err := osFS.Glob(pattern)
if err != nil {
return nil
}
return matches
}
// userdataSecRuleEngineOff reports whether a cPanel userdata file carries
// the per-domain "ModSecurity off" flag. cPanel writes secruleengineoff: 0
// for the enabled state, so presence of the key is not enough.
func userdataSecRuleEngineOff(contents string) bool {
for _, line := range strings.Split(contents, "\n") {
key, value, found := strings.Cut(strings.TrimSpace(line), ":")
if !found || strings.TrimSpace(key) != "secruleengineoff" {
continue
}
if strings.TrimSpace(value) == "1" {
return true
}
}
return false
}
// secRuleEngineOffDirective reports whether an Apache config fragment
// turns the engine off. Files that only carry SecRuleRemoveById lines are
// legitimate false-positive workarounds and leave the engine running.
func secRuleEngineOffDirective(contents string) bool {
for _, line := range strings.Split(contents, "\n") {
fields := strings.Fields(line)
if len(fields) < 2 || strings.HasPrefix(fields[0], "#") {
continue
}
if strings.EqualFold(fields[0], "SecRuleEngine") && strings.EqualFold(fields[1], "Off") {
return true
}
}
return false
}
// modsecDisabledDetailCap bounds how many scopes are named in the finding
// details. The count in the message always reflects the true total.
const modsecDisabledDetailCap = 20
// modsecDisabledFindings reports disabled ModSecurity scopes as a single
// aggregated finding. Hosts routinely accumulate hundreds of these over
// the years, so one finding per scope would drown every other alert.
func modsecDisabledFindings(info platform.Info) []alert.Finding {
scopes := modsecDisabledScopes(info)
if len(scopes) == 0 {
return nil
}
var accounts, domains int
var b strings.Builder
for i, s := range scopes {
if s.Domain == "" {
accounts++
} else {
domains++
}
if i < modsecDisabledDetailCap {
target := s.Domain
if target == "" {
target = "account " + s.User + " (all domains)"
}
b.WriteString(" " + target + " -- " + s.Source + "\n")
}
}
if len(scopes) > modsecDisabledDetailCap {
fmt.Fprintf(&b, " ... and %d more\n", len(scopes)-modsecDisabledDetailCap)
}
return []alert.Finding{{
Severity: alert.High,
Check: "modsec_disabled_vhost",
Message: fmt.Sprintf("ModSecurity is disabled for %d scopes (%d account-wide, %d single-domain)",
len(scopes), accounts, domains),
Details: "Every CSM virtual patch is inert on this traffic and no modsec audit\n" +
"record is written, so attacks there are both unblocked and invisible.\n\n" +
b.String() +
"\nRe-enable per domain with:\n" +
" /usr/local/cpanel/bin/modsecuritydomains enable --domain=<domain>\n" +
"That tool only rewrites the userdata flag. Account-wide and per-domain\n" +
"modsec.conf files must be edited directly in BOTH the std and ssl\n" +
"userdata trees, then rebuild the web server config. Keep any\n" +
"SecRuleRemoveById lines already present -- those are rule exclusions,\n" +
"not an engine switch.",
}}
}
package checks
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"strings"
"sync"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/modsec"
"github.com/pidginhost/csm/internal/store"
)
// ErrModSecReloadNotConfigured reports a CSM rule section that differs from
// the last one the web server reloaded, on a host with no reload command.
var ErrModSecReloadNotConfigured = errors.New("modsec.reload_command is not set, so CSM cannot confirm its updated ModSecurity rules are active; they apply at the next web server reload")
// modsecActiveSectionKey holds the hash of the CSM section on disk at the last
// successful reload. Comparing against it, rather than reloading when a write
// happens, also activates sections written by the installer or an upgrade.
const modsecActiveSectionKey = "modsec:active_section_sha256"
var modsecReloadRunner = modsec.Reload
// vpDestPaths are the ModSecurity user configuration files CSM writes its
// section into, in order of preference.
var vpDestPaths = []string{
"/etc/apache2/conf.d/modsec/modsec2.user.conf",
"/usr/local/apache/conf/modsec2.user.conf",
}
// ModSecReloadReconciler is owned by the daemon and shared by startup and all
// its scans. A CLI opening the store must not implicitly gain reload authority.
type ModSecReloadReconciler struct {
mu sync.Mutex
db *store.DB
active string
pending bool // a successful reload whose metadata write needs retrying
}
type modsecReloadContextKey struct{}
// WithModSecReload lets daemon scans share their startup reconciler. Contexts
// without one can still deploy rules, but leave activation to the daemon.
func WithModSecReload(ctx context.Context, r *ModSecReloadReconciler) context.Context {
return context.WithValue(ctx, modsecReloadContextKey{}, r)
}
func deployAndReconcileModSec(ctx context.Context, command string) error {
r, _ := ctx.Value(modsecReloadContextKey{}).(*ModSecReloadReconciler)
if r == nil {
deployVirtualPatches()
return nil
}
// Serialize the write as well as the reload: a concurrent scan must not
// truncate the configuration while the web server is reading it.
r.mu.Lock()
defer r.mu.Unlock()
deployVirtualPatches()
if strings.TrimSpace(command) == "" {
return nil // only startup warns about an unset command
}
if err := ctx.Err(); err != nil {
return err
}
return r.reconcile(command)
}
// Reconcile activates sections already written by startup or the installer.
// Only failed reloads are repeated; metadata failures retry just the write.
func (r *ModSecReloadReconciler) Reconcile(reloadCommand string) error {
r.mu.Lock()
defer r.mu.Unlock()
return r.reconcile(reloadCommand)
}
func (r *ModSecReloadReconciler) reconcile(reloadCommand string) error {
db := store.Global()
if db == nil {
return nil
}
r.db = db
digest, err := installedVPSectionDigest()
if err != nil {
return err
}
if digest == "" {
return nil
}
if r.active == "" {
active, err := db.ReadMetaString(modsecActiveSectionKey)
if err != nil {
return fmt.Errorf("read active CSM ModSecurity rules: %w", err)
}
r.active = active
}
if r.active == digest {
return r.persistActive()
}
if strings.TrimSpace(reloadCommand) == "" {
return ErrModSecReloadNotConfigured
}
if _, err := modsecReloadRunner(reloadCommand); err != nil {
return fmt.Errorf("web server reload for CSM ModSecurity rules: %w", err)
}
r.active, r.pending = digest, true
csmlog.Info("web server reloaded to activate CSM ModSecurity rules", "command", reloadCommand)
return r.persistActive()
}
func (r *ModSecReloadReconciler) persistActive() error {
if !r.pending {
return nil
}
if err := r.db.SetMetaString(modsecActiveSectionKey, r.active); err != nil {
return fmt.Errorf("web server reloaded, but recording active CSM ModSecurity rules failed: %w", err)
}
r.pending = false
return nil
}
// Deployment can fall back when the preferred file cannot be written. Track
// every installed section so an older preferred copy cannot hide that update.
func installedVPSectionDigest() (string, error) {
var sums []byte
var readErr error
for _, dest := range vpDestPaths {
data, err := osFS.ReadFile(dest)
if err != nil {
if !os.IsNotExist(err) {
readErr = errors.Join(readErr, fmt.Errorf("read CSM ModSecurity rules in %s: %w", dest, err))
}
continue
}
begin, _, ok := markerLineBounds(data, vpBeginMarker)
if !ok {
continue
}
end := vpSectionEnd(data[begin:])
if end < 0 {
readErr = errors.Join(readErr, fmt.Errorf("unterminated CSM ModSecurity section in %s", dest))
continue
}
sum := sha256.Sum256(data[begin : begin+end])
sums = append(sums, sum[:]...)
}
if len(sums) == 0 {
return "", readErr
}
// The usual single section is identified by its own hash.
if len(sums) == sha256.Size {
return hex.EncodeToString(sums), nil
}
sum := sha256.Sum256(sums)
return hex.EncodeToString(sum[:]), nil
}
package checks
import (
"bufio"
"context"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"sync"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
const wpChecksumWorkers = 5 // concurrent wp core verify-checksums
// htaccessMaxLineBytes bounds a single .htaccess line for the token scanner.
// Legitimate directives are far shorter; a line past this is itself an
// anomaly, so the scanner fails closed (flags the file) rather than silently
// truncating the rest.
const htaccessMaxLineBytes = 1 << 20 // 1 MiB
// CheckHtaccess scans for malicious .htaccess directives using pure Go ReadDir.
func CheckHtaccess(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
// Scan each user's document roots
homeDirs := scanHomeDirsWithCoverage(ctx, "htaccess")
for _, homeEntry := range homeDirs {
if ctx.Err() != nil {
return findings
}
if !homeEntry.IsDir() {
continue
}
homeDir := scanHomeDirPath(homeEntry)
docRoot := filepath.Join(homeDir, "public_html")
scanHtaccess(ctx, docRoot, htaccessScanMaxDepth, htaccessSuspiciousPatterns, htaccessSafePatterns, cfg, &findings)
// Also check addon domains
subDirs, err := osFS.ReadDir(homeDir)
markScanReadError(ctx, "htaccess", err)
for _, sd := range subDirs {
if sd.IsDir() && sd.Name() != "public_html" && sd.Name() != "mail" &&
!strings.HasPrefix(sd.Name(), ".") && sd.Name() != "etc" &&
sd.Name() != "logs" && sd.Name() != "ssl" && sd.Name() != "tmp" {
scanHtaccess(ctx, filepath.Join(homeDir, sd.Name()), htaccessScanMaxDepth, htaccessSuspiciousPatterns, htaccessSafePatterns, cfg, &findings)
}
}
}
return findings
}
// htaccessScanMaxDepth is how deep below a document root the scheduled
// .htaccess scan walks. Five levels stopped short of every uploads tree
// (wp-content/uploads/YYYY/MM/<dir> is already five), which is where a
// dropper plants the handler-enabling .htaccess that makes its .jpg run.
// Matched to the rolling content scan's depth so both see the same tree.
const htaccessScanMaxDepth = rollingWalkMaxDepth
func scanHtaccess(ctx context.Context, dir string, maxDepth int, suspicious, safe []string, cfg *config.Config, findings *[]alert.Finding) {
if ctx.Err() != nil {
return
}
if maxDepth <= 0 {
return
}
entries, err := osFS.ReadDir(dir)
if err != nil {
markScanReadError(ctx, "htaccess", err)
return
}
for _, entry := range entries {
if ctx.Err() != nil {
return
}
name := entry.Name()
fullPath := filepath.Join(dir, name)
if entry.IsDir() {
scanHtaccess(ctx, fullPath, maxDepth-1, suspicious, safe, cfg, findings)
continue
}
if name != ".htaccess" {
continue
}
// Skip suppressed paths (bypassed for explicit full-scan / audit requests).
suppressed := false
if scanRespectsIgnores(ctx, cfg) {
for _, ignore := range cfg.Suppressions.IgnorePaths {
if matchGlob(fullPath, ignore) {
suppressed = true
break
}
}
}
if suppressed {
continue
}
checkHtaccessFile(ctx, fullPath, suspicious, safe, findings)
// Run the hardened detector registry alongside the generic
// token scanner so per-pattern findings emit with their own
// names (htaccess_php_in_uploads, htaccess_filesmatch_shield,
// etc.) rather than collapsing into the catch-all categories.
// The two scans can both fire on the same line; downstream
// dedup at alert.Dispatch handles same-key dups.
hardenedFindings, _, complete := auditHtaccessFile(fullPath)
if !complete {
markCheckIncomplete(ctx, "htaccess")
}
for _, finding := range hardenedFindings {
if finding.Check != "htaccess_injection" && finding.Check != "htaccess_handler_abuse" {
*findings = append(*findings, finding)
}
}
}
}
// phpExtension reports whether ext (leading dot, lowercase) is a stock
// PHP-executed extension that legitimately maps to a PHP handler.
func phpExtension(ext string) bool {
return isExecutablePHPName("x" + ext)
}
// phpHandlerRemapsNonPHP reports whether a single lowercased .htaccess line is
// an AddHandler/AddType/SetHandler directive routing a non-PHP file extension
// to a PHP execution handler. cPanel MultiPHP legitimately maps PHP extensions
// to a PHP handler (e.g. application/x-httpd-ea-php74___lsphp .php .php7), so we
// flag only when at least one mapped extension is not PHP-family.
func phpHandlerRemapsNonPHP(lineLower string) bool {
return phpHandlerRemapsNonPHPInContext(lineLower, nil)
}
func phpHandlerRemapsNonPHPInContext(lineLower string, contexts []phpHandlerOverlay) bool {
fields := apacheDirectiveFields(lineLower)
if len(fields) < 2 {
return false
}
directive := fields[0]
switch directive {
case "addhandler", "addtype", "sethandler", "forcetype":
default:
return false
}
handler := fields[1]
if !handlerIsPHP(handler) {
return false
}
exts := normalizedExts(fields[2:])
if len(exts) == 0 {
return directiveHandlerContextTargetsNonPHP(directive, contexts)
}
for _, ext := range exts {
if !phpExtension(ext) {
return true
}
}
return false
}
func directiveHandlerContextTargetsNonPHP(directive string, contexts []phpHandlerOverlay) bool {
if directive != "sethandler" && directive != "forcetype" {
return false
}
for _, ctx := range contexts {
if ctx.unrestricted {
// A name-selecting container hands the PHP handler files that
// no extension list describes.
return true
}
for ext := range ctx.exts {
if !phpExtension(ext) {
return true
}
}
for name := range ctx.names {
if !isExecutablePHPName(name) {
return true
}
}
}
return false
}
func checkHtaccessFile(ctx context.Context, path string, suspicious, safe []string, findings *[]alert.Finding) {
f, err := osFS.Open(path)
if err != nil {
markScanReadError(ctx, "htaccess", err)
return
}
defer func() { _ = f.Close() }()
// The per-line ceiling below bounds one token, not the file: a
// .htaccess of a million short lines still had to be held whole.
if info, statErr := f.Stat(); statErr == nil && info.Size() > htaccessMaxFileBytes {
*findings = append(*findings, alert.Finding{
Severity: alert.High,
Check: "htaccess_injection",
Message: fmt.Sprintf(".htaccess too large to audit: %s", path),
Details: fmt.Sprintf("File: %s\nSize: %d bytes exceeds the %d byte ceiling; a real .htaccess is a few kilobytes. Inspect it by hand.", path, info.Size(), htaccessMaxFileBytes),
FilePath: path,
})
return
}
// Read entire file to check cross-line context (e.g., AddHandler + Options -ExecCGI)
var lines []string
scanner := bufio.NewScanner(f)
// A .htaccess line longer than the default 64 KB token would make
// Scan stop early and silently drop every line after it, letting an
// attacker hide a malicious directive behind one padded line. Raise
// the ceiling, and if a line still exceeds it, fail closed below.
scanner.Buffer(make([]byte, 0, 64*1024), htaccessMaxLineBytes)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
if err := scanner.Err(); err != nil {
markCheckIncomplete(ctx, "htaccess")
// Could not read the whole file (oversized line or I/O error), so
// the cross-line and per-line analysis below is incomplete. Flag
// the file for review rather than reporting a clean partial scan.
*findings = append(*findings, alert.Finding{
Severity: alert.High,
Check: "htaccess_injection",
Message: "Unparseable .htaccess: oversized or unreadable line blocks full analysis",
Details: fmt.Sprintf("File: %s\nError: %v", path, err),
FilePath: path,
})
return
}
legacy, _ := auditHtaccessLegacyContent(path, []byte(strings.Join(lines, "\n")), suspicious, safe)
*findings = append(*findings, legacy...)
}
// CheckWPCore runs wp core verify-checksums for each WordPress installation
// using a bounded worker pool for concurrency.
// Installations that pass verification have their core files cached in
// GlobalCMSCache so the real-time scanner can skip signature matches
// on known-clean CMS files.
func CheckWPCore(ctx context.Context, cfg *config.Config, _ *state.Store) (findings []alert.Finding) {
if ctx == nil {
ctx = context.Background()
}
if incompleteCollectorFrom(ctx) == nil {
ctx, _ = withIncompleteCheckCollector(ctx)
}
wpConfigs := wpCoreScanRoots(ctx)
coverage := newWPVerificationBatch(ctx, store.Global(), "core", logicalOwnerWPCoreVerification, wpConfigs)
defer func() {
err := coverage.finish(ctx, !checkMarkedIncomplete(ctx, "wp_core"))
if _, disabled := disabledLogicalOwners(cfg)[logicalOwnerWPCoreVerification]; !disabled {
findings = append(findings, wpVerificationFindings(ctx, coverage.db, "core", logicalOwnerWPCoreVerification, err)...)
}
}()
if len(wpConfigs) == 0 {
return nil
}
cache := GlobalCMSCache()
var mu sync.Mutex
var wg sync.WaitGroup
var releases wpCoreReleaseChecksums
batch := wpCoreBatches.begin(len(wpConfigs), wpChecksumWorkers)
defer batch.abandon(ctx)
jobs := make(chan int, len(wpConfigs))
for i := 0; i < wpChecksumWorkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for index := range jobs {
wpConfig, work := wpConfigs[index], batch.tasks[index]
work.admit()
stop := false
work.run(ctx, cmdTimeout, func() {
if parentErr := ctx.Err(); parentErr != nil {
work.withdraw(parentErr)
stop = true
return
}
wpPath := filepath.Dir(wpConfig)
user := wpConfigUser(wpPath)
out, err := runCmdCombinedContext(ctx, "wp", "core", "verify-checksums",
"--path="+wpPath, "--allow-root")
work.progress()
if parentErr := ctx.Err(); parentErr != nil {
work.withdraw(parentErr)
stop = true
return
}
if err == nil {
coverage.record(wpPath, store.WPVerificationResult{State: "verified"})
// Verification passed - cache all core files
cacheWPCoreFiles(cache, wpPath)
return
}
coverage.record(wpPath, wpVerificationFailure(err, out))
// Partial integrity output cannot complete a command killed by a signal.
var commandExit *exec.ExitError
if errors.As(err, &commandExit) && commandExit.ExitCode() < 0 {
work.fail()
}
if out == nil {
work.fail()
return
}
outStr := string(out)
completed := wpCoreVerificationCompleted(err, out)
lineEndingOnly := make(map[string]bool)
onlyLineEndings := false
if completed {
lineEndingOnly, onlyLineEndings = releases.lineEndingOnly(wpPath, out)
}
var extraneous []string
reported := false
for _, line := range strings.Split(outStr, "\n") {
reported = reported || wpChecksumModifiedFilePath(line) != "" || strings.Contains(line, "should not exist")
if wpChecksumLineHasExtraneousCoreFile(line) {
extraneous = append(extraneous, strings.TrimSpace(line))
continue
}
// A shipped core file whose bytes changed is where backdoors
// are appended; that is worse than an extra file, and the
// path lets Re-check and the operator go straight to it.
if rel := wpChecksumModifiedCoreFile(line); rel != "" && !lineEndingOnly[rel] {
mu.Lock()
findings = append(findings, alert.Finding{
Severity: wpCoreModifiedSeverity(wpCoreFilePathWithin(wpPath, rel), rel),
Check: "wp_core_integrity",
Message: fmt.Sprintf("WordPress core file modified for %s", user),
Details: fmt.Sprintf("Path: %s\nFile: %s\n%s", wpPath, rel, line),
FilePath: wpCoreFilePathWithin(wpPath, rel),
})
mu.Unlock()
}
}
if len(extraneous) > 0 {
collapsed := wpCoreExtraneousFinding(user, wpPath, extraneous)
// No single file to name, so the install's owner carries
// the identity correlation needs.
if owner, ok := installOwner(wpConfig); ok {
collapsed.TenantID = owner
}
mu.Lock()
findings = append(findings, collapsed)
mu.Unlock()
}
if completed {
state := "modified"
if onlyLineEndings {
state = "verified"
}
coverage.record(wpPath, store.WPVerificationResult{State: state})
}
// wp-cli that ran and refused this tree answered the check.
if !reported && !commandRefused(err) {
work.fail()
}
})
if stop {
return
}
}
}()
}
for index := range wpConfigs {
if ctx.Err() != nil {
break
}
jobs <- index
}
close(jobs)
wg.Wait()
batch.abandon(ctx)
fmt.Fprintf(os.Stderr, "CMS hash cache: %d verified core files cached\n", cache.Size())
return findings
}
// wpCoreExtraneousSampleLimit bounds how many wp-cli lines the collapsed
// extra-file finding quotes. Enough to recognise the shape of the damage
// without turning one broken install into a wall of text.
const wpCoreExtraneousSampleLimit = 15
// wpCoreExtraneousFinding collapses every "should not exist" line wp-cli
// reported for one install into a single finding. The lines describe one
// condition -- this core is not the release it claims to be -- and a core
// rebuilt from an older release reports every file the newer one shipped, so
// emitting them per file buries the rest of the scan. Identity is pinned to
// the install so the row survives the operator deleting the files one by one.
func wpCoreExtraneousFinding(user, wpPath string, lines []string) alert.Finding {
sorted := append([]string(nil), lines...)
sort.Strings(sorted)
var details strings.Builder
fmt.Fprintf(&details, "Path: %s\n", wpPath)
fmt.Fprintf(&details, "Core files reported as extraneous: %d\n", len(sorted))
for _, line := range firstN(sorted, wpCoreExtraneousSampleLimit) {
details.WriteString(line)
details.WriteString("\n")
}
if extra := len(sorted) - wpCoreExtraneousSampleLimit; extra > 0 {
fmt.Fprintf(&details, "... and %d more\n", extra)
}
return alert.Finding{
Severity: alert.High,
Check: "wp_core_integrity",
Message: fmt.Sprintf("WordPress core integrity failure for %s", user),
Details: details.String(),
DedupKey: "extraneous:" + wpPath,
}
}
// cacheWPCoreFiles hashes all PHP files in wp-includes/ and wp-admin/
// for a verified-clean WordPress installation and adds them to the cache.
func cacheWPCoreFiles(cache *CMSHashCache, wpPath string) {
coreDirs := []string{
filepath.Join(wpPath, "wp-includes"),
filepath.Join(wpPath, "wp-admin"),
}
// Also cache root-level WP core files
rootFiles := []string{
"wp-cron.php", "wp-login.php", "wp-settings.php",
"wp-load.php", "wp-blog-header.php", "wp-links-opml.php",
"wp-mail.php", "wp-signup.php", "wp-activate.php",
"wp-comments-post.php", "wp-trackback.php", "xmlrpc.php",
"index.php",
}
for _, name := range rootFiles {
path := filepath.Join(wpPath, name)
if hash := HashFile(path); hash != "" {
if info, err := osFS.Stat(path); err == nil {
cache.Add(hash, info.Size())
}
}
}
for _, dir := range coreDirs {
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
name := strings.ToLower(info.Name())
if strings.HasSuffix(name, ".php") || strings.HasSuffix(name, ".js") {
if hash := HashFile(path); hash != "" {
cache.Add(hash, info.Size())
}
}
return nil
})
}
}
func extractUser(path string) string {
parts := strings.Split(path, "/")
for i, p := range parts {
if p == "home" && i+1 < len(parts) {
return parts[i+1]
}
}
return "unknown"
}
// wpCoreScanRoots lists the WordPress installs to verify. Discovery is shared
// (wpinstalls.go): core files are tampered with in subdomain and nested
// installs as readily as in a primary document root.
func wpCoreScanRoots(ctx context.Context) []string {
installs := wpInstalls(ctx, "wp_core")
out := make([]string, 0, len(installs))
for _, in := range installs {
out = append(out, in.ConfigPath)
}
return out
}
package checks
import (
"path/filepath"
"strings"
)
// Web servers use shell glob syntax, whereas account root patterns use Go's
// filepath syntax. Keep their matching rules separate while checking the same
// directory read errors.
func checkedWebConfigGlob(pattern string) ([]string, bool) {
return checkedPathGlob(pattern, 0, pathGlobSyntax{matchWebConfigGlob, joinWebConfigPath, pathGlobHasMeta})
}
func checkedApacheConfigGlob(pattern string) ([]string, bool) {
return checkedPathGlob(pattern, 0, pathGlobSyntax{matchWebConfigGlob, filepath.Join, apacheIncludeHasGlob})
}
// apr_fnmatch_test lets Apache open literal components without unescaping
// them, even within an include pattern that has wildcards elsewhere.
func apacheIncludeHasGlob(pattern string) bool {
bracket := false
for i := 0; i < len(pattern); i++ {
switch pattern[i] {
case '\\':
i++
case '*', '?':
return true
case '[':
bracket = true
case ']':
if bracket {
return true
}
}
}
return false
}
// nginx leaves symlinks followed by ".." to the filesystem. filepath.Join
// would silently switch to a different file by cleaning those components.
func joinWebConfigPath(parts ...string) string {
return strings.Join(parts, string(filepath.Separator))
}
func matchWebConfigGlob(pattern, name string) (bool, error) {
goPattern, err := webConfigGlobPattern(pattern)
if err != nil {
return false, err
}
// Both httpd's APR_FNM_PERIOD and nginx's glob require an explicit dot.
if strings.HasPrefix(name, ".") && !strings.HasPrefix(pattern, ".") && !strings.HasPrefix(pattern, `\.`) {
return false, nil
}
return filepath.Match(goPattern, name)
}
// Translate the shared shell bracket syntax. Named, collating and equivalence
// classes differ between APR and libc and can depend on the server's locale;
// leave those trees incomplete instead of claiming they matched no files.
func webConfigGlobPattern(pattern string) (string, error) {
var out strings.Builder
for i := 0; i < len(pattern); i++ {
switch pattern[i] {
case '\\':
if i+1 == len(pattern) {
return "", filepath.ErrBadPattern
}
out.WriteString(pattern[i : i+2])
i++
case '[':
out.WriteByte('[')
i++
if i < len(pattern) && (pattern[i] == '!' || pattern[i] == '^') {
out.WriteByte('^')
i++
}
first := true
for ; i < len(pattern); i++ {
c := pattern[i]
if c == ']' && !first {
break
}
if c == '[' && i+1 < len(pattern) && strings.ContainsRune(":.=", rune(pattern[i+1])) {
return "", filepath.ErrBadPattern
}
if c == '\\' {
if i+1 == len(pattern) {
return "", filepath.ErrBadPattern
}
out.WriteByte(c)
i++
} else if c == ']' || c == '-' && (first || i+1 < len(pattern) && pattern[i+1] == ']') {
out.WriteByte('\\')
}
out.WriteByte(pattern[i])
first = false
}
if i == len(pattern) {
return "", filepath.ErrBadPattern
}
out.WriteByte(']')
default:
out.WriteByte(pattern[i])
}
}
return out.String(), nil
}
package checks
import (
"errors"
"os"
"path/filepath"
"strings"
"syscall"
)
// filepath.Glob hides filesystem errors. Expand roots with an explicit
// completeness result so inaccessible roots cannot look like removed ones.
func checkedWebRootGlob(pattern string) ([]string, bool) {
return checkedPathGlob(pattern, 0, pathGlobSyntax{filepath.Match, filepath.Join, pathGlobHasMeta})
}
type pathGlobSyntax struct {
match func(string, string) (bool, error)
join func(...string) string
hasMeta func(string) bool
}
func pathGlobHasMeta(pattern string) bool { return strings.ContainsAny(pattern, `*?[\`) }
func checkedPathGlob(pattern string, depth int, syntax pathGlobSyntax) ([]string, bool) {
// Keep filepath.Glob's bound against stack exhaustion from configuration.
const pathSeparatorsLimit = 10000
if depth == pathSeparatorsLimit {
return nil, false
}
if !syntax.hasMeta(pattern) {
if _, err := osFS.Lstat(pattern); err != nil {
return nil, webRootAbsent(err)
}
return []string{pattern}, true
}
if _, err := syntax.match(pattern, ""); err != nil {
return nil, false
}
dir, leaf := filepath.Split(pattern)
if dir == "" {
dir = "."
} else if dir != string(filepath.Separator) {
dir = strings.TrimSuffix(dir, string(filepath.Separator))
}
parents, complete := checkedPathGlob(dir, depth+1, syntax)
var matches []string
for _, parent := range parents {
info, err := osFS.Stat(parent)
if err != nil {
complete = complete && webRootAbsent(err)
continue
}
if !info.IsDir() {
continue
}
entries, err := osFS.ReadDir(parent)
if err != nil {
complete = complete && webRootAbsent(err)
}
for _, entry := range entries {
matched, err := syntax.match(leaf, entry.Name())
if err != nil {
return nil, false
}
if matched {
matches = append(matches, syntax.join(parent, entry.Name()))
}
}
}
return matches, complete
}
func webRootAbsent(err error) bool {
return os.IsNotExist(err) || errors.Is(err, syscall.ENOTDIR)
}
package checks
import (
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/contenttype"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/wpcheck"
)
const (
// webshellCallerLookback bounds how old a request may be. A quiet site's
// log tail holds lines for days, and they must not keep a caller reported.
webshellCallerLookback = 24 * time.Hour
// webshellPublicClients is the number of distinct clients from which a
// flagged file is read as a public page or an embedded URL rather than an
// operated shell. Its clients are then not reported one by one.
webshellPublicClients = 10
)
// webshellCallers collects, during one domlog pass, the clients that request
// a flagged file.
type webshellCallers struct {
// docroots maps a vhost to its document root.
docroots map[string]string
flagged map[string]bool
// paths holds the hits per flagged file, then per client.
paths map[string]*webshellPathHits
}
type webshellPathHits struct {
clients map[webshellRequestSource]*webshellHits
}
type webshellRequestSource struct {
client, domain, account string
}
type webshellHits struct {
reportable bool
requests, posts int
newest time.Time
sample string
}
// loadWebshellCallers builds the collector for one domlog pass. It is nil,
// and costs the pass nothing, when no file is flagged or the host has no
// per-domain document root map.
func loadWebshellCallers(store *state.Store) *webshellCallers {
if store == nil || !platform.Detect().IsCPanel() {
return nil
}
var latest []alert.Finding
for _, f := range store.LatestFindings() {
if f.Severity == alert.Critical && malwareFileCheck(f.Check) {
latest = append(latest, f)
}
}
removed := RemovedFiles()
if len(latest) == 0 && len(removed) == 0 {
return nil
}
data, err := osFS.ReadFile(userdataDomainsPath)
if err != nil {
return nil
}
vhosts, _ := parseUserdataDomainRootsChecked(string(data))
docroots := make(map[string]string, len(vhosts))
roots := make([]string, 0, len(vhosts))
for _, vh := range vhosts {
docroots[vh.domain] = vh.docroot
roots = append(roots, vh.docroot)
}
flagged := webshellFlaggedFiles(latest, removed, roots)
if flagged == nil {
return nil
}
return &webshellCallers{docroots: docroots, flagged: flagged}
}
// File attribution needs the whole target. Other domlog consumers keep their
// existing retention limit even when this collector shares their scan.
func parseWebshellCallerRecord(line string) (accessLogRecord, bool) {
rec, ok := parseAccessLogRecordWithURILimit(line, len(line))
if len(rec.URI) > accessLogURILimit {
rec.fullURI = rec.URI
rec.URI = rec.URI[:accessLogURILimit]
}
return rec, ok
}
// record counts rec for client when it requests a flagged file.
func (w *webshellCallers) record(rec accessLogRecord, client string, now time.Time, reportable bool) {
if w == nil || rec.Domain == "" || rec.Time.IsZero() || now.Sub(rec.Time) > webshellCallerLookback {
return
}
docroot, ok := w.docroots[strings.ToLower(rec.Domain)]
if !ok {
return
}
uri := rec.URI
if rec.fullURI != "" {
uri = rec.fullURI
}
file, ok := requestedFile(docroot, uri, w.flagged)
if !ok {
return
}
if w.paths == nil {
w.paths = make(map[string]*webshellPathHits)
}
hits := w.paths[file]
if hits == nil {
hits = &webshellPathHits{clients: make(map[webshellRequestSource]*webshellHits)}
w.paths[file] = hits
}
source := webshellRequestSource{client: client, domain: strings.ToLower(rec.Domain), account: rec.Account}
c := hits.clients[source]
if c == nil {
c = &webshellHits{}
hits.clients[source] = c
}
c.reportable = c.reportable || reportable
c.requests++
if rec.Method == "POST" {
c.posts++
}
if !rec.Time.Before(c.newest) {
c.newest = rec.Time
c.sample = rec.Method + " " + truncate(rec.URI, 200) + " " + itoa(rec.Status)
}
}
// webshellCaller is one client's hits on one account.
type webshellCaller struct {
client, account string
domains map[string]bool
files []string
webshellHits
}
// emit returns one finding per client and account, and one per flagged file
// that too many clients request to be an operated shell.
func (w *webshellCallers) emit() []alert.Finding {
if w == nil {
return nil
}
files := make([]string, 0, len(w.paths))
for file := range w.paths {
files = append(files, file)
}
sort.Strings(files)
var out []alert.Finding
callers := make(map[string]*webshellCaller)
var keys []string
for _, file := range files {
hits := w.paths[file]
clients := make(map[string]bool)
domains := make(map[string]bool)
accounts := make(map[string]bool)
for source := range hits.clients {
clients[source.client] = true
domains[source.domain] = true
accounts[source.account] = true
}
if len(clients) >= webshellPublicClients {
domain, account := "", ""
if len(domains) == 1 {
for d := range domains {
domain = d
}
}
if len(accounts) == 1 {
for a := range accounts {
account = a
}
}
out = append(out, alert.Finding{
Severity: alert.Warning,
Check: "webshell_path_public",
TenantID: account,
Domain: domain,
DedupKey: file,
Message: "Flagged PHP file requested by many clients: " + file,
Details: itoa(len(clients)) + " clients requested it in the last 24 hours. The address is public or embedded in a page, " +
"so its clients are not reported one by one.",
})
continue
}
label := file
if w.flagged[file] {
label += " (removed)"
}
for source, c := range hits.clients {
if !c.reportable {
continue
}
scope := source.account
if scope == "" {
scope = source.domain
}
key := source.client + "|" + scope
caller := callers[key]
if caller == nil {
caller = &webshellCaller{client: source.client, account: source.account, domains: make(map[string]bool)}
callers[key] = caller
keys = append(keys, key)
}
caller.domains[source.domain] = true
if len(caller.files) == 0 || caller.files[len(caller.files)-1] != label {
caller.files = append(caller.files, label)
}
caller.requests += c.requests
caller.posts += c.posts
if c.newest.After(caller.newest) || (c.newest.Equal(caller.newest) && c.sample < caller.sample) {
caller.newest, caller.sample = c.newest, c.sample
}
}
}
sort.Strings(keys)
for _, key := range keys {
c := callers[key]
domain := ""
if len(c.domains) == 1 {
for d := range c.domains {
domain = d
}
}
scope := c.account
if scope == "" {
scope = domain
}
requests := itoa(c.requests) + " requests"
if c.requests == 1 {
requests = "1 request"
}
shown := c.files
more := ""
if len(shown) > 3 {
more = "\nand " + itoa(len(shown)-3) + " more"
shown = shown[:3]
}
out = append(out, alert.Finding{
Severity: alert.High,
Check: "webshell_caller",
SourceIP: c.client,
TenantID: c.account,
Domain: domain,
DedupKey: key,
Message: "Requests to a flagged PHP file from " + c.client + " on " + scope,
Details: requests + " (" + itoa(c.posts) + " POST), newest " + c.newest.UTC().Format("2006-01-02 15:04:05 MST") +
"\nFiles:\n" + strings.Join(shown, "\n") + more +
"\nSample: " + c.sample,
})
}
return out
}
// webshellFlaggedFiles returns the files whose requests name a webshell
// caller, keyed by path; the value is true when the path is flagged because
// CSM removed the file. A Critical content finding alone is not enough: it
// also covers a site's own pages with injected code, and their visitors are
// not callers. Nil when nothing is flagged.
func webshellFlaggedFiles(latest []alert.Finding, removed []RemovedFile, docroots []string) map[string]bool {
roots := NewDocrootIndex(docroots)
var flagged map[string]bool
add := func(path string, removal bool) {
if !filepath.IsAbs(path) {
return
}
path = filepath.Clean(path)
if _, ok := flagged[path]; ok || !webshellCallerTarget(path, roots) {
return
}
if flagged == nil {
flagged = make(map[string]bool)
}
flagged[path] = removal
}
for _, f := range latest {
if f.Severity == alert.Critical && malwareFileCheck(f.Check) {
add(f.FilePath, false)
}
}
for _, r := range removed {
// A cleaned file is the site's own and still served. A quarantined
// path that holds a file again counts only through a new finding:
// the site may have been rebuilt.
if r.Action != "quarantine" {
continue
}
if _, err := osFS.Lstat(r.Path); os.IsNotExist(err) {
add(r.Path, true)
}
}
return flagged
}
// webshellCallerTarget excludes known entry points from caller attribution.
// Other flagged PHP files can still be public or embedded in a page.
func webshellCallerTarget(path string, roots *DocrootIndex) bool {
if !contenttype.IsExecutablePHPName(strings.ToLower(filepath.Base(path))) {
return false
}
root := roots.Lookup(path)
if root == "" || path == filepath.Join(root, "index.php") {
return false
}
if shippedByRelease(path) {
return false
}
// An entry file name proves nothing by itself: it is an entry point only
// at the root of an install.
dir := filepath.Dir(path)
if wpcheck.DetectWPRoot(path) == dir {
if _, err := osFS.Stat(filepath.Join(dir, "wp-includes", "version.php")); err == nil {
return false
}
}
return true
}
// requestedFile returns the flagged file a request for uri on the site at
// docroot runs. The request path is compared as the web server resolves it:
// decoded once, without its query, with a script's trailing path info cut and
// a directory answered by its index.
func requestedFile(docroot, uri string, flagged map[string]bool) (string, bool) {
if len(flagged) == 0 || !strings.HasPrefix(uri, "/") {
return "", false
}
if i := strings.IndexAny(uri, "?#"); i >= 0 {
uri = uri[:i]
}
// An encoded slash is decoded too: LiteSpeed runs the file that path names.
decoded, err := url.PathUnescape(uri)
if err != nil || strings.IndexByte(decoded, 0) >= 0 {
return "", false
}
parts := strings.Split(decoded, "/")
clean := make([]string, 0, len(parts))
for _, part := range parts {
switch part {
case "", ".":
case "..":
// Servers reject traversal above the URL root instead of clamping it.
if len(clean) == 0 {
return "", false
}
clean = clean[:len(clean)-1]
default:
clean = append(clean, part)
}
}
if len(clean) == 0 {
return "", false
}
file := docroot + "/" + strings.Join(clean, "/")
end := len(docroot)
for _, part := range clean {
end += 1 + len(part)
// A directory can carry a script's name, so every script-named
// prefix is a candidate, not only the first.
if contenttype.IsExecutablePHPName(strings.ToLower(part)) {
candidate := file[:end]
if _, ok := flagged[candidate]; ok {
return candidate, true
}
}
}
last := parts[len(parts)-1]
if last == "" || last == "." || last == ".." {
file += "/index.php"
if _, ok := flagged[file]; ok {
return file, true
}
}
return "", false
}
package checks
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"net/netip"
"os"
"strings"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
)
// CheckWHMAccess parses the cPanel access log for WHM (port 2087) logins
// and password change API calls from non-infra IPs.
// Only reads the tail of the log - lightweight.
func CheckWHMAccess(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
var findings []alert.Finding
lines := tailFile("/usr/local/cpanel/logs/access_log", 200)
for _, line := range lines {
rec, ok := ParseCpanelAccessRecord(line)
if !ok || rec.Port != "2087" {
continue
}
ip := rec.IP
if !rec.Proxy && (isInfraIP(ip, cfg.InfraIPs) || ip == "127.0.0.1") {
continue
}
source, dedup := "non-infra IP: "+ip, ""
if rec.Proxy {
source, dedup = "a service subdomain (client address unverified)", "proxy:"+ip
}
// Check for password change actions
passwordActions := []string{
"passwd", "change_root_password", "chpasswd",
"force_password_change", "resetpass",
}
// The request names the action; referer and user agent are client text.
if rec.TargetContains(passwordActions...) {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "whm_password_change",
Message: fmt.Sprintf("WHM password change from %s", source),
Details: truncateString(line, 200),
DedupKey: dedup,
})
}
// Check for account management from unknown IPs
accountActions := []string{
"createacct", "killacct", "suspendacct", "unsuspendacct",
}
if rec.TargetContains(accountActions...) {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "whm_account_action",
Message: fmt.Sprintf("WHM account action from %s", source),
Details: truncateString(line, 200),
DedupKey: dedup,
})
}
}
return findings
}
var authLogPath = func() string { return platform.Detect().AuthLogPath() }
// CheckSSHLogins parses the platform authentication log for SSH logins from
// non-infra IPs. With a state store it reads the log forward-only from where
// the previous cycle stopped; without one it falls back to a per-cycle tail.
func CheckSSHLogins(ctx context.Context, cfg *config.Config, store *state.Store) []alert.Finding {
if cfg == nil {
cfg = &config.Config{}
}
if store != nil {
return checkSSHLoginsFollow(cfg, store)
}
var findings []alert.Finding
for _, line := range tailFile(authLogPath(), 100) {
if !strings.Contains(line, "Accepted") {
continue
}
if f, ok := SSHAcceptedLoginFinding(line, cfg); ok {
findings = append(findings, f)
}
}
return findings
}
// SSHAcceptedLoginFinding parses an sshd "Accepted <method> for <user> from
// <ip> port <n>" line and reports it unless the address is infrastructure.
// The daemon's realtime log watcher calls it so a login seen live and the same
// line re-read by CheckSSHLogins carry one identity; without that the state
// store sees two findings and the operator gets one login reported twice.
func SSHAcceptedLoginFinding(line string, cfg *config.Config) (alert.Finding, bool) {
if cfg == nil {
cfg = &config.Config{}
}
if !strings.Contains(line, "Accepted") {
return alert.Finding{}, false
}
user, ip, ok := sshAcceptedRecord(strings.Fields(line))
if !ok || isInfraIP(ip, cfg.InfraIPs) || ip == "127.0.0.1" {
return alert.Finding{}, false
}
// sshd authenticated the name, but only a hosting account is a tenant
// or names an owner; root and service users are neither.
tenant := HostingAccountForUser(user)
f := alert.Finding{
Severity: alert.Critical,
Check: "ssh_login_unknown_ip",
DedupKey: loginRecordKey(line),
Message: fmt.Sprintf("SSH login from non-infra IP: %s (user: %s)", ip, user),
Details: truncateString(line, 200),
SourceIP: ip,
TenantID: tenant,
// sshd's "from <ip> port" field is the transport peer.
Attribution: admission.AttributionPeer,
}
if tenant != "" {
f.Claims = []admission.Claim{{Kind: admission.ClaimAccount, Value: tenant}}
}
return f, true
}
// sshAcceptedRecord reads sshd's own success record, "Accepted <method> for
// <user> from <address> port <port>", right after the syslog header and the
// sshd program token. sshd also logs the login name a client offers, so these
// words count only at these positions; found anywhere else in a line they
// would let any client name any address.
func sshAcceptedRecord(fields []string) (user, ip string, ok bool) {
program := -1
switch {
case len(fields) > 0 && isSSHDProgramToken(fields[0]):
program = 0
case len(fields) >= 2 && isSSHDProgramToken(fields[1]):
// A host name without a timestamp.
program = 1
case len(fields) >= 5 && isSyslogTimestampPrefix(fields) && isSSHDProgramToken(fields[4]):
program = 4
case len(fields) >= 3 && isSSHDProgramToken(fields[2]):
if _, err := time.Parse(time.RFC3339Nano, fields[0]); err == nil {
program = 2
}
}
if program < 0 {
return "", "", false
}
m := fields[program+1:]
if len(m) < 8 || m[0] != "Accepted" || m[2] != "for" || m[4] != "from" || m[6] != "port" ||
m[7] == "" || strings.Trim(m[7], "0123456789") != "" {
return "", "", false
}
if _, err := netip.ParseAddr(m[5]); err != nil {
return "", "", false
}
return m[3], m[5], true
}
func isSSHDProgramToken(field string) bool {
for _, name := range []string{"sshd", "sshd-session"} {
if field == name+":" {
return true
}
pid, ok := strings.CutPrefix(field, name+"[")
if !ok {
continue
}
pid, ok = strings.CutSuffix(pid, "]:")
if ok && pid != "" && strings.Trim(pid, "0123456789") == "" {
return true
}
}
return false
}
// tailFile reads the last N lines of a file efficiently.
func tailFile(path string, maxLines int) []string {
records := tailFileRecords(path, maxLines)
if records == nil {
return nil
}
lines := make([]string, len(records))
for i, r := range records {
lines[i] = r.text
}
return lines
}
// tailFileRecords reads the last N lines of a file with the offset each
// starts at and the file's stream, so a scan that re-reads the window names
// a record the way every reader of the file names it.
func tailFileRecords(path string, maxLines int) []syslogRecord {
if maxLines <= 0 {
return nil
}
f, err := osFS.Open(path)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
// Seek to end and read backwards to find last N lines
info, err := f.Stat()
if err != nil {
return nil
}
var records []syslogRecord
if info.Size() < 1024*1024 {
// For small files, just read all
records = readAllRecords(f, 0, maxLines)
} else if data, start, err := readTailWindow(f, info.Size(), maxLines, maxTailWindowBytes); err != nil {
records = readAllRecords(f, 0, maxLines)
} else {
records = readAllRecords(bytes.NewReader(data), start, maxLines)
}
if id, ok := selfWriteIdentityFromFileInfo(info); ok {
stream := alert.FileStream(id.Device, id.Inode)
for i := range records {
if !records[i].unterminated {
records[i].stream = stream
}
}
}
return records
}
// readTailWindow returns the complete lines of the last maxBytes of the file
// and the offset they start at. A short read cannot be placed in the file,
// so it is an error.
func readTailWindow(f *os.File, size int64, maxLines int, maxBytes int64) ([]byte, int64, error) {
const chunkSize int64 = 256 * 1024
if maxBytes <= 0 {
return nil, size, nil
}
offset := size
newlines := 0
var totalRead int64
chunks := make([][]byte, 0, 4)
for offset > 0 && newlines <= maxLines && totalRead < maxBytes {
n := chunkSize
if offset < n {
n = offset
}
if remaining := maxBytes - totalRead; remaining < n {
n = remaining
}
offset -= n
chunk := make([]byte, n)
read, err := f.ReadAt(chunk, offset)
if err != nil && !errors.Is(err, io.EOF) {
return nil, 0, err
}
if int64(read) != n {
return nil, 0, io.ErrUnexpectedEOF
}
totalRead += int64(read)
newlines += bytes.Count(chunk, []byte{'\n'})
chunks = append(chunks, chunk)
}
total := 0
for _, chunk := range chunks {
total += len(chunk)
}
data := make([]byte, 0, total)
for i := len(chunks) - 1; i >= 0; i-- {
data = append(data, chunks[i]...)
}
if offset > 0 {
if firstNewline := bytes.IndexByte(data, '\n'); firstNewline >= 0 {
return data[firstNewline+1:], offset + int64(firstNewline) + 1, nil
}
return nil, size, nil
}
return data, offset, nil
}
const (
// maxLogLineBytes is the per-line cap for periodic log tailers.
// Oversized records are skipped after the reader advances past the
// terminator so a crafted long line cannot poison the next record.
maxLogLineBytes = 256 * 1024
// maxTailWindowBytes bounds the backward seek window before line
// parsing starts. Without this, a huge unterminated final record makes
// the tail reader cache the whole file while looking for maxLines.
maxTailWindowBytes int64 = 32 * 1024 * 1024
)
func readAllLines(r io.Reader, maxLines int) []string {
records := readAllRecords(r, 0, maxLines)
if records == nil {
return nil
}
lines := make([]string, len(records))
for i, rec := range records {
lines[i] = rec.text
}
return lines
}
// countingReader counts the bytes read through it.
type countingReader struct {
r io.Reader
n int64
}
func (c *countingReader) Read(p []byte) (int, error) {
n, err := c.r.Read(p)
c.n += int64(n)
return n, err
}
// readAllRecords reads the last maxLines lines of r, which starts at offset
// base of its file, with the offset each line starts at. Oversized lines are
// skipped; their bytes still count toward the next line's offset.
func readAllRecords(r io.Reader, base int64, maxLines int) []syslogRecord {
if maxLines <= 0 {
return nil
}
counter := &countingReader{r: r}
br := bufio.NewReaderSize(counter, 64*1024)
var records []syslogRecord
for {
start := base + counter.n - int64(br.Buffered())
line, truncated, err := readBoundedLineLog(br, maxLogLineBytes)
if len(line) > 0 && !truncated {
records = append(records, syslogRecord{text: trimLogLineEnding(line), offset: start, unterminated: err != nil})
}
if err != nil {
break
}
}
if len(records) > maxLines {
return records[len(records)-maxLines:]
}
return records
}
func trimLogLineEnding(line string) string {
line = strings.TrimSuffix(line, "\n")
return strings.TrimSuffix(line, "\r")
}
// readBoundedLineLog reads up to and including the next '\n'. If the
// line exceeds maxBytes the returned data is truncated to maxBytes and
// the reader is advanced past the line's terminating newline so framing
// stays intact. Returns the same error semantics as
// bufio.Reader.ReadString.
func readBoundedLineLog(r *bufio.Reader, maxBytes int) (string, bool, error) {
var b strings.Builder
truncated := false
for {
chunk, err := r.ReadSlice('\n')
if len(chunk) > 0 {
switch {
case truncated:
// drain remainder so the next line is well-framed
case b.Len()+len(chunk) <= maxBytes:
b.Write(chunk)
default:
if room := maxBytes - b.Len(); room > 0 {
b.Write(chunk[:room])
}
truncated = true
}
}
if errors.Is(err, bufio.ErrBufferFull) {
continue
}
return b.String(), truncated, err
}
}
func truncateString(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
}
package checks
import (
"bytes"
"crypto/md5" // #nosec G501 -- wordpress.org publishes MD5 digests for core files
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"unicode"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/safepath"
"github.com/pidginhost/csm/internal/wpcheck"
)
// An FTP client in ASCII mode rewrites the line endings of an uploaded core
// file: LF becomes CRLF, or the reverse on the way back. wp-cli then reports
// the file as modified although its code is the release's code. Only that one
// rewrite is undone before comparing; any other difference, or anything that
// cannot be checked, leaves the report standing.
// wpCoreLineEndingsMaxBytes bounds the read of a flagged file. Shipped core
// files are a few megabytes at most; a larger one is not shown to match.
const wpCoreLineEndingsMaxBytes = 8 << 20
// wpVersionFileMaxBytes bounds the read of version.php, which is about a
// kilobyte in every release.
const wpVersionFileMaxBytes = 64 << 10
var fetchWPCoreChecksums = wpcheck.FetchChecksums
// logWPCoreLineEndings names why a flagged file stays reported; the finding
// alone cannot say whether the file differs or the check could not run.
var logWPCoreLineEndings = csmlog.Info
var (
errWPCoreBeyondLineEndings = errors.New("differs beyond line endings")
errWPCoreFileTooLarge = errors.New("larger than the read limit")
errWPCoreNotInRelease = errors.New("not in the release checksums")
)
// wpCoreReleaseChecksums holds the official checksums of each release for one
// run, so installs on the same release cost one request and a release that
// could not be fetched is not asked again for every file.
type wpCoreReleaseChecksums struct {
mu sync.Mutex
releases map[string]*wpCoreRelease
}
type wpCoreRelease struct {
once sync.Once
checksums map[string]string
err error
}
func (c *wpCoreReleaseChecksums) lookup(version, locale string) (map[string]string, error) {
key := version + "/" + locale
c.mu.Lock()
if c.releases == nil {
c.releases = make(map[string]*wpCoreRelease)
}
release := c.releases[key]
if release == nil {
release = &wpCoreRelease{}
c.releases[key] = release
}
c.mu.Unlock()
release.once.Do(func() { _, release.checksums, release.err = fetchWPCoreChecksums(version, locale) })
return release.checksums, release.err
}
// lineEndingOnly returns the core files in a wp-cli report that differ from the
// install's release only in line endings, and whether nothing else was
// reported. Unrecognized output proves neither claim. A caller may resolve or
// suppress findings only after wpCoreVerificationCompleted accepts the report.
func (c *wpCoreReleaseChecksums) lineEndingOnly(wpPath string, out []byte) (files map[string]bool, only bool) {
files = make(map[string]bool)
only = true
type modifiedFile struct {
rel string
line int
}
var modified []modifiedFile
for i, line := range strings.Split(string(out), "\n") {
trimmed := strings.TrimSpace(line)
if trimmed == "" || strings.EqualFold(trimmed, wpChecksumFailureSummary) {
continue
}
if rel := wpChecksumModifiedFilePath(line); rel != "" {
if coreRel := wpChecksumModifiedCoreFile(line); coreRel != "" {
modified = append(modified, modifiedFile{rel: coreRel, line: i + 1})
} else {
only = false
}
continue
}
// Extra and missing core files are known differences the scan reports
// or tolerates elsewhere; they only keep the install from verifying.
if strings.Contains(line, "should not exist") || strings.Contains(line, "should exist") || strings.Contains(line, "File doesn't exist: ") {
only = false
continue
}
if wpCLIPHPDeprecation(trimmed) {
continue
}
// Only the line number: PHP output can carry secrets.
logWPCoreLineEndings("WordPress core line-ending check stopped at unrecognized wp-cli output", "path", wpPath, "line", i+1)
return files, false
}
if len(modified) == 0 {
return files, false
}
release, checksums, err := c.installChecksums(wpPath)
if err != nil {
logWPCoreLineEndings("WordPress core line-ending check has no release checksums", "path", wpPath, "release", release, "err", err)
return files, false
}
for _, file := range modified {
rel := file.rel
want, listed := checksums[rel]
if !listed {
// The parser also accepts mismatch text embedded in PHP output.
// Only release-listed names are safe to copy into the log.
logWPCoreLineEndings("WordPress core line-ending check cannot identify reported file", "path", wpPath, "line", file.line, "release", release, "err", errWPCoreNotInRelease)
only = false
continue
}
if err := wpCoreFileDiffersOnlyInLineEndings(wpPath, rel, want); err != nil {
logWPCoreLineEndings("WordPress core file not proven to differ only in line endings", "path", wpPath, "file", rel, "release", release, "err", err)
only = false
continue
}
files[rel] = true
}
return files, only && len(files) > 0
}
var (
wpCLIPHPDeprecationLine = regexp.MustCompile(`^(?:PHP )?Deprecated: .+ in .+ on line [0-9]+$`)
// PHP method names can contain Error:: or Warning::; the scope operator
// is not a diagnostic label, but a later single-colon label still is.
wpCLIPHPDeprecationClaim = regexp.MustCompile(`(?i)(?:warning|error|success):(?:[^:]|$)|doesn't verify against checksum|should (?:not )?exist|file doesn't exist`)
)
// wpCLIPHPDeprecation accepts only a standalone notice in PHP's logged or
// displayed format. PHP writes them into wp-cli's output whenever its own log
// file is read-only, as inside the daemon's sandbox, and a site's bundled
// libraries raise them on newer PHP. Site output is untrusted and stdout and
// stderr are combined: even a complete-looking notice must not swallow a
// warning, error, or wp-cli claim. Warnings can mean files were skipped and
// must stay visible.
func wpCLIPHPDeprecation(line string) bool {
if !wpCLIPHPDeprecationLine.MatchString(line) || strings.IndexFunc(line, unicode.IsControl) >= 0 {
return false
}
return !wpCLIPHPDeprecationClaim.MatchString(line)
}
// installChecksums returns the official checksums of the release the install
// names in version.php, and that release as version/locale.
func (c *wpCoreReleaseChecksums) installChecksums(wpPath string) (release string, checksums map[string]string, err error) {
versionFile, err := readWPCoreFile(wpPath, filepath.Join("wp-includes", "version.php"), wpVersionFileMaxBytes)
if err != nil {
return "", nil, fmt.Errorf("reading version.php: %w", err)
}
version, locale, err := wpcheck.ParseVersionContent(versionFile)
if err != nil {
// Parse errors can quote tenant-controlled file contents.
return "", nil, errors.New("invalid release metadata in version.php")
}
checksums, err = c.lookup(version, locale)
return version + "/" + locale, checksums, err
}
// wpCoreFileDiffersOnlyInLineEndings returns nil when one whole-file line
// ending conversion turns the file into the release's file.
func wpCoreFileDiffersOnlyInLineEndings(wpPath, rel, want string) error {
data, err := readWPCoreFile(wpPath, rel, wpCoreLineEndingsMaxBytes)
if err != nil {
return err
}
// Either direction of one whole-file conversion: CRLF copies of an LF
// release, and LF copies of a release that ships CRLF. A file that still
// holds a stray CR is not a clean LF conversion.
var normalized []byte
switch {
case bytes.Contains(data, []byte("\r\n")):
normalized = bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n"))
case bytes.Contains(data, []byte("\n")) && !bytes.Contains(data, []byte("\r")):
normalized = bytes.ReplaceAll(data, []byte("\n"), []byte("\r\n"))
default:
return errWPCoreBeyondLineEndings
}
// #nosec G401 -- compared with the MD5 digests wordpress.org publishes
sum := md5.Sum(normalized)
if got := hex.EncodeToString(sum[:]); got != want {
return fmt.Errorf("%w: converted md5 %s, release md5 %s", errWPCoreBeyondLineEndings, got, want)
}
return nil
}
// readWPCoreFile reads a whole core file; one past the limit is an error.
func readWPCoreFile(wpPath, rel string, limit int64) ([]byte, error) {
data, truncated, err := readWPCoreFileHead(wpPath, rel, limit)
if err == nil && truncated {
err = errWPCoreFileTooLarge
}
return data, err
}
// readWPCoreFileHead keeps every account-controlled component beneath the
// opened install. Production walks by directory descriptor and refuses links;
// injected test filesystems retain the package's bounded-reader behavior.
func readWPCoreFileHead(wpPath, rel string, limit int64) (data []byte, truncated bool, err error) {
path := wpCoreFilePathWithin(wpPath, rel)
if path == "" {
return nil, false, os.ErrPermission
}
if _, production := osFS.(realOS); !production {
return readRegularFileHead(path, limit)
}
target, err := safepath.OpenTarget(wpPath, filepath.Clean(rel), false)
if err != nil {
return nil, false, err
}
defer target.Close()
file, err := target.Parent.OpenFile(target.Name, os.O_RDONLY, 0)
if err != nil {
return nil, false, err
}
defer file.Close()
before, err := file.Stat()
if err != nil {
return nil, false, err
}
if !before.Mode().IsRegular() {
return nil, false, errNonRegularFile
}
data, err = io.ReadAll(io.LimitReader(file, limit+1))
if err != nil {
return nil, false, err
}
after, err := file.Stat()
if err != nil || !sameFileSnapshot(before, after) {
return nil, false, errFileChanged
}
current, err := target.Parent.Stat(target.Name)
if err != nil || !sameFileSnapshot(after, current) {
return nil, false, errFileChanged
}
if err := target.Check(); err != nil {
return nil, false, errFileChanged
}
if int64(len(data)) > limit {
return data[:limit], true, nil
}
return data, false, nil
}
package checks
import (
"context"
"errors"
"fmt"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
type wpVerificationBatch struct {
db *store.DB
kind, owner string
at time.Time
paths map[string]string
mu sync.Mutex
results map[string]store.WPVerificationResult
}
func newWPVerificationBatch(ctx context.Context, db *store.DB, kind, owner string, configs []string) *wpVerificationBatch {
at := time.Now()
if cycle := wpInstallCacheFrom(ctx); cycle != nil {
at = cycle.started
}
b := &wpVerificationBatch{db: db, kind: kind, owner: owner, at: at, paths: make(map[string]string), results: make(map[string]store.WPVerificationResult)}
for _, path := range configs {
b.paths[filepath.Dir(path)] = wpConfigUser(filepath.Dir(path))
}
return b
}
func (b *wpVerificationBatch) record(path string, result store.WPVerificationResult) {
b.mu.Lock()
b.results[path] = result
b.mu.Unlock()
}
// finish runs after workers join, including on partial discovery or shutdown.
// Attempted sites advance; unattempted sites retain their previous evidence.
func (b *wpVerificationBatch) finish(ctx context.Context, complete bool) error {
if b.db == nil {
return nil
}
err := b.db.UpdateWPVerification(b.kind, b.at, AccountFromContext(ctx), b.paths, b.results, complete && ctx.Err() == nil)
if err != nil {
markCheckIncomplete(ctx, b.owner)
}
return err
}
func wpVerificationFindings(ctx context.Context, db *store.DB, kind, owner string, updateErr error) []alert.Finding {
if db == nil {
return nil
}
check, label := "wp_core_unverified", "core verification"
if kind == "plugins" {
check, label = "wp_plugin_inventory_unverified", "plugin inventory"
}
rows, err := db.WPVerification(kind)
if err != nil || updateErr != nil {
markCheckIncomplete(ctx, owner)
return []alert.Finding{{Check: check, Severity: alert.Warning, Message: "WordPress " + label + " history unavailable", Details: "CSM could not read or save verification history. Check the state database; previous coverage findings are retained."}}
}
paths := make([]string, 0, len(rows))
for path := range rows {
paths = append(paths, path)
}
sort.Strings(paths)
scope := AccountFromContext(ctx)
byReason := make(map[string][]string)
for _, path := range paths {
row := rows[path]
if scope != "" && row.Account != scope {
continue
}
if row.State != "unverified" || row.Failures < 2 {
continue
}
byReason[row.Reason] = append(byReason[row.Reason], path)
}
reasons := make([]string, 0, len(byReason))
for reason := range byReason {
reasons = append(reasons, reason)
}
sort.Strings(reasons)
var findings []alert.Finding
for _, reason := range reasons {
group := byReason[reason]
// Only a host scan represents the whole cause. Account scans retain
// installation identities so a scoped result cannot replace or dedup
// against a host summary (or another account's summary).
if scope == "" && len(group) > wpVerificationCollapseCap {
findings = append(findings, wpVerificationCollapsedFinding(check, label, reason, group, rows))
continue
}
for _, path := range group {
row := rows[path]
findings = append(findings, alert.Finding{
Check: check, Severity: alert.Warning,
Message: "WordPress " + label + " repeatedly failed: " + strconv.QuoteToASCII(path),
Details: fmt.Sprintf("Installation: %s\nLast attempt: %s\nReason: %s\nVerification could not complete in consecutive scan cycles. Check wp-cli and this installation locally; raw command output is not stored.", strconv.QuoteToASCII(path), row.AttemptAt.UTC().Format(time.RFC3339), reason),
FilePath: path, TenantID: row.Account, DedupKey: path,
})
}
}
return findings
}
// wpVerificationCollapseCap is how many installations may share one cause
// before the finding stops naming them one alert at a time. A missing wp-cli
// or an unreachable checksum service fails every installation on the host in
// the same cycle, and one alert per installation would spend the whole
// alerts.max_per_hour budget on a single operational fault.
const wpVerificationCollapseCap = 10
// wpVerificationSampleLimit bounds how many installations a collapsed finding
// quotes. Enough to recognise the affected accounts without a wall of paths.
const wpVerificationSampleLimit = 10
// wpVerificationCollapsedFinding reports one cause that stopped verification
// across many installations. The cause is what the operator acts on, so the
// identity is the cause and not any single installation.
func wpVerificationCollapsedFinding(check, label, reason string, group []string, rows map[string]store.WPVerificationRecord) alert.Finding {
var last time.Time
accounts := make(map[string]struct{}, len(group))
for _, path := range group {
row := rows[path]
accounts[row.Account] = struct{}{}
if row.AttemptAt.After(last) {
last = row.AttemptAt
}
}
var details strings.Builder
fmt.Fprintf(&details, "Reason: %s\n", reason)
fmt.Fprintf(&details, "Installations affected: %d\n", len(group))
fmt.Fprintf(&details, "Last attempt: %s\n", last.UTC().Format(time.RFC3339))
for _, path := range firstN(group, wpVerificationSampleLimit) {
fmt.Fprintf(&details, "- %s\n", strconv.QuoteToASCII(path))
}
if extra := len(group) - wpVerificationSampleLimit; extra > 0 {
fmt.Fprintf(&details, "... and %d more\n", extra)
}
details.WriteString("Verification could not complete in consecutive scan cycles. One cause stopped every installation listed, so fix it once; raw command output is not stored.")
finding := alert.Finding{
Check: check, Severity: alert.Warning,
Message: fmt.Sprintf("WordPress %s repeatedly failed for %d installations", label, len(group)),
Details: details.String(),
DedupKey: "reason:" + reason,
}
if len(accounts) == 1 {
for account := range accounts {
finding.TenantID = account
}
}
return finding
}
// wpChecksumFailureSummary is wp-cli's closing line, lowercased, for an
// install it finished checking and found different from the release.
const wpChecksumFailureSummary = "error: wordpress installation doesn't verify against checksums."
// Integrity warnings can precede a later operational error. Only wp-cli's
// expected checksum-mismatch summary establishes a completed negative check.
func wpCoreVerificationCompleted(err error, out []byte) bool {
if !commandRefused(err) {
return false
}
completed := false
for _, line := range strings.Split(strings.ToLower(string(out)), "\n") {
line = strings.TrimSpace(line)
if strings.Contains(line, "fatal error") || strings.Contains(line, "parse error") {
return false
}
if line == wpChecksumFailureSummary {
completed = true
} else if strings.HasPrefix(line, "error:") {
return false
}
}
return completed
}
type wpInventoryError struct {
err error
result store.WPVerificationResult
}
func (e *wpInventoryError) Error() string { return e.result.Reason }
func (e *wpInventoryError) Unwrap() error { return e.err }
// wpVerificationFailure classifies output into fixed reasons. Never copy raw
// PHP output or an exec error into persisted evidence: either may carry secrets.
func wpVerificationFailure(err error, out []byte) store.WPVerificationResult {
var inventory *wpInventoryError
if errors.As(err, &inventory) {
return inventory.result
}
result := store.WPVerificationResult{State: "unverified", Reason: "wp-cli could not complete the check"}
var exit *exec.ExitError
switch {
case errors.Is(err, context.DeadlineExceeded):
result.Reason = "wp-cli timed out"
case errors.Is(err, context.Canceled):
result.Reason = "wp-cli was interrupted"
case errors.Is(err, exec.ErrNotFound):
result.Reason = "wp-cli executable is unavailable"
case errors.Is(err, errWPInventoryParse):
result.Reason = "wp-cli returned invalid plugin inventory JSON"
case errors.Is(err, errWPInventoryNoOutput):
result.Reason = "wp-cli returned no plugin inventory output"
case errors.As(err, &exit) && exit.ExitCode() < 0:
result.Reason = "wp-cli was terminated by a signal"
default:
text := strings.ToLower(string(out))
if len(out) == 0 && exit != nil {
text = strings.ToLower(string(exit.Stderr))
}
switch {
case commandRefused(err) && strings.Contains(text, "error: this does not seem to be a wordpress installation."):
result.State, result.Reason = "not_wordpress", "wp-cli did not find a WordPress installation"
case strings.Contains(text, "fatal error"), strings.Contains(text, "parse error"):
result.Reason = "WordPress configuration or PHP initialization failed"
case strings.Contains(text, "checksum"):
result.Reason = "WordPress checksum data could not be obtained or verified"
case strings.Contains(text, "database connection"):
result.Reason = "WordPress could not connect to its database"
case strings.Contains(text, "permission denied"):
result.Reason = "wp-cli could not read installation files or execute a required command"
case strings.Contains(text, "not found"):
result.Reason = "wp-cli or a required installation file is unavailable"
case len(text) == 0:
result.Reason = "wp-cli returned no diagnostic output"
}
}
return result
}
// CheckWPPluginVerification reports the shared inventory independently of its
// outdated/known-vulnerable consumers. The existing plugin refresh interval and
// disabled_checks controls remain authoritative.
func CheckWPPluginVerification(ctx context.Context, cfg *config.Config, _ *state.Store) []alert.Finding {
if ctx == nil {
ctx = context.Background()
}
if incompleteCollectorFrom(ctx) == nil {
ctx, _ = withIncompleteCheckCollector(ctx)
}
disabled := make(map[string]bool)
if cfg != nil {
for _, name := range cfg.DisabledChecks {
disabled[strings.TrimSpace(name)] = true
}
}
if disabled["outdated_plugins"] && (disabled["vulnerable_plugins"] || (cfg != nil && !cfg.VulnerablePluginScanningEnabled())) {
return nil
}
db := store.Global()
if db == nil {
return nil
}
ensurePluginCacheFresh(ctx, cfg, db)
var updateErr error
if checkMarkedIncomplete(ctx, "wp_plugin_inventory") {
updateErr = errors.New("verification history update failed")
}
return wpVerificationFindings(ctx, db, "plugins", "wp_plugin_inventory", updateErr)
}
package checks
import (
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
var wpCoreBatches = newScanBatchMonitor()
// WPCoreQueueStatus includes selected installations through checksum execution,
// result collection and verified-file caching. Concurrent scans share no cap.
func WPCoreQueueStatus(now time.Time) queuehealth.Status {
status := wpCoreBatches.snapshot(now)
status.DepthUnit = "installations"
return status
}
package checks
import (
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/config"
)
// MigrateWPCronCrontabs upgrades CSM-managed wp-cron crontab lines installed
// by older releases (synchronized */N schedule, no overlap lock) to the
// current staggered format. The perf_wp_cron finding never re-fires once
// DISABLE_WP_CRON is set, so already-fixed accounts can only be reached by
// walking the spool directly. Gated on the same fix_wp_cron opt-in as the
// install path; returns the number of crontabs rewritten.
func MigrateWPCronCrontabs(cfg *config.Config) int {
if cfg == nil || !cfg.AutoResponse.Enabled || !cfg.AutoResponse.FixWPCron {
return 0
}
opts := WPCronFixOptions{
IntervalMinutes: cfg.Performance.WPCronFix.IntervalMinutes,
PHPBin: cfg.Performance.WPCronFix.PHPBin,
}
upgraded := 0
seen := map[string]bool{}
for _, dir := range wpCronSpoolDirs {
entries, err := osFS.ReadDir(dir)
if err != nil {
continue
}
for _, e := range entries {
if e.IsDir() {
continue
}
owner := e.Name()
if seen[owner] || !validCPUser.MatchString(owner) || owner == "root" {
continue
}
data, err := osFS.ReadFile(filepath.Join(dir, owner))
if err != nil {
continue
}
docroots := wpCronManagedDocroots(string(data))
if len(docroots) == 0 {
continue
}
seen[owner] = true
for _, docroot := range docroots {
installed, err := installUserWPCron(owner, docroot, opts)
if err == nil && installed {
upgraded++
}
}
}
}
return upgraded
}
// wpCronManagedDocroots extracts docroots from CSM marker lines. The marker
// is attacker-writable in principle (the spool file belongs to the account),
// so only clean absolute paths may flow into a crontab rewrite.
func wpCronManagedDocroots(crontab string) []string {
var roots []string
for _, line := range strings.Split(crontab, "\n") {
trimmed := strings.TrimSpace(line)
if !strings.HasPrefix(trimmed, wpCronJobMarker) {
continue
}
docroot := strings.TrimSpace(strings.TrimPrefix(trimmed, wpCronJobMarker))
if !safeWPCronDocroot(docroot) {
continue
}
roots = append(roots, docroot)
}
return roots
}
package checks
import (
"errors"
"io/fs"
"path/filepath"
"regexp"
"strings"
)
// cPanel records the MultiPHP version chosen for each vhost in a fixed column
// of /etc/userdatadomains, as an "ea-phpNN" or "alt-phpNN" token.
// Anything outside that shape is not a version we can turn into a path.
var userdataPHPVersionRe = regexp.MustCompile(`^(ea|alt)-php([0-9]{2})$`)
const userdataPHPVersionField = 9
// parseVhostPHPVersion pulls the MultiPHP token out of a /etc/userdatadomains
// row. The field has a fixed position after the IPv6-dedicated flag. Older
// rows may stop before it and newer rows may carry trailing empty fields, so
// accept only the fixed PHP-version column instead of searching attacker-adjacent
// fields for a version-shaped token.
func parseVhostPHPVersion(fields []string) string {
if tok, ok := vhostPHPColumn(fields); ok && userdataPHPVersionRe.MatchString(tok) {
return tok
}
return ""
}
// vhostPHPInherits reports a vhost with no MultiPHP selection of its own.
// cPanel leaves the column empty or writes "inherit"; any other token that is
// not a version is malformed, not an inheritance.
func vhostPHPInherits(fields []string) bool {
tok, ok := vhostPHPColumn(fields)
return ok && (tok == "" || tok == "inherit")
}
func vhostPHPColumn(fields []string) (string, bool) {
if len(fields) <= userdataPHPVersionField {
return "", false
}
for _, trailing := range fields[userdataPHPVersionField+1:] {
if strings.TrimSpace(trailing) != "" {
return "", false
}
}
return strings.TrimSpace(fields[userdataPHPVersionField]), true
}
// phpBinForVersion maps a cPanel MultiPHP version token to its interpreter.
// EasyApache and CloudLinux alt-php lay their trees out differently, so the
// two shapes are built separately rather than by string substitution.
// Returns empty for anything that is not a well-formed version token, which is
// what keeps a malformed or attacker-influenced map out of a crontab line.
func phpBinForVersion(version string) string {
m := userdataPHPVersionRe.FindStringSubmatch(strings.TrimSpace(version))
if m == nil {
return ""
}
if m[1] == "alt" {
return "/opt/alt/php" + m[2] + "/usr/bin/php"
}
return "/opt/cpanel/ea-php" + m[2] + "/root/usr/bin/php"
}
// resolveDocrootPHPBin returns the PHP interpreter the owner's docroot is
// pinned to, or empty when the docroot is unknown, ambiguous, or unusable.
//
// WP-Cron has to run under the same interpreter as the site: a docroot pinned
// to an old MultiPHP version fatal-errors when driven by a newer system
// default, which silently kills scheduled tasks on that site.
//
// The result is deliberately restricted to the two known-good path shapes.
// safeManagedWPCronPHPBin accepts exactly those, so a resolved path never
// makes CSM report its own crontab as an unexpected change.
func resolveDocrootPHPBin(owner, docroot string) string {
content, err := osFS.ReadFile(userdataDomainsPath)
if err != nil {
return ""
}
want := filepath.Clean(docroot)
vhosts, _ := parseUserdataDomainRootsChecked(string(content))
// Match the most specific docroot this account owns that serves the target.
// A WordPress install in a subdirectory is not its own vhost, so the map has
// no entry for it, but cPanel serves it under the enclosing vhost and hence
// that vhost's PHP version. An exact entry always wins, because a subdomain
// docroot can be nested inside the main one.
version, inherit, bestLen := "", false, -1
for _, vh := range vhosts {
if vh.user != owner || !wpCronDocrootCovers(vh.docroot, want) {
continue
}
switch {
case len(vh.docroot) > bestLen:
version, inherit, bestLen = vh.phpVersion, vh.phpInherit, len(vh.docroot)
case len(vh.docroot) == bestLen && (vh.phpVersion != version || vh.phpInherit != inherit):
// Two vhosts claim the same docroot with different selections;
// picking either would pin the wrong interpreter.
version, inherit = "", false
}
}
if inherit {
return inheritedHandlerPHPBin(owner, want)
}
if version == "" {
return ""
}
bin := phpBinForVersion(version)
if bin == "" || !safeManagedWPCronPHPBin(bin) {
return ""
}
return bin
}
// inheritedHandlerPHPBin resolves the interpreter of a vhost with no MultiPHP
// selection of its own. The web server does not fall back to the system
// default there: it applies the nearest cPanel handler block in the .htaccess
// chain, which a copied, restored, or migrated docroot can still carry.
// cPanel's CLI wrapper reads only the domain map, so it disagrees with the site
// in exactly that case. The walk stays inside the owner's home: nothing above
// it is the tenant's to configure.
func inheritedHandlerPHPBin(owner, dir string) string {
home := defaultUIDCache.HomeDir(owner)
if home == "" {
return ""
}
home = filepath.Clean(home)
if dir != home && !strings.HasPrefix(dir, home+string(filepath.Separator)) {
return ""
}
for d := dir; ; d = filepath.Dir(d) {
data, ok, err := readTenantHtaccessBounded(home, d)
switch {
case errors.Is(err, fs.ErrNotExist):
case err != nil || !ok:
// A nearer file that cannot be read may override any ancestor.
return ""
default:
if version, found := cpanelHandlerVersion(string(data)); found {
bin := phpBinForVersion(version)
if bin == "" || !safeManagedWPCronPHPBin(bin) || !wpCronPHPBinInstalled(bin) {
return ""
}
return bin
}
}
if d == home {
return ""
}
}
}
// wpCronPHPBinInstalled reports whether an interpreter is present and
// executable. A handler can outlive the PHP version it names; a cron line on a
// missing binary would fail every run.
var wpCronPHPBinInstalled = func(bin string) bool {
info, err := osFS.Stat(bin)
return err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0
}
const (
cpanelHandlerBegin = "# php -- BEGIN cPanel-generated handler, do not edit"
cpanelHandlerEnd = "# php -- END cPanel-generated handler, do not edit"
)
var (
htaccessAddHandlerRe = regexp.MustCompile(`^AddHandler\s+(\S+)((?:\s+\S+)+)$`)
cpanelPHPHandlerRe = regexp.MustCompile(`^application/x-httpd-((?:ea|alt)-php[0-9]{2})(?:___lsphp)?$`)
)
// cpanelHandlerVersion returns the PHP version the cPanel-generated handler
// blocks of one .htaccess select for .php, and whether a complete block sets
// a handler for .php. The last AddHandler for .php wins, as it does in the
// web server; an unrecognised handler selects no usable version. Directives
// outside a complete block are not cPanel's selection and are ignored.
func cpanelHandlerVersion(content string) (version string, found bool) {
inBlock, blockVersion, blockSets := false, "", false
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
switch {
case line == cpanelHandlerBegin:
inBlock, blockVersion, blockSets = true, "", false
case line == cpanelHandlerEnd:
if inBlock && blockSets {
version, found = blockVersion, true
}
inBlock = false
case inBlock:
m := htaccessAddHandlerRe.FindStringSubmatch(line)
if m == nil || !handlerCoversDotPHP(m[2]) {
continue
}
blockVersion, blockSets = "", true
if h := cpanelPHPHandlerRe.FindStringSubmatch(m[1]); h != nil {
blockVersion = h[1]
}
}
}
return version, found
}
func handlerCoversDotPHP(extensions string) bool {
for _, ext := range strings.Fields(extensions) {
if strings.EqualFold(ext, ".php") {
return true
}
}
return false
}
// wpCronDocrootCovers reports whether vhostRoot serves docroot: either the same
// path or an ancestor of it. Comparison is path-segment aware so
// /home/a/public_html never covers /home/a/public_html_old, and a root shallower
// than /home/<user>/<dir> is rejected so inheritance cannot cross accounts.
func wpCronDocrootCovers(vhostRoot, docroot string) bool {
vhostRoot = filepath.Clean(vhostRoot)
if strings.Count(strings.TrimSuffix(vhostRoot, "/"), "/") < 3 {
return false
}
if vhostRoot == docroot {
return true
}
return strings.HasPrefix(docroot, vhostRoot+"/")
}
// resolveWPCronPHPBin distinguishes an operator override or unambiguous vhost
// mapping from fallback detection. Callers upgrading an existing managed line
// use that provenance to avoid replacing a known-good interpreter when the
// domain map is temporarily unavailable.
func resolveWPCronPHPBin(owner, docroot string, opts WPCronFixOptions) (string, bool) {
if opts.PHPBin != "" {
return opts.PHPBin, true
}
if bin := resolveDocrootPHPBin(owner, docroot); bin != "" {
return bin, true
}
return detectPHPBin(), false
}
// wpCronPHPBin picks the interpreter for a managed cron line. An operator who
// sets php_bin has overridden the choice deliberately, so that wins; otherwise
// the vhost's own version wins; detection is the last resort so a host without
// a usable domain map still gets an installable line.
func wpCronPHPBin(owner, docroot string, opts WPCronFixOptions) string {
bin, _ := resolveWPCronPHPBin(owner, docroot, opts)
return bin
}
package checks
import (
"bytes"
"fmt"
"hash/fnv"
"io"
"os"
"os/user"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"syscall"
)
// WPCronFixOptions carries operator-tunable parameters for the WP-Cron
// remediation. Both the Web UI handler and the daemon auto-response resolve
// these from config before calling the fix, so the remediation core itself
// stays free of config coupling.
type WPCronFixOptions struct {
// IntervalMinutes is how often the installed system cron runs wp-cron.php.
// Clamped to [1,60]; a non-positive value falls back to the 15-minute default.
IntervalMinutes int
// PHPBin is the interpreter the cron line invokes. Empty uses an unambiguous
// cPanel vhost version, then LookPath("php"), then /usr/local/bin/php.
PHPBin string
}
const (
wpCronDefaultIntervalMin = 15
wpCronMaxIntervalMin = 60
// wpCronEditMarker tags the line CSM inserts so the customer can see why
// WP-Cron was disabled and so re-running the fix stays idempotent.
wpCronEditMarker = "// CSM: WP-Cron disabled, served by system cron instead"
// wpCronJobMarker prefixes the managed crontab block for a given docroot.
wpCronJobMarker = "# CSM WP-Cron "
wpCronStopMarker = "stop editing"
)
// wpCronDefineRe matches a define of DISABLE_WP_CRON set to a truthy value,
// matching the detector's view of "already disabled".
var wpCronDefineRe = regexp.MustCompile(`(?i)define\s*\(\s*['"]DISABLE_WP_CRON['"]\s*,\s*['"]?(true|1)['"]?\s*\)`)
// validCPUser guards the username passed to `crontab -u`. cPanel usernames are
// lowercase alnum starting with a letter; rejecting anything else keeps a
// surprising file owner from reaching the crontab argument vector.
var validCPUser = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,31}$`)
var wpCronHeredocStartRe = regexp.MustCompile(`<<<['"]?([A-Za-z_][A-Za-z0-9_]*)['"]?`)
// Crontab installs are read-modify-write; serialize each account in-process.
var wpCronCrontabLocks sync.Map
// wp-config.php edits are read-modify-write too. Deep and periodic scans can
// overlap, so serialize each config path before resolving, reading, and writing.
var wpCronConfigLocks sync.Map
// wpCronOwnerName resolves the account that owns a wp-config.php. It is a var
// so tests can inject a deterministic owner regardless of who runs `go test`.
var wpCronOwnerName = fileOwnerName
// FixDisableWPCron disables WP-Cron in a wp-config.php and installs a real
// per-user system cron that runs wp-cron.php on a fixed interval. It scopes
// writes to the default per-account roots (/home).
func FixDisableWPCron(path string, opts WPCronFixOptions) RemediationResult {
return FixDisableWPCronInRoots(path, fixPerfAllowedRoots, opts)
}
// FixDisableWPCronInRoots is FixDisableWPCron with caller-supplied roots so the
// Web UI can honor configured account_roots and tests can write under t.TempDir().
func FixDisableWPCronInRoots(path string, allowedRoots []string, opts WPCronFixOptions) RemediationResult {
if path == "" {
return RemediationResult{Error: "could not extract file path from finding"}
}
lockPath, err := sanitizeFixPath(path, allowedRoots)
if err != nil {
return RemediationResult{Error: err.Error()}
}
lock := wpCronConfigLock(lockPath)
lock.Lock()
defer lock.Unlock()
resolved, info, err := resolveExistingFixPath(lockPath, allowedRoots)
if err != nil {
return RemediationResult{Error: err.Error()}
}
if info.IsDir() {
return RemediationResult{Error: "refusing to edit a directory"}
}
if filepath.Base(resolved) != "wp-config.php" {
return RemediationResult{Error: fmt.Sprintf("automated WP-Cron fix only applies to wp-config.php (got %s)", filepath.Base(resolved))}
}
target, err := openFixTarget(resolved, info)
if err != nil {
return RemediationResult{Error: fmt.Sprintf("read failed: %v", err)}
}
defer target.Close()
data, err := io.ReadAll(target.File)
if err != nil {
return RemediationResult{Error: fmt.Sprintf("read failed: %v", err)}
}
isWordPress, validateErr := wpCronInstallIsValid(resolved, data)
if validateErr != nil {
return RemediationResult{Error: fmt.Sprintf("WordPress validation failed: %v", validateErr)}
}
if !isWordPress {
return RemediationResult{Error: "refusing WP-Cron fix because the directory is not a complete WordPress install"}
}
var actions []string
needsDefine := !wpCronHasActiveDisableDefine(data)
var rewritten []byte
if needsDefine {
var ok bool
rewritten, ok = insertDisableWPCron(data)
if !ok {
return RemediationResult{Error: "could not find a safe insertion point in wp-config.php (no \"stop editing\" marker or wp-settings.php require)"}
}
}
docroot := filepath.Dir(resolved)
owner, err := wpCronOwnerName(info)
if err != nil {
return RemediationResult{Error: fmt.Sprintf("could not resolve account owner of wp-config.php: %v", err)}
}
cronInstalled, err := installUserWPCron(owner, docroot, opts)
if err != nil {
return RemediationResult{Error: fmt.Sprintf("system cron install failed: %v", err)}
}
if needsDefine {
if werr := writeCleanedFileAtomic(target, rewritten); werr != nil {
if cronInstalled {
return RemediationResult{Error: fmt.Sprintf("system cron installed but wp-config.php update failed: %v", werr)}
}
return RemediationResult{Error: werr.Error()}
}
actions = append(actions, "disabled WP-Cron in wp-config.php")
}
if cronInstalled {
actions = append(actions, fmt.Sprintf("installed every-%d-minute system cron for %s", clampInterval(opts.IntervalMinutes), owner))
}
if len(actions) == 0 {
return RemediationResult{
Success: true,
Action: fmt.Sprintf("wp-cron already configured for %s", docroot),
Description: "WP-Cron already disabled and system cron already present; no change needed",
}
}
return RemediationResult{
Success: true,
Action: fmt.Sprintf("disable WP-Cron + install system cron for %s", docroot),
Description: strings.Join(actions, "; "),
}
}
// insertDisableWPCron returns wp-config.php bytes with the DISABLE_WP_CRON
// define inserted before the "stop editing" marker, or before the
// wp-settings.php require as a fallback. The second return is false when no
// safe insertion point exists, so the caller can refuse rather than append a
// define into an unfamiliar PHP file.
func insertDisableWPCron(data []byte) ([]byte, bool) {
lines := bytes.Split(data, []byte("\n"))
defineLine := []byte("define( 'DISABLE_WP_CRON', true ); " + wpCronEditMarker)
insertAt := wpCronInsertionLine(lines)
if insertAt < 0 {
return nil, false
}
out := make([][]byte, 0, len(lines)+1)
out = append(out, lines[:insertAt]...)
out = append(out, defineLine)
out = append(out, lines[insertAt:]...)
return bytes.Join(out, []byte("\n")), true
}
func wpCronHasActiveDisableDefine(data []byte) bool {
inBlockComment := false
heredocLabel := ""
for _, line := range strings.Split(string(data), "\n") {
code := wpCronActivePHPCode(line, &inBlockComment, &heredocLabel)
if wpCronDefineRe.MatchString(code) {
return true
}
}
return false
}
func wpCronInsertionLine(lines [][]byte) int {
inBlockComment := false
heredocLabel := ""
fallback := -1
for i, line := range lines {
safeAtLineStart := !inBlockComment && heredocLabel == ""
code := wpCronActivePHPCode(string(line), &inBlockComment, &heredocLabel)
if !safeAtLineStart {
continue
}
if bytes.Contains(bytes.ToLower(line), []byte(wpCronStopMarker)) {
return i
}
if fallback < 0 && strings.Contains(code, "wp-settings.php") {
fallback = i
}
}
return fallback
}
func wpCronActivePHPCode(line string, inBlockComment *bool, heredocLabel *string) string {
if *heredocLabel != "" {
if wpCronEndsHeredoc(line, *heredocLabel) {
*heredocLabel = ""
}
return ""
}
var out strings.Builder
quote := byte(0)
escaped := false
for i := 0; i < len(line); i++ {
if *inBlockComment {
if i+1 < len(line) && line[i] == '*' && line[i+1] == '/' {
*inBlockComment = false
i++
}
continue
}
c := line[i]
if quote != 0 {
out.WriteByte(c)
if escaped {
escaped = false
continue
}
if c == '\\' {
escaped = true
continue
}
if c == quote {
quote = 0
}
continue
}
if c == '\'' || c == '"' {
quote = c
out.WriteByte(c)
continue
}
if i+1 < len(line) && c == '/' && line[i+1] == '*' {
*inBlockComment = true
i++
continue
}
if i+1 < len(line) && c == '/' && line[i+1] == '/' {
break
}
if c == '#' {
break
}
out.WriteByte(c)
}
code := out.String()
if match := wpCronHeredocStartRe.FindStringSubmatch(code); len(match) == 2 {
*heredocLabel = match[1]
}
return code
}
func wpCronEndsHeredoc(line, label string) bool {
trimmed := strings.TrimSpace(line)
return trimmed == label || trimmed == label+";"
}
// installUserWPCron ensures the owner's crontab contains a CSM-managed line
// running wp-cron.php for docroot. It returns false (no error) when the line
// is already present. The crontab is rewritten via a spool file because the
// command runner has no stdin channel; `crontab -u <user> <file>` installs and
// validates it atomically.
func installUserWPCron(owner, docroot string, opts WPCronFixOptions) (bool, error) {
if !validCPUser.MatchString(owner) || owner == "root" {
return false, fmt.Errorf("refusing crontab edit for unexpected account name %q", owner)
}
if !safeWPCronDocroot(docroot) {
return false, fmt.Errorf("refusing crontab edit for unsafe WP-Cron docroot %q", docroot)
}
lock := wpCronCrontabLock(owner)
lock.Lock()
defer lock.Unlock()
// Resolve under the account lock so a caller that waited for another
// crontab edit cannot install a PHP mapping captured before that edit.
var authoritativePHPBin bool
opts.PHPBin, authoritativePHPBin = resolveWPCronPHPBin(owner, docroot, opts)
if !safeCronCommandString(opts.PHPBin) {
return false, fmt.Errorf("refusing crontab edit for unsafe WP-Cron php binary %q", opts.PHPBin)
}
existing := ""
if out, err := cmdExec.RunAllowNonZero("crontab", "-u", owner, "-l"); err == nil {
existing = string(out)
}
if !authoritativePHPBin {
if existingPHPBin := currentManagedWPCronPHPBin(existing, owner, docroot); existingPHPBin != "" {
opts.PHPBin = existingPHPBin
}
}
want := wpCronJobLine(owner, docroot, opts)
var buf bytes.Buffer
switch {
case wpCronUpgradeManagedLine(existing, docroot, want, &buf):
// Stale CSM-managed line rewritten in place (legacy synchronized
// schedule, changed interval, or changed php path).
case crontabHasWPCronJob(existing, docroot):
// Current managed line, or a customer-authored wp-cron entry CSM
// must not fight over.
return false, nil
default:
buf.WriteString(strings.TrimRight(existing, "\n"))
if buf.Len() > 0 {
buf.WriteByte('\n')
}
buf.WriteString(wpCronJobMarker + docroot + "\n")
buf.WriteString(want + "\n")
}
tmp, err := os.CreateTemp("", "csm-wpcron-*")
if err != nil {
return false, fmt.Errorf("create crontab spool: %v", err)
}
tmpPath := tmp.Name()
defer func() { _ = os.Remove(tmpPath) }()
if _, err := tmp.Write(buf.Bytes()); err != nil {
_ = tmp.Close()
return false, fmt.Errorf("write crontab spool: %v", err)
}
if err := tmp.Close(); err != nil {
return false, fmt.Errorf("close crontab spool: %v", err)
}
expected := append([]byte(nil), buf.Bytes()...)
preRecordCrontabSelfWrite(owner, expected)
if _, err := cmdExec.Run("crontab", "-u", owner, tmpPath); err != nil {
forgetSelfWrites(crontabSpoolPaths(owner)...)
return false, fmt.Errorf("crontab install: %v", err)
}
recordCrontabSelfWrite(owner, expected)
return true, nil
}
func preRecordCrontabSelfWrite(owner string, expected []byte) {
for _, p := range crontabSpoolPaths(owner) {
RecordSelfWrite(p, expected)
}
}
// recordCrontabSelfWrite registers the just-installed crontab with the
// self-write ledger so the sensitive-file detectors do not flag CSM's own
// change. The on-disk spool content (cron may normalize it) is what the
// detectors hash, so record the spool file rather than our staged buffer.
func recordCrontabSelfWrite(owner string, expected []byte) {
paths := crontabSpoolPaths(owner)
recorded := ""
for _, p := range paths {
data, err := osFS.ReadFile(p)
if err != nil {
continue
}
if crontabContentEqual(data, expected) || crontabExplainedBy(data, expected) {
RecordSelfWrite(p, data)
recorded = p
break
}
}
for _, p := range paths {
if p != recorded {
forgetSelfWrites(p)
}
}
}
// wpCronSpoolDirs lists where cron daemons keep per-user crontabs (cronie,
// then Debian-style cron). A var so tests can point it at a temp dir.
var wpCronSpoolDirs = []string{"/var/spool/cron", "/var/spool/cron/crontabs"}
func crontabSpoolPaths(owner string) []string {
paths := make([]string, 0, len(wpCronSpoolDirs))
for _, dir := range wpCronSpoolDirs {
paths = append(paths, filepath.Join(dir, owner))
}
return paths
}
// crontabExplainedBy reports whether every executable line on disk came from
// the buffer CSM installed. The cPanel crontab wrapper rewrites the spool after
// `crontab -u` -- it prepends its own SHELL= and drops comments -- so the bytes
// never match what was staged, and refusing to record a self-write then left
// CSM's own change looking like a third party's.
//
// This is not a blanket "trust whatever is on disk": a job line that is not in
// the staged buffer, or an environment assignment that could steer execution,
// means something other than the wrapper touched the file, and the change is
// reported instead of vouched for.
func crontabExplainedBy(got, want []byte) bool {
staged := make(map[string]int)
for _, raw := range strings.Split(string(normalizeCrontabLineEndings(want)), "\n") {
line := strings.TrimSpace(raw)
if line != "" && !strings.HasPrefix(line, "#") {
staged[line]++
}
}
if len(staged) == 0 {
return false
}
for _, raw := range strings.Split(string(normalizeCrontabLineEndings(got)), "\n") {
line := strings.TrimSpace(raw)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if staged[line] > 0 {
staged[line]--
continue
}
if name, val, ok := splitCrontabEnv(line); ok &&
strings.EqualFold(name, "SHELL") &&
val == "/usr/local/cpanel/bin/jailshell" {
continue
}
return false
}
for _, remaining := range staged {
if remaining != 0 {
return false
}
}
return true
}
func crontabContentEqual(got, want []byte) bool {
got = normalizeCrontabLineEndings(got)
want = normalizeCrontabLineEndings(want)
if bytes.Equal(got, want) {
return true
}
return bytes.HasSuffix(want, []byte("\n")) && bytes.Equal(got, bytes.TrimSuffix(want, []byte("\n")))
}
func normalizeCrontabLineEndings(data []byte) []byte {
return bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n"))
}
// wpCronJobLine builds the crontab entry. CLI php is used (not an HTTP hit) so
// the job does not tie up a web worker, which is the load source the finding
// flags. max_execution_time caps a runaway cron pass.
//
// The minute field is staggered per account+docroot: a plain */N schedule is
// wall-clock aligned, so every managed site on the host fires in the same
// second and the load spikes once per interval. The offset hash is
// deterministic so reinstalls are idempotent. flock skips a run while the
// previous one still holds the lock; $HOME is expanded by crond, and the lock
// lives in the account home because /tmp is symlink-attackable.
func wpCronJobLine(owner, docroot string, opts WPCronFixOptions) string {
interval := clampInterval(opts.IntervalMinutes)
php := wpCronPHPBin(owner, docroot, opts)
return fmt.Sprintf(`%s * * * * cd %s && flock -n "$HOME/.csm-wpcron-%08x.lock" %s -d max_execution_time=300 wp-cron.php >/dev/null 2>&1`,
wpCronMinuteField(wpCronStaggerOffset(owner, docroot, interval), interval),
shellQuote(docroot), wpCronLockID(docroot), shellQuote(php))
}
// wpCronStaggerOffset spreads managed sites across the interval. Hashing
// owner+docroot (not just docroot) keeps multi-site accounts spread too.
func wpCronStaggerOffset(owner, docroot string, interval int) int {
h := fnv.New32a()
h.Write([]byte(owner))
h.Write([]byte{0})
h.Write([]byte(docroot))
return int(h.Sum32() % uint32(interval)) // #nosec G115 -- interval clamped to [1,60]
}
func wpCronLockID(docroot string) uint32 {
h := fnv.New32a()
h.Write([]byte(docroot))
return h.Sum32()
}
func wpCronMinuteField(offset, interval int) string {
switch {
case interval <= 1:
return "*"
case interval >= 60:
return strconv.Itoa(offset)
case 60%interval == 0:
return fmt.Sprintf("%d-59/%d", offset, interval)
default:
minutes := make([]int, 0, (60+interval-1)/interval)
for minute := 0; minute < 60; minute += interval {
minutes = append(minutes, (minute+offset)%60)
}
sort.Ints(minutes)
parts := make([]string, 0, len(minutes))
for _, minute := range minutes {
parts = append(parts, strconv.Itoa(minute))
}
return strings.Join(parts, ",")
}
}
// wpCronUpgradeManagedLine rewrites the job line under this docroot's CSM
// marker when it differs from want, writing the full crontab into buf. It
// returns false when there is no marker, the managed line already matches, or
// the line after the marker is not a wp-cron job for this docroot (a crontab
// the customer rearranged is left alone rather than guessed at).
func wpCronUpgradeManagedLine(existing, docroot, want string, buf *bytes.Buffer) bool {
lines := strings.Split(existing, "\n")
marker := wpCronJobMarker + docroot
for i, line := range lines {
if strings.TrimSpace(line) != marker {
continue
}
if i+1 >= len(lines) {
lines = append(lines, want)
writeCrontabLines(buf, lines)
return true
}
job := lines[i+1]
if strings.TrimSpace(job) == "" && i+1 == len(lines)-1 {
lines[i+1] = want
writeCrontabLines(buf, lines)
return true
}
if strings.TrimSpace(job) == want {
return false
}
if cmd := crontabCommand(job); cmd == "" || !commandRunsWPCronForDocroot(cmd, docroot) {
return false
}
lines[i+1] = want
writeCrontabLines(buf, lines)
return true
}
return false
}
func currentManagedWPCronPHPBin(existing, owner, docroot string) string {
lines := strings.Split(existing, "\n")
marker := wpCronJobMarker + docroot
for i, line := range lines {
if strings.TrimSpace(line) != marker || i+1 >= len(lines) {
continue
}
job := strings.TrimSpace(lines[i+1])
managedDocroot, ok := csmManagedWPCronJob(job, owner)
if !ok || managedDocroot != docroot {
return ""
}
match := csmManagedWPCronJobRe.FindStringSubmatch(job)
phpBin, ok := unquoteShellSingle(match[4])
if !ok {
return ""
}
return phpBin
}
return ""
}
func writeCrontabLines(buf *bytes.Buffer, lines []string) {
buf.WriteString(strings.TrimRight(strings.Join(lines, "\n"), "\n"))
buf.WriteByte('\n')
}
func safeWPCronDocroot(docroot string) bool {
return docroot != "" && filepath.IsAbs(docroot) && filepath.Clean(docroot) == docroot && safeCronCommandString(docroot)
}
func safeCronCommandString(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] == '%' || s[i] < 0x20 || s[i] == 0x7f {
return false
}
}
return true
}
// crontabHasWPCronJob reports whether the crontab already runs wp-cron.php for
// docroot, regardless of interval or php path, so re-running the fix is a no-op.
func crontabHasWPCronJob(crontab, docroot string) bool {
docroot = filepath.Clean(docroot)
for _, line := range strings.Split(crontab, "\n") {
command := crontabCommand(line)
if command == "" {
continue
}
if commandRunsWPCronForDocroot(command, docroot) {
return true
}
}
return false
}
func wpCronCrontabLock(owner string) *sync.Mutex {
lock, _ := wpCronCrontabLocks.LoadOrStore(owner, &sync.Mutex{})
return lock.(*sync.Mutex)
}
func wpCronConfigLock(path string) *sync.Mutex {
lock, _ := wpCronConfigLocks.LoadOrStore(path, &sync.Mutex{})
return lock.(*sync.Mutex)
}
func crontabCommand(line string) string {
trimmed := strings.TrimSpace(line)
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
return ""
}
fields := strings.Fields(trimmed)
if len(fields) == 0 || strings.Contains(fields[0], "=") {
return ""
}
if strings.HasPrefix(fields[0], "@") {
if len(fields) < 2 {
return ""
}
return crontabCommandBeforeStdin(trimmed[len(fields[0]):])
}
if len(fields) < 6 {
return ""
}
rest := trimmed
for i := 0; i < 5; i++ {
rest = strings.TrimLeft(rest, " \t")
fieldEnd := strings.IndexAny(rest, " \t")
if fieldEnd < 0 {
return ""
}
rest = rest[fieldEnd:]
}
return crontabCommandBeforeStdin(rest)
}
func crontabCommandBeforeStdin(command string) string {
escaped := false
for i := 0; i < len(command); i++ {
switch {
case escaped:
escaped = false
case command[i] == '\\':
escaped = true
case command[i] == '%':
return strings.TrimSpace(command[:i])
}
}
return strings.TrimSpace(command)
}
func commandRunsWPCronForDocroot(command, docroot string) bool {
if !strings.Contains(command, "wp-cron.php") {
return false
}
words := shellWords(command)
wpCronPath := filepath.Clean(filepath.Join(docroot, "wp-cron.php"))
for _, word := range words {
if cleanShellPathWord(word) == wpCronPath {
return true
}
}
for i, word := range words {
if word != "cd" {
continue
}
j := i + 1
for j < len(words) && strings.HasPrefix(words[j], "-") {
j++
}
if j >= len(words) || filepath.Clean(words[j]) != docroot {
continue
}
for _, later := range words[j+1:] {
if filepath.Base(cleanShellPathWord(later)) == "wp-cron.php" {
return true
}
}
}
return false
}
func cleanShellPathWord(word string) string {
if i := strings.IndexAny(word, "?#"); i >= 0 {
word = word[:i]
}
return filepath.Clean(word)
}
func shellWords(command string) []string {
var words []string
var current strings.Builder
quote := byte(0)
escaped := false
flush := func() {
if current.Len() == 0 {
return
}
words = append(words, current.String())
current.Reset()
}
for i := 0; i < len(command); i++ {
c := command[i]
if quote != 0 {
if escaped {
current.WriteByte(c)
escaped = false
continue
}
if c == '\\' && quote == '"' {
escaped = true
continue
}
if c == quote {
quote = 0
continue
}
current.WriteByte(c)
continue
}
if escaped {
current.WriteByte(c)
escaped = false
continue
}
switch c {
case '\\':
escaped = true
case '\'', '"':
quote = c
case ' ', '\t', ';', '&', '|', '(', ')', '<', '>':
flush()
default:
current.WriteByte(c)
}
}
if escaped {
current.WriteByte('\\')
}
flush()
return words
}
func clampInterval(minutes int) int {
if minutes <= 0 {
return wpCronDefaultIntervalMin
}
if minutes > wpCronMaxIntervalMin {
return wpCronMaxIntervalMin
}
return minutes
}
func detectPHPBin() string {
if p, err := cmdExec.LookPath("php"); err == nil && p != "" {
return p
}
return "/usr/local/bin/php"
}
// fileOwnerName resolves the username that owns the wp-config.php so the cron
// runs as the account, not root. The owner is the source of truth for which
// account this WordPress install belongs to.
func fileOwnerName(info os.FileInfo) (string, error) {
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return "", fmt.Errorf("unsupported file info")
}
if stat.Uid == 0 {
// A customer wp-config.php should never be root-owned; installing a
// cron that runs wp-cron.php as root would be a privilege smell.
return "", fmt.Errorf("refusing to install a root-owned cron for wp-config.php")
}
uid := strconv.FormatUint(uint64(stat.Uid), 10)
u, err := user.LookupId(uid)
if err != nil {
return "", fmt.Errorf("uid %s: %v", uid, err)
}
return u.Username, nil
}
package checks
import (
"context"
"errors"
"io/fs"
"path/filepath"
"strings"
"sync"
"time"
)
// wpInstall is one discovered WordPress installation.
type wpInstall struct {
// ConfigPath is the wp-config.php file; DocRoot is the directory holding it.
ConfigPath string
DocRoot string
// Account owns the install, empty when the path is not attributable.
Account string
Served servedState
}
// wpSkipDirNames keep copies of a site out of discovery. A staging or backup
// tree holds a complete WordPress install, but scanning it means querying a
// database the site does not serve and fixing files nobody reaches.
var wpSkipDirNames = map[string]bool{
"cache": true, "backup": true, "backups": true, "staging": true, ".trash": true,
}
// wpDiscovery is one discovery result together with the coverage gap it
// produced. The gap travels with the result so a cached discovery can credit
// every later caller, not only the one that paid for the walk.
type wpDiscovery struct {
installs []wpInstall
panelDomains map[string][]string
// incomplete records that discovery could not see the whole host. Findings
// from an incomplete scan must survive the cycle's purge.
incomplete bool
}
// apply credits the coverage gap to the check that asked for the installs.
// Discovery is shared; completeness is not -- every consumer needs the gap
// recorded under its own name or its findings are purged as if the scan had
// been complete.
func (d wpDiscovery) apply(ctx context.Context, gapCheck string) {
if !d.incomplete || gapCheck == "" {
return
}
markCheckIncomplete(ctx, gapCheck)
}
// wpInstalls returns every WordPress installation visible to CSM, honouring the
// account scope carried by ctx. gapCheck names the check credited with any
// coverage gap discovery hits.
func wpInstalls(ctx context.Context, gapCheck string) []wpInstall {
installs, _ := wpInstallsWithDomains(ctx, gapCheck)
return installs
}
// wpInstallsWithDomains additionally returns the panel's account-to-domain map,
// which the database scan needs for its tenant-boundary checks. The map is nil
// when the panel's own map could not be parsed completely: a partial ownership
// map turns a legitimate domain into a foreign-host finding.
func wpInstallsWithDomains(ctx context.Context, gapCheck string) ([]wpInstall, map[string][]string) {
d := lookupWPInstalls(ctx, "")
d.apply(ctx, gapCheck)
return d.installs, d.panelDomains
}
// wpInstallsForAccount restricts discovery to one account. Fix, drop and
// re-check paths use it: they run outside a scan context but act on the
// account named by the finding they are resolving.
func wpInstallsForAccount(ctx context.Context, gapCheck, account string) []wpInstall {
if !validAccountName.MatchString(account) {
return nil
}
d := lookupWPInstalls(ctx, account)
d.apply(ctx, gapCheck)
return d.installs
}
type wpInstallCacheKey struct{}
// wpInstallCache memoises discovery for the length of one scan cycle. Nine
// WordPress consumers run per cycle and each used to walk every account home
// for itself.
type wpInstallCache struct {
inventory wpInventoryCycle
started time.Time
mu sync.Mutex
byAccount map[string]wpDiscovery
}
// withWPInstallCache attaches the memo to a scan context. Only the runner does
// this: fix, drop and re-check paths build their own context, so they always
// re-discover, which is what a caller that mutates the tree it just walked
// needs. Invalidation is structural -- no cycle context, no cache.
func withWPInstallCache(ctx context.Context) context.Context {
if ctx == nil {
ctx = context.Background()
}
return context.WithValue(ctx, wpInstallCacheKey{}, &wpInstallCache{
byAccount: make(map[string]wpDiscovery),
started: time.Now(),
})
}
func wpInstallCacheFrom(ctx context.Context) *wpInstallCache {
if ctx == nil {
return nil
}
cache, _ := ctx.Value(wpInstallCacheKey{}).(*wpInstallCache)
return cache
}
// lookupWPInstalls answers from the cycle memo when there is one. The cached
// value carries its coverage gap, which every caller re-applies under its own
// check name.
func lookupWPInstalls(ctx context.Context, account string) wpDiscovery {
cache := wpInstallCacheFrom(ctx)
if cache == nil {
return discoverWPInstalls(ctx, account)
}
key := account
if key == "" {
key = AccountFromContext(ctx)
}
// The lock is held across discovery on purpose: two consumers starting at
// once should cost one walk, not two.
cache.mu.Lock()
defer cache.mu.Unlock()
if cached, ok := cache.byAccount[key]; ok {
return cached
}
d := discoverWPInstalls(ctx, account)
cache.byAccount[key] = d
return d
}
// discoverWPInstalls merges cPanel's document-root map with a walk of the
// account home layout. Neither source is sufficient alone: the map reaches
// roots no walk can predict, and the walk reaches roots the panel has stopped
// serving but whose database is still live.
func discoverWPInstalls(ctx context.Context, account string) wpDiscovery {
scope := account
if scope == "" {
scope = AccountFromContext(ctx)
}
var d wpDiscovery
if scope != "" && !validAccountName.MatchString(scope) {
d.incomplete = true
return d
}
seen := make(map[string]bool)
mappedRoots := make(map[string]bool)
panelDomains := make(map[string][]string)
add := func(path, owner string, state servedState, missingIsIncomplete bool) {
if seen[path] {
return
}
info, err := osFS.Lstat(path)
if err != nil {
if missingIsIncomplete || !errors.Is(err, fs.ErrNotExist) {
d.incomplete = true
}
return
}
if !info.Mode().IsRegular() {
// A symlinked wp-config.php is not scannable here. wp-cli runs as
// root and follows it, so an account pointing its config at another
// account's would have that tenant's database inventoried and
// attributed to this one. The install is dropped and the gap
// recorded instead.
d.incomplete = true
return
}
seen[path] = true
if owner == "" {
_, owner, _ = accountRootOf(path)
}
d.installs = append(d.installs, wpInstall{
ConfigPath: path,
DocRoot: filepath.Dir(path),
Account: owner,
Served: state,
})
}
// cPanel publishes its actual domain-to-document-root map. It is
// authoritative for served roots and reaches layouts the home walk below
// cannot see, so it is consulted first.
vhostData, vhostErr := osFS.ReadFile(userdataDomainsPath)
domainMapComplete := false
switch {
case vhostErr == nil:
vhosts, complete := parseUserdataDomainRootsChecked(string(vhostData))
wildcardVhosts, wildcardComplete := parseWildcardUserdataDomainRootsChecked(string(vhostData))
vhosts = append(vhosts, wildcardVhosts...)
domainMapComplete = complete && wildcardComplete && len(vhosts) > 0
if !domainMapComplete {
d.incomplete = true
}
domainOwners := make(map[string]string, len(vhosts))
for _, vh := range vhosts {
root := filepath.Clean(vh.docroot)
if !docrootBelongsToCPanelUser(root, vh.user) {
d.incomplete = true
domainMapComplete = false
continue
}
wildcard := strings.HasPrefix(vh.domain, "*.")
domain := normalizeHost(strings.TrimPrefix(vh.domain, "*."))
if domain == "" {
d.incomplete = true
domainMapComplete = false
} else {
domainKey := domain
if wildcard {
domainKey = "*." + domain
}
owner, exists := domainOwners[domainKey]
if exists && owner != vh.user {
// The map is meant to have one authoritative owner per
// domain. An ambiguous owner cannot safely support a
// tenant-boundary check.
d.incomplete = true
domainMapComplete = false
} else if !exists {
domainOwners[domainKey] = vh.user
panelDomains[vh.user] = append(panelDomains[vh.user], domainKey)
}
}
if scope != "" && vh.user != scope {
continue
}
mappedConfig := filepath.Join(root, "wp-config.php")
wpConfig, err := canonicalWPInstallPath(mappedConfig)
if err != nil {
if errors.Is(err, errWPDocrootMissing) {
// Check the original path too: a symlink target containing
// ".." can resolve differently from its cleaned form. Only
// the same absence accepted by add may permit retirement.
if _, statErr := osFS.Lstat(mappedConfig); errors.Is(statErr, fs.ErrNotExist) {
continue
}
}
d.incomplete = true
continue
}
mappedRoots[wpConfig] = true
add(wpConfig, vh.user, servedByPanel, false)
}
case vhostMapFailureIsIncomplete(vhostErr):
d.incomplete = true
}
if domainMapComplete {
d.panelDomains = panelDomains
}
// The served map is not sufficient on its own. A document root the panel
// has stopped serving still holds a live database, and the compromise this
// scan was widened for sat in exactly such a root -- absent from the domain
// map, from /etc/userdomains, and from vhost userdata alike. Re-pointing the
// domain publishes it again, so the home layout is walked whatever the panel
// says. Anything the map did not name is not served today, but only when the
// map could be read at all.
homeState := notServed
if !domainMapComplete {
homeState = servedUnknown
}
globScope := scope
if globScope == "" {
globScope = "*"
}
if scope != "" {
// The primary document root is checked by name, not by glob. A fixer
// asked about one account must not lose that account's main install
// because the glob returned nothing, and a missing file here is an
// account without WordPress, not a coverage gap.
primary := filepath.Join(accountHomeDir(scope), "public_html", "wp-config.php")
if _, err := osFS.Lstat(primary); err == nil {
canonical, resolveErr := canonicalWPInstallPath(primary)
if resolveErr != nil {
d.incomplete = true
} else {
add(canonical, scope, homeState, false)
}
} else if !errors.Is(err, fs.ErrNotExist) {
d.incomplete = true
}
}
for _, pattern := range []string{
filepath.Join(globScope, "public_html", "wp-config.php"),
filepath.Join(globScope, "public_html", "*", "wp-config.php"),
filepath.Join(globScope, "*", "wp-config.php"),
} {
matches, err := accountHomeGlob(pattern)
if err != nil {
d.incomplete = true
}
for _, path := range matches {
// A symlinked document root (cPanel's www -> public_html) resolves
// to its target first, so one install is not discovered twice and
// an alias is not mistaken for a directory that is never a root.
path, err = canonicalWPInstallPath(path)
if err != nil {
d.incomplete = true
continue
}
if seen[path] || skipWPDiscoveryPath(path) {
continue
}
state := homeState
if mappedRoots[path] {
// Preserve the panel's declaration even if the first Lstat
// failed and the file appeared before the home walk.
state = servedByPanel
}
add(path, "", state, true)
}
}
return d
}
// skipWPDiscoveryPath rejects candidates that are not document roots: account
// data directories, dot-directories, and backup, cache or staging copies.
func skipWPDiscoveryPath(path string) bool {
dir := filepath.Base(filepath.Dir(path))
if nonDocRootDirs[dir] || strings.HasPrefix(dir, ".") {
return true
}
home := wpInstallAccountRoot(path)
if home == "" {
return true
}
rel, err := filepath.Rel(home, path)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return true
}
// Skip names apply inside the account, not to the account name itself.
// cPanel permits accounts named backup, backups, cache, and staging.
for _, part := range strings.Split(rel, string(filepath.Separator)) {
if wpSkipDirNames[strings.ToLower(part)] {
return true
}
}
return false
}
// wpInstallConfigPaths projects installs to their wp-config.php paths for
// callers that work in paths rather than installs.
func wpInstallConfigPaths(installs []wpInstall) []string {
out := make([]string, 0, len(installs))
for _, in := range installs {
out = append(out, in.ConfigPath)
}
return out
}
package checks
import (
"container/heap"
"context"
"crypto/sha256"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/jstaint"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/phptaint"
"github.com/pidginhost/csm/internal/signatures"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
"github.com/pidginhost/csm/internal/yara"
)
var activeYARABackend = yara.Active
var yaraAvailable = yara.Available
// yaraDeepScanMu serializes the persisted host-wide cursors. Manual baseline
// scans can overlap the daemon's scheduled scan; without serialization, the
// older window can overwrite a newer cursor after it returns.
var yaraDeepScanMu sync.Mutex
// yaraDeepNow is indirected so tests can drive the soft-deadline clock.
var yaraDeepNow = time.Now
// yaraDeepDeadlineMargin is how much of the runner budget the walk leaves
// unused so a partial run returns its findings before the context deadline.
// A check that overruns its budget gets every returned finding dropped by
// the runner, so stopping early is the only way partial coverage survives.
const yaraDeepDeadlineMargin = 45 * time.Second
// yaraDeepFullCycleStale bounds how long rolling coverage may go without
// completing a full pass before the check surfaces a warning.
const yaraDeepFullCycleStale = 30 * 24 * time.Hour
// yaraDeepCursorCheck is the host-scope scan-cursor key (account "") under
// which the rolling deep-scan records the YARA consumer's progress.
const yaraDeepCursorCheck = "yara_deep"
// deepScanConsumer is one consumer of the shared ordered deep-content walk.
// Each consumer resumes from its own persisted cursor and advances it
// independently, so a missing backend cannot stall another analyzer's
// coverage and one analyzer's gap cannot move another's resume point.
type deepScanConsumer struct {
name string
dispatch bool
resume string
lastScanned string
cur store.ScanCursorRecord
// staleCheck and label name this consumer in its own staleness finding.
// Coverage degradation must be reported under the consumer that actually
// degraded, never under a sibling's identity.
staleCheck string
label string
}
// wants reports whether this consumer still needs the given path this cycle.
func (c *deepScanConsumer) wants(path string) bool {
return c.dispatch && (c.resume == "" || path > c.resume)
}
// advance is monotonic-max: a leading consumer keeps its resume point while
// the lagging one catches up over the shared walk.
func (c *deepScanConsumer) advance(path string) {
if c.dispatch && path > c.lastScanned {
c.lastScanned = path
}
}
// resetDeepScanCursor clears a disabled consumer's persisted cursor so
// re-enabling it starts a full cycle instead of resuming a stale window.
func resetDeepScanCursor(db *store.DB, check string) {
if db == nil {
return
}
rec, ok, err := db.GetScanCursor("", check)
if err != nil || !ok {
return
}
if rec.LastPath == "" && rec.WrappedAt.IsZero() && rec.LastFullCycleTS.IsZero() {
return
}
var next store.ScanCursorRecord
next.Check = check
if err := db.PutScanCursor(next); err != nil {
fmt.Fprintf(os.Stderr, "%s: cursor reset: %v\n", check, err)
}
}
// scheduledYARADetails describes one deep-scan match. A loader is only half
// a backdoor: the payload it pulls in lives elsewhere and survives a clean-up
// of the matched file alone, so any non-executable file this content includes
// is named alongside the rule.
func scheduledYARADetails(ruleName string, content []byte) string {
return fmt.Sprintf("Scheduled deep scan matched YARA rule %s", ruleName) +
signatures.ReferencedPayloadDetail(content)
}
// scheduledYARAFinding is the finding for one deep-scan match. contentSHA256
// names the bytes the backend actually scanned.
func scheduledYARAFinding(match yara.Match, path string, content []byte, contentSHA256 string) alert.Finding {
return alert.Finding{
Severity: yaraMatchSeverity(match.Meta["severity"]),
Check: "yara_match_scheduled",
Message: fmt.Sprintf("YARA rule match [%s]: %s", match.RuleName, path),
Details: scheduledYARADetails(match.RuleName, content),
FilePath: path,
ContentSHA256: contentSHA256,
DetectLogic: ContentDetectionVersion(),
CoverageScope: deepFileCoverageScope(path),
}
}
// deepFileCoverageScope names one file as the unit a partial deep-scan run
// retires once its window has examined that path.
func deepFileCoverageScope(path string) string {
return "file:" + path
}
type yaraDeepScanEntry struct {
path string
sortKey string
info os.FileInfo
err error
inspected bool
directory bool
}
type yaraDeepScanHeap []yaraDeepScanEntry
func (h yaraDeepScanHeap) Len() int { return len(h) }
func (h yaraDeepScanHeap) Less(i, j int) bool { return h[i].sortKey < h[j].sortKey }
func (h yaraDeepScanHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *yaraDeepScanHeap) Push(value any) { *h = append(*h, value.(yaraDeepScanEntry)) }
func (h *yaraDeepScanHeap) Pop() any {
old := *h
last := len(old) - 1
value := old[last]
old[last] = yaraDeepScanEntry{}
*h = old[:last]
return value
}
// CheckYARADeep is the shared rolling deep-content walk. It reads each file
// once and dispatches the same in-memory snapshot to three consumers with
// independent cursors and completion records: the YARA backend and the JS and
// PHP taint analyzers. A missing, disabled, or failed consumer neither prevents
// nor controls another consumer's progress.
func CheckYARADeep(ctx context.Context, cfg *config.Config, st *state.Store) []alert.Finding {
yaraDeepScanMu.Lock()
defer yaraDeepScanMu.Unlock()
if ctx.Err() != nil {
return nil
}
db := store.Global()
w := &yaraDeepWalk{
ctx: ctx,
yaraConsumer: &deepScanConsumer{name: yaraDeepCursorCheck, staleCheck: "yara_scan_incomplete", label: "YARA"},
jsConsumer: &deepScanConsumer{name: jsTaintDeepCursorCheck, staleCheck: "js_taint_scan_incomplete", label: "JS taint"},
phpConsumer: &deepScanConsumer{name: phpTaintDeepCursorCheck, staleCheck: "php_taint_scan_incomplete", label: "PHP taint"},
}
w.consumers = []*deepScanConsumer{w.yaraConsumer, w.jsConsumer, w.phpConsumer}
yaraOff := yaraDeepConsumerDisabled(cfg)
jsOff := jsTaintDeepConsumerDisabled(cfg)
w.jsConsumer.dispatch = !jsOff
phpOff := phpTaintDeepConsumerDisabled(cfg)
phpReady := phpTaintAnalyzerReady()
// An absent isolated analyzer is not a coverage gap: it means the feature
// is not active on this host. Dispatching anyway would record every
// candidate as unexamined on every scan.
w.phpConsumer.dispatch = !phpOff && phpReady
if !phpOff && !phpReady {
// Keep prior state while the feature is inactive. Treating an inactive
// owner as completed would purge findings without examining their files.
markCheckIncomplete(ctx, logicalOwnerPHPTaintDeep)
}
// Disabled-consumer resets are persistent scan progress too. Commit them
// only on a normal return path so a hard-canceled shared walk writes no
// cursor state for any consumer.
resetDisabledCursors := func() {
if yaraOff {
resetDeepScanCursor(db, yaraDeepCursorCheck)
}
if jsOff {
resetDeepScanCursor(db, jsTaintDeepCursorCheck)
}
if phpOff {
resetDeepScanCursor(db, phpTaintDeepCursorCheck)
}
}
w.backend = activeYARABackend()
yaraReady := w.backend != nil && w.backend.RuleCount() > 0
w.yaraConsumer.dispatch = !yaraOff && yaraReady
if !yaraOff && !yaraReady {
// A missing backend is a YARA coverage gap, never a JS one: mark only
// the YARA owner incomplete and leave its cursor unchanged so it can
// neither purge nor skip its own unscanned range, while the JS pass
// below still covers its cycle.
markCheckIncomplete(ctx, "yara_deep")
if yaraAvailable() {
w.findings = append(w.findings, alert.Finding{
Severity: alert.High,
Check: "yara_scan_incomplete",
Message: "YARA deep scan could not start because no compiled rules are available",
})
}
}
if !w.yaraConsumer.dispatch && !w.jsConsumer.dispatch && !w.phpConsumer.dispatch {
if ctx.Err() != nil {
return nil
}
resetDisabledCursors()
return w.findings
}
w.maxBytes = int64(FullScanMaxFileBytes(cfg))
w.jsMaxBytes = int64(jstaint.MaxSourceBytes)
w.phpMaxBytes = int64(phptaint.MaxSourceBytes)
for _, c := range w.consumers {
if !c.dispatch || db == nil {
continue
}
rec, ok, err := db.GetScanCursor("", c.name)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: cursor read: %v\n", c.name, err)
} else if ok {
c.cur = rec
}
c.resume = c.cur.LastPath
c.lastScanned = c.resume
}
rescanGen := pendingSignatureRescan(cfg, db, w.yaraConsumer.dispatch)
// The shared walk starts at the earliest dispatchable resume point; a
// consumer whose cursor is ahead skips already-covered paths via wants()
// while the lagging consumer catches up on the same snapshot reads.
firstResume := true
for _, c := range w.consumers {
if !c.dispatch {
continue
}
if firstResume || c.resume < w.walkResume {
w.walkResume = c.resume
firstResume = false
}
}
if deadline, ok := ctx.Deadline(); ok {
w.softDeadline = deadline.Add(-yaraDeepDeadlineMargin)
}
// The collector owns both the count and the per-status breakdown, so the
// finding can say what the gaps were instead of only how many.
w.yaraGaps = newYARAGapCollector()
w.yaraGaps.resolveAliases = w.resolveObservedAliases
w.jsGaps = newJSTaintGapCollector()
w.phpGaps = newPHPTaintGapCollector()
w.jsGaps.resolveAliases = w.resolveObservedAliases
w.phpGaps.resolveAliases = w.resolveObservedAliases
w.jsGaps.recordCoverage = func(paths []string) {
recordCoverageGapPaths(ctx, logicalOwnerJSTaintDeep, paths)
}
w.phpGaps.recordCoverage = func(paths []string) {
recordCoverageGapPaths(ctx, logicalOwnerPHPTaintDeep, paths)
}
roots, rootsComplete := deepScanRoots(cfg)
// Restarting the walk on updates would starve paths that sort last.
// Track a lap from the current position instead. Changed or unresolved
// roots invalidate that proof: a new root behind the cursor may not have
// been visited on the return lap. The walk itself keeps its position.
rootsHash := ""
if rootsComplete {
rootsHash = fmt.Sprintf("%x", sha256.Sum256([]byte(strings.Join(roots, "\x00"))))
}
if (rescanGen != 0 && w.yaraConsumer.cur.RulesGeneration != rescanGen) ||
(w.yaraConsumer.cur.RulesGeneration != 0 && (w.yaraConsumer.cur.RulesRootsHash != rootsHash || !rootsComplete)) {
if rescanGen != 0 {
w.yaraConsumer.cur.RulesGeneration = rescanGen
}
w.yaraConsumer.cur.RulesFrom = w.yaraConsumer.resume
w.yaraConsumer.cur.RulesWrapped = false
w.yaraConsumer.cur.RulesRootsHash = rootsHash
}
w.roots = roots
w.walk(roots)
if ctx.Err() != nil {
// The runner drops every finding a check returns after its budget
// expired, so there is nothing worth reporting; leave every cursor
// untouched and let the next run redo this window.
return nil
}
resetDisabledCursors()
now := yaraDeepNow().UTC()
w.saveCursors(db, now, rootsComplete)
if rescanGen != 0 && rootsComplete {
w.clearSignatureRescan(db, rescanGen)
}
w.reportStaleCycles(now)
w.reportCoverage(st)
return w.findings
}
// yaraDeepWalk is one run of the shared deep-content walk: the three
// consumers, their gap collectors, and the single ordered traversal they
// share. The fields below "shared walk state" describe that one traversal.
type yaraDeepWalk struct {
ctx context.Context
backend yara.Backend
yaraConsumer, jsConsumer, phpConsumer *deepScanConsumer
consumers []*deepScanConsumer
maxBytes, jsMaxBytes, phpMaxBytes int64
yaraGaps *yaraGapCollector
jsGaps *jsTaintGapCollector
phpGaps *phpTaintGapCollector
findings []alert.Finding
// Shared walk state. walkResume is the earliest dispatching resume point.
roots []string
walkResume string
softDeadline time.Time
stoppedEarly bool
// observedPath and observedInfo are the identity last observed for a
// path, so a later failure on it can be pinned to that exact file.
observedPath string
observedInfo os.FileInfo
}
// deepFileWants records which consumers still need the file being read.
type deepFileWants struct{ yara, js, php bool }
func (fw deepFileWants) any() bool { return fw.yara || fw.js || fw.php }
func (w *yaraDeepWalk) outOfTime() bool {
return !w.softDeadline.IsZero() && !yaraDeepNow().Before(w.softDeadline)
}
func (w *yaraDeepWalk) observePathIdentity(path string, info os.FileInfo) {
w.observedPath = path
w.observedInfo = info
}
func (w *yaraDeepWalk) resolveObservedAliases(path string) ([]string, bool) {
if path != w.observedPath || w.observedInfo == nil {
return coveragePathAliases(path), false
}
return stableCoveragePathAliases(path, w.observedInfo)
}
func (w *yaraDeepWalk) pathFailureStillAttributable(path string) (os.FileInfo, bool) {
info, err := osFS.Lstat(path)
// Every metadata error is ambiguous, including ENOENT: a delete can race
// this observation and replace the path with a directory immediately after
// it. Only an observed non-directory, non-symlink entry is narrow enough to
// preserve by exact path.
return info, err == nil && !info.IsDir() && info.Mode()&os.ModeSymlink == 0 &&
w.observedPath == path && w.observedInfo != nil && os.SameFile(w.observedInfo, info)
}
func (w *yaraDeepWalk) recordYARAPathGap(path, status string) {
recordCoverageGapPaths(w.ctx, "yara_deep", w.yaraGaps.record(path, status))
}
func (w *yaraDeepWalk) recordYARAScanGap(path string, scanErr error) {
// Inline IPC overflow retries by path. If that path changed into a
// directory or symlink before the retry failed, the unscanned range is no
// longer just the original file: children or a target subtree may now be
// hidden. The post-failure Lstat is inherently racy, so every metadata
// error takes the conservative unknown-range path.
info, attributable := w.pathFailureStillAttributable(path)
if !attributable {
w.yaraGaps.recordUnknownRange(fmt.Sprintf("scanning %s failed after the path changed: %v", path, scanErr))
return
}
w.observePathIdentity(path, info)
w.recordYARAPathGap(path, "scan_error")
}
// recordUnknownRange records a loss whose affected paths are unknowable for
// every consumer in fw.
func (w *yaraDeepWalk) recordUnknownRange(fw deepFileWants, detail string) {
if fw.yara {
w.yaraGaps.recordUnknownRange(detail)
}
if fw.js {
w.jsGaps.recordUnknownRange(detail)
}
if fw.php {
w.phpGaps.recordUnknownRange(detail)
}
}
// recordPathGap records an exact-path gap for every consumer in fw. YARA
// keeps its own status where its report names the failure differently.
func (w *yaraDeepWalk) recordPathGap(fw deepFileWants, path, yaraStatus, status string) {
if fw.yara {
w.recordYARAPathGap(path, yaraStatus)
}
if fw.js {
w.jsGaps.record(path, status)
}
if fw.php {
w.phpGaps.record(path, status)
}
}
func (w *yaraDeepWalk) advanceAll(path string) {
for _, c := range w.consumers {
c.advance(path)
}
}
// deepSubtreePrefix is the sort key every path under dir starts with.
func deepSubtreePrefix(path string) string {
sep := string(filepath.Separator)
path = filepath.Clean(path)
if strings.HasSuffix(path, sep) {
return path
}
return path + sep
}
// deepSubtreeCoveredAfter reports whether every path under dir sorts before
// resume. Children all share the dir+sep prefix, so when resume does not
// itself start with that prefix, any child compares to resume exactly as the
// prefix does. Comparing the bare dir path instead would wrongly skip
// siblings like "ab/" when the cursor sits at "ab.zz" ('/' sorts after '.').
// An exact prefix cursor records a subtree already accounted for this cycle,
// including an empty or unreadable directory.
func deepSubtreeCoveredAfter(resume, dir string) bool {
prefix := deepSubtreePrefix(dir)
return resume != "" && (resume == prefix || (!strings.HasPrefix(resume, prefix) && prefix < resume))
}
// deepScanRoots returns the web roots deduplicated and in walk order, and
// whether every root resolved.
func deepScanRoots(cfg *config.Config) ([]string, bool) {
roots, rootsComplete := resolveWebRootsChecked(cfg)
normalizedRoots := roots[:0]
seenRoots := make(map[string]struct{}, len(roots))
for _, root := range roots {
root = filepath.Clean(root)
if _, exists := seenRoots[root]; exists {
continue
}
seenRoots[root] = struct{}{}
normalizedRoots = append(normalizedRoots, root)
}
roots = normalizedRoots
sort.Slice(roots, func(i, j int) bool { return deepSubtreePrefix(roots[i]) < deepSubtreePrefix(roots[j]) })
return roots, rootsComplete
}
func (w *yaraDeepWalk) walk(roots []string) {
// All roots share one frontier. Walking an outer root to completion
// first can pass a nested symlink root that only an explicit root visit
// follows, falsely placing its unread files inside the retirement window.
ordered := make(yaraDeepScanHeap, len(roots))
rootSet := make(map[string]bool, len(roots))
for i, root := range roots {
rootSet[root] = true
ordered[i] = yaraDeepScanEntry{path: root, sortKey: deepSubtreePrefix(root), inspected: true, directory: true}
}
heap.Init(&ordered)
for ordered.Len() > 0 {
if w.ctx.Err() != nil || w.stoppedEarly {
return
}
if w.outOfTime() {
w.stoppedEarly = true
return
}
item := heap.Pop(&ordered).(yaraDeepScanEntry)
if !item.inspected {
item.info, item.err = osFS.Lstat(item.path)
item.inspected = true
if item.err == nil && item.info.Mode()&os.ModeSymlink == 0 && item.info.IsDir() {
// Bare names are lower bounds. Children of ab/ sort after
// ab.zz, so a discovered directory must be requeued.
item.directory = true
item.sortKey = deepSubtreePrefix(item.path)
heap.Push(&ordered, item)
continue
}
}
if item.directory {
if !deepSubtreeCoveredAfter(w.walkResume, item.path) {
w.enqueueDir(item.path, &ordered, rootSet)
}
continue
}
w.scanFile(item.path, item.info, item.err)
}
}
func (w *yaraDeepWalk) enqueueDir(dir string, ordered *yaraDeepScanHeap, roots map[string]bool) {
entries, err := osFS.ReadDir(dir)
if err != nil {
// The affected paths are unknowable, but a consumer that already
// covered this subtree must not inherit another consumer's gap.
w.recordUnknownRange(deepFileWants{
yara: w.yaraConsumer.dispatch && !deepSubtreeCoveredAfter(w.yaraConsumer.resume, dir),
js: w.jsConsumer.dispatch && !deepSubtreeCoveredAfter(w.jsConsumer.resume, dir),
php: w.phpConsumer.dispatch && !deepSubtreeCoveredAfter(w.phpConsumer.resume, dir),
}, fmt.Sprintf("reading %s: %v", dir, err))
w.advanceAll(deepSubtreePrefix(dir))
return
}
if len(entries) == 0 {
// A prefix cursor covers the entire subtree, so it is safe before
// visiting children only when there are none. Nonempty directories
// advance through their children, whose paths exceed this prefix.
w.advanceAll(deepSubtreePrefix(dir))
}
for _, entry := range entries {
path := filepath.Join(dir, entry.Name())
if !roots[path] {
heap.Push(ordered, yaraDeepScanEntry{path: path, sortKey: path})
}
}
}
// scanFile hands one walked non-directory entry to every consumer that
// still needs it. It sets stoppedEarly when the soft deadline passes before
// the read.
func (w *yaraDeepWalk) scanFile(path string, info os.FileInfo, lstatErr error) {
fw := deepFileWants{
yara: w.yaraConsumer.wants(path),
js: w.jsConsumer.wants(path),
php: w.phpConsumer.wants(path),
}
if !fw.any() {
return
}
if lstatErr != nil {
// A failed Lstat may hide a directory, so the gap range is
// unknowable for either path-based carry-forward.
w.recordUnknownRange(fw, fmt.Sprintf("inspecting %s: %v", path, lstatErr))
w.advanceAll(path)
return
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() || info.Size() == 0 {
w.advanceAll(path)
return
}
w.observePathIdentity(path, info)
if fw.yara && info.Size() > w.maxBytes {
// Oversize files are intentionally not opened, but they still
// count as covered progress for this rolling cycle. The gap
// advances only its own consumer: a soft-deadline stop right
// after this gate must not move the other consumer past a
// file it never received.
w.recordYARAPathGap(path, "oversize")
w.yaraConsumer.advance(path)
fw.yara = false
}
if fw.php && info.Size() > w.phpMaxBytes {
// An oversize file is only a PHP coverage gap if it could have
// been PHP. The deep walk hands every readable file to this
// consumer and the analyzer's own pre-filter normally rejects
// the rest instantly -- but that pre-filter never runs on a
// file too large to send, so without this check every large
// error_log, JSON blob and media file on the host is reported
// as PHP content we failed to examine. Measured on a live
// host: 617 of 660 recorded gaps in one scan, which buries the
// handful that are real.
// Past the soft deadline the walk is stopping and must do
// no further I/O, so the peek is skipped and the gap is
// recorded unexamined -- the same answer an unreadable
// file gets.
if w.outOfTime() || phpFileMayBePHP(path, info) {
w.phpGaps.record(path, phptaint.StatusOversize.String())
}
w.phpConsumer.advance(path)
fw.php = false
}
if fw.js && info.Size() > w.jsMaxBytes {
// An oversize file is only a JS coverage gap if it could
// have been JavaScript. The deep walk hands every readable
// file to this consumer and the analyzer's own pre-filter
// normally rejects the rest instantly -- but that pre-filter
// never runs on a file too large to send, so without this
// check every large image, archive and database on the host
// is reported as JavaScript we failed to examine. This gate
// was missing while the PHP one beside it existed.
//
// This branch used to decide on metadata alone. Peeking
// costs an open, so as with PHP above it is skipped once the
// soft deadline has passed and the gap is recorded
// unexamined.
if w.outOfTime() || jsFileMayBeJS(path, info) {
w.jsGaps.record(path, jstaint.StatusOversize.String())
}
w.jsConsumer.advance(path)
fw.js = false
}
if !fw.any() {
return
}
if w.outOfTime() {
w.stoppedEarly = true
return
}
// Advance before opening so permanently unreadable or unscannable
// files cannot wedge the cursor in place.
w.advanceAll(path)
data, ok := w.readSnapshot(path, &fw)
if !ok {
return
}
fingerprint := sha256.Sum256(data)
contentSHA256 := fmt.Sprintf("%x", fingerprint)
if fw.js {
w.findings = append(w.findings, analyzeJSTaintSnapshot(w.ctx, path, contentSHA256, data, w.jsGaps)...)
}
if fw.php {
w.findings = append(w.findings, analyzePHPTaintSnapshot(w.ctx, path, contentSHA256, data, w.phpGaps)...)
w.phpConsumer.advance(path)
}
if !fw.yara {
return
}
matches, scanErr := yara.ScanBytesChecked(w.backend, path, data)
if scanErr != nil {
w.recordYARAScanGap(path, scanErr)
return
}
for _, match := range matches {
w.findings = append(w.findings, scheduledYARAFinding(match, path, data, contentSHA256))
}
}
// readSnapshot reads path once, capped at the largest limit among the
// consumers in fw. A consumer the file no longer fits, or that cannot get a
// stable snapshot, records its gap and leaves fw. ok is false when no
// consumer is left to receive the content.
func (w *yaraDeepWalk) readSnapshot(path string, fw *deepFileWants) ([]byte, bool) {
file, err := osFS.Open(path)
if err != nil {
if info, attributable := w.pathFailureStillAttributable(path); attributable {
w.observePathIdentity(path, info)
w.recordPathGap(*fw, path, "open_error", "read_error")
} else {
w.recordUnknownRange(*fw, fmt.Sprintf("opening %s: %v", path, err))
}
return nil, false
}
openedInfo, statErr := file.Stat()
if statErr != nil {
_ = file.Close()
w.recordUnknownRange(*fw, fmt.Sprintf("inspecting opened path %s: %v", path, statErr))
return nil, false
}
if !openedInfo.Mode().IsRegular() {
_ = file.Close()
if openedInfo.IsDir() {
// Lstat admitted a regular file, but Open bound a directory.
// Its children were never enumerated, so retaining only the
// exact path would purge findings from an unknown subtree.
w.recordUnknownRange(*fw, fmt.Sprintf("%s changed into a directory while it was being opened", path))
} else {
w.recordPathGap(*fw, path, "changed_during_read", "changed_during_read")
}
return nil, false
}
w.observePathIdentity(path, openedInfo)
if fw.yara && openedInfo.Size() > w.maxBytes {
w.recordYARAPathGap(path, "changed_during_read")
fw.yara = false
}
if fw.php && openedInfo.Size() > w.phpMaxBytes {
w.phpGaps.record(path, "changed_during_read")
fw.php = false
}
if fw.js && openedInfo.Size() > w.jsMaxBytes {
w.jsGaps.record(path, "changed_during_read")
fw.js = false
}
if !fw.any() {
_ = file.Close()
return nil, false
}
readCap := int64(0)
if fw.yara {
readCap = w.maxBytes
}
if fw.js && w.jsMaxBytes > readCap {
readCap = w.jsMaxBytes
}
if fw.php && w.phpMaxBytes > readCap {
readCap = w.phpMaxBytes
}
data, readErr := io.ReadAll(io.LimitReader(file, readCap+1))
closeErr := file.Close()
if readErr != nil || closeErr != nil || int64(len(data)) > readCap {
w.recordPathGap(*fw, path, "read_error", "read_error")
return nil, false
}
if fw.yara && int64(len(data)) > w.maxBytes {
w.recordYARAPathGap(path, "read_error")
fw.yara = false
}
if fw.php && int64(len(data)) > w.phpMaxBytes {
w.phpGaps.record(path, "changed_during_read")
fw.php = false
}
if fw.js && int64(len(data)) > w.jsMaxBytes {
w.jsGaps.record(path, "changed_during_read")
fw.js = false
}
return data, true
}
// saveCursors persists each dispatching consumer's progress. A completed
// walk stamps a full cycle; a stopped one keeps its resume point and the
// start of its partial cycle.
func (w *yaraDeepWalk) saveCursors(db *store.DB, now time.Time, rootsComplete bool) {
if db == nil {
return
}
for _, c := range w.consumers {
if !c.dispatch {
continue
}
var next store.ScanCursorRecord
next.Check = c.name
next.RulesGeneration = c.cur.RulesGeneration
next.RulesFrom = c.cur.RulesFrom
next.RulesWrapped = c.cur.RulesWrapped || (!w.stoppedEarly && rootsComplete && c.cur.RulesGeneration != 0)
next.RulesRootsHash = c.cur.RulesRootsHash
if w.stoppedEarly {
next.LastPath = c.lastScanned
next.LastFullCycleTS = c.cur.LastFullCycleTS
next.WrappedAt = c.cur.WrappedAt
if next.WrappedAt.IsZero() {
next.WrappedAt = now
}
} else {
next.LastFullCycleTS = now
}
if err := db.PutScanCursor(next); err != nil {
fmt.Fprintf(os.Stderr, "%s: cursor write: %v\n", c.name, err)
}
}
}
// clearSignatureRescan retires the queued rescan once the YARA walk has
// passed every file since the rules update that queued it.
func (w *yaraDeepWalk) clearSignatureRescan(db *store.DB, rescanGen uint64) {
// A walk that started from where the update found it has passed every
// file once it completes. Otherwise it must have wrapped and come back
// to that point. A stopped walk cannot have wrapped during this run.
cur := w.yaraConsumer.cur
reachedFrom := w.yaraConsumer.lastScanned >= cur.RulesFrom
if strings.HasSuffix(cur.RulesFrom, string(filepath.Separator)) {
// A prefix cursor covers every child, not just the first path
// that sorts after the prefix. Use the walk's subtree ordering.
reachedFrom = deepSubtreeCoveredAfter(w.yaraConsumer.lastScanned, cur.RulesFrom)
}
if (!w.stoppedEarly && (cur.RulesFrom == "" || cur.RulesWrapped)) ||
(cur.RulesWrapped && reachedFrom) {
cleared, err := db.ClearSignatureRescan(rescanGen)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: clearing signature rescan: %v\n", w.yaraConsumer.name, err)
} else if cleared {
observeSignatureRescan()
}
}
}
// reportStaleCycles warns for each consumer whose rolling cycle has gone too
// long without a full pass.
//
// Rolling-cycle staleness is per consumer. All three keep their own cursor
// and their own WrappedAt, so the warning belongs to whichever cycle
// actually stalled. This was read from the YARA cursor alone, which left a
// stalled PHP or JS cycle silent, and silenced the warning entirely when
// yara_deep was disabled -- a supported configuration.
//
// Enumerated once so the shape is not rediscovered one field at a time.
// Persisted per-consumer state:
//
// cur.LastPath becomes resume and lastScanned. walkResume is the minimum
// active resume; wants and the per-consumer subtree check still gate each
// consumer, and advance plus the cursor write update only that consumer's
// progress.
// cur.WrappedAt starts that consumer's partial cycle and drives only its
// staleCheck/label warning here.
// cur.LastFullCycleTS is preserved or stamped in that consumer's record.
//
// Runtime per-consumer state:
//
// dispatch comes from that consumer's disable/readiness gates and controls
// its cursor, analysis, completion mark, status finding, and carry-forward.
// deepFileWants and the three size limits govern only the matching
// consumer. The shared read cap is their maximum, so no sibling can reduce
// another's snapshot.
// yaraGaps belongs only to YARA; jsGaps only to JS; phpGaps only to PHP.
// Each is reported and used for completion/carry-forward only in its owner
// block of reportCoverage.
//
// Genuinely shared state:
//
// roots, path order, the context/soft deadline, opened snapshots, and
// stoppedEarly describe the one walk. After a consumer's resume point all
// active consumers have the same remaining ordered path space, so a walk
// that stops early cannot have completed one active consumer's cycle while
// leaving another active consumer's cycle unfinished.
//
// Known and deliberately out of scope: php_content_rolling.go writes
// WrappedAt for its per-account cursor and nothing reads it, so that scan
// has no staleness warning of its own.
func (w *yaraDeepWalk) reportStaleCycles(now time.Time) {
if !w.stoppedEarly {
return
}
for _, c := range w.consumers {
if !c.dispatch || c.cur.WrappedAt.IsZero() || now.Sub(c.cur.WrappedAt) <= yaraDeepFullCycleStale {
continue
}
w.findings = append(w.findings, alert.Finding{
Severity: alert.Warning,
Check: c.staleCheck,
Message: fmt.Sprintf("Rolling %s deep scan has not completed a full pass since %s", c.label, c.cur.WrappedAt.Format("2006-01-02")),
Details: "Each run advances the cursor inside its time budget; a pass this stale means the budget is too small for the content volume.",
})
}
}
// reportCoverage closes each dispatching consumer's window: it marks a
// partial window incomplete, reports gaps, and carries forward prior
// findings for known-path gaps when the window may replace the finding set.
func (w *yaraDeepWalk) reportCoverage(st *state.Store) {
if w.yaraConsumer.dispatch {
// Same contract the PHP and JS consumers use. Only a partial window or
// an unknown-range loss suppresses the purge; a gap that names a file
// is handled by carrying that file's prior finding forward, so the
// scan can still retire everything it did examine. Reporting a bare
// count here is what made one permanently unreadable file look
// identical to a lost subtree, and froze every YARA finding on a host.
yaraPartial := w.stoppedEarly || w.yaraConsumer.resume != "" || w.yaraGaps.pathsIncomplete()
if yaraPartial {
markCheckIncomplete(w.ctx, "yara_deep")
}
if !w.yaraGaps.empty() {
w.findings = append(w.findings, w.yaraGaps.finding())
}
if yaraPartial && !w.yaraGaps.pathsIncomplete() && st != nil {
recordCompletedCoverageScopes(w.ctx, "yara_deep",
w.examinedScopes(w.yaraConsumer, st.LatestFindings(), "yara_match_scheduled", w.yaraGaps.hasPath))
}
if !yaraPartial && st != nil {
// Keep the stable carried snapshots in-band so alert deduplication can
// retain its normal ongoing-finding re-alert cadence. Daemon callers
// also publish the path set for race-safe latest-state persistence.
w.findings = append(w.findings, carryForwardYARAFindings(st.LatestFindings(), w.yaraGaps)...)
}
}
if w.phpConsumer.dispatch {
phpPartial := w.stoppedEarly || w.phpConsumer.resume != "" || w.phpGaps.pathsIncomplete()
if phpPartial {
// Only a partial or unknown-range window suppresses the normal
// purge; a known-path gap is handled by the carry-forward below.
markCheckIncomplete(w.ctx, logicalOwnerPHPTaintDeep)
}
if !w.phpGaps.empty() {
w.findings = append(w.findings, w.phpGaps.findings()...)
}
if phpPartial && !w.phpGaps.pathsIncomplete() && st != nil {
recordCompletedCoverageScopes(w.ctx, logicalOwnerPHPTaintDeep,
w.examinedScopes(w.phpConsumer, st.LatestFindings(), "php_remote_taint", w.phpGaps.hasPath))
}
if !phpPartial && st != nil {
// Path-specific carry-forward: this run is eligible to replace the
// PHP finding set, so a known-path coverage gap must re-emit that
// path's prior finding or the purge would clear it. A completed
// negative or missing path stays cleared.
w.findings = append(w.findings, carryForwardPHPTaintFindings(st.LatestFindings(), w.phpGaps)...)
}
}
if w.jsConsumer.dispatch {
jsPartial := w.stoppedEarly || w.jsConsumer.resume != "" || w.jsGaps.pathsIncomplete()
if jsPartial {
// Only a partial or unknown-range window suppresses the normal JS
// purge; a known-path gap is handled by the carry-forward below.
markCheckIncomplete(w.ctx, logicalOwnerJSTaintDeep)
}
if !w.jsGaps.empty() {
w.findings = append(w.findings, w.jsGaps.finding())
}
if jsPartial && !w.jsGaps.pathsIncomplete() && st != nil {
recordCompletedCoverageScopes(w.ctx, logicalOwnerJSTaintDeep,
w.examinedScopes(w.jsConsumer, st.LatestFindings(), "js_keylogger_dataflow", w.jsGaps.hasPath))
}
if !jsPartial && st != nil {
// Path-specific carry-forward: this run is eligible to replace
// the JS finding set, so a known-path coverage gap must re-emit
// that path's prior finding or the purge would clear it. A
// completed negative or missing path stays cleared.
w.findings = append(w.findings, carryForwardJSTaintFindings(st.LatestFindings(), w.jsGaps)...)
}
}
}
// examinedScopes returns the coverage scopes of prior check findings whose
// file this run's window for c examined: under a walked root, past the
// consumer's resume point, and no further than where it stopped. A partial
// run then retires exactly what a complete run would within that window. A
// gap path was not examined, so it keeps its finding.
func (w *yaraDeepWalk) examinedScopes(c *deepScanConsumer, prior []alert.Finding, check string, gap func(string) bool) map[string]bool {
scopes := make(map[string]bool)
for _, f := range prior {
if f.Check != check || f.FilePath == "" || f.CoverageScope != deepFileCoverageScope(f.FilePath) {
continue
}
if !w.underRoot(f.FilePath) || deepPathCoveredBy(c.resume, f.FilePath) ||
(w.stoppedEarly && !deepPathCoveredBy(c.lastScanned, f.FilePath)) || gap(f.FilePath) {
continue
}
scopes[f.CoverageScope] = true
}
return scopes
}
func (w *yaraDeepWalk) underRoot(path string) bool {
for _, root := range w.roots {
if strings.HasPrefix(path, deepSubtreePrefix(root)) {
return true
}
}
return false
}
// deepPathCoveredBy reports whether a cursor has already passed path in walk
// order: a cursor ending in a separator covers that whole subtree, and any
// other cursor covers every path up to and including itself.
func deepPathCoveredBy(cursor, path string) bool {
if cursor == "" {
return false
}
if strings.HasSuffix(cursor, string(filepath.Separator)) && strings.HasPrefix(path, cursor) {
return true
}
return path <= cursor
}
func yaraMatchSeverity(value string) alert.Severity {
switch strings.ToLower(value) {
case "warning", "low", "medium":
return alert.Warning
case "high":
return alert.High
default:
return alert.Critical
}
}
var (
sigRescansTotalOnce sync.Once
sigRescansTotal *metrics.Counter
)
// observeSignatureRescan counts a queued rescan cleared because the walk
// has scanned every file since the rules update that queued it.
func observeSignatureRescan() {
sigRescansTotalOnce.Do(func() {
sigRescansTotal = metrics.NewCounter(
"csm_signature_rescans_total",
"Rescans queued by a signature update that completed. Incremented when the rolling deep YARA walk has scanned every file since the update.",
)
metrics.MustRegister("csm_signature_rescans_total", sigRescansTotal)
})
sigRescansTotal.Inc()
}
// pendingSignatureRescan returns the queued rescan generation the YARA walk
// should cover, or 0 when there is none to track. Disabled rescans pause the
// queue, and an unreadable queue is left for the watcher to repair.
func pendingSignatureRescan(cfg *config.Config, db *store.DB, dispatch bool) uint64 {
if !dispatch || db == nil || !cfg.SignatureRescanEnabled() {
return 0
}
gen, err := db.SignatureRescanPending()
if err != nil {
fmt.Fprintf(os.Stderr, "%s: reading signature rescan: %v\n", yaraDeepCursorCheck, err)
return 0
}
return gen
}
package checks
import (
"sort"
"github.com/pidginhost/csm/internal/alert"
)
// maxYARAGapPaths bounds the exact-path set a single run retains. Past it the
// count keeps rising but the paths are no longer authoritative, which forces
// the run partial rather than letting a purge clear a path it can no longer
// name.
const maxYARAGapPaths = 5000
const yaraGapExampleMaxBytes = 200
// yaraGapCollector records what a YARA deep scan walked but could not examine.
// It mirrors the PHP and JS taint collectors deliberately: those two consumers
// already distinguish a gap that names a file from one that loses an unknown
// range, and YARA reporting only a bare count is what made a permanently
// unreadable file indistinguishable from a lost subtree.
type yaraGapCollector struct {
gapPathSet
}
func newYARAGapCollector() *yaraGapCollector {
// The default resolver must stay symmetric with hasPath, which queries by
// every alias. Recording only the lexical spelling meant a finding stored
// through a symlinked document root never matched its own gap, and was
// purged for a file the scan never read.
return &yaraGapCollector{gapPathSet: newGapPathSet(maxYARAGapPaths, yaraGapExampleMaxBytes,
func(path string) ([]string, bool) { return coveragePathAliases(path), true })}
}
// record notes one file the scan could not examine, under a status naming why.
// It returns the aliases captured for an authoritative retained path so the
// atomic path-scoped purge uses exactly the same identity as carry-forward.
func (g *yaraGapCollector) record(path, status string) []string {
return g.retain(path, status)
}
func (g *yaraGapCollector) finding() alert.Finding {
return g.incompleteFinding(alert.High, "yara_scan_incomplete",
"YARA deep scan could not inspect %d file(s)",
"YARA deep scan could not cover %d location(s)")
}
// carryForwardYARAFindings keeps every distinct prior rule finding for paths
// this cycle could not examine. YARA can emit several rule matches for one
// file, so collapsing by path would silently retire all but one finding even
// though the scan formed no opinion about any of them. Duplicate snapshots of
// the same identity are collapsed by Key, with the newest snapshot winning.
func carryForwardYARAFindings(prior []alert.Finding, gaps *yaraGapCollector) []alert.Finding {
byKey := make(map[string]alert.Finding)
for _, finding := range prior {
if finding.Check != "yara_match_scheduled" || !gaps.hasPath(finding.FilePath) {
continue
}
key := finding.Key()
current, exists := byKey[key]
if !exists || finding.Timestamp.After(current.Timestamp) ||
(finding.Timestamp.Equal(current.Timestamp) && finding.FilePath < current.FilePath) {
byKey[key] = finding
}
}
keys := make([]string, 0, len(byKey))
for key := range byKey {
keys = append(keys, key)
}
sort.Strings(keys)
carried := make([]alert.Finding, 0, len(keys))
for _, key := range keys {
finding := byKey[key]
finding.ScanCarryForward = true
carried = append(carried, finding)
}
return carried
}
package checks
import (
"context"
"fmt"
"os"
"sync"
"time"
"github.com/pidginhost/csm/internal/crashsample"
"github.com/pidginhost/csm/internal/workerproc"
)
// ZipLister lists a tenant zip's entries in a process the daemon can kill.
// The daemon supplies one; there is no in-process fallback.
type ZipLister interface {
ListZip(ctx context.Context, zip *os.File, size int64, maxDirBytes uint64, visit func(name string, dir bool) error) error
}
var (
zipListerMu sync.RWMutex
zipLister ZipLister
// zipStrikes remembers, by SHA-256, zips that made the worker fail.
zipStrikes = workerproc.NewStrikes()
crashSamplesMu sync.RWMutex
crashSamples *crashsample.Store
)
// SetCrashSamples installs the store that keeps zips which crash the parse
// worker. Nil, the default, keeps none.
func SetCrashSamples(s *crashsample.Store) {
crashSamplesMu.Lock()
defer crashSamplesMu.Unlock()
crashSamples = s
}
// saveZipCrashSample keeps a copy of a zip that made the worker fail, when
// crash samples are on. A failed save changes nothing else.
func saveZipCrashSample(sum string, f *os.File, size int64) {
crashSamplesMu.RLock()
store := crashSamples
crashSamplesMu.RUnlock()
if store == nil {
return
}
if err := store.Save("zip-"+sum, []crashsample.File{{Name: "archive.zip", Src: f, Size: size}}); err != nil {
fmt.Fprintf(os.Stderr, "[%s] phishing: keeping crash sample of zip %s: %v\n", time.Now().Format("2006-01-02 15:04:05"), sum, err)
}
}
// SetZipLister installs the supervised lister. Passing nil removes it, after
// which phishing-kit zips are reported as not checked.
func SetZipLister(l ZipLister) {
zipListerMu.Lock()
defer zipListerMu.Unlock()
zipLister = l
}
func currentZipLister() ZipLister {
zipListerMu.RLock()
defer zipListerMu.RUnlock()
return zipLister
}
// Package clientaddr resolves the client address of an HTTP request or
// access-log record from its direct peer and X-Forwarded-For chain. The
// domlog scan and the challenge server share it, so one configured trust
// list decides the address they act on.
package clientaddr
import (
"errors"
"fmt"
"net"
"net/netip"
"strings"
"github.com/pidginhost/csm/internal/netutil"
)
// Bounds on a forwarded chain: the bytes of the whole header, and the hops
// the walk reads up to and including the client. A chain over either is
// refused, never trimmed. Hops a client writes left of its own address are
// never read, so they cannot make it refused.
const (
MaxHops = 32
MaxChainBytes = 4096
)
// Refusals. Each names why no address was resolved; none falls back to the
// proxy, whose address identifies a shared transport, not a client.
var (
ErrPeer = errors.New("direct peer is not an IP address")
ErrNoChain = errors.New("trusted proxy sent no forwarded chain")
ErrMalformed = errors.New("forwarded chain holds an unreadable hop")
ErrTruncated = errors.New("forwarded chain was cut before its client")
ErrOverlong = errors.New("forwarded chain exceeds 4 KiB or 32 hops up to its client")
ErrAllTrusted = errors.New("forwarded chain holds only trusted proxies")
// ErrCatchAllChain refuses a forwarded chain from a peer that only a
// refused catch-all entry listed: the operator put a proxy there, so the
// peer is not a visitor, and its header is not trusted either.
ErrCatchAllChain = errors.New("forwarded chain from a peer only a catch-all entry listed")
)
// ErrCatchAllEntry marks a refused entry that covers every address of a
// family, such as 0.0.0.0/0.
var ErrCatchAllEntry = errors.New("entry covers every address")
// catchAllError refuses a catch-all entry and keeps the range it named.
type catchAllError struct {
msg string
covers netip.Prefix
}
func (e *catchAllError) Error() string { return e.msg }
func (e *catchAllError) Is(target error) bool { return target == ErrCatchAllEntry }
// Trusted is a parsed trusted proxy list.
type Trusted struct {
prefixes []netip.Prefix
// catchAll holds refused catch-all entries; see ErrCatchAllChain.
catchAll []netip.Prefix
}
// Parse reads trusted proxy entries: IP addresses or CIDR ranges. An empty
// entry, a default route or anything else is an error naming its index.
func Parse(entries []string) (Trusted, error) {
var t Trusted
for i, raw := range entries {
entry := strings.TrimSpace(raw)
prefix, err := parseEntry(entry)
if err != nil {
return Trusted{}, fmt.Errorf("[%d]: %q %w", i, entry, err)
}
t.prefixes = append(t.prefixes, prefix)
}
return t, nil
}
// ParseUsable reads entries as Parse does but skips each unusable one,
// returning its error, so a running daemon keeps the proxies listed beside
// it. A skipped catch-all is never trusted; see ErrCatchAllChain.
func ParseUsable(entries []string) (Trusted, []error) {
var t Trusted
var errs []error
for i, raw := range entries {
entry := strings.TrimSpace(raw)
prefix, err := parseEntry(entry)
if err != nil {
var all *catchAllError
if errors.As(err, &all) {
t.catchAll = append(t.catchAll, all.covers)
if all.covers.Addr().Is4In6() {
// Keep both the effective IPv4 catch-all and the
// original range, which may cover native IPv6 peers.
t.catchAll = append(t.catchAll, netip.PrefixFrom(netip.IPv4Unspecified(), 0))
}
}
errs = append(errs, fmt.Errorf("[%d]: %q %w", i, entry, err))
continue
}
t.prefixes = append(t.prefixes, prefix)
}
return t, errs
}
func parseEntry(entry string) (netip.Prefix, error) {
if entry == "" {
return netip.Prefix{}, errors.New("is empty")
}
var prefix netip.Prefix
if strings.Contains(entry, "/") {
// Keep the legacy parser's acceptance of padded prefix lengths.
// Its original address also preserves a broad range's mapped form.
ip, network, err := net.ParseCIDR(entry)
if err != nil {
return netip.Prefix{}, errors.New("is not an IP address or CIDR range")
}
addr, _ := netip.AddrFromSlice(ip)
bits, width := network.Mask.Size()
if width == 32 {
addr = addr.Unmap()
}
prefix = netip.PrefixFrom(addr, bits)
} else {
addr, err := netip.ParseAddr(entry)
if err != nil || addr.Zone() != "" {
return netip.Prefix{}, errors.New("is not an IP address or CIDR range")
}
prefix = netip.PrefixFrom(addr, addr.BitLen())
}
// Masking a broader mapped range first would erase its IPv4 identity
// and silently leave the configured IPv4 proxies uncovered.
if prefix.Addr().Is4In6() {
if prefix.Bits() <= 96 {
return netip.Prefix{}, &catchAllError{"covers every IPv4 address", prefix}
}
prefix = netip.PrefixFrom(prefix.Addr().Unmap(), prefix.Bits()-96)
}
if prefix.Bits() == 0 {
return netip.Prefix{}, &catchAllError{"is a default route and would trust every address", prefix}
}
return prefix.Masked(), nil
}
// Len reports how many entries the list holds.
func (t Trusted) Len() int { return len(t.prefixes) }
// Covers reports whether addr is inside a configured entry, trusted or not.
// A listed address names a shared transport even where Has refuses to trust
// its header, so callers never read it as a client.
func (t Trusted) Covers(addr netip.Addr) bool {
addr = addr.Unmap()
for _, p := range t.prefixes {
if p.Contains(addr) {
return true
}
}
return false
}
// Has reports whether addr is a trusted proxy whose header may be read:
// inside a configured entry and not an address of this host. Any local
// account can connect from a host address and write X-Forwarded-For, so such
// an entry is never trusted; an unread interface set trusts nobody.
func (t Trusted) Has(addr netip.Addr) bool {
if !t.Covers(addr) {
return false
}
local, known := netutil.IsHostLocal(net.IP(addr.Unmap().AsSlice()))
return known && !local
}
// Result is a resolved client. Chained is true when the client came from a
// forwarded chain behind a trusted peer; then Peer is that proxy.
type Result struct {
Client netip.Addr
Peer netip.Addr
Chained bool
}
// Resolve returns the client behind peer. peer is the direct peer as a log or
// socket writes it, with or without a port and without whitespace; xff is the
// X-Forwarded-For value (several header lines joined with commas). Only HTTP
// spaces and tabs may surround a hop. complete is false when the log kept
// only its right end; that suffix must contain whole comma-delimited tokens,
// never a fragment of an address. An untrusted peer is the client and its xff
// is ignored. For a trusted peer the chain is read right to left past trusted
// hops to the first other address; an unreadable hop on the way refuses the
// chain rather than being skipped.
func Resolve(peer, xff string, complete bool, t Trusted) (Result, error) {
p, ok := parsePeer(peer)
if !ok {
return Result{}, ErrPeer
}
if !t.Has(p) {
if t.refusedCatchAll(p) && (!complete || strings.Trim(xff, " \t") != "") {
return Result{}, ErrCatchAllChain
}
return Result{Client: p, Peer: p}, nil
}
return chain(p, xff, complete, t)
}
func (t Trusted) refusedCatchAll(addr netip.Addr) bool {
for _, p := range t.catchAll {
if p.Contains(addr) {
return true
}
}
return false
}
// ResolveBehindProxy returns the client behind a peer that has no address
// but is trusted by how it connects, such as a proxy on a Unix socket only
// its group can open. Its chain is read as Resolve reads a trusted peer's.
func ResolveBehindProxy(xff string, t Trusted) (Result, error) {
return chain(netip.Addr{}, xff, true, t)
}
func chain(p netip.Addr, xff string, complete bool, t Trusted) (Result, error) {
if len(xff) > MaxChainBytes {
return Result{}, ErrOverlong
}
if strings.Trim(xff, " \t") == "" {
return Result{}, ErrNoChain
}
hops := strings.Split(xff, ",")
for i := len(hops) - 1; i >= 0; i-- {
if len(hops)-i > MaxHops {
return Result{}, ErrOverlong
}
hop, err := netip.ParseAddr(strings.Trim(hops[i], " \t"))
if err != nil || hop.Zone() != "" {
return Result{}, ErrMalformed
}
hop = hop.Unmap()
if t.Has(hop) {
continue
}
return Result{Client: hop, Peer: p, Chained: true}, nil
}
// The hops a log cut lie left of every hop read, so they matter only
// when no client was found right of them.
if !complete {
return Result{}, ErrTruncated
}
return Result{}, ErrAllTrusted
}
// parsePeer reads a bare address, an address with a port, or a bracketed
// IPv6 address, and returns it unmapped. Zoned addresses, unmatched brackets
// and bracketed IPv4 are refused. Ports use the unpadded decimal spelling
// emitted by sockets and web servers.
func parsePeer(peer string) (netip.Addr, bool) {
addr, err := netip.ParseAddr(peer)
if err != nil {
if ap, perr := netip.ParseAddrPort(peer); perr == nil {
port := peer[strings.LastIndexByte(peer, ':')+1:]
if len(port) > 1 && port[0] == '0' {
return netip.Addr{}, false
}
addr = ap.Addr()
} else if inner, ok := strings.CutPrefix(peer, "["); ok && strings.HasSuffix(inner, "]") {
addr, err = netip.ParseAddr(strings.TrimSuffix(inner, "]"))
if err != nil || !addr.Is6() {
return netip.Addr{}, false
}
} else {
return netip.Addr{}, false
}
}
if addr.Zone() != "" {
return netip.Addr{}, false
}
return addr.Unmap(), true
}
// Package cms is the single declaration of the content management systems
// CSM supports. Tests compare the taint analyzer's path constants and the
// database adapter owners against this table. Clean-corpus coverage is
// tracked separately.
package cms
import (
"fmt"
"strings"
)
// Kind identifies a supported CMS. Values are the canonical lowercase names
// used in the corpus manifest and in check owner names.
type Kind string
const (
WordPress Kind = "wordpress"
Joomla Kind = "joomla"
Drupal Kind = "drupal"
OpenCart Kind = "opencart"
Magento Kind = "magento"
// PrestaShop covers 8 and 9; 1.7 and 1.6 are best-effort through the
// same schema.
PrestaShop Kind = "prestashop"
)
// Descriptor records what the rest of the tree needs to know about a CMS.
type Descriptor struct {
Kind Kind
// DBContentCheck is the runner owner name of the adapter that scans this
// CMS's database ("db_content" for WordPress, "db_content_joomla", ...).
DBContentCheck string
// PathConstants are the lower-cased PHP constants the CMS defines at
// bootstrap that always hold a local filesystem path. They preserve the
// analyzer's provenance assumptions; they are not proof of a constant's
// runtime value.
PathConstants []string
}
var descriptors = []Descriptor{
{Kind: WordPress, DBContentCheck: "db_content", PathConstants: []string{"abspath"}},
{Kind: Joomla, DBContentCheck: "db_content_joomla", PathConstants: []string{
"jpath_root", "jpath_base", "jpath_site", "jpath_administrator", "jpath_api",
"jpath_cache", "jpath_cli", "jpath_component", "jpath_component_administrator",
"jpath_component_site", "jpath_configuration", "jpath_installation",
"jpath_libraries", "jpath_manifests", "jpath_plugins", "jpath_public", "jpath_themes",
}},
{Kind: Drupal, DBContentCheck: "db_content_drupal", PathConstants: []string{"drupal_root"}},
{Kind: OpenCart, DBContentCheck: "db_content_opencart", PathConstants: []string{
"dir_application", "dir_cache", "dir_catalog", "dir_config", "dir_download",
"dir_extension", "dir_image", "dir_language", "dir_logs", "dir_modification",
"dir_opencart", "dir_root", "dir_session", "dir_storage", "dir_system",
"dir_template", "dir_upload",
}},
{Kind: Magento, DBContentCheck: "db_content_magento", PathConstants: []string{"bp"}},
// _PS_ROOT_DIR_ may come from a server environment variable: server
// configuration, not request input.
{Kind: PrestaShop, DBContentCheck: "db_content_prestashop", PathConstants: []string{
"_ps_cache_dir_", "_ps_class_dir_", "_ps_config_dir_", "_ps_controller_dir_", "_ps_core_dir_",
"_ps_img_dir_", "_ps_module_dir_", "_ps_override_dir_", "_ps_root_dir_", "_ps_tool_dir_",
}},
}
func (d Descriptor) clone() Descriptor {
out := d
out.PathConstants = append([]string(nil), d.PathConstants...)
return out
}
// All returns every supported CMS in declaration order. The result is a
// deep copy; mutating it does not change policy.
func All() []Descriptor {
out := make([]Descriptor, 0, len(descriptors))
for _, d := range descriptors {
out = append(out, d.clone())
}
return out
}
// Lookup returns the descriptor for k, or the zero descriptor and false.
func Lookup(k Kind) (Descriptor, bool) {
for _, d := range descriptors {
if d.Kind == k {
return d.clone(), true
}
}
return Descriptor{}, false
}
// Parse accepts only an exact declared kind. It does not trim or fold case,
// so a manifest or config value must be spelled canonically.
func Parse(s string) (Kind, bool) {
for _, d := range descriptors {
if string(d.Kind) == s {
return d.Kind, true
}
}
return "", false
}
// validateDescriptors reports the first shape violation in ds: kinds and
// owner names must be non-empty and unique, and path constants non-empty,
// lower case and unique within and across descriptors.
func validateDescriptors(ds []Descriptor) error {
seenKind := make(map[Kind]bool, len(ds))
seenOwner := make(map[string]bool, len(ds))
seenConst := make(map[string]Kind)
for _, d := range ds {
if d.Kind == "" {
return fmt.Errorf("descriptor with empty kind")
}
if seenKind[d.Kind] {
return fmt.Errorf("%s: kind declared twice", d.Kind)
}
seenKind[d.Kind] = true
if d.DBContentCheck == "" {
return fmt.Errorf("%s: empty DBContentCheck", d.Kind)
}
if seenOwner[d.DBContentCheck] {
return fmt.Errorf("%s: DBContentCheck %q reused", d.Kind, d.DBContentCheck)
}
seenOwner[d.DBContentCheck] = true
if len(d.PathConstants) == 0 {
return fmt.Errorf("%s: no path constants; local provenance cannot be analysed", d.Kind)
}
for _, c := range d.PathConstants {
if c == "" || c != strings.ToLower(c) {
return fmt.Errorf("%s: constant %q must be non-empty lower case", d.Kind, c)
}
if owner, dup := seenConst[c]; dup {
return fmt.Errorf("constant %q declared by both %s and %s", c, owner, d.Kind)
}
seenConst[c] = d.Kind
}
}
return nil
}
package config
import "github.com/pidginhost/csm/internal/admission"
// DefaultAdmissionCeiling is the admission ledger's hourly ceiling over
// every automatic source when max_blocks_per_hour is unset or zero.
// The legacy hourly counter keeps DefaultMaxBlocksPerHour for the
// same key until routing through the ledger replaces it.
const DefaultAdmissionCeiling = 2000
// Where the admission ceiling came from.
const (
CeilingDefault = "default"
CeilingConfigured = "configured"
CeilingClamped = "clamped"
)
// AdmissionCeiling is the hourly ceiling the admission ledger enforces:
// max_blocks_per_hour, its default when unset or zero, or the largest
// ceiling the ledger accepts when the setting exceeds it. The second
// result names which.
func (c *Config) AdmissionCeiling() (uint32, string) {
v := c.AutoResponse.MaxBlocksPerHour
switch {
case v <= 0 || c.AutoResponse.MaxBlocksPerHourDefaulted:
return DefaultAdmissionCeiling, CeilingDefault
case v > admission.MaxCeiling:
return admission.MaxCeiling, CeilingClamped
}
return uint32(v), CeilingConfigured // #nosec G115 -- 1..MaxCeiling here.
}
package config
import (
"errors"
"fmt"
"gopkg.in/yaml.v3"
)
// Check nodes before decoding: yaml.v3 coerces scalars to strings and drops
// null sequence entries, which would silently change the requested feeds.
func validateAntispamFeedTypes(node yaml.Node) error {
var fields map[string]yaml.Node
if err := node.Decode(&fields); err != nil {
return err
}
if enabled, ok := fields["enabled"]; ok && !yamlNodeIsNull(&enabled) {
if err := validateStringList(enabled, "email_protection.antispam.feeds.enabled"); err != nil {
return err
}
}
if keyFile, ok := fields["spamhaus_dqs_key_file"]; ok && !yamlNodeIsNull(&keyFile) && yamlResolvedTag(&keyFile) != "!!str" {
return errors.New("email_protection.antispam.feeds.spamhaus_dqs_key_file must be a string")
}
return nil
}
// antispamLearningFields checks the learning block's types and returns its
// fields. Enabling learning is consent to reading Spam mail, so only a YAML
// boolean counts; yaml.v3 would otherwise coerce values such as 1, and turn
// any scalar into a string.
func antispamLearningFields(node yaml.Node) (map[string]yaml.Node, error) {
if yamlResolvedTag(&node) != "!!map" {
return nil, errors.New("email_protection.antispam.learning must be a mapping")
}
var fields map[string]yaml.Node
if err := node.Decode(&fields); err != nil {
return nil, err
}
if enabled, ok := fields["enabled"]; ok && !yamlNodeIsNull(&enabled) && yamlResolvedTag(&enabled) != "!!bool" {
return nil, errors.New("email_protection.antispam.learning.enabled must be true or false")
}
for _, key := range []string{"redis_socket", "reporter"} {
if value, ok := fields[key]; ok && !yamlNodeIsNull(&value) && yamlResolvedTag(&value) != "!!str" {
return nil, fmt.Errorf("email_protection.antispam.learning.%s must be a string", key)
}
}
return fields, nil
}
// validateStringList requires a YAML sequence of strings, for the same
// reasons as validateAntispamFeedTypes.
func validateStringList(node yaml.Node, key string) error {
if yamlResolvedTag(&node) != "!!seq" {
return fmt.Errorf("%s must be a list of strings", key)
}
var items []yaml.Node
if err := node.Decode(&items); err != nil {
return err
}
for _, item := range items {
if yamlResolvedTag(&item) != "!!str" {
return fmt.Errorf("%s must contain only strings", key)
}
}
return nil
}
package config
import (
"fmt"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/netutil"
)
// ChallengeSocketExample is the socket form messages and docs point at.
const ChallengeSocketExample = "unix:/run/csm/challenge.sock"
const challengeSocketPrefix = "unix:"
// ChallengeSocketPath reports whether challenge.listen_addr asks for a Unix
// socket and returns its path, unvalidated. Every reader of listen_addr asks
// this first, so socket mode is recognised in one place.
func ChallengeSocketPath(listenAddr string) (string, bool) {
addr := strings.TrimSpace(listenAddr)
if len(addr) < len(challengeSocketPrefix) || !strings.EqualFold(addr[:len(challengeSocketPrefix)], challengeSocketPrefix) {
return "", false
}
return strings.TrimSpace(addr[len(challengeSocketPrefix):]), true
}
// ChallengeSocketMaxPath is the longest socket path a client can connect to:
// sun_path holds 108 bytes including the terminating NUL.
const ChallengeSocketMaxPath = 107
// ChallengeSocketBindName is the name the server binds before renaming the
// socket into place: a fresh directory named ChallengeSocketBindPrefix plus
// up to ten random digits, holding "s". Its length bounds the directory.
const (
ChallengeSocketBindPrefix = ".cs"
challengeSocketBindExtra = len("/"+ChallengeSocketBindPrefix) + 10 + len("/s")
)
// ChallengeSocketPathError reports a socket path the server cannot bind
// reliably: not absolute, not in clean form, or too long for sun_path once
// the bind-time name is counted. Validation, the installer and the server
// share it, so a path never passes one and fails another.
func ChallengeSocketPathError(path string) error {
if !filepath.IsAbs(path) {
return fmt.Errorf("challenge.listen_addr: unix socket path %q must be absolute", path)
}
if filepath.Clean(path) != path {
return fmt.Errorf("challenge.listen_addr: unix socket path %q must be clean (no repeated separators, dot components or trailing slash)", path)
}
if len(path) > ChallengeSocketMaxPath {
return fmt.Errorf("challenge.listen_addr: socket path too long for the %d-byte limit: %s", ChallengeSocketMaxPath, path)
}
if dir := filepath.Dir(path); len(dir)+challengeSocketBindExtra > ChallengeSocketMaxPath {
return fmt.Errorf("challenge.listen_addr: socket directory %s is too long: binding needs %d more bytes within the %d-byte limit", dir, challengeSocketBindExtra, ChallengeSocketMaxPath)
}
return nil
}
// ChallengeLocalTrustError reports a challenge.trusted_proxies entry that
// covers an address of this host. Any local account can connect from such an
// address and write X-Forwarded-For, so trusting it lets it pose as any
// visitor. Validation, the installer and doctor share this text.
func ChallengeLocalTrustError(entries []string) error {
entry, covered, err := netutil.HostLocalProxyEntry(usableTrustedProxyEntries(entries))
if err != nil {
return fmt.Errorf("challenge.trusted_proxies: cannot check the entries: %v", err)
}
if entry == "" {
return nil
}
return fmt.Errorf("challenge.trusted_proxies entry %q covers %s, an address of this host: CSM never trusts it and refuses requests from that proxy. Point a proxy on this host at challenge.listen_addr: %s with challenge.listen_group, or remove trusted_proxies and use direct mode", entry, covered, ChallengeSocketExample)
}
package config
// CanonicalCheckName maps persisted check names to the current finding names.
// Config exclusions, saved mutes and pending work must survive a producer rename.
func CanonicalCheckName(name string) string {
switch name {
case "ftp_login_realtime":
return "ftp_login"
case "ssh_login_realtime":
return "ssh_login_unknown_ip"
default:
return name
}
}
package config
import (
"net"
"os"
"path/filepath"
"strings"
"syscall"
"time"
)
// clamdSocketCandidates are the unix sockets a clamd is normally reachable on.
// The path is set by whoever packaged clamd, not by CSM: RHEL's clamd-scan,
// Debian's clamav-daemon and cPanel's bundled clamd all choose differently, and
// a host whose setting names the wrong one scans no mail at all while every
// health signal still reports the watcher as running.
//
// Every entry is a root-owned service directory. A world-writable location such
// as /tmp is deliberately absent: any account could bind a socket there, and
// CSM would then stream every attachment to it and believe the "OK" it answers.
var clamdSocketCandidates = []string{
"/var/run/clamd.scan/clamd.sock",
"/run/clamd.scan/clamd.sock",
"/var/run/clamav/clamd.ctl",
"/run/clamav/clamd.ctl",
"/var/run/clamav/clamd.sock",
"/run/clamav/clamd.sock",
"/usr/local/cpanel/3rdparty/var/clamav/clamd.sock",
}
// clamdDialTimeout bounds each probe. The candidate list is walked on a health
// path, so a hung socket must not hold the caller. A real clamd that is merely
// busy still answers PING promptly; it is a separate thread from scanning.
const clamdDialTimeout = 2 * time.Second
// ResolveClamdSocket returns the socket to talk to clamd on, and whether it had
// to be discovered because the configured one was not answering.
//
// The configured path always wins when clamd is answering on it. Only when it
// is not does CSM fall back to a well-known location, because refusing to scan
// mail is the worse failure: silence there looks exactly like clean mail.
//
// A discovered socket is only accepted when it is owned by root or by this
// process and sits in a directory no other account can write to, and when
// whatever is listening answers clamd's PING. Discovery must not be a way to
// point mail scanning at something an account controls.
func ResolveClamdSocket(configured string) (string, bool) {
if configured != "" && clamdSocketTrusted(configured) && clamdSocketAnswers(configured) {
return configured, false
}
for _, candidate := range clamdSocketCandidates {
if candidate == configured {
continue
}
if !clamdSocketTrusted(candidate) {
continue
}
if clamdSocketAnswers(candidate) {
return candidate, true
}
}
return configured, false
}
// clamdSocketTrusted reports whether only a privileged account could have put
// this socket here.
//
// What matters is the directory: whoever can write to it decides what the name
// resolves to, and CSM is about to stream every mail attachment to whatever is
// listening and believe the verdict it returns. The socket's own owner is not
// the test -- clamd is packaged to run as its own service user (Debian's
// clamav owns /run/clamav), so requiring root there would reject exactly the
// sockets this discovery exists to find.
//
// A sticky directory is refused rather than allowed: /tmp lets any account
// create the name first, and being unable to delete someone else's socket is
// no help when the attacker's is the one that got there.
func clamdSocketTrusted(path string) bool {
info, err := os.Lstat(path)
if err != nil || info.Mode()&os.ModeSymlink != 0 {
return false
}
return clamdDirTrusted(filepath.Dir(path))
}
// clamdMaxServiceUID is the ceiling for a packaged service account. Hosting
// accounts start well above it on every panel CSM supports.
const clamdMaxServiceUID = 500
func clamdDirTrusted(dir string) bool {
info, err := os.Lstat(dir)
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return false
}
if info.Mode().Perm()&0o022 != 0 {
return false
}
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return false
}
// Owned by root, by this process, or by the service account clamd runs
// as -- never by an account that also hosts websites.
uid := int(stat.Uid)
return uid == 0 || uid == os.Geteuid() || uid < clamdMaxServiceUID
}
// clamdSocketAnswers reports whether clamd itself is listening. Connecting
// proves only that something is there; PING proves it speaks the protocol CSM
// is about to hand mail to.
func clamdSocketAnswers(path string) bool {
conn, err := net.DialTimeout("unix", path, clamdDialTimeout)
if err != nil {
return false
}
defer func() { _ = conn.Close() }()
if setErr := conn.SetDeadline(time.Now().Add(clamdDialTimeout)); setErr != nil {
return false
}
if _, writeErr := conn.Write([]byte("zPING\x00")); writeErr != nil {
return false
}
buf := make([]byte, 16)
n, readErr := conn.Read(buf)
if readErr != nil || n == 0 {
return false
}
return strings.Contains(strings.ToUpper(string(buf[:n])), "PONG")
}
package config
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"syscall"
"gopkg.in/yaml.v3"
)
type confFragment struct {
path string
node *yaml.Node
}
// ValidateConfDir vets an operator-selected conf.d directory before any YAML
// fragments are loaded from it. The returned path is symlink-resolved so later
// reads do not depend on a mutable link name.
func ValidateConfDir(dir string) (string, error) {
if dir == "" {
return "", nil
}
if !filepath.IsAbs(dir) {
return "", fmt.Errorf("conf.d directory must be an absolute path, got %q", dir)
}
resolved, err := filepath.EvalSymlinks(dir)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return "", fmt.Errorf("conf.d directory does not exist: %s", dir)
}
return "", fmt.Errorf("conf.d directory symlink resolution: %w", err)
}
info, err := os.Stat(resolved)
if err != nil {
return "", fmt.Errorf("conf.d directory stat: %w", err)
}
if !info.IsDir() {
return "", fmt.Errorf("conf.d directory is not a directory: %s", resolved)
}
if trustErr := validateConfPathTrust("conf.d directory", resolved, info); trustErr != nil {
return "", trustErr
}
return resolved, nil
}
// LoadConfDir reads every *.yaml file in dir in lexicographic order and
// returns each as a parsed yaml.DocumentNode. A missing directory is not
// an error and returns an empty slice; an unreadable file or invalid YAML
// is fatal so operators see misconfigurations at startup.
func LoadConfDir(dir string) ([]*yaml.Node, error) {
frags, err := loadConfDirFragments(dir)
if err != nil {
return nil, err
}
out := make([]*yaml.Node, 0, len(frags))
for _, frag := range frags {
out = append(out, frag.node)
}
return out, nil
}
// ConfDirFragment is one conf.d drop-in fragment's filename and raw bytes,
// exported so the integrity hasher can cover the same fragment set the loader
// merges without duplicating the enumeration rules.
type ConfDirFragment struct {
Name string
Data []byte
}
// ConfDirFragmentDigestInput returns every non-empty trusted conf.d fragment in
// merge order (sorted .yaml/.yml, symlink-resolved, trust-validated) as
// name+content pairs for integrity hashing. An empty dir or no mergeable
// fragments yields nil so a config without conf.d hashes to the empty digest
// and its baseline is unaffected.
func ConfDirFragmentDigestInput(dir string) ([]ConfDirFragment, error) {
files, err := confDirFragmentFiles(dir)
if err != nil {
return nil, err
}
if len(files) == 0 {
return nil, nil
}
out := make([]ConfDirFragment, 0, len(files))
for _, ff := range files {
if _, ok, err := parseConfFragment(ff); err != nil {
return nil, err
} else if !ok {
continue
}
out = append(out, ConfDirFragment{Name: ff.name, Data: ff.data})
}
return out, nil
}
type confFragmentFile struct {
name string
path string
data []byte
}
// confDirFragmentFiles enumerates trusted conf.d fragment files and returns
// their raw bytes in merge order. Shared by loadConfDirFragments and the
// integrity hasher so both observe exactly the same fragment set.
func confDirFragmentFiles(dir string) ([]confFragmentFile, error) {
if dir == "" {
return nil, nil
}
resolved, err := filepath.EvalSymlinks(dir)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("conf.d directory symlink resolution: %w", err)
}
info, err := os.Stat(resolved)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("conf.d directory stat: %w", err)
}
if !info.IsDir() {
return nil, fmt.Errorf("conf.d directory is not a directory: %s", resolved)
}
if trustErr := validateConfPathTrust("conf.d directory", resolved, info); trustErr != nil {
return nil, trustErr
}
dir = resolved
entries, err := os.ReadDir(dir)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("reading %s: %w", dir, err)
}
names := make([]string, 0, len(entries))
for _, e := range entries {
if e.IsDir() {
continue
}
if !strings.HasSuffix(e.Name(), ".yaml") && !strings.HasSuffix(e.Name(), ".yml") {
continue
}
names = append(names, e.Name())
}
sort.Strings(names)
out := make([]confFragmentFile, 0, len(names))
for _, name := range names {
path := filepath.Join(dir, name)
data, err := readTrustedConfFragment(path)
if err != nil {
return nil, err
}
out = append(out, confFragmentFile{name: name, path: path, data: data})
}
return out, nil
}
func loadConfDirFragments(dir string) ([]confFragment, error) {
files, err := confDirFragmentFiles(dir)
if err != nil {
return nil, err
}
out := make([]confFragment, 0, len(files))
for _, ff := range files {
node, ok, err := parseConfFragment(ff)
if err != nil {
return nil, err
}
if !ok {
continue
}
out = append(out, confFragment{path: ff.path, node: node})
}
return out, nil
}
func parseConfFragment(ff confFragmentFile) (*yaml.Node, bool, error) {
var node yaml.Node
if err := yaml.Unmarshal(ff.data, &node); err != nil {
return nil, false, fmt.Errorf("parsing %s: %w", ff.path, err)
}
// Skip empty files (Unmarshal yields a zero-Content document).
if len(node.Content) == 0 {
return nil, false, nil
}
normalized, err := normalizeYAMLForMerge(&node)
if err != nil {
return nil, false, fmt.Errorf("parsing %s: %w", ff.path, err)
}
node = *normalized
if hasTopLevelKey(&node, "integrity") {
return nil, false, fmt.Errorf("conf.d fragment %s must not set daemon-managed integrity metadata", ff.path)
}
// The confd block decides which fragments the integrity digest covers, so
// a fragment that could set it would be able to exempt itself.
if hasTopLevelKey(&node, "confd") {
return nil, false, fmt.Errorf("conf.d fragment %s must not set the confd policy block; it belongs in the main config", ff.path)
}
return &node, true, nil
}
func hasTopLevelKey(root *yaml.Node, key string) bool {
cur := root
if cur.Kind == yaml.DocumentNode {
if len(cur.Content) == 0 {
return false
}
cur = cur.Content[0]
}
if cur.Kind != yaml.MappingNode {
return false
}
for i := 0; i+1 < len(cur.Content); i += 2 {
if cur.Content[i].Value == key {
return true
}
}
return false
}
func readTrustedConfFragment(path string) ([]byte, error) {
// #nosec G304 -- path is built from an operator-selected conf.d and a directory entry.
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("reading %s: %w", path, err)
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return nil, fmt.Errorf("stat %s: %w", path, err)
}
if !info.Mode().IsRegular() {
return nil, fmt.Errorf("conf.d fragment is not a regular file: %s", path)
}
if trustErr := validateConfPathTrust("conf.d fragment", path, info); trustErr != nil {
return nil, trustErr
}
data, err := readConfigBytesLimited(f)
if errors.Is(err, errConfigTooLarge) {
return nil, fmt.Errorf("conf.d fragment %s exceeds %d byte cap", path, MaxConfigBytes)
}
if err != nil {
return nil, fmt.Errorf("reading %s: %w", path, err)
}
return data, nil
}
func validateConfPathTrust(kind, path string, info os.FileInfo) error {
if mode := info.Mode().Perm(); mode&0022 != 0 {
return fmt.Errorf("%s %s has unsafe mode %04o (group or world writable); set 0750/0640 or stricter", kind, path, mode)
}
sys, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return nil
}
// #nosec G115 -- Linux uid_t is uint32; os.Geteuid returns the kernel
// effective UID and cannot overflow that type on supported hosts.
selfUID := uint32(os.Geteuid())
if sys.Uid != 0 && sys.Uid != selfUID {
return fmt.Errorf("%s %s owner uid=%d is neither root (0) nor process uid=%d; refusing to load untrusted YAML", kind, path, sys.Uid, selfUID)
}
return nil
}
package config
import (
"bytes"
"errors"
"fmt"
"io"
"net"
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"gopkg.in/yaml.v3"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/atomicio"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/stalwartapi"
"github.com/pidginhost/csm/internal/stalwartlog"
)
// WebUIToken is one entry in WebUI.Tokens. Scope must be "admin", "read" or
// "unban". AllowedIPs, when set, limits where the token is accepted from.
type WebUIToken struct {
Name string `yaml:"name"`
Token string `yaml:"token"`
Scope string `yaml:"scope"`
IssuedAt string `yaml:"issued_at"`
ExpiresAt string `yaml:"expires_at,omitempty"`
AllowedIPs []string `yaml:"allowed_ips,omitempty"`
}
const (
// DefaultBlockExpiry is how long an automatic temporary block lasts when
// auto_response.block_expiry is unset.
DefaultBlockExpiry = "24h"
// DefaultMaxBlocksPerHour is the safe hourly cap used when the operator
// leaves auto_response.max_blocks_per_hour unset or sets it to 0.
DefaultMaxBlocksPerHour = 50
// File response limits share one rolling window across automatic cleaners
// and quarantine. Zero selects these defaults, never an unlimited budget.
DefaultMaxFileActionsPerHour = 50
DefaultMaxFileActionsPerAccountPerHour = 10
DefaultMaxFileActionFailuresPerHour = 3
// MaxFileResponseLimit bounds retained hourly safety reservations.
MaxFileResponseLimit = 10000
// DefaultNetBlockThreshold is how many blocked addresses in one IPv4 /24
// or IPv6 /64 escalate to a subnet block when the key is unset.
DefaultNetBlockThreshold = 3
// DefaultNetBlockWindow is how far back blocked addresses count toward
// netblock_threshold when auto_response.netblock_window is unset. Hosts
// that rotate through a /24 keep one address blocked at a time, so only
// counting live blocks never reaches the threshold.
DefaultNetBlockWindow = "168h"
// DefaultPermBlockCount is how many temporary blocks inside the interval
// promote an address to a permanent block when the key is unset.
DefaultPermBlockCount = 4
// DefaultPermBlockInterval is the window used to count temporary blocks
// when auto_response.permblock_interval is unset.
DefaultPermBlockInterval = "24h"
// MinBlockEscalationCount is the lowest meaningful value for either
// counter. Below it there is no pattern to escalate from.
MinBlockEscalationCount = 2
// DefaultMailSprayBlockExpiry is how long a mail login spray block lasts
// when thresholds.mail_spray_block_expiry is unset.
DefaultMailSprayBlockExpiry = "720h"
)
const (
// DefaultExposedFileScanDepth is the number of directory levels below a
// document root inspected by the web-exposed-file detector.
DefaultExposedFileScanDepth = 2
// MaxExposedFileScanDepth prevents a configuration typo from turning the
// hourly detector into an effectively unbounded filesystem traversal.
MaxExposedFileScanDepth = 10
// DefaultDropperUnlinkTTLSec is how long the realtime monitor waits after
// a fresh PHP/executable lands under a web docroot before probing whether
// it self-deleted. Long enough that upgrade staging has settled, short
// enough that a dropper is reported minutes after it vanishes.
DefaultDropperUnlinkTTLSec = 300
// MinDropperUnlinkTTLSec / MaxDropperUnlinkTTLSec bound the operator
// knob: below 30s legitimate installers race the probe, above 1h the
// tracker holds state for little detection gain.
MinDropperUnlinkTTLSec = 30
MaxDropperUnlinkTTLSec = 3600
// DefaultHTTPScannerErrorPct is the fallback percentage gate for the
// URL scanner-profile detector when the operator leaves it unset.
DefaultHTTPScannerErrorPct = 90
// DefaultHTTPScannerMinDistinctPaths is the fallback breadth gate for
// distinct probe paths when the operator leaves it unset.
DefaultHTTPScannerMinDistinctPaths = 10
// HTTPScannerMaxDistinctPaths matches the detector's per-IP path cap.
// Higher configured breadth thresholds can never be reached.
HTTPScannerMaxDistinctPaths = 512
// SlowBruteMinThreshold is the smallest useful long-horizon failure
// threshold: the detector also requires this many distinct mailboxes.
SlowBruteMinThreshold = 3
// SlowBruteMaxThreshold matches the detector's bounded per-IP timestamp
// history. A larger value could never fire and would make the knob lie.
SlowBruteMaxThreshold = 1024
// SlowBruteMaxWindowMin bounds long-lived per-IP state while still allowing
// operators to cover paced attacks over a full week.
SlowBruteMaxWindowMin = 7 * 24 * 60
)
// DefaultHTTPScannerStatusCodes returns a fresh copy of the probe-error
// statuses used when the operator leaves the scanner status list unset.
func DefaultHTTPScannerStatusCodes() []int {
return []int{404, 403}
}
const (
// DefaultHTTPASNCrawlMinIPs is the minimum distinct source IPs from one
// ASN inside the window before http_asn_crawl fires. 0 disables the detector.
DefaultHTTPASNCrawlMinIPs = 25
// DefaultXMLRPCThreshold is the per-IP POST /xmlrpc.php count that trips
// xmlrpc_abuse in access-log based detectors. 0 disables the check; an
// absent key defaults to this. Raised from the legacy 30 so
// Jetpack/WooCommerce sites, which legitimately make many xmlrpc.php calls,
// are not hard-blocked.
DefaultXMLRPCThreshold = 100
// DefaultHTTPASNCrawlMinExpensive is the minimum uncacheable requests from
// the ASN inside the window before http_asn_crawl fires.
DefaultHTTPASNCrawlMinExpensive = 250
// DefaultHTTPASNCrawlMinSharePct is the minimum percentage of a single
// account's total uncacheable requests that must come from one ASN.
DefaultHTTPASNCrawlMinSharePct = 50
// DefaultHTTPASNCrawlHighAmpPct is the percentage threshold above which the
// ASN's share triggers a High-severity finding instead of Warning.
DefaultHTTPASNCrawlHighAmpPct = 50
// DefaultHTTPASNCrawlHighVolMult is the multiplier applied to MinExpensive
// for the High-severity volume threshold.
DefaultHTTPASNCrawlHighVolMult = 4
// DefaultHTTPASNCrawlMaxPrefix is the maximum number of distinct /24
// prefixes (IPv4) or /48 prefixes (IPv6) within the ASN before the finding
// is promoted to Critical (distributed saturation pattern).
DefaultHTTPASNCrawlMaxPrefix = 8
// DefaultHTTPASNCrawl16PrefPct is the percentage of IPs that must share a
// /16 prefix for the condensed-prefix heuristic to apply.
DefaultHTTPASNCrawl16PrefPct = 60
// DefaultHTTPASNCrawlMaxTrackedIPs caps how many source IPs per ASN the
// rolling window keeps in memory.
DefaultHTTPASNCrawlMaxTrackedIPs = 20000
// DefaultHTTPASNCrawlWindowMin is the rolling window length in minutes.
DefaultHTTPASNCrawlWindowMin = 60
// DefaultHTTPASNCrawlTempban is the ban duration applied when the detector
// fires and auto-response is enabled.
DefaultHTTPASNCrawlTempban = "24h"
)
// httpASNCrawlReverseProxySeed is the built-in safety list of reverse-proxy
// CDN ASNs (Cloudflare, Fastly, Akamai). Edge IPs from these never form a
// finding or tempban; real-client attribution requires web_server.trusted_proxies.
var httpASNCrawlReverseProxySeed = []uint{13335, 54113, 20940}
// MailLogsConfig controls how postfix/dovecot logs are read.
//
// source: auto - try file first; fall back to journal if file absent.
// source: file - require log file at the platform-default path.
// source: journal - read from systemd-journald (units must be set).
//
// Units is consulted for journal fallback: the daemon matches each
// systemd unit by name, appending ".service" for bare service names.
type MailLogsConfig struct {
Source string `yaml:"source"` // auto | file | journal
File string `yaml:"file,omitempty"` // override platform default
Units []string `yaml:"units,omitempty"` // for journal source
}
// StalwartConfig locates the Stalwart mail server log CSM reads on hosts
// where Stalwart is the mail server. Ports maps listeners on non-standard
// ports, as "PORT:PROTOCOL" with protocol smtp, imap, pop3, sieve or http.
type StalwartConfig struct {
LogDir string `yaml:"log_dir"`
LogPrefix string `yaml:"log_prefix"`
Ports []string `yaml:"ports,omitempty"`
// APIURL is where `csm stalwart setup` and doctor reach Stalwart's
// admin API.
APIURL string `yaml:"api_url,omitempty"`
// APIKeyFile holds the API key the daemon acts on Stalwart with.
APIKeyFile string `yaml:"api_key_file,omitempty"`
}
type Config struct {
ConfigFile string `yaml:"-"`
ConfigDir string `yaml:"-" hotreload:"restart"` // /etc/csm/conf.d (or operator override); empty means no drop-ins loaded
// RemovedKeys lists settings found in the loaded YAML that no release
// reads any more; Validate turns them into warnings.
RemovedKeys []string `yaml:"-"`
Hostname string `yaml:"hostname" hotreload:"restart"`
// Mode is the operator's posture for this host. "enforce" (default)
// leaves every subsystem under its own switch. "observe" declares that
// CSM must not change host state: the daemon skips the host integration
// files it otherwise deploys at startup, and a config that still enables
// a state-changing subsystem is refused by name instead of being
// silently rewritten. Rewriting would be persisted: the config
// re-signing path marshals the in-memory config back over csm.yaml.
Mode string `yaml:"mode" hotreload:"restart"`
Alerts struct {
Email struct {
Enabled bool `yaml:"enabled"`
To []string `yaml:"to"`
From string `yaml:"from"`
SMTP string `yaml:"smtp"`
DisabledChecks []string `yaml:"disabled_checks"`
} `yaml:"email"`
Webhook struct {
Enabled bool `yaml:"enabled"`
URL string `yaml:"url"`
Type string `yaml:"type"` // slack, discord, generic, phpanel
// HMACSecret is the shared secret used to sign each request when
// Type=="phpanel". Read from this field directly OR via HMACSecretEnv
// (env wins, for secret hygiene).
HMACSecret string `yaml:"hmac_secret,omitempty"`
HMACSecretEnv string `yaml:"hmac_secret_env,omitempty"`
} `yaml:"webhook"`
Heartbeat struct {
Enabled bool `yaml:"enabled"`
URL string `yaml:"url"`
} `yaml:"heartbeat"`
MaxPerHour int `yaml:"max_per_hour"`
// AuditLog ships every (deduplicated) finding to one or more
// SIEM-friendly destinations. Schema is stable: parsers can
// pin on the v=1 contract. Both sub-blocks default off; the
// alert pipeline behaves identically to before when neither
// is enabled.
AuditLog struct {
File struct {
Enabled bool `yaml:"enabled"`
Path string `yaml:"path"` // default: /var/log/csm/audit.jsonl
} `yaml:"file"`
Syslog struct {
Enabled bool `yaml:"enabled"`
Network string `yaml:"network"` // udp | tcp | unix | unixgram | tls
Address string `yaml:"address"` // host:port or filesystem path
Facility string `yaml:"facility"` // default: local0
TLSCAFile string `yaml:"tls_ca"` // optional CA cert for tls
} `yaml:"syslog"`
} `yaml:"audit_log"`
// BlockDigest adds a per-country roll-up of auto-blocked IPs to the
// daily alert summary so operators see when their customers'
// countries get blocked. The hotreload:"restart" tag overrides the
// safe Alerts parent: the collector and its ticker are built once at
// startup.
BlockDigest struct {
Enabled bool `yaml:"enabled"`
Countries []string `yaml:"countries"`
Interval string `yaml:"interval"`
Live bool `yaml:"live"`
SendOn string `yaml:"send_on"`
Channel string `yaml:"channel"`
MinBlock int `yaml:"min_block"`
} `yaml:"block_digest" hotreload:"restart"`
} `yaml:"alerts" hotreload:"safe"`
Integrity struct {
BinaryHash string `yaml:"binary_hash"`
ConfigHash string `yaml:"config_hash"`
// ConfdHash covers the conf.d drop-in fragments merged on top of the
// main config. Empty when no fragments exist, so configs without
// conf.d stay byte-identical to their pre-existing baseline. Without
// it an attacker with write access to conf.d could override any
// setting (auto_response, block_ips, dry_run) without tripping
// integrity verification.
ConfdHash string `yaml:"confd_hash"`
Immutable bool `yaml:"immutable"`
} `yaml:"integrity"`
// ConfD holds operator policy for the conf.d drop-in directory. It lives
// outside the integrity block on purpose: config_hash skips that block,
// so an exemption placed there could be added without tripping
// verification. Here it is covered by config_hash like any other key.
ConfD struct {
// IntegrityExempt names drop-in fragments (bare filenames) whose
// content is left out of integrity.confd_hash. A fragment its owning
// integration rewrites on its own schedule cannot be pinned by a
// static hash without turning every restart into a manual rehash.
// Every other fragment stays covered. Fragments cannot set this key.
IntegrityExempt []string `yaml:"integrity_exempt,omitempty"`
} `yaml:"confd,omitempty" hotreload:"safe"`
Thresholds struct {
MailQueueWarn int `yaml:"mail_queue_warn"`
MailQueueCrit int `yaml:"mail_queue_crit"`
DeepScanIntervalMin int `yaml:"deep_scan_interval_min"`
// ExposedFileScanDepth bounds how many directory levels below each
// docroot the web-exposed-file detector descends (default 2, maximum
// 10). Dumps and backups almost always sit at or just under the web root.
ExposedFileScanDepth int `yaml:"exposed_file_scan_depth"`
MultiIPLoginThreshold int `yaml:"multi_ip_login_threshold"`
MultiIPLoginWindowMin int `yaml:"multi_ip_login_window_min"`
// CredStuffingDistinctAccounts is the number of distinct accounts a
// single source IP must fail against inside the multi_ip_login window
// to raise a credential_stuffing finding. This is the breadth signal
// (one source, many accounts) that the count-based pam_bruteforce
// detector does not catch. Default 5 when unset or <=0, matching the
// always-on posture of the sibling multi_ip_login_threshold.
CredStuffingDistinctAccounts int `yaml:"cred_stuffing_distinct_accounts"`
// PAMBruteforceThreshold is the number of PAM authentication
// failures (SSH, mail, FTP via pam_csm) from one source IP inside
// PAMBruteforceWindowMin minutes before pam_bruteforce fires and the
// address is auto-blocked. Defaults 5 failures in 10 minutes.
PAMBruteforceThreshold int `yaml:"pam_bruteforce_threshold"`
PAMBruteforceWindowMin int `yaml:"pam_bruteforce_window_min"`
PluginCheckIntervalMin int `yaml:"plugin_check_interval_min"`
BruteForceWindow int `yaml:"brute_force_window"`
// DomlogMaxFiles caps how many per-domain access logs the WP brute
// force check scans per cycle. Sites are ranked by recent mtime so
// the cap chops least-active domains. Default 500. Bump on hosts
// with many active domains so late-alphabet sites are not skipped.
DomlogMaxFiles int `yaml:"domlog_max_files"`
// PHPConfigWalkMaxDirs caps how many directories the PHP configuration
// scan walks below a single document root. Reaching it leaves the rest
// of that root unexamined, which is reported as an incomplete scan.
// Shared-hosting accounts routinely carry tens of thousands of
// directories under vendor and node_modules trees, so raise this rather
// than accept the gap. Default 50000.
PHPConfigWalkMaxDirs int `yaml:"php_config_walk_max_dirs"`
// PHPConfigWalkMaxEntries caps how many directory entries that same
// walk examines below one document root. A shallow tree holding very
// many files reaches this before the directory ceiling. Default 500000.
PHPConfigWalkMaxEntries int `yaml:"php_config_walk_max_entries"`
// AccountScanMaxFiles caps how many account and mail-domain paths
// account-scoped scanners iterate per cycle. This covers SSH keys,
// cPanel API tokens, Dovecot shadow files, CMS DB scans, forwarders,
// user crontabs, and account .config backdoor paths. Paths are
// ranked by mtime desc so the cap chops least-active accounts.
// Default 10000 covers typical cPanel hosts with no cap effect.
// Raise on very large multi-tenant hosts where late-mtime accounts
// get skipped.
AccountScanMaxFiles int `yaml:"account_scan_max_files"`
// CrontabBase64BlobMaxBytes caps a single base64 candidate before
// decoding in MatchCrontabPatternsDeep. Default 16384 encoded
// bytes (~12 KiB decoded) comfortably fits any realistic gsocket
// or `base64 -d|bash` payload while bounding work on adversarial
// input. Raise on hosts where csm_checks_crontab_base64_truncated_total
// shows recurring truncation. Must be a multiple of 4: standard
// base64 requires aligned input or the decode fails and the
// candidate is silently skipped.
CrontabBase64BlobMaxBytes int `yaml:"crontab_base64_blob_max_bytes"`
// DomlogTailLines is how many trailing lines the WP brute force
// check reads from each per-domain access log per cycle. Default
// 500 covers roughly 10 minutes of traffic on a busy site. Raise
// on hosts where slow-burn attacks against high-volume domains
// spread across more than 500 lines per scan interval, so the
// per-IP counter window is wide enough to trip a threshold.
DomlogTailLines int `yaml:"domlog_tail_lines"`
// DomlogMaxAgeMin is how many minutes back the WP brute force
// scanner accepts a per-domain access log as "fresh enough to
// scan." Logs whose mtime is older than this are skipped. Default
// 30 keeps the scan focused on currently-active vhosts. Raise on
// low-traffic hosts where a slow-burn dictionary attack against
// a quiet domain still needs to fall inside the window.
DomlogMaxAgeMin int `yaml:"domlog_max_age_min"`
// MailLogTailLines is how many trailing lines CheckMailPerAccount
// reads from /var/log/exim_mainlog per cycle. Default 500. Raise
// on busy mail hosts where a single account's spam burst spreads
// across more than 500 lines per cycle.
MailLogTailLines int `yaml:"mail_log_tail_lines"`
// SyslogMessagesTailLines is kept for direct CheckFTPLogins callers
// that do not pass a state store. The daemon's store-backed FTP detector
// follows /var/log/messages forward and ignores this setting.
SyslogMessagesTailLines int `yaml:"syslog_messages_tail_lines"`
// FTPFailWindowMin is the sliding-window length (minutes) over which the
// store-backed FTP detector accumulates per-IP pure-ftpd auth failures
// before alerting at ftpFailThreshold. 0 means use the built-in default
// (30). Valid range 1..1440.
FTPFailWindowMin int `yaml:"ftp_fail_window_min"`
SMTPBruteForceThreshold int `yaml:"smtp_bruteforce_threshold"`
SMTPBruteForceWindowMin int `yaml:"smtp_bruteforce_window_min"`
SMTPBruteForceSuppressMin int `yaml:"smtp_bruteforce_suppress_min"`
SMTPBruteForceSubnetThresh int `yaml:"smtp_bruteforce_subnet_threshold"`
SMTPAccountSprayThreshold int `yaml:"smtp_account_spray_threshold"`
SMTPBruteForceMaxTracked int `yaml:"smtp_bruteforce_max_tracked"`
// The slow pair catches attackers pacing below the fast window:
// SlowThreshold failures within SlowWindowMin minutes across at
// least three distinct mailboxes from one IP.
SMTPBruteForceSlowThreshold int `yaml:"smtp_bruteforce_slow_threshold"`
SMTPBruteForceSlowWindowMin int `yaml:"smtp_bruteforce_slow_window_min"`
// SMTP probe abuse counts raw inbound SMTP connect events per source
// IP (independent of AUTH outcome) so probe-and-disconnect scanners
// that never reach the AUTH stage are still caught. Threshold sized
// well above any legitimate MUA usage. Explicit 0 disables.
SMTPProbeThreshold int `yaml:"smtp_probe_threshold"`
SMTPProbeWindowMin int `yaml:"smtp_probe_window_min"`
SMTPProbeSuppressMin int `yaml:"smtp_probe_suppress_min"`
SMTPProbeMaxTracked int `yaml:"smtp_probe_max_tracked"`
MailBruteForceThreshold int `yaml:"mail_bruteforce_threshold"`
MailBruteForceWindowMin int `yaml:"mail_bruteforce_window_min"`
MailBruteForceSuppressMin int `yaml:"mail_bruteforce_suppress_min"`
MailBruteForceSubnetThresh int `yaml:"mail_bruteforce_subnet_threshold"`
MailAccountSprayThreshold int `yaml:"mail_account_spray_threshold"`
MailBruteForceMaxTracked int `yaml:"mail_bruteforce_max_tracked"`
// The slow pair mirrors the SMTP one for the dovecot-native path.
MailBruteForceSlowThreshold int `yaml:"mail_bruteforce_slow_threshold"`
MailBruteForceSlowWindowMin int `yaml:"mail_bruteforce_slow_window_min"`
// MailBruteAccountKey selects how the account is extracted from a
// dovecot/postfix log line for per-account brute-force scoring.
// - builtin:dovecot-user (default) - match `user=<...>`
// - builtin:postfix-sasl - match `sasl_username=<...>`
// - regex:<pattern> - capture group 1 is the account
MailBruteAccountKey string `yaml:"mail_brute_account_key,omitempty"`
// Mail login spray detection remembers SMTP and IMAP/POP3 login
// failures and clean logins per IP for a long window. Mode is off,
// observe (findings only) or enforce (block). Explicit 0 disables
// the breadth or the missing-mailbox rule.
MailSprayMode string `yaml:"mail_spray_mode"`
MailSprayWindowMin int `yaml:"mail_spray_window_min"`
MailSprayBreadthMailboxes int `yaml:"mail_spray_breadth_mailboxes"`
MailSprayMissingNetworks int `yaml:"mail_spray_missing_networks"`
// MailSprayBlockExpiry is how long an enforce mode spray block lasts;
// other blocks keep auto_response.block_expiry.
MailSprayBlockExpiry string `yaml:"mail_spray_block_expiry"`
// ModSecEscalationHits is the number of ModSecurity denies from a
// single source IP, inside ModSecEscalationWindowMin, that
// promote the IP from "logged" to "escalated" (Critical finding
// + firewall hand-off). Default 3. Lower it on hosts where
// low-and-slow scanners go below the floor for too long.
ModSecEscalationHits int `yaml:"modsec_escalation_hits"`
// ModSecEscalationWindowMin is the sliding-window size for the
// hit counter. Default 10. Bumping it (e.g. to 60-240) catches
// paced attackers that spread denies across hours without
// changing the trip count.
ModSecEscalationWindowMin int `yaml:"modsec_escalation_window_min"`
// ModSecLowConfidenceEscalationHits is the low-confidence-only
// backstop: how many low-confidence policy/anomaly denies (e.g.
// COMODO content-type/anomaly rules) from one IP within the
// escalation window force a firewall escalation even when the
// burst carries no attack signature. It closes the "only trip
// anomaly rules" bypass without banning a single customer's
// checkout retry. Default 30. Raise it on hosts with high-volume
// legitimate apps that bulk-trip policy rules.
ModSecLowConfidenceEscalationHits int `yaml:"modsec_low_confidence_escalation_hits"`
// HTTPFloodThreshold is the minimum requests per http_flood_window_min
// from one source IP that emits http_request_flood. 0 (default) disables
// the detector. Operators should sample baseline traffic before setting
// a nonzero value.
HTTPFloodThreshold int `yaml:"http_flood_threshold"`
// HTTPFloodWindowMin is the rate window in minutes for HTTPFloodThreshold
// counting. Default 5.
HTTPFloodWindowMin int `yaml:"http_flood_window_min"`
// HTTPUASpoofThreshold is the minimum per-IP per-window count
// of WPSpoofPingback, cache-confirmed negative ClaimedBot,
// ScriptingLang, Headless, or Empty UA requests that emits
// http_ua_spoof. KnownScanner still emits on count=1. Default 30.
HTTPUASpoofThreshold int `yaml:"http_ua_spoof_threshold"`
// HTTPDistributedMinIPs is the number of distinct source IPs that
// must each trip an HTTP-abuse threshold (wp-login / xmlrpc / user
// enumeration / request flood / UA spoof) against one vhost in a
// scan window before a single http_distributed_flood finding is
// emitted for that vhost. Only already-abusive IPs are counted, so
// a popular site's ordinary visitor spread does not trip it. 0
// disables the detector; the shipped sample sets 10.
HTTPDistributedMinIPs int `yaml:"http_distributed_min_ips"`
// HTTPScannerMinRequests is the minimum in-window requests from one
// source IP before the URL scanner-profile detector evaluates the IP.
// The volume gate keeps a visitor following a handful of dead links
// out of scope. 0 (default) disables the detector; the shipped
// sample suggests 30.
HTTPScannerMinRequests int `yaml:"http_scanner_min_requests"`
// HTTPScannerErrorPct is the minimum percentage of in-window
// requests answered with a probe-error status before
// http_scanner_profile fires. Default 90.
HTTPScannerErrorPct int `yaml:"http_scanner_error_pct"`
// HTTPScannerMinDistinctPaths is the minimum count of distinct
// error-status request paths (query strings stripped) before the
// detector fires. Repeated hits on one missing resource (a dead
// bookmark, a broken image) never look like URL enumeration no
// matter the volume. Default 10, maximum 512.
HTTPScannerMinDistinctPaths int `yaml:"http_scanner_min_distinct_paths"`
// HTTPScannerStatusCodes is the set of response statuses counted as
// probe errors. Default [404, 403]. 301 is deliberately excluded:
// http->https and www redirects make every legitimate visitor
// 301-heavy, and a site migration redirects entire domains. Add 301
// here only on hosts where that traffic shape is impossible.
HTTPScannerStatusCodes []int `yaml:"http_scanner_status_codes"`
// HTTPUAScriptingEnabled opts in to flagging scripting-language
// UA strings (curl, python-requests, wget, etc.) as spoof candidates.
// Off by default: many legitimate API integrations use these.
HTTPUAScriptingEnabled bool `yaml:"http_ua_scripting_enabled"`
// HTTPUAHeadlessEnabled opts in to flagging headless-browser UA
// strings (HeadlessChrome, PhantomJS, Playwright, etc.). Off by
// default: headless browsers are used by legitimate monitoring tools.
HTTPUAHeadlessEnabled bool `yaml:"http_ua_headless_enabled"`
// HTTPUAEmptyEnabled opts in to flagging requests with an empty or
// dash User-Agent. Off by default: some CDN health checks omit UA.
HTTPUAEmptyEnabled bool `yaml:"http_ua_empty_enabled"`
// FullScanMaxFileMB caps how large a single file may be (in MiB)
// before a full-scan job skips it and emits a full_scan_file_too_large
// warning. 0 falls back to the default (16 MiB). Raise only when a
// known-large legitimate file must be scanned; the cap exists to
// bound memory use during content decoding.
FullScanMaxFileMB int `yaml:"full_scan_max_file_mb"`
// ScanJobRetention is how many completed full-scan job records the
// job manager keeps before evicting the oldest. Default 20.
ScanJobRetention int `yaml:"scan_job_retention"`
// RollingCoverage enables Phase 3 rolling content-scan coverage: each
// periodic cycle also sweeps a bounded path-sorted slice of files past
// the mtime cap so dormant files are eventually content-scanned.
// Default true; set false to restore pure top-N-by-mtime behavior.
RollingCoverage bool `yaml:"rolling_coverage"`
// DropperDetection gates the realtime self-deleting-dropper detector:
// a PHP/executable file created under a web document root and
// unlinked before the TTL probe. Default true.
DropperDetection bool `yaml:"dropper_detection"`
// DropperUnlinkTTLSec is the tracking TTL in seconds for that
// detector. Default 300.
DropperUnlinkTTLSec int `yaml:"dropper_unlink_ttl_sec"`
// HTTPASNCrawlWindowMin is the rolling window in minutes for the
// single-ASN distributed crawl detector. Default 60.
HTTPASNCrawlWindowMin int `yaml:"http_asn_crawl_window_min"`
// XMLRPCThreshold is the per-IP POST /xmlrpc.php count that trips
// xmlrpc_abuse in access-log based detectors (a hard auto-block). 0
// disables the check; an absent key defaults to 100.
XMLRPCThreshold int `yaml:"xmlrpc_threshold"`
// HTTPASNCrawlMinIPs is the minimum distinct source IPs from one ASN
// inside the window before http_asn_crawl fires. 0 disables the
// detector; an absent key defaults to 25.
HTTPASNCrawlMinIPs int `yaml:"http_asn_crawl_min_ips"`
// HTTPASNCrawlMinExpensive is the minimum uncacheable requests from the
// ASN inside the window. Default 250.
HTTPASNCrawlMinExpensive int `yaml:"http_asn_crawl_min_expensive"`
// HTTPASNCrawlMinSharePct is the minimum percentage of one account's
// total uncacheable requests that must originate from the ASN. 1..100.
// Default 50.
HTTPASNCrawlMinSharePct int `yaml:"http_asn_crawl_min_share_pct"`
// HTTPASNCrawlHighAmpPct is the share percentage above which the finding
// is promoted to High severity. 1..100. Default 50.
HTTPASNCrawlHighAmpPct int `yaml:"http_asn_crawl_high_amp_pct"`
// HTTPASNCrawlHighVolumeMult multiplies MinExpensive to obtain the
// High-severity volume threshold. Default 4.
HTTPASNCrawlHighVolumeMult int `yaml:"http_asn_crawl_high_volume_mult"`
// HTTPASNCrawlSaturation is the lsphp process saturation count that
// triggers a Critical finding. 0 means use
// performance.php_process_warn_per_user.
HTTPASNCrawlSaturation int `yaml:"http_asn_crawl_saturation"`
// HTTPASNCrawlMaxPrefix is the maximum number of distinct /24 (IPv4) or
// /48 (IPv6) prefixes within the ASN before the finding is promoted to
// Critical (distributed saturation). Default 8.
HTTPASNCrawlMaxPrefix int `yaml:"http_asn_crawl_max_prefix"`
// HTTPASNCrawl16PrefPct is the percentage of IPs that must share a /16
// prefix for the condensed-prefix heuristic to apply. 1..100. Default 60.
HTTPASNCrawl16PrefPct int `yaml:"http_asn_crawl_16_pref_pct"`
// HTTPASNCrawlMaxTrackedIPs caps how many source IPs per ASN are kept in
// the rolling window. Default 20000.
HTTPASNCrawlMaxTrackedIPs int `yaml:"http_asn_crawl_max_tracked_ips"`
// HTTPASNCrawlAllowlistASNs is a list of ASNs that are never flagged by
// the detector. Ships empty; operator-supplied list only.
HTTPASNCrawlAllowlistASNs []uint `yaml:"http_asn_crawl_allowlist_asns"`
// HTTPASNCrawlReverseProxyASNs lists CDN/reverse-proxy ASNs whose edge
// IPs are never directly flagged by this detector. Admission also
// refuses HTTP, cPanel and IP reputation forecasts against an address
// the ASN database places in one of these networks.
// Ships with Cloudflare (13335), Fastly (54113), and Akamai (20940).
// Set to [] to clear the seed; absent key retains the built-in list.
HTTPASNCrawlReverseProxyASNs []uint `yaml:"http_asn_crawl_reverse_proxy_asns"`
} `yaml:"thresholds" hotreload:"safe"`
InfraIPs []string `yaml:"infra_ips" hotreload:"restart"`
StatePath string `yaml:"state_path" hotreload:"restart"`
// Detection groups operator-facing knobs that gate individual scanners.
Detection struct {
// DBObjectScanning toggles the MySQL persistence scanner
// (triggers/events/procedures/functions). Tri-state *bool
// matching the existing yara_worker_enabled pattern: nil =
// default-on, *true = explicit on, *false = explicit off.
// When off both the Critical (db_malicious_*) and Warning
// (db_unexpected_*) emit paths fall silent; the manual
// `csm db-clean drop-object` CLI keeps working so operators
// can act on objects discovered by other means.
DBObjectScanning *bool `yaml:"db_object_scanning"`
// DBObjectAllowlist suppresses the Warning tier
// (db_unexpected_*) for objects an operator has reviewed and
// accepted. Entries shaped <account>:<schema>:<type>:<name>.
// The Critical tier (db_malicious_*) ignores this list --
// pattern hits always fire.
DBObjectAllowlist []string `yaml:"db_object_allowlist"`
// VulnerablePluginScanning toggles the known-vulnerable WordPress
// plugin detector (matches installed versions against a curated
// CVE/KEV feed). Tri-state *bool: nil = default-on, *true = on,
// *false = off. Independent of outdated_plugins.
VulnerablePluginScanning *bool `yaml:"vulnerable_plugin_scanning"`
// VulnerablePluginAllow suppresses vulnerable_plugins findings for a
// specific slug@version the operator has reviewed and accepted (for
// example a back-ported/vendor-patched build the feed cannot see).
// Entries shaped <slug>@<version>, case-insensitive presence check.
VulnerablePluginAllow []string `yaml:"vulnerable_plugin_allow"`
// AdminOverlapMinAccounts is the threshold at which the
// cross-account admin email correlator emits a finding. Default
// 2 matches the most common compromise pattern on shared hosting
// -- a contractor account used across multiple customer cPanels
// is a single credential leak away from compromising every site
// they touch. Operators with deliberately shared internal admin
// emails (e.g. one ops team across many sites) can raise the
// threshold to silence the routine overlap.
AdminOverlapMinAccounts int `yaml:"admin_overlap_min_accounts"`
// AdminOverlapTrustedEmails suppresses cross-account admin
// overlap findings for exact, operator-reviewed email addresses.
AdminOverlapTrustedEmails []string `yaml:"admin_overlap_trusted_emails"`
// AdminOverlapTrustedDomains suppresses cross-account admin
// overlap findings for exact email domains used by trusted
// developer or reseller admin accounts.
AdminOverlapTrustedDomains []string `yaml:"admin_overlap_trusted_domains"`
// RescanOnSignatureUpdate queues a rescan of existing files
// when the content of any file under cfg.Signatures.RulesDir
// changes. The rolling deep YARA walk tracks it and clears it
// once every file was scanned after the change. Tri-state *bool:
// nil = default-on, *true = explicit on, *false = explicit off.
// Off pauses the queue; the walk keeps running either way.
RescanOnSignatureUpdate *bool `yaml:"rescan_on_signature_update"`
// AFAlgBackend selects the live AF_ALG (CVE-2026-31431, "Copy
// Fail") detection backend. Empty / "auto" picks BPF LSM if
// the binary was built with -tags bpf and the kernel supports
// it, otherwise the audit-log inotify listener. "bpf" forces
// BPF and disables the audit fallback (no live monitor if BPF
// is unavailable; the periodic critical-tier check still
// runs). "auditd" forces the audit listener even on BPF-
// capable kernels -- a kill switch when a BPF-tagged release
// misbehaves and the operator wants to revert without
// rebuilding. "none" disables the live monitor entirely.
AFAlgBackend string `yaml:"af_alg_backend"`
// ConnectionTrackerBackend selects the live outbound-connection
// tracker. Empty / "auto" tries BPF cgroup/connect4,6 first and
// falls back to the existing /proc/net/tcp polling. "bpf"
// requires BPF (no fallback). "legacy" pins polling. "none"
// disables the live tracker; the periodic check still runs.
ConnectionTrackerBackend string `yaml:"connection_tracker_backend"`
// ConnectionPollInterval is how often the legacy polling backend
// reads /proc/net/tcp(6). Ignored when the BPF backend is active.
// Empty / zero defaults to 30s.
ConnectionPollInterval time.Duration `yaml:"connection_poll_interval"`
// ExecMonitorBackend selects the live process-exec monitor.
// Empty / "auto" tries the sched_process_exec BPF tracepoint and
// falls back to the periodic /proc walk. "bpf" requires BPF
// (no fallback). "legacy" pins polling. "none" disables the live
// monitor; the periodic deep-tier checks still run.
ExecMonitorBackend string `yaml:"exec_monitor_backend"`
// ExecMonitorPollInterval is how often the legacy polling backend
// runs CheckSuspiciousProcesses + CheckFakeKernelThreads. Ignored
// when the BPF backend is active. Empty / zero defaults to 30m.
ExecMonitorPollInterval time.Duration `yaml:"exec_monitor_poll_interval"`
// SensitiveFilesBackend selects the live sensitive-file write
// monitor. Empty / "auto" tries the BPF LSM hook on /etc/shadow
// and friends, falling back to a periodic content-hash check.
// "bpf" requires BPF (no fallback). "legacy" pins polling.
// "none" disables the live monitor; the periodic check still runs.
SensitiveFilesBackend string `yaml:"sensitive_files_backend"`
// SensitiveFilesPollInterval is how often the BPF watchset map
// refreshes (to pick up newly-created files in glob directories
// and handle inode reuse) and how often the legacy polling
// backend runs the content-hash check. Empty / zero defaults to 5m.
SensitiveFilesPollInterval time.Duration `yaml:"sensitive_files_poll_interval"`
// PlacementBackend selects how realtime protection sees files
// renamed or hard-linked into a watched tree. "auto" tries BPF LSM
// rename and link hooks, then a fanotify group that reports
// directory entries (kernel 5.9+), then none. "bpf" requires BPF
// (no fallback). "legacy" pins the fanotify group. "none" leaves
// those files to the deep scan. Chosen once at startup.
PlacementBackend string `yaml:"placement_backend" hotreload:"restart"`
// DirectSMTPEgress flags non-MTA local processes opening
// outbound SMTP connections. Phase 3 of the BPF Incident
// Response Roadmap. Detection-only this phase; Phase 4 will
// add the auto-response action gated by DryRun.
DirectSMTPEgress struct {
Enabled bool `yaml:"enabled"`
Backend string `yaml:"backend"` // auto / bpf / legacy / none
// DryRun, when true (or absent for safety), reports findings
// but takes no detector-scoped action. Phase 3 emits findings
// regardless; the knob exists for the Phase 4 action.
DryRun *bool `yaml:"dry_run,omitempty"`
Ports []int `yaml:"ports,omitempty"`
} `yaml:"direct_smtp_egress" hotreload:"safe"`
// BadASNOutbound flags outbound connections whose destination IP
// resolves (via the GeoLite2-ASN database) to a bad or unexpected
// autonomous system. It is the third leg of the host-takeover
// chain correlator (alongside a new uid-0 account and a planted
// suid binary). Requires the GeoLite2-ASN database. Off by default;
// classification needs operator-supplied ASN lists.
BadASNOutbound struct {
Enabled bool `yaml:"enabled"`
// BlockedASNs are autonomous system numbers always treated as
// bad (e.g. known bulletproof hosters).
BlockedASNs []uint `yaml:"blocked_asns"`
// AllowedASNs, when non-empty, switches to allowlist mode: any
// destination ASN outside this set is treated as bad. Use on
// hosts whose legitimate egress is confined to a few providers.
AllowedASNs []uint `yaml:"allowed_asns"`
} `yaml:"bad_asn_outbound" hotreload:"safe"`
} `yaml:"detection" hotreload:"safe"`
Suppressions struct {
UPCPWindowStart string `yaml:"upcp_window_start"`
UPCPWindowEnd string `yaml:"upcp_window_end"`
KnownAPITokens []string `yaml:"known_api_tokens"`
IgnorePaths []string `yaml:"ignore_paths"`
SuppressWebmail bool `yaml:"suppress_webmail_alerts"` // don't alert on webmail logins
SuppressCpanelLogin bool `yaml:"suppress_cpanel_login_alerts"` // don't alert on cPanel direct logins
TrustedCountries []string `yaml:"trusted_countries"` // ISO 3166-1 alpha-2 codes - suppress cPanel login alerts from these countries
} `yaml:"suppressions" hotreload:"safe"`
AutoResponse struct {
Enabled bool `yaml:"enabled"`
KillProcesses bool `yaml:"kill_processes"`
QuarantineFiles bool `yaml:"quarantine_files"`
MaxFileActionsPerHour int `yaml:"max_file_actions_per_hour"`
MaxFileActionsPerAccountPerHour int `yaml:"max_file_actions_per_account_per_hour"`
MaxFileActionFailuresPerHour int `yaml:"max_file_action_failures_per_hour"`
BlockIPs bool `yaml:"block_ips"`
BlockExpiry string `yaml:"block_expiry"` // e.g. "24h", "12h"
// HTTPASNCrawlTempban is the ban duration for http_asn_crawl findings
// when auto-response is enabled. Default "24h".
HTTPASNCrawlTempban string `yaml:"http_asn_crawl_tempban"`
EnforcePermissions bool `yaml:"enforce_permissions"` // auto-chmod 644 world/group-writable PHP files (default false)
FixWPCron bool `yaml:"fix_wp_cron"` // auto-disable WP-Cron + install per-user system cron on perf_wp_cron findings (default false)
BlockCpanelLogins bool `yaml:"block_cpanel_logins"` // block IPs on cPanel/webmail login alerts (default false)
// HTTPScannerAction selects the response for http_scanner_profile
// findings. "challenge" (default) routes the IP to the PoW
// challenge when the challenge subsystem is enabled and falls
// through to a firewall block when it is not; "block" always
// hard-blocks without offering a challenge.
HTTPScannerAction string `yaml:"http_scanner_action"`
NetBlock bool `yaml:"netblock"` // auto-block IPv4 /24 or IPv6 /64 at threshold
NetBlockThreshold int `yaml:"netblock_threshold"` // IPs from same IPv4 /24 or IPv6 /64 before subnet block (default 3)
NetBlockWindow string `yaml:"netblock_window"` // how far back blocked IPs count toward the threshold (default "168h")
// MaxBlocksPerHour caps per-IP auto-blocks per wall-clock hour.
// 0 uses DefaultMaxBlocksPerHour.
MaxBlocksPerHour int `yaml:"max_blocks_per_hour"`
PermBlock bool `yaml:"permblock"` // auto-promote to permanent after N temp blocks
PermBlockCount int `yaml:"permblock_count"` // temp blocks before permanent (default 4)
PermBlockInterval string `yaml:"permblock_interval"` // window for counting temp blocks (default "24h")
CleanDatabase bool `yaml:"clean_database"` // auto-clean malicious DB injections, revoke sessions, block attacker IPs (default false)
CleanHtaccess bool `yaml:"clean_htaccess"` // auto-clean .htaccess directives flagged by the hardened detectors (default false)
// MaxBlocksPerHourDefaulted records that the operator left
// max_blocks_per_hour unset or zero: the admission ceiling then
// takes its own default (AdmissionCeiling).
MaxBlocksPerHourDefaulted bool `yaml:"-" json:"-"`
// VirtualPatchExposedFiles controls automatic .htaccess "Require all
// denied" rules for confirmed web_exposed_* findings. "off" (default,
// also the value for any unrecognised setting): detection only. "manual":
// deny rules are written only when the operator runs `csm virtual-patch`.
// "auto": deny rules are written on every scan except for warning-only
// sample SQL, gated by DryRun. Use VirtualPatchMode() for the safe default.
VirtualPatchExposedFiles string `yaml:"virtual_patch_exposed_files"`
DisableEnforceAFAlg bool `yaml:"disable_enforce_af_alg"` // suspend periodic AF_ALG enforcement; marker file + detection remain active (default false = enforce when marker present)
CopyFailKillProcess bool `yaml:"copy_fail_kill_process"` // SIGKILL processes caught opening AF_ALG sockets via the live listener (default false; alert-only)
// MailAuthRecovery optionally self-heals the mail auth backend
// (cPanel's cpdoveauthd). CSM always probes the socket, alerts on an
// outage, and pauses mail/SMTP brute-force auto-block while it is down
// (cPanel only); only the service restart is gated here and is off by
// default. A restart runs only after the backend is continuously down
// for DownGrace, so a brief blip during nightly maintenance never trips it.
MailAuthRecovery struct {
RestartEnabled bool `yaml:"restart_enabled"` // run a service restart after a sustained outage (default false)
DownGrace string `yaml:"down_grace"` // continuously-down duration before restarting (default "10m")
MaxRestartsPerHour int `yaml:"max_restarts_per_hour"` // hourly cap on restart attempts (default 3)
RestartCommand string `yaml:"restart_command"` // command to run (default cPanel restartsrv_dovecot)
} `yaml:"mail_auth_recovery" hotreload:"restart"`
// DryRun, when true (or absent - safety default), logs the intended
// action but does NOT touch nftables. Mirrors the PHPRelay.DryRun
// pattern: pointer-bool to distinguish "operator explicitly set false"
// from "operator omitted the key". Implicit nil means dry-run on, so
// flipping block_ips: true alone never causes a real block.
DryRun *bool `yaml:"dry_run,omitempty"`
// PHPRelay controls the auto-freeze behaviour that companion
// email PHP-relay detectors emit findings for. Freeze and DryRun
// are *bool so we can distinguish OMITTED from EXPLICIT FALSE in
// YAML. A plain bool zero-value is false, which would let an
// operator write `freeze: true` and (by forgetting `dry_run`)
// get LIVE freezes against their will. Pointer values: nil =
// "not set in YAML"; *true / *false = explicit. Use the
// FreezeEnabled() / DryRunEnabled() accessors on *Config to
// resolve the safe defaults rather than dereferencing directly.
PHPRelay struct {
Freeze *bool `yaml:"freeze"`
DryRun *bool `yaml:"dry_run"`
MaxActionsPerMinute int `yaml:"max_actions_per_minute"`
} `yaml:"php_relay"`
// VerdictCallback lets phpanel observe each block decision before it's
// applied. CSM POSTs the verdict to the configured URL with HMAC-SHA256
// signing (same scheme as the phpanel webhook); the response is
// advisory - phpanel can attach a tenant_id, return "allow" to keep
// the event audit-only, or omit a response entirely
// (CSM proceeds with its default verdict). NOT a per-tenant nftables
// enforcement: that's a separate, larger feature.
//
// Secret resolution happens at call time (the verdict.Client reads
// HMACSecretEnv per call), so operators can rotate via env without
// restarting the daemon.
VerdictCallback struct {
Enabled bool `yaml:"enabled"`
URL string `yaml:"url"`
HMACSecret string `yaml:"hmac_secret,omitempty"`
HMACSecretEnv string `yaml:"hmac_secret_env,omitempty"`
TimeoutSec int `yaml:"timeout_sec"`
// RequireResponseSignature controls whether the panel must sign
// its response body with the same HMAC scheme used on the
// request (X-CSM-Signature header). Default true: when a secret
// is configured, CSM rejects unsigned or forged responses to
// prevent an on-path attacker from downgrading block to allow.
// Set false only during a phpanel rollout that has not yet
// implemented response signing. In that mode, CSM still checks
// nonce or timestamp fields the panel echoes; responses that
// omit both keep the legacy advisory shape working.
RequireResponseSignature *bool `yaml:"require_response_signature,omitempty"`
// AllowUnsigned opts out of the default fail-closed posture and
// permits the verdict callback to fire without an HMAC secret,
// including advisory "allow" responses. Only set true while
// bootstrapping a new panel or during local testing; production
// deployments must keep this false so the daemon refuses to start
// when the secret env var is empty.
AllowUnsigned bool `yaml:"allow_unsigned,omitempty"`
} `yaml:"verdict_callback"`
} `yaml:"auto_response" hotreload:"safe"`
// BPFEnforcement is the optional in-kernel deny path for matched
// outbound connections. Phase 4 of the BPF Incident Response
// Roadmap. Defaults are all-safe: enforcement off, dry-run on.
// Operators flip live denial only after dry-run telemetry review.
BPFEnforcement struct {
Enabled bool `yaml:"enabled"`
// DryRun, when true (or absent for safety), logs intended
// denials but allows the connect. False = real deny.
DryRun *bool `yaml:"dry_run,omitempty"`
DirectSMTPEgress bool `yaml:"direct_smtp_egress"`
// VerdictCallback, when true, asks auto_response.verdict_callback
// for an advisory ALLOW override before recording a USERSPACE
// action (incident close, audit note). The in-kernel hook NEVER
// waits on this; it would add latency to every connect.
VerdictCallback bool `yaml:"verdict_callback"`
} `yaml:"bpf_enforcement" hotreload:"safe"`
Challenge struct {
Enabled bool `yaml:"enabled"` // enable challenge pages instead of hard block for some IPs
ListenAddr string `yaml:"listen_addr"` // bind address for the challenge listener (default: 127.0.0.1)
ListenPort int `yaml:"listen_port"` // port for challenge server (default: 8439)
// ListenGroup is the group allowed to open the socket when
// listen_addr is unix:<path>; set it to the group your proxy
// runs as. Unused for TCP listeners.
ListenGroup string `yaml:"listen_group"`
// ListenAddr defaults to loopback because the production path
// keeps the listener private until an operator deliberately
// exposes it. The webserver integration redirects browsers to
// challenge.public_url, so installed direct mode needs a
// non-loopback listen address plus TLS material via
// challenge.tls_cert / tls_key, or the webui TLS fallback.
Secret string `yaml:"secret"` // HMAC secret for challenge tokens (auto-generated if empty)
Difficulty int `yaml:"difficulty"` // proof-of-work difficulty 0-5 (default: 2)
TrustedProxies []string `yaml:"trusted_proxies"` // proxies in front of the challenge listener; HTTP checks trust them too (see TrustedProxies)
// TLSCert / TLSKey activate HTTPS on the challenge listener. Empty
// values keep loopback listeners on plain HTTP. Direct/public
// listeners fall back to webui.tls_cert / webui.tls_key so
// single-cert hosts can opt in without duplicating paths.
TLSCert string `yaml:"tls_cert"`
TLSKey string `yaml:"tls_key"`
// PublicURL is the external URL the webserver redirect target
// points at. Operators put it on an existing TLS-valid host so
// the integration does not need a new DNS record or cert.
// Example: https://server.example.com:8439/challenge with
// listen_addr=0.0.0.0, tls_cert/tls_key set to the host's
// cpanel cert.
// When empty the webserver-integration installer refuses to
// run; per-vhost reverse-proxy is no longer supported because
// LSWS proxy emulation does not honor it at server scope.
PublicURL string `yaml:"public_url"`
// CaptchaFallback shows a third-party CAPTCHA widget when JS is
// disabled. All fields default empty; the feature is off until
// the operator supplies provider + keys.
CaptchaFallback struct {
Provider string `yaml:"provider"` // "turnstile" | "hcaptcha" | "" (off)
SiteKey string `yaml:"site_key"` // public key embedded in the HTML widget
SecretKey string `yaml:"secret_key"` // verified server-side against the provider
Timeout time.Duration `yaml:"timeout"` // HTTP timeout for siteverify (default 10s)
} `yaml:"captcha_fallback"`
// VerifiedSession lets operators mint a signed cookie that
// bypasses the PoW for the cookie's TTL. The signing key is
// generated at daemon startup and rotates on restart.
VerifiedSession struct {
Enabled bool `yaml:"enabled"`
CookieName string `yaml:"cookie_name"` // default: csm_admin_session
TTL time.Duration `yaml:"ttl"` // default: 4h
AdminSecret string `yaml:"admin_secret"` // shared secret POST'd to /challenge/admin-token
} `yaml:"verified_session"`
// VerifiedCrawlers allows-passes traffic from search crawlers
// whose IP forward-confirms a reverse-DNS PTR matching one of
// the configured providers.
VerifiedCrawlers struct {
Enabled bool `yaml:"enabled"`
Providers []string `yaml:"providers"` // names: googlebot | bingbot
CacheTTL time.Duration `yaml:"cache_ttl"` // default: 15m
} `yaml:"verified_crawlers"`
// PortGate locks the challenge listener TCP port to specific
// source IPs via nftables. Enabled implies the daemon owns a
// dedicated `csm_chal` inet table whose chain drops all traffic
// to challenge.listen_port except: loopback, operator infra_ips,
// and IPs the IPList has just flagged. The set entry carries
// the same TTL as the challenge entry, so nftables expires the
// allow even if the daemon dies before calling Revoke. Auto-off
// when listen_addr is loopback (gate has no effect there).
PortGate struct {
Enabled bool `yaml:"enabled"`
} `yaml:"port_gate"`
} `yaml:"challenge" hotreload:"restart"`
PHPShield struct {
Enabled bool `yaml:"enabled"` // watch PHP Shield event log for alerts (default: false)
} `yaml:"php_shield" hotreload:"restart"`
Reputation struct {
AbuseIPDBKey string `yaml:"abuseipdb_key"`
Whitelist []string `yaml:"whitelist"` // IPs to never flag as malicious
// Rspamd queries the local rspamd controller for per-IP reject/junk
// counts. Disabled by default. URL must reach the controller HTTP port
// (default 11334). Token is the controller's admin password (rspamadm
// pw -e), supplied via env when possible. Token resolution happens at
// query time so operators can rotate via env without daemon restart.
Rspamd struct {
Enabled bool `yaml:"enabled"`
URL string `yaml:"url"`
Token string `yaml:"token,omitempty"`
TokenEnv string `yaml:"token_env,omitempty"`
} `yaml:"rspamd"`
// Upstream is an HTTP threat-intel source - typically a panel host that
// caches AbuseIPDB / proprietary scores on behalf of every agent in its
// fleet. Disabled by default. Token resolution happens at query time
// (see internal/threatintel/upstream_source.go) so operators can rotate
// the bearer via TokenEnv without restarting the daemon.
Upstream struct {
Enabled bool `yaml:"enabled"`
URL string `yaml:"url"`
Token string `yaml:"token,omitempty"` // discouraged - prefer TokenEnv
TokenEnv string `yaml:"token_env,omitempty"`
CacheTTLMin int `yaml:"cache_ttl_min"`
TimeoutSec int `yaml:"timeout_sec"`
} `yaml:"upstream"`
// BotVerifyEnabled controls async PTR+forward-A verification of
// claimed search-engine bot IPs. *bool so an explicit false
// survives SIGHUP reload without being overwritten by applyDefaults.
// Default true.
BotVerifyEnabled *bool `yaml:"bot_verify_enabled"`
// VerifiedBots extends the built-in good-bot allowlist with operator
// entries: claimed-UA substrings confirmed by forward-confirmed
// reverse DNS against the listed registrable-domain suffixes.
// Additive -- built-in bots (Googlebot etc.) always apply. Lets an
// operator stop a legitimate crawler (typically SEO/backlink bots)
// from tripping the HTTP scanner-profile detector without a code
// change. Suffixes are validated against shared-hosting domains so a
// bad entry cannot turn the allowlist into a bypass.
VerifiedBots []VerifiedBot `yaml:"verified_bots"`
// BotRanges controls the auto-updater that refreshes published
// AI-crawler IP ranges (OpenAI, Perplexity) used to verify those bots
// by address. Embedded snapshots work without it; the updater keeps
// them current via outbound HTTPS to the vendor endpoints. Default on.
BotRanges struct {
AutoUpdate *bool `yaml:"auto_update"` // nil = true
UpdateInterval string `yaml:"update_interval"` // default "24h"; min 1h
} `yaml:"bot_ranges" hotreload:"restart"`
// Report emits signed, minimized abuse reports for confirmed-abuse
// findings to a central abuse database or a private collector.
// Opt-in; default off. Keys/secrets resolve from *_env at startup.
Report struct {
Enabled bool `yaml:"enabled"`
Classes []string `yaml:"classes"` // bruteforce, php_relay, credential_stuffing, bad_asn_egress, spam
SpoolPath string `yaml:"spool_path"` // bbolt file; default <state_dir>/abuse_reports.db
SpoolMax int `yaml:"spool_max"` // bounded queue size; default 10000
Targets []struct {
Name string `yaml:"name"`
URL string `yaml:"url"`
Transport string `yaml:"transport"` // ed25519 | hmac
NodeID string `yaml:"node_id"`
KeyID string `yaml:"key_id"`
KeyEnv string `yaml:"key_env"` // ed25519 hex private key, or hmac secret
TokenEnv string `yaml:"token_env"` // optional bearer for hmac collectors
} `yaml:"targets"`
} `yaml:"report" hotreload:"restart"`
// Central pulls a signed scored-set from the central abuse database and
// acts on it per Action. Opt-in; default off. Central data never hard
// blocks on its own (see Action / firebreaks).
Central struct {
Enabled bool `yaml:"enabled"`
SetURL string `yaml:"set_url"`
PubkeyEnv string `yaml:"pubkey_env"` // env holding the central Ed25519 public key (hex)
AuthTarget string `yaml:"auth_target"` // report target whose node key signs pulls
RefreshInterval string `yaml:"refresh_interval"` // e.g. "6h"; default 6h
Action string `yaml:"action"` // off | challenge | block_if_local_corroborated
BlockThreshold int `yaml:"block_threshold"` // node-local score threshold for block (default 60)
} `yaml:"central" hotreload:"restart"`
} `yaml:"reputation" hotreload:"safe"`
Signatures struct {
RulesDir string `yaml:"rules_dir"`
UpdateURL string `yaml:"update_url"`
AutoUpdate bool `yaml:"auto_update"` // auto-download rules daily (default: true if update_url set)
UpdateInterval string `yaml:"update_interval"` // how often to check (default: "24h")
SigningKey string `yaml:"signing_key"` // hex-encoded ed25519 public key for verifying rule updates
// AllowRuleCountDecrease permits a deliberately reduced, newer signed
// ruleset to bypass the default count-collapse guard.
AllowRuleCountDecrease bool `yaml:"allow_rule_count_decrease"`
YaraForge struct {
Enabled bool `yaml:"enabled"`
Tier string `yaml:"tier"` // "core", "extended", "full" (default: "core")
UpdateInterval string `yaml:"update_interval"` // default: "168h" (weekly)
DownloadURL string `yaml:"download_url"` // signed ZIP URL/template; supports {tier} and {version}
} `yaml:"yara_forge"`
// DisabledRules names rules to switch off: they are stripped from
// YARA-Forge downloads, skipped when the shipped .yml rules load,
// and stripped before the shipped .yar rules compile. `csm validate`
// warns about a name that matches no rule.
DisabledRules []string `yaml:"disabled_rules"`
// YaraWorkerEnabled is a tri-state: nil means "use system default"
// (default-on, per ROADMAP item 2 follow-up), *true means explicit on,
// *false means explicit off. Callers must nil-check before dereferencing;
// daemon.yaraWorkerOn() is the canonical accessor.
YaraWorkerEnabled *bool `yaml:"yara_worker_enabled"`
} `yaml:"signatures" hotreload:"restart"`
WebUI struct {
SessionLifetime string `yaml:"session_lifetime"`
SessionIdleTimeout string `yaml:"session_idle_timeout"`
Enabled bool `yaml:"enabled"`
Listen string `yaml:"listen"`
AuthToken string `yaml:"auth_token"`
AuthTokenIssuedAt string `yaml:"auth_token_issued_at"`
AuthTokenExpiresAt string `yaml:"auth_token_expires_at"`
MetricsToken string `yaml:"metrics_token" hotreload:"safe"` // optional Bearer token for /metrics; rotate via SIGHUP without restart
MetricsTokenIssuedAt string `yaml:"metrics_token_issued_at" hotreload:"safe"`
MetricsTokenExpiresAt string `yaml:"metrics_token_expires_at" hotreload:"safe"`
TLSCert string `yaml:"tls_cert"`
TLSKey string `yaml:"tls_key"`
UIDir string `yaml:"ui_dir"` // path to UI files on disk (default: /opt/csm/ui)
// AllowedOrigins lists extra browser origins ("https://host[:port]")
// whose API requests are accepted besides https://<hostname>:<port>.
// Loopback origins (SSH tunnels) are always accepted. Hot-reloadable.
AllowedOrigins []string `yaml:"allowed_origins,omitempty" hotreload:"safe"`
// Tokens is the multi-credential model added in v2.12.0. Each entry has
// a stable name (for audit), an opaque secret, and a scope that gates
// which endpoints accept it. Legacy AuthToken is preserved during the
// migration window so callers that read it directly keep working;
// applyDefaults populates Tokens from AuthToken when only the legacy
// field is set.
Tokens []WebUIToken `yaml:"tokens,omitempty"`
} `yaml:"webui" hotreload:"restart"`
// MCP is the optional read-only Model Context Protocol endpoint for
// support agents at /mcp on the Web UI listener. It has its own
// credentials, reaches only an allowlist of GET routes dispatched
// in-process, and can change nothing. Disabled by default; the endpoint
// is wired at startup, so the block requires a restart.
MCP MCPConfig `yaml:"mcp" hotreload:"restart"`
EmailAV EmailAVConfig `yaml:"email_av" hotreload:"restart"`
EmailProtection struct {
PasswordCheckIntervalMin int `yaml:"password_check_interval_min"`
HighVolumeSenders []string `yaml:"high_volume_senders"`
RateWarnThreshold int `yaml:"rate_warn_threshold"`
RateCritThreshold int `yaml:"rate_crit_threshold"`
RateWindowMin int `yaml:"rate_window_min"`
KnownForwarders []string `yaml:"known_forwarders"`
// PHPRelay is the operator-tunable knob block for the email
// PHP-relay protection feature (Stage 1). Thresholds default to
// the values set in applyDefaults(), except documented zero
// sentinels such as AccountVolumePerHour auto-derive and
// FanoutDistinctRecipients disabling only that gate.
// IPRecipientSpread and IPRecipientWindowHours tune Path 6 (outside
// recipients per client IP); an explicit zero spread disables it.
// IPRecipientMode is observe (findings only, the default) or
// enforce (challenge, then block).
PHPRelay struct {
// The relay pipeline is wired once at startup; a reload can retune
// it but cannot start or stop it, so toggling it needs a restart.
Enabled bool `yaml:"enabled" hotreload:"restart"`
RateWindowMin int `yaml:"rate_window_min"`
HeaderScoreVolumeMin int `yaml:"header_score_volume_min"`
AbsoluteVolumePerHour int `yaml:"absolute_volume_per_hour"`
AccountVolumePerHour int `yaml:"account_volume_per_hour"`
FanoutDistinctScripts int `yaml:"fanout_distinct_scripts"`
FanoutDistinctRecipients int `yaml:"fanout_distinct_recipients"`
FanoutWindowMin int `yaml:"fanout_window_min"`
IPRecipientSpread int `yaml:"ip_recipient_spread"`
IPRecipientWindowHours int `yaml:"ip_recipient_window_hours"`
IPRecipientMode string `yaml:"ip_recipient_mode"`
PoliciesDir string `yaml:"policies_dir"`
} `yaml:"php_relay"`
// CloudRelay scopes opt-out for the email_cloud_relay_abuse
// detector only. Use this when an operator legitimately runs a
// mailer on a public-cloud VM (Google Cloud, AWS, etc.) and the
// realtime/retro detectors keep false-firing on that mailbox.
// AllowUsers matches full mailboxes (case-insensitive). AllowDomains
// matches the domain part of the AUTH user (case-insensitive),
// covering every mailbox under that domain. Either match exits
// the detector before any window state is updated, so an
// allowlisted mailbox cannot prime the counter for another user.
// Leaving both empty preserves prior behavior. The shared
// EmailProtection.HighVolumeSenders list still applies as well.
CloudRelay struct {
AllowUsers []string `yaml:"allow_users"`
AllowDomains []string `yaml:"allow_domains"`
} `yaml:"cloud_relay"`
// ForwardGuard is the opt-in protection that holds spam/backscatter
// forward copies before they relay to an external provider. Default
// off; dry-run first. The MTA enforces; CSM only generates the rule
// and owns the quarantine, so it is never in the live mail path.
ForwardGuard ForwardGuardConfig `yaml:"forward_guard"`
// Antispam is the opt-in inbound spam filter through rspamd. Default
// off; dry-run first. Each cPanel account's own Spam Filters toggle
// decides whether its mail is filtered. Exim enforces; CSM only
// writes the hooks, so it is never in the live mail path.
Antispam AntispamConfig `yaml:"antispam"`
} `yaml:"email_protection" hotreload:"safe"`
Firewall *firewall.FirewallConfig `yaml:"firewall" hotreload:"restart"`
GeoIP struct {
AccountID string `yaml:"account_id"`
LicenseKey string `yaml:"license_key"`
Editions []string `yaml:"editions"`
AutoUpdate *bool `yaml:"auto_update"` // nil = true when credentials set
UpdateInterval string `yaml:"update_interval"` // default "24h"
} `yaml:"geoip" hotreload:"restart"`
ModSecErrorLog string `yaml:"modsec_error_log" hotreload:"restart"`
ModSec struct {
RulesFile string `yaml:"rules_file"` // path to modsec2.user.conf
OverridesFile string `yaml:"overrides_file"` // path to csm-overrides.conf
ReloadCommand string `yaml:"reload_command"` // e.g. "systemctl restart lsws"
} `yaml:"modsec" hotreload:"restart"`
// WebServer overrides the auto-detected web server paths. Every field is
// optional: anything left blank or empty falls back to what
// platform.Detect() returned at startup. Intended for hosts with a
// custom layout (reverse proxy in front of a second daemon, non-standard
// package locations, chroot, etc.).
WebServer struct {
Type string `yaml:"type"` // "apache", "nginx", "litespeed" -- overrides auto-detect
ConfigDir string `yaml:"config_dir"` // e.g. /etc/apache2 or /etc/nginx
AccessLogs []string `yaml:"access_logs"` // candidate access-log paths, tried in order
ErrorLogs []string `yaml:"error_logs"` // candidate error-log paths (used for modsec denies)
ModSecAudits []string `yaml:"modsec_audit_logs"` // candidate ModSecurity audit-log paths
// TrustedProxies lists the IP addresses or CIDR ranges of proxies
// whose X-Forwarded-For header names the client. It forms one list
// with challenge.trusted_proxies (see TrustedProxies); an address
// of this host is never trusted, and a listed address is never
// read as a client.
TrustedProxies []string `yaml:"trusted_proxies"` // IP/CIDR proxies allowed to supply X-Forwarded-For
// DomlogGlobs overrides the auto-detected per-vhost access-log glob
// patterns. When set, the platform default for the detected panel/OS
// is discarded and only these patterns are used. Leave empty to use
// the auto-detected globs.
DomlogGlobs []string `yaml:"domlog_globs"` // per-vhost access-log globs
} `yaml:"web_server" hotreload:"restart"`
// AccountRoots lets operators point the account-scan based checks at
// non-cPanel web root layouts. Each entry is a glob pattern expanded
// at check time. Validated directories also bound content remediation
// and quarantine restore. Examples:
//
// account_roots:
// - /var/www/*/public
// - /srv/http/*
// - /home/*/public_html # cPanel default (implicit when unset on cPanel)
//
// When unset, CSM uses the cPanel default of /home/*/public_html on
// cPanel hosts and an empty list on non-cPanel hosts (account-scan
// checks skip entirely). See docs/src/configuration.md for the full
// list of checks that consume this.
AccountRoots []string `yaml:"account_roots" hotreload:"restart"`
Performance struct {
Enabled *bool `yaml:"enabled"`
LoadHighMultiplier float64 `yaml:"load_high_multiplier"`
LoadCriticalMultiplier float64 `yaml:"load_critical_multiplier"`
PHPProcessWarnPerUser int `yaml:"php_process_warn_per_user"`
PHPProcessCriticalTotalMult int `yaml:"php_process_critical_total_multiplier"`
ErrorLogWarnSizeMB int `yaml:"error_log_warn_size_mb"`
ErrorLogCriticalSizeMB int `yaml:"error_log_critical_size_mb"`
MySQLJoinBufferMaxMB int `yaml:"mysql_join_buffer_max_mb"`
MySQLWaitTimeoutMax int `yaml:"mysql_wait_timeout_max"`
MySQLMaxConnectionsPerUser int `yaml:"mysql_max_connections_per_user"`
RedisBgsaveMinInterval int `yaml:"redis_bgsave_min_interval"`
RedisLargeDatasetGB int `yaml:"redis_large_dataset_gb"`
WPMemoryLimitMaxMB int `yaml:"wp_memory_limit_max_mb"`
WPTransientWarnMB int `yaml:"wp_transient_warn_mb"`
WPTransientCriticalMB int `yaml:"wp_transient_critical_mb"`
// WPCronFix tunes the WP-Cron remediation (manual fix from the Web UI
// and the daemon auto-response). Disabling WP-Cron without a real cron
// would stop scheduled tasks, so the fix also installs a per-user system
// cron that runs wp-cron.php on this interval.
WPCronFix struct {
IntervalMinutes int `yaml:"interval_minutes"` // system cron frequency; default 15, clamped to [1,60]
PHPBin string `yaml:"php_bin"` // cron interpreter override; empty => per-vhost, then detect
} `yaml:"wp_cron_fix"`
} `yaml:"performance" hotreload:"restart"`
Cloudflare struct {
Enabled bool `yaml:"enabled"`
RefreshHours int `yaml:"refresh_hours"`
} `yaml:"cloudflare" hotreload:"restart"`
C2Blocklist []string `yaml:"c2_blocklist" hotreload:"restart"`
BackdoorPorts []int `yaml:"backdoor_ports" hotreload:"restart"`
// DisabledChecks lists check names that should be skipped entirely by
// the runner (no execution, no finding, no email/webhook/audit). Use
// this when a whole category does not apply to a host (e.g. WAF/web
// checks on DNS-only cPanel servers). Distinct from
// alerts.email.disabled_checks, which only suppresses email but still
// runs the check and emits findings to other sinks.
DisabledChecks []string `yaml:"disabled_checks" hotreload:"safe"`
// Retention bounds bbolt growth in two independent ways:
// - Sweeps (opt-in via Enabled): a daily pass prunes per-bucket entries
// older than the configured TTL. Destructive, so off by default.
// - Compaction (automatic): the daemon reclaims freelist slack at startup
// when the file exceeds CompactMinSizeMB and is less than
// CompactFillRatio full. Non-destructive (it only rewrites the file to
// drop free pages), so it runs regardless of Enabled; set
// CompactMinSizeMB to 0 to disable it.
// All fields are hot-reload:"restart" because the retention goroutine and
// the startup compaction capture these on daemon start.
Retention struct {
Enabled bool `yaml:"enabled"` // opt-in (sweeps only)
FindingsDays int `yaml:"findings_days"` // default 90
HistoryDays int `yaml:"history_days"` // default 30
ReputationDays int `yaml:"reputation_days"` // default 180
SweepInterval string `yaml:"sweep_interval"` // default "24h"
CompactMinSizeMB int `yaml:"compact_min_size_mb"` // default 128; 0 disables auto-compaction
CompactFillRatio float64 `yaml:"compact_fill_ratio"` // default 0.5
} `yaml:"retention" hotreload:"restart"`
// Sentry ships panics and selected errors to a Sentry server for
// aggregation across hosts. Disabled by default; set enabled=true and
// provide a DSN from the Sentry project. Init is one-shot: changes
// require a daemon restart.
Sentry struct {
Enabled bool `yaml:"enabled"`
DSN string `yaml:"dsn"`
Environment string `yaml:"environment"` // e.g. "production", "staging"
SampleRate float64 `yaml:"sample_rate"` // 0 -> 1.0 (capture all errors)
Debug bool `yaml:"debug"` // SDK debug logs to stderr
} `yaml:"sentry" hotreload:"restart"`
// Debug exposes diagnostic endpoints. PprofListen, when set, binds a
// net/http/pprof server (heap/goroutine/CPU profiles) to the given
// host:port. It MUST be a loopback address (127.0.0.1 / ::1 / localhost):
// pprof leaks process internals and lets a caller trigger CPU/heap dumps,
// so it is never exposed off-box. Empty (default) disables it; reach it
// over an SSH tunnel. Restart-required because the listener starts once.
Debug struct {
PprofListen string `yaml:"pprof_listen"`
} `yaml:"debug" hotreload:"restart"`
// MailLogs selects the log source for the postfix/dovecot brute-force
// and relay detectors. Changing the source (file vs. journal) requires
// the daemon to re-attach its reader, so the field is tagged restart.
MailLogs MailLogsConfig `yaml:"mail_logs,omitempty" hotreload:"restart"`
// Stalwart locates the Stalwart log on hosts where Stalwart is the mail
// server. The reader attaches at startup, so changes need a restart.
Stalwart StalwartConfig `yaml:"stalwart,omitempty" hotreload:"restart"`
// Updates controls the upstream release-availability poll surfaced
// in the Web UI top banner. The daemon never downloads or applies
// updates -- it only tells the operator that a newer version
// exists. Disable wholesale on air-gapped hosts.
Updates struct {
// CheckEnabled is a tri-state. nil means default-on; explicit
// false disables the poll entirely (no outbound HTTP, no
// package-manager probe). Use a pointer so the absence of the
// key in YAML is distinguishable from `check_enabled: false`.
CheckEnabled *bool `yaml:"check_enabled"`
// Interval is parsed by time.ParseDuration. Defaults to 24h;
// clamped to a 1h floor by updatecheck.New.
Interval string `yaml:"interval"`
// GitHubAPIURL overrides the default release endpoint. Tests
// and air-gapped mirrors use this; leave empty in production.
GitHubAPIURL string `yaml:"github_api_url,omitempty"`
// PackageName is the apt/dnf package name to query when the
// GitHub call fails. Defaults to "csm".
PackageName string `yaml:"package_name,omitempty"`
} `yaml:"updates" hotreload:"restart"`
// Incidents groups correlator-side knobs the operator can tune
// without code changes. Hot-reload "restart" because the daemon
// captures these on startup; flipping mid-run would race the
// retention loop.
Incidents struct {
// AutoClose resolves Open / Contained incidents whose UpdatedAt
// exceeds the per-kind idle threshold. Default-on with safe
// thresholds; mailbox takeover, mailbox bruteforce, credential
// spray, and web attack expire at 24h, web-account compromise at
// 7d, and host-level kinds never auto-close. Operators who want
// to monitor decisions without writing back can flip dry_run=true.
AutoClose struct {
// Enabled is tri-state: nil (default) means default-on; an
// explicit false in YAML disables. Pointer so absence in YAML
// is distinguishable from "enabled: false".
Enabled *bool `yaml:"enabled"`
DryRun bool `yaml:"dry_run"`
// ByKind maps incident kind -> idle threshold (parsed by
// time.ParseDuration). Kinds absent from the map are never
// auto-closed. Use a string-keyed map so the operator can
// add custom kinds without recompiling. Empty map falls back
// to safe defaults (mailbox_takeover=24h,
// mailbox_bruteforce=24h, credential_spray=24h,
// web_attack=24h, web_account_compromise=7d).
ByKind map[string]string `yaml:"by_kind,omitempty"`
} `yaml:"auto_close"`
// SpraySuppression collapses one source IP brute-forcing many
// distinct mailboxes/accounts into a single credential_spray
// super-incident. Default-OFF + dry_run=TRUE so the path ships
// dark; counters and audit log show what would have happened.
// Operators flip enabled=true and dry_run=false after watching
// the counters on their own infra.
SpraySuppression struct {
Enabled bool `yaml:"enabled"`
DryRun bool `yaml:"dry_run"`
DistinctMailboxes int `yaml:"distinct_mailboxes"`
SeverityEscalateAt int `yaml:"severity_escalate_at"`
PerCheck []string `yaml:"per_check"`
MaxTrackedIPs int `yaml:"max_tracked_ips"`
// BlockAtSeverity drives the firewall hand-off. Empty (default)
// means detection-only: the super-incident opens and counters
// move, but the IP is not blocked. "high" blocks as soon as the
// detector trips at DistinctMailboxes. "critical" waits for the
// severity escalation (SeverityEscalateAt distinct mailboxes)
// before blocking. Auto_response.dry_run and block_ips still
// gate the actual firewall call.
BlockAtSeverity string `yaml:"block_at_severity"`
} `yaml:"spray_suppression"`
// AutoBlock is the generic incident-driven firewall hand-off.
// Independent of SpraySuppression; applies to any non-spray
// incident kind that carries a remote_ip in its correlation
// key. Default-OFF so the path is dormant until an operator
// opts in. Block requests still respect auto_response.enabled
// and auto_response.block_ips at decision time.
AutoBlock struct {
Enabled bool `yaml:"enabled"`
BlockAtSeverity string `yaml:"block_at_severity"`
Kinds []string `yaml:"kinds"`
} `yaml:"auto_block"`
} `yaml:"incidents" hotreload:"restart"`
}
// UpdatesCheckEnabled reports the YAML-level state for the upstream
// release poll. Defaults to TRUE when omitted (most operators want
// the banner). Set `updates.check_enabled: false` to disable.
func (c *Config) UpdatesCheckEnabled() bool {
return c.Updates.CheckEnabled == nil || *c.Updates.CheckEnabled
}
// UpdatesInterval returns the parsed poll interval. Falls back to
// 24h on parse error or when unset; updatecheck applies the floor.
func (c *Config) UpdatesInterval() time.Duration {
if c.Updates.Interval == "" {
return 24 * time.Hour
}
d, err := time.ParseDuration(c.Updates.Interval)
if err != nil {
return 24 * time.Hour
}
return d
}
// minBlockDigestInterval floors BlockDigestInterval: each interval saves the
// daily summary, even with the digest off. A lower configured value runs at
// the floor with a validation warning, so an upgrade never fails startup.
const minBlockDigestInterval = time.Minute
// BlockDigestInterval is how often watched blocks fold into the daily summary
// and the live alert dedup window. Empty, invalid, or non-positive values fall
// back to one hour; values below the floor run at the floor.
func (c *Config) BlockDigestInterval() time.Duration {
if c.Alerts.BlockDigest.Interval == "" {
return time.Hour
}
d, err := time.ParseDuration(c.Alerts.BlockDigest.Interval)
if err != nil || d <= 0 {
return time.Hour
}
return max(d, minBlockDigestInterval)
}
// UpdatesPackageName returns the apt/dnf package name to query.
// Defaults to "csm".
func (c *Config) UpdatesPackageName() string {
if c.Updates.PackageName == "" {
return "csm"
}
return c.Updates.PackageName
}
// PHPRelayFreezeEnabled reports whether auto-freeze should run for the
// email PHP-relay detectors. Defaults to false when freeze was not set
// in YAML — the operator must opt in explicitly.
func (cfg *Config) PHPRelayFreezeEnabled() bool {
return cfg.AutoResponse.PHPRelay.Freeze != nil && *cfg.AutoResponse.PHPRelay.Freeze
}
// IncidentsAutoCloseEnabled reports whether the auto-close path should
// run. Defaults to TRUE when the YAML key is absent so a fresh
// installation drains stale incidents without explicit opt-in. An
// explicit `incidents.auto_close.enabled: false` disables.
func (cfg *Config) IncidentsAutoCloseEnabled() bool {
return cfg.Incidents.AutoClose.Enabled == nil || *cfg.Incidents.AutoClose.Enabled
}
// IncidentsAutoCloseThresholds returns the per-kind idle thresholds in
// parsed form. Built from the operator's by_kind YAML map, falling
// back to safe defaults (mailbox_takeover=24h, mailbox_bruteforce=24h,
// web_account_compromise=7d, credential_spray=24h, web_attack=24h) when the
// operator did not supply a map.
// Unparseable durations are skipped silently so a typo in one entry
// does not disable the rest.
func (cfg *Config) IncidentsAutoCloseThresholds() map[string]time.Duration {
out := defaultIncidentAutoCloseThresholds()
for kind, raw := range cfg.Incidents.AutoClose.ByKind {
if raw == "" {
delete(out, kind)
continue
}
d, err := time.ParseDuration(raw)
if err != nil || d <= 0 {
continue
}
out[kind] = d
}
return out
}
func defaultIncidentAutoCloseThresholds() map[string]time.Duration {
return map[string]time.Duration{
"mailbox_takeover": 24 * time.Hour,
"mailbox_bruteforce": 24 * time.Hour,
"credential_spray": 24 * time.Hour,
"web_account_compromise": 7 * 24 * time.Hour,
"web_attack": 24 * time.Hour,
}
}
// IncidentsAutoBlockKinds returns the configured kinds set in the shape
// the correlator expects. Empty result means "any non-spray kind".
func (cfg *Config) IncidentsAutoBlockKinds() map[string]bool {
src := cfg.Incidents.AutoBlock.Kinds
out := make(map[string]bool, len(src))
for _, k := range src {
if k == "" {
continue
}
out[k] = true
}
return out
}
// IncidentsSpraySuppressionPerCheck returns the configured per-check map
// in the shape the correlator expects. Falls back to a safe default
// when the operator did not supply a list.
func (cfg *Config) IncidentsSpraySuppressionPerCheck() map[string]bool {
src := cfg.Incidents.SpraySuppression.PerCheck
if len(src) == 0 {
// Each name must be a Finding.Check emitted in production code;
// the spray detector gates on Check equality, so an unmatched name
// silently drops that source from spray collapse. email_auth_failure_realtime
// is emitted by daemon/watcher.go; pam_bruteforce and credential_stuffing
// by daemon/pam_listener.go (the latter is the one-IP-many-accounts signal).
src = []string{
"email_auth_failure_realtime",
"pam_bruteforce",
"credential_stuffing",
}
}
out := make(map[string]bool, len(src))
for _, c := range src {
if c == "" {
continue
}
out[c] = true
}
return out
}
// PHPRelayDryRunEnabled reports the YAML-level dry-run state for the
// email PHP-relay auto-freeze. Defaults to TRUE when dry_run was not
// set, which is the safe shipped behaviour: an operator who enables
// freeze without thinking about dry-run gets a dry-run, not a live
// freeze. nil-or-explicit-true => true; explicit-false => false.
func (cfg *Config) PHPRelayDryRunEnabled() bool {
return cfg.AutoResponse.PHPRelay.DryRun == nil || *cfg.AutoResponse.PHPRelay.DryRun
}
// AutoResponseDryRunEnabled mirrors PHPRelayDryRunEnabled: nil-or-true means true.
// When dry_run is absent from YAML the operator gets safe dry-run behaviour;
// explicit false is required to enable live nftables blocking.
func (cfg *Config) AutoResponseDryRunEnabled() bool {
return cfg.AutoResponse.DryRun == nil || *cfg.AutoResponse.DryRun
}
// Virtual-patch modes for auto_response.virtual_patch_exposed_files.
const (
VirtualPatchOff = "off"
VirtualPatchManual = "manual"
VirtualPatchAuto = "auto"
)
// VirtualPatchMode resolves auto_response.virtual_patch_exposed_files to one of
// off/manual/auto. Any unrecognised or empty value is treated as off so a typo
// never silently enables automatic file blocking.
func (cfg *Config) VirtualPatchMode() string {
switch strings.ToLower(strings.TrimSpace(cfg.AutoResponse.VirtualPatchExposedFiles)) {
case VirtualPatchManual:
return VirtualPatchManual
case VirtualPatchAuto:
return VirtualPatchAuto
default:
return VirtualPatchOff
}
}
// SignatureRescanEnabled reports whether rule updates queue a rescan of
// existing files. Tri-state *bool: nil (omitted) defaults to on.
func (cfg *Config) SignatureRescanEnabled() bool {
return cfg == nil || cfg.Detection.RescanOnSignatureUpdate == nil || *cfg.Detection.RescanOnSignatureUpdate
}
// VulnerablePluginScanningEnabled reports whether the known-vulnerable plugin
// detector runs. Tri-state *bool: nil (omitted) defaults to on.
func (cfg *Config) VulnerablePluginScanningEnabled() bool {
return cfg.Detection.VulnerablePluginScanning == nil || *cfg.Detection.VulnerablePluginScanning
}
// DirectSMTPEgressDryRunEnabled reports the YAML-level dry-run state
// for the direct SMTP egress detector. Defaults to TRUE when dry_run
// was omitted (safety default). Operators must explicitly set
// `dry_run: false` to flip the detector to active mode.
func (c *Config) DirectSMTPEgressDryRunEnabled() bool {
if c.Detection.DirectSMTPEgress.DryRun == nil {
return true
}
return *c.Detection.DirectSMTPEgress.DryRun
}
// BPFEnforcementDryRunEnabled reports the YAML-level dry-run state for
// BPF cgroup-deny enforcement. Defaults to TRUE when dry_run is omitted
// (safety default). Operators must explicitly set `dry_run: false` to
// flip the in-kernel program to live denial.
func (c *Config) BPFEnforcementDryRunEnabled() bool {
if c.AutoResponseDryRunEnabled() {
return true
}
if c.BPFEnforcement.DirectSMTPEgress && c.DirectSMTPEgressDryRunEnabled() {
return true
}
if c.BPFEnforcement.DryRun == nil {
return true
}
return *c.BPFEnforcement.DryRun
}
// BotVerifyEnabled reports whether async PTR+forward-A verification of
// claimed search-engine bots is on. Default true; explicit false is
// honored so operators on air-gapped networks can disable DNS calls.
func (c *Config) BotVerifyEnabled() bool {
if c.Reputation.BotVerifyEnabled == nil {
return true
}
return *c.Reputation.BotVerifyEnabled
}
// BotRangesAutoUpdate reports whether the AI-crawler IP-range auto-updater
// runs. Default true (embedded snapshots are used regardless).
func (c *Config) BotRangesAutoUpdate() bool {
if c.Reputation.BotRanges.AutoUpdate == nil {
return true
}
return *c.Reputation.BotRanges.AutoUpdate
}
type defaultPresence struct {
smtpProbeThreshold bool
smtpBruteSlowThreshold bool
mailBruteSlowThreshold bool
mailSprayBreadth bool
mailSprayMissingNetworks bool
forwardGuard forwardGuardPresence
antispam antispamPresence
phpRelay phpRelayPresence
retention retentionPresence
blockDigestMinBlock bool
thresholdsRollingCoverage bool
thresholdsDropperDetection bool
httpASNCrawlMinIPs bool
httpASNCrawlReverseProxy bool
xmlrpcThreshold bool
integrityImmutable bool
backdoorPorts bool
suppressWebmail bool
autoResponse autoResponsePresence
// The trusted_proxies lists as written, null entries kept as "".
webServerProxies, challengeProxies []string
webServerProxiesRead, challengeProxiesRead bool
// firewall records which firewall: keys the operator wrote, so a
// partial block keeps the defaults for everything unlisted. nil means
// the whole block was absent.
firewall map[string]bool
}
// autoResponsePresence distinguishes omitted values from explicit zero or
// empty values that validation must reject while the corresponding feature is
// enabled.
type autoResponsePresence struct {
blockExpiry bool
netBlockThreshold bool
netBlockWindow bool
permBlockCount bool
permBlockInterval bool
}
// forwardGuardPresence records which forward-guard fields were set explicitly,
// so an operator's literal false survives the safe default-true.
type forwardGuardPresence struct {
dryRun bool
retention bool
bounceBackscatter bool
spamFlagged bool
malware bool
badSenderIP bool
authFail bool
}
// antispamPresence records which antispam fields were set to a non-null
// value, so an explicit false or zero survives the defaults.
type antispamPresence struct {
dryRun bool
socket bool
rejectScore bool
tagScore bool
reporter bool
}
// phpRelayPresence records PHP-relay fields where a literal zero has runtime
// meaning and must not be overwritten by defaults.
type phpRelayPresence struct {
fanoutDistinctRecipients bool
ipRecipientSpread bool
}
// retentionPresence records fields where a literal zero has runtime meaning and
// must not be overwritten by defaults.
type retentionPresence struct {
compactMinSizeMB bool
}
// ForwardGuardConfig configures the email forward-guard. Enabled is the master
// switch (default off); DryRun (default true) accounts without holding. Each
// hold signal is individually toggleable; unset signals default on but only
// matter once Enabled.
type ForwardGuardConfig struct {
Enabled bool `yaml:"enabled"`
DryRun bool `yaml:"dry_run"`
HoldSignals ForwardHoldSignals `yaml:"hold_signals"`
QuarantineRetentionDays int `yaml:"quarantine_retention_days"`
}
// AntispamConfig configures inbound spam filtering through rspamd. Enabled is
// the master switch (default off); DryRun (default true) scans and logs
// without changing delivery.
type AntispamConfig struct {
Enabled bool `yaml:"enabled"`
DryRun bool `yaml:"dry_run"`
RspamdSocket string `yaml:"rspamd_socket"`
RejectScore int `yaml:"reject_score"`
TagScore int `yaml:"tag_score"`
Feeds AntispamFeedsConfig `yaml:"feeds"`
Learning AntispamLearningConfig `yaml:"learning"`
// Exempt lists recipient mailboxes and domains never filtered on a
// Postfix host. Exim ignores it; each cPanel account's Spam Filters
// toggle decides there.
Exempt []string `yaml:"exempt"`
}
// AntispamLearningConfig turns on learning from the mail users move to
// their Spam folder. It requires antispam and is off by default.
type AntispamLearningConfig struct {
Enabled bool `yaml:"enabled"`
// RedisSocket is the Unix socket of a Redis used only by learning.
// Empty means none: a cPanel host then uses the Redis rspamd is
// configured with, and a Postfix host cannot learn.
RedisSocket string `yaml:"redis_socket"`
// Reporter decides who casts one learning vote on a Postfix host: each
// mail domain ("domain", the default) or each mailbox ("mailbox").
Reporter string `yaml:"reporter"`
}
// AntispamFeedsConfig selects optional anti-spam data sources. Every feed
// is off until named; the key file is read only for spamhaus_dqs.
type AntispamFeedsConfig struct {
Enabled []string `yaml:"enabled"`
SpamhausDQSKeyFile string `yaml:"spamhaus_dqs_key_file"`
}
// ForwardHoldSignals toggles which layered signals may hold a forward copy.
type ForwardHoldSignals struct {
BounceBackscatter bool `yaml:"bounce_backscatter"`
SpamFlagged bool `yaml:"spam_flagged"`
Malware bool `yaml:"malware"`
BadSenderIP bool `yaml:"bad_sender_ip"`
AuthFail bool `yaml:"auth_fail"`
}
func applyDefaults(cfg *Config, presence defaultPresence) {
if presence.webServerProxiesRead {
cfg.WebServer.TrustedProxies = presence.webServerProxies
}
if presence.challengeProxiesRead {
cfg.Challenge.TrustedProxies = presence.challengeProxies
}
for i, name := range cfg.Alerts.Email.DisabledChecks {
cfg.Alerts.Email.DisabledChecks[i] = CanonicalCheckName(strings.TrimSpace(name))
}
// Defaults
if cfg.StatePath == "" {
cfg.StatePath = "/var/lib/csm/state"
}
if cfg.Mode == "" {
cfg.Mode = ModeEnforce
} else {
cfg.Mode = normalizeMode(cfg.Mode)
}
// Binary immutability defaults on; a config written before the key existed
// must not read as "disable protection". Explicit false is kept.
if !presence.integrityImmutable {
cfg.Integrity.Immutable = true
}
// Webmail login suppression defaults on, as both shipped templates and
// the documentation say; a config that omits the key must not start
// alerting on every webmail login. Explicit false is kept.
if !presence.suppressWebmail {
cfg.Suppressions.SuppressWebmail = true
}
if cfg.Alerts.AuditLog.File.Enabled && cfg.Alerts.AuditLog.File.Path == "" {
cfg.Alerts.AuditLog.File.Path = "/var/log/csm/audit.jsonl"
}
if cfg.Alerts.AuditLog.Syslog.Enabled {
if cfg.Alerts.AuditLog.Syslog.Network == "" {
cfg.Alerts.AuditLog.Syslog.Network = "udp"
}
if cfg.Alerts.AuditLog.Syslog.Facility == "" {
cfg.Alerts.AuditLog.Syslog.Facility = "local0"
}
}
if cfg.Alerts.Webhook.HMACSecretEnv != "" {
if v := os.Getenv(cfg.Alerts.Webhook.HMACSecretEnv); v != "" {
cfg.Alerts.Webhook.HMACSecret = v
}
}
{
bd := &cfg.Alerts.BlockDigest
if bd.Interval == "" {
bd.Interval = "1h"
}
if bd.SendOn == "" {
bd.SendOn = "any"
}
// A literal min_block: 0 means "always show the summary section" and
// must survive; only an absent key falls back to 1.
if bd.MinBlock == 0 && !presence.blockDigestMinBlock {
bd.MinBlock = 1
}
}
if cfg.Signatures.RulesDir == "" {
cfg.Signatures.RulesDir = "/opt/csm/rules"
}
if cfg.Signatures.YaraForge.Tier == "" {
cfg.Signatures.YaraForge.Tier = "core"
}
if cfg.Signatures.YaraForge.UpdateInterval == "" {
cfg.Signatures.YaraForge.UpdateInterval = "168h"
}
if cfg.Reputation.BotRanges.UpdateInterval == "" {
cfg.Reputation.BotRanges.UpdateInterval = "24h"
}
if cfg.WebUI.SessionLifetime == "" {
cfg.WebUI.SessionLifetime = DefaultBrowserSessionLifetime
}
if cfg.WebUI.SessionIdleTimeout == "" {
cfg.WebUI.SessionIdleTimeout = DefaultBrowserSessionIdleTimeout
}
if cfg.WebUI.Listen == "" {
cfg.WebUI.Listen = "0.0.0.0:9443"
}
if cfg.WebUI.AuthToken != "" && len(cfg.WebUI.Tokens) == 0 {
cfg.WebUI.Tokens = []WebUIToken{{
Name: "legacy-auth-token", Token: cfg.WebUI.AuthToken, Scope: "admin",
IssuedAt: cfg.WebUI.AuthTokenIssuedAt, ExpiresAt: cfg.WebUI.AuthTokenExpiresAt,
}}
}
if cfg.Thresholds.MailQueueWarn == 0 {
cfg.Thresholds.MailQueueWarn = 500
}
if cfg.Thresholds.MailQueueCrit == 0 {
cfg.Thresholds.MailQueueCrit = 2000
}
if cfg.Thresholds.DeepScanIntervalMin == 0 {
cfg.Thresholds.DeepScanIntervalMin = 60
}
if cfg.Thresholds.ExposedFileScanDepth == 0 {
cfg.Thresholds.ExposedFileScanDepth = DefaultExposedFileScanDepth
}
if cfg.Thresholds.PluginCheckIntervalMin == 0 {
cfg.Thresholds.PluginCheckIntervalMin = 1440
}
if cfg.Thresholds.BruteForceWindow == 0 {
cfg.Thresholds.BruteForceWindow = 5000
}
if cfg.Thresholds.PHPConfigWalkMaxDirs <= 0 {
cfg.Thresholds.PHPConfigWalkMaxDirs = 50000
}
if cfg.Thresholds.PHPConfigWalkMaxEntries <= 0 {
cfg.Thresholds.PHPConfigWalkMaxEntries = 500000
}
if cfg.Thresholds.DomlogMaxFiles == 0 {
cfg.Thresholds.DomlogMaxFiles = 500
}
if cfg.Thresholds.AccountScanMaxFiles == 0 {
cfg.Thresholds.AccountScanMaxFiles = 10000
}
if cfg.Thresholds.CrontabBase64BlobMaxBytes == 0 {
cfg.Thresholds.CrontabBase64BlobMaxBytes = 16384
}
if cfg.Thresholds.DomlogTailLines == 0 {
cfg.Thresholds.DomlogTailLines = 500
}
if cfg.Thresholds.DomlogMaxAgeMin == 0 {
cfg.Thresholds.DomlogMaxAgeMin = 30
}
if cfg.Thresholds.MailLogTailLines == 0 {
cfg.Thresholds.MailLogTailLines = 500
}
if cfg.Thresholds.SyslogMessagesTailLines == 0 {
cfg.Thresholds.SyslogMessagesTailLines = 200
}
if cfg.Thresholds.FTPFailWindowMin == 0 {
cfg.Thresholds.FTPFailWindowMin = 30
}
if cfg.Thresholds.CredStuffingDistinctAccounts == 0 {
cfg.Thresholds.CredStuffingDistinctAccounts = 5
}
if cfg.Thresholds.PAMBruteforceThreshold == 0 {
cfg.Thresholds.PAMBruteforceThreshold = 5
}
if cfg.Thresholds.PAMBruteforceWindowMin == 0 {
cfg.Thresholds.PAMBruteforceWindowMin = 10
}
if cfg.Thresholds.ModSecEscalationHits == 0 {
cfg.Thresholds.ModSecEscalationHits = 3
}
if cfg.Thresholds.ModSecEscalationWindowMin == 0 {
cfg.Thresholds.ModSecEscalationWindowMin = 10
}
if cfg.Thresholds.ModSecLowConfidenceEscalationHits == 0 {
cfg.Thresholds.ModSecLowConfidenceEscalationHits = 30
}
if cfg.Thresholds.HTTPFloodWindowMin <= 0 {
cfg.Thresholds.HTTPFloodWindowMin = 5
}
// HTTPFloodThreshold has no nonzero default: 0 means disabled and
// that is the shipped behavior.
if cfg.Thresholds.HTTPUASpoofThreshold <= 0 {
cfg.Thresholds.HTTPUASpoofThreshold = 30
}
if cfg.Thresholds.HTTPScannerErrorPct == 0 {
cfg.Thresholds.HTTPScannerErrorPct = DefaultHTTPScannerErrorPct
}
if cfg.Thresholds.HTTPScannerMinDistinctPaths == 0 {
cfg.Thresholds.HTTPScannerMinDistinctPaths = DefaultHTTPScannerMinDistinctPaths
}
if len(cfg.Thresholds.HTTPScannerStatusCodes) == 0 {
cfg.Thresholds.HTTPScannerStatusCodes = DefaultHTTPScannerStatusCodes()
}
if cfg.AutoResponse.HTTPScannerAction == "" {
cfg.AutoResponse.HTTPScannerAction = "challenge"
}
if cfg.Thresholds.SMTPBruteForceThreshold == 0 {
cfg.Thresholds.SMTPBruteForceThreshold = 5
}
if cfg.Thresholds.SMTPBruteForceWindowMin == 0 {
cfg.Thresholds.SMTPBruteForceWindowMin = 10
}
if cfg.Thresholds.SMTPBruteForceSuppressMin == 0 {
cfg.Thresholds.SMTPBruteForceSuppressMin = 60
}
if cfg.Thresholds.SMTPBruteForceSubnetThresh == 0 {
cfg.Thresholds.SMTPBruteForceSubnetThresh = 8
}
if cfg.Thresholds.SMTPAccountSprayThreshold == 0 {
cfg.Thresholds.SMTPAccountSprayThreshold = 12
}
if cfg.Thresholds.SMTPBruteForceMaxTracked == 0 {
cfg.Thresholds.SMTPBruteForceMaxTracked = 20000
}
// Explicit 0 disables the slow-brute signal; absence gets the default.
if cfg.Thresholds.SMTPBruteForceSlowThreshold == 0 && !presence.smtpBruteSlowThreshold {
cfg.Thresholds.SMTPBruteForceSlowThreshold = 40
}
if cfg.Thresholds.SMTPBruteForceSlowWindowMin == 0 {
cfg.Thresholds.SMTPBruteForceSlowWindowMin = 360
}
if cfg.Thresholds.SMTPProbeThreshold == 0 && !presence.smtpProbeThreshold {
cfg.Thresholds.SMTPProbeThreshold = 100
}
if cfg.Thresholds.SMTPProbeWindowMin == 0 {
cfg.Thresholds.SMTPProbeWindowMin = 5
}
if cfg.Thresholds.SMTPProbeSuppressMin == 0 {
cfg.Thresholds.SMTPProbeSuppressMin = 60
}
if cfg.Thresholds.SMTPProbeMaxTracked == 0 {
cfg.Thresholds.SMTPProbeMaxTracked = 20000
}
if cfg.Thresholds.MailBruteForceThreshold == 0 {
cfg.Thresholds.MailBruteForceThreshold = 5
}
if cfg.Thresholds.MailBruteForceWindowMin == 0 {
cfg.Thresholds.MailBruteForceWindowMin = 10
}
if cfg.Thresholds.MailBruteForceSuppressMin == 0 {
cfg.Thresholds.MailBruteForceSuppressMin = 60
}
if cfg.Thresholds.MailBruteForceSubnetThresh == 0 {
cfg.Thresholds.MailBruteForceSubnetThresh = 8
}
if cfg.Thresholds.MailAccountSprayThreshold == 0 {
cfg.Thresholds.MailAccountSprayThreshold = 12
}
if cfg.Thresholds.MailBruteForceMaxTracked == 0 {
cfg.Thresholds.MailBruteForceMaxTracked = 20000
}
// Explicit 0 disables the slow-brute signal; absence gets the default.
if cfg.Thresholds.MailBruteForceSlowThreshold == 0 && !presence.mailBruteSlowThreshold {
cfg.Thresholds.MailBruteForceSlowThreshold = 40
}
if cfg.Thresholds.MailBruteForceSlowWindowMin == 0 {
cfg.Thresholds.MailBruteForceSlowWindowMin = 360
}
if cfg.Thresholds.MailSprayMode == "" {
cfg.Thresholds.MailSprayMode = "observe"
}
if cfg.Thresholds.MailSprayWindowMin == 0 {
cfg.Thresholds.MailSprayWindowMin = 7 * 24 * 60
}
// Explicit 0 disables the rule; absence gets the default.
if cfg.Thresholds.MailSprayBreadthMailboxes == 0 && !presence.mailSprayBreadth {
cfg.Thresholds.MailSprayBreadthMailboxes = 2
}
if cfg.Thresholds.MailSprayMissingNetworks == 0 && !presence.mailSprayMissingNetworks {
cfg.Thresholds.MailSprayMissingNetworks = 3
}
if cfg.Thresholds.MailSprayBlockExpiry == "" {
cfg.Thresholds.MailSprayBlockExpiry = DefaultMailSprayBlockExpiry
}
if cfg.Alerts.MaxPerHour == 0 {
cfg.Alerts.MaxPerHour = 30
}
if cfg.Challenge.ListenAddr == "" {
cfg.Challenge.ListenAddr = "127.0.0.1"
}
if cfg.Challenge.ListenPort == 0 {
cfg.Challenge.ListenPort = 8439
}
if cfg.Challenge.Difficulty == 0 {
cfg.Challenge.Difficulty = 2
}
// An explicit empty list means no port is a backdoor port.
if !presence.backdoorPorts {
cfg.BackdoorPorts = []int{4444, 5555, 55553, 55555, 31337}
}
if cfg.Firewall == nil {
cfg.Firewall = firewall.DefaultConfig()
}
applyFirewallFieldDefaults(cfg.Firewall, presence.firewall)
if len(cfg.GeoIP.Editions) == 0 {
cfg.GeoIP.Editions = []string{"GeoLite2-City", "GeoLite2-ASN"}
}
if cfg.GeoIP.UpdateInterval == "" {
cfg.GeoIP.UpdateInterval = "24h"
}
EmailAVDefaults(&cfg.EmailAV)
if cfg.EmailProtection.PasswordCheckIntervalMin == 0 {
cfg.EmailProtection.PasswordCheckIntervalMin = 1440
}
if cfg.EmailProtection.RateWarnThreshold == 0 {
cfg.EmailProtection.RateWarnThreshold = 50
}
if cfg.EmailProtection.RateCritThreshold == 0 {
cfg.EmailProtection.RateCritThreshold = 100
}
if cfg.EmailProtection.RateWindowMin == 0 {
cfg.EmailProtection.RateWindowMin = 10
}
applyForwardGuardDefaults(&cfg.EmailProtection.ForwardGuard, presence.forwardGuard)
applyAntispamDefaults(&cfg.EmailProtection.Antispam, presence.antispam)
// EmailProtection.PHPRelay defaults. Freeze/DryRun are *bool and
// remain nil here -- accessors resolve the safe defaults
// (PHPRelayFreezeEnabled / PHPRelayDryRunEnabled) so we do NOT
// mutate them. AccountVolumePerHour stays at 0 by default to mark
// "auto-derive from cPanel maxemailsperhour" downstream.
if cfg.EmailProtection.PHPRelay.RateWindowMin == 0 {
cfg.EmailProtection.PHPRelay.RateWindowMin = 5
}
if cfg.EmailProtection.PHPRelay.HeaderScoreVolumeMin == 0 {
cfg.EmailProtection.PHPRelay.HeaderScoreVolumeMin = 5
}
if cfg.EmailProtection.PHPRelay.AbsoluteVolumePerHour == 0 {
cfg.EmailProtection.PHPRelay.AbsoluteVolumePerHour = 30
}
if cfg.EmailProtection.PHPRelay.FanoutDistinctScripts == 0 {
cfg.EmailProtection.PHPRelay.FanoutDistinctScripts = 3
}
if cfg.EmailProtection.PHPRelay.FanoutDistinctRecipients == 0 && !presence.phpRelay.fanoutDistinctRecipients {
cfg.EmailProtection.PHPRelay.FanoutDistinctRecipients = 5
}
if cfg.EmailProtection.PHPRelay.FanoutWindowMin == 0 {
cfg.EmailProtection.PHPRelay.FanoutWindowMin = 5
}
if cfg.EmailProtection.PHPRelay.IPRecipientSpread == 0 && !presence.phpRelay.ipRecipientSpread {
cfg.EmailProtection.PHPRelay.IPRecipientSpread = 5
}
if cfg.EmailProtection.PHPRelay.IPRecipientWindowHours == 0 {
cfg.EmailProtection.PHPRelay.IPRecipientWindowHours = 24
}
if cfg.EmailProtection.PHPRelay.IPRecipientMode == "" {
cfg.EmailProtection.PHPRelay.IPRecipientMode = "observe"
}
if cfg.EmailProtection.PHPRelay.PoliciesDir == "" {
cfg.EmailProtection.PHPRelay.PoliciesDir = "/opt/csm/policies/php_relay"
}
if cfg.AutoResponse.PHPRelay.MaxActionsPerMinute == 0 {
cfg.AutoResponse.PHPRelay.MaxActionsPerMinute = 60
}
if cfg.AutoResponse.MaxFileActionsPerHour == 0 {
cfg.AutoResponse.MaxFileActionsPerHour = DefaultMaxFileActionsPerHour
}
if cfg.AutoResponse.MaxFileActionsPerAccountPerHour == 0 {
cfg.AutoResponse.MaxFileActionsPerAccountPerHour = DefaultMaxFileActionsPerAccountPerHour
}
if cfg.AutoResponse.MaxFileActionFailuresPerHour == 0 {
cfg.AutoResponse.MaxFileActionFailuresPerHour = DefaultMaxFileActionFailuresPerHour
}
if cfg.AutoResponse.MaxBlocksPerHour == 0 {
cfg.AutoResponse.MaxBlocksPerHour = DefaultMaxBlocksPerHour
cfg.AutoResponse.MaxBlocksPerHourDefaulted = true
}
// The block path keeps fallbacks for Config values assembled in code.
// Loaded configs resolve omitted values here so status surfaces report the
// effective settings and validation can reject explicit invalid values.
if cfg.AutoResponse.BlockExpiry == "" && !presence.autoResponse.blockExpiry {
cfg.AutoResponse.BlockExpiry = DefaultBlockExpiry
}
if cfg.AutoResponse.NetBlockThreshold == 0 && !presence.autoResponse.netBlockThreshold {
cfg.AutoResponse.NetBlockThreshold = DefaultNetBlockThreshold
}
if cfg.AutoResponse.NetBlockWindow == "" && !presence.autoResponse.netBlockWindow {
cfg.AutoResponse.NetBlockWindow = DefaultNetBlockWindow
}
if cfg.AutoResponse.PermBlockCount == 0 && !presence.autoResponse.permBlockCount {
cfg.AutoResponse.PermBlockCount = DefaultPermBlockCount
}
if cfg.AutoResponse.PermBlockInterval == "" && !presence.autoResponse.permBlockInterval {
cfg.AutoResponse.PermBlockInterval = DefaultPermBlockInterval
}
if cfg.AutoResponse.MailAuthRecovery.DownGrace == "" {
cfg.AutoResponse.MailAuthRecovery.DownGrace = "10m"
}
if cfg.AutoResponse.MailAuthRecovery.MaxRestartsPerHour == 0 {
cfg.AutoResponse.MailAuthRecovery.MaxRestartsPerHour = 3
}
if cfg.AutoResponse.MailAuthRecovery.RestartCommand == "" {
cfg.AutoResponse.MailAuthRecovery.RestartCommand = "/usr/local/cpanel/scripts/restartsrv_dovecot"
}
// Performance defaults.
// Enabled is a tri-state *bool: nil means "use system default (on)", true means
// explicitly enabled, false means explicitly disabled. We do NOT apply a default
// here so that callers can distinguish "operator left it unset" (nil) from
// "operator set it to true" (&true). All callers must nil-check before dereferencing;
// perfEnabled() in checks/performance.go treats nil as true.
if cfg.Performance.LoadHighMultiplier == 0 {
cfg.Performance.LoadHighMultiplier = 1.0
}
if cfg.Performance.LoadCriticalMultiplier == 0 {
cfg.Performance.LoadCriticalMultiplier = 2.0
}
if cfg.Performance.PHPProcessWarnPerUser == 0 {
cfg.Performance.PHPProcessWarnPerUser = 20
}
if cfg.Performance.PHPProcessCriticalTotalMult == 0 {
cfg.Performance.PHPProcessCriticalTotalMult = 5
}
if cfg.Performance.ErrorLogWarnSizeMB == 0 {
cfg.Performance.ErrorLogWarnSizeMB = 50
}
if cfg.Performance.ErrorLogCriticalSizeMB == 0 {
cfg.Performance.ErrorLogCriticalSizeMB = 1024
}
if cfg.Performance.MySQLJoinBufferMaxMB == 0 {
cfg.Performance.MySQLJoinBufferMaxMB = 64
}
if cfg.Performance.MySQLWaitTimeoutMax == 0 {
cfg.Performance.MySQLWaitTimeoutMax = 3600
}
if cfg.Performance.MySQLMaxConnectionsPerUser == 0 {
cfg.Performance.MySQLMaxConnectionsPerUser = 10
}
if cfg.Performance.RedisBgsaveMinInterval == 0 {
cfg.Performance.RedisBgsaveMinInterval = 900
}
if cfg.Performance.RedisLargeDatasetGB == 0 {
cfg.Performance.RedisLargeDatasetGB = 4
}
if cfg.Performance.WPMemoryLimitMaxMB == 0 {
cfg.Performance.WPMemoryLimitMaxMB = 512
}
if cfg.Performance.WPTransientWarnMB == 0 {
cfg.Performance.WPTransientWarnMB = 1
}
if cfg.Performance.WPTransientCriticalMB == 0 {
cfg.Performance.WPTransientCriticalMB = 10
}
if cfg.Performance.WPCronFix.IntervalMinutes == 0 {
cfg.Performance.WPCronFix.IntervalMinutes = 15
}
if cfg.Cloudflare.RefreshHours == 0 {
cfg.Cloudflare.RefreshHours = 6
}
// Retention: defaults apply whether or not the feature is enabled, so
// that flipping `enabled: true` without further tuning gives the
// documented behaviour.
if cfg.Retention.FindingsDays == 0 {
cfg.Retention.FindingsDays = 90
}
if cfg.Retention.HistoryDays == 0 {
cfg.Retention.HistoryDays = 30
}
if cfg.Retention.ReputationDays == 0 {
cfg.Retention.ReputationDays = 180
}
if cfg.Retention.SweepInterval == "" {
cfg.Retention.SweepInterval = "24h"
}
if cfg.Retention.CompactMinSizeMB == 0 && !presence.retention.compactMinSizeMB {
cfg.Retention.CompactMinSizeMB = 128
}
if cfg.Retention.CompactFillRatio == 0 {
cfg.Retention.CompactFillRatio = 0.5
}
if cfg.MailLogs.Source == "" {
cfg.MailLogs.Source = "auto"
}
if len(cfg.MailLogs.Units) == 0 {
// Debian and Ubuntu run Postfix in the postfix@- instance unit.
// Journal matches are exact, so other instances must be listed.
cfg.MailLogs.Units = []string{"postfix", "postfix@-", "dovecot"}
}
if cfg.Stalwart.LogDir == "" {
cfg.Stalwart.LogDir = "/var/log/stalwart"
}
if cfg.Stalwart.LogPrefix == "" {
cfg.Stalwart.LogPrefix = "stalwart.log"
}
if cfg.Stalwart.APIURL == "" {
cfg.Stalwart.APIURL = "http://127.0.0.1:8080"
}
if cfg.Stalwart.APIKeyFile == "" {
cfg.Stalwart.APIKeyFile = "/etc/csm/stalwart-api.key"
}
if cfg.Updates.Interval == "" {
cfg.Updates.Interval = "24h"
}
if cfg.Updates.PackageName == "" {
cfg.Updates.PackageName = "csm"
}
if cfg.Thresholds.MailBruteAccountKey == "" {
cfg.Thresholds.MailBruteAccountKey = "builtin:dovecot-user"
}
if cfg.Thresholds.FullScanMaxFileMB == 0 {
cfg.Thresholds.FullScanMaxFileMB = 16
}
if cfg.Thresholds.ScanJobRetention == 0 {
cfg.Thresholds.ScanJobRetention = 20
}
// A literal rolling_coverage: false means "disable rolling coverage" and
// must survive; only an absent key falls back to the safe default of true.
if !presence.thresholdsRollingCoverage {
cfg.Thresholds.RollingCoverage = true
}
// Same shape as rolling_coverage: a literal dropper_detection: false is an
// operator opt-out; only an absent key defaults to enabled.
if !presence.thresholdsDropperDetection {
cfg.Thresholds.DropperDetection = true
}
if cfg.Thresholds.DropperUnlinkTTLSec == 0 {
cfg.Thresholds.DropperUnlinkTTLSec = DefaultDropperUnlinkTTLSec
}
// http_asn_crawl: min_ips uses presence so an explicit 0 disables the
// detector; an absent key defaults to 25.
if !presence.httpASNCrawlMinIPs && cfg.Thresholds.HTTPASNCrawlMinIPs == 0 {
cfg.Thresholds.HTTPASNCrawlMinIPs = DefaultHTTPASNCrawlMinIPs
}
// xmlrpc_threshold uses presence so an explicit 0 disables the check; an
// absent key defaults to 100.
if !presence.xmlrpcThreshold && cfg.Thresholds.XMLRPCThreshold == 0 {
cfg.Thresholds.XMLRPCThreshold = DefaultXMLRPCThreshold
}
if cfg.Thresholds.HTTPASNCrawlMinExpensive == 0 {
cfg.Thresholds.HTTPASNCrawlMinExpensive = DefaultHTTPASNCrawlMinExpensive
}
if cfg.Thresholds.HTTPASNCrawlMinSharePct == 0 {
cfg.Thresholds.HTTPASNCrawlMinSharePct = DefaultHTTPASNCrawlMinSharePct
}
if cfg.Thresholds.HTTPASNCrawlHighAmpPct == 0 {
cfg.Thresholds.HTTPASNCrawlHighAmpPct = DefaultHTTPASNCrawlHighAmpPct
}
if cfg.Thresholds.HTTPASNCrawlHighVolumeMult == 0 {
cfg.Thresholds.HTTPASNCrawlHighVolumeMult = DefaultHTTPASNCrawlHighVolMult
}
if cfg.Thresholds.HTTPASNCrawlMaxPrefix == 0 {
cfg.Thresholds.HTTPASNCrawlMaxPrefix = DefaultHTTPASNCrawlMaxPrefix
}
if cfg.Thresholds.HTTPASNCrawl16PrefPct == 0 {
cfg.Thresholds.HTTPASNCrawl16PrefPct = DefaultHTTPASNCrawl16PrefPct
}
if cfg.Thresholds.HTTPASNCrawlMaxTrackedIPs == 0 {
cfg.Thresholds.HTTPASNCrawlMaxTrackedIPs = DefaultHTTPASNCrawlMaxTrackedIPs
}
if cfg.Thresholds.HTTPASNCrawlWindowMin == 0 {
cfg.Thresholds.HTTPASNCrawlWindowMin = DefaultHTTPASNCrawlWindowMin
}
// Reverse-proxy seed: absent key keeps the safety seed; an explicit empty
// list is honored as an operator override.
if !presence.httpASNCrawlReverseProxy && len(cfg.Thresholds.HTTPASNCrawlReverseProxyASNs) == 0 {
cfg.Thresholds.HTTPASNCrawlReverseProxyASNs = append([]uint(nil), httpASNCrawlReverseProxySeed...)
}
if cfg.AutoResponse.HTTPASNCrawlTempban == "" {
cfg.AutoResponse.HTTPASNCrawlTempban = DefaultHTTPASNCrawlTempban
}
if cfg.Reputation.Rspamd.URL == "" {
cfg.Reputation.Rspamd.URL = "http://127.0.0.1:11334"
}
// Token resolution happens at query time (see RspamdSource.Score).
if cfg.Reputation.Upstream.CacheTTLMin == 0 {
cfg.Reputation.Upstream.CacheTTLMin = 15
}
if cfg.Reputation.Upstream.TimeoutSec == 0 {
cfg.Reputation.Upstream.TimeoutSec = 5
}
// Token resolution happens at query time (UpstreamSource.resolveToken).
if cfg.AutoResponse.VerdictCallback.TimeoutSec == 0 {
cfg.AutoResponse.VerdictCallback.TimeoutSec = 2 // tight; the hook is on the block hot path
}
// Secret resolution happens at call time (verdict.Client reads env per call).
// Direct SMTP egress detector defaults. Backend "auto" lets the runtime
// pick BPF where available and fall back to legacy polling. Standard
// submission/relay ports cover the bulk of mass-mail abuse seen in the
// wild; operators can override via YAML to add e.g. 2525.
if cfg.Detection.DirectSMTPEgress.Backend == "" {
cfg.Detection.DirectSMTPEgress.Backend = "auto"
}
if cfg.Detection.PlacementBackend == "" {
cfg.Detection.PlacementBackend = "auto"
}
if len(cfg.Detection.DirectSMTPEgress.Ports) == 0 {
cfg.Detection.DirectSMTPEgress.Ports = []int{25, 465, 587}
}
}
// MaxConfigBytes caps the YAML config body size LoadBytes will parse.
// Real CSM configs (main + every drop-in fragment) top out near 64 KB
// even with verbose comments; 4 MB is several orders of magnitude
// above legitimate use and far below the size at which a malformed
// or attacker-supplied file would force the YAML parser to allocate
// gigabytes of intermediate state.
const MaxConfigBytes = 4 * 1024 * 1024
var errConfigTooLarge = errors.New("config input exceeds byte cap")
func readConfigBytesLimited(r io.Reader) ([]byte, error) {
data, err := io.ReadAll(io.LimitReader(r, MaxConfigBytes+1))
if err != nil {
return nil, err
}
if int64(len(data)) > MaxConfigBytes {
return nil, errConfigTooLarge
}
return data, nil
}
// LoadOptions adjusts validation for callers that load the config without
// starting the daemon. The zero value is the daemon's fail-closed behaviour.
type LoadOptions struct {
// AllowUnsetSecretEnv accepts a secret named by a *_env key whose
// variable is unset. Use only for file integrity checks or redacted
// display, which can run outside the service environment.
AllowUnsetSecretEnv bool
}
// LoadBytes decodes a YAML config body and applies all defaults,
// matching Load. ConfigFile is left empty; the caller sets it.
func LoadBytes(data []byte) (*Config, error) {
return LoadBytesWithOptions(data, LoadOptions{})
}
// LoadBytesWithOptions is LoadBytes with the validation adjusted by opts.
func LoadBytesWithOptions(data []byte, opts LoadOptions) (*Config, error) {
if len(data) > MaxConfigBytes {
return nil, fmt.Errorf("parsing config: input size %d exceeds %d byte cap", len(data), MaxConfigBytes)
}
data, removed, err := stripRemovedKeys(data)
if err != nil {
return nil, fmt.Errorf("parsing config: %w", err)
}
presence, err := defaultPresenceFromYAML(data)
if err != nil {
return nil, fmt.Errorf("parsing config: %w", err)
}
cfg := &Config{RemovedKeys: removed}
dec := yaml.NewDecoder(bytes.NewReader(data))
dec.KnownFields(true)
if err := dec.Decode(cfg); err != nil && !errors.Is(err, io.EOF) {
return nil, fmt.Errorf("parsing config: %w", err)
}
applyDefaults(cfg, presence)
if err := validateMode(cfg); err != nil {
return nil, err
}
if err := validateWebUITokens(cfg); err != nil {
return nil, err
}
if err := validateMCPTokens(cfg); err != nil {
return nil, err
}
if err := validateMailLogs(cfg); err != nil {
return nil, err
}
if _, err := validateStalwartField(cfg); err != nil {
return nil, err
}
if err := validateMailBruteAccountKey(cfg); err != nil {
return nil, err
}
if err := validateReputation(cfg); err != nil {
return nil, err
}
if err := validateVerifiedBotsConfig(cfg); err != nil {
return nil, err
}
if _, err := validateVerdictCallbackField(cfg, opts); err != nil {
return nil, err
}
if err := validateDirectSMTPEgress(cfg); err != nil {
return nil, err
}
if err := validateForwardGuard(cfg); err != nil {
return nil, err
}
if err := validateAntispam(cfg); err != nil {
return nil, err
}
if err := validateHTTPASNCrawl(cfg); err != nil {
return nil, err
}
if err := validateFirewallConfig(cfg); err != nil {
return nil, err
}
return cfg, nil
}
// trustedProxyList reads a trusted_proxies sequence with each null entry kept
// as "". Anything else is left to the typed decode, which reports it.
func trustedProxyList(node yaml.Node) ([]string, bool) {
var items []*string
if yamlResolvedTag(&node) != "!!seq" || node.Decode(&items) != nil {
return nil, false
}
out := make([]string, len(items))
for i, item := range items {
if item != nil {
out[i] = *item
}
}
return out, true
}
func defaultPresenceFromYAML(data []byte) (defaultPresence, error) {
var presence defaultPresence
if len(bytes.TrimSpace(data)) == 0 {
return presence, nil
}
var raw struct {
Thresholds map[string]yaml.Node `yaml:"thresholds"`
EmailProtection struct {
ForwardGuard map[string]yaml.Node `yaml:"forward_guard"`
PHPRelay map[string]yaml.Node `yaml:"php_relay"`
Antispam map[string]yaml.Node `yaml:"antispam"`
} `yaml:"email_protection"`
Alerts struct {
BlockDigest map[string]yaml.Node `yaml:"block_digest"`
} `yaml:"alerts"`
Retention map[string]yaml.Node `yaml:"retention"`
Integrity map[string]yaml.Node `yaml:"integrity"`
Suppressions map[string]yaml.Node `yaml:"suppressions"`
AutoResponse map[string]yaml.Node `yaml:"auto_response"`
Firewall map[string]yaml.Node `yaml:"firewall"`
BackdoorPorts yaml.Node `yaml:"backdoor_ports"`
WebServer map[string]yaml.Node `yaml:"web_server"`
Challenge map[string]yaml.Node `yaml:"challenge"`
}
if err := yaml.Unmarshal(data, &raw); err != nil {
return presence, err
}
// yaml.v3 drops null sequence entries when decoding []string. Keep them
// as "" so Validate reports them instead of losing them silently.
if node, ok := raw.WebServer["trusted_proxies"]; ok {
presence.webServerProxies, presence.webServerProxiesRead = trustedProxyList(node)
}
if node, ok := raw.Challenge["trusted_proxies"]; ok {
presence.challengeProxies, presence.challengeProxiesRead = trustedProxyList(node)
}
if raw.Firewall != nil {
presence.firewall = make(map[string]bool, len(raw.Firewall))
for key, node := range raw.Firewall {
// A null node means "no value", which is what an absent key means,
// so it must not count as present. The distinction matters for
// every field whose zero value is a real setting: `x: null` has to
// keep the shipped default while `x: 0` (or `[]`, or `false`)
// stays the operator's explicit choice.
presence.firewall[key] = !yamlNodeIsNull(&node)
}
}
presence.backdoorPorts = raw.BackdoorPorts.Kind != 0 && !yamlNodeIsNull(&raw.BackdoorPorts)
_, presence.integrityImmutable = raw.Integrity["immutable"]
_, presence.suppressWebmail = raw.Suppressions["suppress_webmail_alerts"]
if node, ok := raw.AutoResponse["block_expiry"]; ok && !yamlNodeIsNull(&node) {
presence.autoResponse.blockExpiry = true
}
if node, ok := raw.AutoResponse["netblock_threshold"]; ok && !yamlNodeIsNull(&node) {
presence.autoResponse.netBlockThreshold = true
}
if node, ok := raw.AutoResponse["netblock_window"]; ok && !yamlNodeIsNull(&node) {
presence.autoResponse.netBlockWindow = true
}
if node, ok := raw.AutoResponse["permblock_count"]; ok && !yamlNodeIsNull(&node) {
presence.autoResponse.permBlockCount = true
}
if node, ok := raw.AutoResponse["permblock_interval"]; ok && !yamlNodeIsNull(&node) {
presence.autoResponse.permBlockInterval = true
}
_, presence.smtpProbeThreshold = raw.Thresholds["smtp_probe_threshold"]
if node, ok := raw.Thresholds["smtp_bruteforce_slow_threshold"]; ok && !yamlNodeIsNull(&node) {
presence.smtpBruteSlowThreshold = true
}
if node, ok := raw.Thresholds["mail_bruteforce_slow_threshold"]; ok && !yamlNodeIsNull(&node) {
presence.mailBruteSlowThreshold = true
}
if node, ok := raw.Thresholds["mail_spray_breadth_mailboxes"]; ok && !yamlNodeIsNull(&node) {
presence.mailSprayBreadth = true
}
if node, ok := raw.Thresholds["mail_spray_missing_networks"]; ok && !yamlNodeIsNull(&node) {
presence.mailSprayMissingNetworks = true
}
_, presence.thresholdsRollingCoverage = raw.Thresholds["rolling_coverage"]
_, presence.thresholdsDropperDetection = raw.Thresholds["dropper_detection"]
_, presence.httpASNCrawlMinIPs = raw.Thresholds["http_asn_crawl_min_ips"]
_, presence.httpASNCrawlReverseProxy = raw.Thresholds["http_asn_crawl_reverse_proxy_asns"]
_, presence.xmlrpcThreshold = raw.Thresholds["xmlrpc_threshold"]
_, presence.phpRelay.fanoutDistinctRecipients = raw.EmailProtection.PHPRelay["fanout_distinct_recipients"]
// yaml.v3 truncates fractions into ints, including zero sentinels, so
// reject non-integer recipient settings before the typed decode.
for _, key := range []string{"ip_recipient_spread", "ip_recipient_window_hours"} {
if node, ok := raw.EmailProtection.PHPRelay[key]; ok && !yamlNodeIsNull(&node) && yamlResolvedTag(&node) != "!!int" {
return presence, fmt.Errorf("email_protection.php_relay.%s must be a whole number", key)
}
}
if node, ok := raw.EmailProtection.PHPRelay["ip_recipient_spread"]; ok && !yamlNodeIsNull(&node) {
presence.phpRelay.ipRecipientSpread = true
}
if node, ok := raw.Alerts.BlockDigest["min_block"]; ok && node.Tag != "!!null" {
presence.blockDigestMinBlock = true
}
if node, ok := raw.Retention["compact_min_size_mb"]; ok && node.Tag != "!!null" {
presence.retention.compactMinSizeMB = true
}
fg := raw.EmailProtection.ForwardGuard
_, presence.forwardGuard.dryRun = fg["dry_run"]
_, presence.forwardGuard.retention = fg["quarantine_retention_days"]
if node, ok := fg["hold_signals"]; ok {
var sig map[string]yaml.Node
if err := node.Decode(&sig); err != nil {
return presence, err
}
_, presence.forwardGuard.bounceBackscatter = sig["bounce_backscatter"]
_, presence.forwardGuard.spamFlagged = sig["spam_flagged"]
_, presence.forwardGuard.malware = sig["malware"]
_, presence.forwardGuard.badSenderIP = sig["bad_sender_ip"]
_, presence.forwardGuard.authFail = sig["auth_fail"]
}
as := raw.EmailProtection.Antispam
set := func(fields map[string]yaml.Node, key string) bool {
node, ok := fields[key]
return ok && !yamlNodeIsNull(&node)
}
// yaml.v3 truncates 6.5 into an int field without an error, so a score
// must be checked as a YAML integer before decoding.
for _, key := range []string{"reject_score", "tag_score"} {
if node, ok := as[key]; ok && !yamlNodeIsNull(&node) && yamlResolvedTag(&node) != "!!int" {
return presence, fmt.Errorf("email_protection.antispam.%s must be a whole number", key)
}
}
// yaml.v3 also converts non-string scalars into strings. Reject these
// before decoding even when socket range checks are disabled.
if node, ok := as["rspamd_socket"]; ok && !yamlNodeIsNull(&node) && yamlResolvedTag(&node) != "!!str" {
return presence, fmt.Errorf("email_protection.antispam.rspamd_socket must be a string")
}
if node, ok := as["feeds"]; ok {
if err := validateAntispamFeedTypes(node); err != nil {
return presence, err
}
}
var learning map[string]yaml.Node
if node, ok := as["learning"]; ok && !yamlNodeIsNull(&node) {
var err error
if learning, err = antispamLearningFields(node); err != nil {
return presence, err
}
}
if node, ok := as["exempt"]; ok && !yamlNodeIsNull(&node) {
if err := validateStringList(node, "email_protection.antispam.exempt"); err != nil {
return presence, err
}
}
presence.antispam = antispamPresence{
dryRun: set(as, "dry_run"),
socket: set(as, "rspamd_socket"),
rejectScore: set(as, "reject_score"),
tagScore: set(as, "tag_score"),
reporter: set(learning, "reporter"),
}
return presence, nil
}
func yamlNodeIsNull(node *yaml.Node) bool {
return yamlResolvedTag(node) == "!!null"
}
// yamlResolvedTag follows aliases and returns the tag of the node they end at,
// or "" for a cycle or an unresolved alias.
func yamlResolvedTag(node *yaml.Node) string {
seen := make(map[*yaml.Node]struct{})
for node != nil && node.Kind == yaml.AliasNode {
if _, ok := seen[node]; ok {
return ""
}
seen[node] = struct{}{}
node = node.Alias
}
if node == nil {
return ""
}
return node.Tag
}
// applyForwardGuardDefaults fills unset forward-guard fields. dry_run and every
// hold signal default on; an explicit false (tracked via presence) is kept.
func applyForwardGuardDefaults(fg *ForwardGuardConfig, p forwardGuardPresence) {
if !p.dryRun {
fg.DryRun = true
}
if !p.bounceBackscatter {
fg.HoldSignals.BounceBackscatter = true
}
if !p.spamFlagged {
fg.HoldSignals.SpamFlagged = true
}
if !p.malware {
fg.HoldSignals.Malware = true
}
if !p.badSenderIP {
fg.HoldSignals.BadSenderIP = true
}
if !p.authFail {
fg.HoldSignals.AuthFail = true
}
if !p.retention {
fg.QuarantineRetentionDays = 14
}
}
var ftpServerInstalled = platform.FTPServerInstalled
// applyFirewallFieldDefaults fills absent firewall: keys from DefaultConfig.
// The whole-pointer default used to be all-or-nothing: any partial firewall
// block ("enabled: true" alone) lost every unlisted default and produced a
// DROP-policy chain with empty accept lists - an instant lockout. Presence
// tracking keeps operator-explicit values, including empty lists, zeros,
// and false. Fields whose default is the zero value need no entry here.
func applyFirewallFieldDefaults(fc *firewall.FirewallConfig, present map[string]bool) {
def := firewall.DefaultConfig()
// The default passive range opens 16k public ports for an FTP server to
// use. Without one it stays closed unless the operator sets either end.
if !present["passive_ftp_start"] && !present["passive_ftp_end"] && !ftpServerInstalled() {
def.PassiveFTPStart, def.PassiveFTPEnd = 0, 0
}
if !present["tcp_in"] {
fc.TCPIn = def.TCPIn
}
if !present["tcp_out"] {
fc.TCPOut = def.TCPOut
}
if !present["udp_in"] {
fc.UDPIn = def.UDPIn
}
if !present["udp_out"] {
fc.UDPOut = def.UDPOut
}
if !present["restricted_tcp"] {
fc.RestrictedTCP = def.RestrictedTCP
}
if !present["passive_ftp_start"] {
fc.PassiveFTPStart = def.PassiveFTPStart
}
if !present["passive_ftp_end"] {
fc.PassiveFTPEnd = def.PassiveFTPEnd
}
if !present["conn_rate_limit"] {
fc.ConnRateLimit = def.ConnRateLimit
}
if !present["syn_flood_protection"] {
fc.SYNFloodProtection = def.SYNFloodProtection
}
if !present["conn_limit"] {
fc.ConnLimit = def.ConnLimit
}
if !present["drop_ipv4_options"] {
fc.DropIPv4Options = def.DropIPv4Options
}
if !present["port_flood"] {
fc.PortFlood = def.PortFlood
}
if !present["udp_flood"] {
fc.UDPFlood = def.UDPFlood
}
if !present["udp_flood_rate"] {
fc.UDPFloodRate = def.UDPFloodRate
}
if !present["udp_flood_burst"] {
fc.UDPFloodBurst = def.UDPFloodBurst
}
if !present["drop_nolog"] {
fc.DropNoLog = def.DropNoLog
}
if !present["deny_ip_limit"] {
fc.DenyIPLimit = def.DenyIPLimit
}
if !present["deny_temp_ip_limit"] {
fc.DenyTempIPLimit = def.DenyTempIPLimit
}
// Every reader already treats a negative limit as 0, and earlier
// releases accepted one, so it loads as 0 with a warning.
if fc.DenyTempIPLimit < 0 {
fc.DenyTempIPLimit, fc.DenyTempIPLimitNegative = 0, true
}
if !present["smtp_allow_users"] {
fc.SMTPAllowUsers = def.SMTPAllowUsers
}
if !present["smtp_ports"] {
fc.SMTPPorts = def.SMTPPorts
}
if !present["log_dropped"] {
fc.LogDropped = def.LogDropped
}
if !present["log_rate"] {
fc.LogRate = def.LogRate
}
}
// applyAntispamDefaults fills omitted or null antispam fields. dry_run defaults
// on and the learning reporter to domain; an explicit false, empty socket,
// empty reporter or zero score is kept for validation. Exemptions are lowered
// to their canonical case before validation.
func applyAntispamDefaults(as *AntispamConfig, p antispamPresence) {
for i, e := range as.Exempt {
as.Exempt[i] = strings.ToLower(e)
}
if !p.dryRun {
as.DryRun = true
}
if !p.socket {
as.RspamdSocket = antispam.DefaultSocket
}
if !p.rejectScore {
as.RejectScore = antispam.DefaultRejectScore
}
if !p.tagScore {
as.TagScore = antispam.DefaultTagScore
}
if !p.reporter {
as.Learning.Reporter = antispam.ReporterDomain
}
}
// validateAntispam checks the values CSM renders into Exim and rspamd. A
// disabled block is not range-checked, so dormant values never stop an
// operator from turning filtering off; YAML type errors still fail the load.
func validateAntispam(cfg *Config) error {
as := cfg.EmailProtection.Antispam
// Checked even when antispam is off: learning has nothing to run on.
if as.Learning.Enabled && !as.Enabled {
return errors.New("email_protection.antispam.learning.enabled requires email_protection.antispam.enabled")
}
if !as.Enabled {
return nil
}
if err := antispam.ValidateSocket(as.RspamdSocket); err != nil {
return fmt.Errorf("email_protection.antispam: %w", err)
}
if err := antispam.ValidateScores(as.RejectScore, as.TagScore); err != nil {
return fmt.Errorf("email_protection.antispam: %w", err)
}
if err := antispam.ValidateFeeds(as.Feeds.Enabled, as.Feeds.SpamhausDQSKeyFile); err != nil {
return fmt.Errorf("email_protection.antispam.feeds: %w", err)
}
if err := antispam.ValidateExempt(as.Exempt); err != nil {
return fmt.Errorf("email_protection.antispam.exempt: %w", err)
}
// Like antispam itself, a disabled learning block is not range-checked.
if !as.Learning.Enabled {
return nil
}
if as.Learning.Reporter != antispam.ReporterDomain && as.Learning.Reporter != antispam.ReporterMailbox {
return errors.New("email_protection.antispam.learning.reporter must be domain or mailbox")
}
if as.Learning.RedisSocket != "" {
if err := antispam.ValidateLearningRedisSocket(as.Learning.RedisSocket); err != nil {
return fmt.Errorf("email_protection.antispam.learning: %w", err)
}
}
return nil
}
// validateForwardGuard rejects nonsensical or unsafe forward-guard configs. The
// guard only matters when enabled, so an off guard never fails validation.
func validateForwardGuard(cfg *Config) error {
fg := cfg.EmailProtection.ForwardGuard
if !fg.Enabled {
return nil
}
if fg.QuarantineRetentionDays <= 0 {
return fmt.Errorf("email_protection.forward_guard.quarantine_retention_days must be > 0 when enabled")
}
// Enforce mode requires a signal exim can actually evaluate at routing time.
// malware/auth_fail are accounted in dry-run but not yet enforceable, so
// enforcing with only those on would silently hold nothing.
if !fg.DryRun && !fg.HoldSignals.BounceBackscatter && !fg.HoldSignals.BadSenderIP && !fg.HoldSignals.SpamFlagged {
return fmt.Errorf("email_protection.forward_guard: enforce mode (dry_run:false) requires bounce_backscatter, bad_sender_ip or spam_flagged enabled; malware/auth_fail are dry-run only")
}
return nil
}
func validateReputation(cfg *Config) error {
up := cfg.Reputation.Upstream
if up.CacheTTLMin != 0 && (up.CacheTTLMin < 1 || up.CacheTTLMin > 1440) {
return fmt.Errorf("reputation.upstream.cache_ttl_min must be between 1 and 1440")
}
if up.TimeoutSec != 0 && (up.TimeoutSec < 1 || up.TimeoutSec > 60) {
return fmt.Errorf("reputation.upstream.timeout_sec must be between 1 and 60")
}
if !up.Enabled {
return nil
}
if strings.TrimSpace(up.URL) == "" {
return fmt.Errorf("reputation.upstream.enabled=true but url is empty")
}
parsed, err := url.Parse(up.URL)
if err != nil {
return fmt.Errorf("reputation.upstream.url: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return fmt.Errorf("reputation.upstream.url must use http or https")
}
if parsed.Host == "" {
return fmt.Errorf("reputation.upstream.url must include host")
}
if parsed.Scheme == "http" && !isLoopbackHost(parsed.Hostname()) {
return fmt.Errorf("reputation.upstream.url must use https for non-loopback hosts (bearer token would otherwise leak in plaintext)")
}
return nil
}
// isLoopbackHost keeps plain HTTP limited to same-host panel deployments.
func isLoopbackHost(host string) bool {
if host == "" {
return false
}
if strings.EqualFold(host, "localhost") {
return true
}
ip := net.ParseIP(host)
if ip == nil {
return false
}
return ip.IsLoopback()
}
func validateVerdictCallbackField(cfg *Config, opts LoadOptions) (string, error) {
vc := cfg.AutoResponse.VerdictCallback
if vc.TimeoutSec != 0 && (vc.TimeoutSec < 1 || vc.TimeoutSec > 30) {
return "auto_response.verdict_callback.timeout_sec", fmt.Errorf("auto_response.verdict_callback.timeout_sec must be between 1 and 30")
}
if !vc.Enabled {
return "", nil
}
rawURL := strings.TrimSpace(vc.URL)
if rawURL == "" {
return "auto_response.verdict_callback.url", fmt.Errorf("auto_response.verdict_callback.enabled=true but url is empty")
}
parsed, err := url.Parse(rawURL)
if err != nil {
return "auto_response.verdict_callback.url", fmt.Errorf("auto_response.verdict_callback.url: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return "auto_response.verdict_callback.url", fmt.Errorf("auto_response.verdict_callback.url must use http or https")
}
if parsed.Host == "" {
return "auto_response.verdict_callback.url", fmt.Errorf("auto_response.verdict_callback.url must include host")
}
if err := validateVerdictCallbackSecret(verdictCallbackForValidation{
HMACSecret: vc.HMACSecret,
HMACSecretEnv: vc.HMACSecretEnv,
AllowUnsigned: vc.AllowUnsigned,
AllowUnsetEnv: opts.AllowUnsetSecretEnv,
}); err != nil {
return verdictCallbackSecretField(vc.HMACSecretEnv), err
}
return "", nil
}
// validateVerdictCallbackSecret enforces fail-closed posture on the
// outbound HMAC: when the callback is enabled, either hmac_secret or the
// hmac_secret_env-named env var must resolve to a non-empty value, OR
// the operator must explicitly set allow_unsigned: true to acknowledge
// that requests and responses will run without integrity protection.
//
// Without this check a misconfigured deployment (env var typoed, secret
// not yet rotated in) silently emits unsigned POSTs while the daemon
// keeps reporting healthy, and any on-path actor can forge or replay
// block decisions.
func validateVerdictCallbackSecret(vc verdictCallbackForValidation) error {
if vc.AllowUnsigned {
return nil
}
if strings.TrimSpace(vc.HMACSecret) != "" {
return nil
}
if vc.HMACSecretEnv != "" {
if vc.AllowUnsetEnv || strings.TrimSpace(os.Getenv(vc.HMACSecretEnv)) != "" {
return nil
}
return fmt.Errorf("auto_response.verdict_callback.enabled=true but env var %q is empty or unset; set the secret, or opt in with allow_unsigned: true", vc.HMACSecretEnv)
}
return fmt.Errorf("auto_response.verdict_callback.enabled=true requires hmac_secret or hmac_secret_env (or allow_unsigned: true to acknowledge unsigned requests and responses)")
}
// verdictCallbackForValidation isolates the fields validateVerdictCallbackSecret
// needs without re-spelling the anonymous struct literal in config.go.
type verdictCallbackForValidation struct {
HMACSecret string
HMACSecretEnv string
AllowUnsigned bool
AllowUnsetEnv bool
}
func verdictCallbackSecretField(hmacSecretEnv string) string {
if hmacSecretEnv != "" {
return "auto_response.verdict_callback.hmac_secret_env"
}
return "auto_response.verdict_callback.hmac_secret"
}
func validateDirectSMTPEgress(cfg *Config) error {
d := cfg.Detection.DirectSMTPEgress
switch strings.ToLower(strings.TrimSpace(d.Backend)) {
case "", "auto", "bpf", "legacy", "none":
default:
return fmt.Errorf("detection.direct_smtp_egress.backend must be auto, bpf, legacy, or none")
}
for i, p := range d.Ports {
if p < 1 || p > 65535 {
return fmt.Errorf("detection.direct_smtp_egress.ports[%d] must be between 1 and 65535", i)
}
}
return nil
}
func validateBPFEnforcement(cfg *Config) error {
if !cfg.BPFEnforcement.Enabled {
return nil
}
switch strings.ToLower(strings.TrimSpace(cfg.Detection.ConnectionTrackerBackend)) {
case "", "auto", "bpf":
case "legacy", "none":
return fmt.Errorf("bpf_enforcement.enabled=true requires detection.connection_tracker_backend=auto or bpf")
default:
return fmt.Errorf("detection.connection_tracker_backend must be auto, bpf, legacy, or none")
}
gates := 0
if cfg.BPFEnforcement.DirectSMTPEgress {
if !cfg.Detection.DirectSMTPEgress.Enabled {
return fmt.Errorf("bpf_enforcement.direct_smtp_egress requires detection.direct_smtp_egress.enabled=true")
}
switch strings.ToLower(strings.TrimSpace(cfg.Detection.DirectSMTPEgress.Backend)) {
case "", "auto", "bpf":
case "legacy", "none":
return fmt.Errorf("bpf_enforcement.direct_smtp_egress requires detection.direct_smtp_egress.backend=auto or bpf")
default:
return fmt.Errorf("detection.direct_smtp_egress.backend must be auto, bpf, legacy, or none")
}
gates++
}
if gates == 0 {
return fmt.Errorf("bpf_enforcement.enabled=true requires at least one feature gate (direct_smtp_egress)")
}
return nil
}
func validateWebUITokens(cfg *Config) error {
if _, _, err := cfg.BrowserSessionDurations(); err != nil {
return err
}
seenNames := make(map[string]struct{}, len(cfg.WebUI.Tokens))
seenTokens := make(map[string]struct{}, len(cfg.WebUI.Tokens))
for i, tok := range cfg.WebUI.Tokens {
name := strings.TrimSpace(tok.Name)
if name == "" {
return fmt.Errorf("webui.tokens[%d]: empty name", i)
}
if tok.Scope != "admin" && tok.Scope != "read" && tok.Scope != "unban" {
return fmt.Errorf("webui.tokens[%d]: unknown scope %q (use admin, read or unban)", i, tok.Scope)
}
for j, src := range tok.AllowedIPs {
if _, err := parseTokenSource(src); err != nil {
return fmt.Errorf("webui.tokens[%d].allowed_ips[%d]: %q is not an IP address or CIDR", i, j, src)
}
}
if tok.Token == "" {
return fmt.Errorf("webui.tokens[%d]: empty token", i)
}
if _, ok := seenNames[name]; ok {
return fmt.Errorf("webui.tokens[%d]: duplicate name %q", i, tok.Name)
}
seenNames[name] = struct{}{}
if _, ok := seenTokens[tok.Token]; ok {
return fmt.Errorf("webui.tokens[%d]: duplicate token", i)
}
seenTokens[tok.Token] = struct{}{}
}
return nil
}
func validateMailLogsField(cfg *Config) (string, error) {
switch cfg.MailLogs.Source {
case "", "auto", "file", "journal":
default:
return "mail_logs.source", fmt.Errorf("mail_logs.source: must be auto, file, or journal (got %q)", cfg.MailLogs.Source)
}
for i, unit := range cfg.MailLogs.Units {
if strings.TrimSpace(unit) == "" {
return "mail_logs.units", fmt.Errorf("mail_logs.units[%d]: empty unit", i)
}
}
return "", nil
}
func validateStalwartField(cfg *Config) (string, error) {
for i, entry := range cfg.Stalwart.Ports {
if _, err := stalwartlog.ParsePorts([]string{entry}); err != nil {
field := fmt.Sprintf("stalwart.ports[%d]", i)
return field, fmt.Errorf("%s: %w", field, err)
}
}
if raw := cfg.Stalwart.APIURL; raw != "" {
const field = "stalwart.api_url"
if err := stalwartapi.ValidateURL(raw); err != nil {
return field, fmt.Errorf("%s: %w", field, err)
}
}
if p := cfg.Stalwart.APIKeyFile; p != "" && (!filepath.IsAbs(p) || filepath.Clean(p) != p || p == "/") {
const field = "stalwart.api_key_file"
return field, fmt.Errorf("%s: %q must be a clean absolute file path", field, p)
}
return "", nil
}
func validateMailLogs(cfg *Config) error {
_, err := validateMailLogsField(cfg)
return err
}
func validateMailBruteAccountKeyField(cfg *Config) (string, error) {
key := cfg.Thresholds.MailBruteAccountKey
switch {
case key == "", key == "builtin:dovecot-user", key == "builtin:postfix-sasl":
// ok
case strings.HasPrefix(key, "regex:"):
re, err := regexp.Compile(strings.TrimPrefix(key, "regex:"))
if err != nil {
return "thresholds.mail_brute_account_key", fmt.Errorf("mail_brute_account_key: invalid regex: %w", err)
}
if re.NumSubexp() < 1 {
return "thresholds.mail_brute_account_key", fmt.Errorf("mail_brute_account_key: regex must contain at least one capture group")
}
default:
return "thresholds.mail_brute_account_key", fmt.Errorf("mail_brute_account_key: %q must be builtin:* or regex:*", key)
}
return "", nil
}
func validateMailBruteAccountKey(cfg *Config) error {
_, err := validateMailBruteAccountKeyField(cfg)
return err
}
// validateHTTPASNCrawl checks http_asn_crawl thresholds and auto-response for
// values that would make the detector mis-behave at runtime. Called from
// LoadBytes after defaults are applied, so percent fields are already 1..100
// unless the operator explicitly set them to out-of-range values.
func validateHTTPASNCrawl(cfg *Config) error {
th := cfg.Thresholds
// Integer fields that must not be negative.
negChecks := []struct {
name string
val int
}{
{"thresholds.http_asn_crawl_window_min", th.HTTPASNCrawlWindowMin},
{"thresholds.http_asn_crawl_min_ips", th.HTTPASNCrawlMinIPs},
{"thresholds.http_asn_crawl_min_expensive", th.HTTPASNCrawlMinExpensive},
{"thresholds.http_asn_crawl_high_volume_mult", th.HTTPASNCrawlHighVolumeMult},
{"thresholds.http_asn_crawl_saturation", th.HTTPASNCrawlSaturation},
{"thresholds.http_asn_crawl_max_prefix", th.HTTPASNCrawlMaxPrefix},
{"thresholds.http_asn_crawl_max_tracked_ips", th.HTTPASNCrawlMaxTrackedIPs},
}
for _, c := range negChecks {
if c.val < 0 {
return fmt.Errorf("%s must not be negative (got %d)", c.name, c.val)
}
}
// Percent fields: after defaults are applied these are always >= 1.
// An operator-supplied value outside 1..100 is rejected.
pctChecks := []struct {
name string
val int
}{
{"thresholds.http_asn_crawl_min_share_pct", th.HTTPASNCrawlMinSharePct},
{"thresholds.http_asn_crawl_high_amp_pct", th.HTTPASNCrawlHighAmpPct},
{"thresholds.http_asn_crawl_16_pref_pct", th.HTTPASNCrawl16PrefPct},
}
for _, c := range pctChecks {
if c.val < 1 || c.val > 100 {
return fmt.Errorf("%s must be in 1..100 (got %d)", c.name, c.val)
}
}
if th.HTTPASNCrawlMinIPs > 0 && th.HTTPASNCrawlMaxTrackedIPs < th.HTTPASNCrawlMinIPs {
return fmt.Errorf("thresholds.http_asn_crawl_max_tracked_ips must be >= thresholds.http_asn_crawl_min_ips when http_asn_crawl is enabled")
}
// Every ASN in both lists must be in 1..4294967295.
asnLists := []struct {
name string
list []uint
}{
{"thresholds.http_asn_crawl_allowlist_asns", th.HTTPASNCrawlAllowlistASNs},
{"thresholds.http_asn_crawl_reverse_proxy_asns", th.HTTPASNCrawlReverseProxyASNs},
}
for _, al := range asnLists {
for _, asn := range al.list {
if asn < 1 || asn > 4294967295 {
return fmt.Errorf("%s contains invalid ASN %d (must be 1..4294967295)", al.name, asn)
}
}
}
// Tempban must be a positive duration.
if tb := cfg.AutoResponse.HTTPASNCrawlTempban; tb != "" {
d, err := time.ParseDuration(tb)
if err != nil {
return fmt.Errorf("auto_response.http_asn_crawl_tempban: unparseable duration %q: %w", tb, err)
}
if d <= 0 {
return fmt.Errorf("auto_response.http_asn_crawl_tempban: duration must be positive (got %q)", tb)
}
}
return nil
}
func Load(path string) (*Config, error) {
// #nosec G304 -- path is operator-supplied config file (CLI flag / env).
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("reading config %s: %w", path, err)
}
defer f.Close()
data, err := readConfigBytesLimited(f)
if errors.Is(err, errConfigTooLarge) {
return nil, fmt.Errorf("config %s exceeds %d byte cap", path, MaxConfigBytes)
}
if err != nil {
return nil, fmt.Errorf("reading config %s: %w", path, err)
}
cfg, err := LoadBytes(data)
if err != nil {
return nil, err
}
cfg.ConfigFile = path
return cfg, nil
}
// LoadWithDir loads the main config file and then merges every YAML fragment
// from confDir on top in lexicographic order. A missing confDir is not an
// error. Unknown fields in fragments are rejected (KnownFields=true).
func LoadWithDir(path, confDir string) (*Config, error) {
return LoadWithDirOptions(path, confDir, LoadOptions{})
}
// LoadWithDirOptions is LoadWithDir with the validation adjusted by opts.
func LoadWithDirOptions(path, confDir string, opts LoadOptions) (*Config, error) {
// #nosec G304 -- path is operator-supplied (CLI flag).
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("reading config %s: %w", path, err)
}
defer f.Close()
mainData, err := readConfigBytesLimited(f)
if errors.Is(err, errConfigTooLarge) {
return nil, fmt.Errorf("config %s exceeds %d byte cap", path, MaxConfigBytes)
}
if err != nil {
return nil, fmt.Errorf("reading config %s: %w", path, err)
}
cfg, err := loadBytesWithDir(mainData, confDir, path, opts)
if err != nil {
return nil, err
}
cfg.ConfigFile = path
cfg.ConfigDir = confDir
return cfg, nil
}
// LoadBytesWithDir decodes an in-memory main config after merging the current
// confDir fragments on top. It is used to validate an edited main file before
// that file is committed to disk. ConfigFile is left empty.
func LoadBytesWithDir(data []byte, confDir string) (*Config, error) {
mergedBytes, err := MergeBytesWithDir(data, confDir)
if err != nil {
return nil, err
}
cfg, err := LoadBytes(mergedBytes)
if err != nil {
return nil, err
}
cfg.ConfigDir = confDir
return cfg, nil
}
// MergeBytesWithDir merges an in-memory main config with the current confDir
// fragments and returns YAML that still preserves whether fields were omitted.
func MergeBytesWithDir(data []byte, confDir string) ([]byte, error) {
if len(data) > MaxConfigBytes {
return nil, fmt.Errorf("parsing config: input size %d exceeds %d byte cap", len(data), MaxConfigBytes)
}
return mergeBytesWithDir(data, confDir, "config input")
}
func loadBytesWithDir(mainData []byte, confDir, source string, opts LoadOptions) (*Config, error) {
mergedBytes, err := mergeBytesWithDir(mainData, confDir, source)
if err != nil {
return nil, err
}
return LoadBytesWithOptions(mergedBytes, opts)
}
func mergeBytesWithDir(mainData []byte, confDir, source string) ([]byte, error) {
var merged yaml.Node
if unmarshalErr := yaml.Unmarshal(mainData, &merged); unmarshalErr != nil {
return nil, fmt.Errorf("parsing %s: %w", source, unmarshalErr)
}
normalized, err := normalizeYAMLForMerge(&merged)
if err != nil {
return nil, fmt.Errorf("parsing %s: %w", source, err)
}
merged = *normalized
frags, err := loadConfDirFragments(confDir)
if err != nil {
return nil, err
}
for _, frag := range frags {
DeepMergeTracked(&merged, frag.node, func(keyPath, oldVal, newVal string) {
fmt.Fprintf(os.Stderr, "confd: %s overrides %s: %q -> %q\n",
frag.path,
keyPath,
redactConfigScalarForLog(keyPath, oldVal),
redactConfigScalarForLog(keyPath, newVal),
)
})
}
mergedBytes, err := yaml.Marshal(&merged)
if err != nil {
return nil, fmt.Errorf("marshaling merged config: %w", err)
}
return mergedBytes, nil
}
func Save(cfg *Config) error {
data, err := yaml.Marshal(cfg)
if err != nil {
return fmt.Errorf("marshaling config: %w", err)
}
path, err := saveTargetPath(cfg.ConfigFile)
if err != nil {
return err
}
// csm.yaml is the daemon's only config: a truncate-in-place write torn
// by a crash would block the next daemon start with a parse error.
return atomicio.AtomicWrite(path, 0600, data)
}
func saveTargetPath(path string) (string, error) {
info, err := os.Lstat(path)
if err != nil {
if os.IsNotExist(err) {
return path, nil
}
return "", fmt.Errorf("checking config path: %w", err)
}
if info.Mode()&os.ModeSymlink == 0 {
return path, nil
}
// FHS migration leaves the legacy config path as a symlink to the
// main config. Save must update the target instead of replacing the link.
resolved, err := filepath.EvalSymlinks(path)
if err != nil {
return "", fmt.Errorf("resolving config symlink: %w", err)
}
return resolved, nil
}
package config
import (
"fmt"
"net"
"net/netip"
"net/url"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/firewall"
)
// outboundDependency is one TCP destination the daemon dials because the
// operator enabled the feature that needs it. what reads as a clause the
// warning can quote, e.g. "alerts.webhook.url dials panel.example.com on
// TCP port 8443".
type outboundDependency struct {
what string
host string // hostname or literal IP; empty when the endpoints are vendor-fixed
port int
daemonRoot bool // the daemon's root UID can use smtp_block's per-UID accepts
}
// builtInHTTPSEndpoints names the outbound HTTPS destinations that are not
// configurable: threat feeds, AbuseIPDB, MaxMind, YARA Forge, AI-crawler
// range feeds and the release check. They share one warning because dropping
// 443 from tcp_out silences all of them at once.
const builtInHTTPSEndpoints = "built-in HTTPS endpoints (threat feeds, signature, GeoIP and bot-range updates, AbuseIPDB, release check)"
// outboundDependencies lists every TCP destination the daemon will dial with
// the features currently enabled. Loopback destinations are left out because
// the output chain accepts loopback ahead of any port rule.
func outboundDependencies(cfg *Config) []outboundDependency {
deps := []outboundDependency{{
what: builtInHTTPSEndpoints + " need TCP port 443 outbound",
port: 443,
daemonRoot: true,
}}
add := func(label, host string, port int) {
if egressLoopbackHost(host) {
return
}
deps = append(deps, outboundDependency{
what: fmt.Sprintf("%s dials %s on TCP port %d", label, host, port),
host: host,
port: port,
daemonRoot: true,
})
}
addURL := func(label, raw string) {
if host, port, ok := urlDialTarget(raw); ok {
add(label, host, port)
}
}
addHostPort := func(label, raw string) {
if host, port, ok := hostPortDialTarget(raw); ok {
add(label, host, port)
}
}
if cfg.Alerts.Email.Enabled && cfg.Alerts.Email.SMTP != "" {
addHostPort("alerts.email.smtp", cfg.Alerts.Email.SMTP)
}
if cfg.Alerts.Webhook.Enabled && cfg.Alerts.Webhook.URL != "" {
addURL("alerts.webhook.url", cfg.Alerts.Webhook.URL)
}
if cfg.Alerts.Heartbeat.Enabled && cfg.Alerts.Heartbeat.URL != "" {
addURL("alerts.heartbeat.url", cfg.Alerts.Heartbeat.URL)
}
syslog := cfg.Alerts.AuditLog.Syslog
if syslog.Enabled && (syslog.Network == "tcp" || syslog.Network == "tls") {
addHostPort("alerts.audit_log.syslog.address", syslog.Address)
}
if cfg.AutoResponse.VerdictCallback.Enabled && cfg.AutoResponse.VerdictCallback.URL != "" {
addURL("auto_response.verdict_callback.url", cfg.AutoResponse.VerdictCallback.URL)
}
if cfg.Reputation.Rspamd.Enabled && cfg.Reputation.Rspamd.URL != "" {
addURL("reputation.rspamd.url", cfg.Reputation.Rspamd.URL)
}
if cfg.Reputation.Upstream.Enabled && cfg.Reputation.Upstream.URL != "" {
addURL("reputation.upstream.url", cfg.Reputation.Upstream.URL)
}
if cfg.Reputation.Report.Enabled {
for i, target := range cfg.Reputation.Report.Targets {
addURL(fmt.Sprintf("reputation.report.targets[%d].url", i), target.URL)
}
}
if cfg.Reputation.Central.Enabled && cfg.Reputation.Central.SetURL != "" {
addURL("reputation.central.set_url", cfg.Reputation.Central.SetURL)
}
// The signature updater runs whenever an update URL is set.
if cfg.Signatures.UpdateURL != "" {
addURL("signatures.update_url", cfg.Signatures.UpdateURL)
}
if cfg.Signatures.YaraForge.Enabled && cfg.Signatures.YaraForge.DownloadURL != "" {
addURL("signatures.yara_forge.download_url", cfg.Signatures.YaraForge.DownloadURL)
}
if cfg.Sentry.Enabled && cfg.Sentry.DSN != "" {
addURL("sentry.dsn", cfg.Sentry.DSN)
}
if cfg.UpdatesCheckEnabled() && cfg.Updates.GitHubAPIURL != "" {
addURL("updates.github_api_url", cfg.Updates.GitHubAPIURL)
}
return deps
}
// urlDialTarget resolves the host and port a URL is dialed on the way the
// HTTP client does: an explicit port wins, otherwise the scheme default. A
// URL that the client cannot dial produces no egress requirement.
func urlDialTarget(raw string) (host string, port int, ok bool) {
u, err := url.Parse(raw)
if err != nil || u.Host == "" {
return "", 0, false
}
host = u.Hostname()
if host == "" {
return "", 0, false
}
scheme := strings.ToLower(u.Scheme)
if scheme != "http" && scheme != "https" {
return "", 0, false
}
if p := u.Port(); p != "" {
port, err = strconv.Atoi(p)
if err != nil || port < 1 || port > 65535 {
return "", 0, false
}
return host, port, true
}
switch scheme {
case "https":
return host, 443, true
case "http":
return host, 80, true
}
return "", 0, false
}
// hostPortDialTarget splits a host:port dial address. Addresses without a
// port cannot be dialed and produce no egress requirement.
func hostPortDialTarget(raw string) (host string, port int, ok bool) {
if raw != strings.TrimSpace(raw) {
return "", 0, false
}
host, portStr, err := net.SplitHostPort(raw)
if err != nil || host == "" {
return "", 0, false
}
// net.Dial accepts both numeric ports and service names. Resolve through
// the same service database so a valid address such as host:smtp cannot
// evade the egress-policy warning.
port, err = net.LookupPort("tcp", portStr)
if err != nil || port < 1 || port > 65535 {
return "", 0, false
}
return host, port, true
}
// firewallEgressResults warns when the enabled firewall's outbound policy
// would refuse a connection the daemon itself needs. The output chain is
// default-drop and ends in a TCP reset, so the failure reads as the far end
// being down while the host looks healthy locally. Like the inbound checks
// these are warnings, never errors: an operator may route through a proxy
// that validation cannot see.
//
// Coverage is limited to what the daemon dials from its own config plus the
// ports declared under firewall.required_tcp_out. A third-party agent's
// egress is only checked when it declares its ports there.
func firewallEgressResults(cfg *Config) []ValidationResult {
fw := cfg.Firewall
if fw == nil || !fw.Enabled {
return nil
}
// Mirror the engine: the output chain is accept-all unless some outbound
// list is set; IPv4 is accepted wholesale when only IPv6 lists are set;
// IPv6 is accepted wholesale unless it is managed.
tcp6Out := fw.TCP6Out
if len(tcp6Out) == 0 {
tcp6Out = fw.TCPOut
}
udp6Out := fw.UDP6Out
if len(udp6Out) == 0 {
udp6Out = fw.UDPOut
}
ipv4Filtered := len(fw.TCPOut) > 0 || len(fw.UDPOut) > 0
ipv6Filtered := fw.IPv6 && (ipv4Filtered || len(tcp6Out) > 0 || len(udp6Out) > 0)
if !ipv4Filtered && !ipv6Filtered {
return nil
}
// smtp_block installs per-UID accepts and then a drop for each mail port
// ahead of both the IPv4 bypass and the configured port rules. The daemon
// runs as root, which is always accepted; required_tcp_out describes a
// separate service whose UID this validator cannot prove is allowed.
smtpRestricted := func(port int) bool {
return fw.SMTPBlock && containsPort(fw.SMTPPorts, port)
}
blocked4 := func(dep outboundDependency) bool {
if smtpRestricted(dep.port) {
return !dep.daemonRoot
}
if outAllowReachesAnyDst(fw, dep.port, false) {
return false
}
return ipv4Filtered && !containsPort(fw.TCPOut, dep.port)
}
blocked6 := func(dep outboundDependency) bool {
if smtpRestricted(dep.port) {
return !dep.daemonRoot
}
if outAllowReachesAnyDst(fw, dep.port, true) {
return false
}
return !containsPort(tcp6Out, dep.port)
}
deps := outboundDependencies(cfg)
for _, port := range fw.RequiredTCPOut {
if port < 1 || port > 65535 {
continue // reported as an error by firewallValueResults
}
deps = append(deps, outboundDependency{
what: fmt.Sprintf("firewall.required_tcp_out declares TCP port %d", port),
port: port,
})
}
var results []ValidationResult
for _, dep := range deps {
wantV4, wantV6 := dialFamilies(dep.host)
isBlocked4 := wantV4 && blocked4(dep)
if isBlocked4 {
results = append(results, ValidationResult{"warn", "firewall.tcp_out",
egressBlockedMessage(dep, "tcp_out", smtpRestricted(dep.port)) +
outAllowScopedNote(fw, dep.port, false)})
}
// An inherited tcp6_out is the same list as tcp_out, so the IPv4
// warning above covers both families only when IPv4 is filtered too.
// With only udp6_out configured, the engine bypasses IPv4 wholesale
// but still drops IPv6 TCP, so that shape needs a tcp6_out warning.
isBlocked6 := wantV6 && ipv6Filtered && blocked6(dep)
if isBlocked6 && (len(fw.TCP6Out) > 0 || !isBlocked4) {
results = append(results, ValidationResult{"warn", "firewall.tcp6_out",
egressBlockedMessage(dep, "tcp6_out", smtpRestricted(dep.port)) +
outAllowScopedNote(fw, dep.port, true)})
}
}
return results
}
// outAllowReachesAnyDst reports whether a tcp_out_allow rule opens the port to
// every destination in the requested family. Only then can it prove the endpoint is
// reachable, because it cannot resolve a hostname to test a scoped prefix.
func outAllowReachesAnyDst(fw *firewall.FirewallConfig, port int, ipv6 bool) bool {
for _, r := range fw.TCPOutAllow {
network := outAllowNetworkForPort(fw, r, port, ipv6)
if network == nil {
continue
}
if ones, _ := network.Mask.Size(); ones == 0 {
return true
}
}
return false
}
// outAllowNetworkForPort mirrors the engine's emission checks and SMTP order
// so a skipped rule cannot hide a lockout or claim to cover a blocked port.
func outAllowNetworkForPort(fw *firewall.FirewallConfig, r firewall.OutAllowRule, port int, ipv6 bool) *net.IPNet {
if !validPort(r.PortStart) || !validPort(r.PortEnd) || port < r.PortStart || port > r.PortEnd {
return nil
}
if ipv6 && !fw.IPv6 || fw.SMTPBlock && containsPort(fw.SMTPPorts, port) {
return nil
}
network, err := firewall.ParseOutAllowDst(r.Dst)
if err != nil || (network.IP.To4() == nil) != ipv6 {
return nil
}
return network
}
// outAllowScopedNote annotates a lockout warning when a scoped tcp_out_allow
// rule covers the port. The warning stands -- silence here would be the silent
// lockout these checks exist to catch -- but the operator is told where to look.
func outAllowScopedNote(fw *firewall.FirewallConfig, port int, ipv6 bool) string {
var dsts []string
for _, r := range fw.TCPOutAllow {
network := outAllowNetworkForPort(fw, r, port, ipv6)
if network == nil {
continue
}
if ones, _ := network.Mask.Size(); ones == 0 {
continue
}
dsts = append(dsts, r.Dst)
}
if len(dsts) == 0 {
return ""
}
return fmt.Sprintf("; tcp_out_allow covers the port for %s only, so confirm the endpoint resolves into that range",
strings.Join(dsts, ", "))
}
func egressBlockedMessage(dep outboundDependency, policy string, smtpRestricted bool) string {
if smtpRestricted {
return fmt.Sprintf("%s but smtp_block restricts TCP port %d to allowed UIDs; required_tcp_out does not identify an allowed user", dep.what, dep.port)
}
if policy == "tcp6_out" {
return fmt.Sprintf("IPv6 is managed and tcp6_out does not allow it: %s", dep.what)
}
return fmt.Sprintf("%s but tcp_out does not allow it; once the firewall applies, connections to that port are refused", dep.what)
}
// dialFamilies reports which IP families a destination can be dialed over. A
// literal address pins one family; a hostname (or the vendor endpoints, which
// have no single host) may resolve to either.
func dialFamilies(host string) (ipv4, ipv6 bool) {
ip, err := netip.ParseAddr(host)
if err != nil {
return true, true
}
if ip.Unmap().Is4() {
return true, false
}
return false, true
}
// egressLoopbackHost recognizes scoped IPv6 literals as well as the common
// forms handled by isLoopbackHost. URL.Hostname and net.SplitHostPort retain
// a zone such as "%lo", which net.ParseIP cannot parse even though the
// kernel still routes ::1 over the output chain's accepted loopback device.
func egressLoopbackHost(host string) bool {
if isLoopbackHost(host) {
return true
}
if strings.EqualFold(strings.TrimSuffix(host, "."), "localhost") {
return true
}
ip, err := netip.ParseAddr(host)
return err == nil && ip.Unmap().IsLoopback()
}
package config
import "time"
// EmailAVConfig holds email antivirus scanning settings.
type EmailAVConfig struct {
Enabled bool `yaml:"enabled"`
ClamdSocket string `yaml:"clamd_socket"`
ScanTimeout string `yaml:"scan_timeout"`
MaxAttachmentSize int64 `yaml:"max_attachment_size"`
MaxArchiveDepth int `yaml:"max_archive_depth"`
MaxArchiveFiles int `yaml:"max_archive_files"`
MaxExtractionSize int64 `yaml:"max_extraction_size"`
QuarantineInfected bool `yaml:"quarantine_infected"`
ScanConcurrency int `yaml:"scan_concurrency"`
// FailMode controls behavior when scanning cannot complete.
// "open" (default): deliver mail when engines are down, scans time out, or quarantine fails.
// "tempfail": defer delivery (Exim retries later) when scanning cannot complete or infected mail cannot be quarantined.
// A message that makes its attachment parse worker fail is deferred in either mode, except in observe mode.
FailMode string `yaml:"fail_mode"`
// KeepCrashSamples keeps copies of messages and tenant zips that crash
// a parse worker, for diagnosis. They hold customer data.
KeepCrashSamples bool `yaml:"keep_crash_samples"`
// WorkerMemoryLimit caps the memory and swap of each attachment parse
// worker, in bytes.
WorkerMemoryLimit int64 `yaml:"worker_memory_limit"`
}
// Attachment parse worker memory cap: the default, and the floor below which
// ordinary messages would fail.
const (
DefaultWorkerMemoryLimit = 512 << 20
MinWorkerMemoryLimit = 128 << 20
)
// ScanTimeoutDuration parses the ScanTimeout string as a time.Duration.
func (c *EmailAVConfig) ScanTimeoutDuration() time.Duration {
if c.ScanTimeout == "" {
return 30 * time.Second
}
d, err := time.ParseDuration(c.ScanTimeout)
if err != nil {
return 30 * time.Second
}
return d
}
// DefaultClamdSocket is the socket assumed when the operator sets none. It is
// RHEL's clamd-scan path; other packagings put it elsewhere, which is what
// ResolveClamdSocket exists to cope with.
const DefaultClamdSocket = "/var/run/clamd.scan/clamd.sock"
// EmailAVDefaults applies default values to an EmailAVConfig.
func EmailAVDefaults(c *EmailAVConfig) {
if c.ClamdSocket == "" {
c.ClamdSocket = DefaultClamdSocket
}
if c.ScanTimeout == "" {
c.ScanTimeout = "30s"
}
if c.MaxAttachmentSize == 0 {
c.MaxAttachmentSize = 25 * 1024 * 1024 // 25 MB
}
if c.MaxArchiveDepth == 0 {
c.MaxArchiveDepth = 1
}
if c.MaxArchiveFiles == 0 {
c.MaxArchiveFiles = 50
}
if c.MaxExtractionSize == 0 {
c.MaxExtractionSize = 100 * 1024 * 1024 // 100 MB
}
if c.ScanConcurrency == 0 {
c.ScanConcurrency = 4
}
if c.WorkerMemoryLimit == 0 {
c.WorkerMemoryLimit = DefaultWorkerMemoryLimit
}
}
package config
import (
"net"
"strings"
"github.com/pidginhost/csm/internal/firewall"
)
// EffectiveFirewallConfig returns the copied configuration handed to the
// firewall engine. Keeping these derived values in one place prevents status
// surfaces from describing the persisted sections instead of the live policy.
func EffectiveFirewallConfig(cfg *Config) *firewall.FirewallConfig {
if cfg == nil {
return nil
}
effective := firewall.FirewallConfig{}
if cfg.Firewall != nil {
effective = *cfg.Firewall
}
effective.InfraIPs = firewall.MergeInfraIPs(cfg.InfraIPs, effective.InfraIPs)
_, socket := ChallengeSocketPath(cfg.Challenge.ListenAddr)
if !cfg.Challenge.Enabled || !cfg.Challenge.PortGate.Enabled || socket ||
cfg.Challenge.ListenPort <= 0 || challengeListenAddrIsLoopback(cfg.Challenge.ListenAddr) {
return &effective
}
effective.TCPIn = appendUniquePort(effective.TCPIn, cfg.Challenge.ListenPort)
effective.RestrictedTCP = removePort(effective.RestrictedTCP, cfg.Challenge.ListenPort)
return &effective
}
func challengeListenAddrIsLoopback(addr string) bool {
addr = strings.TrimSpace(addr)
if addr == "" {
return true
}
host := addr
if h, _, err := net.SplitHostPort(addr); err == nil {
host = h
}
host = strings.Trim(host, "[]")
if host == "" {
return false
}
if strings.EqualFold(host, "localhost") {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
func appendUniquePort(ports []int, port int) []int {
for _, existing := range ports {
if existing == port {
return ports
}
}
out := append([]int(nil), ports...)
return append(out, port)
}
func removePort(ports []int, port int) []int {
out := make([]int, 0, len(ports))
for _, existing := range ports {
if existing != port {
out = append(out, existing)
}
}
return out
}
package config
import (
"reflect"
"sync"
"sync/atomic"
)
// Hot-reload policy (ROADMAP item 7).
//
// Each top-level field of Config carries an optional `hotreload`
// struct tag:
//
// - "safe": a SIGHUP reload swaps the field in place; readers
// on the next tick see the new value.
// - "restart": the field cannot be applied without a full daemon
// restart (fanotify watched roots, bbolt path, web UI
// listener). A SIGHUP that touches a restart field
// logs a warning and leaves the prior config live.
// - (none): treated as restart-required. Tagging every safe
// field explicitly is the strict default; this closes
// the door on accidental hot-swaps of a fresh field
// someone adds without considering the safety of live
// mutation.
const (
TagSafe = "safe"
TagRestart = "restart"
)
// active holds the current live config. Readers on hot paths (check
// tick handlers, alert dispatchers, metrics-auth) call Active() to
// pick up the latest snapshot after a SIGHUP. Writers (daemon
// startup, SIGHUP reload) call SetActive.
var active atomic.Pointer[publication]
// publication is one live config with the response epoch it was
// published under.
type publication struct {
cfg *Config
epoch uint64
// generation grows with every publication.
generation uint64
}
// publishMu orders publications, so each epoch is assigned against the
// config it follows.
var publishMu sync.Mutex
// Active returns the current live Config pointer. Returns nil if
// SetActive has not been called; callers on hot paths are expected
// to nil-check once per call. Daemon startup calls SetActive before
// any tick runs, so a nil return in production is a bug.
func Active() *Config {
if p := active.Load(); p != nil {
return p.cfg
}
return nil
}
// ActiveEpoch returns the live config with its response epoch, both from
// one publication. The epoch grows with every publication that changes
// auto_response or the mode, the settings automatic execution is bound to,
// so a quick change and change back is still seen as two changes. It is
// zero before the first publication.
func ActiveEpoch() (*Config, uint64) {
if p := active.Load(); p != nil {
return p.cfg, p.epoch
}
return nil, 0
}
// ActiveGeneration changes with every publication, so a cache of verdicts
// reached under one config can tell when any setting may have moved.
func ActiveGeneration() uint64 {
if p := active.Load(); p != nil {
return p.generation
}
return 0
}
// SetActive installs cfg as the current live config.
func SetActive(cfg *Config) {
publishMu.Lock()
defer publishMu.Unlock()
next := &publication{cfg: cfg, epoch: 1, generation: 1}
if prev := active.Load(); prev != nil {
next.epoch = prev.epoch
next.generation = prev.generation + 1
if responseChanged(prev.cfg, cfg) {
next.epoch++
}
}
active.Store(next)
}
func responseChanged(a, b *Config) bool {
if a == nil || b == nil {
return a != b
}
return a.ObserveMode() != b.ObserveMode() || !reflect.DeepEqual(a.AutoResponse, b.AutoResponse)
}
// Change describes a single top-level field that differs between an
// old and a new Config.
type Change struct {
// Field is the YAML name (from the `yaml:"..."` struct tag), or
// the Go field name if no yaml tag is set.
Field string
// Tag is the hotreload classification: TagSafe, TagRestart, or
// "" for fields with no explicit tag (treated as TagRestart).
Tag string
}
// ReloadPolicy is the top-level hot-reload manifest exposed to tests and
// operator surfaces that need to explain whether a Settings section can be
// applied live or waits for a restart.
type ReloadPolicy struct {
Field string
Tag string
RestartRequired bool
}
// HotReloadManifest returns every operator-owned top-level Config field with
// its effective reload policy. Fields without an explicit supported tag are
// classified restart-required, matching Diff and RestartRequired.
func HotReloadManifest() []ReloadPolicy {
cfgType := reflect.TypeOf(Config{})
policies := make([]ReloadPolicy, 0, cfgType.NumField())
for i := 0; i < cfgType.NumField(); i++ {
field := cfgType.Field(i)
if !field.IsExported() || isReloadManifestIgnoredRoot(field) {
continue
}
name := yamlFieldName(field)
if name == "" || name == "-" {
continue
}
tag := field.Tag.Get("hotreload")
if tag != TagSafe && tag != TagRestart {
tag = TagRestart
}
policies = append(policies, ReloadPolicy{
Field: name,
Tag: tag,
RestartRequired: tag != TagSafe,
})
}
return policies
}
func isReloadManifestIgnoredRoot(field reflect.StructField) bool {
return field.Name == "ConfigFile" || field.Name == "ConfigDir" || field.Name == "Integrity" || field.Name == "RemovedKeys"
}
// Diff reports which Config fields differ between old and new,
// classified by hotreload tag.
//
// The walk is recursive: if a top-level field is tagged, its tag
// applies to any change inside. If a nested field has its own tag,
// that tag wins over the parent (field-level overrides let a single
// safe field sit inside an otherwise restart-required parent, which
// is how webui.metrics_token can hot-reload even though the rest of
// WebUI needs a restart).
//
// Each Change carries the YAML path from root (e.g. "thresholds" for
// the top-level struct, "webui.metrics_token" for a nested leaf).
// The tag is the nearest tagged ancestor on that path; if nothing on
// the path is tagged, the Change's Tag is "" and the caller should
// treat that as TagRestart.
//
// Granularity rule: if a tagged ancestor classifies the whole
// subtree uniformly (parent tag applies, no nested overrides on
// changed leaves), the Change is reported at the parent level. That
// keeps the common case ("I changed three thresholds") as one
// "thresholds" Change. When a subtree contains a differently-tagged
// leaf, that leaf is reported separately with its own tag, and the
// parent (minus that leaf) is reported with the inherited tag.
func Diff(oldCfg, newCfg *Config) []Change {
if oldCfg == nil || newCfg == nil {
return nil
}
oldV := reflect.ValueOf(*oldCfg)
newV := reflect.ValueOf(*newCfg)
return diffStruct(oldV, newV, "", "")
}
// diffStruct walks two reflect.Values of the same struct type and
// returns Changes for every differing field. parentPath is the
// already-composed YAML dotted path down to this struct (empty at
// the root). parentTag is the effective hotreload tag inherited
// from the nearest tagged ancestor.
func diffStruct(oldV, newV reflect.Value, parentPath, parentTag string) []Change {
var changes []Change
t := oldV.Type()
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
if !field.IsExported() {
continue
}
// ConfigFile / ConfigDir / Integrity are daemon-managed process
// metadata, not operator policy fields.
if parentPath == "" && isReloadManifestIgnoredRoot(field) {
continue
}
oldField := oldV.Field(i).Interface()
newField := newV.Field(i).Interface()
if reflect.DeepEqual(oldField, newField) {
continue
}
// Effective tag for this field: its own explicit tag wins;
// otherwise inherit from the parent path.
tag := field.Tag.Get("hotreload")
if tag != TagSafe && tag != TagRestart {
tag = parentTag
}
name := yamlFieldName(field)
path := name
if parentPath != "" {
path = parentPath + "." + name
}
// If the field is itself a struct (not a pointer, slice,
// map), recurse so nested overrides can surface separately.
// Pointer-to-struct is treated as a leaf because the
// reflect.DeepEqual already told us the pointer target
// changed; re-walking it would produce duplicate noise.
if field.Type.Kind() == reflect.Struct {
nested := diffStruct(oldV.Field(i), newV.Field(i), path, tag)
// If every nested Change carries the same tag and there
// is no mixed classification, collapse to a single
// Change at this level. Operators rarely need the
// granularity "I changed thresholds.mail_queue_warn";
// the collapse keeps the common case clean.
if collapsed, ok := collapseIfUniform(nested, path, tag); ok {
changes = append(changes, collapsed)
} else {
changes = append(changes, nested...)
}
continue
}
changes = append(changes, Change{Field: path, Tag: tag})
}
return changes
}
// collapseIfUniform returns (Change{Field:path, Tag:parentTag}, true)
// when every nested change inherits parentTag (i.e. nothing nested
// overrode it). Returns (_, false) when the subtree contains a
// differently-tagged leaf, which means the caller must keep the
// granular changes.
func collapseIfUniform(nested []Change, path, parentTag string) (Change, bool) {
if len(nested) == 0 {
return Change{}, false
}
for _, c := range nested {
if c.Tag != parentTag {
return Change{}, false
}
}
return Change{Field: path, Tag: parentTag}, true
}
// RestartRequired returns true if any change in the diff carries a
// TagRestart classification (or no tag, which collapses to restart).
func RestartRequired(changes []Change) bool {
for _, c := range changes {
if c.Tag != TagSafe {
return true
}
}
return false
}
// yamlFieldName returns the yaml tag's primary name if set, else the
// Go field name. Strips any `,omitempty` / `,inline` suffix.
func yamlFieldName(f reflect.StructField) string {
tag := f.Tag.Get("yaml")
if tag == "" || tag == "-" {
return f.Name
}
for i := 0; i < len(tag); i++ {
if tag[i] == ',' {
return tag[:i]
}
}
return tag
}
package config
import (
"fmt"
"net/netip"
"strings"
"time"
)
// MCPConfig is the mcp: block. The endpoint answers at /mcp on the Web UI
// listener. Every credential in Tokens is read-only; there is no scope
// field because the endpoint has no other scope.
type MCPConfig struct {
Enabled bool `yaml:"enabled"`
Tokens []MCPToken `yaml:"tokens"`
}
// MCPToken is one entry in MCP.Tokens. It carries the same dates and
// source restriction as WebUIToken but no scope.
type MCPToken struct {
Name string `yaml:"name"`
Token string `yaml:"token"`
IssuedAt string `yaml:"issued_at"`
ExpiresAt string `yaml:"expires_at,omitempty"`
AllowedIPs []string `yaml:"allowed_ips,omitempty"`
}
// AcceptsFrom reports whether the token may be used from addr, as
// WebUIToken.AcceptsFrom does.
func (t MCPToken) AcceptsFrom(addr netip.Addr) bool {
return WebUIToken{AllowedIPs: t.AllowedIPs}.AcceptsFrom(addr)
}
// Current reports whether the credential is usable at now.
func (t MCPToken) Current(now time.Time) bool {
return WebUITokenCurrent(t.IssuedAt, t.ExpiresAt, now)
}
// validateMCPTokens is the hard validation that aborts a config load:
// malformed entries are refused whether or not the endpoint is enabled, so
// a later `enabled: true` cannot start with credentials nobody checked.
func validateMCPTokens(cfg *Config) error {
seenNames := make(map[string]struct{}, len(cfg.MCP.Tokens))
seenTokens := make(map[string]struct{}, len(cfg.MCP.Tokens))
for i, tok := range cfg.MCP.Tokens {
name := strings.TrimSpace(tok.Name)
if name == "" {
return fmt.Errorf("mcp.tokens[%d]: empty name", i)
}
for j, src := range tok.AllowedIPs {
if _, err := parseTokenSource(src); err != nil {
return fmt.Errorf("mcp.tokens[%d].allowed_ips[%d]: %q is not an IP address or CIDR", i, j, src)
}
}
if tok.Token == "" {
return fmt.Errorf("mcp.tokens[%d]: empty token", i)
}
if _, ok := seenNames[name]; ok {
return fmt.Errorf("mcp.tokens[%d]: duplicate name %q", i, tok.Name)
}
seenNames[name] = struct{}{}
if _, ok := seenTokens[tok.Token]; ok {
return fmt.Errorf("mcp.tokens[%d]: duplicate token", i)
}
seenTokens[tok.Token] = struct{}{}
for _, web := range cfg.WebUI.Tokens {
if web.Token != "" && web.Token == tok.Token {
return fmt.Errorf("mcp.tokens[%d]: token is also a webui token; the MCP endpoint needs its own credential", i)
}
}
if cfg.WebUI.MetricsToken != "" && cfg.WebUI.MetricsToken == tok.Token {
return fmt.Errorf("mcp.tokens[%d]: token is also webui.metrics_token; the MCP endpoint needs its own credential", i)
}
}
return nil
}
// mcpResults is the soft validation `csm validate` prints for the block.
func mcpResults(cfg *Config, now time.Time) []ValidationResult {
var results []ValidationResult
for i, tok := range cfg.MCP.Tokens {
if tok.Token != "" && len(tok.Token) < minWebUITokenLength {
results = append(results, ValidationResult{"warn", "mcp.tokens", fmt.Sprintf("token %q is shorter than %d characters and guessable; replace it with a random value", tok.Name, minWebUITokenLength)})
}
if field, problem := webUITokenProblem(tok.IssuedAt, tok.ExpiresAt, now); field != "" {
results = append(results, ValidationResult{"warn", fmt.Sprintf("mcp.tokens[%d].%s", i, field),
"credential disabled: " + problem + "; correct the date or replace the token"})
}
}
if !cfg.MCP.Enabled {
return results
}
if !cfg.WebUI.Enabled {
results = append(results, ValidationResult{"error", "mcp.enabled", "the MCP endpoint reads through the Web UI server; set webui.enabled: true or disable mcp"})
}
usable := 0
for _, tok := range cfg.MCP.Tokens {
if tok.Token != "" && tok.Current(now) {
usable++
}
}
if usable == 0 {
results = append(results, ValidationResult{"error", "mcp.tokens", "mcp enabled but no usable token configured"})
}
results = append(results, ValidationResult{"ok", "mcp", "served at https://" + cfg.WebUI.Listen + "/mcp"})
return results
}
package config
import (
"fmt"
"gopkg.in/yaml.v3"
)
// CollisionFn is invoked when DeepMergeTracked detects a scalar in the
// overlay overwriting a different scalar in the base. Identical-value
// rewrites are not reported because the operator cannot act on them.
// keyPath uses dotted YAML notation rooted at the document
// ("mail_logs.source"); top-level keys have no parent.
type CollisionFn func(keyPath, oldVal, newVal string)
// DeepMerge merges overlay into base in place and returns base.
// Both inputs must be DocumentNodes. Rules:
// - mapping ∩ mapping → key-by-key recurse
// - sequence ∩ sequence → append (base then overlay), with duplicate
// scalar entries removed from all-scalar lists
// - any other combination → overlay replaces base
//
// AliasNodes are treated as opaque scalars: an overlay alias replaces the
// base node; an alias inside base/overlay is not resolved before merging.
func DeepMerge(base, overlay *yaml.Node) *yaml.Node {
return DeepMergeTracked(base, overlay, nil)
}
// DeepMergeTracked is DeepMerge with an optional collision callback. The
// callback fires once per scalar-vs-scalar override across the document
// tree. Callers can pass nil for the previous DeepMerge behaviour.
func DeepMergeTracked(base, overlay *yaml.Node, onCollision CollisionFn) *yaml.Node {
if base == nil || overlay == nil {
return base
}
// An empty yaml.Unmarshal result has Kind==0; treat it as an empty document.
if base.Kind == 0 {
base.Kind = yaml.DocumentNode
}
if overlay.Kind == 0 {
overlay.Kind = yaml.DocumentNode
}
if base.Kind != yaml.DocumentNode || overlay.Kind != yaml.DocumentNode {
return base
}
if len(overlay.Content) == 0 {
return base
}
if len(base.Content) == 0 {
base.Content = overlay.Content
return base
}
mergeNodesAt(base.Content[0], overlay.Content[0], "", onCollision)
return base
}
const maxYAMLMergeExpansionNodes = 100_000
// normalizeYAMLForMerge resolves aliases and YAML merge keys before custom
// main+conf.d merging. Scalar nodes are preserved verbatim: decoding through
// interface{} would coerce date-like strings and other tagged values before
// Config gets its typed decode.
func normalizeYAMLForMerge(root *yaml.Node) (*yaml.Node, error) {
if root == nil || !containsAliasOrMerge(root) {
return root, nil
}
n := yamlMergeNormalizer{active: make(map[*yaml.Node]bool)}
// The initial scan already proved the tree needs normalization. Clone the
// whole tree in one pass so a deeply nested alias cannot make each ancestor
// rescan the same descendants.
return n.normalize(root)
}
func containsAliasOrMerge(node *yaml.Node) bool {
if node == nil {
return false
}
if node.Kind == yaml.AliasNode {
return true
}
if node.Kind == yaml.MappingNode {
for i := 0; i+1 < len(node.Content); i += 2 {
if isYAMLMergeKey(node.Content[i]) {
return true
}
}
}
for _, child := range node.Content {
if containsAliasOrMerge(child) {
return true
}
}
return false
}
type yamlMergeNormalizer struct {
active map[*yaml.Node]bool
created int
}
func (n *yamlMergeNormalizer) normalize(node *yaml.Node) (*yaml.Node, error) {
if node == nil {
return nil, nil
}
if node.Kind == yaml.AliasNode {
if node.Alias == nil {
return nil, fmt.Errorf("YAML alias %q has no anchor", node.Value)
}
if n.active[node.Alias] {
return nil, fmt.Errorf("YAML anchor %q contains itself", node.Value)
}
return n.normalize(node.Alias)
}
if n.active[node] {
return nil, fmt.Errorf("YAML anchor %q contains itself", node.Anchor)
}
if n.created >= maxYAMLMergeExpansionNodes {
return nil, fmt.Errorf("YAML alias expansion exceeds %d nodes", maxYAMLMergeExpansionNodes)
}
n.created++
n.active[node] = true
defer delete(n.active, node)
if node.Kind == yaml.MappingNode {
return n.normalizeMapping(node)
}
clone := *node
clone.Anchor = ""
clone.Alias = nil
clone.Content = make([]*yaml.Node, 0, len(node.Content))
for _, child := range node.Content {
normalized, err := n.normalize(child)
if err != nil {
return nil, err
}
clone.Content = append(clone.Content, normalized)
}
return &clone, nil
}
func (n *yamlMergeNormalizer) normalizeMapping(node *yaml.Node) (*yaml.Node, error) {
clone := *node
clone.Anchor = ""
clone.Alias = nil
clone.Content = make([]*yaml.Node, 0, len(node.Content))
explicit := make(map[string]bool, len(node.Content)/2)
var mergeValue *yaml.Node
for i := 0; i+1 < len(node.Content); i += 2 {
key := node.Content[i]
if isYAMLMergeKey(key) {
if mergeValue != nil {
return nil, fmt.Errorf("YAML mapping contains multiple merge keys")
}
mergeValue = node.Content[i+1]
continue
}
normalizedKey, err := n.normalize(key)
if err != nil {
return nil, err
}
normalizedValue, err := n.normalize(node.Content[i+1])
if err != nil {
return nil, err
}
clone.Content = append(clone.Content, normalizedKey, normalizedValue)
if id, ok := yamlScalarKeyID(normalizedKey); ok {
explicit[id] = true
}
}
if mergeValue == nil {
return &clone, nil
}
mergeMappings, err := n.normalizeMergeValue(mergeValue)
if err != nil {
return nil, err
}
for _, mapping := range mergeMappings {
if err := validateMergeMappingKeys(mapping); err != nil {
return nil, err
}
for i := 0; i+1 < len(mapping.Content); i += 2 {
id, _ := yamlScalarKeyID(mapping.Content[i])
if explicit[id] {
continue
}
explicit[id] = true
clone.Content = append(clone.Content, mapping.Content[i], mapping.Content[i+1])
}
}
return &clone, nil
}
func (n *yamlMergeNormalizer) normalizeMergeValue(node *yaml.Node) ([]*yaml.Node, error) {
normalized, err := n.normalize(node)
if err != nil {
return nil, err
}
switch normalized.Kind {
case yaml.MappingNode:
return []*yaml.Node{normalized}, nil
case yaml.SequenceNode:
mappings := make([]*yaml.Node, 0, len(normalized.Content))
for _, child := range normalized.Content {
if child.Kind != yaml.MappingNode {
return nil, fmt.Errorf("YAML map merge requires a map or sequence of maps")
}
mappings = append(mappings, child)
}
return mappings, nil
default:
return nil, fmt.Errorf("YAML map merge requires a map or sequence of maps")
}
}
func validateMergeMappingKeys(mapping *yaml.Node) error {
seen := make(map[string]bool, len(mapping.Content)/2)
for i := 0; i+1 < len(mapping.Content); i += 2 {
id, ok := yamlScalarKeyID(mapping.Content[i])
if !ok {
return fmt.Errorf("YAML map merge contains a non-scalar key")
}
if seen[id] {
return fmt.Errorf("YAML map merge contains duplicate key %q", mapping.Content[i].Value)
}
seen[id] = true
}
return nil
}
func yamlScalarKeyID(key *yaml.Node) (string, bool) {
if key == nil || key.Kind != yaml.ScalarNode {
return "", false
}
return key.ShortTag() + "\x00" + key.Value, true
}
func isYAMLMergeKey(key *yaml.Node) bool {
return key != nil && key.Kind == yaml.ScalarNode && key.Value == "<<" &&
(key.Tag == "" || key.Tag == "!" || key.ShortTag() == "!!merge")
}
func mergeNodesAt(b, o *yaml.Node, path string, onCollision CollisionFn) {
switch {
case b.Kind == yaml.MappingNode && o.Kind == yaml.MappingNode:
mergeMapAt(b, o, path, onCollision)
case b.Kind == yaml.SequenceNode && o.Kind == yaml.SequenceNode:
b.Content = dedupScalarSequence(append(b.Content, o.Content...))
default:
if onCollision != nil && b.Kind == yaml.ScalarNode && o.Kind == yaml.ScalarNode && b.Value != o.Value {
onCollision(path, b.Value, o.Value)
}
*b = *o
}
}
// dedupScalarSequence removes duplicate scalar entries (by value+tag),
// keeping the first occurrence and preserving order. It only acts when every
// element is a scalar: lists of maps (e.g. webui.tokens) keep every entry,
// where position and identity matter. Idempotent-by-content security lists
// (infra_ips, c2_blocklist, trusted_countries, disabled_checks) merged from a
// fragment that repeats a main-config entry would otherwise carry duplicates
// into validation and enforcement on every load.
func dedupScalarSequence(content []*yaml.Node) []*yaml.Node {
for _, n := range content {
if n.Kind != yaml.ScalarNode {
return content
}
}
seen := make(map[string]struct{}, len(content))
out := content[:0]
for _, n := range content {
key := n.Tag + "\x00" + n.Value
if _, dup := seen[key]; dup {
continue
}
seen[key] = struct{}{}
out = append(out, n)
}
return out
}
func mergeMapAt(b, o *yaml.Node, parent string, onCollision CollisionFn) {
for i := 0; i+1 < len(o.Content); i += 2 {
key := o.Content[i].Value
val := o.Content[i+1]
childPath := key
if parent != "" {
childPath = parent + "." + key
}
if idx := findKey(b, key); idx >= 0 {
mergeNodesAt(b.Content[idx+1], val, childPath, onCollision)
} else {
b.Content = append(b.Content, o.Content[i], val)
}
}
}
func findKey(m *yaml.Node, key string) int {
for i := 0; i+1 < len(m.Content); i += 2 {
if m.Content[i].Value == key {
return i
}
}
return -1
}
package config
import (
"fmt"
"strings"
)
// Operating modes for the Mode field.
const (
// ModeEnforce is the default: every subsystem is governed by its own
// switch and CSM keeps its host integration files current.
ModeEnforce = "enforce"
// ModeObserve declares that CSM must not change host state on this
// host. Detection, correlation, alerting and the audit sinks all run;
// automatic host remediation and integration updates do not. Two probes
// documented in the capability matrix still write: BPF capability
// discovery, and the kcarectl query that refreshes its own cache.
ModeObserve = "observe"
)
// ObserveMode reports whether this host runs in the observe posture.
func (cfg *Config) ObserveMode() bool {
return normalizeMode(cfg.Mode) == ModeObserve
}
func normalizeMode(mode string) string {
return strings.ToLower(strings.TrimSpace(mode))
}
// observeConflict names one config key whose value would let CSM change host
// state, and the value an observe-mode host has to use instead.
type observeConflict struct {
key string
want string
}
// observeConflicts lists switches for automatic changes to host files,
// processes or traffic. New state-changing subsystems belong here so
// validation can report every conflict in one pass.
func observeConflicts(cfg *Config) []observeConflict {
var out []observeConflict
add := func(on bool, key, want string) {
if on {
out = append(out, observeConflict{key: key, want: want})
}
}
add(cfg.AutoResponse.Enabled, "auto_response.enabled", "false")
add(cfg.AutoResponse.CopyFailKillProcess, "auto_response.copy_fail_kill_process", "false")
add(cfg.Firewall != nil && cfg.Firewall.Enabled, "firewall.enabled", "false")
add(cfg.PHPShield.Enabled, "php_shield.enabled", "false")
add(cfg.BPFEnforcement.Enabled, "bpf_enforcement.enabled", "false")
add(cfg.EmailProtection.ForwardGuard.Enabled, "email_protection.forward_guard.enabled", "false")
add(cfg.EmailProtection.Antispam.Enabled, "email_protection.antispam.enabled", "false")
add(cfg.EmailProtection.Antispam.Learning.Enabled, "email_protection.antispam.learning.enabled", "false")
add(cfg.EmailAV.Enabled && cfg.EmailAV.QuarantineInfected, "email_av.quarantine_infected", "false")
add(cfg.EmailAV.Enabled && cfg.EmailAV.FailMode == "tempfail", "email_av.fail_mode", "open")
add(cfg.AutoResponse.PHPRelay.Freeze != nil && *cfg.AutoResponse.PHPRelay.Freeze,
"auto_response.php_relay.freeze", "false")
add(cfg.AutoResponse.MailAuthRecovery.RestartEnabled,
"auto_response.mail_auth_recovery.restart_enabled", "false")
add(cfg.VirtualPatchMode() == VirtualPatchAuto,
"auto_response.virtual_patch_exposed_files", "off or manual")
return out
}
// validateMode rejects an unknown mode, and rejects an observe-mode config
// that still enables a subsystem which writes to the host. Every conflicting
// key is named in one error so an operator fixes the file in one pass.
func validateMode(cfg *Config) error {
switch normalizeMode(cfg.Mode) {
case ModeEnforce:
return nil
case ModeObserve:
default:
return fmt.Errorf("mode: %q is not a valid mode (use %q or %q)", cfg.Mode, ModeEnforce, ModeObserve)
}
conflicts := observeConflicts(cfg)
if len(conflicts) == 0 {
return nil
}
parts := make([]string, 0, len(conflicts))
for _, c := range conflicts {
parts = append(parts, fmt.Sprintf("%s (set %s)", c.key, c.want))
}
return fmt.Errorf("mode: observe forbids changing host state, but these keys still enable it: %s",
strings.Join(parts, ", "))
}
package config
import "slices"
const redactedValue = "***REDACTED***"
// RedactedValue is the placeholder Redact writes over a secret. A settings
// save that sends it back means "keep the stored secret".
const RedactedValue = redactedValue
var sensitiveScalarPaths = map[string]struct{}{
"alerts.webhook.hmac_secret": {},
"auto_response.verdict_callback.hmac_secret": {},
"challenge.captcha_fallback.secret_key": {},
"challenge.secret": {},
"challenge.verified_session.admin_secret": {},
"geoip.license_key": {},
"integrity.binary_hash": {},
"integrity.config_hash": {},
"integrity.confd_hash": {},
"reputation.abuseipdb_key": {},
"reputation.rspamd.token": {},
"reputation.upstream.token": {},
"sentry.dsn": {},
"webui.auth_token": {},
"webui.metrics_token": {},
}
// Redact returns a copy of the config with sensitive fields replaced.
// Empty fields are left empty (not replaced with the redaction marker).
// The original config is not modified.
func Redact(cfg *Config) *Config {
// Shallow copy the struct
c := *cfg
// Redact secrets (only if non-empty)
if c.WebUI.AuthToken != "" {
c.WebUI.AuthToken = redactedValue
}
if c.WebUI.MetricsToken != "" {
c.WebUI.MetricsToken = redactedValue
}
if len(c.WebUI.Tokens) > 0 {
c.WebUI.Tokens = append([]WebUIToken(nil), c.WebUI.Tokens...)
for i := range c.WebUI.Tokens {
if c.WebUI.Tokens[i].Token != "" {
c.WebUI.Tokens[i].Token = redactedValue
}
}
}
if len(c.MCP.Tokens) > 0 {
c.MCP.Tokens = append([]MCPToken(nil), c.MCP.Tokens...)
for i := range c.MCP.Tokens {
if c.MCP.Tokens[i].Token != "" {
c.MCP.Tokens[i].Token = redactedValue
}
}
}
if c.Alerts.Webhook.HMACSecret != "" {
c.Alerts.Webhook.HMACSecret = redactedValue
}
if c.GeoIP.LicenseKey != "" {
c.GeoIP.LicenseKey = redactedValue
}
if c.Reputation.AbuseIPDBKey != "" {
c.Reputation.AbuseIPDBKey = redactedValue
}
if c.Reputation.Rspamd.Token != "" {
c.Reputation.Rspamd.Token = redactedValue
}
if c.Reputation.Upstream.Token != "" {
c.Reputation.Upstream.Token = redactedValue
}
if c.AutoResponse.VerdictCallback.HMACSecret != "" {
c.AutoResponse.VerdictCallback.HMACSecret = redactedValue
}
if c.Challenge.Secret != "" {
c.Challenge.Secret = redactedValue
}
if c.Challenge.CaptchaFallback.SecretKey != "" {
c.Challenge.CaptchaFallback.SecretKey = redactedValue
}
if c.Challenge.VerifiedSession.AdminSecret != "" {
c.Challenge.VerifiedSession.AdminSecret = redactedValue
}
if c.Integrity.BinaryHash != "" {
c.Integrity.BinaryHash = redactedValue
}
if c.Integrity.ConfigHash != "" {
c.Integrity.ConfigHash = redactedValue
}
if c.Integrity.ConfdHash != "" {
c.Integrity.ConfdHash = redactedValue
}
if c.Sentry.DSN != "" {
c.Sentry.DSN = redactedValue
}
// Credential-bearing URLs keep scheme and host only.
c.Stalwart.APIURL = RedactURL(c.Stalwart.APIURL)
c.Alerts.Webhook.URL = RedactURL(c.Alerts.Webhook.URL)
c.Alerts.Heartbeat.URL = RedactURL(c.Alerts.Heartbeat.URL)
c.AutoResponse.VerdictCallback.URL = RedactURL(c.AutoResponse.VerdictCallback.URL)
c.Reputation.Rspamd.URL = RedactURL(c.Reputation.Rspamd.URL)
c.Reputation.Upstream.URL = RedactURL(c.Reputation.Upstream.URL)
if len(c.Reputation.Report.Targets) > 0 {
targets := slices.Clone(c.Reputation.Report.Targets)
for i := range targets {
targets[i].URL = RedactURL(targets[i].URL)
}
c.Reputation.Report.Targets = targets
}
// Deep-copy Firewall pointer so we don't share it with the original
if cfg.Firewall != nil {
fw := *cfg.Firewall
c.Firewall = &fw
}
return &c
}
func redactConfigScalarForLog(keyPath, value string) string {
if value == "" {
return value
}
if _, ok := sensitiveScalarPaths[keyPath]; ok {
return redactedValue
}
if _, ok := urlScalarPaths[keyPath]; ok {
return RedactURL(value)
}
return value
}
package config
import "net/url"
const redactedURLTail = "[REDACTED]"
// RedactURL keeps only the scheme and host of a URL for display. Webhook,
// heartbeat and callback URLs carry their credential in the path, query or
// userinfo (Slack and Discord webhooks, healthcheck pings, token query
// parameters), so everything past the host is replaced. A bare
// scheme://host[:port][/] is returned unchanged; an unparseable value is
// hidden entirely.
func RedactURL(raw string) string {
if raw == "" {
return raw
}
u, err := url.Parse(raw)
if err != nil || u.Scheme == "" || u.Host == "" {
return redactedURLTail
}
bare := u.User == nil && (u.Path == "" || u.Path == "/") && u.RawQuery == "" && u.Fragment == "" && u.RawPath == ""
if bare {
return raw
}
return u.Scheme + "://" + u.Host + "/" + redactedURLTail
}
// urlScalarPaths are config keys whose value is a credential-bearing URL;
// hot-reload diff logging shows them through RedactURL.
var urlScalarPaths = map[string]struct{}{
"stalwart.api_url": {},
"alerts.webhook.url": {},
"alerts.heartbeat.url": {},
"auto_response.verdict_callback.url": {},
"reputation.rspamd.url": {},
"reputation.upstream.url": {},
}
package config
import (
"bytes"
"strings"
"gopkg.in/yaml.v3"
)
// removedKeys are settings an earlier release documented and that nothing
// reads any more. Decoding is strict, so they are dropped before the decoder
// sees them: a host upgraded with one still in csm.yaml or a conf.d fragment
// keeps starting, and Validate warns so the operator can delete the key.
var removedKeys = []string{
"alerts.webhook.per_finding",
"thresholds.state_expiry_hours",
"thresholds.wp_core_check_interval_min",
"thresholds.webshell_scan_interval_min",
"thresholds.filesystem_scan_interval_min",
"email_protection.php_relay.reputation_failures_per_24h",
"email_protection.php_relay.baseline_sigma",
"email_protection.php_relay.baseline_observation_days",
"email_protection.forward_guard.skip_forwarders",
"suppressions.suppress_blocked_alerts",
}
// stripRemovedKeys returns data without any removed key and the dotted paths
// it dropped, in removedKeys order. Data without a removed key is returned
// unchanged, so the common case never re-encodes the document.
func stripRemovedKeys(data []byte) ([]byte, []string, error) {
if len(bytes.TrimSpace(data)) == 0 {
return data, nil, nil
}
var doc yaml.Node
if err := yaml.Unmarshal(data, &doc); err != nil {
return nil, nil, err
}
if doc.Kind != yaml.DocumentNode || len(doc.Content) == 0 {
return data, nil, nil
}
// LoadBytes also serves rehash and Settings without a conf.d merge.
// Resolve YAML merges here too so retired inherited settings are dropped
// without deleting a shared anchor still used by a live setting.
normalized, err := normalizeYAMLForMerge(&doc)
if err != nil {
return nil, nil, err
}
doc = *normalized
var dropped []string
for _, key := range removedKeys {
if deleteMappingPath(doc.Content[0], strings.Split(key, ".")) {
dropped = append(dropped, key)
}
}
if len(dropped) == 0 {
return data, nil, nil
}
out, err := yaml.Marshal(&doc)
if err != nil {
return nil, nil, err
}
return out, dropped, nil
}
// deleteMappingPath removes the key at path from the mapping node and reports
// whether it was present.
func deleteMappingPath(node *yaml.Node, path []string) bool {
if node == nil || node.Kind != yaml.MappingNode || len(path) == 0 {
return false
}
for i := 0; i+1 < len(node.Content); i += 2 {
if node.Content[i].Value != path[0] {
continue
}
if len(path) == 1 {
node.Content = append(node.Content[:i], node.Content[i+2:]...)
return true
}
return deleteMappingPath(node.Content[i+1], path[1:])
}
return false
}
package config
import (
"reflect"
"strings"
"time"
)
// Schema returns a JSON Schema (draft-07-style, partial) describing the
// Config struct via reflection over yaml: tags. Used by phpanel's config
// editor for client-side validation. Not a complete spec implementation -
// covers YAML field names, scalar/container types, and nested objects.
//
// IMPORTANT: This schema is structural only. Imperative validation rules
// enforced by Validate() (e.g., mail_logs.source must be auto/file/journal,
// webui.tokens[].scope must be admin or read) are NOT encoded here.
// Phpanel must still call `csm validate` for the authoritative check.
func Schema() map[string]interface{} {
return reflectStruct(reflect.TypeOf(Config{}))
}
func reflectStruct(t reflect.Type) map[string]interface{} {
if t.Kind() == reflect.Pointer {
t = t.Elem()
}
if t.Kind() != reflect.Struct {
return map[string]interface{}{"type": "object"}
}
props := map[string]interface{}{}
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
tag := f.Tag.Get("yaml")
if tag == "" || tag == "-" {
continue
}
name, _ := splitYAMLTag(tag)
if name == "" {
continue
}
props[name] = reflectField(f.Type)
}
return map[string]interface{}{
"type": "object",
"properties": props,
}
}
func reflectField(t reflect.Type) map[string]interface{} {
for t.Kind() == reflect.Pointer {
t = t.Elem()
}
if t == reflect.TypeOf(time.Duration(0)) {
return map[string]interface{}{"type": "string", "format": "duration"}
}
switch t.Kind() {
case reflect.String:
return map[string]interface{}{"type": "string"}
case reflect.Bool:
return map[string]interface{}{"type": "boolean"}
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return map[string]interface{}{"type": "integer"}
case reflect.Float32, reflect.Float64:
return map[string]interface{}{"type": "number"}
case reflect.Slice, reflect.Array:
return map[string]interface{}{"type": "array", "items": reflectField(t.Elem())}
case reflect.Map:
return map[string]interface{}{"type": "object", "additionalProperties": reflectField(t.Elem())}
case reflect.Struct:
return reflectStruct(t)
default:
return map[string]interface{}{}
}
}
func splitYAMLTag(tag string) (string, []string) {
parts := strings.Split(tag, ",")
return parts[0], parts[1:]
}
package config
import (
"errors"
"fmt"
"github.com/pidginhost/csm/internal/clientaddr"
"github.com/pidginhost/csm/internal/netutil"
)
// TrustedProxies returns the one trusted proxy list every HTTP producer and
// the challenge server resolve clients against: web_server.trusted_proxies
// and challenge.trusted_proxies together. An unusable entry is skipped on
// its own, as earlier releases skipped it, and Validate warns about it.
func TrustedProxies(cfg *Config) clientaddr.Trusted {
if cfg == nil {
return clientaddr.Trusted{}
}
entries := make([]string, 0, len(cfg.WebServer.TrustedProxies)+len(cfg.Challenge.TrustedProxies))
entries = append(entries, cfg.WebServer.TrustedProxies...)
entries = append(entries, cfg.Challenge.TrustedProxies...)
trusted, _ := clientaddr.ParseUsable(entries)
return trusted
}
// WebServerLocalTrustError reports a web_server.trusted_proxies entry that
// covers an address of this host. CSM never trusts such an entry, since any
// local account can connect from that address and write X-Forwarded-For, so
// the requests a proxy there forwards count against nobody.
func WebServerLocalTrustError(entries []string) error {
entry, covered, err := netutil.HostLocalProxyEntry(usableTrustedProxyEntries(entries))
if err != nil {
return fmt.Errorf("web_server.trusted_proxies: cannot check the entries: %v", err)
}
if entry == "" {
return nil
}
return fmt.Errorf("web_server.trusted_proxies entry %q covers %s, an address of this host: CSM never trusts it, so requests a proxy there forwards count against nobody. Have that proxy's web server log the client address, or remove the entry", entry, covered)
}
// UnusableTrustedProxies reports each trusted_proxies entry CSM skips, in
// the words Validate warns with.
func UnusableTrustedProxies(cfg *Config) []ValidationResult {
if cfg == nil {
return nil
}
return append(unusableTrustedProxyResults("warn", "web_server.trusted_proxies", cfg.WebServer.TrustedProxies),
unusableTrustedProxyResults("warn", "challenge.trusted_proxies", cfg.Challenge.TrustedProxies)...)
}
// unusableTrustedProxyResults reports each entry of key that is empty, not
// an IP address or CIDR range, or a catch-all. Validate reports them as
// warnings, since an error stops a daemon that ran with them; Settings save
// refuses them as errors.
func unusableTrustedProxyResults(level, key string, entries []string) []ValidationResult {
_, errs := clientaddr.ParseUsable(entries)
results := make([]ValidationResult, 0, len(errs))
for _, err := range errs {
msg := fmt.Sprintf("%s%v: CSM skips this entry", key, err)
if errors.Is(err, clientaddr.ErrCatchAllEntry) {
msg = fmt.Sprintf("%s%v: CSM never trusts it, and forwarded requests from the peers it covers count against nobody. List your proxies' addresses instead", key, err)
}
results = append(results, ValidationResult{level, key, msg})
}
return results
}
// usableTrustedProxyEntries drops the entries CSM skips. A skipped catch-all
// trusts no address, so it is reported as unusable, not as an address of
// this host.
func usableTrustedProxyEntries(entries []string) []string {
usable := make([]string, 0, len(entries))
for _, entry := range entries {
if _, err := clientaddr.Parse([]string{entry}); err == nil {
usable = append(usable, entry)
}
}
return usable
}
package config
import (
"errors"
"fmt"
"net"
"net/http"
"net/netip"
"net/url"
"os"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
"time"
"golang.org/x/text/language"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/safepath"
"github.com/pidginhost/csm/internal/sshdconf"
)
// ValidationResult represents a single validation finding.
type ValidationResult struct {
Level string // "error", "warn", "ok"
Field string // dotted path matching YAML keys
Message string
}
// String implements the Stringer interface for nice printing.
func (v ValidationResult) String() string {
return fmt.Sprintf("[%s] %s: %s", strings.ToUpper(v.Level), v.Field, v.Message)
}
// Validate checks the config for errors, warnings, and emits OK for valid sections.
func Validate(cfg *Config) []ValidationResult {
var results []ValidationResult
for index, pattern := range cfg.AccountRoots {
if err := platform.ValidateAccountRootPattern(pattern); err != nil {
results = append(results, ValidationResult{"error", fmt.Sprintf("account_roots[%d]", index), err.Error()})
}
}
if cfg.ObserveMode() {
results = append(results, ValidationResult{"ok", "mode", "observe (CSM does not change host state on this host)"})
} else {
results = append(results, ValidationResult{"ok", "mode", ModeEnforce})
}
// --- Hostname ---
if cfg.Hostname == "" || cfg.Hostname == "SET_HOSTNAME_HERE" {
results = append(results, ValidationResult{"error", "hostname", "hostname is not set"})
} else {
results = append(results, ValidationResult{"ok", "hostname", cfg.Hostname})
}
// --- Alerts ---
if !cfg.Alerts.Email.Enabled && !cfg.Alerts.Webhook.Enabled {
results = append(results, ValidationResult{"warn", "alerts", "no alert method enabled; findings only appear in the web UI and API"})
}
// --- Email alerts ---
if cfg.Alerts.Email.Enabled {
if len(cfg.Alerts.Email.To) == 0 {
results = append(results, ValidationResult{"error", "alerts.email.to", "email alerts enabled but no recipients configured"})
} else {
valid := true
for _, to := range cfg.Alerts.Email.To {
if to == "SET_EMAIL_HERE" || !strings.Contains(to, "@") {
results = append(results, ValidationResult{"error", "alerts.email.to", fmt.Sprintf("invalid email recipient: %s", to)})
valid = false
}
}
if valid {
results = append(results, ValidationResult{"ok", "alerts.email.to", strings.Join(cfg.Alerts.Email.To, ", ")})
}
}
if cfg.Alerts.Email.From == "" {
results = append(results, ValidationResult{"error", "alerts.email.from", "email alerts enabled but no from address configured"})
}
if cfg.Alerts.Email.SMTP == "" {
results = append(results, ValidationResult{"error", "alerts.email.smtp", "email alerts enabled but no SMTP server configured"})
} else {
results = append(results, ValidationResult{"ok", "alerts.email.smtp", cfg.Alerts.Email.SMTP})
}
}
// --- Webhook ---
if cfg.Alerts.Webhook.Enabled {
if cfg.Alerts.Webhook.URL == "" {
results = append(results, ValidationResult{"error", "alerts.webhook.url", "webhook alerts enabled but no URL configured"})
} else {
results = append(results, ValidationResult{"ok", "alerts.webhook.url", RedactURL(cfg.Alerts.Webhook.URL)})
}
switch cfg.Alerts.Webhook.Type {
case "", "slack", "discord", "generic", "phpanel":
default:
results = append(results, ValidationResult{"error", "alerts.webhook.type", fmt.Sprintf("unknown webhook type %q", cfg.Alerts.Webhook.Type)})
}
if cfg.Alerts.Webhook.Type == "phpanel" {
secret := cfg.Alerts.Webhook.HMACSecret
if cfg.Alerts.Webhook.HMACSecretEnv != "" {
if v := os.Getenv(cfg.Alerts.Webhook.HMACSecretEnv); v != "" {
secret = v
}
}
if secret == "" {
field := "alerts.webhook.hmac_secret"
if cfg.Alerts.Webhook.HMACSecretEnv != "" {
field = "alerts.webhook.hmac_secret_env"
}
results = append(results, ValidationResult{"error", field, "phpanel webhook enabled but no HMAC secret configured"})
}
}
}
// --- Heartbeat ---
if cfg.Alerts.Heartbeat.Enabled {
if cfg.Alerts.Heartbeat.URL == "" {
results = append(results, ValidationResult{"error", "alerts.heartbeat.url", "heartbeat enabled but no URL configured"})
} else {
results = append(results, ValidationResult{"ok", "alerts.heartbeat.url", RedactURL(cfg.Alerts.Heartbeat.URL)})
}
}
// --- MaxPerHour ---
if cfg.Alerts.MaxPerHour <= 0 {
results = append(results, ValidationResult{"error", "alerts.max_per_hour", "max_per_hour must be > 0"})
}
// --- WebUI ---
results = append(results, shortTokenWarnings(cfg)...)
results = append(results, tokenExpiryWarnings(cfg, time.Now())...)
for _, origin := range cfg.WebUI.AllowedOrigins {
if err := validateBrowserOrigin(origin); err != nil {
results = append(results, ValidationResult{"error", "webui.allowed_origins", fmt.Sprintf("%q: %v", origin, err)})
}
}
if cfg.WebUI.Enabled {
if err := validateWebUITokens(cfg); err != nil {
results = append(results, ValidationResult{"error", "webui.tokens", err.Error()})
}
tokenCount, adminCount := webUITokenCounts(cfg)
if tokenCount == 0 {
results = append(results, ValidationResult{"error", "webui.tokens", "webui enabled but no auth token configured"})
} else {
results = append(results, ValidationResult{"ok", "webui", fmt.Sprintf("listening on %s", cfg.WebUI.Listen)})
if adminCount == 0 {
results = append(results, ValidationResult{"warn", "webui.tokens", "no admin-scope token configured; browser login and admin API calls are disabled"})
}
}
results = append(results, webUIExposureResults(cfg)...)
}
// --- MCP ---
results = append(results, mcpResults(cfg, time.Now())...)
// --- Trusted countries ---
for _, cc := range cfg.Suppressions.TrustedCountries {
if len(cc) != 2 {
results = append(results, ValidationResult{"error", "suppressions.trusted_countries", fmt.Sprintf("invalid country code: %q (expected 2-letter ISO code)", cc)})
}
}
// Credentials only authorize a download; they say nothing about whether a
// database exists. Checking them instead of the database let a wrong key
// pass validation while no database was ever fetched, so the setting was
// inert and reported healthy. Check what the daemon actually reads.
if len(cfg.Suppressions.TrustedCountries) > 0 && !geoIPCityDatabasePresent(cfg.StatePath) {
remedy := "Provision that database, or set geoip.account_id and geoip.license_key and run csm update-geoip"
if cfg.GeoIP.AccountID != "" && cfg.GeoIP.LicenseKey != "" {
remedy = "Credentials are set but no database has been downloaded; run csm update-geoip and check it reports success"
}
results = append(results, ValidationResult{"warn", "suppressions.trusted_countries", "configured but no GeoLite2-City database is installed, so country lookups return nothing and no address is ever treated as trusted. " + remedy})
}
// --- Block digest ---
switch cfg.Alerts.BlockDigest.SendOn {
case "", "any", "customer":
default:
results = append(results, ValidationResult{"error", "alerts.block_digest.send_on", fmt.Sprintf("unknown send_on %q (want any|customer)", cfg.Alerts.BlockDigest.SendOn)})
}
switch cfg.Alerts.BlockDigest.Channel {
case "", "email", "webhook":
default:
results = append(results, ValidationResult{"error", "alerts.block_digest.channel", fmt.Sprintf("unknown channel %q (want email|webhook)", cfg.Alerts.BlockDigest.Channel)})
}
if cfg.Alerts.BlockDigest.Enabled {
switch {
case !cfg.Alerts.Email.Enabled && !cfg.Alerts.Webhook.Enabled:
results = append(results, ValidationResult{"error", "alerts.block_digest.channel", "watched-country summaries require email or webhook alerts to be enabled"})
case cfg.Alerts.BlockDigest.Live && cfg.Alerts.BlockDigest.Channel == "email" && !cfg.Alerts.Email.Enabled:
results = append(results, ValidationResult{"error", "alerts.block_digest.channel", "channel email requires email alerts to be enabled"})
case cfg.Alerts.BlockDigest.Live && cfg.Alerts.BlockDigest.Channel == "webhook" && !cfg.Alerts.Webhook.Enabled:
results = append(results, ValidationResult{"error", "alerts.block_digest.channel", "channel webhook requires webhook alerts to be enabled"})
}
}
if cfg.Alerts.BlockDigest.Interval != "" {
if d, err := time.ParseDuration(cfg.Alerts.BlockDigest.Interval); err != nil {
results = append(results, ValidationResult{"error", "alerts.block_digest.interval", fmt.Sprintf("unparseable duration: %s", cfg.Alerts.BlockDigest.Interval)})
} else if d <= 0 {
results = append(results, ValidationResult{"error", "alerts.block_digest.interval", "interval must be > 0"})
}
}
for _, cc := range cfg.Alerts.BlockDigest.Countries {
if len(cc) != 2 {
results = append(results, ValidationResult{"error", "alerts.block_digest.countries", fmt.Sprintf("invalid country code: %q (expected 2-letter ISO code)", cc)})
}
}
if cfg.Alerts.BlockDigest.MinBlock < 0 {
results = append(results, ValidationResult{"error", "alerts.block_digest.min_block", "min_block must be >= 0"})
}
// --- Duration fields ---
if cfg.AutoResponse.BlockExpiry != "" {
if d, err := time.ParseDuration(cfg.AutoResponse.BlockExpiry); err != nil {
results = append(results, ValidationResult{"error", "auto_response.block_expiry", fmt.Sprintf("unparseable duration: %s", cfg.AutoResponse.BlockExpiry)})
} else if d <= 0 {
results = append(results, ValidationResult{"error", "auto_response.block_expiry", "block_expiry must be a positive duration"})
}
} else if cfg.AutoResponse.Enabled && cfg.AutoResponse.BlockIPs {
results = append(results, ValidationResult{"error", "auto_response.block_expiry", fmt.Sprintf("block_expiry must be a positive duration when IP blocking is enabled; omit the key to use %s", DefaultBlockExpiry)})
}
if v := strings.ToLower(strings.TrimSpace(cfg.AutoResponse.VirtualPatchExposedFiles)); v != "" &&
v != VirtualPatchOff && v != VirtualPatchManual && v != VirtualPatchAuto {
results = append(results, ValidationResult{"error", "auto_response.virtual_patch_exposed_files", fmt.Sprintf("must be off, manual, or auto (got %q)", cfg.AutoResponse.VirtualPatchExposedFiles)})
}
if v := cfg.Thresholds.DropperUnlinkTTLSec; v != 0 && (v < MinDropperUnlinkTTLSec || v > MaxDropperUnlinkTTLSec) {
results = append(results, ValidationResult{"error", "thresholds.dropper_unlink_ttl_sec", fmt.Sprintf("must be between %d and %d seconds (got %d)", MinDropperUnlinkTTLSec, MaxDropperUnlinkTTLSec, v)})
}
if cfg.Signatures.UpdateInterval != "" {
if _, err := time.ParseDuration(cfg.Signatures.UpdateInterval); err != nil {
results = append(results, ValidationResult{"error", "signatures.update_interval", fmt.Sprintf("unparseable duration: %s", cfg.Signatures.UpdateInterval)})
}
}
if cfg.Signatures.YaraForge.UpdateInterval != "" {
if _, err := time.ParseDuration(cfg.Signatures.YaraForge.UpdateInterval); err != nil {
results = append(results, ValidationResult{"error", "signatures.yara_forge.update_interval", fmt.Sprintf("unparseable duration: %s", cfg.Signatures.YaraForge.UpdateInterval)})
}
}
if cfg.EmailAV.ScanTimeout != "" {
if _, err := time.ParseDuration(cfg.EmailAV.ScanTimeout); err != nil {
results = append(results, ValidationResult{"error", "email_av.scan_timeout", fmt.Sprintf("unparseable duration: %s", cfg.EmailAV.ScanTimeout)})
}
}
if cfg.GeoIP.UpdateInterval != "" {
if _, err := time.ParseDuration(cfg.GeoIP.UpdateInterval); err != nil {
results = append(results, ValidationResult{"error", "geoip.update_interval", fmt.Sprintf("unparseable duration: %s", cfg.GeoIP.UpdateInterval)})
}
}
if cfg.Reputation.BotRanges.UpdateInterval != "" {
if d, err := time.ParseDuration(cfg.Reputation.BotRanges.UpdateInterval); err != nil {
results = append(results, ValidationResult{"error", "reputation.bot_ranges.update_interval", fmt.Sprintf("unparseable duration: %s", cfg.Reputation.BotRanges.UpdateInterval)})
} else if d < time.Hour {
results = append(results, ValidationResult{"error", "reputation.bot_ranges.update_interval", "update_interval must be at least 1h"})
}
}
if cfg.AutoResponse.NetBlockWindow != "" {
if d, err := time.ParseDuration(cfg.AutoResponse.NetBlockWindow); err != nil {
results = append(results, ValidationResult{"error", "auto_response.netblock_window", fmt.Sprintf("unparseable duration: %s", cfg.AutoResponse.NetBlockWindow)})
} else if d <= 0 {
results = append(results, ValidationResult{"error", "auto_response.netblock_window", "netblock_window must be a positive duration"})
}
} else if cfg.AutoResponse.NetBlock {
results = append(results, ValidationResult{"error", "auto_response.netblock_window", fmt.Sprintf("netblock_window must be a positive duration when netblock is enabled; omit the key to use %s", DefaultNetBlockWindow)})
}
if cfg.AutoResponse.PermBlockInterval != "" {
if d, err := time.ParseDuration(cfg.AutoResponse.PermBlockInterval); err != nil {
results = append(results, ValidationResult{"error", "auto_response.permblock_interval", fmt.Sprintf("unparseable duration: %s", cfg.AutoResponse.PermBlockInterval)})
} else if d <= 0 {
results = append(results, ValidationResult{"error", "auto_response.permblock_interval", "permblock_interval must be a positive duration"})
}
} else if cfg.AutoResponse.PermBlock {
results = append(results, ValidationResult{"error", "auto_response.permblock_interval", fmt.Sprintf("permblock_interval must be a positive duration when permblock is enabled; omit the key to use %s", DefaultPermBlockInterval)})
}
if cfg.AutoResponse.MailAuthRecovery.DownGrace != "" {
if d, err := time.ParseDuration(cfg.AutoResponse.MailAuthRecovery.DownGrace); err != nil {
results = append(results, ValidationResult{"error", "auto_response.mail_auth_recovery.down_grace", fmt.Sprintf("unparseable duration: %s", cfg.AutoResponse.MailAuthRecovery.DownGrace)})
} else if d <= 0 {
results = append(results, ValidationResult{"error", "auto_response.mail_auth_recovery.down_grace", "down_grace must be > 0"})
}
}
// --- Retention ---
if cfg.Retention.Enabled {
if cfg.Retention.SweepInterval != "" {
if _, err := time.ParseDuration(cfg.Retention.SweepInterval); err != nil {
results = append(results, ValidationResult{"error", "retention.sweep_interval", fmt.Sprintf("unparseable duration: %s", cfg.Retention.SweepInterval)})
}
}
if cfg.Retention.FindingsDays < 0 {
results = append(results, ValidationResult{"error", "retention.findings_days", fmt.Sprintf("findings_days must be >= 0 (0 disables the sweep), got %d", cfg.Retention.FindingsDays)})
}
if cfg.Retention.HistoryDays < 0 {
results = append(results, ValidationResult{"error", "retention.history_days", fmt.Sprintf("history_days must be >= 0, got %d", cfg.Retention.HistoryDays)})
}
if cfg.Retention.ReputationDays < 0 {
results = append(results, ValidationResult{"error", "retention.reputation_days", fmt.Sprintf("reputation_days must be >= 0, got %d", cfg.Retention.ReputationDays)})
}
}
if cfg.Retention.CompactMinSizeMB < 0 {
results = append(results, ValidationResult{"error", "retention.compact_min_size_mb", fmt.Sprintf("compact_min_size_mb must be >= 0, got %d", cfg.Retention.CompactMinSizeMB)})
}
if cfg.Retention.CompactFillRatio < 0 || cfg.Retention.CompactFillRatio > 1 || (cfg.Retention.Enabled && cfg.Retention.CompactFillRatio == 0) {
results = append(results, ValidationResult{"error", "retention.compact_fill_ratio", fmt.Sprintf("compact_fill_ratio must be in (0, 1], got %v", cfg.Retention.CompactFillRatio)})
}
results = append(results, confdResults(cfg)...)
// --- Firewall ---
if cfg.Firewall != nil {
for _, e := range validateDOSExemptRanges(cfg.Firewall.DOSExemptRanges) {
results = append(results, ValidationResult{"error", "firewall.dos_exempt_ranges", e})
}
if _, err := firewall.ParseLocalPortLocks(cfg.Firewall.LocalPortLocks); err != nil {
results = append(results, ValidationResult{"error", "firewall.local_port_locks", err.Error()})
}
if cfg.Firewall.Enabled {
// 0 disables the connection meter: it is what the engine does with
// the value and what the web UI's own help text promises. Absent
// keys are filled from the shipped defaults before validation runs
// (see applyFirewallFieldDefaults), so a 0 reaching here is always
// the operator's explicit choice rather than an unset field.
if cfg.Firewall.ConnRateLimit < 0 {
results = append(results, ValidationResult{"error", "firewall.conn_rate_limit", fmt.Sprintf("conn_rate_limit must be >= 0 when firewall enabled (0 = disabled), got %d", cfg.Firewall.ConnRateLimit)})
}
if cfg.Firewall.ConnLimit < 0 {
results = append(results, ValidationResult{"error", "firewall.conn_limit", "conn_limit must be >= 0 when firewall enabled (0 = disabled)"})
}
if cfg.Firewall.ConnRateLimit >= 0 && cfg.Firewall.ConnLimit >= 0 {
results = append(results, ValidationResult{"ok", "firewall", fmt.Sprintf("enabled, conn_rate_limit=%s, conn_limit=%s",
limitSummary(cfg.Firewall.ConnRateLimit), limitSummary(cfg.Firewall.ConnLimit))})
}
}
results = append(results, firewallLockoutResults(cfg)...)
results = append(results, firewallEgressResults(cfg)...)
results = append(results, firewallValueResults(cfg.Firewall)...)
}
results = append(results, centralActionResults(cfg)...)
results = append(results, centralAuthTargetResults(cfg)...)
results = append(results, blockAtSeverityResults(cfg)...)
// --- Challenge ---
if cfg.Challenge.Difficulty < 0 || cfg.Challenge.Difficulty > 5 {
results = append(results, ValidationResult{"error", "challenge.difficulty", fmt.Sprintf("difficulty must be 0-5, got %d", cfg.Challenge.Difficulty)})
}
socketPath, socket := ChallengeSocketPath(cfg.Challenge.ListenAddr)
if !socket {
if cfg.Challenge.ListenPort < 0 || cfg.Challenge.ListenPort > 65535 {
results = append(results, ValidationResult{"error", "challenge.listen_port", fmt.Sprintf("listen_port must be 0-65535, got %d", cfg.Challenge.ListenPort)})
} else if cfg.Challenge.Enabled && cfg.Challenge.ListenPort == 0 {
results = append(results, ValidationResult{"error", "challenge.listen_port", fmt.Sprintf("listen_port must be 1-65535 when challenge.enabled, got %d", cfg.Challenge.ListenPort)})
}
}
if cfg.Challenge.Enabled {
if socket {
if err := ChallengeSocketPathError(socketPath); err != nil {
results = append(results, ValidationResult{"error", "challenge.listen_addr", err.Error()})
}
if strings.TrimSpace(cfg.Challenge.ListenGroup) == "" {
results = append(results, ValidationResult{"error", "challenge.listen_group", "listen_group must name the group your proxy runs as when challenge.listen_addr is a unix socket"})
}
}
// A warning, not an error: an error aborts daemon start and SIGHUP
// reloads. Request-time attribution refuses the local proxy.
if err := ChallengeLocalTrustError(cfg.Challenge.TrustedProxies); err != nil {
results = append(results, ValidationResult{"warn", "challenge.trusted_proxies", err.Error()})
}
}
// A warning too: an entry of this host is inert, and an error would stop
// a daemon that ran with it before.
if err := WebServerLocalTrustError(cfg.WebServer.TrustedProxies); err != nil {
results = append(results, ValidationResult{"warn", "web_server.trusted_proxies", err.Error()})
}
results = append(results, UnusableTrustedProxies(cfg)...)
// --- EmailAV ---
if cfg.EmailAV.Enabled && cfg.EmailAV.MaxAttachmentSize <= 0 {
results = append(results, ValidationResult{"error", "email_av.max_attachment_size", "max_attachment_size must be > 0 when email_av enabled"})
}
if cfg.EmailAV.WorkerMemoryLimit < 0 || (cfg.EmailAV.WorkerMemoryLimit > 0 && cfg.EmailAV.WorkerMemoryLimit < MinWorkerMemoryLimit) {
results = append(results, ValidationResult{"error", "email_av.worker_memory_limit",
fmt.Sprintf("worker_memory_limit %d is below the %d byte minimum", cfg.EmailAV.WorkerMemoryLimit, MinWorkerMemoryLimit)})
}
if cfg.EmailAV.FailMode != "" && cfg.EmailAV.FailMode != "open" && cfg.EmailAV.FailMode != "tempfail" {
results = append(results, ValidationResult{"error", "email_av.fail_mode",
fmt.Sprintf("invalid fail_mode %q: must be \"open\" or \"tempfail\"", cfg.EmailAV.FailMode)})
}
if cfg.Signatures.UpdateURL != "" && cfg.Signatures.SigningKey == "" {
results = append(results, ValidationResult{"error", "signatures.signing_key",
"signing_key is required when signatures.update_url is configured"})
}
if cfg.Signatures.YaraWorkerEnabled != nil && !*cfg.Signatures.YaraWorkerEnabled {
results = append(results, ValidationResult{"warn", "signatures.yara_worker_enabled",
"false runs YARA-X inside the root daemon, so a crafted file that breaks the scanner reaches a root process; remove it to scan in a locked-down worker"})
}
if cfg.Signatures.YaraForge.Enabled && cfg.Signatures.SigningKey == "" {
results = append(results, ValidationResult{"error", "signatures.signing_key",
"signing_key is required when signatures.yara_forge.enabled is true"})
}
if cfg.Signatures.YaraForge.Enabled && cfg.Signatures.YaraForge.DownloadURL == "" {
results = append(results, ValidationResult{"error", "signatures.yara_forge.download_url",
"download_url is required because upstream YARA Forge releases do not publish CSM detached signatures"})
}
if cfg.Signatures.YaraForge.DownloadURL != "" {
if err := validateSignatureURL(cfg.Signatures.YaraForge.DownloadURL, true); err != nil {
results = append(results, ValidationResult{"error", "signatures.yara_forge.download_url", err.Error()})
}
}
if cfg.Signatures.UpdateURL != "" {
if err := validateSignatureURL(cfg.Signatures.UpdateURL, false); err != nil {
results = append(results, ValidationResult{"error", "signatures.update_url", err.Error()})
}
}
if err := validateDirectSMTPEgress(cfg); err != nil {
results = append(results, ValidationResult{"error", "detection.direct_smtp_egress", err.Error()})
}
if err := validateBPFEnforcement(cfg); err != nil {
results = append(results, ValidationResult{"error", "bpf_enforcement", err.Error()})
}
// Warn only: the daemon runs an unknown value as auto, and the key is
// newer than configs that already load.
switch strings.ToLower(strings.TrimSpace(cfg.Detection.PlacementBackend)) {
case "", "auto", "bpf", "legacy", "none":
default:
results = append(results, ValidationResult{"warn", "detection.placement_backend",
fmt.Sprintf("unknown value %q runs as auto; use auto, bpf, legacy, or none", cfg.Detection.PlacementBackend)})
}
// --- EmailProtection ---
if cfg.EmailProtection.RateWarnThreshold > 0 && cfg.EmailProtection.RateWarnThreshold < 10 {
results = append(results, ValidationResult{"warn", "email_protection.rate_warn_threshold", "rate_warn_threshold < 10 may cause excessive alerts"})
}
if cfg.EmailProtection.RateCritThreshold > 0 && cfg.EmailProtection.RateCritThreshold <= cfg.EmailProtection.RateWarnThreshold {
results = append(results, ValidationResult{"error", "email_protection.rate_crit_threshold", "rate_crit_threshold must be > rate_warn_threshold"})
}
if cfg.EmailProtection.RateWindowMin > 0 && (cfg.EmailProtection.RateWindowMin < 5 || cfg.EmailProtection.RateWindowMin > 60) {
results = append(results, ValidationResult{"error", "email_protection.rate_window_min", "rate_window_min must be between 5 and 60"})
}
if cfg.EmailProtection.PasswordCheckIntervalMin > 0 && cfg.EmailProtection.PasswordCheckIntervalMin < 60 {
results = append(results, ValidationResult{"warn", "email_protection.password_check_interval_min", "password_check_interval_min < 60 may cause high CPU from doveadm"})
}
if cfg.Thresholds.FTPFailWindowMin != 0 &&
(cfg.Thresholds.FTPFailWindowMin < 1 || cfg.Thresholds.FTPFailWindowMin > 1440) {
results = append(results, ValidationResult{"error", "thresholds.ftp_fail_window_min",
fmt.Sprintf("ftp_fail_window_min must be between 1 and 1440, got %d", cfg.Thresholds.FTPFailWindowMin)})
}
// --- EmailProtection.PHPRelay bounds ---
// Bounds checks fire only when the operator has supplied a value
// (zero means "use the applyDefaults value" or, for AccountVolumePerHour,
// "auto-derive from cPanel maxemailsperhour"). PoliciesDir is NOT
// validated here -- filesystem state probes belong in ValidateDeep.
pr := cfg.EmailProtection.PHPRelay
if pr.RateWindowMin != 0 && (pr.RateWindowMin < 1 || pr.RateWindowMin > 60) {
results = append(results, ValidationResult{"error", "email_protection.php_relay.rate_window_min", fmt.Sprintf("rate_window_min must be between 1 and 60, got %d", pr.RateWindowMin)})
}
if pr.HeaderScoreVolumeMin != 0 && (pr.HeaderScoreVolumeMin < 2 || pr.HeaderScoreVolumeMin > 100) {
results = append(results, ValidationResult{"error", "email_protection.php_relay.header_score_volume_min", fmt.Sprintf("header_score_volume_min must be between 2 and 100, got %d", pr.HeaderScoreVolumeMin)})
}
if pr.AbsoluteVolumePerHour != 0 && (pr.AbsoluteVolumePerHour < 10 || pr.AbsoluteVolumePerHour > 1000) {
results = append(results, ValidationResult{"error", "email_protection.php_relay.absolute_volume_per_hour", fmt.Sprintf("absolute_volume_per_hour must be between 10 and 1000, got %d", pr.AbsoluteVolumePerHour)})
}
// AccountVolumePerHour: 0 is the documented "auto-derive" sentinel;
// only reject explicitly out-of-range positive values.
if pr.AccountVolumePerHour < 0 || pr.AccountVolumePerHour > 5000 {
results = append(results, ValidationResult{"error", "email_protection.php_relay.account_volume_per_hour", fmt.Sprintf("account_volume_per_hour must be between 0 (auto-derive) and 5000, got %d", pr.AccountVolumePerHour)})
}
if pr.FanoutDistinctScripts != 0 && (pr.FanoutDistinctScripts < 2 || pr.FanoutDistinctScripts > 20) {
results = append(results, ValidationResult{"error", "email_protection.php_relay.fanout_distinct_scripts", fmt.Sprintf("fanout_distinct_scripts must be between 2 and 20, got %d", pr.FanoutDistinctScripts)})
}
if pr.FanoutDistinctRecipients != 0 && (pr.FanoutDistinctRecipients < 1 || pr.FanoutDistinctRecipients > 100) {
results = append(results, ValidationResult{"error", "email_protection.php_relay.fanout_distinct_recipients", fmt.Sprintf("fanout_distinct_recipients must be between 1 and 100, got %d", pr.FanoutDistinctRecipients)})
}
if pr.FanoutWindowMin != 0 && (pr.FanoutWindowMin < 1 || pr.FanoutWindowMin > 60) {
results = append(results, ValidationResult{"error", "email_protection.php_relay.fanout_window_min", fmt.Sprintf("fanout_window_min must be between 1 and 60, got %d", pr.FanoutWindowMin)})
}
if pr.IPRecipientSpread != 0 && (pr.IPRecipientSpread < 1 || pr.IPRecipientSpread > 100) {
results = append(results, ValidationResult{"error", "email_protection.php_relay.ip_recipient_spread", fmt.Sprintf("ip_recipient_spread must be between 1 and 100, got %d", pr.IPRecipientSpread)})
}
if pr.IPRecipientWindowHours != 0 && (pr.IPRecipientWindowHours < 1 || pr.IPRecipientWindowHours > 168) {
results = append(results, ValidationResult{"error", "email_protection.php_relay.ip_recipient_window_hours", fmt.Sprintf("ip_recipient_window_hours must be between 1 and 168, got %d", pr.IPRecipientWindowHours)})
}
switch pr.IPRecipientMode {
case "", "observe", "enforce":
default:
results = append(results, ValidationResult{"error", "email_protection.php_relay.ip_recipient_mode", fmt.Sprintf("ip_recipient_mode must be observe or enforce, got %q", pr.IPRecipientMode)})
}
// --- AutoResponse.PHPRelay bounds ---
if cfg.AutoResponse.PHPRelay.MaxActionsPerMinute != 0 && (cfg.AutoResponse.PHPRelay.MaxActionsPerMinute < 1 || cfg.AutoResponse.PHPRelay.MaxActionsPerMinute > 600) {
results = append(results, ValidationResult{"error", "auto_response.php_relay.max_actions_per_minute", fmt.Sprintf("max_actions_per_minute must be between 1 and 600, got %d", cfg.AutoResponse.PHPRelay.MaxActionsPerMinute)})
}
for key, value := range map[string]int{
"max_file_actions_per_hour": cfg.AutoResponse.MaxFileActionsPerHour,
"max_file_actions_per_account_per_hour": cfg.AutoResponse.MaxFileActionsPerAccountPerHour,
"max_file_action_failures_per_hour": cfg.AutoResponse.MaxFileActionFailuresPerHour,
} {
if value < 0 || value > MaxFileResponseLimit {
results = append(results, ValidationResult{"error", "auto_response." + key, fmt.Sprintf("must be between 0 and %d (0 uses the default)", MaxFileResponseLimit)})
}
}
if cfg.AutoResponse.MaxBlocksPerHour < 0 {
results = append(results, ValidationResult{"error", "auto_response.max_blocks_per_hour", fmt.Sprintf("max_blocks_per_hour must be >= 0 (0 uses default %d), got %d", DefaultMaxBlocksPerHour, cfg.AutoResponse.MaxBlocksPerHour)})
}
if cfg.AutoResponse.MailAuthRecovery.MaxRestartsPerHour < 0 {
results = append(results, ValidationResult{"error", "auto_response.mail_auth_recovery.max_restarts_per_hour", fmt.Sprintf("max_restarts_per_hour must be >= 0 (0 uses default 3), got %d", cfg.AutoResponse.MailAuthRecovery.MaxRestartsPerHour)})
}
if cfg.AutoResponse.MailAuthRecovery.RestartEnabled && strings.TrimSpace(cfg.AutoResponse.MailAuthRecovery.RestartCommand) == "" {
results = append(results, ValidationResult{"error", "auto_response.mail_auth_recovery.restart_command", "restart_command is required when restart_enabled is true"})
}
// --- SMTP brute-force thresholds ---
t := cfg.Thresholds
if t.ExposedFileScanDepth != 0 &&
(t.ExposedFileScanDepth < 1 || t.ExposedFileScanDepth > MaxExposedFileScanDepth) {
results = append(results, ValidationResult{
"error",
"thresholds.exposed_file_scan_depth",
fmt.Sprintf("exposed_file_scan_depth must be between 1 and %d", MaxExposedFileScanDepth),
})
}
if t.DomlogMaxFiles != 0 && (t.DomlogMaxFiles < 1 || t.DomlogMaxFiles > 100000) {
results = append(results, ValidationResult{"error", "thresholds.domlog_max_files", "domlog_max_files must be between 1 and 100000"})
}
if t.AccountScanMaxFiles != 0 && (t.AccountScanMaxFiles < 1 || t.AccountScanMaxFiles > 100000) {
results = append(results, ValidationResult{"error", "thresholds.account_scan_max_files", "account_scan_max_files must be between 1 and 100000"})
}
if t.FullScanMaxFileMB != 0 && (t.FullScanMaxFileMB < 1 || t.FullScanMaxFileMB > 4096) {
results = append(results, ValidationResult{"error", "thresholds.full_scan_max_file_mb", "full_scan_max_file_mb must be between 1 and 4096"})
}
if t.ScanJobRetention != 0 && (t.ScanJobRetention < 1 || t.ScanJobRetention > 1000) {
results = append(results, ValidationResult{"error", "thresholds.scan_job_retention", "scan_job_retention must be between 1 and 1000"})
}
if t.CrontabBase64BlobMaxBytes != 0 {
if t.CrontabBase64BlobMaxBytes < 1024 || t.CrontabBase64BlobMaxBytes > 1048576 {
results = append(results, ValidationResult{"error", "thresholds.crontab_base64_blob_max_bytes", "crontab_base64_blob_max_bytes must be between 1024 and 1048576"})
} else if t.CrontabBase64BlobMaxBytes%4 != 0 {
results = append(results, ValidationResult{"error", "thresholds.crontab_base64_blob_max_bytes", "crontab_base64_blob_max_bytes must be a multiple of 4 (standard base64 alignment)"})
}
}
if t.DomlogTailLines != 0 && (t.DomlogTailLines < 10 || t.DomlogTailLines > 100000) {
results = append(results, ValidationResult{"error", "thresholds.domlog_tail_lines", "domlog_tail_lines must be between 10 and 100000"})
}
if t.DomlogMaxAgeMin != 0 && (t.DomlogMaxAgeMin < 1 || t.DomlogMaxAgeMin > 1440) {
results = append(results, ValidationResult{"error", "thresholds.domlog_max_age_min", "domlog_max_age_min must be between 1 and 1440"})
}
if t.MailLogTailLines != 0 && (t.MailLogTailLines < 10 || t.MailLogTailLines > 100000) {
results = append(results, ValidationResult{"error", "thresholds.mail_log_tail_lines", "mail_log_tail_lines must be between 10 and 100000"})
}
if t.SyslogMessagesTailLines != 0 && (t.SyslogMessagesTailLines < 10 || t.SyslogMessagesTailLines > 100000) {
results = append(results, ValidationResult{"error", "thresholds.syslog_messages_tail_lines", "syslog_messages_tail_lines must be between 10 and 100000"})
}
if t.CredStuffingDistinctAccounts != 0 && (t.CredStuffingDistinctAccounts < 2 || t.CredStuffingDistinctAccounts > 200) {
results = append(results, ValidationResult{"error", "thresholds.cred_stuffing_distinct_accounts", "cred_stuffing_distinct_accounts must be between 2 and 200"})
}
if t.PAMBruteforceThreshold != 0 && (t.PAMBruteforceThreshold < 2 || t.PAMBruteforceThreshold > 1000) {
results = append(results, ValidationResult{"error", "thresholds.pam_bruteforce_threshold", "pam_bruteforce_threshold must be between 2 and 1000"})
}
if t.PAMBruteforceWindowMin != 0 && (t.PAMBruteforceWindowMin < 1 || t.PAMBruteforceWindowMin > 1440) {
results = append(results, ValidationResult{"error", "thresholds.pam_bruteforce_window_min", "pam_bruteforce_window_min must be between 1 and 1440"})
}
if t.HTTPScannerMinRequests < 0 {
results = append(results, ValidationResult{"error", "thresholds.http_scanner_min_requests", "http_scanner_min_requests must be >= 0 (0 disables the detector)"})
}
if t.HTTPScannerErrorPct != 0 && (t.HTTPScannerErrorPct < 1 || t.HTTPScannerErrorPct > 100) {
results = append(results, ValidationResult{"error", "thresholds.http_scanner_error_pct", "http_scanner_error_pct must be between 1 and 100"})
}
if t.HTTPScannerMinDistinctPaths != 0 && (t.HTTPScannerMinDistinctPaths < 1 || t.HTTPScannerMinDistinctPaths > HTTPScannerMaxDistinctPaths) {
results = append(results, ValidationResult{"error", "thresholds.http_scanner_min_distinct_paths", fmt.Sprintf("http_scanner_min_distinct_paths must be between 1 and %d", HTTPScannerMaxDistinctPaths)})
}
for _, code := range t.HTTPScannerStatusCodes {
if code < 100 || code > 599 {
results = append(results, ValidationResult{"error", "thresholds.http_scanner_status_codes", "http_scanner_status_codes entries must be HTTP status codes between 100 and 599"})
break
}
}
switch cfg.AutoResponse.HTTPScannerAction {
case "", "challenge", "block":
default:
results = append(results, ValidationResult{"error", "auto_response.http_scanner_action", fmt.Sprintf("http_scanner_action must be %q or %q, got %q", "challenge", "block", cfg.AutoResponse.HTTPScannerAction)})
}
if t.SMTPBruteForceThreshold != 0 && (t.SMTPBruteForceThreshold < 2 || t.SMTPBruteForceThreshold > 50) {
results = append(results, ValidationResult{"error", "thresholds.smtp_bruteforce_threshold", "smtp_bruteforce_threshold must be between 2 and 50"})
}
if t.SMTPBruteForceWindowMin != 0 && (t.SMTPBruteForceWindowMin < 1 || t.SMTPBruteForceWindowMin > 60) {
results = append(results, ValidationResult{"error", "thresholds.smtp_bruteforce_window_min", "smtp_bruteforce_window_min must be between 1 and 60"})
}
if t.SMTPBruteForceSuppressMin != 0 && (t.SMTPBruteForceSuppressMin < 1 || t.SMTPBruteForceSuppressMin > 1440) {
results = append(results, ValidationResult{"error", "thresholds.smtp_bruteforce_suppress_min", "smtp_bruteforce_suppress_min must be between 1 and 1440"})
}
if t.SMTPBruteForceSubnetThresh != 0 && (t.SMTPBruteForceSubnetThresh < 2 || t.SMTPBruteForceSubnetThresh > 64) {
results = append(results, ValidationResult{"error", "thresholds.smtp_bruteforce_subnet_threshold", "smtp_bruteforce_subnet_threshold must be between 2 and 64"})
}
if t.SMTPAccountSprayThreshold != 0 && (t.SMTPAccountSprayThreshold < 2 || t.SMTPAccountSprayThreshold > 200) {
results = append(results, ValidationResult{"error", "thresholds.smtp_account_spray_threshold", "smtp_account_spray_threshold must be between 2 and 200"})
}
if t.SMTPBruteForceMaxTracked != 0 && (t.SMTPBruteForceMaxTracked < 1000 || t.SMTPBruteForceMaxTracked > 200000) {
results = append(results, ValidationResult{"error", "thresholds.smtp_bruteforce_max_tracked", "smtp_bruteforce_max_tracked must be between 1000 and 200000"})
}
if t.SMTPBruteForceSlowThreshold != 0 && (t.SMTPBruteForceSlowThreshold < SlowBruteMinThreshold || t.SMTPBruteForceSlowThreshold > SlowBruteMaxThreshold) {
results = append(results, ValidationResult{"error", "thresholds.smtp_bruteforce_slow_threshold", fmt.Sprintf("smtp_bruteforce_slow_threshold must be 0 (disabled) or between %d and %d", SlowBruteMinThreshold, SlowBruteMaxThreshold)})
}
if t.SMTPBruteForceSlowWindowMin != 0 && (t.SMTPBruteForceSlowWindowMin < 1 || t.SMTPBruteForceSlowWindowMin > SlowBruteMaxWindowMin) {
results = append(results, ValidationResult{"error", "thresholds.smtp_bruteforce_slow_window_min", fmt.Sprintf("smtp_bruteforce_slow_window_min must be between 1 and %d", SlowBruteMaxWindowMin)})
}
if t.SMTPProbeThreshold != 0 && (t.SMTPProbeThreshold < 10 || t.SMTPProbeThreshold > 10000) {
results = append(results, ValidationResult{"error", "thresholds.smtp_probe_threshold", "smtp_probe_threshold must be between 10 and 10000"})
}
if t.SMTPProbeWindowMin != 0 && (t.SMTPProbeWindowMin < 1 || t.SMTPProbeWindowMin > 60) {
results = append(results, ValidationResult{"error", "thresholds.smtp_probe_window_min", "smtp_probe_window_min must be between 1 and 60"})
}
if t.SMTPProbeSuppressMin != 0 && (t.SMTPProbeSuppressMin < 1 || t.SMTPProbeSuppressMin > 1440) {
results = append(results, ValidationResult{"error", "thresholds.smtp_probe_suppress_min", "smtp_probe_suppress_min must be between 1 and 1440"})
}
if t.SMTPProbeMaxTracked != 0 && (t.SMTPProbeMaxTracked < 1000 || t.SMTPProbeMaxTracked > 200000) {
results = append(results, ValidationResult{"error", "thresholds.smtp_probe_max_tracked", "smtp_probe_max_tracked must be between 1000 and 200000"})
}
if t.MailBruteForceThreshold != 0 && (t.MailBruteForceThreshold < 2 || t.MailBruteForceThreshold > 50) {
results = append(results, ValidationResult{"error", "thresholds.mail_bruteforce_threshold", "mail_bruteforce_threshold must be between 2 and 50"})
}
if t.MailBruteForceWindowMin != 0 && (t.MailBruteForceWindowMin < 1 || t.MailBruteForceWindowMin > 60) {
results = append(results, ValidationResult{"error", "thresholds.mail_bruteforce_window_min", "mail_bruteforce_window_min must be between 1 and 60"})
}
if t.MailBruteForceSuppressMin != 0 && (t.MailBruteForceSuppressMin < 1 || t.MailBruteForceSuppressMin > 1440) {
results = append(results, ValidationResult{"error", "thresholds.mail_bruteforce_suppress_min", "mail_bruteforce_suppress_min must be between 1 and 1440"})
}
if t.MailBruteForceSubnetThresh != 0 && (t.MailBruteForceSubnetThresh < 2 || t.MailBruteForceSubnetThresh > 64) {
results = append(results, ValidationResult{"error", "thresholds.mail_bruteforce_subnet_threshold", "mail_bruteforce_subnet_threshold must be between 2 and 64"})
}
if t.MailAccountSprayThreshold != 0 && (t.MailAccountSprayThreshold < 2 || t.MailAccountSprayThreshold > 200) {
results = append(results, ValidationResult{"error", "thresholds.mail_account_spray_threshold", "mail_account_spray_threshold must be between 2 and 200"})
}
if t.MailBruteForceMaxTracked != 0 && (t.MailBruteForceMaxTracked < 1000 || t.MailBruteForceMaxTracked > 200000) {
results = append(results, ValidationResult{"error", "thresholds.mail_bruteforce_max_tracked", "mail_bruteforce_max_tracked must be between 1000 and 200000"})
}
if t.MailBruteForceSlowThreshold != 0 && (t.MailBruteForceSlowThreshold < SlowBruteMinThreshold || t.MailBruteForceSlowThreshold > SlowBruteMaxThreshold) {
results = append(results, ValidationResult{"error", "thresholds.mail_bruteforce_slow_threshold", fmt.Sprintf("mail_bruteforce_slow_threshold must be 0 (disabled) or between %d and %d", SlowBruteMinThreshold, SlowBruteMaxThreshold)})
}
if t.MailBruteForceSlowWindowMin != 0 && (t.MailBruteForceSlowWindowMin < 1 || t.MailBruteForceSlowWindowMin > SlowBruteMaxWindowMin) {
results = append(results, ValidationResult{"error", "thresholds.mail_bruteforce_slow_window_min", fmt.Sprintf("mail_bruteforce_slow_window_min must be between 1 and %d", SlowBruteMaxWindowMin)})
}
switch t.MailSprayMode {
case "", "off", "observe", "enforce":
default:
results = append(results, ValidationResult{"error", "thresholds.mail_spray_mode", fmt.Sprintf("mail_spray_mode must be off, observe or enforce, got %q", t.MailSprayMode)})
}
if t.MailSprayWindowMin != 0 && (t.MailSprayWindowMin < 60 || t.MailSprayWindowMin > SlowBruteMaxWindowMin) {
results = append(results, ValidationResult{"error", "thresholds.mail_spray_window_min", fmt.Sprintf("mail_spray_window_min must be between 60 and %d", SlowBruteMaxWindowMin)})
}
if t.MailSprayBreadthMailboxes != 0 && (t.MailSprayBreadthMailboxes < 2 || t.MailSprayBreadthMailboxes > 10) {
results = append(results, ValidationResult{"error", "thresholds.mail_spray_breadth_mailboxes", "mail_spray_breadth_mailboxes must be 0 (disabled) or between 2 and 10"})
}
if t.MailSprayMissingNetworks != 0 && (t.MailSprayMissingNetworks < 1 || t.MailSprayMissingNetworks > 50) {
results = append(results, ValidationResult{"error", "thresholds.mail_spray_missing_networks", "mail_spray_missing_networks must be 0 (disabled) or between 1 and 50"})
}
if t.MailSprayBlockExpiry != "" {
if d, err := time.ParseDuration(t.MailSprayBlockExpiry); err != nil || d < 24*time.Hour || d > 2160*time.Hour {
results = append(results, ValidationResult{"error", "thresholds.mail_spray_block_expiry", fmt.Sprintf("mail_spray_block_expiry must be a duration between 24h and 2160h, got %q", t.MailSprayBlockExpiry)})
}
}
if field, err := validateMailBruteAccountKeyField(cfg); err != nil {
results = append(results, ValidationResult{"error", field, err.Error()})
}
// --- Mail log source ---
if field, err := validateMailLogsField(cfg); err != nil {
results = append(results, ValidationResult{"error", field, err.Error()})
}
if field, err := validateStalwartField(cfg); err != nil {
results = append(results, ValidationResult{"error", field, err.Error()})
}
// --- Reputation.Rspamd ---
if cfg.Reputation.Rspamd.Enabled {
secret := cfg.Reputation.Rspamd.Token
if cfg.Reputation.Rspamd.TokenEnv != "" {
if v := os.Getenv(cfg.Reputation.Rspamd.TokenEnv); v != "" {
secret = v
}
}
// rspamd's default secure_ip lets loopback clients read the controller
// without a password, so only a remote controller needs a token.
u, err := url.Parse(cfg.Reputation.Rspamd.URL)
loopback := err == nil && isLoopbackHost(u.Hostname())
if secret == "" && !loopback {
results = append(results, ValidationResult{"warn", "reputation.rspamd.token", "rspamd enabled but no token configured (rspamd controller history may require auth)"})
}
}
// --- Reputation.Upstream ---
if cfg.Reputation.Upstream.Enabled {
secret := cfg.Reputation.Upstream.Token
if cfg.Reputation.Upstream.TokenEnv != "" {
if v := os.Getenv(cfg.Reputation.Upstream.TokenEnv); v != "" {
secret = v
}
}
if secret == "" {
results = append(results, ValidationResult{"warn", "reputation.upstream.token", "upstream enabled but no token configured (panel endpoint may require auth)"})
}
}
// --- Reputation.VerifiedBots ---
results = append(results, validateVerifiedBots(cfg)...)
// --- AutoResponse.VerdictCallback ---
if field, err := validateVerdictCallbackField(cfg, LoadOptions{}); err != nil {
results = append(results, ValidationResult{"error", field, err.Error()})
}
// --- Debug / pprof ---
if addr := strings.TrimSpace(cfg.Debug.PprofListen); addr != "" {
host, _, err := net.SplitHostPort(addr)
host = strings.TrimSpace(host)
loopback := err == nil && host != "" &&
(strings.EqualFold(host, "localhost") || (net.ParseIP(host) != nil && net.ParseIP(host).IsLoopback()))
if loopback {
results = append(results, ValidationResult{"ok", "debug.pprof_listen", addr})
} else {
results = append(results, ValidationResult{"error", "debug.pprof_listen",
fmt.Sprintf("must be a loopback host:port (127.0.0.1/::1/localhost); %q would expose pprof off-box and is ignored at runtime", addr)})
}
}
results = append(results, blockEscalationResults(cfg)...)
// --- Warnings ---
results = append(results, validateWarnings(cfg)...)
return results
}
// minWebUITokenLength is the shortest Web UI or metrics token that is not
// reported as guessable. The installer generates 64 hex characters.
const minWebUITokenLength = 32
// shortTokenWarnings reports tokens shorter than minWebUITokenLength. They
// are warnings, never errors: startup, `csm validate` and `csm doctor` print
// them, and an existing short token keeps working. Results name the token,
// never its value.
func shortTokenWarnings(cfg *Config) []ValidationResult {
advice := fmt.Sprintf("shorter than %d characters and guessable; replace it with a random value (the installer generates 64 hex characters)", minWebUITokenLength)
var out []ValidationResult
for _, tok := range cfg.WebUI.Tokens {
if tok.Token != "" && len(tok.Token) < minWebUITokenLength {
out = append(out, ValidationResult{"warn", "webui.tokens", fmt.Sprintf("token %q is %s", tok.Name, advice)})
}
}
if len(cfg.WebUI.Tokens) == 0 && cfg.WebUI.AuthToken != "" && len(cfg.WebUI.AuthToken) < minWebUITokenLength {
out = append(out, ValidationResult{"warn", "webui.auth_token", "auth_token is " + advice})
}
if cfg.WebUI.MetricsToken != "" && len(cfg.WebUI.MetricsToken) < minWebUITokenLength {
out = append(out, ValidationResult{"warn", "webui.metrics_token", "metrics_token is " + advice})
}
return out
}
func webUITokenCounts(cfg *Config) (tokens, admins int) {
if len(cfg.WebUI.Tokens) == 0 && cfg.WebUI.AuthToken != "" {
tokens++
admins++
}
for _, tok := range cfg.WebUI.Tokens {
if tok.Token == "" {
continue
}
tokens++
if tok.Scope == "admin" {
admins++
}
}
return tokens, admins
}
// webUIExposureResults warns when the root-equivalent web UI answers on a
// non-loopback address with neither a restrictive CSM firewall policy nor
// address-limited admin tokens in front of it. A host firewall or proxy may
// still restrict it, so this is never an error.
func webUIExposureResults(cfg *Config) []ValidationResult {
v4, v6 := webUIListenFamilies(cfg.WebUI.Listen)
v4 = v4 && !webUIFirewallRestricts(cfg, false)
v6 = v6 && !webUIFirewallRestricts(cfg, true)
if webUIListensOnLoopback(cfg.WebUI.Listen) || !hasUnrestrictedAdminToken(cfg, v4, v6) {
return nil
}
listen := cfg.WebUI.Listen
if listen == "" {
listen = "0.0.0.0:9443"
}
return []ValidationResult{{"warn", "webui.listen",
fmt.Sprintf("web UI listens on %s with an unrestricted admin token and no restrictive CSM firewall policy for that address family; restrict access with a host firewall, the CSM firewall policy, or admin token allowed_ips", listen)}}
}
func webUIFirewallRestricts(cfg *Config, ipv6 bool) bool {
fw := cfg.Firewall
if fw == nil || !fw.Enabled || (ipv6 && !fw.IPv6) {
return false
}
port, known := webUIListenPort(cfg.WebUI.Listen)
if !known || port <= 0 || port > 65535 {
return false
}
// The passive range is accepted independently of restricted_tcp.
if fw.PassiveFTPStart > 0 && port >= fw.PassiveFTPStart && port <= fw.PassiveFTPEnd {
return false
}
if containsPort(fw.RestrictedTCP, port) {
return true
}
ports := fw.TCPIn
if ipv6 && len(fw.TCP6In) > 0 {
ports = fw.TCP6In
}
return !containsPort(ports, port)
}
func webUIListenFamilies(listen string) (v4, v6 bool) {
// Go's TCP listener can bind either wildcard as a dual-stack socket.
// Hostnames can resolve to either family; validation must not depend on
// a DNS answer that can change before the listener starts.
host, _, _ := net.SplitHostPort(listen)
if addr, err := netip.ParseAddr(host); err == nil {
addr = addr.Unmap()
if !addr.IsUnspecified() {
return addr.Is4(), addr.Is6()
}
}
return true, true
}
func webUIListensOnLoopback(listen string) bool {
host, _, err := net.SplitHostPort(listen)
if err != nil {
return false
}
if host == "localhost" {
return true
}
addr, err := netip.ParseAddr(host)
return err == nil && addr.IsLoopback()
}
// hasUnrestrictedAdminToken mirrors webUITokenCounts: a legacy auth_token is
// an admin token, and it cannot carry allowed_ips.
func hasUnrestrictedAdminToken(cfg *Config, v4, v6 bool) bool {
if !v4 && !v6 {
return false
}
if len(cfg.WebUI.Tokens) == 0 && cfg.WebUI.AuthToken != "" {
return true
}
for _, tok := range cfg.WebUI.Tokens {
if tok.Token == "" || tok.Scope != "admin" {
continue
}
if len(tok.AllowedIPs) == 0 {
return true
}
for _, src := range tok.AllowedIPs {
if prefix, err := parseTokenSource(src); err == nil && prefix.Bits() == 0 &&
((v4 && prefix.Addr().Is4()) || (v6 && prefix.Addr().Is6())) {
return true
}
}
}
return false
}
func blockEscalationResults(cfg *Config) []ValidationResult {
var results []ValidationResult
// Escalation counters below 2 describe no pattern: one address is not a
// subnet, and one temporary block is not a repeat offender. Omitted keys
// get defaults during Load, so lower values here were explicit.
if cfg.AutoResponse.NetBlock && cfg.AutoResponse.NetBlockThreshold < MinBlockEscalationCount {
results = append(results, ValidationResult{"error", "auto_response.netblock_threshold",
fmt.Sprintf("netblock_threshold must be at least %d when netblock is enabled; raise the threshold or disable netblock (got %d)", MinBlockEscalationCount, cfg.AutoResponse.NetBlockThreshold)})
}
if cfg.AutoResponse.PermBlock && cfg.AutoResponse.PermBlockCount < MinBlockEscalationCount {
results = append(results, ValidationResult{"error", "auto_response.permblock_count",
fmt.Sprintf("permblock_count must be at least %d when permblock is enabled; raise the count or disable permblock (got %d)", MinBlockEscalationCount, cfg.AutoResponse.PermBlockCount)})
}
return results
}
func hasEffectiveInfraIPs(cfg *Config) bool {
var firewallInfra []string
if cfg.Firewall != nil {
firewallInfra = cfg.Firewall.InfraIPs
}
return len(firewall.MergeInfraIPs(cfg.InfraIPs, firewallInfra)) > 0
}
// validateWarnings checks for non-fatal configuration issues.
func validateWarnings(cfg *Config) []ValidationResult {
var results []ValidationResult
for _, key := range cfg.RemovedKeys {
results = append(results, ValidationResult{"warn", key, "this setting was removed and is ignored; delete it from the config"})
}
if d, err := time.ParseDuration(cfg.Alerts.BlockDigest.Interval); err == nil && d > 0 && d < minBlockDigestInterval {
results = append(results, ValidationResult{"warn", "alerts.block_digest.interval", fmt.Sprintf("%s is below the 1m minimum and runs as 1m; raise it in the config", cfg.Alerts.BlockDigest.Interval)})
}
// GeoIP credentials set but auto_update explicitly false
if cfg.GeoIP.AccountID != "" && cfg.GeoIP.LicenseKey != "" {
if cfg.GeoIP.AutoUpdate != nil && !*cfg.GeoIP.AutoUpdate {
results = append(results, ValidationResult{"warn", "geoip", "GeoIP credentials configured but auto_update is disabled"})
}
}
// Auto-response enabled but no actions
if cfg.AutoResponse.Enabled {
if !cfg.AutoResponse.KillProcesses && !cfg.AutoResponse.QuarantineFiles && !cfg.AutoResponse.BlockIPs {
results = append(results, ValidationResult{"warn", "auto_response", "auto_response enabled but no actions configured (kill/quarantine/block all false)"})
}
}
// block_ips wants to mutate nftables, but the firewall engine that
// would apply those rules is disabled or absent. Without this check
// the daemon happily logs "auto-blocked" actions that never reach
// the kernel, and operators only notice when attackers keep coming
// back.
if cfg.AutoResponse.Enabled && cfg.AutoResponse.BlockIPs {
if cfg.Firewall == nil || !cfg.Firewall.Enabled {
results = append(results, ValidationResult{"warn", "auto_response.block_ips", "auto-response wants to block IPs but firewall is disabled; blocks will be no-ops"})
}
}
// The firewall trims blank entries while merging the two config sections.
// Validation must use that same effective list or placeholders can hide a
// lockout warning even though they create no kernel accept rule.
hasInfra := hasEffectiveInfraIPs(cfg)
if !hasInfra {
results = append(results, ValidationResult{"warn", "infra_ips", "no infra_ips configured in either top-level or firewall section"})
}
// Firewall enabled but no infra IPs (lockout risk)
if cfg.Firewall != nil && cfg.Firewall.Enabled && !hasInfra {
results = append(results, ValidationResult{"warn", "firewall", "firewall enabled but no infra_ips configured - risk of lockout"})
}
return results
}
// confdResults rejects integrity_exempt entries that can never match a
// fragment. The digest compares bare filenames, so a path, a glob or a file
// the loader would not merge leaves the operator believing a fragment is
// exempt while every rewrite of it still fails the next restart.
func confdResults(cfg *Config) []ValidationResult {
var results []ValidationResult
for _, entry := range cfg.ConfD.IntegrityExempt {
if err := validateExemptFragmentName(entry); err != nil {
results = append(results, ValidationResult{"error", "confd.integrity_exempt", err.Error()})
}
}
return results
}
func validateExemptFragmentName(name string) error {
if name == "" {
return errors.New("entries must be conf.d fragment filenames, got an empty entry")
}
if strings.ContainsAny(name, `/\*?[`) || name == "." || name == ".." {
return fmt.Errorf("%q is not a bare fragment filename; list the file name only, without directories or wildcards", name)
}
if !strings.HasSuffix(name, ".yaml") && !strings.HasSuffix(name, ".yml") {
return fmt.Errorf("%q is not a .yaml or .yml fragment, so conf.d never loads it", name)
}
return nil
}
// firewallValueResults rejects firewall values the engine would otherwise
// accept and quietly reinterpret: a port outside 1-65535 cannot select the
// intended service, an inverted passive-FTP range opens nothing, and a
// port_flood proto that is not "udp" is treated as TCP no matter what the
// operator typed.
func firewallValueResults(fw *firewall.FirewallConfig) []ValidationResult {
var results []ValidationResult
for _, list := range []struct {
field string
ports []int
}{
{"firewall.tcp_in", fw.TCPIn},
{"firewall.tcp_out", fw.TCPOut},
{"firewall.udp_in", fw.UDPIn},
{"firewall.udp_out", fw.UDPOut},
{"firewall.tcp6_in", fw.TCP6In},
{"firewall.tcp6_out", fw.TCP6Out},
{"firewall.required_tcp_out", fw.RequiredTCPOut},
{"firewall.udp6_in", fw.UDP6In},
{"firewall.udp6_out", fw.UDP6Out},
{"firewall.restricted_tcp", fw.RestrictedTCP},
{"firewall.drop_nolog", fw.DropNoLog},
{"firewall.smtp_ports", fw.SMTPPorts},
} {
for _, port := range list.ports {
if !validPort(port) {
results = append(results, ValidationResult{"error", list.field,
fmt.Sprintf("port %d is out of range (1-65535)", port)})
}
}
}
// The engine only builds the range rule when both ends are set, so a
// half-configured range is silently inert rather than wrong.
if fw.PassiveFTPStart != 0 || fw.PassiveFTPEnd != 0 {
if !validPort(fw.PassiveFTPStart) {
results = append(results, ValidationResult{"error", "firewall.passive_ftp_start",
fmt.Sprintf("port %d is out of range (1-65535)", fw.PassiveFTPStart)})
}
if !validPort(fw.PassiveFTPEnd) {
results = append(results, ValidationResult{"error", "firewall.passive_ftp_end",
fmt.Sprintf("port %d is out of range (1-65535)", fw.PassiveFTPEnd)})
}
if validPort(fw.PassiveFTPStart) && validPort(fw.PassiveFTPEnd) && fw.PassiveFTPStart > fw.PassiveFTPEnd {
results = append(results, ValidationResult{"error", "firewall.passive_ftp_start",
fmt.Sprintf("passive FTP range starts at %d but ends at %d", fw.PassiveFTPStart, fw.PassiveFTPEnd)})
}
}
for _, code := range fw.CountryBlock {
if !validCountryCode(code) {
results = append(results, ValidationResult{"error", "firewall.country_block",
fmt.Sprintf("%q is not a two-letter ISO country code", code)})
}
}
for i, pf := range fw.PortFlood {
switch {
case !validPort(pf.Port):
results = append(results, ValidationResult{"error", "firewall.port_flood",
fmt.Sprintf("entry %d: port %d is out of range (1-65535)", i, pf.Port)})
// The case handling is deliberately asymmetric. The engine selects UDP
// with an exact `Proto == "udp"` match and treats every other value as
// TCP, so "TCP" resolves to the protocol the operator meant and is
// safe to accept, while "UDP" would silently become TCP and must be
// rejected. Do not "tidy" this into a single case-insensitive compare.
case pf.Proto != "" && !strings.EqualFold(pf.Proto, "tcp") && pf.Proto != "udp":
results = append(results, ValidationResult{"error", "firewall.port_flood",
fmt.Sprintf("entry %d: proto %q must be \"tcp\" or lowercase \"udp\"", i, pf.Proto)})
case pf.Hits <= 0:
results = append(results, ValidationResult{"error", "firewall.port_flood",
fmt.Sprintf("entry %d: hits must be > 0, got %d", i, pf.Hits)})
case pf.Seconds <= 0:
results = append(results, ValidationResult{"error", "firewall.port_flood",
fmt.Sprintf("entry %d: seconds must be > 0, got %d", i, pf.Seconds)})
}
}
if fw.DenyTempIPLimitNegative {
results = append(results, ValidationResult{"warn", "firewall.deny_temp_ip_limit",
"a negative value is read as 0, which sets no limit on temporary blocks; set 0 or a positive limit"})
}
results = append(results, outAllowResults(fw)...)
results = append(results, protocolAllowResults(fw)...)
results = append(results, inAllowResults(fw)...)
return results
}
// inAllowResults validates firewall.tcp_in_allow. The engine renders nothing
// for an entry or source it cannot scope, and that silence reads as a port
// that just does not answer, so every shape error is reported.
func inAllowResults(fw *firewall.FirewallConfig) []ValidationResult {
var results []ValidationResult
for i, r := range fw.TCPInAllow {
if !validPort(r.Port) {
results = append(results, ValidationResult{"error", "firewall.tcp_in_allow",
fmt.Sprintf("entry %d: port %d is out of range (1-65535)", i, r.Port)})
}
prefix := fmt.Sprintf("entry %d (port %d)", i, r.Port)
if len(r.Sources) == 0 {
results = append(results, ValidationResult{"error", "firewall.tcp_in_allow",
fmt.Sprintf("%s: sources is required; list the addresses that may reach the port", prefix)})
}
for _, source := range r.Sources {
network, err := firewall.ParseOutAllowDst(source)
if err != nil {
results = append(results, ValidationResult{"error", "firewall.tcp_in_allow",
fmt.Sprintf("%s: source %q is not an IP address or CIDR", prefix, source)})
continue
}
if ones, _ := network.Mask.Size(); ones == 0 {
results = append(results, ValidationResult{"error", "firewall.tcp_in_allow",
fmt.Sprintf("%s: source %q matches every address; open the port in tcp_in instead", prefix, source)})
continue
}
if network.IP.To4() == nil && !fw.IPv6 {
results = append(results, ValidationResult{"warn", "firewall.tcp_in_allow",
fmt.Sprintf("%s: IPv6 source %q emits no rule while firewall.ipv6 is off", prefix, source)})
}
}
}
return results
}
// inAllowScoped reports whether tcp_in_allow opens port to at least one
// source the engine renders a rule for. Naming a port there is how an
// operator states who reaches the listener, so the lockout checks treat it
// as deliberate, like an infra-only port, and for both IP families: a
// warning would tell the operator to add the port to tcp_in, which undoes
// the scoping.
func inAllowScoped(fw *firewall.FirewallConfig, port int) bool {
for _, r := range fw.TCPInAllow {
if r.Port == port && len(r.Networks(fw.IPv6)) > 0 {
return true
}
}
return false
}
// protocolAllowResults validates firewall.protocol_allow. The engine renders
// nothing for an entry it cannot scope, and for VRRP that silence is both
// members claiming the shared address, so every shape error is reported.
func protocolAllowResults(fw *firewall.FirewallConfig) []ValidationResult {
var results []ValidationResult
names := firewall.ProtocolAllowNames()
for i, r := range fw.ProtocolAllow {
prefix := fmt.Sprintf("entry %d (protocol %q)", i, r.Protocol)
if !slices.Contains(names, r.Protocol) {
results = append(results, ValidationResult{"error", "firewall.protocol_allow",
fmt.Sprintf("%s: unknown protocol; supported: %s (tcp, udp and icmp have their own keys)", prefix, strings.Join(names, ", "))})
}
if len(r.Peers) == 0 {
results = append(results, ValidationResult{"error", "firewall.protocol_allow",
fmt.Sprintf("%s: peers is required; list the addresses of the other members", prefix)})
}
for _, peer := range r.Peers {
network, err := firewall.ParseOutAllowDst(peer)
if err != nil {
results = append(results, ValidationResult{"error", "firewall.protocol_allow",
fmt.Sprintf("%s: peer %q is not an IP address or CIDR", prefix, peer)})
continue
}
if ones, _ := network.Mask.Size(); ones == 0 {
results = append(results, ValidationResult{"error", "firewall.protocol_allow",
fmt.Sprintf("%s: peer %q matches every address; list the members", prefix, peer)})
}
}
}
return results
}
// outAllowResults validates firewall.tcp_out_allow. A malformed entry emits no
// nftables rule at all, so every shape error is reported rather than left to
// fail open as "the fetch just does not work".
func outAllowResults(fw *firewall.FirewallConfig) []ValidationResult {
var results []ValidationResult
for i, r := range fw.TCPOutAllow {
prefix := fmt.Sprintf("entry %d (dst %q)", i, r.Dst)
network, err := firewall.ParseOutAllowDst(r.Dst)
if err != nil {
results = append(results, ValidationResult{"error", "firewall.tcp_out_allow",
fmt.Sprintf("%s: %v", prefix, err)})
continue
}
switch {
case !validPort(r.PortStart):
results = append(results, ValidationResult{"error", "firewall.tcp_out_allow",
fmt.Sprintf("%s: port_start %d is out of range (1-65535)", prefix, r.PortStart)})
continue
case !validPort(r.PortEnd):
results = append(results, ValidationResult{"error", "firewall.tcp_out_allow",
fmt.Sprintf("%s: port_end %d is out of range (1-65535)", prefix, r.PortEnd)})
continue
case r.PortStart > r.PortEnd:
results = append(results, ValidationResult{"error", "firewall.tcp_out_allow",
fmt.Sprintf("%s: range starts at %d but ends at %d, so it matches nothing",
prefix, r.PortStart, r.PortEnd)})
continue
}
// The output chain emits the smtp_block drop before these rules, so an
// overlap cannot actually bypass it. Rejected anyway: rule ordering
// must not be the only guard between a config key and outbound mail.
if fw.SMTPBlock {
for _, port := range fw.SMTPPorts {
if port >= r.PortStart && port <= r.PortEnd {
results = append(results, ValidationResult{"error", "firewall.tcp_out_allow",
fmt.Sprintf("%s: range %d-%d covers smtp port %d while smtp_block is on",
prefix, r.PortStart, r.PortEnd, port)})
break
}
}
}
if ones, bits := network.Mask.Size(); ones == 0 && bits > 0 {
results = append(results, ValidationResult{"warn", "firewall.tcp_out_allow",
fmt.Sprintf("%s: opens ports %d-%d (%d ports) to every destination; scope it to the hosts this server dials",
prefix, r.PortStart, r.PortEnd, r.PortEnd-r.PortStart+1)})
}
if network.IP.To4() == nil && !fw.IPv6 {
results = append(results, ValidationResult{"warn", "firewall.tcp_out_allow",
fmt.Sprintf("%s: IPv6 destination emits no rule while firewall.ipv6 is off", prefix)})
}
}
return results
}
// centralActions mirrors the action constants in internal/reporting, which
// this package cannot import: reporting depends on alert, and alert depends
// on config. TestCentralActionsMatchReportingConstants (external test package,
// so it may import both) fails if the two lists ever drift.
var centralActions = [...]string{"off", "challenge", "block_if_local_corroborated"}
// ValidCentralActions returns every central-intelligence action accepted by
// config validation.
func ValidCentralActions() []string {
return append([]string(nil), centralActions[:]...)
}
// centralActionResults rejects an unrecognised central action. The consumer
// parses it with a default branch that resolves to "challenge", so a typo
// silently turns a corroborated-block policy into a challenge policy and the
// only trace is one daemon log line at startup.
func centralActionResults(cfg *Config) []ValidationResult {
action := cfg.Reputation.Central.Action
if action == "" {
return nil
}
for _, valid := range centralActions {
if action == valid {
return nil
}
}
return []ValidationResult{{"error", "reputation.central.action",
fmt.Sprintf("invalid action %q: must be one of %s", action, strings.Join(centralActions[:], ", "))}}
}
// centralAuthTargetResults checks the report target named to sign central
// pulls. The puller sends that target's node identity, so it must be an
// Ed25519 node target on the same origin as set_url: a signed request never
// goes to a host the node does not report to.
func centralAuthTargetResults(cfg *Config) []ValidationResult {
const field = "reputation.central.auth_target"
name := cfg.Reputation.Central.AuthTarget
if name == "" {
return nil
}
matches := 0
var target reportTargetFields
for _, t := range cfg.Reputation.Report.Targets {
if t.Name == name {
matches++
target = reportTargetFields{t.URL, t.Transport, t.NodeID, t.KeyID, t.KeyEnv}
}
}
var msg string
switch {
case matches != 1:
msg = fmt.Sprintf("%q must name exactly one reputation.report.targets entry, found %d", name, matches)
case target.transport != "ed25519":
msg = fmt.Sprintf("target %q must use transport ed25519", name)
case target.nodeID == "" || target.keyID == "" || target.keyEnv == "":
msg = fmt.Sprintf("target %q needs node_id, key_id and key_env", name)
case !sameOrigin(target.url, cfg.Reputation.Central.SetURL):
msg = fmt.Sprintf("target %q url and reputation.central.set_url must use the same scheme, host and port", name)
default:
return nil
}
return []ValidationResult{{"error", field, msg}}
}
type reportTargetFields struct {
url, transport, nodeID, keyID, keyEnv string
}
// sameOrigin reports whether two URLs share scheme, host and port, with the
// scheme's default port filled in.
func sameOrigin(a, b string) bool {
ua, errA := url.Parse(a)
ub, errB := url.Parse(b)
if errA != nil || errB != nil || !strings.EqualFold(ua.Scheme, ub.Scheme) {
return false
}
hostA, portA, okA := urlDialTarget(a)
hostB, portB, okB := urlDialTarget(b)
if !okA || !okB || portA != portB {
return false
}
// IPv6 spelling can vary, but zone identifiers name case-sensitive
// interfaces and must not inherit DNS hostname case folding.
addrA, ipErrA := netip.ParseAddr(hostA)
addrB, ipErrB := netip.ParseAddr(hostB)
if ipErrA == nil || ipErrB == nil {
return ipErrA == nil && ipErrB == nil && addrA == addrB
}
return strings.EqualFold(hostA, hostB)
}
// blockAtSeverityResults rejects an unrecognised incident block threshold.
// The correlator matches "high" and "critical" and ignores anything else, so
// a typo leaves the operator believing incident blocking is armed when the
// hand-off can never fire.
func blockAtSeverityResults(cfg *Config) []ValidationResult {
var results []ValidationResult
for _, entry := range []struct {
field string
value string
}{
{"incidents.spray_suppression.block_at_severity", cfg.Incidents.SpraySuppression.BlockAtSeverity},
{"incidents.auto_block.block_at_severity", cfg.Incidents.AutoBlock.BlockAtSeverity},
} {
if entry.value == "" {
continue
}
switch strings.ToLower(entry.value) {
case "high", "critical":
default:
results = append(results, ValidationResult{"error", entry.field,
fmt.Sprintf("invalid severity %q: must be \"high\" or \"critical\" (empty disables blocking)", entry.value)})
}
}
return results
}
func validPort(p int) bool { return p >= 1 && p <= 65535 }
func validCountryCode(code string) bool {
if len(code) != 2 {
return false
}
region, err := language.ParseRegion(code)
return err == nil && region.IsCountry()
}
// limitSummary renders a firewall limit for operator-facing output so a
// disabled protection reads as "disabled" instead of as a bare 0 that looks
// like a missing value.
func limitSummary(v int) string {
if v == 0 {
return "disabled"
}
return strconv.Itoa(v)
}
// firewallLockoutResults reports the ways an enabled firewall can cut the
// operator off from the host. The web UI runs the same checks before a save;
// running them here covers hand-edited csm.yaml, `csm doctor`, and daemon
// startup, which previously got no warning at all.
//
// These are warnings, never errors. Fronting the web UI with a reverse proxy
// or reaching it over a VPN are legitimate reasons to leave the port out of
// tcp_in, and validation must not refuse a deliberate configuration.
func firewallLockoutResults(cfg *Config) []ValidationResult {
fw := cfg.Firewall
if fw == nil || !fw.Enabled {
return nil
}
var results []ValidationResult
hasInfra := hasEffectiveInfraIPs(cfg)
port, portKnown := webUIListenPort(cfg.WebUI.Listen)
// Naming the port in restricted_tcp is how an operator asks for an
// infra-only service, and the infra accept rule matches before any port
// rule, so such a port stays reachable without appearing in tcp_in. The
// shipped defaults ship exactly this pair; warning about it would make
// every stock install report WARN. Without infra_ips there is nothing left
// to reach it from, so the warning still stands.
infraOnly := portKnown && hasInfra && containsPort(fw.RestrictedTCP, port)
// A port opened to chosen sources in tcp_in_allow is as deliberate.
infraOnly = infraOnly || (portKnown && inAllowScoped(fw, port))
if cfg.WebUI.Enabled && portKnown && !infraOnly && !containsPort(fw.TCPIn, port) {
results = append(results, ValidationResult{"warn", "firewall.tcp_in",
fmt.Sprintf("web UI listens on %d but tcp_in does not allow it; the next firewall apply drops new web UI connections", port)})
}
// A non-empty tcp6_in overrides tcp_in. When empty, IPv6 inherits tcp_in,
// so the IPv4 check above already covers the effective port policy.
if cfg.WebUI.Enabled && portKnown && !infraOnly && fw.IPv6 && len(fw.TCP6In) > 0 && !containsPort(fw.TCP6In, port) {
results = append(results, ValidationResult{"warn", "firewall.tcp6_in",
fmt.Sprintf("IPv6 is managed and tcp6_in does not allow web UI port %d", port)})
}
// restricted_tcp filters matching ports out of the public allow lists; it
// does not add reachability. Without infra_ips, a port in an effective
// allow list goes from public to unreachable, unless a source-scoped
// accept still opens it: those run ahead of the port rules.
tcp6In := fw.TCP6In
if len(tcp6In) == 0 {
tcp6In = fw.TCPIn
}
if !hasInfra {
var unreachable []int
for _, restricted := range fw.RestrictedTCP {
if inAllowScoped(fw, restricted) {
continue
}
if containsPort(fw.TCPIn, restricted) || (fw.IPv6 && containsPort(tcp6In, restricted)) {
unreachable = append(unreachable, restricted)
}
}
if len(unreachable) > 0 {
msg := "restricted_tcp filters ports out of the public allow list, but no infra_ips are configured, so those ports are reachable from nowhere"
if cfg.WebUI.Enabled && portKnown && containsPort(unreachable, port) {
msg = fmt.Sprintf("web UI port %d is in restricted_tcp but no infra_ips are configured, so the web UI is reachable from nowhere", port)
}
results = append(results, ValidationResult{"warn", "firewall.restricted_tcp", msg})
}
}
return results
}
// sshdConfigPath is where the SSH lockout guard reads the listen ports from.
// It is a test hook, not an operator setting: sshd's own path is fixed.
var sshdConfigPath = sshdconf.DefaultPath
// SetSSHDConfigPath points the SSH lockout guard at another sshd config and
// returns a func restoring the previous path. Tests use it to stay hermetic;
// production always reads sshd's own path.
func SetSSHDConfigPath(path string) func() {
previous := sshdConfigPath
sshdConfigPath = path
return func() { sshdConfigPath = previous }
}
// probeSSHLockout warns when an enabled firewall would not accept the ports
// sshd actually listens on. The shipped tcp_in leaves 22 out, so a host that
// never moved sshd loses SSH on the next apply.
//
// It reads the host, which is why it is a deep probe rather than part of
// Validate: the same csm.yaml must validate identically on any machine.
func probeSSHLockout(cfg *Config) []ValidationResult {
fw := cfg.Firewall
if fw == nil || !fw.Enabled {
return nil
}
// No sshd config means no evidence of a listener. Falling back to the
// compiled default here would warn on every host that does not run sshd.
sshd := sshdconf.Parse(sshdconf.OSFS{}, sshdConfigPath)
if !sshd.Present() {
return nil
}
hasInfra := hasEffectiveInfraIPs(cfg)
var missingTCPIn, missingTCP6In []int
ipv4Ports, ipv6Ports := sshd.RemoteListenPorts()
for _, port := range ipv4Ports {
// Naming the port in restricted_tcp is how an operator asks for an
// infra-only listener. The infra accept rule matches before any port
// rule, so such a port stays reachable without appearing in tcp_in.
if hasInfra && containsPort(fw.RestrictedTCP, port) {
continue
}
// A port opened to chosen sources in tcp_in_allow is as deliberate.
if inAllowScoped(fw, port) {
continue
}
if !containsPort(fw.TCPIn, port) {
missingTCPIn = append(missingTCPIn, port)
}
}
if fw.IPv6 {
for _, port := range ipv6Ports {
if (hasInfra && containsPort(fw.RestrictedTCP, port)) || inAllowScoped(fw, port) {
continue
}
if len(fw.TCP6In) == 0 {
if !containsPort(fw.TCPIn, port) && !containsPort(missingTCPIn, port) {
missingTCPIn = append(missingTCPIn, port)
}
} else if !containsPort(fw.TCP6In, port) {
missingTCP6In = append(missingTCP6In, port)
}
}
}
var results []ValidationResult
if len(missingTCPIn) > 0 {
subject, pronoun := portPhrase(missingTCPIn)
results = append(results, ValidationResult{"warn", "firewall.tcp_in",
fmt.Sprintf("sshd listens on %s but tcp_in does not allow %s; the next firewall apply drops new SSH connections", subject, pronoun)})
}
if len(missingTCP6In) > 0 {
subject, _ := portPhrase(missingTCP6In)
results = append(results, ValidationResult{"warn", "firewall.tcp6_in",
fmt.Sprintf("IPv6 is managed and tcp6_in does not allow sshd %s", subject)})
}
return results
}
// portPhrase renders a port list plus the pronoun that agrees with it.
func portPhrase(ports []int) (subject, pronoun string) {
sort.Ints(ports)
parts := make([]string, 0, len(ports))
for _, p := range ports {
parts = append(parts, strconv.Itoa(p))
}
if len(parts) == 1 {
return "port " + parts[0], "it"
}
return "ports " + strings.Join(parts, ", "), "them"
}
// webUIListenPort extracts the port from a host:port listen string, falling
// back to the shipped default when the field is unset.
func webUIListenPort(listen string) (int, bool) {
if listen == "" {
listen = "0.0.0.0:9443"
}
_, portStr, err := net.SplitHostPort(listen)
if err != nil {
return 0, false
}
port, err := strconv.Atoi(portStr)
if err != nil {
return 0, false
}
return port, true
}
func containsPort(ports []int, want int) bool {
for _, p := range ports {
if p == want {
return true
}
}
return false
}
// ValidateDeep performs connectivity probes against configured services.
// It does NOT call Validate(); the caller should invoke both separately.
func ValidateDeep(cfg *Config) []ValidationResult {
var results []ValidationResult
// State directory
results = append(results, probeStatePath(cfg.StatePath)...)
results = append(results, probeStatePathSandbox(cfg.StatePath)...)
// Signature rules directory
if cfg.Signatures.RulesDir != "" {
results = append(results, probeRulesDir(cfg.Signatures.RulesDir)...)
}
// SMTP
if cfg.Alerts.Email.Enabled && cfg.Alerts.Email.SMTP != "" {
results = append(results, probeSMTP(cfg.Alerts.Email.SMTP)...)
}
// ClamAV socket
if cfg.EmailAV.Enabled && cfg.EmailAV.ClamdSocket != "" {
results = append(results, probeClamd(cfg.EmailAV.ClamdSocket)...)
}
// TLS cert/key (only when custom paths set)
if cfg.WebUI.TLSCert != "" {
if _, err := os.Stat(cfg.WebUI.TLSCert); err != nil {
results = append(results, ValidationResult{"error", "webui.tls_cert", fmt.Sprintf("file not found: %s", cfg.WebUI.TLSCert)})
} else {
results = append(results, ValidationResult{"ok", "webui.tls_cert", cfg.WebUI.TLSCert})
}
}
if cfg.WebUI.TLSKey != "" {
if _, err := os.Stat(cfg.WebUI.TLSKey); err != nil {
results = append(results, ValidationResult{"error", "webui.tls_key", fmt.Sprintf("file not found: %s", cfg.WebUI.TLSKey)})
} else {
results = append(results, ValidationResult{"ok", "webui.tls_key", cfg.WebUI.TLSKey})
}
}
// Webhook
if cfg.Alerts.Webhook.Enabled && cfg.Alerts.Webhook.URL != "" {
results = append(results, probeWebhook(cfg.Alerts.Webhook.URL)...)
}
// GeoIP database files
if cfg.GeoIP.AccountID != "" && cfg.GeoIP.LicenseKey != "" && len(cfg.GeoIP.Editions) > 0 {
results = append(results, probeGeoIPDBs(cfg.StatePath, cfg.GeoIP.Editions)...)
}
// SSH reachability under the configured firewall policy
results = append(results, probeSSHLockout(cfg)...)
return results
}
// ValidateDeepSection runs only the deep probes relevant to the named
// section, so a save to section X does not fail on an unrelated probe
// for section Y. Section names match the webui settings schema IDs.
//
// For any section without deep probes, returns nil.
func ValidateDeepSection(cfg *Config, section string) []ValidationResult {
switch section {
case "alerts":
var results []ValidationResult
if cfg.Alerts.Email.Enabled && cfg.Alerts.Email.SMTP != "" {
results = append(results, probeSMTP(cfg.Alerts.Email.SMTP)...)
}
if cfg.Alerts.Webhook.Enabled && cfg.Alerts.Webhook.URL != "" {
results = append(results, probeWebhook(cfg.Alerts.Webhook.URL)...)
}
return results
case "email_av":
if cfg.EmailAV.Enabled && cfg.EmailAV.ClamdSocket != "" {
return probeClamd(cfg.EmailAV.ClamdSocket)
}
case "geoip":
if cfg.GeoIP.AccountID != "" && cfg.GeoIP.LicenseKey != "" && len(cfg.GeoIP.Editions) > 0 {
return probeGeoIPDBs(cfg.StatePath, cfg.GeoIP.Editions)
}
case "firewall":
return probeSSHLockout(cfg)
case "challenge":
// probeListenPortAvailable is not yet implemented in this codebase.
// When added, append it here. Settings save is where a new proxy
// entry is typed, so an unusable one is refused there.
return unusableTrustedProxyResults("error", "challenge.trusted_proxies", cfg.Challenge.TrustedProxies)
}
return nil
}
// systemdUnitFile is the installed service unit the sandbox probe reads.
var systemdUnitFile = "/etc/systemd/system/csm.service"
// probeStatePathSandbox checks that the daemon, not just this CLI process,
// can write state_path: under ProtectSystem=strict only StateDirectory and
// ReadWritePaths grants are writable, and a state_path outside them makes
// the daemon crash-loop on its first write while validate passes.
func probeStatePathSandbox(statePath string) []ValidationResult {
unit, err := readSystemdUnitAndDropIns(systemdUnitFile)
if err != nil {
return nil
}
covered, known := unitCoversStatePath(unit, statePath)
if !known || covered {
return nil
}
return []ValidationResult{{"error", "state_path", fmt.Sprintf("%s is not writable under the service unit's ProtectSystem and ReadWritePaths settings (unit: %s)", statePath, systemdUnitFile)}}
}
// #nosec G304 -- path is the installed service unit's package-constant path,
// and the drop-ins come from globbing that unit's own .d directory.
func readSystemdUnitAndDropIns(path string) (string, error) {
data, err := os.ReadFile(path)
if err != nil {
return "", err
}
var unit strings.Builder
unit.Write(data)
matches, err := filepath.Glob(filepath.Join(path+".d", "*.conf"))
if err != nil {
return "", err
}
for _, dropIn := range matches {
data, err = os.ReadFile(dropIn)
if err != nil {
return "", err
}
unit.WriteByte('\n')
unit.Write(data)
}
return unit.String(), nil
}
// unitCoversStatePath parses a systemd unit and reports whether statePath
// is writable to the service: covered is true when ProtectSystem is not
// strict or the path sits under a StateDirectory or ReadWritePaths grant;
// known is false when the unit text carries no sandbox directives at all.
func unitCoversStatePath(unit, statePath string) (covered, known bool) {
statePath = filepath.Clean(statePath)
protectSystem := "false"
var stateGrants, pathGrants []string
section := ""
for _, raw := range systemdLogicalLines(unit) {
line := strings.TrimSpace(raw)
if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
section = strings.ToLower(strings.TrimSpace(line[1 : len(line)-1]))
continue
}
if section != "service" {
continue
}
key, value, ok := strings.Cut(line, "=")
if !ok {
continue
}
key = strings.TrimSpace(key)
value = strings.TrimSpace(value)
switch key {
case "ProtectSystem":
known = true
protectSystem = strings.ToLower(strings.Trim(value, "\"'"))
case "StateDirectory":
known = true
if value == "" {
stateGrants = nil
continue
}
words, valid := splitSystemdWords(value)
if !valid {
continue
}
for _, name := range words {
name, _, _ = strings.Cut(name, ":")
if name != "" {
stateGrants = append(stateGrants, filepath.Join("/var/lib", name))
}
}
case "ReadWritePaths":
known = true
if value == "" {
pathGrants = nil
continue
}
words, valid := splitSystemdWords(value)
if !valid {
continue
}
for _, p := range words {
p = strings.TrimLeft(p, "-+!")
if p != "" {
pathGrants = append(pathGrants, filepath.Clean(p))
}
}
}
}
if !known {
return false, false
}
for _, g := range append(stateGrants, pathGrants...) {
if pathContains(g, statePath) {
return true, true
}
}
switch protectSystem {
case "strict":
return false, true
case "full":
return !pathUnderAny(statePath, "/usr", "/boot", "/efi", "/etc"), true
case "true", "yes":
return !pathUnderAny(statePath, "/usr", "/boot", "/efi"), true
default:
return true, true
}
}
func systemdLogicalLines(unit string) []string {
var lines []string
var logical strings.Builder
for _, raw := range strings.Split(unit, "\n") {
raw = strings.TrimSuffix(raw, "\r")
trailingSlashes := 0
for i := len(raw) - 1; i >= 0 && raw[i] == '\\'; i-- {
trailingSlashes++
}
continued := trailingSlashes%2 == 1
if continued {
raw = raw[:len(raw)-1]
}
logical.WriteString(raw)
if continued {
logical.WriteByte(' ')
continue
}
lines = append(lines, logical.String())
logical.Reset()
}
if logical.Len() > 0 {
lines = append(lines, logical.String())
}
return lines
}
func splitSystemdWords(value string) ([]string, bool) {
var words []string
var word strings.Builder
var quote byte
escaped := false
flush := func() {
if word.Len() > 0 {
words = append(words, word.String())
word.Reset()
}
}
for i := 0; i < len(value); i++ {
c := value[i]
if escaped {
word.WriteByte(c)
escaped = false
continue
}
if c == '\\' {
escaped = true
continue
}
if quote != 0 {
if c == quote {
quote = 0
} else {
word.WriteByte(c)
}
continue
}
if c == '\'' || c == '"' {
quote = c
continue
}
if c == ' ' || c == '\t' {
flush()
continue
}
word.WriteByte(c)
}
if escaped {
word.WriteByte('\\')
}
if quote != 0 {
return nil, false
}
flush()
return words, true
}
func pathContains(parent, child string) bool {
return safepath.Within(child, parent)
}
func pathUnderAny(path string, roots ...string) bool {
for _, root := range roots {
if pathContains(root, path) {
return true
}
}
return false
}
// probeStatePath checks that the state directory exists and is writable.
func probeStatePath(path string) []ValidationResult {
info, err := os.Stat(path)
if err != nil {
return []ValidationResult{{"error", "state_path", fmt.Sprintf("directory not found: %s", path)}}
}
if !info.IsDir() {
return []ValidationResult{{"error", "state_path", fmt.Sprintf("not a directory: %s", path)}}
}
probe := filepath.Join(path, ".csm-validate-probe")
// #nosec G304 -- filepath.Join under operator-configured statePath.
f, err := os.Create(probe)
if err != nil {
return []ValidationResult{{"error", "state_path", fmt.Sprintf("directory not writable: %s", path)}}
}
f.Close()
_ = os.Remove(probe)
return []ValidationResult{{"ok", "state_path", path}}
}
// probeRulesDir checks that the rules directory exists and contains rule files.
func probeRulesDir(path string) []ValidationResult {
info, err := os.Stat(path)
if err != nil {
return []ValidationResult{{"error", "signatures.rules_dir", fmt.Sprintf("directory not found: %s", path)}}
}
if !info.IsDir() {
return []ValidationResult{{"error", "signatures.rules_dir", fmt.Sprintf("not a directory: %s", path)}}
}
// Check for rule files
for _, pattern := range []string{"*.yaml", "*.yml", "*.yar", "*.yara"} {
matches, _ := filepath.Glob(filepath.Join(path, pattern))
if len(matches) > 0 {
return []ValidationResult{{"ok", "signatures.rules_dir", fmt.Sprintf("%s (%d rule files)", path, len(matches))}}
}
}
return []ValidationResult{{"error", "signatures.rules_dir", fmt.Sprintf("no rule files (.yaml/.yml/.yar/.yara) found in %s", path)}}
}
// probeSMTP attempts a TCP dial to the SMTP server.
func probeSMTP(addr string) []ValidationResult {
conn, err := net.DialTimeout("tcp", addr, 3*time.Second)
if err != nil {
return []ValidationResult{{"error", "alerts.email.smtp", fmt.Sprintf("cannot connect to %s: %v", addr, err)}}
}
_ = conn.Close()
return []ValidationResult{{"ok", "alerts.email.smtp", fmt.Sprintf("connected to %s", addr)}}
}
// probeClamd attempts to connect to the ClamAV unix socket, falling back to the
// well-known locations so a host whose setting names the wrong path is told
// which one actually answers instead of only that mail is not being scanned.
func probeClamd(socket string) []ValidationResult {
resolved, discovered := ResolveClamdSocket(socket)
if !discovered && socket != "" && !clamdSocketTrusted(socket) {
return []ValidationResult{{"error", "email_av.clamd_socket", fmt.Sprintf(
"%s is in a directory other accounts can write to, so what answers there is not necessarily clamd; move the socket somewhere only root or the clamd service account can write",
socket)}}
}
conn, err := net.DialTimeout("unix", resolved, 3*time.Second)
if err != nil {
return []ValidationResult{{"error", "email_av.clamd_socket", fmt.Sprintf("cannot connect to %s: %v", resolved, err)}}
}
_ = conn.Close()
if discovered {
return []ValidationResult{{"warn", "email_av.clamd_socket", fmt.Sprintf(
"nothing is listening on the configured %s; using %s, which is answering. Set clamd_socket to it so the fallback is not needed",
socket, resolved)}}
}
return []ValidationResult{{"ok", "email_av.clamd_socket", fmt.Sprintf("connected to %s", resolved)}}
}
// probeWebhook performs an HTTP HEAD request to verify the webhook endpoint is reachable.
// DNS/TCP/TLS failures are errors; HTTP status codes (even 401/403/404/405) mean reachable.
func probeWebhook(rawURL string) []ValidationResult {
client := &http.Client{Timeout: 5 * time.Second}
resp, err := client.Head(rawURL)
if err != nil {
var urlErr *url.Error
if errors.As(err, &urlErr) {
err = urlErr.Err
}
return []ValidationResult{{"error", "alerts.webhook.url", fmt.Sprintf("cannot reach %s: %v", RedactURL(rawURL), err)}}
}
resp.Body.Close()
return []ValidationResult{{"ok", "alerts.webhook.url", fmt.Sprintf("reachable (HTTP %d)", resp.StatusCode)}}
}
// validateBrowserOrigin checks that raw has the shape of a browser Origin
// header value the web UI can match: https://host[:port] and nothing else.
func validateBrowserOrigin(raw string) error {
u, err := url.Parse(strings.TrimSpace(raw))
if err != nil {
return fmt.Errorf("parse: %w", err)
}
if !strings.EqualFold(u.Scheme, "https") {
return fmt.Errorf("must be an https origin")
}
if u.Hostname() == "" {
return fmt.Errorf("missing host")
}
if u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" {
return fmt.Errorf("must be a bare https://host[:port] origin")
}
return nil
}
func validateSignatureURL(raw string, allowTemplates bool) error {
candidate := strings.TrimSpace(raw)
if allowTemplates {
candidate = sampleSignatureURLTemplate(candidate)
}
u, err := url.Parse(candidate)
if err != nil {
return fmt.Errorf("parse: %w", err)
}
// Signed content and the rollback guard bound what a tampered download
// can do, but a plain-http mirror still hands an on-path attacker every
// old signed release to replay and every request to observe.
if strings.ToLower(u.Scheme) != "https" {
return fmt.Errorf("signatures URL must be an https URL")
}
host := u.Hostname()
if host == "" {
return fmt.Errorf("missing host in %q", raw)
}
return validateSignaturesHost(host)
}
func sampleSignatureURLTemplate(raw string) string {
raw = strings.ReplaceAll(raw, "{tier}", "core")
raw = strings.ReplaceAll(raw, "{version}", "v1.0.0")
return raw
}
// validateSignaturesHost rejects URL hosts that point at loopback,
// link-local, or RFC1918 / ULA ranges. Scoped IPv6 literals are valid URL
// hosts, so use netip instead of net.ParseIP to keep the zone intact.
func validateSignaturesHost(host string) error {
lower := strings.TrimSuffix(strings.ToLower(host), ".")
if lower == "localhost" || lower == "localhost.localdomain" {
return fmt.Errorf("signatures URL host %q is loopback; refuse for production downloads", host)
}
if addr, err := netip.ParseAddr(lower); err == nil {
addr = addr.Unmap()
if addr.IsLoopback() {
return fmt.Errorf("signatures URL host %s is loopback", host)
}
if addr.IsPrivate() {
return fmt.Errorf("signatures URL host %s is RFC1918 / ULA private", host)
}
if addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() {
return fmt.Errorf("signatures URL host %s is link-local", host)
}
if addr.IsUnspecified() {
return fmt.Errorf("signatures URL host %s is unspecified", host)
}
}
return nil
}
// probeGeoIPDBs checks that expected GeoIP database files exist on disk.
func probeGeoIPDBs(statePath string, editions []string) []ValidationResult {
var results []ValidationResult
allOK := true
for _, edition := range editions {
dbPath := filepath.Join(statePath, "geoip", edition+".mmdb")
if _, err := os.Stat(dbPath); err != nil {
results = append(results, ValidationResult{"error", "geoip", fmt.Sprintf("database not found: %s", dbPath)})
allOK = false
}
}
if allOK {
results = append(results, ValidationResult{"ok", "geoip", fmt.Sprintf("all %d edition databases present", len(editions))})
}
return results
}
// validateDOSExemptRanges checks each entry in the dos_exempt_ranges list.
// Per entry: whitespace is trimmed; empty entries are rejected; CIDR notation
// is parsed via net.ParseCIDR and default routes (/0) are rejected; bare IP
// addresses are accepted as /32 or /128 equivalents; anything else is an error.
// Each error string is prefixed with "firewall.dos_exempt_ranges[<i>]:".
func validateDOSExemptRanges(entries []string) []string {
var errs []string
for i, raw := range entries {
entry := strings.TrimSpace(raw)
if entry == "" {
errs = append(errs, fmt.Sprintf("firewall.dos_exempt_ranges[%d]: empty entry", i))
continue
}
if _, ipnet, err := net.ParseCIDR(entry); err == nil {
ones, _ := ipnet.Mask.Size()
if ones == 0 {
errs = append(errs, fmt.Sprintf("firewall.dos_exempt_ranges[%d]: %s is a default route and cannot be used as an exempt range", i, entry))
}
continue
}
if net.ParseIP(entry) != nil {
continue
}
errs = append(errs, fmt.Sprintf("firewall.dos_exempt_ranges[%d]: %q is not a valid CIDR or IP address", i, entry))
}
return errs
}
// validateFirewallConfig validates firewall fields that are checked at load
// time to prevent invalid configuration from reaching the daemon. It returns
// the first multi-error joined string so LoadBytes can return a single error.
func validateFirewallConfig(cfg *Config) error {
if cfg.Firewall == nil {
return nil
}
errs := validateDOSExemptRanges(cfg.Firewall.DOSExemptRanges)
if _, err := firewall.ParseLocalPortLocks(cfg.Firewall.LocalPortLocks); err != nil {
errs = append(errs, "firewall.local_port_locks: "+err.Error())
}
if len(errs) == 0 {
return nil
}
return errors.New(strings.Join(errs, "; "))
}
// geoIPCityDatabasePresent reports whether the City database the daemon loads
// exists. The daemon opens only the state-path copy, so that is the location
// that decides whether a country lookup can resolve anything.
func geoIPCityDatabasePresent(statePath string) bool {
if statePath == "" {
return false
}
info, err := os.Stat(filepath.Join(statePath, "geoip", "GeoLite2-City.mmdb"))
return err == nil && info.Mode().IsRegular() && info.Size() > 0
}
package config
import (
"fmt"
"strings"
"golang.org/x/net/publicsuffix"
"github.com/pidginhost/csm/internal/netutil"
)
// VerifiedBot is one operator-configured good bot. A request whose UA contains
// any UASubstrings is treated as claiming this identity, and is trusted only
// if the source IP matches one configured verification method.
type VerifiedBot struct {
Name string `yaml:"name" json:"name"`
UASubstrings []string `yaml:"ua_substrings,omitempty" json:"ua_substrings,omitempty"`
RDNSSuffixes []string `yaml:"rdns_suffixes,omitempty" json:"rdns_suffixes,omitempty"`
// IPRanges are published CIDRs (or single IPs) for bots that verify by
// address rather than reverse DNS -- typically AI agents (PerplexityBot,
// GPTBot, ClaudeBot). Membership is checked synchronously; no rDNS lookup.
IPRanges []string `yaml:"ip_ranges,omitempty" json:"ip_ranges,omitempty"`
}
// verifiedBotMinUALen rejects UA substrings short enough to match unrelated
// traffic ("bot", "go"). Real crawler tokens are longer.
const verifiedBotMinUALen = 4
// browserUATokens are substrings that appear in ordinary browser UAs. Entries
// keyed only on these tokens would match real users, so validation rejects
// them unless the substring also carries a crawler-specific token.
var browserUATokens = map[string]bool{
"mozilla": true, "applewebkit": true, "webkit": true, "gecko": true,
"chrome": true, "safari": true, "firefox": true, "edge": true,
"opera": true, "msie": true, "trident": true, "windows": true,
"macintosh": true, "linux": true, "android": true, "iphone": true,
"ipad": true, "x11": true, "mobile": true,
}
var crawlerUATokens = []string{
"bot", "crawler", "spider", "externalhit", "inspectiontool", "lighthouse",
}
// sharedHostingSuffixes are domains where reverse DNS is assigned to whoever
// rents the address, so an attacker can obtain a PTR (and forward A) under
// them. Allowlisting such a suffix would let any tenant spoof a crawler, so
// they are rejected. Matched as a suffix so subdomains are caught too.
var sharedHostingSuffixes = []string{
"amazonaws.com", "googleusercontent.com", "appspot.com", "run.app",
"cloudfront.net", "azurewebsites.net", "cloudapp.azure.com",
"herokuapp.com", "workers.dev", "pages.dev", "netlify.app",
"vercel.app", "ondigitalocean.app", "digitaloceanspaces.com",
"github.io", "gitlab.io", "fastly.net", "akamaitechnologies.com",
"akamai.net", "cloudflare.net", "colocrossing.com", "contabo.net",
"hetzner.com", "your-server.de", "ovh.net", "ip-linodeusercontent.com",
"linode.com", "vultrusercontent.com",
}
// commonPublicSuffixes are multi-label public suffixes that are not
// registrable on their own; a crawler can never legitimately be the whole
// suffix. Bare single-label TLDs are caught by the label-count check.
var commonPublicSuffixes = map[string]bool{
"co.uk": true, "org.uk": true, "gov.uk": true, "ac.uk": true,
"com.au": true, "net.au": true, "org.au": true, "co.jp": true,
"com.br": true, "com.cn": true, "co.in": true, "co.za": true,
"com.tr": true, "com.mx": true, "co.kr": true, "com.sg": true,
}
func validateVerifiedBots(cfg *Config) []ValidationResult {
var results []ValidationResult
seen := map[string]bool{}
seenUA := map[string]string{}
for i, b := range cfg.Reputation.VerifiedBots {
field := fmt.Sprintf("reputation.verified_bots[%d]", i)
name := strings.ToLower(strings.TrimSpace(b.Name))
if name == "" {
results = append(results, ValidationResult{"error", field + ".name", "verified bot name is required"})
continue
}
if seen[name] {
results = append(results, ValidationResult{"error", field + ".name",
fmt.Sprintf("duplicate verified bot name %q", name)})
}
seen[name] = true
hasUA := false
for _, raw := range b.UASubstrings {
s := strings.ToLower(strings.TrimSpace(raw))
if s == "" {
continue
}
hasUA = true
if len(s) < verifiedBotMinUALen {
results = append(results, ValidationResult{"error", field + ".ua_substrings",
fmt.Sprintf("UA substring %q is too short (min %d chars)", s, verifiedBotMinUALen)})
}
if browserUASubstringFootgun(s) {
results = append(results, ValidationResult{"error", field + ".ua_substrings",
fmt.Sprintf("UA substring %q matches ordinary browsers and would allowlist real users", s)})
}
if prev, ok := seenUA[s]; ok && prev != name {
results = append(results, ValidationResult{"error", field + ".ua_substrings",
fmt.Sprintf("UA substring %q is already used by verified bot %q", s, prev)})
}
for prev, prevName := range seenUA {
if prevName == name || prev == s {
continue
}
if strings.Contains(prev, s) || strings.Contains(s, prev) {
results = append(results, ValidationResult{"error", field + ".ua_substrings",
fmt.Sprintf("UA substring %q overlaps verified bot %q substring %q", s, prevName, prev)})
break
}
}
if _, ok := seenUA[s]; !ok {
seenUA[s] = name
}
}
if !hasUA {
results = append(results, ValidationResult{"error", field + ".ua_substrings",
"at least one ua_substring is required"})
}
hasSuffix := false
for _, raw := range b.RDNSSuffixes {
s := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(raw)), ".")
if s == "" {
continue
}
hasSuffix = true
if msg := verifiedBotSuffixError(s); msg != "" {
results = append(results, ValidationResult{"error", field + ".rdns_suffixes", msg})
}
}
hasRange := false
for _, raw := range b.IPRanges {
s := strings.TrimSpace(raw)
if s == "" {
continue
}
hasRange = true
if msg := verifiedBotIPRangeError(s); msg != "" {
results = append(results, ValidationResult{"error", field + ".ip_ranges", msg})
}
}
// A bot needs at least one way to be confirmed: an rDNS suffix (for
// crawlers with forward-confirmable reverse DNS) or an IP range (for
// AI agents that publish address ranges instead of rDNS).
if !hasSuffix && !hasRange {
results = append(results, ValidationResult{"error", field,
"at least one rdns_suffix or ip_range is required"})
}
}
return results
}
// verifiedBotIPRangeError validates an operator-supplied CIDR or single IP.
// It rejects ranges too broad to be a crawler fleet and non-public space, so
// the allowlist cannot be turned into a blanket detection bypass.
func verifiedBotIPRangeError(s string) string {
n := netutil.ParseCIDROrIP(s)
if n == nil {
return fmt.Sprintf("ip_range %q is not a valid CIDR or IP", s)
}
ones, bits := n.Mask.Size()
if bits == 32 && ones < 16 {
return fmt.Sprintf("ip_range %q is too broad (minimum prefix /16 for IPv4)", s)
}
if bits == 128 && ones < 32 {
return fmt.Sprintf("ip_range %q is too broad (minimum prefix /32 for IPv6)", s)
}
if !netutil.IsPublicIP(n.IP) {
return fmt.Sprintf("ip_range %q is not a public address range", s)
}
return ""
}
func browserUASubstringFootgun(s string) bool {
if !containsKnownUAToken(s, browserUATokens) {
return false
}
for _, token := range crawlerUATokens {
if strings.Contains(s, token) {
return false
}
}
return true
}
func containsKnownUAToken(s string, tokens map[string]bool) bool {
for token := range tokens {
for start := 0; start < len(s); {
idx := strings.Index(s[start:], token)
if idx == -1 {
break
}
idx += start
end := idx + len(token)
if uaTokenBoundary(s, idx-1) && uaTokenBoundary(s, end) {
return true
}
start = idx + 1
}
}
return false
}
func uaTokenBoundary(s string, idx int) bool {
if idx < 0 || idx >= len(s) {
return true
}
c := s[idx]
if c >= 'a' && c <= 'z' {
return false
}
if c >= '0' && c <= '9' {
return false
}
return true
}
func validateVerifiedBotsConfig(cfg *Config) error {
for _, r := range validateVerifiedBots(cfg) {
if r.Level == "error" {
return fmt.Errorf("%s: %s", r.Field, r.Message)
}
}
return nil
}
func verifiedBotSuffixError(s string) string {
if strings.ContainsAny(s, " /:") {
return fmt.Sprintf("rdns_suffix %q is not a domain", s)
}
if len(s) > 253 {
return fmt.Sprintf("rdns_suffix %q is too long", s)
}
labels := strings.Split(s, ".")
if len(labels) < 2 {
return fmt.Sprintf("rdns_suffix %q must be a registrable domain (e.g. seranking.com), not a bare TLD", s)
}
for _, l := range labels {
if l == "" {
return fmt.Sprintf("rdns_suffix %q has an empty label", s)
}
if len(l) > 63 {
return fmt.Sprintf("rdns_suffix %q has a label longer than 63 characters", s)
}
if strings.HasPrefix(l, "-") || strings.HasSuffix(l, "-") {
return fmt.Sprintf("rdns_suffix %q has a label starting or ending with hyphen", s)
}
for _, r := range l {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' {
continue
}
return fmt.Sprintf("rdns_suffix %q has invalid domain characters", s)
}
}
if commonPublicSuffixes[s] {
return fmt.Sprintf("rdns_suffix %q is a public suffix, not a registrable domain", s)
}
if _, err := publicsuffix.EffectiveTLDPlusOne(s); err != nil {
return fmt.Sprintf("rdns_suffix %q is a public suffix, not a registrable domain", s)
}
for _, bad := range sharedHostingSuffixes {
if s == bad || strings.HasSuffix(s, "."+bad) {
return fmt.Sprintf("rdns_suffix %q is shared hosting where reverse DNS is attacker-controlled; not allowed", s)
}
}
return ""
}
package config
import (
"fmt"
"time"
)
const DefaultBrowserSessionLifetime = "24h"
const DefaultBrowserSessionIdleTimeout = "30m"
// MaxBrowserSessionLifetime bounds webui.session_lifetime.
const MaxBrowserSessionLifetime = 30 * 24 * time.Hour
// BrowserSessionDurations also accepts an un-defaulted configuration, as used
// by embedded callers. Explicit zero durations never disable session expiry.
func (c *Config) BrowserSessionDurations() (time.Duration, time.Duration, error) {
lifetimeText, idleText := c.WebUI.SessionLifetime, c.WebUI.SessionIdleTimeout
if lifetimeText == "" {
lifetimeText = DefaultBrowserSessionLifetime
}
if idleText == "" {
idleText = DefaultBrowserSessionIdleTimeout
}
lifetime, err := time.ParseDuration(lifetimeText)
if err != nil || lifetime < time.Second || lifetime > MaxBrowserSessionLifetime {
return 0, 0, fmt.Errorf("webui.session_lifetime must be between 1s and 720h")
}
idle, err := time.ParseDuration(idleText)
if err != nil || idle < time.Second || idle > lifetime {
return 0, 0, fmt.Errorf("webui.session_idle_timeout must be between 1s and session_lifetime")
}
return lifetime, idle, nil
}
package config
import (
"fmt"
"net/netip"
"strings"
"time"
)
// WebUITokenCurrent reports whether a credential is usable at now. It expires
// only at its expires_at; issued_at records when it was generated. Credentials
// without dates never expire, as in configurations written before dates
// existed. Malformed or future dates fail closed.
func WebUITokenCurrent(issuedAt, expiresAt string, now time.Time) bool {
field, _ := webUITokenProblem(issuedAt, expiresAt, now)
return field == ""
}
// webUITokenProblem names the date field that disables a credential and why,
// or returns an empty field for a current credential.
func webUITokenProblem(issuedAt, expiresAt string, now time.Time) (field, problem string) {
var issued time.Time
if issuedAt != "" {
var err error
issued, err = time.Parse(time.RFC3339, issuedAt)
if err != nil || issued.IsZero() || now.Before(issued) {
return "issued_at", "issued_at must be a valid RFC3339 issue time that is not in the future"
}
}
if expiresAt == "" {
return "", ""
}
deadline, err := time.Parse(time.RFC3339, expiresAt)
if err != nil || !deadline.After(issued) {
return "expires_at", "expires_at must be a valid RFC3339 time after issued_at"
}
if !now.Before(deadline) {
return "expires_at", "expired at " + deadline.UTC().Format(time.RFC3339)
}
return "", ""
}
func tokenExpiryWarnings(cfg *Config, now time.Time) []ValidationResult {
var out []ValidationResult
check := func(token, issuedAt, expiresAt, prefix string) {
if token == "" {
return
}
if field, problem := webUITokenProblem(issuedAt, expiresAt, now); field != "" {
out = append(out, ValidationResult{"warn", prefix + field,
"credential disabled: " + problem + "; correct the date or replace the token"})
}
}
for i, tok := range cfg.WebUI.Tokens {
check(tok.Token, tok.IssuedAt, tok.ExpiresAt, fmt.Sprintf("webui.tokens[%d].", i))
}
if len(cfg.WebUI.Tokens) == 0 {
check(cfg.WebUI.AuthToken, cfg.WebUI.AuthTokenIssuedAt, cfg.WebUI.AuthTokenExpiresAt, "webui.auth_token_")
}
check(cfg.WebUI.MetricsToken, cfg.WebUI.MetricsTokenIssuedAt, cfg.WebUI.MetricsTokenExpiresAt, "webui.metrics_token_")
return out
}
// parseTokenSource reads one allowed_ips entry: an address or a CIDR.
func parseTokenSource(src string) (netip.Prefix, error) {
if strings.Contains(src, "/") {
p, err := netip.ParsePrefix(src)
if err != nil {
return netip.Prefix{}, err
}
p = p.Masked()
// Peers are unmapped before matching. Keep mapped networks in
// the same address family, including their effective mask width.
if p.Addr().Is4In6() {
p = netip.PrefixFrom(p.Addr().Unmap(), p.Bits()-96)
}
return p, nil
}
addr, err := netip.ParseAddr(src)
if err != nil {
return netip.Prefix{}, err
}
// PrefixFrom drops zones, which would silently widen an entry meant
// to restrict a credential to one interface.
if addr.Zone() != "" {
return netip.Prefix{}, fmt.Errorf("token source addresses must not include a zone")
}
addr = addr.Unmap()
return netip.PrefixFrom(addr, addr.BitLen()), nil
}
// AcceptsFrom reports whether the token may be used from addr. A token
// without AllowedIPs is accepted from any address; a restricted token never
// accepts an invalid address.
func (t WebUIToken) AcceptsFrom(addr netip.Addr) bool {
if len(t.AllowedIPs) == 0 {
return true
}
addr = addr.WithZone("").Unmap()
for _, src := range t.AllowedIPs {
if p, err := parseTokenSource(src); err == nil && p.Contains(addr) {
return true
}
}
return false
}
package config
import (
"bytes"
"fmt"
"reflect"
"sort"
"strings"
"gopkg.in/yaml.v3"
)
// YAMLChange describes a single scalar, list, or map replacement to apply
// to a YAML document. Path is the dotted YAML key path from the document
// root. Value is the Go value to serialise; nil means YAML null.
type YAMLChange struct {
Path []string
Value interface{}
}
// lineIndex maps 1-based line numbers to byte offsets of their first byte.
// lineIndex[i] is the byte offset of line i+1 (i.e. lineIndex[0] == 0 == start of line 1).
type lineIndex []int
func buildLineIndex(data []byte) lineIndex {
idx := lineIndex{0} // line 1 starts at offset 0
for i, b := range data {
if b == '\n' {
idx = append(idx, i+1) // line starts after the newline
}
}
return idx
}
// offset returns the byte offset for a 1-based line and 1-based column.
func (li lineIndex) offset(line, col int) int {
if line < 1 || line > len(li) {
// line beyond end of file -- treat as EOF
if len(li) == 0 {
return 0
}
return li[len(li)-1]
}
return li[line-1] + col - 1
}
// lineEnd returns the byte offset of the '\n' at the end of the given 1-based line,
// or the length of data if the last line has no trailing newline.
func (li lineIndex) lineEnd(line int, data []byte) int {
if line < len(li) {
// next line starts at li[line]; the '\n' is at li[line]-1
return li[line] - 1
}
// last line
return len(data)
}
// lineStart returns the byte offset of the start of 1-based line.
func (li lineIndex) lineStart(line int) int {
if line < 1 {
return 0
}
if line > len(li) {
return li[len(li)-1]
}
return li[line-1]
}
// maxLine returns the maximum Line value in the node subtree.
func maxLine(n *yaml.Node) int {
m := n.Line
for _, child := range n.Content {
if cl := maxLine(child); cl > m {
m = cl
}
}
return m
}
// findNode walks the yaml.Node document tree and returns the key node and value node
// for the last segment of path, plus the parent mapping node.
// Returns (keyNode, valueNode, parentMapping, error).
func findNode(root *yaml.Node, path []string) (keyN, valN, parentMap *yaml.Node, err error) {
// root should be a DocumentNode; its Content[0] is the real root mapping.
cur := root
if cur.Kind == yaml.DocumentNode {
if len(cur.Content) == 0 {
return nil, nil, nil, fmt.Errorf("empty document")
}
cur = cur.Content[0]
}
for i, seg := range path {
if cur.Kind != yaml.MappingNode {
return nil, nil, nil, fmt.Errorf("path %v: segment %q: expected mapping, got kind %d", path[:i+1], seg, cur.Kind)
}
found := false
for j := 0; j+1 < len(cur.Content); j += 2 {
k := cur.Content[j]
v := cur.Content[j+1]
if k.Value == seg {
if i == len(path)-1 {
return k, v, cur, nil
}
cur = v
found = true
break
}
}
if !found {
// segment not found; return the parent mapping so caller can insert
if i == len(path)-1 {
return nil, nil, cur, nil // key missing, parent is cur
}
return nil, nil, nil, fmt.Errorf("path %v: segment %q not found", path[:i+1], seg)
}
}
return nil, nil, nil, fmt.Errorf("empty path")
}
// renderValueInline renders a scalar value to its YAML inline form.
// nil -> "null", bool -> "true"/"false", numbers via fmt, strings quoted if needed.
func renderValueInline(v interface{}) (string, error) {
if v == nil {
return "null", nil
}
// Use yaml.Marshal on a single-value map to get the marshalled scalar,
// then extract just the value part.
type wrapper struct {
V interface{} `yaml:"v"`
}
b, err := yaml.Marshal(wrapper{V: v})
if err != nil {
return "", err
}
// b looks like "v: VALUE\n"
s := strings.TrimPrefix(string(b), "v: ")
s = strings.TrimSuffix(s, "\n")
if strings.ContainsAny(s, "\n\r") {
return "", fmt.Errorf("value of type %T cannot be rendered inline", v)
}
return s, nil
}
// renderKeyInline renders a mapping key string in a YAML-safe form.
// If the key needs quoting (contains special characters, starts with special
// indicators, etc.) yaml.Marshal will add the necessary quotes.
// Returns an error if the key cannot be represented as a single-line YAML key
// (e.g. the key itself contains literal newlines).
func renderKeyInline(key string) (string, error) {
// Marshal a scalar node tagged !!str -- yaml.v3 will add quotes when the
// bare value would be misinterpreted (e.g. "1", ":", " x").
node := &yaml.Node{
Kind: yaml.ScalarNode,
Value: key,
Tag: "!!str",
}
b, err := yaml.Marshal(node)
if err != nil {
return "", err
}
// yaml.Marshal of a scalar node produces "value\n"
rendered := strings.TrimSuffix(string(b), "\n")
// Block literal (|) and folded (>) scalars span multiple lines and cannot
// serve as a simple inline mapping key (key: value on one line).
if strings.ContainsAny(rendered, "\n\r") || strings.HasPrefix(rendered, "|") || strings.HasPrefix(rendered, ">") {
return "", fmt.Errorf("key %q requires multi-line YAML representation and cannot be used as a simple mapping key", key)
}
return rendered, nil
}
// renderValueBlock renders a sequence or mapping value to a block of lines,
// indented at the given column (1-based). Returns lines like:
//
// " - a\n - b\n"
func renderValueBlock(v interface{}, indent int) (string, error) {
raw, err := yaml.Marshal(v)
if err != nil {
return "", err
}
// raw is like "- a\n- b\n" for a sequence, or "key: val\n" for a mapping.
prefix := strings.Repeat(" ", indent-1)
lines := strings.Split(string(raw), "\n")
var sb strings.Builder
for _, line := range lines {
if line == "" {
continue
}
sb.WriteString(prefix)
sb.WriteString(line)
sb.WriteByte('\n')
}
return sb.String(), nil
}
// isBlockValue returns true when the node needs block rendering (sequence or mapping
// that is not on the same line as its key).
func isBlockValue(keyN, valN *yaml.Node) bool {
if valN.Kind == yaml.ScalarNode {
return false
}
return valN.Line > keyN.Line
}
// splice replaces data[start:end] with replacement.
func splice(data []byte, start, end int, replacement []byte) []byte {
var buf bytes.Buffer
buf.Write(data[:start])
buf.Write(replacement)
buf.Write(data[end:])
return buf.Bytes()
}
// edit holds a resolved splice operation.
type edit struct {
start int
end int
replacement []byte
}
// YAMLEdit applies changes to data and returns the new document bytes.
// For every path that already exists, only the value span is rewritten
// at the same indent; untouched bytes (including all comments and
// whitespace) remain byte-identical. For a path that does not exist,
// a new key:value block is appended to the parent mapping at the parent's
// indent. Applies later edits first so earlier offsets remain valid.
func YAMLEdit(data []byte, changes []YAMLChange) ([]byte, error) {
if len(changes) == 0 {
return data, nil
}
seen := make(map[string]struct{}, len(changes))
for _, ch := range changes {
key := strings.Join(ch.Path, "\x00")
if _, dup := seen[key]; dup {
return nil, fmt.Errorf("yamledit: duplicate path %v", ch.Path)
}
seen[key] = struct{}{}
}
var root yaml.Node
if err := yaml.Unmarshal(data, &root); err != nil {
return nil, fmt.Errorf("yamledit: parse: %w", err)
}
li := buildLineIndex(data)
var edits []edit
for _, ch := range changes {
if len(ch.Path) == 0 {
return nil, fmt.Errorf("yamledit: empty path")
}
keyN, valN, parentMap, err := findNode(&root, ch.Path)
if err != nil {
return nil, fmt.Errorf("yamledit: %w", err)
}
if valN == nil {
// Key does not exist -- insert into parentMap.
var ed edit
ed, err = buildInsertEdit(data, li, parentMap, ch.Path[len(ch.Path)-1], ch.Value)
if err != nil {
return nil, fmt.Errorf("yamledit: insert %v: %w", ch.Path, err)
}
edits = append(edits, ed)
continue
}
// Key exists -- replace value span.
var ed edit
ed, err = buildReplaceEdit(data, li, keyN, valN, ch.Value)
if err != nil {
return nil, fmt.Errorf("yamledit: replace %v: %w", ch.Path, err)
}
edits = append(edits, ed)
}
// Sort by start offset descending so we splice end-to-start.
sort.Slice(edits, func(i, j int) bool {
return edits[i].start > edits[j].start
})
result := data
for _, ed := range edits {
result = splice(result, ed.start, ed.end, ed.replacement)
}
// Validate that the output is still parseable YAML. This catches edge cases
// where unusual input formats (complex key notation, etc.) produce invalid output.
var check yaml.Node
if err := yaml.Unmarshal(result, &check); err != nil {
return nil, fmt.Errorf("yamledit: output is not valid YAML: %w", err)
}
return result, nil
}
// buildReplaceEdit computes the splice for replacing an existing value node.
func buildReplaceEdit(data []byte, li lineIndex, keyN, valN *yaml.Node, value interface{}) (edit, error) {
if isBlockValue(keyN, valN) {
// Block sequence or mapping: value occupies one or more complete lines
// starting at valN.Line. Replace from the start of valN.Line to the
// end of the last line in the subtree.
lastLine := maxLine(valN)
start := li.lineStart(valN.Line)
end := li.lineEnd(lastLine, data)
if end < len(data) && data[end] == '\n' {
end++ // include the trailing newline so we replace whole lines
}
rendered, err := renderValueBlock(value, valN.Column)
if err != nil {
return edit{}, fmt.Errorf("path %v: %w", keyN.Value, err)
}
return edit{start: start, end: end, replacement: []byte(rendered)}, nil
}
// Flow / scalar: try inline first. If the new value is a sequence or
// mapping that cannot fit on one line (e.g. replacing `foo: []` with a
// multi-item list) fall back to block rendering and expand the span to
// cover the whole line, so the result is `foo:\n - a\n - b\n`.
start := li.offset(valN.Line, valN.Column)
end := li.lineEnd(valN.Line, data)
rendered, err := renderValueInline(value)
if err == nil {
// An implicit null starts immediately after the colon, before any
// whitespace. Its replacement needs a separator to remain a value.
if valN.Kind == yaml.ScalarNode && valN.Tag == "!!null" && valN.Value == "" {
rendered = " " + rendered
}
return edit{start: start, end: end, replacement: []byte(rendered)}, nil
}
if !needsBlockFallback(value) {
return edit{}, fmt.Errorf("path %v: %w", keyN.Value, err)
}
// Replace from the key's column (beginning of `key:`) to end of the key's
// line, emitting `key:\n<block>`. Using keyN.Column keeps the existing
// indent. A trailing `# comment` on the original line is kept attached
// to the key line so operator annotations survive a multi-select save.
keyStart := li.offset(keyN.Line, keyN.Column)
keyEnd := li.lineEnd(keyN.Line, data)
renderedKey, kerr := renderKeyInline(keyN.Value)
if kerr != nil {
return edit{}, fmt.Errorf("path %v: render key: %w", keyN.Value, kerr)
}
block, berr := renderValueBlock(value, keyN.Column+2)
if berr != nil {
return edit{}, fmt.Errorf("path %v: %w", keyN.Value, berr)
}
trailingComment := extractInlineComment(data[keyStart:keyEnd])
return edit{start: keyStart, end: keyEnd, replacement: []byte(renderedKey + ":" + trailingComment + "\n" + strings.TrimRight(block, "\n"))}, nil
}
// extractInlineComment scans a single YAML line for a trailing `#` comment
// (one preceded by whitespace, outside string/rune literals) and returns it
// with its leading whitespace, e.g. " # keep empty". Returns "" if there
// is no comment. The scanner is deliberately simple — it handles the common
// case of a flow value followed by whitespace + `#`; it does not attempt to
// parse every YAML quoting variant (block scalars etc. do not reach this
// path).
func extractInlineComment(line []byte) string {
inDQ := false
inSQ := false
for i := 0; i < len(line); i++ {
c := line[i]
if c == '"' && !inSQ {
// Count backslashes before this quote to detect escapes.
bs := 0
for j := i - 1; j >= 0 && line[j] == '\\'; j-- {
bs++
}
if bs%2 == 0 {
inDQ = !inDQ
}
continue
}
if c == '\'' && !inDQ {
inSQ = !inSQ
continue
}
if inDQ || inSQ {
continue
}
if c != '#' {
continue
}
// A `#` only starts a comment when preceded by whitespace (or at the
// start of the line). Otherwise it is a legal scalar character.
if i == 0 || line[i-1] == ' ' || line[i-1] == '\t' {
// Include the whitespace run before `#` so output keeps spacing.
start := i
for start > 0 && (line[start-1] == ' ' || line[start-1] == '\t') {
start--
}
return string(line[start:])
}
}
return ""
}
// needsBlockFallback reports whether value is a sequence or mapping type
// that may exceed one line and therefore warrants switching the replacement
// from inline to block rendering.
func needsBlockFallback(value interface{}) bool {
switch value.(type) {
case []string, []interface{}, map[string]interface{}, map[interface{}]interface{}:
return true
}
return false
}
// buildInsertEdit computes the splice for appending a new key to a mapping node.
func buildInsertEdit(data []byte, li lineIndex, parentMap *yaml.Node, key string, value interface{}) (edit, error) {
// Find the insertion point: end of the last content line of parentMap.
// If parentMap has no content (empty mapping), insert after the mapping's own line.
insertLine := parentMap.Line
if len(parentMap.Content) > 0 {
// last child is parentMap.Content[len-1]
last := parentMap.Content[len(parentMap.Content)-1]
insertLine = maxLine(last)
}
insertOff := li.lineEnd(insertLine, data)
needsNewline := false
if insertOff < len(data) && data[insertOff] == '\n' {
insertOff++ // insert after the newline
} else if insertOff > 0 && data[insertOff-1] != '\n' {
// Last line has no trailing newline; we must add one before the new key.
needsNewline = true
}
// Match the indent of existing siblings rather than inferring from the
// parent node's own column, which in a nested mapping does not reflect
// child indentation.
siblingCol := parentMap.Column
if len(parentMap.Content) >= 1 {
siblingCol = parentMap.Content[0].Column
}
prefix := strings.Repeat(" ", siblingCol-1)
renderedKey, err := renderKeyInline(key)
if err != nil {
return edit{}, fmt.Errorf("render key %q: %w", key, err)
}
var rendered string
if valueNeedsBlockInsert(value) {
block, err := renderValueBlock(value, siblingCol+2)
if err != nil {
return edit{}, fmt.Errorf("path %v: %w", key, err)
}
rendered = prefix + renderedKey + ":\n" + block
} else {
inline, err := renderValueInline(value)
if err != nil {
return edit{}, fmt.Errorf("path %v: %w", key, err)
}
rendered = prefix + renderedKey + ": " + inline + "\n"
}
if needsNewline {
rendered = "\n" + rendered
}
return edit{start: insertOff, end: insertOff, replacement: []byte(rendered)}, nil
}
func valueNeedsBlockInsert(value interface{}) bool {
if value == nil {
return false
}
v := reflect.ValueOf(value)
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Struct:
return true
case reflect.Slice:
return v.Type().Elem().Kind() != reflect.Uint8
default:
return false
}
}
// Package contenttype classifies file content shared by the malware scanners.
package contenttype
import (
"bytes"
"path/filepath"
"strings"
)
var compressedArchiveMagics = [][]byte{
{'P', 'K', 0x03, 0x04}, // ZIP local file header
{'P', 'K', 0x05, 0x06}, // ZIP empty archive
{'P', 'K', 0x07, 0x08}, // ZIP spanned data descriptor
{0x1f, 0x8b}, // gzip
{'B', 'Z', 'h'}, // bzip2
{0xfd, '7', 'z', 'X', 'Z', 0x00}, // xz
{'7', 'z', 0xbc, 0xaf, 0x27, 0x1c}, // 7z
{'R', 'a', 'r', '!', 0x1a, 0x07, 0x00}, // RAR 4.x
{'R', 'a', 'r', '!', 0x1a, 0x07, 0x01, 0x00}, // RAR 5.x
}
// IsCompressedArchive reports whether data starts with a supported compressed
// archive signature. Tar and PHAR are intentionally excluded because their raw
// content is executable or otherwise meaningful to the signature engines.
func IsCompressedArchive(data []byte) bool {
if len(data) < 4 {
return false
}
for _, magic := range compressedArchiveMagics {
if bytes.HasPrefix(data, magic) {
return true
}
}
return false
}
// archiveExtensions are the names under which a compressed container is
// stored on a web host: plain archives plus the zip-based document, package
// and extension formats. Lowercase, leading dot.
var archiveExtensions = map[string]struct{}{
".zip": {}, ".zipx": {}, ".jar": {}, ".war": {}, ".ear": {}, ".apk": {},
".docx": {}, ".xlsx": {}, ".pptx": {}, ".odt": {}, ".ods": {}, ".odp": {},
".epub": {}, ".whl": {}, ".egg": {}, ".crx": {}, ".xpi": {}, ".ipa": {},
".nupkg": {}, ".vsix": {},
".gz": {}, ".tgz": {}, ".svgz": {},
".bz2": {}, ".tbz": {}, ".tbz2": {},
".xz": {}, ".txz": {},
".7z": {},
".rar": {},
}
// IsArchiveExt reports whether ext (leading dot, any case) names a compressed
// container format.
func IsArchiveExt(ext string) bool {
_, ok := archiveExtensions[strings.ToLower(ext)]
return ok
}
// IsArchiveFile reports whether both the name and the leading bytes identify a
// compressed container. Bytes alone are not enough: PHP echoes whatever
// precedes its open tag and runs the rest, so any interpreted file can start
// with archive magic and still execute. Only a file that also carries an
// archive extension is left to the extraction-time scan.
func IsArchiveFile(name string, data []byte) bool {
return IsArchiveExt(filepath.Ext(name)) && IsCompressedArchive(data)
}
package contenttype
import (
"bytes"
"encoding/binary"
"strings"
)
// imageExts are the file extensions served as raster images. The set exists
// for path-based dispatch only. Detection never trusts it: every content
// decision goes through ImageContainer, so renaming a payload cannot hide it.
var imageExts = map[string]bool{
".png": true,
".jpg": true,
".jpeg": true,
".jpe": true,
".gif": true,
".webp": true,
".ico": true,
".cur": true,
".bmp": true,
".tif": true,
".tiff": true,
}
// IsImageExt reports whether ext (with the leading dot, any case) names a
// raster image format.
func IsImageExt(ext string) bool {
return imageExts[strings.ToLower(ext)]
}
// ImageContainer identifies the raster image format data begins with and
// returns its display name. A hostile file can carry any extension, so the
// verdict comes from the leading bytes alone.
//
// The signatures are deliberately longer than the shortest unique prefix.
// "BM" and the ICO lead-in are two and four bytes wide, which ordinary text
// reaches by chance, so the structural fields that follow them are checked
// too: an image container that also holds PHP is a strong detection signal
// and a weak magic test would spend it on plain files.
func ImageContainer(data []byte) (string, bool) {
switch {
case bytes.HasPrefix(data, []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}):
return "PNG", true
case bytes.HasPrefix(data, []byte{0xff, 0xd8, 0xff}):
return "JPEG", true
case bytes.HasPrefix(data, []byte("GIF87a")), bytes.HasPrefix(data, []byte("GIF89a")):
return "GIF", true
case len(data) >= 12 && bytes.HasPrefix(data, []byte("RIFF")) && bytes.Equal(data[8:12], []byte("WEBP")):
return "WebP", true
case isICO(data):
return "ICO", true
case isBMP(data):
return "BMP", true
case bytes.HasPrefix(data, []byte{0x49, 0x49, 0x2a, 0x00}), bytes.HasPrefix(data, []byte{0x4d, 0x4d, 0x00, 0x2a}):
return "TIFF", true
}
return "", false
}
// isICO checks the ICONDIR header: two reserved zero bytes, a type of 1 (icon)
// or 2 (cursor), and at least one directory entry.
func isICO(data []byte) bool {
if len(data) < 6 {
return false
}
if data[0] != 0 || data[1] != 0 {
return false
}
imageType := binary.LittleEndian.Uint16(data[2:4])
if imageType != 1 && imageType != 2 {
return false
}
return binary.LittleEndian.Uint16(data[4:6]) > 0
}
// isBMP checks the BITMAPFILEHEADER: the "BM" tag, the two reserved words
// which the format requires to be zero, and a pixel-data offset that lands
// past the header.
func isBMP(data []byte) bool {
if len(data) < 14 || data[0] != 'B' || data[1] != 'M' {
return false
}
if binary.LittleEndian.Uint32(data[6:10]) != 0 {
return false
}
return binary.LittleEndian.Uint32(data[10:14]) >= 14
}
package contenttype
import "strings"
// executablePHPExtensions are the extensions a stock PHP-capable web server
// (Apache mod_php / PHP-FPM via EasyApache4, LiteSpeed LSAPI, Nginx + php-fpm)
// routes to the PHP interpreter by default. Any file with one of these names
// can execute PHP, so a content scan that skipped them would let a webshell
// hide behind a non-".php" name. ".phps" is deliberately excluded: the stock
// handler renders it as highlighted source, it does not execute. Lowercase,
// leading dot.
var executablePHPExtensions = []string{
".php", ".php2", ".php3", ".php4", ".php5", ".php6", ".php7", ".php8",
".phtml", ".pht",
}
// IsExecutablePHPExt reports whether ext (leading dot, any case) is one a
// stock PHP handler executes. Rule sets are written against ".php"; every
// extension here must be matched against the same rules or a payload hides
// behind the name.
func IsExecutablePHPExt(ext string) bool {
ext = strings.ToLower(ext)
for _, e := range executablePHPExtensions {
if ext == e {
return true
}
}
return false
}
// IsExecutablePHPName reports whether a (lowercased) filename has an extension
// that a stock PHP handler executes. Shared by the realtime fanotify path, the
// periodic content scanners and the rule engines so none of them drift apart.
// It is a coarse, default-deny gate for content analysis only; per-directory
// .htaccess handler remappings are layered on top by the checks package.
func IsExecutablePHPName(nameLower string) bool {
for _, ext := range executablePHPExtensions {
if strings.HasSuffix(nameLower, ext) {
return true
}
}
return false
}
// IsPHPSourceName reports whether a file should receive PHP content analysis.
// It deliberately includes .phps even though IsExecutablePHPName does not:
// stock handlers render .phps as source, but the bytes can still hold a staged
// payload that becomes executable after a rename.
func IsPHPSourceName(nameLower string) bool {
return IsExecutablePHPName(nameLower) || strings.HasSuffix(nameLower, ".phps")
}
package contenttype
import "regexp"
// phpOpenTag matches the two PHP openers a web server executes under a stock
// configuration. The bare "<?" short tag is off by default and is also the
// XML and SVG declaration opener, so it is not treated as PHP here.
var phpOpenTag = regexp.MustCompile(`(?i)<\?(?:php(?:[ \t\r\n]|$)|=)`)
// HasPHPOpenTag reports whether data contains a PHP opening tag anywhere. It
// establishes PHP context for content whose name says nothing useful: an
// image carrying a payload, or a rule helper handed a file of unknown type.
func HasPHPOpenTag(data []byte) bool {
return phpOpenTag.Match(data)
}
// Package control defines the wire protocol between the CSM daemon and
// its local command-line client. The daemon listens on a Unix socket and
// accepts one line-framed JSON request per connection, replying with one
// line-framed JSON response. All types defined here are stable enough to
// be imported by both sides.
package control
import (
"encoding/json"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/health"
"github.com/pidginhost/csm/internal/incident"
"github.com/pidginhost/csm/internal/store"
)
// DefaultSocketPath is the Unix socket the daemon binds and the client
// dials. It sits next to pam.sock under /var/run/csm so both follow the
// same directory permissions established by the daemon at startup.
const DefaultSocketPath = "/var/run/csm/control.sock"
// Command names. Use constants so typos fail at compile time.
const (
CmdTierRun = "tier.run"
CmdStatus = "status"
CmdHistoryRead = "history.read"
CmdRulesReload = "rules.reload"
CmdGeoIPReload = "geoip.reload"
CmdBotRangesReload = "botranges.reload"
// Phase 2 additions.
CmdBaseline = "baseline"
CmdFirewallStatus = "firewall.status"
CmdFirewallPorts = "firewall.ports"
CmdFirewallGrep = "firewall.grep"
CmdFirewallAudit = "firewall.audit"
CmdFirewallBlock = "firewall.block"
CmdFirewallUnblock = "firewall.unblock"
CmdFirewallAllow = "firewall.allow"
CmdFirewallRemoveAllow = "firewall.remove_allow"
CmdFirewallAllowPort = "firewall.allow_port"
CmdFirewallRemovePort = "firewall.remove_port"
CmdFirewallTempBan = "firewall.tempban"
CmdFirewallTempAllow = "firewall.tempallow"
CmdFirewallDenySubnet = "firewall.deny_subnet"
CmdFirewallRemoveSubnet = "firewall.remove_subnet"
CmdFirewallDenyFile = "firewall.deny_file"
CmdFirewallAllowFile = "firewall.allow_file"
CmdFirewallFlush = "firewall.flush"
CmdFirewallRestart = "firewall.restart"
CmdFirewallApplyConfirmed = "firewall.apply_confirmed"
CmdFirewallConfirm = "firewall.confirm"
// Durable firewall actions. Actions lists what recovery could not settle,
// and ActionResolve records the outcome an operator established by hand.
CmdFirewallActions = "firewall.actions"
CmdFirewallActionResolve = "firewall.action_resolve"
CmdFirewallRollbackStatus = "firewall.rollback_status"
CmdFirewallRollbackRevert = "firewall.rollback_revert"
CmdFirewallRollbackOK = "firewall.rollback_confirm"
// Backup / restore. Export runs through the daemon (live read of
// bbolt). Import does not -- it requires a stopped daemon, so the
// CLI opens the archive and target paths directly.
CmdStoreExport = "store.export"
// Audit-log backfill. Streams every finding from the history
// bucket newer than the supplied cutoff so SIEM operators can
// seed their pipeline with prior findings the first time they
// enable audit_log:.
CmdHistorySince = "history.since"
CmdPHPRelayStatus = "phprelay.status"
CmdPHPRelayIgnoreScript = "phprelay.ignore_script"
CmdPHPRelayUnignore = "phprelay.unignore"
CmdPHPRelayIgnoreList = "phprelay.ignore_list"
CmdPHPRelayDryRun = "phprelay.dry_run"
CmdPHPRelayThaw = "phprelay.thaw"
// Clears one address's accumulated local threat-scoring state. Kept
// separate from the firewall commands: it changes no block, allow or
// whitelist entry, only the attack record that ranks the address.
CmdThreatForget = "threat.forget"
// CmdAntispamLearn submits one operator file to the antispam learner,
// as spam or to forget what it taught.
CmdAntispamLearn = "antispam.learn"
// CmdAntispamLearnStatus reads recorded outcomes of operator learning
// requests, or lists the unfinished ones.
CmdAntispamLearnStatus = "antispam.learn_status"
// CmdAntispamLearnFresh moves the learning state aside and empties the
// learning ledger, refused while learning is enabled.
CmdAntispamLearnFresh = "antispam.learn_fresh_namespace"
// Phase 2 incident correlation.
CmdIncidentsList = "incidents.list"
CmdIncidentsShow = "incidents.show"
CmdIncidentsStatus = "incidents.status"
CmdIncidentsBulkStatus = "incidents.bulk_status"
// CmdActionShow reads one admitted candidate and its attempts from the
// admission ledger by candidate or action ID.
CmdActionShow = "action.show"
// Full-scan job commands (uncapped-full-scan feature).
CmdScanEnqueue = "scan.enqueue"
CmdScanStatus = "scan.status"
CmdScanReport = "scan.report"
CmdScanCancel = "scan.cancel"
)
// Request is the single JSON object the client sends per connection.
// Args is a raw message so each command defines its own typed payload
// without forcing the transport layer to know about every command.
type Request struct {
Cmd string `json:"cmd"`
Args json.RawMessage `json:"args,omitempty"`
}
// Response is the single JSON object the daemon returns per connection.
// Ok=false means the request did not run to completion; inspect Error.
// Ok=true means the handler ran and Result holds the typed payload.
type Response struct {
OK bool `json:"ok"`
Result json.RawMessage `json:"result,omitempty"`
Error string `json:"error,omitempty"`
}
// TierRunArgs carries parameters for CmdTierRun.
// Tier is one of "critical", "deep", or "all". Alerts=true feeds the
// daemon's normal alert pipeline; false means "scan and report counts
// only" (used by the CLI's dry-run `check-*` commands when they migrate).
type TierRunArgs struct {
Tier string `json:"tier"`
Alerts bool `json:"alerts"`
}
// TierRunResult summarises what the tier run produced. Findings is only
// populated when TierRunArgs.Alerts was false (the dry-run `csm check*`
// path) so live tier runs do not pay the marshalling cost for the
// no-op case.
type TierRunResult struct {
Findings int `json:"findings"`
NewFindings int `json:"new_findings"`
ElapsedMs int64 `json:"elapsed_ms"`
FindingList []alert.Finding `json:"finding_list,omitempty"`
}
// StatusResult mirrors what `csm status` historically printed plus the
// extended health.Snapshot fields used by `csm status --json`. The
// pre-existing fields stay in place; any new client should prefer
// reading Snapshot directly.
type StatusResult struct {
Version string `json:"version"`
UptimeSec int64 `json:"uptime_sec"`
LatestScanTime string `json:"latest_scan_time,omitempty"`
LatestFindings int `json:"latest_findings"`
HistoryCount int `json:"history_count"`
DroppedAlerts int64 `json:"dropped_alerts"`
// Snapshot is the full health view added in v2.12.0. Older clients
// that only knew the six legacy fields above will simply ignore it.
Snapshot *health.Snapshot `json:"snapshot,omitempty"`
}
// HistoryReadArgs carries parameters for CmdHistoryRead.
type HistoryReadArgs struct {
Limit int `json:"limit"`
Offset int `json:"offset"`
}
// HistoryReadResult holds a page of historical findings, newest-first.
type HistoryReadResult struct {
Findings []alert.Finding `json:"findings"`
Total int `json:"total"`
}
// BaselineArgs carries parameters for CmdBaseline.
// Confirm mirrors the CLI's --confirm flag: required when existing
// history would be wiped.
type BaselineArgs struct {
Confirm bool `json:"confirm"`
}
// BaselineResult reports what a fresh baseline wrote.
type BaselineResult struct {
Findings int `json:"findings"`
HistoryCleared int `json:"history_cleared"`
BinaryHash string `json:"binary_hash"`
ConfigHash string `json:"config_hash"`
// NeedsConfirm=true means the daemon refused because Confirm was
// false and HistoryCleared would have been non-zero. The other
// fields are populated so the CLI can print the same warning as
// today without a second round trip.
NeedsConfirm bool `json:"needs_confirm,omitempty"`
}
// FirewallIPArgs is shared by block / unblock / allow / remove-style
// commands that take a single IP and an optional reason. Timeout is
// consumed by tempban/tempallow and MUST parse via time.ParseDuration
// (e.g. "24h", "1h30m", "5m"); empty means permanent. Invalid strings
// are rejected by the handler with a parse error.
type FirewallIPArgs struct {
IP string `json:"ip"`
Reason string `json:"reason,omitempty"`
Timeout string `json:"timeout,omitempty"`
}
// FirewallPortArgs covers allow-port / remove-port.
type FirewallPortArgs struct {
IP string `json:"ip"`
Port int `json:"port"`
Proto string `json:"proto,omitempty"` // "tcp" or "udp"; empty = tcp
Reason string `json:"reason,omitempty"`
}
// FirewallSubnetArgs covers deny-subnet / remove-subnet.
type FirewallSubnetArgs struct {
CIDR string `json:"cidr"`
Reason string `json:"reason,omitempty"`
}
// FirewallFileArgs carries a batch of IPs for deny-file / allow-file.
// The client reads the file locally and sends the contents over the
// socket so the daemon does not need to open arbitrary paths.
type FirewallFileArgs struct {
IPs []string `json:"ips"`
Reason string `json:"reason,omitempty"`
}
// FirewallGrepArgs matches the CLI's positional pattern.
type FirewallGrepArgs struct {
Pattern string `json:"pattern"`
}
// FirewallAuditArgs matches the CLI's optional limit. Limit=0 means
// "use the handler default" (currently 50 lines, matching the old CLI).
type FirewallAuditArgs struct {
Limit int `json:"limit"`
}
// FirewallActionResolveArgs carries an operator decision about one durable
// firewall action. Outcome is "applied" or "rejected", in the operator's own
// terms; Note records the evidence they went on.
type FirewallActionResolveArgs struct {
ID string `json:"id"`
Outcome string `json:"outcome"`
Note string `json:"note,omitempty"`
}
// FirewallApplyConfirmedArgs mirrors the CLI's minutes argument.
type FirewallApplyConfirmedArgs struct {
Minutes int `json:"minutes"`
}
// FirewallAckResult is the minimal ack returned by mutating firewall
// commands that do not need to report state back (block, allow, etc).
// Message is a short human-readable string the CLI can print verbatim.
type FirewallAckResult struct {
Message string `json:"message"`
}
// ThreatForgetResult reports what clearing an address's scoring state
// actually removed. Found distinguishes "cleared a stale record" from
// "there was nothing to clear", which the operator cannot otherwise tell
// apart and which decides whether the alert will stop.
// If legacy spellings created multiple records for the same address, Events
// is their total event count and Score is the highest removed record's score.
// Network is the /64 whose spam evidence went with an IPv6 address.
type ThreatForgetResult struct {
IP string `json:"ip"`
Found bool `json:"found"`
Score int `json:"score"`
Events int `json:"events"`
Network string `json:"network,omitempty"`
Message string `json:"message"`
}
// AntispamLearnArgs carries one file for CmdAntispamLearn. The snapshot
// travels as bytes, never a path the daemon would open. RequestID is
// random per file and reused only by transport retries; SnapshotSHA binds
// it to the bytes. Name is the file's basename, for the action record.
type AntispamLearnArgs struct {
Action string `json:"action"`
RequestID string `json:"request_id"`
Name string `json:"name"`
SnapshotSHA string `json:"snapshot_sha"`
Snapshot []byte `json:"snapshot"`
}
// AntispamLearnCluster is one cluster's result; ClusterID 0 is a part that
// resolved to no cluster (a skip).
type AntispamLearnCluster struct {
ClusterID uint64 `json:"cluster_id,omitempty"`
Result string `json:"result"`
}
// AntispamLearnResult is the outcome of one file. ExitOK is false for any
// skipped, failed, partial or pending file.
type AntispamLearnResult struct {
RequestID string `json:"request_id"`
Clusters []AntispamLearnCluster `json:"clusters,omitempty"`
Digests []string `json:"digests,omitempty"`
Outcome string `json:"outcome"`
ExitOK bool `json:"exit_ok"`
}
// AntispamLearnStatusArgs names request IDs; none lists every unfinished
// request.
type AntispamLearnStatusArgs struct {
RequestIDs []string `json:"request_ids,omitempty"`
}
// AntispamLearnStatusCluster is one cluster's recorded result; Phase is set
// while its publication is unfinished.
type AntispamLearnStatusCluster struct {
ClusterID uint64 `json:"cluster_id,omitempty"`
Result string `json:"result"`
Phase string `json:"phase,omitempty"`
}
// AntispamLearnStatus is what the daemon recorded for one request ID.
type AntispamLearnStatus struct {
RequestID string `json:"request_id"`
Action string `json:"action,omitempty"`
Found bool `json:"found"`
Done bool `json:"done"`
ExitOK bool `json:"exit_ok"`
Outcome string `json:"outcome,omitempty"`
Digests []string `json:"digests,omitempty"`
Clusters []AntispamLearnStatusCluster `json:"clusters,omitempty"`
}
// AntispamLearnStatusResult answers CmdAntispamLearnStatus.
type AntispamLearnStatusResult struct {
Requests []AntispamLearnStatus `json:"requests"`
}
// AntispamLearnFreshResult answers CmdAntispamLearnFresh: where the old
// learning directory went, empty when there was none.
type AntispamLearnFreshResult struct {
Archived string `json:"archived,omitempty"`
}
// FirewallRollbackStatus reports the pending tentative-apply state for
// the firewall settings section. Pending=false means nothing in flight;
// the other fields are zero in that case.
type FirewallRollbackStatus struct {
Pending bool `json:"pending"`
AppliedAtRFC3339 string `json:"applied_at,omitempty"`
ExpiresAtRFC3339 string `json:"expires_at,omitempty"`
SecondsRemaining int64 `json:"seconds_remaining,omitempty"`
AppliedBy string `json:"applied_by,omitempty"`
PrevHash string `json:"prev_hash,omitempty"`
NewHash string `json:"new_hash,omitempty"`
}
// FirewallStatusResult mirrors what `csm firewall status` prints.
// Fields match the CLI output one-to-one so the client can format
// identically without calling firewall.LoadState itself.
type FirewallStatusResult struct {
Enabled bool `json:"enabled"`
TCPIn []string `json:"tcp_in"`
TCPOut []string `json:"tcp_out"`
UDPIn []string `json:"udp_in"`
UDPOut []string `json:"udp_out"`
Restricted []string `json:"restricted"`
PassiveFTPStart int `json:"passive_ftp_start"`
PassiveFTPEnd int `json:"passive_ftp_end"`
TCPOutAllow []string `json:"tcp_out_allow,omitempty"`
ProtocolAllow []string `json:"protocol_allow,omitempty"`
TCPInAllow []string `json:"tcp_in_allow,omitempty"`
InfraIPCount int `json:"infra_ip_count"`
BlockedCount int `json:"blocked_count"`
BlockedNetCount int `json:"blocked_net_count"`
AllowedCount int `json:"allowed_count"`
SYNFlood bool `json:"syn_flood"`
ConnRateLimit int `json:"conn_rate_limit"`
LogDropped bool `json:"log_dropped"`
LogRate int `json:"log_rate"`
RecentBlocked []FirewallBlockedEntry `json:"recent_blocked,omitempty"`
}
// FirewallBlockedEntry is one entry in FirewallStatusResult.RecentBlocked.
type FirewallBlockedEntry struct {
IP string `json:"ip"`
Reason string `json:"reason"`
BlockedAt string `json:"blocked_at"`
ExpiresAt string `json:"expires_at,omitempty"`
}
// FirewallListResult is returned by ports / grep / audit: a freeform
// line-based payload the CLI prints verbatim.
type FirewallListResult struct {
Lines []string `json:"lines"`
}
// StoreExportArgs configures CmdStoreExport. DstPath must be an
// absolute path the daemon can write to; the CLI does not pre-create
// it.
type StoreExportArgs struct {
DstPath string `json:"dst_path"`
// Stage asks the daemon to write the archive under its own state
// directory (the one place its sandbox guarantees writable) instead of
// DstPath; the result's Path names the staged file for the CLI to move.
Stage bool `json:"stage,omitempty"`
}
// HistorySinceArgs configures CmdHistorySince. Since is RFC 3339; an
// empty value yields nothing (caller mistake, not "everything", to
// avoid accidental whole-DB dumps over the socket).
type HistorySinceArgs struct {
Since string `json:"since"`
}
// HistorySinceResult holds every finding newer than Since, oldest
// first so a downstream JSONL writer produces chronologically-sorted
// lines.
type HistorySinceResult struct {
Findings []alert.Finding `json:"findings"`
}
// StoreExportResult summarises a successful export.
type StoreExportResult struct {
Path string `json:"path"`
Bytes int64 `json:"bytes"`
ArchiveSHA256 string `json:"archive_sha256"`
BboltSHA256 string `json:"bbolt_sha256"`
}
// PHPRelayStatusRequest carries no parameters.
type PHPRelayStatusRequest struct{}
// PHPRelayStatusResponse summarises the running detector state.
type PHPRelayStatusResponse struct {
Enabled bool `json:"enabled"`
Platform string `json:"platform"`
DryRun bool `json:"dry_run"`
EffectiveAccountLimit int `json:"effective_account_limit"`
ScriptsTracked int `json:"scripts_tracked"`
IPsTracked int `json:"ips_tracked"`
AccountsTracked int `json:"accounts_tracked"`
MsgIDIndexSize int `json:"msgid_index_size"`
IgnoresActive int `json:"ignores_active"`
RecentFindings map[string]int `json:"recent_findings"` // path -> count last 1h
}
type PHPRelayIgnoreScriptRequest struct {
ScriptKey string `json:"script_key"`
ForHours int `json:"for_hours"` // 0 -> default 7d (168h)
Persist bool `json:"persist"`
Reason string `json:"reason"`
AddedBy string `json:"added_by"`
}
type PHPRelayIgnoreScriptResponse struct {
ExpiresAt time.Time `json:"expires_at"`
}
type PHPRelayUnignoreRequest struct {
ScriptKey string `json:"script_key"`
Persist bool `json:"persist"`
}
type PHPRelayIgnoreEntry struct {
ScriptKey string `json:"script_key"`
ExpiresAt time.Time `json:"expires_at"`
AddedBy string `json:"added_by"`
Reason string `json:"reason"`
}
type PHPRelayIgnoreListResponse struct {
Entries []PHPRelayIgnoreEntry `json:"entries"`
}
type PHPRelayDryRunRequest struct {
Mode string `json:"mode"` // "on" | "off" | "reset"
Persist bool `json:"persist"`
}
type PHPRelayDryRunResponse struct {
Effective bool `json:"effective"`
Source string `json:"source"` // "runtime" | "bbolt" | "csm.yaml"
}
type PHPRelayThawRequest struct {
MsgID string `json:"msg_id"`
By string `json:"by"`
}
type PHPRelayThawResponse struct {
Stderr string `json:"stderr,omitempty"`
}
// ActionShowArgs names the candidate or action CmdActionShow reads.
type ActionShowArgs struct {
ID string `json:"id"`
}
// IncidentShowArgs targets a single incident by id.
type IncidentShowArgs struct {
ID string `json:"id"`
}
// IncidentListArgs pages CmdIncidentsList. All=true returns every
// matching incident and should be reserved for offline export-style use.
type IncidentListArgs struct {
Limit int `json:"limit,omitempty"`
Offset int `json:"offset,omitempty"`
Status string `json:"status,omitempty"` // all / active / open / contained / resolved / dismissed
All bool `json:"all,omitempty"`
}
// IncidentListResult is the bounded list envelope returned by
// CmdIncidentsList. Total is computed before paging so the CLI can show
// the next offset without a second command.
type IncidentListResult struct {
Items []incident.Incident `json:"items"`
Total int `json:"total"`
Offset int `json:"offset"`
Limit int `json:"limit"`
Status string `json:"status"`
}
// IncidentStatusArgs transitions an incident's status.
type IncidentStatusArgs struct {
ID string `json:"id"`
Status string `json:"status"` // open / contained / resolved / dismissed
Details string `json:"details,omitempty"`
}
// IncidentBulkStatusArgs previews or applies the same status transition
// to stale incidents matching the supplied exact filters.
type IncidentBulkStatusArgs struct {
Status string `json:"status,omitempty"` // active / open / contained
To string `json:"to,omitempty"` // resolved / dismissed
OlderThanSeconds int64 `json:"older_than_seconds,omitempty"`
LastSeenBefore time.Time `json:"last_seen_before,omitempty"`
Kind string `json:"kind,omitempty"`
Domain string `json:"domain,omitempty"`
Account string `json:"account,omitempty"`
Mailbox string `json:"mailbox,omitempty"`
Limit int `json:"limit,omitempty"`
Apply bool `json:"apply,omitempty"`
Confirm bool `json:"confirm,omitempty"`
Details string `json:"details,omitempty"`
}
// IncidentBulkStatusResult reports the full match count plus the
// bounded set of incidents that were previewed or changed.
type IncidentBulkStatusResult struct {
DryRun bool `json:"dry_run"`
Matched int `json:"matched"`
Updated int `json:"updated"`
Limit int `json:"limit"`
Status string `json:"status"`
To string `json:"to"`
OlderThanSeconds int64 `json:"older_than_seconds,omitempty"`
LastSeenBefore time.Time `json:"last_seen_before,omitempty"`
Items []incident.BulkStatusItem `json:"items"`
}
// ScanEnqueueRequest carries parameters for CmdScanEnqueue.
// Phase 1 accepts only Scope="account". Target is the cPanel username to scan.
// RespectIgnores gates whether cfg.Suppressions.IgnorePaths apply during the scan.
// Quarantine records the quarantine intent for use by a later phase; no live
// auto-response is wired in Phase 1.
type ScanEnqueueRequest struct {
Scope string `json:"scope"`
Target string `json:"target"`
RespectIgnores bool `json:"respect_ignores"`
Quarantine bool `json:"quarantine"`
}
// ValidScanAccountTarget reports whether target is a single account name, not
// a path. Full-scan handlers join this value under /home, so slashes, dot-only
// names, spaces, and control bytes must be rejected at the protocol boundary.
func ValidScanAccountTarget(target string) bool {
if target == "" || target == "." || target == ".." {
return false
}
for i := 0; i < len(target); i++ {
b := target[i]
if b >= 'a' && b <= 'z' ||
b >= 'A' && b <= 'Z' ||
b >= '0' && b <= '9' ||
b == '_' || b == '-' || b == '.' {
continue
}
return false
}
return true
}
// ScanEnqueueResponse carries the job ID and initial state ("queued").
type ScanEnqueueResponse struct {
JobID string `json:"job_id"`
State string `json:"state"`
}
// ScanStatusRequest carries an optional job ID for CmdScanStatus.
// An empty JobID requests the full job list; a concrete ID returns that job only.
type ScanStatusRequest struct {
JobID string `json:"job_id,omitempty"`
}
// ScanStatusResponse carries a single job or the full list depending on the request.
type ScanStatusResponse struct {
Job *store.ScanJobRecord `json:"job,omitempty"`
Jobs []store.ScanJobRecord `json:"jobs,omitempty"`
}
// ScanReportRequest carries parameters for CmdScanReport.
// Offset and Limit follow the usual slice semantics; Limit=0 returns all findings.
type ScanReportRequest struct {
JobID string `json:"job_id"`
Offset int `json:"offset,omitempty"`
Limit int `json:"limit,omitempty"`
}
// ScanReportResponse carries the job record, the requested page of findings, and
// the total finding count for the job (before paging).
type ScanReportResponse struct {
Job store.ScanJobRecord `json:"job"`
Findings []alert.Finding `json:"findings"`
Total int `json:"total"`
}
// ScanCancelRequest carries the job ID for CmdScanCancel.
type ScanCancelRequest struct {
JobID string `json:"job_id"`
}
// ScanCancelResponse reports the job ID and its state after cancellation.
type ScanCancelResponse struct {
JobID string `json:"job_id"`
State string `json:"state"`
}
package corpusgate
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"strings"
"syscall"
"time"
)
// Only this reserved namespace is tool-owned. Flat caches from older versions
// may share an operator's directory, so they are read-only inputs.
const archiveCacheDir = ".csm-corpus-v1"
func openArchiveCache(parent *os.Root) (*os.Root, *os.File, error) {
if err := parent.Mkdir(archiveCacheDir, 0700); err != nil && !os.IsExist(err) {
return nil, nil, err
}
info, err := parent.Lstat(archiveCacheDir)
if err != nil {
return nil, nil, err
}
if !info.IsDir() {
return nil, nil, fmt.Errorf("corpus archive namespace must be a directory, not a symlink")
}
root, err := parent.OpenRoot(archiveCacheDir)
if err != nil {
return nil, nil, err
}
// OpenRoot follows in-root symlinks. Compare the opened directory to the
// checked inode so a replacement during open cannot redirect pruning.
opened, err := root.Stat(".")
if err != nil {
_ = root.Close()
return nil, nil, err
}
if !os.SameFile(info, opened) {
_ = root.Close()
return nil, nil, fmt.Errorf("corpus archive namespace changed while opening")
}
lock, err := root.OpenFile("lock", os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0600)
if err != nil {
_ = root.Close()
return nil, nil, err
}
// Keep this inode: unlinking it would let concurrent runs hold different
// locks and prune each other's downloads. Closing releases the kernel lock.
// #nosec G115 -- POSIX file descriptors fit in int.
if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
_ = lock.Close()
_ = root.Close()
return nil, nil, fmt.Errorf("corpus archive cache is busy: %w", err)
}
return root, lock, nil
}
func cacheDigest(name, suffix string) bool {
if !strings.HasSuffix(name, suffix) {
return false
}
digest := strings.TrimSuffix(name, suffix)
if len(digest) != sha256.Size*2 || digest != strings.ToLower(digest) {
return false
}
_, err := hex.DecodeString(digest)
return err == nil
}
func dropUnpinned(cache *os.Root, pinned map[string]bool) error {
entries, err := fs.ReadDir(cache.FS(), ".")
if err != nil {
return err
}
for _, e := range entries {
if !e.Type().IsRegular() && e.Type()&os.ModeSymlink == 0 {
continue
}
// Interrupted downloads never become reusable entries. The cache lock
// makes every leftover temporary file safe to remove on the next run.
retired := cacheDigest(e.Name(), ".zip") && !pinned[e.Name()]
if cacheDigest(e.Name(), ".download") || retired {
if err := cache.Remove(e.Name()); err != nil {
return err
}
}
}
return nil
}
func cachedArchive(ctx context.Context, legacy, cache *os.Root, source Source) (*os.File, int64, error) {
digest := strings.ToLower(source.SHA256)
name := digest + ".zip"
f, n, err := verifiedArchive(cache, name, digest)
if err == nil {
return f, n, nil
}
if !os.IsNotExist(err) {
// Remove the entry itself, never a symlink target. A failed repair can
// then be retried without trusting or repeatedly re-reading bad bytes.
if err := cache.Remove(name); err != nil {
return nil, 0, err
}
}
// A valid flat archive still avoids a download. Its name is not proof of
// ownership, so neither importing it nor retiring it may delete the input.
for _, old := range []string{source.ID + "-" + source.SHA256 + ".zip", source.ID + "-" + source.Version + ".zip"} {
if f, _, err := verifiedArchive(legacy, old, digest); err == nil {
_, seekErr := f.Seek(0, io.SeekStart)
if seekErr != nil {
_ = f.Close()
return nil, 0, seekErr
}
imported, n, err := storeArchive(cache, digest, f)
closeErr := f.Close()
if err == nil && closeErr != nil {
_ = imported.Close()
return nil, 0, closeErr
}
return imported, n, err
}
}
return download(ctx, source.URL, cache, digest)
}
func verifiedArchive(cache *os.Root, name, digest string) (*os.File, int64, error) {
info, err := cache.Lstat(name)
if err != nil {
return nil, 0, err
}
if !info.Mode().IsRegular() || info.Size() > maxArchive {
return nil, 0, fmt.Errorf("archive is not a regular file within the size limit")
}
f, err := cache.OpenFile(name, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if err != nil {
return nil, 0, err
}
h := sha256.New()
n, err := io.Copy(h, io.LimitReader(f, maxArchive+1))
if err != nil || n > maxArchive || hex.EncodeToString(h.Sum(nil)) != digest {
_ = f.Close()
return nil, 0, fmt.Errorf("archive checksum/size/read failure")
}
return f, n, nil
}
func download(ctx context.Context, address string, cache *os.Root, digest string) (*os.File, int64, error) {
client := &http.Client{Timeout: 3 * time.Minute, CheckRedirect: func(req *http.Request, via []*http.Request) error {
if req.URL.Scheme != "https" || len(via) >= 10 {
return fmt.Errorf("unsafe corpus redirect")
}
return nil
}}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil)
if err != nil {
return nil, 0, err
}
response, err := client.Do(req)
if err != nil {
return nil, 0, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, 0, fmt.Errorf("corpus download HTTP %d", response.StatusCode)
}
return storeArchive(cache, digest, response.Body)
}
func storeArchive(cache *os.Root, digest string, input io.Reader) (*os.File, int64, error) {
tmp := digest + ".download"
file, err := cache.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600)
if err != nil {
return nil, 0, err
}
defer func() { _ = cache.Remove(tmp) }()
h := sha256.New()
n, copyErr := io.Copy(io.MultiWriter(file, h), io.LimitReader(input, maxArchive+1))
closeErr := file.Close()
if copyErr != nil {
return nil, 0, copyErr
}
if closeErr != nil {
return nil, 0, closeErr
}
if n > maxArchive || hex.EncodeToString(h.Sum(nil)) != digest {
return nil, 0, fmt.Errorf("downloaded archive checksum/size failure")
}
name := digest + ".zip"
if err := cache.Rename(tmp, name); err != nil {
return nil, 0, err
}
// Verify and extract the same open file, including after publication.
return verifiedArchive(cache, name, digest)
}
package corpusgate
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"net/url"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/cms"
)
// ManifestVersion is the only accepted manifest version. Version 2 made the
// per-source cms field mandatory and added the pending list.
const ManifestVersion = 2
// PendingCMS records a supported CMS with no pinned clean source yet and the
// roadmap item that blocks it. It is coverage metadata only: nothing at scan
// time reads it, and it confers no runtime trust.
type PendingCMS struct {
CMS string `json:"cms"`
Reason string `json:"reason"`
}
// Validate checks the manifest's own metadata: the version, every source's
// fields, and that every supported CMS is either sourced or explicitly
// pending, never both. The supported set always comes from internal/cms so a
// caller cannot validate against a smaller one. Archive authentication,
// extraction confinement and inventory checks stay in Prepare.
func (m Manifest) Validate() error {
if m.Version != ManifestVersion {
return fmt.Errorf("corpus manifest version %d is not supported; version %d with a cms field on every source is required", m.Version, ManifestVersion)
}
if len(m.Sources) == 0 {
return fmt.Errorf("corpus manifest has no source")
}
sourced := make(map[cms.Kind]bool)
seenID := make(map[string]bool, len(m.Sources))
for _, s := range m.Sources {
if err := s.validate(); err != nil {
return err
}
if seenID[s.ID] {
return fmt.Errorf("source %q: id declared twice", s.ID)
}
seenID[s.ID] = true
kind, ok := cms.Parse(s.CMS)
if !ok {
return fmt.Errorf("source %q: cms %q is not a supported kind", s.ID, s.CMS)
}
sourced[kind] = true
}
pending := make(map[cms.Kind]bool)
for _, p := range m.Pending {
kind, ok := cms.Parse(p.CMS)
if !ok {
return fmt.Errorf("pending cms %q is not a supported kind", p.CMS)
}
if pending[kind] {
return fmt.Errorf("pending cms %q is declared twice", p.CMS)
}
if strings.TrimSpace(p.Reason) == "" {
return fmt.Errorf("pending cms %q: reason is required", p.CMS)
}
if sourced[kind] {
return fmt.Errorf("cms %q is both sourced and pending", p.CMS)
}
pending[kind] = true
}
for _, d := range cms.All() {
if !sourced[d.Kind] && !pending[d.Kind] {
return fmt.Errorf("cms %q has neither a pinned source nor a pending entry", d.Kind)
}
}
return nil
}
func (s Source) validate() error {
if s.ID == "" || strings.ContainsAny(s.ID, "/\\\x00") || s.ID == "." || s.ID == ".." {
return fmt.Errorf("source %q: id must be a non-empty path-safe name", s.ID)
}
if s.Version == "" || strings.ContainsAny(s.Version, "/\\\x00") {
return fmt.Errorf("source %q: version must be a non-empty path-safe string", s.ID)
}
u, err := url.Parse(s.URL)
if err != nil || u.Scheme != "https" || u.Host == "" {
return fmt.Errorf("source %q: url must be https with a host", s.ID)
}
digest, err := hex.DecodeString(s.SHA256)
if err != nil || len(digest) != sha256.Size {
return fmt.Errorf("source %q: sha256 must be 64 hex characters", s.ID)
}
if s.License == "" {
return fmt.Errorf("source %q: license is required", s.ID)
}
if !filepath.IsLocal(s.LicenseFile) {
return fmt.Errorf("source %q: license_file must be a local path inside the archive", s.ID)
}
if s.Files < 1 || s.Files > maxSourceFiles {
return fmt.Errorf("source %q: files must be between 1 and %d", s.ID, maxSourceFiles)
}
if s.CMS == "" {
return fmt.Errorf("source %q: cms field is required in manifest version %d", s.ID, ManifestVersion)
}
return nil
}
// maxSourceFiles bounds a pinned archive's inventory.
const maxSourceFiles = 30000
// Package corpusgate provisions pinned clean applications for detector tests.
package corpusgate
import (
"archive/zip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
)
type Manifest struct {
Version int `json:"version"`
Sources []Source `json:"sources"`
// Pending lists supported CMSs with no pinned source yet. See Validate.
Pending []PendingCMS `json:"pending,omitempty"`
}
type Source struct {
ID string `json:"id"`
// CMS names the supported kind this archive belongs to; a plugin or
// theme names the CMS it runs on.
CMS string `json:"cms"`
Version string `json:"version"`
URL string `json:"url"`
SHA256 string `json:"sha256"`
License string `json:"license"`
LicenseFile string `json:"license_file"`
Files int `json:"files"`
}
type File struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
Bytes int64 `json:"bytes"`
}
const maxArchive = 512 << 20
const maxExpanded = 1 << 30
// Prepare refuses to reuse extracted trees: only authenticated archive caches
// survive runs, so removed vendor files cannot inflate the next inventory.
func Prepare(ctx context.Context, manifest Manifest, cache, destination string) ([]File, error) {
// Validate before any directory, cache or network side effect so a
// drifted manifest is refused without leaving artifacts behind.
if err := manifest.Validate(); err != nil {
return nil, err
}
if err := os.Mkdir(destination, 0700); err != nil {
return nil, fmt.Errorf("new corpus directory: %w", err)
}
if err := os.MkdirAll(cache, 0700); err != nil {
return nil, err
}
cacheRoot, err := os.OpenRoot(cache)
if err != nil {
return nil, err
}
defer func() { _ = cacheRoot.Close() }()
managed, lock, err := openArchiveCache(cacheRoot)
if err != nil {
return nil, err
}
defer func() { _ = managed.Close() }()
defer func() { _ = lock.Close() }()
pinned := make(map[string]bool, len(manifest.Sources))
for _, s := range manifest.Sources {
pinned[strings.ToLower(s.SHA256)+".zip"] = true
}
// Prune before downloads so failed runs cannot accumulate retired pins.
if err := dropUnpinned(managed, pinned); err != nil {
return nil, err
}
var inventory []File
for _, s := range manifest.Sources {
f, n, err := cachedArchive(ctx, cacheRoot, managed, s)
if err != nil {
return nil, fmt.Errorf("archive %s: %w", s.ID, err)
}
rows, err := extract(f, n, destination, s)
closeErr := f.Close()
if err == nil {
err = closeErr
}
if err != nil {
return nil, fmt.Errorf("extract %s: %w", s.ID, err)
}
inventory = append(inventory, rows...)
}
sort.Slice(inventory, func(i, j int) bool { return inventory[i].Path < inventory[j].Path })
return inventory, nil
}
func extract(archive io.ReaderAt, size int64, destination string, source Source) ([]File, error) {
z, err := zip.NewReader(archive, size)
if err != nil {
return nil, err
}
dir := filepath.Join(destination, source.ID)
if err = os.Mkdir(dir, 0700); err != nil {
return nil, err
}
root, err := os.OpenRoot(dir)
if err != nil {
return nil, err
}
defer func() { _ = root.Close() }()
var rows []File
var expanded int64
license := false
for _, entry := range z.File {
name := strings.TrimSuffix(entry.Name, "/")
if !filepath.IsLocal(name) || strings.Contains(name, "\\") {
return nil, fmt.Errorf("unsafe archive name %q", name)
}
if entry.FileInfo().IsDir() {
if err := root.MkdirAll(name, 0700); err != nil {
return nil, err
}
continue
}
if !entry.Mode().IsRegular() {
return nil, fmt.Errorf("unsupported archive entry %q", name)
}
if len(rows) >= source.Files || entry.UncompressedSize64 > maxExpanded || expanded+int64(entry.UncompressedSize64) > maxExpanded {
return nil, fmt.Errorf("corpus exceeds pinned inventory or size limit")
}
if err := root.MkdirAll(filepath.Dir(name), 0700); err != nil {
return nil, err
}
out, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
if err != nil {
return nil, err
}
in, err := entry.Open()
if err != nil {
_ = out.Close()
return nil, err
}
h := sha256.New()
n, copyErr := io.Copy(io.MultiWriter(out, h), io.LimitReader(in, maxExpanded-expanded+1))
readClose, writeClose := in.Close(), out.Close()
if copyErr != nil {
return nil, copyErr
}
if readClose != nil {
return nil, readClose
}
if writeClose != nil {
return nil, writeClose
}
expanded += n
if expanded > maxExpanded || n != int64(entry.UncompressedSize64) {
return nil, fmt.Errorf("invalid expanded size")
}
if name == source.LicenseFile && n > 0 {
license = true
}
rows = append(rows, File{Path: filepath.ToSlash(filepath.Join(source.ID, name)), SHA256: hex.EncodeToString(h.Sum(nil)), Bytes: n})
}
if len(rows) != source.Files || !license {
return nil, fmt.Errorf("incomplete corpus: files=%d want=%d license=%t", len(rows), source.Files, license)
}
return rows, nil
}
func WriteJSON(path string, value any) error {
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
// #nosec G703 -- The local command or test runner selects this artifact path; vendor content never supplies it.
return os.WriteFile(path, append(data, '\n'), 0600)
}
package corpusgate
import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
)
func Root(key string) (string, error) {
root := os.Getenv(key)
if root == "" && os.Getenv("CSM_CORPUS_REQUIRED") == "1" {
return "", fmt.Errorf("required corpus variable %s is missing", key)
}
return root, nil
}
type Report struct {
Engine string `json:"engine"`
Scanned int `json:"scanned"`
Hits map[string]int `json:"hits"`
Thresholds map[string]int `json:"thresholds"`
Statuses map[string]int `json:"statuses,omitempty"`
StatusThresholds map[string]int `json:"status_thresholds,omitempty"`
}
func (r Report) Validate() error {
if r.Scanned <= 0 {
return fmt.Errorf("%s scanned no files", r.Engine)
}
var bad []string
for rule, count := range r.Hits {
if count > r.Thresholds[rule] {
bad = append(bad, fmt.Sprintf("%s=%d (maximum %d)", rule, count, r.Thresholds[rule]))
}
}
for status, count := range r.Statuses {
if status != "analyzed" && status != "not_candidate" && count > r.StatusThresholds[status] {
bad = append(bad, fmt.Sprintf("status %s=%d (maximum %d)", status, count, r.StatusThresholds[status]))
}
}
sort.Strings(bad)
if len(bad) > 0 {
return fmt.Errorf("%s corpus regressions: %s", r.Engine, strings.Join(bad, ", "))
}
return nil
}
// Save writes before validation, retaining evidence from failing gates too.
func (r Report) Save() error {
dir := os.Getenv("CSM_CORPUS_REPORT_DIR")
if dir == "" {
if os.Getenv("CSM_CORPUS_REQUIRED") == "1" {
return fmt.Errorf("CSM_CORPUS_REPORT_DIR is required")
}
return nil
}
thresholds := make(map[string]int, len(r.Hits))
for rule := range r.Hits {
thresholds[rule] = r.Thresholds[rule]
}
for rule, limit := range r.Thresholds {
thresholds[rule] = limit
}
r.Thresholds = thresholds
return WriteJSON(filepath.Join(dir, r.Engine+".json"), r)
}
// Package crashsample keeps private copies of inputs that crashed a parse
// worker, for diagnosis. The copies hold customer mail and tenant files, so
// the store is bounded in count, bytes and age, and readable by root only.
package crashsample
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
"syscall"
"time"
)
const (
// MaxSamples is how many samples the store keeps; the oldest go first.
MaxSamples = 20
// MaxBytes bounds the store's total size.
MaxBytes = 500 << 20
// MaxAge is how long a sample is kept.
MaxAge = 7 * 24 * time.Hour
tmpPrefix = ".tmp-"
)
var validName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`)
// File is one file of a sample, copied from Src without moving any shared
// file offset.
type File struct {
Name string
Src io.ReaderAt
Size int64
}
// Store keeps samples as directories under dir.
type Store struct {
dir string
maxBytes int64
now func() time.Time
mu sync.Mutex
}
// New returns a store in dir, which is created on first save.
func New(dir string) *Store {
return &Store{dir: dir, maxBytes: MaxBytes, now: time.Now}
}
type sample struct {
name string
mtime time.Time
size int64
}
// Save stores files as sample name, evicting the oldest samples first so the
// store stays within MaxSamples and the byte cap. A sample that already
// exists is kept as it is.
func (s *Store) Save(name string, files []File) error {
if !validName.MatchString(name) {
return fmt.Errorf("crash sample: invalid name %q", name)
}
var total int64
for _, f := range files {
if !validName.MatchString(f.Name) || f.Size < 0 {
return fmt.Errorf("crash sample: invalid file %q", f.Name)
}
if f.Size > s.maxBytes-total {
return fmt.Errorf("crash sample exceeds the %d byte cap", s.maxBytes)
}
total += f.Size
}
s.mu.Lock()
defer s.mu.Unlock()
root, err := s.openDir(true)
if err != nil {
return err
}
defer func() { _ = root.Close() }()
if _, err = s.sweep(root); err != nil {
return err
}
if _, err = root.Lstat(name); err == nil {
return nil
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
if err = s.evict(root, total); err != nil {
return err
}
tmp := tmpPrefix + name
if err = root.Mkdir(tmp, 0o700); err != nil {
return fmt.Errorf("crash sample: %w", err)
}
defer func() { _ = root.RemoveAll(tmp) }()
staging, err := openChildRoot(root, tmp)
if err != nil {
return err
}
defer func() { _ = staging.Close() }()
if err = s.write(staging, files); err != nil {
return err
}
now := s.now()
if err = staging.Chtimes(".", now, now); err != nil {
return fmt.Errorf("crash sample: %w", err)
}
if err = sameDirectory(root, tmp, staging); err != nil {
return err
}
if err = root.Rename(tmp, name); err != nil {
return fmt.Errorf("crash sample: %w", err)
}
return nil
}
func (s *Store) write(staging *os.Root, files []File) error {
for _, f := range files {
out, err := staging.OpenFile(f.Name, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600)
if err != nil {
return fmt.Errorf("crash sample: %w", err)
}
_, err = io.CopyN(out, io.NewSectionReader(f.Src, 0, f.Size), f.Size)
if cerr := out.Close(); err == nil {
err = cerr
}
if err != nil {
return fmt.Errorf("crash sample %s: %w", f.Name, err)
}
}
return nil
}
// Pin the store and staging directories for the whole operation. A path
// check alone would let a replacement link redirect later copies or cleanup.
func (s *Store) openDir(create bool) (*os.Root, error) {
parent, err := openParent(s.dir)
if err != nil {
return nil, err
}
defer func() { _ = parent.Close() }()
name := filepath.Base(s.dir)
if create {
if err = parent.Mkdir(name, 0o700); err != nil && !errors.Is(err, os.ErrExist) {
return nil, err
}
}
root, err := openChildRoot(parent, name)
if err != nil {
return nil, err
}
// Chmod the pinned directory, not its replaceable pathname.
dir, err := root.Open(".")
if err == nil {
err = dir.Chmod(0o700) // #nosec G302 -- directory needs owner search permission.
_ = dir.Close()
}
if err != nil {
_ = root.Close()
return nil, err
}
return root, nil
}
// openParent opens the directory that holds the store. That path belongs to
// root's layout and may contain links (such as /opt on a symlink); only the
// store directory itself is pinned against replacement.
func openParent(path string) (*os.Root, error) {
abs, err := filepath.Abs(filepath.Dir(path))
if err != nil {
return nil, err
}
return os.OpenRoot(abs)
}
func openChildRoot(parent *os.Root, name string) (*os.Root, error) {
info, err := parent.Lstat(name)
if err != nil {
return nil, err
}
if !info.IsDir() {
return nil, fmt.Errorf("crash sample: %q is not a directory", name)
}
root, err := parent.OpenRoot(name)
if err != nil {
return nil, err
}
pinned, err := root.Stat(".")
if err != nil || !os.SameFile(info, pinned) {
_ = root.Close()
return nil, fmt.Errorf("crash sample: directory %q changed while opening", name)
}
return root, nil
}
func sameDirectory(parent *os.Root, name string, root *os.Root) error {
info, err := parent.Lstat(name)
if err != nil {
return err
}
pinned, err := root.Stat(".")
if err != nil {
return err
}
if !info.IsDir() || !os.SameFile(info, pinned) {
return fmt.Errorf("crash sample: directory %q changed while saving", name)
}
return nil
}
// list returns the stored samples, oldest first, and removes leftover
// temporary directories.
func (s *Store) list(root *os.Root) ([]sample, error) {
dir, err := root.Open(".")
if err != nil {
return nil, fmt.Errorf("crash sample: %w", err)
}
defer dir.Close()
entries, err := dir.ReadDir(-1)
if err != nil {
return nil, err
}
var out []sample
for _, e := range entries {
if strings.HasPrefix(e.Name(), tmpPrefix) {
if err := root.RemoveAll(e.Name()); err != nil {
return nil, err
}
continue
}
info, err := root.Lstat(e.Name())
if err != nil {
return nil, err
}
smp := sample{name: e.Name(), mtime: info.ModTime()}
if info.IsDir() {
// Open one component with NOFOLLOW so even a concurrent link
// replacement cannot make retention inspect another tree.
dir, err := root.OpenFile(e.Name(), os.O_RDONLY|syscall.O_DIRECTORY|syscall.O_NOFOLLOW, 0)
if err != nil {
return nil, err
}
files, err := dir.Readdir(-1)
_ = dir.Close()
if err != nil {
return nil, err
}
for _, fi := range files {
smp.size += fi.Size()
}
}
out = append(out, smp)
}
sort.Slice(out, func(i, j int) bool { return out[i].mtime.Before(out[j].mtime) })
return out, nil
}
// evict removes the oldest samples until one more of size bytes fits.
func (s *Store) evict(root *os.Root, size int64) error {
stored, err := s.list(root)
if err != nil {
return err
}
var total int64
for _, smp := range stored {
total += smp.size
}
for len(stored) > 0 && (len(stored) >= MaxSamples || total+size > s.maxBytes) {
if err := root.RemoveAll(stored[0].name); err != nil {
return fmt.Errorf("crash sample: %w", err)
}
total -= stored[0].size
stored = stored[1:]
}
return nil
}
// Sweep removes samples older than MaxAge and leftover temporary
// directories, and reports how many samples it removed.
func (s *Store) Sweep() (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
root, err := s.openDir(false)
if errors.Is(err, os.ErrNotExist) {
return 0, nil
}
if err != nil {
return 0, fmt.Errorf("crash sample: %w", err)
}
defer func() { _ = root.Close() }()
return s.sweep(root)
}
func (s *Store) sweep(root *os.Root) (int, error) {
stored, err := s.list(root)
if err != nil {
return 0, err
}
cutoff := s.now().Add(-MaxAge)
removed := 0
for _, smp := range stored {
if !smp.mtime.Before(cutoff) {
break
}
if err := root.RemoveAll(smp.name); err != nil {
return removed, fmt.Errorf("crash sample: %w", err)
}
removed++
}
return removed, nil
}
package crawlid
import (
"encoding/base64"
"net/netip"
)
// Binding is the client unit a token and the detector's distribution tests
// use: the full IPv4 address, or the IPv6 /64. IPv4-mapped IPv6 is IPv4.
type Binding string
// BindingOf parses a textual client address. Zone-scoped, malformed or
// port-suffixed input is rejected.
func BindingOf(ip string) (Binding, bool) {
a, err := netip.ParseAddr(ip)
if err != nil || a.Zone() != "" {
return "", false
}
a = a.Unmap()
if a.Is4() {
b := a.As4()
return Binding("4" + string(b[:])), true
}
b := a.As16()
return Binding("6" + string(b[:8])), true
}
// String is the unpadded base64url form used in JSON and vectors.
func (b Binding) String() string {
return base64.RawURLEncoding.EncodeToString([]byte(b))
}
package crawlid
// StaticExtensions are path extensions served as static assets. Requests for
// them are neither dynamic nor expensive. http_asn_crawl uses this same list.
var StaticExtensions = map[string]struct{}{
"jpg": {}, "jpeg": {}, "png": {}, "gif": {}, "webp": {}, "svg": {}, "ico": {},
"bmp": {}, "css": {}, "js": {}, "mjs": {}, "map": {}, "woff": {}, "woff2": {},
"ttf": {}, "eot": {}, "otf": {}, "mp4": {}, "webm": {}, "ogg": {}, "mp3": {},
"pdf": {}, "zip": {}, "gz": {}, "avif": {},
}
// Class is the cost heuristic for one request. Dynamic means a GET or HEAD
// for a non-static extension; Expensive additionally needs a non-empty query.
// It is a heuristic, not proof of a cache miss or of PHP execution.
type Class struct {
Dynamic bool
Expensive bool
}
// Classify applies the heuristic. Methods are case-sensitive, as in HTTP.
func Classify(method string, t Target) Class {
if method != "GET" && method != "HEAD" {
return Class{}
}
if _, static := StaticExtensions[t.Ext]; static {
return Class{}
}
return Class{Dynamic: true, Expensive: t.HasQuery}
}
package crawlid
import (
"bytes"
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"errors"
)
// ErrBadKey reports an encoded key that does not decode under this version.
var ErrBadKey = errors.New("crawlid: malformed encoded key")
// Level selects the fields that participate in identity.
type Level uint8
const (
L1 Level = 1 // site, first segment, exact parameter-name set
L2 Level = 2 // site, first segment, non-empty query
L3 Level = 3 // site, any dynamic GET/HEAD
)
// Key identifies one detector counter and one gate policy predicate. Site is
// the canonical site identity from the verified inventory, never a raw Host
// header. Components are borrowed and must remain immutable while used.
type Key struct {
Level Level
Site []byte
Segment []byte
Names [][]byte
}
// KeysFor returns every level a request contributes to, broadest first.
func KeysFor(site []byte, c Class, t Target) []Key {
if !c.Dynamic {
return nil
}
keys := []Key{{Level: L3, Site: site}}
if c.Expensive {
keys = append(keys,
Key{Level: L2, Site: site, Segment: t.Segment},
Key{Level: L1, Site: site, Segment: t.Segment, Names: t.Names})
}
return keys
}
const keyMagic = "ck"
// Encode is the versioned, length-delimited binary form. Every component is
// uvarint-length-prefixed, so no byte value inside a component can forge a
// boundary.
func (k Key) Encode() []byte {
b := append([]byte(keyMagic), byte(Version), byte(k.Level))
b = appendField(b, k.Site)
if k.Level == L1 || k.Level == L2 {
b = appendField(b, k.Segment)
}
if k.Level == L1 {
b = binary.AppendUvarint(b, uint64(len(k.Names)))
for _, n := range k.Names {
b = appendField(b, n)
}
}
return b
}
// ID is a stable short display identifier, not an equality or authority check.
func (k Key) ID() string {
h := sha256.Sum256(k.Encode())
return base64.RawURLEncoding.EncodeToString(h[:16])
}
// DecodeKey parses Encode output and rejects anything else.
func DecodeKey(b []byte) (Key, error) {
if len(b) < 4 || string(b[:2]) != keyMagic || b[2] != Version {
return Key{}, ErrBadKey
}
k := Key{Level: Level(b[3])}
if k.Level < L1 || k.Level > L3 {
return Key{}, ErrBadKey
}
rest := b[4:]
var ok bool
if k.Site, rest, ok = readField(rest); !ok {
return Key{}, ErrBadKey
}
if k.Level != L3 {
if k.Segment, rest, ok = readField(rest); !ok {
return Key{}, ErrBadKey
}
}
if k.Level == L1 {
n, w := binary.Uvarint(rest)
// #nosec G115 -- positive Uvarint width is at most len(rest); the difference is nonnegative.
if w <= 0 || n > uint64(len(rest)-w) {
return Key{}, ErrBadKey
}
rest = rest[w:]
k.Names = make([][]byte, 0, n)
for i := uint64(0); i < n; i++ {
var f []byte
if f, rest, ok = readField(rest); !ok {
return Key{}, ErrBadKey
}
if len(k.Names) > 0 && bytes.Compare(k.Names[len(k.Names)-1], f) >= 0 {
return Key{}, ErrBadKey
}
k.Names = append(k.Names, f)
}
}
if len(rest) != 0 || !bytes.Equal(k.Encode(), b) {
return Key{}, ErrBadKey
}
return k, nil
}
func appendField(b, f []byte) []byte {
b = binary.AppendUvarint(b, uint64(len(f)))
return append(b, f...)
}
func readField(b []byte) (field, rest []byte, ok bool) {
n, w := binary.Uvarint(b)
// #nosec G115 -- positive Uvarint width is at most len(b); the difference is nonnegative.
if w <= 0 || n > uint64(len(b)-w) {
return nil, nil, false
}
// #nosec G115 -- n <= len(b)-w above, so both conversion and sum fit int.
end := w + int(n)
return b[w:end:end], b[end:], true
}
package crawlid
import (
"bytes"
"errors"
"sort"
"strings"
)
var (
ErrEmptyTarget = errors.New("crawlid: empty target")
ErrUnsupportedForm = errors.New("crawlid: target is neither origin-form nor http(s) absolute-form")
ErrTargetTooLong = errors.New("crawlid: target exceeds bound")
)
// Target is the canonical identity of one request target.
type Target struct {
Segment []byte // first path segment, percent-decoded once, case kept
HasQuery bool // a non-empty raw query follows the first '?'
Names [][]byte // normalized parameter names, sorted, deduplicated
Ext string // lower-case extension of the raw final path element
}
// ParseTarget canonicalizes a raw target as it appears in the request line or
// PHP's REQUEST_URI. A target longer than maxLen is an explicit overflow:
// callers must not build an identity from a prefix. The bound counts every
// raw byte, including an absolute-form scheme and authority.
func ParseTarget(raw string, maxLen int) (Target, error) {
switch {
case raw == "":
return Target{}, ErrEmptyTarget
case len(raw) > maxLen:
return Target{}, ErrTargetTooLong
}
p, q, ok := SplitTarget(raw)
if !ok {
return Target{}, ErrUnsupportedForm
}
seg := p[1:]
if i := strings.IndexByte(seg, '/'); i >= 0 {
seg = seg[:i]
}
t := Target{Segment: percentDecode(seg, false), Ext: PathExtension(p)}
if q != "" {
t.HasQuery = true
t.Names = queryNames(q)
}
return t, nil
}
// PathExtension mirrors path.Ext on the raw, undecoded path: the text after the last
// '.' of the final '/'-separated element, ASCII lower-cased.
func PathExtension(p string) string {
elem := p[strings.LastIndexByte(p, '/')+1:]
dot := strings.LastIndexByte(elem, '.')
if dot < 0 {
return ""
}
return string(asciiLower([]byte(elem[dot+1:])))
}
// queryNames splits fields at raw '&' and names at the first raw '=' before
// any decoding. Empty fields are ignored; an explicitly empty name is kept.
// Values are never decoded or retained.
func queryNames(q string) [][]byte {
seen := make(map[string]struct{})
var out [][]byte
for q != "" {
field, rest, _ := strings.Cut(q, "&")
q = rest
if field == "" {
continue
}
name, _, _ := strings.Cut(field, "=")
n := normalizeBrackets(asciiLower(percentDecode(name, true)))
if _, dup := seen[string(n)]; dup {
continue
}
seen[string(n)] = struct{}{}
out = append(out, n)
}
sort.Slice(out, func(i, j int) bool { return bytes.Compare(out[i], out[j]) < 0 })
return out
}
// percentDecode decodes each valid %XX escape once. Invalid or short escapes
// stay literal. With plusSpace, '+' becomes a space (query names only).
func percentDecode(s string, plusSpace bool) []byte {
out := make([]byte, 0, len(s))
for i := 0; i < len(s); i++ {
c := s[i]
switch {
case c == '%' && i+2 < len(s) && isHex(s[i+1]) && isHex(s[i+2]):
out = append(out, unhex(s[i+1])<<4|unhex(s[i+2]))
i += 2
case c == '+' && plusSpace:
out = append(out, ' ')
default:
out = append(out, c)
}
}
return out
}
// normalizeBrackets rewrites every empty or all-digit component of a complete
// trailing bracket chain to "[]": a[0][x] -> a[][x]. A chain needs a non-empty
// base; named components and incomplete chains stay literal.
func normalizeBrackets(n []byte) []byte {
first := bytes.IndexByte(n, '[')
if first <= 0 || bytes.IndexByte(n[:first], ']') >= 0 {
return n
}
out := append([]byte{}, n[:first]...)
for pos := first; pos < len(n); {
if n[pos] != '[' {
return n
}
closeAt := bytes.IndexByte(n[pos+1:], ']')
if closeAt < 0 {
return n
}
end := pos + 1 + closeAt
inner := n[pos+1 : end]
if bytes.IndexByte(inner, '[') >= 0 {
return n
}
out = append(out, '[')
if len(inner) != 0 && !allDigits(inner) {
out = append(out, inner...)
}
out = append(out, ']')
pos = end + 1
}
return out
}
func asciiLower(b []byte) []byte {
out := make([]byte, len(b))
for i, c := range b {
if 'A' <= c && c <= 'Z' {
c += 'a' - 'A'
}
out[i] = c
}
return out
}
func allDigits(b []byte) bool {
for _, c := range b {
if c < '0' || c > '9' {
return false
}
}
return true
}
func isHex(c byte) bool {
return ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')
}
func unhex(c byte) byte {
switch {
case c <= '9':
return c - '0'
case c <= 'F':
return c - 'A' + 10
default:
return c - 'a' + 10
}
}
// SplitTarget returns the raw path and query of an origin-form target or of
// an http or https absolute-form target (scheme compared in ASCII case only).
// Servers must accept absolute-form and hand it to PHP unchanged, so without
// this a client could choose it to leave its identity. The authority is
// dropped: the site comes from verified inventory, never from the request.
// A target without a path, including one that starts with '?', is served at
// the root path. Any other form, such as asterisk, authority-form, another
// scheme or a rootless path, is unsupported.
func SplitTarget(raw string) (path, query string, ok bool) {
rest := raw
if !strings.HasPrefix(raw, "/") && !strings.HasPrefix(raw, "?") {
scheme, after, found := strings.Cut(raw, ":")
if !found || !isHTTPScheme(scheme) {
return "", "", false
}
rest = after
if authority, hasAuthority := strings.CutPrefix(rest, "//"); hasAuthority {
rest = ""
if end := strings.IndexAny(authority, "/?"); end >= 0 {
rest = authority[end:]
}
}
if rest != "" && rest[0] != '/' && rest[0] != '?' {
return "", "", false
}
}
path, query, _ = strings.Cut(rest, "?")
if path == "" {
path = "/"
}
return path, query, true
}
func isHTTPScheme(s string) bool {
if len(s) != len("http") && len(s) != len("https") {
return false
}
lower := string(asciiLower([]byte(s)))
return lower == "http" || lower == "https"
}
package crawlreplay
import (
"errors"
"math"
)
// ErrParams reports a parameter set outside the detector's domain.
var ErrParams = errors.New("crawlreplay: invalid parameters")
// BaselineParams are the hour-of-week profile settings phase 1 freezes.
type BaselineParams struct {
Alpha float64 `json:"alpha"` // EWMA weight of each new complete minute, in (0,1]
MinObs int `json:"min_obs"` // observed minutes a slot needs before it is trusted
MinAge int64 `json:"min_age"` // minutes after a key's first observation before any slot is trusted
FloorPerMin float64 `json:"floor_per_min"` // fixed host-profile floor, requests per minute
}
// Validate checks the domain constraints; it approves no tuning value.
func (p BaselineParams) Validate() error {
switch {
case math.IsNaN(p.Alpha) || p.Alpha <= 0 || p.Alpha > 1,
p.MinObs < 1, p.MinAge < 0,
math.IsNaN(p.FloorPerMin) || math.IsInf(p.FloorPerMin, 0) || p.FloorPerMin <= 0:
return ErrParams
}
return nil
}
type slot struct {
mean float64
obs int
last int64 // minute of the latest observation
}
// Baseline is one key's UTC hour-of-week profile of its total eligible
// non-infrastructure request rate, per minute.
type Baseline struct {
p BaselineParams
first int64 // first observed minute
learned int64 // latest observed minute: each minute is learned at most once
slots [168]slot
}
// NewBaseline starts a cold profile for a key first observed at minute first.
func NewBaseline(p BaselineParams, first int64) *Baseline {
return &Baseline{p: p, first: first}
}
// hourOfWeek maps a Unix minute to its UTC hour-of-week slot, Monday 00:00
// first. The Unix epoch began on a Thursday, three days after a Monday.
func hourOfWeek(minute int64) int {
return int((minute/60 + 72) % 168)
}
// Expected is the per-minute expectation for minute m. Until the key is old
// enough and the slot has enough observations, and whenever the learned
// expectation is zero, the fixed floor applies.
func (b *Baseline) Expected(m int64) float64 {
v, _ := b.expected(m)
return v
}
func (b *Baseline) expected(m int64) (float64, bool) {
return expected(b.p, b.first, m, &b.slots[hourOfWeek(m)])
}
// expected returns the expectation for a slot judged at minute age, and
// whether it came from the trained slot rather than the floor.
func expected(p BaselineParams, first, age int64, s *slot) (float64, bool) {
if age-first < p.MinAge || s.obs < p.MinObs || s.mean <= 0 {
return p.FloorPerMin, false
}
return s.mean, true
}
// Observe folds one complete minute's total into its slot and reports
// whether it learned. A minute at or before the latest learned one is
// refused, so no minute counts twice. Callers skip minutes the detector
// freezes and never pass a missing minute as zero.
func (b *Baseline) Observe(m, total int64) bool {
if m <= b.learned {
return false
}
s := &b.slots[hourOfWeek(m)]
if s.obs == 0 {
s.mean = float64(total)
} else {
// Weight the nonnegative terms separately so a small new total
// is not lost when subtracting it from a much larger old mean.
s.mean = (1-b.p.Alpha)*s.mean + b.p.Alpha*float64(total)
}
s.obs++
s.last, b.learned = m, m
return true
}
// Profile is a baseline pinned when an episode begins: its slots
// and trust as they were then, so neither later learning nor the key coming
// of age during the episode moves the expectation it is judged against.
type Profile struct {
p BaselineParams
first int64
at int64 // the pinning minute
slots [168]slot
}
// Pin copies the profile as it stands at minute at.
func (b *Baseline) Pin(at int64) Profile {
return Profile{p: b.p, first: b.first, at: at, slots: b.slots}
}
// Expected is the pinned expectation for minute m. Minutes before the pin
// keep the trust they had then; later minutes keep the trust of the pinning
// minute.
func (p *Profile) Expected(m int64) float64 {
v, _ := p.expected(m)
return v
}
func (p *Profile) expected(m int64) (float64, bool) {
return expected(p.p, p.first, min(m, p.at), &p.slots[hourOfWeek(m)])
}
package crawlreplay
import (
"bufio"
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"encoding/json/jsontext"
jsonv2 "encoding/json/v2"
"errors"
"fmt"
"io"
"math"
"regexp"
"strings"
)
// ManifestVersion is the bundle manifest format. Version 1 called an
// observed first-to-last extent coverage. Version 2 lacked identity
// digests, so bundles converted against a forked or restored identity
// registry could share a pseudonym for two names unnoticed. Both are
// refused.
const ManifestVersion = 3
// Categories of lines no minute holds, as the manifest and proofs name them.
// LossNoTarget is timed: its lines are records whose volume row counts them.
const (
LossOversized = "oversized"
LossRejected = "rejected"
LossTimeInvalid = "time_invalid"
LossTimeFuture = "time_future"
LossIncomplete = "incomplete"
LossNoTarget = "no_target"
)
var (
// ErrManifest reports a manifest that is not canonical or breaks the
// format. Messages name the field, never the value.
ErrManifest = errors.New("crawlreplay: invalid manifest")
// ErrBundle reports bundle files or rows that do not match their manifest.
ErrBundle = errors.New("crawlreplay: bundle does not match its manifest")
// ErrStrictJSON reports a document outside its closed JSON form. It
// carries no input bytes.
ErrStrictJSON = errors.New("crawlreplay: JSON is not in its closed form")
accountPseudonymRe = regexp.MustCompile(`^acct-[0-9a-f]{6}$`)
sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`)
identityName = regexp.MustCompile(`^(?:dom-([0-9a-f]{6})\.example|acct-([0-9a-f]{6})|e-([0-9a-f]{16}))$`)
saltPrint = regexp.MustCompile(`^[0-9a-f]{12}$`)
revisionHex = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`)
)
// ToolRevision is the source revision a bundle tool was built from.
type ToolRevision struct {
Revision string `json:"revision"`
Dirty bool `json:"dirty"`
GoVersion string `json:"go_version"`
}
// Clean reports a known, unmodified source revision.
func (t ToolRevision) Clean() bool { return !t.Dirty && revisionHex.MatchString(t.Revision) }
// Digest identifies a file by content, never by path.
type Digest struct {
SHA256 string `json:"sha256"`
Bytes int64 `json:"bytes"`
}
func (d Digest) valid() bool { return sha256Hex.MatchString(d.SHA256) && d.Bytes >= 0 }
// BotEvidenceRef identifies the verified-bot evidence a bundle used and the
// verified-bot list revision and configuration it came from.
type BotEvidenceRef struct {
Digest
D2Revision string `json:"d2_revision"`
ConfigSHA256 string `json:"config_sha256"`
}
// Input is one log copy: its bytes as read and its decompressed content.
// DisorderSeconds is the most any timed line's logged time trails an earlier
// timed line of the copy. A copy lists requests in completion order, so this
// is a lower bound on how late a request can be written.
type Input struct {
Site string `json:"site"`
Ordinal int `json:"ordinal"`
SHA256 string `json:"sha256"`
Bytes int64 `json:"bytes"`
ContentSHA256 string `json:"content_sha256"`
ContentBytes int64 `json:"content_bytes"`
Extent *Span `json:"extent,omitempty"`
DisorderSeconds int64 `json:"disorder_seconds"`
}
// Output is one bundle file with its row count.
type Output struct {
Kind string `json:"kind"`
SHA256 string `json:"sha256"`
Bytes int64 `json:"bytes"`
Rows int64 `json:"rows"`
}
// UntimedLoss counts lines of one input and category that carry no usable
// time, between the timed lines logged before and after them (Unix seconds,
// 0 at an end of the input). Logs are written at request completion, so a
// lost request's time lies within those bounds widened by the lateness bound
// a coverage proof supplies.
type UntimedLoss struct {
Input int `json:"input"`
Category string `json:"category"`
After int64 `json:"after"`
Before int64 `json:"before"`
Lines int64 `json:"lines"`
}
// Identity is a site, account or episode pseudonym a bundle issued with the
// full keyed digest it is a prefix of. One pseudonym with two digests is two
// names that one registry would have refused; bundles combined in one
// experiment must agree on every digest.
type Identity struct {
Pseudonym string `json:"pseudonym"`
Digest string `json:"digest"`
}
// valid reports a closed pseudonym whose hexadecimal part begins its digest.
func (id Identity) valid() bool {
m := identityName.FindStringSubmatch(id.Pseudonym)
return m != nil && sha256Hex.MatchString(id.Digest) && strings.HasPrefix(id.Digest, m[1]+m[2]+m[3])
}
// SiteManifest is what the converter observed for one site. Every line read
// is a record or exactly one refused category; record bytes are in volume
// rows and every other byte is unplaced. Extent is diagnostic only.
type SiteManifest struct {
Site string `json:"site"`
Account string `json:"account"`
Extent *Span `json:"extent,omitempty"`
Bytes int64 `json:"bytes"`
UnplacedBytes int64 `json:"unplaced_bytes"`
Lines int64 `json:"lines"`
Records int64 `json:"records"`
Oversized int64 `json:"oversized"`
Rejected int64 `json:"rejected"`
TimeInvalid int64 `json:"time_invalid"`
TimeFuture int64 `json:"time_future"`
OutOfPeriod int64 `json:"out_of_period"`
Incomplete int64 `json:"incomplete"`
NoTarget int64 `json:"no_target"`
AttributionLoss int64 `json:"attribution_loss"`
InvalidClient int64 `json:"invalid_client"`
Infrastructure int64 `json:"infrastructure"`
Labels map[string]int64 `json:"labels"`
Untimed []UntimedLoss `json:"untimed"`
}
// Manifest describes one bundle: provenance, the recording period (inclusive
// Unix minutes), every input and output by digest, and per-site counts.
type Manifest struct {
FormatVersion int `json:"format_version"`
StreamVersion int `json:"stream_version"`
IdentityVersion int `json:"identity_version"`
Tool ToolRevision `json:"tool"`
SaltFingerprint string `json:"salt_fingerprint"`
Period Span `json:"period"`
Inventory Digest `json:"inventory"`
Labels *Digest `json:"labels,omitempty"`
BotEvidence *BotEvidenceRef `json:"bot_evidence,omitempty"`
Inputs []Input `json:"inputs"`
Outputs []Output `json:"outputs"`
Sites []SiteManifest `json:"sites"`
// Identities lists every site, account and episode pseudonym of the
// bundle, ordered by pseudonym.
Identities []Identity `json:"identities"`
digest string
}
// Digest returns the SHA-256 of the manifest bytes DecodeManifest read, the
// value a coverage proof binds to; it is empty for a manifest built in memory.
func (m Manifest) Digest() string { return m.digest }
func manifestError(field string) error { return fmt.Errorf("%w: %s", ErrManifest, field) }
// EncodeManifest validates m and returns its only accepted encoding.
func EncodeManifest(m Manifest) ([]byte, error) {
if err := m.Validate(); err != nil {
return nil, err
}
b, err := json.MarshalIndent(m, "", " ")
if err != nil {
return nil, manifestError("encoding")
}
return append(b, '\n'), nil
}
// DecodeManifest accepts only the exact bytes EncodeManifest produces, so
// unknown, duplicate, differently spelled, null or missing fields, trailing
// data and hand reformatting are all refused.
func DecodeManifest(raw []byte) (Manifest, error) {
var m Manifest
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&m); err != nil {
return Manifest{}, manifestError("syntax")
}
canonical, err := EncodeManifest(m)
if err != nil {
return Manifest{}, err
}
if !bytes.Equal(canonical, raw) {
return Manifest{}, manifestError("not canonical")
}
sum := sha256.Sum256(raw)
m.digest = hex.EncodeToString(sum[:])
return m, nil
}
// Validate checks the manifest against the closed format and its internal
// totals; it does not read the bundle's rows.
func (m Manifest) Validate() error {
switch {
case m.FormatVersion != ManifestVersion:
return manifestError("format_version")
case m.StreamVersion != StreamVersion:
return manifestError("stream_version")
case m.IdentityVersion < 1:
return manifestError("identity_version")
case !m.Tool.Clean():
return manifestError("tool")
case !saltPrint.MatchString(m.SaltFingerprint):
return manifestError("salt_fingerprint")
case m.Period.From <= 0 || m.Period.To < m.Period.From:
return manifestError("period")
case !m.Inventory.valid() || m.Inventory.Bytes == 0:
return manifestError("inventory")
case m.Labels != nil && (!m.Labels.valid() || m.Labels.Bytes == 0):
return manifestError("labels")
case m.BotEvidence != nil && (!m.BotEvidence.valid() || m.BotEvidence.Bytes == 0 ||
!revisionHex.MatchString(m.BotEvidence.D2Revision) || !sha256Hex.MatchString(m.BotEvidence.ConfigSHA256)):
return manifestError("bot_evidence")
case len(m.Sites) == 0:
return manifestError("sites")
}
kinds := map[string]bool{}
for _, o := range m.Outputs {
if (o.Kind != "records" && o.Kind != "volume") || kinds[o.Kind] || !sha256Hex.MatchString(o.SHA256) || o.Bytes < 0 || o.Rows < 0 {
return manifestError("outputs")
}
kinds[o.Kind] = true
}
if len(kinds) != 2 {
return manifestError("outputs")
}
inputs := map[string][]Input{}
read := map[string]int64{}
// Two copies with the same nonempty content would count their requests twice.
content := map[string]bool{}
for _, in := range m.Inputs {
switch {
case in.Ordinal != len(inputs[in.Site]), !sha256Hex.MatchString(in.SHA256), in.Bytes < 0,
!sha256Hex.MatchString(in.ContentSHA256), in.ContentBytes < 0,
in.ContentBytes > 0 && content[in.ContentSHA256],
in.Extent != nil && !within(*in.Extent, m.Period), in.DisorderSeconds < 0:
return manifestError("inputs")
}
inputs[in.Site] = append(inputs[in.Site], in)
var ok bool
if read[in.Site], ok = checkedSum(read[in.Site], in.ContentBytes); !ok {
return manifestError("inputs")
}
if in.ContentBytes > 0 {
content[in.ContentSHA256] = true
}
}
sites := map[string]bool{}
for i := range m.Sites {
s := &m.Sites[i]
if !ValidSite(s.Site) || sites[s.Site] || len(inputs[s.Site]) == 0 {
return manifestError("sites")
}
// Every decompressed byte of a site's copies is a line it counted.
if read[s.Site] != s.Bytes {
return manifestError("bytes")
}
sites[s.Site] = true
if err := s.validate(m.Period, inputs[s.Site]); err != nil {
return err
}
}
if len(inputs) != len(sites) {
return manifestError("inputs")
}
return m.validateIdentities()
}
// validateIdentities requires exactly the manifest's sites and accounts
// plus episode pseudonyms, which only the records can confirm.
func (m Manifest) validateIdentities() error {
named := map[string]bool{}
for i, id := range m.Identities {
if !id.valid() || (i > 0 && id.Pseudonym <= m.Identities[i-1].Pseudonym) {
return manifestError("identities")
}
named[id.Pseudonym] = true
}
used := map[string]bool{}
for _, s := range m.Sites {
if !named[s.Site] || !named[s.Account] {
return manifestError("identities")
}
used[s.Site], used[s.Account] = true, true
}
for _, id := range m.Identities {
if !used[id.Pseudonym] && !strings.HasPrefix(id.Pseudonym, "e-") {
return manifestError("identities")
}
}
return nil
}
func (s *SiteManifest) validate(period Span, inputs []Input) error {
counters := []int64{s.Bytes, s.UnplacedBytes, s.Lines, s.Records, s.Oversized, s.Rejected, s.TimeInvalid,
s.TimeFuture, s.OutOfPeriod, s.Incomplete, s.NoTarget, s.AttributionLoss, s.InvalidClient, s.Infrastructure}
for _, n := range counters {
if n < 0 {
return manifestError("counts")
}
}
lines, ok := checkedSum(s.Records, s.Oversized, s.Rejected, s.TimeInvalid, s.TimeFuture, s.OutOfPeriod, s.Incomplete)
unbound, okUnbound := checkedSum(s.AttributionLoss, s.InvalidClient)
switch {
case !accountPseudonymRe.MatchString(s.Account):
return manifestError("account")
case !ok || lines != s.Lines:
return manifestError("lines")
case !okUnbound || unbound > s.Records, s.NoTarget > s.Records, s.Infrastructure > s.Records:
return manifestError("records")
case s.UnplacedBytes > s.Bytes:
return manifestError("unplaced_bytes")
case s.Bytes < s.Lines, s.UnplacedBytes < s.Lines-s.Records,
s.Lines == s.Records && s.UnplacedBytes != 0:
return manifestError("line bytes")
case (s.Extent == nil) != (s.Records == 0), s.Extent != nil && !within(*s.Extent, period):
return manifestError("extent")
case s.Labels == nil || s.Untimed == nil:
return manifestError("site lists")
}
var labelled int64
for label, n := range s.Labels {
if (label != "" && label != LabelAttack && label != LabelHealthy && label != LabelOverload) || n < 1 {
return manifestError("labels")
}
if labelled, ok = checkedSum(labelled, n); !ok {
return manifestError("labels")
}
}
if labelled != s.Records {
return manifestError("labels")
}
untimed := map[string]int64{}
for _, u := range s.Untimed {
if _, known := untimedCategories[u.Category]; !known || u.Input < 0 || u.Input >= len(inputs) || u.Lines < 1 || u.After < 0 || u.Before < 0 {
return manifestError("untimed")
}
// Both neighbours are timed lines of this copy, even when outside
// the recording period and therefore absent from the record stream.
if u.Before > 0 && u.After-u.Before > inputs[u.Input].DisorderSeconds {
return manifestError("input disorder")
}
if untimed[u.Category], ok = checkedSum(untimed[u.Category], u.Lines); !ok {
return manifestError("untimed")
}
}
for category, counter := range map[string]int64{LossOversized: s.Oversized, LossRejected: s.Rejected,
LossTimeInvalid: s.TimeInvalid, LossTimeFuture: s.TimeFuture, LossIncomplete: s.Incomplete} {
if untimed[category] != counter {
return manifestError("untimed")
}
}
return nil
}
var untimedCategories = map[string]struct{}{LossOversized: {}, LossRejected: {}, LossTimeInvalid: {}, LossTimeFuture: {}, LossIncomplete: {}}
func within(s, outer Span) bool { return s.From >= outer.From && s.To <= outer.To && s.From <= s.To }
// checkedSum adds nonnegative counts, reporting overflow instead of wrapping.
func checkedSum(values ...int64) (int64, bool) {
var total int64
for _, v := range values {
if v < 0 || total > math.MaxInt64-v {
return 0, false
}
total += v
}
return total, true
}
// ReadBundleFile reads one bundle file exactly once. It hashes and counts
// every byte as it is read, decompresses a gzip file (recognized by its magic
// bytes), hands the content to fn and returns the digest of what was read,
// after checking that fn consumed the content to its end.
func ReadBundleFile(r io.Reader, fn func(io.Reader) error) (Digest, error) {
h := sha256.New()
counted := &countingReader{r: io.TeeReader(r, h)}
br := bufio.NewReader(counted)
var content io.Reader = br
head, err := br.Peek(2)
if err != nil && !errors.Is(err, io.EOF) {
// bufio clears the error it reports; the next read would retry.
return Digest{}, fmt.Errorf("%w: read", ErrBundle)
}
if len(head) == 2 && head[0] == 0x1f && head[1] == 0x8b {
zr, gzErr := gzip.NewReader(br)
if gzErr != nil {
return Digest{}, fmt.Errorf("%w: gzip", ErrBundle)
}
content = zr
}
if err = fn(content); err != nil {
return Digest{}, err
}
if n, copyErr := io.Copy(io.Discard, content); copyErr != nil || n != 0 {
return Digest{}, fmt.Errorf("%w: unread content", ErrBundle)
}
if _, err = io.Copy(io.Discard, br); err != nil {
return Digest{}, fmt.Errorf("%w: read", ErrBundle)
}
return Digest{SHA256: hex.EncodeToString(h.Sum(nil)), Bytes: counted.n}, nil
}
type countingReader struct {
r io.Reader
n int64
}
func (c *countingReader) Read(p []byte) (int, error) {
n, err := c.r.Read(p)
c.n += int64(n)
return n, err
}
// DecodeStrictJSON decodes exactly one JSON value into v. Member names must
// match exactly and occur once; unknown members, null values and trailing
// data are refused with ErrStrictJSON.
func DecodeStrictJSON(raw []byte, v any) error {
dec := jsontext.NewDecoder(bytes.NewReader(raw))
for {
tok, err := dec.ReadToken()
if errors.Is(err, io.EOF) {
break
}
if err != nil || tok.Kind() == 'n' {
return ErrStrictJSON
}
}
if err := jsonv2.Unmarshal(raw, v, jsonv2.RejectUnknownMembers(true)); err != nil {
return ErrStrictJSON
}
return nil
}
package crawlreplay
import (
"cmp"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"maps"
"math"
"slices"
"strings"
)
// ProofVersion is the coverage proof format.
const ProofVersion = 1
// Reasons a proof gives for minutes of the recording period it does not
// certify, and the reason ValidateBundle gives for certified minutes that a
// bundle's own loss accounting removes.
const (
ExcludedPartialMinute = "partial_minute"
ExcludedCollectionGap = "collection_gap"
ExcludedLivenessUnknown = "liveness_unknown"
ExcludedTopologyChange = "topology_change"
ExcludedUnknownLoss = "unknown_loss"
)
// Kinds of independent evidence a proof cites by digest.
const (
EvidenceCollection = "collection"
EvidenceLiveness = "logging_liveness"
EvidenceLateness = "lateness"
EvidencePreApplication = "pre_application"
)
// ErrProof reports a coverage proof that breaks its format or does not
// describe the bundle it names.
var ErrProof = errors.New("crawlreplay: invalid coverage proof")
func proofError(field string) error { return fmt.Errorf("%w: %s", ErrProof, field) }
// CoverageProof is the operator's certificate of complete minutes, built
// from evidence independent of the logs: the collection record, logging and
// handler liveness, and the measured completion delay. The logs alone can
// never certify a minute, least of all a silent one.
type CoverageProof struct {
FormatVersion int `json:"format_version"`
ManifestSHA256 string `json:"manifest_sha256"`
// LatenessSeconds bounds how long after its logged time a request can
// be written, which bounds where an untimed lost line belongs.
LatenessSeconds int64 `json:"lateness_seconds"`
Evidence []ProofEvidence `json:"evidence"`
Sites []ProofSite `json:"sites"`
digest string
contentDigest [sha256.Size]byte
}
// ProofEvidence names one independent evidence file by digest; its content
// stays with the operator.
type ProofEvidence struct {
Kind string `json:"kind"`
SHA256 string `json:"sha256"`
}
// ProofSite tiles the recording period for one site: certified spans and
// reasoned exclusions, disjoint, sorted and covering every minute.
type ProofSite struct {
Site string `json:"site"`
Spans []Span `json:"spans"`
Excluded []Exclusion `json:"excluded,omitempty"`
Rejects []RejectWaiver `json:"rejects,omitempty"`
}
// Exclusion is an inclusive range of minutes the proof cannot certify.
type Exclusion struct {
From int64 `json:"from"`
To int64 `json:"to"`
Reason string `json:"reason"`
}
// RejectWaiver states that exactly Lines lost lines of Category within an
// inclusive minute range were rejected before the application ran, as the
// cited pre-application evidence shows. The HTTP status alone is not that
// evidence. A waived line does not remove its minutes from coverage.
type RejectWaiver struct {
From int64 `json:"from"`
To int64 `json:"to"`
Category string `json:"category"`
Lines int64 `json:"lines"`
Evidence string `json:"evidence"`
}
// Digest returns the SHA-256 of the proof bytes DecodeCoverageProof read.
func (p *CoverageProof) Digest() string { return p.digest }
// DecodeCoverageProof decodes and checks a proof's closed format.
func DecodeCoverageProof(raw []byte) (*CoverageProof, error) {
var p CoverageProof
if err := DecodeStrictJSON(raw, &p); err != nil {
return nil, proofError("syntax")
}
if err := p.Validate(); err != nil {
return nil, err
}
sum := sha256.Sum256(raw)
p.digest = hex.EncodeToString(sum[:])
canonical, err := json.Marshal(p)
if err != nil {
return nil, proofError("encoding")
}
p.contentDigest = sha256.Sum256(canonical)
return &p, nil
}
var (
exclusionReasons = map[string]bool{ExcludedPartialMinute: true, ExcludedCollectionGap: true, ExcludedLivenessUnknown: true, ExcludedTopologyChange: true}
evidenceKinds = map[string]bool{EvidenceCollection: true, EvidenceLiveness: true, EvidenceLateness: true, EvidencePreApplication: true}
waivable = map[string]bool{LossNoTarget: true, LossOversized: true}
)
// Validate checks the proof's own format; ValidateBundle checks it against
// a manifest.
func (p *CoverageProof) Validate() error {
if p.digest != "" {
// Proofs permit operator formatting, so retain both the original
// byte digest and a formatting-independent seal of the decoded data.
canonical, err := json.Marshal(p)
if err != nil || sha256.Sum256(canonical) != p.contentDigest {
return proofError("digest")
}
}
switch {
case p.FormatVersion != ProofVersion:
return proofError("format_version")
case !sha256Hex.MatchString(p.ManifestSHA256):
return proofError("manifest_sha256")
case p.LatenessSeconds < 1 || p.LatenessSeconds > math.MaxInt32:
return proofError("lateness_seconds")
case len(p.Sites) == 0:
return proofError("sites")
}
kinds := map[string]bool{}
preApplication := map[string]bool{}
seen := map[ProofEvidence]bool{}
for _, e := range p.Evidence {
if !evidenceKinds[e.Kind] || !sha256Hex.MatchString(e.SHA256) || seen[e] {
return proofError("evidence")
}
seen[e], kinds[e.Kind] = true, true
if e.Kind == EvidencePreApplication {
preApplication[e.SHA256] = true
}
}
if !kinds[EvidenceCollection] || !kinds[EvidenceLiveness] || !kinds[EvidenceLateness] {
return proofError("evidence")
}
sites := map[string]bool{}
for _, s := range p.Sites {
if !ValidSite(s.Site) || sites[s.Site] || s.Spans == nil {
return proofError("sites")
}
sites[s.Site] = true
for i, span := range s.Spans {
if span.From < 1 || span.To < span.From || (i > 0 && span.From <= s.Spans[i-1].To) {
return proofError("spans")
}
}
for _, e := range s.Excluded {
if !exclusionReasons[e.Reason] || e.From < 1 || e.To < e.From {
return proofError("excluded")
}
}
for i, w := range s.Rejects {
if !waivable[w.Category] || !preApplication[w.Evidence] || w.Lines < 1 || w.From < 1 || w.To < w.From {
return proofError("rejects")
}
for _, other := range s.Rejects[:i] {
if other.Category == w.Category && other.From <= w.To && w.From <= other.To {
return proofError("rejects")
}
}
}
}
return nil
}
// BundleInput is a decoded manifest, an optional proof and the two outputs.
type BundleInput struct {
Manifest Manifest // from DecodeManifest
Proof *CoverageProof // nil: coverage is unqualified
Volume io.Reader
Records io.Reader
}
// BundleVisitor receives a bundle's contents as ValidateBundle checks them:
// every volume row, then the validated sites, then every record in stream
// order. Anything built from them must be discarded if ValidateBundle
// returns an error, which it can do after the last callback.
// Sites receives an owned copy, independent of validation and return values.
type BundleVisitor struct {
Volume func(Volume) error
Sites func([]BundleSite) error
Record func(Record) error
}
// BundleSite is one validated site. Extent is the observed first and last
// placed minute. Certified is what the proof certifies and Coverage what a
// replay may use: certified minutes less any with unknown loss. Both are nil
// without a proof; an extent is never coverage.
type BundleSite struct {
Site string
Account string
Extent *Span
Records int64
Certified []Span
Coverage []Span
// Excluded counts period minutes outside Coverage by reason.
Excluded map[string]int64
}
// ValidateBundle checks a bundle against its manifest and, when present,
// the coverage proof bound to that manifest, reading each output once. It
// returns every manifest site, including sites without records. Errors
// wrap ErrManifest, ErrProof, ErrBundle, ErrRecord or ErrDecode and name
// fields only.
func ValidateBundle(in BundleInput, identityVersion int, v BundleVisitor) ([]BundleSite, error) {
c, err := newBundleCheck(in.Manifest, in.Proof, identityVersion)
if err != nil {
return nil, err
}
volume, err := ReadBundleFile(in.Volume, func(r io.Reader) error {
return ReadVolume(r, func(row Volume) error {
if rowErr := c.volume(row); rowErr != nil {
return rowErr
}
if v.Volume != nil {
return v.Volume(row)
}
return nil
})
})
if err != nil {
return nil, err
}
sites, err := c.siteList()
if err != nil {
return nil, err
}
if v.Sites != nil {
if err = v.Sites(cloneBundleSites(sites)); err != nil {
return nil, err
}
}
records, err := ReadBundleFile(in.Records, func(r io.Reader) error {
return ReadRecords(r, func(rec Record) error {
if rowErr := c.record(rec); rowErr != nil {
return rowErr
}
if v.Record != nil {
return v.Record(rec)
}
return nil
})
})
if err != nil {
return nil, err
}
if err = c.finish(records, volume); err != nil {
return nil, err
}
return sites, nil
}
func cloneBundleSites(sites []BundleSite) []BundleSite {
out := slices.Clone(sites)
for i := range out {
s := &out[i]
if s.Extent != nil {
extent := *s.Extent
s.Extent = &extent
}
s.Certified = slices.Clone(s.Certified)
s.Coverage = slices.Clone(s.Coverage)
s.Excluded = maps.Clone(s.Excluded)
}
return out
}
type siteMinute struct {
site string
minute int64
}
type siteState struct {
m *SiteManifest
proof *ProofSite
inputs []inputState
extent *Span
done bool
seq int64
file int
// From records.
records, unbound, infra int64
labels map[string]int64
lines, unboundLines map[int64]int64
// From volume rows.
volumeLines, volumeUnbound, volumeNoTarget map[int64]int64
volumeBytes int64
}
type inputState struct {
m Input
extent *Span
latest int64
disorder int64
}
type bundleCheck struct {
m Manifest
episodes map[string]bool // declared episode pseudonym -> seen in a record
proof *CoverageProof
sites map[string]*siteState
volumes map[siteMinute]bool
l1Parent map[string]string
l2Site map[string]string
current *siteState
vRows int64
rRows int64
}
func bundleError(field string) error { return fmt.Errorf("%w: %s", ErrBundle, field) }
func newBundleCheck(m Manifest, proof *CoverageProof, identityVersion int) (*bundleCheck, error) {
// Exported fields can change after decoding. Rebind the canonical bytes
// and take an owned snapshot before callbacks can touch the caller's data.
raw, err := EncodeManifest(m)
if err != nil {
return nil, err
}
digest := m.digest
m, err = DecodeManifest(raw)
if err != nil {
return nil, err
}
if m.IdentityVersion != identityVersion {
return nil, manifestError("identity_version")
}
if digest != m.digest {
return nil, manifestError("digest")
}
c := &bundleCheck{m: m, proof: proof, sites: map[string]*siteState{}, volumes: map[siteMinute]bool{},
l1Parent: map[string]string{}, l2Site: map[string]string{}, episodes: map[string]bool{}}
for _, id := range m.Identities {
if strings.HasPrefix(id.Pseudonym, "e-") {
c.episodes[id.Pseudonym] = false
}
}
for i := range m.Sites {
c.sites[m.Sites[i].Site] = &siteState{m: &m.Sites[i], labels: map[string]int64{},
lines: map[int64]int64{}, unboundLines: map[int64]int64{},
volumeLines: map[int64]int64{}, volumeUnbound: map[int64]int64{}, volumeNoTarget: map[int64]int64{}}
}
for _, in := range m.Inputs {
s := c.sites[in.Site]
s.inputs = append(s.inputs, inputState{m: in})
}
if proof == nil {
return c, nil
}
if err := proof.Validate(); err != nil {
return nil, err
}
snapshot := *proof
snapshot.Evidence = slices.Clone(proof.Evidence)
snapshot.Sites = slices.Clone(proof.Sites)
for i := range snapshot.Sites {
s := &snapshot.Sites[i]
s.Spans = slices.Clone(s.Spans)
s.Excluded = slices.Clone(s.Excluded)
s.Rejects = slices.Clone(s.Rejects)
}
proof = &snapshot
c.proof = proof
if proof.ManifestSHA256 != m.digest {
return nil, proofError("manifest_sha256")
}
// A bound below the disorder the copies themselves show is false, or
// the copies are not in completion order; either way no bracket built
// from it can be trusted.
for _, in := range m.Inputs {
if in.DisorderSeconds > proof.LatenessSeconds {
return nil, proofError("lateness_seconds")
}
}
if len(proof.Sites) != len(m.Sites) {
return nil, proofError("sites")
}
for i := range proof.Sites {
ps := &proof.Sites[i]
s := c.sites[ps.Site]
if s == nil || s.proof != nil {
return nil, proofError("sites")
}
s.proof = ps
if err := tiles(ps, m.Period); err != nil {
return nil, err
}
}
return c, nil
}
// tiles checks that spans and exclusions cover the period exactly once.
func tiles(ps *ProofSite, period Span) error {
parts := slices.Clone(ps.Spans)
for _, e := range ps.Excluded {
parts = append(parts, Span{From: e.From, To: e.To})
}
slices.SortFunc(parts, func(a, b Span) int { return cmp.Compare(a.From, b.From) })
next := period.From
for _, s := range parts {
if s.From != next || s.To > period.To {
return proofError("period")
}
next = s.To + 1
}
if next != period.To+1 {
return proofError("period")
}
for _, w := range ps.Rejects {
if !within(Span{From: w.From, To: w.To}, period) {
return proofError("rejects")
}
}
return nil
}
func (c *bundleCheck) placed(s *siteState, minute int64) bool {
return s.m.Extent != nil && minute >= s.m.Extent.From && minute <= s.m.Extent.To
}
func (c *bundleCheck) volume(v Volume) error {
s := c.sites[v.Site]
key := siteMinute{v.Site, v.Minute}
switch {
case s == nil:
return bundleError("volume site")
case c.volumes[key]:
return bundleError("duplicate volume row")
case !c.placed(s, v.Minute):
return bundleError("volume minute")
}
c.volumes[key] = true
c.vRows++
var ok bool
if s.volumeBytes, ok = checkedSum(s.volumeBytes, v.Bytes); !ok {
return bundleError("volume bytes")
}
s.volumeLines[v.Minute] = v.Lines
s.volumeUnbound[v.Minute] = v.NoBinding
s.volumeNoTarget[v.Minute] = v.NoTarget
return nil
}
func (c *bundleCheck) record(r Record) error {
s := c.sites[r.Site]
if s == nil {
return bundleError("record site")
}
if s != c.current {
if c.current != nil {
c.current.done = true
}
if s.done {
return bundleError("site order")
}
c.current = s
}
minute := r.T / 60
switch {
case r.Account != s.m.Account:
return bundleError("account")
case r.BotProof != "" && c.m.BotEvidence == nil:
return bundleError("bot evidence")
case r.Episode != "" && !c.declared(r.Episode):
return bundleError("episode identity")
case r.Seq <= s.seq, r.Seq > s.m.Lines, r.File < s.file, r.File >= len(s.inputs):
return bundleError("file order")
case !c.placed(s, minute):
return bundleError("record minute")
}
if r.L2 != "" {
if site, ok := c.l2Site[r.L2]; ok && site != r.Site {
return bundleError("l2 site")
}
c.l2Site[r.L2] = r.Site
if parent, ok := c.l1Parent[r.L1]; ok && parent != r.L2 {
return bundleError("l1 parent")
}
c.l1Parent[r.L1] = r.L2
}
s.seq, s.file = r.Seq, r.File
observeMinute(&s.extent, minute)
in := &s.inputs[r.File]
observeMinute(&in.extent, minute)
if r.T < in.latest {
in.disorder = max(in.disorder, in.latest-r.T)
} else {
in.latest = r.T
}
c.rRows++
s.records++
s.lines[minute]++
s.labels[r.Label]++
if r.Binding == "" {
s.unbound++
s.unboundLines[minute]++
}
if r.Infra {
s.infra++
}
return nil
}
func observeMinute(extent **Span, minute int64) {
if *extent == nil {
*extent = &Span{From: minute, To: minute}
}
(*extent).From = min((*extent).From, minute)
(*extent).To = max((*extent).To, minute)
}
func sameExtent(a, b *Span) bool {
return a == nil && b == nil || a != nil && b != nil && *a == *b
}
func (c *bundleCheck) declared(episode string) bool {
if _, ok := c.episodes[episode]; !ok {
return false
}
c.episodes[episode] = true
return true
}
// finish compares what was read with the manifest's outputs and totals.
func (c *bundleCheck) finish(records, volume Digest) error {
for _, seen := range c.episodes {
if !seen {
return bundleError("episode identity")
}
}
for _, o := range c.m.Outputs {
got, rows := records, c.rRows
if o.Kind == "volume" {
got, rows = volume, c.vRows
}
if got.SHA256 != o.SHA256 || got.Bytes != o.Bytes || rows != o.Rows {
return bundleError("output " + o.Kind)
}
}
for _, sm := range c.m.Sites {
s := c.sites[sm.Site]
if !sameExtent(s.extent, sm.Extent) {
return bundleError("site extent")
}
for _, in := range s.inputs {
// Records are a subset of the copy's timed lines, so their
// disorder cannot exceed the copy's.
if in.disorder > in.m.DisorderSeconds {
return bundleError("input disorder")
}
if !sameExtent(in.extent, in.m.Extent) {
return bundleError("input extent")
}
}
volumeUnbound, noTarget := int64(0), int64(0)
for m, n := range s.volumeUnbound {
volumeUnbound += n
noTarget += s.volumeNoTarget[m]
}
switch {
case s.records != sm.Records, len(s.lines) != len(s.volumeLines):
return bundleError("site records")
case s.volumeBytes != sm.Bytes-sm.UnplacedBytes:
return bundleError("site bytes")
case s.unbound != sm.AttributionLoss+sm.InvalidClient, volumeUnbound != s.unbound:
return bundleError("site bindings")
case noTarget != sm.NoTarget:
return bundleError("site targets")
case s.infra != sm.Infrastructure:
return bundleError("site infrastructure")
}
for m, n := range s.lines {
if s.volumeLines[m] != n || s.volumeUnbound[m] != s.unboundLines[m] {
return bundleError("site minutes")
}
}
for label, n := range sm.Labels {
if s.labels[label] != n {
return bundleError("site labels")
}
}
}
return nil
}
// siteList returns every manifest site with its coverage, which requires every
// volume row: loss is counted per minute there.
func (c *bundleCheck) siteList() ([]BundleSite, error) {
out := make([]BundleSite, 0, len(c.m.Sites))
for _, sm := range c.m.Sites {
s := c.sites[sm.Site]
b := BundleSite{Site: sm.Site, Account: sm.Account, Extent: sm.Extent, Records: sm.Records}
if s.proof != nil {
var err error
if b.Coverage, b.Excluded, err = c.coverage(s); err != nil {
return nil, err
}
b.Certified = slices.Clone(s.proof.Spans)
}
out = append(out, b)
}
return out, nil
}
// coverage removes from the certified spans every minute a lost line may
// belong to, unless pre-application evidence waives exactly that loss.
func (c *bundleCheck) coverage(s *siteState) ([]Span, map[string]int64, error) {
period, late := c.m.Period, c.proof.LatenessSeconds
excluded := map[string]int64{}
for _, e := range s.proof.Excluded {
excluded[e.Reason] += e.To - e.From + 1
}
waived := map[int64]bool{}
var cut []Span
for _, w := range s.proof.Rejects {
var lines int64
switch w.Category {
case LossNoTarget:
for m, n := range s.volumeNoTarget {
if n > 0 && m >= w.From && m <= w.To {
lines += n
waived[m] = true
}
}
case LossOversized:
for _, u := range s.m.Untimed {
if b, ok := bracket(u, period, late); ok && u.Category == LossOversized && b.From >= w.From && b.To <= w.To {
lines += u.Lines
}
}
}
if lines != w.Lines {
return nil, nil, proofError("rejects")
}
}
for m, n := range s.volumeNoTarget {
if (n > 0 && !waived[m]) || s.volumeUnbound[m] > 0 {
cut = append(cut, Span{From: m, To: m})
}
}
for _, u := range s.m.Untimed {
b, ok := bracket(u, period, late)
if !ok {
continue
}
covered := false
for _, w := range s.proof.Rejects {
covered = covered || (u.Category == LossOversized && w.Category == LossOversized && b.From >= w.From && b.To <= w.To)
}
if !covered {
cut = append(cut, b)
}
}
coverage := subtract(s.proof.Spans, cut)
var certified, kept int64
for _, sp := range s.proof.Spans {
certified += sp.To - sp.From + 1
}
for _, sp := range coverage {
kept += sp.To - sp.From + 1
}
if certified > kept {
excluded[ExcludedUnknownLoss] = certified - kept
}
return coverage, excluded, nil
}
// bracket is the minute range an untimed lost line may belong to, within
// the period; ok is false when the range lies wholly outside it.
func bracket(u UntimedLoss, period Span, late int64) (Span, bool) {
lo, hi := period.From, period.To
after, before := u.After, u.Before
if after > 0 && before > 0 && after > before {
// Out-of-order neighbours: take the wider bound of the two.
after, before = before, after
}
if after > 0 {
lo = max(lo, floorMinute(after-late))
}
if before > 0 {
upper, ok := checkedSum(before, late)
if !ok {
upper = math.MaxInt64
}
hi = min(hi, floorMinute(upper))
}
return Span{From: lo, To: hi}, lo <= hi
}
func floorMinute(sec int64) int64 {
if sec < 0 {
return (sec - 59) / 60
}
return sec / 60
}
// subtract removes every minute of cut from sorted disjoint spans.
func subtract(spans, cut []Span) []Span {
slices.SortFunc(cut, func(a, b Span) int { return cmp.Compare(a.From, b.From) })
var out []Span
spansLoop:
for _, s := range spans {
from := s.From
for _, c := range cut {
if c.To < from || c.From > s.To {
continue
}
if c.From > from {
out = append(out, Span{From: from, To: c.From - 1})
}
// No successor is needed when the cut consumes the rest. In
// particular, the last representable minute has no successor.
if c.To >= s.To {
continue spansLoop
}
from = c.To + 1
}
if from <= s.To {
out = append(out, Span{From: from, To: s.To})
}
}
return out
}
// RestrictToCoverage returns the records whose minute lies in coverage, in
// their original order, for a replay over validated coverage.
func RestrictToCoverage(records []Record, coverage []Span) []Record {
var out []Record
for _, r := range records {
m := r.T / 60
i, found := slices.BinarySearchFunc(coverage, m, func(s Span, m int64) int { return cmp.Compare(s.To, m) })
if found || (i < len(coverage) && coverage[i].From <= m) {
out = append(out, r)
}
}
return out
}
package crawlreplay
import (
"cmp"
"errors"
"maps"
"math"
"slices"
"strings"
)
// Margin is a window's distance past A1 and A2; both at least 1 is anomalous.
type Margin struct {
A1 float64 `json:"a1"`
A2 float64 `json:"a2"`
}
func (m Margin) least() float64 { return min(m.A1, m.A2) }
// ErrTruth reports an episode truth table that is malformed or disagrees
// with itself or with the labeled records.
var ErrTruth = errors.New("crawlreplay: invalid episode truth")
// Outcomes of one episode on one site.
const (
OutcomeDetected = "detected" // a correct High transition at or after onset
OutcomeMissed = "missed" // onset in scoring, requests replayed, no correct transition
OutcomeUnscored = "not_scored" // onset outside scoring or no requests replayed in scoring
OutcomeAbsent = "absent" // the site logged no request of the episode
)
// How the scope selected at a detection relates to the detecting key.
const (
ScopeExact = "exact" // the scope names the detecting key
ScopeBroader = "broader" // the scope is a wider level
ScopeNarrower = "narrower" // the scope is a narrower level
ScopeOther = "other" // same level, other keys
ScopeNone = "none" // no scope, or selection refused
)
// EpisodeTruth is the operator's predeclared truth for one episode on one
// site: the keys whose High transition correctly detects it. It is private
// and fixed before replay; the majority label of a window is only a
// suggestion for review, never this truth.
type EpisodeTruth struct {
Episode string `json:"episode"`
Label string `json:"label"`
Site string `json:"site"`
Keys []KeyID `json:"keys"`
}
// EpisodeResult is one episode's outcome on one site. Detection is the
// first correct High transition in a scored minute at or after the onset
// minute; DelaySeconds runs from onset to the end of that minute and
// excludes log flush, lateness watermark and tick delay, which the ledger
// adds from their own measurements.
type EpisodeResult struct {
Episode string `json:"episode"`
Label string `json:"label"`
Site string `json:"site"`
Status string `json:"status"`
Onset int64 `json:"onset,omitempty"` // first counted request, or first request if none counted; Unix seconds
Detected bool `json:"detected"`
DetectMinute int64 `json:"detect_minute,omitempty"`
DelaySeconds int64 `json:"delay_seconds,omitempty"`
Key *KeyID `json:"key,omitempty"`
AtDetection Margin `json:"at_detection"` // margin the decision used
ExactAtDetection Margin `json:"exact_at_detection"` // margin of the exact residuals
Scope *Scope `json:"scope,omitempty"`
ScopeMatch string `json:"scope_match,omitempty"`
ActiveAtOnset []KeyID `json:"active_at_onset,omitempty"` // relevant findings already active: no credit
// Best is the relevant scored window after onset closest to anomalous;
// Worst is the least anomalous among anomalous ones.
Best Margin `json:"best"`
Worst Margin `json:"worst"`
}
// ScoredEvent is a High transition in a scored minute with its review
// fields: the machine-suggested majority class of its window and the
// episodes it correctly detects.
type ScoredEvent struct {
FindingEvent
Class string `json:"class"`
Credited []string `json:"credited,omitempty"`
}
// SiteDay is one site's scored evidence for one UTC day: the denominators
// a false-positive rate needs, and every transition.
type SiteDay struct {
Site string `json:"site"`
Day int64 `json:"day"` // Unix day
Minutes int64 `json:"minutes"` // scored covered minutes
Judged int64 `json:"judged"` // of those, with a complete window
Requests map[string]int64 `json:"requests"` // eligible requests by label, "" unlabeled
Events int `json:"events"` // High transitions
Credited int `json:"credited"` // of those, correct detections
}
// ScoreStart is a finding already active when a site's scoring began.
type ScoreStart struct {
Site string `json:"site"`
ActiveFinding
}
// Scoring is what a scorer found in the scored minutes.
type Scoring struct {
Events []ScoredEvent `json:"events"`
Episodes []EpisodeResult `json:"episodes"`
SiteDays []SiteDay `json:"site_days"`
ScoreStarts []ScoreStart `json:"active_at_score_start"`
}
type episodeSite struct{ episode, site string }
type siteDay struct {
site string
day int64
}
type onset struct {
t int64 // first counted request, or first request when none is counted
counted bool // t is a request the detector counts
minutes []int64 // every minute holding one of the episode's counted requests, ascending
seen bool // the findings active at the onset minute are recorded
}
// Scorer turns a session's ticks into scored evidence. Without a truth
// table it credits a transition to the majority episode of its window: a
// suggestion for synthetic fixtures and review, not a qualified outcome.
type Scorer struct {
p Params
suggest bool
truth map[episodeSite]*EpisodeTruth
labels map[string]string // episode -> label
onsets map[episodeSite]*onset
declared map[string][]Span // scoring spans the segments declared
scored map[string][]Span // scored minutes actually replayed
results map[episodeSite]*EpisodeResult
events []ScoredEvent
days map[siteDay]*SiteDay
wasScored map[string]bool
starts []ScoreStart
}
// NewScorer checks a truth table: pseudonymous sites, episodes and keys,
// episodic labels, and one definition per episode and site with one label
// per episode. It owns a copy of the table. A nil table scores by suggestion.
func NewScorer(p Params, truth []EpisodeTruth) (*Scorer, error) {
if err := p.Validate(); err != nil {
return nil, err
}
sc, err := newScorer(truth)
if err != nil {
return nil, err
}
sc.p = p
return sc, nil
}
// ValidateTruth checks the declaration even when no replay was requested.
func ValidateTruth(truth []EpisodeTruth) error {
_, err := newScorer(truth)
return err
}
func newScorer(truth []EpisodeTruth) (*Scorer, error) {
sc := &Scorer{suggest: truth == nil, truth: map[episodeSite]*EpisodeTruth{}, labels: map[string]string{},
onsets: map[episodeSite]*onset{}, declared: map[string][]Span{}, scored: map[string][]Span{}, results: map[episodeSite]*EpisodeResult{},
days: map[siteDay]*SiteDay{}, wasScored: map[string]bool{}}
for _, tr := range truth {
es := episodeSite{tr.Episode, tr.Site}
if !episodeID.MatchString(tr.Episode) || !ValidSite(tr.Site) || (tr.Label != LabelAttack && tr.Label != LabelOverload) ||
len(tr.Keys) == 0 || sc.truth[es] != nil {
return nil, ErrTruth
}
if l, ok := sc.labels[tr.Episode]; ok && l != tr.Label {
return nil, ErrTruth
}
seen := map[KeyID]bool{}
for _, k := range tr.Keys {
if !validKeyID(k) || seen[k] {
return nil, ErrTruth
}
seen[k] = true
}
sc.labels[tr.Episode] = tr.Label
tr.Keys = slices.Clone(tr.Keys)
sc.truth[es] = &tr
}
return sc, nil
}
// Observe takes a segment's validated records, including those in minutes
// its coverage excludes, before its ticks: an episode's onset is its first
// counted request whether or not that minute can be replayed. With a truth
// table, every episode in a segment with scored minutes needs an entry for
// its site, and a record's label must match its episode's. Onset and the
// replayed evidence come from requests the detector counts: infrastructure
// and ineligible requests inside a labeled range never move the onset. An
// episode with no counted request keeps its first request as onset and is
// never scored. A rejected observation leaves the scorer unchanged.
func (sc *Scorer) Observe(site string, records []Record, score []Span) error {
labels := maps.Clone(sc.labels)
onsets := map[episodeSite]onset{}
minutes := map[episodeSite][]int64{}
for _, r := range records {
if r.Validate() != nil || r.Site != site {
return ErrSite
}
if r.Episode == "" {
continue
}
es := episodeSite{r.Episode, site}
if l, ok := labels[r.Episode]; ok && l != r.Label {
return ErrTruth
}
if !sc.suggest && len(score) > 0 && sc.truth[es] == nil {
return ErrTruth
}
labels[r.Episode] = r.Label
counted := r.Class != ClassOther && !r.Infra
onsets[es] = earlierOnset(onsets[es], onset{t: r.T, counted: counted})
if counted {
minutes[es] = append(minutes[es], r.T/60)
}
}
sc.labels = labels
for es, first := range onsets {
o := sc.onsets[es]
if o == nil {
o = &onset{t: first.t, counted: first.counted}
sc.onsets[es] = o
}
merged := earlierOnset(onset{t: o.t, counted: o.counted}, first)
if !o.counted && merged.counted {
// Earlier segments may have captured findings at a fallback
// onset. Only the first counted request fixes this evidence.
o.seen = false
if r := sc.results[es]; r != nil {
r.ActiveAtOnset = nil
}
}
o.t, o.counted = merged.t, merged.counted
o.minutes = append(o.minutes, minutes[es]...)
slices.Sort(o.minutes)
o.minutes = slices.Compact(o.minutes)
}
sc.declared[site] = append(sc.declared[site], score...)
return nil
}
// earlierOnset prefers a counted request over an uncounted one, and the
// earlier of two of the same kind. The zero onset is empty.
func earlierOnset(a, b onset) onset {
switch {
case a.t == 0:
return b
case b.t == 0, a.counted && !b.counted:
return a
case b.counted && !a.counted, b.t < a.t:
return b
}
return a
}
// relevant reports whether a window of key speaks to episode es: a truth
// key holding the episode's requests or, by suggestion, a window whose
// majority is the episode.
func (sc *Scorer) relevant(es episodeSite, key KeyID, total int64, labels map[string]int64) bool {
label := sc.labels[es.episode]
if sc.suggest {
class, episode := classify(total, labels)
return class == label && episode == es.episode
}
tr := sc.truth[es]
return tr != nil && slices.Contains(tr.Keys, key) && labels[labelKey(label, es.episode)] > 0
}
// near reports whether key is one of the episode's truth keys or an
// ancestor of one: a finding there already holds the episode's traffic.
func (sc *Scorer) near(es episodeSite, key KeyID) bool {
if sc.suggest {
return true
}
tr := sc.truth[es]
if tr == nil {
return false
}
for _, k := range tr.Keys {
if k == key || slices.Contains(ancestors(k), key) {
return true
}
}
return false
}
func (sc *Scorer) result(es episodeSite) *EpisodeResult {
r := sc.results[es]
if r == nil {
r = &EpisodeResult{Episode: es.episode, Label: sc.labels[es.episode], Site: es.site,
Best: Margin{A1: math.Inf(-1), A2: math.Inf(-1)}, Worst: Margin{A1: math.Inf(1), A2: math.Inf(1)}}
sc.results[es] = r
}
return r
}
// siteEpisodes lists the observed episodes of a site in a fixed order, so
// credit lists do not depend on map order.
func (sc *Scorer) siteEpisodes(site string) []episodeSite {
var out []episodeSite
for es := range sc.onsets {
if es.site == site {
out = append(out, es)
}
}
slices.SortFunc(out, func(a, b episodeSite) int { return cmp.Compare(a.episode, b.episode) })
return out
}
// Tick scores one tick of a session fed the observed segments, in order.
func (sc *Scorer) Tick(t Tick) {
episodes := sc.siteEpisodes(t.Site)
for _, es := range episodes {
o := sc.onsets[es]
if o.seen || t.Minute < o.t/60 {
continue
}
o.seen = true
for _, a := range t.Prior {
if a.Since < o.t/60 && sc.near(es, a.Key) {
r := sc.result(es)
r.ActiveAtOnset = append(r.ActiveAtOnset, a.Key)
}
}
}
started := t.Scored && !sc.wasScored[t.Site]
sc.wasScored[t.Site] = t.Scored
if !t.Scored {
return
}
// Declared scoring spans can include coverage gaps. Only a delivered
// scored tick proves that its minute was actually replayed.
spans := sc.scored[t.Site]
if n := len(spans); n > 0 && spans[n-1].To == t.Minute-1 {
spans[n-1].To = t.Minute
} else {
spans = append(spans, Span{From: t.Minute, To: t.Minute})
}
sc.scored[t.Site] = spans
if started {
for _, a := range t.Prior {
if a.Since < t.Minute {
sc.starts = append(sc.starts, ScoreStart{Site: t.Site, ActiveFinding: a})
}
}
}
day := siteDay{t.Site, t.Minute / 1440}
d := sc.days[day]
if d == nil {
d = &SiteDay{Site: t.Site, Day: day.day, Requests: map[string]int64{}}
sc.days[day] = d
}
d.Minutes++
if t.Complete {
d.Judged++
}
for label, n := range t.Requests {
d.Requests[label] += n
}
for _, e := range t.Evaluations {
a1, a2 := sc.p.Margins(e.Residual, e.Distinct, e.Expected)
margin := Margin{A1: a1, A2: a2}
for _, es := range episodes {
if !sc.relevant(es, e.Key, e.Total, e.Labels) {
continue
}
r := sc.result(es)
if margin.least() > r.Best.least() {
r.Best = margin
}
if e.Anomalous && margin.least() < r.Worst.least() {
r.Worst = margin
}
}
}
for _, ev := range t.Events {
ev = cloneFindingEvent(ev)
class, _ := classify(ev.Total, ev.Labels)
se := ScoredEvent{FindingEvent: ev, Class: class}
for _, es := range episodes {
// A window holding the episode's requests cannot precede its
// first request, so relevance also places it after onset.
if !sc.relevant(es, ev.Key, ev.Total, ev.Labels) {
continue
}
se.Credited = append(se.Credited, es.episode)
if r := sc.result(es); !r.Detected {
key, scope := ev.Key, ev.Scope
r.Detected, r.DetectMinute, r.Key, r.Scope = true, t.Minute, &key, &scope
r.DelaySeconds = (t.Minute+1)*60 - sc.onsets[es].t
r.AtDetection, r.ExactAtDetection = ev.Bound, ev.Exact
r.ScopeMatch = scopeMatch(ev.Scope, ev.Key)
}
}
d.Events++
if len(se.Credited) > 0 {
d.Credited++
}
sc.events = append(sc.events, se)
}
}
func scopeMatch(s Scope, key KeyID) string {
switch {
case s.Level == 0:
return ScopeNone
case slices.Contains(s.Keys, key):
return ScopeExact
case s.Level > key.Level:
return ScopeBroader
case s.Level < key.Level:
return ScopeNarrower
}
return ScopeOther
}
// Report finishes the scoring: every truth entry, or every observed episode
// by suggestion, gets an outcome. The returned evidence is independent of
// the scorer and remains unchanged by later ticks.
func (sc *Scorer) Report() Scoring {
out := Scoring{Events: slices.Clone(sc.events), Episodes: []EpisodeResult{}, SiteDays: []SiteDay{}, ScoreStarts: slices.Clone(sc.starts)}
if out.Events == nil {
out.Events = []ScoredEvent{}
}
if out.ScoreStarts == nil {
out.ScoreStarts = []ScoreStart{}
}
for i := range out.Events {
out.Events[i].FindingEvent = cloneFindingEvent(out.Events[i].FindingEvent)
out.Events[i].Credited = slices.Clone(out.Events[i].Credited)
}
keys := slices.Collect(maps.Keys(sc.onsets))
if !sc.suggest {
keys = slices.Collect(maps.Keys(sc.truth))
}
for _, es := range keys {
r := *sc.result(es)
r.ActiveAtOnset = slices.Clone(r.ActiveAtOnset)
if r.Key != nil {
key := *r.Key
r.Key = &key
}
if r.Scope != nil {
scope := cloneScope(*r.Scope)
r.Scope = &scope
}
o := sc.onsets[es]
switch {
case o == nil:
r.Status = OutcomeAbsent
case r.Detected:
r.Status, r.Onset = OutcomeDetected, o.t
// Each segment's declarations are sorted, but they can overlap
// earlier segments' declarations, so their union is not sorted.
case !slices.ContainsFunc(sc.declared[es.site], func(s Span) bool { return s.From <= o.t/60 && o.t/60 <= s.To }),
!slices.ContainsFunc(o.minutes, func(m int64) bool { return inSpans(sc.scored[es.site], m) }):
// It began outside scoring, or none of its requests was
// replayed in a scored minute: not evidence of a miss.
r.Status, r.Onset = OutcomeUnscored, o.t
default:
r.Status, r.Onset = OutcomeMissed, o.t
}
if math.IsInf(r.Worst.A1, 1) {
r.Worst = Margin{}
}
if math.IsInf(r.Best.A1, -1) {
r.Best = Margin{}
}
out.Episodes = append(out.Episodes, r)
}
slices.SortFunc(out.Episodes, func(a, b EpisodeResult) int {
return cmp.Or(cmp.Compare(a.Site, b.Site), cmp.Compare(a.Episode, b.Episode))
})
for _, d := range sc.days {
day := *d
day.Requests = maps.Clone(day.Requests)
out.SiteDays = append(out.SiteDays, day)
}
slices.SortFunc(out.SiteDays, func(a, b SiteDay) int { return cmp.Or(cmp.Compare(a.Site, b.Site), cmp.Compare(a.Day, b.Day)) })
return out
}
func cloneFindingEvent(e FindingEvent) FindingEvent {
e.Labels = maps.Clone(e.Labels)
e.Scope = cloneScope(e.Scope)
return e
}
func cloneScope(s Scope) Scope {
s.Keys = slices.Clone(s.Keys)
return s
}
// classify names the majority label of a window: attack or overload with
// the largest episode, healthy, or unlabeled when no label holds half.
func classify(total int64, labels map[string]int64) (class, episode string) {
var attack, overload int64
best := map[string]int64{}
for key, n := range labels {
label, _, found := strings.Cut(key, "/")
if !found {
continue
}
switch label {
case LabelAttack:
attack += n
case LabelOverload:
overload += n
}
best[key] = n
}
pick := func(label string) string {
name, most := "", int64(-1)
for key, n := range best {
l, ep, _ := strings.Cut(key, "/")
if l == label && (n > most || (n == most && ep < name)) {
name, most = ep, n
}
}
return name
}
switch {
case total == 0:
return "unlabeled", ""
case attack >= total-attack:
return LabelAttack, pick(LabelAttack)
case overload >= total-overload:
return LabelOverload, pick(LabelOverload)
case labels[LabelHealthy] >= total-labels[LabelHealthy]:
return LabelHealthy, ""
}
return "unlabeled", ""
}
// SketchAccuracy compares a sketch session with an exact session fed the
// same segments. The error fields compare bounds with the exact residuals
// of the same window; the decision fields compare the two sessions, whose
// baselines learn independently, so a sketch miss that trains its own
// baseline still counts as lost.
type SketchAccuracy struct {
MaxResidualError int64 `json:"max_residual_error"`
MaxDistinctError int64 `json:"max_distinct_error"`
// Exceeded must stay zero: a bound above the exact value would make
// the prototype unsound.
Exceeded int64 `json:"exceeded"`
LostDecisions int64 `json:"lost_decisions"` // exact session anomalous, sketch session not
ExtraDecisions int64 `json:"extra_decisions"` // sketch session anomalous, exact session not
LostFindings int64 `json:"lost_findings"` // exact transitions with no sketch finding active then
}
// Pairing compares a sketch session's ticks with an exact session's ticks
// for the same segment, which the exact session is fed first.
type Pairing struct {
anomalous map[siteMinute]map[KeyID]bool
transitions map[siteMinute][]KeyID
acc SketchAccuracy
}
// NewPairing starts an empty comparison.
func NewPairing() *Pairing {
return &Pairing{anomalous: map[siteMinute]map[KeyID]bool{}, transitions: map[siteMinute][]KeyID{}}
}
// Exact records one exact-session tick.
func (pr *Pairing) Exact(t Tick) {
sm := siteMinute{t.Site, t.Minute}
for _, e := range t.Evaluations {
if e.Anomalous {
if pr.anomalous[sm] == nil {
pr.anomalous[sm] = map[KeyID]bool{}
}
pr.anomalous[sm][e.Key] = true
}
}
for _, ev := range t.Events {
pr.transitions[sm] = append(pr.transitions[sm], ev.Key)
}
}
// Sketch compares the sketch-session tick for the same site and minute.
func (pr *Pairing) Sketch(t Tick) {
sm := siteMinute{t.Site, t.Minute}
exact := pr.anomalous[sm]
delete(pr.anomalous, sm)
for _, e := range t.Evaluations {
pr.acc.MaxResidualError = max(pr.acc.MaxResidualError, e.ExactResidual-e.Residual)
pr.acc.MaxDistinctError = max(pr.acc.MaxDistinctError, e.ExactDistinct-e.Distinct)
if e.Residual > e.ExactResidual || e.Distinct > e.ExactDistinct {
pr.acc.Exceeded++
}
switch {
case exact[e.Key] && !e.Anomalous:
pr.acc.LostDecisions++
case e.Anomalous && !exact[e.Key]:
pr.acc.ExtraDecisions++
}
delete(exact, e.Key)
}
pr.acc.LostDecisions += int64(len(exact))
for _, key := range pr.transitions[sm] {
if !slices.ContainsFunc(t.Active, func(a ActiveFinding) bool { return a.Key == key }) {
pr.acc.LostFindings++
}
}
delete(pr.transitions, sm)
}
// Accuracy returns the comparison so far; exact decisions no sketch tick
// met count as lost.
func (pr *Pairing) Accuracy() SketchAccuracy {
acc := pr.acc
for _, keys := range pr.anomalous {
acc.LostDecisions += int64(len(keys))
}
for _, keys := range pr.transitions {
acc.LostFindings += int64(len(keys))
}
return acc
}
// Report is one site's replay outcome under one parameter set.
type Report struct {
Episodes []EpisodeResult `json:"episodes"`
Events []ScoredEvent `json:"events"`
// Transitions counts High transitions by suggested class, and
// FalsePositives the healthy-class ones per UTC day (Unix day number):
// findings alert on transitions, not on minutes.
Transitions map[string]int `json:"transitions"`
FalsePositives map[int64]int `json:"false_positives"`
SiteDays []SiteDay `json:"site_days"`
Evaluations int64 `json:"evaluations"`
Anomalous int64 `json:"anomalous"`
ScopeLevels map[uint8]int `json:"scope_levels"` // selected level at minutes with anomalies
MaxActive map[uint8]int `json:"max_active"` // most keys with traffic in one window, per level
Keys map[uint8]int `json:"keys"` // distinct keys seen, per level
MaxBindings int64 `json:"max_bindings"` // most distinct bindings in one key's window
Sketch *SketchAccuracy `json:"sketch,omitempty"`
}
// EvaluateSite replays one site cold, every covered minute scored and in
// normal learning state, and scores it against o.Truth or, without one, by
// suggestion. With a sketch it also runs an exact session and compares the
// two.
func EvaluateSite(site Site, p Params, o Options) (Report, error) {
rep := Report{Transitions: map[string]int{}, FalsePositives: map[int64]int{}, ScopeLevels: map[uint8]int{},
MaxActive: map[uint8]int{}, Keys: map[uint8]int{}}
scorer, err := NewScorer(p, o.Truth)
if err != nil {
return Report{}, err
}
seg := normalSegment(site)
if err = scorer.Observe(seg.Site, site.Records, seg.Score); err != nil {
return Report{}, err
}
var pairing *Pairing
if o.Sketch != nil {
var exact *ReplaySession
if exact, err = NewReplaySession(SessionConfig{Params: p, Shuffle: o.Shuffle}); err != nil {
return Report{}, err
}
pairing = NewPairing()
if err = exact.Feed(seg, func(t Tick) error { pairing.Exact(t); return nil }); err != nil {
return Report{}, err
}
}
s, err := NewReplaySession(SessionConfig{Params: p, Sketch: o.Sketch, Shuffle: o.Shuffle})
if err != nil {
return Report{}, err
}
seen := map[KeyID]bool{}
err = s.Feed(seg, func(t Tick) error {
scorer.Tick(t)
if pairing != nil {
pairing.Sketch(t)
}
perLevel := map[uint8]int{}
anomalous := false
for _, e := range t.Evaluations {
rep.Evaluations++
perLevel[e.Key.Level]++
if !seen[e.Key] {
seen[e.Key] = true
rep.Keys[e.Key.Level]++
}
rep.MaxBindings = max(rep.MaxBindings, e.Bindings)
if e.Anomalous {
rep.Anomalous++
anomalous = true
}
}
for level, n := range perLevel {
rep.MaxActive[level] = max(rep.MaxActive[level], n)
}
if anomalous {
rep.ScopeLevels[t.Scope.Level]++
}
return nil
})
if err != nil {
return Report{}, err
}
scoring := scorer.Report()
rep.Episodes, rep.Events, rep.SiteDays = scoring.Episodes, scoring.Events, scoring.SiteDays
for _, e := range scoring.Events {
rep.Transitions[e.Class]++
if e.Class == LabelHealthy {
rep.FalsePositives[e.Minute/1440]++
}
}
if pairing != nil {
acc := pairing.Accuracy()
rep.Sketch = &acc
}
return rep, nil
}
package crawlreplay
import (
"errors"
"regexp"
)
// ErrFixture reports a fixture outside the generator's domain.
var ErrFixture = errors.New("crawlreplay: invalid fixture")
var fixtureName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`)
// Fixture is a synthetic acceptance workload: Train minutes
// of healthy background on one L1 key, then an attack on a sibling L1 key
// for Minutes, sent by fresh clients of Q requests each. The attack rate
// rises linearly from one request to PerMinute over the first RampMinutes
// (zero starts at full rate) and then holds. Optional heavy padding sources
// on the attack key change every minute when Churn is set.
type Fixture struct {
Name string `json:"name"`
Train int `json:"train"` // background minutes before the attack
Background int `json:"background"` // background requests per minute
Pool int `json:"pool"` // background clients
Minutes int `json:"minutes"` // attack minutes
PerMinute int `json:"per_minute"` // attack requests per minute
RampMinutes int `json:"ramp_minutes"`
Q int `json:"q"` // requests per attacking client
PaddingSources int `json:"padding_sources"`
PaddingPerSource int `json:"padding_per_source"`
Churn bool `json:"churn"`
Seed uint64 `json:"seed"`
}
// Validate checks the generator's domain.
func (f Fixture) Validate() error {
switch {
case !fixtureName.MatchString(f.Name), f.Train < 0, f.Background < 0,
f.Pool < 0, f.Background > 0 && f.Pool < 1, f.Minutes < 1, f.PerMinute < 1, f.Q < 1,
f.RampMinutes < 0, f.RampMinutes > f.Minutes,
f.PaddingSources < 0, f.PaddingPerSource < 0, (f.PaddingSources > 0) != (f.PaddingPerSource > 0):
return ErrFixture
}
return nil
}
// fixtureSiteName is the fixed pseudonym of every synthetic site.
const fixtureSiteName = "dom-000000.example"
// Site builds the fixture's records, starting at a fixed minute so a
// fixture always lands on the same hour-of-week slots.
func (f Fixture) Site() Site {
const start = int64(29_833_000)
s := NewSynth(fixtureSiteName, f.Seed)
parent, background, attack := SynthKey(1), SynthKey(2), SynthKey(3)
end := start + int64(f.Train+f.Minutes) - 1
var recs []Record
if f.Background > 0 {
recs = s.Pool(Traffic{From: start, To: end, PerMinute: f.Background, L2: parent, L1: background, Label: LabelHealthy}, f.Pool)
}
onset := start + int64(f.Train)
attackTraffic := Traffic{From: onset, To: end, PerMinute: f.PerMinute, L2: parent, L1: attack, Label: LabelAttack, Episode: f.Name}
// Keep each client's request count across the ramp-to-steady boundary.
recs = append(recs, s.rotating(attackTraffic, f.Q, func(m int64) int {
elapsed := m - onset
if elapsed < int64(f.RampMinutes) {
return 1 + int(int64(f.PerMinute-1)*elapsed/max(1, int64(f.RampMinutes)-1))
}
return f.PerMinute
})...)
if f.PaddingSources > 0 {
recs = append(recs, s.Heavy(attackTraffic, f.PaddingSources, f.PaddingPerSource, f.Churn)...)
}
return Site{Records: recs, Coverage: []Span{{From: start, To: end}}}
}
package crawlreplay
import (
"cmp"
"errors"
"math"
"slices"
)
// ErrSite reports a replay input that is not one site's covered records.
var ErrSite = errors.New("crawlreplay: invalid site input")
// Params are the detector settings a sweep varies.
type Params struct {
W int `json:"w"` // window, complete minutes
R float64 `json:"r"` // residual rate multiple of the baseline
F float64 `json:"f"` // residual floor, requests per minute
K int `json:"k"` // largest bindings removed
D int `json:"d"` // residual distinct bindings required
C float64 `json:"c"` // scope coverage, percent of anomalous traffic
Baseline BaselineParams `json:"baseline"`
}
// Validate checks each parameter's domain constraints; it approves no value.
func (p Params) Validate() error {
switch {
case p.W < 1, p.K < 1, p.D <= p.K,
math.IsNaN(p.R) || math.IsInf(p.R, 0) || p.R <= 1,
math.IsNaN(p.F) || math.IsInf(p.F, 0) || p.F <= 0,
math.IsNaN(p.C) || p.C <= 0 || p.C > 100:
return ErrParams
}
return p.Baseline.Validate()
}
// Span is an inclusive range of Unix minutes a site's logs cover.
type Span struct {
From int64 `json:"from"`
To int64 `json:"to"`
}
// Site is one site's records and the minutes its logs cover. A minute
// outside Coverage is unknown, never zero traffic.
type Site struct {
Records []Record
Coverage []Span
}
// Options choose how a replay decides.
type Options struct {
// Sketch decides with the prototype sketch bounds; nil uses exact
// residuals. Exact values are always reported beside the bounds.
Sketch *SketchParams
// Shuffle, when nonzero, permutes arrivals inside each minute with this
// seed, to measure how sketch bounds depend on arrival order.
Shuffle uint64
// Truth scores episodes; nil credits a transition to its window's
// majority episode, a suggestion only.
Truth []EpisodeTruth
}
// KeyID names a detector key inside one site's replay: level 3 is the
// site's dynamic traffic, level 2 an L2 key and level 1 an L1 key whose
// L2 ancestor is Parent.
type KeyID struct {
Level uint8 `json:"level"`
Key string `json:"key,omitempty"`
Parent string `json:"parent,omitempty"`
}
func keyOrder(a, b KeyID) int {
return cmp.Or(cmp.Compare(b.Level, a.Level), cmp.Compare(a.Parent, b.Parent), cmp.Compare(a.Key, b.Key))
}
// keysOf lists every level a record contributes to, broadest first.
func keysOf(r *Record) []KeyID {
keys := []KeyID{{Level: 3}}
if r.Class == ClassExpensive {
keys = append(keys, KeyID{Level: 2, Key: r.L2}, KeyID{Level: 1, Key: r.L1, Parent: r.L2})
}
return keys
}
// Evaluation is one key's complete window at one minute.
type Evaluation struct {
Key KeyID `json:"key"`
Total int64 `json:"total"` // window requests, bound or not
Expensive int64 `json:"expensive"` // window requests with a query
Bindings int64 `json:"bindings"` // distinct bindings in the window
Residual int64 `json:"residual"` // residual requests the decision used
Distinct int64 `json:"distinct"` // residual bindings the decision used
ExactResidual int64 `json:"exact_residual"`
ExactDistinct int64 `json:"exact_distinct"`
Expected float64 `json:"expected"` // sum of per-minute expectations over the window
Anomalous bool `json:"anomalous"`
ExactAnomalous bool `json:"exact_anomalous"`
Trusted bool `json:"trusted"` // some window minute used a trained slot, not the floor
Labels map[string]int64 `json:"labels"`
}
// Margins are how far a window is past A1 and A2; both at least 1 means
// anomalous.
func (p Params) Margins(residual, distinct int64, expected float64) (a1, a2 float64) {
need := max(p.R*expected, p.F*float64(p.W))
return float64(residual) / need, float64(distinct) / float64(p.D)
}
func (p Params) anomalous(residual, distinct int64, expected float64) bool {
a1, a2 := p.Margins(residual, distinct, expected)
return a1 >= 1 && a2 >= 1
}
// Count bases of a scope selection.
const (
// BasisExpensive counts only requests with a query.
BasisExpensive = "expensive"
// BasisDynamic counts queryless dynamic requests too: the anomalous
// site key includes them, so every level is measured against them.
BasisDynamic = "dynamic"
)
// Reasons a scope selection refuses to name keys.
const (
// RefusedZero: the anomalous keys hold no requests.
RefusedZero = "zero_denominator"
// RefusedUnknown: a key whose window is incomplete may be anomalous
// outside every known anomalous ancestor, or may be the maximal
// ancestor of one, so the anomalous traffic is not known.
RefusedUnknown = "unknown_denominator"
)
// Scope is the narrowest level whose disjoint anomalous keys cover C
// percent of the anomalous traffic.
type Scope struct {
Level uint8 `json:"level"` // 0 when nothing qualifies
Keys []KeyID `json:"keys,omitempty"`
Covered int64 `json:"covered"`
Denominator int64 `json:"denominator"`
Basis string `json:"basis,omitempty"`
Refused string `json:"refused,omitempty"`
}
// Tick is one covered minute of one site. Evaluations hold every key whose
// window is complete and holds traffic or an active finding; Events hold the
// High transitions of this minute and Active every finding still active
// after it.
type Tick struct {
Site string
Minute int64
Complete bool // the site's last W covered minutes are known
Scored bool
Evaluations []Evaluation
Scope Scope
Events []FindingEvent
Active []ActiveFinding
Prior []ActiveFinding // findings present before this minute was judged
Requests map[string]int64 // this minute's eligible requests by label, "" unlabeled
}
// ReplaySite runs a cold session over one site whose every covered minute
// is in normal learning state, and calls fn for every minute whose W-minute
// window is complete. It suits synthetic fixtures; recorded data declares
// its states through a session.
func ReplaySite(site Site, p Params, o Options, fn func(Tick)) error {
s, err := NewReplaySession(SessionConfig{Params: p, Sketch: o.Sketch, Shuffle: o.Shuffle})
if err != nil {
return err
}
return s.Feed(normalSegment(site), func(t Tick) error {
if t.Complete {
fn(t)
}
return nil
})
}
// ancestors lists the keys whose traffic contains id's, nearest first.
func ancestors(id KeyID) []KeyID {
switch id.Level {
case 1:
return []KeyID{{Level: 2, Key: id.Parent}, {Level: 3}}
case 2:
return []KeyID{{Level: 3}}
}
return nil
}
// selectScope evaluates anomalous L1 keys, then L2, then L3; the first
// level whose union covers C percent of the maximal anomalous traffic
// supplies the disjoint key set. unknown lists keys with traffic whose
// window is incomplete; unless an anomalous ancestor already holds one,
// the anomalous traffic is not known and no scope is chosen.
func selectScope(evals []Evaluation, c float64, unknown []KeyID) Scope {
anomalous := map[KeyID]Evaluation{}
for _, e := range evals {
if e.Anomalous {
anomalous[e.Key] = e
}
}
subsumed := func(id KeyID) bool {
for _, a := range ancestors(id) {
if _, ok := anomalous[a]; ok {
return true
}
}
return false
}
for _, id := range unknown {
if !subsumed(id) {
return Scope{Refused: RefusedUnknown}
}
}
if len(anomalous) == 0 {
return Scope{}
}
basis := BasisExpensive
if site, ok := anomalous[KeyID{Level: 3}]; ok && site.Total > site.Expensive {
basis = BasisDynamic
}
count := func(e Evaluation) int64 {
if basis == BasisDynamic {
return e.Total
}
return e.Expensive
}
var denominator int64
for id, e := range anomalous {
if !subsumed(id) {
denominator += count(e)
}
}
if denominator <= 0 {
return Scope{Refused: RefusedZero}
}
for level := uint8(1); level <= 3; level++ {
var keys []KeyID
var covered int64
for id, e := range anomalous {
if id.Level == level {
keys = append(keys, id)
covered += count(e)
}
}
if len(keys) > 0 && float64(covered)*100 >= c*float64(denominator) {
slices.SortFunc(keys, keyOrder)
return Scope{Level: level, Keys: keys, Covered: covered, Denominator: denominator, Basis: basis}
}
}
return Scope{Denominator: denominator, Basis: basis}
}
// normalSegment is one site's whole coverage, scored and in normal learning
// state, for synthetic replays.
func normalSegment(site Site) ReplaySegment {
name := fixtureSiteName
if len(site.Records) > 0 {
name = site.Records[0].Site
}
seg := ReplaySegment{Site: name, Records: site.Records, Coverage: site.Coverage, Score: site.Coverage}
if n := len(site.Coverage); n > 0 {
seg.States = []StateSpan{{From: site.Coverage[0].From, To: site.Coverage[n-1].To, State: StateNormal}}
}
return seg
}
package crawlreplay
import (
"cmp"
"errors"
"math/rand/v2"
"slices"
)
// ErrSession reports a segment, state declaration, snapshot or call that a
// replay session cannot accept. After a failed Feed callback the session
// refuses every further call: its state may be partly advanced.
var ErrSession = errors.New("crawlreplay: invalid replay session input")
// Learning states a segment declares for a site's or key's minutes.
// The detector freezes learning in every state but normal, and the
// harness also freezes it for anomalous minutes and while a finding is
// active. A minute no declaration covers is unknown and never learned.
const (
StateNormal = "normal"
StateProtected = "protected"
StateDegraded = "degraded"
StateRecoveryHold = "recovery_hold"
)
var learningStates = map[string]bool{StateNormal: true, StateProtected: true, StateDegraded: true, StateRecoveryHold: true}
// StateSpan declares the learning state over inclusive minutes, for one key
// or, when Key is nil, for every key of the segment's site. A key's own
// declaration wins over the site's.
type StateSpan struct {
Key *KeyID `json:"key,omitempty"`
From int64 `json:"from"`
To int64 `json:"to"`
State string `json:"state"`
}
// ReplaySegment is one site's validated records over its coverage. A
// session takes each site's segments in chronological order; windows run on
// across adjacent segments exactly as within one.
type ReplaySegment struct {
Site string
Records []Record // records in Coverage minutes only (RestrictToCoverage)
Coverage []Span // certified minutes without unknown loss
Score []Span // minutes whose ticks are scored; nil trains only
States []StateSpan // after every minute this site was already fed
}
// SessionConfig fixes what a session's state means. A restored snapshot
// keeps only the state this configuration can still interpret.
type SessionConfig struct {
Params Params
Sketch *SketchParams // nil decides with exact residuals
Shuffle uint64 // nonzero permutes arrivals within each minute
IdentityVersion int // the key and binding pseudonym contract
}
func (c SessionConfig) validate() error {
if err := c.Params.Validate(); err != nil {
return err
}
if c.Sketch != nil && (c.Sketch.M <= c.Params.K || c.Sketch.H < c.Params.D+c.Params.K) {
return ErrParams
}
if c.IdentityVersion < 0 {
return ErrParams
}
return nil
}
func (c SessionConfig) clone() SessionConfig {
if c.Sketch != nil {
sketch := *c.Sketch
c.Sketch = &sketch
}
return c
}
// ActiveFinding is a key whose High finding has not cleared. Uncertain
// means its evidence was interrupted, by a coverage gap or an incompatible
// restore, and no complete window has judged it since.
type ActiveFinding struct {
Key KeyID `json:"key"`
Since int64 `json:"since"`
Uncertain bool `json:"uncertain,omitempty"`
}
// FindingEvent is one pseudonymous High transition: a key without an active
// finding whose complete window is anomalous. It carries the window's
// evidence as the detector saw it at that minute.
type FindingEvent struct {
Site string `json:"site"`
Key KeyID `json:"key"`
Minute int64 `json:"minute"`
Total int64 `json:"total"`
Expensive int64 `json:"expensive"`
Expected float64 `json:"expected"`
// Exact is the margin of the exact window residuals; Bound is the
// margin the decision used, the sketch bounds when a sketch decides.
Exact Margin `json:"exact"`
Bound Margin `json:"bound"`
Scope Scope `json:"scope"`
Labels map[string]int64 `json:"labels"` // window requests per label and episode
// CoveredFrom is the first minute of the continuous coverage the window
// lies in; Trusted reports that some window minute was judged against a
// trained slot rather than the cold floor.
CoveredFrom int64 `json:"covered_from"`
Trusted bool `json:"trusted"`
}
// ReplaySession replays sites minute by minute and keeps, per site, the
// state a detector would: hour-of-week baselines with their learned-minute
// watermarks, W-minute windows and summaries, active findings with their
// pinned profiles, and keys whose detail was evicted.
type ReplaySession struct {
cfg SessionConfig
sites map[string]*siteSession
broken bool
}
type siteSession struct {
processedTo int64 // latest minute fed; 0 before the first
coveredFrom int64 // first minute of the current continuous coverage
windowFrom int64 // first minute a window may count: coveredFrom, or later after an incompatible restore
covered []Span
siteStates []StateSpan
keyStates map[KeyID][]StateSpan
keys map[KeyID]*keyState
active map[KeyID]*keyState
retired map[KeyID]bool
}
type keyState struct {
baseline *Baseline
window *Window
minutes map[int64]*minuteCounts
sketches map[int64]*keySketch
learnedTo int64 // latest minute learned, frozen or passed over
// from is the first minute this key's own detail counts from: zero for
// a key whose earlier zero minutes are known, the reintroduction minute
// for a key whose detail was evicted.
from int64
active bool
finding *finding
}
type finding struct {
since int64
pin Profile
uncertain bool
}
// NewReplaySession starts a cold session with its own copy of cfg.
func NewReplaySession(cfg SessionConfig) (*ReplaySession, error) {
if err := cfg.validate(); err != nil {
return nil, err
}
return &ReplaySession{cfg: cfg.clone(), sites: map[string]*siteSession{}}, nil
}
func newSiteSession() *siteSession {
return &siteSession{keyStates: map[KeyID][]StateSpan{}, keys: map[KeyID]*keyState{},
active: map[KeyID]*keyState{}, retired: map[KeyID]bool{}}
}
// Feed replays one segment and calls fn for every covered minute, scored
// or not. Each minute is judged against the state learned before it and
// only then learned, so nothing later in the segment informs an earlier
// decision. Coverage must start after every minute this site was fed.
func (s *ReplaySession) Feed(seg ReplaySegment, fn func(Tick) error) error {
if s.broken {
return ErrSession
}
if !ValidSite(seg.Site) {
return ErrSite
}
buckets, err := bucketSite(seg.Site, seg.Records, seg.Coverage, s.cfg.Shuffle)
if err != nil {
return err
}
site := s.sites[seg.Site]
if site == nil {
site = newSiteSession()
}
if err = checkSegment(seg, site); err != nil {
return err
}
s.sites[seg.Site] = site
for _, st := range seg.States {
if st.Key == nil {
site.siteStates = append(site.siteStates, st)
} else {
// Keep the declaration's serialized key independent of a caller
// reusing the variable that identified this scope.
id := *st.Key
st.Key = &id
site.keyStates[id] = append(site.keyStates[id], st)
}
}
slices.SortFunc(site.siteStates, func(a, b StateSpan) int { return cmp.Compare(a.From, b.From) })
for id := range site.keyStates {
slices.SortFunc(site.keyStates[id], func(a, b StateSpan) int { return cmp.Compare(a.From, b.From) })
}
s.broken = true
for _, span := range seg.Coverage {
for m := span.From; m <= span.To; m++ {
tick := s.minute(seg.Site, site, m, buckets[m], inSpans(seg.Score, m))
if err := fn(tick); err != nil {
return err
}
}
}
s.broken = false
return nil
}
// checkSegment validates scoring spans and state declarations before the
// session changes: every declared minute lies after the minutes already
// fed, and no two declarations for the same scope overlap, including the
// site's earlier ones.
func checkSegment(seg ReplaySegment, site *siteSession) error {
processedTo := site.processedTo
if len(seg.Coverage) > 0 && seg.Coverage[0].From <= processedTo {
return ErrSession
}
for i, sp := range seg.Score {
if sp.From <= 0 || sp.To < sp.From || (i > 0 && sp.From <= seg.Score[i-1].To) {
return ErrSession
}
}
byScope := map[KeyID][]StateSpan{{}: slices.Clone(site.siteStates)}
for id, spans := range site.keyStates {
byScope[id] = slices.Clone(spans)
}
siteScope := KeyID{}
for _, st := range seg.States {
if !learningStates[st.State] || st.From <= processedTo || st.To < st.From {
return ErrSession
}
scope := siteScope
if st.Key != nil {
if !validKeyID(*st.Key) {
return ErrSession
}
scope = *st.Key
}
for _, other := range byScope[scope] {
if st.From <= other.To && other.From <= st.To {
return ErrSession
}
}
byScope[scope] = append(byScope[scope], st)
}
return nil
}
// ValidateStates checks declarations independently of replay coverage. This
// also validates diagnostic experiments, which never create a session.
func ValidateStates(states []StateSpan) error {
return checkSegment(ReplaySegment{States: states}, newSiteSession())
}
// validKeyID reports a key of the closed stream format: the site key, an
// L2 key, or an L1 key with its L2 parent.
func validKeyID(id KeyID) bool {
switch id.Level {
case 3:
return id.Key == "" && id.Parent == ""
case 2:
return keyPseudonym.MatchString(id.Key) && id.Parent == ""
case 1:
return keyPseudonym.MatchString(id.Key) && keyPseudonym.MatchString(id.Parent)
}
return false
}
func inSpans(spans []Span, m int64) bool {
i, found := slices.BinarySearchFunc(spans, m, func(s Span, m int64) int { return cmp.Compare(s.To, m) })
return found || (i < len(spans) && spans[i].From <= m)
}
// bucketSite checks one site's records against its coverage and groups the
// eligible ones by minute in logged order, before anything is replayed.
func bucketSite(name string, records []Record, coverage []Span, shuffle uint64) (map[int64][]*Record, error) {
for i, s := range coverage {
if s.From <= 0 || s.To < s.From || (i > 0 && s.From <= coverage[i-1].To) {
return nil, ErrSite
}
}
buckets := map[int64][]*Record{}
for i := range records {
r := &records[i]
if r.Validate() != nil || r.Site != name || !inSpans(coverage, r.T/60) {
return nil, ErrSite
}
if r.Class == ClassOther || r.Infra {
continue
}
buckets[r.T/60] = append(buckets[r.T/60], r)
}
for m, rs := range buckets {
slices.SortStableFunc(rs, func(a, b *Record) int {
return cmp.Or(cmp.Compare(a.File, b.File), cmp.Compare(a.Seq, b.Seq))
})
if shuffle != 0 {
// #nosec G115 G404 -- positive Unix minute; a reproducible permutation, not a secret.
rng := rand.New(rand.NewPCG(shuffle, uint64(m)))
rng.Shuffle(len(rs), func(i, j int) { rs[i], rs[j] = rs[j], rs[i] })
}
}
return buckets, nil
}
// state is the learning state of key id at minute m: its own declaration,
// else the nearest ancestor's (whose traffic contains its own), else the
// site's.
func (site *siteSession) state(id KeyID, m int64) string {
scopes := [][]StateSpan{site.keyStates[id]}
for _, a := range ancestors(id) {
scopes = append(scopes, site.keyStates[a])
}
for _, spans := range append(scopes, site.siteStates) {
i, found := slices.BinarySearchFunc(spans, m, func(s StateSpan, m int64) int { return cmp.Compare(s.To, m) })
if found || (i < len(spans) && spans[i].From <= m) {
return spans[i].State
}
}
return ""
}
// restart empties every window; findings stay active but uncertain until a
// complete window judges them.
func (site *siteSession) restart() {
for id, ks := range site.active {
ks.window, ks.minutes, ks.sketches = newWindow(), map[int64]*minuteCounts{}, map[int64]*keySketch{}
if ks.finding != nil {
ks.finding.uncertain = true
continue
}
ks.active = false
ks.minutes, ks.sketches = nil, nil
delete(site.active, id)
}
}
// key returns id's state, creating a cold key or reintroducing an evicted
// one, and activates it, first learning the zero minutes it sat idle through.
func (s *ReplaySession) key(site *siteSession, id KeyID, m int64) *keyState {
ks := site.keys[id]
if ks == nil {
ks = &keyState{baseline: NewBaseline(s.cfg.Params.Baseline, m), window: newWindow(), learnedTo: m - 1}
if site.retired[id] {
// Evicted detail lost the minutes it was not tracked, so it
// needs W observed minutes of its own before it can act.
delete(site.retired, id)
ks.from = m
}
site.keys[id] = ks
}
if !ks.active {
site.foldIdle(ks, id, m)
ks.active, ks.minutes, ks.sketches = true, map[int64]*minuteCounts{}, map[int64]*keySketch{}
site.active[id] = ks
}
return ks
}
// foldIdle learns the covered minutes in normal state a key had no traffic
// in, before minute until. Uncovered and frozen minutes are passed over.
func (site *siteSession) foldIdle(ks *keyState, id KeyID, until int64) {
for _, span := range site.covered {
for m := max(ks.learnedTo+1, span.From); m <= min(until-1, span.To); m++ {
if site.state(id, m) == StateNormal {
ks.baseline.Observe(m, 0)
}
}
}
ks.learnedTo = max(ks.learnedTo, until-1)
}
func (s *ReplaySession) minute(name string, site *siteSession, m int64, records []*Record, scored bool) Tick {
p := s.cfg.Params
if site.processedTo == 0 || m != site.processedTo+1 {
site.coveredFrom, site.windowFrom = m, m
site.restart()
}
if n := len(site.covered); n > 0 && site.covered[n-1].To == m-1 {
site.covered[n-1].To = m
} else {
site.covered = append(site.covered, Span{From: m, To: m})
}
requests := map[string]int64{}
for _, r := range records {
requests[r.Label]++
var h uint64
if s.cfg.Sketch != nil && r.Binding != "" {
h = s.cfg.Sketch.hash(r.Binding)
}
for _, id := range keysOf(r) {
ks := s.key(site, id, m)
mc := ks.minutes[m]
if mc == nil {
mc = newMinuteCounts()
ks.minutes[m] = mc
}
mc.add(r)
if s.cfg.Sketch != nil && r.Binding != "" {
sk := ks.sketches[m]
if sk == nil {
sk = newKeySketch(*s.cfg.Sketch)
ks.sketches[m] = sk
}
sk.insert(r.Binding, h)
}
}
}
ids := make([]KeyID, 0, len(site.active))
for id, ks := range site.active {
ids = append(ids, id)
if mc := ks.minutes[m]; mc != nil {
ks.window.apply(mc, 1)
}
if mc := ks.minutes[m-int64(p.W)]; mc != nil {
ks.window.apply(mc, -1)
delete(ks.minutes, m-int64(p.W))
}
delete(ks.sketches, m-int64(p.W))
}
slices.SortFunc(ids, keyOrder)
tick := Tick{Site: name, Minute: m, Complete: m-site.windowFrom+1 >= int64(p.W), Scored: scored, Requests: requests}
var unknown []KeyID
judged := map[KeyID]bool{}
for _, id := range ids {
ks := site.active[id]
if ks.finding != nil {
tick.Prior = append(tick.Prior, ActiveFinding{Key: id, Since: ks.finding.since, Uncertain: ks.finding.uncertain})
}
if m-max(site.windowFrom, ks.from)+1 < int64(p.W) {
if ks.window.Total() > 0 {
unknown = append(unknown, id)
}
continue
}
judged[id] = true
if ks.window.Total() > 0 || ks.finding != nil {
tick.Evaluations = append(tick.Evaluations, s.evaluate(ks, id, m))
}
}
tick.Scope = selectScope(tick.Evaluations, p.C, unknown)
for _, e := range tick.Evaluations {
ks := site.active[e.Key]
switch {
case e.Anomalous && ks.finding == nil:
ks.finding = &finding{since: m, pin: ks.baseline.Pin(m)}
tick.Events = append(tick.Events, s.event(name, site, e, tick.Scope, m))
case e.Anomalous:
ks.finding.uncertain = false
case ks.finding != nil:
ks.finding = nil
}
}
// An anomalous minute always leaves an active finding, so the finding
// freezes learning for it as well as for every later minute until a
// complete window clears it. Traffic no complete window judged may be an
// attack and is never learned; a covered minute without traffic is.
for _, id := range ids {
ks := site.active[id]
var total int64
if mc := ks.minutes[m]; mc != nil {
total = mc.total
}
if ks.finding == nil && (judged[id] || total == 0) && site.state(id, m) == StateNormal {
ks.baseline.Observe(m, total)
}
ks.learnedTo = m
if ks.finding != nil {
tick.Active = append(tick.Active, ActiveFinding{Key: id, Since: ks.finding.since, Uncertain: ks.finding.uncertain})
} else if ks.window.Total() == 0 {
ks.active, ks.minutes, ks.sketches = false, nil, nil
delete(site.active, id)
}
}
site.processedTo = m
return tick
}
func (s *ReplaySession) evaluate(ks *keyState, id KeyID, m int64) Evaluation {
p := s.cfg.Params
e := Evaluation{Key: id, Total: ks.window.Total(), Expensive: ks.window.Expensive(), Bindings: int64(len(ks.window.bindings)), Labels: ks.window.Labels()}
e.ExactResidual, e.ExactDistinct = ks.window.Residual(p.K)
expect := ks.baseline.expected
if ks.finding != nil {
expect = ks.finding.pin.expected
}
for j := m - int64(p.W) + 1; j <= m; j++ {
v, trusted := expect(j)
e.Expected += v
e.Trusted = e.Trusted || trusted
}
e.ExactAnomalous = p.anomalous(e.ExactResidual, e.ExactDistinct, e.Expected)
e.Residual, e.Distinct, e.Anomalous = e.ExactResidual, e.ExactDistinct, e.ExactAnomalous
if s.cfg.Sketch != nil {
minutes := make([]*keySketch, 0, p.W)
for j := m - int64(p.W) + 1; j <= m; j++ {
if sk := ks.sketches[j]; sk != nil {
minutes = append(minutes, sk)
}
}
bounds := composeSketches(minutes, p.K, s.cfg.Sketch.H)
e.Residual, e.Distinct = bounds.LN, bounds.LD
e.Anomalous = p.anomalous(e.Residual, e.Distinct, e.Expected)
}
return e
}
func (s *ReplaySession) event(name string, site *siteSession, e Evaluation, scope Scope, m int64) FindingEvent {
p := s.cfg.Params
exactA1, exactA2 := p.Margins(e.ExactResidual, e.ExactDistinct, e.Expected)
a1, a2 := p.Margins(e.Residual, e.Distinct, e.Expected)
return FindingEvent{Site: name, Key: e.Key, Minute: m, Total: e.Total, Expensive: e.Expensive, Expected: e.Expected,
Exact: Margin{A1: exactA1, A2: exactA2}, Bound: Margin{A1: a1, A2: a2}, Scope: scope, Labels: e.Labels,
CoveredFrom: site.coveredFrom, Trusted: e.Trusted}
}
// Evict drops a key's detail as a bounded detector would: its baseline,
// windows and summaries go, and when its traffic returns it starts cold and
// cannot act before W observed minutes of its own. The site key and keys
// with an active finding are never evicted.
func (s *ReplaySession) Evict(site string, id KeyID) error {
ss := s.sites[site]
if s.broken || ss == nil || id.Level == 3 || !validKeyID(id) {
return ErrSession
}
ks := ss.keys[id]
if ks == nil || ks.finding != nil {
return ErrSession
}
delete(ss.keys, id)
delete(ss.active, id)
ss.retired[id] = true
return nil
}
package crawlreplay
import (
"crypto/hmac"
"crypto/sha256"
"encoding/binary"
"slices"
"strings"
)
// SketchParams size the disposable per-minute summaries that phase 1 uses
// to choose m and h. They are prototypes, not the production
// sketch; a production sketch must reproduce these bounds against the same
// oracle.
type SketchParams struct {
M int `json:"m"` // Space-Saving counters per key-minute; must exceed K
H int `json:"h"` // smallest binding hashes kept per key-minute; at least D+K
Seed uint64 `json:"seed"` // private hash seed
// Hash overrides the keyed binding hash, for forced-collision tests.
Hash func(binding string) uint64 `json:"-"`
}
// bindingHashPurpose separates the cardinality hash from any other use of
// the same seed.
const bindingHashPurpose = "crawlreplay binding cardinality v1"
// hash is the keyed binding hash: HMAC-SHA256 under the big-endian seed
// over the purpose label, a zero byte and the binding, first eight bytes.
// An unkeyed hash would let a client choose bindings whose hashes collide
// or sort low and so steer the bottom-h witness.
func (p SketchParams) hash(binding string) uint64 {
if p.Hash != nil {
return p.Hash(binding)
}
var seed [8]byte
binary.BigEndian.PutUint64(seed[:], p.Seed)
m := hmac.New(sha256.New, seed[:])
m.Write([]byte(bindingHashPurpose))
m.Write([]byte{0})
m.Write([]byte(binding))
return binary.BigEndian.Uint64(m.Sum(nil))
}
type ssEntry struct {
binding string
count int64
err int64
}
// spaceSaving is an insertion-only Space-Saving summary with unit weights.
type spaceSaving struct {
m int
entries []ssEntry
index map[string]int
}
func newSpaceSaving(m int) *spaceSaving {
return &spaceSaving{m: m, index: make(map[string]int, m)}
}
func (s *spaceSaving) add(binding string) {
if i, ok := s.index[binding]; ok {
s.entries[i].count++
return
}
if len(s.entries) < s.m {
s.index[binding] = len(s.entries)
s.entries = append(s.entries, ssEntry{binding: binding, count: 1})
return
}
// Replace the minimum counter. Ties go to the smallest binding, so the
// choice depends on the counters alone, not on their slice order,
// which a restored summary does not keep.
low := 0
for i := range s.entries {
e, l := s.entries[i], s.entries[low]
if e.count < l.count || (e.count == l.count && e.binding < l.binding) {
low = i
}
}
old := s.entries[low]
delete(s.index, old.binding)
s.index[binding] = low
s.entries[low] = ssEntry{binding: binding, count: old.count + 1, err: old.count}
}
// floor is b_j: the minimum counter when the summary is full, else zero.
// An absent binding's true count in this minute is at most floor.
func (s *spaceSaving) floor() int64 {
if len(s.entries) < s.m {
return 0
}
low := s.entries[0].count
for _, e := range s.entries[1:] {
low = min(low, e.count)
}
return low
}
// bottomH keeps the h smallest distinct binding hashes, ascending.
type bottomH struct {
h int
hashes []uint64
}
func (b *bottomH) add(x uint64) {
i, found := slices.BinarySearch(b.hashes, x)
if found || (len(b.hashes) == b.h && i == b.h) {
return
}
b.hashes = slices.Insert(b.hashes, i, x)
if len(b.hashes) > b.h {
b.hashes = b.hashes[:b.h]
}
}
// keySketch is one key's summaries for one minute.
type keySketch struct {
ss *spaceSaving
hs *bottomH
bound int64 // exact count of bound requests this minute
}
func newKeySketch(p SketchParams) *keySketch {
return &keySketch{ss: newSpaceSaving(p.M), hs: &bottomH{h: p.H}}
}
func (k *keySketch) add(p SketchParams, binding string) { k.insert(binding, p.hash(binding)) }
// insert counts one request of binding, whose keyed hash is h.
func (k *keySketch) insert(binding string, h uint64) {
k.bound++
k.ss.add(binding)
k.hs.add(h)
}
// sketchWindow is the composition of one key's W minute summaries.
type sketchWindow struct {
N int64 // exact bound requests in the window
B int64 // sum of the minute floors: bound for any absent binding
Upper map[string]int64 // U(x) for every binding some summary retained
Top int64 // T_U: the K largest U values, padded with B
S int // distinct retained hashes in the window, at most h
LN int64 // max(0, N - T_U)
LD int64 // max(0, S - K)
}
// composeSketches composes W minute summaries into the conservative
// residual bounds. The union is recomputed from whole minutes;
// nothing is subtracted or recompressed.
func composeSketches(minutes []*keySketch, k, h int) sketchWindow {
w := sketchWindow{Upper: map[string]int64{}}
floors := make([]int64, len(minutes))
for j, ks := range minutes {
w.N += ks.bound
floors[j] = ks.ss.floor()
w.B += floors[j]
for _, e := range ks.ss.entries {
w.Upper[e.binding] = 0
}
}
for j, ks := range minutes {
present := make(map[string]int64, len(ks.ss.entries))
for _, e := range ks.ss.entries {
present[e.binding] = e.count
}
for binding := range w.Upper {
if c, ok := present[binding]; ok {
w.Upper[binding] += c
} else {
w.Upper[binding] += floors[j]
}
}
}
us := make([]int64, 0, len(w.Upper)+k)
for _, u := range w.Upper {
us = append(us, u)
}
for len(us) < k {
us = append(us, w.B)
}
slices.Sort(us)
for i := len(us) - 1; i >= len(us)-k; i-- {
w.Top += us[i]
}
var union []uint64
for _, ks := range minutes {
union = append(union, ks.hs.hashes...)
}
slices.Sort(union)
union = slices.Compact(union)
w.S = min(len(union), h)
w.LN, w.LD = max(0, w.N-w.Top), int64(max(0, w.S-k))
return w
}
// sketchState is a minute summary as a snapshot stores it.
type sketchState struct {
Bound int64 `json:"bound"`
Entries []sketchEntry `json:"entries"` // ordered by binding
Hashes []uint64 `json:"hashes"` // ascending
}
type sketchEntry struct {
Binding string `json:"b"`
Count int64 `json:"c"`
Err int64 `json:"e"`
}
func (k *keySketch) state() sketchState {
st := sketchState{Bound: k.bound, Entries: make([]sketchEntry, 0, len(k.ss.entries)), Hashes: slices.Clone(k.hs.hashes)}
for _, e := range k.ss.entries {
st.Entries = append(st.Entries, sketchEntry{Binding: e.binding, Count: e.count, Err: e.err})
}
slices.SortFunc(st.Entries, func(a, b sketchEntry) int { return strings.Compare(a.Binding, b.Binding) })
if st.Hashes == nil {
st.Hashes = []uint64{}
}
return st
}
// restoreSketch rebuilds a summary, refusing one that breaks the
// Space-Saving invariants: counters sum to the bound requests, each
// insertion error is below its counter, and no error exists before the
// summary filled.
func restoreSketch(p SketchParams, st sketchState) (*keySketch, error) {
if st.Bound < 1 || st.Entries == nil || len(st.Entries) == 0 || len(st.Entries) > p.M ||
st.Hashes == nil || len(st.Hashes) == 0 || len(st.Hashes) > p.H {
return nil, ErrSession
}
ks := newKeySketch(p)
var total int64
for i, e := range st.Entries {
if e.Binding == "" || (i > 0 && e.Binding <= st.Entries[i-1].Binding) || e.Count < 1 || e.Err < 0 || e.Err >= e.Count ||
(len(st.Entries) < p.M && e.Err != 0) {
return nil, ErrSession
}
var ok bool
if total, ok = checkedSum(total, e.Count); !ok {
return nil, ErrSession
}
ks.ss.index[e.Binding] = i
ks.ss.entries = append(ks.ss.entries, ssEntry{binding: e.Binding, count: e.Count, err: e.Err})
}
for i := 1; i < len(st.Hashes); i++ {
if st.Hashes[i] <= st.Hashes[i-1] {
return nil, ErrSession
}
}
if total != st.Bound {
return nil, ErrSession
}
ks.bound, ks.hs.hashes = st.Bound, slices.Clone(st.Hashes)
return ks, nil
}
package crawlreplay
import (
"cmp"
"encoding/json"
"maps"
"math"
"slices"
"strings"
)
// SnapshotVersion is the session snapshot format.
const SnapshotVersion = 1
// Invalidation reports which restored state a new configuration could not
// interpret. Invalid windows restart, so a complete new window is needed
// before any key is judged; invalid baselines restart cold; a changed
// identity contract drops every key. Active findings are never dropped:
// they stay active, uncertain, until a complete window judges them.
type Invalidation struct {
Windows bool `json:"windows"`
Baselines bool `json:"baselines"`
Identity bool `json:"identity"`
}
type sessionSnapshot struct {
FormatVersion int `json:"format_version"`
Params Params `json:"params"`
Sketch *SketchParams `json:"sketch,omitempty"`
HashVersion string `json:"hash_version,omitempty"`
Shuffle uint64 `json:"shuffle"`
IdentityVersion int `json:"identity_version"`
Sites []siteSnapshot `json:"sites"`
}
type siteSnapshot struct {
Site string `json:"site"`
ProcessedTo int64 `json:"processed_to"`
CoveredFrom int64 `json:"covered_from"`
WindowFrom int64 `json:"window_from"`
Covered []Span `json:"covered"`
States []StateSpan `json:"states"`
Keys []keySnapshot `json:"keys"`
Retired []KeyID `json:"retired"`
}
type keySnapshot struct {
Key KeyID `json:"key"`
First int64 `json:"first"`
Learned int64 `json:"learned"`
LearnedTo int64 `json:"learned_to"`
From int64 `json:"from"`
Slots []slotSnapshot `json:"slots"`
Minutes []minuteSnapshot `json:"minutes"`
Sketches []sketchSnapshot `json:"sketches"`
Finding *findingSnapshot `json:"finding,omitempty"`
}
type slotSnapshot struct {
Slot int `json:"slot"`
Mean float64 `json:"mean"`
Obs int `json:"obs"`
Last int64 `json:"last"`
}
type countSnapshot struct {
Name string `json:"name"`
N int64 `json:"n"`
}
type minuteSnapshot struct {
Minute int64 `json:"minute"`
Total int64 `json:"total"`
Expensive int64 `json:"expensive"`
Bindings []countSnapshot `json:"bindings"`
Labels []countSnapshot `json:"labels"`
}
type sketchSnapshot struct {
Minute int64 `json:"minute"`
Summary sketchState `json:"summary"`
}
type findingSnapshot struct {
Since int64 `json:"since"`
Uncertain bool `json:"uncertain"`
PinFirst int64 `json:"pin_first"`
PinAt int64 `json:"pin_at"`
PinSlots []slotSnapshot `json:"pin_slots"`
}
// Snapshot serializes the session between segments: configuration, every
// site's coverage and state history, and every key's baseline, window,
// summaries and finding. A session using a test hash override cannot be
// serialized.
func (s *ReplaySession) Snapshot() ([]byte, error) {
if s.broken || (s.cfg.Sketch != nil && s.cfg.Sketch.Hash != nil) {
return nil, ErrSession
}
snap := sessionSnapshot{FormatVersion: SnapshotVersion, Params: s.cfg.Params, Sketch: s.cfg.Sketch,
Shuffle: s.cfg.Shuffle, IdentityVersion: s.cfg.IdentityVersion, Sites: []siteSnapshot{}}
if s.cfg.Sketch != nil {
snap.HashVersion = bindingHashPurpose
}
for _, name := range slices.Sorted(maps.Keys(s.sites)) {
site := s.sites[name]
ss := siteSnapshot{Site: name, ProcessedTo: site.processedTo, CoveredFrom: site.coveredFrom, WindowFrom: site.windowFrom,
Covered: slices.Clone(site.covered), States: slices.Clone(site.siteStates), Keys: []keySnapshot{}, Retired: []KeyID{}}
if ss.Covered == nil {
ss.Covered = []Span{}
}
for _, id := range slices.SortedFunc(maps.Keys(site.keyStates), keyOrder) {
ss.States = append(ss.States, site.keyStates[id]...)
}
if ss.States == nil {
ss.States = []StateSpan{}
}
for _, id := range slices.SortedFunc(maps.Keys(site.keys), keyOrder) {
ss.Keys = append(ss.Keys, snapshotKey(id, site.keys[id]))
}
ss.Retired = append(ss.Retired, slices.SortedFunc(maps.Keys(site.retired), keyOrder)...)
snap.Sites = append(snap.Sites, ss)
}
return json.Marshal(snap)
}
func snapshotSlots(slots *[168]slot) []slotSnapshot {
out := []slotSnapshot{}
for i, sl := range slots {
if sl.obs > 0 {
out = append(out, slotSnapshot{Slot: i, Mean: sl.mean, Obs: sl.obs, Last: sl.last})
}
}
return out
}
func snapshotCounts(counts map[string]int64) []countSnapshot {
out := []countSnapshot{}
for _, name := range slices.Sorted(maps.Keys(counts)) {
out = append(out, countSnapshot{Name: name, N: counts[name]})
}
return out
}
func snapshotKey(id KeyID, ks *keyState) keySnapshot {
k := keySnapshot{Key: id, First: ks.baseline.first, Learned: ks.baseline.learned, LearnedTo: ks.learnedTo, From: ks.from,
Slots: snapshotSlots(&ks.baseline.slots), Minutes: []minuteSnapshot{}, Sketches: []sketchSnapshot{}}
for _, m := range slices.Sorted(maps.Keys(ks.minutes)) {
mc := ks.minutes[m]
k.Minutes = append(k.Minutes, minuteSnapshot{Minute: m, Total: mc.total, Expensive: mc.expensive,
Bindings: snapshotCounts(mc.bindings), Labels: snapshotCounts(mc.labels)})
}
for _, m := range slices.Sorted(maps.Keys(ks.sketches)) {
k.Sketches = append(k.Sketches, sketchSnapshot{Minute: m, Summary: ks.sketches[m].state()})
}
if f := ks.finding; f != nil {
k.Finding = &findingSnapshot{Since: f.since, Uncertain: f.uncertain, PinFirst: f.pin.first, PinAt: f.pin.at, PinSlots: snapshotSlots(&f.pin.slots)}
}
return k
}
// Restore rebuilds a session from Snapshot bytes with its own copy of cfg.
// State cfg cannot interpret is invalidated as Invalidation reports;
// threshold changes (R, F, K, D, C) invalidate nothing and never rewrite a
// pinned profile. Malformed or inconsistent snapshots are refused with
// ErrSession.
func Restore(raw []byte, cfg SessionConfig) (*ReplaySession, Invalidation, error) {
if err := cfg.validate(); err != nil {
return nil, Invalidation{}, err
}
var snap sessionSnapshot
if err := DecodeStrictJSON(raw, &snap); err != nil {
return nil, Invalidation{}, ErrSession
}
stored := SessionConfig{Params: snap.Params, Sketch: snap.Sketch, Shuffle: snap.Shuffle, IdentityVersion: snap.IdentityVersion}
if snap.FormatVersion != SnapshotVersion || stored.validate() != nil || (snap.Sketch == nil) != (snap.HashVersion == "") || snap.Sites == nil {
return nil, Invalidation{}, ErrSession
}
s := &ReplaySession{cfg: cfg.clone(), sites: map[string]*siteSession{}}
for _, ss := range snap.Sites {
if s.sites[ss.Site] != nil || !ValidSite(ss.Site) {
return nil, Invalidation{}, ErrSession
}
site, err := restoreSite(ss, stored, snap.HashVersion == bindingHashPurpose)
if err != nil {
return nil, Invalidation{}, err
}
s.sites[ss.Site] = site
}
var inv Invalidation
inv.Identity = stored.IdentityVersion != cfg.IdentityVersion
inv.Baselines = inv.Identity || stored.Params.Baseline != cfg.Params.Baseline
inv.Windows = inv.Baselines || stored.Params.W != cfg.Params.W || !sameSketch(snap.Sketch, snap.HashVersion, cfg.Sketch)
for _, site := range s.sites {
if inv.Identity {
for id, ks := range site.keys {
if ks.finding == nil {
delete(site.keys, id)
delete(site.active, id)
}
}
clear(site.retired)
clear(site.keyStates)
}
if inv.Windows {
if site.processedTo > 0 {
site.windowFrom = site.processedTo + 1
}
site.restart()
}
if inv.Baselines {
for _, ks := range site.keys {
ks.baseline = NewBaseline(cfg.Params.Baseline, site.processedTo+1)
ks.baseline.learned = site.processedTo
ks.learnedTo = site.processedTo
if ks.finding != nil {
ks.finding.pin = ks.baseline.Pin(site.processedTo + 1)
ks.finding.uncertain = true
}
}
}
}
return s, inv, nil
}
func sameSketch(stored *SketchParams, hashVersion string, current *SketchParams) bool {
if stored == nil || current == nil {
return stored == nil && current == nil
}
return current.Hash == nil && hashVersion == bindingHashPurpose &&
stored.M == current.M && stored.H == current.H && stored.Seed == current.Seed
}
func restoreSite(ss siteSnapshot, cfg SessionConfig, checkHash bool) (*siteSession, error) {
site := newSiteSession()
site.processedTo, site.coveredFrom, site.windowFrom = ss.ProcessedTo, ss.CoveredFrom, ss.WindowFrom
switch {
case ss.ProcessedTo > 0 && len(ss.Covered) == 0:
return nil, ErrSession
case ss.ProcessedTo < 0, ss.Covered == nil, ss.States == nil, ss.Keys == nil, ss.Retired == nil:
return nil, ErrSession
case ss.ProcessedTo == 0 && (len(ss.Covered) > 0 || ss.CoveredFrom != 0 || ss.WindowFrom != 0 || len(ss.Keys) > 0 || len(ss.Retired) > 0):
return nil, ErrSession
}
for i, sp := range ss.Covered {
if sp.From <= 0 || sp.To < sp.From || (i > 0 && sp.From <= ss.Covered[i-1].To+1) {
return nil, ErrSession
}
}
if n := len(ss.Covered); n > 0 {
last := ss.Covered[n-1]
if last.To != ss.ProcessedTo || ss.CoveredFrom < last.From || ss.CoveredFrom > ss.ProcessedTo ||
ss.WindowFrom < ss.CoveredFrom || ss.WindowFrom > ss.ProcessedTo+1 {
return nil, ErrSession
}
}
site.covered = ss.Covered
byScope := map[KeyID][]StateSpan{}
for _, st := range ss.States {
scope := KeyID{}
if st.Key != nil {
if !validKeyID(*st.Key) {
return nil, ErrSession
}
scope = *st.Key
}
if !learningStates[st.State] || st.From <= 0 || st.To < st.From {
return nil, ErrSession
}
for _, other := range byScope[scope] {
if st.From <= other.To && other.From <= st.To {
return nil, ErrSession
}
}
byScope[scope] = append(byScope[scope], st)
if st.Key == nil {
site.siteStates = append(site.siteStates, st)
} else {
site.keyStates[*st.Key] = append(site.keyStates[*st.Key], st)
}
}
for _, spans := range append([][]StateSpan{site.siteStates}, slices.Collect(maps.Values(site.keyStates))...) {
slices.SortFunc(spans, func(a, b StateSpan) int { return cmp.Compare(a.From, b.From) })
}
for _, ks := range ss.Keys {
if !validKeyID(ks.Key) || site.keys[ks.Key] != nil {
return nil, ErrSession
}
state, err := restoreKey(ks, ss, cfg, checkHash)
if err != nil {
return nil, err
}
site.keys[ks.Key] = state
if state.active {
site.active[ks.Key] = state
}
}
for _, id := range ss.Retired {
if !validKeyID(id) || id.Level == 3 || site.keys[id] != nil || site.retired[id] {
return nil, ErrSession
}
site.retired[id] = true
}
return site, nil
}
func restoreSlots(in []slotSnapshot, first, learned int64) ([168]slot, error) {
var out [168]slot
seen := map[int]bool{}
for _, sl := range in {
if sl.Slot < 0 || sl.Slot >= 168 || seen[sl.Slot] || sl.Obs < 1 || math.IsNaN(sl.Mean) || math.IsInf(sl.Mean, 0) || sl.Mean < 0 ||
sl.Last < first || sl.Last > learned || hourOfWeek(sl.Last) != sl.Slot {
return out, ErrSession
}
seen[sl.Slot] = true
out[sl.Slot] = slot{mean: sl.Mean, obs: sl.Obs, last: sl.Last}
}
return out, nil
}
// validLabelKey reports a window label counter name (see labelKey).
func validLabelKey(name string) bool {
label, episode, episodic := strings.Cut(name, "/")
if !episodic {
return label == "" || label == LabelHealthy
}
return (label == LabelAttack || label == LabelOverload) && episodeID.MatchString(episode)
}
func restoreCounts(in []countSnapshot, valid func(string) bool) (map[string]int64, int64, error) {
out := map[string]int64{}
var sum int64
for _, c := range in {
if !valid(c.Name) || c.N < 1 || out[c.Name] != 0 {
return nil, 0, ErrSession
}
out[c.Name] = c.N
var ok bool
if sum, ok = checkedSum(sum, c.N); !ok {
return nil, 0, ErrSession
}
}
return out, sum, nil
}
func restoreKey(k keySnapshot, ss siteSnapshot, cfg SessionConfig, checkHash bool) (*keyState, error) {
w := int64(cfg.Params.W)
switch {
case k.First <= 0, k.Learned < 0, k.Learned > k.LearnedTo, k.LearnedTo > ss.ProcessedTo, k.LearnedTo < k.First-1,
k.From < 0, k.From > ss.ProcessedTo, k.Slots == nil, k.Minutes == nil, k.Sketches == nil:
return nil, ErrSession
}
slots, err := restoreSlots(k.Slots, k.First, k.Learned)
if err != nil {
return nil, err
}
ks := &keyState{baseline: &Baseline{p: cfg.Params.Baseline, first: k.First, learned: k.Learned, slots: slots},
window: newWindow(), learnedTo: k.LearnedTo, from: k.From}
minutes := map[int64]*minuteCounts{}
for _, m := range k.Minutes {
if m.Minute <= ss.ProcessedTo-w || m.Minute > ss.ProcessedTo || m.Minute < ss.WindowFrom || minutes[m.Minute] != nil ||
m.Total < 1 || m.Expensive < 0 || m.Expensive > m.Total || (k.Key.Level < 3 && m.Expensive != m.Total) {
return nil, ErrSession
}
bindings, bound, err := restoreCounts(m.Bindings, bindingPseudonym.MatchString)
if err != nil || bound > m.Total {
return nil, ErrSession
}
labels, labelled, err := restoreCounts(m.Labels, validLabelKey)
if err != nil || labelled != m.Total {
return nil, ErrSession
}
if _, ok := checkedSum(ks.window.Total(), m.Total); !ok {
return nil, ErrSession
}
mc := &minuteCounts{bindings: bindings, total: m.Total, expensive: m.Expensive, labels: labels}
minutes[m.Minute] = mc
ks.window.apply(mc, 1)
}
sketches := map[int64]*keySketch{}
for _, sk := range k.Sketches {
mc := minutes[sk.Minute]
if cfg.Sketch == nil || mc == nil || sketches[sk.Minute] != nil {
return nil, ErrSession
}
restored, err := restoreSketch(*cfg.Sketch, sk.Summary)
if err != nil {
return nil, err
}
var bound int64
for _, n := range mc.bindings {
bound += n
}
for _, e := range restored.ss.entries {
if n := mc.bindings[e.binding]; n == 0 || n > e.count || n < e.count-e.err {
return nil, ErrSession
}
}
witness := &bottomH{h: cfg.Sketch.H}
for binding, n := range mc.bindings {
if _, retained := restored.ss.index[binding]; !retained && n > restored.ss.floor() {
return nil, ErrSession
}
if checkHash {
witness.add(cfg.Sketch.hash(binding))
}
}
// An older hash contract cannot be recomputed here; its windows
// are discarded by Restore before any decision is allowed.
if checkHash && !slices.Equal(witness.hashes, restored.hs.hashes) {
return nil, ErrSession
}
if restored.bound != bound {
return nil, ErrSession
}
sketches[sk.Minute] = restored
}
if cfg.Sketch != nil {
for m, mc := range minutes {
if len(mc.bindings) > 0 && sketches[m] == nil {
return nil, ErrSession
}
}
}
if f := k.Finding; f != nil {
if f.PinSlots == nil || f.Since <= 0 || f.Since > ss.ProcessedTo || f.PinFirst != k.First ||
f.PinAt < max(f.Since, f.PinFirst) || f.PinAt > ss.ProcessedTo+1 {
return nil, ErrSession
}
pinSlots, err := restoreSlots(f.PinSlots, k.First, min(f.PinAt, ss.ProcessedTo))
if err != nil {
return nil, err
}
ks.finding = &finding{since: f.Since, uncertain: f.Uncertain,
pin: Profile{p: cfg.Params.Baseline, first: f.PinFirst, at: f.PinAt, slots: pinSlots}}
}
ks.active = ks.finding != nil || ks.window.Total() > 0
if ks.active {
ks.minutes, ks.sketches = minutes, sketches
} else if len(minutes) > 0 {
return nil, ErrSession
}
return ks, nil
}
package crawlreplay
import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"reflect"
"regexp"
"strings"
)
// StreamVersion is the record and volume stream format version. Version 2
// replaced version 1's range flag with verified-bot proof classes.
const StreamVersion = 2
// Request classes, mirroring crawlid.Classify.
const (
ClassOther uint8 = iota
ClassDynamic
ClassExpensive
)
// Referer classes, mirroring the domlog record parser.
const (
RefNone uint8 = iota
RefMalformed
RefCrossSite
RefSameSite
)
// Verified-bot proof classes of a claimed bot identity, from historical
// evidence of the verified-bot list for that identity at the logged time. A
// claim without one stays unverified.
const (
BotProofRange = "range"
BotProofDNS = "dns"
BotProofNegative = "negative"
)
// Labels an operator assigns before anonymization.
const (
LabelAttack = "attack"
LabelHealthy = "healthy"
LabelOverload = "overload"
)
var (
// ErrRecord reports a stream row that breaks the format. Messages name
// the field, never the value, so a refusal cannot echo private input.
ErrRecord = errors.New("crawlreplay: invalid row")
// ErrDecode reports bytes that are not a stream row.
ErrDecode = errors.New("crawlreplay: undecodable row")
sitePseudonym = regexp.MustCompile(`^dom-[0-9a-f]{6}\.example$`)
accountPseudonym = regexp.MustCompile(`^acct-[0-9a-f]{6}$`)
bindingPseudonym = regexp.MustCompile(`^b-[0-9a-f]{16}$`)
keyPseudonym = regexp.MustCompile(`^k-[0-9a-f]{16}$`)
botIdentity = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`)
episodeID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`)
)
// Record is one logged request with every identity replaced by a salted
// pseudonym. Keys and bindings keep equality, L1 and L2 keep the hierarchy
// (both belong to the record's site), and no target, query value, address,
// user agent or Referer survives. File and Seq keep the logged order.
type Record struct {
T int64 `json:"t"`
File int `json:"f"`
Seq int64 `json:"n"`
Site string `json:"site"`
Account string `json:"acct,omitempty"`
Binding string `json:"b,omitempty"`
L2 string `json:"l2,omitempty"`
L1 string `json:"l1,omitempty"`
Class uint8 `json:"c"`
Status int `json:"s"`
Referer uint8 `json:"r"`
Bot string `json:"bot,omitempty"`
BotProof string `json:"bot_proof,omitempty"`
Infra bool `json:"infra,omitempty"`
Label string `json:"label,omitempty"`
Episode string `json:"ep,omitempty"`
}
// Validate checks every field against the closed stream format.
func (r Record) Validate() error {
switch {
case r.T <= 0:
return fieldError("t")
case r.File < 0:
return fieldError("f")
case r.Seq < 1:
return fieldError("n")
case !sitePseudonym.MatchString(r.Site):
return fieldError("site")
case r.Account != "" && !accountPseudonym.MatchString(r.Account):
return fieldError("acct")
case r.Binding != "" && !bindingPseudonym.MatchString(r.Binding):
return fieldError("b")
case r.Class > ClassExpensive:
return fieldError("c")
case (r.Class == ClassExpensive) != (r.L2 != ""),
r.L2 != "" && !keyPseudonym.MatchString(r.L2):
return fieldError("l2")
case (r.Class == ClassExpensive) != (r.L1 != ""),
r.L1 != "" && !keyPseudonym.MatchString(r.L1):
return fieldError("l1")
case r.Status < 100 || r.Status > 599:
return fieldError("s")
case r.Referer > RefSameSite:
return fieldError("r")
case r.Bot != "" && !botIdentity.MatchString(r.Bot):
return fieldError("bot")
case r.BotProof != "" && (r.Bot == "" || r.Binding == "" || (r.BotProof != BotProofRange && r.BotProof != BotProofDNS && r.BotProof != BotProofNegative)):
return fieldError("bot_proof")
}
episodic := r.Label == LabelAttack || r.Label == LabelOverload
switch {
case r.Label != "" && !episodic && r.Label != LabelHealthy:
return fieldError("label")
case episodic != (r.Episode != ""),
r.Episode != "" && !episodeID.MatchString(r.Episode):
return fieldError("ep")
}
return nil
}
// Volume counts one site's logged lines in one UTC minute, including lines
// that yielded no target or no client binding, for read-budget and coverage
// calibration. Lines the parser refused or that carry no valid time cannot
// be placed in a minute; the converter's manifest counts those.
type Volume struct {
Site string `json:"site"`
Minute int64 `json:"m"`
Lines int64 `json:"lines"`
Bytes int64 `json:"bytes"`
NoTarget int64 `json:"no_target"`
NoBinding int64 `json:"no_binding"`
}
// Validate checks every field against the closed stream format.
func (v Volume) Validate() error {
switch {
case !sitePseudonym.MatchString(v.Site):
return fieldError("site")
case v.Minute <= 0:
return fieldError("m")
case v.Lines < 1:
return fieldError("lines")
case v.Bytes < v.Lines:
return fieldError("bytes")
case v.NoTarget < 0 || v.NoTarget > v.Lines:
return fieldError("no_target")
case v.NoBinding < 0 || v.NoBinding > v.Lines:
return fieldError("no_binding")
}
return nil
}
func fieldError(name string) error { return fmt.Errorf("%w: %s", ErrRecord, name) }
// ValidSite reports whether name is a site pseudonym of the closed format,
// so a bundle's other files can be held to the rule its rows follow.
func ValidSite(name string) bool { return sitePseudonym.MatchString(name) }
type row interface{ Validate() error }
// ReadRecords decodes a JSON record stream and calls fn for each record in
// stream order. Fields must use their exact JSON names, occur at most once,
// and be non-null. Fields without omitempty are required, even when zero.
// Invalid rows stop the read without calling fn for that row.
func ReadRecords(r io.Reader, fn func(Record) error) error { return readRows(r, fn) }
// ReadVolume decodes a JSON volume stream under the same field rules as
// ReadRecords and calls fn for each valid row.
func ReadVolume(r io.Reader, fn func(Volume) error) error { return readRows(r, fn) }
func readRows[T row](r io.Reader, fn func(T) error) error {
// Derive required and optional names from the writer's schema so the
// reader cannot silently invent zero-valued fields or accept aliases.
fields := make(map[string]bool)
typ := reflect.TypeFor[T]()
for i := range typ.NumField() {
name, option, _ := strings.Cut(typ.Field(i).Tag.Get("json"), ",")
fields[name] = option != "omitempty"
}
dec := json.NewDecoder(bufio.NewReader(r))
for n := 1; ; n++ {
var data json.RawMessage
if err := dec.Decode(&data); err != nil {
if errors.Is(err, io.EOF) {
return nil
}
return fmt.Errorf("row %d: %w", n, ErrDecode)
}
var v T
if err := decodeRow(data, &v, fields); err != nil {
return fmt.Errorf("row %d: %w", n, ErrDecode)
}
if err := v.Validate(); err != nil {
return fmt.Errorf("row %d: %w", n, err)
}
if err := fn(v); err != nil {
return err
}
}
}
func decodeRow(data []byte, v any, fields map[string]bool) error {
// Unmarshal alone accepts case-insensitive names, repeated keys and
// null scalars. Check the object before those distinctions are lost.
dec := json.NewDecoder(bytes.NewReader(data))
tok, err := dec.Token()
if err != nil || tok != json.Delim('{') {
return ErrDecode
}
seen := make(map[string]bool, len(fields))
for dec.More() {
tok, err := dec.Token()
if err != nil {
return ErrDecode
}
name, _ := tok.(string)
if _, ok := fields[name]; !ok || seen[name] {
return ErrDecode
}
seen[name] = true
var value json.RawMessage
if err := dec.Decode(&value); err != nil || string(value) == "null" {
return ErrDecode
}
}
if _, err := dec.Token(); err != nil {
return ErrDecode
}
for name, required := range fields {
if required && !seen[name] {
return ErrDecode
}
}
return json.Unmarshal(data, v)
}
// WriteRow validates v and writes it as one JSON line.
func WriteRow[T row](w io.Writer, v T) error {
if err := v.Validate(); err != nil {
return err
}
b, err := json.Marshal(v)
if err != nil {
return err
}
b = append(b, '\n')
n, err := w.Write(b)
if err == nil && n < len(b) {
return io.ErrShortWrite
}
return err
}
package crawlreplay
import (
"fmt"
"math/rand/v2"
)
// Synth builds deterministic synthetic records for fixtures and sweeps.
// Every binding it creates is new unless a pool is named, so one generator
// can model any number of distinct clients.
type Synth struct {
site string
rng *rand.Rand
seq int64
fresh uint64
}
// NewSynth starts a generator for one site pseudonym.
func NewSynth(site string, seed uint64) *Synth {
// #nosec G404 -- fixtures must be reproducible; nothing here is a secret.
return &Synth{site: site, rng: rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))}
}
// SynthKey returns a valid key pseudonym for fixture number n.
func SynthKey(n uint64) string { return fmt.Sprintf("k-%016x", n) }
// synthBinding returns a valid binding pseudonym for client number n.
func synthBinding(n uint64) string { return fmt.Sprintf("b-%016x", n) }
func (s *Synth) newBinding() string {
s.fresh++
return synthBinding(1<<48 | s.fresh)
}
// Traffic describes one workload on one key over a minute range.
type Traffic struct {
From, To int64 // inclusive Unix minutes
PerMinute int // requests per minute
L2, L1 string // key pseudonyms; both empty for queryless dynamic traffic
Label string
Episode string
}
func (s *Synth) record(minute int64, t Traffic, binding string) Record {
s.seq++
r := Record{
T: minute*60 + s.rng.Int64N(60), Seq: s.seq, Site: s.site, Binding: binding,
Class: ClassDynamic, Status: 200, Label: t.Label, Episode: t.Episode,
}
if t.L2 != "" {
r.Class, r.L2, r.L1 = ClassExpensive, t.L2, t.L1
}
return r
}
// Pool sends the traffic from a fixed pool of clients, uniformly.
func (s *Synth) Pool(t Traffic, clients int) []Record {
pool := make([]string, clients)
for i := range pool {
pool[i] = s.newBinding()
}
var out []Record
for m := t.From; m <= t.To; m++ {
for range t.PerMinute {
out = append(out, s.record(m, t, pool[s.rng.IntN(clients)]))
}
}
return out
}
// Rotating sends the traffic from fresh clients that each make exactly q
// requests before the next one starts (the last client of the range may
// make fewer).
func (s *Synth) Rotating(t Traffic, q int) []Record {
return s.rotating(t, q, func(int64) int { return t.PerMinute })
}
func (s *Synth) rotating(t Traffic, q int, perMinute func(int64) int) []Record {
var out []Record
binding, used := s.newBinding(), 0
for m := t.From; m <= t.To; m++ {
for range perMinute(m) {
if used == q {
binding, used = s.newBinding(), 0
}
out = append(out, s.record(m, t, binding))
used++
}
}
return out
}
// Heavy sends sources clients at perSource requests each per minute; with
// churn every minute uses new clients, otherwise the same ones.
func (s *Synth) Heavy(t Traffic, sources, perSource int, churn bool) []Record {
var out []Record
current := make([]string, sources)
for m := t.From; m <= t.To; m++ {
for i := range current {
if churn || current[i] == "" {
current[i] = s.newBinding()
}
for range perSource {
out = append(out, s.record(m, t, current[i]))
}
}
}
return out
}
// Ramp sends fresh q-request clients at a rate rising linearly from
// startPerMin to endPerMin over the range.
func (s *Synth) Ramp(t Traffic, startPerMin, endPerMin, q int) []Record {
span := max(1, t.To-t.From)
return s.rotating(t, q, func(m int64) int {
return startPerMin + int(int64(endPerMin-startPerMin)*(m-t.From)/span)
})
}
package crawlreplay
import (
"cmp"
"maps"
"math"
"slices"
)
// Quantiles are nearest-rank summary statistics of a sample.
type Quantiles struct {
N int `json:"n"`
P50 float64 `json:"p50"`
P99 float64 `json:"p99"`
Max float64 `json:"max"`
}
// SummarizeCounts computes nearest-rank quantiles of a histogram that maps
// each integer value to how often it occurred.
func SummarizeCounts(hist map[int64]int64) Quantiles {
values := make([]int64, 0, len(hist))
var n int64
for v, c := range hist {
if c > 0 {
values = append(values, v)
n += c
}
}
if n == 0 {
return Quantiles{}
}
slices.Sort(values)
rank := func(q float64) float64 {
target := max(1, int64(math.Ceil(q*float64(n))))
var seen int64
for _, v := range values {
if seen += hist[v]; seen >= target {
return float64(v)
}
}
return float64(values[len(values)-1])
}
return Quantiles{N: int(n), P50: rank(0.50), P99: rank(0.99), Max: float64(values[len(values)-1])}
}
// Summarize computes nearest-rank quantiles; an empty sample is all zero.
func Summarize(sample []float64) Quantiles {
if len(sample) == 0 {
return Quantiles{}
}
s := slices.Clone(sample)
slices.Sort(s)
rank := func(q float64) float64 { return s[max(0, int(math.Ceil(q*float64(len(s))))-1)] }
return Quantiles{N: len(s), P50: rank(0.50), P99: rank(0.99), Max: s[len(s)-1]}
}
// HostVolume summarizes logged lines and bytes per minute over every
// minute from the earliest to the latest volume row; a minute no site
// logged counts as zero. SummarizeVolumePeriods restricts that extent to
// the supplied recording periods.
type HostVolume struct {
LinesPerMinute Quantiles `json:"lines_per_minute"`
BytesPerMinute Quantiles `json:"bytes_per_minute"`
SiteLinesPerMinute Quantiles `json:"site_lines_per_minute"` // site-minutes with lines
}
// SummarizeVolume aggregates volume rows across sites.
func SummarizeVolume(rows []Volume) HostVolume {
return SummarizeVolumePeriods(rows, nil)
}
// SummarizeVolumePeriods excludes gaps between recording periods. Periods
// must be chronological and disjoint and contain every row. A nil list
// uses the full observed extent, as SummarizeVolume does.
func SummarizeVolumePeriods(rows []Volume, periods []Span) HostVolume {
if len(rows) == 0 {
return HostVolume{}
}
lines, bytes := map[int64]float64{}, map[int64]float64{}
type siteMinute struct {
site string
minute int64
}
siteLines := map[siteMinute]float64{}
first, last := rows[0].Minute, rows[0].Minute
for _, v := range rows {
lines[v.Minute] += float64(v.Lines)
bytes[v.Minute] += float64(v.Bytes)
siteLines[siteMinute{v.Site, v.Minute}] += float64(v.Lines)
first, last = min(first, v.Minute), max(last, v.Minute)
}
site := make([]float64, 0, len(siteLines))
for _, n := range siteLines {
site = append(site, n)
}
var l, b []float64
if periods == nil {
periods = []Span{{From: first, To: last}}
}
for _, period := range periods {
for m := max(first, period.From); m <= min(last, period.To); m++ {
l = append(l, lines[m])
b = append(b, bytes[m])
}
}
return HostVolume{LinesPerMinute: Summarize(l), BytesPerMinute: Summarize(b), SiteLinesPerMinute: Summarize(site)}
}
// LongestSilence returns the longest run of minutes in span with no logged
// line, given the sorted minutes that have volume rows. A long silence in a
// site's coverage may be missing log data rather than no traffic; the
// ledger reviews it before any baseline trains on it.
func LongestSilence(logged []int64, span Span) int64 {
var longest int64
next := span.From
for _, m := range logged {
if m < span.From || m > span.To {
continue
}
longest = max(longest, m-next)
next = m + 1
}
return max(longest, span.To+1-next)
}
// SiteShape is one site's lateness and key churn, the inputs for choosing
// the lateness watermark and the K1/K2/S detail bounds.
type SiteShape struct {
// Lateness counts records by how many seconds their time trails the
// latest time already logged in the same file, in logged order.
Lateness map[int64]int64
// WindowKeys holds, per level, the distinct keys with traffic in each
// W-minute window ending at a covered minute.
WindowKeys map[uint8][]float64
// WindowBindings holds the distinct bindings in each site-level window.
WindowBindings []float64
// NewKeys holds, per level, the keys first seen in each covered hour.
NewKeys map[uint8][]float64
}
// ShapeSite measures one site's records over its coverage.
func ShapeSite(site Site, w int) SiteShape {
tracker := NewShapeTracker(w)
tracker.Add(site)
return tracker.Report()
}
// ShapeTracker carries one site's diagnostic windows and first-seen keys
// across chronological bundles. Only the current window's records remain
// retained; earlier traffic contributes summary samples and key identities.
type ShapeTracker struct {
w int
shape SiteShape
firstSeen map[KeyID]bool
keyCounts map[KeyID]int
bindCounts map[string]int
window map[int64][]Record
last, coveredFrom, hour int64
}
// NewShapeTracker starts one site's diagnostics with a positive window.
func NewShapeTracker(w int) *ShapeTracker {
return &ShapeTracker{w: w, hour: -1, firstSeen: map[KeyID]bool{},
shape: SiteShape{Lateness: map[int64]int64{}, WindowKeys: map[uint8][]float64{}, NewKeys: map[uint8][]float64{}}}
}
// Add measures a validated bundle segment after every minute already added.
// File numbers belong to this bundle; lateness never joins distinct copies.
func (s *ShapeTracker) Add(site Site) {
ordered := slices.Clone(site.Records)
slices.SortStableFunc(ordered, func(a, b Record) int {
return cmp.Or(cmp.Compare(a.File, b.File), cmp.Compare(a.Seq, b.Seq))
})
latest := map[int]int64{}
byMinute := map[int64][]Record{}
for _, r := range ordered {
if top, ok := latest[r.File]; ok && top > r.T {
s.shape.Lateness[top-r.T]++
} else {
latest[r.File] = r.T
s.shape.Lateness[0]++
}
if r.Class != ClassOther && !r.Infra {
byMinute[r.T/60] = append(byMinute[r.T/60], r)
}
}
for _, span := range site.Coverage {
if s.last == 0 || span.From-1 != s.last {
s.coveredFrom = span.From
s.keyCounts = map[KeyID]int{}
s.bindCounts = map[string]int{}
s.window = map[int64][]Record{}
}
for m := span.From; m <= span.To; m++ {
s.minute(m, byMinute[m])
}
}
}
func (s *ShapeTracker) minute(m int64, records []Record) {
if m/60 != s.hour {
s.hour = m / 60
for level := uint8(1); level <= 3; level++ {
s.shape.NewKeys[level] = append(s.shape.NewKeys[level], 0)
}
}
for _, r := range records {
for _, id := range keysOf(&r) {
s.keyCounts[id]++
if !s.firstSeen[id] {
s.firstSeen[id] = true
counts := s.shape.NewKeys[id.Level]
counts[len(counts)-1]++
}
}
if r.Binding != "" {
s.bindCounts[r.Binding]++
}
}
if len(records) > 0 {
s.window[m] = records
}
old := m - int64(s.w)
for _, r := range s.window[old] {
for _, id := range keysOf(&r) {
if s.keyCounts[id]--; s.keyCounts[id] == 0 {
delete(s.keyCounts, id)
}
}
if r.Binding != "" {
if s.bindCounts[r.Binding]--; s.bindCounts[r.Binding] == 0 {
delete(s.bindCounts, r.Binding)
}
}
}
delete(s.window, old)
if m-s.coveredFrom+1 >= int64(s.w) {
levels := map[uint8]float64{}
for id := range s.keyCounts {
levels[id.Level]++
}
for level := uint8(1); level <= 3; level++ {
s.shape.WindowKeys[level] = append(s.shape.WindowKeys[level], levels[level])
}
s.shape.WindowBindings = append(s.shape.WindowBindings, float64(len(s.bindCounts)))
}
s.last = m
}
// Report returns diagnostics independent of later additions to the tracker.
func (s *ShapeTracker) Report() SiteShape {
out := SiteShape{Lateness: maps.Clone(s.shape.Lateness), WindowBindings: slices.Clone(s.shape.WindowBindings),
WindowKeys: map[uint8][]float64{}, NewKeys: map[uint8][]float64{}}
for level, values := range s.shape.WindowKeys {
out.WindowKeys[level] = slices.Clone(values)
}
for level, values := range s.shape.NewKeys {
out.NewKeys[level] = slices.Clone(values)
}
return out
}
// Footprint estimates the retained bytes of one key's detector state: the
// hour-of-week profile plus W minutes of summaries with m counters and h
// hashes. It is a sizing aid for K1, K2 and S, not a measurement of
// production code, which must be measured against this estimate.
func Footprint(p Params, s SketchParams) int64 {
const bindingBytes = 1 + 8 // family byte and the IPv6 /64, the larger form
profile := int64(168 * (8 + 8))
perMinute := int64(s.M)*(bindingBytes+8+8) + int64(s.H)*8 + 8
return profile + int64(p.W)*perMinute
}
package crawlreplay
import (
"slices"
)
// minuteCounts is one key's traffic in one covered minute.
type minuteCounts struct {
bindings map[string]int64 // requests per client binding
total int64 // all requests, bound or not
expensive int64 // requests with a query (the rest are queryless)
labels map[string]int64 // requests per label key (see labelKey)
}
func newMinuteCounts() *minuteCounts {
return &minuteCounts{bindings: map[string]int64{}, labels: map[string]int64{}}
}
func (m *minuteCounts) add(r *Record) {
m.total++
if r.Class == ClassExpensive {
m.expensive++
}
if r.Binding != "" {
m.bindings[r.Binding]++
}
m.labels[labelKey(r.Label, r.Episode)]++
}
// labelKey folds a record's label and episode into one counter name.
func labelKey(label, episode string) string {
if episode == "" {
return label
}
return label + "/" + episode
}
// Window is the exact traffic of one key over its last W covered minutes.
type Window struct {
bindings map[string]int64
total int64
expensive int64
labels map[string]int64
}
func newWindow() *Window {
return &Window{bindings: map[string]int64{}, labels: map[string]int64{}}
}
// apply adds (sign 1) or removes (sign -1) one minute's counts.
func (w *Window) apply(m *minuteCounts, sign int64) {
w.total += sign * m.total
w.expensive += sign * m.expensive
for b, n := range m.bindings {
if v := w.bindings[b] + sign*n; v != 0 {
w.bindings[b] = v
} else {
delete(w.bindings, b)
}
}
for l, n := range m.labels {
if v := w.labels[l] + sign*n; v != 0 {
w.labels[l] = v
} else {
delete(w.labels, l)
}
}
}
// Total is every request in the window, with or without a binding.
func (w *Window) Total() int64 { return w.total }
// Expensive is the window's requests with a query; L1 and L2 keys hold
// nothing else.
func (w *Window) Expensive() int64 { return w.expensive }
// Bound is the requests that carry a client binding.
func (w *Window) Bound() int64 {
var n int64
for _, c := range w.bindings {
n += c
}
return n
}
// Residual returns the exact requests and distinct bindings left after
// removing the k largest bindings, or every binding if fewer exist.
// Requests without a binding are not residual evidence: they could all
// belong to one heavy source.
func (w *Window) Residual(k int) (requests, bindings int64) {
counts := make([]int64, 0, len(w.bindings))
var bound int64
for _, c := range w.bindings {
counts = append(counts, c)
bound += c
}
slices.Sort(counts)
var top int64
for i := len(counts) - 1; i >= 0 && len(counts)-i <= k; i-- {
top += counts[i]
}
return bound - top, max(0, int64(len(counts)-k))
}
// Labels returns a copy of the window's per-label request counts.
func (w *Window) Labels() map[string]int64 {
out := make(map[string]int64, len(w.labels))
for l, n := range w.labels {
out[l] = n
}
return out
}
package daemon
import (
"context"
"log"
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/reporting"
)
type abuseReportConsumer struct {
stop <-chan struct{}
interval time.Duration
persist func(reporting.Report) error
drain func(context.Context) time.Duration
queue *queuehealth.Channel[reporting.Report]
mu sync.Mutex
closed bool
}
func newAbuseReportConsumer(stop <-chan struct{}, capacity int, interval time.Duration, persist func(reporting.Report) error, drain func(context.Context) time.Duration) *abuseReportConsumer {
return &abuseReportConsumer{
stop: stop, interval: interval, persist: persist, drain: drain,
queue: queuehealth.NewChannel[reporting.Report](capacity, time.Minute),
}
}
func (c *abuseReportConsumer) QueueStatuses(now time.Time) map[string]queuehealth.Status {
return map[string]queuehealth.Status{"ingress": c.queue.Snapshot(now)}
}
func (c *abuseReportConsumer) enqueue(r reporting.Report) bool {
c.mu.Lock()
defer c.mu.Unlock()
if c.closed {
c.queue.Lose(1)
return false
}
select {
case <-c.stop:
c.queue.Lose(1)
return false
default:
return c.queue.TrySend(r)
}
}
func (c *abuseReportConsumer) closeAdmission() {
c.mu.Lock()
defer c.mu.Unlock()
if !c.closed {
c.closed = true
c.queue.Close()
}
}
func (c *abuseReportConsumer) run() {
ctx, cancel := context.WithCancel(context.Background())
canceled := make(chan struct{})
go func() {
defer close(canceled)
select {
case <-c.stop:
cancel()
case <-ctx.Done():
}
}()
var loggedLoss uint64
defer func() {
cancel()
<-canceled
c.closeAdmission()
c.queue.DiscardPending()
c.logDropped(&loggedLoss)
}()
// Each pass says when the next one runs, so a throttled collector is
// retried when it allows while intake keeps running in between.
timer := time.NewTimer(c.interval)
defer timer.Stop()
for {
select {
case <-c.stop:
c.persistRemaining()
return
default:
}
select {
case <-c.stop:
c.persistRemaining()
return
case work := <-c.queue.Items():
c.process(work)
case <-timer.C:
c.logDropped(&loggedLoss)
timer.Reset(c.drain(ctx))
}
}
}
func (c *abuseReportConsumer) persistRemaining() {
// Stop admission before draining: a captured report hook must not append
// work after the final empty check or keep shutdown running indefinitely.
c.closeAdmission()
for work := range c.queue.Items() {
c.process(work)
}
}
func (c *abuseReportConsumer) process(work queuehealth.Work[reporting.Report]) {
work.Ticket.Start(time.Now())
settled := false
defer func() {
if !settled {
work.Ticket.Reject(time.Now())
}
}()
err := c.persist(work.Value)
if err != nil {
work.Ticket.Reject(time.Now())
} else {
work.Ticket.Finish(time.Now())
}
settled = true
if err != nil {
log.Printf("abuse-reporting: report persistence failed: %v", err)
}
}
func (c *abuseReportConsumer) logDropped(previous *uint64) {
total := c.queue.Snapshot(time.Now()).DroppedTotal
if n := total - *previous; n > 0 {
log.Printf("abuse-reporting: %d report(s) incomplete", n)
*previous = total
}
}
package daemon
import (
"crypto/ed25519"
"encoding/hex"
"log"
"net"
"os"
"path/filepath"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/reporting"
)
const (
abuseReportSpoolFile = "abuse_reports.db"
abuseReportSpoolDefault = 10000
abuseReportQueueDefault = abuseReportSpoolDefault
abuseReportDrainEvery = time.Minute
)
// abuseReportFirebreak returns the predicate that keeps protected addresses
// out of abuse reports. Var so tests that report documentation-range
// fixtures can lift it.
var abuseReportFirebreak = func(d *Daemon) func(string) bool { return d.centralFirebreak() }
// startAbuseReporting wires the abuse reporter from config: it sets
// alert.ReportHook so confirmed-abuse findings are gated, minimized, and
// queued for spooling, and returns the spool drain loop to run as a supervised
// goroutine.
// It returns nil when reporting is disabled or misconfigured (logged), leaving
// the alert path untouched.
func (d *Daemon) startAbuseReporting() func() {
alert.SetReportHook(nil)
// Answer the reputation check on every path, once the spam source
// detector, if any, is in place. Scans before this get no answer.
defer checks.SetSharedSendersHealth(d.sharedSendersHealth)
rc := d.cfg.Reputation.Report
if !rc.Enabled {
return nil
}
targets := buildReportTargets(rc.Targets)
if len(targets) == 0 {
log.Printf("abuse-reporting: enabled but no usable targets configured; reporting stays off")
return nil
}
enabled := classSet(rc.Classes)
if len(enabled) == 0 {
log.Printf("abuse-reporting: enabled but no valid classes configured; reporting stays off")
return nil
}
spoolPath := rc.SpoolPath
if spoolPath == "" {
spoolPath = filepath.Join(d.cfg.StatePath, abuseReportSpoolFile)
}
max := rc.SpoolMax
if max <= 0 {
max = abuseReportSpoolDefault
}
spool, err := reporting.NewSpool(spoolPath, "reports", max)
if err != nil {
log.Printf("abuse-reporting: cannot open spool %s: %v; reporting stays off", spoolPath, err)
return nil
}
spooler := reporting.NewSpooler(spool, reporting.NewSender(nil, nil), targets, abuseReportDrainEvery)
// The same firebreak that guards central-intel actions keeps
// infrastructure, Cloudflare edges and verified crawlers out of the
// shared abuse set.
firebreak := abuseReportFirebreak(d)
gate := reporting.Gate{
Enabled: enabled,
Protected: func(ip net.IP) bool { return firebreak(ip.String()) },
}
spam := d.newSpamSources(enabled)
if spam != nil {
gate.SharedSenders = spam.covers
}
stopCh := make(chan struct{})
doneCh := make(chan struct{})
d.abuseReportStop = stopCh
d.abuseReportDone = doneCh
consumer := newAbuseReportConsumer(stopCh, abuseReportQueueSize(max), abuseReportDrainEvery, spooler.Enqueue, spooler.DrainOnce)
d.registerQueueSource("abuse_reporting", abuseReportQueues{ingress: consumer, spool: spool})
alert.SetReportHook(func(f alert.Finding) {
if r, ok := gate.Consider(f); ok {
consumer.enqueue(r)
}
})
if spam != nil {
// Started after the hook, so every finding it raises can be reported.
d.startSpamSources(spam)
}
log.Printf("abuse-reporting: enabled for %d target(s), %d class(es)", len(targets), len(enabled))
return func() {
defer close(doneCh)
defer func() {
alert.SetReportHook(nil)
_ = spool.Close()
}()
consumer.run()
}
}
type abuseReportQueues struct {
ingress *abuseReportConsumer
spool *reporting.Spool
}
func (q abuseReportQueues) QueueStatuses(now time.Time) map[string]queuehealth.Status {
statuses := q.ingress.QueueStatuses(now)
for name, status := range q.spool.QueueStatuses(now) {
statuses[name] = status
}
return statuses
}
func (d *Daemon) stopAbuseReporting() {
// No answer from here on, so a scan during shutdown cannot read the
// detector's absence as a current list and clear a warning.
checks.SetSharedSendersHealth(nil)
// Run has stopped the log watchers by now, so this final save holds every
// spam scan they read.
if r := d.spamSources.Swap(nil); r != nil {
r.stop()
}
if d.abuseReportStop == nil {
return
}
close(d.abuseReportStop)
<-d.abuseReportDone
d.abuseReportStop = nil
d.abuseReportDone = nil
}
func abuseReportQueueSize(spoolMax int) int {
if spoolMax > 0 && spoolMax < abuseReportQueueDefault {
return spoolMax
}
return abuseReportQueueDefault
}
// classSet parses configured class names into the set the gate accepts,
// skipping unknown values with a log line.
func classSet(names []string) map[reporting.Class]bool {
known := map[reporting.Class]bool{
reporting.ClassBruteforce: true,
reporting.ClassPHPRelay: true,
reporting.ClassCredentialStuffing: true,
reporting.ClassBadASNEgress: true,
reporting.ClassSpam: true,
}
out := make(map[reporting.Class]bool)
for _, n := range names {
c := reporting.Class(n)
if known[c] {
out[c] = true
} else {
log.Printf("abuse-reporting: ignoring unknown report class %q", n)
}
}
return out
}
// reportTargetConfig mirrors the per-target config shape; declared so the
// builder takes a concrete slice type from the anonymous struct in config.
type reportTargetConfig = struct {
Name string `yaml:"name"`
URL string `yaml:"url"`
Transport string `yaml:"transport"`
NodeID string `yaml:"node_id"`
KeyID string `yaml:"key_id"`
KeyEnv string `yaml:"key_env"`
TokenEnv string `yaml:"token_env"`
}
// buildReportTargets resolves configured targets into sender targets, reading
// key material from the environment. Invalid targets are skipped with a log
// line rather than failing startup.
func buildReportTargets(cfgTargets []reportTargetConfig) []reporting.Target {
var targets []reporting.Target
for _, ct := range cfgTargets {
if ct.Name == "" || ct.URL == "" || ct.NodeID == "" || ct.KeyID == "" {
log.Printf("abuse-reporting: skipping target with missing name/url/node_id/key_id")
continue
}
if err := reporting.ValidateTargetURL(ct.URL); err != nil {
log.Printf("abuse-reporting: target %q: URL must be HTTPS or loopback HTTP; skipping", ct.Name)
continue
}
t := reporting.Target{
Name: ct.Name,
URL: ct.URL,
NodeID: ct.NodeID,
KeyID: ct.KeyID,
}
secret := os.Getenv(ct.KeyEnv)
switch reporting.Transport(ct.Transport) {
case reporting.TransportEd25519:
raw, err := hex.DecodeString(secret)
if err != nil || len(raw) != ed25519.PrivateKeySize {
log.Printf("abuse-reporting: target %q: configured key_env must hold a 64-byte hex Ed25519 key; skipping", ct.Name)
continue
}
t.Transport = reporting.TransportEd25519
t.Ed25519Key = ed25519.PrivateKey(raw)
case reporting.TransportHMAC:
if secret == "" {
log.Printf("abuse-reporting: target %q: configured key_env is empty; skipping", ct.Name)
continue
}
t.Transport = reporting.TransportHMAC
t.HMACSecret = []byte(secret)
if ct.TokenEnv != "" {
t.BearerToken = os.Getenv(ct.TokenEnv)
}
default:
log.Printf("abuse-reporting: target %q: unknown transport %q; skipping", ct.Name, ct.Transport)
continue
}
targets = append(targets, t)
}
return targets
}
// centralSigner resolves reputation.central.auth_target to the report target
// whose Ed25519 node key signs central pulls. It reports false when the name
// matches no single target or that target cannot sign, so the caller never
// falls back to unsigned pulls.
func centralSigner(name string, cfgTargets []reportTargetConfig) (reporting.Target, bool) {
var found []reportTargetConfig
for _, ct := range cfgTargets {
if ct.Name == name {
found = append(found, ct)
}
}
if len(found) != 1 {
return reporting.Target{}, false
}
ct := found[0]
raw, err := hex.DecodeString(os.Getenv(ct.KeyEnv))
if reporting.Transport(ct.Transport) != reporting.TransportEd25519 || ct.NodeID == "" || ct.KeyID == "" ||
err != nil || len(raw) != ed25519.PrivateKeySize {
return reporting.Target{}, false
}
key := ed25519.PrivateKey(raw)
// A mismatched public-key suffix produces signatures that cannot verify.
if !key.Equal(ed25519.NewKeyFromSeed(raw[:ed25519.SeedSize])) {
return reporting.Target{}, false
}
return reporting.Target{
Name: ct.Name, URL: ct.URL, Transport: reporting.TransportEd25519,
NodeID: ct.NodeID, KeyID: ct.KeyID, Ed25519Key: key,
}, true
}
package daemon
import (
"path/filepath"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
)
// defaultLogDir is the directory the packaged unit creates for CSM's logs.
const defaultLogDir = "/var/log/csm"
// actionLogPath puts the action log beside the SIEM audit log, so an operator
// who moved that file gets both streams in one directory.
func actionLogPath(cfg *config.Config) string {
dir := defaultLogDir
if configured := cfg.Alerts.AuditLog.File.Path; configured != "" {
dir = filepath.Dir(configured)
}
return actionlog.DefaultPath(dir)
}
// installActionLog points the process-wide action recorder at the log file.
// Every subsystem that changes host state records through it, so this runs
// before the watchers and the check scheduler start.
func (d *Daemon) installActionLog() {
path := actionLogPath(d.cfg)
sink := actionlog.NewFileSink(func() string { return path }, func(err error) {
csmlog.Warn("action log write failed", "path", path, "err", err)
})
actionlog.SetSink(sink, d.cfg.Hostname)
}
package daemon
import (
"context"
"errors"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/admissionowner"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/health"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/store"
"github.com/pidginhost/csm/internal/verdict"
)
var _ health.AdmissionProvider = (*Daemon)(nil)
// ownerKeepsDryRunExamples is set once an admission owner that runs blocks
// starts. It then writes, prunes and purges the dry_run_blocks bucket alone,
// and the legacy recorder, prune and purge leave the bucket to
// it.
var ownerKeepsDryRunExamples atomic.Bool
// recordLegacyDryRunBlock is the legacy engine's dry-run recorder: it keeps
// the block a dry run skipped, unless an admission owner keeps the bucket.
func recordLegacyDryRunBlock(ip, reason string, timeout time.Duration) {
if db := store.Global(); db != nil && !ownerKeepsDryRunExamples.Load() {
db.RecordDryRunBlock(ip, reason, timeout)
}
}
// pruneLegacyDryRunBlocks ages out dry_run_blocks records older than a
// week, unless an admission owner keeps the bucket.
func pruneLegacyDryRunBlocks() {
if sdb := store.Global(); sdb != nil && !ownerKeepsDryRunExamples.Load() {
sdb.PurgeDryRunBlocksOlderThan(time.Now().Add(-7 * 24 * time.Hour))
}
}
// setResponseAdmission wires the response funnels; tests observe it.
var setResponseAdmission = checks.SetResponseAdmission
// startAdmission makes the daemon the owner of the admission ledger: the
// one handle on the state database's ledger. Every automatic response the
// legacy funnels select is also handed to it. With the firewall on and
// outside observe mode the owner runs on the firewall engine: a dry-run
// policy forecasts, and a live one previews. Otherwise it only previews.
// Live execution stays disabled while legacy blocking still enforces, so
// the same observation cannot trigger both legacy and admission execution.
func (d *Daemon) startAdmission() {
d.startAdmissionWith(d.admissionOptions(store.Global()))
}
func admissionUsesFirewall(cfg *config.Config) bool {
return cfg != nil && cfg.Firewall != nil && cfg.Firewall.Enabled && !cfg.ObserveMode()
}
func (d *Daemon) admissionOptions(db *store.DB) admissionowner.Options {
// The firewall reads its family settings at start, as this does.
ipv6 := d.cfg.Firewall != nil && d.cfg.Firewall.IPv6
var blocks func() admissionowner.BlockEngine
if admissionUsesFirewall(d.cfg) {
blocks = d.admissionBlocks
}
return admissionowner.Options{
DB: db, StatePath: d.cfg.StatePath,
Ceiling: func() (uint32, string) { return d.currentCfg().AdmissionCeiling() },
Clock: admissionowner.ReadClock,
Inventory: admissionowner.Inventory,
LegacySpend: checks.LegacyBlockSpend,
LegacyHandover: checks.LegacyHandover,
WriteAudit: actionlog.WriteDurableBatch,
Applied: d.consumeAppliedBlocks,
Deliver: d.deliverAdmissionNotices,
Blocks: blocks,
// A dry-run summary is a preview: history only.
DeliverSummary: func(ctx context.Context, f alert.Finding) error {
if err := ctx.Err(); err != nil {
return err
}
return d.store.AppendHistoryDurable([]alert.Finding{f})
},
Expiry: func(c admission.Candidate) time.Duration { return previewExpiry(d.currentCfg(), c) },
BlockExpiry: func() time.Duration { return parseBlockExpiry(d.currentCfg().AutoResponse.BlockExpiry) },
Caps: func() admission.Caps { return admission.Caps{IPv6: ipv6} },
Policy: d.executionPolicy,
}
}
// admissionBlocks is the engine the admission owner runs on: none, as an
// untyped nil, until the firewall has applied its state.
func (d *Daemon) admissionBlocks() admissionowner.BlockEngine {
if e := d.blockEngine.Load(); e != nil {
return e
}
return nil
}
// executionPolicy is the execution policy of the live config, read with its
// response epoch from one publication.
func (d *Daemon) executionPolicy() admissionowner.ExecutionPolicy {
cfg, epoch := config.ActiveEpoch()
if cfg == nil {
return admissionowner.ExecutionPolicy{}
}
limit, source := cfg.AdmissionCeiling()
// The epoch-bound client preserves the configured signature policy.
client := newVerdictClient(cfg)
return admissionowner.ExecutionPolicy{
Epoch: epoch, Observe: cfg.ObserveMode(), DryRun: cfg.AutoResponseDryRunEnabled(),
VerdictEnabled: cfg.AutoResponse.VerdictCallback.Enabled,
VerdictTimeout: time.Duration(cfg.AutoResponse.VerdictCallback.TimeoutSec) * time.Second,
CeilingLimit: limit, CeilingSource: source,
Expiry: func(c admission.Candidate) time.Duration { return previewExpiry(cfg, c) },
BlockExpiry: parseBlockExpiry(cfg.AutoResponse.BlockExpiry),
AskVerdict: func(ctx context.Context, ip, reason string) (verdict.Response, error) {
return client.Ask(ctx, verdict.Request{IP: ip, Reason: reason, Severity: "auto", Source: "auto_response"})
},
// Switches outside auto_response change without a new epoch, so
// queued work is tested against the config live at each turn.
// A queued block of an address that became a trusted proxy or a
// known edge ends too, as the mint would refuse it now.
Allows: func(c admission.Candidate, roots func() []admission.Evidence) bool {
live := config.Active()
return checks.AutomaticResponseAllowed(live, c.Key.Kind, c.Entry, c.Check) && !checks.BlocksAProxy(live, c, roots)
},
}
}
// newVerdictClient is the verdict callback client cfg configures.
func newVerdictClient(cfg *config.Config) *verdict.Client {
vc := cfg.AutoResponse.VerdictCallback
return verdict.New(verdict.Config{
URL: vc.URL,
HMACSecret: vc.HMACSecret,
HMACSecretEnv: vc.HMACSecretEnv,
RequireResponseSignature: vc.RequireResponseSignature,
AllowUnsigned: vc.AllowUnsigned,
Timeout: time.Duration(vc.TimeoutSec) * time.Second,
})
}
// previewExpiry is how long the response c asks for would last under cfg:
// each entry keeps the lifetime its legacy path applies.
func previewExpiry(cfg *config.Config, c admission.Candidate) time.Duration {
switch {
case c.Entry == admission.EntryCentral && c.Key.Kind == admission.KindChallenge:
return centralChallengeTTL
case c.Entry == admission.EntryCentral:
return centralBlockTTL
case c.Key.Kind == admission.KindChallenge:
return checks.ChallengeDuration
case c.Entry == admission.EntryASNCrawl:
return parseBlockExpiry(cfg.AutoResponse.HTTPASNCrawlTempban)
}
return parseBlockExpiry(cfg.AutoResponse.BlockExpiry)
}
func (d *Daemon) startAdmissionWith(opts admissionowner.Options) {
if d.admission != nil {
return
}
// The owner mints from its start: publish the web server state first.
d.refreshHTTPRewrite()
d.admission = admissionowner.Start(opts)
ownerKeepsDryRunExamples.Store(d.admission.OwnsDryRunExamples())
setResponseAdmission(d.admission)
d.registerQueueSource("admission", d.admission)
if st := d.admission.Status(); st != nil && st.Owner != nil && st.Owner.Error != "" {
csmlog.Warn("admission ledger unavailable; retrying", "err", st.Owner.Error)
}
}
// deliverAdmissionNotices is the independent health path: history and a
// direct dispatch, as protection_queue_degraded is sent. The finding
// channel, suppressions and automatic response are never involved, since
// the failure being reported may be in them. A preview is recorded only.
func (d *Daemon) deliverAdmissionNotices(findings []alert.Finding, preview bool) error {
if preview {
return d.store.AppendHistoryDurable(findings)
}
d.store.AppendHistory(findings)
return alert.DispatchNotices(d.currentCfg(), findings)
}
// AdmissionStatus is the owner's last reading of the ledger, refreshed on
// its timer rather than per request.
func (d *Daemon) AdmissionStatus() *health.AdmissionStatus {
if d.admission == nil {
return nil
}
return d.admission.Status()
}
func (d *Daemon) reloadAdmission() {
if d.admission == nil {
return
}
err := d.admission.Reload()
switch {
case errors.Is(err, admissionowner.ErrPolicy):
d.emitReloadFinding(alert.Critical, "config_reload_error",
"config reload: automatic execution is held: "+err.Error())
case err != nil:
csmlog.Warn("admission ceiling not reloaded", "err", err)
}
}
func (d *Daemon) stopAdmission() {
if d.admission != nil {
setResponseAdmission(nil)
d.admission.Stop()
}
}
package daemon
import (
"errors"
"fmt"
"sync"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/attackdb"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/store"
)
// loadAppliedState lets tests act between the state read and the threat write.
var loadAppliedState = firewall.LoadState
// consumeAppliedBlocks consumes verified outcomes before their audit ack.
// A failed state read is not proof of removal; leave the outbox pending.
// The threat rows are chosen from a state read under the threat writer lock,
// so an unblock that removes the firewall row after that read waits for the
// batch to commit and then removes what it wrote.
func (d *Daemon) consumeAppliedBlocks(rows []admission.AuditRow) error {
// A destination the daemon has not opened cannot record the block, and
// acknowledging the rows now would lose their evidence for good.
tdb, adb := checks.GetThreatDB(), attackdb.Global()
switch {
case tdb == nil:
return errors.New("the threat database is not open yet")
case adb == nil:
return errors.New("the attack database is not open yet")
}
var readErr error
var standing []checks.AdmissionBlock
writeErr := tdb.AddTemporaryBatchFrom(func() []store.PermanentBlockEntry {
standing, readErr = d.standingAppliedBlocks(rows)
entries := make([]store.PermanentBlockEntry, 0, len(standing))
for _, b := range standing {
entries = append(entries, store.PermanentBlockEntry{IP: b.IP, Reason: b.Reason, ExpiresAt: b.ExpiresAt})
}
return entries
})
if readErr != nil {
return readErr
}
// The attack database records that the block applied, as the legacy
// path marks it at block time: a block removed or expired before
// delivery is still marked, and the mark does not wait for the threat
// store.
for _, r := range rows {
if addr, ok := r.Target.Addr(); ok {
adb.MarkBlocked(addr.String())
}
}
if writeErr != nil {
return fmt.Errorf("recording applied threat batch: %w", writeErr)
}
if !d.appliedBlocks.add(standing) {
return errors.New("the applied-block consumer is full or stopped")
}
return nil
}
// standingAppliedBlocks returns each row whose action's verified firewall
// row still stands at the row's address and expiry, as that row records it.
func (d *Daemon) standingAppliedBlocks(rows []admission.AuditRow) ([]checks.AdmissionBlock, error) {
state, err := loadAppliedState(d.currentCfg().StatePath)
if err != nil {
return nil, fmt.Errorf("reading verified firewall state for applied blocks: %w", err)
}
standing := make(map[string]firewall.BlockedEntry)
if state != nil {
for _, b := range state.Blocked {
if b.ActionID != "" {
standing[b.ActionID] = b
}
}
}
var blocks []checks.AdmissionBlock
for _, r := range rows {
addr, ok := r.Target.Addr()
if !ok {
continue
}
ip := addr.String()
b, exists := standing[string(r.Attempt.ID)]
if exists && b.IP == ip && b.ExpiresAt.Equal(r.ExpiresAt) {
blocks = append(blocks, checks.AdmissionBlock{IP: ip, Reason: b.Reason, Check: r.Check, FindingID: r.FindingID, BlockedAt: b.BlockedAt, ExpiresAt: b.ExpiresAt})
}
}
return blocks, nil
}
// appliedBlockQueue hands applied admission blocks to the legacy consumers
// (tracker row, permanent-block strike, the auto_block finding) in order, on
// a daemon goroutine: the tracker's lock can be held for a whole scan, and
// the admission owner must not wait for it. Delivery is best effort: blocks
// still queued when the daemon stops are not recorded there.
type appliedBlockQueue struct {
mu sync.Mutex
pending []checks.AdmissionBlock
wake chan struct{}
stopped bool
}
// add never waits for consumers. A full or stopped queue refuses the batch,
// which leaves the durable audit delivery unacknowledged, so the owner
// retries it and the next start redelivers what shutdown left. Shutdown
// discards only work already queued.
func (q *appliedBlockQueue) add(blocks []checks.AdmissionBlock) bool {
q.mu.Lock()
defer q.mu.Unlock()
if len(blocks) == 0 {
return true
}
if q.stopped {
return false
}
if len(blocks) > int(admission.MaxAuditSlots)-len(q.pending) {
return false
}
q.pending = append(q.pending, blocks...)
select {
case q.wake <- struct{}{}:
default:
}
return true
}
// serve records queued blocks until stop closes.
func (q *appliedBlockQueue) serve(d *Daemon, stop <-chan struct{}) {
defer func() {
q.mu.Lock()
q.stopped, q.pending = true, nil
q.mu.Unlock()
}()
for {
select {
case <-stop:
return
default:
}
q.mu.Lock()
blocks := q.pending
q.pending = nil
q.mu.Unlock()
if len(blocks) > 0 {
d.recordAppliedBlocks(checks.RecordAdmissionBlocks(d.currentCfg(), blocks))
}
select {
case <-q.wake:
case <-stop:
return
}
}
}
//go:build linux
package daemon
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"sync/atomic"
"syscall"
"time"
"unsafe"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
)
// auditLogPath is the file the kernel auditd writes events to. Var (not
// const) so tests can redirect it under t.TempDir().
var auditLogPath = "/var/log/audit/audit.log"
// AFAlgAuditListener tails /var/log/audit/audit.log via inotify, parses
// each new line for the csm_af_alg_socket auditd key, and emits a
// Critical alert.Finding (plus optional kill/quarantine reactions)
// within milliseconds of the syscall. Sub-second response on hosts where
// BPF LSM is not available.
//
// The listener seeks to end-of-file at startup — events that pre-date
// the daemon are intentionally not re-alerted; the periodic critical-tier
// CheckAFAlgSocketUsage handles backfill via its (timestamp, serial)
// cursor in state.Store.
type AFAlgAuditListener struct {
alertCh chan<- alert.Finding
cfg *config.Config
path string
inotifyFd int
file *os.File
pos int64
leftover []byte // partial line accumulator across reads
droppedOversize bool // mid-drop of a line that overflowed the cap
cursorAnchor []byte // bytes immediately before pos, used to detect copytruncate rewrites
// Re-open retry state. A rotation (or a broken inotify fd) sets
// reopenPending; each tick retries open() once its backoff has elapsed,
// so one failed re-open no longer blinds the listener forever.
reopenPending bool
reopenBackoff time.Duration
reopenNotBefore time.Time
eventCount atomic.Uint64 // observed by tests / metrics
}
// Re-open backoff bounds. Start small so a normal logrotate race (the
// replacement file lands a beat after the move) recovers within a tick or two,
// then grow to a cap so a genuinely absent audit log does not spin.
const (
afAlgReopenBackoffInitial = 1 * time.Second
afAlgReopenBackoffMax = 30 * time.Second
)
func nextAFAlgReopenBackoff(cur time.Duration) time.Duration {
if cur <= 0 {
return afAlgReopenBackoffInitial
}
next := cur * 2
if next > afAlgReopenBackoffMax {
return afAlgReopenBackoffMax
}
return next
}
// afAlgMaxLeftoverBytes caps the partial-line accumulator. A real auditd
// SYSCALL record is well under 1 KiB; 64 KiB leaves generous headroom while
// bounding memory if a record never terminates.
const afAlgMaxLeftoverBytes = 64 * 1024
// afAlgCursorAnchorBytes is the suffix length remembered at the current read
// cursor. On each tail tick the listener verifies those bytes still sit before
// pos; if copytruncate rewrote the file past the old cursor between ticks, the
// anchor no longer matches and the listener resets to offset 0 instead of
// skipping the fresh records.
const afAlgCursorAnchorBytes = 64
// Mode reports the live-monitor backend kind. Matches the BPF path's
// "bpf-lsm" return so the coordinator and operator-visible logs use a
// stable, machine-readable label.
func (l *AFAlgAuditListener) Mode() string { return "auditd-tail" }
// EventCount returns the number of csm_af_alg_socket events this
// listener has parsed since startup. Operational metric; not exported
// to Prometheus here to keep the listener self-contained.
func (l *AFAlgAuditListener) EventCount() uint64 { return l.eventCount.Load() }
// NewAFAlgAuditListener opens the audit log, seeks to its current end,
// and registers an inotify watch on the file. The watch fires for
// IN_MODIFY (new bytes appended), IN_MOVE_SELF (logrotate moves the
// file), and IN_DELETE_SELF (file unlinked) so we can re-open the
// rotated/replaced file.
//
// Returns an error if /var/log/audit/audit.log is missing — caller is
// expected to log a warning and either skip live detection (with
// periodic check still active) or retry later.
func NewAFAlgAuditListener(alertCh chan<- alert.Finding, cfg *config.Config) (*AFAlgAuditListener, error) {
l := &AFAlgAuditListener{
alertCh: alertCh,
cfg: cfg,
path: auditLogPath,
}
if err := l.open(); err != nil {
return nil, err
}
return l, nil
}
// open initialises (or re-initialises after rotation) the audit log fd
// and inotify watch. Seeks to end-of-file so we tail forward, never
// re-alerting historical events.
func (l *AFAlgAuditListener) open() error {
// #nosec G304 -- l.path is /var/log/audit/audit.log (or t.TempDir()
// equivalent); not user-controlled.
f, err := os.Open(l.path)
if err != nil {
return fmt.Errorf("open %s: %w", l.path, err)
}
end, err := f.Seek(0, 2) // SEEK_END
if err != nil {
_ = f.Close()
return fmt.Errorf("seek %s: %w", l.path, err)
}
fd, err := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK)
if err != nil {
_ = f.Close()
return fmt.Errorf("inotify_init: %w", err)
}
mask := uint32(unix.IN_MODIFY | unix.IN_MOVE_SELF | unix.IN_DELETE_SELF)
if _, err := unix.InotifyAddWatch(fd, l.path, mask); err != nil {
_ = unix.Close(fd)
_ = f.Close()
return fmt.Errorf("inotify_add_watch %s: %w", l.path, err)
}
// Replace any existing fds (rotation case).
if l.file != nil {
_ = l.file.Close()
}
if l.inotifyFd != 0 {
_ = unix.Close(l.inotifyFd)
}
l.file = f
l.pos = end
l.inotifyFd = fd
l.leftover = nil
l.droppedOversize = false
l.captureCursorAnchor()
l.reopenPending = false
l.reopenBackoff = 0
l.reopenNotBefore = time.Time{}
return nil
}
// reopenIfDue services a pending re-open. It returns true when the listener is
// ready to tail this tick: either no re-open was pending, or a due re-open
// succeeded. On a failed re-open it keeps reopenPending set and schedules the
// next attempt with exponential backoff, returning false so the caller skips
// tailing a file that is not yet in place. A re-open still gated by its backoff
// window also returns false without touching the filesystem.
func (l *AFAlgAuditListener) reopenIfDue(now time.Time) bool {
if !l.reopenPending {
return true
}
if now.Before(l.reopenNotBefore) {
return false
}
if err := l.open(); err != nil {
l.reopenBackoff = nextAFAlgReopenBackoff(l.reopenBackoff)
l.reopenNotBefore = now.Add(l.reopenBackoff)
csmlog.Warn("af_alg audit listener: re-open failed; will retry",
"err", err, "retry_in", l.reopenBackoff)
return false
}
return true
}
// Run drains inotify events and tails the audit log until ctx is done.
// Polls the inotify fd every poll interval (matches forwarder_watcher's
// approach — no epoll, easier to reason about).
//
// On rotation (IN_MOVE_SELF / IN_DELETE_SELF) the listener re-opens the
// new audit.log and continues from its end. If the replacement file is not
// on disk yet the re-open is retried with backoff on subsequent ticks
// instead of giving up (one failed re-open used to blind the listener until
// the next successful rotation). A broken inotify fd is recovered the same
// way. There is still a brief window during rotation where events written
// between the move and the re-open can be missed; the periodic critical-tier
// check covers that gap via its persistent cursor.
func (l *AFAlgAuditListener) Run(ctx context.Context) {
defer func() {
if l.inotifyFd != 0 {
_ = unix.Close(l.inotifyFd)
}
if l.file != nil {
_ = l.file.Close()
}
}()
inotifyBuf := make([]byte, 4096)
readBuf := make([]byte, 16*1024)
// 500 ms tick gives sub-second average detection latency. The cost
// is ~2 cheap syscalls/sec (inotify Read returns EAGAIN immediately
// when nothing's queued, file Seek+Read returns EOF cheaply when
// no new bytes). Cheap enough to not bother with epoll/select for
// a v1 — we can refactor to true event-driven if we ever need
// hundred-microsecond latency.
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
// Start by draining anything the kernel may have queued during
// startup.
l.tail(readBuf)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
rotated, ok := l.drainInotify(inotifyBuf)
// A rotation needs a re-open; a broken inotify fd (ok=false) needs
// one too, so a persistently unhealthy watch is rebuilt instead of
// leaving us blind. This is the "safety net" the retry loop provides.
if rotated || !ok {
l.reopenPending = true
}
if !l.reopenIfDue(time.Now()) {
continue
}
l.tail(readBuf)
}
}
}
// drainInotify reads any queued inotify events. Returns:
//
// ok=false on read errors that aren't EAGAIN/EINTR (the listener
// should skip this tick entirely; the next tick re-tries).
// rotated=true if any IN_MOVE_SELF or IN_DELETE_SELF event was seen.
//
// IN_MODIFY events implicitly trigger a tail() in the caller because we
// always read at the end of each tick on success.
func (l *AFAlgAuditListener) drainInotify(buf []byte) (rotated, ok bool) {
for {
n, err := unix.Read(l.inotifyFd, buf)
if err != nil {
// EAGAIN: no more events queued — the normal terminator.
// EINTR: interrupted by signal, also benign — retry next tick.
if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EINTR) {
return rotated, true
}
// Anything else (EBADF after a stray Close, EIO, EFAULT)
// signals the inotify fd is no longer healthy. Surface it so
// the operator notices, and return ok=false: the caller marks a
// re-open pending and the backoff retry rebuilds the watch.
csmlog.Warn("af_alg audit listener: inotify read error", "err", err)
return rotated, false
}
if n <= 0 {
return rotated, true
}
offset := 0
for offset+unix.SizeofInotifyEvent <= n {
// #nosec G103 -- inotify packed binary stream;
// reinterpretation is required and the bound check above
// guarantees the read is in-range.
ev := (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset]))
if ev.Mask&(unix.IN_MOVE_SELF|unix.IN_DELETE_SELF) != 0 {
rotated = true
}
offset += unix.SizeofInotifyEvent + int(ev.Len)
}
}
}
// tail reads any new bytes since l.pos, splits on newlines, and feeds
// each complete line to handleLine. Partial trailing bytes (no newline
// yet) are buffered in l.leftover for the next tick.
func (l *AFAlgAuditListener) tail(buf []byte) {
for {
// Detect in-place truncation (copytruncate logrotate, or a manual
// truncate). When the file shrinks below our read cursor, seeking
// forward would skip the content rewritten from offset 0 and the
// listener would go blind. Size alone misses a fast truncate+rewrite
// that grows past the old cursor before this tick, so verify the
// cursor anchor too.
if fi, err := l.file.Stat(); err == nil {
switch {
case fi.Size() < l.pos:
l.resetTailCursor()
case l.cursorAnchorChanged():
l.resetTailCursor()
}
}
verifiedPos := l.pos
verifiedAnchor := append([]byte(nil), l.cursorAnchor...)
if _, err := l.file.Seek(l.pos, 0); err != nil {
csmlog.Warn("af_alg audit listener: seek failed", "err", err)
return
}
for {
n, err := l.file.Read(buf)
if n > 0 {
if l.cursorAnchorChangedAt(verifiedPos, verifiedAnchor) {
l.resetTailCursor()
break
}
l.feed(buf[:n])
l.pos += int64(n)
}
if err != nil {
if l.cursorAnchorChangedAt(verifiedPos, verifiedAnchor) {
l.resetTailCursor()
break
}
// io.EOF or EAGAIN: out of data for this tick. Re-check the
// old cursor after capturing the new anchor so a copytruncate
// rewrite cannot land in the tiny gap and get treated as the
// new baseline for the next tick.
if !l.refreshCursorAnchorAfterRead(verifiedPos, verifiedAnchor) {
break
}
return
}
}
}
}
func (l *AFAlgAuditListener) resetTailCursor() {
l.pos = 0
l.leftover = nil
l.droppedOversize = false
l.cursorAnchor = nil
}
func (l *AFAlgAuditListener) cursorAnchorChanged() bool {
return l.cursorAnchorChangedAt(l.pos, l.cursorAnchor)
}
func (l *AFAlgAuditListener) cursorAnchorChangedAt(pos int64, anchor []byte) bool {
if l.file == nil || pos <= 0 || len(anchor) == 0 {
return false
}
if int64(len(anchor)) > pos {
return true
}
buf := make([]byte, len(anchor))
n, err := l.file.ReadAt(buf, pos-int64(len(buf)))
if err != nil || n != len(buf) {
return true
}
return !bytes.Equal(buf, anchor)
}
func (l *AFAlgAuditListener) refreshCursorAnchorAfterRead(verifiedPos int64, verifiedAnchor []byte) bool {
if l.cursorAnchorChangedAt(verifiedPos, verifiedAnchor) {
l.resetTailCursor()
return false
}
l.captureCursorAnchor()
if l.cursorAnchorChangedAt(verifiedPos, verifiedAnchor) {
l.resetTailCursor()
return false
}
return true
}
func (l *AFAlgAuditListener) captureCursorAnchor() {
if l.file == nil || l.pos <= 0 {
l.cursorAnchor = nil
return
}
n := afAlgCursorAnchorBytes
if l.pos < int64(n) {
n = int(l.pos)
}
buf := make([]byte, n)
read, err := l.file.ReadAt(buf, l.pos-int64(n))
if err != nil || read != n {
l.cursorAnchor = nil
return
}
l.cursorAnchor = buf
}
// feed appends a chunk of audit-log bytes to the leftover buffer and
// emits a finding for each complete line containing the csm_af_alg_socket
// key. Lines are kept in the leftover until terminated by '\n' so a
// short read at the end of the buffer does not corrupt a multi-byte
// timestamp split across two reads.
func (l *AFAlgAuditListener) feed(chunk []byte) {
l.leftover = append(l.leftover, chunk...)
for {
idx := bytes.IndexByte(l.leftover, '\n')
if idx < 0 {
// No complete line yet. A real audit record fits well under the
// cap; an unterminated buffer past it is garbage (truncated write,
// binary noise, or an attacker-stretched exe= path). Drop it so the
// accumulator cannot grow without bound, and resync at the next
// newline.
if len(l.leftover) > afAlgMaxLeftoverBytes {
l.leftover = nil
l.droppedOversize = true
}
return
}
line := l.leftover[:idx]
l.leftover = l.leftover[idx+1:]
// Skip the remainder of a line whose head was already dropped for
// exceeding the cap: it is a partial record, not a parseable line.
if l.droppedOversize {
l.droppedOversize = false
continue
}
l.handleLine(string(line))
}
}
// handleLine inspects one audit log line. If it carries the
// csm_af_alg_socket key, parse the event and dispatch a finding.
func (l *AFAlgAuditListener) handleLine(line string) {
ev, ok := checks.ParseAFAlgEventLine(line)
if !ok {
return
}
l.eventCount.Add(1)
finding := alert.Finding{
Severity: alert.Critical,
Check: "af_alg_socket_use",
Message: fmt.Sprintf("AF_ALG socket opened by uid=%s exe=%s", ev.UID, ev.Exe),
TenantID: checks.AFAlgOwner(ev),
Timestamp: time.Now(),
Details: fmt.Sprintf(
"Live audit-log detection: timestamp=%s serial=%s\nauid=%s uid=%s comm=%q exe=%q pid=%s\n"+
"AF_ALG is essentially never used by cPanel/PHP workloads. This is\n"+
"the kernel-level exploit signature for CVE-2026-31431 (\"Copy Fail\").\n"+
"This event was caught by the live audit-log listener (sub-second\n"+
"latency); investigate this process immediately.",
ev.Timestamp, ev.Serial, ev.AUID, ev.UID, ev.Comm, ev.Exe, ev.PID,
),
}
// Non-blocking send — the alert dispatcher buffer is sized for bursts;
// dropping a finding under extreme pressure is preferable to blocking
// the listener loop.
if !alert.TryEnqueue(l.alertCh, finding) {
csmlog.Warn("af_alg audit listener: alert channel full; finding dropped", "uid", ev.UID, "exe", ev.Exe)
}
// Optional reactions (kill, quarantine) gated by config; implemented
// in af_alg_react.go so the BPF path can reuse the same logic.
reactToAFAlgEvent(l.cfg, ev)
}
//go:build !(linux && bpf)
package daemon
import (
"context"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/config"
)
// tryStartBPFLSM is the no-tag fallback. The real implementation lives in
// af_alg_bpf.go behind //go:build linux && bpf.
func tryStartBPFLSM(_ context.Context, _ chan<- alert.Finding, _ *config.Config) (AFAlgLiveMonitor, error) {
return nil, bpf.ErrNotBuilt
}
// Code generated by bpf2go; DO NOT EDIT.
//go:build 386 || amd64
package af_alg_bpfprog
import (
"bytes"
_ "embed"
"fmt"
"io"
"structs"
"github.com/cilium/ebpf"
)
type AFAlgAfAlgEvent struct {
_ structs.HostLayout
Uid uint32
Pid uint32
Ppid uint32
Comm [16]uint8
ParentComm [16]uint8
Exe [256]uint8
}
type AFAlgCsmQueueStats struct {
_ structs.HostLayout
Lost uint64
Submitted uint64
}
// Names of all BPF objects in the ELF.
//
// Used for safe lookups in a Collection or CollectionSpec.
const (
AFAlgMapEvents = "events"
AFAlgMapQueueStats = "queue_stats"
AFAlgProgCsmBlockAfAlg = "csm_block_af_alg"
AFAlgVarUnused = "unused"
)
// LoadAFAlg returns the embedded CollectionSpec for AFAlg.
func LoadAFAlg() (*ebpf.CollectionSpec, error) {
reader := bytes.NewReader(_AFAlgBytes)
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
if err != nil {
return nil, fmt.Errorf("can't load AFAlg: %w", err)
}
return spec, err
}
// LoadAFAlgObjects loads AFAlg and converts it into a struct.
//
// The following types are suitable as obj argument:
//
// *AFAlgObjects
// *AFAlgPrograms
// *AFAlgMaps
//
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
func LoadAFAlgObjects(obj any, opts *ebpf.CollectionOptions) error {
spec, err := LoadAFAlg()
if err != nil {
return err
}
return spec.LoadAndAssign(obj, opts)
}
// AFAlgSpecs contains maps and programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type AFAlgSpecs struct {
AFAlgProgramSpecs
AFAlgMapSpecs
AFAlgVariableSpecs
}
// AFAlgProgramSpecs contains programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type AFAlgProgramSpecs struct {
CsmBlockAfAlg *ebpf.ProgramSpec `ebpf:"csm_block_af_alg"`
}
// AFAlgMapSpecs contains maps before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type AFAlgMapSpecs struct {
Events *ebpf.MapSpec `ebpf:"events"`
QueueStats *ebpf.MapSpec `ebpf:"queue_stats"`
}
// AFAlgVariableSpecs contains global variables before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type AFAlgVariableSpecs struct {
Unused *ebpf.VariableSpec `ebpf:"unused"`
}
// AFAlgObjects contains all objects after they have been loaded into the kernel.
//
// It can be passed to LoadAFAlgObjects or ebpf.CollectionSpec.LoadAndAssign.
type AFAlgObjects struct {
AFAlgPrograms
AFAlgMaps
AFAlgVariables
}
func (o *AFAlgObjects) Close() error {
return _AFAlgClose(
&o.AFAlgPrograms,
&o.AFAlgMaps,
)
}
// AFAlgMaps contains all maps after they have been loaded into the kernel.
//
// It can be passed to LoadAFAlgObjects or ebpf.CollectionSpec.LoadAndAssign.
type AFAlgMaps struct {
Events *ebpf.Map `ebpf:"events"`
QueueStats *ebpf.Map `ebpf:"queue_stats"`
}
func (m *AFAlgMaps) Close() error {
return _AFAlgClose(
m.Events,
m.QueueStats,
)
}
// AFAlgVariables contains all global variables after they have been loaded into the kernel.
//
// It can be passed to LoadAFAlgObjects or ebpf.CollectionSpec.LoadAndAssign.
type AFAlgVariables struct {
Unused *ebpf.Variable `ebpf:"unused"`
}
// AFAlgPrograms contains all programs after they have been loaded into the kernel.
//
// It can be passed to LoadAFAlgObjects or ebpf.CollectionSpec.LoadAndAssign.
type AFAlgPrograms struct {
CsmBlockAfAlg *ebpf.Program `ebpf:"csm_block_af_alg"`
}
func (p *AFAlgPrograms) Close() error {
return _AFAlgClose(
p.CsmBlockAfAlg,
)
}
func _AFAlgClose(closers ...io.Closer) error {
for _, closer := range closers {
if err := closer.Close(); err != nil {
return err
}
}
return nil
}
// Do not access this directly.
//
//go:embed afalg_x86_bpfel.o
var _AFAlgBytes []byte
package daemon
import (
"context"
"fmt"
"strings"
"sync"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/metrics"
)
// AFAlgLiveMonitor was the local name for the live-monitor interface;
// it is now an alias of the shared bpf.Backend. Existing call sites in
// reactToAFAlgEvent etc. continue to compile.
type AFAlgLiveMonitor = bpf.Backend
// AFAlgBackend* are the operator-facing cfg.Detection.AFAlgBackend values.
// Reuse bpf constants where the public value already matches.
const (
AFAlgBackendAuto = bpf.BackendAuto
AFAlgBackendBPF = bpf.BackendBPF
AFAlgBackendAuditd = "auditd"
AFAlgBackendNone = bpf.BackendNone
)
var (
afAlgBackendMetricOnce sync.Once
afAlgBackendMetric *metrics.GaugeVec
)
func ensureAFAlgBackendMetric() {
afAlgBackendMetricOnce.Do(func() {
afAlgBackendMetric = metrics.NewGaugeVec(
"csm_af_alg_backend",
"Active AF_ALG (Copy Fail) live-monitor backend; 1 for the selected kind, 0 otherwise.",
[]string{"kind"},
)
metrics.MustRegister("csm_af_alg_backend", afAlgBackendMetric)
})
}
func setAFAlgBackendMetric(active string) {
ensureAFAlgBackendMetric()
for _, k := range []string{"bpf-lsm", "auditd-tail", "none"} {
v := 0.0
if k == active {
v = 1.0
}
afAlgBackendMetric.With(k).Set(v)
}
switch active {
case "bpf-lsm":
bpf.SetActive("af_alg", bpf.BackendBPF)
case "auditd-tail":
bpf.SetActive("af_alg", bpf.BackendLegacy)
default:
bpf.SetActive("af_alg", bpf.BackendNone)
}
}
// StartAFAlgLiveMonitor returns the live monitor selected by
// cfg.Detection.AFAlgBackend. "" / "auto" tries BPF LSM first (when
// compiled in and kernel-supported) and falls back to the audit listener.
// "bpf" requires BPF — no audit fallback if BPF is unavailable, useful for
// hosts where the operator deliberately wants the kernel-side block or
// nothing. "auditd" pins the audit listener even on BPF-capable hosts, the
// kill switch when a BPF-tagged release misbehaves. "none" disables the
// live monitor (the periodic critical-tier check still runs). Returns nil
// when no backend ends up active; the metric csm_af_alg_backend{kind=...}
// reflects whichever path was selected.
func StartAFAlgLiveMonitor(alertCh chan<- alert.Finding, cfg *config.Config) AFAlgLiveMonitor {
choice := strings.ToLower(strings.TrimSpace(cfg.Detection.AFAlgBackend))
if choice == "" {
choice = AFAlgBackendAuto
}
switch choice {
case AFAlgBackendAuto, AFAlgBackendBPF, AFAlgBackendAuditd, AFAlgBackendNone:
default:
csmlog.Warn("af_alg live monitor: unknown backend choice, falling back to auto",
"value", choice,
)
choice = AFAlgBackendAuto
}
if choice == AFAlgBackendNone {
csmlog.Info("af_alg live monitor: disabled by config")
setAFAlgBackendMetric("none")
return nil
}
var bpfErr error
if choice == AFAlgBackendAuto || choice == AFAlgBackendBPF {
if mon, err := tryStartBPFLSMFn(context.Background(), alertCh, cfg); err == nil && mon != nil {
csmlog.Info("af_alg live monitor", "backend", "bpf-lsm", "choice", choice)
setAFAlgBackendMetric("bpf-lsm")
return mon
} else if err != nil {
bpfErr = err
csmlog.Info("af_alg live monitor: BPF LSM unavailable",
"state", "bpf-lsm-unsupported",
"reason", err.Error(),
"choice", choice,
)
if choice == AFAlgBackendBPF {
csmlog.Warn("af_alg live monitor: af_alg_backend=bpf but BPF unavailable; no live detection",
"reason", err.Error(),
)
setAFAlgBackendMetric("none")
emitBPFUnavailableFinding(alertCh, "af_alg", choice, "", err)
return nil
}
}
}
listener, err := NewAFAlgAuditListener(alertCh, cfg) //nolint:staticcheck // The Linux constructor is fallible; the non-Linux stub always returns an error.
if err != nil { //nolint:staticcheck // The Linux constructor can also succeed.
csmlog.Warn("af_alg live monitor: auditd fallback unavailable", "err", err)
setAFAlgBackendMetric("none")
if bpfErr != nil {
emitBPFUnavailableFinding(alertCh, "af_alg", choice, "", fmt.Errorf("BPF unavailable: %w; audit fallback unavailable: %w", bpfErr, err))
}
return nil
}
csmlog.Info("af_alg live monitor", "backend", "auditd-tail", "choice", choice)
setAFAlgBackendMetric("auditd-tail")
if bpfErr != nil {
emitBPFUnavailableFinding(alertCh, "af_alg", choice, "auditd-tail", bpfErr)
}
return listener
}
// tryStartBPFLSMFn is a package-level indirection so tests can substitute a
// fake for the BPF probe path without needing the bpf build tag or kernel
// privileges. Production code goes through tryStartBPFLSM, which is the
// stub on default builds and the real probe on -tags bpf builds.
var tryStartBPFLSMFn = tryStartBPFLSM
//go:build linux
package daemon
import (
"context"
"errors"
"fmt"
"math"
"os"
"strconv"
"strings"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/processhandle"
)
var signalAFAlgProcess = processhandle.Signal
// reactToAFAlgEvent applies opt-in live reactions when an AF_ALG socket
// open is caught by either the audit-log listener or the BPF LSM hook.
// Currently supports a single reaction: SIGKILL the offending process
// (gated by config.AutoResponse.CopyFailKillProcess).
//
// Reactions are intentionally narrow: a critical alert is always emitted
// by the listener itself (this function is for *additional* responses
// beyond alerting). Quarantining the offending exe is a future addition;
// keeping the surface minimal until the kill path has been observed in
// production.
//
// Refuses to act on PID 0/1 to avoid catastrophic mistakes if the parser
// ever returns something unexpected.
func reactToAFAlgEvent(cfg *config.Config, ev checks.AFAlgEvent) {
if cfg == nil || !cfg.AutoResponse.CopyFailKillProcess {
return
}
pid, err := strconv.Atoi(ev.PID)
if err != nil || pid <= 1 {
return
}
refused := false
err = signalAFAlgProcess(context.Background(), pid, unix.SIGKILL, func() error {
_, valid, reason := afAlgKillTarget(ev)
if !valid {
refused = true
return fmt.Errorf("refusing to kill: %s", reason)
}
return nil
})
rec := actionlog.Record{Op: "respond.kill_process", Target: "pid " + strconv.Itoa(pid), Actor: actionlog.Daemon, ActorDetail: ev.Exe, Reason: "AF_ALG socket open", Result: actionlog.Applied}
if err != nil {
rec.Result = actionlog.Failed
rec.Error = err.Error()
}
if refused || errors.Is(err, os.ErrProcessDone) {
rec.Result = actionlog.Refused
}
actionlog.Write(rec)
if err != nil {
csmlog.Warn("af_alg react: kill failed",
"pid", pid, "exe", ev.Exe, "uid", ev.UID,
"err", err,
)
return
}
csmlog.Info("af_alg react: killed offending process",
"pid", pid, "exe", ev.Exe, "uid", ev.UID, "comm", ev.Comm,
)
}
// afAlgKillTarget reports whether ev still names the process it described, and
// may therefore be killed.
//
// The audit record can be a tick old by the time it is read. A PID is recycled
// freely on a busy host, so acting on the number alone means SIGKILLing a
// process that merely inherited it -- as root, on a production server. Three
// facts must agree before the kill: the executable behind the PID is the one
// the record named, the process is owned by the recorded user, and it started
// no later than the event. A process that began after the event cannot be the
// one the event describes.
//
// Anything unverifiable fails closed: an event with no executable, an
// unreadable procfs entry, or a process that has already exited.
func afAlgKillTarget(ev checks.AFAlgEvent) (int, bool, string) {
pid, err := strconv.Atoi(ev.PID)
if err != nil || pid <= 1 {
return 0, false, "implausible pid"
}
if ev.Exe == "" || ev.Exe == "(null)" {
return 0, false, "event carries no executable to verify"
}
eventUID, err := strconv.ParseUint(ev.UID, 10, 32)
if err != nil {
return 0, false, "event carries no uid to verify"
}
exe, err := os.Readlink(fmt.Sprintf("%s/%d/exe", procRootDir, pid))
if err != nil {
return 0, false, "process is gone or its executable is unreadable"
}
// A deleted binary is reported as "<path> (deleted)".
if exe != ev.Exe && strings.TrimSuffix(exe, " (deleted)") != ev.Exe {
return 0, false, "pid now runs a different executable"
}
uid, ok := afAlgProcessUID(pid)
if !ok {
return 0, false, "process ownership unreadable"
}
if uid != eventUID {
return 0, false, "pid now belongs to a different user"
}
eventAt, evErr := strconv.ParseFloat(ev.Timestamp, 64)
if evErr != nil || math.IsNaN(eventAt) || math.IsInf(eventAt, 0) || eventAt <= 0 {
return 0, false, "process start time could not be compared with the event"
}
startedBefore, ok := afAlgProcessStartedBefore(pid, eventAt)
if !ok {
return 0, false, "process start time could not be compared with the event"
}
if !startedBefore {
return 0, false, "pid was recycled after the event"
}
return pid, true, ""
}
func afAlgProcessUID(pid int) (uint64, bool) {
data, err := os.ReadFile(fmt.Sprintf("%s/%d/status", procRootDir, pid))
if err != nil {
return 0, false
}
for _, line := range strings.Split(string(data), "\n") {
rest, found := strings.CutPrefix(line, "Uid:")
if !found {
continue
}
fields := strings.Fields(rest)
if len(fields) != 4 {
return 0, false
}
uid, err := strconv.ParseUint(fields[0], 10, 32)
if err != nil {
return 0, false
}
for _, field := range fields[1:] {
credential, parseErr := strconv.ParseUint(field, 10, 32)
if parseErr != nil || credential != uid {
return 0, false
}
}
return uid, true
}
return 0, false
}
func afAlgProcessStartedBefore(pid int, eventAt float64) (bool, bool) {
statData, err := os.ReadFile(fmt.Sprintf("%s/%d/stat", procRootDir, pid))
if err != nil {
return false, false
}
// The comm field is parenthesised and may contain spaces, so fields are
// counted from after the closing parenthesis.
close := strings.LastIndex(string(statData), ")")
if close < 0 {
return false, false
}
fields := strings.Fields(string(statData)[close+1:])
// starttime is field 22 overall, which is index 19 after pid and comm.
if len(fields) < 20 {
return false, false
}
ticks, err := strconv.ParseUint(fields[19], 10, 64)
if err != nil {
return false, false
}
uptimeData, err := os.ReadFile(fmt.Sprintf("%s/uptime", procRootDir))
if err != nil {
return false, false
}
uptimeFields := strings.Fields(string(uptimeData))
if len(uptimeFields) == 0 {
return false, false
}
uptime, err := strconv.ParseFloat(uptimeFields[0], 64)
if err != nil || math.IsNaN(uptime) || math.IsInf(uptime, 0) || uptime < 0 {
return false, false
}
// Read wall time after uptime so the derived event uptime is conservative.
// Ambiguous boundary cases fail closed instead of accepting a recycled PID.
observedAt := float64(time.Now().UnixNano()) / float64(time.Second)
elapsed := observedAt - eventAt
if elapsed < 0 {
return false, true
}
eventUptime := uptime - elapsed
return eventUptime >= 0 && float64(ticks)/afAlgClockTicks <= eventUptime, true
}
// afAlgClockTicks is USER_HZ, 100 on every architecture Linux ships for the
// platforms CSM runs on.
const afAlgClockTicks = 100.0
package daemon
// Provenance must be available before any detector starts, even when the
// optional BPF monitors never initialize the process-context cache.
func init() { wireAncestryProvenance() }
// pkgManagerComms are the process names CSM treats as evidence that an
// observed sensitive-file write originated from a legitimate root-driven
// package transaction. The list intentionally omits shells (sh, bash) and
// generic utilities (cp, mv) -- attackers reuse those. Matching the package
// manager binary itself anywhere in the parent chain is the discriminator.
var pkgManagerComms = map[string]struct{}{
"dnf": {},
"dnf-3": {},
"microdnf": {},
"yum": {},
"rpm": {},
"dpkg": {},
"apt": {},
"apt-get": {},
"unattended-upgr": {}, // unattended-upgrade is comm-truncated to TASK_COMM_LEN-1.
}
func isPackageManagerComm(comm string) bool {
_, ok := pkgManagerComms[comm]
return ok
}
// initramfsBuilderComms are the tools that assemble a boot image in a
// temporary work directory, copying hundreds of system executables into it.
// The name alone proves nothing; callers also require the process to be root.
var initramfsBuilderComms = map[string]struct{}{
"dracut": {},
"mkinitramfs": {},
}
func isInitramfsBuilderComm(comm string) bool {
_, ok := initramfsBuilderComms[comm]
return ok
}
//go:build !linux || !bpf
package daemon
import "github.com/pidginhost/csm/internal/processctx"
// wireAncestryCache is a no-op on hosts built without the bpf build tag.
// cachedAncestryEvidence stays nil and ancestry falls back to the live /proc
// walk, which is racy for short-lived writers but fails closed.
func wireAncestryCache(*processctx.Cache) {}
package daemon
import (
"context"
"crypto/rand"
"errors"
"net/netip"
"slices"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/antispam/hostfacts"
"github.com/pidginhost/csm/internal/antispam/reconcile"
"github.com/pidginhost/csm/internal/antispam/worker"
"github.com/pidginhost/csm/internal/attackdb"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/ipmap"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/mailstats"
"github.com/pidginhost/csm/internal/netutil"
"github.com/pidginhost/csm/internal/obs"
)
// antispamProbeInterval is how often rspamd's scan socket is probed while
// antispam is enabled.
const antispamProbeInterval = 20 * time.Second
type antispamHealthState struct {
reconcile.Health
// Serialize reloads, log observations and probe completion with latest
// publication, but never hold mu across a probe or notification delivery.
mu sync.Mutex
enabled bool
socket string
generation uint64
latest string
}
// antispamHealthChecks are the conditions the probe loop keeps current in
// the latest set. A recovery is a one-time notice, never a condition.
var antispamHealthChecks = []string{reconcile.CheckRspamdDown, reconcile.CheckScanError}
// antispamPing probes rspamd's scan socket; tests replace it.
var antispamPing = hostfacts.Ping
// The IP map's canary probe, rspamd group lookup and file path; tests
// replace them.
var (
antispamCanary = hostfacts.Canary
antispamRspamdGroup = hostfacts.RspamdGroup
antispamIPMapPath = ipmap.DefaultPath
)
// antispamIPMapInterval is how often the CSM IP map is rebuilt;
// antispamCanaryDelay bounds the wait for a canary check, including
// after writes made by reconciliation and after the load grace expires.
const (
antispamIPMapInterval = 10 * time.Minute
antispamCanaryDelay = 30 * time.Second
)
// antispamHostState reports the host's mail stack and whether CSM's
// antispam integration is installed; tests replace it.
var antispamHostState = func() (antispam.MailStack, bool) {
f := hostfacts.CollectInstalled()
return f.Stack, antispam.Assess(false, antispam.Settings{}, f).Installed
}
// antispamReconciler returns the host the antispam reconcile loop works on;
// tests replace it.
var antispamReconciler = func() reconcile.Reconciler {
return reconcile.Reconciler{
Invalidate: worker.Invalidate,
Collect: hostfacts.Collect,
Installed: hostfacts.CollectInstalled,
Run: worker.Run,
Now: time.Now,
ReadKey: hostfacts.ReadDQSKey,
Probe: hostfacts.QueryDNS,
Feeds: reconcile.NewFeedState(),
Learning: reconcile.NewLearningState(),
EnsureKeys: func(ctx context.Context, dir string) (bool, error) {
newKeypair := func(ctx context.Context) (antispam.Keypair, error) {
return hostfacts.NewFuzzyKeypair(ctx, hostfacts.RunCommand)
}
return hostfacts.EnsureLearningKeys(ctx, dir, newKeypair, rand.Reader)
},
LoadKeys: hostfacts.LoadLearningKeys,
CollectLearning: hostfacts.CollectLearning,
Replaced: func() ([]antispam.Replacement, error) {
return hostfacts.Replaced(antispam.ReplacedDir)
},
}
}
// startAntispam starts antispam at daemon startup on a host it supports,
// and otherwise says why an enabled switch does nothing.
func (d *Daemon) startAntispam() {
d.antispamStarted.Store(true)
if !d.startAntispamLoop() {
warnAntispamUnsupported(d.currentCfg())
}
}
// startAntispamLoop starts antispam reconciliation once, on a host it
// supports: a pass at start, after each config reload, every 15 minutes,
// and 20 seconds after a busy lock. It also starts the
// rspamd probe. Shutdown cancels a running pass or probe. It reports
// whether the host is supported.
func (d *Daemon) startAntispamLoop() bool {
if antispamUnsupported() {
return false
}
if d.scanContext().Err() != nil || !d.antispamClaimed.CompareAndSwap(false, true) {
return true
}
// Health state starts empty; findings kept from before a restart
// describe an outage no longer tracked.
checks.StoreLatestScanFindings(d.store, antispamHealthChecks, nil)
d.store.RearmAbsentDedupFindings(antispamHealthChecks, nil)
r := d.withIPMap(antispamReconciler())
r.Learner = d.learnerProblems
loop := reconcile.NewLoop(func(ctx context.Context) reconcile.Outcome {
d.antispamReconcileMu.Lock()
defer d.antispamReconcileMu.Unlock()
d.pauseLearner()
out := r.Pass(ctx, reconcile.DesiredFromConfig(d.currentCfg()))
d.syncLearner(out)
return out
}, d.reportAntispam)
d.antispamLoop.Store(loop)
// Publish the reload target before reading initial health config so a
// reload during setup is either picked up here or applies itself.
d.syncAntispamHealth()
ctx := d.scanContext()
d.wg.Add(4)
obs.Go("antispam-reconcile", func() {
defer d.wg.Done()
defer d.stopLearner()
loop.Run(ctx)
})
obs.Go("antispam-health", func() {
defer d.wg.Done()
d.antispamHealthLoop(ctx)
})
obs.Go("antispam-ipmap", func() {
defer d.wg.Done()
d.antispamIPMapLoop(ctx)
})
obs.Go("antispam-rspamd-log", func() {
defer d.wg.Done()
d.rspamdLogLoop(ctx)
})
return true
}
// withIPMap gives the reconciler the IP map writer's hooks and shares its
// feed memory with the IP map loop.
func (d *Daemon) withIPMap(r reconcile.Reconciler) reconcile.Reconciler {
if r.Feeds == nil {
r.Feeds = reconcile.NewFeedState()
}
w := ipmap.NewWriter(antispamIPMapPath)
d.antispamFeeds.Store(r.Feeds)
d.ipmapWriter.Store(w)
r.IPMap = reconcile.IPMapHooks{
Write: func(ctx context.Context) error {
d.ipmapMu.Lock()
defer d.ipmapMu.Unlock()
// Without a surviving map, a failed write must not restart
// canary warnings before feed preconditions allow the rule back.
wrote, err := d.writeIPMap(ctx, time.Now())
if wrote || err == nil {
d.ipmapRemoved.Store(false)
}
return err
},
Remove: func() error {
d.ipmapMu.Lock()
defer d.ipmapMu.Unlock()
d.ipmapRemoved.Store(true)
d.ipmapCanaryMisses = 0
return w.Remove()
},
}
return r
}
// antispamIPMapLoop refreshes the CSM IP map at start and every 10
// minutes. Independent canary checks cannot be hidden by a new write
// resetting the load grace, and also see writes made by reconciliation.
func (d *Daemon) antispamIPMapLoop(ctx context.Context) {
ticker := time.NewTicker(antispamIPMapInterval)
defer ticker.Stop()
probe := time.NewTicker(antispamCanaryDelay)
defer probe.Stop()
d.refreshIPMap(ctx, time.Now())
for ctx.Err() == nil {
select {
case <-ctx.Done():
return
case <-ticker.C:
d.refreshIPMap(ctx, time.Now())
case <-probe.C:
d.checkIPMap(ctx, time.Now(), false)
}
}
}
// ipMapWanted reports whether the configuration asks for the CSM IP map.
func ipMapWanted(cfg *config.Config) bool {
as := cfg.EmailProtection.Antispam
return as.Enabled && !cfg.ObserveMode() && slices.Contains(as.Feeds.Enabled, antispam.FeedCSMIPMap)
}
// A rule can reuse a surviving map after both removal and a later write
// fail. Its active state then overrides the earlier removal request.
func (d *Daemon) ipMapRemoved() bool {
return d.ipmapRemoved.Load() && !d.antispamFeeds.Load().IPMapActive()
}
// writeIPMap builds the map from CSM's sources and writes it if changed.
// The caller holds ipmapMu so removal cannot overtake a pending write.
func (d *Daemon) writeIPMap(ctx context.Context, now time.Time) (bool, error) {
return d.ipmapWriter.Load().Write(ipmap.Build(ipmapSources(now, d.antispamInfraIPs(d.currentCfg()))), antispamRspamdGroup(ctx), now)
}
// Reuse the firewall's last successful DNS answers without resolving names
// on the mail log path. Firewall-only entries must not suppress evidence.
func (d *Daemon) antispamInfraIPs(cfg *config.Config) []string {
if d.fwEngine == nil {
return cfg.InfraIPs
}
return slices.Concat(cfg.InfraIPs, d.fwEngine.ResolvedInfraIPs(infraHostnames(cfg.InfraIPs)))
}
// refreshIPMap writes the map while the feed is wanted, runs the canary
// while the rule is in use, and settles antispam_ip_map. It reports
// whether it wrote a new map.
func (d *Daemon) refreshIPMap(ctx context.Context, now time.Time) bool {
return d.checkIPMap(ctx, now, true)
}
func (d *Daemon) checkIPMap(ctx context.Context, now time.Time, rebuild bool) bool {
cfg := d.currentCfg()
d.ipmapMu.Lock()
if !ipMapWanted(cfg) || d.ipMapRemoved() || d.ipmapCanarySocket != cfg.EmailProtection.Antispam.RspamdSocket {
d.ipmapCanaryMisses = 0
}
d.ipmapCanarySocket = cfg.EmailProtection.Antispam.RspamdSocket
if !rebuild && !ipMapWanted(cfg) {
// Only hosts configured for the map pay for the frequent canary
// tick; the 10-minute refresh settles the check everywhere else.
d.ipmapMu.Unlock()
return false
}
var findings []alert.Finding
wrote := false
if ctx.Err() == nil && ipMapWanted(cfg) && !d.ipMapRemoved() {
if rebuild {
wrote, _ = d.writeIPMap(ctx, now)
}
st := d.ipmapWriter.Load().State()
// Until the rule is in use only a write failure counts.
seen, probeErr := st.Gen, error(nil)
if d.antispamFeeds.Load().IPMapActive() {
seen, probeErr = antispamCanary(ctx, cfg.EmailProtection.Antispam.RspamdSocket)
}
p := ipmap.CanaryProblem(st, seen, probeErr, now)
// One bad canary answer, such as during an rspamd reload, is not a
// problem; two in a row are. Write failures count at once.
if p != "" && st.Err == nil {
d.ipmapCanaryMisses++
} else {
d.ipmapCanaryMisses = 0
}
if p != "" && (st.Err != nil || d.ipmapCanaryMisses >= 2) {
findings = append(findings, reconcile.IPMapFinding(p, now))
}
}
d.ipmapMu.Unlock()
if ctx.Err() == nil {
// A dismissal acknowledges this outage, not every future failure
// that shares the map's stable finding identity.
d.store.RearmAbsentDedupFindings([]string{reconcile.CheckIPMap}, findings)
d.processScanFindings(cfg, findings, []string{reconcile.CheckIPMap}, "antispam")
}
return wrote
}
// ipmapSources gathers what the CSM IP map lists. infra
// holds infra_ips entries: like whitelisted and host addresses, nothing in
// them is listed.
func ipmapSources(now time.Time, infra []string) ipmap.Sources {
src := ipmap.Sources{Entries: map[string][]string{}}
for _, raw := range infra {
raw = strings.TrimSpace(raw)
if p, err := netip.ParsePrefix(raw); err == nil {
src.NeverNets = append(src.NeverNets, p.Masked())
} else if a, err := netip.ParseAddr(raw); err == nil && a.Zone() == "" {
a = a.Unmap()
src.NeverNets = append(src.NeverNets, netip.PrefixFrom(a, a.BitLen()))
}
}
if tdb := checks.GetThreatDB(); tdb != nil {
ts := tdb.MapSources()
src.Entries[ipmap.Drop.Name] = slices.Concat(ts.FeedNets["spamhaus-drop"], ts.FeedIPs["spamhaus-drop"])
src.Entries[ipmap.Blocked.Name] = ts.Permanent
src.Entries[ipmap.BlocklistDE.Name] = slices.Concat(ts.FeedIPs["blocklist-de"], ts.FeedNets["blocklist-de"])
src.Entries[ipmap.CINS.Name] = slices.Concat(ts.FeedIPs["cins-army"], ts.FeedNets["cins-army"])
for _, w := range ts.Whitelist {
if a, err := netip.ParseAddr(w); err == nil {
src.Never = append(src.Never, a.Unmap())
}
}
}
if adb := attackdb.Global(); adb != nil {
src.Entries[ipmap.MailAuth.Name], src.Entries[ipmap.Spam.Name] = adb.MapSources(now)
}
src.Never = append(src.Never, netutil.HostAddresses()...)
return src
}
// reconcileAntispam applies a config reload: health tracking follows the
// new switch and socket at once, and a pass reads the new config.
func (d *Daemon) reconcileAntispam() {
// Observe mode and disable stop the learner at once: they cannot wait
// for a pass, a worker lock or alert delivery.
d.syncLearner(reconcile.Outcome{})
if loop := d.antispamLoop.Load(); loop != nil {
d.syncAntispamHealth()
loop.Trigger()
return
}
// Only an enabled switch needs the host check, which can run commands.
cfg := d.currentCfg()
if !cfg.EmailProtection.Antispam.Enabled {
return
}
// Before startup reaches antispam, startup reads the new config itself.
// After it, a host antispam did not support then, for example one where
// Postfix started after CSM, gets its loop now.
supported := !antispamUnsupported()
if d.antispamStarted.Load() {
supported = d.startAntispamLoop()
}
if !supported {
warnAntispamUnsupported(cfg)
}
}
// antispamUnsupported reports a host antispam does not run on: no mail
// stack it integrates with and nothing of CSM's installed. A Postfix host
// can read as no stack while postfix restarts; with CSM's integration
// installed its passes must still run, to keep the integration in place
// and report why.
func antispamUnsupported() bool {
stack, installed := antispamHostState()
return stack == antispam.StackNone && !installed
}
// warnAntispamUnsupported explains an enabled switch that does nothing:
// the integration exists only for cPanel's Exim and for Postfix.
func warnAntispamUnsupported(cfg *config.Config) {
if cfg.EmailProtection.Antispam.Enabled {
csmlog.Warn("antispam is enabled but inactive: it needs a cPanel host running Exim, or a Postfix host")
}
}
// syncAntispamHealth points health tracking at the configured socket while
// antispam is enabled. Disabling forgets every outage and scan error
// without a recovery notice.
func (d *Daemon) syncAntispamHealth() {
h := &d.antispamHealth
h.mu.Lock()
defer h.mu.Unlock()
as := d.currentCfg().EmailProtection.Antispam
if as.Enabled != h.enabled || as.RspamdSocket != h.socket {
// The same socket can be disabled and re-enabled while a probe runs.
// Its result must not count toward the new tracking period.
h.generation++
}
h.enabled, h.socket = as.Enabled, as.RspamdSocket
h.SetEnabled(as.Enabled, as.RspamdSocket)
d.storeAntispamHealthLocked()
}
func (d *Daemon) antispamHealthLoop(ctx context.Context) {
ticker := time.NewTicker(antispamProbeInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case now := <-ticker.C:
d.probeAntispam(ctx, now)
}
}
}
// recordScan counts one scan record, from Exim's main log or rspamd's, for
// the Email Security page and scan health. at is when it was read; the
// counts use the line's own time.
func (d *Daemon) recordScan(s antispam.Scan, line string, at time.Time) []alert.Finding {
d.mailStats.RecordScan(mailstats.LineTime(line, time.Local, at), s)
return d.recordAntispamScan(s, at)
}
func (d *Daemon) recordAntispamScan(s antispam.Scan, now time.Time) []alert.Finding {
h := &d.antispamHealth
h.mu.Lock()
defer h.mu.Unlock()
findings, recovered := h.Scan(s, now)
if recovered {
// Recovery can end before the next probe. Publish it while new log
// lines and probes cannot reactivate the condition being rearmed.
d.storeAntispamHealthLocked()
}
return findings
}
// probeAntispam pings rspamd while antispam is enabled and reports an
// outage or recovery. Current conditions are published before notification
// delivery, which may block on a sink.
func (d *Daemon) probeAntispam(ctx context.Context, now time.Time) {
h := &d.antispamHealth
h.mu.Lock()
enabled, socket, generation := h.enabled, h.socket, h.generation
h.mu.Unlock()
if ctx.Err() != nil {
return
}
var err error
if enabled {
_, err = antispamPing(ctx, socket)
}
h.mu.Lock()
if ctx.Err() != nil || generation != h.generation {
// Shutdown or reload cut this tracking period short.
h.mu.Unlock()
return
}
var findings []alert.Finding
if enabled {
findings = h.Ping(socket, err, now)
}
d.storeAntispamHealthLocked()
h.mu.Unlock()
d.reportAntispamHealth(findings)
}
// The caller holds antispamHealth.mu so a reload cannot clear health and
// then have an old snapshot overwrite the latest set.
func (d *Daemon) storeAntispamHealthLocked() {
h := &d.antispamHealth
active := h.Active()
var state strings.Builder
for _, f := range active {
state.WriteString(f.Key() + "\x00" + f.Details + "\x00")
}
if state.String() == h.latest {
return
}
checks.StoreLatestScanFindings(d.store, antispamHealthChecks, active)
// A dismissal acknowledges one episode. Once the condition clears,
// forget it, so the next episode notifies again.
d.store.RearmAbsentDedupFindings(antispamHealthChecks, active)
h.latest = state.String()
}
// reportAntispamHealth delivers rspamd outage and recovery notices to
// history and notification directly. Alert-state dedup would hide a second
// outage on the same day, and each outage gets one notice.
func (d *Daemon) reportAntispamHealth(findings []alert.Finding) {
if len(findings) == 0 {
return
}
d.store.AppendHistory(findings)
// The probe's hysteresis bounds these notices, so they do not spend the
// hourly budget: a load-driven flap must not mute other alerts.
suppressions := d.store.LoadSuppressions()
if err := alert.DispatchNoticesWithNotificationFilter(d.currentCfg(), findings, func(fs []alert.Finding) []alert.Finding {
return filterUnsuppressedFindings(d.store, fs, suppressions)
}); err != nil {
csmlog.Warn("antispam health notification failed", "err", err)
}
}
// rspamdStatsSource is where the web UI reads rspamd's counters, or why
// it cannot, as the last completed pass found.
type rspamdStatsSource struct {
ep antispam.StatsEndpoint
reason string
}
// reportAntispam keeps the findings of the checks a pass settled in the
// latest set, alerts on new ones, and logs a pass that did not complete.
// A completed pass also publishes where rspamd's counters can be read.
func (d *Daemon) reportAntispam(out reconcile.Outcome) {
if out.Err != nil && !errors.Is(out.Err, context.Canceled) {
csmlog.Warn("antispam reconcile did not complete", "err", out.Err)
}
if len(out.Checks) > 0 {
d.rspamdStats.Store(&rspamdStatsSource{ep: out.Stats, reason: out.StatsReason})
}
if src := out.ScanSource; src != nil {
// Word scan errors for this host before its milters can raise one.
h := &d.antispamHealth
h.mu.Lock()
h.SetStack(src.Stack, src.MilterAction)
d.storeAntispamHealthLocked()
h.mu.Unlock()
d.antispamScanSource.Store(src)
}
d.processScanFindings(d.currentCfg(), out.Findings, out.Checks, "antispam")
}
// rspamdStatsStatus pairs the last completed pass's endpoint with live
// probe health. Dismissing an outage finding must not report recovery.
func (d *Daemon) rspamdStatsStatus() (antispam.StatsEndpoint, string, bool) {
down := slices.ContainsFunc(d.antispamHealth.Active(), func(f alert.Finding) bool {
return f.Check == reconcile.CheckRspamdDown
})
if src := d.rspamdStats.Load(); src != nil {
return src.ep, src.reason, down
}
return antispam.StatsEndpoint{}, "", down
}
package daemon
import (
"context"
"slices"
"strconv"
"sync"
"sync/atomic"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/antispam/learn"
"github.com/pidginhost/csm/internal/antispam/reconcile"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/store"
)
type learnEngine interface {
Run(context.Context)
SetGate(antispam.LearningGate)
Pause()
Publishing() bool
Problems() []antispam.Problem
Operator(context.Context, learn.OperatorRequest) (learn.OperatorResult, error)
SubmitReport(context.Context, learn.Candidate) (learn.Outcome, error)
ProbeStorage(context.Context) error
CancelUnsent(ctx context.Context, uid int, confirm func() (bool, error)) error
ReportUIDs(context.Context) ([]int, error)
Stats() (learn.Stats, error)
}
// newLearnEngine builds the learning engine; tests replace it.
var newLearnEngine = func(cfg learn.EngineConfig) learnEngine { return learn.NewEngine(cfg) }
// learnerRunner is one running engine, its Spam folder watcher and what
// they were built for.
type learnerRunner struct {
target reconcile.LearnerTarget
watch watchConfig
engine learnEngine
watcher learnWatcher
gate atomic.Pointer[antispam.LearningGate]
cancel context.CancelFunc
wg sync.WaitGroup
}
// setGate passes g to the engine and to the watcher's gates.
func (r *learnerRunner) setGate(g antispam.LearningGate) {
// Do not let the watcher read while the engine is still changing gates.
r.gate.Store(nil)
r.engine.SetGate(g)
r.gate.Store(&g)
}
func (r *learnerRunner) currentGate() antispam.LearningGate {
if g := r.gate.Load(); g != nil {
return *g
}
return antispam.LearningGate{}
}
// syncLearner follows a completed reconcile pass: it starts the learning
// engine when the pass names where it runs, passes it the gate, and stops
// it once learning is no longer wanted. Stopping closes the gate first, so
// no new read or send starts, then waits for calls in flight to end within
// their deadlines.
func (d *Daemon) syncLearner(out reconcile.Outcome) {
// Config decides before pass timing: a busy or canceled pass must not
// keep a learner running that observe mode or disable stopped.
d.antispamLearnerMu.Lock()
defer d.antispamLearnerMu.Unlock()
cfg := d.currentCfg()
as := cfg.EmailProtection.Antispam
if !cfg.ObserveMode() && (!as.Enabled || !as.Learning.Enabled) {
d.antispamWatchReset = true
}
if !as.Enabled || !as.Learning.Enabled || cfg.ObserveMode() || d.scanContext().Err() != nil {
d.stopLearnerLocked()
d.confirmRestoreIfDisabled()
if d.antispamWatchReset && !cfg.ObserveMode() && d.scanContext().Err() == nil {
d.resetLearningWatcher()
}
return
}
if d.antispamWatchReset && !d.resetLearningWatcher() {
return
}
if !slices.Contains(out.Checks, reconcile.CheckLearning) {
return
}
cur := d.antispamLearner.Load()
if out.Learner == (reconcile.LearnerTarget{}) {
// Wanted, but this pass named no learner: no controller or keys,
// or learning on Postfix did not resolve on.
if cur != nil {
cur.setGate(antispam.LearningGate{})
}
return
}
watch := watchConfigFor(out.Stack, cfg, cur)
if cur != nil && (cur.target != out.Learner || !cur.watch.equal(watch)) {
d.stopLearnerLocked()
cur = nil
}
if cur == nil {
// Draining the old engine can overlap a config reload.
if !d.learningWanted() {
return
}
if cur = d.startLearning(d.scanContext(), out.Learner, watch, out.LearningGate); cur == nil {
return
}
if !d.learningWanted() {
d.stopLearnerLocked()
}
return
}
if !d.learningWanted() {
d.stopLearnerLocked()
return
}
cur.setGate(out.LearningGate)
}
// learningWanted reports whether the current config asks for learning and
// the daemon is not shutting down.
func (d *Daemon) learningWanted() bool {
cfg := d.currentCfg()
as := cfg.EmailProtection.Antispam
return as.Enabled && as.Learning.Enabled && !cfg.ObserveMode() && d.scanContext().Err() == nil
}
// pauseLearner drains native work before a pass can change rspamd's
// configuration. A busy or canceled pass leaves the gate closed.
func (d *Daemon) pauseLearner() {
d.antispamLearnerMu.Lock()
defer d.antispamLearnerMu.Unlock()
if r := d.antispamLearner.Load(); r != nil {
r.gate.Store(nil)
r.engine.Pause()
}
}
// startLearning applies the first gate before launching either loop, so
// the watcher's immediate pass cannot miss an already open gate.
func (d *Daemon) startLearning(ctx context.Context, target reconcile.LearnerTarget, watch watchConfig, gate antispam.LearningGate) *learnerRunner {
db := store.Global()
if db == nil {
csmlog.Warn("antispam learning cannot start: the state database is not open")
return nil
}
engine := newLearnEngine(learn.EngineConfig{
Client: learn.Client{Controller: target.Controller, RspamdUID: target.RspamdUID},
Ledger: learn.Open(db),
Bodies: &learn.Bodies{Dir: learn.DefaultBodiesDir},
HashGeneration: target.HashGeneration,
Prefix: target.Prefix,
OnReportCommit: func() { learningMetricsSet().Committed.Inc() },
OnStateMismatch: func(detail string) {
csmlog.Warn("antispam learning stopped: its state does not match storage", "detail", detail)
},
OnUnstableAnchor: func(detail string) {
csmlog.Warn("antispam learning dropped a cluster: rspamd no longer hashes its message to the learned digest", "detail", detail)
},
})
runCtx, cancel := context.WithCancel(ctx)
r := &learnerRunner{target: target, watch: watch, engine: engine, cancel: cancel}
r.setGate(gate)
r.watcher = newRunnerWatcher(db, r)
r.wg.Add(2)
d.wg.Add(2)
obs.Go("antispam-learn", func() {
defer d.wg.Done()
defer r.wg.Done()
engine.Run(runCtx)
})
obs.Go("antispam-learnwatch", func() {
defer d.wg.Done()
defer r.wg.Done()
r.watcher.Run(runCtx)
})
d.antispamLearner.Store(r)
return r
}
// stopLearner stops the running engine, if any, and waits for it.
func (d *Daemon) stopLearner() {
d.antispamLearnerMu.Lock()
defer d.antispamLearnerMu.Unlock()
d.stopLearnerLocked()
}
func (d *Daemon) stopLearnerLocked() {
r := d.antispamLearner.Swap(nil)
if r == nil {
return
}
r.setGate(antispam.LearningGate{})
r.cancel()
r.wg.Wait()
}
// learnerProblems is the reconcile pass's view of the engine. Before an
// engine runs, a restored ledger awaiting confirmation already holds
// learning, so restored keys are never installed for a pass.
func (d *Daemon) learnerProblems() []antispam.Problem {
if db := store.Global(); db != nil {
held, err := learn.Open(db).DeclaredRestore()
if err != nil {
return []antispam.Problem{{Code: antispam.CodeLearning, Severity: antispam.Warning, Key: antispam.ReasonStateMismatch,
Detail: "cannot read restored learning state; learning stays off: " + err.Error()}}
}
if held {
return []antispam.Problem{{Code: antispam.CodeLearning, Severity: antispam.Warning, Key: antispam.ReasonStateMismatch,
Detail: "restored learning state awaits confirmation: disable learning, then enable it again, or run csm antispam learn --fresh-namespace"}}
}
}
if r := d.antispamLearner.Load(); r != nil {
problems := withWatcherCapacity(r.engine.Problems(), r.watcher)
// The last pass's registry error stands until a pass reads the
// registry; cPanel's has no finding.
if rep, _ := r.watcher.Last(); r.watch.postfix && rep.RegistryErr != nil {
problems = append(problems, antispam.DovecotProblem(rep.RegistryErr))
}
return problems
}
return nil
}
// withWatcherCapacity adds the accounts whose Spam folder state reached its
// allowance in the last pass to the engine's account_capacity Warning.
func withWatcherCapacity(problems []antispam.Problem, w learnWatcher) []antispam.Problem {
rep, _ := w.Last()
n := rep.Skipped[learn.SkipAccountCapacity]
if n == 0 {
return problems
}
detail := strconv.Itoa(n) + " accounts reached their Spam folder learning allowance in the last pass; their new reports wait"
for i, p := range problems {
if p.Key == antispam.ReasonAccountCapacity {
problems[i].Detail = p.Detail + "; " + detail
return problems
}
}
return append(problems, antispam.Problem{Code: antispam.CodeLearning, Severity: antispam.Warning, Key: antispam.ReasonAccountCapacity, Detail: detail})
}
// confirmRestoreIfDisabled treats learning seen disabled as the operator's
// confirmation of a restore: the next enable resumes the
// restored ledger. Observe mode and shutdown confirm nothing.
func (d *Daemon) confirmRestoreIfDisabled() {
cfg := d.currentCfg()
as := cfg.EmailProtection.Antispam
if (as.Enabled && as.Learning.Enabled) || cfg.ObserveMode() || d.scanContext().Err() != nil {
return
}
if db := store.Global(); db != nil {
if err := learn.Open(db).ClearDeclaredRestore(); err != nil {
csmlog.Warn("antispam learning: cannot confirm the restored learning state", "error", err)
}
}
}
package daemon
import (
"math"
"sync"
"time"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/antispam/learn"
"github.com/pidginhost/csm/internal/antispam/learnwatch"
"github.com/pidginhost/csm/internal/health"
"github.com/pidginhost/csm/internal/metrics"
)
// learningMetrics are the learner's series. No series carries an account,
// mailbox or UID label; skips are labeled by reason only.
type learningMetrics struct {
Admitted *metrics.Counter
Committed *metrics.Counter
Skips *metrics.CounterVec
Pending *metrics.Gauge
BacklogAge *metrics.Gauge
LastPass *metrics.Gauge
LiveClusters *metrics.Gauge
ZeroTargets *metrics.Gauge
BodyBytes *metrics.Gauge
Uncertain *metrics.Gauge
OldestUnchecked *metrics.Gauge
AuditCompleted *metrics.Gauge
Reconstructed *metrics.Gauge
}
var (
learningMetricsOnce sync.Once
learningMetricsInstance *learningMetrics
)
func learningMetricsSet() *learningMetrics {
learningMetricsOnce.Do(func() {
m := &learningMetrics{
Admitted: metrics.NewCounter("csm_antispam_learning_reports_admitted_total", "Spam folder reports the learner accepted."),
Committed: metrics.NewCounter("csm_antispam_learning_contributions_committed_total", "Cluster contributions committed from Spam folder reports."),
Skips: metrics.NewCounterVec("csm_antispam_learning_skips_total", "Spam folder entries and reports skipped, by reason.", []string{"reason"}),
Pending: metrics.NewGauge("csm_antispam_learning_pending_reports", "Reports waiting after the last pass; a lower bound when it was unfinished."),
BacklogAge: metrics.NewGauge("csm_antispam_learning_backlog_age_seconds", "Age of the oldest waiting report at the last pass."),
LastPass: metrics.NewGauge("csm_antispam_learning_last_pass_timestamp_seconds", "When the last Spam folder pass completed."),
LiveClusters: metrics.NewGauge("csm_antispam_learning_live_clusters", "Learned clusters not retired."),
ZeroTargets: metrics.NewGauge("csm_antispam_learning_zero_target_records", "Zero-target guard records."),
BodyBytes: metrics.NewGauge("csm_antispam_learning_body_bytes", "Bytes of stored anchor messages."),
Uncertain: metrics.NewGauge("csm_antispam_learning_uncertain_intents", "Anchor writes with an unknown outcome."),
OldestUnchecked: metrics.NewGauge("csm_antispam_learning_oldest_unchecked_timestamp_seconds", "Latest time every anchor not yet audited in this rotation was checked."),
AuditCompleted: metrics.NewGauge("csm_antispam_learning_audit_completed_timestamp_seconds", "When the anchor audit last completed a rotation."),
Reconstructed: metrics.NewGauge("csm_antispam_learning_reconstructed_clusters", "Anchors scheduled for reconstruction since the learner started."),
}
for name, c := range map[string]metrics.Collector{
"csm_antispam_learning_reports_admitted_total": m.Admitted,
"csm_antispam_learning_contributions_committed_total": m.Committed,
"csm_antispam_learning_skips_total": m.Skips,
"csm_antispam_learning_pending_reports": m.Pending,
"csm_antispam_learning_backlog_age_seconds": m.BacklogAge,
"csm_antispam_learning_last_pass_timestamp_seconds": m.LastPass,
"csm_antispam_learning_live_clusters": m.LiveClusters,
"csm_antispam_learning_zero_target_records": m.ZeroTargets,
"csm_antispam_learning_body_bytes": m.BodyBytes,
"csm_antispam_learning_uncertain_intents": m.Uncertain,
"csm_antispam_learning_oldest_unchecked_timestamp_seconds": m.OldestUnchecked,
"csm_antispam_learning_audit_completed_timestamp_seconds": m.AuditCompleted,
"csm_antispam_learning_reconstructed_clusters": m.Reconstructed,
} {
metrics.MustRegister(name, c)
}
learningMetricsInstance = m
})
return learningMetricsInstance
}
// recordLearningPass folds one completed pass into the series.
func recordLearningPass(r learnwatch.PassReport, now time.Time) {
m := learningMetricsSet()
m.Admitted.Add(float64(r.Submitted))
for reason, n := range r.Skipped {
m.Skips.With(reason).Add(float64(n))
}
m.Pending.Set(float64(r.Pending))
age := 0.0
if !r.OldestPending.IsZero() {
age = now.Sub(r.OldestPending).Seconds()
}
m.BacklogAge.Set(age)
m.LastPass.Set(float64(now.Unix()))
}
func unixOrZero(t time.Time) float64 {
if t.IsZero() {
return 0
}
return float64(t.Unix())
}
// recordLearningStats sets the engine-state series.
func recordLearningStats(s learn.Stats) {
m := learningMetricsSet()
m.LiveClusters.Set(float64(s.LiveClusters))
m.ZeroTargets.Set(float64(s.ZeroTargets))
m.BodyBytes.Set(float64(s.BodyBytes))
m.Uncertain.Set(float64(s.UncertainIntents))
m.OldestUnchecked.Set(unixOrZero(s.OldestUnchecked))
m.AuditCompleted.Set(unixOrZero(s.AuditCompleted))
m.Reconstructed.Set(float64(s.Reconstructed))
}
func invalidateLearningStats() {
m := learningMetricsSet()
for _, g := range []*metrics.Gauge{m.LiveClusters, m.ZeroTargets, m.BodyBytes, m.Uncertain, m.OldestUnchecked, m.AuditCompleted, m.Reconstructed} {
g.Set(math.NaN())
}
}
// LearningStatus is the learner's health for status output, paused or
// not; nil while learning does not run.
func (d *Daemon) LearningStatus() *health.LearningStatus {
r := d.antispamLearner.Load()
if r == nil {
return nil
}
rep, at := r.watcher.Last()
s := &health.LearningStatus{
Publishing: r.engine.Publishing(),
LastPass: at,
PassUnfinished: rep.Unfinished,
OldestPending: rep.OldestPending,
}
if !at.IsZero() && !rep.Unfinished {
n := rep.Pending
s.PendingReports = &n
}
if st, err := r.engine.Stats(); err == nil {
s.LiveClusters, s.ZeroTargets, s.UncertainIntents = &st.LiveClusters, &st.ZeroTargets, &st.UncertainIntents
s.BodyBytes, s.OldestUnchecked, s.AuditCompleted = &st.BodyBytes, st.OldestUnchecked, st.AuditCompleted
} else {
s.StateError = antispam.ReasonStateIO
}
return s
}
package daemon
import (
"context"
"slices"
"time"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/antispam/dovecot"
"github.com/pidginhost/csm/internal/antispam/learnwatch"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/store"
)
type learnWatcher interface {
Run(context.Context)
Pass(context.Context) learnwatch.PassReport
Last() (learnwatch.PassReport, time.Time)
}
// newLearnWatcher builds the Spam folder watcher; tests replace it.
var newLearnWatcher = func(cfg learnwatch.WatcherConfig) learnWatcher { return learnwatch.NewWatcher(cfg) }
// The account registry and global opt-in a cPanel watcher reads, and the
// Dovecot discovery a Postfix watcher reads; tests replace them.
var (
learnwatchSource learnwatch.Source = learnwatch.DefaultSource
learnwatchGlobal = learnwatch.GlobalOptInPath
learnwatchDovecot learnwatch.DovecotSource = dovecot.Dovecot{}
)
// resetLearningWatcher cancels unsent reports on explicit disable, even
// when the daemon started with learning off. Shutdown and observe retain
// the watcher state. The caller holds antispamLearnerMu and has stopped it.
func (d *Daemon) resetLearningWatcher() bool {
db := store.Global()
if db == nil {
return false
}
if err := db.ResetLearnWatch(); err != nil {
csmlog.Warn("antispam learning cannot reset Spam folder baselines", "err", err)
return false
}
d.antispamWatchReset = false
return true
}
// watchConfig is what a runner's watcher reads reporters by: Dovecot with
// a reporter mode and exempt list on Postfix, else cPanel's accounts. The
// daemon compares it apart from the learner target; a change restarts the
// learner.
type watchConfig struct {
postfix bool
reporter string
exempt []string
}
func (w watchConfig) equal(o watchConfig) bool {
return w.postfix == o.postfix && w.reporter == o.reporter && slices.Equal(w.exempt, o.exempt)
}
// watchConfigFor is the watcher a pass on stack asks for under cfg. A pass
// that cannot tell the stack, as while postfix restarts, keeps cur's.
func watchConfigFor(stack antispam.MailStack, cfg *config.Config, cur *learnerRunner) watchConfig {
switch {
case stack == antispam.StackPostfix:
as := cfg.EmailProtection.Antispam
return watchConfig{postfix: true, reporter: as.Learning.Reporter, exempt: slices.Clone(as.Exempt)}
case stack == antispam.StackNone && cur != nil:
return cur.watch
}
return watchConfig{}
}
// newRunnerWatcher builds r's watcher. It lists folders only while r's gate
// allows installs and reads or submits reports only while it also allows
// publishing; stopping r cancels it with the engine.
func newRunnerWatcher(db *store.DB, r *learnerRunner) learnWatcher {
cfg := learnwatch.WatcherConfig{
State: learnwatch.NewState(db, nil),
Engine: r.engine,
CanDiscover: func() bool { return r.currentGate().Install },
CanSubmit: func() bool {
g := r.currentGate()
return g.Install && g.Publish
},
OnPass: func(rep learnwatch.PassReport) {
recordLearningPass(rep, time.Now())
if st, err := r.engine.Stats(); err == nil {
recordLearningStats(st)
} else {
invalidateLearningStats()
}
},
}
if r.watch.postfix {
cfg.Postfix = &learnwatch.Postfix{Dovecot: learnwatchDovecot, Reporter: r.watch.reporter, Exempt: r.watch.exempt}
} else {
cfg.Source, cfg.GlobalPath = learnwatchSource, learnwatchGlobal
}
return newLearnWatcher(cfg)
}
package daemon
import (
"context"
"fmt"
"os"
"strings"
"syscall"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
)
// rspamdLogReopenInterval matches LogWatcher.Run: a log rotated by rename
// is picked up on the next reopen.
const rspamdLogReopenInterval = 5 * time.Minute
// rspamdLogLoop reads scan records from the rspamd log file the last
// reconcile pass named on a Postfix host, following that
// file when a pass names another.
func (d *Daemon) rspamdLogLoop(ctx context.Context) {
var tail rspamdLogTail
defer tail.stop()
poll := time.NewTicker(logWatcherPollInterval)
defer poll.Stop()
reopen := time.NewTicker(rspamdLogReopenInterval)
defer reopen.Stop()
for {
select {
case <-ctx.Done():
return
case <-reopen.C:
if tail.w != nil {
tail.w.reopen()
}
case <-poll.C:
d.followRspamdLog(&tail)
}
}
}
// rspamdLogTail is what rspamdLogLoop keeps between polls.
type rspamdLogTail struct {
w *LogWatcher
// missing is the log last found unreadable, so each outage of a log
// is reported once.
missing string
}
func (t *rspamdLogTail) stop() {
if t.w != nil {
t.w.Stop()
t.w = nil
}
}
// followRspamdLog reads what rspamd logged since the last poll, after
// switching to the log the last pass named, or stopping when there is none.
// A log CSM cannot open is tried again on the next poll.
func (d *Daemon) followRspamdLog(t *rspamdLogTail) {
var path string
if src := d.antispamScanSource.Load(); src != nil && d.currentCfg().EmailProtection.Antispam.Enabled {
path = src.Log
}
if t.w != nil && t.w.path != path {
t.stop()
}
if t.w != nil {
t.w.readNewLines()
return
}
if path == "" {
t.missing = ""
return
}
// Like every log CSM watches, a file is read from where it ends.
w, err := newLogWatcher(path, d.cfg, nil, d.alertCh, openRspamdLog)
if err != nil {
if t.missing != path {
t.missing = path
csmlog.Warn("rspamd log not readable, will retry", "path", path, "err", err)
}
return
}
if t.missing == path {
csmlog.Info("watching rspamd log (appeared after retry)", "path", path)
}
t.missing = ""
w.observed = d.handleRspamdLogLine
t.w = w
}
// openRspamdLog opens rspamd's log without following a symlink or blocking
// on a FIFO, and only as a regular file: rspamd's user owns the log
// directory and can put either there.
func openRspamdLog(path string) (*os.File, error) {
// #nosec G304 -- rspamd's log file as its configuration names it; no-follow and the regular-file check below bound the open.
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if err != nil {
return nil, err
}
info, err := f.Stat()
if err == nil && !info.Mode().IsRegular() {
err = fmt.Errorf("%s is not a regular file", path)
}
if err != nil {
_ = f.Close()
return nil, err
}
return f, nil
}
// handleRspamdLogLine counts a scan record in rspamd's log for scan health
// and the Email Security page. Any local process can have rspamd log a
// genuine record naming a sender of its choice, through rspamd's scan
// listeners, so a record is never evidence about that sender: no attack DB
// verdict, no CSM_IP weight and no spam source report.
func (d *Daemon) handleRspamdLogLine(line string, obs alert.Observation, _ *config.Config) []alert.Finding {
s, ok := antispam.ParseRspamdScan(line)
if !ok {
return nil
}
return d.recordScan(s, line, lineReadAt(obs))
}
// postfixMilterFailure reports a postfix smtpd warning that it could not
// connect to or read from one of milters, named as smtpd_milters names
// them. The warning must open smtpd's message: smtpd quotes client text
// such as HELO and RCPT only after its own words.
func postfixMilterFailure(line string, milters []string) bool {
_, msg := postfixSMTPDMessage(line)
if msg == "" {
return false
}
for _, m := range milters {
if strings.HasPrefix(msg, "warning: milter "+m+": ") || strings.HasPrefix(msg, "warning: connect to Milter service "+m+": ") {
return true
}
}
return false
}
package daemon
import (
"path/filepath"
"regexp"
"strings"
)
// atomicWriteStageRE matches the `.temp.<digits>.<rest>` filename pattern
// emitted by cPanel's fileTransfer service and similar atomic-write
// helpers. The digits are a nanosecond timestamp; <rest> is the original
// basename that will be rename(2)d into place.
var atomicWriteStageRE = regexp.MustCompile(`^\.temp\.\d+\..+`)
// looksLikeAtomicWriteStage reports whether a base filename matches the
// atomic-write staging convention `.temp.<digits>.<name>`. Pass the
// basename, not the full path.
func looksLikeAtomicWriteStage(name string) bool {
return atomicWriteStageRE.MatchString(name)
}
// atomicWriteRenameCandidate maps cPanel's .temp.<timestamp>.<name> staging
// path to its intended final path. It is only a location hint: the probe must
// still validate content or the destination identity before trusting it.
func atomicWriteRenameCandidate(path string) string {
base := filepath.Base(path)
if !looksLikeAtomicWriteStage(base) {
return ""
}
rest := strings.TrimPrefix(base, ".temp.")
dot := strings.IndexByte(rest, '.')
if dot < 0 || dot == len(rest)-1 {
return ""
}
name := rest[dot+1:]
if name == "." || name == ".." {
return ""
}
return filepath.Join(filepath.Dir(path), name)
}
// atomicWriteContentPath supplies a type and checksum lookup hint without
// changing the descriptor or the reported location of the file being scanned.
func atomicWriteContentPath(path string) string {
if candidate := atomicWriteRenameCandidate(path); candidate != "" {
return candidate
}
return path
}
package daemon
import (
"fmt"
"net"
"os"
"github.com/pidginhost/csm/internal/attackdb"
"github.com/pidginhost/csm/internal/netutil"
)
func (d *Daemon) prepareAttackDatabase(adb *attackdb.DB) {
d.registerQueueSource("attackdb", adb)
// Seed from permanent blocklist on first run (when attack DB is empty)
if adb.TotalIPs() == 0 {
if n := adb.SeedFromPermanentBlocklist(d.cfg.StatePath); n > 0 {
fmt.Fprintf(os.Stderr, "[%s] Attack DB seeded %d IPs from permanent blocklist\n", ts(), n)
}
}
if n := forgetInfraAttackRecords(adb, d.cfg.InfraIPs); n > 0 {
_ = adb.Flush()
fmt.Fprintf(os.Stderr, "[%s] Attack DB dropped %d records inside infra_ips\n", ts(), n)
}
fmt.Fprintf(os.Stderr, "[%s] Attack DB initialized (%s)\n", ts(), adb.FormatTopLine())
}
// forgetInfraAttackRecords drops the records of addresses inside infra_ips.
// An address listed after it was recorded would otherwise stay ranked as an
// attacker until its record expires. Hostname entries are skipped: the checks
// match only literal addresses and CIDRs, so a resolved address is still
// recorded and dropping it here would only hide it until the next finding.
func forgetInfraAttackRecords(adb *attackdb.DB, infraIPs []string) int {
var nets []*net.IPNet
for _, entry := range infraIPs {
if n := netutil.ParseCIDROrIP(entry); n != nil {
nets = append(nets, n)
}
}
return adb.ForgetNetworks(nets)
}
package daemon
import (
"context"
"fmt"
"net"
"os/exec"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
)
const (
// cpdoveauthdSocketPath is the cPanel dovecot auth daemon's unix socket.
cpdoveauthdSocketPath = "/usr/local/cpanel/var/cpdoveauthd.sock"
// mailAuthProbeInterval is how often the prober dials the socket.
mailAuthProbeInterval = 20 * time.Second
// mailAuthRestartCooldown is the minimum gap between restart attempts.
mailAuthRestartCooldown = 2 * time.Minute
)
// dialMailAuthBackend probes the cpdoveauthd unix socket. A successful connect
// means the auth backend can answer; a refused/missing socket means it is down.
func dialMailAuthBackend() bool {
c, err := net.DialTimeout("unix", cpdoveauthdSocketPath, 3*time.Second)
if err != nil {
return false
}
_ = c.Close()
return true
}
// restartMailAuthBackend runs the operator-configured restart command.
func restartMailAuthBackend(command string) error {
if strings.TrimSpace(command) == "" {
return fmt.Errorf("restart command is empty")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
// Run through the shell so operators can use normal service commands with
// arguments, such as "systemctl restart dovecot".
// #nosec G204 -- command is operator-configured in root-owned csm.yaml.
return exec.CommandContext(ctx, "/bin/sh", "-c", command).Run()
}
// authBackendHealth tracks reachability of the mail authentication backend
// (cPanel's cpdoveauthd unix socket) via an active probe. While the backend is
// down every mail and SMTP login fails regardless of credentials, so callers
// gate brute-force auto-block on Degraded() to avoid mass-blocking legitimate
// users. chkservd only checks dovecot's listen ports, which stay up during a
// cpdoveauthd outage, so this probe covers a failure mode cPanel's own
// monitoring misses. When restartEnabled, a sustained outage (continuously down
// for at least downGrace) triggers a rate-limited service restart to self-heal.
type authBackendHealth struct {
mu sync.Mutex
now func() time.Time
probe func() bool // true = backend reachable
restart func() error // run the configured restart command
restartEnabled bool
downGrace time.Duration
restartCooldown time.Duration
maxRestartsPerHour int
downSince time.Time // zero == backend currently up
alerted bool // down alert already emitted for the current outage
lastRestart time.Time
restartsThisHour int
hourKey string
}
func newAuthBackendHealth(
now func() time.Time,
probe func() bool,
restart func() error,
restartEnabled bool,
downGrace time.Duration,
restartCooldown time.Duration,
maxRestartsPerHour int,
) *authBackendHealth {
if now == nil {
now = time.Now
}
return &authBackendHealth{
now: now,
probe: probe,
restart: restart,
restartEnabled: restartEnabled,
downGrace: downGrace,
restartCooldown: restartCooldown,
maxRestartsPerHour: maxRestartsPerHour,
}
}
// Degraded reports whether the mail auth backend is currently unreachable.
// Brute-force trackers consult this and suppress auto-block while it is true.
func (h *authBackendHealth) Degraded() bool {
h.mu.Lock()
defer h.mu.Unlock()
return !h.downSince.IsZero()
}
// Observe runs one probe cycle and returns any findings to dispatch: a one-shot
// down alert when an outage starts, and a restart action once the outage is
// sustained past the grace period (subject to cooldown and the hourly cap).
func (h *authBackendHealth) Observe() []alert.Finding {
now := h.now()
reachable := h.probe != nil && h.probe()
h.mu.Lock()
if reachable {
h.downSince = time.Time{}
h.alerted = false
h.mu.Unlock()
return nil
}
var out []alert.Finding
var runRestart bool
var restart func() error
if h.downSince.IsZero() {
h.downSince = now
}
if !h.alerted {
h.alerted = true
out = append(out, alert.Finding{
Severity: alert.Warning,
Check: "mail_auth_backend_degraded",
Message: "Mail auth backend (cpdoveauthd) unreachable; mail and SMTP brute-force auto-block paused",
Details: "CSM could not connect to the cPanel dovecot auth socket. Logins fail regardless of password. Dovecot's listen ports stay up during this fault, so chkservd does not catch it; investigate the auth daemon.",
Timestamp: now,
})
}
if h.restartEnabled && h.restart != nil && now.Sub(h.downSince) >= h.downGrace && h.allowRestart(now) {
h.lastRestart = now
h.restartsThisHour++
restart = h.restart
runRestart = true
}
h.mu.Unlock()
if runRestart {
if err := restart(); err != nil {
out = append(out, alert.Finding{
Severity: alert.High,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-RESTART failed: mail auth backend down >%v and restart errored", h.downGrace),
Details: fmt.Sprintf("Error: %v. Manual intervention required to restore mail authentication.", err),
Timestamp: now,
})
} else {
out = append(out, alert.Finding{
Severity: alert.Warning,
Check: "auto_response",
Message: fmt.Sprintf("AUTO-RESTART: mail auth backend down >%v, restarted the mail service", h.downGrace),
Details: "cpdoveauthd was unreachable beyond the grace period; CSM restarted the mail service to recover authentication.",
Timestamp: now,
})
}
}
return out
}
// allowRestart reports whether a restart may run now, enforcing the per-hour cap
// and the cooldown between attempts. Caller must hold h.mu.
func (h *authBackendHealth) allowRestart(now time.Time) bool {
key := now.Format("2006-01-02T15")
if h.hourKey != key {
h.hourKey = key
h.restartsThisHour = 0
}
if h.maxRestartsPerHour <= 0 {
return false
}
if h.restartsThisHour >= h.maxRestartsPerHour {
return false
}
if !h.lastRestart.IsZero() && now.Sub(h.lastRestart) < h.restartCooldown {
return false
}
return true
}
package daemon
import (
"os"
"sync"
"github.com/pidginhost/csm/internal/integrity"
)
// hashBinary hashes the daemon binary; a var so tests can count reads.
var hashBinary = integrity.HashFile
// binaryHashCache keeps the running binary's hash until the file changes,
// so the status API does not read the whole binary on every poll.
type binaryHashCache struct {
mu sync.Mutex
key string
hash string
}
func (c *binaryHashCache) get(path string) string {
before, err := os.Stat(path)
if err != nil {
return ""
}
key := integrity.FileChangeKey(before)
c.mu.Lock()
defer c.mu.Unlock()
if c.hash != "" && key == c.key {
return c.hash
}
h, err := hashBinary(path)
if err != nil {
return ""
}
// A file replaced while it was read gives a hash of neither version
// for certain; report it but do not keep it.
if after, err := os.Stat(path); err != nil || integrity.FileChangeKey(after) != key {
return h
}
c.key, c.hash = key, h
return h
}
package daemon
import (
"fmt"
"net/netip"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/blockdigest"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/store"
)
var (
blockDigestSendEmail = alert.SendEmail
blockDigestSendWebhookJSON = alert.SendWebhookJSON
)
// buildBlockDigest constructs the collector that sends the daily alert
// summary; with block_digest enabled it also watches blocks by country. Built
// once at startup; block_digest is hotreload:"restart" so its settings do not
// change under a live daemon.
func (d *Daemon) buildBlockDigest(cfg *config.Config) *blockdigest.Collector {
watch := cfg.Alerts.BlockDigest.Enabled
configuredCountries := blockdigest.ResolveCountries(cfg.Alerts.BlockDigest.Countries, nil)
countries := blockdigest.ResolveCountries(cfg.Alerts.BlockDigest.Countries, cfg.Suppressions.TrustedCountries)
if watch && len(countries) == 0 {
csmlog.Warn("block_digest enabled with no countries; watching ALL countries (set alerts.block_digest.countries or suppressions.trusted_countries)")
}
var countriesOf func() []string
if len(configuredCountries) == 0 {
countriesOf = func() []string {
live := d.currentCfg()
if live == nil {
live = cfg
}
return blockdigest.ResolveCountries(nil, live.Suppressions.TrustedCountries)
}
}
email, webhook := d.blockDigestSinks(cfg)
collector := blockdigest.New(blockdigest.Options{
Watch: watch,
Countries: countries,
SendOn: cfg.Alerts.BlockDigest.SendOn,
Interval: cfg.BlockDigestInterval(),
Live: cfg.Alerts.BlockDigest.Live,
LiveChannel: cfg.Alerts.BlockDigest.Channel,
MinBlock: cfg.Alerts.BlockDigest.MinBlock,
Host: cfg.Hostname,
Version: d.version,
CountriesOf: countriesOf,
CountryOf: d.countryOf,
EnrichModSec: d.modsecEnricher(cfg),
EmailSink: email,
WebhookSink: webhook,
LoadSummary: func() ([]byte, error) {
if db := store.Global(); db != nil {
return db.LoadDailySummary()
}
return nil, nil
},
SaveSummary: func(b []byte) error {
if db := store.Global(); db != nil {
return db.SaveDailySummary(b)
}
return nil
},
OnError: func(channel string, err error) {
if channel == "summary" {
csmlog.Warn("daily summary state not loaded or saved", "err", err)
return
}
csmlog.Warn("block_digest delivery failed", "channel", channel, "err", err)
},
DeliveryEnabled: func(event, channel string) bool {
live := d.currentCfg()
if live == nil {
live = cfg
}
if channel == "email" {
return live.Alerts.Email.Enabled
}
return live.Alerts.Webhook.Enabled && !phpanelSkips(live, event)
},
})
d.registerQueueSource("block_digest", collector)
return collector
}
// blockDigestSinks builds both destinations for the daily summary. The
// collector applies block_digest.channel only to live block alerts.
func (d *Daemon) blockDigestSinks(cfg *config.Config) (func(subject, body string) error, func(blockdigest.WebhookPayload) error) {
ch := cfg.Alerts.BlockDigest.Channel
currentCfg := func() *config.Config {
if live := d.currentCfg(); live != nil {
return live
}
return cfg
}
emailSink := func(requireEnabled bool) func(subject, body string) error {
return func(subject, body string) error {
live := currentCfg()
if !live.Alerts.Email.Enabled {
if requireEnabled {
return fmt.Errorf("email alerts disabled")
}
return nil
}
return blockDigestSendEmail(live, subject, body)
}
}
webhookSink := func(requireEnabled bool) func(blockdigest.WebhookPayload) error {
return func(p blockdigest.WebhookPayload) error {
live := currentCfg()
if !live.Alerts.Webhook.Enabled {
if requireEnabled {
return fmt.Errorf("webhook alerts disabled")
}
return nil
}
// Delivery policy already skips these; a reload to phpanel
// between that check and this send must not post one either.
if phpanelSkips(live, p.CSM.Event) {
return nil
}
return blockDigestSendWebhookJSON(live, p)
}
}
return emailSink(ch == "email"), webhookSink(ch == "webhook")
}
// phpanelSkips reports a collector event a phpanel webhook must not receive:
// that endpoint takes signed findings, not CSM's own summary notifications.
func phpanelSkips(cfg *config.Config, event string) bool {
if cfg.Alerts.Webhook.Type != "phpanel" {
return false
}
switch event {
case "daily_summary", "customer_block":
return true
}
return false
}
// countryOf resolves an IP to its ISO country via the loaded GeoIP mmdb. It
// reads through the atomic accessor so it never races the geoip hot-reload swap.
func (d *Daemon) countryOf(ip string) string {
db := getGeoIPDB()
if db == nil {
return ""
}
return db.Lookup(ip).Country
}
// appliedBlockReason returns the reason of an applied auto-block finding, or
// "" for any other finding. PERMBLOCK promotions carry no reason.
func appliedBlockReason(f alert.Finding) string {
if f.Check != "auto_block" || f.Severity != alert.Critical {
return ""
}
return detailsReason(f)
}
// detailsReason is the reason an auto_block notice's details give, without
// the Cloudflare coverage warning.
func detailsReason(f alert.Finding) string {
reason := strings.TrimPrefix(f.Details, "Reason: ")
return strings.TrimSuffix(reason, " (warning: "+firewall.CloudflareCoverageWarning+")")
}
// summaryBlockReason is what the daily summary groups an applied block by.
// Netblock and permanent-block notices carry no reason; their message names
// the kind.
func summaryBlockReason(f alert.Finding) string {
if reason := appliedBlockReason(f); reason != "" {
return reason
}
if f.Check == "auto_block" && f.Severity == alert.Critical {
return f.Message
}
return ""
}
// observeBlocks feeds real auto-block findings to the digest collector,
// then sends one alert naming those that hit known customers. PERMBLOCK
// promotions (no reason details) and non-auto_block findings are skipped by
// the digest; dedup-by-IP in the collector makes multiple call sites safe.
// Each applied block reaches it once, so it alerts once per block.
func (d *Daemon) observeBlocks(actions []alert.Finding) {
if d.blockDigest == nil {
return
}
blocks := d.customerBlocks(actions)
customers := make(map[string]bool, len(blocks))
for _, b := range blocks {
customers[b.Target] = true
}
for _, f := range actions {
reason := appliedBlockReason(f)
if reason == "" {
continue
}
ip := checks.ExtractIPFromFinding(f)
if ip == "" {
continue
}
if customers[ip] {
d.blockDigest.ObserveKnownCustomer(ip, reason, f.Timestamp)
} else {
d.blockDigest.Observe(ip, reason, f.Timestamp)
}
}
d.blockDigest.AlertCustomerBlocks(blocks)
}
// maxCustomerExamples is how many known customers one subnet block names.
const maxCustomerExamples = 5
// blockTarget returns the kind of an auto_block notice (AUTO-BLOCK,
// AUTO-PERMBLOCK, AUTO-BLOCK-SUBNET or AUTO-NETBLOCK), the address or
// network it names, and whether it is a dry-run verdict, which puts
// "[dry-run]" after the kind.
func blockTarget(f alert.Finding) (kind, target string, dryRun bool) {
fields := strings.Fields(f.Message)
switch {
case len(fields) >= 3 && fields[1] == "[dry-run]:":
kind, target, dryRun = fields[0], fields[2], true
case len(fields) >= 2 && strings.HasSuffix(fields[0], ":"):
kind, target = strings.TrimSuffix(fields[0], ":"), fields[1]
default:
return "", "", false
}
switch kind {
case "AUTO-BLOCK", "AUTO-PERMBLOCK":
return kind, checks.ExtractIPFromFinding(f), dryRun
case "AUTO-BLOCK-SUBNET", "AUTO-NETBLOCK":
return kind, target, dryRun
}
return "", "", false
}
// customersAt returns the known customers target covers: an address's own
// last login, or the count and the most recent few inside a network.
func (d *Daemon) customersAt(target string, omit func(blockdigest.CustomerHit) bool) (blockdigest.CustomerBlock, bool) {
if p, err := netip.ParsePrefix(target); err == nil {
// The set keys IPv4 customers unmapped.
if p.Addr().Is4In6() && p.Bits() >= 96 {
p = netip.PrefixFrom(p.Addr().Unmap(), p.Bits()-96)
}
p = p.Masked()
n, sample := d.knownCustomers.Within(p, maxCustomerExamples, omit)
return blockdigest.CustomerBlock{Target: p.String(), Hits: sample, More: n - len(sample)}, n > 0
}
hit, ok := d.knownCustomers.Lookup(target)
if ok && omit != nil && omit(hit) {
ok = false
}
return blockdigest.CustomerBlock{Target: target, Hits: []blockdigest.CustomerHit{hit}}, ok
}
// customerBlock fills in the reason and check of a block notice that hit
// known customers. Netblock and permanent-block notices carry no reason;
// their message says what happened.
func (d *Daemon) customerBlock(f alert.Finding, target string, omit func(blockdigest.CustomerHit) bool) (blockdigest.CustomerBlock, bool) {
b, ok := d.customersAt(target, omit)
if !ok {
return b, false
}
b.Reason = detailsReason(f)
if b.Reason == "" {
b.Reason = f.Message
}
if f.Cause != nil {
b.Check = f.Cause.Check
}
return b, true
}
// blockCauses keeps the login-cause findings dispatch has seen, by finding
// ID, while the login they name can still be a known customer, so a block
// that answers one in a later batch (a queued retry, an incident or a
// central block) leaves that login out as a same-batch block does.
// ponytail: memory only; after a restart a block of an earlier cause alerts.
type blockCauses struct {
mu sync.Mutex
byID map[string]blockCause
}
// blockCause is a cause and when dispatch saw it: a finding may carry no
// timestamp of its own.
type blockCause struct {
f alert.Finding
seen time.Time
}
func (c *blockCauses) note(f alert.Finding, now time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
for id, old := range c.byID {
if knownCustomerExpired(old.seen, now) {
delete(c.byID, id)
}
}
if c.byID == nil {
c.byID = make(map[string]blockCause)
}
c.byID[alert.FindingID(f)] = blockCause{f, now}
}
func (c *blockCauses) lookup(id string, now time.Time) (alert.Finding, bool) {
c.mu.Lock()
defer c.mu.Unlock()
cause, ok := c.byID[id]
return cause.f, ok && !knownCustomerExpired(cause.seen, now)
}
// customerBlocks returns the applied automatic blocks in actions that hit
// known customers. A permanent promotion joins the block of its address from
// the same batch, which the chokepoint emits first. A customer whose login is
// the block's own cause, from this batch or an earlier one, is left out
// (ownLogin).
func (d *Daemon) customerBlocks(actions []alert.Finding) []blockdigest.CustomerBlock {
var blocks []blockdigest.CustomerBlock
byTarget := make(map[string]int)
now := time.Now()
for _, f := range actions {
if f.Check == "mail_account_compromised" || f.Check == "ftp_login_after_bruteforce" {
d.blockCauses.note(f, now)
}
}
for _, f := range actions {
if f.Check != "auto_block" || f.Severity != alert.Critical {
continue
}
kind, target, _ := blockTarget(f)
if i, ok := byTarget[target]; ok && kind == "AUTO-PERMBLOCK" {
blocks[i].Reason += " (promoted to permanent block)"
continue
}
var omit func(blockdigest.CustomerHit) bool
// Range escalations retain the original finding ID without its check.
if cause, found := d.blockCauses.lookup(causeID(f), now); found && (f.Cause.Check == "" || cause.Check == f.Cause.Check) {
omit = func(h blockdigest.CustomerHit) bool { return ownLogin(cause, h) }
}
b, ok := d.customerBlock(f, target, omit)
if !ok {
continue
}
byTarget[target] = len(blocks)
blocks = append(blocks, b)
}
return blocks
}
func causeID(f alert.Finding) string {
if f.Cause == nil {
return ""
}
return f.Cause.FindingID
}
// ownLogin reports whether hit is the login cause reports: a mail account
// compromise or an FTP login after brute force from the same address or IPv6
// /64, through the same kind of login, as the same mailbox or FTP account.
// The cause's Critical finding already names that login.
func ownLogin(cause alert.Finding, hit blockdigest.CustomerHit) bool {
key, ok := customerKey(cause.SourceIP)
if !ok || customerHitIP(key) != hit.IP {
return false
}
switch cause.Check {
case "mail_account_compromised":
switch hit.Kind {
case knownCustomerSMTP, knownCustomerIMAP, knownCustomerPOP:
return hit.Account == normalizeSprayMailbox(cause.Mailbox)
}
case "ftp_login_after_bruteforce":
return hit.Kind == knownCustomerFTP && hit.Account == normalizeSprayMailbox(ftpCustomerAccount(cause))
}
return false
}
// wouldBlock returns the verdict in a summary finding that would have
// blocked known customers: a dry-run auto_block notice, or a spray verdict
// in observe mode. The established-source advisory of the same check is
// not a verdict.
func (d *Daemon) wouldBlock(check string, f alert.Finding) (blockdigest.CustomerBlock, bool) {
switch {
case check == "auto_block":
if _, target, dryRun := blockTarget(f); dryRun {
return d.customerBlock(f, target, nil)
}
case check == "mail_bruteforce_suspected" && strings.Contains(f.Message, "(observe mode, not blocked)"):
b, ok := d.customersAt(f.SourceIP, nil)
b.Reason, b.Check = f.Message, check
return b, ok
}
return blockdigest.CustomerBlock{}, false
}
// noteSummary is the alert summary hook: it counts the summary-class findings
// of one dispatch for the daily summary and lists the verdicts that would
// have blocked known customers. It runs inside dispatch, so it never panics
// into it.
func (d *Daemon) noteSummary(cfg *config.Config, summary []alert.Finding) {
collector := d.blockDigest
if collector == nil || len(summary) == 0 {
return
}
defer func() {
if r := recover(); r != nil {
csmlog.Warn("daily summary dropped findings", "panic", r)
}
}()
muted := make(map[string]bool, len(cfg.Alerts.Email.DisabledChecks))
for _, name := range cfg.Alerts.Email.DisabledChecks {
muted[config.CanonicalCheckName(strings.TrimSpace(name))] = true
}
batch := make([]blockdigest.Handled, 0, len(summary))
for _, f := range summary {
check := config.CanonicalCheckName(f.Check)
h := blockdigest.Handled{
Check: check,
Severity: f.Severity.String(),
IP: checks.ExtractIPFromFinding(f),
BlockReason: summaryBlockReason(f),
Muted: muted[check],
}
if b, ok := d.wouldBlock(check, f); ok {
h.WouldBlock = &b
}
batch = append(batch, h)
}
collector.NoteHandled(batch)
}
package daemon
import (
"context"
"net/http"
"path/filepath"
"sync"
"time"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/threatintel"
)
var (
botRangesRefreshTotal *metrics.CounterVec
botRangesPrefixes *metrics.GaugeVec
botRangesLastSuccess *metrics.Gauge
botRangesMetricsOnce sync.Once
botRangesPrefixMu sync.Mutex
botRangesPrefixBots = map[string]struct{}{}
)
// botRangesMetrics lazily registers and returns the AI-crawler range-updater
// metrics, mirroring the signature/geoip update metrics: a refresh
// success/failure counter, a per-bot prefix-count gauge, and the timestamp of
// the last successful refresh.
func botRangesMetrics() (*metrics.CounterVec, *metrics.GaugeVec, *metrics.Gauge) {
botRangesMetricsOnce.Do(func() {
botRangesRefreshTotal = metrics.NewCounterVec(
"csm_botranges_refresh_total",
"AI-crawler IP-range refresh attempts, labelled by result (success when at least one vendor feed updated).",
[]string{"result"},
)
metrics.MustRegister("csm_botranges_refresh_total", botRangesRefreshTotal)
botRangesPrefixes = metrics.NewGaugeVec(
"csm_botranges_prefixes",
"Current number of published IP prefixes per AI-crawler identity in the active overlay.",
[]string{"bot"},
)
metrics.MustRegister("csm_botranges_prefixes", botRangesPrefixes)
botRangesLastSuccess = metrics.NewGauge(
"csm_botranges_last_success_timestamp_seconds",
"Unix timestamp of the last successful AI-crawler IP-range refresh.",
)
metrics.MustRegister("csm_botranges_last_success_timestamp_seconds", botRangesLastSuccess)
})
return botRangesRefreshTotal, botRangesPrefixes, botRangesLastSuccess
}
// setBotRangesPrefixGauges syncs the per-bot prefix gauge to the active overlay.
func setBotRangesPrefixGauges() {
_, prefixes, _ := botRangesMetrics()
snap := threatintel.FetchedRangesSnapshot()
botRangesPrefixMu.Lock()
defer botRangesPrefixMu.Unlock()
for bot := range botRangesPrefixBots {
if _, ok := snap[bot]; !ok {
prefixes.With(bot).Set(0)
delete(botRangesPrefixBots, bot)
}
}
for bot, nets := range snap {
prefixes.With(bot).Set(float64(len(nets)))
botRangesPrefixBots[bot] = struct{}{}
}
}
// observeBotRangesRefresh records the outcome of a refresh attempt. On success
// it bumps the prefix gauges and last-success timestamp; on failure it only
// increments the failure counter so the previous overlay's gauges stand.
func observeBotRangesRefresh(success bool) {
total, _, lastSuccess := botRangesMetrics()
if !success {
total.With("failure").Inc()
return
}
total.With("success").Inc()
setBotRangesPrefixGauges()
ts := threatintel.LastFetchedRangesRefresh()
if ts.IsZero() {
ts = time.Now()
}
lastSuccess.Set(float64(ts.Unix()))
}
func (d *Daemon) botRangesCachePath() string {
return filepath.Join(d.cfg.StatePath, "botranges.json")
}
// reloadBotRanges republishes the on-disk overlay into the running daemon. It
// is the botranges.reload control handler's worker: `csm update-bot-ranges`
// fetches fresh ranges, writes the cache, then asks the daemon to reload so the
// new ranges take effect without a restart.
func (d *Daemon) reloadBotRanges() error {
if err := threatintel.LoadFetchedRangesRequired(d.botRangesCachePath()); err != nil {
observeBotRangesRefresh(false)
return err
}
observeBotRangesRefresh(true)
return nil
}
// botRangesUpdater periodically refreshes the published AI-crawler IP ranges
// (OpenAI, Perplexity) so GPTBot/ChatGPT-User/OAI-SearchBot/PerplexityBot stay
// verifiable by address without a new release. Embedded snapshots cover the gap
// before the first refresh and whenever a fetch fails.
func (d *Daemon) botRangesUpdater() {
defer d.wg.Done()
if !d.cfg.BotRangesAutoUpdate() {
return
}
cachePath := d.botRangesCachePath()
if err := threatintel.LoadFetchedRanges(cachePath); err != nil {
csmlog.Warn("bot-ranges cache load failed", "err", err)
} else {
// Reflect the cached overlay and its persisted refresh time in the
// metrics before the first live refresh, so a restart does not show the
// ranges as never-refreshed.
setBotRangesPrefixGauges()
if ts := threatintel.LastFetchedRangesRefresh(); !ts.IsZero() {
_, _, lastSuccess := botRangesMetrics()
lastSuccess.Set(float64(ts.Unix()))
}
}
interval := 24 * time.Hour
if s := d.cfg.Reputation.BotRanges.UpdateInterval; s != "" {
if v, err := time.ParseDuration(s); err == nil && v >= time.Hour {
interval = v
}
}
select {
case <-d.stopCh:
return
case <-time.After(5 * time.Minute):
}
d.doBotRangesUpdate(cachePath)
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
d.doBotRangesUpdate(cachePath)
}
}
}
func (d *Daemon) doBotRangesUpdate(cachePath string) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
// Cancel an in-flight fetch promptly on daemon shutdown.
go func() {
select {
case <-d.stopCh:
cancel()
case <-ctx.Done():
}
}()
client := &http.Client{Timeout: 30 * time.Second}
n, err := threatintel.RefreshFetchedRanges(ctx, client, threatintel.DefaultRangeSources(), cachePath)
if err != nil {
csmlog.Warn("bot-ranges refresh error", "err", err)
}
observeBotRangesRefresh(n > 0)
if n > 0 {
csmlog.Info("bot-ranges refreshed", "bots", n)
}
}
package daemon
// PolicyMapPayload is the wire-shape of struct policy_state in the BPF
// program. Mirrors the C layout exactly: three uint32 fields, no
// alignment surprises (all fields are 4 bytes).
type PolicyMapPayload struct {
Enforce uint32
DryRun uint32
ProtectedPorts uint32
}
// policyMapPayload converts the userspace policy into the wire shape
// the BPF program reads. Pure function; tests drive it without loading
// any actual BPF program.
func policyMapPayload(pol BPFEnforcementPolicy) PolicyMapPayload {
return PolicyMapPayload{
Enforce: pol.Enforce,
DryRun: pol.DryRun,
// #nosec G115 -- bounded by protected_ports BPF map max_entries=16
ProtectedPorts: uint32(len(pol.Ports)),
}
}
// staleSafeUIDs returns the installed hosted UIDs (>= 1000) that next no
// longer exempts, so a deleted account's UID cannot stay exempt for whoever
// gets it next. System UIDs stay: hosted accounts never receive them, and
// keeping them means a bad passwd read cannot deny the host's mail daemons.
func staleSafeUIDs(installed []uint32, next map[uint32]bool) []uint32 {
var stale []uint32
for _, uid := range installed {
if uid >= 1000 && !next[uid] {
stale = append(stale, uid)
}
}
return stale
}
package daemon
import (
"sync"
"github.com/pidginhost/csm/internal/metrics"
)
// Decision label values match the BPF DECISION_* numeric codes:
// allow (0), dry_run (1), deny (2). Wire-stable strings; SIEM
// dashboards pin on them.
const (
BPFDecisionAllow = "allow"
BPFDecisionDryRun = "dry_run"
BPFDecisionDeny = "deny"
)
var (
bpfEnfMu sync.Mutex
bpfEnfDecisionsVec *metrics.CounterVec
bpfEnfUIDRefreshTotal *metrics.Counter
bpfEnfUIDRefreshFailures *metrics.Counter
)
// RegisterBPFEnforcementMetrics registers the counters on reg. Production
// callers pass metrics.Default(); tests pass metrics.NewRegistry() to
// keep registration isolated.
func RegisterBPFEnforcementMetrics(reg *metrics.Registry) {
bpfEnfMu.Lock()
defer bpfEnfMu.Unlock()
bpfEnfDecisionsVec = metrics.NewCounterVec(
"csm_bpf_enforcement_decisions_total",
"BPF cgroup-deny decisions by label (allow/dry_run/deny).",
[]string{"decision"},
)
reg.MustRegister("csm_bpf_enforcement_decisions_total", bpfEnfDecisionsVec)
bpfEnfUIDRefreshTotal = metrics.NewCounter(
"csm_bpf_enforcement_uid_map_refresh_total",
"BPF safe-UID map refresh successes.",
)
reg.MustRegister("csm_bpf_enforcement_uid_map_refresh_total", bpfEnfUIDRefreshTotal)
bpfEnfUIDRefreshFailures = metrics.NewCounter(
"csm_bpf_enforcement_uid_map_refresh_failures_total",
"BPF safe-UID map refresh failures.",
)
reg.MustRegister("csm_bpf_enforcement_uid_map_refresh_failures_total", bpfEnfUIDRefreshFailures)
}
// BumpBPFEnforcementDecision advances the per-decision counter. Called
// from the connection consumer when a ConnectionEvent with a decision
// field arrives. Unknown labels are silently ignored (caller bug).
func BumpBPFEnforcementDecision(label string) {
bpfEnfMu.Lock()
cv := bpfEnfDecisionsVec
bpfEnfMu.Unlock()
if cv == nil {
return
}
switch label {
case BPFDecisionAllow, BPFDecisionDryRun, BPFDecisionDeny:
cv.With(label).Inc()
}
}
// BumpUIDRefresh advances the periodic-refresh success counter.
func BumpUIDRefresh() {
bpfEnfMu.Lock()
c := bpfEnfUIDRefreshTotal
bpfEnfMu.Unlock()
if c != nil {
c.Inc()
}
}
// BumpUIDRefreshFailure advances the periodic-refresh failure counter.
func BumpUIDRefreshFailure() {
bpfEnfMu.Lock()
c := bpfEnfUIDRefreshFailures
bpfEnfMu.Unlock()
if c != nil {
c.Inc()
}
}
// resetBPFEnforcementMetricsForTest is a test seam.
func resetBPFEnforcementMetricsForTest() {
bpfEnfMu.Lock()
defer bpfEnfMu.Unlock()
bpfEnfDecisionsVec = nil
bpfEnfUIDRefreshTotal = nil
bpfEnfUIDRefreshFailures = nil
}
package daemon
import (
"bufio"
"os"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
)
// BPFEnforcementPolicy is the userspace-derived state to be loaded into
// the BPF policy + protected_ports + safe_uids maps. Pure data; no IO.
type BPFEnforcementPolicy struct {
Enforce uint32
DryRun uint32
Ports []uint16
}
// BuildBPFEnforcementPolicy translates config into the policy struct
// the BPF program consumes. Disabled config returns zero policy
// (Enforce=0); the in-kernel program then short-circuits to allow.
func BuildBPFEnforcementPolicy(cfg *config.Config) BPFEnforcementPolicy {
if cfg == nil || !cfg.BPFEnforcement.Enabled {
return BPFEnforcementPolicy{}
}
if !connectionTrackerAllowsBPF(cfg) {
return BPFEnforcementPolicy{}
}
p := BPFEnforcementPolicy{Enforce: 1}
if cfg.BPFEnforcementDryRunEnabled() {
p.DryRun = 1
}
if cfg.BPFEnforcement.DirectSMTPEgress && checks.DirectSMTPEgressBackendEnabled(cfg, "bpf") {
for _, port := range cfg.Detection.DirectSMTPEgress.Ports {
if port > 0 && port <= 65535 {
p.Ports = append(p.Ports, uint16(port))
}
}
}
if len(p.Ports) == 0 {
return BPFEnforcementPolicy{}
}
return p
}
func connectionTrackerAllowsBPF(cfg *config.Config) bool {
switch strings.ToLower(strings.TrimSpace(cfg.Detection.ConnectionTrackerBackend)) {
case "", "auto", "bpf":
return true
default:
return false
}
}
// safeUIDsFromPasswd returns a map of UIDs that should be exempt from
// in-kernel deny: UID 0 (root), UIDs <1000 (system accounts), any
// platform-known MTA users that happen to live above 1000 on some
// distros, and accounts listed in firewall.smtp_allow_users. Other hosted
// account UIDs (>=1000) are NOT in the safe map; their connections will be
// evaluated by the in-kernel deny path when enforcement is active.
//
// Caller passes a /etc/passwd path; production wiring uses /etc/passwd.
func safeUIDsFromPasswd(path string, cfg *config.Config) (map[uint32]bool, error) {
f, err := os.Open(path) // #nosec G304 -- caller-controlled; production passes /etc/passwd
if err != nil {
return nil, err
}
defer f.Close()
mta := platform.LocalMTAIdentities(platform.Detect())
out := map[uint32]bool{}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
fields := strings.Split(line, ":")
if len(fields) < 4 {
continue
}
user := fields[0]
uid64, err := strconv.ParseUint(fields[2], 10, 32)
if err != nil {
continue
}
uid := uint32(uid64)
// UID 0 always safe.
if uid == 0 {
out[uid] = true
continue
}
// System UIDs (<1000): always safe. Daemons, services,
// distro-managed users.
if uid < 1000 {
out[uid] = true
continue
}
// Hosted UIDs (>=1000): NOT safe by default. Exceptions:
// platform-known MTA users on distros that put them above 1000,
// and accounts the operator allows to send direct SMTP.
if mta.IsMTAUser(user, uid) || checks.SMTPAllowedUser(cfg, user) {
out[uid] = true
}
}
if err := scanner.Err(); err != nil {
return out, err
}
return out, nil
}
package daemon
import (
"sync"
"sync/atomic"
"time"
)
// UIDRefresherConfig drives a UIDRefresher. Refresh is the function
// called every tick; production wiring re-reads /etc/passwd and
// repopulates the BPF safe_uids map. Interval bounds the period;
// production uses 5 minutes.
type UIDRefresherConfig struct {
Interval time.Duration
Refresh func() error
}
// UIDRefresherStats is a counter snapshot.
type UIDRefresherStats struct {
Refreshes uint64
Failures uint64
}
// UIDRefresher runs the configured Refresh on a fixed interval. Stop
// is idempotent.
type UIDRefresher struct {
cfg UIDRefresherConfig
stop chan struct{}
wg sync.WaitGroup
started atomic.Bool
stopped atomic.Bool
refreshes atomic.Uint64
failures atomic.Uint64
}
// NewUIDRefresher returns a stopped refresher. Call Start to launch.
func NewUIDRefresher(cfg UIDRefresherConfig) *UIDRefresher {
return &UIDRefresher{cfg: cfg, stop: make(chan struct{})}
}
// Start launches the refresh goroutine. Idempotent.
func (r *UIDRefresher) Start() {
if r.started.Swap(true) {
return
}
r.wg.Add(1)
go r.loop()
}
// Stop signals the goroutine and waits. Safe to call multiple times.
func (r *UIDRefresher) Stop() {
if r.stopped.Swap(true) {
return
}
close(r.stop)
r.wg.Wait()
}
// Stats returns a counter snapshot.
func (r *UIDRefresher) Stats() UIDRefresherStats {
return UIDRefresherStats{
Refreshes: r.refreshes.Load(),
Failures: r.failures.Load(),
}
}
func (r *UIDRefresher) loop() {
defer r.wg.Done()
t := time.NewTicker(r.cfg.Interval)
defer t.Stop()
for {
select {
case <-r.stop:
return
case <-t.C:
if err := r.cfg.Refresh(); err != nil {
r.failures.Add(1)
continue
}
r.refreshes.Add(1)
}
}
}
package daemon
import (
"fmt"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
)
// emitBPFUnavailableFinding posts an operator-visible warning when a
// BPF-backed live monitor cannot start the kernel-attached path. fallback is
// the selected non-BPF backend; empty means no live fallback is active.
func emitBPFUnavailableFinding(alertCh chan<- alert.Finding, feature, choice, fallback string, err error) bool {
if alertCh == nil {
return false
}
sev := alert.Warning
if choice == bpf.BackendBPF || fallback == "" {
sev = alert.High
}
status := "no live fallback active"
if fallback != "" {
status = fmt.Sprintf("running on %s fallback", fallback)
}
details := ""
if err != nil {
details = err.Error()
}
f := alert.Finding{
Severity: sev,
Check: "bpf_unavailable",
Message: fmt.Sprintf("BPF backend unavailable for %s (operator choice=%q); %s", feature, choice, status),
Details: details,
Timestamp: time.Now(),
}
return alert.TryEnqueue(alertCh, f)
}
package daemon
import (
"context"
"errors"
"log"
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/reporting"
)
type centralActionConsumer struct {
stop <-chan struct{}
interval time.Duration
refresh func(context.Context) error
perform func(centralQueuedAction) error
queue *queuehealth.Channel[centralQueuedAction]
mu sync.Mutex
closed bool
}
func newCentralActionConsumer(stop <-chan struct{}, capacity int, interval time.Duration, refresh func(context.Context) error, perform func(centralQueuedAction) error) *centralActionConsumer {
return ¢ralActionConsumer{
stop: stop, interval: interval, refresh: refresh, perform: perform,
queue: queuehealth.NewChannel[centralQueuedAction](capacity, time.Minute),
}
}
func (c *centralActionConsumer) QueueStatuses(now time.Time) map[string]queuehealth.Status {
return map[string]queuehealth.Status{"actions": c.queue.Snapshot(now)}
}
func (c *centralActionConsumer) enqueue(a centralQueuedAction) bool {
c.mu.Lock()
defer c.mu.Unlock()
if c.closed {
c.queue.Lose(1)
return false
}
select {
case <-c.stop:
c.queue.Lose(1)
return false
default:
return c.queue.TrySend(a)
}
}
func (c *centralActionConsumer) run() {
ctx, cancel := context.WithCancel(context.Background())
canceled := make(chan struct{})
go func() {
defer close(canceled)
select {
case <-c.stop:
cancel()
case <-ctx.Done():
}
}()
var loggedLoss uint64
defer func() {
cancel()
<-canceled
// A dispatch may retain the old hook after it is uninstalled. Close
// admission with the producer lock before discarding pending work.
c.mu.Lock()
c.closed = true
c.queue.Close()
c.mu.Unlock()
c.queue.DiscardPending()
c.logDropped(&loggedLoss)
}()
if err := c.refresh(ctx); err != nil {
logCentralRefreshError("initial pull failed", err)
}
ticker := time.NewTicker(c.interval)
defer ticker.Stop()
for {
select {
case <-c.stop:
return
default:
}
select {
case <-c.stop:
return
case work := <-c.queue.Items():
select {
case <-c.stop:
work.Ticket.Reject(time.Now())
return
default:
}
c.process(work)
case <-ticker.C:
c.logDropped(&loggedLoss)
if err := c.refresh(ctx); err != nil {
logCentralRefreshError("refresh failed", err)
}
}
}
}
func (c *centralActionConsumer) process(work queuehealth.Work[centralQueuedAction]) {
work.Ticket.Start(time.Now())
settled := false
defer func() {
if !settled {
work.Ticket.Reject(time.Now())
}
}()
err := c.perform(work.Value)
if err != nil && !isCentralBlockRefusal(err) {
work.Ticket.Reject(time.Now())
} else {
work.Ticket.Finish(time.Now())
}
settled = true
if err != nil {
logCentralBlockFailure(work.Value.ip, err)
}
}
// centralUnauthorizedLog names what an operator can fix when the central
// service refuses the node.
const centralUnauthorizedLog = "central-intel: pull unauthorized; check node enrollment, node clock and reputation.central.auth_target"
// logCentralRefreshError logs a failed pull. A refusal of the node gets one
// fixed line; the node keeps its verified set and tries again next interval.
func logCentralRefreshError(what string, err error) {
if errors.Is(err, reporting.ErrPullUnauthorized) {
log.Print(centralUnauthorizedLog)
return
}
log.Printf("central-intel: %s: %v", what, err)
}
func (c *centralActionConsumer) logDropped(previous *uint64) {
total := c.queue.Snapshot(time.Now()).DroppedTotal
if n := total - *previous; n > 0 {
log.Printf("central-intel: %d action(s) incomplete", n)
*previous = total
}
}
package daemon
import (
"context"
"crypto/ed25519"
"encoding/hex"
"errors"
"log"
"net"
"os"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/reporting"
"github.com/pidginhost/csm/internal/threatintel"
)
const (
centralRefreshDefault = 6 * time.Hour
centralBlockThreshold = 60
centralChallengeTTL = 6 * time.Hour
centralBlockTTL = 24 * time.Hour
centralActionQueue = 1024
)
type centralQueuedAction struct {
findingID string
decision reporting.Decision
ip string
// root is the local finding's admission evidence the action answers.
root admission.Evidence
rootFinding alert.Finding
rootErr error
}
// documentationNets are reserved/non-routable ranges (RFC 5737 documentation,
// RFC 3849 IPv6 documentation, RFC 2544 benchmarking) that must never be acted
// on; they are not routable real attackers.
var documentationNets = mustCIDRs(
"192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24", "198.18.0.0/15", "2001:db8::/32",
)
// nonPublicNets are special-purpose ranges (IANA special-purpose registries)
// that net.IP's unicast and private predicates miss: "this network",
// carrier-grade NAT, IETF protocol assignments, the deprecated 6to4 relay
// anycast, the reserved block, deprecated IPv4-compatible and site-local IPv6,
// IPv6 discard-only and dummy sources, IETF IPv6 assignments, local-use
// translation and segment routing. None is a public source to report or act
// on; the central collector refuses them too.
var nonPublicNets = mustCIDRs(
"0.0.0.0/8", "100.64.0.0/10", "192.0.0.0/24", "192.88.99.0/24", "240.0.0.0/4",
"::/96", "fec0::/10", "100::/64", "100:0:0:1::/64", "2001::/23",
"64:ff9b:1::/48", "5f00::/16",
)
// teredoNet is carved out of 2001::/23: Teredo clients are real hosts that can
// send abusive traffic.
var teredoNet = mustCIDRs("2001::/32")[0]
// nat64Net and sixToFourNet embed an IPv4 source (RFC 6052, RFC 3056); the
// firebreak judges the embedded address.
var (
nat64Net = mustCIDRs("64:ff9b::/96")[0]
sixToFourNet = mustCIDRs("2002::/16")[0]
)
func mustCIDRs(cidrs ...string) []*net.IPNet {
out := make([]*net.IPNet, 0, len(cidrs))
for _, c := range cidrs {
if _, n, err := net.ParseCIDR(c); err == nil {
out = append(out, n)
}
}
return out
}
// centralRefresh refreshes store and logs each success with its version, so an
// operator can check from the log that a node's signed pulls work.
func centralRefresh(store *reporting.CentralStore) func(context.Context) error {
return func(ctx context.Context) error {
if err := store.Refresh(ctx); err != nil {
return err
}
// #nosec G706 -- version is uint64 formatted as decimal, with no log control characters.
log.Printf("central-intel: refresh complete (version=%d)", store.Version())
return nil
}
}
// startCentralConsume wires the central scored-set consumer: it pulls and
// verifies the signed set on an interval and installs alert.CentralHook so a
// finding whose IP is in the set is escalated per the configured action. It
// returns the refresh loop, or nil when disabled/misconfigured.
func (d *Daemon) startCentralConsume() func() {
alert.SetCentralHook(nil)
cc := d.cfg.Reputation.Central
if !cc.Enabled {
return nil
}
if cc.SetURL == "" {
log.Printf("central-intel: enabled but set_url is empty; consumer stays off")
return nil
}
pubHex := os.Getenv(cc.PubkeyEnv)
if raw, err := hex.DecodeString(pubHex); err != nil || len(raw) != ed25519.PublicKeySize {
log.Printf("central-intel: %s must hold a 64-hex-char Ed25519 public key; consumer stays off", cc.PubkeyEnv)
return nil
}
policy := reporting.ParseAction(cc.Action)
if cc.Action != "" && !reporting.IsValidAction(cc.Action) {
log.Printf("central-intel: unrecognized action %q, defaulting to challenge", cc.Action)
}
threshold := cc.BlockThreshold
if threshold <= 0 {
threshold = centralBlockThreshold
}
interval := centralRefreshDefault
if cc.RefreshInterval != "" {
if d2, err := time.ParseDuration(cc.RefreshInterval); err == nil && d2 > 0 {
interval = d2
}
}
puller := reporting.NewPuller(nil, cc.SetURL, pubHex)
if cc.AuthTarget != "" {
node, ok := centralSigner(cc.AuthTarget, d.cfg.Reputation.Report.Targets)
if !ok {
log.Printf("central-intel: reputation.central.auth_target %q has no usable Ed25519 node key; consumer stays off", cc.AuthTarget)
return nil
}
puller = reporting.NewSignedPuller(nil, cc.SetURL, pubHex, node, nil)
}
store := reporting.NewCentralStore(puller)
firebreak := d.centralFirebreak()
consumer := newCentralActionConsumer(d.stopCh, centralActionQueue, interval, centralRefresh(store), d.performCentralAction)
d.registerQueueSource("central", consumer)
alert.SetCentralHook(func(f alert.Finding) {
a, ok := d.planCentralAction(store, policy, threshold, firebreak, f)
if !ok {
return
}
consumer.enqueue(a)
})
log.Printf("central-intel: enabled (action=%s, threshold=%d, refresh=%s)", policy, threshold, interval)
return func() {
defer alert.SetCentralHook(nil)
consumer.run()
}
}
// applyCentral escalates a finding's IP when it appears in the central set. A
// finding firing on the IP is the node's local corroboration. Firebreaks and
// the action policy gate what happens; central data never blocks on its own.
func (d *Daemon) applyCentral(store *reporting.CentralStore, action reporting.Action, threshold int, firebreak func(string) bool, f alert.Finding) {
a, ok := d.planCentralAction(store, action, threshold, firebreak, f)
if !ok {
return
}
if err := d.performCentralAction(a); err != nil {
logCentralBlockFailure(a.ip, err)
}
}
func (d *Daemon) planCentralAction(store *reporting.CentralStore, action reporting.Action, threshold int, firebreak func(string) bool, f alert.Finding) (centralQueuedAction, bool) {
// The local finding is the corroboration, so it must name the address
// as an attacker. A response record, a health finding, the remote end
// of an outbound connection, a customer login or an advisory does not.
// A trusted proxy carries every visitor behind it, so it is never
// challenged or blocked for one of them.
ip := f.SourceIP
cfg := d.currentCfg()
if ip == "" || !checks.CheckResponseEnabled(cfg, f.Check) || !checks.AddressEvidence(f.Check, f.Severity) || checks.IsTrustedProxy(ip, cfg) {
return centralQueuedAction{}, false
}
entry, found := store.Lookup(ip)
dec := reporting.Decide(reporting.DecisionInput{
Found: found,
Score: entry.Score,
Protected: firebreak(ip),
LocallyCorroborated: true, // a finding fired on this IP
}, action, threshold)
if dec == reporting.DecisionIgnore {
return centralQueuedAction{}, false
}
root, rootErr := checks.PrepareAdmissionRoot(f)
return centralQueuedAction{decision: dec, ip: ip, findingID: alert.FindingID(f), root: root, rootFinding: f, rootErr: rootErr}, true
}
func (d *Daemon) performCentralAction(a centralQueuedAction) error {
check := a.root.Check()
if check == "" {
check = a.rootFinding.Check
}
cfg := d.currentCfg()
// A reload may have trusted this address while its action was queued.
if !checks.CheckResponseEnabled(cfg, check) || checks.IsTrustedProxy(a.ip, cfg) {
return nil
}
switch a.decision {
case reporting.DecisionChallenge:
if d.ipList != nil {
checks.AnswerPreparedRoot(admission.KindChallenge, a.ip, a.root, admission.EntryCentral, a.rootFinding, a.rootErr, centralChallengeTTL)
d.ipList.AddNonEscalating(a.ip, "central-intel", centralChallengeTTL)
}
case reporting.DecisionBlock:
res, err := checks.ApplyBlock(d.currentCfg(), checks.ApplyBlockRequest{
IP: a.ip,
EngineReason: centralIntelBlockReason,
Reason: centralIntelBlockReason,
TTL: centralBlockTTL,
Source: checks.BlockSourceCentral,
FindingID: a.findingID,
Root: a.root,
Entry: admission.EntryCentral,
RootFinding: a.rootFinding,
RootErr: a.rootErr,
})
d.recordAppliedBlocks(res.Findings)
if err != nil {
return err
}
log.Printf("central-intel: block %s outcome: %s", a.ip, res.Outcome)
}
return nil
}
func logCentralBlockFailure(ip string, err error) {
// Protected IPs are never blockable, a host without a firewall engine
// cannot block, and automatic blocking may be switched off; all are
// expected, not failures.
if isCentralBlockRefusal(err) {
return
}
log.Printf("central-intel: block %s failed: %v", ip, err)
}
func isCentralBlockRefusal(err error) bool {
return isProtectedIPRefusal(err) || errors.Is(err, checks.ErrNoIPBlocker) || errors.Is(err, checks.ErrAutoBlockDisabled) ||
errors.Is(err, checks.ErrBlockTakenByAdmission)
}
// centralFirebreak returns a predicate that reports whether an IP must never be
// acted on from central data or reported to it: anything not global unicast,
// private and other non-public ranges, documentation ranges, or an operator
// infra_ips entry.
func (d *Daemon) centralFirebreak() func(string) bool {
infraEntries := d.cfg.InfraIPs
if d.cfg.Firewall != nil {
infraEntries = mergeInfraIPs(d.cfg.InfraIPs, d.cfg.Firewall.InfraIPs)
}
var infra []*net.IPNet
for _, raw := range infraEntries {
if _, n, err := net.ParseCIDR(raw); err == nil {
infra = append(infra, n)
continue
}
if ip := net.ParseIP(raw); ip != nil {
bits := 32
if ip.To4() == nil {
bits = 128
}
infra = append(infra, &net.IPNet{IP: ip, Mask: net.CIDRMask(bits, bits)})
}
}
var protected func(ip net.IP) bool
protected = func(ip net.IP) bool {
if !ip.IsGlobalUnicast() || ip.IsPrivate() {
return true
}
for _, n := range nonPublicNets {
if n.Contains(ip) && !teredoNet.Contains(ip) {
return true
}
}
for _, n := range documentationNets {
if n.Contains(ip) {
return true
}
}
for _, n := range infra {
if n.Contains(ip) {
return true
}
}
// A Cloudflare edge or a verified crawler in the scored set is not
// an attacker to act on: challenging or blocking it hits every
// visitor behind the edge, or delists the site.
if checks.IsCloudflareIP(ip) || threatintel.IPInAnyVerifiedBotRange(ip) {
return true
}
// Check the original address before its embedded IPv4: operators
// can protect the IPv6 host or prefix independently of that IPv4.
if nat64Net.Contains(ip) {
return protected(net.IPv4(ip[12], ip[13], ip[14], ip[15]))
}
if sixToFourNet.Contains(ip) {
return protected(net.IPv4(ip[2], ip[3], ip[4], ip[5]))
}
return false
}
return func(s string) bool {
ip := net.ParseIP(s)
if ip == nil {
return true // unparseable: never act
}
return protected(ip)
}
}
package daemon
import (
"fmt"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/eximlog"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/store"
)
// --- Cloud-relay compromise detection -----------------------------------
//
// Detects the pattern where a mailbox's SMTP AUTH credentials are being
// abused from a rented botnet of cloud VMs. Characteristic signature:
// multiple authenticated sends in a short window, from several distinct
// cloud-provider IPs (Google Cloud, AWS, Azure, etc.), for the SAME
// mailbox user.
//
// A normal user on a residential/ISP IP never matches (cloud-PTR check).
// A legitimate self-hosted script on a single VPS won't match either
// (requires ≥2 distinct source IPs within the window). Credential abuse
// from a rotating fleet trips the detector within minutes of the first
// distinct IPs showing up.
//
// Action: one Critical finding per user per window. Customer-impacting
// response actions follow the configured auto-response and dry-run settings.
// The finding message embeds a source IP so autoblock can evaluate it.
// cloudProviderPTRSuffixes is an intentionally-conservative list of
// hostname suffixes that strongly indicate a cloud-VM source. Adding to
// this list increases detection coverage; removing reduces it. Keep
// entries specific enough to avoid catching ISP-transit ASNs.
var cloudProviderPTRSuffixes = []string{
// Google Cloud Platform
".googleusercontent.com",
".gce.internal",
// AWS EC2 — public PTRs only. `.compute.internal` is intentionally
// excluded: it is an AWS VPC-internal PTR but also appears on
// corporate VPN and self-hosted lab networks that have nothing to
// do with AWS, so matching it would risk suspending mailboxes
// that just happen to have an internal-looking reverse DNS.
".compute.amazonaws.com",
".compute-1.amazonaws.com",
// Microsoft Azure
".cloudapp.net",
".cloudapp.azure.com",
// Oracle Cloud
".oraclecloud.com",
".oraclevcn.com",
// DigitalOcean
".digitalocean.com",
".digitaloceanspaces.com",
// Linode / Akamai Cloud
".members.linode.com",
".linodeusercontent.com",
// Vultr
".vultr.com",
".vultrusercontent.com",
// Hetzner
".hetzner.com",
".your-server.de",
// OVH / OVHcloud
".ovh.net",
".ovhcloud.com",
".ovh.ca",
".ovh.us",
// Contabo
".contabo.net",
".contabo.host",
".contaboserver.net",
}
// isCloudRelayAllowed reports whether the AUTH user is opted out of the
// email_cloud_relay_abuse detector via the operator-managed allowlists.
// users matches whole mailboxes; domains matches the domain part. Both
// comparisons are case-insensitive. An empty user is never considered
// allowed (defense against malformed log lines).
func isCloudRelayAllowed(user string, users, domains []string) bool {
if user == "" {
return false
}
for _, u := range users {
if strings.EqualFold(user, u) {
return true
}
}
if len(domains) == 0 {
return false
}
at := strings.LastIndexByte(user, '@')
if at < 0 || at >= len(user)-1 {
return false
}
dom := user[at+1:]
for _, d := range domains {
if strings.EqualFold(dom, d) {
return true
}
}
return false
}
// isCloudProviderPTR reports whether the given PTR hostname belongs to a
// recognized public-cloud provider. Case-insensitive suffix match.
func isCloudProviderPTR(ptr string) bool {
if ptr == "" {
return false
}
p := strings.ToLower(ptr)
for _, suffix := range cloudProviderPTRSuffixes {
if strings.HasSuffix(p, suffix) {
return true
}
}
return false
}
// extractEximHostname parses the H=<hostname> field from an exim log line.
// The field often looks like "H=hostname.example (helo.string) [IP]:port"
// — we want the PTR-derived hostname before the HELO-in-parens.
func extractEximHostname(line string) string {
idx, ok := eximlog.HFieldStart(line)
if !ok {
return ""
}
rest := line[idx:]
// Terminate at first space, tab, or opening paren (HELO string).
end := len(rest)
for i, r := range rest {
if r == ' ' || r == '\t' || r == '(' {
end = i
break
}
}
return strings.TrimSpace(rest[:end])
}
// cloudRelayWindow tracks authenticated sends from cloud IPs for one user.
// Bounded so memory can't grow unbounded from a misbehaving log stream.
type cloudRelayWindow struct {
mu sync.Mutex
events []cloudRelayEvent
firedAt time.Time // last Critical emission; dedup guard
lastEvent time.Time // last append — used to garbage-collect idle entries
}
type cloudRelayEvent struct {
at time.Time
ip string
ptr string
}
// cloudRelayWindows tracks per-user cloud-relay activity.
var cloudRelayWindows sync.Map // map[string]*cloudRelayWindow
func lockCloudRelayWindowForUpdate(user string, now time.Time) *cloudRelayWindow {
for {
val, _ := cloudRelayWindows.LoadOrStore(user, &cloudRelayWindow{lastEvent: now})
w, ok := val.(*cloudRelayWindow)
if !ok {
cloudRelayWindows.Delete(user)
continue
}
w.mu.Lock()
// Re-check while holding w.mu so the eviction sweep cannot delete
// a stale entry and orphan a parser update made through that pointer.
current, ok := cloudRelayWindows.Load(user)
currentWindow, currentOK := current.(*cloudRelayWindow)
if ok && currentOK && currentWindow == w {
return w
}
w.mu.Unlock()
}
}
// Detection thresholds. Two OR-combined signals within the same 60-min
// sliding window:
//
// A. Multi-IP burst: ≥ cloudRelayMinEvents sends from ≥ cloudRelayMinDistinctIP
// distinct cloud IPs. Catches rented-fleet abuse rotating IPs per-send
// (the typical credential-stuffing spam pattern).
//
// B. Volume burst: ≥ cloudRelayHighVolumeEvents sends regardless of
// distinct-IP count. Catches paced attacks that deliberately use one
// cloud IP per day to evade signal A. Threshold sits well above any
// legitimate SaaS integration seen on production (SmartBill ~2/hr,
// Nylas ~2/hr, WP transactional ≤3/hr).
//
// Tuning rationale: these values were chosen from the Apr 2026 incident
// analysis. A single-mailbox user with a legit single-VPS cron averaging
// ≤14 mails/hr stays silent; anything above that is either a compromised
// relay or a SaaS integration that should be added to
// `email_protection.high_volume_senders`.
const (
cloudRelayWindow_ = 60 * time.Minute
cloudRelayMinEvents = 3
cloudRelayMinDistinctIP = 2
cloudRelayHighVolumeEvents = 15
cloudRelayDedupCooldown = 60 * time.Minute
cloudRelayMaxEvents = 256 // per-user cap; prevents unbounded growth
)
// parseCloudRelayFinding evaluates an exim acceptance line for the
// cloud-relay compromise pattern. Called from parseEximLogLine. Returns
// zero or one finding. Never auto-suspends on its own — emits a finding
// whose Message embeds the source IP; the existing autoblock + suspend
// pipeline picks it up by check name.
func parseCloudRelayFinding(line string, cfg *config.Config) (findings []alert.Finding) {
// Only care about authenticated outbound acceptance lines.
if !strings.Contains(line, " <= ") || !strings.Contains(line, "A=dovecot_") {
return nil
}
return cloudRelayObserve(cfg, extractAuthUser(line), eximlog.ClientIP(line), extractEximHostname(line), time.Now())
}
// cloudRelayObserve evaluates one authenticated send by user from ip, whose
// reverse DNS name is ptr, for the cloud-relay compromise pattern.
func cloudRelayObserve(cfg *config.Config, user, ip, ptr string, now time.Time) (findings []alert.Finding) {
if user == "" {
return nil
}
if isHighVolumeSender(user, cfg.EmailProtection.HighVolumeSenders) {
return nil
}
if isCloudRelayAllowed(user, cfg.EmailProtection.CloudRelay.AllowUsers, cfg.EmailProtection.CloudRelay.AllowDomains) {
return nil
}
if !isCloudProviderPTR(ptr) {
return nil
}
if ip == "" {
// Without an IP we can't dedup distinct sources; bail silently
// so we don't count half-parsed records toward the threshold.
return nil
}
// Registered before the unlock defer so owner I/O runs after it.
defer func() { stampMailAccountOwner(findings, user) }()
unlock := lockMailResponse(store.MailboxSuspensionID(activeMailBackend.name(), user))
defer unlock()
w := lockCloudRelayWindowForUpdate(user, now)
defer w.mu.Unlock()
// Prune anything older than the window.
cutoff := now.Add(-cloudRelayWindow_)
kept := w.events[:0]
for _, e := range w.events {
if e.at.After(cutoff) {
kept = append(kept, e)
}
}
w.events = kept
// Append this event. At the cap the oldest leaves, so the finding names
// the address of the line it fires on.
if len(w.events) >= cloudRelayMaxEvents {
w.events = append(w.events[:0], w.events[len(w.events)-cloudRelayMaxEvents+1:]...)
}
w.events = append(w.events, cloudRelayEvent{at: now, ip: ip, ptr: ptr})
w.lastEvent = now
// Already fired recently for this user? Dedup.
if !w.firedAt.IsZero() && now.Sub(w.firedAt) < cloudRelayDedupCooldown {
return nil
}
// Evaluate thresholds.
distinctIPs := make(map[string]struct{}, len(w.events))
for _, e := range w.events {
distinctIPs[e.ip] = struct{}{}
}
multiIPBurst := len(w.events) >= cloudRelayMinEvents && len(distinctIPs) >= cloudRelayMinDistinctIP
volumeBurst := len(w.events) >= cloudRelayHighVolumeEvents
if !multiIPBurst && !volumeBurst {
return nil
}
w.firedAt = now
// Build an IP list for the details (newest first, deduped).
seen := make(map[string]struct{}, len(w.events))
ips := make([]string, 0, len(distinctIPs))
for i := len(w.events) - 1; i >= 0; i-- {
e := w.events[i]
if _, dup := seen[e.ip]; dup {
continue
}
seen[e.ip] = struct{}{}
ips = append(ips, e.ip)
}
// The block source is carried in the structured SourceIP field below; the
// IP is included in the message only for operator-facing readability.
message := fmt.Sprintf(
"Email account %s sent %d authenticated messages from %d cloud-provider IPs in %d minutes - credentials compromised - from %s",
user, len(w.events), len(distinctIPs), int(cloudRelayWindow_.Minutes()), ips[0],
)
details := fmt.Sprintf(
"Authenticated SMTP submissions for %s in the last %d minutes:\n"+
" total sends: %d\n"+
" distinct source IPs: %d\n"+
" most recent PTR: %s\n"+
" recent IPs: %s\n\n"+
"Legitimate users do not send mail from rented cloud VMs. "+
"This pattern is characteristic of credential abuse by a bulk "+
"phishing operator. Outgoing mail hold and source-IP blocking "+
"follow the configured auto-response and dry-run settings.",
user,
int(cloudRelayWindow_.Minutes()),
len(w.events),
len(distinctIPs),
ptr,
strings.Join(truncateIPList(ips, 8), ", "),
)
mailbox, domain, _ := splitMailAccount(user)
return []alert.Finding{{
Severity: alert.Critical,
Check: "email_cloud_relay_abuse",
Message: message,
Details: truncateDaemon(details, 800),
SourceIP: ips[0],
Timestamp: now,
// Exim's H=... [ip] of the authenticated session.
Attribution: admission.AttributionPeer,
Mailbox: mailbox,
Domain: domain,
}}
}
func handleCloudRelayCredentialAbuse(cfg *config.Config, authUser string, at time.Time) {
respondToMailboxCompromise(cfg, authUser, "email_cloud_relay_abuse", "authenticated sends from several cloud-provider addresses", at)
}
// retroCloudRelayResponse responds to a startup-scan finding as of the logged
// time of its newest send, so a send from before a release cannot suspend the
// released mailbox again.
func retroCloudRelayResponse(cfg *config.Config, f alert.Finding) {
if sender := extractSenderFromCloudRelayMessage(f.Message); sender != "" {
handleCloudRelayCredentialAbuse(cfg, sender, f.Observation.ObservedAt)
}
}
// forgetCloudRelayEvidence drops a released mailbox's cloud-relay window so
// sends from before the release cannot trigger again.
func forgetCloudRelayEvidence(user string) {
cloudRelayWindows.Range(func(key, val any) bool {
if strings.EqualFold(key.(string), user) {
w := val.(*cloudRelayWindow)
w.mu.Lock()
cloudRelayWindows.CompareAndDelete(key, val)
w.mu.Unlock()
}
return true
})
}
func truncateIPList(ips []string, n int) []string {
if len(ips) <= n {
return ips
}
return ips[:n]
}
// cloudRelayEvictWindow is how long a per-user cloudRelayWindow can stay
// idle before it is evicted. Picked at 2x cloudRelayWindow_ so a user who
// just barely cleared the threshold does not lose their entry before the
// dedup cooldown can suppress a repeat.
const cloudRelayEvictWindow = 2 * cloudRelayWindow_
// StartCloudRelayEviction periodically prunes per-user cloud-relay
// windows that have not seen activity in cloudRelayEvictWindow. Without
// this sweep the cloudRelayWindows sync.Map grows linearly with every
// distinct authenticated sender ever seen, including users deleted by
// the operator. Pairs with the firedAt dedup guard so repeat alerts on
// long-lived attackers still pass.
func StartCloudRelayEviction(stopCh <-chan struct{}) {
obs.Go("cloud-relay-eviction", func() {
ticker := time.NewTicker(10 * time.Minute)
defer ticker.Stop()
for {
select {
case <-stopCh:
return
case now := <-ticker.C:
evictCloudRelayWindows(now)
}
}
})
}
// evictCloudRelayWindows deletes per-user entries whose lastEvent is
// older than cloudRelayEvictWindow. Safe to call from tests.
func evictCloudRelayWindows(now time.Time) {
cutoff := now.Add(-cloudRelayEvictWindow)
cloudRelayWindows.Range(func(key, val any) bool {
w, ok := val.(*cloudRelayWindow)
if !ok {
cloudRelayWindows.Delete(key)
return true
}
w.mu.Lock()
if w.lastEvent.Before(cutoff) {
cloudRelayWindows.CompareAndDelete(key, val)
}
w.mu.Unlock()
return true
})
}
package daemon
import (
"bufio"
"errors"
"fmt"
"io"
"os"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/eximlog"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/store"
)
// --- Retrospective scan for cloud-relay credential abuse -----------------
//
// The realtime watcher in cloud_relay.go only catches traffic arriving
// AFTER CSM starts. A credential-abuse spam run can go for weeks before
// an operator notices (a real incident saw 230 outbound sends from one
// compromised account over 20 days before being flagged). This scanner replays
// the last N hours of exim_mainlog through the same rule, so on CSM
// startup any in-progress or recent compromise is surfaced immediately.
//
// Runs once at daemon startup. Not part of the tiered-check registry
// because it is purely an event-log replay; the realtime watcher owns
// live state thereafter.
// Memory caps for the retro scan. A compromised account or a crafted log
// can otherwise grow byUser without bound. The thresholds are set well
// above the volume detector (cloudRelayHighVolumeEvents=15 in a 60-min
// window) so legitimate detection is unaffected; the caps only kick in
// for pathological volumes that would balloon memory.
const (
cloudRelayScanMaxEventsPerUser = 5000
cloudRelayScanMaxUsers = 10000
)
// CloudRelayScanPath is the log file path scanned at startup. Exported
// via var (not const) for tests.
var CloudRelayScanPath = platform.EximMainLog
// ScanEximHistoryForCloudRelay replays the tail of exim_mainlog for the
// last `lookback` duration and returns a finding per mailbox that
// exceeds the cloud-relay thresholds. Safe to call from goroutines.
//
// The scanner respects EmailProtection.HighVolumeSenders and the
// detector-scoped EmailProtection.CloudRelay.AllowUsers / .AllowDomains
// allowlists, mirrors the realtime detector's thresholds exactly, and
// uses a per-user persistent marker in the global store to avoid
// re-emitting the same finding on successive restarts.
func ScanEximHistoryForCloudRelay(cfg *config.Config, logPath string, now time.Time, lookback time.Duration) []alert.Finding {
if logPath == "" {
logPath = CloudRelayScanPath
}
// #nosec G304 -- logPath is operator-configured / hardcoded to cPanel default.
f, err := os.Open(logPath)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
// A record is named by the file and its offset and bytes, as the realtime
// watcher names the same line.
var stream string
if info, statErr := f.Stat(); statErr == nil {
if id := fileID(info); id.known {
stream = alert.FileStream(id.dev, id.ino)
}
}
since := now.Add(-lookback)
byUser := make(map[string]*cloudRelayScanAccumulator)
// Use bufio.Reader rather than bufio.Scanner: an exim line can
// occasionally exceed whatever fixed Scanner buffer we set (e.g.
// a spam run with a huge Base64 subject). Scanner returns an
// ErrTooLong which aborts the whole loop — missing every later
// compromise event. Reader.ReadString lets us skip oversized
// lines and keep going.
reader := bufio.NewReaderSize(f, 256*1024)
var offset int64
for {
line, rerr := reader.ReadString('\n')
if len(line) > 0 {
lineStart := offset
offset += int64(len(line))
// A line without its newline may still be written: no record yet.
ref := ""
if line[len(line)-1] == '\n' {
line = line[:len(line)-1]
ref = stream
}
processCloudRelayScanRecord(line, cfg, since, byUser, cloudRelayRecord{stream: ref, offset: lineStart})
}
if rerr == nil {
continue
}
if errors.Is(rerr, io.EOF) {
break
}
if errors.Is(rerr, bufio.ErrBufferFull) {
// Line longer than 256 KB — drain it and move on.
// Real exim acceptance lines are well under 10 KB;
// anything longer is almost certainly a pathological
// subject we can't usefully parse anyway.
if drainErr := drainUntilNewline(reader); drainErr != nil {
break
}
continue
}
// Any other I/O error: stop cleanly, don't panic.
break
}
users := make([]string, 0, len(byUser))
for u := range byUser {
users = append(users, u)
}
sort.Strings(users) // stable finding order
var findings []alert.Finding
for _, user := range users {
acc := byUser[user]
if acc == nil || !acc.reportable {
continue
}
maxSends, maxDistinctIPs, fireAt, peakPTR := acc.bestSends, acc.bestDistinctIPs, acc.bestAt, acc.bestPTR
multiIP := maxSends >= cloudRelayMinEvents && maxDistinctIPs >= cloudRelayMinDistinctIP
volume := maxSends >= cloudRelayHighVolumeEvents
if !multiIP && !volume {
continue
}
// Persistent dedup: skip if we've already fired for this user
// and no new event has landed since then.
latestEvent := acc.latestEvent
if alreadyReportedRetro(user, latestEvent) {
continue
}
ips := acc.recentIPs
if len(ips) == 0 {
continue
}
recentIP := ips[0]
msg := fmt.Sprintf(
"RETRO: account %s sent %d authenticated messages from %d cloud-provider IPs (peak 60-min burst) in the last %d hours - credentials compromised - from %s",
user, maxSends, maxDistinctIPs, int(lookback.Hours()), recentIP,
)
details := fmt.Sprintf(
"Retrospective exim_mainlog scan at %s found a cloud-relay pattern:\n"+
" user: %s\n"+
" total cloud-PTR sends (%dh): %d\n"+
" peak 60-min window: %d sends / %d distinct IPs ending at %s\n"+
" peak PTR: %s\n"+
" distinct source IPs observed: %s\n\n"+
"Outgoing mail hold and source-IP blocking follow the configured "+
"auto-response and dry-run settings. Older IPs are left as context "+
"because rented-fleet addresses tend to be recycled outside a 2-hour window.",
now.Format("2006-01-02 15:04:05"),
user,
int(lookback.Hours()),
acc.total,
maxSends, maxDistinctIPs, fireAt.Format("2006-01-02 15:04:05"),
peakPTR,
strings.Join(ips, ", "),
)
mailbox, domain, _ := splitMailAccount(user)
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "email_cloud_relay_abuse",
Message: msg,
Details: truncateDaemon(details, 900),
Timestamp: now,
SourceIP: recentIP,
// Exim's H=... [ip] of the authenticated session.
Attribution: admission.AttributionPeer,
// The newest counted send is the record behind recentIP.
Observation: acc.newest,
Mailbox: mailbox,
Domain: domain,
})
stampMailAccountOwner(findings[len(findings)-1:], user)
markReportedRetro(user, latestEvent)
}
return findings
}
// processCloudRelayScanLine parses a single exim log line and, if it is
// an authenticated cloud-PTR acceptance within the lookback window,
// records it under the AUTH user in `byUser`.
func processCloudRelayScanLine(line string, cfg *config.Config, since time.Time, byUser map[string]*cloudRelayScanAccumulator) {
processCloudRelayScanRecord(line, cfg, since, byUser, cloudRelayRecord{})
}
// processCloudRelayScanRecord is processCloudRelayScanLine for a line read
// at rec, which the newest counted send of its account then names.
func processCloudRelayScanRecord(line string, cfg *config.Config, since time.Time, byUser map[string]*cloudRelayScanAccumulator, rec cloudRelayRecord) {
if !strings.Contains(line, " <= ") || !strings.Contains(line, "A=dovecot_") {
return
}
// A stamp without a zone is in the zone of the caller's clock.
ts, ok := eximlog.LineTime(line, since.Location())
if !ok || ts.Before(since) {
return
}
user := extractAuthUser(line)
if user == "" || isHighVolumeSender(user, cfg.EmailProtection.HighVolumeSenders) {
return
}
if isCloudRelayAllowed(user, cfg.EmailProtection.CloudRelay.AllowUsers, cfg.EmailProtection.CloudRelay.AllowDomains) {
return
}
ptr := extractEximHostname(line)
if !isCloudProviderPTR(ptr) {
return
}
ip := eximlog.ClientIP(line)
if ip == "" {
return
}
acc, exists := byUser[user]
if !exists {
if len(byUser) >= cloudRelayScanMaxUsers {
pruneCloudRelayScanUsers(byUser, ts)
}
if len(byUser) >= cloudRelayScanMaxUsers {
evictOldestCloudRelayScanUser(byUser)
}
if len(byUser) >= cloudRelayScanMaxUsers {
return
}
acc = newCloudRelayScanAccumulator()
byUser[user] = acc
}
acc.record(cloudRelayScanEvent{at: ts, ip: ip, ptr: ptr})
// Forward order: the last counted send is the newest. A send without a
// record leaves none, so the finding never names an older address's line.
acc.newest = alert.Observation{}
if rec.stream != "" {
acc.newest = alert.Observation{Producer: string(checks.ProducerEximHistory), Stream: rec.stream,
Cursor: alert.RecordCursor(rec.offset, line), ObservedAt: ts.UTC()}
}
}
// cloudRelayRecord is where a mainlog line sits: the file's stream, empty for
// a line that may still be written, and the line's start offset.
type cloudRelayRecord struct {
stream string
offset int64
}
// drainUntilNewline reads from reader and discards bytes until a newline
// is consumed or EOF is hit. Returns io.EOF if the reader is exhausted.
func drainUntilNewline(reader *bufio.Reader) error {
for {
_, err := reader.ReadSlice('\n')
if err == nil {
return nil
}
if errors.Is(err, bufio.ErrBufferFull) {
// Still inside the oversized line — keep draining.
continue
}
return err
}
}
// cloudRelayScanEvent is a single timestamped cloud-PTR AUTH send
// replayed from the log.
type cloudRelayScanEvent struct {
at time.Time
ip string
ptr string
}
type cloudRelayScanAccumulator struct {
events []cloudRelayScanEvent
ipCounts map[string]int
recentIPs []string
total int
latestEvent time.Time
bestSends int
bestDistinctIPs int
bestAt time.Time
bestPTR string
reportable bool
// newest names the last counted send's record; zero without one.
newest alert.Observation
}
func newCloudRelayScanAccumulator() *cloudRelayScanAccumulator {
return &cloudRelayScanAccumulator{
ipCounts: make(map[string]int),
}
}
func (acc *cloudRelayScanAccumulator) record(event cloudRelayScanEvent) {
acc.total++
if acc.latestEvent.IsZero() || event.at.After(acc.latestEvent) {
acc.latestEvent = event.at
}
acc.rememberRecentIP(event.ip)
cutoff := event.at.Add(-cloudRelayWindow_)
drop := 0
for drop < len(acc.events) && acc.events[drop].at.Before(cutoff) {
acc.removeWindowIP(acc.events[drop].ip)
drop++
}
if drop > 0 {
clear(acc.events[:drop])
acc.events = acc.events[drop:]
}
if len(acc.events) >= cloudRelayScanMaxEventsPerUser {
acc.removeWindowIP(acc.events[0].ip)
var zero cloudRelayScanEvent
acc.events[0] = zero
acc.events = acc.events[1:]
}
acc.events = append(acc.events, event)
acc.ipCounts[event.ip]++
sends := len(acc.events)
distinctIPs := len(acc.ipCounts)
if sends > acc.bestSends || (sends == acc.bestSends && distinctIPs > acc.bestDistinctIPs) {
acc.bestSends = sends
acc.bestDistinctIPs = distinctIPs
acc.bestAt = event.at
acc.bestPTR = event.ptr
}
if acc.bestSends >= cloudRelayHighVolumeEvents ||
(acc.bestSends >= cloudRelayMinEvents && acc.bestDistinctIPs >= cloudRelayMinDistinctIP) {
acc.reportable = true
}
}
func (acc *cloudRelayScanAccumulator) removeWindowIP(ip string) {
count := acc.ipCounts[ip]
if count <= 1 {
delete(acc.ipCounts, ip)
return
}
acc.ipCounts[ip] = count - 1
}
func (acc *cloudRelayScanAccumulator) rememberRecentIP(ip string) {
if ip == "" {
return
}
for i, existing := range acc.recentIPs {
if existing != ip {
continue
}
copy(acc.recentIPs[1:i+1], acc.recentIPs[:i])
acc.recentIPs[0] = ip
return
}
acc.recentIPs = append(acc.recentIPs, "")
copy(acc.recentIPs[1:], acc.recentIPs[:len(acc.recentIPs)-1])
acc.recentIPs[0] = ip
if len(acc.recentIPs) > 10 {
acc.recentIPs = acc.recentIPs[:10]
}
}
func pruneCloudRelayScanUsers(byUser map[string]*cloudRelayScanAccumulator, now time.Time) {
cutoff := now.Add(-cloudRelayWindow_)
for user, acc := range byUser {
if acc == nil || (!acc.reportable && acc.latestEvent.Before(cutoff)) {
delete(byUser, user)
}
}
}
func evictOldestCloudRelayScanUser(byUser map[string]*cloudRelayScanAccumulator) {
var oldestUser string
var oldestSeen time.Time
for user, acc := range byUser {
if acc == nil {
delete(byUser, user)
return
}
if acc.reportable {
continue
}
if oldestUser == "" || acc.latestEvent.Before(oldestSeen) {
oldestUser = user
oldestSeen = acc.latestEvent
}
}
if oldestUser != "" {
delete(byUser, oldestUser)
}
}
// maxCloudRelayBurst finds the strongest 60-min window in a sorted event
// list. Returns (sends, distinctIPs, peakEnd, peakPTR), where peakEnd is
// the timestamp of the LAST event in the best window so operators see
// when the burst peaked, not when it started.
func maxCloudRelayBurst(events []cloudRelayScanEvent) (int, int, time.Time, string) {
if len(events) == 0 {
return 0, 0, time.Time{}, ""
}
bestSends, bestDistinct := 0, 0
var bestAt time.Time
var bestPTR string
left := 0
ipCounts := make(map[string]int)
for right, event := range events {
ipCounts[event.ip]++
for event.at.Sub(events[left].at) > cloudRelayWindow_ {
leftIP := events[left].ip
if ipCounts[leftIP] <= 1 {
delete(ipCounts, leftIP)
} else {
ipCounts[leftIP]--
}
left++
}
sends := right - left + 1
distinct := len(ipCounts)
// "Best" = highest send count; tie-break by distinct IPs.
if sends > bestSends || (sends == bestSends && distinct > bestDistinct) {
bestSends = sends
bestDistinct = distinct
bestAt = event.at
bestPTR = event.ptr
}
}
return bestSends, bestDistinct, bestAt, bestPTR
}
// alreadyReportedRetro returns true when the latest event for this user
// is older than or equal to the persisted marker (meaning: nothing new
// since we last alerted).
func alreadyReportedRetro(user string, latestEvent time.Time) bool {
db := store.Global()
if db == nil {
return false
}
raw := db.GetMetaString("cloudrelay_retro:" + user)
if raw == "" {
return false
}
prev, err := time.Parse(time.RFC3339, raw)
if err != nil {
return false
}
return !latestEvent.After(prev)
}
// extractSenderFromCloudRelayMessage pulls the sender mailbox out of a
// finding message emitted by ScanEximHistoryForCloudRelay. Returns ""
// when the message is not from this check (defensive — never panics on
// unexpected input).
func extractSenderFromCloudRelayMessage(msg string) string {
const marker = "account "
idx := strings.Index(msg, marker)
if idx < 0 {
return ""
}
rest := msg[idx+len(marker):]
sp := strings.IndexByte(rest, ' ')
if sp <= 0 {
return ""
}
candidate := rest[:sp]
if !strings.Contains(candidate, "@") {
return ""
}
return candidate
}
func markReportedRetro(user string, latestEvent time.Time) {
db := store.Global()
if db == nil {
return
}
_ = db.SetMetaString("cloudrelay_retro:"+user, latestEvent.Format(time.RFC3339Nano))
}
//go:build linux
package daemon
import (
"crypto/sha256"
"encoding/hex"
"golang.org/x/sys/unix"
)
// hashEventFD returns the SHA-256 of exactly expectedSize bytes behind an
// event descriptor. prefix must be the bytes the scanner already read at
// offset zero. Reusing them prevents an in-place overwrite from making the CMS
// decision cover different leading bytes than signature and YARA scanning.
// The fixed expected size bounds the work and prevents a concurrently growing
// file from keeping an analyzer busy indefinitely. An empty result means the
// file could not be read consistently.
func hashEventFD(fd int, prefix []byte, expectedSize int64) string {
if expectedSize < int64(len(prefix)) || expectedSize < 0 {
return ""
}
var before unix.Stat_t
if err := unix.Fstat(fd, &before); err != nil || before.Size != expectedSize {
return ""
}
h := sha256.New()
_, _ = h.Write(prefix)
buf := make([]byte, 64*1024)
offset := int64(len(prefix))
for offset < expectedSize {
want := int64(len(buf))
if remaining := expectedSize - offset; remaining < want {
want = remaining
}
n, err := unix.Pread(fd, buf[:int(want)], offset)
if n > 0 {
_, _ = h.Write(buf[:n])
offset += int64(n)
}
if err != nil {
if err == unix.EINTR && n == 0 {
continue
}
return ""
}
if n == 0 {
return ""
}
}
var after unix.Stat_t
if err := unix.Fstat(fd, &after); err != nil || after.Size != expectedSize {
return ""
}
return hex.EncodeToString(h.Sum(nil))
}
//go:build !(linux && bpf)
package daemon
import (
"context"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/config"
)
// connectionBPF is the no-tag placeholder for the BPF cgroup/connect backend.
// The real type with map handles, links, and the ringbuf reader lives in
// connection_bpf.go behind //go:build linux && bpf. On any other build, the
// coordinator never reaches this stub: startConnectionBPF returns
// bpf.ErrNotBuilt before a value is constructed.
type connectionBPF struct{}
func (c *connectionBPF) Mode() string { return "bpf" }
func (c *connectionBPF) EventCount() uint64 { return 0 }
func (c *connectionBPF) Run(_ context.Context) {}
func startConnectionBPF(_ context.Context, _ chan<- alert.Finding, _ *config.Config) (*connectionBPF, error) {
return nil, bpf.ErrNotBuilt
}
package daemon
import (
"context"
"errors"
"fmt"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/verdict"
)
var errBPFVerdictDisabled = errors.New("verdict callback not configured")
// bpfVerdictEnabled reports whether this event is one the operator's callback
// should be asked about. Cheap enough for the ring-buffer consumer loop; the
// request itself is not, and runs on the enricher's workers.
func bpfVerdictEnabled(cfg *config.Config, ev ConnectionEvent) bool {
if cfg == nil || !cfg.BPFEnforcement.VerdictCallback || !cfg.AutoResponse.VerdictCallback.Enabled {
return false
}
return ev.Decision == 1 || ev.Decision == 2
}
// askBPFVerdict performs one callback. The client is built per call from the
// live configuration so a hot reload of the URL or secret is honoured.
func askBPFVerdict(ctx context.Context, cfg *config.Config, req verdict.Request) (verdict.Response, error) {
if cfg == nil {
return verdict.Response{}, errBPFVerdictDisabled
}
return newVerdictClient(cfg).Ask(ctx, req)
}
// bpfVerdictReason is the callback's reason string for one event.
func bpfVerdictReason(check string, port uint16) string {
return fmt.Sprintf("bpf_enforcement:%s:%d", check, port)
}
func appendFindingDetail(f *alert.Finding, detail string) {
if detail == "" {
return
}
if f.Details == "" {
f.Details = detail
return
}
f.Details += ", " + detail
}
// Code generated by bpf2go; DO NOT EDIT.
//go:build 386 || amd64
package connection_bpfprog
import (
"bytes"
_ "embed"
"fmt"
"io"
"structs"
"github.com/cilium/ebpf"
)
type ConnectionConnEvent struct {
_ structs.HostLayout
Uid uint32
Pid uint32
Family uint32
DstPort uint32
DstIp4 uint32
DstIp6 [16]uint8
Comm [16]uint8
Decision uint32
}
type ConnectionCsmQueueStats struct {
_ structs.HostLayout
Lost uint64
Submitted uint64
}
type ConnectionPolicyState struct {
_ structs.HostLayout
Enforce uint32
DryRun uint32
ProtectedPorts uint32
}
// Names of all BPF objects in the ELF.
//
// Used for safe lookups in a Collection or CollectionSpec.
const (
ConnectionMapEvents = "events"
ConnectionMapPolicy = "policy"
ConnectionMapProtectedPorts = "protected_ports"
ConnectionMapQueueStats = "queue_stats"
ConnectionMapSafeUids = "safe_uids"
ConnectionProgCsmConnect4 = "csm_connect4"
ConnectionProgCsmConnect6 = "csm_connect6"
ConnectionVarUnused = "unused"
ConnectionVarUnusedPolicy = "unused_policy"
)
// LoadConnection returns the embedded CollectionSpec for Connection.
func LoadConnection() (*ebpf.CollectionSpec, error) {
reader := bytes.NewReader(_ConnectionBytes)
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
if err != nil {
return nil, fmt.Errorf("can't load Connection: %w", err)
}
return spec, err
}
// LoadConnectionObjects loads Connection and converts it into a struct.
//
// The following types are suitable as obj argument:
//
// *ConnectionObjects
// *ConnectionPrograms
// *ConnectionMaps
//
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
func LoadConnectionObjects(obj any, opts *ebpf.CollectionOptions) error {
spec, err := LoadConnection()
if err != nil {
return err
}
return spec.LoadAndAssign(obj, opts)
}
// ConnectionSpecs contains maps and programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type ConnectionSpecs struct {
ConnectionProgramSpecs
ConnectionMapSpecs
ConnectionVariableSpecs
}
// ConnectionProgramSpecs contains programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type ConnectionProgramSpecs struct {
CsmConnect4 *ebpf.ProgramSpec `ebpf:"csm_connect4"`
CsmConnect6 *ebpf.ProgramSpec `ebpf:"csm_connect6"`
}
// ConnectionMapSpecs contains maps before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type ConnectionMapSpecs struct {
Events *ebpf.MapSpec `ebpf:"events"`
Policy *ebpf.MapSpec `ebpf:"policy"`
ProtectedPorts *ebpf.MapSpec `ebpf:"protected_ports"`
QueueStats *ebpf.MapSpec `ebpf:"queue_stats"`
SafeUids *ebpf.MapSpec `ebpf:"safe_uids"`
}
// ConnectionVariableSpecs contains global variables before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type ConnectionVariableSpecs struct {
Unused *ebpf.VariableSpec `ebpf:"unused"`
UnusedPolicy *ebpf.VariableSpec `ebpf:"unused_policy"`
}
// ConnectionObjects contains all objects after they have been loaded into the kernel.
//
// It can be passed to LoadConnectionObjects or ebpf.CollectionSpec.LoadAndAssign.
type ConnectionObjects struct {
ConnectionPrograms
ConnectionMaps
ConnectionVariables
}
func (o *ConnectionObjects) Close() error {
return _ConnectionClose(
&o.ConnectionPrograms,
&o.ConnectionMaps,
)
}
// ConnectionMaps contains all maps after they have been loaded into the kernel.
//
// It can be passed to LoadConnectionObjects or ebpf.CollectionSpec.LoadAndAssign.
type ConnectionMaps struct {
Events *ebpf.Map `ebpf:"events"`
Policy *ebpf.Map `ebpf:"policy"`
ProtectedPorts *ebpf.Map `ebpf:"protected_ports"`
QueueStats *ebpf.Map `ebpf:"queue_stats"`
SafeUids *ebpf.Map `ebpf:"safe_uids"`
}
func (m *ConnectionMaps) Close() error {
return _ConnectionClose(
m.Events,
m.Policy,
m.ProtectedPorts,
m.QueueStats,
m.SafeUids,
)
}
// ConnectionVariables contains all global variables after they have been loaded into the kernel.
//
// It can be passed to LoadConnectionObjects or ebpf.CollectionSpec.LoadAndAssign.
type ConnectionVariables struct {
Unused *ebpf.Variable `ebpf:"unused"`
UnusedPolicy *ebpf.Variable `ebpf:"unused_policy"`
}
// ConnectionPrograms contains all programs after they have been loaded into the kernel.
//
// It can be passed to LoadConnectionObjects or ebpf.CollectionSpec.LoadAndAssign.
type ConnectionPrograms struct {
CsmConnect4 *ebpf.Program `ebpf:"csm_connect4"`
CsmConnect6 *ebpf.Program `ebpf:"csm_connect6"`
}
func (p *ConnectionPrograms) Close() error {
return _ConnectionClose(
p.CsmConnect4,
p.CsmConnect6,
)
}
func _ConnectionClose(closers ...io.Closer) error {
for _, closer := range closers {
if err := closer.Close(); err != nil {
return err
}
}
return nil
}
// Do not access this directly.
//
//go:embed connection_x86_bpfel.o
var _ConnectionBytes []byte
package daemon
import (
"net"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
)
var (
directSMTPRDNSOnce sync.Once
directSMTPRDNSCache *checks.RDNSCache
)
// rdnsCache is the daemon-wide rDNS cache used by direct SMTP egress
// detection. TTL 30 min, per-lookup deadline 1 second. Resolver wraps
// net.LookupAddr; negative results cached so a slow upstream does not
// stall the connection consumer.
func rdnsCache() *checks.RDNSCache {
directSMTPRDNSOnce.Do(func() {
directSMTPRDNSCache = checks.NewRDNSCache(checks.RDNSCacheConfig{
TTL: 30 * time.Minute,
ResolveDeadline: time.Second,
Resolve: func(ip net.IP) (string, error) {
names, err := net.LookupAddr(ip.String())
if err != nil || len(names) == 0 {
return "", err
}
return strings.TrimSuffix(names[0], "."), nil
},
})
})
return directSMTPRDNSCache
}
// evaluateConnectionEvent runs every per-event detector and returns the
// findings that should be emitted. Pure-ish: no IO and no alertCh
// access. Caller is responsible for attaching process context (which
// MAY do IO via the enricher) and shipping to alertCh.
//
// The function exists in a non-build-tagged file so unit tests can
// drive a synthetic ConnectionEvent through the same policy logic the
// live BPF Run loop uses, without requiring the linux+bpf build tag.
func evaluateConnectionEvent(cfg *config.Config, mta platform.MTAIdents, ev ConnectionEvent, user string) []alert.Finding {
switch ev.Decision {
case 0:
BumpBPFEnforcementDecision(BPFDecisionAllow)
case 1:
BumpBPFEnforcementDecision(BPFDecisionDryRun)
case 2:
BumpBPFEnforcementDecision(BPFDecisionDeny)
}
// Phase 4 note: bpf_enforcement.verdict_callback is applied by the
// BPF Run loop after this evaluator returns. The in-kernel hook
// NEVER waits on HTTP; cgroup/connect is synchronous and a remote
// callback would add latency to every connect.
now := time.Now()
// Phase 3 note: DryRun knobs are not consulted here. Detection runs
// regardless. The knobs gate the Phase 4 auto-response action that
// has not landed yet.
var out []alert.Finding
if checks.DirectSMTPEgressBackendEnabled(cfg, "bpf") {
// Direct SMTP egress (Phase 3). Distinct Check value; the inbound
// smtp_probe meters never see this traffic.
if f, ok := checks.EvaluateDirectSMTPEgress(cfg, checks.DirectSMTPEgressInput{
UID: ev.UID,
User: user,
PID: ev.PID,
Comm: ev.Comm,
DstIP: ev.DstIP,
DstPort: ev.DstPort,
MTA: mta,
}); ok {
if domain := rdnsCache().Lookup(ev.DstIP); domain != "" {
f.Details += ", Domain: " + domain
}
f.Timestamp = now
out = append(out, f)
checks.BumpDirectSMTPEgressFindings()
}
}
// Pre-existing user_outbound_connection detector. SMTP destinations
// are filtered out by checks.safeRemotePorts inside this evaluator,
// so it does not double-fire for a 25/465/587 connect.
if f, ok := checks.EvaluateConnection(cfg, ev.UID, ev.DstIP, ev.DstPort, 0, protoFromFamily(ev.Family), user); ok {
f.Timestamp = now
out = append(out, f)
}
// Bad-ASN egress (host-takeover chain leg). The BPF program emits
// non-root connects only; root egress is covered by the periodic
// /proc/net scan so root-heavy hosts do not flood the ringbuf.
if ev.UID != 0 && cfg.Detection.BadASNOutbound.Enabled {
if lookup := checks.CurrentASNLookup(); lookup != nil {
asn, org := lookup(ev.DstIP.String())
if f, ok := checks.EvaluateBadASNOutbound(cfg, ev.DstIP, asn, org); ok {
checks.AttributeSocketOwner(&f, ev.UID)
f.Timestamp = now
out = append(out, f)
}
}
}
return out
}
// protoFromFamily maps a sockaddr family int to a string label used in
// finding details. Lives here (not in connection_bpf.go) so the
// evaluator helper compiles on darwin without the bpf tag.
func protoFromFamily(f uint32) string {
if f == 10 {
return "tcp6"
}
return "tcp"
}
package daemon
import (
"encoding/binary"
"errors"
"net"
)
// ConnectionEvent is the userspace shape of a struct conn_event emitted by
// the cgroup/connect BPF program. Field layout matches connection.bpf.c
// byte for byte: scalars are little-endian (host order on amd64/arm64),
// dst_ip4 is network-order, dst_ip6 is the raw 16-byte address.
type ConnectionEvent struct {
UID uint32
PID uint32
Family uint32 // AF_INET=2, AF_INET6=10
DstPort uint16 // host order; BPF program calls bpf_ntohs
DstIP net.IP // resolved from dst_ip4 (v4) or dst_ip6 (v6) per Family
Comm string // null-terminated, up to 16 bytes
Decision uint32 // Phase 4: DECISION_* code (0=allow, 1=dry_run_deny, 2=deny)
}
const connectionEventSize = 4 + 4 + 4 + 4 + 4 + 16 + 16 + 4
func decodeConnectionEvent(b []byte) (ConnectionEvent, error) {
if len(b) < connectionEventSize {
return ConnectionEvent{}, errors.New("connection event short buffer")
}
// The BPF program stores dst_port as __u32 for alignment but calls
// bpf_ntohs() before writing, which guarantees the value fits in 16 bits.
// The narrowing is safe by construction.
dstPort := binary.LittleEndian.Uint32(b[12:16]) & 0xffff
ev := ConnectionEvent{
UID: binary.LittleEndian.Uint32(b[0:4]),
PID: binary.LittleEndian.Uint32(b[4:8]),
Family: binary.LittleEndian.Uint32(b[8:12]),
DstPort: uint16(dstPort), // #nosec G115 -- masked to low 16 bits above
}
switch ev.Family {
case 2: // AF_INET
ipv4 := make(net.IP, 4)
binary.BigEndian.PutUint32(ipv4, binary.BigEndian.Uint32(b[16:20]))
ev.DstIP = ipv4
case 10: // AF_INET6
v6 := make(net.IP, 16)
copy(v6, b[20:36])
ev.DstIP = v6
default:
return ConnectionEvent{}, errors.New("unknown family")
}
ev.Comm = nullTerm(b[36 : 36+16])
ev.Decision = binary.LittleEndian.Uint32(b[52:56])
return ev, nil
}
func indexNull(b []byte) int {
for i, c := range b {
if c == 0 {
return i
}
}
return -1
}
// nullTerm returns the prefix of b up to (but not including) the first NUL,
// or the whole slice if there is none. Helper for fixed-size character
// fields the BPF programs emit (comm[16], filename[256], exe[256]).
func nullTerm(b []byte) string {
if i := indexNull(b); i >= 0 {
return string(b[:i])
}
return string(b)
}
package daemon
import (
"context"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
)
// connectionPoller is the userspace fallback. It runs CheckOutboundUserConnections
// on a fixed interval and forwards any findings to the alert channel. Used
// when the BPF backend is unavailable (no bpf tag, kernel rejects program,
// or operator pinned legacy via Detection.ConnectionTrackerBackend).
type connectionPoller struct {
cfg *config.Config
alertCh chan<- alert.Finding
count atomic.Uint64
}
func newConnectionPoller(cfg *config.Config, alertCh chan<- alert.Finding) *connectionPoller {
return &connectionPoller{cfg: cfg, alertCh: alertCh}
}
func (p *connectionPoller) Mode() string { return "legacy" }
func (p *connectionPoller) EventCount() uint64 { return p.count.Load() }
func (p *connectionPoller) Run(ctx context.Context) {
interval := pollerInterval(p.cfg)
t := time.NewTicker(interval)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
findings := checks.CheckOutboundUserConnections(ctx, activeConnectionCfg(p.cfg), nil)
for _, f := range findings {
p.count.Add(1)
if !alert.TryEnqueue(p.alertCh, f) {
csmlog.Warn("connection legacy: alert channel full, dropping finding")
}
}
}
}
}
// pollerInterval returns the configured polling interval, falling back to a
// 30-second default when Detection.ConnectionPollInterval is unset.
func pollerInterval(cfg *config.Config) time.Duration {
if d := cfg.Detection.ConnectionPollInterval; d > 0 {
return d
}
return 30 * time.Second
}
package daemon
import (
"context"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/processctx"
)
// StartConnectionTracker starts the live connection tracker chosen by
// detection.connection_tracker_backend; see startLiveBackend for the choices.
func StartConnectionTracker(alertCh chan<- alert.Finding, cfg *config.Config) bpf.Backend {
return startLiveBackend("connection_tracker", cfg.Detection.ConnectionTrackerBackend, alertCh,
func() (bpf.Backend, error) { return tryStartConnectionBPFFn(context.Background(), alertCh, cfg) },
func() bpf.Backend { return newConnectionPoller(cfg, alertCh) })
}
func activeConnectionCfg(startup *config.Config) *config.Config {
if cfg := config.Active(); cfg != nil {
return cfg
}
return startup
}
// tryStartConnectionBPFFn is the package-level indirection so tests can
// substitute a fake without the bpf build tag.
var tryStartConnectionBPFFn = tryStartConnectionBPF
func tryStartConnectionBPF(ctx context.Context, ch chan<- alert.Finding, cfg *config.Config) (bpf.Backend, error) {
b, err := startConnectionBPF(ctx, ch, cfg)
if err != nil {
return nil, err
}
return b, nil
}
// attachProcessCtxToFinding sets f.Process from the cache when present, or
// enqueues a /proc enrichment so the next finding for the same PID benefits.
// Cache miss is the common case for short-lived processes; the finding is
// emitted with whatever context already exists (often none).
func attachProcessCtxToFinding(cache *processctx.Cache, enr *processctx.Enricher, f *alert.Finding, ev ConnectionEvent) {
if ev.PID == 0 {
return
}
req := processctxRequestFromConnection(ev)
if pc, needsEnrichment := cache.MaterializeVerifiedSnapshot(req); pc != nil {
f.Process = pc
if pc.Account != "" && (f.Check == "direct_smtp_egress" || f.TenantID == "") {
f.TenantID = pc.Account
}
if needsEnrichment {
enr.Enqueue(req)
}
return
}
enr.Enqueue(req)
}
func processctxRequestFromConnection(ev ConnectionEvent) processctx.EnrichRequest {
pid := int(ev.PID)
return processctx.EnrichRequest{
PID: pid,
UID: int(ev.UID),
UIDKnown: true,
Comm: ev.Comm,
StartedAt: processCtxStartedAt(pid),
}
}
package daemon
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/store"
)
// learningDir holds the learning keys and anchor messages; tests replace
// it.
var learningDir = antispam.DefaultLearningDir
// handleAntispamLearnFresh provisions a fresh learning namespace.
// With learning disabled it moves the learning directory aside, keys
// and anchors included, and empties the ledger and watcher state, so the
// next enable creates new keys and a new prefix. Nothing is deleted and
// Redis is never flushed; the old directory stays until the operator
// removes it.
func (c *ControlListener) handleAntispamLearnFresh(_ json.RawMessage) (any, error) {
rec := actionlog.Record{Op: "integrate.antispam_learn", Action: "fresh_namespace", Actor: actionlog.CLI,
ActorDetail: "csm antispam learn --fresh-namespace", Result: actionlog.Refused}
res, err := c.d.freshLearningNamespace(time.Now())
switch {
case errors.Is(err, errLearnRefused):
rec.Error = err.Error()
case err != nil:
rec.Result, rec.Error = actionlog.Failed, err.Error()
default:
rec.Result = actionlog.Applied
}
if res.Archived != "" {
target, _ := json.Marshal(res.Archived)
rec.Target = string(target)
}
actionlog.Write(rec)
if err != nil {
return nil, err
}
return res, nil
}
func (d *Daemon) freshLearningNamespace(now time.Time) (control.AntispamLearnFreshResult, error) {
// A pass may still be generating keys from before learning was disabled.
// Wait for it and its learner result before replacing the namespace.
d.antispamReconcileMu.Lock()
defer d.antispamReconcileMu.Unlock()
d.antispamLearnerMu.Lock()
defer d.antispamLearnerMu.Unlock()
var res control.AntispamLearnFreshResult
if d.currentCfg().EmailProtection.Antispam.Learning.Enabled || d.antispamLearner.Load() != nil {
return res, fmt.Errorf("%w: disable antispam learning first", errLearnRefused)
}
db := store.Global()
if db == nil {
return res, errors.New("the state database is not open")
}
// Move the keys first: an emptied ledger next to the old keys would let
// the next enable resume the old namespace without its records.
switch _, err := os.Lstat(learningDir); {
case err == nil:
archived := learningDir + ".old-" + now.UTC().Format("20060102T150405Z")
if _, statErr := os.Lstat(archived); !os.IsNotExist(statErr) {
return res, fmt.Errorf("%s already exists; retry in a second", archived)
}
if err = os.Rename(learningDir, archived); err != nil {
return res, fmt.Errorf("moving the learning directory aside: %w", err)
}
res.Archived = archived
case !os.IsNotExist(err):
return res, fmt.Errorf("checking the learning directory: %w", err)
}
restore := func(err error) (control.AntispamLearnFreshResult, error) {
if res.Archived != "" {
if restoreErr := os.Rename(res.Archived, learningDir); restoreErr != nil {
return res, errors.Join(err, fmt.Errorf("restoring the learning directory: %w", restoreErr))
}
res.Archived = ""
err = errors.Join(err, syncLearningParent())
}
return res, err
}
// Persist the move before clearing the ledger: a crash must not bring
// the old keys back beside an empty ledger.
if res.Archived != "" {
if err := syncLearningParent(); err != nil {
return restore(fmt.Errorf("persisting the archived learning directory: %w", err))
}
}
if err := db.ResetLearning(); err != nil {
return restore(fmt.Errorf("clearing learning state: %w", err))
}
return res, nil
}
func syncLearningParent() error {
f, err := os.Open(filepath.Dir(learningDir))
if err != nil {
return err
}
err = f.Sync()
return errors.Join(err, f.Close())
}
package daemon
import (
"context"
"encoding/json"
"errors"
"fmt"
"path/filepath"
"strings"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/antispam/learn"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/store"
)
// learnOperatorTimeout ends the wait before the CLI's own; the learner
// keeps the outcome under the request ID either way.
const learnOperatorTimeout = 4 * time.Minute
type learnOperator interface {
Operator(context.Context, learn.OperatorRequest) (learn.OperatorResult, error)
}
// antispamLearnOperator returns the running learner, or nil; tests replace
// it.
var antispamLearnOperator = func(d *Daemon) learnOperator {
if r := d.antispamLearner.Load(); r != nil {
return r.engine
}
return nil
}
// handleAntispamLearn runs csm antispam learn for one file. Every
// request is recorded in the action log, refusals and failures included,
// with its request ID and digests but never message content.
func (c *ControlListener) handleAntispamLearn(argsRaw json.RawMessage) (any, error) {
var args control.AntispamLearnArgs
parseErr := json.Unmarshal(argsRaw, &args)
name := args.Name
if name != "" {
name = filepath.Base(name)
}
target, _ := json.Marshal(name)
rec := actionlog.Record{Op: "integrate.antispam_learn", Action: args.Action, Actor: actionlog.CLI,
ActorDetail: "csm antispam learn", Target: string(target), Result: actionlog.Refused}
var res learn.OperatorResult
var err error
if parseErr != nil {
err = fmt.Errorf("%w: malformed learning arguments", errLearnRefused)
} else {
res, err = c.antispamLearn(args)
}
switch {
case err != nil:
if !errors.Is(err, errLearnRefused) {
rec.Result = actionlog.Failed
}
rec.Error = err.Error()
case res.ExitOK:
rec.Result = actionlog.Applied
case res.Outcome == learn.ResultDeferred:
rec.Error = "deferred: nothing changed"
default:
rec.Result = actionlog.Failed
rec.Error = "outcome " + res.Outcome
}
if res.Outcome == "" {
res.Outcome = string(rec.Result)
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
res.Outcome = learn.ResultPending
}
}
rec.Reason = learnReason(args.RequestID, res)
for _, d := range res.Digests {
rec.Digests = append(rec.Digests, string(d))
}
actionlog.Write(rec)
if err != nil {
return nil, err
}
out := control.AntispamLearnResult{RequestID: res.RequestID, Outcome: res.Outcome, ExitOK: res.ExitOK}
for _, cl := range res.Clusters {
out.Clusters = append(out.Clusters, control.AntispamLearnCluster{ClusterID: cl.ClusterID, Result: cl.Result})
}
for _, d := range res.Digests {
out.Digests = append(out.Digests, string(d))
}
return out, nil
}
var errLearnRefused = errors.New("antispam learning refused the request")
func (c *ControlListener) antispamLearn(args control.AntispamLearnArgs) (learn.OperatorResult, error) {
cfg := c.d.currentCfg()
switch {
case cfg.ObserveMode():
return learn.OperatorResult{}, fmt.Errorf("%w: observe mode does not learn", errLearnRefused)
case !cfg.EmailProtection.Antispam.Enabled || !cfg.EmailProtection.Antispam.Learning.Enabled:
return learn.OperatorResult{}, fmt.Errorf("%w: antispam learning is not enabled", errLearnRefused)
}
op := antispamLearnOperator(c.d)
if op == nil {
return learn.OperatorResult{}, fmt.Errorf("%w: antispam learning is not running; see the antispam_learning findings", errLearnRefused)
}
ctx, cancel := context.WithTimeout(context.Background(), learnOperatorTimeout)
defer cancel()
res, err := op.Operator(ctx, learn.OperatorRequest{Action: args.Action, RequestID: args.RequestID, Snapshot: args.Snapshot, SnapshotSHA: args.SnapshotSHA})
if errors.Is(err, learn.ErrLearningOff) {
err = fmt.Errorf("%w: learning prerequisites are not met; see the antispam_learning findings", errLearnRefused)
}
return res, err
}
// learnReason names the request, outcome and clusters. Full digests live
// outside Reason because the file sink bounds free-form explanations.
func learnReason(requestID string, res learn.OperatorResult) string {
var b strings.Builder
b.WriteString("request " + requestID)
if res.Outcome != "" {
b.WriteString("; outcome " + res.Outcome)
}
for _, cl := range res.Clusters {
if cl.ClusterID == 0 {
fmt.Fprintf(&b, "; part %s", cl.Result)
} else {
fmt.Fprintf(&b, "; cluster %d %s", cl.ClusterID, cl.Result)
}
}
return b.String()
}
// maxStatusRequests bounds one status query.
const maxStatusRequests = 100
// handleAntispamLearnStatus reads recorded outcomes of operator requests
// from the learning ledger, or lists the unfinished ones. It reads
// only, so it answers in any mode and with no learner running.
func (c *ControlListener) handleAntispamLearnStatus(argsRaw json.RawMessage) (any, error) {
var args control.AntispamLearnStatusArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if len(args.RequestIDs) > maxStatusRequests {
return nil, fmt.Errorf("at most %d request IDs per query", maxStatusRequests)
}
for _, id := range args.RequestIDs {
if len(id) != 32 || strings.Trim(id, "0123456789abcdef") != "" {
return nil, errors.New("a request ID is 32 lowercase hex digits")
}
}
db := store.Global()
if db == nil {
return nil, errors.New("the state database is not open")
}
statuses, err := learn.RequestStatus(learn.Open(db), args.RequestIDs)
if err != nil {
return nil, err
}
res := control.AntispamLearnStatusResult{Requests: []control.AntispamLearnStatus{}}
for _, s := range statuses {
out := control.AntispamLearnStatus{RequestID: s.RequestID, Action: s.Action, Found: s.Found, Done: s.Done, ExitOK: s.ExitOK, Outcome: s.Outcome}
for _, d := range s.Digests {
out.Digests = append(out.Digests, string(d))
}
for _, cl := range s.Clusters {
out.Clusters = append(out.Clusters, control.AntispamLearnStatusCluster{ClusterID: cl.ClusterID, Result: cl.Result, Phase: string(cl.Phase)})
}
res.Requests = append(res.Requests, out)
}
return res, nil
}
package daemon
import (
"encoding/json"
"fmt"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/integrity"
"github.com/pidginhost/csm/internal/store"
)
// handleBaseline clears existing state and captures the current host as
// the new known-good reference. Mirrors the old `csm baseline` flow but
// runs inside the daemon so no external lock coordination is needed.
//
// Concurrency: a sync.Mutex on the daemon serialises baselines against
// each other. The baseline sweep still uses checks.ForceAll to bypass
// throttles; dry-run state is scoped to RunAllDryRun.
func (c *ControlListener) handleBaseline(argsRaw json.RawMessage) (any, error) {
var args control.BaselineArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
c.d.baselineMu.Lock()
defer c.d.baselineMu.Unlock()
histCount := 0
if sdb := store.Global(); sdb != nil {
histCount = sdb.HistoryCount()
}
if histCount > 0 && !args.Confirm {
return control.BaselineResult{
HistoryCleared: histCount,
NeedsConfirm: true,
}, nil
}
// Force-all bypasses throttles for the baseline sweep. Dry-run threads
// through RunAllDryRun so a concurrent periodic scanner running in live
// mode is never silenced by this caller.
prevForceAll := checks.ForceAll
checks.ForceAll = true
defer func() { checks.ForceAll = prevForceAll }()
cfg := c.d.currentCfg()
findings, _ := checks.RunAllDryRun(cfg, c.d.store)
c.d.store.SetBaseline(findings)
binaryHash, err := integrity.HashFile(c.d.binaryPath)
if err != nil {
return nil, fmt.Errorf("hashing binary: %w", err)
}
signed, err := integrity.SignConfigFilePreservingSnapshot(cfg.ConfigFile, cfg.ConfigDir, binaryHash)
if err != nil {
return nil, fmt.Errorf("saving integrity: %w", err)
}
publishSignedBaselineConfig(cfg, signed)
return control.BaselineResult{
Findings: len(findings),
HistoryCleared: histCount,
BinaryHash: binaryHash,
ConfigHash: signed.Integrity.ConfigHash,
}, nil
}
func publishSignedBaselineConfig(live, signedMain *config.Config) {
resynced := *live
resynced.Integrity = signedMain.Integrity
// confd_hash was selected by the on-disk main config's exemption list.
// Publish both together without mutating the config other goroutines read.
resynced.ConfD = signedMain.ConfD
config.SetActive(&resynced)
}
package daemon
import (
"bytes"
"encoding/json"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/atomicio"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/health"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/obs"
)
var (
firewallDeadmanMu sync.Mutex
firewallDeadmanSeq atomic.Uint64
)
// Subnet / batch / meta firewall handlers: operations that either span
// multiple IPs (deny-file/allow-file, subnet ops) or reshape the whole
// ruleset (flush/restart/apply-confirmed/confirm). `restart` and
// `apply-confirmed` require a live fwEngine — a dead engine means
// "systemctl restart csm" rather than rebuild-from-handler.
func (c *ControlListener) handleFirewallDenySubnet(argsRaw json.RawMessage) (any, error) {
var args control.FirewallSubnetArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if _, _, err := net.ParseCIDR(args.CIDR); err != nil {
return nil, fmt.Errorf("invalid cidr: %q", args.CIDR)
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
reason := args.Reason
if reason == "" {
reason = "Blocked via CLI"
}
if err := c.d.fwEngine.BlockSubnet(args.CIDR, reason, 0); err != nil {
return nil, fmt.Errorf("block subnet %s: %w", args.CIDR, err)
}
return control.FirewallAckResult{
Message: fmt.Sprintf("Blocked subnet %s - %s", args.CIDR, reason),
}, nil
}
func (c *ControlListener) handleFirewallRemoveSubnet(argsRaw json.RawMessage) (any, error) {
var args control.FirewallSubnetArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if _, _, err := net.ParseCIDR(args.CIDR); err != nil {
return nil, fmt.Errorf("invalid cidr: %q", args.CIDR)
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
if err := c.d.fwEngine.UnblockSubnet(args.CIDR); err != nil {
return nil, fmt.Errorf("remove subnet %s: %w", args.CIDR, err)
}
return control.FirewallAckResult{
Message: fmt.Sprintf("Removed subnet block %s", args.CIDR),
}, nil
}
func (c *ControlListener) handleFirewallDenyFile(argsRaw json.RawMessage) (any, error) {
var args control.FirewallFileArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if len(args.IPs) == 0 {
return nil, fmt.Errorf("no ips in batch")
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
reason := args.Reason
if reason == "" {
reason = "Bulk block via CLI"
}
blocked, failed, skipped := 0, 0, 0
for _, ip := range args.IPs {
if net.ParseIP(ip) == nil {
skipped++
continue
}
// Operator-initiated batch: bypass auto_response.dry_run gate.
if err := operatorForceBlock(c.d.fwEngine, ip, reason, 0); err != nil {
failed++
continue
}
blocked++
}
msg := fmt.Sprintf("Blocked %d, skipped %d invalid", blocked, skipped)
if failed > 0 {
msg = fmt.Sprintf("Blocked %d, failed %d, skipped %d invalid", blocked, failed, skipped)
}
return control.FirewallAckResult{Message: msg}, nil
}
func (c *ControlListener) handleFirewallAllowFile(argsRaw json.RawMessage) (any, error) {
var args control.FirewallFileArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if len(args.IPs) == 0 {
return nil, fmt.Errorf("no ips in batch")
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
reason := args.Reason
if reason == "" {
reason = "Bulk allow via CLI"
}
allowed, failed, skipped := 0, 0, 0
for _, ip := range args.IPs {
if net.ParseIP(ip) == nil {
skipped++
continue
}
if err := c.d.fwEngine.AllowIP(ip, reason); err != nil {
failed++
continue
}
allowed++
}
msg := fmt.Sprintf("Allowed %d, skipped %d invalid", allowed, skipped)
if failed > 0 {
msg = fmt.Sprintf("Allowed %d, failed %d, skipped %d invalid", allowed, failed, skipped)
}
return control.FirewallAckResult{Message: msg}, nil
}
func (c *ControlListener) handleFirewallFlush(_ json.RawMessage) (any, error) {
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
cfg := c.d.currentCfg()
// Clear the auto-block bookkeeping for the flushed IPs; a surviving
// ThreatDB temp row re-blocks the IP on the next scan and silently
// undoes the flush.
result, err := checks.FlushAutoBlockState(cfg.StatePath, c.d.fwEngine.FlushBlocked)
if result.SnapshotErr != nil {
csmlog.Warn("firewall flush could not snapshot persisted blocks", "err", result.SnapshotErr)
}
if err != nil {
if result.Flushed {
return nil, fmt.Errorf("firewall flushed but auto-block cleanup failed: %w", err)
}
return nil, err
}
message := "Flushed blocked IPs (subnet blocks kept; use remove-subnet)"
if result.SnapshotErr == nil {
message = fmt.Sprintf("Flushed %d blocked IPs (subnet blocks kept; use remove-subnet)", result.BlockedCount)
}
return control.FirewallAckResult{
Message: message,
}, nil
}
func (c *ControlListener) handleFirewallRestart(_ json.RawMessage) (any, error) {
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall engine not running; restart the csm daemon")
}
firewallDeadmanMu.Lock()
defer firewallDeadmanMu.Unlock()
cfg := c.d.currentCfg()
confirmFile, _, _ := firewallRollbackFiles(cfg.StatePath)
if err := rejectPendingFirewallConfirmation(confirmFile); err != nil {
return nil, err
}
// Re-read the firewall block from disk: the engine holds the copy taken
// at daemon start, and a restart that re-applied it reported success
// while the operator's edit stayed unapplied.
previous, err := c.d.refreshFirewallFromDisk()
if err != nil {
return nil, err
}
if err := c.d.fwEngine.Apply(); err != nil {
c.d.fwEngine.SetConfig(previous)
return nil, fmt.Errorf("applying ruleset: %w", err)
}
state, _ := firewall.LoadState(cfg.StatePath)
return control.FirewallAckResult{
Message: fmt.Sprintf("Firewall restarted. %d blocked, %d allowed IPs restored.", len(state.Blocked), len(state.Allowed)),
}, nil
}
func (c *ControlListener) handleFirewallApplyConfirmed(argsRaw json.RawMessage) (any, error) {
var args control.FirewallApplyConfirmedArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
minutes := args.Minutes
if minutes <= 0 || minutes > 60 {
minutes = 3
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall engine not running; restart the csm daemon")
}
cfg := c.d.currentCfg()
confirmFile, rollbackFile, legacyRollbackFile := firewallRollbackFiles(cfg.StatePath)
// Read the edited firewall block only after the deadman has established
// that no other confirmation window owns the rollback files. The
// preparation and snapshot run under one lock so a rejected second
// attempt cannot replace the active window's engine rollback hook.
if err := applyFirewallDeadmanPrepared(confirmFile, rollbackFile, legacyRollbackFile,
time.Duration(minutes)*time.Minute, c.d.refreshFirewallFromDisk,
func(previous *firewall.FirewallConfig) { c.d.fwEngine.SetConfig(previous) },
c.d.fwEngine.Apply); err != nil {
return nil, err
}
state, _ := firewall.LoadState(cfg.StatePath)
return control.FirewallAckResult{
Message: fmt.Sprintf("Firewall applied with %d-minute rollback timer. %d blocked, %d allowed. Run `csm firewall confirm` to keep.", minutes, len(state.Blocked), len(state.Allowed)),
}, nil
}
func (c *ControlListener) handleFirewallConfirm(_ json.RawMessage) (any, error) {
firewallDeadmanMu.Lock()
defer firewallDeadmanMu.Unlock()
cfg := c.d.currentCfg()
confirmFile, rollbackFile, legacyRollbackFile := firewallRollbackFiles(cfg.StatePath)
if _, err := os.Stat(confirmFile); err != nil {
if !os.IsNotExist(err) {
return nil, fmt.Errorf("checking confirm marker: %w", err)
}
if cleanupErr := settleFirewallDeadmanLocked(confirmFile, rollbackFile, legacyRollbackFile); cleanupErr != nil {
return nil, cleanupErr
}
return control.FirewallAckResult{
Message: "No pending confirmation. Firewall is already confirmed.",
}, nil
}
// Confirmation accepts the current engine configuration, even when
// startup could not recover the old configuration.
if err := settleFirewallDeadmanLocked(confirmFile, rollbackFile, legacyRollbackFile); err != nil {
return nil, err
}
return control.FirewallAckResult{
Message: "Firewall confirmed. Rollback timer cancelled.",
}, nil
}
// applyFirewallDeadman runs the tentative-apply protocol: snapshot the live
// ruleset, persist the confirm deadline, then apply the candidate. The
// marker is written BEFORE the kernel apply on purpose: a crash between the
// two leaves a valid deadline on disk for startup recovery to settle,
// whereas the reverse order would leave the candidate applied with no
// record that it was never confirmed (a permanent lockout).
func applyFirewallDeadman(confirmFile, rollbackFile, legacyRollbackFile string, window time.Duration, apply func() error) error {
return applyFirewallDeadmanPrepared(confirmFile, rollbackFile, legacyRollbackFile, window, nil, nil, apply)
}
// applyFirewallDeadmanPrepared checks ownership before prepare installs the
// candidate engine configuration. restore reverses that installation on any
// pre-apply failure and becomes the live rollback hook once the window is
// armed. Both callbacks run while firewallDeadmanMu is held.
func applyFirewallDeadmanPrepared(confirmFile, rollbackFile, legacyRollbackFile string, window time.Duration,
prepare func() (*firewall.FirewallConfig, error), restore func(*firewall.FirewallConfig), apply func() error,
) error {
firewallDeadmanMu.Lock()
defer firewallDeadmanMu.Unlock()
if err := os.MkdirAll(filepath.Dir(rollbackFile), 0700); err != nil {
return fmt.Errorf("creating firewall rollback dir: %w", err)
}
if err := rejectPendingFirewallConfirmation(confirmFile); err != nil {
return err
}
firewallDeadmanConfigErr = nil
// Preparation can swap engine configuration; it shares the window hold.
holdFirewallBatchesLocked(true)
var rollbackConfig *firewall.FirewallConfig
if prepare != nil {
var err error
rollbackConfig, err = prepare()
if err != nil {
holdFirewallBatchesLocked(false)
return err
}
firewallDeadmanOnRollback = func() { restore(rollbackConfig) }
}
// A batch in flight finished before preparation and lands in the
// snapshot; none runs until the window ends.
restorePrepared := func() {
if prepare != nil {
runFirewallDeadmanOnRollbackLocked()
}
if _, err := os.Stat(confirmFile); os.IsNotExist(err) {
holdFirewallBatchesLocked(false)
}
}
if err := removeFirewallRollbackFiles(rollbackFile, legacyRollbackFile); err != nil {
restorePrepared()
return err
}
if err := writeFirewallRollbackFile(rollbackFile); err != nil {
restorePrepared()
return err
}
if err := snapshotFirewallConfig(rollbackFile, rollbackConfig); err != nil {
_ = removeFirewallRollbackFiles(rollbackFile)
restorePrepared()
return err
}
deadline := time.Now().Add(window)
marker := newFirewallConfirmMarker(deadline)
if err := os.WriteFile(confirmFile, marker, 0600); err != nil {
_ = removeFirewallRollbackFiles(confirmFile, rollbackFile)
restorePrepared()
return fmt.Errorf("writing confirm marker: %w", err)
}
if err := apply(); err != nil {
// Apply may have partially mutated the kernel; re-applying the
// snapshot is a no-op when it did not, and the undo when it did.
if restoreErr := applyFirewallRollbackFile(rollbackFile); restoreErr != nil {
// Keep marker + snapshot: the deadline stands, so the armed
// deadman retries the restore at expiry (and startup recovery
// does the same after a crash).
armFirewallDeadman(confirmFile, rollbackFile, marker, time.Until(deadline))
return fmt.Errorf("applying ruleset: %w; rollback restore failed: %v", err, restoreErr)
}
if cleanupErr := removeFirewallRollbackFiles(confirmFile, rollbackFile); cleanupErr != nil {
restorePrepared()
return fmt.Errorf("applying ruleset: %w; %v", err, cleanupErr)
}
restorePrepared()
return fmt.Errorf("applying ruleset: %w; previous ruleset restored", err)
}
armFirewallDeadman(confirmFile, rollbackFile, marker, time.Until(deadline))
return nil
}
// rejectPendingFirewallConfirmation refuses a second change while a window
// is pending, naming only what settles it: waiting helps only while this
// process has a rollback scheduled that can settle the saved configuration.
// Caller holds firewallDeadmanMu.
func rejectPendingFirewallConfirmation(confirmFile string) error {
marker, err := os.ReadFile(confirmFile) // #nosec G304 -- CSM-owned marker under the state dir.
switch {
case os.IsNotExist(err):
return nil
case err != nil:
return fmt.Errorf("checking confirm marker: %w", err)
}
if deadline, parseErr := parseFirewallConfirmDeadline(marker); parseErr == nil && len(marker) > 0 && bytes.Equal(firewallDeadmanArmed, marker) && firewallDeadmanConfigErr == nil {
return fmt.Errorf("firewall confirmation already pending; run `csm firewall confirm` to keep the change, or wait for its rollback at %s, before applying another ruleset", deadline.UTC().Format(time.RFC3339))
}
return fmt.Errorf("firewall confirmation already pending; inspect the firewall, then run `csm firewall confirm` to accept the current state")
}
// firewallConfirmationStatus is the pending window under statePath for
// status and doctor, or nil.
func firewallConfirmationStatus(statePath string) *health.FirewallConfirmation {
firewallDeadmanMu.Lock()
defer firewallDeadmanMu.Unlock()
confirmFile, rollbackFile, _ := firewallRollbackFiles(statePath)
marker, err := os.ReadFile(confirmFile) // #nosec G304 -- CSM-owned marker under the state dir.
switch {
case os.IsNotExist(err):
return nil
case err != nil:
return &health.FirewallConfirmation{Error: "its confirmation marker cannot be read: " + err.Error()}
}
w := &health.FirewallConfirmation{}
w.Deadline, _ = parseFirewallConfirmDeadline(marker)
armed := len(marker) > 0 && bytes.Equal(firewallDeadmanArmed, marker)
switch {
case armed && firewallDeadmanConfigErr != nil:
w.Error = "its saved configuration cannot be read, so its rollback restores the ruleset only: " + firewallDeadmanConfigErr.Error()
case firewallDeadmanConfigErr != nil && firewallRestoreRecorded(rollbackFile, marker):
w.Error = "the previous ruleset was restored, but not its configuration: " + firewallDeadmanConfigErr.Error()
case firewallDeadmanConfigErr != nil:
w.Error = "rollback has not completed and no rollback is scheduled: " + firewallDeadmanConfigErr.Error()
case !armed:
w.Error = "no rollback is scheduled for it"
}
return w
}
// armFirewallDeadman restores the pre-apply ruleset once wait elapses unless
// the operator confirms first. The goroutine lives in the daemon (long-lived,
// so it survives CLI exit); a daemon restart kills it, which is why
// recoverFirewallApplyConfirmed re-arms from the persisted deadline.
// firewallDeadmanOnRollback runs after a rollback restores the kernel
// snapshot (deadline expiry or explicit revert), so the engine's ruleset
// input reverts with it; otherwise the next Apply would rebuild the rules
// the operator never confirmed. Cleared on confirm. Guarded by
// firewallDeadmanMu except in setFirewallDeadmanOnRollback, which takes it.
var firewallDeadmanOnRollback func()
func setFirewallDeadmanOnRollback(fn func()) {
firewallDeadmanMu.Lock()
defer firewallDeadmanMu.Unlock()
firewallDeadmanOnRollback = fn
firewallDeadmanConfigErr = nil
if fn == nil {
firewallDeadmanArmed = nil
}
}
// firewallBatchHold holds or releases admission's block batches on the
// engine: a window holds them from before its snapshot until
// it is confirmed or rolled back, so a rollback cannot erase a row the
// owner wrote inside it. Guarded by firewallDeadmanMu.
var firewallBatchHold func(bool)
// firewallRowGate gates the engine's saved rows on the ledger again. A
// rollback restores state.json as it was when its window opened, without
// the marks and removals the ledger asked for since, so each restore runs
// it before admission is released. Guarded by firewallDeadmanMu.
var firewallRowGate func()
// firewallRetime retimes the engine's blocked sets from its saved rows. A
// rollback restores the kernel ruleset from a dump, which gives each timed
// block the time it had left when the window opened. firewallRetimeDue
// records such a restore; the retiming runs when admission is released,
// after configuration is restored or explicitly accepted by confirmation.
// Both are guarded by firewallDeadmanMu.
var (
firewallRetime func()
firewallRetimeDue bool
)
// setFirewallRestoreHooks installs what follows a rollback's restore: gate
// for the saved rows and retime for the blocked sets.
func setFirewallRestoreHooks(gate, retime func()) {
firewallDeadmanMu.Lock()
defer firewallDeadmanMu.Unlock()
firewallRowGate, firewallRetime = gate, retime
firewallRetimeDue = false
}
// An unreadable configuration proof cannot release admission after
// only the kernel snapshot was restored. Guarded by firewallDeadmanMu.
var firewallDeadmanConfigErr error
// firewallDeadmanArmed is the marker of the window whose rollback this
// process has scheduled; a later window's marker never equals it. Guarded
// by firewallDeadmanMu.
var firewallDeadmanArmed []byte
// settleFirewallDeadmanLocked disarms rollback before clearing the hold.
// Snapshot cleanup can fail after the marker is gone; debris must not keep
// admission held without a pending window. Caller holds firewallDeadmanMu.
func settleFirewallDeadmanLocked(confirmFile, rollbackFile, legacyRollbackFile string) error {
if err := removeFileIfExists(confirmFile); err != nil {
return err
}
firewallDeadmanOnRollback = nil
firewallDeadmanConfigErr = nil
firewallDeadmanArmed = nil
holdFirewallBatchesLocked(false)
return removeFirewallRollbackFiles(rollbackFile, legacyRollbackFile)
}
func setFirewallBatchHold(fn func(bool)) {
firewallDeadmanMu.Lock()
defer firewallDeadmanMu.Unlock()
firewallBatchHold = fn
}
// holdFirewallBatchesLocked: caller holds firewallDeadmanMu.
func holdFirewallBatchesLocked(held bool) {
if !held && firewallRetimeDue {
firewallRetimeDue = false
if firewallRetime != nil {
firewallRetime()
}
}
if firewallBatchHold != nil {
firewallBatchHold(held)
}
}
// holdFirewallBatchesForWindow installs engine's batch hold and holds it
// while a window a restart interrupted is pending, or its marker cannot be
// read; startup recovery then settles the window.
func holdFirewallBatchesForWindow(statePath string, engine *firewall.Engine) {
firewallDeadmanMu.Lock()
defer firewallDeadmanMu.Unlock()
firewallBatchHold = engine.HoldBlockBatches
confirmFile, _, _ := firewallRollbackFiles(statePath)
if _, err := os.Stat(confirmFile); !os.IsNotExist(err) {
engine.HoldBlockBatches(true)
}
}
// runFirewallDeadmanOnRollbackLocked invokes and clears the rollback hook.
// Caller holds firewallDeadmanMu.
func runFirewallDeadmanOnRollbackLocked() {
if firewallDeadmanOnRollback != nil {
firewallDeadmanOnRollback()
firewallDeadmanOnRollback = nil
}
}
// armFirewallDeadman: caller holds firewallDeadmanMu.
func armFirewallDeadman(confirmFile, rollbackFile string, marker []byte, wait time.Duration) {
firewallDeadmanArmed = marker
obs.SafeGo("fw-apply-confirmed-rollback", func() {
time.Sleep(wait)
firewallDeadmanMu.Lock()
defer firewallDeadmanMu.Unlock()
if !bytes.Equal(firewallDeadmanArmed, marker) {
return
}
firewallDeadmanArmed = nil
if err := restoreFirewallRollback(confirmFile, rollbackFile, marker); err != nil {
fmt.Fprintf(os.Stderr, "[%s] Firewall rollback failed: %v\n", ts(), err)
}
})
}
// recoverFirewallApplyConfirmed settles an apply-confirmed window that a
// daemon restart interrupted. Without it the restart kills the deadman
// goroutine and startFirewall re-applies the unconfirmed candidate, making
// permanent exactly the lockout the command exists to prevent. Must run
// after startFirewall so an expired-window restore lands on top of the
// candidate ruleset the startup Apply just re-applied, and before the
// control listener starts so confirm/cancel cannot race the recovery.
func (d *Daemon) recoverFirewallApplyConfirmed() {
firewallDeadmanMu.Lock()
defer firewallDeadmanMu.Unlock()
firewallDeadmanArmed = nil
firewallDeadmanOnRollback = nil
confirmFile, rollbackFile, legacyRollbackFile := firewallRollbackFiles(d.cfg.StatePath)
marker, err := os.ReadFile(confirmFile) // #nosec G304 -- CSM-owned marker under the state dir.
if err != nil {
if !os.IsNotExist(err) {
csmlog.Warn("firewall confirm marker unreadable; leaving apply-confirmed state untouched", "err", err)
return
}
// No marker means no candidate reached the kernel with a pending
// window (the marker is written before the apply), so a leftover
// snapshot is debris from an aborted handler run.
if cleanupErr := settleFirewallDeadmanLocked(confirmFile, rollbackFile, legacyRollbackFile); cleanupErr != nil {
csmlog.Warn("firewall rollback snapshot cleanup failed", "err", cleanupErr)
}
return
}
firewallDeadmanConfigErr = installFirewallConfigRollbackHook(d, rollbackFile)
if firewallDeadmanConfigErr != nil {
csmlog.Warn("firewall configuration rollback snapshot unreadable", "err", firewallDeadmanConfigErr)
}
if firewallRestoreRecorded(rollbackFile, marker) {
firewallDeadmanArmed = nil
d.settleRestoredFirewallWindowLocked(confirmFile, rollbackFile, legacyRollbackFile)
return
}
deadline, parseErr := parseFirewallConfirmDeadline(marker)
if parseErr != nil {
// Fail safe: an unconfirmed ruleset must never outlive its
// deadline, and a corrupt marker gives no deadline to honour.
csmlog.Warn("firewall confirm marker corrupt; restoring previous ruleset", "err", parseErr)
if restoreErr := restoreFirewallRollback(confirmFile, rollbackFile, marker); restoreErr != nil {
csmlog.Warn("firewall rollback failed", "err", restoreErr)
}
return
}
if remaining := time.Until(deadline); remaining > 0 {
// startFirewall already re-applied the candidate, matching the
// kernel state from before the restart, so the operator keeps the
// verification window they asked for; the re-armed deadman still
// bounds it with the original deadline.
armFirewallDeadman(confirmFile, rollbackFile, marker, remaining)
csmlog.Info("firewall apply-confirmed window resumed after restart",
"deadline", deadline.Format(time.RFC3339))
return
}
if restoreErr := restoreFirewallRollback(confirmFile, rollbackFile, marker); restoreErr != nil {
csmlog.Warn("firewall rollback failed", "err", restoreErr)
return
}
csmlog.Warn("firewall apply-confirmed window expired during restart; previous ruleset restored",
"deadline", deadline.Format(time.RFC3339))
}
func newFirewallConfirmMarker(deadline time.Time) []byte {
seq := firewallDeadmanSeq.Add(1)
return []byte(fmt.Sprintf("%s\n%d", deadline.UTC().Format(time.RFC3339Nano), seq))
}
func parseFirewallConfirmDeadline(marker []byte) (time.Time, error) {
text := strings.TrimSpace(string(marker))
if i := strings.IndexByte(text, '\n'); i >= 0 {
text = strings.TrimSpace(text[:i])
}
return time.Parse(time.RFC3339Nano, text)
}
func firewallRollbackFiles(statePath string) (confirmFile, rollbackFile, legacyRollbackFile string) {
firewallDir := filepath.Join(statePath, "firewall")
return filepath.Join(firewallDir, "confirm_pending"),
filepath.Join(firewallDir, "rollback.nft"),
filepath.Join(firewallDir, "rollback.sh")
}
func writeFirewallRollbackFile(rollbackFile string) error {
// #nosec G204 -- "nft list ruleset" is literal.
nftDump, err := exec.Command("nft", "list", "ruleset").Output()
if err != nil {
return fmt.Errorf("capturing rollback ruleset: %w", err)
}
// The dump is nft syntax, so store it as data consumed by nft -f.
// An empty live ruleset still needs a rollback file; flush ruleset
// restores that state.
payload := make([]byte, 0, len("flush ruleset\n")+len(nftDump)+1)
payload = append(payload, "flush ruleset\n"...)
payload = append(payload, nftDump...)
if len(nftDump) > 0 && nftDump[len(nftDump)-1] != '\n' {
payload = append(payload, '\n')
}
// #nosec G306 -- root-only state dir; this is data, not an executable.
if err := os.WriteFile(rollbackFile, payload, 0600); err != nil {
_ = removeFileIfExists(rollbackFile)
return fmt.Errorf("writing rollback ruleset: %w", err)
}
if err := snapshotFirewallState(rollbackFile); err != nil {
_ = removeFileIfExists(rollbackFile)
return err
}
return nil
}
func restoreFirewallRollback(confirmFile, rollbackFile string, expectedMarker []byte) error {
raw, err := os.ReadFile(confirmFile) // #nosec G304 -- CSM-owned marker under the state dir.
if err != nil {
if os.IsNotExist(err) {
firewallDeadmanOnRollback = nil
firewallDeadmanConfigErr = nil
firewallDeadmanArmed = nil
holdFirewallBatchesLocked(false)
return nil
}
return fmt.Errorf("checking confirm marker: %w", err)
}
// A different marker means a newer apply-confirmed window owns the
// files now; this caller's window was superseded and firing would
// roll the newer window back before its own deadline.
if !bytes.Equal(raw, expectedMarker) {
return nil
}
if firewallRestoreRecorded(rollbackFile, raw) {
return fmt.Errorf("rollback already restored this window; confirm the current state or restart after repairing its saved configuration")
}
if err := applyFirewallRollbackFile(rollbackFile); err != nil {
return err
}
if firewallDeadmanConfigErr != nil {
// Restore once: a later restart keeps what
// changed since instead of restoring the window's start again.
if err := recordFirewallRestore(rollbackFile, raw); err != nil {
return fmt.Errorf("restoring firewall configuration: %w; recording the restored ruleset: %v", firewallDeadmanConfigErr, err)
}
return fmt.Errorf("restoring firewall configuration: %w", firewallDeadmanConfigErr)
}
// The kernel is back on the snapshot; put the engine's ruleset input
// back too, or the next Apply rebuilds the unconfirmed candidate.
runFirewallDeadmanOnRollbackLocked()
return settleFirewallDeadmanLocked(confirmFile, rollbackFile, legacyRollbackFileFor(rollbackFile))
}
func applyFirewallRollbackFile(rollbackFile string) (resultErr error) {
rec := actionlog.Record{Op: "operate.manual_firewall", Action: "rollback", Target: rollbackFile, Command: []string{"nft", "-f", rollbackFile}, Result: actionlog.Failed}
defer func() {
if resultErr != nil {
rec.Error = resultErr.Error()
}
actionlog.Write(rec)
}()
if _, err := os.Stat(rollbackFile); err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("rollback ruleset missing")
}
return fmt.Errorf("checking rollback ruleset: %w", err)
}
// #nosec G204 -- nft is hardcoded; rollbackFile is a CSM-written path.
out, err := exec.Command("nft", "-f", rollbackFile).CombinedOutput()
if err != nil {
out = bytes.TrimSpace(out)
if len(out) > 0 {
return fmt.Errorf("restoring rollback ruleset: %w: %s", err, out)
}
return fmt.Errorf("restoring rollback ruleset: %w", err)
}
rec.Result = actionlog.Applied
// Kernel is back on the snapshot; state.json must follow, or the UI
// keeps describing the window's mutations as live.
restored, err := restoreFirewallStateSnapshot(rollbackFile)
if err != nil {
return err
}
if !restored {
return nil
}
if firewallRowGate != nil {
firewallRowGate()
}
firewallRetimeDue = true
return nil
}
func removeFirewallRollbackFiles(paths ...string) error {
for _, path := range paths {
if err := removeFileIfExists(path); err != nil {
return err
}
// A confirmed or superseded window drops its state snapshot with
// its ruleset snapshot.
if filepath.Base(path) == "rollback.nft" {
for _, snapshot := range []string{firewallStateSnapshotPath(path), firewallConfigSnapshotPath(path), firewallRestoredPath(path)} {
if err := removeFileIfExists(snapshot); err != nil {
return err
}
}
}
}
return nil
}
func firewallConfigSnapshotPath(rollbackFile string) string {
return rollbackFile + ".config.json"
}
// firewallRestoredPath records that the window named by its content had its
// kernel and state restored, but not its configuration.
func firewallRestoredPath(rollbackFile string) string {
return rollbackFile + ".restored"
}
// firewallRestoreRecorded reports whether the window marker names already
// had its kernel and state restored.
func firewallRestoreRecorded(rollbackFile string, marker []byte) bool {
raw, err := os.ReadFile(firewallRestoredPath(rollbackFile)) // #nosec G304 -- CSM-owned record under the state dir.
return err == nil && bytes.Equal(raw, firewallRestoreProof(marker))
}
// firewallRestoreProof stays nonempty even when the corrupt confirmation
// marker is empty, so an empty sidecar can never claim completion.
func firewallRestoreProof(marker []byte) []byte {
return append([]byte("restored\n"), marker...)
}
// settleRestoredFirewallWindowLocked handles a window whose kernel and state
// were restored once already: the blocks added since stay. With its
// configuration still unreadable it stays pending and held until
// confirmation; a repaired one is restored and the ruleset rebuilt on the
// current state. Caller holds firewallDeadmanMu.
func (d *Daemon) settleRestoredFirewallWindowLocked(confirmFile, rollbackFile, legacyRollbackFile string) {
if firewallDeadmanConfigErr == nil && firewallDeadmanOnRollback == nil {
firewallDeadmanConfigErr = fmt.Errorf("saved firewall configuration is missing")
}
if firewallDeadmanConfigErr != nil {
csmlog.Warn("firewall apply-confirmed window was rolled back without its configuration; run csm firewall confirm", "err", firewallDeadmanConfigErr)
return
}
runFirewallDeadmanOnRollbackLocked()
if err := applyFirewallEngine(d); err != nil {
firewallDeadmanConfigErr = fmt.Errorf("applying the restored configuration: %w", err)
csmlog.Warn("firewall restored configuration could not be applied", "err", err)
return
}
if err := settleFirewallDeadmanLocked(confirmFile, rollbackFile, legacyRollbackFile); err != nil {
csmlog.Warn("firewall rollback snapshot cleanup failed", "err", err)
}
}
// applyFirewallEngine rebuilds the ruleset from the engine's configuration
// and current state; tests replace it.
var applyFirewallEngine = func(d *Daemon) error {
if d.fwEngine == nil {
return fmt.Errorf("firewall engine is unavailable")
}
return d.fwEngine.Apply()
}
func snapshotFirewallConfig(rollbackFile string, cfg *firewall.FirewallConfig) error {
if cfg == nil {
return nil
}
if err := atomicio.AtomicWriteJSON(firewallConfigSnapshotPath(rollbackFile), 0o600, cfg); err != nil {
return fmt.Errorf("snapshotting firewall configuration: %w", err)
}
return nil
}
var restoreFirewallEngineConfig = func(d *Daemon, cfg *firewall.FirewallConfig) {
if d.fwEngine != nil {
d.fwEngine.SetConfig(cfg)
}
}
// installFirewallConfigRollbackHook restores the engine input after the
// kernel and state snapshots. The persisted copy matters only across daemon
// restart; the original process keeps an equivalent in-memory hook.
func installFirewallConfigRollbackHook(d *Daemon, rollbackFile string) error {
raw, err := os.ReadFile(firewallConfigSnapshotPath(rollbackFile)) // #nosec G304 -- CSM-owned snapshot under the state dir.
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
var cfg firewall.FirewallConfig
if err := json.Unmarshal(raw, &cfg); err != nil {
return fmt.Errorf("decoding snapshot: %w", err)
}
firewallDeadmanOnRollback = func() { restoreFirewallEngineConfig(d, &cfg) }
return nil
}
func legacyRollbackFileFor(rollbackFile string) string {
return filepath.Join(filepath.Dir(rollbackFile), "rollback.sh")
}
func removeFileIfExists(path string) error {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("removing %s: %w", filepath.Base(path), err)
}
return nil
}
// recordFirewallRestore publishes completion only after state.json and
// its directory are synced. The state restore can also remove that file.
func recordFirewallRestore(rollbackFile string, marker []byte) error {
path := firewallStateFileFor(rollbackFile)
// #nosec G304 -- CSM-owned state under the rollback directory.
f, err := os.Open(path)
if err != nil && !os.IsNotExist(err) {
return err
}
if err == nil {
syncErr := f.Sync()
closeErr := f.Close()
if syncErr != nil {
return syncErr
}
if closeErr != nil {
return closeErr
}
}
// #nosec G304 -- parent of the same CSM-owned state file.
dir, err := os.Open(filepath.Dir(path))
if err != nil {
return err
}
syncErr := dir.Sync()
closeErr := dir.Close()
if syncErr != nil {
return syncErr
}
if closeErr != nil {
return closeErr
}
return atomicio.AtomicWrite(firewallRestoredPath(rollbackFile), 0o600, firewallRestoreProof(marker))
}
package daemon
import (
"encoding/json"
"fmt"
"net"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/firewall"
)
// Single-IP firewall mutation handlers. Each validates args, guards on
// c.d.fwEngine != nil, calls the matching engine method, and returns a
// FirewallAckResult with a human-readable message the CLI prints verbatim.
// operatorForceBlock runs an operator-initiated force block and reports it
// to the shared firewall outcome metric alongside auto-response blocks.
func operatorForceBlock(e interface {
BlockIPForce(ip string, reason string, timeout time.Duration) error
}, ip, reason string, timeout time.Duration) error {
err := e.BlockIPForce(ip, reason, timeout)
checks.ObserveOperatorBlock(err, checks.BlockSourceCLI)
return err
}
func (c *ControlListener) handleFirewallBlock(argsRaw json.RawMessage) (any, error) {
var args control.FirewallIPArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if net.ParseIP(args.IP) == nil {
return nil, fmt.Errorf("invalid ip: %q", args.IP)
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
reason := args.Reason
if reason == "" {
reason = "Blocked via CLI"
}
// Operator-initiated: bypass auto_response.dry_run gate.
if err := operatorForceBlock(c.d.fwEngine, args.IP, reason, 0); err != nil {
return nil, fmt.Errorf("block %s: %w", args.IP, err)
}
msg := fmt.Sprintf("Blocked %s - %s", args.IP, reason)
msg += cloudflareCoverageSuffix(c.d.fwEngine, args.IP)
return control.FirewallAckResult{Message: msg}, nil
}
// cloudflareCoverageSuffix warns the operator when a just-blocked IP sits
// inside a Cloudflare allow range: the input chain accepts CF edges on TCP
// 80/443 before the blocked drop, so the block does not stop web traffic.
func cloudflareCoverageSuffix(e interface{ CloudflareCovers(string) bool }, ip string) string {
if e != nil && e.CloudflareCovers(ip) {
return " (warning: " + firewall.CloudflareCoverageWarning + ")"
}
return ""
}
func (c *ControlListener) handleFirewallUnblock(argsRaw json.RawMessage) (any, error) {
var args control.FirewallIPArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if net.ParseIP(args.IP) == nil {
return nil, fmt.Errorf("invalid ip: %q", args.IP)
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
if err := c.d.fwEngine.UnblockIP(args.IP); err != nil {
return nil, fmt.Errorf("unblock %s: %w", args.IP, err)
}
checks.DropAutoBlockThreatRow(args.IP)
msg := fmt.Sprintf("Unblocked %s", args.IP)
// A blocked subnet covering the address keeps dropping it whatever
// happens to the per-IP element; say so instead of reporting success.
if cidr, covered := c.d.fwEngine.BlockedSubnetCovering(args.IP); covered {
msg += fmt.Sprintf(" (still dropped by blocked subnet %s; unblock that subnet to restore access)", cidr)
}
return control.FirewallAckResult{Message: msg}, nil
}
func (c *ControlListener) handleFirewallAllow(argsRaw json.RawMessage) (any, error) {
var args control.FirewallIPArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if net.ParseIP(args.IP) == nil {
return nil, fmt.Errorf("invalid ip: %q", args.IP)
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
reason := args.Reason
if reason == "" {
reason = "Allowed via CLI"
}
if err := c.d.fwEngine.AllowIP(args.IP, reason); err != nil {
return nil, fmt.Errorf("allow %s: %w", args.IP, err)
}
msg := fmt.Sprintf("Allowed %s - %s", args.IP, reason)
if cidr, covered := c.d.fwEngine.BlockedSubnetCovering(args.IP); covered {
msg += fmt.Sprintf(" (WARNING: still dropped by blocked subnet %s; unblock the subnet for this allow to take effect)", cidr)
}
return control.FirewallAckResult{Message: msg}, nil
}
func (c *ControlListener) handleFirewallRemoveAllow(argsRaw json.RawMessage) (any, error) {
var args control.FirewallIPArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if net.ParseIP(args.IP) == nil {
return nil, fmt.Errorf("invalid ip: %q", args.IP)
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
if err := c.d.fwEngine.RemoveAllowIP(args.IP); err != nil {
return nil, fmt.Errorf("remove-allow %s: %w", args.IP, err)
}
return control.FirewallAckResult{Message: fmt.Sprintf("Removed %s from allow list", args.IP)}, nil
}
func (c *ControlListener) handleFirewallAllowPort(argsRaw json.RawMessage) (any, error) {
var args control.FirewallPortArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if net.ParseIP(args.IP) == nil {
return nil, fmt.Errorf("invalid ip: %q", args.IP)
}
if args.Port <= 0 || args.Port > 65535 {
return nil, fmt.Errorf("invalid port: %d", args.Port)
}
proto := args.Proto
if proto == "" {
proto = "tcp"
}
if proto != "tcp" && proto != "udp" {
return nil, fmt.Errorf("invalid proto: %q (want tcp or udp)", args.Proto)
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
reason := args.Reason
if reason == "" {
reason = "Port-allowed via CLI"
}
if err := c.d.fwEngine.AllowIPPort(args.IP, args.Port, proto, reason); err != nil {
return nil, fmt.Errorf("allow-port %s %s:%d: %w", args.IP, proto, args.Port, err)
}
return control.FirewallAckResult{
Message: fmt.Sprintf("Saved allow for %s on %s:%d - %s; takes effect after firewall reload", args.IP, proto, args.Port, reason),
}, nil
}
func (c *ControlListener) handleFirewallRemovePort(argsRaw json.RawMessage) (any, error) {
var args control.FirewallPortArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if net.ParseIP(args.IP) == nil {
return nil, fmt.Errorf("invalid ip: %q", args.IP)
}
if args.Port <= 0 || args.Port > 65535 {
return nil, fmt.Errorf("invalid port: %d", args.Port)
}
proto := args.Proto
if proto == "" {
proto = "tcp"
}
if proto != "tcp" && proto != "udp" {
return nil, fmt.Errorf("invalid proto: %q (want tcp or udp)", args.Proto)
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
if err := c.d.fwEngine.RemoveAllowIPPort(args.IP, args.Port, proto); err != nil {
return nil, fmt.Errorf("remove-port %s %s:%d: %w", args.IP, proto, args.Port, err)
}
return control.FirewallAckResult{
Message: fmt.Sprintf("Saved port-allow removal for %s on %s:%d; takes effect after firewall reload", args.IP, proto, args.Port),
}, nil
}
func (c *ControlListener) handleFirewallTempBan(argsRaw json.RawMessage) (any, error) {
var args control.FirewallIPArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
// Parse timeout FIRST so callers get a duration-parse error before
// the engine-nil check (the unit test depends on this ordering).
if args.Timeout == "" {
return nil, fmt.Errorf("tempban requires timeout")
}
timeout, err := time.ParseDuration(args.Timeout)
if err != nil {
return nil, fmt.Errorf("parsing duration %q: %w", args.Timeout, err)
}
if net.ParseIP(args.IP) == nil {
return nil, fmt.Errorf("invalid ip: %q", args.IP)
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
reason := args.Reason
if reason == "" {
reason = "Temp-banned via CLI"
}
// Operator-initiated: bypass auto_response.dry_run gate.
if err := operatorForceBlock(c.d.fwEngine, args.IP, reason, timeout); err != nil {
return nil, fmt.Errorf("tempban %s: %w", args.IP, err)
}
msg := fmt.Sprintf("Temp-banned %s for %s - %s", args.IP, timeout, reason)
msg += cloudflareCoverageSuffix(c.d.fwEngine, args.IP)
return control.FirewallAckResult{Message: msg}, nil
}
func (c *ControlListener) handleFirewallTempAllow(argsRaw json.RawMessage) (any, error) {
var args control.FirewallIPArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if args.Timeout == "" {
return nil, fmt.Errorf("tempallow requires timeout")
}
timeout, err := time.ParseDuration(args.Timeout)
if err != nil {
return nil, fmt.Errorf("parsing duration %q: %w", args.Timeout, err)
}
if net.ParseIP(args.IP) == nil {
return nil, fmt.Errorf("invalid ip: %q", args.IP)
}
if c.d.fwEngine == nil {
return nil, fmt.Errorf("firewall disabled in csm.yaml")
}
reason := args.Reason
if reason == "" {
reason = "Temp-allowed via CLI"
}
if err := c.d.fwEngine.TempAllowIP(args.IP, reason, timeout); err != nil {
return nil, fmt.Errorf("tempallow %s: %w", args.IP, err)
}
return control.FirewallAckResult{
Message: fmt.Sprintf("Temp-allowed %s for %s - %s", args.IP, timeout, reason),
}, nil
}
package daemon
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/firewall"
)
// Read-only firewall handlers: no state mutation, just surface what
// firewall.LoadState already has.
// fmtPortsSlice converts a slice of int ports into a slice of strings,
// one entry per port. Mirrors the atomic rendering the wire schema
// expects — the CLI joins them with commas for display. Empty input
// returns nil so the JSON wire form is `null` / omitted rather than an
// empty array carrying a placeholder.
func fmtPortsSlice(ports []int) []string {
if len(ports) == 0 {
return nil
}
out := make([]string, len(ports))
for i, p := range ports {
out[i] = strconv.Itoa(p)
}
return out
}
func (c *ControlListener) firewallReadConfig(cfg *config.Config) *firewall.FirewallConfig {
// Re-apply changes only the engine snapshot. Wait for its transaction
// (including recovery on failure) before reporting that policy.
firewallDeadmanMu.Lock()
defer firewallDeadmanMu.Unlock()
if c.d.fwEngine != nil {
if live := c.d.fwEngine.Config(); live != nil {
return live
}
}
return config.EffectiveFirewallConfig(cfg)
}
func (c *ControlListener) handleFirewallStatus(_ json.RawMessage) (any, error) {
cfg := c.d.currentCfg()
// LoadState tolerates a missing file (returns empty state).
state, err := firewall.LoadState(cfg.StatePath)
if err != nil {
return nil, fmt.Errorf("loading firewall state: %w", err)
}
fwCfg := c.firewallReadConfig(cfg)
result := control.FirewallStatusResult{
Enabled: fwCfg.Enabled,
TCPIn: fmtPortsSlice(fwCfg.TCPIn),
TCPOut: fmtPortsSlice(fwCfg.TCPOut),
UDPIn: fmtPortsSlice(fwCfg.UDPIn),
UDPOut: fmtPortsSlice(fwCfg.UDPOut),
Restricted: fmtPortsSlice(fwCfg.RestrictedTCP),
PassiveFTPStart: fwCfg.PassiveFTPStart,
PassiveFTPEnd: fwCfg.PassiveFTPEnd,
TCPOutAllow: fmtOutAllow(fwCfg.TCPOutAllow),
ProtocolAllow: fmtProtocolAllow(fwCfg.ProtocolAllow),
TCPInAllow: fmtInAllow(fwCfg.TCPInAllow),
InfraIPCount: len(fwCfg.InfraIPs),
BlockedCount: len(state.Blocked),
BlockedNetCount: len(state.BlockedNet),
AllowedCount: len(state.Allowed),
SYNFlood: fwCfg.SYNFloodProtection,
ConnRateLimit: fwCfg.ConnRateLimit,
LogDropped: fwCfg.LogDropped,
LogRate: fwCfg.LogRate,
}
// Recent blocked: last 10, newest-first, matching cmd/csm/firewall.go:fwStatus.
// Time values emitted as RFC3339 so the wire schema is not tied to
// Go's time.Time encoding; the CLI renders "N ago" on receive.
shown := 0
for i := len(state.Blocked) - 1; i >= 0 && shown < 10; i-- {
b := state.Blocked[i]
entry := control.FirewallBlockedEntry{
IP: b.IP,
Reason: b.Reason,
BlockedAt: b.BlockedAt.UTC().Format(time.RFC3339),
}
if !b.ExpiresAt.IsZero() {
entry.ExpiresAt = b.ExpiresAt.UTC().Format(time.RFC3339)
}
result.RecentBlocked = append(result.RecentBlocked, entry)
shown++
}
return result, nil
}
func (c *ControlListener) handleFirewallPorts(_ json.RawMessage) (any, error) {
cfg := c.d.currentCfg()
var lines []string
fwCfg := c.firewallReadConfig(cfg)
lines = append(lines, "TCP Inbound (public):")
lines = append(lines, " "+joinPorts(fwCfg.TCPIn))
lines = append(lines, "")
if len(fwCfg.RestrictedTCP) > 0 {
lines = append(lines, "TCP Restricted (infra only):")
lines = append(lines, " "+joinPorts(fwCfg.RestrictedTCP))
lines = append(lines, "")
}
lines = append(lines, "TCP Outbound:")
lines = append(lines, " "+joinPorts(fwCfg.TCPOut))
lines = append(lines, "")
lines = append(lines, "UDP Inbound:")
lines = append(lines, " "+joinPorts(fwCfg.UDPIn))
lines = append(lines, "")
lines = append(lines, "UDP Outbound:")
lines = append(lines, " "+joinPorts(fwCfg.UDPOut))
lines = append(lines, "")
if fwCfg.PassiveFTPStart > 0 {
lines = append(lines, "Passive FTP:")
lines = append(lines, fmt.Sprintf(" %d-%d", fwCfg.PassiveFTPStart, fwCfg.PassiveFTPEnd))
}
if out := fmtOutAllow(fwCfg.TCPOutAllow); len(out) > 0 {
lines = append(lines, "")
lines = append(lines, "TCP Outbound (destination-scoped):")
for _, line := range out {
lines = append(lines, " "+line)
}
}
if allows := fmtProtocolAllow(fwCfg.ProtocolAllow); len(allows) > 0 {
lines = append(lines, "")
lines = append(lines, "Protocols (peer-scoped, in and out):")
for _, line := range allows {
lines = append(lines, " "+line)
}
}
if allows := fmtInAllow(fwCfg.TCPInAllow); len(allows) > 0 {
lines = append(lines, "")
lines = append(lines, "TCP Inbound (source-scoped, ahead of blocks):")
for _, line := range allows {
lines = append(lines, " "+line)
}
}
return control.FirewallListResult{Lines: lines}, nil
}
// fmtInAllow renders tcp_in_allow one entry per line: the port, then the
// sources that reach it.
func fmtInAllow(rules []firewall.InAllowRule) []string {
out := make([]string, 0, len(rules))
for _, r := range rules {
out = append(out, fmt.Sprintf("tcp %d from %s", r.Port, strings.Join(r.Sources, ", ")))
}
return out
}
// fmtProtocolAllow renders protocol_allow one entry per line: the protocol,
// then its peers.
func fmtProtocolAllow(rules []firewall.ProtocolAllowRule) []string {
out := make([]string, 0, len(rules))
for _, r := range rules {
out = append(out, r.Protocol+" "+strings.Join(r.Peers, ", "))
}
return out
}
// fmtOutAllow renders tcp_out_allow one rule per line. A single-port range
// prints as the bare port so the common case does not read as "49152-49152".
func fmtOutAllow(rules []firewall.OutAllowRule) []string {
if len(rules) == 0 {
return nil
}
out := make([]string, 0, len(rules))
for _, r := range rules {
ports := fmt.Sprintf("%d-%d", r.PortStart, r.PortEnd)
if r.PortStart == r.PortEnd {
ports = strconv.Itoa(r.PortStart)
}
out = append(out, fmt.Sprintf("%s tcp %s", r.Dst, ports))
}
return out
}
// joinPorts returns a comma-separated rendering of ports matching
// cmd/csm/firewall.go:fmtPortsWrap's behaviour for the ports handler.
// Wrapping stays client-side; the wire payload keeps the full CSV so
// the CLI can format for whatever terminal width it runs in.
func joinPorts(ports []int) string {
if len(ports) == 0 {
return "(none)"
}
strs := make([]string, len(ports))
for i, p := range ports {
strs[i] = strconv.Itoa(p)
}
return strings.Join(strs, ", ")
}
func (c *ControlListener) handleFirewallGrep(argsRaw json.RawMessage) (any, error) {
var args control.FirewallGrepArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
cfg := c.d.currentCfg()
// Empty pattern matches nothing — the CLI used to require a
// positional arg and exit with usage; mirror that by returning an
// empty result rather than dumping everything.
if args.Pattern == "" {
return control.FirewallListResult{}, nil
}
pattern := strings.ToLower(args.Pattern)
state, err := firewall.LoadState(cfg.StatePath)
if err != nil {
return nil, fmt.Errorf("loading firewall state: %w", err)
}
var lines []string
now := time.Now()
for _, b := range state.Blocked {
if strings.Contains(strings.ToLower(b.IP), pattern) ||
strings.Contains(strings.ToLower(b.Reason), pattern) {
ago := now.Sub(b.BlockedAt).Truncate(time.Minute)
expires := "permanent"
if !b.ExpiresAt.IsZero() {
remaining := b.ExpiresAt.Sub(now).Truncate(time.Minute)
expires = fmt.Sprintf("%s left", remaining)
}
lines = append(lines, fmt.Sprintf("BLOCKED %-18s (%s ago, %s) %s",
b.IP, ago, expires, b.Reason))
}
}
for _, a := range state.Allowed {
if strings.Contains(strings.ToLower(a.IP), pattern) ||
strings.Contains(strings.ToLower(a.Reason), pattern) {
port := ""
if a.Port > 0 {
port = fmt.Sprintf(" port:%d", a.Port)
}
lines = append(lines, fmt.Sprintf("ALLOWED %-18s%s %s",
a.IP, port, a.Reason))
}
}
for _, s := range state.BlockedNet {
if strings.Contains(strings.ToLower(s.CIDR), pattern) ||
strings.Contains(strings.ToLower(s.Reason), pattern) {
ago := now.Sub(s.BlockedAt).Truncate(time.Minute)
lines = append(lines, fmt.Sprintf("SUBNET %-18s (%s ago) %s",
s.CIDR, ago, s.Reason))
}
}
for _, ip := range config.EffectiveFirewallConfig(cfg).InfraIPs {
if strings.Contains(strings.ToLower(ip), pattern) {
lines = append(lines, fmt.Sprintf("INFRA %s", ip))
}
}
return control.FirewallListResult{Lines: lines}, nil
}
func (c *ControlListener) handleFirewallAudit(argsRaw json.RawMessage) (any, error) {
var args control.FirewallAuditArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
limit := args.Limit
if limit <= 0 {
limit = 50
}
cfg := c.d.currentCfg()
entries := firewall.ReadAuditLog(cfg.StatePath, limit)
lines := make([]string, 0, len(entries))
for _, e := range entries {
ts := e.Timestamp.Format("2006-01-02 15:04:05")
dur := ""
if e.Duration != "" {
dur = fmt.Sprintf(" (%s)", e.Duration)
}
reason := ""
if e.Reason != "" {
reason = fmt.Sprintf(" %s", e.Reason)
}
lines = append(lines, fmt.Sprintf("%s %-13s %-18s%s%s",
ts, e.Action, e.IP, dur, reason))
}
return control.FirewallListResult{Lines: lines}, nil
}
package daemon
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/firewall/rollback"
)
func (c *ControlListener) handleFirewallRollbackStatus(_ json.RawMessage) (any, error) {
mgr := rollback.Global()
if mgr == nil {
return control.FirewallRollbackStatus{}, nil
}
st := mgr.Status()
out := control.FirewallRollbackStatus{
Pending: st.Pending,
AppliedBy: st.AppliedBy,
PrevHash: st.PrevHash,
NewHash: st.NewHash,
SecondsRemaining: st.SecondsRemaining,
}
if !st.AppliedAt.IsZero() {
out.AppliedAtRFC3339 = st.AppliedAt.Format(time.RFC3339)
}
if !st.ExpiresAt.IsZero() {
out.ExpiresAtRFC3339 = st.ExpiresAt.Format(time.RFC3339)
}
return out, nil
}
func (c *ControlListener) handleFirewallRollbackConfirm(_ json.RawMessage) (any, error) {
mgr := rollback.Global()
if mgr == nil {
return control.FirewallAckResult{Message: "rollback manager not initialised"}, nil
}
st := mgr.Status()
if !st.Pending {
return control.FirewallAckResult{Message: "no pending rollback"}, nil
}
if err := mgr.ConfirmIfCurrent(st); err != nil {
return nil, fmt.Errorf("confirm rollback: %w", err)
}
return control.FirewallAckResult{Message: "rollback confirmed; pending change is now permanent"}, nil
}
func (c *ControlListener) handleFirewallRollbackRevert(_ json.RawMessage) (any, error) {
mgr := rollback.Global()
if mgr == nil {
return control.FirewallAckResult{Message: "rollback manager not initialised"}, nil
}
st := mgr.Status()
if !st.Pending {
return control.FirewallAckResult{Message: "no pending rollback"}, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := mgr.RevertIfCurrent(ctx, st); err != nil {
return nil, fmt.Errorf("revert rollback: %w", err)
}
return control.FirewallAckResult{Message: "rollback reverted; previous config restored, daemon restart issued"}, nil
}
package daemon
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/health"
"github.com/pidginhost/csm/internal/integrity"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
// dispatch parses a raw request line, routes to the right handler, and
// wraps the handler's (result, error) pair in the response envelope.
// Unknown commands fail cleanly rather than crash the listener.
func (c *ControlListener) dispatch(line []byte) control.Response {
var req control.Request
if err := json.Unmarshal(line, &req); err != nil {
return control.Response{OK: false, Error: fmt.Sprintf("bad request: %v", err)}
}
var (
result any
err error
)
switch req.Cmd {
case control.CmdTierRun:
result, err = c.handleTierRun(req.Args)
case control.CmdStatus:
result, err = c.handleStatus(req.Args)
case control.CmdHistoryRead:
result, err = c.handleHistoryRead(req.Args)
case control.CmdRulesReload:
result, err = c.handleRulesReload(req.Args)
case control.CmdGeoIPReload:
result, err = c.handleGeoIPReload(req.Args)
case control.CmdBotRangesReload:
result, err = c.handleBotRangesReload(req.Args)
case control.CmdBaseline:
result, err = c.handleBaseline(req.Args)
case control.CmdThreatForget:
result, err = c.handleThreatForget(req.Args)
case control.CmdAntispamLearn:
result, err = c.handleAntispamLearn(req.Args)
case control.CmdAntispamLearnStatus:
result, err = c.handleAntispamLearnStatus(req.Args)
case control.CmdAntispamLearnFresh:
result, err = c.handleAntispamLearnFresh(req.Args)
case control.CmdFirewallStatus:
result, err = c.handleFirewallStatus(req.Args)
case control.CmdFirewallPorts:
result, err = c.handleFirewallPorts(req.Args)
case control.CmdFirewallGrep:
result, err = c.handleFirewallGrep(req.Args)
case control.CmdFirewallActions:
result, err = c.handleFirewallActions(req.Args)
case control.CmdFirewallActionResolve:
result, err = c.handleFirewallActionResolve(req.Args)
case control.CmdFirewallAudit:
result, err = c.handleFirewallAudit(req.Args)
case control.CmdFirewallBlock:
result, err = c.handleFirewallBlock(req.Args)
case control.CmdFirewallUnblock:
result, err = c.handleFirewallUnblock(req.Args)
case control.CmdFirewallAllow:
result, err = c.handleFirewallAllow(req.Args)
case control.CmdFirewallRemoveAllow:
result, err = c.handleFirewallRemoveAllow(req.Args)
case control.CmdFirewallAllowPort:
result, err = c.handleFirewallAllowPort(req.Args)
case control.CmdFirewallRemovePort:
result, err = c.handleFirewallRemovePort(req.Args)
case control.CmdFirewallTempBan:
result, err = c.handleFirewallTempBan(req.Args)
case control.CmdFirewallTempAllow:
result, err = c.handleFirewallTempAllow(req.Args)
case control.CmdFirewallDenySubnet:
result, err = c.handleFirewallDenySubnet(req.Args)
case control.CmdFirewallRemoveSubnet:
result, err = c.handleFirewallRemoveSubnet(req.Args)
case control.CmdFirewallDenyFile:
result, err = c.handleFirewallDenyFile(req.Args)
case control.CmdFirewallAllowFile:
result, err = c.handleFirewallAllowFile(req.Args)
case control.CmdFirewallFlush:
result, err = c.handleFirewallFlush(req.Args)
case control.CmdFirewallRestart:
result, err = c.handleFirewallRestart(req.Args)
case control.CmdFirewallApplyConfirmed:
result, err = c.handleFirewallApplyConfirmed(req.Args)
case control.CmdFirewallConfirm:
result, err = c.handleFirewallConfirm(req.Args)
case control.CmdFirewallRollbackStatus:
result, err = c.handleFirewallRollbackStatus(req.Args)
case control.CmdFirewallRollbackOK:
result, err = c.handleFirewallRollbackConfirm(req.Args)
case control.CmdFirewallRollbackRevert:
result, err = c.handleFirewallRollbackRevert(req.Args)
case control.CmdStoreExport:
result, err = c.handleStoreExport(req.Args)
case control.CmdHistorySince:
result, err = c.handleHistorySince(req.Args)
case control.CmdPHPRelayStatus:
result, err = c.handlePHPRelayStatus(req.Args)
case control.CmdPHPRelayIgnoreScript:
result, err = c.handlePHPRelayIgnoreScript(req.Args)
case control.CmdPHPRelayUnignore:
result, err = c.handlePHPRelayUnignore(req.Args)
case control.CmdPHPRelayIgnoreList:
result, err = c.handlePHPRelayIgnoreList(req.Args)
case control.CmdPHPRelayDryRun:
result, err = c.handlePHPRelayDryRun(req.Args)
case control.CmdPHPRelayThaw:
result, err = c.handlePHPRelayThaw(req.Args)
case control.CmdIncidentsList:
result, err = c.handleIncidentsList(req.Args)
case control.CmdIncidentsShow:
result, err = c.handleIncidentsShow(req.Args)
case control.CmdActionShow:
result, err = c.handleActionShow(req.Args)
case control.CmdIncidentsStatus:
result, err = c.handleIncidentsStatus(req.Args)
case control.CmdIncidentsBulkStatus:
result, err = c.handleIncidentsBulkStatus(req.Args)
case control.CmdScanEnqueue:
result, err = c.handleScanEnqueue(req.Args)
case control.CmdScanStatus:
result, err = c.handleScanStatus(req.Args)
case control.CmdScanReport:
result, err = c.handleScanReport(req.Args)
case control.CmdScanCancel:
result, err = c.handleScanCancel(req.Args)
default:
return control.Response{OK: false, Error: fmt.Sprintf("unknown command: %q", req.Cmd)}
}
if err != nil {
return control.Response{OK: false, Error: err.Error()}
}
payload, mErr := json.Marshal(result)
if mErr != nil {
return control.Response{OK: false, Error: "result marshal: " + mErr.Error()}
}
return control.Response{OK: true, Result: payload}
}
// parseTier maps the wire string onto the checks.Tier constants. Kept
// local to the listener so the protocol package does not depend on the
// checks package.
func parseTier(s string) (checks.Tier, error) {
switch s {
case "critical":
return checks.TierCritical, nil
case "deep":
return checks.TierDeep, nil
case "all", "":
return checks.TierAll, nil
}
return "", fmt.Errorf("unknown tier: %q", s)
}
// handleTierRun runs a tier synchronously and reports the result. The
// flow mirrors Daemon.runPeriodicChecks: integrity verify, RunTier,
// purge-and-merge, then hand findings to the alert pipeline. When
// Alerts=false the handler absorbs the old `csm check*` behaviour:
// skip auto-response for this run, append the raw findings to history,
// and return them in the response body so the CLI can render them
// verbatim.
func (c *ControlListener) handleTierRun(argsRaw json.RawMessage) (any, error) {
var args control.TierRunArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
tier, err := parseTier(args.Tier)
if err != nil {
return nil, err
}
// dryRun ties together the three Alerts=false side effects:
// auto-response suppression, the post-run history append, and the
// FindingList in the response. Hoisting it makes the invariant
// "these three happen together" visually obvious.
dryRun := !args.Alerts
if vErr := c.verifyTierRunIntegrity(); vErr != nil {
// Integrity failures are escalated through the normal alert
// pipeline so the on-call path sees them regardless of who
// kicked the tier run. The client also gets an error so the
// systemd timer unit fails loudly.
if args.Alerts {
if !alert.TryEnqueue(c.d.alertCh, alert.Finding{
Severity: alert.Critical,
Check: "integrity",
Message: fmt.Sprintf("BINARY/CONFIG TAMPER DETECTED: %v", vErr),
Timestamp: time.Now(),
}) {
atomic.AddInt64(&c.d.droppedAlerts, 1)
}
}
return nil, fmt.Errorf("integrity verify failed: %w", vErr)
}
// Dry-run threads through RunTierDryRunWithContext, so a concurrent periodic
// scanner running in live mode never sees this caller's dry-run state.
start := time.Now()
var (
findings []alert.Finding
purgeChecks []string
)
cfg := c.d.currentCfg()
scanCtx, gaps := checks.WithCoverageGaps(c.d.scanContext())
if dryRun {
findings, purgeChecks = checks.RunTierDryRunWithContext(scanCtx, cfg, c.d.store, tier)
} else {
findings, purgeChecks = checks.RunTierWithContext(scanCtx, cfg, c.d.store, tier)
}
c.recordTierRunFindings(cfg, findings, purgeChecks, gaps.Snapshot(), !dryRun, args.Alerts)
// Dry-run history + FindingList: the live path writes history via
// Daemon.runPeriodicChecks when the internal scanners fire; the
// pre-phase-2 `csm check*` wrote it via store.AppendHistory in
// cmd/csm/main.go. Preserve that quirk on the socket path.
if dryRun {
c.d.store.AppendHistory(findings)
}
newCount := len(c.d.store.FilterNew(findings))
result := control.TierRunResult{
Findings: len(findings),
NewFindings: newCount,
ElapsedMs: time.Since(start).Milliseconds(),
}
if dryRun {
if findings == nil {
result.FindingList = []alert.Finding{}
} else {
result.FindingList = findings
}
}
return result, nil
}
// recordTierRunFindings persists a control-socket tier run's findings. Auto-fix
// is gated on a live run so a dry run never edits a customer's wp-config.php,
// and the alert push is gated separately on whether the caller asked for alerts.
func (c *ControlListener) recordTierRunFindings(cfg *config.Config, findings []alert.Finding, purgeChecks []string, coverage *state.ScanCoverage, live, alerts bool) {
checks.StoreLatestScanFindingsWithCoverage(c.d.store, purgeChecks, findings, coverage)
if live {
c.d.applyWPCronAutoFix(cfg, findings)
}
if alerts {
c.d.enqueueScanAlerts(findings, "control")
}
}
func (c *ControlListener) verifyTierRunIntegrity() error {
return integrity.Verify(c.d.binaryPath, c.d.currentCfg())
}
// handleStatus reports what `csm status` historically printed from
// disk, sourced from the live daemon instead of re-opening the store.
func (c *ControlListener) handleStatus(_ json.RawMessage) (any, error) {
latest := c.d.store.LatestFindings()
latestTime := c.d.store.LatestScanTime()
var latestStr string
if !latestTime.IsZero() {
latestStr = latestTime.UTC().Format(time.RFC3339)
}
var historyCount int
if sdb := store.Global(); sdb != nil {
historyCount = sdb.HistoryCount()
}
uptime := int64(0)
if !c.d.startTime.IsZero() {
uptime = int64(time.Since(c.d.startTime).Seconds())
}
result := control.StatusResult{
Version: c.d.version,
UptimeSec: uptime,
LatestScanTime: latestStr,
LatestFindings: len(latest),
HistoryCount: historyCount,
DroppedAlerts: c.d.DroppedAlerts(),
}
snap := health.Build(c.d, c.d.version, health.Capabilities())
result.Snapshot = &snap
return result, nil
}
// handleHistoryRead paginates bbolt history. Clamps Limit so a buggy
// client cannot ask for everything at once; 1000 is well above the
// dashboard page size.
func (c *ControlListener) handleHistoryRead(argsRaw json.RawMessage) (any, error) {
var args control.HistoryReadArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if args.Limit <= 0 || args.Limit > 1000 {
args.Limit = 100
}
if args.Offset < 0 {
args.Offset = 0
}
findings, total := c.d.store.ReadHistory(args.Limit, args.Offset)
return control.HistoryReadResult{Findings: findings, Total: total}, nil
}
// handleRulesReload replaces `kill -HUP $(pidof csm)`. Returns after
// the reload completes so the client can confirm it happened.
func (c *ControlListener) handleRulesReload(_ json.RawMessage) (any, error) {
c.d.reloadSignatures()
return map[string]string{"status": "reloaded"}, nil
}
// handleGeoIPReload is the GeoIP equivalent of rules.reload.
func (c *ControlListener) handleGeoIPReload(_ json.RawMessage) (any, error) {
c.d.publishGeoIP()
return map[string]string{"status": "reloaded"}, nil
}
// handleBotRangesReload republishes the on-disk AI-crawler range overlay that
// `csm update-bot-ranges` just refreshed, so the new ranges apply without a
// daemon restart.
func (c *ControlListener) handleBotRangesReload(_ json.RawMessage) (any, error) {
if err := c.d.reloadBotRanges(); err != nil {
return nil, err
}
return map[string]string{"status": "reloaded"}, nil
}
// handleHistorySince streams every history-bucket finding newer than
// the supplied cutoff. Used by `csm export --since` for SIEM backfill;
// the daemon-side bbolt cursor seek is materially faster than
// pagination + client-side filtering for hosts with large histories.
func (c *ControlListener) handleHistorySince(argsRaw json.RawMessage) (any, error) {
var args control.HistorySinceArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if args.Since == "" {
return nil, fmt.Errorf("since is required (RFC 3339)")
}
since, err := time.Parse(time.RFC3339, args.Since)
if err != nil {
return nil, fmt.Errorf("parsing since: %w", err)
}
sdb := store.Global()
if sdb == nil {
return nil, fmt.Errorf("bbolt store not available")
}
findings := sdb.ReadHistorySince(since)
// ReadHistorySince returns newest-first; reverse for chronological
// output so JSONL consumers see the same order they would from a
// live tail.
for i, j := 0, len(findings)-1; i < j; i, j = i+1, j-1 {
findings[i], findings[j] = findings[j], findings[i]
}
return control.HistorySinceResult{Findings: findings}, nil
}
// handleStoreExport writes a tar+zstd backup containing the live bbolt
// snapshot, the state directory, and the signature-rules cache. The
// daemon is the single source of truth for paths; the CLI only supplies
// where to write the archive. Import deliberately does NOT route through
// the socket -- it requires a stopped daemon.
const exportStagingMaxAge = 24 * time.Hour
// prepareExportStagingPath creates a private per-request directory so
// concurrent exports of the same basename cannot overwrite each other. Old
// request directories are removed here so a client that dies after the daemon
// replies cannot leave state storage growing forever.
func prepareExportStagingPath(statePath, dstPath string, now time.Time) (string, error) {
base := filepath.Base(dstPath)
if base == "." || base == ".." || base == string(filepath.Separator) || base == "" {
return "", fmt.Errorf("destination must name an archive file")
}
exportDir := filepath.Join(statePath, "exports")
if err := os.MkdirAll(exportDir, 0o700); err != nil {
return "", err
}
// #nosec G302 -- exportDir is a directory; 0700 is already the tightest
// mode that still lets the daemon traverse into it.
if err := os.Chmod(exportDir, 0o700); err != nil {
return "", err
}
entries, err := os.ReadDir(exportDir)
if err != nil {
return "", err
}
cutoff := now.Add(-exportStagingMaxAge)
for _, entry := range entries {
info, infoErr := entry.Info()
if infoErr == nil && info.ModTime().Before(cutoff) {
_ = os.RemoveAll(filepath.Join(exportDir, entry.Name()))
}
}
requestDir, err := os.MkdirTemp(exportDir, "export-")
if err != nil {
return "", err
}
return filepath.Join(requestDir, base), nil
}
func (c *ControlListener) handleStoreExport(argsRaw json.RawMessage) (any, error) {
var args control.StoreExportArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if args.DstPath == "" {
return nil, fmt.Errorf("dst_path is required")
}
sdb := store.Global()
if sdb == nil {
return nil, fmt.Errorf("bbolt store not available")
}
// The alert path saves state.json at most once a minute; write what it
// holds so the archive carries the current dedup state.
if err := c.d.store.Flush(); err != nil {
return nil, fmt.Errorf("saving state: %w", err)
}
cfg := c.d.currentCfg()
hostname, _ := os.Hostname()
pi := platform.Detect()
dstPath := args.DstPath
staged := false
var err error
if args.Stage {
dstPath, err = prepareExportStagingPath(cfg.StatePath, args.DstPath, time.Now())
if err != nil {
return nil, fmt.Errorf("creating export staging dir: %w", err)
}
staged = true
}
res, err := sdb.Export(store.ExportOptions{
StatePath: cfg.StatePath,
RulesPath: cfg.Signatures.RulesDir,
DstPath: dstPath,
Manifest: store.Manifest{
CSMVersion: c.d.version,
SourceHostname: hostname,
SourcePlatform: map[string]string{
"os": string(pi.OS),
"os_version": pi.OSVersion,
"panel": string(pi.Panel),
"webserver": string(pi.WebServer),
},
},
})
if err != nil {
if staged {
_ = os.RemoveAll(filepath.Dir(dstPath))
}
return nil, err
}
return control.StoreExportResult{
Path: res.Path,
Bytes: res.Bytes,
ArchiveSHA256: res.ArchiveSHA256,
BboltSHA256: res.BboltSHA256,
}, nil
}
package daemon
import (
"encoding/json"
"errors"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/control"
)
// handleActionShow returns one admitted candidate and its attempts from
// the admission owner.
func (c *ControlListener) handleActionShow(argsRaw json.RawMessage) (result any, err error) {
var args control.ActionShowArgs
defer func() { recordActionShow(args.ID, err) }()
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, err
}
}
if c.d == nil || c.d.admission == nil {
return nil, errors.New("admission is not running")
}
return c.d.admission.Action(args.ID)
}
// Reads are operator operations too. Keep their audit outside the
// owner and its outbox so an audit outage cannot disable a diagnostic read.
func recordActionShow(ref string, err error) {
record := actionlog.Record{
Op: "respond.block_ip", Action: "show", Actor: actionlog.CLI,
ActorDetail: "csm action show", Target: ref, Result: "read",
}
if err != nil {
record.Result, record.Error = actionlog.Failed, err.Error()
}
actionlog.Write(record)
}
package daemon
import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/incident"
)
const (
defaultIncidentListLimit = 100
maxIncidentListLimit = 1000
defaultIncidentBulkLimit = 100
maxIncidentBulkLimit = 1000
)
// handleIncidentsList returns a bounded, newest-first incident page.
func (c *ControlListener) handleIncidentsList(argsRaw json.RawMessage) (any, error) {
var args control.IncidentListArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, err
}
}
statuses, statusLabel, err := incidentListStatusFilter(args.Status)
if err != nil {
return nil, err
}
offset := args.Offset
if offset < 0 {
offset = 0
}
limit := args.Limit
if args.All {
limit = 0
} else {
if limit <= 0 {
limit = defaultIncidentListLimit
}
if limit > maxIncidentListLimit {
limit = maxIncidentListLimit
}
}
co := IncidentCorrelator()
items, total := co.SnapshotPageStatuses(statuses, offset, limit)
return control.IncidentListResult{
Items: items,
Total: total,
Offset: offset,
Limit: limit,
Status: statusLabel,
}, nil
}
func incidentListStatusFilter(status string) ([]incident.Status, string, error) {
switch strings.ToLower(strings.TrimSpace(status)) {
case "", "all":
return nil, "all", nil
case "active":
return []incident.Status{incident.StatusOpen, incident.StatusContained}, "active", nil
case string(incident.StatusOpen):
return []incident.Status{incident.StatusOpen}, string(incident.StatusOpen), nil
case string(incident.StatusContained):
return []incident.Status{incident.StatusContained}, string(incident.StatusContained), nil
case string(incident.StatusResolved):
return []incident.Status{incident.StatusResolved}, string(incident.StatusResolved), nil
case string(incident.StatusDismissed):
return []incident.Status{incident.StatusDismissed}, string(incident.StatusDismissed), nil
default:
return nil, "", fmt.Errorf("unknown status: %q", status)
}
}
func incidentBulkStatusFilter(status string) ([]incident.Status, string, error) {
switch strings.ToLower(strings.TrimSpace(status)) {
case "", "active":
return []incident.Status{incident.StatusOpen, incident.StatusContained}, "active", nil
case string(incident.StatusOpen):
return []incident.Status{incident.StatusOpen}, string(incident.StatusOpen), nil
case string(incident.StatusContained):
return []incident.Status{incident.StatusContained}, string(incident.StatusContained), nil
default:
return nil, "", fmt.Errorf("bulk status source must be active, open, or contained")
}
}
// handleIncidentsShow returns one incident by id; ErrIncidentNotFound on miss.
func (c *ControlListener) handleIncidentsShow(argsRaw json.RawMessage) (any, error) {
var args control.IncidentShowArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, err
}
}
co := IncidentCorrelator()
inc, ok := co.Get(args.ID)
if !ok {
return nil, incident.ErrIncidentNotFound
}
return inc, nil
}
// handleIncidentsStatus transitions an incident's status. Returns
// {"ok": true} on success; ErrIncidentNotFound or validation error on
// failure.
func (c *ControlListener) handleIncidentsStatus(argsRaw json.RawMessage) (any, error) {
var args control.IncidentStatusArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, err
}
}
co := IncidentCorrelator()
if err := co.SetStatus(args.ID, incident.Status(args.Status), args.Details); err != nil {
return nil, err
}
return map[string]bool{"ok": true}, nil
}
func (c *ControlListener) handleIncidentsBulkStatus(argsRaw json.RawMessage) (any, error) {
var args control.IncidentBulkStatusArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, err
}
}
statuses, statusLabel, err := incidentBulkStatusFilter(args.Status)
if err != nil {
return nil, err
}
to := incident.Status(strings.ToLower(strings.TrimSpace(args.To)))
if to == "" {
to = incident.StatusResolved
}
if to != incident.StatusResolved && to != incident.StatusDismissed {
return nil, fmt.Errorf("bulk status target must be resolved or dismissed")
}
if args.OlderThanSeconds < 0 {
return nil, fmt.Errorf("older-than must be positive")
}
olderThan := time.Duration(args.OlderThanSeconds) * time.Second
if olderThan <= 0 && args.LastSeenBefore.IsZero() {
return nil, fmt.Errorf("bulk status requires --older-than or --last-seen-before")
}
limit := args.Limit
if limit <= 0 {
limit = defaultIncidentBulkLimit
}
if limit > maxIncidentBulkLimit {
limit = maxIncidentBulkLimit
}
if args.Apply && !args.Confirm {
return nil, fmt.Errorf("bulk status apply requires confirmation")
}
dryRun := !args.Apply
co := IncidentCorrelator()
res, err := co.BulkSetStatus(incident.BulkStatusFilter{
FromStatuses: statuses,
To: to,
OlderThan: olderThan,
LastSeenBefore: args.LastSeenBefore,
Kind: incident.Kind(strings.TrimSpace(args.Kind)),
Domain: strings.TrimSpace(args.Domain),
Account: strings.TrimSpace(args.Account),
Mailbox: strings.TrimSpace(args.Mailbox),
Limit: limit,
DryRun: dryRun,
Details: strings.TrimSpace(args.Details),
})
if err != nil {
return nil, err
}
return control.IncidentBulkStatusResult{
DryRun: dryRun,
Matched: res.Matched,
Updated: res.Updated,
Limit: limit,
Status: statusLabel,
To: string(to),
OlderThanSeconds: args.OlderThanSeconds,
LastSeenBefore: args.LastSeenBefore,
Items: res.Items,
}, nil
}
package daemon
import (
"encoding/json"
"fmt"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/store"
)
// handleScanEnqueue validates the request and submits a new full-scan job to
// the ScanJobManager. Scope="account" scans a single account; Scope="all"
// enqueues a server-wide scan. The control payload is translated into
// checks.AccountScanOptions with the full-scan option set (MaxFiles=0,
// ForceContent=true, ForceFileIndex=true).
func (c *ControlListener) handleScanEnqueue(argsRaw json.RawMessage) (any, error) {
if c.scanJobs == nil {
return nil, fmt.Errorf("scan job manager not available")
}
var req control.ScanEnqueueRequest
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &req); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
cfg := c.d.currentCfg()
opts := checks.FullScanOptions(cfg, req.RespectIgnores)
switch req.Scope {
case "account":
if req.Target == "" {
return nil, fmt.Errorf("target is required")
}
if !control.ValidScanAccountTarget(req.Target) {
return nil, fmt.Errorf("invalid account target %q", req.Target)
}
id, err := c.scanJobs.Enqueue("account", req.Target, opts, req.Quarantine)
if err != nil {
return nil, fmt.Errorf("enqueue: %w", err)
}
return control.ScanEnqueueResponse{JobID: id, State: "queued"}, nil
case "all":
// Target must be empty or the literal "all"; it must not look like an
// account name or path component — the daemon normalises it to "all".
if req.Target != "" && req.Target != "all" {
return nil, fmt.Errorf("invalid target %q for scope \"all\": must be empty or \"all\"", req.Target)
}
// Defense-in-depth (the CLI also rejects this): server-wide quarantine
// would remediate across every account from one audit pass. Refuse it;
// quarantine is a per-account post-review action.
if req.Quarantine {
return nil, fmt.Errorf("quarantine is not supported with scope \"all\"")
}
id, err := c.scanJobs.Enqueue("all", "all", opts, req.Quarantine)
if err != nil {
return nil, fmt.Errorf("enqueue: %w", err)
}
return control.ScanEnqueueResponse{JobID: id, State: "queued"}, nil
default:
return nil, fmt.Errorf("unsupported scope %q: must be \"account\" or \"all\"", req.Scope)
}
}
// handleScanStatus returns the status of one job (when JobID is set) or the
// full list of jobs ordered newest-first. Unknown IDs produce an error.
func (c *ControlListener) handleScanStatus(argsRaw json.RawMessage) (any, error) {
if c.scanJobs == nil {
return nil, fmt.Errorf("scan job manager not available")
}
var req control.ScanStatusRequest
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &req); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if req.JobID != "" {
rec, ok := c.scanJobs.Progress(req.JobID)
if !ok {
return nil, fmt.Errorf("job not found: %q", req.JobID)
}
return control.ScanStatusResponse{Job: &rec}, nil
}
jobs, err := c.scanJobs.ListJobs()
if err != nil {
return nil, fmt.Errorf("listing jobs: %w", err)
}
if jobs == nil {
jobs = []store.ScanJobRecord{}
}
return control.ScanStatusResponse{Jobs: jobs}, nil
}
// handleScanReport returns the job record plus a paginated slice of its
// findings. JobID is required; Offset and Limit follow the usual page semantics
// (Limit=0 returns all findings).
func (c *ControlListener) handleScanReport(argsRaw json.RawMessage) (any, error) {
if c.scanJobs == nil {
return nil, fmt.Errorf("scan job manager not available")
}
var req control.ScanReportRequest
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &req); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if req.JobID == "" {
return nil, fmt.Errorf("job_id is required")
}
rec, ok := c.scanJobs.Progress(req.JobID)
if !ok {
return nil, fmt.Errorf("job not found: %q", req.JobID)
}
if req.Offset < 0 {
req.Offset = 0
}
findings, total, err := c.scanJobs.ListFindings(req.JobID, req.Offset, req.Limit)
if err != nil {
return nil, fmt.Errorf("listing findings: %w", err)
}
if findings == nil {
findings = []alert.Finding{}
}
return control.ScanReportResponse{
Job: rec,
Findings: findings,
Total: total,
}, nil
}
// handleScanCancel cancels the job with the given ID. If the job is queued it
// will be marked canceled before the worker processes it; if it is running its
// context is canceled and partial findings are retained. Unknown or already
// terminal IDs produce an error.
func (c *ControlListener) handleScanCancel(argsRaw json.RawMessage) (any, error) {
if c.scanJobs == nil {
return nil, fmt.Errorf("scan job manager not available")
}
var req control.ScanCancelRequest
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &req); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if req.JobID == "" {
return nil, fmt.Errorf("job_id is required")
}
if err := c.scanJobs.Cancel(req.JobID); err != nil {
return nil, fmt.Errorf("cancel: %w", err)
}
// Return the job's current state. The worker may not have transitioned it
// yet; the caller polls via scan.status for the terminal state.
rec, ok := c.scanJobs.Progress(req.JobID)
state := "canceling"
if ok {
state = rec.State
}
return control.ScanCancelResponse{JobID: req.JobID, State: state}, nil
}
package daemon
import (
"bufio"
"encoding/json"
"fmt"
"net"
"os"
"path/filepath"
"time"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/obs"
)
// controlSocketPath is the Unix socket the daemon binds for
// CLI-to-daemon IPC. A var (not const) so tests can redirect under
// t.TempDir(); production default matches internal/control.DefaultSocketPath.
var controlSocketPath = "/var/run/csm/control.sock"
// controlRequestTimeout caps how long a single client request can block
// the listener. Handlers that legitimately take longer (tier.run on a
// large server) run on the accepting goroutine, so the timeout applies
// to reading the request line and writing the response, not to the
// handler body itself.
const controlRequestTimeout = 2 * time.Second
// ControlListener serves the local command-line client over a Unix
// socket. One request and one response per connection, line-framed JSON.
// The daemon keeps exclusive ownership of the bbolt store; this listener
// is the only reason any CLI command needs to reach into daemon state.
type ControlListener struct {
d *Daemon
listener net.Listener
phprelay *PHPRelayController // wired by Phase O2; may be nil in tests / pre-wiring
scanJobs *ScanJobManager // wired after startScanJobManager(); nil until then
}
// NewControlListener creates the socket, enforces 0600 perms, and
// returns a listener that the daemon wires into its goroutine pool.
func NewControlListener(d *Daemon) (*ControlListener, error) {
socketDir := filepath.Dir(controlSocketPath)
if err := os.MkdirAll(socketDir, 0750); err != nil {
return nil, fmt.Errorf("creating socket dir: %w", err)
}
// Stale socket from a previous crash would make Listen fail with
// EADDRINUSE. Remove before binding; the file is process-owned.
_ = os.Remove(controlSocketPath)
ln, err := net.Listen("unix", controlSocketPath)
if err != nil {
return nil, fmt.Errorf("listening on %s: %w", controlSocketPath, err)
}
// 0600 root-only: the CLI client also runs as root (fanotify,
// nftables, cpanel APIs all require it), so no group is needed.
if err := os.Chmod(controlSocketPath, 0600); err != nil {
_ = ln.Close()
return nil, fmt.Errorf("chmod socket: %w", err)
}
return &ControlListener{d: d, listener: ln}, nil
}
// Run accepts connections until stopCh closes. Each connection is
// handled on its own goroutine so a slow request never stalls the
// accept loop.
func (c *ControlListener) Run(stopCh <-chan struct{}) {
for {
conn, err := c.listener.Accept()
if err != nil {
select {
case <-stopCh:
return
default:
csmlog.Warn("control listener accept error", "err", err)
time.Sleep(100 * time.Millisecond)
continue
}
}
// SO_PEERCRED defence-in-depth: socket perms are already 0600
// (root-only), but if a future install pattern relaxes that --
// or a confused-deputy mount makes the socket reachable from a
// non-root namespace -- the kernel-supplied credentials force
// us to refuse any non-root caller before reading a byte of
// payload. The socket perms remain the primary defence; this
// is an extra rejection layer that the kernel cannot lie about.
if err := verifyControlPeer(conn); err != nil {
csmlog.Warn("control listener rejecting peer", "err", err)
_ = conn.Close()
continue
}
obs.SafeGo("control-conn", func() { c.handleConnection(conn) })
}
}
// verifyControlPeer is defined per-OS:
// - linux: reads SO_PEERCRED and refuses non-root callers
// - other: no-op (the listener is Linux-only in production; macOS dev
// builds still need the package to compile)
// Stop closes the listener and removes the socket file. Safe to call
// after Run has already returned.
func (c *ControlListener) Stop() {
_ = c.listener.Close()
_ = os.Remove(controlSocketPath)
}
// handleConnection reads one request, dispatches it, writes one
// response, and closes. The short timeout applies to I/O only; the
// handler itself can take as long as the underlying work requires.
func (c *ControlListener) handleConnection(conn net.Conn) {
defer func() { _ = conn.Close() }()
// Read deadline. Writes use a separate deadline set after the
// handler returns so slow scans don't count against the reader.
_ = conn.SetReadDeadline(time.Now().Add(controlRequestTimeout))
scanner := bufio.NewScanner(conn)
// Requests and responses are single-line JSON; on a compromised
// server a dry-run tier scan can legitimately return a findings
// list that exceeds the old 1 MiB cap (think: many WordPress
// installs, each with multiple infected files). The socket is
// root-only 0600 so the original DoS guard no longer applies —
// cap the buffer at 16 MiB so large finding lists round-trip.
scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
if !scanner.Scan() {
return
}
line := scanner.Bytes()
resp := c.dispatch(line)
payload, err := json.Marshal(resp)
if err != nil {
// Marshalling a Response should not fail; if it does, fall back
// to a minimal error response the client can still parse.
payload = []byte(`{"ok":false,"error":"internal: response marshal failed"}`)
}
payload = append(payload, '\n')
_ = conn.SetWriteDeadline(time.Now().Add(controlRequestTimeout))
_, _ = conn.Write(payload)
}
//go:build linux
package daemon
import (
"fmt"
"net"
"golang.org/x/sys/unix"
)
var controlPeerRequiredUID uint32
// verifyControlPeer reads SO_PEERCRED and refuses any caller whose
// effective uid is not root. Returns nil when the peer is acceptable.
func verifyControlPeer(conn net.Conn) error {
uc, ok := conn.(*net.UnixConn)
if !ok || uc == nil {
return fmt.Errorf("peer credentials: unsupported connection %T", conn)
}
raw, err := uc.SyscallConn()
if err != nil {
return fmt.Errorf("peer raw conn: %w", err)
}
var ucred *unix.Ucred
var optErr error
if err := raw.Control(func(fd uintptr) {
// #nosec G115 -- POSIX fd fits in int on Linux.
ucred, optErr = unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED)
}); err != nil {
return fmt.Errorf("peer credentials: %w", err)
}
if optErr != nil {
return fmt.Errorf("peer credentials: %w", optErr)
}
if ucred == nil {
return fmt.Errorf("peer credentials: empty result")
}
if ucred.Uid != controlPeerRequiredUID {
return fmt.Errorf("peer uid=%d, want %d", ucred.Uid, controlPeerRequiredUID)
}
return nil
}
package daemon
import (
"encoding/json"
"fmt"
"net"
"github.com/pidginhost/csm/internal/attackdb"
"github.com/pidginhost/csm/internal/control"
)
// handleThreatForget drops one address's record from the attack database.
//
// Most score contributions last until the record expires after 90 days.
// Fixing a detection that attributed events to the wrong address stops new
// events but leaves those contributions behind.
//
// The Web UI's clear and whitelist actions also change enforcement.
// This command leaves enforcement and event history intact, so a cleared
// address is scored again from scratch on its next finding.
func (c *ControlListener) handleThreatForget(argsRaw json.RawMessage) (any, error) {
var args control.FirewallIPArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
ip := net.ParseIP(args.IP)
if ip == nil {
return nil, fmt.Errorf("invalid ip: %q", args.IP)
}
args.IP = ip.String()
network := ""
if ip.To4() == nil {
mask := net.CIDRMask(64, 128)
network = (&net.IPNet{IP: ip.Mask(mask), Mask: mask}).String()
}
adb := attackdb.Global()
if adb == nil {
return nil, fmt.Errorf("attack database unavailable")
}
// Read and remove under one lock so concurrent requests cannot claim
// the same record, or report counts from before a concurrent finding.
res := control.ThreatForgetResult{IP: args.IP}
for _, rec := range adb.ForgetIP(ip) {
res.Found = true
res.Score = max(res.Score, attackdb.ComputeScore(rec))
res.Events += rec.EventCount
// Stored text can be corrupt; only name the expected network.
if network != "" && rec.IP == network {
res.Network = network
}
}
if !res.Found {
res.Message = fmt.Sprintf("No local threat record for %s; nothing cleared", args.IP)
return res, nil
}
// Persist immediately rather than waiting for the 30s background saver:
// Flush does not report persistence errors, so this remains best effort.
// A lookup afterwards cannot verify persistence, and new findings may
// legitimately have created a fresh record by then.
_ = adb.Flush()
res.Message = fmt.Sprintf("Cleared local threat record for %s (was score %d/100, %d attack events)", args.IP, res.Score, res.Events)
if res.Network != "" {
res.Message += " and the spam evidence of " + res.Network
}
res.Message += "; block, allow and whitelist entries are unchanged"
return res, nil
}
package daemon
import (
"net"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/platform"
)
// infraHostnames partitions the cfg.InfraIPs operator list into the
// subset that is hostnames (not literal IPs or CIDRs). Hostnames get
// DNS-refreshed into the engine's infra-block guard so operators can
// list panel hosts by name and have them stay protected as the
// underlying address rotates.
func infraHostnames(entries []string) []string {
out := make([]string, 0, len(entries))
for _, e := range entries {
s := strings.TrimSpace(e)
if s == "" {
continue
}
if _, _, err := net.ParseCIDR(s); err == nil {
continue
}
if ip := net.ParseIP(s); ip != nil {
continue
}
out = append(out, s)
}
return out
}
// containsString returns true when s appears in haystack. Linear scan
// because the call site loops short DynDNS host lists where a map
// would not pay for itself.
func containsString(haystack []string, s string) bool {
for _, h := range haystack {
if h == s {
return true
}
}
return false
}
// expandWithCorrelation runs cross-account correlation over a dispatch
// batch and appends any synthesized findings that are not already present.
// The scan runner may have already produced the same synthetic findings, so
// this helper must be idempotent to avoid double-alerting the first batch.
func expandWithCorrelation(findings []alert.Finding, now time.Time) []alert.Finding {
if len(findings) == 0 {
return findings
}
platform.Detect()
seen := make(map[string]struct{})
for i := range findings {
if !checks.IsDerivedCorrelationCheck(findings[i].Check) {
continue
}
if findings[i].Timestamp.IsZero() {
findings[i].Timestamp = now
}
seen[findings[i].Key()] = struct{}{}
}
res := checks.CorrelateBatchFindings(findings)
for i := range res.Derived {
if res.Derived[i].Timestamp.IsZero() {
res.Derived[i].Timestamp = now
}
key := res.Derived[i].Key()
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
findings = append(findings, res.Derived[i])
}
checks.ReportUnattributedCorrelation(res.Unattributed)
return findings
}
package daemon
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/emailspool"
"gopkg.in/yaml.v3"
)
// userDomainsResolver resolves a cPanel user to the lowercased, IDN-normalised
// set of domains the account owns. TTL-based cache; safe for concurrent use.
type userDomainsResolver struct {
root string
ttl time.Duration
mu sync.Mutex
cache map[string]userDomainsCacheEntry
}
type userDomainsCacheEntry struct {
domains map[string]struct{}
fetched time.Time
err error
}
// newUserDomainsResolver returns a resolver reading from /var/cpanel/userdata/.
// Wired by daemon startup in O2; kept now so that future call sites compile.
//
//nolint:unused // consumed by daemon wiring (Task O2)
func newUserDomainsResolver() *userDomainsResolver {
return newUserDomainsResolverWithRoot("/var/cpanel/userdata", 5*time.Minute)
}
func newUserDomainsResolverWithRoot(root string, ttl time.Duration) *userDomainsResolver {
return &userDomainsResolver{
root: root,
ttl: ttl,
cache: make(map[string]userDomainsCacheEntry),
}
}
// Domains returns the cPanel user's authorised domain set. Returns the
// (possibly cached) error if the user's userdata is unreadable; callers
// must treat an error result as "skip the From-mismatch signal" rather than
// falsely amplifying.
func (r *userDomainsResolver) Domains(user string) (map[string]struct{}, error) {
if user == "" {
return nil, errors.New("empty user")
}
r.mu.Lock()
if e, ok := r.cache[user]; ok && time.Since(e.fetched) < r.ttl {
r.mu.Unlock()
return e.domains, e.err
}
r.mu.Unlock()
set, err := r.read(user)
r.mu.Lock()
r.cache[user] = userDomainsCacheEntry{domains: set, fetched: time.Now(), err: err}
r.mu.Unlock()
return set, err
}
// Invalidate removes the cached entry for user. Callers wire this to
// inotify on /var/cpanel/userdata/<user>/.
func (r *userDomainsResolver) Invalidate(user string) {
r.mu.Lock()
delete(r.cache, user)
r.mu.Unlock()
}
func (r *userDomainsResolver) read(user string) (map[string]struct{}, error) {
path := filepath.Join(r.root, user, "main")
// #nosec G304 -- r.root is fixed at /var/cpanel/userdata/, user is a
// cPanel-managed account name validated by the resolver's caller; the
// resulting path is constrained to the cpanel userdata tree.
data, err := os.ReadFile(path)
if err != nil {
return map[string]struct{}{}, fmt.Errorf("read %s: %w", path, err)
}
var raw struct {
MainDomain string `yaml:"main_domain"`
AddonDomains map[string]string `yaml:"addon_domains"`
ParkedDomains []string `yaml:"parked_domains"`
SubDomains []string `yaml:"sub_domains"`
}
if err := yaml.Unmarshal(data, &raw); err != nil {
return map[string]struct{}{}, fmt.Errorf("parse %s: %w", path, err)
}
set := make(map[string]struct{}, 8)
add := func(d string) {
d = strings.TrimSpace(d)
if d == "" {
return
}
// ExtractDomain handles IDN normalisation and lowercasing for free.
// It expects an addr-style input but happily round-trips bare hosts.
norm := emailspool.ExtractDomain("anyone@" + d)
if norm == "" {
norm = strings.ToLower(d)
}
set[norm] = struct{}{}
}
add(raw.MainDomain)
for k := range raw.AddonDomains {
add(k)
}
for _, d := range raw.ParkedDomains {
add(d)
}
for _, d := range raw.SubDomains {
add(d)
}
return set, nil
}
// IsAuthorisedFromDomain reports whether fromDomain is one of the user's
// domains, accounting for subdomain inclusion (a sub.example.com From is
// authorised if example.com is in the set, but the reverse is NOT true).
func IsAuthorisedFromDomain(fromDomain string, authSet map[string]struct{}) bool {
if fromDomain == "" || len(authSet) == 0 {
return false
}
for base := range authSet {
if emailspool.IsSubdomainOrEqual(fromDomain, base) {
return true
}
}
return false
}
package daemon
import (
"sort"
"time"
)
// credentialStuffingDetector tracks, per source IP, the set of distinct
// accounts hit by auth failures inside a sliding window and flags an IP that
// targets many distinct accounts. This is the breadth signal of credential
// stuffing / password spraying -- one source trying many accounts, often with
// only one or two attempts each -- which the count-based pam_bruteforce
// detector (depth: many failures, any account) does not capture. CSM's auth
// sources never expose the attempted password, so the detector keys on the
// distinct-account behavioral signature rather than a password fingerprint.
//
// Concurrency: callers serialize access (the PAM listener holds its mutex
// across Record), so the detector takes no lock of its own.
type credentialStuffingDetector struct {
distinctAccounts int
window time.Duration
now func() time.Time
perIP map[string]*credStuffState
// maxTrackedIPs bounds the live map under sustained source-IP churn.
// Once at the cap the oldest-by-lastSeen entry is evicted before insert.
maxTrackedIPs int
}
type credStuffState struct {
accounts map[string]time.Time
lastSeen time.Time
// fired is set once the IP crosses the distinct-account threshold so a
// single active window yields one finding, not one per additional account.
fired bool
}
// newCredentialStuffingDetector builds a detector. A distinctAccounts
// threshold below 2 has no breadth meaning and is clamped to 2.
func newCredentialStuffingDetector(distinctAccounts int, window time.Duration, now func() time.Time) *credentialStuffingDetector {
if distinctAccounts < 2 {
distinctAccounts = 2
}
if now == nil {
now = time.Now
}
return &credentialStuffingDetector{
distinctAccounts: distinctAccounts,
window: window,
now: now,
perIP: make(map[string]*credStuffState),
maxTrackedIPs: 10000,
}
}
// Record ingests one auth failure for (ip, account). It returns the sorted
// distinct-account list and true exactly once -- when the IP first crosses
// the distinct-account threshold inside the window. Empty ip or account is
// ignored. A failure whose IP has been silent longer than the window resets
// that IP's distinct set so a fresh campaign does not inherit cold counts.
func (d *credentialStuffingDetector) Record(ip, account string) ([]string, bool) {
if d == nil || ip == "" || account == "" {
return nil, false
}
now := d.now()
state, ok := d.perIP[ip]
if ok {
d.pruneAccountWindow(state, now)
}
if ok && len(state.accounts) == 0 {
state = nil
delete(d.perIP, ip)
}
if state == nil {
if d.maxTrackedIPs > 0 && len(d.perIP) >= d.maxTrackedIPs {
d.evictOldest()
}
state = &credStuffState{accounts: make(map[string]time.Time)}
d.perIP[ip] = state
}
state.accounts[account] = now
state.lastSeen = now
if len(state.accounts) < d.distinctAccounts {
state.fired = false
return nil, false
}
if state.fired {
return nil, false
}
state.fired = true
accounts := make([]string, 0, len(state.accounts))
for a := range state.accounts {
accounts = append(accounts, a)
}
sort.Strings(accounts)
return accounts, true
}
// ClearAccount forgets one account's failures from ip: that account logged
// in, so its earlier failures were the user's own. Failures against other
// accounts stay counted; the entry goes only when none remain, and a set
// that drops back under the threshold may fire again once it regrows.
func (d *credentialStuffingDetector) ClearAccount(ip, account string) {
if d == nil {
return
}
state, ok := d.perIP[ip]
if !ok {
return
}
delete(state.accounts, account)
if len(state.accounts) == 0 {
delete(d.perIP, ip)
return
}
if len(state.accounts) < d.distinctAccounts {
state.fired = false
}
}
// PruneStale drops per-IP entries whose lastSeen is older than the window.
// Called from the PAM listener cleanup loop so the detector does not grow
// without bound between window resets. Returns the number pruned.
func (d *credentialStuffingDetector) PruneStale(now time.Time) int {
if d == nil {
return 0
}
pruned := 0
for ip, state := range d.perIP {
d.pruneAccountWindow(state, now)
if len(state.accounts) == 0 {
delete(d.perIP, ip)
pruned++
continue
}
if len(state.accounts) < d.distinctAccounts {
state.fired = false
}
}
return pruned
}
// Clear removes all tracked breadth state for ip after a successful login.
func (d *credentialStuffingDetector) Clear(ip string) {
if d == nil || ip == "" {
return
}
delete(d.perIP, ip)
}
// Configure applies live threshold/window changes. Callers hold their
// external mutex, matching Record's synchronization contract.
func (d *credentialStuffingDetector) Configure(distinctAccounts int, window time.Duration, now time.Time) {
if d == nil {
return
}
if distinctAccounts < 2 {
distinctAccounts = 2
}
d.distinctAccounts = distinctAccounts
d.window = window
for ip, state := range d.perIP {
d.pruneAccountWindow(state, now)
if len(state.accounts) == 0 {
delete(d.perIP, ip)
continue
}
if len(state.accounts) < d.distinctAccounts {
state.fired = false
}
}
}
func (d *credentialStuffingDetector) pruneAccountWindow(state *credStuffState, now time.Time) {
if state == nil {
return
}
for account, seen := range state.accounts {
if now.Sub(seen) > d.window {
delete(state.accounts, account)
}
}
}
// evictOldest removes the entry with the smallest lastSeen so a fresh insert
// stays within maxTrackedIPs. Linear scan, bounded by the cap.
func (d *credentialStuffingDetector) evictOldest() {
var oldestIP string
var oldestAt time.Time
first := true
for ip, state := range d.perIP {
if first || state.lastSeen.Before(oldestAt) {
oldestIP, oldestAt, first = ip, state.lastSeen, false
}
}
if oldestIP != "" {
delete(d.perIP, oldestIP)
}
}
//go:build linux
package daemon
import "github.com/pidginhost/csm/internal/platform"
// cronSpoolDir returns the per-user crontab directory the watcher marks and
// matches event paths against: cronie's /var/spool/cron, or Debian cron's
// /var/spool/cron/crontabs. platform.Detect caches its answer, so this is
// cheap enough for the per-event path checks.
func cronSpoolDir() string {
if cronSpoolWatchDir != "" {
return cronSpoolWatchDir
}
return platform.Detect().CronSpoolDir()
}
package daemon
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"os"
"os/exec"
"os/signal"
"path/filepath"
"slices"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/admissionowner"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/antispam/reconcile"
"github.com/pidginhost/csm/internal/attackdb"
"github.com/pidginhost/csm/internal/auditd"
"github.com/pidginhost/csm/internal/blockdigest"
"github.com/pidginhost/csm/internal/broadcast"
"github.com/pidginhost/csm/internal/challenge"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/crashsample"
"github.com/pidginhost/csm/internal/emailav"
"github.com/pidginhost/csm/internal/emailspool"
"github.com/pidginhost/csm/internal/eximlog"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/firewall/rollback"
"github.com/pidginhost/csm/internal/geoip"
"github.com/pidginhost/csm/internal/health"
"github.com/pidginhost/csm/internal/incident"
"github.com/pidginhost/csm/internal/integrity"
"github.com/pidginhost/csm/internal/ipmap"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/maillog"
"github.com/pidginhost/csm/internal/mailranges"
"github.com/pidginhost/csm/internal/mailstats"
"github.com/pidginhost/csm/internal/mcp"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/modsec"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/parseworker"
"github.com/pidginhost/csm/internal/phptaintworker"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/sdnotify"
"github.com/pidginhost/csm/internal/sharedsenders"
"github.com/pidginhost/csm/internal/signatures"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
"github.com/pidginhost/csm/internal/threatintel"
"github.com/pidginhost/csm/internal/updatecheck"
"github.com/pidginhost/csm/internal/verdict"
"github.com/pidginhost/csm/internal/webui"
"github.com/pidginhost/csm/internal/yara"
"github.com/pidginhost/csm/internal/yaraworker"
)
const eximMainlogPath = platform.EximMainLog
// Daemon is the main persistent monitoring process.
type Daemon struct {
cfg *config.Config
store *state.Store
lock *state.LockFile
binaryPath string
binaryHash binaryHashCache
logWatchers []*LogWatcher
logWatchersMu sync.Mutex
fileMonitor *FileMonitor
fileMonitorMu sync.RWMutex
hijackDetector *PasswordHijackDetector
staleSession401 *staleSession401s
pamListener *PAMListener
controlListener *ControlListener
spoolWatcher *SpoolWatcher
spoolWatcherMu sync.Mutex
forwarderWatcher *ForwarderWatcher
emailQuarantine *emailav.Quarantine
webServer *webui.Server
challengeServer *challenge.Server
ipList *challenge.IPList
challengeGate challenge.PortGate
fwEngine *firewall.Engine
// blockEngine is fwEngine published for the admission owner, which
// starts before the firewall and reads it on its own goroutine.
blockEngine atomic.Pointer[firewall.Engine]
// fwActions retains the durable-action boundary even after failed startup,
// so pending actions remain recoverable while the firewall is unmanaged.
fwActions firewallActionBoundary
fwStartupError string // finalized before status servers start
fwStateAsideMu sync.RWMutex
fwStateAside string
baselineMu sync.Mutex // serialises CmdBaseline handler runs
geoipDB *geoip.DB
geoipMu sync.Mutex // protects geoipDB for publishGeoIP
version string
blockDigest *blockdigest.Collector
knownCustomers *knownCustomers
blockCauses blockCauses
alertCh chan alert.Finding
alertQueue *queuehealth.Tracker
queueSourcesMu sync.RWMutex
queueSources map[string]queueSource
// alertHold, while open, keeps the dispatcher draining alertCh into its
// batch without dispatching, so realtime producers (which never block)
// lose nothing during the synchronous startup baseline. The ingest queue
// health is held with it, so the baseline is not reported as a stall. Closed by
// releaseAlertDispatch once the baseline has published; nil means the
// dispatcher never holds.
alertHold chan struct{}
alertReleaseOnce sync.Once
droppedAlerts int64 // atomic counter for alert channel backpressure drops
stopCh chan struct{}
phpEscalations chan alert.Finding
scanCtx context.Context
scanCancel context.CancelFunc // cancels in-flight periodic scans on shutdown
modsecReload checks.ModSecReloadReconciler
// modsecRegistry carries what the last rule-action refresh learned, so a
// refresh that changes nothing costs content hashing instead of a
// platform probe plus a full reparse of the vendor rule tree.
modsecRegistry modsecRegistryState
abuseReportStop chan struct{}
abuseReportDone chan struct{}
appliedBlocks appliedBlockQueue
wg sync.WaitGroup
smtpAuthTracker *smtpAuthTracker
smtpProbeTracker *smtpProbeTracker
mailAuthTracker *mailAuthTracker
mailSpray *mailSpray
mailboxes mailboxResolver // nil off cPanel
dovecotSessions *dovecotSessionVerdicts
authBackend *authBackendHealth
// stalwart is the Stalwart log state; nil unless this host's mail server
// is Stalwart.
stalwart *stalwartState
// stalwartUnban lifts Stalwart's bans of released addresses; nil unless
// this host's mail server is Stalwart.
stalwartUnban *stalwartUnban
// smtpAuthFromExim is set before the log readers start when the Exim
// mainlog feeds smtpAuthTracker; Postfix pairing is then skipped so one
// attempt is not counted twice.
smtpAuthFromExim bool
startTime time.Time
// mailStats counts mail decisions for the Email Security page on the
// clock mailStatsNow, which also times its restore, saves and pruning.
mailStats *mailstats.Counter
mailStatsNow func() time.Time
// mailStatsLoadFailing is set from a logged load failure until a retry
// merges, so each outcome is logged once.
mailStatsLoadFailing atomic.Bool
// antispamLoop runs antispam reconcile passes on a host antispam
// supports. It stays nil elsewhere and until startup reaches it.
// antispamStarted records that startup reached antispam, after which a
// reload may start the loop; antispamClaimed lets only one start win.
antispamLoop atomic.Pointer[reconcile.Loop]
antispamStarted atomic.Bool
antispamClaimed atomic.Bool
// Namespace replacement must wait for passes that can still create keys
// or publish a learner target. Reloads can still stop learning at once.
antispamReconcileMu sync.Mutex
// antispamLearner is the running learning engine, nil while learning is
// stopped.
antispamLearner atomic.Pointer[learnerRunner]
// antispamLearnerMu serializes reloads, pass results and shutdown.
antispamLearnerMu sync.Mutex
// antispamWatchReset keeps a failed disable reset pending before any
// re-enable. Protected by antispamLearnerMu.
antispamWatchReset bool
// antispamHealth tracks rspamd's scan socket and Exim's scan markers
// while antispam is enabled.
antispamHealth antispamHealthState
// ipmapWriter writes the CSM IP map; the Exim log path reads its last
// map before startAntispam sets it, so it is atomic.
ipmapWriter atomic.Pointer[ipmap.Writer]
// ipmapMu orders refreshes with reconciliation's writes and removals.
// A removed map stays absent until reconciliation requests it again.
ipmapMu sync.Mutex
ipmapRemoved atomic.Bool
// Canary failures belong to one tracking period and socket; ipmapMu guards both.
ipmapCanaryMisses int
ipmapCanarySocket string
// antispamFeeds is also read by the mail log path before startup finishes.
antispamFeeds atomic.Pointer[reconcile.FeedState]
// rspamdStats is written by reconcile passes and read by web UI requests.
rspamdStats atomic.Pointer[rspamdStatsSource]
// antispamScanSource is written by reconcile passes and read by the mail
// log and rspamd log watchers; nil until a pass says.
antispamScanSource atomic.Pointer[antispam.ScanSource]
// spamSources is set by startAbuseReporting after the log watchers that
// read it have started.
spamSources atomic.Pointer[spamSourceRunner]
// spamSourceDNS is the resolver of the spam source detector; nil means
// the system resolver. Tests set it before startAbuseReporting.
spamSourceDNS sharedsenders.Resolver
// forwardGuardMu prevents overlapping attempts from applying stale config.
forwardGuardMu sync.Mutex
// forwardGuardRetry retains failed or deferred reconcile requests.
forwardGuardRetry atomic.Bool
// forwardGuardWake hands reload requests to the forward-guard loop.
forwardGuardWake chan struct{}
// yaraSup is the supervised YARA-X worker, wired up when
// yaraWorkerOn(cfg) is true (the default; see ROADMAP item 2).
// Nil when the in-process scanner is in use
// (cfg.Signatures.YaraWorkerEnabled explicitly set to false).
yaraSup *yaraworker.Backend
yaraCrashMu sync.Mutex
yaraLastCrashAlert time.Time
realtimeRulesMu sync.Mutex
realtimeRulesState string
// phpTaintSup is the mandatory process boundary around the PHP taint
// parser. It starts its child lazily on the first admitted source.
phpTaintSup *phptaintworker.Supervisor
// zipLister lists tenant zips for the phishing-kit check in a locked-down
// parse worker unit, started on first use.
zipLister *parseworker.Lister
// mailExtractor unpacks email attachments for the spool watcher in
// locked-down parse worker units, one per scan goroutine. Nil when email
// AV is off or the workers could not be set up.
mailExtractor *parseworker.Extractor
// crashSamples is the parse crash sample store; it is swept even when
// email_av.keep_crash_samples is off. Nil without a state path.
crashSamples *crashsample.Store
// parseFindings rate-limits parse worker findings across the daemon and
// its spool watchers, so one condition raises one finding an hour.
parseFindings findingLimiter
// policies holds the email PHP-relay pattern policies
// (suspicious/safe x-mailer classes, HTTP proxy ranges) loaded
// from EmailProtection.PHPRelay.PoliciesDir. Initialised in O2
// (daemon wiring); stays nil until then. The SIGHUP path
// nil-guards the Reload call so this commit is a no-op at
// runtime until O2 lands.
policies *emailspool.Policies
// botVerifier is the async rDNS bot verifier. Retained so the SIGHUP
// path can push reloaded reputation.verified_bots into it. nil when
// bot verification is disabled or no store is available.
botVerifier *threatintel.AsyncBotVerifier
// PHP-relay components wired by startPHPRelay (Linux only). The fields are
// declared cross-platform but stay nil on non-cPanel or non-Linux hosts.
autoFreezer *autoFreezer
phpRelayShutdown []func() // ordered shutdown hooks
// watcherStatus tracks which top-level watchers have successfully attached.
// Keys are short stable names ("fanotify", "audit", "spool", "modsec",
// "afalg"). Values flip from false-to-true when the watcher's setup
// function completes without error. Used by /api/v1/status and the
// sd_notify gate.
//
// watcherChangedAt records the wall-clock time of the most recent state
// transition for the same key. Driven from MarkWatcher; consumed by the
// /api/v1/components endpoint so operators can see how long a watcher
// has been in its current state.
watcherMu sync.RWMutex
watcherStatus map[string]bool
watcherChangedAt map[string]time.Time
// watcherUpstream maps a watcher name to a probe function that reports
// whether the upstream feeding the watcher is still active. The probe
// runs at most once per /api/v1/components scrape; results compose with
// WatcherStatuses to surface "deaf" (attached but no upstream traffic)
// distinct from "idle" (attached and quiet) in the dashboard.
watcherUpstream map[string]UpstreamProbe
// findingBus fans out dispatched findings to passive observers like
// the SSE event stream. Initialized in Run(); closed on shutdown.
findingBus *broadcast.Bus
// updateChecker polls upstream for new CSM releases. Wired in Run()
// when updates.check_enabled is true (default). Nil when disabled
// or before Run starts; UpdateInfo() handles that.
updateChecker *updatecheck.Checker
// scanJobs is the full-scan job manager. Wired in Run() after the global
// bbolt store is open. Nil when the store is unavailable at startup.
scanJobs *ScanJobManager
// admission owns the admission ledger in the state database. Wired in
// Run() before any detector starts; nil without a state database.
admission *admissionowner.Owner
// lastAutomationActionCache memoises the newest automation-emitted
// finding so /api/v1/status does not run a 100-row history cursor on
// every poll. Invalidated after lastAutomationActionTTL elapses.
automationActionMu sync.Mutex
automationActionCache *health.AutomationAction
automationActionCached time.Time
// removedFileSeq is the last file removal whose alert state was cleared.
removedFileSeq atomic.Uint64
}
const lastAutomationActionTTL = 5 * time.Second
var logWatcherRetryInterval = 60 * time.Second
// New creates a new daemon instance.
func New(cfg *config.Config, store *state.Store, lock *state.LockFile, binaryPath string) *Daemon {
d := &Daemon{
cfg: cfg,
store: store,
lock: lock,
binaryPath: binaryPath,
alertCh: make(chan alert.Finding, 500),
stopCh: make(chan struct{}),
appliedBlocks: appliedBlockQueue{wake: make(chan struct{}, 1)},
forwardGuardWake: make(chan struct{}, 1),
mailStats: mailstats.New(time.Now),
mailStatsNow: time.Now,
}
d.phpEscalations = make(chan alert.Finding, cap(d.alertCh))
// Remediation records what it wrote here, so the sensitive-file detectors
// can tell CSM's own change from a third party's after a restart.
checks.SetSelfWriteStore(store)
if store != nil {
d.registerQueueSource("state", store)
}
d.smtpAuthTracker = newSMTPAuthTracker(
cfg.Thresholds.SMTPBruteForceThreshold,
cfg.Thresholds.SMTPBruteForceSubnetThresh,
cfg.Thresholds.SMTPAccountSprayThreshold,
time.Duration(cfg.Thresholds.SMTPBruteForceWindowMin)*time.Minute,
time.Duration(cfg.Thresholds.SMTPBruteForceSuppressMin)*time.Minute,
cfg.Thresholds.SMTPBruteForceSlowThreshold,
time.Duration(cfg.Thresholds.SMTPBruteForceSlowWindowMin)*time.Minute,
cfg.Thresholds.SMTPBruteForceMaxTracked,
time.Now,
)
d.smtpProbeTracker = newSMTPProbeTracker(
cfg.Thresholds.SMTPProbeThreshold,
time.Duration(cfg.Thresholds.SMTPProbeWindowMin)*time.Minute,
time.Duration(cfg.Thresholds.SMTPProbeSuppressMin)*time.Minute,
cfg.Thresholds.SMTPProbeMaxTracked,
time.Now,
smtpProbeBlockExpiryString,
)
d.mailAuthTracker = newMailAuthTracker(
cfg.Thresholds.MailBruteForceThreshold,
cfg.Thresholds.MailBruteForceSubnetThresh,
cfg.Thresholds.MailAccountSprayThreshold,
time.Duration(cfg.Thresholds.MailBruteForceWindowMin)*time.Minute,
time.Duration(cfg.Thresholds.MailBruteForceSuppressMin)*time.Minute,
cfg.Thresholds.MailBruteForceSlowThreshold,
time.Duration(cfg.Thresholds.MailBruteForceSlowWindowMin)*time.Minute,
cfg.Thresholds.MailBruteForceMaxTracked,
time.Now,
)
// time.Now is lock-free; mailSpray calls it under its own mutex while a
// tracker may hold its lock.
d.mailSpray = newMailSpray(sprayConfigFromThresholds(cfg), time.Now)
d.smtpAuthTracker.SetSpray(d.mailSpray)
d.mailAuthTracker.SetSpray(d.mailSpray)
d.dovecotSessions = newDovecotSessionVerdicts()
d.knownCustomers = newKnownCustomers(time.Now)
d.knownCustomers.isInfra = func(ip string) bool { return isInfraIPDaemon(ip, d.currentCfg().InfraIPs) }
d.smtpAuthTracker.SetKnownCustomers(d.knownCustomers)
d.mailAuthTracker.SetKnownCustomers(d.knownCustomers)
// Mail auth backend (cpdoveauthd) health: an active socket probe drives
// brute-force suppression for both trackers and optional self-heal restart.
// The probe goroutine only runs on cPanel (see Start); elsewhere Degraded()
// stays false, so the trackers behave exactly as before.
graceDur := 10 * time.Minute
if g, perr := time.ParseDuration(cfg.AutoResponse.MailAuthRecovery.DownGrace); perr == nil && g > 0 {
graceDur = g
}
restartCmd := cfg.AutoResponse.MailAuthRecovery.RestartCommand
d.authBackend = newAuthBackendHealth(
time.Now,
dialMailAuthBackend,
func() error { return restartMailAuthBackend(restartCmd) },
cfg.AutoResponse.MailAuthRecovery.RestartEnabled,
graceDur,
mailAuthRestartCooldown,
cfg.AutoResponse.MailAuthRecovery.MaxRestartsPerHour,
)
d.mailAuthTracker.SetBackendDownCheck(d.authBackend.Degraded)
d.smtpAuthTracker.SetBackendDownCheck(d.authBackend.Degraded)
return d
}
// reconcileBruteThresholds pushes the live config's SMTP/mail brute-force
// thresholds into the running trackers. The `thresholds` block is tagged
// hotreload:"safe", so a SIGHUP that only changes those fields reports success;
// without this push the trackers would keep their startup values until a full
// restart. Called from the reload success path, mirroring reconcileVerifiedBots.
func (d *Daemon) reconcileBruteThresholds() {
cfg := d.activeOrStartupCfg()
if cfg == nil {
return
}
th := cfg.Thresholds
if d.smtpAuthTracker != nil {
d.smtpAuthTracker.SetThresholds(
th.SMTPBruteForceThreshold,
th.SMTPBruteForceSubnetThresh,
th.SMTPAccountSprayThreshold,
time.Duration(th.SMTPBruteForceWindowMin)*time.Minute,
time.Duration(th.SMTPBruteForceSuppressMin)*time.Minute,
th.SMTPBruteForceSlowThreshold,
time.Duration(th.SMTPBruteForceSlowWindowMin)*time.Minute,
th.SMTPBruteForceMaxTracked,
)
}
if d.smtpProbeTracker != nil {
d.smtpProbeTracker.SetThresholds(
th.SMTPProbeThreshold,
time.Duration(th.SMTPProbeWindowMin)*time.Minute,
time.Duration(th.SMTPProbeSuppressMin)*time.Minute,
th.SMTPProbeMaxTracked,
)
}
if d.mailAuthTracker != nil {
d.mailAuthTracker.SetThresholds(
th.MailBruteForceThreshold,
th.MailBruteForceSubnetThresh,
th.MailAccountSprayThreshold,
time.Duration(th.MailBruteForceWindowMin)*time.Minute,
time.Duration(th.MailBruteForceSuppressMin)*time.Minute,
th.MailBruteForceSlowThreshold,
time.Duration(th.MailBruteForceSlowWindowMin)*time.Minute,
th.MailBruteForceMaxTracked,
)
}
if d.mailSpray != nil {
d.mailSpray.SetConfig(sprayConfigFromThresholds(cfg))
}
}
// SetThresholds swaps the SMTP auth-brute detector thresholds under the
// tracker mutex so a SIGHUP reload of the safe `thresholds` block reaches this
// live tracker. Co-located with reconcileBruteThresholds, its only caller.
func (t *smtpAuthTracker) SetThresholds(perIP, subnet, accountSpray int, window, suppression time.Duration, slowThreshold int, slowWindow time.Duration, maxTracked int) {
t.mu.Lock()
defer t.mu.Unlock()
t.perIPThreshold = perIP
t.subnetThreshold = subnet
t.accountSprayThreshold = accountSpray
t.window = window
t.suppression = suppression
t.slowThreshold = slowThreshold
t.slowWindow = slowWindow
t.maxTracked = maxTracked
}
// SetThresholds swaps the mail auth-brute detector thresholds under the tracker
// mutex for a live SIGHUP reload.
func (t *mailAuthTracker) SetThresholds(perIP, subnet, accountSpray int, window, suppression time.Duration, slowThreshold int, slowWindow time.Duration, maxTracked int) {
t.mu.Lock()
defer t.mu.Unlock()
t.perIPThreshold = perIP
t.subnetThreshold = subnet
t.accountSprayThreshold = accountSpray
t.window = window
t.suppression = suppression
t.slowThreshold = slowThreshold
t.slowWindow = slowWindow
t.maxTracked = maxTracked
}
// SetThresholds swaps the SMTP connect-probe detector thresholds under the
// tracker mutex for a live SIGHUP reload.
func (t *smtpProbeTracker) SetThresholds(threshold int, window, suppression time.Duration, maxTracked int) {
t.mu.Lock()
defer t.mu.Unlock()
t.threshold = threshold
t.window = window
t.suppression = suppression
t.maxTracked = maxTracked
}
// SetVersion sets the application version for display in the web UI.
func (d *Daemon) SetVersion(v string) {
d.version = v
}
// MarkWatcher records the attachment state of a named watcher.
// Call from each watcher's startup path: true on success, false on failure.
// The first record and any subsequent state transition stamps
// watcherChangedAt so the components view can show "since".
func (d *Daemon) MarkWatcher(name string, attached bool) {
d.watcherMu.Lock()
defer d.watcherMu.Unlock()
if d.watcherStatus == nil {
d.watcherStatus = make(map[string]bool)
}
if d.watcherChangedAt == nil {
d.watcherChangedAt = make(map[string]time.Time)
}
prev, existed := d.watcherStatus[name]
if !existed || prev != attached {
d.watcherChangedAt[name] = time.Now()
}
d.watcherStatus[name] = attached
}
// WatcherStatuses returns a snapshot of every recorded watcher.
func (d *Daemon) WatcherStatuses() map[string]bool {
d.watcherMu.RLock()
defer d.watcherMu.RUnlock()
out := make(map[string]bool, len(d.watcherStatus))
for k, v := range d.watcherStatus {
out[k] = v
}
return out
}
// WatcherChangedAt returns the wall-clock time at which each watcher last
// transitioned state. Watchers without a recorded change return the zero
// value.
func (d *Daemon) WatcherChangedAt() map[string]time.Time {
d.watcherMu.RLock()
defer d.watcherMu.RUnlock()
out := make(map[string]time.Time, len(d.watcherChangedAt))
for k, v := range d.watcherChangedAt {
out[k] = v
}
return out
}
// UpstreamProbe reports whether a watcher's upstream input source is
// still feeding it. Return Fresh=false when the watcher is attached but
// can no longer hear from its source (PAM module not installed, log file
// rotated and never reappeared, fanotify marks lost, etc.) so the
// dashboard can surface "deaf" instead of conflating it with "idle".
type UpstreamProbe func() health.UpstreamResult
// RegisterUpstreamProbe wires a probe for a named watcher. Safe to call
// from any watcher's startup path; repeated calls overwrite the previous
// probe. Probes run on the request thread of /api/v1/components, so they
// must be cheap (single stat / atomic load, not a syscall storm).
func (d *Daemon) RegisterUpstreamProbe(name string, probe UpstreamProbe) {
d.watcherMu.Lock()
defer d.watcherMu.Unlock()
if d.watcherUpstream == nil {
d.watcherUpstream = make(map[string]UpstreamProbe)
}
d.watcherUpstream[name] = probe
}
// WatcherUpstream returns a snapshot of every probed watcher's upstream
// state. Watchers without a registered probe are absent from the map; the
// components API treats absence as "no probe wired, do not surface a
// deaf verdict for this watcher".
func (d *Daemon) WatcherUpstream() map[string]health.UpstreamResult {
d.watcherMu.RLock()
probes := make(map[string]UpstreamProbe, len(d.watcherUpstream))
for k, v := range d.watcherUpstream {
probes[k] = v
}
d.watcherMu.RUnlock()
out := make(map[string]health.UpstreamResult, len(probes))
for name, probe := range probes {
if probe == nil {
continue
}
out[name] = probe()
}
return out
}
// buildInfoOnce guards process-wide registration of the build_info
// gauge so repeated daemon construction in tests does not panic.
var buildInfoOnce sync.Once
// storeSizeOnce guards the csm_store_size_bytes gauge hook so tests
// that create multiple daemons share a single registration.
var storeSizeOnce sync.Once
// registerBuildInfo exposes build metadata on /metrics in the
// conventional Prometheus shape: a gauge fixed at 1, with the
// interesting fields as labels so scrapers can join on them.
func (d *Daemon) registerBuildInfo() {
buildInfoOnce.Do(func() {
g := metrics.NewGaugeVec(
"csm_build_info",
"CSM build metadata. Value is always 1; read version from the label.",
[]string{"version"},
)
version := d.version
if version == "" {
version = "unknown"
}
g.With(version).Set(1)
metrics.MustRegister("csm_build_info", g)
})
}
// registerStoreSizeMetric exposes the bbolt on-disk size as a gauge
// that stats the file at scrape time. No caching: the expected scrape
// interval is 15+ seconds and stat is cheap.
func (d *Daemon) registerStoreSizeMetric() {
storeSizeOnce.Do(func() {
metrics.RegisterGaugeFunc(
"csm_store_size_bytes",
"On-disk size of the bbolt state database in bytes.",
func() float64 {
db := store.Global()
if db == nil {
return 0
}
info, err := os.Stat(db.Path())
if err != nil {
return 0
}
return float64(info.Size())
},
)
})
}
// firewallMetricsOnce guards /metrics registration of the firewall +
// blocked-IP gauges so repeated daemon starts in a test binary are
// idempotent.
var firewallMetricsOnce sync.Once
var firewallMetricsMu sync.RWMutex
var firewallMetricsEngine *firewall.Engine
func setFirewallMetricsEngine(engine *firewall.Engine) {
firewallMetricsMu.Lock()
defer firewallMetricsMu.Unlock()
firewallMetricsEngine = engine
}
func firewallMetricsRuleCounts() firewall.RuleCounts {
firewallMetricsMu.RLock()
engine := firewallMetricsEngine
firewallMetricsMu.RUnlock()
if engine == nil {
return firewall.RuleCounts{}
}
return engine.RuleCounts()
}
func (d *Daemon) setFirewallEngine(engine *firewall.Engine) {
// A confirmation window a restart interrupted can still roll the
// firewall back: hold admission's batches before the owner can see the
// engine, until startup recovery settles the window (design C9).
if engine != nil && d.cfg != nil {
holdFirewallBatchesForWindow(d.cfg.StatePath, engine)
}
d.fwEngine = engine
d.blockEngine.Store(engine)
d.fwActions = nil
if boundary, ok := any(engine).(firewallActionBoundary); ok {
d.fwActions = boundary
}
setFirewallMetricsEngine(engine)
}
// registerFirewallMetrics exposes the count of blocked IPs and the
// total number of firewall rules (IPs + allowed + subnets + port
// allow entries). Both gauges read the firewall engine at scrape time:
// the engine state file is authoritative, while the parallel bbolt
// fw:* buckets are written only at migration, so reading the store
// would freeze the gauge at the migration-time snapshot.
func (d *Daemon) registerFirewallMetrics() {
setFirewallMetricsEngine(d.fwEngine)
firewallMetricsOnce.Do(func() {
metrics.RegisterGaugeFunc(
"csm_blocked_ips_total",
"Number of IPs currently on the firewall block list (excluding expired temp bans).",
func() float64 {
return float64(firewallMetricsRuleCounts().Blocked)
},
)
metrics.RegisterGaugeFunc(
"csm_firewall_rules_total",
"Total firewall rules across all categories (blocked IPs, allowed IPs, blocked subnets, port-specific allows).",
func() float64 {
return float64(firewallMetricsRuleCounts().Total())
},
)
})
}
// Run starts the daemon and blocks until stopped.
func (d *Daemon) Run() error {
d.alertQueue = queuehealth.New(cap(d.alertCh), time.Minute)
defer alert.RegisterQueue(d.alertCh, d.alertQueue)()
if err := d.checkObserveStartupRecovery(); err != nil {
return err
}
d.startTime = time.Now()
defer alert.ClosePhpanelQueues()
if d.store != nil {
d.store.EnsureBaseline(d.startTime)
}
// Initialize structured logging from environment (CSM_LOG_FORMAT,
// CSM_LOG_LEVEL). The default text handler preserves the legacy
// "[YYYY-MM-DD HH:MM:SS] msg" format so operators mixing csmlog
// with legacy fmt.Fprintf call sites see a uniform log stream.
// CSM_LOG_FORMAT=json switches to structured JSON for log shipping.
csmlog.Init()
csmlog.Info("CSM daemon starting")
if err := alert.ConfigurePhpanelQueue(d.cfg); err != nil {
return fmt.Errorf("initializing phpanel webhook queue: %w", err)
}
// Expose Go runtime memory/scheduler stats on /metrics (heap growth is
// observable for leak triage), and start the loopback pprof listener when
// the operator opts in via debug.pprof_listen.
metrics.RegisterRuntimeMetrics()
if d.cfg != nil && d.cfg.Debug.PprofListen != "" {
d.startPprofListener(d.cfg.Debug.PprofListen)
}
// Periodic scans use this context so shutdown can abort an in-flight tier
// instead of blocking the worker drain for the full check budget.
scanCtx, scanCancel := context.WithCancel(context.Background())
d.scanCtx = scanCtx
d.scanCancel = scanCancel
defer scanCancel()
// Wire the active config to the incident-singleton's auto-close
// loop BEFORE the singleton is constructed, so the loop reads the
// operator-supplied thresholds on its first sweep. The closure
// captures the Daemon to pick up reloaded configs without restart.
SetIncidentConfigSource(func() *config.Config { return d.currentCfg() })
// Initialize the findings broadcast bus so passive observers (SSE, etc.)
// can subscribe before any findings are dispatched.
d.installFindingBus()
// Install config-supplied platform overrides BEFORE the first Detect()
// call so every check sees the merged view. The daemon command installs
// them even earlier (before the challenge-snippet refresh detects the
// platform); this call re-asserts them and reports if detection ran
// first without them.
InstallPlatformOverrides(d.cfg)
// Log detected platform as a structured record. In text mode this
// comes out as "[ts] platform detected os=X panel=Y ..."; in
// JSON mode as {"msg":"platform detected","os":"X","panel":"Y",...}.
pi := platform.Detect()
csmlog.Info("platform detected",
"os", orUnknown(string(pi.OS)),
"os_version", orUnknown(pi.OSVersion),
"panel", orNone(string(pi.Panel)),
"webserver", orNone(string(pi.WebServer)),
)
// Build the ModSec rule-action registry before any log watcher starts
// so the LiteSpeed classifier can tell pass-action vendor rules apart
// from real denies on the very first parsed line.
d.initModSecRegistry()
// Startup rollback can restore config and restart before reaching the
// watchers or integrity verification, so its sink must already be ready.
d.installActionLog()
// Wire the firewall tentative-apply manager. Recovery has to run
// before integrity.Verify because a pending rollback whose deadline
// passed while the daemon was down restores the previous csm.yaml
// (and its integrity hash) to disk; verifying first would fail
// against the still-on-disk new config the operator never confirmed.
if sdb := store.Global(); sdb != nil {
mgr := rollback.NewManager(sdb, d.cfg.ConfigFile, rollback.SystemctlRestart, time.Now)
rollback.SetGlobal(mgr)
recoveryCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
reverted, rerr := mgr.RecoverOnStartup(recoveryCtx)
cancel()
if rerr != nil {
return fmt.Errorf("firewall rollback recovery failed: %w", rerr)
}
if reverted {
csmlog.Warn("firewall rollback expired during downtime; previous config restored, restart issued")
// systemctl restart csm.service from the manager will tear
// us down momentarily; bail out cleanly so we do not race
// the watchers we are about to start against the restart.
return nil
}
}
// Verify integrity on startup
if err := verifyStartupIntegrity(d.binaryPath, d.cfg); err != nil {
tamper := alert.Finding{
Severity: alert.Critical,
Check: "integrity",
Message: fmt.Sprintf("BINARY/CONFIG TAMPER DETECTED: %v", err),
Timestamp: time.Now(),
}
_ = alert.Dispatch(d.cfg, []alert.Finding{tamper})
return fmt.Errorf("integrity check failed: %w", err)
}
// Publish the verified config as the process-wide live pointer.
// Hot paths (check ticks, alert dispatch, etc.) call
// config.Active() to pick up the current snapshot so a SIGHUP
// reload is visible on the next call without restart.
publishActiveConfig(d.cfg, "startup")
// Install the mail-brute account-key extractor selected by config.
// Validation in config.Load() already rejected invalid specs, so the
// error path here is defense-in-depth only.
if err := installAccountExtractorFromConfig(d.cfg); err != nil {
return err
}
// The threat and attack databases record applied blocks; open them
// before the owner delivers rows a previous run left pending.
checks.InitThreatDB(d.cfg.StatePath, d.cfg.Reputation.Whitelist)
if db := checks.GetThreatDB(); db != nil {
fmt.Fprintf(os.Stderr, "[%s] Threat DB initialized (%d entries)\n", ts(), db.Count())
}
if adb := attackdb.Init(d.cfg.StatePath); adb != nil {
d.prepareAttackDatabase(adb)
}
// Restore the customer logins of the last run before any block can be
// checked against them.
d.loadKnownCustomers()
// Recovered admission notices can dispatch as soon as the owner starts.
// Publish the collector before any producer can read it or call its hook.
d.blockDigest = d.buildBlockDigest(d.currentCfg())
alert.SetSummaryHook(d.noteSummary)
// Own the admission ledger before anything can respond automatically.
// The legacy consumers of its applied blocks run beside it.
d.wg.Add(1)
obs.Go("admission-applied-blocks", func() {
defer d.wg.Done()
d.appliedBlocks.serve(d, d.stopCh)
})
d.startAdmission()
d.applyStartupIntegrations()
// Initialize signature scanners and threat DB (fast, no I/O scan)
d.registerBuildInfo()
d.registerStoreSizeMetric()
d.registerFirewallMetrics()
checks.RegisterDirectSMTPEgressMetrics(metrics.Default())
RegisterBPFEnforcementMetrics(metrics.Default())
mailranges.RegisterMailrangesMetrics(metrics.Default())
if err := d.initYaraBackend(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] YARA backend init: %v\n", ts(), err)
}
// A realtime engine with zero rules is a silent outage; say so now that
// both engines have had their startup load.
d.reportRealtimeRuleCoverageNow()
if err := d.initPHPTaintAnalyzer(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] PHP taint worker init: %v\n", ts(), err)
} else {
defer d.stopPHPTaintAnalyzer()
}
if err := d.initParseWorkers(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] parse worker init: %v\n", ts(), err)
} else {
defer d.stopParseWorkers()
}
d.initCrashSamples()
d.wg.Add(1)
obs.Go("parse-crash-sweep", d.crashSampleSweep)
// Install operator verified-bot ranges before the firewall engine is
// exposed and before the initial auto-response pass. The firewall's
// soft-allow gate consults this registry from its first block decision.
startupBotEntries := verifiedBotEntries(d.cfg)
threatintel.SetOperatorBots(startupBotEntries)
// Load the mail-provider IP range cache synchronously before startFirewall()
// so engine.SetDOSExemptProviderNets receives a full provider set and the
// first Apply() builds dos_exempt_nets with current data. The background
// refresh goroutine is registered here as well.
d.initMailRanges()
// Set before the firewall installs its release hook, which reads it.
d.stalwartUnban = newStalwartUnban(d)
// Start firewall engine if enabled
d.startFirewall()
// Settle any apply-confirmed window the restart interrupted. Must run
// after startFirewall (an expired-window restore has to land on top of
// the candidate ruleset the startup Apply just re-applied) and before
// startControlListener (so confirm/cancel cannot race the recovery).
d.recoverFirewallApplyConfirmed()
// Construct the incident correlator after the firewall starts so the
// credential_spray block hand-off is installed before the singleton
// captures its config. This still happens before any finding is
// dispatched.
_ = IncidentCorrelator()
// Start challenge server if enabled (gray listing)
d.startChallengeServer()
// Start challenge escalation ticker
if d.ipList != nil {
d.wg.Add(1)
obs.Go("challenge-escalator", d.challengeEscalator)
}
// Create password hijack detector
d.hijackDetector = NewPasswordHijackDetector(d.cfg, d.alertCh, d.stopCh)
d.staleSession401 = newStaleSession401s()
// Removals from before a restart keep a returning file recognisable.
// Loaded before anything can respond to or report a file.
checks.LoadRemovedFiles(d.cfg.StatePath)
// The dispatcher, the log watchers and the web UI all act through the
// mail backend, so it is chosen before any of them starts.
selectMailBackend(d.currentCfg)
// Start the alert dispatcher before any producer, held until the baseline
// scan below has published. Producers send non-blocking, so a dispatcher
// that started only after the baseline silently lost every realtime
// finding beyond the channel buffer during those minutes.
d.holdAlertDispatch()
d.wg.Add(1)
obs.Go("alert-dispatcher", d.alertDispatcher)
d.wg.Add(1)
obs.Go("php-escalation", d.phpEscalationWorker)
// Start inotify log watchers
d.startLogWatchers()
// Start PAM listener for real-time brute-force detection
d.startPAMListener()
// Wire the full-scan job manager. Must run after the global bbolt store is
// open so NewScanJobManager can resolve store.Global(). Failure here is
// non-fatal for the rest of the daemon: periodic scans and the webui
// continue; only explicit full-scan commands are unavailable.
if db := store.Global(); db != nil {
if sjm, err := d.startScanJobManager(); err != nil {
csmlog.Warn("scan job manager unavailable", "err", err)
} else {
d.scanJobs = sjm
}
}
// Start control socket listener for the thin-client CLI. The
// daemon is the sole bbolt owner; CLI commands that previously
// raced for the lock now route through this socket.
d.startControlListener()
// Start fanotify file monitor (real-time detection starts immediately)
d.startFileMonitor()
// Start email AV spool watcher (separate fanotify for Exim spool).
// Spool and forwarder watchers are cPanel-only; they watch paths
// (/var/spool/exim, /etc/valiases) that only exist on cPanel hosts.
if platform.Detect().IsCPanel() {
d.startSpoolWatcher()
d.startForwarderWatcher()
}
// Wire the update checker before the Web UI starts so the
// /api/v1/status handler always sees a non-nil checker (the
// goroutine that polls upstream still warms up for 5 minutes
// before the first poll, but UpdateInfo() returns zero values
// without racing on the field assignment).
d.startUpdateChecker()
// Start Web UI server - available immediately, before initial scan
d.startWebUI()
// Wire email quarantine to web server (after both start)
d.syncEmailAVWebState()
// Initialize GeoIP databases (after webServer so SetGeoIPDB can attach)
d.initGeoIP()
// Signal systemd we're up as soon as the real-time watchers and the
// control surfaces are attached. The initial baseline scan, the
// kernel-state probes, and the BPF tracker wiring below all run
// inline but no longer block `systemctl is-active` / `systemctl
// restart`. The watchdog notifier has to start in the same step so
// systemd's WatchdogSec doesn't trip while the baseline scan is
// still running on a large host.
d.wg.Add(1)
obs.Go("watchdog-notifier", d.watchdogNotifier)
d.wg.Add(1)
obs.Go("queue-health", d.monitorQueueHealth)
if sent, err := sdnotify.Ready(); err != nil {
fmt.Fprintf(os.Stderr, "sd_notify READY failed: %v\n", err)
} else if sent {
fmt.Fprintf(os.Stderr, "sd_notify: daemon ready\n")
}
_, _ = sdnotify.Status(fmt.Sprintf("watchers attached: %d", countAttachedWatchers(d.WatcherStatuses())))
// Reconcile the opt-in email forward-guard to the current config (installs
// the exim rule when enabled+enforcing, removes it otherwise), then keep its
// bad-IP lookup fresh. Both no-op off cPanel and fail open on error.
d.reconcileForwardGuard()
d.wg.Add(1)
obs.Go("forward-guard-refresh", d.forwardGuardRefresher)
// Reconcile the opt-in antispam integration and probe rspamd while it is
// enabled. Started after the forward guard's own reconcile, so the two
// startup transactions do not contend for the Exim lock. cPanel and
// Postfix hosts only.
d.startAntispam()
// Snapshot the live config ONCE for the entire initial-scan tick
// (detection + auto-response). Earlier code called d.currentCfg()
// for RunTier and then again later for the auto-response batch;
// a SIGHUP landing between the two reads split the same tick
// across old detection policy and new response policy.
initialCfg := d.currentCfg()
// Start delivery once GeoIP is ready. Findings counted since the collector
// was published are merged with its saved window before catch-up delivery.
bdInterval := initialCfg.BlockDigestInterval()
d.wg.Add(1)
obs.Go("block-digest", func() {
defer d.wg.Done()
t := time.NewTicker(bdInterval)
defer t.Stop()
d.blockDigest.Run(d.stopCh, t.C)
})
d.wg.Add(1)
obs.Go("known-customers", d.runKnownCustomers)
// Run the initial scan synchronously while alert dispatch is held.
fmt.Fprintf(os.Stderr, "[%s] Running initial baseline scan...\n", ts())
initialFindings, initialPurge := checks.RunTierWithContext(d.scanContext(), initialCfg, d.store, checks.TierCritical)
// Seed the attack database with initial scan findings
if adb := attackdb.Global(); adb != nil {
for _, f := range initialFindings {
adb.RecordFinding(f)
}
}
newFindings, permFixedKeys := d.respondToInitialScan(initialCfg, initialFindings)
// Remove auto-fixed findings before storing to UI
if len(permFixedKeys) > 0 {
initialFindings = withoutFixedFindings(initialFindings, permFixedKeys)
}
// Merge initial scan findings into the existing set. Previous deep scan
// results (outdated_plugins, wp_core, etc.) persist across restarts until
// the next deep scan replaces them. ClearLatestFindings is NOT called
// here - it would wipe deep scan findings that haven't re-run yet.
checks.StoreLatestScanFindings(d.store, initialPurge, initialFindings)
csmlog.Info("initial scan complete", "findings", len(initialFindings), "new", len(newFindings))
// NOW let the dispatcher dispatch - no more race with the initial scan,
// and nothing queued while it ran was lost.
d.releaseAlertDispatch()
d.refreshTrustedProxyFinding()
// Findings the previous shutdown drained but could not act on get their
// auto-response and alerts now, through the same pipeline.
d.replayPendingFindings()
// Retrospective cloud-relay scan: replay the last 24h of exim_mainlog
// through the compromise-detection rule so any in-progress credential
// abuse is surfaced within seconds of daemon start, not after the
// realtime watcher sees a new line. Gated on cPanel because
// exim_mainlog is cPanel-specific; safe to run in a goroutine so
// startup isn't delayed by parsing a large log.
if platform.Detect().IsCPanel() {
d.wg.Add(1)
obs.Go("cloud-relay-retro-scan", func() {
defer d.wg.Done()
cfg := d.currentCfg()
retro := ScanEximHistoryForCloudRelay(cfg, "", time.Now(), 24*time.Hour)
for i, f := range retro {
// Enqueue the finding FIRST; only after it is
// accepted by the dispatcher do we trigger the
// account-suspend side-effect. This prevents a
// silent mailbox suspension if the daemon begins
// shutting down between these two operations.
if !alert.Enqueue(d.alertCh, f, d.stopCh) {
alert.RecordQueueLoss(d.alertCh, uint64(len(retro[i+1:])))
return
}
retroCloudRelayResponse(cfg, f)
}
})
}
// Start periodic scanners
d.wg.Add(1)
obs.Go("critical-scanner", d.criticalScanner)
d.wg.Add(1)
obs.Go("deep-scanner", d.deepScanner)
d.wg.Add(1)
obs.Go("companion-sweeper", d.companionSweeper)
// Async bot-rDNS verifier: runs PTR+forward-A verification for
// claimed search-engine bot IPs that are not in a static range.
// Gated on reputation.bot_verify_enabled (default true) and on a
// non-nil store so the result can be persisted.
// Operator-configured verified bots extend the built-in allowlist.
// They were installed before firewall startup so auto-block soft-allow
// checks see them during the initial baseline pass; refresh from the
// current config here in case a SIGHUP landed during startup.
botEntries := verifiedBotEntries(d.currentCfg())
threatintel.SetOperatorBots(botEntries)
if d.cfg.BotVerifyEnabled() {
if db := store.Global(); db != nil {
ver := threatintel.OperatorBotsCacheVersion(threatintel.LogicVersion, botEntries)
if dropped, err := db.EnsureBotVerifyLogicVersion(ver); err != nil {
csmlog.Warn("bot-verify cache version check failed", "err", err)
} else if dropped {
csmlog.Info("bot-verify cache dropped after logic or verified_bots change")
}
d.startBotVerifier(db, botEntries)
}
}
// Auto-clear stale content and web-exposure findings when their re-check
// logic has changed since the last start.
// Runs in a goroutine so it does not block startup; each allowlisted family
// keeps its own fail-closed dismissal invariant.
if db := store.Global(); db != nil && d.store != nil {
token := checks.FindingReverifyVersion()
d.startContentReverifySweepIfChanged(db, token, func() ([]checks.ContentReverifyDismissal, checks.ReverifySweepStats, bool) {
return checks.ReverifyStaleFindingsStats(d.scanContext(), d.store)
})
}
// Live AF_ALG listener (Copy Fail / CVE-2026-31431) — only started
// when the kernel is actually exploitable. Hosts with a KernelCare
// livepatch covering CVE-2026-31431, OR built without the AF_ALG
// aead interface entirely, skip the listener: there's nothing to
// detect, and the inotify watch + 500ms tick would just burn cycles.
// The hardening audit + periodic critical-tier check stay active
// either way, so re-introduction of the vulnerability (e.g., a
// kernel rollback) is still surfaced via the slower path.
kstate := checks.ObserveAFAlgKernelState()
switch {
case !kstate.IsCopyFailExploitable():
csmlog.Info("af_alg live listener: skipped",
"reason", "kernel not exploitable",
"state", kstate.String(),
)
default:
if mon := StartAFAlgLiveMonitor(d.alertCh, d.cfg); mon == nil {
csmlog.Warn("af_alg live listener: not started",
"reason", "no backend available",
"state", kstate.String(),
)
d.MarkWatcher("afalg", false)
} else {
csmlog.Info("af_alg live listener: started",
"backend", mon.Mode(),
"state", kstate.String(),
)
d.registerBackendQueues("bpf.af_alg", mon)
d.wg.Add(1)
obs.Go("af-alg-listener", func() {
defer d.wg.Done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() { <-d.stopCh; cancel() }()
mon.Run(ctx)
})
d.MarkWatcher("afalg", true)
}
}
d.startPHPRelay()
if mon := StartConnectionTracker(d.alertCh, d.cfg); mon != nil {
d.registerBackendQueues("bpf.connection", mon)
csmlog.Info("connection_tracker: started", "backend", mon.Mode())
d.wg.Add(1)
obs.Go("connection-tracker", func() {
defer d.wg.Done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() { <-d.stopCh; cancel() }()
mon.Run(ctx)
})
}
if mon := StartExecMonitor(d.alertCh, d.cfg); mon != nil {
d.registerBackendQueues("bpf.execution", mon)
csmlog.Info("exec_monitor: started", "backend", mon.Mode())
d.wg.Add(1)
obs.Go("exec-monitor", func() {
defer d.wg.Done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() { <-d.stopCh; cancel() }()
mon.Run(ctx)
})
}
if mon := StartSensitiveFileMonitor(d.alertCh, d.cfg, d.store); mon != nil {
d.registerBackendQueues("bpf.sensitive_files", mon)
csmlog.Info("sensitive_files: started", "backend", mon.Mode())
d.wg.Add(1)
obs.Go("sensitive-files", func() {
defer d.wg.Done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() { <-d.stopCh; cancel() }()
mon.Run(ctx)
})
}
// Start automatic signature updates
d.wg.Add(1)
obs.Go("signature-updater", d.signatureUpdater)
// Start signature watcher: queues a rescan when any rule file's
// content changes. Disabled wholesale via
// detection.rescan_on_signature_update: false.
d.wg.Add(1)
obs.Go("signature-watcher", d.signatureWatcher)
d.wg.Add(1)
obs.Go("geoip-updater", d.geoipUpdater)
d.wg.Add(1)
obs.Go("bot-ranges-updater", d.botRangesUpdater)
// Start heartbeat
d.wg.Add(1)
obs.Go("heartbeat", d.heartbeat)
// One-shot: stagger CSM-managed WP-Cron crontab lines left behind by
// older releases. The perf_wp_cron finding never re-fires once a site
// is fixed, so this is the only path that reaches them.
d.wg.Add(1)
obs.Go("wpcron-migrate", func() {
defer d.wg.Done()
if n := checks.MigrateWPCronCrontabs(d.cfg); n > 0 {
csmlog.Info("wp-cron: staggered legacy system cron entries", "upgraded", n)
}
})
// Start abuse reporting (opt-in). startAbuseReporting installs the alert
// hook and returns the spool drain loop, or nil when disabled. The reporter
// stops after the final shutdown alert flush so late findings can still be
// queued before the bbolt spool closes.
if reportLoop := d.startAbuseReporting(); reportLoop != nil {
obs.Go("abuse-reporter", reportLoop)
}
// Start the central scored-set consumer (opt-in). Maintains the verified
// set and escalates findings whose IP is listed; nil when disabled.
if centralLoop := d.startCentralConsume(); centralLoop != nil {
d.wg.Add(1)
obs.Go("central-intel", func() {
defer d.wg.Done()
centralLoop()
})
}
// Start the retention sweep only when opted in. Compaction is not
// run from this goroutine; see internal/daemon/retention.go for why.
if d.cfg != nil && d.cfg.Retention.Enabled {
d.wg.Add(1)
obs.Go("retention-scanner", d.retentionScanner)
}
// Refresh the sd_notify status line now that AF_ALG, BPF trackers,
// and the periodic scanners have all reported in. The initial
// READY=1 fired earlier so systemctl restart didn't have to wait
// on the baseline scan; this is the operator-visible summary.
_, _ = sdnotify.Status(fmt.Sprintf("watchers attached: %d", countAttachedWatchers(d.WatcherStatuses())))
csmlog.Info("CSM daemon running")
// Wait for signals
sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
for sig := range sigCh {
if sig == syscall.SIGHUP {
fmt.Fprintf(os.Stderr, "[%s] SIGHUP received - reloading config and rules\n", ts())
d.reloadConfig()
d.reloadSignatures()
if d.policies != nil {
if err := d.policies.Reload(d.currentCfg().EmailProtection.PHPRelay.PoliciesDir); err != nil {
// Previous valid version stays in effect; surface the failure
// via the existing alert pipeline so operators see partial reload.
d.emitReloadFinding(alert.Warning, "email_php_relay_policies_reload",
fmt.Sprintf("policies/email reload encountered errors: %v", err))
}
}
d.publishGeoIP()
if d.fwEngine != nil {
if err := d.fwEngine.Apply(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] Firewall reload error: %v\n", ts(), err)
} else {
fmt.Fprintf(os.Stderr, "[%s] Firewall rules reloaded\n", ts())
}
}
continue
}
break // SIGTERM or SIGINT
}
csmlog.Info("shutting down")
shutdownStart := time.Now()
close(d.stopCh)
// Abort any in-flight periodic scan so d.wg.Wait below is not held for a
// whole tier. Scanners observe d.stopCh between cycles; this cancels the
// scan currently executing inside RunTier.
if d.scanCancel != nil {
d.scanCancel()
}
// Log watchers own their files and close them when their Run loop exits on
// d.stopCh; d.wg.Wait below blocks until that happens. Calling Stop here
// would race the still-running Run on w.file.
if d.webServer != nil {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
_ = d.webServer.Shutdown(ctx)
cancel()
}
if d.challengeServer != nil {
d.challengeServer.Shutdown()
}
if d.challengeGate != nil {
if err := d.challengeGate.Close(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] challenge port-gate close: %v\n", ts(), err)
}
d.challengeGate = nil
}
if fm := d.getFileMonitor(); fm != nil {
fm.Stop()
}
if sw := d.getSpoolWatcher(); sw != nil {
sw.Stop()
}
if d.pamListener != nil {
d.pamListener.Stop()
}
if d.controlListener != nil {
d.controlListener.Stop()
}
if d.scanJobs != nil {
d.scanJobs.Stop()
}
// The web server and control socket have stopped taking releases.
if d.stalwartUnban != nil {
d.stalwartUnban.stop()
}
d.stopYaraBackend()
d.stopPHPTaintAnalyzer()
csmlog.Info("watchers signalled", "elapsed_ms", time.Since(shutdownStart).Milliseconds())
d.stopParseWorkers()
flushSenderProfiles()
stopProcessCtx()
// Stop the incident auto-close and retention goroutines before the
// admission owner's final drain and before closing the store, so neither
// writes to an already-closed bbolt database.
StopIncidentBackgroundLoops()
// Every producer has stopped: close the ingress generation cleanly.
d.stopAdmission()
csmlog.Info("workers drained", "elapsed_ms", time.Since(shutdownStart).Milliseconds())
// The log watchers have stopped, so this save holds every frozen message
// they reported and the next start does not report them again.
d.persistEximFrozenDedup()
if d.stalwart != nil {
d.drainStalwartVerdicts(d.currentCfg())
}
d.persistMailStats()
// Some producers can finish a tick after alertDispatcher observes stopCh.
// Drain again once tracked workers are gone and before state is closed.
d.flushPendingAlertsOnShutdown()
alert.SetSummaryHook(nil)
alert.ClosePhpanelQueues()
d.stopAbuseReporting()
d.closeFindingBus()
for i := len(d.phpRelayShutdown) - 1; i >= 0; i-- {
d.phpRelayShutdown[i]()
}
d.phpRelayShutdown = nil
if adb := attackdb.Global(); adb != nil {
adb.Stop()
}
if err := d.store.Close(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] error closing state store: %v\n", ts(), err)
}
d.lock.Release()
csmlog.Info("daemon stopped", "elapsed_ms", time.Since(shutdownStart).Milliseconds())
fmt.Fprintf(os.Stderr, "[%s] CSM daemon stopped\n", ts())
return nil
}
type contentLogicVersionStore interface {
ContentLogicVersionChanged(token string) (bool, error)
SetContentLogicVersion(token string) error
}
func (d *Daemon) startContentReverifySweepIfChanged(db contentLogicVersionStore, token string, run func() ([]checks.ContentReverifyDismissal, checks.ReverifySweepStats, bool)) {
changed, err := db.ContentLogicVersionChanged(token)
if err != nil {
csmlog.Warn("finding re-verification version check failed", "err", err)
return
}
if changed {
d.startContentReverifySweep(func() ([]checks.ContentReverifyDismissal, checks.ReverifySweepStats, bool) {
dismissed, stats, complete := run()
if !complete {
return dismissed, stats, false
}
if err := db.SetContentLogicVersion(token); err != nil {
csmlog.Warn("finding re-verification version update failed", "err", err)
return dismissed, stats, false
}
return dismissed, stats, true
})
}
}
func (d *Daemon) startContentReverifySweep(run func() ([]checks.ContentReverifyDismissal, checks.ReverifySweepStats, bool)) {
d.wg.Add(1)
obs.Go("content-reverify-sweep", func() {
defer d.wg.Done()
outcomes, stats, complete := run()
for _, dm := range outcomes {
csmlog.Info(contentReverifyOutcomeMessage(dm),
"check", dm.Check, "path", dm.Path, "detail", dm.Detail)
}
if !complete {
csmlog.Info("finding re-verification sweep will retry on next start")
return
}
// Always log the summary. A sweep that produced nothing used to log
// nothing at all, which made "ran and found nothing" indistinguishable
// from "never ran" and from "failed on every finding" -- and that
// ambiguity cost more than one wrong conclusion about why findings
// were not draining.
csmlog.Info("finding re-verification sweep complete",
"considered", stats.Considered, "cleared", stats.Cleared, "demoted", stats.Demoted,
"promoted", stats.Promoted, "unchecked", stats.Unchecked,
"unchecked_reason", stats.TopUncheckedReason)
})
}
func contentReverifyOutcomeMessage(outcome checks.ContentReverifyDismissal) string {
switch {
case outcome.Promoted:
return "finding severity restored after re-verification"
case outcome.Demoted:
return "remediated finding demoted"
default:
return "stale finding auto-cleared"
}
}
// DroppedAlerts returns the total number of alerts dropped due to
// channel backpressure since the daemon started.
func (d *Daemon) DroppedAlerts() int64 {
if d.alertQueue != nil {
return int64(d.alertQueue.Snapshot(time.Now()).DroppedTotal) // #nosec G115 -- a daemon cannot produce MaxInt64 findings within its lifetime.
}
return atomic.LoadInt64(&d.droppedAlerts)
}
func (d *Daemon) scanContext() context.Context {
if d.scanCtx != nil {
return checks.WithModSecReload(d.scanCtx, &d.modsecReload)
}
return checks.WithModSecReload(context.Background(), &d.modsecReload)
}
// FindingBus returns the per-daemon broadcast.Bus used by passive
// observers like the SSE event stream. Returns nil before Run starts.
func (d *Daemon) FindingBus() *broadcast.Bus {
return d.findingBus
}
// alertDispatcher batches and dispatches alerts.
// alertBatchInterval is how long the dispatcher collects findings before
// dispatching them as one batch. A variable so tests can shorten it.
var alertBatchInterval = 5 * time.Second
// alertHoldMaxBatch bounds the batch the dispatcher accumulates while held.
// Beyond it, further findings are dropped and counted rather than growing
// memory without limit on a host whose baseline runs for minutes.
const alertHoldMaxBatch = 5000
// holdAlertDispatch arms the startup hold. Must run before the dispatcher
// goroutine starts.
func (d *Daemon) holdAlertDispatch() {
d.alertHold = make(chan struct{})
if d.alertQueue != nil {
d.alertQueue.Hold(time.Now())
}
}
// releaseAlertDispatch lets the dispatcher start dispatching its batches.
// Idempotent; a no-op when no hold was armed.
func (d *Daemon) releaseAlertDispatch() {
if d.alertHold == nil {
return
}
d.alertReleaseOnce.Do(func() {
if d.alertQueue != nil {
d.alertQueue.Release(time.Now())
}
close(d.alertHold)
})
}
func (d *Daemon) alertDispatcher() {
defer d.wg.Done()
ticker := time.NewTicker(alertBatchInterval)
defer ticker.Stop()
var batch []alert.Finding
held := d.alertHold // nil: never selected, so the dispatcher runs freely
heldDropped := 0
for {
select {
case <-d.stopCh:
batch = d.drainAlertChannel(batch)
d.persistPendingFindingsOnShutdown(batch)
// Save now: Close waits for the worker drains, and a stop
// killed before it would lose the pending state window.
if d.store != nil {
if err := d.store.Flush(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] state: error saving: %v\n", ts(), err)
}
}
return
case f := <-d.alertCh:
alert.StartQueued(f)
if held != nil && len(batch) >= alertHoldMaxBatch {
alert.RejectQueued(f)
heldDropped++
atomic.AddInt64(&d.droppedAlerts, 1)
continue
}
batch = append(batch, f)
case <-held:
held = nil
if heldDropped > 0 {
fmt.Fprintf(os.Stderr, "[%s] Alert dispatcher: %d realtime findings dropped while dispatch was held for the baseline scan\n", ts(), heldDropped)
}
case <-ticker.C:
if held != nil {
continue
}
if len(batch) > 0 {
d.dispatchBatch(batch)
batch = nil
}
if err := d.store.FlushDue(time.Now()); err != nil {
fmt.Fprintf(os.Stderr, "[%s] state: error saving: %v\n", ts(), err)
}
}
}
}
func (d *Daemon) drainAlertChannel(batch []alert.Finding) []alert.Finding {
for {
select {
case f, ok := <-d.alertCh:
if !ok {
return batch
}
alert.StartQueued(f)
batch = append(batch, f)
default:
return batch
}
}
}
func (d *Daemon) flushPendingAlertsOnShutdown() {
d.persistPendingFindingsOnShutdown(d.drainAlertChannel(nil))
// Log readers and dispatch can note successes after the periodic saver
// observes stopCh. This final save runs once those producers have stopped.
d.persistKnownCustomers()
if d.blockDigest != nil {
d.blockDigest.Flush()
}
}
// persistPendingFindingsOnShutdown records findings still queued at shutdown to
// the history log for forensics and parks them for the next start. It
// deliberately does NOT run the auto-response pipeline (nftables blocks,
// permission fixes, kill/quarantine, DB cleanup) or network alert dispatch
// here: that work blocked the service stop for tens of seconds -- up to twice,
// once here and once in the dispatcher's stop branch -- while systemd waited.
// The baseline scan re-detects scheduled checks after a restart, but a
// realtime-only finding (a webshell written in the last seconds) is seen by
// nothing else, so the parked batch is replayed through dispatchBatch once
// the next start releases the dispatcher. Nothing here is marked sent via
// store.Update, so the replay's dispatch is not suppressed.
func (d *Daemon) persistPendingFindingsOnShutdown(batch []alert.Finding) {
defer alert.FinishQueued(batch)
if len(batch) == 0 {
return
}
alert.FillTimestamps(batch, time.Now())
d.store.AppendHistory(batch)
if err := d.store.AppendPendingFindings(batch); err != nil {
fmt.Fprintf(os.Stderr, "[%s] Could not park %d pending finding(s) for the next start: %v\n", ts(), len(batch), err)
}
}
func isOperatorAlertableCheck(check string) bool {
switch config.CanonicalCheckName(check) {
case "modsec_block_realtime", "modsec_warning_realtime", "modsec_block_escalation", "modsec_csm_block_escalation":
return false // Fully automated and visible on the ModSecurity page.
case "outdated_plugins":
return false // Routine update posture remains on the findings page.
case "email_dkim_failure", "email_spf_rejection":
return false // Operational email authentication issues are informational.
case "email_auth_failure_realtime", "pam_bruteforce", "exim_frozen_realtime":
return false // Failed logins and frozen bounces need no operator action.
case "ftp_login", "cpanel_file_upload_realtime":
// On shared hosting every customer connects from a non-infra address,
// so a successful FTP login or a File Manager write is ordinary use of
// a core feature. Their value is correlation with other evidence on
// the same account, which the findings page, history, incidents and
// the attack database all still get. The brute-force escalations
// (ftp_auth_failure_realtime, ftp_bruteforce,
// ftp_login_after_bruteforce) stay alertable.
return false
default:
return true
}
}
func operatorAlertableFindings(findings []alert.Finding) []alert.Finding {
alertable := make([]alert.Finding, 0, len(findings))
for _, f := range findings {
if isOperatorAlertableCheck(f.Check) {
alertable = append(alertable, f)
}
}
return alertable
}
func (d *Daemon) dispatchBatch(findings []alert.Finding) {
defer alert.FinishQueued(findings)
// Realtime producers may hand over findings without a Timestamp; stamp
// them once here so history, incidents, the latest set and every alert
// sink see the same time.
alert.FillTimestamps(findings, time.Now())
auditSources := append([]alert.Finding(nil), findings...)
// Snapshot the live config once at the top of the batch. Every
// cfg.X read below picks up the last-reloaded value (ROADMAP
// item 7); taking one snapshot avoids the weirder case of a
// SIGHUP landing mid-batch and splitting some auto-response
// actions between old and new policy.
cfg := d.currentCfg()
// A file removed since the last batch may already be back in this one.
d.forgetRemovedFileAlerts(nil)
// Reconcile every observation before deduplication can discard new content
// behind an unchanged, hashless or carried finding with the same key.
findings = d.store.ReconcileDismissedContent(findings)
preserveEscalationResponseProvenance(findings)
findings = alert.Deduplicate(findings)
d.noteFTPCustomers(findings)
suppressions := d.store.LoadSuppressions()
remediableFindings := filterUnsuppressedFindings(d.store, findings, suppressions)
// Suppressed checks do not promise a response, but an already evaluated
// response must survive suppression and deduplication of its finding.
escalationSources := append([]alert.Finding(nil), filterUnsuppressedFindings(d.store, auditSources, suppressions)...)
for _, f := range auditSources {
if f.AutoFileResponseEvaluated {
escalationSources = append(escalationSources, f)
}
}
escalationCandidates := checks.RealtimePHPEscalationCandidates(cfg, escalationSources)
d.queuePHPEscalations(filterUnsuppressedFindings(d.store, d.store.FilterNew(escalationCandidates), suppressions))
// Record ALL findings in attack database (before filtering -
// repeated attacks from the same IP must still be counted even if
// the alert is suppressed by FilterNew or a suppression rule).
if adb := attackdb.Global(); adb != nil {
for _, f := range findings {
adb.RecordFinding(f)
}
}
// Auto-block and permission fix run on ALL findings (not just new ones).
// These must execute BEFORE FilterNew because repeat offender IPs and
// recurring permission issues need to be fixed even if the alert was
// already sent in a previous cycle.
// Challenge routing runs FIRST - claims eligible IPs before hard-blocking.
// One ordered helper guarantees that ordering on every auto-response path.
// Suppression rules do not gate IP responses: they mute a check, and a
// check-wide rule would otherwise leave every attacker it reports
// unblocked. An IP false positive belongs on the allowlist.
challengeActions, blockActions := checks.ChallengeThenBlock(cfg, findings)
permActions, permFixedKeys := checks.AutoFixPermissions(cfg, remediableFindings)
// Mark auto-blocked IPs in attack database
markAppliedBlocks(blockActions)
// Dismiss auto-fixed findings from the Findings page
for _, key := range permFixedKeys {
d.store.DismissLatestFinding(key)
}
// Filter through state - only new findings get alerted and logged
unfilteredNew := d.store.FilterNew(findings)
// responseFindings is every new observation and action, suppressed or not:
// incidents and central enforcement act on it. newFindings drops what
// suppression rules match, which mutes notifications and remediation.
// Suppressions are stored in state/suppressions.json, not in rule files.
responseFindings := append([]alert.Finding(nil), unfilteredNew...)
responseFindings = append(responseFindings, blockActions...)
responseFindings = append(responseFindings, challengeActions...)
responseFindings = append(responseFindings, permActions...)
// PHP-relay AutoFreeze: emit any new findings produced by post-emit
// freeze decisions back into the dispatched batch so operators see
// the action outcome alongside the original finding. Nil-guard for
// non-cPanel / non-linux hosts where wiring is skipped.
if d.autoFreezer != nil {
if freezeFindings := d.autoFreezer.Apply(remediableFindings); len(freezeFindings) > 0 {
responseFindings = append(responseFindings, freezeFindings...)
}
}
// A Critical login after brute force suspends the cracked mailbox; the
// outcome finding tells staff what was done and what is left to them.
responseFindings = append(responseFindings, stolenLoginResponses(cfg, remediableFindings)...)
if len(responseFindings) == 0 {
// A scan applies its own blocks and hands them over as findings,
// which the repeat filter may have judged seen.
d.observeBlocks(findings)
if err := alert.DispatchWithNotificationFilter(cfg, nil, auditSources, nil, d.deliveryCandidates(remediableFindings, nil, suppressions), operatorAlertableFindings, d.store); err != nil {
fmt.Fprintf(os.Stderr, "[%s] Alert dispatch error: %v\n", ts(), err)
}
d.store.Update(findings)
d.forgetRemovedFileAlerts(findings)
return
}
// Copy: with no rules the filter returns its input, and both slices grow.
newFindings := append([]alert.Finding(nil), filterUnsuppressedFindings(d.store, responseFindings, suppressions)...)
d.store.AppendHistory(newFindings)
// Periodic scans run their own auto-response, so their blocks arrive in
// findings; nothing else hands this dispatcher a Critical auto_block.
d.observeBlocks(slices.Concat(findings, blockActions))
// Kill and quarantine only run on new, unsuppressed findings. The
// companion sweep is queued first so a file quarantined below is
// anchored from its live stat.
killActions := checks.AutoKillProcesses(d.scanContext(), cfg, newFindings)
queueCompanionSweeps(newFindings)
quarantineActions := checks.AutoQuarantineFiles(cfg, newFindings)
// Database response also reports WordPress session addresses without
// blocking them. Suppressions stop SQL writes and session revocation,
// but not that report.
dbActions := autoRespondDBMalware(cfg, unfilteredNew, func(f alert.Finding) bool {
return !d.store.IsSuppressed(f, suppressions)
})
for _, actions := range [][]alert.Finding{killActions, quarantineActions, dbActions} {
responseFindings = append(responseFindings, actions...)
newFindings = append(newFindings, filterUnsuppressedFindings(d.store, actions, suppressions)...)
}
// Correlation derives notifications, so it reads only unsuppressed
// findings; its derived findings still reach incidents and enforcement.
uncorrelated := len(newFindings)
newFindings = expandWithCorrelation(newFindings, time.Now())
responseFindings = append(responseFindings, newFindings[uncorrelated:]...)
co := IncidentCorrelator()
for _, f := range alert.Deduplicate(responseFindings) {
_, _, _ = co.OnFinding(f)
}
// Derived findings may themselves be suppressed. Keep them in the
// response set while applying their rules before notification fanout.
newFindings = filterUnsuppressedFindings(d.store, newFindings, suppressions)
// Apply notification policy after the phpanel stream and passive
// observers receive the findings, so muting email does not lose evidence.
auditSources = append(auditSources, responseFindings...)
candidates := d.deliveryCandidates(remediableFindings, newFindings, suppressions)
if err := alert.DispatchWithNotificationFilter(cfg, newFindings, auditSources, responseFindings, candidates, operatorAlertableFindings, d.store); err != nil {
fmt.Fprintf(os.Stderr, "[%s] Alert dispatch error: %v\n", ts(), err)
}
d.store.Update(findings)
d.store.MarkAlerted(newFindings)
d.forgetRemovedFileAlerts(findings)
}
// respondToInitialScan records the baseline scan, runs its auto-response and
// dispatches the resulting alerts. It returns the alerted findings and the keys
// of findings the permission auto-fix repaired.
func (d *Daemon) respondToInitialScan(cfg *config.Config, initialFindings []alert.Finding) ([]alert.Finding, []string) {
// As in dispatchBatch: a record with no parseable time takes its receipt
// time, or the customer set would drop its login as expired.
alert.FillTimestamps(initialFindings, time.Now())
// History and audit retain every observation, including content displaced
// by a newer detection before notification and response.
initialAuditSources := append([]alert.Finding(nil), initialFindings...)
// Before FilterNew and delivery: a dismissed file with new content alerts in this scan.
initialFindings = d.store.ReconcileDismissedContent(initialFindings)
// The realtime watcher starts at the end of the log, so FTP logins made
// while the daemon was down reach the set only through this scan.
d.noteFTPCustomers(initialFindings)
d.store.AppendHistory(initialAuditSources)
d.forgetRemovedFileAlerts(nil)
unfilteredNew := d.store.FilterNew(initialFindings)
suppressions := d.store.LoadSuppressions()
// Copy: with no rules the filter returns its input, and newFindings grows.
newFindings := append([]alert.Finding(nil), filterUnsuppressedFindings(d.store, unfilteredNew, suppressions)...)
// Permission auto-fix runs on ALL findings (not just new) because
// it's safe/idempotent and should fix baseline findings too.
permActions, permFixedKeys := checks.AutoFixPermissions(cfg, filterUnsuppressedFindings(d.store, initialFindings, suppressions))
// Challenge routing runs on ALL findings unconditionally when enabled, so an
// eligible IP is on the challenge list before AutoBlockIPs (below, guarded by
// new findings) checks membership. Not folded into ChallengeThenBlock here:
// challenge must route even with no new findings (re-establishing challenges
// on restart) while the block stage stays gated on new findings. As in
// dispatchBatch, suppression rules do not gate IP responses.
challengeActions := checks.ChallengeRouteIPs(cfg, initialFindings)
// Other auto-response only on new findings. As in dispatchBatch,
// responseFindings keeps suppressed observations for incidents and
// central enforcement while newFindings carries what may notify.
var responseFindings, blockActions []alert.Finding
if len(unfilteredNew) > 0 {
killActions := checks.AutoKillProcesses(d.scanContext(), cfg, newFindings)
queueCompanionSweeps(newFindings)
quarantineActions := checks.AutoQuarantineFiles(cfg, newFindings)
blockActions = checks.AutoBlockIPs(cfg, initialFindings)
responseFindings = append(responseFindings, unfilteredNew...)
for _, actions := range [][]alert.Finding{killActions, quarantineActions, permActions, challengeActions, blockActions} {
responseFindings = append(responseFindings, actions...)
newFindings = append(newFindings, filterUnsuppressedFindings(d.store, actions, suppressions)...)
}
// Cross-account correlation runs on the initial batch too, not
// just on subsequent ticks. Otherwise three account compromises
// landing in the first scan slip past with no synthetic alert.
uncorrelated := len(newFindings)
newFindings = expandWithCorrelation(newFindings, time.Now())
responseFindings = append(responseFindings, newFindings[uncorrelated:]...)
co := IncidentCorrelator()
for _, f := range alert.Deduplicate(responseFindings) {
_, _, _ = co.OnFinding(f)
}
}
// The baseline scan applied its own blocks; they are in initialFindings.
d.observeBlocks(slices.Concat(initialFindings, blockActions))
newFindings = filterUnsuppressedFindings(d.store, newFindings, suppressions)
initialAuditSources = append(initialAuditSources, responseFindings...)
candidates := d.deliveryCandidates(filterUnsuppressedFindings(d.store, initialFindings, suppressions), newFindings, suppressions)
_ = alert.DispatchWithNotificationFilter(cfg, newFindings, initialAuditSources, responseFindings, candidates, operatorAlertableFindings, d.store)
d.store.Update(withoutFixedFindings(initialFindings, permFixedKeys))
d.store.MarkAlerted(newFindings)
d.forgetRemovedFileAlerts(initialFindings)
return newFindings, permFixedKeys
}
// Delivery follows current observations independently of the history reminder.
// Rebuild derived candidates too, so a missed delivery can retry and a live
// correlation does not expire while its source findings are still reported.
func (d *Daemon) deliveryCandidates(observed, newFindings []alert.Finding, suppressions []state.SuppressionRule) []alert.Finding {
candidates := make([]alert.Finding, 0, len(observed)+len(newFindings))
for _, batch := range [][]alert.Finding{observed, newFindings} {
for _, f := range batch {
// Scan summaries precede source policy; history summaries may
// cover only a subset. Rebuild both from eligible observations.
if !checks.IsDerivedCorrelationCheck(f.Check) {
candidates = append(candidates, f)
}
}
}
candidates = alert.Deduplicate(candidates)
var correlationSources []alert.Finding
for _, f := range candidates {
if !d.store.Acknowledged(f) {
correlationSources = append(correlationSources, f)
}
}
derived := checks.CorrelateBatchFindings(correlationSources).Derived
candidates = append(candidates, derived...)
candidates = filterUnsuppressedFindings(d.store, candidates, suppressions)
if candidates == nil {
// The dispatcher reads nil candidates as "reuse the findings", and
// suppression rules can empty the set.
candidates = []alert.Finding{}
}
return candidates
}
// withoutFixedFindings drops findings whose Key is in fixedKeys.
func withoutFixedFindings(findings []alert.Finding, fixedKeys []string) []alert.Finding {
fixed := make(map[string]bool, len(fixedKeys))
for _, k := range fixedKeys {
fixed[k] = true
}
var kept []alert.Finding
for _, f := range findings {
if !fixed[f.Key()] {
kept = append(kept, f)
}
}
return kept
}
// autoFixWPCron lets daemon wiring tests avoid real wp-config.php and crontab
// edits; the checks package covers those side effects directly.
var autoFixWPCron = checks.AutoFixWPCron
// Database wiring tests exercise suppression policy without a live MySQL host.
var autoRespondDBMalware = checks.AutoRespondDBMalwareWithPolicy
// removedFilesSince and removedFileAt let wiring tests stand in for the file
// response ledger.
var (
removedFilesSince = checks.RemovedFilesSince
removedFileAt = checks.RemovedFileAt
)
// forgetRemovedFileAlerts clears the alert state about files CSM removed or
// cleaned, so the file coming back is reported as new. A scan responds before
// alert delivery sees its findings, and another batch may have taken the
// removal by then: observed carries findings whose state Update has just
// written, and those seen no later than their file's removal are cleared too.
func (d *Daemon) forgetRemovedFileAlerts(observed []alert.Finding) {
removed, next := removedFilesSince(d.removedFileSeq.Load())
d.removedFileSeq.Store(next)
for _, r := range removed {
d.store.ForgetFileAlerts(r.Path)
}
var stale []alert.Finding
for _, f := range observed {
if f.FilePath == "" {
continue
}
if r, ok := removedFileAt(f.FilePath); ok && !f.Timestamp.After(r.At) {
stale = append(stale, f)
}
}
d.store.ForgetCleared(stale)
}
// queueCompanionSweeps lets wiring tests see what the dispatcher hands to the
// companion sweep without touching the filesystem.
var queueCompanionSweeps = checks.QueueCompanionSweeps
// companionSweeper reports files planted alongside confirmed malware. Its
// findings take the same path to the dispatcher as a scan's.
func (d *Daemon) companionSweeper() {
defer d.wg.Done()
checks.RunCompanionSweeps(d.scanContext(), d.currentCfg, func(findings []alert.Finding) {
d.enqueueScanAlerts(findings, "companion sweep")
})
}
// processScanFindings handles the output of a deep or periodic scan: it persists
// the findings to the latest-findings surface, runs the auto-responses that act
// on warning-severity perf findings, then forwards the remaining findings to the
// alert dispatcher. Warning-severity perf findings stay off the alert channel so
// they never page an operator; that is exactly why the WP-Cron auto-fix runs
// here and not in dispatchBatch, which only ever sees what the channel carries.
func (d *Daemon) processScanFindings(cfg *config.Config, findings []alert.Finding, purgeChecks []string, label string) {
d.processScanFindingsWithCoverage(cfg, findings, purgeChecks, nil, label)
}
func (d *Daemon) processScanFindingsWithCoverage(cfg *config.Config, findings []alert.Finding, purgeChecks []string, coverage *state.ScanCoverage, label string) {
checks.StoreLatestScanFindingsWithCoverage(d.store, purgeChecks, findings, coverage)
d.applyWPCronAutoFix(cfg, findings)
d.enqueueScanAlerts(findings, label)
}
// scanAlertEnqueueTimeout bounds how long a scan enqueue can make no progress
// while waiting for room on the shared alert channel. The undelivered tail is
// dropped as an absolute last resort when that timeout expires. The
// dispatcher stops draining only while it runs a batch, and auto-response can
// hold that for tens of seconds; a scan burst that fills the buffer in that
// window must apply backpressure rather than silently drop security findings
// the way an earlier non-blocking send did -- a false-positive flood once
// dropped a real rogue-admin finding. Set well above the dispatcher's
// worst-case batch time so a drop here means the dispatcher is genuinely wedged.
const scanAlertEnqueueTimeout = 90 * time.Second
// enqueueScanAlerts forwards a scan's findings to the alert dispatcher.
// Warning-severity perf findings stay off the channel so they never page an
// operator; that is also why the WP-Cron auto-fix runs against the scan
// findings directly rather than in dispatchBatch, which only sees the channel.
//
// Unlike the real-time kernel monitors, which must never block a BPF or
// fanotify handler, the scan path is not latency-sensitive, so it applies
// backpressure when the buffer is full instead of dropping. A finding burst can
// otherwise fill the buffer while the dispatcher is mid-batch and evict
// findings from this or any other producer sharing the channel.
func (d *Daemon) enqueueScanAlerts(findings []alert.Finding, label string) {
d.enqueueScanAlertsWithin(findings, label, scanAlertEnqueueTimeout)
}
func (d *Daemon) enqueueScanAlertsWithin(findings []alert.Finding, label string, timeout time.Duration) {
for i, f := range findings {
if !scanFindingIsAlertable(f) {
continue
}
err := alert.EnqueueWithin(d.alertCh, f, d.stopCh, timeout)
if err == nil {
continue
}
remaining := countAlertableScanFindings(findings[i+1:])
// EnqueueWithin already counted the rejected send, including shutdown.
alert.RecordQueueLoss(d.alertCh, uint64(remaining)) // #nosec G115 -- countAlertableScanFindings returns a nonnegative count bounded by the slice length.
dropped := remaining + 1
atomic.AddInt64(&d.droppedAlerts, int64(dropped))
if errors.Is(err, alert.ErrQueueTimeout) {
fmt.Fprintf(os.Stderr, "[%s] alert channel jammed for %s, dropping %d remaining %s findings (first: %s)\n", ts(), timeout, dropped, label, f.Check)
}
return
}
}
func countAlertableScanFindings(findings []alert.Finding) int {
count := 0
for _, f := range findings {
if scanFindingIsAlertable(f) {
count++
}
}
return count
}
func scanFindingIsAlertable(f alert.Finding) bool {
return !strings.HasPrefix(f.Check, "perf_") || f.Severity != alert.Warning
}
// applyWPCronAutoFix disables WP-Cron and installs a per-user system cron for
// every perf_wp_cron finding, then clears the fixed findings from the
// latest-findings surface and records the actions in history. Findings the
// operator has suppressed are left untouched, so a suppression also stops the
// automated edit of that account's wp-config.php.
func (d *Daemon) applyWPCronAutoFix(cfg *config.Config, findings []alert.Finding) {
if d.store != nil {
if suppressions := d.store.LoadSuppressions(); len(suppressions) > 0 {
findings = filterUnsuppressedFindings(d.store, findings, suppressions)
}
}
actions, fixedKeys := autoFixWPCron(cfg, findings)
for _, key := range fixedKeys {
d.store.DismissLatestFinding(key)
}
if len(actions) > 0 {
d.store.AppendHistory(actions)
}
}
func filterUnsuppressedFindings(store *state.Store, findings []alert.Finding, suppressions []state.SuppressionRule) []alert.Finding {
if len(suppressions) == 0 {
return findings
}
var filtered []alert.Finding
for _, f := range findings {
if !store.IsSuppressed(f, suppressions) {
filtered = append(filtered, f)
}
}
return filtered
}
// criticalScanner runs critical checks every 10 minutes.
func (d *Daemon) criticalScanner() {
defer d.wg.Done()
ticker := time.NewTicker(10 * time.Minute)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
d.runPeriodicChecks(checks.TierCritical)
}
}
}
// deepScanner runs the deep tier at the configured interval (default 60 min).
func (d *Daemon) deepScanner() {
defer d.wg.Done()
// Re-read the interval on each iteration so a SIGHUP that changes
// thresholds.deep_scan_interval_min takes effect on the next scan.
// A ticker captured at startup can't be re-sized cleanly without
// a reset path; time.After recomputes.
for {
interval := time.Duration(d.currentCfg().Thresholds.DeepScanIntervalMin) * time.Minute
if interval <= 0 {
// Defensive: an operator who zeroes the threshold would
// otherwise get a tight spin loop. 60 minutes matches the
// default from config.Load.
interval = 60 * time.Minute
}
select {
case <-d.stopCh:
return
case <-time.After(interval):
// Re-verify findings whose condition someone else resolved. The
// startup sweep is gated on the re-check logic version, which only
// moves on deploy: an operator cleaning a file, or a virtual patch
// closing an exposure, changes the world without changing CSM, and
// a finding gated only on that would keep its severity until the
// next upgrade happened to land.
if d.store != nil {
d.startContentReverifySweep(func() ([]checks.ContentReverifyDismissal, checks.ReverifySweepStats, bool) {
return checks.ReverifyStaleFindingsStats(d.scanContext(), d.store)
})
}
// Update threat intelligence feeds (once per day)
if db := checks.GetThreatDB(); db != nil {
_ = db.UpdateFeeds()
}
// Prune attack DB records past 90 days or kept only for faded spam evidence
if adb := attackdb.Global(); adb != nil {
adb.PruneExpired()
// A flood that stopped left drops no later verdict will report.
now := time.Now()
d.warnSpamEvidenceEvicted(adb.PendingEviction(now), now)
fmt.Fprintf(os.Stderr, "[%s] Attack DB after deep-scan prune (%s)\n", ts(), adb.FormatTopLine())
}
// The full deep tier runs whether or not the file monitor is
// attached; the monitor misses renames, chmod and probe-only
// findings. A rescan queued by a rules update needs nothing extra:
// the rolling YARA walk in this tier tracks and clears it.
cfg := d.currentCfg()
scanCtx, gaps := checks.WithCoverageGaps(d.scanContext())
findings, purgeChecks := checks.RunTierWithContext(scanCtx, cfg, d.store, checks.TierDeep)
d.processScanFindingsWithCoverage(cfg, findings, purgeChecks, gaps.Snapshot(), "deep")
}
}
}
func (d *Daemon) runPeriodicChecks(tier checks.Tier) {
// A web server can start or stop rewriting client addresses without
// telling CSM; re-read it every critical tick.
if tier == checks.TierCritical {
d.refreshHTTPRewrite()
}
// Snapshot the live config ONCE for the whole tick. Calling
// d.currentCfg() twice (once for integrity, once for RunTier)
// lets a SIGHUP land between the two reads and split the tick
// between old-policy integrity verification and new-policy
// detection. Matches the snapshot pattern in dispatchBatch.
cfg := d.currentCfg()
// Verify integrity against the snapshot. A SIGHUP reload re-signs
// integrity.config_hash on disk and updates config.Active; using
// d.cfg (the startup snapshot) here would fire a Critical tamper
// alert on every tick after a successful reload because the stored
// hash in d.cfg is stale. If a reload completes while Verify is
// hashing, retry once against the latest live config to avoid a
// false tamper alert from a stale snapshot.
var err error
cfg, err = d.verifyPeriodicIntegritySnapshot(cfg)
if err != nil {
if !alert.TryEnqueue(d.alertCh, alert.Finding{
Severity: alert.Critical,
Check: "integrity",
Message: fmt.Sprintf("BINARY/CONFIG TAMPER DETECTED: %v", err),
Timestamp: time.Now(),
}) {
atomic.AddInt64(&d.droppedAlerts, 1)
fmt.Fprintf(os.Stderr, "[%s] alert channel full, dropping integrity finding\n", ts())
}
return
}
// Age out stale dry-run-block records so the status surface
// reflects recent activity instead of months-old entries left
// over from a previous dry-run window. Keeping a 7-day rolling
// window matches the operator workflow of reviewing a week of
// would-have-been-blocks before flipping to live.
pruneLegacyDryRunBlocks()
scanCtx, gaps := checks.WithCoverageGaps(d.scanContext())
findings, purgeChecks := checks.RunTierWithContext(scanCtx, cfg, d.store, tier)
d.processScanFindingsWithCoverage(cfg, findings, purgeChecks, gaps.Snapshot(), "periodic")
}
// verifyStartupIntegrity verifies the binary, main config and conf.d, then
// records the per-fragment digests of the verified conf.d. Hosts signed by a
// release without that record get one here, so `csm rehash --fragment` works
// after an upgrade without a full rehash. A record that cannot be written
// only costs the scoped rehash, never the start.
func verifyStartupIntegrity(binaryPath string, cfg *config.Config) error {
if err := integrity.Verify(binaryPath, cfg); err != nil {
return err
}
if err := integrity.RecordVerifiedConfD(cfg); err != nil {
csmlog.Warn("conf.d fragment record not written; csm rehash --fragment needs a full csm rehash first", "err", err)
}
return nil
}
func (d *Daemon) verifyPeriodicIntegritySnapshot(cfg *config.Config) (*config.Config, error) {
if err := integrity.Verify(d.binaryPath, cfg); err != nil {
latest := d.currentCfg()
if latest != nil && latest != cfg {
retryErr := integrity.Verify(d.binaryPath, latest)
if retryErr == nil {
return latest, nil
}
return latest, retryErr
}
return cfg, err
}
return cfg, nil
}
// heartbeat sends periodic pings to dead man's switch.
func (d *Daemon) heartbeat() {
defer d.wg.Done()
ticker := time.NewTicker(10 * time.Minute)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
alert.SendHeartbeat(d.currentCfg())
d.hijackDetector.Cleanup()
// Failed startup can leave only the recovery boundary available.
// Settle pending outcomes before attempting cleanup mutations.
recoverFirewallActions(d.fwActions)
// Clean expired temporary allows
if d.fwEngine != nil {
d.fwEngine.CleanExpiredAllows()
d.fwEngine.CleanExpiredSubnets()
}
// Clean expired temporary whitelist and threat entries
if tdb := checks.GetThreatDB(); tdb != nil {
tdb.PruneExpiredWhitelist()
tdb.PruneExpiredThreats()
}
}
}
}
// startPHPRelay warns on non-cPanel hosts. On cPanel it starts the relay
// only when enabled, locating Exim for automatic actions and wiring the
// spool watcher, pipeline and maintenance tickers on Linux.
func (d *Daemon) startPHPRelay() {
info := platform.Detect()
if !info.IsCPanel() {
alert.TryEnqueue(d.alertCh, alert.Finding{
Severity: alert.Warning,
Check: "email_php_relay_disabled",
Message: "php_relay disabled: not a cPanel host",
Timestamp: time.Now(),
})
return
}
if !d.cfg.EmailProtection.PHPRelay.Enabled {
return
}
if path, err := exec.LookPath("exim"); err == nil {
eximBinary = path
} else {
alert.TryEnqueue(d.alertCh, alert.Finding{
Severity: alert.Warning,
Check: "email_php_relay_no_exim",
Message: "php_relay auto-action disabled: exim binary not in PATH",
Timestamp: time.Now(),
})
}
// Bridge to the linux-only wiring (Phase O2). On non-linux GOOS
// the stub in php_relay_wiring_other.go is a no-op.
startPHPRelayLinux(d)
}
func (d *Daemon) startLogWatchers() {
d.loadMailGoodSource()
d.loadMailSpray()
d.restoreEximFrozenDedup()
d.restoreMailStats()
d.startMailStatsPersistence()
hostInfo := platform.Detect()
watchExim := shouldWatchEximMainlog(hostInfo, os.Stat)
d.smtpAuthFromExim = watchExim
d.mailboxes = newMailboxResolver(hostInfo)
logFiles := d.fixedLogSpecs(hostInfo, d.handleEximLine, watchExim)
if hostInfo.IsCPanel() {
d.wg.Add(1)
obs.Go("stale-session-401-flush", d.flushStaleSession401)
}
if watchExim {
d.startEximFrozenDedupPersistence()
}
if useStalwartReader(platform.DetectMTA()) {
d.startStalwartLogReader(d.cfg)
d.startStalwartAddressList()
} else {
d.startMailLogReader(hostInfo.MailLogPath(), d.handleMailLine)
}
// Only receive PHP Shield events if enabled AND actually installed. A stale
// php_shield.enabled flag (e.g. after an upgrade wiped /opt/csm) would
// otherwise spin the missing-socket retry forever; warn once with
// a remediation hint instead.
if watch, warnNotInstalled := phpShieldWatchDecision(d.cfg.PHPShield.Enabled, phpShieldInstalled()); watch {
d.wg.Add(1)
obs.Go("php-shield-events", d.watchPHPShieldEvents)
} else if warnNotInstalled {
csmlog.Warn(phpShieldMissingScriptWarning, "shield", phpShieldScriptPath)
d.MarkWatcher("php_shield", false)
}
// ModSecurity error log - auto-discover path based on detected web server.
if modsecPath := discoverModSecLogPath(d.cfg); modsecPath != "" {
logFiles = append(logFiles, logWatchSpec{name: "modsec", path: modsecPath, handler: parseModSecLogLineDeduped, producer: checks.ProducerModSecLog})
} else if hostInfo.WebServer != platform.WSNone {
// Only bother with the retry loop if a web server is actually
// present. Headless hosts don't need this.
fmt.Fprintf(os.Stderr, "[%s] ModSecurity error log not found (checked %v), will retry every 60s\n", ts(), hostInfo.ErrorLogPaths)
d.MarkWatcher("modsec", false)
d.wg.Add(1)
obs.Go("logwatch-modsec-retry", func() {
defer d.wg.Done()
ticker := time.NewTicker(logWatcherRetryInterval)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
path := discoverModSecLogPath(d.cfg)
if path == "" {
continue
}
w, err := newObservedLogWatcher(logWatchSpec{path: path, handler: parseModSecLogLineDeduped, producer: checks.ProducerModSecLog}, d.cfg, d.alertCh)
if err != nil {
continue
}
d.logWatchersMu.Lock()
d.logWatchers = append(d.logWatchers, w)
d.logWatchersMu.Unlock()
d.wg.Add(1)
obs.Go("logwatch-modsec", func() {
defer d.wg.Done()
w.Run(d.stopCh)
})
csmlog.Info("watching log (appeared after retry)", "path", path)
d.MarkWatcher("modsec", true)
return
}
}
})
}
// Real-time access log watcher for wp-login/xmlrpc brute force detection.
// Auto-discover path from platform info (Apache/Nginx/cPanel aware).
if accessLogPath := discoverAccessLogPath(); accessLogPath != "" {
logFiles = append(logFiles, logWatchSpec{path: accessLogPath, handler: parseAccessLogBruteForce, handlerForPath: accessLogHandlerForPath, producer: checks.ProducerAccessLog})
} else if hostInfo.WebServer != platform.WSNone && len(hostInfo.AccessLogPaths) > 0 {
csmlog.Warn("access log not found, will retry every 60s", "candidates", fmt.Sprintf("%v", hostInfo.AccessLogPaths))
d.wg.Add(1)
obs.Go("logwatch-access-retry", func() {
d.retryLogWatcherCandidates(hostInfo.AccessLogPaths, logWatchSpec{handler: parseAccessLogBruteForce, handlerForPath: accessLogHandlerForPath, producer: checks.ProducerAccessLog})
})
}
// Start background eviction for modsec dedup/escalation state
StartModSecEviction(d.stopCh, func() *config.Config { return d.currentCfg() })
// Start background eviction for access log brute force state
StartAccessLogEviction(d.stopCh)
// Start background eviction for email rate limiting state
StartEmailRateEviction(d.stopCh)
// Start background eviction for cloud-relay per-user windows so the
// sync.Map does not grow linearly with every distinct authenticated
// sender ever seen.
StartCloudRelayEviction(d.stopCh)
// Save sender histories off the log-reader path and evict idle windows.
d.wg.Add(1)
obs.Go("sender-profile-maintenance", func() {
defer d.wg.Done()
runSenderProfileMaintenance(d.stopCh)
})
// Start background purge for SMTP brute-force tracker
d.wg.Add(1)
obs.Go("smtp-tracker-purge", func() {
defer d.wg.Done()
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
tick := 0
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
if d.smtpAuthTracker != nil {
d.smtpAuthTracker.Purge()
}
if d.smtpProbeTracker != nil {
d.smtpProbeTracker.Purge()
}
// Diagnostic: surface whether the SMTP/mail brute-force
// trackers are actually seeing auth failures and emitting
// blockable findings. A nonzero record_calls with zero
// findings_emitted over a sustained attack means the
// threshold path, not the wiring, is the gap to chase.
tick++
if tick%10 == 0 {
if d.smtpAuthTracker != nil {
sc, se := d.smtpAuthTracker.Stats()
csmlog.Info("smtp brute tracker stats",
"record_calls", sc, "findings_emitted", se, "tracked", d.smtpAuthTracker.Size())
}
if d.mailAuthTracker != nil {
mc, me := d.mailAuthTracker.Stats()
csmlog.Info("mail brute tracker stats",
"record_calls", mc, "findings_emitted", me, "tracked", d.mailAuthTracker.Size())
}
}
}
}
})
// Start background purge for mail (IMAP/POP3) brute-force tracker. It also
// persists established good-source standing every minute and on shutdown, so
// a daemon restart does not re-open the brute-force false-positive
// cold-start window. The login spray memory is purged on the same tick and
// saved every few ticks and on shutdown.
d.wg.Add(1)
obs.Go("mail-tracker-purge", func() {
defer d.wg.Done()
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
tick := 0
for {
select {
case <-d.stopCh:
d.persistMailGoodSource()
d.persistMailSpray()
return
case <-ticker.C:
if d.mailAuthTracker != nil {
d.mailAuthTracker.Purge()
d.persistMailGoodSource()
}
if d.mailSpray != nil {
d.mailSpray.Purge()
}
tick++
if tick%mailSprayPersistTicks == 0 {
d.persistMailSpray()
}
}
}
})
// Mail auth backend probe: actively detect a cpdoveauthd outage that
// chkservd's port checks miss (dovecot stays up while auth is broken),
// suppress brute-force auto-block while it lasts, and optionally self-heal.
// cPanel only -- elsewhere there is no such socket to probe.
if hostInfo.IsCPanel() && d.authBackend != nil {
if !d.emitAuthBackendFindings() {
return
}
d.wg.Add(1)
obs.Go("mail-auth-backend-probe", func() {
defer d.wg.Done()
ticker := time.NewTicker(mailAuthProbeInterval)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
if !d.emitAuthBackendFindings() {
return
}
}
}
})
}
for _, lf := range logFiles {
w, err := newObservedLogWatcher(lf, d.cfg, d.alertCh)
if err != nil {
if os.IsNotExist(err) {
// File doesn't exist yet - retry periodically until it appears
if lf.name != "" {
d.MarkWatcher(lf.name, false)
}
d.wg.Add(1)
spec := lf
obs.Go("logwatch-retry", func() { d.retryLogWatcherNamed(spec) })
} else {
fmt.Fprintf(os.Stderr, "[%s] Warning: could not watch %s: %v\n", ts(), lf.path, err)
if lf.name != "" {
d.MarkWatcher(lf.name, false)
}
}
continue
}
d.logWatchers = append(d.logWatchers, w)
d.wg.Add(1)
watcher := w
obs.Go("logwatch", func() {
defer d.wg.Done()
watcher.Run(d.stopCh)
})
csmlog.Info("watching log", "path", lf.path)
if lf.name != "" {
d.MarkWatcher(lf.name, true)
}
}
}
// recordEximSMTPAuthSuccess feeds authenticated Exim acceptance lines to the
// SMTP tracker as successful logins. extractAuthUser verifies a real
// top-level A= field and names the mailbox; the IP is deliberately read only
// from H=, whose bracketed address is the connecting client rather than a
// HELO/subject/forwarded-header address.
func recordEximSMTPAuthSuccess(line string, cfg *config.Config, tracker *smtpAuthTracker) []alert.Finding {
user := extractAuthUser(line)
if tracker == nil || user == "" {
return nil
}
hStart, hasHField := eximlog.HFieldStart(line)
if !hasHField {
return nil
}
ip := eximlog.HFieldClientIP(line[hStart:])
if ip == "" {
return nil
}
if parsed := net.ParseIP(ip); parsed != nil {
if v4 := parsed.To4(); v4 != nil {
ip = v4.String()
}
}
if isInfraIPDaemon(ip, cfg.InfraIPs) || isPrivateOrLoopback(ip) {
return nil
}
return tracker.recordSuccessAt(ip, user, customerLoginTime(line, tracker.now()), serviceLog)
}
func (d *Daemon) emitAuthBackendFindings() bool {
if d.authBackend == nil {
return true
}
findings := d.authBackend.Observe()
for i, f := range findings {
if !alert.Enqueue(d.alertCh, f, d.stopCh) {
alert.RecordQueueLoss(d.alertCh, uint64(len(findings[i+1:])))
return false
}
}
return true
}
func (d *Daemon) handleMailLogSourceGone(err error) {
d.MarkWatcher("maillog", false)
finding := alert.Finding{
Severity: alert.Warning,
Check: "mail_log_source_unavailable",
Message: fmt.Sprintf("Mail log source unavailable: %v; brute-force and rate detection degraded while attachment is retried", err),
Timestamp: time.Now(),
}
select {
case <-d.stopCh:
return
default:
}
if !alert.TryEnqueue(d.alertCh, finding) {
atomic.AddInt64(&d.droppedAlerts, 1)
fmt.Fprintf(os.Stderr, "[%s] alert channel full, dropping maillog source finding\n", ts())
}
}
func (d *Daemon) handleMailLogSourceRestored() {
d.MarkWatcher("maillog", true)
}
// handleEximLine wraps parseEximLogLine (unchanged) and augments the result
// with smtpAuthTracker findings for dovecot authenticator failures and
// smtpProbeTracker findings for raw connect-rate abuse (scanners that
// probe-and-disconnect without ever reaching AUTH). obs names the line, so a
// subnet spray can name the line that last counted each address.
func (d *Daemon) handleEximLine(line string, obs alert.Observation, cfg *config.Config) []alert.Finding {
findings := parseEximLogLine(line, cfg)
// Connect-rate signal fires before any AUTH attempt.
if probeIP := parseEximSMTPConnectIP(line); probeIP != "" {
if parsed := net.ParseIP(probeIP); parsed != nil {
if v4 := parsed.To4(); v4 != nil {
probeIP = v4.String()
}
}
if !isInfraIPDaemon(probeIP, cfg.InfraIPs) && !isPrivateOrLoopback(probeIP) {
if d.smtpProbeTracker != nil {
findings = append(findings, d.smtpProbeTracker.Record(probeIP)...)
}
}
}
if failure := eximlog.AuthFailure(line); strings.HasPrefix(failure.Authenticator, "dovecot_") {
ip, account := failure.ClientIP, failure.Account
// Canonicalize IPv4-mapped IPv6 (::ffff:a.b.c.d) to plain IPv4 so the
// tracker doesn't double-count the same attacker as two IPs.
if ip != "" {
if parsed := net.ParseIP(ip); parsed != nil {
if v4 := parsed.To4(); v4 != nil {
ip = v4.String()
}
}
}
if ip != "" && !isInfraIPDaemon(ip, cfg.InfraIPs) && !isPrivateOrLoopback(ip) && d.smtpAuthTracker != nil {
// Exim answers 435 when the Dovecot auth socket cannot answer.
// Only a permanent (5xx) reply says the credentials were checked;
// any other reply still counts per IP but is not spray evidence.
if strings.HasPrefix(failure.Code, "5") {
mb := d.sprayMailboxFor(account, mailboxExistenceUnknown)
findings = append(findings, d.smtpAuthTracker.RecordResolvedFailure(ip, account, mb, obs)...)
} else {
findings = append(findings, d.smtpAuthTracker.RecordTemporaryFailure(ip, account, obs)...)
}
}
}
// An authenticated delivery is a successful login on its mailbox: it
// earns trust for that mailbox only, or is reported as a possible
// compromise when it follows failures on other mailboxes.
findings = append(findings, recordEximSMTPAuthSuccess(line, cfg, d.smtpAuthTracker)...)
// CSM's scan hook logs one marker per scan; a failed scan raises
// antispam_scan_error while antispam is enabled.
at := lineReadAt(obs)
d.recordMailStatsExim(line, at)
if s, ok := antispam.ParseScan(line); ok {
// Exim logs the marker with the SMTP client's own address, so it is
// evidence about that sender.
d.recordSpamEvidence(s, at, cfg)
d.recordSpamSource(s, at, cfg)
findings = append(findings, d.recordScan(s, line, at)...)
}
return findings
}
// lineReadAt is when a line was read; some sources carry no observation
// time.
func lineReadAt(obs alert.Observation) time.Time {
if obs.ObservedAt.IsZero() {
return time.Now()
}
return obs.ObservedAt
}
// handleMailLine composes parseDovecotLogLine (preserving email_suspicious_geo)
// with mailAuthTracker augmentation for IMAP/POP3/ManageSieve brute-force,
// subnet spray, account spray, and compromise detection. obs names the line.
// Neither a Postfix SASL warning (it omits the auth status and can reuse a
// stale reason) nor a Dovecot passdb verdict (one database's answer) is SMTP
// credential evidence alone; handleSMTPAuthPairing counts them together. A
// verdict for a login session only tells that session's login summary whether
// its mailbox exists.
func (d *Daemon) handleMailLine(line string, obs alert.Observation, cfg *config.Config) []alert.Finding {
d.recordMailStatsMail(line, obs.ObservedAt)
findings := parseDovecotLogLine(line, cfg)
// Mail postfix could not pass to rspamd has no scan record; the
// failure counts as a failed scan.
if src := d.antispamScanSource.Load(); src != nil && postfixMilterFailure(line, src.Milters) {
return append(findings, d.recordAntispamScan(antispam.Scan{}, lineReadAt(obs))...)
}
if paired, handled := d.handleSMTPAuthPairing(line, obs, cfg); handled {
return append(findings, paired...)
}
if d.noteDovecotSessionVerdict(line, cfg) {
return findings
}
d.spoilDovecotSessions(line)
// Intermediate auth results are not login failures, and their client
// text must not reach the outage detector even without a full verdict.
if isDovecotAuthRequestInfo(line) {
return findings
}
authLine := isMailAuthLine(line)
// Auth-backend failures (dovecot cannot reach the credential backend)
// arrive on their own lines, not login lines. Feed them to the degraded
// gate so a backend outage pauses brute-force auto-block instead of
// mass-blocking every legitimate user whose login now fails.
if d.mailAuthTracker != nil && !authLine && isMailAuthBackendError(line) {
return append(findings, d.mailAuthTracker.RecordBackendFailure()...)
}
if !authLine {
return findings
}
extractor := currentAccountExtractor()
ip, account, success := extractor.extractMailLoginEvent(line)
if ip == "" {
return findings
}
if parsed := net.ParseIP(ip); parsed != nil {
if v4 := parsed.To4(); v4 != nil {
ip = v4.String()
}
}
if isInfraIPDaemon(ip, cfg.InfraIPs) || isPrivateOrLoopback(ip) {
return findings
}
if d.mailAuthTracker == nil {
return findings
}
verdict, withheld := d.dovecotSessions.Take(dovecotLoginSession(line), ip, account, customerLoginTime(line, d.mailAuthTracker.now()))
if success {
kind := knownCustomerIMAP
if strings.HasPrefix(dovecotServiceMessage(line), "pop3-login:") {
kind = knownCustomerPOP
}
findings = append(findings, d.mailAuthTracker.recordSuccessAt(ip, account, customerLoginTime(line, d.mailAuthTracker.now()), kind)...)
} else {
mb := sprayMailbox{name: normalizeSprayMailbox(account), exist: mailboxExistenceUnknown}
if !withheld && extractor.readsWholeLogin() {
mb = d.sprayMailboxFor(account, verdict)
}
findings = append(findings, recordDovecotFailure(d.mailAuthTracker, ip, account, line, mb, obs)...)
}
return findings
}
// noteDovecotSessionVerdict keeps a login session's Dovecot verdict for that
// session's login summary. It reports whether the line was such a verdict,
// which needs no further handling. Log time, as for a login, keeps reading a
// backlog from making an old verdict fresh.
func (d *Daemon) noteDovecotSessionVerdict(line string, cfg *config.Config) bool {
session, ip, account, exist, ok := dovecotSessionRejection(line)
if !ok {
return false
}
if d.mailAuthTracker != nil && d.sprayRulesOn() && !isInfraIPDaemon(ip, cfg.InfraIPs) && !isPrivateOrLoopback(ip) {
d.dovecotSessions.Note(session, ip, account, exist, customerLoginTime(line, d.mailAuthTracker.now()))
}
return true
}
// Unparsed request messages cannot confirm a missing mailbox. They may report
// a different passdb answer, a backend error or a cut verdict. Diagnostics
// still carry on to the outage detector.
func (d *Daemon) spoilDovecotSessions(line string) {
if d.mailAuthTracker == nil || !d.sprayRulesOn() {
return
}
sessions := dovecotAuthTroubleSessions(line)
seg := dovecotAuthSegment(dovecotServiceMessage(line))
if isDovecotAuthRequestInfo(line) && !strings.HasPrefix(seg, "Debug: ") {
sessions = dovecotSessionsInFields(seg)
}
for _, session := range sessions {
d.dovecotSessions.Spoil(session, customerLoginTime(line, d.mailAuthTracker.now()))
}
}
// handleSMTPAuthPairing feeds Postfix SMTP AUTH evidence on hosts whose
// SMTP AUTH results do not come from the Exim mainlog. It reports whether the
// line was a Dovecot verdict or a Postfix result, which need no further
// handling; Dovecot error lines are noted and still reach the outage detector.
func (d *Daemon) handleSMTPAuthPairing(line string, obs alert.Observation, cfg *config.Config) ([]alert.Finding, bool) {
t := d.smtpAuthTracker
if t == nil || d.smtpAuthFromExim {
return nil, false
}
if ip, account, exist, ok := dovecotSessionlessRejection(line); ok {
if !isInfraIPDaemon(ip, cfg.InfraIPs) && !isPrivateOrLoopback(ip) {
t.NoteDovecotRejection(ip, account, exist, smtpAuthLogTime(line, t.now()))
}
return nil, true
}
if ips := dovecotAuthTroubleIPs(line); len(ips) > 0 {
t.NoteAuthTrouble(ips...)
return nil, false
}
process, msg := postfixSMTPDMessage(line)
// A successful login, handled like an authenticated Exim delivery.
if ip, user := postfixSASLSuccess(msg); ip != "" {
if isInfraIPDaemon(ip, cfg.InfraIPs) || isPrivateOrLoopback(ip) {
return nil, true
}
return t.recordSuccessAt(ip, user, customerLoginTime(line, t.now()), syslogFed), true
}
if strings.HasPrefix(msg, "warning: SASL: ") {
// A failed backend reconnect can leave smtpd's previous reply
// intact. This trusted prefix has no client-supplied text.
t.NoteAuthTrouble(process)
return nil, true
}
ip, reply := postfixSASLFailure(msg)
if ip == "" {
return nil, false
}
// A client can send these on a second connection while its real attempt
// waits out Dovecot's failure delay; claiming a verdict here would hide
// that attempt.
if isPostfixSASLProtocolReply(reply) {
return nil, true
}
// Claim the verdict even for a server-blamed reply so it cannot pair
// with a later failure. Recheck infra protection in case configuration
// changed between the two log lines.
credential := !isPostfixSASLNonCredentialReply(reply) && !isInfraIPDaemon(ip, cfg.InfraIPs) && !isPrivateOrLoopback(ip)
account, exist, names, ok := t.PairMTAFailure(ip, process, smtpAuthLogTime(line, t.now()), credential)
if !ok {
return nil, true
}
if names != nil {
return t.RecordAmbiguousFailure(ip, names, obs), true
}
return t.RecordFailure(ip, account, exist, obs), true
}
func (d *Daemon) dispatchMailLogLine(line maillog.Line, handler ObservedLineHandler) bool {
return d.dispatchLogLine(checks.ProducerMailLog, line, handler)
}
// dispatchLogLine runs handler on one positioned line and enqueues its
// findings, each stamped with producer and the line's position.
func (d *Daemon) dispatchLogLine(producer admission.ProducerID, line maillog.Line, handler ObservedLineHandler) bool {
var obs alert.Observation
if line.Position.Stream != "" {
obs = alert.Observation{Producer: string(producer), Stream: line.Position.Stream, Cursor: line.Position.Cursor, ObservedAt: line.Position.ObservedAt}
}
findings := handler(line.Message, obs, d.currentCfg())
for i, f := range findings {
f.Observation = obs
if !alert.Enqueue(d.alertCh, f, d.stopCh) {
alert.RecordQueueLoss(d.alertCh, uint64(len(findings[i+1:])))
return false
}
}
return true
}
// retryLogWatcher polls for a missing log file every 60 seconds.
// When the file appears, it starts a watcher and returns.
func (d *Daemon) retryLogWatcher(path string, handler LogLineHandler) {
d.retryLogWatcherNamed(logWatchSpec{path: path, handler: handler})
}
// fixedLogSpecs lists the logs at fixed paths and the evidence producer each
// feeds.
func (d *Daemon) fixedLogSpecs(hostInfo platform.Info, exim ObservedLineHandler, watchExim bool) []logWatchSpec {
// Generic Linux auth log. RHEL-family uses /var/log/secure, Debian
// family uses /var/log/auth.log. Only register the log appropriate
// for the detected OS so we don't spam "not found, retrying" forever.
specs := []logWatchSpec{{path: "/var/log/secure", handler: parseSecureLogLine, producer: checks.ProducerSSHLog}}
if hostInfo.IsDebianFamily() {
specs[0].path = "/var/log/auth.log"
}
// cPanel-specific logs only watch these on cPanel hosts. On plain
// Ubuntu/AlmaLinux they do not exist and the old code spammed
// "not found, will retry every 60s" forever.
if hostInfo.IsCPanel() {
specs = append(specs,
logWatchSpec{path: "/usr/local/cpanel/logs/session_log", handler: d.cpanelSessionLogHandler},
logWatchSpec{path: "/usr/local/cpanel/logs/access_log", handler: d.cpanelAccessLogHandler,
observed: d.cpanelAccessLogObservedHandler, producer: checks.ProducerCpanelAccessLog},
logWatchSpec{path: "/var/log/messages", handler: parseFTPLogLine, producer: checks.ProducerFTPLog},
)
}
if watchExim {
// Observed: the SMTP tracker records which line named each address.
specs = append(specs, logWatchSpec{path: eximMainlogPath, producer: checks.ProducerEximLog, observed: exim,
handler: func(line string, cfg *config.Config) []alert.Finding { return exim(line, alert.Observation{}, cfg) }})
}
return specs
}
func shouldWatchEximMainlog(hostInfo platform.Info, stat func(string) (os.FileInfo, error)) bool {
if hostInfo.IsCPanel() {
return true
}
if stat == nil {
stat = os.Stat
}
if _, err := stat(eximMainlogPath); err != nil {
return !os.IsNotExist(err)
}
return true
}
func (d *Daemon) retryLogWatcherNamed(spec logWatchSpec) {
csmlog.Warn("log not found, will retry every 60s", "path", spec.path)
d.retryLogWatcherCandidates([]string{spec.path}, spec)
}
func (d *Daemon) startWebUI() {
if !d.cfg.WebUI.Enabled {
if d.cfg.MCP.Enabled {
csmlog.Error("mcp endpoint not served: it needs webui.enabled: true")
}
return
}
srv, err := webui.New(d.cfg, d.store)
if err != nil {
csmlog.Error("webui init error", "err", err)
return
}
// Set version and signature count for status API
srv.SetVersion(d.version)
if scanner := signatures.Global(); scanner != nil {
srv.SetSigCount(scanner.RuleCount())
}
d.webServer = srv
srv.SetHealthProvider(d)
srv.SetMailStats(d.mailStats)
srv.SetRspamdSource(d.rspamdStatsStatus)
srv.SetFindingBus(d.findingBus)
if d.scanJobs != nil {
srv.SetScanJobs(d.scanJobs)
}
srv.SetIncidentCorrelator(IncidentCorrelator())
// Push web-UI verified_bots edits into the live registry + verifier so they
// take effect without a restart, the same path SIGHUP uses.
srv.SetVerifiedBotsReloader(func() error { d.reconcileVerifiedBots(); return nil })
srv.SetSettingsSaveHook(d.refreshHTTPRewrite)
// A settings save publishes the edited config itself; apply it the way a
// SIGHUP reload does so thresholds, whitelists and the forward guard
// change without a restart.
srv.SetConfigAppliedHook(d.applySavedConfig)
srv.SetHealthInfo(d.FanotifyActive, d.LogWatcherCount)
srv.SetMailboxSuspensions(mailboxSuspensionsAPI{})
if d.fwEngine != nil {
srv.SetIPBlocker(d.fwEngine)
}
if d.cfg.MCP.Enabled {
d.attachMCP(srv)
}
d.wg.Add(1)
obs.Go("webui", func() {
defer d.wg.Done()
if err := srv.Start(); err != nil {
csmlog.Error("webui server error", "err", err)
}
})
}
// attachMCP serves the read-only MCP endpoint at /mcp on the Web UI
// listener, reading through that server's read-only dispatcher.
func (d *Daemon) attachMCP(web *webui.Server) {
info := platform.Detect()
startCfg := d.cfg
srv := mcp.New(mcp.Options{
Dispatcher: web.ReadOnly(),
Version: d.version,
Hostname: d.cfg.Hostname,
Platform: mcp.PlatformFacts(info),
ConfigSummary: func() map[string]any {
cfg := config.Active()
if cfg == nil {
cfg = startCfg
}
return mcp.ConfigSummary(cfg, info)
},
})
web.SetMCP(srv)
csmlog.Info("mcp endpoint enabled", "path", "/mcp", "tools", len(srv.Tools()))
}
func (d *Daemon) startPAMListener() {
pl, err := NewPAMListener(d.cfg, d.alertCh)
if err != nil {
csmlog.Warn("PAM listener not available", "err", err)
d.MarkWatcher("pamlistener", false)
return
}
d.pamListener = pl
d.MarkWatcher("pamlistener", true)
d.RegisterUpstreamProbe("pamlistener", pl.UpstreamResult)
d.wg.Add(1)
obs.Go("pam-listener", func() {
defer d.wg.Done()
pl.Run(d.stopCh)
})
csmlog.Info("PAM listener active", "socket", pamSocketPath)
}
func (d *Daemon) startControlListener() {
cl, err := NewControlListener(d)
if err != nil {
// The daemon can still function without the socket — periodic
// scans and webui keep running — but the CLI will hard-error
// because the socket is the expected path. Log loudly.
csmlog.Error("control listener not available", "err", err)
return
}
cl.scanJobs = d.scanJobs
d.controlListener = cl
d.wg.Add(1)
obs.Go("control-listener", func() {
defer d.wg.Done()
cl.Run(d.stopCh)
})
csmlog.Info("control listener active", "socket", controlSocketPath)
}
func (d *Daemon) startFileMonitor() {
fm, err := NewFileMonitor(d.cfg, d.alertCh) //nolint:staticcheck // The Linux constructor is fallible; the non-Linux stub always returns an error.
if err != nil { //nolint:staticcheck // The Linux constructor can also succeed.
csmlog.Warn("fanotify not available, falling back to periodic deep scan", "err", err)
d.MarkWatcher("fanotify", false)
return
}
fm.registerMetrics()
d.setFileMonitor(fm)
// The placement backend feeds the analyzer queue. Joining it before Run
// means a stop during the backend's load cannot close that queue first.
d.startPlacementMonitor(fm)
d.wg.Add(1)
obs.Go("fanotify", func() {
defer d.wg.Done()
fm.Run(d.stopCh)
})
csmlog.Info("fanotify file monitor active", "roots", fm.WatchScopeSummary())
d.MarkWatcher("fanotify", true)
}
func (d *Daemon) startSpoolWatcher() {
if !d.cfg.EmailAV.Enabled {
return
}
// Create ClamAV scanner. The socket path belongs to whoever packaged
// clamd, so a setting naming the wrong one falls back to a location that
// is actually answering: mail that is silently never scanned looks exactly
// like mail that came back clean.
clamdSocket, discovered := config.ResolveClamdSocket(d.cfg.EmailAV.ClamdSocket)
if discovered {
csmlog.Warn("email av: configured clamd socket is not answering; using a discovered one",
"configured", d.cfg.EmailAV.ClamdSocket, "using", clamdSocket)
}
clamScanner := emailav.NewClamdScanner(clamdSocket)
// YARA-X scanner over whichever backend initYaraBackend installs.
// The worker backend can come online after startup through the boot-retry
// path, so resolve yara.Active() lazily instead of capturing a nil backend
// forever while email AV is being wired.
yaraScanner := emailav.NewActiveYaraXScanner()
// Create orchestrator with both engines
scanners := []emailav.Scanner{clamScanner, yaraScanner}
orch := emailav.NewOrchestrator(scanners, d.cfg.EmailAV.ScanTimeoutDuration())
d.registerQueueSource("email_av", orch)
// Create quarantine
quar := emailav.NewQuarantine("/opt/csm/quarantine/email")
d.emailQuarantine = quar
// Create and start spool watcher
sw, err := NewSpoolWatcher(d.cfg, d.alertCh, orch, quar, d.mailStats.RecordQuarantine, d.spoolExtractor(), &d.parseFindings) //nolint:staticcheck // The Linux constructor is fallible; the non-Linux stub always returns an error.
if err != nil { //nolint:staticcheck // The Linux constructor can also succeed.
fmt.Fprintf(os.Stderr, "[%s] Email AV spool watcher not available: %v\n", ts(), err)
d.MarkWatcher("email_av_spool", false)
return
}
d.setSpoolWatcher(sw)
d.MarkWatcher("email_av_spool", true)
d.wg.Add(1)
obs.Go("spool-watcher", func() {
defer d.wg.Done()
d.runSpoolWatcherLoop(sw, orch, quar)
})
// Start quarantine cleanup goroutine
d.wg.Add(1)
obs.Go("email-quarantine-cleanup", d.emailQuarantineCleanup)
fmt.Fprintf(os.Stderr, "[%s] Email AV spool watcher active\n", ts())
}
// superviseWatcherRun runs run() to completion. If daemonStop closes while
// run() is still blocked, stop() is invoked so run() can return. The helper
// goroutine is reaped when run() returns on its own. This guarantees the live
// watcher instance is stopped on shutdown even after a crash-restart swapped a
// fresh instance in, which the external shutdown path (it only stops the
// instance registered via setSpoolWatcher) can miss, hanging wg.Wait forever.
func superviseWatcherRun(daemonStop <-chan struct{}, run, stop func()) {
done := make(chan struct{})
helperDone := make(chan struct{})
go func() {
defer close(helperDone)
select {
case <-daemonStop:
stop()
case <-done:
}
}()
run()
close(done)
<-helperDone
}
type spoolWatcherRuntime interface {
Run()
Stop()
}
func (d *Daemon) runSpoolWatcherLoop(sw *SpoolWatcher, orch *emailav.Orchestrator, quar *emailav.Quarantine) {
d.runSpoolWatcherLoopWithFactory(sw, 2*time.Second, func() (spoolWatcherRuntime, error) {
next, err := NewSpoolWatcher(d.cfg, d.alertCh, orch, quar, d.mailStats.RecordQuarantine, d.spoolExtractor(), &d.parseFindings) //nolint:staticcheck // The Linux constructor is fallible; the non-Linux stub always returns an error.
if err != nil { //nolint:staticcheck // The Linux constructor can also succeed.
return nil, err
}
d.setSpoolWatcher(next)
return next, nil
})
}
func (d *Daemon) runSpoolWatcherLoopWithFactory(current spoolWatcherRuntime, restartDelay time.Duration, newWatcher func() (spoolWatcherRuntime, error)) {
for {
superviseWatcherRun(d.stopCh, current.Run, current.Stop)
select {
case <-d.stopCh:
return
default:
}
fmt.Fprintf(os.Stderr, "[%s] Email AV spool watcher stopped unexpectedly; restarting in %s\n", ts(), restartDelay)
for {
select {
case <-d.stopCh:
return
case <-time.After(restartDelay):
}
next, err := newWatcher()
if err != nil {
fmt.Fprintf(os.Stderr, "[%s] Email AV spool watcher restart failed: %v\n", ts(), err)
continue
}
current = next
break
}
}
}
func (d *Daemon) setSpoolWatcher(sw *SpoolWatcher) {
sw.daemonStop = d.stopCh
sw.setCrashSamples(d.keptCrashSamples())
d.spoolWatcherMu.Lock()
if d.spoolWatcher != nil {
sw.inheritQueueHealth(d.spoolWatcher)
sw.inheritStrikes(d.spoolWatcher)
}
d.spoolWatcher = sw
d.spoolWatcherMu.Unlock()
d.syncEmailAVWebState()
}
func (d *Daemon) getSpoolWatcher() *SpoolWatcher {
d.spoolWatcherMu.Lock()
defer d.spoolWatcherMu.Unlock()
return d.spoolWatcher
}
// startForwarderWatcher starts the inotify watcher for /etc/valiases/.
func (d *Daemon) startForwarderWatcher() {
fw, err := NewForwarderWatcher(d.alertCh, d.cfg.EmailProtection.KnownForwarders) //nolint:staticcheck // The Linux constructor is fallible; the non-Linux stub always returns an error.
if err != nil { //nolint:staticcheck // The Linux constructor can also succeed.
fmt.Fprintf(os.Stderr, "[%s] Warning: forwarder watcher not started: %v\n", ts(), err)
d.MarkWatcher("forwarder", false)
return
}
d.forwarderWatcher = fw
d.registerQueueSource("forwarder", fw)
d.wg.Add(1)
obs.Go("forwarder-watcher", func() {
defer d.wg.Done()
fw.Run(d.stopCh)
})
csmlog.Info("watching log (inotify forwarder watcher)", "path", "/etc/valiases/")
d.MarkWatcher("forwarder", true)
}
func (d *Daemon) syncEmailAVWebState() {
if d.webServer == nil || d.emailQuarantine == nil {
return
}
d.webServer.SetEmailQuarantine(d.emailQuarantine)
if sw := d.getSpoolWatcher(); sw != nil {
if sw.PermissionMode() {
d.webServer.SetEmailAVWatcherMode("permission")
} else {
d.webServer.SetEmailAVWatcherMode("notification")
}
}
}
// emailQuarantineCleanup periodically removes expired quarantined email messages.
func (d *Daemon) emailQuarantineCleanup() {
defer d.wg.Done()
ticker := time.NewTicker(1 * time.Hour)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
if d.emailQuarantine != nil {
cleaned, err := d.emailQuarantine.CleanExpired(30 * 24 * time.Hour)
if err != nil {
fmt.Fprintf(os.Stderr, "[%s] Email quarantine cleanup error: %v\n", ts(), err)
} else if cleaned > 0 {
fmt.Fprintf(os.Stderr, "[%s] Email quarantine cleanup: removed %d expired messages\n", ts(), cleaned)
}
}
}
}
}
func (d *Daemon) startChallengeServer() {
if d.challengeServer != nil {
return
}
// Challenge suppression must fail open until the listener is known to be
// available. This also clears stale package wiring in repeated daemon
// construction paths used by tests and embedding callers.
checks.SetChallengeIPList(nil)
alert.ChallengedIPFunc = nil
if !d.cfg.Challenge.Enabled {
return
}
if d.fwEngine == nil {
fmt.Fprintf(os.Stderr, "[%s] Challenge server requires firewall to be enabled (for escalation). Skipping.\n", ts())
return
}
d.ipList = challenge.NewIPList(challenge.DefaultMapPath)
if db := store.Global(); db != nil {
if err := d.ipList.SetPassStore(db); err != nil {
csmlog.Warn("challenge passes from before this start are unknown", "err", err)
}
}
if platform.Detect().WebServer == platform.WSNginx {
d.ipList.SetNginxMap(challenge.DefaultNginxMapPath, d.reloadChallengeNginxMap)
}
d.attachChallengePortGate()
srv := challenge.New(d.cfg, d.ipList)
listener, err := srv.Listen()
if err != nil {
if d.challengeGate != nil {
if closeErr := d.challengeGate.Close(); closeErr != nil {
fmt.Fprintf(os.Stderr, "[%s] challenge port-gate cleanup: %v\n", ts(), closeErr)
}
d.challengeGate = nil
}
d.ipList = nil
csmlog.Error("challenge server unavailable; challenge routing disabled", "err", err)
return
}
checks.SetChallengeIPList(d.ipList)
alert.ChallengedIPFunc = d.ipList.Contains
d.challengeServer = srv
d.wg.Add(1)
obs.Go("challenge-server", func() {
defer d.wg.Done()
csmlog.Info("challenge server active", "addr", srv.Addr())
if err := srv.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
csmlog.Error("challenge server error", "err", err)
}
})
}
// attachChallengePortGate installs the nftables port-gate for the
// challenge listener when the operator opts in. The gate is silently
// absent when the listener is loopback-only (no off-host traffic can
// reach it anyway) or on non-Linux builds.
func (d *Daemon) attachChallengePortGate() {
if !d.cfg.Challenge.PortGate.Enabled {
return
}
gate, err := challenge.NewPortGate(challenge.PortGateConfig{
ListenAddr: d.cfg.Challenge.ListenAddr,
ListenPort: d.cfg.Challenge.ListenPort,
InfraCIDRs: d.cfg.InfraIPs,
})
if err != nil {
fmt.Fprintf(os.Stderr, "[%s] challenge port-gate install failed: %v (listener stays publicly reachable)\n", ts(), err)
return
}
if gate == nil {
csmlog.Info("challenge port-gate skipped (local listener or non-Linux build)",
"listen_addr", d.cfg.Challenge.ListenAddr)
return
}
d.challengeGate = gate
d.ipList.SetPortGate(gate)
csmlog.Info("challenge port-gate active", "port", d.cfg.Challenge.ListenPort)
}
func (d *Daemon) reloadChallengeNginxMap() error {
// #nosec G204 -- static binary and arguments; no operator input is passed.
out, err := exec.Command("nginx", "-s", "reload").CombinedOutput()
if err != nil {
return fmt.Errorf("nginx -s reload: %w: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
func (d *Daemon) challengeEscalator() {
defer d.wg.Done()
ticker := time.NewTicker(60 * time.Second)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
// Re-read the block expiry each tick so a SIGHUP that
// changes auto_response.block_expiry takes effect on the
// next escalation without requiring a restart.
expiry := parseBlockExpiry(d.currentCfg().AutoResponse.BlockExpiry)
if d.challengeServer != nil {
d.challengeServer.CleanExpired()
}
d.escalateExpiredChallenges(expiry)
}
}
}
// escalateExpiredChallenges hard-blocks every challenge-list entry whose
// window lapsed. Blocks route through the checks.ApplyBlock chokepoint so
// escalations leave the same evidence trail as scan auto-blocks (threat-DB
// row, tracker entry, digest visibility, permblock counting) instead of
// only a stderr line.
func (d *Daemon) escalateExpiredChallenges(expiry time.Duration) {
if d.ipList == nil {
return
}
expired := d.ipList.ExpiredEntries()
if len(expired) == 0 {
return
}
var recorded []alert.Finding
for _, e := range expired {
res, err := checks.ApplyBlock(d.currentCfg(), checks.ApplyBlockRequest{
IP: e.IP,
EngineReason: fmt.Sprintf("CSM challenge-timeout: %s", truncateStr(e.Reason, 100)),
Reason: challengeTimeoutReasonPrefix + truncateStr(e.Reason, 100),
TTL: expiry,
Source: checks.BlockSourceChallenge,
FindingID: e.FindingID,
Root: e.Root,
Entry: admission.EntryChallengeTimeout,
Check: e.Check,
})
recorded = append(recorded, res.Findings...)
if err != nil {
// Own-interface / infra IPs are never blockable, a host
// without a firewall engine cannot escalate, and an operator
// may have turned automatic blocking off; all are expected
// no-ops, not failures worth logging.
if !isExpectedEscalationRefusal(err) {
fmt.Fprintf(os.Stderr, "[%s] challenge-escalate: error blocking %s: %v\n", ts(), e.IP, err)
}
if res.Outcome != firewall.BlockOutcomeLive || !errors.Is(err, firewall.ErrActionAuditPending) {
continue
}
}
observeChallengeEscalated(res.Outcome)
fmt.Fprintf(os.Stderr, "[%s] %s\n", ts(), challengeEscalateLogLine(e.IP, res.Outcome))
}
d.recordAppliedBlocks(recorded)
}
// isExpectedEscalationRefusal reports a challenge escalation that could not
// block for an expected reason, including one left to admission.
func isExpectedEscalationRefusal(err error) bool {
return isProtectedIPRefusal(err) || errors.Is(err, checks.ErrNoIPBlocker) || errors.Is(err, checks.ErrAutoBlockDisabled) ||
errors.Is(err, checks.ErrBlockTakenByAdmission)
}
// markAppliedBlocks sets the attack database's blocked marker for each
// address a Critical auto_block finding names: a block that applied. A
// dry-run "would be blocked" Warning never marks an address blocked
// (only verified full-address blocks call MarkBlocked).
func markAppliedBlocks(findings []alert.Finding) {
adb := attackdb.Global()
if adb == nil {
return
}
for _, f := range findings {
if f.Check != "auto_block" || f.Severity != alert.Critical {
continue
}
if ip := checks.ExtractIPFromFinding(f); ip != "" {
adb.MarkBlocked(ip)
}
}
}
// recordAppliedBlocks routes chokepoint findings from async block sources
// (challenge escalation, central intel, incident spray) through the same
// bookkeeping dispatchBatch gives scan blocks: block digest, attack-DB
// blocked marker, finding history, and alert dispatch, which applies the
// standard suppression rules. Deduplication happens before the fanout so every
// sink sees the same single copy, and a failure in one sink does not skip the
// sinks that follow it.
func (d *Daemon) recordAppliedBlocks(findings []alert.Finding) {
alert.FillTimestamps(findings, time.Now())
findings = alert.Deduplicate(findings)
if len(findings) == 0 {
return
}
d.observeBlocks(findings)
markAppliedBlocks(findings)
if d.store != nil {
d.store.AppendHistory(findings)
}
alertable := findings
if d.store != nil {
alertable = filterUnsuppressedFindings(d.store, findings, d.store.LoadSuppressions())
}
if err := alert.DispatchWithEnforcement(d.currentCfg(), alertable, findings, findings); err != nil {
fmt.Fprintf(os.Stderr, "[%s] Applied-block alert dispatch error: %v\n", ts(), err)
}
}
// applyIncidentBlock is the incident correlator's firewall hand-off for
// incident and spray blocks, routed through the chokepoint so they leave
// evidence and reach the digest. root and entry are the attesting
// observation's admission evidence and the path that decided the block.
func (d *Daemon) applyIncidentBlock(ip, reason string, timeout time.Duration, findingID string, root incident.PreparedRoot, entry admission.Entry) (bool, error) {
res, err := checks.ApplyBlock(d.currentCfg(), checks.ApplyBlockRequest{
IP: ip,
EngineReason: reason,
Reason: reason,
TTL: timeout,
Source: checks.BlockSourceIncident,
FindingID: findingID,
Root: root.Evidence,
Entry: entry,
RootFinding: root.Finding,
RootErr: root.Err,
})
d.recordAppliedBlocks(res.Findings)
// A block admission takes is requested: the correlator latches it once.
if errors.Is(err, checks.ErrBlockTakenByAdmission) {
return true, nil
}
live := res.Outcome == firewall.BlockOutcomeLive && (err == nil || errors.Is(err, firewall.ErrActionAuditPending))
return live, err
}
var (
challengeEscalatedMetric *metrics.CounterVec
challengeEscalatedMetricOnce sync.Once
)
// challengeEscalatedCounter returns the lazily registered metric. Count paths
// use the same Once gate as increments so first status reads cannot race the
// first escalation.
func challengeEscalatedCounter() *metrics.CounterVec {
challengeEscalatedMetricOnce.Do(func() {
challengeEscalatedMetric = metrics.NewCounterVec(
"csm_challenge_escalated_total",
"Challenge-timeout escalations, by firewall outcome (live, noop, dry_run, allowed, allowlisted).",
[]string{"outcome"},
)
metrics.MustRegister("csm_challenge_escalated_total", challengeEscalatedMetric)
})
return challengeEscalatedMetric
}
// observeChallengeEscalated counts one challenge-timeout escalation, labelled by
// the firewall outcome (live=a new hard block landed, noop=the IP was already
// blocked, plus dry_run/allowed/allowlisted). It lets operators see how many
// challenges became real blocks versus no-ops. Registered lazily on first use.
func observeChallengeEscalated(outcome firewall.BlockOutcome) {
challengeEscalatedCounter().With(string(outcome)).Inc()
}
// challengeEscalatedCount returns how many challenge timeouts escalated to a new
// hard block (outcome=live) since daemon start, for the web UI challenge panel.
func challengeEscalatedCount() int {
return int(challengeEscalatedCounter().With(string(firewall.BlockOutcomeLive)).Value())
}
// challengeEscalateLogLine renders the stderr line for one challenge-timeout
// escalation. Only a live block claims "hard-blocked"; a no-op (the IP was
// already hard-blocked, e.g. a confirmed-threat finding blocked it while it sat
// on the challenge list) or a verdict downgrade must not, so incident review is
// not misled by a block that never landed.
func challengeEscalateLogLine(ip string, outcome firewall.BlockOutcome) string {
switch outcome {
case firewall.BlockOutcomeLive:
return fmt.Sprintf("CHALLENGE-ESCALATE: %s timed out, hard-blocked", ip)
case firewall.BlockOutcomeDryRun:
return fmt.Sprintf("CHALLENGE-ESCALATE [dry-run]: %s timed out, would be hard-blocked", ip)
default:
return fmt.Sprintf("challenge-escalate: %s timed out, no new block (outcome: %s)", ip, outcome)
}
}
func parseBlockExpiry(s string) time.Duration {
if s == "" {
return 24 * time.Hour
}
d, err := time.ParseDuration(s)
if err != nil {
return 24 * time.Hour
}
return d
}
func truncateStr(s string, max int) string {
if len(s) <= max {
return s
}
return s[:max]
}
func (d *Daemon) initGeoIP() {
dbDir := filepath.Join(d.cfg.StatePath, "geoip")
db := geoip.Open(dbDir)
if db != nil {
d.geoipDB = db
setGeoIPDB(db) // make available to log watcher handlers for country filtering
if d.webServer != nil {
d.webServer.SetGeoIPDB(db)
}
}
}
// publishGeoIP reloads existing GeoIP databases or creates a new DB
// if databases were downloaded for the first time.
// Mutex-protected: safe to call from geoipUpdater goroutine and SIGHUP handler concurrently.
func (d *Daemon) publishGeoIP() {
d.geoipMu.Lock()
defer d.geoipMu.Unlock()
if d.geoipDB != nil {
if err := d.geoipDB.Reload(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] GeoIP reload error: %v\n", ts(), err)
} else {
fmt.Fprintf(os.Stderr, "[%s] GeoIP databases reloaded\n", ts())
}
return
}
// First-time: no DB existed at startup, try to open freshly downloaded files
dbDir := filepath.Join(d.cfg.StatePath, "geoip")
db := geoip.OpenFresh(dbDir)
if db != nil {
d.geoipDB = db
setGeoIPDB(db)
if d.webServer != nil {
d.webServer.SetGeoIPDB(db)
}
fmt.Fprintf(os.Stderr, "[%s] GeoIP databases loaded for the first time\n", ts())
}
}
// geoipUpdater periodically downloads updated GeoLite2 databases.
func (d *Daemon) geoipUpdater() {
defer d.wg.Done()
// Skip if no credentials configured
if d.cfg.GeoIP.AccountID == "" || d.cfg.GeoIP.LicenseKey == "" {
return
}
// Skip if auto_update is explicitly false
if d.cfg.GeoIP.AutoUpdate != nil && !*d.cfg.GeoIP.AutoUpdate {
return
}
interval := 24 * time.Hour
if d.cfg.GeoIP.UpdateInterval != "" {
if parsed, err := time.ParseDuration(d.cfg.GeoIP.UpdateInterval); err == nil && parsed >= time.Hour {
interval = parsed
}
}
// Wait 5 minutes before first update attempt (let the daemon stabilize)
select {
case <-d.stopCh:
return
case <-time.After(5 * time.Minute):
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
d.doGeoIPUpdate()
select {
case <-d.stopCh:
return
case <-ticker.C:
}
}
}
func (d *Daemon) doGeoIPUpdate() {
results := geoip.Update(
filepath.Join(d.cfg.StatePath, "geoip"),
d.cfg.GeoIP.AccountID,
d.cfg.GeoIP.LicenseKey,
d.cfg.GeoIP.Editions,
)
if results == nil {
return
}
anyUpdated := false
for _, r := range results {
switch r.Status {
case "updated":
fmt.Fprintf(os.Stderr, "[%s] GeoIP auto-update: %s updated\n", ts(), r.Edition)
anyUpdated = true
case "up_to_date":
// silent
case "error":
fmt.Fprintf(os.Stderr, "[%s] GeoIP auto-update: %s error: %v\n", ts(), r.Edition, r.Err)
}
}
if anyUpdated {
d.publishGeoIP()
}
}
func (d *Daemon) autoResponseDryRunEnabled() bool {
return d.activeOrStartupCfg().AutoResponseDryRunEnabled()
}
func (d *Daemon) askVerdictCallback(ctx context.Context, ip, reason string) (string, string, string, error) {
cfg := d.activeOrStartupCfg()
if cfg == nil || !cfg.AutoResponse.VerdictCallback.Enabled {
return "", "", "", nil
}
resp, err := newVerdictClient(cfg).Ask(ctx, verdict.Request{
IP: ip,
Reason: reason,
Severity: "auto",
Source: "auto_response",
})
if err != nil {
return "", "", "", err
}
return resp.Verdict, resp.TenantID, resp.Note, nil
}
func dynDNSUnresolvableFinding(host string) alert.Finding {
return alert.Finding{
Check: "infra_ips_unresolvable",
Severity: alert.Warning,
Message: fmt.Sprintf("dynamic firewall host %s has not resolved within grace period", host),
Details: "Verify DNS for the host or remove it from infra_ips or firewall.dyndns_hosts. While unresolvable, the previous resolved IP remains protected and a rotated IP will not be protected.",
Timestamp: time.Now(),
}
}
// cloudflareRefreshLoop fetches Cloudflare IPs and updates the firewall sets periodically.
func (d *Daemon) cloudflareRefreshLoop() {
defer d.wg.Done()
interval := time.Duration(d.cfg.Cloudflare.RefreshHours) * time.Hour
// Every fetch is cancelled by shutdown. Without this the startup fetch runs
// before the select below is ever reached, so a host that cannot reach
// cloudflare.com holds shutdown for the HTTP timeout.
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
select {
case <-d.stopCh:
cancel()
case <-ctx.Done():
}
}()
// Fetch immediately on startup
d.refreshCloudflareIPs(ctx)
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
d.refreshCloudflareIPs(ctx)
}
}
}
// fetchCloudflareIPs downloads the current Cloudflare ranges. Var so tests
// can feed the refresh an unusable result.
var fetchCloudflareIPs = firewall.FetchCloudflareIPs
func (d *Daemon) refreshCloudflareIPs(ctx context.Context) {
ipv4, ipv6, err := fetchCloudflareIPs(ctx)
if err != nil {
csmlog.Error("cloudflare IP fetch error", "err", err)
}
fresh := len(ipv4) > 0 || len(ipv6) > 0
cached4, cached6 := firewall.LoadCFState(d.cfg.StatePath)
if len(ipv4) == 0 {
ipv4 = cached4
}
if len(ipv6) == 0 {
ipv6 = cached6
}
if len(ipv4) == 0 && len(ipv6) == 0 {
csmlog.Error("cloudflare IP refresh has no fetched or cached ranges")
return
}
// Restore the local auto-block guard even when nftables is unavailable.
// Otherwise the daemon's first refresh failure leaves cached edges
// blockable until the network recovers.
allCF := make([]string, 0, len(ipv4)+len(ipv6))
allCF = append(allCF, ipv4...)
allCF = append(allCF, ipv6...)
checks.SetCloudflareNets(allCF)
if d.fwEngine != nil {
if err := d.fwEngine.UpdateCloudflareSet(ipv4, ipv6); err != nil {
fmt.Fprintf(os.Stderr, "[%s] Cloudflare set update error: %v\n", ts(), err)
return
}
}
if fresh {
if err := firewall.SaveCFState(d.cfg.StatePath, ipv4, ipv6, time.Now()); err != nil {
csmlog.Error("cloudflare state save error", "err", err)
}
}
}
// signatureUpdater periodically downloads new rules and reloads scanners.
func (d *Daemon) signatureUpdater() {
defer d.wg.Done()
yamlEnabled := d.cfg.Signatures.UpdateURL != ""
forgeEnabled := d.cfg.Signatures.YaraForge.Enabled && yara.Available()
if !yamlEnabled && !forgeEnabled {
return
}
select {
case <-d.stopCh:
return
case <-time.After(5 * time.Minute):
}
yamlInterval := 24 * time.Hour
if d.cfg.Signatures.UpdateInterval != "" {
if parsed, err := time.ParseDuration(d.cfg.Signatures.UpdateInterval); err == nil && parsed >= time.Hour {
yamlInterval = parsed
}
}
forgeInterval := 168 * time.Hour
if d.cfg.Signatures.YaraForge.UpdateInterval != "" {
if parsed, err := time.ParseDuration(d.cfg.Signatures.YaraForge.UpdateInterval); err == nil && parsed >= time.Hour {
forgeInterval = parsed
}
}
tickInterval := yamlInterval
if forgeEnabled && forgeInterval < tickInterval {
tickInterval = forgeInterval
}
if !yamlEnabled {
tickInterval = forgeInterval
}
var lastYAML, lastForge time.Time
ticker := time.NewTicker(tickInterval)
defer ticker.Stop()
for {
now := time.Now()
if yamlEnabled && now.Sub(lastYAML) >= yamlInterval {
d.doSignatureUpdate()
lastYAML = now
}
if forgeEnabled && now.Sub(lastForge) >= forgeInterval {
d.doForgeUpdate()
lastForge = now
}
select {
case <-d.stopCh:
return
case <-ticker.C:
}
}
}
func (d *Daemon) doSignatureUpdate() {
count, err := signatures.Update(d.cfg.Signatures.RulesDir, d.cfg.Signatures.UpdateURL, d.cfg.Signatures.SigningKey, signatures.UpdateOptions{
AllowRuleCountDecrease: d.cfg.Signatures.AllowRuleCountDecrease,
})
if err != nil {
d.reportSignatureUpdateError(err)
return
}
fmt.Fprintf(os.Stderr, "[%s] Signature auto-update: %d rules downloaded\n", ts(), count)
d.reloadSignatures()
}
func (d *Daemon) reportSignatureUpdateError(err error) {
fmt.Fprintf(os.Stderr, "[%s] Signature auto-update failed: %v\n", ts(), err)
if errors.Is(err, signatures.ErrUpdateRollback) {
d.emitReloadFinding(alert.Critical, "signature_update_rollback",
fmt.Sprintf("Signed rule update refused by rollback protection: %v", err))
}
}
// forgeRollbackNeeded reports whether a freshly installed Forge ruleset has to
// be undone, given the scanner's rule total after reload and the number of
// rules the new archive carries. A total that cannot even account for the new
// archive means it did not compile in.
//
// The pre-reload total is deliberately not consulted. It includes the Forge
// rules being replaced, so an upstream release carrying fewer rules than the
// installed one reads as a conflict, and the recovery action then deletes the
// entire Forge file -- discarding thousands of working rules over a decrease of
// a few dozen.
func forgeRollbackNeeded(newCount, forgeRuleCount int) bool {
return newCount < forgeRuleCount
}
func (d *Daemon) doForgeUpdate() {
// yara.Active() resolves to the in-process scanner or the worker
// supervisor depending on signatures.yara_worker_enabled; both
// satisfy the Reload/RuleCount calls this routine makes, so the
// forge update path is backend-agnostic.
yaraScanner := yara.Active()
if yaraScanner == nil {
// No YARA backend active (build without yara tag or no rules
// dir). Skip Forge update - rules can't be loaded anyway.
return
}
db := store.Global()
currentVersion := ""
if db != nil {
currentVersion = db.GetMetaString("forge_version_" + d.cfg.Signatures.YaraForge.Tier)
}
// In worker mode the downloaded tier compiles in the compile unit, never
// in this root process.
compileCheck := yara.TestCompile
if d.yaraSup != nil {
compileCheck = d.yaraSup.TestCompile
}
newVersion, count, err := signatures.ForgeUpdateFromURL(
d.cfg.Signatures.RulesDir,
d.cfg.Signatures.YaraForge.Tier,
currentVersion,
d.cfg.Signatures.SigningKey,
d.cfg.Signatures.YaraForge.DownloadURL,
d.cfg.Signatures.DisabledRules,
compileCheck,
)
if err != nil {
fmt.Fprintf(os.Stderr, "[%s] YARA Forge update failed: %v\n", ts(), err)
return
}
if count == 0 {
return
}
fmt.Fprintf(os.Stderr, "[%s] YARA Forge update: %d rules (version %s)\n", ts(), count, newVersion)
forgeFile := filepath.Join(d.cfg.Signatures.RulesDir, fmt.Sprintf("yara-forge-%s.yar", d.cfg.Signatures.YaraForge.Tier))
if !d.settleForgeInstall(yaraScanner, forgeFile, count) {
return // don't store version
}
if db != nil {
_ = db.SetMetaString("forge_version_"+d.cfg.Signatures.YaraForge.Tier, newVersion)
}
}
// forgeReloader is the slice of the YARA backend the Forge settle step uses.
type forgeReloader interface {
Reload() error
RuleCount() int
}
// settleForgeInstall reloads the merged rules directory after a Forge tier
// was written and decides whether the tier stays. It goes when the merged
// reload fails (the tier compiled alone but conflicts with the shipped
// rules) or when the loaded count collapses; either way the file is
// removed, the backend reloads without it, and a Critical finding says so.
// Leaving a non-compiling tier on disk kept the live rules for now but made
// the next worker restart compile the same directory, fail, and run with
// zero rules until an operator deleted the file by hand. Returns true when
// the tier stays and its version may be recorded.
func (d *Daemon) settleForgeInstall(backend forgeReloader, forgeFile string, count int) bool {
reason := ""
if err := backend.Reload(); err != nil && retryableYaraCompile(err) {
// The compile worker did not run, which says nothing about the
// rules. Keep the tier for the next successful reload and leave the
// version unrecorded so the next cycle installs it again.
d.emitReloadFinding(alert.Critical, "yara_backend_unavailable", fmt.Sprintf(
"YARA rules could not be reloaded after a Forge update: the compile worker failed (%v). %s stays installed and loads on the next successful reload; scanning continues on the current rules.",
err, forgeFile))
return false
} else if err != nil {
reason = fmt.Sprintf("the merged rules failed to compile (%v)", err)
} else if newCount := backend.RuleCount(); forgeRollbackNeeded(newCount, count) {
reason = fmt.Sprintf("%d rules downloaded but only %d loaded", count, newCount)
} else {
fmt.Fprintf(os.Stderr, "[%s] Reloaded %d YARA rules after Forge update\n", ts(), newCount)
return true
}
_ = os.Remove(forgeFile)
if err := backend.Reload(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] YARA reload after Forge rollback error: %v\n", ts(), err)
}
// Losing a ruleset this size is a coverage collapse, so it has to be
// alertable rather than a line on stderr that the journal rotates away.
d.emitReloadFinding(alert.Critical, "yara_forge_rollback", fmt.Sprintf(
"YARA Forge update rolled back: %s, so %s was removed. Scanning continues on the remaining rules.",
reason, forgeFile))
return false
}
func (d *Daemon) reloadSignatures() {
if scanner := signatures.Global(); scanner != nil {
if err := scanner.Reload(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] YAML rule reload error: %v\n", ts(), err)
} else {
fmt.Fprintf(os.Stderr, "[%s] Reloaded %d YAML rules (version %d)\n", ts(), scanner.RuleCount(), scanner.Version())
if d.webServer != nil {
d.webServer.SetSigCount(scanner.RuleCount())
}
}
}
yaraRules, yaraActive := 0, false
if yaraScanner := yara.Active(); yaraScanner != nil {
if err := yaraScanner.Reload(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] YARA rule reload error: %v\n", ts(), err)
} else {
fmt.Fprintf(os.Stderr, "[%s] Reloaded %d YARA rule file(s)\n", ts(), yaraScanner.RuleCount())
yaraActive = true
yaraRules = yaraScanner.RuleCount()
}
}
d.reportRealtimeRuleCoverage(yamlRuleCount(), yaraRules, yaraActive)
}
// The startup integrations are indirected so the observe-mode gate around them
// can be tested without a host to write to or a web server to reload.
var (
ensureAuditdRules = auditd.EnsureDeployed
deployHostConfigs = deployConfigs
reconcileModSecReload = (*checks.ModSecReloadReconciler).Reconcile
)
// applyStartupIntegrations refreshes the host-side files CSM owns: the auditd
// rules, the WHM plugin, the ModSecurity section and the deploy script. None
// of them has a switch of its own, so observe mode is what turns them off.
//
// Self-healing the auditd rules matters because package upgrades sometimes
// ship a new csm binary without re-running auditd.Deploy() (postinstall hooks
// differ across apt/dnf and across operator deploy automation), which leaves
// new rules -- including detection layers like csm_af_alg_socket -- silently
// inactive on the upgraded host. Errors are non-fatal: if auditd is absent or
// augenrules fails, the rest of CSM still runs.
func (d *Daemon) applyStartupIntegrations() {
if d.cfg.ObserveMode() {
csmlog.Info("observe mode: skipping host integration deploy (auditd rules, WHM plugin, ModSecurity section, deploy script)")
return
}
if redeployed, err := ensureAuditdRules(); err != nil {
csmlog.Warn("auditd rules ensure failed", "err", err)
} else if redeployed {
csmlog.Info("auditd rules redeployed (drift from embedded constant)")
}
deployHostConfigs()
if err := reconcileModSecReload(&d.modsecReload, d.cfg.ModSec.ReloadCommand); err != nil {
csmlog.Warn("CSM ModSecurity rule activation could not be confirmed", "err", err)
}
}
// deployConfigs writes embedded config files to their system locations on startup.
// Ensures WHM plugin CGI and ModSec rules stay current after binary upgrades.
//
// Every file written here is a system integration point consumed by a
// different process (WHM, Apache, nginx); the permissions intentionally
// allow the right external reader. Gosec G301/G306 warnings on this
// function are suppressed inline with the specific integration target.
func deployConfigs() {
// WHM plugin CGI - embedded in binary
if _, err := os.Stat("/usr/local/cpanel"); err == nil {
dst := "/usr/local/cpanel/whostmgr/docroot/cgi/addon_csm.cgi"
// #nosec G306 -- WHM CGI endpoint; 0755 is required so cPanel's
// webserver can execute it.
if err := os.WriteFile(dst, embeddedWHMCGI, 0755); err == nil {
csmlog.Info("WHM plugin CGI deployed", "path", dst)
}
// Write the AppConfig file, then register it with WHM.
// Writing the file alone does NOT make the plugin appear in the
// sidebar — WHM's AppConfig system maintains a registration
// database that is updated via `register_appconfig`. Skipping
// that step was a long-standing bug; the plugin file existed on
// disk but never showed up in the menu.
// #nosec G301 -- cPanel standard /var/cpanel/apps directory.
_ = os.MkdirAll("/var/cpanel/apps", 0755)
confPath := "/var/cpanel/apps/csm.conf"
// #nosec G306 -- WHM AppConfig; read by cPanel tooling, 0644 is convention.
if err := os.WriteFile(confPath, embeddedWHMConf, 0644); err != nil {
csmlog.Error("WHM AppConfig write failed", "path", confPath, "err", err)
} else if err := registerWHMPlugin(confPath); err != nil {
// Non-fatal: the conf is on disk, register_appconfig failure is
// logged so operators can fix it manually. Most common failure
// is register_appconfig not being in PATH (old cPanel versions).
csmlog.Warn("WHM plugin registration failed", "err", err)
} else {
csmlog.Info("WHM plugin registered with AppConfig")
}
}
// ModSecurity virtual patches. modsec2.user.conf is shared with
// operator-maintained rules, so the embedded rules go through
// checks.MergeModSecUserConfSection: this startup deploy only ever
// creates or rewrites CSM's marker-delimited section and every byte
// outside it is preserved verbatim.
for _, dst := range []string{
"/etc/apache2/conf.d/modsec/modsec2.user.conf",
"/usr/local/apache/conf/modsec2.user.conf",
} {
if _, err := os.Stat(filepath.Dir(dst)); err == nil {
// #nosec G304 -- dst iterates the literal ModSecurity config paths above.
existing, readErr := os.ReadFile(dst)
if readErr != nil && !os.IsNotExist(readErr) {
// Present but unreadable: rewriting blind could destroy
// operator rules, so leave the file alone.
} else if merged, changed := checks.MergeModSecUserConfSection(existing, embeddedModSec); changed {
// #nosec G306 G703 -- dst iterates the literal ModSecurity config
// paths above; Apache reads this config from a different user.
_ = os.WriteFile(dst, merged, 0644)
}
overridesFile := filepath.Join(filepath.Dir(dst), "modsec2.csm-overrides.conf")
modsec.EnsureOverridesInclude(dst, overridesFile)
break
}
}
}
// registerWHMPlugin runs cPanel's register_appconfig helper to add the CSM
// plugin to the WHM sidebar. WHM maintains a cached registration database
// separate from the /var/cpanel/apps/ conf files; without running this
// helper, the plugin file exists on disk but the menu never shows it.
//
// Idempotent: re-running against an already-registered plugin just updates
// the entry. Non-fatal on failure — deployment continues and the operator
// can rerun manually.
func registerWHMPlugin(confPath string) error {
bin := "/usr/local/cpanel/bin/register_appconfig"
if _, err := os.Stat(bin); err != nil {
return fmt.Errorf("register_appconfig not found at %s: %w", bin, err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// #nosec G204 -- bin is the fixed cPanel path validated by os.Stat above;
// confPath was just written by deployConfigs from an embedded constant.
cmd := exec.CommandContext(ctx, bin, confPath)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%s %s: %w: %s", bin, confPath, err, strings.TrimSpace(string(out)))
}
return nil
}
// watchdogInterval keeps a safety margin under the interval systemd expects,
// and never pings faster than every ten seconds: a unit configured with a very
// short WatchdogSec would otherwise spend the daemon's time on keepalives.
func watchdogInterval(timeout time.Duration) time.Duration {
interval := timeout / 2
if interval < 10*time.Second {
interval = 10 * time.Second
}
return interval
}
// watchdogNotifier sends systemd watchdog keepalives on its own ticker.
// Runs at half the WatchdogSec interval so there's always margin.
// Completely independent of scan goroutines — never blocks.
func (d *Daemon) watchdogNotifier() {
defer d.wg.Done()
timeout, configured := sdnotify.WatchdogTimeout()
if !configured || !sdnotify.Enabled() {
return // watchdog not configured, or nothing to notify
}
interval := watchdogInterval(timeout)
csmlog.Info("systemd watchdog active", "interval", interval.String())
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
if _, err := sdnotify.Watchdog(); err != nil {
csmlog.Warn("systemd watchdog notify failed", "err", err)
}
}
}
}
func ts() string {
return time.Now().Format("2006-01-02 15:04:05")
}
func orUnknown(v string) string {
if v == "" {
return "unknown"
}
return v
}
func orNone(v string) string {
if v == "" {
return "none"
}
return v
}
// countAttachedWatchers returns how many watchers are currently attached
// (value == true). Used for the systemd one-line status string.
func countAttachedWatchers(statuses map[string]bool) int {
n := 0
for _, attached := range statuses {
if attached {
n++
}
}
return n
}
package daemon
import (
"context"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/queuehealth"
)
// docrootExecQueueSize bounds the process starts waiting for inspection.
const docrootExecQueueSize = 1024
// docrootExecRefreshInterval is how often the document-root set is resolved
// again, so added and removed accounts are picked up without a restart.
const docrootExecRefreshInterval = 5 * time.Minute
// inspectDocrootExec is indirected so tests can observe what the worker asks.
var inspectDocrootExec = checks.InspectDocrootExec
// docrootExecMonitor judges process starts whose program may live in a
// document root. The exec event loop only filters and queues; reading and
// scanning a program happens on the monitor's own goroutine, so a slow scan
// never holds up other exec events. A start that does not fit the queue is
// counted in its health row, which reports the loss like any other
// protection queue. The program file is still read by the scheduled deep
// scan, and a process that keeps running is judged by the periodic check.
type docrootExecMonitor struct {
cfg *config.Config
alertCh chan<- alert.Finding
roots atomic.Pointer[checks.DocrootIndex]
queue *queuehealth.Channel[ExecEvent]
}
func newDocrootExecMonitor(cfg *config.Config, alertCh chan<- alert.Finding) *docrootExecMonitor {
m := &docrootExecMonitor{
cfg: cfg,
alertCh: alertCh,
queue: queuehealth.NewChannel[ExecEvent](docrootExecQueueSize, time.Minute),
}
m.refreshRoots()
return m
}
func (m *docrootExecMonitor) currentCfg() *config.Config {
if cfg := config.Active(); cfg != nil {
return cfg
}
return m.cfg
}
func (m *docrootExecMonitor) refreshRoots() {
m.roots.Store(checks.NewDocrootIndex(checks.ResolveWebRoots(m.currentCfg())))
}
// observe runs on the exec event loop and does no I/O. Even an absolute
// start name can be an alias into a document root; only /proc can resolve it.
func (m *docrootExecMonitor) observe(ev ExecEvent) {
if ev.UID == 0 {
return
}
m.queue.TrySend(ev)
}
// queueStatuses publishes the inspection queue beside the exec reader's rows.
func (m *docrootExecMonitor) queueStatuses(now time.Time) map[string]queuehealth.Status {
return map[string]queuehealth.Status{"docroot": m.queue.Snapshot(now)}
}
// run inspects queued starts until ctx ends. attach adds process context to a
// finding before it is published.
func (m *docrootExecMonitor) run(ctx context.Context, attach func(*alert.Finding, ExecEvent)) {
refresh := time.NewTicker(docrootExecRefreshInterval)
defer refresh.Stop()
for {
if ctx.Err() != nil {
return
}
select {
case <-ctx.Done():
return
case <-refresh.C:
m.refreshRoots()
case work := <-m.queue.Items():
if ctx.Err() != nil {
work.Ticket.Reject(time.Now())
return
}
work.Process(func(ev ExecEvent) {
f := inspectDocrootExec(m.currentCfg(), int(ev.PID), int(ev.UID), ev.Filename, ev.ParentComm, m.roots.Load())
if f == nil {
return
}
if ctx.Err() != nil {
m.queue.Lose(1)
return
}
attach(f, ev)
if !alert.TryEnqueue(m.alertCh, *f) {
csmlog.Warn("docroot exec: alert channel full, dropping finding")
}
})
}
}
}
// start runs the inspection worker until the returned stop is called. The
// caller stops calling observe before it calls stop; stop then waits for the
// worker, closes the queue and discards the starts still waiting in it.
func (m *docrootExecMonitor) start(ctx context.Context, attach func(*alert.Finding, ExecEvent)) (stop func()) {
ctx, cancel := context.WithCancel(ctx)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
m.run(ctx, attach)
}()
return func() {
cancel()
wg.Wait()
m.queue.Close()
m.queue.DiscardPending()
}
}
package daemon
import (
"container/list"
"slices"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/checks"
)
const (
// dovecotSessionVerdictTTL bounds the gap between a session's last
// verdict and its login summary, by log time.
dovecotSessionVerdictTTL = 5 * time.Minute
// maxDovecotSessionVerdicts caps the sessions held. Eviction loses what a
// session held, so a flood mostly withholds evidence. A lost answer can
// matter only if the flood lands between two answers to one request.
maxDovecotSessionVerdicts = 4096
// maxDovecotSessionNames caps the names one session keeps verdicts for.
maxDovecotSessionNames = 4
)
type dovecotSessionName struct {
name string
exist mailboxExistence
at time.Time
}
type dovecotSession struct {
id string
ip string
at time.Time
spoiled bool
names []dovecotSessionName
}
// dovecotSessionVerdicts keeps what Dovecot's passdb verdicts said about the
// names tried in a login session until the session's login summary claims
// them. One verdict is one database's answer, so it stays here and reaches no
// tracker on its own. A session no summary claims stays until the cap evicts
// it. mu is a leaf: nothing here calls a tracker.
type dovecotSessionVerdicts struct {
mu sync.Mutex
sessions map[string]*list.Element
// order holds one *dovecotSession per entry, least recently noted first,
// so eviction at the cap needs no scan.
order *list.List
}
func newDovecotSessionVerdicts() *dovecotSessionVerdicts {
return &dovecotSessionVerdicts{sessions: make(map[string]*list.Element), order: list.New()}
}
// Note keeps one verdict. A verdict that names no mailbox can pair with no
// login summary and adds nothing, so a flood of them costs nothing. Callers
// filter infra, private and loopback addresses first.
func (v *dovecotSessionVerdicts) Note(session, ip, account string, exist mailboxExistence, at time.Time) {
name := normalizeSprayMailbox(account)
if name == "" {
return
}
v.mu.Lock()
defer v.mu.Unlock()
elem := v.sessions[session]
if elem != nil && at.Sub(elem.Value.(*dovecotSession).at) > dovecotSessionVerdictTTL {
v.remove(elem)
elem = nil
}
if elem == nil {
elem = v.add(session, ip)
}
s := elem.Value.(*dovecotSession)
// A session belongs to one client, and a spoiled one stays spoiled.
if s.spoiled || s.ip != ip {
return
}
if at.After(s.at) {
s.at = at
}
v.order.MoveToBack(elem)
s.note(name, exist, at)
}
// Spoil withholds a session whose request hit an auth error: a later database
// failed, so a rejection before or after it is not the request's answer. The
// tombstone stands in for the session under the same cap and TTL, so a later
// verdict cannot restore it. Client text can name other sessions here, which
// only withholds their verdicts.
func (v *dovecotSessionVerdicts) Spoil(session string, at time.Time) {
v.mu.Lock()
defer v.mu.Unlock()
elem := v.sessions[session]
if elem == nil {
elem = v.add(session, "")
}
s := elem.Value.(*dovecotSession)
if at.After(s.at) {
s.at = at
}
s.spoiled, s.names = true, nil
v.order.MoveToBack(elem)
}
// Take claims the session and returns what its verdicts said about account:
// Unknown when none named it, when ip is not the client that got them, or
// when they expired or were spoiled. withheld reports a session spoiled within
// the TTL, as opposed to an absent verdict: a mailbox lookup must not undo an
// auth-error veto.
func (v *dovecotSessionVerdicts) Take(session, ip, account string, at time.Time) (exist mailboxExistence, withheld bool) {
v.mu.Lock()
defer v.mu.Unlock()
elem, tracked := v.sessions[session]
if !tracked {
return mailboxExistenceUnknown, false
}
s := v.remove(elem)
if s.spoiled && at.Sub(s.at) <= dovecotSessionVerdictTTL {
return mailboxExistenceUnknown, true
}
if s.ip != ip || at.Before(s.at) || at.Sub(s.at) > dovecotSessionVerdictTTL {
return mailboxExistenceUnknown, false
}
name := normalizeSprayMailbox(account)
for _, n := range s.names {
if n.name == name && !at.Before(n.at) && at.Sub(n.at) <= dovecotSessionVerdictTTL {
return n.exist, false
}
}
return mailboxExistenceUnknown, false
}
// add starts an entry, evicting the least recently noted at the cap. Caller
// holds v.mu.
func (v *dovecotSessionVerdicts) add(session, ip string) *list.Element {
if v.order.Len() >= maxDovecotSessionVerdicts {
v.remove(v.order.Front())
}
session = strings.Clone(session)
elem := v.order.PushBack(&dovecotSession{id: session, ip: ip})
v.sessions[session] = elem
return elem
}
// remove drops one session. Caller holds v.mu.
func (v *dovecotSessionVerdicts) remove(elem *list.Element) *dovecotSession {
s := v.order.Remove(elem).(*dovecotSession)
delete(v.sessions, s.id)
return s
}
func (s *dovecotSession) note(name string, exist mailboxExistence, at time.Time) {
for i := range s.names {
if s.names[i].name == name {
n := s.names[i]
if at.Sub(n.at) <= dovecotSessionVerdictTTL {
exist = combineMailboxExistence(n.exist, exist)
}
if n.at.After(at) {
at = n.at
}
s.names[i] = dovecotSessionName{name: n.name, exist: exist, at: at}
return
}
}
// The login summary names the last attempt, so the oldest name goes.
if len(s.names) >= maxDovecotSessionNames {
if s.names[0].exist != mailboxMissing {
s.spoiled, s.names = true, nil
return
}
s.names = slices.Delete(s.names, 0, 1)
}
s.names = append(s.names, dovecotSessionName{name: strings.Clone(name), exist: exist, at: at})
}
// dovecotLoginSession reads the session id of a login summary. user= comes
// first and is client text. A session must be a complete field after the
// server's address fields, not a substring of the user or another field.
func dovecotLoginSession(line string) string {
_, fields := checks.DovecotLoginFields(line)
idx := strings.LastIndex(fields, ", session=<")
if idx < 0 || checks.DovecotRemoteIP(fields[:idx]) == "" {
return ""
}
field, _, _ := strings.Cut(fields[idx+len(", session="):], ", ")
session, ok := dovecotSessionField(field)
if !ok {
return ""
}
return session
}
package daemon
import (
"net"
"strings"
)
// dovecotRejectionVerdicts end a Dovecot auth_verbose line in which one
// password database rejected the credentials. The PAM form is what passdb-pam
// logs for PAM_AUTH_ERR, which a missing user and a wrong password share.
var dovecotRejectionVerdicts = []struct {
suffix string
exist mailboxExistence
}{
{"): Password mismatch", mailboxExists},
{"): unknown user", mailboxMissing},
{"): pam_authenticate() failed: Authentication failure (Password mismatch?)", mailboxExistenceUnknown},
}
// dovecotSessionlessRejection parses a rejection verdict for a request with
// no login session, which is how an MTA such as Postfix asks Dovecot to check
// SMTP AUTH:
//
// auth: sql(alice@example.net,203.0.113.5): Password mismatch
//
// One verdict is one database's answer, not the request's result: a later
// database can still accept the login or fail internally. Callers pair it
// with the MTA's own failure line for the same client.
//
// The client name is attacker text before the fields Dovecot appends, so the
// address and master flag are read from the right; a login session marker
// there never parses as an address. A logged password follows the verdict,
// so lines whose fields hold a second "): " are refused: syslog may have cut
// the line right after a verdict forged inside the password, and Dovecot's
// default name characters never allow that sequence.
func dovecotSessionlessRejection(line string) (ip, account string, exist mailboxExistence, ok bool) {
fields, exist, ok := dovecotRejectionFields(line)
if !ok {
return "", "", mailboxExistenceUnknown, false
}
return dovecotRejectionClient(fields, exist)
}
// dovecotSessionRejection parses a rejection verdict for a request with a
// login session, which is how the IMAP, POP3 and ManageSieve login processes
// ask Dovecot to check a login:
//
// auth: passwd-file(alice@example.net,203.0.113.5,<AbCd1234>): unknown user
//
// Dovecot's fields are user,ip[,master][,<session>]. Like the sessionless
// form, one verdict is one database's answer; callers pair it with the login
// summary of the same session, so the same fields are read from the right
// and the same forged verdicts refused.
func dovecotSessionRejection(line string) (session, ip, account string, exist mailboxExistence, ok bool) {
fields, exist, ok := dovecotRejectionFields(line)
if !ok {
return "", "", "", mailboxExistenceUnknown, false
}
comma := strings.LastIndexByte(fields, ',')
if comma < 0 {
return "", "", "", mailboxExistenceUnknown, false
}
session, ok = dovecotSessionField(fields[comma+1:])
if !ok {
return "", "", "", mailboxExistenceUnknown, false
}
ip, account, exist, ok = dovecotRejectionClient(fields[:comma], exist)
if !ok {
return "", "", "", mailboxExistenceUnknown, false
}
return session, ip, account, exist, true
}
// dovecotSessionField reads a <session> request field.
func dovecotSessionField(field string) (session string, ok bool) {
session, opened := strings.CutPrefix(field, "<")
session, closed := strings.CutSuffix(session, ">")
// Dovecot's LOGIN_MAX_SESSION_ID_LEN also bounds forwarded identifiers.
if !opened || !closed || session == "" || len(session) > 64 {
return "", false
}
for _, c := range session {
if c <= ' ' || c >= 0x7f || strings.ContainsRune("<>,()", c) {
return "", false
}
}
return session, true
}
// dovecotRejectionFields returns the request fields of a rejection verdict,
// which end at the driver's closing parenthesis, and what the verdict says.
func dovecotRejectionFields(line string) (fields string, exist mailboxExistence, ok bool) {
seg := dovecotAuthSegment(dovecotServiceMessage(line))
open := strings.IndexByte(seg, '(')
if open <= 0 || !isDovecotAuthRequestName(seg[:open]) || strings.Contains(seg[:open], ": ") {
return "", mailboxExistenceUnknown, false
}
inner := seg[open+1:]
for _, verdict := range dovecotRejectionVerdicts {
if rest, cut := strings.CutSuffix(inner, verdict.suffix); cut && !strings.Contains(rest, "): ") {
return rest, verdict.exist, true
}
}
return "", mailboxExistenceUnknown, false
}
// dovecotRejectionClient reads user,ip[,master] from the right.
func dovecotRejectionClient(fields string, verdict mailboxExistence) (ip, account string, exist mailboxExistence, ok bool) {
fields = strings.TrimSuffix(fields, ",master")
comma := strings.LastIndexByte(fields, ',')
if comma < 0 {
return "", "", mailboxExistenceUnknown, false
}
addr := net.ParseIP(fields[comma+1:])
if addr == nil {
return "", "", mailboxExistenceUnknown, false
}
account, exist = fields[:comma], verdict
// Dovecot's unknown user is about the exact name it was asked. It says
// nothing about a mailbox whose key spells that name differently.
if exist == mailboxMissing && normalizeSprayMailbox(account) != account {
exist = mailboxExistenceUnknown
}
// Dovecot sanitizes unavailable or invalid name text to '?' and
// abbreviates long names with '...'. Neither identifies the mailbox.
if strings.TrimSpace(account) == "" || strings.Contains(account, "?") || strings.HasSuffix(account, "...") {
account, exist = "", mailboxExistenceUnknown
}
return addr.String(), account, exist, true
}
// dovecotAuthTroubleIPs returns every address named in the request fields of
// a Dovecot auth Error, Warning, Fatal or Panic line. A client name can add
// addresses here, which only withholds evidence for them.
func dovecotAuthTroubleIPs(line string) []string {
var ips []string
for _, field := range dovecotAuthTroubleFields(line) {
if addr := net.ParseIP(field); addr != nil {
ips = append(ips, addr.String())
}
}
return ips
}
// dovecotAuthTroubleSessions returns every session named in the same fields.
// Every one is read, not just the rightmost, because the line's detail text
// can hold fields too and the session Dovecot appended must not be missed. A
// client name can add sessions, which only withholds evidence for them.
func dovecotAuthTroubleSessions(line string) []string {
seg := dovecotAuthSegment(dovecotServiceMessage(line))
if !isDovecotAuthDiagnostic(seg) {
return nil
}
return dovecotSessionsInFields(seg)
}
// Only withholding uses these fields: client text may add a session but must
// never hide a complete one, including a field at the end of a cut log line.
func dovecotSessionsInFields(seg string) []string {
var sessions []string
for i := strings.IndexByte(seg, ','); i >= 0; {
seg = seg[i+1:]
end := strings.IndexAny(seg, ",)")
if end < 0 {
end = len(seg)
}
if session, ok := dovecotSessionField(seg[:end]); ok {
sessions = append(sessions, session)
}
i = strings.IndexByte(seg, ',')
}
return sessions
}
func isDovecotAuthDiagnostic(seg string) bool {
for _, severity := range []string{"Error: ", "Warning: ", "Fatal: ", "Panic: "} {
if strings.HasPrefix(seg, severity) {
return true
}
}
return false
}
// dovecotAuthTroubleFields returns the fields after each comma of an auth
// Error, Warning, Fatal or Panic line, up to the next comma or parenthesis.
func dovecotAuthTroubleFields(line string) []string {
seg := dovecotAuthSegment(dovecotServiceMessage(line))
rest := ""
for _, severity := range []string{"Error: ", "Warning: ", "Fatal: ", "Panic: "} {
if r, ok := strings.CutPrefix(seg, severity); ok {
rest = r
break
}
}
var fields []string
for i := strings.IndexByte(rest, ','); i >= 0; {
rest = rest[i+1:]
end := strings.IndexAny(rest, ",)")
if end < 0 {
break
}
fields = append(fields, rest[:end])
i = strings.IndexByte(rest, ',')
}
return fields
}
// Informational auth messages can contain arbitrary client names and logged
// passwords. Match their trusted prefix, not a verdict in the untrusted body:
// drivers use different verdicts, and syslog can truncate the body anywhere.
func isDovecotAuthRequestInfo(line string) bool {
seg := dovecotAuthSegment(dovecotServiceMessage(line))
if strings.HasPrefix(seg, "Debug: ") {
return true
}
open := strings.IndexByte(seg, '(')
return open > 0 && isDovecotAuthRequestName(seg[:open])
}
// Severity prefixes remain so the caller can preserve error diagnostics.
func dovecotAuthSegment(msg string) string {
if rest, ok := strings.CutPrefix(msg, "auth: "); ok {
return rest
}
rest, ok := strings.CutPrefix(msg, "auth-worker(")
if !ok {
return ""
}
pid, rest, ok := strings.Cut(rest, "): ")
if !ok || !isASCIIDigits(pid) {
return ""
}
// Dovecot 2.3 prefixes worker lines with the auth master connection:
// "conn unix:auth-worker (pid=N,uid=N): auth-worker<N>: driver(...)".
if conn, ok := strings.CutPrefix(rest, "conn unix:auth-worker (pid="); ok {
_, tail, found := strings.Cut(conn, "): auth-worker<")
if !found {
return ""
}
_, tail, found = strings.Cut(tail, ">: ")
if !found {
return ""
}
rest = tail
}
return rest
}
func isDovecotAuthRequestName(s string) bool {
// Dovecot substitutes the configured database name for the driver.
// Only leading severity markers distinguish diagnostics from that name.
for _, severity := range []string{"Error: ", "Warning: ", "Fatal: ", "Panic: "} {
if strings.HasPrefix(s, severity) {
return false
}
}
for i := 0; i < len(s); i++ {
c := s[i]
if c < ' ' || c == 0x7f {
return false
}
}
return s != ""
}
func isASCIIDigits(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] < '0' || s[i] > '9' {
return false
}
}
return s != ""
}
package daemon
import (
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/queuehealth"
)
// maxDropperProbeAttempts bounds how many times an inconclusive probe (a
// permission or transient I/O failure, not a confirmed absence) is retried
// before the candidate is dropped. Dropping is detection-coverage loss, so
// the engine counts it, but retrying forever would pin a permanently
// unreadable path into every probe tick.
const maxDropperProbeAttempts = 5
// dropperFSProber resolves a tracked candidate against the live filesystem.
// The real implementation is platform-specific (statx birth time, quarantine
// ledger); tests inject a fake so the engine's orchestration is verifiable
// without a kernel.
type dropperFSProber interface {
probe(c dropperCandidate) dropperProbe
}
// dropperEmitFn delivers a finding. In production it is bound to
// FileMonitor.sendFileFinding; tests capture the arguments.
type dropperEmitFn func(sev alert.Severity, check, msg, details, path, dedupKey string)
const dropperCheckName = "self_deleting_dropper_realtime"
type dropperEngineConfig struct {
ttl time.Duration
selfPID int32
// ignorePath reports whether a path is covered by
// suppressions.ignore_paths. Nil means nothing is suppressed.
ignorePath func(string) bool
}
// dropperEngine owns the tracker and drives the observe -> probe -> hold ->
// flush lifecycle. admit is called from the analyzer worker pool (via the
// tracker's own locking); probeStep is called only from the single probe
// goroutine, so its attempt bookkeeping needs no lock.
type dropperEngine struct {
tr *dropperTracker
ttl time.Duration
selfPID int32
emit dropperEmitFn
attempts map[queuehealth.Ticket]int
// ignorePath mirrors the suppression every other content check already
// honours. Applied at admit so a suppressed path never consumes tracker
// capacity a real candidate could have used.
ignorePath func(string) bool
}
func newDropperEngine(cfg dropperEngineConfig) *dropperEngine {
return &dropperEngine{
tr: newDropperTracker(cfg.ttl),
ttl: cfg.ttl,
selfPID: cfg.selfPID,
attempts: make(map[queuehealth.Ticket]int),
ignorePath: cfg.ignorePath,
}
}
// admit records a candidate if it passes the freshness/type gate. Returns
// false when the candidate was rejected by the gate or dropped by the
// tracker capacity bound (the caller surfaces the latter as coverage loss).
func (e *dropperEngine) admit(c dropperCandidate) bool {
if e.ignorePath != nil && e.ignorePath(c.Path) {
return false
}
if !shouldTrackDropper(c, e.selfPID, e.ttl) {
return false
}
// Inert files avoid consuming tracker capacity. ContentSuspicious wins: a
// realtime content or signature hit already found structure, and no later
// heuristic may demote that.
if !c.ContentSuspicious && !c.ContentMayExecute && !c.ContentUnsettled && dropperCandidateIsInert(c) {
return false
}
return e.tr.Observe(c)
}
// probeStep resolves every candidate whose TTL elapsed at probeNow, then
// flushes any held findings whose grace window closed at flushNow. Callers
// pass the same clock for both; the two parameters exist so tests can drive
// the grace window independently of the TTL.
func (e *dropperEngine) probeStep(probeNow time.Time, prober dropperFSProber, flushNow time.Time) {
for _, c := range e.tr.Due(probeNow) {
// The work ticket owns the attempt budget across retries.
key := c.ticket
if e.ignorePath != nil && e.ignorePath(c.Path) {
delete(e.attempts, key)
c.ticket.Finish(e.tr.now())
continue
}
verdict := assessDropper(c, prober.probe(c))
if verdict == dropperInconclusive {
if e.attempts[key]+1 >= maxDropperProbeAttempts {
delete(e.attempts, key)
c.ticket.Reject(e.tr.now())
continue
}
attempts := e.attempts[key] + 1
delete(e.attempts, key)
if retained, ok := e.tr.Retry(c); ok {
e.attempts[retained] = max(e.attempts[retained], attempts)
}
continue
}
delete(e.attempts, key)
c.ticket.Finish(e.tr.now())
e.tr.HoldGone(c, verdict, flushNow)
}
for _, f := range e.tr.FlushDue(flushNow) {
e.flushFinding(f)
}
}
func (e *dropperEngine) flushFinding(f dropperFinding) {
defer func() {
for _, item := range f.Items {
item.ticket.Finish(e.tr.now())
}
}()
items := make([]dropperGone, 0, len(f.Items))
for _, item := range f.Items {
if e.ignorePath == nil || !e.ignorePath(item.Cand.Path) {
items = append(items, item)
}
}
// Suppress before deciding burst severity. Otherwise excluded files
// can turn a remaining solitary dropper into a lower-severity burst.
for _, grouped := range groupDropperFindings(f.Docroot, items) {
e.emitFinding(grouped)
}
}
func (e *dropperEngine) emitFinding(f dropperFinding) {
if e.emit != nil {
sev, msg, details, path, dedupKey := dropperAlertParams(f)
e.emit(sev, dropperCheckName, msg, details, path, dedupKey)
}
}
package daemon
import (
"strings"
"github.com/pidginhost/csm/internal/checks"
)
func dropperCandidateIsInert(c dropperCandidate) bool {
// Executable files may be interpreted by a shell rather than PHP.
// PHP comments do not prove that such a script has no commands.
if c.Mode&0o111 != 0 {
return c.Size >= 0 && c.Size <= int64(len(c.Head)) && strings.Trim(string(c.Head), " \t\n") == ""
}
if dropperContentIsInert(c.Head, c.Size) {
return true
}
// A file whose first statement halts the interpreter carries data, not
// code, however large the tail is. Plugins keep state and WAF data in
// .php files of that shape and rewrite them constantly.
//
// Shift-based source encodings cannot reach this shape the way they reach
// a comment: the accepted bytes are the opening tag, PHP whitespace, one
// terminator keyword and a plain-ASCII literal with every encoding-shift
// byte rejected, so an ASCII-transparent encoding leaves them unchanged
// and a non-transparent one never matches the raw opening tag. A whole-file
// transport encoding is the exception: under a BASE64 source encoding this
// header is discarded as padding and an encoded tail becomes the program.
// Refuse the exemption when the tail could be that, which costs at most a
// plugin state file staying a candidate.
end, ok := checks.PHPTerminatesImmediatelyAt(c.Head)
return ok && !dropperTailCouldDecodeToSource(c.Head[end:])
}
// dropperTailCouldDecodeToSource reports whether the unreachable tail is made
// only of transport-encoding alphabet, which a source-encoding conversion
// could turn back into PHP. Real data files carry punctuation or text that no
// such alphabet contains.
func dropperTailCouldDecodeToSource(tail []byte) bool {
digits := 0
for _, b := range tail {
switch {
case b == ' ' || b == '\t' || b == '\n' || b == '\r':
case (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || (b >= '0' && b <= '9') ||
b == '+' || b == '/' || b == '=' || b == '-' || b == '_':
digits++
default:
return false
}
}
// Eight alphabet characters carry the six bytes of a "<?php " opener.
return digits >= 8
}
// dropperContentIsInert only exempts complete blank content. PHP can decode
// source before tokenization, including transfer encodings such as Base64
// and quoted-printable. Even ASCII text that resembles a comment can contain
// statements after that conversion. Without the interpreter's effective
// encoding settings, a PHP comment parser cannot prove those files inert.
func dropperContentIsInert(head []byte, size int64) bool {
return size >= 0 && size <= int64(len(head)) && strings.Trim(string(head), " \t\r\n") == ""
}
package daemon
// dropperUploadExecutionProbes are the exact scripts Really Simple Security
// copies into the uploads directory, requests over HTTP to learn whether PHP
// runs there, and deletes again. The plugin renamed itself once, so both
// shipped versions of the comment are listed.
//
// Only exact bytes qualify. The script is code, and no parser has to judge
// it: any added statement, encoding change or trailing byte stops the match,
// and a fixed byte string cannot carry a payload an attacker chose under any
// source encoding.
var dropperUploadExecutionProbes = []string{
"<?php\n/**\n * Test file for Really Simple SSL to check if uploads directory has code execution permissions\n *\n */\n\necho \"RSSSL CODE EXECUTION MARKER\";\n",
"<?php\n/**\n * Test file for Really Simple Security to check if uploads directory has code execution permissions\n *\n */\n\necho \"RSSSL CODE EXECUTION MARKER\";\n",
}
// dropperCandidateIsKnownProbe reports whether the snapshot is, in full, a
// plugin's server capability test script.
func dropperCandidateIsKnownProbe(c dropperCandidate) bool {
if c.Mode&0o111 != 0 || c.Size != int64(len(c.Head)) {
return false
}
for _, probe := range dropperUploadExecutionProbes {
if string(c.Head) == probe {
return true
}
}
return false
}
// dropperCandidateIsHarmless reports whether the snapshot cannot be a dropper
// payload: it has no executable statement, or it is a known test script.
func dropperCandidateIsHarmless(c dropperCandidate) bool {
return dropperCandidateIsInert(c) || dropperCandidateIsKnownProbe(c)
}
//go:build linux
package daemon
import (
"bytes"
"crypto/md5" // #nosec G501 -- wordpress.org publishes MD5 digests for core files
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"hash"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/contenttype"
"github.com/pidginhost/csm/internal/wpcheck"
)
// dropperDigestMax bounds how many bytes the admission hash covers. A
// self-deleting dropper is small; this cap keeps a large legitimate PHP file
// from stalling the analyzer worker while still giving a full digest for
// realistic candidates. Files past the cap keep DigestKnown=false and rely on
// device/inode identity for rename matching.
const dropperDigestMax = 8 << 20
const dropperDigestChunk = 64 << 10
const dropperPHPHandlerCacheMax = 4096
const dropperPHPHandlerCacheTTL = 15 * time.Second
type dropperPHPHandlerCacheEntry struct {
generation uint64
overlay checks.PHPExecutionOverlay
loaded time.Time
}
// dropperProbeInterval derives the probe cadence from the tracking TTL so the
// loop reacts within a fraction of the window without busy-spinning.
func dropperProbeInterval(ttl time.Duration) time.Duration {
iv := ttl / 4
if iv < 5*time.Second {
return 5 * time.Second
}
if iv > time.Minute {
return time.Minute
}
return iv
}
func (fm *FileMonitor) initDropperDetector(cfg *config.Config) {
if cfg == nil || !cfg.Thresholds.DropperDetection {
return
}
ttl := time.Duration(cfg.Thresholds.DropperUnlinkTTLSec) * time.Second
if ttl <= 0 {
ttl = time.Duration(config.DefaultDropperUnlinkTTLSec) * time.Second
}
// #nosec G115 -- os.Getpid returns this process's PID, bounded by
// /proc/sys/kernel/pid_max (<= 2^22 on Linux), so it always fits in int32.
selfPID := int32(os.Getpid())
e := newDropperEngine(dropperEngineConfig{
ttl: ttl,
selfPID: selfPID,
ignorePath: func(p string) bool {
return checks.PathMatchesIgnore(p, fm.currentCfg().Suppressions.IgnorePaths)
},
})
e.emit = func(sev alert.Severity, check, msg, details, path, dedupKey string) {
fm.sendFileFinding(alert.Finding{Severity: sev, Check: check, Message: msg, Details: details, FilePath: path, DedupKey: dedupKey})
}
fm.dropper = e
fm.dropperQuarantines = newDropperQuarantineLedger(
ttl + time.Duration(maxDropperProbeAttempts)*dropperProbeInterval(ttl) + dropperGraceWindow,
)
fm.dropperHandlerCache = make(map[string]dropperPHPHandlerCacheEntry)
fm.setDropperDocroots(checks.ResolveWebRoots(cfg))
}
func (fm *FileMonitor) setDropperDocroots(docroots []string) {
fm.dropperDocroots.Store(checks.NewDocrootIndex(docroots))
}
func (fm *FileMonitor) dropperDocrootFor(path string) string {
return fm.dropperDocroots.Load().Lookup(path)
}
// isDropperInteresting admits paths that the normal content filter deliberately
// excludes but the dropper detector still needs: inherited .htaccess PHP
// handlers and regular files carrying an executable mode under a document
// root. It does not take ownership of fd.
func (fm *FileMonitor) isDropperInteresting(path string, fd int) (interesting, phpExecutable bool) {
if fm.dropper == nil {
return false, false
}
docroot := fm.dropperDocrootFor(path)
if docroot == "" {
return false, false
}
name := strings.ToLower(filepath.Base(path))
if contenttype.IsExecutablePHPName(name) {
return true, false
}
var st unix.Stat_t
if unix.Fstat(fd, &st) != nil || st.Mode&unix.S_IFMT != unix.S_IFREG {
return false, false
}
if st.Mode&0o111 != 0 {
return true, false
}
phpExecutable = fm.dropperPHPHandlerFor(docroot, filepath.Dir(path)).Executes(name)
return phpExecutable, phpExecutable
}
func (fm *FileMonitor) invalidateDropperPHPHandlerCache(path string) {
if fm.dropper != nil && strings.EqualFold(filepath.Base(path), ".htaccess") {
atomic.AddUint64(&fm.dropperHandlerGeneration, 1)
}
}
func (fm *FileMonitor) dropperPHPHandlerFor(docroot, dir string) checks.PHPExecutionOverlay {
key := docroot + "\x00" + dir
generation := atomic.LoadUint64(&fm.dropperHandlerGeneration)
fm.dropperHandlerMu.Lock()
entry, ok := fm.dropperHandlerCache[key]
fm.dropperHandlerMu.Unlock()
if ok && entry.generation == generation && time.Since(entry.loaded) < dropperPHPHandlerCacheTTL {
return entry.overlay
}
overlay := checks.ResolvePHPExecutionOverlay(docroot, dir)
// A .htaccess event that arrived while the filesystem snapshot was being
// rebuilt invalidates this result. Do not cache it; the next event retries.
if atomic.LoadUint64(&fm.dropperHandlerGeneration) != generation {
return checks.ResolvePHPExecutionOverlay(docroot, dir)
}
fm.dropperHandlerMu.Lock()
if fm.dropperHandlerCache == nil || len(fm.dropperHandlerCache) >= dropperPHPHandlerCacheMax {
fm.dropperHandlerCache = make(map[string]dropperPHPHandlerCacheEntry)
}
fm.dropperHandlerCache[key] = dropperPHPHandlerCacheEntry{
generation: generation,
overlay: overlay,
loaded: time.Now(),
}
fm.dropperHandlerMu.Unlock()
return overlay
}
// observeDropperCandidate snapshots an admitted write into the tracker. It
// runs on the analyzer worker while the event fd is still open. Fstat, statx,
// and Pread do not change the shared file offset, and this function never
// closes the event fd. The returned copy lets downstream content checks add a
// suspicious-content verdict without reading or hashing the file again.
func (fm *FileMonitor) observeDropperCandidate(event fileEvent, procInfo string) *dropperCandidate {
if fm.dropper == nil {
return nil
}
docroot := fm.dropperDocrootFor(event.path)
if docroot == "" {
return nil
}
var st unix.Stat_t
if err := unix.Fstat(event.fd, &st); err != nil {
return nil
}
c := dropperCandidate{
Path: event.path,
Docroot: docroot,
Observed: time.Now(),
Device: uint64(st.Dev),
Inode: st.Ino,
Size: st.Size,
UID: st.Uid,
Mode: uint32(st.Mode),
PID: event.pid,
ProcInfo: procInfo,
PHPExecutable: event.phpExecutable,
}
if birth, ok := statxBirthFromFD(event.fd); ok {
c.Birth = birth
c.BirthKnown = true
}
// Reject non-regular and non-executable names before allocating a head or
// hashing content. HTML, archives, and credential logs also reach the
// analyzer, but they are not dropper candidates.
name := strings.ToLower(filepath.Base(event.path))
if c.Mode&unix.S_IFMT != unix.S_IFREG ||
(!contenttype.IsExecutablePHPName(name) && !c.PHPExecutable && c.Mode&0o111 == 0) ||
c.PID == fm.dropper.selfPID {
return nil
}
if !shouldTrackDropper(c, fm.dropper.selfPID, fm.dropper.ttl) {
return nil
}
if parent, err := statDropperCandidateParent(c.Path, c.Device, c.Inode); err == nil {
c.Parent = parent
}
wpCopy := len(wpUpgradeCopyDestinations(c.Path, c.Docroot)) > 0
staged := wpUpgradeStagedPackageFile(c.Path, c.Docroot)
_, _, core := wpUpgradeCorePackageFile(c.Path, c.Docroot)
_, _, pluginCopy := dropperPluginCopySource(c.Path, c.Docroot)
// Staged files and plugin copies are later compared with an official or
// installed file, so they need a digest of the whole snapshot.
whole := staged || pluginCopy
read, limit := readFromFd, dropperTrackedHeadMax
// Oversized files still need a head for the executable content check,
// even though they cannot supply a complete digest.
if wpCopy || (whole && st.Size <= dropperDigestMax) {
read, limit = readCompleteFromFd, dropperDigestMax
}
// The data proof and retained head must share the opening stat. Separate
// head/body snapshots leave a gap in which a writer can replace a payload.
body, size, stable := readDropperSnapshot(event.fd, st, limit, read)
c.Head, c.Size = body, size
if len(c.Head) > dropperTrackedHeadMax {
c.Head = bytes.Clone(c.Head[:dropperTrackedHeadMax])
}
c.ContentUnsettled = !stable
if whole && body != nil && stable && int64(len(body)) == c.Size {
// Hash the retained snapshot rather than rereading an fd whose bytes
// may already have been replaced.
c.Digest, c.DigestKnown = sha256.Sum256(body), true
if core {
// #nosec G401 -- compared with the MD5 digests wordpress.org publishes
c.CoreMD5, c.CoreMD5Known = md5.Sum(body), true
}
if pluginCopy {
c.PluginRelease = fm.pluginReleaseOf(c)
}
}
// Copy exceptions must check every snapshot: a benign later close-write
// cannot erase an earlier payload. Blank snapshots carry no such evidence.
if wpCopy && (!stable || !dropperContentIsInert(c.Head, c.Size)) {
// Keep only the proof and hash, not a large translation body in each
// tracker entry. Both must describe the same complete snapshot.
if body != nil && stable && int64(len(body)) == c.Size {
c.Digest, c.DigestKnown = sha256.Sum256(body), true
if filepath.Base(c.Path) == "version-current.php" {
c.WPInstallData = checks.IsWPVersionDataBytesComplete(body, true)
if c.WPInstallData {
c.WPCoreRelease = fm.wpCoreReleaseOf(body)
}
} else {
c.WPInstallData = checks.IsWPTranslationCacheBytesComplete(body, true)
}
}
c.WPInstallUnsafe = !c.WPInstallData || c.Mode&0o111 != 0
} else if !wpCopy && !whole && atomicWriteRenameCandidate(c.Path) != "" {
c.Digest, c.DigestKnown = digestFromFD(event.fd, st.Size)
}
// A rewrite of a tracked file updates its entry even when the new bytes
// look inert, so admission's inert gate cannot drop the later snapshot.
if fm.dropper.tr.Refresh(c) {
return &c
}
fm.dropper.admit(c)
// Even an inert snapshot must reach the content pass: a signature hit
// can override the admission gate without another filesystem read.
return &c
}
// readDropperSnapshot reads bytes together with proof that nothing
// changed the file while they were read. Size from before the read cannot
// prove completeness if another writer changed the file in the meantime, even
// when the retained head is empty.
//
// Even a ctime-only change is uncertain: a writer can restore bytes and
// mtime, leaving the same evidence as a harmless unlink or rename. A later
// quiet read cannot establish what executed in that interval.
func readDropperSnapshot(fd int, before unix.Stat_t, limit int, read func(int, int) []byte) ([]byte, int64, bool) {
body := read(fd, limit)
var after unix.Stat_t
stable := unix.Fstat(fd, &after) == nil && sameReadSnapshot(before, after) && before.Mode == after.Mode
return body, before.Size, stable
}
func (fm *FileMonitor) newDropperFSProbe() *dropperFSProbe {
return &dropperFSProbe{quarantines: fm.dropperQuarantines, coreChecksums: fm.wpCache}
}
// wpCoreReleaseOf names the release a version-probe data snapshot declares.
// Asking the cache now starts a missing checksum download in the background,
// so the deletion probe normally finds it cached; the analyzer never waits.
func (fm *FileMonitor) wpCoreReleaseOf(body []byte) *wpcheck.Verification {
if fm.wpCache == nil {
return nil
}
version, locale, err := wpcheck.ParseVersionContent(body)
if err != nil {
return nil
}
// #nosec G401 -- compared with the MD5 digests wordpress.org publishes
sum := md5.Sum(body)
v := &wpcheck.Verification{
Kind: wpcheck.KindCore, Version: version, Locale: locale,
Rel: "wp-includes/version.php", Digest: hex.EncodeToString(sum[:]), Staged: true,
}
fm.wpCache.Verify(*v)
return v
}
// pluginReleaseOf names the release of the plugin a copy comes from, as the
// installed plugin declares it while the copy is written. Asking now starts a
// missing manifest download in the background, so the deletion probe normally
// finds it cached, and an update of the plugin later in the tracking window
// does not change the release the copy is compared with.
func (fm *FileMonitor) pluginReleaseOf(c dropperCandidate) *wpcheck.Verification {
root, pc, ok := dropperPluginCopySource(c.Path, c.Docroot)
if fm.wpCache == nil || !ok || !c.DigestKnown {
return nil
}
// The root pins the slug: official releases of some other plugin, whose
// author could be anyone, never vouch for this copy. The version header
// is account-writable, but it only picks which official release to use.
v := fm.wpCache.Describe(filepath.Join(root, filepath.FromSlash(pc.rel)))
if v.Kind != wpcheck.KindPlugin || v.Root != root || v.Version == "" {
return nil
}
v.Digest = hex.EncodeToString(c.Digest[:])
return &v
}
// dropperProbeLoop probes overdue candidates for deletion and flushes findings.
// It also refreshes the cached docroot set so account changes are picked up.
func (fm *FileMonitor) dropperProbeLoop() {
defer fm.wg.Done()
prober := fm.newDropperFSProbe()
ticker := time.NewTicker(dropperProbeInterval(fm.dropper.ttl))
defer ticker.Stop()
refresh := time.NewTicker(5 * time.Minute)
defer refresh.Stop()
for {
select {
case <-fm.stopCh:
return
case <-refresh.C:
fm.setDropperDocroots(checks.ResolveWebRoots(fm.currentCfg()))
case <-ticker.C:
now := time.Now()
fm.dropper.probeStep(now, prober, now)
fm.reportDropperOverflow()
}
}
}
// reportDropperOverflow surfaces only capacity losses that occurred since the
// previous report and limits a sustained storm to one warning per minute. The
// tracker counter is cumulative for diagnostics.
func (fm *FileMonitor) reportDropperOverflow() {
total := fm.dropper.tr.overflowDropped()
if total <= fm.dropperOverflowReported {
return
}
now := time.Now()
if !fm.lastDropperOverflowReport.IsZero() && now.Sub(fm.lastDropperOverflowReport) < time.Minute {
return
}
dropped := total - fm.dropperOverflowReported
fm.dropperOverflowReported = total
fm.lastDropperOverflowReport = now
fm.sendAlert(alert.Warning, "self_deleting_dropper_overflow",
"self-deleting-dropper tracker is full; some short-lived files were not tracked",
fmt.Sprintf("A create/delete storm exceeded the tracker capacity. %d candidate(s) were dropped since the prior report; those files are only covered by the next scheduled deep scan.", dropped))
}
// statxBirthFromFD returns the file birth time for an open fd when the
// filesystem records it (ext4, xfs, btrfs). Returns ok=false on filesystems
// without STATX_BTIME; those files cannot enter fresh-dropper tracking.
func statxBirthFromFD(fd int) (time.Time, bool) {
var stx unix.Statx_t
if err := unix.Statx(fd, "", unix.AT_EMPTY_PATH|unix.AT_SYMLINK_NOFOLLOW, unix.STATX_BTIME, &stx); err != nil {
return time.Time{}, false
}
if stx.Mask&unix.STATX_BTIME == 0 {
return time.Time{}, false
}
return time.Unix(stx.Btime.Sec, int64(stx.Btime.Nsec)), true
}
// digestFromFD hashes up to dropperDigestMax bytes of the open fd. Files larger
// than the cap return ok=false; rename matching then relies on device/inode
// identity, which still covers rename(2) within a filesystem.
func digestFromFD(fd int, size int64) ([32]byte, bool) {
if size < 0 || size > dropperDigestMax {
return [32]byte{}, false
}
h := sha256.New()
if !hashFDRange(h, fd, size) {
return [32]byte{}, false
}
var after unix.Stat_t
if err := unix.Fstat(fd, &after); err != nil || after.Size != size {
return [32]byte{}, false
}
var sum [sha256.Size]byte
copy(sum[:], h.Sum(nil))
return sum, true
}
func hashFDRange(h hash.Hash, fd int, size int64) bool {
if size == 0 {
return true
}
bufSize := int64(dropperDigestChunk)
if size < bufSize {
bufSize = size
}
buf := make([]byte, int(bufSize))
for offset := int64(0); offset < size; {
want := size - offset
if want > int64(len(buf)) {
want = int64(len(buf))
}
n, err := unix.Pread(fd, buf[:int(want)], offset)
if n > 0 {
_, _ = h.Write(buf[:n])
offset += int64(n)
}
if err != nil && !errors.Is(err, unix.EINTR) {
return false
}
if n == 0 && err == nil {
return false
}
}
return true
}
type dropperPathState struct {
file dropperFileState
mode uint32
}
// statPathToFileState opens path first, then derives identity, birth time,
// size, and an optional digest from that one fd. O_PATH is a fallback for
// symlinks or unreadable objects; it still provides stable identity without
// pretending a digest was available.
func statPathToFileState(path string, includeDigest bool) (dropperPathState, error) {
fd, err := unix.Open(path, unix.O_RDONLY|unix.O_NONBLOCK|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
readable := err == nil
if err != nil {
fd, err = unix.Open(path, unix.O_PATH|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
if err != nil {
return dropperPathState{}, err
}
}
defer func() { _ = unix.Close(fd) }()
return fileStateFromFD(fd, path, includeDigest && readable)
}
// fileStateFromFD describes the file an open descriptor names, under path.
func fileStateFromFD(fd int, path string, includeDigest bool) (dropperPathState, error) {
var st unix.Stat_t
if err := unix.Fstat(fd, &st); err != nil {
return dropperPathState{}, err
}
state := dropperPathState{
file: dropperFileState{
Path: path, Device: uint64(st.Dev), Inode: st.Ino, Size: st.Size,
IsRegular: st.Mode&unix.S_IFMT == unix.S_IFREG,
},
mode: uint32(st.Mode),
}
if birth, ok := statxBirthFromFD(fd); ok {
state.file.Birth = birth
state.file.BirthKnown = true
}
if includeDigest && st.Mode&unix.S_IFMT == unix.S_IFREG {
if digest, ok := digestFromFD(fd, st.Size); ok {
state.file.Digest = digest
state.file.DigestKnown = true
}
}
return state, nil
}
// openDropperDirNoSymlinks opens an absolute directory path one component at
// a time. Refusing symlinks anywhere in the chain is important: reopening a
// path through a retargeted ancestor could otherwise make an unchanged parent
// look replaced and turn attacker-controlled path churn into demotion evidence.
func openDropperDirNoSymlinks(path string) (int, error) {
if !filepath.IsAbs(path) || filepath.Clean(path) != path {
return -1, unix.EINVAL
}
flags := unix.O_PATH | unix.O_DIRECTORY | unix.O_CLOEXEC | unix.O_NOFOLLOW
fd, err := unix.Open(string(filepath.Separator), flags, 0)
if err != nil {
return -1, err
}
for _, component := range strings.Split(strings.TrimPrefix(path, string(filepath.Separator)), string(filepath.Separator)) {
if component == "" {
continue
}
next, openErr := unix.Openat(fd, component, flags, 0)
_ = unix.Close(fd)
if openErr != nil {
return -1, openErr
}
fd = next
}
return fd, nil
}
// statDropperFileNoSymlinks opens a regular file without following a symlink
// anywhere in its path and reports its size and full digest. A plugin file
// reached through a link could live outside the site, where nothing scans the
// bytes it is meant to vouch for.
func statDropperFileNoSymlinks(path string) (dropperFileState, error) {
return statDropperFileNoSymlinksWithDigest(path, digestFromFD)
}
func statDropperFileNoSymlinksWithDigest(path string, digest func(int, int64) ([32]byte, bool)) (dropperFileState, error) {
dir, err := openDropperDirNoSymlinks(filepath.Dir(path))
if err != nil {
return dropperFileState{}, err
}
defer func() { _ = unix.Close(dir) }()
fd, err := unix.Openat(dir, filepath.Base(path), unix.O_RDONLY|unix.O_NONBLOCK|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
if err != nil {
return dropperFileState{}, err
}
defer func() { _ = unix.Close(fd) }()
var st unix.Stat_t
if err := unix.Fstat(fd, &st); err != nil {
return dropperFileState{}, err
}
state := dropperFileState{
Path: path, Device: uint64(st.Dev), Inode: st.Ino, Size: st.Size,
IsRegular: st.Mode&unix.S_IFMT == unix.S_IFREG,
}
if state.IsRegular {
state.Digest, state.DigestKnown = digest(fd, st.Size)
// Size alone cannot show that these bytes still survive in the plugin:
// an account writer can replace them and restore mtime during hashing.
var after unix.Stat_t
state.DigestKnown = state.DigestKnown && unix.Fstat(fd, &after) == nil &&
sameReadSnapshot(st, after) && st.Mode == after.Mode
}
return state, nil
}
// statDropperParent snapshots a real parent directory without following any
// symlink in its path. Symlinked parents deliberately provide no removal
// evidence: a dangling or retargeted link does not prove that the directory
// which contained the event fd was removed.
func statDropperParent(path string) (dropperParentIdentity, error) {
fd, err := openDropperDirNoSymlinks(path)
if err != nil {
return dropperParentIdentity{}, err
}
defer func() { _ = unix.Close(fd) }()
return statDropperParentFD(fd)
}
// statDropperCandidateParent also proves that the candidate fd identity is
// still the entry in this parent. If the path was replaced while the event was
// being admitted, the unrelated successor directory cannot later supply
// removal evidence for the original candidate.
func statDropperCandidateParent(path string, device, inode uint64) (dropperParentIdentity, error) {
fd, err := openDropperDirNoSymlinks(filepath.Dir(path))
if err != nil {
return dropperParentIdentity{}, err
}
defer func() { _ = unix.Close(fd) }()
var child unix.Stat_t
if err := unix.Fstatat(fd, filepath.Base(path), &child, unix.AT_SYMLINK_NOFOLLOW); err != nil {
return dropperParentIdentity{}, err
}
if uint64(child.Dev) != device || child.Ino != inode {
return dropperParentIdentity{}, unix.ESTALE
}
return statDropperParentFD(fd)
}
func statDropperParentFD(fd int) (dropperParentIdentity, error) {
var st unix.Stat_t
if err := unix.Fstat(fd, &st); err != nil {
return dropperParentIdentity{}, err
}
if st.Mode&unix.S_IFMT != unix.S_IFDIR {
return dropperParentIdentity{}, unix.ENOTDIR
}
identity := dropperParentIdentity{Device: uint64(st.Dev), Inode: st.Ino}
if birth, ok := statxBirthFromFD(fd); ok {
identity.BirthKnown = true
identity.BirthNanos = birth.UnixNano()
}
return identity, nil
}
const dropperQuarantineLedgerMax = 4096
type dropperQuarantineRecord struct {
state dropperFileState
expires time.Time
}
type dropperQuarantineLedger struct {
mu sync.Mutex
keepFor time.Duration
count int
byPath map[string][]dropperQuarantineRecord
}
func newDropperQuarantineLedger(keepFor time.Duration) *dropperQuarantineLedger {
return &dropperQuarantineLedger{
keepFor: keepFor,
byPath: make(map[string][]dropperQuarantineRecord),
}
}
func (l *dropperQuarantineLedger) record(originalPath string, state dropperFileState, now time.Time) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
l.pruneLocked(now)
if l.count >= dropperQuarantineLedgerMax {
return
}
l.byPath[originalPath] = append(l.byPath[originalPath], dropperQuarantineRecord{
state: state, expires: now.Add(l.keepFor),
})
l.count++
}
func (l *dropperQuarantineLedger) matched(c dropperCandidate, now time.Time) bool {
if l == nil {
return false
}
l.mu.Lock()
defer l.mu.Unlock()
l.pruneLocked(now)
records := l.byPath[c.Path]
for i, record := range records {
// Require the same inode generation. Digest-only matching would let a
// later identical drop at the same path consume an older quarantine
// record and evade detection.
if !dropperSameIdentity(c, record.state) {
continue
}
records = append(records[:i], records[i+1:]...)
if len(records) == 0 {
delete(l.byPath, c.Path)
} else {
l.byPath[c.Path] = records
}
l.count--
return true
}
return false
}
func (l *dropperQuarantineLedger) pruneLocked(now time.Time) {
for path, records := range l.byPath {
keep := records[:0]
for _, record := range records {
if now.Before(record.expires) {
keep = append(keep, record)
} else {
l.count--
}
}
if len(keep) == 0 {
delete(l.byPath, path)
} else {
l.byPath[path] = keep
}
}
}
// recordDropperQuarantine remembers a file CSM quarantined so the dropper probe
// does not report its removal. Quarantine writes a copy, so the identity comes
// from the detector's descriptor of the original, never from the copy.
func (fm *FileMonitor) recordDropperQuarantine(originalPath string, fd int) {
if fm.dropperQuarantines == nil || fd < 0 {
return
}
state, err := fileStateFromFD(fd, originalPath, false)
if err != nil || state.mode&unix.S_IFMT != unix.S_IFREG {
return
}
fm.dropperQuarantines.record(originalPath, state.file, time.Now())
}
//go:build linux
package daemon
import (
"encoding/hex"
"errors"
"path/filepath"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/wpcheck"
)
// dropperFSProbe resolves a tracked candidate against the live filesystem for
// the probe loop. It distinguishes a confirmed deletion (ENOENT) from a
// permission or transient I/O failure so the engine can requeue the latter
// rather than reporting a phantom self-delete.
type dropperFSProbe struct {
quarantines *dropperQuarantineLedger
// coreChecksums compares a version probe with the official file of the
// release it declares, and a core package file with the release now
// installed. Nil leaves every such candidate unverified.
coreChecksums interface {
Describe(path string) wpcheck.Verification
Verify(wpcheck.Verification) wpcheck.Verdict
}
}
func (p dropperFSProbe) probe(c dropperCandidate) dropperProbe {
state, err := statPathToFileState(c.Path, false)
switch {
case err == nil:
// Every identity field came from one open fd. A matching identity means
// the tracked file survived; a different one means it was replaced.
result := dropperProbe{Conclusive: true, AtPath: &state.file}
if !dropperSameIdentity(c, state.file) {
// A new file at this path does not undo CSM's removal of the
// tracked inode. Its own candidate cannot consume this record.
if p.quarantines.matched(c, time.Now()) {
result.QuarantineMatched = true
return result
}
result.PluginCopy = p.pluginCopy(c)
}
return result
case !errors.Is(err, unix.ENOENT):
// Permission or I/O error: cannot prove deletion. Inconclusive.
return dropperProbe{Conclusive: false}
}
// Confirmed absent. Attribute it to an install rename or a removed docroot
// before treating it as a self-delete.
result := dropperProbe{Conclusive: true}
if p.quarantines.matched(c, time.Now()) {
result.QuarantineMatched = true
return result
}
if target, ts, ok, renameErr := dropperFindRenameTarget(c); renameErr != nil {
return dropperProbe{Conclusive: false}
} else if ok {
result.RenamedTo = target
result.RenameTarget = &ts
}
if c.WPCoreRelease != nil && p.coreChecksums != nil {
result.OfficialWPCoreFile = p.coreChecksums.Verify(*c.WPCoreRelease) == wpcheck.VerdictVerified
}
result.OfficialWPCorePackageFile = p.officialCorePackageFile(c)
result.PluginCopy = p.pluginCopy(c)
var dst unix.Stat_t
if derr := unix.Stat(c.Docroot, &dst); derr != nil && errors.Is(derr, unix.ENOENT) {
result.DocrootRemoved = true
}
if c.Parent.known() {
if current, perr := statDropperParent(filepath.Dir(c.Path)); perr == nil {
result.ParentRemoved = dropperParentChanged(c.Parent, current)
} else if errors.Is(perr, unix.ENOENT) {
result.ParentRemoved = true
}
}
return result
}
// officialCorePackageFile compares a vanished file of an unpacked core
// release with the release installed at its WordPress root. The staged tree,
// and with it the staged version header, is gone by the time of the probe.
// A completed update has installed that release, so its header names the
// manifest to check. An aborted one leaves the old release, whose manifest
// does not match the new bytes, and the candidate stays reported.
func (p dropperFSProbe) officialCorePackageFile(c dropperCandidate) bool {
if p.coreChecksums == nil || !c.CoreMD5Known {
return false
}
wpRoot, rel, ok := wpUpgradeCorePackageFile(c.Path, c.Docroot)
if !ok {
return false
}
v := p.coreChecksums.Describe(filepath.Join(wpRoot, "wp-includes", "version.php"))
if v.Kind != wpcheck.KindCore || v.Root != wpRoot || v.Version == "" {
return false
}
v.Rel, v.Digest = rel, hex.EncodeToString(c.CoreMD5[:])
return p.coreChecksums.Verify(v) == wpcheck.VerdictVerified
}
// pluginCopy compares a vanished plugin copy with the plugin it comes from.
// The wordpress.org manifest of the release named when the copy was written
// decides when it is cached: a match means the copy ran nothing an attacker
// chose, a mismatch means the bytes are not that release's file, whatever the
// installed file now holds. Without a manifest, bytes equal to the installed
// file only show that the content survives on disk, not who wrote it.
func (p dropperFSProbe) pluginCopy(c dropperCandidate) dropperPluginCopyEvidence {
root, pc, ok := dropperPluginCopySource(c.Path, c.Docroot)
if !ok || !c.DigestKnown {
return dropperPluginCopyUnproven
}
if c.PluginRelease != nil && p.coreChecksums != nil {
switch p.coreChecksums.Verify(*c.PluginRelease) {
case wpcheck.VerdictVerified:
return dropperPluginCopyOfficial
case wpcheck.VerdictMismatch:
return dropperPluginCopyUnproven
}
}
installed, err := statDropperFileNoSymlinks(filepath.Join(root, filepath.FromSlash(pc.rel)))
if err != nil || !installed.IsRegular || !installed.DigestKnown ||
installed.Size != c.Size || installed.Digest != c.Digest {
return dropperPluginCopyUnproven
}
return dropperPluginCopyInstalled
}
// dropperFindRenameTarget snapshots the install destinations WordPress and the
// atomic-write helper may move or copy a staged file to. A matching destination
// wins; otherwise the first regular destination is returned as replacement
// evidence.
func dropperFindRenameTarget(c dropperCandidate) (string, dropperFileState, bool, error) {
return dropperFindRenameTargetWithStat(c, statPathToFileState)
}
func dropperFindRenameTargetWithStat(c dropperCandidate, stat func(string, bool) (dropperPathState, error)) (string, dropperFileState, bool, error) {
targets := wpUpgradeInstallDestinations(c.Path, c.Docroot)
if atomic := atomicWriteRenameCandidate(c.Path); atomic != "" {
targets = append(targets, atomic)
}
var firstTarget string
var firstState dropperFileState
var transientErr error
for _, target := range targets {
if !dropperRenameTargetAllowed(c, target) {
continue
}
state, err := stat(target, false)
// An unreachable destination cannot prove a benign move. Treating it
// as transient would let a broken install path exhaust probe retries.
if dropperInstallPathUnreachable(err) {
continue
}
if err != nil {
transientErr = err
continue
}
if state.mode&unix.S_IFMT != unix.S_IFREG {
continue
}
if dropperSameIdentity(c, state.file) {
return target, state.file, true, nil
}
if c.DigestKnown && c.Size == state.file.Size {
state, err = stat(target, true)
if dropperInstallPathUnreachable(err) {
continue
}
if err != nil {
transientErr = err
continue
}
if dropperRenameMatch(c, state.file) {
return target, state.file, true, nil
}
}
if firstTarget == "" {
firstTarget, firstState = target, state.file
}
}
if transientErr != nil {
return "", dropperFileState{}, false, transientErr
}
if firstTarget != "" {
return firstTarget, firstState, true, nil
}
return "", dropperFileState{}, false, nil
}
func dropperInstallPathUnreachable(err error) bool {
return errors.Is(err, unix.ENOENT) || errors.Is(err, unix.ENOTDIR) || errors.Is(err, unix.ELOOP)
}
package daemon
import (
"bytes"
"crypto/sha256"
"fmt"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/contenttype"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/wpcheck"
)
// dropperCandidate captures the fstat/read state of a file at close-write
// time. The fanotify fd is not retained: everything needed for the later
// TTL probe is copied here while the fd is still open, so probe decisions
// never race the attacker deleting or swapping the file.
type dropperCandidate struct {
Path string
Docroot string
Observed time.Time
Birth time.Time
BirthKnown bool
Device uint64
Inode uint64
Size int64
UID uint32
Mode uint32
PID int32
ProcInfo string
// PHPExecutable is set by the Linux analyzer when an inherited
// .htaccess handler makes a non-standard extension executable as PHP.
PHPExecutable bool
// ContentSuspicious prevents FP heuristics from demoting a file whose
// realtime content/signature pass already found malicious structure.
ContentSuspicious bool
// Sticky across refreshes: truncating a previously executable snapshot
// must not turn its later deletion into a harmless empty guard.
ContentMayExecute bool
// ContentUnsettled is sticky: a later harmless snapshot cannot rule out
// code that ran while an earlier read raced a writer.
ContentUnsettled bool
Digest [32]byte
DigestKnown bool
// CoreMD5 is the same snapshot hashed the way wordpress.org publishes
// core checksums. Only files inside an unpacked core release carry it.
CoreMD5 [16]byte
CoreMD5Known bool
// ContentRewritten is sticky: a file whose written content changed
// cannot prove from its last bytes what an earlier version ran.
ContentRewritten bool
// The first nonempty snapshot of a staged package file or a plugin copy,
// kept across empty writes and delayed analyzer verdicts: its digest, or
// that its bytes could not be read whole. The latest snapshot alone cannot
// prove its history.
historySet bool
historyKnown bool
historyDigest [32]byte
historyObserved time.Time
// Observed is the TTL origin; snapshotObserved orders the metadata even
// after a merge has moved Observed back to the earliest event.
snapshotObserved time.Time
// WPInstallData proves the complete, stable snapshot used for Digest was
// a translation return literal or version assignments, with no payload.
WPInstallData bool
// Sticky across rewrites: data copied later cannot erase an earlier
// nonempty snapshot that carried code or whose full content was unknown.
WPInstallUnsafe bool
// WPCoreRelease names the WordPress release a version-probe data snapshot
// declares, with that snapshot's digest in the form wordpress.org
// publishes. It belongs to the same snapshot as WPInstallData.
WPCoreRelease *wpcheck.Verification
// PluginRelease names the plugin release a plugin copy is compared with,
// as the installed plugin declared it when the copy was written, with the
// copy's digest in the form wordpress.org publishes. It belongs to the
// same snapshot as Digest.
PluginRelease *wpcheck.Verification
Head []byte
// Parent identifies the real, non-symlink directory that contained the
// candidate while its event fd was open. The later probe uses this stable
// identity instead of inferring directory removal from two path stats.
Parent dropperParentIdentity
ticket queuehealth.Ticket
}
// dropperParentIdentity is deliberately compact because one is retained for
// every tracked candidate. BirthNanos disambiguates inode reuse when statx
// exposes it; without birth time, a reused identity is treated as unchanged
// and cannot earn a false-positive demotion.
type dropperParentIdentity struct {
Device uint64
Inode uint64
BirthNanos int64
BirthKnown bool
Conflicted bool
}
func (i dropperParentIdentity) known() bool {
return !i.Conflicted && i.Device != 0 && i.Inode != 0
}
func mergeDropperParentIdentity(a, b dropperParentIdentity) dropperParentIdentity {
switch {
case a.Conflicted || b.Conflicted:
return dropperParentIdentity{Conflicted: true}
case !a.known():
return b
case !b.known():
return a
case a.Device != b.Device || a.Inode != b.Inode:
return dropperParentIdentity{Conflicted: true}
case a.BirthKnown && b.BirthKnown && a.BirthNanos != b.BirthNanos:
return dropperParentIdentity{Conflicted: true}
case b.BirthKnown:
return b
default:
return a
}
}
func dropperParentChanged(observed, current dropperParentIdentity) bool {
if !observed.known() || !current.known() {
return false
}
if observed.Device != current.Device || observed.Inode != current.Inode {
return true
}
return observed.BirthKnown && current.BirthKnown && observed.BirthNanos != current.BirthNanos
}
// shouldTrackDropper reports whether a close-write event is a freshly
// created PHP or executable file inside a web document root, i.e. a
// candidate for self-deleting-dropper tracking. A recent statx birth time is
// the only freshness proof, so modifications of pre-existing files and files
// on filesystems without STATX_BTIME are not tracked.
func shouldTrackDropper(c dropperCandidate, selfPID int32, freshFor time.Duration) bool {
if c.Docroot == "" {
return false
}
if c.PID == selfPID {
return false
}
if c.Mode&unixSIFMT != unixSIFREG {
return false
}
name := strings.ToLower(filepath.Base(c.Path))
if !contenttype.IsExecutablePHPName(name) && !c.PHPExecutable && c.Mode&0o111 == 0 {
return false
}
age := c.Observed.Sub(c.Birth)
return c.BirthKnown && age >= 0 && age <= freshFor
}
// S_IFMT constants mirrored from the unix package so this file stays free
// of //go:build linux and the decision logic remains testable on any OS.
const (
unixSIFMT = 0o170000
unixSIFREG = 0o100000
)
// dropperMaxTracked bounds the tracker map. A cPanel package restore or a
// WP Toolkit site clone can close-write tens of thousands of PHP files in
// seconds, and every one stays tracked for the whole unlink TTL; entries
// beyond the cap are dropped (and counted) rather than evicting older
// candidates, because the oldest entries are the ones closest to their probe
// and losing them would blind the detector exactly when a bulk write storm
// provides cover.
const dropperMaxTracked = 16384
// Keep each waiting or detached batch's head-byte budget at 16 MiB.
// Candidate/map/path metadata is additional bounded memory and grows with the
// entry cap; this constant only accounts for copied content. Representative
// Twig and Smarty headers place all required markers inside this window.
const (
dropperTrackedHeadBudget = 16 << 20
dropperTrackedHeadMax = dropperTrackedHeadBudget / dropperMaxTracked
)
type dropperCandidateKey struct {
path string
device uint64
inode uint64
birthNanos int64
birthKnown bool
}
func candidateKey(c dropperCandidate) dropperCandidateKey {
key := dropperCandidateKey{
path: c.Path,
device: c.Device,
inode: c.Inode,
birthKnown: c.BirthKnown,
}
if c.BirthKnown {
key.birthNanos = c.Birth.UnixNano()
}
return key
}
func ownDropperCandidate(c dropperCandidate) dropperCandidate {
if c.snapshotObserved.Before(c.Observed) {
c.snapshotObserved = c.Observed
}
if c.Size != 0 && !c.historySet && dropperKeepsContentHistory(c.Path, c.Docroot) {
c.historySet = true
c.historyKnown, c.historyDigest = c.DigestKnown, c.Digest
c.historyObserved = c.snapshotObserved
}
// Torn bytes that already look like code are evidence, not noise.
c.ContentMayExecute = c.ContentMayExecute || !dropperCandidateIsHarmless(c)
if len(c.Head) > dropperTrackedHeadMax {
c.Head = c.Head[:dropperTrackedHeadMax]
}
c.Head = bytes.Clone(c.Head)
return c
}
// dropperKeepsContentHistory reports whether a later exemption for path
// compares the file's bytes with an official or installed copy. Only then
// must every earlier snapshot be accounted for, not just the last one.
func dropperKeepsContentHistory(path, docroot string) bool {
if wpUpgradeStagedPackageFile(path, docroot) {
return true
}
_, _, ok := dropperPluginCopySource(path, docroot)
return ok
}
// dropperHistoryDiffers reports whether two snapshots of a staged package
// file or a plugin copy may hold different content. Bytes that could not be
// read whole only count once a second snapshot exists to compare them with; a
// single unreadable snapshot moved into place is still one write.
func dropperHistoryDiffers(a, b dropperCandidate) bool {
if !a.historySet || !b.historySet {
return false
}
if !a.historyKnown || !b.historyKnown {
// Analyzer verdicts and detached probes can replay the same unreadable
// snapshot after newer empty writes. Compare the history's own time,
// not the latest metadata time or the merged TTL origin.
return !a.historyObserved.Equal(b.historyObserved)
}
return a.historyDigest != b.historyDigest
}
func mergeDropperCandidate(prev, next dropperCandidate) dropperCandidate {
merged := next
if next.snapshotObserved.Before(prev.snapshotObserved) {
merged = prev
}
merged.Observed = prev.Observed
if next.Observed.Before(prev.Observed) {
merged.Observed = next.Observed
}
merged.PHPExecutable = prev.PHPExecutable || next.PHPExecutable
merged.ContentSuspicious = prev.ContentSuspicious || next.ContentSuspicious
merged.ContentMayExecute = prev.ContentMayExecute || next.ContentMayExecute
merged.ContentUnsettled = prev.ContentUnsettled || next.ContentUnsettled
merged.WPInstallUnsafe = prev.WPInstallUnsafe || next.WPInstallUnsafe
merged.ContentRewritten = prev.ContentRewritten || next.ContentRewritten || dropperHistoryDiffers(prev, next)
history := next
if prev.historySet {
history = prev
}
merged.historySet = history.historySet
merged.historyKnown, merged.historyDigest = history.historyKnown, history.historyDigest
merged.historyObserved = history.historyObserved
merged.Parent = mergeDropperParentIdentity(prev.Parent, next.Parent)
merged.ticket = prev.ticket
return merged
}
// dropperTracker holds candidates between their close-write observation and
// the TTL probe. All methods are safe for concurrent use by the analyzer
// workers and the probe loop.
type dropperTracker struct {
mu sync.Mutex
ttl time.Duration
maxTracked int
entries map[dropperCandidateKey]dropperCandidate
pending []dropperGone
overflow uint64
now func() time.Time
healthOnce sync.Once
health *queuehealth.Tracker
heldHealth *queuehealth.Tracker
}
func newDropperTracker(ttl time.Duration) *dropperTracker {
return &dropperTracker{
ttl: ttl,
maxTracked: dropperMaxTracked,
entries: make(map[dropperCandidateKey]dropperCandidate),
now: time.Now,
}
}
func (t *dropperTracker) initQueueHealth() {
t.healthOnce.Do(func() {
t.health = queuehealth.New(t.maxTracked, time.Minute)
t.heldHealth = queuehealth.New(dropperMaxTracked, time.Minute)
})
}
func (t *dropperTracker) queueStatuses(now time.Time) (queuehealth.Status, queuehealth.Status) {
t.initQueueHealth()
return t.health.Snapshot(now), t.heldHealth.Snapshot(now)
}
// Observe records a candidate. Re-observing the same file identity keeps the
// earliest event time (so rewrites cannot postpone the probe) and the newest
// metadata snapshot. A replacement inode at the same path is a separate
// candidate; otherwise an attacker could overwrite a vanished drop with a
// benign survivor before the probe. The return value reports whether the
// candidate was retained rather than rejected by the capacity bound. A false
// result is detection coverage loss and the Linux wiring must surface it as
// a metric and operator-facing warning.
func (t *dropperTracker) Observe(c dropperCandidate) bool {
t.initQueueHealth()
c = ownDropperCandidate(c)
key := candidateKey(c)
t.mu.Lock()
defer t.mu.Unlock()
if prev, ok := t.entries[key]; ok {
t.entries[key] = mergeDropperCandidate(prev, c)
prev.ticket.RetainQueuedAt(c.Observed.Add(t.ttl))
return true
}
if len(t.entries) >= t.maxTracked {
t.overflow++
t.health.Lose(t.now(), 1)
return false
}
c.ticket = t.health.BeginAt(c.Observed.Add(t.ttl), t.now())
t.entries[key] = c
return true
}
// Retry returns a detached probe to the waiting set. A new observation may
// already occupy its identity or the available slot, so this transfer must
// share the admission lock with Observe.
func (t *dropperTracker) Retry(c dropperCandidate) (queuehealth.Ticket, bool) {
t.mu.Lock()
defer t.mu.Unlock()
key := candidateKey(c)
now := t.now()
if waiting, ok := t.entries[key]; ok {
waiting.ticket.MergeRunning(c.ticket, now)
t.entries[key] = mergeDropperCandidate(waiting, c)
return waiting.ticket, true
}
if len(t.entries) >= t.maxTracked {
t.overflow++
c.ticket.Reject(now)
return queuehealth.Ticket{}, false
}
c.ticket.Requeue(now)
t.entries[key] = c
return c.ticket, true
}
// Refresh updates a previously admitted candidate without creating a new
// entry. A later close-write of a tracked file, or a content verdict on its
// snapshot, must reach the entry even when admission would now refuse the
// snapshot as inert.
func (t *dropperTracker) Refresh(c dropperCandidate) bool {
c = ownDropperCandidate(c)
key := candidateKey(c)
t.mu.Lock()
defer t.mu.Unlock()
if prev, ok := t.entries[key]; ok {
t.entries[key] = mergeDropperCandidate(prev, c)
prev.ticket.RetainQueuedAt(c.Observed.Add(t.ttl))
return true
}
return false
}
// Due removes and returns every candidate whose TTL has elapsed at now.
func (t *dropperTracker) Due(now time.Time) []dropperCandidate {
t.mu.Lock()
defer t.mu.Unlock()
started := t.now()
var due []dropperCandidate
for key, c := range t.entries {
if now.Sub(c.Observed) >= t.ttl {
c.ticket.Start(started)
due = append(due, c)
delete(t.entries, key)
}
}
return due
}
func (t *dropperTracker) trackedCount() int {
t.mu.Lock()
defer t.mu.Unlock()
return len(t.entries)
}
func (t *dropperTracker) overflowDropped() uint64 {
t.mu.Lock()
defer t.mu.Unlock()
return t.overflow
}
// discardPending runs after both the probe loop and analyzer workers join.
// Analyzer work finishing during shutdown can still admit fresh candidates.
func (t *dropperTracker) discardPending(now time.Time) {
t.mu.Lock()
defer t.mu.Unlock()
for _, c := range t.entries {
c.ticket.Reject(now)
}
clear(t.entries)
for _, g := range t.pending {
g.ticket.Reject(now)
}
t.pending = nil
}
// dropperFileState is the identity and content evidence captured when the
// probe opens a path. All fields must come from the same open fd. Device plus
// inode handles rename(2); birth time guards against inode reuse; a full
// digest handles copy-delete moves across filesystems. Head bytes are
// deliberately not identity evidence.
type dropperFileState struct {
Path string
Device uint64
Inode uint64
Size int64
Birth time.Time
BirthKnown bool
Digest [32]byte
DigestKnown bool
// IsRegular distinguishes a file that took over the path from a directory
// or symlink left behind there. Only a regular file can be the result of
// an atomic write.
IsRegular bool
}
// dropperProbe is what the TTL probe learned about a candidate. AtPath and
// RenameTarget carry enough evidence for this platform-free core to validate
// identity. A bare path-exists or destination-exists boolean would let a
// replacement file hide the vanished inode.
type dropperProbe struct {
// Conclusive is set only after the probe distinguished absence from a
// permission, I/O, or other transient failure.
Conclusive bool
AtPath *dropperFileState
// DocrootRemoved is true only for a confirmed ENOENT on the document
// root, not for permission or transient I/O failures.
DocrootRemoved bool
// ParentRemoved is true only when a snapshotted, non-symlink parent is now
// absent or a different directory identity. A file whose whole directory
// went away was not singled out for deletion.
ParentRemoved bool
RenamedTo string
RenameTarget *dropperFileState
// QuarantineMatched requires an exact ledger identity/fingerprint match,
// not merely a prior quarantine entry for the same path.
QuarantineMatched bool
// OfficialWPCoreFile reports that the candidate's WPCoreRelease digest
// equals the wordpress.org checksum for that release. Checksums that are
// not cached yet leave it false.
OfficialWPCoreFile bool
// OfficialWPCorePackageFile reports that the candidate, a file of an
// unpacked core release, is byte for byte the file of that path in the
// release installed at the WordPress root.
OfficialWPCorePackageFile bool
// PluginCopy compares a vanished file that a plugin copies out of its own
// package with the file it was copied from.
PluginCopy dropperPluginCopyEvidence
}
// dropperPluginCopyEvidence grades how well a vanished plugin copy is
// accounted for by the plugin package it came from.
type dropperPluginCopyEvidence int
const (
dropperPluginCopyUnproven dropperPluginCopyEvidence = iota
// The bytes are the file at that path in the wordpress.org release of the
// plugin installed beside the copy.
dropperPluginCopyOfficial
// No release manifest could be consulted, but the bytes equal the
// installed plugin's own file, which stays on disk for the finding to name.
dropperPluginCopyInstalled
)
type dropperVerdict int
const (
dropperBenign dropperVerdict = iota
dropperInconclusive
dropperDemotedTemplate
dropperDemotedAtomicWrite
dropperDemotedWPUpgrade
dropperDemotedDocroot
dropperDemotedDirRemoved
dropperDemotedReplaced
dropperDemotedBackupState
dropperDemotedPluginCopy
dropperSuspect
)
func dropperVerdictDemoted(v dropperVerdict) bool {
return v >= dropperDemotedTemplate && v <= dropperDemotedPluginCopy
}
func dropperSameIdentity(c dropperCandidate, current dropperFileState) bool {
if c.Inode == 0 || current.Inode == 0 || c.Device != current.Device || c.Inode != current.Inode {
return false
}
if c.BirthKnown != current.BirthKnown {
return false
}
return !c.BirthKnown || c.Birth.Equal(current.Birth)
}
// dropperReplacedInPlace reports whether the file now at the candidate's path
// is a regular file that came into existence after the candidate was observed.
// That is an atomic write completing (write temp, rename over the live path),
// which repeats every few minutes for WAF and cache state files. The successor
// stays on disk and is scanned in its own right, and the candidate's own bytes
// were already read by the content pass, so the evidence loss that makes a
// self-delete Critical does not apply.
//
// This does not weaken the detector against an attacker who leaves a benign
// file behind: overwriting the same inode already returns dropperBenign above,
// which is both cheaper and quieter than unlink plus rename. A birth time is
// required, so a filesystem without STATX_BTIME keeps the suspect verdict.
func dropperReplacedInPlace(c dropperCandidate, current dropperFileState) bool {
if !current.IsRegular || !current.BirthKnown {
return false
}
return !current.Birth.Before(c.Observed)
}
// assessDropper turns a probe result into a verdict for one candidate.
func assessDropper(c dropperCandidate, p dropperProbe) dropperVerdict {
if !p.Conclusive {
// Due removed this candidate from the tracker. The probe loop should
// reinsert it with Retry and handle a false capacity result.
return dropperInconclusive
}
if p.QuarantineMatched {
return dropperBenign
}
if p.AtPath != nil {
if p.AtPath.Path != c.Path {
return dropperSuspect
}
if dropperSameIdentity(c, *p.AtPath) {
return dropperBenign
}
}
if _, _, pluginCopy := dropperPluginCopySource(c.Path, c.Docroot); pluginCopy {
// This code ran as a mu-plugin. Other capability probes, directory
// churn and install moves cannot vouch for its bytes or erase rewrites.
if c.ContentSuspicious || c.ContentRewritten {
return dropperSuspect
}
if dropperPluginCopyEligible(c) {
switch p.PluginCopy {
case dropperPluginCopyOfficial:
return dropperBenign
case dropperPluginCopyInstalled:
return dropperDemotedPluginCopy
}
}
if !c.ContentMayExecute && !c.ContentUnsettled && dropperCandidateIsInert(c) {
return dropperBenign
}
return dropperSuspect
}
if p.RenamedTo != "" || p.RenameTarget != nil {
if p.RenamedTo == "" || p.RenameTarget == nil || p.RenameTarget.Path != p.RenamedTo {
return dropperSuspect
}
if dropperRenameTargetAllowed(c, p.RenamedTo) && dropperRenameMatch(c, *p.RenameTarget) {
return dropperBenign
}
}
if c.ContentSuspicious {
return dropperSuspect
}
if dropperOfficialVersionProbe(c, p) {
return dropperBenign
}
if dropperOfficialCorePackageFile(c, p) {
return dropperBenign
}
if !c.ContentMayExecute && !c.ContentUnsettled && dropperCandidateIsHarmless(c) {
return dropperBenign
}
if p.AtPath != nil && dropperReplacedInPlace(c, *p.AtPath) {
return dropperDemotedReplaced
}
if p.DocrootRemoved {
return dropperDemotedDocroot
}
if p.ParentRemoved {
return dropperDemotedDirRemoved
}
if atomicWriteRenameCandidate(c.Path) != "" {
return dropperDemotedAtomicWrite
}
if len(wpUpgradeRenameCandidates(c.Path, c.Docroot)) > 0 {
return dropperDemotedWPUpgrade
}
if looksLikeCompiledTemplate(c.Head) {
return dropperDemotedTemplate
}
if looksLikeBackWPupJobState(c.Path, c.Head) {
return dropperDemotedBackupState
}
return dropperSuspect
}
var backwpupFolderListName = regexp.MustCompile(`^backwpup-[A-Za-z0-9]+-folder\.php$`)
// looksLikeBackWPupJobState recognises the state files the BackWPup plugin
// writes while a backup job runs and deletes when it ends. The file name and
// the head must both match the writer's format: backwpup-working.php is
// "<?php //" plus the job as JSON on one line, and backwpup-<hash>-folder.php
// is "<?php" and then one "//<absolute folder>" comment per line. Every
// visible byte after the opening tag must stay inside those comments; the
// tracked head cannot see the rest of the file, so this only demotes.
func looksLikeBackWPupJobState(path string, head []byte) bool {
name := filepath.Base(path)
switch {
case name == "backwpup-working.php":
rest, ok := bytes.CutPrefix(head, []byte(`<?php //{"`))
return ok && !bytes.ContainsAny(rest, "\r\n") && !bytes.Contains(rest, []byte("?>"))
case backwpupFolderListName.MatchString(name):
rest, ok := bytes.CutPrefix(head, []byte("<?php\n"))
if !ok {
if rest, ok = bytes.CutPrefix(head, []byte("<?php\r\n")); !ok {
return false
}
}
lines := bytes.Split(rest, []byte("\n"))
folders := 0
for i, line := range lines {
last := i == len(lines)-1
// A full retained head may stop between CR and LF. Removing
// only the terminal CR still rejects any bare CR before code.
if !last || len(head) == dropperTrackedHeadMax {
line = bytes.TrimSuffix(line, []byte("\r"))
}
if bytes.ContainsRune(line, '\r') || bytes.Contains(line, []byte("?>")) {
return false
}
switch {
case bytes.HasPrefix(line, []byte("///")):
folders++
case last && len(line) < 3 && bytes.Equal(line, []byte("///")[:len(line)]):
// The head can end inside the next comment marker.
default:
return false
}
}
return folders > 0
}
return false
}
// looksLikeCompiledTemplate recognises template-engine compile artifacts
// (Twig class caches as written by phpMyAdmin/Symfony/Drupal, Smarty
// compile dirs). These are legitimately created and unlinked in short
// windows during cache rebuilds.
func looksLikeCompiledTemplate(head []byte) bool {
head = bytes.TrimSpace(bytes.TrimPrefix(head, []byte{0xef, 0xbb, 0xbf}))
if !bytes.HasPrefix(head, []byte("<?php")) {
return false
}
if bytes.Contains(head, []byte("class __TwigTemplate_")) &&
bytes.Contains(head, []byte(" extends Template")) {
return true
}
if bytes.Contains(head, []byte("/* Smarty version ")) &&
bytes.Contains(head, []byte(", created on ")) &&
bytes.Contains(head, []byte("from '")) {
return true
}
return false
}
// wpUpgradeStagedPath splits a clean absolute path under
// <wpRoot>/wp-content/upgrade/ into the WordPress root and the part below
// upgrade/. The root must be the configured docroot or inside it.
func wpUpgradeStagedPath(path, configuredDocroot string) (wpRoot, rest string, ok bool) {
const marker = "/wp-content/upgrade/"
if !filepath.IsAbs(path) || !filepath.IsAbs(configuredDocroot) ||
filepath.Clean(path) != path || filepath.Clean(configuredDocroot) != configuredDocroot {
return "", "", false
}
idx := strings.Index(path, marker)
if idx < 0 {
return "", "", false
}
wpRoot = path[:idx]
if wpRoot != configuredDocroot && !strings.HasPrefix(wpRoot, configuredDocroot+string(filepath.Separator)) {
return "", "", false
}
return wpRoot, path[idx+len(marker):], true
}
// wpUpgradeRenameCandidates maps a path inside a WordPress upgrade staging
// dir (wp-content/upgrade/<staging>/<package>/<rest>) to the destinations
// WordPress moves it to on success: the plugin and theme dirs, or the
// docroot itself for core packages. Close after write is the only fanotify
// event, and placement is reported only by the placement backend where one
// runs, so a successful rename-based install makes the staged path vanish;
// the probe checks these destinations before calling it a self-deleting drop.
func wpUpgradeRenameCandidates(path, configuredDocroot string) []string {
wpRoot, rest, ok := wpUpgradeStagedPath(path, configuredDocroot)
if !ok {
return nil
}
parts := strings.SplitN(rest, "/", 3)
if len(parts) < 3 || parts[1] == "" || parts[1] == "." || parts[1] == ".." ||
parts[2] == "" || filepath.Clean(parts[2]) != parts[2] {
// A file directly under upgrade/<staging>/ has no package dir to
// move. The flat language-pack copy is handled by
// wpUpgradeInstallDestinations.
return nil
}
pkg, tail := parts[1], parts[2]
if pkg == "wordpress" {
return []string{filepath.Join(wpRoot, tail)}
}
return []string{
filepath.Join(wpRoot, "wp-content", "plugins", pkg, tail),
filepath.Join(wpRoot, "wp-content", "themes", pkg, tail),
}
}
// wpUpgradeStagedPackageFile reports whether path lies inside an unpacked
// core, plugin or theme package under wp-content/upgrade/.
func wpUpgradeStagedPackageFile(path, configuredDocroot string) bool {
return len(wpUpgradeRenameCandidates(path, configuredDocroot)) > 0
}
// wpUpgradeCorePackageFile splits a path inside an unpacked core release,
// <wpRoot>/wp-content/upgrade/<working>/wordpress/<rel>, into the WordPress
// root and the file's key in the release checksum manifest.
func wpUpgradeCorePackageFile(path, configuredDocroot string) (wpRoot, rel string, ok bool) {
wpRoot, rest, ok := wpUpgradeStagedPath(path, configuredDocroot)
if !ok {
return "", "", false
}
parts := strings.SplitN(rest, "/", 3)
if len(parts) < 3 || parts[0] == "" || parts[1] != "wordpress" ||
parts[2] == "" || filepath.Clean(parts[2]) != parts[2] {
return "", "", false
}
return wpRoot, parts[2], true
}
// wpUpgradeInstallDestinations lists every place the WordPress updater puts
// the bytes of a file it wrote under wp-content/upgrade/ before removing it:
// the package-tree moves above, plus two copy-then-delete steps that leave no
// package directory behind. Language packs unzip flat into
// upgrade/<working>/ and are copied into wp-content/languages/ (plugins/ and
// themes/ for those pack types). A core update copies the staged
// wp-includes/version.php to upgrade/version-current.php, reads it, deletes
// it, and later installs the same file as wp-includes/version.php.
//
// Copy destinations also require a complete data-only snapshot. An identical
// payload planted in languages/ or version.php does not prove updater activity.
func wpUpgradeInstallDestinations(path, configuredDocroot string) []string {
if dests := wpUpgradeRenameCandidates(path, configuredDocroot); dests != nil {
return dests
}
return wpUpgradeCopyDestinations(path, configuredDocroot)
}
func wpUpgradeCopyDestinations(path, configuredDocroot string) []string {
wpRoot, rest, ok := wpUpgradeStagedPath(path, configuredDocroot)
if !ok {
return nil
}
parts := strings.Split(rest, "/")
switch {
case len(parts) == 1 && parts[0] == "version-current.php":
return []string{filepath.Join(wpRoot, "wp-includes", "version.php")}
case len(parts) == 2 && strings.HasSuffix(parts[1], ".l10n.php"):
// Clean already rejected empty, "." and ".." components.
languages := filepath.Join(wpRoot, "wp-content", "languages")
return []string{
filepath.Join(languages, parts[1]),
filepath.Join(languages, "plugins", parts[1]),
filepath.Join(languages, "themes", parts[1]),
}
}
return nil
}
func dropperRenameTargetAllowed(c dropperCandidate, target string) bool {
if atomicTarget := atomicWriteRenameCandidate(c.Path); atomicTarget != "" && target == atomicTarget {
return true
}
for _, candidate := range wpUpgradeRenameCandidates(c.Path, c.Docroot) {
if target == candidate {
return true
}
}
if !dropperWPCopyDataEligible(c) {
return false
}
for _, candidate := range wpUpgradeCopyDestinations(c.Path, c.Docroot) {
if target == candidate {
return true
}
}
return false
}
// dropperWPCopyDataEligible reports whether every snapshot of c was complete
// translation or version data with nothing that could have run.
func dropperWPCopyDataEligible(c dropperCandidate) bool {
return c.WPInstallData && !c.WPInstallUnsafe && c.DigestKnown &&
!c.ContentSuspicious && c.Mode&0o111 == 0
}
// dropperOfficialVersionProbe covers a core update that stops after reading
// the new version file, for example on a failed PHP or database requirement.
// The installed version file then belongs to the old release and cannot match,
// so the removed probe must itself be the official file of the release it
// declares.
func dropperOfficialVersionProbe(c dropperCandidate, p dropperProbe) bool {
return p.OfficialWPCoreFile && c.WPCoreRelease != nil && dropperWPCopyDataEligible(c) &&
filepath.Base(c.Path) == "version-current.php" &&
len(wpUpgradeCopyDestinations(c.Path, c.Docroot)) > 0
}
// dropperOfficialCorePackageFile covers the files a core update unpacks but
// never installs: bundled themes and plugins under wp-content/, and files
// unchanged from the running release. WordPress deletes them with the working
// tree. Official release bytes carry nothing an attacker chose, so removing
// them loses no evidence.
func dropperOfficialCorePackageFile(c dropperCandidate, p dropperProbe) bool {
return p.OfficialWPCorePackageFile && c.CoreMD5Known && !c.ContentRewritten &&
!c.ContentUnsettled && c.Mode&0o111 == 0
}
// dropperPluginCopy names a file that a plugin copies out of its own package
// into a directory WordPress loads code from, and deletes again when the
// feature it serves is switched off.
type dropperPluginCopy struct {
// dest is the copy's path below wp-content/.
dest string
slug string
// rel is the source's path below the plugin root, as the plugin's
// wordpress.org release manifest keys it.
rel string
}
var dropperPluginCopies = []dropperPluginCopy{
// Elementor Safe Mode copies its loader into mu-plugins while enabled.
{dest: "mu-plugins/elementor-safe-mode.php", slug: "elementor", rel: "modules/safe-mode/mu-plugin/elementor-safe-mode.php"},
}
// dropperPluginCopySource maps a path where a listed plugin copy lands to the
// root of the plugin it comes from. The plugin must be installed in the same
// wp-content directory, and that directory must lie in the candidate's own
// document root, so a copy is only ever compared with its own site's plugin.
func dropperPluginCopySource(path, docroot string) (string, dropperPluginCopy, bool) {
if !filepath.IsAbs(path) || !filepath.IsAbs(docroot) ||
filepath.Clean(path) != path || filepath.Clean(docroot) != docroot {
return "", dropperPluginCopy{}, false
}
for _, entry := range dropperPluginCopies {
wpContent, found := strings.CutSuffix(path, "/"+entry.dest)
if !found || filepath.Base(wpContent) != "wp-content" {
continue
}
if wpContent != docroot && !strings.HasPrefix(wpContent, docroot+string(filepath.Separator)) {
continue
}
return filepath.Join(wpContent, "plugins", entry.slug), entry, true
}
return "", dropperPluginCopy{}, false
}
// dropperPluginCopyEligible reports whether every snapshot of c was the same
// complete, settled content, so comparing its digest with the plugin's file
// accounts for everything the copy could have run.
func dropperPluginCopyEligible(c dropperCandidate) bool {
_, _, ok := dropperPluginCopySource(c.Path, c.Docroot)
return ok && c.DigestKnown && !c.ContentRewritten && !c.ContentUnsettled &&
c.Mode&0o111 == 0
}
// dropperRenameMatch reports whether a probe of a rename-destination path
// identifies the same file as the tracked candidate: identical device,
// inode, and birth time for rename(2), or identical size plus a full SHA-256
// digest for a copy-delete fallback across filesystems.
func dropperRenameMatch(c dropperCandidate, dest dropperFileState) bool {
// Neither the moved file nor a surviving copy of the last staged snapshot
// can account for an earlier payload overwritten before the move.
staged := wpUpgradeStagedPackageFile(c.Path, c.Docroot)
if staged && c.ContentRewritten {
return false
}
if dropperSameIdentity(c, dest) {
return true
}
if staged && c.ContentUnsettled {
return false
}
return c.DigestKnown && dest.DigestKnown && c.Size == dest.Size && c.Digest == dest.Digest
}
// dropperGraceWindow is how long a vanished candidate is held before its
// finding flushes. The hold lets a bulk operation (plugin upgrade fallback,
// cache purge, deploy rollback) accumulate its siblings so the whole batch
// collapses into one Warning for classified churn or one High signal for an
// unclassified burst, instead of paging Critical per file.
const dropperGraceWindow = 45 * time.Second
// dropperBurstThreshold is the group size at which held candidates from one
// docroot are reported as a single bulk-churn aggregate instead of
// individual findings.
const dropperBurstThreshold = 8
type dropperGone struct {
Cand dropperCandidate
Verdict dropperVerdict
held time.Time
ticket queuehealth.Ticket
}
// dropperFinding is one flush decision: a single vanished file, a removed
// directory group, or a per-docroot aggregate of a create/delete burst.
type dropperFinding struct {
Aggregate bool
Docroot string
Items []dropperGone
// RemovedDir groups files that were each demoted because this directory
// was removed with them: one event, reported once.
RemovedDir string
}
// HoldGone parks a vanished candidate until FlushDue decides whether it is
// reported alone or as part of a bulk-churn aggregate.
func (t *dropperTracker) HoldGone(c dropperCandidate, v dropperVerdict, now time.Time) {
if v == dropperBenign || v == dropperInconclusive {
return
}
t.initQueueHealth()
t.mu.Lock()
defer t.mu.Unlock()
if len(t.pending) >= dropperMaxTracked {
t.heldHealth.Lose(t.now(), 1)
return
}
c.ticket = queuehealth.Ticket{}
t.pending = append(t.pending, dropperGone{
Cand: ownDropperCandidate(c), Verdict: v, held: now,
ticket: t.heldHealth.BeginAt(now.Add(dropperGraceWindow), t.now()),
})
}
// FlushDue emits findings for docroot groups whose oldest held entry has
// aged past the grace window. The whole group flushes together so entries
// arriving late in a burst still fold into the aggregate.
func (t *dropperTracker) FlushDue(now time.Time) []dropperFinding {
t.mu.Lock()
defer t.mu.Unlock()
started := t.now()
type groupKey struct {
docroot string
demoted bool
}
keyFor := func(g dropperGone) groupKey {
return groupKey{docroot: g.Cand.Docroot, demoted: dropperVerdictDemoted(g.Verdict)}
}
oldest := make(map[groupKey]time.Time)
for _, g := range t.pending {
key := keyFor(g)
if first, ok := oldest[key]; !ok || g.held.Before(first) {
oldest[key] = g.held
}
}
groups := make(map[groupKey][]dropperGone)
var keep []dropperGone
for _, g := range t.pending {
key := keyFor(g)
if now.Sub(oldest[key]) >= dropperGraceWindow {
g.ticket.Start(started)
groups[key] = append(groups[key], g)
} else {
keep = append(keep, g)
}
}
t.pending = keep
var out []dropperFinding
for key, items := range groups {
out = append(out, groupDropperFindings(key.docroot, items)...)
}
return out
}
// groupDropperFindings also runs after ignore-path filtering: a burst can
// shrink into removed-directory groups, and a directory group into one file.
// Callers keep docroots and demoted/unclassified batches separate.
func groupDropperFindings(docroot string, items []dropperGone) []dropperFinding {
if len(items) >= dropperBurstThreshold {
return []dropperFinding{{Aggregate: true, Docroot: docroot, Items: items}}
}
var out []dropperFinding
byDir := make(map[string][]dropperGone)
for _, item := range items {
if item.Verdict == dropperDemotedDirRemoved {
dir := filepath.Dir(item.Cand.Path)
byDir[dir] = append(byDir[dir], item)
continue
}
out = append(out, dropperFinding{Docroot: docroot, Items: []dropperGone{item}})
}
for dir, members := range byDir {
f := dropperFinding{Docroot: docroot, Items: members}
if len(members) > 1 {
f.RemovedDir = dir
}
out = append(out, f)
}
return out
}
// dropperHeadExcerptMax caps how many leading file bytes a finding's
// details reproduce as evidence. The bounded tracked head stays in memory
// only until the flush; findings carry just enough to triage without a file
// (the file is gone by definition).
const dropperHeadExcerptMax = 160
// dropperAggregateSampleMax caps how many member paths an aggregate
// finding lists before truncating with a count.
const dropperAggregateSampleMax = 10
const (
dropperPathExcerptMax = 512
dropperProcExcerptMax = 256
)
// dropperAlertParams renders one flush decision into alert parameters:
// severity, message, details, the finding path and its dedup key. Counts,
// sizes, write ages and writers change between reports of one dropper, so
// the key is only what was removed and the severity.
func dropperAlertParams(f dropperFinding) (alert.Severity, string, string, string, string) {
if f.RemovedDir != "" {
var b strings.Builder
b.WriteString("Files created and removed within the tracking TTL together with the directory holding them; each was demoted for that reason:\n")
for i, g := range f.Items {
if i == dropperAggregateSampleMax {
fmt.Fprintf(&b, "... and %d more", len(f.Items)-dropperAggregateSampleMax)
break
}
fmt.Fprintf(&b, "%s (uid=%d size=%d)\n",
dropperPrintable([]byte(g.Cand.Path), dropperPathExcerptMax), g.Cand.UID, g.Cand.Size)
}
msg := fmt.Sprintf("%d short-lived PHP/executable files removed with their directory %s",
len(f.Items), dropperPrintable([]byte(f.RemovedDir), dropperPathExcerptMax))
return alert.Warning, msg, strings.TrimRight(b.String(), "\n"), f.RemovedDir, dropperDedupKey("removed", f.RemovedDir, alert.Warning)
}
if f.Aggregate {
severity := alert.Warning
unclassified := 0
for _, g := range f.Items {
if !dropperVerdictDemoted(g.Verdict) {
unclassified++
}
}
if unclassified > 0 {
severity = alert.High
}
var b strings.Builder
fmt.Fprintf(&b, "Files created and removed within the tracking TTL; %d remained unclassified after false-positive checks:\n", unclassified)
for i, g := range f.Items {
if i == dropperAggregateSampleMax {
fmt.Fprintf(&b, "... and %d more", len(f.Items)-dropperAggregateSampleMax)
break
}
fmt.Fprintf(&b, "%s (uid=%d size=%d)\n",
dropperPrintable([]byte(g.Cand.Path), dropperPathExcerptMax), g.Cand.UID, g.Cand.Size)
}
displayDocroot := dropperPrintable([]byte(f.Docroot), dropperPathExcerptMax)
msg := fmt.Sprintf("%d short-lived PHP/executable files created and removed under %s", len(f.Items), displayDocroot)
return severity, msg, strings.TrimRight(b.String(), "\n"), f.Docroot, dropperDedupKey("docroot", f.Docroot, severity)
}
g := f.Items[0]
c := g.Cand
lifetime := "unknown"
if c.BirthKnown {
lifetime = c.Observed.Sub(c.Birth).Truncate(time.Second).String()
}
details := fmt.Sprintf(
"File appeared and was removed before the TTL probe. uid=%d size=%d mode=%04o write-age=%s",
c.UID, c.Size, c.Mode&0o7777, lifetime)
if c.ProcInfo != "" {
details += " writer=[" + dropperPrintable([]byte(c.ProcInfo), dropperProcExcerptMax) + "]"
}
sev := alert.Critical
if dropperVerdictDemoted(g.Verdict) {
sev = alert.Warning
switch g.Verdict {
case dropperDemotedTemplate:
details += "\nDemoted: content matches a compiled-template artifact (Twig/Smarty cache churn)."
case dropperDemotedAtomicWrite:
details += "\nDemoted: filename matches an atomic-write staging artifact."
case dropperDemotedWPUpgrade:
details += "\nDemoted: path is structurally inside a WordPress upgrade staging tree."
case dropperDemotedDocroot:
details += "\nDemoted: the containing document root was removed before the probe."
case dropperDemotedDirRemoved:
details += "\nDemoted: the original containing directory was removed before the probe."
case dropperDemotedReplaced:
details += "\nDemoted: the path was replaced in place by a newer file (atomic write), not emptied."
case dropperDemotedBackupState:
details += "\nDemoted: content matches BackWPup job state written behind a PHP comment; only the leading bytes were seen."
case dropperDemotedPluginCopy:
if root, pc, ok := dropperPluginCopySource(c.Path, c.Docroot); ok {
source := filepath.Join(root, filepath.FromSlash(pc.rel))
details += "\nDemoted: content is byte for byte the installed plugin's file " +
dropperPrintable([]byte(source), dropperPathExcerptMax) +
", which remains on disk; the plugin's official checksums were not available to prove that file unmodified."
}
}
}
if len(c.Head) > 0 {
details += "\nLeading bytes: " + dropperPrintable(c.Head, dropperHeadExcerptMax)
}
msg := fmt.Sprintf("Self-deleting file under web root: %s",
dropperPrintable([]byte(c.Path), dropperPathExcerptMax))
return sev, msg, details, c.Path, dropperDedupKey("file", c.Path, sev)
}
// dropperDedupKey hashes the path: an attacker names it, and alert state
// keeps one key per condition.
func dropperDedupKey(scope, path string, sev alert.Severity) string {
digest := sha256.Sum256([]byte(path))
return fmt.Sprintf("%s:%x:%s", scope, digest[:16], sev)
}
// dropperPrintable renders up to max bytes of b with control and non-ASCII
// bytes replaced by '.' so binary heads (ELF droppers) cannot corrupt
// alert transports or terminal output.
func dropperPrintable(b []byte, max int) string {
if len(b) > max {
b = b[:max]
}
out := make([]byte, len(b))
for i, c := range b {
if c >= 0x20 && c < 0x7f {
out[i] = c
} else {
out[i] = '.'
}
}
return string(out)
}
package daemon
import (
"fmt"
"os"
"path/filepath"
"sync/atomic"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
)
var escalateRealtimePHP = checks.EscalateRealtimePHP
// A completed analysis can arrive beside a scheduled response for the same
// path. Preserve that response's provenance even when dedup keeps our result.
func preserveEscalationResponseProvenance(findings []alert.Finding) {
evaluated := make(map[string]bool)
for _, f := range findings {
if f.AutoFileResponseEvaluated && f.FilePath != "" {
evaluated[filepath.Clean(f.FilePath)] = true
}
}
for i, f := range findings {
if f.FileResponseIdentity != "" && evaluated[filepath.Clean(f.FilePath)] {
findings[i].AutoFileResponseEvaluated = true
}
}
}
// The queue has the same bound as alert intake. A single worker keeps slow
// regular-file reads out of dispatch without spawning unbounded readers.
func (d *Daemon) queuePHPEscalations(findings []alert.Finding) {
for _, f := range findings {
select {
case <-d.stopCh:
return
default:
}
select {
case d.phpEscalations <- f:
default:
atomic.AddInt64(&d.droppedAlerts, 1)
fmt.Fprintf(os.Stderr, "[%s] PHP escalation queue full; original alert retained\n", ts())
}
}
}
func (d *Daemon) phpEscalationWorker() {
defer d.wg.Done()
for {
select {
case <-d.stopCh:
return
case f := <-d.phpEscalations:
select {
case <-d.stopCh:
return
default:
}
if d.store.IsSuppressed(f, d.store.LoadSuppressions()) {
continue
}
findings := escalateRealtimePHP(d.currentCfg(), []alert.Finding{f})
// Returning through intake gives results normal timestamps, state
// deduplication, history, suppression and incident correlation.
d.enqueueScanAlerts(findings, "PHP escalation")
}
}
}
//go:build !(linux && bpf)
package daemon
import (
"context"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/config"
)
// execBPF is the no-tag placeholder for the BPF exec-monitor backend. The
// real type with a tracepoint link and ringbuf reader lives in exec_bpf.go
// behind //go:build linux && bpf.
type execBPF struct{}
func (e *execBPF) Mode() string { return "bpf" }
func (e *execBPF) EventCount() uint64 { return 0 }
func (e *execBPF) Run(_ context.Context) {}
func startExecBPF(_ context.Context, _ chan<- alert.Finding, _ *config.Config) (*execBPF, error) {
return nil, bpf.ErrNotBuilt
}
// Code generated by bpf2go; DO NOT EDIT.
//go:build 386 || amd64
package exec_bpfprog
import (
"bytes"
_ "embed"
"fmt"
"io"
"structs"
"github.com/cilium/ebpf"
)
type ExecCsmQueueStats struct {
_ structs.HostLayout
Lost uint64
Submitted uint64
}
type ExecExecEvent struct {
_ structs.HostLayout
Uid uint32
Pid uint32
Ppid uint32
Comm [16]uint8
ParentComm [16]uint8
Filename [256]uint8
}
// Names of all BPF objects in the ELF.
//
// Used for safe lookups in a Collection or CollectionSpec.
const (
ExecMapEvents = "events"
ExecMapQueueStats = "queue_stats"
ExecProgCsmOnExec = "csm_on_exec"
ExecVarUnused = "unused"
)
// LoadExec returns the embedded CollectionSpec for Exec.
func LoadExec() (*ebpf.CollectionSpec, error) {
reader := bytes.NewReader(_ExecBytes)
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
if err != nil {
return nil, fmt.Errorf("can't load Exec: %w", err)
}
return spec, err
}
// LoadExecObjects loads Exec and converts it into a struct.
//
// The following types are suitable as obj argument:
//
// *ExecObjects
// *ExecPrograms
// *ExecMaps
//
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
func LoadExecObjects(obj any, opts *ebpf.CollectionOptions) error {
spec, err := LoadExec()
if err != nil {
return err
}
return spec.LoadAndAssign(obj, opts)
}
// ExecSpecs contains maps and programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type ExecSpecs struct {
ExecProgramSpecs
ExecMapSpecs
ExecVariableSpecs
}
// ExecProgramSpecs contains programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type ExecProgramSpecs struct {
CsmOnExec *ebpf.ProgramSpec `ebpf:"csm_on_exec"`
}
// ExecMapSpecs contains maps before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type ExecMapSpecs struct {
Events *ebpf.MapSpec `ebpf:"events"`
QueueStats *ebpf.MapSpec `ebpf:"queue_stats"`
}
// ExecVariableSpecs contains global variables before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type ExecVariableSpecs struct {
Unused *ebpf.VariableSpec `ebpf:"unused"`
}
// ExecObjects contains all objects after they have been loaded into the kernel.
//
// It can be passed to LoadExecObjects or ebpf.CollectionSpec.LoadAndAssign.
type ExecObjects struct {
ExecPrograms
ExecMaps
ExecVariables
}
func (o *ExecObjects) Close() error {
return _ExecClose(
&o.ExecPrograms,
&o.ExecMaps,
)
}
// ExecMaps contains all maps after they have been loaded into the kernel.
//
// It can be passed to LoadExecObjects or ebpf.CollectionSpec.LoadAndAssign.
type ExecMaps struct {
Events *ebpf.Map `ebpf:"events"`
QueueStats *ebpf.Map `ebpf:"queue_stats"`
}
func (m *ExecMaps) Close() error {
return _ExecClose(
m.Events,
m.QueueStats,
)
}
// ExecVariables contains all global variables after they have been loaded into the kernel.
//
// It can be passed to LoadExecObjects or ebpf.CollectionSpec.LoadAndAssign.
type ExecVariables struct {
Unused *ebpf.Variable `ebpf:"unused"`
}
// ExecPrograms contains all programs after they have been loaded into the kernel.
//
// It can be passed to LoadExecObjects or ebpf.CollectionSpec.LoadAndAssign.
type ExecPrograms struct {
CsmOnExec *ebpf.Program `ebpf:"csm_on_exec"`
}
func (p *ExecPrograms) Close() error {
return _ExecClose(
p.CsmOnExec,
)
}
func _ExecClose(closers ...io.Closer) error {
for _, closer := range closers {
if err := closer.Close(); err != nil {
return err
}
}
return nil
}
// Do not access this directly.
//
//go:embed exec_x86_bpfel.o
var _ExecBytes []byte
package daemon
import (
"encoding/binary"
"errors"
)
// ExecEvent matches struct exec_event in exec.bpf.c byte for byte:
// scalars are little-endian (host order on amd64/arm64), comm/parent_comm
// are 16-byte null-padded, filename is 256-byte null-padded.
type ExecEvent struct {
UID uint32
PID uint32
PPID uint32
Comm string
ParentComm string
Filename string
}
const execEventSize = 4 + 4 + 4 + 16 + 16 + 256
func decodeExecEvent(b []byte) (ExecEvent, error) {
if len(b) < execEventSize {
return ExecEvent{}, errors.New("exec event short buffer")
}
ev := ExecEvent{
UID: binary.LittleEndian.Uint32(b[0:4]),
PID: binary.LittleEndian.Uint32(b[4:8]),
PPID: binary.LittleEndian.Uint32(b[8:12]),
}
ev.Comm = nullTerm(b[12:28])
ev.ParentComm = nullTerm(b[28:44])
ev.Filename = nullTerm(b[44 : 44+256])
return ev, nil
}
package daemon
import (
"context"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/state"
)
// execPoller wraps the periodic CheckSuspiciousProcesses + CheckFakeKernelThreads
// in a goroutine. Used when the BPF backend is unavailable or operator-disabled.
// Detection latency for already-running processes equals the poll interval (default
// 30 minutes, matching the existing deep-tier cadence).
type execPoller struct {
cfg *config.Config
alertCh chan<- alert.Finding
count atomic.Uint64
}
func newExecPoller(cfg *config.Config, alertCh chan<- alert.Finding) *execPoller {
return &execPoller{cfg: cfg, alertCh: alertCh}
}
func (p *execPoller) Mode() string { return "legacy" }
func (p *execPoller) EventCount() uint64 { return p.count.Load() }
func (p *execPoller) Run(ctx context.Context) {
interval := execPollerInterval(p.cfg)
t := time.NewTicker(interval)
defer t.Stop()
emit := func(fs []alert.Finding) {
for _, f := range fs {
p.count.Add(1)
if !alert.TryEnqueue(p.alertCh, f) {
csmlog.Warn("exec legacy: alert channel full, dropping finding")
}
}
}
for {
select {
case <-ctx.Done():
return
case <-t.C:
emit(checks.CheckSuspiciousProcesses(ctx, p.cfg, (*state.Store)(nil)))
emit(checks.CheckFakeKernelThreads(ctx, p.cfg, (*state.Store)(nil)))
}
}
}
func execPollerInterval(cfg *config.Config) time.Duration {
if d := cfg.Detection.ExecMonitorPollInterval; d > 0 {
return d
}
return 30 * time.Minute
}
package daemon
import (
"context"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/processctx"
)
// StartExecMonitor starts the live exec monitor chosen by
// detection.exec_monitor_backend; see startLiveBackend for the choices.
func StartExecMonitor(alertCh chan<- alert.Finding, cfg *config.Config) bpf.Backend {
return startLiveBackend("exec_monitor", cfg.Detection.ExecMonitorBackend, alertCh,
func() (bpf.Backend, error) { return tryStartExecBPFFn(context.Background(), alertCh, cfg) },
func() bpf.Backend { return newExecPoller(cfg, alertCh) })
}
// tryStartExecBPFFn is the package-level indirection so tests can substitute
// a fake without the bpf build tag.
var tryStartExecBPFFn = tryStartExecBPF
func tryStartExecBPF(ctx context.Context, ch chan<- alert.Finding, cfg *config.Config) (bpf.Backend, error) {
b, err := startExecBPF(ctx, ch, cfg)
if err != nil {
return nil, err
}
return b, nil
}
// populateProcessCtxFromExec writes the BPF exec event into the process
// context cache without doing identity work in the event loop. Zero PID is
// ignored (synthetic boot-time noise).
func populateProcessCtxFromExec(cache *processctx.Cache, ev ExecEvent, startedAt time.Time) {
if ev.PID == 0 {
return
}
cache.PutFromExecStartedAt(int(ev.PID), int(ev.PPID), int(ev.UID), ev.Comm, ev.Filename, startedAt)
}
func attachProcessCtxToExecFinding(cache *processctx.Cache, f *alert.Finding, ev ExecEvent) {
req := processctx.EnrichRequest{
PID: int(ev.PID),
UID: int(ev.UID),
UIDKnown: true,
Comm: ev.Comm,
StartedAt: processCtxStartedAt(int(ev.PID)),
}
if pc, _ := cache.MaterializeVerifiedSnapshot(req); pc != nil {
f.Process = pc
}
}
// processctxRequestFromExec builds the EnrichRequest snapshot used by the BPF
// exec consumer. Lives here (no build tag) so unit tests on darwin can verify
// the field mapping without a Linux+bpf build.
func processctxRequestFromExec(ev ExecEvent) processctx.EnrichRequest {
pid := int(ev.PID)
return processctx.EnrichRequest{
PID: pid,
UID: int(ev.UID),
UIDKnown: true,
Comm: ev.Comm,
StartedAt: processCtxStartedAt(pid),
}
}
//go:build linux
package daemon
import (
"crypto/sha256"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"runtime"
"runtime/debug"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"unsafe"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/contenttype"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/signatures"
"github.com/pidginhost/csm/internal/wpcheck"
"github.com/pidginhost/csm/internal/yara"
)
// fanotify constants (not all in Go stdlib)
const (
FAN_MARK_ADD = 0x00000001
// FAN_MARK_MOUNT covers a single vfsmount. FAN_MARK_FILESYSTEM marks the
// whole superblock, so a write that reaches the same inode through a bind
// mount is reported too. EL8 backported the flag into 4.18, which is what
// CloudLinux 8 runs, so cages are reachable on production kernels.
FAN_MARK_MOUNT = 0x00000010
FAN_MARK_FILESYSTEM = 0x00000100
FAN_CLOSE_WRITE = 0x00000008
FAN_CLASS_NOTIF = 0x00000000
FAN_CLOEXEC = 0x00000001
FAN_NONBLOCK = 0x00000002
)
// markFunc is the fanotify_mark syscall, passed in so the ladder can be
// exercised without a kernel and without a mutable package-level seam that
// concurrent tests would race on.
type markFunc func(fd int, flags uint, mask uint64, dirFd int, path string) error
// markScope records how widely a watch root ended up being marked.
type markScope int
const (
markScopeNone markScope = iota
markScopeFilesystem
markScopeMount
)
func (s markScope) String() string {
switch s {
case markScopeFilesystem:
return "filesystem"
case markScopeMount:
return "mount"
default:
return "none"
}
}
// markWatchRoot watches path, preferring a filesystem-scoped mark.
//
// A mount-scoped mark sees only the vfsmount it was added to. Every CloudLinux
// CageFS account reaches its files through a bind mount of the same superblock,
// so writes inside a cage produced no event at all and the realtime scanner was
// blind for precisely the accounts most likely to be compromised. Marking the
// superblock covers every mount of it.
//
// Kernels without FAN_MARK_FILESYSTEM fall back to the mount mark. Close
// after write is the only event: FAN_CREATE needs an FID-reporting group,
// which this one is not, so a mask carrying it is always rejected.
func markWatchRoot(fd int, path string, mark markFunc) (markScope, error) {
var lastErr error
for _, attempt := range []struct {
scope markScope
flags uint
}{
{markScopeFilesystem, FAN_MARK_ADD | FAN_MARK_FILESYSTEM},
{markScopeMount, FAN_MARK_ADD | FAN_MARK_MOUNT},
} {
if err := mark(fd, attempt.flags, FAN_CLOSE_WRITE, -1, path); err != nil {
lastErr = err
continue
}
return attempt.scope, nil
}
return markScopeNone, lastErr
}
// watchRootMark records what a watch root ended up being watched by. coveredBy
// names the root whose filesystem-scoped mark covers this one. A later
// filesystem mark can also cover an earlier mount mark.
type watchRootMark struct {
path string
scope markScope
device uint64
coveredBy string
// rootFS records that this root sits on the same filesystem as /, so a
// filesystem-scoped mark on it raises an event for every write on the
// machine. That is device identity, not an inference about containment.
rootFS bool
}
// devStatFunc reports the device a path lives on. Injected so the mark ladder
// can be exercised without a filesystem laid out like a production host.
type devStatFunc func(path string) (uint64, error)
func statDevice(path string) (uint64, error) {
var st unix.Stat_t
if err := unix.Stat(path, &st); err != nil {
return 0, err
}
return uint64(st.Dev), nil // #nosec G115 -- device ids are non-negative on Linux
}
// markWatchRoots skips roots covered by a successful filesystem mark, never
// by a mount mark. The returned error includes partial failures so startup can
// report unwatched roots while continuing with the marks that succeeded.
func markWatchRoots(fd int, roots []string, mark markFunc, statDev devStatFunc) ([]watchRootMark, error) {
var marks []watchRootMark
covering := make(map[uint64]string)
var failures []error
rootDevice, rootErr := statDev("/")
for _, path := range roots {
device, statErr := statDev(path)
if statErr != nil {
if !errors.Is(statErr, os.ErrNotExist) {
failures = append(failures, fmt.Errorf("cannot inspect watch root %s: %w", path, statErr))
}
continue
}
if owner, covered := covering[device]; covered {
marks = append(marks, watchRootMark{
path: path, scope: markScopeFilesystem, device: device, coveredBy: owner,
rootFS: rootErr == nil && device == rootDevice,
})
continue
}
scope, err := markWatchRoot(fd, path, mark)
if err != nil {
failures = append(failures, fmt.Errorf("cannot watch %s: %w", path, err))
continue
}
if scope == markScopeFilesystem {
covering[device] = path
}
marks = append(marks, watchRootMark{
path: path, scope: scope, device: device,
rootFS: rootErr == nil && device == rootDevice,
})
}
// A later filesystem mark makes an earlier mount-only warning obsolete.
for i := range marks {
if owner, covered := covering[marks[i].device]; covered && owner != marks[i].path {
marks[i].scope = markScopeFilesystem
marks[i].coveredBy = owner
}
}
if len(marks) == 0 && len(failures) == 0 {
failures = append(failures, fmt.Errorf("no watch root exists"))
}
return marks, errors.Join(failures...)
}
// EventStats reports completed admission decisions for delivered file events.
type EventStats struct {
Received int64
Admitted int64
Dropped int64
// PlacementUnverified counts placement reports dropped because the path
// never opened as the reported file or directory. They reach no
// admission decision.
PlacementUnverified int64
// PlacementSkipped counts placed files not analysed again because the
// same version was analysed recently under the same conditions.
PlacementSkipped int64
}
// Filtered counts events rejected before queue admission, including events
// whose file descriptor could not be resolved to a path.
func (s EventStats) Filtered() int64 { return s.Received - s.Admitted - s.Dropped }
// EventStats returns a coherent snapshot. In-progress admission decisions are
// published with their outcome so they cannot appear as filtered events.
func (fm *FileMonitor) EventStats() EventStats {
fm.eventStatsMu.Lock()
defer fm.eventStatsMu.Unlock()
stats := fm.eventStats
stats.PlacementUnverified = fm.placementUnverified.Load()
stats.PlacementSkipped = fm.placementSkipped.Load()
return stats
}
func (fm *FileMonitor) recordEvent(admitted, dropped bool) {
fm.eventStatsMu.Lock()
defer fm.eventStatsMu.Unlock()
fm.eventStats.Received++
if fanotifyEventsTotal != nil {
fanotifyEventsTotal.Inc()
}
if admitted {
fm.eventStats.Admitted++
if fanotifyEventsAdmittedTotal != nil {
fanotifyEventsAdmittedTotal.Inc()
}
}
if dropped {
fm.eventStats.Dropped++
if fanotifyDroppedTotal != nil {
fanotifyDroppedTotal.Inc()
}
}
}
// WatchScopeSummary describes what the monitor actually watches.
func (fm *FileMonitor) WatchScopeSummary() string { return watchScopeSummary(fm.watchRoots) }
// A mount point does not establish filesystem containment: symlinks and bind
// mounts can name a subtree of the same superblock. Describe the scope itself
// instead of inferring a boundary by comparing a root's device to its parent's.
func watchScopeSummary(marks []watchRootMark) string {
parts := make([]string, 0, len(marks))
for _, m := range marks {
if m.coveredBy != "" {
parts = append(parts, fmt.Sprintf("%s (covered by %s)", m.path, m.coveredBy))
continue
}
coverage := "whole filesystem, including all bind mounts"
if m.scope == markScopeMount {
coverage = "whole containing mount, excluding other bind mounts"
}
if m.rootFS {
coverage += ", same filesystem as /"
}
parts = append(parts, fmt.Sprintf("%s (%s scope, device %d: %s)", m.path, m.scope, m.device, coverage))
}
return strings.Join(parts, ", ")
}
// fanotifyEventMetadata is the header for each fanotify event.
type fanotifyEventMetadata struct {
EventLen uint32
Vers uint8
Reserved uint8
MetadataLen uint16
Mask uint64
Fd int32
Pid int32
}
const metadataSize = int(unsafe.Sizeof(fanotifyEventMetadata{}))
const htaccessRealtimeMaxBytes = 1 << 20
// M1 - webshells map at package level (avoid per-call allocation)
var knownWebshells = map[string]bool{
"h4x0r.php": true, "c99.php": true, "r57.php": true,
"wso.php": true, "alfa.php": true, "b374k.php": true,
"shell.php": true, "cmd.php": true, "backdoor.php": true,
"webshell.php": true,
}
// M3 - WordPress path stat cache with TTL
type wpPathCacheEntry struct {
exists bool
ts time.Time
}
var wpPathStatCache sync.Map // key: path string → value: wpPathCacheEntry
const wpPathCacheTTL = 5 * time.Minute
// alertDedupTTL is the cooldown period for duplicate alerts on the same
// check+filepath combination. Prevents alert storms from rapid writes.
const alertDedupTTL = 30 * time.Second
// FileMonitor watches mount points for file creation/modification using fanotify.
type FileMonitor struct {
fd int
cfg *config.Config
alertCh chan<- alert.Finding
// panicMu / lastPanicAt rate-limit the realtime_scanner_panic finding
// raised when an analyzer panics on one event (see analyzeFileSafe).
panicMu sync.Mutex
lastPanicAt time.Time
analyzerCh chan fileEvent
queueHealthOnce sync.Once
analyzerHealth *queuehealth.Tracker
reconcileHealth *queuehealth.Tracker
kernelQueueHealth *queuehealth.Tracker
kernelQueue *notificationQueue
// Publish each completed decision atomically with its lifetime totals.
// The minute overflow alert drains droppedEvents independently.
eventStatsMu sync.Mutex
eventStats EventStats
droppedEvents int64
droppedAlerts int64
// placementUnverified and placementSkipped are published through
// EventStats; the process-wide metrics count the same events.
placementUnverified atomic.Int64
placementSkipped atomic.Int64
// recent is nil unless a placement backend runs: nothing else reads it,
// and keeping it costs a read and hash per analysis.
recent *recentAnalyses
// placementCreates holds the running fanotify placement backend's
// creates until their own close-write settles them; nil otherwise.
placementCreates atomic.Pointer[createSettler]
// queueOverflows counts FAN_Q_OVERFLOW events: the kernel notification
// queue filled and events were dropped before userspace ever saw them.
// Distinct from droppedEvents (analyzer-queue backpressure in userspace)
// because a kernel overflow carries no fd, so the affected files are
// unknown and cannot be reconciled by path.
queueOverflows int64
// overflowReportMu rate-limits the operator-facing kernel-overflow finding
// so a sustained storm does not flood the alert channel.
overflowReportMu sync.Mutex
lastOverflowReport time.Time
yaraErrorReportMu sync.Mutex
lastYARAError time.Time
// C4 - pipe for epoll stop signaling
pipeFds [2]int // [0]=read, [1]=write
pipeClosed int32 // atomic flag: 1 = pipe fds closed by drainAndClose
// drainDeadline is the unix-nano instant after which analyzer workers
// stop scanning queued events and only release them. Zero until the
// shutdown drain arms it.
drainDeadline atomic.Int64
// C2 - sync.Once for safe Stop
stopOnce sync.Once
drainOnce sync.Once
stopCh chan struct{} // internal stop channel
wg sync.WaitGroup
// Per-path alert deduplication: "check:filepath" → last alert time
alertDedup sync.Map
// tmpVouches holds the temp work directories a proven writer wrote into.
tmpVouches tmpVouches
// accountRootPatterns and docRootPatterns describe where accounts and their
// document roots live on this platform. The realtime detectors used to
// hardcode /home and /public_html, which made every one of them dead on
// Plesk and DirectAdmin and on cPanel accounts outside /home.
accountRootPatterns []string
docRootPatterns []string
// webRootPatterns is the immutable PHP configuration root set captured at
// startup from account_roots and platform discovery.
webRootPatterns []string
// watchRoots records what each watch root ended up watching, including
// roots an earlier filesystem-scoped mark already covers.
watchRoots []watchRootMark
// WordPress checksum verifier: skips detection on unmodified core and
// plugin files and judges staged update packages file by file.
wpCache wpVerifier
// wpPending holds staged package files whose checksums are still being
// fetched; stagedPackageLoop resolves them once a second.
wpPending *stagedPackageQueue
wpPendingInit sync.Once
// Drop-recovery reconcile: directories that had fanotify events dropped
// because the analyzer queue was full. The overflow reporter schedules
// bounded passes over interesting files within the original recovery
// window so bulk filesystem operations (unzip, backup restore)
// do not blind detection to actual threats landing in the storm.
reconcileMu sync.Mutex
reconcileDirs map[string]reconcileDirectory
// reconcileSig is a buffered cap-1 channel that lets sendEvent's drop
// branch nudge overflowReporter to run reconcileDrops out of cycle
// when sustained drops cross eagerReconcileDropThreshold. The cap-1
// shape collapses multiple triggers in the same window into one and
// keeps sendEvent non-blocking on the event-loop hot path.
reconcileSig chan struct{}
// reconcileRunning is set while a recovery pass is in flight, and
// reconcileLastPass is the unix-nano instant the last one ended. Passes
// are single-flight and rate-limited: recovery that keeps up with a
// storm's trigger rate feeds the storm.
reconcileRunning atomic.Bool
reconcileLastPass atomic.Int64
// Directories renamed or linked into a watched tree wait here for the
// bounded walk that checks the files inside them. placedDirSig wakes
// the walk worker; placedDirWG lets shutdown join it before closing the
// analyzer queue it feeds.
placedDirMu sync.Mutex
placedDirs []placedDir
placedDirSig chan struct{}
placedDirWG sync.WaitGroup
placedDirHealth *queuehealth.Tracker
// placementWG joins the placement backend, whose workers admit into
// analyzerCh, before shutdown closes that channel.
placementWG sync.WaitGroup
// metricsOnce guards one-time registration of the fanotify-scoped
// Prometheus metrics. Each FileMonitor registers its own hooks when
// it first starts; subsequent calls are a no-op.
metricsOnce sync.Once
// dropper drives the self-deleting-dropper detector: candidates are
// admitted from the analyzer path and probed for deletion after a TTL by
// dropperProbeLoop. nil when thresholds.dropper_detection is off.
dropper *dropperEngine
// dropperDocroots indexes the web document roots the admission gate
// matches paths against, refreshed on the probe loop's cadence so
// account add/remove is picked up without a restart.
dropperDocroots atomic.Pointer[checks.DocrootIndex]
// dropperQuarantines records exact snapshots of files CSM moved out of
// their original paths, so the later absence probe does not misclassify
// CSM's own remediation as attacker self-deletion.
dropperQuarantines *dropperQuarantineLedger
// dropperOverflowReported is the last cumulative tracker-overflow count
// surfaced to the operator. Only the probe goroutine accesses it.
dropperOverflowReported uint64
lastDropperOverflowReport time.Time
// dropperHandlerCache shares immutable inherited .htaccess PHP mappings
// across analyzer events. The generation changes as soon as a .htaccess
// event reaches the reader, invalidating every inherited snapshot.
dropperHandlerMu sync.Mutex
dropperHandlerCache map[string]dropperPHPHandlerCacheEntry
dropperHandlerGeneration uint64
}
const (
// reconcileDirCap bounds the dirty-region tracker fed by sendEvent's
// drop branch. A 2026-04-28 cpanel package restore overflowed the
// previous 64-entry cap inside seconds (every wp-content subdir was
// a distinct parent), evicting older dirs before reconcileDrops ran.
// 1024 entries fits typical cpanel restore bursts while bounding the
// retained paths, scan cursors and queue tickets.
reconcileDirCap = 1024
// reconcileWindow scopes which files reconcileDrops will rescan: only
// files whose mtime is within this window of the first pass. The cutoff
// stays fixed across retries. Sized just over
// the minute tick so a drop near the start of a tick is still picked
// up by the reconcile that runs at tick end.
reconcileWindow = 70 * time.Second
// reconcileBudget bounds one recovery pass. The pass reads whole
// directories and scans their recent files inline, outside the analyzer
// pool's backpressure, so an unbounded pass competes with the very
// workers whose backlog caused the drops.
reconcileBudget = 10 * time.Second
// reconcileMinInterval is the shortest gap between recovery passes. The
// eager trigger fires once per few hundred drops, which during a
// sustained storm meant a pass every few seconds: recovery rescanned
// files that were still being written, spent CPU the analyzers needed,
// and produced the next round of drops.
reconcileMinInterval = 30 * time.Second
// analyzerChBufferSize sizes the channel feeding the analyzer pool.
// A cpanel package restore in production observed ~4189 events in a
// few seconds; a 16 KiB buffer absorbs that burst plus headroom
// without ever overflowing. Memory cost is bounded (fileEvent is a
// path string + fd + pid, ~40 bytes each, so <1 MiB at full buffer).
analyzerChBufferSize = 16384
// analyzerDrainBudget bounds how long the analyzer pool keeps scanning
// queued events after shutdown starts. The queue is deep on purpose, so
// a host that is saturated at SIGTERM has thousands of events left; the
// unit gives the daemon 90 seconds in total, and draining that backlog
// to completion spent all of it and ended in SIGKILL.
analyzerDrainBudget = 15 * time.Second
// eagerReconcileDropThreshold triggers an out-of-cycle reconcile
// when sustained drops cross this count within a single minute tick.
// Without this, drops happening just after a tick wait the full
// interval before reconcileDrops walks them - long enough for the
// reconcileWindow to expire on the earliest dropped files.
eagerReconcileDropThreshold = 500
)
// Package-level Prometheus metrics for fanotify. Instantiated once per
// process; one FileMonitor per daemon instance reuses them.
var (
fanotifyEventsTotal *metrics.Counter
fanotifyEventsAdmittedTotal *metrics.Counter
fanotifyDroppedTotal *metrics.Counter
fanotifyKernelOverflowTotal *metrics.Counter
fanotifyPlacementUnverified *metrics.Counter
fanotifyPlacementSkipped *metrics.Counter
fanotifyReconcileDur *metrics.Histogram
contentScanTruncated *metrics.CounterVec
)
// registerFanotifyMetrics is called once per FileMonitor via
// fm.metricsOnce. Safe to call multiple times at the FileMonitor
// layer; the package-level sync.Once guards the actual registrations.
var fanotifyMetricsInit sync.Once
func (fm *FileMonitor) registerMetrics() {
fm.metricsOnce.Do(func() {
fanotifyMetricsInit.Do(func() {
fanotifyEventsTotal = metrics.NewCounter(
"csm_fanotify_events_total",
"Delivered fanotify file events with completed admission decisions, including events rejected before analysis. Excludes kernel queue overflow notifications, which have their own counter.",
)
metrics.MustRegister("csm_fanotify_events_total", fanotifyEventsTotal)
fanotifyEventsAdmittedTotal = metrics.NewCounter(
"csm_fanotify_events_admitted_total",
"Fanotify file events queued for analysis, including dropper tracking. Subtract this and csm_fanotify_events_dropped_total from csm_fanotify_events_total to count events rejected before queue admission.",
)
metrics.MustRegister("csm_fanotify_events_admitted_total", fanotifyEventsAdmittedTotal)
fanotifyDroppedTotal = metrics.NewCounter(
"csm_fanotify_events_dropped_total",
"Fanotify events dropped because the analyzer queue was full. Sustained growth indicates an event storm (bulk unzip, backup restore) or an attack producing more file activity than the scanner can analyse; the reconcile pass still rescans affected directories, so dropped events do not vanish from detection, they arrive delayed.",
)
metrics.MustRegister("csm_fanotify_events_dropped_total", fanotifyDroppedTotal)
fanotifyKernelOverflowTotal = metrics.NewCounter(
"csm_fanotify_kernel_queue_overflow_total",
"FAN_Q_OVERFLOW events: the kernel fanotify notification queue filled and dropped events before userspace read them. Unlike analyzer-queue drops these carry no fd, so the affected files are unknown; the next scheduled deep scan is the backstop. Sustained growth means a storm (bulk unzip, backup restore) or an attack producing more file activity than the reader can drain.",
)
metrics.MustRegister("csm_fanotify_kernel_queue_overflow_total", fanotifyKernelOverflowTotal)
fanotifyPlacementUnverified = metrics.NewCounter(
"csm_fanotify_placement_unverified_total",
"Files and directories renamed, hard-linked or created into watched trees that did not open as the file the placement backend reported, so they reached no admission decision. A file replaced again right after it was placed lands here; sustained growth means placements are being raced.",
)
metrics.MustRegister("csm_fanotify_placement_unverified_total", fanotifyPlacementUnverified)
fanotifyPlacementSkipped = metrics.NewCounter(
"csm_fanotify_placement_skipped_total",
"Files placed into watched trees and not analysed again because the same version was analysed recently under the same conditions, such as an atomic save renamed to its final name.",
)
metrics.MustRegister("csm_fanotify_placement_skipped_total", fanotifyPlacementSkipped)
fanotifyReconcileDur = metrics.NewHistogram(
"csm_fanotify_reconcile_latency_seconds",
"How long the post-overflow reconcile pass takes to walk drop-affected directories and rescan recent files. Buckets sized for the observed range; alert if p99 crosses tens of seconds (reconcile is stealing CPU from real-time analysis).",
[]float64{0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60},
)
metrics.MustRegister("csm_fanotify_reconcile_latency_seconds", fanotifyReconcileDur)
metrics.RegisterGaugeFunc(
"csm_fanotify_queue_depth",
"Current number of queued fanotify events waiting for the analyzer pool. Capacity is 4000; queue approaching that cap means drops are imminent.",
func() float64 {
if fm == nil || fm.analyzerCh == nil {
return 0
}
return float64(len(fm.analyzerCh))
},
)
contentScanTruncated = metrics.NewCounterVec(
"csm_realtime_content_scan_truncated_total",
"Real-time fanotify content checks whose file was larger than the main read window, so the full-rule pass saw only the leading window. Labels: check (phpcontent_inline, phpcontent_uploads, php_check, crontab, htaccess, user_ini, html_phishing, cgi_backdoor).",
[]string{"check"},
)
metrics.MustRegister("csm_realtime_content_scan_truncated_total", contentScanTruncated)
})
})
}
// recordReadTruncation increments csm_realtime_content_scan_truncated_total
// when the file behind fd is larger than maxBytes. Cheap fstat per scan.
// No-op if the counter has not been registered (test setups that skip
// registerMetrics).
func recordReadTruncation(fd int, maxBytes int, check string) {
if contentScanTruncated == nil {
return
}
var st unix.Stat_t
if err := unix.Fstat(fd, &st); err != nil {
return
}
if st.Size > int64(maxBytes) {
contentScanTruncated.With(check).Inc()
}
}
type fileEvent struct {
queueTicket queuehealth.Ticket
path string
fd int
pid int32
mask uint64
dropperOnly bool
phpExecutable bool
// at is when the event was read, the latest moment the writer is known
// to have existed under its PID.
at time.Time
writerChain []int
// tmpAncestry is what the writer's process chain proved when the event
// was admitted, for a root-owned file under a temp root. Nil when no
// capture ran.
tmpAncestry *ancestryEvidence
// Remember rejected captures too: later ownership changes must not trigger a new walk.
tmpCaptured bool
tmpStat *unix.Stat_t
// cacheable is set only by a completed clean content scan. Other
// detector paths may depend on state outside the placement cache key.
cacheable *bool
}
func (fm *FileMonitor) currentCfg() *config.Config {
if cfg := config.Active(); cfg != nil {
return cfg
}
if fm == nil {
return nil
}
return fm.cfg
}
// NewFileMonitor creates a fanotify-based file monitor.
// Returns error if the kernel doesn't support the required features.
func NewFileMonitor(cfg *config.Config, alertCh chan<- alert.Finding) (*FileMonitor, error) {
// H1 - use golang.org/x/sys/unix for fanotify_init
fd, err := unix.FanotifyInit(FAN_CLASS_NOTIF|FAN_CLOEXEC|FAN_NONBLOCK, unix.O_RDONLY)
if err != nil {
return nil, fmt.Errorf("fanotify_init: %w (kernel may not support fanotify)", err)
}
// Mark mount points; M2 - track successful mounts
webRootPatterns := checks.PHPConfigRealtimeRootPatterns(cfg)
mountPaths := fanotifyMountPaths(webRootPatterns)
marks, markErr := markWatchRoots(fd, mountPaths, unix.FanotifyMark, statDevice)
if len(marks) == 0 {
_ = unix.Close(fd)
return nil, fmt.Errorf("no mount points could be watched (tried %v): %w", mountPaths, markErr)
}
if markErr != nil {
fmt.Fprintf(os.Stderr, "[%s] Warning: %v\n", ts(), markErr)
}
var mountScoped []string
for _, m := range marks {
if m.coveredBy == "" && m.scope == markScopeMount {
mountScoped = append(mountScoped, m.path)
}
}
if len(mountScoped) > 0 {
// Worth saying out loud: on these roots a write that arrives through a
// bind mount (a CageFS cage) raises no event, and only the rolling
// content scan will meet it.
fmt.Fprintf(os.Stderr, "[%s] Warning: watching %v per-mount only; writes through bind mounts on them are not seen in real time\n",
ts(), mountScoped)
}
// Directory-scoped watch on /var/spool/cron so any user crontab write
// reaches analyzeFile in real time. Best-effort: cron may live under a
// different path on non-cPanel hosts (the platform layer normalises),
// and we'd rather lose the realtime crontab signal than fail daemon
// startup. The polled CheckCrontabs run still covers this case via
// the next scheduled scan. Mask matches spoolwatch.go (the proven
// production pattern for directory-scoped marks): FAN_CLOSE_WRITE
// alone catches both new and modified crontabs, since the close
// after O_CREAT|O_WRONLY|... fires the close-write event.
if _, statErr := os.Stat(cronSpoolDir()); statErr == nil {
if err := unix.FanotifyMark(fd, FAN_MARK_ADD,
FAN_CLOSE_WRITE|FAN_EVENT_ON_CHILD, -1, cronSpoolDir()); err != nil {
fmt.Fprintf(os.Stderr, "[%s] Warning: cannot watch %s: %v\n", ts(), cronSpoolDir(), err)
}
}
// C4 - create pipe for epoll stop signaling
var pipeFds [2]int
if err := unix.Pipe2(pipeFds[:], unix.O_NONBLOCK|unix.O_CLOEXEC); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("pipe2: %w", err)
}
fm := newFileMonitorCore(cfg, alertCh)
fm.fd = fd
fm.watchRoots = marks
fm.pipeFds = pipeFds
// Admission must match the roots the marks were placed from.
fm.webRootPatterns = webRootPatterns
return fm, nil
}
// newFileMonitorCore builds the monitor state the analyzer needs, without the
// fanotify descriptor, its watch roots or the stop pipe.
func newFileMonitorCore(cfg *config.Config, alertCh chan<- alert.Finding) *FileMonitor {
fm := &FileMonitor{
// -1, not 0: a monitor built without fanotify must never close fd 0.
fd: -1,
pipeFds: [2]int{-1, -1},
cfg: cfg,
alertCh: alertCh,
analyzerCh: make(chan fileEvent, analyzerChBufferSize),
stopCh: make(chan struct{}),
reconcileDirs: make(map[string]reconcileDirectory),
reconcileSig: make(chan struct{}, 1),
placedDirSig: make(chan struct{}, 1),
webRootPatterns: checks.PHPConfigRealtimeRootPatterns(cfg),
accountRootPatterns: checks.AccountHomePatterns(),
docRootPatterns: checks.RealtimeDocumentRootPatterns(cfg),
}
wpCache := wpcheck.NewCache(cfg.StatePath)
wpCache.SetStopCh(fm.stopCh)
fm.wpCache = wpCache
// File responses tell a planted file from a shipped one by these lists.
checks.SetPackageListing(wpCache.PackageListing)
fm.wpPending = newStagedPackageQueue(stagedPackageQueueMax)
fm.initDropperDetector(cfg)
return fm
}
func fanotifyMountPaths(webRootPatterns []string) []string {
paths := []string{"/home", "/tmp", "/dev/shm", "/var/tmp"}
seen := make(map[string]struct{}, len(paths)+len(webRootPatterns))
for _, path := range paths {
seen[path] = struct{}{}
}
for _, pattern := range webRootPatterns {
anchor := fanotifyMountAnchor(pattern)
if anchor == "" {
continue
}
if _, exists := seen[anchor]; exists {
continue
}
seen[anchor] = struct{}{}
paths = append(paths, anchor)
}
return paths
}
func fanotifyMountAnchor(pattern string) string {
if strings.TrimSpace(pattern) == "" {
return ""
}
clean := filepath.Clean(pattern)
if !filepath.IsAbs(clean) {
return ""
}
meta := strings.IndexAny(clean, "*?[")
if meta < 0 {
return clean
}
prefix := clean[:meta]
if strings.HasSuffix(prefix, string(filepath.Separator)) {
return filepath.Clean(prefix)
}
return filepath.Dir(prefix)
}
const (
// minAnalyzerWorkers keeps one write from stalling every other event
// while it is scanned, without putting four content scans on a host that
// has two cores to serve requests with.
minAnalyzerWorkers = 2
// maxAnalyzerWorkers caps the pool on large hosts: past this the workers
// contend on disk rather than finishing sooner.
maxAnalyzerWorkers = 16
)
// analyzerWorkerCount sizes the analyzer pool from the host's core count.
func analyzerWorkerCount(cpus int) int {
if cpus < minAnalyzerWorkers {
return minAnalyzerWorkers
}
if cpus > maxAnalyzerWorkers {
return maxAnalyzerWorkers
}
return cpus
}
// Run starts the file monitor event loop and analyzer workers.
func (fm *FileMonitor) Run(stopCh <-chan struct{}) {
numWorkers := analyzerWorkerCount(runtime.NumCPU())
for i := 0; i < numWorkers; i++ {
fm.wg.Add(1)
obs.Go("fanotify-analyzer", fm.analyzerWorker)
}
// Start overflow reporter
fm.wg.Add(1)
obs.Go("fanotify-overflow", fm.overflowReporter)
// Resolve staged WordPress package files once their checksums land.
fm.wg.Add(1)
obs.Go("fanotify-wp-package", fm.stagedPackageLoop)
// Walk directories placed into watched trees.
fm.placedDirWG.Add(1)
obs.Go("placement-dir-walk", fm.placedDirLoop)
// Start the self-deleting-dropper probe loop when the detector is enabled.
if fm.dropper != nil {
fm.wg.Add(1)
obs.Go("fanotify-dropper", fm.dropperProbeLoop)
}
// C4 - create epoll instance, watch fanotify fd + pipe read end
epfd, err := unix.EpollCreate1(unix.EPOLL_CLOEXEC)
if err != nil {
fmt.Fprintf(os.Stderr, "[%s] epoll_create1 failed: %v, falling back to poll loop\n", ts(), err)
fm.runPollFallback(stopCh)
return
}
defer func() { _ = unix.Close(epfd) }()
// Add fanotify fd to epoll
if err := unix.EpollCtl(epfd, unix.EPOLL_CTL_ADD, fm.fd, &unix.EpollEvent{
Events: unix.EPOLLIN,
// #nosec G115 -- POSIX fd fits in int32 (rlimit caps fds at ~1024).
Fd: int32(fm.fd),
}); err != nil {
fmt.Fprintf(os.Stderr, "[%s] epoll_ctl(fanotify): %v\n", ts(), err)
fm.runPollFallback(stopCh)
return
}
// Add pipe read end to epoll (for stop signaling)
if err := unix.EpollCtl(epfd, unix.EPOLL_CTL_ADD, fm.pipeFds[0], &unix.EpollEvent{
Events: unix.EPOLLIN,
// #nosec G115 -- POSIX fd fits in int32.
Fd: int32(fm.pipeFds[0]),
}); err != nil {
fmt.Fprintf(os.Stderr, "[%s] epoll_ctl(pipe): %v\n", ts(), err)
fm.runPollFallback(stopCh)
return
}
// Forward external stopCh to our internal mechanism
obs.SafeGo("fanotify-stop-forward", func() {
select {
case <-stopCh:
fm.Stop()
case <-fm.stopCh:
}
})
buf := make([]byte, 4096*24) // Large buffer for event batches
events := make([]unix.EpollEvent, 4)
for {
n, err := unix.EpollWait(epfd, events, 500) // 500ms timeout
if err != nil {
if err == unix.EINTR {
continue
}
// Check if we've been stopped
select {
case <-fm.stopCh:
fm.drainAndClose()
return
default:
}
fmt.Fprintf(os.Stderr, "[%s] epoll_wait error: %v\n", ts(), err)
time.Sleep(1 * time.Second)
continue
}
// Check for stop first
select {
case <-fm.stopCh:
fm.drainAndClose()
return
default:
}
for i := 0; i < n; i++ {
// #nosec G115 -- POSIX fd fits in int32; comparing against epoll event fd.
if events[i].Fd == int32(fm.pipeFds[0]) {
// Stop signal received via pipe
fm.drainAndClose()
return
}
// #nosec G115 -- POSIX fd fits in int32.
if events[i].Fd == int32(fm.fd) {
// fanotify events ready — single read per epoll wake
fm.initQueueHealth()
_, readErr := fm.kernelQueue.read(buf, fm.processEvents)
if readErr != nil {
if readErr != unix.EAGAIN && readErr != unix.EINTR {
fmt.Fprintf(os.Stderr, "[%s] fanotify read error: %v\n", ts(), readErr)
}
}
}
}
}
}
// runPollFallback is used when epoll setup fails; falls back to sleep-based polling.
func (fm *FileMonitor) runPollFallback(stopCh <-chan struct{}) {
// Forward external stopCh to our internal mechanism
obs.SafeGo("fanotify-stop-forward", func() {
select {
case <-stopCh:
fm.Stop()
case <-fm.stopCh:
}
})
buf := make([]byte, 4096*24)
for {
select {
case <-fm.stopCh:
fm.drainAndClose()
return
default:
}
fm.initQueueHealth()
_, err := fm.kernelQueue.read(buf, fm.processEvents)
if err != nil {
if err == unix.EAGAIN || err == unix.EINTR {
time.Sleep(100 * time.Millisecond)
continue
}
fmt.Fprintf(os.Stderr, "[%s] fanotify read error: %v\n", ts(), err)
time.Sleep(1 * time.Second)
continue
}
}
}
// processEvents parses a buffer of fanotify event metadata and dispatches each event.
func (fm *FileMonitor) processEvents(buf []byte) {
readAt := time.Now()
offset := 0
for offset+metadataSize <= len(buf) {
// #nosec G103 -- fanotify delivers a packed binary stream on the
// fd; we must reinterpret the byte buffer as the kernel struct.
// The metadataSize bounds check above guarantees we have enough
// bytes for the struct.
event := (*fanotifyEventMetadata)(unsafe.Pointer(&buf[offset]))
eventLen := int(event.EventLen)
if eventLen < metadataSize || offset+eventLen > len(buf) {
break
}
if event.Mask&unix.FAN_Q_OVERFLOW != 0 {
fm.handleQueueOverflow()
} else if event.Fd >= 0 {
fm.handleEventAt(int(event.Fd), event.Pid, event.Mask, readAt)
}
offset += eventLen
}
}
// handleQueueOverflow reacts to a FAN_Q_OVERFLOW record. The kernel dropped
// events we will never see and gave us no fd, so we cannot reconcile the exact
// files. Count it, emit a rate-limited Warning so operators learn coverage was
// lost, and nudge the reconcile pass to rescan directories that also saw
// analyzer-queue drops during the same storm.
func (fm *FileMonitor) handleQueueOverflow() {
fm.initQueueHealth()
fm.kernelQueueHealth.Lose(time.Now(), 1)
atomic.AddInt64(&fm.queueOverflows, 1)
// Dropped writes name no inode, so any recorded analysis may be stale.
fm.recent.Purge()
if fanotifyKernelOverflowTotal != nil {
fanotifyKernelOverflowTotal.Inc()
}
fm.reportQueueOverflow()
if fm.reconcileSig != nil {
select {
case fm.reconcileSig <- struct{}{}:
default:
}
}
}
// reportQueueOverflow emits the kernel-overflow finding at most once per minute.
func (fm *FileMonitor) reportQueueOverflow() {
fm.overflowReportMu.Lock()
if !fm.lastOverflowReport.IsZero() && time.Since(fm.lastOverflowReport) < time.Minute {
fm.overflowReportMu.Unlock()
return
}
fm.lastOverflowReport = time.Now()
fm.overflowReportMu.Unlock()
fm.sendAlert(alert.Warning, "fanotify_kernel_overflow",
"fanotify kernel event queue overflowed; file events were dropped by the kernel and cannot be recovered by path",
"The kernel notification queue filled during a storm (bulk unzip, backup restore, or high-volume attack) and dropped events before the reader could drain them. Files touched during the overflow that are not written again are only covered by the next scheduled deep scan. A reconcile of directories that also saw analyzer-queue drops has been triggered.")
}
// drainAndClose drains the analyzerCh and waits for workers to finish.
// C1 - ensures no fd leak on shutdown.
func (fm *FileMonitor) drainAndClose() {
fm.drainOnce.Do(func() {
fm.drainDeadline.Store(time.Now().Add(analyzerDrainBudget).UnixNano())
// The placement backend admits into analyzerCh and queues walks
// until it sees stopCh; the walk worker then admits until it does.
fm.placementWG.Wait()
fm.placedDirWG.Wait()
close(fm.analyzerCh)
fm.wg.Wait()
fm.tmpVouches.close()
fm.discardReconcilePending()
fm.discardPlacedDirs()
fm.stagedPackages().discardPending(time.Now())
if fm.dropper != nil {
fm.dropper.tr.discardPending(time.Now())
clear(fm.dropper.attempts)
}
// Mark pipe as closed before actually closing, so Stop() won't
// write to an already-closed fd (H2 fix).
atomic.StoreInt32(&fm.pipeClosed, 1)
_ = unix.Close(fm.pipeFds[0])
_ = unix.Close(fm.pipeFds[1])
})
}
// Stop signals the monitor to shut down.
// C2 - sync.Once ensures safe concurrent calls; does not close analyzerCh directly.
func (fm *FileMonitor) Stop() {
fm.stopOnce.Do(func() {
close(fm.stopCh)
// Wake epoll so Run() exits and calls drainAndClose.
// Only write if pipe hasn't been closed by drainAndClose yet.
if atomic.LoadInt32(&fm.pipeClosed) == 0 {
_, _ = unix.Write(fm.pipeFds[1], []byte{0})
}
// Close fanotify fd - causes any pending Read/EpollWait to return
fm.initQueueHealth()
_ = fm.kernelQueue.close()
})
}
func (fm *FileMonitor) handleEvent(fd int, pid int32, mask uint64) {
fm.handleEventAt(fd, pid, mask, time.Now())
}
func (fm *FileMonitor) handleEventAt(fd int, pid int32, mask uint64, readAt time.Time) {
path, unlinked, ok := eventFDPath(fd)
if !ok {
fm.rejectUnresolved(fd)
return
}
// Before admission: a write under a name admission ignores still changes
// what a later rename back would place.
var st unix.Stat_t
var closedWrite bool
if mask&FAN_CLOSE_WRITE != 0 {
if unix.Fstat(fd, &st) == nil {
fm.recent.Invalidate(uint64(st.Dev), st.Ino)
// A descriptor whose name is gone (O_TMPFILE, an unlinked link
// source) is analysed under that name, not the one the inode
// was linked to, so it cannot stand in for the create.
closedWrite = !unlinked
}
}
event, admit := fm.classifyEvent(path, fd)
if fm.admitEvent(fd, event, admit, pid, mask, readAt) && closedWrite {
// A dropped close-write cannot replace the placement scan: its
// mtime-based recovery may exclude an old inode linked into place.
fm.noteClosedWrite(uint64(st.Dev), st.Ino, path)
}
}
// eventFDPath resolves the path an event descriptor was opened at. unlinked
// reports that the name no longer exists.
func eventFDPath(fd int) (path string, unlinked, ok bool) {
raw, err := os.Readlink(fmt.Sprintf("/proc/self/fd/%d", fd))
if err != nil {
return "", false, false
}
path = normalizeFanotifyEventPath(raw)
return path, path != raw, true
}
// rejectUnresolved closes an event descriptor whose path could not be read.
// It still counts as received and filtered.
func (fm *FileMonitor) rejectUnresolved(fd int) {
_ = unix.Close(fd)
fm.recordEvent(false, false)
}
// admitEvent queues the event classifyEvent built for fd and takes
// ownership of fd. It returns whether the analyzer queue accepted it.
func (fm *FileMonitor) admitEvent(fd int, event fileEvent, admit bool, pid int32, mask uint64, readAt time.Time) bool {
var admitted, dropped bool
defer func() { fm.recordEvent(admitted, dropped) }()
if !admit {
_ = unix.Close(fd)
return false
}
// Send to analyzer pool (with backpressure)
fm.initQueueHealth()
ticket := fm.analyzerHealth.Begin(time.Now())
event.queueTicket, event.pid, event.mask, event.at = ticket, pid, mask, readAt
captureRestoreWriterChain(&event)
captureTmpExecAncestry(&event)
select {
case fm.analyzerCh <- event:
admitted = true
default:
// Queue full - drop event, count, and record the parent dir so the
// reconcile pass in overflowReporter can rescan it. Without this
// every file in a bulk burst past buffer capacity is invisible to
// detection forever.
ticket.Reject(time.Now())
dropped = true
n := atomic.AddInt64(&fm.droppedEvents, 1)
if n%100 == 0 {
fmt.Fprintf(os.Stderr, "[%s] fanotify: %d events dropped (analyzer queue full)\n", ts(), n)
}
fm.recordDroppedEvent(event)
fm.maybeTriggerEagerReconcile(n)
_ = unix.Close(fd)
}
return admitted
}
// classifyEvent decides whether a write at path is queued for analysis and
// builds the event the analyzer receives. The caller keeps ownership of fd.
func (fm *FileMonitor) classifyEvent(path string, fd int) (ev fileEvent, admit bool) {
// M5 - skip directory events
if strings.HasSuffix(path, "/") {
return fileEvent{}, false
}
// Recovery copies keep their original names. Re-scanning them repeats
// detections CSM already acted on and spends automatic responses.
if checks.InQuarantine(path) {
return fileEvent{}, false
}
// The dropper tracker also needs arbitrary executable names that have
// no path-only content signal.
fm.invalidateDropperPHPHandlerCache(path)
_, removed := removedFileAt(path)
contentInteresting := removed || fm.isInteresting(path)
// Filtering must not reroute writes retained for dropper tracking: the
// dropper-only path bypasses normal suppressions and checksum verification.
contentNeedsAnalysis := contentInteresting
if contentInteresting && !removed && underTempRoot(path) {
contentNeedsAnalysis = fm.tempRootEventNeedsAnalysis(path, fd)
}
dropperInteresting, phpExecutable := fm.isDropperInteresting(path, fd)
if !contentNeedsAnalysis && !dropperInteresting {
return fileEvent{}, false
}
return fileEvent{
path: path, fd: fd,
dropperOnly: !contentInteresting, phpExecutable: phpExecutable,
}, true
}
// normalizeFanotifyEventPath removes procfs's synthetic " (deleted)" suffix
// when the event fd's directory entry was already unlinked. Linux does not
// disambiguate that marker from a literal filename suffix. Treat it as the
// kernel marker: otherwise an attacker can create a same-inode hardlink with
// the literal suffix and make an immediate self-delete miss the PHP path gate.
func normalizeFanotifyEventPath(path string) string {
const deletedSuffix = " (deleted)"
return strings.TrimSuffix(path, deletedSuffix)
}
// maybeTriggerEagerReconcile nudges overflowReporter to run reconcileDrops
// immediately when sustained drops cross eagerReconcileDropThreshold within
// a single minute window. Delegates to the free function so the trigger
// logic stays testable from a cross-platform test file.
func (fm *FileMonitor) maybeTriggerEagerReconcile(droppedSoFar int64) {
signalEagerReconcile(fm.reconcileSig, droppedSoFar, eagerReconcileDropThreshold)
}
// isInteresting is the fast filter - zero I/O, pure string matching.
func (fm *FileMonitor) isInteresting(path string) bool {
path = atomicWriteContentPath(path)
lower := strings.ToLower(path)
// PHP source files. This is intentionally broader than the executable-PHP
// predicate used by the location and dropper checks: .phps is inert under a
// stock handler, but still needs signature/YARA analysis while staged.
if isPHPSourceExtension(filepath.Base(lower)) {
return true
}
// Webshell extensions
if strings.HasSuffix(lower, ".haxor") || strings.HasSuffix(lower, ".cgix") {
return true
}
// CGI scripts in hosted trees - detect Perl/Python/Bash backdoors.
// An explicit document root may live outside the platform's account homes.
if fm.underAccountOrConfiguredDocRoot(path) {
if strings.HasSuffix(lower, ".pl") || strings.HasSuffix(lower, ".cgi") ||
strings.HasSuffix(lower, ".py") || strings.HasSuffix(lower, ".sh") ||
strings.HasSuffix(lower, ".rb") {
return true
}
}
// .htaccess and .user.ini files (any location), and php.ini under a
// configured or detected web root. An attacker plants php.ini files there
// to weaken disable_functions.
if strings.HasSuffix(lower, ".htaccess") || strings.HasSuffix(lower, ".user.ini") {
return true
}
if filepath.Base(lower) == "php.ini" && pathMatchesWebRootPatterns(path, fm.webRootPatterns) {
return true
}
// HTML files in an account or explicitly configured document tree.
if fm.underAccountOrConfiguredDocRoot(path) &&
(strings.HasSuffix(lower, ".html") || strings.HasSuffix(lower, ".htm")) {
return true
}
// Images in an account or explicitly configured document tree. A real
// image container is a working payload store: PHP appended to a valid
// PNG still opens as a picture, and a one-line include elsewhere in the
// site executes it. Admitting the write is what lets checkImagePayload
// look at the bytes; the extension only routes the event, and the
// verdict comes from the container magic, so a renamed payload is still
// caught by the other branches.
if fm.underAccountOrConfiguredDocRoot(path) && contenttype.IsImageExt(filepath.Ext(lower)) {
return true
}
// Credential log files - known phishing harvest filenames
base := filepath.Base(lower)
if credentialLogNames[base] {
return true
}
// ZIP archives in an account or explicitly configured document tree.
if fm.underAccountOrConfiguredDocRoot(path) && strings.HasSuffix(lower, ".zip") {
return true
}
// Anything in .config directories
if strings.Contains(path, "/.config/") {
return true
}
// User crontabs surfaced via the directory-scoped fanotify mark in
// NewFileMonitor. Each write to /var/spool/cron/<user> dispatches to
// checkCrontab in real time.
if strings.HasPrefix(path, cronSpoolDir()+"/") {
return true
}
// Writes in the shared temporary trees. The path alone cannot say whether
// one carries signal; the reader decides that from the event descriptor in
// tempRootEventNeedsAnalysis.
if underTempRoot(path) {
return true
}
// PHP in sensitive directories that should never contain PHP
if isPHPExtension(strings.ToLower(filepath.Base(path))) && fm.sensitivePHPDir(path) != "" {
return true
}
return false
}
// underAccountRoot reports whether path sits inside a hosting account's tree.
// Falls back to the historical /home spelling when the platform offers no
// patterns, so an unconfigured plain-Linux host keeps the behaviour it had.
func (fm *FileMonitor) underAccountRoot(path string) bool {
if len(fm.accountRootPatterns) == 0 {
return strings.HasPrefix(path, "/home/")
}
return pathMatchesWebRootPatterns(path, fm.accountRootPatterns)
}
// underDocRoot reports whether path sits inside a served document root.
func (fm *FileMonitor) underDocRoot(path string) bool {
if len(fm.docRootPatterns) == 0 {
return strings.Contains(path, "/public_html/")
}
return pathMatchesWebRootPatterns(path, fm.docRootPatterns)
}
func (fm *FileMonitor) underAccountOrConfiguredDocRoot(path string) bool {
return fm.underAccountRoot(path) ||
(len(fm.docRootPatterns) > 0 && fm.underDocRoot(path))
}
// tempRootPrefixes are the shared temporary trees. Anything may be written
// there by anyone, so they are watched, but most of what lands there carries no
// signal any detector acts on.
var tempRootPrefixes = []string{"/tmp/", "/dev/shm/", "/var/tmp/"}
// underTempRoot reports whether path is inside one of those trees.
func underTempRoot(path string) bool {
for _, prefix := range tempRootPrefixes {
if strings.HasPrefix(path, prefix) {
return true
}
}
return false
}
// tmpWorkDir names the directory directly under a temp root that holds path,
// with a trailing slash so it can never equal a file's own path. A file placed
// directly in a temp root has no such directory and is returned unchanged.
func tmpWorkDir(path string) string {
for _, prefix := range tempRootPrefixes {
if rest, ok := strings.CutPrefix(path, prefix); ok {
if i := strings.IndexByte(rest, '/'); i >= 0 {
return path[:len(prefix)+i+1]
}
return path
}
}
return path
}
// sensitivePHPDirs are directories that should never contain PHP, so a PHP
// file in one is reported wherever the directory itself lives.
var sensitivePHPDirs = []string{"/.ssh/", "/.cpanel/", "/.gnupg/", "/.cagefs/"}
// accountMailDir is an account's mail store. Applications ship ordinary
// directories named mail, so only the one directly in an account home counts.
const accountMailDir = "/mail/"
// sensitivePHPDir returns the directory that makes PHP at path Critical on
// location alone, or "" when there is none.
func (fm *FileMonitor) sensitivePHPDir(path string) string {
for _, dir := range sensitivePHPDirs {
if strings.Contains(path, dir) {
return dir
}
}
path = filepath.Clean(path)
for i := 0; ; {
idx := strings.Index(path[i:], accountMailDir)
if idx < 0 {
return ""
}
if fm.isAccountHome(path[:i+idx]) {
return accountMailDir
}
i += idx + 1
}
}
// isAccountHome reports whether dir is an account home itself. cPanel also
// places accounts on further partitions named /home2, /home3 that platform
// detection does not list, so those count wherever /home does.
func (fm *FileMonitor) isAccountHome(dir string) bool {
patterns := fm.accountRootPatterns
if len(patterns) == 0 {
patterns = []string{"/home/*"}
}
for _, pattern := range patterns {
if matched, err := filepath.Match(pattern, dir); err == nil && matched {
return true
}
if pattern != "/home/*" {
continue
}
if rest, ok := strings.CutPrefix(dir, "/home"); ok {
partition, account, ok := strings.Cut(rest, "/")
if ok && partition != "" && strings.Trim(partition, "0123456789") == "" &&
account != "" && !strings.Contains(account, "/") {
return true
}
}
}
return false
}
// tempRootEventNeedsAnalysis decides from the descriptor the reader already
// holds whether a write under a temp root can produce a finding. The analyzer
// reaches the same verdict, but only after the event has taken a queue slot, a
// descriptor and a worker wake-up. Every branch here mirrors a check the
// analyzer runs before or inside its own temp-root branch; an event this
// rejects is one the analyzer would have returned on without reporting.
//
// A descriptor it cannot stat is admitted: losing the mode is not a reason to
// stop looking at a file.
func (fm *FileMonitor) tempRootEventNeedsAnalysis(path string, fd int) bool {
// Judge the file the write is producing, not the staging name it is being
// written under: an editor saving .htaccess writes .temp.1..htaccess first,
// and both isInteresting and the analyzer resolve that before deciding.
content := atomicWriteContentPath(path)
lower := strings.ToLower(content)
name := filepath.Base(lower)
switch {
case strings.HasPrefix(path, cronSpoolDir()+"/"),
isPHPSourceExtension(name),
name == ".htaccess", name == ".user.ini", name == "php.ini",
knownWebshells[name],
strings.HasSuffix(lower, ".haxor"), strings.HasSuffix(lower, ".cgix"),
strings.Contains(content, "/.config/"),
isPHPExtension(name) && fm.sensitivePHPDir(content) != "",
contenttype.IsImageExt(filepath.Ext(lower)) && fm.underAccountOrConfiguredDocRoot(content):
return true
}
var st unix.Stat_t
if err := unix.Fstat(fd, &st); err != nil {
return true
}
return st.Mode&unix.S_IFMT != unix.S_IFDIR && st.Mode&0o111 != 0
}
// credentialLogNames are filenames commonly used by phishing kits to store
// harvested credentials. Checked in isInteresting() for real-time detection.
var credentialLogNames = map[string]bool{
"results.txt": true, "result.txt": true, "log.txt": true,
"logs.txt": true, "emails.txt": true, "data.txt": true,
"passwords.txt": true, "creds.txt": true, "credentials.txt": true,
"victims.txt": true, "output.txt": true, "harvested.txt": true,
"results.log": true, "emails.log": true, "data.log": true,
"results.csv": true, "emails.csv": true, "data.csv": true,
}
// analyzerWorker processes file events from the bounded channel.
// C1 - on channel close, drains remaining events and closes their fds.
// Past the shutdown drain budget the remaining events are released without
// being scanned: their files are still covered by the next scheduled deep
// scan, while a backlog scanned to completion costs the daemon its whole
// stop timeout and ends in SIGKILL.
func (fm *FileMonitor) analyzerWorker() {
defer fm.wg.Done()
for event := range fm.analyzerCh {
if fm.drainBudgetSpent(time.Now()) {
event.queueTicket.Reject(time.Now())
_ = unix.Close(event.fd)
continue
}
fm.analyzeFileSafe(event)
_ = unix.Close(event.fd)
}
}
// drainBudgetSpent reports whether the shutdown drain has run out of time.
func (fm *FileMonitor) drainBudgetSpent(now time.Time) bool {
deadline := fm.drainDeadline.Load()
return deadline != 0 && now.UnixNano() >= deadline
}
// fileAnalyzer analyzes one queued event. Var so tests can substitute a
// panicking analyzer.
var fileAnalyzer = (*FileMonitor).analyzeFile
// analyzeFileSafe runs one event and contains a panic: one crafted file
// must not restart the daemon and reopen the detection gap for every other
// write in flight. The caller still closes the event fd.
func (fm *FileMonitor) analyzeFileSafe(event fileEvent) {
defer func() {
if r := recover(); r != nil {
fm.reportScannerPanic(event.path, r)
}
}()
work := queuehealth.Work[fileEvent]{Value: event, Ticket: event.queueTicket}
work.Process(func(queued fileEvent) {
begun := fm.recent.Begin()
var snapshot analysisSnapshot
var keyed bool
if fm.recent != nil {
snapshot, keyed = snapshotForAnalysis(queued.fd, fm.eventClass(queued))
}
var cacheable bool
queued.cacheable = &cacheable
fileAnalyzer(fm, queued)
// A writer can restore mtime or keep its descriptor open past the
// rename. Only an unchanged scan of the keyed bytes is reusable.
if keyed && cacheable && snapshot.unchanged(queued.fd) {
fm.recent.Record(snapshot.key, begun, time.Now())
}
})
}
// eventClass is everything besides the file version that decides how an
// admitted event is analysed. The content path carries the location and name
// rules, so the class only matches when those are unchanged: a cPanel
// .temp.<n>.<name> write renamed to <name>. The generations retire it on any
// config publication or rule load. Hashed so each remembered analysis costs a
// fixed size whatever the path length.
func (fm *FileMonitor) eventClass(ev fileEvent) string {
if ev.dropperOnly || !isPHPSourceExtension(strings.ToLower(filepath.Base(atomicWriteContentPath(ev.path)))) {
return ""
}
sum := sha256.Sum256(fmt.Appendf(nil, "%s\x00%t\x00%t\x00%t\x00%d\x00%d\x00%d",
atomicWriteContentPath(ev.path), ev.dropperOnly, ev.phpExecutable, fm.suppressedPath(ev.path),
config.ActiveGeneration(), signatures.CacheGeneration(), yara.CacheGeneration()))
return string(sum[:])
}
// reportScannerPanic logs the panic with its stack, forwards it to
// observability and raises a critical finding at most once per ten minutes.
func (fm *FileMonitor) reportScannerPanic(path string, r interface{}) {
obs.CaptureMsg("fanotify-analyzer", fmt.Sprintf("panic analyzing %s: %v", path, r))
fmt.Fprintf(os.Stderr, "[%s] file monitor: recovered panic analyzing %s: %v\n%s", ts(), path, r, debug.Stack())
fm.panicMu.Lock()
defer fm.panicMu.Unlock()
if !fm.lastPanicAt.IsZero() && time.Since(fm.lastPanicAt) < 10*time.Minute {
return
}
fm.lastPanicAt = time.Now()
fm.sendAlertWithPath(alert.Critical, "realtime_scanner_panic",
fmt.Sprintf("Realtime scanner panicked on %s and skipped it: %v", path, r), "", path, "")
}
// readFromFd reads up to maxBytes from a file descriptor at position 0.
// C3 - avoids TOCTOU by reading from the original fanotify event fd.
// readFromFd reads up to maxBytes from the fanotify event fd using pread
// at offset 0. Uses unix.Pread directly to avoid os.NewFile's GC finalizer
// which would close the fd out-of-band, racing with the worker's explicit close.
func readFromFd(fd int, maxBytes int) []byte {
buf := make([]byte, maxBytes)
n, err := unix.Pread(fd, buf, 0)
if n <= 0 || (err != nil && n == 0) {
return nil
}
return buf[:n]
}
func readVersionedFromFd(fd, maxBytes int, stat func(int) os.FileInfo) ([]byte, os.FileInfo) {
before := stat(fd)
data := readFromFd(fd, maxBytes)
if data == nil {
return nil, nil
}
after := stat(fd)
if !checks.SameFileVersion(before, after) {
return data, nil
}
return data, after
}
func readVersionedFileFromFd(fd, maxBytes int) ([]byte, os.FileInfo) {
return readVersionedFromFd(fd, maxBytes, scannedIdentity)
}
const readCompleteMaxInterrupts = 8
// readExactSize reads exactly the snapshotted size. Its buffer is fixed before
// the first read, so a concurrently growing source cannot extend the loop; a
// bounded EINTR retry count also prevents a pathological signal storm from
// pinning an analyzer worker.
func readExactSize(size int64, maxBytes int, pread func([]byte, int64) (int, error)) []byte {
if size <= 0 || maxBytes <= 0 || size > int64(maxBytes) {
return nil
}
buf := make([]byte, int(size))
interrupts := 0
for off := 0; off < len(buf); {
n, err := pread(buf[off:], int64(off))
if n < 0 || n > len(buf)-off {
return nil
}
if n > 0 {
off += n
interrupts = 0
}
if err != nil && !errors.Is(err, unix.EINTR) {
return nil
}
if n > 0 {
continue
}
if !errors.Is(err, unix.EINTR) {
return nil
}
interrupts++
if interrupts > readCompleteMaxInterrupts {
return nil
}
}
return buf
}
func sameReadSnapshot(before, after unix.Stat_t) bool {
return before.Dev == after.Dev && before.Ino == after.Ino && before.Size == after.Size &&
before.Mtim == after.Mtim && before.Ctim == after.Ctim
}
// readCompleteFromFd returns a stable snapshot of the entire file behind fd
// when it fits within maxBytes. A short read, concurrent size/content change,
// or excessive interruption fails closed so whole-file recognizers never
// accept a stale prefix.
func readCompleteFromFd(fd, maxBytes int) []byte {
var before unix.Stat_t
if err := unix.Fstat(fd, &before); err != nil {
return nil
}
buf := readExactSize(before.Size, maxBytes, func(p []byte, off int64) (int, error) {
return unix.Pread(fd, p, off)
})
if buf == nil {
return nil
}
var after unix.Stat_t
if err := unix.Fstat(fd, &after); err != nil || !sameReadSnapshot(before, after) {
return nil
}
return buf
}
func isBenignPHPStubData(fd int, data []byte) bool {
if len(data) == 0 {
return false
}
complete := false
var st unix.Stat_t
if err := unix.Fstat(fd, &st); err == nil {
complete = st.Size <= int64(len(data))
}
return checks.IsBenignPHPStubBytesComplete(data, complete)
}
func isWPTranslationCacheData(fd int, data []byte) bool {
if len(data) == 0 {
return false
}
complete := false
var st unix.Stat_t
if err := unix.Fstat(fd, &st); err == nil {
complete = st.Size <= int64(len(data))
}
return checks.IsWPTranslationCacheBytesComplete(data, complete)
}
// readTailFromFd reads the last maxBytes of a file via its fd using pread.
// Returns nil if the file is smaller than maxBytes (head scan already covers it).
func readTailFromFd(fd int, maxBytes int) []byte {
var stat unix.Stat_t
if err := unix.Fstat(fd, &stat); err != nil {
return nil
}
size := stat.Size
if size <= int64(maxBytes) {
return nil // head read already covers the entire file
}
offset := size - int64(maxBytes)
buf := make([]byte, maxBytes)
n, err := unix.Pread(fd, buf, offset)
if n <= 0 || (err != nil && n == 0) {
return nil
}
return buf[:n]
}
// resolveProcessInfo reads /proc/<pid>/comm and /proc/<pid>/status
// to build a "pid=N cmd=name uid=N" string for alert enrichment.
// Returns empty string on any error (process may have exited).
func resolveProcessInfo(pid int32) string {
if pid <= 0 {
return ""
}
procDir := fmt.Sprintf("/proc/%d", pid)
// Read process name
// #nosec G304 -- /proc/<pid>/comm; kernel pseudo-FS, pid is int32 from fanotify event.
comm, err := os.ReadFile(procDir + "/comm")
if err != nil {
return ""
}
name := strings.TrimSpace(string(comm))
info := fmt.Sprintf("pid=%d cmd=%s", pid, name)
// Read UID from status to map to cPanel username
// #nosec G304 -- /proc/<pid>/status; kernel pseudo-FS.
statusData, err := os.ReadFile(procDir + "/status")
if err != nil {
return info
}
for _, line := range strings.Split(string(statusData), "\n") {
if strings.HasPrefix(line, "Uid:") {
fields := strings.Fields(line)
if len(fields) >= 2 {
info += fmt.Sprintf(" uid=%s", fields[1])
}
break
}
}
return info
}
func (fm *FileMonitor) analyzeFile(event fileEvent) {
path := event.path
contentPath := atomicWriteContentPath(path)
name := filepath.Base(contentPath)
nameLower := strings.ToLower(name)
imageInHostedTree := contenttype.IsImageExt(filepath.Ext(nameLower)) && fm.underAccountOrConfiguredDocRoot(contentPath)
// Resolve process info from PID (best-effort - process may have exited)
procInfo := resolveProcessInfo(event.pid)
// Snapshot this write as a possible self-deleting dropper before the
// per-type checks below can early-return. Reads are positional (Pread/
// Fstat/Statx) so they do not disturb the fd for later content checks.
cand := fm.observeDropperCandidate(event, procInfo)
markDropperContentSuspicious := func() {
if cand == nil {
return
}
cand.ContentSuspicious = true
if !fm.dropper.tr.Refresh(*cand) {
fm.dropper.admit(*cand)
}
}
// Some events are admitted only for dropper tracking. Handler-mapped PHP
// still needs the normal PHP scanner; arbitrary executables retain the
// strongest cheap content signal for the later deletion verdict.
if event.dropperOnly {
if event.phpExecutable {
if fm.checkPHPContent(event.fd, path, procInfo) {
markDropperContentSuspicious()
}
} else if cand != nil && looksLikePHPWebshell(cand.Head) {
markDropperContentSuspicious()
}
return
}
// H2 - suppression path matching using filepath.Match. Read the live config
// (config.Active via currentCfg) so a SIGHUP change to suppressions.ignore_paths
// takes effect without a restart, matching the rest of this analyzer.
if fm.suppressedPath(path) {
return
}
// Skip unmodified WordPress core and plugin files: the hash matches the
// official wordpress.org checksums for the version the install or
// package declares. Stops signature/YARA FPs on stock code, installed or
// staged: a realtime Critical feeds inline quarantine, and a byte-for-byte
// copy of the official release is not what that is for. A cache miss
// triggers a background fetch and falls through to rule evaluation; the
// description is kept for the update-staging branch below, which judges
// a staged package by these verdicts. For atomic writes, the intended
// basename only selects the checksum entry. Trust requires hashing the
// complete original event descriptor.
var wpVerdict wpcheck.Verification
if fm.wpCache != nil {
wpVerdict = fm.wpCache.VerifyFile(event.fd, contentPath)
if wpVerdict.Verdict == wpcheck.VerdictVerified {
// Earlier files may still be waiting for this tree's header.
// Keep the identity even when this file needs no content scan.
if wpVerdict.Staged && wpVerdict.Kind == wpcheck.KindPlugin {
fm.handleStagedPackageFile(path, wpVerdict, procInfo)
}
return
}
}
// Checked after the checksum skip: official release bytes coming back
// are never a response being undone.
if fm.checkRemovedFileRestored(event, procInfo) {
markDropperContentSuspicious()
return
}
// User crontab written under /var/spool/cron/<user>. Scan content
// from the event fd via the shared deep matcher and emit Critical on
// any hit. The polled CheckCrontabs run still tracks root crontab
// hash drift, so we skip root here to avoid duplicate signal.
if strings.HasPrefix(path, cronSpoolDir()+"/") {
fm.checkCrontab(event.fd, path, procInfo)
return
}
// Location-based severity escalation: PHP in dirs that should NEVER have PHP
if isPHPExtension(nameLower) {
if sensitive := fm.sensitivePHPDir(path); sensitive != "" {
fm.sendAlertWithPath(alert.Critical, "php_in_sensitive_dir_realtime",
fmt.Sprintf("PHP file in critical directory: %s", path),
fmt.Sprintf("PHP should never exist in %s - likely webshell or backdoor", sensitive), path, procInfo)
return
}
}
// Known webshell filenames (M1 - package-level var). Filename alone is
// too weak: WordPress core ships wp-includes/Text/Diff/Engine/shell.php
// (the Pear Text_Diff library using shell_exec to call Unix `diff`).
// Confirm with content: the file must also exhibit webshell markers
// (request superglobal flowing into a dangerous function, or an
// eval/assert wrapping a base64/gzinflate decoder).
if knownWebshells[nameLower] {
recordReadTruncation(event.fd, 65536, "phpcontent_inline")
if data, scanned := readVersionedFileFromFd(event.fd, 65536); looksLikePHPWebshell(data) {
markDropperContentSuspicious()
message := fmt.Sprintf("Webshell file created: %s", path)
if confirmedPHPExecutionData(data) {
fm.sendConfirmedPHPFinding(event.fd, data, scanned, "webshell_realtime", message, "", path, procInfo)
} else {
fm.sendAlertWithPath(alert.Critical, "webshell_realtime", message, "", path, procInfo)
}
return
}
}
// Webshell extensions
if strings.HasSuffix(nameLower, ".haxor") || strings.HasSuffix(nameLower, ".cgix") {
fm.sendAlertWithPath(alert.Critical, "webshell_realtime",
fmt.Sprintf("Suspicious CGI file created: %s", path), "", path, procInfo)
return
}
// .htaccess modification - check for injection (C3 - read from fd).
// Checked before the /tmp early-return so a malicious .htaccess anywhere
// (including /tmp) is still analyzed for dangerous directives.
if nameLower == ".htaccess" {
fm.checkTmpExecutable(event, procInfo)
fm.checkHtaccess(event.fd, path, procInfo)
return
}
// .user.ini modification - check for dangerous PHP settings (C3 - read from fd).
// Also checked before /tmp so malicious .user.ini is detected anywhere.
if nameLower == ".user.ini" || nameLower == "php.ini" {
fm.checkTmpExecutable(event, procInfo)
fm.checkUserINI(event.fd, path, procInfo)
return
}
// Executables in .config - checked before the /tmp block so a miner
// dropped at /tmp/.config/* is flagged as executable_in_config_realtime
// (more specific) rather than executable_in_tmp_realtime.
// Uses unix.Fstat on the event fd (not os.Stat by path) for TOCTOU
// safety: an attacker cannot chmod -x or swap the file after the event.
if strings.Contains(path, "/.config/") {
var cfgStat unix.Stat_t
if err := unix.Fstat(event.fd, &cfgStat); err == nil {
isDir := cfgStat.Mode&unix.S_IFMT == unix.S_IFDIR
if !isDir && cfgStat.Mode&0111 != 0 {
fm.sendAlertWithPath(alert.Critical, "executable_in_config_realtime",
fmt.Sprintf("Executable created in .config: %s", path),
fmt.Sprintf("Size: %d", cfgStat.Size), path, procInfo)
}
}
if !imageInHostedTree {
return
}
}
// Executables in /tmp or /dev/shm - detect dropped malware/miners
// Uses unix.Fstat on event fd for TOCTOU safety (attacker can't chmod -x after event)
if underTempRoot(path) {
fm.checkTmpExecutable(event, procInfo)
// Hosted images still need payload checks when the configured root
// lives in a temporary directory, as do PHP source files anywhere.
if !isPHPSourceExtension(nameLower) && !imageInHostedTree {
return
}
}
// PHP in uploads directories.
// Any PHP file here is anomalous: /wp-content/uploads/ is meant for
// media, not code. Plugin-update temp dirs are recognised structurally
// via looksLikePluginUpdate only after content checks have had first
// refusal, so a decoy update directory cannot downgrade a webshell.
// Operators suppress legitimate caching daemons through the path-scoped
// suppressions_api, not an implicit substring allowlist in the daemon.
if strings.Contains(path, "/wp-content/uploads/") && isPHPExtension(nameLower) {
// Content-aware severity: PHP in uploads is anomalous but not
// always malicious (TinyMCE smile_fonts/charmap.php is glyph
// data shipped by WP's bundled editor). Emit Critical for
// direct webshell markers, otherwise run the broader PHP
// content/signature/YARA path before downgrading clean PHP to
// a Warning.
recordReadTruncation(event.fd, 65536, "phpcontent_uploads")
data, scanned := readVersionedFileFromFd(event.fd, 65536)
if looksLikePHPWebshell(data) {
markDropperContentSuspicious()
message := fmt.Sprintf("PHP file created in uploads: %s", path)
details := "Webshell markers in content (request superglobal -> dangerous function, or eval/assert + decoder chain)"
if confirmedPHPExecutionData(data) {
fm.sendConfirmedPHPFinding(event.fd, data, scanned, "php_in_uploads_realtime", message, details, path, procInfo)
} else {
fm.sendAlertWithPath(alert.Critical, "php_in_uploads_realtime", message, details, path, procInfo)
}
} else {
if fm.checkPHPContent(event.fd, path, procInfo) {
markDropperContentSuspicious()
return
}
// Content-shape gate: file whose reachable code is
// whitespace+comments, or that terminates with
// die/exit/__halt_compiler before any statement,
// cannot execute attacker-controlled code via web
// request. BackWPup writes its working-job and
// folder-cache state files this way. The earlier
// signature/YARA pass and the path-only warning
// below are the layers that fire on real droppers;
// a structurally inert stub adds no signal.
if isBenignPHPStubData(event.fd, data) {
return
}
if looksLikePluginUpdate(path) {
// Verified plugin update - emit one low-severity alert per temp directory.
uploadsIdx := strings.Index(path, "/wp-content/uploads/")
afterUploads := path[uploadsIdx+len("/wp-content/uploads/"):]
tempDir := afterUploads
if slashIdx := strings.Index(afterUploads, "/"); slashIdx > 0 {
tempDir = afterUploads[:slashIdx]
}
updateDir := path[:uploadsIdx] + "/wp-content/uploads/" + tempDir
fm.sendAlertWithPath(alert.Warning, "php_in_uploads_realtime",
fmt.Sprintf("Plugin update in uploads: %s", updateDir),
"Verified: matching plugin exists in plugins/", updateDir, procInfo)
return
}
// Suppress the path-only "anomalous location" warning
// when the file is structurally a duplicate (cPanel
// restore staging) or a known plugin probe shape that
// never carries executable input. Signature/YARA scans
// already ran above, so any real malicious content is
// reported through its own pipeline.
if looksLikeCpanelRestoreStaging(path) {
return
}
if looksLikeWPOptimizeProbe(path, data) {
return
}
fm.sendAlertWithPath(alert.Warning, "php_in_uploads_realtime",
fmt.Sprintf("PHP file in uploads (no webshell markers): %s", path),
"Anomalous location for PHP, but content is clean",
path, procInfo)
}
return
}
// PHP in languages/upgrade directories.
// Path-only Critical buried real alerts under location noise (WPML
// translation queues, WP auto-update staging). Run content analysis
// on every file -- a real rule fires Critical, clean real code gets a
// Warning, and inert stubs stay quiet. No filename allowlist: an attacker
// must not be able to hide a backdoor by naming it like a translation or
// index file.
if (strings.Contains(path, "/wp-content/languages/") || strings.Contains(path, "/wp-content/upgrade/")) &&
isPHPExtension(nameLower) {
if fm.checkPHPContent(event.fd, path, procInfo) {
markDropperContentSuspicious()
} else {
// Every staged file reaches content analysis first. Its original
// digest then decides the path-only warning, even for inert files
// absent from the official manifest.
if fm.handleStagedPackageFile(path, wpVerdict, procInfo) {
return
}
// Translation caches and comment-only stubs require a stable,
// complete body. A no-argument PHP terminator is safe from a
// prefix because all following bytes are unreachable, so retain
// the old bounded-head fallback for oversized files.
data := readCompleteFromFd(event.fd, checks.MaxInertPHPScanBytes)
if data != nil && checks.IsBenignPHPStubBytesComplete(data, true) {
return
}
if data == nil && checks.IsBenignPHPStubBytesComplete(readFromFd(event.fd, 65536), false) {
return
}
// WordPress 6.5+ writes *.l10n.php translation caches here as pure
// data return arrays. Suppress by content structure, not filename.
if isWPTranslationCacheData(event.fd, data) {
return
}
// A core update copies the new version file here to read the
// release requirements. Literal assignments to the version
// variables cannot run code; the whole body must be seen.
if data != nil && checks.IsWPVersionDataBytesComplete(data, true) {
return
}
fm.sendAlertWithPath(alert.Warning, "php_in_sensitive_dir_realtime",
fmt.Sprintf("PHP file created in sensitive WP directory (content clean): %s", path), "", path, procInfo)
}
return
}
// PHP content analysis (C3 - read from fd; M4 - 32KB scan size).
// .htaccess, .user.ini, and .config executable checks are handled
// earlier in this function (before the /tmp early-return) so specific
// file types take precedence over the /tmp generic block.
if isPHPSourceExtension(nameLower) {
result := fm.inspectPHPContent(event.fd, path, procInfo)
if result.matched {
markDropperContentSuspicious()
} else if event.cacheable != nil {
*event.cacheable = result.complete
}
return
}
// HTML phishing page detection (uses event fd for content, unix.Fstat for size)
if strings.HasSuffix(nameLower, ".html") || strings.HasSuffix(nameLower, ".htm") {
fm.checkHTMLPhishing(event.fd, path, procInfo)
return
}
// PHP carried inside an image file (uses event fd for content).
if contenttype.IsImageExt(filepath.Ext(nameLower)) {
// Handler-mapped images previously entered through dropperOnly and
// received the full PHP scan. Content admission must retain it.
if event.phpExecutable && fm.checkPHPContent(event.fd, path, procInfo) {
markDropperContentSuspicious()
return
}
if fm.checkImagePayload(event.fd, path, procInfo) {
markDropperContentSuspicious()
}
return
}
// Credential log files (content read from the event fd)
if credentialLogNames[nameLower] {
fm.checkCredentialLog(event.fd, path, procInfo)
return
}
// Phishing kit ZIP archives (path-based)
if strings.HasSuffix(nameLower, ".zip") {
fm.checkPhishingZip(path, nameLower, procInfo)
return
}
// CGI scripts in web-accessible directories (Perl, Python, Bash, Ruby)
// Detect backdoor toolkits like LEVIATHAN that use non-PHP scripts.
if fm.underAccountOrConfiguredDocRoot(path) && isCGIExtension(nameLower) {
fm.checkCGIBackdoor(event.fd, path, procInfo)
return
}
}
// checkTmpExecutable is independent of configuration syntax: a shell can
// execute a file even when its basename routes it to a configuration scanner.
func (fm *FileMonitor) checkTmpExecutable(event fileEvent, procInfo string) {
path := event.path
if !underTempRoot(path) {
return
}
var tmpStat unix.Stat_t
if err := unix.Fstat(event.fd, &tmpStat); err == nil {
isDir := tmpStat.Mode&unix.S_IFMT == unix.S_IFDIR
isExec := tmpStat.Mode&0111 != 0
if !isDir && isExec {
// A directory name proves nothing about the writer, so no
// path is exempt. Root-owned drops from panel maintenance
// or a live package transaction (an initramfs rebuild
// after a kernel update) are rescored to Warning, never
// suppressed. See demoteTmpExec for the gates.
finding := alert.Finding{
Severity: alert.Critical,
Check: "executable_in_tmp_realtime",
Message: fmt.Sprintf("Executable created in %s: %s", filepath.Dir(path), path),
Details: fmt.Sprintf("Size: %d, Mode: %04o, UID: %d", tmpStat.Size, tmpStat.Mode&0777, tmpStat.Uid),
FilePath: path,
ProcessInfo: procInfo,
}
dedupPath := path
now := time.Now()
work := tmpWorkDir(path)
ev := tmpExecEvidence(event, tmpStat.Uid)
ok, reason := tmpExecDemote(tmpStat.Uid, ev, now)
switch {
case ok:
fm.tmpVouches.grant(event, reason, now)
case tmpStat.Uid == 0 && ev.vanished:
// The writer exited before its ancestry could be read. It
// is judged by a writer already proven in the same
// root-private work directory, or by a maintenance
// process working inside it; a writer that is still
// running and proved nothing never is.
vouched, covered := fm.tmpVouches.covers(event, now)
if !covered {
vouched, covered = fm.tmpVouches.occupied(event, now)
}
if covered {
ok, reason = true, vouched+" in this work directory, writer exited"
} else {
finding.Details += " [writer exited before its ancestry was read]"
}
}
if ok {
// Maintenance writes hundreds of executables into one work
// directory. One finding stands for the directory per
// cooldown so the burst cannot crowd other findings out of
// the queue. Only demoted findings share a key: anything
// else in the same directory keeps its own.
finding.Severity = alert.Warning
finding.Details += " [demoted: " + reason + "; one finding per work directory]"
// NUL cannot occur in a file path. Separate warning cooldowns
// even when a top-level file has no directory group.
dedupPath = work + "\x00maintenance"
}
if fm.prepareFileFindingWithDedup(&finding, dedupPath) {
fm.enqueuePreparedFileFinding(finding)
}
}
}
}
// checkCrontab scans a freshly-written /var/spool/cron/<user> file for the
// known persistence-marker patterns (literal + base64-decoded). Reads from
// the event fd, not the path, so an attacker swapping the file post-event
// cannot redirect us. Root crontab drift is tracked separately via
// hash-baseline by the polled CheckCrontabs.
func (fm *FileMonitor) checkCrontab(fd int, path, procInfo string) {
user := filepath.Base(path)
if user == "" || user == "root" || user == filepath.Base(cronSpoolDir()) {
return
}
recordReadTruncation(fd, 65536, "crontab")
data := readFromFd(fd, 65536)
if data == nil {
return
}
matched := checks.MatchCrontabPatternsDeep(string(data), fm.currentCfg())
if len(matched) == 0 {
return
}
fm.sendAlertWithPath(alert.Critical, "suspicious_crontab",
fmt.Sprintf("Suspicious crontab written for user %s: %v", user, matched),
fmt.Sprintf("File: %s\nPatterns matched: %v", path, matched),
path, procInfo)
}
// checkHtaccess reads .htaccess content from the event fd and checks for injection.
// C3 - reads from fd, not path.
func (fm *FileMonitor) checkHtaccess(fd int, path, procInfo string) {
recordReadTruncation(fd, htaccessRealtimeMaxBytes, "htaccess")
data := readFromFd(fd, htaccessRealtimeMaxBytes+1)
if data == nil {
return
}
if len(data) > htaccessRealtimeMaxBytes {
fm.sendAlertWithPath(alert.High, "htaccess_injection_realtime",
fmt.Sprintf(".htaccess too large to inspect in real time: %s", path),
"The file exceeds the real-time .htaccess scan limit and may hide malicious directives.",
path, procInfo)
return
}
findings, _ := checks.AuditHtaccessContent(path, data)
for _, f := range findings {
if f.Check == "htaccess_injection" {
fm.sendAlertWithPath(f.Severity, "htaccess_injection_realtime", f.Message, f.Details, path, procInfo)
} else {
fm.sendAlertWithPath(f.Severity, f.Check, f.Message, f.Details, path, procInfo)
}
}
// Run signature/YARA scanning on .htaccess content
fm.runEventSignatureScan(fd, data, path, ".htaccess", procInfo)
}
// checkUserINI reads the event fd so a path replacement cannot change the
// content being analyzed.
func (fm *FileMonitor) checkUserINI(fd int, path, procInfo string) {
recordReadTruncation(fd, checks.PHPConfigMaxBytes, "user_ini")
data := readFromFd(fd, checks.PHPConfigMaxBytes+1)
if data == nil {
return
}
if len(data) > checks.PHPConfigMaxBytes {
fm.sendAlertWithPath(alert.High, "php_config_realtime",
fmt.Sprintf("PHP configuration too large to inspect: %s", path),
"The file exceeds the PHP configuration scan limit and may hide dangerous directives.",
path, procInfo)
return
}
if dangerous := checks.PHPConfigSecurityBypasses(string(data)); len(dangerous) > 0 {
fm.sendAlertWithPath(alert.Critical, "php_config_realtime",
fmt.Sprintf("PHP security configuration weakened: %s", path),
fmt.Sprintf("Dangerous settings:\n- %s", strings.Join(dangerous, "\n- ")), path, procInfo)
return
}
// Run signature/YARA scanning on PHP configuration content.
fm.runEventSignatureScan(fd, data, path, ".ini", procInfo)
}
var realtimePHPDecoders = [...]string{"base64_decode", "gzinflate", "gzuncompress", "str_rot13", "gzdecode"}
var directPHPDecoderExecutionRE = regexp.MustCompile(
`(?i)\b(?:eval|assert)\s*\(\s*@?\s*(?:` +
strings.Join(realtimePHPDecoders[:], "|") + `)\s*\(`,
)
const packedPHPLiteral = `(?:'[a-z0-9+/=]{12,}'|"[a-z0-9+/=]{12,}")`
var packedLiteralPHPExecutionRE = regexp.MustCompile(
`(?i)^(?:eval|assert)\s*\(\s*@?\s*` +
`(?:base64_decode\s*\(\s*` + packedPHPLiteral + `\s*\)` +
`|(?:gzinflate|gzuncompress|str_rot13|gzdecode)\s*\(\s*` +
`base64_decode\s*\(\s*` + packedPHPLiteral + `\s*\)\s*\))\s*\)`,
)
var confirmedPHPExecNames = map[string]struct{}{
"assert": {}, "create_function": {}, "eval": {}, "exec": {},
"passthru": {}, "popen": {}, "proc_open": {}, "shell_exec": {},
"system": {},
}
var (
phpFunctionImportRE = regexp.MustCompile(`(?is)\buse\s+function\s+([^;]+);`)
phpGroupImportRE = regexp.MustCompile(`(?is)\buse\s+[a-z_\\][a-z0-9_\\]*\s*\{([^{}]*)\}\s*;`)
phpIdentifierRE = regexp.MustCompile(`[a-z_][a-z0-9_]*`)
)
type phpNamespaceScope struct {
id int
start, end int
named bool
}
func directPHPDecoderExecutionMatches(code string) [][]int {
var confirmed [][]int
for _, match := range directPHPDecoderExecutionRE.FindAllStringIndex(code, -1) {
previous := match[0] - 1
for previous >= 0 {
switch code[previous] {
case ' ', '\t', '\n', '\r':
previous--
continue
}
break
}
if previous >= 0 {
switch code[previous] {
case '$', '>', ':':
continue
case '\\':
if strings.HasPrefix(code[match[0]:], "eval") || !isExplicitGlobalPHPCall(code, match[0]) {
continue
}
}
}
confirmed = append(confirmed, match)
}
return confirmed
}
func hasPackedLiteralPHPExecution(executable, structure string) bool {
for _, match := range directPHPDecoderExecutionMatches(structure) {
if packedLiteralPHPExecutionRE.MatchString(executable[match[0]:]) {
return true
}
}
return false
}
func maskUserDispatchedPHPCalls(code string) string {
masked := []byte(code)
scopes := phpNamespaceScopes(code)
importedAliases := importedPHPFunctionAliases(code, scopes)
for i := 0; i < len(code); {
if !isPHPIdentifierStart(code[i]) {
i++
continue
}
end := i + 1
for end < len(code) && isPHPIdentifierPart(code[end]) {
end++
}
name := code[i:end]
if _, dangerous := confirmedPHPExecNames[name]; !dangerous {
i = end
continue
}
next := end
for next < len(code) && isPHPSpace(code[next]) {
next++
}
previous := i - 1
for previous >= 0 && isPHPSpace(code[previous]) {
previous--
}
if next >= len(code) || code[next] != '(' {
i = end
continue
}
unsafeDispatch := previous >= 0 && strings.ContainsRune("$>:", rune(code[previous]))
qualified := previous >= 0 && code[previous] == '\\'
if qualified && (name == "eval" || !isExplicitGlobalPHPCall(code, i)) {
unsafeDispatch = true
}
if !qualified && name != "eval" {
scopeID, named := phpNamespaceAt(scopes, i)
if named || importedAliases[scopeID][name] {
unsafeDispatch = true
}
}
if unsafeDispatch {
for j := i; j < end; j++ {
masked[j] = ' '
}
}
i = end
}
return string(masked)
}
func importedPHPFunctionAliases(code string, scopes []phpNamespaceScope) map[int]map[string]bool {
aliases := make(map[int]map[string]bool)
addAliases := func(scopeID int, clauses string) {
if aliases[scopeID] == nil {
aliases[scopeID] = make(map[string]bool)
}
for clause := range strings.SplitSeq(clauses, ",") {
identifiers := phpIdentifierRE.FindAllString(clause, -1)
if len(identifiers) > 0 {
aliases[scopeID][identifiers[len(identifiers)-1]] = true
}
}
}
for _, match := range phpFunctionImportRE.FindAllStringSubmatchIndex(code, -1) {
if len(match) < 4 || match[2] < 0 {
continue
}
scopeID, _ := phpNamespaceAt(scopes, match[0])
addAliases(scopeID, code[match[2]:match[3]])
}
for _, match := range phpGroupImportRE.FindAllStringSubmatchIndex(code, -1) {
if len(match) < 4 || match[2] < 0 {
continue
}
scopeID, _ := phpNamespaceAt(scopes, match[0])
for clause := range strings.SplitSeq(code[match[2]:match[3]], ",") {
clause = strings.TrimSpace(clause)
const keyword = "function"
if !strings.HasPrefix(clause, keyword) || len(clause) == len(keyword) || !isPHPSpace(clause[len(keyword)]) {
continue
}
addAliases(scopeID, clause[len(keyword):])
}
}
return aliases
}
func phpNamespaceScopes(code string) []phpNamespaceScope {
type declaration struct {
start, body int
brace bool
named bool
}
var declarations []declaration
for i := 0; i < len(code); {
if !isPHPIdentifierStart(code[i]) {
i++
continue
}
end := i + 1
for end < len(code) && isPHPIdentifierPart(code[end]) {
end++
}
if code[i:end] != "namespace" {
i = end
continue
}
previous := i - 1
for previous >= 0 && isPHPSpace(code[previous]) {
previous--
}
if previous >= 0 && code[previous] == '\\' {
i = end
continue
}
cursor := end
for cursor < len(code) && isPHPSpace(code[cursor]) {
cursor++
}
named := false
if cursor < len(code) && isPHPIdentifierStart(code[cursor]) {
named = true
for cursor < len(code) {
if isPHPIdentifierPart(code[cursor]) || code[cursor] == '\\' {
cursor++
continue
}
break
}
for cursor < len(code) && isPHPSpace(code[cursor]) {
cursor++
}
}
if cursor >= len(code) || code[cursor] != ';' && code[cursor] != '{' {
i = end
continue
}
declarations = append(declarations, declaration{start: i, body: cursor + 1, brace: code[cursor] == '{', named: named})
i = cursor + 1
}
scopes := make([]phpNamespaceScope, 0, len(declarations))
for index, declaration := range declarations {
end := len(code)
if declaration.brace {
depth := 1
for cursor := declaration.body; cursor < len(code); cursor++ {
switch code[cursor] {
case '{':
depth++
case '}':
depth--
if depth == 0 {
end = cursor
cursor = len(code)
}
}
}
} else if index+1 < len(declarations) {
end = declarations[index+1].start
}
scopes = append(scopes, phpNamespaceScope{id: index + 1, start: declaration.body, end: end, named: declaration.named})
}
return scopes
}
func phpNamespaceAt(scopes []phpNamespaceScope, offset int) (int, bool) {
for _, scope := range scopes {
if offset >= scope.start && offset < scope.end {
return scope.id, scope.named
}
}
return 0, false
}
func isExplicitGlobalPHPCall(code string, nameStart int) bool {
slash := nameStart - 1
if slash < 0 || code[slash] != '\\' {
return false
}
return slash == 0 || code[slash-1] != '\\' && !isPHPIdentifierPart(code[slash-1])
}
func isPHPIdentifierStart(b byte) bool {
return b == '_' || b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z'
}
func isPHPIdentifierPart(b byte) bool {
return isPHPIdentifierStart(b) || b >= '0' && b <= '9'
}
func isPHPSpace(b byte) bool {
return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}
func confirmedPHPExecution(executable, structure string) bool {
requestFlow := maskUserDispatchedPHPCalls(structure)
return looksLikePHPWebshell([]byte(requestFlow)) || hasPackedLiteralPHPExecution(executable, requestFlow)
}
func confirmedPHPExecutionData(data []byte) bool {
executable := strings.ToLower(checks.PHPExecutableCode(string(data)))
structure := strings.ToLower(checks.PHPRequestFlowStructure(string(data)))
return confirmedPHPExecution(executable, structure)
}
// checkPHPContent reads PHP content from the event fd and checks for malicious patterns.
// C3 - reads from fd, not path. M4 - 32KB scan size.
func (fm *FileMonitor) checkPHPContent(fd int, path, procInfo string) bool {
return fm.inspectPHPContent(fd, path, procInfo).matched
}
// A missing verdict is distinct from a clean scan: only the latter may be
// reused when a file is placed at another name.
type contentScanResult struct {
matched bool
complete bool
}
func (fm *FileMonitor) inspectPHPContent(fd int, path, procInfo string) contentScanResult {
recordReadTruncation(fd, 32768, "php_check")
data, scanned := readVersionedFileFromFd(fd, 32768)
if data == nil {
return contentScanResult{}
}
complete := scanned != nil && int64(len(data)) == min(scanned.Size(), 32768)
content := strings.ToLower(string(data))
executable := strings.ToLower(checks.PHPExecutableCode(string(data)))
structure := strings.ToLower(checks.PHPRequestFlowStructure(string(data)))
confirmedExecution := confirmedPHPExecution(executable, structure)
if looksLikePHPWebshell(data) {
message := fmt.Sprintf("Webshell pattern detected: %s", path)
details := "Request input reaches a dangerous PHP execution primitive"
if confirmedExecution {
fm.sendConfirmedPHPFinding(fd, data, scanned, "webshell_content_realtime", message, details, path, procInfo)
} else {
fm.sendAlertWithPath(alert.Critical, "webshell_content_realtime", message, details, path, procInfo)
}
return contentScanResult{matched: true}
}
// Remote payload fetching — paste sites are always suspicious.
// GitHub raw URLs only flag when combined with a dangerous call on
// the same line (legitimate plugins use GitHub for update checks).
pasteURLs := []string{"pastebin.com/raw", "paste.ee/r/", "ghostbin.co/paste/", "hastebin.com/raw/"}
for _, p := range pasteURLs {
if strings.Contains(content, p) {
fm.sendAlertWithPath(alert.Critical, "php_dropper_realtime",
fmt.Sprintf("PHP dropper with paste site URL: %s", path),
fmt.Sprintf("Fetches from: %s", p), path, procInfo)
return contentScanResult{matched: true}
}
}
githubURLs := []string{"gist.githubusercontent.com", "raw.githubusercontent.com"}
dangerousFns := []string{"file_put_contents(", "fwrite(", "shell_", "passthru(", "popen("}
for _, gh := range githubURLs {
if !strings.Contains(content, gh) {
continue
}
for _, line := range strings.Split(content, "\n") {
if !strings.Contains(line, gh) {
continue
}
for _, fn := range dangerousFns {
if strings.Contains(line, fn) {
fm.sendAlertWithPath(alert.Critical, "php_dropper_realtime",
fmt.Sprintf("PHP dropper fetching from GitHub with dangerous call: %s", path),
fmt.Sprintf("URL: %s, Function: %s", gh, fn), path, procInfo)
return contentScanResult{matched: true}
}
}
}
}
// eval + decoder combo — require same-line nesting to avoid FPs on
// legitimate plugins that use these functions in unrelated contexts.
evalStr := "eval(" // search target for PHP eval function calls
assertStr := "assert(" // search target for PHP assert function calls
decoders := realtimePHPDecoders[:]
// Library docblocks show decoder usage examples on one line, so this
// test reads executable code only. The head starts at byte 0, where tag,
// string and comment boundaries are known; the tail check below stays on
// raw bytes because its window can open inside a string.
code := executable
for _, line := range strings.Split(code, "\n") {
lineHasEval := strings.Contains(line, evalStr) || strings.Contains(line, assertStr)
if !lineHasEval {
continue
}
for _, dec := range decoders {
if strings.Contains(line, dec) {
message := fmt.Sprintf("Obfuscated PHP detected: %s", path)
details := fmt.Sprintf("PHP code execution with %s on same line", dec)
if confirmedExecution {
fm.sendConfirmedPHPFinding(fd, data, scanned, "obfuscated_php_realtime", message, details, path, procInfo)
} else {
fm.sendAlertWithPath(alert.Critical, "obfuscated_php_realtime", message, details, path, procInfo)
}
return contentScanResult{matched: true}
}
}
}
// Fragmented base64 evasion: $a="base"; $b="64_decode"; $c=$a.$b;
if strings.Contains(content, "\"base\"") || strings.Contains(content, "'base'") {
if strings.Contains(content, "64_dec") && strings.Contains(content, evalStr) {
fm.sendAlertWithPath(alert.Critical, "obfuscated_php_realtime",
fmt.Sprintf("Fragmented base64_decode evasion detected: %s", path),
"base64_decode function name split across string variables", path, procInfo)
return contentScanResult{matched: true}
}
}
// Massive variable concatenation payload ($z .= "xxxx"; repeated thousands of times)
concatCount := strings.Count(content, ".= \"")
if concatCount > 50 && strings.Contains(content, evalStr) {
fm.sendAlertWithPath(alert.Critical, "obfuscated_php_realtime",
fmt.Sprintf("Concatenation payload detected: %s (%d concat ops)", path, concatCount),
"Variable built from hundreds of string concatenations then executed", path, procInfo)
return contentScanResult{matched: true}
}
// Shell execution with request input
// Uses containsFunc to avoid substring false positives
// (e.g. "WP_Filesystem(" matching "exec(", "preg_match(" matching "exec(")
shellFuncs := []string{"system(", "passthru(", "exec(", "shell_exec(", "popen("}
requestVars := []string{"$_request", "$_post", "$_get", "$_cookie", "$_server"}
hasShell := false
hasInput := false
for _, sf := range shellFuncs {
if containsFunc(content, sf) {
hasShell = true
}
}
for _, rv := range requestVars {
if strings.Contains(content, rv) {
hasInput = true
}
}
if hasShell && hasInput {
// Require shell function + request variable on the SAME line.
// Same-line narrowing is the actual detection: admin panels with
// both tokens in unrelated contexts stay quiet because they never
// co-occur on one line. A file-wide allowlist (e.g. "skip when
// 'wp_filesystem' appears anywhere") would be forgeable by any
// webshell that pastes the token into a comment.
for _, line := range strings.Split(content, "\n") {
lineHasShell := false
lineHasInput := false
for _, sf := range shellFuncs {
if containsFunc(line, sf) {
lineHasShell = true
break
}
}
for _, rv := range requestVars {
if strings.Contains(line, rv) {
lineHasInput = true
break
}
}
if lineHasShell && lineHasInput {
fm.sendAlertWithPath(alert.Critical, "webshell_content_realtime",
fmt.Sprintf("Webshell pattern detected: %s", path),
fmt.Sprintf("Shell execution with request input on same line: %s", strings.TrimSpace(line)), path, procInfo)
return contentScanResult{matched: true}
}
}
}
// Tail scan: for large files, also check the last 32KB.
// Attackers append payloads (eval+base64) at the end of legitimate PHP files,
// beyond the head scan window. Only do the cheap heuristic checks, not full
// signature scanning (which would be too slow on every large PHP file).
tailData := readTailFromFd(fd, 32768)
if complete && scanned.Size() > 32768 && len(tailData) != 32768 {
complete = false
}
if tailData != nil {
tail := strings.ToLower(string(tailData))
// Check for eval+decoder on same line in tail
for _, line := range strings.Split(tail, "\n") {
lineHasEval := strings.Contains(line, evalStr) || strings.Contains(line, assertStr)
if !lineHasEval {
continue
}
for _, dec := range decoders {
if strings.Contains(line, dec) {
fm.sendAlertWithPath(alert.Critical, "obfuscated_php_realtime",
fmt.Sprintf("Obfuscated PHP appended to file tail: %s", path),
fmt.Sprintf("PHP code execution with %s found at end of file", dec), path, procInfo)
return contentScanResult{matched: true}
}
}
}
// Fragmented base64 in tail
if strings.Contains(tail, "\"base\"") || strings.Contains(tail, "'base'") {
if strings.Contains(tail, "64_dec") && strings.Contains(tail, evalStr) {
fm.sendAlertWithPath(alert.Critical, "obfuscated_php_realtime",
fmt.Sprintf("Fragmented base64_decode evasion in file tail: %s", path),
"Payload appended at end of legitimate PHP file", path, procInfo)
return contentScanResult{matched: true}
}
}
// Concat payload with eval in tail
tailConcatCount := strings.Count(tail, ".= \"")
if tailConcatCount > 50 && strings.Contains(tail, evalStr) {
fm.sendAlertWithPath(alert.Critical, "obfuscated_php_realtime",
fmt.Sprintf("Concatenation payload in file tail: %s (%d concat ops)", path, tailConcatCount),
"Payload appended at end of legitimate PHP file", path, procInfo)
return contentScanResult{matched: true}
}
}
// Skip signature/YARA scanning for verified CMS core files.
// The wp_core periodic check validates files against official checksums;
// if a file's hash matches a known-clean core file, signature matches
// on it are false positives (e.g. $_POST in wp-includes, mail() in
// PHPMailer, fsockopen() in POP3.php).
// Hashed from the event descriptor, not by re-opening the path: the path
// can resolve to clean core content while the bytes just scanned were
// malicious, which would skip signature and YARA scanning for the file
// that was actually examined.
contentSize := int64(len(data))
var stat unix.Stat_t
if err := unix.Fstat(fd, &stat); err == nil && stat.Size > contentSize {
contentSize = stat.Size
}
if !checks.CMSCacheEmpty() && checks.CMSCacheMayContainSize(contentSize) &&
checks.IsVerifiedCMSHash(hashEventFD(fd, data, contentSize)) {
return contentScanResult{complete: complete}
}
// External signature + YARA scanning. The YAML engine sees the complete
// event-file size even though realtime analysis scans a bounded prefix, so
// per-rule file-size limits cannot be defeated by prefix truncation.
result := fm.scanSignatures(data, contentSize, path, filepath.Ext(path), procInfo, fd, scannedIdentity(fd))
result.complete = result.complete && complete
return result
}
// Images fitting the combined read budget are scanned in one piece so a
// payload cannot straddle two independently evaluated windows. Larger files
// get a head and tail window; the middle is left to the deep scan, subject
// to thresholds.full_scan_max_file_mb.
const (
imagePayloadHeadBytes = 65536
imagePayloadTailBytes = 65536
)
// checkImagePayload looks for executable PHP inside a file served as an image.
// Returns true when a finding was raised.
//
// Two shapes reach the same verdict. A polyglot is a genuine image container
// with PHP appended or stored in a metadata chunk: it renders in a browser,
// passes an upload filter that trusts getimagesize, and executes the moment
// any PHP file includes its path. A file that only wears an image name and
// holds PHP source is the same backdoor without the disguise. Neither is
// legitimate under a served tree, so the container is reported as context
// rather than used as a gate.
//
// A PHP opening tag by itself is not evidence. Plugin screenshots quote one
// in their description chunks, so an execution, inclusion or remote-fetch
// construct is required alongside it.
func (fm *FileMonitor) checkImagePayload(fd int, path, procInfo string) bool {
var st unix.Stat_t
if err := unix.Fstat(fd, &st); err != nil || st.Mode&unix.S_IFMT != unix.S_IFREG || st.Size <= 0 {
return false
}
const readBudget = imagePayloadHeadBytes + imagePayloadTailBytes
headBytes := imagePayloadHeadBytes
if st.Size <= readBudget {
headBytes = int(st.Size)
}
recordReadTruncation(fd, readBudget, "image_payload")
head := readFromFd(fd, headBytes)
if len(head) == 0 {
return false
}
container, _ := contenttype.ImageContainer(head)
evidence, found := phpExecutableContent(head)
if !found && st.Size > readBudget {
// The container verdict came from the head, so the tail is examined
// for the payload alone.
if tail := readTailFromFd(fd, imagePayloadTailBytes); tail != nil {
evidence, found = phpExecutableContent(tail)
}
}
if !found {
return false
}
describedContainer := container
if describedContainer == "" {
describedContainer = "none (file is not a valid image)"
}
fm.sendAlertWithPath(alert.Critical, "php_in_image_realtime",
fmt.Sprintf("Executable PHP inside image file: %s", path),
fmt.Sprintf("Container: %s\nEvidence: %s\nRemediation: this path is the payload; find and remove the PHP file that includes it", describedContainer, evidence),
path, procInfo)
return true
}
// checkHTMLPhishing reads an HTML file and checks for phishing indicators:
// brand impersonation + credential input + redirect/exfiltration.
// Uses event fd for content read and unix.Fstat for size (TOCTOU-safe).
func (fm *FileMonitor) checkHTMLPhishing(fd int, path, procInfo string) {
// Only check files in web-accessible directories.
//
// No path-allowlist below this point: the content gates (credential
// inputs + brand impersonation + exfil/trust-badge) reject legitimate
// framework HTML on their own. A previous allowlist for /wp-admin/,
// /wp-content/themes/, /wp-content/plugins/, /node_modules/, /vendor/,
// /.well-known/ let an attacker who compromised any of those dirs drop
// a credential-harvesting page with full suppression.
if !fm.underDocRoot(path) {
return
}
var stat unix.Stat_t
if err := unix.Fstat(fd, &stat); err != nil {
return
}
size := stat.Size
if size < 500 || size > 500000 {
return
}
recordReadTruncation(fd, 16384, "html_phishing")
data := readFromFd(fd, 16384)
if data == nil {
return
}
content := strings.ToLower(string(data))
// Must have a form with credential inputs
if !strings.Contains(content, "<form") && !strings.Contains(content, "<input") {
return
}
hasCredInput := strings.Contains(content, "type=\"email\"") ||
strings.Contains(content, "type=\"password\"") ||
strings.Contains(content, "type='email'") ||
strings.Contains(content, "type='password'") ||
strings.Contains(content, "name=\"email\"") ||
strings.Contains(content, "name=\"password\"") ||
strings.Contains(content, "placeholder=\"you@")
if !hasCredInput {
return
}
// Check for brand impersonation
brands := []struct {
name string
patterns []string
}{
{"Microsoft/SharePoint", []string{"sharepoint", "onedrive", "microsoft 365", "outlook web", "office 365"}},
{"Google", []string{"google drive", "google docs", "accounts.google", "gmail"}},
{"Dropbox", []string{"dropbox"}},
{"DocuSign", []string{"docusign"}},
{"Adobe", []string{"adobe sign", "adobe document"}},
{"Apple/iCloud", []string{"icloud", "apple id"}},
{"Webmail", []string{"roundcube", "horde", "webmail login", "zimbra"}},
{"Generic", []string{"secure access", "verify your", "confirm your identity", "account verification"}},
}
brandMatch := ""
for _, b := range brands {
for _, p := range b.patterns {
if strings.Contains(content, p) {
brandMatch = b.name
break
}
}
if brandMatch != "" {
break
}
}
if brandMatch == "" {
return
}
// Check for redirect/exfiltration patterns
exfilPatterns := []string{
"window.location.href", "window.location.replace", "window.location =",
".workers.dev", "fetch(", "xmlhttprequest",
}
hasExfil := false
for _, p := range exfilPatterns {
if strings.Contains(content, p) {
hasExfil = true
break
}
}
// Also check for trust badges (strong phishing signal)
hasTrustBadge := strings.Contains(content, "secured by microsoft") ||
strings.Contains(content, "secured by google") ||
strings.Contains(content, "256-bit encrypted") ||
strings.Contains(content, "256‑bit encrypted")
if hasExfil || hasTrustBadge {
fm.sendAlertWithPath(alert.Critical, "phishing_realtime",
fmt.Sprintf("Phishing page created (%s impersonation): %s", brandMatch, path),
fmt.Sprintf("Size: %d bytes", size), path, procInfo)
return
}
// Run signature/YARA scanning on HTML content not caught by phishing heuristics
fm.runEventSignatureScan(fd, data, path, ".html", procInfo)
}
// checkCredentialLog reads a text file and checks if it contains harvested
// email:password pairs - output from an active phishing kit. The path is used
// only for the suppression/location checks; the content is read from the
// fanotify event fd (not re-opened by path) so an attacker cannot swap the
// file between the event and the read.
func (fm *FileMonitor) checkCredentialLog(fd int, path, procInfo string) {
if !fm.underDocRoot(path) {
return
}
// Exclude known config file paths - these legitimately contain email-like patterns.
if strings.HasPrefix(path, "/etc/") {
return
}
for _, suffix := range []string{".conf", ".cfg", ".ini", ".yaml", ".yml"} {
if strings.HasSuffix(path, suffix) {
return
}
}
data := readFromFd(fd, 4096)
if data == nil {
return
}
content := string(data)
lines := strings.Split(content, "\n")
credLines := 0
emailCount := 0
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if strings.Contains(line, "@") {
emailCount++
for _, delim := range []string{":", "|", "\t", ","} {
parts := strings.SplitN(line, delim, 3)
if len(parts) >= 2 {
p0 := strings.TrimSpace(parts[0])
p1 := strings.TrimSpace(parts[1])
if strings.Contains(p0, "@") && len(p1) > 0 && !strings.Contains(p1, " ") {
credLines++
break
}
}
}
}
}
if credLines >= 5 {
fm.sendAlertWithPath(alert.Critical, "credential_log_realtime",
fmt.Sprintf("Harvested credential log detected: %s", path),
fmt.Sprintf("%d credential lines (email:password format) found", credLines), path, procInfo)
} else if emailCount >= 10 {
fm.sendAlertWithPath(alert.High, "credential_log_realtime",
fmt.Sprintf("Possible harvested email list: %s", path),
fmt.Sprintf("%d email addresses found in %s", emailCount, filepath.Base(path)), path, procInfo)
}
}
// checkPhishingZip checks if a newly created ZIP file matches known
// phishing-kit archive name patterns. The signal is the COMBINATION of a
// brand name and a phishing-suggestive token in the same filename --
// "office365-login.zip", "paypal-verify.zip", "microsoft-secure.zip".
// Plain plugin distribution backups (google-site-kit.zip, mailchimp.zip)
// have a brand without an action verb and don't fire.
func (fm *FileMonitor) checkPhishingZip(path, nameLower, procInfo string) {
if !fm.underDocRoot(path) {
return
}
// Brand impersonation targets: filenames mimicking a service users log in to.
brands := []string{
"office365", "office 365", "sharepoint", "onedrive",
"microsoft", "outlook", "google", "gmail",
"dropbox", "docusign", "adobe", "wetransfer",
"paypal", "apple", "icloud", "netflix",
"facebook", "instagram", "linkedin",
"webmail", "roundcube", "cpanel",
}
// Phishing-suggestive verbs/nouns. These must co-occur with a brand
// for the rule to fire. "kit" is intentionally NOT here -- many
// official WordPress plugin slugs end in -kit (google-site-kit,
// mailchimp-for-wp-kit) and were the dominant FP source.
phishingIndicators := []string{
"login", "signin", "sign-in", "sign_in",
"verify", "verification",
"secure", "security",
"phish", "scam",
"bank", "account",
"capture", "harvest", "steal",
}
var matchedBrand string
for _, b := range brands {
if strings.Contains(nameLower, b) {
matchedBrand = b
break
}
}
if matchedBrand == "" {
return
}
var matchedIndicator string
for _, p := range phishingIndicators {
if strings.Contains(nameLower, p) {
matchedIndicator = p
break
}
}
if matchedIndicator == "" {
return
}
fm.sendAlertWithPath(alert.High, "phishing_kit_realtime",
fmt.Sprintf("Suspected phishing kit archive uploaded: %s", path),
fmt.Sprintf("Filename combines brand '%s' with phishing indicator '%s'", matchedBrand, matchedIndicator),
path, procInfo)
}
// runSignatureScan runs YAML and YARA signature scanning on file content.
// Returns true if a match was found and an alert was sent.
// Non-critical YAML matches use directory-level dedup to avoid alert floods
// when a plugin directory has many files matching the same rule.
// Critical matches (backdoors, webshells) always alert per-file.
// scannedIdentity describes the object behind an event descriptor. Stat of the
// /proc magic link resolves the open file itself rather than walking the path
// again, so it still names the scanned inode after the path has been replaced.
// os.NewFile is avoided deliberately: its finalizer can close a descriptor the
// daemon still owns.
func scannedIdentity(fd int) os.FileInfo {
info, err := os.Stat(fmt.Sprintf("/proc/self/fd/%d", fd))
if err != nil {
return nil
}
return info
}
func (fm *FileMonitor) runSignatureScan(data []byte, path, ext, procInfo string) bool {
return fm.runSignatureScanWithSize(data, int64(len(data)), path, ext, procInfo, -1, nil)
}
func (fm *FileMonitor) runEventSignatureScan(fd int, data []byte, path, ext, procInfo string) bool {
return fm.runSignatureScanWithSize(data, int64(len(data)), path, ext, procInfo, fd, scannedIdentity(fd))
}
func (fm *FileMonitor) runSignatureScanWithSize(data []byte, contentSize int64, path, ext, procInfo string, fd int, scanned os.FileInfo) bool {
return fm.scanSignatures(data, contentSize, path, ext, procInfo, fd, scanned).matched
}
func (fm *FileMonitor) scanSignatures(data []byte, contentSize int64, path, ext, procInfo string, fd int, scanned os.FileInfo) contentScanResult {
// Both engines see every file. A .yml hit used to end the scan here, so
// a file matching a High .yml rule never met the Critical YARA rule and
// the inline quarantine that only a Critical match triggers. Only a file
// the .yml path already moved to quarantine is not handed to YARA.
matched := false
if scanner := signatures.Global(); scanner != nil {
matches := scanner.ScanContentWithSize(data, ext, contentSize)
if len(matches) > 0 {
matched = true
m := matches[0]
sev := alert.High
if m.Severity == "critical" {
sev = alert.Critical
}
// Non-critical: dedup by rule+directory so 30 files in the same
// plugin matching the same rule produce one alert, not 30.
// Critical matches always alert per-file (real path for quarantine).
suppressed := false
if sev != alert.Critical {
dirKey := m.RuleName + ":" + filepath.Dir(path)
suppressed = !fm.shouldAlert("signature_match_realtime", dirKey)
}
if !suppressed {
details := fmt.Sprintf("Category: %s\nDescription: %s\nMatched: %s",
m.Category, m.Description, strings.Join(m.Matched, ", "))
details += signatures.ReferencedPayloadDetail(data)
finding := alert.Finding{
Severity: sev,
Check: "signature_match_realtime",
Message: fmt.Sprintf("Signature match [%s]: %s", m.RuleName, path),
Details: details,
FilePath: path,
ProcessInfo: procInfo,
}
var qPath string
var quarantined bool
var paused *alert.Finding
if sev == alert.Critical {
// Capture provenance before remediation can remove the source.
checks.StampContentFingerprint(&finding)
qPath, quarantined, paused = checks.InlineQuarantineGatedIdentified(fm.currentCfg(), &finding, path, data, scanned)
}
// Publish after the inline decision so delivery sees its budget
// provenance. A rejected window can still get full-file validation.
fm.sendFileFinding(finding)
if paused != nil && !alert.TryEnqueue(fm.alertCh, *paused) {
atomic.AddInt64(&fm.droppedAlerts, 1)
}
if quarantined {
fm.recordDropperQuarantine(path, fd)
fm.sendAlert(alert.Critical, "auto_response",
fmt.Sprintf("AUTO-QUARANTINE (inline): %s moved to quarantine", path),
fmt.Sprintf("Quarantined to: %s\nRule: %s", qPath, m.RuleName))
return contentScanResult{matched: true, complete: true}
}
}
}
}
if yaraScanner := yara.Active(); yaraScanner != nil {
matches, err := yara.ScanBytesChecked(yaraScanner, path, data)
if err != nil {
fm.reportYARAScanError(path, err)
return contentScanResult{matched: matched}
}
if len(matches) > 0 {
fm.sendFileFinding(alert.Finding{
Severity: alert.Critical,
Check: "yara_match_realtime",
Message: fmt.Sprintf("YARA rule match [%s]: %s", matches[0].RuleName, path),
Details: fmt.Sprintf("Matched %d YARA rule(s)", len(matches)) + signatures.ReferencedPayloadDetail(data),
FilePath: path,
ProcessInfo: procInfo,
ContentSHA256: windowDigest(data, contentSize, scanned),
})
return contentScanResult{matched: true, complete: true}
}
}
return contentScanResult{matched: matched, complete: true}
}
// windowDigest fingerprints the scanned bytes only when they are the whole
// file. A window short of it would never match the whole-file digest Re-check
// compares against, so it is left empty for the later whole-file stamp.
func windowDigest(data []byte, contentSize int64, scanned os.FileInfo) string {
n := int64(len(data))
if n != contentSize || (scanned != nil && scanned.Size() != n) {
return ""
}
return fmt.Sprintf("%x", sha256.Sum256(data))
}
// stopping reports whether this monitor has been signalled to stop. A nil
// stopCh (a monitor built directly in a test) is never stopping, because a
// receive on a nil channel cannot proceed.
func (fm *FileMonitor) stopping() bool {
select {
case <-fm.stopCh:
return true
default:
return false
}
}
// reportYARAScanError names a changed file the scanner could not inspect.
// It is its own check rather than the deep scan's "yara_scan_incomplete",
// which reports scheduled coverage: that report fires for every archive past
// the scan size limit, roughly thirteen times a day forever on a live host,
// and sharing the name left a real scanning outage indistinguishable from
// routine backlog.
//
// Shutdown is not an outage. The daemon stops the YARA backend while this
// monitor's goroutine is still draining events, because the wait for workers
// comes after the teardown, so a clean restart otherwise reported a
// High-severity scanning failure every time. The teardown cannot move after
// that wait, which is unbounded and would hang on a wedged worker. The
// return happens before the rate-limit window is taken, so a suppressed
// shutdown report cannot swallow the first genuine failure afterwards.
func (fm *FileMonitor) reportYARAScanError(path string, err error) {
if fm.stopping() {
return
}
fm.yaraErrorReportMu.Lock()
if !fm.lastYARAError.IsZero() && time.Since(fm.lastYARAError) < time.Minute {
fm.yaraErrorReportMu.Unlock()
return
}
fm.lastYARAError = time.Now()
fm.yaraErrorReportMu.Unlock()
fm.sendAlert(alert.High, "yara_realtime_scan_error",
"YARA real-time scan could not inspect a changed file",
fmt.Sprintf("File: %s\nError: %v", path, err))
}
// M7 - sendAlert uses droppedAlerts counter, separate from droppedEvents.
// No dedup - only used for system-level alerts (overflow reporting) that are
// already ticker-gated. File-related alerts should use sendAlertWithPath.
func (fm *FileMonitor) sendAlert(severity alert.Severity, check, message, details string) {
finding := alert.Finding{
Severity: severity,
Check: check,
Message: message,
Details: details,
Timestamp: time.Now(),
}
if !alert.TryEnqueue(fm.alertCh, finding) {
atomic.AddInt64(&fm.droppedAlerts, 1)
}
}
// sendAlertWithPath is like sendAlert but also sets the FilePath and
// ProcessInfo fields for structured propagation to auto-response.
// Applies per-path deduplication to prevent alert storms from rapid writes.
func (fm *FileMonitor) sendAlertWithPath(severity alert.Severity, check, message, details, filePath, processInfo string) {
fm.sendFileFinding(alert.Finding{
Severity: severity,
Check: check,
Message: message,
Details: details,
FilePath: filePath,
ProcessInfo: processInfo,
})
}
var inlineQuarantineConfirmedPHP = checks.InlineQuarantineConfirmedPHPGatedIdentifiedFD
// sendConfirmedPHPFinding gives the existing automatic file-response policy
// the exact content window and file identity that proved a Critical PHP
// execution finding. The finding is published after the response decision so
// alert delivery cannot retry an attempt that already consumed the budget.
func (fm *FileMonitor) sendConfirmedPHPFinding(fd int, data []byte, scanned os.FileInfo, check, message, details, filePath, processInfo string) {
finding := alert.Finding{
Severity: alert.Critical,
Check: check,
Message: message,
Details: details,
FilePath: filePath,
ProcessInfo: processInfo,
}
if !fm.prepareFileFindingWithDedup(&finding, confirmedPHPDedupPath(filePath, data, scanned)) {
return
}
if scanned == nil {
fm.enqueuePreparedFileFinding(finding)
return
}
qPath, quarantined, paused := inlineQuarantineConfirmedPHP(
fm.currentCfg(), &finding, filePath, data, scanned, fd,
)
fm.enqueuePreparedFileFinding(finding)
if paused != nil && !alert.TryEnqueue(fm.alertCh, *paused) {
atomic.AddInt64(&fm.droppedAlerts, 1)
}
if !quarantined {
return
}
fm.recordDropperQuarantine(filePath, fd)
fm.sendAlert(alert.Critical, "auto_response",
fmt.Sprintf("AUTO-QUARANTINE (inline): %s moved to quarantine", filePath),
fmt.Sprintf("Quarantined to: %s\nDetection: %s", qPath, check))
}
func (fm *FileMonitor) sendFileFinding(finding alert.Finding) {
if !fm.prepareFileFinding(&finding) {
return
}
fm.enqueuePreparedFileFinding(finding)
}
func (fm *FileMonitor) prepareFileFinding(finding *alert.Finding) bool {
dedupPath := finding.FilePath
if finding.DedupKey != "" {
// A later header or digest can change a staged finding's identity
// within the path cooldown. Let persistent dedup see that evidence.
dedupPath = finding.Key()
}
return fm.prepareFileFindingWithDedup(finding, dedupPath)
}
func (fm *FileMonitor) prepareFileFindingWithDedup(finding *alert.Finding, dedupPath string) bool {
if !fm.shouldAlert(finding.Check, dedupPath) {
return false
}
finding.Timestamp = time.Now()
checks.StampContentFingerprint(finding)
return true
}
func confirmedPHPDedupPath(path string, data []byte, scanned os.FileInfo) string {
digest := sha256.Sum256(data)
if scanned == nil {
return fmt.Sprintf("%s\x00unstable:%x", path, digest)
}
if stat, ok := scanned.Sys().(*syscall.Stat_t); ok {
return fmt.Sprintf("%s\x00%d:%d:%d:%d:%d:%x", path, stat.Dev, stat.Ino, stat.Ctim.Sec, stat.Ctim.Nsec, scanned.Size(), digest)
}
return fmt.Sprintf("%s\x00%d:%d:%x", path, scanned.Size(), scanned.ModTime().UnixNano(), digest)
}
func (fm *FileMonitor) enqueuePreparedFileFinding(finding alert.Finding) {
if !alert.TryEnqueue(fm.alertCh, finding) {
atomic.AddInt64(&fm.droppedAlerts, 1)
}
}
// shouldAlert returns true if this check+path combination hasn't been alerted
// recently. Prevents duplicate alerts from rapid writes to the same file.
// Uses LoadOrStore for atomic initial insertion to avoid TOCTOU races
// between concurrent analyzer workers.
func (fm *FileMonitor) shouldAlert(check, filePath string) bool {
return fm.shouldAlertAt(check, filePath, time.Now())
}
func (fm *FileMonitor) shouldAlertAt(check, filePath string, now time.Time) bool {
if filePath == "" {
return true // no path = no dedup possible
}
key := check + ":" + filePath
for {
v, loaded := fm.alertDedup.LoadOrStore(key, now)
if !loaded {
return true
}
if now.Sub(v.(time.Time)) < alertDedupTTL {
return false
}
if fm.alertDedup.CompareAndSwap(key, v, now) {
return true
}
}
}
// M7 - overflowReporter reports dropped events and alerts separately.
//
// Periodic ticks and eager signals feed this loop:
// - 1-minute ticker: emits the periodic fanotify_overflow alert,
// resets drop counters, runs reconcileDrops, and evicts stale alert
// dedup entries.
// - retry ticker: retries pending work even after new drops stop.
// - reconcileSig: out-of-cycle reconcile triggered by sendEvent when
// sustained drops cross eagerReconcileDropThreshold within the
// current tick. Closes the latency gap between a drop and its
// reconcile read so the file's mtime is still inside reconcileWindow.
func (fm *FileMonitor) overflowReporter() {
defer fm.wg.Done()
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
// Retried work must not depend on another drop or a nonzero alert counter.
retry := time.NewTicker(reconcileMinInterval)
defer retry.Stop()
for {
select {
case <-fm.stopCh:
return
case <-retry.C:
fm.startReconcile()
case <-fm.reconcileSig:
// Eager reconcile: do not reset counters, do not emit the
// minute-tick alert. Just walk the tracked dirs and surface
// any interesting file inside reconcileWindow. The minute
// tick will still fire its alert + drain the counters when
// it arrives.
fm.startReconcile()
case <-ticker.C:
droppedEv := atomic.SwapInt64(&fm.droppedEvents, 0)
droppedAl := atomic.SwapInt64(&fm.droppedAlerts, 0)
if droppedEv > 0 {
fm.sendAlert(alert.Warning, "fanotify_overflow",
fmt.Sprintf("fanotify event queue overflowed: %d events dropped in last minute", droppedEv),
"Possible event storm (backup, bulk update) or high-volume attack")
// Recover coverage: scan files in directories that saw drops
// so a threat landing during the storm is still detected.
fm.startReconcile()
}
if droppedAl > 0 {
fmt.Fprintf(os.Stderr, "[%s] alert channel full: %d alerts dropped in last minute\n", ts(), droppedAl)
}
// Evict stale dedup entries every minute
now := time.Now()
fm.alertDedup.Range(func(key, value any) bool {
if now.Sub(value.(time.Time)) > alertDedupTTL {
fm.alertDedup.CompareAndDelete(key, value)
}
return true
})
evictStaleWPPathStatCache(now)
}
}
}
// evictStaleWPPathStatCache bounds the package-level WordPress path stat cache.
// The compare-delete keeps the minute sweep from removing a fresh stat result
// stored by an analyzer worker after Range observed an older entry.
func evictStaleWPPathStatCache(now time.Time) {
cutoff := 2 * wpPathCacheTTL
wpPathStatCache.Range(func(key, value any) bool {
entry, ok := value.(wpPathCacheEntry)
if !ok {
wpPathStatCache.Delete(key)
return true
}
if now.Sub(entry.ts) > cutoff {
wpPathStatCache.CompareAndDelete(key, entry)
}
return true
})
}
// isPHPExtension returns true for all PHP file extensions that can execute code.
// containsFunc checks if content contains a function call that isn't part of
// a longer identifier. Prevents "WP_Filesystem(" matching "exec(" or
// "preg_match(" matching "exec(". Checks the character before the match
// is not a letter, digit, or underscore.
func containsFunc(content, funcCall string) bool {
idx := 0
for {
pos := strings.Index(content[idx:], funcCall)
if pos < 0 {
return false
}
absPos := idx + pos
if absPos == 0 {
return true
}
prev := content[absPos-1]
if (prev < 'a' || prev > 'z') && (prev < 'A' || prev > 'Z') &&
(prev < '0' || prev > '9') && prev != '_' {
return true
}
idx = absPos + len(funcCall)
if idx >= len(content) {
return false
}
}
}
func isPHPExtension(nameLower string) bool {
// Single source of truth shared with the periodic content scanners and the
// rule engines so no path drifts on which extensions execute PHP.
return contenttype.IsExecutablePHPName(nameLower)
}
func isPHPSourceExtension(nameLower string) bool {
return contenttype.IsPHPSourceName(nameLower)
}
func isCGIExtension(nameLower string) bool {
return strings.HasSuffix(nameLower, ".pl") ||
strings.HasSuffix(nameLower, ".cgi") ||
strings.HasSuffix(nameLower, ".py") ||
strings.HasSuffix(nameLower, ".sh") ||
strings.HasSuffix(nameLower, ".rb")
}
// checkCGIBackdoor reads a CGI script and checks for backdoor patterns.
// Detects Perl/Python/Bash backdoors like the LEVIATHAN toolkit.
func (fm *FileMonitor) checkCGIBackdoor(fd int, path, procInfo string) {
recordReadTruncation(fd, 32768, "cgi_backdoor")
data := readFromFd(fd, 32768)
if data == nil {
return
}
content := strings.ToLower(string(data))
// Backdoor indicators in CGI scripts
indicators := 0
var matched []string
// Indicators weighted by suspicion level. Generic patterns like
// "request_method" and "cmd" removed — they match every CGI script.
shellPatterns := []struct {
pattern string
desc string
}{
{"system(", "system() call"},
{"os.popen", "os.popen() call"},
{"`$", "backtick execution with variable"},
{"content_length", "reads POST body length"},
{"base64_decode", "base64 decoding"},
{"$_post", "PHP POST input"},
{"$_get", "PHP GET input"},
{"param(", "CGI parameter read"},
{"qs.parse", "query string parsing"},
}
for _, sp := range shellPatterns {
if strings.Contains(content, sp.pattern) {
indicators++
matched = append(matched, sp.desc)
}
}
// 4+ indicators = likely backdoor
if indicators >= 4 {
fm.sendAlertWithPath(alert.Critical, "cgi_backdoor_realtime",
fmt.Sprintf("CGI backdoor detected: %s", path),
fmt.Sprintf("Indicators (%d): %s", indicators, strings.Join(matched, ", ")), path, procInfo)
return
}
// CGI scripts in unusual locations (images, css, js directories)
if strings.Contains(path, "/img/") || strings.Contains(path, "/images/") ||
strings.Contains(path, "/css/") || strings.Contains(path, "/js/") ||
strings.Contains(path, "/fonts/") || strings.Contains(path, "/icons/") {
fm.sendAlertWithPath(alert.High, "cgi_suspicious_location_realtime",
fmt.Sprintf("CGI script in non-CGI directory: %s", path),
"Scripts should not exist in image/css/js directories", path, procInfo)
return
}
// Run signature scan on the content
fm.runEventSignatureScan(fd, data, path, filepath.Ext(path), procInfo)
}
// suppressedPath reports whether an operator suppression covers path.
func (fm *FileMonitor) suppressedPath(path string) bool {
for _, ignore := range fm.currentCfg().Suppressions.IgnorePaths {
if matchSuppression(ignore, path) {
return true
}
}
return false
}
// matchSuppression checks if a file path matches a suppression glob pattern.
// Supports patterns like "*/cache/*", "*/vendor/*", "*.log".
func matchSuppression(pattern, path string) bool {
if pattern == "" {
return false
}
// Direct match against full path
if m, _ := filepath.Match(pattern, path); m {
return true
}
// Match against basename (e.g. "*.log")
if m, _ := filepath.Match(pattern, filepath.Base(path)); m {
return true
}
if !strings.ContainsAny(pattern, "*?[") {
return strings.Contains(path, pattern)
}
if strings.ContainsAny(pattern, "?[") || !hasLeadingAnyDepthSuppressionGlob(pattern) {
return false
}
residue := strings.ReplaceAll(pattern, "*", "")
if strings.Contains(residue, "/") && strings.Trim(residue, "/") != "" {
return strings.Contains(path, residue)
}
return false
}
func hasLeadingAnyDepthSuppressionGlob(pattern string) bool {
firstSlash := strings.Index(pattern, "/")
if firstSlash <= 0 {
return false
}
return strings.Trim(pattern[:firstSlash], "*") == ""
}
// readHead opens a file by path and reads the first maxBytes.
// Kept for path-based checks (HTML phishing, credential logs, ZIP checks)
// that need os.Stat for file size anyway.
func readHead(path string, maxBytes int) []byte {
if maxBytes <= 0 {
return nil
}
// #nosec G304 -- readHead scans files surfaced by fanotify/scanner;
// reading user files for signature analysis is the daemon's purpose.
f, err := os.Open(path)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
// ReadAll over a LimitReader, not a single f.Read into a pre-sized
// buffer: a short read would hand only a prefix to the detectors and
// silently miss content deeper in the file.
buf, err := io.ReadAll(io.LimitReader(f, int64(maxBytes)))
if err != nil || len(buf) == 0 {
return nil
}
return buf
}
// looksLikePluginUpdate checks if a PHP file in uploads looks like a plugin
// update temp directory (e.g., elementor_t0q9y). Returns true if it matches
// the pattern of a known plugin extracting an update.
// M3 - uses sync.Map cache with 5-minute TTL for plugin directory stat results.
func looksLikePluginUpdate(path string) bool {
// WordPress plugin updates extract to /uploads/{pluginname}_{random}/
// Detect by extracting the directory name under uploads/ and checking
// if a matching plugin exists in wp-content/plugins/.
// No hardcoded whitelist - works for all 60,000+ WP plugins.
uploadsIdx := strings.Index(path, "/wp-content/uploads/")
if uploadsIdx < 0 {
return false
}
wpRoot := path[:uploadsIdx]
afterUploads := path[uploadsIdx+len("/wp-content/uploads/"):]
// Extract the first directory component: "header-footer_7ocsd"
slashIdx := strings.Index(afterUploads, "/")
if slashIdx < 0 {
return false
}
dirName := afterUploads[:slashIdx]
// Strip the random suffix (e.g. "_7ocsd") - WordPress appends _XXXXX
// The plugin name is everything before the last underscore-followed-by-random
pluginName := dirName
if lastUnderscore := strings.LastIndex(dirName, "_"); lastUnderscore > 0 {
suffix := dirName[lastUnderscore+1:]
// Random suffixes are short alphanumeric strings (5-8 chars)
if len(suffix) >= 4 && len(suffix) <= 10 {
pluginName = dirName[:lastUnderscore]
}
}
// Check if a matching plugin directory exists in plugins/
return cachedPathExists(wpRoot + "/wp-content/plugins/" + pluginName)
}
// cachedPathExists answers whether path exists, memoised for wpPathCacheTTL
// when it does and for wpPathNegativeTTL when it does not. The realtime path
// asks this once per file event during an update, so an uncached stat per
// staged file would be paid thousands of times per package; a missing path
// during an update is transient, so its answer must expire quickly.
func cachedPathExists(path string) bool {
if cached, ok := wpPathStatCache.Load(path); ok {
if entry, ok := cached.(wpPathCacheEntry); ok {
ttl := wpPathCacheTTL
if !entry.exists {
ttl = wpPathNegativeTTL
}
if time.Since(entry.ts) < ttl {
return entry.exists
}
}
}
_, err := os.Stat(path)
exists := err == nil
wpPathStatCache.Store(path, wpPathCacheEntry{
exists: exists,
ts: time.Now(),
})
return exists
}
//go:build linux
package daemon
import (
"errors"
"io/fs"
"os"
"path/filepath"
"syscall"
"time"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/queuehealth"
)
type reconcileDirectory struct {
firstDrop time.Time
lastDrop time.Time
ticket queuehealth.Ticket
// Keep the original scan window and progress when a pass yields.
cutoff time.Time
after string
failed bool
// Renames and hard links preserve mtime, so their recovery has no age gate.
placements bool
// A BPF placement can still be waiting for the syscall to commit.
ready time.Time
}
// Repeated drops refresh eviction priority without postponing recovery age.
func (fm *FileMonitor) recordDroppedDir(path string) {
fm.recordDroppedEvent(fileEvent{path: path})
}
func (fm *FileMonitor) recordDroppedEvent(event fileEvent) {
fm.recordDroppedEventAfter(event, time.Time{})
}
func (fm *FileMonitor) recordDroppedEventAfter(event fileEvent, ready time.Time) {
fm.initQueueHealth()
dir := filepath.Dir(event.path)
fm.reconcileMu.Lock()
defer fm.reconcileMu.Unlock()
if fm.reconcileDirs == nil {
fm.reconcileDirs = make(map[string]reconcileDirectory)
}
now := time.Now()
entry, exists := fm.reconcileDirs[dir]
if !exists {
entry.firstDrop = now
entry.ticket = fm.reconcileHealth.Begin(now)
}
// A new write may precede the saved cursor and must be examined again.
entry.after = ""
entry.placements = entry.placements || event.mask&eventPlacedIn != 0
if ready.After(entry.ready) {
entry.ready = ready
}
entry.lastDrop = now
fm.reconcileDirs[dir] = entry
fm.evictOverflowDirLocked(now)
}
// evictOverflowDirLocked drops the least recently refreshed directory once the
// tracker is over its cap. The caller holds reconcileMu.
func (fm *FileMonitor) evictOverflowDirLocked(now time.Time) {
if len(fm.reconcileDirs) <= reconcileDirCap {
return
}
var oldestKey string
var oldestTime time.Time
first := true
for path, candidate := range fm.reconcileDirs {
if first || candidate.lastDrop.Before(oldestTime) {
oldestKey, oldestTime, first = path, candidate.lastDrop, false
}
}
fm.reconcileDirs[oldestKey].ticket.Reject(now)
delete(fm.reconcileDirs, oldestKey)
}
// startReconcile runs one recovery pass off the caller's goroutine. Passes
// never overlap and never run closer together than reconcileMinInterval: the
// eager trigger fires on drop volume, and a storm produces that volume far
// faster than a pass can absorb it.
func (fm *FileMonitor) startReconcile() {
select {
case <-fm.stopCh:
return
default:
}
if !fm.reconcileRunning.CompareAndSwap(false, true) {
return
}
// Acquire single-flight ownership before reading the completion time:
// the previous pass may finish while this caller is being scheduled.
now := time.Now()
if last := fm.reconcileLastPass.Load(); last != 0 && now.Sub(time.Unix(0, last)) < reconcileMinInterval {
fm.reconcileRunning.Store(false)
return
}
fm.reconcileMu.Lock()
pending := len(fm.reconcileDirs) > 0
fm.reconcileMu.Unlock()
if !pending {
fm.reconcileRunning.Store(false)
return
}
// The overflow reporter owns a wg count until its last call returns,
// so shutdown cannot observe zero while this Add is possible.
fm.wg.Add(1)
obs.Go("fanotify-reconcile", func() {
defer fm.wg.Done()
defer func() {
fm.reconcileLastPass.Store(time.Now().UnixNano())
fm.reconcileRunning.Store(false)
}()
fm.reconcileDrops()
})
}
func (fm *FileMonitor) reconcileDrops() {
fm.initQueueHealth()
fm.reconcileMu.Lock()
dirs := fm.reconcileDirs
fm.reconcileDirs = make(map[string]reconcileDirectory)
started := time.Now()
for dir, entry := range dirs {
entry.ticket.Start(started)
if entry.cutoff.IsZero() {
entry.cutoff = started.Add(-reconcileWindow)
}
dirs[dir] = entry
}
fm.reconcileMu.Unlock()
if len(dirs) == 0 {
return
}
// A panic abandons every unfinished directory in this detached batch.
defer func() {
now := time.Now()
for _, entry := range dirs {
entry.ticket.Reject(now)
}
}()
if fanotifyReconcileDur != nil {
defer func() { fanotifyReconcileDur.Observe(time.Since(started).Seconds()) }()
}
deadline := started.Add(reconcileBudget)
for dir, entry := range dirs {
if fm.reconcilePassDone(deadline) {
fm.deferReconcileDirs(dirs)
return
}
if !fm.reconcileDirectory(dir, &entry, deadline) {
// A pending placement can sort before this pass's cursor.
if started.Before(entry.ready) {
entry.after = ""
}
dirs[dir] = entry
fm.deferReconcileDirs(dirs)
return
}
// A pre-commit placement may have landed after this pass listed
// the directory. Keep its obligation for another pass. Still scan
// now so continuous drops cannot postpone all recovery forever.
if started.Before(entry.ready) {
entry.after = ""
fm.deferReconcileDirs(map[string]reconcileDirectory{dir: entry})
delete(dirs, dir)
continue
}
// Write-only recovery cannot recover an obligation outside its scan window.
if !entry.failed && (entry.placements || !entry.firstDrop.Before(entry.cutoff)) {
entry.ticket.Finish(time.Now())
} else {
entry.ticket.Reject(time.Now())
}
delete(dirs, dir)
}
}
// reconcilePassDone reports whether this pass must stop: its budget is spent,
// or the monitor is shutting down.
func (fm *FileMonitor) reconcilePassDone(deadline time.Time) bool {
select {
case <-fm.stopCh:
return true
default:
}
return !time.Now().Before(deadline)
}
// deferReconcileDirs returns directories this pass did not reach to the
// tracker so the next pass takes them, and empties the detached batch so the
// panic guard does not count them as lost. A directory that took a new drop
// while the pass ran is already represented by a waiting ticket; merging keeps
// the older admission age rather than restarting the clock on it.
func (fm *FileMonitor) deferReconcileDirs(dirs map[string]reconcileDirectory) {
now := time.Now()
fm.reconcileMu.Lock()
defer fm.reconcileMu.Unlock()
if fm.reconcileDirs == nil {
fm.reconcileDirs = make(map[string]reconcileDirectory)
}
for dir, entry := range dirs {
delete(dirs, dir)
if waiting, exists := fm.reconcileDirs[dir]; exists {
waiting.ticket.MergeRunning(entry.ticket, now)
if entry.firstDrop.Before(waiting.firstDrop) {
waiting.firstDrop = entry.firstDrop
}
if waiting.cutoff.IsZero() || entry.cutoff.Before(waiting.cutoff) {
waiting.cutoff = entry.cutoff
}
// The new admission may concern a file behind the old cursor.
waiting.after = ""
waiting.failed = waiting.failed || entry.failed
waiting.placements = waiting.placements || entry.placements
if entry.ready.After(waiting.ready) {
waiting.ready = entry.ready
}
fm.reconcileDirs[dir] = waiting
continue
}
entry.ticket.Requeue(now)
fm.reconcileDirs[dir] = entry
fm.evictOverflowDirLocked(now)
}
}
// A tree removed before recovery ran holds nothing left to scan, so the
// kernel loss that started the recovery is the only loss. Reads that failed
// for any other reason left work the operator can still act on.
// Returns false only when the pass yields; read failures stay on the
// obligation until completion so a later pass cannot conceal partial loss.
func (fm *FileMonitor) reconcileDirectory(dir string, work *reconcileDirectory, deadline time.Time) bool {
entries, err := os.ReadDir(dir)
if err != nil {
work.failed = work.failed || !errors.Is(err, fs.ErrNotExist)
return true
}
for _, entry := range entries {
if fm.reconcilePassDone(deadline) {
return false
}
// os.ReadDir sorts names, so continuation does not rescan the prefix.
if entry.Name() <= work.after {
continue
}
work.after = entry.Name()
if entry.IsDir() {
continue
}
path := filepath.Join(dir, entry.Name())
if !work.placements && !fm.isInteresting(path) {
continue
}
info, err := entry.Info()
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
work.failed = true
}
continue
}
if work.placements {
if info.Mode().IsRegular() {
st := info.Sys().(*syscall.Stat_t)
if !fm.reconcilePlacedFile(path, uint64(st.Dev), st.Ino) {
work.failed = true
}
}
continue
}
if info.ModTime().Before(work.cutoff) {
continue
}
if !fm.reconcileFile(path) {
work.failed = true
}
}
return true
}
func (fm *FileMonitor) reconcileFile(path string) bool {
// #nosec G304 -- path is a candidate in a directory recorded after a dropped event; its original event fd is no longer available.
file, err := os.Open(path)
if err != nil {
return errors.Is(err, fs.ErrNotExist)
}
// Close per file, including panic, rather than retaining the whole batch's fds.
defer func() { _ = file.Close() }()
// #nosec G115 -- Linux file descriptors are nonnegative int32 values and fit Go int.
fileAnalyzer(fm, fileEvent{path: path, fd: int(file.Fd())})
return true
}
// Reader and worker producers have joined before shutdown discards this map.
func (fm *FileMonitor) discardReconcilePending() {
fm.reconcileMu.Lock()
defer fm.reconcileMu.Unlock()
now := time.Now()
for dir, entry := range fm.reconcileDirs {
entry.ticket.Reject(now)
delete(fm.reconcileDirs, dir)
}
}
//go:build linux
package daemon
import (
"fmt"
"os"
"strings"
"sync"
"syscall"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/wpcheck"
)
// WordPress unpacks every core, plugin and theme package under
// wp-content/upgrade/<package>/<unpacked>/ and moves the unpacked tree into
// place a few hundred milliseconds later. The analyzer sees each staged PHP
// file in that window, usually before the checksum fetch for the package
// version has finished and sometimes before the header file that names the
// version has even been written.
//
// A staged file is therefore judged by hash, not by where it sits. Files the
// verifier can compare are compared; files whose checksums are still on the
// way wait here and are compared once they land, from the installed location
// if WordPress has moved them by then. Only a file that fails the comparison
// gets its own finding. Packages with no checksum source, and packages the
// fetch cannot resolve in time, collapse to one finding per staging directory.
// wpVerifier is the wordpress.org checksum verifier the analyzer consults.
type wpVerifier interface {
Describe(path string) wpcheck.Verification
Verify(v wpcheck.Verification) wpcheck.Verdict
VerifyFile(fd int, path string) wpcheck.Verification
}
const (
// stagedPackageTimeout bounds how long a staged file waits for its
// package checksums. The fetch is one ZIP download; a package still
// unresolved after this is reported at the directory level instead.
stagedPackageTimeout = 60 * time.Second
// stagedPackageQueueMax caps queued files across all packages. A full
// core or plugin update is a few thousand files; the cap only matters
// when many updates run at once while the fetches stall.
stagedPackageQueueMax = 20000
// stagedPackageIdleTTL retires per-package state that saw no new file.
stagedPackageIdleTTL = 10 * time.Minute
// wpPathNegativeTTL is how long a missing path stays cached. WordPress
// moves the old plugin to upgrade-temp-backup/ and renames the new tree
// in a few hundred milliseconds later, so a negative answer is transient
// by definition and must not hold for the five minutes a positive one does.
wpPathNegativeTTL = 2 * time.Second
)
// wpStagedPackage is a parsed wp-content/upgrade/<package>/<unpacked>/ path.
type wpStagedPackage struct {
dir string
unpacked string
wpRoot string
}
// parseWPStagedPackage identifies the staging directory and the unpacked tree
// a path sits in. Pure string work: the staging name is generated by
// WordPress in several shapes and carries no authority anyway, since anything
// able to write a staged tree chooses both names.
func parseWPStagedPackage(path string) wpStagedPackage {
const marker = "/wp-content/upgrade/"
idx := strings.Index(path, marker)
if idx < 0 {
return wpStagedPackage{}
}
// An installed plugin may ship upgrade-layout fixtures. Its outer root
// owns those files, just as it does when wpcheck describes their manifest.
if root, _ := wpcheck.DetectPluginRoot(path); root != "" && len(root) <= idx {
return wpStagedPackage{}
}
rest := path[idx+len(marker):]
pkg, inner, ok := strings.Cut(rest, "/")
if !ok || pkg == "" {
return wpStagedPackage{} // a file sitting directly in upgrade/ belongs to no package
}
unpacked, _, ok := strings.Cut(inner, "/")
if !ok || unpacked == "" {
return wpStagedPackage{} // the package directory holds no unpacked tree
}
return wpStagedPackage{
dir: path[:idx+len(marker)] + pkg,
unpacked: unpacked,
wpRoot: path[:idx],
}
}
// wpPackageInstalled reports whether the unpacked tree names something the
// site already has: a plugin or theme directory, its rollback copy under
// upgrade-temp-backup/ while an update is mid-flight, or a WordPress root for
// a core package.
func wpPackageInstalled(wpRoot, unpacked string) bool {
if unpacked == "wordpress" {
return cachedPathExists(wpRoot + "/wp-includes/version.php")
}
for _, dir := range []string{
"/wp-content/plugins/", "/wp-content/themes/",
"/wp-content/upgrade-temp-backup/plugins/", "/wp-content/upgrade-temp-backup/themes/",
} {
if cachedPathExists(wpRoot + dir + unpacked) {
return true
}
}
return false
}
type stagedPackageFile struct {
path string
procInfo string
v wpcheck.Verification
pkg wpStagedPackage
key stagedPackageKey
queuedAt time.Time
ticket queuehealth.Ticket
// coreHeader is the installed version.php as it was when a core file
// was queued; a zero stamp means there was none.
coreHeader fileStamp
}
// A pathname can be reused by a later unpack. Retained headers only belong
// to the tree whose directory identity was observed with the queued file.
type stagedPackageKey struct {
root string
dev uint64
ino uint64
}
// stagedPackageInfo is what the directory-level finding says about a package.
// installed is decided on the package's first file, while the old tree or its
// rollback copy is still on disk.
type stagedPackageInfo struct {
installed bool
identity wpcheck.Verification
lastSeen time.Time
}
type stagedPackageQueue struct {
mu sync.Mutex
limit int
files []stagedPackageFile
draining []stagedPackageFile
packages map[stagedPackageKey]stagedPackageInfo
health *queuehealth.Tracker
now func() time.Time
}
func newStagedPackageQueue(limit int) *stagedPackageQueue {
return &stagedPackageQueue{
limit: limit, packages: make(map[stagedPackageKey]stagedPackageInfo),
health: queuehealth.NewSharedCapacity(limit, time.Minute), now: time.Now,
}
}
func (q *stagedPackageQueue) snapshot(now time.Time) queuehealth.Status {
return q.health.Snapshot(now)
}
func (q *stagedPackageQueue) pendingCount() int {
q.mu.Lock()
defer q.mu.Unlock()
return len(q.files) + len(q.draining)
}
// note records a package the analyzer saw a file of and returns its info,
// resolving installed-ness on first sight and the identity as soon as a
// description carries one.
func (q *stagedPackageQueue) note(pkg wpStagedPackage, v wpcheck.Verification, now time.Time) (stagedPackageKey, stagedPackageInfo) {
key := stagedPackageKey{root: pkg.dir + "/" + pkg.unpacked}
// Use the identity captured with the header, before content scanning:
// by now WordPress may already have renamed or replaced this pathname.
installedRoot := pkg.wpRoot + "/wp-content/plugins/" + pkg.unpacked
movedPlugin := v.Staged && v.Kind == wpcheck.KindPlugin && v.Slug == pkg.unpacked && v.Root == installedRoot
if v.RootInfo != nil && v.RootInfo.IsDir() && (v.Root == key.root || movedPlugin) {
if st, ok := v.RootInfo.Sys().(*syscall.Stat_t); ok {
key.dev, key.ino = st.Dev, st.Ino
}
}
return key, q.record(pkg, key, v, now)
}
func (q *stagedPackageQueue) record(pkg wpStagedPackage, key stagedPackageKey, v wpcheck.Verification, now time.Time) stagedPackageInfo {
q.mu.Lock()
defer q.mu.Unlock()
info, ok := q.packages[key]
if !ok {
info.installed = wpPackageInstalled(pkg.wpRoot, pkg.unpacked)
}
if v.Kind != wpcheck.KindNone && (info.identity.Kind == wpcheck.KindNone || v.Version != "" ||
(v.Kind == wpcheck.KindTheme && info.identity.Version == "")) {
info.identity = v
info.identity.Rel, info.identity.Digest = "", ""
// Drains can discover the header at its installed location. Keep
// its identity tied to the original tree for other queued files.
info.identity.Root = key.root
}
info.lastSeen = now
q.packages[key] = info
return info
}
func (q *stagedPackageQueue) info(key stagedPackageKey) stagedPackageInfo {
q.mu.Lock()
defer q.mu.Unlock()
return q.packages[key]
}
func (q *stagedPackageQueue) push(f stagedPackageFile) bool {
q.mu.Lock()
defer q.mu.Unlock()
if len(q.files)+len(q.draining) >= q.limit {
q.health.Lose(q.now(), 1)
return false
}
f.ticket = q.health.BeginAt(f.queuedAt, q.now())
q.files = append(q.files, f)
return true
}
// take hands every queued file to the single drain loop, which requeues the
// ones still waiting. Reserve their capacity until requeue completes; new
// arrivals must not overfill the queue while the slice is detached.
func (q *stagedPackageQueue) take(now time.Time) []stagedPackageFile {
q.mu.Lock()
defer q.mu.Unlock()
files := q.files
q.files = nil
q.draining = files
for _, f := range files {
f.ticket.Start(now)
}
return files
}
func (q *stagedPackageQueue) requeue(files []stagedPackageFile, now time.Time) {
q.mu.Lock()
defer q.mu.Unlock()
keep := make(map[queuehealth.Ticket]bool, len(files))
for _, f := range files {
keep[f.ticket] = true
}
for _, f := range q.draining {
if keep[f.ticket] {
f.ticket.Requeue(now)
} else {
f.ticket.Finish(now)
}
}
q.files = append(files, q.files...)
q.draining = nil
}
// The monitor calls this after joining the analyzer and verifier workers,
// when no producer can append behind the final shutdown accounting.
func (q *stagedPackageQueue) discardPending(now time.Time) {
q.mu.Lock()
defer q.mu.Unlock()
for _, f := range q.files {
f.ticket.Reject(now)
}
q.files = nil
}
func (q *stagedPackageQueue) evictIdle(now time.Time) {
q.mu.Lock()
defer q.mu.Unlock()
for dir, info := range q.packages {
if now.Sub(info.lastSeen) > stagedPackageIdleTTL {
delete(q.packages, dir)
}
}
}
func (fm *FileMonitor) stagedPackages() *stagedPackageQueue {
fm.wpPendingInit.Do(func() {
if fm.wpPending == nil {
fm.wpPending = newStagedPackageQueue(stagedPackageQueueMax)
}
})
return fm.wpPending
}
// handleStagedPackageFile takes over a content-clean PHP file written inside
// a staged package. It returns false when path is not inside one, so the
// caller keeps its per-file warning for loose files under upgrade/.
func (fm *FileMonitor) handleStagedPackageFile(path string, v wpcheck.Verification, procInfo string) bool {
pkg := parseWPStagedPackage(path)
if pkg.dir == "" {
return false
}
q := fm.stagedPackages()
now := q.now()
key, info := q.note(pkg, v, now)
switch v.Verdict {
case wpcheck.VerdictVerified:
case wpcheck.VerdictMismatch, wpcheck.VerdictUnverifiable:
fm.alertStagedFileMismatch(path, pkg, v, procInfo)
case wpcheck.VerdictPending, wpcheck.VerdictNoVersion, wpcheck.VerdictReady:
file := stagedPackageFile{path: path, procInfo: procInfo, v: v, pkg: pkg, key: key, queuedAt: now}
if v.Kind == wpcheck.KindCore {
file.coreHeader, _ = statFileStamp(pkg.wpRoot + "/wp-includes/version.php")
}
if !q.push(file) {
fm.alertStagedPackage(pkg, info, "verification queue full", procInfo)
}
default:
fm.alertStagedPackage(pkg, info, stagedPackageReason(v), procInfo)
}
return true
}
func stagedPackageReason(v wpcheck.Verification) string {
switch {
case v.Verdict == wpcheck.VerdictUnavailable && v.Kind == wpcheck.KindTheme:
return "themes have no checksum source"
case v.Verdict == wpcheck.VerdictUnavailable:
return "not published on wordpress.org"
}
return "no plugin, theme or core header found"
}
func stagedPackageWhat(info stagedPackageInfo, pkg wpStagedPackage) string {
name := stagedPackageName(info.identity.Slug, pkg)
var what string
switch info.identity.Kind {
case wpcheck.KindCore:
what = "WordPress core"
case wpcheck.KindPlugin:
what = "plugin " + name
case wpcheck.KindTheme:
what = "theme " + name
default:
what = "package " + name
}
if info.identity.Version != "" {
what += " " + info.identity.Version
}
return what
}
func stagedPackageKind(kind wpcheck.PackageKind) string {
switch kind {
case wpcheck.KindCore:
return "core"
case wpcheck.KindPlugin:
return "plugin"
case wpcheck.KindTheme:
return "theme"
}
return "package"
}
// stagedPackageName is the package's own name: its slug once a header named
// it, otherwise the unpacked directory the archive created.
func stagedPackageName(slug string, pkg wpStagedPackage) string {
if slug == "" {
return pkg.unpacked
}
return slug
}
// alertStagedPackage raises the one directory-level finding for a package
// that cannot be verified file by file. Every file of the package lands on
// the same finding. The staging directory name is random per upload, so the
// alert identity is the site, package and reason: uploading the same
// unverifiable release again is a repeat, not a new condition. Incomplete
// headers cannot identify a release, so those warnings stay upload-specific.
func (fm *FileMonitor) alertStagedPackage(pkg wpStagedPackage, info stagedPackageInfo, reason, procInfo string) {
what := stagedPackageWhat(info, pkg)
state := "new install"
if info.installed {
state = "updates installed " + strings.TrimSuffix(what, " "+info.identity.Version)
}
var dedupKey string
if info.identity.Kind != wpcheck.KindNone && info.identity.Version != "" {
dedupKey = fmt.Sprintf("staged-package site=%q type=%s name=%q version=%q reason=%q",
pkg.wpRoot, stagedPackageKind(info.identity.Kind), stagedPackageName(info.identity.Slug, pkg),
info.identity.Version, reason)
}
fm.sendFileFinding(alert.Finding{
Severity: alert.Warning,
Check: "php_in_sensitive_dir_realtime",
Message: fmt.Sprintf("WordPress package staged, not verified against wordpress.org: %s", pkg.dir),
Details: fmt.Sprintf("%s; %s; %s. Content findings are reported separately for individual files.", what, state, reason),
DedupKey: dedupKey,
FilePath: pkg.dir,
ProcessInfo: procInfo,
})
}
// stagedFileInstalledPath is where WordPress puts a staged file once the
// package is moved into place.
func stagedFileInstalledPath(pkg wpStagedPackage, v wpcheck.Verification) string {
if v.Rel == "" {
return ""
}
switch v.Kind {
case wpcheck.KindPlugin:
if v.Slug == "" {
return ""
}
return pkg.wpRoot + "/wp-content/plugins/" + v.Slug + "/" + v.Rel
case wpcheck.KindCore:
return pkg.wpRoot + "/" + v.Rel
}
return ""
}
func pathPresent(path string) bool {
_, err := os.Lstat(path)
return err == nil
}
// alertStagedFileMismatch raises the per-file finding for a staged file that
// is not what the official package ships. If WordPress has already moved the
// tree, the finding names the installed path so it can be acted on there.
func (fm *FileMonitor) alertStagedFileMismatch(stagedPath string, pkg wpStagedPackage, v wpcheck.Verification, procInfo string) {
reportPath := stagedPath
moved := false
// A descriptor-linked result already located this event in plugins/.
// The staging pathname may now belong to a different unpacked tree.
if installed := stagedFileInstalledPath(pkg, v); installed != "" && pathPresent(installed) {
identifiedInstalled := v.Staged && v.Kind == wpcheck.KindPlugin && v.Root == pkg.wpRoot+"/wp-content/plugins/"+v.Slug
if identifiedInstalled || !pathPresent(stagedPath) {
reportPath, moved = installed, true
}
}
what := v.Slug + " " + v.Version
if v.Kind == wpcheck.KindCore {
what = "WordPress " + v.Version
}
message := fmt.Sprintf("Staged file does not match wordpress.org %s: %s", what, reportPath)
details := "Content rules found nothing. The file differs from the official package or is not part of it."
if v.Verdict == wpcheck.VerdictUnverifiable {
message = fmt.Sprintf("Staged file could not be verified against wordpress.org %s: %s", what, reportPath)
details = "Content rules found nothing. The file could not be hashed completely, so the official checksum could not be compared."
}
if moved {
details += fmt.Sprintf(" Staged at %s, since installed.", stagedPath)
}
// Identify the file by its place in the package, not the random staging
// directory, and by its digest so different bytes stay a new finding.
// An absent digest is not evidence of equal content across uploads.
file := strings.TrimPrefix(stagedPath, pkg.dir+"/")
var dedupKey string
if v.Digest != "" {
dedupKey = fmt.Sprintf("staged-file site=%q type=%s name=%q version=%q file=%q verdict=%s digest=%q",
pkg.wpRoot, stagedPackageKind(v.Kind), stagedPackageName(v.Slug, pkg), v.Version, file, v.Verdict, v.Digest)
}
fm.sendFileFinding(alert.Finding{
Severity: alert.Warning,
Check: "php_in_sensitive_dir_realtime",
Message: message,
Details: details,
DedupKey: dedupKey,
FilePath: reportPath,
ProcessInfo: procInfo,
})
}
// redescribeStaged retries identifying a file queued before its package
// header existed, from the staged path and then from the installed one.
// The digest taken at event time is kept: a rename does not change content,
// and a later write is a new event.
func (fm *FileMonitor) redescribeStaged(f stagedPackageFile) wpcheck.Verification {
// A later event may have identified this same tree before WordPress
// removed it. Prefer that header to an installed copy that may still be
// the old release; never borrow an identity from a sibling unpacked tree.
identity := fm.stagedPackages().info(f.key).identity
if f.key.ino != 0 && identity.Root == f.v.Root && identity.Version != "" {
identity.Rel, identity.Digest = f.v.Rel, f.v.Digest
// Another file's comparison result says nothing about this digest.
if identity.Kind == wpcheck.KindCore || identity.Kind == wpcheck.KindPlugin {
identity.Verdict = wpcheck.VerdictPending
}
return identity
}
unresolved := func(v wpcheck.Verification) bool {
return v.Verdict == wpcheck.VerdictNoVersion || v.Verdict == wpcheck.VerdictUnknown
}
var st unix.Stat_t
err := unix.Lstat(f.v.Root, &st)
if err == nil && (f.key.ino == 0 || st.Dev != f.key.dev || st.Ino != f.key.ino) {
return f.v // a replacement tree cannot identify an older event
}
v := fm.wpCache.Describe(f.path)
if !unresolved(v) {
if f.v.RootInfo == nil || v.RootInfo == nil || !os.SameFile(f.v.RootInfo, v.RootInfo) {
return f.v // the tree changed during header detection
}
v.Digest = f.v.Digest
return v
}
if !os.IsNotExist(err) {
return f.v
}
if f.v.Kind == wpcheck.KindCore {
return fm.redescribeCoreFromInstalled(f)
}
// Only a renamed plugin directory can link an installed header to this
// staging tree. A plugin copy install loses that link: a recent ctime
// (including chmod or child creation) is not evidence of origin.
if f.v.Kind != wpcheck.KindPlugin || f.v.RootInfo == nil {
return f.v
}
if installed := stagedFileInstalledPath(f.pkg, f.v); installed != "" {
root := f.pkg.wpRoot + "/wp-content/plugins/" + f.v.Slug
before, statErr := os.Lstat(root)
if statErr != nil || !before.IsDir() || !os.SameFile(f.v.RootInfo, before) {
return f.v
}
if iv := fm.wpCache.Describe(installed); !unresolved(iv) {
after, statErr := os.Lstat(root)
if statErr != nil || !after.IsDir() || !os.SameFile(before, after) ||
iv.Root != root || iv.Kind != wpcheck.KindPlugin || iv.Slug != f.v.Slug {
return f.v // the installed tree changed during header detection
}
v = iv
v.Rel = f.v.Rel
}
}
if unresolved(v) {
return f.v
}
v.Digest, v.Staged = f.v.Digest, true
return v
}
// redescribeCoreFromInstalled identifies a core file whose staged tree is gone
// by the installed version.php. A core update copies files into place, so no
// inode links the two trees; the installed header names this release only if
// it was replaced or rewritten since the file was queued. A refused or failed
// update leaves the old release, whose manifest would call every new file
// modified. Borrowing cannot make a modified file pass: its digest must still
// equal the official bytes of that path in the borrowed release.
func (fm *FileMonitor) redescribeCoreFromInstalled(f stagedPackageFile) wpcheck.Verification {
header := f.pkg.wpRoot + "/wp-includes/version.php"
now, ok := statFileStamp(header)
if !ok || now == f.coreHeader {
return f.v // keep waiting: a late event may still identify the tree
}
iv := fm.wpCache.Describe(header)
if iv.Kind != wpcheck.KindCore || iv.Version == "" ||
iv.Verdict == wpcheck.VerdictNoVersion || iv.Verdict == wpcheck.VerdictUnknown {
return f.v
}
if after, ok := statFileStamp(header); !ok || after != now {
return f.v // the header changed during detection
}
iv.Rel, iv.Digest, iv.Staged = f.v.Rel, f.v.Digest, true
return iv
}
// fileStamp identifies one version of a regular file: a replacement changes
// the inode, a rewrite or metadata change moves the change time.
type fileStamp struct {
dev, ino uint64
ctime int64
}
func statFileStamp(path string) (fileStamp, bool) {
var st unix.Stat_t
if err := unix.Lstat(path, &st); err != nil || st.Mode&unix.S_IFMT != unix.S_IFREG {
return fileStamp{}, false
}
return fileStamp{dev: uint64(st.Dev), ino: st.Ino, ctime: st.Ctim.Nano()}, true // #nosec G115 -- device numbers are non-negative
}
// drainStagedPackages resolves every queued file whose checksums have
// arrived and reports packages that ran out of time.
func (fm *FileMonitor) drainStagedPackages(now time.Time) {
q := fm.stagedPackages()
q.evictIdle(now)
files := q.take(time.Now())
if len(files) == 0 {
return
}
var keep []stagedPackageFile
for _, f := range files {
v := f.v
if v.Verdict == wpcheck.VerdictNoVersion || v.Version == "" {
v = fm.redescribeStaged(f)
q.record(f.pkg, f.key, v, now)
}
verdict := v.Verdict
if verdict == wpcheck.VerdictPending || verdict == wpcheck.VerdictReady {
verdict = fm.wpCache.Verify(v)
}
switch verdict {
case wpcheck.VerdictVerified:
case wpcheck.VerdictMismatch, wpcheck.VerdictUnverifiable:
v.Verdict = verdict
fm.alertStagedFileMismatch(f.path, f.pkg, v, f.procInfo)
case wpcheck.VerdictPending, wpcheck.VerdictNoVersion, wpcheck.VerdictReady:
if now.Sub(f.queuedAt) > stagedPackageTimeout {
if _, err := os.Lstat(f.key.root); v.Version == "" && os.IsNotExist(err) {
// Use the normal alert cooldown, not a permanent claim on a
// path that a later unidentified upload may reuse.
fm.alertStagedPackage(f.pkg, q.info(f.key),
"staged tree removed before its release could be identified; installation could not be confirmed", f.procInfo)
continue
}
fm.alertStagedPackage(f.pkg, q.info(f.key),
fmt.Sprintf("checksums not fetched within %s", stagedPackageTimeout), f.procInfo)
continue
}
f.v = v
keep = append(keep, f)
default:
fm.alertStagedPackage(f.pkg, q.info(f.key), stagedPackageReason(v), f.procInfo)
}
}
q.requeue(keep, time.Now())
}
// stagedPackageLoop drains the queue once a second until the monitor stops.
func (fm *FileMonitor) stagedPackageLoop() {
defer fm.wg.Done()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-fm.stopCh:
return
case now := <-ticker.C:
fm.drainStagedPackages(now)
}
}
}
package daemon
import (
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/broadcast"
)
func (d *Daemon) installFindingBus() {
bus := broadcast.NewBus(64)
d.registerQueueSource("events", bus)
d.findingBus = bus
alert.FindingBus = bus
}
package daemon
// closeFindingBus closes the finding broadcast bus at shutdown and leaves it
// installed in alert.FindingBus. Close turns Publish into a no-op, which is
// all shutdown needs; clearing the package-level interface as well raced
// the untracked control-socket and web UI goroutines that read it inside
// alert.Dispatch (a torn interface read is a nil-receiver panic).
func (d *Daemon) closeFindingBus() {
if d.findingBus != nil {
d.findingBus.Close()
}
}
package daemon
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
)
// isolationAlertInterval spaces email_av_isolation_unavailable findings: the
// condition lasts until an operator fixes the host, so once an hour is enough.
const isolationAlertInterval = time.Hour
// findingLimiter emits each check at most once per interval. Only a
// delivered finding uses up the allowance, so a full alert queue cannot hide
// a condition for a whole interval.
type findingLimiter struct {
mu sync.Mutex
last map[string]time.Time
}
func (l *findingLimiter) emit(ch chan<- alert.Finding, check string, severity alert.Severity, interval time.Duration, message string) {
l.mu.Lock()
defer l.mu.Unlock()
if last, ok := l.last[check]; ok && time.Since(last) < interval {
return
}
now := time.Now()
if alert.TryEnqueue(ch, alert.Finding{Severity: severity, Check: check, Message: message, Timestamp: now}) {
if l.last == nil {
l.last = make(map[string]time.Time)
}
l.last[check] = now
}
}
package daemon
import (
"encoding/json"
"fmt"
"strings"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/firewall"
csmlog "github.com/pidginhost/csm/internal/log"
)
// firewallActionBoundary is the durable-action surface an engine exposes once
// a lifecycle is attached. Keeping it an interface means the daemon compiles
// on platforms whose engine has no lifecycle at all.
type firewallActionBoundary interface {
DurableActionsEnabled() bool
RecoverActions() error
PendingActions() ([]firewall.FirewallAction, error)
ResolveAction(id, outcome, detail string) (firewall.FirewallAction, error)
}
// recoverFirewallActions settles outcomes left uncertain by a crash or a
// kernel that could not answer. Until every action is settled the engine
// refuses new mutations, so this runs at startup and on the maintenance tick.
func recoverFirewallActions(engine any) {
boundary, ok := engine.(firewallActionBoundary)
if !ok || boundary == nil || !boundary.DurableActionsEnabled() {
return
}
if err := boundary.RecoverActions(); err != nil {
csmlog.Error("firewall action recovery incomplete", "err", err)
}
}
func (c *ControlListener) firewallActions() (firewallActionBoundary, error) {
if c.d.fwActions == nil || !c.d.fwActions.DurableActionsEnabled() {
return nil, fmt.Errorf("durable firewall actions are not active")
}
return c.d.fwActions, nil
}
// handleFirewallActions reports the actions that still block firewall
// mutations, with the evidence an operator needs to decide what happened.
func (c *ControlListener) handleFirewallActions(json.RawMessage) (any, error) {
boundary, err := c.firewallActions()
if err != nil {
return nil, err
}
pending, err := boundary.PendingActions()
if err != nil {
return nil, err
}
if len(pending) == 0 {
return control.FirewallListResult{Lines: []string{"No firewall actions are waiting for recovery."}}, nil
}
lines := make([]string, 0, len(pending)*2)
for _, a := range pending {
lines = append(lines,
fmt.Sprintf("%s %s %s %s", a.UpdatedAt.Format("2006-01-02 15:04:05"), a.Phase, a.Request.Operation, a.Request.Target),
fmt.Sprintf(" id %s actor %s source %s", a.Request.ID, a.Request.Actor, a.Request.Source),
)
if a.Request.Reason != "" {
lines = append(lines, " reason "+a.Request.Reason)
}
if a.Detail != "" {
lines = append(lines, " detail "+a.Detail)
}
if a.Phase == "unknown" {
lines = append(lines, fmt.Sprintf(" resolve with: csm firewall actions resolve %s applied|rejected", a.Request.ID))
}
}
return control.FirewallListResult{Lines: lines}, nil
}
// handleFirewallActionResolve records what an operator established by hand.
// The engine still prefers kernel evidence when it can prove the outcome.
func (c *ControlListener) handleFirewallActionResolve(argsRaw json.RawMessage) (any, error) {
var args control.FirewallActionResolveArgs
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &args); err != nil {
return nil, fmt.Errorf("parsing args: %w", err)
}
}
if strings.TrimSpace(args.ID) == "" {
return nil, fmt.Errorf("resolve needs the action id")
}
var outcome string
switch args.Outcome {
case "applied":
outcome = "verified"
case "rejected":
outcome = "failed"
default:
return nil, fmt.Errorf("outcome must be applied or rejected, not %q", args.Outcome)
}
boundary, err := c.firewallActions()
if err != nil {
return nil, err
}
detail := "operator resolved via cli"
if note := strings.TrimSpace(args.Note); note != "" {
detail += ": " + note
}
resolved, err := boundary.ResolveAction(args.ID, outcome, detail)
if err != nil {
return nil, err
}
message := fmt.Sprintf("action %s recorded as %s", args.ID, resolved.Phase)
if resolved.Phase != outcome {
message = fmt.Sprintf("action %s was proven %s by the firewall itself; the operator decision was not used", args.ID, resolved.Phase)
}
return control.FirewallAckResult{Message: message}, nil
}
package daemon
import (
"fmt"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
)
// loadEffectiveFirewallFromDisk reads csm.yaml (and conf.d) again and returns
// the firewall block the daemon would build rules from at its next start.
// A SIGHUP reload deliberately leaves restart-required blocks such as
// firewall untouched in the live config, so the re-apply commands, whose
// whole purpose is to apply an edited firewall block under a deadman,
// must read the file rather than the live snapshot.
func loadEffectiveFirewallFromDisk(configFile, confDir string) (*firewall.FirewallConfig, error) {
if configFile == "" {
return nil, fmt.Errorf("config file path unknown")
}
cfg, err := config.LoadWithDir(configFile, confDir)
if err != nil {
return nil, fmt.Errorf("reading %s: %w", configFile, err)
}
for _, res := range config.Validate(cfg) {
if res.Level == "error" && len(res.Field) >= 8 && res.Field[:8] == "firewall" {
return nil, fmt.Errorf("%s: %s", res.Field, res.Message)
}
}
effective := config.EffectiveFirewallConfig(cfg)
if effective == nil || !effective.Enabled {
return nil, fmt.Errorf("firewall is disabled in %s; re-apply would drop the ruleset", configFile)
}
return effective, nil
}
// refreshFirewallFromDisk installs the on-disk firewall block into the
// running engine and returns the configuration it replaced, so a rollback
// can put it back. The previous configuration is returned even when the
// engine holds none (nil) so callers can always restore.
func (d *Daemon) refreshFirewallFromDisk() (previous *firewall.FirewallConfig, err error) {
if d.fwEngine == nil {
return nil, fmt.Errorf("firewall engine not running")
}
cfg := d.currentCfg()
effective, err := loadEffectiveFirewallFromDisk(cfg.ConfigFile, cfg.ConfigDir)
if err != nil {
return nil, err
}
previous = d.fwEngine.Config()
d.fwEngine.SetConfig(effective)
return previous, nil
}
package daemon
import (
"errors"
"fmt"
"os"
"path/filepath"
)
// firewallStateSnapshotPath is where the apply-confirmed window keeps the
// pre-apply copy of state.json, beside the nft ruleset snapshot.
func firewallStateSnapshotPath(rollbackFile string) string {
return rollbackFile + ".state.json"
}
func firewallStateFileFor(rollbackFile string) string {
return filepath.Join(filepath.Dir(rollbackFile), "state.json")
}
// snapshotFirewallState copies state.json next to the rollback ruleset. The
// deadman used to restore the kernel snapshot alone, so an address
// unblocked inside the window came back blocked in the kernel while
// state.json, the UI and `csm firewall status` still said it was free.
// A missing state.json is recorded as an empty snapshot so the restore
// removes whatever the window wrote.
func snapshotFirewallState(rollbackFile string) error {
data, err := os.ReadFile(firewallStateFileFor(rollbackFile)) // #nosec G304 -- CSM-owned state dir.
if err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("reading firewall state for rollback: %w", err)
}
// #nosec G306 G703 -- root-only state dir.
if err := os.WriteFile(firewallStateSnapshotPath(rollbackFile), data, 0o600); err != nil {
return fmt.Errorf("writing firewall state snapshot: %w", err)
}
return nil
}
// restoreFirewallStateSnapshot puts the snapshotted state.json back.
// Keep the snapshot until the window owner disarms rollback: a later
// configuration or marker failure must permit the same restore again.
// No snapshot (an older window) reports no state restore and is not an error.
func restoreFirewallStateSnapshot(rollbackFile string) (bool, error) {
snap := firewallStateSnapshotPath(rollbackFile)
data, err := os.ReadFile(snap) // #nosec G304 -- CSM-owned state dir.
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
return false, fmt.Errorf("reading firewall state snapshot: %w", err)
}
stateFile := firewallStateFileFor(rollbackFile)
if len(data) == 0 {
if err := os.Remove(stateFile); err != nil && !errors.Is(err, os.ErrNotExist) {
return false, fmt.Errorf("removing firewall state written inside the window: %w", err)
}
} else if err := os.WriteFile(stateFile, data, 0o600); err != nil { // #nosec G306 G703 -- root-only state dir.
return false, fmt.Errorf("restoring firewall state: %w", err)
}
return true, nil
}
package daemon
import (
"context"
"errors"
"fmt"
"net"
"os"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/admissionowner"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/mailranges"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/store"
"github.com/pidginhost/csm/internal/threatintel"
)
type firewallStartupOps struct {
newEngine func(*firewall.FirewallConfig, string) (*firewall.Engine, error)
apply func(*firewall.Engine) error
// setAside, when set, replaces (*firewall.Engine).StateSetAside.
setAside func(*firewall.Engine) string
// unproven, when set, replaces readUnprovenBlockActions; settled
// replaces readSettledBlockActions and removeRows
// (*firewall.Engine).RemoveBlockActionRows.
unproven func([]string) ([]string, error)
settled func([]string) ([]string, error)
removeRows func(*firewall.Engine, []string) (int, error)
// retime, when set, replaces (*firewall.Engine).RetimeBlocks.
retime func(*firewall.Engine) error
// leaseFloor, when set, replaces readFirewallLeaseFloor, and
// setLeaseFloor replaces (*firewall.Engine).SetLeaseFloor.
leaseFloor func() (time.Time, error)
setLeaseFloor func(*firewall.Engine, time.Time)
// setBootClock, when set, replaces (*firewall.Engine).SetBootClock.
setBootClock func(*firewall.Engine, func() (string, time.Duration, error))
leaseWarning func(error)
delays []time.Duration
}
func (d *Daemon) startFirewall() {
d.startFirewallUsing(firewallStartupOps{
newEngine: firewall.NewEngine,
apply: (*firewall.Engine).Apply,
delays: []time.Duration{time.Second, 2 * time.Second},
})
}
// Retry before publishing the engine or starting its consumers. Each attempt
// gets a fresh netlink transaction; a failed Apply preserves the kernel rules.
func retryFirewallStartup(stop <-chan struct{}, delays []time.Duration, attempt func() (*firewall.Engine, error)) (*firewall.Engine, error) {
for i := 0; ; i++ {
select {
case <-stop:
return nil, context.Canceled
default:
}
engine, err := attempt()
if err == nil {
return engine, nil
}
// A fresh engine cannot distinguish a missing file from a rename
// whose directory sync failed in the previous attempt.
if errors.Is(err, firewall.ErrStateNotKeptAside) || i == len(delays) {
return nil, err
}
timer := time.NewTimer(delays[i])
select {
case <-stop:
timer.Stop()
return nil, context.Canceled
case <-timer.C:
}
}
}
func (d *Daemon) startFirewallUsing(ops firewallStartupOps) {
if err := checks.InitAutoBlockQueueHealth(d.cfg.StatePath); err != nil {
csmlog.Error("auto-block retry state unreadable", "err", err)
}
effectiveFirewall := config.EffectiveFirewallConfig(d.cfg)
if effectiveFirewall == nil || !effectiveFirewall.Enabled {
return
}
engine, err := retryFirewallStartup(d.stopCh, ops.delays, func() (*firewall.Engine, error) {
return d.prepareFirewall(effectiveFirewall, ops)
})
if aside := d.firewallStateSetAside(); aside != "" {
findings := []alert.Finding{firewallStateSetAsideFinding(aside)}
// Startup health owns its delivery, like queue health: the finding
// queue and routine suppression must not hide missing firewall state.
if d.store != nil {
d.store.AppendHistory(findings)
}
if dispatchErr := alert.Dispatch(d.currentCfg(), findings); dispatchErr != nil {
csmlog.Error("firewall state notification failed", "err", dispatchErr)
}
}
if err != nil {
d.fwStartupError = err.Error()
csmlog.Error("firewall remains unmanaged after startup attempts", "err", err)
return
}
d.fwStartupError = ""
// Apply does not consult the verdict callback. Install the shutdown
// context only after a successful firewall setup so a failed init
// does not leave behind a stopCh waiter.
verdictCtx, cancelVerdict := context.WithCancel(context.Background())
go func() {
<-d.stopCh
cancelVerdict()
}()
engine.SetShutdownContext(verdictCtx)
d.setFirewallEngine(engine)
// A rollback restores the rows of its window's start; they are gated
// again as they were here, and the restored blocks are retimed.
retime := ops.retime
if retime == nil {
retime = (*firewall.Engine).RetimeBlocks
}
setFirewallRestoreHooks(func() { gateFirewallRows(engine, ops) }, func() {
if err := retime(engine); err != nil {
csmlog.Warn("firewall blocks keep the timers of the restored ruleset", "err", err)
}
})
// Set firewall engine for auto-blocking
checks.SetIPBlocker(engine)
// Prune auto-response subnet blocks that now intersect the DoS-exempt set.
// The mail-provider cache is loaded (initMailRanges ran before startFirewall)
// and Apply has completed, so the exempt set is current.
checks.PruneExemptAutoSubnets(d.cfg, engine)
// Wire the incident firewall hand-off through the ApplyBlock chokepoint
// so the correlator distinguishes live mutation from dry-run and no-op
// outcomes AND spray blocks leave the standard evidence trail.
SetIncidentSprayBlocker(d.applyIncidentBlock)
fwState, _ := firewall.LoadState(d.cfg.StatePath)
csmlog.Info("firewall active",
"blocked_ips", len(fwState.Blocked),
"allowed_ips", len(fwState.Allowed),
)
// Start Dynamic DNS resolver if configured. The same resolver
// loop also services hostnames listed under infra_ips so they get
// DNS-refreshed into the engine's infra-block guard; otherwise the
// hostname entries would only protect operators whose IPs never
// move, which defeats the point of listing them by name.
infraHosts := infraHostnames(effectiveFirewall.InfraIPs)
dynHosts := append([]string{}, effectiveFirewall.DynDNSHosts...)
for _, h := range infraHosts {
if !containsString(dynHosts, h) {
dynHosts = append(dynHosts, h)
}
}
if len(dynHosts) > 0 {
resolver := firewall.NewDynDNSResolver(dynHosts, engine)
resolver.SetInfraEngine(engine)
for _, h := range infraHosts {
resolver.RegisterInfraHost(h)
}
resolver.SetFindingSink(func(host string) {
if !alert.TryEnqueue(d.alertCh, dynDNSUnresolvableFinding(host)) {
atomic.AddInt64(&d.droppedAlerts, 1)
fmt.Fprintf(os.Stderr, "[%s] alert channel full, dropping dyndns guard finding: %s\n", ts(), host)
}
})
d.wg.Add(1)
obs.Go("dyndns-resolver", func() {
defer d.wg.Done()
resolver.Run(d.stopCh)
})
csmlog.Info("DynDNS resolver active", "hosts", len(dynHosts), "infra_hosts", len(infraHosts))
}
// Start Cloudflare IP whitelist refresh if configured
if d.cfg.Cloudflare.Enabled {
d.wg.Add(1)
obs.Go("cloudflare-refresh", d.cloudflareRefreshLoop)
csmlog.Info("cloudflare IP whitelist enabled", "refresh_hours", d.cfg.Cloudflare.RefreshHours)
}
}
func (d *Daemon) prepareFirewall(effectiveFirewall *firewall.FirewallConfig, ops firewallStartupOps) (*firewall.Engine, error) {
engine, err := ops.newEngine(effectiveFirewall, d.cfg.StatePath)
if err != nil {
return nil, fmt.Errorf("initializing firewall: %w", err)
}
// Wire dry-run + verdict callbacks BEFORE Apply() and before the
// engine is exposed via d.fwEngine / checks.SetIPBlocker. The
// auto_response.dry_run safety default is "on": if any code path
// reaches engine.BlockIP while these callbacks are still nil, the
// engine treats dry-run as off and the block lands live, defeating
// the operator's stated intent. Wiring before exposure removes the
// boot-time race window entirely.
engine.SetDryRunRecorder(recordLegacyDryRunBlock)
engine.SetDryRunEnabledFunc(d.autoResponseDryRunEnabled)
engine.SetVerdictAsker(d.askVerdictCallback)
// The auto-block path skips published-crawler IPs so a high-volume bot is
// never re-added to blocked_ips behind the operator allowlist. Built-in and
// operator verified_bots ranges both flow through this lookup.
engine.SetSoftAllowChecker(func(ip string) bool {
parsed := net.ParseIP(ip)
return parsed != nil && threatintel.IPInAnyVerifiedBotRange(parsed)
})
// An operator unblock, allow or flush ends the mail spray history of the
// released addresses, so a marker or kept evidence does not block them
// again on their next failed login. A self-service unban ends only the
// re-block.
engine.SetOperatorReleaseHook(d.firewallReleased)
// Push the mail-provider ranges loaded by initMailRanges() into the engine
// before Apply() so the dos_exempt_nets interval sets are populated in the
// first nftables transaction. initMailRanges() runs before startFirewall()
// so ProviderNets() always returns the cached or embedded snapshot here.
engine.SetDOSExemptProviderNets(mailranges.ProviderNets())
// Apply is itself a mutation and refuses pending actions. Recover first,
// retaining the boundary for operator resolution even if Apply fails.
d.fwActions, _ = any(engine).(firewallActionBoundary)
recoverFirewallActions(d.fwActions)
seedFirewallLease(engine, ops)
gateFirewallRows(engine, ops)
err = ops.apply(engine)
// Report a kept file even when this attempt fails: the next attempt
// finds no file and starts empty.
setAside := ops.setAside
if setAside == nil {
setAside = (*firewall.Engine).StateSetAside
}
if aside := setAside(engine); aside != "" {
d.fwStateAsideMu.Lock()
d.fwStateAside = aside
d.fwStateAsideMu.Unlock()
}
if err != nil {
return nil, fmt.Errorf("applying firewall: %w", err)
}
return engine, nil
}
func (d *Daemon) firewallStateSetAside() string {
d.fwStateAsideMu.RLock()
defer d.fwStateAsideMu.RUnlock()
return d.fwStateAside
}
// gateFirewallRows holds back, before Apply restores them, the rows whose
// attempts the ledger has not proven applied. Without a readable ledger
// nothing is proven: the rows are restored as before, the engine never
// evicts them, and status reports why until the next start.
func gateFirewallRows(engine *firewall.Engine, ops firewallStartupOps) {
ids, stateErr := engine.BlockActionIDsForStartup()
read := ops.unproven
if read == nil {
read = readUnprovenBlockActions
}
unproven, err := read(ids)
if err != nil {
engine.SetBlockActionsUnproven(err)
csmlog.Warn("firewall restores automatic blocks without a readable ledger", "err", err)
return
}
if stateErr != nil || len(unproven) == 0 {
return // Apply preserves an unreadable state file.
}
if _, markErr := engine.MarkBlockActionsUnverified(unproven); markErr != nil {
csmlog.Warn("firewall holds unproven blocks back in memory only", "err", markErr)
}
// Settled attempts (failed, or retired after their expiry) applied
// nothing that can remain: their rows go. The removal's write also
// carries the marks a failed write above kept in memory.
readSettled, remove := ops.settled, ops.removeRows
if readSettled == nil {
readSettled = readSettledBlockActions
}
if remove == nil {
remove = (*firewall.Engine).RemoveBlockActionRows
}
settled, err := readSettled(unproven)
if err != nil {
csmlog.Warn("firewall keeps settled blocks held back until the ledger can be read", "err", err)
return
}
if len(settled) == 0 {
return
}
if _, err := remove(engine, settled); err != nil {
csmlog.Warn("firewall keeps settled blocks held back until their rows can be removed", "err", err)
}
}
// seedFirewallLease gives the engine, before Apply restores timed blocks,
// the lower bound supplied by the saved admission clock. It composes with
// the engine's own checkpoint; an unreadable ledger adds no floor.
func seedFirewallLease(engine *firewall.Engine, ops firewallStartupOps) {
// The engine's own checkpoint resumes the lease within one boot; the
// ledger's saved clock bounds it on a new boot.
setBoot := ops.setBootClock
if setBoot == nil {
setBoot = (*firewall.Engine).SetBootClock
}
setBoot(engine, readBootClock)
read, set := ops.leaseFloor, ops.setLeaseFloor
if read == nil {
read = readFirewallLeaseFloor
}
if set == nil {
set = (*firewall.Engine).SetLeaseFloor
}
floor, err := read()
if err != nil {
if ops.leaseWarning != nil {
ops.leaseWarning(err)
} else {
csmlog.Warn("firewall lease clock coverage degraded", "err", err)
}
}
if floor.IsZero() {
return
}
set(engine, floor)
}
func readBootClock() (string, time.Duration, error) {
r, err := admissionowner.ReadClock()
return r.BootID, r.SinceBoot, err
}
func readFirewallLeaseFloor() (time.Time, error) {
db := store.Global()
if db == nil {
return time.Time{}, errors.New("state database unavailable")
}
clock, err := store.ReadAdmissionClock(db)
if err != nil {
return time.Time{}, err
}
reading, err := admissionowner.ReadClock()
if err != nil {
return time.Time{}, err
}
return provenFirewallLease(clock, reading)
}
func provenFirewallLease(clock admission.Clock, reading admission.ClockReading) (time.Time, error) {
if !clock.Initialized() {
return time.Time{}, errors.New("no saved admission clock; no ledger lease floor")
}
_, tick, err := clock.Advance(reading)
if err != nil {
return time.Time{}, err
}
floor := clock.Proven(reading)
if tick.Degraded {
return floor, errors.New("host clock disagrees with the saved checkpoint; using its proven floor")
}
return floor, nil
}
func readSettledBlockActions(ids []string) ([]string, error) {
db := store.Global()
if db == nil {
return nil, errors.New("state database unavailable")
}
return store.ReadSettledBlockActions(db, ids)
}
func readUnprovenBlockActions(ids []string) ([]string, error) {
db := store.Global()
if db == nil {
return nil, errors.New("state database unavailable")
}
return store.ReadUnprovenBlockActions(db, ids)
}
// firewallStateSetAsideFinding reports a state file the engine could not
// read at startup and kept aside before anything could replace it.
func firewallStateSetAsideFinding(path string) alert.Finding {
return alert.Finding{
Check: "firewall_state_unreadable",
Severity: alert.Critical,
Message: "firewall state file could not be read at startup; kept at " + path,
Details: "The blocked and allowed addresses in the kept file could not be restored. Review the kept file and add back the entries that must stay.",
Timestamp: time.Now(),
}
}
package daemon
import (
"errors"
"time"
"github.com/pidginhost/csm/internal/eximlock"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/mailfwd/adapter"
"github.com/pidginhost/csm/internal/mailfwd/guard"
"github.com/pidginhost/csm/internal/mailfwd/quarantine"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/store"
)
// forwardGuardBadIPScore is the reputation score at/above which a sender IP is
// treated as bad for the forward-guard's bad_sender_ip signal.
const forwardGuardBadIPScore = 50
// forwardGuardRefreshInterval is how often the bad-IP lookup file is refreshed
// from the reputation DB. exim reads the lsearch file per lookup, so this only
// rewrites a file -- no exim rebuild or reload.
const forwardGuardRefreshInterval = 15 * time.Minute
// newForwardGuard builds the adapter that runs the forward-guard worker;
// tests replace it.
var newForwardGuard = adapter.NewEximServiceAdapter
// forwardGuardReconciler builds the reconciler for the current host. The guard
// is only active on cPanel/exim; elsewhere Reconcile/RefreshBadIPs are no-ops.
// In observe mode on cPanel, Reconcile only cancels pending worker requests.
func (d *Daemon) forwardGuardReconciler() guard.Reconciler {
cpanel, observe := platform.Detect().IsCPanel(), d.currentCfg().ObserveMode()
return guard.Reconciler{
Guard: newForwardGuard(),
Active: cpanel && !observe,
BadIPs: d.forwardGuardBadIPs,
Observe: cpanel && observe,
}
}
// forwardGuardBadIPs returns sender IPs the reputation DB scores as bad. Empty
// when the store is unavailable -- the guard then simply holds nothing on the
// bad-IP signal (the null-sender signal is unaffected).
func (d *Daemon) forwardGuardBadIPs() []string {
db := store.Global()
if db == nil {
return nil
}
var ips []string
for ip, e := range db.AllReputation() {
if e.Score >= forwardGuardBadIPScore {
ips = append(ips, ip)
}
}
return ips
}
// reconcileForwardGuard installs or removes the exim forward-guard to match the
// current config. Errors are logged, never fatal: a guard failure must not take
// the daemon down or block mail (fail-open). A change refused because another
// CSM transaction, such as an antispam one, holds the Exim lock is logged as
// deferred; it and any failure are retried on the next refresh.
func (d *Daemon) reconcileForwardGuard() {
d.forwardGuardRetry.Store(true)
// No caller waits for an in-flight worker. Reloads reach here through the
// loop, one at a time; any other overlapping call leaves its request for
// the next refresh. Read config only after taking the lock.
if !d.forwardGuardMu.TryLock() {
csmlog.Info("forward-guard reconcile deferred to the next refresh: another reconcile is running",
"refresh_interval", forwardGuardRefreshInterval)
return
}
defer d.forwardGuardMu.Unlock()
// Consume pending work before starting so success cannot erase a reload
// request that arrives while this attempt is running.
d.forwardGuardRetry.Store(false)
fg := d.currentCfg().EmailProtection.ForwardGuard
if err := d.forwardGuardReconciler().Reconcile(fg); err != nil {
d.forwardGuardRetry.Store(true)
if errors.Is(err, eximlock.ErrBusy) {
csmlog.Info("forward-guard reconcile deferred to the next refresh: another CSM transaction holds the Exim lock",
"refresh_interval", forwardGuardRefreshInterval)
return
}
csmlog.Error("forward-guard reconcile failed", "err", err)
return
}
}
// forwardQuarantineDir is a var so tests can prune a temporary Maildir.
var forwardQuarantineDir = adapter.QuarantineDir
// wakeForwardGuard asks the forward-guard loop to reconcile with the latest
// config. It never blocks: a reload must not wait for a worker that can
// rebuild Exim for minutes. Wakes that arrive during a reconcile become one
// more reconcile after it.
func (d *Daemon) wakeForwardGuard() {
select {
case d.forwardGuardWake <- struct{}{}:
default:
}
}
// forwardGuardRefresher reconciles the guard when a reload wakes it, and
// periodically refreshes the bad-IP lookup file while the guard is enforcing
// and ages held copies out of the quarantine.
func (d *Daemon) forwardGuardRefresher() {
defer d.wg.Done()
ticker := time.NewTicker(forwardGuardRefreshInterval)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-d.forwardGuardWake:
// A queued reload can win the select after shutdown starts.
// Do not launch another potentially slow Exim rebuild then.
select {
case <-d.stopCh:
return
default:
}
d.reconcileForwardGuard()
case <-ticker.C:
d.refreshForwardGuard()
}
}
}
// refreshForwardGuard retries a failed reconcile, refreshes the bad-IP
// lookup and ages out held copies.
func (d *Daemon) refreshForwardGuard() {
if d.forwardGuardRetry.Load() {
d.reconcileForwardGuard()
}
fg := d.currentCfg().EmailProtection.ForwardGuard
if err := d.forwardGuardReconciler().RefreshBadIPs(fg); err != nil {
csmlog.Error("forward-guard bad-IP refresh failed", "err", err)
}
d.pruneForwardQuarantine()
}
// pruneForwardQuarantine holds again the copies a release or delete claimed
// and never finished, as when the daemon died mid-release, then deletes held
// forward copies older than quarantine_retention_days. Copies held while the
// guard was on must still age out after it is switched off, so only the
// retention value gates the prune; 0 keeps everything.
func (d *Daemon) pruneForwardQuarantine() {
q := quarantine.New(forwardQuarantineDir)
restored, err := q.RestoreStaleClaims(time.Now())
if err != nil {
csmlog.Error("forward-guard quarantine restore failed", "err", err)
}
if restored > 0 {
csmlog.Warn("forward-guard held copies restored after an interrupted release or delete; a restored release may already have been delivered", "restored", restored)
}
days := d.currentCfg().EmailProtection.ForwardGuard.QuarantineRetentionDays
if days <= 0 {
return
}
removed, err := q.PruneOlderThan(time.Duration(days) * 24 * time.Hour)
if err != nil {
csmlog.Error("forward-guard quarantine prune failed", "err", err)
return
}
if removed > 0 {
csmlog.Info("forward-guard quarantine pruned", "removed", removed, "retention_days", days)
}
}
package daemon
import (
"fmt"
"os"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
)
// parseValiasFileForFindings parses a valiases file and returns findings.
// Used by both the realtime watcher and tests.
func parseValiasFileForFindings(path, domain string, localDomains map[string]bool, knownForwarders []string) []alert.Finding {
return parseValiasFileForFindingsFiltered(path, domain, localDomains, knownForwarders, true)
}
func parseValiasFileForFindingsFiltered(path, domain string, localDomains map[string]bool, knownForwarders []string, includeExternal bool) []alert.Finding {
// #nosec G304 -- path from cPanel valiases directory walk; operator-scoped.
f, err := os.Open(path)
if err != nil {
return nil
}
defer f.Close()
// A read error mid-file still reports the entries parsed before it.
entries, _ := checks.ParseValiasEntries(f, domain)
var findings []alert.Finding
for _, e := range entries {
localPart, mailDomain, d := e.LocalPart, e.Domain, e.Dest
if checks.IsKnownForwarder(localPart, mailDomain, d, knownForwarders) {
continue
}
if checks.IsPipeForwarder(d) {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "email_pipe_forwarder",
Message: fmt.Sprintf("Pipe forwarder detected: %s@%s -> %s", localPart, mailDomain, d),
Details: fmt.Sprintf("Domain: %s\nLocal part: %s\nDestination: %s\nFile: %s", mailDomain, localPart, d, path),
})
continue
}
if d == "/dev/null" {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "email_suspicious_forwarder",
Message: fmt.Sprintf("Mail blackhole: %s@%s -> /dev/null", localPart, mailDomain),
Details: fmt.Sprintf("Domain: %s\nLocal part: %s\nDestination: /dev/null\nFile: %s", mailDomain, localPart, path),
})
continue
}
if includeExternal && checks.IsExternalDest(d, localDomains) {
msg := fmt.Sprintf("External forwarder: %s@%s -> %s", localPart, mailDomain, d)
if localPart == "*" {
msg = fmt.Sprintf("Wildcard catch-all to external: *@%s -> %s", mailDomain, d)
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "email_suspicious_forwarder",
Message: msg,
Details: fmt.Sprintf("Domain: %s\nLocal part: %s\nDestination: %s\nFile: %s", mailDomain, localPart, d, path),
})
}
}
return findings
}
//go:build linux
package daemon
import (
"crypto/sha256"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"unsafe"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/store"
)
// valiasesDir is the inotify watch root. It is a var (not const) so
// tests can redirect it under t.TempDir() without touching the real
// /etc/valiases directory; mirrors cronSpoolWatchDir in fanotify.go.
var valiasesDir = "/etc/valiases"
// ForwarderWatcher watches /etc/valiases/ for changes using inotify.
type ForwarderWatcher struct {
alertCh chan<- alert.Finding
knownForwarders []string
inotifyFd int
queueHealthOnce sync.Once
kernelQueue *notificationQueue
}
// NewForwarderWatcher creates a watcher for the valiases directory.
func NewForwarderWatcher(alertCh chan<- alert.Finding, knownForwarders []string) (*ForwarderWatcher, error) {
fd, err := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK)
if err != nil {
return nil, fmt.Errorf("inotify_init1: %w", err)
}
_, err = unix.InotifyAddWatch(fd, valiasesDir, unix.IN_CLOSE_WRITE)
if err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("inotify_add_watch(%s): %w", valiasesDir, err)
}
return &ForwarderWatcher{
alertCh: alertCh,
knownForwarders: knownForwarders,
inotifyFd: fd,
}, nil
}
// Run starts the watch loop. Blocks until stopCh is closed.
func (fw *ForwarderWatcher) Run(stopCh <-chan struct{}) {
fw.initQueueHealth()
defer func() { _ = fw.kernelQueue.close() }()
buf := make([]byte, 4096)
// Use a polling approach since inotify fd + stopCh coordination
// requires either epoll or periodic polling. Keep it simple.
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
for {
select {
case <-stopCh:
return
case <-ticker.C:
fw.readEvents(buf)
}
}
}
func (fw *ForwarderWatcher) readEvents(buf []byte) {
fw.initQueueHealth()
for {
n, err := fw.kernelQueue.read(buf, fw.processEvents)
if err != nil || n <= 0 {
return // EAGAIN or error - no more events
}
}
}
func (fw *ForwarderWatcher) processEvents(buf []byte) {
n := len(buf)
offset := 0
for offset < n {
if offset+unix.SizeofInotifyEvent > n {
break
}
// #nosec G103 -- inotify returns a packed binary stream;
// reinterpretation is required and bounded by the SizeofInotifyEvent check above.
event := (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset]))
if event.Mask&unix.IN_Q_OVERFLOW != 0 {
fw.kernelQueue.losses.Lose(time.Now(), 1)
}
nameLen := int(event.Len)
if nameLen > 0 && offset+unix.SizeofInotifyEvent+nameLen <= n {
nameBytes := buf[offset+unix.SizeofInotifyEvent : offset+unix.SizeofInotifyEvent+nameLen]
// Trim null bytes
name := strings.TrimRight(string(nameBytes), "\x00")
if name != "" && !strings.HasPrefix(name, ".") {
fw.handleFileChange(name)
}
}
offset += unix.SizeofInotifyEvent + nameLen
}
}
func (fw *ForwarderWatcher) handleFileChange(domain string) {
path := filepath.Join(valiasesDir, domain)
// 2026-04-27: suppress alerts on the first observation of a valiases file.
// Account transfers via WHM rsync write the entire file fresh; alerting
// on every pre-existing forwarder buries operators in noise. Hash the
// file: baseline external destinations on first sight, alert on external
// destinations only when the hash changes, and keep pipe/dev-null checks
// active because they are dangerous even on a first observation. Mirrors
// CheckForwarders' behaviour in internal/checks/forwarder.go.
db := store.Global()
includeExternal := true
if db != nil {
// #nosec G304 -- path is filepath.Join(valiasesDir, domain) where the
// inotify event already restricted domain to a single path component.
data, err := os.ReadFile(path)
if err == nil {
currentHash := fmt.Sprintf("%x", sha256.Sum256(data))
oldHash, found := db.GetForwarderHash("valiases:" + domain)
_ = db.SetForwarderHash("valiases:"+domain, currentHash)
switch {
case !found:
includeExternal = false
case oldHash == currentHash:
return
default:
includeExternal = true
}
}
}
// Load local domains for external detection
localDomains := loadLocalDomainsForWatcher()
findings := parseValiasFileForFindingsFiltered(path, domain, localDomains, fw.knownForwarders, includeExternal)
for _, f := range findings {
f.Timestamp = time.Now()
f.Details += "\n(detected in realtime via inotify)"
if !alert.TryEnqueue(fw.alertCh, f) {
fmt.Fprintf(os.Stderr, "[%s] Warning: alert channel full, dropping forwarder finding for %s\n",
time.Now().Format("2006-01-02 15:04:05"), domain)
}
}
}
// loadLocalDomainsForWatcher reads the host's local domain files.
func loadLocalDomainsForWatcher() map[string]bool {
domains, _ := readLocalDomains()
return domains
}
package daemon
import (
"container/list"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/eximlog"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/store"
)
// eximFrozenDedupTTL is how long an inactive queue ID remains tracked. Queue
// runs refresh the timestamp for messages that are still frozen, so one stuck
// message stays suppressed for its entire frozen lifetime. The expiry only
// bounds stale state when the daemon misses the corresponding unfreeze event,
// for example when a frozen message is removed from the queue, which exim logs
// without an unfreeze line.
const eximFrozenDedupTTL = 24 * time.Hour
// Sweep stale entries periodically instead of walking the entire table for
// every new frozen message. A burst of distinct frozen messages must remain
// O(n), not degrade to O(n^2) scans; eviction at capacity is constant time for
// the same reason.
const eximFrozenDedupPruneInterval = time.Hour
// Bound attacker-influenced queue state. Every tracked ID is a real message
// exim froze (the parser reads only exim's own queue-ID field), so filling
// the table takes that many frozen messages inside one TTL, each of which
// raised its own finding. Reaching the cap evicts the ID no queue run has
// refreshed for the longest, so the cost of overflow is a duplicate finding
// for an old message, never a missed report for a new one.
const eximFrozenDedupMaxEntries = 10_000
// eximFrozenDedupPersistInterval is how often changed dedup state is written
// to the state store. A clean shutdown writes it once more after the log
// watchers stop, so only a crash can lose the IDs first seen inside the last
// interval, and each of those is reported once more after the restart. A var
// so tests can drive the periodic writer without waiting a minute.
var eximFrozenDedupPersistInterval = time.Minute
// eximFrozenDedup keeps the last-observed time per frozen message ID. Exim
// re-logs "Message is frozen" on every queue run for as long as the message
// stays queued, so without this one stuck bounce raises a finding every few
// minutes for days. The table is persisted to the state store: the first
// queue run after a restart re-logs every still-frozen message, and only a
// message the daemon never saw frozen may alert then. Seeding the table from
// the queue at startup instead would hide a message that froze while the
// daemon was down.
var eximFrozenDedup = struct {
mu sync.Mutex
seen map[string]*list.Element
// order holds one *eximFrozenSighting per tracked ID, least recently
// seen first, so eviction at capacity drops the stalest ID without a scan.
order *list.List
nextPrune time.Time
// version counts changes to the table; persisted is the version the state
// store last accepted. They differ while there is something to save.
version uint64
persisted uint64
}{seen: make(map[string]*list.Element), order: list.New()}
type eximFrozenSighting struct {
id string
lastSeen time.Time
}
func resetEximFrozenDedup() {
eximFrozenDedup.mu.Lock()
defer eximFrozenDedup.mu.Unlock()
eximFrozenDedup.seen = make(map[string]*list.Element)
eximFrozenDedup.order = list.New()
eximFrozenDedup.nextPrune = time.Time{}
eximFrozenDedup.version = 0
eximFrozenDedup.persisted = 0
}
type eximFrozenEvent uint8
const (
eximFrozenEventNone eximFrozenEvent = iota
eximFrozenEventFreeze
eximFrozenEventUnfreeze
)
// eximFrozenShouldAlert reports whether a mainlog line is a freeze event that
// deserves a finding. Repeated queue-run notices for the same ID are
// suppressed, while an unfreeze event clears the ID so a later re-freeze is a
// new finding. Freeze-shaped lines with no parseable queue ID fail open.
func eximFrozenShouldAlert(line string, now time.Time) bool {
id, event := parseEximFrozenEvent(line)
if event == eximFrozenEventNone {
return false
}
if event == eximFrozenEventUnfreeze {
if id != "" {
forgetEximFrozenID(id)
}
return false
}
if id == "" {
return true
}
eximFrozenDedup.mu.Lock()
defer eximFrozenDedup.mu.Unlock()
if eximFrozenDedup.nextPrune.IsZero() || !now.Before(eximFrozenDedup.nextPrune) {
cutoff := now.Add(-eximFrozenDedupTTL)
for el := eximFrozenDedup.order.Front(); el != nil; {
next := el.Next()
if sighting := el.Value.(*eximFrozenSighting); !sighting.lastSeen.After(cutoff) {
removeEximFrozenSightingLocked(el)
}
el = next
}
eximFrozenDedup.nextPrune = now.Add(eximFrozenDedupPruneInterval)
}
eximFrozenDedup.version++
if el, ok := eximFrozenDedup.seen[id]; ok {
sighting := el.Value.(*eximFrozenSighting)
suppressed := now.Before(sighting.lastSeen.Add(eximFrozenDedupTTL))
sighting.lastSeen = now
eximFrozenDedup.order.MoveToBack(el)
return !suppressed
}
if eximFrozenDedup.order.Len() >= eximFrozenDedupMaxEntries {
removeEximFrozenSightingLocked(eximFrozenDedup.order.Front())
}
eximFrozenDedup.seen[id] = eximFrozenDedup.order.PushBack(&eximFrozenSighting{id: id, lastSeen: now})
return true
}
func removeEximFrozenSightingLocked(el *list.Element) {
delete(eximFrozenDedup.seen, el.Value.(*eximFrozenSighting).id)
eximFrozenDedup.order.Remove(el)
}
func forgetEximFrozenID(id string) {
eximFrozenDedup.mu.Lock()
defer eximFrozenDedup.mu.Unlock()
if el, ok := eximFrozenDedup.seen[id]; ok {
removeEximFrozenSightingLocked(el)
eximFrozenDedup.version++
}
}
// loadEximFrozenDedup replaces the table with the persisted one. It runs
// before the log watchers start, so the replacement discards nothing they
// recorded. Entries past the TTL, entries that are not exim queue IDs, and
// any excess over the cap (oldest first) are dropped, so the restored table
// obeys the same bounds as one built from the log. A last-seen time ahead of
// now counts as now, so a clock step back cannot stretch suppression past the
// TTL.
func loadEximFrozenDedup(db *store.DB, now time.Time) error {
persisted, err := db.LoadEximFrozenSeen()
if err != nil {
return err
}
cutoff := now.Add(-eximFrozenDedupTTL)
restored := make([]*eximFrozenSighting, 0, len(persisted))
corrected := false
for id, lastSeen := range persisted {
if !eximlog.IsMessageID(id) || !lastSeen.After(cutoff) {
corrected = true
continue
}
if lastSeen.After(now) {
lastSeen = now
corrected = true
}
restored = append(restored, &eximFrozenSighting{id: id, lastSeen: lastSeen})
}
sort.Slice(restored, func(i, j int) bool {
return restored[i].lastSeen.Before(restored[j].lastSeen)
})
if excess := len(restored) - eximFrozenDedupMaxEntries; excess > 0 {
restored = restored[excess:]
corrected = true
}
seen := make(map[string]*list.Element, len(restored))
order := list.New()
for _, sighting := range restored {
seen[sighting.id] = order.PushBack(sighting)
}
eximFrozenDedup.mu.Lock()
eximFrozenDedup.seen = seen
eximFrozenDedup.order = order
if corrected {
// Save cleanup even if no queue run follows. Otherwise each restart
// clamps the same future timestamp again and extends suppression.
eximFrozenDedup.version++
}
eximFrozenDedup.mu.Unlock()
return nil
}
// saveEximFrozenDedup writes the table to the state store when it changed
// since the last successful save. Callers serialize saves: the periodic
// writer exits before the final shutdown save runs.
func saveEximFrozenDedup(db *store.DB) error {
eximFrozenDedup.mu.Lock()
if eximFrozenDedup.version == eximFrozenDedup.persisted {
eximFrozenDedup.mu.Unlock()
return nil
}
version := eximFrozenDedup.version
snapshot := make(map[string]time.Time, eximFrozenDedup.order.Len())
for el := eximFrozenDedup.order.Front(); el != nil; el = el.Next() {
sighting := el.Value.(*eximFrozenSighting)
snapshot[sighting.id] = sighting.lastSeen
}
eximFrozenDedup.mu.Unlock()
if err := db.SaveEximFrozenSeen(snapshot); err != nil {
return err
}
eximFrozenDedup.mu.Lock()
eximFrozenDedup.persisted = version
eximFrozenDedup.mu.Unlock()
return nil
}
// restoreEximFrozenDedup loads the persisted frozen-message table. Without a
// state store the daemon keeps the table in memory only.
func (d *Daemon) restoreEximFrozenDedup() {
if sdb := store.Global(); sdb != nil {
if err := loadEximFrozenDedup(sdb, time.Now()); err != nil {
csmlog.Warn("exim frozen-message dedup load failed", "err", err)
}
}
}
// persistEximFrozenDedup saves the frozen-message table if it changed.
func (d *Daemon) persistEximFrozenDedup() {
if sdb := store.Global(); sdb != nil {
if err := saveEximFrozenDedup(sdb); err != nil {
csmlog.Warn("exim frozen-message dedup persistence failed", "err", err)
}
}
}
// startEximFrozenDedupPersistence saves changed dedup state periodically while
// the exim mainlog is watched. It stops on d.stopCh without a final save; the
// shutdown path saves once more after every log watcher has exited.
func (d *Daemon) startEximFrozenDedupPersistence() {
d.wg.Add(1)
obs.Go("exim-frozen-dedup-persist", func() {
defer d.wg.Done()
ticker := time.NewTicker(eximFrozenDedupPersistInterval)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
d.persistEximFrozenDedup()
}
}
})
}
// releaseEximFrozenDedup re-arms a freeze finding that the log watcher could
// not enqueue. Without this rollback, one full alert channel would discard the
// first finding and suppress every later queue-run reminder for that message.
func releaseEximFrozenDedup(line string) {
id, event := parseEximFrozenEvent(line)
if id == "" || event != eximFrozenEventFreeze {
return
}
forgetEximFrozenID(id)
}
// parseEximFrozenEvent recognizes only Exim's action field, not arbitrary
// occurrences such as an attacker-controlled Subject containing "Frozen".
func parseEximFrozenEvent(line string) (string, eximFrozenEvent) {
if !strings.Contains(line, "frozen") && !strings.Contains(line, "Frozen") {
return "", eximFrozenEventNone
}
p, rest, ok := eximlog.SplitPrefix(line, time.Local)
if !ok {
return "", eximFrozenEventNone
}
fields := strings.Fields(rest)
if p.MessageID != "" {
return p.MessageID, parseEximFrozenAction(fields)
}
// Preserve fail-open behavior for a future message-ID format, but only
// when the field immediately after the candidate ID is an actual Exim
// freeze action. Treating every ID-less occurrence of "frozen" as an event
// lets subjects and router errors manufacture findings.
if event := parseEximFrozenAction(fields); event != eximFrozenEventNone {
return "", event
}
if len(fields) > 0 {
return "", parseEximFrozenAction(fields[1:])
}
return "", eximFrozenEventNone
}
func parseEximFrozenAction(action []string) eximFrozenEvent {
if len(action) == 0 {
return eximFrozenEventNone
}
if strings.EqualFold(action[0], "unfrozen") {
return eximFrozenEventUnfreeze
}
if strings.EqualFold(action[0], "frozen") {
return eximFrozenEventFreeze
}
if len(action) >= 3 &&
strings.EqualFold(action[0], "message") &&
strings.EqualFold(action[1], "is") &&
strings.EqualFold(action[2], "frozen") {
return eximFrozenEventFreeze
}
return eximFrozenEventNone
}
package daemon
import (
"strings"
"sync/atomic"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/geoip"
)
// daemonGeoIPDB is a package-level pointer to the GeoIP database, set once
// during daemon init and read by log watcher handlers for country filtering.
var daemonGeoIPDB atomic.Pointer[geoip.DB]
// setGeoIPDB stores the GeoIP database for daemon-wide use and wires the
// ASN resolver the bad_asn_outbound detector uses. Clearing the DB (nil)
// disables ASN classification.
func setGeoIPDB(db *geoip.DB) {
daemonGeoIPDB.Store(db)
if db == nil {
checks.SetASNLookup(nil)
return
}
checks.SetASNLookup(func(ip string) (uint, string) {
info := db.Lookup(ip)
return info.ASN, info.ASOrg
})
}
// getGeoIPDB returns the daemon's GeoIP database, or nil.
func getGeoIPDB() *geoip.DB {
return daemonGeoIPDB.Load()
}
// isTrustedCountry checks if an IP's country is in the trusted list.
// Returns false if GeoIP is unavailable or country can't be resolved.
func isTrustedCountry(ip string, trustedCountries []string) bool {
if len(trustedCountries) == 0 {
return false
}
db := getGeoIPDB()
if db == nil {
return false
}
info := db.Lookup(ip)
if info.Country == "" {
return false
}
for _, tc := range trustedCountries {
if strings.EqualFold(info.Country, tc) {
return true
}
}
return false
}
package daemon
import (
"fmt"
"strings"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
)
// Enhanced access_log handler - catches File Manager, API failures,
// webmail logins, wp-login brute force, and xmlrpc abuse.
func parseAccessLogLineEnhanced(line string, cfg *config.Config) []alert.Finding {
var findings []alert.Finding
rec, ok := checks.ParseCpanelAccessRecord(line)
if !ok {
return nil
}
ip := rec.IP
// Only a direct peer can claim address-based exemptions.
if !rec.Proxy && (isInfraIPDaemon(ip, cfg.InfraIPs) || ip == "127.0.0.1") {
return nil
}
// Skips need the status the line names unambiguously; counts against
// the sender take any status client text could have forged.
rejected := rec.Status == "401" || rec.Status == "403"
client, source := ip, "non-infra IP: "+ip
sourceIP, dedup := ip, ""
if rec.Proxy {
// Legacy responses and scoring trust SourceIP independently of
// attribution. The log details retain the unverified address.
sourceIP, dedup = "", "proxy:"+ip
client = "a service subdomain (client address unverified)"
source = client
}
// File Manager write operations (port 2083)
// Only match actual write actions - not read-only calls like get_homedir.
// Skip 401/403 responses - the server rejected the request, no write occurred.
// Match against the request target only, not the referer, which can
// contain "upload" in paths.
if rec.Port == "2083" && !rejected {
if rec.TargetContains("fileman/save_file", "fileman/upload_files",
"fileman/paste", "fileman/rename", "fileman/delete") {
findings = append(findings, alert.Finding{
// Warning, not Critical: this fires on every File Manager
// write by any customer, and 401/403 are already skipped
// above, so it only reports authenticated operations. Its
// value is correlation with other findings on the account,
// not the single event.
Severity: alert.Warning,
Check: "cpanel_file_upload_realtime",
Message: fmt.Sprintf("cPanel File Manager write from %s", source),
Details: truncateDaemon(line, 300),
SourceIP: sourceIP,
DedupKey: dedup,
})
}
}
// API authentication failures (401/403)
// Suppress 401s that are stale-session artifacts from a recent password change.
// When a user changes their password, in-flight browser AJAX requests (notification
// polls, etc.) will 401 against the now-invalidated session - that's expected, not
// an attack. Real API abuse won't correlate with a recent purge for the same account.
// A failure counts as brute force against its source and can block it;
// a trusted proxy's address stands for every visitor behind it.
// The scheduled scan counts the same records (checks.CheckAPIAuthFailures).
if rec.APIAuthFailure() && (rec.Proxy || !checks.IsTrustedProxy(ip, cfg)) {
if rec.Proxy || rec.Status != "401" || !purgeTracker.isPostPurge401(ip) {
attribution := admission.AttributionUnknown
if !rec.Proxy {
attribution = admission.AttributionPeer
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "api_auth_failure_realtime",
Message: fmt.Sprintf("cPanel API auth failure from %s", client),
Details: truncateDaemon(line, 300),
SourceIP: sourceIP,
DedupKey: dedup,
Attribution: attribution,
})
}
}
// Webmail login attempts (port 2095/2096), as the scheduled scan counts them.
if !cfg.Suppressions.SuppressWebmail && rec.WebmailLogin() {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "webmail_login_realtime",
Message: fmt.Sprintf("Webmail login attempt from %s", source),
Details: truncateDaemon(line, 200),
SourceIP: sourceIP,
DedupKey: dedup,
})
}
// WHM login attempts (port 2086/2087). CVE-2026-41940 step 1 creates the
// preauth session via a POST to the WHM login endpoint; the CRLF
// injection lands when cpsrvd writes that session file. Surfacing every
// non-infra POST gives ops a brute-force/recon signal even on patched
// hosts. Suppressed under the cPanel-login suppression flag because WHM
// is the admin face of cPanel and shares the same noise profile.
isWHMPort := rec.Port == "2087" || rec.Port == "2086"
if isWHMPort && !cfg.Suppressions.SuppressCpanelLogin {
if rec.Any(func(q checks.CpanelRequest) bool {
return q.MethodIs("POST") && q.TargetHasPrefix("/login/?login_only=1")
}) {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "whm_login_realtime",
Message: fmt.Sprintf("WHM login attempt from %s", source),
Details: truncateDaemon(line, 200),
SourceIP: sourceIP,
DedupKey: dedup,
})
}
}
// CVE-2026-41940 step 4 fingerprint: a tokenless request to a
// token-required WHM path triggers do_token_denied(), which the watchTowr
// PoC abuses to promote a CRLF-injected session record into the JSON
// cache. Legitimate WHM clients always prefix /scripts*/* with the
// /cpsessXXXXXX/ security token, so the bare prefix on a WHM port is a
// hard signature, not a heuristic. Matches both /scripts/ and /scripts2/
// because the do_token_denied() trigger is path-agnostic - the watchTowr
// PoC happens to use listaccts but any token-required endpoint works.
// Fires regardless of suppression - this is an attack IOC, not a login.
if isWHMPort && rec.Any(func(q checks.CpanelRequest) bool { return isUnauthWHMScriptsTarget(q.Target) }) {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "whm_unauth_scripts_realtime",
Message: fmt.Sprintf("Tokenless WHM scripts request from %s (CVE-2026-41940 IOC)", client),
Details: truncateDaemon(line, 300),
SourceIP: sourceIP,
DedupKey: dedup,
})
}
return findings
}
// isUnauthWHMScriptsTarget returns true when the request target is a path
// under /scripts/ or /scripts2/ without a /cpsessXXXXXX/ security-token
// prefix - the literal step-4 fingerprint of CVE-2026-41940. Query strings
// are stripped before comparison.
func isUnauthWHMScriptsTarget(path string) bool {
if q := strings.Index(path, "?"); q >= 0 {
path = path[:q]
}
if strings.Contains(path, "/cpsess") {
return false
}
return strings.HasPrefix(path, "/scripts/") || strings.HasPrefix(path, "/scripts2/")
}
// parseFTPLogLine handles FTP log entries from /var/log/messages.
func parseFTPLogLine(line string, cfg *config.Config) []alert.Finding {
var findings []alert.Finding
ip := checks.PureFTPDClientIP(line)
if ip == "" || isInfraIPDaemon(ip, cfg.InfraIPs) {
return nil
}
// Failed authentication
if strings.Contains(line, "Authentication failed") || strings.Contains(line, "auth failed") {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "ftp_auth_failure_realtime",
Message: fmt.Sprintf("FTP authentication failed from %s", ip),
Details: truncateDaemon(line, 200),
SourceIP: ip,
// The pure-ftpd peer slot (checks.PureFTPDClientIP).
Attribution: admission.AttributionPeer,
})
}
// Successful login from non-infra. The scheduled ftp_logins check reads
// the same file and will meet this line again, so the finding comes from
// the shared builder: identical findings collapse in the state store,
// while a login this watcher never saw is still reported by the scan.
// The builder also drops loopback, which panel transfers use; a local
// relay does not make the auth failure above trustworthy.
if f, ok := checks.FTPLoginFinding(line, cfg); ok {
findings = append(findings, f)
}
return findings
}
package daemon
import (
"fmt"
"net/netip"
"os"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/platform"
)
// Real-time access log handler for detecting wp-login.php brute force and
// xmlrpc.php abuse. Watches the LiteSpeed/Apache Combined Log Format access
// log and tracks per-IP POST counts using a sliding time window.
//
// Emits the same check names as the periodic CheckWPBruteForce
// (wp_login_bruteforce, xmlrpc_abuse) so the existing auto-block pipeline
// handles them automatically.
const (
// Sliding window for counting requests per IP.
accessLogWindow = 5 * time.Minute
// Threshold within the window. Lower than periodic checks because
// we're watching in real-time and want fast response.
accessLogWPLoginThreshold = 10
// Eviction: how often to prune expired trackers.
accessLogEvictInterval = 5 * time.Minute
// Cooldown after an IP is flagged: don't re-alert for this long.
// Prevents alert spam while auto-block processes the finding.
accessLogBlockCooldown = 30 * time.Minute
)
// accessLogTracker tracks POST timestamps per endpoint for a single IP.
type accessLogTracker struct {
mu sync.Mutex
wpLoginTimes []time.Time
xmlrpcTimes []time.Time
adminPanelTimes []time.Time
wpLoginAlerted bool
xmlrpcAlerted bool
adminPanelAlerted bool
lastSeen time.Time
generation uint64
evicting bool
}
// accessLogTrackers holds per-IP state. sync.Map for concurrent handler access.
var accessLogTrackers sync.Map // key: IP string → value: *accessLogTracker
// accessLogTrackerCount approximates the live entry count in
// accessLogTrackers. sync.Map exposes no Len(); maintaining a side
// counter is the canonical workaround. Used to trigger eager
// eviction during a DDoS burst, where the 5-min timer alone would
// let the map grow into the hundreds of thousands of unique IPs
// before the next prune.
var (
accessLogTrackerCount atomic.Int64
accessLogEagerEvictTrip = make(chan struct{}, 1)
)
// accessLogEvictSoftCap is the live-entry threshold above which the
// hot path nudges the eviction goroutine to run sooner than the
// 5-min ticker. Picked to be well below typical memory limits even
// at the worst per-tracker size (~256 bytes) so a 100k entry
// burst stays under 32 MB.
const (
accessLogEvictSoftCap int64 = 50000
accessLogEvictTargetPercent int64 = 95
)
type accessLogEvictionCandidate struct {
key string
tracker *accessLogTracker
lastSeen time.Time
generation uint64
}
// discoverAccessLogPath returns the first access log path that exists,
// consulting the platform detector for OS/web-server specific candidates.
func discoverAccessLogPath() string {
info := platform.Detect()
for _, p := range info.AccessLogPaths {
if _, err := os.Stat(p); err == nil {
return p
}
}
return ""
}
// parseAccessLogBruteForce is the LogLineHandler for the Apache/LiteSpeed
// Combined Log Format access log. It parses each line, tracks per-IP POST
// counts to wp-login.php and xmlrpc.php, and emits findings when thresholds
// are crossed.
func parseAccessLogBruteForce(line string, cfg *config.Config) []alert.Finding {
return parseAccessLogBruteForceForLog(line, cfg, true)
}
func accessLogHandlerForPath(path string) LogLineHandler {
central := platform.Detect().IsCPanelCentralAccessLog(path)
return func(line string, cfg *config.Config) []alert.Finding {
return parseAccessLogBruteForceForLog(line, cfg, central)
}
}
func parseAccessLogBruteForceForLog(line string, cfg *config.Config, central bool) []alert.Finding {
// Fast reject: only care about POST requests to known attack targets.
if !strings.Contains(line, "POST") {
return nil
}
raw, method, path, ok := accessLogIPMethodPath(line)
if !ok {
return nil
}
// The web server writes the peer as a bare address; act only on its
// canonical form, so a block names what the firewall matches.
peer, err := netip.ParseAddr(raw)
if err != nil || peer.Zone() != "" {
return nil
}
ip := peer.Unmap().String()
// The panel's own log carries requests a cPanel proxy subdomain passed
// through the web server. The reader identifies the log's provenance.
if checks.IsProxiedPanelRequest(path, central) {
return nil
}
// Skip infra IPs and loopback.
if ip == "127.0.0.1" || ip == "::1" || isInfraIPDaemon(ip, cfg.InfraIPs) {
return nil
}
if method != "POST" {
return nil
}
isWPLogin := strings.Contains(path, "wp-login.php")
isXMLRPC := strings.Contains(path, "xmlrpc.php")
isAdminPanel := isAdminPanelPath(path)
xmlrpcThreshold := effectiveAccessLogXMLRPCThreshold(cfg)
if isXMLRPC && xmlrpcThreshold <= 0 {
isXMLRPC = false
}
if !isWPLogin && !isXMLRPC && !isAdminPanel {
return nil
}
// A trusted proxy's address stands for every visitor behind it. This
// path reads only the direct peer, so it leaves forwarded requests to
// the scheduled scan, which reads the client from X-Forwarded-For when
// the log records it.
if checks.IsTrustedProxy(ip, cfg) {
return nil
}
now := time.Now()
tracker := loadAccessLogTracker(ip, now)
defer tracker.mu.Unlock()
tracker.lastSeen = now
tracker.generation++
cutoff := now.Add(-accessLogWindow)
var results []alert.Finding
// Once a per-tier alert has fired, skip pruneAndAppend until cooldown
// clears the `alerted` flag. The slice would otherwise grow on every
// event during a sustained burst (potentially tens of thousands of
// entries for a 5-min window at 100 rps), wasting CPU on prune passes
// whose result is never consumed: the `alerted` flag already prevents
// re-alerts, and the eviction loop trims the slice on its own schedule.
//
// Safety: evictAccessLogState resets `alerted` once `lastSeen` is older
// than `cooldownCutoff` (30 min of silence by default). By that point
// the same eviction call has also pruned the slice to empty (window is
// 5 min, so any remaining timestamp is far past cutoff), so the next
// matching event correctly starts a fresh count from 1.
if isWPLogin && !tracker.wpLoginAlerted {
tracker.wpLoginTimes = pruneAndAppend(tracker.wpLoginTimes, cutoff, now)
if len(tracker.wpLoginTimes) >= accessLogWPLoginThreshold {
tracker.wpLoginAlerted = true
results = append(results, alert.Finding{
Severity: alert.Critical,
Check: "wp_login_bruteforce",
Message: fmt.Sprintf("WordPress login brute force from %s: %d POSTs in %v (real-time)", ip, len(tracker.wpLoginTimes), accessLogWindow),
Details: "Real-time detection: high rate of POST requests to wp-login.php",
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionHTTPPeer,
})
}
}
if isXMLRPC && !tracker.xmlrpcAlerted {
tracker.xmlrpcTimes = pruneAndAppend(tracker.xmlrpcTimes, cutoff, now)
if len(tracker.xmlrpcTimes) >= xmlrpcThreshold {
tracker.xmlrpcAlerted = true
results = append(results, alert.Finding{
Severity: alert.Critical,
Check: "xmlrpc_abuse",
Message: fmt.Sprintf("XML-RPC abuse from %s: %d POSTs in %v (real-time)", ip, len(tracker.xmlrpcTimes), accessLogWindow),
Details: "Real-time detection: high rate of POST requests to xmlrpc.php (brute force or amplification)",
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionHTTPPeer,
})
}
}
if isAdminPanel && !tracker.adminPanelAlerted {
tracker.adminPanelTimes = pruneAndAppend(tracker.adminPanelTimes, cutoff, now)
if len(tracker.adminPanelTimes) >= accessLogWPLoginThreshold {
tracker.adminPanelAlerted = true
results = append(results, alert.Finding{
Severity: alert.Critical,
Check: "admin_panel_bruteforce",
Message: fmt.Sprintf("Admin panel brute force from %s: %d POSTs in %v (real-time)", ip, len(tracker.adminPanelTimes), accessLogWindow),
Details: "Real-time detection: high rate of POST requests to common admin panel login paths (phpMyAdmin / Joomla)",
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionHTTPPeer,
})
}
}
return results
}
func effectiveAccessLogXMLRPCThreshold(cfg *config.Config) int {
if cfg == nil {
return config.DefaultXMLRPCThreshold
}
return cfg.Thresholds.XMLRPCThreshold
}
func loadAccessLogTracker(ip string, now time.Time) *accessLogTracker {
for {
val, loaded := accessLogTrackers.LoadOrStore(ip, &accessLogTracker{lastSeen: now})
tracker := val.(*accessLogTracker)
if !loaded && accessLogTrackerCount.Add(1) > accessLogEvictSoftCap {
signalAccessLogEagerEviction()
}
tracker.mu.Lock()
if !tracker.evicting {
return tracker
}
tracker.mu.Unlock()
if accessLogTrackers.CompareAndDelete(ip, tracker) {
decrementAccessLogTrackerCount()
}
}
}
func signalAccessLogEagerEviction() {
select {
case accessLogEagerEvictTrip <- struct{}{}:
default:
}
}
// pruneAndAppend removes entries older than cutoff and appends now.
func pruneAndAppend(times []time.Time, cutoff, now time.Time) []time.Time {
recent := times[:0]
for _, t := range times {
if !t.Before(cutoff) {
recent = append(recent, t)
}
}
return append(recent, now)
}
// StartAccessLogEviction starts a background goroutine that prunes expired
// tracker entries to prevent unbounded memory growth. Same pattern as
// StartModSecEviction.
func StartAccessLogEviction(stopCh <-chan struct{}) {
obs.Go("accesslog-eviction", func() {
ticker := time.NewTicker(accessLogEvictInterval)
defer ticker.Stop()
for {
select {
case <-stopCh:
return
case now := <-ticker.C:
evictAccessLogState(now)
case <-accessLogEagerEvictTrip:
// Soft-cap signal from the hot path. Run an
// immediate eviction so a DDoS burst of unique
// IPs cannot grow the tracker map past memory
// budget before the next 5-min tick.
evictAccessLogState(time.Now())
}
}
})
}
func evictAccessLogState(now time.Time) {
evictAccessLogStateWithCap(now, accessLogEvictSoftCap)
}
func evictAccessLogStateWithCap(now time.Time, cap int64) {
cutoff := now.Add(-accessLogWindow)
cooldownCutoff := now.Add(-accessLogBlockCooldown)
candidates := make([]accessLogEvictionCandidate, 0)
accessLogTrackers.Range(func(key, value any) bool {
ip := key.(string)
tracker := value.(*accessLogTracker)
tracker.mu.Lock()
if tracker.evicting {
tracker.mu.Unlock()
return true
}
// Prune old timestamps.
tracker.wpLoginTimes = pruneSlice(tracker.wpLoginTimes, cutoff)
tracker.xmlrpcTimes = pruneSlice(tracker.xmlrpcTimes, cutoff)
tracker.adminPanelTimes = pruneSlice(tracker.adminPanelTimes, cutoff)
// Reset alerted flags after cooldown so the IP can be re-detected
// if it comes back after the block expires.
if tracker.wpLoginAlerted && tracker.lastSeen.Before(cooldownCutoff) {
tracker.wpLoginAlerted = false
}
if tracker.xmlrpcAlerted && tracker.lastSeen.Before(cooldownCutoff) {
tracker.xmlrpcAlerted = false
}
if tracker.adminPanelAlerted && tracker.lastSeen.Before(cooldownCutoff) {
tracker.adminPanelAlerted = false
}
empty := len(tracker.wpLoginTimes) == 0 && len(tracker.xmlrpcTimes) == 0 &&
len(tracker.adminPanelTimes) == 0 && tracker.lastSeen.Before(cooldownCutoff)
if empty {
deleteAccessLogTrackerLocked(ip, tracker)
} else {
candidates = append(candidates, accessLogEvictionCandidate{
key: ip,
tracker: tracker,
lastSeen: tracker.lastSeen,
generation: tracker.generation,
})
}
tracker.mu.Unlock()
return true
})
enforceAccessLogTrackerCap(candidates, cap)
}
func enforceAccessLogTrackerCap(candidates []accessLogEvictionCandidate, cap int64) {
if cap <= 0 || accessLogTrackerCount.Load() <= cap {
return
}
sort.Slice(candidates, func(i, j int) bool {
return candidates[i].lastSeen.Before(candidates[j].lastSeen)
})
target := cap * accessLogEvictTargetPercent / 100
for _, candidate := range candidates {
if accessLogTrackerCount.Load() <= target {
return
}
tracker := candidate.tracker
tracker.mu.Lock()
if !tracker.evicting &&
tracker.generation == candidate.generation &&
tracker.lastSeen.Equal(candidate.lastSeen) {
deleteAccessLogTrackerLocked(candidate.key, tracker)
}
tracker.mu.Unlock()
}
}
func deleteAccessLogTrackerLocked(key string, tracker *accessLogTracker) {
tracker.evicting = true
if accessLogTrackers.CompareAndDelete(key, tracker) {
decrementAccessLogTrackerCount()
}
}
func decrementAccessLogTrackerCount() {
for {
current := accessLogTrackerCount.Load()
if current <= 0 {
return
}
if accessLogTrackerCount.CompareAndSwap(current, current-1) {
return
}
}
}
// accessLogIPMethodPath extracts client IP, request method, and request path
// from an Apache/LiteSpeed Combined Log Format line without allocating a
// string slice. Hot path: each domlog line that survives the "POST" prefilter
// hits this. strings.Fields allocates len(fields)+1 strings per call; this
// scanner only returns sub-strings that share the input's backing array.
func accessLogIPMethodPath(line string) (ip, method, path string, ok bool) {
n := len(line)
i := 0
for i < n && isAccessLogSpace(line[i]) {
i++
}
start := i
for i < n && !isAccessLogSpace(line[i]) {
i++
}
ip = line[start:i]
if ip == "" {
return "", "", "", false
}
// A remote user may contain spaces and brackets. Its quotes are escaped,
// so only the timestamp ends with a bracket, blanks and a literal quote.
for ; i < n; i++ {
if line[i] != ']' {
continue
}
j := i + 1
for j < n && isAccessLogSpace(line[j]) {
j++
}
if j > i+1 && j < n && line[j] == '"' {
i = j
break
}
}
var fields [2]string
for f := range fields {
for i < n && isAccessLogSpace(line[i]) {
i++
}
if i >= n {
return "", "", "", false
}
start = i
for i < n && !isAccessLogSpace(line[i]) {
i++
}
fields[f] = line[start:i]
}
method = fields[0]
for len(method) > 0 && method[0] == '"' {
method = method[1:]
}
for len(method) > 0 && method[len(method)-1] == '"' {
method = method[:len(method)-1]
}
return ip, method, fields[1], true
}
func isAccessLogSpace(b byte) bool {
return b == ' ' || b == '\t' || b == '\n' || b == '\v' || b == '\f' || b == '\r'
}
// isAdminPanelPath returns true for high-confidence non-WP admin panel login
// paths suitable for hard-block auto-response. Drupal /user/login, Tomcat
// /manager/html, generic /admin/login.php, /mysql/ are intentionally EXCLUDED
// because they're either too generic (FP risk on shared hosting) or use a
// different attack shape (Basic auth vs. POST forms).
func isAdminPanelPath(path string) bool {
return strings.Contains(path, "/phpmyadmin/index.php") ||
strings.Contains(path, "/pma/index.php") ||
strings.Contains(path, "/phpMyAdmin/index.php") ||
strings.Contains(path, "/administrator/index.php")
}
func pruneSlice(times []time.Time, cutoff time.Time) []time.Time {
recent := times[:0]
for _, t := range times {
if !t.Before(cutoff) {
recent = append(recent, t)
}
}
return recent
}
package daemon
import (
"fmt"
"net"
"os"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/geoip"
"github.com/pidginhost/csm/internal/store"
)
const (
geoHistoryMaxAge = 30 * 24 * 60 * 60 // 30 days in seconds
geoMinLoginCount = 5 // minimum logins before alerting on new country
geoAlertCooldownH = 24 // hours between alerts per account
)
// geoLookup resolves an IP through the daemon's GeoIP database, returning an
// empty Info when no database is loaded. A variable so tests can drive the geo
// detector without an mmdb file.
var geoLookup = func(ip string) geoip.Info {
db := getGeoIPDB()
if db == nil {
return geoip.Info{}
}
return db.Lookup(ip)
}
// parseDovecotLogLine handles Dovecot login lines from /var/log/maillog.
// It tracks per-mailbox login countries and alerts on new-country logins.
func parseDovecotLogLine(line string, cfg *config.Config) []alert.Finding {
// Only process successful login lines. dovecotLoginSucceeded accepts both
// the classic "Login: user=<" and the production "Logged in: user=<"
// formats so this detector is not silently dead on whichever one the host
// emits.
if !dovecotLoginSucceeded(line) {
return nil
}
user, ip := parseDovecotLoginFields(line)
if user == "" || ip == "" {
return nil
}
// Skip private/loopback IPs
if isPrivateOrLoopback(ip) {
return nil
}
// Skip infra IPs
if isInfraIPDaemon(ip, cfg.InfraIPs) {
return nil
}
info := geoLookup(ip)
country := info.Country
if country == "" {
return nil
}
// A trusted country suppresses the alert, not the login: the logins from
// home are what carry a mailbox past the login floor and teach its
// baseline. Returning before the history update left every mailbox whose
// owner lives in a trusted country stuck at zero logins, so its first
// foreign login was recorded as a known country and never alerted.
trusted := countryTrusted(country, cfg.Suppressions.TrustedCountries)
// Load history from bbolt
boltDB := store.Global()
if boltDB == nil {
return nil
}
now := time.Now().Unix()
history, _ := boltDB.GetGeoHistory(user)
if history.Countries == nil {
history.Countries = make(map[string]int64)
}
// Prune old country entries
history.Countries = pruneOldCountries(history.Countries, now, geoHistoryMaxAge)
// Increment login count
history.LoginCount++
// Check if this is a new country
_, countryKnown := history.Countries[country]
isNewCountry := !trusted && !countryKnown && history.LoginCount >= geoMinLoginCount
// Update country timestamp
history.Countries[country] = now
// Persist updated history
if err := boltDB.SetGeoHistory(user, history); err != nil {
fmt.Fprintf(os.Stderr, "[%s] Warning: failed to save geo history for %s: %v\n",
time.Now().Format("2006-01-02 15:04:05"), user, err)
}
if !isNewCountry {
return nil
}
// Rate limit: max 1 alert per account per 24h
alertKey := "email:geo_alert:" + user
lastAlertStr := boltDB.GetMetaString(alertKey)
if lastAlertStr != "" {
if lastAlert, err := time.Parse(time.RFC3339, lastAlertStr); err == nil {
if time.Since(lastAlert) < time.Duration(geoAlertCooldownH)*time.Hour {
return nil
}
}
}
// Record alert time
_ = boltDB.SetMetaString(alertKey, time.Now().Format(time.RFC3339))
// Build "previously seen" country list
var knownCountries []string
for c := range history.Countries {
if c != country {
knownCountries = append(knownCountries, c)
}
}
previousList := "none"
if len(knownCountries) > 0 {
previousList = strings.Join(knownCountries, ", ")
}
countryName := info.CountryName
if countryName == "" {
countryName = country
}
mailbox, domain, _ := splitMailAccount(user)
tenant := mailAccountOwner(user)
return []alert.Finding{{
Severity: alert.High,
Check: "email_suspicious_geo",
Message: fmt.Sprintf("Suspicious email login for %s from %s (%s) - previously seen: %s",
user, countryName, ip, previousList),
Details: fmt.Sprintf("Country: %s (%s)\nIP: %s\nLogin count: %d\nPreviously seen countries: %s",
country, countryName, ip, history.LoginCount, previousList),
SourceIP: ip,
Mailbox: mailbox,
Domain: domain,
TenantID: tenant,
Timestamp: time.Now(),
}}
}
// parseDovecotLoginFields extracts user and remote IP from a Dovecot login line.
// Expected format: "... Login: user=<user@example.com>, ... rip=203.0.113.5, ..."
func parseDovecotLoginFields(line string) (user, ip string) {
user, _ = checks.DovecotLoginFields(line)
ip = checks.DovecotRemoteIP(line)
if user == "" || ip == "" {
return "", ""
}
return user, ip
}
// isPrivateOrLoopback returns true if the IP is loopback, RFC1918 private or
// an IPv6 unique local address.
func isPrivateOrLoopback(ipStr string) bool {
ip := net.ParseIP(ipStr)
if ip == nil {
return true // invalid = skip
}
return ip.IsLoopback() || ip.IsPrivate()
}
// pruneOldCountries removes country entries older than maxAge seconds.
func pruneOldCountries(countries map[string]int64, now, maxAge int64) map[string]int64 {
pruned := make(map[string]int64, len(countries))
cutoff := now - maxAge
for c, ts := range countries {
if ts >= cutoff {
pruned[c] = ts
}
}
return pruned
}
package daemon
import (
"fmt"
"net"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/modsec"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/store"
)
// modsecDenyEvent is one ModSecurity event recorded in an IP's sliding window.
// Warnings (isBlock=false) are recorded only when high-confidence or unknown,
// to supply fail-secure evidence to a later low-confidence anomaly-threshold
// deny in the same burst (anomaly-scoring WAFs log the specific attack rule as
// a warning, then deny on the points-threshold rule).
type modsecDenyEvent struct {
t time.Time
rule int
conf modsecConfidence
isBlock bool
}
// modsecIPCounter tracks recent ModSecurity events for a single IP.
type modsecIPCounter struct {
mu sync.Mutex
events []modsecDenyEvent
escalated bool // latched once escalation fires; reset when the window drains
lastEscalated time.Time // lets a sustained source refresh an expired firewall block
lowBurstEmitted bool // latched once the low-confidence burst finding fires
}
// modsecEscalationOutcome reports what a single recorded event should produce.
type modsecEscalationOutcome struct {
escalate bool // fire the Critical auto-block escalation finding
lowConfBurst bool // fire the non-actioned low-confidence-burst visibility finding
classifierGap bool // fire the non-actioned classifier-gap finding (new unknown rule)
singleHit bool // escalation came from an exploit-specific deny, not the hit count
}
type modsecWindowSummary struct {
totalBlock int
lowConfBlock int
highEvidence bool
unknownEvidence bool
singleHit bool
}
var (
modsecDedup sync.Map // key: "IP:ruleID" → value: time.Time
modsecBlockCount sync.Map // key: IP → value: *modsecIPCounter
)
// Classifier gaps describe the confidence table, not a source, so they are
// tracked per rule ID for the whole host.
var (
modsecGapMu sync.Mutex
modsecGapReported = map[int]time.Time{}
)
const (
modsecDedupTTL = 60 * time.Second
modsecEvictInterval = 10 * time.Minute
modsecDefaultEscalationWin = 10 * time.Minute
modsecDefaultEscalationHits = 3
// modsecClassifierGapInterval is how long one unknown rule stays quiet after
// its classifier-gap finding before an unresolved gap is reported again.
modsecClassifierGapInterval = 24 * time.Hour
)
// modsecEscalationParams returns the operator-tuned (hits, window) pair,
// falling back to the shipped defaults when either is unset or
// non-positive. nil cfg returns the defaults so test wiring without a
// config still behaves predictably.
func modsecEscalationParams(cfg *config.Config) (int, time.Duration) {
hits := modsecDefaultEscalationHits
win := modsecDefaultEscalationWin
if cfg == nil {
return hits, win
}
if cfg.Thresholds.ModSecEscalationHits > 0 {
hits = cfg.Thresholds.ModSecEscalationHits
}
if cfg.Thresholds.ModSecEscalationWindowMin > 0 {
win = time.Duration(cfg.Thresholds.ModSecEscalationWindowMin) * time.Minute
}
return hits, win
}
// modsecDefaultLowConfEscalationHits is the shipped low-confidence-only backstop:
// how many low-confidence policy/anomaly denies from one IP within the
// escalation window force a firewall escalation even with no attack signature.
// It closes the deliberate "only trip anomaly/content-type rules" bypass while
// staying far above any single legitimate checkout-retry pattern.
const modsecDefaultLowConfEscalationHits = 30
const modsecDefaultEscalationRearmInterval = 24 * time.Hour
// modsecLowConfEscalationHits returns the operator-tuned backstop count,
// falling back to the shipped default when unset or non-positive. Raise it on
// hosts with high-volume legitimate apps that trip policy/anomaly rules in
// bulk; it is not meant to be disabled (that would reopen the bypass).
func modsecLowConfEscalationHits(cfg *config.Config) int {
if cfg == nil {
return modsecDefaultLowConfEscalationHits
}
if cfg.Thresholds.ModSecLowConfidenceEscalationHits > 0 {
return cfg.Thresholds.ModSecLowConfidenceEscalationHits
}
return modsecDefaultLowConfEscalationHits
}
// modsecEscalationRearmInterval mirrors the firewall auto-block expiry so a
// sustained attacker can refresh the ban after it would expire, without
// re-emitting duplicate escalation findings during the active block window.
func modsecEscalationRearmInterval(cfg *config.Config) time.Duration {
if cfg == nil || cfg.AutoResponse.BlockExpiry == "" {
return modsecDefaultEscalationRearmInterval
}
d, err := time.ParseDuration(cfg.AutoResponse.BlockExpiry)
if err != nil || d <= 0 {
return modsecDefaultEscalationRearmInterval
}
return d
}
// parseModSecLogLine parses a ModSecurity log line from Apache or LiteSpeed
// error logs and returns findings for blocked requests or warnings.
func parseModSecLogLine(line string, cfg *config.Config) []alert.Finding {
// Fast reject: not a ModSecurity line.
logMessage, isLiteSpeed := modsec.ErrorLogMessage(line)
if logMessage == "" {
return nil
}
ip := modsec.ErrorLogClientIP(line)
rule, ruleKnown := modsec.ErrorLogRule(line)
ruleID, msg, hostname, uri := rule.ID, rule.Msg, rule.Hostname, rule.URI
// Skip infra and loopback IPs - consistent with other realtime handlers
// (handlers.go:22, autoblock.go:149). Prevents noisy findings and
// false escalation from proxied or locally forwarded Apache traffic.
if ip != "" && (isInfraIPDaemon(ip, cfg.InfraIPs) || ip == "127.0.0.1" || ip == "::1") {
return nil
}
// Determine check name.
//
// Apache mod_security writes the action verbatim into the message, so
// "Access denied" at the start of its message is a block signal. Request
// fields later in the line can contain the same words. LiteSpeed's mod_security
// front-end writes every match as "triggered!" with no action context,
// regardless of whether the rule's declared action denied the request
// or merely incremented a counter. Without further context every match
// would be counted as a deny, escalating to a 24-hour auto-block after
// three pass-action triggers from the same IP. Consult the rule-action
// registry built at daemon start: pass/allow rules produce a warning,
// deny/drop and the other disruptive actions produce a block, block
// follows the rule's SecDefaultAction, and an unknown rule ID in a
// populated registry stays conservative. When the registry is empty, the
// line remains a warning until a refresh loads rule actions.
check := "modsec_warning_realtime"
if strings.HasPrefix(logMessage, "Access denied") {
check = "modsec_block_realtime"
} else if isLiteSpeed && strings.HasPrefix(logMessage, "mod_security rule ") && strings.Contains(logMessage, "triggered!") {
check = classifyLiteSpeedTrigger(ruleID)
}
ruleNum := 0
if n, err := strconv.Atoi(ruleID); err == nil {
ruleNum = n
}
conf := classifyModSecRuleConfidence(ruleNum, rule.File)
// Determine severity from confidence. Individual low-confidence and unknown
// blocks stay Warning so the confidence-gated escalation path remains the
// only firewall-grade signal for those rules.
// Individual blocks are informational - ModSecurity already denied the
// request. Only the escalation finding (3+ from same IP) is CRITICAL
// because it triggers auto-block at the firewall level.
severity := alert.Warning
if conf == modsecConfHigh {
severity = alert.High // specific attack/probe evidence, informational
}
// Build message.
label := ruleID
if !ruleKnown {
label = "unknown (ambiguous or incomplete log fields)"
}
message := fmt.Sprintf("ModSecurity rule %s", label)
if check == "modsec_block_realtime" {
message = fmt.Sprintf("ModSecurity blocked request: rule %s", label)
}
if ip != "" {
message += fmt.Sprintf(" from %s", ip)
}
if hostname != "" {
message += fmt.Sprintf(" on %s", hostname)
}
if uri != "" {
message += fmt.Sprintf(" uri=%s", uri)
}
if msg != "" {
message += fmt.Sprintf(" - %s", msg)
}
// Store structured details so the web UI can extract fields consistently
// regardless of whether the source was Apache or LiteSpeed format.
details := fmt.Sprintf("Rule: %s\nMessage: %s\nHostname: %s\nURI: %s\nRaw: %s",
ruleID, msg, hostname, uri, truncateDaemon(line, 300))
return []alert.Finding{{
Severity: severity,
Check: check,
Message: message,
Details: details,
SourceIP: ip,
Domain: domainOrEmpty(hostname),
}}
}
// domainOrEmpty returns hostname unless it parses as a bare IP address
// (v4 or v6, with or without surrounding brackets). Vhosts served on a
// raw IP would otherwise key the incident bucket on the IP literal,
// causing two unrelated victim sites that happen to be reachable over
// their public IPs to merge into a single bucket.
func domainOrEmpty(hostname string) string {
if hostname == "" {
return ""
}
probe := strings.TrimPrefix(hostname, "[")
probe = strings.TrimSuffix(probe, "]")
if net.ParseIP(probe) != nil {
return ""
}
return hostname
}
// classifyLiteSpeedTrigger decides whether a LiteSpeed mod_security
// "triggered!" line represents a real deny (block_realtime) or merely an
// informational pass-action match (warning_realtime), based on the rule's
// declared action in the rule-action registry. A populated registry with an
// unknown rule defaults to block; a nil or empty registry cannot distinguish
// pass-action matches from denies, so ambiguous lines stay warnings until a
// refresh loads rule actions.
func classifyLiteSpeedTrigger(ruleID string) string {
num, err := strconv.Atoi(ruleID)
if err != nil {
return "modsec_block_realtime"
}
reg := modsec.Global()
// No rule-action knowledge available: the registry has not been built yet,
// or the vendor rule tree was transiently empty (cPanel modsec_assemble
// mid-rewrite, or a boot-time web-server mis-detection). We cannot tell a
// pass-action scoring rule -- e.g. Comodo CWAF 210710 / 214930, which only
// add anomaly points and never deny -- from a real deny. Defaulting every
// "triggered" line to a block in this state false-escalates benign hits
// into 24h auto-bans of real visitors. Explicit "Access denied" lines are
// classified as blocks before this function is reached. LiteSpeed deny
// rules that log only "triggered!" are degraded to warning during this
// empty-registry window, but ambiguous lines must not auto-escalate while
// the registry is unavailable.
if reg == nil || reg.Len() == 0 {
return "modsec_warning_realtime"
}
action, known := reg.Action(num)
if !known {
// Registry IS populated but this specific rule is unrecognised -- stay
// conservative and treat it as a block so a genuinely unknown deny
// rule still escalates.
return "modsec_block_realtime"
}
if modsec.IsBlockingAction(action) {
return "modsec_block_realtime"
}
return "modsec_warning_realtime"
}
// modsecRuleFileBase returns the base name of the rule file a line names.
func modsecRuleFileBase(path string) string {
if path == "" {
return ""
}
return filepath.Base(path)
}
// parseModSecLogLineDeduped wraps parseModSecLogLine with dedup and block
// threshold escalation. It is the handler registered with the log watcher.
//
// Order of operations (critical for correctness):
// 1. Parse the raw line.
// 2. ALWAYS increment the block escalation counter (even if dedup will suppress).
// 3. Then check dedup - suppress the base finding if a duplicate, but still
// return any escalation finding from step 2.
func parseModSecLogLineDeduped(line string, cfg *config.Config) []alert.Finding {
raw := parseModSecLogLine(line, cfg)
if len(raw) == 0 {
return nil
}
f := raw[0]
now := time.Now()
var results []alert.Finding
// --- Step 1: block escalation (before dedup) ---
// Extract IP and rule ID directly from the raw log line - NOT from the
// finding message, which could be manipulated via log injection.
ip := modsec.ErrorLogClientIP(line)
rule, ruleKnown := modsec.ErrorLogRule(line)
ruleID := rule.ID
ruleNum, _ := strconv.Atoi(ruleID)
isBlock := f.Check == "modsec_block_realtime"
isCSM := isBlock && ruleNum >= 900000 && ruleNum <= 900999
// Record hit for per-rule stats (24h hourly buckets)
if ruleNum >= 900000 && ruleNum <= 900999 {
if sdb := store.Global(); sdb != nil {
sdb.IncrModSecRuleHit(ruleNum, now)
}
}
// Classify attack confidence from the rule ID, file, and registry metadata.
// This decides whether a deny may auto-escalate to a firewall ban
// (high/unknown) or only feeds the low-confidence visibility/backstop path
// (low).
conf := classifyModSecRuleConfidence(ruleNum, rule.File)
// Feed the per-IP window with every blocking deny, plus high/unknown
// warnings, which supply attack evidence to a later anomaly-threshold deny
// in the same burst. LiteSpeed often omits msg/tag on pass-action attack
// rules; unknown must fail secure instead of disappearing before the low
// anomaly rule denies. Operator-excluded rules are not recorded at all.
// A trusted proxy's address stands for every visitor behind it: its
// denies stay visible below but never escalate to a block of it.
// A line whose request text imitates rule fields names no rule: it still
// counts, as an unknown rule, so the imitation cannot dodge escalation.
if ip != "" && (ruleID != "" || !ruleKnown) && (isBlock || conf == modsecConfHigh || conf == modsecConfUnknown) &&
!checks.IsTrustedProxy(ip, cfg) {
hits, win := modsecEscalationParams(cfg)
lowConfHits := modsecLowConfEscalationHits(cfg)
rearmAfter := modsecEscalationRearmInterval(cfg)
outcome := recordModSecEventWithRearm(ip, now, ruleNum, conf, isBlock, hits, lowConfHits, win, rearmAfter)
switch {
case outcome.escalate:
check := "modsec_block_escalation"
label := "ModSecurity"
if isCSM {
check = "modsec_csm_block_escalation"
label = "CSM rule"
}
results = append(results, alert.Finding{
Severity: alert.Critical,
Check: check,
Message: modsecEscalationMessage(label, ip, hits, win, outcome.singleHit),
Details: f.Details,
SourceIP: ip,
// modsec.ErrorLogClientIP: the server's own client field.
Attribution: admission.AttributionHTTPPeer,
Domain: f.Domain,
Claims: modsecRequestNameClaims(f.Domain),
})
case outcome.lowConfBurst:
results = append(results, alert.Finding{
Severity: alert.Warning,
Check: "modsec_low_confidence_burst",
Message: fmt.Sprintf("ModSecurity low-confidence burst: %d+ policy/anomaly denies from %s within %v (no attack signature; not auto-blocked)",
hits, ip, win),
Details: f.Details,
SourceIP: ip,
Domain: f.Domain,
})
}
if outcome.classifierGap && ruleID != "" {
results = append(results, alert.Finding{
Severity: alert.Warning,
Check: "modsec_classifier_gap",
Message: fmt.Sprintf("ModSecurity rule %s from %s is unclassified (escalation-eligible; add it to the confidence table if it is a known policy/anomaly rule)",
ruleID, ip),
Details: f.Details,
SourceIP: ip,
Domain: f.Domain,
})
}
}
// --- Step 2: Dedup ---
dedupKey := ip + ":" + ruleID
if prev, loaded := modsecDedup.Load(dedupKey); loaded {
if now.Sub(prev.(time.Time)) < modsecDedupTTL {
// Suppress the base finding but still return any escalation.
if len(results) > 0 {
return results
}
return nil
}
}
modsecDedup.Store(dedupKey, now)
results = append(results, f)
return results
}
// recordModSecEvent records one ModSecurity event for an IP's sliding window
// and decides what it should produce. Confidence-gated:
//
// - escalate (Critical auto-block) fires when total blocking denies reach
// hits AND the window has high-confidence or unknown evidence; OR when
// low-confidence-only denies reach the lowConfHits backstop; OR when an
// exploit-specific rule in modsecSingleHitRules denies a request.
// - lowConfBurst (non-actioned visibility) fires when the hit count is reached
// with only low-confidence evidence and the backstop is not yet met.
// - classifierGap (non-actioned visibility) fires once per unknown rule ID
// host-wide per modsecClassifierGapInterval so new vendor packs are noticed
// instead of silently taking the low-confidence path.
//
// Repeating one high-confidence rule still escalates: diversity is never
// required when high-confidence evidence is present. hits/lowConfHits/window are
// operator knobs; callers pull defaults via modsecEscalationParams.
func recordModSecEvent(ip string, now time.Time, rule int, conf modsecConfidence, isBlock bool, hits, lowConfHits int, window time.Duration) modsecEscalationOutcome {
return recordModSecEventWithRearm(ip, now, rule, conf, isBlock, hits, lowConfHits, window, modsecDefaultEscalationRearmInterval)
}
func recordModSecEventWithRearm(ip string, now time.Time, rule int, conf modsecConfidence, isBlock bool, hits, lowConfHits int, window, rearmAfter time.Duration) modsecEscalationOutcome {
excluded, err := modsecNoEscalateRules()
// The optional seed must not stop the daemon, but unreadable operator
// exclusions cannot authorize a firewall ban. HTTP deny reporting continues.
if err != nil || excluded[rule] {
return modsecEscalationOutcome{}
}
val, _ := modsecBlockCount.LoadOrStore(ip, &modsecIPCounter{})
ctr := val.(*modsecIPCounter)
ctr.mu.Lock()
defer ctr.mu.Unlock()
// Prune expired and operator-excluded evidence. Re-arm latches from the
// pre-append window; otherwise a fresh event that takes the IP from below the
// trigger back to the trigger would keep the stale latch and never re-emit.
ctr.events = pruneModSecEvents(ctr.events, now.Add(-window), excluded)
if len(ctr.events) == 0 {
ctr.escalated = false
ctr.lastEscalated = time.Time{}
ctr.lowBurstEmitted = false
} else {
before := summarizeModSecEvents(ctr.events)
normalBefore, backstopBefore := modsecTriggerStates(before, hits, lowConfHits)
if !normalBefore && !backstopBefore {
ctr.escalated = false
ctr.lastEscalated = time.Time{}
}
}
ctr.events = append(ctr.events, modsecDenyEvent{t: now, rule: rule, conf: conf, isBlock: isBlock})
// Aggregate the current window.
summary := summarizeModSecEvents(ctr.events)
var out modsecEscalationOutcome
if conf == modsecConfUnknown {
out.classifierGap = claimModSecClassifierGap(rule, now)
}
normalFire, backstopFire := modsecTriggerStates(summary, hits, lowConfHits)
if ctr.escalated && !modsecEscalationRearmDue(ctr.lastEscalated, now, rearmAfter) {
return out
}
if normalFire || backstopFire {
ctr.escalated = true
ctr.lastEscalated = now
out.escalate = true
out.singleHit = summary.singleHit && summary.totalBlock < hits
return out
}
// Low-confidence-only burst at the normal bar: visibility, never a ban.
if summary.totalBlock >= hits && !summary.highEvidence && !summary.unknownEvidence && !ctr.lowBurstEmitted {
ctr.lowBurstEmitted = true
out.lowConfBurst = true
}
return out
}
func modsecNoEscalateRules() (map[int]bool, error) {
if db := store.Global(); db != nil {
return db.GetModSecNoEscalateRules()
}
return nil, nil
}
// Exclusions apply to retained evidence too, or another rule's event could
// renew a ban based solely on an exploit deny the operator has since excluded.
func pruneModSecEvents(events []modsecDenyEvent, cutoff time.Time, excluded map[int]bool) []modsecDenyEvent {
kept := events[:0]
for _, e := range events {
if !e.t.Before(cutoff) && !excluded[e.rule] {
kept = append(kept, e)
}
}
return kept
}
func summarizeModSecEvents(events []modsecDenyEvent) modsecWindowSummary {
var summary modsecWindowSummary
for _, e := range events {
if e.conf == modsecConfHigh {
summary.highEvidence = true
}
if e.conf == modsecConfUnknown {
summary.unknownEvidence = true
}
if !e.isBlock {
continue
}
summary.totalBlock++
if modsecSingleHitRules[e.rule] {
summary.singleHit = true
}
if e.conf == modsecConfLow {
summary.lowConfBlock++
}
}
return summary
}
func modsecTriggerStates(summary modsecWindowSummary, hits, lowConfHits int) (normalFire, backstopFire bool) {
normalFire = summary.singleHit || (summary.totalBlock >= hits && (summary.highEvidence || summary.unknownEvidence))
backstopFire = !summary.highEvidence && !summary.unknownEvidence && lowConfHits > 0 && summary.lowConfBlock >= lowConfHits
return normalFire, backstopFire
}
func modsecEscalationMessage(label, ip string, hits int, win time.Duration, singleHit bool) string {
if singleHit {
return fmt.Sprintf("%s escalation: exploit-specific deny from %s", label, ip)
}
return fmt.Sprintf("%s escalation: %d+ denies from %s within %v", label, hits, ip, win)
}
func modsecEscalationRearmDue(lastEscalated, now time.Time, rearmAfter time.Duration) bool {
if lastEscalated.IsZero() || rearmAfter <= 0 {
return false
}
return !now.Before(lastEscalated.Add(rearmAfter))
}
// StartModSecEviction starts a background goroutine that prunes expired dedup
// and counter entries every modsecEvictInterval to prevent unbounded memory
// growth. It returns when stopCh is closed. cfgFn supplies the live
// thresholds at each tick so SIGHUP edits to the escalation window take
// effect without restarting the evictor.
func StartModSecEviction(stopCh <-chan struct{}, cfgFn func() *config.Config) {
if cfgFn == nil {
cfgFn = func() *config.Config { return nil }
}
obs.Go("modsec-eviction", func() {
ticker := time.NewTicker(modsecEvictInterval)
defer ticker.Stop()
for {
select {
case <-stopCh:
return
case now := <-ticker.C:
cfg := cfgFn()
hits, win := modsecEscalationParams(cfg)
lowConfHits := modsecLowConfEscalationHits(cfg)
evictModSecStateWithLowConf(now, hits, lowConfHits, win)
}
}
})
}
// discoverModSecLogPath returns the path to the web server error log that
// CSM should tail for ModSecurity denies. Config override wins, then the
// first candidate from platform detection that actually exists.
func discoverModSecLogPath(cfg *config.Config) string {
if cfg.ModSecErrorLog != "" {
return cfg.ModSecErrorLog
}
return firstExistingPath(platform.Detect().ErrorLogPaths)
}
// firstExistingPath returns the first path in the list that exists on disk,
// or "" if none do. Pure function so tests can exercise it directly.
func firstExistingPath(candidates []string) string {
for _, p := range candidates {
if _, err := os.Stat(p); err == nil {
return p
}
}
return ""
}
func evictModSecState(now time.Time, hits int, window time.Duration) {
evictModSecStateWithLowConf(now, hits, modsecDefaultLowConfEscalationHits, window)
}
// evictModSecStateWithLowConf prunes expired entries from modsecDedup and
// modsecBlockCount. hits, lowConfHits, and window mirror the live thresholds so
// latch reset matches what recordModSecEvent would compute on the next event.
func evictModSecStateWithLowConf(now time.Time, hits, lowConfHits int, window time.Duration) {
// Prune dedup entries older than modsecDedupTTL.
modsecDedup.Range(func(key, value any) bool {
if now.Sub(value.(time.Time)) >= modsecDedupTTL {
modsecDedup.Delete(key)
}
return true
})
// Prune counter entries.
cutoff := now.Add(-window)
// Even if exclusions are unreadable, expired evidence still needs eviction.
// Recording and escalation wait for a successful read in recordModSecEvent.
excluded, _ := modsecNoEscalateRules()
modsecBlockCount.Range(func(key, value any) bool {
ctr := value.(*modsecIPCounter)
ctr.mu.Lock()
kept := pruneModSecEvents(ctr.events, cutoff, excluded)
ctr.events = kept
empty := len(kept) == 0
summary := summarizeModSecEvents(kept)
normalFire, backstopFire := modsecTriggerStates(summary, hits, lowConfHits)
if !normalFire && !backstopFire {
ctr.escalated = false
ctr.lastEscalated = time.Time{}
}
if empty {
ctr.lastEscalated = time.Time{}
ctr.lowBurstEmitted = false
}
ctr.mu.Unlock()
if empty {
modsecBlockCount.Delete(key)
}
return true
})
modsecGapMu.Lock()
for rule, reported := range modsecGapReported {
if now.Sub(reported) >= modsecClassifierGapInterval {
delete(modsecGapReported, rule)
}
}
modsecGapMu.Unlock()
}
// claimModSecClassifierGap reports whether this hit of an unknown rule should
// raise the classifier-gap finding: the first hit host-wide, then again once
// per modsecClassifierGapInterval while the rule stays unclassified.
func claimModSecClassifierGap(rule int, now time.Time) bool {
modsecGapMu.Lock()
defer modsecGapMu.Unlock()
if reported, ok := modsecGapReported[rule]; ok && now.Sub(reported) < modsecClassifierGapInterval {
return false
}
modsecGapReported[rule] = now
return true
}
// modsecRequestNameClaims names the host the client asked for. The client
// chose it, so it is a request name and never verifies an owner.
func modsecRequestNameClaims(host string) []admission.Claim {
if host == "" {
return nil
}
return []admission.Claim{{Kind: admission.ClaimRequestName, Value: host}}
}
package daemon
import (
"context"
"maps"
"strings"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/challenge"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall/rollback"
"github.com/pidginhost/csm/internal/health"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/processhandle"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/store"
"github.com/pidginhost/csm/internal/updatecheck"
)
// QueueStatuses reports protection work independently of alert delivery.
func (d *Daemon) QueueStatuses() map[string]queuehealth.Status {
return d.queueStatuses(time.Now())
}
func (d *Daemon) queueStatuses(now time.Time) map[string]queuehealth.Status {
out := d.registeredQueueStatuses(now)
for name, status := range checks.AutoBlockQueueStatuses(now) {
out["auto_block."+name] = status
}
if incidentCorrelator != nil {
for name, status := range incidentCorrelator.QueueStatuses(now) {
out["incident."+name] = status
}
}
out["actionlog.writes"] = actionlog.QueueStatus(now)
out["phpanel.spool"] = alert.PhpanelQueueStatus(now)
out["checks.executions"] = checks.CheckExecutionQueueStatus(now)
out["checks.plugin_inventory"] = checks.PluginInventoryQueueStatus(now)
out["checks.wordpress_core"] = checks.WPCoreQueueStatus(now)
out["checks.reputation_queries"] = checks.ReputationQueueStatus(now)
out["php_shield.eval_sites"] = phpShieldEvalSiteQueueStatus(now)
for name, status := range checks.FileIndexQueueStatuses(now) {
out["checks.file_index."+name] = status
}
out["checks.dispatch"] = checks.CheckDispatchQueueStatus(now)
for name, status := range rdnsCache().QueueStatuses(now) {
out["smtp_rdns."+name] = status
}
for name, status := range checks.EmailPasswordQueueStatuses(now) {
out["email_password."+name] = status
}
if enr := processCtxPublished.Load(); enr != nil {
for name, state := range enr.QueueStatuses(now) {
out["processctx."+name] = state
}
}
if d.alertQueue != nil {
out["findings.ingest"] = d.alertQueue.Snapshot(now)
}
if fm := d.getFileMonitor(); fm != nil {
maps.Copy(out, fm.queueStatuses(now))
}
if sw := d.getSpoolWatcher(); sw != nil {
maps.Copy(out, sw.queueStatuses(now))
}
return out
}
// Hostname implements health.Provider.
func (d *Daemon) Hostname() string {
cfg := d.currentCfg()
if cfg == nil {
return ""
}
return cfg.Hostname
}
// StartedAt implements health.Provider.
func (d *Daemon) StartedAt() time.Time {
return d.startTime
}
// LatestScan implements health.Provider.
func (d *Daemon) LatestScan() time.Time {
if d.store == nil {
return time.Time{}
}
return d.store.LatestScanTime()
}
// BaselineAt implements health.Provider. Returns the persisted first-start
// timestamp recorded by EnsureBaseline on the daemon's first successful
// boot against this state directory. Reinstalls and upgrades preserve
// the original value.
func (d *Daemon) BaselineAt() time.Time {
if d.store == nil {
return time.Time{}
}
return d.store.BaselineAt()
}
// StoreHealthy implements health.Provider.
func (d *Daemon) StoreHealthy() bool {
s := store.Global()
if s == nil {
return false
}
return s.IsHealthy()
}
// StoreSizeMB implements health.Provider.
func (d *Daemon) StoreSizeMB() float64 {
s := store.Global()
if s == nil {
return 0
}
return float64(s.SizeBytes()) / (1024 * 1024)
}
// SeverityCounts implements health.Provider.
// Buckets the latest findings by severity name.
func (d *Daemon) SeverityCounts() map[string]int {
out := map[string]int{"critical": 0, "high": 0, "warning": 0}
if d.store == nil {
return out
}
for _, f := range d.store.LatestFindings() {
switch f.Severity {
case alert.Critical:
out["critical"]++
case alert.High:
out["high"]++
default:
out["warning"]++
}
}
return out
}
// BlocklistSize implements health.Provider.
//
// The firewall engine is the authoritative source: production code never
// writes the parallel bbolt `fw:blocked` bucket the previous implementation
// read, so /api/v1/status reported a stale count (a production host showed 25
// against 909 in the real engine state). Engine.BlockedCount() reads the
// same state file Status() and `csm firewall status` use, with expired
// entries pruned.
func (d *Daemon) BlocklistSize() int {
if d.fwEngine == nil {
return 0
}
return d.fwEngine.BlockedCount()
}
// IncidentsOpen implements health.Provider. Returns the count of
// open + contained incidents in the correlator. Falls back to 0 if
// the correlator has not been constructed yet (e.g. very early
// startup or shutdown).
func (d *Daemon) IncidentsOpen() int {
if incidentCorrelator == nil {
return 0
}
return incidentCorrelator.OpenCount()
}
// BPFEnforcementActive implements health.Provider. Reports the
// configured enforcement state. Phase 4 of the BPF Incident Response
// Roadmap. Reads the live config via Daemon.currentCfg(); falls back
// to false if config is nil (very early startup).
func (d *Daemon) BPFEnforcementActive() bool {
cfg := d.currentCfg()
if cfg == nil {
return false
}
return cfg.BPFEnforcement.Enabled &&
cfg.BPFEnforcement.DirectSMTPEgress &&
bpf.ActiveKind("connection_tracker") == bpf.BackendBPF
}
// HistoryCount implements health.Provider.
func (d *Daemon) HistoryCount() int {
s := store.Global()
if s == nil {
return 0
}
return s.HistoryCount()
}
// ConfigHash implements health.Provider.
func (d *Daemon) ConfigHash() string {
cfg := d.currentCfg()
if cfg == nil {
return ""
}
return cfg.Integrity.ConfigHash
}
// BinaryHash implements health.Provider. The binary is read again only
// when the file changes; returns empty on error.
func (d *Daemon) BinaryHash() string {
if d.binaryPath == "" {
return ""
}
return d.binaryHash.get(d.binaryPath)
}
// CorrelationAttribution implements health.Provider. Nil until the first
// active-set merge, so a fresh daemon does not claim a clean state it has
// not yet observed.
func (d *Daemon) CorrelationAttribution() *health.CorrelationAttribution {
h := checks.AttributionHealth()
if h.ActiveSetUpdates == 0 {
return nil
}
return &health.CorrelationAttribution{
Current: h.Current,
Cumulative: h.Cumulative,
ActiveSetUpdates: h.ActiveSetUpdates,
Since: h.Since,
}
}
// DryRunBlocksCount implements health.Provider.
// Returns the number of firewall blocks that were intercepted by dry_run.
func (d *Daemon) DryRunBlocksCount() int {
s := store.Global()
if s == nil {
return 0
}
return s.DryRunBlocksCount()
}
// AutomationStatus implements health.Provider.
func (d *Daemon) AutomationStatus() health.AutomationStatus {
cfg := d.currentCfg()
out := health.AutomationStatus{
DryRunBlocks: d.DryRunBlocksCount(),
LastAction: d.lastAutomationAction(),
}
if cfg != nil {
out.AutoResponseEnabled = cfg.AutoResponse.Enabled
out.AutoResponseBlockIPs = cfg.AutoResponse.BlockIPs
out.AutoResponseDryRun = cfg.AutoResponseDryRunEnabled()
// Configured termination that the kernel cannot perform safely is
// inoperative, not merely unused: report the capability either way.
out.ProcessKillEnabled = cfg.AutoResponse.Enabled && cfg.AutoResponse.KillProcesses
if err := processhandle.Available(); err != nil {
out.ProcessSignalError = err.Error()
} else {
out.ProcessSignalSupported = true
}
out.ChallengeEnabled = cfg.Challenge.Enabled
out.ChallengePortGateEnabled = cfg.Challenge.PortGate.Enabled
}
if d.ipList != nil {
out.ChallengePending = d.ipList.Count()
}
out.ChallengeEscalated = challengeEscalatedCount()
out.ChallengePassed = challenge.PassedCount()
out.ChallengePortGateActive = d.challengeGate != nil
if cfg != nil {
out.FirewallConfirmation = firewallConfirmationStatus(cfg.StatePath)
}
if cfg != nil && cfg.Firewall != nil {
out.FirewallEnabled = cfg.Firewall.Enabled
if out.FirewallEnabled {
out.FirewallStartupError = d.fwStartupError
out.FirewallStateSetAside = d.firewallStateSetAside()
}
}
// FirewallManaged is true only when a live engine is wired. Reporting it
// (alongside FirewallEnabled) lets monitoring detect "enabled but not
// managed" -- the engine-failed-to-apply condition that previously left the
// firewall silently unmanaged.
if engine := d.fwEngine; engine != nil {
out.FirewallManaged = true
if err := engine.BlockActionsUnproven(); err != nil {
out.FirewallLedgerError = err.Error()
}
rc := engine.RuleCounts()
out.FirewallBlockedIPs = rc.Blocked
out.FirewallBlockedSubnets = rc.Subnets
out.FirewallLocalPortLocks = engine.LockedLocalPorts()
}
if mgr := rollback.Global(); mgr != nil {
st := mgr.Status()
out.FirewallRollbackPending = st.Pending
out.FirewallRollbackSecondsRemain = st.SecondsRemaining
}
return out
}
func (d *Daemon) lastAutomationAction() *health.AutomationAction {
if d.store == nil {
return nil
}
d.automationActionMu.Lock()
defer d.automationActionMu.Unlock()
if !d.automationActionCached.IsZero() && time.Since(d.automationActionCached) < lastAutomationActionTTL {
return d.automationActionCache
}
d.automationActionCache = d.computeLastAutomationAction()
d.automationActionCached = time.Now()
return d.automationActionCache
}
func (d *Daemon) computeLastAutomationAction() *health.AutomationAction {
var (
best alert.Finding
ok bool
)
consider := func(findings []alert.Finding) {
for _, f := range findings {
if !isAutomationActionCheck(f.Check) {
continue
}
if !ok || f.Timestamp.After(best.Timestamp) {
best = f
ok = true
}
}
}
consider(d.store.LatestFindings())
if history, _ := d.store.ReadHistory(100, 0); len(history) > 0 {
consider(history)
}
if !ok {
return nil
}
return &health.AutomationAction{
Check: best.Check,
Message: best.Message,
Timestamp: best.Timestamp,
}
}
func isAutomationActionCheck(check string) bool {
switch check {
case "auto_block", "auto_response", "challenge_route":
return true
}
return strings.HasPrefix(check, "email_php_relay_action_")
}
// Mode implements health.Provider. It reports the operator's posture so
// status, the API and doctor all show whether this host is allowed to change
// its own state.
func (d *Daemon) Mode() string {
cfg := config.Active()
if cfg == nil {
cfg = d.cfg
}
if cfg == nil {
return config.ModeEnforce
}
if cfg.ObserveMode() {
return config.ModeObserve
}
return config.ModeEnforce
}
// UpdateInfo implements health.Provider. Returns the latest cached
// release-check result, or zero value if the checker was disabled
// or has not completed its first poll yet.
func (d *Daemon) UpdateInfo() health.UpdateInfo {
if d.updateChecker == nil {
return health.UpdateInfo{}
}
info := d.updateChecker.Latest()
return health.UpdateInfo{
LatestVersion: info.LatestVersion,
Available: info.Available,
Source: info.Source,
CheckedAt: info.CheckedAt,
Err: info.Err,
}
}
// startUpdateChecker wires the updatecheck.Checker. No-op when
// updates.check_enabled is false (operator opt-out, e.g. air-gapped
// deployments) or when the running binary is "dev" (still useful but
// the banner will always show).
func (d *Daemon) startUpdateChecker() {
cfg := d.cfg
if cfg == nil || !cfg.UpdatesCheckEnabled() {
return
}
interval := cfg.UpdatesInterval()
pkgProbe := selectPackageProbe(cfg.UpdatesPackageName())
d.updateChecker = updatecheck.New(updatecheck.Options{
CurrentVersion: d.version,
Interval: interval,
GitHubAPIURL: cfg.Updates.GitHubAPIURL,
PackageProbe: pkgProbe,
LogErr: func(source string, err error) {
csmlog.Debug("update check probe failed", "source", source, "err", err)
},
})
d.wg.Add(1)
obs.Go("update-checker", func() {
defer d.wg.Done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() { <-d.stopCh; cancel() }()
d.updateChecker.Run(ctx)
})
}
// selectPackageProbe returns an apt or dnf probe based on the
// detected OS family, or nil when the host runs neither (binary
// installs, source builds, etc.).
func selectPackageProbe(packageName string) updatecheck.PackageProbe {
info := platform.Detect()
switch {
case info.IsDebianFamily():
return updatecheck.AptProbe(packageName)
case info.IsRHELFamily():
return updatecheck.DnfProbe(packageName)
default:
return nil
}
}
// compile-time check: Daemon satisfies health.Provider.
var _ health.Provider = (*Daemon)(nil)
package daemon
import (
"github.com/pidginhost/csm/internal/health"
"github.com/pidginhost/csm/internal/store"
)
// WordPressVerification supplies the same persisted coverage to the CLI and API.
func (d *Daemon) WordPressVerification() map[string]health.WPVerificationCounts {
db := store.Global()
if db == nil {
return nil
}
result := make(map[string]health.WPVerificationCounts)
for _, kind := range []string{"core", "plugins"} {
rows, err := db.WPVerification(kind)
if err != nil {
result[kind] = health.WPVerificationCounts{Error: "verification history unavailable"}
continue
}
if len(rows) == 0 {
continue
}
var counts health.WPVerificationCounts
for _, row := range rows {
switch row.State {
case "verified":
counts.Verified++
case "modified":
counts.Modified++
case "unverified":
counts.Unverified++
case "not_wordpress":
counts.NotWordPress++
default:
counts.Unknown++
}
if row.AttemptAt.After(counts.LastAttempt) {
counts.LastAttempt = row.AttemptAt
}
}
result[kind] = counts
}
return result
}
package daemon
import (
"fmt"
"os"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/integrity"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/store"
)
// Reload outcome labels for csm_config_reloads_total. Keep these in
// sync with docs/src/metrics.md.
const (
reloadResultSuccess = "success"
reloadResultError = "error"
reloadResultRestartRequired = "restart_required"
reloadResultNoop = "noop"
)
var (
reloadMetric *metrics.CounterVec
reloadMetricOnce sync.Once
)
// recordReloadResult bumps csm_config_reloads_total by one for the
// given outcome label, registering the metric on first use.
func recordReloadResult(result string) {
reloadMetricOnce.Do(func() {
reloadMetric = metrics.NewCounterVec(
"csm_config_reloads_total",
"SIGHUP config reload attempts, by outcome. result=success when safe fields were swapped in place; restart_required when the edit touched a field that needs a full restart (live settings unchanged); error on YAML parse, validation, re-hash, or apply failure (live settings unchanged); noop when the edit was semantically identical to the running config.",
[]string{"result"},
)
metrics.MustRegister("csm_config_reloads_total", reloadMetric)
})
reloadMetric.With(result).Inc()
}
// reloadConfig re-reads the on-disk csm.yaml plus configured drop-ins,
// validates it, diffs against the current live config, and - if every
// change is marked safe for live reload - installs the new config via
// config.SetActive. Only the main config file is re-signed; drop-ins are
// never written back into csm.yaml.
//
// Failure modes leave the live settings untouched. If signing succeeds but
// applying settings fails, the live integrity metadata follows the signed file.
//
// - YAML parse error: Critical `config_reload_error` finding.
// - Validation error: Critical `config_reload_error` finding.
// - Restart-required fields changed: Warning
// `config_reload_restart_required` finding, listing the offending
// field names.
// - Re-signing failure: Critical `config_reload_error` finding.
//
// ROADMAP item 7.
func (d *Daemon) reloadConfig() {
// Every reload attempt re-reads the web server, including one that
// changes no setting or fails: an operator who changed the web server
// reloads CSM to have it read again.
d.refreshHTTPRewrite()
// Settings saves use the same lock through signing and publication. A
// reload must not sign their bytes or overwrite their newer live snapshot.
configMu := integrity.ConfigWriteMutex()
configMu.Lock()
defer configMu.Unlock()
oldCfg := d.activeOrStartupCfg()
cfgPath := oldCfg.ConfigFile
fmt.Fprintf(os.Stderr, "[%s] SIGHUP: reloading config from %s\n", ts(), cfgPath)
newCfg, err := config.LoadWithDir(cfgPath, oldCfg.ConfigDir)
if err != nil {
recordReloadResult(reloadResultError)
d.emitReloadFinding(alert.Critical, "config_reload_error",
fmt.Sprintf("SIGHUP reload: parse failed (%v); keeping old config", err))
return
}
for _, r := range config.Validate(newCfg) {
if r.Level == "error" {
recordReloadResult(reloadResultError)
d.emitReloadFinding(alert.Critical, "config_reload_error",
fmt.Sprintf("SIGHUP reload: validation error on %q: %s; keeping old config",
r.Field, r.Message))
return
}
}
changes := config.Diff(oldCfg, newCfg)
if len(changes) == 0 {
// An edit that changes no setting (a comment, a retired key) still
// moves the stored hash. Adopt it, or the periodic integrity check
// reports a tamper.
if err := d.signAndSaveReloadedConfig(oldCfg, newCfg); err != nil {
recordReloadResult(reloadResultError)
d.emitReloadFinding(alert.Critical, "config_reload_error",
fmt.Sprintf("SIGHUP reload: re-hashing config failed: %v; live config unchanged", err))
return
}
if newCfg.Integrity != oldCfg.Integrity {
publishResyncedIntegrity(oldCfg, newCfg, cfgPath)
}
recordReloadResult(reloadResultNoop)
fmt.Fprintf(os.Stderr, "[%s] SIGHUP: no config changes detected\n", ts())
// An operator who fixed a host condition that blocked antispam, or
// started Postfix after CSM, reloads to have it checked again; do
// not wait for the interval.
d.reconcileAntispam()
return
}
if config.RestartRequired(changes) {
var offenders []string
for _, c := range changes {
if c.Tag != config.TagSafe {
offenders = append(offenders, c.Field)
}
}
// The edit passed Validate, so the file on disk is
// loadable. Re-sign integrity.config_hash to match the
// edited content -- otherwise the next daemon restart
// (systemd, manual, crash recovery) trips the startup
// integrity check and crash-loops. Update the live cfg's
// stored ConfigHash in lock-step so the periodic
// integrity.Verify(currentCfg()) does not see a disk /
// memory divergence and fire spurious tamper alerts.
//
// Any error re-signing degrades to "live config unchanged,
// file unchanged, operator must rehash manually"; we still
// emit the warning so they know to act.
if err := d.signAndSaveReloadedConfig(oldCfg, newCfg); err == nil {
publishResyncedIntegrity(oldCfg, newCfg, cfgPath)
} else {
fmt.Fprintf(os.Stderr, "[%s] config_reload_restart_required: re-hash failed (%v); file and live hash remain mismatched until operator runs `csm rehash`\n",
ts(), err)
}
recordReloadResult(reloadResultRestartRequired)
d.emitReloadFinding(alert.Warning, "config_reload_restart_required",
fmt.Sprintf("SIGHUP reload: restart-required fields changed: %v; live config unchanged, main config re-hashed if needed for next restart",
offenders))
return
}
accountExtractor, extractorErr := NewAccountExtractor(newCfg.Thresholds.MailBruteAccountKey)
if extractorErr != nil {
recordReloadResult(reloadResultError)
d.emitReloadFinding(alert.Critical, "config_reload_error",
fmt.Sprintf("SIGHUP reload: account extractor update failed: %v; live config unchanged", extractorErr))
return
}
if err := d.signAndSaveReloadedConfig(oldCfg, newCfg); err != nil {
recordReloadResult(reloadResultError)
d.emitReloadFinding(alert.Critical, "config_reload_error",
fmt.Sprintf("SIGHUP reload: re-hashing config failed: %v; live config unchanged", err))
return
}
newCfg.ConfigFile = cfgPath
newCfg.ConfigDir = oldCfg.ConfigDir
phpanelEnabled := newCfg.Alerts.Webhook.Enabled && newCfg.Alerts.Webhook.Type == "phpanel"
if phpanelEnabled {
if err := alert.ConfigurePhpanelQueue(newCfg); err != nil {
publishResyncedIntegrity(oldCfg, newCfg, cfgPath)
recordReloadResult(reloadResultError)
d.emitReloadFinding(alert.Critical, "config_reload_error",
fmt.Sprintf("SIGHUP reload: phpanel queue update failed: %v; live config unchanged", err))
return
}
}
SetAccountExtractor(accountExtractor)
publishActiveConfig(newCfg, "SIGHUP")
if !phpanelEnabled {
_ = alert.ConfigurePhpanelQueue(newCfg)
}
recordReloadResult(reloadResultSuccess)
var names []string
for _, c := range changes {
names = append(names, c.Field)
}
fmt.Fprintf(os.Stderr, "[%s] SIGHUP: config reloaded; safe fields updated: %v\n", ts(), names)
d.reconcileHotReloadables()
}
// reconcileHotReloadables pushes the safe-field values of the live config into
// the subsystems that cache them. Both reload paths end here: a SIGHUP after
// the edited file is published, and a web UI save after the handler has
// published the edited config.
func (d *Daemon) reconcileHotReloadables() {
// email_protection.antispam is a safe field: health follows the new
// switch at once, and a reconcile pass applies the new settings. It goes
// first, so a forward-guard transaction rebuilding Exim cannot delay it.
d.reconcileAntispam()
// A forward-guard config change is a safe (hot-reload) field; re-reconcile
// so enabling/disabling or toggling enforce takes effect without a restart.
// Its own loop runs the worker, so the reload does not wait for it.
d.wakeForwardGuard()
// reputation.verified_bots is a safe field too; push the new list into the
// live verifier and re-stamp the PTR cache so changes take effect at once.
d.reconcileVerifiedBots()
// thresholds is a safe block; push the new SMTP/mail brute-force thresholds
// into the live trackers so a reload takes effect without a restart.
d.reconcileBruteThresholds()
// reputation.whitelist is a safe field; replace the threat database's
// configured whitelist so additions and removals apply at once.
d.reconcileReputationWhitelist()
// auto_response.max_blocks_per_hour sets the admission ceiling.
d.reloadAdmission()
d.refreshTrustedProxyFinding()
}
// applySavedConfig applies a config the web UI has already saved and
// published. The file is signed and the live config swapped by then, so a
// failing step is logged rather than refused: the operator sees the saved
// values and the log explains what did not reach a subsystem.
func (d *Daemon) applySavedConfig() {
cfg := d.activeOrStartupCfg()
if err := installAccountExtractorFromConfig(cfg); err != nil {
csmlog.Error("settings save: account extractor update failed", "err", err)
}
if err := alert.ConfigurePhpanelQueue(cfg); err != nil {
csmlog.Error("settings save: phpanel queue update failed", "err", err)
}
d.reconcileHotReloadables()
}
// activeOrStartupCfg returns the current live config, falling back
// to d.cfg (the startup snapshot) if SetActive has not yet been
// called. Reload paths use this so the first reload diffs against
// the startup config, and every subsequent reload diffs against
// whatever the last successful reload installed.
//
// Also used by the tier-run hot paths (via currentCfg below) so a
// SIGHUP-driven threshold change reaches the next tick without a
// restart.
func (d *Daemon) activeOrStartupCfg() *config.Config {
if c := config.Active(); c != nil {
return c
}
return d.cfg
}
// currentCfg is the per-tick config accessor for hot paths. See
// ROADMAP item 7 for the threshold-tuning motivation.
func (d *Daemon) currentCfg() *config.Config {
return d.activeOrStartupCfg()
}
func publishActiveConfig(cfg *config.Config, source string) {
config.SetActive(cfg)
purgeDryRunBlocksIfAutoResponseLive(cfg, source)
}
func purgeDryRunBlocksIfAutoResponseLive(cfg *config.Config, source string) {
// Startup publishes the config before the owner starts. Reserve its
// examples now so even that first purge uses bounded owner maintenance.
if cfg == nil || cfg.AutoResponseDryRunEnabled() || ownerKeepsDryRunExamples.Load() || admissionUsesFirewall(cfg) {
return
}
if sdb := store.Global(); sdb != nil {
if removed := sdb.PurgeAllDryRunBlocks(); removed > 0 {
fmt.Fprintf(os.Stderr, "[%s] %s: purged %d dry_run_blocks records (auto-response live)\n", ts(), source, removed)
}
}
}
// publishResyncedIntegrity publishes the live config unchanged except for
// the integrity hashes the reload signed, for a reload that applies no
// setting.
func publishResyncedIntegrity(oldCfg, newCfg *config.Config, cfgPath string) {
resynced := *oldCfg
resynced.Integrity = newCfg.Integrity
// confd_hash was computed under the edited exemption list, so
// the live copy must verify under that same list.
resynced.ConfD = newCfg.ConfD
resynced.ConfigFile = cfgPath
resynced.ConfigDir = oldCfg.ConfigDir
publishActiveConfig(&resynced, "SIGHUP")
}
// signAndSaveReloadedConfig re-computes integrity.config_hash for the main
// config file (and integrity.confd_hash for the conf.d fragments) when either
// changed across a SIGHUP. Drop-in fragment CONTENT is intentionally not
// written back into csm.yaml; only their digest is folded into confd_hash so
// a later Verify still passes. The binary hash is preserved from the prior
// live config; a SIGHUP reload cannot upgrade the binary, so it must not drift.
func (d *Daemon) signAndSaveReloadedConfig(oldCfg, newCfg *config.Config) error {
currentHash, err := integrity.HashConfigStable(oldCfg.ConfigFile)
if err != nil {
return err
}
currentConfd, err := integrity.HashConfDir(oldCfg.ConfigDir, newCfg.ConfD.IntegrityExempt)
if err != nil {
return err
}
if currentHash == oldCfg.Integrity.ConfigHash && currentConfd == oldCfg.Integrity.ConfdHash {
newCfg.Integrity = oldCfg.Integrity
return nil
}
// An operator may already have run rehash. Adopt verified hashes without
// requiring another write, while retaining the running binary's identity.
if currentHash == newCfg.Integrity.ConfigHash && currentConfd == newCfg.Integrity.ConfdHash && newCfg.Integrity.BinaryHash == oldCfg.Integrity.BinaryHash {
return nil
}
configHash, confdHash, err := integrity.SignConfigFilePreserving(oldCfg.ConfigFile, oldCfg.ConfigDir, oldCfg.Integrity.BinaryHash)
if err != nil {
return err
}
newCfg.Integrity.BinaryHash = oldCfg.Integrity.BinaryHash
newCfg.Integrity.ConfigHash = configHash
newCfg.Integrity.ConfdHash = confdHash
return nil
}
// emitReloadFinding logs to stderr and pushes a Finding into the
// daemon's alert channel. Non-blocking on channel saturation; the
// daemon's existing drop counter tracks those.
func (d *Daemon) emitReloadFinding(sev alert.Severity, check, msg string) {
fmt.Fprintf(os.Stderr, "[%s] %s: %s\n", ts(), check, msg)
finding := alert.Finding{
Severity: sev,
Check: check,
Message: msg,
Timestamp: time.Now(),
}
if !alert.TryEnqueue(d.alertCh, finding) {
atomic.AddInt64(&d.droppedAlerts, 1)
}
}
package daemon
import (
"sync"
"github.com/pidginhost/csm/internal/checks"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/platform"
)
// detectHostHTTPRewrite reads this host's web server configuration.
var detectHostHTTPRewrite = func() checks.HTTPRewrite { return checks.DetectHTTPRewrite(platform.Detect()) }
// rewriteMu orders each read of the web server with its publication, so a
// slower read never overwrites a newer one. The state is process-wide.
var rewriteMu sync.Mutex
// refreshHTTPRewrite publishes whether the web server rewrites the client
// address it logs, and logs a change.
func (d *Daemon) refreshHTTPRewrite() {
rewriteMu.Lock()
defer rewriteMu.Unlock()
r := detectHostHTTPRewrite()
if prev := checks.CurrentHTTPRewrite(); prev != r {
csmlog.Info("web server client address rewriting", "state", r.State.String(), "server", string(r.Server), "directive", r.Directive, "file", r.File, "reason", r.Reason)
}
checks.SetHTTPRewrite(r)
}
package daemon
import (
"regexp"
"github.com/pidginhost/csm/internal/contenttype"
)
// imagePayloadConstructs are the constructs that turn PHP bytes riding inside
// an image container into a working backdoor: a code or command sink, or a
// fetch of remote code to run. Each entry is reported verbatim as the
// finding's evidence, so the operator sees why the file was flagged.
//
// Every entry is a multi-character ASCII identifier. Compressed pixel data is
// random bytes, so a short punctuation shape turns up in ordinary plugin
// artwork by chance and cannot carry a rule: a shell-backtick arm measured
// here fired on a stock plugin layout preview whose pixel data also happened
// to hold a three-byte short-echo tag.
var imagePayloadConstructs = []struct {
name string
re *regexp.Regexp
}{
{"code execution sink", regexp.MustCompile(`(?i)(?:^|[^A-Za-z0-9_>:$])(?:eval|assert|create_function|call_user_func(?:_array)?)\s*\(`)},
{"command execution sink", regexp.MustCompile(`(?i)(?:^|[^A-Za-z0-9_>:$])(?:system|exec|shell_exec|passthru|proc_open|popen|pcntl_exec)\s*\(`)},
{"file inclusion sink", regexp.MustCompile(`(?i)(?:^|[^A-Za-z0-9_>:$])(?:include|require)(?:_once)?\s*(?:\(\s*)?(?:@\s*)?[$'"]`)},
{"file write sink", regexp.MustCompile(`(?i)(?:^|[^A-Za-z0-9_>:$])(?:file_put_contents|fwrite|fputs|move_uploaded_file)\s*\(`)},
{"remote code fetch", regexp.MustCompile(`(?i)(?:^|[^A-Za-z0-9_>:$])(?:curl_init|curl_exec|curl_setopt(?:_array)?|fsockopen|stream_context_create)\s*\(`)},
{"remote code fetch", regexp.MustCompile(`(?i)(?:^|[^A-Za-z0-9_>:$])(?:file_get_contents|fopen|readfile)\s*\(\s*['"]?(?:https?|ftp|php):`)},
{"request-controlled input", regexp.MustCompile(`\$_(?:GET|POST|REQUEST|COOKIE|FILES)\s*\[`)},
}
// phpExecutableContent reports a PHP opening tag accompanied by a construct
// that executes or fetches code, and names the construct.
//
// A PHP opening tag alone is not evidence. Plugin screenshots quote one in
// their description chunks, and three bytes of it turn up in compressed pixel
// data by chance, so the construct beside it is what makes the verdict.
//
// It carries no opinion about the file type, so a caller that already knows a
// path has no business holding PHP can apply it to a tail read, where the
// container magic at offset zero is out of view.
func phpExecutableContent(data []byte) (string, bool) {
if !contenttype.HasPHPOpenTag(data) {
return "", false
}
for _, construct := range imagePayloadConstructs {
if construct.re.Match(data) {
return construct.name, true
}
}
return "", false
}
package daemon
import (
"errors"
"net"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/incident"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/store"
"github.com/pidginhost/csm/internal/threatintel"
)
var (
incidentOnce sync.Once
incidentCorrelator *incident.Correlator
incidentRegistry = metrics.Default
incidentRetentionCancel func()
incidentAutoCloseCancel func()
// incidentSprayBlocker is the firewall hand-off the daemon wires in
// before IncidentCorrelator() is first invoked. The bool reports
// whether nftables was actually mutated; dry-run, already-blocked,
// and verdict-allow outcomes return false so incident timelines do
// not record a live block that never landed.
// nil means "no blocker wired" (early startup or unit tests); the
// singleton then skips wiring OnSprayBlock and the spray detector
// stays detection-only even with BlockAtSeverity set.
incidentSprayBlocker func(ip, reason string, timeout time.Duration, findingID string, root incident.PreparedRoot, entry admission.Entry) (bool, error)
)
// autoResponseBlockExpiry is the operator's configured block duration, the
// first rung of the incident auto-block escalation ladder. An unset or
// unparseable value falls back to a day, matching the firewall default.
func autoResponseBlockExpiry(cfg *config.Config) time.Duration {
if cfg == nil {
return 24 * time.Hour
}
timeout, err := time.ParseDuration(cfg.AutoResponse.BlockExpiry)
if err != nil || timeout <= 0 {
return 24 * time.Hour
}
return timeout
}
// SetIncidentSprayBlocker installs the firewall-side hand-off used by the
// incident auto-block paths. Call once after the firewall engine is built
// and before the first IncidentCorrelator() call.
// Passing nil clears the binding.
func SetIncidentSprayBlocker(fn func(ip, reason string, timeout time.Duration, findingID string, root incident.PreparedRoot, entry admission.Entry) (bool, error)) {
incidentSprayBlocker = fn
}
// incidentAutoCloseInterval is how often the daemon scans Open / Contained
// incidents for staleness. One hour is fast enough that 24h-idle incidents
// close within ~25h worst-case, but slow enough that the per-kind walk
// stays cheap on hosts with thousands of incidents.
const incidentAutoCloseInterval = 1 * time.Hour
// incidentAutoCloseWarmup delays the first sweep just long enough for the
// startup finding burst to settle. Incidents are restored synchronously
// before the loop starts, so a long warm-up only parks a stale backlog as
// "open" after every restart (observed on a frequently-upgraded prod host:
// thousands of >24h incidents sitting open for the full warm-up). Short.
const incidentAutoCloseWarmup = 2 * time.Minute
// incidentAutoCloseDrainDelay is the cadence used when a sweep hit its
// per-sweep cap, so a large backlog drains over a few quick passes instead
// of waiting a full interval between each capped sweep.
const incidentAutoCloseDrainDelay = 30 * time.Second
// incidentAutoCloseMaxPerSweep bounds live closes per sweep so a big
// post-restart backlog does not hold the correlator lock or burst
// thousands of bbolt persists in one tick.
const incidentAutoCloseMaxPerSweep = 1000
// incidentClosedRetention is how long resolved/dismissed incidents are kept
// before compaction prunes them. Named value per project convention; config
// exposure deferred until operators ask.
var incidentClosedRetention = incident.ClosedRetention{
Operator: 30 * 24 * time.Hour,
Auto: 7 * 24 * time.Hour,
}
// incidentOpenThreshold is the number of correlated findings required
// before a finding subject to the threshold opens an incident. Two means an
// isolated probe (a single dictionary-attack guess, one modsec hit
// from a wandering scanner) is treated as a finding only and never
// promoted to an incident on its own; the next correlated event
// inside the merge window does the promotion. Declared as var so
// tests that exercise the correlator wiring (where one finding is
// expected to land in one incident immediately) can pin it to 1
// via resetIncidentForTest. Production code never mutates this.
var incidentOpenThreshold = 2
// IncidentCorrelator returns the daemon-wide incident correlator.
// On first call: builds the correlator, restores prior state from
// the bbolt store (when available), and registers metrics. Safe for
// concurrent callers.
func IncidentCorrelator() *incident.Correlator {
incidentOnce.Do(func() {
db := store.Global()
var persist func(incident.Incident) error
if db != nil {
persist = db.SaveIncident
}
// Resolve spray-suppression knobs from the active config. nil
// config (early test wiring) leaves the detector disabled.
var spray incident.SpraySuppressionConfig
var autoBlock incident.IncidentAutoBlockConfig
var whitelisted func(string) bool
var onSprayBlock func(ip, reason string, ttl time.Duration, findingID string, root incident.PreparedRoot) bool
var onIncidentBlock func(ip, reason string, ttl time.Duration, findingID string, root incident.PreparedRoot) bool
if cfg := globalCfgForIncidents(); cfg != nil {
spray = incident.SpraySuppressionConfig{
Enabled: cfg.Incidents.SpraySuppression.Enabled,
DryRun: cfg.Incidents.SpraySuppression.DryRun,
DistinctMailboxes: cfg.Incidents.SpraySuppression.DistinctMailboxes,
BlockExpiry: autoResponseBlockExpiry(cfg),
SeverityEscalateAt: cfg.Incidents.SpraySuppression.SeverityEscalateAt,
PerCheck: cfg.IncidentsSpraySuppressionPerCheck(),
MaxTrackedIPs: cfg.Incidents.SpraySuppression.MaxTrackedIPs,
BlockAtSeverity: cfg.Incidents.SpraySuppression.BlockAtSeverity,
}
// Only wire the firewall hand-off when block-on-spray is
// configured and the daemon has a blocker installed. The live
// auto_response gate is checked at decision time so SIGHUP
// changes to enabled/block_ips take effect without rebuilding
// the singleton.
if spray.BlockAtSeverity != "" && incidentSprayBlocker != nil {
blocker := incidentSprayBlocker
onSprayBlock = func(ip, reason string, ttl time.Duration, findingID string, root incident.PreparedRoot) bool {
liveCfg := globalCfgForIncidents()
if liveCfg == nil || !liveCfg.AutoResponse.Enabled || !liveCfg.AutoResponse.BlockIPs {
return false
}
// ttl comes from the correlator's escalation ladder; zero
// is a permanent block, which the engine understands.
live, err := blocker(ip, sprayBlockReasonPrefix+reason, ttl, findingID, root, admission.EntryIncidentSpray)
if err != nil {
if live && errors.Is(err, firewall.ErrActionAuditPending) {
csmlog.Warn("credential_spray block audit delivery pending", "ip", ip, "err", err)
return true
}
if !isProtectedIPRefusal(err) && !errors.Is(err, checks.ErrAutoBlockDisabled) {
csmlog.Warn("credential_spray block failed", "ip", ip, "err", err)
}
return false
}
return live
}
}
// Generic incident-driven auto-block. Reuses the same firewall
// blocker as the spray path; the reason prefix differs so audit
// log rows distinguish which detector triggered the block.
kindsRaw := cfg.IncidentsAutoBlockKinds()
kinds := make(map[incident.Kind]bool, len(kindsRaw))
for k := range kindsRaw {
kinds[incident.Kind(k)] = true
}
autoBlock = incident.IncidentAutoBlockConfig{
Enabled: cfg.Incidents.AutoBlock.Enabled,
BlockAtSeverity: cfg.Incidents.AutoBlock.BlockAtSeverity,
BlockExpiry: autoResponseBlockExpiry(cfg),
Kinds: kinds,
}
if autoBlock.Enabled && autoBlock.BlockAtSeverity != "" && incidentSprayBlocker != nil {
blocker := incidentSprayBlocker
onIncidentBlock = func(ip, reason string, ttl time.Duration, findingID string, root incident.PreparedRoot) bool {
liveCfg := globalCfgForIncidents()
if liveCfg == nil || !liveCfg.AutoResponse.Enabled || !liveCfg.AutoResponse.BlockIPs {
return false
}
live, err := blocker(ip, incidentReasonPrefix+reason, ttl, findingID, root, admission.EntryIncident)
if err != nil {
if live && errors.Is(err, firewall.ErrActionAuditPending) {
csmlog.Warn("incident auto-block audit delivery pending", "ip", ip, "err", err)
return true
}
// Own-interface / infra IPs are intentionally never
// blockable; the incident still opened, so the operator is
// alerted to activity attributed to a protected address
// (e.g. a compromised site pivoting through the server IP)
// without the noise of a failed-block warning.
if !isProtectedIPRefusal(err) && !errors.Is(err, checks.ErrAutoBlockDisabled) {
csmlog.Warn("incident auto-block failed", "ip", ip, "err", err)
}
return false
}
return live
}
}
// Whitelist accessor: reputation.whitelist from the live config
// (a hot-reload field), then the bbolt-backed whitelist operators
// add at runtime. Resolve the config per call, but cache the last
// immutable snapshot so findings do not scan the list.
var configWhitelist reputationWhitelist
whitelisted = func(ip string) bool {
if ip == "" {
return false
}
if configWhitelist.contains(globalCfgForIncidents(), ip) {
return true
}
if d := store.Global(); d != nil && d.IsWhitelisted(ip) {
return true
}
// Backstop: a verified-crawler IP from a published range
// (Googlebot/Bingbot/Applebot) should never anchor a
// correlated incident. CDN edge ranges are intentionally
// excluded -- see threatintel.IPInAnyBot.
return threatintel.DefaultRanges().IPInAnyBot(net.ParseIP(ip))
}
}
incidentCorrelator = incident.NewCorrelator(incident.CorrelatorConfig{
Persist: persist,
OpenThreshold: incidentOpenThreshold,
SpraySuppression: spray,
AutoBlock: autoBlock,
IsWhitelisted: whitelisted,
AddressEvidence: checks.AddressEvidence,
Root: prepareIncidentRoot,
CanSprayBlock: func() bool {
cfg := globalCfgForIncidents()
return cfg != nil && cfg.AutoResponse.Enabled && cfg.AutoResponse.BlockIPs
},
CanIncidentBlock: func() bool {
cfg := globalCfgForIncidents()
return cfg != nil && cfg.AutoResponse.Enabled && cfg.AutoResponse.BlockIPs
},
OnSprayBlock: onSprayBlock,
OnIncidentBlock: onIncidentBlock,
})
if db != nil {
list, err := db.ListIncidents()
if err != nil {
csmlog.Warn("incident restore failed", "err", err)
} else {
incidentCorrelator.Restore(list)
}
}
incident.RegisterMetrics(incidentRegistry(), incidentCorrelator)
incidentRetentionCancel = startIncidentRetentionLoop(incidentCorrelator)
// Auto-close runs on its own hourly ticker so the daily retention
// loop is not coupled to the close cadence; a 24h-idle incident
// closes within at most ~25h.
if cfg := globalCfgForIncidents(); cfg != nil {
incidentAutoCloseCancel = startIncidentAutoCloseLoop(incidentCorrelator, cfg)
}
})
return incidentCorrelator
}
// globalCfgForIncidents is overridden in tests to plug a synthetic config
// without touching package-level state. Production wiring sets this to a
// closure over the daemon's loaded config; until that wiring lands the
// auto-close loop simply does not start (no panic). The retention loop
// keeps running unchanged.
var globalCfgForIncidents = func() *config.Config { return nil }
// SetIncidentConfigSource wires the daemon-loaded config so the
// incident singleton can resolve auto-close thresholds at construction.
// Called once from cmd/csm/serve before IncidentCorrelator() is first
// invoked. Subsequent calls overwrite the source so reload paths can
// rebind without restarting the singleton.
func SetIncidentConfigSource(get func() *config.Config) {
if get == nil {
globalCfgForIncidents = func() *config.Config { return nil }
return
}
globalCfgForIncidents = get
}
// startIncidentAutoCloseLoop launches the per-kind idle scan that
// auto-resolves stale incidents. Returns a cancel func. Logs every run
// at info when work was done; silent when nothing closed.
func startIncidentAutoCloseLoop(c *incident.Correlator, cfg *config.Config) func() {
stop := make(chan struct{})
stopped := make(chan struct{})
go func() {
defer close(stopped)
// Single resettable timer: first sweep after a short warm-up, then
// the normal interval -- unless a sweep reports a remaining backlog,
// in which case the next sweep fires on the fast drain cadence so a
// post-restart backlog clears in minutes rather than over hours.
timer := time.NewTimer(incidentAutoCloseWarmup)
defer timer.Stop()
for {
select {
case <-stop:
return
case <-timer.C:
more := runIncidentAutoClose(c, cfg)
next := incidentAutoCloseInterval
if more {
next = incidentAutoCloseDrainDelay
}
timer.Reset(next)
}
}
}()
// The cancel waits for the goroutine to actually exit, not just signal it,
// so the daemon's shutdown sequence can guarantee no sweep is mid-bbolt-
// write when it closes the store immediately after cancelling.
return func() {
close(stop)
<-stopped
}
}
// runIncidentAutoClose is one tick of the auto-close loop. Gated on
// the operator's config and on the per-kind threshold map. dry_run=true
// only increments counters; live mode flips Status -> resolved and
// records "auto:stale" attribution. Returns more=true when the per-sweep
// cap left stale incidents unclosed, so the caller schedules a prompt
// follow-up sweep instead of waiting the full interval.
func runIncidentAutoClose(c *incident.Correlator, cfg *config.Config) (more bool) {
return runIncidentAutoCloseAt(c, cfg, time.Now())
}
// runIncidentAutoCloseAt is runIncidentAutoClose with an explicit clock.
func runIncidentAutoCloseAt(c *incident.Correlator, cfg *config.Config, now time.Time) (more bool) {
// Sub-threshold findings and spray-detector state age out after the
// merge window. Pruning them only in the daily retention compaction
// held a day of stale entries, each carrying a full Finding, on a host
// with sustained one-shot scanner traffic.
_ = c.PruneStalePending(now)
_ = c.PruneStaleSpray(now)
// The safety cap runs on every tick regardless of the operator's
// auto-close toggle or per-kind thresholds. It is a hard backstop against
// unbounded growth of Open/Contained incidents (in memory and bbolt) on a
// host under sustained attack when auto-close is off or a kind is omitted.
capMore := runIncidentSafetyCap(c)
if cfg == nil || !cfg.IncidentsAutoCloseEnabled() {
return capMore
}
rawThresholds := cfg.IncidentsAutoCloseThresholds()
if len(rawThresholds) == 0 {
return capMore
}
thresholds := make(map[incident.Kind]time.Duration, len(rawThresholds))
for k, v := range rawThresholds {
thresholds[incident.Kind(k)] = v
}
dryRun := cfg.Incidents.AutoClose.DryRun
closed, dryRunCount, scanned, more := c.CloseStaleLimited(now, thresholds, dryRun, incidentAutoCloseMaxPerSweep)
if closed > 0 || dryRunCount > 0 {
csmlog.Info("incident auto-close",
"closed", closed,
"dry_run_decisions", dryRunCount,
"scanned", scanned,
"dry_run", dryRun,
"backlog_remaining", more,
)
}
return more || capMore
}
// incidentSafetyMaxAge is the hard age cap: any Open/Contained incident idle
// longer than this is force-closed regardless of auto-close config.
const incidentSafetyMaxAge = 30 * 24 * time.Hour
// incidentSafetyMaxActive bounds how many Open/Contained incidents are held in
// memory at once; the oldest over this are force-closed.
const incidentSafetyMaxActive = 50000
// runIncidentSafetyCap force-closes incidents past the age cap and trims the
// active set back under the size ceiling. Always runs, independent of the
// operator's auto-close settings. Returns more=true if either sweep left a
// backlog so the loop schedules a prompt follow-up.
func runIncidentSafetyCap(c *incident.Correlator) (more bool) {
now := time.Now()
byAge, ageMore := c.CloseStaleByAge(now, incidentSafetyMaxAge, incidentAutoCloseMaxPerSweep)
byCap, capMore := c.EnforceActiveCap(now, incidentSafetyMaxActive, incidentAutoCloseMaxPerSweep)
if byAge > 0 || byCap > 0 {
csmlog.Info("incident safety cap",
"closed_by_age", byAge,
"closed_by_active_cap", byCap,
"backlog_remaining", ageMore || capMore,
)
}
return ageMore || capMore
}
// startIncidentRetentionLoop runs a daily compaction sweep against the
// store. Started from IncidentCorrelator() once after the singleton is
// constructed. Logs errors but never panics. Returns the cancel func.
// The first sweep fires after one hour so the daemon has time to settle
// before touching the store under retention rules.
func startIncidentRetentionLoop(c *incident.Correlator) func() {
stop := make(chan struct{})
stopped := make(chan struct{})
go func() {
defer close(stopped)
t := time.NewTicker(24 * time.Hour)
defer t.Stop()
first := time.NewTimer(time.Hour)
defer first.Stop()
for {
select {
case <-stop:
return
case <-first.C:
runIncidentCompaction(c)
case <-t.C:
runIncidentCompaction(c)
}
}
}()
// Wait for the goroutine to exit, so a compaction in flight finishes
// before the daemon closes the store.
return func() {
close(stop)
<-stopped
}
}
// runIncidentCompaction prunes resolved/dismissed incidents older than
// the retention window and bumps the compacted_total counter so the
// metric reflects actual store work. Errors are logged, never fatal.
func runIncidentCompaction(c *incident.Correlator) {
db := store.Global()
if db == nil {
return
}
runIncidentCompactionWith(c, time.Now(), db.CompactIncidents)
}
func runIncidentCompactionWith(c *incident.Correlator, now time.Time, compact func(time.Time, incident.ClosedRetention) (int, error)) {
pruned, err := compact(now, incidentClosedRetention)
c.IncrementCompactedTotal(pruned)
if err != nil {
csmlog.Warn("incident retention compaction failed", "pruned", pruned, "err", err)
return
}
_ = c.PruneClosedOlderThan(now, incidentClosedRetention)
_ = c.PruneStalePending(now)
_ = c.PruneStaleSpray(now)
if pruned > 0 {
csmlog.Info("incident retention compaction", "pruned", pruned)
}
}
// StopIncidentBackgroundLoops cancels the incident auto-close and retention
// goroutines. The daemon calls it during shutdown, before closing the store,
// so neither loop performs a bbolt write against an already-closed database.
// Safe to call when the singleton was never constructed (both cancels nil).
func StopIncidentBackgroundLoops() {
if incidentRetentionCancel != nil {
incidentRetentionCancel()
incidentRetentionCancel = nil
}
if incidentAutoCloseCancel != nil {
incidentAutoCloseCancel()
incidentAutoCloseCancel = nil
}
if incidentCorrelator != nil {
if flushed := incidentCorrelator.FlushPendingPersists(); flushed > 0 {
csmlog.Info("incident pending persists flushed", "count", flushed)
}
}
}
// resetIncidentForTest is a test seam. Stops any retention worker, zeros
// the singleton, and pins the registry to a private one so subsequent
// IncidentCorrelator() calls do not collide on metrics.Default.
func resetIncidentForTest() {
resetIncidentForTestWithThreshold(1)
}
// resetIncidentForTestWithThreshold is the same seam but lets a test pin
// the open threshold to a specific value. Used by the wiring test that
// proves the production default (2) is honored end-to-end through the
// IncidentCorrelator() singleton constructor.
func resetIncidentForTestWithThreshold(threshold int) {
if incidentRetentionCancel != nil {
incidentRetentionCancel()
incidentRetentionCancel = nil
}
if incidentAutoCloseCancel != nil {
incidentAutoCloseCancel()
incidentAutoCloseCancel = nil
}
incidentCorrelator = nil
incidentOnce = sync.Once{}
incidentRegistry = metrics.NewRegistry
globalCfgForIncidents = func() *config.Config { return nil }
incidentSprayBlocker = nil
// Most tests assert that one finding lands in one incident; the
// production threshold of 2 would defer creation to the second
// correlated event and break those wiring assertions. Pin to the
// caller-supplied value; production callers never invoke this seam.
incidentOpenThreshold = threshold
}
// prepareIncidentRoot mints the admission root of an attesting finding.
// A timeline holds one per attesting event, so it keeps of the finding only
// what a later refusal counts: its check and severity.
func prepareIncidentRoot(f alert.Finding) incident.PreparedRoot {
root, err := checks.PrepareAdmissionRoot(f)
return incident.PreparedRoot{Evidence: root, Finding: alert.Finding{Check: f.Check, Severity: f.Severity}, Err: err}
}
package daemon
import (
"net/netip"
"strings"
"sync"
"github.com/pidginhost/csm/internal/config"
)
// Config publication replaces immutable snapshots. Retain only the last one
// so whitelist lookup stays constant-time without retaining reload history.
type reputationWhitelist struct {
mu sync.Mutex
cfg *config.Config
ips map[netip.Addr]bool
literals map[string]bool
}
func (w *reputationWhitelist) contains(cfg *config.Config, addr string) bool {
if cfg == nil || addr == "" {
return false
}
w.mu.Lock()
defer w.mu.Unlock()
if w.cfg != cfg {
w.ips = make(map[netip.Addr]bool, len(cfg.Reputation.Whitelist))
w.literals = make(map[string]bool)
for _, entry := range cfg.Reputation.Whitelist {
entry = strings.TrimSpace(entry)
if entry == "" {
continue
}
if ip, err := netip.ParseAddr(entry); err == nil && ip.Zone() == "" {
w.ips[ip.Unmap()] = true
} else {
// Subnet-spray findings can name a literal CIDR. Preserve
// exact matches without granting a host-wide subnet exemption.
w.literals[entry] = true
}
}
w.cfg = cfg
}
if ip, err := netip.ParseAddr(addr); err == nil && ip.Zone() == "" {
return w.ips[ip.Unmap()]
}
return w.literals[addr]
}
//go:build linux
package daemon
import (
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/queuehealth"
)
type inotifyDescriptor struct{ fanotifyDescriptor }
func (fd inotifyDescriptor) Pending() (int, error) {
return unix.IoctlGetInt(int(fd.fanotifyDescriptor), unix.TIOCINQ)
}
func newInotifyQueue(fd int) *notificationQueue {
q := newNotificationQueue(inotifyDescriptor{fanotifyDescriptor(fd)}, queuehealth.New(0, time.Minute), queuehealth.New(1, time.Minute))
q.sampled = queuehealth.NewSampled(0, "bytes", time.Minute)
q.variableRecords = true
return q
}
func (fw *ForwarderWatcher) initQueueHealth() {
fw.queueHealthOnce.Do(func() { fw.kernelQueue = newInotifyQueue(fw.inotifyFd) })
}
func (fw *ForwarderWatcher) QueueStatuses(_ time.Time) map[string]queuehealth.Status {
fw.initQueueHealth()
kernel, reader := fw.kernelQueue.snapshot(time.Now)
return map[string]queuehealth.Status{"kernel": kernel, "reader": reader}
}
func (w *spoolWatcher) initQueueHealth() {
w.queueHealthOnce.Do(func() { w.kernelQueue = newInotifyQueue(w.fd) })
}
func (w *spoolWatcher) QueueStatuses(_ time.Time) map[string]queuehealth.Status {
w.initQueueHealth()
kernel, reader := w.kernelQueue.snapshot(time.Now)
return map[string]queuehealth.Status{"kernel": kernel, "reader": reader}
}
// Set before publishing or running the replacement. Its descriptor has fresh
// occupancy, but a restart must not clear losses from this daemon's lifetime.
func (w *spoolWatcher) inheritQueueHealth(previous *spoolWatcher) {
previous.initQueueHealth()
w.initQueueHealth()
w.kernelQueue.losses = previous.kernelQueue.losses
w.kernelQueue.batches = previous.kernelQueue.batches
}
package daemon
import (
"container/heap"
"net/netip"
"slices"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/blockdigest"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/eximlog"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/store"
)
// knownCustomerTTL is how long one successful login keeps an address known.
const knownCustomerTTL = 30 * 24 * time.Hour
// maxKnownCustomers bounds the set, split in two fixed halves by login
// source. Any local user can write syslog with logger, so each half trims
// only itself: a flood read from one source never evicts the other. A full
// half trims in one batch, down to 95%, so each new address does not pay for
// a scan.
const maxKnownCustomers = 50000
// maxSyslogCustomers bounds the logins read from syslog.
const maxSyslogCustomers = maxKnownCustomers / 2
// maxServiceCustomers bounds the logins read from session_log and the Exim
// mainlog.
const maxServiceCustomers = maxKnownCustomers - maxSyslogCustomers
// maxKnownCustomerIPsPerAccount bounds the addresses one account holds from
// one login source; logins without an account name form one group. Rotating
// credentials or addresses cannot take over the set.
const maxKnownCustomerIPsPerAccount = 64
// Kinds of successful customer login.
const (
knownCustomerCpanel = "cpanel"
knownCustomerWebmail = "webmail"
knownCustomerSMTP = "smtp"
knownCustomerIMAP = "imap"
knownCustomerPOP = "pop"
knownCustomerFTP = "ftp"
)
// loginSource is the log a login was read from.
type loginSource bool
const (
serviceLog loginSource = false // written only by the service: session_log, the Exim mainlog
syslogFed loginSource = true // syslog, which any local user can write with logger
)
// loginFromSyslog reports whether a login of kind was read from syslog.
// cPanel and webmail logins come only from session_log, FTP, IMAP and POP
// only from syslog; SMTP comes from the Exim mainlog or from Postfix over
// syslog, as src says. ok is false for an unknown kind.
func loginFromSyslog(kind string, src loginSource) (syslog, ok bool) {
switch kind {
case knownCustomerCpanel, knownCustomerWebmail:
return false, true
case knownCustomerFTP, knownCustomerIMAP, knownCustomerPOP:
return true, true
case knownCustomerSMTP:
return bool(src), true
}
return false, false
}
// customerGroup is what one account holds from one login source. A full set
// trims its largest groups first, so a flood loses its own entries before
// anyone else's.
type customerGroup struct {
syslog bool
account string
}
// knownCustomers remembers addresses that logged in successfully as a
// customer, with the last such login. Only real successes are noted, so an
// attacker who can only fail logins never enters the set. An IPv6 host is
// its /64: privacy addresses rotate inside it. Its mutex is a leaf: the
// infra lookup and store I/O run outside it.
type knownCustomers struct {
now func() time.Time
// isInfra reports addresses that are never stored. Set once before log
// readers start.
isInfra func(ip string) bool
mu sync.Mutex
entries map[netip.Addr]store.KnownCustomer
groups map[customerGroup]map[netip.Addr]struct{}
syslogCount int // entries read from syslog
stored map[netip.Addr]struct{} // addresses loaded or handed to the writer
dirty map[netip.Addr]struct{} // to write at the next save
deleted map[string]struct{} // stored keys to remove at the next save
}
func newKnownCustomers(now func() time.Time) *knownCustomers {
return &knownCustomers{
now: now,
entries: make(map[netip.Addr]store.KnownCustomer),
groups: make(map[customerGroup]map[netip.Addr]struct{}),
stored: make(map[netip.Addr]struct{}),
dirty: make(map[netip.Addr]struct{}),
deleted: make(map[string]struct{}),
}
}
func knownCustomerExpired(at, now time.Time) bool {
return now.Sub(at) >= knownCustomerTTL
}
// customerKey is the key ip is stored under: the address, or for IPv6 its
// /64.
func customerKey(ip string) (netip.Addr, bool) {
addr, err := netip.ParseAddr(ip)
if err != nil || addr.Zone() != "" {
return netip.Addr{}, false
}
return networkKey(addr.Unmap()), true
}
func networkKey(addr netip.Addr) netip.Addr {
if addr.Is6() {
return netip.PrefixFrom(addr, 64).Masked().Addr()
}
return addr
}
// storedKey reports whether raw is a key as Changes writes it.
func storedKey(raw string) (netip.Addr, bool) {
key, ok := customerKey(raw)
return key, ok && key.String() == raw
}
// customerAddr is the key a login from ip is stored under, or false when ip
// can never be a customer address.
func (k *knownCustomers) customerAddr(ip string) (netip.Addr, bool) {
addr, err := netip.ParseAddr(ip)
if err != nil || addr.Zone() != "" {
return netip.Addr{}, false
}
addr = addr.Unmap()
if !addr.IsGlobalUnicast() || addr.IsPrivate() || (k.isInfra != nil && k.isInfra(addr.String())) {
return netip.Addr{}, false
}
return networkKey(addr), true
}
// Note records a successful login of kind from ip at at, read from src. A
// login older than the TTL is ignored and one stamped in the future counts
// as now.
func (k *knownCustomers) Note(ip, kind, account string, at time.Time, src loginSource) {
if k == nil {
return
}
syslog, ok := loginFromSyslog(kind, src)
if !ok {
return
}
addr, ok := k.customerAddr(ip)
if !ok {
return
}
now := k.now()
if at.After(now) {
at = now
}
if knownCustomerExpired(at, now) {
return
}
c := store.KnownCustomer{Kind: kind, Account: strings.Clone(normalizeSprayMailbox(account)), At: at, Syslog: syslog}
k.mu.Lock()
defer k.mu.Unlock()
k.putLocked(addr, c, now)
}
// putLocked stores c for addr, then trims the half c belongs to. An expired
// entry counts as absent.
func (k *knownCustomers) putLocked(addr netip.Addr, c store.KnownCustomer, now time.Time) {
if old, exists := k.entries[addr]; exists {
if !knownCustomerExpired(old.At, now) {
switch {
case !old.Syslog && c.Syslog:
// A forged syslog line would relabel the entry into the
// half a syslog flood can trim.
return
case old.Syslog && !c.Syslog:
// A service log is the stronger record; keep the later time.
if old.At.After(c.At) {
c.At = old.At
}
case old.At.After(c.At):
return
}
}
k.unlinkLocked(addr, old)
}
k.entries[addr] = c
k.linkLocked(addr, c)
k.dirty[addr] = struct{}{}
delete(k.deleted, addr.String())
k.trimGroupLocked(customerGroup{c.Syslog, c.Account})
if c.Syslog && k.syslogCount > maxSyslogCustomers {
k.evictLocked(true, k.syslogCount-(maxSyslogCustomers*95/100+1))
}
if service := len(k.entries) - k.syslogCount; !c.Syslog && service > maxServiceCustomers {
k.evictLocked(false, service-(maxServiceCustomers*95/100+1))
}
}
func (k *knownCustomers) linkLocked(addr netip.Addr, c store.KnownCustomer) {
g := customerGroup{c.Syslog, c.Account}
held := k.groups[g]
if held == nil {
held = make(map[netip.Addr]struct{})
k.groups[g] = held
}
held[addr] = struct{}{}
if c.Syslog {
k.syslogCount++
}
}
func (k *knownCustomers) unlinkLocked(addr netip.Addr, c store.KnownCustomer) {
g := customerGroup{c.Syslog, c.Account}
delete(k.groups[g], addr)
if len(k.groups[g]) == 0 {
delete(k.groups, g)
}
if c.Syslog {
k.syslogCount--
}
}
// trimGroupLocked drops the oldest entry of a group over its cap, which may
// be the one just noted.
func (k *knownCustomers) trimGroupLocked(g customerGroup) {
held := k.groups[g]
if len(held) <= maxKnownCustomerIPsPerAccount {
return
}
var oldest customerEntry
for a := range held {
if e := (customerEntry{a, k.entries[a]}); !oldest.addr.IsValid() || compareCustomerEntries(e, oldest) < 0 {
oldest = e
}
}
k.removeLocked(oldest.addr)
}
// customerEntry is one address with its last login.
type customerEntry struct {
addr netip.Addr
c store.KnownCustomer
}
// compareCustomerEntries orders the oldest login first.
func compareCustomerEntries(a, b customerEntry) int {
if c := a.c.At.Compare(b.c.At); c != 0 {
return c
}
return a.addr.Compare(b.addr)
}
// evictLocked removes n entries of one half, each the oldest of the largest
// remaining group.
func (k *knownCustomers) evictLocked(syslog bool, n int) {
byGroup := make(map[customerGroup][]customerEntry)
for a, c := range k.entries {
if c.Syslog == syslog {
g := customerGroup{c.Syslog, c.Account}
byGroup[g] = append(byGroup[g], customerEntry{a, c})
}
}
h := make(evictionHeap, 0, len(byGroup))
for _, members := range byGroup {
slices.SortFunc(members, compareCustomerEntries)
h = append(h, members)
}
heap.Init(&h)
for ; n > 0 && h.Len() > 0; n-- {
members := h[0]
k.removeLocked(members[0].addr)
if len(members) == 1 {
heap.Pop(&h)
continue
}
h[0] = members[1:]
heap.Fix(&h, 0)
}
}
// evictionHeap holds groups, oldest entry first within each, ordered largest
// group first and then by oldest entry.
type evictionHeap [][]customerEntry
func (h evictionHeap) Len() int { return len(h) }
func (h evictionHeap) Less(i, j int) bool {
if len(h[i]) != len(h[j]) {
return len(h[i]) > len(h[j])
}
return compareCustomerEntries(h[i][0], h[j][0]) < 0
}
func (h evictionHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *evictionHeap) Push(x any) { *h = append(*h, x.([]customerEntry)) }
func (h *evictionHeap) Pop() any {
old := *h
last := old[len(old)-1]
*h = old[:len(old)-1]
return last
}
func (k *knownCustomers) removeLocked(addr netip.Addr) {
c, ok := k.entries[addr]
if !ok {
return
}
delete(k.entries, addr)
k.unlinkLocked(addr, c)
delete(k.dirty, addr)
if _, stored := k.stored[addr]; stored && !k.queueDeleteLocked(addr.String()) {
delete(k.stored, addr)
}
}
// queueDeleteLocked marks a stored key for deletion at the next save and
// reports whether it did. Pending deletes stay bounded; a key past the bound
// stays on disk until a later load drops it again.
func (k *knownCustomers) queueDeleteLocked(raw string) bool {
if _, pending := k.deleted[raw]; !pending && len(k.deleted) >= maxKnownCustomers {
return false
}
k.deleted[raw] = struct{}{}
return true
}
// customerHit names an IPv6 entry by its /64.
func customerHit(addr netip.Addr, c store.KnownCustomer) blockdigest.CustomerHit {
return blockdigest.CustomerHit{IP: customerHitIP(addr), Kind: c.Kind, Account: c.Account, At: c.At}
}
func customerHitIP(addr netip.Addr) string {
if addr.Is6() {
return netip.PrefixFrom(addr, 64).String()
}
return addr.String()
}
// entryPrefix is the network one entry covers.
func entryPrefix(addr netip.Addr) netip.Prefix {
if addr.Is6() {
return netip.PrefixFrom(addr, 64)
}
return netip.PrefixFrom(addr, 32)
}
// Lookup returns the last customer login from ip, or from its IPv6 /64,
// within the TTL.
func (k *knownCustomers) Lookup(ip string) (blockdigest.CustomerHit, bool) {
if k == nil {
return blockdigest.CustomerHit{}, false
}
key, ok := customerKey(ip)
if !ok {
return blockdigest.CustomerHit{}, false
}
now := k.now()
k.mu.Lock()
c, ok := k.entries[key]
k.mu.Unlock()
if !ok || knownCustomerExpired(c.At, now) {
return blockdigest.CustomerHit{}, false
}
return customerHit(key, c), true
}
// Within counts the known customers that p covers or overlaps and returns up
// to limit of them, the most recent first. An omitted customer counts toward
// neither the total nor the sample. omit must not take locks or perform I/O.
func (k *knownCustomers) Within(p netip.Prefix, limit int, omit func(blockdigest.CustomerHit) bool) (n int, sample []blockdigest.CustomerHit) {
if k == nil || !p.IsValid() {
return 0, nil
}
p = p.Masked()
now := k.now()
var matches []customerEntry
k.mu.Lock()
// A linear scan of at most maxKnownCustomers entries per call; index by
// network if subnet blocks ever become frequent.
for addr, c := range k.entries {
if !p.Overlaps(entryPrefix(addr)) || knownCustomerExpired(c.At, now) {
continue
}
if omit != nil && omit(customerHit(addr, c)) {
continue
}
n++
if limit > 0 {
matches = append(matches, customerEntry{addr, c})
}
}
k.mu.Unlock()
if limit <= 0 {
return n, nil
}
slices.SortFunc(matches, func(a, b customerEntry) int { return compareCustomerEntries(b, a) })
for _, m := range matches[:min(limit, len(matches))] {
sample = append(sample, customerHit(m.addr, m.c))
}
return n, sample
}
// Purge drops entries past the TTL.
func (k *knownCustomers) Purge() {
now := k.now()
k.mu.Lock()
defer k.mu.Unlock()
for addr, c := range k.entries {
if knownCustomerExpired(c.At, now) {
k.removeLocked(addr)
}
}
}
// Load merges saved entries, keeping the newer login per key. Saved keys it
// cannot use (unparsable, not a customer address, an unknown kind, past the
// TTL) or that are not keys as Changes writes them are queued for deletion,
// so the bucket sheds them.
func (k *knownCustomers) Load(saved map[string]store.KnownCustomer) {
type entry struct {
raw string
addr netip.Addr
c store.KnownCustomer
}
now := k.now()
valid := make([]entry, 0, len(saved))
var dropped []string
for raw, c := range saved {
syslog, kindOK := loginFromSyslog(c.Kind, loginSource(c.Syslog))
addr, ok := k.customerAddr(raw)
if !kindOK || !ok || knownCustomerExpired(c.At, now) {
dropped = append(dropped, raw)
continue
}
if c.At.After(now) {
c.At = now
}
c.Account = strings.Clone(normalizeSprayMailbox(c.Account))
c.Syslog = syslog
valid = append(valid, entry{raw, addr, c})
}
k.mu.Lock()
defer k.mu.Unlock()
for _, raw := range dropped {
k.queueDeleteLocked(raw)
}
// Index all stored keys before admission: a later record may evict an
// earlier one, and both still need deletion from the saved set.
for _, e := range valid {
if e.raw == e.addr.String() {
k.stored[e.addr] = struct{}{}
}
}
slices.SortFunc(valid, func(a, b entry) int {
return compareCustomerEntries(customerEntry{a.addr, a.c}, customerEntry{b.addr, b.c})
})
for _, e := range valid {
canonical := e.raw == e.addr.String()
if !canonical {
k.queueDeleteLocked(e.raw)
}
k.putLocked(e.addr, e.c, now)
// The store already holds an entry loaded unchanged.
if cur, ok := k.entries[e.addr]; ok && canonical && sameKnownCustomer(cur, saved[e.raw]) {
delete(k.dirty, e.addr)
}
}
}
func sameKnownCustomer(a, b store.KnownCustomer) bool {
return a.Kind == b.Kind && a.Account == b.Account && a.At.Equal(b.At) && a.Syslog == b.Syslog
}
// Changes returns what changed since the last call: entries to write and
// stored keys to delete.
func (k *knownCustomers) Changes() (upserts map[string]store.KnownCustomer, deletes []string) {
k.mu.Lock()
defer k.mu.Unlock()
upserts = make(map[string]store.KnownCustomer, len(k.dirty))
for addr := range k.dirty {
upserts[addr.String()] = k.entries[addr]
k.stored[addr] = struct{}{}
}
deletes = make([]string, 0, len(k.deleted))
for raw := range k.deleted {
deletes = append(deletes, raw)
if addr, ok := storedKey(raw); ok {
delete(k.stored, addr)
}
}
slices.Sort(deletes)
clear(k.dirty)
clear(k.deleted)
return upserts, deletes
}
// requeue marks the keys of a failed save for the next one: a key still held
// is written with its current value, any other is deleted.
func (k *knownCustomers) requeue(upserts map[string]store.KnownCustomer, deletes []string) {
k.mu.Lock()
defer k.mu.Unlock()
mark := func(raw string) {
addr, isKey := storedKey(raw)
if isKey {
if _, ok := k.entries[addr]; ok {
k.stored[addr] = struct{}{}
k.dirty[addr] = struct{}{}
return
}
}
queued := k.queueDeleteLocked(raw)
if !isKey {
return
}
if queued {
// A failed delete may still be on disk even if a new login
// temporarily cancels this pending deletion.
k.stored[addr] = struct{}{}
} else {
delete(k.stored, addr)
}
}
for raw := range upserts {
mark(raw)
}
for _, raw := range deletes {
mark(raw)
}
}
// customerLoginTime keeps buffered records from renewing an old login.
// Timestamp-free journal messages use the receipt time.
func customerLoginTime(line string, now time.Time) time.Time {
if strings.HasPrefix(line, "[") {
if at, ok := checks.CPanelLogTime(line); ok {
return at
}
}
if at := smtpAuthLogTime(line, now); !at.IsZero() {
return at
}
if at, ok := eximlog.LineTime(line, now.Location()); ok {
return at
}
return now
}
// noteFTPCustomers remembers the addresses of successful FTP logins.
func (d *Daemon) noteFTPCustomers(findings []alert.Finding) {
for _, f := range findings {
switch f.Check {
case "ftp_login", "ftp_login_after_bruteforce":
d.knownCustomers.Note(f.SourceIP, knownCustomerFTP, ftpCustomerAccount(f), f.Timestamp, syslogFed)
}
}
}
// ftpCustomerAccount is the account an FTP login is known under: the hosting
// account that owns the FTP account, or the login name when no account owns
// it.
func ftpCustomerAccount(f alert.Finding) string {
if f.TenantID != "" {
return f.TenantID
}
if f.LoginAccount != "" {
return f.LoginAccount
}
return checks.FTPLoginAccount(f.Details)
}
// loadKnownCustomers seeds the set from the last run. The store still
// returns the records when it could not delete the unusable ones.
func (d *Daemon) loadKnownCustomers() {
sdb := store.Global()
if sdb == nil {
return
}
saved, err := sdb.LoadKnownCustomers()
if err != nil {
csmlog.Warn("known customer load failed", "err", err)
}
d.knownCustomers.Load(saved)
}
// persistKnownCustomers purges expired entries and saves what changed.
func (d *Daemon) persistKnownCustomers() {
if d.knownCustomers == nil {
return
}
d.knownCustomers.Purge()
sdb := store.Global()
if sdb == nil {
return
}
upserts, deletes := d.knownCustomers.Changes()
if len(upserts)+len(deletes) == 0 {
return
}
if err := sdb.SaveKnownCustomers(upserts, deletes); err != nil {
d.knownCustomers.requeue(upserts, deletes)
csmlog.Warn("known customer save failed", "err", err)
}
}
// runKnownCustomers saves the set every minute. The last save runs in the
// shutdown flush, once log readers and dispatch have stopped noting logins.
func (d *Daemon) runKnownCustomers() {
defer d.wg.Done()
t := time.NewTicker(time.Minute)
defer t.Stop()
for {
select {
case <-d.stopCh:
return
case <-t.C:
d.persistKnownCustomers()
}
}
}
package daemon
import (
"errors"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
csmlog "github.com/pidginhost/csm/internal/log"
)
// startLiveBackend selects the backend for one live monitor from the
// operator's choice and host capability:
//
// "auto" (default) -- try BPF, fall back to legacy polling, else none.
// "bpf" -- require BPF; return nil if unavailable (no fallback).
// "legacy" -- pin legacy polling.
// "none" -- disable the live monitor (the periodic check still runs).
//
// Unknown values fall back to "auto" with a warning. The metric
// csm_bpf_backend{feature=<feature>, kind="..."} reflects the chosen path.
func startLiveBackend(feature, configured string, alertCh chan<- alert.Finding,
tryBPF func() (bpf.Backend, error), legacy func() bpf.Backend) bpf.Backend {
choice := strings.ToLower(strings.TrimSpace(configured))
if choice == "" {
choice = bpf.BackendAuto
}
switch choice {
case bpf.BackendAuto, bpf.BackendBPF, bpf.BackendLegacy, bpf.BackendNone:
default:
csmlog.Warn(feature+": unknown backend choice, using auto", "value", choice)
choice = bpf.BackendAuto
}
if choice == bpf.BackendNone {
csmlog.Info(feature + ": disabled by config")
bpf.SetActive(feature, bpf.BackendNone)
return nil
}
var bpfErr error
if choice == bpf.BackendAuto || choice == bpf.BackendBPF {
if b, err := tryBPF(); err == nil && b != nil {
csmlog.Info(feature, "backend", "bpf", "choice", choice)
bpf.SetActive(feature, bpf.BackendBPF)
return b
} else if err != nil {
bpfErr = err
level := "bpf-unsupported"
if errors.Is(err, bpf.ErrNotBuilt) {
level = "bpf-not-built"
}
csmlog.Info(feature+": BPF unavailable", "state", level, "reason", err.Error(), "choice", choice)
if choice == bpf.BackendBPF {
csmlog.Warn(feature+": backend=bpf but BPF unavailable; no live monitor", "reason", err.Error())
bpf.SetActive(feature, bpf.BackendNone)
emitBPFUnavailableFinding(alertCh, feature, choice, "", err)
return nil
}
}
}
poller := legacy()
if poller == nil {
// Only a legacy backend that probes the kernel can fail to start.
// Its caller knows why, so it decides whether that is a finding.
csmlog.Info(feature+": no legacy backend available", "choice", choice)
bpf.SetActive(feature, bpf.BackendNone)
return nil
}
csmlog.Info(feature, "backend", "legacy", "choice", choice)
bpf.SetActive(feature, bpf.BackendLegacy)
if bpfErr != nil {
emitBPFUnavailableFinding(alertCh, feature, choice, bpf.BackendLegacy, bpfErr)
}
return poller
}
package daemon
import "github.com/pidginhost/csm/internal/config"
// liveConfigFn returns an accessor for the live daemon config: the active
// (hot-reloaded) config once one is published, otherwise the startup
// snapshot. Long-lived components that used to capture the startup pointer
// read through it so a reload actually reaches them.
func liveConfigFn(startup *config.Config) func() *config.Config {
return func() *config.Config {
if active := config.Active(); active != nil {
return active
}
return startup
}
}
package daemon
import (
"time"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/obs"
)
// Retry the whole candidate list: a fallback can appear before the preferred
// log when the web server starts after the daemon. spec names the handler and
// producer; its path is ignored.
func (d *Daemon) retryLogWatcherCandidates(paths []string, spec logWatchSpec) {
defer d.wg.Done()
ticker := time.NewTicker(logWatcherRetryInterval)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
for _, path := range paths {
candidate := spec
candidate.path = path
w, err := newObservedLogWatcher(candidate, d.cfg, d.alertCh)
if err != nil {
continue
}
d.logWatchersMu.Lock()
d.logWatchers = append(d.logWatchers, w)
d.logWatchersMu.Unlock()
d.wg.Add(1)
obs.Go("logwatch-late", func() {
defer d.wg.Done()
w.Run(d.stopCh)
})
csmlog.Info("watching log (appeared after retry)", "path", path)
if spec.name != "" {
d.MarkWatcher(spec.name, true)
}
return
}
}
}
}
package daemon
import (
"slices"
"strings"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/eximlog"
)
// mailAccountOwner accepts an authenticated identity, never an envelope
// sender. Bare names need the same hosting-account validation as spool users.
func mailAccountOwner(account string) string {
if strings.Contains(account, "@") {
return checks.MailOwner(account)
}
return checks.HostingAccountForUser(account)
}
// Call after releasing tracker locks: the resolver may refresh host files.
func stampMailAccountOwner(findings []alert.Finding, account string) {
if len(findings) == 0 {
return
}
owner := mailAccountOwner(account)
// A mailbox's domain owner is useful for correlation, but authentication
// of the mailbox does not establish the hosting account's identity.
var claims []admission.Claim
if strings.Contains(account, "@") {
claims = append(claims, admission.Claim{Kind: admission.ClaimMailbox, Value: account})
} else if owner != "" {
claims = append(claims, admission.Claim{Kind: admission.ClaimAccount, Value: owner})
}
for i := range findings {
findings[i].TenantID = owner
findings[i].Claims = slices.Clone(claims)
}
}
// Only cPanel's own permission records establish a held local account.
// Peer names, subjects and remote delivery replies are untrusted log data.
func mailPermissionLogText(line string) string {
p, rest, ok := eximlog.SplitPrefix(line, time.Local)
if !ok {
return ""
}
fields := strings.Fields(rest)
if p.MessageID == "" {
if len(fields) >= 2 && (fields[0] == "Sender" || fields[0] == "Domain") {
return strings.Join(fields, " ")
}
return ""
}
if len(fields) < 4 || (fields[0] != "==" && fields[0] != "**") || fields[2] != "R=enforce_mail_permissions" {
return ""
}
return strings.Join(fields[3:], " ")
}
package daemon
import (
"context"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/maillog"
"github.com/pidginhost/csm/internal/obs"
)
func (d *Daemon) startMailLogReader(platformDefault string, handler ObservedLineHandler) {
d.MarkWatcher("maillog", false)
queue := maillog.NewQueue()
d.registerQueueSource("mail", queue)
ctx, cancel := context.WithCancel(context.Background())
d.wg.Add(2)
obs.Go("maillog-stop", func() {
defer d.wg.Done()
select {
case <-d.stopCh:
case <-ctx.Done():
}
cancel()
})
obs.Go("maillog-supervisor", func() {
defer d.wg.Done()
defer cancel()
defer d.MarkWatcher("maillog", false)
maillog.Supervise(ctx, func() (maillog.Reader, error) {
return maillog.New(d.currentCfg().MailLogs, platformDefault, queue)
}, func(err error) {
if err != nil {
csmlog.Warn("mail log source unavailable; retrying", "err", err)
d.handleMailLogSourceGone(err)
} else {
d.handleMailLogSourceRestored()
}
}, func(line maillog.Line) bool {
return d.dispatchMailLogLine(line, handler)
})
})
}
package daemon
// mailboxExistence is what a rejected login says about its mailbox.
type mailboxExistence uint8
const (
mailboxExistenceUnknown mailboxExistence = iota
mailboxExists
mailboxMissing
)
// sprayMailbox is the mailbox a failed login counts against. name is already
// normalised; a typo resolves to the real mailbox.
type sprayMailbox struct {
name string
exist mailboxExistence
}
// combineMailboxExistence merges two answers about the same mailbox. Each
// database in a Dovecot passdb chain answers on its own, and one database
// that knows the mailbox outweighs any number that do not.
func combineMailboxExistence(a, b mailboxExistence) mailboxExistence {
if a == b {
return a
}
if a == mailboxExists || b == mailboxExists {
return mailboxExists
}
return mailboxExistenceUnknown
}
package daemon
import (
"errors"
"slices"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/antispam/dovecot"
"github.com/pidginhost/csm/internal/antispam/learnwatch"
"github.com/pidginhost/csm/internal/checks"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/platform"
)
const (
// mailboxResolverTTL is how long a read of cPanel's state answers. A new
// or deleted mailbox can get its old answer for this long.
mailboxResolverTTL = 2 * time.Minute
// maxResolverAccounts bounds the accounts whose mailbox lists are held.
maxResolverAccounts = 4096
)
// mailboxResolver answers whether a failed login name is a real mailbox.
type mailboxResolver interface {
Resolve(name string) sprayMailbox
}
// Platform detection must follow startup's config overrides, so construction
// takes the already detected host instead of probing in New.
func newMailboxResolver(host platform.Info) mailboxResolver {
if !host.IsCPanel() {
return nil
}
return newCPanelMailboxResolver(newLiveMailboxSource(learnwatch.DefaultSource), time.Now)
}
// resolveSprayMailbox gives the mailbox a failed login on name counts against.
// A verdict from the log is about this login while the resolver can be a TTL
// behind, so the resolver cannot overturn it, except that a Missing name near
// a real mailbox is a typo of that mailbox.
func resolveSprayMailbox(r mailboxResolver, name string, verdict mailboxExistence) sprayMailbox {
key := normalizeSprayMailbox(name)
if r == nil || verdict == mailboxExists {
return sprayMailbox{name: key, exist: verdict}
}
mb := r.Resolve(name)
if verdict == mailboxExistenceUnknown || (mb.exist == mailboxExists && mb.name != key) {
return mb
}
return sprayMailbox{name: key, exist: mailboxMissing}
}
// sprayMailboxFor is resolveSprayMailbox with the daemon's resolver. Callers
// hold no tracker lock. The lookup runs only while the spray rules are on.
func (d *Daemon) sprayMailboxFor(name string, verdict mailboxExistence) sprayMailbox {
if !d.sprayRulesOn() {
return resolveSprayMailbox(nil, name, verdict)
}
return resolveSprayMailbox(d.mailboxes, name, verdict)
}
// sprayRulesOn reports whether anything reads mailbox existence.
func (d *Daemon) sprayRulesOn() bool {
return d.mailSpray != nil && d.mailSpray.Mode() != sprayOff
}
// cpanelMailboxSource reads cPanel's accounts, domains and mailboxes.
type cpanelMailboxSource interface {
Inventory() (checks.HostingSnapshot, error)
// Mailboxes returns the local parts of each domain the last Inventory
// lists for account, sorted, distinct and in the spelling of
// normalizeSprayMailbox, as are the domain keys. complete is false when
// the read skipped anything.
Mailboxes(account string) (lists map[string][]string, complete bool, err error)
}
// accountMailboxes belongs to one inventory snapshot. Failed and partial reads
// are kept too, so each account costs at most one read before the next snapshot.
type accountMailboxes struct {
lists map[string][]string
usable bool
}
// cpanelMailboxResolver answers from cached cPanel state. A typed name is
// only compared with that state: the source is only ever asked for owners
// named in the snapshot.
type cpanelMailboxResolver struct {
src cpanelMailboxSource
now func() time.Time
// mu is a leaf lock held across source reads, so a burst of logins costs
// one read and the source needs no lock of its own.
mu sync.Mutex
snapshot checks.HostingSnapshot
snapshotUsable bool
snapshotAt time.Time
typoIndex mailboxTypoIndex
accounts map[string]accountMailboxes
}
func newCPanelMailboxResolver(src cpanelMailboxSource, now func() time.Time) *cpanelMailboxResolver {
return &cpanelMailboxResolver{src: src, now: now, accounts: map[string]accountMailboxes{}}
}
// Resolve takes the original login name: normalizing before lookup would lose
// evidence of truncation. An answer it cannot give is Unknown. It returns the
// normalized key, except that a typo match takes the real mailbox's spelling,
// so one mailbox is one key.
func (r *cpanelMailboxResolver) Resolve(input string) sprayMailbox {
name := normalizeSprayMailbox(input)
if name == "" {
return sprayMailbox{}
}
// A cut after whitespace can leave a valid key shorter than the limit.
if len(strings.TrimSpace(input)) >= maxSprayMailboxBytes || strings.HasPrefix(name, "__cpanel__") {
return sprayMailbox{name: name, exist: mailboxExistenceUnknown}
}
r.mu.Lock()
defer r.mu.Unlock()
r.refreshSnapshot(r.now())
if !r.snapshotUsable {
return sprayMailbox{name: name, exist: mailboxExistenceUnknown}
}
at := strings.LastIndexByte(name, '@')
if at < 0 {
return sprayMailbox{name: name, exist: r.accountNameExistence(name)}
}
local, domain := name[:at], name[at+1:]
if !dovecot.ValidMailbox(local) || !dovecot.ValidDomain(domain) || r.snapshot.Unresolved[domain] {
return sprayMailbox{name: name, exist: mailboxExistenceUnknown}
}
if owner, hosted := r.snapshot.Domains[domain]; hosted {
lists, usable := r.accountLists(owner)
if !usable {
return sprayMailbox{name: name, exist: mailboxExistenceUnknown}
}
if _, listed := slices.BinarySearch(lists[domain], local); listed || local == owner {
return sprayMailbox{name: name, exist: mailboxExists}
}
if hasAddressExtension(local) {
return sprayMailbox{name: name, exist: mailboxExistenceUnknown}
}
}
return r.resolveTypo(name, local, domain)
}
// accountNameExistence answers for a login with no domain.
func (r *cpanelMailboxResolver) accountNameExistence(name string) mailboxExistence {
if !admission.ValidAccountName(name) {
return mailboxExistenceUnknown
}
if _, found := slices.BinarySearch(r.snapshot.Accounts, name); found {
return mailboxExists
}
return mailboxMissing
}
// hasAddressExtension reports a name that may be a mailbox spelled another
// way: + and % can start an extension or stand in for the @ of a login.
func hasAddressExtension(s string) bool {
return strings.ContainsAny(s, "+%")
}
func (r *cpanelMailboxResolver) refreshSnapshot(now time.Time) {
if !r.snapshotAt.IsZero() && now.Sub(r.snapshotAt) <= mailboxResolverTTL {
return
}
// Before its first read the lookup counts as up, so a first failure is news
// and a first good read is not.
wasDown := !r.snapshotAt.IsZero() && !r.snapshotUsable
r.snapshotAt = now
snapshot, err := r.src.Inventory()
// A host taking logins has accounts and domains. A read without them says
// nothing about the names asked, and outage traffic is never evidence.
r.snapshot = snapshot
r.snapshotUsable = err == nil && len(snapshot.Accounts) > 0 && len(snapshot.Domains) > 0
r.typoIndex = newMailboxTypoIndex(snapshot)
// Lists were read for the previous ownership and account identities. Even
// a younger list cannot answer for domains introduced by this snapshot.
clear(r.accounts)
switch {
case wasDown && r.snapshotUsable:
csmlog.Info("mailbox lookup available again")
case !wasDown && !r.snapshotUsable:
if err == nil {
err = errors.New("no accounts or domains listed")
}
csmlog.Warn("mailbox lookup unavailable, failed logins stay unknown", "err", err)
}
}
func (r *cpanelMailboxResolver) accountLists(account string) (map[string][]string, bool) {
if cached, ok := r.accounts[account]; ok {
return cached.lists, cached.usable
}
// Evicting live entries lets rotating login names force repeated reads.
// Capacity is available again when the inventory expires.
if len(r.accounts) >= maxResolverAccounts {
return nil, false
}
entry := accountMailboxes{}
if lists, complete, err := r.src.Mailboxes(account); err == nil && complete {
entry.lists, entry.usable = lists, true
}
r.accounts[account] = entry
return entry.lists, entry.usable
}
package daemon
import (
"fmt"
"slices"
"github.com/pidginhost/csm/internal/antispam/dovecot"
"github.com/pidginhost/csm/internal/antispam/learnwatch"
"github.com/pidginhost/csm/internal/checks"
)
// liveMailboxSource reads cPanel's own files. It holds no lock: the resolver
// serialises its calls.
type liveMailboxSource struct {
source learnwatch.Source
inventory func() (checks.HostingSnapshot, error)
// accounts and domains are the last Inventory's. Only these homes are
// opened and only these domains are read.
accounts map[string]learnwatch.Account
domains map[string]string
}
func newLiveMailboxSource(accounts learnwatch.Source) cpanelMailboxSource {
return &liveMailboxSource{source: accounts, inventory: checks.HostingInventory}
}
func (s *liveMailboxSource) Inventory() (checks.HostingSnapshot, error) {
accounts, _, err := learnwatch.Accounts(s.source)
if err != nil {
return checks.HostingSnapshot{}, err
}
s.accounts = make(map[string]learnwatch.Account, len(accounts))
for _, a := range accounts {
s.accounts[a.Name] = a
}
snapshot, err := s.inventory()
s.domains = snapshot.Domains
return snapshot, err
}
func (s *liveMailboxSource) Mailboxes(account string) (map[string][]string, bool, error) {
a, ok := s.accounts[account]
if !ok {
return nil, false, fmt.Errorf("account %q is not in the account list", account)
}
home, err := learnwatch.OpenHome(a)
if err != nil {
return nil, false, err
}
defer func() { _ = home.Close() }()
boxes, skips, err := home.DomainMailboxes(s.hostedDomains(account))
if err != nil {
return nil, false, err
}
lists := map[string][]string{}
for _, b := range boxes {
domain := normalizeSprayMailbox(b.Domain)
lists[domain] = append(lists[domain], normalizeSprayMailbox(b.Box))
}
// Folding can reorder a list and merge entries.
for domain, list := range lists {
slices.Sort(list)
lists[domain] = slices.Compact(list)
}
return lists, len(skips) == 0, nil
}
// hostedDomains lists the valid domains the last Inventory gave account, in
// its lower-case spelling. ~/etc is not listed.
//
// ponytail: a directory spelled differently from its domain is not seen, so its
// mailboxes answer Missing; list ~/etc for case variants if a host ever has one.
func (s *liveMailboxSource) hostedDomains(account string) []string {
var domains []string
for domain, owner := range s.domains {
if owner == account && dovecot.ValidDomain(domain) {
domains = append(domains, domain)
}
}
slices.Sort(domains)
return domains
}
package daemon
import (
"errors"
"fmt"
"os"
"slices"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/store"
"github.com/pidginhost/csm/internal/webui"
)
// compromiseEvidence says what a trigger saw. Sending evidence means abuse is
// already going out; login evidence means only that a password was cracked.
type compromiseEvidence uint8
const (
evidenceSending compromiseEvidence = iota + 1
evidenceLogin
)
// mailboxCompromise is one trigger's request to stop an abused credential.
// Subject is a mailbox ("user@domain"), a bare hosting account (no '@' or
// '.'), or a domain whose owning account is held.
type mailboxCompromise struct {
Subject string
Ref string
Check string
Reason string
Evidence compromiseEvidence
At time.Time
Previous store.MailboxSuspension
}
// mailBackend is the mail server's side of a suspension. suspend fills Kind,
// Account, Ref and Applied; the caller stamps the rest of the record.
type mailBackend interface {
name() string
suspend(c mailboxCompromise) (store.MailboxSuspension, error)
release(rec store.MailboxSuspension, actor string) error
active(recs []store.MailboxSuspension) (map[string]bool, error)
}
var activeMailBackend mailBackend = cpanelMailBackend{}
// Keep the server mutation, record update and evidence reset in one critical
// section. References include waiters so a lock cannot be replaced underneath
// them; idle subjects do not accumulate in this map.
var mailResponseLocks = struct {
sync.Mutex
entries map[string]*mailResponseLock
}{entries: make(map[string]*mailResponseLock)}
type mailResponseLock struct {
mu sync.Mutex
refs int
}
func lockMailResponse(key string) func() {
mailResponseLocks.Lock()
l := mailResponseLocks.entries[key]
if l == nil {
l = &mailResponseLock{}
mailResponseLocks.entries[key] = l
}
l.refs++
mailResponseLocks.Unlock()
l.mu.Lock()
return func() {
l.mu.Unlock()
mailResponseLocks.Lock()
l.refs--
if l.refs == 0 {
delete(mailResponseLocks.entries, key)
}
mailResponseLocks.Unlock()
}
}
// respondToCompromise is the one entry point for stopping an abused mail
// credential. It honours the auto-response switch and dry-run, never repeats
// an active suspension, and after a release acts again only on evidence
// dated after it.
func respondToCompromise(cfg *config.Config, c mailboxCompromise) (stopped bool, findings []alert.Finding) {
now := time.Now()
stamp := now.Format("2006-01-02 15:04:05")
if cfg == nil || !cfg.AutoResponse.Enabled {
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: would suspend %s (auto_response disabled)\n", stamp, c.Subject)
return false, nil
}
if cfg.AutoResponseDryRunEnabled() {
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: would suspend %s (dry-run)\n", stamp, c.Subject)
actionlog.Write(actionlog.Record{Op: dryRunMailOp(c.Subject), Target: c.Subject, Reason: c.Reason, Result: actionlog.DryRun})
return false, nil
}
backend := activeMailBackend
id := store.MailboxSuspensionID(backend.name(), c.Subject)
if b, ok := backend.(stalwartMailBackend); ok {
var err error
c.Ref, err = b.resolve(c.Subject)
if err != nil {
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: could not resolve %s: %v\n", stamp, c.Subject, err)
if c.Evidence == evidenceLogin {
return false, []alert.Finding{loginResponseFinding(c, store.MailboxSuspension{Backend: backend.name()}, err)}
}
return false, nil
}
id = stalwartSuspensionID(c.Ref)
}
unlock := lockMailResponse(id)
defer unlock()
now = time.Now()
db := store.Global()
var uncertain store.MailboxSuspension
if db != nil {
prev, ok, err := db.GetMailboxSuspension(id)
switch {
case err != nil:
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: reading suspension record for %s: %v\n", stamp, c.Subject, err)
case ok && prev.ReleasedAt == nil:
// Trust the row only while the mail server agrees: a suspension
// lifted outside CSM, or one whose state cannot be read, must
// not stop a new response. Suspending again is idempotent.
live, statusErr := backend.active([]store.MailboxSuspension{prev})
still, known := live[prev.ID]
if known && still {
return true, nil
}
if statusErr != nil {
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: %v; suspending %s again\n", stamp, statusErr, c.Subject)
}
if !known {
uncertain = prev
}
case ok && !c.At.After(*prev.ReleasedAt):
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: %s was released at %s, after this evidence; not suspending again\n",
stamp, c.Subject, prev.ReleasedAt.Format(time.RFC3339))
return false, nil
}
}
c.Previous = uncertain
applied, err := backend.suspend(c)
if errors.Is(err, errMailAlreadyRestricted) {
return true, nil
}
if err != nil {
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: could not stop %s: %v\n", stamp, c.Subject, err)
if c.Evidence == evidenceLogin {
return false, []alert.Finding{loginResponseFinding(c, store.MailboxSuspension{Backend: backend.name()}, err)}
}
return false, nil
}
if uncertain.Account == applied.Account {
parts := slices.Clone(uncertain.Applied)
for _, part := range applied.Applied {
if !slices.Contains(parts, part) {
parts = append(parts, part)
}
}
applied.Applied = parts
}
if slices.Contains(applied.Applied, store.AppliedHoldOutgoing) {
applied.Kind = store.SuspensionKindAccount
}
rec := store.MailboxSuspension{
ID: id,
Backend: backend.name(),
Subject: c.Subject,
Kind: applied.Kind,
Account: applied.Account,
Ref: applied.Ref,
Applied: applied.Applied,
Check: c.Check,
Reason: c.Reason,
SuspendedAt: now,
}
if db != nil {
if err := db.PutMailboxSuspension(rec); err != nil {
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: recording suspension of %s: %v\n", stamp, c.Subject, err)
}
}
if c.Evidence == evidenceLogin {
return true, []alert.Finding{loginResponseFinding(c, rec, nil)}
}
return true, nil
}
// loginResponseFinding tells staff what CSM did about a cracked password and
// what they still have to do. Sending triggers need none: their own finding
// already says the mailbox is suspended.
func loginResponseFinding(c mailboxCompromise, rec store.MailboxSuspension, err error) alert.Finding {
mailbox, domain, tenant := splitMailAccount(c.Subject)
if rec.Account != "" {
tenant = rec.Account
} else if tenant == "" && domain != "" && rec.Backend != "stalwart" {
tenant = lookupCPanelUser(domain)
}
f := alert.Finding{
Severity: alert.Critical,
Check: "email_mailbox_response",
Mailbox: mailbox,
Domain: domain,
TenantID: tenant,
Timestamp: time.Now(),
}
switch {
case err != nil:
f.Message = fmt.Sprintf("Could not suspend %s after its password was cracked: %v", c.Subject, err)
f.Details = "Protection could not be confirmed. Change its password now."
case rec.Kind == store.SuspensionKindAccount && mailbox == "":
f.Message = fmt.Sprintf("Outgoing mail held for cPanel account %s after its own password was cracked", rec.Account)
f.Details = "That password unlocks every mailbox and the cPanel login. Change the cPanel password, then release the hold on the Email page, Suspended tab."
default:
f.Message = fmt.Sprintf("Mailbox %s suspended after its password was cracked", c.Subject)
switch {
case slices.Contains(rec.Applied, store.AppliedDisableSignIn):
f.Details = "Sign-ins are stopped, which also stops sending. Sessions that are already open stay open until they end."
case !slices.Contains(rec.Applied, store.AppliedSuspendLogin):
f.Details = "Only outgoing mail is stopped; logins may still work."
case !slices.Contains(rec.Applied, store.AppliedSuspendOutgoing):
f.Details = "Only logins are stopped; outgoing mail was not suspended."
default:
f.Details = "Logins and outgoing mail are stopped."
}
f.Details += " Have the customer change the mailbox password, then release it on the Email page, Suspended tab."
}
return f
}
// hostHasMailBackend reports whether this host has a mail server CSM can
// suspend accounts on. selectMailBackend replaces it on a Stalwart host. var
// so tests can stand in for a cPanel host.
var hostHasMailBackend = func() bool { return platform.Detect().IsCPanel() }
// stolenLoginResponses suspends the mailbox behind each Critical login after
// brute force. High findings (an established multi-mailbox source, spray
// advisories) stay alert-only.
func stolenLoginResponses(cfg *config.Config, findings []alert.Finding) []alert.Finding {
var out []alert.Finding
for _, f := range findings {
if f.Check != "mail_account_compromised" || f.Severity != alert.Critical {
continue
}
subject := f.Mailbox
// These spellings belong to cPanel. On Stalwart a bare login
// containing '+' or '%' must not name another account's mailbox.
if activeMailBackend.name() == "cpanel" {
subject = loginMailbox(subject)
}
if subject == "" || !hostHasMailBackend() {
continue
}
_, response := respondToCompromise(cfg, mailboxCompromise{
Subject: subject, Check: f.Check, Reason: f.Message, Evidence: evidenceLogin, At: f.Timestamp,
})
out = append(out, response...)
}
return out
}
// dryRunMailOp names the operation a dry run would have performed.
func dryRunMailOp(subject string) string {
if strings.Contains(subject, "@") {
return "respond.suspend_mailbox"
}
return "respond.hold_outgoing_mail"
}
// loginMailbox turns cPanel's alternative login forms, user+domain and
// user%domain, into the mailbox address they name.
func loginMailbox(name string) string {
if strings.Contains(name, "@") {
return name
}
if i := strings.LastIndexAny(name, "+%"); i > 0 && strings.Contains(name[i+1:], ".") {
return name[:i] + "@" + name[i+1:]
}
return name
}
// releasedOutsideCSM marks a suspension the mail server no longer applies,
// lifted by someone outside CSM.
const releasedOutsideCSM = "outside CSM"
var errSuspensionAlreadyReleased = errors.New("mailbox suspension already released")
// mailboxSuspensionStatus is a stored suspension with its live state.
type mailboxSuspensionStatus struct {
store.MailboxSuspension
Active bool
StatusUnknown bool
}
// listMailboxSuspensions returns every stored suspension with the mail
// server's live state. One the server no longer applies is marked released
// outside CSM; one whose state cannot be read stays as it was. Only the
// backend that made a row can read it: after the host's mail server changes,
// the other backend's rows show an unknown state.
func listMailboxSuspensions() ([]mailboxSuspensionStatus, error) {
db := store.Global()
if db == nil {
return nil, errors.New("state store not open")
}
rows, err := db.ListMailboxSuspensions()
if err != nil {
return nil, err
}
backend := activeMailBackend
var open []store.MailboxSuspension
for _, r := range rows {
if r.ReleasedAt == nil && r.Backend == backend.name() {
open = append(open, r)
}
}
live := map[string]bool{}
if len(open) > 0 {
var statusErr error
if live, statusErr = backend.active(open); statusErr != nil {
fmt.Fprintf(os.Stderr, "[%s] mailbox suspensions: %v\n", time.Now().Format("2006-01-02 15:04:05"), statusErr)
}
}
out := make([]mailboxSuspensionStatus, 0, len(rows))
for _, r := range rows {
unlock := lockMailResponse(r.ID)
current, found, readErr := db.GetMailboxSuspension(r.ID)
if readErr != nil {
unlock()
return nil, readErr
}
if !found {
unlock()
continue
}
st := mailboxSuspensionStatus{MailboxSuspension: current}
if current.ReleasedAt == nil {
active, known := live[r.ID]
switch {
case !current.SuspendedAt.Equal(r.SuspendedAt):
// The batched status predates a new suspension. Never apply
// it to that generation, even if it reported no restrictions.
st.StatusUnknown = true
case !known:
st.StatusUnknown = true
case active:
st.Active = true
default:
now := time.Now()
if err := finishMailboxRelease(db, current, now, releasedOutsideCSM); err != nil {
st.StatusUnknown = true
break
}
at := now
st.ReleasedAt, st.ReleasedBy = &at, releasedOutsideCSM
}
}
unlock()
out = append(out, st)
}
return out, nil
}
// releaseMailboxSuspension lifts a suspension CSM made, at an operator's
// request, and forgets the sending evidence behind it so sends from before
// the release cannot suspend the mailbox again.
func releaseMailboxSuspension(id, actor string) error {
unlock := lockMailResponse(id)
defer unlock()
db := store.Global()
if db == nil {
return errors.New("state store not open")
}
rec, ok, err := db.GetMailboxSuspension(id)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("%w: %s", store.ErrMailboxSuspensionNotFound, id)
}
if rec.ReleasedAt != nil {
return errSuspensionAlreadyReleased
}
backend := activeMailBackend
if rec.Backend != backend.name() {
return fmt.Errorf("suspension was made through %s; this host uses %s", rec.Backend, backend.name())
}
if err := backend.release(rec, actor); err != nil {
return err
}
return finishMailboxRelease(db, rec, time.Now(), actor)
}
// The caller holds the subject lock through both evidence and record updates.
func finishMailboxRelease(db *store.DB, rec store.MailboxSuspension, now time.Time, actor string) error {
if err := forgetSenderEvidence(rec.Subject, now); err != nil {
return err
}
forgetCloudRelayEvidence(rec.Subject)
if err := db.MarkMailboxSuspensionReleased(rec.ID, now, actor); err != nil {
return err
}
if !strings.Contains(rec.Subject, "@") && strings.Contains(rec.Subject, ".") {
// The spam-outbreak governor held this domain; restart its dedup
// window so the same blast does not hold it again at once.
recordRecentOutgoingMailHold(rec.Subject)
}
return nil
}
// mailboxSuspensionsAPI serves the web UI's suspension list and Release.
type mailboxSuspensionsAPI struct{}
func (mailboxSuspensionsAPI) ListSuspensions() ([]webui.MailboxSuspensionRow, error) {
rows, err := listMailboxSuspensions()
if err != nil {
return nil, err
}
out := make([]webui.MailboxSuspensionRow, 0, len(rows))
for _, r := range rows {
out = append(out, webui.MailboxSuspensionRow{MailboxSuspension: r.MailboxSuspension, Active: r.Active, StatusUnknown: r.StatusUnknown})
}
return out, nil
}
func (mailboxSuspensionsAPI) ReleaseSuspension(id, actor string) error {
err := releaseMailboxSuspension(id, actor)
if errors.Is(err, errSuspensionAlreadyReleased) {
return webui.ErrSuspensionReleased
}
return err
}
package daemon
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"os/exec"
"path/filepath"
"regexp"
"slices"
"strings"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/store"
"github.com/pidginhost/csm/internal/systemdrun"
)
// uapiTimeout bounds one uapi call. cPanel's API binary loads its Perl stack
// and the account's data on every invocation, so several seconds is normal,
// and a hung call must not stall the Exim log watcher for long.
const uapiTimeout = 20 * time.Second
// cPanel ships uapi here; the /usr/bin symlink is not present on every host
// and the service PATH is minimal, so the absolute path is used.
const uapiPath = "/usr/local/cpanel/bin/uapi"
// uapiExec runs cPanel's uapi with the given arguments and returns its
// JSON stdout. A var so tests can observe calls without cPanel.
var uapiExec = func(ctx context.Context, args ...string) ([]byte, error) {
return runUAPI(ctx, exec.LookPath, uapiCommand, args...)
}
func uapiCommand(ctx context.Context, name string, args ...string) ([]byte, error) {
// #nosec G204 -- name is systemd-run or a cPanel API binary; args are
// fixed API words plus a cPanel-managed account and an authenticated Exim
// identity. No shell interprets them; systemdrun escapes variable expansion.
cmd := exec.CommandContext(ctx, name, args...)
// Killing the command need not close pipes inherited by children.
// Bound the drain too, so those children cannot stall the log watcher.
cmd.WaitDelay = time.Second
out, err := cmd.Output()
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
err = fmt.Errorf("%w: %s", err, truncateDaemon(strings.TrimSpace(string(exitErr.Stderr)), 200))
}
return out, err
}
// runUAPI starts uapi as a transient unit forked by PID 1. uapi writes the
// suspension markers under the account home and into cPanel's own caches,
// which the daemon's sandbox does not grant; where systemd-run is unavailable
// the command runs directly.
func runUAPI(ctx context.Context, lookPath systemdrun.LookPathFunc, run systemdrun.RunnerFunc, args ...string) ([]byte, error) {
opt := systemdrun.Options{Pipe: true}
if deadline, ok := ctx.Deadline(); ok {
opt.RuntimeMax = time.Until(deadline)
}
return systemdrun.Run(ctx, lookPath, run, opt, uapiPath, args...)
}
// uapiValue encodes v for uapi's command line, which URI-decodes every value:
// a + or % in a mailbox would otherwise reach cPanel as something else. A
// space goes in as %20, which reads the same whether or not the decoder takes
// + for a space.
func uapiValue(v string) string {
return strings.ReplaceAll(url.QueryEscape(v), "+", "%20")
}
// uapiResultError reads the outcome of a `uapi --output=json` call. uapi exits
// non-zero on an API failure, so the JSON status is the source of truth and
// the exit error only adds context when no reply can be read.
func uapiResultError(out []byte) error {
var reply struct {
Result struct {
Status int `json:"status"`
Errors []string `json:"errors"`
} `json:"result"`
}
if err := json.Unmarshal(out, &reply); err != nil {
return fmt.Errorf("unreadable uapi reply: %s", truncateDaemon(strings.TrimSpace(string(out)), 200))
}
if reply.Result.Status == 1 {
return nil
}
if len(reply.Result.Errors) == 0 {
return errors.New("uapi reported failure without an error message")
}
return errors.New(strings.Join(reply.Result.Errors, "; "))
}
// maybeSuspendMailbox stops one abused credential seen sending: it suspends
// the mailbox's logins, which also ends its SMTP authentication, and its
// outgoing mail. at is when the abuse was logged. It returns true when
// sending from the mailbox was stopped.
func maybeSuspendMailbox(cfg *config.Config, mailbox, check, reason string, at time.Time) bool {
stopped, _ := respondToCompromise(cfg, mailboxCompromise{
Subject: mailbox, Check: check, Reason: reason, Evidence: evidenceSending, At: at,
})
return stopped
}
// cpanelMailBackend suspends through cPanel's uapi and whmapi1.
type cpanelMailBackend struct{}
func (cpanelMailBackend) name() string { return "cpanel" }
func (cpanelMailBackend) suspend(c mailboxCompromise) (store.MailboxSuspension, error) {
switch {
case strings.Contains(c.Subject, "@"):
return suspendCPanelMailbox(c)
case c.Evidence == evidenceLogin:
// A cracked login names a mailbox or the account itself, never a
// domain: login evidence must not reach a domain-wide hold.
return holdBareCPanelAccount(c.Subject)
case strings.Contains(c.Subject, "."):
// A domain names its owning account; only the account hold fits.
return holdOwnerAccount(c.Subject)
default:
return holdBareCPanelAccount(c.Subject)
}
}
// release undoes exactly what was applied, last first. A retry may repeat
// successful unsuspends, which cPanel accepts even when already lifted.
func (cpanelMailBackend) release(rec store.MailboxSuspension, actor string) error {
for i := len(rec.Applied) - 1; i >= 0; i-- {
var err error
switch part := rec.Applied[i]; part {
case store.AppliedSuspendLogin, store.AppliedSuspendOutgoing:
err = runMailboxRelease(rec.Account, rec.Subject, "un"+part, actor)
case store.AppliedHoldOutgoing:
err = releaseCPanelAccountHold(rec.Account, rec.Subject, actor)
default:
err = fmt.Errorf("unknown suspension part %q", part)
}
if err != nil {
return err
}
}
return nil
}
// active reads cPanel's live state: the outgoing hold list for account holds
// and one mailbox listing per account for mailbox suspensions. A record whose
// status could not be read is left out of the map.
func (cpanelMailBackend) active(recs []store.MailboxSuspension) (map[string]bool, error) {
live := make(map[string]bool, len(recs))
unknown := make(map[string]bool)
byAccount := map[string][]store.MailboxSuspension{}
var errs []error
for _, r := range recs {
live[r.ID] = false
if slices.Contains(r.Applied, store.AppliedHoldOutgoing) {
held, err := outgoingMailHoldState(r.Account)
if err != nil {
errs = append(errs, fmt.Errorf("outgoing hold list: %w", err))
unknown[r.ID] = true
}
live[r.ID] = held
}
if slices.Contains(r.Applied, store.AppliedSuspendLogin) || slices.Contains(r.Applied, store.AppliedSuspendOutgoing) {
byAccount[r.Account] = append(byAccount[r.Account], r)
}
}
for account, rows := range byAccount {
flags, err := cpanelMailboxFlags(account)
if err != nil {
errs = append(errs, fmt.Errorf("mailbox status for cPanel account %s: %w", account, err))
for _, r := range rows {
unknown[r.ID] = true
}
continue
}
for _, r := range rows {
f, listed := flags[strings.ToLower(r.Subject)]
live[r.ID] = live[r.ID] || listed && f.stillApplied(r.Applied)
}
}
for id := range unknown {
if !live[id] {
delete(live, id)
}
}
return live, errors.Join(errs...)
}
// mailboxFlags is a mailbox's suspension state as cPanel reports it.
type mailboxFlags struct{ login, outgoing bool }
func (f mailboxFlags) stillApplied(applied []string) bool {
for _, part := range applied {
if part == store.AppliedSuspendLogin && f.login || part == store.AppliedSuspendOutgoing && f.outgoing {
return true
}
}
return false
}
// cpanelFlag reads a cPanel 0/1 flag sent as a number, a string or a bool.
type cpanelFlag bool
func (f *cpanelFlag) UnmarshalJSON(b []byte) error {
switch strings.Trim(string(b), `"`) {
case "1", "true":
*f = true
case "0", "false":
*f = false
default:
return fmt.Errorf("unexpected cPanel flag %s", b)
}
return nil
}
// cpanelMailboxFlags lists one account's mailboxes with their suspensions.
// get_restrictions=1 makes cPanel include the outgoing suspension flag.
func cpanelMailboxFlags(account string) (map[string]mailboxFlags, error) {
ctx, cancel := context.WithTimeout(context.Background(), uapiTimeout)
defer cancel()
out, execErr := uapiExec(ctx, "--output=json", "--user="+account, "Email", "list_pops_with_disk", "get_restrictions=1")
if err := uapiResultError(out); err != nil {
if execErr != nil {
err = fmt.Errorf("%w: %v", execErr, err)
}
return nil, err
}
var reply struct {
Result struct {
Data []struct {
Email string `json:"email"`
SuspendedLogin *cpanelFlag `json:"suspended_login"`
SuspendedOutgoing *cpanelFlag `json:"suspended_outgoing"`
} `json:"data"`
} `json:"result"`
}
if err := json.Unmarshal(out, &reply); err != nil {
return nil, fmt.Errorf("unreadable mailbox listing: %w", err)
}
if reply.Result.Data == nil {
return nil, errors.New("mailbox listing has no data")
}
flags := make(map[string]mailboxFlags, len(reply.Result.Data))
for _, m := range reply.Result.Data {
if m.Email == "" || m.SuspendedLogin == nil || m.SuspendedOutgoing == nil {
return nil, errors.New("mailbox listing has incomplete restrictions")
}
flags[strings.ToLower(m.Email)] = mailboxFlags{login: bool(*m.SuspendedLogin), outgoing: bool(*m.SuspendedOutgoing)}
}
return flags, nil
}
// runMailboxRelease lifts one uapi Email suspension at an operator's request.
func runMailboxRelease(user, mailbox, fn, actor string) error {
command, err := runUAPIEmail(user, mailbox, fn)
rec := actionlog.Record{
Op: "operate.release_mailbox",
Action: fn,
Actor: actionlog.WebUI,
ActorDetail: actor,
Target: mailbox,
Account: user,
Command: command,
Result: actionlog.Applied,
}
if err != nil {
rec.Result, rec.Error = actionlog.Failed, err.Error()
}
actionlog.Write(rec)
return err
}
// releaseCPanelAccountHold lifts an account's outgoing hold at an
// operator's request.
func releaseCPanelAccountHold(user, subject, actor string) error {
if user == "" {
return fmt.Errorf("no cPanel account recorded for %s", subject)
}
unlock := lockMailResponse("cpanel-account:" + user)
defer unlock()
out, err := whmapi1ReleaseExec(user)
if err == nil {
err = whmapi1ResultError(out)
}
rec := actionlog.Record{
Op: "operate.release_mailbox",
Action: "release_outgoing_email",
Actor: actionlog.WebUI,
ActorDetail: actor,
Target: subject,
Account: user,
Command: []string{"whmapi1", "release_outgoing_email", "user=" + user, "--output=json"},
Result: actionlog.Applied,
}
if err != nil {
err = fmt.Errorf("whmapi1 release_outgoing_email for %s: %w: %s", user, err, truncateDaemon(strings.TrimSpace(string(out)), 200))
rec.Result, rec.Error = actionlog.Failed, err.Error()
}
actionlog.Write(rec)
return err
}
// A successful process exit is not evidence that WHM accepted the operation.
func whmapi1ResultError(out []byte) error {
var reply struct {
Metadata struct {
Result int `json:"result"`
Reason string `json:"reason"`
} `json:"metadata"`
}
if err := json.Unmarshal(out, &reply); err != nil {
return fmt.Errorf("unreadable whmapi1 reply: %w", err)
}
if reply.Metadata.Result == 1 {
return nil
}
return fmt.Errorf("whmapi1 refused operation: %s", reply.Metadata.Reason)
}
// suspendCPanelMailbox suspends one mailbox's logins and outgoing mail. An
// account-wide hold punishes every mailbox under the account for one stolen
// password, so it is the fallback only for sending evidence, when neither
// suspension applies or no cPanel account owns the mailbox.
func suspendCPanelMailbox(c mailboxCompromise) (store.MailboxSuspension, error) {
stamp := time.Now().Format("2006-01-02 15:04:05")
domain := extractDomainFromEmail(c.Subject)
user := lookupCPanelUser(domain)
if user == "" {
if c.Evidence == evidenceLogin {
return store.MailboxSuspension{}, fmt.Errorf("no cPanel account owns %s", domain)
}
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: no cPanel account owns %s, falling back to the account hold\n", stamp, domain)
return holdOwnerAccount(c.Subject)
}
var applied []string
var errs []error
flags, statusErr := cpanelMailboxFlags(user)
before := flags[strings.ToLower(c.Subject)]
if statusErr != nil {
// Protection comes first: an unreadable state must not leave a
// cracked mailbox open. CSM then records the suspension as its own,
// even if someone else had already applied it.
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: mailbox state for %s unreadable (%v); suspending anyway\n", stamp, c.Subject, statusErr)
}
for _, fn := range []string{store.AppliedSuspendLogin, store.AppliedSuspendOutgoing} {
owned := c.Previous.Account == user && slices.Contains(c.Previous.Applied, fn)
if statusErr == nil && before.stillApplied([]string{fn}) {
if owned {
applied = append(applied, fn)
}
continue
}
if err := runMailboxSuspension(user, c.Subject, fn, c.Reason); err != nil {
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: uapi Email %s failed for %s: %v\n", stamp, fn, c.Subject, err)
errs = append(errs, fmt.Errorf("%s: %w", fn, err))
continue
}
applied = append(applied, fn)
}
if len(applied) > 0 {
fmt.Fprintf(os.Stderr, "[%s] AUTO-SUSPEND: mailbox %s suspended (cPanel user %s)\n", stamp, c.Subject, user)
return store.MailboxSuspension{Kind: store.SuspensionKindMailbox, Account: user, Applied: applied}, nil
}
if before.login || before.outgoing {
return store.MailboxSuspension{}, errMailAlreadyRestricted
}
if c.Evidence == evidenceLogin {
return store.MailboxSuspension{}, errors.Join(errs...)
}
return holdOwnerAccount(c.Subject)
}
// holdOwnerAccount holds outgoing mail for the account owning a domain or a
// mailbox's domain.
func holdOwnerAccount(subject string) (store.MailboxSuspension, error) {
domain := subject
if at := strings.LastIndexByte(domain, '@'); at >= 0 {
domain = domain[at+1:]
}
user := lookupCPanelUser(domain)
unlock := lockMailResponse("cpanel-account:" + user)
defer unlock()
if err := checkCPanelHoldOwnership(user); err != nil {
return store.MailboxSuspension{}, err
}
if !autoSuspendOutgoingMail(subject) {
return store.MailboxSuspension{}, fmt.Errorf("outgoing mail hold for %s failed", subject)
}
return store.MailboxSuspension{Kind: store.SuspensionKindAccount, Account: user, Applied: []string{store.AppliedHoldOutgoing}}, nil
}
// cpanelUsersDir lists one file per cPanel account. var so tests can point it
// at a fixture.
var cpanelUsersDir = "/var/cpanel/users"
// cpanelUserRe bounds a bare login to a cPanel account name before it reaches
// a path or a whmapi1 argument.
var cpanelUserRe = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,31}$`)
var errMailAlreadyRestricted = errors.New("mail is already restricted outside this suspension")
func checkCPanelHoldOwnership(user string) error {
held, err := outgoingMailHoldState(user)
if err != nil {
return fmt.Errorf("cannot read outgoing mail hold for %s: %w", user, err)
}
if held {
return errMailAlreadyRestricted
}
return nil
}
// holdBareCPanelAccount holds the account's outgoing mail when a login named
// the cPanel account itself: that password unlocks every mailbox.
func holdBareCPanelAccount(user string) (store.MailboxSuspension, error) {
if !cpanelUserRe.MatchString(user) {
return store.MailboxSuspension{}, fmt.Errorf("%q is not a cPanel account name", user)
}
if _, err := os.Stat(filepath.Join(cpanelUsersDir, user)); err != nil {
return store.MailboxSuspension{}, fmt.Errorf("no cPanel account %s", user)
}
unlock := lockMailResponse("cpanel-account:" + user)
defer unlock()
if err := checkCPanelHoldOwnership(user); err != nil {
return store.MailboxSuspension{}, err
}
if !holdCPanelAccount(user) {
return store.MailboxSuspension{}, fmt.Errorf("outgoing mail hold for cPanel account %s failed", user)
}
return store.MailboxSuspension{Kind: store.SuspensionKindAccount, Account: user, Applied: []string{store.AppliedHoldOutgoing}}, nil
}
// runUAPIEmail runs one uapi Email function on a mailbox and returns the
// argv it ran.
func runUAPIEmail(user, mailbox, fn string) ([]string, error) {
args := []string{"--output=json", "--user=" + user, "Email", fn, "email=" + uapiValue(mailbox)}
ctx, cancel := context.WithTimeout(context.Background(), uapiTimeout)
defer cancel()
out, execErr := uapiExec(ctx, args...)
err := uapiResultError(out)
if err != nil && execErr != nil {
err = fmt.Errorf("%w: %v", execErr, err)
}
return append([]string{uapiPath}, args...), err
}
// runMailboxSuspension applies one uapi Email suspension and records it.
func runMailboxSuspension(user, mailbox, fn, reason string) error {
command, err := runUAPIEmail(user, mailbox, fn)
rec := actionlog.Record{
Op: "respond.suspend_mailbox",
Action: fn,
Target: mailbox,
Account: user,
Reason: reason,
Command: command,
Undo: fmt.Sprintf("%s %s Email un%s %s", uapiPath, mailboxShellArg("--user="+user), fn, mailboxShellArg("email="+uapiValue(mailbox))),
Result: actionlog.Applied,
}
if err != nil {
rec.Result = actionlog.Failed
rec.Error = err.Error()
}
actionlog.Write(rec)
return err
}
// Undo is pasted into an operator's shell; the parser's authenticated identity
// guarantee does not make mailbox punctuation safe for shell interpretation.
func mailboxShellArg(s string) string {
return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'"
}
package daemon
import "github.com/pidginhost/csm/internal/checks"
// maxTypoEdits is how far a login name can be from a real mailbox and still
// be that mailbox, counting local part and domain edits together.
const maxTypoEdits = 2
// editsWithin returns the optimal string alignment distance between a and b
// when it is at most k: an insert, delete, substitution or swap of adjacent
// bytes costs one, and no byte is edited twice. Equal leading bytes are always
// aligned, and each edit spends the budget, so the walk stays in the band k
// wide around the diagonal and gives up when the budget runs out.
//
// ponytail: branches four ways per edit, cheap at k = 2; a banded table if k
// ever grows.
func editsWithin(a, b string, k int) (int, bool) {
if k <= 0 {
return 0, k == 0 && a == b
}
for a != "" && b != "" && a[0] == b[0] {
a, b = a[1:], b[1:]
}
if a == "" || b == "" {
return len(a) + len(b), len(a)+len(b) <= k
}
if len(a)-len(b) > k || len(b)-len(a) > k {
return 0, false
}
best, found := 0, false
edit := func(x, y string) {
if d, ok := editsWithin(x, y, k-1); ok {
// Only a cheaper answer is worth finding from here on.
best, found, k = d+1, true, d
}
}
edit(a[1:], b[1:])
edit(a[1:], b)
edit(a, b[1:])
if len(a) > 1 && len(b) > 1 && a[0] == b[1] && a[1] == b[0] {
edit(a[2:], b[2:])
}
return best, found
}
// mailboxTypoIndex holds the hosted domains by byte length, the unresolved ones
// too: a domain within k edits differs in length by at most k. It is built with
// each snapshot and lookups only read it, so typed names add nothing to it.
type mailboxTypoIndex map[int][]string
func newMailboxTypoIndex(snapshot checks.HostingSnapshot) mailboxTypoIndex {
index := mailboxTypoIndex{}
for domain := range snapshot.Domains {
index[len(domain)] = append(index[len(domain)], domain)
}
for domain := range snapshot.Unresolved {
index[len(domain)] = append(index[len(domain)], domain)
}
return index
}
// typoMatch is the nearest real mailbox seen so far. Ties go to the smaller
// address, so the answer does not depend on the order of the scan.
type typoMatch struct {
address string
edits int
}
// scan weighs the mailboxes of domain, the owner name among them, against
// local. domainEdits is what the domain already spent of maxTypoEdits.
func (m *typoMatch) scan(local, domain, owner string, locals []string, domainEdits int) {
weigh := func(candidate string) {
edits, near := editsWithin(local, candidate, maxTypoEdits-domainEdits)
if !near {
return
}
edits += domainEdits
address := candidate + "@" + domain
if m.address == "" || edits < m.edits || (edits == m.edits && address < m.address) {
m.address, m.edits = address, edits
}
}
for _, candidate := range locals {
weigh(candidate)
}
weigh(owner)
}
// answer gives the match, or otherwise when nothing was near.
func (m *typoMatch) answer(otherwise sprayMailbox) sprayMailbox {
if m.address == "" {
return otherwise
}
// Truncating the real address would create a different mailbox key.
if len(m.address) > maxSprayMailboxBytes {
return sprayMailbox{name: otherwise.name, exist: mailboxExistenceUnknown}
}
return sprayMailbox{name: m.address, exist: mailboxExists}
}
// resolveTypo answers for a name no hosted domain lists as typed. The nearest
// mailbox within maxTypoEdits wins, whether on the typed domain, which is a
// candidate at no cost when hosted, or on a near hosted domain. A near domain
// whose mailboxes cannot be read might hold the mailbox meant, so it turns an
// answer of Missing into Unknown, but cannot outweigh a match. An extension
// name only reaches this on an unhosted domain, where it proves no absence.
func (r *cpanelMailboxResolver) resolveTypo(name, local, domain string) sprayMailbox {
var nearest typoMatch
otherwise := sprayMailbox{name: name, exist: mailboxMissing}
if hasAddressExtension(local) {
otherwise.exist = mailboxExistenceUnknown
}
// ponytail: linear in hosted domains of similar length for every name that
// is not an exact match, under the mutex; a deletion-neighbourhood index if
// a host with far more domains feels it.
for size := len(domain) - maxTypoEdits; size <= len(domain)+maxTypoEdits; size++ {
for _, candidate := range r.typoIndex[size] {
domainEdits, near := editsWithin(domain, candidate, maxTypoEdits)
if !near {
continue
}
owner, resolved := r.snapshot.Domains[candidate]
if !resolved {
otherwise.exist = mailboxExistenceUnknown
continue
}
lists, usable := r.accountLists(owner)
if !usable {
otherwise.exist = mailboxExistenceUnknown
continue
}
nearest.scan(local, candidate, owner, lists[candidate], domainEdits)
}
}
return nearest.answer(otherwise)
}
package daemon
import (
"fmt"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
"unicode/utf8"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/store"
)
// mailIPEntry tracks failed-auth timestamps, successful-login timestamps, the
// mailboxes seen on each side, and suppression state for one IP. The mailbox
// context keeps unrelated successful logins from hiding a mailbox attack.
type mailIPEntry struct {
times []time.Time
succ []time.Time
successAccounts map[string][]time.Time
failedAccounts map[string][]time.Time
// Long-horizon state for the slow-brute signal: bounded failure timestamps,
// bounded per-mailbox last-failure times, and the most recent failure not
// represented in the mailbox map. An unattributed or overflowed failure
// prevents the bounded named-mailbox set from exempting activity it cannot
// vouch for. Login trust lives in mailSpray.
slowTimes []time.Time
slowAccounts map[string]time.Time
slowUnvouched time.Time
// goodFirst/goodLast record, per mailbox, the earliest and most recent
// successful auth from this IP within mailGoodSourceTTL. They outlive the
// short failure window so an established legitimate sender (e.g. a working
// POP3 profile) is not mistaken for an attacker when a second misconfigured
// profile produces a burst of auth failures.
goodFirst map[string]time.Time
goodLast map[string]time.Time
suppressed time.Time
// suspectedSuppressed rate-limits the advisory mail_bruteforce_suspected
// finding independently of the auto-block clock, so a misconfigured client
// is surfaced once per window without firing repeatedly, yet a later
// escalation to a real spray can still reach the auto-block path.
suspectedSuppressed time.Time
lastSeen time.Time
}
// recordGoodAuth notes a successful auth for account at now, maintaining the
// established-sender window. A success after a gap longer than the TTL starts a
// fresh relationship. Caller must hold t.mu.
func (e *mailIPEntry) recordGoodAuth(account string, now time.Time) {
if account == "" {
return
}
if e.goodFirst == nil {
e.goodFirst = make(map[string]time.Time)
e.goodLast = make(map[string]time.Time)
}
if last, ok := e.goodLast[account]; !ok || now.Sub(last) > mailGoodSourceTTL {
e.goodFirst[account] = now
}
e.goodLast[account] = now
if len(e.goodLast) > mailGoodSourceMaxAccountsPerIP {
e.evictOldestGoodSource()
}
}
// evictOldestGoodSource drops the least-recently-successful mailbox so a single
// IP's good-source records stay bounded. Caller must hold t.mu.
func (e *mailIPEntry) evictOldestGoodSource() {
var oldestAcct string
var oldest time.Time
for acct, ts := range e.goodLast {
if oldestAcct == "" || ts.Before(oldest) {
oldestAcct, oldest = acct, ts
}
}
delete(e.goodLast, oldestAcct)
delete(e.goodFirst, oldestAcct)
}
// pruneGoodSource drops good-source records whose most recent success is older
// than cutoff (now - mailGoodSourceTTL). Caller must hold t.mu.
func (e *mailIPEntry) pruneGoodSource(cutoff time.Time) {
for account, last := range e.goodLast {
if last.Before(cutoff) {
delete(e.goodLast, account)
delete(e.goodFirst, account)
}
}
}
// establishedGood reports whether this IP is an established legitimate sender
// for account: it authenticated successfully longer ago than the failure window
// (so the current failure burst is not its first contact) and recently enough
// to still be live. Caller must hold t.mu.
func (e *mailIPEntry) establishedGood(account string, now time.Time, window time.Duration) bool {
last, ok := e.goodLast[account]
if !ok || now.Sub(last) > mailGoodSourceTTL {
return false
}
return now.Sub(e.goodFirst[account]) >= window
}
// establishedOtherGoodAccounts counts mailboxes other than account for which
// this IP holds established good standing from before the active attack.
// Standing learned alongside its failures cannot vouch for that attack even
// after it ages. Caller must hold t.mu.
func (e *mailIPEntry) establishedOtherGoodAccounts(account string, now time.Time, window time.Duration, failuresSince time.Time) int {
n := 0
for acct := range e.goodLast {
if acct == account {
continue
}
if e.establishedGood(acct, now, window) && (failuresSince.IsZero() || e.goodFirst[acct].Before(failuresSince)) {
n++
}
}
return n
}
// establishedGoodConfined reports whether this IP looks like a misconfigured
// legitimate client rather than a brute-forcer: every current failure is
// attributed to a named mailbox, and every one of those failing mailboxes is a
// mailbox this IP holds ESTABLISHED good standing for (it authenticated
// successfully to that same mailbox longer ago than the failure window and
// recently enough to still be live). When true the per-IP signal is downgraded
// to an advisory rather than a firewall block, so a source whose own working
// mailbox now fails on a stale saved password (e.g. POP3 succeeds while IMAP
// fails on the same box) is not locked out.
//
// Good standing is scoped to the (IP, mailbox) identity that earned it: standing
// on mailbox A does NOT vouch for failures on a different mailbox B the IP has
// never signed into. The earlier count-only bound (failing count <= good count)
// was identity-blind and let a source with good standing on N of its own
// mailboxes brute-force N unrelated victim mailboxes indefinitely without ever
// auto-blocking. A fresh same-window "success" is not established, so padding
// earns no downgrade, and the compromise and spray detectors stay fully armed.
// Caller must hold t.mu and must have pruned
// e.times/e.failedAccounts/e.successAccounts to the window.
func (e *mailIPEntry) establishedGoodConfined(now time.Time, window time.Duration) bool {
failing := len(e.failedAccounts)
if failing == 0 {
return false
}
named := 0
for account, times := range e.failedAccounts {
named += len(times)
if !e.establishedGood(account, now, window) {
return false
}
}
// Accountless failures (no user= in the log line) could target anything; a
// good source's standing must not hide them.
return named == len(e.times)
}
// Only ASCII case is an alias. Unicode lowercasing can turn a distinct login
// name into an ASCII mailbox and lend it that mailbox's trust.
func normalizeMailAuthAccount(account string) string {
// Replacing invalid bytes with a Unicode replacement character would
// merge malformed attempts with a different, valid login name.
if !utf8.ValidString(account) {
return ""
}
return strings.Map(func(r rune) rune {
if r >= 'A' && r <= 'Z' {
return r + ('a' - 'A')
}
return r
}, strings.TrimSpace(account))
}
// slowConfinedToEstablishedGood reports whether every mailbox the slow-window
// failures target is one this IP holds established good standing for -- the
// shared-device-with-stale-credentials shape the fast path downgrades to an
// advisory. Caller must hold t.mu and must have pruned e.slowAccounts.
func (e *mailIPEntry) slowConfinedToEstablishedGood(now time.Time, window time.Duration) bool {
if len(e.slowAccounts) == 0 || !e.slowUnvouched.IsZero() {
return false
}
for acct := range e.slowAccounts {
if !e.establishedGood(acct, now, window) {
return false
}
}
return true
}
// recentGoodCount returns how many distinct mailboxes this IP succeeded on
// recently enough to still be on record (last success within
// mailGoodSourceTTL), regardless of whether that standing has aged past the
// failure window. Unlike establishedGood it does not require the
// relationship to predate the burst, so it recognizes a legitimate sender whose
// standing is only seconds old -- a cold-started daemon before persisted
// standing re-ages, or a customer returning after the snapshot expired. Caller
// must hold t.mu.
func (e *mailIPEntry) recentGoodCount(now time.Time) int {
n := 0
for _, last := range e.goodLast {
if now.Sub(last) <= mailGoodSourceTTL {
n++
}
}
return n
}
// looksLikeFreshGoodSourceFP reports the cold-start/misconfiguration
// false-positive shape: the same confined, named, non-cracking failure set that
// establishedGoodConfined downgrades, but bounded by the IP's recent-success
// footprint instead of its established (aged) one. It is true exactly when the
// only reason the burst was not downgraded to an advisory is that the source's
// good standing is too fresh to have aged past the window. The auto-block still
// fires -- a fresh success cannot be trusted to grant a brute-force bypass, or a
// padding login would buy an attacker slack -- but the finding is annotated so
// an operator can spot a likely false positive without reconstructing the mail
// log by hand. Caller must hold t.mu and must have pruned the window state.
func (e *mailIPEntry) looksLikeFreshGoodSourceFP(now time.Time, window time.Duration) bool {
failing := len(e.failedAccounts)
if failing == 0 {
return false
}
named := 0
for account, times := range e.failedAccounts {
named += len(times)
if e.isCrackInProgress(account, now, window) {
return false
}
}
if named != len(e.times) {
return false
}
return failing <= e.recentGoodCount(now)
}
// mailFailTarget is one mailbox this IP's in-window failures hit, with the
// failure count for that mailbox.
type mailFailTarget struct {
Account string
Count int
}
// failTargets summarizes which mailboxes this IP's in-window failures targeted.
// It returns the per-mailbox counts sorted by count (desc) then account name
// (asc) for deterministic output, plus how many failures carried no mailbox
// (dovecot lines with no user=). Caller must hold t.mu and must have pruned
// e.times and e.failedAccounts to the window.
func (e *mailIPEntry) failTargets() ([]mailFailTarget, int) {
named := 0
targets := make([]mailFailTarget, 0, len(e.failedAccounts))
for account, times := range e.failedAccounts {
named += len(times)
targets = append(targets, mailFailTarget{Account: account, Count: len(times)})
}
sort.Slice(targets, func(i, j int) bool {
if targets[i].Count != targets[j].Count {
return targets[i].Count > targets[j].Count
}
return targets[i].Account < targets[j].Account
})
return targets, len(e.times) - named
}
// Unsafe account bytes are hex-escaped so a crafted user= value cannot add
// fake target separators, new alert lines, or path-looking tokens.
func formatMailFailTargetAccount(account string) string {
truncated := false
if len(account) > maxMailTargetAccountDisplayBytes {
account = account[:maxMailTargetAccountDisplayBytes]
truncated = true
}
if isPlainMailFailTargetAccount(account) {
if truncated {
return account + "..."
}
return account
}
var b strings.Builder
b.WriteString(`account="`)
for i := 0; i < len(account); i++ {
c := account[i]
if isPlainMailFailTargetAccountByte(c) {
b.WriteByte(c)
continue
}
fmt.Fprintf(&b, `\x%02x`, c)
}
if truncated {
b.WriteString("...")
}
b.WriteByte('"')
return b.String()
}
func isPlainMailFailTargetAccount(account string) bool {
if account == "" {
return false
}
for i := 0; i < len(account); i++ {
if !isPlainMailFailTargetAccountByte(account[i]) {
return false
}
}
return true
}
func isPlainMailFailTargetAccountByte(c byte) bool {
return c >= 'a' && c <= 'z' ||
c >= 'A' && c <= 'Z' ||
c >= '0' && c <= '9' ||
strings.ContainsRune("@._+-=%*", rune(c))
}
// formatMailFailTargets renders the target summary for a mail brute-force
// finding: the mailboxes hit with their failure counts (at most max named, the
// rest collapsed into "(+N more)") and a count of failures that named no
// mailbox. Returns "" when there is nothing to report.
func formatMailFailTargets(targets []mailFailTarget, accountless, max int) string {
if len(targets) == 0 && accountless <= 0 {
return ""
}
listed := targets
remainder := 0
if max > 0 && len(targets) > max {
listed = targets[:max]
remainder = len(targets) - max
}
var b strings.Builder
b.WriteString("Targets: ")
for i, tg := range listed {
if i > 0 {
b.WriteString(", ")
}
fmt.Fprintf(&b, "%s (%d)", formatMailFailTargetAccount(tg.Account), tg.Count)
}
if remainder > 0 {
fmt.Fprintf(&b, " (+%d more)", remainder)
}
if accountless > 0 {
if len(listed) > 0 {
b.WriteString("; ")
}
fmt.Fprintf(&b, "%d with no mailbox", accountless)
}
return b.String()
}
// successDominant reports whether this IP behaves like a legit busy client
// rather than a brute-forcer: it has successful logins in the window, at least
// as many successes as failures, and the successful mailboxes explain the failed
// mailbox set. Caller must hold t.mu and must have pruned the slices and account
// maps to the current window first.
func (e *mailIPEntry) successDominant() bool {
if len(e.succ) == 0 || len(e.succ) < len(e.times) || len(e.failedAccounts) == 0 {
return false
}
named := 0
for account, failures := range e.failedAccounts {
named += len(failures)
if len(e.successAccounts[account]) < len(failures) {
return false
}
}
return named == len(e.times)
}
func (e *mailIPEntry) accountSuccessDominant(account string) bool {
failures := len(e.failedAccounts[account])
return failures > 0 && len(e.successAccounts[account]) >= failures
}
// isCrackInProgress reports whether same-account successes mixed with failures
// look like a guessing breakthrough rather than a flaky legitimate client.
//
// An account the IP holds established good standing for (it has owned the
// mailbox longer than the window) is exempt: a mix of successes and failures
// there is an intermittent or stale-on-one-device client, the same mistype
// signal RecordSuccess uses to suppress compromise. Only a fresh, non-established
// in-window success that does not dominate the failures is treated as a crack:
// that is the attacker-just-guessed-it shape. Caller must hold t.mu.
func (e *mailIPEntry) isCrackInProgress(account string, now time.Time, window time.Duration) bool {
if e.establishedGood(account, now, window) {
return false
}
cutoff := now.Add(-window)
for _, ts := range e.successAccounts[account] {
if ts.After(cutoff) {
return !e.accountSuccessDominant(account)
}
}
return false
}
// goodSourceTimes is the persisted established-sender window for one mailbox:
// the earliest and most recent successful auth from an IP.
type goodSourceTimes struct {
First time.Time
Last time.Time
}
// goodSourceSnapshot maps ip -> account -> goodSourceTimes. It is persisted
// across daemon restarts so established good standing survives a restart and the
// post-restart cold-start window does not re-open the brute-force false-positive
// window (a customer's working profile would otherwise have to re-authenticate
// and re-age past the failure window before its stale-password profile stops
// being mistaken for an attacker).
type goodSourceSnapshot map[string]map[string]goodSourceTimes
// ExportGoodSource snapshots the established-sender records for persistence.
// Only good-source state is exported; short-lived failure/success window state
// is intentionally not persisted.
func (t *mailAuthTracker) ExportGoodSource() goodSourceSnapshot {
t.mu.Lock()
defer t.mu.Unlock()
snap := make(goodSourceSnapshot)
for ip, e := range t.ips {
if len(e.goodLast) == 0 {
continue
}
accts := make(map[string]goodSourceTimes, len(e.goodLast))
for acct, last := range e.goodLast {
accts[acct] = goodSourceTimes{First: e.goodFirst[acct], Last: last}
}
snap[ip] = accts
}
return snap
}
// LoadGoodSource seeds established-sender records from a persisted snapshot,
// dropping any whose most recent success is already older than the good-source
// TTL. Intended for one-time startup seeding; a record newer than the snapshot
// (a success already observed this run) keeps its Last time, while colliding
// normalized mailbox records keep the earliest First time.
func (t *mailAuthTracker) LoadGoodSource(snap goodSourceSnapshot, now time.Time) {
t.mu.Lock()
defer t.mu.Unlock()
cutoff := now.Add(-mailGoodSourceTTL)
for ip, accts := range snap {
for acct, ts := range accts {
acct = normalizeMailAuthAccount(acct)
if ip == "" || acct == "" || !validGoodSourceTimes(ts, cutoff) {
continue
}
e, ok := t.ips[ip]
if !ok {
e = &mailIPEntry{}
t.ips[ip] = e
t.aliased = t.aliased || sprayAddr(ip) != ip
}
if e.goodFirst == nil {
e.goodFirst = make(map[string]time.Time)
e.goodLast = make(map[string]time.Time)
}
if cur, ok := e.goodFirst[acct]; !ok || ts.First.Before(cur) {
e.goodFirst[acct] = ts.First
}
if cur, ok := e.goodLast[acct]; !ok || ts.Last.After(cur) {
e.goodLast[acct] = ts.Last
}
if e.goodLast[acct].After(e.lastSeen) {
e.lastSeen = e.goodLast[acct]
}
if len(e.goodLast) > mailGoodSourceMaxAccountsPerIP {
e.evictOldestGoodSource()
}
}
}
t.enforceMaxTracked("")
}
func validGoodSourceTimes(ts goodSourceTimes, cutoff time.Time) bool {
return !ts.First.IsZero() &&
!ts.Last.IsZero() &&
!ts.First.After(ts.Last) &&
!ts.Last.Before(cutoff)
}
// loadMailGoodSource seeds the tracker before log readers start, so first
// post-startup auth failures see the persisted standing instead of cold state.
func (d *Daemon) loadMailGoodSource() {
if d.mailAuthTracker == nil {
return
}
if sdb := store.Global(); sdb != nil {
snap, err := sdb.LoadMailGoodSource()
if err != nil {
csmlog.Warn("mail good-source load failed", "err", err)
return
}
d.mailAuthTracker.LoadGoodSource(storeToGoodSourceSnapshot(snap), d.mailAuthTracker.now())
}
}
// persistMailGoodSource writes the tracker's established good-source snapshot to
// the store so it survives a restart. No-op when the store or tracker is absent.
func (d *Daemon) persistMailGoodSource() {
if d.mailAuthTracker == nil {
return
}
if sdb := store.Global(); sdb != nil {
snap := goodSourceSnapshotToStore(d.mailAuthTracker.ExportGoodSource())
if err := sdb.SaveMailGoodSource(snap); err != nil {
csmlog.Warn("mail good-source persistence failed", "err", err)
}
}
}
func storeToGoodSourceSnapshot(in map[string]map[string]store.GoodSourcePair) goodSourceSnapshot {
out := make(goodSourceSnapshot, len(in))
for ip, accts := range in {
m := make(map[string]goodSourceTimes, len(accts))
for a, p := range accts {
m[a] = goodSourceTimes{First: p.First, Last: p.Last}
}
out[ip] = m
}
return out
}
func goodSourceSnapshotToStore(in goodSourceSnapshot) map[string]map[string]store.GoodSourcePair {
out := make(map[string]map[string]store.GoodSourcePair, len(in))
for ip, accts := range in {
m := make(map[string]store.GoodSourcePair, len(accts))
for a, ts := range accts {
m[a] = store.GoodSourcePair{First: ts.First, Last: ts.Last}
}
out[ip] = m
}
return out
}
// mailSubnetEntry tracks unique attacker IPs within a /24.
type mailSubnetEntry struct {
ips map[string]subnetSighting
suppressed time.Time
lastSeen time.Time
}
// mailAccountEntry tracks unique attacker IPs per mailbox, plus a separate
// suppression clock for compromise findings emitted by RecordSuccess.
type mailAccountEntry struct {
ips map[string]time.Time
suppressed time.Time
compromiseSuppressed time.Time
lastSeen time.Time
}
// mailBackendDegradedThreshold is how many auth-backend failure observations
// (dovecot unable to reach the credential backend, e.g. cPanel's cpdoveauthd)
// within the tracker window mark the mail auth subsystem as degraded. A healthy
// host produces zero of these; a backend outage produces thousands. While
// degraded, every login fails regardless of credentials, so the per-IP and
// per-subnet brute signals are suppressed to avoid auto-blocking legitimate
// users en masse.
const mailBackendDegradedThreshold = 10
// mailGoodSourceTTL is how long a successful authentication keeps an (IP,
// mailbox) pair on record as an established legitimate sender. A real owner
// authenticates successfully at least this often; once the last success ages
// past the TTL the standing is forgotten, so an IP cannot be permanently
// whitelisted by a single old success.
const mailGoodSourceTTL = 24 * time.Hour
// mailGoodSourceMaxAccountsPerIP caps how many distinct mailboxes one IP keeps
// good-source records for. A carrier-grade NAT can carry many legitimate
// mailboxes; the cap bounds memory without affecting detection (eviction is
// least-recently-successful first).
const mailGoodSourceMaxAccountsPerIP = 256
// mailEstablishedSourceAccounts is how many OTHER mailboxes an IP must hold
// established good standing on before a compromise finding for one more
// mailbox is downgraded to an advisory (office/agency device pattern).
const mailEstablishedSourceAccounts = 2
// maxMailTargetsListed caps how many mailboxes a brute-force finding names
// before collapsing the rest into a "(+N more)" suffix, so a wide spray does
// not dump dozens of names into one alert.
const maxMailTargetsListed = 5
// maxMailTargetAccountDisplayBytes bounds attacker-controlled account text in
// brute-force Details while keeping normal mailbox names intact.
const maxMailTargetAccountDisplayBytes = 128
// mailAuthTracker aggregates dovecot IMAP/POP3/ManageSieve auth events into
// four detection signals: per-IP brute force, per-/24 password spray,
// per-mailbox account spray, and per-account compromise (success after
// recent failures).
//
// Thread-safe; Record/RecordSuccess may be called concurrently from multiple
// log readers.
type mailAuthTracker struct {
mu sync.Mutex
perIPThreshold int
subnetThreshold int
accountSprayThreshold int
window time.Duration
suppression time.Duration
slowThreshold int
slowWindow time.Duration
maxTracked int
now func() time.Time
ips map[string]*mailIPEntry
subnets map[string]*mailSubnetEntry
accounts map[string]*mailAccountEntry
// Diagnostic counters (guarded by mu): cumulative Record invocations and
// findings emitted, logged periodically by the daemon to pin whether the
// non-cPanel dovecot brute-force path sees traffic and escalates.
recordCalls int64
findingsEmitted int64
// backendErr holds recent auth-backend failure timestamps. While the
// windowed count is at or above mailBackendDegradedThreshold the auth
// subsystem is treated as down and brute/subnet auto-block signals are
// suppressed. backendWarnUntil rate-limits the operator warning to one per
// suppression window.
backendErr []time.Time
backendWarnUntil time.Time
// backendDownFn, when set, reports whether the active socket probe currently
// sees the mail auth backend down. It augments the log-derived backendErr
// heuristic so suppression also triggers on the authoritative probe signal.
backendDownFn func() bool
// spray, when set, is the long-memory login spray detector shared with
// the SMTP tracker. It holds per-mailbox login trust. Set once at startup
// before log readers begin.
spray *mailSpray
// knownCustomers, when set, remembers each successful login's address.
// Set once at startup before log readers begin; read without mu.
knownCustomers *knownCustomers
// aliased is set once ips holds a key that is not its address's
// canonical spelling. Until then a release looks up the canonical key
// instead of scanning every entry.
aliased bool
}
// SetBackendDownCheck installs the active-probe callback the tracker consults to
// learn whether the mail auth backend is down. When it returns true, brute-force
// and subnet auto-block are suppressed. Set once at startup before log readers
// begin.
func (t *mailAuthTracker) SetBackendDownCheck(fn func() bool) {
t.mu.Lock()
defer t.mu.Unlock()
t.backendDownFn = fn
}
// SetSpray installs the shared spray detector. Set once at startup before log
// readers begin.
func (t *mailAuthTracker) SetSpray(m *mailSpray) {
t.mu.Lock()
defer t.mu.Unlock()
t.spray = m
}
// SetKnownCustomers installs the known-customer set. Set once at startup
// before log readers begin.
func (t *mailAuthTracker) SetKnownCustomers(k *knownCustomers) {
t.knownCustomers = k
}
// newMailAuthTracker constructs a tracker. `now` is injected so tests can
// use deterministic clocks; pass `time.Now` in production.
func newMailAuthTracker(
perIPThreshold int,
subnetThreshold int,
accountSprayThreshold int,
window time.Duration,
suppression time.Duration,
slowThreshold int,
slowWindow time.Duration,
maxTracked int,
now func() time.Time,
) *mailAuthTracker {
if now == nil {
now = time.Now
}
return &mailAuthTracker{
perIPThreshold: perIPThreshold,
subnetThreshold: subnetThreshold,
accountSprayThreshold: accountSprayThreshold,
window: window,
suppression: suppression,
slowThreshold: slowThreshold,
slowWindow: slowWindow,
maxTracked: maxTracked,
now: now,
ips: make(map[string]*mailIPEntry),
subnets: make(map[string]*mailSubnetEntry),
accounts: make(map[string]*mailAccountEntry),
}
}
// Size returns the total number of tracked entities (IPs + subnets + accounts).
func (t *mailAuthTracker) Size() int {
t.mu.Lock()
defer t.mu.Unlock()
return len(t.ips) + len(t.subnets) + len(t.accounts)
}
// Record processes one dovecot IMAP/POP3/ManageSieve auth-failure observation.
// Returns zero or more findings that callers should append.
//
// ip MUST be non-private, non-loopback, and non-infra — callers enforce this
// before invoking Record.
func (t *mailAuthTracker) Record(ip, account string) []alert.Finding {
return t.RecordObserved(ip, account, alert.Observation{})
}
// RecordObserved is Record for a failure read from a positioned line: obs
// names that line, so a subnet spray names the line that last counted each
// address.
func (t *mailAuthTracker) RecordObserved(ip, account string, obs alert.Observation) []alert.Finding {
return t.RecordResolvedObserved(ip, account, sprayMailbox{name: normalizeSprayMailbox(account), exist: mailboxExistenceUnknown}, obs)
}
// RecordResolvedObserved is RecordObserved with the mailbox the failure
// resolved to: spray memory counts it against mb, the tracker against the
// typed account.
func (t *mailAuthTracker) RecordResolvedObserved(ip, account string, mb sprayMailbox, obs alert.Observation) []alert.Finding {
if ip == "" {
return nil
}
account = normalizeMailAuthAccount(account)
t.mu.Lock()
defer t.mu.Unlock()
t.recordCalls++
now := t.now()
cutoff := now.Add(-t.window)
var findings []alert.Finding
// During an auth-backend outage every login fails regardless of password, so
// the failure-rate signals are meaningless and would mass-block real users.
// Either the log-derived heuristic or the authoritative socket probe trips it.
degraded := t.backendDegraded(now)
if !degraded && t.backendDownFn != nil {
degraded = t.backendDownFn()
}
// --- Per-IP tracker ---
e, ok := t.ips[ip]
if !ok {
e = &mailIPEntry{}
t.ips[ip] = e
t.aliased = t.aliased || sprayAddr(ip) != ip
}
e.times = pruneTimes(e.times, cutoff)
pruneMailAccountTimes(e.failedAccounts, cutoff)
e.times = append(e.times, now)
e.failedAccounts = appendMailAccountTime(e.failedAccounts, account, now)
e.lastSeen = now
if t.perIPThreshold > 0 && len(e.times) >= t.perIPThreshold {
e.succ = pruneTimes(e.succ, cutoff)
pruneMailAccountTimes(e.successAccounts, cutoff)
if !e.successDominant() && !degraded {
switch {
case e.establishedGoodConfined(now, t.window):
// Established good source fat-fingering a confined set of its own
// mailboxes: surface for visibility but do not auto-block, so a
// stale saved password does not lock out a real customer. A real
// attack from the same source still blocks via the compromise and
// spray detectors. Rate-limited on its own clock so escalation to a
// wider spray can still reach the block path below.
if now.Before(e.suspectedSuppressed) {
break
}
e.suspectedSuppressed = now.Add(t.suppression)
details := "Failures are confined to a few named mailboxes from a source with established successful mail auth history; likely a stale saved password, not a brute-force. Visibility only - no auto-block. Compromise and spray detectors remain active."
targets, accountless := e.failTargets()
if targetSummary := formatMailFailTargets(targets, accountless, maxMailTargetsListed); targetSummary != "" {
details += " " + targetSummary + "."
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "mail_bruteforce_suspected",
Message: fmt.Sprintf("Suspected mail misconfiguration from %s: %d failed auths in %v from an established good source (not auto-blocked)",
ip, len(e.times), t.window),
Details: details,
Timestamp: now,
SourceIP: ip,
})
case !now.Before(e.suppressed):
e.suppressed = now.Add(t.suppression)
details := "Real-time detection of dovecot imap/pop3/managesieve auth failures."
if e.looksLikeFreshGoodSourceFP(now, t.window) {
details += " Note: this source has recent successful mail auth for one or more other mailboxes, so the failures may be a misconfigured client with a stale saved password rather than an attack. Verify before treating it as a confirmed brute-force."
}
targets, accountless := e.failTargets()
if targetSummary := formatMailFailTargets(targets, accountless, maxMailTargetsListed); targetSummary != "" {
details += " " + targetSummary + "."
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "mail_bruteforce",
Message: fmt.Sprintf("Mail auth brute force from %s: %d failed auths in %v",
ip, len(e.times), t.window),
Details: details,
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionPeer,
})
}
}
}
// --- Long-horizon slow-brute tracker ---
// Catches attackers pacing below the fast window. The distinct-mailbox
// requirement separates a mailbox walk from a stale saved password. A
// source whose paced failures stay confined to mailboxes it holds
// established good standing or aged login trust for keeps the fast path's
// advisory treatment instead of escalating to a block.
if t.slowThreshold > 0 && t.slowWindow > 0 {
if degraded {
// Backend failures say nothing about credentials. Discard the
// long-lived evidence so outage traffic cannot trigger a delayed
// block as soon as the backend recovers.
e.slowTimes = nil
e.slowAccounts = nil
e.slowUnvouched = time.Time{}
} else {
slowCutoff := now.Add(-t.slowWindow)
e.slowTimes = pruneTimes(e.slowTimes, slowCutoff)
e.slowTimes = appendSlowFailure(e.slowTimes, now)
var trackedAccount bool
e.slowAccounts, trackedAccount = recordSlowAccount(e.slowAccounts, account, now, slowCutoff)
if !trackedAccount {
e.slowUnvouched = now
}
if e.slowUnvouched.Before(slowCutoff) {
e.slowUnvouched = time.Time{}
}
if (len(e.slowTimes) >= t.slowThreshold || len(e.slowAccounts) >= slowBruteWalkAccounts) &&
len(e.slowAccounts) >= slowBruteMinAccounts {
safeSource := e.slowConfinedToEstablishedGood(now, t.window) ||
slowTrusted(t.spray, ip, e.slowAccounts, e.slowUnvouched)
switch {
case safeSource && !now.Before(e.suspectedSuppressed):
e.suspectedSuppressed = now.Add(t.suppression)
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "mail_bruteforce_suspected",
Message: fmt.Sprintf("Suspected mail misconfiguration from %s: %d paced auth failures across %d mailboxes in %v (not auto-blocked)",
ip, len(e.slowTimes), len(e.slowAccounts), t.slowWindow),
Details: "Long-horizon failures came from a source with established good standing or aged login trust on every named target. Visibility only - no auto-block; compromise and spray detectors remain active.",
Timestamp: now,
SourceIP: ip,
})
case !safeSource && !now.Before(e.suppressed):
e.suppressed = now.Add(t.suppression)
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "mail_bruteforce",
Message: fmt.Sprintf("Mail auth brute force from %s: %d failed auths across %d mailboxes in %v",
ip, len(e.slowTimes), len(e.slowAccounts), t.slowWindow),
Details: "Long-horizon detection of paced imap/pop3/managesieve auth failures that stay below the fast per-IP window",
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionPeer,
})
}
}
}
}
// --- Long-memory login spray (shared with SMTP) ---
if degraded && t.spray != nil {
// The outage may have begun a while before it was detected.
t.spray.ForgetRecentFailures(ip, cutoff)
}
if !degraded && t.spray != nil {
if v := t.spray.Failure(ip, mb.name, mb.exist); v != nil {
f := sprayFinding(protoMail, ip, v, t.spray.Mode(), now)
// One block per suppression window: a Dovecot line can carry
// several attempts, so the spray block shares the per-IP clock
// with the fast and slow paths. A spray block with its own length
// takes the place of a per-IP block from this same failure. The
// evidence is recorded either way.
perIPBlock := slices.IndexFunc(findings, func(p alert.Finding) bool { return p.Check == "mail_bruteforce" })
switch {
case f.Check != "mail_bruteforce":
findings = append(findings, f)
case f.BlockTTL > 0 && perIPBlock >= 0:
findings[perIPBlock] = f
case !now.Before(e.suppressed):
e.suppressed = now.Add(t.suppression)
findings = append(findings, f)
}
}
}
// --- Per-/24 subnet tracker (IPv4 only) ---
if prefix := extractPrefix24Daemon(ip); prefix != "" {
s, ok := t.subnets[prefix]
if !ok {
s = &mailSubnetEntry{ips: make(map[string]subnetSighting)}
t.subnets[prefix] = s
}
for ipKey, seen := range s.ips {
if seen.at.Before(cutoff) {
delete(s.ips, ipKey)
}
}
s.ips[ip] = subnetSighting{at: now, obs: obs}
s.lastSeen = now
if t.subnetThreshold > 0 && len(s.ips) >= t.subnetThreshold && !now.Before(s.suppressed) && !degraded {
s.suppressed = now.Add(t.suppression)
cidr := prefix + ".0/24"
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "mail_subnet_spray",
Message: fmt.Sprintf("Mail password spray from %s.0/24: %d unique IPs in %v",
prefix, len(s.ips), t.window),
Details: "Real-time detection of mail auth failures from many IPs in one /24",
Timestamp: now,
CIDRs: []string{cidr},
SprayConstituents: sprayConstituents(s.ips),
})
}
}
// --- Per-account spray tracker ---
if account != "" {
a, ok := t.accounts[account]
if !ok {
a = &mailAccountEntry{ips: make(map[string]time.Time)}
t.accounts[account] = a
}
for ipKey, ts := range a.ips {
if ts.Before(cutoff) {
delete(a.ips, ipKey)
}
}
a.ips[ip] = now
a.lastSeen = now
if t.accountSprayThreshold > 0 && len(a.ips) >= t.accountSprayThreshold && !now.Before(a.suppressed) {
a.suppressed = now.Add(t.suppression)
_, acctDomain := alert.SplitEmail(account)
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "mail_account_spray",
Message: fmt.Sprintf("Mail password spray targeting %s: %d unique IPs in %v",
account, len(a.ips), t.window),
Details: "Distributed login attempts across many IPs against one mailbox (visibility only — no auto-block).",
Timestamp: now,
SourceIP: ip,
Domain: acctDomain,
Mailbox: account,
})
}
}
t.enforceMaxTracked(ip)
t.findingsEmitted += int64(len(findings))
return findings
}
// Forget drops ip's failure history after an operator release, or every
// address's when ip is empty. Successful logins and good standing stay:
// they describe the mailboxes it logs in to, not the block.
func (t *mailAuthTracker) Forget(ip string) {
ip = sprayAddr(ip)
t.mu.Lock()
defer t.mu.Unlock()
forget := func(e *mailIPEntry) {
e.times, e.failedAccounts = nil, nil
e.slowTimes, e.slowAccounts, e.slowUnvouched = nil, nil, time.Time{}
e.suppressed, e.suspectedSuppressed = time.Time{}, time.Time{}
}
if ip != "" && !t.aliased {
if e := t.ips[ip]; e != nil {
forget(e)
}
return
}
// Log readers and persisted good standing may use different spellings
// of one address. Release every matching entry without discarding trust.
for addr, e := range t.ips {
if ip == "" || sprayAddr(addr) == ip {
forget(e)
}
}
}
// Stats returns cumulative Record invocations and findings emitted since
// startup. Used by the daemon's periodic diagnostic log.
func (t *mailAuthTracker) Stats() (calls, emits int64) {
t.mu.Lock()
defer t.mu.Unlock()
return t.recordCalls, t.findingsEmitted
}
// RecordBackendFailure records one auth-backend failure observation (dovecot
// could not reach the credential backend). It returns a one-shot operator
// warning the first time the subsystem crosses into "degraded" within a
// suppression window, so an outage is visible rather than silently pausing
// detection. Callers feed this from log lines matched by isMailAuthBackendError.
func (t *mailAuthTracker) RecordBackendFailure() []alert.Finding {
t.mu.Lock()
defer t.mu.Unlock()
now := t.now()
t.recordBackendFailureTime(now)
if len(t.backendErr) < mailBackendDegradedThreshold || now.Before(t.backendWarnUntil) {
return nil
}
t.backendWarnUntil = now.Add(t.suppression)
t.findingsEmitted++
return []alert.Finding{{
Severity: alert.Warning,
Check: "mail_auth_backend_degraded",
Message: fmt.Sprintf("Mail auth backend degraded: %d backend failures in %v; mail brute-force auto-block paused",
len(t.backendErr), t.window),
Details: "Dovecot could not reach the credential backend (e.g. cpdoveauthd). Logins fail regardless of password, so brute-force and subnet auto-blocks are paused to avoid locking out legitimate users. Investigate the auth daemon.",
Timestamp: now,
}}
}
// recordBackendFailureTime keeps only the newest observations needed to prove
// the degraded invariant. Backend outages can flood logs; tracking more than
// the threshold would add memory pressure without changing detection.
func (t *mailAuthTracker) recordBackendFailureTime(now time.Time) {
t.backendErr = pruneTimes(t.backendErr, now.Add(-t.window))
if len(t.backendErr) == 0 {
t.backendErr = nil
}
if len(t.backendErr) >= mailBackendDegradedThreshold {
const keep = mailBackendDegradedThreshold - 1
if cap(t.backendErr) > mailBackendDegradedThreshold {
recent := make([]time.Time, mailBackendDegradedThreshold)
copy(recent, t.backendErr[len(t.backendErr)-keep:])
recent[keep] = now
t.backendErr = recent
return
}
copy(t.backendErr, t.backendErr[len(t.backendErr)-keep:])
t.backendErr = t.backendErr[:mailBackendDegradedThreshold]
t.backendErr[keep] = now
return
}
t.backendErr = append(t.backendErr, now)
}
// backendDegraded reports whether the mail auth backend looks down: at least
// mailBackendDegradedThreshold backend-failure observations within the window.
// Caller must hold t.mu.
func (t *mailAuthTracker) backendDegraded(now time.Time) bool {
t.backendErr = pruneTimes(t.backendErr, now.Add(-t.window))
if len(t.backendErr) == 0 {
t.backendErr = nil
}
return len(t.backendErr) >= mailBackendDegradedThreshold
}
func pruneMailAccountTimes(accounts map[string][]time.Time, cutoff time.Time) {
for account, times := range accounts {
times = pruneTimes(times, cutoff)
if len(times) == 0 {
delete(accounts, account)
continue
}
accounts[account] = times
}
}
func appendMailAccountTime(accounts map[string][]time.Time, account string, ts time.Time) map[string][]time.Time {
if account == "" {
return accounts
}
if accounts == nil {
accounts = make(map[string][]time.Time)
}
accounts[account] = append(accounts[account], ts)
return accounts
}
// RecordSuccess processes a successful mail login. Emits mail_account_compromised
// when the successful IP has repeated recent failed auths for the same account
// and that mailbox is failure-dominant from this IP. Prior successful logins for
// the same mailbox look like a legit client that merely mistyped, so they are
// not flagged. A login the spray memory does not call clean, one that follows
// failures on other mailboxes, earns no good standing and may be reported too;
// one login raises at most one compromise finding, the stronger.
//
// ip and account MUST both be non-empty. Caller filters infra/private/loopback
// IPs before invoking.
func (t *mailAuthTracker) RecordSuccess(ip, account string) (findings []alert.Finding) {
return t.recordSuccessAt(ip, account, t.now(), knownCustomerIMAP)
}
// recordSuccessAt retains the source time for customer history while the
// brute-force tracker continues to process the event at receipt time. kind
// is imap or pop; Dovecot logs both through syslog.
func (t *mailAuthTracker) recordSuccessAt(ip, account string, at time.Time, kind string) (findings []alert.Finding) {
// Before t.mu: the customer set takes its own lock.
t.knownCustomers.Note(ip, kind, account, at, syslogFed)
// A tracker key is not an authenticated identity. Keep the logged local
// part or hosting-account case for owner lookup and mailbox response.
loginAccount := strings.TrimSpace(account)
account = normalizeMailAuthAccount(account)
if ip == "" || account == "" {
return nil
}
if local, _, ok := strings.Cut(loginAccount, "@"); ok && local != "" {
if _, domain, _ := strings.Cut(account, "@"); domain != "" {
loginAccount = local + "@" + domain
}
}
// Registered before tracker cleanup so resolution sees no held locks.
// The one finding is stamped for the mailbox it names.
defer func() {
if len(findings) > 0 {
stampMailAccountOwner(findings, findings[0].Mailbox)
}
}()
t.mu.Lock()
defer t.mu.Unlock()
// Successes create per-IP entries too; keep the tracker bounded on every
// path (runs under the held lock, before Unlock).
defer t.enforceMaxTracked(ip)
now := t.now()
cutoff := now.Add(-t.window)
// Classify the login before recording it: a guessed password must not
// become success-dominant activity that exempts later failures.
e, ok := t.ips[ip]
if !ok {
e = &mailIPEntry{}
t.ips[ip] = e
t.aliased = t.aliased || sprayAddr(ip) != ip
}
e.succ = pruneTimes(e.succ, cutoff)
pruneMailAccountTimes(e.successAccounts, cutoff)
pruneMailAccountTimes(e.failedAccounts, cutoff)
// A guessed-password success should not train the long-lived good-source cache.
degraded := t.backendDegraded(now) || (t.backendDownFn != nil && t.backendDownFn())
recordGoodSource := !degraded
var sprayed *alert.Finding
sprayMailbox := normalizeSprayMailbox(account)
if !degraded && t.spray != nil {
clean, v := t.spray.Success(ip, sprayMailbox)
if !clean && t.spray.Enabled() {
recordGoodSource = false
}
if v != nil {
v.Mailbox = loginAccount
f := sprayFinding(protoMail, ip, v, t.spray.Mode(), now)
sprayed = &f
}
}
defer func() {
if recordGoodSource {
e.succ = append(e.succ, now)
e.successAccounts = appendMailAccountTime(e.successAccounts, account, now)
e.recordGoodAuth(account, now)
}
e.lastSeen = now
switch {
case sprayed == nil:
case len(findings) == 0:
t.findingsEmitted++
findings = []alert.Finding{*sprayed}
// At equal severity a spray block keeps its own block length.
case sprayed.Severity > findings[0].Severity || (sprayed.Severity == findings[0].Severity && sprayed.BlockTTL > 0):
findings[0] = *sprayed
}
}()
// A mailbox that already succeeds from this IP is a legit owner who mistyped,
// not a takeover. Genuine password guessing is failure-dominant for the same
// mailbox. Use the per-IP evidence: account entries can be evicted before it.
e.times = pruneTimes(e.times, cutoff)
targetFailures := len(e.failedAccounts[account])
if targetFailures < 2 || e.accountSuccessDominant(account) || e.establishedGood(account, now, t.window) {
return nil
}
recordGoodSource = false
delete(e.goodFirst, account)
delete(e.goodLast, account)
// The spray memory saw failures on this mailbox only and called the login
// clean; a cracked password earns no trust there either.
if !degraded && t.spray != nil {
t.spray.RevokeTrust(ip, sprayMailbox)
}
a, ok := t.accounts[account]
if !ok {
a = &mailAccountEntry{ips: make(map[string]time.Time), lastSeen: now}
t.accounts[account] = a
}
if now.Before(a.compromiseSuppressed) {
return nil
}
a.compromiseSuppressed = now.Add(t.suppression)
_, compDomain := alert.SplitEmail(loginAccount)
t.findingsEmitted++
severity := alert.Critical
message := fmt.Sprintf("Mail account compromise: successful login for %s from %s after recent auth failures",
loginAccount, ip)
details := "Attacker succeeded after repeated failed attempts from the same IP for this mailbox. Rotate password and revoke sessions."
// An IP with established standing on several other mailboxes is far more
// likely a shared office or agency device carrying one stale credential
// than a takeover: real credential attacks come from sources with no
// legitimate multi-mailbox history on this host. Keep the finding for
// visibility but downgrade it below the auto-block bar.
var failuresSince time.Time
if t.spray != nil {
failuresSince = t.spray.FirstUntrustedFailure(ip)
}
if n := e.establishedOtherGoodAccounts(account, now, t.window, failuresSince); n >= mailEstablishedSourceAccounts {
severity = alert.High
message += " (established multi-mailbox source)"
details = fmt.Sprintf("Source IP holds established successful logins to %d other mailboxes on this host, so this is more likely a shared office or agency device with a stale credential than a takeover. Verify with the customer before acting; not auto-blocked.", n)
}
return []alert.Finding{{
Severity: severity,
Check: "mail_account_compromised",
Message: message,
Details: details,
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionPeer,
Domain: compDomain,
Mailbox: loginAccount,
}}
}
// Purge removes stale tracker entries. Good-source-only IPs live until
// mailGoodSourceTTL; failure and short success history uses the detector window.
// Called from a background goroutine every minute.
func (t *mailAuthTracker) Purge() {
t.mu.Lock()
defer t.mu.Unlock()
now := t.now()
activityCutoff := now.Add(-(t.window + t.suppression))
windowCutoff := now.Add(-t.window)
goodCutoff := now.Add(-mailGoodSourceTTL)
slowCutoff := now.Add(-t.slowWindow)
for k, e := range t.ips {
e.times = pruneTimes(e.times, windowCutoff)
e.succ = pruneTimes(e.succ, windowCutoff)
e.slowTimes = pruneTimes(e.slowTimes, slowCutoff)
pruneSlowAccounts(e.slowAccounts, slowCutoff)
if e.slowUnvouched.Before(slowCutoff) {
e.slowUnvouched = time.Time{}
}
pruneMailAccountTimes(e.successAccounts, windowCutoff)
pruneMailAccountTimes(e.failedAccounts, windowCutoff)
e.pruneGoodSource(goodCutoff)
if len(e.times) == 0 && len(e.succ) == 0 && len(e.successAccounts) == 0 &&
len(e.failedAccounts) == 0 && len(e.goodLast) == 0 &&
len(e.slowTimes) == 0 && !e.lastSeen.After(activityCutoff) {
delete(t.ips, k)
}
}
for k, s := range t.subnets {
for ip, seen := range s.ips {
if seen.at.Before(windowCutoff) {
delete(s.ips, ip)
}
}
if len(s.ips) == 0 && !s.lastSeen.After(activityCutoff) {
delete(t.subnets, k)
}
}
for k, a := range t.accounts {
for ip, ts := range a.ips {
if ts.Before(windowCutoff) {
delete(a.ips, ip)
}
}
if len(a.ips) == 0 && !a.lastSeen.After(activityCutoff) {
delete(t.accounts, k)
}
}
t.backendErr = pruneTimes(t.backendErr, windowCutoff)
if len(t.backendErr) == 0 {
t.backendErr = nil
}
}
// enforceMaxTracked evicts the least-recently-seen entries until total tracked
// state is <= 95% of maxTracked. Batch target avoids re-sorting on every
// subsequent insert. Caller must hold t.mu.
// keepIP is the source entry the caller just wrote. It is never evicted:
// dropping the entry a Record is currently accumulating into means a single
// source can never reach its threshold while the table is under pressure, so
// a flood would switch detection off for the very source causing it.
func (t *mailAuthTracker) enforceMaxTracked(keepIP string) {
total := len(t.ips) + len(t.subnets) + len(t.accounts)
if total <= t.maxTracked {
return
}
// Evict to 95% of cap so subsequent inserts don't re-trigger the sort.
target := t.maxTracked * 95 / 100
now := t.now()
// Victims are ranked by what they cost to lose, then by age. Account
// and subnet keys are attacker-chosen (any string after "user=<...>",
// any /24), so a flood of unique mailbox names used to run one LRU over
// the IP entries too and evict a source's good-source standing or
// slow-brute evidence, exactly the state that keeps a legitimate client
// exempt from auto-block. Accounts go first, then subnets, then sources
// with no live failures, and sources with in-window failure or
// slow-brute evidence last.
type victim struct {
kind string // "ip" | "subnet" | "account"
key string
seen time.Time
rank int
}
victims := make([]victim, 0, total)
for k, v := range t.ips {
if k == keepIP {
continue
}
victims = append(victims, victim{"ip", k, v.lastSeen, v.evictionRank(now, t.window, t.slowWindow)})
}
for k, v := range t.subnets {
victims = append(victims, victim{"subnet", k, v.lastSeen, evictionRankSubnet})
}
for k, v := range t.accounts {
victims = append(victims, victim{"account", k, v.lastSeen, evictionRankAccount})
}
sort.Slice(victims, func(i, j int) bool {
if victims[i].rank != victims[j].rank {
return victims[i].rank < victims[j].rank
}
return victims[i].seen.Before(victims[j].seen)
})
for i := 0; i < len(victims); i++ {
if len(t.ips)+len(t.subnets)+len(t.accounts) <= target {
break
}
v := victims[i]
switch v.kind {
case "ip":
delete(t.ips, v.key)
case "subnet":
delete(t.subnets, v.key)
case "account":
delete(t.accounts, v.key)
}
}
}
// Eviction ranks, lowest evicted first.
const (
evictionRankAccount = iota
evictionRankSubnet
evictionRankIdleIP
evictionRankActiveIP
evictionRankProtectedIP
)
// evictionRank keeps established legitimate sources and meaningful slow
// evidence behind disposable one-shot failure entries. Otherwise an attacker
// can send one failure from each fresh IP, make every attacker entry "active",
// and evict the older good-source standing that prevents false blocks.
func (e *mailIPEntry) evictionRank(now time.Time, window, slowWindow time.Duration) int {
for _, last := range e.goodLast {
if !last.Before(now.Add(-mailGoodSourceTTL)) {
return evictionRankProtectedIP
}
}
if slowWindow > 0 && countTimesAtOrAfter(e.slowTimes, now.Add(-slowWindow)) > 1 {
return evictionRankProtectedIP
}
if hasTimeAtOrAfter(e.times, now.Add(-window)) {
return evictionRankActiveIP
}
return evictionRankIdleIP
}
func hasTimeAtOrAfter(times []time.Time, cutoff time.Time) bool {
for _, ts := range times {
if !ts.Before(cutoff) {
return true
}
}
return false
}
func countTimesAtOrAfter(times []time.Time, cutoff time.Time) int {
n := 0
for _, ts := range times {
if !ts.Before(cutoff) {
n++
}
}
return n
}
// isMailAuthBackendError reports whether a dovecot log line shows the auth
// backend itself failing (could not verify ANY credential), as opposed to an
// ordinary wrong-password failure. During such an outage every login fails, so
// these events must drive the degraded gate, not the brute-force counters.
func isMailAuthBackendError(line string) bool {
if isMailAuthLine(line) {
return false
}
msg := dovecotServiceMessage(line)
if msg == "" {
return false
}
detail := msg
if strings.HasPrefix(detail, "auth-worker") {
detail = authWorkerDetail(detail)
}
lowerDetail := strings.ToLower(detail)
if strings.Contains(lowerDetail, "cpdoveauthd.sock") {
return strings.Contains(lowerDetail, "failed to connect") ||
strings.Contains(lowerDetail, "socket error") ||
strings.Contains(lowerDetail, "connection refused")
}
if strings.Contains(lowerDetail, "temporary authentication failure") {
return strings.HasPrefix(msg, "auth-worker") || strings.Contains(strings.ToLower(msg), "auth:")
}
if !strings.HasPrefix(msg, "auth-worker") {
return false
}
return strings.Contains(lowerDetail, "connection refused") ||
strings.Contains(lowerDetail, "internal error")
}
func dovecotServiceMessage(line string) string {
return checks.DovecotServiceMessage(line)
}
func authWorkerDetail(msg string) string {
parenDepth := 0
angleDepth := 0
for i := 0; i < len(msg); i++ {
switch msg[i] {
case '(':
if angleDepth == 0 {
parenDepth++
}
case ')':
if angleDepth == 0 && parenDepth > 0 {
parenDepth--
}
case '<':
if parenDepth == 0 {
angleDepth++
}
case '>':
if parenDepth == 0 && angleDepth > 0 {
angleDepth--
}
case ':':
if parenDepth == 0 && angleDepth == 0 {
return strings.TrimSpace(msg[i+1:])
}
}
}
return ""
}
// isMailAuthLine returns true for dovecot imap/pop3/managesieve login events.
func isMailAuthLine(line string) bool {
return checks.DovecotLoginMessage(line) != ""
}
// dovecotLoginSucceeded reports whether a dovecot imap/pop3/managesieve line
// records a successful login. Dovecot emits two success formats depending on
// version and configuration:
//
// "<proto>-login: Logged in: user=<...>" (observed on production cPanel)
// "<proto>-login: Login: user=<...>" (classic dovecot)
//
// Both the mailbrute compromise detector and the geo new-country detector must
// accept BOTH; keying each consumer off a different single marker left one of
// them silently dead on whichever format the deployment happened to use.
func dovecotLoginSucceeded(line string) bool {
return checks.DovecotLoginSucceeded(line)
}
// extractMailLoginEvent parses a dovecot login line and returns
// (ip, account, success). Returns empty strings and false on parse failure.
//
// Real dovecot wire format (validated against production logs):
//
// Success: "imap-login: Logged in: user=<alice@x.ro>, method=PLAIN, rip=..."
// Failure: "imap-login: Login aborted: ... (auth failed, N attempts ...): user=<...>, method=..., rip=..."
//
// Success is matched via dovecotLoginSucceeded so both dovecot success formats
// count (an earlier version keyed only off "Logged in" and silently skipped
// every classic-format login, which broke RecordSuccess compromise detection).
// The failure marker is "(auth failed" with the opening paren, which
// distinguishes real auth failures from Login-aborted reasons like
// "no auth attempts" or TLS handshake errors.
func (e *AccountExtractor) extractMailLoginEvent(line string) (ip, account string, success bool) {
switch {
case dovecotLoginSucceeded(line):
success = true
case strings.Contains(line, "(auth failed"):
success = false
default:
return "", "", false
}
// Keep the key and its whole-login guarantee on the same config snapshot.
account = e.Extract(line)
ip = checks.DovecotRemoteIP(line)
return ip, account, success
}
// dovecotAttemptsCap bounds how many failures one "Login aborted" line may
// record, so a forged or absurd count cannot flood the tracker.
const dovecotAttemptsCap = 20
// dovecotFailedAttempts returns the number of password attempts a Dovecot
// "Login aborted ... (auth failed, N attempts in S secs)" line reports, or 1
// when the line carries no count. Dovecot writes one such line per
// connection; counting it once let a client that tries many passwords per
// connection stay under every per-IP threshold.
func dovecotFailedAttempts(line string) int {
const marker = "(auth failed, "
i := strings.Index(line, marker)
if i < 0 {
return 1
}
rest := line[i+len(marker):]
end := strings.IndexByte(rest, ' ')
if end <= 0 || !strings.HasPrefix(strings.TrimSpace(rest[end:]), "attempts") {
return 1
}
n, err := strconv.Atoi(rest[:end])
if err != nil || n < 1 {
return 1
}
if n > dovecotAttemptsCap {
return dovecotAttemptsCap
}
return n
}
// recordDovecotFailure records one failure per attempt the Dovecot line
// reports and returns every finding those records produced.
func recordDovecotFailure(t *mailAuthTracker, ip, account, line string, mb sprayMailbox, obs alert.Observation) []alert.Finding {
var findings []alert.Finding
for i := dovecotFailedAttempts(line); i > 0; i-- {
findings = append(findings, t.RecordResolvedObserved(ip, account, mb, obs)...)
}
return findings
}
package daemon
import (
"fmt"
"regexp"
"strings"
"sync/atomic"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
)
// AccountExtractor pulls the account/mailbox identifier out of a mail
// server log line. Used by mailbrute for per-account scoring. Selected
// by cfg.Thresholds.MailBruteAccountKey at daemon startup and SIGHUP
// reload.
type AccountExtractor struct {
mode string
re *regexp.Regexp
}
// NewAccountExtractor parses the spec string from cfg.Thresholds.MailBruteAccountKey.
// Empty spec defaults to "builtin:dovecot-user" (matches the legacy behavior).
func NewAccountExtractor(spec string) (*AccountExtractor, error) {
switch {
case spec == "" || spec == "builtin:dovecot-user":
return &AccountExtractor{mode: "dovecot-user"}, nil
case spec == "builtin:postfix-sasl":
return &AccountExtractor{mode: "postfix-sasl"}, nil
case strings.HasPrefix(spec, "regex:"):
re, err := regexp.Compile(strings.TrimPrefix(spec, "regex:"))
if err != nil {
return nil, fmt.Errorf("invalid regex: %w", err)
}
if re.NumSubexp() < 1 {
return nil, fmt.Errorf("regex must contain at least one capture group")
}
return &AccountExtractor{mode: "regex", re: re}, nil
default:
return nil, fmt.Errorf("unknown extractor spec: %s", spec)
}
}
// Extract returns the account/mailbox key, or "" when no match.
func (e *AccountExtractor) Extract(line string) string {
switch e.mode {
case "dovecot-user":
user, _ := checks.DovecotLoginFields(line)
return user
case "postfix-sasl":
return extractEqualsValue(line, "sasl_username=")
case "regex":
m := e.re.FindStringSubmatch(line)
if len(m) >= 2 {
return m[1]
}
}
return ""
}
// readsWholeLogin reports whether Extract gives the login name as typed. A
// regex can capture part of it, and a part says nothing about the mailbox.
func (e *AccountExtractor) readsWholeLogin() bool {
return e.mode == "dovecot-user"
}
// extractEqualsValue matches `key=<value>` where value is delimited by
// whitespace or comma (postfix log format).
func extractEqualsValue(line, key string) string {
idx := strings.Index(line, key)
if idx < 0 {
return ""
}
rest := line[idx+len(key):]
end := strings.IndexAny(rest, " ,\t\n")
if end < 0 {
return rest
}
return rest[:end]
}
// defaultAccountExtractor is the package-level singleton set at daemon
// startup and safe config reloads.
var defaultAccountExtractor atomic.Pointer[AccountExtractor]
func installAccountExtractorFromConfig(cfg *config.Config) error {
ex, err := NewAccountExtractor(cfg.Thresholds.MailBruteAccountKey)
if err != nil {
return fmt.Errorf("invalid mail_brute_account_key: %w", err)
}
SetAccountExtractor(ex)
return nil
}
// SetAccountExtractor installs the configured extractor; called from
// Daemon.Run() and safe reload after applyDefaults has set the spec.
func SetAccountExtractor(ex *AccountExtractor) {
defaultAccountExtractor.Store(ex)
}
// currentAccountExtractor returns the installed extractor, or lazily
// initializes the default so test code that doesn't call SetAccountExtractor
// gets the legacy dovecot-user behavior.
func currentAccountExtractor() *AccountExtractor {
if ex := defaultAccountExtractor.Load(); ex != nil {
return ex
}
ex, _ := NewAccountExtractor("")
defaultAccountExtractor.Store(ex)
return ex
}
package daemon
import (
"context"
"net"
"path/filepath"
"time"
"github.com/pidginhost/csm/internal/firewall"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/mailranges"
"github.com/pidginhost/csm/internal/obs"
)
const (
mailRangesRefreshInterval = 12 * time.Hour
mailRangesStartupDelay = 5 * time.Minute
)
// mailRangesResolver is the DNS resolver for the periodic SPF-based refresh.
// Replaced in tests to avoid live network calls.
var mailRangesResolver mailranges.Resolver = net.DefaultResolver
// mailRangesReapplyFn applies updated provider nets to the firewall engine's
// DoS-exempt sets. Replaced in tests to inject reapply failures without
// requiring a live nftables connection.
var mailRangesReapplyFn = func(e *firewall.Engine, nets []*net.IPNet) error {
return e.RefreshDOSExemptSets(nets)
}
func (d *Daemon) mailRangesCachePath() string {
return filepath.Join(d.cfg.StatePath, "mailranges.json")
}
// initMailRanges loads the on-disk mail-provider range cache synchronously.
// This must be called before startFirewall() so that engine.SetDOSExemptProviderNets
// receives a full provider set and Apply() builds dos_exempt_nets from day zero.
//
// LoadCache errors are non-fatal: when the cache is absent or corrupt the
// embedded snapshot is published and the daemon continues. The error is
// logged so operators know the fallback is active.
func (d *Daemon) initMailRanges() {
if err := mailranges.LoadCache(d.mailRangesCachePath()); err != nil {
csmlog.Warn("mailranges: cache load failed, using embedded snapshot", "err", err)
}
d.wg.Add(1)
obs.Go("mailranges-refresh", d.mailRangesRefreshLoop)
}
// mailRangesRefreshLoop periodically refreshes the mail-provider IP ranges
// (Google, Microsoft) so the firewall's DoS-exempt sets remain current.
// A short startup delay mirrors the botranges updater so the daemon does not
// hit external DNS before the host is fully settled.
func (d *Daemon) mailRangesRefreshLoop() {
defer d.wg.Done()
select {
case <-d.stopCh:
return
case <-time.After(mailRangesStartupDelay):
}
d.doMailRangesRefresh()
ticker := time.NewTicker(mailRangesRefreshInterval)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
d.doMailRangesRefresh()
}
}
}
// doMailRangesRefresh runs one refresh cycle: resolves SPF records, updates
// the on-disk cache, and reapplies the firewall DoS-exempt sets. On reapply
// failure it restores the previous provider snapshot so the in-memory state
// stays consistent with what is actually live in nftables.
func (d *Daemon) doMailRangesRefresh() {
cachePath := d.mailRangesCachePath()
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
// Cancel in-flight DNS promptly on daemon shutdown.
go func() {
select {
case <-d.stopCh:
cancel()
case <-ctx.Done():
}
}()
// Snapshot the provider set before refresh so it can be restored if the
// subsequent firewall reapply fails. The snapshot and the reapply must be
// consistent: ProviderNets() must reflect what is in nftables.
prev := mailranges.ProviderSnapshot()
n, err := mailranges.Refresh(ctx, mailRangesResolver, cachePath)
if err != nil {
csmlog.Warn("mailranges: refresh error", "err", err)
}
if n == 0 {
// All providers failed; Refresh did not update the active snapshot.
return
}
csmlog.Info("mailranges: providers refreshed", "prefixes", n)
if d.fwEngine == nil {
return
}
if reapplyErr := mailRangesReapplyFn(d.fwEngine, mailranges.ProviderNets()); reapplyErr != nil {
// Restore the previous snapshot so ProviderNets() reflects what is
// actually in nftables, not the candidate update that failed to apply.
// Auto-response subnets for the failed candidate are not pruned.
mailranges.PublishProviderSnapshot(prev)
csmlog.Warn("mailranges: firewall reapply failed; previous snapshot restored", "err", reapplyErr)
}
}
package daemon
import (
"cmp"
"iter"
"maps"
"math"
"net/netip"
"slices"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/store"
)
// sprayMode says what the spray rules do with a verdict. Evidence and trust
// are recorded in every mode, so trust keeps closing the slow-brute hole even
// with the rules off.
type sprayMode uint8
const (
sprayOff sprayMode = iota
sprayObserve
sprayEnforce
)
func parseSprayMode(s string) sprayMode {
switch s {
case "observe":
return sprayObserve
case "enforce":
return sprayEnforce
}
return sprayOff
}
type sprayConfig struct {
Mode sprayMode
Window time.Duration // evidence and trust memory; 0 disables the component
BreadthMailboxes int // untrusted mailboxes that block; 0 disables
MissingNetworks int // networks that must try a missing name first; 0 disables
TrustAge time.Duration // trust counts only after this age
Suppression time.Duration // least gap between verdicts for one IP
MaxIPs int // tracked IP cap
BlockTTL time.Duration // length of an enforce mode spray block
}
// sprayMinTrackedPerTracker is the validated floor of each tracker's
// max_tracked, used when a config that skipped defaults leaves one at 0.
const sprayMinTrackedPerTracker = 1000
// sprayConfigFromThresholds derives the spray settings from the thresholds
// block. Trust ages and verdicts repeat on the slower of the two trackers,
// and the IP cap is their combined capacity. A block length that is unset
// or does not parse takes the default.
func sprayConfigFromThresholds(cfg *config.Config) sprayConfig {
th := &cfg.Thresholds
maxTracked := func(n int) int {
if n <= 0 {
return sprayMinTrackedPerTracker
}
return n
}
blockTTL, err := time.ParseDuration(th.MailSprayBlockExpiry)
if err != nil || blockTTL <= 0 {
blockTTL, _ = time.ParseDuration(config.DefaultMailSprayBlockExpiry)
}
return sprayConfig{
Mode: parseSprayMode(th.MailSprayMode),
Window: time.Duration(th.MailSprayWindowMin) * time.Minute,
BreadthMailboxes: th.MailSprayBreadthMailboxes,
MissingNetworks: th.MailSprayMissingNetworks,
TrustAge: time.Duration(max(th.SMTPBruteForceWindowMin, th.MailBruteForceWindowMin)) * time.Minute,
Suppression: time.Duration(max(th.SMTPBruteForceSuppressMin, th.MailBruteForceSuppressMin)) * time.Minute,
MaxIPs: maxTracked(th.SMTPBruteForceMaxTracked) + maxTracked(th.MailBruteForceMaxTracked),
BlockTTL: blockTTL,
}
}
type sprayRule string
const (
sprayRuleMissingMailbox sprayRule = "missing_mailbox"
sprayRuleBreadth sprayRule = "breadth"
sprayRuleCompromise sprayRule = "compromise"
sprayRuleWeakCompromise sprayRule = "weak_compromise"
sprayRuleSingleMailboxSuccess sprayRule = "single_mailbox_success"
sprayRuleRepeat sprayRule = "repeat"
)
type sprayVerdict struct {
Rule sprayRule
Block bool // missing_mailbox, breadth, repeat, and compromise unless downgraded
Mailbox string // the login's mailbox (success rules) or the missing one
Targets []string // untrusted names in memory, sorted; "" is an accountless failure
Extra int // untrusted names beyond Targets
Networks int // networks that tried the missing name
BlockTTL time.Duration // block length; set only on enforce mode block verdicts
}
// sprayStaleSpread is how long before a login the failures on its mailbox
// must have started for the stale password advisory (4d). A user who
// mistypes and retries within the hour is not news.
const sprayStaleSpread = time.Hour
const (
maxSprayNamesPerIP = 16
maxSprayTrustPerIP = mailGoodSourceMaxAccountsPerIP
maxSprayTrustIPsPerMailbox = 64
maxSprayCorroborationNames = 20000
maxSprayNetworksPerName = 50
maxSprayMarkers = 50000
)
type sprayFailure struct {
first time.Time // first failure since the name was last idle for a whole window
last time.Time
count int // failures since first
exist mailboxExistence
}
type sprayTrust struct{ first, last time.Time }
type sprayTrustKey struct{ ip, mailbox string }
// sprayIP is the failure evidence of one evidence key (sprayEvidenceKey).
type sprayIP struct {
failed map[string]sprayFailure
weakAlerted map[string]time.Time // success mailbox to its last compromise advisory
revoked map[string]time.Time // known cracks cannot earn trust during the window
sources map[string]spraySource // verdict cadence by address
untracked time.Time // latest untrusted failure refused at the name cap
revokedOverflow time.Time // latest crack refused at the name cap; survives release
verdictAt time.Time
observedAt time.Time // last observe mode block verdict
observedNames int // untrusted names it listed
lastSeen time.Time
}
// spraySource is one address's verdict cadence. Blocks target the address,
// so an IPv6 /64 shares evidence but each of its addresses draws and is
// suppressed from verdicts on its own.
type spraySource struct{ blockUntil, compromiseUntil time.Time }
func (s spraySource) lapse() time.Time {
if s.blockUntil.After(s.compromiseUntil) {
return s.blockUntil
}
return s.compromiseUntil
}
// sprayName is the corroboration for one missing mailbox name: which networks
// tried it and when.
type sprayName struct {
networks map[string]time.Time
last time.Time
}
// mailSpray is the long-memory login spray detector shared by the SMTP and
// IMAP/POP trackers. Trackers call it while holding their own lock, so it
// must never call back into a tracker or take any other lock.
//
// Mailbox names are taken as given: callers pass them through
// normalizeSprayMailbox once, at their boundary.
//
// Evidence and trust are two pools, each capped at MaxIPs entries and
// evicted on its own, so a flood of failures cannot evict trust and a flood
// of clean logins (anyone can own a mailbox on a shared host) cannot evict
// evidence. One IP may sit in both. Evidence is kept per IPv4 address and
// per IPv6 /64; trust is kept per address.
type mailSpray struct {
mu sync.Mutex
cfg sprayConfig
now func() time.Time
ips map[string]*sprayIP // evidence pool, by sprayEvidenceKey
names map[string]*sprayName
trust map[sprayTrustKey]sprayTrust // trust pool
trustByIP map[string]map[string]struct{} // ip to its trusted mailboxes
trustByMailbox map[string]map[string]struct{} // mailbox to the IPs it trusts
trustByKey map[string]map[string]struct{} // evidence key to its IPs holding trust
// Block markers: address to the end of its spray block. A failure from
// the address up to a window past that end is blocked again (rule 5).
// Kept per address, indexed and capped per evidence key.
blocks map[string]time.Time
blocksByKey map[string]map[string]struct{}
// Eviction floors of the evidence and corroboration pools, set when a
// full pool's batch eviction stops before entries over a bar (see
// sprayEvict). Ranks change with time and config, so Purge, SetConfig
// and Load clear them.
ipFloor, nameFloor int
// Persistence bookkeeping, so a save writes only what changed. full
// makes the next save rewrite the whole bucket instead: at start, after
// Load or a reset, and if the bookkeeping outgrows its bound.
full bool
ipChanges sprayChanges // evidence IPs
trustChanges sprayChanges // trust IPs
nameChanges sprayChanges // corroboration names
blockChanges sprayChanges // block markers
}
// sprayChanges tracks one kind of saved key: those changed since the last
// save and those the bucket holds.
type sprayChanges struct{ dirty, saved map[string]struct{} }
func newSprayChanges() sprayChanges {
return sprayChanges{dirty: make(map[string]struct{}), saved: make(map[string]struct{})}
}
func newMailSpray(cfg sprayConfig, now func() time.Time) *mailSpray {
return &mailSpray{
cfg: cfg,
now: now,
ips: make(map[string]*sprayIP),
names: make(map[string]*sprayName),
trust: make(map[sprayTrustKey]sprayTrust),
trustByIP: make(map[string]map[string]struct{}),
trustByMailbox: make(map[string]map[string]struct{}),
trustByKey: make(map[string]map[string]struct{}),
blocks: make(map[string]time.Time),
blocksByKey: make(map[string]map[string]struct{}),
full: true,
ipChanges: newSprayChanges(),
trustChanges: newSprayChanges(),
nameChanges: newSprayChanges(),
blockChanges: newSprayChanges(),
}
}
// SetConfig applies a reloaded config. Turning the component off drops its
// memory.
func (m *mailSpray) SetConfig(cfg sprayConfig) {
m.mu.Lock()
defer m.mu.Unlock()
m.cfg = cfg
m.ipFloor, m.nameFloor = 0, 0
if cfg.Window <= 0 {
clear(m.ips)
clear(m.names)
clear(m.trust)
clear(m.trustByIP)
clear(m.trustByMailbox)
clear(m.trustByKey)
clear(m.blocks)
clear(m.blocksByKey)
m.resetChanges()
return
}
m.evictIPs("", m.now())
m.evictTrust(sprayTrustKey{})
}
// Enabled reports whether the component keeps memory: with no window it
// records nothing, so its verdicts say nothing about a login.
func (m *mailSpray) Enabled() bool {
m.mu.Lock()
defer m.mu.Unlock()
return m.cfg.Window > 0
}
func (m *mailSpray) Mode() sprayMode {
m.mu.Lock()
defer m.mu.Unlock()
if m.cfg.Window <= 0 {
return sprayOff
}
return m.cfg.Mode
}
// Failure records a failed login on a normalised mailbox ("" when the log
// names none) and returns a block verdict for rule 2, rule 3 or, in enforce
// mode, rule 5 (an address blocked for spraying is back).
func (m *mailSpray) Failure(ip, mailbox string, exist mailboxExistence) *sprayVerdict {
ip = sprayAddr(ip)
m.mu.Lock()
defer m.mu.Unlock()
if m.cfg.Window <= 0 {
return nil
}
if mailbox == "" {
exist = mailboxExistenceUnknown
}
now := m.now()
key := sprayEvidenceKey(ip)
e := m.touch(key, now)
// Rank the current IP after its verdict, including it as an eviction
// candidate so fresh noise cannot displace stronger evidence.
defer m.evictIPs(key, now)
stored := m.recordFailure(e, key, mailbox, exist, now)
networks := 0
if exist == mailboxMissing {
networks = m.corroborate(mailbox, ip, now)
}
src := e.sources[ip]
if m.cfg.Mode == sprayOff || now.Before(src.blockUntil) {
return nil
}
untrusted := m.untrusted(e, key, now)
if end, ok := m.blocks[ip]; ok && m.cfg.Mode == sprayEnforce && !now.After(end.Add(m.cfg.Window)) {
src.blockUntil = now.Add(m.cfg.Suppression)
e.setSource(ip, src)
return m.verdict(e, ip, sprayRuleRepeat, true, "", untrusted, now)
}
missing := m.cfg.MissingNetworks > 0 && exist == mailboxMissing &&
networks >= m.cfg.MissingNetworks && m.keyTrustCount(key, "", 1, now) == 0
if !missing && (m.cfg.BreadthMailboxes <= 0 || !m.breadth(key, mailbox, stored, untrusted, now)) {
return nil
}
// Observe mode blocks nothing, so an IP over the bar keeps failing.
if m.cfg.Mode == sprayObserve && !e.noteObserved(len(untrusted), now, m.cfg.Window) {
return nil
}
src.blockUntil = now.Add(m.cfg.Suppression)
e.setSource(ip, src)
if missing {
v := m.verdict(e, ip, sprayRuleMissingMailbox, true, mailbox, untrusted, now)
v.Networks = networks
return v
}
return m.verdict(e, ip, sprayRuleBreadth, true, "", untrusted, now)
}
// breadth reports whether the key's untrusted names reach the rule 3 bar. A
// name refused at the per-IP cap still counts, and a full IP satisfies any
// bar above the cap.
func (m *mailSpray) breadth(key, mailbox string, stored bool, untrusted []string, now time.Time) bool {
n := len(untrusted)
if !stored && !m.excused(key, mailbox, now) {
n++
}
return n >= min(m.cfg.BreadthMailboxes, maxSprayNamesPerIP)
}
// Success records a successful login on a normalised mailbox. clean is true
// when the login earned or refreshed trust. A login that follows failures on
// other untrusted mailboxes earns nothing and may be a compromise, unless
// they all name one mailbox not known to be missing (one stale device) and
// the login's own mailbox did not fail. A login never adds to the evidence
// pool.
func (m *mailSpray) Success(ip, mailbox string) (clean bool, v *sprayVerdict) {
ip = sprayAddr(ip)
m.mu.Lock()
defer m.mu.Unlock()
if m.cfg.Window <= 0 || mailbox == "" {
return false, nil
}
now := m.now()
k := sprayTrustKey{ip, mailbox}
key := sprayEvidenceKey(ip)
e := m.ips[key]
if m.agedTrust(ip, mailbox, now) || m.trustPredatesFailures(e, k, key, now) {
t := m.trust[k]
t.last = now
m.trust[k] = t
m.markTrust(ip)
return true, nil
}
// An IPv6 host rotates its address. A login to a mailbox another
// address of its /64 holds aged trust on is the same user: clean, and
// the new address starts trust that ages on its own.
if m.excused(key, mailbox, now) {
if e != nil {
if at, ok := e.revoked[mailbox]; ok && now.Sub(at) <= m.cfg.Window {
return false, nil
}
}
m.recordTrust(k, now)
return true, nil
}
if e != nil {
m.markIP(key)
}
var untrusted []string
if e != nil {
untrusted = m.untrusted(e, key, now)
}
others := slices.DeleteFunc(slices.Clone(untrusted), func(n string) bool { return n == mailbox })
// One office device with a stale saved password can fail all day on one
// other mailbox; that alone taints no login to a mailbox that did not
// fail. A missing name, a failure that names no mailbox, a second
// mailbox, or failures on the login's own mailbox still do.
stale := len(untrusted) == 1 && len(others) == 1 && others[0] != "" && e.failed[others[0]].exist != mailboxMissing
if (len(others) == 0 || stale) && (e == nil || !e.hasUntracked(now, m.cfg.Window)) {
if e != nil {
if at, ok := e.revoked[mailbox]; ok && now.Sub(at) <= m.cfg.Window {
return false, nil
}
}
hadTrust := m.recordTrust(k, now)
// 4d fires once per trust relationship; later logins while the new
// trust ages are the same user.
if m.cfg.Mode == sprayOff || hadTrust || e == nil || !e.staleFailures(mailbox, now, m.cfg.Window) {
return true, nil
}
return true, m.verdict(e, ip, sprayRuleSingleMailboxSuccess, false, mailbox, []string{mailbox}, now)
}
src := e.sources[ip]
if m.cfg.Mode == sprayOff || now.Before(src.compromiseUntil) {
return false, nil
}
rule, block := sprayRuleWeakCompromise, false
if slices.ContainsFunc(others, func(n string) bool { return m.corroboratedMissing(e, n, now) }) {
rule = sprayRuleCompromise
block = m.keyTrustCount(key, mailbox, mailEstablishedSourceAccounts, now) < mailEstablishedSourceAccounts
}
// A polling client logs in every few minutes: an advisory fires once per
// mailbox per window, not once per suppression period.
if (rule == sprayRuleWeakCompromise || m.cfg.Mode != sprayEnforce) && !e.noteWeakAlert(mailbox, now, m.cfg.Window) {
return false, nil
}
src.compromiseUntil = now.Add(m.cfg.Suppression)
e.setSource(ip, src)
return false, m.verdict(e, ip, rule, block, mailbox, untrusted, now)
}
// trustPredatesFailures reports whether k holds live trust that began
// before every untrusted failure on another mailbox: a clean login those
// failures came after, so they cannot taint it.
func (m *mailSpray) trustPredatesFailures(e *sprayIP, k sprayTrustKey, key string, now time.Time) bool {
t, ok := m.trust[k]
if !ok || now.Sub(t.last) > m.cfg.Window {
return false
}
if e == nil {
return true
}
// A failure refused at the name cap has no first time to compare.
if e.hasUntracked(now, m.cfg.Window) {
return false
}
for name, f := range e.failed {
if name != k.mailbox && now.Sub(f.last) <= m.cfg.Window && !f.first.After(t.first) && !m.excused(key, name, now) {
return false
}
}
return true
}
// RevokeTrust drops trust on a normalised mailbox for every address of the
// IP's evidence key, for a login the caller knows was a guessed password. Remember the crack for the evidence
// window so another login, on either protocol, cannot immediately restore it.
func (m *mailSpray) RevokeTrust(ip, mailbox string) {
ip = sprayAddr(ip)
m.mu.Lock()
defer m.mu.Unlock()
key := sprayEvidenceKey(ip)
m.dropKeyTrust(key, mailbox)
if m.cfg.Window <= 0 {
return
}
now := m.now()
m.touch(key, now).noteRevoked(mailbox, now, m.cfg.Window)
m.evictIPs(key, now)
}
// dropKeyTrust drops every address of evidence key's trust on mailbox: a
// crack must not stay excused through a sibling address of the /64.
func (m *mailSpray) dropKeyTrust(key, mailbox string) {
var ips []string
for ip := range m.trustByMailbox[mailbox] {
if _, ok := m.trustByKey[key][ip]; ok {
ips = append(ips, ip)
}
}
for _, ip := range ips {
m.dropTrust(sprayTrustKey{ip, mailbox})
}
}
// ForgetRecentFailures drops the IP's failures since "since": an outage
// starts before it is detected, and its failures are no evidence. A name
// that also failed before keeps only its first failure. Trust, verdicts
// and corroboration stay.
func (m *mailSpray) ForgetRecentFailures(ip string, since time.Time) {
ip = sprayAddr(ip)
m.mu.Lock()
defer m.mu.Unlock()
key := sprayEvidenceKey(ip)
e := m.ips[key]
if e == nil {
return
}
changed := false
for name, f := range e.failed {
switch {
case f.last.Before(since):
continue
case f.first.Before(since):
f.last, f.count = f.first, 1
e.failed[name] = f
default:
delete(e.failed, name)
}
changed = true
}
if changed {
m.markIP(key)
}
}
// ForgetBlock drops ip's block marker and nothing else, for a self-service
// unban.
func (m *mailSpray) ForgetBlock(ip string) {
ip = sprayAddr(ip)
m.mu.Lock()
defer m.mu.Unlock()
if _, ok := m.blocks[ip]; ok {
m.dropBlock(ip)
}
}
// Forget drops what an operator release of ip ends: its block marker, its
// verdict cadence and the failure evidence of its evidence key. For IPv6
// that is the whole /64, since evidence kept for the /64 would block the
// released address again on its next failure. Trust and crack records are
// about mailboxes, not the block, and stay, as do the markers and cadence
// of the key's other addresses. An entry left with no records goes at the
// next Purge.
func (m *mailSpray) Forget(ip string) {
ip = sprayAddr(ip)
m.mu.Lock()
defer m.mu.Unlock()
if _, ok := m.blocks[ip]; ok {
m.dropBlock(ip)
}
key := sprayEvidenceKey(ip)
e := m.ips[key]
if e == nil {
return
}
delete(e.sources, ip)
e.forgetFailures()
m.markIP(key)
// The entry may now rank below the eviction floor.
m.ipFloor = 0
}
// ForgetAll is Forget for every address, after a firewall flush, and also
// drops corroboration. A flush is rare, so the next save rewrites the
// bucket.
func (m *mailSpray) ForgetAll() {
m.mu.Lock()
defer m.mu.Unlock()
for _, e := range m.ips {
e.sources = nil
e.forgetFailures()
}
clear(m.names)
clear(m.blocks)
clear(m.blocksByKey)
m.ipFloor, m.nameFloor = 0, 0
m.resetChanges()
}
// TrustedAll reports whether the IP holds aged trust on every one of the
// normalised mailboxes. An empty list is not trusted.
func (m *mailSpray) TrustedAll(ip string, mailboxes []string) bool {
ip = sprayAddr(ip)
m.mu.Lock()
defer m.mu.Unlock()
if m.cfg.Window <= 0 || len(mailboxes) == 0 {
return false
}
now := m.now()
for _, mb := range mailboxes {
if !m.agedTrust(ip, mb, now) {
return false
}
}
return true
}
// FirstUntrustedFailure returns the start of the address's active untrusted
// failures. Trust earned beside one stale mailbox vouches for its own login,
// but cannot establish the source as legitimate for an attack already underway.
func (m *mailSpray) FirstUntrustedFailure(ip string) time.Time {
m.mu.Lock()
defer m.mu.Unlock()
return m.firstUntrustedFailure(sprayEvidenceKey(sprayAddr(ip)), m.now())
}
// Purge drops failures, trust and corroboration older than the window, and
// block markers a window past their end.
func (m *mailSpray) Purge() {
m.mu.Lock()
defer m.mu.Unlock()
if m.cfg.Window <= 0 {
return
}
m.ipFloor, m.nameFloor = 0, 0
cutoff := m.now().Add(-m.cfg.Window)
for ip, e := range m.ips {
records := len(e.failed) + len(e.weakAlerted) + len(e.revoked)
e.pruneFailed(cutoff)
maps.DeleteFunc(e.weakAlerted, func(_ string, at time.Time) bool { return at.Before(cutoff) })
maps.DeleteFunc(e.revoked, func(_ string, at time.Time) bool { return at.Before(cutoff) })
if !e.untracked.IsZero() && e.untracked.Before(cutoff) {
e.untracked = time.Time{}
records++
}
if !e.revokedOverflow.IsZero() && e.revokedOverflow.Before(cutoff) {
e.revokedOverflow = time.Time{}
records++
}
changed := records != len(e.failed)+len(e.weakAlerted)+len(e.revoked)
// ForgetRecentFailures can leave an entry with no records, so an
// entry dropped here may not have changed in this pass.
if len(e.failed) == 0 && e.untracked.IsZero() && e.revokedOverflow.IsZero() && len(e.weakAlerted) == 0 && len(e.revoked) == 0 {
delete(m.ips, ip)
changed = true
}
if changed {
m.markIP(ip)
}
}
for k, t := range m.trust {
if t.last.Before(cutoff) {
m.dropTrust(k)
}
}
for name, n := range m.names {
networks := len(n.networks)
maps.DeleteFunc(n.networks, func(_ string, ts time.Time) bool { return ts.Before(cutoff) })
if len(n.networks) == 0 {
delete(m.names, name)
}
if len(n.networks) != networks {
m.markName(name)
}
}
for ip, end := range m.blocks {
if end.Before(cutoff) {
m.dropBlock(ip)
}
}
}
// Size returns the number of IPs in the evidence pool.
func (m *mailSpray) Size() int {
m.mu.Lock()
defer m.mu.Unlock()
return len(m.ips)
}
// TrustSize returns the number of (IP, mailbox) records in the trust pool.
func (m *mailSpray) TrustSize() int {
m.mu.Lock()
defer m.mu.Unlock()
return len(m.trust)
}
// Export returns a copy of the whole memory.
func (m *mailSpray) Export() store.MailSprayState {
m.mu.Lock()
defer m.mu.Unlock()
return m.exportAll()
}
// ExportChanges returns what changed since the last save and starts a new
// change set. full reports that the save must replace the whole bucket; Put
// then holds the whole memory. Only changed entries are copied under the
// lock otherwise. A caller whose save fails hands the delta to RetryChanges.
func (m *mailSpray) ExportChanges() (d store.MailSprayDelta, full bool) {
m.mu.Lock()
defer m.mu.Unlock()
if m.full {
m.full = false
m.ipChanges.saveAll(maps.Keys(m.ips))
m.trustChanges.saveAll(maps.Keys(m.trustByIP))
m.nameChanges.saveAll(maps.Keys(m.names))
m.blockChanges.saveAll(maps.Keys(m.blocks))
return store.MailSprayDelta{Put: m.exportAll()}, true
}
d.Put = store.MailSprayState{
IPs: make(map[string]store.MailSprayIP, len(m.ipChanges.dirty)),
Trust: make(map[string]map[string]store.GoodSourcePair, len(m.trustChanges.dirty)),
Networks: make(map[string]map[string]time.Time, len(m.nameChanges.dirty)),
Blocked: make(map[string]time.Time, len(m.blockChanges.dirty)),
}
d.DeleteIPs = m.ipChanges.export(func(ip string) bool {
e := m.ips[ip]
if e != nil {
d.Put.IPs[ip] = exportSprayIP(e)
}
return e != nil
})
d.DeleteTrust = m.trustChanges.export(func(ip string) bool {
if len(m.trustByIP[ip]) == 0 {
return false
}
d.Put.Trust[ip] = m.exportTrust(ip)
return true
})
d.DeleteNames = m.nameChanges.export(func(name string) bool {
n := m.names[name]
if n != nil {
d.Put.Networks[name] = maps.Clone(n.networks)
}
return n != nil
})
d.DeleteBlocked = m.blockChanges.export(func(ip string) bool {
end, ok := m.blocks[ip]
if ok {
d.Put.Blocked[ip] = end
}
return ok
})
return d, false
}
// RetryChanges takes back the changes of a failed save, so the next save
// writes those keys again with their values at that time.
func (m *mailSpray) RetryChanges(d store.MailSprayDelta, full bool) {
m.mu.Lock()
defer m.mu.Unlock()
if m.full {
return
}
if full {
m.resetChanges()
return
}
for _, k := range []struct {
changes sprayChanges
puts iter.Seq[string]
deletes []string
mark func(string)
}{
{m.ipChanges, maps.Keys(d.Put.IPs), d.DeleteIPs, m.markIP},
{m.trustChanges, maps.Keys(d.Put.Trust), d.DeleteTrust, m.markTrust},
{m.nameChanges, maps.Keys(d.Put.Networks), d.DeleteNames, m.markName},
{m.blockChanges, maps.Keys(d.Put.Blocked), d.DeleteBlocked, m.markBlock},
} {
// The deletes did not happen: the bucket still holds those keys.
for _, key := range k.deletes {
k.changes.saved[key] = struct{}{}
k.mark(key)
if m.full {
return
}
}
for key := range k.puts {
k.mark(key)
if m.full {
return
}
}
}
}
func (m *mailSpray) markIP(ip string) {
m.mark(m.ipChanges, ip, m.ips[ip] != nil, m.cfg.MaxIPs)
}
func (m *mailSpray) markTrust(ip string) {
m.mark(m.trustChanges, ip, len(m.trustByIP[ip]) > 0, m.cfg.MaxIPs)
}
func (m *mailSpray) markName(name string) {
m.mark(m.nameChanges, name, m.names[name] != nil, maxSprayCorroborationNames)
}
func (m *mailSpray) markBlock(ip string) {
_, live := m.blocks[ip]
m.mark(m.blockChanges, ip, live, maxSprayMarkers)
}
// mark records that key changed; live says whether memory still holds it.
// A key gone from memory that the bucket never held needs no delete, so it
// leaves the change set: attacker churn of short-lived entries cannot grow
// the set or force a rewrite of idle entries. The sets stay within the pool
// plus the bucket while saves succeed; saves that keep failing can grow
// them, so past twice the pool the next save rewrites the bucket instead.
func (m *mailSpray) mark(c sprayChanges, key string, live bool, pool int) {
if m.full {
return
}
if _, saved := c.saved[key]; live || saved {
c.dirty[key] = struct{}{}
} else {
delete(c.dirty, key)
}
if len(c.dirty) > 2*pool || len(c.saved) > 2*pool {
m.resetChanges()
}
}
func (m *mailSpray) resetChanges() {
m.full = true
for _, c := range []sprayChanges{m.ipChanges, m.trustChanges, m.nameChanges, m.blockChanges} {
clear(c.dirty)
clear(c.saved)
}
}
// saveAll records a full save of keys.
func (c sprayChanges) saveAll(keys iter.Seq[string]) {
clear(c.dirty)
clear(c.saved)
for k := range keys {
c.saved[k] = struct{}{}
}
}
// export hands each changed key to put, which copies a key memory still
// holds and reports whether it did. It returns the keys to delete and
// records them as saved or gone, assuming the save succeeds.
func (c sprayChanges) export(put func(string) bool) (deletes []string) {
for k := range c.dirty {
if put(k) {
c.saved[k] = struct{}{}
} else {
deletes = append(deletes, k)
delete(c.saved, k)
}
}
clear(c.dirty)
return deletes
}
func (m *mailSpray) exportAll() store.MailSprayState {
st := store.MailSprayState{
IPs: make(map[string]store.MailSprayIP, len(m.ips)),
Trust: make(map[string]map[string]store.GoodSourcePair, len(m.trustByIP)),
Networks: make(map[string]map[string]time.Time, len(m.names)),
Blocked: maps.Clone(m.blocks),
}
for ip, e := range m.ips {
st.IPs[ip] = exportSprayIP(e)
}
for ip := range m.trustByIP {
st.Trust[ip] = m.exportTrust(ip)
}
for name, n := range m.names {
st.Networks[name] = maps.Clone(n.networks)
}
return st
}
func exportSprayIP(e *sprayIP) store.MailSprayIP {
s := store.MailSprayIP{
Untracked: e.untracked,
RevokedOverflow: e.revokedOverflow,
VerdictAt: e.verdictAt,
ObservedAt: e.observedAt,
ObservedNames: e.observedNames,
LastSeen: e.lastSeen,
}
if len(e.failed) > 0 {
s.Failed = make(map[string]store.MailSprayFailure, len(e.failed))
for name, f := range e.failed {
s.Failed[name] = store.MailSprayFailure{First: f.first, Last: f.last, Count: f.count, Exist: uint8(f.exist)}
}
}
if len(e.weakAlerted) > 0 {
s.WeakAlerted = maps.Clone(e.weakAlerted)
}
if len(e.revoked) > 0 {
s.Revoked = maps.Clone(e.revoked)
}
if len(e.sources) > 0 {
s.Sources = make(map[string]store.MailSpraySource, len(e.sources))
for ip, src := range e.sources {
s.Sources[ip] = store.MailSpraySource{SuppressedUntil: src.blockUntil, CompromiseUntil: src.compromiseUntil}
}
}
return s
}
func (m *mailSpray) exportTrust(ip string) map[string]store.GoodSourcePair {
out := make(map[string]store.GoodSourcePair, len(m.trustByIP[ip]))
for mb := range m.trustByIP[ip] {
t := m.trust[sprayTrustKey{ip, mb}]
out[mb] = store.GoodSourcePair{First: t.first, Last: t.last}
}
return out
}
// Load replaces memory before log readers start. The reader visits trust before evidence
// so name eviction can prefer untrusted failures. Each pool is capped during
// replay, rather than after allocating a copy of an untrusted bucket.
func (m *mailSpray) Load(read func(func(store.MailSprayState)) error) error {
m.mu.Lock()
defer m.mu.Unlock()
if m.cfg.Window <= 0 {
return nil
}
now := m.now()
cutoff := now.Add(-m.cfg.Window)
clear(m.ips)
clear(m.names)
clear(m.trust)
clear(m.trustByIP)
clear(m.trustByMailbox)
clear(m.trustByKey)
clear(m.blocks)
clear(m.blocksByKey)
// Entries load with no newcomer to check against the eviction floors,
// so a floor a load batch sets may not hold once it is done.
defer func() { m.ipFloor, m.nameFloor = 0, 0 }()
// The bucket may hold entries this load dropped: rewrite it whole.
m.resetChanges()
return read(func(st store.MailSprayState) {
for saved, mbs := range st.Trust {
if !sprayValidIP(saved) {
continue
}
ip := sprayAddr(saved)
for mb, p := range mbs {
mb = strings.Clone(normalizeSprayMailbox(mb))
if mb == "" || p.First.After(now) || p.Last.After(now) ||
!validGoodSourceTimes(goodSourceTimes{First: p.First, Last: p.Last}, cutoff) {
continue
}
m.loadTrust(sprayTrustKey{ip, mb}, sprayTrust{p.First, p.Last})
}
}
for saved, s := range st.IPs {
key := sprayLoadEvidenceKey(saved)
if key == "" {
continue
}
if e := m.loadIP(m.ips[key], key, s, cutoff, now); e != nil {
m.ips[key] = e
m.evictIPs("", now)
}
}
for name, nets := range st.Networks {
name = strings.Clone(normalizeSprayMailbox(name))
if name == "" {
continue
}
n := m.names[name]
if n == nil {
n = &sprayName{networks: make(map[string]time.Time)}
}
for key, at := range nets {
// A future observation cannot corroborate a real failure now.
if key = sprayLoadNetwork(key); key == "" || at.Before(cutoff) || at.After(now) {
continue
}
sprayKeepLatest(n.networks, key, at, maxSprayNetworksPerName)
if at.After(n.last) {
n.last = at
}
}
if len(n.networks) > 0 {
m.names[name] = n
m.evictNames("", now)
}
}
// A block cannot end later than one starting now would.
for saved, end := range st.Blocked {
if !sprayValidIP(saved) || end.Before(cutoff) {
continue
}
ip := sprayAddr(saved)
m.noteBlock(ip, sprayLatest(m.blocks[ip], sprayNotAfter(end, now.Add(m.cfg.BlockTTL))))
}
})
}
// loadTrust keeps the most recent relationships even when disk key order
// differs from login order. Live addTrust always receives the latest login.
func (m *mailSpray) loadTrust(k sprayTrustKey, t sprayTrust) {
if old, ok := m.trust[k]; ok {
t.first = sprayNotAfter(t.first, old.first)
if old.last.After(t.last) {
t.last = old.last
}
m.trust[k] = t
return
}
if mbs := m.trustByIP[k.ip]; len(mbs) >= maxSprayTrustPerIP {
oldest := m.oldestTrust(mbs, func(mb string) sprayTrustKey { return sprayTrustKey{k.ip, mb} })
if t.last.Before(m.trust[oldest].last) {
return
}
}
if ips := m.trustByMailbox[k.mailbox]; len(ips) >= maxSprayTrustIPsPerMailbox {
oldest := m.oldestTrust(ips, func(ip string) sprayTrustKey { return sprayTrustKey{ip, k.mailbox} })
if t.last.Before(m.trust[oldest].last) {
return
}
}
m.addTrust(k, t)
m.evictTrust(sprayTrustKey{})
}
// loadIP merges one saved entry into the evidence of key, e (nil when there
// is none yet), and returns it, or nil when nothing is left in the window, as
// Purge would drop it. Damaged times and counts are repaired toward weaker
// evidence.
func (m *mailSpray) loadIP(e *sprayIP, key string, s store.MailSprayIP, cutoff, now time.Time) *sprayIP {
if e == nil {
e = &sprayIP{}
}
e.verdictAt = sprayLatest(e.verdictAt, sprayNotAfter(s.VerdictAt, now))
e.lastSeen = sprayLatest(e.lastSeen, sprayNotAfter(s.LastSeen, now))
if u := sprayNotAfter(s.Untracked, now); !u.Before(cutoff) {
e.untracked = sprayLatest(e.untracked, u)
}
if r := sprayNotAfter(s.RevokedOverflow, now); !r.Before(cutoff) {
e.revokedOverflow = sprayLatest(e.revokedOverflow, r)
}
if o := sprayNotAfter(s.ObservedAt, now); !o.Before(cutoff) {
e.observedAt = sprayLatest(e.observedAt, o)
e.observedNames = max(e.observedNames, min(max(s.ObservedNames, 0), maxSprayNamesPerIP))
}
limit := now.Add(m.cfg.Suppression)
for addr, saved := range s.Sources {
if !sprayValidIP(addr) || sprayEvidenceKey(addr) != key {
continue
}
ip := sprayAddr(addr)
src := e.sources[ip]
src.blockUntil = sprayLatest(src.blockUntil, sprayNotAfter(saved.SuppressedUntil, limit))
src.compromiseUntil = sprayLatest(src.compromiseUntil, sprayNotAfter(saved.CompromiseUntil, limit))
e.setSource(ip, src)
}
// Revoke before choosing failures at the name cap, including revocations
// whose names will be represented only by the overflow marker.
for mb, at := range s.Revoked {
mb = strings.Clone(normalizeSprayMailbox(mb))
at = sprayNotAfter(at, now)
if mb == "" || at.Before(cutoff) {
continue
}
m.dropKeyTrust(key, mb)
if old := e.revoked[mb]; old.After(at) {
at = old
}
e.noteRevoked(mb, at, m.cfg.Window)
}
for name, f := range s.Failed {
last := sprayNotAfter(f.Last, now)
if last.Before(cutoff) {
continue
}
first := sprayNotAfter(f.First, now)
if first.IsZero() || first.After(last) {
first = last
}
exist := mailboxExistence(f.Exist)
if exist > mailboxMissing {
exist = mailboxExistenceUnknown
}
count := max(f.Count, 1)
name = strings.Clone(normalizeSprayMailbox(name))
if cur, ok := e.failed[name]; ok {
first = sprayNotAfter(first, cur.first)
if cur.last.After(last) {
last = cur.last
}
count = max(count, cur.count)
exist = foldObservedExistence(exist, cur.exist)
}
if _, ok := e.failed[name]; !ok && !m.roomForName(e, key, name, last, now) {
continue
}
if e.failed == nil {
e.failed = make(map[string]sprayFailure)
}
e.failed[name] = sprayFailure{first: first, last: last, count: count, exist: exist}
}
for mb, at := range s.WeakAlerted {
mb = strings.Clone(normalizeSprayMailbox(mb))
at = sprayNotAfter(at, now)
if mb == "" || at.Before(cutoff) {
continue
}
if e.weakAlerted == nil {
e.weakAlerted = make(map[string]time.Time)
}
sprayKeepLatest(e.weakAlerted, mb, at, maxSprayNamesPerIP)
}
if len(e.failed) == 0 && e.untracked.IsZero() && e.revokedOverflow.IsZero() && len(e.weakAlerted) == 0 && len(e.revoked) == 0 {
return nil
}
return e
}
// sprayKeepLatest merges a timestamp without retaining an unbounded map or
// letting an older disk entry displace newer evidence at the cap.
func sprayKeepLatest(dst map[string]time.Time, key string, at time.Time, limit int) {
if old, ok := dst[key]; ok {
if at.After(old) {
dst[key] = at
}
return
}
if len(dst) >= limit {
oldest := ""
for k, t := range dst {
if oldest == "" || t.Before(dst[oldest]) {
oldest = k
}
}
if at.Before(dst[oldest]) {
return
}
delete(dst, oldest)
}
dst[key] = at
}
// sprayLoadNetwork accepts only a key sprayNetworkKey could have made.
func sprayLoadNetwork(key string) string {
p, err := netip.ParsePrefix(key)
if err != nil || sprayNetworkKey(p.Addr().String()) != key {
return ""
}
return key
}
// sprayLoadEvidenceKey returns the evidence key a saved one belongs to.
// Older state keyed IPv6 evidence by address, which folds into the /64. A
// key that is neither an address nor a /64 sprayEvidenceKey could have made
// returns "".
func sprayLoadEvidenceKey(saved string) string {
if a, err := netip.ParseAddr(saved); err == nil {
if a.Zone() != "" {
return ""
}
return sprayEvidenceKey(saved)
}
p, err := netip.ParsePrefix(saved)
if err != nil || sprayEvidenceKey(p.Addr().String()) != saved {
return ""
}
return saved
}
func sprayLatest(a, b time.Time) time.Time {
if b.After(a) {
return b
}
return a
}
func sprayValidIP(ip string) bool {
a, err := netip.ParseAddr(ip)
return err == nil && a.Zone() == ""
}
func sprayNotAfter(t, limit time.Time) time.Time {
if t.After(limit) {
return limit
}
return t
}
// JSON can escape every mailbox byte into six bytes. The largest live
// record is one IP's trust map; allow its timestamps and field names too.
const maxSprayEntryBytes = max(maxSprayTrustPerIP, 3*maxSprayNamesPerIP, maxSprayNetworksPerName)*(6*maxSprayMailboxBytes+256) + 512
// mailSprayPersistTicks is how many one-minute purge ticks pass between
// saves of the spray memory, which is far larger than good-source standing.
const mailSprayPersistTicks = 5
// loadMailSpray seeds the spray memory before the log readers start.
func (d *Daemon) loadMailSpray() {
sdb := store.Global()
if d.mailSpray == nil || sdb == nil {
return
}
if err := d.mailSpray.Load(func(visit func(store.MailSprayState)) error {
return sdb.LoadMailSpray(maxSprayEntryBytes, visit)
}); err != nil {
csmlog.Warn("mail spray memory load failed", "err", err)
}
}
// persistMailSpray saves the spray memory so a restart keeps it: the whole
// memory the first time, then only what changed.
func (d *Daemon) persistMailSpray() {
sdb := store.Global()
if d.mailSpray == nil || sdb == nil {
return
}
delta, full := d.mailSpray.ExportChanges()
var err error
switch {
case full:
err = sdb.SaveMailSpray(delta.Put)
case !delta.Empty():
err = sdb.ApplyMailSprayDelta(delta)
}
if err != nil {
d.mailSpray.RetryChanges(delta, full)
csmlog.Warn("mail spray memory persistence failed", "err", err)
}
}
// firewallReleased is the firewall's release hook. It runs with the
// firewall lock released, and each lock here is taken alone. A self-service
// unban drops only the address's re-block marker so the unban sticks; its
// failure history and per-IP counts stay, so renewed spraying is blocked
// again on the usual rules. Every other release is an operator release.
// Either kind of release of one address also lifts Stalwart's own ban of it.
func (d *Daemon) firewallReleased(ip string, kind firewall.ReleaseKind) {
if a, err := netip.ParseAddr(ip); err == nil && d.stalwartUnban != nil {
d.stalwartUnban.add(a.Unmap().String())
}
if kind != firewall.ReleaseUnban {
d.operatorReleased(ip)
return
}
if d.mailSpray != nil {
d.mailSpray.ForgetBlock(ip)
}
}
// operatorReleased ends the spray history of an address an operator
// unblocked, allowed or released by undo, or of every address after a
// flush ("").
func (d *Daemon) operatorReleased(ip string) {
if d.mailSpray != nil {
if ip == "" {
d.mailSpray.ForgetAll()
} else {
d.mailSpray.Forget(ip)
}
}
if d.smtpAuthTracker != nil {
d.smtpAuthTracker.Forget(ip)
}
if d.mailAuthTracker != nil {
d.mailAuthTracker.Forget(ip)
}
}
// sprayEvidenceKey is the evidence entry a source's failures go to: the
// address for IPv4 (IPv4-mapped included) and the /64 for IPv6, since one
// IPv6 host can rotate through its /64. Trust and blocks stay per address.
func sprayEvidenceKey(ip string) string {
a, err := netip.ParseAddr(ip)
if err != nil {
return ip
}
a = a.Unmap().WithZone("")
if a.Is4() {
return a.String()
}
p, _ := a.Prefix(64)
return p.String()
}
// sprayAddr is the one spelling of an address that its marker, verdict
// cadence and trust are kept under, so every spelling of it shares them:
// IPv4-mapped IPv6 as IPv4, IPv6 lower case and compressed. An ip that
// does not parse is kept as given.
func sprayAddr(ip string) string {
a, err := netip.ParseAddr(ip)
if err != nil {
return ip
}
return a.Unmap().String()
}
// sprayNetworkKey is the network a source counts as for corroboration: the
// IPv4 /16 (IPv4-mapped included) or the IPv6 /32. "" if ip does not parse.
func sprayNetworkKey(ip string) string {
a, err := netip.ParseAddr(ip)
if err != nil {
return ""
}
a = a.Unmap().WithZone("")
bits := 32
if a.Is4() {
bits = 16
}
p, err := a.Prefix(bits)
if err != nil {
return ""
}
return p.String()
}
// The helpers below run with m.mu held.
func (m *mailSpray) touch(ip string, now time.Time) *sprayIP {
e := m.ips[ip]
if e == nil {
e = &sprayIP{}
m.ips[ip] = e
}
e.lastSeen = now
m.markIP(ip)
return e
}
// recordFailure keeps untrusted names ahead of stale-password noise. When
// no name fits, retain its time so losing the name cannot earn new trust.
func (m *mailSpray) recordFailure(e *sprayIP, key, name string, exist mailboxExistence, now time.Time) bool {
f, ok := e.failed[name]
if !ok && !m.roomForName(e, key, name, now, now) {
return false
}
if !ok || now.Sub(f.last) > m.cfg.Window {
f = sprayFailure{first: now, exist: exist}
} else {
f.exist = foldObservedExistence(f.exist, exist)
}
if f.count < math.MaxInt {
f.count++
}
f.last = now
if e.failed == nil {
e.failed = make(map[string]sprayFailure)
}
e.failed[name] = f
return true
}
// roomForName makes room in e.failed for a new name that failed at "at",
// dropping aged-trust names first. With no room it reports false, and an
// untrusted name leaves its time in untracked.
func (m *mailSpray) roomForName(e *sprayIP, key, name string, at, now time.Time) bool {
if len(e.failed) < maxSprayNamesPerIP {
return true
}
e.pruneFailed(now.Add(-m.cfg.Window))
if len(e.failed) >= maxSprayNamesPerIP {
oldest := ""
for mb, failure := range e.failed {
if m.excused(key, mb, now) && (oldest == "" || failure.last.Before(e.failed[oldest].last)) {
oldest = mb
}
}
if oldest != "" {
delete(e.failed, oldest)
}
}
if len(e.failed) < maxSprayNamesPerIP {
return true
}
if !m.excused(key, name, now) && at.After(e.untracked) {
e.untracked = at
}
return false
}
// foldObservedExistence merges what separate failures say about one name
// over time. Unlike combineMailboxExistence, the fold for one request across
// a passdb chain, unknown here means no information (an IMAP failure carries
// none), so a missing answer outweighs it. One exists answer outweighs all.
func foldObservedExistence(a, b mailboxExistence) mailboxExistence {
switch {
case a == mailboxExists || b == mailboxExists:
return mailboxExists
case a == mailboxMissing || b == mailboxMissing:
return mailboxMissing
}
return mailboxExistenceUnknown
}
func (e *sprayIP) pruneFailed(cutoff time.Time) {
maps.DeleteFunc(e.failed, func(_ string, f sprayFailure) bool { return f.last.Before(cutoff) })
}
// forgetFailures drops e's failure evidence and the record of its verdicts.
func (e *sprayIP) forgetFailures() {
e.failed = nil
e.untracked = time.Time{}
e.verdictAt = time.Time{}
e.observedAt, e.observedNames = time.Time{}, 0
}
// noteWeakAlert reports whether a compromise advisory for mailbox may fire, at
// most once per window, and records it. The record is capped like failed,
// oldest first, and grants no trust.
func (e *sprayIP) noteWeakAlert(mailbox string, now time.Time, window time.Duration) bool {
at, ok := e.weakAlerted[mailbox]
if ok && now.Sub(at) <= window {
return false
}
if e.weakAlerted == nil {
e.weakAlerted = make(map[string]time.Time)
}
if !ok && len(e.weakAlerted) >= maxSprayNamesPerIP {
oldest := ""
for mb, t := range e.weakAlerted {
if oldest == "" || t.Before(e.weakAlerted[oldest]) {
oldest = mb
}
}
delete(e.weakAlerted, oldest)
}
e.weakAlerted[mailbox] = now
return true
}
// noteObserved reports whether an observe mode block verdict naming n
// untrusted mailboxes may fire, and records it: once per window, or sooner
// when n grows past what the last one listed.
func (e *sprayIP) noteObserved(n int, now time.Time, window time.Duration) bool {
if !e.observedAt.IsZero() && now.Sub(e.observedAt) <= window && n <= e.observedNames {
return false
}
e.observedAt, e.observedNames = now, n
return true
}
// noteRevoked remembers a crack on mailbox at "at" for the window. Losing a
// cracked mailbox at the name cap must not turn its next login into clean
// evidence. Crack overflow stays separate from failed-login overflow so
// an operator release can clear failures without restoring cracked trust.
func (e *sprayIP) noteRevoked(mailbox string, at time.Time, window time.Duration) {
maps.DeleteFunc(e.revoked, func(_ string, t time.Time) bool { return at.Sub(t) > window })
if _, ok := e.revoked[mailbox]; ok || len(e.revoked) < maxSprayNamesPerIP {
if e.revoked == nil {
e.revoked = make(map[string]time.Time)
}
e.revoked[mailbox] = at
return
}
if at.After(e.revokedOverflow) {
e.revokedOverflow = at
}
}
// setSource records ip's verdict cadence. At the cap the record that lapses
// first goes; losing a live one only lets an address already reported draw
// another verdict.
func (e *sprayIP) setSource(ip string, src spraySource) {
if _, ok := e.sources[ip]; !ok && len(e.sources) >= maxSprayNamesPerIP {
first := ""
for a, s := range e.sources {
if first == "" || s.lapse().Before(e.sources[first].lapse()) {
first = a
}
}
delete(e.sources, first)
}
if e.sources == nil {
e.sources = make(map[string]spraySource)
}
e.sources[ip] = src
}
func (e *sprayIP) hasUntracked(now time.Time, window time.Duration) bool {
return (!e.untracked.IsZero() && now.Sub(e.untracked) <= window) ||
(!e.revokedOverflow.IsZero() && now.Sub(e.revokedOverflow) <= window)
}
// staleFailures reports whether name failed at least twice in the window,
// the first time at least sprayStaleSpread ago: a saved password gone
// stale, not a typo (4d).
func (e *sprayIP) staleFailures(name string, now time.Time, window time.Duration) bool {
f, ok := e.failed[name]
return ok && now.Sub(f.last) <= window && f.count >= 2 && now.Sub(f.first) >= sprayStaleSpread
}
// recordTrust records a clean login and reports whether a live trust record
// already existed. A record idle for a whole window starts over.
func (m *mailSpray) recordTrust(k sprayTrustKey, now time.Time) bool {
t, ok := m.trust[k]
live := ok && now.Sub(t.last) <= m.cfg.Window
if !live {
t.first = now
}
t.last = now
if ok {
m.trust[k] = t
m.markTrust(k.ip)
} else {
m.addTrust(k, t)
}
m.evictTrust(k)
return live
}
// addTrust stores a new record. It makes room in its IP's and its mailbox's
// share first, so one owned mailbox cannot fill the pool from fresh IPs.
func (m *mailSpray) addTrust(k sprayTrustKey, t sprayTrust) {
if mbs := m.trustByIP[k.ip]; len(mbs) >= maxSprayTrustPerIP {
m.dropTrust(m.oldestTrust(mbs, func(mb string) sprayTrustKey { return sprayTrustKey{k.ip, mb} }))
}
if ips := m.trustByMailbox[k.mailbox]; len(ips) >= maxSprayTrustIPsPerMailbox {
m.dropTrust(m.oldestTrust(ips, func(ip string) sprayTrustKey { return sprayTrustKey{ip, k.mailbox} }))
}
setAdd(m.trustByIP, k.ip, k.mailbox)
setAdd(m.trustByMailbox, k.mailbox, k.ip)
setAdd(m.trustByKey, sprayEvidenceKey(k.ip), k.ip)
m.trust[k] = t
m.markTrust(k.ip)
}
func (m *mailSpray) dropTrust(k sprayTrustKey) {
_, ok := m.trust[k]
delete(m.trust, k)
setDel(m.trustByIP, k.ip, k.mailbox)
setDel(m.trustByMailbox, k.mailbox, k.ip)
if ok && len(m.trustByIP[k.ip]) == 0 {
setDel(m.trustByKey, sprayEvidenceKey(k.ip), k.ip)
}
if ok {
m.markTrust(k.ip)
}
}
// oldestTrust returns the record with the oldest last success among set.
func (m *mailSpray) oldestTrust(set map[string]struct{}, key func(string) sprayTrustKey) sprayTrustKey {
var oldest sprayTrustKey
found := false
for s := range set {
k := key(s)
if !found || m.trust[k].last.Before(m.trust[oldest].last) {
oldest, found = k, true
}
}
return oldest
}
// noteBlock records that ip is blocked for spraying until end. One evidence
// key holds at most maxSprayNamesPerIP markers, so one IPv6 host cycling
// its /64 cannot evict other networks' markers: past that its
// soonest-ending marker goes, and a marker ending before all of them is not
// kept. The pool holds at most maxSprayMarkers, soonest-ending out first.
// Losing a marker only costs the re-block shortcut.
func (m *mailSpray) noteBlock(ip string, end time.Time) {
key := sprayEvidenceKey(ip)
if _, ok := m.blocks[ip]; !ok && len(m.blocksByKey[key]) >= maxSprayNamesPerIP {
first := ""
for a := range m.blocksByKey[key] {
if first == "" || m.blocks[a].Before(m.blocks[first]) {
first = a
}
}
if end.Before(m.blocks[first]) {
return
}
m.dropBlock(first)
}
m.blocks[ip] = end
setAdd(m.blocksByKey, key, ip)
m.markBlock(ip)
sprayEvict(m.blocks, maxSprayMarkers, ip, new(int),
func(time.Time) int { return 0 },
func(end time.Time) time.Time { return end },
m.dropBlock)
}
func (m *mailSpray) dropBlock(ip string) {
delete(m.blocks, ip)
setDel(m.blocksByKey, sprayEvidenceKey(ip), ip)
m.markBlock(ip)
}
func setAdd(idx map[string]map[string]struct{}, k, v string) {
if idx[k] == nil {
idx[k] = make(map[string]struct{})
}
idx[k][v] = struct{}{}
}
func setDel(idx map[string]map[string]struct{}, k, v string) {
delete(idx[k], v)
if len(idx[k]) == 0 {
delete(idx, k)
}
}
// agedTrust is trust that is still live and older than TrustAge (4b).
func (m *mailSpray) agedTrust(ip, mailbox string, now time.Time) bool {
t, ok := m.trust[sprayTrustKey{ip, mailbox}]
return ok && now.Sub(t.last) <= m.cfg.Window && now.Sub(t.first) >= m.cfg.TrustAge
}
// excused reports whether an address of evidence key holds aged trust on
// mailbox. Trust is earned per address, but an IPv6 /64 is one network's
// evidence, so trust any of its addresses earned excuses failures on that
// mailbox for all of them. An IPv4 key holds only its own address.
func (m *mailSpray) excused(key, mailbox string, now time.Time) bool {
a, b := m.trustByKey[key], m.trustByMailbox[mailbox]
if len(a) > len(b) {
a, b = b, a
}
for ip := range a {
if _, ok := b[ip]; ok && m.agedTrust(ip, mailbox, now) {
return true
}
}
return false
}
func (m *mailSpray) firstUntrustedFailure(key string, now time.Time) time.Time {
var first time.Time
if e := m.ips[key]; e != nil {
for name, f := range e.failed {
if now.Sub(f.last) <= m.cfg.Window && !m.excused(key, name, now) && (first.IsZero() || f.first.Before(first)) {
first = f.first
}
}
}
return first
}
// keyTrustCount counts, up to limit, the mailboxes other than except that
// addresses of evidence key hold aged trust on from before its active
// untrusted failures. The scan is bounded by the trust pool and stops at limit.
func (m *mailSpray) keyTrustCount(key, except string, limit int, now time.Time) int {
first := m.firstUntrustedFailure(key, now)
seen := make(map[string]struct{}, limit)
for ip := range m.trustByKey[key] {
for mb := range m.trustByIP[ip] {
if mb == except || !m.agedTrust(ip, mb, now) {
continue
}
if !first.IsZero() && !m.trust[sprayTrustKey{ip, mb}].first.Before(first) {
continue
}
seen[mb] = struct{}{}
if len(seen) >= limit {
return len(seen)
}
}
}
return len(seen)
}
// untrusted returns the key's failed names in the window that no address of
// the key holds aged trust on, sorted.
func (m *mailSpray) untrusted(e *sprayIP, key string, now time.Time) []string {
var out []string
for name, f := range e.failed {
if now.Sub(f.last) <= m.cfg.Window && !m.excused(key, name, now) {
out = append(out, name)
}
}
slices.Sort(out)
return out
}
// verdict builds ip's verdict. An enforce mode block with a length leaves
// a marker for ip until the block ends.
func (m *mailSpray) verdict(e *sprayIP, ip string, rule sprayRule, block bool, mailbox string, untrusted []string, now time.Time) *sprayVerdict {
v := &sprayVerdict{Rule: rule, Block: block, Mailbox: mailbox, Targets: untrusted}
if block && m.cfg.Mode == sprayEnforce {
v.BlockTTL = m.cfg.BlockTTL
if v.BlockTTL > 0 {
m.noteBlock(ip, now.Add(v.BlockTTL))
}
}
if len(untrusted) > maxMailTargetsListed {
v.Targets = untrusted[:maxMailTargetsListed:maxMailTargetsListed]
v.Extra = len(untrusted) - maxMailTargetsListed
}
e.verdictAt = now
return v
}
// corroborate notes that ip's network tried the missing name and returns how
// many networks tried it within the window.
func (m *mailSpray) corroborate(name, ip string, now time.Time) int {
key := sprayNetworkKey(ip)
n := m.names[name]
if key == "" {
return m.networkCount(n, now)
}
if n == nil {
n = &sprayName{networks: make(map[string]time.Time)}
m.names[name] = n
}
n.note(key, now)
m.markName(name)
m.evictNames(name, now)
return m.networkCount(n, now)
}
// note records that network key tried the name at "at", dropping the oldest
// network at the cap.
func (n *sprayName) note(key string, at time.Time) {
if _, ok := n.networks[key]; !ok && len(n.networks) >= maxSprayNetworksPerName {
oldest := ""
for k, ts := range n.networks {
if oldest == "" || ts.Before(n.networks[oldest]) {
oldest = k
}
}
delete(n.networks, oldest)
}
n.networks[key] = at
if at.After(n.last) {
n.last = at
}
}
func (m *mailSpray) networkCount(n *sprayName, now time.Time) int {
if n == nil {
return 0
}
c := 0
for _, ts := range n.networks {
if now.Sub(ts) <= m.cfg.Window {
c++
}
}
return c
}
// corroboratedMissing reports whether name is missing for this IP and tried
// from enough networks for rule 2.
func (m *mailSpray) corroboratedMissing(e *sprayIP, name string, now time.Time) bool {
return m.cfg.MissingNetworks > 0 && e.failed[name].exist == mailboxMissing &&
m.networkCount(m.names[name], now) >= m.cfg.MissingNetworks
}
// Evidence eviction ranks, lowest dropped first. Attacker-chosen IPs and
// names come cheap, so one-shot entries go before multi-failure evidence and
// IPs that drew a verdict.
const (
sprayRankOneShot = iota
sprayRankRepeat
sprayRankVerdict
)
func (m *mailSpray) ipRank(e *sprayIP, now time.Time) int {
if !e.verdictAt.IsZero() && now.Sub(e.verdictAt) <= m.cfg.Window {
return sprayRankVerdict
}
for _, at := range e.revoked {
if now.Sub(at) <= m.cfg.Window {
return sprayRankVerdict
}
}
if e.hasUntracked(now, m.cfg.Window) {
return sprayRankRepeat
}
n := 0
for _, f := range e.failed {
if now.Sub(f.last) <= m.cfg.Window {
if f.count >= 2-n {
return sprayRankRepeat
}
n += f.count
}
}
return sprayRankOneShot
}
func (m *mailSpray) evictIPs(keep string, now time.Time) {
sprayEvict(m.ips, m.cfg.MaxIPs, keep, &m.ipFloor,
func(e *sprayIP) int { return m.ipRank(e, now) },
func(e *sprayIP) time.Time { return e.lastSeen },
func(ip string) {
delete(m.ips, ip)
m.markIP(ip)
})
}
// evictTrust drops the oldest last success first: trust is earned with a real
// password, so recency is all that tells records apart.
func (m *mailSpray) evictTrust(keep sprayTrustKey) {
sprayEvict(m.trust, m.cfg.MaxIPs, keep, new(int),
func(sprayTrust) int { return 0 },
func(t sprayTrust) time.Time { return t.last },
m.dropTrust)
}
// evictNames keeps names already over the rule 2 bar ahead of a flood of
// invented ones. Such a name ranks with verdict evidence.
func (m *mailSpray) evictNames(keep string, now time.Time) {
sprayEvict(m.names, maxSprayCorroborationNames, keep, &m.nameFloor,
func(n *sprayName) int {
if m.cfg.MissingNetworks > 0 && m.networkCount(n, now) >= m.cfg.MissingNetworks {
return sprayRankVerdict
}
return sprayRankOneShot
},
func(n *sprayName) time.Time { return n.last },
func(name string) {
delete(m.names, name)
m.markName(name)
})
}
type sprayVictim[K comparable] struct {
key K
rank int
seen time.Time
}
// sprayEvict drops the lowest rank first, then the least recently seen.
// keep wins ties within its rank, but cannot displace a higher rank. Once
// within limit, batch removal goes on to 95% of it, through one-shot and
// multi-failure entries alike, but stops before entries over a bar
// (sprayRankVerdict): spare capacity must not cost them. It then records
// their rank in *floor, as every entry left ranks at least that high.
// While that holds, a newcomer ranked below the floor is the one entry a
// sort would drop, so it is dropped without sorting the pool. One admitted
// into spare room instead ends the promise. drop also updates indexes.
func sprayEvict[K comparable, V any](m map[K]V, limit int, keep K, floor *int, rank func(V) int, seen func(V) time.Time, drop func(K)) {
if v, ok := m[keep]; ok && *floor > 0 && rank(v) < *floor {
if len(m) <= limit {
*floor = 0
return
}
drop(keep)
}
if len(m) <= limit {
return
}
victims := make([]sprayVictim[K], 0, len(m))
for k, v := range m {
victims = append(victims, sprayVictim[K]{k, rank(v), seen(v)})
}
slices.SortFunc(victims, func(a, b sprayVictim[K]) int {
if c := cmp.Compare(a.rank, b.rank); c != 0 {
return c
}
if c := a.seen.Compare(b.seen); c != 0 {
return c
}
if a.key == keep && b.key != keep {
return 1
}
if b.key == keep && a.key != keep {
return -1
}
return 0
})
target := limit * 95 / 100
if limit > 0 {
target = max(1, target)
}
lastRank := victims[0].rank
for _, v := range victims {
if len(m) <= target {
break
}
if len(m) <= limit && v.rank > lastRank && v.rank >= sprayRankVerdict {
*floor = v.rank
break
}
drop(v.key)
lastRank = v.rank
}
}
package daemon
import (
"fmt"
"strings"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
)
// mailProto is the login protocol a spray verdict came from.
type mailProto uint8
const (
protoSMTP mailProto = iota
protoMail
)
func (p mailProto) String() string {
if p == protoSMTP {
return "SMTP"
}
return "IMAP/POP"
}
// sprayFinding turns a mailSpray verdict into a finding under the existing
// check names. The missing-mailbox, breadth and repeat rules block only in
// enforce mode; observe mode reports them as mail_bruteforce_suspected. A
// compromise is Critical only when the verdict blocks in enforce mode. Every
// other verdict is a High advisory. Only the enforce mode blocks carry the
// verdict's block length.
func sprayFinding(proto mailProto, ip string, v *sprayVerdict, mode sprayMode, now time.Time) alert.Finding {
enforce := mode == sprayEnforce
observe := ""
if !enforce {
observe = " Observe mode: reported only, not blocked."
}
targets := strings.TrimSpace(formatSprayTargets(v.Targets, v.Extra) + " " + sprayScope(ip))
if v.Block && v.Rule != sprayRuleCompromise {
reason := fmt.Sprintf("failed logins on %d mailboxes it has no trusted login for", len(v.Targets)+v.Extra)
details := "Long-memory spray detection: failed logins on several mailboxes this address has no trusted login for."
switch v.Rule {
case sprayRuleMissingMailbox:
reason = fmt.Sprintf("failed login to missing mailbox %s, tried by %d networks", sprayMailboxText(v.Mailbox), v.Networks)
details = "Long-memory spray detection: a failed login to a mailbox that does not exist, which several networks also tried."
case sprayRuleRepeat:
reason = "failed login by an address blocked earlier for login spraying"
details = "Long-memory spray detection: a failed login by an address blocked earlier for login spraying, so it is blocked again."
}
check, severity := "mail_bruteforce_suspected", alert.High
message := fmt.Sprintf("Suspected %s password spray from %s: %s (observe mode, not blocked)", proto, ip, reason)
var ttl time.Duration
switch {
case !enforce:
case proto == protoSMTP:
check, severity, ttl = "smtp_bruteforce", alert.Critical, v.BlockTTL
message = fmt.Sprintf("SMTP brute force from %s: %s", ip, reason)
default:
check, severity, ttl = "mail_bruteforce", alert.Critical, v.BlockTTL
message = fmt.Sprintf("Mail auth brute force from %s: %s", ip, reason)
}
return alert.Finding{
Severity: severity,
Check: check,
Message: message,
Details: strings.TrimSpace(details + observe + " " + targets),
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionPeer,
BlockTTL: ttl,
}
}
severity := alert.High
mailbox := sprayMailboxText(v.Mailbox)
var message, details string
var ttl time.Duration
switch v.Rule {
case sprayRuleCompromise:
message = fmt.Sprintf("Mail account compromise: successful %s login for %s from %s after failed logins on other mailboxes", proto, mailbox, ip)
details = "The address logged in after failing on other mailboxes, one of them a missing mailbox that several networks tried. Rotate the password and revoke sessions."
switch {
case !v.Block:
message += " (established multi-mailbox source)"
details = "The address also holds trusted logins to several other mailboxes, so this may be a shared office device. Verify with the mailbox owner; not auto-blocked."
case enforce:
severity, ttl = alert.Critical, v.BlockTTL
default:
message += " (observe mode, not blocked)"
details += observe
}
case sprayRuleWeakCompromise:
message = fmt.Sprintf("Possible mail account compromise: successful %s login for %s from %s after failed logins on other mailboxes", proto, mailbox, ip)
details = "The address failed on other mailboxes before this login. Verify with the mailbox owner; not auto-blocked."
default:
message = fmt.Sprintf("Mail login after repeated failures: successful %s login for %s from %s after failures on the same mailbox", proto, mailbox, ip)
details = "Repeated failures on this mailbox, the first at least an hour before this successful login, usually mean a saved password was fixed. Verify with the mailbox owner; not auto-blocked."
}
_, domain := alert.SplitEmail(v.Mailbox)
return alert.Finding{
Severity: severity,
Check: "mail_account_compromised",
Message: message,
Details: strings.TrimSpace(details + " " + targets),
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionPeer,
Domain: domain,
Mailbox: v.Mailbox,
BlockTTL: ttl,
}
}
// sprayScope says that an IPv6 source's targets and failure history are
// those of its /64, while a block names only the source address.
func sprayScope(ip string) string {
key := sprayEvidenceKey(ip)
if !strings.Contains(key, "/") {
return ""
}
return fmt.Sprintf("Targets and failure history cover %s; any block applies to %s only.", key, ip)
}
// sprayMailboxText renders a client-chosen mailbox name for a finding.
func sprayMailboxText(name string) string {
if name == "" {
return "(no mailbox)"
}
return formatMailFailTargetAccount(name)
}
// formatSprayTargets lists a verdict's untrusted names, the rest collapsed
// into "(+N more)".
func formatSprayTargets(targets []string, extra int) string {
if len(targets) == 0 {
return ""
}
names := make([]string, len(targets))
for i, name := range targets {
names[i] = sprayMailboxText(name)
}
s := "Targets: " + strings.Join(names, ", ")
if extra > 0 {
s += fmt.Sprintf(" (+%d more)", extra)
}
return s + "."
}
package daemon
import (
"strings"
"unicode"
"unicode/utf8"
"github.com/pidginhost/csm/internal/antispam/dovecot"
)
// maxSprayMailboxBytes is the longest mailbox name kept as a spray key. It
// matches the longest address SMTP allows.
const maxSprayMailboxBytes = 254
// normalizeSprayMailbox is the one spelling of a mailbox name for both failed
// and successful logins, so trust earned by a login matches the failures
// logged for that mailbox. Names are client-chosen, so the key is bounded.
// It is idempotent: a cut that lands after a space drops that space.
func normalizeSprayMailbox(s string) string {
s = normalizeMailAuthAccount(s)
if len(s) > maxSprayMailboxBytes {
n := maxSprayMailboxBytes
for n > 0 && !utf8.RuneStart(s[n]) {
n--
}
s = strings.TrimRightFunc(s[:n], unicode.IsSpace)
}
// The resolver drops a domain's final dot. Login trust and restored
// evidence must use that same key, even when no lookup is available.
if local, domain, ok := strings.Cut(s, "@"); ok && strings.HasSuffix(domain, ".") &&
dovecot.ValidMailbox(local) && dovecot.ValidDomain(strings.TrimSuffix(domain, ".")) {
s = strings.TrimSuffix(s, ".")
}
return s
}
package daemon
import (
"strings"
"time"
"github.com/pidginhost/csm/internal/eximlog"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/mailstats"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/store"
)
// mailStatsPersistInterval is how often the mail decision counter is saved.
// A clean shutdown saves once more after the log watchers stop. A var so
// tests can drive the periodic writer without waiting a minute.
var mailStatsPersistInterval = time.Minute
// recordMailStatsExim counts an Exim main log line for the Email Security
// page; recordScan counts scan markers. Every Exim line comes through here,
// so a parser runs only for a line that holds its marker. The login count
// does not depend on the IP or infra
// filters, so it matches a grep of the log. An arrival counts as a send of
// its submitter's hosting account; inbound mail has no submitter. A
// null-sender arrival counts only when its submitter resolves to an
// account: Exim's own bounces come from mailnull, which resolves to none,
// and an account must not hide its mail behind <>.
func (d *Daemon) recordMailStatsExim(line string, at time.Time) {
if _, sender, ok := eximlog.Arrival(line); ok {
if id := eximlog.Submitter(line); id != "" {
owner := mailAccountOwner(id)
if owner != "" || sender != "<>" {
d.mailStats.RecordSend(mailstats.LineTime(line, time.Local, at), owner)
}
}
}
if strings.Contains(line, " rejected ") {
if r, ok := mailstats.ParseReject(line, time.Local, at); ok {
d.mailStats.RecordReject(r)
}
}
if strings.Contains(line, " authenticator failed for ") && strings.HasPrefix(eximlog.AuthFailure(line).Authenticator, "dovecot_") {
d.mailStats.RecordLoginFailure(mailstats.LineTime(line, time.Local, at), false, 1)
}
}
// recordMailStatsMail counts the password attempts of a failed Dovecot
// login, whatever the client address, and on a Postfix host the rejects and
// sends of the mail log. A zero at means the source gave no observation
// time.
func (d *Daemon) recordMailStatsMail(line string, at time.Time) {
if at.IsZero() {
at = d.mailStatsNow()
}
// Where the Exim main log is watched it already counts this mail.
if !d.smtpAuthFromExim && strings.Contains(line, "postfix") {
d.recordMailStatsPostfix(line, at)
}
if !strings.Contains(line, "(auth failed") || !isMailAuthLine(line) || dovecotLoginSucceeded(line) {
return
}
d.mailStats.RecordLoginFailure(mailstats.LineTime(line, time.Local, at), true, dovecotFailedAttempts(line))
}
// recordMailStatsPostfix counts a Postfix reject, or a send: the line smtpd
// logs when it accepts mail from a client that logged in. The login is the
// sender; mail accepted without one is inbound or relayed for a trusted
// host, and the line cannot tell which.
func (d *Daemon) recordMailStatsPostfix(line string, at time.Time) {
if strings.Contains(line, "reject: ") {
if r, ok := mailstats.ParsePostfixReject(line, time.Local, at); ok {
d.mailStats.RecordReject(r)
return
}
}
_, msg := postfixSMTPDMessage(line)
if _, login := postfixSASLSuccess(msg); login != "" {
d.mailStats.RecordSend(mailstats.LineTime(line, time.Local, at), strings.ToLower(login))
}
}
// restoreMailStats merges the stored counts into the counter before the log
// watchers start. Counting starts now, not when the counter was built, so
// the stop the load records ends here; a retried load keeps that end.
func (d *Daemon) restoreMailStats() {
d.mailStats.SetStart(d.mailStatsNow())
if sdb := store.Global(); sdb != nil {
d.loadMailStats(sdb)
}
}
// loadMailStats merges the stored counts once. Until a load succeeds the
// counter runs in memory only, since Flush writes nothing to the store
// before a load has merged; every save retries, and a load that already
// succeeded does nothing. Only the first failure and the recovery are
// logged.
func (d *Daemon) loadMailStats(sdb *store.DB) {
err := d.mailStats.Load(sdb, d.mailStatsNow())
switch {
case err != nil && !d.mailStatsLoadFailing.Load():
d.mailStatsLoadFailing.Store(true)
csmlog.Warn("mail stats load failed, counting in memory until a retry succeeds", "err", err)
case err == nil && d.mailStatsLoadFailing.Load():
d.mailStatsLoadFailing.Store(false)
csmlog.Info("mail stats load succeeded on retry")
}
}
// persistMailStats saves the counter, loading the stored counts first if
// that has not succeeded yet. Without a state store it only prunes, so
// memory stays bounded.
func (d *Daemon) persistMailStats() {
sdb := store.Global()
if sdb == nil {
d.mailStats.Prune(d.mailStatsNow())
return
}
d.loadMailStats(sdb)
if err := d.mailStats.Flush(sdb, d.mailStatsNow()); err != nil {
csmlog.Warn("mail stats persistence failed", "err", err)
}
}
// startMailStatsPersistence saves the counter periodically. It stops on
// d.stopCh without a final save; the shutdown path saves once more after
// every log watcher has exited.
func (d *Daemon) startMailStatsPersistence() {
d.wg.Add(1)
obs.Go("mail-stats-persist", func() {
defer d.wg.Done()
ticker := time.NewTicker(mailStatsPersistInterval)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
d.persistMailStats()
}
}
})
}
package daemon
import (
"strings"
"github.com/pidginhost/csm/internal/modsec"
)
// modsecConfidence classifies how strongly a single ModSecurity deny indicates
// a real attack, independent of the rule's blocking action. It drives whether a
// deny may auto-escalate to a 24h firewall ban (high/unknown) or only counts
// toward the low-confidence visibility/backstop path (low).
type modsecConfidence int
const (
// modsecConfUnknown is a blocking deny CSM cannot classify from the parsed
// rule ID, message, or tags. Treated as escalation-eligible at the normal
// bar (fail-secure) so a new vendor rule never gets a silent no-ban path.
modsecConfUnknown modsecConfidence = iota
// modsecConfLow is a policy/anomaly/scoring deny whose own rule is not proof
// of hostile intent (content-type policy, anomaly score threshold). Never
// auto-bans on its own at the normal bar; only via the low-confidence
// backstop.
modsecConfLow
// modsecConfHigh is a specific attack/probe signal (SQLi, RCE, traversal,
// URL-encoding abuse, CSM custom deny, scanner). Escalation-eligible at the
// normal bar even when it is the only distinct rule.
modsecConfHigh
)
func (c modsecConfidence) String() string {
switch c {
case modsecConfLow:
return "low"
case modsecConfHigh:
return "high"
default:
return "unknown"
}
}
// modsecSingleHitRules are CSM rules that match one exploit's request shape,
// which legitimate clients do not send. One deny is enough evidence, so they
// escalate on the first hit whatever the operator's hit threshold. The per-rule
// escalation exclusion still applies. Only exact CSM rule IDs.
// Endpoint-only rules such as 900107 can also match legitimate requests and
// must stay on the normal hit-count path.
var modsecSingleHitRules = map[int]bool{
900122: true, // wp2shell exploit tool User-Agent
900133: true, // PHP2Shell upload progress field, non-browser requests
}
// modsecKnownLowConfRules are vendor policy/anomaly/scoring rule IDs verified to
// fire on legitimate-but-unusual traffic as often as on attacks. They are
// low-confidence by exact ID, but a high-confidence attack signal in the same
// message/tags still overrides (see classifyModSecConfidence ordering). Do NOT
// add broad rule-ID ranges here; only exact, fixture-verified IDs.
var modsecKnownLowConfRules = map[int]bool{
// COMODO CWAF policy/anomaly.
210710: true, // request content-type not allowed by policy
214930: true, // inbound points exceeded (anomaly threshold)
211170: true, // outbound points / scoring
211220: true, // outbound points / scoring
// OWASP CRS policy/anomaly.
920100: true, // invalid HTTP request line (protocol enforcement)
920420: true, // request content-type not allowed
920430: true, // HTTP protocol version not allowed (policy)
920440: true, // URL file extension restricted by policy
949110: true, // inbound anomaly score exceeded
959100: true, // outbound anomaly score exceeded
980130: true, // anomaly score reporting
}
// modsecAttackMsgKeywords are lowercase substrings of vendor messages that name
// a specific attack/probe class. Presence means high-confidence.
var modsecAttackMsgKeywords = []string{
"sql injection", "sqli", "remote command", "command injection",
"os command", "remote code execution", "code execution", "code injection",
"file inclusion", "lfi", "rfi", "cross-site scripting",
"xss", "path traversal", "directory traversal", "url encoding abuse",
"web shell", "webshell", "backdoor", "shellshock", "ssrf",
"server-side request forgery", "session fixation", "remote file",
"request smuggling", "response splitting", "crlf injection",
"object injection", "template injection", "xxe", "xml external entity",
"ldap injection", "nosql injection", "scanner", "exploit",
"injection attack", "deserializ",
}
// modsecAttackTagKeywords are lowercase substrings of the ModSecurity rule tag
// taxonomy that name a specific attack class. "attack-protocol" and
// "attack-generic" are deliberately excluded: protocol/anomaly policy rules
// carry them but are low-confidence.
var modsecAttackTagKeywords = []string{
"attack-sqli", "attack-rce", "attack-xss", "attack-lfi", "attack-rfi",
"attack-injection", "attack-disclosure", "attack-fixation", "attack-ssrf",
"attack-shell", "application-attack",
}
// modsecPolicyAnomalyKeywords are lowercase substrings that name a
// policy/anomaly/scoring decision (low-confidence) when no attack signal is
// present.
var modsecPolicyAnomalyKeywords = []string{
"anomaly", "inbound points", "outbound points", "points exceeded",
"total incoming points", "total inbound points", "content-type",
"content type", "not allowed by policy", "not allowed by the policy",
"protocol version", "score exceeded",
}
// classifyModSecRuleConfidence uses the existing rule registry for message
// and tag evidence. Logged values may contain client-controlled macros even
// when field parsing succeeds. Missing metadata falls back to the known IDs
// and the validated rule file; everything else stays unknown and eligible.
func classifyModSecRuleConfidence(ruleNum int, ruleFile string) modsecConfidence {
m, _ := modsec.Global().Metadata(ruleNum)
return classifyModSecConfidence(ruleNum, m.Message, m.Tags, modsecRuleFileBase(ruleFile))
}
// classifyModSecConfidence classifies trusted rule metadata. Ordering is deliberate:
// a high-confidence attack signal always wins; CSM custom rules are high;
// otherwise an exact known-low ID or policy/anomaly wording (with no attack
// signal) is low; anything else is unknown (fail-secure, escalation-eligible).
//
// The modsec [severity] field is intentionally not an input: anomaly-scoring
// WAFs (COMODO CWAF) emit CRITICAL severity on benign policy/anomaly rules, so
// it is too noisy to separate attacks from policy hits. Rule ID, message, and
// tags from configuration carry the reliable signal.
func classifyModSecConfidence(ruleNum int, msg, tags, ruleFile string) modsecConfidence {
lc := strings.ToLower(msg + " " + tags)
// OWASP CRS names each rule file after the tag taxonomy of the rules in it
// (REQUEST-942-APPLICATION-ATTACK-SQLI.conf). LiteSpeed logs only the rule
// ID and file, so the file name is the only place that evidence survives.
attackTags := strings.ToLower(tags + " " + ruleFile)
// CSM custom rules are purpose-built attack/probe detections.
if ruleNum >= 900000 && ruleNum <= 900999 {
return modsecConfHigh
}
// Specific attack/probe evidence wins over any low signal.
if containsAny(lc, modsecAttackMsgKeywords) || containsAny(attackTags, modsecAttackTagKeywords) {
return modsecConfHigh
}
// Positive low evidence only: exact known-low ID or policy/anomaly wording.
if modsecKnownLowConfRules[ruleNum] {
return modsecConfLow
}
if containsAny(lc, modsecPolicyAnomalyKeywords) {
return modsecConfLow
}
return modsecConfUnknown
}
func containsAny(s string, subs []string) bool {
for _, sub := range subs {
if strings.Contains(s, sub) {
return true
}
}
return false
}
package daemon
import (
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/store"
)
const (
// modsecContextScanCap bounds the history walk per enriched IP. Denies that
// drive one escalation number in the dozens, so a few thousand is ample
// headroom while keeping the bbolt read cheap on a hammered host.
modsecContextScanCap = 2000
// modsecContextMaxItems caps the domains and URIs surfaced per block so the
// digest line stays readable.
modsecContextMaxItems = 5
)
// modsecEnricher returns the EnrichModSec lookup wired into the block digest.
// It reuses already-parsed modsec findings from the history store, so the
// digest can name targeted domains and request paths without re-reading the
// (multi-hundred-MB) audit log. The lookback matches the escalation window
// that produced the block. The lookup reads the live threshold so safe reloads
// of the ModSecurity escalation window also change the digest lookback.
func (d *Daemon) modsecEnricher(cfg *config.Config) func(ip string) (domains, uris []string) {
return func(ip string) ([]string, []string) {
live := d.currentCfg()
if live == nil {
live = cfg
}
_, win := modsecEscalationParams(live)
return aggregateModSecContext(ip, time.Now().Add(-win))
}
}
// aggregateModSecContext reads the per-deny modsec findings for one IP since the
// cutoff and returns its most-hit customer domains and request URIs. Escalation
// findings (no per-deny URI) and other IPs are skipped. Returns nil slices when
// no store is loaded or nothing matched.
func aggregateModSecContext(ip string, since time.Time) (domains, uris []string) {
db := store.Global()
if db == nil {
return nil, nil
}
findings := db.SearchHistorySince(since, modsecContextScanCap, func(f alert.Finding) bool {
return f.Check == "modsec_block_realtime" && f.SourceIP == ip
})
domainCounts := make(map[string]int)
uriCounts := make(map[string]int)
for _, f := range findings {
if f.Domain != "" {
domainCounts[f.Domain]++
}
if uri := modsecURIFromDetails(f.Details); uri != "" {
uriCounts[uri]++
}
}
return topByCount(domainCounts, modsecContextMaxItems), topByCount(uriCounts, modsecContextMaxItems)
}
// modsecURIFromDetails pulls the URI line out of a structured modsec finding
// Details blob ("Rule: ...\nURI: ...\n..."). Returns "" when absent.
func modsecURIFromDetails(details string) string {
for _, line := range strings.Split(details, "\n") {
if v, ok := strings.CutPrefix(line, "URI: "); ok {
return v
}
}
return ""
}
// topByCount returns up to n keys ordered by descending count, breaking ties on
// the key itself so output is deterministic. Returns nil when the map is empty.
func topByCount(counts map[string]int, n int) []string {
if len(counts) == 0 {
return nil
}
keys := make([]string, 0, len(counts))
for k := range counts {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
if counts[keys[i]] != counts[keys[j]] {
return counts[keys[i]] > counts[keys[j]]
}
return keys[i] < keys[j]
})
if len(keys) > n {
keys = keys[:n]
}
return keys
}
package daemon
import (
"time"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/modsec"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/platform"
)
// modsecRegistryRefresh controls how often the rule-action registry is
// checked against disk. ModSec rule files change rarely (vendor pack updates,
// cPanel modsec_assemble nightly run), so a coarse interval keeps the cost
// negligible while still picking up operator edits within minutes.
const modsecRegistryRefresh = 5 * time.Minute
// Seams: the refresh probes the platform and parses the rule tree, and tests
// need to count both without a web server on the host.
var (
// modsecProbeRuleDirs re-runs detection (not the cached Detect) so a
// web-server mis-detection at boot -- LiteSpeed probed before lsws
// finished starting, which points RuleDirs at non-existent directories --
// self-heals on a later refresh instead of staying wrong for the daemon's
// lifetime. Each call forks one process per candidate unit, which is why
// refreshModSecRegistry stops calling it once a rule set has loaded.
modsecProbeRuleDirs = func() []string { return modsec.RuleDirs(platform.DetectFreshWithOverrides()) }
modsecBuildRegistry = modsec.BuildRegistry
)
// modsecRegistryState carries what the last refresh learned. Only the refresh
// path touches it: once at startup, then from the refresh goroutine.
type modsecRegistryState struct {
dirs []string
fingerprint string
// loaded records whether the last build produced a non-empty rule set
// from these dirs. Otherwise the platform is probed on every refresh,
// because an empty registry is exactly the symptom of detection having
// resolved the wrong directories.
loaded bool
}
// initModSecRegistry builds the rule-action registry once at startup and
// installs it as the package-level singleton. The registry tells the
// LiteSpeed log-line classifier which "triggered!" matches actually denied
// the request and which were pass-action informational rules. Without this,
// pass-action vendor rules (Comodo CWAF id 210710, 214930, ...) would be
// counted as denies, falsely escalating to a 24-hour auto-block of any IP
// that hits them three times in ten minutes.
//
// The build is failure-soft: missing rule directories yield an empty
// registry. With no prior healthy registry, ambiguous LiteSpeed "triggered!"
// lines are warnings until a later refresh loads rule actions.
func (d *Daemon) initModSecRegistry() {
d.refreshModSecRegistry()
d.wg.Add(1)
obs.Go("modsec-registry-refresh", d.modsecRegistryRefreshLoop)
}
func (d *Daemon) refreshModSecRegistry() {
state := &d.modsecRegistry
if !state.loaded || len(state.dirs) == 0 {
state.dirs = modsecProbeRuleDirs()
}
fingerprint, present := modsec.RuleTreeFingerprint(state.dirs)
if state.loaded && !present {
// The directories the last detection resolved are gone: the web
// server was swapped out, or the vendor pack removed. Detection has
// to run again rather than keep reporting the old rule actions.
state.loaded = false
state.dirs = modsecProbeRuleDirs()
fingerprint, _ = modsec.RuleTreeFingerprint(state.dirs)
}
if state.loaded && fingerprint != "" && fingerprint == state.fingerprint {
// Only a complete fingerprint can establish that the rule contents
// match the last successful build.
return
}
reg, err := modsecBuildRegistry(state.dirs)
if err != nil {
csmlog.Warn("modsec rule-action registry build had errors", "err", err, "rules_loaded", reg.Len())
}
state.loaded = reg.Len() > 0
// An empty fingerprint means the build could not read every rule file, so
// the next refresh has to look again. A parse failure over bytes that were
// read in full keeps its fingerprint: reparsing them changes nothing.
state.fingerprint = ""
// ReplaceGlobal keeps a previously-healthy registry rather than blanking
// it to empty: the vendor rule tree is briefly empty during cPanel's
// modsec_assemble rewrite, and a blank registry loses known pass and deny
// actions.
if !modsec.ReplaceGlobal(reg) {
previousRules := 0
if prev := modsec.Global(); prev != nil {
previousRules = prev.Len()
}
csmlog.Warn("modsec rule-action registry refresh returned 0 rules; keeping previous rule actions",
"previous_rules", previousRules, "dirs", len(state.dirs))
return
}
state.fingerprint = reg.Fingerprint()
csmlog.Info("modsec rule-action registry loaded", "rules", reg.Len(), "dirs", len(state.dirs))
}
func (d *Daemon) modsecRegistryRefreshLoop() {
defer d.wg.Done()
ticker := time.NewTicker(modsecRegistryRefresh)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
d.refreshModSecRegistry()
}
}
}
//go:build linux
package daemon
import (
"fmt"
"sync"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/queuehealth"
)
type notificationSource interface {
Pending() (int, error)
Read([]byte) (int, error)
Write([]byte) (int, error)
Close() error
}
type fanotifyDescriptor int
func (fd fanotifyDescriptor) Pending() (int, error) {
bytes, err := unix.IoctlGetInt(int(fd), unix.TIOCINQ)
if err != nil {
return 0, err
}
// Fanotify's FIONREAD counts metadata headers, including overflow records.
if bytes < 0 || bytes%metadataSize != 0 {
return 0, fmt.Errorf("invalid fanotify pending byte count: %d", bytes)
}
return bytes / metadataSize, nil
}
func (fd fanotifyDescriptor) Read(buf []byte) (int, error) { return unix.Read(int(fd), buf) }
func (fd fanotifyDescriptor) Write(buf []byte) (int, error) { return unix.Write(int(fd), buf) }
func (fd fanotifyDescriptor) Close() error { return unix.Close(int(fd)) }
// Notification descriptors are nonblocking. Serialize their syscalls with
// close so health reads and permission responses cannot use a recycled fd.
// Parsing runs outside that lock and keeps its own tracked batch.
type notificationQueue struct {
mu sync.Mutex
source notificationSource
sampled *queuehealth.Sampled
unmeasured queuehealth.Dwell
losses *queuehealth.Tracker
batches *queuehealth.Tracker
consumed uint64
closed bool
closeErr error
unavailable bool
variableRecords bool
}
func newNotificationQueue(source notificationSource, losses, batches *queuehealth.Tracker) *notificationQueue {
return ¬ificationQueue{
source: source, losses: losses, batches: batches,
sampled: queuehealth.NewSampled(0, "records", time.Minute),
}
}
func (q *notificationQueue) snapshot(now func() time.Time) (kernel, reader queuehealth.Status) {
q.mu.Lock()
defer q.mu.Unlock()
if !q.closed {
pending, err := q.source.Pending()
q.unavailable = err != nil || pending < 0
if !q.unavailable {
q.sampled.Observe(now(), pending, q.consumed)
}
}
at := now()
kernel = q.sampled.Snapshot(at)
losses := q.losses.Snapshot(at)
kernel.DroppedTotal, kernel.RecentDrops = losses.DroppedTotal, losses.RecentDrops
// The group limit is not exposed. The current sysctl can differ from the
// value copied at creation. An overflow marker also omits its loss count.
kernel.CapacityUnavailable, kernel.DroppedLowerBound = true, true
// A live reading can be retried, so one failure is not yet a degradation.
// The reading taken at close is all there will ever be.
unmeasured := q.unmeasured.Held(at, q.unavailable, queuehealth.MeasurementWindow) || q.unavailable && q.closed
if q.unavailable && !q.closed {
// This reading invalidates the depth the sampled reason came from.
kernel.Depth, kernel.LagSeconds = 0, 0
kernel.DepthUnavailable, kernel.LagBasis = true, "unavailable"
kernel.Status, kernel.Reason = "ok", ""
}
switch {
case losses.Status == "degraded" && kernel.Reason == "":
// Records the kernel already dropped outrank an unreadable depth.
kernel.Status, kernel.Reason = losses.Status, losses.Reason
case unmeasured:
kernel.Status, kernel.Reason = "degraded", "measurement_unavailable"
}
reader = q.batches.Snapshot(at)
reader.DepthUnit = "batches"
return kernel, reader
}
func (q *notificationQueue) read(buf []byte, process func([]byte)) (int, error) {
q.mu.Lock()
if q.closed {
q.mu.Unlock()
return 0, unix.EBADF
}
n, err := q.source.Read(buf)
if err != nil || n <= 0 {
q.mu.Unlock()
return n, err
}
q.consumed++
work := queuehealth.Work[[]byte]{Value: buf[:n], Ticket: q.batches.Begin(time.Now())}
q.mu.Unlock()
work.Process(process)
return n, err
}
func (q *notificationQueue) write(buf []byte) error {
q.mu.Lock()
defer q.mu.Unlock()
if q.closed {
return unix.EBADF
}
_, err := q.source.Write(buf)
return err
}
// Watch changes and readiness polling share the same descriptor lifetime as reads.
func (q *notificationQueue) useDescriptor(fn func() (int, error)) (int, error) {
q.mu.Lock()
defer q.mu.Unlock()
if q.closed {
return 0, unix.EBADF
}
return fn()
}
func (q *notificationQueue) close() error {
q.mu.Lock()
defer q.mu.Unlock()
if q.closed {
return q.closeErr
}
pending, err := q.source.Pending()
if err != nil || pending < 0 {
q.unavailable = true
} else {
q.unavailable = false
// Events can still arrive before close. Retain only the known minimum.
if q.variableRecords {
// A byte count cannot identify the number of variable-length records.
if pending > 0 {
q.losses.Lose(time.Now(), 1)
}
} else {
q.losses.Lose(time.Now(), uint64(pending))
}
}
q.closeErr = q.source.Close()
q.closed = true
q.sampled.Observe(time.Now(), 0, q.consumed)
return q.closeErr
}
package daemon
import (
"fmt"
"os"
"github.com/pidginhost/csm/internal/store"
)
// A pending manual apply must be settled before switching posture. Recovering
// it could restore an enforcing config or ruleset; ignoring it would abandon
// the operator's rollback deadline.
func (d *Daemon) checkObserveStartupRecovery() error {
if !d.cfg.ObserveMode() {
return nil
}
if db := store.Global(); db != nil {
if _, pending := db.GetFirewallRollback(); pending {
return fmt.Errorf("mode: observe cannot start with pending firewall settings recovery; resolve the pending apply in enforce mode before switching modes")
}
}
marker, _, _ := firewallRollbackFiles(d.cfg.StatePath)
if _, err := os.Stat(marker); err == nil {
return fmt.Errorf("mode: observe cannot start with pending firewall rules recovery; resolve the pending apply in enforce mode before switching modes")
} else if !os.IsNotExist(err) {
return fmt.Errorf("mode: observe cannot check pending firewall recovery: %w", err)
}
return nil
}
package daemon
import (
"bufio"
"fmt"
"net"
"net/netip"
"os"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/health"
"github.com/pidginhost/csm/internal/obs"
)
const pamSocketPath = "/var/run/csm/pam.sock"
const (
defaultPAMFailureThreshold = 5
defaultPAMFailureWindowMin = 10
defaultCredStuffingDistinctAccounts = 5
)
// PAMListener listens on a Unix socket for authentication events from the
// pam_csm.so PAM module. Tracks failures per IP and triggers CSM auto-blocking.
type PAMListener struct {
cfg *config.Config
alertCh chan<- alert.Finding
listener net.Listener
mu sync.Mutex
failures map[string]*pamFailureTracker
// serviceFailures counts failures from PAM services other than sshd.
// They are visibility only: those services have their own log
// producers, so counting them as SSH evidence would see one event twice.
serviceFailures map[string]*pamServiceFailures
// stopCh is set by Run so emit can abort a send when the daemon is
// shutting down. Per-connection goroutines are not tracked by a
// WaitGroup, so without this escape a goroutine blocked on the
// undrained alert channel would leak past shutdown. Nil for
// hand-constructed listeners in tests, where emit keeps blocking-send
// semantics.
stopCh <-chan struct{}
// useActiveConfig is true for the daemon-owned listener, so SIGHUP
// threshold changes apply without rebuilding the socket. Unit tests that
// assemble a listener by hand keep using their local cfg.
useActiveConfig bool
// stuffing flags one source IP failing against many distinct accounts
// (credential stuffing / password spraying breadth) -- complementary to
// the per-IP failure-count brute-force trigger below. Guarded by mu.
stuffing *credentialStuffingDetector
// startedAt anchors the upstream-probe verdict so the dashboard does
// not flag a freshly-started daemon as "deaf" before the PAM module
// has had a chance to emit anything. Compared against lastPeerNanos
// to decide whether silence is informative.
startedAt time.Time
// lastPeerNanos is an atomic UnixNano timestamp updated on every
// inbound connection (regardless of payload validity). Used by the
// upstream probe to detect a missing PAM module hook.
lastPeerNanos atomic.Int64
// hostNameWarned limits the "remote host is not an address" log to one
// line per daemon run; sshd with UseDNS sends one on every login.
hostNameWarned atomic.Bool
}
type pamFailureTracker struct {
count int
firstSeen time.Time
lastSeen time.Time
users map[string]bool
services map[string]bool
accounts map[string]*pamAccountFailures
blocked bool
}
// pamServiceFailures tracks one address's failures from PAM services other
// than sshd inside the window.
type pamServiceFailures struct {
count int
firstSeen time.Time
lastSeen time.Time
services map[string]bool
reported bool
}
type pamAccountFailures struct {
count int
firstSeen time.Time
lastSeen time.Time
services map[string]bool
}
// NewPAMListener creates a Unix socket listener for PAM events.
func NewPAMListener(cfg *config.Config, alertCh chan<- alert.Finding) (*PAMListener, error) {
// Ensure socket directory exists
if err := os.MkdirAll("/var/run/csm", 0750); err != nil {
return nil, fmt.Errorf("creating socket dir: %w", err)
}
// Remove stale socket
_ = os.Remove(pamSocketPath)
listener, err := net.Listen("unix", pamSocketPath)
if err != nil {
return nil, fmt.Errorf("listening on %s: %w", pamSocketPath, err)
}
// The PAM module runs inside privileged auth stacks, so the socket can stay
// root-only instead of accepting arbitrary local writers.
_ = os.Chmod(pamSocketPath, 0600)
_, window, distinct := pamThresholds(cfg)
return &PAMListener{
cfg: cfg,
alertCh: alertCh,
listener: listener,
failures: make(map[string]*pamFailureTracker),
startedAt: time.Now(),
useActiveConfig: true,
stuffing: newCredentialStuffingDetector(distinct, window, nil),
}, nil
}
// UpstreamProbe returns an UpstreamResult describing whether the PAM
// module hook is feeding the socket. The probe is cheap (single atomic
// load + clock read) so it is safe to wire into the components API.
//
// Fresh verdict:
// - At least one inbound connection within pamUpstreamFreshWindow.
// - Or the daemon has been up for less than pamUpstreamGracePeriod
// (so a freshly-started daemon is not flagged before the first
// real auth happens).
//
// LastActivity is the most recent connection time, or the daemon start
// time when no connection has arrived. Reason explains a !Fresh verdict
// to operators.
func (p *PAMListener) UpstreamResult() health.UpstreamResult {
last := time.Unix(0, p.lastPeerNanos.Load())
if p.lastPeerNanos.Load() == 0 && time.Since(p.startedAt) < pamUpstreamGracePeriod {
return health.UpstreamResult{Fresh: true, LastActivity: p.startedAt}
}
if p.lastPeerNanos.Load() != 0 && time.Since(last) < pamUpstreamFreshWindow {
return health.UpstreamResult{Fresh: true, LastActivity: last}
}
reason := "no PAM module hook feeding the socket; install pam_csm.so and add `session optional pam_csm.so` to the relevant /etc/pam.d/ files"
activity := last
if p.lastPeerNanos.Load() == 0 {
activity = p.startedAt
}
return health.UpstreamResult{Fresh: false, LastActivity: activity, Reason: reason}
}
const (
// pamUpstreamFreshWindow is how recently a peer connection must have
// arrived before the upstream is considered alive. Sized to comfortably
// span the longest realistic gap between auth events on a host that
// has the PAM module installed.
pamUpstreamFreshWindow = 24 * time.Hour
// pamUpstreamGracePeriod is the post-start window during which a
// silent socket is not yet flagged as deaf.
pamUpstreamGracePeriod = 15 * time.Minute
)
// Run accepts connections and processes PAM events.
func (p *PAMListener) Run(stopCh <-chan struct{}) {
p.stopCh = stopCh
// Start cleanup goroutine to expire old failure records
obs.Go("pam-cleanup", func() { p.cleanupLoop(stopCh) })
// Accept connections
obs.Go("pam-accept", func() {
for {
conn, err := p.listener.Accept()
if err != nil {
select {
case <-stopCh:
return
default:
fmt.Fprintf(os.Stderr, "[%s] PAM listener accept error: %v\n", ts(), err)
time.Sleep(100 * time.Millisecond)
continue
}
}
obs.SafeGo("pam-conn", func() { p.handleConnection(conn) })
}
})
<-stopCh
}
// Stop closes the listener and removes the socket file.
func (p *PAMListener) Stop() {
_ = p.listener.Close()
_ = os.Remove(pamSocketPath)
}
func (p *PAMListener) handleConnection(conn net.Conn) {
defer func() { _ = conn.Close() }()
// Record the connection moment regardless of peer trust so the
// upstream probe can distinguish "PAM hook not installed" (no
// connections at all) from "PAM hook present but rejected as
// untrusted" (connections happen but never deliver payload).
p.lastPeerNanos.Store(time.Now().UnixNano())
if !isTrustedPAMPeer(conn) {
return
}
_ = conn.SetDeadline(time.Now().Add(1 * time.Second))
scanner := bufio.NewScanner(conn)
for scanner.Scan() {
line := scanner.Text()
p.processEvent(line)
}
}
// processEvent handles a single PAM event line.
// Format: FAIL ip=1.2.3.4 user=root service=sshd
//
// OK ip=1.2.3.4 user=root service=sshd
func (p *PAMListener) processEvent(line string) {
parts := strings.SplitN(strings.TrimSpace(line), " ", 2)
if len(parts) < 2 {
return
}
eventType := parts[0]
kvPart := parts[1]
var ip, user, service, pid, stamp string
for _, kv := range strings.Fields(kvPart) {
switch {
case strings.HasPrefix(kv, "ip="):
ip = kv[3:]
case strings.HasPrefix(kv, "user="):
user = kv[5:]
case strings.HasPrefix(kv, "service="):
service = kv[8:]
case strings.HasPrefix(kv, "pid="):
pid = kv[4:]
case strings.HasPrefix(kv, "ts="):
stamp = kv[3:]
}
}
if ip == "" || ip == "-" {
return
}
addr, err := netip.ParseAddr(ip)
if err != nil {
if p.hostNameWarned.CompareAndSwap(false, true) {
fmt.Fprintf(os.Stderr, "[%s] PAM listener: remote host %q is not an IP address; its logins are not tracked (set UseDNS no in sshd_config)\n", ts(), ip)
}
return
}
addr = addr.Unmap().WithZone("")
if addr.IsLoopback() {
return
}
ip = addr.String()
// Skip infra IPs. Read from the live config like the thresholds are: an
// infrastructure address added by reload must stop counting at once.
if isInfraIP(ip, p.currentCfg().InfraIPs) {
return
}
switch eventType {
case "FAIL":
findings := p.recordFailure(ip, user, service)
obs := pamObservation(pid, stamp)
for i := range findings {
findings[i].Observation = obs
}
p.emit(findings)
case "OK":
if service == "sshd" {
p.clearFailuresForUser(ip, user)
}
// Successful login from non-infra IP - informational alert
p.emit([]alert.Finding{{
Severity: alert.High,
Check: "pam_login",
Message: fmt.Sprintf("Login success from non-infra IP: %s (user: %s, service: %s)", ip, user, service),
Timestamp: time.Now(),
SourceIP: ip,
}})
}
}
// emit forwards findings to the alert channel. It must be called WITHOUT
// p.mu held: a stalled alert consumer blocks the send, and holding the lock
// across that send would wedge recordFailure, clearFailures, and the cleanup
// loop, letting failure trackers grow without bound.
func (p *PAMListener) emit(findings []alert.Finding) {
for i, f := range findings {
if !alert.Enqueue(p.alertCh, f, p.stopCh) {
alert.RecordQueueLoss(p.alertCh, uint64(len(findings[i+1:])))
// Shutting down and the dispatcher has stopped draining;
// drop the remaining findings rather than leak this
// goroutine. A nil stopCh (hand-constructed listener)
// makes this case unselectable, preserving blocking send.
return
}
}
}
// recordFailure updates per-IP failure state and returns any findings the
// update produced. Findings are returned rather than sent so the caller can
// emit them after releasing p.mu (see emit).
func (p *PAMListener) recordFailure(ip, user, service string) []alert.Finding {
p.mu.Lock()
defer p.mu.Unlock()
var findings []alert.Finding
cfg := p.currentCfg()
threshold, window, distinct := pamThresholds(cfg)
now := time.Now()
if service != "sshd" {
return p.recordServiceFailureLocked(ip, service, threshold, window, now)
}
tracker, exists := p.failures[ip]
if !exists {
tracker = &pamFailureTracker{
firstSeen: now,
users: make(map[string]bool),
services: make(map[string]bool),
accounts: make(map[string]*pamAccountFailures),
}
p.failures[ip] = tracker
}
if tracker.accounts == nil {
tracker.accounts = make(map[string]*pamAccountFailures)
}
account := tracker.accounts[user]
if account == nil {
account = &pamAccountFailures{firstSeen: now, services: make(map[string]bool)}
tracker.accounts[user] = account
}
tracker.count++
tracker.lastSeen = now
tracker.users[user] = true
tracker.services[service] = true
account.count++
account.lastSeen = now
account.services[service] = true
// Credential-stuffing breadth signal: one source IP failing against many
// distinct accounts. Independent of the per-IP failure-count brute-force
// trigger below, so a low-and-slow campaign that stays under the count
// threshold per account is still caught. Fires once per window.
p.ensureCredentialStuffingDetectorLocked(distinct, window, now)
if accounts, fire := p.stuffing.Record(ip, user); fire {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "credential_stuffing",
Message: fmt.Sprintf("Credential stuffing: %s failed logins against %d distinct accounts", ip, len(accounts)),
Details: fmt.Sprintf("Accounts targeted: %s\nService(s): %s",
strings.Join(accounts, ", "), strings.Join(sortedBoolKeys(tracker.services), ", ")),
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionPeer,
SprayTargets: append([]string(nil), accounts...),
})
}
// Only block if within the time window
if now.Sub(tracker.firstSeen) > window {
// Window expired - reset tracker
tracker.count = 1
tracker.firstSeen = now
tracker.users = map[string]bool{user: true}
tracker.services = map[string]bool{service: true}
tracker.accounts = map[string]*pamAccountFailures{user: {
count: 1, firstSeen: now, lastSeen: now, services: map[string]bool{service: true},
}}
tracker.blocked = false
return findings
}
if tracker.count >= threshold && !tracker.blocked {
tracker.blocked = true
users := sortedBoolKeys(tracker.users)
services := sortedBoolKeys(tracker.services)
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "pam_bruteforce",
Message: fmt.Sprintf("PAM brute-force detected: %s (%d failures in %ds)", ip, tracker.count, int(now.Sub(tracker.firstSeen).Seconds())),
Details: fmt.Sprintf("Users targeted: %s\nServices: %s",
strings.Join(users, ", "), strings.Join(services, ", ")),
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionPeer,
SprayTargets: users,
})
}
return findings
}
// recordServiceFailureLocked counts a failure from a PAM service other than
// sshd and reports the address once per window above the PAM threshold. The
// report never blocks: the service's own log carries the evidence.
func (p *PAMListener) recordServiceFailureLocked(ip, service string, threshold int, window time.Duration, now time.Time) []alert.Finding {
if p.serviceFailures == nil {
p.serviceFailures = make(map[string]*pamServiceFailures)
}
tracker := p.serviceFailures[ip]
if tracker == nil || now.Sub(tracker.firstSeen) > window {
tracker = &pamServiceFailures{firstSeen: now, services: make(map[string]bool)}
p.serviceFailures[ip] = tracker
}
tracker.count++
tracker.lastSeen = now
tracker.services[service] = true
if tracker.count < threshold || tracker.reported {
return nil
}
tracker.reported = true
return []alert.Finding{{
Severity: alert.High,
Check: "pam_auth_failures",
Message: fmt.Sprintf("PAM login failures from %s: %d in %ds (not blocked)", ip, tracker.count, int(now.Sub(tracker.firstSeen).Seconds())),
Details: fmt.Sprintf("Service(s): %s\nThese services have their own log detectors, which carry the evidence for blocking.", strings.Join(sortedBoolKeys(tracker.services), ", ")),
Timestamp: now,
SourceIP: ip,
}}
}
// clearFailuresForUser forgets the failures attributed to user from ip once
// that user logged in. Failures against other users stay, and so does the
// credential-stuffing breadth: an attacker walking many accounts who finally
// lands one (or who owns one valid account) must not reset the record against
// every other account. The tracker goes only when no failed user remains.
func (p *PAMListener) clearFailuresForUser(ip, user string) {
p.mu.Lock()
defer p.mu.Unlock()
p.stuffing.ClearAccount(ip, user)
tracker, ok := p.failures[ip]
if !ok {
return
}
delete(tracker.users, user)
delete(tracker.accounts, user)
if len(tracker.users) == 0 {
delete(p.failures, ip)
p.stuffing.Clear(ip)
return
}
if tracker.accounts != nil {
tracker.count = 0
tracker.firstSeen = time.Time{}
tracker.lastSeen = time.Time{}
tracker.services = make(map[string]bool)
for _, account := range tracker.accounts {
tracker.count += account.count
if tracker.firstSeen.IsZero() || account.firstSeen.Before(tracker.firstSeen) {
tracker.firstSeen = account.firstSeen
}
if account.lastSeen.After(tracker.lastSeen) {
tracker.lastSeen = account.lastSeen
}
for service := range account.services {
tracker.services[service] = true
}
}
threshold, _, _ := pamThresholds(p.currentCfg())
if tracker.count < threshold {
tracker.blocked = false
}
}
}
// cleanupLoop removes expired failure trackers every minute.
func (p *PAMListener) cleanupLoop(stopCh <-chan struct{}) {
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
for {
select {
case <-stopCh:
return
case <-ticker.C:
p.cleanupAt(time.Now())
}
}
}
func (p *PAMListener) cleanupAt(now time.Time) {
_, window, distinct := pamThresholds(p.currentCfg())
cutoff := now.Add(-window)
p.mu.Lock()
defer p.mu.Unlock()
for ip, tracker := range p.failures {
if tracker.lastSeen.Before(cutoff) {
delete(p.failures, ip)
}
}
for ip, tracker := range p.serviceFailures {
if tracker.lastSeen.Before(cutoff) {
delete(p.serviceFailures, ip)
}
}
if p.stuffing != nil {
p.stuffing.Configure(distinct, window, now)
p.stuffing.PruneStale(now)
}
}
func (p *PAMListener) currentCfg() *config.Config {
if p.useActiveConfig {
if cfg := config.Active(); cfg != nil {
return cfg
}
}
return p.cfg
}
func pamThresholds(cfg *config.Config) (threshold int, window time.Duration, distinct int) {
threshold = defaultPAMFailureThreshold
windowMin := defaultPAMFailureWindowMin
distinct = defaultCredStuffingDistinctAccounts
if cfg != nil {
if cfg.Thresholds.PAMBruteforceThreshold > 0 {
threshold = cfg.Thresholds.PAMBruteforceThreshold
}
if cfg.Thresholds.PAMBruteforceWindowMin > 0 {
windowMin = cfg.Thresholds.PAMBruteforceWindowMin
}
if cfg.Thresholds.CredStuffingDistinctAccounts > 0 {
distinct = cfg.Thresholds.CredStuffingDistinctAccounts
}
}
return threshold, time.Duration(windowMin) * time.Minute, distinct
}
func (p *PAMListener) ensureCredentialStuffingDetectorLocked(distinct int, window time.Duration, now time.Time) {
if p.stuffing == nil {
p.stuffing = newCredentialStuffingDetector(distinct, window, nil)
return
}
p.stuffing.Configure(distinct, window, now)
}
func sortedBoolKeys(m map[string]bool) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// isInfraIP checks if an IP is in the configured infra IP ranges.
// Duplicated here to avoid import cycle with checks package.
// isInfraIP defers to the shared matcher so an infra_ips entry written as a
// bare address ("203.0.113.5") counts here the way it does everywhere else;
// the local copy accepted CIDRs only and silently ignored bare entries.
func isInfraIP(ip string, infraNets []string) bool {
return checks.IsInfraIP(ip, infraNets)
}
// pamBootID names this boot: a pid and a clock reading from the module name
// one event only within it. Tests replace it.
var pamBootID = sync.OnceValue(func() string { return readPAMBootID(pamBootIDPath) })
const pamBootIDPath = "/proc/sys/kernel/random/boot_id"
func readPAMBootID(path string) string {
data, err := os.ReadFile(path) // #nosec G304 -- fixed procfs path; tests pass a temp file.
if err != nil {
return ""
}
id := strings.TrimSpace(string(data))
if len(id) != 36 || strings.Trim(id, "0123456789abcdef-") != "" {
return ""
}
for i := range id {
if (i == 8 || i == 13 || i == 18 || i == 23) != (id[i] == '-') {
return ""
}
}
return id
}
// pamObservation names one module event by its pid and clock reading on this
// boot. A line from an older module, or one whose fields do not parse,
// carries none.
func pamObservation(pid, ts string) alert.Observation {
const digits = "0123456789"
sec, frac, ok := strings.Cut(ts, ".")
if pid == "" || len(pid) > 10 || strings.Trim(pid, digits) != "" ||
!ok || sec == "" || len(sec) > 19 || strings.Trim(sec, digits) != "" ||
len(frac) != 9 || strings.Trim(frac, digits) != "" {
return alert.Observation{}
}
s, err := strconv.ParseInt(sec, 10, 64)
if err != nil || s <= 0 {
return alert.Observation{}
}
ns, err := strconv.ParseInt(frac, 10, 64)
boot := pamBootID()
if err != nil || boot == "" {
return alert.Observation{}
}
at := time.Unix(s, ns)
// Parked observations use time.Time's RFC3339 JSON encoding.
if at.Year() > 9999 {
return alert.Observation{}
}
return alert.Observation{
Producer: string(checks.ProducerPAMSocket),
Stream: "pam:" + boot,
Cursor: pid + ":" + ts,
ObservedAt: at,
}
}
//go:build linux
package daemon
import (
"net"
"golang.org/x/sys/unix"
)
func isTrustedPAMPeer(conn net.Conn) bool {
unixConn, ok := conn.(*net.UnixConn)
if !ok {
return false
}
rawConn, err := unixConn.SyscallConn()
if err != nil {
return false
}
trusted := false
controlErr := rawConn.Control(func(fd uintptr) {
// #nosec G115 -- socket fd from net.Conn.SyscallConn; POSIX fd fits in int.
cred, err := unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED)
if err == nil && cred != nil && cred.Uid == 0 {
trusted = true
}
})
return controlErr == nil && trusted
}
package daemon
import (
"errors"
"os"
"path/filepath"
"strings"
"syscall"
)
// deletedExeSuffix is what procfs appends to /proc/<pid>/exe once the binary
// behind a running process has been unlinked.
const deletedExeSuffix = " (deleted)"
// panelToolExeTrusted reports whether exe is an executable the control panel
// installed and only root can replace: a regular file inside one of roots,
// owned by root, writable by nobody else.
//
// Provenance reads the resolved exe rather than comm because comm is set by
// the process itself -- any user can run prctl(PR_SET_NAME, "upcp"). An exe
// path cannot be faked without actually executing that file, and a root-owned
// file with no group or other write bit cannot be put under the panel root by
// an unprivileged attacker. A binary unlinked after it started no longer has
// inspectable bytes behind it, so it proves nothing and is refused.
func panelToolExeTrusted(exe string, roots []string, mode os.FileMode, uid uint32) bool {
if len(roots) == 0 || uid != 0 {
return false
}
if !mode.IsRegular() || mode.Perm()&0o022 != 0 {
return false
}
return exeInPanelRoot(exe, roots)
}
// exeInPanelRoot reports whether exe resolves inside one of roots. Split out
// of panelToolExeTrusted so the walker can reject a path before stat()ing it:
// an ancestor's exe can point anywhere, including a hung network or FUSE
// mount, and a path test costs nothing.
func exeInPanelRoot(exe string, roots []string) bool {
if strings.HasSuffix(exe, deletedExeSuffix) || !filepath.IsAbs(exe) {
return false
}
clean := filepath.Clean(exe)
for _, root := range roots {
if strings.HasPrefix(clean, filepath.Clean(root)+string(filepath.Separator)) {
return true
}
}
return false
}
// exeStat resolves an executable path to the mode and owner uid used for the
// trust decision. A var so tests can describe a root-owned binary they cannot
// create.
var exeStat = statExe
func statExe(path string) (os.FileMode, uint32, error) {
// Stat, not Lstat: procfs hands back the kernel's resolved path for the
// running binary, and a root-owned target is one only root could place
// whichever way the path reached it.
fi, err := os.Stat(path)
if err != nil {
return 0, 0, err
}
st, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return 0, 0, errNoStat
}
return fi.Mode(), st.Uid, nil
}
var errNoStat = errors.New("daemon: file info carries no owner")
package daemon
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/crashsample"
emime "github.com/pidginhost/csm/internal/mime"
"github.com/pidginhost/csm/internal/parseworker"
"github.com/pidginhost/csm/internal/workerproc"
)
// parseWorkerUnitPrefix starts every parse worker unit name, so leftovers of
// a daemon that died can be found and killed.
const parseWorkerUnitPrefix = "csm-parse-worker-"
// stopParseWorkerUnits is a seam for tests.
var stopParseWorkerUnits = workerproc.StopUnits
// checkParseWorkerCommand is a seam for tests whose binary path does not exist.
var checkParseWorkerCommand = workerproc.CheckUnitCommand
// parseWorkerLauncher is a seam for tests that must not start units.
var parseWorkerLauncher = workerproc.UnitLauncher
// parseWorkerHiddenPaths is a seam for tests: the account home volumes
// ProtectHome does not hide from workers.
var parseWorkerHiddenPaths = checks.AccountHomeVolumes
func (d *Daemon) initParseWorkers() error {
if d.binaryPath == "" {
return errors.New("parse worker: daemon binary unknown")
}
logf := func(format string, args ...any) {
fmt.Fprintf(os.Stderr, "[%s] parse-worker: "+format+"\n", append([]any{ts()}, args...)...)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
if err := stopParseWorkerUnits(ctx, parseWorkerUnitPrefix); err != nil {
logf("stop leftover units: %v", err)
}
cancel()
// Refused here, once, rather than at every launch: a unit's dynamic user
// cannot run an owner-only binary, so each worker would fail at exec.
if err := checkParseWorkerCommand(d.binaryPath); err != nil {
d.reportIsolationUnavailable(err)
return fmt.Errorf("parse worker: %w", err)
}
registerParseWorkerMetrics()
hidden := parseWorkerHiddenPaths()
unitConfig := func(name string, memoryLimit, fileSizeLimit uint64) workerproc.UnitConfig {
cfg := parseworker.UnitConfig(d.binaryPath, name, memoryLimit, fileSizeLimit, logf)
cfg.InaccessiblePaths = hidden
return cfg
}
launchZip := countParseWorkerStarts("zip", parseWorkerLauncher(unitConfig(parseWorkerUnitPrefix+"zip0", parseworker.DefaultMemoryLimit, 64<<20)))
lister, err := parseworker.NewLister(parseworker.ListerConfig{
Launch: func(ctx context.Context) (workerproc.Process, error) {
proc, err := launchZip(ctx)
// Descriptor exhaustion bypasses the failure hook and breaker,
// but still leaves archives without an isolated parser.
if err != nil && ctx.Err() == nil {
d.reportIsolationUnavailable(err)
}
return proc, err
},
Log: logf,
OnJob: func(outcome string, memory uint64) { recordParseJob("zip", outcome, memory) },
})
if err != nil {
return err
}
d.zipLister = lister
d.registerQueueSource("archive_list", lister)
checks.SetZipLister(lister)
if d.cfg == nil || !d.cfg.EmailAV.Enabled {
return nil
}
// The spool watcher runs one scan goroutine per slot, sized the same way.
extractor, err := parseworker.NewExtractor(parseworker.ExtractorConfig{
Size: d.cfg.EmailAV.ScanConcurrency,
Launch: func(slot int) workerproc.Launcher {
name := fmt.Sprintf("%smail%d", parseWorkerUnitPrefix, slot)
return countParseWorkerStarts("mail", parseWorkerLauncher(unitConfig(name, mailWorkerMemoryLimit(d.cfg), parseworker.FileSizeLimit(d.cfg.EmailAV.MaxAttachmentSize))))
},
Log: logf,
OnJob: func(outcome string, memory uint64) { recordParseJob("mail", outcome, memory) },
})
if err != nil {
return err
}
d.mailExtractor = extractor
d.registerQueueSource("email_extract", extractor)
return nil
}
// mailWorkerMemoryLimit is the configured memory cap of each mail parse
// worker; validation keeps it at or above the minimum.
func mailWorkerMemoryLimit(cfg *config.Config) uint64 {
if cfg.EmailAV.WorkerMemoryLimit < config.MinWorkerMemoryLimit {
return parseworker.DefaultMemoryLimit
}
return uint64(cfg.EmailAV.WorkerMemoryLimit)
}
// reportIsolationUnavailable raises email_av_isolation_unavailable for parse
// workers that cannot run. It shares the spool watcher's hourly allowance.
func (d *Daemon) reportIsolationUnavailable(err error) {
d.parseFindings.emit(d.alertCh, "email_av_isolation_unavailable", alert.Critical, isolationAlertInterval,
fmt.Sprintf("Parse worker units cannot run: %q. Email attachments are handled as incomplete scans under email_av.fail_mode, and phishing-kit zips are reported as not checked.", err))
}
// mailExtractor is what the spool watcher needs from the mail parse workers;
// tests fake it.
type mailExtractor interface {
Size() int
Extract(ctx context.Context, slot int, header, body *os.File, limits emime.Limits) (*emime.ExtractionResult, error)
}
// spoolExtractor is the mail extractor for a new spool watcher, or nil.
// A nil pointer must not become a non-nil interface.
func (d *Daemon) spoolExtractor() mailExtractor {
// Panic cleanup can clear the extractor while a watcher is restarting.
d.spoolWatcherMu.Lock()
defer d.spoolWatcherMu.Unlock()
if d.mailExtractor == nil {
return nil
}
return d.mailExtractor
}
func (d *Daemon) stopParseWorkers() {
// During shutdown a watcher may still be draining or finishing a
// restart, and must keep its pool until it returns. Before shutdown is
// signalled (a panic unwinding Run) nothing tells those goroutines to
// stop: waiting would hang the daemon instead of letting it crash and be
// restarted.
select {
case <-d.stopCh:
d.wg.Wait()
default:
}
checks.SetZipLister(nil)
d.spoolWatcherMu.Lock()
extractor := d.mailExtractor
d.mailExtractor = nil
d.spoolWatcherMu.Unlock()
if extractor != nil {
_ = extractor.Stop()
}
if d.zipLister == nil {
return
}
_ = d.zipLister.Stop()
d.zipLister = nil
}
// initCrashSamples sets up the parse crash sample store. Phishing zips are
// kept only when email_av.keep_crash_samples is on.
func (d *Daemon) initCrashSamples() {
if d.cfg == nil || d.cfg.StatePath == "" {
return
}
d.crashSamples = crashsample.New(filepath.Join(d.cfg.StatePath, "parse-crashes"))
checks.SetCrashSamples(d.keptCrashSamples())
}
// keptCrashSamples is the store new crash samples go to, or nil when
// email_av.keep_crash_samples is off.
func (d *Daemon) keptCrashSamples() *crashsample.Store {
if d.crashSamples == nil || d.cfg == nil || !d.cfg.EmailAV.KeepCrashSamples {
return nil
}
return d.crashSamples
}
// crashSampleSweep removes expired crash samples at start and then hourly.
// It runs whether or not samples are kept, so turning the setting off never
// leaves customer data behind.
func (d *Daemon) crashSampleSweep() {
defer d.wg.Done()
d.sweepCrashSamples()
ticker := time.NewTicker(time.Hour)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
d.sweepCrashSamples()
}
}
}
func (d *Daemon) sweepCrashSamples() {
if d.crashSamples == nil {
return
}
n, err := d.crashSamples.Sweep()
if err != nil {
fmt.Fprintf(os.Stderr, "[%s] parse crash samples: sweep: %v\n", ts(), err)
} else if n > 0 {
fmt.Fprintf(os.Stderr, "[%s] parse crash samples: removed %d expired\n", ts(), n)
}
}
package daemon
import (
"context"
"sync"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/workerproc"
)
// Parse worker metrics, per pool ("mail" or "zip"). Nil until
// registerParseWorkerMetrics runs; callers nil-check.
var (
parseWorkerMetricsOnce sync.Once
parseJobsTotal *metrics.CounterVec
parseStartsTotal *metrics.CounterVec
parseMemoryBytes *metrics.HistogramVec
)
func newParseWorkerMetrics() (jobs, starts *metrics.CounterVec, memory *metrics.HistogramVec) {
jobs = metrics.NewCounterVec("csm_parse_worker_jobs_total",
"Parse worker jobs by pool and outcome: ok, timeout, oom, crash, protocol (reply rejected), spawn (unit did not start), refused (failing repeatedly) or skipped (the message killed workers before).",
[]string{"pool", "outcome"})
starts = metrics.NewCounterVec("csm_parse_worker_starts_total",
"Parse worker units started, by pool.",
[]string{"pool"})
memory = metrics.NewHistogramVec("csm_parse_worker_memory_bytes",
"Parse worker memory and swap in use after a successful job, by pool.",
[]string{"pool"},
[]float64{32 << 20, 64 << 20, 128 << 20, 256 << 20, 384 << 20, 512 << 20, 768 << 20, 1024 << 20})
// Both labels come from fixed sets.
jobs.SetMaxChildren(32)
starts.SetMaxChildren(4)
memory.SetMaxChildren(4)
return jobs, starts, memory
}
func registerParseWorkerMetrics() {
parseWorkerMetricsOnce.Do(func() {
parseJobsTotal, parseStartsTotal, parseMemoryBytes = newParseWorkerMetrics()
metrics.MustRegister("csm_parse_worker_jobs_total", parseJobsTotal)
metrics.MustRegister("csm_parse_worker_starts_total", parseStartsTotal)
metrics.MustRegister("csm_parse_worker_memory_bytes", parseMemoryBytes)
})
}
// recordParseJob counts one job of pool and, for a successful one with a
// known memory use, records that use.
func recordParseJob(pool, outcome string, memory uint64) {
if parseJobsTotal != nil {
parseJobsTotal.With(pool, outcome).Inc()
}
if outcome == workerproc.OutcomeOK && memory > 0 && parseMemoryBytes != nil {
parseMemoryBytes.With(pool).Observe(float64(memory))
}
}
// countParseWorkerStarts counts each unit of pool that launch starts.
func countParseWorkerStarts(pool string, launch workerproc.Launcher) workerproc.Launcher {
return func(ctx context.Context) (workerproc.Process, error) {
proc, err := launch(ctx)
if err == nil && parseStartsTotal != nil {
parseStartsTotal.With(pool).Inc()
}
return proc, err
}
}
package daemon
import (
"fmt"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
)
// PasswordHijackDetector tracks WHM password changes from non-infra IPs
// and correlates them with subsequent cPanel logins to detect the attack
// pattern: attacker changes password via WHM → immediately logs in.
//
// Legitimate flow (excluded):
//
// Portal (infra IP) changes password via xml-api → user logs in
//
// Attack flow (detected):
//
// Attacker (non-infra IP) changes password via whostmgr → logs in within 60s
type PasswordHijackDetector struct {
mu sync.Mutex
recentChanges map[string]*passwordChange // account -> change info
cfg *config.Config
alertCh chan<- alert.Finding
stopCh <-chan struct{} // closed on daemon shutdown; lets sends escape
}
type passwordChange struct {
account string
ip string
timestamp time.Time
}
const hijackWindow = 120 * time.Second // time window to correlate password change + login
// NewPasswordHijackDetector creates a new detector.
func NewPasswordHijackDetector(cfg *config.Config, alertCh chan<- alert.Finding, stopCh <-chan struct{}) *PasswordHijackDetector {
return &PasswordHijackDetector{
recentChanges: make(map[string]*passwordChange),
cfg: cfg,
alertCh: alertCh,
stopCh: stopCh,
}
}
// emit delivers a finding without ever blocking the (synchronous) session-log
// watcher goroutine forever. It applies backpressure while the daemon is
// running, but gives up the moment shutdown is signaled -- after the alert
// dispatcher stops draining, a plain send would otherwise wedge d.wg.Wait().
// Callers must NOT hold d.mu while calling this.
func (d *PasswordHijackDetector) emit(f alert.Finding) {
alert.Enqueue(d.alertCh, f, d.stopCh)
}
// HandlePasswordChange records a WHM password change from a non-infra IP.
func (d *PasswordHijackDetector) HandlePasswordChange(account, ip string) {
if isInfraIPDaemon(ip, d.cfg.InfraIPs) || ip == "127.0.0.1" || ip == "internal" {
return // legitimate - portal or admin action
}
d.mu.Lock()
d.recentChanges[account] = &passwordChange{
account: account,
ip: ip,
timestamp: time.Now(),
}
d.mu.Unlock()
// Alert on the password change itself - non-infra WHM password change is
// always suspicious. Emit outside the lock so a saturated alert channel
// can never wedge the mutex.
d.emit(alert.Finding{
Severity: alert.Critical,
Check: "whm_password_change_noninfra",
Message: fmt.Sprintf("WHM password change from non-infra IP: %s (account: %s)", ip, account),
Details: "Password was changed via WHM from an IP outside your infrastructure. This is a strong indicator of account takeover.",
Timestamp: time.Now(),
SourceIP: ip,
TenantID: account,
})
}
// HandleLogin checks if a cPanel login matches a recent non-infra password change.
func (d *PasswordHijackDetector) HandleLogin(account, loginIP string) {
if isInfraIPDaemon(loginIP, d.cfg.InfraIPs) {
return
}
d.mu.Lock()
change, exists := d.recentChanges[account]
if exists {
delete(d.recentChanges, account)
}
d.mu.Unlock()
if !exists {
return
}
// Check if within the hijack window
if time.Since(change.timestamp) > hijackWindow {
return
}
// CONFIRMED ATTACK: password changed from non-infra IP, login within 120s
d.emit(alert.Finding{
Severity: alert.Critical,
Check: "password_hijack_confirmed",
Message: fmt.Sprintf("CONFIRMED ACCOUNT HIJACK: %s - password changed from %s, login from %s within %ds", account, change.ip, loginIP, int(time.Since(change.timestamp).Seconds())),
Details: fmt.Sprintf("Attack pattern: WHM password change from non-infra IP followed by immediate cPanel login.\nPassword change IP: %s\nLogin IP: %s\nTime between: %ds\n\nBoth IPs should be permanently blocked.", change.ip, loginIP, int(time.Since(change.timestamp).Seconds())),
Timestamp: time.Now(),
SourceIP: loginIP,
TenantID: account,
})
}
// Cleanup removes expired entries.
func (d *PasswordHijackDetector) Cleanup() {
d.mu.Lock()
defer d.mu.Unlock()
for account, change := range d.recentChanges {
if time.Since(change.timestamp) > hijackWindow*2 {
delete(d.recentChanges, account)
}
}
}
// ParseSessionLineForHijack extracts password change and login events
// from session log lines and feeds them to the detector.
func ParseSessionLineForHijack(line string, detector *PasswordHijackDetector) {
r, ok := checks.ParseSessionRecord(line)
if !ok {
return
}
addr, isAddr := r.ClientAddr()
if !isAddr || addr.IsLoopback() {
return
}
switch {
case r.Service == "whostmgr" && r.Event == "PURGE" && r.Reason() == "password_change":
// WHM password change: [timestamp] info [whostmgr] IP PURGE account:token password_change
detector.HandlePasswordChange(r.Account, addr.String())
case r.Service == "cpaneld" && r.Event == "NEW" && !r.CreatedByAPI():
// cPanel login, not an API session: [timestamp] info [cpaneld] IP NEW account:token ...
detector.HandleLogin(r.Account, addr.String())
}
}
package daemon
import (
"fmt"
"os"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
)
// replayPendingFindings runs the batch parked by the previous shutdown
// through the normal dispatch pipeline: auto-response, history, correlation
// and alerting. Called once the dispatcher has been released, so the replay
// sees the same ordering as any live batch.
func (d *Daemon) replayPendingFindings() {
if d.store == nil {
return
}
err := d.store.ReplayPendingFindings(func(pending []alert.Finding) {
active := make([]alert.Finding, 0, len(pending))
for _, f := range pending {
if !checks.IsRetiredThreatScoreFinding(f) {
active = append(active, f)
}
}
if len(active) == 0 {
return
}
fmt.Fprintf(os.Stderr, "[%s] Replaying %d finding(s) left queued by the previous shutdown\n", ts(), len(active))
d.dispatchBatch(active)
})
if err != nil {
fmt.Fprintf(os.Stderr, "[%s] Cannot replay pending findings: %v\n", ts(), err)
}
}
package daemon
import (
"errors"
"fmt"
"net"
"net/url"
"os"
"path"
"strings"
"syscall"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/phpshield"
"golang.org/x/sys/unix"
)
const (
phpEventMaxBytes = 64 * 1024
phpEventArchiveMaxBytes = 10 * 1024 * 1024
phpEventSocketMode = 0o222
// phpShieldURIMaxBytes is the length the Shield cuts REQUEST_URI to before
// sending it.
phpShieldURIMaxBytes = 200
)
var (
phpEventsLogPath = phpshield.EventLogPath
phpEventsSocketPath = phpshield.EventSocketPath
phpEventRetryInterval = 2 * time.Second
phpShieldEventListener = listenPHPShieldEventSocket
errPHPEventArchiveFull = errors.New("PHP Shield event archive reached its size cap; rotate it before events can be archived again")
)
type phpEventPacketListener interface {
Read([]byte) (int, error)
SetReadDeadline(time.Time) error
Close() error
}
type phpEventUnixgramListener struct {
*net.UnixConn
path string
info os.FileInfo
}
func (l *phpEventUnixgramListener) Close() error {
err := l.UnixConn.Close()
if info, statErr := os.Lstat(l.path); statErr == nil && os.SameFile(l.info, info) {
_ = os.Remove(l.path)
}
return err
}
func listenPHPShieldEventSocket(path string) (phpEventPacketListener, error) {
addr := &net.UnixAddr{Name: path, Net: "unixgram"}
if info, err := os.Lstat(path); err == nil {
if info.Mode()&os.ModeSocket == 0 {
return nil, fmt.Errorf("PHP Shield event socket path is not a socket")
}
// Before this release the directory was tenant-writable. Do not trust an
// active socket a tenant may have planted there before the installer
// hardened it: otherwise Shield events could be delivered to that tenant.
stat, statOK := info.Sys().(*syscall.Stat_t)
foreignSocket := os.Geteuid() == 0 && statOK && stat.Uid != 0
if !foreignSocket {
probe, probeErr := net.DialUnix("unixgram", nil, addr)
if probeErr == nil {
_ = probe.Close()
return nil, fmt.Errorf("PHP Shield event socket is already active")
}
}
if removeErr := os.Remove(path); removeErr != nil {
return nil, fmt.Errorf("removing stale PHP Shield event socket: %w", removeErr)
}
} else if !os.IsNotExist(err) {
return nil, fmt.Errorf("checking PHP Shield event socket: %w", err)
}
conn, err := net.ListenUnixgram("unixgram", addr)
if err != nil {
return nil, fmt.Errorf("listening on PHP Shield event socket: %w", err)
}
// #nosec G302 -- write-only for every uid: tenant PHP processes must send
// events, and only the daemon can read them.
if chmodErr := os.Chmod(path, phpEventSocketMode); chmodErr != nil {
_ = conn.Close()
_ = os.Remove(path)
return nil, fmt.Errorf("setting PHP Shield event socket mode: %w", chmodErr)
}
info, err := os.Lstat(path)
if err != nil {
_ = conn.Close()
_ = os.Remove(path)
return nil, fmt.Errorf("checking PHP Shield event socket after bind: %w", err)
}
return &phpEventUnixgramListener{UnixConn: conn, path: path, info: info}, nil
}
func processPHPShieldEventPacket(data []byte, archivePath string, _ *config.Config, alertCh chan<- alert.Finding) (bool, error) {
if len(data) == 0 || len(data) > phpEventMaxBytes {
return false, nil
}
line := strings.TrimSuffix(string(data), "\n")
if line == "" || strings.ContainsAny(line, "\r\n") {
return false, nil
}
finding, quiet := parsePHPShieldEventLine(line)
if finding == nil {
return false, nil
}
_, archiveErr := appendPHPShieldEventArchive(archivePath, line)
if !quiet {
if finding.Timestamp.IsZero() {
finding.Timestamp = time.Now()
}
if !alert.TryEnqueue(alertCh, *finding) {
fmt.Fprintln(os.Stderr, "Warning: alert channel full, dropping PHP Shield finding")
}
}
return true, archiveErr
}
func appendPHPShieldEventArchive(path, line string) (archived bool, retErr error) {
// #nosec G304 G302 -- fixed root-owned archive path; O_NOFOLLOW rejects
// symlink replacement and mode 0600 keeps tenants from reading/truncating it.
fd, err := unix.Open(path, unix.O_WRONLY|unix.O_APPEND|unix.O_CREAT|unix.O_NONBLOCK|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0o600)
if err != nil {
return false, err
}
f := os.NewFile(uintptr(fd), path) // #nosec G115 -- unix.Open returned a non-negative fd
if f == nil {
_ = unix.Close(fd)
return false, fmt.Errorf("opening PHP Shield event archive")
}
defer func() {
if err := f.Close(); err != nil && retErr == nil {
retErr = err
}
}()
var stat unix.Stat_t
if err := unix.Fstat(fd, &stat); err != nil {
return false, err
}
if stat.Mode&unix.S_IFMT != unix.S_IFREG || stat.Nlink != 1 {
return false, fmt.Errorf("PHP Shield event archive is not a single-link regular file")
}
if err := f.Chmod(0o600); err != nil {
return false, err
}
record := line + "\n"
if stat.Size > phpEventArchiveMaxBytes-int64(len(record)) {
return false, errPHPEventArchiveFull
}
if _, err := f.WriteString(record); err != nil {
return false, err
}
return true, nil
}
func waitPHPShieldEventRetry(stopCh <-chan struct{}) bool {
timer := time.NewTimer(phpEventRetryInterval)
defer timer.Stop()
select {
case <-stopCh:
return false
case <-timer.C:
return true
}
}
func (d *Daemon) watchPHPShieldEvents() {
defer d.wg.Done()
buffer := make([]byte, phpEventMaxBytes+1)
lastError := ""
var listener phpEventPacketListener
defer func() {
if listener != nil {
_ = listener.Close()
}
}()
for {
select {
case <-d.stopCh:
return
default:
}
if listener == nil {
var err error
listener, err = phpShieldEventListener(phpEventsSocketPath)
if err != nil {
d.MarkWatcher("php_shield", false)
if err.Error() != lastError {
csmlog.Warn("PHP Shield event socket unavailable", "path", phpEventsSocketPath, "err", err)
lastError = err.Error()
}
if !waitPHPShieldEventRetry(d.stopCh) {
return
}
continue
}
lastError = ""
d.MarkWatcher("php_shield", true)
}
if err := listener.SetReadDeadline(time.Now().Add(phpEventRetryInterval)); err != nil {
_ = listener.Close()
listener = nil
d.MarkWatcher("php_shield", false)
if err.Error() != lastError {
csmlog.Warn("PHP Shield event socket deadline failed", "path", phpEventsSocketPath, "err", err)
lastError = err.Error()
}
if !waitPHPShieldEventRetry(d.stopCh) {
return
}
continue
}
n, err := listener.Read(buffer)
if err == nil {
processed, processErr := processPHPShieldEventPacket(buffer[:n], phpEventsLogPath, d.cfg, d.alertCh)
if processErr != nil {
if processErr.Error() != lastError {
csmlog.Warn("PHP Shield event archive unavailable", "path", phpEventsLogPath, "err", processErr)
lastError = processErr.Error()
}
} else if processed {
lastError = ""
}
continue
}
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
continue
}
_ = listener.Close()
listener = nil
d.MarkWatcher("php_shield", false)
if err.Error() != lastError {
csmlog.Warn("PHP Shield event socket read failed", "path", phpEventsSocketPath, "err", err)
lastError = err.Error()
}
if !waitPHPShieldEventRetry(d.stopCh) {
return
}
}
}
// parsePHPShieldLogLine wraps parsePHPShieldLine for the log watcher handler signature.
func parsePHPShieldLogLine(line string, _ *config.Config) []alert.Finding { //nolint:unparam
f := parsePHPShieldLine(line)
if f == nil {
return nil
}
return []alert.Finding{*f}
}
// parsePHPShieldLine parses a line from the PHP shield event log and returns
// a finding if it represents a security event.
//
// Format: [2026-03-25 10:00:00] EVENT_TYPE sha256=H ip=X script=Y uri=Z ua=A details=B
// Older Shields omit sha256; those observations cannot be quieted.
func parsePHPShieldLine(line string) *alert.Finding {
finding, quiet := parsePHPShieldEventLine(line)
if quiet {
return nil
}
return finding
}
// Keep the observation even when its alert is quiet: a route mismatch does
// not prove the requested file was absent, or that downstream CMS code is safe.
func parsePHPShieldEventLine(line string) (*alert.Finding, bool) {
line = strings.TrimSpace(line)
if line == "" || !strings.HasPrefix(line, "[") {
return nil, false
}
// Extract event type (first word after the timestamp bracket)
closeBracket := strings.Index(line, "]")
if closeBracket < 0 || closeBracket+2 >= len(line) {
return nil, false
}
rest := strings.TrimSpace(line[closeBracket+1:])
fields := strings.SplitN(rest, " ", 2)
if len(fields) < 1 {
return nil, false
}
eventType := fields[0]
// Extract key=value pairs. The URI and user agent are what identify the
// request: "/alfacgiapi/perl.alfa" from a "Mozlila" agent names the scanner,
// where the bare parameter name does not. Both were parsed and discarded.
var digest, ip, script, uri, ua, details string
if len(fields) > 1 {
kvPart := fields[1]
// Only the producer's first field is content evidence. A URI or user
// agent containing sha256= must not forge proof for a legacy event.
if first, rest, ok := strings.Cut(kvPart, " "); ok && strings.HasPrefix(first, "sha256=") {
digest = strings.TrimPrefix(first, "sha256=")
kvPart = rest
}
for _, kv := range splitKV(kvPart) {
switch kv[0] {
case "ip":
ip = kv[1]
case "script":
script = kv[1]
case "uri":
uri = kv[1]
case "ua":
ua = kv[1]
case "details":
details = kv[1]
}
}
}
context := phpShieldDetails(ip, uri, ua, details)
switch eventType {
case "BLOCK_PATH":
return &alert.Finding{
Severity: alert.Critical,
Check: "php_shield_block",
SourceIP: ip,
FilePath: script,
Message: fmt.Sprintf("PHP Shield blocked execution from dangerous path: %s", script),
Details: context,
}, false
case "WEBSHELL_PARAM":
// Observation, not a denial: for a document-root script the Shield never
// reaches its deny branch, so nothing was blocked. Every public site
// receives these daily, and rating them Critical buries the real blocks.
// A rewrite can reach a real shell, even one called index.php. Use the
// scanner's verified content cache and PHP's event-time fingerprint;
// reopening the path here could inspect a replacement file instead.
quiet := !phpShieldRequestReachedScript(script, uri) && checks.IsVerifiedCMSHash(digest)
return &alert.Finding{
Severity: alert.Warning,
Check: "php_shield_webshell",
SourceIP: ip,
FilePath: script,
Message: fmt.Sprintf("PHP Shield observed a webshell command parameter: %s", script),
Details: context,
}, quiet
case "BLOCK_WEBSHELL":
// Not gated on the request path: the Shield blocks on the executing
// script's own source, so a rewrite into a planted shell is still a
// stopped webshell.
return &alert.Finding{
Severity: alert.Critical,
Check: "php_shield_webshell",
SourceIP: ip,
FilePath: script,
Message: fmt.Sprintf("PHP Shield blocked a webshell signature: %s", script),
Details: context,
}, false
case "EVAL_FATAL":
f := &alert.Finding{
Severity: alert.High,
Check: "php_shield_eval",
SourceIP: ip,
FilePath: script,
Message: fmt.Sprintf("PHP Shield detected eval() chain failure: %s", script),
Details: context,
}
if phpShieldEvalSiteIsSystemCode(script) {
f.Severity = alert.Warning
f.Message = fmt.Sprintf("PHP Shield reported eval() failure at a root-owned system site: %s", script)
f.Details = strings.TrimPrefix(context+"\n"+phpShieldSystemEvalNote, "\n")
}
return f, false
}
return nil, false
}
// phpShieldRequestReachedScript reports whether the request URI names the
// script that executed, i.e. whether a command parameter was delivered to the
// script the client asked for.
//
// Scanners send cmd= to paths that do not exist. CMS rewrite rules (or a
// 404 handler) can answer with the site's front controller. This is only a
// routing hint: a rewritten request can also execute a real shell. The caller
// must establish content evidence before quieting an alert, and still archive
// the observation. The request names the executing script when a
// leading run of its path segments is a trailing part of the script path (this
// covers PATH_INFO such as /shell.php/extra), or when it names the directory
// holding the script, which is then served as its directory index. A request
// for "/" therefore still fires on the document-root index.php: the client did
// ask for that script, and a shell injected into index.php is reached exactly
// that way.
//
// A leading /~user segment is dropped as well, since that is how a userdir URL
// maps onto the account's document root.
//
// When the path cannot be judged (no URI, a form other than an origin or
// absolute path, a bad escape, or a path cut short by the Shield's truncation)
// the event is kept: silence has to be earned by a path that clearly names a
// different script.
func phpShieldRequestReachedScript(script, uri string) bool {
if !path.IsAbs(script) || uri == "" || uri == "-" {
return true
}
rawPath, _, hasQuery := strings.Cut(uri, "?")
if !strings.HasPrefix(rawPath, "/") {
parsed, err := url.ParseRequestURI(uri)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil {
return true
}
if strings.HasPrefix(parsed.Host, "[") && net.ParseIP(parsed.Hostname()) == nil {
return true
}
rawPath = parsed.EscapedPath()
if rawPath == "" {
rawPath = "/"
}
}
decoded, err := url.PathUnescape(rawPath)
if err != nil {
return true
}
for _, c := range decoded {
if c <= ' ' || c == 0x7f || c == '\\' || c == '#' {
return true
}
}
requested := path.Clean(decoded)
script = path.Clean(script)
candidates := []string{requested}
if first, rest, _ := strings.Cut(requested[1:], "/"); strings.HasPrefix(first, "~") {
candidates = append(candidates, "/"+rest)
}
for _, candidate := range candidates {
if phpShieldPathNamesScript(script, candidate) {
return true
}
}
return !hasQuery && len(uri) >= phpShieldURIMaxBytes
}
// phpShieldPathNamesScript applies the matching rule described on
// phpShieldRequestReachedScript to one cleaned request path.
func phpShieldPathNamesScript(script, requested string) bool {
if strings.HasSuffix(path.Dir(script), strings.TrimSuffix(requested, "/")) {
return true
}
for end := len(requested); end > 0; end = strings.LastIndexByte(requested[:end], '/') {
if strings.HasSuffix(script, requested[:end]) {
return true
}
}
return false
}
// phpShieldDetails renders the context an operator needs to judge a Shield
// event: who sent it, what they asked for, and what they claimed to be. Empty
// fields are omitted rather than printed as blanks.
func phpShieldDetails(ip, uri, ua, details string) string {
var b strings.Builder
for _, field := range [][2]string{
{"IP", ip},
{"URI", uri},
{"User-Agent", ua},
} {
if field[1] == "" {
continue
}
if b.Len() > 0 {
b.WriteString("\n")
}
fmt.Fprintf(&b, "%s: %s", field[0], field[1])
}
if details != "" {
if b.Len() > 0 {
b.WriteString("\n")
}
b.WriteString(details)
}
return b.String()
}
// splitKV splits "key1=val1 key2=val2" respecting values with spaces.
func splitKV(s string) [][2]string {
var result [][2]string
keys := []string{"ip=", "script=", "uri=", "ua=", "details="}
for i, key := range keys {
idx := strings.Index(s, key)
if idx < 0 {
continue
}
valStart := idx + len(key)
// Value ends at the next key or end of string
valEnd := len(s)
for _, nextKey := range keys[i+1:] {
nextIdx := strings.Index(s[valStart:], " "+nextKey)
if nextIdx >= 0 {
valEnd = valStart + nextIdx
break
}
}
val := s[valStart:valEnd]
// Preserve the URI byte count and ambiguous whitespace. Trimming a
// truncated path can make it look complete enough to suppress an alert.
if key != "uri=" {
val = strings.TrimSpace(val)
}
keyName := strings.TrimSuffix(key, "=")
result = append(result, [2]string{keyName, val})
}
return result
}
package daemon
import (
"os"
"path"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
// phpEvalCodeSuffix is how PHP names code run by eval(): the file of the
// eval() call, its line in parentheses, then this suffix.
const phpEvalCodeSuffix = ") : eval()'d code"
const phpShieldSystemEvalNote = "The reported eval() site is a root-owned file no account can change. The evaluated code and event sender were not verified."
var (
// phpShieldEvalSiteLstat is replaceable so tests can describe root-owned
// trees they cannot create.
phpShieldEvalSiteLstat = os.Lstat
// phpShieldEvalSiteTimeout bounds how long one event may hold the Shield
// event reader while its eval site is inspected.
phpShieldEvalSiteTimeout = 500 * time.Millisecond
// phpShieldEvalSiteProbe admits one inspection at a time. A probe stuck in
// a hung mount keeps it, so later events are graded at once instead of
// each stalling the reader and leaving another blocked thread behind.
phpShieldEvalSiteProbe = make(chan struct{}, 1)
phpShieldEvalSiteHealth = queuehealth.New(0, time.Minute)
)
// phpShieldEvalSiteIsSystemCode reports whether errorFile names a single
// reported eval() call in a root-owned file that no account can change.
//
// WP Toolkit's bundled wp-cli running "wp eval" as the account is the common
// case. An account that runs that script with its own code gains nothing from
// the lower grade: the Shield is loaded by that account's own PHP process,
// which can already turn it off. The event fields do not prove who sent it
// or which code ran. A web request that reaches such an eval() is still
// reported at Warning. A reported site in a file an account can write must
// stay High, so ownership is checked on the live filesystem at receipt for
// the file and every directory above it, without following symlinks. cPanel
// account homes are owned by the account, so no path under one qualifies.
// Anything this cannot prove keeps the High grade.
func phpShieldEvalSiteIsSystemCode(errorFile string) bool {
site, ok := strings.CutSuffix(errorFile, phpEvalCodeSuffix)
if !ok {
return false
}
open := strings.LastIndexByte(site, '(')
if open < 0 {
return false
}
file, line := site[:open], site[open+1:]
if line == "" || strings.Trim(line, "0123456789") != "" {
return false
}
// PHP reports resolved paths, so anything else did not come from PHP. A
// nested eval() blames the outer eval()'d code, which is no file, so the
// walk refuses it.
if !path.IsAbs(file) || path.Clean(file) != file {
return false
}
return phpShieldEvalSiteProven(file)
}
// phpShieldEvalSiteProven runs the ownership walk with a deadline. Event
// fields are forgeable by any local user, so a path on a hung network mount
// must cost the reader one timeout at most, and then nothing until it clears.
func phpShieldEvalSiteProven(file string) bool {
work := acquirePHPShieldEvalSiteWork()
if work == nil {
return false
}
defer work.release()
result := make(chan bool, 1)
go func() {
work.run(file, result)
}()
timer := time.NewTimer(phpShieldEvalSiteTimeout)
defer timer.Stop()
select {
case proven := <-result:
return proven
case <-timer.C:
work.fail()
return false
}
}
type phpShieldEvalSiteWork struct {
ticket queuehealth.Ticket
stats *queuehealth.Tracker
remaining atomic.Int32
failOnce sync.Once
}
func acquirePHPShieldEvalSiteWork() *phpShieldEvalSiteWork {
stats := phpShieldEvalSiteHealth
ticket := stats.Begin(time.Now())
select {
case phpShieldEvalSiteProbe <- struct{}{}:
work := &phpShieldEvalSiteWork{ticket: ticket, stats: stats}
work.remaining.Store(2)
return work
default:
ticket.Reject(time.Now())
return nil
}
}
func (w *phpShieldEvalSiteWork) fail() {
w.failOnce.Do(func() { w.stats.Lose(time.Now(), 1) })
}
func (w *phpShieldEvalSiteWork) release() {
// Neither a caller timeout nor a buffered outcome ends ownership alone.
if w.remaining.Add(-1) == 0 {
w.ticket.Finish(time.Now())
}
}
func (w *phpShieldEvalSiteWork) run(file string, result chan<- bool) {
w.ticket.Start(time.Now())
completed := false
defer func() {
if !completed {
w.fail()
}
w.release()
}()
proven := func() bool {
// Release before publishing, even if the walk exits abnormally.
// The next event must not depend on the caller being scheduled.
defer func() { <-phpShieldEvalSiteProbe }()
return rootOwnedUnwritableChain(file)
}()
result <- proven
completed = true
}
func phpShieldEvalSiteQueueStatus(now time.Time) queuehealth.Status {
status := phpShieldEvalSiteHealth.Snapshot(now)
// One filesystem walk is bounded, but concurrent callers can still hold
// completed results after it releases its slot.
status.CapacityUnavailable = true
status.Advisory = true
switch {
case status.LagSeconds >= phpShieldEvalSiteTimeout.Seconds():
status.Status, status.Reason = "degraded", "backlog_lag"
case status.ProcessingSeconds >= phpShieldEvalSiteTimeout.Seconds():
status.Status, status.Reason = "degraded", "processing_lag"
}
return status
}
// rootOwnedUnwritableChain reports whether file is a regular file and it and
// every directory from / down to it are real (not symlinks), owned by root,
// and without group or other write permission.
func rootOwnedUnwritableChain(file string) bool {
current := "/"
components := strings.Split(file[1:], "/")
for i := -1; i < len(components); i++ {
if i >= 0 {
current = path.Join(current, components[i])
}
info, err := phpShieldEvalSiteLstat(current)
if err != nil {
return false
}
last := i == len(components)-1
if (last && !info.Mode().IsRegular()) || (!last && !info.Mode().IsDir()) {
return false
}
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok || stat.Uid != 0 || info.Mode().Perm()&0o022 != 0 {
return false
}
}
return true
}
package daemon
import (
"bufio"
"bytes"
"encoding/gob"
"fmt"
"io"
"net"
"net/mail"
"net/url"
"os"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/contenttype"
"github.com/pidginhost/csm/internal/crawlid"
"github.com/pidginhost/csm/internal/emailspool"
"github.com/pidginhost/csm/internal/eximlog"
"github.com/pidginhost/csm/internal/store"
)
var phpRelayEvaluatorRef atomic.Pointer[evaluator]
// SetPHPRelayEvaluator is called by daemon wiring once everything is up.
// nil disables php_relay path dispatch from parseEximLogLine.
func SetPHPRelayEvaluator(e *evaluator) {
phpRelayEvaluatorRef.Store(e)
}
// PHPRelayEvaluator returns the registered evaluator, or nil if none.
func PHPRelayEvaluator() *evaluator {
return phpRelayEvaluatorRef.Load()
}
// scriptKey = host(X-PHP-Script) + ":" + path(X-PHP-Script)
type scriptKey = string
// scriptEvent is one accepted outbound mail from a PHP script. The booleans
// are computed at acceptance time (Flow A); evaluatePaths counts them in
// the sliding window.
type scriptEvent struct {
At time.Time
MsgID string
Subject string
FromMismatch bool
AdditionalSignal bool // Reply-To external mismatch OR X-Mailer suspicious-not-safe
SourceIP string
}
// rejectionEvent records a remote-MTA policy-block rejection for Path 3.
// Stage 1 defines the type for completeness; Stage 2 wires it.
//
//nolint:unused // wired by Path 3 in Stage 2
type rejectionEvent struct {
At time.Time
MsgID string
MTACode string
Snippet string
}
const (
phpRelayMaxEventsPerScript = 256
phpRelayMaxRejectionsPerScript = 64
phpRelayMaxActiveMsgsPerScript = 4096
phpRelayScriptIdleHorizon = 25 * time.Hour //nolint:unused // consumed by Flow E in Task O2
)
// recipientTracker records the distinct envelope recipients seen in some
// window and whether that window is safe to gate on. A window is only safe
// when at least one recipient parse succeeded and no parse gap was seen in
// that same window, so a caller reading it can fail open. Callers hold their
// own lock; the tracker has none.
type recipientTracker struct {
recipients map[string]time.Time
lastAt time.Time
lastUnknownAt time.Time
}
// record folds one message's recipients in. An empty list or any invalid
// address marks a parse gap. A domain skipped by skipDomain still counts as
// parsed, but its recipients are left out of the distinct count.
func (t *recipientTracker) record(recipients []string, at time.Time, max int, skipDomain func(string) bool) {
recorded := false
complete := len(recipients) > 0
if len(recipients) > 0 {
if t.recipients == nil {
t.recipients = make(map[string]time.Time, 8)
}
for _, raw := range recipients {
r, domain := parseRelayRecipient(raw)
if r == "" {
complete = false
continue
}
recorded = true
if skipDomain != nil && skipDomain(domain) {
continue
}
if _, exists := t.recipients[r]; !exists && len(t.recipients) >= max {
evictOldestRecipient(t.recipients)
}
if at.After(t.recipients[r]) {
t.recipients[r] = at
}
}
}
if recorded && at.After(t.lastAt) {
t.lastAt = at
}
if !complete && at.After(t.lastUnknownAt) {
t.lastUnknownAt = at
}
}
// distinctSince returns how many distinct recipients were seen no earlier than
// since, and whether that answer is trustworthy. known=false means fail open.
func (t *recipientTracker) distinctSince(since time.Time) (count int, known bool) {
known = seenAtOrAfter(t.lastAt, since) && !seenAtOrAfter(t.lastUnknownAt, since)
for _, last := range t.recipients {
if !last.Before(since) {
count++
}
}
return count, known
}
// scriptState tracks one script's recent activity. All fields read or
// written through the embedded mutex.
type scriptState struct {
mu sync.Mutex
events []scriptEvent
rejections []rejectionEvent //nolint:unused // wired by Path 3 in Stage 2
firedAt map[string]time.Time
rcpts recipientTracker
activeMsgs map[string]time.Time // msgID -> acceptedAt
activeMsgsCapped bool
lastEvent time.Time
maxEvents int
maxRejections int //nolint:unused // wired by Path 3 in Stage 2
maxActiveMsgs int
}
func newScriptState() *scriptState {
return &scriptState{
firedAt: make(map[string]time.Time, 4),
activeMsgs: make(map[string]time.Time, 64),
maxEvents: phpRelayMaxEventsPerScript,
maxRejections: phpRelayMaxRejectionsPerScript,
maxActiveMsgs: phpRelayMaxActiveMsgsPerScript,
}
}
func (s *scriptState) append(e scriptEvent) {
s.mu.Lock()
defer s.mu.Unlock()
s.appendLocked(e)
}
func (s *scriptState) appendLocked(e scriptEvent) {
if len(s.events) >= s.maxEvents {
s.events = s.events[1:]
}
s.events = append(s.events, e)
if e.At.After(s.lastEvent) {
s.lastEvent = e.At
}
}
// appendMessage records an event and its recipient parse result as one state
// transition. This prevents a concurrent evaluator from counting the event
// while still trusting recipient data from only the preceding messages.
func (s *scriptState) appendMessage(e scriptEvent, recipients []string) {
s.mu.Lock()
defer s.mu.Unlock()
s.appendLocked(e)
s.rcpts.record(recipients, e.At, maxTrackedRecipients, nil)
}
// distinctRecipientsSince reports the distinct recipients this script reached
// no earlier than since. known=false means callers must fail open.
func (s *scriptState) distinctRecipientsSince(since time.Time) (count int, known bool) {
s.mu.Lock()
defer s.mu.Unlock()
return s.rcpts.distinctSince(since)
}
// qualifyingCount returns the number of events whose At is at or after
// since AND for which match returns true.
func (s *scriptState) qualifyingCount(since time.Time, match func(scriptEvent) bool) int {
s.mu.Lock()
defer s.mu.Unlock()
n := 0
for _, e := range s.events {
if e.At.Before(since) {
continue
}
if match(e) {
n++
}
}
return n
}
// volumeCount returns events on or after since regardless of signal flags.
//
//nolint:unused // consumed by Path 2 evaluator in Task F2
func (s *scriptState) volumeCount(since time.Time) int {
return s.qualifyingCount(since, func(scriptEvent) bool { return true })
}
func (s *scriptState) relayHit(k scriptKey, since time.Time, match func(scriptEvent) bool) (alert.RelayScriptHit, bool) {
s.mu.Lock()
defer s.mu.Unlock()
hit := alert.RelayScriptHit{ScriptKey: string(k)}
var sampleAt time.Time
for _, e := range s.events {
if e.At.Before(since) || !match(e) {
continue
}
hit.Hits++
if e.At.After(hit.LastSeen) {
hit.LastSeen = e.At
}
if e.Subject != "" && (sampleAt.IsZero() || e.At.After(sampleAt)) {
hit.SampleSubject = truncateDaemon(e.Subject, phpRelayBreakdownSubjectMax)
sampleAt = e.At
}
}
if hit.Hits == 0 {
return alert.RelayScriptHit{}, false
}
return hit, true
}
func (s *scriptState) recordActive(msgID string, at time.Time) {
s.mu.Lock()
defer s.mu.Unlock()
if _, exists := s.activeMsgs[msgID]; !exists && len(s.activeMsgs) >= s.maxActiveMsgs {
// Drop oldest.
var oldestID string
var oldest time.Time
first := true
for id, t := range s.activeMsgs {
if first || t.Before(oldest) {
oldestID = id
oldest = t
first = false
}
}
if oldestID != "" {
delete(s.activeMsgs, oldestID)
}
s.activeMsgsCapped = true
}
s.activeMsgs[msgID] = at
}
func (s *scriptState) removeActive(msgID string) {
s.mu.Lock()
delete(s.activeMsgs, msgID)
s.mu.Unlock()
}
// snapshotActiveMsgs returns a copy of activeMsgs keys and the capped flag.
// The returned slice is independent of internal state; callers may mutate
// it without affecting the scriptState.
func (s *scriptState) snapshotActiveMsgs() ([]string, bool) {
s.mu.Lock()
defer s.mu.Unlock()
out := make([]string, 0, len(s.activeMsgs))
for id := range s.activeMsgs {
out = append(out, id)
}
return out, s.activeMsgsCapped
}
// pruneActiveMsgsOlderThan drops activeMsgs entries older than cutoff.
// Called by Flow E's GC to bound the lifetime of unreaped ids.
func (s *scriptState) pruneActiveMsgsOlderThan(cutoff time.Time) int {
s.mu.Lock()
defer s.mu.Unlock()
n := 0
for id, at := range s.activeMsgs {
if at.Before(cutoff) {
delete(s.activeMsgs, id)
n++
}
}
return n
}
// shouldFire returns true and updates firedAt if the cooldown for path has
// elapsed since the last fire.
//
//nolint:unused // consumed by Path 1/2/3 evaluators in Tasks F1/F2/G1
func (s *scriptState) shouldFire(path string, now time.Time, cooldown time.Duration) bool {
s.mu.Lock()
defer s.mu.Unlock()
if last, ok := s.firedAt[path]; ok && now.Sub(last) < cooldown {
return false
}
s.firedAt[path] = now
return true
}
// perScriptWindow keeps a scriptState per scriptKey.
type perScriptWindow struct {
states sync.Map // map[scriptKey]*scriptState
}
func newPerScriptWindow() *perScriptWindow { return &perScriptWindow{} }
func (w *perScriptWindow) getOrCreate(k scriptKey) *scriptState {
if v, ok := w.states.Load(k); ok {
return v.(*scriptState)
}
fresh := newScriptState()
actual, _ := w.states.LoadOrStore(k, fresh)
return actual.(*scriptState)
}
// SweepIdle drops scriptState entries whose lastEvent is before cutoff.
// Returns the number of entries dropped. Called by Flow E.
//
//nolint:unused // consumed by Flow E GC in Task O2
func (w *perScriptWindow) SweepIdle(cutoff time.Time) int {
n := 0
w.states.Range(func(k, v any) bool {
s := v.(*scriptState)
s.mu.Lock()
idle := s.lastEvent.Before(cutoff)
s.mu.Unlock()
if idle {
w.states.Delete(k)
n++
}
return true
})
return n
}
// PruneActiveMsgs iterates retained scriptStates and prunes activeMsgs
// entries older than cutoff. Used by Flow E so still-active scripts don't
// accumulate ghost activeMsgs whose corresponding messages have left the
// queue without a "Completed" log line being parsed. Returns the total
// number of activeMsgs entries removed across all scripts.
func (w *perScriptWindow) PruneActiveMsgs(cutoff time.Time) int {
n := 0
w.states.Range(func(_, v any) bool {
n += v.(*scriptState).pruneActiveMsgsOlderThan(cutoff)
return true
})
return n
}
// Snapshot returns the current per-script states (for csm phprelay status).
//
//nolint:unused // consumed by csm phprelay status command in Task M1
func (w *perScriptWindow) Snapshot() map[scriptKey]*scriptState {
out := make(map[scriptKey]*scriptState)
w.states.Range(func(k, v any) bool {
out[k.(scriptKey)] = v.(*scriptState)
return true
})
return out
}
type ipState struct {
mu sync.Mutex
scripts map[scriptKey]*ipScriptState
rcpts recipientTracker
lastEvent time.Time
}
type ipScriptState struct {
lastSeen time.Time
sampleAt time.Time
sampleSubject string
}
type perIPWindow struct {
states sync.Map // map[string]*ipState
capPerIP int
}
func newPerIPWindow(capPerIP int) *perIPWindow {
if capPerIP <= 0 {
capPerIP = 64
}
return &perIPWindow{capPerIP: capPerIP}
}
func (w *perIPWindow) append(ip string, k scriptKey, at time.Time, subject ...string) {
if ip == "" {
return
}
v, _ := w.states.LoadOrStore(ip, &ipState{scripts: make(map[scriptKey]*ipScriptState, 8)})
s := v.(*ipState)
s.mu.Lock()
defer s.mu.Unlock()
s.appendLocked(k, at, w.capPerIP, subject...)
}
func (s *ipState) appendLocked(k scriptKey, at time.Time, capPerIP int, subject ...string) {
if _, exists := s.scripts[k]; !exists && len(s.scripts) >= capPerIP {
var oldestK scriptKey
var oldest time.Time
first := true
for kk, ss := range s.scripts {
if first || ss.lastSeen.Before(oldest) {
oldestK = kk
oldest = ss.lastSeen
first = false
}
}
delete(s.scripts, oldestK)
}
ss := s.scripts[k]
if ss == nil {
ss = &ipScriptState{}
s.scripts[k] = ss
}
sampleSubject := ""
if len(subject) > 0 {
sampleSubject = truncateDaemon(subject[0], phpRelayBreakdownSubjectMax)
}
if at.After(ss.lastSeen) {
ss.lastSeen = at
}
if sampleSubject != "" && (ss.sampleAt.IsZero() || at.After(ss.sampleAt)) {
ss.sampleAt = at
ss.sampleSubject = sampleSubject
}
if at.After(s.lastEvent) {
s.lastEvent = at
}
}
// appendMessage records a script hit and its recipient parse result while
// holding the same IP-state lock, keeping the Path 4 gate fail-open when
// evaluation runs concurrently with spool processing. Recipients in the
// sending account's listed domains are parsed but not counted. Subdomains
// need their own entry: they can belong to a different hosting account. An
// empty ownDomains set counts every recipient.
func (w *perIPWindow) appendMessage(ip string, k scriptKey, at time.Time, subject string, recipients []string, ownDomains map[string]struct{}) {
if ip == "" {
return
}
v, _ := w.states.LoadOrStore(ip, &ipState{scripts: make(map[scriptKey]*ipScriptState, 8)})
s := v.(*ipState)
s.mu.Lock()
defer s.mu.Unlock()
s.appendLocked(k, at, w.capPerIP, subject)
s.rcpts.record(recipients, at, maxTrackedRecipients, func(domain string) bool {
_, own := ownDomains[domain]
return own
})
}
func (w *perIPWindow) distinctScriptsSince(ip string, since time.Time) int {
v, ok := w.states.Load(ip)
if !ok {
return 0
}
s := v.(*ipState)
s.mu.Lock()
defer s.mu.Unlock()
n := 0
for _, ss := range s.scripts {
if !ss.lastSeen.Before(since) {
n++
}
}
return n
}
// maxTrackedRecipients bounds each script or source-IP recipient set. A
// genuine high-diversity relay crosses the gate threshold before this cap, so
// the bound protects memory without hiding the current high diversity.
const maxTrackedRecipients = 256
// distinctRecipientsSince returns the number of distinct recipients seen from
// ip no earlier than since, and whether any recipient data was recorded within
// that window. known=false means recipients are unknown; callers must fail open.
func (w *perIPWindow) distinctRecipientsSince(ip string, since time.Time) (count int, known bool) {
v, ok := w.states.Load(ip)
if !ok {
return 0, false
}
s := v.(*ipState)
s.mu.Lock()
defer s.mu.Unlock()
return s.rcpts.distinctSince(since)
}
func seenAtOrAfter(t, since time.Time) bool {
return !t.IsZero() && !t.Before(since)
}
func evictOldestRecipient(m map[string]time.Time) {
var oldestK string
var oldest time.Time
first := true
for k, t := range m {
if first || t.Before(oldest) {
oldestK, oldest, first = k, t, false
}
}
if !first {
delete(m, oldestK)
}
}
// Envelope recipients are mailboxes, not display-name headers. Wrapping the
// input forces addr-spec parsing, so angle brackets inside a quoted local
// part cannot substitute a different domain for the actual destination.
func parseRelayRecipient(r string) (recipient, domain string) {
r = strings.TrimSpace(r)
if strings.HasPrefix(r, "<") && strings.HasSuffix(r, ">") {
r = strings.TrimSpace(r[1 : len(r)-1])
}
addr, err := mail.ParseAddress("<" + r + ">")
if err != nil {
return "", ""
}
at := strings.LastIndexByte(addr.Address, '@')
if at < 0 {
return "", ""
}
domain = strings.TrimSuffix(emailspool.ExtractDomain("recipient"+addr.Address[at:]), ".")
return strings.ToLower(addr.Address[:at]) + "@" + domain, domain
}
func (w *perIPWindow) relaySamplesSince(ip string, since time.Time) []alert.RelayScriptHit {
v, ok := w.states.Load(ip)
if !ok {
return nil
}
s := v.(*ipState)
s.mu.Lock()
defer s.mu.Unlock()
out := make([]alert.RelayScriptHit, 0, len(s.scripts))
for k, ss := range s.scripts {
if ss.lastSeen.Before(since) {
continue
}
sampleSubject := ""
if !ss.sampleAt.Before(since) {
sampleSubject = ss.sampleSubject
}
out = append(out, alert.RelayScriptHit{
ScriptKey: string(k),
Hits: 1,
LastSeen: ss.lastSeen,
SampleSubject: sampleSubject,
})
}
return out
}
func (w *perIPWindow) SweepIdle(cutoff time.Time) int {
n := 0
w.states.Range(func(k, v any) bool {
s := v.(*ipState)
s.mu.Lock()
idle := s.lastEvent.Before(cutoff)
s.mu.Unlock()
if idle {
w.states.Delete(k)
n++
}
return true
})
return n
}
type accountState struct {
mu sync.Mutex
events []time.Time
firedAt time.Time
lastEvent time.Time
maxEvents int
}
type perAccountWindow struct {
states sync.Map
cap int
}
func newPerAccountWindow(capPerAccount int) *perAccountWindow {
if capPerAccount <= 0 {
capPerAccount = 5000
}
return &perAccountWindow{cap: capPerAccount}
}
func (w *perAccountWindow) append(user string, at time.Time) {
if user == "" {
return
}
v, _ := w.states.LoadOrStore(user, &accountState{maxEvents: w.cap})
s := v.(*accountState)
s.mu.Lock()
defer s.mu.Unlock()
if len(s.events) >= s.maxEvents {
s.events = s.events[1:]
}
s.events = append(s.events, at)
if at.After(s.lastEvent) {
s.lastEvent = at
}
}
func (w *perAccountWindow) volumeSince(user string, since time.Time) int {
v, ok := w.states.Load(user)
if !ok {
return 0
}
s := v.(*accountState)
s.mu.Lock()
defer s.mu.Unlock()
n := 0
for _, t := range s.events {
if !t.Before(since) {
n++
}
}
return n
}
func (w *perAccountWindow) shouldFire(user string, now time.Time, cooldown time.Duration) bool {
v, ok := w.states.Load(user)
if !ok {
return false
}
s := v.(*accountState)
s.mu.Lock()
defer s.mu.Unlock()
if !s.firedAt.IsZero() && now.Sub(s.firedAt) < cooldown {
return false
}
s.firedAt = now
return true
}
func (w *perAccountWindow) SweepIdle(cutoff time.Time) int {
n := 0
w.states.Range(func(k, v any) bool {
s := v.(*accountState)
s.mu.Lock()
idle := s.lastEvent.Before(cutoff)
s.mu.Unlock()
if idle {
w.states.Delete(k)
n++
}
return true
})
return n
}
// signals is the per-event boolean fingerprint Flow A appends to
// scriptState. The numeric scriptKey lookup happens against the same
// emailspool helpers used elsewhere so subdomain handling is consistent.
type signals struct {
ScriptKey scriptKey
SourceIP string
FromMismatch bool
AdditionalSignal bool
XMailer string
}
// computeSignals resolves the per-event flags for an accepted message.
// authDomains is the cPanel user's authorised domain set (empty on
// resolver error -- caller treats that as "skip From-mismatch contribution"
// by passing isAuthDomainsKnown=false).
func computeSignals(h emailspool.Headers, authDomains map[string]struct{}, pol *emailspool.Policies) signals {
sk, sourceIP := parseXPHPScript(h.XPHPScript)
s := signals{
ScriptKey: sk,
SourceIP: sourceIP,
XMailer: h.XMailer,
}
if len(authDomains) > 0 {
fromDomain := emailspool.ExtractDomain(h.From)
if fromDomain != "" && !IsAuthorisedFromDomain(fromDomain, authDomains) {
s.FromMismatch = true
}
}
// Reply-To external mismatch contribution.
var replyToDomainMismatch bool
if h.ReplyTo != "" && h.From != "" {
rd := emailspool.ExtractDomain(h.ReplyTo)
fd := emailspool.ExtractDomain(h.From)
if rd != "" && fd != "" && rd != fd {
replyToDomainMismatch = true
}
}
// X-Mailer suspicious contribution.
var mailerSuspicious bool
if pol != nil {
if pol.MailerSuspicious(h.XMailer) && !pol.MailerSafe(h.XMailer) {
mailerSuspicious = true
}
}
s.AdditionalSignal = replyToDomainMismatch || mailerSuspicious
return s
}
// parseXPHPScript splits an X-PHP-Script header value into (scriptKey, sourceIP).
// cPanel's EA-PHP header is `<SERVER_NAME><PHP_SELF> for <REMOTE_ADDR>`, or
// `... for <X-Forwarded-For>, <REMOTE_ADDR>` when the client sent
// X-Forwarded-For. Only the last element
// is server-observed; earlier ones are client-supplied. sourceIP is that
// element in canonical form, or "" when it is missing or not an IP.
// Returns ("", "") on an empty header.
func parseXPHPScript(v string) (scriptKey, string) {
v = strings.TrimSpace(v)
if v == "" {
return "", ""
}
forIdx := strings.Index(v, " for ")
var url, ip string
if forIdx > 0 {
// Forwarding text must never become part of the script key. The
// header does not escape delimiters in PHP_SELF, so an ambiguous
// path is grouped under its prefix before the first delimiter.
url = strings.TrimSpace(v[:forIdx])
// Parse the peer independently: PHP_SELF can contain PATH_INFO
// with commas and " for ", even when there is no forwarding list.
list := v[strings.LastIndex(v, " for ")+5:]
last := strings.TrimSpace(list[strings.LastIndexByte(list, ',')+1:])
if parsed := net.ParseIP(last); parsed != nil {
ip = parsed.String()
}
} else {
url = v
}
// Strip any query string.
if q := strings.IndexByte(url, '?'); q > 0 {
url = url[:q]
}
slash := strings.IndexByte(url, '/')
if slash < 0 {
// Bare host with no path.
return scriptKey(url + ":/"), ip
}
host := url[:slash]
path := url[slash:]
return scriptKey(host + ":" + path), ip
}
const (
phpRelayPathCooldown = 30 * time.Minute
phpRelayBreakdownSubjectMax = 160
)
// evaluator combines windows + config + alerter in one object so the
// detector code path is callable from inotify watcher, retro scan, and
// startup spool walker without rebuilding the dependency graph each time.
type evaluator struct {
scripts *perScriptWindow
ips *perIPWindow
accounts *perAccountWindow
cfgFn func() *config.Config // live config; see liveConfigFn
metrics *phpRelayMetrics // optional; nil in unit tests
policies *emailspool.Policies
msgIndex *msgIDIndex // optional; nil in unit tests
effectiveAccountLimit int
accountLimitFn func(*config.Config) int
form *formRelayWindow // Path 6 state
}
func newEvaluator(s *perScriptWindow, i *perIPWindow, a *perAccountWindow, cfg *config.Config, m *phpRelayMetrics) *evaluator {
form := newFormRelayWindow(formRelayMaxIPs)
if m != nil {
form.evictions = m.FormRelayEvictions
}
return &evaluator{scripts: s, ips: i, accounts: a, cfgFn: liveConfigFn(cfg), metrics: m, form: form}
}
// scriptPHPPath returns k's path lowercased and cut after its first PHP component,
// so PATH_INFO cannot change which script the key names.
func scriptPHPPath(k scriptKey) string {
slash := strings.IndexByte(k, '/')
if slash < 0 {
return ""
}
p := strings.ToLower(k[slash:])
for i := 1; i < len(p); i++ {
if p[i] == '/' && contenttype.IsExecutablePHPName(p[:i]) {
return p[:i]
}
}
return p
}
// scriptID names k's script for Path 6 findings and ignores: the host
// lowercased plus scriptPHPPath. Host case and PATH_INFO come from the
// client, so they must neither split one form nor reach an operator hint.
func scriptID(k scriptKey) scriptKey {
slash := strings.IndexByte(k, '/')
if slash < 0 {
return strings.ToLower(k)
}
return strings.ToLower(k[:slash]) + scriptPHPPath(k)
}
// isAuthenticatedAdminScript reports a WordPress admin page that needs a
// login, in any install directory. admin-ajax.php, admin-post.php,
// profile.php and index.php are left out: logged-out visitors or
// subscribers reach them.
func isAuthenticatedAdminScript(k scriptKey) bool {
p := scriptPHPPath(k)
slash := strings.LastIndexByte(p, '/')
if !strings.HasSuffix(p[:slash+1], "/wp-admin/") {
return false
}
file := p[slash+1:]
switch name := strings.TrimSuffix(file, ".php"); name {
case "", file, "admin-ajax", "admin-post", "profile", "index":
return false
}
return true
}
// isWPCronScript reports WordPress's scheduled-job runner, which mails
// whatever its trigger request queued: the recipients are not that
// request's input. Cron services stay countable, so they cannot relay.
func isWPCronScript(k scriptKey) bool {
return strings.HasSuffix(scriptPHPPath(k), "/wp-cron.php")
}
// formRelayDomlogRoot holds cPanel's per-account access logs, one file per
// vhost and scheme. A var so tests can point it at a temporary directory.
var formRelayDomlogRoot = "/usr/local/apache/domlogs"
// formRelayDomlogTail bounds how much of each access log a crossing reads,
// formRelayDomlogLine each line it parses.
const (
formRelayDomlogTail = 4 << 20
formRelayDomlogLine = 64 << 10
)
// formRelayStaffAccounts returns the accounts among shares whose current
// access logs show client key loading a WordPress admin page that needs a
// login: a GET answered 200. Order-list quick actions and other ajax admin
// work mail without passing such a page, so only this evidence spares the
// staff using them. A missing or unreadable log is no evidence.
func formRelayStaffAccounts(key string, shares map[scriptKey]formRelayShare, now time.Time) map[string]time.Time {
found := make(map[string]time.Time)
read := make(map[string]bool)
for id, share := range shares {
user := share.CPUser
host, _, _ := strings.Cut(id, ":/")
if !domlogName(user) || !domlogName(host) {
continue
}
names := []string{host, host + "-ssl_log"}
if bare, ok := strings.CutPrefix(host, "www."); ok && domlogName(bare) {
names = append(names, bare, bare+"-ssl_log")
}
for _, name := range names {
path := filepath.Join(formRelayDomlogRoot, user, name)
if read[path] {
continue
}
read[path] = true
if at := domlogStaffSeenAt(path, key, now); at.After(found[user]) {
found[user] = at
}
}
}
return found
}
// domlogName reports a single path element. The host comes from the
// client's Host header.
func domlogName(s string) bool {
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\x00")
}
// domlogStaffSeenAt scans the last formRelayDomlogTail bytes of path. The
// line the tail starts inside is skipped, and so is any line longer than
// formRelayDomlogLine: a piece of either may be client text shaped like a
// log line.
func domlogStaffSeenAt(path, key string, now time.Time) time.Time {
fi, err := os.Stat(path)
if err != nil || !fi.Mode().IsRegular() {
return time.Time{} // also keeps a FIFO from blocking the open
}
// #nosec G304 -- fixed cPanel log root; both names are checked by domlogName.
f, err := os.Open(path)
if err != nil {
return time.Time{}
}
defer func() { _ = f.Close() }()
off := max(fi.Size()-formRelayDomlogTail, 0)
r := bufio.NewReaderSize(io.NewSectionReader(f, off, fi.Size()-off), formRelayDomlogLine)
skip := off > 0
var latest time.Time
cutoff := now.Add(-formRelayStaffTTL)
for {
line, more, err := r.ReadLine()
for more && err == nil {
skip = true
_, more, err = r.ReadLine()
}
if err == io.EOF {
return latest
}
if err != nil {
return time.Time{}
}
if !skip {
ip, _, _ := strings.Cut(string(line), " ")
if formRelayKey(ip) == key {
rec, ok := checks.ParseCrawlLogLine(string(line), nil)
if ok && rec.TimeOK && !rec.Time.Before(cutoff) && !rec.Time.After(now) &&
rec.Time.After(latest) && isStaffAdminRequest(rec) {
latest = rec.Time
}
}
}
skip = false
}
}
// Log evidence needs a known staff page, not just an apparent admin script:
// public endpoints return 200 too, and servers may normalize the raw path.
// admin.php also serves subscribers; plugin capabilities are not in the log.
func isStaffAdminRequest(rec checks.CrawlLogRecord) bool {
if rec.Method != "GET" || rec.Status != 200 || rec.Target == "" || rec.TargetInvalid || rec.TargetOverflow {
return false
}
path, query, ok := crawlid.SplitTarget(rec.Target)
if !ok || strings.ContainsAny(path, "%\\#") || scriptPHPPath(path) != strings.ToLower(path) {
return false
}
for segment := range strings.SplitSeq(path, "/") {
if segment == "." || segment == ".." {
return false
}
}
slash := strings.LastIndexByte(path, '/')
if !strings.HasSuffix(path[:slash+1], "/wp-admin/") {
return false
}
switch path[slash+1:] {
case "edit.php", "post.php", "post-new.php", "upload.php",
"edit-comments.php", "users.php", "options-general.php", "options-writing.php",
"options-reading.php", "options-discussion.php", "options-media.php",
"options-permalink.php", "options-privacy.php", "plugins.php", "themes.php",
"edit-tags.php", "nav-menus.php", "tools.php", "import.php", "export.php", "site-health.php":
return true
case "admin.php":
return isOrderListQuery(query)
}
return false
}
// isOrderListQuery reports admin.php's WooCommerce HPOS order list, which
// WooCommerce opens only to order managers; every other admin.php page may
// serve subscribers. PHP drops leading spaces and truncates names at NUL;
// the last duplicate wins, so any second name PHP would read as page voids it.
func isOrderListQuery(query string) bool {
// A later page parameter may be discarded by PHP's site-specific input
// limit, leaving a subscriber-accessible admin.php request instead.
first, _, _ := strings.Cut(query, "&")
firstName, _, _ := strings.Cut(first, "=")
if name, err := url.QueryUnescape(firstName); err != nil || name != "page" {
return false
}
q, err := url.ParseQuery(query)
if err != nil {
return false
}
pages := 0
for name, values := range q {
base := strings.TrimLeft(name, " ")
if i := strings.IndexAny(base, "[\x00"); i >= 0 {
base = base[:i]
}
if base == "page" {
pages += len(values)
}
}
page := q.Get("page")
return pages == 1 && len(q["page"]) == 1 && (page == "wc-orders" || strings.HasPrefix(page, "wc-orders--"))
}
// config returns the live config so a reload that retunes or disables the
// relay detector reaches every evaluation; cfg was a startup snapshot before.
func (e *evaluator) config() *config.Config { return e.cfgFn() }
// SetPolicies is called by daemon wiring once the policies file has loaded.
func (e *evaluator) SetPolicies(p *emailspool.Policies) { e.policies = p }
// evaluatePaths inspects the script's window state (and IP window) and
// returns the set of findings that fire at this moment. Cooldowns prevent
// duplicate emissions per (script, path).
func (e *evaluator) evaluatePaths(k scriptKey, sourceIP, cpuser string, now time.Time) []alert.Finding {
cfg := e.config()
relayCfg := cfg.EmailProtection.PHPRelay
if !relayCfg.Enabled {
return nil
}
var findings []alert.Finding
s := e.scripts.getOrCreate(k)
// Path 1: sustained qualifying events.
win := time.Duration(relayCfg.RateWindowMin) * time.Minute
qualifying := s.qualifyingCount(now.Add(-win), func(ev scriptEvent) bool {
return ev.FromMismatch && ev.AdditionalSignal
})
if qualifying >= relayCfg.HeaderScoreVolumeMin {
if s.shouldFire("header", now, phpRelayPathCooldown) {
f := e.makeFinding(k, "header", sourceIP, cpuser, s, fmtHeaderMessage(qualifying, win), now)
f.RelayTotal = qualifying
f.RelayBreakdown = e.scriptRelayBreakdown(k, now.Add(-win), func(ev scriptEvent) bool {
return ev.FromMismatch && ev.AdditionalSignal
})
if e.metrics != nil {
e.metrics.Findings.With("header").Inc()
}
findings = append(findings, f)
}
}
// Path 2: absolute volume per script in the last 60 min.
absVol := s.volumeCount(now.Add(-60 * time.Minute))
if absVol >= relayCfg.AbsoluteVolumePerHour &&
!e.scriptIsLowDiversityNotification(s, now.Add(-60*time.Minute), relayCfg.FanoutDistinctRecipients) {
if s.shouldFire("volume", now, phpRelayPathCooldown) {
f := e.makeFinding(k, "volume", sourceIP, cpuser, s,
fmt.Sprintf("Path 2: %d outbound mails from one script in last 60 min", absVol), now)
f.RelayTotal = absVol
f.RelayBreakdown = e.scriptRelayBreakdown(k, now.Add(-60*time.Minute), func(scriptEvent) bool {
return true
})
if e.metrics != nil {
e.metrics.Findings.With("volume").Inc()
}
findings = append(findings, f)
}
}
// Path 4: HTTP-IP fanout. Skipped silently for proxy IPs.
if sourceIP != "" {
if e.policies == nil || !e.policies.IsProxyIP(sourceIP) {
fwin := time.Duration(relayCfg.FanoutWindowMin) * time.Minute
distinct := e.ips.distinctScriptsSince(sourceIP, now.Add(-fwin))
if distinct >= relayCfg.FanoutDistinctScripts &&
!e.fanoutIsLowDiversityNotification(sourceIP, now.Add(-fwin), relayCfg.FanoutDistinctRecipients) {
if s.shouldFire("fanout", now, phpRelayPathCooldown) {
f := e.makeFinding(k, "fanout", sourceIP, cpuser, s,
fmt.Sprintf("Path 4: HTTP source IP %s triggered %d distinct scripts in last %s", sourceIP, distinct, fwin), now)
f.RelayTotal = distinct
f.RelayBreakdown = e.fanoutRelayBreakdown(sourceIP, now.Add(-fwin))
if e.metrics != nil {
e.metrics.Findings.With("fanout").Inc()
}
findings = append(findings, f)
}
}
}
}
return findings
}
// formRelayFindings emits the client finding once per mode per window when ip's
// outside-recipient count reaches spread, plus one finding per script that
// reached at least two of them, for the operator to fix. Outside enforce
// mode the client finding uses a check that aims no response.
func (e *evaluator) formRelayFindings(k scriptKey, ip, cpuser string, spread int, window time.Duration, now time.Time, ignored func(scriptKey) bool) []alert.Finding {
mode := e.config().EmailProtection.PHPRelay.IPRecipientMode
key := formRelayKey(ip)
since := now.Add(-window)
hours := fmt.Sprintf("%dh", window/time.Hour)
snapshot := e.form.evaluate(key, now, window, spread, ignored, mode, false)
if snapshot.FireClient {
// The logs are read outside the store lock; the claim comes after,
// on what is left once staff accounts drop out.
for user, at := range formRelayStaffAccounts(key, snapshot.Shares, now) {
e.form.markStaff(user, key, at)
}
snapshot = e.form.evaluate(key, now, window, spread, ignored, mode, true)
}
if snapshot.Count < spread {
return nil
}
shares := snapshot.Shares
scripts := make([]scriptKey, 0, len(shares))
for sk := range shares {
scripts = append(scripts, sk)
}
sort.Slice(scripts, func(i, j int) bool {
if a, b := shares[scripts[i]].Count, shares[scripts[j]].Count; a != b {
return a > b
}
return scripts[i] < scripts[j]
})
var out []alert.Finding
if snapshot.FireClient {
var details strings.Builder
details.WriteString("Outside recipients per script:")
var subjects []string
for _, sk := range scripts {
fmt.Fprintf(&details, "\n %s (account %s): %d", sk, shares[sk].CPUser, shares[sk].Count)
if s := e.formRelaySubject(shares[sk].Raw, key, since); s != "" && len(subjects) < 3 && !slices.Contains(subjects, s) {
subjects = append(subjects, s)
}
}
if len(subjects) > 0 {
details.WriteString("\nSample subjects:")
for _, s := range subjects {
fmt.Fprintf(&details, "\n %s", s)
}
}
if domains := snapshot.Domains; len(domains) > 0 {
fmt.Fprintf(&details, "\nRecipient domains: %s", strings.Join(domains, ", "))
}
if e.metrics != nil {
e.metrics.Findings.With("form_relay").Inc()
}
check := "email_php_form_relay"
message := fmt.Sprintf("Path 6: client %s made hosted sites mail %d different outside addresses in %s", ip, snapshot.Count, hours)
if mode != "enforce" {
check = "email_php_form_relay_observed"
message += " (observe mode, not challenged)"
}
out = append(out, alert.Finding{
Severity: alert.High,
Check: check,
Path: "form_relay",
Message: message,
Details: details.String(),
ScriptKey: scriptID(k),
SourceIP: ip,
CPUser: cpuser,
TenantID: checks.HostingAccountForUser(cpuser),
Timestamp: now,
})
}
for _, sk := range scripts {
share := shares[sk]
if share.Count < 2 || !e.form.claimScript(sk, now, window) {
continue
}
if e.metrics != nil {
e.metrics.Findings.With("form_relay_script").Inc()
}
out = append(out, alert.Finding{
Severity: alert.High,
Check: "email_php_form_relay_script",
Path: "form_relay_script",
Message: fmt.Sprintf("Bots use %s to mail outside addresses", sk),
Details: fmt.Sprintf("Client %s reached %d outside addresses through this script in %s.\n"+
"Add a captcha to the form or turn off its automatic reply to the submitter. "+
"If this is a legitimate bulk sender: csm phprelay ignore-script %s", ip, share.Count, hours, shellQuote(sk)),
// The details name the latest client; the script is the identity,
// so later clients update this finding instead of adding one.
DedupKey: string(sk),
ScriptKey: string(sk),
CPUser: share.CPUser,
TenantID: checks.HostingAccountForUser(share.CPUser),
Timestamp: now,
})
}
return out
}
// shellQuote single-quotes s for a POSIX shell, so a pasted hint runs no
// part of s.
func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" }
// formRelaySubject returns the latest subject script k sent for client key.
func (e *evaluator) formRelaySubject(k scriptKey, key string, since time.Time) string {
v, ok := e.scripts.states.Load(k)
if !ok {
return ""
}
hit, _ := v.(*scriptState).relayHit(k, since, func(ev scriptEvent) bool {
return formRelayKey(ev.SourceIP) == key
})
return hit.SampleSubject
}
// scriptIsLowDiversityNotification reports whether a script's hourly volume is
// notification mail rather than relay abuse. A security or e-commerce plugin
// can legitimately emit hundreds of mails an hour to the same one or two
// admin addresses; relay abuse reaches many distinct victims. Unknown or
// partially parsed recipients leave the gate failing open.
func (e *evaluator) scriptIsLowDiversityNotification(s *scriptState, since time.Time, minRcpt int) bool {
if minRcpt <= 0 || s == nil {
return false
}
count, known := s.distinctRecipientsSince(since)
return known && count < minRcpt
}
// fanoutIsLowDiversityNotification reports whether a script fanout from this
// source IP looks like WordPress notification mail (comment moderation, contact
// forms): many distinct scripts but a small fixed recipient set. Mail to each
// sending site's own domains is not counted, so one client commenting on many
// hosted sites does not add up its owners into a relay. It returns true only
// when recipient data is known AND the distinct recipient count is below the
// configured minimum, so Path 4 still fires whenever recipients are diverse
// (real relay) or unknown (recipient parsing gap -- fail open). A non-positive
// threshold disables the gate and preserves the original behavior.
func (e *evaluator) fanoutIsLowDiversityNotification(sourceIP string, since time.Time, minRcpt int) bool {
if minRcpt <= 0 || e.ips == nil {
return false
}
count, known := e.ips.distinctRecipientsSince(sourceIP, since)
return known && count < minRcpt
}
// makeFinding builds a Critical Finding for the given path.
func (e *evaluator) makeFinding(k scriptKey, path, sourceIP, cpuser string, s *scriptState, message string, now time.Time) alert.Finding {
msgIDs, _ := s.snapshotActiveMsgs()
// Cap the sample shown in the finding so the alert payload stays bounded;
// AutoFreezePHPRelayQueue takes its own complete snapshot.
if len(msgIDs) > 10 {
msgIDs = msgIDs[:10]
}
return alert.Finding{
Severity: alert.Critical,
Check: "email_php_relay_abuse",
Path: path,
Message: message,
ScriptKey: string(k),
SourceIP: sourceIP,
CPUser: cpuser,
TenantID: checks.HostingAccountForUser(cpuser),
MsgIDs: msgIDs,
Timestamp: now,
}
}
func (e *evaluator) scriptRelayBreakdown(k scriptKey, since time.Time, match func(scriptEvent) bool) []alert.RelayScriptHit {
hit, ok := e.scripts.getOrCreate(k).relayHit(k, since, match)
if !ok {
return nil
}
return []alert.RelayScriptHit{hit}
}
func (e *evaluator) fanoutRelayBreakdown(sourceIP string, since time.Time) []alert.RelayScriptHit {
if e.ips == nil || sourceIP == "" {
return nil
}
samples := e.ips.relaySamplesSince(sourceIP, since)
if len(samples) == 0 {
return nil
}
out := make([]alert.RelayScriptHit, 0, len(samples))
for _, sample := range samples {
hit := sample
if e.scripts != nil {
if v, ok := e.scripts.states.Load(scriptKey(sample.ScriptKey)); ok {
if counted, ok := v.(*scriptState).relayHit(scriptKey(sample.ScriptKey), since, func(ev scriptEvent) bool {
return ev.SourceIP == sourceIP
}); ok {
hit = counted
}
}
}
out = append(out, hit)
}
sortRelayScriptHits(out)
return out
}
func sortRelayScriptHits(out []alert.RelayScriptHit) {
sort.Slice(out, func(i, j int) bool {
if out[i].Hits != out[j].Hits {
return out[i].Hits > out[j].Hits
}
if !out[i].LastSeen.Equal(out[j].LastSeen) {
return out[i].LastSeen.After(out[j].LastSeen)
}
return out[i].ScriptKey < out[j].ScriptKey
})
}
func fmtHeaderMessage(qualifying int, win time.Duration) string {
return fmt.Sprintf("Path 1: %d qualifying outbound mails (From-mismatch AND suspicious header) in last %s", qualifying, win)
}
type cpanelLimitStatus int
const (
cpanelLimitOK cpanelLimitStatus = iota
cpanelLimitMissing
cpanelLimitUnparsable
cpanelLimitDisabled
)
// readCpanelHourlyLimit returns (parsed-value, status). The key in
// /var/cpanel/cpanel.config is `maxemailsperhour` (no underscores, matches
// internal/checks/hardening_audit.go usage).
//
// OK -> integer > 0; the cap is in force.
// Missing -> file or key absent; caller assumes default 100 + Warning.
// Unparsable -> key present but not a number; caller assumes default 100 + Warning.
// Disabled -> key present and == 0; cpanel hourly limit explicitly off.
func readCpanelHourlyLimit(path string) (int, cpanelLimitStatus) {
// #nosec G304 -- path is the cPanel config path (default /var/cpanel/cpanel.config); operator-controlled, root-owned.
f, err := os.Open(path)
if err != nil {
return 0, cpanelLimitMissing
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
line := sc.Text()
eq := strings.IndexByte(line, '=')
if eq < 0 {
continue
}
if strings.TrimSpace(line[:eq]) != "maxemailsperhour" {
continue
}
val := strings.TrimSpace(line[eq+1:])
n, err := strconv.Atoi(val)
if err != nil {
return 0, cpanelLimitUnparsable
}
if n == 0 {
return 0, cpanelLimitDisabled
}
return n, cpanelLimitOK
}
return 0, cpanelLimitMissing
}
// deriveEffectiveAccountLimit derives the per-account hourly limit from
// cPanel's limit and the operator setting. Returns (effective, enabled, cappedFromOperator).
// - cpanelLimit/status come from readCpanelHourlyLimit.
// - missing/unparsable callers should also emit a Warning startup finding.
// - returned enabled=false means Path 2b should not run this session.
func deriveEffectiveAccountLimit(cfg *config.Config, cpanelLimit int, status cpanelLimitStatus) (effective int, enabled bool, capped bool) {
op := cfg.EmailProtection.PHPRelay.AccountVolumePerHour
// Step 1: classify the cPanel limit.
var assumed int
var known bool
switch status {
case cpanelLimitOK:
assumed = cpanelLimit
known = true
case cpanelLimitMissing, cpanelLimitUnparsable:
// Caller emits Warning; we use the cPanel default 100.
assumed = 100
known = true
case cpanelLimitDisabled:
known = false
}
// Step 2: derive effective.
if known {
cap := assumed * 95 / 100
if cap < 1 {
cap = 1
}
if op == 0 {
target := assumed * 60 / 100
if target < 20 {
target = 20
}
if target > 60 {
target = 60
}
effective = target
if effective > cap {
effective = cap
}
} else {
effective = op
if effective > cap {
effective = cap
capped = true
}
}
if effective <= 0 {
return 0, false, false
}
return effective, true, capped
}
// Cpanel limit explicitly disabled.
if op > 0 {
return op, true, false
}
return 0, false, false
}
const (
phpRelayAccountWindowDur = 60 * time.Minute
phpRelayAccountFireCooldown = 30 * time.Minute
)
// SetEffectiveAccountLimit is called by daemon wiring after derivation.
// Tests may also call it directly. A non-positive value disables Path 2b.
func (e *evaluator) SetEffectiveAccountLimit(n int) {
e.effectiveAccountLimit = n
e.accountLimitFn = nil
}
// SetAccountLimitSource keeps the cPanel limit fixed while deriving the
// operator-controlled threshold from the live config on each evaluation.
func (e *evaluator) SetAccountLimitSource(cpanelLimit int, status cpanelLimitStatus) {
e.accountLimitFn = func(cfg *config.Config) int {
effective, enabled, _ := deriveEffectiveAccountLimit(cfg, cpanelLimit, status)
if !enabled {
return 0
}
return effective
}
e.effectiveAccountLimit = e.accountLimit(e.config())
}
func (e *evaluator) accountLimit(cfg *config.Config) int {
if e.accountLimitFn != nil {
return e.accountLimitFn(cfg)
}
return e.effectiveAccountLimit
}
// SetMsgIndex wires the msgID->script index so exim queue-completion log lines
// can reap the corresponding activeMsgs entry.
func (e *evaluator) SetMsgIndex(idx *msgIDIndex) { e.msgIndex = idx }
// reapCompletedMsg drops an activeMsgs entry when exim logs that its message
// completed delivery and left the queue, so a delivered message is not later
// frozen as if still queued nor counted against the freeze rate-limit budget.
func (e *evaluator) reapCompletedMsg(line string) {
if e.msgIndex == nil || e.scripts == nil {
return
}
msgID, ok := eximCompletedMsgID(line)
if !ok {
return
}
entry, found := e.msgIndex.Get(msgID)
if !found {
return
}
if v, loaded := e.scripts.states.Load(scriptKey(entry.ScriptKey)); loaded {
v.(*scriptState).removeActive(msgID)
}
}
// eximCompletedMsgID returns the message id from an exim queue-completion line
// ("<time> [<zone>] [<pid>] <msgid> Completed[ ...]") and true, or ("", false)
// for any other line. Requiring the "Completed" verb to immediately follow the
// id keeps an attacker-controlled Subject that merely contains the word from
// reaping a live message.
func eximCompletedMsgID(line string) (string, bool) {
p, rest, ok := eximlog.SplitPrefix(line, time.Local)
if !ok || p.MessageID == "" || (rest != "Completed" && !strings.HasPrefix(rest, "Completed ")) {
return "", false
}
return p.MessageID, true
}
// parsePHPRelayAccountVolume processes one outbound `<= ` exim_mainlog line
// seen live. Returns zero or one finding (per cooldown).
func (e *evaluator) parsePHPRelayAccountVolume(line string, now time.Time) []alert.Finding {
// Reap on the live path only: queue-completion lines free the msgID from
// activeMsgs. The retro history replay never sees the live index.
e.reapCompletedMsg(line)
return e.parsePHPRelayAccountVolumeAt(line, now, now)
}
// parsePHPRelayAccountVolumeAt records the account event at eventTime and
// evaluates the volume window relative to now. The live watcher passes
// eventTime == now; the startup history replay passes the line's real exim
// timestamp so days-old log entries are not all stamped "now" and miscounted as
// a single last-hour burst (a false Path 2b Critical on every restart).
func (e *evaluator) parsePHPRelayAccountVolumeAt(line string, eventTime, now time.Time) []alert.Finding {
cfg := e.config()
limit := e.accountLimit(cfg)
if !cfg.EmailProtection.PHPRelay.Enabled || limit <= 0 || e.accounts == nil {
return nil
}
// Only Exim's own submission metadata names the account: a message a
// hosting account's process submitted locally, such as PHP mail().
user := eximlog.LocalSubmitter(line)
if user == "" {
return nil
}
tenant := checks.HostingAccountForUser(user)
if tenant == "" {
return nil
}
if _, sender, ok := eximlog.Arrival(line); !ok || sender == "<>" {
return nil
}
if !sendsOffHost(eximlog.Recipients(line)) {
return nil
}
e.accounts.append(user, eventTime)
volume := e.accounts.volumeSince(user, now.Add(-phpRelayAccountWindowDur))
if volume < limit {
return nil
}
if !e.accounts.shouldFire(user, now, phpRelayAccountFireCooldown) {
return nil
}
if e.metrics != nil {
e.metrics.Findings.With("volume_account").Inc()
}
return []alert.Finding{{
Severity: alert.Critical,
Check: "email_php_relay_abuse",
Path: "volume_account",
Message: fmt.Sprintf("Path 2b: account %s sent >= %d outbound mails in last hour", user, limit),
CPUser: user,
TenantID: tenant,
RelayTotal: volume,
Timestamp: now,
}}
}
// sendsOffHost reports whether any recipient is outside this host's mail
// domains. Mail to the host's own mailboxes is not outbound volume, and with
// the domain list unreadable nothing counts rather than everything.
func sendsOffHost(recipients []string) bool {
for _, rcpt := range recipients {
// Exim can cut the final address mid-domain when its log buffer
// fills. That fragment cannot prove an off-host destination.
if strings.HasSuffix(rcpt, "<trunc>") {
continue
}
at := strings.LastIndexByte(rcpt, '@')
if at < 0 || at == len(rcpt)-1 {
continue
}
local, known := phpRelayIsLocalDomain(strings.ToLower(rcpt[at+1:]))
if !known {
return false
}
if !local {
return true
}
}
return false
}
// ignoreEntry records an operator-issued ignore on a script. A zero
// ExpiresAt means "never expires"; otherwise Has/List/SweepExpired drop
// the entry once now > ExpiresAt.
type ignoreEntry struct {
ScriptKey string
AddedAt time.Time
ExpiresAt time.Time
AddedBy string
Reason string
}
// ignoreList is the in-memory operator allowlist for php_relay scripts.
// L2 (bbolt persistence) wraps this with --persist semantics; Flow E
// (O2) calls SweepExpired periodically.
type ignoreList struct {
mu sync.Mutex
entries map[string]ignoreEntry
db *store.DB
}
func newIgnoreList() *ignoreList {
return &ignoreList{entries: make(map[string]ignoreEntry)}
}
func (il *ignoreList) Add(k scriptKey, expiresAt time.Time, by, reason string) {
il.mu.Lock()
defer il.mu.Unlock()
il.entries[string(k)] = ignoreEntry{
ScriptKey: string(k),
AddedAt: time.Now(),
ExpiresAt: expiresAt,
AddedBy: by,
Reason: reason,
}
}
func (il *ignoreList) Remove(k scriptKey) {
il.mu.Lock()
delete(il.entries, string(k))
il.mu.Unlock()
}
func (il *ignoreList) Has(k scriptKey) bool {
il.mu.Lock()
defer il.mu.Unlock()
e, ok := il.entries[string(k)]
if !ok {
return false
}
if !e.ExpiresAt.IsZero() && time.Now().After(e.ExpiresAt) {
delete(il.entries, string(k))
return false
}
return true
}
// Covers reports whether k is ignored under its own key or its scriptID,
// so one ignore on the name a Path 6 finding prints covers every variant.
func (il *ignoreList) Covers(k scriptKey) bool { return il.Has(k) || il.Has(scriptID(k)) }
func (il *ignoreList) List() []ignoreEntry {
il.mu.Lock()
defer il.mu.Unlock()
out := make([]ignoreEntry, 0, len(il.entries))
now := time.Now()
for k, e := range il.entries {
if !e.ExpiresAt.IsZero() && now.After(e.ExpiresAt) {
delete(il.entries, k)
continue
}
out = append(out, e)
}
return out
}
// SweepExpired drops expired entries. Called by Flow E ticker.
//
//nolint:unused // wired in O2 Flow E ticker
func (il *ignoreList) SweepExpired(now time.Time) int {
il.mu.Lock()
defer il.mu.Unlock()
n := 0
for k, e := range il.entries {
if !e.ExpiresAt.IsZero() && now.After(e.ExpiresAt) {
delete(il.entries, k)
n++
}
}
return n
}
const ignoreBucket = "phprelay:ignore"
func (il *ignoreList) SetStore(db *store.DB) { il.db = db }
func (il *ignoreList) AddPersist(k scriptKey, expiresAt time.Time, by, reason string) error {
il.Add(k, expiresAt, by, reason)
if il.db == nil {
return nil
}
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(ignoreEntry{
ScriptKey: string(k), AddedAt: time.Now(),
ExpiresAt: expiresAt, AddedBy: by, Reason: reason,
}); err != nil {
return err
}
return il.db.PHPRelayPut(ignoreBucket, string(k), buf.Bytes())
}
//nolint:unused // wired in M2/O2
func (il *ignoreList) RemovePersist(k scriptKey) error {
il.Remove(k)
if il.db == nil {
return nil
}
return il.db.PHPRelayDelete(ignoreBucket, string(k))
}
// Restore re-populates the in-memory list from bbolt at daemon start.
// Corrupt rows are skipped silently; expired rows are skipped (the bbolt
// row stays put until SweepBolt prunes it on the next Flow E tick).
func (il *ignoreList) Restore() error {
if il.db == nil {
return nil
}
rows, err := il.db.PHPRelayList(ignoreBucket)
if err != nil {
return err
}
now := time.Now()
for _, raw := range rows {
var e ignoreEntry
if err := gob.NewDecoder(bytes.NewReader(raw)).Decode(&e); err != nil {
continue
}
if !e.ExpiresAt.IsZero() && now.After(e.ExpiresAt) {
continue
}
il.Add(scriptKey(e.ScriptKey), e.ExpiresAt, e.AddedBy, e.Reason)
}
return nil
}
// SweepBolt drops expired bbolt entries on the Flow E ticker. Corrupt
// rows are also dropped so the bucket stays healthy.
//
//nolint:unused // wired in M2/O2
func (il *ignoreList) SweepBolt(now time.Time) (int, error) {
if il.db == nil {
return 0, nil
}
return il.db.PHPRelaySweep(ignoreBucket, func(_, value []byte) bool {
var e ignoreEntry
if err := gob.NewDecoder(bytes.NewReader(value)).Decode(&e); err != nil {
return true // drop corrupt rows
}
return !e.ExpiresAt.IsZero() && now.After(e.ExpiresAt)
})
}
package daemon
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/emailspool"
"github.com/pidginhost/csm/internal/systemdrun"
)
// msgIDPattern guards exim -Mf invocations against header-injected garbage
// that slipped past parseHeaders. Exim msgIDs are <= 23 chars in practice
// but we accept up to 32 to allow for future format changes; the lower
// bound of 16 rules out any short string an attacker could try to slip in.
var msgIDPattern = regexp.MustCompile(`^[A-Za-z0-9-]{16,32}$`)
// eximBinary is resolved at module init via exec.LookPath. Empty means
// auto-action is permanently disabled (a Warning finding is emitted at
// startup -- see Phase O).
//
//nolint:unused // populated in K5 by AutoFreezePHPRelayQueue init via exec.LookPath
var eximBinary string
// actionRateLimiter is a sliding-window counter of exim -M* invocations.
// At most maxPerMinute actions in any rolling 60s window.
type actionRateLimiter struct {
mu sync.Mutex
maxPerMin int
bucket int
refilledAt time.Time
now func() time.Time
}
func newActionRateLimiter(maxPerMin int) *actionRateLimiter {
return &actionRateLimiter{
maxPerMin: maxPerMin,
bucket: maxPerMin,
now: time.Now,
}
}
// consumeUpTo grants up to n tokens and returns how many were available and
// consumed (0..n). Partial grants let a real outbreak freeze as many messages
// as the per-minute budget allows instead of freezing none the moment the batch
// size exceeds the cap.
func (rl *actionRateLimiter) consumeUpTo(n int) int {
rl.mu.Lock()
defer rl.mu.Unlock()
return rl.consumeUpToLocked(n, rl.maxPerMin)
}
func (rl *actionRateLimiter) consumeUpToLimit(n, maxPerMin int) int {
rl.mu.Lock()
defer rl.mu.Unlock()
return rl.consumeUpToLocked(n, maxPerMin)
}
func (rl *actionRateLimiter) consumeUpToLocked(n, maxPerMin int) int {
if maxPerMin <= 0 {
maxPerMin = 60
}
now := rl.now()
if maxPerMin != rl.maxPerMin {
used := rl.maxPerMin - rl.bucket
rl.maxPerMin = maxPerMin
rl.bucket = maxPerMin - used
if rl.bucket < 0 {
rl.bucket = 0
}
}
if rl.refilledAt.IsZero() || now.Sub(rl.refilledAt) >= time.Minute {
rl.bucket = rl.maxPerMin
rl.refilledAt = now
}
if n <= 0 {
return 0
}
grant := n
if grant > rl.bucket {
grant = rl.bucket
}
rl.bucket -= grant
return grant
}
// freezeErrIsAlreadyGone matches the Exim stderr fragments emitted when
// the message has already left the queue between snapshot and freeze.
// Those are not action failures -- they are normal queue churn.
func freezeErrIsAlreadyGone(stderr string) bool {
s := strings.ToLower(stderr)
return strings.Contains(s, "message not found") ||
strings.Contains(s, "spool file not found") ||
strings.Contains(s, "no such message")
}
// spoolScanMatchingScript walks every -H file under spoolRoot, parses
// headers, and returns msgIDs whose X-PHP-Script host:path matches
// scriptKey. Used by AutoFreezePHPRelayQueue when activeMsgs was capped
// or when a late reputation finding has no in-memory activeMsgs left.
//
// Handles BOTH spool layouts:
//
// 1. Split (cPanel default + Exim's split_spool_directory=true): each
// msgID-H lives under spoolRoot/<hash-char>/. We must descend one
// level into each subdir.
// 2. Unsplit (some self-hosted Exim builds, smaller cPanel installs
// where the operator has disabled split_spool_directory): -H files
// live directly in spoolRoot.
//
// Both layouts occur in the field. We probe each
// entry: if it's a regular -H file at the root, scan it; if it's a
// directory, descend. No probing of /etc/exim or spool config -- the
// filesystem layout is the source of truth.
func spoolScanMatchingScript(spoolRoot string, k scriptKey) []string {
var out []string
// #nosec G304 -- spoolRoot is operator-configured / hardcoded to cPanel default.
entries, err := os.ReadDir(spoolRoot)
if err != nil {
return nil
}
inspect := func(full string, name string) {
if !strings.HasSuffix(name, "-H") {
return
}
h, err := emailspool.ParseHeaders(full)
if err != nil || h.XPHPScript == "" {
return
}
sk, _ := parseXPHPScript(h.XPHPScript)
if sk != k {
return
}
id := strings.TrimSuffix(name, "-H")
if msgIDPattern.MatchString(id) {
out = append(out, id)
}
}
for _, e := range entries {
full := filepath.Join(spoolRoot, e.Name())
if e.IsDir() {
// Split layout: descend one level.
// #nosec G304 -- spoolRoot is operator-configured / hardcoded to cPanel default.
files, err := os.ReadDir(full)
if err != nil {
continue
}
for _, f := range files {
inspect(filepath.Join(full, f.Name()), f.Name())
}
continue
}
// Unsplit layout: -H files at the root of spoolRoot.
inspect(full, e.Name())
}
return out
}
// runner abstracts exec.CommandContext so tests can inject a stub.
type runner interface {
Run(ctx context.Context, bin string, args []string) (stderr string, err error)
}
type defaultRunner struct{}
var (
defaultRunnerLookPath = exec.LookPath
defaultRunnerCommand = func(ctx context.Context, name string, args ...string) ([]byte, error) {
// #nosec G204 -- name is either the resolved systemd-run path or the
// resolved Exim binary; args are fixed flags plus validated message IDs.
cmd := exec.CommandContext(ctx, name, args...)
// Only stderr is returned: the freeze audit log records this under
// "stderr" and operators read it as the failure reason. --pipe connects
// the transient unit's stderr to ours, so wrapping does not change what
// is captured.
var stderr bytes.Buffer
cmd.Stderr = &stderr
err := cmd.Run()
return stderr.Bytes(), err
}
)
func (defaultRunner) Run(ctx context.Context, bin string, args []string) (string, error) {
opt := systemdrun.Options{Pipe: true}
if deadline, ok := ctx.Deadline(); ok {
opt.RuntimeMax = time.Until(deadline)
}
out, err := systemdrun.Run(ctx, defaultRunnerLookPath, defaultRunnerCommand, opt, bin, args...)
return string(out), err
}
// auditEntry is the per-action record written by the auditor. K6 will add a
// JSONL serialiser; K5 only constructs the in-memory shape.
type auditEntry struct {
Ts time.Time
MsgID string
ScriptKey string
Path string
DryRun bool
Exit int
Stderr string
Action string // "freeze" | "thaw" | "freeze_dry_run"
}
type auditor interface {
Write(e auditEntry)
}
// autoFreezer holds the wiring needed to translate findings into exim -Mf
// invocations. Constructed once at daemon start; Apply is invoked per
// post-emit AutoResponse pass.
//
// dryRunFn returns the EFFECTIVE dry-run state from the same config snapshot
// Apply uses for enablement and rate limits. The CLI's runtime override,
// bbolt override and csm.yaml fallback are resolved by the controller so
// `csm phprelay dry-run on|off|reset` changes freeze behaviour immediately.
//
//nolint:unused // wired in O2 by daemon controller
type autoFreezer struct {
scripts *perScriptWindow
cfgFn func() *config.Config // live config; see liveConfigFn
spoolRoot string
eximBin string
runner runner
auditor auditor
rateLim *actionRateLimiter
metrics *phpRelayMetrics
dryRunFn func(*config.Config) bool
}
// config returns the live config so a reload that disables auto-response or
// the freeze action stops the next Apply; cfg was a startup snapshot before.
func (a *autoFreezer) config() *config.Config { return a.cfgFn() }
//nolint:unused // wired in O2 by daemon controller
func newAutoFreezer(scripts *perScriptWindow, cfg *config.Config, spoolRoot, eximBin string, r runner, a auditor, m *phpRelayMetrics, dryRunFn func(*config.Config) bool) *autoFreezer {
if r == nil {
r = defaultRunner{}
}
rl := newActionRateLimiter(cfg.AutoResponse.PHPRelay.MaxActionsPerMinute)
if cfg.AutoResponse.PHPRelay.MaxActionsPerMinute <= 0 {
rl = newActionRateLimiter(60)
}
if dryRunFn == nil {
// Defensive default: if no resolver wired, fall back to the safe
// YAML-level dry-run state (PHPRelayDryRunEnabled defaults to TRUE).
dryRunFn = func(liveCfg *config.Config) bool { return liveCfg.PHPRelayDryRunEnabled() }
}
return &autoFreezer{
scripts: scripts, cfgFn: liveConfigFn(cfg), spoolRoot: spoolRoot, eximBin: eximBin,
runner: r, auditor: a, rateLim: rl, metrics: m, dryRunFn: dryRunFn,
}
}
// Apply iterates findings, snapshots each script's activeMsgs, optionally
// extends with a spool-scan fallback, and freezes via exim -Mf. Returns
// any new findings produced (Warning/Critical for action outcomes). Pure
// from the perspective of finding emission -- caller forwards them to the
// alert pipeline.
//
//nolint:unused // wired in O2 by daemon controller
func (a *autoFreezer) Apply(findings []alert.Finding) []alert.Finding {
var emitted []alert.Finding
cfg := a.config()
if !cfg.AutoResponse.Enabled || !cfg.PHPRelayFreezeEnabled() {
return nil
}
if a.eximBin == "" {
return nil
}
dryRun := a.dryRunFn(cfg)
for _, f := range findings {
if f.Check != "email_php_relay_abuse" {
continue
}
if !canActOnPath(f.Path) {
emitted = append(emitted, alert.Finding{
Severity: alert.Warning,
Check: "email_php_relay_action_skipped",
Path: f.Path,
Message: fmt.Sprintf("AutoFreeze skipped: path %q has no scriptKey", f.Path),
Timestamp: time.Now(),
})
continue
}
s := a.scripts.getOrCreate(scriptKey(f.ScriptKey))
ids, capped := s.snapshotActiveMsgs()
if capped || (len(ids) == 0 && f.Path == "reputation") {
if a.metrics != nil {
if capped {
a.metrics.SpoolScanFallbacks.With("capped").Inc()
} else {
a.metrics.SpoolScanFallbacks.With("reputation").Inc()
}
}
extra := spoolScanMatchingScript(a.spoolRoot, scriptKey(f.ScriptKey))
ids = unionStrings(ids, extra)
}
if len(ids) == 0 {
continue
}
if dryRun {
for _, id := range ids {
a.auditor.Write(auditEntry{
Ts: time.Now(), MsgID: id, ScriptKey: f.ScriptKey,
Path: f.Path, DryRun: true, Action: "freeze_dry_run",
})
}
emitted = append(emitted, alert.Finding{
Severity: alert.Warning,
Check: "email_php_relay_action_dry_run",
Path: f.Path,
Message: fmt.Sprintf("AutoFreeze dry-run: would freeze %d msgs from %s", len(ids), f.ScriptKey),
ScriptKey: f.ScriptKey,
Timestamp: time.Now(),
})
continue
}
requested := len(ids)
grant := a.rateLim.consumeUpToLimit(requested, cfg.AutoResponse.PHPRelay.MaxActionsPerMinute)
if grant < requested {
emitted = append(emitted, alert.Finding{
Severity: alert.Warning,
Check: "email_php_relay_rate_limit_hit",
Path: f.Path,
Message: fmt.Sprintf("AutoFreeze rate limit deferred %d of %d freezes for %s", requested-grant, requested, f.ScriptKey),
ScriptKey: f.ScriptKey,
Timestamp: time.Now(),
})
}
if grant == 0 {
continue
}
ids = ids[:grant]
var failed []string
for _, id := range ids {
if !msgIDPattern.MatchString(id) {
continue
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
stderr, err := a.runner.Run(ctx, a.eximBin, []string{"-Mf", id})
cancel()
entry := auditEntry{
Ts: time.Now(), MsgID: id, ScriptKey: f.ScriptKey,
Path: f.Path, DryRun: false, Stderr: stderr, Action: "freeze",
}
if err != nil {
if freezeErrIsAlreadyGone(stderr) {
if a.metrics != nil {
a.metrics.ActionGone.Inc()
}
a.auditor.Write(entry)
continue
}
entry.Exit = 1
failed = append(failed, id)
if a.metrics != nil {
a.metrics.Actions.With("freeze", "fail").Inc()
}
}
a.auditor.Write(entry)
// On successful freeze, drop the id from activeMsgs so we
// don't re-freeze it on the next finding emission.
if err == nil {
if a.metrics != nil {
a.metrics.Actions.With("freeze", "ok").Inc()
}
s.removeActive(id)
}
}
if len(failed) > 0 {
emitted = append(emitted, alert.Finding{
Severity: alert.Critical,
Check: "email_php_relay_action_failed",
Path: f.Path,
Message: fmt.Sprintf("exim -Mf failed for %d msgs from %s", len(failed), f.ScriptKey),
ScriptKey: f.ScriptKey,
MsgIDs: failed,
Timestamp: time.Now(),
})
}
}
return emitted
}
// canActOnPath reports whether AutoFreeze can act on a finding's Path.
// volume_account fires per-cpuser without scriptKey; baseline / reputation /
// header / volume / fanout all carry scriptKey.
//
//nolint:unused // wired in O2 by daemon controller
func canActOnPath(p string) bool {
switch p {
case "header", "volume", "fanout", "baseline", "reputation":
return true
}
return false
}
//nolint:unused // wired in O2 by daemon controller
func unionStrings(a, b []string) []string {
seen := make(map[string]struct{}, len(a)+len(b))
out := make([]string, 0, len(a)+len(b))
for _, s := range append(a, b...) {
if _, ok := seen[s]; ok {
continue
}
seen[s] = struct{}{}
out = append(out, s)
}
return out
}
type structuredAuditor struct {
mu sync.Mutex
w io.Writer
}
func newStructuredAuditor(w io.Writer) *structuredAuditor { return &structuredAuditor{w: w} }
func (a *structuredAuditor) Write(e auditEntry) {
payload := struct {
Ts time.Time `json:"ts"`
MsgID string `json:"msg_id"`
ScriptKey string `json:"script_key"`
Path string `json:"path"`
Action string `json:"action"`
DryRun bool `json:"dry_run"`
Exit int `json:"exit"`
Stderr string `json:"stderr,omitempty"`
}{
Ts: e.Ts.UTC(), MsgID: e.MsgID, ScriptKey: e.ScriptKey,
Path: e.Path, Action: e.Action, DryRun: e.DryRun,
Exit: e.Exit, Stderr: e.Stderr,
}
line, err := json.Marshal(payload)
if err != nil {
return
}
a.mu.Lock()
defer a.mu.Unlock()
_, _ = a.w.Write(line)
_, _ = a.w.Write([]byte("\n"))
}
package daemon
import (
"io"
"os"
"sync"
"github.com/pidginhost/csm/internal/config"
)
// eximAuditWriterAt returns the writer used by the structured JSONL auditor.
// When auto-freeze is disabled, it returns a lazy writer so daemon startup
// does not create a 0-byte orphan log file, while manual thaw commands can
// still create the file on their first real audit entry. When auto-freeze is
// enabled, the file is opened at startup to preserve the existing live-action
// failure visibility.
func eximAuditWriterAt(cfg *config.Config, path string) io.Writer {
if cfg == nil || !cfg.PHPRelayFreezeEnabled() {
return &lazyEximAuditWriter{path: path}
}
return openEximAuditWriterAt(path)
}
type lazyEximAuditWriter struct {
mu sync.Mutex
path string
w io.Writer
}
func (w *lazyEximAuditWriter) Write(p []byte) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()
if w.w == nil {
w.w = openEximAuditWriterAt(w.path)
}
return w.w.Write(p)
}
func openEximAuditWriterAt(path string) io.Writer {
// #nosec G304 G302 -- G304: path is the compile-time constant phpRelayAuditPath in production; tests pass t.TempDir-derived paths. G302: 0640 is intentional; SIEM log shippers (Vector, Filebeat, Fluentbit) commonly run as a non-root user that needs group-read access. 0600 would force the shipper to run as root. Same rationale as internal/alert/audit_jsonl.go.
if f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640); err == nil {
return f
}
return os.Stderr
}
package daemon
import (
"context"
"encoding/json"
"errors"
"fmt"
"sync"
"time"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/store"
)
// PHPRelayController aggregates the phprelay-state references the
// control-socket handlers need. One instance per running daemon;
// constructed in daemon.New (Phase O2) and assigned to ControlListener.phprelay.
//
// The spool-pipeline reference (linux-only spoolPipeline) is intentionally
// not held here so this file stays cross-platform. Phase O2 can attach
// the pipeline through a separate linux-gated wiring file if needed.
type PHPRelayController struct {
eng *evaluator
msgIndex *msgIDIndex
ignores *ignoreList
actionDryRun *runtimeBool
db *store.DB
runner runner
eximBin string
auditor auditor
enabled bool
platform string
}
// runtimeBool is the in-memory dry-run override; effective-value precedence
// resolves CLI > bbolt > csm.yaml at read time.
type runtimeBool struct {
mu sync.Mutex
set bool
value bool
}
func (r *runtimeBool) Set(v bool) { r.mu.Lock(); r.set = true; r.value = v; r.mu.Unlock() }
func (r *runtimeBool) Reset() { r.mu.Lock(); r.set = false; r.mu.Unlock() }
func (r *runtimeBool) Get() (value, set bool) {
r.mu.Lock()
defer r.mu.Unlock()
return r.value, r.set
}
// effectiveDryRun resolves precedence: runtime > bbolt > csm.yaml.
// Returns (effective, source) where source identifies the winning input.
func (c *PHPRelayController) effectiveDryRun() (bool, string) {
var cfg *config.Config
if c.eng != nil {
cfg = c.eng.config()
}
return c.effectiveDryRunForConfig(cfg)
}
func (c *PHPRelayController) effectiveDryRunForConfig(cfg *config.Config) (bool, string) {
if v, set := c.actionDryRun.Get(); set {
return v, "runtime"
}
if c.db != nil {
if v, ok, err := readDryRunOverride(c.db); err == nil && ok {
return v, "bbolt"
}
}
if cfg != nil {
return cfg.PHPRelayDryRunEnabled(), "csm.yaml"
}
return true, "default"
}
// Status returns a snapshot of detector state for `csm phprelay status`.
func (c *PHPRelayController) Status(_ context.Context, _ control.PHPRelayStatusRequest) (control.PHPRelayStatusResponse, error) {
cfg := c.eng.config()
resp := control.PHPRelayStatusResponse{
Enabled: c.enabled,
Platform: c.platform,
EffectiveAccountLimit: c.eng.accountLimit(cfg),
IgnoresActive: len(c.ignores.List()),
RecentFindings: map[string]int{}, // populated by metrics in Phase N
}
eff, _ := c.effectiveDryRunForConfig(cfg)
resp.DryRun = eff
if c.eng.scripts != nil {
resp.ScriptsTracked = len(c.eng.scripts.Snapshot())
}
if c.msgIndex != nil {
resp.MsgIDIndexSize = c.msgIndex.Len()
}
return resp, nil
}
// handlePHPRelayStatus is the dispatcher-side adapter that bridges the
// json.RawMessage args to the typed Status method.
func (c *ControlListener) handlePHPRelayStatus(argsRaw json.RawMessage) (any, error) {
if c.phprelay == nil {
return nil, c.phpRelayUnavailable()
}
var req control.PHPRelayStatusRequest
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &req); err != nil {
return nil, fmt.Errorf("bad args: %w", err)
}
}
return c.phprelay.Status(context.Background(), req)
}
func (c *PHPRelayController) IgnoreScript(_ context.Context, req control.PHPRelayIgnoreScriptRequest) (control.PHPRelayIgnoreScriptResponse, error) {
if req.ScriptKey == "" {
return control.PHPRelayIgnoreScriptResponse{}, errors.New("script_key required")
}
hours := req.ForHours
if hours == 0 {
hours = 24 * 7
}
expires := time.Now().Add(time.Duration(hours) * time.Hour)
by := req.AddedBy
if by == "" {
by = "operator"
}
if req.Persist {
if err := c.ignores.AddPersist(scriptKey(req.ScriptKey), expires, by, req.Reason); err != nil {
return control.PHPRelayIgnoreScriptResponse{}, err
}
} else {
c.ignores.Add(scriptKey(req.ScriptKey), expires, by, req.Reason)
}
return control.PHPRelayIgnoreScriptResponse{ExpiresAt: expires}, nil
}
func (c *PHPRelayController) Unignore(_ context.Context, req control.PHPRelayUnignoreRequest) (struct{}, error) {
if req.ScriptKey == "" {
return struct{}{}, errors.New("script_key required")
}
if req.Persist {
if err := c.ignores.RemovePersist(scriptKey(req.ScriptKey)); err != nil {
return struct{}{}, err
}
} else {
c.ignores.Remove(scriptKey(req.ScriptKey))
}
return struct{}{}, nil
}
func (c *PHPRelayController) IgnoreList(_ context.Context, _ struct{}) (control.PHPRelayIgnoreListResponse, error) {
raw := c.ignores.List()
out := make([]control.PHPRelayIgnoreEntry, 0, len(raw))
for _, e := range raw {
out = append(out, control.PHPRelayIgnoreEntry{
ScriptKey: e.ScriptKey, ExpiresAt: e.ExpiresAt,
AddedBy: e.AddedBy, Reason: e.Reason,
})
}
return control.PHPRelayIgnoreListResponse{Entries: out}, nil
}
func (c *ControlListener) handlePHPRelayIgnoreScript(argsRaw json.RawMessage) (any, error) {
if c.phprelay == nil {
return nil, c.phpRelayUnavailable()
}
var req control.PHPRelayIgnoreScriptRequest
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &req); err != nil {
return nil, fmt.Errorf("bad args: %w", err)
}
}
return c.phprelay.IgnoreScript(context.Background(), req)
}
func (c *ControlListener) handlePHPRelayUnignore(argsRaw json.RawMessage) (any, error) {
if c.phprelay == nil {
return nil, c.phpRelayUnavailable()
}
var req control.PHPRelayUnignoreRequest
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &req); err != nil {
return nil, fmt.Errorf("bad args: %w", err)
}
}
return c.phprelay.Unignore(context.Background(), req)
}
func (c *ControlListener) handlePHPRelayIgnoreList(argsRaw json.RawMessage) (any, error) {
if c.phprelay == nil {
return nil, c.phpRelayUnavailable()
}
_ = argsRaw // no args expected
return c.phprelay.IgnoreList(context.Background(), struct{}{})
}
func (c *PHPRelayController) DryRun(_ context.Context, req control.PHPRelayDryRunRequest) (control.PHPRelayDryRunResponse, error) {
switch req.Mode {
case "on":
c.actionDryRun.Set(true)
if req.Persist {
if err := writeDryRunOverride(c.db, true, "operator"); err != nil {
return control.PHPRelayDryRunResponse{}, err
}
}
case "off":
c.actionDryRun.Set(false)
if req.Persist {
if err := writeDryRunOverride(c.db, false, "operator"); err != nil {
return control.PHPRelayDryRunResponse{}, err
}
}
case "reset":
c.actionDryRun.Reset()
if req.Persist {
if err := deleteDryRunOverride(c.db); err != nil {
return control.PHPRelayDryRunResponse{}, err
}
}
default:
return control.PHPRelayDryRunResponse{}, errors.New("mode must be on|off|reset")
}
eff, src := c.effectiveDryRun()
return control.PHPRelayDryRunResponse{Effective: eff, Source: src}, nil
}
// DryRunFn evaluates the precedence chain against the operation's config
// snapshot. Daemon wiring passes it to newAutoFreezer so `csm phprelay
// dry-run` changes freeze behaviour without rebuilding the freezer.
func (c *PHPRelayController) DryRunFn() func(*config.Config) bool {
return func(cfg *config.Config) bool {
v, _ := c.effectiveDryRunForConfig(cfg)
return v
}
}
func (c *ControlListener) handlePHPRelayDryRun(argsRaw json.RawMessage) (any, error) {
if c.phprelay == nil {
return nil, c.phpRelayUnavailable()
}
var req control.PHPRelayDryRunRequest
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &req); err != nil {
return nil, fmt.Errorf("bad args: %w", err)
}
}
return c.phprelay.DryRun(context.Background(), req)
}
// Thaw runs `exim -Mt <msg_id>` to release a frozen message back to the
// queue. msgIDPattern validation guards against header-injected garbage
// even though only operators can hit this endpoint. The audit entry is
// written for both success and failure so an operator can later prove
// what was thawed.
//
// req.By is accepted on the wire for forward compatibility (future
// auditEntry.By field) but is not used by the M4 handler.
func (c *PHPRelayController) Thaw(ctx context.Context, req control.PHPRelayThawRequest) (control.PHPRelayThawResponse, error) {
if !msgIDPattern.MatchString(req.MsgID) {
return control.PHPRelayThawResponse{}, errors.New("invalid msg_id")
}
sub, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
stderr, err := c.runner.Run(sub, c.eximBin, []string{"-Mt", req.MsgID})
c.auditor.Write(auditEntry{
Ts: time.Now(), MsgID: req.MsgID, Action: "thaw",
Stderr: stderr,
})
if err != nil {
return control.PHPRelayThawResponse{Stderr: stderr}, err
}
return control.PHPRelayThawResponse{Stderr: stderr}, nil
}
func (c *ControlListener) handlePHPRelayThaw(argsRaw json.RawMessage) (any, error) {
if c.phprelay == nil {
return nil, c.phpRelayUnavailable()
}
var req control.PHPRelayThawRequest
if len(argsRaw) > 0 {
if err := json.Unmarshal(argsRaw, &req); err != nil {
return nil, fmt.Errorf("bad args: %w", err)
}
}
return c.phprelay.Thaw(context.Background(), req)
}
package daemon
import (
"os"
"strings"
"sync"
"time"
)
// phpRelayIsLocalDomain reports whether domain is one of this host's mail
// domains. known is false when the domain inventory could not be read, so callers
// that tell outbound from local mail can stay quiet instead of guessing.
var phpRelayIsLocalDomain = phpRelayLocalDomains.has
var phpRelayLocalDomains = &localDomainCache{ttl: time.Minute}
// Tests redirect these system files into a temporary directory.
var localDomainPaths = []string{"/etc/localdomains", "/etc/virtualdomains"}
// localDomainCache rereads the host's domain files at most once per ttl, so a
// per-message check does not read them for every log line.
type localDomainCache struct {
mu sync.Mutex
ttl time.Duration
loadedAt time.Time
domains map[string]bool
known bool
}
func (c *localDomainCache) has(domain string) (local, known bool) {
c.mu.Lock()
defer c.mu.Unlock()
if c.loadedAt.IsZero() || time.Since(c.loadedAt) >= c.ttl {
c.domains, c.known = readLocalDomains()
c.loadedAt = time.Now()
}
return c.domains[strings.ToLower(domain)], c.known
}
// readLocalDomains reads /etc/localdomains and /etc/virtualdomains. Missing
// optional files are normal, but a read failure can hide local domains.
// ok requires at least one readable file and no failures on present files.
func readLocalDomains() (domains map[string]bool, ok bool) {
domains = make(map[string]bool)
failed := false
for _, path := range localDomainPaths {
// #nosec G304 -- paths are fixed system files, redirected only by tests.
data, err := os.ReadFile(path)
if err != nil {
if !os.IsNotExist(err) {
failed = true
}
continue
}
ok = true
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if idx := strings.IndexByte(line, ':'); idx > 0 {
line = strings.TrimSpace(line[:idx])
}
domains[strings.ToLower(line)] = true
}
}
return domains, ok && !failed
}
package daemon
import (
"net/netip"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/metrics"
)
const (
formRelayMaxIPs = 20000
formRelayStaffTTL = 7 * 24 * time.Hour
formRelayStaffPerAccount = 64
)
// formRelayUnparsed prefixes the entry an unparsable message adds, keyed by
// its message ID. A parsed address never holds a NUL, so it cannot collide.
const formRelayUnparsed = "\x00"
// formRelayWindow is Path 6 state: the distinct outside recipients each
// client reached over the detection window, the clients seen on an
// authenticated admin page per account, and when each script was last
// reported. It is kept apart from perIPWindow, which Flow E sweeps after
// one idle hour, because bots post about hourly and the window is a day.
// All methods are safe for concurrent use.
type formRelayWindow struct {
mu sync.Mutex
maxIPs int
ips map[string]*formRelayClient
staff map[string]map[string]time.Time // cpuser -> client key -> last admin mail
// scriptFired is keyed by scriptID, which may name no script window.
scriptFired map[scriptKey]time.Time
hitCap bool // evicted a client since the last sweep
now time.Time // latest observation or sweep; replay cannot rewind expiry
evictions *metrics.Counter // optional
}
type formRelayClient struct {
lastSeen time.Time
firedAt time.Time
observedAt time.Time
rcpts map[string]formRelayRcpt
}
type formRelayRcpt struct {
at time.Time
script scriptKey
cpuser string
domain string // empty for an unparsed message
}
// formRelayShare is one script's part of a client's outside recipients.
// Raw is the newest recipient's script key, whose window holds its subjects.
type formRelayShare struct {
Count int
CPUser string
Raw scriptKey
}
type formRelaySnapshot struct {
Count int
Shares map[scriptKey]formRelayShare
Domains []string // sorted; full addresses belong to third parties
FireClient bool
}
// evaluate captures one qualifying window and, with claim, claims its
// client cooldown for mode under the same lock. Expiry must not remove evidence
// between counting it and building the finding. Current exemptions also
// apply to older mail. Without claim, FireClient reports whether a claim
// would succeed.
func (w *formRelayWindow) evaluate(key string, at time.Time, window time.Duration, spread int, ignored func(scriptKey) bool, mode string, claim bool) formRelaySnapshot {
w.mu.Lock()
defer w.mu.Unlock()
cutoff := w.advanceLocked(at).Add(-window)
c := w.ips[key]
if c == nil || at.Before(cutoff) {
return formRelaySnapshot{}
}
c.expire(cutoff)
for r, v := range c.rcpts {
if seenAtOrAfter(w.staff[v.cpuser][key], at.Add(-formRelayStaffTTL)) || ignored != nil && ignored(v.script) {
delete(c.rcpts, r)
}
}
if len(c.rcpts) < spread {
return formRelaySnapshot{}
}
out := formRelaySnapshot{Count: len(c.rcpts), Shares: make(map[scriptKey]formRelayShare)}
domains := make(map[string]struct{})
newest := make(map[scriptKey]time.Time)
for _, v := range c.rcpts {
id := scriptID(v.script)
share := out.Shares[id]
share.Count++
share.CPUser = v.cpuser
if last := newest[id]; share.Raw == "" || v.at.After(last) || v.at.Equal(last) && v.script < share.Raw {
newest[id], share.Raw = v.at, v.script
}
out.Shares[id] = share
if v.domain != "" {
domains[v.domain] = struct{}{}
}
}
for domain := range domains {
out.Domains = append(out.Domains, domain)
}
sort.Strings(out.Domains)
// An observe report must not consume the first enforce response after
// reload. Keep both cooldowns so toggling cannot repeatedly re-arm one.
firedAt := &c.observedAt
if mode == "enforce" {
firedAt = &c.firedAt
}
if !seenAtOrAfter(*firedAt, at.Add(-window)) {
if claim {
*firedAt = at
}
out.FireClient = true
}
return out
}
func newFormRelayWindow(maxIPs int) *formRelayWindow {
return &formRelayWindow{
maxIPs: maxIPs,
ips: make(map[string]*formRelayClient),
staff: make(map[string]map[string]time.Time),
scriptFired: make(map[scriptKey]time.Time),
}
}
// formRelayKey is the client unit Path 6 counts: the IPv4 address, or the
// IPv6 /64, since one IPv6 customer gets a whole /64.
func formRelayKey(ip string) string {
a, err := netip.ParseAddr(ip)
if err != nil {
return ""
}
if a = a.Unmap(); a.Is4() {
return a.String()
}
return netip.PrefixFrom(a, 64).Masked().String()
}
// classifyFormRelayRecipients returns the lowercased recipients outside the
// account's listed domains (a subdomain needs its own entry, as in
// perIPWindow.appendMessage) and whether any recipient failed to parse. The
// attacker picks the address, so a parse failure, including a recipient
// block the spool parser could not validate (empty list), must still count.
func classifyFormRelayRecipients(rcpts []string, own map[string]struct{}) (outside []string, unparsed bool) {
if len(rcpts) == 0 {
return nil, true
}
for _, raw := range rcpts {
r, domain := parseRelayRecipient(raw)
if r == "" {
unparsed = true
continue
}
if _, ok := own[domain]; !ok {
outside = append(outside, r)
}
}
return outside, unparsed
}
// record adds one message's outside recipients for key and returns their
// distinct count, capped at limit, in the latest observed window. Delayed
// messages cannot rewind that window. An unparsed message adds one entry
// keyed by msgID. Retaining the newest recipients preserves later crossings
// when older evidence expires. An empty key is never stored.
func (w *formRelayWindow) record(key, cpuser string, k scriptKey, msgID string, at time.Time, outside []string, unparsed bool, window time.Duration, limit int) int {
if key == "" {
return 0
}
w.mu.Lock()
defer w.mu.Unlock()
cutoff := w.advanceLocked(at).Add(-window)
c := w.ips[key]
if c == nil {
if at.Before(cutoff) || len(outside) == 0 && !unparsed {
return 0
}
c = w.addLocked(key, cutoff)
}
c.expire(cutoff)
c.trim(limit)
if at.Before(cutoff) {
return len(c.rcpts)
}
if !at.Before(c.lastSeen) {
c.lastSeen = at
}
for _, r := range outside {
c.put(r, formRelayRcpt{at: at, script: k, cpuser: cpuser, domain: r[strings.LastIndexByte(r, '@')+1:]}, limit)
}
if unparsed {
c.put(formRelayUnparsed+msgID, formRelayRcpt{at: at, script: k, cpuser: cpuser}, limit)
}
return len(c.rcpts)
}
// put stores r, or refreshes it so the latest message's script owns it.
func (c *formRelayClient) put(r string, v formRelayRcpt, limit int) {
if old, ok := c.rcpts[r]; ok {
if !v.at.Before(old.at) {
c.rcpts[r] = v
}
return
}
c.rcpts[r] = v
c.trim(limit)
}
func (c *formRelayClient) trim(limit int) {
for len(c.rcpts) > limit {
var oldest string
var at time.Time
for r, v := range c.rcpts {
if oldest == "" || v.at.Before(at) {
oldest, at = r, v.at
}
}
delete(c.rcpts, oldest)
}
}
func (c *formRelayClient) expire(cutoff time.Time) {
for r, v := range c.rcpts {
if v.at.Before(cutoff) {
delete(c.rcpts, r)
}
}
}
func (w *formRelayWindow) advanceLocked(at time.Time) time.Time {
if at.After(w.now) {
w.now = at
}
return w.now
}
func (w *formRelayWindow) expireClientsLocked(cutoff time.Time) {
for key, c := range w.ips {
c.expire(cutoff)
if len(c.rcpts) == 0 && !seenAtOrAfter(c.firedAt, cutoff) && !seenAtOrAfter(c.observedAt, cutoff) {
delete(w.ips, key)
}
}
}
func (w *formRelayWindow) addLocked(key string, cutoff time.Time) *formRelayClient {
if len(w.ips) >= w.maxIPs {
w.expireClientsLocked(cutoff)
}
if len(w.ips) >= w.maxIPs {
w.evictLocked()
}
c := &formRelayClient{rcpts: make(map[string]formRelayRcpt, 4)}
w.ips[key] = c
return c
}
// evictLocked drops the client with the fewest recipients, the oldest among
// equals. That minimum is never a client at or above half the threshold
// while a lower one exists, so flooding the table with one-recipient
// clients cannot push out one close to crossing; when every client is at or
// above half, the lowest and oldest still goes.
func (w *formRelayWindow) evictLocked() {
// ponytail: O(n) scan only at the cap; bucket by count if floods make it hot.
var victim string
var low *formRelayClient
for key, c := range w.ips {
if low == nil || len(c.rcpts) < len(low.rcpts) ||
len(c.rcpts) == len(low.rcpts) && c.lastSeen.Before(low.lastSeen) {
victim, low = key, c
}
}
if low != nil {
delete(w.ips, victim)
w.hitCap = true
if w.evictions != nil {
w.evictions.Inc()
}
}
}
// claimScript reports true once per script id per window.
func (w *formRelayWindow) claimScript(id scriptKey, at time.Time, window time.Duration) bool {
w.mu.Lock()
defer w.mu.Unlock()
if seenAtOrAfter(w.scriptFired[id], at.Add(-window)) {
return false
}
w.scriptFired[id] = at
return true
}
// markStaff records that key sent mail through an authenticated admin page
// of cpuser. Each account keeps its newest formRelayStaffPerAccount keys.
func (w *formRelayWindow) markStaff(cpuser, key string, at time.Time) {
if key == "" {
return
}
w.mu.Lock()
defer w.mu.Unlock()
keys := w.staff[cpuser]
if keys == nil {
keys = make(map[string]time.Time, 1)
w.staff[cpuser] = keys
}
if at.After(keys[key]) {
keys[key] = at
}
if len(keys) > formRelayStaffPerAccount {
evictOldestRecipient(keys)
}
}
func (w *formRelayWindow) isStaff(cpuser, key string, at time.Time) bool {
if key == "" {
return false
}
w.mu.Lock()
defer w.mu.Unlock()
return seenAtOrAfter(w.staff[cpuser][key], at.Add(-formRelayStaffTTL))
}
// sweep drops expired recipients, clients with nothing left in the window,
// script reports older than the window and expired staff, and reports
// whether the table hit its cap since the last sweep.
func (w *formRelayWindow) sweep(now time.Time, window time.Duration) (evictedFull bool) {
w.mu.Lock()
defer w.mu.Unlock()
now = w.advanceLocked(now)
w.expireClientsLocked(now.Add(-window))
for id, at := range w.scriptFired {
if !seenAtOrAfter(at, now.Add(-window)) {
delete(w.scriptFired, id)
}
}
staffCutoff := now.Add(-formRelayStaffTTL)
for user, keys := range w.staff {
for key, at := range keys {
if at.Before(staffCutoff) {
delete(keys, key)
}
}
if len(keys) == 0 {
delete(w.staff, user)
}
}
evictedFull, w.hitCap = w.hitCap, false
return evictedFull
}
package daemon
import (
"sync"
"github.com/pidginhost/csm/internal/metrics"
)
// phpRelayMetrics holds every series the module emits via the local
// internal/metrics OpenMetrics implementation.
//
// Defined as a struct of pointers so callers can pass nil and skip
// observation. All increments at call sites are guarded by
// `if e.metrics != nil` checks.
type phpRelayMetrics struct {
Findings *metrics.CounterVec // labels: path
Actions *metrics.CounterVec // labels: action, result
PathSkipped *metrics.CounterVec // labels: path, reason
WindowsActive *metrics.GaugeVec // labels: kind (script/ip/account)
MsgIDIndexSize *metrics.GaugeVec // labels: layer (memory/bbolt)
MsgindexPersistDropped *metrics.Counter
MsgindexPersistErrors *metrics.Counter
InotifyOverflows *metrics.Counter
SpoolReadErrors *metrics.Counter
UserdataErrors *metrics.Counter
ActiveMsgsCapped *metrics.Counter
SpoolScanFallbacks *metrics.CounterVec // labels: reason
ActionGone *metrics.Counter
FormRelayEvictions *metrics.Counter
}
var (
phpRelayMetricsOnce sync.Once
phpRelayMetricsInstance *phpRelayMetrics
)
// newPHPRelayMetrics constructs (and registers via the package default
// registry) the singleton metric set. Subsequent calls return the same
// instance -- sync.Once protects against the duplicate-name panic from
// metrics.MustRegister.
func newPHPRelayMetrics() *phpRelayMetrics {
phpRelayMetricsOnce.Do(func() {
m := &phpRelayMetrics{
Findings: metrics.NewCounterVec("csm_php_relay_findings_total", "Findings emitted by php_relay paths.", []string{"path"}),
Actions: metrics.NewCounterVec("csm_php_relay_actions_total", "AutoFreeze actions attempted.", []string{"action", "result"}),
PathSkipped: metrics.NewCounterVec("csm_php_relay_path_skipped_total", "Path evaluation skipped.", []string{"path", "reason"}),
WindowsActive: metrics.NewGaugeVec("csm_php_relay_windows_active", "Active windows per kind.", []string{"kind"}),
MsgIDIndexSize: metrics.NewGaugeVec("csm_php_relay_msgid_index_size", "msgIDIndex size by storage layer.", []string{"layer"}),
MsgindexPersistDropped: metrics.NewCounter("csm_php_relay_msgindex_persist_dropped_total", "Persist queue overflow drops."),
MsgindexPersistErrors: metrics.NewCounter("csm_php_relay_msgindex_persist_errors_total", "bbolt commit failures."),
InotifyOverflows: metrics.NewCounter("csm_php_relay_inotify_overflows_total", "IN_Q_OVERFLOW events."),
SpoolReadErrors: metrics.NewCounter("csm_php_relay_spool_read_errors_total", "Spool -H read errors."),
UserdataErrors: metrics.NewCounter("csm_php_relay_userdata_errors_total", "cpanelUserDomains read errors."),
ActiveMsgsCapped: metrics.NewCounter("csm_php_relay_active_msgs_capped_total", "scriptState.activeMsgs cap-hit events."),
SpoolScanFallbacks: metrics.NewCounterVec("csm_php_relay_spool_scan_fallbacks_total", "AutoFreeze spool-scan fallback invocations.", []string{"reason"}),
ActionGone: metrics.NewCounter("csm_php_relay_action_gone_total", "Messages already absent at exim -Mf time."),
FormRelayEvictions: metrics.NewCounter("csm_php_relay_form_evictions_total", "Path 6 clients dropped because the client table was full."),
}
metrics.MustRegister("csm_php_relay_findings_total", m.Findings)
metrics.MustRegister("csm_php_relay_actions_total", m.Actions)
metrics.MustRegister("csm_php_relay_path_skipped_total", m.PathSkipped)
metrics.MustRegister("csm_php_relay_windows_active", m.WindowsActive)
metrics.MustRegister("csm_php_relay_msgid_index_size", m.MsgIDIndexSize)
metrics.MustRegister("csm_php_relay_msgindex_persist_dropped_total", m.MsgindexPersistDropped)
metrics.MustRegister("csm_php_relay_msgindex_persist_errors_total", m.MsgindexPersistErrors)
metrics.MustRegister("csm_php_relay_inotify_overflows_total", m.InotifyOverflows)
metrics.MustRegister("csm_php_relay_spool_read_errors_total", m.SpoolReadErrors)
metrics.MustRegister("csm_php_relay_userdata_errors_total", m.UserdataErrors)
metrics.MustRegister("csm_php_relay_active_msgs_capped_total", m.ActiveMsgsCapped)
metrics.MustRegister("csm_php_relay_spool_scan_fallbacks_total", m.SpoolScanFallbacks)
metrics.MustRegister("csm_php_relay_action_gone_total", m.ActionGone)
metrics.MustRegister("csm_php_relay_form_evictions_total", m.FormRelayEvictions)
phpRelayMetricsInstance = m
})
return phpRelayMetricsInstance
}
package daemon
import (
"sync"
"time"
)
// indexEntry maps a message ID to the per-message attribution recorded at
// acceptance. Used by Path 3 (Stage 2) to map delivery-failure log lines
// back to the originating script. Public field names because gob-encoded
// for bbolt persistence in Task C3.
type indexEntry struct {
ScriptKey string
HeaderScore int
SourceIP string
CPUser string
At time.Time
}
// msgIDIndex stores indexEntry per msgID, served from memory.
// Bounded by maxEntries; overflow drops the oldest entry by acceptance time.
// Persistence to bbolt is handled by msgIndexPersister (Task C3).
type msgIDIndex struct {
mu sync.Mutex
entries map[string]indexEntry
maxEntries int
persister *msgIndexPersister // nil in unit tests; real in production
}
func newMsgIDIndex(persister *msgIndexPersister, maxEntries int) *msgIDIndex {
if maxEntries <= 0 {
maxEntries = 200_000
}
return &msgIDIndex{
entries: make(map[string]indexEntry, 4096),
maxEntries: maxEntries,
persister: persister,
}
}
// Put records an entry. If the in-memory map exceeds maxEntries, the
// oldest entry by At is evicted from memory. The persister (if non-nil)
// receives the put asynchronously; persistence failure does not affect
// in-memory correctness.
func (i *msgIDIndex) Put(msgID string, e indexEntry) {
i.mu.Lock()
if _, ok := i.entries[msgID]; !ok && len(i.entries) >= i.maxEntries {
i.evictOldestLocked()
}
i.entries[msgID] = e
i.mu.Unlock()
if i.persister != nil {
i.persister.Enqueue(msgID, e)
}
}
// Get returns the entry and whether it was present.
func (i *msgIDIndex) Get(msgID string) (indexEntry, bool) {
i.mu.Lock()
e, ok := i.entries[msgID]
i.mu.Unlock()
return e, ok
}
// Has reports whether msgID is present in memory.
func (i *msgIDIndex) Has(msgID string) bool {
i.mu.Lock()
_, ok := i.entries[msgID]
i.mu.Unlock()
return ok
}
// Len returns the number of entries currently in memory.
func (i *msgIDIndex) Len() int {
i.mu.Lock()
defer i.mu.Unlock()
return len(i.entries)
}
// SweepMemory drops entries whose At is at or before cutoff.
// Called by Flow E's 1-min ticker with cutoff = now - 4h.
func (i *msgIDIndex) SweepMemory(cutoff time.Time) int {
i.mu.Lock()
defer i.mu.Unlock()
n := 0
for id, e := range i.entries {
if !e.At.After(cutoff) {
delete(i.entries, id)
n++
}
}
return n
}
func (i *msgIDIndex) evictOldestLocked() {
var oldestID string
var oldestAt time.Time
first := true
for id, e := range i.entries {
if first || e.At.Before(oldestAt) {
oldestID = id
oldestAt = e.At
first = false
}
}
if oldestID != "" {
delete(i.entries, oldestID)
}
}
package daemon
import (
"bytes"
"encoding/gob"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/store"
)
const msgIndexBucket = "phprelay:msgindex"
// All bbolt access goes through `*store.DB`'s phprelay helpers. The
// underlying `*bolt.DB` is unexported by design (internal/store/db.go);
// daemon code never imports go.etcd.io/bbolt directly.
// msgIndexPersister persists msgIDIndex entries to bbolt off the hot path.
// Enqueue, Flush and Stop are safe for concurrent callers after Start.
// Persistence failure does not affect the in-memory index; it degrades recovery
// after a restart and
// emits a Critical via the alerter callback.
//
// SetErrorCallback must be invoked BEFORE Start to establish a
// happens-before relationship between the writer of `onError` and the
// goroutine that reads it; concurrent callers after Start are not safe.
type msgIndexPersister struct {
db *store.DB
queue chan persistOp
flushEvery time.Duration
batchSize int
stopCh chan struct{}
doneCh chan struct{}
flushReqCh chan chan struct{}
queueStats *queuehealth.Tracker
admission sync.Mutex
stopped bool
droppedTotal uint64
errorsTotal uint64
onError func(alert.Finding)
metrics *phpRelayMetrics
}
type persistOp struct {
msgID string
entry indexEntry
ticket queuehealth.Ticket
}
func newMsgIndexPersister(db *store.DB, queueSize int, flushEvery time.Duration) *msgIndexPersister {
if queueSize <= 0 {
queueSize = 4096
}
if flushEvery <= 0 {
flushEvery = 100 * time.Millisecond
}
return &msgIndexPersister{
db: db,
queue: make(chan persistOp, queueSize),
flushEvery: flushEvery,
batchSize: 256,
stopCh: make(chan struct{}),
doneCh: make(chan struct{}),
flushReqCh: make(chan chan struct{}),
queueStats: queuehealth.New(queueSize, time.Minute),
onError: func(alert.Finding) {},
}
}
// SetErrorCallback wires Critical findings emission for bbolt failures.
// Optional -- nil disables emission (used by tests). Must be called before
// Start; concurrent invocation after Start is not safe.
func (p *msgIndexPersister) SetErrorCallback(fn func(alert.Finding)) {
if fn != nil {
p.onError = fn
}
}
// SetMetrics wires the phpRelayMetrics sink. Optional -- nil disables
// observation (used by tests). Must be called before Start; concurrent
// invocation after Start is not safe.
func (p *msgIndexPersister) SetMetrics(m *phpRelayMetrics) {
p.metrics = m
}
// Start launches the single writer. Call it once, after configuring callbacks.
func (p *msgIndexPersister) Start() {
go p.run()
}
func (p *msgIndexPersister) Stop() {
p.admission.Lock()
if !p.stopped {
p.stopped = true
close(p.queue)
close(p.stopCh)
}
p.admission.Unlock()
<-p.doneCh
}
// Enqueue is non-blocking. Returns immediately; the op is dropped if the
// queue is full or stopped (in which case DroppedCount increments).
func (p *msgIndexPersister) Enqueue(msgID string, e indexEntry) {
p.admission.Lock()
defer p.admission.Unlock()
if p.stopped {
p.queueStats.Lose(time.Now(), 1)
} else {
ticket := p.queueStats.Begin(time.Now())
select {
case p.queue <- persistOp{msgID: msgID, entry: e, ticket: ticket}:
return
default:
ticket.Reject(time.Now())
}
}
atomic.AddUint64(&p.droppedTotal, 1)
if p.metrics != nil {
p.metrics.MsgindexPersistDropped.Inc()
}
}
// Flush blocks until the persister has drained whatever was already
// enqueued at call time. For tests and shutdown.
func (p *msgIndexPersister) Flush() {
done := make(chan struct{})
select {
case p.flushReqCh <- done:
<-done
case <-p.stopCh:
<-p.doneCh
}
}
func (p *msgIndexPersister) QueueStatuses(now time.Time) map[string]queuehealth.Status {
return map[string]queuehealth.Status{"persistence": p.queueStats.Snapshot(now)}
}
func (p *msgIndexPersister) DroppedCount() uint64 {
return atomic.LoadUint64(&p.droppedTotal)
}
func (p *msgIndexPersister) ErrorCount() uint64 {
return atomic.LoadUint64(&p.errorsTotal)
}
// Lookup reads an entry from bbolt by msgID.
func (p *msgIndexPersister) Lookup(msgID string) (indexEntry, bool, error) {
raw, ok, err := p.db.PHPRelayGet(msgIndexBucket, msgID)
if err != nil || !ok {
return indexEntry{}, false, err
}
var e indexEntry
if err := gob.NewDecoder(bytes.NewReader(raw)).Decode(&e); err != nil {
return indexEntry{}, false, fmt.Errorf("decode %s: %w", msgID, err)
}
return e, true, nil
}
// SweepBolt deletes phprelay:msgindex entries whose At <= cutoff.
// Returns the number of entries removed. Called by Flow E's 1-min ticker.
// Corrupt rows (decode failure) are also dropped to keep the bucket
// healthy.
func (p *msgIndexPersister) SweepBolt(cutoff time.Time) (int, error) {
return p.db.PHPRelaySweep(msgIndexBucket, func(_, value []byte) bool {
var e indexEntry
if err := gob.NewDecoder(bytes.NewReader(value)).Decode(&e); err != nil {
return true // drop corrupt rows
}
return !e.At.After(cutoff)
})
}
func (p *msgIndexPersister) run() {
defer close(p.doneCh)
ticker := time.NewTicker(p.flushEvery)
defer ticker.Stop()
var pending []persistOp
for {
select {
case op, ok := <-p.queue:
if !ok {
p.commitBatch(pending)
return
}
op.ticket.Start(time.Now())
pending = append(pending, op)
if len(pending) >= p.batchSize {
p.commitBatch(pending)
pending = pending[:0]
}
case <-ticker.C:
if len(pending) > 0 {
p.commitBatch(pending)
pending = pending[:0]
}
case done := <-p.flushReqCh:
p.flushPending(pending)
pending = pending[:0]
close(done)
}
}
}
func (p *msgIndexPersister) flushPending(pending []persistOp) {
// There is one consumer. Snapshot admission so concurrent producers cannot
// extend this flush, and keep each transaction within the writer's limit.
for queued := len(p.queue); queued > 0; queued-- {
op := <-p.queue
op.ticket.Start(time.Now())
pending = append(pending, op)
if len(pending) >= p.batchSize {
p.commitBatch(pending)
pending = pending[:0]
}
}
p.commitBatch(pending)
}
func (p *msgIndexPersister) commitBatch(ops []persistOp) {
if len(ops) == 0 {
return
}
// A failed error callback must not leave its batch reported as running.
defer rejectPersistenceOps(ops)
kvs := make([]store.PHPRelayKV, 0, len(ops))
var buf bytes.Buffer
for i := range ops {
op := &ops[i]
buf.Reset()
if err := gob.NewEncoder(&buf).Encode(&op.entry); err != nil {
// Encoding failure is a code bug, not a transient I/O issue.
// Skip the offending op and continue with the rest of the batch.
op.ticket.Reject(time.Now())
op.ticket = queuehealth.Ticket{}
atomic.AddUint64(&p.errorsTotal, 1)
if p.metrics != nil {
p.metrics.MsgindexPersistErrors.Inc()
}
p.onError(alert.Finding{
Severity: alert.Critical,
Check: "email_php_relay_msgindex_persist_failed",
Timestamp: time.Now(),
Message: fmt.Sprintf("encode %s: %v", op.msgID, err),
})
continue
}
kvs = append(kvs, store.PHPRelayKV{
Key: []byte(op.msgID),
Value: append([]byte(nil), buf.Bytes()...),
})
}
if err := p.db.PHPRelayPutBatch(msgIndexBucket, kvs); err != nil {
rejectPersistenceOps(ops)
atomic.AddUint64(&p.errorsTotal, 1)
if p.metrics != nil {
p.metrics.MsgindexPersistErrors.Inc()
}
p.onError(alert.Finding{
Severity: alert.Critical,
Check: "email_php_relay_msgindex_persist_failed",
Timestamp: time.Now(),
Message: fmt.Sprintf("phprelay:msgindex commit failed (%d ops): %v", len(kvs), err),
})
return
}
for i := range ops {
ops[i].ticket.Finish(time.Now())
ops[i].ticket = queuehealth.Ticket{}
}
}
func rejectPersistenceOps(ops []persistOp) {
for i := range ops {
ops[i].ticket.Reject(time.Now())
ops[i].ticket = queuehealth.Ticket{}
}
}
package daemon
import (
"bufio"
"bytes"
"context"
"errors"
"io"
"os"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/eximlog"
)
const phpRelayHistoryMaxLineBytes = 1024 * 1024
// ScanEximHistoryForPHPRelayAccountVolume replays exim_mainlog through the
// Path 2b parser, used at daemon startup to populate perAccountWindow with
// recent outbound activity. Each accepted finding is delivered via emit.
//
// Reads the file lazily and skips a single oversized line instead of
// abandoning later entries. Caller passes `now` to keep retro replays
// deterministic for tests; production passes time.Now() once and the parser
// uses it for window math.
//
// ctx scopes the scan to daemon lifetime; nil leaves the scan unbounded for
// direct helper callers.
func ScanEximHistoryForPHPRelayAccountVolume(ctx context.Context, path string, eng *evaluator, now time.Time, emit func(alert.Finding)) {
// #nosec G304 -- path is operator-configured / hardcoded to cPanel default.
f, err := os.Open(path)
if err != nil {
return
}
defer func() { _ = f.Close() }()
reader := bufio.NewReaderSize(f, 64*1024)
var line []byte
oversized := false
for {
if phpRelayScanContextDone(ctx) {
return
}
part, rerr := reader.ReadSlice('\n')
if len(part) > 0 && !oversized {
if len(line)+len(part) > phpRelayHistoryMaxLineBytes {
oversized = true
line = nil
} else {
line = append(line, part...)
}
}
switch {
case rerr == nil:
if !oversized {
emitPHPRelayHistoryLine(line, eng, now, emit)
}
line = nil
oversized = false
case errors.Is(rerr, bufio.ErrBufferFull):
continue
case errors.Is(rerr, io.EOF):
if len(line) > 0 && !oversized {
emitPHPRelayHistoryLine(line, eng, now, emit)
}
return
default:
return
}
}
}
func phpRelayScanContextDone(ctx context.Context) bool {
if ctx == nil {
return false
}
select {
case <-ctx.Done():
return true
default:
return false
}
}
func emitPHPRelayHistoryLine(line []byte, eng *evaluator, now time.Time, emit func(alert.Finding)) {
line = bytes.TrimSuffix(line, []byte("\n"))
line = bytes.TrimSuffix(line, []byte("\r"))
s := string(line)
// Replay only lines inside the account detection window, stamped with their
// real exim timestamp. Without this every historical line was stamped `now`,
// so a whole day of sends collapsed into one hour and fired a false
// "account sent >= N in the last hour" Critical on every daemon start.
ts, ok := eximlog.LineTime(s, time.Local)
if !ok || ts.Before(now.Add(-phpRelayAccountWindowDur)) || ts.After(now) {
return
}
for _, ev := range eng.parsePHPRelayAccountVolumeAt(s, ts, now) {
emit(ev)
}
}
package daemon
import (
"bytes"
"encoding/gob"
"errors"
"time"
"github.com/pidginhost/csm/internal/store"
)
const (
settingsBucket = "phprelay:settings"
dryRunOverrideKey = "dry_run_override"
)
type dryRunOverrideRow struct {
Value bool
UpdatedAt time.Time
UpdatedBy string
}
func writeDryRunOverride(db *store.DB, val bool, by string) error {
if db == nil {
return errors.New("db nil")
}
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(dryRunOverrideRow{Value: val, UpdatedAt: time.Now(), UpdatedBy: by}); err != nil {
return err
}
return db.PHPRelayPut(settingsBucket, dryRunOverrideKey, buf.Bytes())
}
func deleteDryRunOverride(db *store.DB) error {
if db == nil {
return nil
}
return db.PHPRelayDelete(settingsBucket, dryRunOverrideKey)
}
func readDryRunOverride(db *store.DB) (bool, bool, error) {
if db == nil {
return false, false, nil
}
raw, ok, err := db.PHPRelayGet(settingsBucket, dryRunOverrideKey)
if err != nil || !ok {
return false, false, err
}
var row dryRunOverrideRow
if err := gob.NewDecoder(bytes.NewReader(raw)).Decode(&row); err != nil {
return false, false, err
}
return row.Value, true, nil
}
//go:build linux
package daemon
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"unsafe"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/emailspool"
)
// spoolWatcher watches /var/spool/exim/input for new -H files. cPanel hashes
// msgIDs into 64+ subdirs; the watcher enumerates them at start and watches
// IN_CREATE on the parent so subdirs that appear later are also picked up.
//
// On every IN_CLOSE_WRITE / IN_MOVED_TO whose name ends in "-H", the
// supplied callback is invoked synchronously with the absolute path. The
// callback must not block long; spawn worker goroutines if needed.
type spoolWatcher struct {
root string
onFile func(path string)
fd int
parentW int
mu sync.Mutex
subDirs map[int]string // watch descriptor -> path
queueHealthOnce sync.Once
kernelQueue *notificationQueue
overflowCount uint64
onOverflow func() // invoked from Run() the moment IN_Q_OVERFLOW arrives
metrics *phpRelayMetrics
}
// SetOverflowHandler wires the recovery scan + Critical finding emission
// into the watcher. Caller passes a closure that calls runRecoveryScan
// against the spool root and emits findings via the daemon alerter.
func (w *spoolWatcher) SetOverflowHandler(fn func()) { w.onOverflow = fn }
// SetMetrics wires the phpRelayMetrics sink. Optional -- nil disables
// observation (used by tests). Must be called before Run; concurrent
// invocation after Run is not safe.
func (w *spoolWatcher) SetMetrics(m *phpRelayMetrics) { w.metrics = m }
func newSpoolWatcher(root string, onFile func(path string)) (*spoolWatcher, error) {
fd, err := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK)
if err != nil {
return nil, fmt.Errorf("inotify_init1: %w", err)
}
parentMask := uint32(unix.IN_CREATE | unix.IN_MOVED_TO)
parentW, err := unix.InotifyAddWatch(fd, root, parentMask)
if err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("inotify_add_watch %s: %w", root, err)
}
w := &spoolWatcher{
root: root,
onFile: onFile,
fd: fd,
parentW: parentW,
subDirs: make(map[int]string),
}
// Enumerate existing subdirs.
entries, err := os.ReadDir(root)
if err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("readdir %s: %w", root, err)
}
for _, e := range entries {
if !e.IsDir() {
continue
}
if err := w.addSubdir(filepath.Join(root, e.Name())); err != nil {
// Non-fatal -- continue with what we have.
continue
}
}
return w, nil
}
func (w *spoolWatcher) addSubdir(path string) error {
mask := uint32(unix.IN_CLOSE_WRITE | unix.IN_MOVED_TO)
w.initQueueHealth()
wd, err := w.kernelQueue.useDescriptor(func() (int, error) {
return unix.InotifyAddWatch(w.fd, path, mask)
})
if err != nil {
return err
}
w.mu.Lock()
w.subDirs[wd] = path
w.mu.Unlock()
return nil
}
func (w *spoolWatcher) Close() error {
w.initQueueHealth()
return w.kernelQueue.close()
}
// Run drains inotify events until ctx is cancelled.
func (w *spoolWatcher) Run(ctx context.Context) {
w.initQueueHealth()
defer func() { _ = w.Close() }()
buf := make([]byte, 16*1024)
for {
select {
case <-ctx.Done():
return
default:
}
_, err := w.kernelQueue.read(buf, w.processEvents)
if err != nil {
if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EINTR) {
// Briefly yield via select so cancellation is responsive.
select {
case <-ctx.Done():
return
default:
_, _ = w.kernelQueue.useDescriptor(func() (int, error) {
// #nosec G115 -- Linux descriptors are nonnegative int32 values.
return unix.Poll([]unix.PollFd{{Fd: int32(w.fd), Events: unix.POLLIN}}, 100)
})
continue
}
}
// Treat other errors as fatal; supervisor will restart us.
return
}
}
}
func (w *spoolWatcher) processEvents(buf []byte) {
n := len(buf)
offset := 0
for offset+unix.SizeofInotifyEvent <= n {
// #nosec G103 -- bounds-checked above; standard inotify decode pattern.
ev := (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset]))
nameBytes := buf[offset+unix.SizeofInotifyEvent : offset+unix.SizeofInotifyEvent+int(ev.Len)]
name := strings.TrimRight(string(nameBytes), "\x00")
offset += unix.SizeofInotifyEvent + int(ev.Len)
if ev.Mask&unix.IN_Q_OVERFLOW != 0 {
w.kernelQueue.losses.Lose(time.Now(), 1)
w.overflowCount++
if w.metrics != nil {
w.metrics.InotifyOverflows.Inc()
}
if w.onOverflow != nil {
w.onOverflow()
}
continue
}
if int(ev.Wd) == w.parentW {
if ev.Mask&(unix.IN_CREATE|unix.IN_MOVED_TO) != 0 && name != "" {
full := filepath.Join(w.root, name)
if fi, err := os.Stat(full); err == nil && fi.IsDir() {
_ = w.addSubdir(full)
}
}
continue
}
w.mu.Lock()
dir, ok := w.subDirs[int(ev.Wd)]
w.mu.Unlock()
if !ok || name == "" {
continue
}
if !strings.HasSuffix(name, "-H") {
continue
}
w.onFile(filepath.Join(dir, name))
}
}
// OverflowCount returns the number of IN_Q_OVERFLOW events observed.
// Used by the daemon to drive recovery scans (Task I3).
//
//nolint:unused // consumed by daemon wiring (Task O2)
func (w *spoolWatcher) OverflowCount() uint64 {
return w.overflowCount
}
// spoolPipeline ties together: parse headers -> compute signals -> update
// windows -> evaluate paths -> emit findings via alerter callback.
type spoolPipeline struct {
eng *evaluator
domains *userDomainsResolver
policies *emailspool.Policies
msgIndex *msgIDIndex
ignores *ignoreList
alerter func(alert.Finding)
rebuilding atomic.Bool
}
func newSpoolPipeline(eng *evaluator, domains *userDomainsResolver, pol *emailspool.Policies, idx *msgIDIndex, ignores *ignoreList, alerter func(alert.Finding)) *spoolPipeline {
eng.SetPolicies(pol)
return &spoolPipeline{
eng: eng, domains: domains, policies: pol, msgIndex: idx, ignores: ignores, alerter: alerter,
}
}
// SetRebuilding gates finding emission during the startup spool-walker
// rebuild pass. When true: state is updated, findings are NOT emitted.
func (p *spoolPipeline) SetRebuilding(v bool) { p.rebuilding.Store(v) }
// OnFile is the inotify callback for live spool events: it stamps the message
// at the current time.
func (p *spoolPipeline) OnFile(path string) {
p.onFileAt(path, time.Now())
}
// onFileAt parses, signals, updates state, and evaluates a single -H file,
// attributing the message to event time `at`. Live callers pass time.Now(); the
// startup walker passes the -H file's ModTime so already-queued mail keeps its
// real age instead of being compressed into one instant.
func (p *spoolPipeline) onFileAt(path string, at time.Time) {
h, err := emailspool.ParseHeaders(path)
if err != nil {
if p.eng != nil && p.eng.metrics != nil {
p.eng.metrics.SpoolReadErrors.Inc()
}
return
}
if h.XPHPScript == "" {
return
}
msgID := msgIDFromPath(path)
if msgID == "" {
return
}
if p.msgIndex != nil && p.msgIndex.Has(msgID) {
return // queue-runner re-write dedup
}
auth, domainErr := p.domains.Domains(h.EnvelopeUser)
sig := computeSignals(h, auth, p.policies)
if sig.ScriptKey == "" {
return
}
if p.ignores != nil && p.ignores.Covers(sig.ScriptKey) {
return
}
now := at
if p.msgIndex != nil {
p.msgIndex.Put(msgID, indexEntry{
ScriptKey: string(sig.ScriptKey),
SourceIP: sig.SourceIP,
CPUser: h.EnvelopeUser,
At: now,
})
}
state := p.eng.scripts.getOrCreate(sig.ScriptKey)
// Path 2 includes cron-driven mail without an HTTP source IP, so every
// script event carries its recipient parse outcome into the script window.
state.appendMessage(scriptEvent{
At: now,
MsgID: msgID,
Subject: truncateDaemon(h.Subject, phpRelayBreakdownSubjectMax),
FromMismatch: sig.FromMismatch,
AdditionalSignal: sig.AdditionalSignal,
SourceIP: sig.SourceIP,
}, h.Recipients)
state.recordActive(msgID, now)
if p.policies == nil || !p.policies.IsProxyIP(sig.SourceIP) {
fanoutRecipients := h.Recipients
if domainErr != nil || len(auth) == 0 {
// Missing ownership data must not let earlier own-domain
// notifications suppress this message's fanout window.
fanoutRecipients = nil
}
p.eng.ips.appendMessage(sig.SourceIP, sig.ScriptKey, now, h.Subject, fanoutRecipients, auth)
}
formEligible := p.recordFormRelay(h, sig, auth, domainErr, msgID, now)
if p.rebuilding.Load() {
return
}
findings := p.eng.evaluatePaths(sig.ScriptKey, sig.SourceIP, h.EnvelopeUser, now)
relayCfg := p.eng.config().EmailProtection.PHPRelay
if formEligible && relayCfg.Enabled && relayCfg.IPRecipientSpread > 0 {
window := time.Duration(relayCfg.IPRecipientWindowHours) * time.Hour
var ignored func(scriptKey) bool
if p.ignores != nil {
ignored = p.ignores.Covers
}
findings = append(findings, p.eng.formRelayFindings(sig.ScriptKey, sig.SourceIP, h.EnvelopeUser, relayCfg.IPRecipientSpread, window, now, ignored)...)
}
for _, f := range findings {
if f.Check == "email_php_form_relay" {
// Exim's message ID keeps retries tied to the same observation.
f.Observation = alert.Observation{
Producer: string(checks.ProducerPHPRelaySpool),
Stream: "exim_spool",
Cursor: msgID,
ObservedAt: now,
}
}
p.alerter(f)
}
}
// recordFormRelay feeds Path 6 the message's outside recipients for its
// client. Every skip comes before classifying: classification counts an
// empty recipient list as one unparsed recipient, and a failed domain
// lookup leaves nothing to tell outside recipients from the account's own.
// Runs during the startup rebuild too; only emission waits. The result
// permits evaluation only for messages eligible to contribute evidence.
func (p *spoolPipeline) recordFormRelay(h emailspool.Headers, sig signals, auth map[string]struct{}, domainErr error, msgID string, at time.Time) bool {
cfg := p.eng.config()
relayCfg := cfg.EmailProtection.PHPRelay
ip := sig.SourceIP
if relayCfg.IPRecipientSpread <= 0 || ip == "" || domainErr != nil || len(auth) == 0 ||
isWPCronScript(sig.ScriptKey) ||
p.policies != nil && p.policies.IsProxyIP(ip) ||
checks.IsInfraIP(ip, cfg.InfraIPs) { // also the Cloudflare ranges the daemon refreshes
return false
}
key := formRelayKey(ip)
if isAuthenticatedAdminScript(sig.ScriptKey) {
p.eng.form.markStaff(h.EnvelopeUser, key, at)
return false
}
if p.eng.form.isStaff(h.EnvelopeUser, key, at) {
return false
}
outside, unparsed := classifyFormRelayRecipients(h.Recipients, auth)
p.eng.form.record(key, h.EnvelopeUser, sig.ScriptKey, msgID, at, outside, unparsed,
time.Duration(relayCfg.IPRecipientWindowHours)*time.Hour, relayCfg.IPRecipientSpread)
return len(outside) > 0 || unparsed
}
// msgIDFromPath returns the msgID portion of a /path/<msgID>-H file.
func msgIDFromPath(path string) string {
base := filepath.Base(path)
if !strings.HasSuffix(base, "-H") {
return ""
}
return strings.TrimSuffix(base, "-H")
}
// runRecoveryScan walks every -H file under spoolRoot/*/, sorts by mtime
// (oldest first), invokes onFile up to maxFiles. Returns the number scanned
// and whether the cap was hit.
//
//nolint:unused // consumed by daemon wiring (Task O2)
func runRecoveryScan(spoolRoot string, maxFiles int, onFile func(string)) (int, bool) {
type entry struct {
path string
mod time.Time
}
var entries []entry
subs, err := os.ReadDir(spoolRoot)
if err != nil {
return 0, false
}
for _, sub := range subs {
if !sub.IsDir() {
continue
}
subPath := filepath.Join(spoolRoot, sub.Name())
files, err := os.ReadDir(subPath)
if err != nil {
continue
}
for _, f := range files {
if !strings.HasSuffix(f.Name(), "-H") {
continue
}
full := filepath.Join(subPath, f.Name())
fi, err := os.Stat(full)
if err != nil {
continue
}
entries = append(entries, entry{path: full, mod: fi.ModTime()})
}
}
sort.Slice(entries, func(i, j int) bool { return entries[i].mod.Before(entries[j].mod) })
truncated := false
if len(entries) > maxFiles {
entries = entries[:maxFiles]
truncated = true
}
for _, e := range entries {
onFile(e.path)
}
return len(entries), truncated
}
// phpRelayStartupWalkMax bounds how many recent -H files the startup walker
// replays so a stuffed queue cannot stall daemon startup. The live watcher and
// the Path 2b history scan backstop anything beyond the cap.
const phpRelayStartupWalkMax = 20000
// maxDetectionWindow returns the widest window any php_relay path evaluates
// over. The startup spool walker skips -H files older than this: such mail can
// no longer contribute to any current detection and only costs parse time.
// Lives in this linux-only file because the walker is its sole caller.
func (e *evaluator) maxDetectionWindow() time.Duration {
maxMin := 60 // Path 2 absolute-volume window is hardcoded at 60 min
relayCfg := e.config().EmailProtection.PHPRelay
if v := relayCfg.RateWindowMin; v > maxMin {
maxMin = v
}
if v := relayCfg.FanoutWindowMin; v > maxMin {
maxMin = v
}
if v := relayCfg.IPRecipientWindowHours * 60; relayCfg.IPRecipientSpread > 0 && v > maxMin {
maxMin = v
}
return time.Duration(maxMin) * time.Minute
}
// runStartupSpoolWalker walks the currently-queued -H files through the pipeline
// in REBUILD mode, then performs one re-evaluation pass over the reconstructed
// scriptStates. Findings are emitted ONLY in the re-evaluation pass, so the
// rebuild itself never produces duplicate findings for the same in-queue mail.
//
// Each message is attributed to its -H file ModTime, so mail queued over days is
// not compressed into one instant (which used to fabricate Path 1/2/4 bursts and
// mass-freeze the legit queue). Files older than the widest detection window are
// skipped, and the walk is bounded to the newest phpRelayStartupWalkMax files so
// a stuffed queue cannot stall startup.
func runStartupSpoolWalker(spoolRoot string, p *spoolPipeline) {
p.SetRebuilding(true)
now := time.Now()
cutoff := now.Add(-p.eng.maxDetectionWindow())
type walkEntry struct {
path string
mod time.Time
}
var entries []walkEntry
if subs, err := os.ReadDir(spoolRoot); err == nil {
for _, sub := range subs {
if !sub.IsDir() {
continue
}
subPath := filepath.Join(spoolRoot, sub.Name())
files, ferr := os.ReadDir(subPath)
if ferr != nil {
continue
}
for _, f := range files {
if !strings.HasSuffix(f.Name(), "-H") {
continue
}
full := filepath.Join(subPath, f.Name())
fi, serr := os.Stat(full)
if serr != nil {
continue
}
mod := fi.ModTime()
if mod.After(now) {
mod = now
}
if mod.Before(cutoff) {
continue // skip mail older than any detection window
}
entries = append(entries, walkEntry{path: full, mod: mod})
}
}
}
// Newest first, then cap: keep the most recent (most relevant) messages and
// bound startup work on a stuffed queue.
sort.Slice(entries, func(i, j int) bool { return entries[i].mod.After(entries[j].mod) })
if len(entries) > phpRelayStartupWalkMax {
entries = entries[:phpRelayStartupWalkMax]
}
// Replay chronologically so bounded per-script/per-account rings retain the
// newest events after their own caps are applied.
sort.Slice(entries, func(i, j int) bool { return entries[i].mod.Before(entries[j].mod) })
for _, e := range entries {
p.onFileAt(e.path, e.mod)
}
p.SetRebuilding(false)
// Re-evaluation pass.
snap := p.eng.scripts.Snapshot()
for k, s := range snap {
// We don't have per-script source IP in the snapshot; pass empty
// sourceIP. Path 4 (HTTP-IP fanout) is keyed off perIPWindow which
// was already populated in OnFile, so an empty SourceIP here just
// means the per-script Path 4 finding doesn't carry an IP -- the
// window itself still triggers correctly via direct OnFile calls
// during normal operation.
cpuser := ""
// Best-effort cpuser: read it from any active msgID's index entry.
if p.msgIndex != nil {
if ids, _ := s.snapshotActiveMsgs(); len(ids) > 0 {
if e, ok := p.msgIndex.Get(ids[0]); ok {
cpuser = e.CPUser
}
}
}
for _, f := range p.eng.evaluatePaths(k, "", cpuser, now) {
p.alerter(f)
}
}
}
// spoolSupervisor wraps a goroutine that may panic. After maxRestarts
// consecutive panics, it stops trying and invokes OnFailed (used to emit
// a Critical finding email_php_relay_watcher_failed).
type spoolSupervisor struct {
fn func(ctx context.Context)
maxRestarts int
OnFailed func()
}
//nolint:unused // consumed by daemon wiring (Task O2)
func newSpoolSupervisor(fn func(ctx context.Context), maxRestarts int) *spoolSupervisor {
return &spoolSupervisor{fn: fn, maxRestarts: maxRestarts, OnFailed: func() {}}
}
func (s *spoolSupervisor) Run(ctx context.Context) {
backoff := 100 * time.Millisecond
for attempt := 0; attempt <= s.maxRestarts; attempt++ {
select {
case <-ctx.Done():
return
default:
}
func() {
defer func() {
if r := recover(); r != nil {
_ = r // Panic recovered; loop will sleep + retry.
}
}()
s.fn(ctx)
}()
if ctx.Err() != nil {
return
}
if attempt == s.maxRestarts {
s.OnFailed()
return
}
select {
case <-ctx.Done():
return
case <-time.After(backoff):
}
if backoff < 5*time.Second {
backoff *= 2
}
}
}
package daemon
import (
"errors"
"fmt"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
)
// phpRelayUnavailableError explains why no PHP-relay controller exists.
//
// Every handler used to answer "phprelay controller not wired (Phase O2)",
// which reads as "this feature has not been written yet". It has been: the
// wiring runs from startPHPRelay, and startControlListener runs before it, so
// the controller is attached whenever it is built. What stops it being built
// is the platform gate -- a non-cPanel host, or the feature switched off (its
// default, and absent entirely from a config that predates the key).
//
// Reporting the gate that actually closed lets the operator act. Naming a
// phase number sends them to read source they do not have.
func phpRelayUnavailableError(cfg *config.Config, isCPanel bool) error {
if !isCPanel {
return errors.New("php_relay guard is inactive: it supports cPanel hosts only")
}
if cfg == nil || !cfg.EmailProtection.PHPRelay.Enabled {
return fmt.Errorf("php_relay guard is disabled: set email_protection.php_relay.enabled: true in csm.yaml, then %s", rehashAndRestartHint)
}
// Enabled on a supported platform and still absent: startup failed. Do
// not blame a setting that is already correct.
return errors.New("php_relay guard is enabled but did not start; check the daemon log for php_relay errors at startup")
}
// rehashAndRestartHint is the standard follow-up after a csm.yaml edit. The
// config hash is part of the integrity baseline, so a restart without a
// rehash makes the daemon refuse to start.
const rehashAndRestartHint = "run `csm rehash && systemctl restart csm`"
// phpRelayUnavailable is the handler-side wrapper: handlers hold a daemon,
// not a platform verdict.
func (c *ControlListener) phpRelayUnavailable() error {
var cfg *config.Config
if c.d != nil {
cfg = c.d.cfg
}
return phpRelayUnavailableError(cfg, platform.Detect().IsCPanel())
}
//go:build linux
package daemon
import (
"context"
"fmt"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/emailspool"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/store"
)
// phpRelayAuditPath is the production audit file. Linux-only because the
// writer is only attached from startPHPRelayLinux; tests pass t.TempDir
// paths directly to eximAuditWriterAt.
const phpRelayAuditPath = "/var/log/csm/php_relay_audit.jsonl"
type phpRelayPaths struct {
cpanelConfig string
spool string
historyLog string
auditLog string
}
// startPHPRelayLinux completes the PHP-relay wiring after the platform
// gate (in startPHPRelay) has confirmed cPanel + located exim. It is
// split out from daemon.go so the heavy linux-only types stay in this
// file; the cross-platform stub in php_relay_wiring_other.go keeps the
// darwin build clean.
//
// The whole pipeline is built up exactly once at daemon start; SIGHUP
// only reloads policies (handled in daemon.go where d.policies is
// already wired). Callers must not invoke this twice.
func startPHPRelayLinux(d *Daemon) {
startPHPRelayLinuxAt(d, phpRelayPaths{
cpanelConfig: "/var/cpanel/cpanel.config",
spool: "/var/spool/exim/input",
historyLog: platform.EximMainLog,
auditLog: phpRelayAuditPath,
})
}
// Explicit paths let startup tests exercise the real workers without touching
// the host's cPanel configuration, mail spool or audit log.
func startPHPRelayLinuxAt(d *Daemon, paths phpRelayPaths) {
// 1. cPanel hourly limit + Path 2b derivation.
limit, status := readCpanelHourlyLimit(paths.cpanelConfig)
switch status {
case cpanelLimitMissing, cpanelLimitUnparsable:
emitPHPRelayFinding(d, alert.Warning, "email_php_relay_cpanel_limit_unreadable",
"cpanel.config maxemailsperhour unreadable; assuming 100")
}
// 2. Policies (suspicious mailers, HTTP-proxy ranges). LoadPolicies
// returns a usable Policies even on partial-failure so we always
// have something to install on the daemon for SIGHUP reloads.
pol, err := emailspool.LoadPolicies(d.cfg.EmailProtection.PHPRelay.PoliciesDir)
if err != nil {
emitPHPRelayFinding(d, alert.Warning, "email_php_relay_policies_reload",
fmt.Sprintf("policies/email load encountered errors at startup: %v", err))
}
d.policies = pol
// 3. Window state (per-script, per-IP, per-account).
psw := newPerScriptWindow()
pip := newPerIPWindow(64)
pacct := newPerAccountWindow(5000)
// 4. Evaluator. Wires the cPanel-derived effective account limit so
// Path 2b activates as soon as the first message arrives.
prMetrics := newPHPRelayMetrics()
eng := newEvaluator(psw, pip, pacct, d.cfg, prMetrics)
eng.SetPolicies(pol)
_, enabled, capped := deriveEffectiveAccountLimit(d.cfg, limit, status)
if !enabled {
emitPHPRelayFinding(d, alert.Warning, "email_php_relay_path2b_disabled",
"Path 2b disabled: cPanel limit off and no operator override")
}
if capped {
emitPHPRelayFinding(d, alert.Warning, "email_php_relay_account_volume_capped",
"operator AccountVolumePerHour capped to 95% of cPanel hourly limit")
}
eng.SetAccountLimitSource(limit, status)
SetPHPRelayEvaluator(eng)
// 5. msgIDIndex + persister. Bbolt access is via store.Global() --
// the daemon does not hold a *store.DB directly; the global handle
// is the same singleton the rest of the codebase uses (sigWatcher,
// retention, etc.).
bdb := store.Global()
persister := newMsgIndexPersister(bdb, 4096, 100*time.Millisecond)
persister.SetErrorCallback(func(f alert.Finding) {
alert.TryEnqueue(d.alertCh, f)
})
persister.SetMetrics(prMetrics)
d.registerQueueSource("phprelay.index", persister)
persister.Start()
d.phpRelayShutdown = append(d.phpRelayShutdown, persister.Stop)
idx := newMsgIDIndex(persister, 200_000)
// Let queue-completion log lines reap activeMsgs so delivered mail is not
// re-frozen nor charged against the freeze rate-limit budget.
eng.SetMsgIndex(idx)
// 6. ignoreList with bbolt-backed restore.
ignores := newIgnoreList()
ignores.SetStore(bdb)
_ = ignores.Restore()
// 7. cpanel user domains resolver (Path 4 helper).
domains := newUserDomainsResolver()
// 8. Controller (constructed before the freezer so DryRunFn can
// thread the runtime/bbolt/yaml precedence into freeze decisions).
runner := defaultRunner{}
auditor := newStructuredAuditor(eximAuditWriterAt(d.cfg, paths.auditLog))
controller := &PHPRelayController{
eng: eng,
msgIndex: idx,
ignores: ignores,
actionDryRun: &runtimeBool{},
db: bdb,
runner: runner,
eximBin: eximBinary,
auditor: auditor,
enabled: true,
platform: "cpanel",
}
if d.controlListener != nil {
d.controlListener.phprelay = controller
}
// 9. Spool pipeline (Flow A) + autoFreezer (post-emit hook).
pipeline := newSpoolPipeline(eng, domains, pol, idx, ignores, func(f alert.Finding) {
alert.TryEnqueue(d.alertCh, f)
})
freezer := newAutoFreezer(psw, d.cfg, paths.spool, eximBinary,
runner, auditor, prMetrics, controller.DryRunFn())
d.autoFreezer = freezer
// 10. Startup walker BEFORE the watcher to rebuild script state for
// messages already on the spool when the daemon starts.
runStartupSpoolWalker(paths.spool, pipeline)
var previousWatcher *spoolWatcher
watcherFn := func(ctx context.Context) {
w, err := newSpoolWatcher(paths.spool, pipeline.OnFile)
if err != nil {
d.MarkWatcher("phprelay", false)
emitPHPRelayFinding(d, alert.Critical, "email_php_relay_watcher_failed", err.Error())
return
}
if previousWatcher != nil {
w.inheritQueueHealth(previousWatcher)
}
previousWatcher = w
d.registerQueueSource("phprelay", w)
d.MarkWatcher("phprelay", true)
w.SetMetrics(prMetrics)
w.SetOverflowHandler(func() {
emitPHPRelayFinding(d, alert.Critical, "email_php_relay_inotify_overflow",
"inotify queue overflow; running bounded recovery scan")
const phpRelayOverflowScanMax = 1000
n, truncated := runRecoveryScan(paths.spool, phpRelayOverflowScanMax, pipeline.OnFile)
if truncated {
emitPHPRelayFinding(d, alert.Critical, "email_php_relay_overflow_scan_truncated",
fmt.Sprintf("overflow recovery capped at %d files; older messages skipped (Path 2b backstops)", phpRelayOverflowScanMax))
}
emitPHPRelayFinding(d, alert.Warning, "email_php_relay_inotify_overflow_recovered",
fmt.Sprintf("recovery scan processed %d -H files", n))
})
w.Run(ctx)
}
sup := newSpoolSupervisor(watcherFn, 5)
sup.OnFailed = func() {
d.MarkWatcher("phprelay", false)
emitPHPRelayFinding(d, alert.Critical, "email_php_relay_watcher_failed", "supervisor exhausted restarts")
}
ctx := stopChContext(d)
d.wg.Add(1)
obs.Go("php-relay-supervisor", func() {
defer d.wg.Done()
sup.Run(ctx)
})
// 11. Retrospective Path 2b scan over exim_mainlog so account-volume
// alerts fire on the first hour boundary even after a daemon
// restart that lost in-memory state. Threaded through ctx so a
// large mainlog on a busy host cannot outlive shutdown.
d.wg.Add(1)
obs.Go("php-relay-history-scan", func() {
defer d.wg.Done()
ScanEximHistoryForPHPRelayAccountVolume(ctx, paths.historyLog, eng, time.Now(), func(f alert.Finding) {
alert.TryEnqueue(d.alertCh, f)
})
})
// 12. Flow E maintenance ticker.
d.wg.Add(1)
obs.Go("php-relay-flow-e", func() {
defer d.wg.Done()
runPHPRelayFlowE(d, ctx, eng, psw, pip, pacct, idx, persister, ignores, prMetrics)
})
}
// runPHPRelayFlowE drives Phase E maintenance (TTL sweeps + metric
// gauges). Single source of truth for php_relay TTLs; runs until ctx is
// cancelled (which happens when d.stopCh closes).
func runPHPRelayFlowE(
d *Daemon,
ctx context.Context,
eng *evaluator,
psw *perScriptWindow,
pip *perIPWindow,
pacct *perAccountWindow,
idx *msgIDIndex,
persister *msgIndexPersister,
ignores *ignoreList,
m *phpRelayMetrics,
) {
minTicker := time.NewTicker(1 * time.Minute)
fiveMinTicker := time.NewTicker(5 * time.Minute)
defer minTicker.Stop()
defer fiveMinTicker.Stop()
var formFullReported time.Time
for {
select {
case <-ctx.Done():
return
case <-minTicker.C:
now := time.Now()
_ = idx.SweepMemory(now.Add(-4 * time.Hour))
if m != nil {
m.MsgIDIndexSize.With("memory").Set(float64(idx.Len()))
}
if _, err := persister.SweepBolt(now.Add(-25 * time.Hour)); err != nil {
emitPHPRelayFinding(d, alert.Warning, "email_php_relay_sweep_failed", err.Error())
}
if _, err := ignores.SweepBolt(now); err != nil {
emitPHPRelayFinding(d, alert.Warning, "email_php_relay_sweep_failed", err.Error())
}
ignores.SweepExpired(now)
case <-fiveMinTicker.C:
now := time.Now()
cutoff25h := now.Add(-25 * time.Hour)
psw.PruneActiveMsgs(cutoff25h)
psw.SweepIdle(cutoff25h)
pip.SweepIdle(now.Add(-1 * time.Hour))
pacct.SweepIdle(now.Add(-24 * time.Hour))
formFullReported = sweepFormRelay(d, eng, now, formFullReported)
}
}
}
// sweepFormRelay expires Path 6 state and, when the client table hit its
// cap since the last sweep, reports it at most once a day. It returns when
// the table was last reported full.
func sweepFormRelay(d *Daemon, eng *evaluator, now, lastFull time.Time) time.Time {
window := time.Duration(eng.config().EmailProtection.PHPRelay.IPRecipientWindowHours) * time.Hour
if !eng.form.sweep(now, window) || !lastFull.IsZero() && now.Sub(lastFull) < 24*time.Hour {
return lastFull
}
emitPHPRelayFinding(d, alert.Warning, "email_php_form_relay_table_full",
fmt.Sprintf("Form relay client table reached its cap of %d; clients with the fewest outside recipients were dropped", eng.form.maxIPs))
return now
}
// Nonblocking delivery keeps a full findings channel from stopping mail
// supervision; the shared queue tracker retains any delivery loss.
func emitPHPRelayFinding(d *Daemon, sev alert.Severity, check, msg string) {
alert.TryEnqueue(d.alertCh, alert.Finding{
Severity: sev,
Check: check,
Message: msg,
Timestamp: time.Now(),
})
}
// stopChContext bridges the daemon's stopCh (chan struct{}) to a
// context.Context for components that take a ctx (spool watcher, Flow E
// ticker). The returned context is cancelled when stopCh closes.
func stopChContext(d *Daemon) context.Context {
ctx, cancel := context.WithCancel(context.Background())
go func() {
<-d.stopCh
cancel()
}()
return ctx
}
package daemon
import (
"os"
"github.com/pidginhost/csm/internal/phpshield"
)
const phpShieldScriptPath = phpshield.ScriptPath
const phpShieldMissingScriptWarning = "php_shield is enabled but not installed; protection is inactive until reinstalled; " +
"run `csm enable --php-shield`, restart PHP or the web server (e.g. systemctl restart lsws || apachectl graceful), " +
"then restart csm; or set php_shield.enabled: false"
var phpShieldStat = os.Stat
// phpShieldInstalled reports whether the PHP shield script is deployed on disk.
func phpShieldInstalled() bool {
_, err := phpShieldStat(phpShieldScriptPath)
return err == nil
}
// phpShieldWatchDecision decides, from the config flag and whether the shield
// script is installed, whether the daemon should receive Shield event
// datagrams and whether it should warn that the Shield is enabled but absent.
//
// When php_shield.enabled is true but the shield was never installed (or an
// upgrade wiped /opt/csm), opening the event socket would retry forever. Instead
// we warn once with a remediation hint, so the
// misprovision is surfaced rather than masked or spammed.
func phpShieldWatchDecision(enabled, scriptExists bool) (watch, warnNotInstalled bool) {
if !enabled {
return false, false
}
if scriptExists {
return true, false
}
return false, true
}
package daemon
import (
"context"
"errors"
"fmt"
"os"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/phptaintworker"
"github.com/pidginhost/csm/internal/workerproc"
)
// phpTaintWorkerTimeout is shorter than the check-side outer deadline. The
// supervisor must have time to kill and reap a stuck parser before the shared
// deep-content walk gives up on the request.
const phpTaintWorkerTimeout = 20 * time.Second
// phpTaintWorkerUnitPrefix names the PHP taint worker units. Startup kills
// any left by a daemon that died, matching on the prefix plus "-".
const phpTaintWorkerUnitPrefix = "csm-phptaint-worker"
// phpTaintWorkerLauncher is a seam for tests that must not start units.
var phpTaintWorkerLauncher = workerproc.UnitLauncher
func (d *Daemon) initPHPTaintAnalyzer() error {
if d.binaryPath == "" {
return errors.New("phptaint worker: daemon binary unknown")
}
logf := func(format string, args ...any) {
fmt.Fprintf(os.Stderr, "[%s] phptaint-worker: "+format+"\n", append([]any{ts()}, args...)...)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
if err := stopParseWorkerUnits(ctx, phpTaintWorkerUnitPrefix+"-"); err != nil {
logf("stop leftover units: %v", err)
}
cancel()
unit := phptaintworker.UnitConfig(d.binaryPath, phpTaintWorkerUnitPrefix, logf)
unit.InaccessiblePaths = parseWorkerHiddenPaths()
launch := phpTaintWorkerLauncher(unit)
sup, err := phptaintworker.NewSupervisor(phptaintworker.SupervisorConfig{
Launch: func(ctx context.Context) (workerproc.Process, error) {
// Keep preflight failures inside the supervisor: an absent analyzer
// disables dispatch and would hide these files' coverage gaps.
if err := checkParseWorkerCommand(unit.Command); err != nil {
return nil, fmt.Errorf("phptaint worker: %w", err)
}
return launch(ctx)
},
Timeout: phpTaintWorkerTimeout,
Log: logf,
})
if err != nil {
return err
}
d.phpTaintSup = sup
d.registerQueueSource("php_taint", sup)
checks.SetPHPTaintAnalyzer(sup)
return nil
}
func (d *Daemon) stopPHPTaintAnalyzer() {
checks.SetPHPTaintAnalyzer(nil)
if d.phpTaintSup == nil {
return
}
_ = d.phpTaintSup.Stop()
d.phpTaintSup = nil
}
//go:build linux
package daemon
import (
"errors"
"fmt"
"io/fs"
"math"
"os"
"strconv"
"strings"
"syscall"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/platform"
)
// procRootDir is the procfs mount used by the ancestry walker. A var so
// tests can point it at a synthetic tree.
var procRootDir = "/proc"
// maxAncestryDepth bounds the PPid walk. Real package-manager chains are
// short (cpio <- weak-modules <- dnf <- systemd); a cap keeps a hostile or
// corrupt PPid chain from turning event handling into an unbounded scan.
const maxAncestryDepth = 16
// Injection points for tests; defaults are the real implementations.
var (
tmpExecPkgWindow = checks.PkgManagerRecentlyActive
tmpExecAncestry = ancestryEvidenceFor
tmpExecDemote = demoteTmpExec
)
// ancestryEvidence is what one walk of a process chain proved. The signals
// are kept apart because they are corroborated differently: a comm can be set
// by the process itself, so a package manager's needs a package transaction
// in the same window and an initramfs builder's needs a live root process chain,
// while an exe under a panel root cannot be faked and stands on its own.
type ancestryEvidence struct {
packageManager bool
panelTool bool
initramfsBuilder bool
// vanished reports that the walk stopped at a process that had already
// exited, with no non-root or unknown credentials encountered first.
// Read errors alone do not establish exit: see procGone.
vanished bool
}
// procGone reports whether a procfs read failed because the process no longer
// exists. Opening a file of a process that has been reaped fails with ENOENT;
// a process that exits between the open and the read fails the read with
// ESRCH. Any other error says nothing about the process.
func procGone(err error) bool {
return errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ESRCH)
}
func (e ancestryEvidence) none() bool {
return !e.packageManager && !e.panelTool && !e.initramfsBuilder
}
// procAncestryEvidence walks pid's PPid chain through procfs and reports what
// the chain proves. Any read or parse failure ends the walk with what was
// collected so far: a vanished process cannot prove provenance, so a chain
// that proved nothing leaves the alert at its original severity.
func procAncestryEvidence(pid int32, panelRoots []string) ancestryEvidence {
var ev ancestryEvidence
rootChain := true
for depth := 0; depth < maxAncestryDepth && pid > 1; depth++ {
dir := fmt.Sprintf("%s/%d", procRootDir, pid)
// #nosec G304 -- procfs pseudo-files under a fixed root; pid is from the fanotify event.
commRaw, err := os.ReadFile(dir + "/comm")
if err != nil {
ev.vanished = procGone(err) && rootChain
return ev
}
comm := strings.TrimSpace(string(commRaw))
if isPackageManagerComm(comm) {
ev.packageManager = true
}
if !ev.panelTool && procExeIsPanelTool(dir, panelRoots) {
ev.panelTool = true
}
if ev.packageManager && ev.panelTool {
return ev
}
// #nosec G304 -- procfs pseudo-files under a fixed root; pid is from the fanotify event.
status, err := os.ReadFile(dir + "/status")
if err != nil {
ev.vanished = procGone(err) && rootChain
return ev
}
ppid := int32(-1)
root := false
statusComm := ""
for _, line := range strings.Split(string(status), "\n") {
switch {
case strings.HasPrefix(line, "Name:"):
statusComm = strings.TrimSpace(strings.TrimPrefix(line, "Name:"))
case strings.HasPrefix(line, "PPid:"):
n, convErr := strconv.ParseInt(strings.TrimSpace(strings.TrimPrefix(line, "PPid:")), 10, 32)
if convErr != nil {
return ev
}
ppid = int32(n)
case strings.HasPrefix(line, "Uid:"):
root = statusUIDIsRoot(line)
}
}
// A privileged ancestor does not vouch for descendants that dropped
// privileges. Read the name with the credentials, not from an earlier
// comm lookup that may have addressed a previous occupant of the PID.
rootChain = rootChain && root
if rootChain && isInitramfsBuilderComm(statusComm) {
ev.initramfsBuilder = true
}
// The kernel prints parent 0 for a task it is releasing while the
// status is read. Only init and the kernel thread parent, PIDs 1
// and 2, really have no parent.
if ppid == 0 && pid > 2 {
ev.vanished = rootChain
return ev
}
pid = ppid
}
return ev
}
// statusUIDIsRoot reports whether a /proc status Uid line names root as both
// the real and the effective user. A set-uid helper started by another user
// keeps that user's real id, so it never qualifies.
func statusUIDIsRoot(line string) bool {
ids := strings.Fields(strings.TrimPrefix(line, "Uid:"))
return len(ids) >= 2 && ids[0] == "0" && ids[1] == "0"
}
// procExeIsPanelTool reports whether the process at procDir is running a
// binary the control panel installed. The path is tested before the file is
// stat()ed because an ancestor's exe can point anywhere.
func procExeIsPanelTool(procDir string, roots []string) bool {
if len(roots) == 0 {
return false
}
exe, err := os.Readlink(procDir + "/exe")
if err != nil || !exeInPanelRoot(exe, roots) {
return false
}
mode, uid, err := exeStat(exe)
if err != nil {
return false
}
return panelToolExeTrusted(exe, roots, mode, uid)
}
// panelToolRoots returns the detected panel's own tool directories. Detection
// is cached, so this is cheap enough to call per event. A var so tests can
// describe a cPanel host without one.
var panelToolRoots = func() []string { return platform.Detect().PanelToolRoots() }
// wireAncestryProvenance installs the ancestry hook the checks package uses to
// rescore sensitive-file findings. The /proc walk needs no BPF, so this runs
// on every Linux host; wireAncestryCache only adds a cache that survives
// process exit.
func wireAncestryProvenance() { checks.AncestryProvenance = ancestryProvenanceReason }
// ancestryEvidenceFor preserves cached panel and package evidence after
// process exit. Initramfs evidence requires a live root process chain;
// missing credentials leave that arm at the original severity.
func ancestryEvidenceFor(pid int32) ancestryEvidence {
if pid <= 0 {
return ancestryEvidence{}
}
roots := panelToolRoots()
var cached ancestryEvidence
if probe := cachedAncestryEvidence; probe != nil {
cached = probe(uint32(pid), roots)
if cached.panelTool {
return cached
}
}
// The cache has neither both current credentials nor a pinned parent
// chain. Only live procfs evidence can authorize initramfs demotion.
live := procAncestryEvidence(pid, roots)
live.packageManager = live.packageManager || cached.packageManager
return live
}
// cachedAncestryEvidence reads the same evidence out of the BPF processctx
// cache. Nil on hosts built or running without BPF.
var cachedAncestryEvidence func(pid uint32, panelRoots []string) ancestryEvidence
const (
pkgAncestryReason = "ancestor is package manager"
panelToolAncestryReason = "ancestor is control panel maintenance"
)
// ancestryProvenanceReason names the trusted component behind pid, or "" when
// the chain proves nothing. Panel tooling is reported first: its evidence is
// an executable an unprivileged user cannot place, while a package-manager
// comm is only a name.
func ancestryProvenanceReason(pid uint32) string {
if pid == 0 || pid > math.MaxInt32 {
return ""
}
ev := ancestryEvidenceFor(int32(pid))
switch {
case ev.panelTool:
return panelToolAncestryReason
case ev.packageManager:
return pkgAncestryReason
default:
return ""
}
}
// demoteTmpExec decides whether an executable_in_tmp_realtime finding is
// demoted from Critical to Warning on what its writer's ancestry proved. The
// file must be root-owned in every case -- a non-root attacker can never
// qualify -- and then one of three provenance arms has to hold:
//
// - the writer descends from the control panel's own tooling, proven by an
// ancestor's executable path,
// - or it descends from a package manager AND a package-manager log was
// touched within the provenance window. The comm behind that arm is
// attacker-settable, so it is never the sole gate,
// - or its live ancestry through an initramfs builder runs entirely as root.
//
// The finding is rescored, never dropped by this decision, so the evidence
// trail survives even if the heuristic is wrong.
func demoteTmpExec(uid uint32, ev ancestryEvidence, now time.Time) (bool, string) {
if uid != 0 {
return false, ""
}
window := tmpExecPkgWindow(now)
panelRooted := len(panelToolRoots()) > 0
switch {
case panelRooted && ev.panelTool:
return true, "control panel maintenance ancestry"
case window && ev.packageManager:
return true, "package manager ancestry during active package window"
case ev.initramfsBuilder:
return true, "initramfs build ancestry"
}
return false, ""
}
//go:build linux
package daemon
import (
"errors"
"fmt"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/checks"
)
// placementKind names how a directory entry appeared in a watched tree.
type placementKind uint8
const (
placementRename placementKind = iota + 1
placementLink
placementCreate
)
// placementEvent is a file or directory renamed, linked or created into a
// watched tree. Close-write never fires for these, so a placement backend
// reports them with the identity the kernel saw at the time.
type placementEvent struct {
Path string
Dev, Ino uint64
IsDir bool
PID int32
Kind placementKind
At time.Time
}
// eventPlacedIn is the synthetic mask bit for a placed-in file. No fanotify
// event uses bit 63. The file's content is complete: it was written elsewhere
// before it appeared at this path.
const eventPlacedIn uint64 = 1 << 63
var errPlacementUnverified = errors.New("placed file not found with the reported identity")
// placementRetryDelays cover a backend that reports the entry before the
// rename or link is visible to a lookup.
var placementRetryDelays = [...]time.Duration{50 * time.Millisecond, 200 * time.Millisecond}
// handlePlacement runs a placed-in file through the same admission, analysis
// and response as a write at that path. It waits out a rename or link the
// kernel reported before it took effect.
func (fm *FileMonitor) handlePlacement(ev placementEvent) {
fm.verifyPlacement(ev, placementRetryDelays[:])
}
// handleResolvedPlacement is handlePlacement for a backend that read the
// identity from the entry after the event: a mismatch is a replacement made
// since, which no wait undoes.
func (fm *FileMonitor) handleResolvedPlacement(ev placementEvent) {
fm.verifyPlacement(ev, nil)
}
func (fm *FileMonitor) verifyPlacement(ev placementEvent, retryDelays []time.Duration) {
// Recovery copies keep their original names; analysing them repeats
// detections CSM already acted on.
if checks.InQuarantine(ev.Path) {
return
}
fd, isDir, err := openPlacedFile(ev.Path, ev.Dev, ev.Ino, retryDelays)
if err != nil {
fm.placementUnverified.Add(1)
if fanotifyPlacementUnverified != nil {
fanotifyPlacementUnverified.Inc()
}
return
}
if isDir {
_ = unix.Close(fd)
fm.queuePlacedDir(ev.Path, ev.Dev, ev.Ino, ev.PID, ev.At, time.Time{})
return
}
fm.handlePlacedFile(fd, ev.PID, ev.At)
}
// recoverDroppedPlacement hands a placement a full backend queue refused to
// overflow recovery: a rescan of a file's directory in placement mode, or the
// walk of a moved-in directory, which the rescan does not descend into.
func (fm *FileMonitor) recoverDroppedPlacement(ev placementEvent) {
// The ring reports intent before the syscall commits. Recovery has no
// descriptor to prove completion, so give it the same visibility window
// as the identity-checking workers without blocking the ring reader.
ready := time.Now()
for _, delay := range placementRetryDelays {
ready = ready.Add(delay)
}
fm.recoverPlacementAfter(ev, ready)
}
func (fm *FileMonitor) recoverPlacementAfter(ev placementEvent, ready time.Time) {
if ev.IsDir {
fm.queuePlacedDir(ev.Path, ev.Dev, ev.Ino, ev.PID, ev.At, ready)
return
}
fm.recordDroppedEventAfter(fileEvent{path: ev.Path, mask: eventPlacedIn}, ready)
}
// handlePlacedFile takes ownership of fd, a regular file whose identity the
// caller already verified.
func (fm *FileMonitor) handlePlacedFile(fd int, pid int32, at time.Time) {
path, _, ok := eventFDPath(fd)
if !ok {
fm.rejectUnresolved(fd)
return
}
event, admit := fm.classifyEvent(path, fd)
if admit && fm.analyzedUnchanged(event) {
fm.placementSkipped.Add(1)
if fanotifyPlacementSkipped != nil {
fanotifyPlacementSkipped.Inc()
}
// Only the content analysis is redundant. The deletion probe checks
// the path a candidate was tracked at, and this path is new.
event.pid, event.mask, event.at = pid, eventPlacedIn, at
fm.observeDropperCandidate(event, resolveProcessInfo(pid))
_ = unix.Close(fd)
return
}
fm.admitEvent(fd, event, admit, pid, eventPlacedIn, at)
}
// analyzedUnchanged reports whether this file version was analysed recently
// with the same outcome-relevant class it gets at its new path.
func (fm *FileMonitor) analyzedUnchanged(ev fileEvent) bool {
// Restore detection keys on the destination: a payload renamed back over
// the path it was removed from is the restore, whatever ran before.
if _, removed := removedFileAt(ev.path); removed {
return false
}
snapshot, ok := snapshotForAnalysis(ev.fd, fm.eventClass(ev))
return ok && fm.recent.Seen(snapshot.key, time.Now())
}
// openPlacedFile opens path without following a final symlink and returns the
// descriptor only when it is the regular file or directory the kernel
// reported. A different inode at the path is never analysed in its place.
func openPlacedFile(path string, dev, ino uint64, retryDelays []time.Duration) (fd int, isDir bool, err error) {
for attempt := 0; ; attempt++ {
fd, isDir, retry, err := openPlacedOnce(path, dev, ino)
if err == nil {
return fd, isDir, nil
}
if !retry || attempt == len(retryDelays) {
return -1, false, fmt.Errorf("%w: %s: %v", errPlacementUnverified, path, err)
}
time.Sleep(retryDelays[attempt])
}
}
// openPlacedOnce reports retry for the two states a pending rename or link
// can leave behind: no entry yet, or the old inode still in place.
func openPlacedOnce(path string, dev, ino uint64) (fd int, isDir, retry bool, err error) {
return openPlacedAt(unix.AT_FDCWD, path, dev, ino)
}
// openPlacedAt is openPlacedOnce for name relative to dirfd.
func openPlacedAt(dirfd int, name string, dev, ino uint64) (fd int, isDir, retry bool, err error) {
// A pending rename may replace any entry, including a socket or device.
// O_PATH lets identity checks precede I/O and keeps mismatches retryable.
probe, err := unix.Openat(dirfd, name, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
if err != nil {
return -1, false, errors.Is(err, unix.ENOENT), err
}
defer func() { _ = unix.Close(probe) }()
var st unix.Stat_t
if err = unix.Fstat(probe, &st); err != nil {
return -1, false, false, err
}
if uint64(st.Dev) != dev || st.Ino != ino {
return -1, false, true, errors.New("identity mismatch")
}
switch st.Mode & unix.S_IFMT {
case unix.S_IFREG:
case unix.S_IFDIR:
isDir = true
default:
return -1, false, false, errors.New("not a regular file or directory")
}
// Follow only our own procfs descriptor link, not the mutable entry.
// The pinned inode cannot become a special file between stat and open.
fd, err = unix.Open(fmt.Sprintf("/proc/self/fd/%d", probe), unix.O_RDONLY|unix.O_NONBLOCK|unix.O_CLOEXEC, 0)
if err != nil {
return -1, false, false, err
}
return fd, isDir, false, nil
}
// Recovery runs inline under the existing reconcile budget; enqueueing again
// would lose the file to the same backpressure that started recovery.
func (fm *FileMonitor) reconcilePlacedFile(path string, dev, ino uint64) bool {
fd, isDir, _, err := openPlacedOnce(path, dev, ino)
if err != nil {
return errors.Is(err, unix.ENOENT)
}
defer func() { _ = unix.Close(fd) }()
if isDir {
return true
}
if event, admit := fm.classifyEvent(path, fd); admit {
event.mask, event.at = eventPlacedIn, time.Now()
fileAnalyzer(fm, event)
}
return true
}
//go:build !(linux && bpf)
package daemon
import (
"context"
"github.com/pidginhost/csm/internal/bpf"
)
// placementBPF is the no-tag placeholder for the BPF placement monitor in
// placement_bpf.go.
type placementBPF struct{}
func (p *placementBPF) Mode() string { return "bpf" }
func (p *placementBPF) EventCount() uint64 { return 0 }
func (p *placementBPF) Run(_ context.Context) {}
func startPlacementBPF(_ context.Context, _ *FileMonitor) (*placementBPF, error) {
return nil, bpf.ErrNotBuilt
}
// Code generated by bpf2go; DO NOT EDIT.
//go:build 386 || amd64
package placement_bpfprog
import (
"bytes"
_ "embed"
"fmt"
"io"
"structs"
"github.com/cilium/ebpf"
)
type PlacementCsmQueueStats struct {
_ structs.HostLayout
Lost uint64
Submitted uint64
}
type PlacementPlacementEvent struct {
_ structs.HostLayout
Kind uint8
IsDir uint8
Depth uint8
Truncated uint8
Pid uint32
Dev uint64
Ino uint64
Path [4096]int8
}
type PlacementPlacementSlot struct {
_ structs.HostLayout
Busy uint32
_ [4]byte
Ev PlacementPlacementEvent
}
// Names of all BPF objects in the ELF.
//
// Used for safe lookups in a Collection or CollectionSpec.
const (
PlacementMapDevsWatched = "devs_watched"
PlacementMapEvents = "events"
PlacementMapQueueStats = "queue_stats"
PlacementMapScratch = "scratch"
PlacementProgCsmInodeLink = "csm_inode_link"
PlacementProgCsmInodeRename = "csm_inode_rename"
PlacementVarUnused = "unused"
)
// LoadPlacement returns the embedded CollectionSpec for Placement.
func LoadPlacement() (*ebpf.CollectionSpec, error) {
reader := bytes.NewReader(_PlacementBytes)
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
if err != nil {
return nil, fmt.Errorf("can't load Placement: %w", err)
}
return spec, err
}
// LoadPlacementObjects loads Placement and converts it into a struct.
//
// The following types are suitable as obj argument:
//
// *PlacementObjects
// *PlacementPrograms
// *PlacementMaps
//
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
func LoadPlacementObjects(obj any, opts *ebpf.CollectionOptions) error {
spec, err := LoadPlacement()
if err != nil {
return err
}
return spec.LoadAndAssign(obj, opts)
}
// PlacementSpecs contains maps and programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type PlacementSpecs struct {
PlacementProgramSpecs
PlacementMapSpecs
PlacementVariableSpecs
}
// PlacementProgramSpecs contains programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type PlacementProgramSpecs struct {
CsmInodeLink *ebpf.ProgramSpec `ebpf:"csm_inode_link"`
CsmInodeRename *ebpf.ProgramSpec `ebpf:"csm_inode_rename"`
}
// PlacementMapSpecs contains maps before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type PlacementMapSpecs struct {
DevsWatched *ebpf.MapSpec `ebpf:"devs_watched"`
Events *ebpf.MapSpec `ebpf:"events"`
QueueStats *ebpf.MapSpec `ebpf:"queue_stats"`
Scratch *ebpf.MapSpec `ebpf:"scratch"`
}
// PlacementVariableSpecs contains global variables before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type PlacementVariableSpecs struct {
Unused *ebpf.VariableSpec `ebpf:"unused"`
}
// PlacementObjects contains all objects after they have been loaded into the kernel.
//
// It can be passed to LoadPlacementObjects or ebpf.CollectionSpec.LoadAndAssign.
type PlacementObjects struct {
PlacementPrograms
PlacementMaps
PlacementVariables
}
func (o *PlacementObjects) Close() error {
return _PlacementClose(
&o.PlacementPrograms,
&o.PlacementMaps,
)
}
// PlacementMaps contains all maps after they have been loaded into the kernel.
//
// It can be passed to LoadPlacementObjects or ebpf.CollectionSpec.LoadAndAssign.
type PlacementMaps struct {
DevsWatched *ebpf.Map `ebpf:"devs_watched"`
Events *ebpf.Map `ebpf:"events"`
QueueStats *ebpf.Map `ebpf:"queue_stats"`
Scratch *ebpf.Map `ebpf:"scratch"`
}
func (m *PlacementMaps) Close() error {
return _PlacementClose(
m.DevsWatched,
m.Events,
m.QueueStats,
m.Scratch,
)
}
// PlacementVariables contains all global variables after they have been loaded into the kernel.
//
// It can be passed to LoadPlacementObjects or ebpf.CollectionSpec.LoadAndAssign.
type PlacementVariables struct {
Unused *ebpf.Variable `ebpf:"unused"`
}
// PlacementPrograms contains all programs after they have been loaded into the kernel.
//
// It can be passed to LoadPlacementObjects or ebpf.CollectionSpec.LoadAndAssign.
type PlacementPrograms struct {
CsmInodeLink *ebpf.Program `ebpf:"csm_inode_link"`
CsmInodeRename *ebpf.Program `ebpf:"csm_inode_rename"`
}
func (p *PlacementPrograms) Close() error {
return _PlacementClose(
p.CsmInodeLink,
p.CsmInodeRename,
)
}
func _PlacementClose(closers ...io.Closer) error {
for _, closer := range closers {
if err := closer.Close(); err != nil {
return err
}
}
return nil
}
// Do not access this directly.
//
//go:embed placement_x86_bpfel.o
var _PlacementBytes []byte
//go:build linux
package daemon
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/queuehealth"
)
const (
// placedDirEntryBudget bounds what one placed directory may cost
// realtime protection; the deep scan covers anything past it.
placedDirEntryBudget = 20000
// placedDirTimeBudget bounds one pass so a slow tree cannot hold back
// the directories queued behind it. The walk resumes on a later pass.
placedDirTimeBudget = 30 * time.Second
// placedDirQueueCap bounds waiting directories. Eviction is oldest
// first and always reported.
placedDirQueueCap = 256
)
// Vars so tests can lower the budgets.
var (
placedDirEntryLimit = placedDirEntryBudget
placedDirTimeLimit = placedDirTimeBudget
placedDirQueueLimit = placedDirQueueCap
)
const (
// placedDirHeadroomPoll is how often a walk waiting for analyzer
// headroom looks again; a channel offers no wake on free space.
placedDirHeadroomPoll = 10 * time.Millisecond
// placedDirReadBatch bounds each directory read, so a listing stops
// soon after it outgrows the entry budget.
placedDirReadBatch = 256
)
// placedDirReadNames lets tests count what a walk reads.
var placedDirReadNames = (*os.File).Readdirnames
const placedDirIncompleteCheck = "placed_dir_incomplete_realtime"
// placedDir is a directory renamed or linked into a watched tree. Its files
// raised no events of their own, so the walk is their realtime check.
type placedDir struct {
path string
dev, ino uint64
pid int32
// at is when the directory was placed. Walked files carry it, not the
// walk time: restore detection checks the writer started before it.
at time.Time
procInfo string
ticket queuehealth.Ticket
// Unverified BPF overflow events wait for the rename before walking.
ready time.Time
// cursor is the last entry visited, as components below path. Names
// are walked in sorted order, so a resumed pass skips up to it.
cursor []string
entries int
failed bool
// overflowed is sticky: an over-budget listing walks a readdir-order
// subset, so names below a later cursor may never have been read.
overflowed bool
// Only time waiting for analyzer room counts as starvation. Preserve it
// across passes so slow directory reads cannot cause endless retries.
headroomWait time.Duration
}
// queuePlacedDir hands a placed-in directory to the walk worker. It never
// walks on the caller's goroutine.
func (fm *FileMonitor) queuePlacedDir(path string, dev, ino uint64, pid int32, at, ready time.Time) {
fm.initQueueHealth()
// Resolved now: by the time a walk is cut the renamer may be gone.
procInfo := resolveProcessInfo(pid)
now := time.Now()
fm.placedDirMu.Lock()
fm.placedDirs = append(fm.placedDirs, placedDir{
path: path, dev: dev, ino: ino, pid: pid, at: at, procInfo: procInfo,
ticket: fm.placedDirHealth.Begin(now), ready: ready,
})
evicted := fm.evictPlacedDirsLocked(now)
fm.placedDirMu.Unlock()
fm.reportEvictedPlacedDirs(evicted)
select {
case fm.placedDirSig <- struct{}{}:
default:
}
}
// evictPlacedDirsLocked drops the oldest waiting directories past the cap.
// The caller holds placedDirMu and reports what it returns.
func (fm *FileMonitor) evictPlacedDirsLocked(now time.Time) []placedDir {
over := len(fm.placedDirs) - placedDirQueueLimit
if over <= 0 {
return nil
}
evicted := slices.Clone(fm.placedDirs[:over])
fm.placedDirs = slices.Delete(fm.placedDirs, 0, over)
for _, dir := range evicted {
dir.ticket.Reject(now)
}
return evicted
}
func (fm *FileMonitor) reportEvictedPlacedDirs(evicted []placedDir) {
for _, dir := range evicted {
fm.reportPlacedDirIncomplete(dir, fmt.Sprintf("more than %d moved-in directories were waiting", placedDirQueueLimit))
}
}
func (fm *FileMonitor) reportPlacedDirIncomplete(dir placedDir, reason string) {
fm.sendAlertWithPath(alert.High, placedDirIncompleteCheck,
fmt.Sprintf("Realtime protection did not finish checking a directory moved into a site: %s", dir.path),
fmt.Sprintf("Realtime protection stopped before it reached every file in this directory: %s. The next deep scan covers the files it did not check.", reason),
dir.path, dir.procInfo)
}
// placedDirLoop is the single walk worker. Run starts it; drainAndClose
// waits for it before closing the analyzer queue the walk feeds.
func (fm *FileMonitor) placedDirLoop() {
defer fm.placedDirWG.Done()
for {
select {
case <-fm.stopCh:
return
case <-fm.placedDirSig:
}
for fm.walkNextPlacedDir() {
}
}
}
// walkNextPlacedDir runs one bounded pass over the oldest waiting directory
// and reports whether there was one.
func (fm *FileMonitor) walkNextPlacedDir() bool {
select {
case <-fm.stopCh:
return false
default:
}
fm.placedDirMu.Lock()
if len(fm.placedDirs) == 0 {
fm.placedDirMu.Unlock()
return false
}
dir := fm.placedDirs[0]
fm.placedDirs = slices.Delete(fm.placedDirs, 0, 1)
fm.placedDirMu.Unlock()
dir.ticket.Start(time.Now())
if wait := time.Until(dir.ready); wait > 0 {
timer := time.NewTimer(wait)
select {
case <-fm.stopCh:
timer.Stop()
dir.ticket.Reject(time.Now())
return false
case <-timer.C:
}
}
yield, reason := fm.walkPlacedDirSafe(&dir)
now := time.Now()
switch {
case yield:
// Behind newer arrivals, so one large tree cannot starve them.
dir.ticket.Requeue(now)
fm.placedDirMu.Lock()
fm.placedDirs = append(fm.placedDirs, dir)
evicted := fm.evictPlacedDirsLocked(now)
fm.placedDirMu.Unlock()
fm.reportEvictedPlacedDirs(evicted)
case reason != "":
dir.ticket.Reject(now)
fm.reportPlacedDirIncomplete(dir, reason)
default:
dir.ticket.Finish(now)
}
return true
}
// walkPlacedDirSafe contains a panic the way analyzeFileSafe does: one
// crafted tree must not stop the walks queued behind it.
func (fm *FileMonitor) walkPlacedDirSafe(dir *placedDir) (yield bool, reason string) {
defer func() {
if r := recover(); r != nil {
fm.reportScannerPanic(dir.path, r)
yield, reason = false, "the walk failed"
}
}()
w := placedDirWalk{fm: fm, dir: dir, deadline: time.Now().Add(placedDirTimeLimit)}
yield, reason = w.walk(nil, dir.dev, dir.ino, dir.cursor)
switch {
case yield && w.starved && w.files == 0:
// A whole pass without analyzer headroom: realtime is saturated,
// so say so instead of retrying this directory indefinitely.
return false, "the realtime analyzers stayed busy for a whole walk pass"
case yield || reason != "":
return yield, reason
case dir.overflowed:
return false, fmt.Sprintf("it holds more than %d entries", placedDirEntryLimit)
case dir.failed:
return false, "some entries could not be read"
}
return false, ""
}
type placedDirWalk struct {
fm *FileMonitor
dir *placedDir
deadline time.Time
visited int
// files counts files handed to analysis this pass; starved records
// that the pass ended waiting for analyzer headroom.
files int
starved bool
}
// yield reports whether this pass must stop: the monitor is stopping, or
// the time budget is spent after at least one entry, so every pass advances.
func (w *placedDirWalk) yield() bool {
return w.stopping() || w.visited > 0 && !time.Now().Before(w.deadline)
}
func (w *placedDirWalk) stopping() bool {
select {
case <-w.fm.stopCh:
return true
default:
}
return false
}
// headroom waits until the analyzer queue is at most half full, so a walk
// never admits faster than the analyzers drain and leaves room for live
// events. It reports false when the pass must end first.
func (w *placedDirWalk) headroom() bool {
limit := max(cap(w.fm.analyzerCh)/2, 1)
started := time.Now()
deadline := w.deadline
// A slow listing may already have spent this pass. Give the first
// pending file its remaining congestion budget before repeating that I/O.
if !started.Before(deadline) {
deadline = started.Add(placedDirTimeLimit - w.dir.headroomWait)
}
for len(w.fm.analyzerCh) >= limit {
waited := w.dir.headroomWait + time.Since(started)
if waited >= placedDirTimeLimit {
w.starved = true
return false
}
wait := min(time.Until(deadline), placedDirTimeLimit-waited)
if wait <= 0 {
w.dir.headroomWait = waited
return false
}
select {
case <-w.fm.stopCh:
return false
case <-time.After(min(wait, placedDirHeadroomPoll)):
}
}
w.dir.headroomWait = 0
return true
}
// walk visits rel, the directory below the placed root that its parent's
// listing saw as dev+ino, from after, the rest of the resume cursor.
func (w *placedDirWalk) walk(rel []string, dev, ino uint64, after []string) (yield bool, reason string) {
fd, ok := w.open(rel, dev, ino)
if !ok {
return false, ""
}
defer func() {
if fd >= 0 {
_ = unix.Close(fd)
}
}()
from := ""
if len(after) > 0 {
from = after[0]
}
names, over, err := readDirNames(fd, from, placedDirEntryLimit-w.dir.entries)
if err != nil {
w.dir.failed = true
return false, ""
}
w.dir.overflowed = w.dir.overflowed || over
budgetCut := fmt.Sprintf("it holds more than %d entries", placedDirEntryLimit)
for _, name := range names {
var below []string
revisit := false
if len(after) > 0 {
if name < after[0] {
continue
}
if name == after[0] {
revisit, below = true, after[1:]
}
}
path := append(slices.Clip(rel), name)
if !revisit {
if w.yield() {
return true, ""
}
if w.dir.entries >= placedDirEntryLimit {
return false, budgetCut
}
}
var st unix.Stat_t
statErr := unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW)
regular := statErr == nil && st.Mode&unix.S_IFMT == unix.S_IFREG
if !revisit {
// Yield before the file, so the next pass starts with it.
if regular && !w.headroom() {
return true, ""
}
w.dir.entries++
w.visited++
w.dir.cursor = path
}
if statErr != nil {
w.dir.failed = w.dir.failed || !errors.Is(statErr, unix.ENOENT)
continue
}
switch st.Mode & unix.S_IFMT {
case unix.S_IFREG:
if !revisit {
w.file(fd, name, &st)
}
case unix.S_IFDIR:
// Hold no descriptor for this level while below it, so a deep
// tree cannot exhaust the daemon's descriptors.
_ = unix.Close(fd)
fd = -1
if yield, reason = w.walk(path, uint64(st.Dev), st.Ino, below); yield || reason != "" {
return yield, reason
}
if fd, ok = w.open(rel, dev, ino); !ok {
return false, ""
}
}
}
return false, ""
}
// open reopens a directory of the walk by path. Only the final component is
// opened without following a symlink; the identity check then accepts only
// the directory the walk listed, so a swapped ancestor makes the reopen fail
// rather than read another tree. A directory moved away or replaced raised
// its own event and is not lost.
func (w *placedDirWalk) open(rel []string, dev, ino uint64) (int, bool) {
path := filepath.Join(append([]string{w.dir.path}, rel...)...)
fd, isDir, retry, err := openPlacedOnce(path, dev, ino)
if err != nil {
w.dir.failed = w.dir.failed || !retry
return -1, false
}
if !isDir {
_ = unix.Close(fd)
return -1, false
}
return fd, true
}
// file runs one walked file through the placed-file path, opened relative
// to its listed directory with the identity the listing saw.
func (w *placedDirWalk) file(dirfd int, name string, st *unix.Stat_t) {
fd, isDir, retry, err := openPlacedAt(dirfd, name, uint64(st.Dev), st.Ino)
if err != nil {
w.dir.failed = w.dir.failed || !retry
return
}
if isDir {
_ = unix.Close(fd)
return
}
w.files++
w.fm.handlePlacedFile(fd, w.dir.pid, w.dir.at)
}
// readDirNames lists the names of fd's directory from from on, sorted,
// without consuming fd. It stops reading once more than limit names past
// from are held: those alone exceed the entry budget, so the walk is cut and
// their order no longer matters. Names before the cursor count against the
// full entry budget too, or a growing directory could make a resume unbounded.
func readDirNames(fd int, from string, limit int) (names []string, over bool, err error) {
dup, err := unix.Openat(fd, ".", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
if err != nil {
return nil, false, err
}
// #nosec G115 -- Openat returned a non-negative descriptor because err is nil.
dir := os.NewFile(uintptr(dup), ".")
defer func() { _ = dir.Close() }()
fresh, read := 0, 0
for {
batch, err := placedDirReadNames(dir, placedDirReadBatch)
for _, name := range batch {
read++
if name >= from {
names = append(names, name)
if name > from {
fresh++
}
}
if fresh > limit || read > placedDirEntryLimit {
slices.Sort(names)
return names, true, nil
}
}
if errors.Is(err, io.EOF) {
slices.Sort(names)
return names, false, nil
}
if err != nil {
return nil, false, err
}
}
}
// discardPlacedDirs counts directories still waiting at shutdown as lost.
// The walk worker has exited before this runs.
func (fm *FileMonitor) discardPlacedDirs() {
fm.placedDirMu.Lock()
defer fm.placedDirMu.Unlock()
now := time.Now()
for _, dir := range fm.placedDirs {
dir.ticket.Reject(now)
}
fm.placedDirs = nil
}
package daemon
import (
"bytes"
"encoding/binary"
"errors"
"time"
)
// Wire layout of struct placement_event in _placement.bpf.c.
const (
placementEventHeaderSize = 24
placementEventPathSize = 4096
)
// PlacementRawEvent is one decoded ring record. Components are the new
// entry's names from the leaf up to the filesystem root, so userspace can
// join them under that filesystem's mount. Dev is the kernel dev_t. At is
// when the ring reader decoded it, however long a worker takes to reach it.
type PlacementRawEvent struct {
Kind uint8
IsDir bool
Truncated bool
PID uint32
Dev, Ino uint64
Components []string
At time.Time
}
func decodePlacementEvent(b []byte) (PlacementRawEvent, error) {
if len(b) < placementEventHeaderSize {
return PlacementRawEvent{}, errors.New("placement event short buffer")
}
ev := PlacementRawEvent{
Kind: b[0],
IsDir: b[1] != 0,
Truncated: b[3] != 0,
PID: binary.LittleEndian.Uint32(b[4:8]),
Dev: binary.LittleEndian.Uint64(b[8:16]),
Ino: binary.LittleEndian.Uint64(b[16:24]),
At: time.Now(),
}
depth := int(b[2])
names := b[placementEventHeaderSize:]
ev.Components = make([]string, 0, depth)
for range depth {
// The reservation is not zeroed, so only counted names are path.
end := bytes.IndexByte(names, 0)
if end < 0 {
return PlacementRawEvent{}, errors.New("placement event path shorter than its depth")
}
ev.Components = append(ev.Components, string(names[:end]))
names = names[end+1:]
}
return ev, nil
}
//go:build linux
package daemon
import (
"bytes"
"container/list"
"context"
"encoding/binary"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/queuehealth"
)
const (
// fidCreateSettle is how long a create waits for its own close-write.
// open(O_CREAT) raises FAN_CREATE and then CLOSE_WRITE; a hard link
// raises only FAN_CREATE.
fidCreateSettle = 2 * time.Second
// fidPendingCap bounds creates waiting out their window. A busy host
// creates thousands of files a second; past the cap the oldest create
// is analysed early, which costs at most a duplicate analysis.
fidPendingCap = 16384
// maxFileHandleSize is the kernel's MAX_HANDLE_SZ.
maxFileHandleSize = 128
// fidPollMillis bounds how late an expired create is released.
fidPollMillis = 250
)
var errFIDMalformed = errors.New("malformed fanotify FID record")
// fidRecord is one event from a FAN_REPORT_DFID_NAME group: an entry Name
// in the directory Handle identifies on the filesystem Fsid. Handle aliases
// the read buffer.
type fidRecord struct {
Mask uint64
PID int32
Fsid unix.Fsid
HandleType int32
Handle []byte
Name string
}
// parseFIDRecord decodes the event at the start of b. size is the event's
// length whenever the metadata frames it, so a caller can skip a record it
// rejects; 0 means the rest of b cannot be framed.
func parseFIDRecord(b []byte) (rec fidRecord, size int, err error) {
ne := binary.NativeEndian
if len(b) < metadataSize || b[4] != unix.FANOTIFY_METADATA_VERSION {
return rec, 0, errFIDMalformed
}
size = int(ne.Uint32(b[0:4]))
if size < metadataSize || size > len(b) {
return rec, 0, errFIDMalformed
}
metaLen := int(ne.Uint16(b[6:8]))
if metaLen < metadataSize || metaLen > size {
return rec, size, errFIDMalformed
}
rec.Mask = ne.Uint64(b[8:16])
rec.PID = int32(ne.Uint32(b[20:24])) // #nosec G115 -- the kernel writes a pid_t
if rec.Mask&unix.FAN_Q_OVERFLOW != 0 {
return rec, size, nil
}
found := false
for off := metaLen; off < size; {
if size-off < 4 {
return rec, size, errFIDMalformed
}
infoLen := int(ne.Uint16(b[off+2 : off+4]))
if infoLen < 4 || infoLen > size-off {
return rec, size, errFIDMalformed
}
if b[off] == unix.FAN_EVENT_INFO_TYPE_DFID_NAME {
if found || rec.decodeDirName(b[off:off+infoLen]) != nil {
return rec, size, errFIDMalformed
}
found = true
}
off += infoLen
}
if !found {
return rec, size, errFIDMalformed
}
return rec, size, nil
}
// decodeDirName reads a DFID_NAME info record: header, fsid, struct
// file_handle, then the NUL-terminated entry name.
func (rec *fidRecord) decodeDirName(info []byte) error {
ne := binary.NativeEndian
const fixed = 4 + 8 + 8
if len(info) < fixed {
return errFIDMalformed
}
rec.Fsid = unix.Fsid{Val: [2]int32{
int32(ne.Uint32(info[4:8])), // #nosec G115 -- __kernel_fsid_t holds two ints
int32(ne.Uint32(info[8:12])), // #nosec G115 -- __kernel_fsid_t holds two ints
}}
n := int(ne.Uint32(info[12:16]))
rec.HandleType = int32(ne.Uint32(info[16:20])) // #nosec G115 -- handle_type is an int
if n == 0 || n > maxFileHandleSize || n > len(info)-fixed {
return errFIDMalformed
}
rec.Handle = info[fixed : fixed+n]
name := info[fixed+n:]
end := bytes.IndexByte(name, 0)
if end <= 0 {
return errFIDMalformed
}
rec.Name = string(name[:end])
// The name is joined under a resolved directory: anything but one
// component would place the event at a different path.
if rec.Name == "." || rec.Name == ".." || strings.Contains(rec.Name, "/") {
return errFIDMalformed
}
return nil
}
type createKey struct {
dev, ino uint64
path string
}
type pendingCreate struct {
ev placementEvent
held time.Time
ticket queuehealth.Ticket
}
// createSettler holds creates until their window ends. The close-write path
// settles a create by dev+inode+path; reader and close-write goroutines share it.
type createSettler struct {
mu sync.Mutex
cap int
order *list.List // *pendingCreate, oldest first
pending map[createKey]*list.Element
// A drop here is a create analysed before its window ended, never one
// skipped, except at shutdown.
health *queuehealth.Tracker
}
func newCreateSettler(capacity int) *createSettler {
return &createSettler{
cap: capacity, order: list.New(), pending: make(map[createKey]*list.Element),
health: queuehealth.New(capacity, time.Minute),
}
}
// add holds ev and returns the create the cap pushed out, if any.
func (s *createSettler) add(ev placementEvent, now time.Time) (placementEvent, bool) {
s.mu.Lock()
defer s.mu.Unlock()
key := createKey{ev.Dev, ev.Ino, ev.Path}
if old, ok := s.pending[key]; ok {
// Repeated publication of the same inode must not postpone its scan.
old.Value.(*pendingCreate).ev = ev
return placementEvent{}, false
}
var released placementEvent
full := s.order.Len() >= s.cap
if full {
oldest := s.removeLocked(s.order.Front())
oldest.ticket.Reject(now)
released = oldest.ev
}
s.pending[key] = s.order.PushBack(&pendingCreate{ev: ev, held: now, ticket: s.health.Begin(now)})
return released, full
}
func (s *createSettler) removeLocked(e *list.Element) *pendingCreate {
p := s.order.Remove(e).(*pendingCreate)
delete(s.pending, createKey{p.ev.Dev, p.ev.Ino, p.ev.Path})
return p
}
// settle drops a create whose own close-write analyses the file.
func (s *createSettler) settle(key createKey, now time.Time) {
s.mu.Lock()
defer s.mu.Unlock()
if e, ok := s.pending[key]; ok {
s.removeLocked(e).ticket.Finish(now)
}
}
// expire returns the creates whose window ended without a close-write.
func (s *createSettler) expire(now time.Time) []placementEvent {
s.mu.Lock()
defer s.mu.Unlock()
var due []placementEvent
for e := s.order.Front(); e != nil; e = s.order.Front() {
p := e.Value.(*pendingCreate)
if now.Sub(p.held) < fidCreateSettle {
break
}
s.removeLocked(e).ticket.Finish(now)
due = append(due, p.ev)
}
return due
}
func (s *createSettler) discard(now time.Time) {
s.mu.Lock()
defer s.mu.Unlock()
for e := s.order.Front(); e != nil; e = s.order.Front() {
s.removeLocked(e).ticket.Reject(now)
}
}
// noteClosedWrite settles only the name the close-write analyses. Another
// hard link can have different admission and location-dependent checks.
func (fm *FileMonitor) noteClosedWrite(dev, ino uint64, path string) {
if s := fm.placementCreates.Load(); s != nil {
s.settle(createKey{dev, ino, path}, time.Now())
}
}
type fidMount struct {
fd int
path string
// dev keys the unresolved-placement cooldown, as the BPF path does.
dev uint64
}
// placementFID reports placements through a second fanotify group that
// reports directory handles and entry names. It needs kernel 5.9.
type placementFID struct {
fd int
// Handles identify an inode, not the watched mount path. Keep each
// watched view so a bind alias gets its own location-dependent analysis.
mounts map[unix.Fsid][]fidMount
// roots uses resolved names, as /proc/self/fd does for delivered events.
roots []watchRootMark
fm *FileMonitor
handle func(placementEvent)
out *queuehealth.Channel[placementEvent]
creates *createSettler
openDir func(int, unix.FileHandle, int) (int, error)
// kernel counts FAN_Q_OVERFLOW: placements the kernel dropped unseen.
kernel *queuehealth.Tracker
// unresolved counts records that could not be turned into a path.
unresolved *queuehealth.Tracker
count atomic.Uint64
}
func newPlacementFID(fm *FileMonitor, fd int, mounts map[unix.Fsid][]fidMount) *placementFID {
return &placementFID{
fd: fd, mounts: mounts, roots: slices.Clone(fm.watchRoots), fm: fm, handle: fm.handleResolvedPlacement,
out: queuehealth.NewChannel[placementEvent](256, time.Minute),
creates: newCreateSettler(fidPendingCap),
openDir: unix.OpenByHandleAt,
kernel: queuehealth.New(0, time.Minute),
unresolved: queuehealth.New(0, time.Minute),
}
}
// startPlacementFID marks each watched filesystem once. An error (EINVAL
// before 5.9, EPERM without CAP_SYS_ADMIN) leaves no group behind.
func startPlacementFID(_ context.Context, fm *FileMonitor) (*placementFID, error) {
fd, err := unix.FanotifyInit(FAN_CLASS_NOTIF|FAN_CLOEXEC|FAN_NONBLOCK|unix.FAN_REPORT_DFID_NAME, unix.O_RDONLY|unix.O_CLOEXEC)
if err != nil {
return nil, fidInitError(err)
}
p := newPlacementFID(fm, fd, make(map[unix.Fsid][]fidMount))
all, err := readMountinfo()
if err != nil {
p.close()
return nil, fmt.Errorf("read mountinfo: %w", err)
}
skipped, err := p.markFilesystemsWith(all, unix.FanotifyMark)
if err != nil {
p.close()
return nil, err
}
p.reportCoverage(all, skipped)
return p, nil
}
// reportCoverage names the roots the running group cannot see: those on a
// filesystem this kernel cannot mark (one Warning), and any root left out of
// the marked mounts (a High each).
func (p *placementFID) reportCoverage(all, skipped []mountEntry) {
p.fm.reportUnmarkablePlacementRoots(all, skipped, p.roots)
p.fm.reportUncoveredPlacementRoots(all, append(p.markedMounts(all), skipped...), p.roots)
}
// markedMounts returns the mounts the group resolves handles through. Today
// every root's own mount is marked or the start fails; the report guards
// against a root left out.
func (p *placementFID) markedMounts(all []mountEntry) []mountEntry {
var marked []mountEntry
for _, mounts := range p.mounts {
for _, m := range mounts {
if i := visibleMount(all, m.path); i >= 0 {
marked = append(marked, all[i])
}
}
}
return marked
}
// fidInitError marks the errors a kernel without directory-entry reports
// returns (EINVAL before 5.9, ENOSYS without fanotify) as missing support.
func fidInitError(err error) error {
if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.ENOSYS) {
return fmt.Errorf("%w: fanotify_init with directory handles: %w", errPlacementFIDUnsupported, err)
}
return fmt.Errorf("fanotify_init with directory handles: %w", err)
}
// markFilesystemsWith marks the filesystem under every watch root. A root on
// a filesystem this kernel cannot report handles for (tmpfs before 6.7 gives
// ENODEV or EOPNOTSUPP; some btrfs subvolumes give EXDEV) is skipped and
// returned; any other failure, or skipping every root, fails the group.
func (p *placementFID) markFilesystemsWith(all []mountEntry, mark markFunc) ([]mountEntry, error) {
roots, err := resolveWatchRootLinks(p.roots)
if err != nil {
return nil, err
}
p.roots = roots
var skipped []mountEntry
var skipErr error
seen := make(map[mountEntry]bool)
devices := make(map[unix.Fsid]uint64)
for _, r := range p.roots {
i := placementVisibleMount(all, r)
if i < 0 {
return nil, fmt.Errorf("watch root %s has no visible mount on its recorded device", r.path)
}
m := all[i]
if seen[m] {
continue
}
seen[m] = true
anchor := m.MountPoint
// Not O_PATH: fanotify_mark and, on older kernels,
// open_by_handle_at reject an O_PATH descriptor with EBADF.
dirfd, err := unix.Open(anchor, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
if err != nil {
return nil, fmt.Errorf("open %s: %w", anchor, err)
}
var st unix.Stat_t
if err = unix.Fstat(dirfd, &st); err != nil || uint64(st.Dev) != r.device {
_ = unix.Close(dirfd)
return nil, fmt.Errorf("watch mount %s changed or could not be inspected: %v", anchor, err)
}
var sfs unix.Statfs_t
if err = unix.Fstatfs(dirfd, &sfs); err != nil {
_ = unix.Close(dirfd)
return nil, fmt.Errorf("statfs %s: %w", r.path, err)
}
device, marked := devices[sfs.Fsid]
if marked && device != r.device {
_ = unix.Close(dirfd)
return nil, fmt.Errorf("watch root %s shares a filesystem id with another device", r.path)
}
if !marked {
err = mark(p.fd, FAN_MARK_ADD|FAN_MARK_FILESYSTEM, unix.FAN_MOVED_TO|unix.FAN_CREATE|unix.FAN_ONDIR, dirfd, "")
if err != nil {
_ = unix.Close(dirfd)
err = fmt.Errorf("mark %s for placements: %w", r.path, err)
if !errors.Is(err, unix.ENODEV) && !errors.Is(err, unix.EOPNOTSUPP) && !errors.Is(err, unix.EXDEV) {
return nil, err
}
skipped, skipErr = append(skipped, m), err
continue
}
devices[sfs.Fsid] = r.device
}
p.mounts[sfs.Fsid] = append(p.mounts[sfs.Fsid], fidMount{fd: dirfd, path: anchor, dev: r.device})
}
if len(p.mounts) == 0 {
if skipErr != nil {
return nil, fmt.Errorf("%w: no watched filesystem accepts directory-entry marks: %w", errPlacementFIDUnsupported, skipErr)
}
return nil, errors.New("no watch root to mark")
}
return skipped, nil
}
func (p *placementFID) close() {
for _, mounts := range p.mounts {
for _, m := range mounts {
_ = unix.Close(m.fd)
}
}
_ = unix.Close(p.fd)
}
func (p *placementFID) Mode() string { return "legacy" }
func (p *placementFID) EventCount() uint64 { return p.count.Load() }
func (p *placementFID) QueueStatuses(now time.Time) map[string]queuehealth.Status {
return map[string]queuehealth.Status{
"output": p.out.Snapshot(now),
"create_settle": p.creates.health.Snapshot(now),
"kernel": p.kernel.Snapshot(now),
"unresolved": p.unresolved.Snapshot(now),
}
}
// Run reads until ctx ends. Records resolve on this goroutine; the placement
// handler runs on the shared workers.
func (p *placementFID) Run(ctx context.Context) {
p.fm.placementCreates.Store(p.creates)
waitWorkers := startPlacementWorkers(ctx, p.fm, p.out.Items(), func(ev placementEvent) []placementEvent { return []placementEvent{ev} }, p.handle)
p.read(ctx)
p.fm.placementCreates.CompareAndSwap(p.creates, nil)
waitWorkers()
now := time.Now()
p.out.Close()
p.out.DiscardPending()
p.creates.discard(now)
p.close()
}
func (p *placementFID) read(ctx context.Context) {
buf := make([]byte, 64*1024)
fds := []unix.PollFd{{Fd: int32(p.fd), Events: unix.POLLIN}} // #nosec G115 -- POSIX fd fits in int32
for ctx.Err() == nil {
p.expire(time.Now())
if _, err := unix.Poll(fds, fidPollMillis); err != nil && !errors.Is(err, unix.EINTR) {
csmlog.Warn("placement: fanotify poll failed", "err", err)
time.Sleep(time.Second)
continue
}
for ctx.Err() == nil {
n, err := unix.Read(p.fd, buf)
if errors.Is(err, unix.EINTR) {
continue
}
if err != nil || n <= 0 {
if err != nil && !errors.Is(err, unix.EAGAIN) {
csmlog.Warn("placement: fanotify read failed", "err", err)
time.Sleep(time.Second)
}
break
}
now := time.Now()
p.dispatch(buf[:n], now)
p.expire(now)
}
}
}
func (p *placementFID) dispatch(buf []byte, now time.Time) {
for len(buf) > 0 {
rec, size, err := parseFIDRecord(buf)
if size == 0 {
p.unresolved.Lose(now, 1)
return
}
buf = buf[size:]
if err != nil {
p.unresolved.Lose(now, 1)
continue
}
p.count.Add(1)
if rec.Mask&unix.FAN_Q_OVERFLOW != 0 {
p.kernel.Lose(now, 1)
p.fm.reportPlacementOverflow()
continue
}
mounts := p.mounts[rec.Fsid]
if len(mounts) == 0 {
p.unresolved.Lose(now, 1)
continue
}
for _, m := range mounts {
ev, ok, err := p.resolve(rec, m, now)
if err != nil {
p.unresolved.Lose(now, 1)
// A systematic failure, such as a refused open_by_handle_at,
// would otherwise hide every placement on this filesystem.
p.fm.reportUnresolvedOnDevice(m.dev, func() string {
return fmt.Sprintf("Realtime protection did not check entry %q in %s because it could not open the directory or the entry: %v. Placed by PID %d. The next deep scan covers it.",
rec.Name, m.path, err, rec.PID)
})
continue
}
if ok {
p.route(ev, now)
}
}
}
}
// resolve finds the entry a record names. ok is false, with no error, when
// nothing analysable is left there: the entry or its directory is gone, it
// is not a regular file or directory, or it is outside the watch roots.
func (p *placementFID) resolve(rec fidRecord, mount fidMount, at time.Time) (ev placementEvent, ok bool, err error) {
dirfd, err := p.openDir(mount.fd, unix.NewFileHandle(rec.HandleType, rec.Handle), unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC)
if errors.Is(err, unix.ESTALE) || errors.Is(err, unix.ENOENT) {
return ev, false, nil
}
if err != nil {
return ev, false, err
}
defer func() { _ = unix.Close(dirfd) }()
// Raw, not eventFDPath: a live directory can be named "x (deleted)".
// A removed one is empty, so the lookup below finds nothing in it.
dir, err := os.Readlink(fmt.Sprintf("/proc/self/fd/%d", dirfd))
if err != nil {
return ev, false, err
}
path := filepath.Join(dir, rec.Name)
if !pathWithin(dir, mount.path) || !p.withinWatchRoots(path) {
return ev, false, nil
}
var st unix.Stat_t
if err = unix.Fstatat(dirfd, rec.Name, &st, unix.AT_SYMLINK_NOFOLLOW); err != nil {
if errors.Is(err, unix.ENOENT) {
return ev, false, nil
}
return ev, false, err
}
ev = placementEvent{
Path: path, Dev: uint64(st.Dev), Ino: st.Ino, // #nosec G115 -- device ids are non-negative on Linux
PID: rec.PID, Kind: placementRename, At: at,
}
switch st.Mode & unix.S_IFMT {
case unix.S_IFDIR:
// A new directory is empty; its files raise their own events.
ev.IsDir = true
return ev, rec.Mask&unix.FAN_MOVED_TO != 0, nil
case unix.S_IFREG:
default:
return ev, false, nil
}
if rec.Mask&unix.FAN_MOVED_TO == 0 {
// A second name identifies a hard link, which needs no settle window.
ev.Kind = placementCreate
if st.Nlink > 1 {
ev.Kind = placementLink
}
}
return ev, true, nil
}
func (p *placementFID) withinWatchRoots(path string) bool {
return placementWithinRoots(p.roots, path, false)
}
// route holds creates for their own close-write and queues the rest.
func (p *placementFID) route(ev placementEvent, now time.Time) {
if ev.Kind == placementCreate {
if released, ok := p.creates.add(ev, now); ok {
p.send(released)
}
return
}
p.send(ev)
}
func (p *placementFID) expire(now time.Time) {
for _, ev := range p.creates.expire(now) {
p.send(ev)
}
}
// send queues ev for the workers. A full queue counts the drop and hands ev
// to overflow recovery, as the BPF ring reader does.
func (p *placementFID) send(ev placementEvent) {
if !p.out.TrySend(ev) {
p.fm.recoverPlacementAfter(ev, time.Time{})
}
}
// reportPlacementOverflow tells the operator that the kernel dropped
// placement events: no path survives an overflow, so no recovery can
// rescan them.
func (fm *FileMonitor) reportPlacementOverflow() {
const check = "fanotify_kernel_overflow"
if !fm.shouldAlert(check, "placement") {
return
}
fm.sendAlert(alert.Warning, check,
"fanotify placement event queue overflowed; files moved or linked into watched trees during the overflow were not checked",
"The kernel queue for rename and hard-link events filled and dropped events before the reader could drain them. Files placed during the overflow are only covered by the next scheduled deep scan.")
}
package daemon
import (
"context"
"errors"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
)
// StartPlacementMonitor starts the monitor for files renamed or hard-linked
// into fm's watch roots, chosen by detection.placement_backend: the BPF LSM
// hooks, else the fanotify directory-entry group ("legacy"), else none.
func StartPlacementMonitor(fm *FileMonitor, alertCh chan<- alert.Finding, cfg *config.Config) bpf.Backend {
ctx := context.Background()
var bpfErr, fidErr error
mon := startLiveBackend("placement", cfg.Detection.PlacementBackend, alertCh,
func() (bpf.Backend, error) {
b, err := tryStartPlacementBPFFn(ctx, fm)
bpfErr = err
return b, err
},
func() bpf.Backend {
b, err := tryStartPlacementFIDFn(ctx, fm)
if err != nil {
fidErr = err
csmlog.Info("placement: fanotify directory-entry group unavailable", "reason", err.Error())
return nil
}
return b
})
if mon != nil {
return mon
}
// A kernel without support is expected on older hosts and shows in
// status; an unexpected failure is a finding. Both ran only under auto.
switch {
case bpfErr != nil && fidErr != nil:
if !placementUnsupported(bpfErr, fidErr) {
emitBPFUnavailableFinding(alertCh, "placement", bpf.BackendAuto, "", errors.Join(bpfErr, fidErr))
}
case fidErr != nil:
if !errors.Is(fidErr, errPlacementFIDUnsupported) {
emitBPFUnavailableFinding(alertCh, "placement", bpf.BackendLegacy, "", fidErr)
}
}
// An explicit bpf choice already warned.
if !strings.EqualFold(strings.TrimSpace(cfg.Detection.PlacementBackend), bpf.BackendBPF) {
csmlog.Warn("placement: no live backend; files renamed or hard-linked into watched trees, and directories moved into them, are checked only by the deep scan")
}
return nil
}
// errPlacementFIDUnsupported marks a kernel that cannot report directory
// entries through fanotify at all.
var errPlacementFIDUnsupported = errors.New("kernel lacks fanotify directory-entry events")
func placementUnsupported(bpfErr, fidErr error) bool {
return (errors.Is(bpfErr, bpf.ErrUnsupported) || errors.Is(bpfErr, bpf.ErrNotBuilt)) &&
errors.Is(fidErr, errPlacementFIDUnsupported)
}
// startPlacementMonitor runs the placement backend beside the file monitor
// it feeds. The file monitor stops it before closing its analyzer queue.
func (d *Daemon) startPlacementMonitor(fm *FileMonitor) {
mon := StartPlacementMonitor(fm, d.alertCh, d.cfg)
if mon == nil {
return
}
d.registerBackendQueues("bpf.placement", mon)
fm.runPlacement(mon)
csmlog.Info("placement: started", "backend", mon.Mode())
}
// Package-level indirections so tests can substitute fakes.
var (
tryStartPlacementBPFFn = tryStartPlacementBPF
tryStartPlacementFIDFn = tryStartPlacementFID
)
func tryStartPlacementBPF(ctx context.Context, fm *FileMonitor) (bpf.Backend, error) {
p, err := startPlacementBPF(ctx, fm)
if err != nil {
return nil, err
}
return p, nil
}
// tryStartPlacementFID returns nil, never a typed nil, with the error.
func tryStartPlacementFID(ctx context.Context, fm *FileMonitor) (bpf.Backend, error) {
p, err := startPlacementFID(ctx, fm)
if err != nil {
return nil, err
}
return p, nil
}
//go:build linux
package daemon
import (
"bufio"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
csmlog "github.com/pidginhost/csm/internal/log"
)
// mountEntry is one /proc/self/mountinfo line. Dev uses the stat(2)
// encoding; Root is the directory of the superblock this mount shows.
type mountEntry struct {
Dev uint64
Root string
MountPoint string
}
func readMountinfo() ([]mountEntry, error) {
f, err := os.Open("/proc/self/mountinfo")
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
return parseMountinfo(f)
}
func parseMountinfo(r io.Reader) ([]mountEntry, error) {
var mounts []mountEntry
sc := bufio.NewScanner(r)
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
for sc.Scan() {
fields := strings.Fields(sc.Text())
if len(fields) < 5 {
return nil, fmt.Errorf("mountinfo line has %d fields", len(fields))
}
major, minor, ok := strings.Cut(fields[2], ":")
if !ok {
return nil, fmt.Errorf("mountinfo device %q", fields[2])
}
maj, err := strconv.ParseUint(major, 10, 32)
if err != nil {
return nil, fmt.Errorf("mountinfo device %q: %w", fields[2], err)
}
mnr, err := strconv.ParseUint(minor, 10, 32)
if err != nil {
return nil, fmt.Errorf("mountinfo device %q: %w", fields[2], err)
}
mounts = append(mounts, mountEntry{
Dev: unix.Mkdev(uint32(maj), uint32(mnr)),
Root: unescapeMountinfo(fields[3]),
MountPoint: unescapeMountinfo(fields[4]),
})
}
return mounts, sc.Err()
}
// unescapeMountinfo reverses the kernel's octal escaping of space, tab,
// newline and backslash in mountinfo paths.
func unescapeMountinfo(s string) string {
if !strings.Contains(s, `\`) {
return s
}
var b strings.Builder
for i := 0; i < len(s); i++ {
if s[i] == '\\' && i+4 <= len(s) {
if v, err := strconv.ParseUint(s[i+1:i+4], 8, 8); err == nil {
b.WriteByte(byte(v))
i += 3
continue
}
}
b.WriteByte(s[i])
}
return b.String()
}
// bpfPlacementMounts returns the mounts the BPF backend joins kernel paths
// under, and the watch roots spelled as those joined paths spell them.
func bpfPlacementMounts(all []mountEntry, roots []watchRootMark) ([]mountEntry, []watchRootMark, error) {
roots, err := resolveWatchRootLinks(roots)
if err != nil {
return nil, nil, err
}
return placementMounts(all, roots), roots, nil
}
// resolveWatchRootLinks returns a copy of roots with symlinks resolved.
// statDevice and fanotify_mark follow watch-root symlinks, but mountinfo
// and the paths placement events resolve to never contain those aliases.
func resolveWatchRootLinks(roots []watchRootMark) ([]watchRootMark, error) {
resolved := slices.Clone(roots)
for i := range resolved {
path, err := filepath.EvalSymlinks(resolved[i].path)
if err != nil {
return nil, fmt.Errorf("resolve watch root %s: %w", resolved[i].path, err)
}
resolved[i].path = path
}
return resolved, nil
}
// placementMounts picks every whole-superblock mount holding a watch root.
// Even on one device, admission and child mounts depend on the mount path.
func placementMounts(all []mountEntry, roots []watchRootMark) []mountEntry {
var picked []mountEntry
seen := make(map[mountEntry]bool)
for _, r := range roots {
best := placementRootMount(all, r)
if best >= 0 && !seen[all[best]] {
seen[all[best]] = true
picked = append(picked, all[best])
}
}
return picked
}
// A deeper mount hides its ancestors even when it shows a different device
// or a subdirectory of the same filesystem. Filtering those mounts first
// would incorrectly treat a hidden whole-filesystem mount as usable.
func placementRootMount(all []mountEntry, root watchRootMark) int {
best := placementVisibleMount(all, root)
if best < 0 || all[best].Root != "/" {
return -1
}
return best
}
// placementVisibleMount returns the mount that shows root, if it is on the
// root's recorded device.
func placementVisibleMount(all []mountEntry, root watchRootMark) int {
best := visibleMount(all, root.path)
if best < 0 || all[best].Dev != root.device {
return -1
}
return best
}
// visibleMount returns the index of the mount that shows path: the deepest
// mount point above it, the later of two at one point being on top.
func visibleMount(all []mountEntry, path string) int {
best := -1
for i, m := range all {
if pathWithin(path, m.MountPoint) && (best < 0 || len(m.MountPoint) >= len(all[best].MountPoint)) {
best = i
}
}
return best
}
func pathWithin(path, dir string) bool {
return dir == "/" || path == dir || strings.HasPrefix(path, dir+"/")
}
// placementWithinRoots reports whether an entry placed at path can put a
// file under a watch root: it is inside one, or is a directory holding one.
func placementWithinRoots(roots []watchRootMark, path string, isDir bool) bool {
for _, r := range roots {
if pathWithin(path, r.path) || isDir && pathWithin(r.path, path) {
return true
}
}
return false
}
// resolvePlacementPath joins leaf-first names under the whole-superblock
// mount of dev. Bind mounts of a subdirectory (a CageFS cage) share dev but
// show only part of the tree, so they cannot place a superblock-rooted path.
func resolvePlacementPath(dev uint64, leafFirst []string, mounts []mountEntry) (string, bool) {
if len(leafFirst) == 0 {
return "", false
}
for _, m := range mounts {
if m.Dev != dev || m.Root != "/" {
continue
}
parts := make([]string, 0, len(leafFirst)+1)
parts = append(parts, m.MountPoint)
for i := len(leafFirst) - 1; i >= 0; i-- {
parts = append(parts, leafFirst[i])
}
return filepath.Join(parts...), true
}
return "", false
}
const placementUnresolvedCheck = "placement_unresolved_realtime"
// placementFromRaw turns a kernel record into the event handlePlacement
// takes. A record whose path cannot be rebuilt is reported, not dropped
// silently: nesting deeper than the kernel walk would otherwise hide a file.
func (fm *FileMonitor) placementFromRaw(raw PlacementRawEvent, mounts []mountEntry, at time.Time) (placementEvent, bool) {
dev := userDeviceID(raw.Dev)
path, ok := "", false
if !raw.Truncated {
path, ok = resolvePlacementPath(dev, raw.Components, mounts)
}
if !ok {
fm.reportUnresolvedPlacement(dev, raw)
return placementEvent{}, false
}
return placementEvent{
Path: path, Dev: dev, Ino: raw.Ino, IsDir: raw.IsDir,
PID: int32(raw.PID), // #nosec G115 -- kernel pids fit in pid_t
Kind: placementKind(raw.Kind), At: at,
}, true
}
func (fm *FileMonitor) reportUnresolvedPlacement(dev uint64, raw PlacementRawEvent) {
fm.reportUnresolvedOnDevice(dev, func() string {
names := make([]string, 0, len(raw.Components)+1)
names = append(names, "...")
for i := len(raw.Components) - 1; i >= 0; i-- {
names = append(names, raw.Components[i])
}
reason := "the filesystem has no whole mount to resolve the path under"
if raw.Truncated {
reason = "the path is deeper or longer than realtime protection follows"
}
return fmt.Sprintf("Realtime protection did not check it because %s. Nearest names: %s; inode %d on that device, placed by PID %d. The next deep scan covers it.",
reason, strings.Join(names, "/"), raw.Ino, raw.PID)
})
}
// reportUnresolvedOnDevice keys its cooldown on the device, not the names:
// a writer can make every placement unresolvable. details runs only when
// the finding is sent.
func (fm *FileMonitor) reportUnresolvedOnDevice(dev uint64, details func() string) {
device := deviceName(dev)
if !fm.shouldAlert(placementUnresolvedCheck, "dev "+device) {
return
}
fm.sendAlert(alert.High, placementUnresolvedCheck,
fmt.Sprintf("Realtime protection could not resolve the path of a file or directory placed on device %s", device),
details())
}
// reportUncoveredPlacementRoots names roots that the running backend's
// mounts, through which it turns events into paths, do not show.
func (fm *FileMonitor) reportUncoveredPlacementRoots(all, mounts []mountEntry, roots []watchRootMark) {
for _, r := range roots {
if i := visibleMount(all, r.path); i >= 0 && slices.Contains(mounts, all[i]) {
continue
}
csmlog.Warn("placement: watch root is not visible through the placement mounts", "root", r.path, "device", deviceName(r.device))
fm.sendAlert(alert.High, placementUnresolvedCheck,
fmt.Sprintf("Realtime protection cannot see files moved or linked into %s (device %s)", r.path, deviceName(r.device)),
"Realtime protection resolves placements on this filesystem through a mount that does not show this watch root, so it cannot match them to this location. The deep scan covers them.")
}
}
// reportUnmarkablePlacementRoots raises one Warning per start for the roots
// whose filesystem this kernel cannot watch for placements: a kernel limit,
// not a fault.
func (fm *FileMonitor) reportUnmarkablePlacementRoots(all, skipped []mountEntry, watched []watchRootMark) {
var roots []string
for _, r := range watched {
if i := placementVisibleMount(all, r); i >= 0 && slices.Contains(skipped, all[i]) {
roots = append(roots, r.path)
}
}
if len(roots) == 0 {
return
}
csmlog.Warn("placement: kernel cannot watch these roots for moved or linked files", "roots", strings.Join(roots, ", "))
noun := "watch roots"
if len(roots) == 1 {
noun = "watch root"
}
fm.sendAlert(alert.Warning, placementUnresolvedCheck,
fmt.Sprintf("Realtime protection cannot see files moved or linked into %d %s on this kernel", len(roots), noun),
fmt.Sprintf("This kernel cannot report renames and hard links on the filesystems of: %s. Files moved or linked there are checked by the deep scan.", strings.Join(roots, ", ")))
}
func deviceName(dev uint64) string {
return fmt.Sprintf("%d:%d", unix.Major(dev), unix.Minor(dev))
}
package daemon
import (
"container/list"
"slices"
"sync"
"time"
)
const (
recentAnalysisCap = 65536
recentAnalysisTTL = 10 * time.Minute
)
// analysisKey identifies one analysis of one file version. A rename keeps
// dev, inode, content, size, mtime, mode and owner; Class holds everything else the
// analysis outcome depends on, so a key match means the same analysis
// already ran.
type analysisKey struct {
Dev, Ino uint64
Size, MtimeNs int64
Mode uint32
Uid, Gid uint32
Class string
Digest [32]byte
}
type analysedInode struct{ dev, ino uint64 }
// recentAnalyses remembers files analysed in the last ttl. Losing an entry
// to capacity, expiry or invalidation only costs a second analysis, never a
// skip.
type recentAnalyses struct {
mu sync.Mutex
capacity int
ttl time.Duration
order *list.List // of *recentAnalysis, most recently recorded first
byKey map[analysisKey]*list.Element
byFile map[analysedInode][]*list.Element
// seq counts invalidations. invalidated holds the last one per inode,
// and floor is the newest one no longer held there, so a record begun
// before floor cannot be checked and is dropped.
seq uint64
floor uint64
invalidated map[analysedInode]uint64
}
type recentAnalysis struct {
key analysisKey
at time.Time
}
func newRecentAnalyses(capacity int, ttl time.Duration) *recentAnalyses {
r := &recentAnalyses{capacity: capacity, ttl: ttl}
r.reset()
return r
}
func (r *recentAnalyses) reset() {
r.order = list.New()
r.byKey = make(map[analysisKey]*list.Element)
r.byFile = make(map[analysedInode][]*list.Element)
r.floor = r.seq
r.invalidated = make(map[analysedInode]uint64)
}
// Begin marks the start of an analysis; pass the result to Record.
func (r *recentAnalyses) Begin() uint64 {
if r == nil {
return 0
}
r.mu.Lock()
defer r.mu.Unlock()
return r.seq
}
// Record notes an analysis that started at begun and has completed. It is
// dropped when the inode was invalidated after begun: the analysis may have
// read content older than the write that invalidated it. A nil receiver
// records nothing.
func (r *recentAnalyses) Record(k analysisKey, begun uint64, now time.Time) {
if r == nil {
return
}
r.mu.Lock()
defer r.mu.Unlock()
if begun < r.floor || r.invalidated[analysedInode{k.Dev, k.Ino}] > begun {
return
}
if el, ok := r.byKey[k]; ok {
el.Value.(*recentAnalysis).at = now
r.order.MoveToFront(el)
return
}
el := r.order.PushFront(&recentAnalysis{key: k, at: now})
r.byKey[k] = el
id := analysedInode{k.Dev, k.Ino}
r.byFile[id] = append(r.byFile[id], el)
if r.order.Len() > r.capacity {
r.removeLocked(r.order.Back())
}
}
// Seen reports whether k was recorded less than ttl before now.
func (r *recentAnalyses) Seen(k analysisKey, now time.Time) bool {
if r == nil {
return false
}
r.mu.Lock()
defer r.mu.Unlock()
el, ok := r.byKey[k]
if !ok {
return false
}
if now.Sub(el.Value.(*recentAnalysis).at) < r.ttl {
return true
}
r.removeLocked(el)
return false
}
// Invalidate forgets every analysis of the inode: its content may have
// changed in a way the key cannot see.
func (r *recentAnalyses) Invalidate(dev, ino uint64) {
if r == nil {
return
}
r.mu.Lock()
defer r.mu.Unlock()
id := analysedInode{dev, ino}
for _, el := range r.byFile[id] {
r.order.Remove(el)
delete(r.byKey, el.Value.(*recentAnalysis).key)
}
delete(r.byFile, id)
r.seq++
if len(r.invalidated) >= r.capacity {
r.floor, r.invalidated = r.seq-1, make(map[analysedInode]uint64)
}
r.invalidated[id] = r.seq
}
// Purge forgets everything.
func (r *recentAnalyses) Purge() {
if r == nil {
return
}
r.mu.Lock()
defer r.mu.Unlock()
r.seq++
r.reset()
}
func (r *recentAnalyses) removeLocked(el *list.Element) {
k := el.Value.(*recentAnalysis).key
r.order.Remove(el)
delete(r.byKey, k)
id := analysedInode{k.Dev, k.Ino}
if rest := slices.DeleteFunc(r.byFile[id], func(e *list.Element) bool { return e == el }); len(rest) > 0 {
r.byFile[id] = rest
} else {
delete(r.byFile, id)
}
}
//go:build linux
package daemon
import (
"crypto/sha256"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/checks"
)
type analysisSnapshot struct {
key analysisKey
stat unix.Stat_t
}
// Content, not mtime, proves equivalence across renames. The existing complete
// content budget bounds this optimization; larger files always get analysed.
func snapshotForAnalysis(fd int, class string) (analysisSnapshot, bool) {
var st unix.Stat_t
if class == "" || unix.Fstat(fd, &st) != nil || st.Mode&unix.S_IFMT != unix.S_IFREG || st.Mode&0o111 != 0 {
return analysisSnapshot{}, false
}
data := readExactSize(st.Size, checks.MaxInertPHPScanBytes, func(p []byte, off int64) (int, error) {
return unix.Pread(fd, p, off)
})
if data == nil {
return analysisSnapshot{}, false
}
snapshot := analysisSnapshot{
key: analysisKey{
Dev: uint64(st.Dev), Ino: st.Ino, Size: st.Size, MtimeNs: st.Mtim.Nano(),
Mode: st.Mode, Uid: st.Uid, Gid: st.Gid, Class: class, Digest: sha256.Sum256(data),
},
stat: st,
}
return snapshot, snapshot.unchanged(fd)
}
// Ctime is excluded from the cache key because a rename changes it. During a
// scan it must stay fixed, including when a writer changes bytes and puts them
// back before the scan ends.
func (s analysisSnapshot) unchanged(fd int) bool {
var after unix.Stat_t
return unix.Fstat(fd, &after) == nil && sameReadSnapshot(s.stat, after) &&
s.stat.Mode == after.Mode && s.stat.Uid == after.Uid && s.stat.Gid == after.Gid
}
//go:build linux
package daemon
import (
"context"
"sync"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/queuehealth"
)
// placementWorkers bounds the goroutines blocked in handlePlacement, which
// waits up to its retry budget for a pending rename. A backend's reader
// never waits on them: a full output queue counts the event and hands it to
// overflow recovery.
const placementWorkers = 4
// startPlacementWorkers drains events with placementWorkers goroutines until
// ctx ends or events closes. resolve returns the placements to check at
// each watched location; an empty slice leaves nothing to check. The
// returned function waits for every worker to stop.
func startPlacementWorkers[T any](ctx context.Context, fm *FileMonitor, events <-chan queuehealth.Work[T], resolve func(T) []placementEvent, handle func(placementEvent)) func() {
var workers sync.WaitGroup
for range placementWorkers {
workers.Go(func() {
for {
select {
case <-ctx.Done():
return
case work, ok := <-events:
if !ok {
return
}
processPlacement(fm, work, resolve, handle)
}
}
})
}
return workers.Wait
}
// processPlacement contains a panic the way analyzeFileSafe does: one
// placement must not take the monitor down with it.
func processPlacement[T any](fm *FileMonitor, work queuehealth.Work[T], resolve func(T) []placementEvent, handle func(placementEvent)) {
var path string
defer func() {
if r := recover(); r != nil {
fm.reportScannerPanic(path, r)
}
}()
work.Process(func(item T) {
for _, ev := range resolve(item) {
path = ev.Path
handle(ev)
}
})
}
// runPlacement runs b until the monitor stops. drainAndClose waits for it, so
// call it before Run: an Add after that Wait admits into a closed queue.
func (fm *FileMonitor) runPlacement(b bpf.Backend) {
fm.recent = newRecentAnalyses(recentAnalysisCap, recentAnalysisTTL)
ctx, cancel := context.WithCancel(context.Background())
fm.placementWG.Add(1)
obs.Go("placement", func() {
defer fm.placementWG.Done()
defer cancel()
obs.SafeGo("placement-stop-forward", func() {
select {
case <-fm.stopCh:
cancel()
case <-ctx.Done():
}
})
b.Run(ctx)
})
}
package daemon
import (
"fmt"
"os"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/platform"
)
// PlatformOverridesFrom maps the operator's web_server: block to platform
// overrides. WebServer is only set when a type was given, so an empty type
// leaves the detected webserver alone.
func PlatformOverridesFrom(cfg *config.Config) platform.Overrides {
var wsOverride *platform.WebServer
if t := cfg.WebServer.Type; t != "" {
ws := platform.WebServer(t)
wsOverride = &ws
}
return platform.Overrides{
WebServer: wsOverride,
ApacheConfigDir: cfg.WebServer.ConfigDir,
NginxConfigDir: cfg.WebServer.ConfigDir,
AccessLogPaths: cfg.WebServer.AccessLogs,
ErrorLogPaths: cfg.WebServer.ErrorLogs,
ModSecAuditLogPaths: cfg.WebServer.ModSecAudits,
DomlogGlobs: cfg.WebServer.DomlogGlobs,
}
}
// InstallPlatformOverrides installs the config-supplied platform overrides.
// It must run before anything in the process calls platform.Detect: a
// detection cached without them silently discards the operator's remedy for
// a wrong probe, so the daemon's log watchers attach to the wrong files. A
// lost override is reported loudly; the daemon keeps running on the probe.
func InstallPlatformOverrides(cfg *config.Config) bool {
if platform.SetOverrides(PlatformOverridesFrom(cfg)) {
return true
}
msg := "platform overrides from web_server: were ignored because platform detection ran first; log watchers follow the probe, not the config"
fmt.Fprintf(os.Stderr, "[ERROR] %s\n", msg)
obs.CaptureMsg("platform", msg)
return false
}
package daemon
import (
"net"
"strconv"
"strings"
)
// postfixSMTPDMessage returns the process tag and message of a postfix smtpd
// log line. Backend warnings omit the client address, so their process tag
// must be retained to withhold that process's subsequent failures.
func postfixSMTPDMessage(line string) (process, msg string) {
tagEnd := strings.Index(line, ": ")
if tagEnd < 0 {
return "", ""
}
process = line[strings.LastIndexAny(line[:tagEnd], " \t")+1 : tagEnd]
if !isPostfixSMTPDTag(process) {
return "", ""
}
return process, line[tagEnd+2:]
}
// postfixSASLFailure reports the client address of a postfix smtpd
// "warning: name[addr]: SASL <mech> authentication failed: <reply>" line and
// the reply text. It returns "" for any other line.
//
// The client field must open the warning as one space-free token: smtpd
// echoes client-supplied text (RCPT, HELO) into other warnings, and a match
// anywhere in the message would let a client name the address to block.
//
// The line alone is never evidence: Postfix logs a Dovecot temporary failure
// the same way as a wrong password, and a failed reconnect can repeat an
// earlier reply. Callers pair it with Dovecot's verdict for the request.
func postfixSASLFailure(msg string) (ip, reply string) {
msg, ok := strings.CutPrefix(msg, "warning: ")
if !ok {
return "", ""
}
client, rest, ok := strings.Cut(msg, ": SASL ")
if !ok {
return "", ""
}
ip = postfixClientIP(client)
if ip == "" {
return "", ""
}
mech, rest, ok := strings.Cut(rest, " ")
if !ok || mech == "" {
return "", ""
}
// SASL mechanism names cannot contain quoting or whitespace. Postfix
// echoes invalid AUTH tokens too, so an embedded failure is not a result.
for _, c := range mech {
if c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '_' {
continue
}
return "", ""
}
if rest == "authentication failed" {
return ip, ""
}
reply, ok = strings.CutPrefix(rest, "authentication failed:")
if !ok {
return "", ""
}
// Newer smtpd versions append the attempted username. It is untrusted
// text and must not turn a credential failure into a backend outage.
reply, _, _ = strings.Cut(reply, ", sasl_username=")
return ip, strings.ToLower(strings.TrimSpace(reply))
}
// postfixSASLSuccess reports the client address and login name of a postfix
// smtpd "QUEUEID: client=name[addr], sasl_method=MECH, sasl_username=USER"
// line, which smtpd logs when it accepts the first recipient from a client
// that authenticated. It returns "" for any other line, and an empty user
// when the line has no login name.
//
// The queue ID must open the message and be followed directly by the client
// field. Rejects and header warnings that echo client text put other words
// after the queue ID.
func postfixSASLSuccess(msg string) (ip, user string) {
queueID, rest, ok := strings.Cut(msg, ": client=")
if !ok || !isASCIIAlnum(queueID) {
return "", ""
}
// postfixClientIP refuses whitespace, so the method field must follow
// the client field directly.
client, rest, ok := strings.Cut(rest, ", sasl_method=")
if !ok {
return "", ""
}
if ip = postfixClientIP(client); ip == "" {
return "", ""
}
// smtpd may append the AUTH= sender and XFORWARD fields, in this order,
// after the login name. Only the field immediately after the method
// can name the authenticated user; later fields may contain client text.
_, rest, _ = strings.Cut(rest, ", ")
user, ok = strings.CutPrefix(rest, "sasl_username=")
if !ok {
return ip, ""
}
for _, field := range []string{", sasl_sender=", ", orig_queue_id=", ", orig_client="} {
user, _, _ = strings.Cut(user, field)
}
return ip, user
}
func isASCIIAlnum(s string) bool {
for i := 0; i < len(s); i++ {
c := s[i]
if c < '0' || c > '9' && c < 'A' || c > 'Z' && c < 'a' || c > 'z' {
return false
}
}
return s != ""
}
func isPostfixSASLNonCredentialReply(reply string) bool {
// smtpd_sasl_glue logs the SASL reason, not the SMTP status. Dovecot's
// malformed backend replies and Cyrus's temporary errors do not all
// include "temporary" or "authentication server" in that reason.
switch reply {
case "authentication backend error", "generic failure", "no memory available", "transient failure (e.g., weak key)":
return true
}
return strings.Contains(reply, "authentication server") || strings.Contains(reply, "temporar")
}
// isPostfixSASLProtocolReply reports protocol refusals from either smtpd or
// Dovecot. They do not establish a credential check and must not claim a
// parallel request's pending verdict.
func isPostfixSASLProtocolReply(reply string) bool {
switch reply {
case "invalid authentication mechanism", "invalid base64 data in initial response", "invalid base64 data in continued response":
return true
case "authentication request timed out", "unexpected continuation",
"client didn't present valid ssl certificate", "ssl certificate didn't contain username":
// Dovecot sends these before processing the SASL response. Postfix
// forwards them in the same warning as a wrong password.
return true
}
return strings.HasPrefix(reply, "invalid authentication mechanism:")
}
func postfixClientIP(client string) string {
if strings.ContainsAny(client, " \t\r\n") {
return ""
}
end := strings.LastIndexByte(client, ']')
if end < 0 {
return ""
}
// smtpd_client_port_logging appends a decimal port to the client field.
if tail := client[end+1:]; tail != "" {
port, ok := strings.CutPrefix(tail, ":")
if !ok {
return ""
}
if _, err := strconv.ParseUint(port, 10, 16); err != nil {
return ""
}
}
start := strings.LastIndexByte(client[:end], '[')
if start <= 0 {
return ""
}
ip := net.ParseIP(client[start+1 : end])
if ip == nil {
return ""
}
return ip.String()
}
// isPostfixSMTPDTag accepts "postfix/smtpd[pid]" and the per-service
// syslog_name form "postfix/submission/smtpd[pid]".
func isPostfixSMTPDTag(tag string) bool {
i := strings.LastIndex(tag, "/smtpd[")
if i < 0 || !strings.HasPrefix(tag, "postfix") || !strings.HasSuffix(tag, "]") {
return false
}
pid := tag[i+len("/smtpd[") : len(tag)-1]
if pid == "" {
return false
}
for i := 0; i < len(pid); i++ {
if pid[i] < '0' || pid[i] > '9' {
return false
}
}
return true
}
package daemon
import (
"net"
"net/http"
"net/http/pprof"
"runtime"
"strings"
"sync"
"time"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/obs"
)
const (
// mutexProfileFraction samples one in every N mutex contention events.
// Sampling limits stack collection overhead on busy hosts.
mutexProfileFraction = 100
// blockProfileRate samples one blocking event per this many nanoseconds
// spent blocked, so a goroutine parked for 10 microseconds is recorded
// with probability one. Shorter waits are sampled with proportionally lower
// probability, not excluded. Even unsampled waits incur timing overhead.
blockProfileRate = 10_000
)
// The runtime rates are process-wide. A listener stopping must not switch off
// another listener that is still serving profiles.
var contentionProfiles struct {
sync.Mutex
listeners int
}
// enableContentionProfiles turns on the sampling the mutex and block profiles
// depend on for the lifetime of successfully bound listeners.
func enableContentionProfiles() {
contentionProfiles.Lock()
defer contentionProfiles.Unlock()
if contentionProfiles.listeners == 0 {
runtime.SetMutexProfileFraction(mutexProfileFraction)
runtime.SetBlockProfileRate(blockProfileRate)
}
contentionProfiles.listeners++
}
func disableContentionProfiles() {
contentionProfiles.Lock()
defer contentionProfiles.Unlock()
contentionProfiles.listeners--
if contentionProfiles.listeners == 0 {
runtime.SetMutexProfileFraction(0)
runtime.SetBlockProfileRate(0)
}
}
// pprofListen is replaceable in tests so listener lifecycle coverage does not
// depend on the test sandbox allowing real sockets.
var pprofListen = net.Listen
// isLoopbackPprofAddr reports whether addr (host:port) binds only to a loopback
// interface. The pprof server exposes process internals and lets a caller
// trigger CPU/heap dumps, so it must never listen off-box. An empty or wildcard
// host ("" / ":6060") is rejected because it would bind all interfaces.
func isLoopbackPprofAddr(addr string) bool {
host, _, err := net.SplitHostPort(strings.TrimSpace(addr))
if err != nil {
return false
}
host = strings.TrimSpace(host)
if host == "" {
return false
}
if strings.EqualFold(host, "localhost") {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
func newPprofMux() *http.ServeMux {
mux := http.NewServeMux()
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
return mux
}
// startPprofListener starts a net/http/pprof server on addr, but only when addr
// is a loopback bind. A dedicated mux keeps the pprof handlers off
// http.DefaultServeMux so nothing else can serve them. The listener closes on
// daemon shutdown. It reports whether a listener was actually started.
func (d *Daemon) startPprofListener(addr string) bool {
if !isLoopbackPprofAddr(addr) {
csmlog.Warn("debug.pprof_listen ignored: not a loopback bind; pprof exposes process internals and must use 127.0.0.1/::1/localhost",
"addr", addr)
return false
}
ln, err := pprofListen("tcp", strings.TrimSpace(addr))
if err != nil {
csmlog.Warn("debug.pprof_listen ignored: listener failed", "addr", addr, "err", err)
return false
}
// Sampling starts with the listener, not at daemon start: a host with no
// pprof bind pays nothing for profiles nobody can fetch.
enableContentionProfiles()
srv := &http.Server{
Addr: addr,
Handler: newPprofMux(),
ReadHeaderTimeout: 10 * time.Second,
}
done := make(chan struct{})
d.wg.Add(2)
obs.Go("pprof-listener", func() {
defer d.wg.Done()
defer close(done)
csmlog.Info("pprof debug listener started", "addr", ln.Addr().String())
if err := srv.Serve(ln); err != nil && err != http.ErrServerClosed {
csmlog.Warn("pprof listener stopped", "err", err)
}
})
obs.Go("pprof-shutdown", func() {
defer d.wg.Done()
defer disableContentionProfiles()
select {
case <-d.stopCh:
_ = srv.Close()
case <-done:
}
})
return true
}
package daemon
import (
"math"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/processctx"
)
const (
processCtxCacheCap = 16384
processCtxCacheTTL = 30 * time.Minute
processCtxEnrichWorkers = 2
processCtxEnrichQueueCap = 1024
processCtxProcRoot = "/proc"
processCtxProcReadDeadline = 10 * time.Millisecond
)
var (
processCtxOnce sync.Once
processCtxCache *processctx.Cache
processCtxEnr *processctx.Enricher
processCtxPublished atomic.Pointer[processctx.Enricher]
processCtxRegistry = metrics.Default
)
var processCtxReadStartedAt = defaultProcessCtxReadStartedAt
func defaultProcessCtxReadStartedAt(pid int) (time.Time, bool) {
return processctx.NewProcReader(processCtxProcRoot, processCtxProcReadDeadline).ReadStartedAt(pid)
}
func processCtxStartedAt(pid int) time.Time {
t, ok := processCtxReadStartedAt(pid)
if !ok {
return time.Time{}
}
return t
}
// ProcessCtx returns the daemon-wide process-context cache and enricher,
// constructing them on first call and registering metrics on the default
// registry. Safe for concurrent callers.
func ProcessCtx() (*processctx.Cache, *processctx.Enricher) {
processCtxOnce.Do(func() {
processCtxCache = processctx.NewCache(processCtxCacheCap, processCtxCacheTTL)
reader := processctx.NewProcReader(processCtxProcRoot, processCtxProcReadDeadline)
processCtxEnr = processctx.NewEnricher(processCtxCache, reader, processctx.EnricherConfig{
Workers: processCtxEnrichWorkers,
QueueCap: processCtxEnrichQueueCap,
Resolver: daemonProcessIdentityResolver{},
})
processctx.RegisterMetrics(processCtxRegistry(), processCtxCache, processCtxEnr)
processCtxPublished.Store(processCtxEnr)
processCtxEnr.Start()
wireAncestryCache(processCtxCache)
})
return processCtxCache, processCtxEnr
}
func stopProcessCtx() {
if enr := processCtxPublished.Load(); enr != nil {
enr.Stop()
}
}
type daemonProcessIdentityResolver struct{}
func (daemonProcessIdentityResolver) Resolve(uid int) (string, string) {
if uid < 0 || uid > math.MaxUint32 {
return "", ""
}
user := checks.LookupUser(uint32(uid))
account := resolveLocalAccountForUID(uid, user)
return user, account
}
func resolveLocalAccountForUID(uid int, user string) string {
// First phase: for normal hosted account UIDs, the username is the account
// on cPanel and plain Linux fallback hosts. Later phases can replace this
// helper with a platform-backed account enumerator without changing the
// processctx package.
if uid >= 1000 && user != "" && !strings.HasPrefix(user, "uid:") {
return user
}
return ""
}
// resetProcessCtxForTest is a test seam. Callers in tests must run with
// t.Setenv or similar isolation; production code never invokes this.
func resetProcessCtxForTest() {
if processCtxEnr != nil {
processCtxEnr.Stop()
}
processCtxPublished.Store(nil)
processCtxOnce = sync.Once{}
processCtxCache = nil
processCtxEnr = nil
processCtxRegistry = metrics.NewRegistry
processCtxReadStartedAt = defaultProcessCtxReadStartedAt
}
package daemon
import (
"errors"
"github.com/pidginhost/csm/internal/firewall"
)
func isProtectedIPRefusal(err error) bool {
return errors.Is(err, firewall.ErrIPProtected)
}
package daemon
import (
"sync"
"time"
)
const purgeSuppressionWindow = 60 * time.Second
// purgeTracker correlates password purge events with subsequent
// stale-session 401 errors to suppress false-positive alerts.
//
// When a cPanel user changes their password, all existing sessions are
// invalidated. Any in-flight browser requests (AJAX polls, notifications,
// etc.) will return 401 - these are expected side effects, not attacks.
//
// Flow: login (NEW) records IP→account, PURGE records account→time,
// 401 handler checks IP→account→purgeTime to decide suppression.
var purgeTracker = &purgeState{
purges: make(map[string]time.Time),
sessions: make(map[string]string),
}
type purgeState struct {
mu sync.Mutex
purges map[string]time.Time // account → last purge time
sessions map[string]string // IP → last known account
}
// recordLogin tracks which account an IP most recently logged into.
func (ps *purgeState) recordLogin(ip, account string) {
ps.mu.Lock()
defer ps.mu.Unlock()
ps.sessions[ip] = account
ps.cleanupLocked()
}
// recordPurge records a password purge event for an account.
func (ps *purgeState) recordPurge(account string) {
ps.mu.Lock()
defer ps.mu.Unlock()
ps.purges[account] = time.Now()
}
// isPostPurge401 returns true if the IP's 401 is likely a stale session
// artifact from a recent password change (within the suppression window).
func (ps *purgeState) isPostPurge401(ip string) bool {
ps.mu.Lock()
defer ps.mu.Unlock()
account, ok := ps.sessions[ip]
if !ok {
return false
}
purgeTime, ok := ps.purges[account]
if !ok {
return false
}
return time.Since(purgeTime) < purgeSuppressionWindow
}
// cleanupLocked removes stale entries. Caller must hold ps.mu.
func (ps *purgeState) cleanupLocked() {
cutoff := time.Now().Add(-2 * purgeSuppressionWindow)
for k, t := range ps.purges {
if t.Before(cutoff) {
delete(ps.purges, k)
}
}
// Cap sessions map to prevent unbounded growth (keep most recent 500)
if len(ps.sessions) > 500 {
ps.sessions = make(map[string]string)
}
}
package daemon
import (
"fmt"
"maps"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/queuehealth"
)
type queueSource interface {
QueueStatuses(time.Time) map[string]queuehealth.Status
}
func (d *Daemon) registerBackendQueues(prefix string, backend bpf.Backend) {
if source, ok := backend.(queueSource); ok {
d.registerQueueSource(prefix, source)
}
}
func (d *Daemon) registerQueueSource(prefix string, source queueSource) {
d.queueSourcesMu.Lock()
defer d.queueSourcesMu.Unlock()
if d.queueSources == nil {
d.queueSources = make(map[string]queueSource)
}
d.queueSources[prefix] = source
}
func (d *Daemon) registeredQueueStatuses(now time.Time) map[string]queuehealth.Status {
d.queueSourcesMu.RLock()
sources := maps.Clone(d.queueSources)
d.queueSourcesMu.RUnlock()
statuses := make(map[string]queuehealth.Status)
for prefix, source := range sources {
for name, status := range source.QueueStatuses(now) {
statuses[prefix+"."+name] = status
}
}
return statuses
}
func (d *Daemon) setFileMonitor(fm *FileMonitor) {
d.fileMonitorMu.Lock()
d.fileMonitor = fm
d.fileMonitorMu.Unlock()
}
func (d *Daemon) getFileMonitor() *FileMonitor {
d.fileMonitorMu.RLock()
defer d.fileMonitorMu.RUnlock()
return d.fileMonitor
}
// FanotifyActive reports whether the fanotify file monitor is running.
func (d *Daemon) FanotifyActive() bool {
return d.getFileMonitor() != nil
}
// LogWatcherCount returns the number of running log watchers, including
// those a retry started after a missing log appeared.
func (d *Daemon) LogWatcherCount() int {
d.logWatchersMu.Lock()
defer d.logWatchersMu.Unlock()
return len(d.logWatchers)
}
func (d *Daemon) monitorQueueHealth() {
defer d.wg.Done()
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
var reporter queuehealth.Reporter
for {
select {
case <-d.stopCh:
return
case now := <-ticker.C:
d.reportQueueHealth(now, &reporter)
}
}
}
func (d *Daemon) reportQueueHealth(now time.Time, reporter *queuehealth.Reporter) {
events := reporter.Events(now, d.queueStatuses(now))
if len(events) == 0 {
return
}
findings := make([]alert.Finding, 0, len(events))
for _, event := range events {
check, message := "protection_queue_degraded", "Protection work is delayed or being dropped"
if event.Recovered {
check, message = "protection_queue_recovered", "Protection queue recovered"
}
s := event.Current
findings = append(findings, alert.Finding{
Check: check, Severity: alert.Warning, Message: message,
DedupKey: event.Name, Timestamp: now,
Details: fmt.Sprintf("queue=%s reason=%s %s", event.Name, s.Reason, s.Evidence()),
})
}
// This loop owns delivery: the failing ingest channel cannot carry its
// own alarm. History and passive observers receive transitions without
// running automatic response or advancing the last completed scan time.
d.store.AppendHistory(findings)
if err := alert.Dispatch(d.currentCfg(), findings); err != nil {
csmlog.Warn("queue health notification failed", "err", err)
}
}
//go:build linux
package daemon
import (
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
func (fm *FileMonitor) initQueueHealth() {
fm.queueHealthOnce.Do(func() {
fm.analyzerHealth = queuehealth.New(cap(fm.analyzerCh), time.Minute)
fm.reconcileHealth = queuehealth.New(reconcileDirCap, time.Minute)
fm.placedDirHealth = queuehealth.New(placedDirQueueCap, time.Minute)
fm.kernelQueueHealth = queuehealth.New(0, time.Minute)
fm.kernelQueue = newNotificationQueue(fanotifyDescriptor(fm.fd), fm.kernelQueueHealth, queuehealth.New(1, time.Minute))
})
}
func (fm *FileMonitor) queueStatuses(now time.Time) map[string]queuehealth.Status {
fm.initQueueHealth()
kernel, reader := fm.kernelQueue.snapshot(time.Now)
reconcile := fm.reconcileHealth.Snapshot(now)
reconcile.DepthUnit = "directories"
placedDirs := fm.placedDirHealth.Snapshot(now)
placedDirs.DepthUnit = "directories"
statuses := map[string]queuehealth.Status{
"fanotify.analyzer": fm.analyzerHealth.Snapshot(now),
"fanotify.kernel": kernel,
"fanotify.reader": reader,
"fanotify.reconcile": reconcile,
"fanotify.staged_packages": fm.stagedPackages().snapshot(now),
"placement.dir_walk": placedDirs,
}
if fm.dropper != nil {
statuses["fanotify.dropper"], statuses["fanotify.dropper_findings"] = fm.dropper.tr.queueStatuses(now)
}
return statuses
}
func (sw *SpoolWatcher) initQueueHealth() {
sw.queueHealthOnce.Do(func() {
sw.scannerHealth = queuehealth.New(cap(sw.scanCh), time.Minute)
sw.kernelQueueHealth = queuehealth.New(0, time.Minute)
sw.kernelQueue = newNotificationQueue(fanotifyDescriptor(sw.fd), sw.kernelQueueHealth, queuehealth.New(1, time.Minute))
})
}
// A restarted watcher belongs to the same daemon lifetime. Carry its work
// accounting forward so repeated crashes cannot clear the recent loss window.
func (sw *SpoolWatcher) inheritQueueHealth(previous *SpoolWatcher) {
previous.initQueueHealth()
sw.queueHealthOnce.Do(func() {
sw.scannerHealth = previous.scannerHealth
sw.kernelQueueHealth = previous.kernelQueueHealth
sw.kernelQueue = newNotificationQueue(fanotifyDescriptor(sw.fd), sw.kernelQueueHealth, previous.kernelQueue.batches)
})
}
func (sw *SpoolWatcher) queueStatuses(now time.Time) map[string]queuehealth.Status {
sw.initQueueHealth()
kernel, reader := sw.kernelQueue.snapshot(time.Now)
return map[string]queuehealth.Status{
"spool.scanner": sw.scannerHealth.Snapshot(now),
"spool.kernel": kernel,
"spool.reader": reader,
}
}
package daemon
import (
"fmt"
"strings"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/signatures"
"github.com/pidginhost/csm/internal/yara"
)
// reportRealtimeRuleCoverage raises a finding when a realtime engine is
// running with no rules. Both scanners treat a missing or empty rules
// directory as a successful load of nothing, so a mistyped rules_dir or an
// empty rule sync left every file write "scanned" against zero rules while
// the daemon looked healthy. yaraActive is false when no YARA backend is
// installed or its rules failed to compile; those states raise their own
// findings and are not repeated here.
func (d *Daemon) reportRealtimeRuleCoverage(yamlRules, yaraRules int, yaraActive bool) {
var empty []string
if yamlRules == 0 {
empty = append(empty, "YAML")
}
if yaraActive && yaraRules == 0 {
empty = append(empty, "YARA")
}
if len(empty) == 0 {
d.realtimeRulesMu.Lock()
d.realtimeRulesState = ""
d.realtimeRulesMu.Unlock()
return
}
rulesDir := "the configured rules directory"
if cfg := d.currentCfg(); cfg != nil && cfg.Signatures.RulesDir != "" {
rulesDir = cfg.Signatures.RulesDir
}
state := strings.Join(empty, ",") + "\x00" + rulesDir
d.realtimeRulesMu.Lock()
defer d.realtimeRulesMu.Unlock()
if d.realtimeRulesState == state {
return
}
if d.emitYaraFinding(alert.High, "realtime_rules_missing",
fmt.Sprintf("Real-time file scanning has no %s rules loaded from %s; every file write is scanned against nothing until rules are installed and reloaded.",
strings.Join(empty, " or "), rulesDir)) {
d.realtimeRulesState = state
}
}
// reportRealtimeRuleCoverageNow reads the live engines and reports on them.
func (d *Daemon) reportRealtimeRuleCoverageNow() {
yaraRules, yaraActive := 0, false
if b := yara.Active(); b != nil {
yaraActive = true
yaraRules = b.RuleCount()
}
d.reportRealtimeRuleCoverage(yamlRuleCount(), yaraRules, yaraActive)
}
// yamlRuleCount returns the number of YAML rules the global scanner holds;
// no scanner at all counts as zero rules.
func yamlRuleCount() int {
if s := signatures.Global(); s != nil {
return s.RuleCount()
}
return 0
}
//go:build linux
package daemon
import (
"fmt"
"os"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
)
// checkRemovedFileRestored reports a write that puts back, byte for byte, the
// content CSM quarantined or cleaned at the same path. A process doing that is
// undoing a response, typically a loop that rewrites a webshell whenever it
// goes missing, so the report does not depend on what the content detectors
// make of the bytes.
func (fm *FileMonitor) checkRemovedFileRestored(event fileEvent, procInfo string) bool {
removed, ok := removedFileAt(event.path)
if !ok || int(event.pid) == os.Getpid() {
return false
}
var st unix.Stat_t
if err := unix.Fstat(event.fd, &st); err != nil || st.Mode&unix.S_IFMT != unix.S_IFREG || st.Size != removed.Size {
return false
}
before := scannedIdentity(event.fd)
// A removal too large to hash and a failed read both leave no digest.
if digest := hashEventFD(event.fd, nil, st.Size); digest == "" || digest != removed.SHA256 {
return false
}
after := scannedIdentity(event.fd)
current, err := os.Lstat(event.path)
if err != nil || !checks.SameFileVersion(before, after) || !checks.SameFileVersion(after, current) {
return false
}
// An operator restore or another response may have invalidated the
// ledger entry while this descriptor was being hashed.
latest, ok := removedFileAt(event.path)
if !ok || latest != removed {
return false
}
f := removedFileRestoredFinding(event.path, removed, procInfo, event.pid)
f.WriterChain, f.WriterObservedAt = event.writerChain, event.at
fm.sendFileFinding(f)
return true
}
// Capture before analyzer queuing: the writer may exit during content analysis,
// but its still-running parents must remain eligible for the response.
func captureRestoreWriterChain(event *fileEvent) {
if event.pid <= 1 || int(event.pid) == os.Getpid() {
return
}
if _, ok := removedFileAt(event.path); ok {
event.writerChain = restoreWriterChain(int(event.pid), event.at)
}
}
// restoreWriterChain lets tests observe what the detector captures.
var restoreWriterChain = checks.RestoreWriterChain
func removedFileRestoredFinding(path string, removed checks.RemovedFile, procInfo string, pid int32) alert.Finding {
return alert.Finding{
Severity: alert.Critical,
Check: checks.RemovedFileRestoredCheck,
Message: fmt.Sprintf("File CSM removed was written back: %s", path),
Details: fmt.Sprintf("CSM %s this content at %s; a process wrote the same bytes back.\nSHA-256: %s",
removedActionPast(removed.Action), removed.At.UTC().Format(time.RFC3339), removed.SHA256),
FilePath: path,
ProcessInfo: procInfo,
PID: int(pid),
// One report per removal: a file written back after CSM removed it
// again is a new event, not a repeat of the first. The removal time,
// unlike its sequence, stays unique across daemon restarts.
DedupKey: removed.RestoreKey(),
}
}
func removedActionPast(action string) string {
if action == "clean" {
return "cleaned"
}
return "quarantined"
}
package daemon
import "github.com/pidginhost/csm/internal/checks"
// reconcileReputationWhitelist pushes the live config's reputation.whitelist
// into the running threat database. The field is tagged safe for hot reload,
// so a SIGHUP that only changes it reports success; without this push the
// threat database kept the startup list until a full restart. Called from the
// reload success path, mirroring reconcileVerifiedBots.
func (d *Daemon) reconcileReputationWhitelist() {
cfg := d.activeOrStartupCfg()
if cfg == nil {
return
}
if db := checks.GetThreatDB(); db != nil {
db.SetConfigWhitelist(cfg.Reputation.Whitelist)
}
}
package daemon
import (
"bytes"
"github.com/pidginhost/csm/internal/checks"
)
// signalEagerReconcile fires a non-blocking notification on sig the first
// time count reaches threshold. Cross-platform helper extracted so the
// trigger can be unit-tested from a non-linux test file.
//
// - sig is a buffered cap-1 channel. The send is non-blocking (default
// branch) so a stalled receiver never wedges the caller.
// - The trigger fires only on the exact threshold (not >=). A long
// burst above threshold within the same window must not refire
// after the signal has been drained; the next window's first count
// reaching threshold rearms it once the receiver has reset counters.
// - A nil sig is a no-op (some unit tests construct partial structs
// that omit it).
func signalEagerReconcile(sig chan struct{}, count, threshold int64) {
if sig == nil {
return
}
if count != threshold {
return
}
select {
case sig <- struct{}{}:
default:
}
}
// Recognisers that suppress the lowest-tier "anomalous PHP location"
// warning for two specific shapes, without skipping content scanning:
//
// 1. Files inside cPanel's pkgacct/restorepkg staging tree. cPanel
// extracts the user backup as root into /home/cpanelpkgrestore.TMP.
// work.<id>/ for inspection, then re-extracts it under the user
// identity into /home/<account>/. Both extractions raise fanotify
// events; the user-context one carries the real signal, so the
// staging-side warning is a duplicate. The signature/YARA scanners
// still run on the staging file - only the path-only warning is
// dropped.
//
// 2. WP-Optimize probe files at wp-content/uploads/wpo/*. The plugin
// writes tiny <?php files to test whether the host honours certain
// Apache/Nginx directives (Server-Signature, mod_headers, mod_rewrite).
// They contain no input handling and no execution primitives; the
// anomalous-location warning is noise on every site running this
// plugin. As above, the signature/YARA scanners still run.
// looksLikeCpanelRestoreStaging delegates to the shared recogniser in
// internal/checks/sitedetect.go. The deep-scan path uses the same
// helper so realtime and scheduled scans agree on which files are
// duplicates of the user-context extraction.
func looksLikeCpanelRestoreStaging(path string) bool {
return checks.LooksLikeCpanelRestoreStaging(path)
}
// wpOptimizeProbeMaxSize bounds the size of files the recogniser will
// accept. WP-Optimize probes are header()/echo one-liners; anything
// larger fails the shape gate and falls through to the standard
// anomalous-location warning.
const wpOptimizeProbeMaxSize = 512
// wpOptimizeProbeDangerous is the deny list of byte sequences that, if
// present, disqualify a file from being treated as a WP-Optimize probe.
// Probes never use PHP superglobals or execution primitives; an attacker
// payload that does (the only realistic way to abuse a 512-byte file in
// /uploads/wpo/) trips this gate and continues to the standard alert.
//
// Tokens are matched case-insensitively against the file body. They are
// kept separate from the signature scanner above this recogniser so the
// gate stays valid even if a future signature update changes coverage.
var wpOptimizeProbeDangerous = [][]byte{
[]byte("$_"), // any PHP superglobal: $_POST, $_GET, $_REQUEST, $_COOKIE, $_SERVER...
[]byte("ev" + "al"), // split to keep the source-tree security hook happy
[]byte("ass" + "ert"),
[]byte("include"),
[]byte("require"),
[]byte("sys" + "tem"),
[]byte("p" + "assthru"),
[]byte("sh" + "ell_exec"),
[]byte("po" + "pen"),
[]byte("proc_open"),
[]byte("e" + "xec"), // plain exec() and any *exec* variant
[]byte("base64"), // any encoder/decoder pair
[]byte("phpinfo"), // information disclosure
[]byte("create_func"), // create_function deprecated lambda primitive
[]byte("file_get"), // file_get_contents (file disclosure)
[]byte("file_put"), // file_put_contents (write primitive)
[]byte("fwrite"),
[]byte("readfile"),
[]byte("`"), // backtick command substitution
}
// looksLikeWPOptimizeProbe is the realtime, content-aware check.
// It applies the shared path-only gate from internal/checks/sitedetect.go
// (path under /uploads/wpo/, basename test.php, plugin installed) and
// then adds two content-shape gates the deep-scan path cannot apply:
//
// - File body fits in wpOptimizeProbeMaxSize bytes.
// - File body contains none of wpOptimizeProbeDangerous.
//
// All gates together prevent a webshell hidden under /uploads/wpo/test.php
// from silencing the realtime warning: any payload large or interesting
// enough to be useful trips one of the content gates. The
// signature/YARA scanners run before this recogniser, so any existing
// rule still fires on its own pipeline regardless of suppression here.
func looksLikeWPOptimizeProbe(path string, content []byte) bool {
if !checks.LooksLikeWPOptimizeProbeByPath(path) {
return false
}
if len(content) > wpOptimizeProbeMaxSize {
return false
}
lower := bytes.ToLower(content)
for _, danger := range wpOptimizeProbeDangerous {
if bytes.Contains(lower, danger) {
return false
}
}
return true
}
package daemon
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/store"
)
// RetentionResult reports how many entries each sweep removed in a single
// RunRetentionOnce invocation.
type RetentionResult struct {
History int
AttackEvents int
Reputation int
FirewallActions int
Errors []error
}
// Deleted returns the total number of entries deleted across all sweeps.
func (r RetentionResult) Deleted() int {
return r.History + r.AttackEvents + r.Reputation + r.FirewallActions
}
// bucketMap describes which config knob drives which bucket sweep. Keeping
// this explicit rather than inferring it from field names keeps the mapping
// documented next to the orchestrator.
//
// Retention bucket mapping:
// - HistoryDays → `history` bucket (the finding archive every scan appends to)
// - FindingsDays → `attacks:events` bucket (per-attack event trail feeding scoring)
// - ReputationDays → `reputation` bucket (AbuseIPDB / local lookup cache, keyed by IP)
// - HistoryDays → `fw:actions` journal (proven firewall outcomes; this is
// also how far back a durable action can be undone)
//
// Blocked IPs are deliberately NOT on a TTL: `fw:blocked` is pruned when
// an operator or auto-response unblocks an IP, and temp-ban expiry is
// already handled by LoadFirewallState.
// RunRetentionOnce performs one sweep cycle over the retention-managed
// buckets. Safe to call when retention is disabled or when inputs are
// nil — it no-ops and returns a zero result.
//
// Setting a bucket's *Days value to zero means "don't sweep this bucket
// on this cycle"; negative values are treated as zero (validation catches
// them at config load).
func RunRetentionOnce(db *store.DB, cfg *config.Config, now time.Time) RetentionResult {
var result RetentionResult
if db == nil || cfg == nil || !cfg.Retention.Enabled {
return result
}
if cfg.Retention.HistoryDays > 0 {
cutoff := now.Add(-time.Duration(cfg.Retention.HistoryDays) * 24 * time.Hour)
n, err := db.SweepHistoryOlderThan(cutoff)
if err != nil {
result.Errors = append(result.Errors, err)
}
result.History = n
}
if cfg.Retention.HistoryDays > 0 {
cutoff := now.Add(-time.Duration(cfg.Retention.HistoryDays) * 24 * time.Hour)
n, err := db.SweepFirewallActionsOlderThan(cutoff)
if err != nil {
result.Errors = append(result.Errors, err)
}
result.FirewallActions = n
}
if cfg.Retention.FindingsDays > 0 {
cutoff := now.Add(-time.Duration(cfg.Retention.FindingsDays) * 24 * time.Hour)
n, err := db.SweepAttackEventsOlderThan(cutoff)
if err != nil {
result.Errors = append(result.Errors, err)
}
result.AttackEvents = n
}
if cfg.Retention.ReputationDays > 0 {
cutoff := now.Add(-time.Duration(cfg.Retention.ReputationDays) * 24 * time.Hour)
n, err := db.SweepReputationOlderThan(cutoff)
if err != nil {
result.Errors = append(result.Errors, err)
}
result.Reputation = n
}
return result
}
// retentionSweepDurationOnce guards /metrics registration of the
// retention-cycle counter so repeated daemon starts in a test binary are
// idempotent.
var retentionSweepDurationOnce sync.Once
var retentionSweepCounter *metrics.Counter
var retentionDeletedCounter *metrics.Counter
func registerRetentionMetrics() {
retentionSweepDurationOnce.Do(func() {
retentionSweepCounter = metrics.NewCounter(
"csm_retention_sweeps_total",
"Number of retention sweep cycles completed since daemon start.",
)
metrics.MustRegister("csm_retention_sweeps_total", retentionSweepCounter)
retentionDeletedCounter = metrics.NewCounter(
"csm_retention_deleted_total",
"Number of bucket entries deleted by the retention sweep.",
)
metrics.MustRegister("csm_retention_deleted_total", retentionDeletedCounter)
})
}
// retentionScanner is the daemon goroutine that drives RunRetentionOnce
// on the configured SweepInterval. Started from Run() only when
// cfg.Retention.Enabled is true; absent that, the sweep is dormant and
// no timer fires.
//
// Compaction is NOT triggered from here: reclaiming space safely requires
// exclusive access to the bbolt file, which only startup and `csm store
// compact` with the daemon stopped have. This goroutine instead emits an info
// log when the startup compaction rule says a restart would reclaim space.
func (d *Daemon) retentionScanner() {
defer d.wg.Done()
registerRetentionMetrics()
// First sweep happens after a short settle period so a restart storm
// does not hammer bbolt. Subsequent sweeps use the full interval.
settle := 5 * time.Minute
timer := time.NewTimer(settle)
defer timer.Stop()
for {
select {
case <-d.stopCh:
return
case <-timer.C:
d.runRetentionTick()
timer.Reset(d.retentionInterval())
}
}
}
// retentionInterval parses Retention.SweepInterval from the live config,
// falling back to 24h if the duration is malformed or non-positive. The
// live config is re-read each tick so SIGHUP can adjust cadence without
// restart (the retention struct itself stays hotreload:"restart", but the
// ticker can pick up edits on the next cycle).
func (d *Daemon) retentionInterval() time.Duration {
cfg := d.currentCfg()
if cfg == nil {
return 24 * time.Hour
}
ival, err := time.ParseDuration(cfg.Retention.SweepInterval)
if err != nil || ival <= 0 {
return 24 * time.Hour
}
return ival
}
// runRetentionTick runs one sweep + size check and emits metrics/logs.
func (d *Daemon) runRetentionTick() {
cfg := d.currentCfg()
db := store.Global()
result := RunRetentionOnce(db, cfg, time.Now())
retentionSweepCounter.Inc()
if n := result.Deleted(); n > 0 {
retentionDeletedCounter.Add(float64(n))
csmlog.Info("retention sweep",
"deleted_total", n,
"history", result.History,
"attacks_events", result.AttackEvents,
"reputation", result.Reputation,
)
}
for _, err := range result.Errors {
csmlog.Warn("retention sweep bucket error", "err", err)
}
// Compaction hint: the daemon auto-compacts at the next startup
// (maybeCompactStateAtStartup), and `csm store compact` does it now with
// the daemon stopped. Say so only when that startup check would act.
if size, free, due := compactionHintDue(db, cfg); due {
csmlog.Info("retention: state db is mostly free space; it will be auto-compacted on the next restart (or run `csm store compact` now with the daemon stopped)",
"size_bytes", size,
"free_bytes", free,
)
}
}
// compactionHintDue applies the startup compaction rule to the live db. A
// large file whose pages are still in use is not compacted, so it gets no hint.
func compactionHintDue(db *store.DB, cfg *config.Config) (size, free int64, due bool) {
if cfg == nil || db == nil {
return 0, 0, false
}
size, err := db.Size()
if err != nil {
return 0, 0, false
}
free, err = db.FreeBytes()
if err != nil {
return 0, 0, false
}
return size, free, store.CompactionDue(size, free, cfg.Retention.CompactMinSizeMB, cfg.Retention.CompactFillRatio)
}
package daemon
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
// scanJobQueueDepth is the maximum number of jobs that may wait in the queue
// while the single worker is busy. Enqueue returns an error when full.
const scanJobQueueDepth = 8
// maxScanJobFindingsPerJob caps how many findings a single job persists. A
// server-wide scan can emit tens of thousands of findings; storing them all
// bloats the state db and no operator triages that many per job. Findings past
// the cap are counted (FindingCount) but not stored, and the record's
// FindingsTruncated flag lets the UI show "showing first N of M". Declared as a
// var so tests can pin a small cap. Production never mutates it.
var maxScanJobFindingsPerJob = 10000
// scanJobFindingFlushBatch bounds how many findings are written per bbolt
// transaction. Batching amortizes the per-commit fsync while keeping any single
// transaction (and its memory) bounded on a huge scan.
const scanJobFindingFlushBatch = 1000
// maxRetainedScanJobFindings bounds the cumulative finding rows kept across all
// retained jobs, independent of the job-count retention. Without it, retaining
// the newest N jobs still lets finding rows grow without bound when scans are
// finding-heavy.
const maxRetainedScanJobFindings = 50000
// scanJobIDCounter is a process-wide monotonic counter used to make job IDs
// unique when two Enqueue calls occur within the same millisecond.
var scanJobIDCounter atomic.Int64
// newScanJobID returns a lexically time-sortable, collision-free job ID.
// Format: "sj-<unix-ms-hex>-<counter-hex>".
// Lexical sort on the hex timestamp gives newest-first ordering consistent
// with the store's ListScanJobs sort key.
func newScanJobID() string {
ms := time.Now().UnixMilli()
seq := scanJobIDCounter.Add(1)
return fmt.Sprintf("sj-%016x-%08x", ms, seq)
}
// scanJobRunner is the function the worker calls to perform the actual scan.
// It is a field on ScanJobManager so tests can substitute a blocking fixture
// without any sleep races.
type scanJobRunner func(ctx context.Context, cfg *config.Config, st *state.Store, target string, opts checks.AccountScanOptions) []alert.Finding
// accountEnumerator lists cPanel accounts for scope="all" jobs.
// It is a field on ScanJobManager so tests can substitute a fake without
// touching the filesystem.
type accountEnumerator func(cfg *config.Config) ([]string, error)
// scanJobRequest is what Enqueue pushes into the work channel.
type scanJobRequest struct {
work *scanJobWork
id string
opts checks.AccountScanOptions
quarantine bool // when true, annotateQuarantine runs on each finding
cancelCtx context.Context // per-job cancellable context
cancelFn context.CancelFunc // allows Cancel() to stop the runner
// remediated records the successful disposition per file path within this
// job, so a second check flagging the same file reports the action already
// taken instead of attempting it again. The map is shared by every copy of
// the request; only the worker goroutine touches it. It can contain at most
// maxScanJobFindingsPerJob entries because
// annotateQuarantine runs only for findings admitted under that cap.
remediated map[string]scanJobRemediation
}
type scanJobRemediation struct {
status string
detail string
}
// ScanJobManager runs full-scan jobs as background work in the daemon.
// A single worker goroutine drains a bounded channel of job requests so
// jobs are strictly serialised; Phase 1 does not fan out across accounts.
//
// Lifecycle: the caller owns a sync.WaitGroup slot (d.wg). Stop() cancels
// all in-flight work and blocks until the worker goroutine exits.
type ScanJobManager struct {
st *state.Store
cfg *config.Config
db scanJobStore // bbolt handle; resolved from store.Global() at construction
health *scanJobHealth
// runAccountScan is the runner called for each job. Tests replace this
// with a fixture to control blocking / return values without sleep races.
runAccountScan scanJobRunner
// enumerateAccounts lists accounts for scope="all" jobs. Tests replace it.
enumerateAccounts accountEnumerator
// quarantineFile performs the pure file quarantine for a single finding.
// Defaults to checks.QuarantineFindingFile; tests replace it to operate on
// a temp dir without touching real /home or /opt/csm.
quarantineFile func(f alert.Finding) (checks.RemediationResult, bool)
workCh chan scanJobRequest // bounded; Enqueue returns an error when full
stopCh chan struct{} // closed by Stop()
// cancelMu guards stopped and cancelFns across enqueue, cancel, run,
// drain, and stop. Enqueue holds it until the work item is buffered so
// Stop cannot close the worker before seeing the new job context.
cancelMu sync.Mutex
stopped bool
cancelFns map[string]context.CancelFunc
// workerDone is closed when the worker goroutine exits.
workerDone chan struct{}
// stopOnce ensures Stop() is idempotent: a second call is a no-op rather
// than a panic on double-close of stopCh.
stopOnce sync.Once
}
// NewScanJobManager creates a ScanJobManager and reconciles any job left in
// state "queued" or "running" from a previous daemon crash to "error" with reason
// "daemon_restarted". Returns an error if the global bbolt handle is nil.
func NewScanJobManager(st *state.Store, cfg *config.Config) (*ScanJobManager, error) {
db := store.Global()
if db == nil {
return nil, errors.New("scan-job manager: global bbolt store is nil")
}
m := &ScanJobManager{
st: st,
cfg: cfg,
db: db,
health: newScanJobHealth(),
runAccountScan: defaultScanRunner,
enumerateAccounts: checks.EnumerateScanAccounts,
quarantineFile: checks.QuarantineFindingFile,
workCh: make(chan scanJobRequest, scanJobQueueDepth),
stopCh: make(chan struct{}),
cancelFns: make(map[string]context.CancelFunc),
workerDone: make(chan struct{}),
}
if err := m.reconcileInterruptedJobs(); err != nil {
return nil, fmt.Errorf("scan-job manager: reconcile: %w", err)
}
// The worker goroutine is tracked in its own WaitGroup so Stop() can
// drain it independently of the daemon's wg. The daemon wires this
// manager via d.wg + obs.Go when it calls startScanJobManager().
obs.Go("scan-job-worker", m.worker)
return m, nil
}
// defaultScanRunner delegates to checks.RunAccountScanWithOptions.
func defaultScanRunner(ctx context.Context, cfg *config.Config, st *state.Store, target string, opts checks.AccountScanOptions) []alert.Finding {
return checks.RunAccountScanWithOptions(ctx, cfg, st, target, opts)
}
// reconcileInterruptedJobs marks unfinished persisted jobs as "error" with
// reason "daemon_restarted" because their in-memory requests cannot be resumed.
// Called once on construction before the worker
// starts, so no concurrent writes race with this read-modify-write pass.
func (m *ScanJobManager) reconcileInterruptedJobs() error {
jobs, err := m.db.ListScanJobs()
if err != nil {
return err
}
for _, rec := range jobs {
if rec.State != "running" && rec.State != "queued" {
continue
}
rec.State = "error"
rec.Error = "daemon_restarted"
rec.Finished = time.Now().UTC()
if putErr := m.db.PutScanJob(rec); putErr != nil {
return putErr
}
m.health.losses.Lose(time.Now(), 1)
csmlog.Warn("scan job reconciled after restart", "job_id", rec.ID)
}
return nil
}
// Enqueue creates a new queued job record and pushes it into the work channel.
// Returns the job ID. Returns an error when the queue is full.
// A quarantine flag is recorded on the job record for use by a later phase;
// no live auto-response is wired in Phase 1.
func (m *ScanJobManager) Enqueue(scope, target string, opts checks.AccountScanOptions, quarantine bool) (string, error) {
admission := m.health.admission.Begin(time.Now())
completed := false
defer func() {
if completed {
admission.Finish(time.Now())
} else {
admission.Reject(time.Now())
}
}()
m.cancelMu.Lock()
defer m.cancelMu.Unlock()
admission.Start(time.Now())
if m.stopped {
completed = true
return "", errors.New("scan-job manager stopped")
}
id := newScanJobID()
// Options map for storage (human-readable; not used by the worker).
optsMap := map[string]any{
"max_files": opts.MaxFiles,
"force_content": opts.ForceContent,
"force_file_index": opts.ForceFileIndex,
"respect_ignores": opts.RespectIgnores,
"max_file_bytes": opts.MaxFileBytes,
"quarantine": quarantine,
}
rec := store.ScanJobRecord{
ID: id,
Scope: scope,
Target: target,
State: "queued",
Created: time.Now().UTC(),
Options: optsMap,
}
if err := m.db.PutScanJob(rec); err != nil {
return "", fmt.Errorf("scan-job enqueue: persist: %w", err)
}
// Build a per-job cancellable context. Cancel is called either by Cancel()
// or by Stop(), which cancels all live job contexts before waiting for the
// worker goroutine to exit.
jobCtx, jobCancel := context.WithCancel(context.Background())
jobCtx, progress := checks.WithCheckDispatchProgress(jobCtx)
m.cancelFns[id] = jobCancel
req := scanJobRequest{
work: m.health.begin(progress.Snapshot),
id: id,
opts: opts,
quarantine: quarantine,
remediated: make(map[string]scanJobRemediation),
cancelCtx: jobCtx,
cancelFn: jobCancel,
}
select {
case m.workCh <- req:
completed = true
return id, nil
default:
req.work.finish(false)
completed = true
// Queue full -- remove the persisted record and cancel the context.
jobCancel()
delete(m.cancelFns, id)
// Mark the just-persisted record as error so it does not look queued.
rec.State = "error"
rec.Error = "queue_full"
rec.Finished = time.Now().UTC()
_ = m.db.PutScanJob(rec)
return "", errors.New("scan-job queue is full")
}
}
// Cancel cancels the job with the given ID. If the job is still queued the
// worker will transition it to "canceled" without running the scanner. If the
// job is running, its context is canceled and the worker persists any findings
// the runner returns after ctx.Done() before setting state "canceled".
// Returns an error when the ID is unknown or already in a terminal state.
func (m *ScanJobManager) Cancel(id string) error {
m.cancelMu.Lock()
fn, ok := m.cancelFns[id]
m.cancelMu.Unlock()
if !ok {
return fmt.Errorf("scan-job cancel: unknown or already terminal job %q", id)
}
fn()
return nil
}
// ListJobs returns all scan job records ordered newest-first.
// It is the encapsulated accessor for handleScanStatus and the WebUI Phase 2b
// handler so neither needs to reach the manager's private db field.
func (m *ScanJobManager) ListJobs() ([]store.ScanJobRecord, error) {
return m.db.ListScanJobs()
}
// ListFindings returns a paginated slice of findings for the given job ID plus
// the total number of findings stored for that job. offset and limit follow the
// usual page semantics (limit=0 returns all findings).
// It is the encapsulated accessor for handleScanReport and the WebUI Phase 2b
// handler so neither needs to reach the manager's private db field.
func (m *ScanJobManager) ListFindings(id string, offset, limit int) ([]alert.Finding, int, error) {
return m.db.ListScanJobFindings(id, offset, limit)
}
// Progress returns the current record for the given job ID.
// ok is false when the ID is not found in the store.
func (m *ScanJobManager) Progress(id string) (store.ScanJobRecord, bool) {
rec, ok, err := m.db.GetScanJob(id)
if err != nil || !ok {
return store.ScanJobRecord{}, false
}
return rec, true
}
// Stop cancels all in-flight or queued work and waits for the worker goroutine
// to exit. After Stop returns the manager must not be used. Stop is idempotent:
// a sync.Once guards the close of stopCh so repeated calls are safe.
// Canceling all live job contexts before blocking on workerDone ensures that
// any scan currently executing inside runAccountScan returns promptly; the
// runner honors ctx and the job lands in terminal state "canceled" with its
// partial findings retained.
func (m *ScanJobManager) Stop() {
m.stopOnce.Do(func() {
m.cancelMu.Lock()
m.stopped = true
close(m.stopCh)
for _, fn := range m.cancelFns {
fn()
}
m.cancelMu.Unlock()
})
<-m.workerDone
}
// worker is the single serialised goroutine that processes scan jobs.
// It holds no references to closed resources after Stop() returns because:
// 1. Stop() closes stopCh and cancels all live job contexts before blocking
// on workerDone; any in-flight scan honors ctx.Done() and returns promptly
// with partial findings, landing in terminal state "canceled".
// 2. The select below exits as soon as stopCh fires, and all remaining
// queued jobs are drained as "canceled" -- none write to the store after
// Close() because the daemon closes the store only after Stop() returns.
func (m *ScanJobManager) worker() {
defer m.workerExited()
for {
select {
case <-m.stopCh:
// Drain the queue: mark any pending jobs canceled without running.
m.drainQueueOnStop()
return
case req := <-m.workCh:
req.work.run(func() { m.runJob(req) })
}
}
}
func (m *ScanJobManager) workerExited() {
// A panic or Goexit can bypass the normal drain. Refuse new admissions
// under the same lock used to publish them before releasing queued work.
m.cancelMu.Lock()
defer m.cancelMu.Unlock()
defer close(m.workerDone)
m.stopped = true
for id, cancel := range m.cancelFns {
cancel()
delete(m.cancelFns, id)
}
for {
select {
case req := <-m.workCh:
req.work.finish(false)
default:
return
}
}
}
// drainQueueOnStop consumes all remaining items in workCh after stopCh closes
// and marks each as "canceled". Nothing writes to the store after this returns.
func (m *ScanJobManager) drainQueueOnStop() {
for {
select {
case req := <-m.workCh:
req.work.run(func() {
req.cancelFn()
m.cancelMu.Lock()
delete(m.cancelFns, req.id)
m.cancelMu.Unlock()
m.setTerminal(req, "canceled", "")
})
default:
return
}
}
}
// runJob executes a single scan job to completion and persists the result.
// The job context (req.cancelCtx) may be canceled externally by Cancel() or
// by Stop() -> drainQueueOnStop. This function handles both cases:
// - If the context is already canceled when we check, skip the scan entirely.
// - After the runner returns, inspect ctx.Err() to choose the terminal state.
func (m *ScanJobManager) runJob(req scanJobRequest) {
db := req.trackedStore(m.db)
defer func() {
// Always remove the cancel function entry when the job is done.
m.cancelMu.Lock()
delete(m.cancelFns, req.id)
m.cancelMu.Unlock()
req.cancelFn()
}()
// If the job was canceled before the worker got to it (e.g. Cancel()
// called while it was queued), skip the scan and go straight to terminal.
if req.cancelCtx.Err() != nil {
m.setTerminal(req, "canceled", "")
return
}
// Transition to "running".
rec, ok, err := db.GetScanJob(req.id)
if err != nil || !ok {
csmlog.Warn("scan job missing at run time", "job_id", req.id)
return
}
rec.State = "running"
rec.Started = time.Now().UTC()
if putErr := db.PutScanJob(rec); putErr != nil {
csmlog.Warn("scan job state update failed", "job_id", req.id, "err", putErr)
return
}
cfg := m.cfg
if cfg == nil {
cfg = &config.Config{}
}
// Branch on scope: "all" runs every account; anything else (default
// "account") runs the single target as before.
var findingCount, findingsStored int
var truncated bool
var jobState string
if rec.Scope == "all" {
findingCount, findingsStored, truncated, jobState = m.runAllAccounts(req, rec, cfg)
} else {
// Run the scan. The runner blocks until complete or ctx is canceled.
req.work.progressed()
findings := m.runAccountScan(req.cancelCtx, cfg, m.st, rec.Target, req.opts)
req.work.progressed()
// Persist findings (batched, capped), even if canceled mid-scan, so the
// "cancel keeps partial" guarantee holds. Quarantine is applied only to
// findings that will be stored; truncated findings must not trigger
// unaudited file moves.
findingsStored, truncated = m.persistFindings(req, 0, findings, func(f alert.Finding) alert.Finding {
return m.annotateQuarantine(req, f)
})
findingCount = len(findings)
jobState = "done"
if req.cancelCtx.Err() != nil {
jobState = "canceled"
}
}
// Refresh the record before writing the terminal state so FilesScanned
// and FindingCount reflect any in-progress updates (future phases may
// update these mid-scan via callbacks; for now we set them from findings).
rec2, ok2, err2 := db.GetScanJob(req.id)
if err2 != nil || !ok2 {
// Fall back to the snapshot we already have.
rec2 = rec
}
rec2.State = jobState
rec2.Finished = time.Now().UTC()
rec2.FindingCount = findingCount
rec2.FindingsStored = findingsStored
rec2.FindingsTruncated = truncated
rec2.CurrentAccount = "" // clear transient progress field on completion
if putErr := db.PutScanJob(rec2); putErr != nil {
csmlog.Warn("scan job terminal state failed", "job_id", req.id, "err", putErr)
}
// Prune oldest jobs to keep the store within the configured retention,
// bounding both the job count and the cumulative finding rows.
retention := cfg.Thresholds.ScanJobRetention
if retention <= 0 {
retention = 20
}
if _, pruneErr := db.PruneScanJobs(retention, maxRetainedScanJobFindings); pruneErr != nil {
csmlog.Warn("scan job prune failed", "err", pruneErr)
}
}
// persistFindings writes findings for a job in batched transactions, honoring
// the per-job findings cap. stored is how many findings the job has already
// persisted (so the cap spans every account of a scope="all" job). prepare is
// applied only to the findings that fit under the cap. It returns how many
// findings from this slice were persisted and whether the cap or a write error
// dropped any of them.
func (m *ScanJobManager) persistFindings(req scanJobRequest, stored int, findings []alert.Finding, prepare func(alert.Finding) alert.Finding) (written int, truncated bool) {
db := req.trackedStore(m.db)
jobID := req.id
room := len(findings)
if maxScanJobFindingsPerJob > 0 {
remaining := maxScanJobFindingsPerJob - stored
if remaining <= 0 {
return 0, len(findings) > 0
}
if room > remaining {
room = remaining
truncated = true
}
}
for off := 0; off < room; off += scanJobFindingFlushBatch {
end := off + scanJobFindingFlushBatch
if end > room {
end = room
}
batch := findings[off:end]
if prepare != nil {
batch = append([]alert.Finding(nil), batch...)
for i := range batch {
req.work.progressed()
batch[i] = prepare(batch[i])
req.work.progressed()
}
}
if err := db.AppendScanJobFindings(jobID, stored+off, batch); err != nil {
csmlog.Warn("scan job finding batch persist failed", "job_id", jobID, "seq", stored+off, "err", err)
return written, true
}
written += len(batch)
}
return written, truncated
}
// runAllAccounts executes the scan body for a scope="all" job: enumerates
// accounts, scans each with panic isolation, persists findings incrementally,
// and returns (findingCount, terminalState). Terminal state is "error" if
// enumeration fails, "canceled" if the context is done after the loop,
// "done" otherwise.
//
// For cancel mid-iteration: the check at the top of the per-account loop
// detects a canceled context before starting the next account, so partial
// findings already persisted from completed accounts are retained.
func (m *ScanJobManager) runAllAccounts(req scanJobRequest, rec store.ScanJobRecord, cfg *config.Config) (findingCount, findingsStored int, truncated bool, jobState string) {
db := req.trackedStore(m.db)
req.work.progressed()
accounts, err := m.enumerateAccounts(cfg)
req.work.progressed()
if err != nil {
req.work.fail()
// Persist error state immediately so the outer runJob terminal write
// picks up the correct state (it will overwrite State/Finished again,
// but we set Error here via a direct PutScanJob call first).
recErr, ok2, getErr := db.GetScanJob(req.id)
if getErr == nil && ok2 {
recErr.State = "error"
recErr.Error = err.Error()
recErr.Finished = time.Now().UTC()
_ = db.PutScanJob(recErr)
}
return 0, 0, false, "error"
}
// Set AccountsTotal and persist immediately so progress is visible.
rec.AccountsTotal = len(accounts)
if putErr := db.PutScanJob(rec); putErr != nil {
csmlog.Warn("scan job accounts_total persist failed", "job_id", req.id, "err", putErr)
}
findingCount = 0
findingsStored = 0
for _, account := range accounts {
// Cancel check: stop before starting next account.
if req.cancelCtx.Err() != nil {
break
}
// Update progress: current account being scanned.
rec.CurrentAccount = account
if putErr := db.PutScanJob(rec); putErr != nil {
csmlog.Warn("scan job progress persist failed", "job_id", req.id, "err", putErr)
}
// Run the account scan with panic isolation so one bad account cannot
// abort the entire server-wide job.
req.work.progressed()
findings := m.runAccountScanIsolated(req, cfg, account)
req.work.progressed()
// Attribute each finding to the account when the check did not already
// set TenantID, then persist the account's findings as one batch. The
// per-job cap spans accounts via the running findingsStored counter.
now := time.Now().UTC()
for i := range findings {
if findings[i].TenantID == "" {
findings[i].TenantID = account
}
if findings[i].Timestamp.IsZero() {
findings[i].Timestamp = now
}
}
written, trunc := m.persistFindings(req, findingsStored, findings, func(f alert.Finding) alert.Finding {
return m.annotateQuarantine(req, f)
})
findingsStored += written
truncated = truncated || trunc
findingCount += len(findings)
rec.AccountsDone++
rec.FindingCount = findingCount
rec.FindingsStored = findingsStored
rec.FindingsTruncated = truncated
if putErr := db.PutScanJob(rec); putErr != nil {
csmlog.Warn("scan job accounts_done persist failed", "job_id", req.id, "err", putErr)
}
}
// Clear the transient CurrentAccount field before terminal state is written.
rec.CurrentAccount = ""
jobState = "done"
if req.cancelCtx.Err() != nil {
jobState = "canceled"
}
return findingCount, findingsStored, truncated, jobState
}
// runAccountScanIsolated calls m.runAccountScan inside a recover() so a panic
// in any account's scanner is converted to a synthetic account_scan_error
// finding rather than crashing the worker goroutine and aborting the job.
func (m *ScanJobManager) runAccountScanIsolated(req scanJobRequest, cfg *config.Config, account string) (findings []alert.Finding) {
defer func() {
if r := recover(); r != nil {
req.work.fail()
findings = []alert.Finding{{
Severity: alert.High,
Check: "account_scan_error",
Message: fmt.Sprintf("scan failed for account %s", account),
Details: fmt.Sprintf("%v", r),
TenantID: account,
Timestamp: time.Now().UTC(),
}}
}
}()
return m.runAccountScan(req.cancelCtx, cfg, m.st, account, req.opts)
}
// setTerminal writes a terminal state for a job without running any scan.
// Used to transition queued-but-canceled jobs during drainQueueOnStop.
func (m *ScanJobManager) setTerminal(req scanJobRequest, jobState, errMsg string) {
db := req.trackedStore(m.db)
id := req.id
rec, ok, err := db.GetScanJob(id)
if err != nil || !ok {
return
}
rec.State = jobState
rec.Error = errMsg
rec.Finished = time.Now().UTC()
_ = db.PutScanJob(rec)
}
// annotateQuarantine runs the full-scan file remediation on f when the job's
// quarantine flag is set, and stamps the finding's RemediationStatus /
// RemediationDetail fields with the outcome. When the flag is false
// (report-only) f is returned unchanged so existing consumers see no JSON diff.
//
// Eligible standalone malware files are quarantined, while eligible WordPress
// core, plugin and theme files are cleaned in place. Ineligible findings such
// as process kills, DB cleanup and htaccess edits are marked "left_for_review".
// This method never calls alert.Dispatch, state.AppendHistory, or
// StoreLatestScanFindings; remediation is not an alert.
func (m *ScanJobManager) annotateQuarantine(req scanJobRequest, f alert.Finding) alert.Finding {
if !req.quarantine {
return f
}
// A cancelled job keeps its partial findings but stops acting on them:
// the operator withdrew consent, and findings produced during teardown
// are the least verified of the batch.
if req.cancelCtx != nil && req.cancelCtx.Err() != nil {
return f
}
if prior, done := req.remediated[f.FilePath]; done && f.FilePath != "" {
f.RemediationStatus = prior.status
f.RemediationDetail = prior.detail
return f
}
result, eligible := m.quarantineFile(f)
if !eligible {
f.RemediationStatus = "left_for_review"
return f
}
if result.Success {
f.RemediationStatus = result.RemediationStatus
if f.RemediationStatus == "" {
f.RemediationStatus = "quarantined"
}
f.RemediationDetail = result.Action
if req.remediated != nil {
req.remediated[f.FilePath] = scanJobRemediation{
status: f.RemediationStatus,
detail: result.Action,
}
}
} else {
req.work.fail()
f.RemediationStatus = "failed"
f.RemediationDetail = result.Error
}
return f
}
// startScanJobManager initialises the ScanJobManager and wires it into the
// daemon's lifecycle. Call this from Daemon.Run() after store.Global() is set
// and before startControlListener() so the control socket can immediately
// accept job submissions.
func (d *Daemon) startScanJobManager() (*ScanJobManager, error) {
m, err := NewScanJobManager(d.store, d.cfg)
if err != nil {
return nil, err
}
d.registerQueueSource("scans", m)
// Track the manager's lifetime in the daemon wait-group using the same
// obs.Go + defer d.wg.Done() pattern every other background worker uses.
// NewScanJobManager already started the worker goroutine via its own
// obs.Go call; this entry just blocks d.wg.Wait() until that goroutine
// finishes draining. Stop() must be called before d.wg.Wait() so that
// workerDone is already closed by the time we reach here.
d.wg.Add(1)
obs.Go("scan-job-manager", func() {
defer d.wg.Done()
<-m.workerDone
})
return m, nil
}
package daemon
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/queuehealth"
)
const scanJobControlBudget = time.Minute
type scanJobHealth struct {
mu sync.Mutex
pending map[*scanJobWork]struct{}
losses *queuehealth.Tracker
admission *queuehealth.Tracker
progress time.Time
fullSince time.Time
}
type scanJobWork struct {
owner *scanJobHealth
checks func(time.Time) checks.DispatchProgressSnapshot
started time.Time
progress time.Time
failed bool
}
func newScanJobHealth() *scanJobHealth {
return &scanJobHealth{
pending: make(map[*scanJobWork]struct{}),
losses: queuehealth.New(0, scanJobControlBudget),
admission: queuehealth.New(0, scanJobControlBudget),
}
}
func (h *scanJobHealth) begin(progress func(time.Time) checks.DispatchProgressSnapshot) *scanJobWork {
h.mu.Lock()
defer h.mu.Unlock()
now := time.Now()
if len(h.pending) == 0 {
h.progress = now
}
w := &scanJobWork{owner: h, checks: progress, progress: now}
h.pending[w] = struct{}{}
h.updateFull(now)
return w
}
func (h *scanJobHealth) updateFull(now time.Time) {
waiting := 0
for w := range h.pending {
if w.started.IsZero() {
waiting++
}
}
if waiting >= scanJobQueueDepth {
if h.fullSince.IsZero() {
h.fullSince = now
}
} else {
h.fullSince = time.Time{}
}
}
func (w *scanJobWork) progressed() {
w.owner.mu.Lock()
defer w.owner.mu.Unlock()
w.progress = time.Now()
w.owner.progress = w.progress
}
func (w *scanJobWork) fail() {
w.owner.mu.Lock()
defer w.owner.mu.Unlock()
w.failLocked(time.Now())
}
func (w *scanJobWork) failLocked(now time.Time) {
if !w.failed {
w.failed = true
w.owner.losses.Lose(now, 1)
}
}
func (w *scanJobWork) finish(completed bool) {
h := w.owner
h.mu.Lock()
defer h.mu.Unlock()
now := time.Now()
if !completed {
w.failLocked(now)
}
delete(h.pending, w)
if !w.started.IsZero() {
h.progress = now
}
h.updateFull(now)
}
func (w *scanJobWork) run(fn func()) {
h := w.owner
h.mu.Lock()
w.started = time.Now()
w.progress = w.started
h.progress = w.started
h.updateFull(w.started)
h.mu.Unlock()
completed := false
defer func() { w.finish(completed) }()
fn()
completed = true
}
func (h *scanJobHealth) snapshot(now time.Time) queuehealth.Status {
h.mu.Lock()
defer h.mu.Unlock()
s := h.losses.Snapshot(now)
s.Capacity = scanJobQueueDepth
s.LagBasis = "consumer_progress"
progress := h.progress
var overdue bool
for w := range h.pending {
if w.started.IsZero() {
s.Depth++
continue
}
s.InFlight++
s.ProcessingSeconds = max(s.ProcessingSeconds, now.Sub(w.started).Seconds())
child := w.checks(now)
latest := w.progress
if child.LastProgress.After(latest) {
latest = child.LastProgress
}
if latest.After(progress) {
progress = latest
}
if child.Active {
overdue = overdue || child.Overdue
} else {
overdue = overdue || now.Sub(latest) >= scanJobControlBudget
}
}
if s.Depth > 0 {
s.LagSeconds = max(0, now.Sub(progress).Seconds())
}
switch {
case overdue:
s.Reason = "processing_lag"
case s.Depth > 0 && s.InFlight == 0 && now.Sub(progress) >= scanJobControlBudget:
s.Reason = "backlog_lag"
case !h.fullSince.IsZero() && now.Sub(h.fullSince) >= 30*time.Second:
s.Reason = "queue_full"
}
if s.Reason != "" {
s.Status = "degraded"
}
return s
}
// QueueStatuses retains work through persistence and worker cleanup. It never
// takes the cancellation lock or asks the store for progress.
func (m *ScanJobManager) QueueStatuses(now time.Time) map[string]queuehealth.Status {
admission := m.health.admission.Snapshot(now)
admission.CapacityUnavailable = true
return map[string]queuehealth.Status{"jobs": m.health.snapshot(now), "admission": admission}
}
package daemon
import (
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/store"
)
type scanJobStore interface {
PutScanJob(store.ScanJobRecord) error
GetScanJob(string) (store.ScanJobRecord, bool, error)
ListScanJobs() ([]store.ScanJobRecord, error)
AppendScanJobFindings(string, int, []alert.Finding) error
ListScanJobFindings(string, int, int) ([]alert.Finding, int, error)
PruneScanJobs(int, int) (int, error)
}
// Only worker operations advance the job clock. Status polling through the
// manager's store must not conceal stalled worker persistence.
type scanJobWorkStore struct {
scanJobStore
work *scanJobWork
}
func (req scanJobRequest) trackedStore(db scanJobStore) scanJobWorkStore {
return scanJobWorkStore{scanJobStore: db, work: req.work}
}
func (db scanJobWorkStore) PutScanJob(rec store.ScanJobRecord) error {
db.work.progressed()
err := db.scanJobStore.PutScanJob(rec)
if err != nil {
db.work.fail()
}
db.work.progressed()
return err
}
func (db scanJobWorkStore) GetScanJob(id string) (store.ScanJobRecord, bool, error) {
db.work.progressed()
rec, ok, err := db.scanJobStore.GetScanJob(id)
if err != nil || !ok {
db.work.fail()
}
db.work.progressed()
return rec, ok, err
}
func (db scanJobWorkStore) AppendScanJobFindings(id string, seq int, findings []alert.Finding) error {
db.work.progressed()
err := db.scanJobStore.AppendScanJobFindings(id, seq, findings)
if err != nil {
db.work.fail()
}
db.work.progressed()
return err
}
func (db scanJobWorkStore) PruneScanJobs(keep, maxFindings int) (int, error) {
db.work.progressed()
n, err := db.scanJobStore.PruneScanJobs(keep, maxFindings)
if err != nil {
db.work.fail()
}
db.work.progressed()
return n, err
}
package daemon
import (
"fmt"
"maps"
"net"
"os"
"slices"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/eximlog"
"github.com/pidginhost/csm/internal/store"
)
// --- Authenticated sender profile ---------------------------------------
//
// A stolen mailbox password is used politely: well under any global rate
// limit, from residential addresses with no cloud PTR, with each message to
// a different victim. What gives it away is the mailbox's own history. The
// owner sends from one or two addresses a day in one country to a stable set
// of correspondents; the thief rotates addresses per message, appears from a
// country the mailbox never sent from, or mails more distinct recipients in a
// day than the owner does in a month. Credential checkers that validate a
// stolen password send one probe each from several countries within minutes.
//
// Each authenticated arrival updates a rolling one-hour window in memory and
// the mailbox's per-day record in the store. Thresholds are the larger of a
// floor and twice the mailbox's own prior maximum, so a busy shared mailbox
// earns a higher bar than a one-person box. Address blocking is deliberately
// not used: the sources rotate and may include the owner's own address, so a
// Critical finding suspends the mailbox instead.
const (
senderHourWindow = time.Hour
senderMinCountriesHour = 3 // distinct source countries in one hour; no traveller reaches it
senderIPFloorHour = 4 // distinct source addresses in one hour
senderIPFloorDay = 6 // distinct source addresses in one UTC day
senderRcptFloorDay = 50 // distinct recipients in one UTC day, each in its own message
senderBaselineDays = 14 // prior days that form the mailbox's own maxima
senderBaselineMinDays = 3 // prior active days before an unseen country counts as new
senderDedupCooldown = time.Hour
senderMaxHourEvents = 1024
senderFlushInterval = time.Minute
senderMaxDaySet = 512 // per-day cap on distinct addresses and recipients kept
)
type senderEvent struct {
at time.Time
ip string
}
type senderWindow struct {
mu sync.Mutex
events []senderEvent
firedAt time.Time
firedSeverity alert.Severity
lastEvent time.Time
countries map[string]time.Time
profile store.SenderProfile
baseline senderBaselineStats
dayKey string
db *store.DB
loaded bool
dirty bool
loadAfter time.Time
}
// senderWindows holds the rolling hour per authenticated mailbox.
var senderWindows sync.Map // map[string]*senderWindow
func lockSenderWindow(user string, now time.Time) *senderWindow {
for {
val, _ := senderWindows.LoadOrStore(user, &senderWindow{lastEvent: now})
w := val.(*senderWindow)
w.mu.Lock()
// Re-check under the lock so an eviction sweep cannot delete the
// entry between the load and the update.
if current, ok := senderWindows.Load(user); ok && current == val {
return w
}
w.mu.Unlock()
}
}
type senderBaselineStats struct {
activeDays int
maxHourIPs int
maxDayIPs int
maxRecipients int
countries map[string]bool
}
// senderBaseline summarises the prior days inside the baseline window.
func senderBaseline(p store.SenderProfile, now time.Time) senderBaselineStats {
b := senderBaselineStats{countries: map[string]bool{}}
today := senderDayKey(now)
oldest := senderDayKey(now.UTC().AddDate(0, 0, -senderBaselineDays))
for key, d := range p.Days {
if d == nil || key >= today || key < oldest {
continue
}
b.activeDays++
b.maxHourIPs = max(b.maxHourIPs, d.MaxHourIPs)
b.maxDayIPs = max(b.maxDayIPs, len(d.IPs))
b.maxRecipients = max(b.maxRecipients, len(d.Recipients))
for _, c := range d.Countries {
b.countries[c] = true
}
}
return b
}
func senderDayKey(t time.Time) string { return t.UTC().Format("2006-01-02") }
// senderThreshold is the floor or twice the mailbox's own maximum plus one,
// whichever is higher.
func senderThreshold(floor, own int) int {
return max(floor, 2*own+1)
}
func addDistinct(set []string, v string, limit int) []string {
if slices.Contains(set, v) || len(set) >= limit {
return set
}
return append(set, v)
}
func countryTrusted(country string, trusted []string) bool {
for _, tc := range trusted {
if strings.EqualFold(country, tc) {
return true
}
}
return false
}
func sortedKeys[V any](m map[string]V) []string {
return slices.Sorted(maps.Keys(m))
}
func recentSenderIPs(events []senderEvent, n int) []string {
var ips []string
for i := len(events) - 1; i >= 0 && len(ips) < n; i-- {
ips = addDistinct(ips, events[i].ip, n)
}
return ips
}
// parseSenderProfileFinding evaluates one authenticated Exim arrival against
// the sender's own history. It returns at most one email_compromised_account
// finding and never acts on its own; the caller responds to Critical ones.
func parseSenderProfileFinding(line string, cfg *config.Config, now time.Time) (findings []alert.Finding) {
if !strings.Contains(line, " <= ") || !strings.Contains(line, "A=dovecot_") {
return nil
}
user := extractAuthUser(line)
if user == "" {
return nil
}
ip := eximlog.ClientIP(line)
if ip == "" {
return nil
}
if parsed := net.ParseIP(ip); parsed != nil {
ip = parsed.String()
}
return senderProfileObserve(cfg, user, ip, eximlog.Recipients(line), now)
}
// senderProfileObserve judges one authenticated send by user against the
// mailbox's own history. ip is the client address, or "" for a local
// submission, which counts toward volume and recipients but adds no address
// or country.
func senderProfileObserve(cfg *config.Config, user, ip string, recipients []string, now time.Time) (findings []alert.Finding) {
if user == "" || isHighVolumeSender(user, cfg.EmailProtection.HighVolumeSenders) {
return nil
}
country := ""
if ip != "" && !isPrivateOrLoopback(ip) && !isInfraIPDaemon(ip, cfg.InfraIPs) {
country = geoLookup(ip).Country
}
db := store.Global()
// Registered before the unlock defer so owner I/O runs after it.
defer func() { stampMailAccountOwner(findings, user) }()
unlock := lockMailResponse(store.MailboxSuspensionID(activeMailBackend.name(), user))
defer unlock()
w := lockSenderWindow(user, now)
defer w.mu.Unlock()
if !w.loaded || w.db != db {
var profile store.SenderProfile
if db != nil {
if now.Before(w.loadAfter) {
return nil
}
var err error
profile, err = db.GetSenderProfile(user)
if err != nil {
w.loadAfter = now.Add(senderFlushInterval)
fmt.Fprintf(os.Stderr, "[%s] Warning: failed to load sender profile for %s: %v\n",
now.Format("2006-01-02 15:04:05"), user, err)
return nil
}
}
w.profile = profile
w.db, w.loaded, w.dayKey = db, true, ""
}
if w.profile.Days == nil {
w.profile.Days = map[string]*store.SenderDay{}
}
// Concurrent readers can capture their timestamps in the opposite order
// to acquiring this lock. Never move the cache's day or idle time back.
if now.Before(w.lastEvent) {
now = w.lastEvent
}
w.lastEvent = now
w.observeSource(now, ip, country)
hourIPs := len(w.events)
hourCountries := w.countries
today := senderDayKey(now)
if w.dayKey != today {
oldest := senderDayKey(now.UTC().AddDate(0, 0, -senderBaselineDays))
for key := range w.profile.Days {
if key < oldest {
delete(w.profile.Days, key)
}
}
w.baseline = senderBaseline(w.profile, now)
w.dayKey = today
}
day := w.profile.Days[today]
if day == nil {
day = &store.SenderDay{}
w.profile.Days[today] = day
}
day.Sends++
if ip != "" {
day.IPs = addDistinct(day.IPs, ip, senderMaxDaySet)
}
if country != "" {
day.Countries = addDistinct(day.Countries, country, senderMaxDaySet)
}
var messageRecipients []string
for _, r := range recipients {
r = strings.ToLower(r)
day.Recipients = addDistinct(day.Recipients, r, senderMaxDaySet)
// Only the distinction between one and multiple recipients matters.
messageRecipients = addDistinct(messageRecipients, r, 2)
}
if len(messageRecipients) == 1 {
day.SingleRecipients = addDistinct(day.SingleRecipients, messageRecipients[0], senderMaxDaySet)
}
day.MaxHourIPs = max(day.MaxHourIPs, hourIPs)
w.dirty = true
base := w.baseline
var reasons []string
if len(hourCountries) >= senderMinCountriesHour {
reasons = append(reasons, fmt.Sprintf("sent from %d countries within an hour (%s)",
len(hourCountries), strings.Join(sortedKeys(hourCountries), ", ")))
}
critical := len(reasons) > 0
var churn []string
if hourIPs >= senderThreshold(senderIPFloorHour, base.maxHourIPs) {
churn = append(churn, fmt.Sprintf("sent from %d addresses within an hour (own history: at most %d)",
hourIPs, base.maxHourIPs))
}
if len(day.IPs) >= senderThreshold(senderIPFloorDay, base.maxDayIPs) {
churn = append(churn, fmt.Sprintf("sent from %d addresses today (own history: at most %d a day)",
len(day.IPs), base.maxDayIPs))
}
if rcptThreshold := senderThreshold(senderRcptFloorDay, base.maxRecipients); len(day.SingleRecipients) >= rcptThreshold {
churn = append(churn, fmt.Sprintf("mailed %d recipients individually today (own history: at most %d a day)",
len(day.SingleRecipients), base.maxRecipients))
}
if len(churn) == 0 && !critical {
return nil
}
reasons = append(reasons, churn...)
if len(churn) > 0 && base.activeDays >= senderBaselineMinDays {
for _, seenCountry := range day.Countries {
if !base.countries[seenCountry] && !countryTrusted(seenCountry, cfg.Suppressions.TrustedCountries) {
reasons = append(reasons, fmt.Sprintf("country %s never seen for this mailbox in %d active days (known: %s)",
seenCountry, base.activeDays, strings.Join(sortedKeys(base.countries), ", ")))
critical = true
break
}
}
}
severity := alert.High
if critical {
severity = alert.Critical
}
// One finding per severity per cooldown; an escalation passes.
if !w.firedAt.IsZero() && now.Sub(w.firedAt) < senderDedupCooldown && severity <= w.firedSeverity {
return nil
}
w.firedAt, w.firedSeverity = now, severity
message := fmt.Sprintf("Email account %s: %s", user, strings.Join(reasons, "; "))
if critical {
message += " - credentials compromised"
}
details := fmt.Sprintf(
"Authenticated SMTP submissions for %s:\n"+
" last hour: %d source addresses, %d countries\n"+
" today (UTC): %d messages, %d source addresses, %d recipients\n"+
" own history (%d active days of %d): at most %d addresses an hour, %d a day, %d recipients a day\n"+
" recent addresses: %s\n\n"+
"The mailbox's own history sets the bar, so a shared or travelling mailbox is judged "+
"against its habit rather than a global number. Source addresses are not blocked because "+
"they rotate and may include the owner's own; a Critical finding suspends the mailbox's "+
"logins and outgoing mail under the auto-response settings.",
user, hourIPs, len(hourCountries),
day.Sends, len(day.IPs), len(day.Recipients),
base.activeDays, senderBaselineDays, base.maxHourIPs, base.maxDayIPs, base.maxRecipients,
strings.Join(recentSenderIPs(w.events, 8), ", "))
mailbox, domain, _ := splitMailAccount(user)
return []alert.Finding{{
Severity: severity,
Check: "email_compromised_account",
Message: message,
Details: truncateDaemon(details, 900),
Mailbox: mailbox,
Domain: domain,
Timestamp: now,
}}
}
// respondToMailboxCompromise suspends the abused credential and marks its
// domain so rate alerts stay quiet while the response settles. The domain
// marker is correlation state, not an auto-response action, so it is kept
// even when the suspension is disabled or dry-run gated. It returns true when
// sending from the mailbox was stopped.
func respondToMailboxCompromise(cfg *config.Config, authUser, check, reason string, at time.Time) (stopped bool) {
domain := extractDomainFromEmail(authUser)
if domain == "" {
return false
}
if hostHasMailBackend() {
stopped = maybeSuspendMailbox(cfg, authUser, check, reason, at)
} else {
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: no mailbox suspension on this mail server; %s left active\n",
time.Now().Format("2006-01-02 15:04:05"), authUser)
}
RecordCompromisedDomain(domain)
return stopped
}
// forgetSenderEvidence drops a released mailbox's sending evidence: its
// rolling hour and today's counts. Sends from before a release must not
// suspend the mailbox again.
func forgetSenderEvidence(user string, now time.Time) error {
if !strings.Contains(user, "@") {
return nil
}
db := store.Global()
keys := map[string]bool{}
if db != nil {
stored, err := db.SenderProfileKeys(user)
if err != nil {
return err
}
for _, key := range stored {
keys[key] = true
}
}
senderWindows.Range(func(key, _ any) bool {
if strings.EqualFold(key.(string), user) {
keys[key.(string)] = true
}
return true
})
for key := range keys {
if err := forgetSenderWindow(key, now, db); err != nil {
return err
}
}
return nil
}
func forgetSenderWindow(user string, now time.Time, db *store.DB) error {
w := lockSenderWindow(user, now)
defer w.mu.Unlock()
if (!w.loaded || w.db != db) && db != nil {
profile, err := db.GetSenderProfile(user)
if err != nil {
return fmt.Errorf("load sender profile for %s: %w", user, err)
} else {
w.profile = profile
w.db, w.loaded, w.dayKey = db, true, ""
}
}
w.events, w.countries = nil, nil
w.firedAt, w.firedSeverity = time.Time{}, 0
if w.loaded {
delete(w.profile.Days, senderDayKey(now))
w.dirty = true
if !w.flush(user) {
return fmt.Errorf("save cleared sender profile for %s", user)
}
}
return nil
}
// senderEvictWindow is how long a per-user window may stay idle before it is
// evicted: twice the rolling hour, so the dedup guard outlives the window.
const senderEvictWindow = 2 * senderHourWindow
// runSenderProfileMaintenance saves dirty profiles independently of mail
// throughput. Shutdown performs a final save after the log readers stop.
func runSenderProfileMaintenance(stopCh <-chan struct{}) {
ticker := time.NewTicker(senderFlushInterval)
defer ticker.Stop()
for {
select {
case <-stopCh:
return
case now := <-ticker.C:
flushSenderProfiles()
evictSenderWindows(now)
}
}
}
func flushSenderProfiles() {
senderWindows.Range(func(key, val any) bool {
w := val.(*senderWindow)
w.mu.Lock()
w.flush(key.(string))
w.mu.Unlock()
return true
})
}
// Caller holds mu through the save: clearing dirty must never erase a
// concurrent submission, and eviction must not discard an unsaved profile.
func (w *senderWindow) flush(user string) bool {
if !w.dirty || w.db == nil {
return true
}
if err := w.db.SetSenderProfile(user, w.profile); err != nil {
fmt.Fprintf(os.Stderr, "[%s] Warning: failed to save sender profile for %s: %v\n",
time.Now().Format("2006-01-02 15:04:05"), user, err)
return false
}
w.dirty = false
return true
}
func evictSenderWindows(now time.Time) {
cutoff := now.Add(-senderEvictWindow)
senderWindows.Range(func(key, val any) bool {
w := val.(*senderWindow)
w.mu.Lock()
if w.lastEvent.Before(cutoff) && w.flush(key.(string)) {
senderWindows.CompareAndDelete(key, val)
}
w.mu.Unlock()
return true
})
pruneSenderHistory(now)
}
var senderHistoryPrune struct {
sync.Mutex
db *store.DB
day string
}
func pruneSenderHistory(now time.Time) {
db := store.Global()
if db == nil {
return
}
senderHistoryPrune.Lock()
defer senderHistoryPrune.Unlock()
today := senderDayKey(now)
if senderHistoryPrune.db == db && senderHistoryPrune.day == today {
return
}
oldest := senderDayKey(now.UTC().AddDate(0, 0, -senderBaselineDays))
if err := db.PruneSenderProfiles(oldest); err != nil {
fmt.Fprintf(os.Stderr, "[%s] Warning: failed to expire sender history: %v\n",
now.Format("2006-01-02 15:04:05"), err)
return
}
senderHistoryPrune.db, senderHistoryPrune.day = db, today
}
func (w *senderWindow) observeSource(now time.Time, ip, country string) {
cutoff := now.Add(-senderHourWindow)
kept := w.events[:0]
for _, e := range w.events {
if e.at.After(cutoff) && e.ip != ip {
kept = append(kept, e)
}
}
// Store the last observation of each source, so repeated sends cannot
// fill the cap or displace another source's still-active evidence.
if ip != "" {
if len(kept) == senderMaxHourEvents {
copy(kept, kept[1:])
kept = kept[:len(kept)-1]
}
kept = append(kept, senderEvent{at: now, ip: ip})
}
w.events = kept
if w.countries == nil {
w.countries = make(map[string]time.Time)
}
for c, at := range w.countries {
if !at.After(cutoff) {
delete(w.countries, c)
}
}
// Countries have their own clock: source-cap eviction must not erase
// evidence of travel while that country is still inside the hour.
if country != "" {
w.countries[country] = now
}
}
//go:build !(linux && bpf)
package daemon
import (
"context"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/config"
)
// sensitiveFileBPF is the no-tag placeholder for the BPF sensitive-file
// monitor. The real type with BPF map handles, link, and the ringbuf reader
// lives in sensitive_file_bpf.go behind //go:build linux && bpf.
type sensitiveFileBPF struct{}
func (s *sensitiveFileBPF) Mode() string { return "bpf" }
func (s *sensitiveFileBPF) EventCount() uint64 { return 0 }
func (s *sensitiveFileBPF) Run(_ context.Context) {}
func startSensitiveFileBPF(_ context.Context, _ chan<- alert.Finding, _ *config.Config) (*sensitiveFileBPF, error) {
return nil, bpf.ErrNotBuilt
}
// Code generated by bpf2go; DO NOT EDIT.
//go:build 386 || amd64
package sensitive_file_bpfprog
import (
"bytes"
_ "embed"
"fmt"
"io"
"structs"
"github.com/cilium/ebpf"
)
type SensitiveFileCsmQueueStats struct {
_ structs.HostLayout
Lost uint64
Submitted uint64
}
type SensitiveFileFileid struct {
_ structs.HostLayout
Dev uint64
Ino uint64
}
type SensitiveFileSensitiveEvent struct {
_ structs.HostLayout
Uid uint32
Pid uint32
Mask uint32
_ [4]byte
Dev uint64
Ino uint64
Comm [16]uint8
}
// Names of all BPF objects in the ELF.
//
// Used for safe lookups in a Collection or CollectionSpec.
const (
SensitiveFileMapEvents = "events"
SensitiveFileMapQueueStats = "queue_stats"
SensitiveFileMapWatched = "watched"
SensitiveFileProgCsmFilePerm = "csm_file_perm"
SensitiveFileVarUnused = "unused"
)
// LoadSensitiveFile returns the embedded CollectionSpec for SensitiveFile.
func LoadSensitiveFile() (*ebpf.CollectionSpec, error) {
reader := bytes.NewReader(_SensitiveFileBytes)
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
if err != nil {
return nil, fmt.Errorf("can't load SensitiveFile: %w", err)
}
return spec, err
}
// LoadSensitiveFileObjects loads SensitiveFile and converts it into a struct.
//
// The following types are suitable as obj argument:
//
// *SensitiveFileObjects
// *SensitiveFilePrograms
// *SensitiveFileMaps
//
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
func LoadSensitiveFileObjects(obj any, opts *ebpf.CollectionOptions) error {
spec, err := LoadSensitiveFile()
if err != nil {
return err
}
return spec.LoadAndAssign(obj, opts)
}
// SensitiveFileSpecs contains maps and programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type SensitiveFileSpecs struct {
SensitiveFileProgramSpecs
SensitiveFileMapSpecs
SensitiveFileVariableSpecs
}
// SensitiveFileProgramSpecs contains programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type SensitiveFileProgramSpecs struct {
CsmFilePerm *ebpf.ProgramSpec `ebpf:"csm_file_perm"`
}
// SensitiveFileMapSpecs contains maps before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type SensitiveFileMapSpecs struct {
Events *ebpf.MapSpec `ebpf:"events"`
QueueStats *ebpf.MapSpec `ebpf:"queue_stats"`
Watched *ebpf.MapSpec `ebpf:"watched"`
}
// SensitiveFileVariableSpecs contains global variables before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type SensitiveFileVariableSpecs struct {
Unused *ebpf.VariableSpec `ebpf:"unused"`
}
// SensitiveFileObjects contains all objects after they have been loaded into the kernel.
//
// It can be passed to LoadSensitiveFileObjects or ebpf.CollectionSpec.LoadAndAssign.
type SensitiveFileObjects struct {
SensitiveFilePrograms
SensitiveFileMaps
SensitiveFileVariables
}
func (o *SensitiveFileObjects) Close() error {
return _SensitiveFileClose(
&o.SensitiveFilePrograms,
&o.SensitiveFileMaps,
)
}
// SensitiveFileMaps contains all maps after they have been loaded into the kernel.
//
// It can be passed to LoadSensitiveFileObjects or ebpf.CollectionSpec.LoadAndAssign.
type SensitiveFileMaps struct {
Events *ebpf.Map `ebpf:"events"`
QueueStats *ebpf.Map `ebpf:"queue_stats"`
Watched *ebpf.Map `ebpf:"watched"`
}
func (m *SensitiveFileMaps) Close() error {
return _SensitiveFileClose(
m.Events,
m.QueueStats,
m.Watched,
)
}
// SensitiveFileVariables contains all global variables after they have been loaded into the kernel.
//
// It can be passed to LoadSensitiveFileObjects or ebpf.CollectionSpec.LoadAndAssign.
type SensitiveFileVariables struct {
Unused *ebpf.Variable `ebpf:"unused"`
}
// SensitiveFilePrograms contains all programs after they have been loaded into the kernel.
//
// It can be passed to LoadSensitiveFileObjects or ebpf.CollectionSpec.LoadAndAssign.
type SensitiveFilePrograms struct {
CsmFilePerm *ebpf.Program `ebpf:"csm_file_perm"`
}
func (p *SensitiveFilePrograms) Close() error {
return _SensitiveFileClose(
p.CsmFilePerm,
)
}
func _SensitiveFileClose(closers ...io.Closer) error {
for _, closer := range closers {
if err := closer.Close(); err != nil {
return err
}
}
return nil
}
// Do not access this directly.
//
//go:embed sensitivefile_x86_bpfel.o
var _SensitiveFileBytes []byte
//go:build linux
package daemon
import "golang.org/x/sys/unix"
// kernelDeviceID converts the device number stat(2) reports, which uses the
// glibc encoding (major<<8)|(minor&0xff)|((minor&~0xff)<<12), into the
// kernel's own dev_t (major<<20)|minor. That is what an LSM program reads from
// inode->i_sb->s_dev, so a BPF map keyed by device must use this form. The two
// encodings agree only for major 0 (tmpfs, overlay), which is why a map keyed
// with the raw stat value works in test rigs and misses every file on a real
// block device.
func kernelDeviceID(statDev uint64) uint64 {
return uint64(unix.Major(statDev))<<20 | uint64(unix.Minor(statDev))
}
// userDeviceID is the inverse of kernelDeviceID: a dev_t read in BPF back to
// the stat(2) form that fstat on the same file reports.
func userDeviceID(kernelDev uint64) uint64 {
return unix.Mkdev(uint32(kernelDev>>20), uint32(kernelDev&(1<<20-1))) // #nosec G115 -- the kernel dev_t is 32 bits
}
package daemon
import (
"encoding/binary"
"errors"
)
// SensitiveFileEvent matches struct sensitive_event in sensitive_file.bpf.c
// byte for byte. Userspace looks up (Dev, Ino) in the in-memory mirror of
// the BPF watchset map to recover the path string at finding time.
type SensitiveFileEvent struct {
UID uint32
PID uint32
Mask uint32
Dev uint64
Ino uint64
Comm string
}
const sensitiveFileEventSize = 4 + 4 + 4 + 4 + 8 + 8 + 16
func decodeSensitiveFileEvent(b []byte) (SensitiveFileEvent, error) {
if len(b) < sensitiveFileEventSize {
return SensitiveFileEvent{}, errors.New("sensitive file event short buffer")
}
ev := SensitiveFileEvent{
UID: binary.LittleEndian.Uint32(b[0:4]),
PID: binary.LittleEndian.Uint32(b[4:8]),
Mask: binary.LittleEndian.Uint32(b[8:12]),
Dev: binary.LittleEndian.Uint64(b[16:24]),
Ino: binary.LittleEndian.Uint64(b[24:32]),
}
ev.Comm = nullTerm(b[32 : 32+16])
return ev, nil
}
package daemon
import (
"context"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/state"
)
// sensitiveFilePoller wraps checks.CheckSensitiveFiles in a goroutine.
// Used when the BPF backend is unavailable or operator-disabled. Detection
// latency equals the poll interval (default 5 minutes).
type sensitiveFilePoller struct {
cfg *config.Config
store *state.Store
alertCh chan<- alert.Finding
count atomic.Uint64
}
func newSensitiveFilePoller(cfg *config.Config, store *state.Store, alertCh chan<- alert.Finding) *sensitiveFilePoller {
return &sensitiveFilePoller{cfg: cfg, store: store, alertCh: alertCh}
}
func (p *sensitiveFilePoller) Mode() string { return "legacy" }
func (p *sensitiveFilePoller) EventCount() uint64 { return p.count.Load() }
func (p *sensitiveFilePoller) Run(ctx context.Context) {
interval := sensitiveFilePollerInterval(p.cfg)
t := time.NewTicker(interval)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
for _, f := range checks.CheckSensitiveFiles(ctx, p.cfg, p.store) {
p.count.Add(1)
if !alert.TryEnqueue(p.alertCh, f) {
csmlog.Warn("sensitive_file legacy: alert channel full, dropping finding")
}
}
}
}
}
func sensitiveFilePollerInterval(cfg *config.Config) time.Duration {
if d := cfg.Detection.SensitiveFilesPollInterval; d > 0 {
return d
}
return 5 * time.Minute
}
package daemon
import (
"context"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/state"
)
// StartSensitiveFileMonitor starts the live sensitive-file write monitor chosen by
// detection.sensitive_files_backend; see startLiveBackend for the choices.
func StartSensitiveFileMonitor(alertCh chan<- alert.Finding, cfg *config.Config, store *state.Store) bpf.Backend {
return startLiveBackend("sensitive_files", cfg.Detection.SensitiveFilesBackend, alertCh,
func() (bpf.Backend, error) { return tryStartSensitiveFileBPFFn(context.Background(), alertCh, cfg) },
func() bpf.Backend { return newSensitiveFilePoller(cfg, store, alertCh) })
}
var tryStartSensitiveFileBPFFn = tryStartSensitiveFileBPF
func tryStartSensitiveFileBPF(ctx context.Context, ch chan<- alert.Finding, cfg *config.Config) (bpf.Backend, error) {
b, err := startSensitiveFileBPF(ctx, ch, cfg)
if err != nil {
return nil, err
}
return b, nil
}
package daemon
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/store"
)
// Signature-update-driven retroactive rescan.
//
// CSM's signature rules update independently of the deep-tier
// scanner. The rolling deep YARA walk reaches every existing file once
// per pass, but a pass spans many deep ticks, so a rules update is only
// applied to all existing files when the walk has come back round to
// where it stood at the update. This watcher makes that point visible.
//
// The watcher polls cfg.Signatures.RulesDir every sigWatchInterval,
// stat()s every *.yaml / *.yml / *.yar / *.yara file, and queues a
// rescan in the store whenever any tracked file's content changes. The
// rolling deep YARA walk (checks.CheckYARADeep) tracks the queued
// generation and clears it once it has scanned every file since then.
//
// A file is hashed only when its mtime or size moved. Package upgrades
// and the rules updater rewrite files whose content is unchanged, so a
// moved mtime alone is not a reason to queue a rescan.
//
// The per-file state is persisted in bbolt (sig_watch bucket) so a
// daemon restart does not look like "all files are new" and trigger a
// phantom rescan on first tick.
//
// The queued rescan is persisted in the same transaction as the state
// that caused it, so a restart cannot keep the new state and drop the
// rescan it calls for.
const sigWatchInterval = 60 * time.Second
var sigWatchExtensions = []string{".yaml", ".yml", ".yar", ".yara"}
// sigWatcher carries the watcher's loop state. The daemon owns one
// instance; the goroutine in (*Daemon).signatureWatcher drives it.
//
// cfg and store are re-resolved per tick, not captured at
// construction. Originally we cached cfg.Signatures.RulesDir and
// store.Global() into struct fields and discovered two ways that
// could go wrong: a hot-reload that changed the rules dir would
// silently keep walking the old path, and a daemon ordering quirk
// where store.Global() is nil at goroutine spawn would leave the
// watcher persistence-blind for the rest of its lifetime. Live
// resolution closes both.
type sigWatcher struct {
cfgFunc func() *config.Config
storeFunc func() *store.DB
alertCh chan<- alert.Finding
interval time.Duration
hashFile func(string, os.FileInfo) (string, error)
// Initialised on first tick from store.GetSignatureFiles(); the
// in-memory map is the authoritative working copy for the loop.
last map[string]store.SignatureFileState
// loaded distinguishes a successful store read from a baseline built
// while the store was unavailable. Failed startup reads are retried.
loaded bool
// persisted is false until last has been written to bbolt, and again
// after it changes, so an unchanged tick commits nothing.
persisted bool
// queueRescan is true from a detected change until the queued rescan
// is written with the state, so a failed write is retried with it.
queueRescan bool
}
// newSigWatcher constructs a watcher with production defaults.
// cfgFunc and storeFunc are called per tick so config hot-reloads
// and lazy bbolt initialisation are picked up automatically.
// Callers can override the interval after construction for tests.
func newSigWatcher(cfgFunc func() *config.Config, storeFunc func() *store.DB, alertCh chan<- alert.Finding) *sigWatcher {
return &sigWatcher{
cfgFunc: cfgFunc,
storeFunc: storeFunc,
alertCh: alertCh,
interval: sigWatchInterval,
hashFile: hashRulesFile,
}
}
// loadInitial restores saved work even if earlier ticks ran without a store.
// A corrupt queue must be replaced before accepting the current rule state.
func (w *sigWatcher) loadInitial(sdb *store.DB) {
_, queueErr := sdb.SignatureRescanPending()
if queueErr != nil {
csmlog.Warn("sig_watch: loading queued rescan", "err", queueErr)
if errors.Is(queueErr, store.ErrSignatureRescanCorrupt) {
w.queueRescan = true
}
}
got, err := sdb.GetSignatureFiles()
if err != nil {
csmlog.Warn("sig_watch: loading persisted state", "err", err)
return
}
if got == nil {
got = make(map[string]store.SignatureFileState)
}
w.persisted = !w.queueRescan
// Saved comparisons take precedence over a first observation made
// without the store. Already detected in-memory changes are retained
// with their owed queue, and newly observed paths keep their baseline.
for path, state := range w.last {
if _, exists := got[path]; !exists || w.queueRescan {
got[path] = state
w.persisted = false
}
}
w.last = got
w.loaded = queueErr == nil || errors.Is(queueErr, store.ErrSignatureRescanCorrupt)
}
// tick performs one walk of the rules dir and queues a rescan when any
// tracked file's content changed. Removed files drop out of
// the persisted map without triggering a rescan -- only a change to
// an existing file is a trigger.
func (w *sigWatcher) tick() {
cfg := w.cfgFunc()
if !cfg.SignatureRescanEnabled() {
return
}
rulesDir := cfg.Signatures.RulesDir
if rulesDir == "" {
return
}
sdb := w.storeFunc()
if !w.loaded && sdb != nil {
w.loadInitial(sdb)
}
if w.last == nil {
w.last = map[string]store.SignatureFileState{}
}
current := walkRulesDir(rulesDir)
next := make(map[string]store.SignatureFileState, len(current))
var changed []sigWatchChange
for path, file := range current {
info := file.info
old, seen := w.last[path]
if seen && old.SHA256 != "" && old.Size == info.Size() && old.Mtime.Equal(info.ModTime()) {
next[path] = old
continue
}
digest, err := w.hashFile(path, info)
if err != nil {
csmlog.Warn("sig_watch: hashing rules", "path", path, "err", err)
if !seen {
continue
}
// Keep the last good comparison point, including its stamp, so
// the next tick retries instead of accepting an unreadable update.
if old.SHA256 != "" {
next[path] = old
continue
}
// With no recorded hash, preserve the legacy mtime fallback
// even when this read cannot establish a content baseline.
}
state := store.SignatureFileState{Mtime: info.ModTime(), Size: info.Size(), SHA256: digest}
next[path] = state
switch {
case !seen:
// New file. First observation is a
// non-event so a fresh `update-rules` install does not
// cause a rescan when the daemon also starts cold.
case old.SHA256 != "":
if old.SHA256 != state.SHA256 {
changed = append(changed, sigWatchChange{Path: path, Old: old.Mtime, New: state.Mtime})
}
case old.Size < 0 && old.Mtime.Equal(file.legacyMtime),
old.Size == state.Size && old.Mtime.Equal(state.Mtime):
// A legacy record with the same stamp gains a hash without
// treating the first tick after upgrade as a rules change.
default:
// The stamp moved and the contents cannot be compared, so
// this is treated as a change, as it was before content
// hashes were tracked.
changed = append(changed, sigWatchChange{Path: path, Old: old.Mtime, New: state.Mtime})
}
}
if !sameSignatureState(w.last, next) {
w.persisted = false
}
if len(changed) > 0 {
w.queueRescan = true
}
w.last = next
if sdb != nil && !w.persisted {
var err error
if w.queueRescan {
_, err = sdb.PutSignatureFilesWithRescan(next)
} else {
err = sdb.PutSignatureFiles(next)
}
if err != nil {
csmlog.Warn("sig_watch: persisting state", "err", err)
} else {
w.persisted = true
w.queueRescan = false
}
}
for _, c := range changed {
alert.TryEnqueue(w.alertCh, alert.Finding{
Severity: alert.Warning,
Check: "signature_update_rescan_queued",
Message: fmt.Sprintf("Signature update detected, rescan of existing files queued: %s", filepath.Base(c.Path)),
Details: fmt.Sprintf("File: %s\nOld mtime: %s\nNew mtime: %s", c.Path, c.Old.UTC().Format(time.RFC3339), c.New.UTC().Format(time.RFC3339)),
FilePath: c.Path,
Timestamp: time.Now(),
})
}
}
// hashRulesFile accepts a hash only while the walked file, the open file and
// the final pathname still identify the same regular file and stamp.
func hashRulesFile(path string, expected os.FileInfo) (string, error) {
if !expected.Mode().IsRegular() {
return "", fmt.Errorf("rules file is not regular")
}
// A replacement by a FIFO between walk and open must not stall the watcher.
f, err := os.OpenFile(path, os.O_RDONLY|unix.O_NONBLOCK, 0) // #nosec G304 -- path comes from walking the operator-configured rules dir and the opened identity is checked below.
if err != nil {
return "", err
}
defer func() { _ = f.Close() }()
before, err := f.Stat()
if err != nil {
return "", err
}
if !sameRulesFile(expected, before) {
return "", fmt.Errorf("rules file changed before hashing or is not regular")
}
h := sha256.New()
n, err := io.Copy(h, io.LimitReader(f, before.Size()+1))
if err != nil {
return "", err
}
after, err := f.Stat()
if err != nil {
return "", err
}
current, err := os.Stat(path)
if err != nil {
return "", err
}
if n != before.Size() || !sameRulesFile(before, after) || !sameRulesFile(after, current) {
return "", fmt.Errorf("rules file changed while hashing")
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func sameRulesFile(a, b os.FileInfo) bool {
return a.Mode().IsRegular() && b.Mode().IsRegular() && os.SameFile(a, b) &&
a.Size() == b.Size() && a.ModTime().Equal(b.ModTime())
}
func sameSignatureState(a, b map[string]store.SignatureFileState) bool {
if len(a) != len(b) {
return false
}
for path, x := range a {
y, ok := b[path]
if !ok || x.Size != y.Size || x.SHA256 != y.SHA256 || !x.Mtime.Equal(y.Mtime) {
return false
}
}
return true
}
type sigWatchFile struct {
info os.FileInfo
// Old builds recorded the link's mtime for symlinks, not the target's.
legacyMtime time.Time
}
// walkRulesDir returns the target file info of every signature file under dir.
// Sub-directories are walked too -- the YARA Forge updater puts
// files under tier-named subfolders. Errors during walk (missing
// dir, EACCES on a sub-tree) are swallowed; we want one bad path
// not to crash the watcher or stop sibling traversal.
func walkRulesDir(dir string) map[string]sigWatchFile {
out := map[string]sigWatchFile{}
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
// Missing dir or EACCES on a sub-tree -- ignore so the
// watcher does not crash. We deliberately swallow err
// instead of returning it; filepath.SkipDir is the
// idiomatic alternative but we want to keep walking
// siblings, not the descendants of one bad path.
return filepath.SkipDir
}
if info.IsDir() {
return nil
}
ext := strings.ToLower(filepath.Ext(path))
if !sigWatchExtMatches(ext) {
return nil
}
file := sigWatchFile{info: info, legacyMtime: info.ModTime()}
if info.Mode()&os.ModeSymlink != 0 {
if target, err := os.Stat(path); err == nil {
file.info = target
}
}
out[path] = file
return nil
})
return out
}
// sigWatchExtMatches returns true when ext is one of the file
// extensions the watcher tracks. Lower-case input expected.
func sigWatchExtMatches(ext string) bool {
for _, want := range sigWatchExtensions {
if ext == want {
return true
}
}
return false
}
// sigWatchChange records one changed file for the alert detail
// message.
type sigWatchChange struct {
Path string
Old time.Time
New time.Time
}
// signatureWatcher is the daemon's signature-watch goroutine. Runs
// until d.stopCh is closed; ticks every sigWatchInterval and queues a
// rescan when any tracked rule file's content changes.
//
// Cfg and store are accessed via getter closures (not captured
// values) so a hot-reload of signatures.rules_dir takes effect on
// the next tick and a late-initialised bbolt is picked up
// automatically.
func (d *Daemon) signatureWatcher() {
defer d.wg.Done()
w := newSigWatcher(
func() *config.Config { return d.currentCfg() },
store.Global,
d.alertCh,
)
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
// Initial tick on start so the watcher converges quickly when
// the daemon comes up shortly after an `update-rules` invocation.
w.tick()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
w.tick()
}
}
}
package daemon
import (
"slices"
"strings"
"time"
)
const (
// smtpAuthPairWindow bounds the gap between Dovecot's verdict and the
// MTA's failure line for one request. Dovecot delays failure replies by
// auth_failure_delay (2 s by default) after logging the verdict.
smtpAuthPairWindow = 30 * time.Second
// smtpAuthPairMaxPending caps unmatched verdicts kept per client.
smtpAuthPairMaxPending = 16
)
type smtpRejectionTime struct{ at, loggedAt time.Time }
type smtpPendingVerdict struct {
smtpRejectionTime
account string
exist mailboxExistence
name string // original rejected name, before ambiguity removes attribution
batch *smtpVerdictBatch // shared once pending answers disagree
}
type smtpBatchName struct {
smtpRejectionTime
since time.Time // receipt boundary excluding duplicates from an earlier consumed window
name string
consumed bool
}
// smtpVerdictBatch retains distinct rejections even when duplicate passdb
// answers fill the reply queue. Each name keeps its own pairing bounds and
// consumption state. Guarded by the tracker's mutex and capped like the queue.
type smtpVerdictBatch struct{ names []smtpBatchName }
func (b *smtpVerdictBatch) add(name string, at smtpRejectionTime) {
fresh := smtpBatchName{smtpRejectionTime: at, since: at.at, name: name}
for i := range b.names {
if b.names[i].name == name {
// Duplicate answers must not extend consumption indefinitely
// when overlapping requests keep the batch alive.
if b.names[i].consumed {
if b.names[i].at.Before(at.at.Add(-smtpAuthPairWindow)) {
b.names[i] = fresh
}
} else {
b.names[i].smtpRejectionTime = at
}
return
}
}
if len(b.names) >= smtpAuthPairMaxPending {
b.names = slices.Delete(b.names, 0, 1)
}
b.names = append(b.names, fresh)
}
// take returns an independent snapshot. Another log reader may append a
// rejection after pairing releases the tracker lock and before recording it.
func (b *smtpVerdictBatch) take(pending []smtpPendingVerdict, cutoff, loggedAt time.Time, credential bool) []string {
names := make([]string, 0, len(b.names))
for i := range b.names {
n := &b.names[i]
if n.consumed {
continue
}
// An outage reply cannot identify which of the names it invalidates.
if !credential {
n.consumed = true
continue
}
// Buffered logs can put the latest rejection after this reply's
// source time. An older queued rejection may still qualify.
matches := n.matches(cutoff, loggedAt) || slices.ContainsFunc(pending, func(p smtpPendingVerdict) bool {
return p.name == n.name && !p.at.Before(n.since) && p.matches(cutoff, loggedAt)
})
if matches {
n.consumed = true
names = append(names, n.name)
}
}
return names
}
func (p smtpRejectionTime) matches(cutoff, loggedAt time.Time) bool {
return !p.at.Before(cutoff) && (p.loggedAt.IsZero() || loggedAt.IsZero() ||
(!loggedAt.Before(p.loggedAt) && loggedAt.Sub(p.loggedAt) <= smtpAuthPairWindow))
}
// NoteDovecotRejection keeps a sessionless Dovecot rejection until the MTA's
// failure line for the same client claims it. Callers filter infra, private
// and loopback addresses first.
func (t *smtpAuthTracker) NoteDovecotRejection(ip, account string, exist mailboxExistence, loggedAt time.Time) {
t.mu.Lock()
defer t.mu.Unlock()
now := t.now()
pending, exists := t.pendingVerdicts[ip]
if !exists && len(t.pendingVerdicts) >= t.maxTracked {
t.prunePairing(now)
if len(t.pendingVerdicts) >= t.maxTracked {
// Dropping a verdict only withholds evidence.
return
}
}
if t.pendingVerdicts == nil {
t.pendingVerdicts = make(map[string][]smtpPendingVerdict)
}
name := normalizeSprayMailbox(account)
at := smtpRejectionTime{at: now, loggedAt: loggedAt}
for len(pending) > 0 && pending[0].at.Before(now.Add(-smtpAuthPairWindow)) {
pending = pending[1:]
}
// Every passdb in a chain answers for the same request, so pending
// answers for one mailbox share what any of them learned about it.
// Answers for different mailboxes say nothing about either, but each
// name they rejected is kept for the spray memory. Pending answers
// always agree with each other, so the oldest speaks for all.
var batch *smtpVerdictBatch
if len(pending) > 0 {
batch = pending[0].batch
if batch == nil && pending[0].account != account {
batch = &smtpVerdictBatch{}
for _, p := range pending {
batch.add(p.name, p.smtpRejectionTime)
}
}
}
if batch != nil {
batch.add(name, at)
account, exist = "", mailboxExistenceUnknown
} else if len(pending) > 0 {
exist = combineMailboxExistence(exist, pending[0].exist)
}
// Preserve ambiguity even when the queue drops an older answer or a
// different request finishes first.
for i := range pending {
pending[i].account, pending[i].exist, pending[i].batch = account, exist, batch
}
if len(pending) >= smtpAuthPairMaxPending {
pending = pending[1:]
}
t.pendingVerdicts[ip] = append(pending, smtpPendingVerdict{smtpRejectionTime: at, account: account, exist: exist, name: name, batch: batch})
t.aliased = t.aliased || sprayAddr(ip) != ip
}
// NoteAuthTrouble marks Dovecot client addresses or Postfix process tags.
// Their failures inside the pairing window are not credential evidence.
// Process tags and canonical IP addresses cannot collide.
func (t *smtpAuthTracker) NoteAuthTrouble(keys ...string) {
t.mu.Lock()
defer t.mu.Unlock()
now := t.now()
for _, key := range keys {
if _, exists := t.authTrouble[key]; !exists && len(t.authTrouble) >= t.maxTracked {
t.prunePairing(now)
if len(t.authTrouble) >= t.maxTracked {
// Losing a mark could turn an outage into evidence, so
// withhold all pairing until the marks would have expired.
t.authTroubleOverflowUntil = now.Add(smtpAuthPairWindow)
continue
}
}
if t.authTrouble == nil {
t.authTrouble = make(map[string]time.Time)
}
t.authTrouble[key] = now
}
}
// PairMTAFailure claims the oldest pending rejection for ip. There is no
// request ID in these lines, and concurrent requests can finish out of order,
// so account attribution requires all pending answers to agree; when they do
// not, names holds the eligible names not consumed yet. Dovecot client errors,
// Postfix process errors and non-credential replies discard ambiguous names.
func (t *smtpAuthTracker) PairMTAFailure(ip, process string, loggedAt time.Time, credential bool) (account string, exist mailboxExistence, names []string, ok bool) {
t.mu.Lock()
defer t.mu.Unlock()
now := t.now()
cutoff := now.Add(-smtpAuthPairWindow)
pending := t.pendingVerdicts[ip]
for len(pending) > 0 {
if pending[0].matches(cutoff, loggedAt) {
break
}
pending = pending[1:]
}
if len(pending) == 0 {
delete(t.pendingVerdicts, ip)
return "", mailboxExistenceUnknown, nil, false
}
account, exist = pending[0].account, pending[0].exist
if len(pending) == 1 {
delete(t.pendingVerdicts, ip)
} else {
t.pendingVerdicts[ip] = pending[1:]
}
if !now.After(t.authTroubleOverflowUntil) {
credential = false
}
for _, key := range []string{ip, process} {
if at, marked := t.authTrouble[key]; marked && !at.Before(cutoff) {
credential = false
}
}
if batch := pending[0].batch; batch != nil {
// The probe gate also withholds spray evidence, including pending
// names whose source times belong to a later buffered reply. Keep
// counting the current reply in the rate trackers as before.
sprayCredential := credential && (t.backendDownFn == nil || !t.backendDownFn())
names = batch.take(pending, cutoff, loggedAt, sprayCredential)
}
if !credential {
return "", mailboxExistenceUnknown, nil, false
}
return account, exist, names, true
}
// smtpAuthLogTime keeps buffered file input from extending the pairing
// window. Bare messages have no source time; their receipt-time bound still
// applies. Traditional syslog omits the year and uses the host's timezone.
func smtpAuthLogTime(line string, now time.Time) time.Time {
if end := strings.IndexAny(line, " \t"); end > 0 {
if at, err := time.Parse(time.RFC3339Nano, line[:end]); err == nil {
return at
}
}
if len(line) < len(time.Stamp) {
return time.Time{}
}
at, err := time.ParseInLocation(time.Stamp, line[:len(time.Stamp)], now.Location())
if err != nil {
return time.Time{}
}
at = time.Date(now.Year(), at.Month(), at.Day(), at.Hour(), at.Minute(), at.Second(), 0, now.Location())
// A December line read in January belongs to the previous year. Allow
// a day's clock skew rather than mistaking it for a year-old message.
if at.After(now.AddDate(0, 0, 1)) {
at = at.AddDate(-1, 0, 0)
}
return at
}
// prunePairing drops expired verdicts and error marks. Caller holds t.mu.
func (t *smtpAuthTracker) prunePairing(now time.Time) {
cutoff := now.Add(-smtpAuthPairWindow)
for ip, pending := range t.pendingVerdicts {
if pending[len(pending)-1].at.Before(cutoff) {
delete(t.pendingVerdicts, ip)
}
}
for ip, at := range t.authTrouble {
if at.Before(cutoff) {
delete(t.authTrouble, ip)
}
}
}
package daemon
import (
"fmt"
"maps"
"slices"
"sort"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
)
// smtpIPEntry tracks failed-auth timestamps and suppression state for one IP.
// slowTimes/slowAccounts hold the long-horizon failure history that catches
// attackers pacing below the fast window; slowAccounts maps each targeted
// mailbox to its most recent failure so distinct-target counting can prune.
// slowUnvouched is the latest slow failure the mailbox map does not hold (no
// mailbox, or the map was full), which no login trust can vouch for.
// Successful logins leave no state here: login trust lives in mailSpray.
type smtpIPEntry struct {
times []time.Time
slowTimes []time.Time
slowAccounts map[string]time.Time
slowUnvouched time.Time
suppressed time.Time
lastSeen time.Time
}
// evictionRank keeps meaningful accumulated evidence behind disposable
// one-shot failure entries. This prevents a fresh one-failure-per-IP flood
// from evicting slow-brute evidence. Login trust has its own pool in
// mailSpray, out of reach of this flood.
func (e *smtpIPEntry) evictionRank(now time.Time, window, slowWindow time.Duration) int {
if slowWindow > 0 && countTimesAtOrAfter(e.slowTimes, now.Add(-slowWindow)) > 1 {
return evictionRankProtectedIP
}
if hasTimeAtOrAfter(e.times, now.Add(-window)) {
return evictionRankActiveIP
}
return evictionRankIdleIP
}
// smtpSubnetEntry tracks unique attacker IPs within a /24.
type smtpSubnetEntry struct {
ips map[string]subnetSighting // ip -> last sighting in window
suppressed time.Time
lastSeen time.Time
}
// smtpAccountEntry tracks unique attacker IPs per mailbox.
type smtpAccountEntry struct {
ips map[string]time.Time
suppressed time.Time
lastSeen time.Time
}
// slowBruteMinAccounts is how many distinct mailboxes a single IP's
// long-horizon failures must target before the slow-brute signal fires. A
// misconfigured client with a stale saved password hammers one mailbox; a
// mailbox walk touches several.
const slowBruteMinAccounts = config.SlowBruteMinThreshold
// Slow-brute state is nested inside a tracked IP, so maxTracked cannot bound
// it. These caps match the largest accepted threshold and keep a high-rate
// source or a stream of unique attacker-controlled mailbox names bounded.
const (
slowBruteMaxTimesPerIP = config.SlowBruteMaxThreshold
slowBruteMaxAccountsPerIP = config.SlowBruteMaxThreshold
)
// slowBruteWalkAccounts fires the slow signal on distinct-mailbox breadth
// alone. A walk probing one or two passwords per mailbox stays under any
// failure-count floor forever (observed live: 34 failures across 26
// mailboxes), but no legitimate client fails against this many distinct
// mailboxes from one address without a single success.
const slowBruteWalkAccounts = 10
// pruneSlowAccounts drops per-mailbox last-failure records older than cutoff.
func pruneSlowAccounts(accounts map[string]time.Time, cutoff time.Time) {
for acct, ts := range accounts {
if ts.Before(cutoff) {
delete(accounts, acct)
}
}
}
// appendSlowFailure retains the newest bounded history. Validation caps the
// configured threshold at the same value, so a reachable threshold is never
// discarded. Reslicing avoids copying the full history on every event.
func appendSlowFailure(times []time.Time, ts time.Time) []time.Time {
times = append(times, ts)
if len(times) > slowBruteMaxTimesPerIP {
times = times[len(times)-slowBruteMaxTimesPerIP:]
}
return times
}
// recordSlowAccount prunes before insertion and refuses new keys after the
// per-IP cap. Three distinct live keys are sufficient for detection, so a
// capped map preserves the signal while preventing unique-name memory growth.
// The bool reports whether this event's account is represented in the map.
func recordSlowAccount(accounts map[string]time.Time, account string, now, cutoff time.Time) (map[string]time.Time, bool) {
pruneSlowAccounts(accounts, cutoff)
if account == "" {
return accounts, false
}
if accounts == nil {
accounts = make(map[string]time.Time)
}
if _, exists := accounts[account]; exists || len(accounts) < slowBruteMaxAccountsPerIP {
accounts[account] = now
return accounts, true
}
return accounts, false
}
// smtpAuthTracker aggregates dovecot auth-failure events into three
// detection signals: per-IP brute force, per-/24 password spray, and
// per-mailbox account spray.
//
// Thread-safe; Record may be called concurrently from multiple log readers.
type smtpAuthTracker struct {
mu sync.Mutex
perIPThreshold int
subnetThreshold int
accountSprayThreshold int
window time.Duration
suppression time.Duration
slowThreshold int
slowWindow time.Duration
maxTracked int
now func() time.Time
ips map[string]*smtpIPEntry
subnets map[string]*smtpSubnetEntry
accounts map[string]*smtpAccountEntry
// Diagnostic counters (guarded by mu): cumulative Record invocations and
// findings emitted. The daemon logs these so a "zero smtp_bruteforce in
// production despite thousands of auth failures" can be pinned to either
// "Record never called" or "called but never crosses threshold".
recordCalls int64
findingsEmitted int64
// backendDownFn, when set, reports whether the active socket probe currently
// sees the mail auth backend down. SMTP-AUTH (exim->dovecot) fails the same
// way during a cpdoveauthd outage, so suppress brute/subnet auto-block then.
backendDownFn func() bool
// spray, when set, is the long-memory login spray detector shared with
// the IMAP/POP tracker. It holds per-mailbox login trust. Set once at
// startup before log readers begin.
spray *mailSpray
// knownCustomers, when set, remembers each successful login's address.
// Set once at startup before log readers begin; read without mu.
knownCustomers *knownCustomers
// Postfix SMTP AUTH pairing state (see smtp_auth_pairing.go), bounded by
// maxTracked entries each and expired after smtpAuthPairWindow.
pendingVerdicts map[string][]smtpPendingVerdict
authTrouble map[string]time.Time
authTroubleOverflowUntil time.Time
// aliased is set once ips or pendingVerdicts holds a key that is not
// its address's canonical spelling. Until then a release looks up the
// canonical key instead of scanning every entry.
aliased bool
}
// SetBackendDownCheck installs the active-probe callback the tracker consults to
// learn whether the mail auth backend is down. When it returns true, brute-force
// and subnet auto-block are suppressed. Set once at startup before log readers
// begin.
func (t *smtpAuthTracker) SetBackendDownCheck(fn func() bool) {
t.mu.Lock()
defer t.mu.Unlock()
t.backendDownFn = fn
}
// SetSpray installs the shared spray detector. Set once at startup before log
// readers begin.
func (t *smtpAuthTracker) SetSpray(m *mailSpray) {
t.mu.Lock()
defer t.mu.Unlock()
t.spray = m
}
// SetKnownCustomers installs the known-customer set. Set once at startup
// before log readers begin.
func (t *smtpAuthTracker) SetKnownCustomers(k *knownCustomers) {
t.knownCustomers = k
}
// newSMTPAuthTracker constructs a tracker. `now` is injected so tests can
// use deterministic clocks; pass `time.Now` in production.
func newSMTPAuthTracker(
perIPThreshold int,
subnetThreshold int,
accountSprayThreshold int,
window time.Duration,
suppression time.Duration,
slowThreshold int,
slowWindow time.Duration,
maxTracked int,
now func() time.Time,
) *smtpAuthTracker {
if now == nil {
now = time.Now
}
return &smtpAuthTracker{
perIPThreshold: perIPThreshold,
subnetThreshold: subnetThreshold,
accountSprayThreshold: accountSprayThreshold,
window: window,
suppression: suppression,
slowThreshold: slowThreshold,
slowWindow: slowWindow,
maxTracked: maxTracked,
now: now,
ips: make(map[string]*smtpIPEntry),
subnets: make(map[string]*smtpSubnetEntry),
accounts: make(map[string]*smtpAccountEntry),
}
}
// Size returns the total number of tracked entities (IPs + subnets + accounts).
func (t *smtpAuthTracker) Size() int {
t.mu.Lock()
defer t.mu.Unlock()
return len(t.ips) + len(t.subnets) + len(t.accounts)
}
// Record processes one dovecot auth-failure observation. Returns zero or more
// findings that callers should append to their finding slice.
//
// ip MUST be non-private, non-loopback, and non-infra — callers enforce this
// before invoking Record.
func (t *smtpAuthTracker) Record(ip, account string) []alert.Finding {
return t.RecordObserved(ip, account, alert.Observation{})
}
// RecordObserved is Record for a failure read from a positioned line: obs
// names that line, so a subnet spray names the line that last counted each
// address.
func (t *smtpAuthTracker) RecordObserved(ip, account string, obs alert.Observation) []alert.Finding {
return t.RecordFailure(ip, account, mailboxExistenceUnknown, obs)
}
// RecordFailure is RecordObserved for a failure whose log said whether the
// mailbox exists.
func (t *smtpAuthTracker) RecordFailure(ip, account string, exist mailboxExistence, obs alert.Observation) []alert.Finding {
return t.recordFailure(ip, account, exist, obs, true, nil)
}
// RecordResolvedFailure is RecordFailure with the mailbox the failure resolved
// to: spray memory counts it against mb, the tracker against the typed account.
func (t *smtpAuthTracker) RecordResolvedFailure(ip, account string, mb sprayMailbox, obs alert.Observation) []alert.Finding {
return t.recordFailure(ip, account, mb.exist, obs, true, []string{mb.name})
}
// RecordAmbiguousFailure is RecordFailure for a failure whose pending
// answers named different mailboxes. It counts with no mailbox, and the
// spray memory gets the names claimed during pairing, existence unknown.
func (t *smtpAuthTracker) RecordAmbiguousFailure(ip string, names []string, obs alert.Observation) []alert.Finding {
return t.recordFailure(ip, "", mailboxExistenceUnknown, obs, true, names)
}
// RecordTemporaryFailure is RecordObserved for a failure the MTA answered
// with a temporary reply: the auth backend, not the password, failed. It
// counts as before but is never spray evidence.
func (t *smtpAuthTracker) RecordTemporaryFailure(ip, account string, obs alert.Observation) []alert.Finding {
return t.recordFailure(ip, account, mailboxExistenceUnknown, obs, false, nil)
}
func (t *smtpAuthTracker) recordFailure(ip, account string, exist mailboxExistence, obs alert.Observation, credential bool, names []string) []alert.Finding {
if ip == "" {
return nil
}
t.mu.Lock()
defer t.mu.Unlock()
t.recordCalls++
now := t.now()
cutoff := now.Add(-t.window)
var findings []alert.Finding
// During a mail-auth-backend outage exim->dovecot SMTP AUTH fails for every
// user regardless of password, so the failure-rate signals would mass-block
// legitimate senders. Suppress them while the probe reports the backend down.
degraded := t.backendDownFn != nil && t.backendDownFn()
// --- Per-IP tracker ---
e, ok := t.ips[ip]
if !ok {
e = &smtpIPEntry{}
t.ips[ip] = e
t.aliased = t.aliased || sprayAddr(ip) != ip
}
e.times = pruneTimes(e.times, cutoff)
e.times = append(e.times, now)
e.lastSeen = now
if t.perIPThreshold > 0 && len(e.times) >= t.perIPThreshold && !now.Before(e.suppressed) && !degraded {
e.suppressed = now.Add(t.suppression)
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "smtp_bruteforce",
Message: fmt.Sprintf("SMTP brute force from %s: %d failed auths in %v",
ip, len(e.times), t.window),
Details: "Real-time detection of dovecot_login auth failures",
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionPeer,
})
}
// --- Long-horizon slow-brute tracker ---
// Catches attackers pacing below the fast window (e.g. one failure every
// few minutes for hours). Requiring several distinct target mailboxes
// separates a mailbox walk from a misconfigured client retrying one stale
// saved password, which fails against a single mailbox no matter how long
// it runs.
if t.slowThreshold > 0 && t.slowWindow > 0 {
if degraded {
// Backend failures say nothing about credentials. Discard the
// long-lived evidence so outage traffic cannot trigger a delayed
// block as soon as the backend recovers.
e.slowTimes = nil
e.slowAccounts = nil
e.slowUnvouched = time.Time{}
} else {
slowCutoff := now.Add(-t.slowWindow)
e.slowTimes = pruneTimes(e.slowTimes, slowCutoff)
e.slowTimes = appendSlowFailure(e.slowTimes, now)
var trackedAccount bool
e.slowAccounts, trackedAccount = recordSlowAccount(e.slowAccounts, normalizeMailAuthAccount(account), now, slowCutoff)
if !trackedAccount {
e.slowUnvouched = now
}
if e.slowUnvouched.Before(slowCutoff) {
e.slowUnvouched = time.Time{}
}
if (len(e.slowTimes) >= t.slowThreshold || len(e.slowAccounts) >= slowBruteWalkAccounts) &&
len(e.slowAccounts) >= slowBruteMinAccounts &&
!now.Before(e.suppressed) &&
!slowTrusted(t.spray, ip, e.slowAccounts, e.slowUnvouched) {
e.suppressed = now.Add(t.suppression)
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "smtp_bruteforce",
Message: fmt.Sprintf("SMTP brute force from %s: %d failed auths across %d mailboxes in %v",
ip, len(e.slowTimes), len(e.slowAccounts), t.slowWindow),
Details: "Long-horizon detection of paced dovecot_login auth failures that stay below the fast per-IP window",
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionPeer,
})
}
}
}
// --- Long-memory login spray (shared with IMAP/POP) ---
if degraded && t.spray != nil {
// The outage may have begun a while before it was detected.
t.spray.ForgetRecentFailures(ip, cutoff)
}
if !degraded && credential && t.spray != nil {
if names == nil {
names = []string{normalizeSprayMailbox(account)}
}
var v *sprayVerdict
for _, name := range names {
// Every name is evidence; one failure still raises one verdict.
if nv := t.spray.Failure(ip, name, exist); v == nil {
v = nv
}
}
if v != nil {
f := sprayFinding(protoSMTP, ip, v, t.spray.Mode(), now)
// One failure raises one block: the evidence is recorded either
// way. A per-IP block above already covers this address, unless
// the spray block carries its own length and takes its place.
perIPBlock := slices.IndexFunc(findings, func(p alert.Finding) bool { return p.Check == "smtp_bruteforce" })
switch {
case f.Check != "smtp_bruteforce" || perIPBlock < 0:
findings = append(findings, f)
case f.BlockTTL > 0:
findings[perIPBlock] = f
}
}
}
// --- Per-/24 subnet tracker (IPv4 only) ---
if prefix := extractPrefix24Daemon(ip); prefix != "" {
s, ok := t.subnets[prefix]
if !ok {
s = &smtpSubnetEntry{ips: make(map[string]subnetSighting)}
t.subnets[prefix] = s
}
pruneSubnetIPs(s, cutoff)
s.ips[ip] = subnetSighting{at: now, obs: obs}
s.lastSeen = now
if t.subnetThreshold > 0 && len(s.ips) >= t.subnetThreshold && !now.Before(s.suppressed) && !degraded {
s.suppressed = now.Add(t.suppression)
cidr := prefix + ".0/24"
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "smtp_subnet_spray",
Message: fmt.Sprintf("SMTP password spray from %s.0/24: %d unique IPs in %v",
prefix, len(s.ips), t.window),
Details: "Real-time detection of dovecot_login auth failures from many IPs in one /24",
Timestamp: now,
CIDRs: []string{cidr},
SprayConstituents: sprayConstituents(s.ips),
})
}
}
// --- Per-account spray tracker ---
// Keyed on the trimmed, ASCII-folded mailbox: "User@X.RO", "user@x.ro"
// and a padded spelling are one target, and a spray across those
// variants must reach the distinct-source threshold as one account.
account = normalizeMailAuthAccount(account)
if account != "" {
a, ok := t.accounts[account]
if !ok {
a = &smtpAccountEntry{ips: make(map[string]time.Time)}
t.accounts[account] = a
}
pruneAccountIPs(a, cutoff)
a.ips[ip] = now
a.lastSeen = now
if t.accountSprayThreshold > 0 && len(a.ips) >= t.accountSprayThreshold && !now.Before(a.suppressed) {
a.suppressed = now.Add(t.suppression)
_, acctDomain := alert.SplitEmail(account)
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "smtp_account_spray",
Message: fmt.Sprintf("SMTP password spray targeting %s: %d unique IPs in %v",
account, len(a.ips), t.window),
Details: "Distributed login attempts across many IPs against one mailbox (visibility only — no auto-block).",
Timestamp: now,
SourceIP: ip,
Domain: acctDomain,
Mailbox: account,
})
}
}
t.enforceMaxTracked(ip)
t.findingsEmitted += int64(len(findings))
return findings
}
// slowTrusted reports whether ip holds aged login trust on every mailbox its
// slow failures name: a known client's stale saved passwords, not a walk. A
// slow failure the mailbox map does not hold (unvouched set) names nothing
// trust could vouch for. Both trackers call it with their lock held.
func slowTrusted(spray *mailSpray, ip string, accounts map[string]time.Time, unvouched time.Time) bool {
if spray == nil || !unvouched.IsZero() {
return false
}
mailboxes := make([]string, 0, len(accounts))
for account := range accounts {
mailboxes = append(mailboxes, normalizeSprayMailbox(account))
}
return spray.TrustedAll(ip, mailboxes)
}
// RecordSuccess notes a successful SMTP login on account from ip. A clean
// login earns trust for that mailbox only; a login after failures on other
// mailboxes earns none and may be reported as a compromise. The tracker keeps
// no state of its own for it. Callers filter infra/private/loopback IPs first.
func (t *smtpAuthTracker) RecordSuccess(ip, account string) (findings []alert.Finding) {
return t.recordSuccessAt(ip, account, t.now(), syslogFed)
}
// recordSuccessAt retains the source time for customer history while the
// brute-force tracker continues to process the event at receipt time. src
// names the log the login came from: the Exim mainlog or Postfix over syslog.
func (t *smtpAuthTracker) recordSuccessAt(ip, account string, at time.Time, src loginSource) (findings []alert.Finding) {
// Before t.mu: the customer set takes its own lock.
t.knownCustomers.Note(ip, knownCustomerSMTP, account, at, src)
if ip == "" {
return nil
}
mailbox := normalizeSprayMailbox(account)
// Registered before the lock so owner resolution runs after unlock.
defer func() { stampMailAccountOwner(findings, mailbox) }()
t.mu.Lock()
defer t.mu.Unlock()
// During an auth outage the failures were kept out of spray memory, so
// a login then must not earn trust or raise a finding either.
if t.spray == nil || (t.backendDownFn != nil && t.backendDownFn()) {
return nil
}
_, v := t.spray.Success(ip, mailbox)
if v == nil {
return nil
}
t.findingsEmitted++
return []alert.Finding{sprayFinding(protoSMTP, ip, v, t.spray.Mode(), t.now())}
}
// Forget drops ip's failure counts and pending rejections after an operator
// release, or every address's when ip is empty. Backend outage marks stay.
func (t *smtpAuthTracker) Forget(ip string) {
ip = sprayAddr(ip)
t.mu.Lock()
defer t.mu.Unlock()
if ip != "" && !t.aliased {
delete(t.ips, ip)
delete(t.pendingVerdicts, ip)
return
}
// A canonical firewall release also covers alternate log spellings.
maps.DeleteFunc(t.ips, func(addr string, _ *smtpIPEntry) bool {
return ip == "" || sprayAddr(addr) == ip
})
// Delayed MTA replies must not restore evidence from before the release.
maps.DeleteFunc(t.pendingVerdicts, func(addr string, _ []smtpPendingVerdict) bool {
return ip == "" || sprayAddr(addr) == ip
})
}
// Stats returns cumulative Record invocations and findings emitted since
// startup. Used by the daemon's periodic diagnostic log.
func (t *smtpAuthTracker) Stats() (calls, emits int64) {
t.mu.Lock()
defer t.mu.Unlock()
return t.recordCalls, t.findingsEmitted
}
// pruneTimes drops timestamps older than cutoff. Reuses the backing array.
func pruneTimes(times []time.Time, cutoff time.Time) []time.Time {
recent := times[:0]
for _, ts := range times {
if !ts.Before(cutoff) {
recent = append(recent, ts)
}
}
return recent
}
// extractPrefix24Daemon returns the first three octets of an IPv4 address as
// "a.b.c", or "" if the input isn't an IPv4 address in dotted-quad form.
func extractPrefix24Daemon(ip string) string {
parts := 0
end := 0
for i := 0; i < len(ip); i++ {
if ip[i] == '.' {
parts++
if parts == 3 {
end = i
break
}
}
}
if parts != 3 {
return ""
}
// Reject IPv6 mapped or containing colons.
for i := 0; i < end; i++ {
if ip[i] == ':' {
return ""
}
}
return ip[:end]
}
// pruneSubnetIPs drops per-/24 IP entries whose last-seen is older than cutoff.
func pruneSubnetIPs(s *smtpSubnetEntry, cutoff time.Time) {
for ip, seen := range s.ips {
if seen.at.Before(cutoff) {
delete(s.ips, ip)
}
}
}
// pruneAccountIPs drops per-account IP entries whose last-seen is older than cutoff.
func pruneAccountIPs(a *smtpAccountEntry, cutoff time.Time) {
for ip, ts := range a.ips {
if ts.Before(cutoff) {
delete(a.ips, ip)
}
}
}
// Purge removes entries with no recent activity (older than window + suppression).
// Called from a background goroutine every minute.
func (t *smtpAuthTracker) Purge() {
t.mu.Lock()
defer t.mu.Unlock()
now := t.now()
activityCutoff := now.Add(-(t.window + t.suppression))
slowCutoff := now.Add(-t.slowWindow)
for k, e := range t.ips {
e.times = pruneTimes(e.times, now.Add(-t.window))
e.slowTimes = pruneTimes(e.slowTimes, slowCutoff)
pruneSlowAccounts(e.slowAccounts, slowCutoff)
if e.slowUnvouched.Before(slowCutoff) {
e.slowUnvouched = time.Time{}
}
if len(e.times) == 0 && len(e.slowTimes) == 0 && !e.lastSeen.After(activityCutoff) {
delete(t.ips, k)
}
}
for k, s := range t.subnets {
pruneSubnetIPs(s, now.Add(-t.window))
if len(s.ips) == 0 && !s.lastSeen.After(activityCutoff) {
delete(t.subnets, k)
}
}
t.prunePairing(now)
for k, a := range t.accounts {
pruneAccountIPs(a, now.Add(-t.window))
if len(a.ips) == 0 && !a.lastSeen.After(activityCutoff) {
delete(t.accounts, k)
}
}
}
// enforceMaxTracked evicts the least-recently-seen entries until the total
// number of tracked entities (IPs + subnets + accounts) is <= maxTracked.
// Caller must hold t.mu.
// keepIP is never evicted; see the mail tracker for why the entry a caller
// just wrote must survive its own bookkeeping.
func (t *smtpAuthTracker) enforceMaxTracked(keepIP string) {
total := len(t.ips) + len(t.subnets) + len(t.accounts)
if total <= t.maxTracked {
return
}
now := t.now()
// Ranked like the mail tracker: attacker-chosen account and subnet keys
// go first, idle sources next, sources with in-window failures or
// slow-brute evidence last, so a flood of unique mailbox names cannot
// evict the evidence that governs a source's auto-block decision.
type victim struct {
kind string // "ip" | "subnet" | "account"
key string
seen time.Time
rank int
}
victims := make([]victim, 0, total)
for k, v := range t.ips {
if k == keepIP {
continue
}
victims = append(victims, victim{"ip", k, v.lastSeen, v.evictionRank(now, t.window, t.slowWindow)})
}
for k, v := range t.subnets {
victims = append(victims, victim{"subnet", k, v.lastSeen, evictionRankSubnet})
}
for k, v := range t.accounts {
victims = append(victims, victim{"account", k, v.lastSeen, evictionRankAccount})
}
sort.Slice(victims, func(i, j int) bool {
if victims[i].rank != victims[j].rank {
return victims[i].rank < victims[j].rank
}
return victims[i].seen.Before(victims[j].seen)
})
// Evict to 95% of cap so subsequent inserts don't re-trigger the sort.
target := t.maxTracked * 95 / 100
for i := 0; i < len(victims); i++ {
if len(t.ips)+len(t.subnets)+len(t.accounts) <= target {
break
}
v := victims[i]
switch v.kind {
case "ip":
delete(t.ips, v.key)
case "subnet":
delete(t.subnets, v.key)
case "account":
delete(t.accounts, v.key)
}
}
}
package daemon
import (
"fmt"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/eximlog"
)
// smtpProbeBlockExpiryString returns the configured block expiry string when
// auto-response will actually block the source IP for an `smtp_probe_abuse`
// finding (auto_response.enabled AND block_ips true AND dry_run false), or ""
// otherwise.
// The returned string is what the operator put in csm.yaml ("24h", "12h"...)
// so the alert text matches the value they configured rather than Go's
// canonical Duration formatting.
func smtpProbeBlockExpiryString() string {
cfg := config.Active()
if cfg == nil {
return ""
}
if !cfg.AutoResponse.Enabled || !cfg.AutoResponse.BlockIPs || cfg.AutoResponseDryRunEnabled() {
return ""
}
if cfg.AutoResponse.BlockExpiry == "" {
return "24h"
}
return cfg.AutoResponse.BlockExpiry
}
// smtpProbeEntry records connect timestamps and suppression for one IP.
type smtpProbeEntry struct {
times []time.Time
suppressed time.Time
lastSeen time.Time
}
// smtpProbeTracker counts raw SMTP connect events per source IP and emits an
// `smtp_probe_abuse` finding when an IP exceeds the threshold inside the
// rolling window.
//
// This is the connection-rate complement to smtpAuthTracker: scanners that
// probe-and-disconnect (no AUTH attempt) never trigger the auth tracker, so
// they need their own signal. The thresholds are deliberately set well above
// any legitimate MUA usage; Thunderbird/iPhone bursts of 10-15 parallel
// sessions per send fall comfortably under, scanner storms with hundreds of
// connect/min are caught.
type smtpProbeTracker struct {
mu sync.Mutex
threshold int
window time.Duration
suppression time.Duration
maxTracked int
now func() time.Time
// expiryStrFn returns the operator-visible block expiry (e.g. "24h") when
// live auto-blocking is enabled, or "" when no auto-block will run. Read
// at finding time so a SIGHUP reload of auto_response.* is reflected in
// the next emitted finding's Details.
expiryStrFn func() string
ips map[string]*smtpProbeEntry
}
func newSMTPProbeTracker(threshold int, window, suppression time.Duration, maxTracked int, now func() time.Time, expiryStrFn func() string) *smtpProbeTracker {
if now == nil {
now = time.Now
}
return &smtpProbeTracker{
threshold: threshold,
window: window,
suppression: suppression,
maxTracked: maxTracked,
now: now,
expiryStrFn: expiryStrFn,
ips: make(map[string]*smtpProbeEntry),
}
}
// Size returns the number of tracked source IPs.
func (t *smtpProbeTracker) Size() int {
t.mu.Lock()
defer t.mu.Unlock()
return len(t.ips)
}
// Record observes one SMTP connect event. ip MUST be non-private,
// non-loopback, and non-infra. Callers enforce this before invoking Record.
// Returns zero or one finding (no per-call multiplication).
func (t *smtpProbeTracker) Record(ip string) []alert.Finding {
if ip == "" {
return nil
}
t.mu.Lock()
defer t.mu.Unlock()
// threshold is read under the lock: it is mutable via SetThresholds on a
// SIGHUP reload, so a lockless fast-path read here would race the swap.
if t.threshold <= 0 {
return nil
}
now := t.now()
cutoff := now.Add(-t.window)
e, ok := t.ips[ip]
if !ok {
e = &smtpProbeEntry{}
t.ips[ip] = e
}
e.times = pruneTimes(e.times, cutoff)
e.times = append(e.times, now)
e.lastSeen = now
var findings []alert.Finding
if len(e.times) >= t.threshold && !now.Before(e.suppressed) {
e.suppressed = now.Add(t.suppression)
// The Details message is computed here, before AutoBlockIPs runs in
// dispatchBatch. We can only report the *intent* (scheduled for
// auto-block) - the actual outcome (blocked / rate-limited / already
// blocked / challenged) is published by the companion `auto_block`
// finding emitted by checks.AutoBlockIPs in the same batch.
details := "Sustained SMTP connect rate above the configured threshold. Likely scanner / dictionary probe;"
if t.expiryStrFn != nil {
if exp := t.expiryStrFn(); exp != "" {
details += fmt.Sprintf(" scheduled for auto-block (%s).", exp)
} else {
details += " consider manual block."
}
} else {
details += " consider manual block."
}
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "smtp_probe_abuse",
Message: fmt.Sprintf("SMTP probe abuse from %s: %d connections in %v",
ip, len(e.times), t.window),
Details: details,
Timestamp: now,
SourceIP: ip,
Attribution: admission.AttributionPeer,
// The connection count grows while one scanner keeps probing.
DedupKey: ip,
})
}
t.enforceMaxTracked()
return findings
}
// Purge removes IPs with no activity since (window + suppression) ago.
// Called periodically to prevent unbounded growth.
func (t *smtpProbeTracker) Purge() {
t.mu.Lock()
defer t.mu.Unlock()
now := t.now()
activityCutoff := now.Add(-(t.window + t.suppression))
for k, e := range t.ips {
e.times = pruneTimes(e.times, now.Add(-t.window))
if len(e.times) == 0 && !e.lastSeen.After(activityCutoff) {
delete(t.ips, k)
}
}
}
// enforceMaxTracked evicts the least-recently-seen IPs to keep memory bounded.
// Caller must hold t.mu.
func (t *smtpProbeTracker) enforceMaxTracked() {
if t.maxTracked <= 0 || len(t.ips) <= t.maxTracked {
return
}
type victim struct {
key string
seen time.Time
}
victims := make([]victim, 0, len(t.ips))
for k, e := range t.ips {
victims = append(victims, victim{k, e.lastSeen})
}
sort.Slice(victims, func(i, j int) bool { return victims[i].seen.Before(victims[j].seen) })
target := t.maxTracked * 95 / 100
for i := 0; i < len(victims) && len(t.ips) > target; i++ {
delete(t.ips, victims[i].key)
}
}
// parseEximSMTPConnectIP extracts the connecting source IP from an exim
// mainlog "SMTP connection from ..." line. Returns "" when the line is not
// a connect event.
//
// Exim formats vary:
//
// SMTP connection from [1.2.3.4]:65417 (TCP/IP connection count = 7)
// SMTP connection from (helo.example.com) [1.2.3.4]:43018 lost D=5s
// SMTP connection from ([helo-as-ip]) [1.2.3.4]:38294 lost D=15s
// SMTP connection from ([192.168.0.94]) [1.2.3.4]:64547 D=5s closed by QUIT
//
// Client attribution is delegated to eximlog so HELO address literals and
// malformed bracketed tokens follow the same rules as every other consumer.
func parseEximSMTPConnectIP(line string) string {
const marker = "SMTP connection from "
markerStart := strings.Index(line, marker)
if markerStart < 0 {
return ""
}
if hStart, ok := eximlog.HFieldStart(line); ok && hStart-len(" H=") < markerStart {
return ""
}
return eximlog.ClientIP(line)
}
package daemon
import (
"fmt"
"net/netip"
"os"
"slices"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/attackdb"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/ipmap"
"github.com/pidginhost/csm/internal/netutil"
)
// spamEvidence reads one scan as a verdict about the sender of the scanned
// message. never excludes whitelisted and the host's own
// addresses; weight is the CSM_IP score the last written map gives the
// sender, in tenths.
func spamEvidence(s antispam.Scan, as config.AntispamConfig, never func(netip.Addr) bool, weight func(netip.Addr) int) (string, ipmap.Verdict) {
if !as.Enabled || !ipmap.Recordable(s.IP, nil) || never(s.IP) {
return "", ipmap.NoVerdict
}
return s.IP.String(), ipmap.Judge(s.OK, s.Score, weight(s.IP), as.RejectScore, as.TagScore)
}
// recordSpamEvidence stores one verdict on the sender's attack DB record.
func (d *Daemon) recordSpamEvidence(s antispam.Scan, at time.Time, cfg *config.Config) {
adb := attackdb.Global()
if adb == nil {
return
}
weight := func(netip.Addr) int { return 0 }
w := d.ipmapWriter.Load()
removed := d.ipMapRemoved()
if !removed && slices.Contains(cfg.EmailProtection.Antispam.Feeds.Enabled, antispam.FeedCSMIPMap) && (w == nil || w.State().Gen == "") {
// rspamd can retain a map across daemon restarts. Until its local
// lookup is rebuilt, treating the unknown contribution as zero
// would let the map reinforce its own spam evidence.
return
}
if w != nil && !removed {
weight = w.Weight
}
never := func(a netip.Addr) bool { return d.spamSenderExcluded(a, cfg) }
ip, v := spamEvidence(s, cfg.EmailProtection.Antispam, never, weight)
if v != ipmap.NoVerdict {
d.recordMailVerdict(adb, ip, v == ipmap.SpamVerdict, at)
}
}
// recordMailVerdict stores one verdict and warns when it made the attack
// database drop spam evidence. The database hands each report out once, so
// every verdict is recorded through here.
func (d *Daemon) recordMailVerdict(adb *attackdb.DB, ip string, spam bool, at time.Time) {
d.warnSpamEvidenceEvicted(adb.RecordMailVerdict(ip, spam, at), at)
}
// warnSpamEvidenceEvicted raises the warning for one report of the attack
// database; nil is no report.
func (d *Daemon) warnSpamEvidenceEvicted(ev *attackdb.Eviction, at time.Time) {
if ev == nil {
return
}
finding := alert.Finding{
Severity: alert.Warning,
Check: "spam_evidence_evicted",
Message: fmt.Sprintf("Spam evidence budget reached: dropped %d records, weakest evidence first; the most crowded network, %s, held %d",
ev.Dropped, ev.Network, ev.NetworkRecords),
Details: "Records with attack findings, a block or a recent mail login attack are never dropped. Records with two or more verdicts are dropped only when such records fill nine tenths of the budget. The named network held the most spam-only records when the drop began. That count alone does not show who sent the flood; a small count means the senders are spread over many networks.",
Timestamp: at,
}
// Logged first: a full alert channel refuses the finding.
fmt.Fprintf(os.Stderr, "[%s] %s: %s\n", ts(), finding.Check, finding.Message)
if !alert.TryEnqueue(d.alertCh, finding) {
atomic.AddInt64(&d.droppedAlerts, 1)
}
}
// spamSenderExcluded reports a sender no spam evidence may name: whitelisted,
// the host's own or an infra_ips address.
func (d *Daemon) spamSenderExcluded(a netip.Addr, cfg *config.Config) bool {
return notRecordable(a) || isInfraIPDaemon(a.String(), d.antispamInfraIPs(cfg))
}
// notRecordable reports a whitelisted or host-owned address.
func notRecordable(a netip.Addr) bool {
if tdb := checks.GetThreatDB(); tdb != nil && tdb.IsWhitelisted(a.String()) {
return true
}
return netutil.IsHostAddress(a.String())
}
package daemon
import (
"net/netip"
"slices"
"sync"
"time"
"github.com/pidginhost/csm/internal/store"
)
const (
spamSourceOccasions = 2
spamSourceOccasionGap = time.Hour
spamSourceWindow = 7 * 24 * time.Hour
spamSourceCleanWindow = 30 * 24 * time.Hour
spamSourceThrottle = 24 * time.Hour
spamSourceMaxIPs = 50_000
// spamSourceRejectReserve is the percent of the bound that IPs with
// recent clean mail cannot take from IPs without it.
spamSourceRejectReserve = 10
)
type spamSourceEntry struct {
store.SpamSource
}
// spamSourceTracker holds per-IP spam scan history for at most max IPs. To
// make room for a new IP it drops one without recent clean mail first, but
// keeps a reserve of the bound for those IPs.
type spamSourceTracker struct {
mu sync.Mutex
max int
entries map[netip.Addr]*spamSourceEntry
// Pending checks retain their entry identity across eviction. They are
// never saved: a restart cannot finish work from the previous process.
checking map[netip.Addr]*spamSourceEntry
}
func newSpamSourceTracker(max int) *spamSourceTracker {
return &spamSourceTracker{
max: max, entries: make(map[netip.Addr]*spamSourceEntry),
checking: make(map[netip.Addr]*spamSourceEntry),
}
}
// observe records one scan. It returns true when ip qualifies, is not
// throttled and has no check in flight, and marks the check in flight.
func (t *spamSourceTracker) observe(ip netip.Addr, reject, clean bool, at time.Time) bool {
if !reject && !clean {
return false
}
ip = ip.Unmap()
t.mu.Lock()
defer t.mu.Unlock()
e := t.entries[ip]
if e == nil {
if len(t.entries) >= t.max {
t.evict(at)
}
e = &spamSourceEntry{}
t.entries[ip] = e
}
if at.After(e.Seen) {
e.Seen = at
}
if clean && at.After(e.LastClean) {
e.LastClean = at
}
if !reject {
return false
}
// One message split into several deliveries, or a re-read log line, lands
// inside the occasion it belongs to and is counted once.
if n := len(e.Occasions); n == 0 || !at.Before(e.Occasions[n-1].Add(spamSourceOccasionGap)) {
e.Occasions = append(e.Occasions, at)
}
cut := at.Add(-spamSourceWindow)
e.Occasions = slices.DeleteFunc(e.Occasions, func(o time.Time) bool { return o.Before(cut) })
if n := len(e.Occasions); n > spamSourceOccasions {
e.Occasions = e.Occasions[n-spamSourceOccasions:]
}
if t.checking[ip] != nil || !e.qualifies(at) {
return false
}
t.checking[ip] = e
return true
}
// forget drops ip's history. A check in flight for it then ends unreported,
// because finish finds no entry.
func (t *spamSourceTracker) forget(ip netip.Addr) {
t.mu.Lock()
defer t.mu.Unlock()
delete(t.entries, ip.Unmap())
}
// finish ends the check in flight. If ip still qualifies, emit runs under
// the lock; when it returns true the throttle starts at at.
func (t *spamSourceTracker) finish(ip netip.Addr, at time.Time, emit func() bool) bool {
t.mu.Lock()
defer t.mu.Unlock()
ip = ip.Unmap()
e := t.checking[ip]
delete(t.checking, ip)
// A completion belongs to the entry it checked, never a replacement
// inserted for the same address after eviction.
if e == nil || t.entries[ip] != e {
return false
}
// Clean mail that arrived during the check cancels the finding here.
if !e.qualifies(at) || !emit() {
return false
}
e.LastFinding = at
return true
}
// qualifies treats zero times as long past, so an IP with no clean mail or no
// finding yet passes those two tests.
func (e *spamSourceEntry) qualifies(at time.Time) bool {
from := at.Add(-spamSourceWindow)
n := 0
for _, o := range e.Occasions {
if !o.Before(from) && !o.After(at) {
n++
}
}
return n >= spamSourceOccasions &&
e.LastClean.Before(at.Add(-spamSourceCleanWindow)) &&
!e.LastFinding.After(at.Add(-spamSourceThrottle))
}
// evict makes room for one IP. Recent clean mail is what keeps a mostly good
// sender from being reported, so the least recently seen IP without clean mail
// in the clean window before now goes first. Clean IPs still may not fill the
// reserve: past their share the least recently seen clean IP goes, as it does
// when no other IP is left.
//
// One pass under the lock counts the clean IPs exactly and tracks the least
// recently seen IP of each class. It runs only when a new IP arrives with the
// tracker full, so it costs a scan of the bound per such insert and nothing on
// the per-message path otherwise. A cached count would drift as clean mail ages
// out. Move to one LRU list per class if profiles show this scan; entries would
// then change lists, and the count would change, as their clean mail ages out.
func (t *spamSourceTracker) evict(now time.Time) {
cut := now.Add(-spamSourceCleanWindow)
var oldestClean, oldestOther netip.Addr
var cleanSeen, otherSeen time.Time
clean, other := 0, 0
for ip, e := range t.entries {
if !e.LastClean.Before(cut) {
if clean == 0 || e.Seen.Before(cleanSeen) {
oldestClean, cleanSeen = ip, e.Seen
}
clean++
} else {
if other == 0 || e.Seen.Before(otherSeen) {
oldestOther, otherSeen = ip, e.Seen
}
other++
}
}
// Over the share means clean > max*(100-reserve)/100, kept in integers
// as clean*100 > max*(100-reserve) so it is exact for every bound.
if other == 0 || clean*100 > t.max*(100-spamSourceRejectReserve) {
delete(t.entries, oldestClean)
return
}
delete(t.entries, oldestOther)
}
func (t *spamSourceTracker) snapshot() map[string]store.SpamSource {
t.mu.Lock()
defer t.mu.Unlock()
out := make(map[string]store.SpamSource, len(t.entries))
for ip, e := range t.entries {
src := e.SpamSource
src.Occasions = slices.Clone(src.Occasions)
out[ip.String()] = src
}
return out
}
// load replaces the tracker state with m. An IP unseen for the clean window
// carries nothing that can still block or count toward a finding, so it is
// dropped along with keys that are not IPs.
func (t *spamSourceTracker) load(m map[string]store.SpamSource, now time.Time) {
cut := now.Add(-spamSourceCleanWindow)
entries := make(map[netip.Addr]*spamSourceEntry, len(m))
for key, src := range m {
ip, err := netip.ParseAddr(key)
if err != nil || src.Seen.Before(cut) {
continue
}
src.Occasions = slices.Clone(src.Occasions)
entries[ip.Unmap()] = &spamSourceEntry{SpamSource: src}
}
t.mu.Lock()
defer t.mu.Unlock()
t.entries = entries
clear(t.checking)
t.trimLoaded(now)
}
// trimLoaded applies the same eviction policy to oversized saved state.
// Sort once rather than rescanning the whole map for each excess entry.
func (t *spamSourceTracker) trimLoaded(now time.Time) {
if len(t.entries) <= t.max {
return
}
cut := now.Add(-spamSourceCleanWindow)
var clean, other []netip.Addr
for ip, e := range t.entries {
if !e.LastClean.Before(cut) {
clean = append(clean, ip)
} else {
other = append(other, ip)
}
}
oldest := func(a, b netip.Addr) int {
if order := t.entries[a].Seen.Compare(t.entries[b].Seen); order != 0 {
return order
}
return a.Compare(b)
}
slices.SortFunc(clean, oldest)
slices.SortFunc(other, oldest)
for len(t.entries) > t.max {
if len(other) == 0 || len(clean)*100 > t.max*(100-spamSourceRejectReserve) {
delete(t.entries, clean[0])
clean = clean[1:]
} else {
delete(t.entries, other[0])
other = other[1:]
}
}
}
package daemon
import (
"context"
"log"
"net"
"net/netip"
"path/filepath"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/ipmap"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/reporting"
"github.com/pidginhost/csm/internal/sharedsenders"
"github.com/pidginhost/csm/internal/store"
)
const (
sharedSendersCacheFile = "shared_senders.json"
// spamSourceCandidates bounds the senders waiting for their DNS check.
spamSourceCandidates = 64
spamSourceCheckBudget = 5 * time.Second
spamSourceRefreshBudget = time.Minute
spamSourceSaveEvery = 10 * time.Minute
spamSourceRetryAfter = 10 * time.Minute
)
// spamSourceObservation classifies one scan for the detector. It reads the
// marker's own decision and raw score: mail delivered with filtering off
// never counts as a reject, but its clean scans still clear the sender.
func spamSourceObservation(s antispam.Scan, as config.AntispamConfig) (reject, clean bool) {
if !s.OK {
return false, false
}
return s.Decision == "reject", s.Score < as.TagScore*10
}
type spamCandidate struct {
IP netip.Addr
At time.Time
}
// spamSourceRunner checks each sender the tracker qualifies against the
// shared mail services, off the log path, and raises mail_spam_source for the
// rest.
type spamSourceRunner struct {
tracker *spamSourceTracker
guard *sharedsenders.Guard
resolver sharedsenders.Resolver
candidates chan spamCandidate
db *store.DB
cancel context.CancelFunc
wg sync.WaitGroup
// refreshed is set once a refresh of the guard has ended, well or not.
refreshed atomic.Bool
}
// newSpamSources prepares the detector when the spam class is enabled: it
// loads the saved tracker and the shared-sender cache. It returns nil when the
// detector stays off.
func (d *Daemon) newSpamSources(enabled map[reporting.Class]bool) *spamSourceRunner {
if !enabled[reporting.ClassSpam] {
return nil
}
if !d.cfg.EmailProtection.Antispam.Enabled {
log.Printf("abuse-reporting: the spam class needs email_protection.antispam.enabled; spam source detection stays off")
return nil
}
db := store.Global()
if db == nil {
log.Printf("abuse-reporting: no state database; spam source detection stays off")
return nil
}
now := time.Now()
saved, err := db.LoadSpamSources()
if err != nil {
// An empty tracker would forget recent clean mail and throttles.
log.Printf("abuse-reporting: cannot load spam source state: %v; spam source detection stays off", err)
return nil
}
guard := sharedsenders.New(filepath.Join(d.cfg.StatePath, sharedSendersCacheFile))
if err := guard.Load(now); err != nil {
log.Printf("abuse-reporting: %v", err)
}
if !guard.Ready(now) {
log.Printf("abuse-reporting: shared mail services not loaded; spam reports wait for a refresh")
}
resolver := d.spamSourceDNS
if resolver == nil {
resolver = net.DefaultResolver
}
r := &spamSourceRunner{
tracker: newSpamSourceTracker(spamSourceMaxIPs),
guard: guard,
resolver: resolver,
candidates: make(chan spamCandidate, spamSourceCandidates),
db: db,
}
r.tracker.load(saved, now)
return r
}
// startSpamSources runs r's loop under d.wg and starts feeding it log lines.
func (d *Daemon) startSpamSources(r *spamSourceRunner) {
ctx, cancel := context.WithCancel(context.Background())
r.cancel = cancel
// Run waits for d.wg before stopAbuseReporting, so the daemon stop must
// also cancel a DNS check in flight.
go func() {
select {
case <-d.stopCh:
cancel()
case <-ctx.Done():
}
}()
d.wg.Add(1)
r.wg.Add(1)
obs.Go("spam-sources", func() {
defer d.wg.Done()
defer r.wg.Done()
r.run(ctx, d.alertCh)
})
d.spamSources.Store(r)
}
// covers is the report gate's shared-sender check. It asks no DNS, so it is
// true for a provider prefix and for every address while the guard is not
// ready.
func (r *spamSourceRunner) covers(ip net.IP) bool {
a, ok := netip.AddrFromSlice(ip)
return !ok || r.guard.Covers(a.Unmap(), time.Now())
}
func (r *spamSourceRunner) run(ctx context.Context, alertCh chan<- alert.Finding) {
refresh := time.NewTimer(mailRangesStartupDelay)
defer refresh.Stop()
save := time.NewTicker(spamSourceSaveEvery)
defer save.Stop()
for ctx.Err() == nil {
select {
case <-ctx.Done():
case c := <-r.candidates:
r.check(ctx, alertCh, c)
case <-save.C:
r.save()
case <-refresh.C:
refresh.Reset(r.refresh(ctx))
}
}
}
// check ends the tracker's check for c. A shared or unresolved sender, or a
// full alert queue, raises nothing and starts no throttle, so the sender's
// next reject is checked again.
func (r *spamSourceRunner) check(ctx context.Context, alertCh chan<- alert.Finding, c spamCandidate) {
ctx, cancel := context.WithTimeout(ctx, spamSourceCheckBudget)
defer cancel()
shared := r.guard.Shared(ctx, r.resolver, c.IP, time.Now())
r.tracker.finish(c.IP, c.At, func() bool {
return !shared && alert.TryEnqueue(alertCh, spamSourceFinding(c))
})
}
// refresh resolves the shared mail services again and returns when to run
// next.
func (r *spamSourceRunner) refresh(ctx context.Context) time.Duration {
ctx, cancel := context.WithTimeout(ctx, spamSourceRefreshBudget)
defer cancel()
// Set after Refresh has published, so a reader that sees it also sees
// what this refresh made ready.
defer r.refreshed.Store(true)
if err := r.guard.Refresh(ctx, r.resolver, time.Now()); err != nil {
log.Printf("abuse-reporting: shared mail services refresh: %v", err)
return spamSourceRetryAfter
}
return mailRangesRefreshInterval
}
// sharedSendersHealth tells the reputation check whether spam source reports
// wait on a shared mail service list that should be current by now. Before
// the first refresh ends there is no answer: it runs a few minutes after
// start, and a fresh install has no list until then.
func (d *Daemon) sharedSendersHealth(now time.Time) checks.SharedSendersHealth {
r := d.spamSources.Load()
if r == nil {
return checks.SharedSendersOK
}
// Read before Ready: a refresh ending between the two is then seen as
// ready, never as stale.
refreshed := r.refreshed.Load()
switch {
case r.guard.Ready(now):
return checks.SharedSendersOK
case !refreshed:
return checks.SharedSendersUnknown
default:
return checks.SharedSendersStale
}
}
func (r *spamSourceRunner) save() {
if err := r.db.SaveSpamSources(r.tracker.snapshot()); err != nil {
log.Printf("abuse-reporting: saving spam source state: %v", err)
}
}
// stop ends the loop and saves the tracker. A candidate still queued needs no
// finish: in-flight marks are never saved.
func (r *spamSourceRunner) stop() {
r.cancel()
r.wg.Wait()
r.save()
}
// recordSpamSource feeds one scan to the tracker and hands a sender that
// qualifies to the DNS check. It never blocks the log path: with the queue
// full the check ends unreported, and the sender's next reject starts
// another.
func (d *Daemon) recordSpamSource(s antispam.Scan, at time.Time, cfg *config.Config) {
r := d.spamSources.Load()
if r == nil || !cfg.EmailProtection.Antispam.Enabled || !ipmap.Recordable(s.IP, nil) {
return
}
reject, clean := spamSourceObservation(s, cfg.EmailProtection.Antispam)
if (!reject && !clean) || d.spamSenderExcluded(s.IP, cfg) {
return
}
// A provider sender is never reported, so it holds no tracker slot. History
// kept from before the guard knew its prefix goes too: it would count
// against the sender if a later refresh drops the prefix. While the guard
// is not ready no sender is a provider, so every one is tracked.
if r.guard.ProviderPrefix(s.IP, time.Now()) {
r.tracker.forget(s.IP)
return
}
if !r.tracker.observe(s.IP, reject, clean, at) {
return
}
select {
case r.candidates <- spamCandidate{IP: s.IP, At: at}:
default:
r.tracker.finish(s.IP, at, func() bool { return false })
}
}
func spamSourceFinding(c spamCandidate) alert.Finding {
ip := c.IP.String()
return alert.Finding{
Severity: alert.High,
Check: "mail_spam_source",
Message: "Mail server repeatedly rejected as spam with no recent clean mail: " + ip,
SourceIP: ip,
Timestamp: c.At,
}
}
package daemon
import (
"os"
"path/filepath"
"sort"
)
// eximSplitSpoolDirName reports whether name is one of the hash
// subdirectories Exim creates under input/ when split_spool_directory is
// on: a single character from [0-9A-Za-z], taken from the message ID.
func eximSplitSpoolDirName(name string) bool {
if len(name) != 1 {
return false
}
c := name[0]
return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
}
// spoolMarkTargets returns the directories a spool watcher has to mark to
// see every message file under root: the root itself (flat spool) and every
// split-spool hash subdirectory present right now, sorted. A missing root
// yields nil. Exim creates hash directories lazily, so callers re-run this
// periodically and mark whatever is new.
func spoolMarkTargets(root string) []string {
entries, err := os.ReadDir(root)
if err != nil {
return nil
}
targets := []string{root}
for _, e := range entries {
if e.IsDir() && eximSplitSpoolDirName(e.Name()) {
targets = append(targets, filepath.Join(root, e.Name()))
}
}
sort.Strings(targets[1:])
return targets
}
//go:build linux
package daemon
import (
"errors"
"fmt"
"os"
"path/filepath"
"runtime/debug"
"strings"
"sync"
"sync/atomic"
"time"
"unsafe"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/crashsample"
"github.com/pidginhost/csm/internal/emailav"
"github.com/pidginhost/csm/internal/metrics"
emime "github.com/pidginhost/csm/internal/mime"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/workerproc"
)
// spoolQueueOverflowTotal counts FAN_Q_OVERFLOW records on the spool watcher.
// Registered once per process; hand-built test watchers that skip
// registerSpoolMetrics leave it nil, so callers must nil-check.
var (
spoolQueueOverflowTotal *metrics.Counter
spoolHoldExpiredTotal *metrics.CounterVec
spoolSourceWaitsTotal *metrics.CounterVec
spoolMetricsOnce sync.Once
)
var spoolFanotifyInit = unix.FanotifyInit
func registerSpoolMetrics() {
spoolMetricsOnce.Do(func() {
spoolQueueOverflowTotal = metrics.NewCounter(
"csm_spool_fanotify_queue_overflow_total",
"FAN_Q_OVERFLOW events on the Exim spool watcher: the kernel queue filled and dropped spool open events. In permission mode the kernel allows the overflowing opens, so mail may be delivered without an AV scan. Any sustained growth needs investigation.",
)
metrics.MustRegister("csm_spool_fanotify_queue_overflow_total", spoolQueueOverflowTotal)
spoolHoldExpiredTotal = metrics.NewCounterVec(
"csm_spool_hold_expired_total",
"Mail opens answered under email_av.fail_mode because the hold budget ran out, by where the message was: queued (no scan slot yet; counts toward the hold bypass), waiting (behind another message from the same sender) or scanning.",
[]string{"stage"},
)
metrics.MustRegister("csm_spool_hold_expired_total", spoolHoldExpiredTotal)
spoolSourceWaitsTotal = metrics.NewCounterVec(
"csm_spool_source_waits_total",
"Mail opens whose sender was already scanning on its share of the slots: waited (parked until one of the sender's scans finished) or overflow (too many already waiting; scanned at once without waiting its turn).",
[]string{"outcome"},
)
metrics.MustRegister("csm_spool_source_waits_total", spoolSourceWaitsTotal)
})
}
// fanotify constants for permission events (not in Go stdlib).
const (
FAN_CLASS_CONTENT = 0x00000004
FAN_OPEN_PERM = 0x00010000
FAN_ALLOW = 0x01
FAN_DENY = 0x02
FAN_EVENT_ON_CHILD = 0x08000000
)
// fanotifyResponse is the struct written back to the fanotify fd
// to allow or deny a permission event.
type fanotifyResponse struct {
Fd int32
Response uint32
}
const responseSize = int(unsafe.Sizeof(fanotifyResponse{}))
// SpoolWatcher monitors Exim spool directories for new messages using a
// dedicated fanotify instance with permission events (FAN_OPEN_PERM).
// It is completely separate from the FileMonitor.
type SpoolWatcher struct {
fd int
cfg *config.Config
alertCh chan<- alert.Finding
orchestrator *emailav.Orchestrator
quarantine *emailav.Quarantine
permissionMode bool // true if using FAN_OPEN_PERM, false if fallback to FAN_CLOSE_WRITE
// eventMask is the fanotify mask every spool directory is marked with.
// spoolRoots are the Exim input directories found at start; marked
// records each pathname seen so periodic rescans can report newly found
// split-spool hash subdirectories without growing beyond the fixed Exim
// hash alphabet. Every rescan still refreshes the kernel mark because an
// unlink drops the inode-bound mark while a recreated path stays recorded.
eventMask uint64
spoolRoots []string
markedMu sync.Mutex
marked map[string]struct{}
// extractor unpacks attachments in parse worker units; nil means none
// could be set up, and every message is an incomplete scan.
extractor mailExtractor
// onQuarantined is called with the time and Exim message ID of each
// message the watcher quarantined. Nil for hand-built test watchers.
onQuarantined func(time.Time, string)
// selfPID is os.Getpid(). fanotify permission events generated by CSM's
// own reads of the spool -D file (during MIME parsing) must be allowed
// without re-scanning, or the scan re-enters fanotify and deadlocks every
// worker. See dispatchEvent.
selfPID int
scanCh chan spoolEvent
pipeFds [2]int
stopOnce sync.Once
drainOnce sync.Once
stopCh chan struct{}
wg sync.WaitGroup
// daemonStop distinguishes final shutdown from a watcher restart, which
// must still scan queued mail before the replacement reuses its slots.
daemonStop <-chan struct{}
pipeClosed int32 // atomic
fdClosed int32 // atomic - guards sw.fd against double-close
runActive int32 // atomic - Run owns scanCh shutdown while set
degradedMu sync.Mutex
lastDegradedAt time.Time
encryptedMu sync.Mutex
lastEncryptedAt time.Time
panicMu sync.Mutex
lastPanicAt time.Time
// findings rate-limits parse worker findings, shared with the daemon;
// ownFindings stands in for hand-built test watchers.
findings *findingLimiter
ownFindings findingLimiter
// queueOverflows counts FAN_Q_OVERFLOW records. In permission mode a kernel
// queue overflow means opens were let through without a scan verdict, so
// mail may have been delivered unscanned. overflowMu rate-limits the
// operator finding so a storm does not flood the alert channel.
queueOverflows int64 // atomic
overflowMu sync.Mutex
lastOverflowAt time.Time
// holds tracks hold-budget expiries and decides when to stop holding mail.
holds holdWatchdog
// sources lets one sender scan on at most half the slots.
sources sourceGate
// mailStrikes counts worker kills per message ID; sourceKills counts
// them per source. Nil on hand-built test watchers.
mailStrikes *workerproc.Strikes
sourceKills *workerproc.Strikes
// crashSamples keeps messages that kill a worker; nil when
// email_av.keep_crash_samples is off. Set before Run.
crashSamples *crashsample.Store
queueHealthOnce sync.Once
scannerHealth *queuehealth.Tracker
kernelQueueHealth *queuehealth.Tracker
kernelQueue *notificationQueue
}
type spoolEvent struct {
queueTicket queuehealth.Ticket
path string
fd int // fanotify event fd (for permission response)
pid int32
needResp bool // true if permission event requiring response
// slot is the parse worker slot of the scan goroutine handling this
// event; set by scanWorker.
slot int
// guard bounds how long this open stays suspended. Nil for events built
// outside dispatchEvent, which answer directly from needResp.
guard *holdGuard
// source is the sender key from the message's -H, "" when unknown; set
// by the scan goroutine.
source string
}
// finish hands the kernel a verdict for this event, once, and closes the
// event fd. The guard may already have answered when the scan outran the
// hold budget; then this verdict only affects quarantine, not delivery.
func (evt spoolEvent) finish(sw *SpoolWatcher, response uint32) {
switch {
case evt.guard != nil:
evt.guard.finish(response)
return
case evt.needResp:
spoolWriteResponse(sw, int32(evt.fd), response) // #nosec G115 -- POSIX fd fits in int32.
}
_ = unix.Close(evt.fd)
}
// NewSpoolWatcher creates a dedicated fanotify instance for Exim spool scanning.
// Attempts FAN_CLASS_CONTENT with FAN_OPEN_PERM first; falls back to
// FAN_CLASS_NOTIF with FAN_CLOSE_WRITE if permission events are unavailable.
func NewSpoolWatcher(cfg *config.Config, alertCh chan<- alert.Finding, orch *emailav.Orchestrator, quar *emailav.Quarantine, onQuarantined func(time.Time, string), extractor mailExtractor, findings *findingLimiter) (*SpoolWatcher, error) {
if cfg == nil {
return nil, errors.New("spool watcher: nil config")
}
removeLegacyEmailAVTempDir(cfg.StatePath)
registerSpoolMetrics()
sw := &SpoolWatcher{
cfg: cfg,
alertCh: alertCh,
orchestrator: orch,
quarantine: quar,
extractor: extractor,
findings: findings,
onQuarantined: onQuarantined,
selfPID: os.Getpid(),
scanCh: make(chan spoolEvent, 256),
stopCh: make(chan struct{}),
}
sw.initStrikes()
// Try permission-capable class first
fd, err := spoolFanotifyInit(FAN_CLASS_CONTENT|FAN_CLOEXEC|FAN_NONBLOCK, unix.O_RDONLY|unix.O_CLOEXEC)
if err == nil {
sw.fd = fd
sw.permissionMode = true
fmt.Fprintf(os.Stderr, "[%s] spool watcher: permission events enabled (FAN_OPEN_PERM)\n", ts())
} else {
// Fallback to notification-only
fd, err = spoolFanotifyInit(FAN_CLASS_NOTIF|FAN_CLOEXEC|FAN_NONBLOCK, unix.O_RDONLY|unix.O_CLOEXEC)
if err != nil {
return nil, fmt.Errorf("fanotify_init: %w (neither permission nor notification mode available)", err)
}
sw.fd = fd
sw.permissionMode = false
fmt.Fprintf(os.Stderr, "[%s] spool watcher: WARNING - permission events unavailable, using notification mode (small delivery race window possible)\n", ts())
if sw.cfg.EmailAV.FailMode == "tempfail" {
fmt.Fprintf(os.Stderr, "[%s] spool watcher: WARNING - fail_mode=tempfail requested but cannot be honoured without FAN_OPEN_PERM; operating fail-open\n", ts())
}
}
// Mark spool directories
spoolDirs := []string{"/var/spool/exim/input", "/var/spool/exim4/input"}
var eventMask uint64
if sw.permissionMode {
eventMask = FAN_OPEN_PERM | FAN_EVENT_ON_CHILD
} else {
eventMask = FAN_CLOSE_WRITE | FAN_EVENT_ON_CHILD
}
sw.eventMask = eventMask
sw.marked = make(map[string]struct{})
marked := 0
for _, dir := range spoolDirs {
if _, err := os.Stat(dir); err != nil {
continue
}
sw.spoolRoots = append(sw.spoolRoots, dir)
marked += sw.markSpoolTargets(dir)
}
if marked == 0 {
_ = unix.Close(sw.fd)
return nil, fmt.Errorf("no Exim spool directories found to watch")
}
// Create pipe for stop signaling
if err := unix.Pipe2(sw.pipeFds[:], unix.O_NONBLOCK|unix.O_CLOEXEC); err != nil {
_ = unix.Close(sw.fd)
return nil, fmt.Errorf("creating pipe: %w", err)
}
return sw, nil
}
// spoolRescanInterval bounds how long a split-spool hash directory Exim
// created after start stays unwatched.
var spoolRescanInterval = time.Minute
// markSpoolTargets marks root and every split-spool hash subdirectory under
// it, and returns how many pathnames were marked for the first time.
// FAN_EVENT_ON_CHILD on a directory mark covers only its
// direct children, so on a split spool (the cPanel default) the -D files,
// which live one level down, are only seen through the subdirectory marks.
// Uses FAN_MARK_ADD (not FAN_MARK_MOUNT) to scope to the directory.
func (sw *SpoolWatcher) markSpoolTargets(root string) int {
added := 0
for _, dir := range spoolMarkTargets(root) {
sw.markedMu.Lock()
_, done := sw.marked[dir]
sw.markedMu.Unlock()
if err := unix.FanotifyMark(sw.fd, FAN_MARK_ADD, sw.eventMask, -1, dir); err != nil {
fmt.Fprintf(os.Stderr, "[%s] spool watcher: cannot watch %s: %v\n", ts(), dir, err)
continue
}
if !done {
sw.markedMu.Lock()
sw.marked[dir] = struct{}{}
sw.markedMu.Unlock()
added++
fmt.Fprintf(os.Stderr, "[%s] spool watcher: watching %s\n", ts(), dir)
}
}
return added
}
// rescanSpoolDirs marks split-spool hash directories that appeared since
// the last pass.
func (sw *SpoolWatcher) rescanSpoolDirs() {
for _, root := range sw.spoolRoots {
sw.markSpoolTargets(root)
}
}
// Run starts the event loop and scanner workers. Blocks until Stop() is called.
func (sw *SpoolWatcher) Run() {
// Event loop. Set up epoll before starting any workers so a setup
// failure has nothing to unwind beyond drainAndClose: workers started
// first would park on scanCh forever and hang daemon shutdown.
epfd, err := unix.EpollCreate1(unix.EPOLL_CLOEXEC)
if err != nil {
fmt.Fprintf(os.Stderr, "[%s] spool watcher: epoll_create: %v\n", ts(), err)
sw.drainAndClose()
return
}
defer func() { _ = unix.Close(epfd) }()
// #nosec G115 -- POSIX fd fits in int32 (rlimit ~1024). Same for all fd→int32 in this file.
if err := unix.EpollCtl(epfd, unix.EPOLL_CTL_ADD, sw.fd, &unix.EpollEvent{Events: unix.EPOLLIN, Fd: int32(sw.fd)}); err != nil {
fmt.Fprintf(os.Stderr, "[%s] spool watcher: epoll_ctl(fanotify fd): %v\n", ts(), err)
sw.drainAndClose()
return
}
// #nosec G115 -- POSIX fd fits in int32.
if err := unix.EpollCtl(epfd, unix.EPOLL_CTL_ADD, sw.pipeFds[0], &unix.EpollEvent{Events: unix.EPOLLIN, Fd: int32(sw.pipeFds[0])}); err != nil {
fmt.Fprintf(os.Stderr, "[%s] spool watcher: epoll_ctl(pipe fd): %v\n", ts(), err)
sw.drainAndClose()
return
}
// Start scanner workers
atomic.StoreInt32(&sw.runActive, 1)
defer atomic.StoreInt32(&sw.runActive, 0)
sw.startScanWorkers()
events := make([]unix.EpollEvent, 16)
buf := make([]byte, 4096)
lastRescan := time.Now()
for {
select {
case <-sw.stopCh:
sw.drainAndClose()
return
default:
}
if time.Since(lastRescan) >= spoolRescanInterval {
sw.rescanSpoolDirs()
lastRescan = time.Now()
}
n, err := unix.EpollWait(epfd, events, 500)
if err != nil {
if err == unix.EINTR {
continue
}
select {
case <-sw.stopCh:
sw.drainAndClose()
return
default:
continue
}
}
for i := 0; i < n; i++ {
// #nosec G115 -- POSIX fd fits in int32.
if events[i].Fd == int32(sw.pipeFds[0]) {
sw.drainAndClose()
return
}
// #nosec G115 -- POSIX fd fits in int32.
if events[i].Fd == int32(sw.fd) {
sw.readEvents(buf)
}
}
}
}
func (sw *SpoolWatcher) readEvents(buf []byte) {
sw.initQueueHealth()
for {
n, err := sw.kernelQueue.read(buf, sw.parseEvents)
if err != nil || n < metadataSize {
return
}
}
}
// parseEvents walks a raw fanotify buffer and dispatches each record. A
// FAN_Q_OVERFLOW record (Fd == FAN_NOFD, FAN_Q_OVERFLOW bit set) is handled
// explicitly so a kernel queue overflow is not silently swallowed.
func (sw *SpoolWatcher) parseEvents(buf []byte) {
offset := 0
for offset+metadataSize <= len(buf) {
// #nosec G103 -- fanotify delivers a packed binary stream;
// reinterpretation is required and bounded by metadataSize above.
meta := (*fanotifyEventMetadata)(unsafe.Pointer(&buf[offset]))
if meta.EventLen < uint32(metadataSize) || int(meta.EventLen) > len(buf)-offset {
break
}
if meta.Mask&unix.FAN_Q_OVERFLOW != 0 {
sw.handleQueueOverflow()
} else if meta.Fd >= 0 {
sw.dispatchEvent(meta.Fd, meta.Pid)
}
offset += int(meta.EventLen)
}
}
// handleQueueOverflow reacts to a FAN_Q_OVERFLOW record on the spool watcher.
// In permission mode the kernel resolves opens it cannot queue by allowing
// them, so an overflow means some inbound messages were delivered without an
// AV scan. Count it and emit a rate-limited Warning that says so.
func (sw *SpoolWatcher) handleQueueOverflow() {
sw.initQueueHealth()
sw.kernelQueueHealth.Lose(time.Now(), 1)
atomic.AddInt64(&sw.queueOverflows, 1)
if spoolQueueOverflowTotal != nil {
spoolQueueOverflowTotal.Inc()
}
sw.overflowMu.Lock()
if !sw.lastOverflowAt.IsZero() && time.Since(sw.lastOverflowAt) < time.Minute {
sw.overflowMu.Unlock()
return
}
sw.lastOverflowAt = time.Now()
sw.overflowMu.Unlock()
sw.emitFinding("email_av_queue_overflow", alert.Warning,
"Spool fanotify queue overflowed: the kernel dropped spool open events. In permission mode overflowing opens are allowed through, so one or more messages may have been delivered unscanned during the storm. Investigate the mail spike and consider a manual rescan of recent messages.")
}
// dispatchEvent decides what to do with a single fanotify event: allow our own
// reads immediately, allow non -D opens, or hand the -D open to a scan worker.
// It owns the event fd: every path either enqueues it (worker closes it) or
// responds and closes it here.
func (sw *SpoolWatcher) dispatchEvent(fd int32, pid int32) {
// MAIL-02: never scan CSM's own reads. When a scan worker opens the -D
// file to parse it, that open re-enters this fanotify instance as a
// FAN_OPEN_PERM event for our own pid. With the scan blocked waiting on a
// verdict, self-recursion deadlocks the workers and stalls Exim delivery
// server-wide until a restart. fanotify(7) documents excluding getpid()
// for exactly this reason. Allow immediately, without scanning.
if int(pid) == sw.selfPID {
if sw.permissionMode {
sw.writeResponse(fd, FAN_ALLOW)
}
_ = unix.Close(int(fd))
return
}
path, err := os.Readlink(fmt.Sprintf("/proc/self/fd/%d", fd))
if err != nil {
// Must respond even on error paths (permission mode)
if sw.permissionMode {
sw.writeResponse(fd, FAN_ALLOW)
}
_ = unix.Close(int(fd))
return
}
// Only process *-D files (message body)
if !strings.HasSuffix(path, "-D") {
if sw.permissionMode {
sw.writeResponse(fd, FAN_ALLOW)
}
_ = unix.Close(int(fd))
return
}
// While the scanner is behind, release opens instead of queueing them.
if sw.dispatchBypass(fd, sw.permissionMode) {
return
}
// Hand to a scan worker. The kernel keeps the opener suspended meanwhile,
// so the guard's budget starts here: time spent queued counts against the
// same deadline as time spent scanning.
sw.initQueueHealth()
evt := spoolEvent{
queueTicket: sw.scannerHealth.Begin(time.Now()),
path: path,
fd: int(fd),
pid: pid,
needResp: sw.permissionMode,
guard: sw.newHoldGuard(fd, sw.permissionMode),
}
select {
case sw.scanCh <- evt:
// Worker will handle response and fd close
case <-sw.stopCh:
// Shutting down - allow and close
evt.queueTicket.Reject(time.Now())
evt.finish(sw, FAN_ALLOW)
default:
// Never wait in the reader: later events in this batch (including our
// own parser opens) are also suspended and do not yet have timers.
evt.queueTicket.Reject(time.Now())
evt.guard.expire()
evt.finish(sw, FAN_ALLOW)
}
}
// startScanWorkers starts the scan goroutines and returns how many. With an
// extractor there is one per worker slot: parts live in the memory cgroup of
// the worker that wrote them, so two scans must never share a worker.
func (sw *SpoolWatcher) startScanWorkers() int {
n := sw.cfg.EmailAV.ScanConcurrency
if sw.extractor != nil {
n = sw.extractor.Size()
}
if n < 1 {
n = 4
}
sw.sources.setLimit(n)
for slot := range n {
sw.wg.Add(1)
obs.Go("spool-scanner", func() { sw.scanWorker(slot) })
}
return n
}
// scanWorker processes spool events on its parse worker slot: extract, scan,
// quarantine/allow. It also handles the events parked behind each source it
// is scanning.
func (sw *SpoolWatcher) scanWorker(slot int) {
defer sw.wg.Done()
handle := func(evt spoolEvent) { sw.handleFair(slot, evt) }
// drainAndClose closes scanCh after shutdown. Keep draining queued events
// until then: dispatchEvent may enqueue a permission fd while Stop is racing,
// and the worker owns the response/close path for every enqueued fd.
for {
select {
case evt, ok := <-sw.scanCh:
if !ok {
return
}
handle(evt)
case <-sw.stopCh:
if atomic.LoadInt32(&sw.runActive) != 0 {
for evt := range sw.scanCh {
handle(evt)
}
return
}
for {
select {
case evt, ok := <-sw.scanCh:
if !ok {
return
}
handle(evt)
default:
return
}
}
}
}
}
// spoolEventHandler processes one queued spool event. Var so tests can
// substitute a panicking handler.
var spoolEventHandler = (*SpoolWatcher).handleSpoolEvent
// handleSpoolEventSafe contains panics and guarantees event cleanup even if a
// handler fails before installing its own defer. Shared guard ownership makes
// this fallback safe after a handler has already answered and closed the fd.
// Re-raising would restart the daemon, and Exim would redeliver the same
// message into the same panic.
func (sw *SpoolWatcher) handleSpoolEventSafe(evt spoolEvent) {
if evt.guard == nil {
evt.guard = &holdGuard{sw: sw, fd: int32(evt.fd), needResp: evt.needResp} // #nosec G115 -- POSIX fd fits in int32.
}
defer func() {
evt.finish(sw, FAN_ALLOW)
if r := recover(); r != nil {
sw.reportScannerPanic(evt.path, r)
}
}()
work := queuehealth.Work[spoolEvent]{Value: evt, Ticket: evt.queueTicket}
work.Process(func(queued spoolEvent) { spoolEventHandler(sw, queued) })
}
// reportScannerPanic logs the panic with its stack, forwards it to
// observability and raises a critical finding at most once per ten minutes.
func (sw *SpoolWatcher) reportScannerPanic(path string, r interface{}) {
obs.CaptureMsg("spool-scanner", fmt.Sprintf("panic scanning %s: %v", path, r))
fmt.Fprintf(os.Stderr, "[%s] spool watcher: recovered panic scanning %s: %v\n%s", ts(), path, r, debug.Stack())
sw.panicMu.Lock()
defer sw.panicMu.Unlock()
if !sw.lastPanicAt.IsZero() && time.Since(sw.lastPanicAt) < 10*time.Minute {
return
}
sw.lastPanicAt = time.Now()
sw.emitFinding("email_av_scanner_panic", alert.Critical,
fmt.Sprintf("Email AV scanner panicked on %s and let it through unscanned: %v", filepath.Base(path), r))
}
func (sw *SpoolWatcher) handleSpoolEvent(evt spoolEvent) {
// CRITICAL: deferred FAN_ALLOW - every code path must allow by default.
// Only overridden to FAN_DENY when policy requires deferral or quarantine.
response := uint32(FAN_ALLOW)
defer func() {
evt.finish(sw, response)
// finish joins the winning verdict; checking before it could miss a
// timer that wins concurrently, or mistake a tempfail for delivery.
released := evt.guard != nil && evt.guard.timedOut() && evt.guard.timeoutVerdict == FAN_ALLOW
if released && response == FAN_DENY {
sw.emitFinding("email_av_late_verdict", alert.Warning,
fmt.Sprintf("Message %s was allowed before its scan finished (hold budget %s) and the scan then asked to stop delivery. A copy may already have been delivered; see the scan and quarantine findings for the final outcome.",
strings.TrimSuffix(filepath.Base(evt.path), "-D"), spoolHoldBudget))
}
}()
// Daemon shutdown releases the fanotify opens. Drain their descriptors
// without starting extractions or reporting shutdown as a scan outage.
select {
case <-sw.daemonStop:
return
default:
}
// Derive message ID: strip -D suffix and directory
base := filepath.Base(evt.path)
msgID := strings.TrimSuffix(base, "-D")
spoolDir := filepath.Dir(evt.path)
headerPath := filepath.Join(spoolDir, msgID+"-H")
// MIME parse in a parse worker
limits := emime.Limits{
MaxAttachmentSize: sw.cfg.EmailAV.MaxAttachmentSize,
MaxArchiveDepth: sw.cfg.EmailAV.MaxArchiveDepth,
MaxArchiveFiles: sw.cfg.EmailAV.MaxArchiveFiles,
MaxExtractionSize: sw.cfg.EmailAV.MaxExtractionSize,
MaxParts: emime.MaxParts,
}
tempfail := sw.cfg.EmailAV.FailMode == "tempfail" && evt.needResp
extraction, messageFault, err := sw.extractSpoolMessage(evt, msgID, headerPath, limits)
if err != nil {
// MAIL-03: Exim opens the -D body file before it writes the matching
// -H header file (the -H lands only after the message is accepted).
// fanotify delivers that reception-time open to us while the -H is
// legitimately absent. Allow silently: emitting a parse-error Warning
// here would fire once per inbound message, and deferring (tempfail)
// would defer 100% of inbound mail. Any ENOENT on a spool file is safe
// to allow -- a message whose spool files are gone cannot be scanned.
if errors.Is(err, os.ErrNotExist) {
return
}
fmt.Fprintf(os.Stderr, "[%s] spool watcher: cannot open spool files of %s: %v\n", ts(), msgID, err)
sw.emitFinding("email_av_parse_error", alert.Warning, fmt.Sprintf("Spool files of message %s could not be opened for scanning: %v", msgID, err))
if tempfail {
response = FAN_DENY // tempfail: Exim retries later
}
return
}
defer extraction.Close()
sw.emitEncryptedArchiveWarning(msgID, extraction.EncryptedEntries, extraction.EncryptedEntriesOmitted)
if extraction.Partial {
partialResult := &emailav.ScanResult{PartialExtraction: true}
switch {
case messageFault && evt.needResp && !sw.cfg.ObserveMode():
// fail_mode open covers the host failing to scan. Mail built to
// break the worker would otherwise choose to pass unscanned.
response = FAN_DENY
sw.emitDegradedWarning("message_fault_deferred", fmt.Sprintf("Incomplete email attachment extraction for message %s (%q) - delivery deferred (the message made the parse worker fail)", msgID, partialExtractionReason(extraction)))
return
case shouldTempfailEmailDelivery(tempfail, partialResult, nil):
response = FAN_DENY
sw.emitDegradedWarning("partial_extraction_deferred", fmt.Sprintf("Incomplete email attachment extraction for message %s (%q) - delivery deferred (tempfail mode)", msgID, partialExtractionReason(extraction)))
return
}
sw.emitDegradedWarning("partial_extraction_allowed", fmt.Sprintf("Incomplete email attachment extraction for message %s (%q) - delivery allowed (fail-open mode)", msgID, partialExtractionReason(extraction)))
}
if len(extraction.Parts) == 0 {
return // No attachments to scan - allow
}
// Scan
result := sw.orchestrator.ScanParts(msgID, extraction.Parts, extraction.Partial)
// Emit degraded/timeout findings for operator visibility
if result.AllEnginesDown {
if shouldTempfailEmailDelivery(tempfail, result, nil) {
response = FAN_DENY // tempfail: defer delivery until engines recover
sw.emitDegradedWarning("engines_down_deferred", fmt.Sprintf("All AV engines unavailable - message %s deferred (tempfail mode)", msgID))
return
}
sw.emitDegradedWarning("engines_down_unscanned", fmt.Sprintf("All AV engines unavailable - message %s delivered unscanned", msgID))
}
if len(result.TimedOutEngines) > 0 {
sw.emitFinding("email_av_timeout", alert.Warning,
fmt.Sprintf("Scan timeout for message %s on engine(s): %s", msgID, strings.Join(result.TimedOutEngines, ", ")))
if shouldTempfailEmailDelivery(tempfail, result, nil) {
sw.emitDegradedWarning("scan_timeout_deferred", fmt.Sprintf("Incomplete AV scan - message %s deferred after engine timeout", msgID))
response = FAN_DENY
return
}
}
if len(result.ErroredEngines) > 0 {
sw.emitFinding("email_av_scan_error", alert.Warning,
fmt.Sprintf("Scan error for message %s on engine(s): %s", msgID, strings.Join(result.ErroredEngines, ", ")))
if shouldTempfailEmailDelivery(tempfail, result, nil) {
sw.emitDegradedWarning("scan_error_deferred", fmt.Sprintf("Incomplete AV scan - message %s deferred after engine error", msgID))
response = FAN_DENY
return
}
}
if !result.Infected {
return // Clean - allow
}
// Infected - attempt quarantine
env := emailav.QuarantineEnvelope{
From: extraction.From,
To: extraction.To,
Subject: extraction.Subject,
Direction: extraction.Direction,
}
if sw.cfg.EmailAV.QuarantineInfected {
if err := sw.quarantineSpoolEvent(evt, msgID, spoolDir, result, env); err != nil {
fmt.Fprintf(os.Stderr, "[%s] spool watcher: quarantine failed for %s: %v\n", ts(), msgID, err)
sw.emitFinding("email_av_quarantine_error", alert.Warning,
fmt.Sprintf("Quarantine failed for infected message %s: %v", msgID, err))
if shouldTempfailEmailDelivery(tempfail, result, err) {
sw.emitDegradedWarning("quarantine_failed_deferred", fmt.Sprintf("Quarantine failed for infected message %s - delivery deferred (tempfail mode)", msgID))
response = FAN_DENY
}
} else {
// Quarantine succeeded - deny the open so Exim can't deliver
response = FAN_DENY
if sw.onQuarantined != nil {
sw.onQuarantined(time.Now(), msgID)
}
}
}
// Emit alert finding
sigNames := make([]string, len(result.Findings))
for i, f := range result.Findings {
sigNames[i] = fmt.Sprintf("%s(%s)", f.Signature, f.Engine)
}
msg := fmt.Sprintf("Malware detected in %q email from %q to %q: %s [subject: %q]",
extraction.Direction, extraction.From, strings.Join(extraction.To, ","),
strings.Join(sigNames, ", "), extraction.Subject)
sw.emitFinding("email_malware", alert.Critical, msg)
}
func partialExtractionReason(extraction *emime.ExtractionResult) string {
if extraction.PartialReason != "" {
return extraction.PartialReason
}
return "partial extraction"
}
func (sw *SpoolWatcher) writeResponse(fd int32, response uint32) {
resp := fanotifyResponse{Fd: fd, Response: response}
// #nosec G103 -- serializing the fanotify response struct for the
// kernel write; unsafe cast to a byte slice of the exact struct size.
respBytes := (*[responseSize]byte)(unsafe.Pointer(&resp))[:]
sw.initQueueHealth()
err := sw.kernelQueue.write(respBytes)
if err != nil {
// The kernel holds blocked processes until a response is written or
// the fanotify fd is closed. A failed write means the fd is broken -
// close it to release ALL pending permission events (fail-open),
// then signal the event loop to exit so the daemon can restart us.
fmt.Fprintf(os.Stderr, "[%s] spool watcher: FATAL - fanotify response write failed: %v - closing fd to release pending events\n", ts(), err)
sw.closeFd()
sw.Stop()
}
}
// closeFd closes the fanotify fd exactly once, even if called from multiple paths.
func (sw *SpoolWatcher) closeFd() {
if atomic.CompareAndSwapInt32(&sw.fdClosed, 0, 1) {
sw.initQueueHealth()
_ = sw.kernelQueue.close()
}
}
func (sw *SpoolWatcher) emitFinding(check string, severity alert.Severity, message string) bool {
if alert.TryEnqueue(sw.alertCh, alert.Finding{
Severity: severity,
Check: check,
Message: message,
Timestamp: time.Now(),
}) {
return true
} else {
// Alert channel full - drop
return false
}
}
// encryptedArchiveAlertInterval rate-limits the encrypted-archive finding.
// Password-protected archives are a steady part of ordinary business mail, so
// this is a recurring condition rather than an incident; a per-minute limit
// would put over a thousand findings a day in front of the operator.
const encryptedArchiveAlertInterval = time.Hour
// emitEncryptedArchiveWarning reports attachments delivered without being
// scanned because their archive entries are encrypted. This is deliberately
// not an email_av_degraded finding: nothing is degraded, and no retry will
// make the content readable. Keeping it separate lets an operator set policy
// on unscannable mail without losing the signal that scanning itself broke.
func (sw *SpoolWatcher) emitEncryptedArchiveWarning(msgID string, entries []emime.EncryptedArchiveEntry, omitted int) {
if len(entries) == 0 && omitted == 0 {
return
}
sw.encryptedMu.Lock()
defer sw.encryptedMu.Unlock()
if time.Since(sw.lastEncryptedAt) < encryptedArchiveAlertInterval {
return
}
named := make([]string, 0, len(entries))
for _, e := range entries {
named = append(named, fmt.Sprintf("%q in %q", e.Filename, e.ArchiveName))
}
if omitted > 0 {
named = append(named, fmt.Sprintf("%d additional encrypted member(s) (names omitted)", omitted))
}
if sw.emitFinding("email_av_encrypted_archive", alert.Warning,
fmt.Sprintf("Encrypted archive attachment could not be scanned for message %s: %s",
msgID, strings.Join(named, ", "))) {
// Only delivered warnings consume the allowance; queue pressure must
// not hide this condition for an hour after the queue recovers.
sw.lastEncryptedAt = time.Now()
}
}
// emitDegradedWarning emits an email_av_degraded finding, rate-limited to
// once per minute to avoid flooding the alert channel when clamd is down.
// kind names the condition and its outcome; the message names the message
// that hit it, which differs on every report of the same condition.
func (sw *SpoolWatcher) emitDegradedWarning(kind, message string) {
sw.degradedMu.Lock()
if time.Since(sw.lastDegradedAt) < time.Minute {
sw.degradedMu.Unlock()
return
}
sw.lastDegradedAt = time.Now()
sw.degradedMu.Unlock()
alert.TryEnqueue(sw.alertCh, alert.Finding{
Severity: alert.Warning,
Check: "email_av_degraded",
Message: message,
DedupKey: kind,
Timestamp: time.Now(),
})
}
func (sw *SpoolWatcher) drainAndClose() {
sw.drainOnce.Do(func() {
close(sw.scanCh)
sw.wg.Wait()
sw.closeFd()
if atomic.CompareAndSwapInt32(&sw.pipeClosed, 0, 1) {
_ = unix.Close(sw.pipeFds[0])
_ = unix.Close(sw.pipeFds[1])
}
})
}
// PermissionMode returns true if using FAN_OPEN_PERM, false if FAN_CLOSE_WRITE fallback.
func (sw *SpoolWatcher) PermissionMode() bool {
return sw.permissionMode
}
// Stop signals the event loop to exit.
func (sw *SpoolWatcher) Stop() {
sw.stopOnce.Do(func() {
close(sw.stopCh)
if atomic.LoadInt32(&sw.pipeClosed) == 0 {
_, _ = unix.Write(sw.pipeFds[1], []byte{0})
}
sw.closeFd()
})
}
//go:build linux
package daemon
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/crashsample"
emime "github.com/pidginhost/csm/internal/mime"
"github.com/pidginhost/csm/internal/parseworker"
"github.com/pidginhost/csm/internal/workerproc"
)
// spoolExtractTimeout backstops one extraction. The worker's own deadline
// (parseworker.ExtractTimeout) catches a hang and counts it against the
// worker; this one also covers starting the worker's unit. Var for tests.
var spoolExtractTimeout = parseworker.ExtractTimeout + 10*time.Second
// errNoExtractor stands for parse workers that could not be set up.
var errNoExtractor = errors.New("no mail parse worker")
const (
// repeatFailureReason is the partial reason of a message that is not
// unpacked again because it made workers fail before.
repeatFailureReason = "parse worker failed on this message before"
// outcomeSkipped is the job outcome of such a message.
outcomeSkipped = "skipped"
// sourceKillLimit worker kills from one source, each within
// sourceKillWindow of the one before, raise a High finding.
sourceKillLimit = 3
sourceKillWindow = 10 * time.Minute
)
func (sw *SpoolWatcher) initStrikes() {
sw.mailStrikes = workerproc.NewStrikes()
sw.sourceKills = workerproc.NewStrikesWith(sourceKillLimit, sourceKillWindow)
}
func (sw *SpoolWatcher) inheritStrikes(previous *SpoolWatcher) {
if previous.mailStrikes != nil {
sw.mailStrikes = previous.mailStrikes
sw.sourceKills = previous.sourceKills
}
}
// countedKill reports a failure the worker's input caused: a timeout, a
// memory kill, a crash or a rejected reply. Failed starts, the breaker,
// cancellation and shutdown are not the message's doing.
func countedKill(err error) bool {
switch workerproc.FailureReason(err) {
case workerproc.ReasonTimeout, workerproc.ReasonOOM, workerproc.ReasonCrash, workerproc.ReasonProtocol:
return true
}
return false
}
// openSpoolHeader opens the -H file of the message whose -D is open on
// bodyFD. It follows no link and does not block on a FIFO, and the result
// must be a regular file on the body's filesystem: the worker is given
// whatever this opens. A missing file keeps its ENOENT for the reception race
// (MAIL-03).
func openSpoolHeader(path string, bodyFD int) (*os.File, error) {
fd, err := unix.Open(path, unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_CLOEXEC|unix.O_NONBLOCK, 0)
if err != nil {
return nil, &os.PathError{Op: "open", Path: path, Err: err}
}
var hs, bs unix.Stat_t
err = unix.Fstat(fd, &hs)
if err == nil {
err = unix.Fstat(bodyFD, &bs)
}
switch {
case err != nil:
case hs.Mode&unix.S_IFMT != unix.S_IFREG:
err = fmt.Errorf("spool header %s is not a regular file", path)
case hs.Dev != bs.Dev:
err = fmt.Errorf("spool header %s is not on the spool filesystem", path)
default:
err = unix.SetNonblock(fd, false)
}
if err != nil {
_ = unix.Close(fd)
return nil, err
}
return os.NewFile(uintptr(fd), path), nil
}
// dupEventBody gives extraction independent ownership of the event's body
// descriptor while preserving close-on-exec on the copy.
func dupEventBody(fd int) (*os.File, error) {
dup, err := unix.FcntlInt(uintptr(fd), unix.F_DUPFD_CLOEXEC, 0) // #nosec G115 -- POSIX fd is non-negative.
if err != nil {
return nil, fmt.Errorf("duplicating spool body descriptor: %w", err)
}
var st unix.Stat_t
if err = unix.Fstat(dup, &st); err == nil && st.Mode&unix.S_IFMT != unix.S_IFREG {
err = errors.New("spool body is not a regular file")
}
if err != nil {
_ = unix.Close(dup)
return nil, err
}
return os.NewFile(uintptr(dup), "spool body"), nil // #nosec G115 -- POSIX fd is non-negative.
}
// extractSpoolMessage extracts the event's message in the event's worker
// slot. An error means the spool files could not be opened; a worker that
// fails, or no worker at all, gives a partial result naming why. messageFault
// marks a partial result the message caused: it made a worker fail now, or
// often enough before that it is not unpacked again.
func (sw *SpoolWatcher) extractSpoolMessage(evt spoolEvent, msgID, headerPath string, limits emime.Limits) (extraction *emime.ExtractionResult, messageFault bool, err error) {
body, err := dupEventBody(evt.fd)
if err != nil {
return nil, false, err
}
defer func() { _ = body.Close() }()
header, err := openSpoolHeader(headerPath, evt.fd)
if err != nil {
return nil, false, err
}
defer func() { _ = header.Close() }()
if sw.extractor == nil {
return sw.extractFailed(msgID, evt, errNoExtractor), false, nil
}
if sw.mailStrikes != nil && sw.mailStrikes.Blocked(msgID) {
recordParseJob("mail", outcomeSkipped, 0)
fmt.Fprintf(os.Stderr, "[%s] spool watcher: not unpacking %s again: parse workers failed on it before\n", ts(), msgID)
return &emime.ExtractionResult{Partial: true, PartialReason: repeatFailureReason}, true, nil
}
ctx, cancel := context.WithTimeout(context.Background(), spoolExtractTimeout)
defer cancel()
extraction, err = sw.extractor.Extract(ctx, evt.slot, header, body, limits)
if err != nil {
killed := countedKill(err)
if killed {
sw.recordWorkerKill(msgID, evt.source)
sw.saveCrashSample(msgID, header, body)
}
return sw.extractFailed(msgID, evt, err), killed, nil
}
return extraction, false, nil
}
// recordWorkerKill charges a worker kill to the message and its source. A
// source whose messages keep killing workers is named in a High finding.
func (sw *SpoolWatcher) recordWorkerKill(msgID, source string) {
if sw.mailStrikes == nil {
return
}
sw.mailStrikes.Record(msgID)
if source == "" {
return
}
sw.sourceKills.Record(source)
if sw.sourceKills.Blocked(source) {
sw.emitLimited("email_av_parse_worker_source", alert.High, sourceKillWindow,
fmt.Sprintf("Messages from source %q killed the attachment parse worker %d times, each within %s of the one before. Their attachments were not scanned. This looks like crafted mail; consider blocking the source.", source, sourceKillLimit, sourceKillWindow))
}
}
func (sw *SpoolWatcher) setCrashSamples(s *crashsample.Store) { sw.crashSamples = s }
// saveCrashSample keeps a copy of a message that killed a worker, when crash
// samples are on. A failed save changes nothing else.
func (sw *SpoolWatcher) saveCrashSample(msgID string, header, body *os.File) {
if sw.crashSamples == nil {
return
}
var files []crashsample.File
for _, f := range []struct {
name string
file *os.File
}{{"body", body}, {"header", header}} {
info, err := f.file.Stat()
if err != nil {
fmt.Fprintf(os.Stderr, "[%s] spool watcher: keeping crash sample of %s: %v\n", ts(), msgID, err)
return
}
files = append(files, crashsample.File{Name: f.name, Src: f.file, Size: info.Size()})
}
if err := sw.crashSamples.Save(msgID, files); err != nil {
fmt.Fprintf(os.Stderr, "[%s] spool watcher: keeping crash sample of %s: %v\n", ts(), msgID, err)
}
}
// sourceText names a message source for a finding.
func sourceText(source string) string {
if source == "" {
return "an unknown source"
}
return fmt.Sprintf("source %q", source)
}
// extractFailed reports a failed extraction and turns it into a partial
// result, which is never clean.
func (sw *SpoolWatcher) extractFailed(msgID string, evt spoolEvent, err error) *emime.ExtractionResult {
reason, isolation := extractFailureReason(err)
fmt.Fprintf(os.Stderr, "[%s] spool watcher: extraction failed for %s: %q\n", ts(), msgID, err)
switch {
case errors.Is(err, workerproc.ErrStopped):
// The daemon is shutting down; that is not an outage.
case isolation:
sw.emitLimited("email_av_isolation_unavailable", alert.Critical, isolationAlertInterval,
fmt.Sprintf("Email attachments cannot be unpacked in an isolated worker (%s); message %s was handled as an incomplete scan under email_av.fail_mode: %q", reason, msgID, err))
default:
outcome := "the message was handled as an incomplete scan under email_av.fail_mode"
if countedKill(err) {
switch {
case sw.cfg.ObserveMode():
outcome = "the message caused the failure; delivery is allowed in observe mode"
case !evt.needResp:
outcome = "the message caused the failure; delivery cannot be deferred without fanotify permission events"
default:
// The hold may already have answered, so this is a request,
// not proof that the message was stopped.
outcome = "the message caused the failure; delivery deferral is requested regardless of email_av.fail_mode"
}
}
sw.emitLimited("email_av_parse_worker_failed", alert.Warning, time.Minute,
fmt.Sprintf("Parse worker failed on message %s from %s (%s); %s", msgID, sourceText(evt.source), reason, outcome))
if workerproc.FailureReason(err) == workerproc.ReasonProtocol {
sw.emitLimited("email_av_parse_worker_rejected", alert.Critical, time.Hour,
fmt.Sprintf("Parse worker sent an invalid reply for message %s and was killed; a crafted message may have taken it over", msgID))
}
}
return &emime.ExtractionResult{Partial: true, PartialReason: reason}
}
// extractFailureReason names an extraction failure for the partial result,
// and says whether isolation itself is unavailable rather than one worker
// having failed.
func extractFailureReason(err error) (reason string, isolation bool) {
switch {
case errors.Is(err, workerproc.ErrWorkerFailed):
switch workerproc.FailureReason(err) {
case workerproc.ReasonTimeout:
return "parse worker timed out", false
case workerproc.ReasonOOM:
return "parse worker ran out of memory", false
case workerproc.ReasonProtocol:
return "parse worker reply rejected", false
default:
return "parse worker crashed", false
}
case errors.Is(err, context.DeadlineExceeded):
return "parse worker timed out", false
case errors.Is(err, workerproc.ErrBreakerOpen):
return "parse worker failing repeatedly", true
default:
return "parse isolation unavailable", true
}
}
// emitLimited emits a finding at most once per interval per check.
func (sw *SpoolWatcher) emitLimited(check string, severity alert.Severity, interval time.Duration, message string) {
l := sw.findings
if l == nil {
l = &sw.ownFindings
}
l.emit(sw.alertCh, check, severity, interval, message)
}
// removeLegacyEmailAVTempDir removes the attachment staging directory of
// earlier versions. A link in its place is removed, never followed.
func removeLegacyEmailAVTempDir(statePath string) {
if statePath == "" {
return
}
dir := filepath.Join(statePath, "emailav-tmp")
if _, err := os.Lstat(dir); err != nil {
return
}
if err := os.RemoveAll(dir); err != nil {
fmt.Fprintf(os.Stderr, "[%s] spool watcher: removing old attachment directory %s: %v\n", ts(), dir, err)
return
}
fmt.Fprintf(os.Stderr, "[%s] spool watcher: removed old attachment directory %s\n", ts(), dir)
}
//go:build linux
package daemon
import (
"fmt"
"os"
"runtime/debug"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/emailspool"
"github.com/pidginhost/csm/internal/obs"
)
// maxSourceWaiting bounds how many messages may wait behind one source's
// running scans. Real senders stay far below it; past it, messages are
// scanned at once without waiting their turn.
const maxSourceWaiting = 32
// sourceGate lets one source scan on at most limit slots at a time, so one
// sender cannot take every scan slot. limit is half the slots, not one: a
// single key can stand for a filtering gateway, unauthenticated mail from
// localhost, or the bounces Exim writes itself. Messages parked behind a
// source hold no slot; the goroutines scanning that source handle them next.
type sourceGate struct {
mu sync.Mutex
limit int
busy map[string]*sourceScans
}
type sourceScans struct {
running int
waiting []spoolEvent
}
// setLimit sets the scans one source may run at once to half of slots, at
// least one.
func (g *sourceGate) setLimit(slots int) {
g.mu.Lock()
defer g.mu.Unlock()
g.limit = max(1, slots/2)
}
// enter claims a scan of src for evt. run means scan evt now; otherwise evt
// is parked, unless overflow says the source already has maxSourceWaiting
// waiting.
func (g *sourceGate) enter(src string, evt spoolEvent) (run, overflow bool) {
g.mu.Lock()
defer g.mu.Unlock()
if g.busy == nil {
g.busy = make(map[string]*sourceScans)
}
scans := g.busy[src]
if scans == nil {
scans = &sourceScans{}
g.busy[src] = scans
}
if scans.running < max(1, g.limit) {
scans.running++
return true, false
}
if len(scans.waiting) >= maxSourceWaiting {
return false, true
}
scans.waiting = append(scans.waiting, evt)
return false, false
}
// leave hands over the next parked message of src to the scan that just
// ended, or ends that scan when none is waiting.
func (g *sourceGate) leave(src string) (spoolEvent, bool) {
g.mu.Lock()
defer g.mu.Unlock()
scans := g.busy[src]
if scans == nil {
return spoolEvent{}, false
}
if len(scans.waiting) > 0 {
next := scans.waiting[0]
scans.waiting[0] = spoolEvent{}
scans.waiting = scans.waiting[1:]
return next, true
}
scans.running--
if scans.running <= 0 {
delete(g.busy, src)
}
return spoolEvent{}, false
}
// readSpoolSource is a seam for tests.
var readSpoolSource = emailspool.ReadSource
// safeSpoolSource is spoolSource behind a panic guard: it parses
// attacker-influenced bytes before handleSpoolEventSafe's own guard, and a
// parser bug must not crash the daemon on every redelivery. A message whose
// source cannot be read is scanned without the gate.
func (sw *SpoolWatcher) safeSpoolSource(evt spoolEvent) (src string) {
defer func() {
if r := recover(); r != nil {
obs.CaptureMsg("spool-scanner", fmt.Sprintf("panic reading the source of %s: %v", evt.path, r))
fmt.Fprintf(os.Stderr, "[%s] spool watcher: recovered panic reading the source of %s: %v\n%s", ts(), evt.path, r, debug.Stack())
src = ""
}
}()
return sw.spoolSource(evt)
}
// spoolSource reads the source key of the event's message from its -H file,
// with the same refusals as extraction. It returns "" when the -H cannot be
// read, which includes the reception open before Exim writes it (MAIL-03).
func (sw *SpoolWatcher) spoolSource(evt spoolEvent) string {
if !strings.HasSuffix(evt.path, "-D") {
return ""
}
header, err := openSpoolHeader(strings.TrimSuffix(evt.path, "-D")+"-H", evt.fd)
if err != nil {
return ""
}
defer func() { _ = header.Close() }()
src, err := readSpoolSource(header)
if err != nil {
return ""
}
return src
}
// handleFair scans evt on slot, at most the gate's limit of scans per
// source at a time. Messages parked behind a source are handled by one of
// that source's scanning goroutines, on its slot, once a scan finishes.
func (sw *SpoolWatcher) handleFair(slot int, evt spoolEvent) {
evt.slot = slot
evt.guard.setStage(holdScanning)
src := sw.safeSpoolSource(evt)
if src == "" {
sw.handleSpoolEventSafe(evt)
return
}
evt.source = src
// Mark the hold as waiting before parking: once parked, the source's
// goroutine may take the event and mark it scanning at any time. A hold
// that expires while waiting follows fail_mode without counting toward
// the bypass.
evt.guard.setStage(holdWaiting)
run, overflow := sw.sources.enter(src, evt)
switch {
case overflow:
sw.sourceOverflow(evt)
evt.guard.setStage(holdScanning)
sw.handleSpoolEventSafe(evt)
return
case !run:
if spoolSourceWaitsTotal != nil {
spoolSourceWaitsTotal.With("waited").Inc()
}
return
}
evt.guard.setStage(holdScanning)
for more := true; more; {
func() {
// A panic must not leave the source busy.
defer func() { evt, more = sw.sources.leave(src) }()
sw.handleSpoolEventSafe(evt)
}()
if more {
evt.slot = slot
evt.guard.setStage(holdScanning)
}
}
}
// sourceOverflow reports an event whose source already has the maximum
// number of messages waiting. The event is then scanned at once, outside the
// gate: a source key can stand for many senders, so answering it unscanned
// would deliver their mail unchecked.
func (sw *SpoolWatcher) sourceOverflow(evt spoolEvent) {
if spoolSourceWaitsTotal != nil {
spoolSourceWaitsTotal.With("overflow").Inc()
}
sw.emitLimited("email_av_source_backlog", alert.Warning, time.Minute,
fmt.Sprintf("More than %d messages from source %q are waiting to be scanned. Further messages from it are scanned without waiting their turn and can take every scan slot until the backlog clears. Block the source if it should not send this much mail.", maxSourceWaiting, evt.source))
}
//go:build linux
package daemon
import (
"fmt"
"os"
"sync"
"sync/atomic"
"time"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/alert"
)
// spoolHoldBudget bounds how long one mail open may stay suspended waiting for
// a scan verdict. The kernel keeps the opening process frozen until we answer,
// so an unbounded wait turns slow scanning into frozen mail: a saturated
// scanner once held 167 Exim processes and stalled outbound mail host-wide.
// The scan continues after the deadline and can still quarantine.
var spoolHoldBudget = 5 * time.Second
const (
// spoolHoldExpiryWindow and spoolHoldExpiryThreshold define the burst that
// means the scanner cannot keep up with mail rather than one slow message.
spoolHoldExpiryWindow = time.Minute
spoolHoldExpiryThreshold = 10
// spoolBypassCooldown is how long opens are released unheld after a burst.
spoolBypassCooldown = 5 * time.Minute
)
// spoolWriteResponse is the seam to the kernel verdict write. Var so tests can
// observe verdicts without a live fanotify descriptor.
var spoolWriteResponse = (*SpoolWatcher).writeResponse
// holdStage is where a suspended open is when its hold budget runs out. Only
// an open still queued for a scan slot shows that scan capacity is exhausted.
type holdStage int32
const (
holdQueued holdStage = iota
holdWaiting
holdScanning
)
func (s holdStage) String() string {
switch s {
case holdWaiting:
return "waiting"
case holdScanning:
return "scanning"
default:
return "queued"
}
}
// holdGuard owns the verdict for one suspended open. Whoever gets there first
// wins -- the scan, or the budget timer -- and the kernel is answered once.
type holdGuard struct {
sw *SpoolWatcher
fd int32
needResp bool
once sync.Once
closeOnce sync.Once
timer *time.Timer
expired atomic.Bool
stage atomic.Int32
timeoutVerdict uint32 // read only after respond has joined once
}
// setStage records where the open is now. Safe on a nil guard.
func (g *holdGuard) setStage(s holdStage) {
if g != nil {
g.stage.Store(int32(s))
}
}
// newHoldGuard arms the budget timer for a suspended open. Outside permission
// mode nothing is suspended, so there is no verdict to write and no timer.
func (sw *SpoolWatcher) newHoldGuard(fd int32, needResp bool) *holdGuard {
g := &holdGuard{sw: sw, fd: fd, needResp: needResp && sw.permissionMode}
if g.needResp {
g.timer = time.AfterFunc(spoolHoldBudget, g.expire)
}
return g
}
// timeoutResponse is the verdict used when the budget runs out. fail_mode
// tempfail means the operator chose never to deliver unscanned mail, so defer
// and let Exim retry; otherwise release the message and keep scanning.
func (sw *SpoolWatcher) timeoutResponse() uint32 {
if sw.cfg.EmailAV.FailMode == "tempfail" {
return FAN_DENY
}
return FAN_ALLOW
}
func (g *holdGuard) expire() {
if !g.needResp {
return
}
g.once.Do(func() {
// The scan can advance while the kernel response is being written.
stage := holdStage(g.stage.Load())
g.timeoutVerdict = g.sw.timeoutResponse()
g.expired.Store(true)
spoolWriteResponse(g.sw, g.fd, g.timeoutVerdict)
// Keep all callback work inside once so finishing an event also joins
// its timer before the watcher or test hooks can be torn down.
if spoolHoldExpiredTotal != nil {
spoolHoldExpiredTotal.With(stage.String()).Inc()
}
// A slow scan, or a message waiting behind its own sender, is not a
// capacity problem: one sender must not be able to switch holding off.
if stage == holdQueued {
g.sw.noteHoldExpiry(time.Now())
}
})
}
// respond hands the kernel the scan's verdict, unless the budget already
// answered for this event.
func (g *holdGuard) respond(response uint32) {
if g.timer != nil {
g.timer.Stop()
}
if !g.needResp {
return
}
g.once.Do(func() {
spoolWriteResponse(g.sw, g.fd, response)
})
}
func (g *holdGuard) finish(response uint32) {
g.closeOnce.Do(func() {
// once waits for a timer's in-progress write before the fd can be
// closed and recycled. Panic cleanup may finish the same event again.
g.respond(response)
_ = unix.Close(int(g.fd))
})
}
// timedOut reports whether the budget answered before the scan did.
func (g *holdGuard) timedOut() bool { return g.expired.Load() }
// holdWatchdog tracks opens that never got a scan slot: holds that expired in
// the queue and opens refused by a full queue. A burst of them means scanning
// cannot keep up with mail, and holding every further message for the full
// budget only spreads the delay, so the watcher stops holding for a cooldown.
type holdWatchdog struct {
mu sync.Mutex
expiries []time.Time
bypassUntil time.Time
}
// recordExpiry notes one expiry and reports whether it started a bypass.
func (w *holdWatchdog) recordExpiry(now time.Time) bool {
w.mu.Lock()
defer w.mu.Unlock()
w.expiries = append(pruneTimes(w.expiries, now.Add(-spoolHoldExpiryWindow)), now)
if len(w.expiries) < spoolHoldExpiryThreshold || now.Before(w.bypassUntil) {
return false
}
w.bypassUntil = now.Add(spoolBypassCooldown)
w.expiries = w.expiries[:0]
return true
}
func (w *holdWatchdog) bypassing(now time.Time) bool {
w.mu.Lock()
defer w.mu.Unlock()
return now.Before(w.bypassUntil)
}
// noteHoldExpiry records one open that could not start scanning in time and
// reports the first transition into bypass.
func (sw *SpoolWatcher) noteHoldExpiry(now time.Time) {
if !sw.holds.recordExpiry(now) {
return
}
action := "allowed without scanning"
if sw.timeoutResponse() == FAN_DENY {
action = "deferred without scanning (tempfail mode)"
}
fmt.Fprintf(os.Stderr, "[%s] spool watcher: scan capacity repeatedly exhausted - mail %s for %s\n", ts(), action, spoolBypassCooldown)
sw.emitFinding("email_av_hold_bypass", alert.Critical,
fmt.Sprintf("Email AV scanning fell behind mail delivery: scan capacity was exhausted. %d messages within %s could not start scanning within the %s hold budget, or found the scan queue full. New messages are %s for the next %s; these messages are not queued for a later scan. Scans already running continue. Investigate scanner load.",
spoolHoldExpiryThreshold, spoolHoldExpiryWindow, spoolHoldBudget, action, spoolBypassCooldown))
}
// dispatchBypass answers an open immediately while bypassing, without
// queueing it. Queueing is what left Exim waiting behind a saturated scanner.
// Reports whether it handled the event.
func (sw *SpoolWatcher) dispatchBypass(fd int32, needResp bool) bool {
if !sw.holds.bypassing(time.Now()) {
return false
}
if needResp && sw.permissionMode {
spoolWriteResponse(sw, fd, sw.timeoutResponse())
}
_ = unix.Close(int(fd))
sw.initQueueHealth()
sw.scannerHealth.Lose(time.Now(), 1)
return true
}
package daemon
import "github.com/pidginhost/csm/internal/emailav"
func shouldTempfailEmailDelivery(tempfail bool, result *emailav.ScanResult, quarantineErr error) bool {
if !tempfail {
return false
}
if quarantineErr != nil {
return true
}
if result == nil {
return false
}
return result.PartialExtraction ||
result.AllEnginesDown ||
len(result.TimedOutEngines) > 0 ||
len(result.ErroredEngines) > 0
}
//go:build linux
package daemon
import (
"fmt"
"io"
"os"
"path/filepath"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/emailav"
)
// A verdict deadline can let Exim start delivery while we scan. Take its body
// lock before touching either spool file, even if the deadline has not fired
// yet: it may fire during the move. An OFD lock also excludes other workers
// and survives unrelated parser closes, unlike process-wide F_SETLK locks.
func (sw *SpoolWatcher) quarantineSpoolEvent(evt spoolEvent, msgID, spoolDir string, result *emailav.ScanResult, env emailav.QuarantineEnvelope) error {
fd, err := unix.Open(evt.path, unix.O_RDWR|unix.O_NOFOLLOW|unix.O_CLOEXEC|unix.O_NONBLOCK, 0)
if err != nil {
return fmt.Errorf("opening spool body for quarantine: %w", err)
}
defer func() { _ = unix.Close(fd) }()
lock := unix.Flock_t{Type: unix.F_WRLCK, Whence: int16(io.SeekStart)}
// #nosec G115 -- successful unix.Open returned a nonnegative POSIX fd.
if err := unix.FcntlFlock(uintptr(fd), unix.F_OFD_SETLK, &lock); err != nil {
return fmt.Errorf("locking spool body for quarantine: %w", err)
}
var original, current unix.Stat_t
if err := unix.Fstat(evt.fd, &original); err != nil {
return fmt.Errorf("checking event body for quarantine: %w", err)
}
if err := unix.Lstat(evt.path, ¤t); err != nil {
return fmt.Errorf("checking spool body for quarantine: %w", err)
}
var locked unix.Stat_t
if err := unix.Fstat(fd, &locked); err != nil {
return fmt.Errorf("checking locked body for quarantine: %w", err)
}
if current.Mode&unix.S_IFMT != unix.S_IFREG || original.Dev != current.Dev || original.Ino != current.Ino || locked.Dev != current.Dev || locked.Ino != current.Ino {
return fmt.Errorf("spool body changed before quarantine")
}
// A delivery journal contains recipients already delivered. The quarantine
// format stores only H/D; separating a surviving journal would lose that
// state on release and could send duplicates. Leave it for Exim recovery.
if _, err := os.Lstat(filepath.Join(spoolDir, msgID+"-J")); !os.IsNotExist(err) {
if err != nil {
return fmt.Errorf("checking delivery journal: %w", err)
}
return fmt.Errorf("spool delivery journal requires Exim recovery before quarantine")
}
if _, err := os.Lstat(filepath.Join(spoolDir, msgID+"-H")); err != nil {
return fmt.Errorf("checking spool header for quarantine: %w", err)
}
return sw.quarantine.QuarantineMessage(msgID, spoolDir, result, env)
}
package daemon
import (
"sort"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
)
// subnetSighting is one address of a /24 inside the spray window: when it was
// last seen and the line that last named it.
type subnetSighting struct {
at time.Time
obs alert.Observation
}
// sprayConstituents lists the addresses a subnet spray counted, sorted. A /24
// holds at most 256 addresses, which bounds the list.
func sprayConstituents(ips map[string]subnetSighting) []alert.SprayConstituent {
out := make([]alert.SprayConstituent, 0, len(ips))
for ip, s := range ips {
// Both mail trackers count the transport peer their parser read.
out = append(out, alert.SprayConstituent{Address: ip, LastSeen: s.at, Observation: s.obs, Attribution: admission.AttributionPeer})
}
sort.Slice(out, func(i, j int) bool { return out[i].Address < out[j].Address })
return out
}
package daemon
import (
"fmt"
"os"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
)
// staleSession401Hold is how long a session-URL 401 waits for the evidence
// that would explain it. An anonymous request may need a named request at
// the opposite edge of the denial's match window. Allow both windows plus
// one watcher poll and one poll of read slack. Unexplained failures leave
// correlation at this deadline and retain the original finding.
const staleSession401Hold = 2*checks.StaleSessionMatchWindow + 2*logWatcherPollInterval
// staleSessionDenialRetention bounds memory only. Matching is by log time,
// so keeping a denial longer never widens what it explains; it only has to
// outlast a watcher that falls behind. cPanel writes a denial only for a
// live session, so the set stays small.
const staleSessionDenialRetention = 10 * time.Minute
// Two requests explained by one denial can be two match windows apart.
// Keep the rejection through the later request's watcher delay and hold.
const staleSessionRejectionRetention = 2*checks.StaleSessionMatchWindow + 2*logWatcherPollInterval + staleSession401Hold
// staleSession401s keeps a stale browser tab from being reported, and
// blocked, as an API authentication failure. A tab left open after its
// account logged in again elsewhere keeps the old URL security token; its
// next page load fails on every API call until cPanel purges the session
// for token failures. The 401 lines and the session_log purge are read by
// separate watchers, so a session-URL 401 is held until the purge could
// have been read, then reported unchanged if nothing explained it.
type staleSession401s struct {
mu sync.Mutex
denials map[sessionEvidenceKey]observedExplanation
rejected map[sessionEvidenceKey]map[string]observedExplanation
explained map[sessionEvidenceKey]observedExplanation
held []heldAPI401
}
// cPanel timestamps have second precision. Named evidence is keyed by
// account, anonymous evidence by URL token; neither can cross an address.
type sessionEvidenceKey struct {
ip string
name string
at int64
}
type observedExplanation struct {
seen time.Time
expires time.Time
}
type heldAPI401 struct {
req checks.StaleSessionRequest
finding alert.Finding
due time.Time
}
func newStaleSession401s() *staleSession401s {
return &staleSession401s{
denials: make(map[sessionEvidenceKey]observedExplanation),
rejected: make(map[sessionEvidenceKey]map[string]observedExplanation),
explained: make(map[sessionEvidenceKey]observedExplanation),
}
}
// filter sees every access_log line with the findings the handler made for
// it. It records session-URL 401s that name a user as evidence, and drops
// or holds the API auth failure for a line that evidence explains now or
// may explain once the session_log is read. Other findings pass through.
func (s *staleSession401s) filter(line string, findings []alert.Finding, now time.Time) []alert.Finding {
req, candidate := checks.ParseStaleSessionRequest(line)
if !candidate {
return findings
}
s.mu.Lock()
defer s.mu.Unlock()
if req.User != "-" {
key := sessionEvidenceKey{ip: req.IP, name: req.User, at: req.At.Unix()}
if s.rejected[key] == nil {
s.rejected[key] = make(map[string]observedExplanation)
}
r := observedExplanation{seen: now, expires: now.Add(staleSessionRejectionRetention)}
rememberExplanation(s.rejected[key], req.Token, r)
window := int64(checks.StaleSessionMatchWindow / time.Second)
for at := key.at - window; at <= key.at+window; at++ {
dKey := sessionEvidenceKey{ip: req.IP, name: req.User, at: at}
if d, ok := s.denials[dKey]; ok && d.expires.After(now) {
s.explainTokenLocked(req.IP, req.Token, at, d, r)
}
}
}
out := findings[:0:0]
for _, f := range findings {
if f.Check != "api_auth_failure_realtime" {
out = append(out, f)
continue
}
if s.explainsLocked(req, now, now) {
continue
}
if f.Timestamp.IsZero() {
f.Timestamp = now
}
s.held = append(s.held, heldAPI401{req: req, finding: f, due: now.Add(staleSession401Hold)})
}
return out
}
// observeSessionLine joins a denial with matching user-named requests.
// Held requests are resolved in batches by due, never rescanned per line.
func (s *staleSession401s) observeSessionLine(line string, now time.Time) {
d, ok := checks.ParseSessionTokenDenial(line)
if !ok {
return
}
s.mu.Lock()
defer s.mu.Unlock()
key := sessionEvidenceKey{ip: d.IP, name: d.Account, at: d.At.Unix()}
observed := observedExplanation{seen: now, expires: now.Add(staleSessionDenialRetention)}
rememberExplanation(s.denials, key, observed)
window := int64(checks.StaleSessionMatchWindow / time.Second)
for at := key.at - window; at <= key.at+window; at++ {
for token, r := range s.rejected[sessionEvidenceKey{ip: d.IP, name: d.Account, at: at}] {
if r.expires.After(now) {
s.explainTokenLocked(d.IP, token, key.at, observed, r)
}
}
}
}
// due returns the held findings nothing explained within their hold and
// forgets evidence past retention.
func (s *staleSession401s) due(now time.Time) []alert.Finding {
s.mu.Lock()
defer s.mu.Unlock()
var out []alert.Finding
kept := s.held[:0]
for _, h := range s.held {
before := now
if !now.Before(h.due) {
// Evidence arriving at or after the deadline cannot retract a
// finding, even when the flush goroutine has not run yet.
before = h.due.Add(-time.Nanosecond)
}
if s.explainsLocked(h.req, h.due.Add(-staleSession401Hold), before) {
continue
}
if now.Before(h.due) {
kept = append(kept, h)
continue
}
out = append(out, h.finding)
}
clear(s.held[len(kept):])
s.held = kept
if len(s.held) == 0 {
s.held = nil
} else if cap(s.held) > 2*len(s.held) {
s.held = append([]heldAPI401(nil), s.held...)
}
pruneExplanations(s.denials, now)
pruneExplanations(s.explained, now)
for key, tokens := range s.rejected {
pruneExplanations(tokens, now)
if len(tokens) == 0 {
delete(s.rejected, key)
}
}
return out
}
func (s *staleSession401s) explainTokenLocked(ip, token string, at int64, d, r observedExplanation) {
joined := observedExplanation{seen: d.seen, expires: d.expires}
if r.seen.After(joined.seen) {
joined.seen = r.seen
}
if r.expires.Before(joined.expires) {
joined.expires = r.expires
}
rememberExplanation(s.explained, sessionEvidenceKey{ip: ip, name: token, at: at}, joined)
}
func (s *staleSession401s) explainsLocked(req checks.StaleSessionRequest, start, before time.Time) bool {
evidence, name := s.denials, req.User
if req.User == "-" {
evidence, name = s.explained, req.Token
}
at := req.At.Unix()
window := int64(checks.StaleSessionMatchWindow / time.Second)
for second := at - window; second <= at+window; second++ {
e, ok := evidence[sessionEvidenceKey{ip: req.IP, name: name, at: second}]
if ok && !e.seen.After(before) && e.expires.After(start) {
return true
}
}
return false
}
// Repeated lines in one logged second need one piece of evidence, rather
// than one allocation per request. Preserve when it first became usable.
func rememberExplanation[K comparable](entries map[K]observedExplanation, key K, observed observedExplanation) {
if old, ok := entries[key]; ok && old.expires.After(observed.seen) {
if old.seen.Before(observed.seen) {
observed.seen = old.seen
}
if old.expires.After(observed.expires) {
observed.expires = old.expires
}
}
entries[key] = observed
}
func pruneExplanations[K comparable](entries map[K]observedExplanation, now time.Time) {
for key, e := range entries {
if !e.expires.After(now) {
delete(entries, key)
}
}
}
// cpanelSessionLogHandler feeds session_log to the password hijack detector,
// the stale-session correlation and the known-customer set before the
// regular session handling.
func (d *Daemon) cpanelSessionLogHandler(line string, cfg *config.Config) []alert.Finding {
ParseSessionLineForHijack(line, d.hijackDetector)
d.staleSession401.observeSessionLine(line, time.Now())
if app, ip, account, ok := parseSessionNew(line); ok {
kind := knownCustomerCpanel
if app == "webmaild" {
kind = knownCustomerWebmail
}
d.knownCustomers.Note(ip, kind, account, customerLoginTime(line, time.Now()), serviceLog)
}
return parseSessionLogLine(line, cfg)
}
// cpanelAccessLogHandler holds back the API auth failures a stale browser
// session may explain.
func (d *Daemon) cpanelAccessLogHandler(line string, cfg *config.Config) []alert.Finding {
return d.cpanelAccessLogObservedHandler(line, alert.Observation{}, cfg)
}
// cpanelAccessLogObservedHandler stamps the line's observation before the
// hold, so a held finding still names its line when it is emitted.
func (d *Daemon) cpanelAccessLogObservedHandler(line string, obs alert.Observation, cfg *config.Config) []alert.Finding {
findings := parseAccessLogLineEnhanced(line, cfg)
for i := range findings {
if findings[i].Observation == (alert.Observation{}) {
findings[i].Observation = obs
}
}
return d.staleSession401.filter(line, findings, time.Now())
}
// Keep the same queue contract as LogWatcher: backpressure is counted and
// logged without stopping expiry of attacker-controlled correlation state.
func (d *Daemon) emitDueStaleSession401(now time.Time) {
dropped := 0
for _, f := range d.staleSession401.due(now) {
if !alert.TryEnqueue(d.alertCh, f) {
dropped++
}
}
if dropped > 0 {
fmt.Fprintf(os.Stderr, "[%s] Warning: alert channel full, dropped %d held cPanel API findings\n", ts(), dropped)
}
}
func (d *Daemon) flushStaleSession401() {
defer d.wg.Done()
ticker := time.NewTicker(logWatcherPollInterval)
defer ticker.Stop()
for {
select {
case <-d.stopCh:
return
case <-ticker.C:
// A queued tick can be old after a large batch was processed.
d.emitDueStaleSession401(time.Now())
}
}
}
package daemon
import (
"fmt"
"net/netip"
"sort"
"sync"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/stalwartlog"
)
// stalwartAuth feeds one Stalwart login event to the tracker for its
// listener's protocol. SMTP, IMAP, POP3 and ManageSieve listeners log the
// connecting peer, so their events reach the trackers that may block. An
// HTTP listener (JMAP, webmail) can log a forwarded address: its failures
// only alert, and its successes are not evidence of anything.
func (d *Daemon) stalwartAuth(ev stalwartlog.Event, s stalwartlog.Session, obs alert.Observation, cfg *config.Config) []alert.Finding {
ip := s.RemoteIP.String()
if isInfraIPDaemon(ip, cfg.InfraIPs) || isPrivateOrLoopback(ip) {
return nil
}
account := ev.Fields["accountName"]
success := ev.Name == "auth.success"
if success && account == "" {
return nil
}
switch proto := d.stalwart.ports.Of(s.LocalPort); proto {
case stalwartlog.ProtoSMTP:
if d.smtpAuthTracker == nil {
return nil
}
if success {
return d.smtpAuthTracker.recordSuccessAt(ip, account, ev.Time, serviceLog)
}
return d.smtpAuthTracker.RecordObserved(ip, account, obs)
case stalwartlog.ProtoIMAP, stalwartlog.ProtoPOP3, stalwartlog.ProtoSieve:
if d.mailAuthTracker == nil {
return nil
}
if success {
kind := knownCustomerIMAP
if proto == stalwartlog.ProtoPOP3 {
kind = knownCustomerPOP
}
return d.mailAuthTracker.recordSuccessAt(ip, account, ev.Time, kind)
}
return d.mailAuthTracker.RecordObserved(ip, account, obs)
case stalwartlog.ProtoHTTP:
if success {
return nil
}
return d.stalwart.webmail.record(ip, ev.Time, cfg)
}
return nil
}
// stalwartWebmailMaxTracked bounds the tracker when no mail brute-force cap
// is configured.
const stalwartWebmailMaxTracked = 20000
// stalwartWebmailTracker counts failed logins on Stalwart's HTTP listener per
// address, with the mail brute-force threshold, window and suppression. Its
// findings carry no attribution and have no block policy.
type stalwartWebmailTracker struct {
mu sync.Mutex
ips map[string]*stalwartWebmailEntry
}
type stalwartWebmailEntry struct {
times []time.Time
lastSeen time.Time
quietUntil time.Time
}
func newStalwartWebmailTracker() *stalwartWebmailTracker {
return &stalwartWebmailTracker{ips: map[string]*stalwartWebmailEntry{}}
}
func (t *stalwartWebmailTracker) size() int {
t.mu.Lock()
defer t.mu.Unlock()
return len(t.ips)
}
// record counts one failure from ip at now and returns a finding when the
// address reaches the threshold within the window.
func (t *stalwartWebmailTracker) record(ip string, now time.Time, cfg *config.Config) []alert.Finding {
th := cfg.Thresholds
window := time.Duration(th.MailBruteForceWindowMin) * time.Minute
if th.MailBruteForceThreshold <= 0 || window <= 0 {
return nil
}
maxTracked := th.MailBruteForceMaxTracked
if maxTracked <= 0 {
maxTracked = stalwartWebmailMaxTracked
}
t.mu.Lock()
defer t.mu.Unlock()
e := t.ips[ip]
if e == nil {
t.evictFor(maxTracked)
e = &stalwartWebmailEntry{}
t.ips[ip] = e
}
e.lastSeen = now
if now.Before(e.quietUntil) {
return nil
}
e.times = append(pruneTimes(e.times, now.Add(-window)), now)
n := len(e.times)
if n < th.MailBruteForceThreshold {
return nil
}
e.times = nil
e.quietUntil = now.Add(time.Duration(th.MailBruteForceSuppressMin) * time.Minute)
return []alert.Finding{{
Severity: alert.High,
Check: "stalwart_webmail_bruteforce",
Message: fmt.Sprintf("Webmail or JMAP login failures from %s: %d in %d minutes (Stalwart)", ip, n, th.MailBruteForceWindowMin),
Details: "Stalwart's HTTP listener can log an address taken from a forwarded header, " +
"so CSM reports it and does not block it. Stalwart applies its own throttling and bans to this address.",
Timestamp: now,
SourceIP: ip,
}}
}
// evictFor makes room for one more address. At the cap it drops the least
// recently seen tenth in one pass, so a flood of new addresses costs one sort
// per tenth of the cap rather than a full scan per address.
func (t *stalwartWebmailTracker) evictFor(maxTracked int) {
if len(t.ips) < maxTracked {
return
}
type seen struct {
ip string
at time.Time
}
all := make([]seen, 0, len(t.ips))
for ip, e := range t.ips {
all = append(all, seen{ip, e.lastSeen})
}
sort.Slice(all, func(i, j int) bool { return all[i].at.Before(all[j].at) })
drop := len(all) - maxTracked*9/10
if drop < 1 {
drop = 1
}
for _, s := range all[:min(drop, len(all))] {
delete(t.ips, s.ip)
}
}
// stalwartBanReasons names the Stalwart bans CSM follows. A loiter ban is
// left out because slow legitimate mail servers trigger it.
var stalwartBanReasons = map[string]string{
"security.authentication-ban": "repeated failed logins",
"security.abuse-ban": "abuse",
"security.scan-ban": "scanning",
}
// stalwartBan turns a Stalwart ban on a mail listener into a finding that
// blocks the address at CSM's firewall. A ban on the HTTP listener or an
// unmapped port may name a forwarded address and is not followed.
func (d *Daemon) stalwartBan(ev stalwartlog.Event, s stalwartlog.Session, cfg *config.Config) []alert.Finding {
what, ok := stalwartBanReasons[ev.Name]
if !ok {
return nil
}
proto := d.stalwart.ports.Of(s.LocalPort)
switch proto {
case stalwartlog.ProtoSMTP, stalwartlog.ProtoIMAP, stalwartlog.ProtoPOP3, stalwartlog.ProtoSieve:
default:
return nil
}
ip := s.RemoteIP.String()
if isInfraIPDaemon(ip, cfg.InfraIPs) || isPrivateOrLoopback(ip) {
return nil
}
if reason := ev.Fields["reason"]; reason != "" {
what += ": " + reason
}
f := alert.Finding{
Severity: alert.Critical,
Check: "stalwart_ban",
Message: fmt.Sprintf("Stalwart banned %s on its %s listener (port %d) for %s", ip, proto, s.LocalPort, what),
Details: "CSM blocks the address at its firewall for the auto-response block length. Stalwart's own ban stays in place.",
Timestamp: ev.Time,
SourceIP: ip,
Attribution: admission.AttributionPeer,
}
// A failed-login ban of an address a customer logged in from recently is
// most likely an office network with one stale password, or failures
// another listener attributed to it. Only Critical bans block, so this
// one is reported and not blocked. Scanning and abuse are never a
// customer's own mistake. On a Stalwart host the customer logins come
// from Stalwart's log, which only stalwart and root can write. Stalwart's
// bans expire, so only the first ban within the known-customer window is
// excused: an address banned again keeps failing and is blocked.
if _, known := d.knownCustomers.Lookup(ip); known && ev.Name == "security.authentication-ban" &&
d.stalwart.excused.first(s.RemoteIP, ev.Time) {
f.Severity = alert.High
f.Message += "; not blocked, a customer logged in from this address recently"
f.Details = "A customer logged in successfully from this address within the last 30 days, so CSM reports " +
"Stalwart's ban without blocking the address at its firewall. Stalwart's own ban still refuses its mail connections."
}
return []alert.Finding{f}
}
// excusedBans remembers when a known customer's address last had a
// failed-login ban excused.
type excusedBans struct {
mu sync.Mutex
at map[netip.Addr]time.Time
}
// first reports whether ip has had no ban excused within knownCustomerTTL
// of now, and records now when it has not. It holds at most
// maxKnownCustomers addresses, dropping expired ones first, then the oldest.
func (e *excusedBans) first(ip netip.Addr, now time.Time) bool {
ip = ip.Unmap()
e.mu.Lock()
defer e.mu.Unlock()
if last, ok := e.at[ip]; ok && now.Sub(last) < knownCustomerTTL {
return false
}
if e.at == nil {
e.at = map[netip.Addr]time.Time{}
}
if _, ok := e.at[ip]; !ok && len(e.at) >= maxKnownCustomers {
e.evict(now)
}
e.at[ip] = now
return true
}
// evict drops expired entries and, if that frees nothing, the oldest tenth,
// so a full set costs one sort per tenth of the cap rather than one per ban.
func (e *excusedBans) evict(now time.Time) {
for ip, at := range e.at {
if now.Sub(at) >= knownCustomerTTL {
delete(e.at, ip)
}
}
if len(e.at) < maxKnownCustomers {
return
}
type entry struct {
ip netip.Addr
at time.Time
}
all := make([]entry, 0, len(e.at))
for ip, at := range e.at {
all = append(all, entry{ip, at})
}
sort.Slice(all, func(i, j int) bool { return all[i].at.Before(all[j].at) })
for _, x := range all[:len(all)-maxKnownCustomers*9/10] {
delete(e.at, x.ip)
}
}
func (e *excusedBans) size() int {
e.mu.Lock()
defer e.mu.Unlock()
return len(e.at)
}
package daemon
import (
"context"
"os"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/stalwartapi"
)
// stalwartAPI is the part of the Stalwart client the daemon acts through.
type stalwartAPI interface {
AccountByAddress(ctx context.Context, address string) (stalwartapi.Account, bool, error)
Account(ctx context.Context, id string) (stalwartapi.Account, bool, error)
DisableSignIn(ctx context.Context, a stalwartapi.Account) error
EnableSignIn(ctx context.Context, id string) error
BlockedIPs(ctx context.Context, address string) ([]stalwartapi.BlockedIP, error)
DeleteBlockedIPs(ctx context.Context, ids []string) error
ReloadBlockedIPs(ctx context.Context) error
SpamSettings(ctx context.Context) (stalwartapi.SpamSettings, error)
SpamFilterStageOff(ctx context.Context) (bool, error)
SpamRule(ctx context.Context, name string) (stalwartapi.SpamRule, bool, error)
LookupKeys(ctx context.Context, namespace string) ([]stalwartapi.LookupKey, error)
CreateLookupKeys(ctx context.Context, namespace string, keys []string) error
DeleteLookupKeys(ctx context.Context, ids []string) error
ReloadLookupStores(ctx context.Context) error
}
// newStalwartKeyClient is a var so tests can see what the client is built
// with.
var newStalwartKeyClient = stalwartapi.NewKey
// stalwartClient reads the key file on every call so a rotated key takes
// effect without a restart. var so tests can replace the client.
var stalwartClient = func(cfg *config.Config) (stalwartAPI, error) {
key, err := stalwartapi.ReadKeyFile(cfg.Stalwart.APIKeyFile)
if err != nil {
return nil, err
}
return newStalwartKeyClient(cfg.Stalwart.APIURL, key, stalwartapi.CheckPeer), nil
}
// stalwartKeyPresent reports whether the key file exists, without reading
// it. A symlink counts so that ReadKeyFile reports why it is refused.
func stalwartKeyPresent(cfg *config.Config) bool {
_, err := os.Lstat(cfg.Stalwart.APIKeyFile)
return err == nil
}
// stalwartMethodForbidden also checks reload errors joined to an earlier write failure.
func stalwartMethodForbidden(err error) bool {
switch e := err.(type) {
case *stalwartapi.MethodError:
return e.Type == "forbidden"
case interface{ Unwrap() []error }:
for _, child := range e.Unwrap() {
if stalwartMethodForbidden(child) {
return true
}
}
case interface{ Unwrap() error }:
return stalwartMethodForbidden(e.Unwrap())
}
return false
}
package daemon
import (
"fmt"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/platform"
)
const (
// stalwartHealthTick is how often the Stalwart log health is judged.
stalwartHealthTick = 5 * time.Minute
// stalwartHealthSlots holds one hour of ticks.
stalwartHealthSlots = 12
// stalwartUnparsedMinLines is how many lines an hour must deliver, none
// of them in Stalwart's layout, before CSM calls the layout changed.
stalwartUnparsedMinLines = 50
// stalwartStallAfter is how long a running Stalwart may leave its log
// unchanged before CSM reports it. Scanners alone keep a public mail
// server's log moving.
stalwartStallAfter = 6 * time.Hour
// stalwartUnattributedMin is how many SMTP sends an hour may go without
// a login to name their sender before CSM reports it. Only sessions in
// progress when CSM started, or a flood of logins evicting remembered
// connections, leave a send unnamed.
stalwartUnattributedMin = 20
)
// stalwartServiceActive reports whether Stalwart is running. var so tests
// can stand in for systemd.
var stalwartServiceActive = func() bool {
return platform.ServiceActive("stalwart") || platform.ServiceActive("stalwart-mail")
}
// stalwartHealth is the last hour of line counts, touched only by the
// health ticker.
type stalwartHealth struct {
lastRead, lastParsed, lastUnattributed int64
read, parsed, unattributed [stalwartHealthSlots]int64
next int
}
// health judges one tick. It reports a log whose last hour held many lines
// and none in Stalwart's layout (a layout change after an upgrade), a burst
// of SMTP sends with no login to name their sender, and a log that stopped
// growing while Stalwart runs.
func (s *stalwartState) health(now time.Time, serviceActive bool) []alert.Finding {
h := &s.hourly
read, parsed, unattributed := s.stats.read.Load(), s.stats.parsed.Load(), s.stats.unattributed.Load()
h.read[h.next], h.parsed[h.next] = read-h.lastRead, parsed-h.lastParsed
h.unattributed[h.next] = unattributed - h.lastUnattributed
h.lastRead, h.lastParsed, h.lastUnattributed = read, parsed, unattributed
h.next = (h.next + 1) % stalwartHealthSlots
var hourRead, hourParsed, hourUnattributed int64
for i := range h.read {
hourRead += h.read[i]
hourParsed += h.parsed[i]
hourUnattributed += h.unattributed[i]
}
var out []alert.Finding
if hourRead >= stalwartUnparsedMinLines && hourParsed == 0 {
out = append(out, alert.Finding{
Severity: alert.Warning,
Check: "stalwart_log_degraded",
Message: fmt.Sprintf("Stalwart log: %d lines in the last hour, none in the expected format; Stalwart may have changed its log layout, so CSM sees no mail logins or sends", hourRead),
DedupKey: "stalwart_log_degraded:unparsed",
Timestamp: now,
})
}
if hourUnattributed >= stalwartUnattributedMin {
out = append(out, alert.Finding{
Severity: alert.Warning,
Check: "stalwart_log_degraded",
Message: fmt.Sprintf("Stalwart log: %d authenticated SMTP sends in the last hour could not be tied to their login, so CSM cannot judge who sent them", hourUnattributed),
DedupKey: "stalwart_log_degraded:unattributed",
Timestamp: now,
})
}
if last := time.Unix(s.stats.lastLine.Load(), 0); serviceActive && now.Sub(last) >= stalwartStallAfter {
out = append(out, alert.Finding{
Severity: alert.Warning,
Check: "stalwart_log_degraded",
Message: "Stalwart log has not grown for 6 hours while the stalwart service is running, so CSM sees no mail logins or sends",
DedupKey: "stalwart_log_degraded:stalled",
Timestamp: now,
})
}
return out
}
package daemon
import (
"context"
"errors"
"fmt"
"os"
"strings"
"time"
"unicode"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/ipmap"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/stalwartapi"
"github.com/pidginhost/csm/internal/stalwartsetup"
)
const (
stalwartCheckAddressList = "stalwart_address_list"
stalwartCheckFilterOff = "stalwart_spam_filter_off"
stalwartListTickLimit = 5 * time.Minute
)
// stalwartListState retains confirmed filter state and unfinished reloads between ticks.
type stalwartListState struct {
m *ipmap.Map
filterOff, ruleOff bool
reloadPending bool
}
// startStalwartAddressList syncs at start and every 10 minutes.
func (d *Daemon) startStalwartAddressList() {
ctx := d.scanContext()
d.wg.Add(1)
obs.Go("stalwart-address-list", func() {
defer d.wg.Done()
ticker := time.NewTicker(antispamIPMapInterval)
defer ticker.Stop()
for {
tickCtx, cancel := context.WithTimeout(ctx, stalwartListTickLimit)
d.stalwartListTick(tickCtx, time.Now())
cancel()
select {
case <-d.stopCh:
return
case <-ticker.C:
}
}
})
}
// stalwartListTick syncs the address list and settles its findings.
func (d *Daemon) stalwartListTick(ctx context.Context, now time.Time) []alert.Finding {
cfg := d.currentCfg()
m := ipmap.Build(ipmapSources(now, d.antispamInfraIPs(cfg)))
st := stalwartListState{m: &m}
if s := d.stalwart; s != nil {
if previous := s.list.Load(); previous != nil {
st.filterOff, st.ruleOff, st.reloadPending = previous.filterOff, previous.ruleOff, previous.reloadPending
}
}
// Store the map before reading Stalwart so a hung read leaves evidence guarded.
d.storeStalwartList(st)
api, findings, filterKnown := stalwartListRead(ctx, cfg, &st, now)
if filterKnown && st.filterOff {
findings = append(findings, alert.Finding{
Severity: alert.Warning, Check: stalwartCheckFilterOff, DedupKey: "filter_off",
Message: "Stalwart's spam filter is off",
Details: "Stalwart does not score incoming mail, so CSM's address list adds no score. Switch the spam filter on in Stalwart.",
Timestamp: now,
})
}
d.storeStalwartList(st)
// Observe mode reads but leaves Stalwart's lookup lists as they are.
if api != nil && !cfg.ObserveMode() {
want := m.StalwartKeys()
if st.ruleOff {
want = nil
}
var err error
st.reloadPending, err = syncStalwartKeys(ctx, api, want, st.reloadPending)
if err != nil {
findings = append(findings, stalwartListFinding(err, now))
}
d.storeStalwartList(st)
}
// A timed-out tick still reports; only shutdown leaves findings as they are.
if !errors.Is(ctx.Err(), context.Canceled) {
var owned []string
// Observe mode cannot confirm recovery from a failed reload.
if api == nil || !cfg.ObserveMode() || !st.reloadPending {
owned = append(owned, stalwartCheckAddressList)
}
if filterKnown {
owned = append(owned, stalwartCheckFilterOff)
}
d.store.RearmAbsentDedupFindings(owned, findings)
d.processScanFindings(cfg, findings, owned, "stalwart")
}
return findings
}
func (d *Daemon) storeStalwartList(st stalwartListState) {
if s := d.stalwart; s != nil {
s.list.Store(&st)
}
}
// stalwartListRead returns the client only after reading the rule permits a sync.
func stalwartListRead(ctx context.Context, cfg *config.Config, st *stalwartListState, now time.Time) (stalwartAPI, []alert.Finding, bool) {
// Without a key, setup and doctor report what is missing.
if _, err := os.Lstat(cfg.Stalwart.APIKeyFile); err != nil {
if errors.Is(err, os.ErrNotExist) {
st.filterOff, st.ruleOff = false, false
return nil, nil, true
}
return nil, []alert.Finding{stalwartListFinding(err, now)}, false
}
api, err := stalwartClient(cfg)
if err != nil {
return nil, []alert.Finding{stalwartListFinding(err, now)}, false
}
off, known := stalwartFilterOff(ctx, api)
if known {
st.filterOff = off
}
readCtx, cancel := context.WithTimeout(ctx, stalwartCallTimeout)
rule, found, err := api.SpamRule(readCtx, stalwartsetup.AddressListRule)
cancel()
if err != nil {
return nil, []alert.Finding{stalwartListFinding(err, now)}, known
}
st.ruleOff = !found || rule.Type != "Any" || !rule.Enabled
return api, nil, known
}
// Either switch proves the filter off; both must be readable to prove it on.
func stalwartFilterOff(ctx context.Context, api stalwartAPI) (off, known bool) {
readCtx, cancel := context.WithTimeout(ctx, stalwartCallTimeout)
s, settingsErr := api.SpamSettings(readCtx)
cancel()
if settingsErr == nil && !s.Enabled {
return true, true
}
readCtx, cancel = context.WithTimeout(ctx, stalwartCallTimeout)
off, stageErr := api.SpamFilterStageOff(readCtx)
cancel()
if stageErr == nil && off {
return true, true
}
return false, settingsErr == nil && stageErr == nil
}
func stalwartListFinding(err error, now time.Time) alert.Finding {
msg := "CSM cannot keep its address list in Stalwart up to date"
if errors.Is(err, stalwartapi.ErrForbidden) || stalwartMethodForbidden(err) {
msg = "Stalwart refused CSM's key access to the address list; run csm stalwart setup"
}
// Bound server text before persisting it or sending it to alert sinks.
details := strings.ToValidUTF8(truncateStr(err.Error(), 1024), "")
details = strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return ' '
}
return r
}, details)
return alert.Finding{
Severity: alert.Warning, Check: stalwartCheckAddressList, DedupKey: "address_list",
Message: msg, Details: details, Timestamp: now,
}
}
// syncStalwartKeys owns each namespace and returns whether a reload still needs retrying.
func syncStalwartKeys(ctx context.Context, api stalwartAPI, want map[string][]string, reloadPending bool) (pending bool, err error) {
changed := false
added, removed := 0, 0
for _, ns := range ipmap.StalwartNamespaces() {
var have []stalwartapi.LookupKey
callCtx, cancel := context.WithTimeout(ctx, stalwartCallTimeout)
have, err = api.LookupKeys(callCtx, ns)
cancel()
if err != nil {
break
}
wanted := make(map[string]bool, len(want[ns]))
for _, k := range want[ns] {
wanted[k] = true
}
var stale, missing []string
for _, k := range have {
if wanted[k.Key] && k.Glob == strings.Contains(k.Key, "*") {
delete(wanted, k.Key)
} else {
stale = append(stale, k.ID)
}
}
for _, k := range want[ns] {
if wanted[k] {
missing = append(missing, k)
delete(wanted, k)
}
}
// A failed write may have applied in part, so it counts as a change.
if len(stale) > 0 {
changed = true
callCtx, cancel := context.WithTimeout(ctx, stalwartCallTimeout)
err = api.DeleteLookupKeys(callCtx, stale)
cancel()
if err != nil {
break
}
removed += len(stale)
}
if len(missing) > 0 {
changed = true
callCtx, cancel := context.WithTimeout(ctx, stalwartCallTimeout)
err = api.CreateLookupKeys(callCtx, ns, missing)
cancel()
if err != nil {
break
}
added += len(missing)
}
}
if !changed && !reloadPending {
return false, err
}
if added+removed > 0 {
csmlog.Info("stalwart address list updated", "added", added, "removed", removed)
}
// Reload even after a failed write so the loaded list matches the stored keys.
var reloadErr error
for range 2 {
reloadCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
reloadErr = api.ReloadLookupStores(reloadCtx)
cancel()
if reloadErr == nil {
return false, err
}
}
return true, errors.Join(err, fmt.Errorf("reloading Stalwart's lookup lists: %w", reloadErr))
}
package daemon
import (
"context"
"fmt"
"os"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/maillog"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/stalwartlog"
)
// stalwartLogWatcher is the watcher name the Stalwart log reader reports.
const stalwartLogWatcher = "stalwart_log"
// stalwartLogStats counts what the reader delivered, for the health check.
type stalwartLogStats struct {
read atomic.Int64 // lines delivered
parsed atomic.Int64 // lines in Stalwart's layout
lastLine atomic.Int64 // unix seconds of the last delivered line, or the start
unattributed atomic.Int64 // SMTP sends on a connection with no remembered login
}
// stalwartState is what the Stalwart log handler keeps between lines.
type stalwartState struct {
ports stalwartlog.Ports
stats stalwartLogStats
webmail *stalwartWebmailTracker
sessions *stalwartSessions
hourly stalwartHealth
excused excusedBans
verdicts *stalwartVerdicts
// list is the last address list tick's state, nil before the first.
list atomic.Pointer[stalwartListState]
}
func newStalwartState(ports stalwartlog.Ports, start time.Time) *stalwartState {
s := &stalwartState{ports: ports, webmail: newStalwartWebmailTracker(), sessions: newStalwartSessions(), verdicts: newStalwartVerdicts()}
s.stats.lastLine.Store(start.Unix())
return s
}
// useStalwartReader reports whether this host reads the Stalwart log instead
// of the Postfix/Dovecot mail log. A Stalwart host has neither of those logs.
func useStalwartReader(mta platform.MTAKind) bool { return mta == platform.MTAStalwart }
// startStalwartLogReader follows the newest Stalwart log file and feeds each
// line to handleStalwartLine.
func (d *Daemon) startStalwartLogReader(cfg *config.Config) {
ports, err := stalwartlog.ParsePorts(cfg.Stalwart.Ports)
if err != nil {
// Validation already refused bad entries; keep the standard ports.
csmlog.Warn("stalwart port mapping ignored", "err", err)
ports, _ = stalwartlog.ParsePorts(nil)
}
d.stalwart = newStalwartState(ports, time.Now())
dir, prefix := cfg.Stalwart.LogDir, cfg.Stalwart.LogPrefix
d.MarkWatcher(stalwartLogWatcher, false)
queue := maillog.NewQueue()
d.registerQueueSource("stalwart", queue)
ctx, cancel := context.WithCancel(context.Background())
d.wg.Add(3)
obs.Go("stalwart-log-health", func() {
defer d.wg.Done()
ticker := time.NewTicker(stalwartHealthTick)
defer ticker.Stop()
verdictTicker := time.NewTicker(stalwartVerdictTick)
defer verdictTicker.Stop()
for {
select {
case <-d.stopCh:
return
case now := <-verdictTicker.C:
d.sweepStalwartVerdicts(now, d.currentCfg())
case now := <-ticker.C:
// A missing log already reports itself as unavailable.
active := d.WatcherStatuses()[stalwartLogWatcher] && stalwartServiceActive()
for _, f := range d.stalwart.health(now, active) {
if !alert.TryEnqueue(d.alertCh, f) {
atomic.AddInt64(&d.droppedAlerts, 1)
}
}
}
}
})
obs.Go("stalwart-log-stop", func() {
defer d.wg.Done()
select {
case <-d.stopCh:
case <-ctx.Done():
}
cancel()
})
obs.Go("stalwart-log-supervisor", func() {
defer d.wg.Done()
defer cancel()
defer d.MarkWatcher(stalwartLogWatcher, false)
maillog.Supervise(ctx, func() (maillog.Reader, error) {
return maillog.NewResolvedFileReader(func() (string, error) { return stalwartlog.NewestFile(dir, prefix, time.Now()) }, queue)
}, func(err error) {
if err != nil {
csmlog.Warn("stalwart log unavailable; retrying", "err", err)
d.handleStalwartLogGone(err)
} else {
d.handleStalwartLogRestored()
}
}, func(line maillog.Line) bool {
return d.dispatchLogLine(checks.ProducerStalwartLog, line, d.handleStalwartLine)
})
})
}
// handleStalwartLine parses one Stalwart log line. Lines outside Stalwart's
// layout and events CSM does not use produce nothing.
func (d *Daemon) handleStalwartLine(line string, obs alert.Observation, cfg *config.Config) []alert.Finding {
s := d.stalwart
if s == nil {
return nil
}
s.stats.read.Add(1)
s.stats.lastLine.Store(time.Now().Unix())
ev, ok := stalwartlog.Parse(line)
if !ok {
return nil
}
s.stats.parsed.Add(1)
d.sweepStalwartVerdicts(time.Now(), cfg)
switch ev.Name {
case "auth.failed", "auth.success":
session, ok := ev.Session()
if !ok {
return nil
}
// A login names its connection's sends, a loopback one included:
// sites submit locally, often on a port CSM does not map.
if account := ev.Fields["accountName"]; ev.Name == "auth.success" && account != "" && stalwartCarriesSends(s.ports.Of(session.LocalPort)) {
s.sessions.remember(stalwartSessionKey(session), account, ev.Time)
}
return d.stalwartAuth(ev, session, obs, cfg)
case "queue.authenticated-message-queued":
s.verdicts.invalidate(ev.Fields["queueId"])
session, ok := ev.Session()
if !ok {
return nil
}
return d.stalwartSend(ev, session, cfg)
case "security.authentication-ban", "security.abuse-ban", "security.scan-ban":
session, ok := ev.Session()
if !ok {
return nil
}
return d.stalwartBan(ev, session, cfg)
case "queue.message-queued":
if session, ok := ev.Session(); ok {
d.stalwartQueued(ev, session)
} else {
s.verdicts.invalidate(ev.Fields["queueId"])
}
case "message-ingest.ham", "message-ingest.spam":
if id := ev.Fields["queueId"]; id != "" {
s.verdicts.ingested(id, ev.Name == "message-ingest.spam")
}
case "spam.classify":
if session, ok := ev.Session(); ok {
d.stalwartClassify(ev, session, cfg)
}
}
return nil
}
func (d *Daemon) handleStalwartLogGone(err error) {
d.MarkWatcher(stalwartLogWatcher, false)
finding := alert.Finding{
Severity: alert.Warning,
Check: "mail_log_source_unavailable",
Message: fmt.Sprintf("Stalwart log unavailable: %v; mail login and send detection paused while CSM retries", err),
Timestamp: time.Now(),
}
select {
case <-d.stopCh:
return
default:
}
if !alert.TryEnqueue(d.alertCh, finding) {
atomic.AddInt64(&d.droppedAlerts, 1)
fmt.Fprintf(os.Stderr, "[%s] alert channel full, dropping stalwart log source finding\n", ts())
}
}
func (d *Daemon) handleStalwartLogRestored() {
d.MarkWatcher(stalwartLogWatcher, true)
}
package daemon
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"strings"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/store"
)
// stalwartCallTimeout bounds each backend call: suspends run on the log
// reader's path and status reads on a web UI request.
const stalwartCallTimeout = 10 * time.Second
// detectMTA is a var so tests can stand in for the host's mail server.
var detectMTA = platform.DetectMTA
// selectMailBackend picks the mail server CSM suspends mailboxes on. It runs
// once at daemon start, before anything reads the backend.
func selectMailBackend(cfg func() *config.Config) {
if platform.Detect().IsCPanel() || detectMTA() != platform.MTAStalwart {
return
}
activeMailBackend = stalwartMailBackend{cfg: cfg}
// The key file is checked on each use, so running setup takes effect
// without a restart.
hostHasMailBackend = func() bool { return stalwartKeyPresent(cfg()) }
}
// stalwartMailBackend suspends a Stalwart account by denying it sign-in,
// which also ends its SMTP authentication.
type stalwartMailBackend struct{ cfg func() *config.Config }
func (stalwartMailBackend) name() string { return "stalwart" }
// Account IDs, unlike login addresses, survive alias and domain changes and
// must not be case-folded. Every alias shares the same response lock and row.
func stalwartSuspensionID(ref string) string {
sum := sha256.Sum256([]byte("stalwart\x00" + ref))
return hex.EncodeToString(sum[:8])
}
func (b stalwartMailBackend) resolve(subject string) (string, error) {
if !strings.Contains(subject, "@") {
return "", errors.New("CSM does not suspend Stalwart account names without a domain; change its password in Stalwart")
}
client, err := stalwartClient(b.cfg())
if err != nil {
return "", err
}
ctx, cancel := context.WithTimeout(context.Background(), stalwartCallTimeout)
defer cancel()
acct, found, err := client.AccountByAddress(ctx, subject)
if err != nil {
return "", err
}
if !found {
return "", fmt.Errorf("no Stalwart account has the address %s; change its password in Stalwart", subject)
}
return acct.ID, nil
}
func (b stalwartMailBackend) suspend(c mailboxCompromise) (store.MailboxSuspension, error) {
client, err := stalwartClient(b.cfg())
if err != nil {
return store.MailboxSuspension{}, err
}
ctx, cancel := context.WithTimeout(context.Background(), stalwartCallTimeout)
defer cancel()
// DisableSignIn needs the account as read now: a stale permission type
// would replace the account's whole permission list.
acct, found, err := client.Account(ctx, c.Ref)
switch {
case err != nil:
return store.MailboxSuspension{}, err
case !found:
return store.MailboxSuspension{}, fmt.Errorf("no Stalwart account has the address %s; change its password in Stalwart", c.Subject)
case acct.Admin:
return store.MailboxSuspension{}, fmt.Errorf("%s is a Stalwart admin account, which CSM does not suspend; change its password in Stalwart", c.Subject)
case acct.SignInDisabled:
if c.Previous.Ref != acct.ID {
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: %s already has sign-ins stopped outside CSM; not recorded\n",
time.Now().Format("2006-01-02 15:04:05"), c.Subject)
}
return store.MailboxSuspension{}, errMailAlreadyRestricted
}
err = client.DisableSignIn(ctx, acct)
if stalwartMethodForbidden(err) {
// Stalwart refuses a key that lacks one of the account's own
// permissions, as with a custom admin role.
err = fmt.Errorf("the CSM key is not allowed to change %s; change its password in Stalwart", c.Subject)
}
rec := actionlog.Record{
Op: "respond.suspend_mailbox",
Action: store.AppliedDisableSignIn,
Target: c.Subject,
Account: acct.ID,
Reason: c.Reason,
Result: actionlog.Applied,
}
if err != nil {
rec.Result, rec.Error = actionlog.Failed, err.Error()
}
actionlog.Write(rec)
if err != nil {
return store.MailboxSuspension{}, err
}
return store.MailboxSuspension{Kind: store.SuspensionKindMailbox, Ref: acct.ID, Applied: []string{store.AppliedDisableSignIn}}, nil
}
// release enables sign-in on the account CSM suspended, found by its id: the
// address may name another account by now.
func (b stalwartMailBackend) release(rec store.MailboxSuspension, actor string) error {
client, err := stalwartClient(b.cfg())
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), stalwartCallTimeout)
defer cancel()
err = client.EnableSignIn(ctx, rec.Ref)
entry := actionlog.Record{
Op: "operate.release_mailbox",
Action: "enable_signin",
Actor: actionlog.WebUI,
ActorDetail: actor,
Target: rec.Subject,
Account: rec.Ref,
Result: actionlog.Applied,
}
if err != nil {
entry.Result, entry.Error = actionlog.Failed, err.Error()
}
actionlog.Write(entry)
return err
}
// active reads each account's sign-in state. A deleted account is no longer
// suspended; one that cannot be read is left out of the map.
func (b stalwartMailBackend) active(recs []store.MailboxSuspension) (map[string]bool, error) {
client, err := stalwartClient(b.cfg())
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), stalwartCallTimeout)
defer cancel()
live := make(map[string]bool, len(recs))
var errs []error
for _, r := range recs {
acct, found, err := client.Account(ctx, r.Ref)
if err != nil {
errs = append(errs, fmt.Errorf("sign-in state of %s: %w", r.Subject, err))
continue
}
live[r.ID] = found && acct.SignInDisabled
}
return live, errors.Join(errs...)
}
package daemon
import (
"fmt"
"net"
"os"
"sort"
"strconv"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/stalwartlog"
"github.com/pidginhost/csm/internal/store"
)
const (
// stalwartSessionTTL is how long a login names the sends on its
// connection; each send refreshes it.
stalwartSessionTTL = time.Hour
// stalwartSessionMax bounds the connections remembered.
stalwartSessionMax = 4096
)
// What CSM did about a compromised Stalwart mailbox, for staff reading the
// finding.
const (
stalwartSuspendedContext = "CSM stopped this mailbox's sign-ins; release it on the Email page, Suspended tab."
stalwartNoKeyContext = "CSM has no Stalwart API key, so this mailbox stays active; run csm stalwart setup, or change the mailbox password in Stalwart."
stalwartNotSuspendedContext = "CSM did not suspend this mailbox (see the CSM log); change its password in Stalwart if it is still in use."
)
// stalwartPTR is the reverse DNS name of an address. var so tests can stand
// in for DNS.
var stalwartPTR = func(ip string) string {
parsed := net.ParseIP(ip)
if parsed == nil {
return ""
}
return rdnsCache().Lookup(parsed)
}
// stalwartSessions remembers which account logged in on each connection.
// Stalwart logs a queued message with the connection but not the account,
// so the login on the same listener, local port, client address and client
// port names the sender.
type stalwartSessions struct {
mu sync.Mutex
m map[string]stalwartSession
}
type stalwartSession struct {
account string
at time.Time
}
func newStalwartSessions() *stalwartSessions {
return &stalwartSessions{m: map[string]stalwartSession{}}
}
func stalwartSessionKey(s stalwartlog.Session) string {
return s.Listener + "|" + strconv.Itoa(int(s.LocalPort)) + "|" + s.RemoteIP.String() + "|" + strconv.Itoa(int(s.RemotePort))
}
func (s *stalwartSessions) size() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.m)
}
// remember records that account logged in on the connection key at at.
func (s *stalwartSessions) remember(key, account string, at time.Time) {
s.mu.Lock()
defer s.mu.Unlock()
if _, ok := s.m[key]; !ok && len(s.m) >= stalwartSessionMax {
s.evictLocked(at)
}
s.m[key] = stalwartSession{account: account, at: at}
}
// account returns the account logged in on key, refreshing its age.
func (s *stalwartSessions) account(key string, now time.Time) (string, bool) {
s.mu.Lock()
defer s.mu.Unlock()
e, ok := s.m[key]
if !ok || now.Sub(e.at) >= stalwartSessionTTL {
return "", false
}
if now.After(e.at) {
e.at = now
s.m[key] = e
}
return e.account, true
}
// evictLocked drops expired connections, then the oldest tenth if the map
// is still full, so a burst of logins costs one sort per tenth of the cap.
func (s *stalwartSessions) evictLocked(now time.Time) {
for k, e := range s.m {
if now.Sub(e.at) >= stalwartSessionTTL {
delete(s.m, k)
}
}
if len(s.m) < stalwartSessionMax {
return
}
keys := make([]string, 0, len(s.m))
for k := range s.m {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool { return s.m[keys[i]].at.Before(s.m[keys[j]].at) })
for _, k := range keys[:len(keys)-stalwartSessionMax*9/10] {
delete(s.m, k)
}
}
// stalwartCarriesSends reports whether sends on a listener are matched to
// its logins: SMTP, and unmapped ports such as a local submission listener.
// An HTTP connection from the web server's proxy can carry several webmail
// users, and IMAP, POP3 and ManageSieve never send.
func stalwartCarriesSends(p stalwartlog.Protocol) bool {
return p == stalwartlog.ProtoSMTP || p == stalwartlog.ProtoUnknown
}
// stalwartSend feeds one authenticated send to the rate, sender-profile and
// cloud-relay detectors. Only a send on an SMTP listener from a public
// address carries an address signal; a local send or one on an unmapped
// listener counts toward volume and recipients only.
func (d *Daemon) stalwartSend(ev stalwartlog.Event, s stalwartlog.Session, cfg *config.Config) []alert.Finding {
st := d.stalwart
proto := st.ports.Of(s.LocalPort)
if !stalwartCarriesSends(proto) {
return nil
}
smtp := proto == stalwartlog.ProtoSMTP
account, ok := st.sessions.account(stalwartSessionKey(s), ev.Time)
if !ok {
// An SMTP send always follows a login on its own connection; the
// health check reports a burst of sends CSM cannot name.
if smtp {
st.stats.unattributed.Add(1)
}
return nil
}
signalIP := ""
if ip := s.RemoteIP.String(); smtp && !isPrivateOrLoopback(ip) && !isInfraIPDaemon(ip, cfg.InfraIPs) {
signalIP = ip
}
now := time.Now()
findings := checkEmailRate(account, cfg)
for _, f := range senderProfileObserve(cfg, account, signalIP, ev.Lists["to"], now) {
if f.Severity == alert.Critical {
f.Context = respondToStalwartCompromise(cfg, account, "email_compromised_account", "authenticated sends outside the mailbox's own source and recipient history", ev.Time)
} else if !hostHasMailBackend() {
f.Context = stalwartNoKeyContext
}
findings = append(findings, f)
}
if signalIP != "" {
for _, f := range cloudRelayObserve(cfg, account, signalIP, stalwartPTR(signalIP), now) {
f.Context = respondToStalwartCompromise(cfg, account, "email_cloud_relay_abuse", "authenticated sends from several cloud-provider addresses", ev.Time)
findings = append(findings, f)
}
}
return findings
}
// respondToStalwartCompromise responds to a compromised Stalwart account and
// returns the finding's context saying what CSM did. A bare account name has
// no domain to mark, and the envelope sender is the sender's own choice:
// marking its domain would only quiet other mailboxes' rate alerts. The
// finding stands either way.
func respondToStalwartCompromise(cfg *config.Config, account, check, reason string, at time.Time) string {
stopped := false
if extractDomainFromEmail(account) != "" {
stopped = respondToMailboxCompromise(cfg, account, check, reason, at)
} else if hostHasMailBackend() {
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: Stalwart account %s has no domain; not suspended\n",
time.Now().Format("2006-01-02 15:04:05"), account)
}
switch {
case stopped && holdsStalwartSuspension(account):
return stalwartSuspendedContext
case !hostHasMailBackend():
return stalwartNoKeyContext
default:
return stalwartNotSuspendedContext
}
}
// holdsStalwartSuspension reports whether CSM holds an unreleased suspension
// of subject. Sign-ins stopped outside CSM also count as stopped, but leave
// nothing to release.
func holdsStalwartSuspension(subject string) bool {
db := store.Global()
if db == nil {
return false
}
backend, ok := activeMailBackend.(stalwartMailBackend)
if !ok {
return false
}
ref, err := backend.resolve(subject)
if err != nil {
return false
}
rec, ok, err := db.GetMailboxSuspension(stalwartSuspensionID(ref))
return err == nil && ok && rec.ReleasedAt == nil
}
package daemon
import (
"net/netip"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/attackdb"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/ipmap"
"github.com/pidginhost/csm/internal/mailstats"
"github.com/pidginhost/csm/internal/stalwartlog"
)
const (
// stalwartVerdictHold is how long a queued message waits for its
// delivery lines, from when CSM read it, so a backlog read in a burst
// still joins.
stalwartVerdictHold = 2 * time.Minute
// stalwartVerdictMax bounds the messages held.
stalwartVerdictMax = 8192
// stalwartVerdictTick is how often held messages are booked while the
// log is quiet.
stalwartVerdictTick = 30 * time.Second
)
// stalwartVerdict is what Stalwart's spam filter did with one message.
type stalwartVerdict struct {
ip netip.Addr
at time.Time
junk bool
unmatched bool
}
// stalwartVerdicts joins a queued message, which names the sending address,
// to its delivery lines, which say whether the filter filed it to Junk.
// Both carry Stalwart's queue id and nothing else in common.
type stalwartVerdicts struct {
mu sync.Mutex
m map[string]*stalwartHeld
order []*stalwartHeld // arrival order, so the oldest are at the front
}
type stalwartHeld struct {
id string
session stalwartlog.Session
read time.Time
ingested bool
v stalwartVerdict
}
func newStalwartVerdicts() *stalwartVerdicts {
return &stalwartVerdicts{m: map[string]*stalwartHeld{}}
}
// queued holds the message id from s, logged at at and read at now. It
// reports false only when the memory is full. Repeated lines keep their
// original expiry; conflicting identities make the join unusable.
func (v *stalwartVerdicts) queued(id string, s stalwartlog.Session, at, now time.Time) bool {
v.mu.Lock()
defer v.mu.Unlock()
if h, ok := v.m[id]; ok {
if h.session != s || !h.v.at.Equal(at) {
h.v.unmatched = true
}
return true
}
if len(v.m) >= stalwartVerdictMax {
return false
}
// The id is a slice of the whole log line; holding it would pin the line.
id = strings.Clone(id)
h := &stalwartHeld{id: id, session: s, read: now, v: stalwartVerdict{ip: s.RemoteIP, at: at}}
v.m[id] = h
v.order = append(v.order, h)
return true
}
// invalidate prevents an excluded queued message from borrowing a held
// sender's identity. Keep the slot until expiry so later ingests cannot
// revive an ambiguous join.
func (v *stalwartVerdicts) invalidate(id string) {
v.mu.Lock()
defer v.mu.Unlock()
if h, ok := v.m[id]; ok {
h.v.unmatched = true
}
}
// ingested notes one delivery of message id. Junk for one recipient is Junk
// for the message.
func (v *stalwartVerdicts) ingested(id string, junk bool) {
v.mu.Lock()
defer v.mu.Unlock()
if h, ok := v.m[id]; ok {
h.ingested = true
h.v.junk = h.v.junk || junk
}
}
// due forgets every message read at least stalwartVerdictHold before now
// and returns delivered messages and ambiguous joins.
func (v *stalwartVerdicts) due(now time.Time) []stalwartVerdict {
v.mu.Lock()
defer v.mu.Unlock()
n := 0
for n < len(v.order) && now.Sub(v.order[n].read) >= stalwartVerdictHold {
n++
}
return v.popLocked(n)
}
// drain forgets every message and returns delivered messages and ambiguous joins.
func (v *stalwartVerdicts) drain() []stalwartVerdict {
v.mu.Lock()
defer v.mu.Unlock()
return v.popLocked(len(v.order))
}
func (v *stalwartVerdicts) popLocked(n int) []stalwartVerdict {
var out []stalwartVerdict
for _, h := range v.order[:n] {
delete(v.m, h.id)
if h.ingested || h.v.unmatched {
out = append(out, h.v)
}
}
clear(v.order[:n])
v.order = v.order[n:]
return out
}
// stalwartQueued holds an unauthenticated message until its delivery lines
// say what the filter did with it.
func (d *Daemon) stalwartQueued(ev stalwartlog.Event, s stalwartlog.Session) {
st := d.stalwart
id := ev.Fields["queueId"]
if id == "" {
return
}
// An HTTP listener can log an address from a forwarded header.
if st.ports.Of(s.LocalPort) == stalwartlog.ProtoHTTP {
st.verdicts.invalidate(id)
return
}
if !st.verdicts.queued(id, s, ev.Time, time.Now()) {
d.mailStats.RecordFilterVerdict(ev.Time, mailstats.FilterUnmatched)
}
}
// stalwartClassify books a message the filter rejected or discarded. Stalwart
// logs no queued or delivery line for it, so the line's own session names
// the sender.
func (d *Daemon) stalwartClassify(ev stalwartlog.Event, s stalwartlog.Session, cfg *config.Config) {
if d.stalwart.ports.Of(s.LocalPort) == stalwartlog.ProtoHTTP {
return
}
var outcome string
switch ev.Fields["result"] {
case "reject":
outcome = mailstats.FilterRejected
case "discard":
outcome = mailstats.FilterDiscarded
default:
return
}
d.mailStats.RecordFilterVerdict(ev.Time, outcome)
d.recordStalwartVerdict(s.RemoteIP, true, ev.Time, cfg)
}
// sweepStalwartVerdicts books the messages whose hold has ended.
func (d *Daemon) sweepStalwartVerdicts(now time.Time, cfg *config.Config) {
d.bookStalwartVerdicts(d.stalwart.verdicts.due(now), cfg)
}
// drainStalwartVerdicts books every delivered message without waiting, at
// shutdown.
func (d *Daemon) drainStalwartVerdicts(cfg *config.Config) {
d.bookStalwartVerdicts(d.stalwart.verdicts.drain(), cfg)
}
func (d *Daemon) bookStalwartVerdicts(vs []stalwartVerdict, cfg *config.Config) {
for _, v := range vs {
if v.unmatched {
d.mailStats.RecordFilterVerdict(v.at, mailstats.FilterUnmatched)
continue
}
outcome := mailstats.FilterDelivered
if v.junk {
outcome = mailstats.FilterJunk
}
d.mailStats.RecordFilterVerdict(v.at, outcome)
d.recordStalwartVerdict(v.ip, v.junk, v.at, cfg)
}
}
// recordStalwartVerdict records Stalwart's verdict about a sending address
// as spam evidence. Stalwart filters on its own, so this does not depend on
// CSM's antispam settings.
func (d *Daemon) recordStalwartVerdict(ip netip.Addr, spam bool, at time.Time, cfg *config.Config) {
adb := attackdb.Global()
if adb == nil || !ipmap.Recordable(ip, nil) || d.spamSenderExcluded(ip, cfg) {
return
}
// A verdict influenced by CSM's own list says more about the list than the sender.
if s := d.stalwart; s != nil {
if st := s.list.Load(); st != nil && (st.filterOff || (!st.ruleOff && st.m.Weight(ip) > 0)) {
return
}
}
d.recordMailVerdict(adb, ip.String(), spam, at)
}
package daemon
import (
"context"
"fmt"
"net/netip"
"os"
"sync"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/platform"
)
// stalwartUnban lifts Stalwart's own bans of addresses CSM released, away
// from the request that released them. Any firewall release of one address
// adds it: an operator's, a hosting panel's self-service unban, or a DynDNS
// allow. Each address is pending at most once, so the set needs no cap. It has
// its own lifetime: adds come from request paths that can outlive the
// daemon's final wait, so its drain cannot join that wait.
type stalwartUnban struct {
cfg func() *config.Config
// ctx ends the call in flight when stop runs.
ctx context.Context
cancel context.CancelFunc
mu sync.Mutex
pending map[string]struct{}
running bool
stopped bool
// done is closed when the last drain started has ended.
done chan struct{}
}
// newStalwartUnban returns nil unless this host's mail server is Stalwart.
func newStalwartUnban(d *Daemon) *stalwartUnban {
if detectMTA() != platform.MTAStalwart {
return nil
}
ctx, cancel := context.WithCancel(context.Background())
return &stalwartUnban{cfg: d.currentCfg, ctx: ctx, cancel: cancel, pending: map[string]struct{}{}}
}
// add queues ip and returns at once. After stop it does nothing.
func (u *stalwartUnban) add(ip string) {
u.mu.Lock()
defer u.mu.Unlock()
if u.stopped {
return
}
u.pending[ip] = struct{}{}
if u.running {
return
}
u.running = true
u.done = make(chan struct{})
obs.Go("stalwart-unban", u.drain)
}
// stop ends the call in flight and returns once the drain has ended. No
// drain starts after it. It is safe to call more than once.
func (u *stalwartUnban) stop() {
u.mu.Lock()
u.stopped = true
u.cancel()
done := u.done
u.mu.Unlock()
if done != nil {
<-done
}
}
func (u *stalwartUnban) drain() {
for {
ip, ok := u.next()
if !ok {
return
}
// A call ended by stop is not a failure to report.
if err := u.clear(u.ctx, ip); err != nil && u.ctx.Err() == nil {
fmt.Fprintf(os.Stderr, "[%s] Stalwart may still refuse %s: %v\n", ts(), ip, err)
}
}
}
// next takes one pending address. When none is left or stop has run, it
// ends the drain under the lock, so add and stop always see a drain that is
// either still running or fully done.
func (u *stalwartUnban) next() (string, bool) {
u.mu.Lock()
defer u.mu.Unlock()
if !u.stopped {
for ip := range u.pending {
delete(u.pending, ip)
return ip, true
}
}
u.running = false
close(u.done)
return "", false
}
// automaticBanReasons are the reasons Stalwart gives the bans it makes on
// its own. Any other reason, including one a later Stalwart adds, is left
// for the Stalwart admin.
var automaticBanReasons = map[string]bool{
"rcptToFailure": true,
"authFailure": true,
"loitering": true,
"portScanning": true,
}
// clear lifts Stalwart's automatic bans of ip. Any other block is the
// Stalwart admin's decision and stays.
func (u *stalwartUnban) clear(ctx context.Context, ip string) error {
client, err := stalwartClient(u.cfg())
if err != nil {
return err
}
lookupCtx, cancel := context.WithTimeout(ctx, stalwartCallTimeout)
bans, err := client.BlockedIPs(lookupCtx, ip)
cancel()
if err != nil {
return err
}
want, _ := netip.ParseAddr(ip)
var ids, reasons []string
foreign := false
kept := map[string]bool{}
for _, b := range bans {
// A lookup is an exact match, but an entry for another address must
// never be lifted on this one's behalf.
if a, parseErr := netip.ParseAddr(b.Address); parseErr != nil || a.Unmap() != want.Unmap() {
foreign = true
continue
}
if !automaticBanReasons[b.Reason] {
kept[describeBanReason(b.Reason)] = true
continue
}
ids, reasons = append(ids, b.ID), append(reasons, b.Reason)
}
if foreign {
fmt.Fprintf(os.Stderr, "[%s] Stalwart answered the lookup for %s with entries for other addresses; left them\n", ts(), ip)
}
for reason := range kept {
fmt.Fprintf(os.Stderr, "[%s] Stalwart keeps its block of %s (%s)\n", ts(), ip, reason)
}
if len(ids) == 0 {
// A lift whose reload was lost leaves a ban Stalwart enforces but
// no longer lists. Reloading here lets a second unblock finish it.
reloadCtx, cancelReload := context.WithTimeout(ctx, stalwartCallTimeout)
defer cancelReload()
if reloadErr := client.ReloadBlockedIPs(reloadCtx); reloadErr != nil {
return fmt.Errorf("reloading Stalwart's ban list: %w", reloadErr)
}
return nil
}
deleteCtx, cancel := context.WithTimeout(ctx, stalwartCallTimeout)
err = client.DeleteBlockedIPs(deleteCtx, ids)
cancel()
for _, reason := range reasons {
rec := actionlog.Record{
Op: "operate.clear_stalwart_ban",
Action: "delete_blocked_ip",
Target: ip,
Reason: reason,
Result: actionlog.Applied,
}
if err != nil {
rec.Result, rec.Error = actionlog.Failed, err.Error()
}
actionlog.Write(rec)
}
return err
}
// describeBanReason names a kept entry's reason for the log. Stalwart's
// reasons are plain words; anything else is not echoed.
func describeBanReason(reason string) string {
if reason == "" {
return "an unknown reason"
}
for _, c := range reason {
if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') {
return "an unknown reason"
}
}
return "reason " + reason
}
//go:build linux
package daemon
import (
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/sys/unix"
)
// tmpExecEvidence returns what the writer's ancestry proved: the capture taken
// when the event was admitted, else a walk now. A file that is not root-owned
// or has no writer PID proves nothing and is never walked.
func tmpExecEvidence(event fileEvent, uid uint32) ancestryEvidence {
if uid != 0 || event.pid <= 0 {
return ancestryEvidence{}
}
if event.tmpCaptured {
var st unix.Stat_t
if event.tmpStat == nil || unix.Fstat(event.fd, &st) != nil || !sameTmpFile(*event.tmpStat, st) {
return ancestryEvidence{}
}
}
if event.tmpAncestry != nil {
return *event.tmpAncestry
}
return tmpExecAncestry(event.pid)
}
// captureTmpExecAncestry reads the writer's ancestry while the event is being
// admitted. Copy and link tools exit within moments of closing the file, long
// before an analyzer worker reaches the event. Only a root-owned file can be
// demoted, so nothing else pays for the walk.
func captureTmpExecAncestry(event *fileEvent) {
if event.pid <= 0 || !underTempRoot(event.path) {
return
}
event.tmpCaptured = true
var st unix.Stat_t
if unix.Fstat(event.fd, &st) != nil || st.Uid != 0 || st.Mode&unix.S_IFMT != unix.S_IFREG {
return
}
event.tmpStat = &st
ev := tmpExecAncestry(event.pid)
event.tmpAncestry = &ev
}
const (
// tmpVouchTTL is how long a work directory stays vouched after the last
// proven writer. Builds prove themselves again every few seconds.
tmpVouchTTL = 2 * time.Minute
// tmpVouchCap bounds the table. Concurrent maintenance uses a handful of
// work directories.
tmpVouchCap = 32
// tmpOccupantScanGap spaces out process-table scans for one work
// directory in which no running maintenance process was found.
tmpOccupantScanGap = 5 * time.Second
)
// sameTmpFile reports whether two stats name the same file under the same
// owner. Size, mode and times are left out on purpose: a build goes on
// writing, re-moding and hard-linking a file after the close that raised the
// event, and each of those moves its ctime. Only root can change the owner of a
// root-owned file, so an owner changed away and back is not what this guards.
func sameTmpFile(a, b unix.Stat_t) bool {
return a.Dev == b.Dev && a.Ino == b.Ino && a.Uid == b.Uid && a.Gid == b.Gid
}
// tmpVouches pins each vouched inode until expiry or eviction, so removing a
// directory cannot transfer its proof to a new directory that reuses its inode.
// The path is also bound: renames and mount aliases do not carry proof with them.
type tmpVouches struct {
mu sync.Mutex
entries []tmpVouch
// scanMu admits one occupant scan at a time; scanned holds, per work
// directory, when the next scan may run after one that found nothing.
scanMu sync.Mutex
scanned map[string]time.Time
}
type tmpVouch struct {
dir *os.File
path string
dev, ino uint64
reason string
until time.Time
}
// tmpVouchDirectory follows only root-controlled directories, without symlinks
// or mounts below the temporary root. The final file must be the event's open
// file, not a replacement at its old name. O_PATH never opens a FIFO or device
// for IO. Mount IDs from fdinfo also work on the supported older kernels.
//
// Mount IDs are compared only along this walk. The event descriptor belongs to
// the writer's mount, and a daemon that systemd gave its own mount namespace
// never shares a mount ID with the processes it watches; the descriptor is
// bound to the walked file by device and inode instead.
func tmpVouchDirectory(event fileEvent) *os.File {
work := tmpWorkDir(event.path)
if !strings.HasSuffix(work, "/") || filepath.Clean(event.path) != event.path {
return nil
}
temp := filepath.Dir(strings.TrimSuffix(work, "/"))
fd, err := unix.Open("/", unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
if err != nil {
return nil
}
defer func() { _ = unix.Close(fd) }()
var workFD *os.File
defer func() {
if workFD != nil {
_ = workFD.Close()
}
}()
current := ""
tempMount := ""
parts := strings.Split(strings.TrimPrefix(event.path, "/"), "/")
for i, part := range parts {
if i == len(parts)-1 {
if tempMount == "" || !tmpEventFilePlaced(fd, part, event.fd, tempMount) {
return nil
}
result := workFD
workFD = nil
return result
}
next, err := unix.Openat(fd, part, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
if err != nil {
return nil
}
_ = unix.Close(fd)
fd = next
if tempMount != "" && tmpMountID(fd) != tempMount {
return nil
}
current += "/" + part
var st unix.Stat_t
if unix.Fstat(fd, &st) != nil || st.Uid != 0 {
return nil
}
if st.Mode&unix.S_IFMT != unix.S_IFDIR {
return nil
}
if current == temp {
// Shared roots need sticky semantics; all children must be root-private.
if st.Mode&0o022 != 0 && st.Mode&unix.S_ISVTX == 0 {
return nil
}
tempMount = tmpMountID(fd)
if tempMount == "" {
return nil
}
} else if st.Mode&0o022 != 0 {
return nil
}
if current+"/" == work {
pin, err := unix.FcntlInt(uintptr(fd), unix.F_DUPFD_CLOEXEC, 0)
if err != nil {
return nil
}
workFD = os.NewFile(uintptr(pin), current)
}
}
return nil
}
// tmpEventFilePlaced reports whether the event's open file is the root-owned
// regular file that name refers to in dir, or was until it was removed. An
// installer may create a file, remove it and have a copy tool write it again:
// the removed file has no name left to compare, its path was reported by the
// kernel when the write was seen, and only root can remove a name from a
// directory that passed the checks above.
func tmpEventFilePlaced(dir int, name string, eventFD int, mount string) bool {
var original, parent unix.Stat_t
if unix.Fstat(eventFD, &original) != nil || unix.Fstat(dir, &parent) != nil {
return false
}
if original.Mode&unix.S_IFMT != unix.S_IFREG || original.Uid != 0 || original.Mode&0o022 != 0 ||
original.Dev != parent.Dev {
return false
}
if original.Nlink == 0 {
return true
}
fd, err := unix.Openat(dir, name, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
if err != nil {
return false
}
defer func() { _ = unix.Close(fd) }()
var st unix.Stat_t
return tmpMountID(fd) == mount && unix.Fstat(fd, &st) == nil && sameTmpFile(st, original)
}
// fdinfo belongs to our own pinned O_PATH descriptor, not to the writer.
func tmpMountID(fd int) string {
// #nosec G304 -- fixed procfs metadata path for a descriptor opened by this process.
data, err := os.ReadFile("/proc/self/fdinfo/" + strconv.Itoa(fd))
if err != nil {
return ""
}
for _, line := range strings.Split(string(data), "\n") {
if value, ok := strings.CutPrefix(line, "mnt_id:"); ok {
return strings.TrimSpace(value)
}
}
return ""
}
func (v *tmpVouches) prune(now time.Time) {
kept := v.entries[:0]
for _, e := range v.entries {
if now.Before(e.until) {
kept = append(kept, e)
} else {
_ = e.dir.Close()
}
}
clear(v.entries[len(kept):])
v.entries = kept
}
// grant accepts only direct ancestry proof, dated when that proof was captured.
func (v *tmpVouches) grant(event fileEvent, reason string, now time.Time) {
at := event.at
if at.IsZero() {
at = now
}
v.grantAt(event, reason, at, now)
}
// grantAt records proof read at the given time for the event's work directory.
func (v *tmpVouches) grantAt(event fileEvent, reason string, at, now time.Time) {
if !now.Before(at.Add(tmpVouchTTL)) {
return
}
dir := tmpVouchDirectory(event)
if dir == nil {
return
}
var st unix.Stat_t
if unix.Fstat(int(dir.Fd()), &st) != nil {
_ = dir.Close()
return
}
work := tmpWorkDir(event.path)
v.mu.Lock()
defer v.mu.Unlock()
v.prune(now)
for i, e := range v.entries {
if e.path == work {
// Workers may finish out of order. Older evidence cannot replace newer proof.
if e.until.After(at.Add(tmpVouchTTL)) {
_ = dir.Close()
return
}
_ = e.dir.Close()
v.entries = append(v.entries[:i], v.entries[i+1:]...)
break
}
}
if len(v.entries) >= tmpVouchCap {
_ = v.entries[0].dir.Close()
v.entries = v.entries[1:]
}
v.entries = append(v.entries, tmpVouch{dir: dir, path: work, dev: uint64(st.Dev), ino: st.Ino, reason: reason, until: at.Add(tmpVouchTTL)})
}
func (v *tmpVouches) covers(event fileEvent, now time.Time) (string, bool) {
dir := tmpVouchDirectory(event)
if dir == nil {
return "", false
}
defer func() { _ = dir.Close() }()
var st unix.Stat_t
if unix.Fstat(int(dir.Fd()), &st) != nil {
return "", false
}
v.mu.Lock()
defer v.mu.Unlock()
v.prune(now)
for _, e := range v.entries {
if e.path == tmpWorkDir(event.path) && e.dev == uint64(st.Dev) && e.ino == st.Ino {
return e.reason, true
}
}
return "", false
}
func (v *tmpVouches) close() {
v.mu.Lock()
defer v.mu.Unlock()
for _, e := range v.entries {
_ = e.dir.Close()
}
v.entries = nil
}
// occupied looks for a running maintenance process working inside the event's
// work directory and vouches for the directory when it finds one. A build
// driver such as make stays in its work directory while the short-lived tools
// it starts write the files, so no writer there may ever be proven.
func (v *tmpVouches) occupied(event fileEvent, now time.Time) (string, bool) {
dir := tmpVouchDirectory(event)
if dir == nil {
return "", false
}
defer func() { _ = dir.Close() }()
var st unix.Stat_t
if unix.Fstat(int(dir.Fd()), &st) != nil {
return "", false
}
work := tmpWorkDir(event.path)
// Workers that queue behind a scan find the vouch it left.
v.scanMu.Lock()
defer v.scanMu.Unlock()
if reason, ok := v.covers(event, now); ok {
return reason, true
}
v.mu.Lock()
for path, until := range v.scanned {
if !now.Before(until) {
delete(v.scanned, path)
}
}
// Never evict an active cooldown: a burst across more directories than
// the table holds would otherwise turn every write into a process scan.
wait := now.Before(v.scanned[work]) || len(v.scanned) >= tmpVouchCap
v.mu.Unlock()
if wait {
return "", false
}
reason, ok := provenOccupant(uint64(st.Dev), st.Ino, now)
v.mu.Lock()
if v.scanned == nil {
v.scanned = make(map[string]time.Time)
}
if ok {
delete(v.scanned, work)
} else {
v.scanned[work] = now.Add(tmpOccupantScanGap)
}
v.mu.Unlock()
if !ok {
return "", false
}
v.grantAt(event, reason, now, now)
return reason, true
}
// provenOccupant reports the reason a running root process with proven
// maintenance ancestry gives, when its working directory is inside the work
// directory with this device and inode. The directory is matched by identity,
// so a second mount of the same temporary filesystem does not hide it.
func provenOccupant(dev, ino uint64, now time.Time) (string, bool) {
entries, err := os.ReadDir(procRootDir)
if err != nil {
return "", false
}
for _, entry := range entries {
pid, err := strconv.ParseInt(entry.Name(), 10, 32)
if err != nil || pid <= 1 {
continue
}
cwdPath := filepath.Join(procRootDir, entry.Name(), "cwd")
cwd, err := os.Readlink(cwdPath)
if err != nil || !underTempRoot(cwd+"/") {
continue
}
var st unix.Stat_t
work := strings.TrimSuffix(tmpWorkDir(cwd+"/"), "/")
if unix.Lstat(work, &st) != nil || uint64(st.Dev) != dev || st.Ino != ino {
continue
}
if !tmpOccupantInside(cwdPath, dev, ino) || !procIsRoot(int32(pid)) {
continue
}
if ok, reason := tmpExecDemote(0, tmpExecAncestry(int32(pid)), now); ok {
return reason, true
}
}
return "", false
}
// tmpOccupantInside follows the kernel's cwd reference, not just its printed
// name. Every directory back to the event's work directory must be controlled
// by root. A writable child cannot lend its occupant's proof to safe siblings.
func tmpOccupantInside(cwdPath string, dev, ino uint64) bool {
fd, err := unix.Open(cwdPath, unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
if err != nil {
return false
}
defer func() { _ = unix.Close(fd) }()
mount := tmpMountID(fd)
if mount == "" {
return false
}
for {
var st unix.Stat_t
if unix.Fstat(fd, &st) != nil || st.Uid != 0 || st.Mode&0o022 != 0 {
return false
}
if uint64(st.Dev) == dev && st.Ino == ino {
return true
}
parent, err := unix.Openat(fd, "..", unix.O_PATH|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
if err != nil {
return false
}
_ = unix.Close(fd)
fd = parent
var up unix.Stat_t
if tmpMountID(fd) != mount || unix.Fstat(fd, &up) != nil || (st.Dev == up.Dev && st.Ino == up.Ino) {
return false
}
}
}
// procIsRoot reports whether pid runs as root by real and effective user.
func procIsRoot(pid int32) bool {
// #nosec G304 -- procfs pseudo-file under a fixed root; pid is a numeric procfs entry.
status, err := os.ReadFile(filepath.Join(procRootDir, strconv.Itoa(int(pid)), "status"))
if err != nil {
return false
}
for _, line := range strings.Split(string(status), "\n") {
if strings.HasPrefix(line, "Uid:") {
return statusUIDIsRoot(line)
}
}
return false
}
package daemon
import (
"crypto/sha256"
"fmt"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/config"
)
const checkTrustedProxyUnusable = "trusted_proxy_unusable"
// trustedProxyFindings reports skipped entries that can leave proxies
// counted as clients or forwarded requests without attribution.
func trustedProxyFindings(cfg *config.Config) []alert.Finding {
skipped := config.UnusableTrustedProxies(cfg)
if len(skipped) == 0 {
return nil
}
reasons := make([]string, len(skipped))
for i, r := range skipped {
reasons[i] = r.Message
}
details := strings.Join(reasons, "\n")
// Explicit identities let recovery re-arm dismissals. Include the errors
// so acknowledging one configuration does not hide a different mistake.
identity := sha256.Sum256([]byte(details))
return []alert.Finding{{
Severity: alert.High,
Check: checkTrustedProxyUnusable,
DedupKey: fmt.Sprintf("%x", identity),
Message: fmt.Sprintf("CSM skips %d trusted_proxies entries; a proxy they meant to list can be blocked as a client", len(skipped)),
Details: details +
"\nFix or remove these entries and restart CSM. Requests may count against a proxy or against nobody.",
Timestamp: time.Now(),
}}
}
// refreshTrustedProxyFinding keeps the finding in step with the running
// config: raised while it skips an entry, resolved once a config without
// one runs.
func (d *Daemon) refreshTrustedProxyFinding() {
if d.store == nil {
return
}
cfg := d.currentCfg()
findings := trustedProxyFindings(cfg)
// A dismissal covers one episode; a fixed and later repeated mistake
// notifies again.
d.store.RearmAbsentDedupFindings([]string{checkTrustedProxyUnusable}, findings)
d.processScanFindings(cfg, findings, []string{checkTrustedProxyUnusable}, "config")
}
package daemon
import (
"context"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/verdict"
)
// verdictAskFunc asks the operator's callback about one destination.
type verdictAskFunc func(ctx context.Context, req verdict.Request) (verdict.Response, error)
type verdictEnricherOpts struct {
Ask verdictAskFunc
Workers int
Queue int
TTL time.Duration
}
type verdictEntry struct {
tenantID string
verdict string
note string
at time.Time
}
type verdictKey struct {
ip string
reason string
severity string
}
type verdictJob struct {
key verdictKey
ticket queuehealth.Ticket
}
// verdictEnricher annotates findings with the operator's verdict without ever
// standing between the BPF ring buffer and the alert channel.
//
// The callback used to run inline in the consumer loop: one denied connection
// could hold the reader for the callback's whole timeout while a 256-slot ring
// overflowed, so an optional annotation cost real security events. Enrichment
// now happens on a bounded worker pool behind a short-lived cache. A finding is
// dispatched immediately either way; what a cache miss loses is the annotation
// on that first event, not the event.
type verdictEnricher struct {
ask verdictAskFunc
ttl time.Duration
jobs chan verdictJob
workers int
cacheCap int
wg sync.WaitGroup
mu sync.Mutex
cache map[verdictKey]verdictEntry
inFlight map[verdictKey]bool
dropped atomic.Int64
queueStats *queuehealth.Tracker
ctx context.Context
stopped bool
stopOnce sync.Once
}
func newVerdictEnricher(opts verdictEnricherOpts) *verdictEnricher {
if opts.Workers <= 0 {
opts.Workers = 2
}
if opts.Workers > 4 {
opts.Workers = 4
}
if opts.Queue <= 0 {
opts.Queue = 64
}
if opts.TTL <= 0 {
opts.TTL = time.Minute
}
return &verdictEnricher{
ask: opts.Ask,
ttl: opts.TTL,
jobs: make(chan verdictJob, opts.Queue),
workers: opts.Workers,
cacheCap: opts.Queue,
cache: make(map[verdictKey]verdictEntry),
inFlight: make(map[verdictKey]bool),
queueStats: queuehealth.New(opts.Queue, time.Minute),
}
}
func (e *verdictEnricher) start(ctx context.Context) {
e.mu.Lock()
e.ctx = ctx
e.mu.Unlock()
for i := 0; i < e.workers; i++ {
e.wg.Add(1)
go func() {
defer e.wg.Done()
e.work(ctx)
}()
}
}
func (e *verdictEnricher) wait() {
e.stopOnce.Do(func() {
e.mu.Lock()
e.stopped = true
close(e.jobs)
e.mu.Unlock()
e.wg.Wait()
e.mu.Lock()
defer e.mu.Unlock()
for job := range e.jobs {
delete(e.inFlight, job.key)
job.ticket.Reject(time.Now())
e.dropped.Add(1)
}
})
}
// QueueStatuses reports annotation work. The row is advisory: the finding is
// dispatched with or without a verdict, so a stalled lookup delays context
// rather than protection.
func (e *verdictEnricher) QueueStatuses(now time.Time) map[string]queuehealth.Status {
status := e.queueStats.Snapshot(now)
status.Advisory = true
return map[string]queuehealth.Status{"verdict": status}
}
func (e *verdictEnricher) droppedEnrichments() int64 { return e.dropped.Load() }
// annotate applies a cached verdict to f and reports whether it could. On a
// miss it queues the lookup and returns immediately: the caller dispatches the
// finding now and later events for the same destination carry the answer.
func (e *verdictEnricher) annotate(f *alert.Finding, ip, reason, severity string) bool {
if e == nil || f == nil || e.ask == nil {
return false
}
key := verdictKey{ip: ip, reason: reason, severity: severity}
e.mu.Lock()
entry, ok := e.cache[key]
fresh := ok && time.Since(entry.at) < e.ttl
if ok && !fresh {
delete(e.cache, key)
}
if fresh {
e.mu.Unlock()
applyVerdictEntry(f, entry)
return true
}
defer e.mu.Unlock()
if e.stopped || (e.ctx != nil && e.ctx.Err() != nil) {
e.queueStats.Lose(time.Now(), 1)
e.dropped.Add(1)
return false
}
if e.inFlight[key] {
return false
}
ticket := e.queueStats.Begin(time.Now())
e.inFlight[key] = true
select {
case e.jobs <- verdictJob{key: key, ticket: ticket}:
default:
// Saturated: the annotation is what we give up, never the finding.
delete(e.inFlight, key)
ticket.Reject(time.Now())
e.dropped.Add(1)
}
return false
}
func (e *verdictEnricher) work(ctx context.Context) {
for {
if ctx.Err() != nil {
return
}
select {
case <-ctx.Done():
return
case job, ok := <-e.jobs:
if !ok {
return
}
if err := e.processJob(ctx, job); err != nil {
csmlog.Warn("bpf enforcement verdict callback failed", "err", err, "dst", job.key.ip)
}
}
}
}
func (e *verdictEnricher) processJob(ctx context.Context, job verdictJob) error {
job.ticket.Start(time.Now())
completed := false
defer func() {
e.mu.Lock()
defer e.mu.Unlock()
delete(e.inFlight, job.key)
if completed {
job.ticket.Finish(time.Now())
} else {
job.ticket.Reject(time.Now())
e.dropped.Add(1)
}
}()
resp, err := e.ask(ctx, verdict.Request{
IP: job.key.ip,
Reason: job.key.reason,
Severity: job.key.severity,
Source: "bpf_enforcement",
})
if err != nil {
// A failure is not an answer: the next event must be able to retry.
return err
}
e.mu.Lock()
now := time.Now()
e.pruneCacheLocked(now)
e.cache[job.key] = verdictEntry{
tenantID: resp.TenantID,
verdict: resp.Verdict,
note: resp.Note,
at: now,
}
e.mu.Unlock()
completed = true
return nil
}
func (e *verdictEnricher) pruneCacheLocked(now time.Time) {
for key, entry := range e.cache {
if now.Sub(entry.at) >= e.ttl {
delete(e.cache, key)
}
}
if len(e.cache) < e.cacheCap {
return
}
var oldestKey verdictKey
var oldestAt time.Time
for key, entry := range e.cache {
if oldestAt.IsZero() || entry.at.Before(oldestAt) {
oldestKey = key
oldestAt = entry.at
}
}
delete(e.cache, oldestKey)
}
func applyVerdictEntry(f *alert.Finding, entry verdictEntry) {
if entry.tenantID != "" && f.TenantID == "" {
f.TenantID = entry.tenantID
}
if entry.verdict != "" {
appendFindingDetail(f, "Verdict callback: "+entry.verdict)
}
if entry.tenantID != "" {
appendFindingDetail(f, "Verdict tenant: "+entry.tenantID)
}
if entry.note != "" {
appendFindingDetail(f, "Verdict note: "+entry.note)
}
}
package daemon
import (
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/store"
"github.com/pidginhost/csm/internal/threatintel"
)
// verifiedBotEntries converts the operator-configured reputation.verified_bots
// list into the threatintel registry shape.
func verifiedBotEntries(cfg *config.Config) []threatintel.BotEntry {
if cfg == nil {
return nil
}
out := make([]threatintel.BotEntry, 0, len(cfg.Reputation.VerifiedBots))
for _, b := range cfg.Reputation.VerifiedBots {
out = append(out, threatintel.BotEntry{
Name: b.Name,
UASubstrings: b.UASubstrings,
RDNSSuffixes: b.RDNSSuffixes,
IPRanges: b.IPRanges,
})
}
return out
}
// reconcileVerifiedBots re-applies reputation.verified_bots after a SIGHUP so
// operators can add or change good bots without a restart. Re-stamping the
// PTR-verdict cache with the new list drops cached verdicts when the list
// changed, so a previously-spoofed IP is re-checked under the new suffixes
// instead of staying pinned for the cache TTL.
func (d *Daemon) reconcileVerifiedBots() {
cfg := d.activeOrStartupCfg()
entries := verifiedBotEntries(cfg)
threatintel.SetOperatorBots(entries)
if !cfg.BotVerifyEnabled() {
return
}
db := store.Global()
if db == nil {
return
}
ver := threatintel.OperatorBotsCacheVersion(threatintel.LogicVersion, entries)
if dropped, err := db.EnsureBotVerifyLogicVersion(ver); err == nil && dropped {
csmlog.Info("bot-verify cache dropped after verified_bots change")
}
if d.botVerifier == nil {
d.startBotVerifier(db, entries)
return
}
d.botVerifier.SetOperatorEntries(entries)
}
func (d *Daemon) startBotVerifier(db *store.DB, entries []threatintel.BotEntry) {
bv := threatintel.NewAsyncBotVerifier(db.PutBotVerify, db)
bv.SetOperatorEntries(entries)
d.registerQueueSource("bot_verification", bv)
d.botVerifier = bv
d.wg.Add(1)
obs.Go("bot-verify", func() {
defer d.wg.Done()
bv.Run(d.stopCh)
})
checks.SetBotVerifier(bv, db.GetBotVerify)
}
package daemon
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"strings"
"sync"
"syscall"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/eximlog"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/store"
)
const (
recentOutgoingMailHoldWindow = 2 * time.Hour
logWatcherMaxLineBytes = 256 * 1024
logWatcherOffsetMarkerBytes = 256
)
// LogLineHandler parses a log line and returns findings (if any).
type LogLineHandler func(line string, cfg *config.Config) []alert.Finding
// ObservedLineHandler is a LogLineHandler that also receives the line's
// observation, for a handler that keeps a finding and emits it later.
type ObservedLineHandler func(line string, obs alert.Observation, cfg *config.Config) []alert.Finding
// logWatchSpec is one log the daemon watches: the handler, and the evidence
// producer its findings come from (empty for a log that feeds none).
type logWatchSpec struct {
name string
path string
handler LogLineHandler
handlerForPath func(string) LogLineHandler
observed ObservedLineHandler
producer admission.ProducerID
}
// LogWatcher tails a log file using inotify and processes new lines.
type LogWatcher struct {
path string
cfg *config.Config
handler LogLineHandler
alertCh chan<- alert.Finding
file *os.File
offset int64
fileID logFileID
marker []byte
closeOnce sync.Once
// producer names the evidence producer this log feeds; empty leaves
// findings without an observation.
producer admission.ProducerID
// observed, when set, replaces handler and also gets the observation.
observed ObservedLineHandler
// open opens path for reading; nil means os.Open.
open func(path string) (*os.File, error)
// stream caches the stream name of streamOf, so an observed log does not
// format it for every line.
stream string
streamOf logFileID
// unframed is set while the offset is inside a line that began before
// this watcher read the file: that remainder names no observation.
unframed bool
}
type logFileID struct {
dev uint64
ino uint64
known bool
}
func fileID(info os.FileInfo) logFileID {
if info == nil {
return logFileID{}
}
if st, ok := info.Sys().(*syscall.Stat_t); ok {
return logFileID{
dev: uint64(st.Dev), // #nosec G115 -- device IDs are non-negative on supported Unix hosts
ino: uint64(st.Ino), // #nosec G115 -- inode numbers are non-negative
known: st.Dev != 0 || st.Ino != 0,
}
}
return logFileID{}
}
func (id logFileID) same(other logFileID) bool {
return id.known && other.known && id.dev == other.dev && id.ino == other.ino
}
func readOffsetMarker(f *os.File, offset int64) ([]byte, bool, error) {
if f == nil || offset <= 0 {
return nil, true, nil
}
n := int64(logWatcherOffsetMarkerBytes)
if offset < n {
n = offset
}
buf := make([]byte, n)
read, err := f.ReadAt(buf, offset-n)
if err != nil && !errors.Is(err, io.EOF) {
return nil, false, err
}
return buf[:read], read == len(buf), nil
}
// logWatcherPollInterval is how often each LogWatcher reads new lines.
const logWatcherPollInterval = 2 * time.Second
// NewLogWatcher creates a watcher for a log file.
func NewLogWatcher(path string, cfg *config.Config, handler LogLineHandler, alertCh chan<- alert.Finding) (*LogWatcher, error) {
return newLogWatcher(path, cfg, handler, alertCh, nil)
}
// newLogWatcher creates a watcher that opens the log with open, or with
// os.Open when open is nil, every time it opens it.
func newLogWatcher(path string, cfg *config.Config, handler LogLineHandler, alertCh chan<- alert.Finding, open func(string) (*os.File, error)) (*LogWatcher, error) {
w := &LogWatcher{path: path, cfg: cfg, handler: handler, alertCh: alertCh, open: open}
f, err := w.openFile()
if err != nil {
return nil, err
}
// Seek to end - only process new lines
offset, err := f.Seek(0, io.SeekEnd)
if err != nil {
_ = f.Close()
return nil, err
}
info, err := f.Stat()
if err != nil {
_ = f.Close()
return nil, err
}
marker, markerOK, err := readOffsetMarker(f, offset)
if err != nil {
_ = f.Close()
return nil, fmt.Errorf("read offset marker for %s: %w", path, err)
}
if !markerOK {
// The file rotated or shrank between Seek and ReadAt. Start without a
// marker rather than fail: a constructor error would disable this
// watcher until daemon restart, while readNewLines treats the saved
// position as untrusted and reads from the beginning on the next tick.
marker = nil
}
w.file, w.offset, w.fileID, w.marker = f, offset, fileID(info), marker
// Only a newline before the start offset makes the next byte a record start.
w.unframed = len(marker) > 0 && marker[len(marker)-1] != '\n'
return w, nil
}
func (w *LogWatcher) openFile() (*os.File, error) {
if w.open != nil {
return w.open(w.path)
}
// #nosec G304 -- path is operator-configured log path from csm.yaml.
return os.Open(w.path)
}
// newObservedLogWatcher watches a log as the evidence producer spec names.
func newObservedLogWatcher(spec logWatchSpec, cfg *config.Config, alertCh chan<- alert.Finding) (*LogWatcher, error) {
handler := spec.handler
if spec.handlerForPath != nil {
handler = spec.handlerForPath(spec.path)
}
w, err := NewLogWatcher(spec.path, cfg, handler, alertCh)
if err != nil {
return nil, err
}
w.producer = spec.producer
w.observed = spec.observed
return w, nil
}
// observation names the line that starts at lineStart in the current file
// by its offset and bytes; zero when the log feeds no producer.
func (w *LogWatcher) observation(lineStart int64, line string, at time.Time) alert.Observation {
if w.producer == "" || !w.fileID.known {
return alert.Observation{}
}
if w.stream == "" || w.streamOf != w.fileID {
w.stream = alert.FileStream(w.fileID.dev, w.fileID.ino)
w.streamOf = w.fileID
}
return alert.Observation{
Producer: string(w.producer),
Stream: w.stream,
Cursor: alert.RecordCursor(lineStart, line),
ObservedAt: at,
}
}
// currentCfg returns the live daemon config so SIGHUP changes to thresholds,
// infra_ips, trusted_countries, and suppression settings reach the log-line
// handlers without a restart. Falls back to the startup snapshot before the
// first hot-reload publishes an active config.
func (w *LogWatcher) currentCfg() *config.Config {
if cfg := config.Active(); cfg != nil {
return cfg
}
return w.cfg
}
// Run starts watching the log file. Uses polling (every 2 seconds) instead of
// inotify to avoid complexity with log rotation. Simple, reliable, low overhead.
func (w *LogWatcher) Run(stopCh <-chan struct{}) {
// Run owns the file for its lifetime. Closing it here, rather than from a
// separate shutdown goroutine, keeps w.file single-threaded: a concurrent
// Stop() close used to race readNewLines/reopen, and the freed fd could be
// reused by another goroutine mid-Stat/Read.
defer w.closeFile()
ticker := time.NewTicker(logWatcherPollInterval)
defer ticker.Stop()
// Also reopen the file every 5 minutes to handle log rotation
reopenTicker := time.NewTicker(5 * time.Minute)
defer reopenTicker.Stop()
for {
select {
case <-stopCh:
return
case <-reopenTicker.C:
w.reopen()
case <-ticker.C:
w.readNewLines()
}
}
}
// Stop closes the watcher's file. Safe to call when Run was never started
// (unit tests open a watcher just to drive readNewLines directly). When Run is
// active it closes the file itself on stopCh, so the daemon must not call Stop
// concurrently with a running Run.
func (w *LogWatcher) Stop() {
w.closeFile()
}
func (w *LogWatcher) closeFile() {
w.closeOnce.Do(func() {
if w.file != nil {
_ = w.file.Close()
}
})
}
func (w *LogWatcher) readNewLines() {
if w.file == nil {
w.reopen()
if w.file == nil {
return
}
}
info, err := w.file.Stat()
if err != nil {
w.reopen()
return
}
// File was truncated or rotated (smaller than our offset)
if info.Size() < w.offset {
w.reopen()
return
}
if !w.offsetMarkerMatches(w.file) {
w.offset = 0
w.marker = nil
w.unframed = false
}
// No new data
if info.Size() == w.offset {
return
}
// Seek to where we left off
_, err = w.file.Seek(w.offset, io.SeekStart)
if err != nil {
return
}
reader := bufio.NewReaderSize(w.file, 64*1024)
committedOffset := w.offset
for {
lineStart := committedOffset
rawLine, truncated, readErr := readBoundedWatcherLine(reader, logWatcherMaxLineBytes)
if len(rawLine) > 0 && readErr != nil {
break
}
framed := !w.unframed
if readErr == nil {
// This line ends here, so the next one starts a record.
w.unframed = false
if current, seekErr := w.file.Seek(0, io.SeekCurrent); seekErr == nil {
committedOffset = current - int64(reader.Buffered())
}
}
if truncated {
fmt.Fprintf(os.Stderr, "[%s] Warning: skipped oversized log line from %s at %d bytes\n", ts(), w.path, logWatcherMaxLineBytes)
}
if len(rawLine) > 0 && !truncated {
line := trimWatcherLineEnding(rawLine)
if line == "" {
if readErr != nil {
break
}
continue
}
now := time.Now()
var findings []alert.Finding
var obs alert.Observation
if w.observed != nil {
if framed {
obs = w.observation(lineStart, line, now)
}
findings = w.observed(line, obs, w.currentCfg())
} else if findings = w.handler(line, w.currentCfg()); len(findings) > 0 && framed {
obs = w.observation(lineStart, line, now)
}
for _, f := range findings {
if f.Timestamp.IsZero() {
f.Timestamp = now
}
f.Observation = obs
if !alert.TryEnqueue(w.alertCh, f) {
if f.Check == "exim_frozen_realtime" {
releaseEximFrozenDedup(line)
}
// Channel full - drop (backpressure)
fmt.Fprintf(os.Stderr, "[%s] Warning: alert channel full, dropping finding from %s\n", ts(), w.path)
}
}
}
if readErr != nil {
break
}
}
w.offset = committedOffset
w.refreshOffsetMarker()
}
func readBoundedWatcherLine(r *bufio.Reader, maxBytes int) (string, bool, error) {
var b strings.Builder
truncated := false
for {
chunk, err := r.ReadSlice('\n')
if len(chunk) > 0 {
switch {
case truncated:
case b.Len()+len(chunk) <= maxBytes:
b.Write(chunk)
default:
if room := maxBytes - b.Len(); room > 0 {
b.Write(chunk[:room])
}
truncated = true
}
}
if errors.Is(err, bufio.ErrBufferFull) {
continue
}
return b.String(), truncated, err
}
}
func trimWatcherLineEnding(line string) string {
line = strings.TrimSuffix(line, "\n")
return strings.TrimSuffix(line, "\r")
}
func (w *LogWatcher) reopen() {
if w.file != nil {
_ = w.file.Close()
// Drop the closed handle so a failed open below doesn't leave a dead
// fd behind for the next readNewLines to Stat in a loop.
w.file = nil
}
f, err := w.openFile()
if err != nil {
return
}
info, err := f.Stat()
if err != nil {
_ = f.Close()
return
}
w.file = f
id := fileID(info)
switch {
case w.fileID.known && id.known && !w.fileID.same(id):
// Rotated by rename+create: new file, read from the start regardless
// of its size.
w.offset = 0
case info.Size() < w.offset:
// Truncated in place (copytruncate rotation).
w.offset = 0
case !w.offsetMarkerMatches(f):
// The saved offset now points into different content. This catches a
// truncate-and-regrow between polling ticks and cheap inode reuse.
w.offset = 0
}
if w.offset == 0 {
w.marker = nil
w.unframed = false
} else {
w.refreshOffsetMarker()
}
// Same file, size >= offset, matching marker: keep w.offset so lines
// written since the last read tick are not skipped. readNewLines seeks
// before every read.
w.fileID = id
}
func (w *LogWatcher) offsetMarkerMatches(f *os.File) bool {
if w.offset == 0 {
return true
}
if len(w.marker) == 0 {
return false
}
marker, ok, err := readOffsetMarker(f, w.offset)
return err == nil && ok && bytes.Equal(marker, w.marker)
}
func (w *LogWatcher) refreshOffsetMarker() {
marker, ok, err := readOffsetMarker(w.file, w.offset)
if err != nil || !ok {
w.marker = nil
return
}
w.marker = marker
}
// --- Log line handlers ---
func parseSessionLogLine(line string, cfg *config.Config) []alert.Finding {
r, ok := checks.ParseSessionRecord(line)
if !ok {
return nil
}
var findings []alert.Finding
// cPanel login from non-infra IP - only alert on direct form login,
// not API-created sessions (from portal create_user_session)
if r.Service == "cpaneld" && r.Event == "NEW" {
addr, isAddr := r.ClientAddr()
// Track IP→account for purge correlation (before any filtering)
if isAddr {
purgeTracker.recordLogin(addr.String(), r.Account)
}
switch {
case cfg.Suppressions.SuppressCpanelLogin:
// Skip all cPanel login alerts
case r.CreatedByAPI():
// Portal-created session - no alert
case !isAddr || addr.IsLoopback():
// No client to report, as in the scheduled scan
default:
ip := addr.String()
if !isInfraIPDaemon(ip, cfg.InfraIPs) && !isTrustedCountry(ip, cfg.Suppressions.TrustedCountries) {
// WARNING severity - logins are useful for audit trail but
// not paging-level. Multi-IP correlation and brute-force
// stay at CRITICAL/HIGH via their own checks.
method := r.Attr("method")
switch method {
case "handle_form_login":
method = "direct form login"
case "":
method = "unknown"
}
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "cpanel_login_realtime",
Message: fmt.Sprintf("cPanel direct login from non-infra IP: %s (account: %s, method: %s)", ip, r.Account, method),
Details: truncateDaemon(line, 300),
SourceIP: ip,
TenantID: r.Account,
})
}
}
}
// Password purge
if r.Event == "PURGE" && r.Reason() == "password_change" {
purgeTracker.recordPurge(r.Account)
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "cpanel_password_purge_realtime",
Message: fmt.Sprintf("cPanel password purge for: %s", r.Account),
TenantID: r.Account,
})
}
return findings
}
// parseSecureLogLine reports accepted SSH logins from the authentication log.
// The scheduled ssh_logins check reads the same file and will meet this line
// again, so the finding comes from the shared builder: identical findings
// collapse in the state store, while a login this watcher never saw is still
// reported by the scan.
func parseSecureLogLine(line string, cfg *config.Config) []alert.Finding {
if f, ok := checks.SSHAcceptedLoginFinding(line, cfg); ok {
return []alert.Finding{f}
}
return nil
}
func parseEximLogLine(line string, cfg *config.Config) []alert.Finding {
var findings []alert.Finding
// 1. Frozen bounces - spam indicator. The capitalized form is the initial
// freeze event ("Frozen (delivery error message)"); the lowercase form is
// exim re-logging "Message is frozen" on every queue run. Dedup by queue
// ID so one stuck message alerts once, and unfreeze events never do.
if eximFrozenShouldAlert(line, time.Now()) {
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "exim_frozen_realtime",
Message: "Exim frozen message detected",
Details: truncateDaemon(line, 200),
})
}
// 2. Outgoing mail hold - account is held by cPanel.
// Format: "Sender office@example.com has an outgoing mail hold"
// or: "Domain example.org has an outgoing mail hold"
//
// Exim emits this rejection from the enforce_mail_permissions router on
// EVERY queued-message retry while the hold is active, so re-applying the
// hold here creates a feedback loop: an operator who clears a
// false-positive hold (e.g. caused by external transit defers like the
// 2026-05-11 Microsoft edge outage) sees CSM re-set the hold within
// seconds because old queued messages keep retrying. cPanel's
// TailWatch::Eximstats is the authoritative source for setting
// the hold. CSM records the hold so later retry-limit noise from
// the held domain is not promoted to a fresh spam outbreak.
permissionText := mailPermissionLogText(line)
if strings.Contains(permissionText, "outgoing mail hold") {
sender := extractMailHoldSender(permissionText)
domain := extractDomainFromEmail(sender)
if domain == "" {
domain = sender // may already be a bare domain
}
if domain != "" {
recordRecentOutgoingMailHold(domain)
RecordCompromisedDomain(domain)
}
// Alert only once per domain per hour
dedupKey := "email_hold:" + domain
if db := store.Global(); db != nil {
lastAlert := db.GetMetaString(dedupKey)
if lastAlert != "" && !isDedupExpired(lastAlert, 1*time.Hour) {
// Already alerted for this domain recently - skip finding
} else {
_ = db.SetMetaString(dedupKey, time.Now().Format(time.RFC3339))
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "email_compromised_account",
Message: fmt.Sprintf("Account %s is on cPanel outgoing mail hold", sender),
Details: truncateDaemon(line, 300),
Mailbox: mailboxOnly(sender),
Domain: domain,
TenantID: checks.MailOwner(domain),
})
}
}
}
// 3. Max defers/failures exceeded.
//
// cPanel's TailWatch::Eximstats has already throttled the domain by the
// time exim emits this line from enforce_mail_permissions, so the line is
// not independent evidence of an outbound spam blast: the same governor
// trips on inbound junk, full mailboxes, and forwarder bounces. Escalate
// to a compromise (CRITICAL + auto-hold) only when CSM's own
// authenticated-send rate window corroborates a real outbound blast for
// the domain. Otherwise report a deliverability event and leave the hold
// to cPanel, so an operator who clears a false-positive hold is not
// immediately re-held.
if strings.Contains(permissionText, "max defers and failures per hour") {
domain := extractEximDomain(permissionText)
if recentOutgoingMailHold(domain) {
return findings
}
if domainHasOutboundBlast(domain, cfg) {
held := maybeHoldOutgoingMail(cfg, domain)
if held {
recordRecentOutgoingMailHold(domain)
}
message := fmt.Sprintf("Spam outbreak: %s exceeded max defers/failures with high outbound volume", domain)
if held {
message += " - outgoing mail auto-suspended"
}
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "email_spam_outbreak",
Message: message,
Details: truncateDaemon(line, 300),
Domain: domain,
TenantID: checks.MailOwner(domain),
})
if domain != "" {
RecordCompromisedDomain(domain)
}
} else {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "email_defer_fail_governor",
Message: fmt.Sprintf("%s hit the cPanel defer/fail governor; no outbound spam volume observed", domain),
Details: truncateDaemon(line, 300),
Domain: domain,
})
}
}
// 4. SMTP credentials leaked in subject - compromised account
// Pattern: T="...host:port,user@domain,PASSWORD..." in the subject field
if subject, ok := eximlog.Subject(line); ok {
_, sender, _ := eximlog.Arrival(line)
subjectLower := strings.ToLower(subject)
// Detect credential patterns: host:port,user,password or
// SMTP credentials in subject (common in credential stuffing attacks)
if (strings.Contains(subject, ":587,") || strings.Contains(subject, ":465,") ||
strings.Contains(subject, ":25,")) &&
strings.Contains(subject, "@") {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "email_credential_leak",
Message: fmt.Sprintf("SMTP credentials leaked in email subject from %s", sender),
Details: fmt.Sprintf("The email subject contains what appears to be SMTP credentials (host:port,user,password). This account is likely compromised by a bulk mail service.\nSubject: %s", truncateDaemon(subject, 100)),
Mailbox: mailboxOnly(sender),
Domain: extractDomainFromEmail(sender),
TenantID: mailAccountOwner(extractAuthUser(line)),
})
}
// Also detect common spam subject patterns
if strings.Contains(subjectLower, "password") && strings.Contains(subjectLower, "smtp") {
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "email_credential_leak",
Message: fmt.Sprintf("Suspicious email subject with SMTP/password keywords from %s", sender),
Details: truncateDaemon(line, 300),
Mailbox: mailboxOnly(sender),
Domain: extractDomainFromEmail(sender),
TenantID: mailAccountOwner(extractAuthUser(line)),
})
}
}
// 5. Authentication from known bulk mail services. The login and the
// connecting host come from Exim's metadata; the subject is sender text.
if authUser := extractAuthUser(line); authUser != "" {
if service := bulkMailServiceHost(line); service != "" {
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "email_compromised_account",
Message: fmt.Sprintf("Compromised email account %s authenticated from bulk mail service %s", authUser, service),
Details: truncateDaemon(line, 300),
Mailbox: mailboxOnly(authUser),
Domain: extractDomainFromEmail(authUser),
TenantID: mailAccountOwner(authUser),
})
}
}
// 6. Dovecot auth failure - brute force indicator
// Format: "dovecot_login authenticator failed for (HELO) [IP]:port: 535 ... (set_id=user@domain)"
if failure := eximlog.AuthFailure(line); strings.HasPrefix(failure.Authenticator, "dovecot_") {
ip, account := failure.ClientIP, failure.Account
msg := "Email authentication failure"
if account != "" {
msg += " for " + account
}
if ip != "" {
msg += " from " + ip
}
// cPanel-local mailboxes log set_id as a bare local part with no
// "@domain"; treating it as a Mailbox would leave the structured
// field empty (mailboxOnly drops bare names) and force the
// correlator to fall back to SourceIP, splitting one targeted
// account across many attacker IPs. Route the bare form to
// TenantID so the incident groups by account.
mailbox, domain, tenant := splitMailAccount(account)
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "email_auth_failure_realtime",
Message: msg,
Details: truncateDaemon(line, 300),
SourceIP: ip,
Mailbox: mailbox,
Domain: domain,
TenantID: tenant,
})
}
// 7. DKIM signing failures
if dkimDomain := parseDKIMFailureDomain(line); dkimDomain != "" {
dedupKey := "dkim_fail:" + dkimDomain
if db := store.Global(); db != nil {
lastAlert := db.GetMetaString(dedupKey)
if lastAlert == "" || isDedupExpired(lastAlert, 24*time.Hour) {
_ = db.SetMetaString(dedupKey, time.Now().Format(time.RFC3339))
findings = append(findings, alert.Finding{
Severity: alert.Warning,
Check: "email_dkim_failure",
Message: fmt.Sprintf("DKIM signing failed for %s - check key file and DNS TXT record", dkimDomain),
Details: truncateDaemon(line, 300),
Timestamp: time.Now(),
Domain: dkimDomain,
})
}
}
}
// 8. SPF/DMARC outbound rejections
if spfDomain, spfReason := parseSPFDMARCRejection(line); spfDomain != "" {
dedupKey := "spf_reject:" + spfDomain
if db := store.Global(); db != nil {
lastAlert := db.GetMetaString(dedupKey)
if lastAlert == "" || isDedupExpired(lastAlert, 24*time.Hour) {
_ = db.SetMetaString(dedupKey, time.Now().Format(time.RFC3339))
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "email_spf_rejection",
Message: fmt.Sprintf("Outbound mail from %s rejected due to SPF/DMARC failure", spfDomain),
Details: fmt.Sprintf("Reason: %s\n%s", spfReason, truncateDaemon(line, 200)),
Timestamp: time.Now(),
Domain: spfDomain,
})
}
}
}
// 9. Outbound rate limiting for authenticated users
if strings.Contains(line, " <= ") && strings.Contains(line, "A=dovecot_") {
authUser := extractAuthUser(line)
if authUser != "" {
rateFindings := checkEmailRate(authUser, cfg)
findings = append(findings, rateFindings...)
}
}
// 10. Cloud-relay credential abuse (multiple authenticated sends from
// distinct cloud-provider IPs for the same mailbox).
// Use the AUTH identity (A=dovecot_*:<user>), not the envelope-from:
// the envelope-from can be forged by the attacker, while the AUTH
// identity is the credential actually being abused and the one we
// must lock out.
for _, f := range parseCloudRelayFinding(line, cfg) {
handleCloudRelayCredentialAbuse(cfg, extractAuthUser(line), f.Timestamp)
findings = append(findings, f)
}
// 11. Authenticated sender profile: country diversity, source address
// churn and recipient fan-out judged against the mailbox's own history.
for _, f := range parseSenderProfileFinding(line, cfg, time.Now()) {
if f.Severity == alert.Critical {
respondToMailboxCompromise(cfg, extractAuthUser(line), "email_compromised_account", "authenticated sends outside the mailbox's own source and recipient history", f.Timestamp)
}
findings = append(findings, f)
}
if eng := PHPRelayEvaluator(); eng != nil {
findings = append(findings, eng.parsePHPRelayAccountVolume(line, time.Now())...)
}
return findings
}
// extractEximDomain extracts a domain from an exim log line mentioning
// "Domain X has exceeded".
func extractEximDomain(line string) string {
idx := strings.Index(line, "Domain ")
if idx < 0 {
return ""
}
rest := line[idx+7:]
if sp := strings.IndexByte(rest, ' '); sp > 0 {
return rest[:sp]
}
return rest
}
// parseSessionNew reads a new cPanel or webmail session from session_log.
// cpsrvd writes it only for an authenticated session.
func parseSessionNew(line string) (app, ip, account string, ok bool) {
r, ok := checks.ParseSessionRecord(line)
if !ok || r.Event != "NEW" || (r.Service != "cpaneld" && r.Service != "webmaild") {
return "", "", "", false
}
addr, ok := r.ClientAddr()
if !ok {
return "", "", "", false
}
return r.Service, addr.String(), r.Account, true
}
// isInfraIPDaemon is the realtime path's infra test. It delegates to the
// checks package so the two cannot drift: an earlier copy here lacked the
// Cloudflare branch, so realtime findings carried edge addresses that the
// scan path filters, and those were exported and blocked as attackers.
func isInfraIPDaemon(ip string, infraNets []string) bool {
return checks.IsInfraIP(ip, infraNets)
}
// mergeInfraIPs combines top-level infra IPs with firewall-specific ones,
// deduplicating entries. This allows the firewall to include additional CIDRs
// (e.g. server's own range) that need port access but shouldn't suppress alerts.
func mergeInfraIPs(topLevel, fwSpecific []string) []string {
return firewall.MergeInfraIPs(topLevel, fwSpecific)
}
// outgoingMailHoldUsersPath is the cPanel file listing users currently
// under OUTGOING_MAIL_HOLD. Read-only; cPanel/WHM owns mutation. var
// (not const) so tests can point it at a fixture.
var outgoingMailHoldUsersPath = "/etc/outgoing_mail_hold_users"
// whmapi1HoldExec invokes `whmapi1 hold_outgoing_email user=<user>`.
// Declared as var so tests can replace it without spawning whmapi1.
var whmapi1HoldExec = func(user string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return uapiCommand(ctx, "whmapi1", "hold_outgoing_email", "user="+user, "--output=json")
}
// whmapi1ReleaseExec invokes `whmapi1 release_outgoing_email user=<user>`.
// var so tests can replace it without spawning whmapi1.
var whmapi1ReleaseExec = func(user string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return uapiCommand(ctx, "whmapi1", "release_outgoing_email", "user="+user, "--output=json")
}
// userOnOutgoingMailHold reports whether the cPanel user already appears
// in /etc/outgoing_mail_hold_users. Used to short-circuit redundant
// whmapi1 calls when exim re-emits "exceeded max defers/failures" every
// retry hour while the hold is already active.
func userOnOutgoingMailHold(user string) bool {
held, _ := outgoingMailHoldState(user)
return held
}
// outgoingMailHoldState reads whether cPanel holds the user's outgoing mail.
// A missing list means no account is held; any other read failure is an
// error, so a caller can tell "not held" from "cannot tell".
func outgoingMailHoldState(user string) (bool, error) {
if user == "" {
return false, nil
}
f, err := os.Open(outgoingMailHoldUsersPath)
if errors.Is(err, fs.ErrNotExist) {
return false, nil
}
if err != nil {
return false, err
}
defer func() { _ = f.Close() }()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
if strings.TrimSpace(scanner.Text()) == user {
return true, nil
}
}
return false, scanner.Err()
}
// maybeHoldOutgoingMail holds outgoing mail for the account that owns a
// domain after sustained mail abuse. It goes through the same entry point as
// a mailbox suspension, so it honours auto-response and dry-run, is recorded,
// and can be released. It returns true only when a hold was applied or is
// already active, so callers can keep their hold-dedup bookkeeping accurate.
func maybeHoldOutgoingMail(cfg *config.Config, domainOrEmail string) bool {
subject := domainOrEmail
if at := strings.LastIndexByte(subject, '@'); at >= 0 {
subject = subject[at+1:]
}
stopped, _ := respondToCompromise(cfg, mailboxCompromise{
Subject: subject,
Check: "email_spam_outbreak",
Reason: "outbound spam blast with deferred and failed deliveries",
Evidence: evidenceSending,
At: time.Now(),
})
return stopped
}
// autoSuspendOutgoingMail calls whmapi1 to hold outgoing mail for the cPanel
// account that owns the given domain or email address. It returns true when
// the hold is applied or already active. Declared as var so tests can swap in
// a recorder without spawning whmapi1.
var autoSuspendOutgoingMail = autoSuspendOutgoingMailReal
func autoSuspendOutgoingMailReal(domainOrEmail string) bool {
if domainOrEmail == "" {
return false
}
domain := domainOrEmail
if atIdx := strings.LastIndexByte(domain, '@'); atIdx >= 0 {
domain = domain[atIdx+1:]
}
user := lookupCPanelUser(domain)
if user == "" {
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: could not find cPanel user for domain %s\n",
time.Now().Format("2006-01-02 15:04:05"), domain)
return false
}
return holdCPanelAccount(user)
}
// holdCPanelAccount holds one cPanel account's outgoing mail. var so tests
// can swap in a recorder without spawning whmapi1.
var holdCPanelAccount = holdCPanelAccountReal
func holdCPanelAccountReal(user string) bool {
// Skip if cPanel already lists this user as held. Re-issuing the
// hold has no operational effect, but on a sustained exim retry
// loop (queued bounces keep the defer/fail ratio above threshold
// hour after hour) the redundant whmapi1 calls produce a stream
// of "AUTO-SUSPEND" log lines that look like a fresh incident.
if userOnOutgoingMailHold(user) {
return true
}
out, err := whmapi1HoldExec(user)
if err == nil {
err = whmapi1ResultError(out)
}
rec := actionlog.Record{
Op: "respond.hold_outgoing_mail",
Action: "hold_outgoing_email",
Target: user,
Account: user,
Command: []string{"whmapi1", "hold_outgoing_email", "user=" + user, "--output=json"},
Undo: "whmapi1 release_outgoing_email " + mailboxShellArg("user="+user),
Result: actionlog.Applied,
}
if err != nil {
rec.Result, rec.Error = actionlog.Failed, err.Error()
actionlog.Write(rec)
fmt.Fprintf(os.Stderr, "[%s] auto-suspend: whmapi1 hold_outgoing_email failed for %s: %v\n%s\n",
time.Now().Format("2006-01-02 15:04:05"), user, err, string(out))
return false
}
actionlog.Write(rec)
fmt.Fprintf(os.Stderr, "[%s] AUTO-SUSPEND: outgoing mail held for cPanel user %s\n",
time.Now().Format("2006-01-02 15:04:05"), user)
return true
}
// userdomainsPath is the cPanel domain→user map file. var (not const)
// so tests can point it at a fixture under t.TempDir(). Production must
// not mutate at runtime.
var userdomainsPath = "/etc/userdomains"
// lookupCPanelUser finds the cPanel username that owns a domain.
// Reads userdomainsPath which maps "domain: user" per line.
func lookupCPanelUser(domain string) string {
f, err := os.Open(userdomainsPath)
if err != nil {
return ""
}
defer func() { _ = f.Close() }()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
parts := strings.SplitN(line, ":", 2)
if len(parts) != 2 {
continue
}
d := strings.TrimSpace(parts[0])
u := strings.TrimSpace(parts[1])
if strings.EqualFold(d, domain) {
return u
}
}
return ""
}
// extractMailHoldSender extracts the account/domain from outgoing mail hold messages.
//
// Two formats:
//
// "Sender user@domain has an outgoing mail hold" -> "user@domain"
// "Domain example.com has an outgoing mail hold" -> "example.com"
func extractMailHoldSender(line string) string {
// Try "Sender user@domain" first
if idx := strings.Index(line, "Sender "); idx >= 0 {
rest := line[idx+7:]
if sp := strings.IndexByte(rest, ' '); sp > 0 {
return rest[:sp]
}
return rest
}
// Try "Domain example.com" format
if idx := strings.Index(line, "Domain "); idx >= 0 {
rest := line[idx+7:]
if sp := strings.IndexByte(rest, ' '); sp > 0 {
return rest[:sp]
}
return rest
}
return ""
}
func recordRecentOutgoingMailHold(domain string) {
if domain == "" {
return
}
db := store.Global()
if db == nil {
return
}
_ = db.SetMetaString("email_hold_seen:"+domain, time.Now().Format(time.RFC3339))
}
func recentOutgoingMailHold(domain string) bool {
if domain == "" {
return false
}
db := store.Global()
if db == nil {
return false
}
stored := db.GetMetaString("email_hold_seen:" + domain)
return stored != "" && !isDedupExpired(stored, recentOutgoingMailHoldWindow)
}
func truncateDaemon(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
}
// parseDKIMFailureDomain extracts domain from "DKIM: signing failed for {domain}"
func parseDKIMFailureDomain(line string) string {
const prefix = "DKIM: signing failed for "
idx := strings.Index(line, prefix)
if idx < 0 {
return ""
}
rest := line[idx+len(prefix):]
end := strings.IndexAny(rest, ": \t\n")
if end < 0 {
return rest
}
return rest[:end]
}
// parseSPFDMARCRejection extracts SENDER domain and rejection reason from
// exim ** permanent failure lines. Sender comes from <envelope_sender>.
func parseSPFDMARCRejection(line string) (senderDomain, reason string) {
starIdx := strings.Index(line, " ** ")
if starIdx < 0 {
return "", ""
}
// Extract envelope sender from <sender@domain> - search AFTER the **
// marker to avoid matching earlier <> fields (e.g. H=<hostname>).
rest := line[starIdx:]
ltIdx := strings.Index(rest, "<")
gtIdx := strings.Index(rest, ">")
if ltIdx < 0 || gtIdx < 0 || gtIdx <= ltIdx+1 {
return "", ""
}
sender := rest[ltIdx+1 : gtIdx]
atIdx := strings.LastIndexByte(sender, '@')
if atIdx < 0 || atIdx >= len(sender)-1 {
return "", ""
}
domain := sender[atIdx+1:]
// Extract rejection reason after last " : "
colonIdx := strings.LastIndex(line, " : ")
if colonIdx < 0 {
return "", ""
}
reason = strings.TrimSpace(line[colonIdx+3:])
if !isSPFDMARCRelated(reason) {
return "", ""
}
if len(reason) > 200 {
reason = reason[:200]
}
return domain, reason
}
// isSPFDMARCRelated checks if a rejection reason is SPF/DMARC related.
// Generic 5.7.1 alone is NOT sufficient - requires explicit auth keywords.
func isSPFDMARCRelated(reason string) bool {
if reason == "" {
return false
}
lower := strings.ToLower(reason)
for _, kw := range []string{"spf", "dmarc", "dkim"} {
if strings.Contains(lower, kw) {
return true
}
}
for _, code := range []string{"5.7.23", "5.7.25", "5.7.26"} {
if strings.Contains(lower, code) {
return true
}
}
if strings.Contains(lower, "5.7.1") {
if strings.Contains(lower, "authentication") || strings.Contains(lower, "ptr record") ||
strings.Contains(lower, "sender policy") || strings.Contains(lower, "alignment") {
return true
}
}
return false
}
// isDedupExpired checks if a stored RFC3339 timestamp is older than the given duration.
func isDedupExpired(stored string, window time.Duration) bool {
t, err := time.Parse(time.RFC3339, stored)
if err != nil {
return true
}
return time.Since(t) > window
}
// --- Outbound email rate limiting ---
// rateWindow tracks send timestamps for a single authenticated user.
type rateWindow struct {
mu sync.Mutex
times []time.Time
alerted string // last threshold level alerted ("warn" or "crit") - prevents repeated alerts per window
}
// add appends a timestamp to the window.
func (rw *rateWindow) add(t time.Time) {
rw.times = append(rw.times, t)
}
// countInWindow returns the number of timestamps within the window duration.
// Caller must hold rw.mu.
func (rw *rateWindow) countInWindow(now time.Time, window time.Duration) int {
cutoff := now.Add(-window)
count := 0
for _, t := range rw.times {
if t.After(cutoff) {
count++
}
}
return count
}
// prune removes timestamps older than the window duration and resets the
// alerted flag when the count drops below thresholds. Caller must hold rw.mu.
func (rw *rateWindow) prune(now time.Time, window time.Duration) {
rw.times = pruneTimes(rw.times, now.Add(-window))
}
// emailRateWindows tracks per-user send rate windows.
var emailRateWindows sync.Map // map[string]*rateWindow
// extractAuthUser shares the Exim identity parser with scheduled mail checks.
func extractAuthUser(line string) string {
return eximlog.AuthenticatedUser(line)
}
// bulkMailServiceHost returns the known bulk mail service named by the
// connecting host or its greeting in the arrival's H= field, or "".
func bulkMailServiceHost(line string) string {
start, ok := eximlog.HFieldStart(line)
if !ok {
return ""
}
_, end := eximlog.HFieldClientIPAndEnd(line[start:])
if end == 0 {
return ""
}
host := strings.ToLower(line[start : start+end])
for _, service := range []string{
"truelist.io", "sendinblue.com", "mailspree.co",
"bulkmailer.", "massmailsoftware.", "sendblaster.",
} {
if strings.Contains(host, service) {
return service
}
}
return ""
}
// isHighVolumeSender checks if a user is in the high-volume senders allowlist.
func isHighVolumeSender(user string, allowlist []string) bool {
for _, allowed := range allowlist {
if strings.EqualFold(user, allowed) {
return true
}
}
return false
}
// extractDomainFromEmail returns the domain part of an email address.
func extractDomainFromEmail(email string) string {
idx := strings.LastIndexByte(email, '@')
if idx < 0 || idx >= len(email)-1 {
return ""
}
return email[idx+1:]
}
// mailboxOnly returns the input only when it looks like a full mailbox
// (contains '@'); otherwise returns "". Used by realtime emit sites that
// receive either "user@domain" or a bare domain — the bare domain belongs
// in the Domain field, not Mailbox, so the correlator does not collapse
// distinct mailboxes onto a domain key.
func mailboxOnly(s string) string {
if strings.IndexByte(s, '@') < 0 {
return ""
}
return s
}
// splitMailAccount classifies an authenticated mail account string into
// the three correlation fields. A full mailbox ("user@domain") routes to
// Mailbox + Domain. A bare local part (cPanel-style, no '@') routes to
// TenantID so the incident correlator groups by account, not by attacker
// SourceIP. An empty input returns three empty strings.
func splitMailAccount(account string) (mailbox, domain, tenant string) {
if account == "" {
return "", "", ""
}
if strings.IndexByte(account, '@') < 0 {
return "", "", account
}
return account, extractDomainFromEmail(account), ""
}
// hasRecentCompromisedFinding checks if there's a recent email_compromised_account
// or email_spam_outbreak finding for the given domain (suppresses rate alerts).
func hasRecentCompromisedFinding(domain string) bool {
emailRateSuppressed.mu.Lock()
defer emailRateSuppressed.mu.Unlock()
if ts, ok := emailRateSuppressed.domains[domain]; ok {
if time.Since(ts) < time.Hour {
return true
}
delete(emailRateSuppressed.domains, domain)
}
return false
}
// emailRateSuppressed tracks domains with recent compromised/spam findings.
var emailRateSuppressed = struct {
mu sync.Mutex
domains map[string]time.Time
}{domains: make(map[string]time.Time)}
// RecordCompromisedDomain marks a domain as having a recent compromised finding.
// Called from parseEximLogLine when email_compromised_account or email_spam_outbreak fires.
func RecordCompromisedDomain(domain string) {
emailRateSuppressed.mu.Lock()
defer emailRateSuppressed.mu.Unlock()
emailRateSuppressed.domains[domain] = time.Now()
}
// domainHasOutboundBlast reports whether authenticated senders under the given
// domain have produced enough outbound volume within the rate window to
// corroborate an actual spam outbreak. A cPanel defer/fail governor trip alone
// is not such evidence. Returns false when rate thresholds are unconfigured, so
// an operator who has not tuned the rate window never auto-holds on a bare
// governor line.
func domainHasOutboundBlast(domain string, cfg *config.Config) bool {
if domain == "" || cfg == nil {
return false
}
threshold := cfg.EmailProtection.RateWarnThreshold
windowDur := time.Duration(cfg.EmailProtection.RateWindowMin) * time.Minute
if threshold <= 0 || windowDur <= 0 {
return false
}
now := time.Now()
total := 0
emailRateWindows.Range(func(key, val any) bool {
user, ok := key.(string)
if !ok || !strings.EqualFold(extractDomainFromEmail(user), domain) {
return true
}
rw, ok := val.(*rateWindow)
if !ok {
return true
}
rw.mu.Lock()
total += rw.countInWindow(now, windowDur)
rw.mu.Unlock()
return total < threshold // stop iterating once corroborated
})
return total >= threshold
}
// checkEmailRate processes an outbound email for rate limiting.
// Returns findings if thresholds are exceeded.
func checkEmailRate(user string, cfg *config.Config) (findings []alert.Finding) {
// Registered before the unlock defer so owner I/O runs after it.
defer func() { stampMailAccountOwner(findings, user) }()
// Guard: skip if thresholds are zero (misconfigured or disabled)
if cfg.EmailProtection.RateWarnThreshold <= 0 || cfg.EmailProtection.RateCritThreshold <= 0 {
return nil
}
if isHighVolumeSender(user, cfg.EmailProtection.HighVolumeSenders) {
return nil
}
// Load or create rate window for this user
val, _ := emailRateWindows.LoadOrStore(user, &rateWindow{})
rw := val.(*rateWindow)
now := time.Now()
windowDur := time.Duration(cfg.EmailProtection.RateWindowMin) * time.Minute
rw.mu.Lock()
defer rw.mu.Unlock()
// Check domain suppression BEFORE adding to window - prevents
// phantom rate inflation for suppressed domains.
domain := extractDomainFromEmail(user)
if domain != "" && hasRecentCompromisedFinding(domain) {
return nil
}
rw.add(now)
count := rw.countInWindow(now, windowDur)
// Reset alerted state when count drops below warn threshold -
// allows re-alerting on the next burst after the window slides.
if count < cfg.EmailProtection.RateWarnThreshold {
rw.alerted = ""
}
mailbox, domain, _ := splitMailAccount(user)
if count >= cfg.EmailProtection.RateCritThreshold {
if rw.alerted != "crit" {
rw.alerted = "crit"
findings = append(findings, alert.Finding{
Severity: alert.Critical,
Check: "email_rate_critical",
Message: fmt.Sprintf("Email rate CRITICAL: %s sent %d messages in %d minutes (threshold: %d)", user, count, cfg.EmailProtection.RateWindowMin, cfg.EmailProtection.RateCritThreshold),
Details: fmt.Sprintf("User: %s\nMessages in window: %d\nWindow: %d minutes\nThreshold: %d", user, count, cfg.EmailProtection.RateWindowMin, cfg.EmailProtection.RateCritThreshold),
Mailbox: mailbox,
Domain: domain,
})
}
} else if count >= cfg.EmailProtection.RateWarnThreshold {
if rw.alerted != "warn" && rw.alerted != "crit" {
rw.alerted = "warn"
findings = append(findings, alert.Finding{
Severity: alert.High,
Check: "email_rate_warning",
Message: fmt.Sprintf("Email rate WARNING: %s sent %d messages in %d minutes (threshold: %d)", user, count, cfg.EmailProtection.RateWindowMin, cfg.EmailProtection.RateWarnThreshold),
Details: fmt.Sprintf("User: %s\nMessages in window: %d\nWindow: %d minutes\nThreshold: %d", user, count, cfg.EmailProtection.RateWindowMin, cfg.EmailProtection.RateWarnThreshold),
Mailbox: mailbox,
Domain: domain,
})
}
}
return findings
}
// StartEmailRateEviction starts a background goroutine that prunes expired
// rate windows every 10 minutes. Same pattern as StartModSecEviction.
func StartEmailRateEviction(stopCh <-chan struct{}) {
obs.Go("email-rate-eviction", func() {
ticker := time.NewTicker(10 * time.Minute)
defer ticker.Stop()
for {
select {
case <-stopCh:
return
case now := <-ticker.C:
evictEmailRateWindows(now)
}
}
})
}
// evictEmailRateWindows prunes all per-user rate windows and deletes empty entries.
func evictEmailRateWindows(now time.Time) {
// Use a generous 60-minute eviction window to avoid premature deletion.
// The actual rate window is checked during rate evaluation.
evictWindow := 60 * time.Minute
emailRateWindows.Range(func(key, val any) bool {
rw := val.(*rateWindow)
rw.mu.Lock()
rw.prune(now, evictWindow)
empty := len(rw.times) == 0
if empty {
rw.alerted = ""
}
rw.mu.Unlock()
if empty {
emailRateWindows.Delete(key)
}
return true
})
// Also prune the suppressed domains map
emailRateSuppressed.mu.Lock()
for domain, ts := range emailRateSuppressed.domains {
if time.Since(ts) > time.Hour {
delete(emailRateSuppressed.domains, domain)
}
}
emailRateSuppressed.mu.Unlock()
}
package daemon
import (
"path/filepath"
"strings"
)
// maxWebRootGlobs bounds the per-call glob table kept on the stack. Hosts
// configure a handful of account and document root globs; a longer list
// still matches correctly, through the unfiltered path.
const maxWebRootGlobs = 16
// pathMatchesWebRootPatterns reports whether any directory containing path,
// from its parent up to the root, matches one of the root globs.
//
// It runs for every watched write that reaches the account and document root
// tests, so it avoids matching a glob against a directory it cannot match: a
// glob with no character class and no escape matches only a name with as many
// separators as itself, because its wildcards never match one. A glob with a
// class or an escape may match a separator and is tried at every level.
func pathMatchesWebRootPatterns(path string, patterns []string) bool {
if len(patterns) > maxWebRootGlobs {
return matchWebRootsAtEveryLevel(path, patterns)
}
var globs [maxWebRootGlobs]string
var depths [maxWebRootGlobs]int
for i, pattern := range patterns {
globs[i] = filepath.Clean(pattern)
depths[i] = -1
if !strings.ContainsAny(globs[i], `[\`) {
depths[i] = strings.Count(globs[i], "/")
}
}
dir := filepath.Clean(filepath.Dir(path))
depth := strings.Count(dir, "/")
for {
for i := range patterns {
if depths[i] >= 0 && depths[i] != depth {
continue
}
if matched, err := filepath.Match(globs[i], dir); err == nil && matched {
return true
}
}
// dir is clean, so its parent is the prefix before the last
// separator; filepath.Dir would clean each one again.
switch i := strings.LastIndexByte(dir, '/'); {
case i > 0:
dir, depth = dir[:i], depth-1
case dir == "/" || dir == ".":
return false
case i == 0:
dir, depth = "/", 1
default:
dir, depth = ".", 0
}
}
}
func matchWebRootsAtEveryLevel(path string, patterns []string) bool {
dir := filepath.Clean(filepath.Dir(path))
for {
for _, pattern := range patterns {
if matched, err := filepath.Match(filepath.Clean(pattern), dir); err == nil && matched {
return true
}
}
parent := filepath.Dir(dir)
if parent == dir {
return false
}
dir = parent
}
}
package daemon
import "regexp"
// looksLikePHPWebshell returns true when PHP file content exhibits the
// canonical realtime-detectable webshell shapes:
// 1. A request superglobal ($_GET / $_POST / $_REQUEST / $_COOKIE /
// php://input) flowing into a code-execution primitive
// (eval / assert / system / passthru / exec / shell_exec / proc_open
// / popen / create_function), with optional decoder layers
// (base64_decode / gzinflate / str_rot13).
// 2. eval/assert wrapping a decoder of an arbitrary base64 / gzinflate
// blob (the obfuscated-payload primitive).
//
// Returns false on legitimate code that uses dangerous functions in
// non-attack contexts (Pear Text_Diff/Engine/shell.php's shell_exec call
// to Unix `diff`, TinyMCE charmap.php's static glyph data array).
func looksLikePHPWebshell(data []byte) bool {
if len(data) == 0 {
return false
}
// No inner byte cap. The fanotify callers own the read window and
// record when that window is hit. A duplicate cap here would silently
// shorten any caller that legitimately passed a larger buffer, hiding
// truncation from that caller without adding protection beyond the
// upstream reader limit.
for _, re := range webshellContentRegexes {
if re.Match(data) {
return true
}
}
return requestVariableFlowsToDangerousFunction(data)
}
// webshellContentRegexes are the realtime-detection-grade content patterns
// for looksLikePHPWebshell. Compiled once at package init.
var webshellContentRegexes = []*regexp.Regexp{
// Request superglobal directly piped into a code-execution primitive
// in the same expression (with optional decoder layers).
regexp.MustCompile(`(?i)\b(?:eval|assert|system|passthru|exec|shell_exec|proc_open|popen|create_function)\s*\(\s*(?:gzinflate\s*\(\s*|str_rot13\s*\(\s*|base64_decode\s*\(\s*|@\s*)*\s*\$_(?:GET|POST|REQUEST|COOKIE|FILES|SERVER)\b`),
// php://input piped into eval/assert/system in the same expression.
regexp.MustCompile(`(?i)\b(?:eval|assert|system|passthru|exec|shell_exec|proc_open|popen|create_function)\s*\(\s*(?:gzinflate\s*\(\s*|str_rot13\s*\(\s*|base64_decode\s*\(\s*|@\s*)*\s*file_get_contents\s*\(\s*['"]php://input`),
// eval/assert wrapping a base64/gzinflate/str_rot13 decoder of a
// long literal blob (obfuscated-payload primitive).
regexp.MustCompile(`(?i)\b(?:eval|assert)\s*\(\s*(?:gzinflate\s*\(\s*|str_rot13\s*\(\s*)?base64_decode\s*\(\s*['"][A-Za-z0-9+/=]{40,}`),
}
var (
requestAssignmentRegex = regexp.MustCompile(`(?is)\$([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(?:@?\s*)?(?:\$_(?:GET|POST|REQUEST|COOKIE|FILES|SERVER)\b(?:\s*\[[^\]]{0,200}\])?|file_get_contents\s*\(\s*['"]php://input['"]\s*\))[^;]{0,200};`)
dangerousVariableCallRegex = regexp.MustCompile(`(?is)\b(?:eval|assert|system|passthru|exec|shell_exec|proc_open|popen|create_function)\s*\(\s*(?:@?\s*)?\$([A-Za-z_][A-Za-z0-9_]*)\b`)
)
func requestVariableFlowsToDangerousFunction(data []byte) bool {
matches := requestAssignmentRegex.FindAllSubmatchIndex(data, -1)
for _, match := range matches {
if len(match) < 4 {
continue
}
varName := string(data[match[2]:match[3]])
windowEnd := match[1] + 800
if windowEnd > len(data) {
windowEnd = len(data)
}
for _, call := range dangerousVariableCallRegex.FindAllSubmatch(data[match[1]:windowEnd], -1) {
if len(call) == 2 && string(call[1]) == varName {
return true
}
}
}
return false
}
package daemon
import (
"context"
"errors"
"fmt"
"os"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/workerproc"
"github.com/pidginhost/csm/internal/yara"
"github.com/pidginhost/csm/internal/yaraworker"
)
// yaraMetricsOnce guards registration of the yara-worker failure
// counter hook so a baseline re-run or a second daemon instance in the
// same test binary does not panic with "duplicate registration".
var yaraMetricsOnce sync.Once
// yaraWorkerOn reports whether the daemon should run YARA-X in a
// supervised child process. The field is a *bool tri-state: nil means
// "use system default" (true, per ROADMAP item 2 follow-up), *true is
// explicit opt-in, *false is explicit opt-out.
//
// A nil cfg falls back to false so a pathological caller does not
// accidentally spin up a worker; production never passes nil.
func yaraWorkerOn(cfg *config.Config) bool {
if cfg == nil {
return false
}
if cfg.Signatures.YaraWorkerEnabled == nil {
return true
}
return *cfg.Signatures.YaraWorkerEnabled
}
// yaraWorkerWatcher names the YARA-X worker in the watcher status map.
const yaraWorkerWatcher = "yara_worker"
// Limits for the YARA worker units. A scan gets as long as the old socket
// client allowed. A fresh worker loading Forge full takes about 3.4 s, and
// compiling it about 10 s.
const (
yaraScanTimeout = 30 * time.Second
yaraLoadTimeout = 60 * time.Second
yaraCompileTimeout = 2 * time.Minute
)
// yaraBootCompileTimeout bounds the first compile, which runs before the
// daemon reports ready to systemd. A compile still running then is handed to
// the background retry, which has the full compile timeout.
var yaraBootCompileTimeout = 45 * time.Second
// yaraRetryMinBackoff is the first wait of the background compile retry; a
// seam so tests reach the persistent-failure finding quickly. Init reads it
// once: a retry left running by an earlier test must not race a change.
var yaraRetryMinBackoff = time.Second
// yaraWorkerLauncher is a seam for tests that must not start units.
var yaraWorkerLauncher = workerproc.UnitLauncher
// yaraEngineAvailable is a seam: plain-build tests force the yara build's
// answer.
var yaraEngineAvailable = yara.Available
// maxYaraFindingErr bounds the worker error text a finding quotes.
const maxYaraFindingErr = 512
// initYaraBackend wires up either the locked-down worker backend (default)
// or the in-process scanner (when config.Signatures.YaraWorkerEnabled is
// explicitly *false). Both register as the yara package's active backend.
//
// In worker mode the daemon never compiles or scans: rules compile in a
// short-lived compile unit and scans run in a scan unit, both without root,
// network or host files. The backend is installed before the first compile,
// so a compile that cannot run leaves scans failing closed rather than YARA
// skipped.
func (d *Daemon) initYaraBackend() error {
if !yaraWorkerOn(d.cfg) {
if yaraScanner := yara.Init(d.cfg.Signatures.RulesDir, d.cfg.Signatures.DisabledRules...); yaraScanner != nil {
fmt.Fprintf(os.Stderr, "[%s] YARA-X scanner active: %d rule file(s)\n", ts(), yaraScanner.RuleCount())
}
return nil
}
logf := func(format string, args ...any) {
fmt.Fprintf(os.Stderr, "[%s] yara-worker: "+format+"\n", append([]any{ts()}, args...)...)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
for _, prefix := range []string{yaraworker.ScanUnitPrefix, yaraworker.CompileUnitPrefix} {
if err := stopParseWorkerUnits(ctx, prefix+"-"); err != nil {
logf("stop leftover units: %v", err)
}
}
cancel()
hidden := parseWorkerHiddenPaths()
compileUnit := yaraworker.CompileUnitConfig(d.binaryPath, logf)
compileUnit.InaccessiblePaths = hidden
b, err := yaraworker.NewBackend(yaraworker.BackendConfig{
ScanLaunch: d.yaraScanLauncher(hidden, logf),
CompileLaunch: d.checkedYaraLauncher(yaraWorkerLauncher(compileUnit)),
RulesDir: d.cfg.Signatures.RulesDir,
DisabledRules: d.cfg.Signatures.DisabledRules,
ScanTimeout: yaraScanTimeout,
LoadTimeout: yaraLoadTimeout,
CompileTimeout: yaraCompileTimeout,
OnFailure: d.onYaraWorkerFailure,
OnJob: d.onYaraWorkerJob,
Log: logf,
WarmAfterReload: true,
})
if err != nil {
return fmt.Errorf("creating yara worker backend: %w", err)
}
d.yaraSup = b
yara.SetActive(b)
d.registerQueueSource("yara", b)
yaraMetricsOnce.Do(func() {
metrics.RegisterCounterFunc(
"csm_yara_worker_restarts_total",
"Number of counted YARA-X scan worker failures, including failed starts.",
func() float64 {
var total uint64
for _, n := range b.Stats().Kills {
total += n
}
return float64(total)
},
)
})
if !yaraEngineAvailable() {
fmt.Fprintf(os.Stderr, "[%s] %s\n", ts(), yaraWorkerStatusLine(0))
return nil
}
bootCtx, cancelBoot := context.WithTimeout(context.Background(), yaraBootCompileTimeout)
err = b.ReloadContext(bootCtx)
cancelBoot()
switch {
case err == nil:
d.activateYaraBackend(b)
return nil
case retryableYaraCompile(err):
// The compile worker could not run: every YARA scan fails until it
// does. Report it where doctor and the status API look, and keep
// trying in the background.
d.MarkWatcher(yaraWorkerWatcher, false)
minBackoff := yaraRetryMinBackoff
obs.Go("yara-init-retry", func() { d.retryYaraStart(b, minBackoff) })
return fmt.Errorf("compiling yara rules (retrying in background): %w", err)
default:
d.MarkWatcher(yaraWorkerWatcher, false)
fmt.Fprintf(os.Stderr, "[%s] %s\n", ts(), yaraWorkerStatusLine(0))
d.reportYaraCompileStatus(b.CompileError())
d.reportRealtimeRuleCoverage(yamlRuleCount(), 0, false)
return nil
}
}
func retryableYaraCompile(err error) bool {
// Descriptor pressure belongs to the daemon, so workerproc deliberately
// leaves it out of worker failure accounting. It still needs a retry.
// A boot compile that outlived its bound is retried with the full
// compile timeout.
return workerproc.FailureReason(err) != "" || errors.Is(err, workerproc.ErrSpawn) ||
errors.Is(err, syscall.EMFILE) || errors.Is(err, syscall.ENFILE) ||
errors.Is(err, context.DeadlineExceeded)
}
// yaraScanLauncher starts scan workers under the live config's size caps:
// thresholds reload without a restart, and the worker reads a whole file or
// mail part into memory.
func (d *Daemon) yaraScanLauncher(hidden []string, logf func(string, ...any)) workerproc.Launcher {
var mu sync.Mutex
var limit uint64
var launch workerproc.Launcher
current := func() workerproc.Launcher {
mu.Lock()
defer mu.Unlock()
if want := yaraScanMemoryLimit(d.activeOrStartupCfg()); launch == nil || want != limit {
unit := yaraworker.ScanUnitConfig(d.binaryPath, want, logf)
unit.InaccessiblePaths = hidden
limit, launch = want, d.checkedYaraLauncher(yaraWorkerLauncher(unit))
}
return launch
}
current()
return func(ctx context.Context) (workerproc.Process, error) { return current()(ctx) }
}
// yaraScanMemoryLimit is the scan worker's memory cap: the loaded rules plus
// the largest content it may be handed, a file or, with mail scanning on, a
// mail part.
func yaraScanMemoryLimit(cfg *config.Config) uint64 {
largest := checks.FullScanMaxFileBytes(cfg)
if cfg != nil && cfg.EmailAV.Enabled && cfg.EmailAV.MaxAttachmentSize > largest {
largest = cfg.EmailAV.MaxAttachmentSize
}
// #nosec G115 -- both caps are positive: validation and defaults.
return yaraworker.ScanMemoryBase + uint64(largest)
}
// checkedYaraLauncher refuses each launch while the dynamic user cannot run
// the binary, so the failure reaches the scan as an error instead of every
// unit failing at exec.
func (d *Daemon) checkedYaraLauncher(launch workerproc.Launcher) workerproc.Launcher {
return func(ctx context.Context) (workerproc.Process, error) {
if err := checkParseWorkerCommand(d.binaryPath); err != nil {
return nil, fmt.Errorf("yara worker: %w", err)
}
return launch(ctx)
}
}
// activateYaraBackend reports a backend whose rules compiled. The watcher is
// left to the scan worker's own requests, so activation cannot erase a
// failure seen in the meantime.
func (d *Daemon) activateYaraBackend(b *yaraworker.Backend) {
fmt.Fprintf(os.Stderr, "[%s] %s\n", ts(), yaraWorkerStatusLine(b.RuleCount()))
d.reportRealtimeRuleCoverage(yamlRuleCount(), b.RuleCount(), true)
}
// yaraWorkerStatusLine describes the compiled rules at startup. A worker with
// zero rules matches nothing, so it must not be called active: that wording
// let a host report a healthy scanner while every scan checked against an
// empty rule set.
func yaraWorkerStatusLine(ruleCount int) string {
if ruleCount == 0 {
return "YARA-X worker has 0 rules compiled - scanning nothing until rules load"
}
return fmt.Sprintf("YARA-X worker active: %d rule(s) compiled, scanning in a locked-down unit", ruleCount)
}
// retryYaraStart re-attempts a first compile whose worker could not run, with
// capped exponential backoff, until it runs or the daemon stops, raising one
// finding once the failure is persistent. A compile that runs and finds the
// rules broken ends the retries: recompiling would fail the same way.
func (d *Daemon) retryYaraStart(b *yaraworker.Backend, minBackoff time.Duration) {
var rulesErr error
ok := retryStartWithStopContext(
func(ctx context.Context) error {
err := b.ReloadContext(ctx)
if err != nil && !retryableYaraCompile(err) {
rulesErr = err
return nil
}
return err
},
d.stopCh,
minBackoff, 60*time.Second, 3,
func(attempt int, err error) {
d.emitYaraFinding(alert.Critical, "yara_backend_unavailable",
fmt.Sprintf("YARA-X rules could not be compiled: the compile worker failed %d times (%v). Files and mail parts are reported as not scanned while CSM keeps retrying.", attempt, err))
},
)
if !ok {
return
}
if rulesErr != nil {
d.reportYaraCompileStatus(b.CompileError())
return
}
fmt.Fprintf(os.Stderr, "[%s] yara rules compiled after retrying\n", ts())
if b.RuleCount() == 0 {
// No scan worker is needed, and the compile worker just ran.
d.MarkWatcher(yaraWorkerWatcher, true)
}
d.activateYaraBackend(b)
}
func retryStartWithStopContext(start func(context.Context) error, stop <-chan struct{}, minBackoff, maxBackoff time.Duration, alertAfter int, onPersistent func(attempt int, err error)) bool {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
retryDone := make(chan struct{})
obs.Go("yara-init-retry-stop", func() {
select {
case <-stop:
cancel()
case <-retryDone:
}
})
ok := retryStart(
func() error { return start(ctx) },
stop,
minBackoff, maxBackoff, alertAfter,
onPersistent,
)
close(retryDone)
return ok
}
// retryStart calls start with capped exponential backoff until it returns nil,
// stop is signalled, or forever. onPersistent, if set, fires exactly once
// after alertAfter consecutive failures so a persistent outage raises a single
// finding. Returns true if start eventually succeeded.
func retryStart(start func() error, stop <-chan struct{}, minBackoff, maxBackoff time.Duration, alertAfter int, onPersistent func(attempt int, err error)) bool {
backoff := minBackoff
for attempt := 1; ; attempt++ {
select {
case <-time.After(backoff):
case <-stop:
return false
}
if stopRequested(stop) {
return false
}
if err := start(); err == nil {
return !stopRequested(stop)
} else if attempt == alertAfter && onPersistent != nil {
if stopRequested(stop) {
return false
}
onPersistent(attempt, err)
}
backoff *= 2
if backoff > maxBackoff {
backoff = maxBackoff
}
}
}
func stopRequested(stop <-chan struct{}) bool {
if stop == nil {
return false
}
select {
case <-stop:
return true
default:
return false
}
}
// reportYaraCompileStatus raises a finding when the rules failed to compile
// and none are loaded. A clean compile ("") is silent.
func (d *Daemon) reportYaraCompileStatus(compileErr string) {
if compileErr == "" {
return
}
d.emitYaraFinding(alert.Critical, "yara_worker_compile_failed",
fmt.Sprintf("YARA-X rules failed to compile: %s. Files and mail parts are reported as not scanned by YARA until the rules are fixed and reloaded.", compileErr))
}
// emitYaraFinding pushes a finding without blocking; a saturated alert channel
// is accounted for by the daemon's general drop counter.
func (d *Daemon) emitYaraFinding(sev alert.Severity, check, msg string) bool {
if stopRequested(d.stopCh) {
return false
}
finding := alert.Finding{Severity: sev, Check: check, Message: msg, Timestamp: time.Now()}
if alert.TryEnqueue(d.alertCh, finding) {
return true
} else {
atomic.AddInt64(&d.droppedAlerts, 1)
return false
}
}
// stopYaraBackend is called during daemon shutdown. Safe to call when
// worker mode is off.
func (d *Daemon) stopYaraBackend() {
if d.yaraSup == nil {
return
}
if err := d.yaraSup.Stop(); err != nil {
fmt.Fprintf(os.Stderr, "[%s] yara-worker stop: %v\n", ts(), err)
}
yara.SetActive(nil)
}
// onYaraWorkerJob marks the worker attached after a served request. It runs
// under the worker supervisor's lock and only takes the watcher lock.
func (d *Daemon) onYaraWorkerJob(outcome string, _ uint64) {
if outcome != workerproc.OutcomeOK || stopRequested(d.stopCh) {
return
}
d.MarkWatcher(yaraWorkerWatcher, true)
}
// onYaraWorkerFailure is called once per counted scan worker failure. It
// marks the watcher failed and emits a Critical finding at most once a
// minute: a file that kills the worker on every scan must not become one
// alert per scan. It runs under the worker supervisor's lock and must not
// block.
func (d *Daemon) onYaraWorkerFailure(reason string, err error) {
// stopCh closes at the top of shutdown; workers stopped from then on are
// not failures. A nil stopCh (zero-value Daemon, several tests) means
// running.
if stopRequested(d.stopCh) {
return
}
now := time.Now()
d.yaraCrashMu.Lock()
d.MarkWatcher(yaraWorkerWatcher, false)
// Only an emitted alert starts the quiet minute: failures seconds apart
// must not keep pushing it forward and silence a persistent fault.
if last := d.yaraLastCrashAlert; !last.IsZero() && now.Sub(last) < time.Minute {
d.yaraCrashMu.Unlock()
return
}
d.yaraLastCrashAlert = now
d.yaraCrashMu.Unlock()
detail := err.Error()
if len(detail) > maxYaraFindingErr {
detail = strings.ToValidUTF8(detail[:maxYaraFindingErr], "") + "..."
}
alert.TryEnqueue(d.alertCh, alert.Finding{
Severity: alert.Critical,
Check: "yara_worker_crashed",
Timestamp: now,
Message: fmt.Sprintf("YARA-X worker failed (%s: %s); files and mail parts are reported as not scanned until a replacement worker loads the rules.", reason, detail),
})
}
package emailav
import (
"encoding/binary"
"fmt"
"io"
"net"
"strings"
"time"
)
// ClamdScanner scans files via the clamd Unix socket using INSTREAM.
type ClamdScanner struct {
socketPath string
}
// NewClamdScanner creates a ClamdScanner that connects to clamd at the given socket path.
func NewClamdScanner(socketPath string) *ClamdScanner {
return &ClamdScanner{socketPath: socketPath}
}
func (s *ClamdScanner) Name() string { return "clamav" }
// Available checks if clamd is reachable by attempting a connection.
func (s *ClamdScanner) Available() bool {
conn, err := net.DialTimeout("unix", s.socketPath, 2*time.Second)
if err != nil {
return false
}
_ = conn.Close()
return true
}
// Scan streams the part to clamd via INSTREAM and returns the verdict.
func (s *ClamdScanner) Scan(part Part) (Verdict, error) {
if part.Size <= 0 {
return Verdict{}, fmt.Errorf("invalid part size: %d", part.Size)
}
f := io.NewSectionReader(part.File, 0, part.Size)
conn, err := net.DialTimeout("unix", s.socketPath, 5*time.Second)
if err != nil {
return Verdict{}, fmt.Errorf("connecting to clamd: %w", err)
}
defer func() { _ = conn.Close() }()
err = conn.SetDeadline(time.Now().Add(30 * time.Second))
if err != nil {
return Verdict{}, fmt.Errorf("setting deadline: %w", err)
}
// Send INSTREAM command
_, err = conn.Write([]byte("nINSTREAM\n"))
if err != nil {
return Verdict{}, fmt.Errorf("sending INSTREAM: %w", err)
}
// Stream file content in chunks
buf := make([]byte, 8192)
lenBuf := make([]byte, 4)
var sent int64
for {
n, readErr := f.Read(buf)
sent += int64(n)
if n > 0 {
// #nosec G115 -- n is bounded by len(buf)=8192; fits in uint32.
binary.BigEndian.PutUint32(lenBuf, uint32(n))
_, err = conn.Write(lenBuf)
if err != nil {
return Verdict{}, fmt.Errorf("sending chunk length: %w", err)
}
_, err = conn.Write(buf[:n])
if err != nil {
return Verdict{}, fmt.Errorf("sending chunk data: %w", err)
}
}
if readErr == io.EOF {
if sent != part.Size {
return Verdict{}, fmt.Errorf("reading part: got %d of %d bytes: %w", sent, part.Size, io.ErrUnexpectedEOF)
}
break
}
if readErr != nil {
return Verdict{}, fmt.Errorf("reading file: %w", readErr)
}
}
// Send terminator (4 zero bytes)
binary.BigEndian.PutUint32(lenBuf, 0)
_, err = conn.Write(lenBuf)
if err != nil {
return Verdict{}, fmt.Errorf("sending terminator: %w", err)
}
// Read response
resp := make([]byte, 4096)
n, err := conn.Read(resp)
if err != nil && err != io.EOF {
return Verdict{}, fmt.Errorf("reading response: %w", err)
}
return parseClamdResponse(string(resp[:n]))
}
// parseClamdResponse parses a clamd INSTREAM response line.
// "stream: OK\n" → clean
// "stream: Win.Trojan.Agent-123 FOUND\n" → infected
func parseClamdResponse(resp string) (Verdict, error) {
resp = strings.TrimSpace(resp)
if strings.HasSuffix(resp, "OK") {
return Verdict{Infected: false}, nil
}
if strings.HasSuffix(resp, "FOUND") {
// Extract signature: "stream: <sig> FOUND"
resp = strings.TrimPrefix(resp, "stream: ")
sig := strings.TrimSuffix(resp, " FOUND")
return Verdict{
Infected: true,
Signature: sig,
Severity: "critical",
}, nil
}
return Verdict{}, fmt.Errorf("unexpected clamd response: %q", resp)
}
package emailav
import (
"context"
"fmt"
"os"
"sync"
"time"
emime "github.com/pidginhost/csm/internal/mime"
"github.com/pidginhost/csm/internal/obs"
)
// Orchestrator runs multiple scanners in parallel against extracted email parts.
type Orchestrator struct {
scanners []Scanner
scanTimeout time.Duration
health *scanQueue
}
// NewOrchestrator creates an orchestrator with the given scanners and per-scan timeout.
func NewOrchestrator(scanners []Scanner, scanTimeout time.Duration) *Orchestrator {
return &Orchestrator{
scanners: scanners,
scanTimeout: scanTimeout,
health: newScanQueue(),
}
}
// ScanParts scans all extracted parts with all available engines.
// Fail-open: unavailable engines, timeouts, and errors are recorded but do not
// mark the message as infected.
func (o *Orchestrator) ScanParts(messageID string, parts []emime.ExtractedPart, partial bool) *ScanResult {
result := &ScanResult{
MessageID: messageID,
ScannedAt: time.Now(),
PartialExtraction: partial,
}
// Determine which engines are available
var available []Scanner
for _, s := range o.scanners {
if s.Available() {
available = append(available, s)
result.EnginesUsed = append(result.EnginesUsed, s.Name())
} else {
result.FailedEngines = append(result.FailedEngines, s.Name())
fmt.Fprintf(os.Stderr, "[emailav] engine %s unavailable\n", s.Name())
}
}
if len(available) == 0 {
// fail-open: no engines available - rate-limit the warning
result.AllEnginesDown = true
return result
}
// Scan each part with all available engines
for _, part := range parts {
findings, timedOut, errored := o.scanPart(part, available)
result.Findings = append(result.Findings, findings...)
result.TimedOutEngines = append(result.TimedOutEngines, timedOut...)
result.ErroredEngines = append(result.ErroredEngines, errored...)
}
result.Infected = len(result.Findings) > 0
return result
}
// scanPart scans a single part with all available engines concurrently.
// Returns findings and lists of engine names that timed out or errored.
func (o *Orchestrator) scanPart(part emime.ExtractedPart, scanners []Scanner) ([]Finding, []string, []string) {
ctx, cancel := context.WithTimeout(context.Background(), o.scanTimeout)
defer cancel()
results := make(chan engineScanResult, len(scanners))
var wg sync.WaitGroup
work := make([]*scanWork, 0, len(scanners))
defer func() {
for _, w := range work {
w.finishDelivery(false)
}
}()
for _, s := range scanners {
work = append(work, o.startScan(ctx, s, Part{File: part.File, Size: part.Size}, results, &wg))
}
// Close results channel when all scans complete
obs.SafeGo("emailav-drain", func() {
wg.Wait()
close(results)
})
var findings []Finding
var timedOut []string
var errored []string
for r := range results {
r.work.received()
if r.err != nil {
fmt.Fprintf(os.Stderr, "[emailav] %s scan error on %s: %v\n", r.engine, part.Filename, r.err)
if r.timedOut {
timedOut = append(timedOut, r.engine)
} else {
errored = append(errored, r.engine)
}
r.work.finishDelivery(true)
continue // fail-open
}
if r.verdict.Infected {
f := Finding{
Filename: part.Filename,
Engine: r.engine,
Signature: r.verdict.Signature,
Severity: r.verdict.Severity,
}
if part.Nested {
f.Filename = part.ArchiveName + "/" + part.Filename
}
findings = append(findings, f)
}
r.work.finishDelivery(true)
}
return findings, timedOut, errored
}
type engineScanResult struct {
engine string
verdict Verdict
err error
timedOut bool
work *scanWork
}
func (o *Orchestrator) startScan(ctx context.Context, scanner Scanner, part Part, results chan<- engineScanResult, wg *sync.WaitGroup) *scanWork {
deadline, _ := ctx.Deadline()
w := o.health.begin(deadline)
wg.Add(1)
obs.SafeGo("emailav-scan", func() {
defer wg.Done()
published := false
defer func() {
if !published {
w.finishDelivery(false)
}
}()
done := make(chan engineScanResult, 1)
obs.SafeGo("emailav-engine", func() {
success := false
defer func() { w.finishEngine(success) }()
w.start()
v, err := scanner.Scan(part)
done <- engineScanResult{engine: scanner.Name(), verdict: v, err: err, work: w}
success = err == nil
})
var r engineScanResult
select {
case r = <-done:
case <-ctx.Done():
r = engineScanResult{engine: scanner.Name(), err: fmt.Errorf("scan timeout"), timedOut: true, work: w}
}
w.publish(r.err != nil)
results <- r
published = true
})
return w
}
package emailav
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/pidginhost/csm/internal/mailfwd/quarantine"
"github.com/pidginhost/csm/internal/quarantinefs"
"github.com/pidginhost/csm/internal/safepath"
)
type movedFile struct {
src string
dst string
}
// vars so tests can force EXDEV and source swaps without depending on
// the host filesystem layout.
var (
moveFileRename = renameSpoolFile
moveFileAfterCrossDeviceCopy = func(string, string) error { return nil }
moveFileSyncDir = quarantinefs.SyncDir
)
// QuarantineEnvelope holds the email envelope info for quarantine metadata.
type QuarantineEnvelope struct {
From string
To []string
Subject string
Direction string
}
// QuarantineMetadata is the JSON sidecar written with quarantined messages.
// Subject is stored as the message carried it and read as
// quarantine.CleanSubject leaves it.
type QuarantineMetadata struct {
MessageID string `json:"message_id"`
Direction string `json:"direction"`
From string `json:"from"`
To []string `json:"to"`
Subject string `json:"subject"`
QuarantinedAt time.Time `json:"quarantined_at"`
OriginalSpoolDir string `json:"original_spool_dir"`
Findings []Finding `json:"findings"`
PartialScan bool `json:"partial_scan"`
EnginesUsed []string `json:"engines_used"`
// Size is the -D file's size, taken with stat whenever the entry is
// read; the value stored in the sidecar is never used.
Size int64 `json:"size"`
}
// Quarantine manages the per-message email quarantine directory.
type Quarantine struct {
baseDir string // e.g. /opt/csm/quarantine/email
allowedSpoolDirs []string
}
// NewQuarantine creates a quarantine manager for the given base directory.
func NewQuarantine(baseDir string) *Quarantine {
return &Quarantine{
baseDir: baseDir,
allowedSpoolDirs: []string{"/var/spool/exim/input", "/var/spool/exim4/input"},
}
}
// QuarantineMessage moves spool files into a per-message quarantine directory
// and writes metadata.json.
func (q *Quarantine) QuarantineMessage(msgID, spoolDir string, result *ScanResult, env QuarantineEnvelope) error {
msgID, err := cleanQuarantineMessageID(msgID)
if err != nil {
return err
}
msgDir := filepath.Join(q.baseDir, msgID)
meta := QuarantineMetadata{
MessageID: msgID,
Direction: env.Direction,
From: env.From,
To: env.To,
Subject: env.Subject,
QuarantinedAt: time.Now(),
OriginalSpoolDir: spoolDir,
Findings: result.Findings,
PartialScan: result.PartialExtraction,
EnginesUsed: result.EnginesUsed,
}
metaData, err := json.MarshalIndent(meta, "", " ")
if err != nil {
return fmt.Errorf("marshaling metadata: %w", err)
}
if err := quarantinefs.EnsureDir(q.baseDir, 0700); err != nil {
return fmt.Errorf("creating quarantine root: %w", err)
}
if err := os.Mkdir(msgDir, 0700); err != nil {
return fmt.Errorf("creating quarantine dir: %w", err)
}
if err := quarantinefs.SyncDir(q.baseDir); err != nil {
return fmt.Errorf("syncing quarantine root: %w", err)
}
metaPath := filepath.Join(msgDir, "metadata.json")
if err := quarantinefs.WriteExclusive(metaPath, bytes.NewReader(metaData), 0600); err != nil {
_ = os.RemoveAll(msgDir)
return fmt.Errorf("writing metadata before moving spool files: %w", err)
}
var moved []movedFile
for _, suffix := range []string{"-H", "-D"} {
src := filepath.Join(spoolDir, msgID+suffix)
dst := filepath.Join(msgDir, msgID+suffix)
if err := moveFile(src, dst); err != nil {
var partial *partialSpoolMove
if errors.As(err, &partial) {
return fmt.Errorf("quarantine partly applied; recovery files and metadata retained at %s: %w", msgDir, err)
}
if !os.IsNotExist(err) {
rollbackErr := rollbackMovedFiles(moved)
if rollbackErr != nil {
return fmt.Errorf("moving spool file %s: %w (rollback failed, recovery files retained at %s: %v)", suffix, err, msgDir, rollbackErr)
}
_ = os.RemoveAll(msgDir)
return fmt.Errorf("moving spool file %s: %w", suffix, err)
}
continue
}
moved = append(moved, movedFile{src: src, dst: dst})
}
if len(moved) == 0 {
_ = os.RemoveAll(msgDir)
return fmt.Errorf("no spool files found for %s", msgID)
}
return nil
}
// ListMessages returns all quarantined email messages.
func (q *Quarantine) ListMessages() ([]QuarantineMetadata, error) {
entries, err := os.ReadDir(q.baseDir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("reading quarantine dir: %w", err)
}
var msgs []QuarantineMetadata
for _, entry := range entries {
if !entry.IsDir() {
continue
}
meta, err := q.readMetadata(entry.Name())
if err != nil {
continue
}
msgs = append(msgs, *meta)
}
return msgs, nil
}
// GetMessage returns the metadata for a single quarantined message.
func (q *Quarantine) GetMessage(msgID string) (*QuarantineMetadata, error) {
msgID, err := cleanQuarantineMessageID(msgID)
if err != nil {
return nil, err
}
return q.readMetadata(msgID)
}
// ReleaseMessage moves spool files back to the original spool directory
// and removes the quarantine directory.
func (q *Quarantine) ReleaseMessage(msgID string) error {
msgID, err := cleanQuarantineMessageID(msgID)
if err != nil {
return err
}
meta, err := q.readMetadata(msgID)
if err != nil {
return fmt.Errorf("reading metadata: %w", err)
}
spoolDir, err := q.validateReleaseSpoolDir(meta.OriginalSpoolDir, msgID)
if err != nil {
return err
}
msgDir := filepath.Join(q.baseDir, msgID)
// The queue header makes the message visible to Exim; restore its body first.
for _, suffix := range []string{"-D", "-H"} {
src := filepath.Join(msgDir, msgID+suffix)
dst := filepath.Join(spoolDir, msgID+suffix)
if err := moveFile(src, dst); err != nil {
// If source doesn't exist, skip (partial quarantine)
if os.IsNotExist(err) {
continue
}
return fmt.Errorf("moving %s back to spool: %w", suffix, err)
}
}
if err := os.RemoveAll(msgDir); err != nil {
return fmt.Errorf("message released but quarantine cleanup failed: %w", err)
}
return quarantinefs.SyncDir(q.baseDir)
}
// DeleteMessage permanently removes a quarantined message. Unlike
// ReleaseMessage it does not require metadata: deleting an orphaned or
// partially-written entry is a legitimate cleanup operation.
func (q *Quarantine) DeleteMessage(msgID string) error {
msgID, err := cleanQuarantineMessageID(msgID)
if err != nil {
return err
}
msgDir := filepath.Join(q.baseDir, msgID)
return os.RemoveAll(msgDir)
}
// CleanExpired removes quarantine directories older than maxAge.
// Returns the number of directories cleaned.
func (q *Quarantine) CleanExpired(maxAge time.Duration) (int, error) {
entries, err := os.ReadDir(q.baseDir)
if err != nil {
if os.IsNotExist(err) {
return 0, nil
}
return 0, err
}
cleaned := 0
var firstErr error
cutoff := time.Now().Add(-maxAge)
for _, entry := range entries {
if !entry.IsDir() {
continue
}
meta, err := q.readMetadata(entry.Name())
if err != nil {
continue
}
if meta.QuarantinedAt.Before(cutoff) {
// Keep sweeping past a stuck entry, but report it: a silent
// failure here leaves the quarantine growing while the sweep
// claims success.
if err := os.RemoveAll(filepath.Join(q.baseDir, entry.Name())); err != nil {
if firstErr == nil {
firstErr = fmt.Errorf("removing expired entry %s: %w", entry.Name(), err)
}
continue
}
cleaned++
}
}
return cleaned, firstErr
}
func (q *Quarantine) readMetadata(msgID string) (*QuarantineMetadata, error) {
msgID, err := cleanQuarantineMessageID(msgID)
if err != nil {
return nil, err
}
metaPath := filepath.Join(q.baseDir, msgID, "metadata.json")
// #nosec G304 -- msgID is restricted to a single path segment under baseDir.
data, err := os.ReadFile(metaPath)
if err != nil {
return nil, err
}
var meta QuarantineMetadata
if err := json.Unmarshal(data, &meta); err != nil {
return nil, err
}
meta.Subject = quarantine.CleanSubject(meta.Subject)
meta.Size = 0
if info, err := os.Lstat(filepath.Join(q.baseDir, msgID, msgID+"-D")); err == nil && info.Mode().IsRegular() {
meta.Size = info.Size()
}
return &meta, nil
}
func cleanQuarantineMessageID(msgID string) (string, error) {
if msgID == "" {
return "", fmt.Errorf("message id is required")
}
if msgID == "." || msgID == ".." || strings.ContainsAny(msgID, `/\`) {
return "", fmt.Errorf("invalid message id")
}
if filepath.Base(msgID) != msgID {
return "", fmt.Errorf("invalid message id")
}
return msgID, nil
}
// moveFile renames src to dst, falling back to a fd-bound copy when
// rename fails (cross-device or other EXDEV-style errors). Callers
// construct dst by filepath.Join under the quarantine base dir
// (config-owned) plus a validated single-segment identifier.
//
// The cross-device path opens src with O_NOFOLLOW, copies content
// from the open fd into a freshly-created dst, and unlinks src by the
// path only after verifying the path still names the same inode. An
// attacker who swaps src for a symlink or replaces the file
// mid-copy is rejected.
func moveFile(src, dst string) error {
if err := quarantinefs.SyncFilePath(src); err != nil {
return err
}
if err := moveFileRename(src, dst); err == nil {
return syncSpoolMove(src, dst)
} else if !errors.Is(err, syscall.EXDEV) {
// Non-EXDEV rename errors are not a cross-device condition;
// surface them directly so the caller does not silently
// fall back to copy semantics for permission or path errors.
return err
}
// #nosec G304 -- src is mail queue path from scanner walk;
// O_NOFOLLOW plus the identity check below catch symlink swaps.
fd, err := os.OpenFile(src, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
if err != nil {
return fmt.Errorf("opening cross-device source: %w", err)
}
defer fd.Close()
srcInfo, err := fd.Stat()
if err != nil {
return fmt.Errorf("stat cross-device source: %w", err)
}
if !srcInfo.Mode().IsRegular() {
return fmt.Errorf("refusing cross-device copy of non-regular %s", src)
}
// #nosec G304 G306 -- dst is constructed under the quarantine baseDir;
// 0600 keeps the copy private.
dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return fmt.Errorf("creating cross-device destination: %w", err)
}
removeDst := true
defer func() {
if removeDst {
_ = os.Remove(dst)
}
}()
if _, copyErr := io.Copy(dstFile, fd); copyErr != nil {
_ = dstFile.Close()
return fmt.Errorf("cross-device copy body: %w", copyErr)
}
stat := srcInfo.Sys().(*syscall.Stat_t)
if ownerErr := dstFile.Chown(int(stat.Uid), int(stat.Gid)); ownerErr != nil {
_ = dstFile.Close()
return fmt.Errorf("cross-device ownership: %w", ownerErr)
}
// Chown can clear special mode bits, so permissions follow ownership.
if modeErr := dstFile.Chmod(srcInfo.Mode()); modeErr != nil {
_ = dstFile.Close()
return fmt.Errorf("cross-device permissions: %w", modeErr)
}
if timeErr := safepath.SetModTime(dstFile, srcInfo.ModTime()); timeErr != nil {
_ = dstFile.Close()
return fmt.Errorf("cross-device modification time: %w", timeErr)
}
if syncErr := dstFile.Sync(); syncErr != nil {
_ = dstFile.Close()
return fmt.Errorf("cross-device sync: %w", syncErr)
}
if closeErr := dstFile.Close(); closeErr != nil {
return fmt.Errorf("cross-device close: %w", closeErr)
}
if hookErr := moveFileAfterCrossDeviceCopy(src, dst); hookErr != nil {
return fmt.Errorf("cross-device post-copy check: %w", hookErr)
}
pathInfo, err := os.Lstat(src)
if err != nil {
return fmt.Errorf("stat cross-device source path: %w", err)
}
if !sameUnixInode(srcInfo, pathInfo) {
return fmt.Errorf("cross-device source %s changed during copy", src)
}
if err := moveFileSyncDir(filepath.Dir(dst)); err != nil {
return fmt.Errorf("syncing cross-device destination: %w", err)
}
if err := os.Remove(src); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("removing cross-device source: %w", err)
}
removeDst = false
if err := moveFileSyncDir(filepath.Dir(src)); err != nil {
return &partialSpoolMove{src: src, dst: dst, cause: err}
}
return nil
}
type partialSpoolMove struct {
src, dst string
cause error
}
func (e *partialSpoolMove) Error() string {
return fmt.Sprintf("moved %s to %s, but directory sync failed; inspect both paths before retrying: %v", e.src, e.dst, e.cause)
}
func (e *partialSpoolMove) Unwrap() error { return e.cause }
func syncSpoolMove(src, dst string) error {
for _, dir := range []string{filepath.Dir(dst), filepath.Dir(src)} {
if err := moveFileSyncDir(dir); err != nil {
return &partialSpoolMove{src: src, dst: dst, cause: err}
}
}
return nil
}
func renameSpoolFile(src, dst string) error {
source, err := safepath.OpenDir(filepath.Dir(src))
if err != nil {
return err
}
defer func() { _ = source.Close() }()
destination, err := safepath.OpenDir(filepath.Dir(dst))
if err != nil {
return err
}
defer func() { _ = destination.Close() }()
return source.RenameTo(filepath.Base(src), destination, filepath.Base(dst))
}
// sameUnixInode compares two FileInfos via underlying syscall.Stat_t so
// the cross-device move can verify the source path still names the
// same inode it opened. Falls back to false if either Sys() does not
// expose Stat_t (e.g. non-Linux builds).
func sameUnixInode(a, b os.FileInfo) bool {
if a == nil || b == nil {
return false
}
if os.SameFile(a, b) {
return true
}
as, aok := a.Sys().(*syscall.Stat_t)
bs, bok := b.Sys().(*syscall.Stat_t)
return aok && bok && as.Dev == bs.Dev && as.Ino == bs.Ino
}
func rollbackMovedFiles(moved []movedFile) error {
for i := len(moved) - 1; i >= 0; i-- {
if err := moveFile(moved[i].dst, moved[i].src); err != nil {
return err
}
}
return nil
}
func (q *Quarantine) validateReleaseSpoolDir(spoolDir, msgID string) (string, error) {
cleanDir := filepath.Clean(spoolDir)
if cleanDir == "" || !filepath.IsAbs(cleanDir) {
return "", fmt.Errorf("invalid original spool directory")
}
// Fail closed when the supplied path will not resolve. The previous
// behaviour fell back to the unresolved literal, which let a path
// that does not currently exist on disk still match an allowed-list
// entry that also does not exist (e.g. on a host where only one of
// the default exim spool defaults is installed). A release that
// targets a path we cannot resolve cannot have its identity
// confirmed, so it must not proceed.
resolvedDir, err := filepath.EvalSymlinks(cleanDir)
if err != nil {
return "", fmt.Errorf("resolving original spool directory %q: %w", cleanDir, err)
}
for _, allowed := range q.allowedSpoolDirs {
cleanAllowed := filepath.Clean(allowed)
// Allowed entries are operator-config trusted defaults; the
// shipped list intentionally covers both exim and exim4, so a
// host that only has one of them must silently skip the
// non-installed entry rather than falling back to the literal
// (which would silently widen the trust boundary). The input
// path was already required to resolve above, so a missing
// allowed entry can never alias the resolved input.
resolvedAllowed, err := filepath.EvalSymlinks(cleanAllowed)
if err != nil {
continue
}
if resolvedDir == resolvedAllowed {
return resolvedDir, nil
}
// With split_spool_directory (the cPanel default) Exim keeps the
// message in a hash subdirectory named after the sixth character
// of its ID, and looks for it nowhere else under that root.
if len(msgID) > 5 && resolvedDir == filepath.Join(resolvedAllowed, msgID[5:6]) {
return resolvedDir, nil
}
}
return "", fmt.Errorf("original spool directory is not trusted: %s", cleanDir)
}
package emailav
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
type scanQueue struct {
mu sync.Mutex
pending map[*scanWork]struct{}
losses *queuehealth.Tracker
}
type scanWork struct {
queue *scanQueue
queued, deadline time.Time
started, returned, ready, receiving time.Time
engineDone, deliveryDone, failed bool
}
func newScanQueue() *scanQueue {
return &scanQueue{pending: make(map[*scanWork]struct{}), losses: queuehealth.New(0, time.Minute)}
}
func (q *scanQueue) begin(deadline time.Time) *scanWork {
w := &scanWork{queue: q, queued: time.Now(), deadline: deadline}
q.mu.Lock()
q.pending[w] = struct{}{}
q.mu.Unlock()
return w
}
func (w *scanWork) start() {
w.queue.mu.Lock()
w.started = time.Now()
w.queue.mu.Unlock()
}
func (w *scanWork) finishEngine(success bool) {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
w.engineDone = true
w.returned = time.Now()
if !success {
w.failLocked()
}
w.releaseLocked()
}
func (w *scanWork) publish(failed bool) {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
w.ready = time.Now()
if failed {
w.failLocked()
}
}
func (w *scanWork) received() {
w.queue.mu.Lock()
w.receiving = time.Now()
w.queue.mu.Unlock()
}
func (w *scanWork) finishDelivery(success bool) {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
if w.deliveryDone {
return
}
w.deliveryDone = true
if !success {
w.failLocked()
}
w.releaseLocked()
}
func (w *scanWork) failLocked() {
if !w.failed {
w.failed = true
w.queue.losses.Lose(time.Now(), 1)
}
}
func (w *scanWork) releaseLocked() {
// Timeout releases result delivery while Scan may still run. Conversely,
// a returned engine can leave work in either buffered result handoff.
if w.engineDone && w.deliveryDone {
delete(w.queue.pending, w)
}
}
// QueueStatuses reads memory without probing an engine or taking its locks.
func (o *Orchestrator) QueueStatuses(now time.Time) map[string]queuehealth.Status {
q := o.health
q.mu.Lock()
defer q.mu.Unlock()
s := q.losses.Snapshot(now)
// Concurrent ScanParts calls and engines outliving their callers have no
// fixed global admission limit, despite each result channel being bounded.
s.CapacityUnavailable = true
var waitingLate, runningLate bool
for w := range q.pending {
origin, deadline, running := w.queued, w.deadline, false
switch {
case !w.engineDone:
if !w.started.IsZero() {
origin, running = w.started, true
}
case !w.receiving.IsZero():
origin, running = w.receiving, true
deadline = origin.Add(time.Minute)
default:
origin = w.returned
if !w.ready.IsZero() && w.ready.Before(origin) {
origin = w.ready
}
deadline = origin.Add(time.Minute)
}
if running {
s.InFlight++
s.ProcessingSeconds = max(s.ProcessingSeconds, now.Sub(origin).Seconds())
runningLate = runningLate || !now.Before(deadline)
} else {
s.Depth++
s.LagSeconds = max(s.LagSeconds, now.Sub(origin).Seconds())
waitingLate = waitingLate || !now.Before(deadline)
}
}
switch {
case waitingLate:
s.Reason = "backlog_lag"
case runningLate:
s.Reason = "processing_lag"
}
if s.Reason != "" {
s.Status = "degraded"
}
return map[string]queuehealth.Status{"scans": s}
}
//go:build !yara
package emailav
import "github.com/pidginhost/csm/internal/yara"
// YaraXScanner is a no-op stub when YARA-X is not compiled in. The
// constructor still accepts a yara.Backend so callers can pass
// yara.Active() uniformly across build tags.
type YaraXScanner struct{}
// NewYaraXScanner returns a scanner that is never available.
func NewYaraXScanner(_ yara.Backend) *YaraXScanner {
return &YaraXScanner{}
}
// NewActiveYaraXScanner returns the same no-op scanner in non-YARA builds.
func NewActiveYaraXScanner() *YaraXScanner {
return &YaraXScanner{}
}
func (s *YaraXScanner) Name() string { return "yara-x" }
func (s *YaraXScanner) Available() bool { return false }
func (s *YaraXScanner) Scan(_ Part) (Verdict, error) {
return Verdict{}, nil
}
package emailspool
import (
"bufio"
"errors"
"fmt"
"io"
"os"
"strconv"
"strings"
"golang.org/x/net/idna"
)
// ExtractDomain returns the lowercased, IDN-normalised domain portion of an
// RFC 5322 address or display-name form. Returns "" on parse failure.
//
// Quoted local parts ("a@b"@example.com) are handled by treating the address
// as the substring after the LAST unquoted '@'.
func ExtractDomain(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return ""
}
if i := strings.LastIndex(s, "<"); i >= 0 {
if j := strings.LastIndex(s, ">"); j > i {
s = s[i+1 : j]
}
}
at := lastUnquotedAt(s)
if at < 0 {
return ""
}
domain := strings.TrimSpace(s[at+1:])
domain = strings.ToLower(domain)
// On IDNA failure keep the raw lowercased domain rather than returning "".
// Callers gate mismatch checks on a non-empty domain, so an empty return
// would let a malformed attacker-controlled label bypass those checks.
if ascii, err := idna.ToASCII(domain); err == nil {
domain = ascii
}
return domain
}
// lastUnquotedAt returns the index of the rightmost '@' character that is not
// inside a double-quoted segment, or -1 if none.
func lastUnquotedAt(s string) int {
inQuote := false
last := -1
for i := 0; i < len(s); i++ {
c := s[i]
switch c {
case '\\':
i++ // skip the next byte (quoted-pair)
case '"':
inQuote = !inQuote
case '@':
if !inQuote {
last = i
}
}
}
return last
}
// IsSubdomainOrEqual reports whether candidate is base or a subdomain of base.
// Both inputs are case-insensitive. Empty inputs return false.
func IsSubdomainOrEqual(candidate, base string) bool {
if candidate == "" || base == "" {
return false
}
c := strings.ToLower(strings.TrimSuffix(candidate, "."))
b := strings.ToLower(strings.TrimSuffix(base, "."))
if c == b {
return true
}
return strings.HasSuffix(c, "."+b)
}
// MaxSpoolHeaderBytes bounds how much of an Exim -H file we read.
// 32 KiB covers rich messages with DKIM, ARC, and folded MIME headers
// without unbounded memory.
const MaxSpoolHeaderBytes = 32 * 1024
// Headers is the parsed envelope + interesting RFC 5322 fields from a
// cPanel-Exim spool -H file. EnvelopeUser comes from the file's line 2
// (the local UID under which Exim accepted the message); the RFC 5322
// fields come from the message's mail headers section. Empty string means
// "header absent" -- callers should not synthesise defaults from absence.
type Headers struct {
EnvelopeUser string
EnvelopeUID int
From string
ReplyTo string
Subject string
XPHPScript string
XMailer string
UserAgent string
MessageID string
// Recipients holds the envelope recipient addresses when the Exim -H
// recipient block can be unambiguously located; empty when the block is
// absent or its shape cannot be validated. Consumers that gate on
// recipient diversity must treat empty as "unknown" and fail open.
Recipients []string
}
// ParseHeaders reads the given Exim -H file and returns a Headers.
// Returns the parse error if the file cannot be opened or is structurally
// invalid; otherwise missing individual headers leave the corresponding
// Headers field empty.
func ParseHeaders(path string) (Headers, error) {
// #nosec G304 -- path is supplied by the daemon's Exim spool
// watcher and resolved from /var/spool/exim/input/, an
// operator-trusted directory enumerated by the spool walker.
f, err := os.Open(path)
if err != nil {
return Headers{}, err
}
defer f.Close()
h, err := ParseHeadersReader(f)
if err != nil {
return Headers{}, fmt.Errorf("parse %s: %w", path, err)
}
return h, nil
}
// ParseHeadersReader is the io.Reader form of ParseHeaders for callers that
// already have the spool bytes in memory or behind a custom seam (e.g. the
// checks package's osFS abstraction). It applies the same Exim -H parsing
// rules as ParseHeaders -- envelope preamble, blank-line separator,
// "NNNX " prefixed RFC 5322 headers -- and is bounded by MaxSpoolHeaderBytes
// per token; oversize input returns bufio.ErrTooLong.
func ParseHeadersReader(r io.Reader) (Headers, error) {
var h Headers
// Per-line memory is bounded by the scanner's max buffer
// (MaxSpoolHeaderBytes); a token larger than that returns
// bufio.ErrTooLong. We deliberately do NOT wrap r in an io.LimitReader:
// when LimitReader returns EOF mid-token, bufio.Scanner emits the
// partial token without error and oversize spool files are silently
// truncated. That hides the failure from operators.
sc := bufio.NewScanner(r)
sc.Buffer(make([]byte, 0, 8192), MaxSpoolHeaderBytes)
// Line 1: msgID-H (we ignore the value; presence is enough)
if !sc.Scan() {
return Headers{}, errors.New("empty spool file")
}
// Line 2: "<user> <uid> <gid>"
if !sc.Scan() {
return Headers{}, errors.New("missing envelope user line")
}
fields := splitFields(sc.Text())
if len(fields) < 2 {
return Headers{}, fmt.Errorf("malformed envelope user line: %q", sc.Text())
}
h.EnvelopeUser = fields[0]
if uid, err := strconv.Atoi(fields[1]); err == nil {
h.EnvelopeUID = uid
}
// Skip remaining envelope metadata until the blank line that separates
// it from the RFC 5322 header section. Exim's -H format places mail
// headers AFTER a blank line that follows the recipient list; recipients
// are preceded by a numeric count line.
inHeaders := false
var preamble []string
lastHeader := ""
skippingDeleted := false
for sc.Scan() {
line := sc.Text()
if !inHeaders {
if line == "" {
inHeaders = true
continue
}
preamble = append(preamble, line)
continue
}
// RFC 5322 header section. Each header line in Exim's -H format
// starts with "<count><flag> <name>: <value>" where count is a
// variable-width decimal byte count for the full stored header
// (including folded continuation lines) and flag is a single Exim
// marker ('T', 'F', 'R', '*', space, etc.).
if isFoldedHeaderLine(line) {
if !skippingDeleted && lastHeader != "" {
appendEximHeaderContinuation(&h, lastHeader, line)
}
continue
}
lastHeader = ""
skippingDeleted = false
name, value, deleted, ok := parseEximHeaderLine(line)
if deleted {
skippingDeleted = true
continue
}
if !ok {
continue
}
if canonical, handled := setEximHeaderValue(&h, name, value); handled {
lastHeader = canonical
}
}
if err := sc.Err(); err != nil {
return Headers{}, fmt.Errorf("scan spool: %w", err)
}
if !inHeaders {
return Headers{}, errors.New("missing header section separator")
}
h.Recipients = extractEximRecipients(preamble)
return h, nil
}
// extractEximRecipients recovers the envelope recipient addresses from the
// Exim -H preamble (the lines between the envelope-user line and the blank
// header separator). Exim writes the recipient block last: a line holding only
// the recipient count N, then exactly N recipient lines that run to the end of
// the preamble. Anchoring on "index + 1 + N == len(preamble)" plus an
// address-shape check on every claimed recipient locates the block without a
// full grammar and tolerates option/ACL lines above it. Any ambiguity, including
// a malformed anchored candidate, returns nil so callers treat recipients as
// unknown and fail open.
func extractEximRecipients(preamble []string) []string {
var candidate []string
ambiguous := false
for i, line := range preamble {
n, ok := parseBareUint(line)
if !ok || n < 1 || i+1+n != len(preamble) {
continue
}
rcpts := make([]string, 0, n)
valid := true
for _, r := range preamble[i+1:] {
addr := firstField(r)
if addr == "" || !strings.Contains(addr, "@") {
valid = false
break
}
rcpts = append(rcpts, addr)
}
if !valid {
ambiguous = true
continue
}
if candidate != nil {
ambiguous = true
continue
}
candidate = rcpts
}
if ambiguous {
return nil
}
return candidate
}
// parseBareUint reports whether s is a single non-negative integer token with
// no other characters. The length cap rejects timestamps and other long
// numeric option values that are not recipient counts.
func parseBareUint(s string) (int, bool) {
s = strings.TrimSpace(s)
if s == "" || len(s) > 9 {
return 0, false
}
for i := 0; i < len(s); i++ {
if s[i] < '0' || s[i] > '9' {
return 0, false
}
}
n, err := strconv.Atoi(s)
if err != nil {
return 0, false
}
return n, true
}
func firstField(s string) string {
s = strings.TrimSpace(s)
if i := strings.IndexAny(s, " \t"); i >= 0 {
s = s[:i]
}
return s
}
// parseEximHeaderLine returns the header name, value, deletion marker, and
// prefix-valid marker for an Exim -H header line of the form
// "NNNF Header-Name: value". The prefix is a variable-width decimal length
// (Exim writes the byte count of the full stored header, including folded
// continuation lines), followed by a flag byte ('T', 'F', 'R', '*', space,
// etc.) and a space separator; the whole prefix is stripped before splitting
// on the colon.
func parseEximHeaderLine(line string) (name string, value string, deleted bool, ok bool) {
i := 0
for i < len(line) && isDigit(line[i]) {
i++
}
if i == 0 || i+1 >= len(line) {
return "", "", false, false
}
// After the digit run: a flag byte (letter, or space for unflagged
// headers such as X-PHP-Script in real cPanel-Exim spool output) then a
// single space separator.
if !isEximHeaderFlag(line[i]) || line[i+1] != ' ' {
return "", "", false, false
}
if line[i] == '*' {
return "", "", true, true
}
rest := line[i+2:]
colon := indexByte(rest, ':')
if colon < 0 {
return "", "", false, true
}
name = rest[:colon]
if colon+1 < len(rest) {
value = trimLeadingSpace(rest[colon+1:])
}
return name, value, false, true
}
func isFoldedHeaderLine(line string) bool {
return len(line) > 0 && (line[0] == ' ' || line[0] == '\t')
}
func isEximHeaderFlag(b byte) bool {
return b == ' ' || b == '*' || isLetter(b)
}
func setEximHeaderValue(h *Headers, name, value string) (string, bool) {
switch strings.ToLower(name) {
case "from":
h.From = value
return "from", true
case "reply-to":
h.ReplyTo = value
return "reply-to", true
case "subject":
h.Subject = value
return "subject", true
case "x-php-script":
h.XPHPScript = value
return "x-php-script", true
case "x-mailer":
h.XMailer = value
return "x-mailer", true
case "user-agent":
h.UserAgent = value
return "user-agent", true
case "message-id":
h.MessageID = value
return "message-id", true
default:
return "", false
}
}
func appendEximHeaderContinuation(h *Headers, name, line string) {
value := trimLeadingSpace(line)
switch name {
case "from":
h.From = appendFoldedHeaderValue(h.From, value)
case "reply-to":
h.ReplyTo = appendFoldedHeaderValue(h.ReplyTo, value)
case "subject":
h.Subject = appendFoldedHeaderValue(h.Subject, value)
case "x-php-script":
h.XPHPScript = appendFoldedHeaderValue(h.XPHPScript, value)
case "x-mailer":
h.XMailer = appendFoldedHeaderValue(h.XMailer, value)
case "user-agent":
h.UserAgent = appendFoldedHeaderValue(h.UserAgent, value)
case "message-id":
h.MessageID = appendFoldedHeaderValue(h.MessageID, value)
}
}
func appendFoldedHeaderValue(current, continuation string) string {
if current == "" {
return continuation
}
if continuation == "" {
return current
}
return current + " " + continuation
}
func isDigit(b byte) bool { return b >= '0' && b <= '9' }
func isLetter(b byte) bool { return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') }
func indexByte(s string, b byte) int {
for i := 0; i < len(s); i++ {
if s[i] == b {
return i
}
}
return -1
}
func trimLeadingSpace(s string) string {
i := 0
for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
i++
}
return s[i:]
}
func splitFields(s string) []string {
var out []string
start := -1
for i := 0; i < len(s); i++ {
if s[i] == ' ' || s[i] == '\t' {
if start >= 0 {
out = append(out, s[start:i])
start = -1
}
} else if start < 0 {
start = i
}
}
if start >= 0 {
out = append(out, s[start:])
}
return out
}
package emailspool
import (
_ "embed"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"sync"
"gopkg.in/yaml.v3"
)
// Policies is the loaded contents of the php_relay policies_dir. Used by
// Stage 1 (mailer classes, http_proxy_ranges) and Stage 2 (policy_blocks). Each
// data file has its own schema; failing to parse one file does not abort
// the whole load -- each category degrades independently.
type Policies struct {
mu sync.RWMutex
suspiciousMail []string
safeMail []string
proxyNets []*net.IPNet
// selfIPs are the host's own interface addresses. PHP-relay Path 4
// (HTTP-IP fanout) treats them as proxy-equivalent so WordPress cron
// and any other local loopback-to-public traffic does not page on
// "one IP triggered N scripts". Populated by RefreshSelfIPs.
selfIPs []net.IP
}
// hostIPsFunc returns the set of non-loopback host IPs to treat as self.
// Package-level so tests can inject deterministic addresses.
var hostIPsFunc = enumerateHostIPs
// enumerateHostIPs reads every IPv4/IPv6 address bound to a non-loopback
// interface. Errors from net.InterfaceAddrs (extremely rare; only on syscall
// failure) drop us to an empty list; loopback fallback in IsProxyIP still
// covers 127/8 and ::1.
func enumerateHostIPs() []net.IP {
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil
}
out := make([]net.IP, 0, len(addrs))
for _, a := range addrs {
var ip net.IP
switch v := a.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
continue
}
out = append(out, ip)
}
return out
}
// RefreshSelfIPs re-enumerates host addresses and replaces the cached set.
// Safe to call concurrently with IsProxyIP. LoadPolicies / Reload call this
// automatically so SIGHUP picks up cPanel alias IP additions without code
// changes to callers; exposed for tests and the daemon Flow E ticker.
func (p *Policies) RefreshSelfIPs() {
ips := hostIPsFunc()
p.mu.Lock()
p.selfIPs = ips
p.mu.Unlock()
}
type mailerClassesYAML struct {
Version int `yaml:"version"`
Suspicious []string `yaml:"suspicious"`
Safe []string `yaml:"safe"`
}
//nolint:unused // consumed by E2/E3
type httpProxyRangesYAML struct {
Version int `yaml:"version"`
// yaml.v3 drops nulls from []string. Keep them so an unfinished list
// cannot be mistaken for an explicitly empty one.
CIDRs []*string `yaml:"cidrs"`
}
// defaultProxyRanges is the built-in http_proxy_ranges.yaml. Packages
// create policies_dir empty, so without it every per-IP path would see
// CDN edges and payment webhooks as visitors.
//
//go:embed embed/http_proxy_ranges.yaml
var defaultProxyRanges []byte
var builtInProxyRanges = mustProxyRanges(defaultProxyRanges)
func mustProxyRanges(data []byte) httpProxyRangesYAML {
var raw httpProxyRangesYAML
if err := yaml.Unmarshal(data, &raw); err != nil {
panic("embedded http_proxy_ranges.yaml: " + err.Error())
}
return raw
}
// parseProxyCIDRs parses http_proxy_ranges.yaml entries; a bare IP counts
// as /32 or /128. Invalid entries are reported and skipped. The result is
// never nil, so an explicit empty list stays distinguishable from a failed
// load.
func parseProxyCIDRs(cidrs []*string, setErr func(error)) []*net.IPNet {
nets := make([]*net.IPNet, 0, len(cidrs))
for _, c := range cidrs {
if c == nil {
setErr(errors.New("null CIDR in http_proxy_ranges.yaml"))
continue
}
s := strings.TrimSpace(*c)
if ip := net.ParseIP(s); ip != nil {
bits := 128
if v4 := ip.To4(); v4 != nil {
ip, bits = v4, 32
}
nets = append(nets, &net.IPNet{IP: ip, Mask: net.CIDRMask(bits, bits)})
continue
}
_, n, err := net.ParseCIDR(s)
if err != nil {
setErr(fmt.Errorf("invalid CIDR %q in http_proxy_ranges.yaml: %w", *c, err))
continue
}
nets = append(nets, n)
}
return nets
}
// LoadPolicies reads all known policy files in dir. A missing
// mailer_classes.yaml means no entries; a missing, unreadable or corrupt
// http_proxy_ranges.yaml means the built-in list. Corrupt files return an
// error per file but do not abort the load. The returned Policies is safe
// for concurrent use.
func LoadPolicies(dir string) (*Policies, error) {
p := &Policies{}
if err := p.load(dir); err != nil {
return p, err
}
return p, nil
}
func (p *Policies) load(dir string) error {
p.mu.Lock()
// Refresh self IPs alongside file-backed policies so SIGHUP picks up
// any cPanel alias IP additions that landed since startup.
p.selfIPs = hostIPsFunc()
defer p.mu.Unlock()
var firstErr error
setErr := func(err error) {
if firstErr == nil {
firstErr = err
}
}
// mailer_classes.yaml
// #nosec G304 -- dir is the operator-supplied policy directory; filename is a fixed literal under it.
if data, err := os.ReadFile(filepath.Join(dir, "mailer_classes.yaml")); err == nil {
var raw mailerClassesYAML
if uerr := yaml.Unmarshal(data, &raw); uerr != nil {
setErr(fmt.Errorf("parse mailer_classes.yaml: %w", uerr))
} else {
p.suspiciousMail = lowerList(raw.Suspicious)
p.safeMail = lowerList(raw.Safe)
}
} else if !os.IsNotExist(err) {
setErr(fmt.Errorf("read mailer_classes.yaml: %w", err))
}
// http_proxy_ranges.yaml: an operator file replaces the built-in list,
// and a missing one means the built-in list. An unreadable or unparsable
// file keeps the last good list (the built-in one on first load), so a
// broken file never empties the exemption list. A file without a cidrs
// key (empty, truncated) or without one valid entry is unparsable;
// "cidrs: []" is a deliberate empty list.
var nets []*net.IPNet
// #nosec G304 -- dir is the operator-supplied policy directory; filename is a fixed literal under it.
data, err := os.ReadFile(filepath.Join(dir, "http_proxy_ranges.yaml"))
switch {
case os.IsNotExist(err):
nets = parseProxyCIDRs(builtInProxyRanges.CIDRs, setErr)
case err != nil:
setErr(fmt.Errorf("read http_proxy_ranges.yaml: %w", err))
default:
var op httpProxyRangesYAML
if uerr := yaml.Unmarshal(data, &op); uerr != nil {
setErr(fmt.Errorf("parse http_proxy_ranges.yaml: %w", uerr))
} else if op.CIDRs == nil {
setErr(errors.New("parse http_proxy_ranges.yaml: no cidrs list"))
} else {
var entryErr error
nets = parseProxyCIDRs(op.CIDRs, func(err error) {
if entryErr == nil {
entryErr = err
}
})
// An entry error alone would read as one skipped entry.
if len(nets) == 0 && len(op.CIDRs) > 0 {
entryErr = fmt.Errorf("http_proxy_ranges.yaml rejected: no valid cidrs (first error: %w)", entryErr)
nets = nil
}
if entryErr != nil {
setErr(entryErr)
}
}
}
// proxyNets is nil only before the first load.
if nets == nil && p.proxyNets == nil {
nets = parseProxyCIDRs(builtInProxyRanges.CIDRs, setErr)
}
if nets != nil {
p.proxyNets = nets
}
return firstErr
}
// Reload refreshes from dir while keeping previous values for any category
// whose new file is corrupt (existing reload-error contract). Used by SIGHUP.
//
//nolint:unused // consumed by E3
func (p *Policies) Reload(dir string) error {
return p.load(dir)
}
// MailerSuspicious reports whether x-mailer header matches any suspicious
// substring. Substrings, not exact match.
func (p *Policies) MailerSuspicious(xMailer string) bool {
if xMailer == "" {
return false
}
p.mu.RLock()
defer p.mu.RUnlock()
low := strings.ToLower(xMailer)
for _, s := range p.suspiciousMail {
if strings.Contains(low, s) {
return true
}
}
return false
}
// MailerSafe reports whether x-mailer matches any safe substring.
func (p *Policies) MailerSafe(xMailer string) bool {
if xMailer == "" {
return false
}
p.mu.RLock()
defer p.mu.RUnlock()
low := strings.ToLower(xMailer)
for _, s := range p.safeMail {
if strings.Contains(low, s) {
return true
}
}
return false
}
// IsProxyIP reports whether ip falls within any configured CDN/proxy CIDR,
// is one of the host's own interface addresses, or is a loopback address.
// Used by Path 4 to skip fanout counting for IPs that are CDN front IPs,
// the local host (WordPress cron, panel-internal callbacks), or 127/::1.
func (p *Policies) IsProxyIP(ip string) bool {
if ip == "" {
return false
}
parsed := net.ParseIP(ip)
if parsed == nil {
return false
}
if parsed.IsLoopback() {
return true
}
p.mu.RLock()
defer p.mu.RUnlock()
for _, n := range p.proxyNets {
if n.Contains(parsed) {
return true
}
}
for _, self := range p.selfIPs {
if self.Equal(parsed) {
return true
}
}
return false
}
func lowerList(in []string) []string {
out := make([]string, 0, len(in))
for _, s := range in {
s = strings.ToLower(strings.TrimSpace(s))
if s != "" {
out = append(out, s)
}
}
return out
}
package emailspool
import (
"net/netip"
"slices"
"strings"
)
// ProxyProvider is one block of the built-in shared-caller list, named by its
// "# Provider:" heading.
type ProxyProvider struct {
Name string
Prefixes []netip.Prefix
}
// BuiltInProxyProviders returns the built-in http_proxy_ranges.yaml blocks
// in file order, so a caller can select providers by name. A bare address
// is a single-address range. An operator's replacement file is not read:
// it carries no provider headings the loader could trust.
// The returned providers and prefix slices are independent copies.
func BuiltInProxyProviders() []ProxyProvider {
providers := slices.Clone(builtInProxyProviders)
for i := range providers {
providers[i].Prefixes = slices.Clone(providers[i].Prefixes)
}
return providers
}
var builtInProxyProviders = mustProxyProviders(defaultProxyRanges)
func mustProxyProviders(data []byte) []ProxyProvider {
var out []ProxyProvider
for _, line := range strings.Split(string(data), "\n") {
l := strings.TrimSpace(line)
if name, ok := strings.CutPrefix(l, "# Provider:"); ok {
out = append(out, ProxyProvider{Name: strings.TrimSpace(name)})
continue
}
entry, ok := strings.CutPrefix(l, "- ")
if !ok {
continue
}
if len(out) == 0 {
panic("embedded http_proxy_ranges.yaml: range before any provider heading")
}
prefix, err := parseProviderEntry(strings.Trim(strings.TrimSpace(entry), `"'`))
if err != nil {
panic("embedded http_proxy_ranges.yaml: " + err.Error())
}
out[len(out)-1].Prefixes = append(out[len(out)-1].Prefixes, prefix)
}
return out
}
func parseProviderEntry(s string) (netip.Prefix, error) {
if addr, err := netip.ParseAddr(s); err == nil {
addr = addr.Unmap()
return netip.PrefixFrom(addr, addr.BitLen()), nil
}
p, err := netip.ParsePrefix(s)
if err != nil {
return netip.Prefix{}, err
}
return p.Masked(), nil
}
package emailspool
import (
"bufio"
"errors"
"io"
"net/netip"
"strconv"
"strings"
)
// maxSourcePreamble bounds how much of an -H file ReadSource reads. The
// options preamble is a few hundred bytes; ACL values can make it larger.
const maxSourcePreamble = 64 << 10
// maxSourceKey bounds the length of a source key.
const maxSourceKey = 255
// ReadSource returns the key of the party that sent a message, read from the
// options preamble of its Exim -H file: "auth:<id>" for an authenticated
// client, else "ip:<addr>" for a remote host (an IPv6 host as its /64), else
// "uid:<n>" for the local user that submitted it. It reads at most
// maxSourcePreamble bytes and errors when no key can be found, including a
// network message whose host address cannot be read.
func ReadSource(r io.Reader) (string, error) {
br := bufio.NewReader(io.LimitReader(r, maxSourcePreamble))
var uid, auth, host string
sawHost := false
for lineNo := 1; ; lineNo++ {
line, err := br.ReadString('\n')
// A partial identity at EOF or the read limit is not a source key.
// Otherwise padding could split one sender across several gates.
if err != nil {
break
}
line = strings.TrimSuffix(line, "\n")
if lineNo <= 4 {
if lineNo == 2 {
uid = sourceUID(line)
}
continue
}
name, value, ok := sourceOption(line)
if !ok {
break
}
if name == "acl" || name == "aclc" || name == "aclm" {
// The value follows on the next lines and may hold newlines, so
// skip it by its length, plus the newline after it.
n, perr := strconv.Atoi(lastSourceField(value))
if perr != nil || n < 0 || n >= maxSourcePreamble {
break
}
if _, cerr := io.CopyN(io.Discard, br, int64(n)+1); cerr != nil {
break
}
continue
}
switch name {
case "auth_id":
if auth == "" {
auth = sourceAuth(value)
}
case "host_address":
sawHost = true
if host == "" {
host = sourceHost(value)
}
}
}
switch {
case auth != "":
return auth, nil
case host != "":
return host, nil
case sawHost:
// Line 2 of a message received over the network names Exim's own
// user, which all such mail shares; it is not this message's source.
return "", errors.New("emailspool: unreadable host address in header")
case uid != "":
return uid, nil
}
return "", errors.New("emailspool: no message source in header")
}
// sourceOption splits an options line into its name and value. Exim marks a
// tainted value with a second dash and may name its quoter in parentheses.
func sourceOption(line string) (name, value string, ok bool) {
rest, ok := strings.CutPrefix(line, "-")
if !ok {
return "", "", false
}
rest = strings.TrimPrefix(rest, "-")
if strings.HasPrefix(rest, "(") {
end := strings.IndexByte(rest, ')')
if end < 0 {
return "", "", false
}
rest = rest[end+1:]
}
name, value, _ = strings.Cut(rest, " ")
return name, value, name != ""
}
func lastSourceField(s string) string {
s = strings.TrimRight(s, " ")
if i := strings.LastIndexByte(s, ' '); i >= 0 {
return s[i+1:]
}
return s
}
// sourceUID reads the uid from line 2, "<login> <uid> <gid>".
func sourceUID(line string) string {
fields := strings.Fields(line)
if len(fields) < 2 {
return ""
}
n, err := strconv.ParseUint(fields[1], 10, 32)
if err != nil {
return ""
}
return "uid:" + strconv.FormatUint(n, 10)
}
// sourceAuth accepts only printable ASCII ids that fit the key.
func sourceAuth(id string) string {
if id == "" || len("auth:")+len(id) > maxSourceKey {
return ""
}
for i := 0; i < len(id); i++ {
if id[i] < 0x20 || id[i] > 0x7e {
return ""
}
}
return "auth:" + strings.ToLower(id)
}
// sourceHost parses "[<addr>]:<port>", as Exim 4.100 writes it, or
// "<addr>.<port>", as older releases did. One IPv6 customer gets a /64, so
// an IPv6 host is keyed by its /64.
func sourceHost(value string) string {
var host, port string
if rest, ok := strings.CutPrefix(value, "["); ok {
var found bool
if host, port, found = strings.Cut(rest, "]:"); !found {
return ""
}
} else {
i := strings.LastIndexByte(value, '.')
if i < 0 {
return ""
}
host, port = value[:i], value[i+1:]
}
if _, err := strconv.ParseUint(port, 10, 16); err != nil {
return ""
}
addr, err := netip.ParseAddr(host)
if err != nil {
return ""
}
addr = addr.Unmap().WithZone("")
if addr.Is4() {
return "ip:" + addr.String()
}
return "ip:" + netip.PrefixFrom(addr, 64).Masked().String()
}
// Package eximlock serializes CSM's edits and rebuilds of shared Exim
// configuration. Every CSM worker that writes Exim files or runs buildeximconf
// takes this lock first.
package eximlock
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"syscall"
"time"
)
// Dir holds the lock file. Forward guard created this directory first; every
// worker must use the same inode, so the path stays where it is.
const Dir = "/var/lib/csm/forward_guard"
// ErrBusy means another CSM transaction holds the lock. The caller retries
// later; it is not a failed transaction.
var ErrBusy = errors.New("exim mutation lock is held by another CSM transaction")
// With runs mutate while holding dir/mutation.lock, without waiting for it.
// The lock does not exclude cPanel, which edits the same files on its own.
func With(dir string, mutate func() error) error {
// #nosec G301 -- Exim reads forward guard's bad-IP lookup in this directory as the mail transport user.
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
// #nosec G304 -- fixed CSM state directory; callers cannot supply a path through a worker request.
file, err := os.OpenFile(filepath.Join(dir, "mutation.lock"), os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return err
}
// Keep the inode after close: unlinking a lock allows two callers to lock
// different inodes. Close releases the lock even after a failed transaction.
defer func() { _ = file.Close() }()
// #nosec G115 -- POSIX file descriptors fit in int.
if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
if errors.Is(err, syscall.EWOULDBLOCK) {
return fmt.Errorf("%w: %v", ErrBusy, err)
}
return fmt.Errorf("exim mutation lock: %w", err)
}
return mutate()
}
// RecordWait bounds how long CSM waits for the lock before it records a
// request or starts a worker transaction. The daemon's own records hold the
// lock only for a small unsynced write or a rename; a worker transaction holds
// it for seconds or minutes and still reads as busy.
const RecordWait = time.Second
// WithRetry runs mutate under the lock like With, retrying a busy lock until
// wait has passed or ctx is canceled. Once the lock is acquired, mutate runs
// at most once; its errors are never retried.
func WithRetry(ctx context.Context, dir string, wait time.Duration, mutate func() error) error {
deadline := time.Now().Add(wait)
for {
if err := ctx.Err(); err != nil {
return err
}
acquired := false
err := With(dir, func() error {
acquired = true
if err := ctx.Err(); err != nil {
return err
}
return mutate()
})
if acquired || !errors.Is(err, ErrBusy) {
return err
}
remaining := time.Until(deadline)
if remaining <= 0 {
return err
}
timer := time.NewTimer(min(10*time.Millisecond, remaining))
select {
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case <-timer.C:
}
if ctxErr := ctx.Err(); ctxErr != nil {
return ctxErr
}
if !time.Now().Before(deadline) {
return err
}
}
}
package eximlog
import "strings"
// Arrival returns the message ID and envelope sender of an Exim arrival
// ("<=") record. The marker counts only after the complete mainlog prefix,
// so a "<=" inside a delivery reply or subject is not an arrival. A bounce
// has the sender "<>".
func Arrival(line string) (id, sender string, ok bool) {
accept := strings.Index(line, " <= ")
if accept < 0 || !arrivalPrefix(line[:accept]) {
return "", "", false
}
prefix := strings.Fields(line[:accept])
rest := line[accept+len(" <= "):]
end := envelopeEnd(rest)
if end <= 0 {
return "", "", false
}
return prefix[len(prefix)-1], rest[:end], true
}
// IsSubmission reports whether an arrival was submitted by a local process
// or authenticated SMTP client. Unlike Submitter, it does not require a
// tenant identity: Exim can authenticate without logging one, and local
// processes can submit SMTP on stdin instead of using P=local.
func IsSubmission(line string) bool {
fields, ok := submissionFields(line)
if !ok {
return false
}
return (!fields.remote && fields.user != "") ||
fields.protocol == "esmtpa" || fields.protocol == "esmtpsa"
}
package eximlog
import (
"regexp"
"strings"
)
// Only the record prefix can establish a failure. Searching the whole line
// would accept an authenticator marker in HELO, a subject or an SMTP command.
var authFailurePrefix = regexp.MustCompile(`^(?:\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?(?: [+-]\d{4})?(?: \[\d+\])? )?([A-Za-z0-9_]+) authenticator failed for `)
type AuthFailureRecord struct {
Authenticator string
ClientIP string
Code string
Account string
}
// AuthFailure reads one Exim authentication failure. Missing or unreadable
// reply codes remain empty; callers must not treat them as credential checks.
func AuthFailure(line string) AuthFailureRecord {
match := authFailurePrefix.FindStringSubmatchIndex(line)
if match == nil {
return AuthFailureRecord{}
}
ip, details := hostAndIdentClient(line[match[1]:])
out := AuthFailureRecord{Authenticator: line[match[2]:match[3]], ClientIP: ip}
// The attempted identity follows the reply, never the greeting. Preserve
// its complete spelling so a delimiter in the name cannot alias trust.
if _, account, ok := strings.Cut(details, " (set_id="); ok {
if account, ok = strings.CutSuffix(account, ")"); ok {
out.Account = account
}
}
if len(details) < 4 || details[3] != ' ' {
return out
}
for i := 0; i < 3; i++ {
if details[i] < '0' || details[i] > '9' {
return out
}
}
out.Code = details[:3]
return out
}
// Package eximlog extracts the connecting client from Exim main log lines.
//
// Exim renders the peer as `hostname (HELO) [IP]:port`, prefixed with `H=`
// in most records but bare in a few (authenticator failures, TLS errors).
// Everything before the bracketed address is attacker-influenced. The HELO
// may be an RFC 5321 address literal such as `[203.0.113.9]`, and servers that
// accept junk HELO values can log delimiter-like text there. The message
// Subject can carry brackets too. Every consumer that turns a log line into
// an IP for blocking or reputation scoring must therefore go through this
// package rather than grabbing the first bracketed token.
package eximlog
import (
"net"
"strings"
)
// ClientIP returns the connecting client's IP from an Exim log line, or ""
// when the line carries none. It reads either a real H= field or one of the
// known records that Exim writes through host_and_ident without an H= prefix.
func ClientIP(line string) string {
hStart, hasHField := HFieldStart(line)
bareStart, bareMarker := unprefixedClientStart(line)
if bareMarker >= 0 && (!hasHField || bareMarker < hFieldMarkerStart(line, hStart)) {
return hostAndIdentClientIP(line[bareStart:])
}
if hasHField {
return HFieldClientIP(line[hStart:])
}
return ""
}
func hFieldMarkerStart(line string, valueStart int) int {
if strings.HasPrefix(line, "H=") {
return 0
}
return valueStart - len(" H=")
}
// unprefixedClientStart recognizes the Exim records that render
// host_and_ident(FALSE) directly. A marker inside T= is message data, not a
// peer field. markerStart is returned separately so ClientIP can prefer a
// genuine H= field that occurs earlier on the line.
func unprefixedClientStart(line string) (start, markerStart int) {
markerStart = -1
offset, fields := peerFields(line)
t := strings.Index(fields, " T=")
for _, marker := range []string{
"authenticator failed for ",
"TLS error on connection from ",
"SMTP connection from ",
} {
idx := strings.Index(fields, marker)
if idx < 0 || (t >= 0 && t < idx) {
continue
}
if markerStart < 0 || offset+idx < markerStart {
markerStart = offset + idx
start = offset + idx + len(marker)
}
}
return start, markerStart
}
// peerFields bounds host-field searches to reception metadata on arrivals.
// The sender, authentication and post-size fields can contain client data.
// Other records are searched from the start.
func peerFields(line string) (int, string) {
accept := strings.Index(line, " <= ")
if accept < 0 || !arrivalPrefix(line[:accept]) {
return 0, line
}
sender := accept + len(" <= ")
offset := sender + envelopeEnd(line[sender:])
fields := line[offset:]
for _, boundary := range []string{" A=", " S="} {
if end := strings.Index(fields, boundary); end >= 0 {
fields = fields[:end]
}
}
return offset, fields
}
// HFieldStart returns the offset just past the H= marker and true when the
// line carries a real H= field. An H= that appears after T= is inside the
// Subject and is ignored, as is one in an arrival's message data.
func HFieldStart(line string) (int, bool) {
if strings.HasPrefix(line, "H=") {
return len("H="), true
}
offset, fields := peerFields(line)
if h := strings.Index(fields, " H="); h >= 0 {
if t := strings.Index(fields, " T="); t >= 0 && t < h {
return 0, false
}
return offset + h + len(" H="), true
}
return 0, false
}
// HFieldClientIP returns the connecting address inside an H= value.
func HFieldClientIP(s string) string {
ip, _ := HFieldClientIPAndEnd(s)
return ip
}
// HFieldClientIPAndEnd returns the connecting address and the byte offset
// immediately after its closing bracket. The offset lets callers discard the
// entire attacker-controlled H= value before parsing later Exim fields. A
// candidate must be followed by a real H= boundary, and a second plausible
// candidate makes the field ambiguous instead of letting junk HELO text win.
func HFieldClientIPAndEnd(s string) (string, int) {
// Remote ident follows the peer. A U= marker before a candidate means
// greeting delimiters hid the real peer and ident boundary. Text after
// the peer is not checked this way: subjects, addresses and login names
// placed there by any sender must not remove the connecting address.
identStart := strings.Index(s, " U=")
parenDepth := 0
quoted := false
client := ""
clientEnd := 0
for i := 0; i < len(s); i++ {
if quoted {
if s[i] == '\\' && i+1 < len(s) {
i++
continue
}
if s[i] == '"' {
quoted = false
}
continue
}
// Authentication and later message fields can contain client data.
// Their address literals and delimiters do not describe the peer.
if parenDepth == 0 && (strings.HasPrefix(s[i:], " A=") || strings.HasPrefix(s[i:], " S=")) {
break
}
switch s[i] {
case '(':
parenDepth++
case ')':
if parenDepth == 0 {
return "", 0
}
parenDepth--
case '"':
if parenDepth == 0 {
quoted = true
}
case '[':
end := strings.IndexByte(s[i+1:], ']')
if end < 0 {
return "", 0
}
if parenDepth == 0 && !interfaceAddressAt(s, i) {
candidate := s[i+1 : i+1+end]
after := s[i+1+end+1:]
if net.ParseIP(candidate) != nil && hFieldClientIPTerminated(after) {
if client != "" || (identStart >= 0 && identStart < i) {
return "", 0
}
client = candidate
clientEnd = i + end + 2
}
}
i += end + 1
}
}
if parenDepth != 0 || quoted {
return "", 0
}
return client, clientEnd
}
func beginsNextField(s string) bool {
if len(s) == 0 || (s[0] != ' ' && s[0] != '\t') {
return false
}
rest := strings.TrimLeft(s, " \t")
if strings.HasPrefix(rest, "for ") {
return true
}
eq := strings.IndexByte(rest, '=')
if eq <= 0 || eq > 3 {
return false
}
for i := 0; i < eq; i++ {
c := rest[i]
if (c < 'A' || c > 'Z') && (c < 'a' || c > 'z') {
return false
}
}
return true
}
func hFieldClientIPTerminated(s string) bool {
rest := withoutLoggedPort(s)
if after, ok := strings.CutPrefix(rest, " TFO"); ok {
rest = strings.TrimPrefix(after, "*")
}
return rest == "" || beginsNextField(rest) ||
strings.HasPrefix(rest, " authenticator failed") ||
strings.HasPrefix(rest, " rejected RCPT")
}
func hostAndIdentClientIPTerminated(s string) bool {
rest := withoutLoggedPort(s)
if rest == "" || beginsNextField(rest) || strings.HasPrefix(rest, ": ") {
return true
}
for _, suffix := range []string{" (", " lost", " D=", " closed"} {
if strings.HasPrefix(rest, suffix) {
return true
}
}
return false
}
func withoutLoggedPort(s string) string {
if len(s) < 2 || s[0] != ':' || s[1] < '0' || s[1] > '9' {
return s
}
i := 2
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
i++
}
return s[i:]
}
// hostAndIdentClientIP returns the connecting address from Exim's unprefixed
// host_and_ident output. Exim encloses the HELO in parentheses before the
// client, so an address literal inside that group is attacker text. More than
// one plausible peer or malformed parentheses are rejected; otherwise a junk
// HELO could make CSM block an address supplied by the peer.
func hostAndIdentClientIP(s string) string {
ip, _ := hostAndIdentClient(s)
return ip
}
// hostAndIdentClient is hostAndIdentClientIP that also returns the failure
// details after the peer, when they follow it.
func hostAndIdentClient(s string) (ip, details string) {
// host_and_ident writes remote U= after the peer, as in an H= record.
identStart := strings.Index(s, " U=")
parenDepth := 0
quoted := false
client := ""
for i := 0; i < len(s); i++ {
if quoted {
if s[i] == '\\' && i+1 < len(s) {
i++
continue
}
if s[i] == '"' {
quoted = false
}
continue
}
switch s[i] {
case '(':
parenDepth++
case ')':
if parenDepth == 0 {
return "", ""
}
parenDepth--
case '"':
if parenDepth == 0 {
quoted = true
}
case '[':
end := strings.IndexByte(s[i+1:], ']')
if end < 0 {
return "", ""
}
if parenDepth == 0 && !interfaceAddressAt(s, i) {
candidate := s[i+1 : i+1+end]
after := s[i+1+end+1:]
if net.ParseIP(candidate) != nil && hostAndIdentClientIPTerminated(after) {
if client != "" || (identStart >= 0 && identStart < i) {
return "", ""
}
// Failure details, including the attempted login name,
// follow the peer and are not host information.
if details, ok := failureDetails(after); ok {
return candidate, details
}
client = candidate
}
}
i += end + 1
}
}
if parenDepth != 0 || quoted {
return "", ""
}
return client, ""
}
// failureDetails returns the failure details when s, the text after a peer's
// closing bracket, holds only the logged port, local interface and
// connection ID before the ": " that starts them. Remote ident text can
// contain that separator, so a U= field never qualifies.
func failureDetails(s string) (string, bool) {
rest := withoutLoggedPort(s)
if strings.HasPrefix(rest, " I=[") {
end := strings.IndexByte(rest, ']')
if end < 0 || net.ParseIP(rest[len(" I=["):end]) == nil {
return "", false
}
rest = withoutLoggedPort(rest[end+1:])
}
if strings.HasPrefix(rest, " Ci=") {
digits := rest[len(" Ci="):]
n := 0
for n < len(digits) && digits[n] >= '0' && digits[n] <= '9' {
n++
}
if n == 0 {
return "", false
}
rest = digits[n:]
}
details, ok := strings.CutPrefix(rest, ": ")
return details, ok
}
func interfaceAddressAt(s string, bracket int) bool {
return bracket >= len(" I=") && s[bracket-len(" I="):bracket] == " I="
}
package eximlog
import (
"strings"
"time"
)
// AuthenticatedUser reads the Dovecot authentication identity on an Exim
// arrival record. The envelope sender, HELO and quoted fields are not proof
// of authentication. The complete mainlog prefix must identify an arrival.
func AuthenticatedUser(line string) string {
fields, ok := submissionFields(line)
if !ok {
return ""
}
return fields.auth
}
// Submitter returns an authenticated identity, or the local Exim caller on
// a non-network P=local arrival. Records containing a remote U= identity
// are ambiguous and cannot prove a submitter. The caller must resolve the
// returned identity against the local inventory before assigning a tenant.
func Submitter(line string) string {
fields, ok := submissionFields(line)
if !ok {
return ""
}
if fields.auth != "" {
return fields.auth
}
if !fields.remote && !fields.authSeen && fields.protocol == "local" {
return fields.user
}
return ""
}
// LocalSubmitter returns the local Exim caller of an arrival a process on
// this host submitted without authenticating: P=local, or local SMTP on stdin
// such as `sendmail -bs`. Network and authenticated arrivals return "". The
// caller must resolve the name against the local account inventory.
func LocalSubmitter(line string) string {
fields, ok := submissionFields(line)
if !ok || fields.remote || fields.authSeen {
return ""
}
if fields.protocol != "local" && !strings.HasPrefix(fields.protocol, "local-") {
return ""
}
return fields.user
}
type submitFields struct {
auth, user, protocol string
remote, authSeen bool
}
func submissionFields(line string) (submitFields, bool) {
var out submitFields
accept := strings.Index(line, " <= ")
if accept < 0 || !arrivalPrefix(line[:accept]) {
return out, false
}
rest := line[accept+len(" <= "):]
end := envelopeEnd(rest)
if end <= 0 || end == len(rest) {
return out, false
}
rest = rest[end:]
seen := map[string]bool{}
metadata:
for rest != "" {
rest = strings.TrimLeft(rest, " \t\r\n")
// Exim writes submission metadata before the message size. Later
// fields contain message data, including addr-spec message IDs with
// quoted words that can resemble authentication or local-user fields.
if rest == "" || strings.HasPrefix(rest, "S=") || strings.HasPrefix(rest, "T=") || strings.HasPrefix(rest, "for ") {
break
}
// Only a top-level host field establishes a network submission.
// Searching ahead would mistake H= inside message data for metadata.
if strings.HasPrefix(rest, "H=") {
_, clientEnd := HFieldClientIPAndEnd(rest[len("H="):])
if out.remote || clientEnd == 0 {
return submitFields{}, false
}
out.remote = true
rest = rest[len("H=")+clientEnd:]
continue
}
end := fieldEnd(rest)
field := rest[:end]
key, value, hasValue := strings.Cut(field, "=")
if hasValue && (key == "A" || key == "U" || key == "P") {
if seen[key] {
return submitFields{}, false
}
seen[key] = true
switch key {
case "A":
out.authSeen = true
authenticator, identity, ok := strings.Cut(value, ":")
if ok {
// smtp_mailauth can append an envelope identity after the
// authenticated identity. Only the second A= item is trusted.
auth, _, mailauth := strings.Cut(identity, ":")
if (authenticator == "dovecot_login" || authenticator == "dovecot_plain") &&
strings.Count(auth, "@") <= 1 && !strings.HasPrefix(auth, "@") &&
!strings.HasSuffix(auth, "@") && !strings.ContainsAny(auth, " \t\r\n\"\\") {
out.auth = auth
}
if mailauth {
// The optional envelope is client-supplied xtext,
// not additional submission metadata.
break metadata
}
}
case "U":
out.user = value
case "P":
out.protocol = value
}
}
rest = rest[end:]
}
// Exim appends remote RFC 1413 ident text without quoting spaces. It
// can therefore imitate all following metadata, including A= and S=.
// Neither an apparent authentication field nor its order proves an
// identity on such a record. Local U= still comes from the server.
if out.remote && seen["U"] {
return submitFields{}, false
}
return out, true
}
// MessagePrefix reports whether prefix is exactly what Exim writes before a
// message's log text: date, time, optional zone and PID, and message ID.
func MessagePrefix(prefix string) bool {
return arrivalPrefix(prefix)
}
// An arrival must start with a date, time, optional zone/PID and message ID.
// A <= fragment in a delivery reply or subject cannot supply an identity.
func arrivalPrefix(prefix string) bool {
p, rest, ok := SplitPrefix(prefix, time.UTC)
return ok && p.MessageID != "" && strings.TrimSpace(rest) == ""
}
func envelopeEnd(s string) int {
quoted := false
for i := 0; i < len(s); i++ {
switch {
case s[i] == '\\' && i+1 < len(s):
i++
case s[i] == '"':
quoted = !quoted
case !quoted && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n'):
return i
}
}
return len(s)
}
// fieldEnd consumes quoted values together so their contents cannot
// masquerade as authentication or local-user metadata. Quotes can begin
// within a value, including an optional mailbox appended to an A= field.
func fieldEnd(s string) int {
// Apostrophes are address characters, not quoting delimiters. Exim
// also preserves quoted pairs outside double-quoted local parts.
return envelopeEnd(s)
}
package eximlog
import "time"
// LineTime is the time Exim wrote at the start of line: the date, the time
// with any fraction log_selector +millisec adds, and the zone log_timezone
// adds, else the time in local. An explicit zone that is not a valid offset
// dates nothing; client text later in the line is never read.
func LineTime(line string, local *time.Location) (time.Time, bool) {
p, _, ok := SplitPrefix(line, local)
return p.Time, ok
}
// validOffset reports whether zone is +hhmm or -hhmm with hh below 24 and
// mm below 60.
func validOffset(zone string) bool {
if len(zone) != 5 {
return false
}
for _, c := range zone[1:] {
if c < '0' || c > '9' {
return false
}
}
return zone[1:3] < "24" && zone[3:5] < "60"
}
package eximlog
import (
"strings"
"time"
)
// Prefix is what Exim writes before the text of a main log line: the time,
// the process ID when log_selector has +pid, and the message ID on lines
// about one message.
type Prefix struct {
Time time.Time
PID string
MessageID string
}
// SplitPrefix reads the prefix that starts line and returns it with the text
// after it. The time carries the zone log_timezone writes, else local. An
// explicit zone that is not a valid offset dates nothing. A field that does
// not have the shape of a PID or message ID is left in the text.
func SplitPrefix(line string, local *time.Location) (Prefix, string, bool) {
var p Prefix
date, rest := nextField(line)
clock, rest := nextField(rest)
field, after := nextField(rest)
if isPID(field) {
// Exim writes the zone with the time, then the PID. A zone after
// the PID is read too.
p.PID, rest = field[1:len(field)-1], after
field, after = nextField(rest)
}
stamp := date + " " + clock
var err error
if field != "" && (field[0] == '+' || field[0] == '-') {
if !validOffset(field) {
return Prefix{}, "", false
}
p.Time, err = time.Parse(time.DateTime+" -0700", stamp+" "+field)
rest = after
} else {
p.Time, err = time.ParseInLocation(time.DateTime, stamp, local)
}
if err != nil {
return Prefix{}, "", false
}
if field, after := nextField(rest); p.PID == "" && isPID(field) {
p.PID, rest = field[1:len(field)-1], after
}
if field, after := nextField(rest); IsMessageID(field) {
p.MessageID, rest = field, after
}
return p, strings.TrimLeft(rest, " \t"), true
}
func nextField(s string) (field, rest string) {
s = strings.TrimLeft(s, " \t")
if i := strings.IndexAny(s, " \t"); i >= 0 {
return s[:i], s[i:]
}
return s, ""
}
func isPID(field string) bool {
return len(field) >= 3 && field[0] == '[' && field[len(field)-1] == ']' &&
strings.Trim(field[1:len(field)-1], "0123456789") == ""
}
// IsMessageID reports whether field has the shape of an Exim message ID:
// three dash-separated base62 parts. Exim writes 6-6-2 or, from 4.97,
// 6-11-4; in a log line the position, not the length, proves the field is
// server-written.
func IsMessageID(field string) bool {
parts := strings.Split(field, "-")
if len(parts) != 3 {
return false
}
for _, part := range parts {
if part == "" || strings.Trim(part, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") != "" {
return false
}
}
return true
}
package eximlog
import "strings"
// Recipients returns the envelope recipients Exim logged after the top-level
// "for" of an arrival record. Quoted subjects, HELO text and quoted envelope
// senders can carry the same word, so fields are consumed with the quoting
// rules of the submission metadata. Nil when the record is not an arrival or
// logs no recipient list.
func Recipients(line string) []string {
accept := strings.Index(line, " <= ")
if accept < 0 || !arrivalPrefix(line[:accept]) {
return nil
}
rest := line[accept+len(" <= "):]
end := envelopeEnd(rest)
if end <= 0 || end == len(rest) {
return nil
}
rest = rest[end:]
remote := false
for {
rest = strings.TrimLeft(rest, " \t\r\n")
if rest == "" {
return nil
}
if strings.HasPrefix(rest, "H=") {
_, clientEnd := HFieldClientIPAndEnd(rest[len("H="):])
if clientEnd == 0 {
return nil
}
remote = true
rest = rest[len("H=")+clientEnd:]
continue
}
end := fieldEnd(rest)
field := rest[:end]
// Remote ident and the optional MAIL AUTH envelope are logged as
// unquoted client text. Neither can prove where the list begins.
if (remote && strings.HasPrefix(field, "U=")) ||
(strings.HasPrefix(field, "A=") && strings.Count(field, ":") >= 2) {
return nil
}
if field == "for" {
return recipientAddresses(rest[end:])
}
rest = rest[end:]
}
}
// A quoted local part is one recipient even when it contains whitespace.
// Reject incomplete quoting instead of counting fragments of an address.
func recipientAddresses(rest string) []string {
var recipients []string
for {
rest = strings.TrimLeft(rest, " \t\r\n")
if rest == "" {
return recipients
}
end := envelopeEnd(rest)
address := rest[:end]
quoted := false
for i := 0; i < len(address); i++ {
if address[i] == '\\' && i+1 < len(address) {
i++
} else if address[i] == '"' {
quoted = !quoted
}
}
if quoted {
return nil
}
recipients = append(recipients, address)
rest = rest[end:]
}
}
package eximlog
import "strings"
// Subject returns the T= subject of an Exim arrival record. It reads only
// the arrival's own fields, so a T=" inside the sender, HELO or message ID
// is skipped. Records with unquoted remote ident or MAIL AUTH text are
// ambiguous and return no subject. Quote and backslash escapes are decoded;
// escapes for non-printing bytes stay as written.
// A subject cut off by a truncated line is returned as far as it goes.
func Subject(line string) (string, bool) {
accept := strings.Index(line, " <= ")
if accept < 0 || !arrivalPrefix(line[:accept]) {
return "", false
}
rest := line[accept+len(" <= "):]
end := envelopeEnd(rest)
if end <= 0 {
return "", false
}
rest = rest[end:]
remote := false
for {
rest = strings.TrimLeft(rest, " \t\r\n")
if rest == "" || strings.HasPrefix(rest, "for ") {
return "", false
}
if strings.HasPrefix(rest, "H=") {
_, clientEnd := HFieldClientIPAndEnd(rest[len("H="):])
if clientEnd == 0 {
return "", false
}
remote = true
rest = rest[len("H=")+clientEnd:]
continue
}
if quoted, ok := strings.CutPrefix(rest, `T="`); ok {
return unescapeSubject(quoted), true
}
end := fieldEnd(rest)
field := rest[:end]
// These values are logged without quoting spaces, so their text
// can imitate every later field, including the size and subject.
if (remote && strings.HasPrefix(field, "U=")) ||
(strings.HasPrefix(field, "A=") && strings.Count(field, ":") >= 2) {
return "", false
}
rest = rest[end:]
}
}
// unescapeSubject undoes only the \" and \\ quoting receive.c adds. The
// non-printing escapes stay as text rather than becoming terminal controls
// or line breaks in a finding. Literal tabs pass through as Exim logs them.
func unescapeSubject(s string) string {
var b strings.Builder
for i := 0; i < len(s); i++ {
switch {
case s[i] == '"':
return b.String()
case s[i] == '\\' && i+1 < len(s) && (s[i+1] == '"' || s[i+1] == '\\'):
i++
}
b.WriteByte(s[i])
}
return b.String()
}
package firewall
import (
"encoding/json"
"time"
)
// RetiredActionState is the committed state an unknown block action retires
// to as expired_unproven: every member target is timed and expired,
// and every row its Before and After states disagree on is a verified timed
// block that has expired at at. Committed state keeps the rows both
// states agree on. Any other action reports false and waits for an
// operator's Resolve: one that is not a block, changes anything but blocked
// rows, or has an effect that is permanent, unverified or still live.
func RetiredActionState(a FirewallAction, at time.Time) (FirewallState, bool) {
if at.IsZero() || a.Phase != "unknown" {
return FirewallState{}, false
}
for _, m := range a.ActionMembers() {
if m.Request.Operation != "block" || m.Request.TTL <= 0 {
return FirewallState{}, false
}
found := false
for _, row := range a.After.Blocked {
if sameAddress(row.IP, m.Request.Target) {
if row.Unverified || row.ExpiresAt.IsZero() || row.ExpiresAt.After(at) {
return FirewallState{}, false
}
found = true
}
}
if !found {
return FirewallState{}, false
}
}
if a.Request.Operation != "block" || !sameRows(a.Before.Allowed, a.After.Allowed) || !sameRows(a.Before.BlockedNet, a.After.BlockedNet) || !sameRows(a.Before.PortAllowed, a.After.PortAllowed) {
return FirewallState{}, false
}
expired := func(row BlockedEntry) bool {
return !row.Unverified && !row.ExpiresAt.IsZero() && !row.ExpiresAt.After(at)
}
unmatched := make(map[BlockedEntry]int, len(a.After.Blocked))
for _, row := range a.After.Blocked {
unmatched[blockedRowKey(row)]++
}
state := FirewallState{Allowed: a.Before.Allowed, BlockedNet: a.Before.BlockedNet, PortAllowed: a.Before.PortAllowed}
for _, row := range a.Before.Blocked {
if key := blockedRowKey(row); unmatched[key] > 0 {
unmatched[key]--
state.Blocked = append(state.Blocked, row)
} else if !expired(row) {
return FirewallState{}, false
}
}
for _, row := range a.After.Blocked {
if key := blockedRowKey(row); unmatched[key] > 0 {
unmatched[key]--
if !expired(row) {
return FirewallState{}, false
}
}
}
return state, true
}
// blockedRowKey identifies a row by value whatever location its times carry.
func blockedRowKey(row BlockedEntry) BlockedEntry {
row.BlockedAt, row.ExpiresAt = row.BlockedAt.UTC(), row.ExpiresAt.UTC()
return row
}
// sameRows compares two lists by value; an empty list and none are equal.
func sameRows[T any](a, b []T) bool {
if len(a) == 0 || len(b) == 0 {
return len(a) == len(b)
}
left, err := json.Marshal(a)
if err != nil {
return false
}
right, err := json.Marshal(b)
return err == nil && string(left) == string(right)
}
package firewall
import (
"errors"
"net"
"slices"
"sync"
"time"
)
var (
ErrActionDryRun = errors.New("firewall action suppressed by dry-run policy")
ErrActionAuditPending = errors.New("firewall action verified; audit delivery pending")
ErrActionMissing = errors.New("firewall action not found")
ErrActionUnknown = errors.New("firewall action outcome requires recovery")
ErrActionFailed = errors.New("firewall action was not applied")
ErrScanBudget = errors.New("firewall scan budget exhausted")
)
// ActionRequest is the identity and policy input for one firewall decision.
// ID identifies a request, not an individual attempt to contact the kernel.
type ActionRequest struct {
ID string `json:"id"`
Operation string `json:"operation"`
Target string `json:"target"`
Reason string `json:"reason"`
Actor string `json:"actor"`
ActorDetail string `json:"actor_detail,omitempty"`
Source string `json:"source"`
FindingID string `json:"finding_id,omitempty"`
IncidentID string `json:"incident_id,omitempty"`
UndoOf string `json:"undo_of,omitempty"`
Automatic bool `json:"automatic,omitempty"`
TTL time.Duration `json:"ttl,omitempty"`
// LoginSpray marks an automatic mail spray block: its row is the first
// eviction victim at the temporary cap.
LoginSpray bool `json:"login_spray,omitempty"`
}
// ScanAdmission uses the existing scan hourly policy. A nil admission is
// exempt; callers must not charge non-scan sources to the scan budget.
type ScanAdmission struct {
Window string `json:"window"`
Limit int `json:"limit"`
}
// MaxActionMembers bounds one batch manifest. It matches the block batch
// applier, so one durable batch can carry one applier batch.
const MaxActionMembers = MaxBlockBatchMembers
// ActionMember is one immutable single-target request inside a batch, with
// its admission data and the effects its plan records.
type ActionMember struct {
Request ActionRequest `json:"request"`
Budget *ScanAdmission `json:"budget,omitempty"`
// Victims are the rows this member removes to make room, exactly as they
// were, so undo can prove they are still current.
Victims []BlockedEntry `json:"victims,omitempty"`
}
// FirewallAction retains both complete states until the kernel outcome can be
// proved. Admission does not publish After as committed firewall state.
type FirewallAction struct {
Ruleset *ActionRuleset `json:"ruleset,omitempty"`
KernelBefore []ActionSet `json:"kernel_before,omitempty"`
KernelAfter []ActionSet `json:"kernel_after,omitempty"`
Request ActionRequest `json:"request"`
// Members is the batch manifest. Without it the action is one member, its
// Request and Budget, as every record written before manifests is. With
// it, a batch of several members uses Request as its header.
Members []ActionMember `json:"members,omitempty"`
Before FirewallState `json:"before"`
After FirewallState `json:"after"`
Revision uint64 `json:"revision"`
Phase string `json:"phase"`
// Retired marks an unknown outcome recovery settled as expired_unproven:
// every effect had expired and the kernel held the retired state. The
// outcome stays unknown, but it no longer holds later mutations.
Retired bool `json:"retired,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Detail string `json:"detail,omitempty"`
Budget *ScanAdmission `json:"budget,omitempty"`
AuditVersion uint64 `json:"audit_version"`
AuditAck uint64 `json:"audit_ack"`
}
// ActionMembers returns the batch manifest, deriving the single member of an
// action that carries none. Callers must not modify the result.
func (a FirewallAction) ActionMembers() []ActionMember {
if a.Members == nil {
return []ActionMember{{Request: a.Request, Budget: a.Budget}}
}
return a.Members
}
// SameManifest reports whether b is the same batch as a: the same header, the
// same members in the same order and the same effects. Admission data is not
// compared: a replay in a later window keeps the charge taken first.
func (a FirewallAction) SameManifest(b FirewallAction) bool {
return a.Request == b.Request && slices.EqualFunc(a.ActionMembers(), b.ActionMembers(), func(x, y ActionMember) bool {
return x.Request == y.Request && slices.EqualFunc(x.Victims, y.Victims, SameBlockedEntry)
})
}
// removedVictims returns the victims of m that a took out: rows its Before
// state held exactly, at an address no row of its After state names. A batch
// keeps every unexpired row, so a victim stored on a batch member was never
// evicted; it is neither audited as an eviction nor restored by undo.
func removedVictims(a FirewallAction, m ActionMember) []BlockedEntry {
if len(a.Members) > 1 {
return nil
}
var out []BlockedEntry
for _, v := range m.Victims {
held := slices.ContainsFunc(a.Before.Blocked, func(row BlockedEntry) bool { return SameBlockedEntry(row, v) })
kept := slices.ContainsFunc(a.After.Blocked, func(row BlockedEntry) bool { return sameAddress(row.IP, v.IP) })
if held && !kept {
out = append(out, v)
}
}
return out
}
// sameAddress reports whether two spellings name one IP address.
func sameAddress(a, b string) bool {
x := net.ParseIP(a)
return x != nil && x.Equal(net.ParseIP(b))
}
// IsRequest reports whether a is the one-member action for exactly req, so a
// batch whose header happens to match is never replayed as one request.
func (a FirewallAction) IsRequest(req ActionRequest) bool {
members := a.ActionMembers()
return a.Request == req && len(members) == 1 && members[0].Request == req
}
// ActionStore owns atomic admission, accounting, outcome and audit delivery
// state. Implementations must keep kernel and audit I/O outside transactions.
type ActionStore interface {
StateStore
AdmitFirewallAction(FirewallAction) (FirewallAction, bool, error)
ReadFirewallAction(string) (FirewallAction, error)
// ReadFirewallBatchMember returns the batch of two or more members that
// indexes a member ID, or ErrActionMissing.
ReadFirewallBatchMember(string) (FirewallAction, error)
PendingFirewallActions() ([]FirewallAction, error)
TransitionFirewallAction(string, string, string, time.Time) (FirewallAction, error)
// RetireFirewallAction commits the state an unknown action retires to
// at the given instant (RetiredActionState) and marks it retired.
RetireFirewallAction(string, time.Time) (FirewallAction, error)
FirewallAuditPending() ([]FirewallAction, error)
AcknowledgeFirewallAudit(string, uint64) error
ReadFirewallScanBudget(string) (int, error)
}
type ActionObservation struct{ Before, After bool }
// ActionKernel compares the affected targets, including provenance and expiry.
// Recovery only observes; it never calls ApplyFirewallAction.
type ActionKernel interface {
ObserveFirewallAction(FirewallAction) (ActionObservation, error)
ApplyFirewallAction(FirewallAction) error
}
// Lifecycle is owned by one engine. Its lock serializes execution with recovery
// and undo, while the store protects admission against competing requests.
type Lifecycle struct {
mu sync.Mutex
Store ActionStore
Audit func(FirewallAction) error
// RetirementClockTrusted runs with the engine lock held and must not
// call back into the engine. Nil or false refuses retirement; an error
// also reaches recovery's caller, even if the returned bool is true.
RetirementClockTrusted func() (bool, error)
}
// ActionSet is detached recovery evidence for one complete nftables set.
// Expiry is absolute so neither retry nor undo can renew a timed element.
type ActionSet struct {
Exists bool `json:"exists"`
Name string `json:"name"`
Elements []ActionElement `json:"elements"`
}
type ActionElement struct {
Key []byte `json:"key"`
End bool `json:"end,omitempty"`
Comment string `json:"comment,omitempty"`
ExpiresAt time.Time `json:"expires_at,omitempty"`
}
// ActionRuleset proves an atomic whole-ruleset application. An unrelated
// namespace edit invalidates this conservative recovery proof.
type ActionRuleset struct {
Generation uint32 `json:"generation_before"`
Marker string `json:"marker"`
}
package firewall
import (
"errors"
"time"
"github.com/pidginhost/csm/internal/actionlog"
)
// recordFirewallAction mirrors successful legacy entries. Paths with multiple
// outcomes record at their public operation boundary instead.
func recordFirewallAction(action, ip, reason, source string, duration time.Duration) {
recordFirewallResult(action, ip, reason, source, duration, actionlog.Applied, nil)
}
func firewallRecord(action, ip, reason, source string, duration time.Duration) actionlog.Record {
op, actor := firewallActionOp(action, source)
rec := actionlog.Record{Op: op, Action: action, Actor: actor, Target: ip, Reason: reason, Result: actionlog.Applied}
if duration > 0 {
rec.ActorDetail = "expires in " + duration.String()
}
// A block/allow reversal needs the prior state. Creating an allow is not
// an undo: it also suppresses future automatic blocks for this address.
return rec
}
func recordFirewallResult(action, ip, reason, source string, duration time.Duration, result actionlog.Result, err error) {
recordFirewallFindingResult(action, ip, reason, source, duration, result, err, "")
}
func recordFirewallFindingResult(action, ip, reason, source string, duration time.Duration, result actionlog.Result, err error, findingID string) {
rec := firewallRecord(action, ip, reason, source, duration)
rec.FindingID = findingID
rec.Result = result
if err != nil {
rec.Error = err.Error()
rec.Result = actionlog.Failed
if errors.Is(err, ErrIPProtected) {
rec.Result = actionlog.Refused
}
}
actionlog.Write(rec)
}
func recordFirewallFailure(action, ip, reason, source string, duration time.Duration, err error) {
if err != nil {
recordFirewallResult(action, ip, reason, source, duration, actionlog.Failed, err)
}
}
func recordBlockOutcome(ip, reason string, duration time.Duration, outcome BlockOutcome, err error, manual bool, findingID string) {
if !manual && outcome == BlockOutcomeNoop && err == nil {
return
}
rec := firewallRecord("block", ip, reason, InferProvenance("block", reason), duration)
rec.FindingID = findingID
rec.Op = "respond.block_ip"
if manual {
rec.Op = "operate.manual_firewall"
}
switch outcome {
case BlockOutcomeDryRun:
rec.Result = actionlog.DryRun
case BlockOutcomeAllowed, BlockOutcomeAllowlisted:
rec.Result = actionlog.Refused
}
if err != nil {
rec.Result = actionlog.Failed
rec.Error = err.Error()
if errors.Is(err, ErrIPProtected) {
rec.Result = actionlog.Refused
}
}
actionlog.Write(rec)
}
func firewallActionOp(action, source string) (string, actionlog.Actor) {
actor := actionlog.DefaultActor()
switch source {
case SourceCLI:
actor = actionlog.CLI
case SourceWebUI:
actor = actionlog.WebUI
}
switch action {
case "apply", "restart":
return "integrate.firewall_ruleset", actor
}
if source == SourceCLI || source == SourceWebUI || actor == actionlog.CLI {
return "operate.manual_firewall", actor
}
switch action {
case "flush", "unblock", "remove_allow", "allow_port", "remove_port_allow", "unblock_subnet":
return "operate.manual_firewall", actor
}
return "respond.block_ip", actor
}
//go:build linux
package firewall
import (
"fmt"
"time"
"github.com/google/nftables"
)
// Persist intent before touching the kernel so a restart converges to the
// requested policy. A failed atomic kernel batch restores the previous intent.
// The caller holds e.mu across both writes and the kernel transaction.
func (e *Engine) commitAllowedRemovals(prior, next FirewallState, removeIPs []string, req ActionRequest) error {
if e.lifecycle != nil {
return e.runDurableLocked(req, nil, next)
}
if err := e.persistFirewallIntent(prior, next); err != nil {
return fmt.Errorf("persisting allow removal: %w", err)
}
if err := e.deleteAllowedElements(removeIPs, next.Allowed); err != nil {
if restoreErr := e.saveState(&prior); restoreErr != nil {
return fmt.Errorf("partial failure: %w (state restore failed: %w)", err, restoreErr)
}
return err
}
return nil
}
func (e *Engine) deleteAllowedElements(ips []string, remaining []AllowedEntry) error {
if len(ips) == 0 {
return nil
}
conn := e.newMutationConn()
sets := make(map[*nftables.Set]bool)
for _, ip := range ips {
set, key, err := e.resolveIPSet(ip, e.setAllowed, e.setAllowed6)
if err != nil || set == nil {
continue
}
if err := conn.SetDeleteElements(set, []nftables.SetElement{{Key: key}}); err != nil {
return fmt.Errorf("removing allow for %s: %w", ip, err)
}
sets[set] = true
}
if err := conn.Flush(); err != nil {
if !isNftNotFound(err) {
return fmt.Errorf("removing allows: %w", err)
}
// An element already absent aborts the whole delete batch. Rebuild the
// affected sets atomically so one missing element cannot wedge cleanup.
return e.replaceAllowedSets(sets, remaining)
}
return nil
}
func (e *Engine) replaceAllowedSets(sets map[*nftables.Set]bool, entries []AllowedEntry) error {
conn := e.newMutationConn()
elements := make(map[*nftables.Set][]nftables.SetElement)
seen := make(map[string]bool)
now := time.Now()
for _, entry := range entries {
if !entry.ExpiresAt.IsZero() && !now.Before(entry.ExpiresAt) {
continue
}
ip, ok := canonicalIPKey(entry.IP)
if !ok || seen[ip] {
continue
}
set, key, err := e.resolveIPSet(ip, e.setAllowed, e.setAllowed6)
if err != nil || !sets[set] {
continue
}
seen[ip] = true
elements[set] = append(elements[set], nftables.SetElement{Key: key})
}
for set := range sets {
conn.FlushSet(set)
if err := addElementsChunked(conn, set, elements[set]); err != nil {
return err
}
}
if err := conn.Flush(); err != nil {
return fmt.Errorf("rebuilding allowed sets: %w", err)
}
return nil
}
// A separate non-lasting connection discards queued messages on any error.
// Its constructor cannot fail because it does not dial until Flush.
func (e *Engine) newMutationConn() *nftables.Conn {
conn, _ := nftables.New(nftables.WithSockOptions(applyNFTSocketBuffer), nftables.WithNetNSFd(e.conn.NetNS), nftables.WithTestDial(e.conn.TestDial))
return conn
}
package firewall
import (
"bufio"
"encoding/json"
"log"
"os"
"path/filepath"
"time"
)
const maxAuditFileSize = 10 * 1024 * 1024 // 10 MB
// AuditEntry records a firewall modification for compliance and forensics.
type AuditEntry struct {
Timestamp time.Time `json:"timestamp"`
Action string `json:"action"` // block, unblock, allow, remove_allow, flush, apply
IP string `json:"ip,omitempty"`
Reason string `json:"reason,omitempty"`
Source string `json:"source,omitempty"`
Duration string `json:"duration,omitempty"`
}
// AppendAudit writes an audit entry to the JSONL audit log.
// Rotates the log when it exceeds 10 MB.
func AppendAudit(statePath, action, ip, reason, source string, duration time.Duration) {
if source == "" {
source = InferProvenance(action, reason)
}
recordFirewallAction(action, ip, reason, source, duration)
appendAudit(statePath, action, ip, reason, source, duration)
}
func appendAudit(statePath, action, ip, reason, source string, duration time.Duration) {
if source == "" {
source = InferProvenance(action, reason)
}
entry := AuditEntry{
Timestamp: time.Now(),
Action: action,
IP: ip,
Reason: reason,
Source: source,
}
if duration > 0 {
entry.Duration = duration.String()
}
path := filepath.Join(statePath, "audit.jsonl")
data, err := json.Marshal(entry)
if err != nil {
return
}
data = append(data, '\n')
// Rotate if file exceeds max size
if info, statErr := os.Stat(path); statErr == nil && info.Size() > maxAuditFileSize {
_ = os.Rename(path, path+".1")
}
// #nosec G304 -- path is filepath.Join under operator-configured statePath.
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
// Without this log, perm/disk-full/inode-exhaust modes drop the
// audit entry with no operator-visible signal -- the Write/Close
// branches below already log, so a silent Open path was the only
// remaining hole in the audit pipeline.
log.Printf("firewall: audit open failed for %s: %v", path, err)
return
}
if _, writeErr := f.Write(data); writeErr != nil {
_ = f.Close()
log.Printf("firewall: audit write failed for %s: %v", path, writeErr)
return
}
// Close error on a writable file is the disk-full / fsync signal --
// without it, a dropped audit entry leaves no record anywhere.
if closeErr := f.Close(); closeErr != nil {
log.Printf("firewall: audit close failed for %s: %v", path, closeErr)
}
}
// ReadAuditLog returns the last N audit entries from the log.
func ReadAuditLog(statePath string, limit int) []AuditEntry {
path := filepath.Join(statePath, "firewall", "audit.jsonl")
// #nosec G304 -- filepath.Join under operator-configured statePath.
f, err := os.Open(path)
if err != nil {
return nil
}
defer f.Close()
var all []AuditEntry
scanner := bufio.NewScanner(f)
for scanner.Scan() {
var entry AuditEntry
if json.Unmarshal(scanner.Bytes(), &entry) == nil {
all = append(all, entry)
}
}
if limit > 0 && len(all) > limit {
all = all[len(all)-limit:]
}
return all
}
package firewall
import (
"errors"
"time"
)
// The block batch API compiles on every platform, so portable code can
// compose it; only Linux runs it.
// MaxBlockBatchMembers bounds one block batch, as the admission ledger
// bounds an execution batch.
const MaxBlockBatchMembers = 64
// MaxBlockReasonBytes bounds the reason a batch member writes on its row.
const MaxBlockReasonBytes = 256
// BlockMember is one timed full-address block an admission attempt may
// apply: the attempt's ID, the canonical address, the reason written on
// its row and the absolute expiry its element never outlives. Victim is
// the admission block whose slot it takes, as the ledger claimed it; zero
// when it takes a free slot.
type BlockMember struct {
Action string
IP string
Reason string
ExpiresAt time.Time
Victim BlockVictim
}
// BlockVictim is an admission block a member removes to take its slot:
// its action, its canonical address and the absolute expiry
// it was applied with.
type BlockVictim struct {
Action string
IP string
ExpiresAt time.Time
}
// BlockRefusal says why a batch member cannot run; empty means it can.
type BlockRefusal string
const (
// BlockRefusedInvalid: a malformed action ID, address, reason or
// expiry, or an address not in canonical form.
BlockRefusedInvalid BlockRefusal = "invalid"
// BlockRefusedExpired: less than a millisecond of the block is left, or
// a durable batch member too short to outlive its verification.
BlockRefusedExpired BlockRefusal = "expired"
// BlockRefusedUnsupported: the address family is not enabled.
BlockRefusedUnsupported BlockRefusal = "unsupported"
// BlockRefusedProtected: infra, the host's own address or one that is
// never a packet's source.
BlockRefusedProtected BlockRefusal = "protected"
// BlockRefusedAllowed: an operator allow or a verified crawler.
BlockRefusedAllowed BlockRefusal = "allowlisted"
// BlockRefusedDuplicate: an earlier member names the same address or
// action.
BlockRefusedDuplicate BlockRefusal = "duplicate"
// BlockRefusedExisting: a verified row and its kernel element already
// block the address for at least the member's lifetime: an untimed
// element for a permanent row, or one with the time a timed row has
// left when that row ends no earlier than the member.
BlockRefusedExisting BlockRefusal = "existing_effect"
// BlockRefusedPending: a row or a kernel element names the address
// without proving an enforced block that covers the member: the row is
// unverified, has no element or one with another lifetime, an element
// has no row, or the proven block ends before the member. Recovery or
// that block's end has to settle it first.
BlockRefusedPending BlockRefusal = "pending_recovery"
// BlockRefusedFull: no temporary deny slot is left.
BlockRefusedFull BlockRefusal = "set_full"
)
// BlockEffect is what a batch did to one member.
type BlockEffect string
const (
// BlockApplied: the kernel holds the member's element as encoded.
BlockApplied BlockEffect = "applied"
// BlockFailed: proven not applied; nothing of it is in the kernel.
BlockFailed BlockEffect = "failed"
// BlockUnknown: the element may or may not be in the kernel. Its row
// stays, marked unverified.
BlockUnknown BlockEffect = "unknown"
)
// BlockVictimEffect is what a batch did to a member's victim.
type BlockVictimEffect string
const (
// VictimRemoved: deleted in the member's transaction, and the readback
// proved it gone.
VictimRemoved BlockVictimEffect = "removed"
// VictimSpared: never deleted, because the member did not run, the
// kernel rejected the transaction or the victim no longer qualified.
VictimSpared BlockVictimEffect = "spared"
// VictimUnproven: the deletion was sent, but neither an uncertain
// reply nor the readback proves it.
VictimUnproven BlockVictimEffect = "unproven"
)
// BlockResult is one member's outcome: its effect, the check that refused
// it before the kernel batch, or the error behind a failed or unknown one.
// Existing names the block behind an existing-effect refusal. Victim is
// what the batch did to the member's victim; empty when it named none.
type BlockResult struct {
Effect BlockEffect
Refusal BlockRefusal
Existing ExistingBlock
Err error
Victim BlockVictimEffect
}
// ExistingBlock names the block that already holds an address, so a member
// refused as an existing effect can link to it: the admission
// action that wrote its row, or "eff_" and a digest of an operator or
// earlier row's identity, and when it ends (zero: permanent). The same row
// gets the same ID on every check and after a restart.
type ExistingBlock struct {
ID string
ExpiresAt time.Time
}
// BlockAction names the row an admission attempt wrote: its action ID and
// address.
type BlockAction struct {
Action, IP string
}
// BlockCheck is the batch check's answer for one member: its refusal and,
// for an existing effect, the block that holds the address.
type BlockCheck struct {
Refusal BlockRefusal
Existing ExistingBlock
}
// ErrBlockBatchRejected marks a member that failed because the kernel
// rejected the whole transaction; a failure before the send does not carry
// it. A failed state restore carries ErrBlockBatchState instead. The owner
// retries rejected members one at a time.
var ErrBlockBatchRejected = errors.New("kernel rejected the block batch")
// ErrBlockBatchState requires the owner to hold automatic execution while
// the engine's state safety changes lack a durable acknowledgement.
var ErrBlockBatchState = errors.New("firewall batch state is unproven")
// blockReadbackSlack covers the kernel's timeout granularity and the
// clock reads around the batch.
const blockReadbackSlack = time.Second
// CoversExpiry reports whether an effect ending at held lasts as long as
// one asked to end at wanted: a permanent effect, zero held, or one ending
// no earlier than wanted within the readback slack. A dry-run forecast
// judges its own blocks by it, so they cover exactly as real ones do.
func CoversExpiry(held, wanted time.Time) bool {
return held.IsZero() || !held.Before(wanted.Add(-blockReadbackSlack))
}
// MaxShadowBlocks bounds the forecast addresses one admission snapshot
// counts: a dry-run forecast models at most this many active blocks.
const MaxShadowBlocks = 20000
// MaxHeldBlockActions bounds the blocks one held check names: as many as
// a forecast's shadow list. Larger ledgers rotate through bounded batches.
const MaxHeldBlockActions = MaxShadowBlocks
// BlockAnswer is an admission snapshot's answer for one member, naming
// the member it answers.
type BlockAnswer struct {
Action, IP string
BlockCheck
}
// BlockAdmission is a read-only answer for one admission turn, from one
// snapshot of the state file and the kernel: each member's local checks
// and the temporary occupancy. No member takes a temporary slot and no
// repeated member is refused: the caller decides which members are ready
// and checks them with ReadyCapacity.
type BlockAdmission struct {
// Answers holds one answer per member, in member order.
Answers []BlockAnswer
// Temporary is how many timed elements the blocked sets hold, both
// address families together; Limit is the configured temporary limit,
// zero for none.
Temporary, Limit int
// The timed occupants by recorded ownership: Admission
// counts the elements of rows admission actions wrote, Imported those
// of rows legacy automatic blocks marked, and Unclaimed every other
// timed element plus two elements per timed subnet row, which
// admission never evicts.
Admission, Imported, Unclaimed int
// Drain bounds how long the last unclaimed or imported occupant has left
// after collection and member checks; zero when no time remains,
// -time.Second when a kernel expiry is missing.
Drain time.Duration
// ShadowHeld names the forecast addresses the sets already hold as a
// timed or an untimed element.
ShadowHeld map[string]bool
// Unproven says startup could not prove admission outcomes: no batch
// removes a block for a member until a restart proves them.
Unproven bool
// Imports lists the imports a member may evict, soonest expiry first,
// at most MaxBlockImports; none while startup could not prove
// admission outcomes.
Imports []BlockImport
}
// MaxBlockImports bounds the imports one snapshot lists: the soonest to
// expire, which admission evicts first among imports.
const MaxBlockImports = 256
// BlockImport is a known automatic block admission may evict: a
// timed, verified row a legacy automatic block stamped, alone at its
// canonical address, whose element the snapshot proves live. ID is its
// effect name, which stays the same while the row does.
type BlockImport struct {
ID string
IP string
ExpiresAt time.Time
}
// Full reports whether the temporary set has no slot left for any member.
func (a BlockAdmission) Full() bool { return a.Limit > 0 && a.Temporary >= a.Limit }
// ReadyCapacity answers the members a turn is ready to run, in order, as a
// batch check answers its capacity: a member repeating an earlier one's
// address or action is a duplicate, and once the occupancy reaches the
// limit the rest find the set full. Only a member that passes takes a slot.
func (a BlockAdmission) ReadyCapacity(ready []BlockMember) []BlockRefusal {
out := make([]BlockRefusal, len(ready))
seenIP, seenAction := map[string]bool{}, map[string]bool{}
temp := a.Temporary
for i, m := range ready {
switch {
case seenIP[m.IP] || seenAction[m.Action]:
out[i] = BlockRefusedDuplicate
case a.Limit > 0 && temp >= a.Limit:
out[i] = BlockRefusedFull
default:
temp++
seenIP[m.IP], seenAction[m.Action] = true, true
}
}
return out
}
//go:build linux
package firewall
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"maps"
"net"
"slices"
"strings"
"syscall"
"time"
"unicode"
"unicode/utf8"
"github.com/google/nftables"
"github.com/google/nftables/expr"
"github.com/mdlayher/netlink"
"golang.org/x/sys/unix"
)
func validBlockAction(id string) bool {
if id == "" || len(id) > 64 {
return false
}
for _, c := range id {
if c != '_' && c != '-' && (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') {
return false
}
}
return true
}
func validBlockReason(reason string) bool {
if len(reason) > MaxBlockReasonBytes || !utf8.ValidString(reason) {
return false
}
for _, r := range reason {
if unicode.IsControl(r) {
return false
}
}
return true
}
// blockSnapshot is the one view a batch is checked against: the state
// file and the kernel's blocked sets. blocked and allowed key the checked
// rows by canonical address once, and live the kernel's elements, so each
// member check is a lookup.
type blockSnapshot struct {
at time.Time
prior FirewallState
state FirewallState
blocked map[string]BlockedEntry
allowed map[string]bool
live map[string]blockObservation
elapsed time.Time
temp int
// rules are the input rule handles a batch checks again before writing.
rules []uint64
}
// proves reports whether row is an enforced block the kernel shows: a
// verified row whose element is untimed when the row is permanent, or has
// the time the row has left, within the readback slack, when it is timed.
type blockObservation struct {
el nftables.SetElement
before, after time.Time
// temporary classifies the element as the temporary count does.
temporary bool
}
func (snap blockSnapshot) proves(row BlockedEntry, observed blockObservation, present bool) bool {
el := observed.el
switch {
case row.Unverified || !present:
return false
case row.ExpiresAt.IsZero():
return el.Timeout == 0 && el.Expires == 0
}
left := row.ExpiresAt.Sub(snap.at)
if left <= 0 || el.Timeout <= 0 || el.Expires <= 0 {
return false
}
// Netlink reports time left, not when in the dump it was sampled.
// Bound that instant and age both bounds through the last set read.
earliest := el.Expires - snap.elapsed.Sub(observed.before)
latest := el.Expires - snap.elapsed.Sub(observed.after)
return earliest > 0 && earliest-left <= blockReadbackSlack && latest-left >= -blockReadbackSlack
}
// HoldBlockBatches holds admission's member checks, batches, probes and
// retirements while an apply-confirmed window is open: its rollback
// restores the firewall from a snapshot and would erase the rows they
// write. The read-only admission snapshot of a forecast still
// answers. It waits for a call in flight.
func (e *Engine) HoldBlockBatches(held bool) {
e.mu.Lock()
defer e.mu.Unlock()
e.blockBatchesHeld = held
}
// blockBatchSnapshotLocked is the snapshot of a call that leads to or makes
// a firewall change, refused while batches are held or while the rules
// that drop blocked addresses are missing.
func (e *Engine) blockBatchSnapshotLocked() (blockSnapshot, error) {
if e.blockBatchesHeld {
return blockSnapshot{}, ErrConfirmPending
}
// Preserve readiness refusals before opening the rule dump socket.
if e.blockBatchStateErr != nil {
return blockSnapshot{}, e.blockBatchStateErr
}
if e.lifecycle != nil {
return blockSnapshot{}, errDurableBlockRows
}
if e.setBlocked == nil || (e.liveBlockedDump == nil && e.conn == nil) {
return blockSnapshot{}, errors.New("blocked sets unavailable")
}
rules, err := e.blockRuleHandlesLocked()
if err != nil {
return blockSnapshot{}, err
}
snap, err := e.blockSnapshotLocked()
if err != nil {
return blockSnapshot{}, err
}
snap.rules = rules
return snap, nil
}
// blockRuleHandlesLocked returns the handles of the input chain's rules in
// order, up to and including the last drop of a filtered blocked set.
// Set readback proves membership, not enforcement: a ruleset reloaded, a
// drop removed or a rule inserted ahead of the drops outside CSM changes
// these handles. CSM's own rule changes run under e.mu.
func (e *Engine) blockRuleHandlesLocked() ([]uint64, error) {
if e.setBlocked == nil {
return nil, fmt.Errorf("%w: blocked sets unavailable", ErrBlockRules)
}
rules, err := e.blockRulesLocked()
if err != nil {
return nil, fmt.Errorf("%w: listing the input rules: %v", ErrBlockRules, err)
}
want := map[string]byte{e.setBlocked.Name: 2}
if e.setBlocked6 != nil && e.cfg.IPv6 {
want[e.setBlocked6.Name] = 10
}
last := -1
handles := make([]uint64, 0, len(rules))
for i, rule := range rules {
handles = append(handles, rule.Handle)
for name, family := range want {
if set, ok := blockDropSet(rule, family); ok && set == name {
delete(want, name)
last = i
}
}
}
if len(want) > 0 {
return nil, fmt.Errorf("%w: no rule drops %s", ErrBlockRules, strings.Join(slices.Sorted(maps.Keys(want)), ", "))
}
return handles[:last+1], nil
}
// blockDropSet recognizes the source-address drop emitted by Apply.
// Lookup transaction IDs are not stable dump fields. The register aliases
// below name the same bytes; every other semantic field must match.
func blockDropSet(rule *nftables.Rule, family byte) (string, bool) {
if len(rule.Exprs) != 5 {
return "", false
}
meta, ok := rule.Exprs[0].(*expr.Meta)
if !ok || meta.SourceRegister || meta.Key != expr.MetaKeyNFPROTO || !blockSourceRegister(meta.Register) {
return "", false
}
cmp, ok := rule.Exprs[1].(*expr.Cmp)
if !ok || cmp.Op != expr.CmpOpEq || !blockSourceRegister(cmp.Register) || !slices.Equal(cmp.Data, []byte{family}) {
return "", false
}
offset, length := uint32(12), uint32(4)
if family == 10 {
offset, length = 8, 16
}
payload, ok := rule.Exprs[2].(*expr.Payload)
if !ok || payload.OperationType != expr.PayloadLoad || !blockSourceRegister(payload.DestRegister) ||
payload.Base != expr.PayloadBaseNetworkHeader || payload.Offset != offset || payload.Len != length {
return "", false
}
lookup, ok := rule.Exprs[3].(*expr.Lookup)
if !ok || lookup.Invert || lookup.IsDestRegSet || !blockSourceRegister(lookup.SourceRegister) || lookup.SetName == "" {
return "", false
}
verdict, ok := rule.Exprs[4].(*expr.Verdict)
return lookup.SetName, ok && verdict.Kind == expr.VerdictDrop
}
func blockSourceRegister(register uint32) bool {
return register == unix.NFT_REG_1 || register == unix.NFT_REG32_00
}
func (e *Engine) blockSnapshotLocked() (blockSnapshot, error) {
return e.readBlockSnapshotLocked(e.loadStateFileRawLocked)
}
func (e *Engine) readBlockSnapshotLocked(loadState func() (FirewallState, bool)) (blockSnapshot, error) {
if e.blockBatchStateErr != nil {
return blockSnapshot{}, e.blockBatchStateErr
}
if e.lifecycle != nil {
return blockSnapshot{}, errDurableBlockRows
}
if e.setBlocked == nil || (e.liveBlockedDump == nil && e.conn == nil) {
return blockSnapshot{}, errors.New("blocked sets unavailable")
}
prior, ok := loadState()
if !ok {
return blockSnapshot{}, fmt.Errorf("%w: firewall state unavailable", ErrBlockBatchState)
}
e.overlayUnverifiedBlockActionsLocked(&prior)
var conn *nftables.Conn
if e.liveBlockedDump == nil {
var err error
conn, err = newLifecycleConn(e)
if err != nil {
return blockSnapshot{}, err
}
}
now := e.expiryNow()
at := e.leaseAtLocked(now)
// Expired rows still own kernel cleanup, notably untimed subnet elements.
// Only the check view is pruned; intent and restore retain the full image.
state := copyFirewallState(prior)
// Recovery needs every action's row, including rows sharing an address.
// Only the admission check view may merge those rows.
normalizeFirewallStateIPs(&state)
state.Blocked, _ = pruneBlocked(state.Blocked, at)
state.BlockedNet, _ = pruneBlockedNet(state.BlockedNet, now)
state.Allowed, _ = pruneAllowed(state.Allowed, now)
snap := blockSnapshot{at: at, prior: prior, state: state, blocked: map[string]BlockedEntry{}, allowed: map[string]bool{}, live: map[string]blockObservation{}}
for _, row := range state.Blocked {
snap.blocked[stateIPKey(row.IP)] = row
}
for _, row := range state.Allowed {
snap.allowed[stateIPKey(row.IP)] = true
}
for _, row := range state.PortAllowed {
snap.allowed[stateIPKey(row.IP)] = true
}
tempByIP := blockedStateTempByIP(snap.state)
for _, set := range []*nftables.Set{e.setBlocked, e.setBlocked6} {
if set == nil || (set == e.setBlocked6 && !e.cfg.IPv6) {
continue
}
e.leaseNowLocked()
before := e.lease.at
elements, err := e.blockSnapshotElementsLocked(conn, set)
e.leaseNowLocked()
after := e.lease.at
if err != nil {
return blockSnapshot{}, fmt.Errorf("listing blocked set %s: %w", set.Name, err)
}
var valid []nftables.SetElement
for _, el := range elements {
if ip, ok := blockSetIP(set, el.Key); ok {
snap.live[ip] = blockObservation{el: el, before: before, after: after, temporary: liveBlockTemporary(el, ip, tempByIP)}
valid = append(valid, el)
}
}
_, temp := countLiveBlockElements(valid, tempByIP)
snap.temp += temp
}
snap.at = e.leaseNowLocked()
snap.elapsed = e.lease.at
snap.state.Blocked, _ = pruneBlocked(snap.state.Blocked, snap.at)
clear(snap.blocked)
for _, row := range snap.state.Blocked {
snap.blocked[stateIPKey(row.IP)] = row
}
return snap, nil
}
// Snapshot dumps use transient deadline sockets; the legacy connection
// may be lasting and has no bound on a reply.
func (e *Engine) blockSnapshotElementsLocked(conn *nftables.Conn, set *nftables.Set) ([]nftables.SetElement, error) {
if e.liveBlockedDump != nil {
return e.liveBlockedDump(set)
}
return conn.GetSetElements(set)
}
// verifiedRanges asks the verified-crawler callback outside the engine
// lock, as the automatic block path does.
func (e *Engine) verifiedRanges(members []BlockMember) []bool {
out := make([]bool, len(members))
for i, m := range members {
if net.ParseIP(m.IP) != nil {
out[i] = e.autoBlockVerifiedRange(m.IP)
}
}
return out
}
// checkBlockMembersLocked answers every member against one snapshot. An
// invalid member is refused alone: it takes no slot and names nothing a
// later member could collide with. With evicts the temporary set's
// capacity applies: a member whose victim qualifies (evicts[i]) takes that
// victim's slot, any other a free one, or it is refused set_full. The
// complete result must fit the current limit even for a replacement. Without
// it capacity is left to the apply, since the ledger claims victims only
// when it reserves.
func (e *Engine) checkBlockMembersLocked(members []BlockMember, verified []bool, snap blockSnapshot, now time.Time, evicts []bool) []BlockRefusal {
out := make([]BlockRefusal, len(members))
seenIP, seenAction := map[string]bool{}, map[string]bool{}
temp := snap.temp
for i, m := range members {
out[i] = e.blockRefusalLocked(m, verified[i], snap, now, seenIP, seenAction)
if out[i] != "" {
continue
}
switch {
case evicts == nil || (evicts[i] && (e.cfg.DenyTempIPLimit <= 0 || temp <= e.cfg.DenyTempIPLimit)):
case e.cfg.DenyTempIPLimit > 0 && temp >= e.cfg.DenyTempIPLimit:
out[i] = BlockRefusedFull
continue
default:
temp++
}
seenIP[m.IP], seenAction[m.Action] = true, true
}
e.ageExistingLocked(members, out, snap)
return out
}
// victimsLocked reports, per member, whether its victim is still the
// block the ledger claimed, so the batch may remove it: a
// well-formed action at an address no member targets, named by no
// earlier member, whose row there carries that action and expiry and
// whose timed element snap proves live at that canonical address, or an
// import whose row there still has that name and expiry. Retained rows
// there must agree; normalization alone cannot prove ownership.
func (e *Engine) victimsLocked(members []BlockMember, snap blockSnapshot) []bool {
out := make([]bool, len(members))
if e.blockActionsUnproven != nil {
return out
}
imports := snap.importRows()
conflicted := map[string]bool{}
rows, _ := pruneBlocked(snap.prior.Blocked, snap.at)
for _, row := range rows {
key := stateIPKey(row.IP)
selected := snap.blocked[key]
if row.Unverified || row.ActionID != selected.ActionID || !row.ExpiresAt.Equal(selected.ExpiresAt) {
conflicted[key] = true
}
}
targets, taken := map[string]bool{}, map[string]bool{}
for _, m := range members {
targets[stateIPKey(m.IP)] = true
}
for i, m := range members {
v := m.Victim
if v.ExpiresAt.IsZero() {
continue
}
key := stateIPKey(v.IP)
var row BlockedEntry
var held bool
switch {
case admissionAction(v.Action):
row, held = snap.blocked[key]
held = held && row.ActionID == v.Action
case importName(v.Action):
row, held = imports[key]
held = held && blockEffectID(row) == v.Action
}
observed, live := snap.live[v.IP]
if targets[key] || taken[key] || conflicted[key] || !held || !row.ExpiresAt.Equal(v.ExpiresAt) || !snap.proves(row, observed, live) {
continue
}
out[i], taken[key] = true, true
}
return out
}
// importRows maps each canonical address to its candidate import row: a
// row a legacy automatic block stamped, with no action and no other
// retained row at the kernel address, including a zoned alias retirement
// would treat as the same address. Callers still prove its timed element,
// which also requires the row verified.
func (snap blockSnapshot) importRows() map[string]BlockedEntry {
rows, _ := pruneBlocked(snap.prior.Blocked, snap.at)
count := make(map[string]int, len(rows))
for _, row := range rows {
count[retirementIPKey(row.IP)]++
}
out := make(map[string]BlockedEntry)
for _, row := range rows {
key := stateIPKey(row.IP)
if count[retirementIPKey(row.IP)] == 1 && row.Automatic && row.ActionID == "" {
out[key] = row
}
}
return out
}
// imports lists the import rows whose temporary element snap proves live,
// soonest expiry first, then name, at most MaxBlockImports.
func (snap blockSnapshot) imports(at, elapsed time.Time) []BlockImport {
rows := snap.importRows()
// Age proof without reclassifying occupants: an alias expiring during
// member checks must not promote an unclaimed occupant into the list.
snap.at, snap.elapsed = at, elapsed
var out []BlockImport
for key, row := range rows {
observed, live := snap.live[key]
if !live || !observed.temporary || !snap.proves(row, observed, live) {
continue
}
out = append(out, BlockImport{ID: blockEffectID(row), IP: key, ExpiresAt: row.ExpiresAt})
}
slices.SortFunc(out, func(a, b BlockImport) int {
if c := a.ExpiresAt.Compare(b.ExpiresAt); c != 0 {
return c
}
return strings.Compare(a.ID, b.ID)
})
return out[:min(len(out), MaxBlockImports)]
}
// ageExistingLocked proves each existing effect again at a fresh lease
// reading: protection checks can wait for interface lookups after the set
// dumps, so every proof ages through all members before the caller can
// end a pick.
func (e *Engine) ageExistingLocked(members []BlockMember, out []BlockRefusal, snap blockSnapshot) {
snap.at = e.leaseNowLocked()
snap.elapsed = e.lease.at
for i, refusal := range out {
if refusal != BlockRefusedExisting {
continue
}
row := snap.blocked[stateIPKey(members[i].IP)]
observed, present := snap.live[members[i].IP]
if !snap.proves(row, observed, present) {
out[i] = BlockRefusedPending
}
}
}
func (e *Engine) blockRefusalLocked(m BlockMember, verified bool, snap blockSnapshot, now time.Time, seenIP, seenAction map[string]bool) BlockRefusal {
canon, err := canonicalFirewallIP(m.IP)
if err != nil || canon != m.IP || !validBlockAction(m.Action) || !validBlockReason(m.Reason) || m.ExpiresAt.IsZero() {
return BlockRefusedInvalid
}
if left := m.ExpiresAt.Sub(now); !now.Add(left).Equal(m.ExpiresAt) || m.ExpiresAt.UTC().Year() < 0 || m.ExpiresAt.UTC().Year() > 9999 {
return BlockRefusedInvalid
}
if _, ok := blockTimeout(m.ExpiresAt, now); !ok {
return BlockRefusedExpired
}
parsed := net.ParseIP(m.IP)
if parsed.To4() == nil && (!e.cfg.IPv6 || e.setBlocked6 == nil) {
return BlockRefusedUnsupported
}
if e.blockProtectionLocked(parsed, m.IP) != nil {
return BlockRefusedProtected
}
key := stateIPKey(m.IP)
if verified || snap.allowed[key] {
return BlockRefusedAllowed
}
if seenIP[m.IP] || seenAction[m.Action] {
return BlockRefusedDuplicate
}
el, live := snap.live[m.IP]
row, held := snap.blocked[key]
switch {
case held && snap.proves(row, el, live) && covers(row, m):
return BlockRefusedExisting
case held || live:
return BlockRefusedPending
}
return ""
}
// covers reports whether row lasts as long as member m (only
// the exact effect is already enforced). A shorter block holds the address
// until it ends.
func covers(row BlockedEntry, m BlockMember) bool { return CoversExpiry(row.ExpiresAt, m.ExpiresAt) }
// CheckBlockBatch answers, before anything is reserved, which members of a
// batch can run: each is checked on its own against one snapshot of the
// state file and the kernel, and only the members that fail are refused.
// An error refuses the whole call: an empty or oversized batch, durable
// state, or no snapshot.
func (e *Engine) CheckBlockBatch(members []BlockMember) ([]BlockRefusal, error) {
checks, err := e.CheckBlockBatchEffects(members)
if err != nil {
return nil, err
}
out := make([]BlockRefusal, len(checks))
for i, c := range checks {
out[i] = c.Refusal
}
return out, nil
}
// CheckBlockBatchEffects is CheckBlockBatch with the block that holds the
// address of each member refused as an existing effect.
func (e *Engine) CheckBlockBatchEffects(members []BlockMember) ([]BlockCheck, error) {
if len(members) == 0 || len(members) > MaxBlockBatchMembers {
return nil, fmt.Errorf("a block batch has 1 to %d members", MaxBlockBatchMembers)
}
verified := e.verifiedRanges(members)
e.mu.Lock()
defer e.mu.Unlock()
snap, err := e.blockBatchSnapshotLocked()
if err != nil {
return nil, err
}
out := make([]BlockCheck, len(members))
for i, refusal := range e.checkBlockMembersLocked(members, verified, snap, snap.at, nil) {
out[i] = BlockCheck{Refusal: refusal, Existing: snap.existing(members[i], refusal)}
}
return out, nil
}
// BlockAdmissionSnapshot answers an admission turn without changing
// anything: each member's local checks against one snapshot of the state
// file and the kernel, as a batch check makes them, and the temporary
// occupancy from the same snapshot. No member takes a slot and no repeat
// is a duplicate (BlockAdmission.ReadyCapacity checks the members the
// caller is ready to run). shadow names the canonical addresses a
// forecast holds, at most MaxShadowBlocks; the answer counts those the
// sets already hold. An error refuses the whole call: too many members or
// addresses, a malformed or repeated address, durable state, or no
// snapshot. Remote checks run outside the engine lock.
func (e *Engine) BlockAdmissionSnapshot(members []BlockMember, shadow []string) (BlockAdmission, error) {
if len(members) > MaxBlockBatchMembers || len(shadow) > MaxShadowBlocks {
return BlockAdmission{}, fmt.Errorf("an admission snapshot has at most %d members and %d forecast addresses", MaxBlockBatchMembers, MaxShadowBlocks)
}
seen := make(map[string]bool, len(shadow))
for _, ip := range shadow {
if canon, err := canonicalFirewallIP(ip); err != nil || canon != ip || seen[ip] {
return BlockAdmission{}, errors.New("a forecast address is malformed or repeated")
}
seen[ip] = true
}
verified := e.verifiedRanges(members)
e.mu.Lock()
defer e.mu.Unlock()
snap, err := e.blockSnapshotLocked()
if err != nil {
return BlockAdmission{}, err
}
refusals := make([]BlockRefusal, len(members))
for i, m := range members {
// Without seen members nothing is a duplicate.
refusals[i] = e.blockRefusalLocked(m, verified[i], snap, snap.at, nil, nil)
}
e.ageExistingLocked(members, refusals, snap)
snap.elapsed = e.lease.at
out := BlockAdmission{Answers: make([]BlockAnswer, len(members)), Temporary: snap.temp, Limit: e.cfg.DenyTempIPLimit, ShadowHeld: map[string]bool{}}
for i, m := range members {
out.Answers[i] = BlockAnswer{Action: m.Action, IP: m.IP, BlockCheck: BlockCheck{Refusal: refusals[i], Existing: snap.existing(m, refusals[i])}}
}
snap.occupants(&out, e.expiryNow())
out.Unproven = e.blockActionsUnproven != nil
if !out.Unproven {
// Import proof must age the row and the kernel sample to the same
// instant, including time spent checking members and any wall step.
at := e.leaseNowLocked()
out.Imports = snap.imports(at, e.lease.at)
}
for _, ip := range shadow {
if _, present := snap.live[ip]; present {
out.ShadowHeld[ip] = true
}
}
return out, nil
}
// occupants classes the snapshot's timed elements and timed subnet rows
// into out, subnet expiry read at now.
func (snap blockSnapshot) occupants(out *BlockAdmission, now time.Time) {
owners := occupantOwners(snap.prior.Blocked, snap.at)
for ip, observed := range snap.live {
if !observed.temporary {
continue
}
row, held := owners[stateIPKey(ip)]
switch {
case held && row.ActionID != "":
out.Admission++
continue
case held && row.Automatic && !row.Unverified:
out.Imported++
default:
out.Unclaimed++
}
if observed.el.Expires <= 0 {
out.Drain = -time.Second
} else if out.Drain >= 0 {
// The sample can be as late as the end of its dump. Age that
// upper bound through later dumps and member checks without
// letting a wall-clock step change a kernel timeout.
out.Drain = max(out.Drain, observed.el.Expires-snap.elapsed.Sub(observed.after))
}
}
for _, row := range snap.state.BlockedNet {
if !row.ExpiresAt.After(now) {
continue
}
// A prefix encodes as at most two interval elements.
out.Unclaimed += 2
if out.Drain >= 0 {
out.Drain = max(out.Drain, row.ExpiresAt.Sub(now))
}
}
}
// occupantOwners requires retained raw rows at an address to agree on ownership.
// Address normalization may select a longer-lived row for protection checks;
// that selection must not turn an unknown occupant into an automatic import.
func occupantOwners(rows []BlockedEntry, at time.Time) map[string]BlockedEntry {
rows, _ = pruneBlocked(rows, at)
owners := make(map[string]BlockedEntry, len(rows))
for _, row := range rows {
ip := stateIPKey(row.IP)
if old, found := owners[ip]; found && (!old.ExpiresAt.After(at) || old.ActionID != row.ActionID || (row.ActionID == "" && (old.Automatic != row.Automatic || old.Unverified != row.Unverified))) {
row = BlockedEntry{}
}
if !row.ExpiresAt.After(at) {
row = BlockedEntry{}
}
owners[ip] = row
}
return owners
}
// existing names the block that holds m's address when m is refused as an
// existing effect.
func (snap blockSnapshot) existing(m BlockMember, refusal BlockRefusal) ExistingBlock {
if refusal != BlockRefusedExisting {
return ExistingBlock{}
}
row := snap.blocked[stateIPKey(m.IP)]
return ExistingBlock{ID: blockEffectID(row), ExpiresAt: row.ExpiresAt}
}
// blockEffectID names the block row holds: the admission action that wrote
// it, or for an operator or earlier row a digest of its identity, which
// stays the same while the row does.
func blockEffectID(row BlockedEntry) string {
if admissionAction(row.ActionID) {
return row.ActionID
}
identity := strings.Join([]string{stateIPKey(row.IP), row.Source, row.BlockedAt.UTC().Format(time.RFC3339Nano), row.ExpiresAt.UTC().Format(time.RFC3339Nano)}, "\x00")
sum := sha256.Sum256([]byte(identity))
return "eff_" + hex.EncodeToString(sum[:16])
}
// admissionAction reports whether id has the form of an admission action
// ID: "act_" and 32 lowercase hex digits.
func admissionAction(id string) bool { return effectName(id, "act_") }
// importName reports whether id has the form of an import's name, the
// effect name of a stamped legacy row: "eff_" and 32 lowercase hex digits.
func importName(id string) bool { return effectName(id, "eff_") }
// effectName reports whether id is prefix and 32 lowercase hex digits.
func effectName(id, prefix string) bool {
rest, ok := strings.CutPrefix(id, prefix)
if !ok || len(rest) != 32 {
return false
}
for _, c := range rest {
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
return false
}
}
return true
}
// kernelRejected follows error provenance: a definitive reply proves the
// atomic transaction failed even if later acknowledgements were lost.
// A local receive error only says the reply could not be read.
func kernelRejected(err error) bool {
if timed, ok := err.(interface{ Timeout() bool }); ok && timed.Timeout() {
return false
}
switch cause := err.(type) {
case *netlink.OpError:
if cause.Op == "receive" {
errno, ok := cause.Err.(syscall.Errno)
return ok && kernelRejected(errno)
}
return cause.Op == "send-messages" && kernelRejected(cause.Err)
case syscall.Errno:
switch cause {
case syscall.EINTR, syscall.EAGAIN, syscall.ENOBUFS, syscall.ETIMEDOUT:
return false
}
return cause != 0
case interface{ Unwrap() []error }:
for _, child := range cause.Unwrap() {
if kernelRejected(child) {
return true
}
}
case interface{ Unwrap() error }:
return kernelRejected(cause.Unwrap())
}
return false
}
// A lasting connection dials before returning, so dial and socket-option
// errors are known before any send. The deadline covers send and readback.
func newBlockBatchConn(e *Engine, extra ...nftables.ConnOption) (*nftables.Conn, error) {
if e.conn == nil {
return nil, errors.New("firewall batch transport unavailable")
}
opts := []nftables.ConnOption{
nftables.AsLasting(),
nftables.WithNetNSFd(e.conn.NetNS),
nftables.WithTestDial(e.conn.TestDial),
nftables.WithSockOptions(applyNFTSocketBuffer),
}
if e.conn.TestDial == nil {
opts = append(opts, nftables.WithSockOptions(applyLifecycleSocketDeadline))
}
opts = append(opts, extra...)
return nftables.New(opts...)
}
// readbackMatches bounds time left between the encoded timeout and that
// timeout minus elapsed send/readback time and slack. The timeout itself
// must match independently of elapsed time, allowing kernel tick truncation.
// An older element with a longer timeout can have the same time left.
func readbackMatches(el nftables.SetElement, encoded, window time.Duration) bool {
if el.Expires <= 0 || el.Expires > encoded || encoded-el.Expires > window {
return false
}
return el.Timeout > 0 && el.Timeout <= encoded && encoded-el.Timeout <= blockReadbackSlack && el.Expires <= el.Timeout
}
func (e *Engine) readbackLocked(conn *nftables.Conn, set *nftables.Set) ([]nftables.SetElement, error) {
if e.liveBlockedDump != nil {
return e.liveBlockedDump(set)
}
return conn.GetSetElements(set)
}
// ApplyBlockBatch runs the members of one batch: it checks them again
// against the current state and kernel, writes every remaining row (with
// its action ID) in one state write, and in one netlink transaction on a
// fresh connection with a deadline deletes each qualifying victim and adds
// the members' elements; then it reads the sets back. A member is applied
// only when the readback shows its element as encoded, and its victim is
// removed only when the readback of the victim's set shows it gone; a
// member whose removal is unproven is unknown. A victim's row stays until
// the owner retires it after the ledger records the outcome, so a restart
// before that restores it. A victim that could expire before the kernel
// runs the transaction is spared and its member alone fails set_full. A
// kernel rejection, including the ENOENT of a victim removed before the
// send, fails every member and restores the state; the owner retries with
// new charged attempts. Any other uncertainty leaves the members unknown, their rows
// marked unverified. An error means nothing was written or sent.
func (e *Engine) ApplyBlockBatch(members []BlockMember) ([]BlockResult, error) {
if len(members) == 0 || len(members) > MaxBlockBatchMembers {
return nil, fmt.Errorf("a block batch has 1 to %d members", MaxBlockBatchMembers)
}
verified := e.verifiedRanges(members)
e.mu.Lock()
defer e.mu.Unlock()
snap, err := e.blockBatchSnapshotLocked()
if err != nil {
return nil, err
}
now := snap.at
results := make([]BlockResult, len(members))
defer func() {
// Other members can spend a proved block's remaining lifetime.
snap.at = e.leaseNowLocked()
snap.elapsed = e.lease.at
for i, result := range results {
if result.Refusal != BlockRefusedExisting {
continue
}
row := snap.blocked[stateIPKey(members[i].IP)]
observed, present := snap.live[members[i].IP]
if !snap.proves(row, observed, present) {
results[i].Refusal, results[i].Existing = BlockRefusedPending, ExistingBlock{}
}
}
}()
next := copyFirewallState(snap.prior)
encoded := make(map[int]time.Duration)
evicts := e.victimsLocked(members, snap)
var run []int
var elems4, elems6 []nftables.SetElement
for i, refusal := range e.checkBlockMembersLocked(members, verified, snap, now, evicts) {
if refusal != "" {
results[i] = BlockResult{Effect: BlockFailed, Refusal: refusal, Existing: snap.existing(members[i], refusal), Victim: spared(members[i])}
continue
}
m := members[i]
upsertBlockedEntryInState(&next, BlockedEntry{IP: m.IP, Reason: m.Reason, Source: SourceAutoResponse, BlockedAt: now, ExpiresAt: m.ExpiresAt.UTC(), ActionID: m.Action})
run = append(run, i)
}
if len(run) == 0 {
return results, nil
}
// The last check before the write: the rules that enforce the blocks
// are the ones the snapshot read.
rules, ruleErr := e.blockRuleHandlesLocked()
if ruleErr != nil {
return nil, ruleErr
}
if !slices.Equal(rules, snap.rules) {
return nil, fmt.Errorf("%w: the input rules changed during the batch", ErrBlockRules)
}
if persistErr := e.persistFirewallIntent(snap.prior, next); persistErr != nil {
e.suppressFailedBlockActionsLocked(blockActions(members, run))
e.blockBatchStateErr = errors.Join(ErrBlockBatchState, persistErr)
for _, i := range run {
results[i] = BlockResult{Effect: BlockFailed, Err: e.blockBatchStateErr, Victim: spared(members[i])}
}
return results, nil
}
conn, err := newBlockBatchConn(e)
var dels4, dels6 []nftables.SetElement
if err == nil {
defer func() { _ = conn.CloseLasting() }()
// State persistence can consume a member's remaining lifetime.
// Refresh at one instant immediately before queueing the send.
sendAt := e.leaseNowLocked()
var ready []int
for _, i := range run {
m := members[i]
timeout, ok := blockTimeout(m.ExpiresAt, sendAt)
if !ok {
results[i] = BlockResult{Effect: BlockFailed, Refusal: BlockRefusedExpired, Victim: spared(m)}
e.suppressFailedBlockActionsLocked([]string{m.Action})
removeBlockedIPFromState(&next, m.IP)
continue
}
// A victim that could expire before the kernel runs the
// transaction would reject it whole with ENOENT and fail every
// member: its member fails alone and the victim is spared.
if evicts[i] && !m.Victim.ExpiresAt.After(sendAt.Add(blockBatchMinLifetime)) {
results[i] = BlockResult{Effect: BlockFailed, Refusal: BlockRefusedFull, Victim: VictimSpared}
e.suppressFailedBlockActionsLocked([]string{m.Action})
removeBlockedIPFromState(&next, m.IP)
continue
}
encoded[i] = timeout
parsed := net.ParseIP(m.IP)
if ip4 := parsed.To4(); ip4 != nil {
elems4 = append(elems4, nftables.SetElement{Key: ip4, Timeout: timeout})
} else {
elems6 = append(elems6, nftables.SetElement{Key: parsed.To16(), Timeout: timeout})
}
// The victim leaves in the same transaction as its member's add.
if evicts[i] {
victim := net.ParseIP(m.Victim.IP)
if ip4 := victim.To4(); ip4 != nil {
dels4 = append(dels4, nftables.SetElement{Key: ip4})
} else {
dels6 = append(dels6, nftables.SetElement{Key: victim.To16()})
}
}
ready = append(ready, i)
}
run = ready
if len(run) == 0 {
return results, nil
}
}
if err == nil && len(dels4) > 0 {
err = conn.SetDeleteElements(e.setBlocked, dels4)
}
if err == nil && len(dels6) > 0 {
err = conn.SetDeleteElements(e.setBlocked6, dels6)
}
if err == nil && len(elems4) > 0 {
err = conn.SetAddElements(e.setBlocked, elems4)
}
if err == nil && len(elems6) > 0 {
err = conn.SetAddElements(e.setBlocked6, elems6)
}
sent := err == nil
start := time.Now()
if sent {
err = conn.Flush()
}
if err != nil && (!sent || kernelRejected(err)) {
if restoreErr := e.saveState(&snap.prior); restoreErr != nil {
e.suppressFailedBlockActionsLocked(blockActions(members, run))
e.blockBatchStateErr = errors.Join(ErrBlockBatchState, restoreErr)
err = errors.Join(err, fmt.Errorf("state restore failed: %w", e.blockBatchStateErr))
} else if sent {
err = errors.Join(ErrBlockBatchRejected, err)
}
for _, i := range run {
results[i] = BlockResult{Effect: BlockFailed, Err: err, Victim: spared(members[i])}
}
return results, nil
}
var found map[string]nftables.SetElement
var read blockFamilies
if err == nil {
found, read, err = e.readbackBatchLocked(conn, len(elems4)+len(dels4) > 0, len(elems6)+len(dels6) > 0)
}
window := time.Since(start) + blockReadbackSlack
var unknown []int
for _, i := range run {
m := members[i]
victim := VictimUnproven
switch {
case m.Victim == (BlockVictim{}):
victim = ""
case !evicts[i]:
victim = VictimSpared
case read.has(m.Victim.IP):
if _, live := found[m.Victim.IP]; !live {
victim = VictimRemoved
e.legacyFileAuditLocked("evict_temp", m.Victim.IP, evictTempReason, SourceSystem, 0)
}
}
// An unproven removal leaves the member unknown even when its add is
// proven: the victim may still hold the slot.
if el, ok := found[m.IP]; ok && readbackMatches(el, encoded[i], window) && victim != VictimUnproven {
results[i] = BlockResult{Effect: BlockApplied, Victim: victim}
e.legacyFileAuditLocked("block", m.IP, m.Reason, SourceAutoResponse, encoded[i])
continue
}
results[i] = BlockResult{Effect: BlockUnknown, Err: err, Victim: victim}
unknown = append(unknown, i)
}
if len(unknown) > 0 {
if markErr := e.flagUnknownRowsLocked(members, unknown, next); markErr != nil {
for _, i := range unknown {
results[i].Err = errors.Join(results[i].Err, markErr)
}
}
}
return results, nil
}
// spared is the victim effect of a member that never ran: none without a
// victim.
func spared(m BlockMember) BlockVictimEffect {
if m.Victim == (BlockVictim{}) {
return ""
}
return VictimSpared
}
// blockFamilies says which address families a readback read.
type blockFamilies struct {
v4, v6 bool
}
// has reports whether ip's family is set.
func (f blockFamilies) has(ip string) bool {
if net.ParseIP(ip).To4() != nil {
return f.v4
}
return f.v6
}
// readbackBatchLocked reads the sets of the families asked for and merges
// their elements; read says which reads succeeded, and the error joins the
// failures.
func (e *Engine) readbackBatchLocked(conn *nftables.Conn, v4, v6 bool) (map[string]nftables.SetElement, blockFamilies, error) {
found := map[string]nftables.SetElement{}
var read blockFamilies
var readErr error
for _, r := range []struct {
set *nftables.Set
used bool
ok *bool
}{{e.setBlocked, v4, &read.v4}, {e.setBlocked6, v6, &read.v6}} {
if !r.used {
continue
}
elements, err := e.readbackLocked(conn, r.set)
if err != nil {
readErr = errors.Join(readErr, fmt.Errorf("reading back blocked set %s: %w", r.set.Name, err))
continue
}
*r.ok = true
for _, el := range elements {
if ip, ok := blockSetIP(r.set, el.Key); ok {
found[ip] = el
}
}
}
return found, read, readErr
}
// flagUnknownRowsLocked persists unknown marks; failed persistence keeps
// live suppression and exposes the shared state-integrity hold.
func (e *Engine) flagUnknownRowsLocked(members []BlockMember, unknown []int, next FirewallState) error {
flag := make(map[string]bool, len(unknown))
for _, i := range unknown {
flag[members[i].Action] = true
}
for i, row := range next.Blocked {
if flag[row.ActionID] {
next.Blocked[i].Unverified = true
}
}
e.rememberUnverifiedBlockActionsLocked(next)
if err := e.saveState(&next); err != nil {
e.blockBatchStateErr = errors.Join(ErrBlockBatchState, err)
return e.blockBatchStateErr
}
return nil
}
func blockActions(members []BlockMember, indices []int) []string {
out := make([]string, 0, len(indices))
for _, i := range indices {
out = append(out, members[i].Action)
}
return out
}
// A mapped key in an IPv6 set cannot enforce an IPv4 packet.
func blockSetIP(set *nftables.Set, key []byte) (string, bool) {
if len(key) != int(set.KeyType.Bytes) || (len(key) == net.IPv6len && net.IP(key).To4() != nil) {
return "", false
}
return setElementIPString(key)
}
// ProbeBlockState proves the current snapshot without sending an action.
func (e *Engine) ProbeBlockState() error {
e.mu.Lock()
defer e.mu.Unlock()
_, err := e.blockBatchSnapshotLocked()
return err
}
//go:build linux
package firewall
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
)
// HeldBlockActions reports, for each verified admission block, whether
// what it applied may remain in the kernel: retirement's answer reversed,
// with no state write. An element still at the address that
// no other action's verified row proves answers held, even after a
// forward wall step makes the block's own row look expired, and so does
// an address whose family the snapshot cannot read; only a removal
// answers false. An error answers nothing.
func (e *Engine) HeldBlockActions(actions []BlockAction) ([]bool, error) {
if len(actions) > MaxHeldBlockActions {
return nil, fmt.Errorf("a held check names at most %d blocks", MaxHeldBlockActions)
}
e.mu.Lock()
defer e.mu.Unlock()
canonical, unread, err := e.canonicalBlockActions(actions, "checking")
if err != nil {
return nil, err
}
if e.blockBatchesHeld {
return nil, ErrConfirmPending
}
snap, err := e.readBlockSnapshotLocked(e.readHeldStateLocked)
if err != nil {
return nil, err
}
held := clearBlockActions(snap, canonical, unread)
for i := range held {
held[i] = !held[i]
}
return held, nil
}
// The ordinary raw loader may rename unreadable state for recovery. A
// held check must leave it untouched and keep refusing subsequent reads.
func (e *Engine) readHeldStateLocked() (FirewallState, bool) {
if e.stateAsideErr != nil {
return FirewallState{}, false
}
// #nosec G304 -- filepath.Join under operator-configured statePath.
data, err := os.ReadFile(filepath.Join(e.statePath, "state.json"))
if err != nil {
return FirewallState{}, os.IsNotExist(err)
}
var state FirewallState
err = json.Unmarshal(data, &state)
return state, err == nil
}
//go:build linux
package firewall
import (
"errors"
"fmt"
"os"
"path/filepath"
"github.com/google/nftables"
)
// RetimeBlocks replaces the elements of the blocked address sets with the
// ones the saved rows give, in one transaction: each timed block gets the
// time its row has left. A ruleset restored from a dump gives a timed
// block the time it had left when the dump was taken, so the block would
// outlast its row by the dump's age and no row would prove its element.
// As at Apply, a row held back as unverified gets no element and an
// element no row names goes. Nothing is sent when the rows could not be
// read, or under the durable lifecycle, which changes the kernel only
// through its own actions. The transaction has its own connection, so one
// that cannot be finished is dropped whole.
func (e *Engine) RetimeBlocks() error {
e.mu.Lock()
defer e.mu.Unlock()
if e.lifecycle != nil {
return nil
}
if e.setBlocked == nil {
return errors.New("blocked sets unavailable")
}
conn, err := newLifecycleConn(e)
if err != nil {
return err
}
if e.stateAsideErr != nil {
return e.stateAsideErr
}
// Missing state after preservation is not proof of an empty policy.
if e.stateSetAside != "" {
if _, err := os.Stat(filepath.Join(e.statePath, "state.json")); err != nil {
return fmt.Errorf("checking preserved firewall state: %w", err)
}
}
state, readable := e.loadStateFileRawLocked()
if !readable {
return errors.Join(errors.New("firewall state unreadable"), e.stateAsideErr)
}
e.overlayUnverifiedBlockActionsLocked(&state)
normalizeFirewallStateIPs(&state)
initial := e.initialBlockStateFromRowsLocked(state, e.expiryNow())
conn.FlushSet(e.setBlocked)
if err := addElementsChunked(conn, e.setBlocked, initial.blocked4); err != nil {
return err
}
blocked6 := e.setBlocked6
if e.cfg.IPv6 {
// A restart inside a window with IPv6 disabled creates no IPv6
// reference, even when the rollback restores IPv6 filtering.
if blocked6 == nil {
blocked6 = &nftables.Set{
Table: e.table, Name: "blocked_ips6",
KeyType: nftables.TypeIP6Addr, HasTimeout: true,
}
}
conn.FlushSet(blocked6)
if err := addElementsChunked(conn, blocked6, initial.blocked6); err != nil {
return err
}
}
if err := conn.Flush(); err != nil {
return fmt.Errorf("retiming blocked sets: %w", err)
}
e.setBlocked6 = blocked6
return nil
}
//go:build linux
package firewall
import (
"errors"
"fmt"
"net/netip"
"slices"
)
// RemoveBlockActionRows removes, in one state write, the rows the named
// attempts wrote and reports how many went. Startup calls it with the
// ledger's settled attempts (failed, or retired after their expiry) once
// every unproven one is marked: nothing they applied can remain, so their
// rows only hold an address. An error reports no completed removal; the
// next start reads the saved state and tries again.
func (e *Engine) RemoveBlockActionRows(actions []string) (int, error) {
e.mu.Lock()
defer e.mu.Unlock()
if e.lifecycle != nil {
return 0, errDurableBlockRows
}
named := make(map[string]bool, len(actions))
for _, id := range actions {
if id != "" {
named[id] = true
}
}
if len(named) == 0 && !e.blockRemovalPending {
return 0, nil
}
state, ok := e.loadStateFileRawLocked()
if !ok {
return 0, errors.New("firewall state unavailable")
}
kept := state.Blocked[:0]
removed := 0
for _, row := range state.Blocked {
if named[row.ActionID] {
removed++
continue
}
kept = append(kept, row)
}
if removed == 0 && !e.blockRemovalPending {
return 0, nil
}
if err := e.checkpointBlockRemovalLocked(); err != nil {
return 0, err
}
state.Blocked = kept
e.overlayUnverifiedBlockActionsLocked(&state)
if err := e.saveBlockRemovalLocked(state); err != nil {
return 0, err
}
e.forgetBlockActionsLocked(named)
return removed, nil
}
// RetireBlockActions reports, for each expired unknown attempt or evicted
// import, whether nothing it may have applied can remain in the kernel:
// its address has no element, or the element belongs to a
// verified row of another action. An import stays held while its own row
// proves the live element, even if an alias does too. Each clear row
// (same action ID or import name and address) goes in one state write, under the same
// engine lock as the snapshot, so no batch writes the address in between.
// An element still there otherwise keeps the attempt held, as after a
// forward wall step that made its expiry look past, and so does an address
// whose family the snapshot cannot read. An error answers nothing.
func (e *Engine) RetireBlockActions(actions []BlockAction) ([]bool, error) {
if len(actions) > MaxBlockBatchMembers {
return nil, fmt.Errorf("a retirement checks at most %d attempts", MaxBlockBatchMembers)
}
e.mu.Lock()
defer e.mu.Unlock()
if e.lifecycle != nil {
return nil, errDurableBlockRows
}
canonical, unread, err := e.canonicalBlockActions(actions, "retiring")
if err != nil {
return nil, err
}
if e.blockBatchesHeld {
return nil, ErrConfirmPending
}
if err = e.checkpointBlockRemovalLocked(); err != nil {
return nil, err
}
snap, err := e.blockSnapshotLocked()
if err != nil {
return nil, err
}
out := clearBlockActions(snap, canonical, unread)
gone := make(map[BlockAction]bool, len(actions))
for i, m := range canonical {
if out[i] {
gone[m] = true
}
}
next := copyFirewallState(snap.prior)
kept := next.Blocked[:0]
for _, row := range next.Blocked {
if gone[BlockAction{Action: retireID(row), IP: retirementIPKey(row.IP)}] {
continue
}
kept = append(kept, row)
}
if len(kept) == len(next.Blocked) && !e.blockRemovalPending {
return out, nil
}
next.Blocked = kept
if err = e.saveBlockRemovalLocked(next); err != nil {
return nil, err
}
return out, nil
}
// canonicalBlockActions is actions at their canonical addresses, and which
// of them the snapshot cannot read: it skips a missing or disabled IPv6
// set, so it cannot prove such an address absent. A malformed address
// refuses the whole call, named with verb.
func (e *Engine) canonicalBlockActions(actions []BlockAction, verb string) ([]BlockAction, []bool, error) {
canonical := make([]BlockAction, len(actions))
unread := make([]bool, len(actions))
for i, m := range actions {
addr, err := netip.ParseAddr(m.IP)
if err != nil {
return nil, nil, fmt.Errorf("%s %s: %w", verb, m.Action, err)
}
addr = addr.WithZone("").Unmap()
unread[i] = addr.Is6() && (!e.cfg.IPv6 || e.setBlocked6 == nil)
canonical[i] = BlockAction{Action: m.Action, IP: addr.String()}
}
return canonical, unread, nil
}
// clearBlockActions reports, for each canonical action, whether nothing it
// may have applied can remain in the kernel: its address has no element,
// or the element belongs to a verified row of another action. An unread
// address is never clear. Aliased imports can prove the same element under
// different names; neither is clear while its own proof holds.
func clearBlockActions(snap blockSnapshot, actions []BlockAction, unread []bool) []bool {
owners := make(map[string][]string)
for _, row := range snap.prior.Blocked {
ip := retirementIPKey(row.IP)
observed, live := snap.live[ip]
if snap.proves(row, observed, live) {
owners[ip] = append(owners[ip], retireID(row))
}
}
out := make([]bool, len(actions))
for i, m := range actions {
if unread[i] || (importName(m.Action) && slices.Contains(owners[m.IP], m.Action)) {
continue
}
_, live := snap.live[m.IP]
out[i] = !live
for _, owner := range owners[m.IP] {
if owner != m.Action {
out[i] = true
break
}
}
}
return out
}
// retireID is the name retirement knows row by: the admission action that
// wrote it or, for a row a legacy automatic block stamped, its effect name
// (an evicted import). Every other row keeps its empty action, which no
// retirement names.
func retireID(row BlockedEntry) string {
if row.ActionID == "" && row.Automatic {
return blockEffectID(row)
}
return row.ActionID
}
// A failed rename durability check can leave no rows for the retry to
// remove. Keep requiring a full state write until one confirms durability.
func (e *Engine) saveBlockRemovalLocked(state FirewallState) error {
err := e.writeStateLocked(state)
e.blockRemovalPending = err != nil
return err
}
// forgetBlockActionsLocked drops the in-memory marks of attempts whose rows
// are gone.
func (e *Engine) forgetBlockActionsLocked(actions map[string]bool) {
for id := range actions {
delete(e.unverifiedBlockActions, id)
delete(e.failedBlockActions, id)
}
}
// Recovery compares addresses in the engine's namespace; a legacy row's
// textual zone does not change the address stored in the blocked set.
func retirementIPKey(raw string) string {
addr, err := netip.ParseAddr(raw)
if err != nil {
return raw
}
return addr.WithZone("").Unmap().String()
}
// Preserve the lease proof before dropping rows or authorizing retirement.
// A fresh startup engine must resume the old checkpoint before replacing it.
func (e *Engine) checkpointBlockRemovalLocked() error {
if err := e.resumeLeaseLocked(); err != nil {
// Cleanup retries must not treat a failed read as a completed resume.
e.leaseResumed = false
return fmt.Errorf("resuming removal lease: %w", err)
}
if err := e.saveLeaseLocked(); err != nil {
return fmt.Errorf("saving removal lease: %w", err)
}
return nil
}
//go:build linux
package firewall
import "errors"
// BlockedEntries returns active, proven rows from the engine's state owner.
// Consumers of block provenance must not infer it from kernel membership or
// a separate scan tracker, which can lag behind a committed firewall action.
func (e *Engine) BlockedEntries() ([]BlockedEntry, error) {
e.mu.Lock()
defer e.mu.Unlock()
state := e.loadStateFile()
if err := errors.Join(e.stateReadErr, e.stateAsideErr); err != nil {
return nil, err
}
rows := state.Blocked[:0]
for _, row := range state.Blocked {
if !row.Unverified {
rows = append(rows, row)
}
}
return rows, nil
}
// errDurableBlockRows refuses batch row changes while a durable lifecycle
// owns the state: its batch manifest is not built yet.
var errDurableBlockRows = errors.New("firewall state is owned by the durable lifecycle")
func unverifiedRows(entries []BlockedEntry) int {
n := 0
for _, entry := range entries {
if entry.Unverified {
n++
}
}
return n
}
// MarkBlockActionsUnverified flags the rows written by the named attempts
// as unproven, in one state write, and reports how many changed. Startup
// calls it with the ledger's unresolved attempts before Apply, so their
// rows are neither restored nor reported as blocks. While a failed state
// write holds batches it writes the safe state even when nothing is named,
// and a successful write releases the hold.
func (e *Engine) MarkBlockActionsUnverified(actions []string) (int, error) {
e.mu.Lock()
defer e.mu.Unlock()
if e.lifecycle != nil {
return 0, errDurableBlockRows
}
named := make(map[string]bool, len(actions))
for _, id := range actions {
if id != "" {
named[id] = true
}
}
held := e.blockBatchStateErr != nil
if len(named) == 0 && !held {
return 0, nil
}
state, ok := e.loadStateFileRawLocked()
if !ok {
e.blockBatchStateErr = errors.Join(ErrBlockBatchState, errors.New("firewall state unavailable"))
return 0, e.blockBatchStateErr
}
marked := 0
pending := false
for i, entry := range state.Blocked {
if named[entry.ActionID] && !entry.Unverified {
state.Blocked[i].Unverified = true
marked++
}
if named[entry.ActionID] && e.unverifiedBlockActions[entry.ActionID] {
pending = true
}
}
if marked == 0 && !pending && !held {
return 0, nil
}
e.rememberUnverifiedBlockActionsLocked(state)
// Recovery owns action identities, not the normalized address view.
// Merging aliases here could erase a different action before retirement.
e.overlayUnverifiedBlockActionsLocked(&state)
if err := e.writeStateLocked(state); err != nil {
e.blockBatchStateErr = errors.Join(ErrBlockBatchState, err)
return marked, e.blockBatchStateErr
}
return marked, nil
}
// SetBlockActionsUnproven records, in memory only, why startup could not
// read the ledger. The rows its attempts wrote are restored as before, but
// none is ever chosen to make room for a new block: nothing proves which
// of them the kernel enforces. The next start reads the ledger again.
func (e *Engine) SetBlockActionsUnproven(cause error) {
e.mu.Lock()
defer e.mu.Unlock()
e.blockActionsUnproven = cause
}
// BlockActionsUnproven is the cause SetBlockActionsUnproven recorded, or nil.
func (e *Engine) BlockActionsUnproven() error {
e.mu.Lock()
defer e.mu.Unlock()
return e.blockActionsUnproven
}
// filteredRowsLocked is st without the blocked rows whose address family
// has no blocked set, as after IPv6 filtering was turned off.
func (e *Engine) filteredRowsLocked(st FirewallState) FirewallState {
kept := make([]BlockedEntry, 0, len(st.Blocked))
for _, row := range st.Blocked {
if set, _, err := e.resolveIPSet(row.IP, e.setBlocked, e.setBlocked6); err == nil && set != nil {
kept = append(kept, row)
}
}
st.Blocked = kept
return st
}
// heldRows is st without the blocked rows whose address is not in live,
// the addresses the limit count found in the kernel: replacing a row with
// no element frees no slot. A nil live, from a count that read no kernel
// image, keeps every row.
func heldRows(st FirewallState, live map[string]bool) FirewallState {
if live == nil {
return st
}
kept := make([]BlockedEntry, 0, len(st.Blocked))
for _, row := range st.Blocked {
if live[stateIPKey(row.IP)] {
kept = append(kept, row)
}
}
st.Blocked = kept
return st
}
// evictionVictimLocked picks the temporary block a new one replaces at the
// cap. It leaves out the rows admission actions wrote: no legacy block,
// automatic or operator, removes a block admission owns. An automatic block
// replaces only rows that recorded an automatic origin, so automatic work
// can never lift an operator's temporary block.
// Under the durable lifecycle, where no admission action writes rows,
// only an automatic block or unproven rows leave its action rows out. Among
// the rows left, a mail spray row goes first, so a spray wave replaces its
// own blocks before any other.
func (e *Engine) evictionVictimLocked(st FirewallState, excludeIP string, automatic bool) (string, bool) {
addresses := make(map[string]int)
if e.lifecycle != nil {
for _, row := range st.Blocked {
addresses[stateIPKey(row.IP)]++
}
}
if e.lifecycle == nil || e.blockActionsUnproven != nil || automatic {
var kept []BlockedEntry
for _, row := range st.Blocked {
if row.ActionID == "" && (!automatic || row.Automatic) {
kept = append(kept, row)
}
}
st.Blocked = kept
}
if e.lifecycle != nil {
// The saved Before must identify exactly one row at this address.
at := e.leaseNowLocked()
var recordable []BlockedEntry
for _, row := range st.Blocked {
if e.unverifiedBlockActions[row.ActionID] || e.failedBlockActions[row.ActionID] {
continue
}
if row.IP == stateIPKey(row.IP) && addresses[stateIPKey(row.IP)] == 1 && row.ExpiresAt.After(at) {
recordable = append(recordable, row)
}
}
st.Blocked = recordable
}
spray := FirewallState{}
for _, row := range st.Blocked {
if row.LoginSpray {
spray.Blocked = append(spray.Blocked, row)
}
}
if victim, ok := soonestExpiringTempIP(spray, excludeIP); ok {
return victim, true
}
return soonestExpiringTempIP(st, excludeIP)
}
// The in-memory mark survives a failed save and cache replacement until a
// successful full-state write records it. Only then can the hold clear.
func (e *Engine) rememberUnverifiedBlockActionsLocked(state FirewallState) {
if e.unverifiedBlockActions == nil {
e.unverifiedBlockActions = make(map[string]bool)
}
for _, row := range state.Blocked {
if row.Unverified && row.ActionID != "" {
e.unverifiedBlockActions[row.ActionID] = true
}
}
if e.stateCache != nil {
e.overlayUnverifiedBlockActionsLocked(e.stateCache)
e.rebuildIndexLocked()
}
}
func (e *Engine) suppressFailedBlockActionsLocked(actions []string) {
if e.failedBlockActions == nil {
e.failedBlockActions = make(map[string]bool)
}
for _, id := range actions {
if id != "" {
e.failedBlockActions[id] = true
}
}
if e.stateCache != nil {
e.overlayUnverifiedBlockActionsLocked(e.stateCache)
e.rebuildIndexLocked()
}
}
func (e *Engine) overlayUnverifiedBlockActionsLocked(state *FirewallState) {
blocked := state.Blocked[:0]
for _, row := range state.Blocked {
if e.failedBlockActions[row.ActionID] {
continue
}
if e.unverifiedBlockActions[row.ActionID] {
row.Unverified = true
}
blocked = append(blocked, row)
}
state.Blocked = blocked
}
//go:build linux
package firewall
import (
"encoding/binary"
"errors"
"fmt"
"github.com/google/nftables"
"github.com/google/nftables/expr"
"github.com/mdlayher/netlink"
"github.com/mdlayher/netlink/nltest"
"golang.org/x/sys/unix"
)
func (e *Engine) blockRulesLocked() ([]*nftables.Rule, error) {
if e.blockRuleDump != nil {
return e.blockRuleDump()
}
if e.conn == nil {
return nil, errors.New("firewall lifecycle transport unavailable")
}
var socket *netlink.Conn
if e.conn.TestDial != nil {
socket = nltest.Dial(e.conn.TestDial)
} else {
var err error
socket, err = netlink.Dial(unix.NETLINK_NETFILTER, &netlink.Config{NetNS: e.conn.NetNS})
if err != nil {
return nil, err
}
if err = applyLifecycleSocketDeadline(socket); err != nil {
return nil, err
}
_ = applyNFTSocketBuffer(socket)
}
defer func() { _ = socket.Close() }()
table := e.setBlocked.Table
attrs, err := netlink.MarshalAttributes([]netlink.Attribute{
{Type: unix.NFTA_RULE_TABLE, Data: []byte(table.Name + "\x00")},
{Type: unix.NFTA_RULE_CHAIN, Data: []byte("input\x00")},
})
if err != nil {
return nil, err
}
messages, err := socket.Execute(netlink.Message{
Header: netlink.Header{Type: unix.NFNL_SUBSYS_NFTABLES<<8 | unix.NFT_MSG_GETRULE, Flags: netlink.Request | netlink.Dump},
Data: append([]byte{byte(table.Family), 0, 0, 0}, attrs...),
})
if err != nil {
return nil, err
}
rules := make([]*nftables.Rule, 0, len(messages))
for _, message := range messages {
rule, err := decodeBlockRule(message)
if err != nil {
return nil, err
}
rules = append(rules, rule)
}
return rules, nil
}
// GetRules silently discards unknown expression types. Enforcement proof
// must count the raw expressions, including any predicates or register
// writes its decoder cannot represent. Other rules still supply handles.
func decodeBlockRule(message netlink.Message) (*nftables.Rule, error) {
if message.Header.Type != unix.NFNL_SUBSYS_NFTABLES<<8|unix.NFT_MSG_NEWRULE || len(message.Data) < 4 {
return nil, errors.New("invalid input rule message")
}
ad, err := netlink.NewAttributeDecoder(message.Data[4:])
if err != nil {
return nil, err
}
ad.ByteOrder = binary.BigEndian
rule := new(nftables.Rule)
for ad.Next() {
switch ad.Type() {
case unix.NFTA_RULE_HANDLE:
rule.Handle = ad.Uint64()
case unix.NFTA_RULE_EXPRESSIONS:
rule.Exprs, err = decodeBlockExpressions(message.Data[0], ad.Bytes())
if err != nil {
return nil, err
}
}
}
if err := ad.Err(); err != nil {
return nil, err
}
if rule.Handle == 0 {
return nil, errors.New("input rule has no handle")
}
return rule, nil
}
func decodeBlockExpressions(family byte, data []byte) ([]expr.Any, error) {
attrs, err := netlink.UnmarshalAttributes(data)
if err != nil {
return nil, err
}
names := []string{"meta", "cmp", "payload", "lookup", "immediate"}
if len(attrs) != len(names) {
return nil, nil
}
expressions := []expr.Any{&expr.Meta{}, &expr.Cmp{}, &expr.Payload{}, &expr.Lookup{}, &expr.Immediate{}}
for i, attr := range attrs {
ad, err := netlink.NewAttributeDecoder(attr.Data)
if err != nil {
return nil, err
}
var name string
var body []byte
for ad.Next() {
switch ad.Type() {
case unix.NFTA_EXPR_NAME:
name = ad.String()
case unix.NFTA_EXPR_DATA:
body = ad.Bytes()
}
}
if err := ad.Err(); err != nil {
return nil, err
}
if name != names[i] || len(body) == 0 {
return nil, nil
}
if err := expr.Unmarshal(family, body, expressions[i]); err != nil {
return nil, fmt.Errorf("decoding block %s: %w", name, err)
}
if immediate, ok := expressions[i].(*expr.Immediate); ok && immediate.Register == unix.NFT_REG_VERDICT && len(immediate.Data) == 0 {
verdict := new(expr.Verdict)
if err := expr.Unmarshal(family, body, verdict); err != nil {
return nil, err
}
expressions[i] = verdict
}
}
return expressions, nil
}
//go:build linux
package firewall
import (
"encoding/json"
"errors"
"os"
"path/filepath"
)
// BlockActionIDsForStartup includes every saved row that can hold an address:
// LoadState hides unverified rows and their address aliases, but removing a
// settled hold can expose another action at that address. An expired row
// no earlier start marked is never restored and is left to expiry, so it
// cannot hold back another block at its address.
func (e *Engine) BlockActionIDsForStartup() ([]string, error) {
e.mu.Lock()
defer e.mu.Unlock()
// #nosec G304 -- filepath.Join under operator-configured statePath.
data, err := os.ReadFile(filepath.Join(e.statePath, "state.json"))
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
if err != nil {
return nil, err
}
var state FirewallState
if err = json.Unmarshal(data, &state); err != nil {
return nil, err
}
var ids []string
seen := make(map[string]bool)
// Apply must retain this expiry decision if the wall clock moves back
// before installation, or it could restore an action the ledger never saw.
now := e.leaseNowLocked()
for _, row := range state.Blocked {
expired := !row.ExpiresAt.IsZero() && !row.ExpiresAt.After(now)
if row.ActionID != "" && !seen[row.ActionID] && (row.Unverified || !expired) {
seen[row.ActionID] = true
ids = append(ids, row.ActionID)
}
}
return ids, nil
}
package firewall
import "time"
// blockTimeout is the kernel timeout that keeps a timed element no longer
// than its absolute expiry: the time left at now, truncated to the
// kernel's millisecond precision. It reports false when less than a
// millisecond is left: nftables would send a zero timeout, which the
// kernel takes as no timeout at all, and a negative one wraps.
func blockTimeout(expiresAt, now time.Time) (time.Duration, bool) {
left := expiresAt.Sub(now)
if !now.Add(left).Equal(expiresAt) {
return 0, false
}
left = left.Truncate(time.Millisecond)
if left < time.Millisecond {
return 0, false
}
return left, true
}
//go:build linux
package firewall
import "time"
// BlockIPForUndo captures both sides of a forced operator block under the
// mutation lock. A later CLI decision must never become part of this undo.
func (e *Engine) BlockIPForUndo(ip, reason string, ttl time.Duration) (before, after *BlockedEntry, resultErr error) {
defer func() {
if e.shouldLegacyOutcome(resultErr) {
recordBlockOutcome(ip, reason, ttl, BlockOutcomeLive, resultErr, true, "")
}
}()
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return nil, nil, err
}
ip = canonical
e.mu.Lock()
defer e.mu.Unlock()
state := e.loadStateFile()
if e.stateReadErr != nil {
return nil, nil, e.stateReadErr
}
if entry, exists := blockedStateEntry(state, ip); exists {
before = &entry
}
_, err = e.blockIPRequestLocked(ip, reason, ttl, false, false,
ActionRequest{Operation: "block", Target: ip, Reason: reason, TTL: ttl}, nil, true, time.Time{})
if err != nil {
return nil, nil, err
}
state = e.loadStateFile()
entry, _ := blockedStateEntry(state, ip)
return before, &entry, nil
}
// UnblockIPForUndo captures the removed block in the same critical section.
func (e *Engine) UnblockIPForUndo(ip string) (before *BlockedEntry, resultErr error) {
defer func() { e.legacyFirewallFailure("unblock", ip, "", "", 0, resultErr) }()
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return nil, err
}
ip = canonical
defer func() { e.released(ip, ReleaseOperator, resultErr) }()
e.mu.Lock()
defer e.mu.Unlock()
state := e.loadStateFile()
if e.stateReadErr != nil {
return nil, e.stateReadErr
}
if entry, exists := blockedStateEntry(state, ip); exists {
before = &entry
}
if err := e.unblockIPLocked(ip); err != nil {
return nil, err
}
return before, nil
}
// RestoreBlockIfUnchanged applies a saved lifetime only while the action's
// firewall snapshot still matches. The comparison and mutation share e.mu.
func (e *Engine) RestoreBlockIfUnchanged(ip string, expected, prior *BlockedEntry) (resultErr error) {
var action, reason string
var duration time.Duration
defer func() {
switch action {
case "block":
if e.shouldLegacyOutcome(resultErr) {
recordBlockOutcome(ip, reason, duration, BlockOutcomeLive, resultErr, true, "")
}
case "unblock":
e.legacyFirewallFailure("unblock", ip, "", "", 0, resultErr)
e.released(ip, ReleaseOperator, resultErr)
}
}()
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return err
}
ip = canonical
e.mu.Lock()
defer e.mu.Unlock()
state := e.loadStateFile()
if e.stateReadErr != nil {
return e.stateReadErr
}
current, exists := blockedStateEntry(state, ip)
if exists != (expected != nil) || (exists && !SameBlockedEntry(current, *expected)) {
return ErrBlockChanged
}
if !exists {
live, err := e.isBlockedLiveLocked(ip)
if err != nil {
return err
}
if live {
return ErrBlockChanged
}
}
if prior != nil {
ttl := time.Duration(0)
if !prior.ExpiresAt.IsZero() {
var ok bool
ttl, ok = blockTimeout(prior.ExpiresAt, e.leaseNowLocked())
if !ok {
prior = nil
}
}
if prior != nil {
action, reason, duration = "block", prior.Reason, ttl
// Spray undo keeps its eviction preference; other legacy undo
// continues to create an operator-owned row.
spray := prior.LoginSpray && prior.Automatic
_, err := e.blockIPRequestLocked(ip, prior.Reason, ttl, false, false,
ActionRequest{Operation: "block", Target: ip, Reason: prior.Reason, TTL: ttl,
Automatic: spray, LoginSpray: spray}, nil, false, prior.ExpiresAt)
return err
}
}
if !exists {
return nil
}
action = "unblock"
return e.unblockIPLocked(ip)
}
package firewall
import "net"
// LiveBlockedSnapshot is a point-in-time membership view of the kernel's
// blocked IP sets, taken by dumping each configured family set once.
//
// HasV4 / HasV6 record which families the snapshot actually covers. A set can
// be absent -- firewall.ipv6 disabled leaves the v6 set nil -- and an
// uncovered family must never read as "not blocked", or a reconcile pass
// would prune every tracked block of that family on its first cycle.
type LiveBlockedSnapshot struct {
V4 map[string]struct{}
V6 map[string]struct{}
HasV4 bool
HasV6 bool
}
// Contains reports whether ip is in the live blocked set. known is false when
// the snapshot does not cover the IP's address family, in which case the
// caller must keep its cached answer rather than treat ip as unblocked.
//
// Family selection mirrors the per-IP path: an IPv4-mapped IPv6 address
// resolves against the v4 set, because that is the set the block path keys it
// into. A malformed IP is reported as definitively absent, matching
// IsBlockedLive.
func (s LiveBlockedSnapshot) Contains(ip string) (blocked, known bool) {
parsed := net.ParseIP(ip)
if parsed == nil {
return false, true
}
if v4 := parsed.To4(); v4 != nil {
if !s.HasV4 {
return false, false
}
_, ok := s.V4[v4.String()]
return ok, true
}
if !s.HasV6 {
return false, false
}
_, ok := s.V6[parsed.To16().String()]
return ok, true
}
//go:build linux
package firewall
import (
"fmt"
"net"
"os"
"github.com/google/nftables"
)
// UpdateCloudflareSet flushes and repopulates the Cloudflare nftables sets.
func (e *Engine) UpdateCloudflareSet(ipv4, ipv6 []string) error {
e.mu.Lock()
defer e.mu.Unlock()
if err := e.lifecycleReadyLocked(); err != nil {
return err
}
// Require both sets -- FlushSet/SetAddElements panic on a nil set, and
// neither exists before createSets has run.
if e.setCFWhitelist == nil || e.setCFWhitelist6 == nil {
return fmt.Errorf("cf_whitelist sets not initialized")
}
// Flush existing entries
e.conn.FlushSet(e.setCFWhitelist)
e.conn.FlushSet(e.setCFWhitelist6)
// Populate IPv4 CIDRs
var elems4 []nftables.SetElement
for _, cidr := range ipv4 {
_, network, err := net.ParseCIDR(cidr)
if err != nil {
continue
}
start := network.IP.To4()
end := lastIPInRange(network)
if start == nil || end == nil {
continue
}
elems4 = appendIntervalSetElements(elems4, start, end)
}
if len(elems4) > 0 {
if err := e.conn.SetAddElements(e.setCFWhitelist, normalizeIntervalElements(elems4)); err != nil {
return fmt.Errorf("adding CF IPv4 elements: %w", err)
}
}
// Populate IPv6 CIDRs
var elems6 []nftables.SetElement
for _, cidr := range ipv6 {
_, network, err := net.ParseCIDR(cidr)
if err != nil {
continue
}
if network.IP.To4() != nil {
continue
}
start := network.IP.To16()
end := lastIPInRange(network)
if start == nil || end == nil {
continue
}
elems6 = appendIntervalSetElements(elems6, start, end)
}
if len(elems6) > 0 {
if err := e.conn.SetAddElements(e.setCFWhitelist6, normalizeIntervalElements(elems6)); err != nil {
return fmt.Errorf("adding CF IPv6 elements: %w", err)
}
}
if err := e.conn.Flush(); err != nil {
return fmt.Errorf("flushing CF whitelist: %w", err)
}
fmt.Fprintf(os.Stderr, "firewall: cloudflare whitelist updated: %d IPv4, %d IPv6 CIDRs\n",
len(ipv4), len(ipv6))
return nil
}
// CloudflareIPs returns the currently configured Cloudflare CIDRs from the cached state.
func (e *Engine) CloudflareIPs() (ipv4, ipv6 []string) {
return LoadCFState(e.statePath)
}
// CloudflareCovers reports whether ip is inside a cached Cloudflare range,
// meaning a block of it leaves TCP 80/443 reachable (the CF accept precedes
// the blocked drop in the input chain).
func (e *Engine) CloudflareCovers(ip string) bool {
v4, v6 := LoadCFState(e.statePath)
return CloudflareRangesCover(v4, v6, ip)
}
package firewall
import (
"bufio"
"context"
"errors"
"fmt"
"net"
"net/http"
"os"
"strings"
"time"
"github.com/pidginhost/csm/internal/atomicio"
)
const (
cfIPv4URL = "https://www.cloudflare.com/ips-v4"
cfIPv6URL = "https://www.cloudflare.com/ips-v6"
// CloudflareCoverageWarning is shared by every operator-facing block
// response so API, CLI, and finding wording cannot drift apart.
CloudflareCoverageWarning = "IP is inside a Cloudflare allow range; ports 80/443 from it are still accepted"
)
// FetchCloudflareIPs downloads the current Cloudflare IP ranges, honouring
// ctx. A caller shutting down cancels it rather than waiting out the HTTP
// timeout: on a host that cannot reach cloudflare.com -- an egress-restricted
// server among them -- that wait is the full timeout, twice.
func FetchCloudflareIPs(ctx context.Context) (ipv4, ipv6 []string, err error) {
client := &http.Client{Timeout: 30 * time.Second}
return fetchCloudflareIPs(ctx, client)
}
func fetchCloudflareIPs(ctx context.Context, client *http.Client) (ipv4, ipv6 []string, err error) {
ipv4, ipv4Err := fetchCIDRList(ctx, client, cfIPv4URL)
if ipv4Err != nil {
ipv4Err = fmt.Errorf("fetching CF IPv4: %w", ipv4Err)
}
if ctx.Err() != nil {
// Cancelled between the two fetches: do not start the second.
return ipv4, nil, errors.Join(ipv4Err, ctx.Err())
}
ipv6, ipv6Err := fetchCIDRList(ctx, client, cfIPv6URL)
if ipv6Err != nil {
ipv6Err = fmt.Errorf("fetching CF IPv6: %w", ipv6Err)
}
return ipv4, ipv6, errors.Join(ipv4Err, ipv6Err)
}
// fetchCIDRList fetches a URL and parses one CIDR per line.
func fetchCIDRList(ctx context.Context, client *http.Client, url string) ([]string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d from %s", resp.StatusCode, url)
}
scanner := bufio.NewScanner(resp.Body)
cidrs := parseCloudflareResponse(scanner)
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("reading %s: %w", url, err)
}
// A 200 with no ranges is an interstitial, a proxy page or a truncated
// body, never an empty Cloudflare list. Publishing it would flush every
// Cloudflare guard, so the previous list must stay in force.
if len(cidrs) == 0 {
return nil, fmt.Errorf("no CIDR ranges in response from %s", url)
}
return cidrs, nil
}
// parseCloudflareResponse parses lines from a scanner, returning valid CIDRs.
// Skips blank lines, comments, and invalid entries.
func parseCloudflareResponse(scanner *bufio.Scanner) []string {
var cidrs []string
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
_, _, err := net.ParseCIDR(line)
if err != nil {
continue
}
cidrs = append(cidrs, line)
}
return cidrs
}
// SaveCFState persists the Cloudflare CIDRs for status and coverage checks.
func SaveCFState(statePath string, ipv4, ipv6 []string, refreshed time.Time) error {
path := statePath
if !strings.HasSuffix(path, "/firewall") {
path += "/firewall"
}
if err := os.MkdirAll(path, 0700); err != nil {
return fmt.Errorf("creating Cloudflare state directory: %w", err)
}
var sb strings.Builder
fmt.Fprintf(&sb, "# refreshed: %s\n", refreshed.Format(time.RFC3339))
sb.WriteString("# ipv4\n")
for _, cidr := range ipv4 {
sb.WriteString(cidr)
sb.WriteByte('\n')
}
sb.WriteString("# ipv6\n")
for _, cidr := range ipv6 {
sb.WriteString(cidr)
sb.WriteByte('\n')
}
file := path + "/cf_whitelist.txt"
if err := atomicio.AtomicWrite(file, 0600, []byte(sb.String())); err != nil {
return fmt.Errorf("writing Cloudflare state: %w", err)
}
return nil
}
// CloudflareRangesCover reports whether ip falls inside any of the given
// Cloudflare CIDR ranges. The input chain accepts Cloudflare edges on TCP
// 80/443 before the blocked-IP drop, so blocking a covered IP does not stop
// its web traffic; callers use this to warn the operator at block time.
// Malformed CIDRs are skipped so one bad cache line cannot hide coverage.
func CloudflareRangesCover(v4, v6 []string, ip string) bool {
parsed := net.ParseIP(ip)
if parsed == nil {
return false
}
ranges := v4
if parsed.To4() == nil {
ranges = v6
}
for _, cidr := range ranges {
_, network, err := net.ParseCIDR(cidr)
if err != nil {
continue
}
if network.Contains(parsed) {
return true
}
}
return false
}
// LoadCFState reads the cached Cloudflare CIDRs.
func LoadCFState(statePath string) (ipv4, ipv6 []string) {
path := statePath
if !strings.HasSuffix(path, "/firewall") {
path += "/firewall"
}
file := path + "/cf_whitelist.txt"
// #nosec G304 -- fixed filename under operator-configured statePath.
f, err := os.Open(file)
if err != nil {
return nil, nil
}
defer f.Close()
scanner := bufio.NewScanner(f)
section := "ipv4"
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "# ipv4" {
section = "ipv4"
continue
}
if line == "# ipv6" {
section = "ipv6"
continue
}
if line == "" || strings.HasPrefix(line, "#") {
continue
}
switch section {
case "ipv4":
ipv4 = append(ipv4, line)
case "ipv6":
ipv6 = append(ipv6, line)
}
}
return ipv4, ipv6
}
// LoadCFRefreshTime reads the last CF refresh time from state.
func LoadCFRefreshTime(statePath string) time.Time {
path := statePath
if !strings.HasSuffix(path, "/firewall") {
path += "/firewall"
}
file := path + "/cf_whitelist.txt"
// #nosec G304 -- fixed filename under operator-configured statePath.
f, err := os.Open(file)
if err != nil {
return time.Time{}
}
defer f.Close()
scanner := bufio.NewScanner(f)
if scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "# refreshed: ") {
ts := strings.TrimPrefix(line, "# refreshed: ")
if t, err := time.Parse(time.RFC3339, ts); err == nil {
return t
}
}
}
return time.Time{}
}
package firewall
import (
"errors"
"fmt"
"net"
"slices"
"strings"
)
// FirewallConfig defines the nftables firewall configuration.
type FirewallConfig struct {
Enabled bool `yaml:"enabled"`
// Open ports (IPv4)
TCPIn []int `yaml:"tcp_in"`
TCPOut []int `yaml:"tcp_out"`
UDPIn []int `yaml:"udp_in"`
UDPOut []int `yaml:"udp_out"`
// IPv6 - enable dual-stack filtering
IPv6 bool `yaml:"ipv6"`
TCP6In []int `yaml:"tcp6_in"` // if empty, uses tcp_in
TCP6Out []int `yaml:"tcp6_out"` // if empty, uses tcp_out
UDP6In []int `yaml:"udp6_in"` // if empty, uses udp_in
UDP6Out []int `yaml:"udp6_out"` // if empty, uses udp_out
// Ports restricted to infra IPs only
RestrictedTCP []int `yaml:"restricted_tcp"`
// RequiredTCPOut declares outbound TCP ports a service on this host needs.
// It is checked, never merged: validation warns when an effective outbound
// family policy omits one, so an integration can state its
// requirement in its own conf.d fragment and have `csm doctor` verify it
// against the effective policy.
RequiredTCPOut []int `yaml:"required_tcp_out"`
// TCPOutAllow permits outbound TCP to a specific destination on a port
// range. tcp_out is []int and cannot express a range, so a host acting as
// a client for a range-using protocol (passive FTP data channels) has no
// way to state its need without opening every high port to the internet.
// Empty means no destination-scoped egress, which is the prior behaviour.
TCPOutAllow []OutAllowRule `yaml:"tcp_out_allow"`
// ProtocolAllow accepts an IP protocol that has no ports, such as VRRP
// between the members of a high-availability pair, to and from listed
// peers only. tcp/udp/icmp have their own keys and are not accepted here.
ProtocolAllow []ProtocolAllowRule `yaml:"protocol_allow"`
// TCPInAllow opens one TCP port to listed sources only. The accepts
// precede the blocklists, so a listed source keeps reaching that port
// when it is blocked, and stays blockable on every other port.
TCPInAllow []InAllowRule `yaml:"tcp_in_allow"`
// Passive FTP range
PassiveFTPStart int `yaml:"passive_ftp_start"`
PassiveFTPEnd int `yaml:"passive_ftp_end"`
// Infra IPs (CIDR notation)
InfraIPs []string `yaml:"infra_ips"`
// Rate limiting (per-source nftables meters). SYN/conn-rate/UDP are
// dual-stack (IPv6 keyed per /64); ConnLimit is IPv4-only (per-source
// ct count uses nf_conncount, whose GC has a kernel UAF on the el8 kernel).
ConnRateLimit int `yaml:"conn_rate_limit"` // new connections per minute per source (IPv6 per /64)
SYNFloodProtection bool `yaml:"syn_flood_protection"`
ConnLimit int `yaml:"conn_limit"` // max concurrent connections per IPv4 source, IPv4 only (0 = disabled)
// DropIPv4Options drops inbound IPv4 TCP packets that carry IP options
// before conntrack, NAT and every accept. Ordinary TCP never sets them,
// and options on a connection request are what CVE-2026-98432 needs on
// an unpatched kernel. Off only for workloads that rely on IP options,
// such as CIPSO labels.
DropIPv4Options bool `yaml:"drop_ipv4_options"`
// Per-port flood protection - per-source rate limit per port and IP family.
PortFlood []PortFloodRule `yaml:"port_flood"`
// UDP flood protection - per-source rate limit on UDP packets (IPv6 per /64)
UDPFlood bool `yaml:"udp_flood"`
UDPFloodRate int `yaml:"udp_flood_rate"` // packets per second
UDPFloodBurst int `yaml:"udp_flood_burst"` // burst allowance
// Country blocking
CountryBlock []string `yaml:"country_block"` // ISO country codes
CountryDBPath string `yaml:"country_db_path"`
// Ports to drop silently without logging (reduces log noise from scanners)
DropNoLog []int `yaml:"drop_nolog"`
// Max blocked IPs (prevents memory exhaustion, 0 = unlimited)
DenyIPLimit int `yaml:"deny_ip_limit"`
DenyTempIPLimit int `yaml:"deny_temp_ip_limit"`
// DenyTempIPLimitNegative records a negative deny_temp_ip_limit the
// loader read as 0, so validation can warn about what was written.
DenyTempIPLimitNegative bool `yaml:"-" json:"-"`
// Outbound SMTP restriction - block outgoing mail except from allowed users
SMTPBlock bool `yaml:"smtp_block"`
SMTPAllowUsers []string `yaml:"smtp_allow_users"` // usernames allowed to send
SMTPPorts []int `yaml:"smtp_ports"`
// LocalPortLocks lets only the listed users (and root) open new
// connections to a port on this host, as "PORT:user,user".
LocalPortLocks []string `yaml:"local_port_locks"`
// Dynamic DNS - resolve hostnames to IPs, update allowed set periodically
DynDNSHosts []string `yaml:"dyndns_hosts"`
// Logging
LogDropped bool `yaml:"log_dropped"`
LogRate int `yaml:"log_rate"` // log entries per minute
// DoS-exempt ranges - source CIDRs excluded from connection/mail-port
// meters and auto-subnet escalation. Intended for shared-source ranges
// such as carrier CGNAT blocks and well-known mail-provider egress.
DOSExemptRanges []string `yaml:"dos_exempt_ranges"`
DOSExemptKnownMailProviders *bool `yaml:"dos_exempt_known_mail_providers"`
}
// MergeInfraIPs returns the effective firewall infra list in configuration
// order. Equivalent IP spellings share one entry because nftables enforces the
// parsed address or network, not the operator's original text.
func MergeInfraIPs(topLevel, firewallSpecific []string) []string {
const infraIPHintCap = 1 << 16
hint := len(topLevel) + len(firewallSpecific)
if hint < 0 || hint > infraIPHintCap {
hint = infraIPHintCap
}
seen := make(map[string]struct{}, hint)
merged := make([]string, 0, hint)
appendEntries := func(entries []string) {
for _, raw := range entries {
entry := strings.TrimSpace(raw)
if entry == "" {
continue
}
key := canonicalInfraIPKey(entry)
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
merged = append(merged, entry)
}
}
appendEntries(topLevel)
appendEntries(firewallSpecific)
if len(merged) == 0 {
return nil
}
return merged
}
func canonicalInfraIPKey(entry string) string {
if _, network, err := net.ParseCIDR(entry); err == nil {
return "network:" + network.String()
}
if ip := net.ParseIP(entry); ip != nil {
bits := net.IPv6len * 8
if ip.To4() != nil {
bits = net.IPv4len * 8
}
return "network:" + (&net.IPNet{IP: ip, Mask: net.CIDRMask(bits, bits)}).String()
}
return "host:" + strings.ToLower(strings.TrimSuffix(entry, "."))
}
func effectiveIPv6Ports(ipv4, ipv6 []int) []int {
if len(ipv6) > 0 {
return ipv6
}
return ipv4
}
// restrictedOutputNeedsIPv4Bypass reports whether the restricted output chain
// would drop all IPv4 egress. effectiveIPv6Ports mirrors IPv4 out-ports onto
// IPv6 but never the reverse, so configuring only IPv6 egress ports puts the
// shared inet chain into DROP mode with no IPv4 accept rules. Callers invoke
// this only on the restricted path and, when true, must accept IPv4 wholesale
// after the SMTP guard so smtp_block still applies to IPv4.
func (c *FirewallConfig) restrictedOutputNeedsIPv4Bypass() bool {
return len(c.TCPOut) == 0 && len(c.UDPOut) == 0
}
// ExemptKnownMailProviders reports whether bundled mail-provider ranges are
// included in the DoS-exempt set. Defaults to true when unset.
func (c *FirewallConfig) ExemptKnownMailProviders() bool {
if c.DOSExemptKnownMailProviders == nil {
return true
}
return *c.DOSExemptKnownMailProviders
}
// ParseOutAllowDst accepts a bare IP or a CIDR and returns it as a network.
// A bare address becomes a host route; mapped IPv4 prefixes use IPv4 widths.
func ParseOutAllowDst(dst string) (*net.IPNet, error) {
if dst == "" {
return nil, errors.New("dst is required")
}
if _, network, err := net.ParseCIDR(dst); err == nil {
if ip4 := network.IP.To4(); ip4 != nil {
// ParseCIDR retains a 128-bit mask for mapped IPv4 prefixes.
// The engine loads only four bytes, so normalize both together.
network.IP = ip4
network.Mask = network.Mask[len(network.Mask)-net.IPv4len:]
}
return network, nil
}
ip := net.ParseIP(dst)
if ip == nil {
return nil, fmt.Errorf("dst %q is neither an IP address nor a CIDR", dst)
}
if ip4 := ip.To4(); ip4 != nil {
return &net.IPNet{IP: ip4, Mask: net.CIDRMask(32, 32)}, nil
}
return &net.IPNet{IP: ip.To16(), Mask: net.CIDRMask(128, 128)}, nil
}
// OutAllowRule permits outbound TCP to one destination on a port range.
// Dst is an IP or CIDR; 0.0.0.0/0 and ::/0 mean any destination in that family.
type OutAllowRule struct {
Dst string `yaml:"dst"`
PortStart int `yaml:"port_start"`
PortEnd int `yaml:"port_end"`
}
// ProtocolAllowRule accepts one portless IP protocol from and to its peers.
// Each peer is an IP or CIDR, parsed as a tcp_out_allow destination.
type ProtocolAllowRule struct {
Protocol string `yaml:"protocol"`
Peers []string `yaml:"peers"`
}
// InAllowRule accepts inbound TCP on one port from its sources.
// Each source is an IP or CIDR, parsed as a tcp_out_allow destination.
type InAllowRule struct {
Port int `yaml:"port"`
Sources []string `yaml:"sources"`
}
// Networks returns the sources the engine renders an accept for: none when
// the port is out of range, and otherwise every parseable source that is not
// world-wide, IPv6 ones only while dual-stack filtering is on. Validation
// reports what this leaves out; callers must not widen it.
func (r InAllowRule) Networks(ipv6Enabled bool) []*net.IPNet {
if r.Port < 1 || r.Port > 65535 {
return nil
}
var out []*net.IPNet
for _, source := range r.Sources {
network, err := ParseOutAllowDst(source)
if err != nil {
continue
}
if ones, _ := network.Mask.Size(); ones == 0 {
continue
}
if network.IP.To4() == nil && !ipv6Enabled {
continue
}
out = append(out, network)
}
return out
}
// allowedProtocol is an IP protocol protocol_allow can name, with the
// link-local multicast groups it advertises to.
type allowedProtocol struct {
number byte
groups []net.IP
}
var allowedProtocols = map[string]allowedProtocol{
// VRRP (RFC 5798) advertises to 224.0.0.18 and, as VRRPv3, ff02::12.
"vrrp": {number: 112, groups: []net.IP{net.IPv4(224, 0, 0, 18).To4(), net.ParseIP("ff02::12")}},
}
// ProtocolAllowNames lists the protocols firewall.protocol_allow accepts.
func ProtocolAllowNames() []string {
names := make([]string, 0, len(allowedProtocols))
for name := range allowedProtocols {
names = append(names, name)
}
slices.Sort(names)
return names
}
// PortFloodRule defines per-port connection rate limiting.
type PortFloodRule struct {
Port int `yaml:"port"`
Proto string `yaml:"proto"` // "tcp" or "udp"
Hits int `yaml:"hits"` // max new connections
Seconds int `yaml:"seconds"` // time window in seconds
}
// DefaultConfig returns a sensible default firewall configuration
// matching a typical cPanel server.
func DefaultConfig() *FirewallConfig {
return &FirewallConfig{
Enabled: false,
// SSH (22) is intentionally absent. Many cPanel hosts move sshd to
// 2087 or other alt ports; operators who keep sshd on 22 must add
// it explicitly. TCP 853 enables DNS-over-TLS; UDP 853 enables
// DNS-over-QUIC.
TCPIn: []int{
20, 21, 25, 26, 53, 80, 110, 143, 443, 465, 587,
853, 993, 995, 2077, 2078, 2079, 2080, 2082, 2083,
2091, 2095, 2096,
},
TCPOut: []int{
20, 21, 25, 26, 37, 43, 53, 80, 110, 113, 443,
465, 587, 853, 873, 993, 995, 2082, 2083, 2086, 2087,
2089, 2195, 2325, 2703,
},
UDPIn: []int{53, 443, 853},
// 6277/24441 are DCC/Pyzor network checks used by SpamAssassin.
// Without them outbound spam-scoring queries silently fail.
UDPOut: []int{53, 113, 123, 443, 853, 873, 6277, 24441},
RestrictedTCP: []int{2086, 2087, 2325, 9443},
TCPOutAllow: nil,
PassiveFTPStart: 49152,
PassiveFTPEnd: 65534,
// 200 new connections per minute per source (IPv6 aggregated per /64).
// Sized to tolerate shared CGNAT egress (residential ISPs / mobile
// carriers) where hundreds of subscribers share a single public
// address; the original 30/min default produced spurious drops on
// such ranges.
ConnRateLimit: 200,
SYNFloodProtection: true,
// 400 concurrent connections per IPv4 source. Sized for power users
// (multi-tab webmail + IMAP IDLE on multiple devices + Thunderbird
// parallel send + HTTPS browsing) plus headroom for shared CGNAT
// egress IPs. Operators with very heavy IDLE-style workloads can
// raise it further.
ConnLimit: 400,
DropIPv4Options: true,
// 600 hits / 300 s = 120 new connections per minute per source IP.
// Sized to tolerate normal MUA bursts (Thunderbird/iPhone/Outlook each
// open 5-15 parallel sessions when sending one email or syncing IMAP
// after suspend) while still catching true single-IP floods. Detection
// of low-and-slow scanners belongs to userspace, not this rule.
PortFlood: []PortFloodRule{
{Port: 25, Proto: "tcp", Hits: 600, Seconds: 300},
{Port: 465, Proto: "tcp", Hits: 600, Seconds: 300},
{Port: 587, Proto: "tcp", Hits: 600, Seconds: 300},
},
UDPFlood: true,
UDPFloodRate: 100,
UDPFloodBurst: 500,
DropNoLog: []int{23, 67, 68, 111, 113, 135, 136, 137, 138, 139, 445, 500, 513, 520},
DenyIPLimit: 3000,
DenyTempIPLimit: 500,
SMTPBlock: false,
// cPanel's own sender and Mailman; users missing on a host are skipped.
SMTPAllowUsers: []string{"cpanel", "mailman"},
SMTPPorts: []int{25, 465, 587},
LogDropped: true,
LogRate: 5,
}
}
package firewall
import "path/filepath"
// CountryDBDir returns the directory holding the per-country CIDR files:
// country_db_path when set, otherwise the geoip directory under the state
// path, which is where `csm firewall update-geoip` writes. Every consumer
// (engine, CLI update and lookup) resolves through here so an empty setting
// cannot leave country blocking armed on one surface and inert on another.
func CountryDBDir(cfg *FirewallConfig, statePath string) string {
if cfg != nil && cfg.CountryDBPath != "" {
return cfg.CountryDBPath
}
return filepath.Join(statePath, "geoip")
}
// countryBlockingActive reports whether the operator asked for country
// blocking. The DB directory always resolves, so the code list alone decides.
func countryBlockingActive(cfg *FirewallConfig) bool {
return cfg != nil && len(cfg.CountryBlock) > 0
}
package firewall
import (
"context"
"fmt"
"net"
"os"
"sort"
"sync"
"time"
)
// hostHealth tracks per-host resolution state for the re-resolve guard.
type hostHealth struct {
lastSuccess time.Time
firstFailure time.Time
findingEmitted bool
}
// DynDNSResolver periodically resolves hostnames and updates the firewall allowed set.
type DynDNSResolver struct {
mu sync.Mutex
hosts []string
resolved map[string][]string // hostname -> all resolved IPs
// infraHosts is the subset of hosts that also feed the engine's
// infra-IP guard. When a host appears here, every successful
// resolution additionally calls engine.UpdateInfraResolved so the
// hostname stays blockable-refusable even when the IP behind it
// rotates. Hosts not in this map only feed the allowed-IPs set.
infraHosts map[string]struct{}
engine interface {
AllowIP(ip string, reason string) error
RemoveAllowIPBySource(ip string, source string) error
}
// infraEngine receives infra-mode updates. Optional - separate
// interface so the AllowIP/RemoveAllowIPBySource consumers don't
// have to grow when an operator does not declare any infra hosts.
infraEngine interface {
UpdateInfraResolved(host string, ips []string)
DropInfraResolved(host string)
}
// lookupFn is the context-bound DNS lookup function. Defaults to
// net.DefaultResolver.LookupHost so callers can bound a stuck
// resolver by cancelling the parent context. Tests may replace this
// with a stub.
lookupFn func(ctx context.Context, host string) ([]string, error)
// Guard fields for the DNS re-resolve guard.
muGuard sync.RWMutex
hostHealth map[string]*hostHealth
gracePeriod time.Duration
unresolvable map[string]struct{}
findingSink func(name string)
}
// NewDynDNSResolver creates a resolver for the given hostnames.
func NewDynDNSResolver(hosts []string, engine interface {
AllowIP(ip string, reason string) error
RemoveAllowIPBySource(ip string, source string) error
}) *DynDNSResolver {
r := &DynDNSResolver{
hosts: hosts,
resolved: make(map[string][]string),
engine: engine,
hostHealth: make(map[string]*hostHealth),
unresolvable: make(map[string]struct{}),
gracePeriod: 10 * time.Minute,
}
r.lookupFn = net.DefaultResolver.LookupHost
return r
}
// AddHost appends a hostname to the resolver's host list.
// It is safe to call concurrently. Used in tests to add hosts after construction.
func (d *DynDNSResolver) AddHost(host string) {
d.mu.Lock()
defer d.mu.Unlock()
d.hosts = append(d.hosts, host)
}
// RegisterInfraHost marks host as an infra hostname. Every subsequent
// successful resolution will, in addition to AllowIP, call
// engine.UpdateInfraResolved so the resolved IPs feed the infra-block
// guard. Idempotent. Wire an infra engine via SetInfraEngine before
// the resolver's first tick to make this effective.
func (d *DynDNSResolver) RegisterInfraHost(host string) {
d.mu.Lock()
defer d.mu.Unlock()
if d.infraHosts == nil {
d.infraHosts = make(map[string]struct{})
}
d.infraHosts[host] = struct{}{}
}
// SetInfraEngine wires the engine that receives infra-mode resolution
// updates. Setting it to nil disables infra routing without affecting
// the regular allowed-IPs path.
func (d *DynDNSResolver) SetInfraEngine(eng interface {
UpdateInfraResolved(host string, ips []string)
DropInfraResolved(host string)
}) {
d.mu.Lock()
defer d.mu.Unlock()
d.infraEngine = eng
}
// markLastSuccess seeds the lastSuccess timestamp for a host to now.
// Used in tests to simulate a prior successful resolution without a real
// DNS lookup.
func (d *DynDNSResolver) markLastSuccess(host string) {
d.muGuard.Lock()
defer d.muGuard.Unlock()
hh := d.hostHealth[host]
if hh == nil {
hh = &hostHealth{}
d.hostHealth[host] = hh
}
hh.lastSuccess = time.Now()
}
// SetFindingSink installs the callback invoked when a host has been
// unresolvable for longer than gracePeriod. Called from the daemon at
// startup, after the alert pipeline is wired.
func (d *DynDNSResolver) SetFindingSink(sink func(host string)) {
d.muGuard.Lock()
defer d.muGuard.Unlock()
d.findingSink = sink
}
// UnresolvableHosts lists infra_ips hostnames currently failing to resolve
// beyond the grace period.
func (d *DynDNSResolver) UnresolvableHosts() []string {
d.muGuard.RLock()
defer d.muGuard.RUnlock()
out := make([]string, 0, len(d.unresolvable))
for h := range d.unresolvable {
out = append(out, h)
}
sort.Strings(out)
return out
}
// Run starts the periodic resolver. Blocks until stopCh is closed.
func (d *DynDNSResolver) Run(stopCh <-chan struct{}) {
parent, cancelParent := context.WithCancel(context.Background())
defer cancelParent()
done := make(chan struct{})
go func() {
select {
case <-stopCh:
cancelParent()
case <-done:
}
}()
defer close(done)
select {
case <-stopCh:
cancelParent()
return
default:
}
d.runTick(parent)
ticker := time.NewTicker(5 * time.Minute)
defer ticker.Stop()
for {
select {
case <-parent.Done():
return
case <-ticker.C:
d.runTick(parent)
}
}
}
// runTick is the periodic Run helper. It bounds a single resolution
// cycle so a stuck DNS server cannot hold the ticker beyond its budget
// and stack late ticks. The per-tick budget is purposely larger than
// the default 30-second per-resolver timeout but smaller than the
// 5-minute ticker period.
func (d *DynDNSResolver) runTick(parent context.Context) {
ctx, cancel := context.WithTimeout(parent, dyndnsTickBudget)
defer cancel()
d.tickOnce(ctx)
}
const dyndnsTickBudget = 60 * time.Second
// tickOnce performs one resolution cycle over all registered hosts.
// The periodic Run loop calls this; tests can call it directly. The
// context bounds the whole cycle so a single stuck DNS server cannot
// hold the tick for the resolver's implicit timeout (~30s) and pile
// late ticks on top of the configured 5-minute period.
func (d *DynDNSResolver) tickOnce(ctx context.Context) {
d.mu.Lock()
hosts := make([]string, len(d.hosts))
copy(hosts, d.hosts)
d.mu.Unlock()
for _, host := range hosts {
if ctx.Err() != nil {
return
}
d.resolveHost(ctx, host)
}
}
// resolveAll is retained for backward compatibility with existing tests.
func (d *DynDNSResolver) resolveAll() {
d.tickOnce(context.Background())
}
func (d *DynDNSResolver) resolveHost(ctx context.Context, host string) {
newIPs, err := d.lookupFn(ctx, host)
if err != nil || len(newIPs) == 0 {
if ctx.Err() == context.Canceled {
return
}
fmt.Fprintf(os.Stderr, "dyndns: failed to resolve %s: %v\n", host, err)
d.updateGuardFailure(host)
return
}
sort.Strings(newIPs)
d.mu.Lock()
oldIPs := d.resolved[host]
d.mu.Unlock()
oldSet := make(map[string]bool)
for _, ip := range oldIPs {
oldSet[ip] = true
}
newSet := make(map[string]bool)
var infraIPs []string
for _, ip := range newIPs {
newSet[ip] = true
if parsed := net.ParseIP(ip); parsed != nil {
infraIPs = append(infraIPs, parsed.String())
}
}
var successIPs []string
// Keep failed removals tracked so the next DNS refresh retries them.
for _, ip := range oldIPs {
if !newSet[ip] {
if err := d.engine.RemoveAllowIPBySource(ip, SourceDynDNS); err != nil {
successIPs = append(successIPs, ip)
fmt.Fprintf(os.Stderr, "dyndns: error removing %s (%s): %v\n", ip, host, err)
continue
}
fmt.Fprintf(os.Stderr, "dyndns: %s removed %s (no longer resolves)\n", host, ip)
}
}
// Add new IPs
reason := fmt.Sprintf("dyndns: %s", host)
for _, ip := range newIPs {
if oldSet[ip] {
successIPs = append(successIPs, ip) // already allowed
continue
}
if err := d.engine.AllowIP(ip, reason); err != nil {
fmt.Fprintf(os.Stderr, "dyndns: error allowing %s (%s): %v\n", ip, host, err)
continue
}
successIPs = append(successIPs, ip)
fmt.Fprintf(os.Stderr, "dyndns: %s resolved to %s (added)\n", host, ip)
}
// Retain active allows and old addresses whose removal still needs retrying.
d.mu.Lock()
d.resolved[host] = successIPs
isInfra := false
if _, ok := d.infraHosts[host]; ok {
isInfra = true
}
infraEngine := d.infraEngine
d.mu.Unlock()
// Infra mode feeds the block guard from DNS itself, not from the
// allow-list mutation result. A transient nftables write failure
// should not make a resolved management hostname blockable.
if isInfra && infraEngine != nil {
infraEngine.UpdateInfraResolved(host, infraIPs)
}
// Successful resolution: clear any guard state.
d.updateGuardSuccess(host)
}
// updateGuardSuccess marks a host as successfully resolved in the guard state.
func (d *DynDNSResolver) updateGuardSuccess(host string) {
d.muGuard.Lock()
hh := d.hostHealth[host]
if hh == nil {
hh = &hostHealth{}
d.hostHealth[host] = hh
}
hh.lastSuccess = time.Now()
hh.firstFailure = time.Time{}
if _, was := d.unresolvable[host]; was {
delete(d.unresolvable, host)
fmt.Fprintf(os.Stderr, "dyndns: %s recovered (resolution succeeded)\n", host)
}
hh.findingEmitted = false
d.muGuard.Unlock()
}
// updateGuardFailure updates guard state after a failed resolution for a host.
// If the host has been unresolvable beyond gracePeriod and no finding has been
// emitted yet, it marks the host unresolvable and invokes the finding sink.
func (d *DynDNSResolver) updateGuardFailure(host string) {
d.muGuard.Lock()
hh := d.hostHealth[host]
if hh == nil {
hh = &hostHealth{}
d.hostHealth[host] = hh
}
now := time.Now()
if hh.firstFailure.IsZero() {
hh.firstFailure = now
}
since := hh.lastSuccess
if since.IsZero() {
since = hh.firstFailure
}
var sinkToCall func(string)
if now.Sub(since) > d.gracePeriod &&
!hh.findingEmitted {
d.unresolvable[host] = struct{}{}
hh.findingEmitted = true
sinkToCall = d.findingSink
}
d.muGuard.Unlock()
// Invoke sink outside the lock to prevent deadlock if sink calls back in.
if sinkToCall != nil {
sinkToCall(host)
}
}
package firewall
import "net"
// EffectiveDOSExemptNets returns the union of operator ranges and (when enabled)
// provider ranges, split into IPv4 and IPv6 *net.IPNet slices. providerNets may
// be nil. Invalid operator entries are skipped (validation already rejected them
// at load; this is defense in depth).
func EffectiveDOSExemptNets(cfg *FirewallConfig, providerNets []*net.IPNet) (v4, v6 []*net.IPNet) {
var all []*net.IPNet
if cfg != nil {
for _, s := range cfg.DOSExemptRanges {
n := parseExemptNet(s)
if n != nil {
all = append(all, n)
}
}
}
// nil cfg treats ExemptKnownMailProviders as true (safe default).
if cfg == nil || cfg.ExemptKnownMailProviders() {
all = append(all, providerNets...)
}
// Split and copy. To4() returns non-nil for IPv4, including
// IPv4-mapped-in-IPv6 addresses, so it correctly classifies both.
// Copies ensure callers cannot mutate the returned slices back into
// any shared state held by the caller or this package.
for _, n := range all {
if n == nil {
continue
}
ip := make(net.IP, len(n.IP))
copy(ip, n.IP)
mask := make(net.IPMask, len(n.Mask))
copy(mask, n.Mask)
c := &net.IPNet{IP: ip, Mask: mask}
if n.IP.To4() != nil {
v4 = append(v4, c)
} else {
v6 = append(v6, c)
}
}
return v4, v6
}
// parseExemptNet parses s as a CIDR or a bare IP (becomes /32 or /128).
// Returns nil for invalid input.
func parseExemptNet(s string) *net.IPNet {
if _, n, err := net.ParseCIDR(s); err == nil {
return n
}
ip := net.ParseIP(s)
if ip == nil {
return nil
}
if ip4 := ip.To4(); ip4 != nil {
return &net.IPNet{IP: ip4, Mask: net.CIDRMask(32, 32)}
}
return &net.IPNet{IP: ip.To16(), Mask: net.CIDRMask(128, 128)}
}
//go:build linux
package firewall
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
"github.com/mdlayher/netlink"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/atomicio"
)
// Engine manages the nftables firewall ruleset.
// Manages the nftables ruleset via netlink.
type Engine struct {
lifecycle *Lifecycle
unverifiedBlockActions map[string]bool
failedBlockActions map[string]bool
blockActionsUnproven error
blockBatchStateErr error
blockBatchesHeld bool
blockRemovalPending bool
stateRevision uint64
stateReadErr error
stateSetAside string
stateAsideErr error
renameStateAside func(string, string) error
syncStateAside func(string) error
mu sync.Mutex
conn *nftables.Conn
// listTables, when set, replaces conn.ListTables (tests inject a
// failing listing).
listTables func() ([]*nftables.Table, error)
cfg *FirewallConfig
// Captured only after our own successful ruleset transaction. Config-file
// hashes cannot authenticate the rules currently installed in the kernel.
appliedRuleset string
lockedLocalPorts []int
readRuleset func() (string, error)
// dryRunRecorder is called by BlockIP when auto_response.dry_run is
// active. Set by SetDryRunRecorder after construction so the firewall
// package does not import internal/store (which would be a cycle).
dryRunRecorder func(ip, reason string, timeout time.Duration)
// dryRunEnabled reports whether auto_response.dry_run is active.
// Set by the daemon so this package does not import internal/config.
dryRunEnabled func() bool
// verdictAsker, when set, is consulted after local block validation and
// before the dry-run gate. The callback returns (verdict, tenantID, note,
// error). Verdict "allow" short-circuits the block; "block" or empty
// proceeds with the default flow.
// Errors are fail-open: the daemon proceeds with the default block and
// logs the failure. Daemon owns the underlying verdict.Client; this
// package stays free of the internal/verdict import.
verdictAsker func(ctx context.Context, ip, reason string) (verdict, tenantID, note string, err error)
// softAllowFn, when set, reports whether an IP belongs to a verified-bot
// range (built-in/auto-updated crawler snapshots plus operator
// reputation.verified_bots IP-range entries). The auto-block path consults
// it so a published crawler the auto-blocker would otherwise re-add to
// blocked_ips is left out. Operator `firewall deny` (BlockIPForce) bypasses
// this gate. Set by the daemon so this package does not import
// internal/threatintel.
softAllowFn func(ip string) bool
// operatorReleaseFn, when set, is told of each successful release of an
// address (unblock, unban, allow, temporary allow, undo) and of a flush
// ("", every address), with its kind, so the daemon can drop that
// address's mail spray history. Set by the daemon so this package does
// not import it.
operatorReleaseFn func(ip string, kind ReleaseKind)
// shutdownCtx, when set, scopes the lifetime of any in-flight verdict
// callback to daemon shutdown. Without it, BlockIPOutcome used
// context.Background() and a wedged panel callback kept the
// auto-block caller waiting for the full http.Client.Timeout during
// graceful restart. Nil falls back to context.Background() so unit
// tests that build the Engine literal without a daemon keep working.
shutdownCtx context.Context
table *nftables.Table
chainIn *nftables.Chain
chainOut *nftables.Chain
setBlocked *nftables.Set
setBlockedNet *nftables.Set
setAllowed *nftables.Set
setInfra *nftables.Set
setCountry *nftables.Set
// Cloudflare IP whitelist sets (interval for CIDR matching)
setCFWhitelist *nftables.Set // IPv4
setCFWhitelist6 *nftables.Set // IPv6
// DoS-exempt ranges (interval sets for CIDR matching).
// Sources in these sets bypass per-IP rate-limit / conn-limit / port-flood
// rules. setDOSExempt6 is nil when IPv6 is disabled.
setDOSExempt *nftables.Set
setDOSExempt6 *nftables.Set
// dosExemptProviderNets holds the mail-provider overlay pushed by the
// daemon. Written by SetDOSExemptProviderNets and RefreshDOSExemptSets;
// read by createSets() and RefreshDOSExemptSets. Always guarded by mu.
dosExemptProviderNets []*net.IPNet
// IPv6 sets (nil if IPv6 disabled)
setBlocked6 *nftables.Set
setBlockedNet6 *nftables.Set
setAllowed6 *nftables.Set
setInfra6 *nftables.Set
setCountry6 *nftables.Set
// Meters for per-IP rate limiting
meterSYN *nftables.Set
meterConn *nftables.Set
meterUDP *nftables.Set
meterConnlim *nftables.Set
// IPv6 siblings of the rate meters (keyed on the /64 prefix). There is
// deliberately no v6 connlimit: per-source ct count uses nf_conncount,
// whose GC has a kernel use-after-free on the el8 kernel, so concurrent
// connection limiting stays IPv4-only.
meterSYN6 *nftables.Set
meterConn6 *nftables.Set
meterUDP6 *nftables.Set
meterPortFlood4 map[string]*nftables.Set
meterPortFlood6 map[string]*nftables.Set
statePath string
// Cached parsed firewall state. Population is lazy: the first call
// to loadStateFile under e.mu reads + parses state.json once, then
// subsequent calls return a deep copy from the in-memory cache as
// long as the on-disk metadata key is unchanged.
//
// Caching avoids repeated disk reads and linear scans on frequent
// block and allow checks.
//
// All four fields are written only while e.mu is held. The index
// maps are rebuilt every time stateCache is repopulated so
// O(1) lookups stay coherent with the cached slices. blockedIPIndex
// stores the slice position for each canonical IP.
stateCache *FirewallState
stateCacheKey stateFileCacheKey
blockedIPIndex map[string]int
allowedIPIndex map[string]struct{}
portAllowedIndex map[string]struct{}
blockedCIDRIndex map[string]struct{}
// expiryDue is the earliest moment an entry of stateCache can expire.
// Lookups skip the expiry scan until then: the netblock and auto-block
// passes check thousands of addresses per cycle, and scanning thousands
// of cached entries on every check made each pass quadratic.
// expiryClock, when non-nil, replaces time.Now for cache expiry so
// tests can pin expiry boundaries. expiryScans counts full scans.
expiryDue expiryDeadline
expiryClock func() time.Time
expiryScans uint64
// lease is the last reading of the clock that turns absolute expiries
// into kernel timeouts (lease_clock_linux.go). leaseClock, when
// non-nil, samples the wall and elapsed clocks so tests can step one
// without the other.
lease leaseReading
leaseClock func() (wall, elapsed time.Time)
// leaseLeadReported is set once a lead over the wall clock is logged.
leaseLeadReported bool
// bootClock reads the boot ID and the time since boot for the lease
// checkpoint (lease_checkpoint_linux.go); leaseResumed is set once the
// first Apply has read it.
bootClock func() (bootID string, sinceBoot time.Duration, err error)
leaseResumed bool
liveBlockLookup func(set *nftables.Set, key []byte) (bool, error)
// liveBlockedDump, when non-nil, replaces the netlink dump of one
// blocked set. Tests inject it to avoid a real nft connection.
liveBlockedDump func(set *nftables.Set) ([]nftables.SetElement, error)
// blockRuleDump, when non-nil, replaces the netlink dump of the input
// chain's rules that block batches check.
blockRuleDump func() ([]*nftables.Rule, error)
// liveBlockCounts, when non-nil, returns the live (perm, temp)
// element counts across blocked v4 + v6 sets. Tests inject this
// to avoid spinning up a real nft connection. Nil falls back to
// GetSetElements queries.
liveBlockCounts func() (perm, temp int, err error)
// infraResolved maps a hostname declared under cfg.InfraIPs to its
// last successfully-resolved set of IPs. blockIPTarget refuses to
// block any of these so a transient DNS pause cannot let an
// attacker block CSM's own panel hostname into a lockout. Mutated
// under e.mu by UpdateInfraResolved / DropInfraResolved from the
// DynDNS resolver.
infraResolved map[string]map[string]struct{}
// localAddrs caches the host's own non-loopback interface addresses.
// The block guard refuses to block any of these regardless of
// cfg.InfraIPs so a misconfigured config or a stray scan that loops
// back to the daemon cannot brick the host. Refreshed lazily under
// e.mu when localAddrsExpiresAt has elapsed.
localAddrs map[string]struct{}
localAddrsExpiresAt time.Time
// localAddrsLookup, when non-nil, replaces net.InterfaceAddrs() for
// tests. Returning an error leaves the cache untouched.
localAddrsLookup func() ([]string, error)
}
type stateFileCacheKey struct {
modTime time.Time
changeTime time.Time
size int64
dev uint64
ino uint64
}
func stateFileCacheKeyFromInfo(info os.FileInfo) stateFileCacheKey {
key := stateFileCacheKey{
modTime: info.ModTime(),
size: info.Size(),
}
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
key.changeTime = time.Unix(stat.Ctim.Sec, stat.Ctim.Nsec)
key.dev = uint64(stat.Dev)
key.ino = uint64(stat.Ino)
}
return key
}
func (k stateFileCacheKey) matches(info os.FileInfo) bool {
other := stateFileCacheKeyFromInfo(info)
return k.size == other.size &&
k.dev == other.dev &&
k.ino == other.ino &&
k.modTime.Equal(other.modTime) &&
k.changeTime.Equal(other.changeTime)
}
// BlockedEntry represents a blocked IP with metadata.
type BlockedEntry struct {
IP string `json:"ip"`
Reason string `json:"reason"`
Source string `json:"source,omitempty"`
BlockedAt time.Time `json:"blocked_at"`
ExpiresAt time.Time `json:"expires_at"` // zero = permanent
// ActionID names the admission attempt that wrote the row; empty for
// every other block. Older builds ignore it.
ActionID string `json:"action_id,omitempty"`
// Unverified marks a row whose kernel element is unproven: the attempt
// ended unknown or has not been settled. It is not reported as a block
// and not restored into the kernel.
Unverified bool `json:"unverified,omitempty"`
// Automatic marks a timed row a legacy automatic block wrote: explicit
// provenance, where Source is read from the reason text.
// Admission rows carry ActionID instead. Older builds ignore it.
Automatic bool `json:"automatic,omitempty"`
// LoginSpray marks an automatic row a mail spray block wrote. These rows
// are evicted first at the temporary cap. Older builds ignore it.
LoginSpray bool `json:"login_spray,omitempty"`
}
// AllowedEntry represents an allowed IP with metadata.
type AllowedEntry struct {
IP string `json:"ip"`
Reason string `json:"reason"`
Source string `json:"source,omitempty"`
Port int `json:"port,omitempty"` // 0 = all ports
ExpiresAt time.Time `json:"expires_at,omitempty"` // zero = permanent
}
// SubnetEntry represents a blocked CIDR range.
type SubnetEntry struct {
CIDR string `json:"cidr"`
Reason string `json:"reason"`
Source string `json:"source,omitempty"`
BlockedAt time.Time `json:"blocked_at"`
ExpiresAt time.Time `json:"expires_at,omitempty"`
// Automatic records request intent; older Source values came from reason text.
Automatic bool `json:"automatic,omitempty"`
}
// PortAllowEntry represents a port-specific IP allow (e.g. tcp|in|d=PORT|s=IP).
type PortAllowEntry struct {
IP string `json:"ip"`
Port int `json:"port"`
Proto string `json:"proto"` // "tcp" or "udp"
Reason string `json:"reason"`
Source string `json:"source,omitempty"`
}
// FirewallState is persisted to disk for restore on restart.
type FirewallState struct {
Blocked []BlockedEntry `json:"blocked"`
BlockedNet []SubnetEntry `json:"blocked_nets"`
Allowed []AllowedEntry `json:"allowed"`
PortAllowed []PortAllowEntry `json:"port_allowed"`
}
// portU16 converts an operator-configured int port to the uint16 nftables
// expects. Returns 0 for out-of-range values; 0 is an unroutable TCP/UDP port
// so a misconfigured rule fails closed (no traffic matches) rather than
// wrapping silently to a valid-but-wrong port.
func portU16(p int) uint16 {
if p < 0 || p > 65535 {
return 0
}
// #nosec G115 -- bounded above.
return uint16(p)
}
// nftSocketBufferBytes is the netlink socket read/write buffer CSM requests for
// its nftables connections. The default OS netlink buffer
// (net.core.rmem_default, typically ~208 KiB) cannot hold the kernel ack stream
// for a full ruleset Apply on a host with a large blocklist; the apply then
// fails with ENOBUFS ("netlink receive: recvmsg: no buffer space available")
// and the firewall is left unmanaged. SetReadBuffer uses SO_RCVBUFFORCE under
// CAP_NET_ADMIN, so the larger size holds regardless of net.core.rmem_max.
const nftSocketBufferBytes = 8 << 20 // 8 MiB
// applyNFTSocketBuffer enlarges a netlink connection's receive and send
// buffers. It is best-effort: a resize failure is ignored so the connection
// still works with the default buffer (sufficient for small rulesets), and it
// returns nil so it never aborts the dial inside WithSockOptions.
func applyNFTSocketBuffer(nc *netlink.Conn) error {
_ = nc.SetReadBuffer(nftSocketBufferBytes)
_ = nc.SetWriteBuffer(nftSocketBufferBytes)
return nil
}
// newNFTConn opens an nftables netlink connection with enlarged socket buffers.
// WithSockOptions runs on every (re)dial, including the transient per-Flush
// sockets of a non-lasting connection, so each Apply uses the larger buffer.
func newNFTConn() (*nftables.Conn, error) {
return nftables.New(nftables.WithSockOptions(applyNFTSocketBuffer))
}
// NewEngine creates a new nftables firewall engine.
func NewEngine(cfg *FirewallConfig, statePath string) (*Engine, error) {
conn, err := newNFTConn()
if err != nil {
return nil, fmt.Errorf("nftables connection: %w", err)
}
e := &Engine{
conn: conn,
cfg: cfg,
statePath: filepath.Join(statePath, "firewall"),
}
_ = os.MkdirAll(e.statePath, 0700)
return e, nil
}
// SetDryRunRecorder installs a callback that is invoked by BlockIP whenever
// auto_response.dry_run is active. The daemon calls this after construction
// to wire in store.RecordDryRunBlock without creating an import cycle between
// internal/firewall and internal/store.
// errTableListing marks an Apply aborted because the kernel table listing
// failed; nothing was changed.
var errTableListing = errors.New("listing nftables tables")
// listTablesFn returns the table lister: the injected seam when a test set
// one, else the live connection.
func (e *Engine) listTablesFn() func() ([]*nftables.Table, error) {
if e.listTables != nil {
return e.listTables
}
return e.conn.ListTables
}
// SetConfig replaces the ruleset input the next Apply builds from. The
// engine was constructed with a value copy of the firewall block taken at
// daemon start, so every re-apply path (`csm firewall restart`,
// `apply-confirmed`, SIGHUP) rebuilt the old rules while reporting success
// and the operator's csm.yaml edit landed unprotected at the next restart.
// A nil cfg is ignored.
func (e *Engine) SetConfig(cfg *FirewallConfig) {
if cfg == nil {
return
}
e.mu.Lock()
defer e.mu.Unlock()
e.cfg = cfg
}
// Config returns the ruleset input the engine currently applies.
func (e *Engine) Config() *FirewallConfig {
e.mu.Lock()
defer e.mu.Unlock()
return e.cfg
}
func (e *Engine) SetDryRunRecorder(fn func(ip, reason string, timeout time.Duration)) {
e.mu.Lock()
e.dryRunRecorder = fn
e.mu.Unlock()
}
// SetDryRunEnabledFunc installs the callback BlockIP uses to decide whether
// auto_response.dry_run should intercept an automatic block. Nil means live.
func (e *Engine) SetDryRunEnabledFunc(fn func() bool) {
e.mu.Lock()
e.dryRunEnabled = fn
e.mu.Unlock()
}
// SetVerdictAsker installs the verdict callback the daemon constructs at
// startup. Nil disables the verdict callback (the gate skips entirely).
func (e *Engine) SetVerdictAsker(fn func(ctx context.Context, ip, reason string) (string, string, string, error)) {
e.mu.Lock()
e.verdictAsker = fn
e.mu.Unlock()
}
func (e *Engine) verdictAskerFn() func(ctx context.Context, ip, reason string) (string, string, string, error) {
e.mu.Lock()
fn := e.verdictAsker
e.mu.Unlock()
return fn
}
// SetSoftAllowChecker installs the callback the auto-block path consults to
// decide whether an IP belongs to a verified-bot range. Nil disables the
// verified-bot side of the soft-allow gate (operator allowed_ips is still
// honoured). The daemon wires this to threatintel so the firewall package
// stays free of that import.
func (e *Engine) SetSoftAllowChecker(fn func(ip string) bool) {
e.mu.Lock()
e.softAllowFn = fn
e.mu.Unlock()
}
// SetOperatorReleaseHook installs the callback told of each successful
// release with the canonical address, or "" after a flush, and its kind. It
// runs with the engine lock released, so it may call back into the engine.
func (e *Engine) SetOperatorReleaseHook(fn func(ip string, kind ReleaseKind)) {
e.mu.Lock()
e.operatorReleaseFn = fn
e.mu.Unlock()
}
// released runs the release hook for ip when err is nil. Callers defer it
// before taking e.mu, so it runs once the lock is released.
func (e *Engine) released(ip string, kind ReleaseKind, err error) {
if err != nil {
return
}
e.mu.Lock()
fn := e.operatorReleaseFn
e.mu.Unlock()
if fn != nil {
fn(ip, kind)
}
}
// autoBlockSoftAllowed reports whether an automatic block of ip must be
// skipped because ip is on a soft-allow list: an operator full-IP allow,
// an operator port-specific allow, or a verified-bot range. Only the
// auto-block path (BlockIPOutcome) consults this; BlockIPForce bypasses it so
// an explicit operator deny still wins.
func (e *Engine) autoBlockSoftAllowed(ip string) bool {
if e.operatorSoftAllowed(ip) {
return true
}
return e.autoBlockVerifiedRange(ip)
}
func (e *Engine) autoBlockVerifiedRange(ip string) bool {
e.mu.Lock()
fn := e.softAllowFn
e.mu.Unlock()
return fn != nil && fn(ip)
}
func (e *Engine) logAutoBlockSoftAllowed(ip string) {
fmt.Fprintf(os.Stderr, "[%s] auto-block: %s is allowlisted or a verified bot - not blocking\n",
time.Now().Format("2006-01-02 15:04:05"), ip)
}
// SetShutdownContext installs a context whose cancellation aborts any
// in-flight verdict callback. The daemon ties this to its stopCh so a
// graceful shutdown does not have to wait for an unresponsive panel
// callback to return.
func (e *Engine) SetShutdownContext(ctx context.Context) {
e.mu.Lock()
e.shutdownCtx = ctx
e.mu.Unlock()
}
func (e *Engine) verdictContext() context.Context {
e.mu.Lock()
ctx := e.shutdownCtx
e.mu.Unlock()
if ctx == nil {
return context.Background()
}
return ctx
}
// SetDOSExemptProviderNets stores the mail-provider IP ranges used by the
// dos_exempt_nets set. The daemon calls this before Apply() and on each
// provider refresh. Nil is valid when the daemon has no provider data yet;
// createSets() will produce an empty set in that case.
func (e *Engine) SetDOSExemptProviderNets(nets []*net.IPNet) {
e.mu.Lock()
e.dosExemptProviderNets = cloneIPNetSlice(nets)
e.mu.Unlock()
}
func cloneIPNetSlice(nets []*net.IPNet) []*net.IPNet {
if len(nets) == 0 {
return nil
}
out := make([]*net.IPNet, len(nets))
for i, n := range nets {
if n == nil {
continue
}
ip := make(net.IP, len(n.IP))
copy(ip, n.IP)
mask := make(net.IPMask, len(n.Mask))
copy(mask, n.Mask)
out[i] = &net.IPNet{IP: ip, Mask: mask}
}
return out
}
// dosExemptIntervalElems builds nftables interval set elements from nets.
// Callers must pass nets from a single IP family (all IPv4 or all IPv6).
func dosExemptIntervalElems(nets []*net.IPNet) []nftables.SetElement {
if len(nets) == 0 {
return nil
}
var elems []nftables.SetElement
for _, n := range nets {
if n == nil {
continue
}
var start net.IP
if n.IP.To4() != nil {
start = n.IP.To4()
} else {
start = n.IP.To16()
}
end := lastIPInRange(n)
if start == nil || end == nil {
continue
}
elems = appendIntervalSetElements(elems, start, end)
}
return normalizeIntervalElements(elems)
}
// RefreshDOSExemptSets repopulates the dos_exempt_nets[6] interval sets with
// a new provider overlay in a single batched kernel transaction. If the kernel
// batch fails the previous set contents remain active and dosExemptProviderNets
// is not updated, preserving the last-known good state.
func (e *Engine) RefreshDOSExemptSets(providerNets []*net.IPNet) error {
e.mu.Lock()
defer e.mu.Unlock()
if err := e.lifecycleReadyLocked(); err != nil {
return err
}
if e.setDOSExempt == nil {
return fmt.Errorf("dos_exempt_nets set not initialized")
}
v4, v6 := EffectiveDOSExemptNets(e.cfg, providerNets)
// Queue flush + repopulate for IPv4. Nothing is sent to the kernel yet.
e.conn.FlushSet(e.setDOSExempt)
if elems4 := dosExemptIntervalElems(v4); len(elems4) > 0 {
if err := e.conn.SetAddElements(e.setDOSExempt, elems4); err != nil {
return fmt.Errorf("queuing dos_exempt_nets elements: %w", err)
}
}
// Queue flush + repopulate for IPv6 when the set exists.
if e.setDOSExempt6 != nil {
e.conn.FlushSet(e.setDOSExempt6)
if elems6 := dosExemptIntervalElems(v6); len(elems6) > 0 {
if err := e.conn.SetAddElements(e.setDOSExempt6, elems6); err != nil {
return fmt.Errorf("queuing dos_exempt_nets6 elements: %w", err)
}
}
}
// Atomic commit. On failure the kernel keeps the previous elements.
if err := e.conn.Flush(); err != nil {
return fmt.Errorf("refreshing dos_exempt sets: %w", err)
}
// Update overlay only after the kernel confirmed the transaction.
e.dosExemptProviderNets = cloneIPNetSlice(providerNets)
return nil
}
// dosExemptV4Lookup returns a two-expression sequence that loads the IPv4
// source address (network-header offset 12, 4 bytes) into reg and performs an
// inverted set lookup against dos_exempt_nets. The caller must place the result
// after an IPv4 family guard and before the meter expressions. The Ct and
// Payload expressions that follow reload reg independently, so register reuse
// is safe.
func (e *Engine) dosExemptV4Lookup(reg uint32) []expr.Any {
return []expr.Any{
&expr.Payload{DestRegister: reg, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4},
&expr.Lookup{SourceRegister: reg, SetName: e.setDOSExempt.Name, SetID: e.setDOSExempt.ID, Invert: true},
}
}
// dosExemptV6Lookup is the IPv6 analogue of dosExemptV4Lookup, loading the
// IPv6 source address (network-header offset 8, 16 bytes) against
// dos_exempt_nets6. Use only in rules that carry IPv6 traffic.
func (e *Engine) dosExemptV6Lookup(reg uint32) []expr.Any {
return []expr.Any{
&expr.Payload{DestRegister: reg, Base: expr.PayloadBaseNetworkHeader, Offset: 8, Len: 16},
&expr.Lookup{SourceRegister: reg, SetName: e.setDOSExempt6.Name, SetID: e.setDOSExempt6.ID, Invert: true},
}
}
// Apply builds and atomically applies the complete nftables ruleset.
// All operations (delete old table + create new table/rules +
// populate persisted block/allow entries) are batched into a single
// netlink transaction. If the flush fails, the kernel keeps whatever
// ruleset was running before - the server is never left without a
// firewall. Equally important: the new ruleset never appears with
// EMPTY blocked sets between the table-swap and the persisted-state
// load; an attacker IP from state.json is blocked from the moment
// the new table becomes the live one.
func (e *Engine) Apply() (resultErr error) {
defer func() {
if e.shouldLegacyOutcome(resultErr) {
recordFirewallResult("apply", "csm", "", "", 0, actionlog.Applied, resultErr)
}
}()
e.mu.Lock()
defer e.mu.Unlock()
// Durable planning and verification need the same clock as installation.
if err := e.resumeLeaseLocked(); err != nil {
fmt.Fprintf(os.Stderr, "firewall: restoring timed blocks without the lease checkpoint: %v\n", err)
}
if err := e.lifecycleReadyLocked(); err != nil {
return err
}
if e.lifecycle != nil {
return e.applyDurableLocked()
}
return e.applyRulesetLocked("")
}
func (e *Engine) applyRulesetLocked(marker string) (resultErr error) {
// Compute the elements to seed each set from persisted state
// BEFORE touching nft. Pure computation; if state.json is
// missing or malformed the slices stay empty and the new table
// still applies (no firewall regression on a fresh install).
initial := e.computeInitialBlockStateLocked()
// Even a failed base install must stop fresh-engine startup retries:
// an unsynced rename can leave no file for the next engine to inspect.
defer func() {
if e.stateAsideErr != nil && !errors.Is(resultErr, ErrStateNotKeptAside) {
resultErr = errors.Join(e.stateAsideErr, resultErr)
}
}()
// Check if existing CSM table needs replacing.
// If so, include the delete in the same atomic batch as the new table.
// A listing that fails must abort: AddTable is create-only, so without
// the delete every rule below would be appended a second time to the
// live chains (doubled meters, halved rate limits).
tables, err := e.listTablesFn()()
if err != nil {
return fmt.Errorf("%w: %v", errTableListing, err)
}
var running *nftables.Table
for _, t := range tables {
if t.Name == "csm" && t.Family == nftables.TableFamilyINet {
running = t
break
}
}
// State the engine could neither read nor keep aside seeds nothing and
// fails startup. A running ruleset keeps its blocks; after a reboot the
// kernel has none, so the base rules still go in.
if e.stateAsideErr != nil && running != nil {
return e.stateAsideErr
}
if running != nil {
e.conn.DelTable(running)
}
// Create table - all operations below are batched, nothing is sent until Flush()
e.table = e.conn.AddTable(&nftables.Table{
Name: "csm",
Family: nftables.TableFamilyINet,
})
// Create IP sets
if err := e.createSets(); err != nil {
return fmt.Errorf("creating sets: %w", err)
}
// Create chains and rules
e.createPreroutingChain()
e.createInputChain()
e.createOutputChain()
// Queue initial set elements from persisted state into the same
// netlink batch as the table+set+chain creation above. Without
// this, Apply previously Flushed an empty-set ruleset, then a
// separate loadState() Flush populated the sets - leaving a
// brief window where the new table existed without the
// persisted blocks.
if err := e.queueInitialBlockStateLocked(initial); err != nil {
return fmt.Errorf("queueing initial firewall state: %w", err)
}
if marker != "" {
e.conn.AddChain(&nftables.Chain{Name: marker, Table: e.table})
}
// Apply atomically - if this fails, nftables keeps whatever was running before
if err := e.conn.Flush(); err != nil {
return fmt.Errorf("applying ruleset: %w", err)
}
e.lockedLocalPorts = configuredLockPorts(e.cfg)
if err := e.saveLeaseLocked(); err != nil {
fmt.Fprintf(os.Stderr, "firewall: lease checkpoint not saved: %v\n", err)
}
// A failed capture leaves monitoring visibly unbaselined; it must neither
// undo an applied firewall nor bless a later, possibly external edit.
snapshot, captureErr := e.readRulesetLocked()
e.appliedRuleset = snapshot
if captureErr != nil {
e.appliedRuleset = ""
}
return nil
}
// createSets creates the nftables named sets for IP management.
func (e *Engine) createSets() error {
// Blocked IPs set. HasTimeout enables per-element timeouts for temporary
// blocks; there must be NO set-level default timeout: the kernel assigns
// the default to every element added without its own timeout (the netlink
// library omits zero element timeouts), so a default would silently expire
// permanent blocks (deny, promote) while state.json still says blocked.
e.setBlocked = &nftables.Set{
Table: e.table,
Name: "blocked_ips",
KeyType: nftables.TypeIPAddr,
HasTimeout: true,
}
if err := e.conn.AddSet(e.setBlocked, nil); err != nil {
return fmt.Errorf("blocked set: %w", err)
}
// Blocked subnets set (interval for CIDR ranges, permanent)
e.setBlockedNet = &nftables.Set{
Table: e.table,
Name: "blocked_nets",
KeyType: nftables.TypeIPAddr,
Interval: true,
}
if err := e.conn.AddSet(e.setBlockedNet, nil); err != nil {
return fmt.Errorf("blocked nets set: %w", err)
}
// Allowed IPs set
e.setAllowed = &nftables.Set{
Table: e.table,
Name: "allowed_ips",
KeyType: nftables.TypeIPAddr,
}
if err := e.conn.AddSet(e.setAllowed, nil); err != nil {
return fmt.Errorf("allowed set: %w", err)
}
// Infra IPs set (interval for CIDR support) - split IPv4 and IPv6
e.setInfra = &nftables.Set{
Table: e.table,
Name: "infra_ips",
KeyType: nftables.TypeIPAddr,
Interval: true,
}
var infraElements []nftables.SetElement
var infra6Elements []nftables.SetElement
for _, cidr := range e.cfg.InfraIPs {
_, network, err := net.ParseCIDR(cidr)
if err != nil {
ip := net.ParseIP(cidr)
if ip == nil {
continue
}
if ip4 := ip.To4(); ip4 != nil {
infraElements = appendIntervalSetElements(infraElements, ip4, ip4)
} else if e.cfg.IPv6 {
ip16 := ip.To16()
infra6Elements = appendIntervalSetElements(infra6Elements, ip16, ip16)
}
continue
}
if network.IP.To4() != nil {
start := network.IP.To4()
end := lastIPInRange(network)
if start != nil && end != nil {
infraElements = appendIntervalSetElements(infraElements, start, end)
}
} else if e.cfg.IPv6 {
start := network.IP.To16()
end := lastIPInRange(network)
if start != nil && end != nil {
infra6Elements = appendIntervalSetElements(infra6Elements, start, end)
}
}
}
if err := e.conn.AddSet(e.setInfra, normalizeIntervalElements(infraElements)); err != nil {
return fmt.Errorf("infra set: %w", err)
}
// Cloudflare IP whitelist sets (interval for CIDR ranges, accept on 80/443 only)
e.setCFWhitelist = &nftables.Set{
Table: e.table,
Name: "cf_whitelist",
KeyType: nftables.TypeIPAddr,
Interval: true,
}
if err := e.conn.AddSet(e.setCFWhitelist, nil); err != nil {
return fmt.Errorf("cf_whitelist set: %w", err)
}
e.setCFWhitelist6 = &nftables.Set{
Table: e.table,
Name: "cf_whitelist6",
KeyType: nftables.TypeIP6Addr,
Interval: true,
}
if err := e.conn.AddSet(e.setCFWhitelist6, nil); err != nil {
return fmt.Errorf("cf_whitelist6 set: %w", err)
}
// Country-blocked IPs set (interval for CIDR ranges)
if countryBlockingActive(e.cfg) {
e.setCountry = &nftables.Set{
Table: e.table,
Name: "country_blocked",
KeyType: nftables.TypeIPAddr,
Interval: true,
}
var countryElements []nftables.SetElement
for _, code := range e.cfg.CountryBlock {
countryElements = append(countryElements, loadCountryCIDRs(CountryDBDir(e.cfg, e.statePath), code)...)
}
if err := e.conn.AddSet(e.setCountry, normalizeIntervalElements(countryElements)); err != nil {
fmt.Fprintf(os.Stderr, "firewall: warning creating country set: %v\n", err)
e.setCountry = nil
} else if len(countryElements) > 0 {
fmt.Fprintf(os.Stderr, "firewall: loaded %d country block ranges for %v\n",
len(countryElements)/2, e.cfg.CountryBlock)
}
}
// IPv6 sets
if e.cfg.IPv6 {
// No set-level default timeout; see the blocked_ips comment above.
e.setBlocked6 = &nftables.Set{
Table: e.table, Name: "blocked_ips6",
KeyType: nftables.TypeIP6Addr, HasTimeout: true,
}
if err := e.conn.AddSet(e.setBlocked6, nil); err != nil {
return fmt.Errorf("blocked6 set: %w", err)
}
e.setBlockedNet6 = &nftables.Set{
Table: e.table, Name: "blocked_nets6",
KeyType: nftables.TypeIP6Addr, Interval: true,
}
if err := e.conn.AddSet(e.setBlockedNet6, nil); err != nil {
return fmt.Errorf("blocked_nets6 set: %w", err)
}
e.setAllowed6 = &nftables.Set{
Table: e.table, Name: "allowed_ips6",
KeyType: nftables.TypeIP6Addr,
}
if err := e.conn.AddSet(e.setAllowed6, nil); err != nil {
return fmt.Errorf("allowed6 set: %w", err)
}
e.setInfra6 = &nftables.Set{
Table: e.table, Name: "infra_ips6",
KeyType: nftables.TypeIP6Addr, Interval: true,
}
if err := e.conn.AddSet(e.setInfra6, normalizeIntervalElements(infra6Elements)); err != nil {
return fmt.Errorf("infra6 set: %w", err)
}
// IPv6 country-block set, mirroring the IPv4 set above. Without it an
// attacker on an IPv6 address from a blocked country was never dropped.
if countryBlockingActive(e.cfg) {
e.setCountry6 = &nftables.Set{
Table: e.table, Name: "country_blocked6",
KeyType: nftables.TypeIP6Addr, Interval: true,
}
var country6Elements []nftables.SetElement
for _, code := range e.cfg.CountryBlock {
country6Elements = append(country6Elements, loadCountryCIDRs6(CountryDBDir(e.cfg, e.statePath), code)...)
}
if err := e.conn.AddSet(e.setCountry6, normalizeIntervalElements(country6Elements)); err != nil {
fmt.Fprintf(os.Stderr, "firewall: warning creating IPv6 country set: %v\n", err)
e.setCountry6 = nil
} else if len(country6Elements) > 0 {
fmt.Fprintf(os.Stderr, "firewall: loaded %d IPv6 country block ranges for %v\n",
len(country6Elements)/2, e.cfg.CountryBlock)
}
}
}
// DoS-exempt sets: sources here bypass connection meters and mail-port
// flood rules. Populated from operator dos_exempt_ranges + provider
// overlay (pushed by the daemon before Apply).
v4Exempt, v6Exempt := EffectiveDOSExemptNets(e.cfg, e.dosExemptProviderNets)
e.setDOSExempt = &nftables.Set{
Table: e.table,
Name: "dos_exempt_nets",
KeyType: nftables.TypeIPAddr,
Interval: true,
}
if err := e.conn.AddSet(e.setDOSExempt, dosExemptIntervalElems(v4Exempt)); err != nil {
return fmt.Errorf("dos_exempt_nets set: %w", err)
}
if e.cfg.IPv6 {
e.setDOSExempt6 = &nftables.Set{
Table: e.table,
Name: "dos_exempt_nets6",
KeyType: nftables.TypeIP6Addr,
Interval: true,
}
if err := e.conn.AddSet(e.setDOSExempt6, dosExemptIntervalElems(v6Exempt)); err != nil {
return fmt.Errorf("dos_exempt_nets6 set: %w", err)
}
}
// Meter sets for per-IP rate limiting (dynamic sets). The IPv6 siblings key
// on the /64 prefix; see ipv6MeterSet.
if e.cfg.SYNFloodProtection {
e.meterSYN = &nftables.Set{
Table: e.table, Name: "meter_syn", KeyType: nftables.TypeIPAddr,
Dynamic: true, HasTimeout: true, Timeout: time.Minute,
}
_ = e.conn.AddSet(e.meterSYN, nil)
if e.cfg.IPv6 {
e.meterSYN6 = e.ipv6MeterSet("meter_syn6")
}
}
if e.cfg.ConnRateLimit > 0 {
e.meterConn = &nftables.Set{
Table: e.table, Name: "meter_conn", KeyType: nftables.TypeIPAddr,
Dynamic: true, HasTimeout: true, Timeout: time.Minute,
}
_ = e.conn.AddSet(e.meterConn, nil)
if e.cfg.IPv6 {
e.meterConn6 = e.ipv6MeterSet("meter_conn6")
}
}
if e.cfg.UDPFlood && e.cfg.UDPFloodRate > 0 {
e.meterUDP = &nftables.Set{
Table: e.table, Name: "meter_udp", KeyType: nftables.TypeIPAddr,
Dynamic: true, HasTimeout: true, Timeout: time.Minute,
}
_ = e.conn.AddSet(e.meterUDP, nil)
if e.cfg.IPv6 {
e.meterUDP6 = e.ipv6MeterSet("meter_udp6")
}
}
if e.cfg.ConnLimit > 0 {
e.meterConnlim = &nftables.Set{
Table: e.table, Name: "meter_connlimit", KeyType: nftables.TypeIPAddr,
Dynamic: true,
}
_ = e.conn.AddSet(e.meterConnlim, nil)
}
if portFloodNeedsMeter(e.cfg.PortFlood) {
e.meterPortFlood4 = make(map[string]*nftables.Set)
if e.cfg.IPv6 {
e.meterPortFlood6 = make(map[string]*nftables.Set)
}
for _, pf := range e.cfg.PortFlood {
if !usablePortFloodRule(pf) {
continue
}
name4 := portFloodMeterName(pf, portFloodIPv4)
if _, ok := e.meterPortFlood4[name4]; !ok {
set := &nftables.Set{
Table: e.table, Name: name4, KeyType: nftables.TypeIPAddr,
Dynamic: true, HasTimeout: true, Timeout: time.Minute,
}
_ = e.conn.AddSet(set, nil)
e.meterPortFlood4[name4] = set
}
if e.cfg.IPv6 {
name6 := portFloodMeterName(pf, portFloodIPv6)
if _, ok := e.meterPortFlood6[name6]; !ok {
set := &nftables.Set{
Table: e.table, Name: name6, KeyType: nftables.TypeIP6Addr,
Dynamic: true, HasTimeout: true, Timeout: time.Minute,
}
_ = e.conn.AddSet(set, nil)
e.meterPortFlood6[name6] = set
}
}
}
}
return nil
}
// portFloodNeedsMeter reports whether any port_flood rule has a usable rate,
// so the meter set is only created when at least one rule will reference it.
func portFloodNeedsMeter(rules []PortFloodRule) bool {
for _, pf := range rules {
if usablePortFloodRule(pf) {
return true
}
}
return false
}
func usablePortFloodRule(pf PortFloodRule) bool {
return pf.Hits > 0 && pf.Seconds > 0 && pf.Port > 0
}
// createPreroutingChain drops inbound IPv4 TCP packets that carry IP
// options. Priority -310 runs it before conntrack (-200) and the DNAT that
// hands published ports to containers (-100): forwarded traffic never reaches
// the input chain. A drop in any base chain is final, so no accept elsewhere,
// infra included, can let such a packet through.
func (e *Engine) createPreroutingChain() {
if !e.cfg.DropIPv4Options {
return
}
policy := nftables.ChainPolicyAccept
chain := e.conn.AddChain(&nftables.Chain{
Name: "prerouting",
Table: e.table,
Type: nftables.ChainTypeFilter,
Hooknum: nftables.ChainHookPrerouting,
Priority: nftables.ChainPriorityRef(-310),
Policy: &policy,
})
exprs := append(ipv4NFProtoGuard(),
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}},
// IHL is the low nibble of the first header byte; 5 words is a
// header without options.
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 0, Len: 1},
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 1, Mask: []byte{0x0f}, Xor: []byte{0}},
&expr.Cmp{Op: expr.CmpOpGt, Register: 1, Data: []byte{5}},
&expr.Counter{},
&expr.Verdict{Kind: expr.VerdictDrop},
)
e.conn.AddRule(&nftables.Rule{Table: e.table, Chain: chain, Exprs: exprs})
}
// createInputChain builds the input filter chain with proper rule ordering.
func (e *Engine) createInputChain() {
policy := nftables.ChainPolicyDrop
e.chainIn = e.conn.AddChain(&nftables.Chain{
Name: "input",
Table: e.table,
Type: nftables.ChainTypeFilter,
Hooknum: nftables.ChainHookInput,
Priority: nftables.ChainPriorityFilter,
Policy: &policy,
})
// Rule 1: Allow loopback
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte("lo\x00"),
},
&expr.Verdict{Kind: expr.VerdictAccept},
},
})
if !e.cfg.IPv6 {
e.conn.AddRule(&nftables.Rule{Table: e.table, Chain: e.chainIn, Exprs: familyBypassRuleExprs(10)})
}
// Rule 2: Drop INVALID conntrack state (malformed packets)
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: []expr.Any{
&expr.Ct{Register: 1, SourceRegister: false, Key: expr.CtKeySTATE},
&expr.Bitwise{
SourceRegister: 1, DestRegister: 1, Len: 4,
Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitINVALID),
Xor: binaryutil.NativeEndian.PutUint32(0),
},
&expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(0)},
&expr.Verdict{Kind: expr.VerdictDrop},
},
})
// Rule 3: Allow infra IPs FIRST - infra must NEVER be blocked, even accidentally
e.addSetMatchRule(e.setInfra, expr.VerdictAccept)
e.addSetMatchRuleV6(e.setInfra6, expr.VerdictAccept)
// Peer-scoped protocol allows (VRRP between HA members) also precede the
// blocklists. A member that stops hearing its peer claims the shared
// address too, so a block on a peer must not cut its advertisements; the
// accept covers only that protocol from that peer.
protocolIn, _ := protocolAllowRuleExprs(e.cfg.ProtocolAllow, e.cfg.IPv6)
for _, exprs := range protocolIn {
e.conn.AddRule(&nftables.Rule{Table: e.table, Chain: e.chainIn, Exprs: exprs})
}
// Source-scoped port allows share that position: a listed source keeps
// reaching its port when a blocklist or the country block covers it, and
// reaches nothing else through this rule.
for _, exprs := range inAllowRuleExprs(e.cfg.TCPInAllow, e.cfg.IPv6) {
e.conn.AddRule(&nftables.Rule{Table: e.table, Chain: e.chainIn, Exprs: exprs})
}
// Rule 4: Cloudflare IP whitelist - accept on TCP 80/443 only.
// CF IPs can still be blocked on other ports (unlike infra).
e.addCFWhitelistRule(e.setCFWhitelist, false)
e.addCFWhitelistRule(e.setCFWhitelist6, true)
// Rule 5: Drop blocked IPs before established/related so active
// keep-alive connections do not bypass a new block.
e.addSetMatchRule(e.setBlocked, expr.VerdictDrop)
e.addSetMatchRuleV6(e.setBlocked6, expr.VerdictDrop)
// Rule 6: Drop blocked subnets (interval set for CIDR ranges)
e.addSetMatchRule(e.setBlockedNet, expr.VerdictDrop)
e.addSetMatchRuleV6(e.setBlockedNet6, expr.VerdictDrop)
// Rule 7: Allow established/related connections after block checks.
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: []expr.Any{
&expr.Ct{Register: 1, SourceRegister: false, Key: expr.CtKeySTATE},
&expr.Bitwise{
SourceRegister: 1,
DestRegister: 1,
Len: 4,
Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
Xor: binaryutil.NativeEndian.PutUint32(0),
},
&expr.Cmp{
Op: expr.CmpOpNeq,
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(0),
},
&expr.Verdict{Kind: expr.VerdictAccept},
},
})
// Rule 8: Allow explicitly allowed IPs
e.addSetMatchRule(e.setAllowed, expr.VerdictAccept)
e.addSetMatchRuleV6(e.setAllowed6, expr.VerdictAccept)
// Rule 9: Port-specific allows (IP+port, e.g. MySQL access for specific IPs)
state := e.loadStateFile()
for _, pa := range state.PortAllowed {
exprs := buildPortAllowExprs(pa, e.cfg.IPv6)
if exprs == nil {
continue
}
e.conn.AddRule(&nftables.Rule{Table: e.table, Chain: e.chainIn, Exprs: exprs})
}
// ICMPv4 echo-request (type 8)
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{1}}, // ICMPv4
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{8}}, // echo-request
&expr.Verdict{Kind: expr.VerdictAccept},
},
})
// ICMPv6 echo-request (type 128) + neighbor discovery (types 133-137, required for IPv6)
if e.cfg.IPv6 {
for _, icmp6Type := range []byte{128, 133, 134, 135, 136, 137} {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{58}}, // ICMPv6
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{icmp6Type}},
&expr.Verdict{Kind: expr.VerdictAccept},
},
})
}
}
// Per-IP SYN flood protection via meter
if e.cfg.SYNFloodProtection && e.meterSYN != nil {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: e.synFloodRuleExprs(),
})
if e.meterSYN6 != nil {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: e.synFloodRuleExprs6(),
})
}
}
// Per-IP new connection rate limit via meter
if e.cfg.ConnRateLimit > 0 && e.meterConn != nil {
// #nosec G115 -- ConnRateLimit is an operator-configured int (typical 10–1000);
// /2 is non-negative and well below uint32 max.
burst := uint32(e.cfg.ConnRateLimit / 2)
if burst < 5 {
burst = 5
}
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: e.connMeterRuleExprs(uint64(e.cfg.ConnRateLimit), burst),
})
if e.meterConn6 != nil {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: e.connMeterRuleExprs6(uint64(e.cfg.ConnRateLimit), burst),
})
}
}
// Per-IP concurrent connection limit (CONNLIMIT)
if e.cfg.ConnLimit > 0 && e.meterConnlim != nil {
// #nosec G115 -- ConnLimit is operator-configured non-negative int; fits in uint32.
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: e.connlimitRuleExprs(uint32(e.cfg.ConnLimit)),
})
}
// Rule 9: Country block - drop traffic from blocked countries
if e.setCountry != nil {
e.addSetMatchRule(e.setCountry, expr.VerdictDrop)
}
if e.setCountry6 != nil {
e.addSetMatchRuleV6(e.setCountry6, expr.VerdictDrop)
}
// Per-port flood protection - rate-limit new connections per source IP.
// Each rule has separate IPv4 and IPv6 meters so ports and families do not
// consume each other's token buckets.
for _, pf := range e.cfg.PortFlood {
items := []struct {
family portFloodIPFamily
meter *nftables.Set
}{
{family: portFloodIPv4, meter: e.meterPortFlood4[portFloodMeterName(pf, portFloodIPv4)]},
}
if e.cfg.IPv6 {
items = append(items, struct {
family portFloodIPFamily
meter *nftables.Set
}{family: portFloodIPv6, meter: e.meterPortFlood6[portFloodMeterName(pf, portFloodIPv6)]})
}
for _, item := range items {
exprs := e.portFloodRuleExprs(pf, item.meter, item.family)
if exprs == nil {
continue
}
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: exprs,
})
}
}
// Per-IP UDP flood protection via meter
if e.cfg.UDPFlood && e.cfg.UDPFloodRate > 0 && e.meterUDP != nil {
// #nosec G115 -- UDPFloodBurst is operator-configured non-negative int.
burst := uint32(e.cfg.UDPFloodBurst)
if burst < 10 {
burst = 10
}
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: e.udpFloodRuleExprs(uint64(e.cfg.UDPFloodRate), burst),
})
if e.meterUDP6 != nil {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: e.udpFloodRuleExprs6(uint64(e.cfg.UDPFloodRate), burst),
})
}
}
// Build restricted port set - these are only reachable via infra IPs (rule 4)
restricted := make(map[int]bool)
for _, p := range e.cfg.RestrictedTCP {
restricted[p] = true
}
// Open TCP ports (public) - restricted ports excluded
for _, port := range e.cfg.TCPIn {
if restricted[port] {
continue
}
e.addPortAcceptRule(port, true, 2)
}
// Open UDP ports (public)
for _, port := range e.cfg.UDPIn {
e.addPortAcceptRule(port, false, 2)
}
if e.cfg.IPv6 {
for _, port := range effectiveIPv6Ports(e.cfg.TCPIn, e.cfg.TCP6In) {
if !restricted[port] {
e.addPortAcceptRule(port, true, 10)
}
}
for _, port := range effectiveIPv6Ports(e.cfg.UDPIn, e.cfg.UDP6In) {
e.addPortAcceptRule(port, false, 10)
}
}
// Passive FTP range
if e.cfg.PassiveFTPStart > 0 && e.cfg.PassiveFTPEnd > 0 {
e.addPortRangeAcceptRule(e.cfg.PassiveFTPStart, e.cfg.PassiveFTPEnd, true, 2)
if e.cfg.IPv6 {
e.addPortRangeAcceptRule(e.cfg.PassiveFTPStart, e.cfg.PassiveFTPEnd, true, 10)
}
}
// Silent drop for commonly-scanned ports (no logging)
for _, port := range e.cfg.DropNoLog {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}}, // TCP
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(port))},
&expr.Verdict{Kind: expr.VerdictDrop},
},
})
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{17}}, // UDP
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(port))},
&expr.Verdict{Kind: expr.VerdictDrop},
},
})
}
// Rate-limited log for remaining dropped packets
if e.cfg.LogDropped {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: []expr.Any{
&expr.Limit{
Type: expr.LimitTypePkts,
Rate: uint64(max(e.cfg.LogRate, 1)),
Unit: expr.LimitTimeMinute,
Burst: 5,
},
&expr.Log{Key: 1, Data: []byte("CSM-DROP: ")},
},
})
}
// Default policy is DROP - anything not matched above is dropped
}
// createOutputChain builds the output filter chain.
// Restricts outbound to configured ports only (prevents C2 on non-standard ports).
func (e *Engine) createOutputChain() {
tcp6Out := effectiveIPv6Ports(e.cfg.TCPOut, e.cfg.TCP6Out)
udp6Out := effectiveIPv6Ports(e.cfg.UDPOut, e.cfg.UDP6Out)
if len(e.cfg.TCPOut) == 0 && len(e.cfg.UDPOut) == 0 && (!e.cfg.IPv6 || len(tcp6Out) == 0 && len(udp6Out) == 0) {
// No outbound restrictions configured - accept all
policy := nftables.ChainPolicyAccept
e.chainOut = e.conn.AddChain(&nftables.Chain{
Name: "output",
Table: e.table,
Type: nftables.ChainTypeFilter,
Hooknum: nftables.ChainHookOutput,
Priority: nftables.ChainPriorityFilter,
Policy: &policy,
})
e.addLocalPortLockRules()
return
}
// Outbound filtering enabled
policy := nftables.ChainPolicyDrop
e.chainOut = e.conn.AddChain(&nftables.Chain{
Name: "output",
Table: e.table,
Type: nftables.ChainTypeFilter,
Hooknum: nftables.ChainHookOutput,
Priority: nftables.ChainPriorityFilter,
Policy: &policy,
})
// Allow established/related outbound
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainOut,
Exprs: []expr.Any{
&expr.Ct{Register: 1, SourceRegister: false, Key: expr.CtKeySTATE},
&expr.Bitwise{
SourceRegister: 1,
DestRegister: 1,
Len: 4,
Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
Xor: binaryutil.NativeEndian.PutUint32(0),
},
&expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(0)},
&expr.Verdict{Kind: expr.VerdictAccept},
},
})
// Port locks decide new loopback connections before loopback is accepted.
e.addLocalPortLockRules()
// Allow loopback outbound
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainOut,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte("lo\x00")},
&expr.Verdict{Kind: expr.VerdictAccept},
},
})
if !e.cfg.IPv6 {
e.conn.AddRule(&nftables.Rule{Table: e.table, Chain: e.chainOut, Exprs: familyBypassRuleExprs(10)})
}
// Peer-scoped protocol allows: advertisements to each peer and to the
// protocol's multicast group.
_, protocolOut := protocolAllowRuleExprs(e.cfg.ProtocolAllow, e.cfg.IPv6)
for _, exprs := range protocolOut {
e.conn.AddRule(&nftables.Rule{Table: e.table, Chain: e.chainOut, Exprs: exprs})
}
// SMTP block - restrict outbound mail to allowed users only.
// resolveSMTPAllowedUIDs unconditionally includes root and mailnull
// (exim's queue runner); without mailnull on the allow list queued
// mail is silently dropped while CSM still reports healthy.
smtpBlocked := make(map[int]bool)
if e.cfg.SMTPBlock && len(e.cfg.SMTPPorts) > 0 {
allowedUIDs := resolveSMTPAllowedUIDs(e.cfg.SMTPAllowUsers)
for _, port := range e.cfg.SMTPPorts {
smtpBlocked[port] = true
// Accept from each allowed UID
for _, uid := range allowedUIDs {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainOut,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(port))},
&expr.Meta{Key: expr.MetaKeySKUID, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(uid)},
&expr.Verdict{Kind: expr.VerdictAccept},
},
})
}
// Drop SMTP from everyone else
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainOut,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(port))},
&expr.Verdict{Kind: expr.VerdictDrop},
},
})
}
}
// The operator restricted egress for another family (e.g. IPv6 only) but
// configured no IPv4 out-ports. Accept IPv4 egress wholesale rather than let
// the chain's DROP policy sever it. Placed after the SMTP guard above so
// smtp_block still governs IPv4 mail.
if e.cfg.restrictedOutputNeedsIPv4Bypass() {
e.conn.AddRule(&nftables.Rule{Table: e.table, Chain: e.chainOut, Exprs: familyBypassRuleExprs(2)})
}
// Destination-scoped outbound allows. Deliberately emitted AFTER the
// smtp_block guard above: nftables takes the first verdict, so the drop on
// each smtp port is already decided and no range here can bypass it. The
// config validator rejects an overlapping range as well; rule order must
// not be the only guard between a config key and outbound mail.
for _, r := range e.cfg.TCPOutAllow {
exprs := buildOutAllowExprs(r, e.cfg.IPv6)
if exprs == nil {
continue
}
e.conn.AddRule(&nftables.Rule{Table: e.table, Chain: e.chainOut, Exprs: exprs})
}
// Allow configured outbound TCP ports (skip SMTP-blocked ports - handled above)
for _, port := range e.cfg.TCPOut {
if smtpBlocked[port] {
continue
}
e.addOutboundPortRule(port, true, 2)
}
// Allow configured outbound UDP ports
for _, port := range e.cfg.UDPOut {
e.addOutboundPortRule(port, false, 2)
}
if e.cfg.IPv6 {
for _, port := range tcp6Out {
if !smtpBlocked[port] {
e.addOutboundPortRule(port, true, 10)
}
}
for _, port := range udp6Out {
e.addOutboundPortRule(port, false, 10)
}
}
// Allow only safe ICMP outbound (echo-reply + echo-request, block dest-unreachable)
// Blocking ICMP type 3 (dest-unreachable) prevents leaking closed port info to scanners
for _, icmpType := range []byte{0, 8} { // 0=echo-reply, 8=echo-request
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainOut,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{1}}, // ICMP
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{icmpType}},
&expr.Verdict{Kind: expr.VerdictAccept},
},
})
}
// ICMPv6 outbound - allow echo-reply (129) + echo-request (128) + ND (133-137)
if e.cfg.IPv6 {
for _, icmp6Type := range []byte{128, 129, 133, 134, 135, 136, 137} {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainOut,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{58}}, // ICMPv6
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{icmp6Type}},
&expr.Verdict{Kind: expr.VerdictAccept},
},
})
}
}
// REJECT outbound TCP with RST (faster failure than silent DROP)
// UDP still silently drops via chain policy.
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainOut,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}},
&expr.Reject{Type: 1}, // NFT_REJECT_TCP_RST
},
})
}
// --- Helper methods ---
// resolveIPSet returns the appropriate set and key bytes for an IP address.
// Falls back to IPv6 set if the IP is not IPv4.
func (e *Engine) resolveIPSet(ip string, set4, set6 *nftables.Set) (*nftables.Set, []byte, error) {
parsed := net.ParseIP(ip)
if parsed == nil {
return nil, nil, fmt.Errorf("invalid IP: %s", ip)
}
if ip4 := parsed.To4(); ip4 != nil {
return set4, ip4, nil
}
if set6 == nil {
return nil, nil, fmt.Errorf("IPv6 not enabled in firewall config: %s", ip)
}
return set6, parsed.To16(), nil
}
func canonicalFirewallIP(ip string) (string, error) {
parsed := net.ParseIP(ip)
if parsed == nil {
return "", fmt.Errorf("invalid IP: %s", ip)
}
return parsed.String(), nil
}
func canonicalIPKey(ip string) (string, bool) {
parsed := net.ParseIP(ip)
if parsed == nil {
return "", false
}
return parsed.String(), true
}
func stateIPKey(ip string) string {
if key, ok := canonicalIPKey(ip); ok {
return key
}
return ip
}
func sameIPString(a, b string) bool {
if a == b {
return true
}
aKey, aOK := canonicalIPKey(a)
if !aOK {
return false
}
bKey, bOK := canonicalIPKey(b)
return bOK && aKey == bKey
}
func preferLongerLivedBlock(current, candidate BlockedEntry) BlockedEntry {
if current.Unverified != candidate.Unverified {
if current.Unverified {
return current
}
return candidate
}
if current.ExpiresAt.IsZero() {
return current
}
if candidate.ExpiresAt.IsZero() || candidate.ExpiresAt.After(current.ExpiresAt) {
return candidate
}
return current
}
func preferLongerLivedAllow(current, candidate AllowedEntry) AllowedEntry {
if current.ExpiresAt.IsZero() {
return current
}
if candidate.ExpiresAt.IsZero() || candidate.ExpiresAt.After(current.ExpiresAt) {
return candidate
}
return current
}
func normalizeFirewallStateIPs(state *FirewallState) {
blockedPos := make(map[string]int, len(state.Blocked))
blocked := state.Blocked[:0]
for _, entry := range state.Blocked {
key, ok := canonicalIPKey(entry.IP)
if !ok {
blocked = append(blocked, entry)
continue
}
entry.IP = key
if i, exists := blockedPos[key]; exists {
// An automatic alias cannot erase an operator's ownership.
automatic := blocked[i].Automatic && entry.Automatic
blocked[i] = preferLongerLivedBlock(blocked[i], entry)
blocked[i].Automatic = automatic
continue
}
blockedPos[key] = len(blocked)
blocked = append(blocked, entry)
}
state.Blocked = blocked
type allowedKey struct {
ip string
source string
}
allowedPos := make(map[allowedKey]int, len(state.Allowed))
allowed := state.Allowed[:0]
for _, entry := range state.Allowed {
key, ok := canonicalIPKey(entry.IP)
if !ok {
allowed = append(allowed, entry)
continue
}
entry.IP = key
dedupKey := allowedKey{ip: key, source: entry.Source}
if i, exists := allowedPos[dedupKey]; exists {
allowed[i] = preferLongerLivedAllow(allowed[i], entry)
continue
}
allowedPos[dedupKey] = len(allowed)
allowed = append(allowed, entry)
}
state.Allowed = allowed
type portAllowKey struct {
ip string
port int
proto string
}
portAllowedPos := make(map[portAllowKey]struct{}, len(state.PortAllowed))
portAllowed := state.PortAllowed[:0]
for _, entry := range state.PortAllowed {
key, ok := canonicalIPKey(entry.IP)
if !ok {
portAllowed = append(portAllowed, entry)
continue
}
entry.IP = key
dedupKey := portAllowKey{ip: key, port: entry.Port, proto: entry.Proto}
if _, exists := portAllowedPos[dedupKey]; exists {
continue
}
portAllowedPos[dedupKey] = struct{}{}
portAllowed = append(portAllowed, entry)
}
state.PortAllowed = portAllowed
}
// addSetMatchRule adds an IPv4 source-IP set match rule on the input chain.
func (e *Engine) addSetMatchRule(set *nftables.Set, verdict expr.VerdictKind) {
exprs := buildSetMatchRuleExprs(set, verdict, 2, 12, 4)
if exprs == nil {
return
}
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: exprs,
})
}
// addSetMatchRuleV6 adds an IPv6 source-IP set match rule on the input chain.
func (e *Engine) addSetMatchRuleV6(set *nftables.Set, verdict expr.VerdictKind) {
exprs := buildSetMatchRuleExprs(set, verdict, 10, 8, 16)
if exprs == nil {
return
}
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: exprs,
})
}
func buildSetMatchRuleExprs(set *nftables.Set, verdict expr.VerdictKind, nfproto byte, sourceOffset, sourceLen uint32) []expr.Any {
if set == nil {
return nil
}
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{nfproto}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: sourceOffset, Len: sourceLen},
&expr.Lookup{SourceRegister: 1, SetName: set.Name, SetID: set.ID},
&expr.Verdict{Kind: verdict},
}
}
// ipv4NFProtoGuard returns the NFPROTO==IPV4 family guard prepended to every
// per-IP meter rule. The meters live in the dual-stack inet table and key on a
// raw IPv4 network-header source load (offset 12, len 4); on an IPv6 packet that
// offset reads bytes 4..7 of the 16-byte IPv6 source and writes a fake IPv4 key
// into the IPv4-typed meter set. The guard makes the meter rules run on IPv4
// only. Register 1 is reloaded by the expression that follows, so the guard does
// not disturb later register use.
func ipv4NFProtoGuard() []expr.Any {
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{2}}, // NFPROTO_IPV4
}
}
// ipv6NFProtoGuard is the NFPROTO==IPV6 sibling of ipv4NFProtoGuard, prepended
// to the IPv6 meter rules so they only run on IPv6 packets.
func ipv6NFProtoGuard() []expr.Any {
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{10}}, // NFPROTO_IPV6
}
}
func buildPortAllowExprs(pa PortAllowEntry, ipv6Enabled bool) []expr.Any {
parsed := net.ParseIP(pa.IP)
if parsed == nil || pa.Port <= 0 || pa.Port > 65535 {
return nil
}
proto := byte(6)
if pa.Proto == "udp" {
proto = 17
}
common := func(ip net.IP, offset, length uint32) []expr.Any {
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{proto}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: offset, Len: length},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ip},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(pa.Port))},
&expr.Verdict{Kind: expr.VerdictAccept},
}
}
if ip4 := parsed.To4(); ip4 != nil {
return append(ipv4NFProtoGuard(), common(ip4, 12, 4)...)
}
if !ipv6Enabled {
return nil
}
return append(ipv6NFProtoGuard(), common(parsed.To16(), 8, 16)...)
}
// ipv6Saddr64Key loads the IPv6 source address into register 1 and masks it to
// its /64 prefix. IPv6 hosts routinely own a whole /64 and rotate addresses
// within it for free, so the flood meters key on the /64 rather than the full
// /128, which a rotating source would trivially evade.
func ipv6Saddr64Key() []expr.Any {
return []expr.Any{
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 8, Len: 16},
&expr.Bitwise{
SourceRegister: 1, DestRegister: 1, Len: 16,
Mask: []byte{
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
},
Xor: make([]byte, 16),
},
}
}
// ipv6MeterSet builds a dynamic IPv6 meter set keyed on the /64 prefix, with the
// same one-minute element timeout as the v4 meters.
func (e *Engine) ipv6MeterSet(name string) *nftables.Set {
set := &nftables.Set{
Table: e.table, Name: name, KeyType: nftables.TypeIP6Addr,
Dynamic: true, HasTimeout: true, Timeout: time.Minute,
}
_ = e.conn.AddSet(set, nil)
return set
}
// synFloodRuleExprs6 is the IPv6 sibling of synFloodRuleExprs: same TCP-SYN
// match, keyed on the /64 source prefix into the v6 meter.
func (e *Engine) synFloodRuleExprs6() []expr.Any {
exprs := append(ipv6NFProtoGuard(), []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}}, // TCP
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 13, Len: 1},
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 1, Mask: []byte{0x12}, Xor: []byte{0x00}},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{0x02}}, // SYN only
}...)
exprs = append(exprs, ipv6Saddr64Key()...)
return append(exprs,
&expr.Dynset{
SrcRegKey: 1, SetName: e.meterSYN6.Name, SetID: e.meterSYN6.ID, Operation: 1,
Exprs: []expr.Any{
&expr.Limit{Type: expr.LimitTypePkts, Rate: 25, Unit: expr.LimitTimeSecond, Burst: 100, Over: true},
},
},
&expr.Verdict{Kind: expr.VerdictDrop},
)
}
// udpFloodRuleExprs6 is the IPv6 sibling of udpFloodRuleExprs.
func (e *Engine) udpFloodRuleExprs6(rate uint64, burst uint32) []expr.Any {
exprs := append(ipv6NFProtoGuard(), []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{17}}, // UDP
}...)
exprs = append(exprs, ipv6Saddr64Key()...)
return append(exprs,
&expr.Dynset{
SrcRegKey: 1, SetName: e.meterUDP6.Name, SetID: e.meterUDP6.ID, Operation: 1,
Exprs: []expr.Any{
&expr.Limit{Type: expr.LimitTypePkts, Rate: rate, Unit: expr.LimitTimeSecond, Burst: burst, Over: true},
},
},
&expr.Verdict{Kind: expr.VerdictDrop},
)
}
// connMeterRuleExprs6 is the IPv6 sibling of connMeterRuleExprs. The exempt
// lookup matches the full v6 source (exempt ranges are CIDRs); the meter keys on
// the /64 prefix.
func (e *Engine) connMeterRuleExprs6(rate uint64, burst uint32) []expr.Any {
core := []expr.Any{
&expr.Ct{Register: 1, SourceRegister: false, Key: expr.CtKeySTATE},
&expr.Bitwise{
SourceRegister: 1, DestRegister: 1, Len: 4,
Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW),
Xor: binaryutil.NativeEndian.PutUint32(0),
},
&expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(0)},
}
core = append(core, ipv6Saddr64Key()...)
core = append(core,
&expr.Dynset{
SrcRegKey: 1, SetName: e.meterConn6.Name, SetID: e.meterConn6.ID, Operation: 1,
Exprs: []expr.Any{
&expr.Limit{Type: expr.LimitTypePkts, Rate: rate, Unit: expr.LimitTimeMinute, Burst: burst, Over: true},
},
},
&expr.Verdict{Kind: expr.VerdictDrop},
)
if e.setDOSExempt6 != nil {
core = append(e.dosExemptV6Lookup(1), core...)
}
return append(ipv6NFProtoGuard(), core...)
}
// synFloodRuleExprs builds the expression list for the per-IP SYN flood
// rate-limit rule. The rule is intentionally NOT exempt-aware: SYN flood
// protection targets TCP half-open storms and is applied without a dos_exempt
// guard (unlike connection meters).
func (e *Engine) synFloodRuleExprs() []expr.Any {
return append(ipv4NFProtoGuard(), []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}}, // TCP
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 13, Len: 1},
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 1, Mask: []byte{0x12}, Xor: []byte{0x00}},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{0x02}}, // SYN only
// Load source IP for per-IP metering
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4},
&expr.Dynset{
SrcRegKey: 1,
SetName: e.meterSYN.Name,
SetID: e.meterSYN.ID,
Operation: 1, // NFT_DYNSET_OP_UPDATE
Exprs: []expr.Any{
&expr.Limit{Type: expr.LimitTypePkts, Rate: 25, Unit: expr.LimitTimeSecond, Burst: 100, Over: true},
},
},
&expr.Verdict{Kind: expr.VerdictDrop},
}...)
}
// udpFloodRuleExprs builds the expression list for the per-IP UDP flood
// rate-limit rule. Like synFloodRuleExprs, this rule carries no dos_exempt
// guard.
func (e *Engine) udpFloodRuleExprs(rate uint64, burst uint32) []expr.Any {
return append(ipv4NFProtoGuard(), []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{17}}, // UDP
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4},
&expr.Dynset{
SrcRegKey: 1,
SetName: e.meterUDP.Name,
SetID: e.meterUDP.ID,
Operation: 1,
Exprs: []expr.Any{
&expr.Limit{Type: expr.LimitTypePkts, Rate: rate, Unit: expr.LimitTimeSecond, Burst: burst, Over: true},
},
},
&expr.Verdict{Kind: expr.VerdictDrop},
}...)
}
// connMeterRuleExprs builds the expression list for the per-IP new-connection
// rate-limit rule. When setDOSExempt is non-nil, an inverted source-set lookup
// is prepended so sources in dos_exempt_nets bypass the rule entirely.
//
// Register-reuse safety: dosExemptV4Lookup writes saddr into reg 1. The
// following Ct and Payload exprs reload reg 1 with ct-state and saddr
// respectively, so the exempt check can never interfere with the meter logic.
func (e *Engine) connMeterRuleExprs(rate uint64, burst uint32) []expr.Any {
exprs := []expr.Any{
&expr.Ct{Register: 1, SourceRegister: false, Key: expr.CtKeySTATE},
&expr.Bitwise{
SourceRegister: 1, DestRegister: 1, Len: 4,
Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW),
Xor: binaryutil.NativeEndian.PutUint32(0),
},
&expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(0)},
// Load source IP for per-IP metering
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4},
&expr.Dynset{
SrcRegKey: 1,
SetName: e.meterConn.Name,
SetID: e.meterConn.ID,
Operation: 1,
Exprs: []expr.Any{
&expr.Limit{Type: expr.LimitTypePkts, Rate: rate, Unit: expr.LimitTimeMinute, Burst: burst, Over: true},
},
},
&expr.Verdict{Kind: expr.VerdictDrop},
}
if e.setDOSExempt != nil {
exprs = append(e.dosExemptV4Lookup(1), exprs...)
}
// The IPv4 family guard must precede the exempt lookup, which also loads an
// IPv4 network-header source and would otherwise misread IPv6 packets.
return append(ipv4NFProtoGuard(), exprs...)
}
// connlimitRuleExprs builds the expression list for the per-IP concurrent
// connection limit rule. When setDOSExempt is non-nil, an inverted source-set
// lookup is prepended so sources in dos_exempt_nets bypass the rule.
func (e *Engine) connlimitRuleExprs(limit uint32) []expr.Any {
exprs := []expr.Any{
&expr.Ct{Register: 1, SourceRegister: false, Key: expr.CtKeySTATE},
&expr.Bitwise{
SourceRegister: 1, DestRegister: 1, Len: 4,
Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW),
Xor: binaryutil.NativeEndian.PutUint32(0),
},
&expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(0)},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4},
&expr.Dynset{
SrcRegKey: 1,
SetName: e.meterConnlim.Name,
SetID: e.meterConnlim.ID,
Operation: 1,
Exprs: []expr.Any{
&expr.Connlimit{Count: limit, Flags: 1}, // 1 = over
},
},
&expr.Verdict{Kind: expr.VerdictDrop},
}
if e.setDOSExempt != nil {
exprs = append(e.dosExemptV4Lookup(1), exprs...)
}
// The IPv4 family guard must precede the exempt lookup, which also loads an
// IPv4 network-header source and would otherwise misread IPv6 packets.
return append(ipv4NFProtoGuard(), exprs...)
}
// addCFWhitelistRule adds an accept rule for Cloudflare IPs on TCP ports 80 and 443.
// Equivalent to: ip saddr @cf_whitelist tcp dport {80, 443} accept
func (e *Engine) addCFWhitelistRule(set *nftables.Set, ipv6 bool) {
if set == nil {
return
}
for _, port := range []uint16{80, 443} {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: cfWhitelistRuleExprs(set, ipv6, port),
})
}
}
// cfWhitelistRuleExprs builds the expression list for one Cloudflare whitelist
// accept rule: <family> saddr @set tcp dport <port> accept. The NFPROTO guard
// must precede the raw network-header source load: without it the v4 load also
// runs on IPv6 packets, where offset 12 reads bytes 4..7 of the IPv6 source,
// so a colliding IPv6 address would be accepted ahead of the block rules.
func cfWhitelistRuleExprs(set *nftables.Set, ipv6 bool, port uint16) []expr.Any {
var exprs []expr.Any
if ipv6 {
exprs = append(ipv6NFProtoGuard(),
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 8, Len: 16},
)
} else {
exprs = append(ipv4NFProtoGuard(),
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4},
)
}
return append(exprs,
&expr.Lookup{SourceRegister: 1, SetName: set.Name, SetID: set.ID},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}}, // TCP
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(port)},
&expr.Verdict{Kind: expr.VerdictAccept},
)
}
func buildFamilyPortRuleExprs(nfproto byte, port int, tcp bool) []expr.Any {
proto := byte(6) // TCP
if !tcp {
proto = 17 // UDP
}
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{nfproto}},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{proto}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(port))},
&expr.Verdict{Kind: expr.VerdictAccept},
}
}
// buildOutAllowExprs builds one destination-scoped outbound accept rule:
// nfproto guard, TCP, destination address, destination port range, accept.
//
// The neighbouring buildPortAllowExprs matches the SOURCE address because it
// serves the input chain. This rule serves the output chain and matches the
// DESTINATION, at offset 16 for IPv4 and 24 for IPv6.
//
// Returns nil when the rule cannot be expressed -- an unparseable destination,
// or an IPv6 destination while dual-stack filtering is off. Callers must emit
// nothing rather than a rule matching more than the operator asked for.
func buildOutAllowExprs(r OutAllowRule, ipv6Enabled bool) []expr.Any {
network, err := ParseOutAllowDst(r.Dst)
if err != nil {
return nil
}
if !validOutAllowRange(r) {
return nil
}
var (
exprs []expr.Any
addr net.IP
mask net.IPMask
offset uint32
length uint32
)
if ip4 := network.IP.To4(); ip4 != nil {
exprs, addr, mask, offset, length = ipv4NFProtoGuard(), ip4, network.Mask, 16, 4
} else {
if !ipv6Enabled {
return nil
}
exprs, addr, mask, offset, length = ipv6NFProtoGuard(), network.IP.To16(), network.Mask, 24, 16
}
exprs = append(exprs,
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}}, // TCP
)
// A zero-length prefix means every destination, and the honest encoding of
// that is no address match at all rather than a compare against nothing.
exprs = append(exprs, networkMatchExprs(addr, mask, offset, length)...)
return append(exprs,
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpGte, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(r.PortStart))},
&expr.Cmp{Op: expr.CmpOpLte, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(r.PortEnd))},
&expr.Verdict{Kind: expr.VerdictAccept},
)
}
// networkMatchExprs matches the network-header address at offset against
// addr/mask: a load, a mask when the prefix is shorter than the address, and
// a compare. A zero-length prefix matches every address and emits nothing.
func networkMatchExprs(addr net.IP, mask net.IPMask, offset, length uint32) []expr.Any {
ones, _ := mask.Size()
if ones == 0 {
return nil
}
exprs := []expr.Any{&expr.Payload{
DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: offset, Len: length,
}}
if ones < int(length)*8 {
exprs = append(exprs, &expr.Bitwise{
SourceRegister: 1, DestRegister: 1, Len: length,
Mask: mask, Xor: make([]byte, length),
})
}
return append(exprs, &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: addr.Mask(mask)})
}
// protocolAllowRuleExprs renders firewall.protocol_allow: per peer, an input
// accept on the source and an output accept on the destination, plus one
// output accept per family to each multicast group the protocol advertises
// to. Inbound advertisements to a group come from a peer and match the
// source rule. An unknown protocol, an unparseable or world-wide peer, and an
// IPv6 peer while dual-stack filtering is off render nothing: validation
// reports the first three, and the family bypass already passes IPv6.
func protocolAllowRuleExprs(rules []ProtocolAllowRule, ipv6Enabled bool) (input, output [][]expr.Any) {
type groupKey struct {
proto byte
family byte
}
grouped := make(map[groupKey]bool)
var groups [][]expr.Any
match := func(proto, family byte, addr net.IP, mask net.IPMask, offset, length uint32) []expr.Any {
exprs := []expr.Any{
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{family}},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{proto}},
}
exprs = append(exprs, networkMatchExprs(addr, mask, offset, length)...)
return append(exprs, &expr.Verdict{Kind: expr.VerdictAccept})
}
for _, r := range rules {
p, known := allowedProtocols[r.Protocol]
if !known {
continue
}
for _, peer := range r.Peers {
network, err := ParseOutAllowDst(peer)
if err != nil {
continue
}
if ones, _ := network.Mask.Size(); ones == 0 {
continue
}
family, addr, srcOffset, dstOffset, length := byte(2), network.IP.To4(), uint32(12), uint32(16), uint32(4)
if addr == nil {
if !ipv6Enabled {
continue
}
family, addr, srcOffset, dstOffset, length = 10, network.IP.To16(), 8, 24, 16
}
input = append(input, match(p.number, family, addr, network.Mask, srcOffset, length))
output = append(output, match(p.number, family, addr, network.Mask, dstOffset, length))
if grouped[groupKey{p.number, family}] {
continue
}
grouped[groupKey{p.number, family}] = true
for _, group := range p.groups {
if (group.To4() != nil) == (family == 2) {
bits := int(length) * 8
groups = append(groups, match(p.number, family, group, net.CIDRMask(bits, bits), dstOffset, length))
}
}
}
}
return input, append(output, groups...)
}
// inAllowRuleExprs renders firewall.tcp_in_allow: per source, an input accept
// on that source and the entry's destination port. InAllowRule.Networks
// decides which sources render; a rule wider than the operator asked for
// must never be the fallback.
func inAllowRuleExprs(rules []InAllowRule, ipv6Enabled bool) [][]expr.Any {
var out [][]expr.Any
for _, r := range rules {
for _, network := range r.Networks(ipv6Enabled) {
exprs, addr, offset, length := ipv4NFProtoGuard(), network.IP.To4(), uint32(12), uint32(4)
if addr == nil {
exprs, addr, offset, length = ipv6NFProtoGuard(), network.IP.To16(), 8, 16
}
exprs = append(exprs,
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}}, // TCP
)
exprs = append(exprs, networkMatchExprs(addr, network.Mask, offset, length)...)
out = append(out, append(exprs,
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(r.Port))},
&expr.Verdict{Kind: expr.VerdictAccept},
))
}
}
return out
}
// validOutAllowRange mirrors the config validator so a rule that slipped past
// validation still emits nothing rather than a range that matches everything.
func validOutAllowRange(r OutAllowRule) bool {
return r.PortStart >= 1 && r.PortStart <= 65535 &&
r.PortEnd >= 1 && r.PortEnd <= 65535 &&
r.PortStart <= r.PortEnd
}
func familyBypassRuleExprs(nfproto byte) []expr.Any {
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{nfproto}},
&expr.Verdict{Kind: expr.VerdictAccept},
}
}
func (e *Engine) addPortAcceptRule(port int, tcp bool, nfproto byte) {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: buildFamilyPortRuleExprs(nfproto, port, tcp),
})
}
func (e *Engine) addPortRangeAcceptRule(startPort, endPort int, tcp bool, nfproto byte) {
proto := byte(6)
if !tcp {
proto = 17
}
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainIn,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{nfproto}},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{proto}},
// Load dest port once, check range
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpGte, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(startPort))},
&expr.Cmp{Op: expr.CmpOpLte, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(endPort))},
&expr.Verdict{Kind: expr.VerdictAccept},
},
})
}
func (e *Engine) addOutboundPortRule(port int, tcp bool, nfproto byte) {
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainOut,
Exprs: buildFamilyPortRuleExprs(nfproto, port, tcp),
})
}
// --- Public API ---
// BlockIP adds an IP to the blocked set with optional timeout.
// timeout 0 = permanent block.
//
// Thin wrapper over BlockIPOutcome that discards the outcome. Existing
// callers that only need success/error semantics keep working; auto-
// response callers should use BlockIPOutcome so they can suppress local
// side effects (state mutation, AUTO-BLOCK alert) when the kernel was
// not actually touched.
func (e *Engine) BlockIP(ip string, reason string, timeout time.Duration) error {
_, err := e.BlockIPOutcome(ip, reason, timeout)
return err
}
// BlockIPOutcome is the AUTO-RESPONSE entry point. It performs the same
// guards, verdict-callback consultation, and dry-run gating as BlockIP,
// but additionally reports which path was taken via BlockOutcome so the
// caller can decide whether to record local state. See the BlockOutcome
// godoc for the meaning of each return value.
//
// Operator-initiated commands (csm firewall block, Web UI manual block) must
// call BlockIPForce instead, which skips the dry-run gate unconditionally.
func (e *Engine) BlockIPOutcome(ip string, reason string, timeout time.Duration) (outcome BlockOutcome, resultErr error) {
return e.BlockIPOutcomeWithFindingID(ip, reason, timeout, "")
}
// BlockIPOutcomeWithFindingID preserves the originating audit identity across
// every automatic outcome without changing the block policy or result.
func (e *Engine) BlockIPOutcomeWithFindingID(ip, reason string, timeout time.Duration, findingID string) (outcome BlockOutcome, resultErr error) {
return e.BlockIPRequest(ActionRequest{Operation: "block", Target: ip, Reason: reason, TTL: timeout, FindingID: findingID, Automatic: true, Actor: string(actionlog.Daemon)}, nil)
}
func (e *Engine) blockIPOutcomeRequest(req ActionRequest, budget *ScanAdmission) (outcome BlockOutcome, resultErr error) {
ip, reason, timeout, findingID := req.Target, req.Reason, req.TTL, req.FindingID
defer func() {
if e.shouldLegacyOutcome(resultErr) || (resultErr == nil && outcome != BlockOutcomeLive) {
recordBlockOutcome(ip, reason, timeout, outcome, resultErr, false, findingID)
}
}()
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return BlockOutcomeNoop, err
}
ip = canonical
// Soft-allow gate: an automatic block must never re-add an operator
// full-IP/port allow or a verified-bot range to blocked_ips. The nftables
// input chain drops @blocked_ips before it accepts operator allows, so the
// only safe fix is to keep the IP out of the blocked set. Checked before
// the local validation guards so a soft-allowed IP is skipped cleanly even
// when the deny limit is full. Operator `firewall deny` uses BlockIPForce,
// which never reaches this path, so an explicit deny still overrides a
// soft-allow.
if e.autoBlockSoftAllowed(ip) {
e.logAutoBlockSoftAllowed(ip)
return BlockOutcomeAllowlisted, nil
}
// Local safety checks always run before consulting the external callback.
// The callback can downgrade a block decision, but it cannot bypass
// malformed-IP, IPv6-disabled, infra-IP, or block-limit guards.
alreadyBlocked, err := e.validateBlockIP(ip, timeout, true)
if err != nil {
return BlockOutcomeNoop, err
}
if alreadyBlocked {
return BlockOutcomeNoop, nil
}
// Verdict gate: consult the panel after local validation and before the
// dry-run gate so that an "allow" verdict short-circuits everything.
// Fail-open: errors proceed with the default block. Nil callback skips
// the gate entirely.
if asker := e.verdictAskerFn(); asker != nil {
v, tenant, note, verr := asker(e.verdictContext(), ip, reason)
switch {
case verr != nil:
fmt.Fprintf(os.Stderr, "[%s] verdict callback failed for %s: %v - proceeding with default block\n",
time.Now().Format("2006-01-02 15:04:05"), ip, verr)
case v == "allow":
fmt.Fprintf(os.Stderr, "[%s] verdict callback returned allow for %s (tenant=%q note=%q) - not blocking\n",
time.Now().Format("2006-01-02 15:04:05"), ip, tenant, note)
return BlockOutcomeAllowed, nil
case tenant != "" || note != "":
fmt.Fprintf(os.Stderr, "[%s] verdict callback returned block for %s (tenant=%q note=%q) - proceeding with default block\n",
time.Now().Format("2006-01-02 15:04:05"), ip, tenant, note)
}
// "block" / empty / error -> proceed with default flow.
}
if e.autoBlockSoftAllowed(ip) {
e.logAutoBlockSoftAllowed(ip)
return BlockOutcomeAllowlisted, nil
}
// Dry-run gate: the daemon callback reads the current daemon config at
// call time so a SIGHUP takes effect without a daemon restart. Nil
// callback means live.
if e.autoResponseDryRunEnabled() {
fmt.Fprintf(os.Stderr, "[%s] auto_response dry_run: would have blocked %s (%s)\n",
time.Now().Format("2006-01-02 15:04:05"), ip, reason)
e.recordDryRunBlock(ip, reason, timeout)
return BlockOutcomeDryRun, nil
}
if e.autoBlockVerifiedRange(ip) {
e.logAutoBlockSoftAllowed(ip)
return BlockOutcomeAllowlisted, nil
}
lockedOutcome, err := e.blockIPLockedRequest(ip, reason, timeout, true, true, req, budget)
if err != nil {
return lockedOutcome, err
}
if lockedOutcome == BlockOutcomeAllowlisted {
e.logAutoBlockSoftAllowed(ip)
return BlockOutcomeAllowlisted, nil
}
return lockedOutcome, nil
}
func (e *Engine) autoResponseDryRunEnabled() bool {
e.mu.Lock()
fn := e.dryRunEnabled
e.mu.Unlock()
return fn != nil && fn()
}
// BlockIPForce adds an IP to the blocked set unconditionally, bypassing the
// auto_response.dry_run gate. Use this for operator-initiated commands (CLI,
// Web UI manual block) where the operator has explicitly decided to block.
func (e *Engine) BlockIPForce(ip string, reason string, timeout time.Duration) (resultErr error) {
defer func() {
if e.shouldLegacyOutcome(resultErr) {
recordBlockOutcome(ip, reason, timeout, BlockOutcomeLive, resultErr, true, "")
}
}()
return e.blockIPLocked(ip, reason, timeout, false)
}
// BlockIPForcePreserveLifetime is the Web UI timed-block path. The guard
// and replacement share the engine lock, including concurrent CLI promotions.
func (e *Engine) BlockIPForcePreserveLifetime(ip, reason string, timeout time.Duration) (resultErr error) {
defer func() {
if e.shouldLegacyOutcome(resultErr) {
recordBlockOutcome(ip, reason, timeout, BlockOutcomeLive, resultErr, true, "")
}
}()
_, err := e.blockIPLockedRequestGuarded(ip, reason, timeout, false, false,
ActionRequest{Operation: "block", Target: ip, Reason: reason, TTL: timeout}, nil, true)
return err
}
// PromoteToPermanentBlock upgrades an existing temporary block on ip to a
// permanent one: it clears the kernel timeout by deleting the timed element
// and re-adding it without a timeout, and zeroes ExpiresAt in state. The
// ordinary block path cannot do this during PermBlock escalation because it
// skips an already-blocked IP, so the kernel timeout would otherwise expire
// the block the operator wanted made permanent. Returns an error if the IP is
// not currently blocked (nothing to promote).
func (e *Engine) PromoteToPermanentBlock(ip, reason string) (resultErr error) {
return e.PromoteToPermanentBlockWithFindingID(ip, reason, "")
}
// PromoteToPermanentBlockWithFindingID ties escalation to its triggering
// observation while preserving the existing permanent-block transaction.
func (e *Engine) PromoteToPermanentBlockWithFindingID(ip, reason, findingID string) (resultErr error) {
source := InferProvenance("permblock", reason)
defer func() {
if e.shouldLegacyOutcome(resultErr) {
recordFirewallFindingResult("permblock", ip, reason, source, 0, actionlog.Applied, resultErr, findingID)
}
}()
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return err
}
ip = canonical
// This path deliberately bypasses blockIPLocked, so it does not inherit
// that path's address guard. An entry that reached the set before the
// guard existed must not be made permanent here.
if isUnblockableAddress(ip) {
return ipProtectedErrorf("refusing to block non-routable address: %s", ip)
}
e.mu.Lock()
defer e.mu.Unlock()
targetSet, key, err := e.resolveIPSet(ip, e.setBlocked, e.setBlocked6)
if err != nil {
return err
}
priorState := e.loadStateFile()
entry := BlockedEntry{IP: ip, Reason: reason, Source: SourceSystem, BlockedAt: time.Now()}
found := false
wasTemporary := false
for _, b := range priorState.Blocked {
if sameIPString(b.IP, ip) && !b.Unverified {
if b.ActionID != "" {
return fmt.Errorf("cannot promote %s: %w", ip, ErrAdmissionOwnsBlock)
}
found = true
wasTemporary = !b.ExpiresAt.IsZero()
entry.BlockedAt = b.BlockedAt
if b.Source != "" {
entry.Source = b.Source
}
break
}
}
if !found {
return fmt.Errorf("cannot promote %s: not currently blocked", ip)
}
if e.cfg != nil && wasTemporary && e.cfg.DenyIPLimit > 0 {
perm, _, _, ok := e.livePermTempCountsLocked(priorState)
if !ok {
perm = countPermanentBlockedEntries(priorState)
}
if perm >= e.cfg.DenyIPLimit {
return fmt.Errorf("permanent deny limit reached (%d)", e.cfg.DenyIPLimit)
}
}
// ExpiresAt left zero: permanent.
nextState := copyFirewallState(priorState)
upsertBlockedEntryInState(&nextState, entry)
if e.lifecycle != nil {
return e.runDurableLocked(ActionRequest{Operation: "promote", Target: ip, Reason: reason, FindingID: findingID, Automatic: findingID != ""}, nil, nextState)
}
if err := e.persistFirewallIntent(priorState, nextState); err != nil {
return fmt.Errorf("persisting permanent promotion for %s: %w", ip, err)
}
// Delete the timed element and re-add it without a timeout in one
// transaction, so the address is never unblocked in between.
if err := e.conn.SetDeleteElements(targetSet, []nftables.SetElement{{Key: key}}); err != nil {
if restoreErr := e.restoreBlockStateAfterFailureLocked(priorState, ip); restoreErr != nil {
return fmt.Errorf("promoting %s: delete timed element: %w (state restore failed: %v)", ip, err, restoreErr)
}
return fmt.Errorf("promoting %s: delete timed element: %w", ip, err)
}
if err := e.conn.SetAddElements(targetSet, []nftables.SetElement{{Key: key}}); err != nil {
if restoreErr := e.restoreBlockStateAfterFailureLocked(priorState, ip); restoreErr != nil {
return fmt.Errorf("promoting %s: re-add permanent element: %w (state restore failed: %v)", ip, err, restoreErr)
}
return fmt.Errorf("promoting %s: re-add permanent element: %w", ip, err)
}
if err := e.conn.Flush(); err != nil {
if restoreErr := e.restoreBlockStateAfterFailureLocked(priorState, ip); restoreErr != nil {
return fmt.Errorf("promoting %s: flush: %w (state restore failed: %v)", ip, err, restoreErr)
}
return fmt.Errorf("promoting %s: flush: %w", ip, err)
}
source = entry.Source
e.legacyFileAuditLocked("permblock", ip, reason, entry.Source, 0)
return nil
}
// recordDryRunBlock persists a dry-run record through the daemon-installed
// recorder so operators can review the count via /api/v1/status.
// No-op when no recorder is installed.
func (e *Engine) recordDryRunBlock(ip, reason string, timeout time.Duration) {
e.mu.Lock()
recorder := e.dryRunRecorder
e.mu.Unlock()
if recorder != nil {
recorder(ip, reason, timeout)
}
}
// blockIPLocked is the real implementation called by both BlockIP and BlockIPForce.
func (e *Engine) blockIPLocked(ip string, reason string, timeout time.Duration, skipExisting bool) error {
_, err := e.blockIPLockedMaybeSoftAllowed(ip, reason, timeout, skipExisting, false)
return err
}
func (e *Engine) blockIPLockedMaybeSoftAllowed(ip string, reason string, timeout time.Duration, skipExisting bool, enforceSoftAllow bool) (BlockOutcome, error) {
return e.blockIPLockedRequest(ip, reason, timeout, skipExisting, enforceSoftAllow, ActionRequest{Operation: "block", Target: ip, Reason: reason, TTL: timeout, Automatic: enforceSoftAllow}, nil)
}
func (e *Engine) blockIPLockedRequest(ip string, reason string, timeout time.Duration, skipExisting bool, enforceSoftAllow bool, req ActionRequest, budget *ScanAdmission) (BlockOutcome, error) {
return e.blockIPLockedRequestGuarded(ip, reason, timeout, skipExisting, enforceSoftAllow, req, budget, false)
}
func (e *Engine) blockIPLockedRequestGuarded(ip string, reason string, timeout time.Duration, skipExisting bool, enforceSoftAllow bool, req ActionRequest, budget *ScanAdmission, preserveLifetime bool) (BlockOutcome, error) {
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return BlockOutcomeNoop, err
}
ip = canonical
e.mu.Lock()
defer e.mu.Unlock()
return e.blockIPRequestLocked(ip, reason, timeout, skipExisting, enforceSoftAllow, req, budget, preserveLifetime, time.Time{})
}
// blockIPRequestLocked runs with e.mu held, including snapshot-checked undo.
func (e *Engine) blockIPRequestLocked(ip string, reason string, timeout time.Duration, skipExisting bool, enforceSoftAllow bool, req ActionRequest, budget *ScanAdmission, preserveLifetime bool, expiresAt time.Time) (BlockOutcome, error) {
req.Target = ip
req = normalizeActionRequest(req)
if found, replayErr := e.replayActionLocked(req); found || replayErr != nil {
return BlockOutcomeNoop, replayErr
}
if readyErr := e.lifecycleReadyLocked(); readyErr != nil {
return BlockOutcomeNoop, readyErr
}
priorState, policyState, readErr := e.blockPlanningStatesLocked()
if readErr != nil {
return BlockOutcomeNoop, readErr
}
if preserveLifetime && timeout > 0 {
if e.stateReadErr != nil {
return BlockOutcomeNoop, e.stateReadErr
}
// A legacy lifetime or failed kernel probe must not mask the address
// refusal. Capacity checks still follow lifetime checks for usable IPs.
if isUnblockableAddress(ip) {
return BlockOutcomeNoop, ipProtectedErrorf("refusing to block non-routable address: %s", ip)
}
if entry, found := blockedStateEntry(policyState, ip); found {
if entry.ExpiresAt.IsZero() {
return BlockOutcomeNoop, ErrPermanentBlock
}
if entry.ExpiresAt.Sub(e.leaseNowLocked()) > timeout {
return BlockOutcomeNoop, ErrLongerBlock
}
} else {
set, key, err := e.resolveIPSet(ip, e.setBlocked, e.setBlocked6)
if err != nil {
return BlockOutcomeNoop, err
}
live, temporary, classified := e.liveBlockElementKindLocked(set, key)
if !classified {
return BlockOutcomeNoop, fmt.Errorf("cannot verify existing block lifetime for %s", ip)
}
if live && !temporary {
return BlockOutcomeNoop, ErrPermanentBlock
}
}
}
// Safety, capacity, eviction and the resulting state share one snapshot.
if enforceSoftAllow && (ipSetIndexContains(e.allowedIPIndex, ip) || ipSetIndexContains(e.portAllowedIndex, ip)) {
return BlockOutcomeAllowlisted, nil
}
targetSet, key, alreadyBlocked, evictTempIP, err := e.blockIPTargetFromStates(ip, timeout, skipExisting, policyState, priorState)
if err != nil {
return BlockOutcomeNoop, err
}
if alreadyBlocked {
return BlockOutcomeNoop, nil
}
// Persist to state BEFORE adding the kernel element. state.json is the
// seed source on the next Apply, so a crash after this point but before the
// kernel add leaves a durable record that Apply re-applies. The previous
// ordering (kernel first, then state) left a window where a process kill
// produced a permanent kernel block with no state row: it never expired
// (timeout 0) yet a later state-seeded Apply would silently drop it.
// Zero ExpiresAt means permanent.
at := e.leaseNowLocked()
entry := BlockedEntry{
IP: ip,
Reason: reason,
Source: InferProvenance("block", reason),
BlockedAt: at,
}
if timeout > 0 {
entry.ExpiresAt = at.Add(timeout)
}
if !expiresAt.IsZero() {
entry.ExpiresAt = expiresAt
}
expiresAt = entry.ExpiresAt
entry.Automatic = req.Automatic && !expiresAt.IsZero()
entry.LoginSpray = req.LoginSpray && entry.Automatic
var evictSet *nftables.Set
var evictKey []byte
if evictTempIP != "" {
evictSet, evictKey, err = e.resolveIPSet(evictTempIP, e.setBlocked, e.setBlocked6)
if err != nil {
return BlockOutcomeNoop, fmt.Errorf("resolving temp block eviction target %s: %w", evictTempIP, err)
}
}
// A forced block over an address that is already blocked changes the
// timeout (deny over an auto-block, tempban over a permanent deny).
// nf_tables treats NEWSETELEM without NLM_F_EXCL on an existing key as
// an acknowledged no-op that keeps the old timeout, so the element must be
// deleted and re-added in the same batch, as PromoteToPermanentBlock does;
// otherwise state.json and every CSM surface disagree with the kernel.
replaceExisting := !skipExisting && firewallStateHasBlocked(policyState, ip)
if !skipExisting && !replaceExisting {
// State is normally written before the kernel batch, but older builds
// and partial recovery can leave a live kernel element without a state
// row. A plain add would be acknowledged without replacing its timeout,
// recreating the state/kernel drift this path is meant to repair.
live, liveErr := e.isBlockedLiveLocked(ip)
if liveErr != nil {
return BlockOutcomeNoop, fmt.Errorf("checking existing block for %s: %w", ip, liveErr)
}
replaceExisting = live
}
nextState := copyFirewallState(priorState)
if evictTempIP != "" {
removeBlockedIPFromState(&nextState, evictTempIP)
}
upsertBlockedEntryInState(&nextState, entry)
// A new deadline may use a forward-ratcheted lease. Save its clock
// before either intent so a crash cannot restore more than authorized.
if !expiresAt.IsZero() {
if err := e.saveLeaseLocked(); err != nil {
return BlockOutcomeNoop, fmt.Errorf("saving block lease: %w", err)
}
}
if e.lifecycle != nil {
plan := FirewallAction{Request: req, Budget: budget, After: nextState}
if evictTempIP != "" {
for _, row := range priorState.Blocked {
if row.IP == evictTempIP {
plan.Members = []ActionMember{{Request: req, Budget: budget, Victims: []BlockedEntry{row}}}
break
}
}
}
if _, err := e.runDurablePlanLocked(plan); err != nil {
if errors.Is(err, ErrActionAuditPending) {
return BlockOutcomeLive, err
}
return BlockOutcomeNoop, err
}
return BlockOutcomeLive, nil
}
if err := e.persistFirewallIntent(priorState, nextState); err != nil {
return BlockOutcomeNoop, fmt.Errorf("persisting block for %s: %w", ip, err)
}
if !expiresAt.IsZero() {
var ok bool
if timeout, ok = blockTimeout(expiresAt, e.leaseNowLocked()); !ok {
restoreErr := e.restoreBlockStateAfterFailureLocked(priorState, ip)
return BlockOutcomeNoop, errors.Join(errors.New("restored block expired before send"), restoreErr)
}
}
elem := []nftables.SetElement{{Key: key, Timeout: timeout}}
if replaceExisting {
if err := e.conn.SetDeleteElements(targetSet, []nftables.SetElement{{Key: key}}); err != nil {
if restoreErr := e.restoreBlockStateAfterFailureLocked(priorState, ip); restoreErr != nil {
return BlockOutcomeNoop, fmt.Errorf("replacing blocked element: %w (state restore failed: %v)", err, restoreErr)
}
return BlockOutcomeNoop, fmt.Errorf("replacing blocked element: %w", err)
}
}
if err := e.conn.SetAddElements(targetSet, elem); err != nil {
if restoreErr := e.restoreBlockStateAfterFailureLocked(priorState, ip); restoreErr != nil {
return BlockOutcomeNoop, fmt.Errorf("adding to blocked set: %w (state restore failed: %v)", err, restoreErr)
}
return BlockOutcomeNoop, fmt.Errorf("adding to blocked set: %w", err)
}
if evictTempIP != "" {
if err := e.conn.SetDeleteElements(evictSet, []nftables.SetElement{{Key: evictKey}}); err != nil {
if restoreErr := e.restoreBlockStateAfterFailureLocked(priorState, ip); restoreErr != nil {
return BlockOutcomeNoop, fmt.Errorf("evicting temp block %s: %w (state restore failed: %v)", evictTempIP, err, restoreErr)
}
return BlockOutcomeNoop, fmt.Errorf("evicting temp block %s: %w", evictTempIP, err)
}
}
if err := e.conn.Flush(); err != nil {
if (replaceExisting || evictSet != nil) && isNftNotFound(err) {
// state.json said blocked but the kernel had already expired the
// replacement or eviction element, so the whole batch was rejected
// on that delete. Retry without whichever stale delete failed.
err = e.retryBlockAddAfterMissingElement(targetSet, elem, evictSet, evictKey, replaceExisting, expiresAt)
}
if err != nil {
if restoreErr := e.restoreBlockStateAfterFailureLocked(priorState, ip); restoreErr != nil {
return BlockOutcomeNoop, fmt.Errorf("flushing: %w (state restore failed: %v)", err, restoreErr)
}
return BlockOutcomeNoop, fmt.Errorf("flushing: %w", err)
}
}
if evictTempIP != "" {
e.legacyActionAuditLocked("evict_temp", evictTempIP, evictTempReason, SourceSystem, 0)
}
e.legacyFileAuditLocked("block", ip, reason, entry.Source, timeout)
return BlockOutcomeLive, nil
}
// retryBlockAddAfterMissingElement re-queues a block batch without the
// delete of an element the kernel no longer holds. Caller must hold e.mu.
func (e *Engine) retryBlockAddAfterMissingElement(targetSet *nftables.Set, elem []nftables.SetElement, evictSet *nftables.Set, evictKey []byte, replaceExisting bool, expiresAt time.Time) error {
refresh := func() error {
if !expiresAt.IsZero() {
timeout, ok := blockTimeout(expiresAt, e.leaseNowLocked())
if !ok {
return errors.New("restored block expired before retry")
}
elem[0].Timeout = timeout
}
return nil
}
queueAdd := func() error {
if err := refresh(); err != nil {
return err
}
if err := e.conn.SetAddElements(targetSet, elem); err != nil {
return fmt.Errorf("retry adding to blocked set: %w", err)
}
return nil
}
flush := func(stage string) error {
if err := e.conn.Flush(); err != nil {
return fmt.Errorf("%s: %w", stage, err)
}
return nil
}
if !replaceExisting {
// The only delete in the rejected batch was the expired eviction
// victim. Re-add the new target without repeating that stale delete.
if err := queueAdd(); err != nil {
return err
}
return flush("retry flushing block without eviction")
}
// First assume the replacement target expired but the eviction victim is
// still live. Adding the target and evicting the victim remains atomic.
if err := queueAdd(); err != nil {
return err
}
if evictSet != nil {
if err := e.conn.SetDeleteElements(evictSet, []nftables.SetElement{{Key: evictKey}}); err != nil {
return fmt.Errorf("retry evicting temp block: %w", err)
}
}
if err := flush("retry flushing block after missing replacement"); err == nil {
return nil
} else if !isNftNotFound(err) {
return err
}
// The eviction victim was the stale element instead. Replace the target
// without repeating the eviction delete, keeping a live target blocked for
// the whole atomic transaction. Validate the lease before queueing a delete.
if err := refresh(); err != nil {
return err
}
if err := e.conn.SetDeleteElements(targetSet, []nftables.SetElement{{Key: elem[0].Key}}); err != nil {
return fmt.Errorf("retry deleting replacement target: %w", err)
}
if err := e.conn.SetAddElements(targetSet, elem); err != nil {
return fmt.Errorf("retry adding to blocked set: %w", err)
}
if err := flush("retry flushing block without eviction"); err == nil {
return nil
} else if !isNftNotFound(err) {
return err
}
// Both deletes raced expired elements. No live target remains to preserve,
// so an add-only transaction is the final bounded retry.
if err := queueAdd(); err != nil {
return err
}
return flush("retry flushing add-only block")
}
func (e *Engine) validateBlockIP(ip string, timeout time.Duration, skipExisting bool) (bool, error) {
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return false, err
}
ip = canonical
e.mu.Lock()
defer e.mu.Unlock()
_, _, alreadyBlocked, _, err := e.blockIPTarget(ip, timeout, skipExisting)
return alreadyBlocked, err
}
// blockPlanningStatesLocked keeps saved rows separate from active policy
// rows, both from one read. Expiry must not erase victim identity or make
// an expired row occupy a policy slot.
func (e *Engine) blockPlanningStatesLocked() (FirewallState, FirewallState, error) {
if e.lifecycle == nil {
state := e.loadStateFile()
return state, state, nil
}
state, _, err := e.readCommittedStateLocked()
if err != nil {
return FirewallState{}, FirewallState{}, err
}
e.installCommittedCache(state)
return state, copyFirewallState(*e.stateCache), nil
}
//nolint:unparam // keeps the result shape of blockIPTargetFromStates, which it forwards
func (e *Engine) blockIPTarget(ip string, timeout time.Duration, skipExisting bool) (*nftables.Set, []byte, bool, string, error) {
state, policyState, err := e.blockPlanningStatesLocked()
if err != nil {
return nil, nil, false, "", err
}
return e.blockIPTargetFromStates(ip, timeout, skipExisting, policyState, state)
}
func (e *Engine) blockIPTargetFromStates(ip string, timeout time.Duration, skipExisting bool, st, victimState FirewallState) (*nftables.Set, []byte, bool, string, error) {
parsed := net.ParseIP(ip)
if parsed == nil {
return nil, nil, false, "", fmt.Errorf("invalid IP: %s", ip)
}
ip = parsed.String()
if err := e.blockProtectionLocked(parsed, ip); err != nil {
return nil, nil, false, "", err
}
targetSet, key, err := e.resolveIPSet(ip, e.setBlocked, e.setBlocked6)
if err != nil {
return nil, nil, false, "", err
}
cachedBlockMissingLive := false
if skipExisting && firewallStateHasBlocked(st, ip) {
// An automatic block never replaces a row an admission action
// wrote, even one whose element is gone or unproven: the owner
// retires it.
if row, _ := blockedStateEntry(st, ip); row.ActionID != "" {
return targetSet, key, true, "", nil
}
liveBlocked, liveErr := e.isBlockedLiveLocked(ip)
if liveErr != nil || liveBlocked {
// Treat a probe error as "still blocked" so we never demote a
// cached block on transient netlink trouble. Returning nil
// here is intentional and the conservative posture.
return targetSet, key, true, "", nil //nolint:nilerr // intentional fail-safe on netlink probe error
}
cachedBlockMissingLive = true
}
// Enforce deny IP limits. Prefer counts from the live nft set
// so an entry the kernel already expired no longer counts
// against the cap. Fall back to the cached state.json count
// (existing behaviour) if the live query is unavailable.
if e.cfg.DenyIPLimit > 0 || e.cfg.DenyTempIPLimit > 0 {
// A row of a family the firewall does not filter holds no element:
// it fills no slot and no block can replace it.
st, victimState = e.filteredRowsLocked(st), e.filteredRowsLocked(victimState)
var perm, temp int
// The addresses the count found in the kernel, when it read them.
var live map[string]bool
if timeout > 0 && !skipExisting && e.lifecycle == nil && hasAdmissionRows(st) {
// Count ownership and occupancy from one image, including replacement.
temp, live = e.operatorTempCountLocked(st, ip)
} else {
var ok bool
perm, temp, live, ok = e.livePermTempCountsLocked(st)
stateEntry, replacingStateEntry := blockedStateEntry(st, ip)
if ok && !skipExisting {
if replacingStateEntry && !stateEntry.Unverified {
// Replacing an element already occupying a limit slot is not a new
// block. Only subtract it when the kernel confirms the state entry
// is still live; stale state must not hide some other live element.
if live, liveErr := e.isBlockedLiveLocked(ip); liveErr == nil && live {
if stateEntry.ExpiresAt.IsZero() && perm > 0 {
perm--
} else if !stateEntry.ExpiresAt.IsZero() && temp > 0 {
temp--
}
}
} else if live, temporary, classified := e.liveBlockElementKindLocked(targetSet, key); classified && live {
// Recovery may leave a kernel element with no state row. It still
// occupies one of the counted slots, so replacing it must release
// that same slot before enforcing the limit.
if temporary && temp > 0 {
temp--
} else if !temporary && perm > 0 {
perm--
}
}
}
if !ok {
excludeIP := ""
if cachedBlockMissingLive || (!skipExisting && replacingStateEntry) {
excludeIP = ip
}
perm, temp = blockedStatePermTempCounts(st, excludeIP)
}
}
if timeout == 0 && e.cfg.DenyIPLimit > 0 && perm >= e.cfg.DenyIPLimit {
return nil, nil, false, "", fmt.Errorf("permanent deny limit reached (%d)", e.cfg.DenyIPLimit)
}
if timeout > 0 && e.cfg.DenyTempIPLimit > 0 && temp >= e.cfg.DenyTempIPLimit {
// Validation must stay read-only because verdict and dry-run gates run
// after it. The live block path applies this eviction target.
victim, ok := e.evictionVictimLocked(heldRows(victimState, live), ip, skipExisting)
if !ok {
return nil, nil, false, "", fmt.Errorf("temporary deny limit reached (%d) and no temp entry to evict", e.cfg.DenyTempIPLimit)
}
return targetSet, key, false, victim, nil
}
}
return targetSet, key, false, "", nil
}
// BlockProtected reports whether no block may name ip: infra, the current
// address of an infra hostname, the host's own interfaces or an address that
// is never a packet's source. Callers about to act on an address in another
// way ask it so they skip what a block would refuse.
func (e *Engine) BlockProtected(ip string) bool {
parsed := net.ParseIP(strings.TrimSpace(ip))
if parsed == nil {
return false
}
e.mu.Lock()
defer e.mu.Unlock()
return e.blockProtectionLocked(parsed, parsed.String()) != nil
}
// blockProtectionLocked refuses an address no block may name: infra,
// the host's own interfaces and addresses that are never a packet's source.
func (e *Engine) blockProtectionLocked(parsed net.IP, ip string) error {
// SAFETY: never block infra IPs - prevents admin lockout.
// Runs before resolveIPSet so that an IPv6-disabled config (set6 == nil)
// cannot bypass the infra guard when the caller passes the canonical
// IPv6 form of a listed infra address.
for _, cidr := range e.cfg.InfraIPs {
_, network, cidrErr := net.ParseCIDR(cidr)
if cidrErr != nil {
if infraIP := net.ParseIP(cidr); infraIP != nil && infraIP.String() == parsed.String() {
return ipProtectedErrorf("refusing to block infra IP: %s", ip)
}
continue
}
if network.Contains(parsed) {
return ipProtectedErrorf("refusing to block infra IP: %s (in %s)", ip, cidr)
}
}
// Hostnames in cfg.InfraIPs are resolved by the DynDNS loop and
// pushed in via UpdateInfraResolved. Without this check a hostname
// listed as infra would only be honoured when the operator also
// pinned the IP, so a moving panel IP would silently drop out of
// the lockout guard.
if host, ok := e.infraIPResolvedHostLocked(ip); ok {
return ipProtectedErrorf("refusing to block infra IP: %s (resolved from %s)", ip, host)
}
// Daemon's own interface addresses are always off-limits. Without
// this guard a stray request from the host to itself (cron, panel
// callback, internal probe) could trigger an auto-block that
// firewalls every customer hosted on the same IP.
if e.isLocalAddrLocked(ip) {
return ipProtectedErrorf("refusing to block local host IP: %s (own interface address)", ip)
}
// Loopback and link-local are excluded from the interface set above, so
// they need their own refusal: the panel's own proxied requests appear as
// loopback, and a WAF denial count for it must not become a block.
if isUnblockableAddress(ip) {
return ipProtectedErrorf("refusing to block non-routable address: %s", ip)
}
return nil
}
// evictTempReason is the reason every eviction record gives.
const evictTempReason = "temp deny limit reached; evicted soonest-expiring entry"
// soonestExpiringTempIP returns the IP of the temporary block closest to
// expiry, skipping permanent blocks, unverified rows and excludeIP. Pure
// helper so the eviction policy is unit-testable without nftables. An
// unverified row is never an automatic victim: its effect is unproven and
// only the owner's retirement may remove it.
func soonestExpiringTempIP(st FirewallState, excludeIP string) (string, bool) {
var best string
var bestExp time.Time
found := false
for _, b := range st.Blocked {
if sameIPString(b.IP, excludeIP) || b.ExpiresAt.IsZero() || b.Unverified {
continue
}
if !found || b.ExpiresAt.Before(bestExp) {
best, bestExp, found = b.IP, b.ExpiresAt, true
}
}
return best, found
}
func firewallStateHasBlocked(state FirewallState, ip string) bool {
_, ok := blockedStateEntry(state, ip)
return ok
}
func hasAdmissionRows(state FirewallState) bool {
for _, row := range state.Blocked {
if row.ActionID != "" {
return true
}
}
return false
}
// operatorTempCountLocked counts non-admission occupants from one complete
// live image and returns the addresses in it. Partial dumps fall back as
// a whole to lease-pruned policy and return no addresses.
// The aggregate liveBlockCounts test seam cannot describe ownership.
func (e *Engine) operatorTempCountLocked(state FirewallState, excludeIP string) (int, map[string]bool) {
var conn *nftables.Conn
readable := e.liveBlockedDump != nil
if !readable {
var err error
conn, err = newLifecycleConn(e)
readable = err == nil
}
live := make(map[string]nftables.SetElement)
gotAny := false
for _, set := range []*nftables.Set{e.setBlocked, e.setBlocked6} {
if !readable || set == nil || (set == e.setBlocked6 && !e.cfg.IPv6) {
continue
}
elements, err := e.blockSnapshotElementsLocked(conn, set)
if err != nil {
readable = false
break
}
gotAny = true
for _, el := range elements {
if ip, valid := blockSetIP(set, el.Key); valid {
live[ip] = el
}
}
}
// A lease may end while the last family is read.
at := e.leaseNowLocked()
state = copyFirewallState(state)
state.Blocked, _ = pruneBlocked(state.Blocked, at)
owners := make(map[string]string)
for _, row := range state.Blocked {
ip := stateIPKey(row.IP)
id := row.ActionID
if !row.ExpiresAt.After(at) {
id = ""
}
if prior, exists := owners[ip]; exists && prior != id {
id = ""
}
owners[ip] = id
}
// Use the same permanent-over-timed precedence as the policy view.
normalizeFirewallStateIPs(&state)
types := blockedStateTempByIP(state)
if readable && gotAny {
n := 0
held := make(map[string]bool, len(live))
for ip, el := range live {
held[ip] = true
if !sameIPString(ip, excludeIP) && liveBlockTemporary(el, ip, types) && owners[ip] == "" {
n++
}
}
return n, held
}
var other []BlockedEntry
for _, row := range state.Blocked {
if owners[stateIPKey(row.IP)] == "" {
other = append(other, row)
}
}
state.Blocked = other
_, n := blockedStatePermTempCounts(state, excludeIP)
return n, nil
}
func blockedStateEntry(state FirewallState, ip string) (BlockedEntry, bool) {
for _, entry := range state.Blocked {
if sameIPString(entry.IP, ip) {
return entry, true
}
}
return BlockedEntry{}, false
}
func countPermanentBlockedEntries(state FirewallState) int {
perm, _ := blockedStatePermTempCounts(state, "")
return perm
}
func blockedStatePermTempCounts(state FirewallState, excludeIP string) (perm, temp int) {
seen := make(map[string]bool, len(state.Blocked))
for _, entry := range state.Blocked {
key, ok := canonicalIPKey(entry.IP)
if !ok || entry.Unverified {
continue
}
if excludeIP != "" && sameIPString(key, excludeIP) {
continue
}
isTemp := !entry.ExpiresAt.IsZero()
if priorTemp, exists := seen[key]; exists {
if priorTemp && !isTemp {
seen[key] = false
}
continue
}
seen[key] = isTemp
}
for _, isTemp := range seen {
if isTemp {
temp++
} else {
perm++
}
}
return perm, temp
}
// UnblockIP removes an IP from the blocked set and state.
func (e *Engine) UnblockIP(ip string) error {
return e.unblockIP(ip, ReleaseOperator)
}
// UnbanIP is UnblockIP for a self-service unban: the release hook is told
// ReleaseUnban, so the address keeps its failure history.
func (e *Engine) UnbanIP(ip string) error {
return e.unblockIP(ip, ReleaseUnban)
}
func (e *Engine) unblockIP(ip string, kind ReleaseKind) (resultErr error) {
defer func() { e.legacyFirewallFailure("unblock", ip, "", "", 0, resultErr) }()
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return err
}
ip = canonical
defer func() { e.released(ip, kind, resultErr) }()
e.mu.Lock()
defer e.mu.Unlock()
return e.unblockIPLocked(ip)
}
func (e *Engine) unblockIPLocked(ip string) error {
targetSet, key, err := e.resolveIPSet(ip, e.setBlocked, e.setBlocked6)
if err != nil {
return err
}
// Remove from state BEFORE the kernel delete: a crash between the two
// must converge to the operator's intent (unblocked) on the next Apply,
// not silently re-block the IP from a stale state row. On kernel failure
// the prior state is restored so the entry stays visible for a retry.
priorState := e.loadStateFile()
nextState := copyFirewallState(priorState)
removeBlockedIPFromState(&nextState, ip)
if e.lifecycle != nil {
return e.runDurableLocked(ActionRequest{Operation: "unblock", Target: ip}, nil, nextState)
}
if err := e.persistFirewallIntent(priorState, nextState); err != nil {
return fmt.Errorf("persisting unblock for %s: %w", ip, err)
}
if err := e.conn.SetDeleteElements(targetSet, []nftables.SetElement{{Key: key}}); err != nil {
if restoreErr := e.saveState(&priorState); restoreErr != nil {
return fmt.Errorf("removing from blocked set: %w (state restore failed: %v)", err, restoreErr)
}
return fmt.Errorf("removing from blocked set: %w", err)
}
if err := e.conn.Flush(); err != nil {
if isNftNotFound(err) {
// The delete target already disappeared from nft, so the
// persisted unblock is the only remaining state to keep.
e.legacyActionAuditLocked("unblock", ip, "", "", 0)
return nil
}
if restoreErr := e.saveState(&priorState); restoreErr != nil {
return fmt.Errorf("flushing: %w (state restore failed: %v)", err, restoreErr)
}
return fmt.Errorf("flushing: %w", err)
}
e.legacyActionAuditLocked("unblock", ip, "", "", 0)
return nil
}
// IsBlocked returns true if the IP is currently in the engine's blocked state.
// Uses the persisted state file (which is cleaned of expired entries on load).
//
// The lookup is O(1) via the blockedIPIndex map populated from the
// cached state. Linear scans over the parsed slice are gone -- on
// hosts with hundreds of persisted blocks the scan was the dominant
// cost of every connection-handler IsBlocked check.
func (e *Engine) IsBlocked(ip string) bool {
e.mu.Lock()
defer e.mu.Unlock()
e.ensureStateCacheLocked()
if _, ok := e.blockedIPIndex[ip]; ok {
return true
}
// The index is keyed by canonical form (blockIPLocked stores
// net.ParseIP(ip).String()). A caller passing a non-canonical form -- an
// IPv4-mapped IPv6 like "::ffff:1.2.3.4" from a dual-stack listener --
// would otherwise miss the block. Retry with the canonical form.
if parsed := net.ParseIP(ip); parsed != nil {
if canon := parsed.String(); canon != ip {
_, ok := e.blockedIPIndex[canon]
return ok
}
}
return false
}
// IsAllowed reports whether ip is on the operator allowed_ips set, read from
// the in-memory cache built from state.json. Mirrors IsBlocked, including the
// canonical-form retry for IPv4-mapped IPv6 callers.
func (e *Engine) IsAllowed(ip string) bool {
e.mu.Lock()
defer e.mu.Unlock()
e.ensureStateCacheLocked()
return ipSetIndexContains(e.allowedIPIndex, ip)
}
func (e *Engine) operatorSoftAllowed(ip string) bool {
e.mu.Lock()
defer e.mu.Unlock()
return e.operatorSoftAllowedLocked(ip)
}
func (e *Engine) operatorSoftAllowedLocked(ip string) bool {
e.ensureStateCacheLocked()
return ipSetIndexContains(e.allowedIPIndex, ip) || ipSetIndexContains(e.portAllowedIndex, ip)
}
func ipSetIndexContains(index map[string]struct{}, ip string) bool {
if _, ok := index[ip]; ok {
return true
}
if parsed := net.ParseIP(ip); parsed != nil {
if canon := parsed.String(); canon != ip {
_, ok := index[canon]
return ok
}
}
return false
}
// IsBlockedLive queries the live nftables set, not the in-memory cache built
// from state.json. The cache can drift from the kernel when nft auto-expires
// entries faster than CSM rewrites state.json, or when an out-of-band flush
// happens. Reconcile loops should consult this method so the local tracker
// shrinks in lock-step with the kernel; per-packet hot paths should stay on
// IsBlocked since this issues a netlink RTT.
//
// Malformed IPs are reported as absent. Netlink and engine-initialization
// failures are returned so callers can keep their cached answer instead of
// deleting local state on a transient lookup failure.
func (e *Engine) IsBlockedLive(ip string) (bool, error) {
e.mu.Lock()
defer e.mu.Unlock()
return e.isBlockedLiveLocked(ip)
}
// livePermTempCountsLocked returns the count of permanent and temporary
// entries across the blocked v4 + v6 nft sets. Used by blockIPTarget so
// the deny limits trip against the kernel's actual state instead of
// stale state.json entries that the kernel already expired. Live keys
// with verified CSM state are classified from state because nft
// timeout attributes can reflect inherited/default set behaviour rather
// than the operator's block intent. Out-of-state live keys fall back to
// the kernel expiration attributes. live is the addresses it read from
// the sets, nil when a seam supplied the counts.
//
// Must be called with e.mu held.
func (e *Engine) livePermTempCountsLocked(state FirewallState) (perm, temp int, live map[string]bool, ok bool) {
if e.liveBlockCounts != nil {
p, t, err := e.liveBlockCounts()
if err != nil {
return 0, 0, nil, false
}
return p, t, nil, true
}
if e.conn == nil && e.liveBlockedDump == nil {
return 0, 0, nil, false
}
stateTempByIP := blockedStateTempByIP(state)
gotAny := false
live = make(map[string]bool)
for _, set := range []*nftables.Set{e.setBlocked, e.setBlocked6} {
// An IPv6 set an earlier ruleset left behind is not in the kernel.
if set == nil || (set == e.setBlocked6 && !e.cfg.IPv6) {
continue
}
elements, err := e.dumpBlockedSetLocked(set)
if err != nil {
return 0, 0, nil, false
}
gotAny = true
p, t := countLiveBlockElements(elements, stateTempByIP)
perm += p
temp += t
for _, el := range elements {
if ip, valid := blockSetIP(set, el.Key); valid {
live[ip] = true
}
}
}
if !gotAny {
return 0, 0, nil, false
}
return perm, temp, live, true
}
func blockedStateTempByIP(state FirewallState) map[string]bool {
byIP := make(map[string]bool, len(state.Blocked)*2)
for _, entry := range state.Blocked {
if entry.IP == "" || entry.Unverified {
continue
}
temp := !entry.ExpiresAt.IsZero()
byIP[entry.IP] = temp
if parsed := net.ParseIP(entry.IP); parsed != nil {
byIP[parsed.String()] = temp
}
}
return byIP
}
func countLiveBlockElements(elements []nftables.SetElement, stateTempByIP map[string]bool) (perm, temp int) {
for _, el := range elements {
ip, _ := setElementIPString(el.Key)
if liveBlockTemporary(el, ip, stateTempByIP) {
temp++
} else {
perm++
}
}
return perm, temp
}
// liveBlockTemporary classifies a blocked-set element with address ip
// ("" when its key is none): by the verified row that names the address,
// otherwise by the element's own timeout.
func liveBlockTemporary(el nftables.SetElement, ip string, stateTempByIP map[string]bool) bool {
if stateTemp, found := stateTempByIP[ip]; found {
return stateTemp
}
return el.Timeout > 0 || el.Expires > 0
}
func (e *Engine) liveBlockElementKindLocked(set *nftables.Set, key []byte) (live, temporary, classified bool) {
if set == nil || (e.liveBlockedDump == nil && e.conn == nil) {
return false, false, false
}
elements, err := e.dumpBlockedSetLocked(set)
if err != nil {
return false, false, false
}
for _, el := range elements {
if bytes.Equal(el.Key, key) {
return true, el.Timeout > 0 || el.Expires > 0, true
}
}
return false, false, true
}
func setElementIPString(key []byte) (string, bool) {
switch len(key) {
case net.IPv4len:
return net.IP(key).String(), true
case net.IPv6len:
return net.IP(key).String(), true
default:
return "", false
}
}
func (e *Engine) isBlockedLiveLocked(ip string) (bool, error) {
parsed := net.ParseIP(ip)
if parsed == nil {
return false, nil
}
var (
set *nftables.Set
key []byte
)
if ip4 := parsed.To4(); ip4 != nil {
set = e.setBlocked
key = ip4
} else {
set = e.setBlocked6
key = parsed.To16()
}
if set == nil {
return false, fmt.Errorf("blocked set unavailable for %s", ip)
}
if e.liveBlockLookup != nil {
return e.liveBlockLookup(set, key)
}
if e.conn == nil {
return false, fmt.Errorf("nftables connection unavailable")
}
elements, err := e.conn.GetSetElements(set)
if err != nil {
return false, fmt.Errorf("listing blocked set: %w", err)
}
for _, el := range elements {
if bytes.Equal(el.Key, key) {
return true, nil
}
}
return false, nil
}
// LiveBlockedSet dumps the blocked v4 and v6 sets once and returns a
// membership snapshot. IsBlockedLive issues a full set dump per call, so a
// reconcile pass over N tracked IPs cost N dumps of the entire set; callers
// testing more than one IP should snapshot instead.
//
// A nil set is left uncovered rather than reported empty, and so is one whose
// dump failed: the returned snapshot always describes whichever families did
// answer, with the failures joined into the error. Callers keep their cached
// view for uncovered families instead of erasing local state on a transient
// netlink error, while a family that dumped cleanly stays reconciled.
func (e *Engine) LiveBlockedSet() (LiveBlockedSnapshot, error) {
e.mu.Lock()
defer e.mu.Unlock()
if e.setBlocked == nil && e.setBlocked6 == nil {
return LiveBlockedSnapshot{}, fmt.Errorf("blocked sets unavailable")
}
if e.liveBlockedDump == nil && e.conn == nil {
return LiveBlockedSnapshot{}, fmt.Errorf("nftables connection unavailable")
}
var snap LiveBlockedSnapshot
var dumpErrs []error
for _, target := range []struct {
set *nftables.Set
members *map[string]struct{}
covered *bool
}{
{e.setBlocked, &snap.V4, &snap.HasV4},
{e.setBlocked6, &snap.V6, &snap.HasV6},
} {
if target.set == nil {
continue
}
elements, err := e.dumpBlockedSetLocked(target.set)
if err != nil {
// One family failing does not invalidate the other. Leaving this
// family uncovered keeps its callers on their cached answer while
// the family that did dump stays reconciled against the kernel.
dumpErrs = append(dumpErrs, fmt.Errorf("listing blocked set %s: %w", target.set.Name, err))
continue
}
members := make(map[string]struct{}, len(elements))
for _, el := range elements {
// Keys come from the kernel; anything that is not an address
// width cannot be matched against and is not ours to interpret.
if len(el.Key) != net.IPv4len && len(el.Key) != net.IPv6len {
continue
}
members[net.IP(el.Key).String()] = struct{}{}
}
*target.members = members
*target.covered = true
}
return snap, errors.Join(dumpErrs...)
}
func (e *Engine) dumpBlockedSetLocked(set *nftables.Set) ([]nftables.SetElement, error) {
if e.liveBlockedDump != nil {
return e.liveBlockedDump(set)
}
return e.conn.GetSetElements(set)
}
// AllowIP adds an IP to the allowed set and persists it.
// If the IP is currently blocked, the block is removed first.
func (e *Engine) AllowIP(ip string, reason string) error {
return e.allowIP(ip, reason, 0, "allow")
}
// TempAllowIP adds a temporary allow with expiry. Uses the same allowed set
// but tracks expiry in state - CleanExpiredAllows removes them periodically.
func (e *Engine) TempAllowIP(ip string, reason string, timeout time.Duration) error {
return e.allowIP(ip, reason, timeout, "temp_allow")
}
func (e *Engine) allowIP(ip string, reason string, timeout time.Duration, action string) (resultErr error) {
defer func() {
e.legacyFirewallFailure(action, ip, reason, InferProvenance(action, reason), timeout, resultErr)
}()
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return err
}
ip = canonical
defer func() { e.released(ip, ReleaseOperator, resultErr) }()
e.mu.Lock()
defer e.mu.Unlock()
blockedSet, blockedKey, _ := e.resolveIPSet(ip, e.setBlocked, e.setBlocked6)
allowedSet, allowedKey, err := e.resolveIPSet(ip, e.setAllowed, e.setAllowed6)
if err != nil {
return err
}
if allowedSet == nil {
return fmt.Errorf("allowed set unavailable for %s", ip)
}
entry := AllowedEntry{IP: ip, Reason: reason, Source: InferProvenance(action, reason)}
if action == "temp_allow" && timeout > 0 {
entry.ExpiresAt = time.Now().Add(timeout)
}
priorState := e.loadStateFile()
nextState := copyFirewallState(priorState)
removeBlockedIPFromState(&nextState, ip)
upsertAllowedEntryInState(&nextState, entry)
if e.lifecycle != nil {
return e.runDurableLocked(ActionRequest{Operation: action, Target: ip, Reason: reason, TTL: timeout}, nil, nextState)
}
if err := e.persistFirewallIntent(priorState, nextState); err != nil {
return fmt.Errorf("persisting %s for %s: %w", action, ip, err)
}
conn := e.newMutationConn()
if blockedSet != nil {
if err := conn.SetDeleteElements(blockedSet, []nftables.SetElement{{Key: blockedKey}}); err != nil {
if restoreErr := e.saveState(&priorState); restoreErr != nil {
return fmt.Errorf("removing from blocked set: %w (state restore failed: %v)", err, restoreErr)
}
logNftSetOpErr(action+" remove from blocked", ip, err)
return fmt.Errorf("removing from blocked set: %w", err)
}
}
if err := conn.SetAddElements(allowedSet, []nftables.SetElement{{Key: allowedKey}}); err != nil {
if restoreErr := e.saveState(&priorState); restoreErr != nil {
return fmt.Errorf("adding to allowed set: %w (state restore failed: %v)", err, restoreErr)
}
return fmt.Errorf("adding to allowed set: %w", err)
}
if err := conn.Flush(); err != nil {
retryErr := e.retryAllowAfterBenignFlushError(allowedSet, allowedKey, err)
if retryErr == nil {
e.legacyActionAuditLocked(action, ip, reason, entry.Source, timeout)
return nil
}
if restoreErr := e.saveState(&priorState); restoreErr != nil {
return fmt.Errorf("flushing: %w (state restore failed: %v)", err, restoreErr)
}
if isNftNotFound(err) {
return fmt.Errorf("flushing: %w (retry failed: %w)", err, retryErr)
}
return fmt.Errorf("flushing: %w", err)
}
e.legacyActionAuditLocked(action, ip, reason, entry.Source, timeout)
return nil
}
func (e *Engine) retryAllowAfterBenignFlushError(allowedSet *nftables.Set, allowedKey []byte, flushErr error) error {
if !isNftNotFound(flushErr) {
return flushErr
}
// Only the blocked delete can be absent when the allowed set exists.
// Retrying the add alone keeps the retry atomic and avoids a committed
// delete followed by a failed add.
conn := e.newMutationConn()
if err := conn.SetAddElements(allowedSet, []nftables.SetElement{{Key: allowedKey}}); err != nil {
return fmt.Errorf("retry adding to allowed set: %w", err)
}
if err := conn.Flush(); err != nil {
return fmt.Errorf("retry flushing allowed add: %w", err)
}
return nil
}
// CleanExpiredAllows removes expired temporary allows from the set and state.
// An IP is only removed from nftables if no non-expired entries remain for it.
// Called periodically by the daemon.
func (e *Engine) CleanExpiredAllows() int {
e.mu.Lock()
defer e.mu.Unlock()
prior, ok := e.loadStateFileRawLocked()
if !ok {
return 0
}
next := copyFirewallState(prior)
next.Allowed = nil
var expired []AllowedEntry
expiredIPs := make(map[string]bool)
now := time.Now()
for _, entry := range prior.Allowed {
if !entry.ExpiresAt.IsZero() && !now.Before(entry.ExpiresAt) {
expired = append(expired, entry)
expiredIPs[stateIPKey(entry.IP)] = true
} else {
next.Allowed = append(next.Allowed, entry)
}
}
if len(expired) == 0 {
return 0
}
for _, entry := range next.Allowed {
delete(expiredIPs, stateIPKey(entry.IP))
}
var removeIPs []string
for ip := range expiredIPs {
removeIPs = append(removeIPs, ip)
}
if err := e.commitAllowedRemovals(prior, next, removeIPs, ActionRequest{Operation: "temp_allow_expired", Target: "*", Actor: "daemon", Source: SourceSystem}); err != nil {
for _, entry := range expired {
if e.lifecycle == nil {
recordFirewallFailure("temp_allow_expired", entry.IP, "", SourceSystem, 0, err)
}
}
fmt.Fprintf(os.Stderr, "firewall: expired-allow cleanup failed: %v\n", err)
return 0
}
for _, entry := range expired {
e.legacyActionAuditLocked("temp_allow_expired", entry.IP, "", SourceSystem, 0)
}
return len(expired)
}
// CleanExpiredSubnets removes expired temporary subnet blocks from nftables and state.
func (e *Engine) CleanExpiredSubnets() int {
e.mu.Lock()
defer e.mu.Unlock()
prior, ok := e.loadStateFileRawLocked()
if !ok {
return 0
}
next := copyFirewallState(prior)
var active, expired []SubnetEntry
now := time.Now()
for _, entry := range prior.BlockedNet {
if !entry.ExpiresAt.IsZero() && !now.Before(entry.ExpiresAt) {
expired = append(expired, entry)
} else {
active = append(active, entry)
}
}
if len(expired) == 0 {
return 0
}
next.BlockedNet = active
if err := e.updateSubnetMutation(prior, next, ActionRequest{Operation: "temp_subnet_expired", Target: "*", Actor: "daemon", Source: SourceSystem}, nil); err != nil {
for _, entry := range expired {
if e.lifecycle == nil {
recordFirewallFailure("temp_subnet_expired", entry.CIDR, "", SourceSystem, 0, err)
}
}
fmt.Fprintf(os.Stderr, "firewall: expired-subnet cleanup failed: %v\n", err)
return 0
}
for _, entry := range expired {
e.legacyActionAuditLocked("temp_subnet_expired", entry.CIDR, "", SourceSystem, 0)
}
return len(expired)
}
// RemoveAllowIP removes an IP from the allowed set and state.
func (e *Engine) RemoveAllowIP(ip string) (resultErr error) {
defer func() { e.legacyFirewallFailure("remove_allow", ip, "", "", 0, resultErr) }()
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return err
}
ip = canonical
e.mu.Lock()
defer e.mu.Unlock()
prior := e.loadStateFile()
next := copyFirewallState(prior)
next.Allowed = nil
for _, entry := range prior.Allowed {
if !sameIPString(entry.IP, ip) {
next.Allowed = append(next.Allowed, entry)
}
}
if err := e.commitAllowedRemovals(prior, next, []string{ip}, ActionRequest{Operation: "remove_allow", Target: ip}); err != nil {
return err
}
e.legacyActionAuditLocked("remove_allow", ip, "", "", 0)
return nil
}
// RemoveAllowIPBySource removes only allow entries from a specific source.
// The IP is only removed from the nftables set if no other sources remain.
func (e *Engine) RemoveAllowIPBySource(ip, source string) (resultErr error) {
defer func() { e.legacyFirewallFailure("remove_allow", ip, "", source, 0, resultErr) }()
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return err
}
ip = canonical
e.mu.Lock()
defer e.mu.Unlock()
prior := e.loadStateFile()
next := copyFirewallState(prior)
_, ipGone := removeAllowedSourceFromState(&next, ip, source)
var removeIPs []string
if ipGone {
removeIPs = []string{ip}
}
if err := e.commitAllowedRemovals(prior, next, removeIPs, ActionRequest{Operation: "remove_allow", Target: ip, Reason: "source: " + source, Source: source}); err != nil {
return err
}
e.legacyActionAuditLocked("remove_allow", ip, "source: "+source, source, 0)
return nil
}
// AllowIPPort adds a port-specific IP allow. The rule is persisted to state
// and applied on the next Apply(). For immediate effect, call Apply() after.
func (e *Engine) AllowIPPort(ip string, port int, proto string, reason string) (resultErr error) {
defer func() {
e.legacyFirewallFailure("allow_port", fmt.Sprintf("%s:%d/%s", ip, port, proto), reason, InferProvenance("allow_port", reason), 0, resultErr)
}()
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return err
}
ip = canonical
if port < 1 || port > 65535 {
return fmt.Errorf("invalid port: %d", port)
}
if proto != "tcp" && proto != "udp" {
proto = "tcp"
}
e.mu.Lock()
defer e.mu.Unlock()
st := e.loadStateFile()
// Deduplicate
for _, existing := range st.PortAllowed {
if sameIPString(existing.IP, ip) && existing.Port == port && existing.Proto == proto {
return e.savePortPolicyLocked(&st, ActionRequest{Operation: "configure_port_allow", Target: fmt.Sprintf("%s:%d/%s", ip, port, proto), Reason: reason}) // Confirm durability even after a prior uncertain write.
}
}
st.PortAllowed = append(st.PortAllowed, PortAllowEntry{
IP: ip, Port: port, Proto: proto, Reason: reason, Source: InferProvenance("allow_port", reason),
})
if err := e.savePortPolicyLocked(&st, ActionRequest{Operation: "configure_port_allow", Target: fmt.Sprintf("%s:%d/%s", ip, port, proto), Reason: reason}); err != nil {
return fmt.Errorf("persisting port allow change: %w", err)
}
e.legacyActionAuditLocked("allow_port", fmt.Sprintf("%s:%d/%s", ip, port, proto), reason, InferProvenance("allow_port", reason), 0)
return nil
}
// RemoveAllowIPPort removes a port-specific IP allow from state.
func (e *Engine) RemoveAllowIPPort(ip string, port int, proto string) (resultErr error) {
defer func() {
e.legacyFirewallFailure("remove_port_allow", fmt.Sprintf("%s:%d/%s", ip, port, proto), "", "", 0, resultErr)
}()
canonical, err := canonicalFirewallIP(ip)
if err != nil {
return err
}
ip = canonical
e.mu.Lock()
defer e.mu.Unlock()
st := e.loadStateFile()
var remaining []PortAllowEntry
found := false
for _, entry := range st.PortAllowed {
if sameIPString(entry.IP, ip) && entry.Port == port && entry.Proto == proto {
found = true
continue
}
remaining = append(remaining, entry)
}
if !found {
return fmt.Errorf("port allow not found: %s:%d/%s", ip, port, proto)
}
st.PortAllowed = remaining
if err := e.savePortPolicyLocked(&st, ActionRequest{Operation: "remove_port_allow", Target: fmt.Sprintf("%s:%d/%s", ip, port, proto)}); err != nil {
return fmt.Errorf("persisting port allow change: %w", err)
}
e.legacyActionAuditLocked("remove_port_allow", fmt.Sprintf("%s:%d/%s", ip, port, proto), "", "", 0)
return nil
}
// FlushBlocked removes all IPs from the blocked set and clears persisted state.
func (e *Engine) FlushBlocked() (resultErr error) {
defer func() { e.legacyFirewallFailure("flush", "", "", "", 0, resultErr) }()
defer func() { e.released("", ReleaseOperator, resultErr) }()
e.mu.Lock()
defer e.mu.Unlock()
// Persist the operator's intent before changing nftables. If the process
// dies between these steps, the next Apply must converge to an empty block
// set instead of restoring every flushed IP from stale state.
priorState := e.loadStateFile()
nextState := copyFirewallState(priorState)
nextState.Blocked = nil
if e.lifecycle != nil {
return e.runDurableLocked(ActionRequest{Operation: "flush", Target: "*"}, nil, nextState)
}
if err := e.persistFirewallIntent(priorState, nextState); err != nil {
return fmt.Errorf("persisting flush: %w", err)
}
e.conn.FlushSet(e.setBlocked)
if e.setBlocked6 != nil {
e.conn.FlushSet(e.setBlocked6)
}
if err := e.conn.Flush(); err != nil {
if restoreErr := e.saveState(&priorState); restoreErr != nil {
return fmt.Errorf("flushing blocked set: %w (state restore failed: %v)", err, restoreErr)
}
return fmt.Errorf("flushing blocked set: %w", err)
}
e.legacyActionAuditLocked("flush", "", fmt.Sprintf("cleared %d entries", len(priorState.Blocked)), SourceSystem, 0)
return nil
}
// subnetSafetyGuardLocked refuses a CIDR block that would firewall traffic the
// daemon must keep reachable: infra IPs or ranges, DNS-resolved infra hosts,
// local interface addresses, full-IP allows, port-specific allows, or the
// default route. Subnet blocks cover many addresses, and the output chain has
// no infra carve-out, so an unsafe subnet can lock out operators or kill the
// daemon's own egress.
// Must be called with e.mu held.
func (e *Engine) subnetSafetyGuardLocked(network *net.IPNet) error {
state := e.loadStateFile()
if e.lifecycle != nil && e.stateReadErr != nil {
return e.stateReadErr
}
return e.subnetSafetyGuardStateLocked(network, state)
}
func (e *Engine) subnetSafetyGuardStateLocked(network *net.IPNet, state FirewallState) error {
if ones, _ := network.Mask.Size(); ones == 0 {
return ipProtectedErrorf("refusing to block default route: %s", network.String())
}
// An unspecified or broadcast host is not a usable target, but its
// containing range can be: operators may block 0.0.0.0/8 or 240.0.0.0/4
// as bogon ranges.
if ones, bits := network.Mask.Size(); ones == bits && (network.IP.IsUnspecified() || network.IP.Equal(net.IPv4bcast)) {
return ipProtectedErrorf("refusing to block non-routable range: %s", network.String())
}
// Checking only the first address misses larger ranges covering a
// protected one. Interface enumeration omits loopback and link-local, and
// no packet reaching the input chain carries a multicast source.
for _, protected := range unblockableRanges {
if network.Contains(protected.IP) || protected.Contains(network.IP) {
return ipProtectedErrorf("refusing to block subnet %s: overlaps protected range %s", network, protected)
}
}
for _, raw := range e.cfg.InfraIPs {
if _, infraNet, cidrErr := net.ParseCIDR(raw); cidrErr == nil {
if network.Contains(infraNet.IP) || infraNet.Contains(network.IP) {
return ipProtectedErrorf("refusing to block subnet %s: overlaps infra range %s", network.String(), raw)
}
continue
}
if infraIP := net.ParseIP(raw); infraIP != nil && network.Contains(infraIP) {
return ipProtectedErrorf("refusing to block subnet %s: contains infra IP %s", network.String(), raw)
}
}
for host, set := range e.infraResolved {
for key := range set {
if ip := net.ParseIP(key); ip != nil && network.Contains(ip) {
return ipProtectedErrorf("refusing to block subnet %s: contains infra IP %s (resolved from %s)", network.String(), key, host)
}
}
}
e.refreshLocalAddrsLocked()
for key := range e.localAddrs {
if ip := net.ParseIP(key); ip != nil && network.Contains(ip) {
return ipProtectedErrorf("refusing to block subnet %s: contains local host IP %s", network.String(), key)
}
}
for _, entry := range state.Allowed {
if ip := net.ParseIP(entry.IP); ip != nil && network.Contains(ip) {
return ipProtectedErrorf("refusing to block subnet %s: contains allowed IP %s", network.String(), entry.IP)
}
}
for _, entry := range state.PortAllowed {
if ip := net.ParseIP(entry.IP); ip != nil && network.Contains(ip) {
return ipProtectedErrorf("refusing to block subnet %s: contains port-allowed IP %s", network.String(), entry.IP)
}
}
return nil
}
var unblockableRanges = []*net.IPNet{
{IP: net.IPv4(127, 0, 0, 0), Mask: net.CIDRMask(8, 32)},
{IP: net.IPv4(169, 254, 0, 0), Mask: net.CIDRMask(16, 32)},
{IP: net.IPv4(224, 0, 0, 0), Mask: net.CIDRMask(4, 32)},
{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
{IP: net.ParseIP("fe80::"), Mask: net.CIDRMask(10, 128)},
{IP: net.ParseIP("ff00::"), Mask: net.CIDRMask(8, 128)},
}
func (e *Engine) subnetBlockPlanLocked(cidr string, state FirewallState) (*net.IPNet, bool, error) {
_, network, err := net.ParseCIDR(cidr)
if err != nil {
return nil, false, fmt.Errorf("invalid CIDR: %s", cidr)
}
if err := e.subnetSafetyGuardStateLocked(network, state); err != nil {
return nil, false, err
}
for _, entry := range state.BlockedNet {
if entry.CIDR == network.String() {
return network, true, nil
}
}
targetSet, _, _ := e.resolveSubnetSet(network)
if targetSet == nil {
return nil, false, fmt.Errorf("no matching set for %s (IPv6 disabled?)", cidr)
}
return network, false, nil
}
// ValidateSubnetBlock runs the same safety and capability checks as
// BlockSubnet without changing persisted or kernel firewall state.
func (e *Engine) ValidateSubnetBlock(cidr string) error {
e.mu.Lock()
defer e.mu.Unlock()
state := e.loadStateFile()
if e.lifecycle != nil && e.stateReadErr != nil {
return e.stateReadErr
}
_, _, err := e.subnetBlockPlanLocked(cidr, state)
return err
}
// BlockSubnet adds a CIDR range to the blocked subnets set (IPv4 or IPv6).
// timeout 0 = permanent block.
func (e *Engine) BlockSubnet(cidr string, reason string, timeout time.Duration) (resultErr error) {
return e.BlockSubnetWithFindingID(cidr, reason, timeout, "")
}
// BlockSubnetWithFindingID records a causal finding for an automatic subnet
// decision. Manual and maintenance callers use BlockSubnet without one.
func (e *Engine) BlockSubnetWithFindingID(cidr, reason string, timeout time.Duration, findingID string) (resultErr error) {
return e.BlockSubnetRequest(ActionRequest{Operation: "block_subnet", Target: cidr, Reason: reason, TTL: timeout, FindingID: findingID, Automatic: findingID != ""}, nil)
}
func (e *Engine) BlockSubnetRequest(req ActionRequest, budget *ScanAdmission) (resultErr error) {
cidr, reason, timeout, findingID := req.Target, req.Reason, req.TTL, req.FindingID
// Keep display provenance aligned with the request's automatic intent.
// Free-form operator reasons cannot establish automatic ownership.
source := req.Source
if source == "" {
source = InferProvenance("block_subnet", reason)
}
if req.Automatic {
source = SourceAutoResponse
} else if source == SourceAutoResponse {
source = SourceUnknown
}
defer func() {
if e.shouldLegacyOutcome(resultErr) {
result, auditErr := actionlog.Applied, resultErr
if errors.Is(resultErr, ErrActionDryRun) {
result, auditErr = actionlog.DryRun, nil
}
recordFirewallFindingResult("block_subnet", cidr, reason, source, timeout, result, auditErr, findingID)
}
}()
_, network, parseErr := net.ParseCIDR(req.Target)
if parseErr != nil {
return fmt.Errorf("invalid CIDR: %s", req.Target)
}
req.Operation = "block_subnet"
req.Target = network.String()
req = normalizeActionRequest(req)
dryRun := req.Automatic && e.autoResponseDryRunEnabled()
e.mu.Lock()
defer e.mu.Unlock()
if found, replayErr := e.replayActionLocked(req); found || replayErr != nil {
return replayErr
}
if readyErr := e.lifecycleReadyLocked(); readyErr != nil {
return readyErr
}
priorState := e.loadStateFile()
if e.lifecycle != nil && e.stateReadErr != nil {
return e.stateReadErr
}
network, alreadyBlocked, err := e.subnetBlockPlanLocked(cidr, priorState)
if err != nil {
return err
}
if dryRun {
return ErrActionDryRun
}
cidr = network.String()
if alreadyBlocked {
if e.lifecycle != nil {
return nil
}
// A prior write may be visible despite a durability error, before the
// kernel changed. Retrying must reconcile that saved intent as well.
if err := e.updateSubnetMutation(priorState, priorState, req, budget); err != nil {
return err
}
e.legacyFileAuditLocked("block_subnet", network.String(), reason, source, timeout)
return nil
}
entry := SubnetEntry{
CIDR: network.String(),
Reason: reason,
Source: source,
BlockedAt: time.Now(),
Automatic: req.Automatic,
}
if timeout > 0 {
entry.ExpiresAt = time.Now().Add(timeout)
}
// Persist to state BEFORE the kernel add, mirroring blockIPLocked: state
// seeds the next Apply, so a crash between the kernel add and a later
// state write would otherwise leave a kernel block that silently
// disappears on restart. On kernel failure the prior state is restored.
nextState := copyFirewallState(priorState)
addSubnetEntryIfMissingInState(&nextState, entry)
if err := e.updateSubnetMutation(priorState, nextState, req, budget); err != nil {
return err
}
e.legacyFileAuditLocked("block_subnet", network.String(), reason, entry.Source, timeout)
return nil
}
// IsSubnetBlocked returns true if the CIDR is present in the persisted subnet block state.
func (e *Engine) IsSubnetBlocked(cidr string) bool {
_, network, err := net.ParseCIDR(cidr)
if err != nil {
return false
}
e.mu.Lock()
defer e.mu.Unlock()
return e.isSubnetBlockedStateLocked(network.String())
}
// BlockedSubnetCovering reports the blocked CIDR (if any) that contains ip.
// The input-chain drops blocked_nets before the allowed_ips accept, so an
// allow on an IP inside a blocked subnet has no effect: the subnet drop still
// fires. Callers surface this so an operator is not told an IP is reachable
// when a subnet rule still blocks it. The subnet block stays authoritative by
// design (see subnetSafetyGuardLocked); this only reports, it does not unblock.
func (e *Engine) BlockedSubnetCovering(ip string) (string, bool) {
e.mu.Lock()
defer e.mu.Unlock()
return subnetCovering(e.loadStateFile().BlockedNet, ip)
}
// UnblockSubnet removes a CIDR range from the blocked subnets set (IPv4 or IPv6).
func (e *Engine) UnblockSubnet(cidr string) (resultErr error) {
return e.unblockSubnet(cidr, nil)
}
// UnblockSubnetIfUnchanged refuses to remove a replacement decision after a
// caller checked the subnet's provenance in a snapshot.
func (e *Engine) UnblockSubnetIfUnchanged(expected SubnetEntry) error {
return e.unblockSubnet(expected.CIDR, &expected)
}
func (e *Engine) unblockSubnet(cidr string, expected *SubnetEntry) (resultErr error) {
defer func() { e.legacyFirewallFailure("unblock_subnet", cidr, "", "", 0, resultErr) }()
e.mu.Lock()
defer e.mu.Unlock()
_, network, err := net.ParseCIDR(cidr)
if err != nil {
return fmt.Errorf("invalid CIDR: %s", cidr)
}
targetSet, _, _ := e.resolveSubnetSet(network)
if targetSet == nil {
return fmt.Errorf("no matching set for %s (IPv6 disabled?)", cidr)
}
prior := e.loadStateFile()
if expected != nil {
if e.stateReadErr != nil {
return e.stateReadErr
}
found := false
for _, entry := range prior.BlockedNet {
if entry.CIDR != network.String() {
continue
}
if entry.Reason != expected.Reason || entry.Source != expected.Source || entry.Automatic != expected.Automatic ||
!entry.BlockedAt.Equal(expected.BlockedAt) || !entry.ExpiresAt.Equal(expected.ExpiresAt) {
return ErrBlockChanged
}
found = true
}
if !found {
return ErrBlockChanged
}
}
next := copyFirewallState(prior)
remaining := next.BlockedNet[:0]
for _, entry := range next.BlockedNet {
if entry.CIDR != network.String() {
remaining = append(remaining, entry)
}
}
next.BlockedNet = remaining
if err := e.updateSubnetMutation(prior, next, ActionRequest{Operation: "unblock_subnet", Target: network.String()}, nil); err != nil {
return err
}
e.legacyActionAuditLocked("unblock_subnet", network.String(), "", "", 0)
return nil
}
// resolveSubnetSet returns the correct blocked_nets set and start/end keys
// for a CIDR. Returns (nil, nil, nil) when IPv6 is disabled for a v6 CIDR,
// or when lastIPInRange cannot produce an interval end (malformed
// net.IPNet whose IP is neither 4 nor 16 bytes) -- callers already check
// for a nil set, so this also short-circuits the degenerate case where
// nextIP(nil) would feed an empty Key to the kernel.
func (e *Engine) resolveSubnetSet(network *net.IPNet) (*nftables.Set, net.IP, net.IP) {
end := lastIPInRange(network)
if end == nil {
return nil, nil, nil
}
if start := network.IP.To4(); start != nil {
return e.setBlockedNet, start, end
}
if e.setBlockedNet6 != nil {
return e.setBlockedNet6, network.IP.To16(), end
}
return nil, nil, nil
}
func intervalSetElements(start, end net.IP) []nftables.SetElement {
if start == nil || end == nil {
return nil
}
return appendIntervalSetElements(nil, start, end)
}
func appendIntervalSetElements(dst []nftables.SetElement, start, end net.IP) []nftables.SetElement {
if start == nil || end == nil {
return dst
}
endMarker, ok := nextIntervalKey(end)
if !ok {
// nftables represents an interval through the last address with an
// open end. Wrapping its exclusive end to zero would change its meaning.
return append(dst, nftables.SetElement{Key: start})
}
return append(dst,
nftables.SetElement{Key: start},
nftables.SetElement{Key: endMarker, IntervalEnd: true},
)
}
// UpdateInfraResolved records the IP set last resolved for an infra
// hostname. Replaces any previous entry for that host so the resolver's
// per-tick refresh leaves no stale ghost IPs. Pass an empty ips slice
// to remove the host entirely (e.g. when DNS stopped resolving).
func (e *Engine) UpdateInfraResolved(host string, ips []string) {
e.mu.Lock()
defer e.mu.Unlock()
if host == "" {
return
}
if e.infraResolved == nil {
e.infraResolved = make(map[string]map[string]struct{})
}
if len(ips) == 0 {
delete(e.infraResolved, host)
return
}
set := make(map[string]struct{}, len(ips))
for _, ip := range ips {
// Normalise via net.ParseIP so canonical form is stored
// (collapses IPv6 forms and drops malformed values).
if parsed := net.ParseIP(ip); parsed != nil {
set[parsed.String()] = struct{}{}
}
}
if len(set) == 0 {
delete(e.infraResolved, host)
return
}
e.infraResolved[host] = set
}
// DropInfraResolved clears all resolved IPs for a host. Equivalent to
// UpdateInfraResolved(host, nil); separate name surfaces operator
// intent at call sites that purposefully retire a hostname.
func (e *Engine) DropInfraResolved(host string) {
e.UpdateInfraResolved(host, nil)
}
// infraIPResolvedHostLocked reports whether ip matches any IP recorded
// for any tracked infra hostname. Must be called with e.mu held; the
// existing blockIPTarget path already does so. The lookup normalizes ip
// to the same canonical form the storage path applies (net.ParseIP
// collapses IPv6 and rewrites ::ffff:1.2.3.4 to 1.2.3.4), so a caller
// passing the IPv4-mapped or uncanonical form still hits the guard.
func (e *Engine) infraIPResolvedHostLocked(ip string) (string, bool) {
if e.infraResolved == nil {
return "", false
}
parsed := net.ParseIP(ip)
if parsed == nil {
return "", false
}
key := parsed.String()
for host, set := range e.infraResolved {
if _, ok := set[key]; ok {
return host, true
}
}
return "", false
}
// localAddrsCacheTTL bounds how stale the host-own-IP set can get before
// the next block call rebuilds it. The trade-off: a newly assigned local
// address could be auto-blocked for up to this window if a flagged source
// happens to share that address. Local-address changes (operator running
// `ip addr add`) are rare, so 60s keeps both the FP window and the steady
// per-block syscall cost negligible. Refreshing every miss instead would
// pin the cache to permanently-fresh under any scan storm.
const localAddrsCacheTTL = 60 * time.Second
// refreshLocalAddrsLocked rebuilds the cache of host-own interface
// addresses when the TTL has expired. Must be called with e.mu held.
// Failure leaves the previous cache in place so a transient netlink
// hiccup cannot demote the lockout guard.
func (e *Engine) refreshLocalAddrsLocked() {
if e.localAddrs != nil && !e.localAddrsExpiresAt.IsZero() && time.Now().Before(e.localAddrsExpiresAt) {
return
}
var ips []string
if e.localAddrsLookup != nil {
got, err := e.localAddrsLookup()
if err != nil {
return
}
ips = got
} else {
addrs, err := net.InterfaceAddrs()
if err != nil {
return
}
for _, addr := range addrs {
ipnet, ok := addr.(*net.IPNet)
if !ok {
continue
}
ips = append(ips, ipnet.IP.String())
}
}
set := make(map[string]struct{}, len(ips))
for _, raw := range ips {
key, ok := localAddrGuardKey(raw)
if !ok {
continue
}
set[key] = struct{}{}
}
e.localAddrs = set
e.localAddrsExpiresAt = time.Now().Add(localAddrsCacheTTL)
}
// isUnblockableAddress reports addresses that can never legitimately be
// blocked, independent of what the interface enumeration returned.
//
// The local-address guard is built from localAddrGuardKey, which drops
// loopback and link-local, so those were the one class the guard did not
// cover -- and loopback is precisely what a control panel's own proxied
// requests appear as. A block of 127.0.0.1 was therefore accepted rather than
// refused; nothing broke only because the input chain accepts "iifname lo"
// before reaching the blocked set, leaving an entry that looked effective
// while doing nothing.
//
// Multicast and the limited broadcast address are never a packet's source:
// the kernel drops such packets before the input chain. A block on one
// matches nothing and only takes a slot in the deny set.
func isUnblockableAddress(ip string) bool {
parsed := net.ParseIP(ip)
if parsed == nil {
return false
}
return parsed.IsLoopback() ||
parsed.IsUnspecified() ||
parsed.IsLinkLocalUnicast() ||
parsed.IsMulticast() ||
parsed.Equal(net.IPv4bcast)
}
func localAddrGuardKey(raw string) (string, bool) {
parsed := net.ParseIP(raw)
if parsed == nil {
return "", false
}
if isUnblockableAddress(raw) {
return "", false
}
return parsed.String(), true
}
// isLocalAddrLocked reports whether ip is one of the daemon's own host
// addresses. Must be called with e.mu held.
func (e *Engine) isLocalAddrLocked(ip string) bool {
parsed := net.ParseIP(ip)
if parsed == nil {
return false
}
e.refreshLocalAddrsLocked()
if len(e.localAddrs) == 0 {
return false
}
_, ok := e.localAddrs[parsed.String()]
return ok
}
// BlockedCount returns the number of live blocked IP entries the engine
// is enforcing. Sourced from the same state file Status() uses, so
// `/api/v1/status` and `csm firewall status` agree on the number. Expired
// entries are pruned by loadStateFile before being counted.
func (e *Engine) BlockedCount() int {
e.mu.Lock()
defer e.mu.Unlock()
s := e.loadStateFile()
return countBlockedRules(s.Blocked, e.cfg != nil && e.cfg.IPv6)
}
// RuleCounts returns the cardinality of every firewall rule category from
// the engine state file with expired temp bans pruned. Callers needing a
// live count (e.g. Prometheus gauges) must use this rather than the bbolt
// store, which holds only the migration-time snapshot.
func (e *Engine) RuleCounts() RuleCounts {
e.mu.Lock()
defer e.mu.Unlock()
s := e.loadStateFile()
return countRuleEntries(s, e.cfg != nil && e.cfg.IPv6)
}
// BlockedSubnets returns a snapshot of the active persisted subnet blocks.
// The returned slice is a fresh copy: loadStateFile reuses the warm shared
// state cache, so the slice it returns can alias internal engine state. Copy
// it here so a caller mutating the result cannot corrupt the cache. SubnetEntry
// is a value type (strings + time.Time, no reference fields), so a slice copy
// is a sufficient deep copy.
func (e *Engine) BlockedSubnets() []SubnetEntry {
e.mu.Lock()
defer e.mu.Unlock()
s := e.loadStateFile()
return append([]SubnetEntry(nil), s.BlockedNet...)
}
// Status returns current firewall statistics.
//
// Takes e.mu so the cached state can be read coherently. Before the
// cache existed loadStateFile was lock-free because every call did its
// own ReadFile + Unmarshal; now that loadStateFile mutates the shared
// cache + index, the lock is required.
func (e *Engine) Status() map[string]interface{} {
e.mu.Lock()
defer e.mu.Unlock()
state := e.loadStateFile()
return map[string]interface{}{
"enabled": e.cfg.Enabled,
"tcp_in": e.cfg.TCPIn,
"tcp_out": e.cfg.TCPOut,
"udp_in": e.cfg.UDPIn,
"udp_out": e.cfg.UDPOut,
"infra_ips": e.cfg.InfraIPs,
"blocked": len(state.Blocked) - unverifiedRows(state.Blocked),
"allowed": len(state.Allowed),
"log_dropped": e.cfg.LogDropped,
}
}
// --- State persistence ---
// initialBlockState is the pre-computed pool of nft set elements to
// seed a freshly-built csm table from persisted state. Populated by
// computeInitialBlockStateLocked and consumed by
// queueInitialBlockStateLocked.
type initialBlockState struct {
leaseAt time.Time
blocked4, blocked6 []nftables.SetElement
allowed4, allowed6 []nftables.SetElement
blockedNet4, blockedNet6 []nftables.SetElement
}
// computeInitialBlockStateLocked reads state.json and returns the
// nft elements needed to repopulate the blocked / allowed / blocked-
// net sets. Pure computation; does not touch nft. Safe to call from
// Apply before any AddTable / AddSet so the result can be queued
// into the same atomic netlink batch.
func (e *Engine) computeInitialBlockStateLocked() initialBlockState {
now := e.expiryNow()
state := e.loadStateFileAtLocked(now)
return e.initialBlockStateFromRowsLocked(state, now)
}
// initialBlockStateFromRowsLocked also accepts a strictly read rollback image.
func (e *Engine) initialBlockStateFromRowsLocked(state FirewallState, now time.Time) initialBlockState {
// Timeouts come from the lease clock, which a step back of the wall
// clock cannot rewind: a reapplied block keeps only the time it had.
lease := e.leaseAtLocked(now)
if e.lifecycle != nil {
sets := desiredActionSets(state, e.cfg.IPv6, now)
return initialBlockState{leaseAt: lease, blocked4: actionElements(sets["blocked_ips"], lease), blocked6: actionElements(sets["blocked_ips6"], lease), allowed4: actionElements(sets["allowed_ips"], now), allowed6: actionElements(sets["allowed_ips6"], now), blockedNet4: actionElements(sets["blocked_nets"], now), blockedNet6: actionElements(sets["blocked_nets6"], now)}
}
// One reading for every row: a row must not pass the expiry check and
// then encode the time left after a later reading.
ibs := initialBlockState{leaseAt: lease}
restoredBlocked := make(map[string]bool)
for _, entry := range state.Blocked {
if entry.Unverified {
continue
}
timeout := time.Duration(0)
if !entry.ExpiresAt.IsZero() {
var ok bool
if timeout, ok = blockTimeout(entry.ExpiresAt, lease); !ok {
continue
}
}
key, ok := canonicalIPKey(entry.IP)
if !ok {
continue
}
if restoredBlocked[key] {
continue
}
parsed := net.ParseIP(entry.IP)
if ip4 := parsed.To4(); ip4 != nil {
ibs.blocked4 = append(ibs.blocked4, nftables.SetElement{Key: ip4, Timeout: timeout})
} else if e.cfg.IPv6 {
ibs.blocked6 = append(ibs.blocked6, nftables.SetElement{Key: parsed.To16(), Timeout: timeout})
}
restoredBlocked[key] = true
}
restoredAllowed := make(map[string]bool)
for _, entry := range state.Allowed {
if !entry.ExpiresAt.IsZero() && now.After(entry.ExpiresAt) {
continue
}
key, ok := canonicalIPKey(entry.IP)
if !ok {
continue
}
if restoredAllowed[key] {
continue
}
parsed := net.ParseIP(entry.IP)
if ip4 := parsed.To4(); ip4 != nil {
ibs.allowed4 = append(ibs.allowed4, nftables.SetElement{Key: ip4})
} else if e.cfg.IPv6 {
ibs.allowed6 = append(ibs.allowed6, nftables.SetElement{Key: parsed.To16()})
}
restoredAllowed[key] = true
}
ibs.blockedNet4, ibs.blockedNet6 = subnetIntervalElements(state.BlockedNet, e.cfg.IPv6, now)
return ibs
}
// queueInitialBlockStateLocked queues the previously-computed
// elements into the still-pending Apply netlink batch. Apply Flushes
// the whole batch as one transaction. No Flush here.
func (e *Engine) queueInitialBlockStateLocked(ibs initialBlockState) error {
// Table discovery and rule construction can consume the saved lifetime.
at := e.leaseNowLocked()
refresh := func(elements []nftables.SetElement) []nftables.SetElement {
live := elements[:0]
for _, el := range elements {
if el.Timeout > 0 {
timeout, ok := blockTimeout(ibs.leaseAt.Add(el.Timeout), at)
if !ok {
continue
}
el.Timeout = timeout
}
live = append(live, el)
}
return live
}
ibs.blocked4, ibs.blocked6 = refresh(ibs.blocked4), refresh(ibs.blocked6)
if err := e.addElementsChunked(e.setBlocked, ibs.blocked4); err != nil {
return err
}
if e.setBlocked6 != nil {
if err := e.addElementsChunked(e.setBlocked6, ibs.blocked6); err != nil {
return err
}
}
if err := e.addElementsChunked(e.setAllowed, ibs.allowed4); err != nil {
return err
}
if e.setAllowed6 != nil {
if err := e.addElementsChunked(e.setAllowed6, ibs.allowed6); err != nil {
return err
}
}
if err := e.addElementsChunked(e.setBlockedNet, ibs.blockedNet4); err != nil {
return err
}
if e.setBlockedNet6 != nil {
if err := e.addElementsChunked(e.setBlockedNet6, ibs.blockedNet6); err != nil {
return err
}
}
return nil
}
// addElementsChunked issues SetAddElements in fixed-size chunks. A single
// SetAddElements call encodes all elements into one netlink message whose
// size scales linearly with len(elems); the kernel's netlink socket rmem
// (typically 208 KB, tunable via net.core.rmem_max) caps how big that
// message can be before the receive path refuses it with ENOBUFS. At
// ~28 bytes per element worst-case a 1000-element chunk is ~28 KB, well
// under the default rmem and comfortably below any realistic rmem_max.
// The batch size must stay even so merged interval boundary pairs never
// split across chunks. Only the final range can have an open upper end.
func (e *Engine) addElementsChunked(s *nftables.Set, elems []nftables.SetElement) error {
return addElementsChunked(e.conn, s, elems)
}
func addElementsChunked(conn *nftables.Conn, s *nftables.Set, elems []nftables.SetElement) error {
const chunk = 1000
for i := 0; i < len(elems); i += chunk {
end := i + chunk
if end > len(elems) {
end = len(elems)
}
if err := conn.SetAddElements(s, elems[i:end]); err != nil {
op := fmt.Sprintf("add elements to set %q chunk %d-%d", s.Name, i, end)
logNftSetOpErr(op, "initial restore", err)
return fmt.Errorf("adding initial elements to set %q chunk %d-%d: %w", s.Name, i, end, err)
}
}
return nil
}
// logNftSetOpErr keeps operator-visible nft error logs grep-friendly.
func logNftSetOpErr(op, target string, err error) {
fmt.Fprintf(os.Stderr, "firewall: nft %s for %s failed: %v\n", op, target, err)
}
// loadStateFile returns a deep copy of the cached firewall state with
// expired entries pruned. The on-disk state.json is re-read only when
// the file metadata key differs from the cached key (or the cache is empty).
//
// All callers must hold e.mu. The returned value is safe to mutate
// without affecting the cache; mutators write back via saveState which
// rebuilds the cache from the passed-in struct.
//
// Hot read paths (IsBlocked, IsSubnetBlocked, IsAllowed) bypass this
// allocation by consulting the index maps directly.
func (e *Engine) loadStateFile() FirewallState {
return e.loadStateFileAtLocked(e.expiryNow())
}
func (e *Engine) loadStateFileAtLocked(now time.Time) FirewallState {
e.ensureStateCacheAtLocked(now)
if e.stateCache == nil {
return FirewallState{}
}
s := e.stateCache
return FirewallState{
Blocked: append([]BlockedEntry(nil), s.Blocked...),
BlockedNet: append([]SubnetEntry(nil), s.BlockedNet...),
Allowed: append([]AllowedEntry(nil), s.Allowed...),
PortAllowed: append([]PortAllowEntry(nil), s.PortAllowed...),
}
}
// loadStateFileRawLocked reads state.json without pruning expired entries.
// Expiry cleanup needs the stale rows so it can remove matching nftables
// elements before writing the active state back.
func (e *Engine) loadStateFileRawLocked() (FirewallState, bool) {
if e.lifecycle != nil {
state, _, err := e.readCommittedStateLocked()
return state, err == nil
}
if e.stateAsideErr != nil {
return FirewallState{}, false
}
stateFile := filepath.Join(e.statePath, "state.json")
data, err := os.ReadFile(stateFile) // #nosec G304 -- filepath.Join under operator-configured statePath.
if err != nil {
if os.IsNotExist(err) {
return FirewallState{}, true
}
e.ensureStateCacheLocked()
return FirewallState{}, false
}
var state FirewallState
if err := json.Unmarshal(data, &state); err != nil {
e.keepPriorStateCacheAtLocked(e.expiryNow(), stateFile, err)
return FirewallState{}, false
}
return state, true
}
// ensureStateCacheLocked populates or refreshes e.stateCache. Cheap on
// cache-hit (one stat). On cache-miss it does the full ReadFile +
// json.Unmarshal that the pre-cache implementation did on every call.
//
// Expired entries are pruned in-place after load so IsBlocked and the
// other index-backed lookups never report a stale block. Pruning
// updates the index maps via rebuildIndexLocked.
func (e *Engine) ensureStateCacheLocked() {
e.ensureStateCacheAtLocked(e.expiryNow())
}
func (e *Engine) ensureStateCacheAtLocked(now time.Time) {
if e.lifecycle != nil {
state, _, err := e.readCommittedStateLocked()
if err == nil {
e.installCommittedCacheAtLocked(state, now)
}
return
}
// Preservation failure is terminal for this engine. In particular, the
// port-allow read during a base install must not adopt replacement state.
if e.stateAsideErr != nil {
return
}
stateFile := filepath.Join(e.statePath, "state.json")
info, statErr := os.Stat(stateFile)
if statErr == nil && e.stateCache != nil && e.stateCacheKey.matches(info) {
e.applyExpiryAtLocked(now)
return
}
var fresh FirewallState
switch {
case statErr == nil:
// #nosec G304 -- filepath.Join under operator-configured statePath.
data, readErr := os.ReadFile(stateFile)
if readErr != nil {
e.keepPriorStateCacheAtLocked(now, stateFile, readErr)
return
}
if err := json.Unmarshal(data, &fresh); err != nil {
e.keepPriorStateCacheAtLocked(now, stateFile, err)
return
}
e.stateCacheKey = stateFileCacheKeyFromInfo(info)
case os.IsNotExist(statErr):
e.stateCacheKey = stateFileCacheKey{}
default:
if e.stateCache != nil {
e.applyExpiryAtLocked(now)
} else {
e.stateAsideErr = fmt.Errorf("%w: stat: %v", ErrStateNotKeptAside, statErr)
}
return
}
e.overlayUnverifiedBlockActionsLocked(&fresh)
normalizeFirewallStateIPs(&fresh)
e.stateCache = &fresh
e.applyExpiryAtLocked(now)
e.rebuildIndexLocked()
}
// keepPriorStateCacheAtLocked answers a state file the engine cannot read.
// A held cache stays, and the next write puts it back. Without one, the
// next write would replace the file and every entry in it, so the file is
// first renamed aside and the engine starts empty. If it cannot be moved,
// Apply refuses.
func (e *Engine) keepPriorStateCacheAtLocked(now time.Time, path string, cause error) {
if e.stateCache != nil {
e.applyExpiryAtLocked(now)
return
}
base := path + ".unreadable-" + now.UTC().Format("20060102T150405Z")
rename := e.renameStateAside
if rename == nil {
rename = func(old, kept string) error {
return unix.Renameat2(unix.AT_FDCWD, old, unix.AT_FDCWD, kept, unix.RENAME_NOREPLACE)
}
}
aside := base
for collision := 0; ; collision++ {
if collision > 0 {
aside = fmt.Sprintf("%s-%d", base, collision)
}
if err := rename(path, aside); err != nil {
if errors.Is(err, os.ErrExist) {
continue
}
e.stateAsideErr = fmt.Errorf("%w: %v; %v", ErrStateNotKeptAside, cause, err)
return
}
break
}
e.stateSetAside = aside
// The kept name must survive a crash before another state write.
syncDir := e.syncStateAside
if syncDir == nil {
syncDir = syncKeptStateDirectory
}
if err := syncDir(filepath.Dir(path)); err != nil {
e.stateAsideErr = fmt.Errorf("%w: sync kept state: %v", ErrStateNotKeptAside, err)
return
}
e.stateAsideErr = nil
e.stateCache = &FirewallState{}
e.stateCacheKey = stateFileCacheKey{}
e.rebuildIndexLocked()
}
func syncKeptStateDirectory(path string) error {
// #nosec G304 -- parent of the operator-configured state path.
dir, err := os.Open(path)
if err != nil {
return err
}
return errors.Join(dir.Sync(), dir.Close())
}
// StateSetAside names where an unreadable state file was kept when the
// engine first read its state, or "" when it read it.
func (e *Engine) StateSetAside() string {
e.mu.Lock()
defer e.mu.Unlock()
return e.stateSetAside
}
// applyExpiryAtLocked prunes expired entries from the cached state in
// place and rebuilds the index maps when something was dropped. It scans
// only when an entry is due or the cache was replaced since the last scan.
func (e *Engine) applyExpiryAtLocked(now time.Time) {
if e.stateCache == nil {
return
}
s := e.stateCache
if !e.expiryDue.reached(s, now) {
return
}
e.expiryScans++
changed := false
if pruned, dropped := pruneBlocked(s.Blocked, now); dropped {
s.Blocked = pruned
changed = true
}
if pruned, dropped := pruneBlockedNet(s.BlockedNet, now); dropped {
s.BlockedNet = pruned
changed = true
}
if pruned, dropped := pruneAllowed(s.Allowed, now); dropped {
s.Allowed = pruned
changed = true
}
if changed {
e.rebuildIndexLocked()
}
e.expiryDue = nextExpiryDeadline(s)
}
func (e *Engine) expiryNow() time.Time {
if e.expiryClock != nil {
return e.expiryClock()
}
return time.Now()
}
// expiryDeadline is bound to the cache snapshot it was computed for, so
// replacing stateCache (reload, save, committed install) forces a fresh
// scan without each writer having to reset it. Only applyExpiryAtLocked
// changes a cached snapshot in place, and it recomputes the deadline.
//
// A cached ExpiresAt carries a monotonic reading when the entry was made
// in this process and none when it was decoded from disk. The prune
// compares on the monotonic clock only when both sides carry one. Keep
// separate minima for those clocks, plus the wall minimum of all entries
// for a lookup without a monotonic reading. Use the original timestamps:
// synthesizing a bound with Sub/Add loses clock information and saturates
// for distant expiries, causing late expiry or unnecessary scans.
type expiryDeadline struct {
state *FirewallState
mono time.Time
wallOnly time.Time
wall time.Time
}
func (d expiryDeadline) reached(s *FirewallState, now time.Time) bool {
if d.state != s {
return true
}
wallNow := now.Round(0)
if now == wallNow { //nolint:staticcheck // Equal ignores monotonic metadata; struct equality detects its absence.
return !d.wall.IsZero() && !wallNow.Before(d.wall)
}
return (!d.mono.IsZero() && !now.Before(d.mono)) ||
(!d.wallOnly.IsZero() && !wallNow.Before(d.wallOnly))
}
func nextExpiryDeadline(s *FirewallState) expiryDeadline {
d := expiryDeadline{state: s}
consider := func(at time.Time) {
if at.IsZero() {
return
}
wall := at.Round(0)
if d.wall.IsZero() || wall.Before(d.wall) {
d.wall = wall
}
if at != wall {
if d.mono.IsZero() || at.Before(d.mono) {
d.mono = at
}
} else if d.wallOnly.IsZero() || wall.Before(d.wallOnly) {
d.wallOnly = wall
}
}
for _, entry := range s.Blocked {
if !entry.Unverified {
consider(entry.ExpiresAt)
}
}
for _, entry := range s.BlockedNet {
consider(entry.ExpiresAt)
}
for _, entry := range s.Allowed {
consider(entry.ExpiresAt)
}
return d
}
// rebuildIndexLocked refreshes the three lookup maps from the cached
// state. Must be called any time e.stateCache is mutated.
func (e *Engine) rebuildIndexLocked() {
if e.stateCache == nil {
e.blockedIPIndex = nil
e.allowedIPIndex = nil
e.portAllowedIndex = nil
e.blockedCIDRIndex = nil
return
}
s := e.stateCache
blocked := make(map[string]int, len(s.Blocked))
for i, entry := range s.Blocked {
if entry.Unverified {
continue
}
blocked[entry.IP] = i
if key, ok := canonicalIPKey(entry.IP); ok {
blocked[key] = i
}
}
e.blockedIPIndex = blocked
allowed := make(map[string]struct{}, len(s.Allowed))
for _, entry := range s.Allowed {
allowed[entry.IP] = struct{}{}
if key, ok := canonicalIPKey(entry.IP); ok {
allowed[key] = struct{}{}
}
}
e.allowedIPIndex = allowed
portAllowed := make(map[string]struct{}, len(s.PortAllowed))
for _, entry := range s.PortAllowed {
portAllowed[entry.IP] = struct{}{}
if key, ok := canonicalIPKey(entry.IP); ok {
portAllowed[key] = struct{}{}
}
}
e.portAllowedIndex = portAllowed
subnets := make(map[string]struct{}, len(s.BlockedNet))
for _, entry := range s.BlockedNet {
subnets[entry.CIDR] = struct{}{}
}
e.blockedCIDRIndex = subnets
}
// pruneBlocked retains unverified rows until their owner retires the
// matching attempt; expiry alone cannot settle an unknown kernel effect.
// Return the original slice when no enforced entry expired.
func pruneBlocked(in []BlockedEntry, now time.Time) ([]BlockedEntry, bool) {
expired := 0
for _, entry := range in {
if !entry.Unverified && !entry.ExpiresAt.IsZero() && !entry.ExpiresAt.After(now) {
expired++
}
}
if expired == 0 {
return in, false
}
out := make([]BlockedEntry, 0, len(in)-expired)
for _, entry := range in {
if !entry.Unverified && !entry.ExpiresAt.IsZero() && !entry.ExpiresAt.After(now) {
continue
}
out = append(out, entry)
}
return out, true
}
func pruneBlockedNet(in []SubnetEntry, now time.Time) ([]SubnetEntry, bool) {
expired := 0
for _, entry := range in {
if !entry.ExpiresAt.IsZero() && !entry.ExpiresAt.After(now) {
expired++
}
}
if expired == 0 {
return in, false
}
out := make([]SubnetEntry, 0, len(in)-expired)
for _, entry := range in {
if !entry.ExpiresAt.IsZero() && !entry.ExpiresAt.After(now) {
continue
}
out = append(out, entry)
}
return out, true
}
func pruneAllowed(in []AllowedEntry, now time.Time) ([]AllowedEntry, bool) {
expired := 0
for _, entry := range in {
if !entry.ExpiresAt.IsZero() && !entry.ExpiresAt.After(now) {
expired++
}
}
if expired == 0 {
return in, false
}
out := make([]AllowedEntry, 0, len(in)-expired)
for _, entry := range in {
if !entry.ExpiresAt.IsZero() && !entry.ExpiresAt.After(now) {
continue
}
out = append(out, entry)
}
return out, true
}
var writeFirewallStateJSON = atomicio.AtomicWriteJSON
// saveState writes the firewall state to disk atomically (write to .tmp,
// rename into place) and rebuilds the in-memory cache to reflect the
// just-written snapshot. Callers must hold e.mu.
//
// The cache rebuild deep-copies the input slices so a caller that
// keeps mutating the local FirewallState after saveState returns cannot
// corrupt the cache.
func (e *Engine) saveState(s *FirewallState) error {
if e.lifecycle != nil {
return e.runDurableLocked(ActionRequest{Operation: "state", Target: "*"}, nil, *s)
}
if e.stateCache == nil {
_, _ = e.loadStateFileRawLocked()
}
if e.stateAsideErr != nil {
return e.stateAsideErr
}
state := copyFirewallState(*s)
e.overlayUnverifiedBlockActionsLocked(&state)
normalizeFirewallStateIPs(&state)
return e.writeStateLocked(state)
}
// writeStateLocked persists a prepared state without merging rows. Callers
// must first load the prior state and preserve any pending action marks.
func (e *Engine) writeStateLocked(state FirewallState) error {
path := filepath.Join(e.statePath, "state.json")
if err := writeFirewallStateJSON(path, 0o600, &state); err != nil {
if firewallStateFileMatches(path, 0o600, &state) {
e.setStateCacheLocked(path, &state)
return &stateDurabilityError{cause: err}
}
fmt.Fprintf(os.Stderr, "firewall: persist state.json failed: %v\n", err)
// A failed write must not make a held engine fresh again: unreadable
// external state still needs the last known snapshot for the retry.
e.stateCacheKey = stateFileCacheKey{}
return err
}
e.setStateCacheLocked(path, &state)
e.unverifiedBlockActions = nil
e.failedBlockActions = nil
e.blockBatchStateErr = nil
e.blockRemovalPending = false
return nil
}
func firewallStateFileMatches(path string, perm os.FileMode, s *FirewallState) bool {
info, err := os.Stat(path)
if err != nil || info.Mode().Perm() != perm {
return false
}
want, err := json.MarshalIndent(s, "", " ")
if err != nil {
return false
}
// #nosec G304 -- path is the engine-owned state file path.
got, err := os.ReadFile(path)
if err != nil {
return false
}
return bytes.Equal(got, want)
}
func (e *Engine) setStateCacheLocked(path string, s *FirewallState) {
var cacheKey stateFileCacheKey
if info, statErr := os.Stat(path); statErr == nil {
cacheKey = stateFileCacheKeyFromInfo(info)
}
state := copyFirewallState(*s)
e.overlayUnverifiedBlockActionsLocked(&state)
normalizeFirewallStateIPs(&state)
e.stateCache = &FirewallState{
Blocked: append([]BlockedEntry(nil), state.Blocked...),
BlockedNet: append([]SubnetEntry(nil), state.BlockedNet...),
Allowed: append([]AllowedEntry(nil), state.Allowed...),
PortAllowed: append([]PortAllowEntry(nil), state.PortAllowed...),
}
e.stateCacheKey = cacheKey
e.rebuildIndexLocked()
}
func (e *Engine) clearStateCacheLocked() {
e.stateCache = nil
e.stateCacheKey = stateFileCacheKey{}
e.rebuildIndexLocked()
}
func copyFirewallState(s FirewallState) FirewallState {
return FirewallState{
Blocked: append([]BlockedEntry(nil), s.Blocked...),
BlockedNet: append([]SubnetEntry(nil), s.BlockedNet...),
Allowed: append([]AllowedEntry(nil), s.Allowed...),
PortAllowed: append([]PortAllowEntry(nil), s.PortAllowed...),
}
}
func upsertBlockedEntryInState(state *FirewallState, entry BlockedEntry) {
if key, ok := canonicalIPKey(entry.IP); ok {
entry.IP = key
}
out := state.Blocked[:0]
written := false
for _, existing := range state.Blocked {
if sameIPString(existing.IP, entry.IP) {
if !written {
out = append(out, entry)
written = true
}
continue
}
out = append(out, existing)
}
if !written {
out = append(out, entry)
}
state.Blocked = out
}
func removeBlockedIPFromState(state *FirewallState, ip string) {
remaining := state.Blocked[:0]
for _, entry := range state.Blocked {
if sameIPString(entry.IP, ip) {
continue
}
remaining = append(remaining, entry)
}
state.Blocked = remaining
}
func upsertAllowedEntryInState(state *FirewallState, entry AllowedEntry) {
if key, ok := canonicalIPKey(entry.IP); ok {
entry.IP = key
}
out := state.Allowed[:0]
written := false
for _, existing := range state.Allowed {
if sameIPString(existing.IP, entry.IP) && existing.Source == entry.Source {
if !written {
out = append(out, entry)
written = true
}
continue
}
out = append(out, existing)
}
if !written {
out = append(out, entry)
}
state.Allowed = out
}
func addSubnetEntryIfMissingInState(state *FirewallState, entry SubnetEntry) bool {
for _, existing := range state.BlockedNet {
if existing.CIDR == entry.CIDR {
return false
}
}
state.BlockedNet = append(state.BlockedNet, entry)
return true
}
func isNftNotFound(err error) bool {
return errors.Is(err, syscall.ENOENT)
}
func (e *Engine) restoreBlockStateAfterFailureLocked(state FirewallState, ip string) error {
if err := e.saveState(&state); err != nil {
fmt.Fprintf(os.Stderr, "firewall: restore state after failed block for %s failed: %v\n", ip, err)
return err
}
return nil
}
func (e *Engine) saveBlockedEntry(entry BlockedEntry) error {
if entry.Source == "" {
entry.Source = InferProvenance("block", entry.Reason)
}
if key, ok := canonicalIPKey(entry.IP); ok {
entry.IP = key
}
state := e.loadStateFile()
upsertBlockedEntryInState(&state, entry)
return e.saveState(&state)
}
func (e *Engine) removeBlockedState(ip string) error {
state := e.loadStateFile()
var remaining []BlockedEntry
for _, entry := range state.Blocked {
if !sameIPString(entry.IP, ip) {
remaining = append(remaining, entry)
}
}
state.Blocked = remaining
return e.saveState(&state)
}
func (e *Engine) saveAllowedEntry(entry AllowedEntry) error {
if entry.Source == "" {
entry.Source = InferProvenance("allow", entry.Reason)
}
if key, ok := canonicalIPKey(entry.IP); ok {
entry.IP = key
}
state := e.loadStateFile()
upsertAllowedEntryInState(&state, entry)
return e.saveState(&state)
}
func (e *Engine) removeAllowedState(ip string) error {
state := e.loadStateFile()
var remaining []AllowedEntry
for _, entry := range state.Allowed {
if !sameIPString(entry.IP, ip) {
remaining = append(remaining, entry)
}
}
state.Allowed = remaining
return e.saveState(&state)
}
// removeAllowedStateBySource reports whether the final source was removed.
func (e *Engine) removeAllowedStateBySource(ip, source string) (bool, error) {
state := e.loadStateFile()
found, ipGone := removeAllowedSourceFromState(&state, ip, source)
if !found {
return false, nil
}
if err := e.saveState(&state); err != nil {
return false, err
}
return ipGone, nil
}
func removeAllowedSourceFromState(state *FirewallState, ip, source string) (found, ipGone bool) {
remaining := state.Allowed[:0]
ipGone = true
for _, entry := range state.Allowed {
if sameIPString(entry.IP, ip) && entry.Source == source {
found = true
continue
}
remaining = append(remaining, entry)
if sameIPString(entry.IP, ip) {
ipGone = false
}
}
state.Allowed = remaining
return found, ipGone
}
func (e *Engine) saveSubnetEntry(entry SubnetEntry) error {
if entry.Source == "" {
entry.Source = InferProvenance("block_subnet", entry.Reason)
}
state := e.loadStateFile()
if !addSubnetEntryIfMissingInState(&state, entry) {
return nil
}
return e.saveState(&state)
}
func (e *Engine) isSubnetBlockedStateLocked(cidr string) bool {
e.ensureStateCacheLocked()
_, ok := e.blockedCIDRIndex[cidr]
return ok
}
func (e *Engine) removeSubnetState(cidr string) error {
state := e.loadStateFile()
var remaining []SubnetEntry
for _, entry := range state.BlockedNet {
if entry.CIDR != cidr {
remaining = append(remaining, entry)
}
}
state.BlockedNet = remaining
return e.saveState(&state)
}
// IP helpers (nextIP, lastIPInRange, fileExistsFirewall) moved to ip_helpers.go (no build tag).
// loadCountryCIDRs reads CIDR ranges from a country file.
// Expected format: one CIDR per line in {dbPath}/{CODE}.cidr
func loadCountryCIDRs(dbPath, countryCode string) []nftables.SetElement {
file := filepath.Join(dbPath, strings.ToUpper(countryCode)+".cidr")
// #nosec G304 -- filepath.Join under operator-configured GeoIP dbPath.
data, err := os.ReadFile(file)
if err != nil {
return nil
}
var elements []nftables.SetElement
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
_, network, err := net.ParseCIDR(line)
if err != nil {
continue
}
start := network.IP.To4()
end := lastIPInRange(network)
if start != nil && end != nil {
elements = appendIntervalSetElements(elements, start, end)
}
}
return elements
}
// loadCountryCIDRs6 reads IPv6 CIDR ranges from {dbPath}/{CODE}.cidr6 and
// builds interval set elements. v4 CIDRs in the file (if any) are skipped so
// only IPv6 ranges reach the IPv6 set.
func loadCountryCIDRs6(dbPath, countryCode string) []nftables.SetElement {
file := filepath.Join(dbPath, strings.ToUpper(countryCode)+".cidr6")
// #nosec G304 -- filepath.Join under operator-configured GeoIP dbPath.
data, err := os.ReadFile(file)
if err != nil {
return nil
}
var elements []nftables.SetElement
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
_, network, err := net.ParseCIDR(line)
if err != nil {
continue
}
if network.IP.To4() != nil {
continue // not an IPv6 range
}
start := network.IP.To16()
end := lastIPInRange(network)
if start != nil && end != nil {
elements = appendIntervalSetElements(elements, start, end.To16())
}
}
return elements
}
//go:build linux
package firewall
import (
"errors"
"fmt"
"net"
"slices"
"time"
"unicode/utf8"
)
// BlockBatchResult is what a durable block batch did: the action it ran or
// replayed, and, in call order, why each member was left out (empty when the
// member ran). A replayed batch reports no refusals.
type BlockBatchResult struct {
Action FirewallAction
Refusals []BlockRefusal
}
// BlockBatchRequest runs automatic temporary blocks as one durable batch.
// Each member is checked on its own against one committed
// snapshot and refused members are left out; the rest share one admission,
// one kernel transaction and one outcome. A lone valid member runs as an
// ordinary one-request action under its own ID. A repeated call naming a
// stored batch observes it and never plans it again. Nothing evicts for a
// batch: a full temporary set refuses the member.
func (e *Engine) BlockBatchRequest(header ActionRequest, members []ActionRequest, budget *ScanAdmission) (BlockBatchResult, error) {
if !validBlockBatchHeader(header) || len(members) == 0 || len(members) > MaxActionMembers {
return BlockBatchResult{}, errors.New("invalid firewall block batch")
}
// Dry run refuses new work only; read it before taking the lock.
dryRun := e.autoResponseDryRunEnabled()
members = slices.Clone(members)
// The verified-range callback runs outside the engine lock, as it does
// for a single automatic block.
verified := make([]bool, len(members))
for i := range members {
if canonical, err := canonicalFirewallIP(members[i].Target); err == nil {
members[i].Target = canonical
verified[i] = e.autoBlockVerifiedRange(canonical)
}
}
e.mu.Lock()
defer e.mu.Unlock()
if e.lifecycle == nil {
return BlockBatchResult{}, errors.New("firewall lifecycle unavailable")
}
if replayed, result, err := e.replayBlockBatchLocked(header, members); replayed || err != nil {
return result, err
}
if replayed, result, err := e.replayBlockSingleLocked(header, members); replayed || err != nil {
return result, err
}
if dryRun {
return BlockBatchResult{}, ErrActionDryRun
}
if err := e.lifecycleReadyLocked(); err != nil {
return BlockBatchResult{}, err
}
prior, _, err := e.readCommittedStateLocked()
if err != nil {
return BlockBatchResult{}, err
}
now := e.expiryNow()
at := e.leaseAtLocked(now)
result := BlockBatchResult{Refusals: e.checkBlockBatchLocked(header, members, verified, prior, at, now)}
next := copyFirewallState(prior)
var ran []ActionMember
for i, m := range members {
if result.Refusals[i] != "" {
continue
}
ran = append(ran, ActionMember{Request: m, Budget: budget})
next.Blocked = append(next.Blocked, BlockedEntry{IP: m.Target, Reason: m.Reason, Source: InferProvenance("block", m.Reason), BlockedAt: at, ExpiresAt: at.Add(m.TTL), ActionID: m.ID, Automatic: m.Automatic, LoginSpray: m.LoginSpray})
}
if len(ran) == 0 {
return result, nil
}
// The new deadlines may use a forward-ratcheted lease, as a single
// block's does: save its clock before the intent.
if leaseErr := e.saveLeaseLocked(); leaseErr != nil {
return result, fmt.Errorf("saving block lease: %w", leaseErr)
}
plan := FirewallAction{Request: header, Members: ran, After: next, CreatedAt: at}
if len(ran) == 1 {
// A lone member writes the record a single block writes.
next.Blocked[len(next.Blocked)-1].ActionID = ""
plan = FirewallAction{Request: ran[0].Request, Budget: budget, After: next, CreatedAt: at}
}
result.Action, err = e.runDurablePlanLocked(plan)
return result, err
}
// blockBatchMinLifetime is the shortest member lifetime a batch plans. It
// covers the kernel's timeout granularity and the admission, apply and
// readback that run before verification: a member that expires first leaves
// the batch unknown, holding every durable mutation until it is resolved.
const blockBatchMinLifetime = blockReadbackSlack + 2*time.Second
// validBlockBatchHeader holds a batch header to its required shape: an
// automatic block with no target, link, lifetime or charge of its own.
func validBlockBatchHeader(h ActionRequest) bool {
return validBlockAction(h.ID) && h.Operation == "block" && h.Automatic && h.Actor != "" && h.Source != "" && utf8.ValidString(h.Actor) && utf8.ValidString(h.ActorDetail) && utf8.ValidString(h.Source) &&
h.Target == "" && h.Reason == "" && h.FindingID == "" && h.IncidentID == "" && h.UndoOf == "" && h.TTL == 0
}
// replayBlockBatchLocked observes a batch already stored under the header's
// ID. The call must name that batch's members, in order, among its own.
func (e *Engine) replayBlockBatchLocked(header ActionRequest, members []ActionRequest) (bool, BlockBatchResult, error) {
stored, err := e.lifecycle.Store.ReadFirewallAction(header.ID)
if errors.Is(err, ErrActionMissing) {
return false, BlockBatchResult{}, nil
}
if err != nil {
return true, BlockBatchResult{}, err
}
next := 0
for _, m := range members {
if next < len(stored.Members) && stored.Members[next].Request == m {
next++
}
}
if stored.Request != header || len(stored.Members) < 2 || next != len(stored.Members) {
return true, BlockBatchResult{}, ErrStateConflict
}
a, executeErr := e.lifecycle.Execute(stored, engineActionKernel{e})
current, _, readErr := e.readCommittedStateLocked()
if readErr == nil {
e.installCommittedCache(current)
}
return true, BlockBatchResult{Action: a}, errors.Join(durableActionOutcome(a, executeErr), readErr)
}
// replayBlockSingleLocked handles the single-request shape whose header was
// never stored. The known member is immutable; other IDs do not become
// part of the stored action merely because policy now permits them. Group
// occurrences before reading so each ID is decoded once without retaining
// every stored action's snapshots. A later occurrence may be the exact request.
func (e *Engine) replayBlockSingleLocked(header ActionRequest, members []ActionRequest) (bool, BlockBatchResult, error) {
var stored FirewallAction
found, known := false, false
occurrences := make(map[string][]ActionRequest, len(members))
for _, m := range members {
occurrences[m.ID] = append(occurrences[m.ID], m)
}
for _, m := range members {
candidates, unread := occurrences[m.ID]
if !unread {
continue
}
delete(occurrences, m.ID)
a, err := e.lifecycle.Store.ReadFirewallAction(m.ID)
if errors.Is(err, ErrActionMissing) {
continue
}
if err != nil {
return true, BlockBatchResult{}, err
}
known = true
matched := false
for _, candidate := range candidates {
if !validBlockBatchMember(header, candidate) || !a.IsRequest(candidate) {
continue
}
matched = true
break
}
if !matched {
continue
}
if found && stored.Request.ID != a.Request.ID {
return true, BlockBatchResult{}, ErrStateConflict
}
stored, found = a, true
}
if !found {
if known {
return true, BlockBatchResult{}, ErrStateConflict
}
return false, BlockBatchResult{}, nil
}
a, executeErr := e.lifecycle.Execute(stored, engineActionKernel{e})
current, _, readErr := e.readCommittedStateLocked()
if readErr == nil {
e.installCommittedCache(current)
}
return true, BlockBatchResult{Action: a}, errors.Join(durableActionOutcome(a, executeErr), readErr)
}
// checkBlockBatchLocked answers every member against the committed snapshot.
// A refused member takes no slot and names nothing a later member collides
// with.
func (e *Engine) checkBlockBatchLocked(header ActionRequest, members []ActionRequest, verified []bool, prior FirewallState, at, now time.Time) []BlockRefusal {
// Reading the cache per member would replace the planning revision and
// could admit an old After against newer state. Allows share the snapshot;
// their expiry still follows wall time, independently of block leases.
allowed := make(map[string]bool, len(prior.Allowed)+len(prior.PortAllowed))
for _, row := range prior.Allowed {
if row.ExpiresAt.IsZero() || row.ExpiresAt.After(now) {
allowed[stateIPKey(row.IP)] = true
}
}
for _, row := range prior.PortAllowed {
allowed[stateIPKey(row.IP)] = true
}
rows := make(map[string]BlockedEntry, len(prior.Blocked))
temp := 0
for _, row := range prior.Blocked {
// Expiry cannot settle an unverified row; its owner must.
if row.Unverified || row.ExpiresAt.IsZero() || row.ExpiresAt.After(at) {
if key := stateIPKey(row.IP); !rows[key].Unverified {
rows[key] = row
}
}
if !row.Unverified && row.ExpiresAt.After(at) {
temp++
}
}
out := make([]BlockRefusal, len(members))
seenIP, seenID := map[string]bool{}, map[string]bool{header.ID: true}
for i, m := range members {
out[i] = e.blockBatchRefusalLocked(header, m, verified[i] || allowed[m.Target], rows, at, seenIP, seenID)
if out[i] != "" {
continue
}
if e.cfg.DenyTempIPLimit > 0 && temp >= e.cfg.DenyTempIPLimit {
out[i] = BlockRefusedFull
continue
}
temp++
seenIP[m.Target], seenID[m.ID] = true, true
}
return out
}
func validBlockBatchMember(header, m ActionRequest) bool {
parsed := net.ParseIP(m.Target)
if parsed == nil || !validBlockAction(m.ID) || !validBlockReason(m.Reason) || !utf8.ValidString(m.FindingID) || !utf8.ValidString(m.IncidentID) ||
m.Operation != "block" || !m.Automatic || m.TTL <= 0 || m.UndoOf != "" || m.Actor != header.Actor || m.ActorDetail != header.ActorDetail || m.Source != header.Source {
return false
}
return true
}
func (e *Engine) blockBatchRefusalLocked(header, m ActionRequest, softAllowed bool, rows map[string]BlockedEntry, at time.Time, seenIP, seenID map[string]bool) BlockRefusal {
parsed := net.ParseIP(m.Target)
if !validBlockBatchMember(header, m) {
return BlockRefusedInvalid
}
if _, ok := blockTimeout(at.Add(m.TTL), at); !ok || m.TTL < blockBatchMinLifetime {
return BlockRefusedExpired
}
if parsed.To4() == nil && (!e.cfg.IPv6 || e.setBlocked6 == nil) {
return BlockRefusedUnsupported
}
if e.blockProtectionLocked(parsed, m.Target) != nil {
return BlockRefusedProtected
}
if softAllowed {
return BlockRefusedAllowed
}
if seenIP[m.Target] || seenID[m.ID] {
return BlockRefusedDuplicate
}
if row, held := rows[m.Target]; held {
if !row.Unverified && CoversExpiry(row.ExpiresAt, at.Add(m.TTL)) {
return BlockRefusedExisting
}
return BlockRefusedPending
}
return ""
}
//go:build linux
package firewall
import (
"bytes"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net"
"reflect"
"strings"
"time"
"github.com/google/nftables"
"github.com/pidginhost/csm/internal/actionlog"
)
// AttachLifecycle is a construction-time injection, not a migration. The owner
// must already have initialized the complete store and own the daemon lock.
func (e *Engine) AttachLifecycle(l *Lifecycle) error {
e.mu.Lock()
defer e.mu.Unlock()
if l == nil || l.Store == nil || e.lifecycle != nil {
return errors.New("invalid firewall lifecycle attachment")
}
state, revision, err := l.Store.ReadFirewallState()
if err != nil {
return err
}
if l.Audit == nil {
l.Audit = writeFirewallActionAudit
}
e.lifecycle = l
e.stateRevision = revision
e.installCommittedCache(state)
return nil
}
// writeFirewallActionAudit records one event per member and one evict_temp
// event per row a member removed to make room. Several events go in one
// durable write, each with the action's outcome and version.
func writeFirewallActionAudit(a FirewallAction) error {
var records []actionlog.Record
for _, m := range a.ActionMembers() {
records = append(records, firewallAuditRecord(a, m.Request))
for _, v := range removedVictims(a, m) {
r := firewallAuditRecord(a, m.Request)
r.Action, r.Target, r.Reason = "evict_temp", v.IP, evictTempReason
records = append(records, r)
}
}
if len(records) == 1 {
return actionlog.WriteDurable(records[0])
}
return actionlog.WriteDurableBatch(records)
}
func firewallAuditRecord(a FirewallAction, req ActionRequest) actionlog.Record {
result := actionlog.Result(a.Phase)
if a.Retired {
result = "expired_unproven"
}
return actionlog.Record{ActionID: req.ID, ActionVersion: a.AuditVersion, IncidentID: req.IncidentID, UndoOf: req.UndoOf, Timestamp: a.UpdatedAt, Op: firewallAuditOperation(req), Action: req.Operation, Actor: actionlog.Actor(req.Actor), ActorDetail: req.ActorDetail, FindingID: req.FindingID, Target: req.Target, Reason: req.Reason, Result: result, Error: a.Detail}
}
func (e *Engine) lifecycleEnabled() bool { e.mu.Lock(); defer e.mu.Unlock(); return e.lifecycle != nil }
func (e *Engine) installCommittedCache(state FirewallState) {
e.installCommittedCacheAtLocked(state, e.expiryNow())
}
func (e *Engine) installCommittedCacheAtLocked(state FirewallState, now time.Time) {
state = copyFirewallState(state)
e.overlayUnverifiedBlockActionsLocked(&state)
e.stateCache = &state
e.applyExpiryAtLocked(now)
e.rebuildIndexLocked()
}
func (e *Engine) lifecycleReadyLocked() error {
if e.lifecycle == nil {
return nil
}
pending, err := e.lifecycle.Store.PendingFirewallActions()
if err != nil {
return err
}
if len(pending) > 0 {
return fmt.Errorf("%w: %s", ErrActionUnknown, pending[0].Request.ID)
}
state, _, err := e.readCommittedStateLocked()
if err != nil {
return err
}
e.installCommittedCache(state)
return nil
}
func (e *Engine) RecoverActions() error {
e.mu.Lock()
defer e.mu.Unlock()
if e.lifecycle == nil {
return errors.New("firewall lifecycle unavailable")
}
recoverErr := e.lifecycle.Recover(engineActionKernel{e})
state, _, readErr := e.readCommittedStateLocked()
if readErr == nil {
e.installCommittedCache(state)
}
return errors.Join(recoverErr, readErr)
}
// PendingActions reports the actions recovery could not settle. While any
// exists the engine refuses new mutations, so this is what an operator needs
// to see before deciding anything.
func (e *Engine) PendingActions() ([]FirewallAction, error) {
e.mu.Lock()
defer e.mu.Unlock()
if e.lifecycle == nil {
return nil, errors.New("firewall lifecycle unavailable")
}
return e.lifecycle.Store.PendingFirewallActions()
}
// ResolveAction records an outcome an operator established by hand for an
// action the kernel cannot prove. The committed cache follows the outcome,
// whether it came from the operator or from the kernel proving it after all.
func (e *Engine) ResolveAction(id, outcome, detail string) (FirewallAction, error) {
e.mu.Lock()
defer e.mu.Unlock()
if e.lifecycle == nil {
return FirewallAction{}, errors.New("firewall lifecycle unavailable")
}
a, err := e.lifecycle.Resolve(id, outcome, detail, engineActionKernel{e})
state, _, readErr := e.readCommittedStateLocked()
if readErr == nil {
e.installCommittedCache(state)
}
return a, errors.Join(durableActionOutcome(a, err), readErr)
}
func (e *Engine) UndoAction(req ActionRequest) (FirewallAction, error) {
e.mu.Lock()
defer e.mu.Unlock()
if e.lifecycle == nil {
return FirewallAction{}, errors.New("firewall lifecycle unavailable")
}
a, err := e.lifecycle.undoWithClock(req, engineActionKernel{e}, e.leaseNowLocked)
err = durableActionOutcome(a, err)
if a.Phase == "verified" {
current, _, readErr := e.readCommittedStateLocked()
if readErr != nil {
return a, errors.Join(err, readErr)
}
e.installCommittedCache(current)
}
return a, err
}
func (e *Engine) runDurableLocked(req ActionRequest, budget *ScanAdmission, next FirewallState) error {
_, err := e.runDurablePlanLocked(FirewallAction{Request: req, After: next, Budget: budget})
return err
}
// runDurablePlanLocked admits and executes a plan the caller made from the
// committed snapshot it last read. Its request, members, budget and After
// state are the caller's; Before, revision and kernel evidence are read here.
// Only an admitted attempt's error speaks for its durable outcome; a refusal
// before admission keeps the caller's refusal record.
func (e *Engine) runDurablePlanLocked(plan FirewallAction) (FirewallAction, error) {
if e.stateReadErr != nil {
return FirewallAction{}, e.stateReadErr
}
expectedRevision := e.stateRevision
if err := e.lifecycleReadyLocked(); err != nil {
return FirewallAction{}, err
}
prior, revision, err := e.readCommittedStateLocked()
if err != nil {
return FirewallAction{}, err
}
if revision != expectedRevision {
return FirewallAction{}, fmt.Errorf("%w: planning snapshot changed", ErrStateConflict)
}
if plan.Request.ID == "" {
plan.Request.ID = rand.Text()
}
plan.Request = normalizeActionRequest(plan.Request)
if len(plan.Members) == 1 {
// A one-member manifest repeats its header exactly.
plan.Members[0].Request = plan.Request
}
plan.Before, plan.Revision = prior, revision
at := plan.CreatedAt
if at.IsZero() {
at = e.expiryNow()
}
if prepareErr := e.prepareActionKernelAt(&plan, at); prepareErr != nil {
return FirewallAction{}, prepareErr
}
a, err := e.lifecycle.Execute(plan, engineActionKernel{e})
if a.Phase == "verified" {
e.installCommittedCache(a.After)
}
return a, durableActionOutcome(a, err)
}
func stateFingerprint(value any) string {
raw, _ := json.Marshal(value)
sum := sha256.Sum256(raw)
return "csm:" + hex.EncodeToString(sum[:])
}
func (e *Engine) actionSet(name string) *nftables.Set {
for _, set := range []*nftables.Set{e.setBlocked, e.setBlocked6, e.setAllowed, e.setAllowed6, e.setBlockedNet, e.setBlockedNet6} {
if set != nil && set.Name == name {
return set
}
}
return nil
}
func actionSetNames(ipv6 bool) []string {
names := []string{"blocked_ips", "allowed_ips", "blocked_nets"}
if ipv6 {
names = append(names, "blocked_ips6", "allowed_ips6", "blocked_nets6")
}
return names
}
func desiredActionSets(state FirewallState, ipv6 bool, now time.Time) map[string][]ActionElement {
out := make(map[string][]ActionElement)
for _, name := range actionSetNames(ipv6) {
out[name] = nil
}
for _, b := range state.Blocked {
if b.Unverified || (!b.ExpiresAt.IsZero() && !b.ExpiresAt.After(now)) {
continue
}
name, key, ok := blockedSetKey(b.IP, ipv6)
if !ok {
continue
}
out[name] = append(out[name], ActionElement{Key: key, Comment: stateFingerprint(b), ExpiresAt: b.ExpiresAt})
}
allowed := make(map[string][]AllowedEntry)
for _, a := range state.Allowed {
if a.ExpiresAt.IsZero() || a.ExpiresAt.After(now) {
allowed[a.IP] = append(allowed[a.IP], a)
}
}
for address, rows := range allowed {
ip := net.ParseIP(address)
if ip == nil {
continue
}
name := "allowed_ips"
key := ip.To4()
if key == nil {
if !ipv6 {
continue
}
name += "6"
key = ip.To16()
}
out[name] = append(out[name], ActionElement{Key: key, Comment: stateFingerprint(rows)})
}
var activeSubnets []SubnetEntry
for _, entry := range state.BlockedNet {
if entry.ExpiresAt.IsZero() || entry.ExpiresAt.After(now) {
activeSubnets = append(activeSubnets, entry)
}
}
v4, v6 := subnetIntervalElements(activeSubnets, ipv6, now)
for i, elems := range [][]nftables.SetElement{v4, v6} {
name := "blocked_nets"
if i == 1 {
name += "6"
}
for _, elem := range elems {
comment := ""
// Interval-end sentinels cannot carry element userdata. The start
// marker identifies the complete union, including source expiry.
if !elem.IntervalEnd {
comment = stateFingerprint(activeSubnets)
}
out[name] = append(out[name], ActionElement{Key: elem.Key, End: elem.IntervalEnd, Comment: comment})
}
}
return out
}
// blockedSetKey names the blocked set and element key for an address.
func blockedSetKey(address string, ipv6 bool) (string, []byte, bool) {
ip := net.ParseIP(address)
if ip == nil {
return "", nil, false
}
if key := ip.To4(); key != nil {
return "blocked_ips", key, true
}
if !ipv6 {
return "", nil, false
}
return "blocked_ips6", ip.To16(), true
}
func (e *Engine) prepareActionKernel(a *FirewallAction) error {
return e.prepareActionKernelAt(a, e.expiryNow())
}
// prepareActionKernelAt keeps a planner's chosen instant for all its evidence.
func (e *Engine) prepareActionKernelAt(a *FirewallAction, now time.Time) error {
a.CreatedAt = now
desired := desiredActionSets(a.After, e.setBlocked6 != nil, now)
expected := desiredActionSets(a.Before, e.setBlocked6 != nil, now)
unpruned := desiredActionSets(a.Before, e.setBlocked6 != nil, time.Time{})
conn, connErr := newLifecycleConn(e)
if connErr != nil {
return connErr
}
names := requiredActionSets(*a)
a.KernelBefore = nil
a.KernelAfter = nil
for _, name := range names {
at := e.leaseNowLocked()
actual, readErr := readActionSet(conn, name)
if readErr != nil {
return readErr
}
elems := actual.elements
if !actual.exists && e.actionSet(name) != nil {
return fmt.Errorf("%w: set %s disappeared", ErrActionUnknown, name)
}
if !matchActionElements(expected[name], elems, at) && !matchActionElements(unpruned[name], elems, at) {
return fmt.Errorf("%w: live set %s differs from committed state", ErrActionUnknown, name)
}
before := ActionSet{Name: name, Exists: actual.exists}
now := e.leaseNowLocked()
for _, elem := range elems {
entry := ActionElement{Key: bytes.Clone(elem.Key), End: elem.IntervalEnd, Comment: elem.Comment}
if elem.Timeout > 0 {
entry.ExpiresAt = now.Add(elem.Expires)
for _, known := range expected[name] {
if bytes.Equal(known.Key, entry.Key) && known.Comment == entry.Comment {
entry.ExpiresAt = known.ExpiresAt
break
}
}
}
before.Elements = append(before.Elements, entry)
}
a.KernelBefore = append(a.KernelBefore, before)
a.KernelAfter = append(a.KernelAfter, ActionSet{Name: name, Exists: actual.exists, Elements: desired[name]})
}
return nil
}
type engineActionKernel struct{ e *Engine }
func (k engineActionKernel) ApplyFirewallAction(a FirewallAction) error {
if a.Request.Operation == "apply" {
conn, err := newLifecycleConn(k.e)
if err != nil {
return err
}
priorConn := k.e.conn
k.e.conn = conn
defer func() { k.e.conn = priorConn }()
return k.e.applyRulesetLocked(a.Ruleset.Marker)
}
if err := k.applyActionSets(a, true); err != nil {
if len(a.Members) > 1 || !isNftNotFound(err) {
return err
}
// The kernel expires timed elements on its own, so a delete can name an
// element that is already gone. That batch changed nothing, so rewrite
// the complete set instead of reporting an uncertain outcome.
return k.applyActionSets(a, false)
}
return nil
}
// applyActionSets writes the intended effect of one action. A whole-set rewrite
// costs one message per retained element, which on a busy host is most of the
// work a single block does, so unchanged elements are left alone where the set
// allows it.
func (k engineActionKernel) applyActionSets(a FirewallAction, delta bool) error {
conn, connErr := newLifecycleConn(k.e)
if connErr != nil {
return connErr
}
now := k.e.leaseNowLocked()
for i, state := range a.KernelAfter {
if !state.Exists {
continue
}
set := k.e.actionSet(state.Name)
if set == nil {
return fmt.Errorf("firewall recovery set unavailable: %s", state.Name)
}
if !delta || i >= len(a.KernelBefore) || !deltaApplicable(a.KernelBefore[i], state) {
conn.FlushSet(set)
if err := addElementsChunked(conn, set, actionElements(state.Elements, now)); err != nil {
return err
}
continue
}
add, remove := actionElementDelta(a.KernelBefore[i], state, now)
// Each element list has a uint16 netlink attribute length. Bound
// deletion messages too, while keeping all chunks in one transaction.
for offset := 0; offset < len(remove); offset += 1000 {
end := min(offset+1000, len(remove))
if err := conn.SetDeleteElements(set, remove[offset:end]); err != nil {
return err
}
}
if err := addElementsChunked(conn, set, add); err != nil {
return err
}
}
return conn.Flush()
}
// deltaApplicable reports whether a set can be changed element by element.
// Interval sets carry paired start and end markers whose union changes shape
// when any member changes, so those are always rewritten whole.
func deltaApplicable(before, after ActionSet) bool {
if !before.Exists || before.Name != after.Name {
return false
}
for _, set := range []ActionSet{before, after} {
for _, elem := range set.Elements {
if elem.End {
return false
}
}
}
return true
}
func actionElementKey(elem ActionElement) string {
return fmt.Sprintf("%x/%t", elem.Key, elem.End)
}
// actionElementDelta returns the elements to add and to remove so the live set
// matches the intended effect. An element whose comment or expiry changed is
// removed and re-added in the same batch, which nftables applies atomically.
func actionElementDelta(before, after ActionSet, now time.Time) (add, remove []nftables.SetElement) {
live := make(map[string]ActionElement, len(before.Elements))
for _, elem := range before.Elements {
live[actionElementKey(elem)] = elem
}
intended := make(map[string]ActionElement, len(after.Elements))
var additions []ActionElement
for _, entry := range after.Elements {
key := actionElementKey(entry)
prior, held := live[key]
// Millisecond encoding must not make an unchanged absolute expiry
// look like a replacement on every later mutation.
unchanged := held && prior.Comment == entry.Comment && prior.ExpiresAt.Equal(entry.ExpiresAt)
if !entry.ExpiresAt.IsZero() {
if _, ok := blockTimeout(entry.ExpiresAt, now); !ok {
// The kernel expires an unchanged element on its own. A delete
// naming one it already dropped fails the whole transaction.
if unchanged {
intended[key] = entry
}
continue
}
}
intended[key] = entry
if unchanged {
continue
}
if held {
remove = append(remove, nftables.SetElement{Key: prior.Key, IntervalEnd: prior.End})
}
additions = append(additions, entry)
}
for _, elem := range before.Elements {
if _, wanted := intended[actionElementKey(elem)]; wanted {
continue
}
remove = append(remove, nftables.SetElement{Key: elem.Key, IntervalEnd: elem.End})
}
return actionElements(additions, now), remove
}
func actionElements(entries []ActionElement, now time.Time) []nftables.SetElement {
var out []nftables.SetElement
for _, entry := range entries {
timeout := time.Duration(0)
if !entry.ExpiresAt.IsZero() {
var ok bool
if timeout, ok = blockTimeout(entry.ExpiresAt, now); !ok {
continue
}
}
out = append(out, nftables.SetElement{Key: entry.Key, IntervalEnd: entry.End, Comment: entry.Comment, Timeout: timeout})
}
return out
}
func matchActionElements(expected []ActionElement, actual []nftables.SetElement, at time.Time) bool {
remaining := make(map[string]ActionElement)
for _, entry := range expected {
if !entry.ExpiresAt.IsZero() && !entry.ExpiresAt.After(at) {
continue
}
remaining[fmt.Sprintf("%x/%t", entry.Key, entry.End)] = entry
}
for _, elem := range actual {
key := fmt.Sprintf("%x/%t", elem.Key, elem.IntervalEnd)
want, ok := remaining[key]
if !ok || want.Comment != elem.Comment {
return false
}
if want.ExpiresAt.IsZero() {
if elem.Timeout != 0 {
return false
}
} else {
if elem.Timeout <= 0 {
return false
}
delta := at.Add(elem.Expires).Sub(want.ExpiresAt)
if delta > 2*time.Second || delta < -2*time.Second {
return false
}
}
delete(remaining, key)
}
return len(remaining) == 0
}
func (k engineActionKernel) ObserveFirewallAction(a FirewallAction) (ActionObservation, error) {
if err := k.ValidateEvidence(a); err != nil {
return ActionObservation{}, err
}
var generation uint32
if a.Request.Operation == "apply" {
var genErr error
generation, genErr = k.e.actionGeneration()
if genErr != nil {
return ActionObservation{}, genErr
}
}
result := ActionObservation{Before: true, After: true}
for i, before := range a.KernelBefore {
after := a.KernelAfter[i]
if before.Name != after.Name {
return ActionObservation{}, errors.New("mismatched firewall kernel evidence")
}
conn, connErr := newLifecycleConn(k.e)
if connErr != nil {
return ActionObservation{}, connErr
}
started := time.Now()
at := k.e.leaseNowLocked()
actual, err := readActionSet(conn, before.Name)
if err != nil {
return ActionObservation{}, err
}
elems := actual.elements
if actual.exists != before.Exists {
result.Before = false
}
if actual.exists != after.Exists {
result.After = false
}
if time.Since(started) > 2*time.Second {
return ActionObservation{}, errors.New("firewall verification took too long")
}
result.Before = result.Before && matchActionElements(before.Elements, elems, at)
result.After = result.After && matchActionElements(after.Elements, elems, at)
}
if len(a.Members) > 1 {
at := k.e.leaseNowLocked()
for _, m := range a.Members {
for _, row := range a.After.Blocked {
// A row can outlive the journal entry that reserved its ID.
if row.ActionID != m.Request.ID || row.IP != m.Request.Target || !row.ExpiresAt.After(a.CreatedAt) {
continue
}
if _, live := blockTimeout(row.ExpiresAt, at); !live {
// Absence cannot distinguish a missed effect from an expired one.
return ActionObservation{}, nil
}
}
}
}
if a.Request.Operation == "apply" {
rules, err := k.e.rulesetEvidenceMatches(a, generation)
if err != nil {
return ActionObservation{}, err
}
result.Before = result.Before && rules.Before
result.After = result.After && rules.After
}
return result, nil
}
// BlockIPRequest carries durable request identity and source-specific admission
// policy through the existing automatic and manual safety gates.
func (e *Engine) BlockIPRequest(req ActionRequest, budget *ScanAdmission) (BlockOutcome, error) {
canonical, err := canonicalFirewallIP(req.Target)
if err != nil {
recordBlockOutcome(req.Target, req.Reason, req.TTL, BlockOutcomeNoop, err, !req.Automatic, req.FindingID)
return BlockOutcomeNoop, err
}
req.Target = canonical
req.Operation = "block"
req = normalizeActionRequest(req)
e.mu.Lock()
if found, replayErr := e.replayActionLocked(req); found || replayErr != nil {
e.mu.Unlock()
return BlockOutcomeNoop, replayErr
}
readyErr := e.lifecycleReadyLocked()
e.mu.Unlock()
if readyErr != nil {
return BlockOutcomeNoop, readyErr
}
if req.Automatic {
return e.blockIPOutcomeRequest(req, budget)
}
outcome, err := e.blockIPLockedRequest(req.Target, req.Reason, req.TTL, false, false, req, budget)
if e.shouldLegacyOutcome(err) {
recordBlockOutcome(req.Target, req.Reason, req.TTL, outcome, err, true, req.FindingID)
}
return outcome, err
}
func firewallAuditOperation(req ActionRequest) string {
if req.Automatic {
return "respond.block_ip"
}
return "operate.manual_firewall"
}
func (e *Engine) DurableActionsEnabled() bool { return e.lifecycleEnabled() }
func (e *Engine) FirewallScanBudget(window string) (int, error) {
e.mu.Lock()
defer e.mu.Unlock()
if e.lifecycle == nil {
return 0, errors.New("durable firewall budget unavailable")
}
return e.lifecycle.Store.ReadFirewallScanBudget(window)
}
type liveActionSet struct {
exists bool
elements []nftables.SetElement
}
func readActionSet(conn *nftables.Conn, name string) (liveActionSet, error) {
set, err := conn.GetSetByName(&nftables.Table{Name: "csm", Family: nftables.TableFamilyINet}, name)
if err != nil {
if isNftNotFound(err) {
return liveActionSet{}, nil
}
return liveActionSet{}, err
}
elems, err := conn.GetSetElements(set)
return liveActionSet{exists: true, elements: elems}, err
}
func (e *Engine) applyDurableLocked() error {
state, revision, err := e.readCommittedStateLocked()
if err != nil {
return err
}
req := ActionRequest{ID: rand.Text(), Operation: "apply", Target: "csm", Actor: string(actionlog.DefaultActor()), Source: SourceSystem}
plan := FirewallAction{Request: req, Before: state, After: state, Revision: revision, CreatedAt: time.Now()}
if evidenceErr := e.prepareRulesetEvidence(&plan); evidenceErr != nil {
return evidenceErr
}
plan.CreatedAt = e.expiryNow()
desired := desiredActionSets(state, e.cfg.IPv6, plan.CreatedAt)
conn, err := newLifecycleConn(e)
if err != nil {
return err
}
for _, name := range actionSetNames(true) {
actual, readErr := readActionSet(conn, name)
if readErr != nil {
return readErr
}
before := ActionSet{Name: name, Exists: actual.exists}
now := e.leaseNowLocked()
for _, elem := range actual.elements {
entry := ActionElement{Key: bytes.Clone(elem.Key), End: elem.IntervalEnd, Comment: elem.Comment}
if elem.Timeout > 0 {
entry.ExpiresAt = now.Add(elem.Expires)
}
before.Elements = append(before.Elements, entry)
}
elements, exists := desired[name]
plan.KernelBefore = append(plan.KernelBefore, before)
plan.KernelAfter = append(plan.KernelAfter, ActionSet{Name: name, Exists: exists, Elements: elements})
}
generation, genErr := e.actionGeneration()
if genErr != nil {
return genErr
}
if generation != plan.Ruleset.Generation {
return fmt.Errorf("%w: ruleset changed during planning", ErrActionUnknown)
}
a, executeErr := e.lifecycle.Execute(plan, engineActionKernel{e})
return durableActionOutcome(a, executeErr)
}
func requiredActionSets(a FirewallAction) []string {
if a.Request.Operation == "apply" {
return actionSetNames(true)
}
// A removal must prove the target set even when committed state already
// omits it. Otherwise an untracked live element is reported as removed.
blocked := a.Request.Operation == "unblock" || a.Request.Operation == "flush"
allowed := a.Request.Operation == "remove_allow"
subnets := a.Request.Operation == "unblock_subnet"
var names []string
if blocked || !reflect.DeepEqual(a.Before.Blocked, a.After.Blocked) {
names = append(names, "blocked_ips", "blocked_ips6")
}
if allowed || !reflect.DeepEqual(a.Before.Allowed, a.After.Allowed) {
names = append(names, "allowed_ips", "allowed_ips6")
}
if subnets || !reflect.DeepEqual(a.Before.BlockedNet, a.After.BlockedNet) {
names = append(names, "blocked_nets", "blocked_nets6")
}
return names
}
func (k engineActionKernel) ValidateEvidence(a FirewallAction) error {
names := requiredActionSets(a)
if len(names) != len(a.KernelBefore) || len(names) != len(a.KernelAfter) {
return errors.New("incomplete firewall kernel evidence")
}
expected := desiredActionSets(a.After, true, a.CreatedAt)
for i, name := range names {
before, after := a.KernelBefore[i], a.KernelAfter[i]
if before.Name != name || after.Name != name {
return errors.New("mismatched firewall kernel evidence")
}
if !strings.HasSuffix(name, "6") && !after.Exists {
return errors.New("missing IPv4 firewall set")
}
for _, set := range []ActionSet{before, after} {
if !set.Exists && len(set.Elements) != 0 {
return errors.New("absent set has elements")
}
keys := make(map[string]bool)
for _, elem := range set.Elements {
size := 4
if strings.HasSuffix(name, "6") {
size = 16
}
key := fmt.Sprintf("%x/%t", elem.Key, elem.End)
if len(elem.Key) != size || keys[key] {
return errors.New("invalid or duplicate firewall element")
}
keys[key] = true
}
}
if after.Exists && !sameActionElements(after.Elements, expected[name]) {
return errors.New("kernel evidence differs from intended effect")
}
}
return validateBatchMemberEvidence(a)
}
// validateBatchMemberEvidence requires every member of a batch to add its own
// element: before the batch the kernel held nothing for its address, and
// after it the element carries the fingerprint of that member's row. Retained
// rows from an older use of an ID are not evidence for the current member.
func validateBatchMemberEvidence(a FirewallAction) error {
if len(a.Members) < 2 {
return nil
}
targets := make(map[string]string, len(a.Members))
for _, m := range a.Members {
targets[m.Request.ID] = m.Request.Target
}
rows := make(map[string]BlockedEntry, len(a.Members))
for _, row := range a.After.Blocked {
if target, member := targets[row.ActionID]; member && row.IP == target && row.ExpiresAt.After(a.CreatedAt) {
rows[row.ActionID] = row
}
}
sets := make(map[string][2]ActionSet, len(a.KernelBefore))
for i, before := range a.KernelBefore {
sets[before.Name] = [2]ActionSet{before, a.KernelAfter[i]}
}
for _, m := range a.Members {
row, ok := rows[m.Request.ID]
if !ok {
return errors.New("batch member has no row")
}
name, key, ok := blockedSetKey(row.IP, true)
evidence, held := sets[name]
if !ok || !held {
return errors.New("batch member has no kernel evidence")
}
for _, elem := range evidence[0].Elements {
if bytes.Equal(elem.Key, key) {
return errors.New("batch member already held before the batch")
}
}
added := false
for _, elem := range evidence[1].Elements {
added = added || bytes.Equal(elem.Key, key) && elem.Comment == stateFingerprint(row)
}
if !added {
return errors.New("batch member element missing after the batch")
}
}
return nil
}
func sameActionElements(a, b []ActionElement) bool {
if len(a) != len(b) {
return false
}
entries := make(map[string]ActionElement, len(a))
for _, entry := range a {
entries[fmt.Sprintf("%x/%t", entry.Key, entry.End)] = entry
}
for _, entry := range b {
prior, ok := entries[fmt.Sprintf("%x/%t", entry.Key, entry.End)]
if !ok || prior.Comment != entry.Comment || !prior.ExpiresAt.Equal(entry.ExpiresAt) {
return false
}
}
return true
}
// ProveFirewallRetirement reports whether the kernel holds exactly the state
// a retires to at the lease clock's current instant, and that instant. A
// lease clock running ahead of the kernel finds an element the kernel still
// holds, so it proves nothing.
func (k engineActionKernel) ProveFirewallRetirement(a FirewallAction) (time.Time, bool, error) {
if k.e.lifecycle == nil || k.e.lifecycle.RetirementClockTrusted == nil {
return time.Time{}, false, nil
}
trusted, trustErr := k.e.lifecycle.RetirementClockTrusted()
if trustErr != nil || !trusted {
return time.Time{}, false, trustErr
}
if err := k.ValidateEvidence(a); err != nil {
return time.Time{}, false, err
}
at := k.e.leaseNowLocked()
state, ok := RetiredActionState(a, at)
if !ok {
return at, false, nil
}
desired := desiredActionSets(state, true, at)
conn, err := newLifecycleConn(k.e)
if err != nil {
return at, false, err
}
blockedProof := false
for i, set := range a.KernelAfter {
blockedProof = blockedProof || set.Name == "blocked_ips"
if a.KernelBefore[i].Exists != set.Exists {
return at, false, nil
}
started := time.Now()
readAt := k.e.leaseNowLocked()
actual, err := readActionSet(conn, set.Name)
if err != nil {
return at, false, err
}
if time.Since(started) > 2*time.Second {
return at, false, errors.New("firewall retirement verification took too long")
}
if actual.exists != set.Exists || !matchActionElements(desired[set.Name], actual.elements, readAt) {
return at, false, nil
}
}
return at, blockedProof, nil
}
func (k engineActionKernel) PrepareFirewallUndo(plan *FirewallAction) error {
at := k.e.leaseNowLocked()
plan.After = withoutExpiredRows(plan.After, at)
return k.e.prepareActionKernelAt(plan, at)
}
// The caller holds e.mu. Durable paths deliver their versioned journal event.
func (e *Engine) legacyActionAuditLocked(action, target, reason, source string, ttl time.Duration) {
if e.lifecycle == nil {
AppendAudit(e.statePath, action, target, reason, source, ttl)
}
}
func (e *Engine) legacyFileAuditLocked(action, target, reason, source string, ttl time.Duration) {
if e.lifecycle == nil {
appendAudit(e.statePath, action, target, reason, source, ttl)
}
}
// Failure defers run after the engine lock is released. Cleanup loops already
// hold that lock and use their durable outcome instead.
func (e *Engine) legacyFirewallFailure(action, target, reason, source string, ttl time.Duration, err error) {
if e.shouldLegacyOutcome(err) {
recordFirewallFailure(action, target, reason, source, ttl, err)
}
}
func normalizeActionRequest(req ActionRequest) ActionRequest {
if req.Source == "" {
req.Source = InferProvenance(req.Operation, req.Reason)
}
if req.Actor == "" {
_, actor := firewallActionOp(req.Operation, req.Source)
req.Actor = string(actor)
}
return req
}
// A retry observes the original plan and refreshes current committed state.
// Historical After must never overwrite cache state from a later action.
func (e *Engine) replayActionLocked(req ActionRequest) (bool, error) {
if e.lifecycle == nil || req.ID == "" {
return false, nil
}
previous, err := e.lifecycle.Store.ReadFirewallAction(req.ID)
if errors.Is(err, ErrActionMissing) {
return false, nil
}
if err != nil {
return false, err
}
if !previous.IsRequest(req) {
return true, ErrStateConflict
}
result, executeErr := e.lifecycle.Execute(previous, engineActionKernel{e})
if executeErr != nil && result.Phase != "verified" {
return true, durableActionOutcome(result, executeErr)
}
current, _, err := e.readCommittedStateLocked()
if err == nil {
e.installCommittedCache(current)
}
return true, errors.Join(durableActionOutcome(result, executeErr), err)
}
func (e *Engine) savePortPolicyLocked(state *FirewallState, req ActionRequest) error {
if e.lifecycle != nil {
return e.runDurableLocked(req, nil, *state)
}
return e.saveState(state)
}
// An admitted action owns its versioned outcome; rejected attempts retain the
// existing best-effort refusal record at their public operation boundary.
type durableAttemptError struct{ error }
func (err *durableAttemptError) Unwrap() error { return err.error }
func durableActionOutcome(a FirewallAction, err error) error {
if err == nil {
return nil
}
if a.Phase == "verified" {
return &durableAttemptError{errors.Join(ErrActionAuditPending, err)}
}
if a.Request.ID != "" || errors.Is(err, ErrActionUnknown) || errors.Is(err, ErrStateCommitUncertain) {
return &durableAttemptError{err}
}
return err
}
func (e *Engine) shouldLegacyOutcome(err error) bool {
if !e.lifecycleEnabled() {
return true
}
var admitted *durableAttemptError
return err != nil && !errors.As(err, &admitted)
}
// Pin mutation plans to the snapshot they read. A later successful read must
// not silently attach an older After state to a newer committed revision.
func (e *Engine) readCommittedStateLocked() (FirewallState, uint64, error) {
state, revision, err := e.lifecycle.Store.ReadFirewallState()
e.stateReadErr = err
if err == nil {
e.stateRevision = revision
}
return state, revision, err
}
package firewall
import (
"bufio"
"fmt"
"io"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
const (
geoIPBaseURL = "https://raw.githubusercontent.com/herrbischoff/country-ip-blocks/master/ipv4/"
geoIPBaseURLv6 = "https://raw.githubusercontent.com/herrbischoff/country-ip-blocks/master/ipv6/"
)
// UpdateGeoIPDB downloads country CIDR lists from a public source.
// Creates one file per country code per family: {dbPath}/{CC}.cidr (IPv4)
// and {dbPath}/{CC}.cidr6 (IPv6). IPv6 is best-effort so a country with no v6
// allocation does not fail the update. The return value is the number of CIDR
// files refreshed.
func UpdateGeoIPDB(dbPath string, countryCodes []string) (int, error) {
client := &http.Client{Timeout: 30 * time.Second}
return updateGeoIPDBWithClient(dbPath, countryCodes, client)
}
func updateGeoIPDBWithClient(dbPath string, countryCodes []string, client *http.Client) (int, error) {
if err := os.MkdirAll(dbPath, 0700); err != nil {
return 0, fmt.Errorf("creating geoip directory: %w", err)
}
updated := 0
for _, code := range countryCodes {
code = strings.ToLower(strings.TrimSpace(code))
if len(code) != 2 {
continue
}
cc := strings.ToUpper(code)
if downloadCIDRFile(client, geoIPBaseURL+code+".cidr", filepath.Join(dbPath, cc+".cidr")) {
updated++
}
if downloadCIDRFile(client, geoIPBaseURLv6+code+".cidr", filepath.Join(dbPath, cc+".cidr6")) {
updated++
}
}
return updated, nil
}
// countryCIDRMaxBytes bounds one country CIDR download.
const countryCIDRMaxBytes = 16 << 20
// countCIDRLines returns how many lines of path parse as a CIDR, skipping
// blanks and comments.
func countCIDRLines(path string) int {
// #nosec G304 -- path is the download's own temp file under dbPath.
data, err := os.ReadFile(path)
if err != nil {
return 0
}
count := 0
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if _, _, err := net.ParseCIDR(line); err == nil {
count++
}
}
return count
}
// downloadCIDRFile fetches url into outPath atomically. Returns false (and
// logs) on any HTTP, write, or invalid-payload condition so the caller can
// treat each family independently.
func downloadCIDRFile(client *http.Client, url, outPath string) bool {
resp, err := client.Get(url)
if err != nil {
fmt.Fprintf(os.Stderr, "geoip: error downloading %s: %v\n", url, err)
return false
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
fmt.Fprintf(os.Stderr, "geoip: %s returned HTTP %d\n", url, resp.StatusCode)
return false
}
tmpPath := outPath + ".tmp"
// #nosec G304 -- filepath.Join under operator-configured dbPath; code from fixed list.
f, err := os.Create(tmpPath)
if err != nil {
fmt.Fprintf(os.Stderr, "geoip: error creating %s: %v\n", tmpPath, err)
return false
}
// Bounded: a country list is well under a megabyte; an upstream that
// streams more is not serving the list.
n, copyErr := io.Copy(f, io.LimitReader(resp.Body, countryCIDRMaxBytes+1))
closeErr := f.Close()
if copyErr != nil {
_ = os.Remove(tmpPath)
fmt.Fprintf(os.Stderr, "geoip: error writing %s: %v\n", tmpPath, copyErr)
return false
}
if closeErr != nil {
_ = os.Remove(tmpPath)
fmt.Fprintf(os.Stderr, "geoip: error closing %s: %v\n", tmpPath, closeErr)
return false
}
if n > countryCIDRMaxBytes {
_ = os.Remove(tmpPath)
fmt.Fprintf(os.Stderr, "geoip: %s exceeds %d bytes, keeping previous file\n", url, countryCIDRMaxBytes)
return false
}
// Validate before install: a 200 with no parseable CIDR (an HTML
// interstitial, a moved path) must not replace the last good file, or
// the next restart builds an empty country set while the update
// reported success.
if cidrs := countCIDRLines(tmpPath); cidrs == 0 {
_ = os.Remove(tmpPath)
fmt.Fprintf(os.Stderr, "geoip: %s holds no CIDR entries (%d bytes), keeping previous file\n", url, n)
return false
}
if err := os.Rename(tmpPath, outPath); err != nil {
_ = os.Remove(tmpPath)
fmt.Fprintf(os.Stderr, "geoip: error installing %s: %v\n", outPath, err)
return false
}
fmt.Fprintf(os.Stderr, "geoip: updated %s (%d bytes)\n", outPath, n)
return true
}
// LookupIP finds which country CIDR files contain the given IP.
// Returns matching country codes.
func LookupIP(dbPath string, ip string) []string {
parsed := net.ParseIP(ip)
if parsed == nil {
return nil
}
// IPv4 (incl. v4-mapped) matches against .cidr files; IPv6 against .cidr6.
var suffix string
var needle net.IP
if ip4 := parsed.To4(); ip4 != nil {
suffix = ".cidr"
needle = ip4
} else {
suffix = ".cidr6"
needle = parsed.To16()
}
if needle == nil {
return nil
}
entries, err := os.ReadDir(dbPath)
if err != nil {
return nil
}
var matches []string
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), suffix) {
continue
}
code := strings.TrimSuffix(entry.Name(), suffix)
if containsIP(filepath.Join(dbPath, entry.Name()), needle) {
matches = append(matches, code)
}
}
return matches
}
func containsIP(cidrFile string, ip net.IP) bool {
// #nosec G304 -- cidrFile is filepath.Join under operator-configured dbPath.
f, err := os.Open(cidrFile)
if err != nil {
return false
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
_, network, err := net.ParseCIDR(line)
if err != nil {
continue
}
if network.Contains(ip) {
return true
}
}
return false
}
//go:build linux
package firewall
import "slices"
// ResolvedInfraIPs copies the last successful DNS answers for the requested
// infrastructure hosts. Callers choose which configured hosts they protect.
func (e *Engine) ResolvedInfraIPs(hosts []string) []string {
e.mu.Lock()
defer e.mu.Unlock()
var ips []string
for _, host := range hosts {
for ip := range e.infraResolved[host] {
ips = append(ips, ip)
}
}
slices.Sort(ips)
return slices.Compact(ips)
}
//go:build linux
package firewall
import (
"bytes"
"fmt"
"net"
"slices"
"time"
"github.com/google/nftables"
)
// normalizeIntervalElements unions independent start/end pairs of one address
// family. Stored source entries stay separate; only their kernel view is merged.
// A start without an end represents a range through the family's last address.
func normalizeIntervalElements(elements []nftables.SetElement) []nftables.SetElement {
type ipRange struct{ start, end []byte }
ranges := make([]ipRange, 0, len(elements)/2+1)
for i := 0; i < len(elements); i++ {
r := ipRange{start: elements[i].Key}
if i+1 < len(elements) && elements[i+1].IntervalEnd {
i++
r.end = bytes.Clone(elements[i].Key)
for j := len(r.end) - 1; j >= 0; j-- {
r.end[j]--
if r.end[j] != 0xff {
break
}
}
} else {
r.end = bytes.Repeat([]byte{0xff}, len(r.start))
}
ranges = append(ranges, r)
}
slices.SortFunc(ranges, func(a, b ipRange) int { return bytes.Compare(a.start, b.start) })
var out []nftables.SetElement
for i := 0; i < len(ranges); {
merged := ranges[i]
i++
for i < len(ranges) {
next, hasNext := nextIntervalKey(merged.end)
if hasNext && bytes.Compare(ranges[i].start, next) > 0 {
break
}
if bytes.Compare(ranges[i].end, merged.end) > 0 {
merged.end = ranges[i].end
}
i++
}
out = appendIntervalSetElements(out, merged.start, merged.end)
}
return out
}
// Preserve the set's key width even when an IPv6 boundary has the numeric
// shape of an IPv4-mapped address.
func nextIntervalKey(key []byte) ([]byte, bool) {
next := bytes.Clone(key)
for i := len(next) - 1; i >= 0; i-- {
next[i]++
if next[i] != 0 {
return next, true
}
}
return nil, false
}
func subnetIntervalElements(entries []SubnetEntry, ipv6 bool, now time.Time) (v4, v6 []nftables.SetElement) {
for _, entry := range entries {
if !entry.ExpiresAt.IsZero() && !now.Before(entry.ExpiresAt) {
continue
}
_, network, err := net.ParseCIDR(entry.CIDR)
if err != nil {
continue
}
// The public block path refuses default routes. Old state must obey
// that same lockout guard when it is restored at startup.
if ones, _ := network.Mask.Size(); ones == 0 {
continue
}
end := lastIPInRange(network)
if start := network.IP.To4(); start != nil {
v4 = appendIntervalSetElements(v4, start, end)
} else if ipv6 {
v6 = appendIntervalSetElements(v6, network.IP.To16(), end)
}
}
return normalizeIntervalElements(v4), normalizeIntervalElements(v6)
}
// Rebuild the union in one transaction. Deleting the original boundaries of
// an overlapping entry could otherwise remove a different source's protection.
func (e *Engine) replaceBlockedSubnetSets(entries []SubnetEntry) error {
// A pending config reload may differ from the installed ruleset. Keep
// every family that still has a live set until Apply replaces the table.
v4, v6 := subnetIntervalElements(entries, e.setBlockedNet6 != nil, time.Now())
// A non-lasting connection only dials at Flush and cannot fail here. A
// separate batch also makes any queue error discard the entire update.
conn, _ := nftables.New(nftables.WithSockOptions(applyNFTSocketBuffer), nftables.WithNetNSFd(e.conn.NetNS), nftables.WithTestDial(e.conn.TestDial))
for _, family := range []struct {
set *nftables.Set
elements []nftables.SetElement
}{{e.setBlockedNet, v4}, {e.setBlockedNet6, v6}} {
if family.set == nil {
continue
}
conn.FlushSet(family.set)
if err := addElementsChunked(conn, family.set, family.elements); err != nil {
return err
}
}
if err := conn.Flush(); err != nil {
return fmt.Errorf("replacing subnet sets: %w", err)
}
return nil
}
func (e *Engine) updateSubnetMutation(prior, next FirewallState, req ActionRequest, budget *ScanAdmission) error {
if e.lifecycle != nil {
return e.runDurableLocked(req, budget, next)
}
if err := e.persistFirewallIntent(prior, next); err != nil {
return fmt.Errorf("persisting subnet change: %w", err)
}
if err := e.replaceBlockedSubnetSets(next.BlockedNet); err != nil {
if restoreErr := e.saveState(&prior); restoreErr != nil {
return fmt.Errorf("%w (state restore failed: %v)", err, restoreErr)
}
return err
}
return nil
}
package firewall
import (
"net"
"os"
)
// subnetCovering returns the first CIDR in entries that contains ip, if any.
// Pure helper for BlockedSubnetCovering so the containment logic is testable
// without a kernel-attached engine.
func subnetCovering(entries []SubnetEntry, ip string) (string, bool) {
parsed := net.ParseIP(ip)
if parsed == nil {
return "", false
}
for _, entry := range entries {
_, network, err := net.ParseCIDR(entry.CIDR)
if err != nil {
continue
}
if network.Contains(parsed) {
return entry.CIDR, true
}
}
return "", false
}
// nextIP returns the IP address immediately following the given IP.
// When ip is the all-ones address for its family, nextIP clamps to ip
// instead of wrapping to all-zeros. nftables interval keys use a separate
// width-preserving helper because their address family is already fixed.
func nextIP(ip net.IP) net.IP {
next, _ := nextIPSafe(ip)
return next
}
// nextIPSafe returns the successor of ip plus whether the successor
// exists in the same address family.
func nextIPSafe(ip net.IP) (net.IP, bool) {
next := canonicalIPBytes(ip)
if next == nil {
return nil, false
}
for i := len(next) - 1; i >= 0; i-- {
if next[i] != 0xff {
next[i]++
return next, true
}
next[i] = 0
}
return canonicalIPBytes(ip), false
}
func canonicalIPBytes(ip net.IP) net.IP {
if ip4 := ip.To4(); ip4 != nil {
out := make(net.IP, net.IPv4len)
copy(out, ip4)
return out
}
if ip16 := ip.To16(); ip16 != nil {
out := make(net.IP, net.IPv6len)
copy(out, ip16)
return out
}
return nil
}
// lastIPInRange returns the last IP address in a CIDR range.
func lastIPInRange(network *net.IPNet) net.IP {
ip := network.IP.To4()
if ip == nil {
ip = network.IP.To16()
}
if ip == nil {
return nil
}
mask := network.Mask
if len(ip) == net.IPv4len && len(mask) == net.IPv6len {
mask = mask[net.IPv6len-net.IPv4len:]
}
last := make(net.IP, len(ip))
for i := range ip {
if i < len(mask) {
last[i] = ip[i] | ^mask[i]
} else {
last[i] = ip[i]
}
}
return last
}
// fileExistsFirewall checks if a file exists.
func fileExistsFirewall(path string) bool {
_, err := os.Stat(path)
return err == nil
}
//go:build linux
package firewall
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"time"
"github.com/pidginhost/csm/internal/atomicio"
)
// leaseCheckpointFile, beside the state file, keeps the lease clock across
// restarts. Older builds ignore it.
const leaseCheckpointFile = "lease.json"
// leaseCheckpoint is the lease clock as last saved, with the boot it was
// read in and the time since that boot. Within one boot a restart resumes
// at the checkpoint plus the time elapsed since it, whatever the wall clock
// did meanwhile; a new boot proves no downtime and resumes at the
// checkpoint itself. Clock sampling latency can only shorten
// restoration.
type leaseCheckpoint struct {
Lease time.Time `json:"lease"`
BootID string `json:"boot_id"`
SinceBoot time.Duration `json:"since_boot"`
}
// SetBootClock gives the engine the boot identity and the time since boot,
// read together. Without it the engine keeps no lease checkpoint.
func (e *Engine) SetBootClock(read func() (bootID string, sinceBoot time.Duration, err error)) {
e.mu.Lock()
defer e.mu.Unlock()
e.bootClock = read
}
// resumeLeaseLocked moves the lease clock forward to what the saved
// checkpoint proves, once per engine, before its first Apply restores timed
// blocks. A missing checkpoint proves nothing; a damaged one is reported
// and changes nothing.
func (e *Engine) resumeLeaseLocked() error {
if e.bootClock == nil || e.leaseResumed {
return nil
}
e.leaseResumed = true
data, err := os.ReadFile(filepath.Join(e.statePath, leaseCheckpointFile))
if errors.Is(err, fs.ErrNotExist) {
return nil
}
if err != nil {
return err
}
var ck leaseCheckpoint
if err = json.Unmarshal(data, &ck); err != nil || ck.Lease.IsZero() || ck.BootID == "" || ck.SinceBoot < 0 {
return fmt.Errorf("lease checkpoint is damaged: %w", errors.Join(err, errors.New("invalid checkpoint")))
}
// Anchor before the boot read: its latency must not be given back
// when the resumed floor is later advanced by elapsed time.
e.leaseNowLocked()
boot, since, err := e.bootClock()
if err != nil {
return err
}
floor := ck.Lease
if boot == ck.BootID && since >= ck.SinceBoot {
floor = ck.Lease.Add(since - ck.SinceBoot)
}
if floor.After(e.lease.now) {
e.lease.now = floor.Round(0)
}
return nil
}
// saveLeaseLocked records the lease clock and the boot reading taken with
// it. Apply saves after installation; timed block intents save first so
// their new absolute deadlines cannot outlive the authorized duration.
func (e *Engine) saveLeaseLocked() error {
if e.bootClock == nil {
return nil
}
// Pair a later lease sample with an earlier boot sample. A slow
// boot read may shorten restoration, but cannot renew a lease.
boot, since, err := e.bootClock()
if err != nil {
return err
}
lease := e.leaseNowLocked()
return atomicio.AtomicWriteJSON(filepath.Join(e.statePath, leaseCheckpointFile), 0o600, leaseCheckpoint{Lease: lease.UTC(), BootID: boot, SinceBoot: since})
}
//go:build linux
package firewall
import (
"fmt"
"os"
"time"
)
// leaseReading is the lease clock's last reading: the time it gave and the
// elapsed-clock sample it was taken from.
type leaseReading struct {
now, at time.Time
}
// leaseNowLocked is the time absolute expiries are turned into kernel
// timeouts against (a clock step must not lengthen a
// lease). It never runs backward: it is the later of the wall clock and
// the last reading advanced by the elapsed time since. A forward step is
// kept, which only shortens leases.
func (e *Engine) leaseNowLocked() time.Time {
return e.leaseAtLocked(e.expiryNow())
}
// leaseAtLocked reads the lease clock from sample, a reading of the
// engine's clock: the wall time without its monotonic component, and the
// sample itself, whose differences are monotonic in production. A caller
// that already read the clock passes that reading, so one instant serves
// both. leaseClock replaces the sample in tests.
func (e *Engine) leaseAtLocked(sample time.Time) time.Time {
wall, elapsed := sample.Round(0), sample
if e.leaseClock != nil {
wall, elapsed = e.leaseClock()
}
now := wall
if !e.lease.at.IsZero() {
if since := e.lease.now.Add(elapsed.Sub(e.lease.at)); since.After(now) {
now = since
}
}
e.lease = leaseReading{now: now, at: elapsed}
// After a step back the lease stays ahead of the wall clock, and so do
// the expiry times of new rows; say so once.
if lead := now.Sub(wall); lead > blockReadbackSlack && !e.leaseLeadReported {
e.leaseLeadReported = true
fmt.Fprintf(os.Stderr, "firewall: system clock is %s behind the lease clock; temporary blocks keep their lifetime, new ones show expiry times that much later\n", lead.Round(time.Second))
}
return now
}
// LeaseNow reads the lease clock for a caller outside the engine: the
// admission owner sets new expiries on it, so a wall clock step back cannot
// shorten them once the engine encodes their timeouts.
func (e *Engine) LeaseNow() time.Time {
e.mu.Lock()
defer e.mu.Unlock()
return e.leaseNowLocked()
}
// SetLeaseFloor moves the lease clock forward to floor when it is behind
// it. The daemon passes the admission clock's saved time before the first
// Apply: a lower bound for a new boot, where the lease checkpoint proves no
// downtime (lease_checkpoint_linux.go).
func (e *Engine) SetLeaseFloor(floor time.Time) {
e.mu.Lock()
defer e.mu.Unlock()
e.raiseLeaseLocked(floor)
}
func (e *Engine) raiseLeaseLocked(floor time.Time) {
if now := e.leaseNowLocked(); floor.After(now) {
e.lease.now = floor.Round(0)
}
}
package firewall
import (
"encoding/json"
"errors"
"fmt"
"slices"
"time"
)
func sameActionState(a, b FirewallState) bool {
left, err := json.Marshal(a)
if err != nil {
return false
}
right, err := json.Marshal(b)
return err == nil && string(left) == string(right)
}
func actionResult(a FirewallAction) error {
switch a.Phase {
case "verified":
return nil
case "failed":
return fmt.Errorf("%w: %s", ErrActionFailed, a.Request.ID)
default:
return fmt.Errorf("%w: %s", ErrActionUnknown, a.Request.ID)
}
}
// Execute admits once and executes only a fresh request. Repeated IDs inspect
// the original evidence; they never renew the plan or replay a kernel batch.
func (l *Lifecycle) Execute(plan FirewallAction, kernel ActionKernel) (FirewallAction, error) {
l.mu.Lock()
defer l.mu.Unlock()
return l.execute(plan, kernel)
}
func (l *Lifecycle) execute(plan FirewallAction, kernel ActionKernel) (FirewallAction, error) {
a, fresh, err := l.Store.AdmitFirewallAction(plan)
if err != nil {
return FirewallAction{}, err
}
if !fresh {
if a.Phase != "verified" && a.Phase != "failed" && !a.Retired {
a, err = l.reconcile(a, kernel, nil)
} else {
err = actionResult(a)
}
return a, errors.Join(err, l.deliverAudit())
}
a, err = l.Store.TransitionFirewallAction(a.Request.ID, "executing", "", time.Now())
if err != nil {
return FirewallAction{}, fmt.Errorf("%w: execution admission: %w", ErrActionUnknown, err)
}
observed, inspectErr := kernel.ObserveFirewallAction(a)
if inspectErr != nil || !observed.Before {
cause := errors.New("target changed before execution")
if inspectErr != nil {
cause = inspectErr
}
result, unknownErr := l.markUnknown(a, cause)
return result, errors.Join(unknownErr, l.deliverAudit())
}
kernelErr := kernel.ApplyFirewallAction(a)
if kernelErr == nil {
a, err = l.Store.TransitionFirewallAction(a.Request.ID, "applied", "", time.Now())
if err != nil {
return FirewallAction{}, fmt.Errorf("%w: applied outcome persistence: %w", ErrActionUnknown, err)
}
}
a, err = l.reconcile(a, kernel, kernelErr)
return a, errors.Join(err, l.deliverAudit())
}
func (l *Lifecycle) markUnknown(a FirewallAction, cause error) (FirewallAction, error) {
result, err := l.Store.TransitionFirewallAction(a.Request.ID, "unknown", cause.Error(), time.Now())
return result, errors.Join(fmt.Errorf("%w: %s: %w", ErrActionUnknown, a.Request.ID, cause), err)
}
var errFirewallUnproven = errors.New("kernel does not prove a complete before or after state")
func (l *Lifecycle) reconcile(a FirewallAction, kernel ActionKernel, kernelErr error) (FirewallAction, error) {
observed, err := kernel.ObserveFirewallAction(a)
if err != nil {
return l.markUnknown(a, err)
}
phase := ""
switch {
case observed.After && !observed.Before:
phase = "verified"
case observed.Before && !observed.After:
phase = "failed"
case observed.After && observed.Before && a.Phase == "applied":
phase = "verified"
case observed.After && observed.Before && a.Phase == "planned":
phase = "failed"
default:
return l.markUnknown(a, errFirewallUnproven)
}
detail := ""
if kernelErr != nil {
detail = kernelErr.Error()
}
result, err := l.Store.TransitionFirewallAction(a.Request.ID, phase, detail, time.Now())
if err != nil {
return FirewallAction{}, fmt.Errorf("%w: outcome persistence: %w", ErrActionUnknown, err)
}
return result, actionResult(result)
}
// Recover proves incomplete outcomes without executing host mutations. A
// proven rejected action is recovered successfully, though its result is failed.
func (l *Lifecycle) Recover(kernel ActionKernel) error {
l.mu.Lock()
defer l.mu.Unlock()
actions, err := l.Store.PendingFirewallActions()
if err != nil {
return err
}
var failures, uncertain []error
var unknown []FirewallAction
for _, a := range actions {
result, err := l.reconcile(a, kernel, nil)
switch {
case err == nil || errors.Is(err, ErrActionFailed):
case result.Phase == "unknown":
unknown, uncertain = append(unknown, result), append(uncertain, err)
default:
failures = append(failures, err)
}
}
auditErr := l.deliverAudit()
if auditErr == nil {
// A retirement follows the delivered outcome it settles.
retired := false
for i, a := range unknown {
ok, err := l.retire(a, kernel)
if ok {
retired = true
if errors.Is(uncertain[i], errFirewallUnproven) {
uncertain[i] = nil
}
} else if err != nil {
failures = append(failures, err)
}
}
if retired {
auditErr = l.deliverAudit()
}
}
return errors.Join(append(append(failures, uncertain...), auditErr)...)
}
// retire settles an unknown action as expired_unproven when the kernel
// proves the state it retires to. It never writes the
// kernel; a kernel that cannot prove it leaves the action pending.
func (l *Lifecycle) retire(a FirewallAction, kernel ActionKernel) (bool, error) {
prover, ok := kernel.(interface {
ProveFirewallRetirement(FirewallAction) (time.Time, bool, error)
})
if !ok {
return false, nil
}
at, proven, err := prover.ProveFirewallRetirement(a)
if err != nil || !proven {
return false, err
}
if _, err := l.Store.RetireFirewallAction(a.Request.ID, at); err != nil {
return false, err
}
return true, nil
}
// Resolve records the outcome an operator established by hand for an action
// the kernel could not prove or retire. Kernel evidence still wins: a proven
// outcome is recorded instead of the asserted one, and a settled action is
// never rewritten. Detail names who decided, and reaches the audit trail.
func (l *Lifecycle) Resolve(id, outcome, detail string, kernel ActionKernel) (FirewallAction, error) {
l.mu.Lock()
defer l.mu.Unlock()
if outcome != "verified" && outcome != "failed" {
return FirewallAction{}, fmt.Errorf("firewall action outcome must be verified or failed, not %q", outcome)
}
if detail == "" {
return FirewallAction{}, errors.New("firewall action resolution must record who decided")
}
a, err := l.Store.ReadFirewallAction(id)
if err != nil {
return FirewallAction{}, err
}
if a.Phase != "unknown" || a.Retired {
phase := a.Phase
if a.Retired {
phase = "retired as expired_unproven"
}
return FirewallAction{}, fmt.Errorf("%w: action %s is %s, not uncertain", ErrStateConflict, id, phase)
}
proven, reconcileErr := l.reconcile(a, kernel, nil)
if proven.Phase == "verified" || proven.Phase == "failed" {
return proven, errors.Join(ignoreActionResult(reconcileErr), l.deliverAudit())
}
// Only a durably recorded unknown permits an operator assertion. A
// failed transition can mean the kernel proved the opposite outcome;
// a storage error is not permission to replace that evidence.
if proven.Phase != "unknown" {
return proven, reconcileErr
}
result, err := l.Store.TransitionFirewallAction(id, outcome, detail, time.Now())
if err != nil {
return FirewallAction{}, fmt.Errorf("%w: operator outcome persistence: %w", ErrActionUnknown, err)
}
return result, l.deliverAudit()
}
// A resolution reports what the outcome turned out to be. A proven rejection
// is a complete answer to the operator's question, not a failure to answer it.
func ignoreActionResult(err error) error {
if errors.Is(err, ErrActionFailed) {
return nil
}
return err
}
func (l *Lifecycle) DeliverAudit() error {
l.mu.Lock()
defer l.mu.Unlock()
return l.deliverAudit()
}
func (l *Lifecycle) deliverAudit() error {
actions, err := l.Store.FirewallAuditPending()
if err != nil {
return err
}
for _, a := range actions {
if l.Audit == nil {
return errors.New("durable firewall audit writer unavailable")
}
if err := l.Audit(a); err != nil {
return fmt.Errorf("firewall audit pending for %s: %w", a.Request.ID, err)
}
if err := l.Store.AcknowledgeFirewallAudit(a.Request.ID, a.AuditVersion); err != nil {
return err
}
}
return nil
}
// Undo admits an inverse state transition, never a command string. A whole
// action needs its complete result current; a member needs its own row current.
func (l *Lifecycle) Undo(req ActionRequest, kernel ActionKernel) (FirewallAction, error) {
return l.undoWithClock(req, kernel, time.Now)
}
// The engine supplies its lease clock so an expired alias cannot prevent a
// member undo after a backward wall-clock step.
func (l *Lifecycle) undoWithClock(req ActionRequest, kernel ActionKernel, now func() time.Time) (FirewallAction, error) {
l.mu.Lock()
defer l.mu.Unlock()
if req.Operation != "undo" || req.UndoOf == "" {
return FirewallAction{}, errors.New("invalid firewall undo request")
}
if existing, err := l.Store.ReadFirewallAction(req.ID); err == nil {
if !existing.IsRequest(req) {
return FirewallAction{}, ErrStateConflict
}
return l.execute(existing, kernel)
} else if !errors.Is(err, ErrActionMissing) {
return FirewallAction{}, err
}
original, err := l.Store.ReadFirewallAction(req.UndoOf)
if errors.Is(err, ErrActionMissing) {
return l.undoMember(req, kernel, now)
}
if err != nil {
return FirewallAction{}, err
}
if original.Request.Operation == "apply" {
return FirewallAction{}, errors.New("ruleset configuration undo requires the confirmed-apply rollback protocol")
}
if original.Phase != "verified" {
return FirewallAction{}, errors.New("only a verified firewall action can be undone")
}
state, revision, err := l.Store.ReadFirewallState()
if err != nil {
return FirewallAction{}, err
}
if !sameActionState(state, original.After) {
return FirewallAction{}, fmt.Errorf("%w: undo state changed", ErrStateConflict)
}
observed, err := kernel.ObserveFirewallAction(original)
if err != nil {
return FirewallAction{}, err
}
if !observed.After {
return FirewallAction{}, fmt.Errorf("%w: undo target changed", ErrStateConflict)
}
at := now()
plan := FirewallAction{Request: req, Before: state, After: withoutExpiredRows(original.Before, at), Revision: revision, CreatedAt: at, KernelBefore: original.KernelAfter, KernelAfter: original.KernelBefore}
return l.executeUndo(plan, kernel)
}
// undoMember removes one member of a verified batch, planned against current
// state: the batch's other members and later live changes stay. Only that
// member's own row must still be current, with no other live or unverified
// row on its address. Batches evict no live victim to restore.
func (l *Lifecycle) undoMember(req ActionRequest, kernel ActionKernel, now func() time.Time) (FirewallAction, error) {
batch, err := l.Store.ReadFirewallBatchMember(req.UndoOf)
if err != nil {
return FirewallAction{}, err
}
if batch.Phase != "verified" {
return FirewallAction{}, errors.New("only a verified firewall action can be undone")
}
i := slices.IndexFunc(batch.Members, func(m ActionMember) bool { return m.Request.ID == req.UndoOf })
if i < 0 {
return FirewallAction{}, ErrStateCorrupt
}
member := batch.Members[i].Request
j := slices.IndexFunc(batch.After.Blocked, func(row BlockedEntry) bool {
return row.ActionID == member.ID && row.IP == member.Target && row.ExpiresAt.After(batch.CreatedAt)
})
if j < 0 {
return FirewallAction{}, ErrStateCorrupt
}
row := batch.After.Blocked[j]
state, revision, err := l.Store.ReadFirewallState()
if err != nil {
return FirewallAction{}, err
}
at := now()
next := FirewallState{BlockedNet: state.BlockedNet, Allowed: state.Allowed, PortAllowed: state.PortAllowed}
current, others := false, 0
for _, r := range state.Blocked {
switch {
case !sameAddress(r.IP, row.IP):
next.Blocked = append(next.Blocked, r)
case !current && SameBlockedEntry(r, row):
current = true
case !r.Unverified && !r.ExpiresAt.IsZero() && !r.ExpiresAt.After(at):
// An expired row on the address blocks nothing; it goes too.
default:
others++
}
}
if !current || others > 0 {
return FirewallAction{}, fmt.Errorf("%w: undo state changed", ErrStateConflict)
}
return l.executeUndo(FirewallAction{Request: req, Before: state, After: withoutExpiredRows(next, at), Revision: revision, CreatedAt: at}, kernel)
}
func (l *Lifecycle) executeUndo(plan FirewallAction, kernel ActionKernel) (FirewallAction, error) {
if planner, ok := kernel.(interface{ PrepareFirewallUndo(*FirewallAction) error }); ok {
if err := planner.PrepareFirewallUndo(&plan); err != nil {
return FirewallAction{}, err
}
}
return l.execute(plan, kernel)
}
// withoutExpiredRows copies state without the timed rows that have expired
// at at, so undo never puts one back; a restored row keeps its absolute
// expiry. Expired rows are omitted even when their outcome is unverified.
func withoutExpiredRows(state FirewallState, at time.Time) FirewallState {
live := func(expires time.Time) bool { return expires.IsZero() || expires.After(at) }
out := FirewallState{PortAllowed: slices.Clone(state.PortAllowed)}
for _, row := range state.Blocked {
if live(row.ExpiresAt) {
out.Blocked = append(out.Blocked, row)
}
}
for _, row := range state.Allowed {
if live(row.ExpiresAt) {
out.Allowed = append(out.Allowed, row)
}
}
for _, row := range state.BlockedNet {
if live(row.ExpiresAt) {
out.BlockedNet = append(out.BlockedNet, row)
}
}
return out
}
//go:build linux
package firewall
import (
"encoding/binary"
"errors"
"fmt"
"math"
"github.com/google/nftables"
"github.com/mdlayher/netlink"
"github.com/mdlayher/netlink/nltest"
"golang.org/x/sys/unix"
)
// The namespace generation advances on a committed nftables transaction.
// Together with a unique inert chain in our atomic Apply batch it proves both
// that the intended batch committed and that no later ruleset edit intervened.
// Unrelated namespace edits conservatively require recovery review too.
func (e *Engine) actionGeneration() (uint32, error) {
var socket *netlink.Conn
if e.conn.TestDial != nil {
socket = nltest.Dial(e.conn.TestDial)
} else {
var err error
socket, err = netlink.Dial(unix.NETLINK_NETFILTER, &netlink.Config{NetNS: e.conn.NetNS})
if err != nil {
return 0, err
}
if err = applyLifecycleSocketDeadline(socket); err != nil {
return 0, err
}
}
defer func() { _ = socket.Close() }()
messages, err := socket.Execute(netlink.Message{Header: netlink.Header{Type: netlink.HeaderType(unix.NFNL_SUBSYS_NFTABLES<<8 | unix.NFT_MSG_GETGEN), Flags: netlink.Request}, Data: []byte{unix.AF_UNSPEC, 0, 0, 0}})
if err != nil {
return 0, err
}
for _, message := range messages {
if len(message.Data) < 4 {
continue
}
attrs, decodeErr := netlink.UnmarshalAttributes(message.Data[4:])
if decodeErr != nil {
return 0, decodeErr
}
for _, attr := range attrs {
if attr.Type&0x3fff == unix.NFTA_GEN_ID && len(attr.Data) == 4 {
return binary.BigEndian.Uint32(attr.Data), nil
}
}
}
return 0, errors.New("missing nftables generation evidence")
}
func (e *Engine) prepareRulesetEvidence(a *FirewallAction) error {
generation, err := e.actionGeneration()
if err != nil {
return err
}
if generation == math.MaxUint32 {
return errors.New("nftables generation rollover requires a new recovery baseline")
}
a.Ruleset = &ActionRuleset{Generation: generation, Marker: "csm_action_" + stateFingerprint(a.Request)[4:]}
return nil
}
func (e *Engine) rulesetEvidenceMatches(a FirewallAction, generation uint32) (ActionObservation, error) {
if a.Ruleset == nil || a.Ruleset.Marker != "csm_action_"+stateFingerprint(a.Request)[4:] || a.Ruleset.Generation == math.MaxUint32 {
return ActionObservation{}, errors.New("invalid ruleset recovery evidence")
}
conn, err := newLifecycleConn(e)
if err != nil {
return ActionObservation{}, err
}
chains, err := conn.ListChainsOfTableFamily(nftables.TableFamilyINet)
if err != nil {
return ActionObservation{}, err
}
marker := false
for _, chain := range chains {
if chain.Table.Name == "csm" && chain.Name == a.Ruleset.Marker {
marker = true
}
}
after, err := e.actionGeneration()
if err != nil {
return ActionObservation{}, err
}
if after != generation {
return ActionObservation{}, fmt.Errorf("%w: ruleset changed during verification", ErrActionUnknown)
}
return ActionObservation{Before: generation == a.Ruleset.Generation && !marker, After: generation == a.Ruleset.Generation+1 && marker}, nil
}
//go:build linux
package firewall
import (
"errors"
"fmt"
"time"
"github.com/google/nftables"
"github.com/mdlayher/netlink"
)
// Bound the actual socket exchange, including both request writes and replies.
// This option runs on every transient dial, so a reused nftables connection
// receives a fresh deadline for each operation.
func applyLifecycleSocketDeadline(connection *netlink.Conn) error {
if err := connection.SetDeadline(time.Now().Add(2 * time.Second)); err != nil {
// nftables does not close a socket when one of its socket options fails.
_ = connection.Close()
return fmt.Errorf("setting firewall lifecycle socket deadline: %w", err)
}
return nil
}
func newLifecycleConn(engine *Engine) (*nftables.Conn, error) {
if engine == nil || engine.conn == nil {
return nil, errors.New("firewall lifecycle transport unavailable")
}
options := []nftables.ConnOption{
nftables.WithNetNSFd(engine.conn.NetNS),
nftables.WithTestDial(engine.conn.TestDial),
nftables.WithSockOptions(applyNFTSocketBuffer),
}
// The explicitly injected nltest transport has no OS socket or deadline
// support. Never suppress deadline errors on a live transport.
if engine.conn.TestDial == nil {
options = append(options, nftables.WithSockOptions(applyLifecycleSocketDeadline))
}
return nftables.New(options...)
}
package firewall
import (
"fmt"
"os"
"os/user"
"regexp"
"strconv"
"strings"
)
// LocalPortLock lets only Users, and root, open new connections to Port on
// this host.
type LocalPortLock struct {
Port int
Users []string
}
var lockUserName = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,31}$`)
// ParseLocalPortLocks reads firewall.local_port_locks entries of the form
// "PORT:user[,user...]". A port may appear once.
func ParseLocalPortLocks(entries []string) ([]LocalPortLock, error) {
locks := make([]LocalPortLock, 0, len(entries))
seen := map[int]bool{}
for i, entry := range entries {
portText, usersText, ok := strings.Cut(entry, ":")
if !ok {
return nil, fmt.Errorf("entry %d %q: want PORT:user[,user...]", i, entry)
}
port, err := strconv.Atoi(strings.TrimSpace(portText))
if err != nil || port < 1 || port > 65535 {
return nil, fmt.Errorf("entry %d %q: port must be 1-65535", i, entry)
}
if seen[port] {
return nil, fmt.Errorf("entry %d %q: port %d is listed twice", i, entry, port)
}
seen[port] = true
var users []string
for _, name := range strings.Split(usersText, ",") {
name = strings.TrimSpace(name)
if !lockUserName.MatchString(name) {
return nil, fmt.Errorf("entry %d %q: %q is not a user name", i, entry, name)
}
users = append(users, name)
}
locks = append(locks, LocalPortLock{Port: port, Users: users})
}
return locks, nil
}
// localPortLockLookupUser resolves lock user names; tests swap a fixture.
var localPortLockLookupUser = user.Lookup
// resolveLockUIDs returns root plus each listed user's UID. Unknown users are
// reported and skipped: the lock still applies, so a typo denies that user
// rather than opening the port.
func resolveLockUIDs(port int, users []string) []uint32 {
out := []uint32{0}
seen := map[uint32]bool{0: true}
for _, name := range users {
u, err := localPortLockLookupUser(name)
if err != nil {
fmt.Fprintf(os.Stderr, "firewall: local_port_locks: port %d: unknown user %q\n", port, name)
continue
}
uid, err := strconv.ParseUint(u.Uid, 10, 32)
if err != nil {
fmt.Fprintf(os.Stderr, "firewall: local_port_locks: port %d: invalid uid for %s: %v\n", port, name, err)
continue
}
if !seen[uint32(uid)] {
seen[uint32(uid)] = true
out = append(out, uint32(uid))
}
}
return out
}
//go:build linux
package firewall
import (
"fmt"
"os"
"slices"
"sort"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
)
// LockedLocalPorts reports installed locks only while the live rules still
// match the installation. A rollback or external edit invalidates that proof.
func (e *Engine) LockedLocalPorts() []int {
e.mu.Lock()
defer e.mu.Unlock()
if len(e.lockedLocalPorts) == 0 || e.appliedRuleset == "" {
return nil
}
current, err := e.readRulesetLocked()
if err != nil || current != e.appliedRuleset {
return nil
}
return slices.Clone(e.lockedLocalPorts)
}
func configuredLockPorts(cfg *FirewallConfig) []int {
if cfg == nil {
return nil
}
locks, err := ParseLocalPortLocks(cfg.LocalPortLocks)
if err != nil {
return nil
}
ports := make([]int, 0, len(locks))
for _, l := range locks {
ports = append(ports, l.Port)
}
sort.Ints(ports)
return ports
}
// addLocalPortLockRules accepts new loopback TCP connections to each locked
// port from root and the listed users, and resets everyone else's. Packets
// of established connections and traffic to other hosts do not match.
func (e *Engine) addLocalPortLockRules() {
locks, err := ParseLocalPortLocks(e.cfg.LocalPortLocks)
if err != nil {
// Config load rejects this; never reached from a validated config.
fmt.Fprintf(os.Stderr, "firewall: local_port_locks: %v\n", err)
return
}
for _, l := range locks {
for _, uid := range resolveLockUIDs(l.Port, l.Users) {
exprs := append(localPortLockMatch(l.Port),
&expr.Meta{Key: expr.MetaKeySKUID, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(uid)},
&expr.Verdict{Kind: expr.VerdictAccept},
)
e.conn.AddRule(&nftables.Rule{Table: e.table, Chain: e.chainOut, Exprs: exprs})
}
e.conn.AddRule(&nftables.Rule{
Table: e.table,
Chain: e.chainOut,
Exprs: append(localPortLockMatch(l.Port), &expr.Reject{Type: 1}), // NFT_REJECT_TCP_RST
})
}
}
func localPortLockMatch(port int) []expr.Any {
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte("lo\x00")},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(port))},
// SYN set and ACK clear opens a connection. The TCP flags, unlike
// conntrack state, still match where loopback traffic is untracked.
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 13, Len: 1},
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 1, Mask: []byte{0x12}, Xor: []byte{0}},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{0x02}},
}
}
//go:build linux
package firewall
import (
"errors"
"fmt"
)
type stateDurabilityError struct{ cause error }
func (e *stateDurabilityError) Error() string {
return fmt.Sprintf("state file matches requested change but durability is unconfirmed: %v", e.cause)
}
func (e *stateDurabilityError) Unwrap() error { return e.cause }
// A write can fail after rename made the next state visible. Restore the
// prior intent before returning so background cleanup retains its retry rows.
// Both failures must be exposed if storage also refuses the rollback.
func (e *Engine) persistFirewallIntent(prior, next FirewallState) error {
err := e.saveState(&next)
if err == nil {
return nil
}
var uncertain *stateDurabilityError
if !errors.As(err, &uncertain) {
return err
}
if restoreErr := e.saveState(&prior); restoreErr != nil {
return fmt.Errorf("partial failure: %w (state restore failed: %w)", err, restoreErr)
}
return fmt.Errorf("intent write failed; previous state restored: %w", err)
}
//go:build linux
package firewall
import (
"fmt"
"strings"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
)
type portFloodIPFamily struct {
name string
nfproto byte
sourceOffset uint32
sourceLen uint32
}
var (
portFloodIPv4 = portFloodIPFamily{name: "v4", nfproto: 2, sourceOffset: 12, sourceLen: 4}
portFloodIPv6 = portFloodIPFamily{name: "v6", nfproto: 10, sourceOffset: 8, sourceLen: 16}
)
// buildPortFloodExprs returns the nftables expressions for one per-port
// flood-protection rule. The rule rate-limits new TCP/UDP connections to
// pf.Port per source address by updating a dynamic meter set. The caller
// supplies a family-specific meter, so IPv4 and IPv6 do not share buckets.
//
// Returning nil signals the caller to skip rule creation (zero rate, missing
// meter, or zero-port).
func buildPortFloodExprs(pf PortFloodRule, meter *nftables.Set, family portFloodIPFamily) []expr.Any {
if meter == nil || pf.Hits <= 0 || pf.Seconds <= 0 || pf.Port <= 0 {
return nil
}
proto := byte(6) // TCP
if pf.Proto == "udp" {
proto = 17
}
// hits/seconds -> packets per minute (multiply first to keep precision).
ratePerMin := uint64(pf.Hits) * 60 / uint64(pf.Seconds)
if ratePerMin < 1 {
ratePerMin = 1
}
burst := uint32(ratePerMin / 4)
if burst < 2 {
burst = 2
}
return []expr.Any{
// Restrict to the family that matches the meter key type.
&expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{family.nfproto}},
// Match new connections only.
&expr.Ct{Register: 1, SourceRegister: false, Key: expr.CtKeySTATE},
&expr.Bitwise{
SourceRegister: 1, DestRegister: 1, Len: 4,
Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW),
Xor: binaryutil.NativeEndian.PutUint32(0),
},
&expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(0)},
// L4 protocol filter (TCP or UDP).
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{proto}},
// Destination port.
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(portU16(pf.Port))},
// Load source address into reg1; this is the meter key.
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: family.sourceOffset, Len: family.sourceLen},
// Update meter entry for this source IP, evaluating its own token bucket.
&expr.Dynset{
SrcRegKey: 1,
SetName: meter.Name,
SetID: meter.ID,
Operation: 1, // NFT_DYNSET_OP_UPDATE
Exprs: []expr.Any{
&expr.Limit{Type: expr.LimitTypePkts, Rate: ratePerMin, Unit: expr.LimitTimeMinute, Burst: burst, Over: true},
},
},
&expr.Verdict{Kind: expr.VerdictDrop},
}
}
func (e *Engine) portFloodRuleExprs(pf PortFloodRule, meter *nftables.Set, family portFloodIPFamily) []expr.Any {
exprs := buildPortFloodExprs(pf, meter, family)
if exprs == nil || !isMailTCP(pf) {
return exprs
}
var exempt []expr.Any
switch {
case family == portFloodIPv4 && e.setDOSExempt != nil:
exempt = e.dosExemptV4Lookup(1)
case family == portFloodIPv6 && e.setDOSExempt6 != nil:
exempt = e.dosExemptV6Lookup(1)
default:
return exprs
}
guarded := make([]expr.Any, 0, len(exprs)+len(exempt))
guarded = append(guarded, exprs[:2]...)
guarded = append(guarded, exempt...)
return append(guarded, exprs[2:]...)
}
func portFloodMeterName(pf PortFloodRule, family portFloodIPFamily) string {
return fmt.Sprintf("meter_pf_%s_%d_%s", portFloodProto(pf), pf.Port, family.name)
}
func portFloodProto(pf PortFloodRule) string {
if pf.Proto == "udp" {
return "udp"
}
return "tcp"
}
// isMailTCP reports whether pf is a TCP rule for a standard mail relay or
// submission port (25, 465, 587). Only these ports receive the DoS-exempt
// inverted-lookup guard; all other TCP ports and UDP rules are left unchanged.
func isMailTCP(pf PortFloodRule) bool {
if !strings.EqualFold(pf.Proto, "tcp") {
return false
}
switch pf.Port {
case 25, 465, 587:
return true
default:
return false
}
}
//go:build linux
package firewall
import "fmt"
type ipProtectedError struct {
msg string
}
func (e ipProtectedError) Error() string {
return e.msg
}
func (e ipProtectedError) Unwrap() error {
return ErrIPProtected
}
func ipProtectedErrorf(format string, args ...any) error {
return ipProtectedError{msg: fmt.Sprintf(format, args...)}
}
package firewall
import "strings"
const (
SourceUnknown = "unknown"
SourceWebUI = "web_ui"
SourceCLI = "cli"
SourceAutoResponse = "auto_response"
SourceChallenge = "challenge"
SourceWhitelist = "whitelist"
SourceDynDNS = "dyndns"
SourceSystem = "system"
SourceConfig = "config"
)
// InferProvenance classifies a firewall entry source from structured action/reason text.
// This keeps provenance logic centralized instead of spreading fragile string checks
// throughout the web UI and firewall call sites.
func InferProvenance(action, reason string) string {
action = strings.ToLower(strings.TrimSpace(action))
reason = strings.ToLower(strings.TrimSpace(reason))
switch {
case strings.Contains(reason, "dyndns:"):
return SourceDynDNS
case strings.Contains(reason, "passed challenge"),
strings.Contains(reason, "challenge-timeout"),
strings.Contains(reason, "challenge timeout"):
return SourceChallenge
case strings.Contains(reason, "temp whitelist"),
strings.Contains(reason, "whitelist"),
strings.Contains(reason, "bulk whitelist"),
strings.Contains(reason, "customer ip"):
return SourceWhitelist
case strings.Contains(reason, "auto-block"),
strings.Contains(reason, "permbblock"),
strings.Contains(reason, "permblock"),
strings.Contains(reason, "auto-netblock"):
return SourceAutoResponse
case strings.Contains(reason, "via cli"):
return SourceCLI
case strings.Contains(reason, "via csm web ui"),
strings.Contains(reason, "via ui"),
strings.Contains(reason, "allowed from firewall lookup"),
strings.Contains(reason, "manual block"):
return SourceWebUI
case action == "temp_allow_expired":
return SourceSystem
case action == "flush":
return SourceSystem
default:
return SourceUnknown
}
}
// Package rollback implements the firewall settings tentative-apply
// workflow: a save with a deadline that auto-reverts unless the operator
// confirms before the timer expires. The manager survives daemon restarts
// (state is persisted in bbolt) so that the apply itself can take down the
// daemon for a config reload without losing the rollback intent.
package rollback
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"os/exec"
"sync"
"time"
"github.com/pidginhost/csm/internal/actionlog"
"github.com/pidginhost/csm/internal/integrity"
"github.com/pidginhost/csm/internal/store"
)
// MinTimeout and MaxTimeout bound the operator-supplied window. The lower
// bound exists so a misclick cannot leave the operator one second to react;
// the upper bound caps how long a botched apply can sit on disk before
// auto-recovery kicks in.
const (
MinTimeout = 1 * time.Minute
MaxTimeout = 30 * time.Minute
DefaultTimeout = 5 * time.Minute
)
// Restarter performs a daemon restart. Production wires this to a
// systemctl exec; tests substitute a fake.
type Restarter func(ctx context.Context) error
// Status describes the current pending rollback for status APIs and the
// Web UI banner. AppliedAt and ExpiresAt are UTC; SecondsRemaining is a
// derived hint computed at call time.
type Status struct {
Pending bool `json:"pending"`
AppliedAt time.Time `json:"applied_at,omitzero"`
ExpiresAt time.Time `json:"expires_at,omitzero"`
SecondsRemaining int64 `json:"remaining_seconds,omitempty"`
AppliedBy string `json:"applied_by,omitempty"`
PrevHash string `json:"prev_hash,omitempty"`
NewHash string `json:"new_hash,omitempty"`
identity string
}
// Manager owns the active timer and serialises Apply/Confirm/Revert/Recover
// against each other. Storage and restart are injected so the manager can be
// driven in tests without a real bbolt or systemctl.
type Manager struct {
mu sync.Mutex
db *store.DB
configPath string
restart Restarter
now func() time.Time
timer *time.Timer
}
// Process-wide singleton. The daemon installs one Manager at startup; the
// Web UI handlers and the control-socket commands look it up by calling
// Global so they do not need to thread the manager through every layer.
var (
globalMu sync.Mutex
global *Manager
)
// SetGlobal installs the process-wide manager. Safe to call from
// daemon startup; subsequent SetGlobal calls overwrite.
func SetGlobal(m *Manager) {
globalMu.Lock()
global = m
globalMu.Unlock()
}
// Global returns the installed manager or nil when none has been set
// (e.g. inside CLI commands that load config but never start the
// daemon). Callers must nil-check.
func Global() *Manager {
globalMu.Lock()
defer globalMu.Unlock()
return global
}
// NewManager wires a manager. Use SystemctlRestart for the production
// restart path. now defaults to time.Now when nil.
func NewManager(db *store.DB, configPath string, restart Restarter, now func() time.Time) *Manager {
if now == nil {
now = time.Now
}
return &Manager{
db: db,
configPath: configPath,
restart: restart,
now: now,
}
}
// SystemctlRestart issues `systemctl restart csm.service`. The context
// timeout caps how long we wait for systemctl itself; the daemon restart
// it triggers is asynchronous from systemctl's perspective.
func SystemctlRestart(ctx context.Context) error {
// #nosec G204 -- fixed argv, no operator input interpolated.
out, err := exec.CommandContext(ctx, "systemctl", "restart", "csm.service").CombinedOutput()
if err != nil {
return fmt.Errorf("systemctl restart csm: %w (%s)", err, string(out))
}
return nil
}
// HashYAML returns the sha256 hex digest of yaml bytes. Used so the
// Web UI and CLI can show operators the before/after hash without the
// full file contents.
func HashYAML(data []byte) string {
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])
}
// clampTimeout returns a timeout within [MinTimeout, MaxTimeout]; it
// substitutes DefaultTimeout for zero so callers can pass 0 to mean
// "use the default."
func clampTimeout(d time.Duration) time.Duration {
if d == 0 {
return DefaultTimeout
}
if d < MinTimeout {
return MinTimeout
}
if d > MaxTimeout {
return MaxTimeout
}
return d
}
// Apply records prevYAML as the snapshot to restore on expiry, computes
// the expiry deadline, persists the rollback entry, and arms the local
// timer. The caller is responsible for holding ConfigWriteMutex across Apply
// and the newYAML write, then triggering the restart.
//
// applyBy is logged with the rollback record (e.g. token name or "cli")
// so audits can trace the source.
func (m *Manager) Apply(prevYAML, newYAML []byte, timeout time.Duration, applyBy string) (Status, error) {
m.mu.Lock()
defer m.mu.Unlock()
if existing, ok := m.db.GetFirewallRollback(); ok {
return statusFromRecord(existing, m.now()), fmt.Errorf("rollback already pending; confirm or revert first")
}
timeout = clampTimeout(timeout)
now := m.now().UTC()
rb := store.FirewallRollback{
PrevYAML: prevYAML,
PrevHash: HashYAML(prevYAML),
NewHash: HashYAML(newYAML),
AppliedAt: now,
ExpiresAt: now.Add(timeout),
AppliedBy: applyBy,
}
if err := m.db.SaveFirewallRollback(rb); err != nil {
return Status{}, fmt.Errorf("persist rollback: %w", err)
}
m.armTimerLocked(timeout)
return statusFromRecord(rb, m.now()), nil
}
// Confirm drops the pending rollback. The new config stays on disk;
// no daemon restart is required. Idempotent: confirming with no
// pending entry is a no-op.
func (m *Manager) Confirm() error {
expected := m.Status()
if !expected.Pending {
return nil
}
return m.ConfirmIfCurrent(expected)
}
// ConfirmIfCurrent confirms only the rollback represented by expected. A
// confirmation request can race another operator confirming the old record
// and applying a replacement; it must not silently clear the newer record.
// Taking the config writer lock also keeps confirmation from landing between
// tentative-apply's rollback staging and its csm.yaml write.
func (m *Manager) ConfirmIfCurrent(expected Status) error {
configMu := integrity.ConfigWriteMutex()
configMu.Lock()
defer configMu.Unlock()
return m.confirmIfCurrent(expected)
}
// AbortApplyIfCurrent drops a staged rollback when tentative-apply could not
// write the new config. The caller must already hold ConfigWriteMutex for the
// complete staging/write transaction.
func (m *Manager) AbortApplyIfCurrent(expected Status) error {
return m.confirmIfCurrent(expected)
}
func (m *Manager) confirmIfCurrent(expected Status) error {
m.mu.Lock()
defer m.mu.Unlock()
rb, ok := m.db.GetFirewallRollback()
if !ok || !sameRollbackStatus(expected, rb) {
return fmt.Errorf("rollback changed before confirm; refresh and retry")
}
if m.timer != nil {
m.timer.Stop()
m.timer = nil
}
return m.db.ClearFirewallRollback()
}
// Revert restores the snapshot to disk and triggers a daemon restart.
// Returns an error if there is no pending rollback so callers can
// surface a clean "nothing to revert" message instead of silently
// succeeding.
func (m *Manager) Revert(ctx context.Context) error {
expected := m.Status()
if !expected.Pending {
return fmt.Errorf("no pending rollback")
}
return m.RevertIfCurrent(ctx, expected)
}
// RevertIfCurrent restores only the rollback represented by expected. The
// manager re-checks after acquiring the config writer lock because a blocked
// revert must not restore a replacement rollback created while it waited.
func (m *Manager) RevertIfCurrent(ctx context.Context, expected Status) error {
configMu := integrity.ConfigWriteMutex()
configMu.Lock()
defer configMu.Unlock()
m.mu.Lock()
defer m.mu.Unlock()
rb, ok := m.db.GetFirewallRollback()
if !ok {
return fmt.Errorf("no pending rollback")
}
if !sameRollbackStatus(expected, rb) {
return fmt.Errorf("rollback changed before revert; refresh and retry")
}
return m.applyRevertLocked(ctx, rb)
}
// RecoverOnStartup is called once during daemon startup. If a pending
// rollback exists, the manager either fires the revert immediately
// (timer already expired while the daemon was down) or rearms a
// time.AfterFunc for the remaining window.
//
// The bool return is true when an immediate revert was performed so
// the caller can decide whether to bail out of further startup work
// while the restart it triggers takes effect.
func (m *Manager) RecoverOnStartup(ctx context.Context) (reverted bool, err error) {
m.mu.Lock()
rb, ok := m.db.GetFirewallRollback()
if !ok {
m.mu.Unlock()
return false, nil
}
now := m.now()
if !now.Before(rb.ExpiresAt) {
m.mu.Unlock()
configMu := integrity.ConfigWriteMutex()
configMu.Lock()
defer configMu.Unlock()
m.mu.Lock()
defer m.mu.Unlock()
rb, ok = m.db.GetFirewallRollback()
if !ok {
return false, nil
}
now = m.now()
if now.Before(rb.ExpiresAt) {
m.armTimerLocked(rb.ExpiresAt.Sub(now))
return false, nil
}
err := m.applyRevertLocked(ctx, rb)
if err != nil {
return false, err
}
return true, nil
}
remaining := rb.ExpiresAt.Sub(now)
m.armTimerLocked(remaining)
m.mu.Unlock()
return false, nil
}
// Status reports the pending rollback for /api/v1/.../rollback and
// the CLI status command. Pending=false means "nothing in flight".
func (m *Manager) Status() Status {
m.mu.Lock()
defer m.mu.Unlock()
rb, ok := m.db.GetFirewallRollback()
if !ok {
return Status{}
}
return statusFromRecord(rb, m.now())
}
func (m *Manager) armTimerLocked(d time.Duration) {
if m.timer != nil {
m.timer.Stop()
}
m.timer = time.AfterFunc(d, func() {
// Build a fresh context so the AfterFunc goroutine has a
// usable deadline; the original ctx from RecoverOnStartup
// would have been cancelled by the time this fires.
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := m.timerExpired(ctx); err != nil {
fmt.Fprintf(os.Stderr, "rollback: timer expiry failed: %v\n", err)
}
})
}
func (m *Manager) timerExpired(ctx context.Context) error {
configMu := integrity.ConfigWriteMutex()
configMu.Lock()
defer configMu.Unlock()
m.mu.Lock()
defer m.mu.Unlock()
rb, ok := m.db.GetFirewallRollback()
if !ok {
return nil
}
now := m.now()
if now.Before(rb.ExpiresAt) {
m.armTimerLocked(rb.ExpiresAt.Sub(now))
return nil
}
return m.applyRevertLocked(ctx, rb)
}
// applyRevertLocked restores the snapshot bytes to the config path, clears
// the pending record, and only then triggers the daemon restart. The order
// is load-bearing: systemctl restart kills this very process, so anything
// sequenced after a successful restart never runs. Clearing after the
// restart left the record in place, and every subsequent boot found it
// expired, reverted, and restarted again - a permanent restart loop. Once
// the snapshot is on disk any later daemon start converges on the reverted
// config, so a failed restart (surfaced as an error) loses nothing but
// immediacy. A failed clear aborts before the restart for the same reason:
// restarting with the record still present is what loops.
// Caller must hold m.mu.
func (m *Manager) applyRevertLocked(ctx context.Context, rb store.FirewallRollback) (resultErr error) {
rec := actionlog.Record{Op: "operate.manual_firewall", Action: "rollback_config", Target: m.configPath, Before: actionlog.Stat(m.configPath), Result: actionlog.Failed}
recorded := false
defer func() {
if !recorded {
if resultErr != nil {
rec.Error = resultErr.Error()
}
actionlog.Write(rec)
}
}()
if m.timer != nil {
m.timer.Stop()
m.timer = nil
}
if len(rb.PrevYAML) == 0 {
return fmt.Errorf("rollback record has empty prev_yaml; cannot restore")
}
if err := integrity.WriteConfigBytesAtomic(m.configPath, rb.PrevYAML); err != nil {
return fmt.Errorf("restore previous config: %w", err)
}
rec.Result = actionlog.Applied
rec.After = actionlog.Stat(m.configPath)
if err := m.db.ClearFirewallRollback(); err != nil {
return fmt.Errorf("clear rollback before restart: %w", err)
}
// Restart may terminate this process before it returns. The config change
// has completed and must be recorded before requesting the restart.
actionlog.Write(rec)
recorded = true
if m.restart != nil {
if err := m.restart(ctx); err != nil {
return fmt.Errorf("trigger restart after revert (config already restored on disk): %w", err)
}
}
return nil
}
func statusFromRecord(rb store.FirewallRollback, now time.Time) Status {
remaining := int64(rb.ExpiresAt.Sub(now).Seconds())
if remaining < 0 {
remaining = 0
}
return Status{
Pending: true,
AppliedAt: rb.AppliedAt,
ExpiresAt: rb.ExpiresAt,
SecondsRemaining: remaining,
AppliedBy: rb.AppliedBy,
PrevHash: rb.PrevHash,
NewHash: rb.NewHash,
identity: rollbackIdentity(rb),
}
}
func sameRollbackStatus(expected Status, current store.FirewallRollback) bool {
return expected.Pending && expected.identity != "" && expected.identity == rollbackIdentity(current)
}
func rollbackIdentity(rb store.FirewallRollback) string {
payload := fmt.Sprintf("%x\n%q\n%q\n%q\n%q\n%q",
rb.PrevYAML, rb.PrevHash, rb.NewHash,
rb.AppliedAt.UTC().Format(time.RFC3339Nano),
rb.ExpiresAt.UTC().Format(time.RFC3339Nano), rb.AppliedBy)
sum := sha256.Sum256([]byte(payload))
return hex.EncodeToString(sum[:])
}
package firewall
// RuleCounts holds firewall rule cardinalities sourced from the engine
// state file, which is the authoritative store. The parallel bbolt
// fw:* buckets are written only during migration, so anything counting
// live rules must read the engine, not the store. Expired temp bans are
// excluded.
type RuleCounts struct {
Blocked int
Allowed int
Subnets int
PortAllowed int
}
// Total returns the sum across all rule categories.
func (c RuleCounts) Total() int {
return c.Blocked + c.Allowed + c.Subnets + c.PortAllowed
}
//go:build linux
package firewall
import "net"
func countRuleEntries(state FirewallState, ipv6Enabled bool) RuleCounts {
return RuleCounts{
Blocked: countBlockedRules(state.Blocked, ipv6Enabled),
Allowed: countAllowedRules(state.Allowed, ipv6Enabled),
Subnets: countSubnetRules(state.BlockedNet, ipv6Enabled),
PortAllowed: countPortAllowRules(state.PortAllowed, ipv6Enabled),
}
}
func countBlockedRules(entries []BlockedEntry, ipv6Enabled bool) int {
seen := make(map[string]struct{}, len(entries))
for _, entry := range entries {
key, ok := ruleIPKey(entry.IP, ipv6Enabled)
if !ok || entry.Unverified {
continue
}
seen[key] = struct{}{}
}
return len(seen)
}
func countAllowedRules(entries []AllowedEntry, ipv6Enabled bool) int {
seen := make(map[string]struct{}, len(entries))
for _, entry := range entries {
key, ok := ruleIPKey(entry.IP, ipv6Enabled)
if !ok {
continue
}
seen[key] = struct{}{}
}
return len(seen)
}
func countSubnetRules(entries []SubnetEntry, ipv6Enabled bool) int {
seen := make(map[string]struct{}, len(entries))
for _, entry := range entries {
_, network, err := net.ParseCIDR(entry.CIDR)
if err != nil {
continue
}
if network.IP.To4() == nil && !ipv6Enabled {
continue
}
seen[network.String()] = struct{}{}
}
return len(seen)
}
func countPortAllowRules(entries []PortAllowEntry, ipv6Enabled bool) int {
count := 0
for _, entry := range entries {
if _, ok := ruleIPKey(entry.IP, ipv6Enabled); !ok {
continue
}
count++
}
return count
}
func ruleIPKey(raw string, ipv6Enabled bool) (string, bool) {
parsed := net.ParseIP(raw)
if parsed == nil {
return "", false
}
if ip4 := parsed.To4(); ip4 != nil {
return "4:" + net.IP(ip4).String(), true
}
if !ipv6Enabled {
return "", false
}
ip16 := parsed.To16()
if ip16 == nil {
return "", false
}
return "6:" + net.IP(ip16).String(), true
}
//go:build linux
package firewall
import (
"context"
"os/exec"
"time"
)
// RulesetSnapshot returns the live rules and the baseline captured immediately
// after the last successful Apply. The engine lock keeps our own concurrent
// reconfiguration from splitting the pair. An empty baseline means capture
// failed or this engine attached to an existing table without applying it.
func (e *Engine) RulesetSnapshot() (current, applied string, err error) {
e.mu.Lock()
defer e.mu.Unlock()
current, err = e.readRulesetLocked()
return current, e.appliedRuleset, err
}
func (e *Engine) readRulesetLocked() (string, error) {
if e.readRuleset != nil {
out, err := e.readRuleset()
return StableRuleset(out), err
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
// Terse/stateless output omits members, counters, and elapsed timeouts.
// Keep transient traffic state out of both the baseline and its memory cost.
out, err := exec.CommandContext(ctx, "nft", "-s", "-t", "list", "table", "inet", "csm").Output()
if err != nil {
return "", err
}
return StableRuleset(string(out)), nil
}
package firewall
import "regexp"
// setElementCount matches the live element count that nft prints beside a
// set's size on newer kernels. It moves with traffic; the size is structure.
var setElementCount = regexp.MustCompile(`(?m)^([ \t]*size [0-9]+)[ \t]*# count [0-9]+$`)
// StableRuleset removes set element counts from a terse nft listing, so two
// listings of unchanged rules compare equal.
func StableRuleset(listing string) string {
return setElementCount.ReplaceAllString(listing, "$1")
}
package firewall
import (
"fmt"
"os"
"os/user"
"strconv"
)
// smtpAllowlistLookupUser is the user-database accessor that
// resolveSMTPAllowedUIDs goes through. Production binds it to
// user.Lookup; tests can swap a fixture so unit tests do not depend on
// /etc/passwd of the host they run on.
var smtpAllowlistLookupUser = user.Lookup
// resolveSMTPAllowedUIDs returns the deduplicated list of UIDs that are
// allowed to open outbound SMTP connections when smtp_block is on.
//
// Two UIDs are unconditional:
// - 0 (root): operator commands, csm itself, system tools.
// - mailnull: exim's queue-runner runs as this user on cPanel; if it
// is dropped, queued mail never leaves the host even though cPanel
// thinks it released the hold. Silently breaking outbound mail is
// the worst possible failure mode for a security firewall, so we
// allow it unconditionally rather than rely on the operator
// remembering to list it under smtp_allow_users.
//
// `allowUsers` is the operator-supplied set; each entry is resolved
// through smtpAllowlistLookupUser. Unknown or unparseable entries are
// reported to stderr (matching the legacy behavior in createOutputChain)
// and skipped, so a typo in the YAML does not crash the firewall engine.
func resolveSMTPAllowedUIDs(allowUsers []string) []uint32 {
// Cap the size hint so a pathological config (or future caller bug)
// cannot drive a multi-gigabyte allocation; the +2 covers root and
// mailnull which are added unconditionally below.
const smtpAllowHintCap = 1 << 16
hint := len(allowUsers)
if hint > smtpAllowHintCap {
hint = smtpAllowHintCap
}
seen := make(map[uint32]struct{}, hint+2)
out := make([]uint32, 0, hint+2)
add := func(uid uint32) {
if _, ok := seen[uid]; ok {
return
}
seen[uid] = struct{}{}
out = append(out, uid)
}
add(0)
if u, err := smtpAllowlistLookupUser("mailnull"); err == nil {
if uid, parseErr := strconv.ParseUint(u.Uid, 10, 32); parseErr == nil {
add(uint32(uid))
}
}
for _, name := range allowUsers {
u, err := smtpAllowlistLookupUser(name)
if err != nil {
fmt.Fprintf(os.Stderr, "firewall: smtp_allow_users: unknown user %q\n", name)
continue
}
uid, err := strconv.ParseUint(u.Uid, 10, 32)
if err != nil {
fmt.Fprintf(os.Stderr, "firewall: smtp_allow_users: invalid uid for %s: %v\n", name, err)
continue
}
add(uint32(uid))
}
return out
}
package firewall
import (
"encoding/json"
"errors"
"net"
"os"
"path/filepath"
"time"
)
// ErrPermanentBlock requires an explicit unblock before a timed Web UI block.
var ErrPermanentBlock = errors.New("IP is permanently blocked; unblock it explicitly before applying a timed block")
// ErrLongerBlock requires an explicit unblock before shortening a timed block.
var ErrLongerBlock = errors.New("IP has a longer block; unblock it explicitly before shortening its lifetime")
// ErrConfirmPending refuses admission's block batches while an
// apply-confirmed window can roll the firewall back (design C9).
var ErrConfirmPending = errors.New("a firewall change awaits confirmation")
// ErrBlockRules refuses admission's block batches while the rules that drop
// blocked addresses are missing, or changed during a batch.
var ErrBlockRules = errors.New("the firewall rules that drop blocked addresses are missing or changed")
// ErrAdmissionOwnsBlock refuses an automatic rewrite of a block an
// admission action wrote: its owner retires it (design C8).
var ErrAdmissionOwnsBlock = errors.New("an admission action owns the block")
// ErrBlockChanged prevents undo from replacing a later firewall decision.
var ErrBlockChanged = errors.New("block changed since the action; undo is no longer available")
// ErrStateNotKeptAside requires a new startup after unreadable state could
// not be preserved. Retrying with a fresh engine can miss an unsynced rename.
var ErrStateNotKeptAside = errors.New("firewall state unreadable and not kept aside")
// SameBlockedEntry compares snapshots without relying on time.Time locations.
func SameBlockedEntry(a, b BlockedEntry) bool {
return a.IP == b.IP && a.Reason == b.Reason && a.Source == b.Source &&
a.BlockedAt.Equal(b.BlockedAt) && a.ExpiresAt.Equal(b.ExpiresAt) &&
a.ActionID == b.ActionID && a.Unverified == b.Unverified && a.Automatic == b.Automatic &&
a.LoginSpray == b.LoginSpray
}
// LoadState reads the authoritative firewall state file directly without requiring
// a running engine. A missing state file is a valid fresh-host state and returns
// an empty FirewallState. Only enforced blocks are returned: an unverified row is
// held for recovery and never restored to the kernel.
func LoadState(statePath string) (*FirewallState, error) {
stateFile := filepath.Join(statePath, "firewall", "state.json")
// #nosec G304 -- filepath.Join under operator-configured statePath.
data, err := os.ReadFile(stateFile)
if err != nil {
if os.IsNotExist(err) {
return &FirewallState{}, nil
}
return nil, err
}
var state FirewallState
if err := json.Unmarshal(data, &state); err != nil {
return nil, err
}
// The engine keeps an unverified row over its address aliases, even
// after expiry. Filtering it alone would expose a duplicate it never seeds.
held := make(map[string]bool)
for _, entry := range state.Blocked {
if !entry.Unverified {
continue
}
if ip := net.ParseIP(entry.IP); ip != nil {
held[ip.String()] = true
}
}
// Clean expired entries
now := time.Now()
var active []BlockedEntry
for _, entry := range state.Blocked {
if len(held) > 0 {
if ip := net.ParseIP(entry.IP); ip != nil && held[ip.String()] {
continue
}
}
if !entry.Unverified && (entry.ExpiresAt.IsZero() || now.Before(entry.ExpiresAt)) {
active = append(active, entry)
}
}
state.Blocked = active
var activeNets []SubnetEntry
for _, entry := range state.BlockedNet {
if entry.ExpiresAt.IsZero() || now.Before(entry.ExpiresAt) {
activeNets = append(activeNets, entry)
}
}
state.BlockedNet = activeNets
var activeAllowed []AllowedEntry
for _, entry := range state.Allowed {
if entry.ExpiresAt.IsZero() || now.Before(entry.ExpiresAt) {
activeAllowed = append(activeAllowed, entry)
}
}
state.Allowed = activeAllowed
return &state, nil
}
//go:build linux
package firewall
// Supported reports whether this build includes the nftables engine.
func Supported() bool { return true }
// Package forensic produces evidence archives for incident response.
//
// A snapshot bundles the structured outputs an operator needs to hand a
// customer after a database-layer compromise: full trigger / event /
// routine definitions, the admin user roster, active session metadata,
// and the recent-mtime list under the account's document roots. Wrapped
// in a tar+gzip with a manifest and a SHA256 sidecar.
//
// The snapshot intentionally excludes credentials. Password rotation is
// a separate runbook step; bundling new credentials with the evidence
// archive would conflate two opposing flows (hand-to-customer for
// audit vs hand-to-ops for rotation) and risk credential leakage if
// the archive is later shared casually.
package forensic
import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"syscall"
"time"
"github.com/pidginhost/csm/internal/safepath"
)
// SchemaTarget identifies one MySQL schema to include in the snapshot
// along with the WordPress table prefix needed to enumerate the user
// roster correctly. Discovered from wp-config.php in the account's
// document roots by the production hook.
type SchemaTarget struct {
Schema string
TablePrefix string
ConfigPath string
}
// DiscoveryAudit records why a snapshot captured the targets it did.
// It is written into the manifest so operators can validate the
// evidence boundary without inspecting the host again.
type DiscoveryAudit struct {
AccountRoot string
PrivatePathsExcluded bool
PrivateTopPaths []string
SkippedPaths []SkippedPath
}
// SkippedPath records one discovery path that was not safe or useful
// to capture.
type SkippedPath struct {
Path string
Reason string
}
// Sources lets the caller swap each I/O dependency for a test double.
// Production wiring lives in cmd/csm; tests pass deterministic
// closures.
type Sources struct {
DiscoverTargets func(account string) []SchemaTarget
DumpSchema func(schema string) ([]byte, error)
ListAdmins func(schema, tablePrefix string) ([]byte, error)
ListSessions func(schema, tablePrefix string) ([]byte, error)
ListRecentFiles func(accountRoot string, since time.Time) ([]byte, error)
}
// Snapshot is the operator-facing configuration. Account and OutPath
// are required; Sources is required when running outside the default
// production wiring (see cmd/csm for the defaults).
type Snapshot struct {
Account string
OutPath string
Timestamp time.Time
DiscoveryAudit DiscoveryAudit
Sources Sources
}
var accountNamePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,32}$`)
var schemaNamePattern = regexp.MustCompile(`^[A-Za-z0-9_@]+$`)
var tablePrefixPattern = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
// AccountNameValid keeps the account string conservative enough to use
// in archive entry names, manifest keys, and shell-free SQL queries.
func AccountNameValid(name string) bool {
return accountNamePattern.MatchString(name)
}
func schemaNameValid(name string) bool {
return schemaNamePattern.MatchString(name)
}
func tablePrefixValid(name string) bool {
return tablePrefixPattern.MatchString(name)
}
// Write builds the archive at s.OutPath and a `<out>.sha256` sidecar,
// returning the archive path and the SHA256 hex digest. Errors abort
// before any partial state is written.
func (s Snapshot) Write() (string, string, error) {
if !AccountNameValid(s.Account) {
return "", "", fmt.Errorf("invalid account name: %q", s.Account)
}
if s.OutPath == "" {
return "", "", errors.New("forensic snapshot: OutPath required")
}
if err := ValidateOutPath(s.Account, s.OutPath); err != nil {
return "", "", err
}
if s.Sources.DiscoverTargets == nil {
return "", "", errors.New("forensic snapshot: Sources.DiscoverTargets required")
}
ts := s.Timestamp
if ts.IsZero() {
ts = time.Now().UTC()
}
targets := s.Sources.DiscoverTargets(s.Account)
sort.Slice(targets, func(i, j int) bool { return targets[i].Schema < targets[j].Schema })
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
tw := tar.NewWriter(gw)
var manifestB strings.Builder
fmt.Fprintf(&manifestB, "account=%s\n", s.Account)
fmt.Fprintf(&manifestB, "timestamp=%s\n", ts.UTC().Format(time.RFC3339))
fmt.Fprintf(&manifestB, "schema_count=%d\n", len(targets))
writeDiscoveryAudit(&manifestB, s.DiscoveryAudit)
var validTargets, invalidTargets int
var dumpOK, dumpErr int
var adminsOK, adminsErr int
var sessionsOK, sessionsErr int
recentStatus := "disabled"
for i, tgt := range targets {
if !schemaNameValid(tgt.Schema) || !tablePrefixValid(tgt.TablePrefix) {
invalidTargets++
fmt.Fprintf(&manifestB, "invalid_target=%d schema=%q table_prefix=%q\n", i, tgt.Schema, tgt.TablePrefix)
name := "schema/invalid-target-" + strconv.Itoa(i) + ".err"
data := []byte(fmt.Sprintf("invalid schema target: schema=%q table_prefix=%q\n", tgt.Schema, tgt.TablePrefix))
if err := writeArchiveEntry(tw, name, data, ts); err != nil {
return "", "", err
}
continue
}
validTargets++
fmt.Fprintf(&manifestB, "schema=%s table_prefix=%s", tgt.Schema, tgt.TablePrefix)
if tgt.ConfigPath != "" {
fmt.Fprintf(&manifestB, " config_path=%q", tgt.ConfigPath)
}
fmt.Fprint(&manifestB, "\n")
// Schema dump.
if s.Sources.DumpSchema != nil {
data, err := s.Sources.DumpSchema(tgt.Schema)
name := "schema/" + tgt.Schema + "-routines.sql"
if err != nil {
dumpErr++
name += ".err"
data = []byte(err.Error() + "\n")
} else {
dumpOK++
}
if werr := writeArchiveEntry(tw, name, data, ts); werr != nil {
return "", "", werr
}
}
// Admin roster.
if s.Sources.ListAdmins != nil {
data, err := s.Sources.ListAdmins(tgt.Schema, tgt.TablePrefix)
name := "schema/" + tgt.Schema + "-admins.tsv"
if err != nil {
adminsErr++
name += ".err"
data = []byte(err.Error() + "\n")
} else {
adminsOK++
}
if werr := writeArchiveEntry(tw, name, data, ts); werr != nil {
return "", "", werr
}
}
// Sessions.
if s.Sources.ListSessions != nil {
data, err := s.Sources.ListSessions(tgt.Schema, tgt.TablePrefix)
name := "schema/" + tgt.Schema + "-sessions.tsv"
if err != nil {
sessionsErr++
name += ".err"
data = []byte(err.Error() + "\n")
} else {
sessionsOK++
}
if werr := writeArchiveEntry(tw, name, data, ts); werr != nil {
return "", "", werr
}
}
}
// Recent files.
if s.Sources.ListRecentFiles != nil {
data, err := s.Sources.ListRecentFiles("/home/"+s.Account, ts.Add(-7*24*time.Hour))
name := "files/recent-mtimes.tsv"
if err != nil {
recentStatus = "error"
name += ".err"
data = []byte(err.Error() + "\n")
} else {
recentStatus = "ok"
}
if werr := writeArchiveEntry(tw, name, data, ts); werr != nil {
return "", "", werr
}
}
fmt.Fprintf(&manifestB, "valid_target_count=%d\n", validTargets)
fmt.Fprintf(&manifestB, "invalid_target_count=%d\n", invalidTargets)
fmt.Fprintf(&manifestB, "dump_success_count=%d\n", dumpOK)
fmt.Fprintf(&manifestB, "dump_error_count=%d\n", dumpErr)
fmt.Fprintf(&manifestB, "admins_success_count=%d\n", adminsOK)
fmt.Fprintf(&manifestB, "admins_error_count=%d\n", adminsErr)
fmt.Fprintf(&manifestB, "sessions_success_count=%d\n", sessionsOK)
fmt.Fprintf(&manifestB, "sessions_error_count=%d\n", sessionsErr)
fmt.Fprintf(&manifestB, "recent_mtimes_status=%s\n", recentStatus)
// Manifest last so the schema list reflects what was actually
// processed.
if err := writeArchiveEntry(tw, "manifest.txt", []byte(manifestB.String()), ts); err != nil {
return "", "", err
}
if err := tw.Close(); err != nil {
return "", "", fmt.Errorf("closing tar: %w", err)
}
if err := gw.Close(); err != nil {
return "", "", fmt.Errorf("closing gzip: %w", err)
}
if err := writeNewFile(s.OutPath, buf.Bytes()); err != nil {
return "", "", fmt.Errorf("writing archive: %w", err)
}
sum := sha256.Sum256(buf.Bytes())
hexSum := hex.EncodeToString(sum[:])
sidecar := s.OutPath + ".sha256"
sidecarBody := fmt.Sprintf("%s %s\n", hexSum, filepath.Base(s.OutPath))
if err := writeNewFile(sidecar, []byte(sidecarBody)); err != nil {
if removeErr := os.Remove(s.OutPath); removeErr != nil {
return "", "", fmt.Errorf("writing sidecar: %w; removing incomplete archive: %v", err, removeErr)
}
return "", "", fmt.Errorf("writing sidecar: %w", err)
}
return s.OutPath, hexSum, nil
}
// writeNewFile creates path as a brand-new 0600 file and writes data to it.
// The destination is operator-chosen and often sits in a world-writable
// directory, where the compromised account can pre-plant a symlink; an
// exclusive, non-following create refuses anything already at the path
// instead of writing evidence through it.
func writeNewFile(path string, data []byte) error {
// #nosec G304 -- path is the operator's --out argument; O_EXCL|O_NOFOLLOW
// refuse a pre-existing file or symlink.
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600)
if err != nil {
return err
}
if _, err := f.Write(data); err != nil {
_ = f.Close()
_ = os.Remove(path)
return err
}
if err := f.Close(); err != nil {
_ = os.Remove(path)
return err
}
return nil
}
func writeDiscoveryAudit(b *strings.Builder, audit DiscoveryAudit) {
if audit.AccountRoot != "" {
fmt.Fprintf(b, "discovery_root=%q\n", audit.AccountRoot)
}
if audit.PrivatePathsExcluded {
fmt.Fprintln(b, "private_paths_excluded=true")
}
if len(audit.PrivateTopPaths) > 0 {
paths := append([]string(nil), audit.PrivateTopPaths...)
sort.Strings(paths)
fmt.Fprintf(b, "private_top_excluded=%q\n", strings.Join(paths, ","))
}
for i, skipped := range audit.SkippedPaths {
fmt.Fprintf(b, "skipped_path=%d path=%q reason=%q\n", i, skipped.Path, skipped.Reason)
}
}
// ValidateOutPath rejects destinations that would land inside the
// target account's home directory. Writing forensic evidence somewhere
// the suspect user can read defeats the point.
func ValidateOutPath(account, outPath string) error {
if !AccountNameValid(account) {
return fmt.Errorf("invalid account name: %q", account)
}
abs, err := filepath.Abs(outPath)
if err != nil {
return fmt.Errorf("resolving out path: %w", err)
}
home := filepath.Clean("/home/" + account)
homes := []string{home}
if realHome, err := filepath.EvalSymlinks(home); err == nil {
homes = append(homes, realHome)
}
paths := []string{abs}
if realAbs, err := filepath.EvalSymlinks(abs); err == nil {
paths = append(paths, realAbs)
}
if realParent, err := filepath.EvalSymlinks(filepath.Dir(abs)); err == nil {
paths = append(paths, filepath.Join(realParent, filepath.Base(abs)))
}
for _, h := range homes {
for _, p := range paths {
if pathWithin(h, p) {
return fmt.Errorf("out path must not be inside /home/%s/", account)
}
}
}
return nil
}
func pathWithin(root, path string) bool {
return safepath.Within(path, root)
}
// writeArchiveEntry adds a single file to the tar stream with a fixed
// mtime so identical inputs produce byte-identical archives. The mode
// is 0600 because forensic content is operator-only.
func writeArchiveEntry(tw *tar.Writer, name string, data []byte, ts time.Time) error {
if unsafeArchiveEntryName(name) {
return fmt.Errorf("unsafe archive entry name: %q", name)
}
hdr := &tar.Header{
Name: name,
Size: int64(len(data)),
Mode: 0o600,
ModTime: ts.UTC(),
}
if err := tw.WriteHeader(hdr); err != nil {
return fmt.Errorf("tar header %s: %w", name, err)
}
if _, err := tw.Write(data); err != nil {
return fmt.Errorf("tar body %s: %w", name, err)
}
return nil
}
func unsafeArchiveEntryName(name string) bool {
if name == "" || filepath.IsAbs(name) || strings.Contains(name, `\`) {
return true
}
clean := filepath.Clean(name)
return clean != name || clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator))
}
package geoip
// KnownEditions returns the MaxMind database editions CSM supports in the
// settings UI. The first slice is the free GeoLite2 family; the second is
// the paid GeoIP2 family. The lists are curated to what MaxMind actually
// publishes via the geoipupdate protocol; adding an edition here makes it
// selectable in the Settings → GeoIP → Database editions dropdown.
func KnownEditions() (free, commercial []string) {
free = []string{
"GeoLite2-City",
"GeoLite2-Country",
"GeoLite2-ASN",
}
commercial = []string{
"GeoIP2-City",
"GeoIP2-Country",
"GeoIP2-ISP",
"GeoIP2-Domain",
"GeoIP2-Connection-Type",
"GeoIP2-Anonymous-IP",
"GeoIP2-Enterprise",
}
return free, commercial
}
// Package geoip provides IP geolocation via MaxMind GeoLite2 databases
// and on-demand RDAP lookups for detailed ISP/org information.
package geoip
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/netip"
"os"
"path/filepath"
"sync"
"time"
"github.com/oschwald/maxminddb-golang/v2"
)
// Info contains geolocation and network information for an IP.
type Info struct {
IP string `json:"ip"`
Country string `json:"country"` // ISO 3166-1 alpha-2 (e.g. "US")
CountryName string `json:"country_name"` // Full name (e.g. "United States")
City string `json:"city,omitempty"`
ASN uint `json:"asn,omitempty"` // Autonomous System Number
ASOrg string `json:"as_org,omitempty"` // AS Organization (ISP)
Network string `json:"network,omitempty"` // CIDR range
RDAPOrg string `json:"rdap_org,omitempty"` // Detailed org from RDAP (on-demand)
RDAPName string `json:"rdap_name,omitempty"` // Network name from RDAP
RDAPCountry string `json:"rdap_country,omitempty"` // Country from RDAP
}
// DB holds the MaxMind database readers.
type DB struct {
mu sync.RWMutex
cityDB *maxminddb.Reader
asnDB *maxminddb.Reader
dbDir string
rdapMu sync.Mutex
rdapTTL map[string]rdapCacheEntry
}
type rdapCacheEntry struct {
info Info
fetched time.Time
// failed marks a lookup that did not complete (transport error,
// non-200, undecodable body). Such an entry is retried after
// rdapNegativeTTL instead of standing as an empty answer for a day.
failed bool
}
// rdapBaseURL is the RDAP bootstrap endpoint; var so tests can point it at
// a local server. rdapNegativeTTL bounds how long a failed lookup is kept
// before the address is tried again.
var (
rdapBaseURL = "https://rdap.org/ip/"
rdapNegativeTTL = 10 * time.Minute
rdapPositiveTTL = 24 * time.Hour
rdapMaxResponseBytes = int64(1 << 20)
errRDAPLookupIncomplete = errors.New("rdap lookup did not complete")
)
// MaxMind GeoLite2 record structures
type cityRecord struct {
Country struct {
ISOCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"country"`
City struct {
Names map[string]string `maxminddb:"names"`
} `maxminddb:"city"`
}
type asnRecord struct {
ASN uint `maxminddb:"autonomous_system_number"`
Org string `maxminddb:"autonomous_system_organization"`
}
// Open loads MaxMind databases from the given directory.
// Expects GeoLite2-City.mmdb and/or GeoLite2-ASN.mmdb.
// Returns nil if no databases found (graceful degradation).
func Open(dbDir string) *DB {
if dbDir == "" {
return nil
}
db := &DB{
dbDir: dbDir,
rdapTTL: make(map[string]rdapCacheEntry),
}
cityPath := filepath.Join(dbDir, "GeoLite2-City.mmdb")
if r, err := maxminddb.Open(cityPath); err == nil {
db.cityDB = r
fmt.Fprintf(os.Stderr, "geoip: loaded %s\n", cityPath)
}
asnPath := filepath.Join(dbDir, "GeoLite2-ASN.mmdb")
if r, err := maxminddb.Open(asnPath); err == nil {
db.asnDB = r
fmt.Fprintf(os.Stderr, "geoip: loaded %s\n", asnPath)
}
if db.cityDB == nil && db.asnDB == nil {
fmt.Fprintf(os.Stderr, "geoip: no databases found in %s (download GeoLite2-City.mmdb and GeoLite2-ASN.mmdb)\n", dbDir)
return nil
}
return db
}
// Close releases database resources.
func (db *DB) Close() {
if db == nil {
return
}
db.mu.Lock()
defer db.mu.Unlock()
if db.cityDB != nil {
_ = db.cityDB.Close()
}
if db.asnDB != nil {
_ = db.asnDB.Close()
}
}
// Reload opens new database readers and swaps them in atomically.
// Opens replacement readers first - if both fail, the old readers stay in place.
// If one succeeds and the other fails, only the successful one is swapped.
func (db *DB) Reload() error {
if db == nil {
return fmt.Errorf("geoip: cannot reload nil DB")
}
cityPath := filepath.Join(db.dbDir, "GeoLite2-City.mmdb")
asnPath := filepath.Join(db.dbDir, "GeoLite2-ASN.mmdb")
// Open replacements before taking lock
newCity, cityErr := maxminddb.Open(cityPath)
newASN, asnErr := maxminddb.Open(asnPath)
if cityErr != nil && asnErr != nil {
return fmt.Errorf("geoip: reload failed - city: %v, asn: %v", cityErr, asnErr)
}
db.mu.Lock()
if newCity != nil {
if db.cityDB != nil {
_ = db.cityDB.Close()
}
db.cityDB = newCity
}
if newASN != nil {
if db.asnDB != nil {
_ = db.asnDB.Close()
}
db.asnDB = newASN
}
db.mu.Unlock()
if cityErr != nil {
fmt.Fprintf(os.Stderr, "geoip: reload warning - city DB failed: %v\n", cityErr)
}
if asnErr != nil {
fmt.Fprintf(os.Stderr, "geoip: reload warning - ASN DB failed: %v\n", asnErr)
}
return nil
}
// OpenFresh creates a new DB from databases on disk.
// Use when no DB existed at startup and databases have since been downloaded.
// Returns nil if no databases found (same behavior as Open).
func OpenFresh(dbDir string) *DB {
return Open(dbDir)
}
// Lookup returns geolocation info for an IP from local MaxMind databases.
// Fast (microseconds), no network calls.
func (db *DB) Lookup(ip string) Info {
info := Info{IP: ip}
if db == nil {
return info
}
addr, err := netip.ParseAddr(ip)
if err != nil {
return info
}
db.mu.RLock()
defer db.mu.RUnlock()
if db.cityDB != nil {
var record cityRecord
result := db.cityDB.Lookup(addr)
if err := result.Decode(&record); err == nil {
info.Country = record.Country.ISOCode
info.CountryName = record.Country.Names["en"]
info.City = record.City.Names["en"]
if prefix := result.Prefix(); prefix.IsValid() {
info.Network = prefix.String()
}
}
}
if db.asnDB != nil {
var record asnRecord
result := db.asnDB.Lookup(addr)
if err := result.Decode(&record); err == nil {
info.ASN = record.ASN
info.ASOrg = record.Org
}
}
return info
}
// LookupWithRDAP returns geolocation info plus on-demand RDAP details.
// The RDAP lookup is cached for 24 hours.
func (db *DB) LookupWithRDAP(ip string) Info {
info := db.Lookup(ip)
// Check RDAP cache
db.rdapMu.Lock()
if cached, ok := db.rdapTTL[ip]; ok {
ttl := rdapPositiveTTL
if cached.failed {
ttl = rdapNegativeTTL
}
if time.Since(cached.fetched) < ttl {
db.rdapMu.Unlock()
info.RDAPOrg = cached.info.RDAPOrg
info.RDAPName = cached.info.RDAPName
info.RDAPCountry = cached.info.RDAPCountry
return info
}
}
db.rdapMu.Unlock()
// Fetch from RDAP. A failure is cached only briefly so an outage does
// not blank the registry data for a day.
rdapInfo, err := fetchRDAP(ip)
info.RDAPOrg = rdapInfo.RDAPOrg
info.RDAPName = rdapInfo.RDAPName
info.RDAPCountry = rdapInfo.RDAPCountry
// Cache
db.rdapMu.Lock()
db.rdapTTL[ip] = rdapCacheEntry{info: rdapInfo, fetched: time.Now(), failed: err != nil}
db.evictRDAPLocked()
db.rdapMu.Unlock()
return info
}
// maxRDAPCacheEntries hard-caps the RDAP cache. Caller holds db.rdapMu.
const maxRDAPCacheEntries = 10000
// evictRDAPLocked bounds the RDAP cache. It first drops expired entries; if the
// map is still over the cap (every entry fresh, e.g. a burst of distinct
// lookups within the 24h TTL), it evicts the oldest entries until at the cap so
// the map cannot grow without bound. Caller holds db.rdapMu.
func (db *DB) evictRDAPLocked() {
if len(db.rdapTTL) <= maxRDAPCacheEntries {
return
}
for k, v := range db.rdapTTL {
if time.Since(v.fetched) > 24*time.Hour {
delete(db.rdapTTL, k)
}
}
for len(db.rdapTTL) > maxRDAPCacheEntries {
var oldestKey string
var oldest time.Time
for k, v := range db.rdapTTL {
if oldestKey == "" || v.fetched.Before(oldest) {
oldestKey = k
oldest = v.fetched
}
}
if oldestKey == "" {
break
}
delete(db.rdapTTL, oldestKey)
}
}
// RDAP lookup - fetches from the appropriate RIR
func fetchRDAP(ip string) (Info, error) {
var info Info
url := rdapBaseURL + ip
client := &http.Client{Timeout: 5 * time.Second}
resp, err := client.Get(url)
if err != nil {
return info, fmt.Errorf("%w: %v", errRDAPLookupIncomplete, err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return info, fmt.Errorf("%w: HTTP %d", errRDAPLookupIncomplete, resp.StatusCode)
}
var rdap struct {
Name string `json:"name"`
Country string `json:"country"`
Handle string `json:"handle"`
Entities []struct {
VCardArray []interface{} `json:"vcardArray"`
Roles []string `json:"roles"`
} `json:"entities"`
}
// Read one byte past the cap so a complete JSON prefix with an oversized
// trailing body cannot be accepted and cached as a positive answer.
data, err := io.ReadAll(io.LimitReader(resp.Body, rdapMaxResponseBytes+1))
if err != nil {
return info, fmt.Errorf("%w: %v", errRDAPLookupIncomplete, err)
}
if int64(len(data)) > rdapMaxResponseBytes {
return info, fmt.Errorf("%w: response exceeds %d bytes", errRDAPLookupIncomplete, rdapMaxResponseBytes)
}
if err := json.Unmarshal(data, &rdap); err != nil {
return info, fmt.Errorf("%w: %v", errRDAPLookupIncomplete, err)
}
info.RDAPName = rdap.Name
info.RDAPCountry = rdap.Country
// Extract org name from entities
for _, entity := range rdap.Entities {
for _, role := range entity.Roles {
if role == "registrant" || role == "abuse" {
if len(entity.VCardArray) >= 2 {
if props, ok := entity.VCardArray[1].([]interface{}); ok {
for _, prop := range props {
if arr, ok := prop.([]interface{}); ok && len(arr) >= 4 {
if name, ok := arr[0].(string); ok && name == "fn" {
if val, ok := arr[3].(string); ok {
info.RDAPOrg = val
}
}
}
}
}
}
}
}
}
return info, nil
}
package geoip
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/oschwald/maxminddb-golang/v2"
)
const (
maxMindBaseURL = "https://download.maxmind.com/geoip/databases"
maxDownloadSize = 150 * 1024 * 1024 // 150MB
// maxExtractedSize bounds the size of the .mmdb entry we extract from
// the tar.gz. Real GeoLite2 files are ~70MB; 500MiB is generous. The
// check sits on the tar header and prevents io.Copy from writing a
// bomb-compressed entry to disk.
maxExtractedSize = 500 * 1024 * 1024
downloadTimeout = 120 * time.Second
)
// EditionResult reports the outcome of updating a single GeoLite2 edition.
type EditionResult struct {
Edition string // e.g. "GeoLite2-City"
Status string // "updated", "up_to_date", "error"
Err error // nil unless Status == "error"
}
// Update downloads GeoLite2 databases from MaxMind's direct download API.
// Returns one EditionResult per edition. Returns nil if credentials are empty.
func Update(dbDir, accountID, licenseKey string, editions []string) []EditionResult {
if accountID == "" || licenseKey == "" {
return nil
}
if err := os.MkdirAll(dbDir, 0700); err != nil {
result := make([]EditionResult, len(editions))
for i, ed := range editions {
result[i] = EditionResult{Edition: ed, Status: "error", Err: fmt.Errorf("creating directory: %w", err)}
}
return result
}
client := &http.Client{Timeout: downloadTimeout}
results := make([]EditionResult, len(editions))
for i, edition := range editions {
results[i] = updateEdition(client, dbDir, accountID, licenseKey, edition)
}
return results
}
func updateEdition(client *http.Client, dbDir, accountID, licenseKey, edition string) EditionResult {
return updateEditionWithURL(client, dbDir, accountID, licenseKey, edition, maxMindBaseURL)
}
func updateEditionWithURL(client *http.Client, dbDir, accountID, licenseKey, edition, baseURL string) EditionResult {
url := fmt.Sprintf("%s/%s/download?suffix=tar.gz", baseURL, edition)
markerPath := filepath.Join(dbDir, ".last-modified-"+edition)
// Read stored Last-Modified
storedLM := ""
// #nosec G304 -- filepath.Join under operator-configured dbDir.
if data, err := os.ReadFile(markerPath); err == nil {
storedLM = strings.TrimSpace(string(data))
}
// HEAD request to check Last-Modified
headReq, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return EditionResult{Edition: edition, Status: "error", Err: err}
}
headReq.SetBasicAuth(accountID, licenseKey)
headResp, err := client.Do(headReq)
if err != nil {
return EditionResult{Edition: edition, Status: "error", Err: fmt.Errorf("HEAD request: %w", err)}
}
headResp.Body.Close()
if headResp.StatusCode == 401 {
return EditionResult{Edition: edition, Status: "error", Err: fmt.Errorf("invalid MaxMind credentials")}
}
if headResp.StatusCode == 429 {
return EditionResult{Edition: edition, Status: "error", Err: fmt.Errorf("rate limited by MaxMind")}
}
if headResp.StatusCode != 200 {
return EditionResult{Edition: edition, Status: "error", Err: fmt.Errorf("HEAD returned HTTP %d", headResp.StatusCode)}
}
remoteLM := headResp.Header.Get("Last-Modified")
if storedLM != "" && remoteLM != "" && storedLM == remoteLM {
return EditionResult{Edition: edition, Status: "up_to_date"}
}
// GET request to download
getReq, err := http.NewRequest("GET", url, nil)
if err != nil {
return EditionResult{Edition: edition, Status: "error", Err: err}
}
getReq.SetBasicAuth(accountID, licenseKey)
getResp, err := client.Do(getReq)
if err != nil {
return EditionResult{Edition: edition, Status: "error", Err: fmt.Errorf("download: %w", err)}
}
defer getResp.Body.Close()
if getResp.StatusCode != 200 {
return EditionResult{Edition: edition, Status: "error", Err: fmt.Errorf("download returned HTTP %d", getResp.StatusCode)}
}
// Reject oversized responses upfront if Content-Length is known
if getResp.ContentLength > maxDownloadSize {
return EditionResult{Edition: edition, Status: "error", Err: fmt.Errorf("download too large: %d bytes (max %d)", getResp.ContentLength, maxDownloadSize)}
}
// LimitReader as safety net for responses without Content-Length
mmdbTmpPath := filepath.Join(dbDir, edition+".mmdb.tmp")
if err := extractMMDB(io.LimitReader(getResp.Body, maxDownloadSize), mmdbTmpPath, edition); err != nil {
_ = os.Remove(mmdbTmpPath)
return EditionResult{Edition: edition, Status: "error", Err: fmt.Errorf("extract: %w", err)}
}
if err := validateMMDB(mmdbTmpPath); err != nil {
_ = os.Remove(mmdbTmpPath)
return EditionResult{Edition: edition, Status: "error", Err: fmt.Errorf("validate: %w", err)}
}
// Atomic install
destPath := filepath.Join(dbDir, edition+".mmdb")
if err := os.Rename(mmdbTmpPath, destPath); err != nil {
_ = os.Remove(mmdbTmpPath)
return EditionResult{Edition: edition, Status: "error", Err: fmt.Errorf("install: %w", err)}
}
// Save Last-Modified marker
if remoteLM != "" {
_ = os.WriteFile(markerPath, []byte(remoteLM), 0600)
}
return EditionResult{Edition: edition, Status: "updated"}
}
func validateMMDB(path string) error {
db, err := maxminddb.Open(path)
if err != nil {
return err
}
return db.Close()
}
// extractMMDB reads a tar.gz stream and extracts the .mmdb file to destPath.
// MaxMind tar.gz archives contain a single directory with the .mmdb inside,
// e.g. GeoLite2-City_20260328/GeoLite2-City.mmdb
func extractMMDB(r io.Reader, destPath, edition string) error {
gz, err := gzip.NewReader(r)
if err != nil {
return fmt.Errorf("gzip: %w", err)
}
defer func() { _ = gz.Close() }()
tr := tar.NewReader(gz)
suffix := edition + ".mmdb"
for {
header, err := tr.Next()
if err == io.EOF {
return fmt.Errorf("no %s found in archive", suffix)
}
if err != nil {
return fmt.Errorf("reading tar: %w", err)
}
if header.Typeflag != tar.TypeReg {
continue
}
if !strings.HasSuffix(header.Name, suffix) {
continue
}
if header.Size > maxExtractedSize {
return fmt.Errorf("archive entry too large: %d bytes", header.Size)
}
// #nosec G304 -- destPath is filepath.Join under operator-configured dbDir.
f, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return fmt.Errorf("creating %s: %w", destPath, err)
}
_, copyErr := io.Copy(f, io.LimitReader(tr, maxExtractedSize))
closeErr := f.Close()
if copyErr != nil {
return fmt.Errorf("writing mmdb: %w", copyErr)
}
if closeErr != nil {
return fmt.Errorf("closing mmdb: %w", closeErr)
}
return nil
}
}
package health
import (
"maps"
"slices"
"time"
"github.com/pidginhost/csm/internal/admission"
)
// AdmissionStatus is the admission ledger's status and the ingress health.
// CheckedAt is when the daemon read them; doctor judges the
// Critical-gap rule from it.
type AdmissionStatus struct {
CheckedAt time.Time `json:"checked_at"`
Ledger *admission.LedgerStatus `json:"ledger,omitempty"`
Ingress *admission.IngressHealth `json:"ingress,omitempty"`
Owner *AdmissionOwner `json:"owner,omitempty"`
}
// AdmissionOwner is what the ledger's owner knows that the ledger cannot
// show: why it is not running, its clock readings, where the ceiling came
// from, the legacy import and the inventory reads.
type AdmissionOwner struct {
// AuditError is the last delivery failure, including applied consumers.
// It clears only when a delivery pass succeeds.
AuditError string `json:"audit_error,omitempty"`
// Executing is set when admission runs automatic blocks rather than
// previewing or forecasting them: an outage of admission is then an
// outage of automatic blocking, and doctor fails where a preview only
// warns.
Executing bool `json:"executing"`
// Settled is set once this process's owner has completed a turn. The
// check that follows an upgrade runs within seconds of the start, and
// a failure there rolls the binary back to one that may refuse the
// ledger; until the first turn, and for two minutes at most, doctor
// rates an executing owner's outages as warnings.
Settled bool `json:"settled"`
// Error is why the owner is not running; empty once it is.
Error string `json:"error,omitempty"`
// TurnError is why the owner's last turn failed while the owner runs.
// The turn is retried from one second, backing off to once a minute;
// it clears when a turn succeeds.
TurnError string `json:"turn_error,omitempty"`
// RetirementError is the last fault reported while retiring uncertain
// blocks. Each affected block stays held and is retried; it clears once
// none is left faulted.
RetirementError string `json:"retirement_error,omitempty"`
// RetirementHeldBy names what keeps unknown outcomes from retiring
// while the ledger holds some: "engine" for an owner without the
// firewall engine, "policy" for a refused execution policy. Their
// addresses stay held until retirement can run.
RetirementHeldBy string `json:"retirement_held_by,omitempty"`
// TickError is why the last clock reading or its live start failed.
// Nothing is admitted until both succeed.
TickError string `json:"tick_error,omitempty"`
// ClockDegraded is set while the wall clock is behind the ledger's
// high-water mark or disagrees with the time since boot.
ClockDegraded bool `json:"clock_degraded"`
LastTick time.Time `json:"last_tick,omitempty"`
// CeilingSource says whether the ceiling is max_blocks_per_hour, its
// default or the largest ceiling the ledger accepts.
CeilingSource string `json:"ceiling_source,omitempty"`
// Import describes this process's import or retained charges found at
// startup, or why the legacy state could not be read when execution
// began. A read error is process-local; restart restores a known
// horizon without re-reading the legacy file.
Import *AdmissionImport `json:"import,omitempty"`
InventoryAt time.Time `json:"inventory_at,omitempty"`
InventoryError string `json:"inventory_error,omitempty"`
// DamageError is the latest drain cause that discarded damaged arrivals.
// It survives recovery from any later failure in that drain and is kept
// until the owner restarts.
DamageError string `json:"damage_error,omitempty"`
// ExecutionHeld is why automatic execution is held: block work waits,
// deferred as an unavailable engine, until a probe succeeds.
ExecutionHeld string `json:"execution_held,omitempty"`
// ExecutionHeldBy names what holds execution: "policy" for a refused
// execution policy, "firewall" for the firewall engine.
ExecutionHeldBy string `json:"execution_held_by,omitempty"`
// Verdict is the panel verdict stage of an owner with an engine.
Verdict *AdmissionVerdict `json:"verdict,omitempty"`
// DryRun is the dry-run forecast of an owner with an engine.
DryRun *AdmissionDryRun `json:"dry_run,omitempty"`
// Set combines the last scheduling turn's non-admission snapshot with
// current ledger holds outside a dry run; SetError is why that turn
// could not read the set.
Set *AdmissionSet `json:"set,omitempty"`
SetError string `json:"set_error,omitempty"`
// PreviewReplacements counts this process's previews that took the
// temporary slot of a lower-priority automatic block, which live
// admission would replace; PreviewReplacementsLastHour counts those of
// the last sixty one-minute buckets of process monotonic time.
PreviewReplacements uint64 `json:"preview_replacements"`
PreviewReplacementsLastHour uint64 `json:"preview_replacements_last_hour"`
// PreviewRotations and PreviewRotationsLastHour count the previews
// that took the slot of an equal-priority automatic block, which live
// admission would rotate out, in the same way.
PreviewRotations uint64 `json:"preview_rotations"`
PreviewRotationsLastHour uint64 `json:"preview_rotations_last_hour"`
}
// AdmissionSet is the temporary block set as automatic admission divides
// it: the firewall's entries and the ledger's automatic blocks,
// pending and verified. Capacity is the limit, or 10000 for an unlimited
// set; Unclaimed counts the entries admission never evicts (operator
// blocks, blocks of unknown origin and two per timed subnet) and Imported
// the legacy automatic blocks; DrainSeconds is how long the last of them
// has left, or -1 when a kernel expiry is unavailable. The partitions'
// free slots bound the block work a schedule serves.
type AdmissionSet struct {
Limit int `json:"limit"`
Unlimited bool `json:"unlimited"`
Capacity int `json:"capacity"`
Unclaimed int `json:"unclaimed"`
Imported int `json:"imported"`
DrainSeconds int64 `json:"drain_seconds"`
General AdmissionDryRunPartition `json:"general"`
Reserved AdmissionDryRunPartition `json:"reserved"`
}
// AdmissionDryRun is an executing owner's dry-run forecast: a
// model of the hourly ceiling and the temporary block capacity, never a
// record of applied blocks. The session fields and counts are the current
// forecast session's; a session starts at startup, on entry into dry run
// or after its model failed, and leaving dry run clears the forecast but
// keeps the last session's identity. The loss and error counters count
// this process and survive mode changes. LastHour sums sixty one-minute
// buckets of process monotonic time, so a wall-clock change cannot age
// them and a restart starts them at zero.
type AdmissionDryRun struct {
// Active is set while the owner forecasts instead of running blocks.
Active bool `json:"active"`
SessionID *string `json:"session_id"`
Since *time.Time `json:"since"`
ResetReason *string `json:"reset_reason"`
// ModelledKinds and PredictionScope name what the model covers.
ModelledKinds []string `json:"modelled_kinds"`
PredictionScope string `json:"prediction_scope"`
Occupancy *AdmissionDryRunOccupancy `json:"occupancy"`
Ceiling *AdmissionDryRunCeiling `json:"ceiling"`
// Picks and LastHour count the session's predictions by name, every
// prediction present.
Picks map[string]uint64 `json:"picks"`
LastHour map[string]uint64 `json:"last_hour"`
ExamplesLost uint64 `json:"examples_lost"`
SummariesLost uint64 `json:"summaries_lost"`
RecordErrors uint64 `json:"record_errors"`
// ExamplesStale is set while a purge of earlier sessions' examples is
// pending.
ExamplesStale bool `json:"examples_stale"`
// ForecastError is a fixed code for why forecasting last failed,
// cleared by a good turn; LastErrorAt keeps its time until the next
// session starts.
ForecastError *string `json:"forecast_error"`
LastErrorAt *time.Time `json:"last_error_at"`
// ComparisonIncomplete is set after a restart or a lost example,
// summary or comparison row.
ComparisonIncomplete bool `json:"comparison_incomplete"`
}
// AdmissionDryRunOccupancy is the temporary block capacity as the last
// snapshot of the forecast saw it: the real timed blocks, the forecast
// blocks the real sets do not hold, the listed legacy automatic blocks the
// forecast replaced, and what is left against the limit. Unlimited marks
// a zero limit; Limited marks a model that holds as many forecast blocks
// as it can.
type AdmissionDryRunOccupancy struct {
Actual int `json:"actual"`
ShadowOnly int `json:"shadow_only"`
Replaced int `json:"replaced"`
Predicted int `json:"predicted"`
Limit int `json:"limit"`
Unlimited bool `json:"unlimited"`
Limited bool `json:"limited"`
// Capacity is the finite capacity automatic admission is modelled
// within: the limit, or 10000 for an unlimited set. Unclaimed counts
// the real timed entries it never evicts (operator blocks, blocks of
// unknown origin and two per timed subnet) and Imported the real
// legacy automatic blocks; DrainSeconds is how long the last of them
// has left, or -1 when a kernel expiry is unavailable.
Capacity int `json:"capacity"`
Unclaimed int `json:"unclaimed"`
Imported int `json:"imported"`
DrainSeconds int64 `json:"drain_seconds"`
// General and Reserved divide the slots the unclaimed entries leave:
// a fifth, rounded up, is reserved for direct or corroborated work.
General AdmissionDryRunPartition `json:"general"`
Reserved AdmissionDryRunPartition `json:"reserved"`
// Shares is how the forecast shares each partition among its scopes.
Shares AdmissionDryRunShares `json:"shares"`
}
// AdmissionDryRunShares is how the forecast shares each partition among
// its scopes for equal-tier rotation.
type AdmissionDryRunShares struct {
General AdmissionDryRunShare `json:"general"`
Reserved AdmissionDryRunShare `json:"reserved"`
}
// AdmissionDryRunShare is one partition's sharing: the scopes holding its
// forecast blocks or waiting at the last turn, and the fewest slots each
// is due, its size divided among them; the remainder goes one each to the
// scopes after the session's cursor. Both are zero without a scope.
type AdmissionDryRunShare struct {
Scopes int `json:"scopes"`
Share int `json:"share"`
}
// AdmissionDryRunPartition is one modelled partition of the automatic
// slots: its size and the real and forecast automatic blocks in it.
type AdmissionDryRunPartition struct {
Size int `json:"size"`
Used int `json:"used"`
Free int `json:"free"`
}
// AdmissionDryRunCeiling is the forecast's own copy of the hourly ceiling.
type AdmissionDryRunCeiling struct {
Limit uint32 `json:"limit"`
GeneralUsed uint32 `json:"general_used"`
ReservedUsed uint32 `json:"reserved_used"`
GeneralBudget uint32 `json:"general_budget"`
ReservedBudget uint32 `json:"reserved_budget"`
}
// clone copies d, its collections and pointers included.
func (d *AdmissionDryRun) clone() *AdmissionDryRun {
if d == nil {
return nil
}
out := *d
out.SessionID, out.Since, out.ResetReason = clonePtr(d.SessionID), clonePtr(d.Since), clonePtr(d.ResetReason)
out.ForecastError, out.LastErrorAt = clonePtr(d.ForecastError), clonePtr(d.LastErrorAt)
out.Occupancy, out.Ceiling = clonePtr(d.Occupancy), clonePtr(d.Ceiling)
out.ModelledKinds = slices.Clone(d.ModelledKinds)
out.Picks, out.LastHour = maps.Clone(d.Picks), maps.Clone(d.LastHour)
return &out
}
func clonePtr[T any](p *T) *T {
if p == nil {
return nil
}
v := *p
return &v
}
// AdmissionVerdict is the verdict callback stage. The gauges
// are its current state. The counters and the failure window count this
// process's calls and start from zero at a restart; committed refusals
// stay in the ledger. Errors are fixed classes, never response text.
type AdmissionVerdict struct {
Enabled bool `json:"enabled"`
Epoch uint64 `json:"epoch"`
Pending int `json:"pending"`
InFlight uint64 `json:"in_flight"`
Ready int `json:"ready"`
StageRetiring bool `json:"stage_retiring"`
// Asked counts calls a worker started; each ends as one response,
// failure or cancellation.
Asked uint64 `json:"asked"`
ResponsesAllow uint64 `json:"responses_allow"`
ResponsesDefault uint64 `json:"responses_default"`
Failed uint64 `json:"failed"`
Cancelled uint64 `json:"cancelled"`
// Discarded counts jobs cancelled before their call and answers no
// longer usable when they arrived or were due.
Discarded uint64 `json:"discarded"`
ReadyEvicted uint64 `json:"ready_evicted"`
// AllowEndings counts committed verdict_allow refusals.
AllowEndings uint64 `json:"allow_endings"`
LastError *string `json:"last_error"`
LastErrorAt *time.Time `json:"last_error_at"`
// LastHourFailed counts failures in the sixty minutes of process
// monotonic time up to now.
LastHourFailed uint64 `json:"last_hour_failed"`
}
// AdmissionImport is the legacy hour's spend imported as charges. They
// count at least until an hour after At, the end of that legacy hour;
// conservative elapsed-time retention can hold them longer.
type AdmissionImport struct {
Units uint32 `json:"units"`
At time.Time `json:"at,omitempty"`
// Error is why the legacy counter could not be read: the ledger then
// started without credit.
Error string `json:"error,omitempty"`
}
// AdmissionProvider is implemented by a daemon that owns an admission
// ledger. Without one the snapshot carries no admission view, which must
// not read as a healthy ledger.
type AdmissionProvider interface {
AdmissionStatus() *AdmissionStatus
}
// Clone returns a snapshot whose pointers and collections belong to the caller.
func (in *AdmissionStatus) Clone() *AdmissionStatus {
if in == nil {
return nil
}
out := *in
if in.Ledger != nil {
ledger := *in.Ledger
ledger.Queue.Occupancy = slices.Clone(ledger.Queue.Occupancy)
ledger.Counters.Rows = slices.Clone(ledger.Counters.Rows)
ledger.Outcomes.Hour = slices.Clone(ledger.Outcomes.Hour)
ledger.Outcomes.Day = slices.Clone(ledger.Outcomes.Day)
ledger.Outcomes.Month = slices.Clone(ledger.Outcomes.Month)
ledger.Notices.Records = slices.Clone(ledger.Notices.Records)
out.Ledger = &ledger
}
if in.Ingress != nil {
ingress := *in.Ingress
out.Ingress = &ingress
}
if in.Owner != nil {
owner := *in.Owner
if in.Owner.Import != nil {
imp := *in.Owner.Import
owner.Import = &imp
}
if in.Owner.Verdict != nil {
v := *in.Owner.Verdict
if v.LastError != nil {
class := *v.LastError
v.LastError = &class
}
if v.LastErrorAt != nil {
at := *v.LastErrorAt
v.LastErrorAt = &at
}
owner.Verdict = &v
}
if in.Owner.Set != nil {
set := *in.Owner.Set
owner.Set = &set
}
owner.DryRun = in.Owner.DryRun.clone()
out.Owner = &owner
}
return &out
}
package health
import (
"maps"
"time"
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/queuehealth"
)
// Provider is the contract the daemon (or a stub for tests) implements
// so the snapshot builder doesn't depend on internal/daemon directly.
type Provider interface {
Hostname() string
StartedAt() time.Time
LatestScan() time.Time
BaselineAt() time.Time
WatcherStatuses() map[string]bool
StoreHealthy() bool
StoreSizeMB() float64
SeverityCounts() map[string]int
BlocklistSize() int
IncidentsOpen() int
BPFEnforcementActive() bool
HistoryCount() int
ConfigHash() string
BinaryHash() string
DryRunBlocksCount() int
AutomationStatus() AutomationStatus
UpdateInfo() UpdateInfo
Mode() string
// CorrelationAttribution is nil until the first active-set merge.
CorrelationAttribution() *CorrelationAttribution
QueueStatuses() map[string]queuehealth.Status
}
// Build assembles a Snapshot from the provider plus the static version
// string and capability list. Safe to call from any goroutine; the
// provider's accessors are expected to be lock-protected internally.
func Build(p Provider, version string, capabilities []string) Snapshot {
started := p.StartedAt()
uptime := int64(0)
if !started.IsZero() {
uptime = int64(time.Since(started).Seconds())
}
caps := append([]string(nil), capabilities...)
var wordpress map[string]WPVerificationCounts
if wp, ok := p.(WordPressVerificationProvider); ok {
wordpress = maps.Clone(wp.WordPressVerification())
}
var adm *AdmissionStatus
if ap, ok := p.(AdmissionProvider); ok {
adm = ap.AdmissionStatus().Clone()
}
var learning *LearningStatus
if lp, ok := p.(LearningProvider); ok {
learning = lp.LearningStatus().Clone()
}
return Snapshot{
WordPressVerification: wordpress,
Queues: maps.Clone(p.QueueStatuses()),
Version: version,
Hostname: p.Hostname(),
StartedAt: started,
UptimeSec: uptime,
LatestScan: p.LatestScan(),
BaselineAt: p.BaselineAt(),
BlocklistSize: p.BlocklistSize(),
IncidentsOpen: p.IncidentsOpen(),
BPFEnforcementActive: p.BPFEnforcementActive(),
HistoryCount: p.HistoryCount(),
Severities: cloneIntMap(p.SeverityCounts()),
Watchers: cloneBoolMap(p.WatcherStatuses()),
StoreHealthy: p.StoreHealthy(),
StoreSizeMB: p.StoreSizeMB(),
ConfigHash: p.ConfigHash(),
BinaryHash: p.BinaryHash(),
Capabilities: caps,
Backends: bpf.ActiveKinds(),
DryRunBlocks: p.DryRunBlocksCount(),
Automation: p.AutomationStatus(),
Update: p.UpdateInfo(),
Mode: p.Mode(),
CorrelationAttribution: cloneCorrelationAttribution(p.CorrelationAttribution()),
Admission: adm,
AntispamLearning: learning,
}
}
func cloneCorrelationAttribution(in *CorrelationAttribution) *CorrelationAttribution {
if in == nil {
return nil
}
return &CorrelationAttribution{
Current: cloneIntMap(in.Current),
Cumulative: cloneIntMap(in.Cumulative),
ActiveSetUpdates: in.ActiveSetUpdates,
Since: in.Since,
}
}
func cloneIntMap(m map[string]int) map[string]int {
out := make(map[string]int, len(m))
for k, v := range m {
out[k] = v
}
return out
}
func cloneBoolMap(m map[string]bool) map[string]bool {
out := make(map[string]bool, len(m))
for k, v := range m {
out[k] = v
}
return out
}
package health
import (
"github.com/pidginhost/csm/internal/bpf"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/maillog"
)
// Capabilities is the static list of features this build supports. Phpanel
// reads it via /api/v1/capabilities to feature-detect without version
// sniffing. Add a string here when shipping a feature; remove when ripping
// one out. Keep the base order stable; build-tag gated capabilities are
// appended only when this binary actually supports them.
//
// BPF capability strings (`bpf-...`) are appended dynamically based on the
// running kernel's accepted BPF program types. Their presence depends on
// build tag and host kernel and is therefore not stable across deployments.
func Capabilities() []string {
caps := []string{
"confd.dropins.v1", // P1
"profile.phpanel-agent.v1", // P1
"status.json.v1", // P2
"capabilities.v1", // P2
"doctor.v1", // P2
"config.schema.v1", // P2
"sd_notify.ready", // P2
"audit.fields.tenant.v1", // P3
"webhook.phpanel.v1", // P3
"events.sse.v1", // P3
"token.scope.readonly.v1", // P3
"mail.brute.account_key.v1", // P4
"ti.source.rspamd.v1", // P4
"auto_response.dry_run.v1", // P5
"infra_ips.guard.v1", // P5
"store.backup.v1", // P5
"ti.source.upstream.v1", // P6
"verdict.callback.v1", // P7
"systemd.dropin.example.v1", // P7
"incidents.v1",
"bpf_enforcement.available.v1",
"webui.prefs.v1", // operator preferences + saved views
"webui.undo.v1", // bulk-action undo
"mail.filter.exfil.v1", // BEC mail-filter exfiltration detection
"mail.queue.composition.v1",
"mail.forward_guard.v1", // opt-in MTA-native forward-guard (hold spam/backscatter forward copies)
"mail.antispam.v1", // opt-in inbound spam filtering through rspamd with per-account cPanel opt-out
"mail.antispam.postfix.v1", // the same filtering on Postfix hosts that pass mail to rspamd, with an operator exempt list
"mail.antispam.learning.v1", // opt-in learning from Spam folders and operator reports, health in status
"mail.antispam.learning.postfix.v1", // the same learning on Postfix hosts from Dovecot Junk folders, with a dedicated Redis
"detect.http_scanner_profile.v1", // URL scanner-profile detector + challenge/block action
"challenge.stats.v1",
"verified_bots.editor.v1", // operator-managed verified-bot allowlist (rDNS + IP ranges) with web editor
"status.firewall_health.v1", // status snapshot reports firewall enabled/managed state + block counts
"mode.observe.v1", // observe posture: detection and alerting without host changes
"status.queue_health.v1",
"status.admission.v1", // status carries the admission ledger, ingress and owner view
"token.expires_at.v1", // per-credential expiry
"token.scope.unban.v1", // firewall check and unban only
"token.allowed_ips.v1", // per-token source addresses
"confd.rehash.fragment.v1", // csm rehash --fragment re-signs one integration's conf.d files
"admission.action_show.v1",
"mail.forwarders.delete.v1",
"mcp.readonly.v1", // optional read-only MCP endpoint for support agents (mcp: block)
"alerts.daily_summary.v1", // handled attacks reach operators in one daily summary at 08:00
"alerts.customer_block.v1", // an automatic block of a known customer alerts at once (event customer_block)
}
if firewall.Supported() {
caps = append(caps,
"firewall.rollback.v1", // timed config rollback: apply, confirm, revert, survives a restart
"firewall.dos_exempt.v1", // ranges exempt from connection-rate and flood metering
"firewall.protocol_allow.v1", // peer-scoped portless protocols such as VRRP
"firewall.drop_ipv4_options.v1", // drops inbound IPv4 TCP that carries IP options
"firewall.tcp_in_allow.v1", // one TCP port opened to listed sources, ahead of the blocklists
)
}
if maillog.JournalSupported() {
caps = append(caps, "mail.source.journal.v1")
}
caps = appendBPFCaps(caps)
caps = appendActiveBPFFeatures(caps)
return caps
}
// appendActiveBPFFeatures adds one capability string per BPF-backed live
// monitor that is currently running on the kernel-side path (as opposed to
// its userspace fallback). Phases 2-4 extend this with their own feature
// keys; the test for each phase asserts the expected toggling.
func appendActiveBPFFeatures(out []string) []string {
if bpf.ActiveKind("connection_tracker") == bpf.BackendBPF {
out = append(out, "bpf-connection-tracker")
}
if bpf.ActiveKind("af_alg") == bpf.BackendBPF {
out = append(out, "bpf-af-alg-live")
}
if bpf.ActiveKind("exec_monitor") == bpf.BackendBPF {
out = append(out, "bpf-exec-monitor")
}
if bpf.ActiveKind("sensitive_files") == bpf.BackendBPF {
out = append(out, "bpf-sensitive-files")
}
if bpf.ActiveKind("placement") == bpf.BackendBPF {
out = append(out, "bpf-file-placement")
}
return out
}
// bpfCapabilities returns the cached probe result. Tests use this to assert
// that capability strings stay in sync with the shared BPF probe.
func bpfCapabilities() bpf.Capabilities { return bpf.Probe() }
// appendBPFCaps adds one capability string per BPF program type the kernel
// accepts. Phases 1-4 add a second helper alongside this one for per-feature
// "live monitor is currently running on BPF" strings.
func appendBPFCaps(out []string) []string {
caps := bpf.Probe()
if caps.LSMAttach {
out = append(out, "bpf-lsm-attach")
}
if caps.CgroupSock {
out = append(out, "bpf-cgroup-sock")
}
if caps.Tracepoint {
out = append(out, "bpf-tracepoint")
}
if caps.Ringbuf {
out = append(out, "bpf-ringbuf")
}
return out
}
package health
import "time"
// LearningStatus is the anti-spam learner's health while it runs, paused
// or not. PendingReports is nil when the last pass did not enumerate every
// folder, so an unknown backlog never reads as zero.
type LearningStatus struct {
Publishing bool `json:"publishing"`
LastPass time.Time `json:"last_pass,omitzero"`
PassUnfinished bool `json:"pass_unfinished,omitempty"`
PendingReports *int `json:"pending_reports,omitempty"`
OldestPending time.Time `json:"oldest_pending,omitzero"`
// StateError is a fixed code. Ledger counts are absent if they could
// not be read, so unavailable state never looks empty.
StateError string `json:"state_error,omitempty"`
LiveClusters *int `json:"live_clusters,omitempty"`
ZeroTargets *int `json:"zero_target_records,omitempty"`
UncertainIntents *int `json:"uncertain_intents,omitempty"`
BodyBytes *int64 `json:"body_bytes,omitempty"`
OldestUnchecked time.Time `json:"oldest_unchecked,omitzero"`
AuditCompleted time.Time `json:"audit_completed,omitzero"`
}
// LearningProvider is implemented by a daemon that can run the learner.
type LearningProvider interface {
LearningStatus() *LearningStatus
}
// Clone copies s, including its optional counts.
func (s *LearningStatus) Clone() *LearningStatus {
if s == nil {
return nil
}
out := *s
out.PendingReports = clonePtr(s.PendingReports)
out.LiveClusters = clonePtr(s.LiveClusters)
out.ZeroTargets = clonePtr(s.ZeroTargets)
out.UncertainIntents = clonePtr(s.UncertainIntents)
out.BodyBytes = clonePtr(s.BodyBytes)
return &out
}
package health
import (
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
// Snapshot is the unified machine-readable health view assembled from the
// running daemon (or, on a cold lookup, from on-disk state). It is the
// single source of truth for /api/v1/status, csm status --json, csm doctor,
// and the sd_notify readiness gate.
type Snapshot struct {
WordPressVerification map[string]WPVerificationCounts `json:"wordpress_verification,omitempty"`
Queues map[string]queuehealth.Status `json:"queues,omitempty"`
Version string `json:"version"`
Hostname string `json:"hostname"`
// Mode is the operator's posture: "enforce" or "observe". An observe
// host runs detection and alerting but changes no host state.
Mode string `json:"mode,omitempty"`
StartedAt time.Time `json:"started_at"`
UptimeSec int64 `json:"uptime_seconds"`
LatestScan time.Time `json:"latest_scan,omitzero"`
BaselineAt time.Time `json:"baseline_at,omitzero"`
BlocklistSize int `json:"blocklist_size"`
IncidentsOpen int `json:"incidents_open"`
BPFEnforcementActive bool `json:"bpf_enforcement_active"`
HistoryCount int `json:"history_count"`
Severities map[string]int `json:"severities"` // "critical","high","warning"
Watchers map[string]bool `json:"watchers"` // name -> attached
StoreHealthy bool `json:"store_healthy"`
StoreSizeMB float64 `json:"store_size_mb"`
ConfigHash string `json:"config_hash,omitempty"`
BinaryHash string `json:"binary_hash,omitempty"`
Capabilities []string `json:"capabilities,omitempty"`
// Backends names the backend each live monitor selected at startup:
// bpf, legacy or none.
Backends map[string]string `json:"backends"`
// DryRunBlocks is the count of firewall blocks that were intercepted by
// auto_response.dry_run and logged rather than applied to nftables.
// Cleared whenever auto-response is live; dry-run mode keeps a recent
// rolling window for operator review.
DryRunBlocks int `json:"dry_run_blocks,omitempty"`
// Automation is the operator-facing safety surface for automatic action
// rollout. It groups dry-run state, challenge routing, pending firewall
// rollback, and the last recorded automation action in one stable payload.
Automation AutomationStatus `json:"automation,omitempty"`
// CorrelationAttribution reports which checks feed cross-account
// correlation findings without a hosting owner. Nil until the daemon has
// merged an active set, and on daemons that predate the block.
CorrelationAttribution *CorrelationAttribution `json:"correlation_attribution,omitempty"`
// Update reports whether a newer CSM release is available upstream.
// Populated by internal/updatecheck. Zero value means the checker has
// not yet completed a poll (very early startup) or is disabled in
// config.
Update UpdateInfo `json:"update,omitempty"`
// Admission is the admission ledger and ingress view; nil when the
// daemon owns no ledger.
Admission *AdmissionStatus `json:"admission,omitempty"`
// AntispamLearning is the learner's health; nil while learning does not
// run.
AntispamLearning *LearningStatus `json:"antispam_learning,omitempty"`
}
// AutomationStatus summarizes the live automation safety state. It is
// intentionally compact so status clients can decide whether the host is
// observe-only, actively mutating the firewall, or waiting for operator
// confirmation after a tentative firewall apply.
type AutomationStatus struct {
AutoResponseEnabled bool `json:"auto_response_enabled"`
AutoResponseBlockIPs bool `json:"auto_response_block_ips"`
AutoResponseDryRun bool `json:"auto_response_dry_run"`
// Termination needs a kernel process handle. A kernel that cannot pin one
// leaves configured automatic killing inoperative, so the capability and
// its cause travel with the status instead of staying in the log.
ProcessKillEnabled bool `json:"process_kill_enabled"`
ProcessSignalSupported bool `json:"process_signal_supported"`
ProcessSignalError string `json:"process_signal_error,omitempty"`
DryRunBlocks int `json:"dry_run_blocks"`
ChallengeEnabled bool `json:"challenge_enabled"`
ChallengePortGateEnabled bool `json:"challenge_port_gate_enabled"`
ChallengePortGateActive bool `json:"challenge_port_gate_active"`
ChallengePending int `json:"challenge_pending"`
ChallengeEscalated int `json:"challenge_escalated"`
ChallengePassed int `json:"challenge_passed"`
// FirewallEnabled reflects firewall.enabled in config. FirewallManaged is
// true only when the daemon has a live nftables engine wired. The
// combination FirewallEnabled && !FirewallManaged means the firewall is
// configured on but the daemon is NOT managing it (e.g. the engine failed
// to apply at startup) -- a condition monitoring should alert on.
// FirewallStateSetAside names where startup kept a state file it could
// not read; the firewall then runs without the entries that file held.
// FirewallLedgerError says why startup could not read the ledger: rows
// its attempts wrote were restored without proof of their outcome.
FirewallEnabled bool `json:"firewall_enabled"`
FirewallManaged bool `json:"firewall_managed"`
FirewallStartupError string `json:"firewall_startup_error,omitempty"`
FirewallStateSetAside string `json:"firewall_state_set_aside,omitempty"`
FirewallLedgerError string `json:"firewall_ledger_error,omitempty"`
FirewallBlockedIPs int `json:"firewall_blocked_ips"`
FirewallBlockedSubnets int `json:"firewall_blocked_subnets"`
// FirewallLocalPortLocks are the ports the live engine lets only listed
// users open new local connections to; empty without a live engine.
FirewallLocalPortLocks []int `json:"firewall_local_port_locks,omitempty"`
FirewallRollbackPending bool `json:"firewall_rollback_pending"`
FirewallRollbackSecondsRemain int64 `json:"firewall_rollback_remaining_seconds,omitempty"`
// FirewallConfirmation is a pending `csm firewall apply-confirmed`
// window; nil when none is.
FirewallConfirmation *FirewallConfirmation `json:"firewall_confirmation,omitempty"`
LastAction *AutomationAction `json:"last_action,omitempty"`
}
// FirewallConfirmation is a pending CLI confirmation window: when its
// rollback is due, and why it cannot settle without `csm firewall confirm`
// when it cannot.
type FirewallConfirmation struct {
Deadline time.Time `json:"deadline,omitzero"`
Error string `json:"error,omitempty"`
}
// AutomationAction is the newest action-like finding CSM recorded.
type AutomationAction struct {
Check string `json:"check"`
Message string `json:"message"`
Timestamp time.Time `json:"timestamp"`
}
// CorrelationAttribution is the operator-facing view of cross-account
// correlation attribution. Current is the per-check count of qualifying
// findings in the latest-state active set that carry no hosting owner and
// are inside the correlation window at its most recent merge. Unstamped
// legacy rows also count. A later merge clears attributed or expired rows.
// Cumulative sums every unattributed row reported since the daemon started,
// across active-set merges and per-batch derivations, so a producer that
// recovered stays visible as having failed. Kept as its own type so
// internal/health does not import internal/checks.
type CorrelationAttribution struct {
Current map[string]int `json:"current"`
Cumulative map[string]int `json:"cumulative"`
ActiveSetUpdates int `json:"active_set_updates"`
Since time.Time `json:"since"`
}
// UpdateInfo mirrors updatecheck.Info for the health snapshot. Kept
// as a separate type so internal/health does not import
// internal/updatecheck and create a cycle.
type UpdateInfo struct {
LatestVersion string `json:"latest_version,omitempty"`
Available bool `json:"available,omitempty"`
Source string `json:"source,omitempty"`
CheckedAt time.Time `json:"checked_at,omitempty"`
Err string `json:"err,omitempty"`
}
// TotalFindings returns the sum across all severity buckets.
func (s Snapshot) TotalFindings() int {
total := 0
for _, v := range s.Severities {
total += v
}
return total
}
// AllWatchersAttached reports whether every registered watcher is attached.
// An empty Watchers map returns false (we never claim ready before probing).
func (s Snapshot) AllWatchersAttached() bool {
if len(s.Watchers) == 0 {
return false
}
for _, attached := range s.Watchers {
if !attached {
return false
}
}
return true
}
// OverallStatus collapses the snapshot into one of: "ok", "degraded", "down".
// - "down" if the snapshot was zero-valued (never assembled)
// - "degraded" if a watcher is detached, the store is unhealthy, an enabled
// firewall is unmanaged, enabled termination has no safe kernel path,
// or a protection queue is degraded. Advisory queues carry best-effort
// work and are reported without changing the host status.
// - "ok" otherwise
func (s Snapshot) OverallStatus() string {
if s.StartedAt.IsZero() && len(s.Watchers) == 0 {
return "down"
}
if !s.StoreHealthy || !s.AllWatchersAttached() || s.Automation.FirewallEnabled && !s.Automation.FirewallManaged ||
s.Automation.ProcessKillEnabled && !s.Automation.ProcessSignalSupported {
return "degraded"
}
for _, q := range s.Queues {
if q.Status == "degraded" && !q.Advisory {
return "degraded"
}
}
return "ok"
}
package incident
import (
"fmt"
"sort"
"strings"
"time"
)
// BulkStatusFilter selects stale active incidents for a bounded operator
// transition. The caller must supply at least one age guard and a positive
// limit so a broad filter cannot accidentally close every incident.
type BulkStatusFilter struct {
FromStatuses []Status
To Status
OlderThan time.Duration
LastSeenBefore time.Time
Kind Kind
Domain string
Account string
Mailbox string
Limit int
DryRun bool
Details string
Now time.Time
}
// BulkStatusItem is a small preview row for bulk incident status changes.
type BulkStatusItem struct {
ID string `json:"id"`
Kind string `json:"kind"`
Status string `json:"status"`
NewStatus string `json:"new_status"`
Severity string `json:"severity"`
Domain string `json:"domain,omitempty"`
Account string `json:"account,omitempty"`
Mailbox string `json:"mailbox,omitempty"`
CreatedAt time.Time `json:"created_at"`
LastSeenAt time.Time `json:"last_seen_at"`
}
// BulkStatusResult reports how many incidents matched the filter and how
// many were changed. Items is capped by BulkStatusFilter.Limit.
type BulkStatusResult struct {
Matched int
Updated int
Items []BulkStatusItem
}
// BulkSetStatus previews or applies one closing transition to matching
// incidents. Matching and mutation happen under the correlator lock so a
// fresh finding cannot update LastSeen between filter evaluation and close.
func (c *Correlator) BulkSetStatus(filter BulkStatusFilter) (BulkStatusResult, error) {
if filter.To != StatusResolved && filter.To != StatusDismissed {
return BulkStatusResult{}, fmt.Errorf("incident: bulk target status must be resolved or dismissed")
}
if filter.OlderThan <= 0 && filter.LastSeenBefore.IsZero() {
return BulkStatusResult{}, fmt.Errorf("incident: bulk status requires older-than or last-seen-before")
}
if filter.Limit <= 0 {
return BulkStatusResult{}, fmt.Errorf("incident: bulk status requires a positive limit")
}
now := filter.Now
if now.IsZero() {
now = c.now()
}
statusSet := make(map[Status]struct{}, len(filter.FromStatuses))
for _, status := range filter.FromStatuses {
if !validStatus(status) {
return BulkStatusResult{}, fmt.Errorf("incident: invalid status %q", status)
}
statusSet[status] = struct{}{}
}
if len(statusSet) == 0 {
return BulkStatusResult{}, fmt.Errorf("incident: bulk status requires a source status")
}
var persist []*queuedPersist
result := BulkStatusResult{Items: make([]BulkStatusItem, 0, filter.Limit)}
c.mu.Lock()
matched := make([]*Incident, 0, len(c.incidents))
for _, inc := range c.incidents {
if bulkStatusMatches(inc, filter, statusSet, now) {
matched = append(matched, inc)
}
}
sort.Slice(matched, func(i, j int) bool {
if !matched[i].UpdatedAt.Equal(matched[j].UpdatedAt) {
return matched[i].UpdatedAt.Before(matched[j].UpdatedAt)
}
return matched[i].ID < matched[j].ID
})
result.Matched = len(matched)
for _, inc := range matched {
if len(result.Items) >= filter.Limit {
break
}
from := inc.Status
result.Items = append(result.Items, bulkStatusItem(inc, filter.To))
if filter.DryRun {
continue
}
inc.Status = filter.To
inc.UpdatedAt = now
inc.ClosedAt = now
inc.ClosedBy = "operator"
inc.Actions = append(inc.Actions, IncidentAction{
Time: now,
Action: "incident_status_changed",
Result: "ok",
Details: string(from) + " -> " + string(filter.To) + ": " + filter.Details,
})
c.counters.statusChangedTotal.Add(1)
c.unbindLocked(inc.ID)
if c.spray != nil {
c.spray.UnbindIncident(inc.ID)
}
if req, ok := c.queuePersistLocked(*inc); ok {
persist = append(persist, req)
}
result.Updated++
}
c.mu.Unlock()
c.runQueuedPersists(persist)
return result, nil
}
func bulkStatusMatches(inc *Incident, filter BulkStatusFilter, statusSet map[Status]struct{}, now time.Time) bool {
if _, ok := statusSet[inc.Status]; !ok {
return false
}
if filter.OlderThan > 0 {
cutoff := now.Add(-filter.OlderThan)
if inc.UpdatedAt.After(cutoff) {
return false
}
}
if !filter.LastSeenBefore.IsZero() && inc.UpdatedAt.After(filter.LastSeenBefore) {
return false
}
if filter.Kind != "" && inc.Kind != filter.Kind {
return false
}
if filter.Domain != "" && !strings.EqualFold(inc.Domain, filter.Domain) {
return false
}
if filter.Account != "" && !strings.EqualFold(inc.Account, filter.Account) {
return false
}
if filter.Mailbox != "" && !strings.EqualFold(inc.Mailbox, filter.Mailbox) {
return false
}
return true
}
func bulkStatusItem(inc *Incident, to Status) BulkStatusItem {
return BulkStatusItem{
ID: inc.ID,
Kind: string(inc.Kind),
Status: string(inc.Status),
NewStatus: string(to),
Severity: inc.Severity.String(),
Domain: inc.Domain,
Account: inc.Account,
Mailbox: inc.Mailbox,
CreatedAt: inc.CreatedAt,
LastSeenAt: inc.UpdatedAt,
}
}
package incident
import (
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"net"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
)
// ErrIncidentNotFound is returned when SetStatus or other lookups
// target an unknown incident id.
var ErrIncidentNotFound = errors.New("incident: not found")
// maxIncidentFindings and maxIncidentTimeline cap the per-incident
// fingerprint slice and operator-visible timeline so a long-running
// open incident with sustained low-severity traffic does not grow
// memory and persistence payloads without bound. Eviction keeps the
// first half (incident-opening context an operator needs to root-
// cause the incident) and the most recent half (so the timeline
// reflects current activity). Operators reading the timeline see a
// gap marker via the appended IncidentEvent that the cap fires.
const (
maxIncidentFindings = 5000
maxIncidentTimeline = 500
incidentFingerprintTruncatedMark = "...truncated:"
incidentTimelineTruncatedKind = "truncated"
)
// incidentMergeWindow is the time gap inside which two findings with the
// same correlation key are considered the same incident. Named constant
// per project convention; config exposure deferred until operators ask.
const incidentMergeWindow = 15 * time.Minute
// incidentPersistDebounce bounds how often a bookkeeping-only merge (a
// finding that appends fingerprint/timeline but does not change status,
// severity, or kind) rewrites the whole incident blob to the store. A busy
// incident under sustained attack would otherwise fsync a ~100-300KB blob on
// every finding; instead those writes coalesce to at most one per window.
// State-changing transitions always persist synchronously. Quiet bookkeeping
// waits for the next mutation or explicit flush; there is no periodic writer.
const incidentPersistDebounce = 5 * time.Second
// CorrelatorConfig is reserved for future tunables and the persistence
// hook used by the daemon to write incidents to bbolt.
type CorrelatorConfig struct {
// Persist receives ordered immutable snapshots; bookkeeping may coalesce.
// Implementations must be quick and idempotent. Errors are logged and
// counted without rolling back in-memory transitions. nil is memory-only.
Persist func(Incident) error
// OpenThreshold is the number of correlated findings required before
// a thresholded finding opens an incident. Critical-severity findings and
// check-specific first-hit signals always open immediately. Values <= 0
// default to 1 (open on first finding) for backwards compatibility with
// callers that expect the original behavior; the daemon explicitly
// configures 2 to suppress one-shot scanner noise.
OpenThreshold int
// SpraySuppression turns on the credential-spray super-incident
// path. When zero the detector is not constructed and OnFinding
// follows the legacy per-mailbox correlation path. Default-off.
SpraySuppression SpraySuppressionConfig
// IsWhitelisted is consulted before a source-IP finding can anchor
// incident correlation and by the spray detector to skip IPs the
// operator has marked as known-good (e.g. internal mail relays).
// nil short-circuits to "no IPs whitelisted".
IsWhitelisted func(ip string) bool
// CanSprayBlock is consulted immediately before recording a
// credential_spray block request. nil means "allowed" when
// OnSprayBlock is present. Implementations must be quick and must
// not call back into the correlator.
CanSprayBlock func() bool
// OnSprayBlock is invoked once per IP when the credential_spray
// detector decides the IP should be hard-blocked, based on
// SpraySuppression.BlockAtSeverity. The callback runs after the
// correlator mutex is released so firewall or verdict latency does
// not stall incident ingestion. nil disables the hand-off; the spray
// super-incident still opens and escalates, but no firewall action
// fires. The return value reports whether the firewall actually
// recorded the block: false means dry-run, transient failure, or an
// upstream gate refused, and the audit "credential_spray_block_requested"
// action is not appended in that case so operators cannot mistake a
// declined request for an enforced block. findingID identifies the latest
// eligible source observation, or is empty for an older unlinked timeline.
// root retains that observation's evidence or mint refusal; the zero
// value means the incident kept none (restored incidents keep none).
OnSprayBlock func(ip, reason string, ttl time.Duration, findingID string, root PreparedRoot) bool
// AutoBlock turns on the generic incident-driven firewall hand-off
// for non-spray kinds. Independent of SpraySuppression; applies when
// an incident has exactly one unambiguous remote IP and its kind is
// allowed by AutoBlock.Kinds (empty = any). Default-zero means the
// path is dormant.
AutoBlock IncidentAutoBlockConfig
// CanIncidentBlock is consulted immediately before recording a
// generic incident block request. nil means "allowed" when
// OnIncidentBlock is present. Lets the daemon recheck
// auto_response.enabled / block_ips at decision time so SIGHUP
// edits take effect without rebuilding the correlator.
CanIncidentBlock func() bool
// AddressEvidence reports whether a finding of check at sev names an
// attacker by its address; the daemon passes checks.AddressEvidence.
// Automatic incident and spray blocks only request an address such a
// finding named. nil refuses every block.
AddressEvidence func(check string, sev alert.Severity) bool
// OnIncidentBlock fires when the generic auto-block gate trips. The
// callback runs after the correlator mutex is released and returns
// true only when a live block request was accepted. Dry-run,
// disabled, and failed attempts must return false so the correlator
// can retry on the next finding instead of permanently latching the
// incident. nil disables the path even when AutoBlock is configured.
// findingID and root carry the same source attribution as OnSprayBlock.
OnIncidentBlock func(ip, reason string, ttl time.Duration, findingID string, root PreparedRoot) bool
// Root prepares the finding's admission evidence and keeps a failed
// mint with its finding until a later block can count that refusal.
// The block answers this observation instead of minting its own root.
// It is kept in memory only; nil keeps no roots.
Root func(f alert.Finding) PreparedRoot
}
// IncidentAutoBlockConfig drives the generic incident-driven firewall
// hand-off independent of credential-spray suppression. Operators turn
// it on once they have validated that incident severity is trustworthy
// (the daemon does not promote a finding to High/Critical without
// either an explicit per-check signal or the correlator's threshold
// gate).
type IncidentAutoBlockConfig struct {
Enabled bool
// BlockAtSeverity is the minimum incident severity that triggers
// a firewall hand-off. "" / "high" / "critical". Comparison is
// case-insensitive. Any other value is ignored so operator typos
// cannot accidentally engage blocking.
BlockAtSeverity string
// BlockExpiry is the operator's configured auto-response block duration,
// the first rung of the escalation ladder. Zero falls back to 24h.
BlockExpiry time.Duration
// Kinds, when non-empty, restricts the auto-block path to the
// listed incident kinds. Empty means "every kind that carries one
// unambiguous remote IP". Credential_spray is implicitly excluded
// since the dedicated spray hand-off owns it.
Kinds map[Kind]bool
}
// IsZero reports whether the config is unset; the correlator treats a
// zero value as "generic auto-block disabled" without touching
// defaults.
func (c IncidentAutoBlockConfig) IsZero() bool {
return !c.Enabled && c.BlockAtSeverity == "" && len(c.Kinds) == 0
}
// counters holds the atomic tallies exposed via RegisterMetrics. Kept
// on the Correlator so a single instance owns its own metric state and
// tests can build isolated correlators without touching globals.
type counters struct {
createdTotal atomic.Uint64
severityChangedTotal atomic.Uint64
statusChangedTotal atomic.Uint64
findingsMergedTotal atomic.Uint64
compactedTotal atomic.Uint64
autoClosedTotal atomic.Uint64
autoCloseDryRunTotal atomic.Uint64
sprayOpenedTotal atomic.Uint64
spraySuppressedTotal atomic.Uint64
sprayDryRunTotal atomic.Uint64
}
// Correlator groups findings into incidents. In-memory state; the
// daemon is responsible for wiring it to a store via CorrelatorConfig.Persist.
type Correlator struct {
mu sync.Mutex
persistence *persistQueue
cfg CorrelatorConfig
incidents map[string]*Incident
byKey map[string]string
pending map[string]pendingFinding
// A false pending value keeps the in-flight slot occupied but discards
// its result after an incident closes, even if it is reopened meanwhile.
pendingSprayBlocks map[string]bool
pendingIncidentBlocks map[string]bool
openThreshold int
now func() time.Time
counters counters
spray *sprayDetector
// lastPersistAt records the wall-clock time of the most recent scheduled
// store write per incident id, used to debounce bookkeeping-only merges.
// Written under c.mu; cleared when the incident leaves c.incidents.
lastPersistAt map[string]time.Time
}
// pendingFinding is a finding seen for a key that has not yet met the
// open threshold. Stored only on the create path; merge into open
// incidents stays unconditional.
type pendingFinding struct {
finding alert.Finding
at time.Time
}
// NewCorrelator returns a ready Correlator. Nothing to start; this type
// is purely callback-driven.
func NewCorrelator(cfg CorrelatorConfig) *Correlator {
threshold := cfg.OpenThreshold
if threshold < 1 {
threshold = 1
}
c := &Correlator{
cfg: cfg,
persistence: newPersistQueue(),
incidents: map[string]*Incident{},
byKey: map[string]string{},
pending: map[string]pendingFinding{},
pendingSprayBlocks: map[string]bool{},
pendingIncidentBlocks: map[string]bool{},
openThreshold: threshold,
now: time.Now,
lastPersistAt: map[string]time.Time{},
}
c.spray = newSprayDetector(cfg.SpraySuppression, incidentMergeWindow, func() time.Time { return c.now() }, cfg.IsWhitelisted)
return c
}
// OnFinding ingests a Finding. Returns the incident id (if attributable)
// and whether a new incident was created. Unattributable findings yield
// ("", false, nil). Findings subject to the threshold whose key has fewer
// than OpenThreshold prior findings inside the merge window are stashed in the
// pending map and yield ("", false, nil) too; they will only open an incident
// if the threshold is met inside the window.
func (c *Correlator) OnFinding(f alert.Finding) (string, bool, error) {
key := KeyFor(f)
if key.IsEmpty() {
return "", false, nil
}
// A spray names its subnet in CIDRs; it is whitelisted and recorded as
// the remote address exactly as when SourceIP held it.
src := alert.AttackerAddress(f)
if key.Host == "" && src != "" && c.cfg.IsWhitelisted != nil && c.cfg.IsWhitelisted(src) {
return "", false, nil
}
var afterUnlock func()
c.mu.Lock()
defer func() {
c.mu.Unlock()
if afterUnlock != nil {
afterUnlock()
}
}()
keyStr := keyString(key)
now := c.now()
// Credential-spray super-incident path. When one source IP brute-forces
// many distinct mailboxes/accounts inside the merge window, collapse
// the per-mailbox fan-out into a single credential_spray incident
// keyed on RemoteIP. The detector returns sprayDecisionNone for
// non-spray traffic so the legacy correlation continues unchanged.
if c.spray != nil {
decision, hits := c.spray.Decide(f)
switch decision {
case sprayDecisionOpen:
sprayKey := Key{RemoteIP: src}
id, created := c.promoteOrCreateSprayLocked(sprayKey, f, now, hits)
c.spray.BindIncident(src, id)
c.counters.sprayOpenedTotal.Add(1)
if cb := c.maybeBlockSprayLocked(c.incidents[id], src, hits, now, "spray opened"); cb != nil {
afterUnlock = cb
}
return id, created, nil
case sprayDecisionSuppress:
id := c.spray.IncidentForIP(src)
inc, ok := c.incidents[id]
if ok && incidentStatusActive(inc.Status) {
// Fold the finding and apply the spray-specific escalation
// before a single persist so the escalating finding does not
// write the blob twice (mutate + escalate in one write).
transition := c.mutateWithFindingLocked(inc, f, now)
c.counters.findingsMergedTotal.Add(1)
if hits >= c.spray.cfg.SeverityEscalateAt && inc.Severity < alert.Critical {
from := inc.Severity
inc.Severity = alert.Critical
c.counters.severityChangedTotal.Add(1)
inc.Actions = append(inc.Actions, IncidentAction{
Time: now,
Action: "incident_severity_changed",
Result: "ok",
Details: from.String() + " -> CRITICAL: spray sustained " + strconv.Itoa(hits) + " mailboxes",
})
transition = true
}
c.persistAfterMergeLocked(inc, now, true, transition)
// Re-evaluate the block gate on every merged finding. The
// configured BlockAtSeverity may have been armed AFTER the
// incident was opened or escalated, in which case the
// transition-time hook already fired without effect; the
// helper is idempotent via triggerSprayBlockLocked's
// action-presence and in-flight guards so a no-op call is
// harmless.
if cb := c.maybeBlockSprayLocked(inc, src, hits, now, "spray ongoing"); cb != nil {
afterUnlock = cb
}
c.counters.spraySuppressedTotal.Add(1)
return id, false, nil
}
// Bound incident vanished (purged) or is no longer active
// (operator resolved/dismissed). Clear the perIP binding so
// subsequent findings don't keep falling into the same dead
// lookup, then fall through to legacy so the finding still
// produces an incident rather than silently disappearing.
if id != "" {
c.spray.UnbindIncident(id)
}
case sprayDecisionNone:
if c.spray.cfg.DryRun && hits >= c.spray.cfg.DistinctMailboxes {
c.counters.sprayDryRunTotal.Add(1)
}
}
}
if id, ok := c.byKey[keyStr]; ok {
// A quiet interval must not erase an active incident's block ladder.
// Only closing the incident ends that episode, including after restore.
if inc, exists := c.incidents[id]; exists && (now.Sub(inc.UpdatedAt) <= incidentMergeWindow || inc.AutoBlock.Count > 0) {
c.mergeLocked(inc, f, now, true)
delete(c.pending, keyStr)
if cb := c.maybeBlockIncidentLocked(inc, now, "merge"); cb != nil {
afterUnlock = cb
}
return id, false, nil
}
// Stale binding -- the incident is older than the merge window.
// Drop the binding and fall through to create so a fresh incident
// owns the key going forward.
delete(c.byKey, keyStr)
}
// Retain pending evidence even when the next finding opens immediately.
// Otherwise the incident can lose the only attestation of its address.
if c.openThreshold > 1 {
if pf, ok := c.pending[keyStr]; ok && now.Sub(pf.at) <= incidentMergeWindow {
delete(c.pending, keyStr)
id := c.createIncidentLocked(key, keyStr, pf.finding, pf.at)
inc := c.incidents[id]
// The second finding is what satisfied the open threshold, so it
// is part of incident creation and must be durable even if it
// lands inside the bookkeeping debounce window.
c.mergeAndPersistLocked(inc, f, now, true)
if cb := c.maybeBlockIncidentLocked(inc, now, "threshold promote"); cb != nil {
afterUnlock = cb
}
return id, true, nil
}
if !opensIncidentImmediately(f) {
c.pending[keyStr] = pendingFinding{finding: f, at: now}
return "", false, nil
}
}
id := c.createIncidentLocked(key, keyStr, f, now)
delete(c.pending, keyStr)
if cb := c.maybeBlockIncidentLocked(c.incidents[id], now, "incident opened"); cb != nil {
afterUnlock = cb
}
return id, true, nil
}
// promoteOrCreateSprayLocked opens the credential_spray incident for key.
// The first failures from an IP usually opened an ordinary per-IP incident
// under the same RemoteIP key before the spray threshold tripped; creating
// a second incident under that key stole the key index and left the earlier
// incident Open with nothing able to merge into or close it. An active
// incident already holding the key is promoted in place, keeping its
// findings and timeline. Caller must hold c.mu.
func (c *Correlator) promoteOrCreateSprayLocked(key Key, f alert.Finding, now time.Time, hits int) (string, bool) {
if existingID, ok := c.byKey[keyString(key)]; ok {
if inc, live := c.incidents[existingID]; live && incidentStatusActive(inc.Status) {
fromKind := inc.Kind
inc.Kind = KindCredentialSpray
if inc.Severity < alert.High {
fromSeverity := inc.Severity
inc.Severity = alert.High
c.counters.severityChangedTotal.Add(1)
inc.Actions = append(inc.Actions, IncidentAction{
Time: now,
Action: "incident_severity_changed",
Result: "ok",
Details: fromSeverity.String() + " -> HIGH: promoted to credential spray",
})
}
inc.Actions = append(inc.Actions, IncidentAction{
Time: now,
Action: "credential_spray_opened",
Result: "ok",
Details: key.RemoteIP + " hit " + strconv.Itoa(hits) + " distinct mailboxes inside window; promoted from " + string(fromKind),
})
// A kind change must be durable, not left to the bookkeeping
// debounce.
c.mergeAndPersistLocked(inc, f, now, true)
return existingID, false
}
}
return c.createSprayIncidentLocked(key, f, now, hits), true
}
// createSprayIncidentLocked builds a credential_spray incident keyed on
// the source IP. Caller must hold c.mu. Severity is HIGH at trip and
// escalates to CRITICAL once the merge path observes
// SpraySuppressionConfig.SeverityEscalateAt distinct mailboxes.
func (c *Correlator) createSprayIncidentLocked(key Key, f alert.Finding, now time.Time, hits int) string {
id := newIncidentID()
sev := f.Severity
if sev < alert.High {
sev = alert.High
}
inc := &Incident{
ID: id,
Kind: KindCredentialSpray,
Status: StatusOpen,
Severity: sev,
CorrelationKey: cloneKey(key),
Findings: []string{},
Timeline: []IncidentEvent{},
Actions: []IncidentAction{{
Time: now,
Action: "credential_spray_opened",
Result: "ok",
Details: key.RemoteIP + " hit " + strconv.Itoa(hits) + " distinct mailboxes inside window",
}},
CreatedAt: now,
UpdatedAt: now,
}
c.incidents[id] = inc
keyStr := keyString(key)
c.byKey[keyStr] = id
c.counters.createdTotal.Add(1)
c.mergeLocked(inc, f, now, false)
return id
}
// createIncidentLocked builds a new Incident, registers it in the maps,
// and seeds it with the given finding via mergeLocked. Caller must hold
// c.mu. mergeLocked is the single source of truth for Persist
// invocations, avoiding double-fire on create.
func (c *Correlator) createIncidentLocked(key Key, keyStr string, f alert.Finding, now time.Time) string {
id := newIncidentID()
displayMailbox, displayDomain := displayMailboxDomain(f.Mailbox, f.Domain)
if displayMailbox == "" && displayDomain == "" {
displayMailbox, displayDomain = key.Mailbox, key.Domain
}
inc := &Incident{
ID: id,
Kind: ClassifyKind(f),
Status: StatusOpen,
Severity: f.Severity,
Account: key.Account,
Domain: displayDomain,
Mailbox: displayMailbox,
CorrelationKey: cloneKey(key),
Findings: []string{},
Timeline: []IncidentEvent{},
Actions: []IncidentAction{},
CreatedAt: now,
UpdatedAt: now,
}
c.incidents[id] = inc
c.byKey[keyStr] = id
c.counters.createdTotal.Add(1)
c.mergeLocked(inc, f, now, false)
return id
}
// PendingCount returns the number of findings currently held in the
// threshold-gate pending map. Exposed for metrics and tests.
func (c *Correlator) PendingCount() int {
c.mu.Lock()
defer c.mu.Unlock()
return len(c.pending)
}
// PruneStalePending removes pending findings whose age relative to now
// exceeds the merge window. Returns the number pruned. Called by the
// daemon's retention loop so a host with sustained one-shot scanner
// traffic does not grow the pending map without bound.
func (c *Correlator) PruneStalePending(now time.Time) int {
c.mu.Lock()
defer c.mu.Unlock()
cutoff := now.Add(-incidentMergeWindow)
pruned := 0
for k, pf := range c.pending {
if pf.at.Before(cutoff) {
delete(c.pending, k)
pruned++
}
}
return pruned
}
// PruneStaleSpray clears spray-detector state whose lastSeen is older
// than the merge window. Wired into the same retention sweep as
// PruneStalePending.
func (c *Correlator) PruneStaleSpray(now time.Time) int {
c.mu.Lock()
defer c.mu.Unlock()
return c.spray.PruneStale(now)
}
// SprayTrackedIPs reports the count of source IPs currently held in
// the spray detector. Safe to call when the detector is disabled.
func (c *Correlator) SprayTrackedIPs() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.spray.TrackedIPs()
}
// OpenCount returns the number of incidents in Open or Contained
// status. Used by the csm_incidents_open gauge; computed at scrape
// time so the value never drifts from in-memory state.
func (c *Correlator) OpenCount() int {
c.mu.Lock()
defer c.mu.Unlock()
n := 0
for _, inc := range c.incidents {
if inc.Status == StatusOpen || inc.Status == StatusContained {
n++
}
}
return n
}
// OpenCountsBySeverity returns Open and Contained incidents keyed by
// lowercase severity. Contained incidents still need operator attention,
// so they remain part of the dashboard's active-incident posture.
func (c *Correlator) OpenCountsBySeverity() map[string]int {
c.mu.Lock()
defer c.mu.Unlock()
counts := map[string]int{}
for _, inc := range c.incidents {
if inc.Status != StatusOpen && inc.Status != StatusContained {
continue
}
switch inc.Severity {
case alert.Critical:
counts["critical"]++
case alert.High:
counts["high"]++
default:
counts["warning"]++
}
}
return counts
}
// Get returns a snapshot of the incident by id.
func (c *Correlator) Get(id string) (Incident, bool) {
c.mu.Lock()
defer c.mu.Unlock()
inc, ok := c.incidents[id]
if !ok {
return Incident{}, false
}
return cloneIncident(*inc), true
}
// SnapshotPage returns a page of incidents matching status (empty
// string means all statuses), starting at offset, with at most limit
// items. The returned total is the number of records that match the
// filter regardless of the page bounds, so the caller can render an
// accurate "X of Y" header.
//
// limit <= 0 returns the rest of the filtered set after offset. The
// caller (web UI / phpanel) is expected to cap the page at a sane
// ceiling; this primitive only enforces correct slicing. Negative
// offset is clamped to zero.
//
// Items are deep-copied so callers may mutate the returned slice
// without affecting subsequent calls.
func (c *Correlator) SnapshotPage(status Status, offset, limit int) ([]Incident, int) {
if status == "" {
return c.SnapshotPageStatuses(nil, offset, limit)
}
return c.SnapshotPageStatuses([]Status{status}, offset, limit)
}
// SnapshotPageStatuses returns a page of incidents matching any status
// in statuses. An empty status list means all statuses. Sorting and
// slicing happen against internal pointers first; only the returned
// page is deep-copied.
func (c *Correlator) SnapshotPageStatuses(statuses []Status, offset, limit int) ([]Incident, int) {
c.mu.Lock()
defer c.mu.Unlock()
statusSet := make(map[Status]struct{}, len(statuses))
for _, st := range statuses {
if st != "" {
statusSet[st] = struct{}{}
}
}
matched := make([]*Incident, 0, len(c.incidents))
for _, inc := range c.incidents {
if len(statusSet) > 0 {
if _, ok := statusSet[inc.Status]; !ok {
continue
}
}
matched = append(matched, inc)
}
sortIncidentRefs(matched)
total := len(matched)
if offset < 0 {
offset = 0
}
if offset >= total {
return []Incident{}, total
}
end := total
if limit > 0 && offset+limit < end {
end = offset + limit
}
out := make([]Incident, 0, end-offset)
for _, inc := range matched[offset:end] {
out = append(out, cloneIncident(*inc))
}
return out, total
}
func sortIncidentRefs(refs []*Incident) {
sort.Slice(refs, func(i, j int) bool {
return incidentRefLess(refs[i], refs[j])
})
}
func incidentRefLess(a, b *Incident) bool {
if !a.UpdatedAt.Equal(b.UpdatedAt) {
return a.UpdatedAt.After(b.UpdatedAt)
}
return a.ID > b.ID
}
// Snapshot returns every incident sorted by UpdatedAt descending. Safe
// for concurrent callers; produces a deep-copy slice so the API layer
// can serialize it without coordinating with mutators.
func (c *Correlator) Snapshot() []Incident {
c.mu.Lock()
defer c.mu.Unlock()
refs := make([]*Incident, 0, len(c.incidents))
for _, inc := range c.incidents {
refs = append(refs, inc)
}
sortIncidentRefs(refs)
out := make([]Incident, 0, len(refs))
for _, inc := range refs {
out = append(out, cloneIncident(*inc))
}
return out
}
// mergeLocked folds f into inc. merged=true means this is a join into
// an existing incident (bumps findings_merged_total); merged=false means
// the caller already created the incident and is using mergeLocked only
// to seed the first finding -- in that case the create path owns the
// "did a new incident appear" tally.
func (c *Correlator) mergeLocked(inc *Incident, f alert.Finding, now time.Time, merged bool) {
c.mergeLockedWithPersistence(inc, f, now, merged, false)
}
func (c *Correlator) mergeAndPersistLocked(inc *Incident, f alert.Finding, now time.Time, merged bool) {
c.mergeLockedWithPersistence(inc, f, now, merged, true)
}
func (c *Correlator) mergeLockedWithPersistence(inc *Incident, f alert.Finding, now time.Time, merged, forcePersist bool) {
if merged {
c.counters.findingsMergedTotal.Add(1)
}
transition := c.mutateWithFindingLocked(inc, f, now)
c.persistAfterMergeLocked(inc, now, merged, transition || forcePersist)
}
// mutateWithFindingLocked folds f into inc and reports whether the fold caused
// a state-changing transition (kind, severity or address evidence). It does not
// persist: the caller decides synchronous vs debounced persistence via
// persistAfterMergeLocked. Splitting mutation from persistence lets the
// credential_spray path apply its own escalation before a single persist,
// avoiding a redundant second write.
func (c *Correlator) mutateWithFindingLocked(inc *Incident, f alert.Finding, now time.Time) (transition bool) {
// Re-classify before appending so timeline-aware compound rules
// see the unchanged history; the new finding is passed in
// explicitly so its Check participates in the compound check.
priorKind := inc.Kind
maybeReclassifyKind(inc, f)
if inc.Kind != priorKind {
inc.Actions = append(inc.Actions, IncidentAction{
Time: now,
Action: "incident_kind_changed",
Result: "ok",
Details: string(priorKind) + " -> " + string(inc.Kind),
})
transition = true
}
inc.Findings = appendCappedFingerprint(inc.Findings, f.Fingerprint())
ev := IncidentEvent{
FindingID: alert.FindingID(f),
Time: f.Timestamp,
Kind: "finding",
Check: f.Check,
Severity: f.Severity.String(),
Message: f.Message,
}
if f.Process != nil {
ev.PID = f.Process.PID
ev.UID = f.Process.UID
ev.Process = f.Process.Comm
}
if f.FilePath != "" {
ev.Path = f.FilePath
}
if src := alert.AttackerAddress(f); src != "" {
ev.RemoteIP = src
if c.cfg.Root != nil && c.cfg.AddressEvidence != nil && c.cfg.AddressEvidence(f.Check, f.Severity) {
root := c.cfg.Root(f)
ev.root = &root
}
if (!inc.RemoteIPEvidence || inc.RemoteIPEvidenceFinding == "") && c.cfg.AddressEvidence != nil && inc.CorrelationKey != nil && c.cfg.AddressEvidence(f.Check, f.Severity) {
if key := normalizeIncidentRemoteIP(inc.CorrelationKey.RemoteIP); key != "" && key == normalizeIncidentRemoteIP(src) {
inc.RemoteIPEvidence = true
inc.RemoteIPEvidenceFinding = ev.FindingID
inc.RemoteIPEvidenceCheck = ev.Check
inc.RemoteIPEvidenceSeverity = ev.Severity
transition = true
}
}
}
inc.Timeline = appendCappedTimeline(inc.Timeline, ev)
if f.Severity > inc.Severity {
from := inc.Severity
inc.Severity = f.Severity
c.counters.severityChangedTotal.Add(1)
inc.Actions = append(inc.Actions, IncidentAction{
Time: now,
Action: "incident_severity_changed",
Result: "ok",
Details: from.String() + " -> " + f.Severity.String(),
})
transition = true
}
inc.UpdatedAt = now
return transition
}
// persistAfterMergeLocked persists a just-merged incident. Incident creation
// (merged==false) and synchronous merge writes persist immediately so a restart
// never loses an open/escalate/kind-change/threshold-promotion. A pure
// bookkeeping merge is debounced so a sustained-attack incident does not fsync
// its whole blob per finding.
func (c *Correlator) persistAfterMergeLocked(inc *Incident, now time.Time, merged, syncPersist bool) {
if !merged || syncPersist {
c.markPersistedLocked(inc.ID, now)
c.persistLocked(*inc)
return
}
c.persistDebouncedLocked(inc, now)
}
func (c *Correlator) markPersistedLocked(id string, at time.Time) {
c.lastPersistAt[id] = at
}
// persistDebouncedLocked writes inc only when at least incidentPersistDebounce
// has elapsed since its last merge-path store write; otherwise the update stays
// in memory (durable state is still captured by the next transition or close).
// Caller holds c.mu.
func (c *Correlator) persistDebouncedLocked(inc *Incident, now time.Time) {
if last, ok := c.lastPersistAt[inc.ID]; ok && now.Sub(last) < incidentPersistDebounce {
if c.cfg.Persist != nil {
c.persistence.deferWrite(inc.ID)
}
return
}
c.markPersistedLocked(inc.ID, now)
c.persistLocked(*inc)
}
// FlushPendingPersists writes incidents whose latest bookkeeping-only merge was
// skipped by the debounce window. It is intentionally cheap when no incidents
// are dirty and is used by shutdown hooks before the store closes.
func (c *Correlator) FlushPendingPersists() int {
var persist []*queuedPersist
c.mu.Lock()
now := c.now()
for _, id := range c.persistence.deferredIDs() {
inc, ok := c.incidents[id]
if !ok {
c.persistence.discardDeferred(id)
delete(c.lastPersistAt, id)
continue
}
c.markPersistedLocked(id, now)
if req, ok := c.queuePersistLocked(*inc); ok {
persist = append(persist, req)
}
}
c.mu.Unlock()
c.runQueuedPersists(persist)
return len(persist)
}
// appendCappedFingerprint appends fp to fps and trims to
// maxIncidentFindings via first-half + last-half retention when the
// cap is crossed. Keeps the original opening signals plus the most
// recent traffic, dropping the middle.
func appendCappedFingerprint(fps []string, fp string) []string {
fps = append(fps, fp)
if len(fps) <= maxIncidentFindings {
return fps
}
real := make([]string, 0, len(fps))
elided := 0
for _, existing := range fps {
if n, ok := fingerprintTruncationCount(existing); ok {
elided += n
continue
}
real = append(real, existing)
}
if len(real) <= maxIncidentFindings {
return fingerprintsWithTruncationMarker(real, elided)
}
half := maxIncidentFindings / 2
tailLen := maxIncidentFindings - half
elided += len(real) - half - tailLen
head := append([]string(nil), real[:half]...)
tail := append([]string(nil), real[len(real)-tailLen:]...)
gap := []string{formatFingerprintTruncation(elided)}
return append(append(head, gap...), tail...)
}
// appendCappedTimeline behaves the same as appendCappedFingerprint
// for the operator-visible IncidentEvent slice; the truncation marker
// is rendered as a synthetic "truncated" event so the UI can show a
// "X events elided" row.
func appendCappedTimeline(events []IncidentEvent, ev IncidentEvent) []IncidentEvent {
events = append(events, ev)
if len(events) <= maxIncidentTimeline {
return events
}
real := make([]IncidentEvent, 0, len(events))
elided := 0
var markerTime time.Time
for _, existing := range events {
if n, ok := timelineTruncationCount(existing); ok {
elided += n
if markerTime.IsZero() || existing.Time.Before(markerTime) {
markerTime = existing.Time
}
continue
}
real = append(real, existing)
}
if len(real) <= maxIncidentTimeline {
return timelineWithTruncationMarker(real, elided, markerTime)
}
half := maxIncidentTimeline / 2
tailLen := maxIncidentTimeline - half
if markerTime.IsZero() {
markerTime = real[half].Time
}
elided += len(real) - half - tailLen
head := append([]IncidentEvent(nil), real[:half]...)
tail := append([]IncidentEvent(nil), real[len(real)-tailLen:]...)
gap := []IncidentEvent{timelineTruncationMarker(elided, markerTime)}
return append(append(head, gap...), tail...)
}
func fingerprintsWithTruncationMarker(fps []string, elided int) []string {
if elided == 0 {
return fps
}
half := len(fps) / 2
out := make([]string, 0, len(fps)+1)
out = append(out, fps[:half]...)
out = append(out, formatFingerprintTruncation(elided))
out = append(out, fps[half:]...)
return out
}
func fingerprintTruncationCount(fp string) (int, bool) {
if !strings.HasPrefix(fp, incidentFingerprintTruncatedMark) {
return 0, false
}
rest := strings.TrimPrefix(fp, incidentFingerprintTruncatedMark)
fields := strings.Fields(rest)
if len(fields) == 0 {
return 1, true
}
n, err := strconv.Atoi(fields[0])
if err != nil || n < 1 {
return 1, true
}
return n, true
}
func formatFingerprintTruncation(count int) string {
return incidentFingerprintTruncatedMark + strconv.Itoa(count) + " findings elided"
}
func timelineWithTruncationMarker(events []IncidentEvent, elided int, markerTime time.Time) []IncidentEvent {
if elided == 0 {
return events
}
if markerTime.IsZero() && len(events) > 0 {
markerTime = events[len(events)/2].Time
}
half := len(events) / 2
out := make([]IncidentEvent, 0, len(events)+1)
out = append(out, events[:half]...)
out = append(out, timelineTruncationMarker(elided, markerTime))
out = append(out, events[half:]...)
return out
}
func timelineTruncationCount(ev IncidentEvent) (int, bool) {
if ev.Kind != incidentTimelineTruncatedKind {
return 0, false
}
fields := strings.Fields(ev.Message)
if len(fields) == 0 {
return 1, true
}
n, err := strconv.Atoi(fields[0])
if err != nil || n < 1 {
return 1, true
}
return n, true
}
func timelineTruncationMarker(count int, at time.Time) IncidentEvent {
return IncidentEvent{
Time: at,
Kind: incidentTimelineTruncatedKind,
Message: strconv.Itoa(count) + " events elided to cap incident size",
}
}
// persistLocked invokes the Persist callback while temporarily releasing
// the correlator mutex so a re-entrant Persist that reads Correlator
// state does not deadlock. The caller MUST already hold c.mu; the
// deferred re-Lock keeps the "mu held on return" contract that
// mergeLocked's callers rely on.
func (c *Correlator) persistLocked(snap Incident) {
req, ok := c.queuePersistLocked(snap)
if !ok {
return
}
c.mu.Unlock()
defer c.mu.Lock()
c.runQueuedPersist(req)
}
// RecordOperatorBlock notes on the incident that an operator blocked its
// address from the incident view, and settles the automatic escalation ladder
// to match: the hand-off has no business re-requesting a block for an address
// the operator just blocked. A zero ttl is a permanent block, which the ladder
// records as never lapsing.
func (c *Correlator) RecordOperatorBlock(id, ip string, ttl time.Duration) error {
c.mu.Lock()
defer c.mu.Unlock()
inc, ok := c.incidents[id]
if !ok {
return ErrIncidentNotFound
}
ip = normalizeIncidentRemoteIP(ip)
if ip == "" || ip != incidentBlockCandidate(inc) {
return errors.New("incident: blocked address does not match incident source")
}
now := c.now()
if incidentStatusActive(inc.Status) {
count := inc.AutoBlock.Count
// Refreshing an existing block is not a recurrence after expiry.
if inc.AutoBlock.lapsed(now) {
count++
}
inc.AutoBlock = AutoBlockState{Count: count, LastAt: now}
if ttl > 0 {
inc.AutoBlock.ExpiresAt = now.Add(ttl)
}
} else {
// A closed record now carries an operator decision and must keep
// that decision for the operator retention window.
inc.UpdatedAt = now
inc.ClosedAt = now
inc.ClosedBy = "operator"
}
inc.Actions = append(inc.Actions, IncidentAction{
Time: now,
Action: "operator_block",
Result: "ok",
Details: ip + " blocked by operator " + blockDurationLabel(ttl),
})
c.markPersistedLocked(id, now)
c.persistLocked(*inc)
return nil
}
// resetAutoBlockLocked ends the ladder and invalidates any callback still
// running for this episode without releasing its concurrency guard early.
func (c *Correlator) resetAutoBlockLocked(inc *Incident) {
inc.AutoBlock = AutoBlockState{}
if _, ok := c.pendingSprayBlocks[inc.ID]; ok {
c.pendingSprayBlocks[inc.ID] = false
}
if _, ok := c.pendingIncidentBlocks[inc.ID]; ok {
c.pendingIncidentBlocks[inc.ID] = false
}
}
// SetStatus transitions an incident's status. On Resolved/Dismissed
// the incident is unbound from the active byKey index so future
// findings for the same correlation key start a fresh incident.
// Returns ErrIncidentNotFound if id is unknown.
func (c *Correlator) SetStatus(id string, status Status, details string) error {
if !validStatus(status) {
return fmt.Errorf("incident: invalid status %q", status)
}
c.mu.Lock()
defer c.mu.Unlock()
inc, ok := c.incidents[id]
if !ok {
return ErrIncidentNotFound
}
if inc.Status == status && (incidentStatusActive(status) || inc.ClosedBy == "operator") {
return nil
}
now := c.now()
from := inc.Status
inc.Status = status
inc.UpdatedAt = now
inc.Actions = append(inc.Actions, IncidentAction{
Time: now,
Action: "incident_status_changed",
Result: "ok",
Details: string(from) + " -> " + string(status) + ": " + details,
})
c.counters.statusChangedTotal.Add(1)
if status == StatusResolved || status == StatusDismissed {
// SetStatus is an operator decision, including confirmation or
// dismissal of a record the daemon already closed.
inc.ClosedAt = now
inc.ClosedBy = "operator"
// Closing ends the episode. A later recurrence is new activity and
// starts at the bottom of the escalation ladder, rather than jumping
// to a permanent block off the back of a long-closed incident.
c.resetAutoBlockLocked(inc)
c.unbindLocked(id)
if c.spray != nil {
c.spray.UnbindIncident(id)
}
} else {
// Reverting from resolved/dismissed back to open or contained
// clears the close attribution so future closes attribute correctly.
inc.ClosedAt = time.Time{}
inc.ClosedBy = ""
c.bindLocked(inc)
}
c.markPersistedLocked(id, now)
c.persistLocked(*inc)
return nil
}
// CloseStale auto-resolves Open / Contained incidents whose UpdatedAt is
// older than the per-kind threshold in `idleThresholds`. Kinds absent
// from the map are never closed (the caller decides which kinds expire).
// dryRun=true counts decisions without mutating state, so an operator
// can validate thresholds before flipping the live switch. Returns
// (closed, dryRun, total-scanned).
//
// Closing unbinds both the incident key and any spray detector state,
// so future findings are evaluated as new activity instead of merging
// into the closed incident.
func (c *Correlator) CloseStale(now time.Time, idleThresholds map[Kind]time.Duration, dryRun bool) (closed, dryRunCount, scanned int) {
closed, dryRunCount, scanned, _ = c.CloseStaleLimited(now, idleThresholds, dryRun, 0)
return closed, dryRunCount, scanned
}
// CloseStaleLimited is CloseStale with a per-call cap on the number of
// incidents it resolves. limit <= 0 means unbounded (the CloseStale
// behaviour). When the cap is hit, more=true signals the caller that
// stale incidents remain so it can schedule a prompt follow-up sweep
// instead of waiting the full auto-close interval. Bounding the work
// keeps a large post-restart backlog from holding c.mu and bursting
// thousands of bbolt persists in a single tick. The cap applies only to
// live closes; a dry-run pass always scans the full set so its counters
// stay accurate.
func (c *Correlator) CloseStaleLimited(now time.Time, idleThresholds map[Kind]time.Duration, dryRun bool, limit int) (closed, dryRunCount, scanned int, more bool) {
if len(idleThresholds) == 0 {
return 0, 0, 0, false
}
var persist []*queuedPersist
c.mu.Lock()
for id, inc := range c.incidents {
if inc.Status != StatusOpen && inc.Status != StatusContained {
continue
}
threshold, ok := idleThresholds[inc.Kind]
if !ok || threshold <= 0 {
continue
}
idle := now.Sub(inc.UpdatedAt)
if idle <= threshold {
continue
}
if !dryRun && limit > 0 && closed >= limit {
more = true
break
}
scanned++
if dryRun {
c.counters.autoCloseDryRunTotal.Add(1)
dryRunCount++
continue
}
from := inc.Status
inc.Status = StatusResolved
inc.UpdatedAt = now
inc.ClosedAt = now
inc.ClosedBy = "auto:stale"
c.resetAutoBlockLocked(inc)
inc.Actions = append(inc.Actions, IncidentAction{
Time: now,
Action: "incident_auto_closed",
Result: "ok",
Details: string(from) + " -> resolved: stale " + idle.Truncate(time.Second).String(),
})
c.counters.statusChangedTotal.Add(1)
c.counters.autoClosedTotal.Add(1)
c.unbindLocked(id)
if c.spray != nil {
c.spray.UnbindIncident(id)
}
c.markPersistedLocked(id, now)
if req, ok := c.queuePersistLocked(*inc); ok {
persist = append(persist, req)
}
closed++
}
c.mu.Unlock()
c.runQueuedPersists(persist)
return closed, dryRunCount, scanned, more
}
// closeIncidentLocked force-resolves an active incident with the given
// attribution. Caller holds c.mu and must queue/run the persist of *inc.
func (c *Correlator) closeIncidentLocked(inc *Incident, id string, now time.Time, by, detail string) {
c.resetAutoBlockLocked(inc)
from := inc.Status
inc.Status = StatusResolved
inc.UpdatedAt = now
inc.ClosedAt = now
inc.ClosedBy = by
inc.Actions = append(inc.Actions, IncidentAction{
Time: now,
Action: "incident_auto_closed",
Result: "ok",
Details: string(from) + " -> resolved: " + detail,
})
c.counters.statusChangedTotal.Add(1)
c.counters.autoClosedTotal.Add(1)
c.unbindLocked(id)
if c.spray != nil {
c.spray.UnbindIncident(id)
}
c.markPersistedLocked(id, now)
}
// CloseStaleByAge is a kind-agnostic safety cap. It force-closes any Open or
// Contained incident whose UpdatedAt is older than maxAge, independent of the
// operator's per-kind auto-close thresholds. Without it, disabling auto-close
// (or omitting a kind from the threshold map) lets active incidents accumulate
// without bound in memory and bbolt on a host under sustained attack. limit
// bounds closures per call (0 = unbounded); more reports a remaining backlog.
func (c *Correlator) CloseStaleByAge(now time.Time, maxAge time.Duration, limit int) (closed int, more bool) {
if maxAge <= 0 {
return 0, false
}
var persist []*queuedPersist
c.mu.Lock()
for id, inc := range c.incidents {
if inc.Status != StatusOpen && inc.Status != StatusContained {
continue
}
if now.Sub(inc.UpdatedAt) <= maxAge {
continue
}
if limit > 0 && closed >= limit {
more = true
break
}
c.closeIncidentLocked(inc, id, now, "auto:age_cap", "stale age cap "+maxAge.Truncate(time.Second).String())
if req, ok := c.queuePersistLocked(*inc); ok {
persist = append(persist, req)
}
closed++
}
c.mu.Unlock()
c.runQueuedPersists(persist)
return closed, more
}
// EnforceActiveCap bounds how many Open/Contained incidents are held in
// memory. When the active count exceeds maxActive it force-closes the oldest
// (by UpdatedAt) until the count is back at maxActive or the per-call limit is
// reached. This protects against a flood of distinct incidents arriving within
// the age-cap window. maxActive <= 0 disables the cap; limit <= 0 is
// unbounded. more reports that incidents remained over the cap after limit.
func (c *Correlator) EnforceActiveCap(now time.Time, maxActive, limit int) (closed int, more bool) {
if maxActive <= 0 {
return 0, false
}
c.mu.Lock()
type activeRef struct {
id string
updated time.Time
}
var active []activeRef
for id, inc := range c.incidents {
if inc.Status == StatusOpen || inc.Status == StatusContained {
active = append(active, activeRef{id: id, updated: inc.UpdatedAt})
}
}
if len(active) <= maxActive {
c.mu.Unlock()
return 0, false
}
sort.Slice(active, func(i, j int) bool { return active[i].updated.Before(active[j].updated) })
overflow := len(active) - maxActive
var persist []*queuedPersist
for _, a := range active {
if overflow <= 0 {
break
}
if limit > 0 && closed >= limit {
more = true
break
}
inc := c.incidents[a.id]
if inc == nil || (inc.Status != StatusOpen && inc.Status != StatusContained) {
continue
}
c.closeIncidentLocked(inc, a.id, now, "auto:active_cap", "active incident cap "+strconv.Itoa(maxActive))
if req, ok := c.queuePersistLocked(*inc); ok {
persist = append(persist, req)
}
closed++
overflow--
}
c.mu.Unlock()
c.runQueuedPersists(persist)
return closed, more
}
// validStatus reports whether s is one of the four incident statuses.
// Guards SetStatus against arbitrary strings reaching the persisted
// timeline; control-socket and webui handlers also reject early but
// the correlator owns the type and must not trust callers.
func validStatus(s Status) bool {
switch s {
case StatusOpen, StatusContained, StatusResolved, StatusDismissed:
return true
}
return false
}
func incidentStatusActive(s Status) bool {
return s == StatusOpen || s == StatusContained
}
// IncrementCompactedTotal bumps the compaction counter by n. Called
// from the daemon-side retention scheduler after store.CompactIncidents
// removes records. Negative inputs are ignored so a buggy caller cannot
// underflow the monotonic counter.
func (c *Correlator) IncrementCompactedTotal(n int) {
if n < 0 {
return
}
c.counters.compactedTotal.Add(uint64(n))
}
// PruneClosedOlderThan removes resolved/dismissed incidents older than
// retention from the in-memory map. Store compaction removes the durable
// records; this keeps API/control snapshots from serving stale incidents
// until the next daemon restart.
func (c *Correlator) PruneClosedOlderThan(now time.Time, retention ClosedRetention) int {
c.mu.Lock()
defer c.mu.Unlock()
pruned := 0
var prunedIDs []string
prunedSet := make(map[string]struct{})
for id, inc := range c.incidents {
if !retention.Expired(*inc, now) {
continue
}
delete(c.incidents, id)
delete(c.lastPersistAt, id)
c.persistence.discardDeferred(id)
prunedSet[id] = struct{}{}
if c.spray != nil {
prunedIDs = append(prunedIDs, id)
}
pruned++
}
// Walk the active index once. Scanning it for each expired record
// stalls all incident activity during a large retention backlog.
for key, id := range c.byKey {
if _, ok := prunedSet[id]; ok {
delete(c.byKey, key)
}
}
if c.spray != nil {
c.spray.UnbindIncidents(prunedIDs)
}
return pruned
}
// Restore re-hydrates correlator state from a list previously loaded
// from the store. Open and Contained incidents are bound to the
// byKey index so a finding arriving inside the merge window joins the
// existing incident; Resolved/Dismissed incidents are loaded into the
// id map only (Get still returns them) but do NOT claim their key, so
// future findings for the same key start a fresh incident.
func (c *Correlator) Restore(incidents []Incident) {
c.mu.Lock()
defer c.mu.Unlock()
for i := range incidents {
inc := incidents[i]
// Recheck stored authority against the current policy before a
// merge can trim the source event.
c.restoreAddressEvidence(&inc)
c.incidents[inc.ID] = &inc
delete(c.lastPersistAt, inc.ID)
c.persistence.discardDeferred(inc.ID)
if inc.Status != StatusOpen && inc.Status != StatusContained {
continue
}
c.bindLocked(&inc)
// credential_spray super-incidents persist in bbolt but the spray
// detector's perIP map is in-memory only. Without this rehydration
// step a daemon restart while an attacker is mid-spray causes the
// detector to re-trip and open a duplicate super-incident even
// though the original is still active.
if c.spray != nil && inc.Kind == KindCredentialSpray && inc.CorrelationKey != nil {
ip := inc.CorrelationKey.RemoteIP
if ip != "" {
c.spray.Rehydrate(ip, inc.ID, inc.UpdatedAt)
}
}
}
}
func (c *Correlator) bindLocked(inc *Incident) {
key, ok := incidentKey(*inc)
if !ok {
return
}
c.byKey[keyString(key)] = inc.ID
}
func (c *Correlator) unbindLocked(id string) {
// Scan-and-delete by value rather than rebuilding the key: incidents
// can be keyed by account, domain, mailbox, process, remote IP, or a
// combination. byKey only holds active incidents so the scan is bounded.
for k, v := range c.byKey {
if v == id {
delete(c.byKey, k)
}
}
}
func incidentKey(inc Incident) (Key, bool) {
if inc.CorrelationKey != nil && !inc.CorrelationKey.IsEmpty() {
return canonicalizeKey(*inc.CorrelationKey), true
}
key := Key{Account: inc.Account, Domain: inc.Domain, Mailbox: inc.Mailbox}
key = canonicalizeKey(key)
if key.IsEmpty() {
return Key{}, false
}
return key, true
}
func cloneIncident(in Incident) Incident {
out := in
out.Findings = append([]string(nil), in.Findings...)
out.Timeline = append([]IncidentEvent(nil), in.Timeline...)
out.Actions = append([]IncidentAction(nil), in.Actions...)
if in.CorrelationKey != nil {
key := *in.CorrelationKey
out.CorrelationKey = &key
}
return out
}
func cloneKey(k Key) *Key {
if k.IsEmpty() {
return nil
}
key := k
return &key
}
// keyString serializes a Key into a stable string for the byKey map.
// All fields selected by KeyFor must be encoded so distinct findings
// (e.g. different PID-only processes or different remote IPs) do not
// collapse to the same bucket and falsely merge.
func keyString(k Key) string {
return fmt.Sprintf("%d:%s|%d:%s|%d:%s|%d:%s|%d|%d|%d:%s",
len(k.Host), k.Host,
len(k.Account), k.Account,
len(k.Mailbox), k.Mailbox,
len(k.Domain), k.Domain,
k.UID,
k.PID,
len(k.RemoteIP), k.RemoteIP,
)
}
func opensIncidentImmediately(f alert.Finding) bool {
if f.Severity >= alert.Critical {
return true
}
// A copy-forward is graded Warning when its owner cannot be distinguished
// from an attacker by rule shape alone. It still needs first-hit incident
// review; otherwise lowering alert severity silently removes the existing
// mailbox-takeover correlation path.
if f.Check == "email_filter_exfil" {
return true
}
// Reputation severity describes the observed surface, not confidence in
// the threat-intel match. Preserve the pre-grading first-hit incident path.
if f.Check == "ip_reputation" {
return true
}
return f.Severity >= alert.High && ClassifyKind(f) == KindHostIntegrityRisk
}
func newIncidentID() string {
var buf [6]byte
_, _ = rand.Read(buf[:])
return "inc_" + hex.EncodeToString(buf[:])
}
// maybeBlockSprayLocked is the single decision point for the
// credential_spray firewall hand-off. Unlike a transition-only hook,
// this runs on every spray decision (open, merge, escalate) so an
// operator who arms BlockAtSeverity AFTER an incident has already
// reached the configured severity still gets a block on the next
// matching finding. Idempotency is provided by
// triggerSprayBlockLocked's action-presence and in-flight guards, so
// calling this helper repeatedly against the same incident emits one
// live firewall call at a time.
//
// Returns the callback that the caller must invoke after releasing
// c.mu (matches the existing sprayDecisionOpen contract), or nil when
// no block is owed.
func (c *Correlator) maybeBlockSprayLocked(inc *Incident, ip string, hits int, now time.Time, reason string) func() {
if inc == nil || c.spray == nil || c.cfg.OnSprayBlock == nil {
return nil
}
if !incidentStatusActive(inc.Status) {
return nil
}
if !c.sprayBlockAllowed() {
return nil
}
// Like every incident block, the address needs a finding with address
// evidence; raw failures that only count mailboxes never block.
if target := normalizeIncidentRemoteIP(ip); target == "" || !c.blockAddressAttested(inc, target) {
return nil
}
switch strings.ToLower(c.spray.cfg.BlockAtSeverity) {
case "high":
if inc.Severity < alert.High {
return nil
}
case "critical":
if inc.Severity < alert.Critical {
return nil
}
default:
return nil
}
return c.triggerSprayBlockLocked(inc, ip, hits, now, reason)
}
func (c *Correlator) sprayBlockAllowed() bool {
return c.cfg.CanSprayBlock == nil || c.cfg.CanSprayBlock()
}
// maybeBlockIncidentLocked is the decision point for the generic
// incident-driven firewall hand-off. Runs on every create / merge so
// an operator who arms AutoBlock AFTER an incident has already crossed
// the gate still gets a block on the next finding. Idempotent via the
// action-presence and in-flight guards in triggerIncidentBlockLocked.
//
// Skips credential_spray incidents -- the dedicated spray hand-off owns
// those so we avoid double-firing.
//
// Returns the callback that the caller must invoke after releasing
// c.mu, or nil when no block is owed.
func (c *Correlator) maybeBlockIncidentLocked(inc *Incident, now time.Time, why string) func() {
if inc == nil || c.cfg.OnIncidentBlock == nil || !c.cfg.AutoBlock.Enabled {
return nil
}
if !incidentStatusActive(inc.Status) {
return nil
}
if inc.Kind == KindCredentialSpray {
return nil
}
if c.sprayOwnsIncident(inc) {
return nil
}
if !c.incidentBlockAllowed() {
return nil
}
ip := incidentBlockCandidate(inc)
if ip == "" || !c.blockAddressAttested(inc, ip) {
return nil
}
if len(c.cfg.AutoBlock.Kinds) > 0 && !c.cfg.AutoBlock.Kinds[inc.Kind] {
return nil
}
switch strings.ToLower(c.cfg.AutoBlock.BlockAtSeverity) {
case "high":
if inc.Severity < alert.High {
return nil
}
case "critical":
if inc.Severity < alert.Critical {
return nil
}
default:
return nil
}
return c.triggerIncidentBlockLocked(inc, ip, now, why)
}
func (c *Correlator) incidentBlockAllowed() bool {
return c.cfg.CanIncidentBlock == nil || c.cfg.CanIncidentBlock()
}
// triggerIncidentBlockLocked is the per-incident emit point for the
// generic auto-block path. It marks the incident as in-flight, returns
// the deferred callback, then appends "incident_block_requested" only
// when the callback reports a live block request. Dry-run attempts are
// intentionally not latched so a later finding can retry after the
// operator disables dry-run.
func (c *Correlator) triggerIncidentBlockLocked(inc *Incident, ip string, now time.Time, why string) func() {
if inc == nil || c.cfg.OnIncidentBlock == nil {
return nil
}
reason := "incident " + string(inc.Kind) + " " + inc.Severity.String() + " (" + why + ")"
return c.requestBlockLocked(inc, ip, now, c.pendingIncidentBlocks, c.cfg.AutoBlock.BlockExpiry,
reason, "incident_block_requested", c.cfg.OnIncidentBlock)
}
// requestBlockLocked marks inc as having a block in flight in pending and
// returns the callback that asks onBlock for it, or nil when the previous
// block has not lapsed or a request is already in flight. Caller holds c.mu
// and must invoke the callback after unlocking. action is appended only when
// onBlock reports a live block request, so a declined attempt can retry on a
// later finding once the in-flight marker is cleared.
func (c *Correlator) requestBlockLocked(inc *Incident, ip string, now time.Time, pending map[string]bool, expiry time.Duration, reason, action string, onBlock func(ip, reason string, ttl time.Duration, findingID string, root PreparedRoot) bool) func() {
if !inc.AutoBlock.lapsed(now) {
return nil
}
if _, ok := c.pendingIncidentBlocks[inc.ID]; ok {
return nil
}
if _, ok := c.pendingSprayBlocks[inc.ID]; ok {
return nil
}
pending[inc.ID] = true
prior := inc.AutoBlock
incidentID := inc.ID
attempt := inc.AutoBlock.Count + 1
ttl := blockTTLForAttempt(attempt, expiry)
if attempt > 1 {
reason += "; block " + strconv.Itoa(attempt) + " after the previous one lapsed"
}
findingID, root := c.attesting(inc, ip)
return func() {
var live bool
callbackReturned := false
// The in-flight slot must clear even if onBlock panics. Otherwise,
// later findings keep seeing the incident as already in flight and
// skip the auto-block path for the rest of the incident lifetime.
defer func() {
c.mu.Lock()
defer c.mu.Unlock()
valid := pending[incidentID]
delete(pending, incidentID)
if !valid || !callbackReturned || !live {
return
}
current, ok := c.incidents[incidentID]
if !ok || current != inc || !incidentStatusActive(current.Status) || current.AutoBlock != prior {
return
}
current.AutoBlock = AutoBlockState{Count: attempt, LastAt: now}
if ttl > 0 {
current.AutoBlock.ExpiresAt = now.Add(ttl)
}
current.Actions = append(current.Actions, IncidentAction{
Time: now,
Action: action,
Result: "ok",
Details: ip + " " + reason + " " + blockDurationLabel(ttl),
})
c.markPersistedLocked(incidentID, c.now())
c.persistLocked(*current)
}()
c.mu.Lock()
valid := pending[incidentID] && c.incidents[incidentID] == inc && incidentStatusActive(inc.Status) && inc.AutoBlock == prior
c.mu.Unlock()
if !valid {
return
}
live = onBlock(ip, reason, ttl, findingID, root)
callbackReturned = true
}
}
// blockAddressAttested reports whether a finding whose address is attacker
// evidence named ip. The correlation key's address keeps that across
// timeline trimming; any other candidate exists only while the timeline is
// whole, so its events decide.
func (c *Correlator) blockAddressAttested(inc *Incident, ip string) bool {
// Without a gate nothing attests an address: fail closed.
if c.cfg.AddressEvidence == nil {
return false
}
if c.rememberedAddressEvidence(inc, ip) {
return true
}
for _, ev := range inc.Timeline {
if ev.Kind == "finding" && normalizeIncidentRemoteIP(ev.RemoteIP) == ip && c.eventAttests(ev) {
return true
}
}
return false
}
// rememberedAddressEvidence rechecks authority kept beyond timeline
// trimming. An old bit without its attesting policy identity is no proof.
func (c *Correlator) rememberedAddressEvidence(inc *Incident, ip string) bool {
if !inc.RemoteIPEvidence || inc.RemoteIPEvidenceCheck == "" || c.cfg.AddressEvidence == nil || inc.CorrelationKey == nil || normalizeIncidentRemoteIP(inc.CorrelationKey.RemoteIP) != ip {
return false
}
return c.eventAttests(IncidentEvent{Check: inc.RemoteIPEvidenceCheck, Severity: inc.RemoteIPEvidenceSeverity})
}
// restoreAddressEvidence migrates verifiable old proof from a retained
// event. Missing or retired proof cannot authorize a new block; installed
// blocks and the escalation ladder remain untouched.
func (c *Correlator) restoreAddressEvidence(inc *Incident) {
if inc.CorrelationKey != nil {
ip := normalizeIncidentRemoteIP(inc.CorrelationKey.RemoteIP)
if ip != "" && c.rememberedAddressEvidence(inc, ip) {
if inc.RemoteIPEvidenceFinding == "" {
for i := len(inc.Timeline) - 1; i >= 0; i-- {
ev := inc.Timeline[i]
if ev.Kind == "finding" && ev.FindingID != "" && normalizeIncidentRemoteIP(ev.RemoteIP) == ip && c.eventAttests(ev) {
inc.RemoteIPEvidenceFinding = ev.FindingID
inc.RemoteIPEvidenceCheck = ev.Check
inc.RemoteIPEvidenceSeverity = ev.Severity
break
}
}
}
return
}
}
inc.RemoteIPEvidence = false
inc.RemoteIPEvidenceFinding = ""
inc.RemoteIPEvidenceCheck = ""
inc.RemoteIPEvidenceSeverity = ""
if c.cfg.AddressEvidence == nil || inc.CorrelationKey == nil {
return
}
ip := normalizeIncidentRemoteIP(inc.CorrelationKey.RemoteIP)
if ip == "" {
return
}
for i := len(inc.Timeline) - 1; i >= 0; i-- {
ev := inc.Timeline[i]
if ev.Kind == "finding" && normalizeIncidentRemoteIP(ev.RemoteIP) == ip && c.eventAttests(ev) {
inc.RemoteIPEvidence = true
inc.RemoteIPEvidenceFinding = ev.FindingID
inc.RemoteIPEvidenceCheck = ev.Check
inc.RemoteIPEvidenceSeverity = ev.Severity
return
}
}
}
// eventAttests reports whether a timeline finding is address evidence.
// It runs only behind a non-nil gate.
func (c *Correlator) eventAttests(ev IncidentEvent) bool {
sev, ok := parseSeverity(ev.Severity)
if ev.Severity == "" {
// Older events lack severity. They can attest checks with no
// severity floor, but cannot establish a Critical-only signal.
sev, ok = alert.Warning, true
}
return ok && c.cfg.AddressEvidence(ev.Check, sev)
}
// attesting names the finding a block of ip rests on and the admission root
// kept for it: the newest timeline finding with address evidence for it,
// or the one recorded with RemoteIPEvidence, without a root, once timeline
// trimming dropped that event.
func (c *Correlator) attesting(inc *Incident, ip string) (string, PreparedRoot) {
ip = normalizeIncidentRemoteIP(ip)
if ip == "" {
return "", PreparedRoot{}
}
for i := len(inc.Timeline) - 1; i >= 0; i-- {
ev := inc.Timeline[i]
if ev.Kind == "finding" && ev.FindingID != "" && normalizeIncidentRemoteIP(ev.RemoteIP) == ip && c.eventAttests(ev) {
if ev.root == nil {
return ev.FindingID, PreparedRoot{}
}
return ev.FindingID, *ev.root
}
}
if inc.RemoteIPEvidence && inc.CorrelationKey != nil && normalizeIncidentRemoteIP(inc.CorrelationKey.RemoteIP) == ip {
return inc.RemoteIPEvidenceFinding, PreparedRoot{}
}
return "", PreparedRoot{}
}
func incidentBlockCandidate(inc *Incident) string {
if inc == nil {
return ""
}
if inc.CorrelationKey != nil {
if ip := normalizeIncidentRemoteIP(inc.CorrelationKey.RemoteIP); ip != "" {
return ip
}
}
var candidate string
for _, ev := range inc.Timeline {
if ev.Kind == incidentTimelineTruncatedKind {
return ""
}
ip := normalizeIncidentRemoteIP(ev.RemoteIP)
if ip == "" {
continue
}
if candidate == "" {
candidate = ip
continue
}
if candidate != ip {
return ""
}
}
return candidate
}
func (c *Correlator) sprayOwnsIncident(inc *Incident) bool {
if c == nil || c.spray == nil || inc == nil {
return false
}
owned := false
for _, ev := range inc.Timeline {
if ev.RemoteIP == "" {
continue
}
if !c.spray.cfg.PerCheck[ev.Check] {
return false
}
owned = true
}
return owned
}
func normalizeIncidentRemoteIP(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
if host, _, err := net.SplitHostPort(raw); err == nil {
raw = host
}
raw = strings.Trim(raw, "[]")
ip := net.ParseIP(raw)
if ip == nil || ip.IsLoopback() || ip.IsUnspecified() {
return ""
}
return ip.String()
}
// blockDurationLabel renders a block's lifetime for the incident action so an
// operator reading the timeline can see the ladder escalating.
func blockDurationLabel(ttl time.Duration) string {
if ttl <= 0 {
return "(permanent)"
}
return "(" + ttl.String() + ")"
}
func hasIncidentAction(actions []IncidentAction, action string) bool {
for _, a := range actions {
if a.Action == action {
return true
}
}
return false
}
// triggerSprayBlockLocked invokes the operator-supplied OnSprayBlock
// callback at most once while a spray block is already in flight and
// appends an audit action only when the callback reports the firewall
// actually applied the block. Caller holds c.mu. The returned callback
// must be invoked after unlocking. Idempotent after success; declined
// callbacks can retry on a later finding after the in-flight marker is
// cleared.
func (c *Correlator) triggerSprayBlockLocked(inc *Incident, ip string, hits int, now time.Time, why string) func() {
if inc == nil || c.cfg.OnSprayBlock == nil {
return nil
}
reason := "credential_spray: " + strconv.Itoa(hits) + " distinct mailboxes (" + why + ")"
return c.requestBlockLocked(inc, ip, now, c.pendingSprayBlocks, c.spray.cfg.BlockExpiry,
reason, "credential_spray_block_requested", c.cfg.OnSprayBlock)
}
package incident
import (
"sort"
"time"
"github.com/pidginhost/csm/internal/alert"
)
// IncidentGroupsScanCap is the hard upper bound on matching incidents
// grouped per BuildGroups call. Rows excluded by status/kind filters do
// not consume the cap.
const IncidentGroupsScanCap = 10000
// Group is one row of the grouped incident view: a (kind, source)
// bucket plus rolled-up counters. Source identifies what the bucket is
// keyed on -- IP for credential-spray patterns, account/domain/mailbox
// when the source IP is unknown.
type Group struct {
Key string `json:"key"`
Kind Kind `json:"kind"`
SourceKind string `json:"source_kind"`
Source string `json:"source"`
IncidentCount int `json:"incident_count"`
OpenCount int `json:"open_count"`
ContainedCount int `json:"contained_count"`
ResolvedCount int `json:"resolved_count"`
DismissedCount int `json:"dismissed_count"`
SeverityMax alert.Severity `json:"-"`
SeverityLabel string `json:"severity_max"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
SampleIDs []string `json:"sample_ids"`
}
// GroupsResponse is what BuildGroups returns. /api/v1/incidents/groups
// sends Groups as its items, with the counts next to them.
type GroupsResponse struct {
Groups []Group `json:"groups"`
TotalGroups int `json:"total_groups"`
ScannedIncidents int `json:"scanned_incidents"`
Truncated bool `json:"truncated"`
}
// GroupFilter narrows what BuildGroups buckets. Empty fields mean "no
// filter on that dimension".
type GroupFilter struct {
// StatusSet, when non-empty, restricts incidents to the listed
// statuses. Pass {StatusOpen, StatusContained} to surface only
// active incidents (the default web UI mode).
StatusSet []Status
// Kind, when non-empty, restricts incidents to a specific kind.
Kind Kind
// Offset is the starting index into the sorted group list. Used by
// the UI to page through groups when TotalGroups exceeds MaxGroups.
Offset int
// MaxGroups caps the returned slice. Zero or negative means "no
// cap"; the handler still applies a sane upper bound.
MaxGroups int
}
// BuildGroups buckets the supplied incidents by (kind, source) and
// returns the rolled-up groups sorted by incident_count desc, then
// severity_max desc, then last_seen desc. SampleIDs holds up to three
// of the most recently updated members of each group so the UI can
// drill in without a follow-up call.
//
// `incidents` may be the full correlator snapshot. The function caps
// its scan at IncidentGroupsScanCap after status/kind filtering; the
// returned `truncated` flag reports whether the cap clipped matching
// incidents. Callers fed by Correlator.Snapshot() get an already-newest-
// first slice; sort stability there means the truncation drops the oldest
// matching entries first, which is what an operator wants.
func BuildGroups(incidents []Incident, filter GroupFilter) GroupsResponse {
statusAllowed := func(Status) bool { return true }
if len(filter.StatusSet) > 0 {
set := make(map[Status]struct{}, len(filter.StatusSet))
for _, s := range filter.StatusSet {
set[s] = struct{}{}
}
statusAllowed = func(s Status) bool {
_, ok := set[s]
return ok
}
}
type sampleEntry struct {
id string
updatedAt time.Time
}
type aggregator struct {
group Group
samples []sampleEntry
statusMap map[Status]int
}
bucketKey := func(kind Kind, sourceKind, source string) string {
return string(kind) + "|" + sourceKind + ":" + source
}
buckets := make(map[string]*aggregator)
scanned := 0
truncated := false
for _, inc := range incidents {
if !statusAllowed(inc.Status) {
continue
}
if filter.Kind != "" && inc.Kind != filter.Kind {
continue
}
if scanned >= IncidentGroupsScanCap {
truncated = true
break
}
scanned++
sourceKind, source := groupSource(inc)
k := bucketKey(inc.Kind, sourceKind, source)
agg, ok := buckets[k]
if !ok {
agg = &aggregator{
group: Group{
Key: k,
Kind: inc.Kind,
SourceKind: sourceKind,
Source: source,
FirstSeen: inc.CreatedAt,
LastSeen: inc.UpdatedAt,
},
statusMap: map[Status]int{},
}
buckets[k] = agg
}
agg.group.IncidentCount++
agg.statusMap[inc.Status]++
if inc.Severity > agg.group.SeverityMax {
agg.group.SeverityMax = inc.Severity
}
if inc.CreatedAt.Before(agg.group.FirstSeen) || agg.group.FirstSeen.IsZero() {
agg.group.FirstSeen = inc.CreatedAt
}
if inc.UpdatedAt.After(agg.group.LastSeen) {
agg.group.LastSeen = inc.UpdatedAt
}
agg.samples = append(agg.samples, sampleEntry{id: inc.ID, updatedAt: inc.UpdatedAt})
}
out := make([]Group, 0, len(buckets))
for _, agg := range buckets {
agg.group.OpenCount = agg.statusMap[StatusOpen]
agg.group.ContainedCount = agg.statusMap[StatusContained]
agg.group.ResolvedCount = agg.statusMap[StatusResolved]
agg.group.DismissedCount = agg.statusMap[StatusDismissed]
agg.group.SeverityLabel = agg.group.SeverityMax.String()
// Top-3 most recently updated members.
sort.SliceStable(agg.samples, func(i, j int) bool {
return agg.samples[i].updatedAt.After(agg.samples[j].updatedAt)
})
n := len(agg.samples)
if n > 3 {
n = 3
}
ids := make([]string, n)
for i := 0; i < n; i++ {
ids[i] = agg.samples[i].id
}
agg.group.SampleIDs = ids
out = append(out, agg.group)
}
sort.SliceStable(out, func(i, j int) bool {
if out[i].IncidentCount != out[j].IncidentCount {
return out[i].IncidentCount > out[j].IncidentCount
}
if out[i].SeverityMax != out[j].SeverityMax {
return out[i].SeverityMax > out[j].SeverityMax
}
return out[i].LastSeen.After(out[j].LastSeen)
})
totalGroups := len(out)
if filter.Offset > 0 {
if filter.Offset >= len(out) {
out = out[:0]
} else {
out = out[filter.Offset:]
}
}
if filter.MaxGroups > 0 && len(out) > filter.MaxGroups {
out = out[:filter.MaxGroups]
}
return GroupsResponse{
Groups: out,
TotalGroups: totalGroups,
ScannedIncidents: scanned,
Truncated: truncated,
}
}
// groupSource derives the (source_kind, source) pair the UI uses to
// label and drill into a group. Cascade order: host > remote_ip >
// account > domain > mailbox > "_unkeyed". The IP path is the most
// useful grouping for credential-spray patterns and the most common
// shape on busy hosts.
func groupSource(inc Incident) (sourceKind, source string) {
if inc.CorrelationKey != nil && inc.CorrelationKey.Host != "" {
return "host", inc.CorrelationKey.Host
}
if inc.CorrelationKey != nil && inc.CorrelationKey.RemoteIP != "" {
return "ip", inc.CorrelationKey.RemoteIP
}
if ip := timelineRemoteIP(inc); ip != "" {
return "ip", ip
}
if inc.Account != "" {
return "account", inc.Account
}
if inc.Domain != "" {
return "domain", inc.Domain
}
if inc.Mailbox != "" {
return "mailbox", inc.Mailbox
}
return "_unkeyed", ""
}
func timelineRemoteIP(inc Incident) string {
counts := make(map[string]int)
for _, ev := range inc.Timeline {
if ev.RemoteIP != "" {
counts[ev.RemoteIP]++
}
}
best := ""
bestCount := 0
for ip, count := range counts {
if count > bestCount || count == bestCount && (best == "" || ip < best) {
best = ip
bestCount = count
}
}
return best
}
package incident
import (
"strings"
"github.com/pidginhost/csm/internal/alert"
)
// Key is the correlation key derived from a Finding. Empty fields mean
// "not provided"; the correlator uses the most specific non-empty fields.
// Host is the synthetic local-host actor for findings whose blast radius
// is the machine itself rather than one tenant, process, or remote IP.
type Key struct {
Host string `json:"host,omitempty"`
Account string `json:"account,omitempty"`
Domain string `json:"domain,omitempty"`
Mailbox string `json:"mailbox,omitempty"`
UID int `json:"uid,omitempty"`
PID int `json:"pid,omitempty"`
RemoteIP string `json:"remote_ip,omitempty"`
}
// excludedFromIncidents reports whether a finding records posture, a response,
// or degraded visibility rather than an event that needs containment.
func excludedFromIncidents(check string) bool {
switch check {
case "vulnerable_plugins", "outdated_plugins":
// An installed weakness is not evidence that the account was entered.
return true
case "auto_block", "auto_response", "auto_response_paused", "challenge_route",
"reputation_quota_exhausted", "threat_feed_stale", "shared_senders_stale",
"wp_core_unverified", "wp_plugin_inventory_unverified":
// CSM output must not re-enter the incident response path. In
// particular, response findings may gain attacker attribution later.
return true
default:
return false
}
}
// IsEmpty reports whether the key has nothing to correlate on. Such
// findings are emitted normally but do not join an incident.
func (k Key) IsEmpty() bool {
return k.Host == "" && k.Account == "" && k.Domain == "" && k.Mailbox == "" && k.UID == 0 && k.PID == 0 && k.RemoteIP == ""
}
// KeyFor extracts a correlation key from a Finding. Host-integrity
// findings are keyed to the local host so unattributed root/system events
// still become incidents. TenantID, Process.Account, CPUser, and a
// /home[N]/<account>/ heuristic provide account attribution. Domain and
// Mailbox come directly from the finding. Process UID/PID and SourceIP are
// fallback identities: they should not split account/domain/mailbox
// incidents.
//
// Mailbox + Domain are canonicalised so emitters that set either the
// full local@domain form or the split (Mailbox=local, Domain=site)
// form land on the same key. Without that, two findings about the
// same mailbox split into two incidents whenever the emitters use
// different conventions.
func KeyFor(f alert.Finding) Key {
if excludedFromIncidents(f.Check) {
return Key{}
}
switch ClassifyKind(f) {
case KindHostIntegrityRisk:
return Key{Host: "host"}
case KindWebAttack, KindMailboxBruteforce:
// Inbound attacks correlate on the attacker IP; the victim
// domain/account/mailbox is the target, not the key. This
// collapses one attacker's hits across many victims into a single
// incident. ClassifyKind only returns these kinds when a source IP
// is present.
return Key{RemoteIP: alert.AttackerAddress(f)}
}
mailbox, domain := canonicalizeMailboxDomain(f.Mailbox, f.Domain)
k := Key{
Account: f.TenantID,
Domain: domain,
Mailbox: mailbox,
}
if f.Process != nil && f.Process.Account != "" && k.Account == "" {
k.Account = f.Process.Account
}
if k.Account == "" && f.CPUser != "" {
k.Account = f.CPUser
}
if k.Account == "" {
k.Account = accountFromHomePath(f.FilePath)
}
if f.Process != nil && !hasStableActor(k) {
if f.Process.UID != 0 {
k.UID = f.Process.UID
}
if k.UID == 0 {
k.PID = f.Process.PID
}
}
if k.Account == "" && k.Domain == "" && k.Mailbox == "" && k.UID == 0 && k.PID == 0 {
k.RemoteIP = f.SourceIP
}
return k
}
func hasStableActor(k Key) bool {
return k.Account != "" || k.Domain != "" || k.Mailbox != ""
}
// canonicalizeMailboxDomain merges Mailbox+Domain into a stable
// (Mailbox, Domain) key pair regardless of which emit convention the
// caller used. Rules:
//
// - If Mailbox already contains "@", treat it as authoritative;
// drop Domain to avoid double-keying on conflicting site.
// - If Mailbox lacks "@" and Domain is set, splice them into the
// full form. Domain is then dropped from the key (it's already
// encoded in Mailbox).
// - Domain-only findings (no Mailbox) keep the domain as the key.
//
// Domain names are case-insensitive, so only the domain component is
// lower-cased. The local part is left intact.
func canonicalizeMailboxDomain(mailbox, domain string) (string, string) {
mailbox = strings.TrimSpace(mailbox)
domain = normalizeDomainForKey(domain)
if mailbox == "" {
return "", domain
}
if local, mailboxDomain := splitMailboxForKey(mailbox); mailboxDomain != "" {
return local + "@" + mailboxDomain, ""
}
if strings.Contains(mailbox, "@") {
return mailbox, ""
}
if domain == "" {
return mailbox, ""
}
return mailbox + "@" + domain, ""
}
func canonicalizeKey(k Key) Key {
k.Mailbox, k.Domain = canonicalizeMailboxDomain(k.Mailbox, k.Domain)
return k
}
func displayMailboxDomain(mailbox, domain string) (string, string) {
mailbox = strings.TrimSpace(mailbox)
domain = strings.TrimSpace(domain)
if mailbox == "" {
return "", domain
}
if local, mailboxDomain := splitMailboxForKey(mailbox); mailboxDomain != "" {
return local + "@" + mailboxDomain, mailboxDomain
}
if domain == "" {
return mailbox, ""
}
if strings.Contains(mailbox, "@") {
return mailbox, domain
}
normalizedDomain := normalizeDomainForKey(domain)
return mailbox + "@" + normalizedDomain, normalizedDomain
}
func splitMailboxForKey(mailbox string) (string, string) {
at := strings.LastIndexByte(mailbox, '@')
if at <= 0 || at == len(mailbox)-1 {
return "", ""
}
return mailbox[:at], normalizeDomainForKey(mailbox[at+1:])
}
func normalizeDomainForKey(domain string) string {
return strings.ToLower(strings.TrimSpace(domain))
}
// accountFromHomePath parses /home[N]/<account>/... paths. Returns the
// account segment or "" if the path does not match the cPanel-style home
// layout. Pure string parsing; does not walk the filesystem.
func accountFromHomePath(p string) string {
if p == "" {
return ""
}
parts := strings.SplitN(p, "/", 4)
if len(parts) < 3 {
return ""
}
if parts[0] != "" {
return ""
}
if !strings.HasPrefix(parts[1], "home") {
return ""
}
for _, ch := range parts[1][len("home"):] {
if ch < '0' || ch > '9' {
return ""
}
}
return parts[2]
}
package incident
import (
"strings"
"github.com/pidginhost/csm/internal/alert"
)
// ClassifyKind returns the incident Kind for a Finding using rule
// precedence: host-integrity signals first, then inbound attacks keyed on the
// attacker IP (web_attack, mailbox_bruteforce), then mailbox-takeover signals,
// then ephemeral-path process exec, then remote-IP reputation, then the
// account-scoped web-compromise default.
func ClassifyKind(f alert.Finding) Kind {
check := strings.ToLower(f.Check)
hasAttacker := strings.TrimSpace(alert.AttackerAddress(f)) != ""
// Host integrity -- daemon/kernel-level signals whose blast radius is
// the host itself, not a single tenant or an inbound attacker. Listed
// explicitly so account-attributed checks that share a substring
// (e.g. suspicious_crontab on a per-user spool) stay in the tenant
// bucket.
if isHostIntegrityCheck(check) {
return KindHostIntegrityRisk
}
// Inbound attack keyed on the attacker IP. A defended web hit (WAF
// block, scanner probe, login brute-force) or a failed mailbox login
// names a victim domain/account/mailbox, but that is the attack
// target, not evidence the account is compromised. Classify by the
// attacker so defended traffic does not inflate the compromise /
// takeover buckets or inherit their long retention. Requires an
// attacker IP to key on; without one the finding falls through to the
// account-scoped tiers below. Genuine compromise is recognised by the
// on-disk and behavioural signals handled lower down, not by inbound
// hits.
if hasAttacker {
if isMailAuthAttackCheck(check) {
return KindMailboxBruteforce
}
if isInboundWebAttackCheck(check) {
return KindWebAttack
}
}
// Mailbox takeover -- a Mailbox attribute or a post-authentication
// mail-abuse check name. Mailbox attribution is the strongest single
// tenant signal so it wins over the account-scoped default below. Some
// authenticated-mail findings route bare cPanel-local accounts to
// TenantID instead of Mailbox; the check-name list keeps those here
// without sweeping in domain/config mail checks or PHP relay findings.
if f.Mailbox != "" {
return KindMailboxTakeover
}
if isMailboxTakeoverCheck(check) {
return KindMailboxTakeover
}
// Post-exploit process -- exe under ephemeral paths is a strong
// indicator of staged-then-executed payloads (cryptominers, reverse
// shells) regardless of which tenant owns the parent.
if f.Process != nil && (strings.HasPrefix(f.Process.Exe, "/tmp/") ||
strings.HasPrefix(f.Process.Exe, "/var/tmp/") ||
strings.HasPrefix(f.Process.Exe, "/dev/shm/")) {
return KindPostExploitProcess
}
// Remote-IP reputation / threat-score with no victim attribution -- an
// attacking source IP flagged by reputation, not a compromised tenant.
// Kept on the strict remote-IP-keyed gate: a reputation hit tied to an
// account is about that account, not an anonymous attacker. A hit on an
// address found failing mailbox logins is a mail attack.
if isRemoteIPThreatCheck(check) && isRemoteIPKeyed(f) {
if f.MailLoginFailure {
return KindMailboxBruteforce
}
return KindWebAttack
}
// Default -- account-scoped web compromise. Most CSM findings are
// tenant-attributed web/PHP issues, so this fallback matches the
// modal incident shape operators see.
return KindWebAccountCompromise
}
// isRemoteIPKeyed reports whether a finding would correlate on its source
// IP alone -- i.e. it has no account (tenant, cPanel user, process
// account, or /home/<account>/ path), no domain, no mailbox, and no
// stable process actor (UID/PID). It mirrors KeyFor's RemoteIP fallback
// without calling KeyFor (KeyFor depends on ClassifyKind, so the reverse
// call would recurse). Callers reach this only after the host-integrity,
// mailbox, and post-exploit-process tiers have been ruled out.
func isRemoteIPKeyed(f alert.Finding) bool {
if strings.TrimSpace(f.SourceIP) == "" {
return false
}
mailbox, domain := canonicalizeMailboxDomain(f.Mailbox, f.Domain)
if mailbox != "" || domain != "" {
return false
}
if f.TenantID != "" || f.CPUser != "" {
return false
}
if f.Process != nil && f.Process.Account != "" {
return false
}
if accountFromHomePath(f.FilePath) != "" {
return false
}
if f.Process != nil && (f.Process.UID != 0 || f.Process.PID != 0) {
return false
}
return true
}
// isRemoteIPThreatCheck covers remote-IP reputation / threat-score signals
// that flag an attacking source IP rather than a compromised tenant. Like
// inbound web attacks these correlate on the source IP alone, so they belong
// in web_attack with attacker-grade retention, not the account-compromise
// bucket. Add future remote-IP reputation checks here.
func isRemoteIPThreatCheck(check string) bool {
switch strings.ToLower(strings.TrimSpace(check)) {
case "ip_reputation", "mail_spam_source":
return true
default:
return false
}
}
// isMailAuthAttackCheck covers failed mail-authentication and pre-auth mail
// probe signals. These are attacker attempts keyed on the source IP, not
// evidence that the targeted mailbox was taken over.
func isMailAuthAttackCheck(check string) bool {
switch strings.ToLower(strings.TrimSpace(check)) {
case "email_auth_failure_realtime",
"mail_account_spray",
"mail_bruteforce",
"mail_bruteforce_suspected",
"mail_subnet_spray",
"smtp_account_spray",
"smtp_bruteforce",
"smtp_probe_abuse",
"smtp_subnet_spray",
"stalwart_ban",
"stalwart_webmail_bruteforce":
return true
default:
return false
}
}
func isInboundWebAttackCheck(check string) bool {
check = strings.ToLower(strings.TrimSpace(check))
if strings.HasPrefix(check, "http_") || strings.HasPrefix(check, "modsec_") {
return true
}
switch check {
case "admin_panel_bruteforce",
"email_php_form_relay",
"email_php_form_relay_observed",
"wp_login_bruteforce",
"wp_user_enumeration",
"xmlrpc_abuse",
"waf_attack_blocked",
"webshell_caller",
"webshell_path_public",
"api_auth_failure",
"api_auth_failure_realtime",
"webmail_bruteforce",
"webmail_login_realtime",
"whm_login_realtime",
"whm_unauth_scripts_realtime":
return true
default:
return false
}
}
// hostIntegrityChecks lists check names whose scope is the host itself
// (kernel modules, system daemon configs, root-owned credential stores)
// rather than a single tenant. Findings matching one of these jump
// straight to KindHostIntegrityRisk so incident severity reflects the
// blast radius.
var hostIntegrityChecks = map[string]bool{
"bulk_password_change": true,
"sensitive_file_modified": true,
"fake_kernel_thread": true,
"integrity": true,
"shadow_change": true,
"sshd_config_change": true,
"root_password_change": true,
"uid0_account": true,
"suid_binary": true,
"bad_asn_outbound": true,
"kernel_module": true,
"crontab_change": true,
"crond_change": true,
"firewall_ipv6_unmanaged": true,
"mail_auth_backend_degraded": true,
}
func isHostIntegrityCheck(check string) bool {
return hostIntegrityChecks[strings.ToLower(strings.TrimSpace(check))]
}
func isMailboxTakeoverCheck(check string) bool {
switch check {
case "email_cloud_relay_abuse",
"email_compromised_account",
"email_credential_leak",
"email_rate_critical",
"email_rate_warning",
"email_spam_outbreak",
"email_suspicious_geo",
"mail_account_compromised",
"mail_per_account":
return true
default:
return false
}
}
package incident
import "github.com/pidginhost/csm/internal/metrics"
// RegisterMetrics binds the correlator's counters to reg. Production
// callers should pass metrics.Default(); tests pass metrics.NewRegistry()
// to keep registration isolated.
func RegisterMetrics(reg *metrics.Registry, c *Correlator) {
reg.RegisterGaugeFunc(
"csm_incidents_open",
"Open and Contained incidents currently in correlator state.",
func() float64 { return float64(c.OpenCount()) },
)
reg.RegisterCounterFunc(
"csm_incidents_created_total",
"Total incidents created by the correlator.",
func() float64 { return float64(c.counters.createdTotal.Load()) },
)
reg.RegisterCounterFunc(
"csm_incidents_severity_changed_total",
"Incident severity escalations (severity does not downgrade, so this is monotonic).",
func() float64 { return float64(c.counters.severityChangedTotal.Load()) },
)
reg.RegisterCounterFunc(
"csm_incidents_status_changed_total",
"Incident status transitions (open/contained/resolved/dismissed).",
func() float64 { return float64(c.counters.statusChangedTotal.Load()) },
)
reg.RegisterCounterFunc(
"csm_incidents_findings_merged_total",
"Findings merged into an existing incident (not counted on incident create).",
func() float64 { return float64(c.counters.findingsMergedTotal.Load()) },
)
reg.RegisterCounterFunc(
"csm_incidents_compacted_total",
"Incidents pruned by retention compaction (resolved/dismissed beyond TTL).",
func() float64 { return float64(c.counters.compactedTotal.Load()) },
)
reg.RegisterGaugeFunc(
"csm_incidents_pending",
"Findings held in the threshold gate, awaiting a second correlated finding before opening an incident.",
func() float64 { return float64(c.PendingCount()) },
)
reg.RegisterCounterFunc(
"csm_incidents_auto_closed_total",
"Open or contained incidents auto-resolved after exceeding their per-kind idle threshold.",
func() float64 { return float64(c.counters.autoClosedTotal.Load()) },
)
reg.RegisterCounterFunc(
"csm_incidents_auto_close_dry_run_total",
"Auto-close decisions counted while dry_run was active (state unchanged).",
func() float64 { return float64(c.counters.autoCloseDryRunTotal.Load()) },
)
reg.RegisterCounterFunc(
"csm_credential_spray_opened_total",
"Credential-spray super-incidents opened (one source IP brute-forcing many mailboxes).",
func() float64 { return float64(c.counters.sprayOpenedTotal.Load()) },
)
reg.RegisterCounterFunc(
"csm_credential_spray_suppressed_mailbox_takeover_total",
"Per-mailbox incidents suppressed because a credential_spray incident already covers the source IP.",
func() float64 { return float64(c.counters.spraySuppressedTotal.Load()) },
)
reg.RegisterCounterFunc(
"csm_credential_spray_dry_run_total",
"Spray decisions counted while dry_run was active (routing unchanged).",
func() float64 { return float64(c.counters.sprayDryRunTotal.Load()) },
)
reg.RegisterGaugeFunc(
"csm_credential_spray_tracked_ips",
"Source IPs currently held in the spray detector's per-IP map.",
func() float64 { return float64(c.SprayTrackedIPs()) },
)
}
package incident
import (
"sync"
"time"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/queuehealth"
)
type queuedPersist struct {
previous <-chan struct{}
done chan struct{}
snap Incident
persist func(Incident) error
at time.Time
running, failed bool
}
type persistQueue struct {
mu sync.Mutex
tail chan struct{}
waiting map[*queuedPersist]struct{}
active *queuedPersist
idleSince time.Time
// Deferred bookkeeping waits for the next mutation or explicit flush.
// Its age is visibility, not a promised delivery deadline.
deferred map[string]time.Time
waitingLoss, activeLoss *queuehealth.Tracker
}
func newPersistQueue() *persistQueue {
tail := make(chan struct{})
close(tail)
return &persistQueue{
tail: tail,
waiting: make(map[*queuedPersist]struct{}),
deferred: make(map[string]time.Time),
waitingLoss: queuehealth.New(0, time.Minute),
activeLoss: queuehealth.New(1, time.Minute),
}
}
func (q *persistQueue) deferWrite(id string) {
q.mu.Lock()
defer q.mu.Unlock()
if _, pending := q.deferred[id]; !pending {
q.deferred[id] = time.Now()
}
}
func (q *persistQueue) discardDeferred(id string) {
q.mu.Lock()
delete(q.deferred, id)
q.mu.Unlock()
}
func (q *persistQueue) deferredIDs() []string {
q.mu.Lock()
defer q.mu.Unlock()
ids := make([]string, 0, len(q.deferred))
for id := range q.deferred {
ids = append(ids, id)
}
return ids
}
// queuePersistLocked reserves this write's place in mutation order while
// c.mu is still held. The returned callback must run after c.mu is released.
func (c *Correlator) queuePersistLocked(snap Incident) (*queuedPersist, bool) {
persist := c.cfg.Persist
q := c.persistence
if persist == nil {
q.discardDeferred(snap.ID)
return nil, false
}
req := &queuedPersist{snap: cloneIncident(snap), persist: persist, done: make(chan struct{}), at: time.Now()}
q.mu.Lock()
defer q.mu.Unlock()
// A full immutable snapshot supersedes deferred bookkeeping atomically
// with publication, so health cannot lose the owner during the transfer.
delete(q.deferred, snap.ID)
if q.active == nil && len(q.waiting) == 0 {
q.idleSince = req.at
}
req.previous = q.tail
q.tail = req.done
q.waiting[req] = struct{}{}
return req, true
}
func (q *persistQueue) loseLocked(req *queuedPersist) {
if req.failed {
return
}
req.failed = true
tracker := q.waitingLoss
if req.running {
tracker = q.activeLoss
}
tracker.Lose(time.Now(), 1)
}
func (q *persistQueue) finish(req *queuedPersist, completed bool) {
q.mu.Lock()
defer q.mu.Unlock()
if !completed {
q.loseLocked(req)
}
if req.running {
q.active = nil
q.idleSince = time.Now()
} else {
delete(q.waiting, req)
}
// Publish completion with owner removal; a successor cannot become active
// while health still attributes the slot to its predecessor.
close(req.done)
}
func (c *Correlator) runQueuedPersist(req *queuedPersist) {
q := c.persistence
<-req.previous
completed := false
defer func() { q.finish(req, completed) }()
q.mu.Lock()
delete(q.waiting, req)
q.active = req
req.running = true
req.at = time.Now()
q.mu.Unlock()
if err := req.persist(req.snap); err != nil {
q.mu.Lock()
q.loseLocked(req)
q.mu.Unlock()
// The in-memory transition has already advanced. Count failed durable
// work before logging, which can itself wait on an output writer.
csmlog.Warn("incident persist failed", "id", req.snap.ID, "kind", string(req.snap.Kind), "status", string(req.snap.Status), "err", err)
}
completed = true
}
func (c *Correlator) runQueuedPersists(batch []*queuedPersist) {
next := 0
defer func() {
// The batch reserved contiguous ordering links under c.mu. The active
// callback has finished its cleanup before this defer, so abandoning
// its unstarted tail cannot overtake an executing predecessor.
for _, req := range batch[next:] {
c.persistence.finish(req, false)
}
}()
for next < len(batch) {
req := batch[next]
next++
c.runQueuedPersist(req)
}
}
// QueueStatuses reads only queue memory, independently of correlator state and
// persistence callbacks. Waiting writes use progress of the shared writer.
func (c *Correlator) QueueStatuses(now time.Time) map[string]queuehealth.Status {
q := c.persistence
q.mu.Lock()
defer q.mu.Unlock()
waiting := q.waitingLoss.Snapshot(now)
waiting.CapacityUnavailable = true
active := q.activeLoss.Snapshot(now)
stalled := q.active == nil && !q.idleSince.IsZero() && now.Sub(q.idleSince) >= time.Minute
if req := q.active; req != nil {
active.InFlight = 1
active.ProcessingSeconds = max(0, now.Sub(req.at).Seconds())
if active.ProcessingSeconds >= time.Minute.Seconds() {
active.Status, active.Reason = "degraded", "processing_lag"
stalled = true
}
}
for req := range q.waiting {
waiting.Depth++
waiting.LagSeconds = max(waiting.LagSeconds, now.Sub(req.at).Seconds())
if stalled {
waiting.Status, waiting.Reason = "degraded", "backlog_lag"
}
}
deferred := queuehealth.Status{Status: "ok", CapacityUnavailable: true, LagBasis: "deferred_checkpoint"}
for _, at := range q.deferred {
deferred.Depth++
deferred.LagSeconds = max(deferred.LagSeconds, now.Sub(at).Seconds())
}
return map[string]queuehealth.Status{"persist.waiting": waiting, "persist.active": active, "persist.deferred": deferred}
}
package incident
import (
"strings"
"github.com/pidginhost/csm/internal/alert"
)
// kindRank orders Kind values from weakest to strongest so the merge
// path can upgrade an incident's Kind when a stronger pattern appears
// later, but never downgrades. Higher number = stronger / more
// operator-attention.
var kindRank = map[Kind]int{
KindWebAttack: 0,
KindMailboxBruteforce: 0,
KindWebAccountCompromise: 1,
KindMailboxTakeover: 2,
KindPostExploitProcess: 3,
KindHostIntegrityRisk: 4,
KindCredentialSpray: 3,
KindHostTakeover: 5,
}
var compoundPostExploitWebChecks = map[string]struct{}{
"webshell": {},
"webshell_realtime": {},
"webshell_content_realtime": {},
"new_webshell_file": {},
"obfuscated_php": {},
"obfuscated_php_realtime": {},
"php_shield_webshell": {},
}
var compoundPostExploitNetworkChecks = map[string]struct{}{
"c2_connection": {},
"backdoor_port": {},
"backdoor_port_outbound": {},
}
// compoundHostPrivescUID0Checks, compoundHostPrivescSUIDChecks, and
// compoundHostPrivescBadASNChecks are the three host-takeover legs the
// takeover rule correlates: a new uid-0 account, a planted suid binary, and
// an outbound connection to a bad/unexpected ASN.
var compoundHostPrivescUID0Checks = map[string]struct{}{
"uid0_account": {},
}
var compoundHostPrivescSUIDChecks = map[string]struct{}{
"suid_binary": {},
}
var compoundHostPrivescBadASNChecks = map[string]struct{}{
"bad_asn_outbound": {},
}
// allCompoundFlagsSet reports whether every compound signal is already
// recorded, so the timeline hydrate loop can stop early.
func allCompoundFlagsSet(f CompoundFlags) bool {
return f.Webshell && f.C2 && f.UID0 && f.SUID && f.BadASNOutbound
}
// hostTakeoverLegs counts how many of the three distinct host-takeover legs
// an incident has observed. Two or more legs escalate to KindHostTakeover.
func hostTakeoverLegs(f CompoundFlags) int {
n := 0
if f.UID0 {
n++
}
if f.SUID {
n++
}
if f.BadASNOutbound {
n++
}
return n
}
// maybeReclassifyKind upgrades inc.Kind in place when the new finding
// classifies as a stronger Kind, or when the incident's sticky
// CompoundFlags plus the new finding cover a compound pattern that the
// per-finding classifier cannot see. Compound rules at this time:
// webshell + outbound C2 connection -> PostExploitProcess;
// uid0_account + suid_binary -> HostTakeover. Idempotent: calling with
// weaker findings is a no-op.
//
// CompoundFlags are mutated here so callers do not need a separate
// pass; they survive timeline trimming so an early webshell still
// arms the rule when a much later C2 finding arrives.
func maybeReclassifyKind(inc *Incident, f alert.Finding) {
if inc == nil {
return
}
// Hydrate sticky flags from the current timeline so incidents
// restored from bbolt (predating sticky flags) or built directly
// in tests still arm the compound rule. Timeline scan is bounded
// by maxIncidentTimeline so the cost is constant.
hydrateCompoundFlagsFromTimeline(&inc.CompoundFlags, inc.Timeline)
updateCompoundFlags(&inc.CompoundFlags, f.Check)
if newKind := ClassifyKind(f); kindRank[newKind] > kindRank[inc.Kind] && canPromoteKind(inc, f) {
inc.Kind = newKind
}
if kindRank[KindPostExploitProcess] > kindRank[inc.Kind] && inc.CompoundFlags.Webshell && inc.CompoundFlags.C2 && canPromoteKind(inc, f) {
inc.Kind = KindPostExploitProcess
}
// Host takeover: any two of the three distinct legs (new uid-0 account,
// planted suid binary, outbound connection to a bad ASN) on the same
// host inside the window.
if kindRank[KindHostTakeover] > kindRank[inc.Kind] && hostTakeoverLegs(inc.CompoundFlags) >= 2 && canPromoteKind(inc, f) {
inc.Kind = KindHostTakeover
}
}
func canPromoteKind(inc *Incident, f alert.Finding) bool {
if !remoteIPOnlyIncidentKey(inc.CorrelationKey) {
return true
}
k := KeyFor(f)
return !remoteIPOnlyKey(k)
}
func remoteIPOnlyIncidentKey(k *Key) bool {
if k == nil {
return false
}
return remoteIPOnlyKey(*k)
}
func remoteIPOnlyKey(k Key) bool {
return k.RemoteIP != "" && k.Host == "" && k.Account == "" && k.Domain == "" &&
k.Mailbox == "" && k.UID == 0 && k.PID == 0
}
// hydrateCompoundFlagsFromTimeline OR-merges timeline-derived signals
// into flags. Used as a one-shot migration for legacy/persisted
// incidents that have webshell or C2 events in their timeline but no
// CompoundFlags yet. Trimmed timelines may miss events, but anything
// still present remains a valid signal.
func hydrateCompoundFlagsFromTimeline(flags *CompoundFlags, events []IncidentEvent) {
if flags == nil || allCompoundFlagsSet(*flags) {
return
}
for _, ev := range events {
updateCompoundFlags(flags, ev.Check)
if allCompoundFlagsSet(*flags) {
return
}
}
}
// updateCompoundFlags sets sticky compound flags based on a Finding's
// check name. Once true a flag stays true so reclassify decisions are not
// silently disarmed by later trimming.
func updateCompoundFlags(flags *CompoundFlags, check string) {
if flags == nil {
return
}
check = strings.ToLower(strings.TrimSpace(check))
if _, ok := compoundPostExploitWebChecks[check]; ok {
flags.Webshell = true
}
if _, ok := compoundPostExploitNetworkChecks[check]; ok {
flags.C2 = true
}
if _, ok := compoundHostPrivescUID0Checks[check]; ok {
flags.UID0 = true
}
if _, ok := compoundHostPrivescSUIDChecks[check]; ok {
flags.SUID = true
}
if _, ok := compoundHostPrivescBadASNChecks[check]; ok {
flags.BadASNOutbound = true
}
}
package incident
import (
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
)
// SpraySuppressionConfig is the operator-tunable knob set for the
// credential-spray detector. Defaults are conservative: enabled=false and
// dry_run=true so the detector ships dark, increments counters, and only
// changes incident routing once an operator opts in.
type SpraySuppressionConfig struct {
Enabled bool
DryRun bool
DistinctMailboxes int
SeverityEscalateAt int
PerCheck map[string]bool
MaxTrackedIPs int
// BlockAtSeverity gates the firewall hand-off. Values:
// "" / unset - detection-only (legacy behavior).
// "high" - block on incident open (DistinctMailboxes trip).
// "critical" - block on severity escalation (SeverityEscalateAt).
// Comparison is case-insensitive. Any other value is ignored so an
// operator typo cannot accidentally engage blocking.
BlockAtSeverity string
// BlockExpiry is the operator's configured auto-response block duration,
// used as the first rung of the escalation ladder. Zero falls back to 24h.
BlockExpiry time.Duration
}
// IsZero reports whether the config is unset; the correlator treats a
// zero value as "spray detection disabled" without touching defaults.
func (c SpraySuppressionConfig) IsZero() bool {
return !c.Enabled && !c.DryRun && c.DistinctMailboxes == 0 && c.SeverityEscalateAt == 0 && len(c.PerCheck) == 0 && c.MaxTrackedIPs == 0 && c.BlockAtSeverity == ""
}
// sprayDecision is the verdict the detector returns to OnFinding for a
// candidate spray-class finding.
type sprayDecision int
const (
// sprayDecisionNone means the finding is not spray traffic; the
// caller should run the normal per-mailbox correlation path.
sprayDecisionNone sprayDecision = iota
// sprayDecisionOpen means this finding tipped the IP over the
// distinct-mailbox threshold; the caller must open a new
// credential_spray incident keyed on RemoteIP and report the id back
// to the detector via BindIncident.
sprayDecisionOpen
// sprayDecisionSuppress means an active spray incident already
// exists for the IP and the finding should be attached to it
// instead of opening a per-mailbox incident.
sprayDecisionSuppress
)
// ipSprayState tracks the distinct-mailbox set hit by a single source IP
// inside the merge window, plus the bound spray incident id once the
// threshold trips.
type ipSprayState struct {
mailboxes map[string]struct{}
firstSeen time.Time
lastSeen time.Time
// incident is the bound credential_spray incident id once the
// threshold has tripped. Empty until then.
incident string
}
// sprayDetector keeps a per-IP sliding window of distinct mailboxes hit
// by spray-class checks and decides whether a finding should open a new
// credential_spray super-incident, attach to an existing one, or fall
// through to the normal per-mailbox correlator path.
//
// Concurrency: the correlator's mutex serializes all calls into the
// detector. The detector itself does not take a separate lock so the
// "single mutex protects all correlator state" invariant holds.
type sprayDetector struct {
cfg SpraySuppressionConfig
window time.Duration
now func() time.Time
isWhitelisted func(string) bool
// perIP is the live state map. Bounded by cfg.MaxTrackedIPs; entries
// that fall outside the window during Decide are pruned in place,
// and once the map exceeds the cap the oldest-by-lastSeen entry is
// evicted before insert.
perIP map[string]*ipSprayState
}
// newSprayDetector returns a detector wired to the supplied config.
// Returns nil when cfg is zero-valued so the correlator can no-op the
// fast path on hosts that have not opted in.
func newSprayDetector(cfg SpraySuppressionConfig, window time.Duration, now func() time.Time, isWhitelisted func(string) bool) *sprayDetector {
if cfg.IsZero() {
return nil
}
if cfg.MaxTrackedIPs <= 0 {
cfg.MaxTrackedIPs = 10000
}
if cfg.DistinctMailboxes <= 0 {
cfg.DistinctMailboxes = 10
}
if cfg.SeverityEscalateAt <= cfg.DistinctMailboxes {
// Default to 5x threshold so a CRITICAL bump only fires on
// genuinely sustained sprays, not on the immediate trip.
cfg.SeverityEscalateAt = cfg.DistinctMailboxes * 5
}
if isWhitelisted == nil {
isWhitelisted = func(string) bool { return false }
}
if now == nil {
now = time.Now
}
return &sprayDetector{
cfg: cfg,
window: window,
now: now,
isWhitelisted: isWhitelisted,
perIP: make(map[string]*ipSprayState),
}
}
// Decide consumes a spray-candidate finding and returns the decision
// the correlator should apply. The detector mutates internal state
// (records the mailbox, tracks the lastSeen timestamp) regardless of
// the dry_run flag so counters and audit logs reflect what the live
// path would have done; only the returned decision is gated by
// dry_run. Caller must hold the correlator mutex.
//
// hitCount is the number of distinct mailboxes the IP has hit inside
// the window after this finding is recorded. The caller uses it to
// decide severity for sprayDecisionOpen and to escalate severity on
// merge-into-existing-spray.
func (d *sprayDetector) Decide(f alert.Finding) (decision sprayDecision, hitCount int) {
if d == nil || !d.cfg.Enabled && !d.cfg.DryRun {
return sprayDecisionNone, 0
}
ip := alert.AttackerAddress(f)
if ip == "" {
return sprayDecisionNone, 0
}
if !d.cfg.PerCheck[f.Check] {
return sprayDecisionNone, 0
}
if d.isWhitelisted(ip) {
return sprayDecisionNone, 0
}
targets := sprayTargets(f)
if len(targets) == 0 {
return sprayDecisionNone, 0
}
now := d.now()
state, ok := d.perIP[ip]
if ok {
// Window expiration: a state whose lastSeen fell outside the
// window is stale; reset before recording the new hit so a
// fresh attack does not inherit cold mailbox counts. Bound
// entries stay until the correlator closes or unbinds their
// incident; otherwise a quiet but still-open super-incident can
// lose suppression and duplicate on the next finding.
if state.incident == "" && now.Sub(state.lastSeen) > d.window {
state = nil
delete(d.perIP, ip)
}
}
if state == nil {
// Eviction: keep the live set bounded. Drop the oldest-by-lastSeen
// entry before inserting if we are at the cap. O(N) scan; cheap
// at the configured cap (10k) and only runs at insert time.
if len(d.perIP) >= d.cfg.MaxTrackedIPs {
d.evictOldestLocked()
}
state = &ipSprayState{
mailboxes: make(map[string]struct{}),
firstSeen: now,
}
d.perIP[ip] = state
}
for _, target := range targets {
state.mailboxes[target] = struct{}{}
}
state.lastSeen = now
hitCount = len(state.mailboxes)
// Bound to existing spray incident? Continue suppressing.
if state.incident != "" {
if d.cfg.DryRun {
return sprayDecisionNone, hitCount
}
return sprayDecisionSuppress, hitCount
}
if hitCount < d.cfg.DistinctMailboxes {
return sprayDecisionNone, hitCount
}
// Threshold tripped. Live mode opens a new spray incident; dry_run
// only counts the decision so the operator can observe the workload
// without changing routing.
if d.cfg.DryRun {
return sprayDecisionNone, hitCount
}
return sprayDecisionOpen, hitCount
}
// sprayTargets returns the identity dimension used for the distinct-target set:
// mailbox, tenant id, cPanel user, aggregate auth targets, then message text.
func sprayTargets(f alert.Finding) []string {
target, _ := canonicalizeMailboxDomain(f.Mailbox, f.Domain)
if target != "" {
return []string{target}
}
if f.TenantID != "" {
return []string{f.TenantID}
}
if f.CPUser != "" {
return []string{f.CPUser}
}
if targets := cleanSprayTargets(f.SprayTargets); len(targets) > 0 {
return targets
}
message := strings.TrimSpace(f.Message)
if message != "" {
return []string{message}
}
return nil
}
func cleanSprayTargets(targets []string) []string {
if len(targets) == 0 {
return nil
}
seen := make(map[string]struct{}, len(targets))
out := make([]string, 0, len(targets))
for _, target := range targets {
target = strings.TrimSpace(target)
if target == "" {
continue
}
if _, ok := seen[target]; ok {
continue
}
seen[target] = struct{}{}
out = append(out, target)
}
return out
}
// BindIncident records the spray incident id the correlator just
// created in response to sprayDecisionOpen. Subsequent findings from
// the same IP return sprayDecisionSuppress while the incident stays
// bound.
// Caller holds the correlator mutex.
func (d *sprayDetector) BindIncident(ip, id string) {
if d == nil {
return
}
if state, ok := d.perIP[ip]; ok {
state.incident = id
}
}
// Rehydrate seeds the perIP map at daemon startup so an open
// credential_spray incident restored from bbolt continues to suppress
// new per-mailbox fan-out instead of allowing a duplicate super-incident
// to open. The seeded state carries no per-mailbox set: the operator
// already saw the trip on the open incident, and the suppress path only
// reads state.incident. lastSeen is set so metrics and future unbound
// expiry have a stable anchor if the incident later closes.
// Caller holds the correlator mutex.
func (d *sprayDetector) Rehydrate(ip, id string, lastSeen time.Time) {
if d == nil || ip == "" || id == "" {
return
}
d.perIP[ip] = &ipSprayState{
mailboxes: make(map[string]struct{}),
firstSeen: lastSeen,
lastSeen: lastSeen,
incident: id,
}
}
// IncidentForIP returns the bound spray incident id for ip, or "" if no
// spray is currently bound. Used by Decide's suppress path to tell the
// caller which incident to merge the finding into.
func (d *sprayDetector) IncidentForIP(ip string) string {
if d == nil {
return ""
}
if state, ok := d.perIP[ip]; ok {
return state.incident
}
return ""
}
// UnbindIncident drops detector state for every IP currently bound to
// id, so a finding from one of those IPs cannot reach the closed or
// missing incident through the suppress merge path. Dropping the state
// also resets the distinct-mailbox threshold for future activity after
// an operator closes or dismisses the incident. Caller holds the
// correlator mutex. Linear scan over perIP, same shape as the byKey
// unbind in correlator.unbindLocked: the active set is bounded by
// MaxTrackedIPs.
func (d *sprayDetector) UnbindIncident(id string) {
if d == nil || id == "" {
return
}
for ip, state := range d.perIP {
if state.incident == id {
delete(d.perIP, ip)
}
}
}
// UnbindIncidents drops detector state for every IP bound to one of the
// supplied incident ids. Caller holds the correlator mutex.
func (d *sprayDetector) UnbindIncidents(ids []string) {
if d == nil || len(ids) == 0 {
return
}
if len(ids) == 1 {
d.UnbindIncident(ids[0])
return
}
idSet := make(map[string]struct{}, len(ids))
for _, id := range ids {
if id != "" {
idSet[id] = struct{}{}
}
}
if len(idSet) == 0 {
return
}
for ip, state := range d.perIP {
if _, ok := idSet[state.incident]; ok {
delete(d.perIP, ip)
}
}
}
// PruneStale clears entries whose lastSeen is older than the window.
// Called by the daemon retention loop alongside PruneStalePending so
// the detector does not grow without bound on hosts with churning
// attacker IPs. Entries still bound to an open spray incident are
// kept regardless of age -- evicting them lets a new spray finding
// open a duplicate per-mailbox incident while the super-incident is
// still active in the correlator.
func (d *sprayDetector) PruneStale(now time.Time) int {
if d == nil {
return 0
}
pruned := 0
for ip, state := range d.perIP {
if state.incident != "" {
continue
}
if now.Sub(state.lastSeen) > d.window {
delete(d.perIP, ip)
pruned++
}
}
return pruned
}
// TrackedIPs returns the count of source IPs currently held in the
// detector. Surfaced via the csm_credential_spray_tracked_ips gauge.
func (d *sprayDetector) TrackedIPs() int {
if d == nil {
return 0
}
return len(d.perIP)
}
// evictOldestLocked drops the perIP entry with the smallest lastSeen.
// Caller holds the correlator mutex. Linear scan over MaxTrackedIPs.
func (d *sprayDetector) evictOldestLocked() {
var oldestIP string
var oldestAt time.Time
first := true
for ip, state := range d.perIP {
if first || state.lastSeen.Before(oldestAt) {
oldestIP = ip
oldestAt = state.lastSeen
first = false
}
}
if oldestIP != "" {
delete(d.perIP, oldestIP)
}
}
// Counters for the credential-spray decisions live on the Correlator's
// existing `counters` struct (see correlator.go). Keeping them there
// means there is exactly one place that owns counter mutations, which
// keeps the metrics story coherent and avoids surprising operators
// who already grep for `csm_incidents_*`.
// Package incident groups related security findings into a single
// "story" with a timeline. Original findings are not mutated or
// suppressed; the Incident is layered on top so operators read one
// escalating object instead of stitching findings together by hand.
package incident
import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
)
// Status is the lifecycle position of an incident.
type Status string
const (
StatusOpen Status = "open"
StatusContained Status = "contained"
StatusResolved Status = "resolved"
StatusDismissed Status = "dismissed"
)
// ClosedRetention is how long resolved and dismissed incidents are kept after
// their last update. Operator applies to incidents an operator closed and to
// rows closed before close attribution existed; Auto applies to incidents the
// daemon closed on its own, which carry no operator decision worth keeping as
// long.
type ClosedRetention struct {
Operator time.Duration
Auto time.Duration
}
// Expired reports whether a closed incident has outlived its retention.
// Active incidents never expire here.
func (r ClosedRetention) Expired(inc Incident, now time.Time) bool {
if inc.Status != StatusResolved && inc.Status != StatusDismissed {
return false
}
keep := r.Operator
updated := inc.UpdatedAt
if strings.HasPrefix(inc.ClosedBy, closedByAutoPrefix) {
keep = r.Auto
// Older operator writers left automatic attribution on closed
// records. Preserve those decisions during the first upgraded sweep;
// a later automatic closure starts a new retention episode.
for i := len(inc.Actions) - 1; i >= 0; i-- {
action := inc.Actions[i]
if action.Action == "incident_auto_closed" {
break
}
if (action.Action == "incident_status_changed" || action.Action == "operator_block") && !action.Time.Before(inc.ClosedAt) {
keep = r.Operator
if action.Time.After(updated) {
updated = action.Time
}
}
}
}
return updated.Before(now.Add(-keep))
}
// closedByAutoPrefix marks ClosedBy values the daemon writes when it closes
// an incident itself ("auto:stale", "auto:age_cap", "auto:active_cap").
const closedByAutoPrefix = "auto:"
// Kind is the high-level taxonomy a correlator assigns at create time.
// Stable strings; downstream tooling pins on these.
type Kind string
const (
KindWebAccountCompromise Kind = "web_account_compromise"
// KindWebAttack is an inbound web attack: a WAF hit, scanner probe,
// or login brute-force from a remote source, plus remote-IP
// reputation/threat-score signals. When such a finding names a victim
// domain or account, that is the attack target, not evidence the
// account is compromised, so these correlate on the attacker source IP
// and get a short attacker-grade retention. Keeping them out of
// web_account_compromise stops defended inbound traffic from inflating
// the account-compromise count and the 7-day review window. Genuine
// compromise is recognised by on-disk and behavioural signals
// (webshell, suspicious PHP, post-exploit process), not by inbound hits.
KindWebAttack Kind = "web_attack"
// KindMailboxBruteforce is a failed-authentication brute-force attempt
// or pre-auth mail probe against one or more mailboxes from a remote
// source. A failed login is an attack attempt, not a takeover, so it
// correlates on the attacker source with short attacker-grade retention.
// Post-authentication abuse (outbound spam, cloud relay,
// compromised-account, suspicious geo) stays in mailbox_takeover.
KindMailboxBruteforce Kind = "mailbox_bruteforce"
KindMailboxTakeover Kind = "mailbox_takeover"
KindPostExploitProcess Kind = "post_exploit_process"
KindHostIntegrityRisk Kind = "host_integrity_risk"
// KindCredentialSpray collapses a single source IP that is brute-forcing
// many distinct mailboxes/accounts inside the merge window into one
// super-incident keyed on the source IP. Prevents the per-mailbox fan-out
// that turns one attacker into thousands of mailbox_bruteforce incidents.
KindCredentialSpray Kind = "credential_spray" // #nosec G101 -- taxonomy label, not a secret
// KindHostTakeover is the compound escalation when more than one
// host-privilege-escalation leg (a new uid-0 account, a planted suid
// binary, or bad-ASN outbound connection) is seen for the same host
// inside the merge window. It ranks above KindHostIntegrityRisk so a
// confirmed multi-leg takeover stands out from a single host-integrity
// finding.
KindHostTakeover Kind = "host_takeover"
)
// Incident is the wire shape every consumer (API, control socket,
// audit propagation) sees. omitempty fields are absent from JSON when
// zero so consumers ignore optional context cleanly.
type Incident struct {
ID string `json:"id"`
Kind Kind `json:"kind"`
Status Status `json:"status"`
Severity alert.Severity `json:"severity"`
Account string `json:"account,omitempty"`
Domain string `json:"domain,omitempty"`
Mailbox string `json:"mailbox,omitempty"`
CorrelationKey *Key `json:"correlation_key,omitempty"`
Summary string `json:"summary,omitempty"`
Confidence int `json:"confidence,omitempty"`
Findings []string `json:"findings,omitempty"`
Timeline []IncidentEvent `json:"timeline,omitempty"`
Actions []IncidentAction `json:"actions,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
// ClosedAt records the latest closure or operator decision on a closed
// incident. Reopening clears it.
ClosedAt time.Time `json:"closed_at,omitzero"`
// ClosedBy is "operator" for manual decisions on closed incidents and
// "auto:<reason>" for daemon closures. Empty for active or legacy rows.
ClosedBy string `json:"closed_by,omitempty"`
// CompoundFlags carries sticky bits used by the timeline-aware
// reclassifier. Once set, they survive timeline trimming so an
// early webshell or C2 signal still drives the compound rule when
// the matching counterpart arrives much later.
CompoundFlags CompoundFlags `json:"compound_flags,omitzero"`
// AutoBlock records what the automatic firewall hand-off already did for
// this incident. The block it applies expires; without this the marker
// saying "already blocked" did not, so an attack that outlasted its
// expiry was never blocked again.
AutoBlock AutoBlockState `json:"auto_block,omitzero"`
// RemoteIPEvidence records that a finding whose address is attacker
// evidence named the correlation key's remote IP. Like CompoundFlags it
// survives timeline trimming, which may drop that finding's event.
RemoteIPEvidence bool `json:"remote_ip_evidence,omitempty"`
// RemoteIPEvidenceFinding is the FindingID of the finding that set
// RemoteIPEvidence, so a block names its evidence after trimming.
RemoteIPEvidenceFinding string `json:"remote_ip_evidence_finding,omitempty"`
// The attesting policy identity is retained so restored proof can be
// revalidated even after its source event leaves the timeline.
RemoteIPEvidenceCheck string `json:"remote_ip_evidence_check,omitempty"`
RemoteIPEvidenceSeverity string `json:"remote_ip_evidence_severity,omitempty"`
}
// AutoBlockState is the escalation ladder's memory for one incident. Count is
// how many blocks the hand-off has requested, ExpiresAt when the most recent
// one lapses, and a zero ExpiresAt with a nonzero Count means that block is
// permanent and nothing re-requests it. Reset when the incident leaves an
// active status, so a later recurrence starts from the bottom of the ladder.
type AutoBlockState struct {
Count int `json:"count,omitempty"`
ExpiresAt time.Time `json:"expires_at,omitzero"`
LastAt time.Time `json:"last_at,omitzero"`
}
// lapsed reports whether the hand-off may request another block: never
// blocked, or the last block has expired. A permanent block never lapses.
func (s AutoBlockState) lapsed(now time.Time) bool {
if s.Count == 0 {
return true
}
if s.ExpiresAt.IsZero() {
return false
}
return !now.Before(s.ExpiresAt)
}
// blockTTLForAttempt escalates the hand-off: the first block uses the
// operator's configured expiry, the second a week, and any later one is
// permanent (the firewall reads a zero timeout as permanent). An attacker who
// outlasts one expiry pays more each time, while a single false positive
// still ages out on its own.
func blockTTLForAttempt(attempt int, configured time.Duration) time.Duration {
switch {
case attempt <= 1:
if configured <= 0 {
return 24 * time.Hour
}
return configured
case attempt == 2:
return 7 * 24 * time.Hour
default:
return 0
}
}
// CompoundFlags records the union of compound-pattern signals an
// Incident has ever observed. Fields are sticky once true; they are
// not derived from the (possibly trimmed) timeline so reclassify is
// not silently disarmed by head+tail eviction.
type CompoundFlags struct {
Webshell bool `json:"webshell,omitempty"`
C2 bool `json:"c2,omitempty"`
// UID0, SUID, and BadASNOutbound record the three host-takeover legs:
// a new uid-0 account, a planted suid binary, and an outbound connection
// to a bad/unexpected ASN. When any two are set on one incident the
// reclassifier escalates to KindHostTakeover.
UID0 bool `json:"uid0,omitempty"`
SUID bool `json:"suid,omitempty"`
BadASNOutbound bool `json:"bad_asn_outbound,omitempty"`
}
// MarshalJSON renders Severity as its uppercase string form
// ("HIGH", "CRITICAL", "WARNING") instead of the underlying int.
// alert.Severity is an int enum, so default marshaling would emit
// numbers; consumers (web UI, control socket, audit propagation)
// expect the same human-readable token already produced by
// audit_sink and webhook dispatch.
func (i Incident) MarshalJSON() ([]byte, error) {
type wireIncident Incident
return json.Marshal(struct {
wireIncident
Severity string `json:"severity"`
}{
wireIncident: wireIncident(i),
Severity: i.Severity.String(),
})
}
// UnmarshalJSON decodes the wire shape produced by MarshalJSON. Severity
// is read from its string form ("WARNING"/"HIGH"/"CRITICAL") and converted
// back to alert.Severity. Unknown strings return an error so SIEM-side
// schema drift is loud, not silent.
func (i *Incident) UnmarshalJSON(data []byte) error {
type wireIncident Incident
aux := struct {
*wireIncident
Severity string `json:"severity"`
}{wireIncident: (*wireIncident)(i)}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
if aux.Severity == "" {
// allow the zero-severity case for partial decodes (tests, partial
// JSON snippets in the API). Severity stays at zero value (Warning).
return nil
}
sev, ok := parseSeverity(aux.Severity)
if !ok {
return fmt.Errorf("incident: unknown severity %q", aux.Severity)
}
i.Severity = sev
return nil
}
// parseSeverity reads the token alert.Severity.String produces.
func parseSeverity(s string) (alert.Severity, bool) {
switch s {
case "WARNING":
return alert.Warning, true
case "HIGH":
return alert.High, true
case "CRITICAL":
return alert.Critical, true
}
return 0, false
}
// IncidentEvent is one entry in an incident's timeline. Built from a
// Finding when it joins the incident; carries enough context to
// render the timeline without re-reading the original record.
type IncidentEvent struct {
Time time.Time `json:"time"`
Kind string `json:"kind"`
Check string `json:"check,omitempty"`
Severity string `json:"severity,omitempty"`
Message string `json:"message"`
FindingID string `json:"finding_id,omitempty"`
PID int `json:"pid,omitempty"`
UID int `json:"uid,omitempty"`
Process string `json:"process,omitempty"`
Path string `json:"path,omitempty"`
RemoteIP string `json:"remote_ip,omitempty"`
// root retains the attesting finding's prepared evidence or mint
// refusal for the block it may lead to; never stored. It is a pointer
// because timelines are copied on every read: most events carry none.
root *PreparedRoot
}
// PreparedRoot retains the attesting finding's evidence, or its mint
// refusal with what the refusal counts of the finding, until a response is
// selected. Incident events keep it only in memory.
type PreparedRoot struct {
admission.Evidence
Finding alert.Finding
Err error
}
// IncidentAction is an automated or operator action that touched the
// incident. Appended to the timeline; surfaced separately so dashboards
// can filter by what the system did vs what it observed.
type IncidentAction struct {
Time time.Time `json:"time"`
Action string `json:"action"`
Result string `json:"result"`
Details string `json:"details,omitempty"`
}
package webserver
import (
"context"
"fmt"
"os/exec"
"time"
"github.com/pidginhost/csm/internal/platform"
)
// apacheHandler covers cPanel + plain Apache. cPanel ships `apachectl`
// pointing at the EasyApache binary; plain Apache on Debian/Ubuntu has
// `apache2ctl`. The selector picks the right one at construction.
type apacheHandler struct {
siteIncluder
snippetPath string
ctlBinary string // "apachectl" or "apache2ctl"
reloadAction []string
cmdRunner cmdRunner
}
func newApacheHandler(info platform.Info, r cmdRunner) *apacheHandler {
h := &apacheHandler{cmdRunner: r}
switch {
case info.IsCPanel():
// cPanel always runs EasyApache, conf.d is the canonical drop-in.
h.siteIncluder = newCPanelSites(r)
h.snippetPath = "/etc/apache2/conf.d/csm-challenge.conf"
h.ctlBinary = "apachectl"
h.reloadAction = []string{"apachectl", "graceful"}
case info.IsDebianFamily():
// Debian/Ubuntu use apache2 + conf-enabled.
h.snippetPath = "/etc/apache2/conf-enabled/csm-challenge.conf"
h.ctlBinary = "apache2ctl"
h.reloadAction = []string{"systemctl", "reload", "apache2"}
default:
// RHEL family without cPanel: httpd + /etc/httpd/conf.d.
h.snippetPath = "/etc/httpd/conf.d/csm-challenge.conf"
h.ctlBinary = "apachectl"
h.reloadAction = []string{"systemctl", "reload", "httpd"}
}
if h.siteIncluder == nil {
h.siteIncluder = &plainApacheSites{ctl: h.ctlBinary, runner: r}
}
return h
}
func (h *apacheHandler) Kind() string { return "apache" }
func (h *apacheHandler) SnippetPath() string { return h.snippetPath }
func (h *apacheHandler) Template() string { return apacheTemplate }
// PostInstallInstructions names the per-site step plain Apache needs: its
// site blocks never apply the server-wide snippet on their own.
func (h *apacheHandler) PostInstallInstructions() string {
if h.SiteConfigPath() != "" {
return ""
}
return "Apache applies the server-wide challenge rule inside no <VirtualHost> on its own.\n" +
"Add this line inside every <VirtualHost>, HTTP and HTTPS:\n\n" +
" Include " + plainApacheSiteSnippetPath + "\n\n" +
"Then run: " + h.ctlBinary + " configtest and reload Apache. `csm doctor challenge` warns while a site block lacks it."
}
func (h *apacheHandler) Validate() error {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
out, err := h.cmdRunner.Run(ctx, h.ctlBinary, "configtest")
if err != nil {
return fmt.Errorf("apache configtest failed: %v\n%s", err, out)
}
return nil
}
func (h *apacheHandler) Reload() error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
out, err := h.cmdRunner.Run(ctx, h.reloadAction[0], h.reloadAction[1:]...)
if err != nil {
return fmt.Errorf("apache reload failed: %v\n%s", err, out)
}
return nil
}
// cmdRunner is the injection seam tests use to mock exec. The real
// implementation in realCmdRunner just shells out via os/exec.
type cmdRunner interface {
Run(ctx context.Context, name string, args ...string) ([]byte, error)
}
type realCmdRunner struct{}
func (realCmdRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
// #nosec G204 -- name + args come from the installer's static handler
// definitions (apachectl/apache2ctl/nginx/lswsctrl + verbs). No
// user-controlled strings reach this path.
return exec.CommandContext(ctx, name, args...).CombinedOutput()
}
package webserver
import (
"fmt"
"os"
"strconv"
"strings"
)
// Each occurrence matters: one shared file can be loaded in several sites.
// Apache's dump supplies the parent line and indentation, including only
// includes whose conditions passed. Text alone cannot establish that.
type apacheInclude struct {
path string
line int
indent int
children []*apacheInclude
}
func parseApacheIncludes(dump string) (*apacheInclude, error) {
var root *apacheInclude
var stack []*apacheInclude
for _, raw := range strings.Split(dump, "\n") {
line := strings.TrimLeft(raw, " \t")
if !strings.HasPrefix(line, "(") {
continue
}
label, path, ok := strings.Cut(line, ") ")
path = strings.TrimSpace(path)
if !ok || !strings.HasPrefix(path, "/") {
return nil, fmt.Errorf("invalid Apache include record: %q", line)
}
node := &apacheInclude{path: path, indent: len(raw) - len(line)}
if label == "(*" {
if root != nil {
return nil, fmt.Errorf("multiple Apache include roots")
}
root = node
} else {
n, err := strconv.Atoi(strings.TrimPrefix(label, "("))
if err != nil || n <= 0 {
return nil, fmt.Errorf("invalid Apache include line: %q", label)
}
node.line = n
for len(stack) > 0 && stack[len(stack)-1].indent >= node.indent {
stack = stack[:len(stack)-1]
}
if len(stack) == 0 {
return nil, fmt.Errorf("apache include has no parent: %q", path)
}
parent := stack[len(stack)-1]
if nchildren := len(parent.children); nchildren > 0 && parent.children[nchildren-1].line > n {
return nil, fmt.Errorf("apache includes are out of order: %q", path)
}
parent.children = append(parent.children, node)
}
stack = append(stack, node)
}
if root == nil {
return nil, fmt.Errorf("apache did not report its loaded configuration files")
}
return root, nil
}
type apacheSiteCoverage struct {
inSite, included, missing bool
}
func (c *apacheSiteCoverage) visit(file *apacheInclude) error {
if file.path == plainApacheSiteSnippetPath {
if c.inSite {
c.included = true
}
return nil
}
// #nosec G304 -- a configuration file Apache itself reports loading.
data, err := os.ReadFile(file.path)
if err != nil {
return err
}
child := 0
var logical string
for i, line := range strings.Split(string(data), "\n") {
logical += strings.TrimSuffix(line, "\r")
if strings.HasSuffix(logical, "\\") {
logical = strings.TrimSuffix(logical, "\\")
continue
}
fields := strings.Fields(logical)
logical = ""
if len(fields) > 0 {
switch strings.ToLower(fields[0]) {
case "<virtualhost":
if c.inSite {
return fmt.Errorf("nested VirtualHost in %s", file.path)
}
c.inSite, c.included = true, false
case "</virtualhost>":
if !c.inSite {
return fmt.Errorf("unmatched VirtualHost end in %s", file.path)
}
c.missing = c.missing || !c.included
c.inSite = false
}
}
for child < len(file.children) && file.children[child].line <= i+1 {
if err := c.visit(file.children[child]); err != nil {
return err
}
child++
}
}
if child != len(file.children) || logical != "" {
return fmt.Errorf("apache configuration changed while checking %s", file.path)
}
return nil
}
// Package webserver auto-installs the CSM challenge webserver glue
// (Apache / LSWS / Nginx) with a write-validate-reload-or-revert flow.
// The operator runs `csm webserver-integration {install|upgrade|...}`;
// the package picks the right handler for the host and never reloads
// the webserver with a snippet that does not pass configtest.
package webserver
import (
"bytes"
"errors"
"fmt"
"io"
"net"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"text/template"
"github.com/pidginhost/csm/internal/challenge"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
)
// templateHeader is prepended to every rendered snippet so the
// installer can read the version back without parsing the body. The
// header is a single comment line whose value uniquely identifies
// "CSM owns this file". Manual edits that wipe or change the header
// trip ErrManualEdits at upgrade time and the file is left untouched.
const templateHeaderPrefix = "# csm-managed-version: "
// RenderConfig contains the daemon settings that have to be baked
// into webserver snippets. Values come from csm.yaml at install time
// where they are operator-configurable.
type RenderConfig struct {
ChallengeMapPath string
ChallengeNginxMapPath string
ChallengeListenAddr string
ChallengeListenGroup string
// ChallengePublicURL is the fully-qualified redirect target the
// webserver snippet emits. Required on install; empty value blocks
// the installer with a clear error. Operator typically points it at
// the host's TLS-valid control-panel domain on a CSM-owned port,
// e.g. https://server.example.com:8439/challenge.
ChallengePublicURL string
// ChallengeTrustedProxies is checked so the installer never wires
// redirects to a listener that trusts an address of this host.
ChallengeTrustedProxies []string
}
type templateData struct {
ChallengeMapPath string
ChallengeNginxMapPath string
ChallengePublicURL string
}
// ErrMissingPublicURL is returned by Install when challenge.public_url
// is empty. The webserver snippet has no fallback redirect target so
// the integration cannot function without it.
var ErrMissingPublicURL = errors.New("webserver integration: challenge.public_url is not set")
// ErrInvalidPublicURL is returned when challenge.public_url cannot be
// used as the direct browser redirect target.
var ErrInvalidPublicURL = errors.New("webserver integration: challenge.public_url must be an absolute http(s) URL ending in /challenge")
// ErrLoopbackPublicURL is returned when the installed snippets would
// redirect browsers to a TCP listener that only accepts loopback traffic.
var ErrLoopbackPublicURL = errors.New("webserver integration: challenge.listen_addr must be non-loopback when challenge.public_url redirects are enabled; for a proxy on this host use challenge.listen_addr: " + config.ChallengeSocketExample)
// Result is the structured outcome of an installer run. JSON-friendly
// shape so the CLI can render either human text or `--json` output.
type Result struct {
Action string `json:"action"` // "install" | "upgrade" | "remove" | "status" | "validate"
Status string `json:"status"` // "ok" | "no-op" | "skipped" | "fail"
Webserver string `json:"webserver"` // detected handler kind, "" if none
SnippetPath string `json:"snippet_path"` // "" if no handler
// SitePath is the per-site snippet on stacks whose HTTPS sites need it.
SitePath string `json:"site_snippet_path,omitempty"`
OnDiskVer int `json:"on_disk_version,omitempty"`
ShippedVer int `json:"shipped_version,omitempty"`
Message string `json:"message,omitempty"`
// FollowUp is the operator-facing setup the integration could not
// automate on this stack. Empty when install is fully automatic.
// The CLI prints it after the result block on successful install.
FollowUp string `json:"follow_up,omitempty"`
}
// Installer drives the install / upgrade / status / remove flow. All
// I/O goes through injected hooks so unit tests can run on darwin or
// against a temp tree without touching real webserver paths.
type Installer struct {
Handler Handler
Config RenderConfig
MkdirAll func(path string, mode os.FileMode) error
WriteAt func(path string, data []byte, mode os.FileMode) error
ReadAt func(path string) ([]byte, error)
StatAt func(path string) (os.FileInfo, error)
RemoveAt func(path string) error
Stderr io.Writer
// LegacySnippetPath is the pre-integration snippet Install retires
// in the same configtest and reload as its own write. Set only when
// the handler's configtest covers that file; empty elsewhere.
LegacySnippetPath string
// RebuildSiteIncludes lets Install regenerate the web server
// configuration once so site blocks read the per-site snippet. Only the
// explicit command sets it; the daemon never rebuilds on its own.
RebuildSiteIncludes bool
}
// LegacySnippetPath is where the package installer used to deploy a
// proxy-mode challenge snippet on cPanel Apache and LiteSpeed. It proxied
// challenged visitors to the loopback listener, which sees them all as
// loopback unless challenge.trusted_proxies lists it, and it doubled the
// integration snippet in the same directory.
const LegacySnippetPath = "/etc/apache2/conf.d/csm_challenge.conf"
// legacySnippetFor returns LegacySnippetPath when the handler's snippet
// shares its directory, so the handler's configtest and reload cover a
// change to the legacy file; otherwise "".
func legacySnippetFor(h Handler) string {
if filepath.Dir(h.SnippetPath()) == filepath.Dir(LegacySnippetPath) {
return LegacySnippetPath
}
return ""
}
// New returns an Installer wired for live operation: real filesystem
// reads, atomic writes, real exec runner. The handler is auto-selected
// from platform.Detect(); pass info to override for tests.
func New(info platform.Info, cfg *config.Config) (*Installer, error) {
h, err := pickHandler(info, realCmdRunner{})
if err != nil {
return nil, err
}
return &Installer{
Handler: h,
Config: renderConfigFrom(cfg),
MkdirAll: os.MkdirAll,
WriteAt: atomicWrite,
ReadAt: os.ReadFile,
StatAt: os.Stat,
RemoveAt: os.Remove,
Stderr: os.Stderr,
LegacySnippetPath: legacySnippetFor(h),
}, nil
}
// Install writes the snippet for the first time (or overwrites a
// stale one) with the safe rollback flow:
//
// 1. Stash existing bytes (or note absence).
// 2. Write new snippet atomically.
// 3. Validate via the webserver's own configtest.
// 4. On pass: reload + done.
// 5. On fail: restore previous bytes + return error.
//
// A legacy snippet beside it is removed before step 3, so one configtest
// and one reload cover both changes and a rollback restores both.
//
// Reload failure after a passing configtest is restored the same way,
// then a recovery reload is attempted so the host returns to the
// last-known-good state.
func (i *Installer) Install() (Result, error) {
res := Result{
Action: "install",
Webserver: i.Handler.Kind(),
SnippetPath: i.Handler.SnippetPath(),
ShippedVer: TemplateVersion,
}
if err := validateChallengePublicURL(i.Config); err != nil {
res.Status = "fail"
res.Message = err.Error()
return res, err
}
// Kept out of validateChallengePublicURL so doctor, which reports this
// in its own check, does not report it twice.
if err := config.ChallengeLocalTrustError(i.Config.ChallengeTrustedProxies); err != nil {
err = fmt.Errorf("webserver integration: %w", err)
res.Status = "fail"
res.Message = err.Error()
return res, err
}
prevBytes, prevExists, prevVer, err := i.readSnippet()
if err != nil && !errors.Is(err, os.ErrNotExist) {
res.Status = "fail"
res.Message = err.Error()
return res, err
}
res.OnDiskVer = prevVer
if prevExists && prevVer == 0 {
res.Status = "fail"
res.Message = ErrManualEdits.Error() + ": " + i.Handler.SnippetPath()
return res, ErrManualEdits
}
legacyBytes, legacyExists, err := i.readLegacySnippet()
if err != nil {
res.Status = "fail"
res.Message = err.Error()
return res, err
}
if mapErr := i.ensureChallengeMapFiles(); mapErr != nil {
res.Status = "fail"
res.Message = "runtime files: " + mapErr.Error()
return res, mapErr
}
rendered, rerr := i.renderTemplate()
if rerr != nil {
res.Status = "fail"
res.Message = "render: " + rerr.Error()
return res, rerr
}
site, err := i.prepareSite()
if err != nil {
res.Status = "fail"
res.Message = err.Error()
return res, err
}
if site != nil {
res.SitePath = site.path
}
// Only a stack with generated site configuration can be rebuilt; on
// others the operator adds the includes and the note says how.
canRebuild := site != nil && site.sites.SiteConfigPath() != ""
rebuild := canRebuild && i.RebuildSiteIncludes && !site.active
if canRebuild && i.RebuildSiteIncludes && site.activeErr != nil {
res.Status = "fail"
res.Message = "read site includes: " + site.activeErr.Error()
return res, site.activeErr
}
current := prevExists && bytes.Equal(prevBytes, rendered)
if current && !legacyExists && (site == nil || site.current) && !rebuild {
res.Status = "no-op"
res.Message = "snippet already current" + site.pendingNote()
return res, nil
}
// A second rebuild is not a rollback: it can fail or regenerate the same
// bad configuration. Save the exact active file before changing anything.
var configBefore []byte
var configMode os.FileMode
if rebuild {
configBefore, err = i.ReadAt(site.sites.SiteConfigPath())
if err == nil {
statAt := i.StatAt
if statAt == nil {
statAt = os.Stat
}
var info os.FileInfo
info, err = statAt(site.sites.SiteConfigPath())
if err == nil {
configMode = info.Mode().Perm()
}
}
if err != nil {
res.Status = "fail"
res.Message = "save site configuration: " + err.Error()
return res, err
}
}
if !current {
if err := i.WriteAt(i.Handler.SnippetPath(), rendered, 0o644); err != nil {
res.Status = "fail"
res.Message = "write: " + err.Error()
return res, err
}
}
siteWritten := false
legacyRemoved := false
rebuilt := false
rollback := func() error {
var errs []error
if !current {
errs = append(errs, i.restore(prevBytes, prevExists))
}
if siteWritten {
errs = append(errs, i.restoreAt(site.path, site.prevBytes, site.prevExists))
}
if legacyRemoved {
errs = append(errs, i.restoreLegacySnippet(legacyBytes))
}
if rebuilt {
if err := i.WriteAt(site.sites.SiteConfigPath(), configBefore, configMode); err != nil {
errs = append(errs, fmt.Errorf("rollback site configuration: %w", err))
}
}
return errors.Join(errs...)
}
if site != nil && !site.current {
err := i.mkdirAll(filepath.Dir(site.path), 0o755)
if err == nil {
err = i.WriteAt(site.path, site.rendered, 0o644)
}
if err != nil {
err = errors.Join(err, rollback())
res.Status = "fail"
res.Message = "write site snippet: " + err.Error()
return res, err
}
siteWritten = true
}
if legacyExists {
if err := i.RemoveAt(i.LegacySnippetPath); err != nil {
err = errors.Join(err, rollback())
res.Status = "fail"
res.Message = "delete legacy snippet: " + err.Error()
return res, err
}
legacyRemoved = true
}
if rebuild {
// Even a failed command may have replaced httpd.conf.
rebuilt = true
site.rebuilt = true
if err := site.sites.RebuildSiteIncludes(); err != nil {
err = errors.Join(err, rollback())
res.Status = "fail"
res.Message = err.Error()
return res, err
}
site.active, site.activeErr = site.sites.SiteIncludeActive()
}
if verr := i.Handler.Validate(); verr != nil {
verr = errors.Join(verr, rollback())
res.Status = "fail"
res.Message = "configtest: " + verr.Error()
return res, verr
}
// Plain Apache cannot dump loaded includes until a missing required
// snippet exists. Report the configuration that just passed configtest.
if siteWritten && !canRebuild {
site.active, site.activeErr = site.sites.SiteIncludeActive()
}
if rerr := i.Handler.Reload(); rerr != nil {
res.Status = "fail"
if rollbackErr := rollback(); rollbackErr != nil {
err := errors.Join(rerr, rollbackErr)
res.Message = "reload: " + err.Error()
return res, err
}
if rebuilt {
if err := i.Handler.Validate(); err != nil {
err = errors.Join(rerr, fmt.Errorf("rollback configtest: %w", err))
res.Message = err.Error()
return res, err
}
}
// Only reload after all changed files are back to their previous content.
if err := i.Handler.Reload(); err != nil {
err = errors.Join(rerr, fmt.Errorf("recovery reload: %w", err))
res.Message = err.Error()
return res, err
}
res.Message = "reload: " + rerr.Error() + " (rolled back)"
return res, rerr
}
res.Status = "ok"
switch {
case current:
res.Message = fmt.Sprintf("snippet already current (v%d)", TemplateVersion)
case prevExists:
res.Message = fmt.Sprintf("snippet upgraded v%d -> v%d", prevVer, TemplateVersion)
default:
res.Message = fmt.Sprintf("snippet installed (v%d)", TemplateVersion)
}
if legacyExists {
res.Message += "; legacy snippet " + i.LegacySnippetPath + " removed"
}
if siteWritten {
res.Message += "; site snippet " + site.path + " written"
}
if rebuilt {
res.Message += "; httpd.conf rebuilt"
}
res.Message += site.pendingNote()
res.FollowUp = i.Handler.PostInstallInstructions()
return res, nil
}
// siteChange is the per-site snippet Install manages beside the server one.
type siteChange struct {
sites siteIncluder
path string
prevBytes []byte
prevExists bool
rendered []byte
current bool
active bool
activeErr error
rebuilt bool
}
// pendingNote tells the operator when HTTPS sites still miss the rule. Safe
// on a nil receiver.
func (s *siteChange) pendingNote() string {
switch {
case s == nil || s.active:
return ""
case s.activeErr != nil:
return "; could not tell whether " + s.sites.SiteScope() + " read " + s.path + ": " + s.activeErr.Error()
case s.rebuilt:
return "; " + s.sites.SiteScope() + " still do not read " + s.path + " after the rebuild: check cPanel's Apache virtual host templates"
default:
return "; " + s.sites.SiteScope() + " are redirected only after they read " + s.path + ": " + s.sites.SiteFix()
}
}
// SiteGuidance names, on stacks with a per-site snippet, the site blocks
// that need it and what the operator does while they do not read it.
func (i *Installer) SiteGuidance() (scope, fix string) {
if sites := i.siteSnippets(); sites != nil {
return sites.SiteScope(), sites.SiteFix()
}
return "", ""
}
// SiteCoverage reports, on stacks with a per-site snippet, whether the site
// blocks read it. applicable is false on other stacks.
func (i *Installer) SiteCoverage() (applicable, active bool, path string, err error) {
sites := i.siteSnippets()
if sites == nil {
return false, false, "", nil
}
path = sites.SiteSnippetPath()
body, readErr := i.ReadAt(path)
if errors.Is(readErr, os.ErrNotExist) || bytes.Equal(body, []byte(disabledSiteSnippet)) {
return true, false, path, nil
}
if readErr != nil {
return true, false, path, readErr
}
active, err = sites.SiteIncludeActive()
return true, active && err == nil, path, err
}
// siteSnippets returns the handler's per-site include support, or nil.
func (i *Installer) siteSnippets() siteIncluder {
s, ok := i.Handler.(siteIncluder)
if !ok || s.SiteSnippetPath() == "" {
return nil
}
return s
}
// prepareSite reads and renders the per-site snippet, or returns nil when
// the handler has none.
func (i *Installer) prepareSite() (*siteChange, error) {
sites := i.siteSnippets()
if sites == nil {
return nil, nil
}
path := sites.SiteSnippetPath()
prev, exists, ver, err := i.readManaged(path)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
if exists && ver == 0 {
return nil, fmt.Errorf("%w: %s", ErrManualEdits, path)
}
rendered, err := i.renderBody(i.Handler.Kind()+"-site", sites.SiteTemplate())
if err != nil {
return nil, fmt.Errorf("render site snippet: %w", err)
}
active, activeErr := sites.SiteIncludeActive()
return &siteChange{
sites: sites,
path: path,
prevBytes: prev,
prevExists: exists,
rendered: rendered,
current: exists && bytes.Equal(prev, rendered),
active: active && activeErr == nil,
activeErr: activeErr,
}, nil
}
func (i *Installer) mkdirAll(path string, mode os.FileMode) error {
if i.MkdirAll == nil {
return os.MkdirAll(path, mode)
}
return i.MkdirAll(path, mode)
}
// Upgrade is an alias for Install with a more honest CLI verb. The
// underlying flow is the same: idempotent install + version compare.
func (i *Installer) Upgrade() (Result, error) {
res, err := i.Install()
res.Action = "upgrade"
return res, err
}
// Status returns the current integration state without writing
// anything. Used by post-upgrade hooks and operator-facing diagnostic
// commands to detect drift.
func (i *Installer) Status() (Result, error) {
res := Result{
Action: "status",
Webserver: i.Handler.Kind(),
SnippetPath: i.Handler.SnippetPath(),
ShippedVer: TemplateVersion,
}
_, exists, ver, err := i.readSnippet()
if err != nil && !errors.Is(err, os.ErrNotExist) {
res.Status = "fail"
res.Message = err.Error()
return res, err
}
res.OnDiskVer = ver
res.Status, res.Message = classifyStatus(exists, ver, TemplateVersion)
// A newer template must not become an automatic downgrade merely
// because the legacy file is present after a binary rollback.
if res.Status == "ok" && ver == TemplateVersion {
_, legacyExists, err := i.readLegacySnippet()
if err != nil {
res.Status = "fail"
res.Message = err.Error()
return res, err
}
if legacyExists {
res.Status = "stale"
res.Message = fmt.Sprintf("legacy snippet %s still present; run `csm webserver-integration upgrade`", i.LegacySnippetPath)
}
}
if sites := i.siteSnippets(); sites != nil && ver <= TemplateVersion && (res.Status == "ok" || res.Status == "stale") {
path := sites.SiteSnippetPath()
res.SitePath = path
_, exists, siteVer, err := i.readManaged(path)
if err != nil && !errors.Is(err, os.ErrNotExist) {
res.Status = "fail"
res.Message = err.Error()
return res, err
}
switch {
case siteVer > TemplateVersion:
res.Status = "ok"
res.Message = "site snippet is newer than this binary; leaving both snippets unchanged"
case !exists || (siteVer > 0 && siteVer < TemplateVersion):
res.Status = "stale"
res.Message = fmt.Sprintf("site snippet %s missing or old; run `csm webserver-integration upgrade`", path)
case siteVer == 0:
res.Status = "modified"
res.Message = ErrManualEdits.Error() + ": " + path
}
}
return res, nil
}
// classifyStatus is the pure-logic version compare extracted so the
// stale / modified / ok branches can be unit-tested without depending
// on the current TemplateVersion constant.
func classifyStatus(exists bool, onDisk, shipped int) (status, message string) {
switch {
case !exists:
return "missing", "no snippet installed; run `csm webserver-integration install`"
case onDisk == 0:
return "modified", ErrManualEdits.Error()
case onDisk < shipped:
return "stale", fmt.Sprintf("on-disk v%d < shipped v%d; run `csm webserver-integration upgrade`", onDisk, shipped)
default:
return "ok", fmt.Sprintf("snippet at v%d", onDisk)
}
}
// disabledSiteSnippet remains managed so a later install can replace it.
const disabledSiteSnippet = "# csm-managed-version: 1\n# Challenge redirects disabled; keep this file for cPanel's required include.\n"
// Remove disables the snippets, runs configtest to confirm the webserver
// is happy without it, and reloads. Mirrors the install rollback
// discipline: if removing the file makes configtest fail, restore the
// original and exit non-zero.
func (i *Installer) Remove() (Result, error) {
res := Result{
Action: "remove",
Webserver: i.Handler.Kind(),
SnippetPath: i.Handler.SnippetPath(),
ShippedVer: TemplateVersion,
}
prevBytes, prevExists, prevVer, err := i.readSnippet()
if err != nil && !errors.Is(err, os.ErrNotExist) {
res.Status = "fail"
res.Message = err.Error()
return res, err
}
res.OnDiskVer = prevVer
if prevExists && prevVer == 0 {
res.Status = "fail"
res.Message = ErrManualEdits.Error() + ": refusing to delete an operator-edited file"
return res, ErrManualEdits
}
var (
sitePath string
siteBytes []byte
siteExists bool
)
if sites := i.siteSnippets(); sites != nil {
sitePath = sites.SiteSnippetPath()
var siteVer int
siteBytes, siteExists, siteVer, err = i.readManaged(sitePath)
if err != nil && !errors.Is(err, os.ErrNotExist) {
res.Status = "fail"
res.Message = err.Error()
return res, err
}
if siteExists && siteVer == 0 {
res.Status = "fail"
res.Message = ErrManualEdits.Error() + ": refusing to delete an operator-edited file: " + sitePath
return res, ErrManualEdits
}
}
res.SitePath = sitePath
changeSite := siteExists && !bytes.Equal(siteBytes, []byte(disabledSiteSnippet))
if !prevExists && !changeSite {
res.Status = "no-op"
res.Message = "challenge redirects not installed"
return res, nil
}
serverRemoved, siteWritten := false, false
restoreAll := func() error {
var errs []error
if serverRemoved {
errs = append(errs, i.restore(prevBytes, true))
}
if siteWritten {
errs = append(errs, i.restoreAt(sitePath, siteBytes, true))
}
return errors.Join(errs...)
}
if prevExists {
if err := i.RemoveAt(i.Handler.SnippetPath()); err != nil {
res.Status = "fail"
res.Message = "delete: " + err.Error()
return res, err
}
serverRemoved = true
}
if changeSite {
// cPanel uses Include, not IncludeOptional. Keep a comment-only
// match so removing the last snippet cannot break server startup.
if err := i.WriteAt(sitePath, []byte(disabledSiteSnippet), 0o644); err != nil {
err = errors.Join(err, restoreAll())
res.Status = "fail"
res.Message = "disable site snippet: " + err.Error()
return res, err
}
siteWritten = true
}
if verr := i.Handler.Validate(); verr != nil {
verr = errors.Join(verr, restoreAll())
res.Status = "fail"
res.Message = "configtest after remove: " + verr.Error()
return res, verr
}
if rerr := i.Handler.Reload(); rerr != nil {
res.Status = "fail"
if err := restoreAll(); err != nil {
err = errors.Join(rerr, err)
res.Message = "reload: " + err.Error()
return res, err
}
if err := i.Handler.Validate(); err != nil {
err = errors.Join(rerr, fmt.Errorf("rollback configtest: %w", err))
res.Message = err.Error()
return res, err
}
if err := i.Handler.Reload(); err != nil {
err = errors.Join(rerr, fmt.Errorf("recovery reload: %w", err))
res.Message = err.Error()
return res, err
}
res.Message = "reload: " + rerr.Error() + " (rolled back)"
return res, rerr
}
res.Status = "ok"
res.Message = "snippet removed"
return res, nil
}
// Validate is a dry-run that exercises the webserver's own configtest
// against the current on-disk state. No writes, no reload.
func (i *Installer) Validate() (Result, error) {
res := Result{
Action: "validate",
Webserver: i.Handler.Kind(),
SnippetPath: i.Handler.SnippetPath(),
ShippedVer: TemplateVersion,
}
if err := i.Handler.Validate(); err != nil {
res.Status = "fail"
res.Message = err.Error()
return res, err
}
res.Status = "ok"
res.Message = "configtest passed"
return res, nil
}
// renderTemplate prefixes the handler's body with the version marker
// the installer reads back at status/upgrade time.
func (i *Installer) renderTemplate() ([]byte, error) {
return i.renderBody(i.Handler.Kind(), i.Handler.Template())
}
// renderBody renders one snippet template behind the version marker.
func (i *Installer) renderBody(name, body string) ([]byte, error) {
tpl, err := template.New(name).Parse(body)
if err != nil {
return nil, err
}
var b strings.Builder
b.WriteString(templateHeaderPrefix)
b.WriteString(strconv.Itoa(TemplateVersion))
b.WriteByte('\n')
if err := tpl.Execute(&b, i.templateData()); err != nil {
return nil, err
}
return []byte(b.String()), nil
}
func renderConfigFrom(cfg *config.Config) RenderConfig {
rc := RenderConfig{
ChallengeMapPath: challenge.DefaultMapPath,
ChallengeNginxMapPath: challenge.DefaultNginxMapPath,
ChallengeListenAddr: "127.0.0.1",
}
if cfg == nil {
return rc
}
rc.ChallengeListenAddr = strings.TrimSpace(cfg.Challenge.ListenAddr)
if rc.ChallengeListenAddr == "" {
rc.ChallengeListenAddr = "127.0.0.1"
}
rc.ChallengeListenGroup = strings.TrimSpace(cfg.Challenge.ListenGroup)
rc.ChallengePublicURL = strings.TrimSpace(cfg.Challenge.PublicURL)
rc.ChallengeTrustedProxies = cfg.Challenge.TrustedProxies
return rc
}
// RenderConfigFromConfig exposes the daemon-to-template projection for
// diagnostics that need to validate the same inputs before install.
func RenderConfigFromConfig(cfg *config.Config) RenderConfig {
return renderConfigFrom(cfg)
}
func (i *Installer) templateData() templateData {
mapPath := strings.TrimSpace(i.Config.ChallengeMapPath)
if mapPath == "" {
mapPath = challenge.DefaultMapPath
}
nginxMapPath := strings.TrimSpace(i.Config.ChallengeNginxMapPath)
if nginxMapPath == "" {
nginxMapPath = challenge.DefaultNginxMapPath
}
return templateData{
ChallengeMapPath: mapPath,
ChallengeNginxMapPath: nginxMapPath,
ChallengePublicURL: strings.TrimSpace(i.Config.ChallengePublicURL),
}
}
func (i *Installer) ensureChallengeMapFiles() error {
data := i.templateData()
mkdirAll := i.MkdirAll
if mkdirAll == nil {
mkdirAll = os.MkdirAll
}
for _, f := range []struct {
path string
body []byte
}{
{path: data.ChallengeMapPath, body: []byte("# Generated by CSM.\n")},
{path: data.ChallengeNginxMapPath, body: []byte("# Generated by CSM.\n")},
} {
if err := mkdirAll(filepath.Dir(f.path), 0o755); err != nil {
return err
}
statAt := i.StatAt
if statAt == nil {
statAt = os.Stat
}
if _, err := statAt(f.path); err == nil {
continue
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
if err := i.WriteAt(f.path, f.body, 0o644); err != nil {
return err
}
}
return nil
}
func validateChallengePublicURL(rc RenderConfig) error {
raw := strings.TrimSpace(rc.ChallengePublicURL)
if raw == "" {
return ErrMissingPublicURL
}
u, err := url.Parse(raw)
if err != nil || !u.IsAbs() || u.Host == "" {
return fmt.Errorf("%w: %q", ErrInvalidPublicURL, raw)
}
if u.Scheme != "http" && u.Scheme != "https" {
return fmt.Errorf("%w: unsupported scheme %q", ErrInvalidPublicURL, u.Scheme)
}
if u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path != "/challenge" {
return fmt.Errorf("%w: %q", ErrInvalidPublicURL, raw)
}
if isLoopbackHost(u.Hostname()) {
return fmt.Errorf("%w: host %q is loopback", ErrInvalidPublicURL, u.Hostname())
}
// The installer loads config without running config.Validate. Check
// socket settings before accepting redirects to a listener that cannot
// start, and before the "unix:" prefix can be parsed as host:port.
if path, socket := config.ChallengeSocketPath(rc.ChallengeListenAddr); socket {
if err := config.ChallengeSocketPathError(path); err != nil {
return fmt.Errorf("webserver integration: %w", err)
}
if strings.TrimSpace(rc.ChallengeListenGroup) == "" {
return errors.New("webserver integration: challenge.listen_group must name the group your proxy runs as when challenge.listen_addr is a unix socket")
}
return nil
}
if _, _, err := net.SplitHostPort(strings.TrimSpace(rc.ChallengeListenAddr)); err == nil {
return errors.New("webserver integration: challenge.listen_addr must contain only a host; set the port in challenge.listen_port")
}
if isLoopbackListenAddr(rc.ChallengeListenAddr) {
return ErrLoopbackPublicURL
}
return nil
}
// ValidateChallengePublicURL applies the same direct-redirect validation used
// by Install so diagnostics and tests do not drift from installer behavior.
func ValidateChallengePublicURL(rc RenderConfig) error {
return validateChallengePublicURL(rc)
}
func isLoopbackListenAddr(addr string) bool {
return isLoopbackHost(challenge.ListenHost(addr))
}
func isLoopbackHost(host string) bool {
host = strings.TrimSpace(strings.Trim(host, "[]"))
if strings.EqualFold(host, "localhost") {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
// readSnippet parses the on-disk snippet header to recover the
// embedded version. Returns the raw bytes, a presence flag, and the
// parsed version (zero when the file exists but lacks the marker).
func (i *Installer) readSnippet() ([]byte, bool, int, error) {
return i.readManaged(i.Handler.SnippetPath())
}
// readManaged reads a managed snippet at path like readSnippet does.
func (i *Installer) readManaged(path string) ([]byte, bool, int, error) {
data, err := i.ReadAt(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, false, 0, err
}
return nil, false, 0, err
}
ver := parseHeaderVersion(data)
return data, true, ver, nil
}
// restore writes the previous bytes back (or removes the new file if
// there were none) after a failed validate/reload step. Best-effort:
// I/O errors are also returned so Install can refuse a recovery reload
// when either part of the configuration could not be restored.
func (i *Installer) restore(prevBytes []byte, prevExists bool) error {
return i.restoreAt(i.Handler.SnippetPath(), prevBytes, prevExists)
}
// restoreAt puts path back to its previous state like restore does.
func (i *Installer) restoreAt(path string, prevBytes []byte, prevExists bool) error {
if !prevExists {
if err := i.RemoveAt(path); err != nil && !errors.Is(err, os.ErrNotExist) {
fmt.Fprintf(i.Stderr, "webserver integration: rollback delete failed: %v\n", err)
return fmt.Errorf("rollback delete: %w", err)
}
return nil
}
if err := i.WriteAt(path, prevBytes, 0o644); err != nil {
fmt.Fprintf(i.Stderr, "webserver integration: rollback write failed: %v\n", err)
return fmt.Errorf("rollback write: %w", err)
}
return nil
}
// readLegacySnippet returns the legacy snippet's bytes and whether it
// exists. Always absent when the handler cannot validate its removal.
func (i *Installer) readLegacySnippet() ([]byte, bool, error) {
if i.LegacySnippetPath == "" {
return nil, false, nil
}
data, err := i.ReadAt(i.LegacySnippetPath)
if errors.Is(err, os.ErrNotExist) {
return nil, false, nil
}
if err != nil {
return nil, false, fmt.Errorf("read legacy snippet: %w", err)
}
return data, true, nil
}
// restoreLegacySnippet puts the legacy snippet back after a failed
// validate/reload step, reporting I/O errors like restore does.
func (i *Installer) restoreLegacySnippet(data []byte) error {
if err := i.WriteAt(i.LegacySnippetPath, data, 0o644); err != nil {
fmt.Fprintf(i.Stderr, "webserver integration: rollback of legacy snippet %s failed: %v\n", i.LegacySnippetPath, err)
return fmt.Errorf("rollback legacy snippet %s: %w", i.LegacySnippetPath, err)
}
return nil
}
func parseHeaderVersion(data []byte) int {
scanner := bytes.SplitN(data, []byte("\n"), 2)
if len(scanner) == 0 {
return 0
}
line := strings.TrimSpace(string(scanner[0]))
if !strings.HasPrefix(line, templateHeaderPrefix) {
return 0
}
rest := strings.TrimSpace(strings.TrimPrefix(line, templateHeaderPrefix))
v, err := strconv.Atoi(rest)
if err != nil {
return 0
}
return v
}
func pickHandler(info platform.Info, r cmdRunner) (Handler, error) {
switch info.WebServer {
case platform.WSApache:
return newApacheHandler(info, r), nil
case platform.WSLiteSpeed:
return newLSWSHandler(info, r), nil
case platform.WSNginx:
return newNginxHandler(r), nil
default:
return nil, ErrUnknownWebserver
}
}
// atomicWrite writes data to a sibling temp file then renames it into
// place so the webserver never sees a half-written snippet. fsync on
// the directory is best-effort; rename + fsync on the file before
// rename gives crash safety on every common Linux filesystem.
func atomicWrite(path string, data []byte, mode os.FileMode) error {
dir := filepath.Dir(path)
tmp, err := os.CreateTemp(dir, ".csm-ws-install-*")
if err != nil {
return err
}
tmpName := tmp.Name()
cleanup := func() { _ = os.Remove(tmpName) }
if _, werr := tmp.Write(data); werr != nil {
_ = tmp.Close()
cleanup()
return werr
}
if serr := tmp.Sync(); serr != nil {
_ = tmp.Close()
cleanup()
return serr
}
if cerr := tmp.Close(); cerr != nil {
cleanup()
return cerr
}
if merr := os.Chmod(tmpName, mode); merr != nil {
cleanup()
return merr
}
return os.Rename(tmpName, path)
}
package webserver
import (
"context"
"fmt"
"time"
"github.com/pidginhost/csm/internal/platform"
)
// lswsHandler manages the LiteSpeed integration. The right snippet path
// depends on how LSWS is wired:
//
// - cPanel + LSWS: LSWS runs with <loadApacheConf>1</loadApacheConf>
// and reads cPanel's Apache config tree. The snippet drops at
// /etc/apache2/conf.d/csm-challenge.conf, same as plain Apache,
// and LSWS picks it up automatically.
//
// - Plain LSWS (no cPanel): the operator runs LSWS in native mode
// with /usr/local/lsws/conf/httpd_config.xml. There is no auto-
// include dir for text-style rewrite rules; the snippet goes in
// /usr/local/lsws/conf/templates/ and the operator must include
// it manually via the LSWS WebAdmin Console -> Server -> General
// -> Rewrite -> External Rewrite Rules. The installer writes the
// file but emits a stderr note pointing at the manual step.
type lswsHandler struct {
*cpanelSites
cmdRunner cmdRunner
cpanel bool
}
func newLSWSHandler(info platform.Info, r cmdRunner) *lswsHandler {
h := &lswsHandler{cmdRunner: r, cpanel: info.IsCPanel()}
if h.cpanel {
h.cpanelSites = newCPanelSites(r)
}
return h
}
func (h *lswsHandler) Kind() string { return "lsws" }
func (h *lswsHandler) SnippetPath() string {
if h.cpanel {
return "/etc/apache2/conf.d/csm-challenge.conf"
}
return "/usr/local/lsws/conf/templates/csm-challenge.conf"
}
func (h *lswsHandler) Template() string { return lswsTemplate }
func (h *lswsHandler) Validate() error {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
// LSWS has no `lswsctrl conftest` verb (the documented surface is
// start|stop|restart|reload|condrestart|try-restart|status). The
// equivalent configtest is `lshttpd -t`, which parses the active
// config and exits non-zero on syntax errors without touching the
// running listener.
out, err := h.cmdRunner.Run(ctx, "/usr/local/lsws/bin/lshttpd", "-t")
if err != nil {
return fmt.Errorf("lsws lshttpd -t failed: %v\n%s", err, out)
}
return nil
}
// PostInstallInstructions returns operator-facing follow-up after a
// successful install. v4 of the snippet redirects challenged IPs to
// challenge.public_url directly; no LSWS External App is required, so
// the previous WebAdmin Console steps are obsolete.
func (h *lswsHandler) PostInstallInstructions() string { return "" }
func (h *lswsHandler) Reload() error {
// LSWS does not have a graceful reload equivalent; `restart` is the
// supported way to pick up new config without dropping established
// listener sockets (LSWS's internal supervisor handles the hand-
// off). The full-restart path is what the operator's own toolchain
// invokes on config change too.
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
out, err := h.cmdRunner.Run(ctx, "/usr/local/lsws/bin/lswsctrl", "restart")
if err != nil {
return fmt.Errorf("lsws restart failed: %v\n%s", err, out)
}
return nil
}
package webserver
import (
"context"
"fmt"
"os/exec"
"time"
"github.com/pidginhost/csm/internal/systemdrun"
)
// nginxHandler manages the nginx integration. The snippet lands in
// /etc/nginx/conf.d/ where stock nginx auto-includes everything via
// the default http{} include glob. Validation uses `nginx -t`, run
// outside the service sandbox; reload uses `systemctl reload nginx` so
// existing connections drain gracefully.
type nginxHandler struct {
cmdRunner cmdRunner
lookPath systemdrun.LookPathFunc
}
func newNginxHandler(r cmdRunner) *nginxHandler {
return &nginxHandler{cmdRunner: r, lookPath: exec.LookPath}
}
func (h *nginxHandler) Kind() string { return "nginx" }
func (h *nginxHandler) SnippetPath() string { return "/etc/nginx/conf.d/csm-challenge.conf" }
func (h *nginxHandler) Template() string { return nginxTemplate }
// PostInstallInstructions reminds the operator that the http{}
// snippet only ships the shared map; each guarded server{} block has
// to opt in with a one-line if-redirect. Nginx cannot apply this
// safely from an http{} drop-in.
func (h *nginxHandler) PostInstallInstructions() string {
return `Nginx detected. Per-server opt-in required so http{} drop-ins do
not blind-redirect already-protected hosts. For each server{} that
should respect the challenge:
server {
...
if ($csm_challenged) {
return 302 <challenge.public_url>?dest=$scheme://$host$request_uri;
}
}
Then run: nginx -t && systemctl reload nginx`
}
const nginxConfigtestTimeout = 15 * time.Second
func (h *nginxHandler) Validate() error {
// The availability probe must not consume nginx's configtest budget.
run := func(ctx context.Context, name string, args ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(ctx, nginxConfigtestTimeout)
defer cancel()
return h.cmdRunner.Run(ctx, name, args...)
}
// nginx -t opens its log files for writing, which the
// daemon's read-only service sandbox refuses.
out, err := systemdrun.Run(context.Background(), h.lookPath, run,
systemdrun.Options{Pipe: true, RuntimeMax: nginxConfigtestTimeout}, "nginx", "-t")
if err != nil {
return fmt.Errorf("nginx configtest failed: %v\n%s", err, out)
}
return nil
}
func (h *nginxHandler) Reload() error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
out, err := h.cmdRunner.Run(ctx, "systemctl", "reload", "nginx")
if err != nil {
return fmt.Errorf("nginx reload failed: %v\n%s", err, out)
}
return nil
}
package webserver
import (
"context"
"fmt"
"os"
"strings"
"time"
)
// siteIncluder is a handler whose site blocks do not inherit the
// server-scope snippet and read the rule from a per-site include instead.
// SiteSnippetPath is empty when the host has no such include.
type siteIncluder interface {
SiteSnippetPath() string
SiteTemplate() string
// SiteConfigPath is the generated configuration changed by a rebuild,
// or empty where the operator adds the includes and nothing rebuilds.
SiteConfigPath() string
// SiteScope names the site blocks that need the include.
SiteScope() string
// SiteFix is what the operator does while they do not read it.
SiteFix() string
// SiteIncludeActive reports whether the site blocks read the include.
SiteIncludeActive() (bool, error)
// RebuildSiteIncludes regenerates the web server configuration so its
// site blocks read the include.
RebuildSiteIncludes() error
}
const (
cpanelSiteSnippetPath = "/etc/apache2/conf.d/userdata/csm-challenge.conf"
cpanelSiteInclude = `Include "/etc/apache2/conf.d/userdata/*.conf"`
cpanelHTTPDConf = "/etc/apache2/conf/httpd.conf"
cpanelRebuildHTTPD = "/usr/local/cpanel/scripts/rebuildhttpdconf"
)
// cpanelSites covers cPanel's site blocks, under Apache or LiteSpeed.
// Port-80 blocks inherit the server-scope snippet; HTTPS blocks do not, so
// the rule also goes in the per-site include. cPanel adds that include to
// every block only when it regenerates httpd.conf with a file present.
// SiteSnippetPath returns an empty path on a nil receiver to opt out.
type cpanelSites struct {
httpdConf string
runner cmdRunner
}
func newCPanelSites(r cmdRunner) *cpanelSites {
return &cpanelSites{httpdConf: cpanelHTTPDConf, runner: r}
}
func (c *cpanelSites) SiteSnippetPath() string {
if c == nil {
return ""
}
return cpanelSiteSnippetPath
}
func (c *cpanelSites) SiteTemplate() string { return siteTemplate }
func (c *cpanelSites) SiteConfigPath() string { return c.httpdConf }
func (c *cpanelSites) SiteScope() string { return "HTTPS sites" }
func (c *cpanelSites) SiteFix() string { return "run `csm webserver-integration upgrade`" }
func (c *cpanelSites) SiteIncludeActive() (bool, error) {
// #nosec G304 -- fixed cPanel httpd.conf path, or a test's temp file.
data, err := os.ReadFile(c.httpdConf)
if err != nil {
return false, err
}
return cpanelHTTPSIncludes(string(data)), nil
}
// cpanelUserSiteMarker is the per-user include path cPanel writes into every
// user site block, active or as a commented example. The server's own
// hostname blocks carry none and never get the global include, so they do
// not count against coverage.
const cpanelUserSiteMarker = "/etc/apache2/conf.d/userdata/ssl/"
// cPanel emits its sites inline. A global include, a commented example, or
// an HTTP-only include does not establish coverage of the HTTPS sites.
func cpanelHTTPSIncludes(conf string) bool {
inSite, tls, userSite, included := false, false, false, false
for _, line := range strings.Split(conf, "\n") {
if inSite && strings.Contains(line, cpanelUserSiteMarker) {
userSite = true
}
fields := strings.Fields(line)
if len(fields) == 0 || strings.HasPrefix(fields[0], "#") {
continue
}
switch strings.ToLower(fields[0]) {
case "<virtualhost":
inSite, tls, userSite, included = true, false, false, false
for _, addr := range fields[1:] {
if strings.HasSuffix(strings.TrimSuffix(addr, ">"), ":443") {
tls = true
}
}
case "</virtualhost>":
if inSite && tls && userSite {
if !included {
return false
}
}
inSite = false
case "sslengine":
if inSite && len(fields) == 2 && strings.EqualFold(fields[1], "on") {
tls = true
}
case "include", "includeoptional":
if inSite && len(fields) == 2 && strings.Trim(fields[1], "\"'") == "/etc/apache2/conf.d/userdata/*.conf" {
included = true
}
}
}
// No user HTTPS site means nothing to cover yet: a site added later gets
// the include when cPanel builds its block with the file present.
return !inSite
}
func (c *cpanelSites) RebuildSiteIncludes() error {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
out, err := c.runner.Run(ctx, cpanelRebuildHTTPD)
if err != nil {
return fmt.Errorf("rebuild httpd.conf: %v\n%s", err, out)
}
return nil
}
// plainApacheSiteSnippetPath lives beside the challenge maps: the daemon may
// write there, it survives daemon stops, and Apache never loads it on its
// own, only through the Include line each site block carries.
const plainApacheSiteSnippetPath = "/var/cache/csm/apache-challenge-site.conf"
// plainApacheSites covers Apache without cPanel. No site block inherits the
// server-wide snippet and CSM does not edit operator site files, so the
// operator adds the Include line to each <VirtualHost>.
type plainApacheSites struct {
ctl string
runner cmdRunner
}
func (p *plainApacheSites) SiteSnippetPath() string { return plainApacheSiteSnippetPath }
func (p *plainApacheSites) SiteTemplate() string { return siteTemplate }
func (p *plainApacheSites) SiteConfigPath() string { return "" }
func (p *plainApacheSites) SiteScope() string { return "Apache sites" }
func (p *plainApacheSites) SiteFix() string {
return "add `Include " + plainApacheSiteSnippetPath + "` inside every <VirtualHost>, then reload Apache"
}
func (p *plainApacheSites) RebuildSiteIncludes() error {
return fmt.Errorf("apache site blocks need `Include %s` added by the operator", plainApacheSiteSnippetPath)
}
// SiteIncludeActive reads every file Apache loads and requires each
// <VirtualHost> in them to include the snippet. A server with no site block
// has nothing to cover.
func (p *plainApacheSites) SiteIncludeActive() (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
out, err := p.runner.Run(ctx, p.ctl, "-t", "-D", "DUMP_INCLUDES")
if err != nil {
return false, fmt.Errorf("%s -t -D DUMP_INCLUDES: %v\n%s", p.ctl, err, out)
}
root, err := parseApacheIncludes(string(out))
if err != nil {
return false, err
}
coverage := apacheSiteCoverage{}
if err := coverage.visit(root); err != nil {
return false, err
}
return !coverage.inSite && !coverage.missing, nil
}
package integrity
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"github.com/pidginhost/csm/internal/config"
)
// The per-fragment record sits beside the main config instead of inside its
// integrity block: the config loader rejects unknown keys, so a new key there
// would stop an older binary from starting after a downgrade.
const fragmentRecordSuffix = ".confd-fragments.json"
// ErrNoFragmentBaseline means no per-fragment record matches the signed
// conf.d state, so a scoped rehash cannot tell which fragments changed. A
// full `csm rehash` signs the current state and records one.
var ErrNoFragmentBaseline = errors.New("no per-fragment conf.d record matches the recorded state")
// UnnamedChangesError refuses a scoped rehash because something other than
// the named fragments changed since the last signed state.
type UnnamedChangesError struct {
Changed []string
}
func (e *UnnamedChangesError) Error() string {
return "changed besides the named fragments: " + strings.Join(e.Changed, ", ")
}
// The original bytes are needed to recompute the signed aggregate digest:
// independent per-file hashes cannot prove membership in that digest. Like
// the main config and its backup, this record can contain credentials and
// must stay private to the daemon user.
type fragmentRecord struct {
ConfdHash string `json:"confd_hash"`
Fragments map[string][]byte `json:"fragments"`
}
// FragmentRecordPath is where the per-fragment record for configPath lives:
// beside the resolved main config, so a legacy symlinked path and its target
// share one record.
func FragmentRecordPath(configPath string) (string, error) {
target, err := atomicWriteTarget(configPath)
if err != nil {
return "", err
}
return target + fragmentRecordSuffix, nil
}
func writeFragmentRecord(configPath, confdHash string, contents map[string][]byte) error {
path, err := FragmentRecordPath(configPath)
if err != nil {
return err
}
data, err := json.Marshal(fragmentRecord{ConfdHash: confdHash, Fragments: contents})
if err != nil {
return fmt.Errorf("encoding conf.d fragment record: %w", err)
}
if err := atomicWriteFile(path, data, 0o600); err != nil {
return fmt.Errorf("writing conf.d fragment record: %w", err)
}
return nil
}
func readFragmentRecord(configPath string) (fragmentRecord, error) {
var rec fragmentRecord
path, err := FragmentRecordPath(configPath)
if err != nil {
return rec, err
}
data, err := os.ReadFile(path) // #nosec G304 -- derived from the operator-configured config path.
if errors.Is(err, os.ErrNotExist) {
return rec, ErrNoFragmentBaseline
}
if err != nil {
return rec, fmt.Errorf("reading conf.d fragment record: %w", err)
}
rec, err = decodeFragmentRecord(data)
if err != nil {
return rec, fmt.Errorf("decoding conf.d fragment record %s: %w", path, err)
}
return rec, nil
}
func decodeFragmentRecord(data []byte) (fragmentRecord, error) {
var wire struct {
ConfdHash *string `json:"confd_hash"`
Fragments map[string][]byte `json:"fragments"`
}
if err := json.Unmarshal(data, &wire); err != nil {
return fragmentRecord{}, err
}
if wire.ConfdHash == nil {
return fragmentRecord{}, errors.New("missing conf.d digest")
}
rec := fragmentRecord{ConfdHash: *wire.ConfdHash, Fragments: wire.Fragments}
return rec, rec.validate()
}
func (rec fragmentRecord) validate() error {
if rec.Fragments == nil {
return errors.New("missing fragments")
}
frags := make([]config.ConfDirFragment, 0, len(rec.Fragments))
for _, name := range fragmentNames(rec.Fragments, nil) {
if !validFragmentName(name) || len(rec.Fragments[name]) == 0 {
return fmt.Errorf("invalid fragment %q", name)
}
frags = append(frags, config.ConfDirFragment{Name: name, Data: rec.Fragments[name]})
}
digest, _ := confdSnapshot(frags, nil)
if digest != rec.ConfdHash {
return errors.New("fragment contents do not match the recorded digest")
}
return nil
}
// RecordVerifiedConfD records the covered bytes of a conf.d state the
// caller just verified, so hosts signed before records existed get one
// without a rehash. It re-reads the fragments and records nothing unless they
// still hash to the signed digest. A config that was never signed has
// nothing verified to record.
func RecordVerifiedConfD(cfg *config.Config) error {
if cfg.Integrity.BinaryHash == "" || cfg.Integrity.ConfigHash == "" {
return nil
}
frags, err := config.ConfDirFragmentDigestInput(cfg.ConfigDir)
if err != nil {
return fmt.Errorf("hashing conf.d: %w", err)
}
digest, contents := confdSnapshot(frags, cfg.ConfD.IntegrityExempt)
if digest != cfg.Integrity.ConfdHash {
return fmt.Errorf("%w: conf.d changed after verification", ErrConfdHashMismatch)
}
if rec, readErr := readFragmentRecord(cfg.ConfigFile); readErr == nil && rec.ConfdHash == digest {
return nil
}
return writeFragmentRecord(cfg.ConfigFile, digest, contents)
}
// RehashFragments re-signs conf.d after an integration rewrote the named
// fragments (bare file names in confDir). It refuses with an
// UnnamedChangesError when anything else differs from the last signed state:
// the binary, the main config, or any other covered fragment added, removed
// or edited. A named fragment may be new, edited or deleted; one that exists
// neither on disk nor in the record is an error. Fragments under
// confd.integrity_exempt stay outside the signed state either way. Without a
// record matching the signed state it returns ErrNoFragmentBaseline.
func RehashFragments(binaryPath, configPath, confDir string, names []string) error {
if len(names) == 0 {
return errors.New("name at least one conf.d fragment")
}
named := make(map[string]bool, len(names))
for _, name := range names {
if !validFragmentName(name) {
return fmt.Errorf("invalid fragment name %q: give the bare .yaml or .yml file name inside the conf.d directory", name)
}
named[name] = true
}
data, err := os.ReadFile(configPath) // #nosec G304 -- operator-configured config path.
if err != nil {
return fmt.Errorf("read config: %w", err)
}
cfg, err := decodeForSigning(data)
if err != nil {
return err
}
cfg.ConfigFile, cfg.ConfigDir = configPath, confDir
if cfg.Integrity.BinaryHash == "" || cfg.Integrity.ConfigHash == "" {
return fmt.Errorf("%w: the config has no recorded hashes", ErrNoFragmentBaseline)
}
rec, err := readFragmentRecord(configPath)
if err != nil {
return err
}
if rec.ConfdHash != cfg.Integrity.ConfdHash {
return fmt.Errorf("%w: the record belongs to another recorded state", ErrNoFragmentBaseline)
}
exempt := cfg.ConfD.IntegrityExempt
for name := range named {
if _, recorded := rec.Fragments[name]; recorded || isExemptFragment(name, exempt) {
continue
}
_, statErr := os.Lstat(filepath.Join(confDir, name))
if errors.Is(statErr, os.ErrNotExist) {
return fmt.Errorf("fragment %s does not exist in %s and is not part of the recorded state", name, confDir)
}
if statErr != nil {
return fmt.Errorf("fragment %s: %w", name, statErr)
}
}
var changed []string
binaryHash, err := HashFile(binaryPath)
if err != nil {
return fmt.Errorf("hashing binary: %w", err)
}
if binaryHash != cfg.Integrity.BinaryHash {
changed = append(changed, "binary "+binaryPath)
}
if HashConfigStableBytes(data) != cfg.Integrity.ConfigHash {
changed = append(changed, "main config "+configPath)
}
frags, err := config.ConfDirFragmentDigestInput(confDir)
if err != nil {
return fmt.Errorf("hashing conf.d: %w", err)
}
_, current := confdSnapshot(frags, exempt)
for _, name := range fragmentNames(rec.Fragments, current) {
if named[name] {
continue
}
was, had := rec.Fragments[name]
now, has := current[name]
switch {
case !had:
changed = append(changed, "fragment "+name+" (added)")
case !has:
changed = append(changed, "fragment "+name+" (removed)")
case !bytes.Equal(was, now):
changed = append(changed, "fragment "+name+" (edited)")
}
}
if len(changed) > 0 {
return &UnnamedChangesError{Changed: changed}
}
return signAndSavePreserving(configPath, confDir, data, cfg, cfg.Integrity.BinaryHash, frags)
}
// validFragmentName accepts what the conf.d loader would merge: a plain
// .yaml or .yml file name with no directory part.
func validFragmentName(name string) bool {
// Newlines can impersonate boundaries in the legacy aggregate digest.
return (strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")) && filepath.Base(name) == name && !strings.ContainsAny(name, "\x00\r\n")
}
func fragmentNames(a, b map[string][]byte) []string {
names := make([]string, 0, len(a)+len(b))
for name := range a {
names = append(names, name)
}
for name := range b {
if _, dup := a[name]; !dup {
names = append(names, name)
}
}
slices.Sort(names)
return names
}
package integrity
import (
"bytes"
"fmt"
"os"
"reflect"
"strings"
"sync"
"github.com/pidginhost/csm/internal/config"
)
var configWriteMu sync.Mutex
// ConfigWriteMutex serializes complete read-validate-write transactions for
// the main configuration. Callers must take it before reading csm.yaml, not
// only around the final rename, or an ETag check can race another writer.
func ConfigWriteMutex() *sync.Mutex {
return &configWriteMu
}
// WriteConfigBytesAtomic writes data to path with the same atomic-rename
// semantics SignAndSaveAtomic uses. Intended for paths that ship pre-signed
// bytes (e.g. restoring a snapshot whose hash already matches its content)
// where re-signing would mutate the integrity block we want to preserve.
func WriteConfigBytesAtomic(path string, data []byte) error {
return atomicWriteFile(path, data, 0o600)
}
// SignAndSavePreserving writes editedBytes to path after patching
// integrity.binary_hash and integrity.config_hash inside the byte
// stream itself, not by re-marshaling the cfg. Operator comments and
// untouched formatting outside the integrity block survive
// byte-for-byte.
//
// Verifies the final bytes decode as a config and match
// intendedClone under reflect.DeepEqual (with integrity fields
// normalised). Mismatch aborts the write.
//
// Atomic write semantics match SignAndSaveAtomic: same-directory
// tempfile, fsync, rename. On success, intendedClone.Integrity.BinaryHash
// and .ConfigHash are updated in place to reflect the hashes written to
// disk. intendedClone.ConfigFile must equal path and intendedClone.ConfigDir
// must equal confDir.
func SignAndSavePreserving(path, confDir string, editedBytes []byte, intendedClone *config.Config, binaryHash string) error {
frags, err := config.ConfDirFragmentDigestInput(confDir)
if err != nil {
return fmt.Errorf("hashing conf.d: %w", err)
}
return signAndSavePreserving(path, confDir, editedBytes, intendedClone, binaryHash, frags)
}
// signAndSavePreserving signs over fragments the caller already read, so a
// caller that checked those bytes signs exactly what it checked.
func signAndSavePreserving(path, confDir string, editedBytes []byte, intendedClone *config.Config, binaryHash string, frags []config.ConfDirFragment) error {
if intendedClone == nil {
return fmt.Errorf("intendedClone is nil")
}
if intendedClone.ConfigFile != path {
return fmt.Errorf("intendedClone.ConfigFile=%q does not match path=%q", intendedClone.ConfigFile, path)
}
if intendedClone.ConfigDir != confDir {
return fmt.Errorf("intendedClone.ConfigDir=%q does not match confDir=%q", intendedClone.ConfigDir, confDir)
}
// Hash the operator-edited bytes before the integrity scalars are
// rewritten. HashConfigStableBytes ignores the integrity block, so
// the stored hash still matches the final file after the integrity
// patch.
newConfigHash := HashConfigStableBytes(editedBytes)
// Cover the conf.d fragments merged on top of this main config. Empty
// when there are none, leaving conf.d-free configs byte-identical to
// their prior baseline.
newConfdHash, fragmentContents := confdSnapshot(frags, intendedClone.ConfD.IntegrityExempt)
patched, err := config.YAMLEdit(editedBytes, []config.YAMLChange{
{Path: []string{"integrity", "binary_hash"}, Value: binaryHash},
{Path: []string{"integrity", "config_hash"}, Value: newConfigHash},
{Path: []string{"integrity", "confd_hash"}, Value: newConfdHash},
})
if err != nil {
return fmt.Errorf("patch integrity scalars: %w", err)
}
if stripIntegrityBlock(string(patched)) != stripIntegrityBlock(string(editedBytes)) {
return fmt.Errorf("integrity patch drift: bytes outside integrity block changed")
}
decoded, err := decodeForSigning(patched)
if err != nil {
return fmt.Errorf("verify decode: %w", err)
}
decoded.ConfigFile = path
decoded.ConfigDir = confDir
expected := *intendedClone
expected.Integrity.BinaryHash = binaryHash
expected.Integrity.ConfigHash = newConfigHash
expected.Integrity.ConfdHash = newConfdHash
if !reflect.DeepEqual(decoded, &expected) {
return fmt.Errorf("yaml rewrite drift: decoded config does not match intended clone")
}
intendedClone.Integrity.BinaryHash = binaryHash
intendedClone.Integrity.ConfigHash = newConfigHash
intendedClone.Integrity.ConfdHash = newConfdHash
if err := keepPreviousConfig(path, patched); err != nil {
return err
}
// The record goes first: if the config write then fails, the record
// names a digest the config does not carry and is ignored, rather than
// a signed config being left without one.
if err := writeFragmentRecord(path, newConfdHash, fragmentContents); err != nil {
return err
}
return atomicWriteFile(path, patched, 0o600)
}
// keepPreviousConfig copies the current file to <path>.bak before it is
// replaced, so a bad edit from the UI, a rehash or the installer can be put
// back by hand. A missing file (first install) leaves no backup, and an
// unchanged file keeps the backup it already has.
func keepPreviousConfig(path string, next []byte) error {
// Legacy paths can be symlinks into /etc/csm. The service can write the
// resolved directory, but not necessarily the directory holding the link.
var err error
path, err = atomicWriteTarget(path)
if err != nil {
return err
}
current, err := os.ReadFile(path) // #nosec G304 -- operator-configured config path.
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("read previous config: %w", err)
}
if bytes.Equal(current, next) {
return nil
}
return atomicWriteFile(path+".bak", current, 0o600)
}
// SignConfigFilePreserving signs path in place without re-marshaling the
// config. It updates only the operator-owned main config file, but folds the
// conf.d fragments under confDir into integrity.confd_hash so a later edit to
// any fragment is detected by Verify.
func SignConfigFilePreserving(path, confDir, binaryHash string) (configHash, confdHash string, err error) {
cfg, err := SignConfigFilePreservingSnapshot(path, confDir, binaryHash)
if err != nil {
return "", "", err
}
return cfg.Integrity.ConfigHash, cfg.Integrity.ConfdHash, nil
}
// SignConfigFilePreservingSnapshot signs path and returns the decoded main
// config whose conf.d exemption policy selected the new ConfdHash. Callers
// that mirror the new integrity metadata into a running config must carry
// ConfD from this same snapshot so later Verify calls use the matching list.
func SignConfigFilePreservingSnapshot(path, confDir, binaryHash string) (*config.Config, error) {
// #nosec G304 -- operator-configured config path.
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read config: %w", err)
}
cfg, err := decodeForSigning(data)
if err != nil {
return nil, err
}
cfg.ConfigFile = path
cfg.ConfigDir = confDir
if err := SignAndSavePreserving(path, confDir, data, cfg, binaryHash); err != nil {
return nil, err
}
return cfg, nil
}
// stripIntegrityBlock removes the top-level `integrity:` mapping and
// its indented children from s, then returns the remaining text.
// Used both by SignAndSavePreserving's drift guard and by tests, so
// both compare using the same definition of "outside the integrity
// block".
func stripIntegrityBlock(s string) string {
lines := strings.Split(s, "\n")
var out []string
inIntegrity := false
for _, line := range lines {
if strings.HasPrefix(line, "integrity:") {
inIntegrity = true
continue
}
if inIntegrity {
if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") || line == "" {
continue
}
inIntegrity = false
}
out = append(out, line)
}
return strings.Join(out, "\n")
}
// decodeForSigning decodes config bytes this package signs or compares.
// Signing never uses runtime secrets, so a secret named by a *_env key may be
// unset here (rehash runs from package scripts without the service
// environment). Callers that act on the config load it fail-closed first.
func decodeForSigning(data []byte) (*config.Config, error) {
return config.LoadBytesWithOptions(data, config.LoadOptions{AllowUnsetSecretEnv: true})
}
package integrity
import (
"fmt"
"os"
)
// FileChangeKey changes when the file behind info is replaced or rewritten:
// its size and modification time, plus on Linux its device, inode and change
// time, which a rewrite cannot keep even when size and mtime match.
func FileChangeKey(info os.FileInfo) string {
return fmt.Sprintf("%d:%d:%s", info.Size(), info.ModTime().UnixNano(), fileChangeIdentity(info))
}
//go:build linux
package integrity
import (
"fmt"
"os"
"syscall"
)
// fileChangeIdentity adds the device, inode and change time, which a
// replaced or rewritten file cannot keep even if its size and mtime match.
func fileChangeIdentity(info os.FileInfo) string {
st, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return ""
}
return fmt.Sprintf("%d:%d:%d.%d", st.Dev, st.Ino, st.Ctim.Sec, st.Ctim.Nsec)
}
package integrity
import (
"bufio"
"bytes"
"crypto/sha256"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
"github.com/pidginhost/csm/internal/config"
)
// Sentinel causes for a Verify failure. Callers such as `csm doctor` match on
// them to suggest the right remedy; the wrapped messages carry the hashes.
var (
ErrBinaryHashMismatch = errors.New("binary hash mismatch")
ErrConfigHashMismatch = errors.New("config hash mismatch")
ErrConfdHashMismatch = errors.New("conf.d hash mismatch")
)
// HashFile returns the SHA256 hash of a file.
func HashFile(path string) (string, error) {
// #nosec G304 -- integrity hashing of operator-configured binary/config paths.
f, err := os.Open(path)
if err != nil {
return "", err
}
defer func() { _ = f.Close() }()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return fmt.Sprintf("sha256:%x", h.Sum(nil)), nil
}
// HashConfigStable hashes the config file excluding the integrity section,
// so that writing hashes back to the config doesn't change the hash.
//
// Line length is capped at 1 MiB; anything longer is treated as a
// corrupted config and the digest reflects whatever was scanned up to
// the truncation point. A corruption-driven hash will trip
// integrity.Verify on the next start, which is the intended response.
func HashConfigStable(path string) (string, error) {
// #nosec G304 -- operator-supplied config file path.
data, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("reading config: %w", err)
}
return HashConfigStableBytes(data), nil
}
// HashConfigStableBytes is the in-memory counterpart to
// HashConfigStable: compute the stable hash from an already-serialised
// config body without touching disk. Used by the SIGHUP reload path
// and `csm rehash` so both can hash a prospective file content before
// committing it, avoiding the two-pass-write dance that could leave
// integrity.config_hash blank on disk if the second save failed.
//
// Returns the same "sha256:..." string shape as HashConfigStable.
// The scanner is sized to cover config lines well beyond any
// realistic CSM yaml; a line longer than 1 MiB is treated as
// corruption and the final digest reflects whatever was scanned
// before the truncation (the resulting mismatch against any stored
// hash will trip integrity.Verify, which is the correct response).
func HashConfigStableBytes(data []byte) string {
h := sha256.New()
scanner := bufio.NewScanner(bytes.NewReader(data))
scanner.Buffer(make([]byte, 0, 64*1024), 1<<20)
inIntegrity := false
for scanner.Scan() {
line := scanner.Text()
// Skip the integrity section
if strings.HasPrefix(line, "integrity:") {
inIntegrity = true
continue
}
if inIntegrity {
// Still inside integrity block (indented lines)
if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") || line == "" {
continue
}
inIntegrity = false
}
_, _ = h.Write([]byte(line + "\n"))
}
return fmt.Sprintf("sha256:%x", h.Sum(nil))
}
// SignAndSaveAtomic re-computes integrity.config_hash for cfg and
// writes the result to cfg.ConfigFile atomically. Atomicity means:
// the on-disk file either carries the prior content (operation
// failed) or the fully-signed new content (operation succeeded).
// There is no window where the file exists on disk with an empty or
// stale config_hash, so a crash between the two passes of the
// previous two-save dance can no longer put the daemon into a
// crash-loop on next startup.
//
// The integrity.binary_hash is set to the supplied binaryHash. The
// CALLER is responsible for picking the right value: `csm rehash`
// hashes /opt/csm/csm afresh; SIGHUP reload preserves the prior
// daemon's binary hash because a reload cannot upgrade the binary.
//
// Implementation: marshal the config with a blank ConfigHash, hash
// the stable form of those bytes, store the hash, marshal again,
// write to a sibling temp file, rename into place. The YAML hashing
// strips the integrity block so both marshals round-trip to the
// same stable hash.
func SignAndSaveAtomic(cfg *config.Config, binaryHash string) error {
cfg.Integrity.BinaryHash = binaryHash
cfg.Integrity.ConfigHash = ""
confdHash, err := HashConfDir(cfg.ConfigDir, cfg.ConfD.IntegrityExempt)
if err != nil {
return fmt.Errorf("hashing conf.d: %w", err)
}
cfg.Integrity.ConfdHash = confdHash
preHash, err := yaml.Marshal(cfg)
if err != nil {
return fmt.Errorf("marshal (pre-hash): %w", err)
}
cfg.Integrity.ConfigHash = HashConfigStableBytes(preHash)
final, err := yaml.Marshal(cfg)
if err != nil {
return fmt.Errorf("marshal (post-hash): %w", err)
}
return atomicWriteFile(cfg.ConfigFile, final, 0o600)
}
// atomicWriteFile writes data to a temp file in the same directory
// as path, fsyncs and closes the temp, then renames it onto path.
// Rename is atomic on POSIX when source and destination are on the
// same filesystem (which is why the temp is created in the target's
// dir, not /tmp). Permission is applied before the rename.
func atomicWriteFile(path string, data []byte, perm os.FileMode) error {
targetPath, err := atomicWriteTarget(path)
if err != nil {
return err
}
dir := filepath.Dir(targetPath)
tmp, err := os.CreateTemp(dir, ".csm-cfg-*.tmp")
if err != nil {
return fmt.Errorf("create temp: %w", err)
}
tmpName := tmp.Name()
// Best-effort cleanup: if any error below leaves the temp
// behind, unlink it so we do not fill the dir with orphans.
cleanup := func() { _ = os.Remove(tmpName) }
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
cleanup()
return fmt.Errorf("write temp: %w", err)
}
if err := tmp.Chmod(perm); err != nil {
_ = tmp.Close()
cleanup()
return fmt.Errorf("chmod temp: %w", err)
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
cleanup()
return fmt.Errorf("fsync temp: %w", err)
}
if err := tmp.Close(); err != nil {
cleanup()
return fmt.Errorf("close temp: %w", err)
}
if err := os.Rename(tmpName, targetPath); err != nil {
cleanup()
return fmt.Errorf("rename: %w", err)
}
if err := syncDirectory(dir); err != nil {
return err
}
return nil
}
func atomicWriteTarget(path string) (string, error) {
info, err := os.Lstat(path)
if err != nil {
if os.IsNotExist(err) {
return path, nil
}
return "", fmt.Errorf("stat config path: %w", err)
}
if info.Mode()&os.ModeSymlink == 0 {
return path, nil
}
resolved, err := filepath.EvalSymlinks(path)
if err != nil {
return "", fmt.Errorf("resolve config symlink: %w", err)
}
return resolved, nil
}
func syncDirectory(dir string) error {
// #nosec G304 -- dir is derived from the caller-owned config path.
d, err := os.Open(dir)
if err != nil {
return fmt.Errorf("open dir: %w", err)
}
if err := d.Sync(); err != nil {
_ = d.Close()
return fmt.Errorf("fsync dir: %w", err)
}
if err := d.Close(); err != nil {
return fmt.Errorf("close dir: %w", err)
}
return nil
}
// HashConfDir returns a stable digest over every conf.d drop-in fragment that
// would be merged on top of the main config, in merge order. It returns the
// empty string when there are no fragments, so a config without conf.d keeps
// the empty digest and its pre-existing baseline still verifies after upgrade.
//
// Each fragment is domain-separated by name and length so two fragments cannot
// collide by shuffling bytes across the filename boundary.
//
// Fragments named in exempt (confd.integrity_exempt) hash as if absent: their
// owning integration rewrites them on its own schedule, and pinning them would
// turn every one of those rewrites into a failed restart.
func HashConfDir(confDir string, exempt []string) (string, error) {
frags, err := config.ConfDirFragmentDigestInput(confDir)
if err != nil {
return "", err
}
digest, _ := confdSnapshot(frags, exempt)
return digest, nil
}
// confdSnapshot retains the covered bytes so a per-fragment record can be
// checked against the existing conf.d digest without trusting its label.
func confdSnapshot(frags []config.ConfDirFragment, exempt []string) (string, map[string][]byte) {
h := sha256.New()
perFragment := make(map[string][]byte, len(frags))
for _, f := range frags {
if isExemptFragment(f.Name, exempt) {
continue
}
fmt.Fprintf(h, "confd-fragment:%s:%d\n", f.Name, len(f.Data))
_, _ = h.Write(f.Data)
perFragment[f.Name] = f.Data
}
if len(perFragment) == 0 {
return "", perFragment
}
return fmt.Sprintf("sha256:%x", h.Sum(nil)), perFragment
}
// hashOrNone keeps "expected , got ..." out of operator-facing errors when one
// side of a conf.d comparison is the empty no-fragments digest.
func hashOrNone(h string) string {
if h == "" {
return "none"
}
return h
}
func isExemptFragment(name string, exempt []string) bool {
for _, e := range exempt {
if e == name {
return true
}
}
return false
}
// Verify checks the binary and config file integrity.
func Verify(binaryPath string, cfg *config.Config) error {
if cfg.Integrity.BinaryHash == "" {
return nil // Not yet baselined
}
currentHash, err := HashFile(binaryPath)
if err != nil {
return fmt.Errorf("hashing binary: %w", err)
}
if currentHash != cfg.Integrity.BinaryHash {
return fmt.Errorf("%w: expected %s, got %s; run `csm rehash` after an intentional binary upgrade",
ErrBinaryHashMismatch, cfg.Integrity.BinaryHash, currentHash)
}
if cfg.Integrity.ConfigHash != "" {
configHash, err := HashConfigStable(cfg.ConfigFile)
if err != nil {
return fmt.Errorf("hashing config: %w", err)
}
if configHash != cfg.Integrity.ConfigHash {
return fmt.Errorf("%w: expected %s, got %s; run `csm rehash` after an intentional csm.yaml change",
ErrConfigHashMismatch, cfg.Integrity.ConfigHash, configHash)
}
// conf.d fragments are merged on top of the main config at load time,
// so they must be covered too. A symmetric comparison closes the gap
// both ways: a tampered or added fragment makes the computed digest
// diverge from the stored one, and a baseline taken without conf.d
// stays empty == empty. Operators who already use conf.d must re-run
// `csm rehash` once after upgrade to populate confd_hash.
confdHash, err := HashConfDir(cfg.ConfigDir, cfg.ConfD.IntegrityExempt)
if err != nil {
return fmt.Errorf("hashing conf.d: %w", err)
}
if confdHash != cfg.Integrity.ConfdHash {
return fmt.Errorf("%w: a drop-in under %s changed since the hashes were last recorded (expected %s, got %s); run `csm rehash` after an intentional conf.d change, or list fragments an integration rewrites under confd.integrity_exempt",
ErrConfdHashMismatch, cfg.ConfigDir, hashOrNone(cfg.Integrity.ConfdHash), hashOrNone(confdHash))
}
}
return nil
}
// Package ipmap builds and writes the CSM IP map: one rspamd multimap file
// listing CSM's own evidence about sender addresses.
package ipmap
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"math"
"net/netip"
"slices"
"strconv"
)
// Symbol is one CSM_IP symbol and its multiplier on RuleScore.
type Symbol struct {
Name string
Multiplier float64
}
// The CSM_IP rule's symbols, strongest first. rspamd scores a
// symbol not in the rule's list at the rule's full score, so the writer
// emits only these.
var (
Drop = Symbol{"CSM_IP_DROP", 0.8}
Blocked = Symbol{"CSM_IP_BLOCKED", 0.6}
MailAuth = Symbol{"CSM_IP_MAIL_AUTH", 0.6}
Spam = Symbol{"CSM_IP_SPAM", 0.6}
BlocklistDE = Symbol{"CSM_IP_BLOCKLISTDE", 0.4}
CINS = Symbol{"CSM_IP_CINS", 0.4}
Canary = Symbol{"CSM_IP_CANARY", 0}
)
// Symbols lists the rule's symbols in rule order.
func Symbols() []Symbol { return []Symbol{Drop, Blocked, MailAuth, Spam, BlocklistDE, CINS, Canary} }
const (
// RuleScore is the CSM_IP rule's score; each line scales it.
RuleScore = 5.0
// CanaryAddr is the documentation address of the canary line.
CanaryAddr = "192.0.2.255"
// DefaultPath is where rspamd reads the map.
DefaultPath = "/var/lib/csm-rspamd/csm_ip.map"
)
// Tenths is the symbol's score in Exim's integer tenths.
func (s Symbol) Tenths() int { return int(math.Round(RuleScore * s.Multiplier * 10)) }
func rank(name string) int {
return slices.IndexFunc(Symbols(), func(s Symbol) bool { return s.Name == name })
}
// Sources are the entries per symbol name (addresses or CIDRs) and the
// addresses that must never be listed: whitelisted and the host's own.
type Sources struct {
Entries map[string][]string
Never []netip.Addr
// NeverNets are networks nothing inside may be listed from, such as
// infra_ips ranges.
NeverNets []netip.Prefix
}
type entry struct {
prefix netip.Prefix
symbol Symbol
}
// Map is one built map: its lines and a most-specific-match lookup.
type Map struct {
lines []entry
exact map[netip.Addr]Symbol
nets []entry // longest prefix first
}
var canaryAddr = netip.MustParseAddr(CanaryAddr)
// Include both the non-listable ranges and the reserved canary. Check
// overlap too: a listable network address can still contain private hosts.
var excludedNetworks = []netip.Prefix{
netip.MustParsePrefix("0.0.0.0/32"),
netip.MustParsePrefix("10.0.0.0/8"),
netip.MustParsePrefix("127.0.0.0/8"),
netip.MustParsePrefix("169.254.0.0/16"),
netip.MustParsePrefix("172.16.0.0/12"),
netip.MustParsePrefix("192.168.0.0/16"),
netip.MustParsePrefix("224.0.0.0/4"),
netip.MustParsePrefix("255.255.255.255/32"),
netip.MustParsePrefix("::/128"),
netip.MustParsePrefix("::1/128"),
netip.MustParsePrefix("fc00::/7"),
netip.MustParsePrefix("fe80::/10"),
netip.MustParsePrefix("ff00::/8"),
netip.PrefixFrom(canaryAddr, canaryAddr.BitLen()),
}
// listable reports whether a can ever be listed: global unicast, not
// private, and not the canary, which only the canary line names.
func listable(a netip.Addr) bool {
return a.IsGlobalUnicast() && !a.IsPrivate() && a != canaryAddr
}
// Build makes the map from src: one line per address or network, with the
// strongest symbol winning and excluded addresses omitted. A nested address
// or network is kept only when its symbol is stronger, because rspamd uses
// the most specific match.
func Build(src Sources) Map {
never := make([]netip.Addr, len(src.Never))
for i, a := range src.Never {
never[i] = a.WithZone("").Unmap()
}
neverNets := make([]netip.Prefix, len(src.NeverNets))
for i, p := range src.NeverNets {
neverNets[i] = unmapPrefix(p)
}
best := map[netip.Prefix]Symbol{}
for _, s := range Symbols() {
if s == Canary {
continue
}
for _, raw := range src.Entries[s.Name] {
p, ok := parse(raw)
if !ok || !listable(p.Addr()) {
continue
}
if slices.ContainsFunc(excludedNetworks, p.Overlaps) || slices.ContainsFunc(neverNets, p.Overlaps) || slices.ContainsFunc(never, p.Contains) {
continue
}
if cur, ok := best[p]; !ok || rank(s.Name) < rank(cur.Name) {
best[p] = s
}
}
}
m := Map{exact: map[netip.Addr]Symbol{}}
var nets []entry
for p, s := range best {
if !p.IsSingleIP() {
nets = append(nets, entry{p, s})
}
}
// Check every ancestor: the nearest one may itself be removed because
// a broader network has a stronger symbol.
covered := func(p netip.Prefix, s Symbol) bool {
for _, n := range nets {
if n.prefix.Bits() < p.Bits() && n.prefix.Contains(p.Addr()) && rank(n.symbol.Name) <= rank(s.Name) {
return true
}
}
return false
}
for p, s := range best {
if covered(p, s) {
continue
}
m.lines = append(m.lines, entry{p, s})
if p.IsSingleIP() {
m.exact[p.Addr()] = s
} else {
m.nets = append(m.nets, entry{p, s})
}
}
slices.SortFunc(m.lines, func(a, b entry) int {
if c := a.prefix.Addr().Compare(b.prefix.Addr()); c != 0 {
return c
}
return a.prefix.Bits() - b.prefix.Bits()
})
slices.SortFunc(m.nets, func(a, b entry) int { return b.prefix.Bits() - a.prefix.Bits() })
return m
}
// parse reads one address or network. A zoned IPv6 address is refused:
// netip keeps any text after the zone separator, newlines included, and the
// map is a line-oriented file.
func parse(raw string) (netip.Prefix, bool) {
if p, err := netip.ParsePrefix(raw); err == nil {
p = unmapPrefix(p)
return p, p.IsValid()
}
a, err := netip.ParseAddr(raw)
if err != nil || a.Zone() != "" {
return netip.Prefix{}, false
}
a = a.Unmap()
return netip.PrefixFrom(a, a.BitLen()), true
}
// Keep IPv4 entries in the same form as sender lookups and exclusions.
func unmapPrefix(p netip.Prefix) netip.Prefix {
if p.Addr().Is4In6() {
if p.Bits() < 96 {
return netip.Prefix{}
}
p = netip.PrefixFrom(p.Addr().Unmap(), p.Bits()-96)
}
return p.Masked()
}
// Len is the number of data lines.
func (m Map) Len() int { return len(m.lines) }
// Weight is the score in tenths rspamd gives a, from its most specific
// entry; 0 when unlisted.
func (m Map) Weight(a netip.Addr) int {
a = a.Unmap()
if s, ok := m.exact[a]; ok {
return s.Tenths()
}
for _, n := range m.nets {
if n.prefix.Contains(a) {
return n.symbol.Tenths()
}
}
return 0
}
// Render returns the file and its generation: the first 12 hex digits of
// the SHA-256 of the data lines, which the canary line carries.
func (m Map) Render() ([]byte, string) {
var data bytes.Buffer
for _, e := range m.lines {
target := e.prefix.String()
if e.prefix.IsSingleIP() {
target = e.prefix.Addr().String()
}
data.WriteString(target + " " + e.symbol.Name + ":" + strconv.FormatFloat(e.symbol.Multiplier, 'f', -1, 64) + "\n")
}
sum := sha256.Sum256(data.Bytes())
gen := hex.EncodeToString(sum[:])[:12]
var out bytes.Buffer
out.WriteString(CanaryAddr + " " + Canary.Name + ":0:g" + gen + "\n")
out.Write(data.Bytes())
return out.Bytes(), gen
}
package ipmap
import "time"
// CanaryGrace is how long rspamd may take to load a new map before the
// canary counts against it.
const CanaryGrace = 5 * time.Minute
// CanaryProblem judges the writer's state and one canary answer: seen is
// the generation rspamd reported ("" when CSM_IP_CANARY was absent). It
// returns what is wrong, or "".
func CanaryProblem(st State, seen string, probeErr error, now time.Time) string {
if st.Err != nil {
return "CSM could not write the IP map (" + st.Err.Error() + "); rspamd keeps using the previous map"
}
if st.Gen == "" || seen == st.Gen || now.Sub(st.WrittenAt) < CanaryGrace {
return ""
}
if probeErr != nil {
return "CSM could not ask rspamd which IP map it uses: " + probeErr.Error()
}
if seen == "" {
return "rspamd is not using the current CSM IP map: its canary answer carries no CSM_IP_CANARY; check the CSM_IP rule and that rspamd can read the map file"
}
return "rspamd is not using the current CSM IP map: it still reports generation " + seen + "; check that rspamd can read the map file"
}
package ipmap
import (
"net/netip"
"slices"
"strconv"
"strings"
)
// StalwartNamespace is the lookup namespace holding s's keys for one address family.
func StalwartNamespace(s Symbol, v6 bool) string {
ns := "csm-ip-"
if v6 {
ns = "csm-ip6-"
}
return ns + strings.ReplaceAll(strings.ToLower(strings.TrimPrefix(s.Name, "CSM_IP_")), "_", "-")
}
// StalwartNamespaces lists every CSM namespace in rule order, IPv4 then IPv6 per symbol.
func StalwartNamespaces() []string {
var out []string
for _, s := range Symbols() {
out = append(out, StalwartNamespace(s, false), StalwartNamespace(s, true))
}
return out
}
// StalwartKeys returns sorted keys in every namespace, with CanaryAddr in the IPv4 canary namespace.
func (m Map) StalwartKeys() map[string][]string {
out := map[string][]string{}
for _, ns := range StalwartNamespaces() {
out[ns] = nil
}
out[StalwartNamespace(Canary, false)] = []string{CanaryAddr}
for _, e := range m.lines {
ns := StalwartNamespace(e.symbol, e.prefix.Addr().Is6())
out[ns] = append(out[ns], stalwartKeys(e.prefix)...)
}
for _, keys := range out {
slices.Sort(keys)
}
return out
}
// Masked prefixes split at the next octet or nibble boundary, varying only the last unit.
func stalwartKeys(p netip.Prefix) []string {
a := p.Addr()
var units []int
width := 8
if a.Is4() {
for _, o := range a.As4() {
units = append(units, int(o))
}
} else {
width = 4
for _, o := range a.As16() {
units = append(units, int(o>>4), int(o&0xf))
}
}
n := (p.Bits() + width - 1) / width
var keys []string
for i := range 1 << (n*width - p.Bits()) {
u := slices.Clone(units[:n])
u[n-1] += i
keys = append(keys, stalwartKey(u, a.Is4(), n == len(units)))
}
return keys
}
func stalwartKey(units []int, v4, exact bool) string {
parts := make([]string, len(units))
if v4 {
// IPv4 remote_ip is dotted text, so networks need forward globs instead of CIDRs.
for i, u := range units {
parts[i] = strconv.Itoa(u)
}
if exact {
return strings.Join(parts, ".")
}
return strings.Join(parts, ".") + ".*"
}
// Compressed IPv6 text prevents forward globs, so keys use reversed nibble names.
for i, u := range units {
parts[len(units)-1-i] = strconv.FormatInt(int64(u), 16)
}
if exact {
return strings.Join(parts, ".")
}
return "*." + strings.Join(parts, ".")
}
package ipmap
import (
"net/netip"
"slices"
)
// Verdict is what one scan says about its sender.
type Verdict int
const (
NoVerdict Verdict = iota
SpamVerdict
HamVerdict
)
// Judge turns one scan into a verdict about its sender: the
// score, in Exim tenths, less the weight CSM's own map gave the sender; spam
// at or above the reject score, ham below the tag score, nothing between or
// on a failed scan. Dry run, enforce and an opted-out recipient judge the
// same way: the verdict is about the sender.
func Judge(scanOK bool, score, mapTenths, rejectScore, tagScore int) Verdict {
if !scanOK {
return NoVerdict
}
adjusted := score - mapTenths
switch {
case adjusted >= rejectScore*10:
return SpamVerdict
case adjusted < tagScore*10:
return HamVerdict
}
return NoVerdict
}
// Recordable reports whether evidence about a may be recorded: only
// addresses the map could ever list.
func Recordable(a netip.Addr, never []netip.Addr) bool {
if a.Zone() != "" {
return false
}
a = a.Unmap()
return listable(a) && !slices.ContainsFunc(never, func(excluded netip.Addr) bool {
return excluded.Unmap() == a
})
}
package ipmap
import (
"errors"
"fmt"
"io/fs"
"net/netip"
"os"
"path/filepath"
"sync"
"time"
)
// State is what the writer last did.
type State struct {
Gen string
WrittenAt time.Time
Err error
}
// Writer writes the map when its content changes and keeps the last written
// map so spam evidence can subtract that map's score for the sender.
type Writer struct {
Path string
chown func(path string, gid int) error
syncDir func(path string) error
mu sync.Mutex
last Map
state State
file os.FileInfo
gid int
}
// NewWriter returns a writer for path.
func NewWriter(path string) *Writer {
return &Writer{Path: path, chown: func(p string, gid int) error { return os.Chown(p, -1, gid) }, syncDir: syncDirectory}
}
// Write replaces the map when its content, service group or file metadata no
// longer match what it last wrote, or the file is missing. Failures before
// rename keep the previous file and lookup. After rename, wrote is true and
// the lookup follows the published file even if syncing the directory fails;
// err then reports uncertain durability. gid is rspamd's group; without it
// the file would be unreadable to rspamd.
func (w *Writer) Write(m Map, gid int, now time.Time) (bool, error) {
content, gen := m.Render()
w.mu.Lock()
defer w.mu.Unlock()
if gid < 0 {
w.state.Err = errors.New("rspamd's group is unknown")
return false, w.state.Err
}
if gen == w.state.Gen && gid == w.gid && w.state.Err == nil && w.file != nil {
// Content alone does not prove the map is still present and readable
// after a package change or an operator replacing the file.
if info, err := os.Lstat(w.Path); err == nil && os.SameFile(info, w.file) &&
info.Mode() == w.file.Mode() && info.Size() == w.file.Size() && info.ModTime().Equal(w.file.ModTime()) {
return false, nil
}
}
wrote, err := w.replace(content, gid)
if wrote {
w.last, w.state, w.gid = m, State{Gen: gen, WrittenAt: now}, gid
// A lost stat only disables the no-op check on the next write.
w.file, _ = os.Lstat(w.Path)
}
w.state.Err = err
return wrote, err
}
func (w *Writer) replace(content []byte, gid int) (bool, error) {
dir := filepath.Dir(w.Path)
tmp, err := os.CreateTemp(dir, "."+filepath.Base(w.Path)+".*")
if err != nil {
return false, w.writeErr(err)
}
name := tmp.Name()
defer func() { _ = os.Remove(name) }()
if _, err = tmp.Write(content); err == nil {
err = tmp.Chmod(0o640) // #nosec G302 -- rspamd's service group must read the published map.
}
if err == nil {
err = w.chown(name, gid)
}
if err == nil {
err = tmp.Sync()
}
if cerr := tmp.Close(); err == nil {
err = cerr
}
if err == nil {
err = os.Rename(name, w.Path)
}
if err != nil {
return false, w.writeErr(err)
}
if err := w.syncDir(dir); err != nil {
return true, fmt.Errorf("syncing %s: %w", dir, err)
}
return true, nil
}
// writeErr names the map in err instead of the random temporary file, so a
// persistent failure reads the same on every attempt.
func (w *Writer) writeErr(err error) error {
var pe *fs.PathError
var le *os.LinkError
switch {
case errors.As(err, &pe):
err = fmt.Errorf("%s: %w", pe.Op, pe.Err)
case errors.As(err, &le):
err = fmt.Errorf("%s: %w", le.Op, le.Err)
}
return fmt.Errorf("writing %s: %w", w.Path, err)
}
func syncDirectory(dir string) error {
d, err := os.Open(dir) // #nosec G304 -- the map's own directory
if err != nil {
return err
}
defer func() { _ = d.Close() }()
return d.Sync()
}
// Weight looks a up in the last written map, in tenths.
func (w *Writer) Weight(a netip.Addr) int {
w.mu.Lock()
defer w.mu.Unlock()
return w.last.Weight(a)
}
// State reports the last write.
func (w *Writer) State() State {
w.mu.Lock()
defer w.mu.Unlock()
return w.state
}
// Remove deletes the map and forgets it; a missing file is fine.
func (w *Writer) Remove() error {
w.mu.Lock()
defer w.mu.Unlock()
if err := os.Remove(w.Path); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
w.last, w.state = Map{}, State{}
w.file = nil
return nil
}
package jstaint
import "github.com/tdewolff/parse/v2/js"
// canonicalVar follows the parser's Link chain to the declaration identity. The
// parser sets Link when it merges an undeclared use with its later declaration
// or an outer-scope binding, so following it to the end yields one stable
// identity per variable. Shadowed declarations keep distinct identities because
// each is its own declared Var with a nil Link.
func canonicalVar(v *js.Var) *js.Var {
for v.Link != nil {
v = v.Link
}
return v
}
// tdewolff's walker reaches ClassElementName but does not descend into its
// computed expression. Walk it explicitly so limits and semantic visitors see
// code that JavaScript executes while defining a class.
func walkComputedClassName(v js.IVisitor, node js.INode) bool {
name, ok := node.(*js.ClassElementName)
if !ok {
return false
}
js.Walk(v, name.Computed)
return true
}
// staticStringOrIdent returns the literal identifier or string value of expr, if
// expr is a literal. It accepts both concrete literal forms the AST uses: a
// DotExpr member name arrives as a LiteralExpr value, while a bracket index or
// call argument arrives as a *LiteralExpr pointer. Handling only the pointer
// form would silently drop every dotted member name.
//
// String tokens are unquoted but not unescaped: escape spellings such as
// \x6b are out of version 1 scope, so the raw bytes between the quotes are
// returned verbatim.
func staticStringOrIdent(expr js.IExpr) (string, bool) {
data, ok := staticBytesOrIdent(expr)
if !ok {
return "", false
}
return string(data), true
}
// staticBytesOrIdent is the allocation-free form of staticStringOrIdent for
// AST walkers that only need to compare or render the literal bytes.
func staticBytesOrIdent(expr js.IExpr) ([]byte, bool) {
switch lit := expr.(type) {
case *js.LiteralExpr:
return literalBytes(lit.TokenType, lit.Data)
case js.LiteralExpr:
return literalBytes(lit.TokenType, lit.Data)
default:
return nil, false
}
}
func literalText(tt js.TokenType, data []byte) (string, bool) {
literal, ok := literalBytes(tt, data)
if !ok {
return "", false
}
return string(literal), true
}
func literalBytes(tt js.TokenType, data []byte) ([]byte, bool) {
switch tt {
case js.IdentifierToken:
return data, true
case js.StringToken:
if len(data) < 2 {
return nil, false
}
q := data[0]
if (q != '\'' && q != '"') || data[len(data)-1] != q {
return nil, false
}
return data[1 : len(data)-1], true
default:
return nil, false
}
}
package jstaint
import (
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
const (
// maxEvidencePaths bounds the returned evidence flows. TotalResults still
// reports the true count, so a caller can name the additional paths.
maxEvidencePaths = 8
// maxViaSegments bounds a displayed laundering chain: the first headViaSegments
// segments, one omission marker, and the final tailViaSegments segments.
maxViaSegments = 32
headViaSegments = 16
tailViaSegments = 15
// maxSegmentBytes bounds one displayed source, via, or sink segment so a long
// attacker-controlled identifier cannot create unbounded findings.
maxSegmentBytes = 64
)
// finalizeResults returns the sorted evidence flows capped to maxEvidencePaths,
// the true pre-cap flow count, and whether any evidence was shortened. Every
// displayed segment is sanitized and length-bounded, and a chain longer than
// maxViaSegments keeps its head and tail around one omission marker.
func (a *analysis) finalizeResults() ([]Result, int, bool) {
sorted := a.sortedResults()
total := len(sorted)
truncated := false
if total > maxEvidencePaths {
truncated = true
sorted = sorted[:maxEvidencePaths]
}
for i := range sorted {
source, cutSource := boundSegment(sorted[i].Source)
sink, cutSink := boundSegment(sorted[i].Sink)
via, cutVia := truncateVia(sorted[i].Via)
sorted[i].Source = source
sorted[i].Sink = sink
sorted[i].Via = via
truncated = truncated || cutSource || cutSink || cutVia
}
return sorted, total, truncated
}
// truncateVia bounds each segment and, for a chain longer than maxViaSegments,
// retains the head and tail around one marker that names the omitted count.
func truncateVia(via []string) ([]string, bool) {
if len(via) == 0 {
return via, false
}
truncated := false
bounded := make([]string, len(via))
for i, s := range via {
seg, cut := boundSegment(s)
bounded[i] = seg
truncated = truncated || cut
}
if len(bounded) <= maxViaSegments {
return bounded, truncated
}
omitted := len(bounded) - (headViaSegments + tailViaSegments)
out := make([]string, 0, maxViaSegments)
out = append(out, bounded[:headViaSegments]...)
out = append(out, "["+strconv.Itoa(omitted)+" segments omitted]")
out = append(out, bounded[len(bounded)-tailViaSegments:]...)
return out, true
}
// boundSegment sanitizes one display segment and bounds it to maxSegmentBytes at a
// rune boundary. It reports whether the length bound shortened the segment;
// sanitizing invalid bytes alone does not count as truncation.
func boundSegment(s string) (string, bool) {
clean := sanitizeSegment(s)
if len(clean) <= maxSegmentBytes {
return clean, false
}
const marker = "..."
cut := maxSegmentBytes - len(marker)
for cut > 0 && !utf8.RuneStart(clean[cut]) {
cut--
}
return clean[:cut] + marker, true
}
// sanitizeSegment emits valid UTF-8 and replaces control bytes so evidence text
// stays printable and searchable.
func sanitizeSegment(s string) string {
for i := 0; i < len(s); {
r, size := utf8.DecodeRuneInString(s[i:])
if (r == utf8.RuneError && size == 1) || unicode.IsControl(r) {
var clean strings.Builder
clean.Grow(len(s))
clean.WriteString(s[:i])
for i < len(s) {
r, size = utf8.DecodeRuneInString(s[i:])
if (r == utf8.RuneError && size == 1) || unicode.IsControl(r) {
clean.WriteByte('?')
} else {
clean.WriteString(s[i : i+size])
}
i += size
}
return clean.String()
}
i += size
}
return s
}
package jstaint
import (
"bytes"
"encoding/json"
"io"
"strconv"
"strings"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/parse/v2/css"
"github.com/tdewolff/parse/v2/html"
)
// isNonJSDocument is only used after JavaScript parsing fails. The deep walk
// supplies every file regardless of extension, so token matches in data and
// templates must not be reported as JavaScript coverage failures. Ambiguous
// content stays a parse failure; filenames never decide which files to skip.
func isNonJSDocument(src []byte) bool {
src = bytes.TrimSpace(bytes.TrimPrefix(src, []byte("\xef\xbb\xbf")))
if opensWithPHPTag(src) || json.Valid(src) {
return true
}
lexer := html.NewLexer(parse.NewInputBytes(src[:len(src):len(src)]))
for {
token, data := lexer.Next()
switch token {
case html.CommentToken:
// HTML comments can precede the document's first tag.
continue
case html.TextToken:
if len(bytes.TrimSpace(data)) == 0 {
continue
}
case html.StartTagToken, html.DoctypeToken:
return true
}
break
}
return isStylesheet(src) || isTranslationCatalog(src)
}
func opensWithPHPTag(src []byte) bool {
if bytes.HasPrefix(src, []byte("<?=")) {
return true
}
const tag = "<?php"
if len(src) < len(tag) || !bytes.EqualFold(src[:len(tag)], []byte(tag)) {
return false
}
// The long opening tag requires whitespace, unlike the echo tag.
return len(src) == len(tag) || bytes.ContainsAny(src[len(tag):len(tag)+1], " \t\r\n")
}
func isStylesheet(src []byte) bool {
// CSS parsers recover at EOF, even from unclosed strings and blocks. Only
// classify a complete token stream so malformed JS retains its warning.
if !completeCSSTokens(src) {
return false
}
p := css.NewParser(parse.NewInputBytes(src[:len(src):len(src)]), false)
sawRule := false
for {
grammar, _, _ := p.Next()
switch grammar {
case css.ErrorGrammar:
return p.Err() == io.EOF && sawRule
case css.AtRuleGrammar, css.BeginAtRuleGrammar:
sawRule = true
case css.BeginRulesetGrammar:
if !plausibleCSSSelector(p.Values()) {
return false
}
sawRule = true
}
}
}
// The CSS grammar parser accepts arbitrary selector tokens. Reject JS
// assignments and calls rather than treating their blocks as style rules.
func plausibleCSSSelector(tokens []css.Token) bool {
brackets, functions := 0, 0
previous := css.EmptyToken
for _, token := range tokens {
switch token.TokenType {
case css.WhitespaceToken, css.CommentToken:
continue
case css.LeftBracketToken:
brackets++
case css.RightBracketToken:
brackets--
case css.FunctionToken:
if previous != css.ColonToken && functions == 0 {
return false
}
functions++
case css.LeftParenthesisToken:
if functions == 0 {
return false
}
functions++
case css.RightParenthesisToken:
functions--
case css.DelimToken:
if brackets == 0 && !bytes.ContainsAny(token.Data, ".*>+~|&") {
return false
}
case css.SemicolonToken, css.AtKeywordToken:
return false
}
previous = token.TokenType
}
return true
}
func completeCSSTokens(src []byte) bool {
l := css.NewLexer(parse.NewInputBytes(src[:len(src):len(src)]))
var closes []css.TokenType
for {
token, data := l.Next()
switch token {
case css.ErrorToken:
return l.Err() == io.EOF && len(closes) == 0
case css.BadStringToken, css.BadURLToken:
return false
case css.StringToken:
if len(data) < 2 || data[len(data)-1] != data[0] {
return false
}
case css.CommentToken:
if !bytes.HasSuffix(data, []byte("*/")) {
return false
}
case css.URLToken:
if !bytes.HasSuffix(data, []byte(")")) {
return false
}
case css.LeftBraceToken:
closes = append(closes, css.RightBraceToken)
case css.LeftBracketToken:
closes = append(closes, css.RightBracketToken)
case css.LeftParenthesisToken, css.FunctionToken:
closes = append(closes, css.RightParenthesisToken)
case css.RightBraceToken, css.RightBracketToken, css.RightParenthesisToken:
if len(closes) == 0 || closes[len(closes)-1] != token {
return false
}
closes = closes[:len(closes)-1]
}
}
}
// Catalog strings can contain entire JavaScript programs. Require every
// non-comment line to be a gettext directive or a quoted continuation, so a
// catalog-like prefix cannot discard a trailing program or a parse failure.
func isTranslationCatalog(src []byte) bool {
sawID, sawTranslation, inString := false, false, false
for line := range strings.Lines(string(src)) {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if line[0] != '"' {
end := strings.IndexAny(line, " \t")
if end < 0 {
return false
}
key, value := line[:end], line[end:]
switch {
case key == "msgid":
sawID = true
case key == "msgstr":
sawTranslation = true
case strings.HasPrefix(key, "msgstr[") && strings.HasSuffix(key, "]"):
index := key[len("msgstr[") : len(key)-1]
if index == "" || strings.Trim(index, "0123456789") != "" {
return false
}
sawTranslation = true
case key == "msgctxt", key == "msgid_plural":
default:
return false
}
line = strings.TrimSpace(value)
inString = true
}
if !inString || len(line) < 2 || line[0] != '"' {
return false
}
if _, err := strconv.Unquote(line); err != nil {
return false
}
}
return sawID && sawTranslation
}
package jstaint
import "github.com/tdewolff/parse/v2/js"
// allocate installs a fresh instance at site and returns a value referencing it.
// Inside a loop the instance is summarized, because the site can produce many
// runtime objects; otherwise it is the current singleton for this control-flow
// path. A prior current instance at the same site is promoted to the summary
// class first, so a re-allocation cannot clear taint on an earlier instance that
// was aliased or published.
func (a *analysis) allocate(st *state, site int, inLoop bool) value {
a.promoteCurrent(st, site)
return a.installLiteral(st, site, inLoop, &object{})
}
// installLiteral publishes a fully evaluated object or array literal at its
// allocation site. Building it separately preserves construction semantics
// before loop instances are merged into a summary.
func (a *analysis) installLiteral(st *state, site int, inLoop bool, fresh *object) value {
id := allocID{site: site, summary: inLoop}
if inLoop {
if old, ok := st.heap[id]; ok {
fresh = mergeObject(old, fresh)
}
}
st.installObject(id, fresh)
a.fact()
return value{allocs: allocSet{{id: id}: true}, allocOnly: true}
}
// promoteCurrent moves any current instance at site into the summary class and
// rewrites every reference to it. This is how two recency classes model an
// unbounded number of runtime objects: the escaped or aliased earlier object
// survives in the summary while a fresh current instance takes its place.
func (a *analysis) promoteCurrent(st *state, site int) {
from := allocID{site: site, summary: false}
cur, ok := st.heap[from]
if !ok {
return
}
to := allocID{site: site, summary: true}
if ex, ok := st.heap[to]; ok {
st.installObject(to, mergeObject(ex, cur))
} else {
// A plain move within one state preserves whatever exclusivity the
// object already had.
st.ownHeap()
st.heap[to] = cur
}
st.dropObject(from)
rewriteAlloc(st, from, to)
}
func rewriteAlloc(st *state, from, to allocID) {
for k, v := range st.env {
if hasAllocID(v.allocs, from) {
v.allocs = replaceAlloc(v.allocs, from, to)
st.setEnv(k, v)
}
}
for node, v := range st.captures {
if hasAllocID(v.allocs, from) {
v.allocs = replaceAlloc(v.allocs, from, to)
st.setCapture(node, v)
}
}
// Read-only scan first: mutObject may replace st.heap with an owned copy,
// which must not happen while ranging over the map being replaced.
var hit []allocID
for id, o := range st.heap {
if objectRefersToAlloc(o, from) {
hit = append(hit, id)
}
}
for _, id := range hit {
o := st.mutObject(id)
for fk, fv := range o.fields {
if hasAllocID(fv.allocs, from) {
fv.allocs = replaceAlloc(fv.allocs, from, to)
o.fields[fk] = fv
}
}
if hasAllocID(o.wild.allocs, from) {
o.wild.allocs = replaceAlloc(o.wild.allocs, from, to)
}
if hasAllocID(o.wildReq.allocs, from) {
o.wildReq.allocs = replaceAlloc(o.wildReq.allocs, from, to)
}
if hasAllocID(o.elem.allocs, from) {
o.elem.allocs = replaceAlloc(o.elem.allocs, from, to)
}
}
}
func objectRefersToAlloc(o *object, id allocID) bool {
for _, fv := range o.fields {
if hasAllocID(fv.allocs, id) {
return true
}
}
return hasAllocID(o.wild.allocs, id) || hasAllocID(o.wildReq.allocs, id) ||
hasAllocID(o.elem.allocs, id)
}
func replaceAlloc(s allocSet, from, to allocID) allocSet {
out := make(allocSet, len(s))
for ref := range s {
if ref.id == from {
ref.id = to
out[ref] = true
} else {
out[ref] = true
}
}
return out
}
func valueCarriesDepthZero(st *state, v value) bool {
if taintCarriesDepthZero(v.scalar) {
return true
}
seen := map[allocRef]bool{}
stack := make([]allocRef, 0, len(v.allocs))
for ref := range v.allocs {
stack = append(stack, ref)
}
push := func(parent allocRef, child value) bool {
child = applyRefDepth(child, parent)
for fact := range child.scalar {
if fact.callDepth == 0 {
return true
}
}
for ref := range child.allocs {
if !seen[ref] {
stack = append(stack, ref)
}
}
return false
}
for len(stack) > 0 {
ref := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if seen[ref] {
continue
}
seen[ref] = true
o := st.heap[ref.id]
if o == nil {
continue
}
for _, field := range o.fields {
if push(ref, field) {
return true
}
}
if push(ref, o.elem) || push(ref, o.wild) || push(ref, o.wildReq) {
return true
}
}
return false
}
func taintCarriesDepth(ts taintSet, depth int) bool {
for fact := range ts {
if int(fact.callDepth) == depth {
return true
}
}
return false
}
func taintCarriesDepthZero(ts taintSet) bool {
return taintCarriesDepth(ts, 0)
}
// resetAllocRefDepth records that an allocation now carries a fact created in
// the current invocation. Existing field facts retain their own depth, but
// aliases must no longer impose an older return-path barrier on the new write.
func resetAllocRefDepth(st *state, ids map[allocID]bool) {
needsReset := func(v value) bool {
for ref := range v.allocs {
if ids[ref.id] && (ref.minDepth != 0 || ref.advance != 0) {
return true
}
}
return false
}
reset := func(v value) value {
refs := make(allocSet, len(v.allocs))
for ref := range v.allocs {
if ids[ref.id] {
ref.minDepth = 0
ref.advance = 0
}
refs[ref] = true
}
v.allocs = refs
return v
}
for cv, v := range st.env {
if needsReset(v) {
st.setEnv(cv, reset(v))
}
}
for node, v := range st.captures {
if needsReset(v) {
st.setCapture(node, reset(v))
}
}
// Read-only scan first: mutObject may replace st.heap with an owned copy,
// which must not happen while ranging over the map being replaced.
var hit []allocID
for id, o := range st.heap {
if objectNeedsDepthReset(o, needsReset) {
hit = append(hit, id)
}
}
for _, id := range hit {
o := st.mutObject(id)
for name, v := range o.fields {
if needsReset(v) {
o.fields[name] = reset(v)
}
}
if needsReset(o.elem) {
o.elem = reset(o.elem)
}
if needsReset(o.wild) {
o.wild = reset(o.wild)
}
if needsReset(o.wildReq) {
o.wildReq = reset(o.wildReq)
}
}
}
func objectNeedsDepthReset(o *object, needsReset func(value) bool) bool {
for _, v := range o.fields {
if needsReset(v) {
return true
}
}
return needsReset(o.elem) || needsReset(o.wild) || needsReset(o.wildReq)
}
func hasAllocID(s allocSet, id allocID) bool {
for ref := range s {
if ref.id == id {
return true
}
}
return false
}
func allocsConstrained(s allocSet) bool {
for ref := range s {
if ref.minDepth != 0 || ref.advance != 0 {
return true
}
}
return false
}
// writeField taints one field on the receiver's allocations. A clean write is a
// strong update (it can clear taint) only when the receiver is exactly one
// current instance; a summary field or a receiver with several possible
// allocations is weak-updated, so it cannot lose taint an aliased path still
// carries. Statically known array indexes are distinct fields; unresolved array
// elements and wildcard writes stay weak because they can represent many keys.
func (a *analysis) writeField(st *state, recv value, key fieldKey, rhs value) {
ids := uniqueAllocIDs(recv.allocs)
strong := recv.allocOnly && len(ids) == 1 && soleCurrent(recv.allocs)
for id := range ids {
o := st.mutObject(id)
if o == nil {
o = &object{}
st.installObject(id, o)
}
switch key.kind {
case fieldNamed:
o.setNamed(key.name, rhs, strong && !id.summary)
case fieldElem:
if key.name != "" {
o.setNamed(key.name, rhs, strong && !id.summary)
} else {
o.weakElem(rhs, strong && !id.summary)
}
default:
o.writeWild(rhs, strong && !id.summary)
}
}
if a.callDepth == 0 && allocsConstrained(recv.allocs) && valueCarriesDepthZero(st, rhs) {
resetAllocRefDepth(st, ids)
}
a.fact()
}
// deleteField removes one statically known field when the receiver is a lone
// current instance. A delete through a summary or ambiguous receiver is a weak
// update, so it cannot prove that every represented runtime field disappeared.
func (a *analysis) deleteField(st *state, recv value, key fieldKey) {
ids := uniqueAllocIDs(recv.allocs)
if len(ids) != 1 || !soleCurrent(recv.allocs) {
return
}
for id := range ids {
o := st.mutObject(id)
if o == nil {
return
}
switch key.kind {
case fieldNamed:
o.deleteNamed(key.name)
case fieldElem:
if key.name == "" {
return
}
o.deleteNamed(key.name)
default:
return
}
a.fact()
}
}
func soleCurrent(s allocSet) bool {
for ref := range s {
if ref.id.summary {
return false
}
}
return len(s) != 0
}
func uniqueAllocIDs(s allocSet) map[allocID]bool {
ids := make(map[allocID]bool, len(s))
for ref := range s {
ids[ref.id] = true
}
return ids
}
// readField reads one field across every allocation the receiver may reference.
func (a *analysis) readField(st *state, recv value, key fieldKey) value {
var out value
have := false
definiteAlloc := recv.allocOnly && len(recv.allocs) != 0
definiteField := definiteAlloc
for ref := range recv.allocs {
o := st.heap[ref.id]
if o == nil {
definiteAlloc = false
definiteField = false
continue
}
fv, definite := getField(o, key)
fv = applyRefDepth(fv, ref)
out, have = mergePresentValue(out, have, fv), true
if !definite || !fv.allocOnly {
definiteAlloc = false
}
if !definite {
definiteField = false
}
}
if !have {
return value{}
}
out.allocOnly = definiteAlloc
if !definiteField {
out.scheme = schemeState{}
}
return out
}
func fieldDefinitelyPresent(st *state, recv value, key fieldKey) bool {
if !recv.allocOnly || len(recv.allocs) == 0 {
return false
}
seen := map[allocID]bool{}
for ref := range recv.allocs {
if seen[ref.id] {
continue
}
seen[ref.id] = true
o := st.heap[ref.id]
if o == nil {
return false
}
if _, definite := getField(o, key); !definite {
return false
}
}
return len(seen) != 0
}
// serializeStringify returns the scalar taint JSON.stringify(v) would carry.
// Reachable field taint is folded in by an iterative graph walk. An
// allocation whose reachable graph contains a cycle contributes no value, because
// the runtime throws before producing output; a union that also holds an acyclic
// alternative keeps the acyclic taint. The walk is iterative because a heap graph
// built through assignments can be far deeper than the AST, and a recursive walk
// could overflow the Go stack, a fatal error recover cannot intercept.
func (a *analysis) serializeStringify(st *state, v value) taintSet {
out := v.scalar
for ref := range v.allocs {
if ts, cyclic := a.walkStringify(st, ref); !cyclic {
out = mergeTaint(out, ts)
}
}
return out
}
type stringifySlot struct {
value value
optional bool
collection bool
}
func stringifyValues(o *object, ref allocRef) []stringifySlot {
values := make([]stringifySlot, 0, len(o.fields)+3)
for name, fv := range o.fields {
if !o.array || isArrayIndexName(name) {
values = append(values, stringifySlot{value: applyRefDepth(fv, ref), optional: !o.must[name]})
}
}
if o.elemMay {
values = append(values, stringifySlot{
value: applyRefDepth(o.elem, ref), optional: !o.elemMust, collection: true,
})
}
if o.wildMay {
values = append(values, stringifySlot{value: applyRefDepth(o.wild, ref), optional: true})
}
if o.wildMust && !o.array {
values = append(values, stringifySlot{value: applyRefDepth(o.wildReq, ref)})
}
return values
}
func arrayNeighbors(o *object, ref allocRef, addScalar func(taintSet)) []allocRef {
if !o.array {
return nil
}
var nbrs []allocRef
collect := func(v value) {
v = applyRefDepth(v, ref)
addScalar(v.scalar)
for aid := range v.allocs {
nbrs = append(nbrs, aid)
}
}
for name, fv := range o.fields {
if isArrayIndexName(name) {
collect(fv)
}
}
if o.elemMay {
collect(o.elem)
}
if o.wildMay {
collect(o.wild)
}
return nbrs
}
func (a *analysis) walkStringify(st *state, root allocRef) (taintSet, bool) {
// Preserve each field's allocation set as one group: its members are runtime
// alternatives, while separate fields and collected array elements must all
// serialize. Starting with leaf allocations and satisfying dependent groups
// computes the acyclic choices without mistaking a diamond for a cycle.
nodes := map[allocRef][]stringifySlot{}
stack := []allocRef{root}
for len(stack) > 0 {
if !a.alive() {
return nil, false
}
ref := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if _, ok := nodes[ref]; ok {
continue
}
var values []stringifySlot
if o := st.heap[ref.id]; o != nil {
values = stringifyValues(o, ref)
}
nodes[ref] = values
for _, slot := range values {
for aid := range slot.value.allocs {
if _, seen := nodes[aid]; !seen {
stack = append(stack, aid)
}
}
}
}
type dependencyGroup struct {
owner allocRef
satisfied bool
}
var groups []dependencyGroup
pending := make(map[allocRef]int, len(nodes))
reverse := map[allocRef][]int{}
for ref, values := range nodes {
for _, slot := range values {
fv := slot.value
if slot.optional {
continue
}
if len(fv.allocs) == 0 || (!slot.collection && (!fv.allocOnly || len(fv.scalar) != 0)) {
continue
}
if slot.collection {
for aid := range fv.allocs {
group := len(groups)
groups = append(groups, dependencyGroup{owner: ref})
pending[ref]++
reverse[aid] = append(reverse[aid], group)
}
continue
}
group := len(groups)
groups = append(groups, dependencyGroup{owner: ref})
pending[ref]++
for aid := range fv.allocs {
reverse[aid] = append(reverse[aid], group)
}
}
}
serializable := make(map[allocRef]bool, len(nodes))
queue := make([]allocRef, 0, len(nodes))
for ref := range nodes {
if pending[ref] == 0 {
queue = append(queue, ref)
}
}
for len(queue) > 0 {
if !a.alive() {
return nil, false
}
ref := queue[len(queue)-1]
queue = queue[:len(queue)-1]
if serializable[ref] {
continue
}
serializable[ref] = true
for _, group := range reverse[ref] {
if groups[group].satisfied {
continue
}
groups[group].satisfied = true
owner := groups[group].owner
pending[owner]--
if pending[owner] == 0 {
queue = append(queue, owner)
}
}
}
if !serializable[root] {
return nil, true
}
var out taintSet
done := map[allocRef]bool{}
stack = append(stack, root)
for len(stack) > 0 {
if !a.alive() {
return out, false
}
ref := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if done[ref] || !serializable[ref] {
continue
}
done[ref] = true
for _, slot := range nodes[ref] {
fv := slot.value
out = mergeTaint(out, fv.scalar)
for aid := range fv.allocs {
if serializable[aid] && !done[aid] {
stack = append(stack, aid)
}
}
}
}
return out, false
}
// serializeArray returns the scalar taint a join or array serialization carries.
// A self-referential element contributes no taint when revisited, but other
// tainted elements still propagate and the walk always terminates. It is
// iterative for the same stack-safety reason as serializeStringify.
func (a *analysis) serializeArray(st *state, v value) taintSet {
var out taintSet
done := map[allocRef]bool{}
stack := make([]allocRef, 0, len(v.allocs))
for ref := range v.allocs {
stack = append(stack, ref)
}
for len(stack) > 0 {
if !a.alive() {
return out
}
ref := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if done[ref] {
continue
}
done[ref] = true
o := st.heap[ref.id]
if o == nil || !o.array {
continue
}
for _, aid := range arrayNeighbors(o, ref, func(ts taintSet) { out = mergeTaint(out, ts) }) {
if !done[aid] {
stack = append(stack, aid)
}
}
}
return out
}
// numberAllocSites assigns a deterministic site id to every object, array, new
// expression, and modeled createElement call in source order. The ids drive the
// two-class recency model.
func numberAllocSites(ast *js.AST) map[js.INode]int {
n := &siteNumberer{sites: map[js.INode]int{}}
js.Walk(n, ast)
return n.sites
}
type siteNumberer struct {
sites map[js.INode]int
next int
}
func (n *siteNumberer) Exit(js.INode) {}
func (n *siteNumberer) Enter(node js.INode) js.IVisitor {
if walkComputedClassName(n, node) {
return n
}
switch e := node.(type) {
case *js.ObjectExpr, *js.ArrayExpr, *js.NewExpr:
n.sites[node] = n.next
n.next++
case *js.CallExpr:
// A createElement call allocates a fresh element, so it needs its own site
// for the resource-element receiver model.
if prop, _, ok := memberAccess(ungroupExpr(e.X)); ok && prop == "createElement" {
n.sites[node] = n.next
n.next++
}
}
return n
}
package jstaint
import "github.com/tdewolff/parse/v2/js"
// funcInfo is a normalized view of a function expression, declaration, or arrow
// function. The two AST shapes (FuncDecl and ArrowFunc) share nothing but these
// fields for the analyzer's purposes.
type funcInfo struct {
params js.Params
body *js.BlockStmt
generator bool
async bool
}
// handlerSite is one discovered keyboard-event handler registration. eventVar is
// the canonical identity of the handler's first parameter, or nil when that
// parameter is not a plain identifier (a destructured or absent parameter).
type handlerSite struct {
fn *funcInfo
eventVar *js.Var
}
// discoverHandlers finds every statically resolvable keyboard-handler
// registration in the parse unit and returns one site per resolved function.
// When a handler identifier resolves to more than one function value, each is
// returned so their analyses can be unioned.
func discoverHandlers(ast *js.AST) []handlerSite {
funcs := collectFuncValues(ast)
var sites []handlerSite
seen := map[*funcInfo]bool{}
v := &handlerVisitor{funcs: funcs, add: func(fn *funcInfo) {
if seen[fn] {
return
}
seen[fn] = true
sites = append(sites, handlerSite{fn: fn, eventVar: firstParamVar(fn.params)})
}}
js.Walk(v, ast)
return sites
}
type handlerVisitor struct {
funcs map[*js.Var][]*funcInfo
add func(*funcInfo)
}
func (v *handlerVisitor) Exit(js.INode) {}
func (v *handlerVisitor) Enter(n js.INode) js.IVisitor {
if walkComputedClassName(v, n) {
return v
}
switch e := n.(type) {
case *js.BinaryExpr:
if e.Op == js.EqToken && isKeyHandlerProperty(e.X) {
v.emit(e.Y)
}
case *js.CallExpr:
if name, fn, ok := addEventListenerHandler(e); ok && isDOMEventName(name) {
v.emit(fn)
}
case *js.Property:
if e.Name != nil && !e.Name.IsComputed() && isReactHandlerProp(e.Name.String()) {
v.emit(e.Value)
}
}
return v
}
// emit resolves expr to zero or more concrete functions and records a handler
// site for each. A generator is never a handler.
func (v *handlerVisitor) emit(expr js.IExpr) {
for _, fn := range resolveFuncValues(expr, v.funcs) {
if !fn.generator {
v.add(fn)
}
}
}
// isKeyHandlerProperty reports whether target is a member access naming a DOM
// keyboard on* property, in either dot or static-bracket form.
func isKeyHandlerProperty(target js.IExpr) bool {
name, _, ok := memberAccess(target)
return ok && isDOMHandlerProp(name)
}
// DOM on* properties are lowercase; the DOM does not fire a camelCase
// element.onKeyDown assignment.
func isDOMHandlerProp(name string) bool {
switch name {
case "onkeydown", "onkeypress", "onkeyup":
return true
default:
return false
}
}
// DOM event type strings passed to addEventListener are case-sensitive
// lowercase.
func isDOMEventName(name string) bool {
switch name {
case "keydown", "keypress", "keyup":
return true
default:
return false
}
}
// React object-literal handler props are camelCase.
func isReactHandlerProp(name string) bool {
switch name {
case "onKeyDown", "onKeyPress", "onKeyUp":
return true
default:
return false
}
}
// addEventListenerHandler matches both the receiver form el.addEventListener and
// the bare unshadowed-global form, returning the event name and handler value.
func addEventListenerHandler(call *js.CallExpr) (string, js.IExpr, bool) {
if len(call.Args.List) < 2 {
return "", nil, false
}
if call.Args.List[0].Rest || call.Args.List[1].Rest {
return "", nil, false
}
switch callee := call.X.(type) {
case *js.DotExpr:
if name, ok := staticStringOrIdent(callee.Y); !ok || name != "addEventListener" {
return "", nil, false
}
case *js.Var:
if !isGlobalRef(callee, "addEventListener") {
return "", nil, false
}
default:
return "", nil, false
}
eventName, ok := staticStringOrIdent(ungroupExpr(call.Args.List[0].Value))
if !ok {
return "", nil, false
}
return eventName, call.Args.List[1].Value, true
}
func ungroupExpr(expr js.IExpr) js.IExpr {
for {
group, ok := expr.(*js.GroupExpr)
if !ok {
return expr
}
expr = group.X
}
}
// isGlobalRef reports whether v is an unshadowed reference to the named global
// binding. The parser leaves free identifiers undeclared, so a NoDecl canonical
// identity with the expected name is a global; a local declaration of the same
// name has a declaration type and is therefore not the platform global.
func isGlobalRef(v *js.Var, name string) bool {
n, ok := globalName(v)
return ok && n == name
}
func firstParamVar(params js.Params) *js.Var {
if len(params.List) == 0 {
return nil
}
if v, ok := params.List[0].Binding.(*js.Var); ok {
return canonicalVar(v)
}
return nil
}
// resolveFuncValues returns the concrete functions expr can denote: a direct,
// possibly parenthesized function or arrow literal, or a same-file identifier
// whose collected values are functions.
func resolveFuncValues(expr js.IExpr, funcs map[*js.Var][]*funcInfo) []*funcInfo {
expr = ungroupExpr(expr)
if fn := literalFuncValue(expr); fn != nil {
return []*funcInfo{fn}
}
switch e := expr.(type) {
case *js.Var:
return funcs[canonicalVar(e)]
default:
return nil
}
}
// collectFuncValues maps each canonical variable to the function values assigned
// to it anywhere in the file: named declarations, declaration initializers, and
// plain assignments. Multiple values accumulate so a later union covers every
// possibility.
func collectFuncValues(ast *js.AST) map[*js.Var][]*funcInfo {
c := &funcValueCollector{funcs: map[*js.Var][]*funcInfo{}}
js.Walk(c, ast)
return c.funcs
}
type funcValueCollector struct {
funcs map[*js.Var][]*funcInfo
}
func (c *funcValueCollector) Exit(js.INode) {}
func (c *funcValueCollector) Enter(n js.INode) js.IVisitor {
if walkComputedClassName(c, n) {
return c
}
switch e := n.(type) {
case *js.FuncDecl:
if e.Name != nil {
c.bind(e.Name, newFuncInfo(e.Params, &e.Body, e.Generator, e.Async))
}
case *js.VarDecl:
for i := range e.List {
be := e.List[i]
target, ok := be.Binding.(*js.Var)
if !ok || be.Default == nil {
continue
}
for _, fn := range literalFuncValues(be.Default) {
c.bind(target, fn)
}
}
case *js.BinaryExpr:
if e.Op != js.EqToken {
return c
}
if target, ok := e.X.(*js.Var); ok {
for _, fn := range literalFuncValues(e.Y) {
c.bind(target, fn)
}
}
}
return c
}
func (c *funcValueCollector) bind(v *js.Var, fn *funcInfo) {
key := canonicalVar(v)
c.funcs[key] = append(c.funcs[key], fn)
}
// literalFuncValues returns a function value only for a direct, possibly
// parenthesized function or arrow literal. It does not chase identifier aliases,
// so binding collection cannot recurse without bound.
func literalFuncValues(expr js.IExpr) []*funcInfo {
if fn := literalFuncValue(ungroupExpr(expr)); fn != nil {
return []*funcInfo{fn}
}
return nil
}
func literalFuncValue(expr js.IExpr) *funcInfo {
switch e := expr.(type) {
case *js.FuncDecl:
return newFuncInfo(e.Params, &e.Body, e.Generator, e.Async)
case *js.ArrowFunc:
return newFuncInfo(e.Params, &e.Body, false, e.Async)
default:
}
return nil
}
func newFuncInfo(params js.Params, body *js.BlockStmt, generator, async bool) *funcInfo {
return &funcInfo{params: params, body: body, generator: generator, async: async}
}
// collectFunctionLocals records bindings owned by one invocation. They must not
// escape into the caller state when an inline callee returns.
func collectFunctionLocals(body *js.BlockStmt) map[*js.Var]bool {
c := &functionLocalCollector{locals: map[*js.Var]bool{}}
js.Walk(c, body)
return c.locals
}
type functionLocalCollector struct {
locals map[*js.Var]bool
}
func (c *functionLocalCollector) Exit(js.INode) {}
func (c *functionLocalCollector) Enter(n js.INode) js.IVisitor {
switch x := n.(type) {
case *js.BlockStmt:
for _, v := range x.Declared {
c.locals[canonicalVar(v)] = true
}
case *js.FuncDecl, *js.ArrowFunc, *js.MethodDecl:
return nil
}
return c
}
package jstaint
import (
"math"
"math/big"
"strconv"
"strings"
"sync/atomic"
"github.com/tdewolff/parse/v2/js"
)
// allocID identifies one abstract allocation instance: a deterministic source
// site plus a recency class. The two classes per site are the current instance
// and a summary of older instances, so the analyzer never invents an unbounded
// runtime-object identity.
type allocID struct {
site int
summary bool
}
// allocRef carries the call depth of the path through which an allocation was
// obtained. Heap objects remain keyed by allocation identity.
type allocRef struct {
id allocID
minDepth uint8
advance uint8
}
type allocSet map[allocRef]bool
// value is an abstract value at a program point: scalar taint carried directly
// plus the set of allocation instances the value may reference. Network sinks
// consume only the scalar part, because a URL, body, or header must be a string;
// an object reaches a sink only after a modeled serializer turns its reachable
// field taint into scalar taint.
type value struct {
scalar taintSet
allocs allocSet
// allocOnly is true only when every represented runtime alternative is an
// allocation. A branch that may instead produce an untracked clean primitive
// clears it, preventing an object spread or cycle check from acting definite.
allocOnly bool
// scheme is the URL scheme this value definitely carries when used as a
// destination, independent of taint. It gates whether a tainted URL is a
// network sink.
scheme schemeState
}
// object is the abstract contents of one allocation: statically named fields, an
// array-element field, and a wildcard field for writes whose key is not
// statically known.
type object struct {
fields map[string]value
must map[string]bool
elem value
wild value
wildReq value
elemMay bool
wildMay bool
elemMust bool
wildMust bool
array bool
// Receiver provenance for network-sink method calls. kind names the platform
// object this allocation represents; a generic object is never a sink.
kind objectKind
// XMLHttpRequest path state: whether an open has been seen on this path, and
// the taint remembered from that open's URL and any setRequestHeader values.
xhrOpened bool
xhrURL taintSet
xhrHeader taintSet
xhrScheme schemeState
// WebSocket path state: possibly open once a later callback can observe it,
// definitely closed only when closed on every merged path.
wsMaybeOpen bool
wsClosed bool
// owner is the writer token of the single state allowed to mutate this
// object in place. Zero means frozen: every state must copy before writing.
// clone() leaves it zero; installers set it. It is bookkeeping, not
// semantic state, so objectEqual and mergeObject ignore it.
owner uint64
}
func (o *object) clone() *object {
n := &object{
elem: o.elem,
wild: o.wild,
wildReq: o.wildReq,
elemMay: o.elemMay,
wildMay: o.wildMay,
elemMust: o.elemMust,
wildMust: o.wildMust,
array: o.array,
kind: o.kind,
xhrOpened: o.xhrOpened,
xhrURL: o.xhrURL,
xhrHeader: o.xhrHeader,
xhrScheme: o.xhrScheme,
wsMaybeOpen: o.wsMaybeOpen,
wsClosed: o.wsClosed,
}
if len(o.fields) != 0 {
n.fields = make(map[string]value, len(o.fields))
for k, v := range o.fields {
n.fields[k] = v
}
}
if len(o.must) != 0 {
n.must = make(map[string]bool, len(o.must))
for k := range o.must {
n.must[k] = true
}
}
return n
}
// stateMutSeq issues globally unique writer tokens. Uniqueness across
// goroutines is all that matters; the values never reach output.
var stateMutSeq atomic.Uint64
func nextMut() uint64 { return stateMutSeq.Add(1) }
// state is the flow-sensitive abstract store at a program point: per-variable
// values, transient receiver captures, and the heap of allocation contents.
// Branch forks clone it; merges union it; loop fixed points compare it.
//
// Clones share the three maps copy-on-write, so clone is O(1) instead of a
// deep heap copy at every branch fork. The shared flags mark maps another
// state may still reach, and the per-object owner token marks objects this
// state may mutate in place. Every write must go through setEnv/delEnv,
// setCapture/delCapture, mutObject/installObject/shareObject/dropObject, or
// an own* barrier; a direct map write on a shared state corrupts its siblings.
type state struct {
env map[*js.Var]value
// captures holds transient receivers across computed keys, right-hand sides,
// and call arguments. It is part of state so allocation promotion rewrites
// captured identities on every branch before the operation uses its receiver.
captures map[js.INode]value
heap map[allocID]*object
// mut is this state's writer token. An object is mutable in place only
// when o.owner == s.mut. Cloning hands the maps to a second state, so both
// sides take fresh tokens and thereby abandon in-place rights on every
// previously owned object. Invariant: heapShared implies no object has
// owner == s.mut, because installing an owned object first unshares the
// map and cloning re-tokens s.
mut uint64
// envVer, capsVer, and heapVer identify the underlying map storage. Two
// states with equal versions hold the same map, so merge and equality
// short-circuit; own* stamps a fresh version when it copies.
envVer uint64
capsVer uint64
heapVer uint64
envShared bool
capsShared bool
heapShared bool
continues bool
}
func newState() *state {
return &state{
env: map[*js.Var]value{},
captures: map[js.INode]value{},
heap: map[allocID]*object{},
mut: nextMut(),
envVer: nextMut(),
capsVer: nextMut(),
heapVer: nextMut(),
continues: true,
}
}
func (s *state) clone() *state {
s.mut = nextMut()
s.envShared, s.capsShared, s.heapShared = true, true, true
return &state{
env: s.env, captures: s.captures, heap: s.heap,
mut: nextMut(),
envVer: s.envVer, capsVer: s.capsVer, heapVer: s.heapVer,
envShared: true, capsShared: true, heapShared: true,
continues: s.continues,
}
}
// replaceWith overwrites s with the contents of src, keeping a stable *state
// identity across a may-execute merge. It adopts src's storage and writer
// token, so src must be a temporary that is never used again.
func (s *state) replaceWith(src *state) {
s.env, s.captures, s.heap = src.env, src.captures, src.heap
s.mut = src.mut
s.envVer, s.capsVer, s.heapVer = src.envVer, src.capsVer, src.heapVer
s.envShared, s.capsShared, s.heapShared = src.envShared, src.capsShared, src.heapShared
s.continues = src.continues
}
func (s *state) ownEnv() {
if !s.envShared {
return
}
env := make(map[*js.Var]value, len(s.env))
for k, v := range s.env {
env[k] = v
}
s.env = env
s.envVer = nextMut()
s.envShared = false
}
func (s *state) ownCaptures() {
if !s.capsShared {
return
}
caps := make(map[js.INode]value, len(s.captures))
for k, v := range s.captures {
caps[k] = v
}
s.captures = caps
s.capsVer = nextMut()
s.capsShared = false
}
func (s *state) ownHeap() {
if !s.heapShared {
return
}
heap := make(map[allocID]*object, len(s.heap))
for k, o := range s.heap {
heap[k] = o
}
s.heap = heap
s.heapVer = nextMut()
s.heapShared = false
}
func (s *state) setEnv(cv *js.Var, v value) {
s.ownEnv()
s.env[cv] = v
}
func (s *state) delEnv(cv *js.Var) {
if _, ok := s.env[cv]; !ok {
return
}
s.ownEnv()
delete(s.env, cv)
}
func (s *state) setCapture(node js.INode, v value) {
s.ownCaptures()
s.captures[node] = v
}
func (s *state) delCapture(node js.INode) {
if _, ok := s.captures[node]; !ok {
return
}
s.ownCaptures()
delete(s.captures, node)
}
// mutObject returns the object at id with in-place write rights for s, copying
// it first when any other state may still reach it.
func (s *state) mutObject(id allocID) *object {
o := s.heap[id]
if o == nil {
return nil
}
if o.owner != s.mut {
o = o.clone()
o.owner = s.mut
s.ownHeap()
s.heap[id] = o
}
return o
}
// installObject publishes an object s built or merged exclusively for itself,
// granting in-place write rights.
func (s *state) installObject(id allocID, o *object) {
o.owner = s.mut
s.ownHeap()
s.heap[id] = o
}
// shareObject publishes an object owned elsewhere without copying. Freezing
// the owner makes every later writer, including s, copy first.
func (s *state) shareObject(id allocID, o *object) {
o.owner = 0
s.ownHeap()
s.heap[id] = o
}
func (s *state) dropObject(id allocID) {
if _, ok := s.heap[id]; !ok {
return
}
s.ownHeap()
delete(s.heap, id)
}
func mergeAllocs(a, b allocSet) allocSet {
if len(a) == 0 {
return b
}
if len(b) == 0 {
return a
}
out := make(allocSet, len(a)+len(b))
for k := range a {
out[k] = true
}
for k := range b {
out[k] = true
}
return out
}
func mergeValue(a, b value) value {
return value{
scalar: mergeTaint(a.scalar, b.scalar),
allocs: mergeAllocs(a.allocs, b.allocs),
allocOnly: a.allocOnly && b.allocOnly,
scheme: mergeScheme(a.scheme, b.scheme),
}
}
// widenAbsentValue adds an untracked non-allocation alternative to v. A value
// missing on one path cannot retain a definite allocation or URL scheme.
func widenAbsentValue(v value) value {
v.allocOnly = false
v.scheme = schemeState{}
return v
}
// advanceCallValue moves a value across one user-defined call edge. Facts past
// depth 1 are discarded. Allocation identities remain available at a blocked
// depth so the callee can still apply object mutations without propagating taint.
func advanceCallValue(v value) value {
v.scalar = applyTaintDepth(v.scalar, 0, 1)
if len(v.allocs) != 0 {
refs := make(allocSet, len(v.allocs))
for ref := range v.allocs {
if ref.advance < 2 {
ref.advance++
}
refs[ref] = true
}
v.allocs = refs
}
return v
}
// returnValueAtDepthOne materializes the lazy allocation constraints accumulated
// while a depth-1 callee used an argument, then records the return edge without
// counting the same call twice.
func returnValueAtDepthOne(v value) value {
v.scalar = applyTaintDepth(v.scalar, 1, 0)
if len(v.allocs) == 0 {
return v
}
refs := make(allocSet, len(v.allocs))
for ref := range v.allocs {
refs[composeAllocRef(ref, allocRef{minDepth: 1})] = true
}
v.allocs = refs
return v
}
func applyRefDepth(v value, parent allocRef) value {
v.scalar = applyTaintDepth(v.scalar, parent.minDepth, parent.advance)
if len(v.allocs) != 0 && (parent.minDepth != 0 || parent.advance != 0) {
refs := make(allocSet, len(v.allocs))
for ref := range v.allocs {
refs[composeAllocRef(ref, parent)] = true
}
v.allocs = refs
}
return v
}
// composeAllocRef applies parent after child. Each constraint represents
// max(depth, minDepth) + advance, so a later minimum is discounted by an
// advance the child has already applied.
func composeAllocRef(child, parent allocRef) allocRef {
requiredMin := uint8(0)
if parent.minDepth > child.advance {
requiredMin = parent.minDepth - child.advance
}
if child.minDepth < requiredMin {
child.minDepth = requiredMin
}
child.advance += parent.advance
if child.advance > 2 {
child.advance = 2
}
return child
}
func applyTaintDepth(ts taintSet, minDepth, advance uint8) taintSet {
if len(ts) == 0 || (minDepth == 0 && advance == 0) {
return ts
}
out := make(taintSet, len(ts))
for fact, chain := range ts {
if fact.callDepth < minDepth {
fact.callDepth = minDepth
}
fact.callDepth += advance
if fact.callDepth <= 1 {
if existing, ok := out[fact]; !ok || shorterChain(chain, existing) {
out[fact] = chain
}
}
}
return out
}
func mergePresentValue(current value, present bool, next value) value {
if !present {
return next
}
return mergeValue(current, next)
}
func mergeObject(a, b *object) *object {
n := &object{
elemMay: a.elemMay || b.elemMay,
wildMay: a.wildMay || b.wildMay,
elemMust: a.elemMust && b.elemMust,
wildMust: a.wildMust && b.wildMust,
array: a.array,
// The two operands are the same allocation site, so kind agrees; a fresh
// generic instance from one path defers to the provenance of the other.
kind: mergeKind(a.kind, b.kind),
xhrOpened: a.xhrOpened || b.xhrOpened,
xhrURL: mergeTaint(a.xhrURL, b.xhrURL),
xhrHeader: mergeTaint(a.xhrHeader, b.xhrHeader),
xhrScheme: mergeXHRScheme(a, b),
wsMaybeOpen: a.wsMaybeOpen || b.wsMaybeOpen,
wsClosed: a.wsClosed && b.wsClosed,
}
if a.elemMay {
n.elem = a.elem
}
if b.elemMay {
n.elem = mergePresentValue(n.elem, a.elemMay, b.elem)
}
if a.wildMay {
n.wild = a.wild
}
if b.wildMay {
n.wild = mergePresentValue(n.wild, a.wildMay, b.wild)
}
if n.wildMust {
n.wildReq = mergeValue(a.wildReq, b.wildReq)
}
n.fields = make(map[string]value, len(a.fields)+len(b.fields))
for k, v := range a.fields {
if bv, ok := b.fields[k]; ok {
n.fields[k] = mergeValue(v, bv)
} else if b.wildMay {
n.fields[k] = mergeValue(v, b.wild)
} else {
n.fields[k] = v
}
}
for k, v := range b.fields {
if _, ok := n.fields[k]; !ok {
if a.wildMay {
n.fields[k] = mergeValue(a.wild, v)
} else {
n.fields[k] = v
}
}
}
for k := range a.must {
if b.must[k] {
if n.must == nil {
n.must = map[string]bool{}
}
n.must[k] = true
}
}
return n
}
// mergeXHRScheme joins only paths with an active request. A path where open has
// not run cannot contribute a destination scheme because send cannot issue a
// network request there.
func mergeXHRScheme(a, b *object) schemeState {
switch {
case !a.xhrOpened:
return b.xhrScheme
case !b.xhrOpened:
return a.xhrScheme
default:
return mergeScheme(a.xhrScheme, b.xhrScheme)
}
}
func mergeState(a, b *state) *state {
if !a.continues && b.continues {
return b.clone()
}
if a.continues && !b.continues {
return a.clone()
}
n := &state{mut: nextMut(), continues: a.continues || b.continues}
// Equal versions mean both sides still hold the identical shared map, so
// its self-merge is itself and the result becomes one more holder.
if a.envVer == b.envVer {
n.env, n.envVer = a.env, a.envVer
n.envShared, a.envShared, b.envShared = true, true, true
} else {
n.env = make(map[*js.Var]value, len(a.env)+len(b.env))
n.envVer = nextMut()
for k, v := range a.env {
if bv, ok := b.env[k]; ok {
if merged := mergeValue(v, bv); storable(merged) {
n.env[k] = merged
}
} else {
v = widenAbsentValue(v)
if storable(v) {
n.env[k] = v
}
}
}
for k, v := range b.env {
if _, ok := a.env[k]; !ok {
v = widenAbsentValue(v)
if storable(v) {
n.env[k] = v
}
}
}
}
if a.capsVer == b.capsVer {
n.captures, n.capsVer = a.captures, a.capsVer
n.capsShared, a.capsShared, b.capsShared = true, true, true
} else {
n.captures = make(map[js.INode]value, len(a.captures)+len(b.captures))
n.capsVer = nextMut()
for node, v := range a.captures {
if bv, ok := b.captures[node]; ok {
n.captures[node] = mergeValue(v, bv)
} else {
n.captures[node] = widenAbsentValue(v)
}
}
for node, v := range b.captures {
if _, ok := a.captures[node]; !ok {
n.captures[node] = widenAbsentValue(v)
}
}
}
if a.heapVer == b.heapVer {
n.heap, n.heapVer = a.heap, a.heapVer
n.heapShared, a.heapShared, b.heapShared = true, true, true
return n
}
n.heap = make(map[allocID]*object, len(a.heap)+len(b.heap))
n.heapVer = nextMut()
// Objects present on one side, or pointer-identical on both, are shared
// into the merged state frozen; only genuinely diverged objects are merged
// into a fresh object the result owns.
for k, oa := range a.heap {
if ob, ok := b.heap[k]; ok {
if oa == ob {
oa.owner = 0
n.heap[k] = oa
} else {
m := mergeObject(oa, ob)
m.owner = n.mut
n.heap[k] = m
}
} else {
oa.owner = 0
n.heap[k] = oa
}
}
for k, ob := range b.heap {
if _, ok := a.heap[k]; !ok {
ob.owner = 0
n.heap[k] = ob
}
}
return n
}
func allocsEqual(a, b allocSet) bool {
if len(a) != len(b) {
return false
}
for k := range a {
if !b[k] {
return false
}
}
return true
}
func valueEqual(a, b value) bool {
return a.allocOnly == b.allocOnly && a.scheme == b.scheme &&
taintEqual(a.scalar, b.scalar) && allocsEqual(a.allocs, b.allocs)
}
func objectEqual(a, b *object) bool {
if a.array != b.array || a.elemMay != b.elemMay || a.wildMay != b.wildMay ||
a.elemMust != b.elemMust || a.wildMust != b.wildMust ||
a.kind != b.kind || a.xhrOpened != b.xhrOpened ||
a.wsMaybeOpen != b.wsMaybeOpen || a.wsClosed != b.wsClosed ||
a.xhrScheme != b.xhrScheme ||
!taintEqual(a.xhrURL, b.xhrURL) || !taintEqual(a.xhrHeader, b.xhrHeader) ||
len(a.must) != len(b.must) || !valueEqual(a.elem, b.elem) ||
!valueEqual(a.wildReq, b.wildReq) ||
!valueEqual(a.wild, b.wild) || len(a.fields) != len(b.fields) {
return false
}
for k := range a.must {
if !b.must[k] {
return false
}
}
for k, va := range a.fields {
vb, ok := b.fields[k]
if !ok || !valueEqual(va, vb) {
return false
}
}
return true
}
func stateEqual(a, b *state) bool {
if a.continues != b.continues || len(a.env) != len(b.env) ||
len(a.captures) != len(b.captures) || len(a.heap) != len(b.heap) {
return false
}
if a.envVer != b.envVer {
for k, va := range a.env {
vb, ok := b.env[k]
if !ok || !valueEqual(va, vb) {
return false
}
}
}
if a.capsVer != b.capsVer {
for node, va := range a.captures {
vb, ok := b.captures[node]
if !ok || !valueEqual(va, vb) {
return false
}
}
}
if a.heapVer != b.heapVer {
for k, oa := range a.heap {
ob, ok := b.heap[k]
if !ok {
return false
}
if oa == ob {
continue
}
if !objectEqual(oa, ob) {
return false
}
}
}
return true
}
// fieldKind selects which slot of an object a member access reaches.
type fieldKind uint8
const (
fieldNamed fieldKind = iota
fieldElem
fieldWild
)
type fieldKey struct {
kind fieldKind
name string
}
// fieldKeyOf classifies a member-access key expression. A static string or
// identifier names a field, a numeric literal index is an array element, and an
// unresolved computed key is the wildcard.
func fieldKeyOf(key js.IExpr) fieldKey {
key = ungroupExpr(key)
if name, ok := staticStringOrIdent(key); ok {
if isArrayIndexName(name) {
return fieldKey{kind: fieldElem, name: name}
}
return fieldKey{kind: fieldNamed, name: name}
}
if name, ok := numericPropertyNameOf(key); ok {
if isArrayIndexName(name) {
return fieldKey{kind: fieldElem, name: name}
}
return fieldKey{kind: fieldNamed, name: name}
}
return fieldKey{kind: fieldWild}
}
func numericPropertyNameOf(expr js.IExpr) (string, bool) {
negative := false
for {
u, ok := ungroupExpr(expr).(*js.UnaryExpr)
if !ok || (u.Op != js.PosToken && u.Op != js.NegToken) {
break
}
if u.Op == js.NegToken {
negative = !negative
}
expr = u.X
}
lit, ok := ungroupExpr(expr).(*js.LiteralExpr)
if !ok || (lit.TokenType != js.IntegerToken && lit.TokenType != js.DecimalToken) {
return "", false
}
name := numericPropertyName(lit)
if name == "" {
return "", false
}
if negative && name != "0" {
name = "-" + name
}
return name, true
}
func numericPropertyName(lit *js.LiteralExpr) string {
raw := strings.ReplaceAll(string(lit.Data), "_", "")
if strings.HasSuffix(raw, "n") {
integer := new(big.Int)
if _, ok := integer.SetString(strings.TrimSuffix(raw, "n"), 0); ok {
return integer.String()
}
return ""
}
if lit.TokenType == js.IntegerToken {
if n, err := strconv.ParseUint(raw, 0, 64); err == nil {
return jsNumberPropertyName(float64(n))
}
}
n, err := strconv.ParseFloat(raw, 64)
if err != nil || math.IsInf(n, 0) || math.IsNaN(n) {
return ""
}
return jsNumberPropertyName(n)
}
func jsNumberPropertyName(n float64) string {
if n == 0 {
return "0"
}
abs := math.Abs(n)
if abs >= 1e-6 && abs < 1e21 {
return strconv.FormatFloat(n, 'f', -1, 64)
}
s := strconv.FormatFloat(n, 'e', -1, 64)
parts := strings.SplitN(s, "e", 2)
exponent, err := strconv.Atoi(parts[1])
if err != nil {
return ""
}
return parts[0] + "e" + fmtSignedExponent(exponent)
}
func fmtSignedExponent(exponent int) string {
if exponent >= 0 {
return "+" + strconv.Itoa(exponent)
}
return strconv.Itoa(exponent)
}
// isArrayIndexName reports whether a string is a canonical JavaScript array
// index. Bracket access coerces both 0 and "0" to the same property key.
func isArrayIndexName(name string) bool {
if name == "" || (len(name) > 1 && name[0] == '0') {
return false
}
for i := range name {
if name[i] < '0' || name[i] > '9' {
return false
}
}
n, err := strconv.ParseUint(name, 10, 32)
return err == nil && n < 1<<32-1
}
// getField reads one field of an object. A static named read and a wildcard read
// both consume the wildcard, because a wildcard write may have set any property.
func getField(o *object, key fieldKey) (value, bool) {
switch key.kind {
case fieldNamed:
if v, ok := o.fields[key.name]; ok {
return v, o.must[key.name]
}
if o.wildMay {
return o.wild, false
}
return value{}, false
case fieldElem:
if key.name != "" {
if v, ok := o.fields[key.name]; ok {
return v, o.must[key.name]
}
}
var out value
have := false
if o.elemMay {
out, have = mergePresentValue(out, have, o.elem), true
}
if o.wildMay {
out = mergePresentValue(out, have, o.wild)
}
return out, false
default:
var out value
have := false
if o.elemMay {
out, have = mergePresentValue(out, have, o.elem), true
}
if o.wildMay {
out, have = mergePresentValue(out, have, o.wild), true
}
for _, fv := range o.fields {
out, have = mergePresentValue(out, have, fv), true
}
return out, false
}
}
func (o *object) setNamed(name string, v value, strong bool) {
if o.fields == nil {
o.fields = map[string]value{}
}
if strong {
o.fields[name] = v
// A preceding wildcard write may have selected this exact property, so it
// is no longer certain that an unresolved field remains after overwrite.
o.wildMust = false
o.wildReq = value{}
if o.must == nil {
o.must = map[string]bool{}
}
o.must[name] = true
} else {
if o.wildMust {
// The weak write may select the required unresolved property. Preserve
// both its old value and the overwrite as runtime alternatives.
o.wildReq = mergeValue(o.wildReq, v)
}
old, ok := o.fields[name]
if !ok {
old = o.wild
}
o.fields[name] = mergeValue(old, v)
}
}
func (o *object) deleteNamed(name string) {
// The required unresolved property may be the deleted name. Other unresolved
// properties remain possible, but none remains certain after this delete.
o.wildMust = false
o.wildReq = value{}
if o.wildMay {
// Keep a clean tombstone so the deleted property does not expose an older
// unresolved write. A later wildcard write folds into the tombstone again.
if o.fields == nil {
o.fields = map[string]value{}
}
o.fields[name] = value{}
} else {
delete(o.fields, name)
}
delete(o.must, name)
}
func (o *object) weakElem(v value, definite bool) {
o.elem, o.elemMay = mergePresentValue(o.elem, o.elemMay, v), true
if definite {
o.elemMust = true
}
}
// writeWild applies a write whose key is unresolved. Existing named fields must
// absorb it because the key may select any of them; a later strong named write
// can then replace that field without clearing the wildcard for other names.
func (o *object) writeWild(v value, definite bool) {
o.wild, o.wildMay = mergePresentValue(o.wild, o.wildMay, v), true
if definite {
o.wildMust = true
o.wildReq = v
} else if o.wildMust {
o.wildReq = mergeValue(o.wildReq, v)
}
for k, fv := range o.fields {
o.fields[k] = mergeValue(fv, v)
}
}
package jstaint
import (
"strconv"
"github.com/tdewolff/parse/v2/js"
)
// rootSite is a reachable execution root: a callback whose body is analyzed with
// callback-published shared state. eventVar is the canonical keyboard-event
// parameter for a keyboard handler, or nil for a timer, listener, or socket
// callback whose parameters are not keystroke sources.
type rootSite struct {
fn *funcInfo
eventVar *js.Var
argCount int
}
type reachableFunction struct {
body *js.BlockStmt
params *js.Params
scanBody bool
scanBindings bool
defaultFrom int
defaultTo int
}
type reachableCall struct {
fn *funcInfo
provided int
}
// discoverReachableRoots returns the callback roots reachable from the module top
// level. Reachability follows callback registrations and direct same-file calls,
// but never descends into a nested function body until that function is itself
// registered or called. A function that is only declared is therefore not a root,
// so a sink reachable only inside it is never analyzed.
func (a *analysis) discoverReachableRoots(ast *js.AST) []rootSite {
var roots []rootSite
seenRoot := map[*funcInfo]int{}
bodyScanned := map[*funcInfo]bool{}
bindingsScanned := map[*funcInfo]bool{}
defaultFrom := map[*funcInfo]int{}
queue := []reachableFunction{{body: &ast.BlockStmt, scanBody: true}}
enqueue := func(fn *funcInfo, provided int) {
if fn.generator || fn.body == nil {
return
}
job := reachableFunction{body: fn.body, params: &fn.params}
if !bodyScanned[fn] {
bodyScanned[fn] = true
job.scanBody = true
}
if !bindingsScanned[fn] {
bindingsScanned[fn] = true
job.scanBindings = true
}
if provided < 0 {
provided = 0
}
if provided > len(fn.params.List) {
provided = len(fn.params.List)
}
previous, ok := defaultFrom[fn]
if !ok {
previous = len(fn.params.List)
}
if provided < previous {
job.defaultFrom = provided
job.defaultTo = previous
defaultFrom[fn] = provided
}
if job.scanBody || job.scanBindings || job.defaultFrom < job.defaultTo {
queue = append(queue, job)
}
}
for len(queue) > 0 {
current := queue[len(queue)-1]
queue = queue[:len(queue)-1]
regs, calls := a.scanFunctionLevel(current)
for _, r := range regs {
if index, ok := seenRoot[r.fn]; ok {
if roots[index].eventVar == nil && r.eventVar != nil {
roots[index].eventVar = r.eventVar
}
if r.argCount < roots[index].argCount {
roots[index].argCount = r.argCount
}
} else {
seenRoot[r.fn] = len(roots)
roots = append(roots, r)
}
enqueue(r.fn, r.argCount)
}
for _, call := range calls {
enqueue(call.fn, call.provided)
}
}
return roots
}
// scanFunctionLevel walks one function body's own code, collecting callback
// registrations and directly called same-file functions. It does not descend
// into nested function literals, whose bodies belong to their own reachability.
func (a *analysis) scanFunctionLevel(fn reachableFunction) ([]rootSite, []reachableCall) {
s := &funcLevelScanner{a: a}
if fn.params != nil {
if fn.scanBindings {
s.scanParamBindings(fn.params)
}
for i := fn.defaultFrom; i < fn.defaultTo; i++ {
js.Walk(s, fn.params.List[i].Default)
}
}
if fn.scanBody {
for i := range fn.body.List {
js.Walk(s, fn.body.List[i])
}
}
return s.regs, s.calls
}
type funcLevelScanner struct {
a *analysis
regs []rootSite
calls []reachableCall
}
func (s *funcLevelScanner) Exit(js.INode) {}
func (s *funcLevelScanner) Enter(n js.INode) js.IVisitor {
if !s.a.alive() {
return nil
}
if walkComputedClassName(s, n) {
return s
}
switch e := n.(type) {
case *js.FuncDecl:
// A nested function's body is reachable only when it is registered or
// called, which is decided at this level, not by lexical nesting.
return nil
case *js.ArrowFunc:
return nil
case *js.MethodDecl:
js.Walk(s, e.Name.Computed)
return nil
case *js.ClassDecl:
s.scanClassDefinition(e)
return nil
case *js.BinaryExpr:
if e.Op == js.EqToken {
s.registerAssignment(e)
}
case *js.CallExpr:
s.registerCall(e)
case *js.Property:
if e.Name != nil && !e.Name.IsComputed() && isReactHandlerProp(e.Name.String()) {
s.addRoots(e.Value, true, 1)
}
}
return s
}
// scanParamBindings visits computed keys and nested defaults, which can execute
// even when the invocation supplied the containing top-level argument.
func (s *funcLevelScanner) scanParamBindings(params *js.Params) {
for i := range params.List {
s.scanParamBinding(params.List[i].Binding)
}
s.scanParamBinding(params.Rest)
}
func (s *funcLevelScanner) scanParamBinding(binding js.IBinding) {
switch b := binding.(type) {
case *js.BindingArray:
for i := range b.List {
element := &b.List[i]
js.Walk(s, element.Default)
s.scanParamBinding(element.Binding)
}
s.scanParamBinding(b.Rest)
case *js.BindingObject:
for i := range b.List {
item := &b.List[i]
if item.Key != nil {
js.Walk(s, item.Key.Computed)
}
js.Walk(s, item.Value.Default)
s.scanParamBinding(item.Value.Binding)
}
s.scanParamBinding(b.Rest)
}
}
// scanClassDefinition visits only the expressions that run while a class is
// defined. Instance field initializers and method bodies do not execute until a
// construction or method call that version 1 does not model.
func (s *funcLevelScanner) scanClassDefinition(class *js.ClassDecl) {
js.Walk(s, class.Extends)
for i := range class.List {
item := &class.List[i]
if item.Method != nil {
js.Walk(s, item.Method.Name.Computed)
} else if item.StaticBlock == nil {
js.Walk(s, item.Name.Computed)
}
}
for i := range class.List {
item := &class.List[i]
if item.StaticBlock != nil {
js.Walk(s, item.StaticBlock)
} else if item.Method == nil && item.Static {
js.Walk(s, item.Init)
}
}
}
// registerAssignment records a callback assigned to a DOM keyboard on-property or
// to a static onopen/onmessage socket property.
func (s *funcLevelScanner) registerAssignment(e *js.BinaryExpr) {
name, _, ok := memberAccess(e.X)
if !ok {
return
}
switch {
case isDOMHandlerProp(name):
s.addRoots(e.Y, true, 1)
case isSocketCallbackProp(name):
s.addRoots(e.Y, false, 1)
}
}
func (s *funcLevelScanner) registerCall(call *js.CallExpr) {
if name, fn, ok := addEventListenerHandler(call); ok {
s.addRoots(fn, isDOMEventName(name), 1)
return
}
if fn, args, ok := scheduledCallback(call); ok {
s.addRoots(fn, false, args)
return
}
// A direct call to a same-file function makes that function reachable, so its
// own registrations count.
provided := definiteArgCount(call.Args.List)
for _, fn := range resolveFuncValues(ungroupExpr(call.X), s.a.funcs) {
s.calls = append(s.calls, reachableCall{fn: fn, provided: provided})
}
}
func definiteArgCount(args []js.Arg) int {
for i := range args {
if args[i].Rest {
return 0
}
}
return len(args)
}
// addRoots resolves an expression to its concrete callback functions and records
// each as a root. keyHandler marks whether the first parameter is a keystroke
// event.
func (s *funcLevelScanner) addRoots(expr js.IExpr, keyHandler bool, argCount int) {
for _, fn := range resolveFuncValues(expr, s.a.funcs) {
if fn.generator {
continue
}
var ev *js.Var
if keyHandler {
ev = firstParamVar(fn.params)
}
s.regs = append(s.regs, rootSite{fn: fn, eventVar: ev, argCount: argCount})
}
}
// isSocketCallbackProp reports whether a property assignment installs a socket
// event callback. Receiver provenance is refined later; here it only widens the
// reachable set, and findings still require a keystroke to reach a sink.
func isSocketCallbackProp(name string) bool {
switch name {
case "onopen", "onmessage":
return true
default:
return false
}
}
// scheduledCallback returns the function scheduled by a timer or microtask
// registration whose first argument is a callback.
func scheduledCallback(call *js.CallExpr) (js.IExpr, int, bool) {
name, ok := scheduledCallbackName(call.X)
if !ok {
return nil, 0, false
}
if len(call.Args.List) == 0 || call.Args.List[0].Rest {
return nil, 0, false
}
if name == "requestAnimationFrame" {
return call.Args.List[0].Value, 1, true
}
if name == "queueMicrotask" {
return call.Args.List[0].Value, 0, true
}
if len(call.Args.List) <= 2 {
return call.Args.List[0].Value, 0, true
}
return call.Args.List[0].Value, definiteArgCount(call.Args.List[2:]), true
}
func scheduledCallbackName(expr js.IExpr) (string, bool) {
switch callee := ungroupExpr(expr).(type) {
case *js.Var:
name, ok := globalName(callee)
return name, ok && isScheduledCallbackName(name)
case *js.DotExpr, *js.IndexExpr:
name, base, ok := memberAccess(expr)
return name, ok && isGlobalObject(base) && isScheduledCallbackName(name)
default:
return "", false
}
}
func isScheduledCallbackName(name string) bool {
switch name {
case "setTimeout", "setInterval", "queueMicrotask", "requestAnimationFrame":
return true
default:
return false
}
}
// sharedState projects a state onto its file-scope portion: the shared variables
// plus the heap reachable from them. This is the may-state that callbacks publish
// to and consume from one another.
func (a *analysis) sharedState(src *state) *state {
out := newState()
var roots []value
for cv, v := range src.env {
if a.isShared(cv) {
out.setEnv(cv, v)
roots = append(roots, v)
}
}
copyReachableHeap(src, out, roots)
return out
}
// publishShared merges a callback's exit shared writes into the global may-state
// and returns the updated may-state.
func (a *analysis) publishShared(global, src *state) *state {
out := global.clone()
var roots []value
for cv, v := range out.env {
if !a.isShared(cv) {
continue
}
if _, ok := src.env[cv]; ok {
continue
}
v = widenAbsentValue(v)
if storable(v) {
out.setEnv(cv, v)
} else {
out.delEnv(cv)
}
}
for cv, v := range src.env {
if !a.isShared(cv) {
continue
}
if ex, ok := out.env[cv]; ok {
merged := mergeValue(ex, v)
if storable(merged) {
out.setEnv(cv, merged)
} else {
out.delEnv(cv)
}
} else {
v = widenAbsentValue(v)
if storable(v) {
out.setEnv(cv, v)
}
}
roots = append(roots, v)
}
mergeReachableHeap(src, out, roots)
return out
}
func (a *analysis) isShared(cv *js.Var) bool {
return a.sharedVars[cv] || cv.Decl == js.NoDecl
}
// copyReachableHeap copies into dst every heap object reachable from the given
// values in src.
func copyReachableHeap(src, dst *state, roots []value) {
for _, id := range reachableAllocs(src, roots) {
if o := src.heap[id]; o != nil {
dst.shareObject(id, o)
}
}
}
// mergeReachableHeap merges into dst every heap object reachable from the given
// values in src, unioning with any object already present.
func mergeReachableHeap(src, dst *state, roots []value) {
for _, id := range reachableAllocs(src, roots) {
o := src.heap[id]
if o == nil {
continue
}
if ex, ok := dst.heap[id]; ok {
if ex != o {
dst.installObject(id, mergeObject(ex, o))
}
} else {
dst.shareObject(id, o)
}
}
}
// reachableAllocs returns every allocation reachable from roots through the heap.
func reachableAllocs(st *state, roots []value) []allocID {
seen := map[allocID]bool{}
var order []allocID
var stack []allocID
push := func(v value) {
for ref := range v.allocs {
id := ref.id
if !seen[id] {
seen[id] = true
order = append(order, id)
stack = append(stack, id)
}
}
}
for _, v := range roots {
push(v)
}
for len(stack) > 0 {
id := stack[len(stack)-1]
stack = stack[:len(stack)-1]
o := st.heap[id]
if o == nil {
continue
}
for _, fv := range o.fields {
push(fv)
}
push(o.elem)
push(o.wild)
push(o.wildReq)
}
return order
}
// evalUserCall inlines a same-file function call at depth 1. It returns the
// call's tainted return value and true when the callee is a modeled user
// function; otherwise it returns false so the caller falls back to built-in
// handling. A fact already at depth 1 cannot enter another callee.
func (a *analysis) evalUserCall(callee js.IExpr, args []value, st *state) (value, bool) {
if a.callDepth != 0 {
return value{}, false
}
fns := resolveFuncValues(callee, a.funcs)
if len(fns) == 0 {
return value{}, false
}
var ret value
haveRet := false
handled := false
unmodeledAlternative := false
base := st.clone()
effects := base.clone()
for _, fn := range fns {
if fn.generator || fn.body == nil || a.inProgress[fn] {
unmodeledAlternative = true
continue
}
handled = true
branch := base.clone()
branchRet := a.analyzeCallee(fn, args, branch)
if fn.async {
branchRet = value{}
}
ret, haveRet = mergePresentValue(ret, haveRet, branchRet), true
effects = mergeState(effects, branch)
if !a.alive() {
break
}
}
if handled {
st.replaceWith(effects)
if unmodeledAlternative {
ret = mergeValue(ret, value{})
}
}
// A user call is not a scheme-preserving operation. Receiver allocations and
// scalar taint return, but destination classification becomes unknown.
ret.scheme = schemeState{}
return ret, handled
}
// analyzeCallee analyzes one callee body at depth 1 with its parameters bound to
// the argument values, then publishes its heap effects back to the caller and
// returns the union of its return values.
func (a *analysis) analyzeCallee(fn *funcInfo, args []value, st *state) value {
caller := st.clone()
sub := st.clone()
suspensionStart := len(a.suspensions)
previousDepth := a.callDepth
a.callDepth = 1
a.inProgress[fn] = true
a.retStack = append(a.retStack, value{})
a.returnExits = append(a.returnExits, nil)
a.bindParams(fn.params, args, sub)
sub = a.analyzeBlock(fn.body, sub)
exits := a.popReturnExits(sub)
a.retStack = a.retStack[:len(a.retStack)-1]
var ret value
haveRet := false
for i := range exits {
var exitRet value
if exits[i].returned {
exitRet = exits[i].ret
}
ret, haveRet = mergePresentValue(ret, haveRet, exitRet), true
}
// The callee shares the caller's heap identities, so its object mutations and
// outer-variable writes flow back by unioning its exit state. Invocation-local
// bindings are removed so they cannot seed a later call.
effects := mergeExitStates(exits)
if fn.async && len(a.suspensions) > suspensionStart {
// The complete exit state is callback-published because the continuation
// can run later. A second pass stops each path at its first await and keeps
// no-await alternatives, which is the state visible when the promise returns.
a.suspensions = append(a.suspensions, effects)
effects = a.analyzeAsyncPrefix(fn, args, caller)
}
a.inProgress[fn] = false
a.callDepth = previousDepth
a.removeFunctionLocals(fn, effects)
out := mergeState(caller, effects)
st.replaceWith(out)
return returnValueAtDepthOne(ret)
}
func (a *analysis) analyzeAsyncPrefix(fn *funcInfo, args []value, caller *state) *state {
sub := caller.clone()
previousStop := a.stopAtAwait
a.stopAtAwait = true
a.retStack = append(a.retStack, value{})
a.returnExits = append(a.returnExits, nil)
a.bindParams(fn.params, args, sub)
sub = a.analyzeBlock(fn.body, sub)
exits := a.popReturnExits(sub)
a.retStack = a.retStack[:len(a.retStack)-1]
a.stopAtAwait = previousStop
return mergeExitStates(exits)
}
func mergeExitStates(exits []functionExit) *state {
if len(exits) == 0 {
return newState()
}
out := exits[0].state.clone()
for i := 1; i < len(exits); i++ {
out = mergeState(out, exits[i].state)
}
return out
}
func (a *analysis) removeFunctionLocals(fn *funcInfo, st *state) {
for cv := range a.functionLocals(fn) {
if !a.isShared(cv) {
st.delEnv(cv)
}
}
}
func (a *analysis) functionLocals(fn *funcInfo) map[*js.Var]bool {
if locals, ok := a.localCache[fn.body]; ok {
return locals
}
locals := collectFunctionLocals(fn.body)
a.localCache[fn.body] = locals
return locals
}
// bindParams applies positional, destructured, defaulted, and rest bindings as
// strong invocation-local updates. Rest arguments use a bounded synthetic array
// allocation so only explicit serialization turns their fields into a scalar.
func (a *analysis) bindParams(params js.Params, args []value, st *state) {
for i := range params.List {
p := ¶ms.List[i]
v, ok := p.Binding.(*js.Var)
if !ok {
switch {
case i < len(args):
a.bindParamPattern(p.Binding, advanceCallValue(args[i]), st)
case p.Default != nil:
a.bindParamPattern(p.Binding, a.evalExpr(p.Default, st), st)
default:
a.bindParamPattern(p.Binding, value{}, st)
}
continue
}
cv := canonicalVar(v)
switch {
case i < len(args):
a.bindVar(st, cv, advanceCallValue(args[i]))
case p.Default != nil:
a.bindVar(st, cv, a.evalExpr(p.Default, st))
default:
st.delEnv(cv)
}
}
if params.Rest != nil {
start := len(params.List)
if start > len(args) {
start = len(args)
}
a.bindRestParam(params.Rest, args[start:], st)
}
}
func (a *analysis) bindParamPattern(binding js.IBinding, arg value, st *state) {
switch b := binding.(type) {
case *js.Var:
a.bindVar(st, canonicalVar(b), arg)
case *js.BindingArray:
for i := range b.List {
be := &b.List[i]
if be.Binding == nil {
continue
}
key := fieldKey{kind: fieldElem, name: strconv.Itoa(i)}
a.bindParamElement(be, a.readField(st, arg, key), fieldDefinitelyPresent(st, arg, key), st)
}
if b.Rest != nil {
a.bindArrayRestParam(b.Rest, arg, len(b.List), st)
}
case *js.BindingObject:
excluded := make(map[string]bool, len(b.List))
for i := range b.List {
item := &b.List[i]
key := a.bindingObjectKey(item, st)
if key.kind != fieldWild {
excluded[key.name] = true
}
a.bindParamElement(
&item.Value,
a.readField(st, arg, key),
fieldDefinitelyPresent(st, arg, key),
st,
)
}
if b.Rest != nil {
a.bindObjectRestParam(b.Rest, arg, excluded, st)
}
}
}
func (a *analysis) bindingObjectKey(item *js.BindingObjectItem, st *state) fieldKey {
if item.Key != nil {
if item.Key.Computed != nil {
a.evalExpr(item.Key.Computed, st)
return fieldKeyOf(ungroupExpr(item.Key.Computed))
}
return fieldKeyOf(&item.Key.Literal)
}
if v, ok := item.Value.Binding.(*js.Var); ok {
return fieldKey{kind: fieldNamed, name: string(v.Name())}
}
return fieldKey{kind: fieldWild}
}
func (a *analysis) bindParamElement(be *js.BindingElement, arg value, present bool, st *state) {
if be.Binding == nil {
return
}
if be.Default == nil || present {
a.bindParamPattern(be.Binding, arg, st)
return
}
defaultSt := st.clone()
a.bindParamPattern(be.Binding, a.evalExpr(be.Default, defaultSt), defaultSt)
argSt := st.clone()
a.bindParamPattern(be.Binding, arg, argSt)
st.replaceWith(mergeState(defaultSt, argSt))
}
func (a *analysis) bindRestParam(binding js.IBinding, args []value, st *state) {
fresh := &object{array: true}
for i := range args {
fresh.setNamed(strconv.Itoa(i), advanceCallValue(args[i]), true)
a.fact()
}
a.bindRestObject(binding, fresh, st)
}
func (a *analysis) bindArrayRestParam(binding js.IBinding, arg value, start int, st *state) {
fresh := &object{array: true}
if len(arg.scalar) != 0 {
fresh.weakElem(value{scalar: arg.scalar}, false)
}
ustart := uint64(start) // #nosec G115 -- start is a parameter index, always non-negative
for ref := range arg.allocs {
o := st.heap[ref.id]
if o == nil || !o.array {
continue
}
for name, field := range o.fields {
if !isArrayIndexName(name) {
continue
}
index, err := strconv.ParseUint(name, 10, 32)
if err != nil || index < ustart {
continue
}
fresh.setNamed(strconv.FormatUint(index-ustart, 10), applyRefDepth(field, ref), false)
}
if o.elemMay {
fresh.weakElem(applyRefDepth(o.elem, ref), false)
}
if o.wildMay {
fresh.weakElem(applyRefDepth(o.wild, ref), false)
}
}
a.bindRestObject(binding, fresh, st)
}
func (a *analysis) bindObjectRestParam(binding js.IBinding, arg value, excluded map[string]bool, st *state) {
fresh := &object{}
if len(arg.scalar) != 0 {
fresh.writeWild(value{scalar: arg.scalar}, false)
}
for ref := range arg.allocs {
o := st.heap[ref.id]
if o == nil {
continue
}
for name, field := range o.fields {
if excluded[name] {
continue
}
fresh.setNamed(name, applyRefDepth(field, ref), false)
}
if o.elemMay {
fresh.weakElem(applyRefDepth(o.elem, ref), false)
}
if o.wildMay {
fresh.writeWild(applyRefDepth(o.wild, ref), false)
}
}
a.bindRestObject(binding, fresh, st)
}
func (a *analysis) bindRestObject(binding js.IBinding, fresh *object, st *state) {
site, ok := a.restSites[binding]
if !ok {
a.evalBindingPattern(binding, st)
return
}
a.promoteCurrent(st, site)
rest := a.installLiteral(st, site, a.loopDepth > 0, fresh)
a.bindParamPattern(binding, rest, st)
}
func numberRestSites(ast *js.AST) map[js.IBinding]int {
n := &restSiteNumberer{sites: map[js.IBinding]int{}, next: -1}
js.Walk(n, ast)
return n.sites
}
type restSiteNumberer struct {
sites map[js.IBinding]int
next int
}
func (n *restSiteNumberer) Exit(js.INode) {}
func (n *restSiteNumberer) Enter(node js.INode) js.IVisitor {
switch binding := node.(type) {
case *js.Params:
n.number(binding.Rest)
case *js.BindingArray:
n.number(binding.Rest)
case *js.BindingObject:
if binding.Rest != nil {
n.number(binding.Rest)
}
}
return n
}
func (n *restSiteNumberer) number(binding js.IBinding) {
if binding == nil {
return
}
if _, exists := n.sites[binding]; exists {
return
}
n.sites[binding] = n.next
n.next--
}
// Package jstaint reports keystroke values that reach a network sink in a
// JavaScript source file.
//
// It exists because the regex keylogger rule can only match a syntactically
// direct capture. A keylogger that moves the keystroke through variables needs
// variable identity to detect, which requires backreferences that neither
// YARA-X nor RE2 provides.
//
// The package is pure: bytes in, report out. It touches no filesystem, config,
// store, or process global, so callers own every I/O and persistence decision.
package jstaint
import (
"bytes"
"context"
"errors"
"fmt"
"strings"
"unicode"
"unicode/utf8"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/parse/v2/js"
)
// Status is the outcome of an analysis attempt. Callers must not infer a clean
// file from an empty result slice: only StatusAnalyzed means the content was
// examined end to end. StatusNotCandidate excludes unsupported documents and
// content without flow tokens; all remaining statuses are coverage gaps.
type Status uint8
const (
// StatusNotCandidate means the content lacks the required flow tokens or
// is a recognized non-JavaScript document. Embedded scripts are not examined.
StatusNotCandidate Status = iota
// StatusAnalyzed means the content was parsed and examined to completion.
StatusAnalyzed
StatusOversize
StatusParseError
StatusResourceLimit
StatusCanceled
StatusPanic
)
// String names the status for metrics labels and operator-facing text.
func (s Status) String() string {
switch s {
case StatusNotCandidate:
return "not_candidate"
case StatusAnalyzed:
return "analyzed"
case StatusOversize:
return "oversize"
case StatusParseError:
return "parse_error"
case StatusResourceLimit:
return "resource_limit"
case StatusCanceled:
return "canceled"
case StatusPanic:
return "panic"
}
return "unknown"
}
// MaxSourceBytes bounds the complete source an analysis accepts. Parsing
// allocates on the order of 18x the input, so this also bounds transient
// memory. Callers should read one byte past it to tell an exact-limit file
// from a truncated prefix.
const MaxSourceBytes = 2 << 20
// MaxReasonBytes bounds Report.Reason. Parser diagnostics can contain
// attacker-controlled text, so reports retain only sanitized, bounded context.
const MaxReasonBytes = 256
const (
// maxAnalysisDepth must cover real minified bundles, whose parse trees
// reach depth 905, while staying below the parser's pinned 1000 nesting
// limit so a parseable fixture can still distinguish this limit from a
// parser error.
maxAnalysisDepth = 950
maxPropagatedFacts = 200_000
)
type analysisLimitError uint8
const (
errAnalysisDepthLimit analysisLimitError = iota
errFactLimit
errNodeLimit
)
func (e analysisLimitError) Error() string {
switch e {
case errAnalysisDepthLimit:
return "maximum AST recursion depth exceeded"
case errNodeLimit:
return "maximum AST node count exceeded"
default:
return "maximum propagated fact count exceeded"
}
}
// Result is one keystroke-to-sink flow.
type Result struct {
// Source is the keyboard property read, such as "e.which".
Source string
// Via lists the canonical names the value passed through, in order.
Via []string
// Sink names the network operation that received the value.
Sink string
}
// Report is the outcome of analysing one source file.
type Report struct {
Status Status
// Results is non-empty only when Status is StatusAnalyzed.
Results []Result
// TotalResults counts every distinct flow before evidence truncation. It is
// non-zero only when Status is StatusAnalyzed.
TotalResults int
// Reason carries bounded diagnostic context for a non-analyzed status.
Reason string
// EvidenceTruncated reports that a flow or display segment exceeded an
// evidence limit.
EvidenceTruncated bool
}
// Analyze examines src for keystroke data reaching a network sink.
//
// It never panics: a panic anywhere inside is converted to StatusPanic so one
// malformed input cannot take down a scan.
func Analyze(ctx context.Context, src []byte) Report {
return analyzeWithPass(ctx, src, taintPass)
}
type analysisPass func(context.Context, *js.AST, *resourceBudget) (results []Result, total int, truncated bool, err error)
func analyzeWithPass(ctx context.Context, src []byte, pass analysisPass) (report Report) {
defer func() {
if r := recover(); r != nil {
report = Report{Status: StatusPanic, Reason: "recovered panic during analysis"}
}
report = finalizeReport(report)
}()
if err := ctx.Err(); err != nil {
return Report{Status: StatusCanceled, Reason: cancellationReason(err)}
}
// The size gate runs before the pre-filter so an oversize file is reported
// as a coverage gap even when its first bytes look uninteresting. Deciding
// "not a candidate" from a prefix would let padding hide a payload.
if len(src) > MaxSourceBytes {
return Report{Status: StatusOversize}
}
if !isCandidate(src) {
return Report{Status: StatusNotCandidate}
}
ast, err := js.Parse(parse.NewInputBytes(src), js.Options{})
if err != nil {
// A document marker inside a JavaScript literal or comment must never
// suppress analysis. Classify other formats only after JS parsing fails.
if isNonJSDocument(src) {
return Report{Status: StatusNotCandidate}
}
return Report{Status: StatusParseError, Reason: parseFailureContext(err)}
}
results, total, evidenceTruncated, err := pass(ctx, ast, &resourceBudget{})
if err != nil {
status := analysisErrorStatus(err)
return Report{
Status: status,
Results: results,
TotalResults: total,
Reason: analysisErrorReason(status, err),
EvidenceTruncated: evidenceTruncated,
}
}
return Report{
Status: StatusAnalyzed,
Results: results,
TotalResults: total,
EvidenceTruncated: evidenceTruncated,
}
}
// MayBeJSSource reports whether a prefix could be JavaScript source at all.
// It exists for files too large to analyze: the deep walk hands every
// readable file to this analyzer, and without a content check each oversize
// one became "JavaScript we failed to examine" -- 118,688 claimed skips over
// eight weeks on a live host, whose examples were .jpg, .png, .zip and
// .mmdb.
//
// The test is deliberately "is this source at all" rather than "is this a
// candidate". Analyze runs its size gate ahead of isCandidate on purpose, so
// padding cannot hide a payload behind an uninteresting prefix; deciding
// candidacy from a prefix here would reintroduce exactly that. NUL suggests
// binary content but is legal inside JS literals and comments. Only reject
// binary bytes the lexer encounters outside those tokens; incomplete tokens
// and ambiguous syntax must keep their coverage gap.
func MayBeJSSource(prefix []byte) bool {
if bytes.IndexByte(prefix, 0) < 0 {
return true
}
// The parser input appends a sentinel. Cap the slice so it cannot write
// into a caller's source beyond the peek, including concurrent readers.
input := parse.NewInputBytes(prefix[:len(prefix):len(prefix)])
lexer := js.NewLexer(input)
for {
token, data := lexer.Next()
switch token {
case js.ErrorToken:
// EOF can split a literal, comment or UTF-8 sequence. Other
// syntax errors are not proof of binary content either.
if input.Err() != nil {
return true
}
if !utf8.Valid(data) {
return false
}
return len(data) != 1 || (data[0] >= 0x20 && data[0] != 0x7f)
case js.DivToken, js.DivEqToken:
// Distinguishing division from a regexp needs parser context.
// Either may lead to a literal containing binary bytes.
return true
}
}
}
// isCandidate reports whether src carries both a key-handler token and a sink
// token. Matching is ASCII-case-insensitive so React's onKeyDown and a quoted
// "KeyDown" event name are admitted alongside the lowercase spellings.
func isCandidate(src []byte) bool {
return containsAnyASCIIFold(src, "keydown", "keypress", "keyup") &&
containsAnyASCIIFold(src, "fetch", "sendbeacon", "send", "open", "src", "websocket")
}
func containsAnyASCIIFold(src []byte, tokens ...string) bool {
for i, c := range src {
c = asciiLower(c)
for _, token := range tokens {
if c != token[0] || len(src)-i < len(token) {
continue
}
matched := true
for j := 1; j < len(token); j++ {
if asciiLower(src[i+j]) != token[j] {
matched = false
break
}
}
if matched {
return true
}
}
}
return false
}
func asciiLower(c byte) byte {
if c >= 'A' && c <= 'Z' {
return c + 'a' - 'A'
}
return c
}
type resourceBudget struct {
depth int
facts int
nodes int
}
func (b *resourceBudget) enterAST() error {
if b.depth >= maxAnalysisDepth {
return errAnalysisDepthLimit
}
b.depth++
return nil
}
func (b *resourceBudget) leaveAST() {
b.depth--
}
func (b *resourceBudget) addFact() error {
if b.facts >= maxPropagatedFacts {
return errFactLimit
}
b.facts++
return nil
}
type limitVisitor struct {
ctx context.Context
budget *resourceBudget
err error
}
func (v *limitVisitor) Enter(node js.INode) js.IVisitor {
if v.err != nil {
return nil
}
if err := v.ctx.Err(); err != nil {
v.err = err
return nil
}
if err := v.budget.enterAST(); err != nil {
v.err = err
return nil
}
// Count each parser node once, here, so the taint pass need not recount as it
// revisits basic blocks during fixed-point iteration.
v.budget.nodes++
if v.budget.nodes > maxASTNodes {
v.err = errNodeLimit
v.budget.leaveAST()
return nil
}
walkComputedClassName(v, node)
return v
}
func (v *limitVisitor) Exit(js.INode) {
v.budget.leaveAST()
}
func analysisErrorStatus(err error) Status {
switch {
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
return StatusCanceled
case errors.Is(err, errAnalysisDepthLimit), errors.Is(err, errFactLimit), errors.Is(err, errNodeLimit):
return StatusResourceLimit
default:
panic(fmt.Sprintf("unexpected analyzer error: %v", err))
}
}
func analysisErrorReason(status Status, err error) string {
if status == StatusCanceled {
return cancellationReason(err)
}
return err.Error()
}
func cancellationReason(err error) string {
if errors.Is(err, context.DeadlineExceeded) {
return context.DeadlineExceeded.Error()
}
return context.Canceled.Error()
}
func parseFailureContext(err error) string {
var parseErr *parse.Error
if errors.As(err, &parseErr) {
return fmt.Sprintf("invalid JavaScript at line %d, column %d", parseErr.Line, parseErr.Column)
}
return "invalid JavaScript"
}
func finalizeReport(report Report) Report {
if report.Status == StatusAnalyzed {
report.Reason = ""
return report
}
report.Results = nil
report.TotalResults = 0
report.EvidenceTruncated = false
detail := sanitizeReason(report.Reason)
report.Reason = report.Status.String()
if detail != "" {
report.Reason += ": " + detail
}
report.Reason = boundReason(report.Reason)
return report
}
func sanitizeReason(reason string) string {
reason = strings.ToValidUTF8(reason, "?")
reason = strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return ' '
}
return r
}, reason)
return strings.Join(strings.Fields(reason), " ")
}
func boundReason(reason string) string {
reason = sanitizeReason(reason)
if len(reason) <= MaxReasonBytes {
return reason
}
cut := MaxReasonBytes - 3
for cut > 0 && !utf8.RuneStart(reason[cut]) {
cut--
}
return reason[:cut] + "..."
}
package jstaint
import "github.com/tdewolff/parse/v2/js"
// objectKind names the platform object an allocation represents. Receiver
// provenance is what separates a real network sink from a generic application
// object that merely has a method named send, open, or src.
type objectKind uint8
const (
kindGeneric objectKind = iota
kindXHR
kindWebSocket
kindResource // Image or a typed resource element from createElement
)
func mergeKind(a, b objectKind) objectKind {
if a != kindGeneric {
return a
}
return b
}
// Provenance sink display strings.
const (
sinkXHRBody = "XMLHttpRequest.send body argument"
sinkXHRURL = "XMLHttpRequest.open url argument"
sinkXHRHeader = "XMLHttpRequest.setRequestHeader value argument"
sinkWSURL = "WebSocket url argument"
sinkWSProtocol = "WebSocket protocols argument"
sinkWSSend = "WebSocket.send argument"
sinkResourceSrc = "resource element src assignment"
)
// schemeState is the URL-scheme lattice element. The zero value is the unknown,
// possibly-networked scheme (a relative or dynamic URL). A set scheme names a
// single definite scheme; a merge of two different schemes returns to unknown.
type schemeState struct {
set bool
name string
}
func mergeScheme(a, b schemeState) schemeState {
if a == b {
return a
}
return schemeState{}
}
// isNonNetworkScheme reports whether a definite scheme never performs a network
// fetch, so a captured key placed after it is not exfiltrated.
func (s schemeState) isNonNetwork() bool {
if !s.set {
return false
}
switch s.name {
case "about", "blob", "data", "file", "javascript":
return true
default:
return false
}
}
// schemeOfLiteral extracts the definite scheme of a string literal's value. A URL
// scheme is an ASCII letter followed by letters, digits, +, -, or ., ending at
// the first colon. Anything else (a relative path, a dynamic prefix) is unknown.
func schemeOfLiteral(expr js.IExpr) schemeState {
lit, ok := expr.(*js.LiteralExpr)
if !ok || lit.TokenType != js.StringToken || len(lit.Data) < 2 {
return schemeState{}
}
q := lit.Data[0]
if (q != '\'' && q != '"') || lit.Data[len(lit.Data)-1] != q {
return schemeState{}
}
return schemeOfBytes(lit.Data[1 : len(lit.Data)-1])
}
func schemeOfBytes(b []byte) schemeState {
if len(b) == 0 || !isASCIILetter(b[0]) {
return schemeState{}
}
for i := 0; i < len(b); i++ {
c := b[i]
if c == ':' {
if i == 0 {
return schemeState{}
}
return schemeState{set: true, name: asciiFoldString(b[:i])}
}
if !isSchemeChar(c) {
return schemeState{}
}
}
return schemeState{}
}
func isASCIILetter(c byte) bool {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}
func isSchemeChar(c byte) bool {
return isASCIILetter(c) || (c >= '0' && c <= '9') || c == '+' || c == '-' || c == '.'
}
func asciiFoldString(b []byte) string {
out := make([]byte, len(b))
for i := range b {
out[i] = asciiLower(b[i])
}
return string(out)
}
// templateScheme reads the definite scheme from a template literal's leading
// text, or from its first substitution when no literal prefix precedes it.
func templateScheme(x *js.TemplateExpr, first schemeState) schemeState {
if x.Tag != nil {
return schemeState{}
}
if len(x.List) == 0 {
return schemeOfBytes(templateCooked(x.Tail))
}
leading := templateCooked(x.List[0].Value)
if scheme := schemeOfBytes(leading); scheme.set {
return scheme
}
if len(leading) == 0 {
return first
}
return schemeState{}
}
// templateCooked strips the surrounding backtick or `}`/`${` delimiters from a
// template part's raw bytes so only the literal text remains.
func templateCooked(raw []byte) []byte {
if len(raw) > 0 && (raw[0] == '`' || raw[0] == '}') {
raw = raw[1:]
}
if n := len(raw); n >= 2 && raw[n-2] == '$' && raw[n-1] == '{' {
raw = raw[:n-2]
} else if n >= 1 && raw[n-1] == '`' {
raw = raw[:n-1]
}
return raw
}
// resourceElementTag reports whether a createElement tag names an element that
// fetches its src attribute, compared ASCII case-insensitively.
func resourceElementTag(name string) bool {
switch asciiFoldString([]byte(name)) {
case "audio", "embed", "iframe", "img", "script", "source", "track", "video":
return true
default:
return false
}
}
// allocateKinded installs a fresh current platform-object allocation. Recency is
// retained inside loops because constructor-local protocol state must not inherit
// from older instances; promoteCurrent still bounds each site to two identities.
func (a *analysis) allocateKinded(st *state, site int, kind objectKind) value {
a.promoteCurrent(st, site)
fresh := &object{kind: kind}
return a.installLiteral(st, site, false, fresh)
}
// evalNewSink handles new-expression receivers with network provenance. It
// returns the constructed value and true when it recognized one; the caller
// evaluates arguments before calling this.
func (a *analysis) evalNewSink(x *js.NewExpr, args []value, site int, st *state) (value, bool) {
switch {
case a.isGlobalCallee(x.X, "XMLHttpRequest"):
return a.allocateKinded(st, site, kindXHR), true
case a.isGlobalCallee(x.X, "Image"):
return a.allocateKinded(st, site, kindResource), true
case a.isGlobalCallee(x.X, "WebSocket"):
if len(args) == 0 {
return value{}, true
}
a.recordURL(args[0], x, 0, sinkWSURL)
if args[0].scheme.isNonNetwork() {
return value{}, true
}
if len(args) >= 2 {
a.record(argScalar(args, 1), x, 1, sinkWSProtocol)
a.record(a.serializeArray(st, args[1]), x, 1, sinkWSProtocol)
}
return a.allocateKinded(st, site, kindWebSocket), true
}
return value{}, false
}
// createElementValue returns a typed resource-element allocation when the call is
// document.createElement with a static resource tag, and true when it applied.
func (a *analysis) createElementValue(call *js.CallExpr, st *state) (value, bool) {
prop, base, ok := memberAccess(ungroupExpr(call.X))
if !ok || prop != "createElement" || !a.isDocument(base) {
return value{}, false
}
if len(call.Args.List) == 0 {
return value{}, false
}
tag, ok := staticStringOrIdent(ungroupExpr(call.Args.List[0].Value))
if !ok || !resourceElementTag(tag) {
return value{}, false
}
site, ok := a.sites[call]
if !ok {
return value{}, false
}
return a.allocateKinded(st, site, kindResource), true
}
// isDocument reports whether expr is the unshadowed global document object.
func (a *analysis) isDocument(expr js.IExpr) bool {
switch v := ungroupExpr(expr).(type) {
case *js.Var:
return isGlobalRef(v, "document")
case *js.DotExpr, *js.IndexExpr:
prop, base, ok := memberAccess(expr)
return ok && prop == "document" && isGlobalObject(base)
default:
return false
}
}
// xhrOrWSMethod applies a method call on an XHR or WebSocket receiver and records
// any sink. It returns true when the receiver's provenance claimed the call.
func (a *analysis) xhrOrWSMethod(call *js.CallExpr, recv value, args []value, st *state) bool {
prop, _, ok := memberAccess(ungroupExpr(call.X))
if !ok {
return false
}
ids := xhrWSTargets(st, recv)
if len(ids) == 0 {
return false
}
strong := recv.allocOnly && len(uniqueAllocIDs(recv.allocs)) == 1 && soleCurrent(recv.allocs)
handled := false
for id := range ids {
o := st.mutObject(id)
switch o.kind {
case kindXHR:
handled = a.applyXHRMethod(st, o, prop, call, args, recv, id, strong && !id.summary) || handled
case kindWebSocket:
handled = a.applyWSMethod(o, prop, call, args, strong && !id.summary) || handled
}
}
return handled
}
// xhrWSTargets returns the receiver allocations that carry XHR or WebSocket
// provenance.
func xhrWSTargets(st *state, recv value) map[allocID]bool {
var ids map[allocID]bool
for ref := range recv.allocs {
o := st.heap[ref.id]
if o == nil || (o.kind != kindXHR && o.kind != kindWebSocket) {
continue
}
if ids == nil {
ids = map[allocID]bool{}
}
ids[ref.id] = true
}
return ids
}
func (a *analysis) applyXHRMethod(
st *state,
o *object,
prop string,
call *js.CallExpr,
args []value,
recv value,
id allocID,
strong bool,
) bool {
switch prop {
case "open":
if len(args) < 2 {
return true
}
wasOpened := o.xhrOpened
// A later open reinitializes the request. Only a lone current receiver can
// prove the reset clears an earlier path's remembered URL and headers.
o.xhrOpened = true
switch {
case strong:
o.xhrURL = argScalar(args, 1)
o.xhrHeader = nil
o.xhrScheme = args[1].scheme
case !wasOpened:
o.xhrURL = argScalar(args, 1)
o.xhrScheme = args[1].scheme
default:
o.xhrURL = mergeTaint(o.xhrURL, argScalar(args, 1))
o.xhrScheme = mergeScheme(o.xhrScheme, args[1].scheme)
}
a.resetReceiverDepth(st, id, argScalar(args, 1))
return true
case "setRequestHeader":
if o.xhrOpened {
o.xhrHeader = mergeTaint(o.xhrHeader, argScalar(args, 1))
a.resetReceiverDepth(st, id, argScalar(args, 1))
}
return true
case "send":
if o.xhrOpened && !o.xhrScheme.isNonNetwork() {
a.record(argScalar(args, 0), call, 0, sinkXHRBody)
a.record(receiverTaint(recv, id, o.xhrURL), call, 1, sinkXHRURL)
a.record(receiverTaint(recv, id, o.xhrHeader), call, 2, sinkXHRHeader)
}
return true
case "abort":
if strong {
o.xhrOpened = false
o.xhrURL = nil
o.xhrHeader = nil
o.xhrScheme = schemeState{}
}
return true
}
return false
}
func (a *analysis) resetReceiverDepth(st *state, id allocID, ts taintSet) {
if taintCarriesDepth(ts, a.callDepth) {
resetAllocRefDepth(st, map[allocID]bool{id: true})
}
}
// receiverTaint applies every call-depth constraint carried by receiver aliases
// to state stored on one allocation.
func receiverTaint(recv value, id allocID, ts taintSet) taintSet {
var out taintSet
for ref := range recv.allocs {
if ref.id == id {
out = mergeTaint(out, applyTaintDepth(ts, ref.minDepth, ref.advance))
}
}
return out
}
func (a *analysis) applyWSMethod(o *object, prop string, call *js.CallExpr, args []value, strong bool) bool {
switch prop {
case "send":
if o.wsMaybeOpen && !o.wsClosed {
a.record(argScalar(args, 0), call, 0, sinkWSSend)
}
return true
case "close":
if strong {
o.wsClosed = true
}
return true
}
return false
}
// markSocketsObservable marks every WebSocket allocation in a shared state as
// possibly open. A socket published to file-scope state can be observed open by a
// later callback, which is exactly when a send on it becomes a network sink.
func markSocketsObservable(st *state) {
var hit []allocID
for id, o := range st.heap {
if o.kind == kindWebSocket && !o.wsMaybeOpen {
hit = append(hit, id)
}
}
for _, id := range hit {
st.mutObject(id).wsMaybeOpen = true
}
}
// resourceSrcSink records a src-assignment sink when the receiver is a typed
// resource element, gated by the assigned value's URL scheme.
func (a *analysis) resourceSrcSink(recv value, key fieldKey, rhs value, node js.INode, st *state) {
if key.kind != fieldNamed || key.name != "src" {
return
}
for ref := range recv.allocs {
if o := st.heap[ref.id]; o != nil && o.kind == kindResource {
a.recordURL(rhs, node, 0, sinkResourceSrc)
return
}
}
}
// recordURL records a destination-URL sink unless the destination is proven to
// use a non-network scheme.
func (a *analysis) recordURL(dest value, sink js.INode, arg int, text string) {
if dest.scheme.isNonNetwork() {
return
}
a.record(dest.scalar, sink, arg, text)
}
// recordBody records a body, data, or header sink carried alongside a
// destination. A proven non-network destination controls the whole operation, so
// its body is not exfiltrated either.
func (a *analysis) recordBody(dest, payload value, sink js.INode, arg int, text string) {
if dest.scheme.isNonNetwork() {
return
}
a.record(payload.scalar, sink, arg, text)
}
package jstaint
import "github.com/tdewolff/parse/v2/js"
// Sink display strings. They are stable because they appear in deterministic
// evidence output.
const (
sinkFetchURL = "fetch url argument"
sinkFetchBody = "fetch body option"
sinkFetchReferrer = "fetch referrer option"
sinkBeaconURL = "navigator.sendBeacon url argument"
sinkBeaconData = "navigator.sendBeacon data argument"
)
// evalCall evaluates a call's receiver and arguments once, records any network
// sink the call represents, and returns the value the call propagates.
func (a *analysis) evalCall(call *js.CallExpr, st *state) value {
callee := ungroupExpr(call.X)
recv := a.evalCallCallee(callee, st)
st.setCapture(call, recv)
isFetch := a.isGlobalCallee(callee, "fetch")
isBeacon := a.isBeaconCallee(callee)
argsSt := st
argsMayBeSkipped := call.Optional || optionalChainMaySkip(call.X)
if argsMayBeSkipped {
argsSt = st.clone()
}
args := make([]value, len(call.Args.List))
for i := range call.Args.List {
if isFetch && i == 1 {
a.evalFetchInit(call, args[0], call.Args.List[i].Value, argsSt)
continue
}
args[i] = a.evalExpr(call.Args.List[i].Value, argsSt)
}
if argsMayBeSkipped {
st.replaceWith(mergeState(st, argsSt))
}
recv = st.captures[call]
st.delCapture(call)
a.checkCallSink(call, args, isFetch, isBeacon)
if a.xhrOrWSMethod(call, recv, args, st) {
return value{}
}
if v, ok := a.createElementValue(call, st); ok {
return v
}
if ret, ok := a.evalUserCall(callee, args, st); ok {
return ret
}
return a.evalCallReturn(st, callee, args, recv)
}
func optionalChainMaySkip(expr js.IExpr) bool {
switch x := expr.(type) {
case *js.DotExpr:
return x.Optional || optionalChainMaySkip(x.X)
case *js.IndexExpr:
return x.Optional || optionalChainMaySkip(x.X)
case *js.CallExpr:
return x.Optional || optionalChainMaySkip(x.X)
case *js.TemplateExpr:
return x.Optional || optionalChainMaySkip(x.Tag)
default:
return false
}
}
// evalCallCallee evaluates a call's receiver and returns its value, which the
// array-method and string-method handling consume.
func (a *analysis) evalCallCallee(callee js.IExpr, st *state) value {
switch c := callee.(type) {
case *js.DotExpr:
return a.evalExpr(c.X, st)
case *js.IndexExpr:
recv := a.evalExpr(c.X, st)
st.setCapture(c, recv)
if c.Optional || optionalChainMaySkip(c.X) {
skipped := st.clone()
taken := st.clone()
a.evalExpr(c.Y, taken)
st.replaceWith(mergeState(skipped, taken))
} else {
a.evalExpr(c.Y, st)
}
recv = st.captures[c]
st.delCapture(c)
return recv
default:
a.evalExpr(callee, st)
return value{}
}
}
// checkCallSink records a finding when a network-call sink receives a tainted
// scalar. Only the scalar part of an argument counts: a URL or data argument must
// be a string, so an unserialized object never taints a sink.
func (a *analysis) checkCallSink(call *js.CallExpr, args []value, isFetch, isBeacon bool) {
if isFetch {
if len(args) >= 1 {
a.recordURL(args[0], call, 0, sinkFetchURL)
}
return
}
if isBeacon {
if len(args) >= 1 {
a.recordURL(args[0], call, 0, sinkBeaconURL)
}
if len(args) >= 2 {
a.recordBody(argValue(args, 0), args[1], call, 1, sinkBeaconData)
}
}
}
// evalFetchInit records a tainted scalar body or referrer in fetch's second
// argument, which must be a plain object literal in version 1. Evaluating the
// literal through the heap preserves computed, spread, and duplicate-property
// semantics without evaluating any property twice.
func (a *analysis) evalFetchInit(call *js.CallExpr, dest value, initExpr js.IExpr, st *state) {
_, ok := ungroupExpr(initExpr).(*js.ObjectExpr)
if !ok {
a.evalExpr(initExpr, st)
return
}
init := a.evalExpr(initExpr, st)
body := a.readField(st, init, fieldKey{kind: fieldNamed, name: "body"})
referrer := a.readField(st, init, fieldKey{kind: fieldNamed, name: "referrer"})
a.recordBody(dest, body, call, 1, sinkFetchBody)
a.recordBody(dest, referrer, call, 1, sinkFetchReferrer)
}
// evalCallReturn returns the value a call propagates for the value-preserving
// built-ins and serializers version 1 models, and applies the array-mutating
// effect of push. Any other call returns clean.
func (a *analysis) evalCallReturn(st *state, callee js.IExpr, args []value, recv value) value {
for _, name := range []string{"String", "encodeURIComponent", "encodeURI", "escape", "btoa"} {
if a.isGlobalCallee(callee, name) {
return value{scalar: argScalar(args, 0)}
}
}
prop, base, ok := memberAccess(callee)
if !ok {
return value{}
}
if prop == "fromCharCode" && a.isGlobalCallee(base, "String") {
return value{scalar: unionArgScalars(args)}
}
if prop == "stringify" && a.isGlobalCallee(base, "JSON") {
return value{scalar: a.serializeStringify(st, argValue(args, 0))}
}
switch prop {
case "toString", "trim", "charAt", "slice", "substr", "substring":
return value{scalar: recv.scalar}
case "concat":
if isDefiniteNonString(base) || (recv.allocOnly && len(recv.allocs) != 0) {
return value{}
}
return value{scalar: mergeTaint(recv.scalar, unionArgScalars(args))}
case "push":
a.arrayPush(st, recv, args)
return value{}
case "join":
return value{scalar: a.serializeArray(st, recv)}
default:
return value{}
}
}
// arrayPush taints the array-element field of the receiver's allocations with the
// pushed values.
func (a *analysis) arrayPush(st *state, recv value, args []value) {
if len(args) == 0 {
return
}
v := args[0]
for i := 1; i < len(args); i++ {
v = mergeValue(v, args[i])
}
a.writeArrayElements(st, recv, v, true)
}
func (a *analysis) writeArrayElements(st *state, recv value, v value, definite bool) {
ids := uniqueAllocIDs(recv.allocs)
strong := len(ids) == 1 && soleCurrent(recv.allocs)
for id := range ids {
if o := st.heap[id]; o == nil || !o.array {
continue
}
st.mutObject(id).weakElem(v, definite && strong && !id.summary)
}
if a.callDepth == 0 && allocsConstrained(recv.allocs) && valueCarriesDepthZero(st, v) {
resetAllocRefDepth(st, ids)
}
a.fact()
}
func argValue(args []value, i int) value {
if i >= len(args) {
return value{}
}
return args[i]
}
func argScalar(args []value, i int) taintSet {
return argValue(args, i).scalar
}
func unionArgScalars(args []value) taintSet {
var out taintSet
for i := range args {
out = mergeTaint(out, args[i].scalar)
}
return out
}
// isDefiniteNonString reports whether concat's receiver is provably not a string,
// so array concat is not treated as string laundering.
func isDefiniteNonString(expr js.IExpr) bool {
switch x := ungroupExpr(expr).(type) {
case *js.LiteralExpr:
return x.TokenType != js.StringToken
case *js.ArrayExpr, *js.ObjectExpr, *js.NewExpr, *js.ClassDecl, *js.FuncDecl, *js.ArrowFunc:
return true
default:
return false
}
}
// isGlobalCallee reports whether callee is the named unshadowed global function,
// either bare or as a property of window/self/globalThis.
func (a *analysis) isGlobalCallee(callee js.IExpr, name string) bool {
switch c := ungroupExpr(callee).(type) {
case *js.Var:
return isGlobalRef(c, name)
case *js.DotExpr, *js.IndexExpr:
prop, base, ok := memberAccess(callee)
return ok && prop == name && isGlobalObject(base)
default:
return false
}
}
func (a *analysis) isBeaconCallee(callee js.IExpr) bool {
prop, base, ok := memberAccess(callee)
return ok && prop == "sendBeacon" && a.isNavigator(base)
}
// isNavigator reports whether expr is the unshadowed global navigator object.
func (a *analysis) isNavigator(expr js.IExpr) bool {
switch v := ungroupExpr(expr).(type) {
case *js.Var:
return isGlobalRef(v, "navigator")
case *js.DotExpr, *js.IndexExpr:
prop, base, ok := memberAccess(expr)
return ok && prop == "navigator" && isGlobalObject(base)
default:
return false
}
}
// isGlobalObject reports whether expr is one of the unshadowed global object
// aliases.
func isGlobalObject(expr js.IExpr) bool {
v, ok := ungroupExpr(expr).(*js.Var)
if !ok {
return false
}
name, ok := globalName(v)
if !ok {
return false
}
switch name {
case "window", "self", "globalThis":
return true
default:
return false
}
}
// globalName returns the name of an unshadowed global reference. The parser
// leaves free identifiers undeclared, so a NoDecl canonical identity is a global
// binding rather than a local of the same spelling.
func globalName(v *js.Var) (string, bool) {
c := canonicalVar(v)
if c.Decl != js.NoDecl {
return "", false
}
return string(c.Name()), true
}
package jstaint
import (
"bytes"
"strings"
"github.com/tdewolff/parse/v2/js"
)
// isKeyboardProp reports whether name is an event property whose value is a
// keystroke. e.target is deliberately absent: its .value is the input's current
// text, which legitimate search-as-you-type widgets read and post.
func isKeyboardProp(name []byte) bool {
return bytes.Equal(name, []byte("key")) ||
bytes.Equal(name, []byte("keyCode")) ||
bytes.Equal(name, []byte("charCode")) ||
bytes.Equal(name, []byte("which")) ||
bytes.Equal(name, []byte("code"))
}
// The only member accesses allowed between the event variable and a keyboard
// property are these framework wrappers, for example e.originalEvent.key.
func isEventWrapperProp(name []byte) bool {
return bytes.Equal(name, []byte("originalEvent")) ||
bytes.Equal(name, []byte("nativeEvent"))
}
// keyboardSource reports whether expr statically reads a keyboard property off
// the handler's event variable, possibly through originalEvent/nativeEvent
// wrappers, and returns a dotted display string for evidence. The boolean result
// of a comparison is not handled here; barriers are enforced during propagation.
func keyboardSource(expr js.IExpr, eventVar *js.Var) (string, bool) {
name, base, ok := memberAccessBytes(expr)
if !ok || !isKeyboardProp(name) {
return "", false
}
if !resolvesToEventBase(base, eventVar) {
return "", false
}
return memberDisplay(expr), true
}
// resolvesToEventBase reports whether base is the event variable itself or a
// wrapper chain rooted at it.
func resolvesToEventBase(base js.IExpr, eventVar *js.Var) bool {
if eventVar == nil {
return false
}
return eventBaseVar(base) == canonicalVar(eventVar)
}
// eventBaseVar returns the canonical event-variable candidate at the root of a
// supported wrapper chain.
func eventBaseVar(base js.IExpr) *js.Var {
for {
base = ungroupExpr(base)
if v, ok := base.(*js.Var); ok {
return canonicalVar(v)
}
name, inner, ok := memberAccessBytes(base)
if !ok || !isEventWrapperProp(name) {
return nil
}
base = inner
}
}
// memberAccess splits a dot or static-bracket member access into its property
// name and base expression.
func memberAccess(expr js.IExpr) (name string, base js.IExpr, ok bool) {
data, base, ok := memberAccessBytes(expr)
if !ok {
return "", nil, false
}
return string(data), base, true
}
func memberAccessBytes(expr js.IExpr) (name []byte, base js.IExpr, ok bool) {
switch e := ungroupExpr(expr).(type) {
case *js.DotExpr:
if n, ok := staticBytesOrIdent(ungroupExpr(e.Y)); ok {
return n, e.X, true
}
case *js.IndexExpr:
if n, ok := staticBytesOrIdent(ungroupExpr(e.Y)); ok {
return n, e.X, true
}
}
return nil, nil, false
}
// memberDisplay renders a member-access chain as dotted names for evidence,
// normalizing bracket access to dot form. An unresolvable base is shown as "?".
func memberDisplay(expr js.IExpr) string {
var names [][]byte
for {
expr = ungroupExpr(expr)
if v, ok := expr.(*js.Var); ok {
return joinMemberDisplay(v.Name(), names)
}
name, base, ok := memberAccessBytes(expr)
if !ok {
return joinMemberDisplay([]byte("?"), names)
}
names = append(names, name)
expr = base
}
}
func joinMemberDisplay(base []byte, reversedNames [][]byte) string {
size := len(base)
for _, name := range reversedNames {
size += 1 + len(name)
}
var display strings.Builder
display.Grow(size)
display.Write(base)
for i := len(reversedNames) - 1; i >= 0; i-- {
display.WriteByte('.')
display.Write(reversedNames[i])
}
return display.String()
}
// binaryOpPropagates reports whether a binary operator's result carries taint
// from its operands. Comparisons are barriers because their boolean result does
// not contain the captured key; logical operators propagate because JavaScript
// returns one of the operand values.
func binaryOpPropagates(op js.TokenType) bool {
switch op {
case js.AddToken, js.SubToken, js.MulToken, js.DivToken, js.ModToken, js.ExpToken,
js.LtLtToken, js.GtGtToken, js.GtGtGtToken,
js.BitAndToken, js.BitOrToken, js.BitXorToken,
js.AndToken, js.OrToken, js.NullishToken:
return true
default:
return false
}
}
// unaryOpPropagates reports whether a unary operator's result carries taint.
// !, typeof, void, and delete are barriers; +, -, ~, increment/decrement, and
// await keep a value derived from the operand.
func unaryOpPropagates(op js.TokenType) bool {
switch op {
case js.PosToken, js.NegToken, js.BitNotToken, js.AwaitToken,
js.PreIncrToken, js.PreDecrToken, js.PostIncrToken, js.PostDecrToken:
return true
default:
return false
}
}
package jstaint
import (
"context"
"sort"
"github.com/tdewolff/parse/v2/js"
)
// maxASTNodes bounds the parser nodes the structural pass visits. It is a safety
// ceiling: exceeding it fails the whole analysis rather than returning a partial
// decision. Real minified bundles average 4-6 bytes per node, so the cap must
// admit about 2.6 bytes per node at the 2 MiB input limit or real files become
// permanent coverage gaps; it still rejects pathological synthetic nesting.
const maxASTNodes = 800_000
// taintChain is the shortest known list of via display names a captured value
// passed through to reach a program point.
type taintChain = []string
// taintFact identifies one source at one permitted user-call depth. Depth is
// part of the fact so a returned value cannot enter a second user-defined
// callee after control returns to a root.
type taintFact struct {
source int
callDepth uint8
}
// taintSet maps a source/depth fact to its shortest via chain.
type taintSet map[taintFact]taintChain
// flowKey is the identity of one source-to-sink flow: which source occurrence,
// sink occurrence, sink kind, and argument. Multiple propagation routes to the
// same endpoints collapse to one result with the shortest chain.
type flowKey struct {
source int
sink js.INode
arg int
kind string
}
type sourceOccurrence struct {
id int
display string
}
type functionExit struct {
state *state
ret value
returned bool
}
// analysis holds the mutable state for one Analyze call.
type analysis struct {
ctx context.Context
budget *resourceBudget
sources map[js.IExpr]sourceOccurrence
sites map[js.INode]int
funcs map[*js.Var][]*funcInfo
sharedVars map[*js.Var]bool
inProgress map[*funcInfo]bool
localCache map[*js.BlockStmt]map[*js.Var]bool
restSites map[js.IBinding]int
retStack []value
returnExits [][]functionExit
suspensions []*state
display map[int]string
results map[flowKey]Result
loopDepth int
callDepth int
stopAtAwait bool
err error
}
// taintPass is the production analysis pass wired into Analyze. It first enforces
// the structural depth, node, and cancellation limits with a bounded walk, then
// runs the taint analysis on the within-limit tree.
func taintPass(ctx context.Context, ast *js.AST, budget *resourceBudget) ([]Result, int, bool, error) {
lv := &limitVisitor{ctx: ctx, budget: budget}
js.Walk(lv, ast)
if lv.err != nil {
return nil, 0, false, lv.err
}
sharedVars := map[*js.Var]bool{}
for _, v := range ast.Declared {
sharedVars[canonicalVar(v)] = true
}
a := &analysis{
ctx: ctx,
budget: budget,
sites: numberAllocSites(ast),
funcs: collectFuncValues(ast),
sharedVars: sharedVars,
inProgress: map[*funcInfo]bool{},
localCache: map[*js.BlockStmt]map[*js.Var]bool{},
restSites: numberRestSites(ast),
results: map[flowKey]Result{},
}
roots := a.discoverReachableRoots(ast)
eventVars := map[*js.Var]bool{}
for _, r := range roots {
if r.eventVar != nil {
eventVars[r.eventVar] = true
}
}
a.sources, a.display = numberSources(ast, eventVars)
a.analyzeReachable(ast, roots)
if a.err != nil {
return nil, 0, false, a.err
}
results, total, truncated := a.finalizeResults()
return results, total, truncated, nil
}
// analyzeReachable runs the top level once, then every reachable callback root to
// a fixed point over the file-scope may-state that callbacks publish to and read
// from one another.
func (a *analysis) analyzeReachable(ast *js.AST, roots []rootSite) {
// The top level runs once with a clean published state, so a request that runs
// before any event cannot observe a taint a later handler produces.
top := newState()
top = a.analyzeBlock(&ast.BlockStmt, top)
if !a.alive() {
return
}
global := a.sharedState(top)
// A socket that survives to file-scope state can be observed open by a later
// callback, which is when a send on it becomes a network sink.
markSocketsObservable(global)
for a.alive() {
changed := false
for i := range roots {
st := global.clone()
a.suspensions = make([]*state, 0)
a.bindRootParams(roots[i], st)
a.returnExits = append(a.returnExits, nil)
st = a.analyzeBlock(roots[i].fn.body, st)
exits := a.popReturnExits(st)
if !a.alive() {
return
}
next := global
for _, exit := range exits {
next = a.publishShared(next, exit.state)
}
for _, suspended := range a.suspensions {
next = a.publishShared(next, suspended)
}
a.suspensions = nil
markSocketsObservable(next)
if !stateEqual(next, global) {
global = next
changed = true
}
}
if !changed || !a.fact() {
return
}
}
}
// popReturnExits closes the current function-flow frame. Explicit return states
// and the normal fallthrough state are separate execution alternatives; code
// after a return must not inherit writes from the returned path.
func (a *analysis) popReturnExits(normalExit *state) []functionExit {
index := len(a.returnExits) - 1
exits := a.returnExits[index]
a.returnExits = a.returnExits[:index]
if normalExit.continues {
exits = append(exits, functionExit{state: normalExit})
}
return exits
}
// bindRootParams binds a callback's parameters before its body. A keyboard
// handler's first parameter is the event object whose keystroke reads are the
// numbered sources, so only its later parameters are bound here.
func (a *analysis) bindRootParams(r rootSite, st *state) {
start := 0
if r.eventVar != nil {
start = 1
}
for i := start; i < len(r.fn.params.List); i++ {
if i < r.argCount {
a.bindParamPattern(r.fn.params.List[i].Binding, value{}, st)
continue
}
a.analyzeBinding(&r.fn.params.List[i], st, true)
}
}
// sortedResults returns the recorded flows in deterministic content order.
func (a *analysis) sortedResults() []Result {
out := make([]Result, 0, len(a.results))
for _, r := range a.results {
out = append(out, r)
}
sort.Slice(out, func(i, j int) bool { return lessResult(out[i], out[j]) })
return out
}
func lessResult(x, y Result) bool {
if x.Source != y.Source {
return x.Source < y.Source
}
xv, yv := joinChain(x.Via), joinChain(y.Via)
if xv != yv {
return xv < yv
}
return x.Sink < y.Sink
}
func joinChain(c []string) string {
out := ""
for i, s := range c {
if i != 0 {
out += "\x00"
}
out += s
}
return out
}
// record inserts a source-to-sink flow, keeping the shortest via chain when a
// route to the same endpoints already exists. Ties break lexicographically by
// the joined via names so output does not depend on map iteration order.
func (a *analysis) record(ts taintSet, sink js.INode, arg int, sinkText string) {
for fact, chain := range ts {
if !a.alive() {
return
}
key := flowKey{source: fact.source, sink: sink, arg: arg, kind: sinkText}
cand := Result{Source: a.display[fact.source], Via: append([]string(nil), chain...), Sink: sinkText}
if ex, ok := a.results[key]; ok {
if shorterChain(cand.Via, ex.Via) {
a.results[key] = cand
}
continue
}
if !a.fact() {
return
}
a.results[key] = cand
}
}
func shorterChain(cand, existing []string) bool {
if len(cand) != len(existing) {
return len(cand) < len(existing)
}
return joinChain(cand) < joinChain(existing)
}
func (a *analysis) fact() bool {
if a.err != nil {
return false
}
if err := a.budget.addFact(); err != nil {
a.err = err
return false
}
return true
}
func (a *analysis) alive() bool {
if a.err != nil {
return false
}
if err := a.ctx.Err(); err != nil {
a.err = err
return false
}
return true
}
// numberSources assigns a deterministic occurrence id to every keyboard-source
// read whose base resolves to a discovered event variable. The traversal order
// does not affect output because results are content-sorted; it only needs to be
// stable across identical inputs.
func numberSources(ast *js.AST, eventVars map[*js.Var]bool) (map[js.IExpr]sourceOccurrence, map[int]string) {
n := &sourceNumberer{eventVars: eventVars, out: map[js.IExpr]sourceOccurrence{}, display: map[int]string{}}
js.Walk(n, ast)
return n.out, n.display
}
type sourceNumberer struct {
eventVars map[*js.Var]bool
out map[js.IExpr]sourceOccurrence
display map[int]string
next int
}
func (n *sourceNumberer) Exit(js.INode) {}
func (n *sourceNumberer) Enter(node js.INode) js.IVisitor {
if walkComputedClassName(n, node) {
return n
}
expr, ok := node.(js.IExpr)
if !ok {
return n
}
name, base, ok := memberAccessBytes(expr)
if !ok || !isKeyboardProp(name) || !n.resolves(base) {
return n
}
disp := memberDisplay(expr)
n.out[expr] = sourceOccurrence{id: n.next, display: disp}
n.display[n.next] = disp
n.next++
return n
}
func (n *sourceNumberer) resolves(base js.IExpr) bool {
v := eventBaseVar(base)
return v != nil && n.eventVars[v]
}
// mergeTaint unions two taint sets, keeping the shortest chain per source.
func mergeTaint(a, b taintSet) taintSet {
if len(a) == 0 {
return b
}
if len(b) == 0 {
return a
}
out := make(taintSet, len(a)+len(b))
for fact, ch := range a {
out[fact] = ch
}
for fact, ch := range b {
if ex, ok := out[fact]; !ok || shorterChain(ch, ex) {
out[fact] = ch
}
}
return out
}
// appendVia extends every chain in ts with name, skipping a redundant repeat so
// loop fixed points do not grow chains without bound.
func appendVia(ts taintSet, name string) taintSet {
if len(ts) == 0 {
return nil
}
out := make(taintSet, len(ts))
for fact, ch := range ts {
if len(ch) > 0 && ch[len(ch)-1] == name {
out[fact] = ch
continue
}
nc := make(taintChain, len(ch)+1)
copy(nc, ch)
nc[len(ch)] = name
out[fact] = nc
}
return out
}
func taintEqual(a, b taintSet) bool {
if len(a) != len(b) {
return false
}
for fact, ca := range a {
cb, ok := b[fact]
if !ok || !chainEqual(ca, cb) {
return false
}
}
return true
}
func chainEqual(a, b taintChain) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
package jstaint
import (
"strconv"
"github.com/tdewolff/parse/v2/js"
)
// srcValue is the abstract value of a keyboard-source read: scalar taint tagged
// with the source occurrence and an empty laundering chain.
func (a *analysis) srcValue(id int) value {
depth := uint8(a.callDepth) // #nosec G115 -- callDepth is 0 or 1 under the depth-1 interprocedural cap
return value{scalar: taintSet{{source: id, callDepth: depth}: taintChain{}}}
}
// analyzeBlock threads the state through a statement list in source order.
func (a *analysis) analyzeBlock(block *js.BlockStmt, st *state) *state {
if block == nil {
return st
}
for i := range block.List {
if !a.alive() || !st.continues {
return st
}
st = a.analyzeStmt(block.List[i], st)
}
return st
}
func (a *analysis) analyzeStmt(stmt js.IStmt, st *state) *state {
if !a.alive() || !st.continues {
return st
}
switch s := stmt.(type) {
case *js.VarDecl:
for i := range s.List {
a.analyzeBinding(&s.List[i], st, s.TokenType != js.VarToken)
}
case *js.ClassDecl:
a.evalClass(s, st)
case *js.ExprStmt:
a.analyzeExprStmt(s.Value, st)
case *js.BlockStmt:
st = a.analyzeBlock(s, st)
case *js.IfStmt:
a.evalExpr(s.Cond, st)
thenSt := a.analyzeStmt(s.Body, st.clone())
var elseSt *state
if s.Else != nil {
elseSt = a.analyzeStmt(s.Else, st.clone())
} else {
elseSt = st.clone()
}
st = mergeState(thenSt, elseSt)
case *js.ForStmt:
a.analyzeInit(s.Init, st)
st = a.analyzeLoop(s.Body, s.Cond, s.Post, st)
case *js.WhileStmt:
st = a.analyzeLoop(s.Body, s.Cond, nil, st)
case *js.DoWhileStmt:
st = a.analyzeDoLoop(s.Body, s.Cond, st)
case *js.ForInStmt:
a.evalExpr(s.Value, st)
st = a.analyzeIterationLoop(s.Body, s.Init, value{}, false, st)
case *js.ForOfStmt:
iter := a.evalExpr(s.Value, st)
st = a.analyzeIterationLoop(s.Body, s.Init, iter, true, st)
case *js.SwitchStmt:
st = a.analyzeSwitch(s, st)
case *js.TryStmt:
st = a.analyzeTry(s, st)
case *js.ReturnStmt:
var rv value
if s.Value != nil {
rv = a.evalExpr(s.Value, st)
}
if n := len(a.returnExits); n > 0 {
a.returnExits[n-1] = append(a.returnExits[n-1], functionExit{
state: st.clone(), ret: rv, returned: true,
})
st.continues = false
}
case *js.ThrowStmt:
a.evalExpr(s.Value, st)
case *js.WithStmt:
a.evalExpr(s.Cond, st)
bodySt := a.analyzeStmt(s.Body, st.clone())
st = mergeState(st, bodySt)
case *js.LabelledStmt:
st = a.analyzeStmt(s.Value, st)
case *js.BranchStmt:
// break/continue: no data effect for this model.
}
return st
}
// analyzeInit handles a for-loop initializer, which may be a var declaration or
// an expression.
func (a *analysis) analyzeInit(init js.IExpr, st *state) {
if init == nil {
return
}
if vd, ok := init.(*js.VarDecl); ok {
for i := range vd.List {
a.analyzeBinding(&vd.List[i], st, vd.TokenType != js.VarToken)
}
return
}
a.evalExpr(init, st)
}
// analyzeBinding applies a declaration binding. A tainted initializer taints the
// bound variable, while a clean initializer is a strong update. An absent lexical
// initializer writes undefined; an absent var initializer is only a declaration.
func (a *analysis) analyzeBinding(be *js.BindingElement, st *state, clearAbsent bool) {
v, ok := be.Binding.(*js.Var)
if !ok {
if be.Default != nil {
a.evalExpr(be.Default, st)
}
a.evalBindingPattern(be.Binding, st)
return
}
cv := canonicalVar(v)
if be.Default == nil {
if clearAbsent {
st.delEnv(cv)
}
return
}
rhs := a.evalExpr(be.Default, st)
a.bindVar(st, cv, rhs)
}
// bindVar performs a strong update binding a variable to a value, appending the
// variable name to the laundering chain and preserving aliased allocations.
func (a *analysis) bindVar(st *state, cv *js.Var, rhs value) {
nt := value{
scalar: appendVia(rhs.scalar, string(cv.Name())),
allocs: rhs.allocs, allocOnly: rhs.allocOnly, scheme: rhs.scheme,
}
if !storable(nt) {
st.delEnv(cv)
return
}
st.setEnv(cv, nt)
a.fact()
}
// storable reports whether a value carries information worth keeping in the
// environment: taint, an allocation identity, or a definite URL scheme.
func storable(v value) bool {
return len(v.scalar) != 0 || len(v.allocs) != 0 || v.scheme.set
}
func (a *analysis) analyzeExprStmt(expr js.IExpr, st *state) {
expr = ungroupExpr(expr)
if be, ok := expr.(*js.BinaryExpr); ok && isAssignOp(be.Op) {
a.handleAssign(be, st)
return
}
a.evalExpr(expr, st)
}
// handleAssign updates state for target = rhs. A plain assignment is a strong
// update that can clear taint; a compound assignment reads the old value and
// combines it. A logical assignment writes only when its short-circuit condition
// allows, so the write and its right side are may-state.
func (a *analysis) handleAssign(be *js.BinaryExpr, st *state) value {
target := ungroupExpr(be.X)
if isLogicalAssignOp(be.Op) {
return a.handleLogicalAssign(be, target, st)
}
switch t := target.(type) {
case *js.Var:
cv := canonicalVar(t)
old := st.env[cv]
rhs := a.evalExpr(be.Y, st)
return a.assignVar(st, cv, old, rhs, be.Op)
case *js.DotExpr, *js.IndexExpr:
return a.assignMember(be, target, st)
default:
a.evalExpr(target, st)
return a.evalExpr(be.Y, st)
}
}
func (a *analysis) assignVar(st *state, cv *js.Var, old, rhs value, op js.TokenType) value {
name := string(cv.Name())
var nt value
if op == js.EqToken {
nt = value{scalar: appendVia(rhs.scalar, name), allocs: rhs.allocs, allocOnly: rhs.allocOnly, scheme: rhs.scheme}
} else {
// A compound assignment coerces to a string or number, so the result is a
// scalar and carries no allocation identity.
nt = value{scalar: appendVia(mergeTaint(old.scalar, rhs.scalar), name)}
if op == js.AddEqToken {
nt.scheme = old.scheme
}
}
if !storable(nt) {
st.delEnv(cv)
return value{}
}
st.setEnv(cv, nt)
a.fact()
return nt
}
func (a *analysis) assignMember(be *js.BinaryExpr, target js.IExpr, st *state) value {
recv, key := a.evalMemberTarget(target, st)
st.setCapture(be, recv)
var old value
if be.Op != js.EqToken {
// Compound assignment captures the current property value before the RHS
// runs. The RHS may overwrite the same field.
old = a.readField(st, recv, key)
}
rhs := a.evalExpr(be.Y, st)
recv = st.captures[be]
st.delCapture(be)
writeVal := rhs
if be.Op != js.EqToken {
writeVal = value{scalar: mergeTaint(old.scalar, rhs.scalar)}
if be.Op == js.AddEqToken {
writeVal.scheme = old.scheme
}
}
a.resourceSrcSink(recv, key, writeVal, be, st)
a.writeField(st, recv, key, writeVal)
return writeVal
}
// handleLogicalAssign models &&=, ||=, and ??=, whose right side and write are
// only reached on the short-circuit path, so both are merged as may-state.
func (a *analysis) handleLogicalAssign(be *js.BinaryExpr, target js.IExpr, st *state) value {
switch t := target.(type) {
case *js.Var:
cv := canonicalVar(t)
skipped := st.clone()
taken := st.clone()
rhs := a.evalExpr(be.Y, taken)
a.assignVar(taken, cv, taken.env[cv], rhs, js.EqToken)
st.replaceWith(mergeState(skipped, taken))
return st.env[cv]
default:
recv, key := a.evalMemberTarget(target, st)
st.setCapture(be, recv)
skipped := st.clone()
taken := st.clone()
rhs := a.evalExpr(be.Y, taken)
takenRecv := taken.captures[be]
resultRecv := mergeValue(skipped.captures[be], takenRecv)
taken.delCapture(be)
skipped.delCapture(be)
a.resourceSrcSink(takenRecv, key, rhs, be, taken)
a.writeField(taken, takenRecv, key, rhs)
st.replaceWith(mergeState(skipped, taken))
return a.readField(st, resultRecv, key)
}
}
// evalMemberTarget evaluates the receiver and key of a member assignment target
// before the right-hand side, and returns the receiver value and field key.
func (a *analysis) evalMemberTarget(target js.IExpr, st *state) (value, fieldKey) {
switch t := ungroupExpr(target).(type) {
case *js.DotExpr:
recv := a.evalExpr(t.X, st)
return recv, fieldKeyOf(ungroupExpr(t.Y))
case *js.IndexExpr:
recv := a.evalExpr(t.X, st)
st.setCapture(t, recv)
a.evalExpr(t.Y, st)
recv = st.captures[t]
st.delCapture(t)
return recv, fieldKeyOf(ungroupExpr(t.Y))
}
return value{}, fieldKey{}
}
// analyzeLoop iterates the body to a fixed point over the loop state.
func (a *analysis) analyzeLoop(body js.IStmt, cond, post js.IExpr, st *state) *state {
a.loopDepth++
defer func() { a.loopDepth-- }()
cur := st.clone()
for {
if !a.alive() {
return cur
}
if cond != nil {
a.evalExpr(cond, cur)
}
bodyOut := a.analyzeStmt(body, cur.clone())
if post != nil {
a.evalExpr(post, bodyOut)
}
merged := mergeState(cur, bodyOut)
if stateEqual(merged, cur) {
return merged
}
if !a.fact() {
return merged
}
cur = merged
}
}
// analyzeDoLoop applies the first body execution before merging later iterations,
// because a do-while body always runs at least once.
func (a *analysis) analyzeDoLoop(body js.IStmt, cond js.IExpr, st *state) *state {
a.loopDepth++
defer func() { a.loopDepth-- }()
cur := a.analyzeStmt(body, st.clone())
if !cur.continues {
return cur
}
if cond != nil {
a.evalExpr(cond, cur)
}
for {
if !a.alive() {
return cur
}
bodyOut := a.analyzeStmt(body, cur.clone())
if cond != nil {
a.evalExpr(cond, bodyOut)
}
merged := mergeState(cur, bodyOut)
if stateEqual(merged, cur) {
return merged
}
if !a.fact() {
return merged
}
cur = merged
}
}
// analyzeIterationLoop applies the implicit iteration assignment before each body
// execution. The incoming state remains an exit alternative because an iterable
// can be empty.
func (a *analysis) analyzeIterationLoop(
body js.IStmt,
init js.IExpr,
iter value,
forOf bool,
st *state,
) *state {
a.loopDepth++
defer func() { a.loopDepth-- }()
cur := st.clone()
for {
if !a.alive() {
return cur
}
bodyIn := cur.clone()
elem := value{}
if forOf {
elem = a.iterationElement(bodyIn, iter)
}
a.applyIterationBinding(init, elem, bodyIn)
bodyOut := a.analyzeStmt(body, bodyIn)
merged := mergeState(cur, bodyOut)
if stateEqual(merged, cur) {
return merged
}
if !a.fact() {
return merged
}
cur = merged
}
}
// iterationElement is the value each element of a for-of iterable can carry:
// scalar taint from iterating a tainted string, plus the iterable's element
// field for an array of values.
func (a *analysis) iterationElement(st *state, iter value) value {
return a.collectArrayElements(st, iter)
}
func (a *analysis) applyIterationBinding(init js.IExpr, elem value, st *state) {
if decl, ok := init.(*js.VarDecl); ok {
if len(decl.List) != 0 {
a.assignIterationVar(decl.List[0].Binding, elem, st)
}
return
}
if v, ok := ungroupExpr(init).(*js.Var); ok {
a.bindVar(st, canonicalVar(v), elem)
return
}
target := ungroupExpr(init)
switch target.(type) {
case *js.DotExpr, *js.IndexExpr:
recv, key := a.evalMemberTarget(target, st)
a.resourceSrcSink(recv, key, elem, target, st)
a.writeField(st, recv, key, elem)
default:
a.evalExpr(target, st)
}
}
func (a *analysis) assignIterationVar(binding js.IBinding, elem value, st *state) {
if v, ok := binding.(*js.Var); ok {
a.bindVar(st, canonicalVar(v), elem)
return
}
a.evalBindingPattern(binding, st)
}
// evalBindingPattern evaluates computed property names and nested defaults. This
// phase does not bind values extracted from a destructuring pattern, but the
// pattern's expressions still execute and can contain sinks or scalar writes.
func (a *analysis) evalBindingPattern(binding js.IBinding, st *state) {
switch b := binding.(type) {
case *js.BindingArray:
for i := range b.List {
a.evalNestedBinding(&b.List[i], st)
}
if b.Rest != nil {
a.evalBindingPattern(b.Rest, st)
}
case *js.BindingObject:
for i := range b.List {
item := &b.List[i]
if item.Key != nil {
a.evalExpr(item.Key.Computed, st)
}
a.evalNestedBinding(&item.Value, st)
}
}
}
func (a *analysis) evalNestedBinding(be *js.BindingElement, st *state) {
if be.Default != nil {
taken := st.clone()
a.evalExpr(be.Default, taken)
st.replaceWith(mergeState(st, taken))
}
a.evalBindingPattern(be.Binding, st)
}
func (a *analysis) analyzeSwitch(s *js.SwitchStmt, st *state) *state {
a.evalExpr(s.Init, st)
out := st.clone()
var fall *state
for i := range s.List {
cl := &s.List[i]
if cl.Cond != nil {
a.evalExpr(cl.Cond, st)
}
in := st.clone()
if fall != nil {
in = mergeState(in, fall)
}
for j := range cl.List {
in = a.analyzeStmt(cl.List[j], in)
}
fall = in
out = mergeState(out, in)
}
return out
}
// analyzeTry models that an exception can occur anywhere in the body, so the
// catch clause sees the pre-body state merged with taint the body may have set,
// and the finally clause applies to every outgoing edge.
func (a *analysis) analyzeTry(s *js.TryStmt, st *state) *state {
frame := len(a.returnExits) - 1
returnStart := 0
if frame >= 0 {
returnStart = len(a.returnExits[frame])
}
bodySt := a.analyzeBlock(s.Body, st.clone())
merged := bodySt
if s.Catch != nil {
catchIn := mergeState(st, bodySt)
catchSt := a.analyzeBlock(s.Catch, catchIn)
merged = mergeState(bodySt, catchSt)
}
if s.Finally != nil {
var pending []functionExit
if frame >= 0 {
pending = append(pending, a.returnExits[frame][returnStart:]...)
a.returnExits[frame] = a.returnExits[frame][:returnStart]
}
if merged.continues {
merged = a.analyzeBlock(s.Finally, merged.clone())
}
if frame >= 0 {
for i := range pending {
out := a.analyzeBlock(s.Finally, pending[i].state.clone())
if out.continues {
pending[i].state = out
a.returnExits[frame] = append(a.returnExits[frame], pending[i])
}
}
}
}
return merged
}
func (a *analysis) evalClass(class *js.ClassDecl, st *state) {
a.evalExpr(class.Extends, st)
// Every computed key is evaluated while the class elements are defined. Static
// fields and blocks initialize only after all keys have been computed.
for i := range class.List {
item := &class.List[i]
if item.Method != nil {
a.evalExpr(item.Method.Name.Computed, st)
} else if item.StaticBlock == nil {
a.evalExpr(item.Name.Computed, st)
}
}
for i := range class.List {
item := &class.List[i]
if item.StaticBlock != nil {
blockSt := a.analyzeBlock(item.StaticBlock, st.clone())
st.replaceWith(blockSt)
} else if item.Method == nil && item.Static {
a.evalExpr(item.Init, st)
}
}
}
// evalExpr returns the abstract value of expr and records any sink it reaches. It
// mutates state only through an assignment used as a value or a may-state merge.
func (a *analysis) evalExpr(expr js.IExpr, st *state) value {
if !a.alive() || !st.continues {
return value{}
}
expr = ungroupExpr(expr)
switch x := expr.(type) {
case nil:
return value{}
case *js.Var:
return st.env[canonicalVar(x)]
case *js.LiteralExpr:
return value{scheme: schemeOfLiteral(x)}
case *js.DotExpr:
if occ, ok := a.sources[expr]; ok {
return a.srcValue(occ.id)
}
base := a.evalExpr(x.X, st)
name, ok := staticStringOrIdent(ungroupExpr(x.Y))
if !ok {
return value{}
}
return a.readField(st, base, fieldKey{kind: fieldNamed, name: name})
case *js.IndexExpr:
if occ, ok := a.sources[expr]; ok {
return a.srcValue(occ.id)
}
base := a.evalExpr(x.X, st)
st.setCapture(x, base)
key := a.evalReadIndexKey(x, st)
base = st.captures[x]
st.delCapture(x)
return a.readField(st, base, key)
case *js.BinaryExpr:
if isAssignOp(x.Op) {
return a.handleAssign(x, st)
}
lx := a.evalExpr(x.X, st)
var rx value
if isLogicalOp(x.Op) {
skipped := st.clone()
taken := st.clone()
rx = a.evalExpr(x.Y, taken)
st.replaceWith(mergeState(skipped, taken))
} else {
rx = a.evalExpr(x.Y, st)
}
if binaryOpPropagates(x.Op) {
if isLogicalOp(x.Op) {
// ||, &&, and ?? return one operand value, so both allocations and
// scalars can flow through. The operator itself is not one of the
// scheme-preserving forms, so destination classification is reset.
out := mergeValue(lx, rx)
out.scheme = schemeState{}
return out
}
out := value{scalar: mergeTaint(lx.scalar, rx.scalar)}
if x.Op == js.AddToken {
// A concatenation's URL scheme is fixed by its leftmost prefix.
out.scheme = lx.scheme
}
return out
}
return value{}
case *js.UnaryExpr:
return a.evalUnary(x, st)
case *js.CondExpr:
a.evalExpr(x.Cond, st)
thenSt := st.clone()
elseSt := st.clone()
thenVal := a.evalExpr(x.X, thenSt)
elseVal := a.evalExpr(x.Y, elseSt)
st.replaceWith(mergeState(thenSt, elseSt))
return mergeValue(thenVal, elseVal)
case *js.TemplateExpr:
return a.evalTemplate(x, st)
case *js.CommaExpr:
var last value
for i := range x.List {
last = a.evalExpr(x.List[i], st)
}
last.scheme = schemeState{}
return last
case *js.CallExpr:
return a.evalCall(x, st)
case *js.NewExpr:
return a.evalNew(x, st)
case *js.ArrayExpr:
return a.evalArray(x, st)
case *js.ObjectExpr:
return a.evalObject(x, st)
case *js.ClassDecl:
a.evalClass(x, st)
return value{}
default:
return value{}
}
}
func (a *analysis) evalUnary(x *js.UnaryExpr, st *state) value {
if x.Op == js.DeleteToken {
a.evalDelete(x.X, st)
return value{}
}
if isUpdateOp(x.Op) {
return a.evalUpdate(x.X, x.Op, st)
}
xt := a.evalExpr(x.X, st)
if x.Op == js.AwaitToken {
if a.stopAtAwait {
st.continues = false
return xt
}
if a.suspensions != nil {
a.suspensions = append(a.suspensions, st.clone())
}
xt.scheme = schemeState{}
return xt
}
if unaryOpPropagates(x.Op) {
return value{scalar: xt.scalar}
}
return value{}
}
func (a *analysis) evalDelete(target js.IExpr, st *state) {
target = ungroupExpr(target)
switch target.(type) {
case *js.DotExpr, *js.IndexExpr:
if optionalChainMaySkip(target) {
skipped := st.clone()
taken := st.clone()
recv, key := a.evalMemberTarget(target, taken)
a.deleteField(taken, recv, key)
st.replaceWith(mergeState(skipped, taken))
return
}
recv, key := a.evalMemberTarget(target, st)
a.deleteField(st, recv, key)
default:
a.evalExpr(target, st)
}
}
func (a *analysis) evalUpdate(target js.IExpr, op js.TokenType, st *state) value {
target = ungroupExpr(target)
switch t := target.(type) {
case *js.Var:
cv := canonicalVar(t)
old := st.env[cv]
return a.assignVar(st, cv, old, value{}, op)
case *js.DotExpr, *js.IndexExpr:
recv, key := a.evalMemberTarget(target, st)
old := a.readField(st, recv, key)
updated := value{scalar: old.scalar}
a.writeField(st, recv, key, updated)
return updated
default:
old := a.evalExpr(target, st)
return value{scalar: old.scalar}
}
}
func isUpdateOp(op js.TokenType) bool {
switch op {
case js.PreIncrToken, js.PreDecrToken, js.PostIncrToken, js.PostDecrToken:
return true
default:
return false
}
}
func (a *analysis) evalTemplate(x *js.TemplateExpr, st *state) value {
if x.Tag != nil {
a.evalExpr(x.Tag, st)
valuesSt := st
if x.Optional {
valuesSt = st.clone()
}
for i := range x.List {
a.evalExpr(x.List[i].Expr, valuesSt)
}
if x.Optional {
st.replaceWith(mergeState(st, valuesSt))
}
// A tag function's return value is not modeled, so the result is clean.
return value{}
}
var ts taintSet
var firstScheme schemeState
for i := range x.List {
v := a.evalExpr(x.List[i].Expr, st)
if i == 0 {
firstScheme = v.scheme
}
ts = mergeTaint(ts, v.scalar)
}
return value{scalar: ts, scheme: templateScheme(x, firstScheme)}
}
// evalReadIndexKey evaluates a read index expression, honoring optional-chain
// short-circuit for its side effects, and returns the field key it selects.
func (a *analysis) evalReadIndexKey(x *js.IndexExpr, st *state) fieldKey {
if x.Optional || optionalChainMaySkip(x.X) {
skipped := st.clone()
taken := st.clone()
a.evalExpr(x.Y, taken)
st.replaceWith(mergeState(skipped, taken))
} else {
a.evalExpr(x.Y, st)
}
return fieldKeyOf(ungroupExpr(x.Y))
}
func (a *analysis) evalNew(x *js.NewExpr, st *state) value {
isArray := a.isGlobalCallee(x.X, "Array")
a.evalExpr(x.X, st)
var args []value
if x.Args != nil {
args = make([]value, len(x.Args.List))
for i := range x.Args.List {
args[i] = a.evalExpr(x.Args.List[i].Value, st)
}
}
site, ok := a.sites[x]
if !ok {
return value{}
}
if v, handled := a.evalNewSink(x, args, site, st); handled {
return v
}
if !isArray {
return a.allocate(st, site, a.loopDepth > 0)
}
a.promoteCurrent(st, site)
fresh := &object{array: true}
if len(args) != 1 || !isNumericLiteralExpr(x.Args.List[0].Value) {
for i := range args {
fresh.setNamed(strconv.Itoa(i), args[i], true)
a.fact()
}
}
return a.installLiteral(st, site, a.loopDepth > 0, fresh)
}
func isNumericLiteralExpr(expr js.IExpr) bool {
_, ok := numericPropertyNameOf(expr)
return ok
}
func (a *analysis) evalArray(x *js.ArrayExpr, st *state) value {
site, ok := a.sites[x]
if !ok {
for i := range x.List {
if x.List[i].Value != nil {
a.evalExpr(x.List[i].Value, st)
}
}
return value{}
}
a.promoteCurrent(st, site)
fresh := &object{array: true}
unknownIndex := false
for i := range x.List {
el := &x.List[i]
if el.Value == nil {
continue
}
ev := a.evalExpr(el.Value, st)
if el.Spread {
fresh.weakElem(a.spreadElements(st, ev), false)
unknownIndex = true
a.fact()
continue
}
if unknownIndex {
fresh.weakElem(ev, true)
} else {
fresh.setNamed(strconv.Itoa(i), ev, true)
}
a.fact()
}
return a.installLiteral(st, site, a.loopDepth > 0, fresh)
}
func (a *analysis) evalObject(x *js.ObjectExpr, st *state) value {
site, ok := a.sites[x]
if !ok {
a.evalObjectSideEffects(x, st)
return value{}
}
// A literal's properties are evaluated into one fresh runtime object. Only
// after the final property is known is that object merged into a loop summary.
a.promoteCurrent(st, site)
fresh := &object{}
for i := range x.List {
p := &x.List[i]
if p.Spread {
sv := a.evalExpr(p.Value, st)
a.spreadInto(st, fresh, sv)
continue
}
if p.Name != nil && p.Name.Computed != nil {
a.evalExpr(p.Name.Computed, st)
}
pv := a.evalExpr(p.Value, st)
if p.Init != nil {
a.evalExpr(p.Init, st)
}
key := fieldKey{kind: fieldWild}
if p.Name != nil {
if p.Name.IsComputed() {
key = fieldKeyOf(ungroupExpr(p.Name.Computed))
} else {
key = fieldKeyOf(&p.Name.Literal)
}
}
switch key.kind {
case fieldNamed:
fresh.setNamed(key.name, pv, true)
case fieldElem:
if key.name != "" {
fresh.setNamed(key.name, pv, true)
} else {
fresh.weakElem(pv, true)
}
default:
fresh.writeWild(pv, true)
}
a.fact()
}
return a.installLiteral(st, site, a.loopDepth > 0, fresh)
}
// evalObjectSideEffects evaluates an object literal's expressions without
// recording fields, used only when the literal had no allocation site assigned.
func (a *analysis) evalObjectSideEffects(x *js.ObjectExpr, st *state) {
for i := range x.List {
p := &x.List[i]
if p.Name != nil && p.Name.Computed != nil {
a.evalExpr(p.Name.Computed, st)
}
a.evalExpr(p.Value, st)
if p.Init != nil {
a.evalExpr(p.Init, st)
}
}
}
// spreadElements collapses a spread source's element taint into one value for
// insertion into the target array element field.
func (a *analysis) spreadElements(st *state, src value) value {
return a.collectArrayElements(st, src)
}
func (a *analysis) collectArrayElements(st *state, src value) value {
var out value
have := false
if len(src.scalar) != 0 {
out, have = mergePresentValue(out, have, value{scalar: src.scalar}), true
}
for ref := range src.allocs {
o := st.heap[ref.id]
if o == nil || !o.array {
continue
}
for name, fv := range o.fields {
if isArrayIndexName(name) {
out, have = mergePresentValue(out, have, applyRefDepth(fv, ref)), true
}
}
if o.elemMay {
out, have = mergePresentValue(out, have, applyRefDepth(o.elem, ref)), true
}
if o.wildMay {
out, have = mergePresentValue(out, have, applyRefDepth(o.wild, ref)), true
}
}
if !have {
return value{}
}
if !src.allocOnly {
out.allocOnly = false
}
return out
}
// spreadInto copies a spread source onto a fresh object literal. A named field
// present on every possible source definitely overwrites the earlier property;
// partial and unresolved fields remain weak updates.
func (a *analysis) spreadInto(st *state, dst *object, src value) {
fields := map[string]value{}
definite := map[string]int{}
elemDefinite := 0
wildDefinite := 0
var elem, wild, wildReq value
elemMay := false
wildMay := false
wildReqSeen := false
for ref := range src.allocs {
so := st.heap[ref.id]
if so == nil {
continue
}
for k, fv := range so.fields {
fv = applyRefDepth(fv, ref)
if old, ok := fields[k]; ok {
fields[k] = mergeValue(old, fv)
} else {
fields[k] = fv
}
if so.must[k] {
definite[k]++
}
}
if so.elemMay {
elem, elemMay = mergePresentValue(elem, elemMay, applyRefDepth(so.elem, ref)), true
}
if so.wildMay {
wild, wildMay = mergePresentValue(wild, wildMay, applyRefDepth(so.wild, ref)), true
}
if so.elemMust {
elemDefinite++
}
if so.wildMust {
wildDefinite++
if wildReqSeen {
wildReq = mergeValue(wildReq, applyRefDepth(so.wildReq, ref))
} else {
wildReq = applyRefDepth(so.wildReq, ref)
wildReqSeen = true
}
}
}
// A scalar spread contributes character properties at numeric keys.
if len(src.scalar) != 0 {
elem, elemMay = mergePresentValue(elem, elemMay, value{scalar: src.scalar}), true
}
allSources := src.allocOnly && len(src.allocs) != 0
if wildMay {
dst.writeWild(wild, false)
}
if allSources && wildDefinite == len(src.allocs) {
dst.wildMust = true
dst.wildReq = wildReq
}
for k, fv := range fields {
dst.setNamed(k, fv, allSources && definite[k] == len(src.allocs))
}
if elemMay {
dst.weakElem(elem, allSources && elemDefinite == len(src.allocs))
}
a.fact()
}
func isLogicalOp(op js.TokenType) bool {
return op == js.AndToken || op == js.OrToken || op == js.NullishToken
}
func isLogicalAssignOp(op js.TokenType) bool {
return op == js.AndEqToken || op == js.OrEqToken || op == js.NullishEqToken
}
func isAssignOp(op js.TokenType) bool {
switch op {
case js.EqToken, js.AddEqToken, js.SubEqToken, js.MulEqToken, js.DivEqToken,
js.ModEqToken, js.ExpEqToken, js.LtLtEqToken, js.GtGtEqToken, js.GtGtGtEqToken,
js.BitAndEqToken, js.BitOrEqToken, js.BitXorEqToken,
js.AndEqToken, js.OrEqToken, js.NullishEqToken:
return true
default:
return false
}
}
// Package log provides a structured-logging wrapper around log/slog.
//
// Rationale: CSM's daemon currently emits timestamped log lines via direct
// fmt.Fprintf(os.Stderr, ...) calls. That works for journalctl but loses
// structure when operators ship logs to Loki, ELK, or Datadog. This
// package provides a drop-in replacement that:
//
// - Emits the legacy "[YYYY-MM-DD HH:MM:SS] msg" format in text mode so
// mixing csmlog calls with legacy fmt.Fprintf calls produces a
// uniform log stream (important during incremental migration)
// - Switches to JSON on CSM_LOG_FORMAT=json for log-shipping pipelines,
// emitting the slog-native level/msg/time/fields structure
// - Honors CSM_LOG_LEVEL={debug|info|warn|error} (default: info)
//
// Usage (preferred for new code):
//
// log.Info("daemon starting", "version", v, "pid", os.Getpid())
// log.Warn("log not found, will retry", "path", path)
// log.Error("alert dispatch failed", "err", err)
//
// Legacy call sites that still use fmt.Fprintf will keep working —
// migration is incremental. See docs/src/development.md for guidance.
package log
import (
"context"
"io"
"log/slog"
"os"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"unicode"
"unicode/utf8"
)
// global is the package-level logger, loaded lazily on first use.
// atomic.Pointer so Init can swap it without data races.
var global atomic.Pointer[slog.Logger]
// Init configures the global logger from environment variables:
//
// CSM_LOG_FORMAT = "text" (default) | "json"
// CSM_LOG_LEVEL = "debug" | "info" (default) | "warn" | "error"
//
// Safe to call multiple times. Returns the installed logger so callers can
// also pass it into subsystems that take a *slog.Logger.
func Init() *slog.Logger {
level := parseLevel(os.Getenv("CSM_LOG_LEVEL"))
handler := buildHandler(os.Getenv("CSM_LOG_FORMAT"), level)
logger := slog.New(handler)
global.Store(logger)
slog.SetDefault(logger)
return logger
}
// L returns the current global logger, initializing it on first call.
// Cheap on the hot path (single atomic load).
func L() *slog.Logger {
if l := global.Load(); l != nil {
return l
}
return Init()
}
// Helpers that mirror slog's method set on the global logger. Provided so
// call sites don't need to write log.L().Info(...) on every line.
func Debug(msg string, args ...any) { L().Debug(msg, args...) }
func Info(msg string, args ...any) { L().Info(msg, args...) }
func Warn(msg string, args ...any) { L().Warn(msg, args...) }
func Error(msg string, args ...any) { L().Error(msg, args...) }
func parseLevel(s string) slog.Level {
switch strings.ToLower(strings.TrimSpace(s)) {
case "debug":
return slog.LevelDebug
case "warn", "warning":
return slog.LevelWarn
case "error", "err":
return slog.LevelError
default:
return slog.LevelInfo
}
}
func buildHandler(format string, level slog.Level) slog.Handler {
opts := &slog.HandlerOptions{Level: level}
switch strings.ToLower(strings.TrimSpace(format)) {
case "json":
return slog.NewJSONHandler(os.Stderr, opts)
default:
return newLegacyTextHandler(os.Stderr, level)
}
}
// legacyTextHandler emits log records in CSM's historical "[timestamp] msg"
// format so callers migrating from fmt.Fprintf produce the same output. Key
// differences from slog.NewTextHandler:
//
// - No "time=... level=... msg=..." prefix; just "[YYYY-MM-DD HH:MM:SS] msg"
// - Structured fields are appended as " key=value" when present
// - Level is prepended as "WARN:" / "ERROR:" only for non-info records
//
// This lets operators mix csmlog calls with the ~180 remaining fmt.Fprintf
// call sites in the daemon without introducing a mixed-format log stream.
type legacyTextHandler struct {
w io.Writer
mu *sync.Mutex
level slog.Level
attrs []slog.Attr
group string
}
// NewTextHandler returns the handler Init installs for the text format,
// writing to w.
func NewTextHandler(w io.Writer, level slog.Level) slog.Handler {
return newLegacyTextHandler(w, level)
}
func newLegacyTextHandler(w io.Writer, level slog.Level) *legacyTextHandler {
return &legacyTextHandler{
w: w,
mu: &sync.Mutex{},
level: level,
}
}
func (h *legacyTextHandler) Enabled(_ context.Context, level slog.Level) bool {
return level >= h.level
}
func (h *legacyTextHandler) Handle(_ context.Context, r slog.Record) error {
var sb strings.Builder
sb.Grow(128)
ts := r.Time
if ts.IsZero() {
ts = time.Now()
}
sb.WriteByte('[')
sb.WriteString(ts.Format("2006-01-02 15:04:05"))
sb.WriteString("] ")
// Prepend a level marker only for non-info records so the info path
// exactly matches the legacy "[ts] msg" format.
switch r.Level {
case slog.LevelWarn:
sb.WriteString("WARN: ")
case slog.LevelError:
sb.WriteString("ERROR: ")
case slog.LevelDebug:
sb.WriteString("DEBUG: ")
}
sb.WriteString(r.Message)
// Append pre-bound attrs then record attrs as " key=value" pairs.
for _, a := range h.attrs {
writeAttr(&sb, a)
}
r.Attrs(func(a slog.Attr) bool {
writeAttr(&sb, a)
return true
})
sb.WriteByte('\n')
h.mu.Lock()
defer h.mu.Unlock()
_, err := io.WriteString(h.w, sb.String())
return err
}
func writeAttr(sb *strings.Builder, a slog.Attr) {
if a.Key == "" {
return
}
sb.WriteString(" ")
sb.WriteString(a.Key)
sb.WriteByte('=')
s := a.Value.Resolve().String()
// Values can name tenant files. Quote and escape any value with
// whitespace, a quote or an unprintable byte, so the key=value pairs stay
// parseable and a value can neither end the line nor forge another.
if strings.ContainsFunc(s, needsQuoting) || !utf8.ValidString(s) {
sb.WriteString(strconv.Quote(s))
} else {
sb.WriteString(s)
}
}
func needsQuoting(r rune) bool {
return r == '"' || unicode.IsSpace(r) || !unicode.IsPrint(r)
}
func (h *legacyTextHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
merged := make([]slog.Attr, 0, len(h.attrs)+len(attrs))
merged = append(merged, h.attrs...)
merged = append(merged, attrs...)
return &legacyTextHandler{
w: h.w,
mu: h.mu,
level: h.level,
attrs: merged,
group: h.group,
}
}
func (h *legacyTextHandler) WithGroup(name string) slog.Handler {
// Groups flatten into the attr key via a prefix; simple implementation
// that's sufficient for CSM's usage (we don't use groups today).
return &legacyTextHandler{
w: h.w,
mu: h.mu,
level: h.level,
attrs: h.attrs,
group: name,
}
}
// Ensure legacyTextHandler satisfies the slog.Handler contract at compile time.
var _ slog.Handler = (*legacyTextHandler)(nil)
// Package adapter renders and applies the MTA-native forward-guard rule. On
// cPanel/exim it writes a router + transport into the cPanel-preserved
// /etc/exim.conf.local include sections and regenerates exim.conf via
// buildeximconf, so the rule survives cPanel exim rebuilds. CSM is never in the
// live mail path: exim evaluates the rule and writes held copies to the
// CSM-owned quarantine Maildir itself.
//
// The exact router/transport here was validated on a real cPanel exim 4.99
// host (null-sender forward held while the local copy delivers; normal mail
// forwarded unchanged; Remove restores normal forwarding).
package adapter
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/mailfwd/policy"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/systemdrun"
)
// Status reports whether the guard rule is currently installed.
type Status struct {
Installed bool `json:"installed"`
}
// ForwardGuard renders and (un)installs the MTA forward-guard rule.
type ForwardGuard interface {
Apply(cfg policy.Config, badIPs []string) error
Remove() error
Status() (Status, error)
RefreshBadIPs(badIPs []string) error
}
// QuarantineDir is the Maildir where the exim transport appends held forward
// copies. The daemon prunes it and the web UI lists it.
const QuarantineDir = "/var/lib/csm/forward_quarantine/held"
// Default on-disk locations (overridable in tests).
const (
defaultLocalConf = "/etc/exim.conf.local"
defaultStateRoot = "/var/lib/csm"
defaultBadIPsPath = "/var/lib/csm/forward_guard/bad_ips"
defaultQuarantineDir = QuarantineDir
// transportUser delivers held copies. It must NOT be root: cPanel lists
// root on exim's never_users, so an appendfile as root fails. mailnull is
// exim's own non-root identity and exists on every cPanel host.
transportUser = "mailnull"
)
// Managed-block sentinels. Apply replaces whatever is between them, so a
// re-apply is idempotent and Remove can strip the block cleanly.
const (
routerBegin = "# CSM-FORWARD-GUARD ROUTER BEGIN (managed by csm; do not edit)"
routerEnd = "# CSM-FORWARD-GUARD ROUTER END"
transportBegin = "# CSM-FORWARD-GUARD TRANSPORT BEGIN (managed by csm; do not edit)"
transportEnd = "# CSM-FORWARD-GUARD TRANSPORT END"
)
// EximAdapter is the cPanel/exim ForwardGuard.
type EximAdapter struct {
localConf string
stateRoot string
badIPsPath string
quarantineDir string
// Injected side effects (real implementations on a live host; fakes in tests).
rebuild func() error // runs buildeximconf
chown func(path, user string) error // chowns the quarantine dir
mkdirAll func(path string, perm os.FileMode) error
}
// NewEximAdapter returns an adapter targeting the standard cPanel locations.
func NewEximAdapter() *EximAdapter {
return &EximAdapter{
localConf: defaultLocalConf,
stateRoot: defaultStateRoot,
badIPsPath: defaultBadIPsPath,
quarantineDir: defaultQuarantineDir,
rebuild: runBuildEximConf,
chown: chownToUser,
mkdirAll: os.MkdirAll,
}
}
// Apply installs (or refreshes) the forward-guard rule for an enabled,
// non-dry-run policy. It is transactional: on any failure the previous
// exim.conf.local is restored and exim is rebuilt back to its prior state, so a
// failed apply never leaves a half-installed rule.
func (a *EximAdapter) Apply(cfg policy.Config, badIPs []string) error {
if !cfg.Enabled {
return fmt.Errorf("forward-guard adapter: cannot apply disabled policy")
}
if cfg.DryRun {
return fmt.Errorf("forward-guard adapter: cannot apply dry-run policy")
}
router, err := a.renderRouter(cfg.HoldSignals)
if err != nil {
return err
}
prev, hadPrev, err := a.readLocalConf()
if err != nil {
return err
}
base := prev
if !hadPrev || strings.TrimSpace(base) == "" {
base = platform.EximLocalSkeleton
}
next, err := injectBlock(base, "@ROUTERSTART@", router)
if err != nil {
return err
}
next, err = injectBlock(next, "@TRANSPORTSTART@", a.renderTransport(cfg.HoldSignals))
if err != nil {
return err
}
// Quarantine dir must exist and be writable by the transport user before
// exim can deliver into it.
if err := a.mkdirAll(a.quarantineDir, 0700); err != nil {
return fmt.Errorf("creating quarantine dir: %w", err)
}
if err := a.chown(a.quarantineDir, transportUser); err != nil {
return fmt.Errorf("chowning quarantine dir to %s: %w", transportUser, err)
}
if err := a.writeBadIPs(badIPs); err != nil {
return err
}
// exim appends held copies as the transport user and reads the bad-IP
// lookup as its own non-root user. Older systemd applies StateDirectoryMode
// only when it creates the state root, so a host installed while that mode
// was 0700 keeps a root-only directory above both paths unless opened here.
for _, dir := range []string{filepath.Dir(a.quarantineDir), filepath.Dir(a.badIPsPath)} {
if err := makeSearchable(a.stateRoot, dir); err != nil {
return err
}
}
if err := writeFileAtomic(a.localConf, []byte(next)); err != nil {
return err
}
if err := a.rebuild(); err != nil {
// Roll back to the prior config so mail keeps flowing as before.
if restoreErr := a.restore(prev, hadPrev); restoreErr != nil {
return fmt.Errorf("buildeximconf failed: %w; rollback failed: %v", err, restoreErr)
}
return fmt.Errorf("buildeximconf failed, rolled back: %w", err)
}
return nil
}
// Remove strips the managed blocks and rebuilds, restoring normal forwarding.
func (a *EximAdapter) Remove() error {
cur, had, err := a.readLocalConf()
if err != nil {
return err
}
if !had {
return nil // nothing installed
}
stripped := stripBlock(cur, routerBegin, routerEnd)
stripped = stripBlock(stripped, transportBegin, transportEnd)
if stripped == cur {
return nil // not installed; no rebuild needed
}
if err := writeFileAtomic(a.localConf, []byte(stripped)); err != nil {
return err
}
if err := a.rebuild(); err != nil {
if restoreErr := a.restore(cur, true); restoreErr != nil {
return fmt.Errorf("buildeximconf failed during remove: %w; rollback failed: %v", err, restoreErr)
}
return fmt.Errorf("buildeximconf failed during remove, rolled back: %w", err)
}
return nil
}
// Status reports whether both managed blocks are present.
func (a *EximAdapter) Status() (Status, error) {
cur, had, err := a.readLocalConf()
if err != nil {
return Status{}, err
}
if !had {
return Status{}, nil
}
routerInstalled := strings.Contains(cur, routerBegin)
transportInstalled := strings.Contains(cur, transportBegin)
if routerInstalled != transportInstalled {
return Status{}, fmt.Errorf("forward-guard adapter: partial install in exim.conf.local (router=%t transport=%t)", routerInstalled, transportInstalled)
}
return Status{Installed: routerInstalled}, nil
}
// holdClause is one enabled hold signal and its Exim condition.
type holdClause struct{ reason, cond string }
// holdClauses lists the enabled routing-time signals in a fixed order. The
// router holds when any matches; the transport records the ones that did.
func (a *EximAdapter) holdClauses(sig policy.HoldSignals) []holdClause {
var out []holdClause
if sig.BounceBackscatter {
out = append(out, holdClause{"bounce_backscatter", "eq{$sender_address}{}"})
}
if sig.BadSenderIP {
out = append(out, holdClause{"bad_sender_ip", fmt.Sprintf("eq{${lookup{$sender_host_address}lsearch{%s}{1}{0}}}{1}", a.badIPsPath)})
}
if sig.SpamFlagged {
// Antispam strips inbound X-Spam headers from the mail it scans before
// tagging; on other mail a forged header holds only the forger's copy.
// Exim joins repeated headers with newlines. A preceding verdict must
// not hide a later Yes, and a word starting with Yes is not a verdict.
out = append(out, holdClause{"spam_flagged", `match{$h_x-spam-status:}{\N(?im)^Yes[[:blank:]]*(,|$)\N}`})
}
return out
}
func (a *EximAdapter) renderRouter(sig policy.HoldSignals) (string, error) {
var clauses []string
for _, c := range a.holdClauses(sig) {
clauses = append(clauses, "{"+c.cond+"}")
}
if len(clauses) == 0 {
// Config validation forbids enforce mode with no such signal; guard here
// so the adapter never installs a router that holds everything or nothing.
return "", fmt.Errorf("forward-guard adapter: no routing-time-enforceable signal enabled (need bounce_backscatter, bad_sender_ip or spam_flagged)")
}
cond := fmt.Sprintf("${if and{ {def:parent_local_part} {or{ %s } } }{yes}{no}}", strings.Join(clauses, " "))
return strings.Join([]string{
routerBegin,
"csm_forward_guard:",
" driver = accept",
" domains = ! +local_domains",
" condition = " + cond,
" transport = csm_forward_hold",
routerEnd,
}, "\n"), nil
}
func (a *EximAdapter) renderTransport(sig policy.HoldSignals) string {
var reasons strings.Builder
for _, c := range a.holdClauses(sig) {
reasons.WriteString("${if " + c.cond + "{" + c.reason + ",}}")
}
headers := strings.Join([]string{
"X-CSM-Forwarder: $parent_local_part@$parent_domain",
"X-CSM-Recipient: $local_part@$domain",
"X-CSM-Sender: $sender_address",
"X-CSM-Reasons: " + reasons.String(),
}, "\\n")
// Exim unquotes the option before expanding it; regex escapes must
// survive both stages.
return strings.Join([]string{
transportBegin,
"csm_forward_hold:",
" driver = appendfile",
" directory = " + a.quarantineDir,
" maildir_format",
" create_directory",
" directory_mode = 0700",
" mode = 0600",
" user = " + transportUser,
" headers_add = " + strconv.Quote(headers),
transportEnd,
}, "\n")
}
// RefreshBadIPs rewrites only the bad-IP lookup file. exim reads the lsearch
// file at lookup time, so the change takes effect immediately with no rebuild
// or reload -- cheap to call on a schedule as the attack DB changes.
func (a *EximAdapter) RefreshBadIPs(ips []string) error {
return a.writeBadIPs(ips)
}
func (a *EximAdapter) writeBadIPs(ips []string) error {
dir := filepath.Dir(a.badIPsPath)
if err := a.mkdirAll(dir, 0711); err != nil {
return fmt.Errorf("creating bad IP lookup dir: %w", err)
}
// Refresh runs inside the daemon sandbox. It may recreate this directory
// under a restrictive umask, but must not change the shared state root.
if err := makeSearchable(dir, dir); err != nil {
return err
}
var buf bytes.Buffer
for _, ip := range ips {
ip = strings.TrimSpace(ip)
if ip == "" || strings.ContainsAny(ip, " \t\r\n:") {
continue // lsearch keys are one bare token per line
}
fmt.Fprintf(&buf, "%s: 1\n", ip)
}
return writeFileAtomic(a.badIPsPath, buf.Bytes())
}
func (a *EximAdapter) readLocalConf() (string, bool, error) {
data, err := os.ReadFile(a.localConf) // #nosec G304 -- operator-fixed exim.conf.local path
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return "", false, nil
}
return "", false, fmt.Errorf("reading %s: %w", a.localConf, err)
}
return string(data), true, nil
}
func (a *EximAdapter) restore(prev string, had bool) error {
if had {
if err := writeFileAtomic(a.localConf, []byte(prev)); err != nil {
return fmt.Errorf("restoring exim.conf.local: %w", err)
}
} else {
if err := os.Remove(a.localConf); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("removing new exim.conf.local: %w", err)
}
}
if err := a.rebuild(); err != nil {
return fmt.Errorf("rebuilding restored exim config: %w", err)
}
return nil
}
// makeSearchable adds the group and other search bits to dir and to every
// directory above it up to and including root. It only adds bits, so a mode an
// operator widened on purpose is left alone, and nothing becomes listable.
func makeSearchable(root, dir string) error {
root, dir = filepath.Clean(root), filepath.Clean(dir)
rel, err := filepath.Rel(root, dir)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return fmt.Errorf("%s is not under %s", dir, root)
}
var paths []string
for p := dir; ; p = filepath.Dir(p) {
paths = append(paths, p)
if p == root {
break
}
}
// These are root-controlled directories. Reject symlinks before changing
// any mode: a link into the private state directory must not expose it.
modes := make([]os.FileMode, len(paths))
for i := len(paths) - 1; i >= 0; i-- {
p := paths[i]
info, err := os.Lstat(p)
if err != nil {
return fmt.Errorf("checking %s: %w", p, err)
}
if !info.IsDir() {
return fmt.Errorf("%s must be a directory, not a symlink or file", p)
}
modes[i] = info.Mode()
}
for i, p := range paths {
if modes[i]&0011 != 0011 {
if err := os.Chmod(p, modes[i]|0011); err != nil {
return fmt.Errorf("opening %s to the exim user: %w", p, err)
}
}
}
return nil
}
// injectBlock removes any existing managed block of the same kind, then inserts
// block immediately after the marker line. Idempotent: re-injecting yields the
// same file.
func injectBlock(conf, marker, block string) (string, error) {
// Strip a prior copy of this block so re-apply doesn't duplicate it.
begin, end := blockSentinels(block)
conf = stripBlock(conf, begin, end)
markerLine := marker + "\n"
idx := strings.Index(conf, markerLine)
if idx < 0 {
return "", fmt.Errorf("exim.conf.local missing %s marker", marker)
}
at := idx + len(markerLine)
return conf[:at] + block + "\n" + conf[at:], nil
}
var blockSentinelRe = regexp.MustCompile(`^(# CSM-FORWARD-GUARD \w+ BEGIN)`)
func blockSentinels(block string) (begin, end string) {
lines := strings.SplitN(block, "\n", 2)
begin = lines[0]
// Derive the END sentinel from the BEGIN kind.
if m := blockSentinelRe.FindStringSubmatch(begin); m != nil {
kind := strings.Fields(m[1])[2] // ROUTER or TRANSPORT
return begin, "# CSM-FORWARD-GUARD " + kind + " END"
}
return begin, ""
}
// stripBlock removes the inclusive begin..end region (and a trailing newline).
func stripBlock(conf, begin, end string) string {
for {
bi := strings.Index(conf, begin)
if bi < 0 {
return conf
}
ei := strings.Index(conf[bi:], end)
if ei < 0 {
return conf
}
stop := bi + ei + len(end)
if stop < len(conf) && conf[stop] == '\n' {
stop++
}
conf = conf[:bi] + conf[stop:]
}
}
func writeFileAtomic(path string, data []byte) error {
return writeFileAtomicMode(path, data, 0644)
}
// writeFileAtomicMode is writeFileAtomic with the file mode chosen by the
// caller, for CSM state that only root reads.
func writeFileAtomicMode(path string, data []byte, perm os.FileMode) error {
dir := filepath.Dir(path)
f, err := os.CreateTemp(dir, "."+filepath.Base(path)+".*.csmtmp") // #nosec G304 -- caller owns the destination path
if err != nil {
return fmt.Errorf("opening temp file for %s: %w", path, err)
}
tmp := f.Name()
if err := f.Chmod(perm); err != nil {
_ = f.Close()
_ = os.Remove(tmp)
return fmt.Errorf("chmod temp file for %s: %w", path, err)
}
if n, err := f.Write(data); err != nil {
_ = f.Close()
_ = os.Remove(tmp)
return fmt.Errorf("writing %s: %w", path, err)
} else if n != len(data) {
_ = f.Close()
_ = os.Remove(tmp)
return fmt.Errorf("writing %s: %w", path, io.ErrShortWrite)
}
if err := f.Close(); err != nil {
_ = os.Remove(tmp)
return fmt.Errorf("closing temp file for %s: %w", path, err)
}
if err := os.Rename(tmp, path); err != nil {
_ = os.Remove(tmp)
return fmt.Errorf("committing %s: %w", path, err)
}
return nil
}
// buildEximConfScript is cPanel's exim config builder (/scripts is the standard
// symlink to /usr/local/cpanel/scripts on cPanel hosts).
const (
buildEximConfScript = "/scripts/buildeximconf"
buildEximConfTimeout = 120 * time.Second
)
// buildEximConfArgv returns the argv that rebuilds exim. buildeximconf is a
// cPanel script that writes an unbounded set of system paths (exim.conf,
// exim.pl.local, cPanel state, etc.), which csm.service's ProtectSystem=strict
// allow-list cannot reasonably enumerate, so the rebuild runs as a transient
// unit forked by PID 1.
func buildEximConfArgv(systemdRunPath string) (string, []string) {
return systemdrun.Argv(systemdRunPath, systemdrun.Options{
RuntimeMax: buildEximConfTimeout,
}, buildEximConfScript)
}
type commandRunner func(context.Context, string, ...string) ([]byte, error)
func runBuildEximConf() error {
ctx, cancel := context.WithTimeout(context.Background(), buildEximConfTimeout)
defer cancel()
systemdRun, _ := exec.LookPath("systemd-run")
return runBuildEximConfCommand(ctx, systemdRun, runCommand)
}
func runBuildEximConfCommand(ctx context.Context, systemdRunPath string, run commandRunner) error {
lookPath := func(string) (string, error) {
if systemdRunPath == "" {
return "", exec.ErrNotFound
}
return systemdRunPath, nil
}
output, err := systemdrun.Run(ctx, lookPath, systemdrun.RunnerFunc(run), systemdrun.Options{
RuntimeMax: buildEximConfTimeout,
}, buildEximConfScript)
if err != nil {
return commandFailure(buildEximConfScript, output, err)
}
return nil
}
func runCommand(ctx context.Context, name string, args ...string) ([]byte, error) {
// #nosec G204 -- argv is built from fixed constants and the resolved
// systemd-run path; no attacker-controlled input.
return exec.CommandContext(ctx, name, args...).CombinedOutput()
}
func commandFailure(name string, output []byte, err error) error {
trimmed := strings.TrimSpace(string(output))
if trimmed == "" {
return fmt.Errorf("%s failed: %w", name, err)
}
return fmt.Errorf("%s failed: %w: %s", name, err, trimmed)
}
func chownToUser(path, user string) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// #nosec G204 -- user is the constant transportUser ("mailnull") and path is
// the operator-fixed quarantine dir; neither is attacker-controlled.
return exec.CommandContext(ctx, "chown", "-R", user+":"+user, path).Run()
}
package adapter
import (
"bytes"
"context"
"crypto/rand"
"encoding/json"
"errors"
"fmt"
"io"
"net/netip"
"os"
"os/exec"
"path/filepath"
"time"
"github.com/pidginhost/csm/internal/eximlock"
"github.com/pidginhost/csm/internal/mailfwd/policy"
"github.com/pidginhost/csm/internal/systemdrun"
)
const eximMutationLimit = 8 << 20
const eximMutationTimeout = 5 * time.Minute
// workerBusyExit is the worker's exit status when another CSM transaction
// holds the Exim lock (EX_TEMPFAIL). systemd-run --pipe returns the unit's
// exit status unchanged, and systemd's own exit codes for a unit it could
// not start begin at 200.
const workerBusyExit = 75
// WorkerExitCode is the forward-guard worker's exit status for the error
// HandleEximMutation returned.
func WorkerExitCode(err error) int {
switch {
case err == nil:
return 0
case errors.Is(err, eximlock.ErrBusy), errors.Is(err, errSuperseded):
return workerBusyExit
default:
return 1
}
}
// The helper accepts policy data, never a destination path or command.
type eximMutation struct {
RequestID string `json:"request_id,omitempty"`
Operation string `json:"operation"`
Config policy.Config `json:"config"`
BadIPs []string `json:"bad_ips"`
}
// applyTo is called only after validation at the worker boundary.
func (r eximMutation) applyTo(target ForwardGuard) error {
if r.Operation == "apply" {
return target.Apply(r.Config, r.BadIPs)
}
return target.Remove()
}
type eximServiceAdapter struct {
*EximAdapter
mutate func(eximMutation) error
}
// NewEximServiceAdapter keeps the entire config transaction, including rollback,
// outside the daemon's mount namespace. Status reads, request records and
// lookup refreshes still use the daemon's existing access to the CSM state
// directory.
func NewEximServiceAdapter() ForwardGuard {
return &eximServiceAdapter{EximAdapter: NewEximAdapter(), mutate: runEximMutation}
}
func (a *eximServiceAdapter) Apply(cfg policy.Config, ips []string) error {
return a.submit(eximMutation{Operation: "apply", Config: cfg, BadIPs: ips})
}
// Invalidate cancels pending requests without waiting for the Exim lock. A
// busy lock must not defer cancellation and let an old worker run as soon as
// the lock is released. Workers that already checked their request finish
// their transaction; the atomic replacement fences all later checks.
func (a *eximServiceAdapter) Invalidate() error {
// #nosec G301 -- Exim reads the bad-IP lookup in this directory as the mail transport user.
if err := os.MkdirAll(filepath.Dir(a.badIPsPath), 0755); err != nil {
return err
}
return writeFileAtomicMode(a.requestPath(), []byte("null"), 0o600)
}
func (a *eximServiceAdapter) Remove() error {
return a.submit(eximMutation{Operation: "remove"})
}
// submit records request as the latest one under the shared Exim lock, then
// hands it to the root worker. A worker started for an older request, such
// as one from before a daemon restart, finds the newer record and changes
// nothing. A removal with no rule installed needs no worker. A busy lock
// defers the request, and a request that cannot be recorded starts no
// worker, because the worker could not tell that it is current.
func (a *eximServiceAdapter) submit(request eximMutation) error {
// Settings can repeat across restarts; each submission must still
// supersede earlier workers with the same policy and sender list.
request.RequestID = rand.Text()
body, err := json.Marshal(request)
if err != nil {
return err
}
done := false
record := func() error {
if writeErr := writeFileAtomicMode(a.requestPath(), body, 0o600); writeErr != nil {
return writeErr
}
if request.Operation == "remove" {
status, statusErr := a.Status()
done = statusErr == nil && !status.Installed
}
return nil
}
if err = eximlock.WithRetry(context.Background(), filepath.Dir(a.badIPsPath), eximlock.RecordWait, record); err != nil || done {
return err
}
return a.mutate(request)
}
func runEximMutation(request eximMutation) error {
binary, err := os.Executable()
if err != nil {
return err
}
body, err := json.Marshal(request)
if err != nil {
return err
}
if len(body) > eximMutationLimit {
return fmt.Errorf("forward-guard request exceeds limit")
}
ctx, cancel := context.WithTimeout(context.Background(), eximMutationTimeout)
defer cancel()
run := func(ctx context.Context, name string, args ...string) ([]byte, error) {
// #nosec G204 -- command is this executable with a fixed helper subcommand, or systemd-run with fixed flags.
cmd := exec.CommandContext(ctx, name, args...)
cmd.Stdin = bytes.NewReader(body)
return cmd.CombinedOutput()
}
return executeEximMutation(ctx, binary, exec.LookPath, run)
}
func executeEximMutation(ctx context.Context, binary string, lookup systemdrun.LookPathFunc, run systemdrun.RunnerFunc) error {
output, err := systemdrun.Run(ctx, lookup, run, systemdrun.Options{Pipe: true, RuntimeMax: eximMutationTimeout}, binary, "forward-guard-worker")
var exit *exec.ExitError
if errors.As(err, &exit) && exit.ExitCode() == workerBusyExit {
return fmt.Errorf("forward-guard worker: %w", eximlock.ErrBusy)
}
if err != nil {
return commandFailure("forward-guard worker", output, err)
}
return nil
}
// HandleEximMutation handles one bounded request at the privileged CLI boundary.
// Only the fixed cPanel locations in NewEximAdapter are available to callers.
func HandleEximMutation(input io.Reader) error {
return handleEximMutation(input, lockedEximAdapter{NewEximAdapter()}.mutate)
}
func handleEximMutation(input io.Reader, mutate func(eximMutation) error) error {
body, err := io.ReadAll(io.LimitReader(input, eximMutationLimit+1))
if err != nil {
return err
}
if len(body) > eximMutationLimit {
return fmt.Errorf("forward-guard request exceeds limit")
}
var request eximMutation
decoder := json.NewDecoder(bytes.NewReader(body))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&request); err != nil {
return fmt.Errorf("invalid forward-guard request: %w", err)
}
if err := decoder.Decode(new(any)); !errors.Is(err, io.EOF) {
return fmt.Errorf("forward-guard request has trailing data")
}
switch request.Operation {
case "apply":
if !request.Config.Enabled || request.Config.DryRun || (!request.Config.HoldSignals.BounceBackscatter && !request.Config.HoldSignals.BadSenderIP && !request.Config.HoldSignals.SpamFlagged) {
return fmt.Errorf("forward-guard request must enable an enforceable policy")
}
for _, ip := range request.BadIPs {
addr, err := netip.ParseAddr(ip)
if err != nil || addr.Zone() != "" {
return fmt.Errorf("forward-guard request has an invalid address")
}
}
case "remove":
if request.Config != (policy.Config{}) || len(request.BadIPs) != 0 {
return fmt.Errorf("remove request must not contain policy data")
}
default:
return fmt.Errorf("unsupported forward-guard operation")
}
return mutate(request)
}
package adapter
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"github.com/pidginhost/csm/internal/eximlock"
)
// errSuperseded reports a request the daemon replaced before this worker
// checked it. The newer request decides, so it is not a failure.
var errSuperseded = errors.New("a newer forward-guard request replaced this one")
// requestFile holds the latest request the daemon handed to the worker.
// Submissions replace it under the shared lock. Observe-mode cancellation
// replaces it atomically without the lock, so busy transactions cannot delay it.
const requestFile = "request.json"
// lockedEximAdapter runs one worker transaction under the shared Exim lock,
// only if its request is current when the transaction begins.
type lockedEximAdapter struct{ *EximAdapter }
func (a lockedEximAdapter) mutate(request eximMutation) error {
return eximlock.WithRetry(context.Background(), filepath.Dir(a.badIPsPath), eximlock.RecordWait, func() error {
latest, err := a.isLatest(request)
if err != nil {
return err
}
if !latest {
return errSuperseded
}
return request.applyTo(a.EximAdapter)
})
}
func (a *EximAdapter) requestPath() string {
return filepath.Join(filepath.Dir(a.badIPsPath), requestFile)
}
// isLatest reports whether request is the one the daemon recorded last. The
// daemon records the same encoding it sends, including the submission ID.
// Only a request without an ID can predate request recording.
func (a *EximAdapter) isLatest(request eximMutation) (bool, error) {
recorded, err := os.ReadFile(a.requestPath()) // #nosec G304 -- fixed CSM state path next to the lock
if errors.Is(err, os.ErrNotExist) {
return request.RequestID == "", nil
}
if err != nil {
return false, err
}
body, err := json.Marshal(request)
if err != nil {
return false, err
}
return bytes.Equal(recorded, body), nil
}
// Package guard glues the operator config to the pure forward-guard policy.
// It exists so internal/mailfwd/policy stays dependency-free of internal/config
// (policy is a low-level leaf; only this glue knows about both).
package guard
import (
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mailfwd/policy"
)
// PolicyFromConfig projects the operator's forward-guard config onto the policy
// input. Only the fields the verdict needs are mapped; skip-list and retention
// are consumed by the adapter and quarantine, not by Verdict.
func PolicyFromConfig(fg config.ForwardGuardConfig) policy.Config {
return policy.Config{
Enabled: fg.Enabled,
DryRun: fg.DryRun,
HoldSignals: policy.HoldSignals{
BounceBackscatter: fg.HoldSignals.BounceBackscatter,
SpamFlagged: fg.HoldSignals.SpamFlagged,
Malware: fg.HoldSignals.Malware,
BadSenderIP: fg.HoldSignals.BadSenderIP,
AuthFail: fg.HoldSignals.AuthFail,
},
}
}
package guard
import (
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/mailfwd/adapter"
)
// Reconciler drives the exim forward-guard from operator config. The daemon
// calls Reconcile on startup and on every config reload, and RefreshBadIPs on a
// schedule. It is the only thing that decides apply-vs-remove, so the live mail
// path and the dry-run path can never both be active.
type Reconciler struct {
// Guard is the MTA adapter (nil on platforms without one).
Guard adapter.ForwardGuard
// Active permits mail changes; the daemon sets it true only on a
// cPanel/exim host in enforce mode.
Active bool
// BadIPs supplies the current bad-sender-IP set (from the reputation DB).
BadIPs func() []string
// Observe is set on a cPanel host in observe mode. Reconcile then only
// cancels pending requests. Workers that already checked their request
// finish their transaction.
Observe bool
}
// pendingCanceler is a guard that can cancel requests still waiting for a
// worker without starting one.
type pendingCanceler interface{ Invalidate() error }
// Reconcile installs the guard when it is enabled and enforcing, and removes it
// otherwise (disabled or dry-run). Dry-run never installs an MTA rule -- its
// accounting is CSM-side only.
func (r Reconciler) Reconcile(fg config.ForwardGuardConfig) error {
if r.Guard == nil {
return nil
}
if !r.Active {
if canceler, ok := r.Guard.(pendingCanceler); ok && r.Observe {
return canceler.Invalidate()
}
return nil
}
if fg.Enabled && !fg.DryRun {
return r.Guard.Apply(PolicyFromConfig(fg), r.badIPs())
}
return r.Guard.Remove()
}
// RefreshBadIPs rewrites the bad-IP lookup file while the guard is enforcing.
// It is a no-op when the guard is not installed, so it is safe to call on a
// timer regardless of config.
func (r Reconciler) RefreshBadIPs(fg config.ForwardGuardConfig) error {
if !r.Active || r.Guard == nil || !fg.Enabled || fg.DryRun {
return nil
}
return r.Guard.RefreshBadIPs(r.badIPs())
}
func (r Reconciler) badIPs() []string {
if r.BadIPs == nil {
return nil
}
return r.BadIPs()
}
package intel
import (
"context"
"errors"
"fmt"
"strings"
"time"
)
// FlushResult reports the observable outcome of a queue flush.
type FlushResult struct {
Removed int `json:"removed"`
// Targeted is the number of message IDs submitted for removal. It is kept
// off the wire because the public response historically exposes only the
// post-action count, but callers need it to audit an unconfirmed outcome.
Targeted int `json:"-"`
}
// QueueFlusher removes safe-to-delete backscatter from the mail queue.
type QueueFlusher interface {
FlushBackscatter() (FlushResult, error)
}
// FrozenBackscatterIDs returns the message IDs of messages that are BOTH frozen
// AND null-sender (<>) in `exim -bp` output. This is the only set the flush
// touches: a frozen null-sender message is undeliverable bounce backscatter,
// so removing it cannot lose a real sender's mail or interrupt a live retry.
// A message exim could not fully read is left for the operator.
func FrozenBackscatterIDs(out string) []string {
msgs, err := parseQueueEntries(out, 0)
if err != nil {
return nil
}
return frozenBackscatterIDs(msgs)
}
func frozenBackscatterIDs(msgs []QueueMessage) []string {
var ids []string
for _, m := range msgs {
if m.Bounce && m.Frozen && m.Unreadable == "" {
ids = append(ids, m.ID)
}
}
return ids
}
// eximRemoveBatch bounds how many message IDs are passed to one `exim -Mrm`
// invocation so a huge queue cannot overflow the command line.
const eximRemoveBatch = 100
// EximQueueFlusher lists the queue, selects frozen null-sender messages, and
// removes them with `exim -Mrm`.
type EximQueueFlusher struct {
list func() ([]byte, error)
remove func(ids []string) error
}
// NewEximQueueFlusher returns a flusher backed by the live exim binary.
func NewEximQueueFlusher() *EximQueueFlusher {
return &EximQueueFlusher{list: runEximBp, remove: runEximRemove}
}
// survivorsListed bounds how many surviving message IDs are named in the
// error text so a large stuck queue cannot produce an unreadable message.
const survivorsListed = 5
// FlushBackscatter removes every frozen null-sender message currently queued.
//
// The queue, not the exit status, decides the outcome. exim -Mrm has been
// observed removing every message it was handed and still exiting non-zero,
// which reported a failure for work that had already completed and skipped
// the caller's audit record. Re-reading the queue afterwards establishes
// what actually left, so a lying exit code can neither invent a failure nor
// hide a removal that silently did nothing.
//
// Removed is the number of targeted IDs absent from the post-action queue and
// stays meaningful alongside a non-nil error. A concurrent delivery can also
// make an ID disappear, so audit callers must describe it as no longer queued,
// not necessarily deleted by this command.
func (f *EximQueueFlusher) FlushBackscatter() (FlushResult, error) {
out, err := f.list()
if err != nil {
return FlushResult{}, err
}
msgs, err := parseQueueEntries(string(out), 0)
if err != nil {
return FlushResult{}, err
}
ids := frozenBackscatterIDs(msgs)
if len(ids) == 0 {
return FlushResult{}, nil
}
result := FlushResult{Targeted: len(ids)}
removeErr := f.remove(ids)
after, listErr := f.list()
var remaining []QueueMessage
if listErr == nil {
remaining, listErr = parseQueueEntries(string(after), 0)
}
if listErr != nil {
confirmErr := fmt.Errorf("removal could not be confirmed: %w", listErr)
if removeErr != nil {
return result, errors.Join(fmt.Errorf("removal command failed: %w", removeErr), confirmErr)
}
return result, confirmErr
}
targeted := make(map[string]struct{}, len(ids))
for _, id := range ids {
targeted[id] = struct{}{}
}
stillQueued := make(map[string]struct{}, len(ids))
for _, m := range remaining {
if _, isTargeted := targeted[m.ID]; isTargeted {
stillQueued[m.ID] = struct{}{}
}
}
var survived []string
for _, id := range ids {
if _, ok := stillQueued[id]; ok {
survived = append(survived, id)
}
}
result.Removed = len(ids) - len(survived)
if len(survived) == 0 {
return result, nil
}
named := survived
if len(named) > survivorsListed {
named = named[:survivorsListed]
}
msg := fmt.Sprintf("%d of %d frozen backscatter messages still queued after removal (%s)",
len(survived), len(ids), strings.Join(named, " "))
if removeErr != nil {
return result, fmt.Errorf("%s: %w", msg, removeErr)
}
return result, errors.New(msg)
}
func runEximRemove(ids []string) error {
var firstErr error
failedBatches := 0
for start := 0; start < len(ids); start += eximRemoveBatch {
end := start + eximRemoveBatch
if end > len(ids) {
end = len(ids)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
args := append([]string{"-Mrm"}, ids[start:end]...)
_, err := runEximCommand(ctx, 30*time.Second, args...)
cancel()
if err != nil {
// Exim can remove every ID and still return non-zero. Continue so an
// unreliable status from one batch cannot prevent later attempts;
// the queue re-read decides which messages remain. Retaining only the
// first error keeps the eventual operator message bounded.
failedBatches++
if firstErr == nil {
firstErr = err
}
}
}
if firstErr != nil {
return fmt.Errorf("%d removal batch(es) reported an error; first: %w", failedBatches, firstErr)
}
return nil
}
// Package intel turns exim_mainlog deferral lines into operator-facing
// reputation signals: which outbound IPs are being throttled, by which mail
// providers, and for what stated reason. It answers "why is the queue backing
// up" without CSM sitting in the mail path -- it only reads the log the MTA
// already writes.
//
// The parser is deliberate about attacker-controlled content: a deferral line
// echoes a remote server's free-text error, so every parsed field is bounded
// and the line is rejected unless it has the exact exim deferral shape.
package intel
import (
"net"
"regexp"
"sort"
"strings"
"time"
"unicode/utf8"
"github.com/pidginhost/csm/internal/eximlog"
"github.com/pidginhost/csm/internal/mailfwd/inventory"
)
// eximTimeLayout is exim_mainlog's default timestamp ("2026-06-07 10:15:23").
const eximTimeLayout = "2006-01-02 15:04:05"
// maxTextLen bounds the stored remote error text so a hostile MTA cannot bloat
// the report with a multi-kilobyte error string.
const (
maxAddressLen = 254
maxDomainLen = 253
maxHostLen = 253
maxIPLen = 45
maxTextLen = 240
)
// Deferral is one parsed exim "==" deferral event.
type Deferral struct {
Time time.Time
Recipient string
Domain string
Provider inventory.ProviderClass
RemoteHost string // the deferring MX (from H=)
RemoteIP string // the deferring MX address
OutboundIP string // this server's sending IP, as echoed in the error, "" if absent
SMTPCode string // "421", "" if none
ReasonCode string // bracketed code (TSS04) or a keyword label (spamhaus, rate_limit)
Text string // bounded remote error text
}
var (
hostBoundaryRe = regexp.MustCompile(`\bH=(\S{1,253})\s+\[([0-9a-fA-F:.]{1,45})\]:\s*`)
smtpCodeRe = regexp.MustCompile(`\b([45]\d{2})\b`)
reasonRe = regexp.MustCompile(`\[([A-Za-z][A-Za-z0-9]{1,7})\]`)
ipv4Re = regexp.MustCompile(`\b\d{1,3}(?:\.\d{1,3}){3}\b`)
whitespaceR = regexp.MustCompile(`\s+`)
)
// parseDeferralLine parses one exim_mainlog line. ok is false for anything that
// is not a deferral (delivery "=>", arrival "<=", failure "**", blanks, junk).
func parseDeferralLine(line string) (Deferral, bool) {
p, rest, ok := eximlog.SplitPrefix(line, time.Local)
if !ok {
return Deferral{}, false
}
return parseDeferral(line, p, rest)
}
// parseDeferral is parseDeferralLine for a line whose prefix is already read.
func parseDeferral(line string, p eximlog.Prefix, rest string) (Deferral, bool) {
// <msgid> == <recipient> ...
fields := strings.Fields(rest)
if p.MessageID == "" || len(fields) < 2 || fields[0] != "==" {
return Deferral{}, false
}
recipient := strings.Trim(fields[1], "<>")
if recipient == "" || len(recipient) > maxAddressLen || !strings.Contains(recipient, "@") {
return Deferral{}, false
}
d := Deferral{Recipient: recipient}
if at := strings.LastIndexByte(recipient, '@'); at >= 0 && at < len(recipient)-1 {
d.Domain = strings.ToLower(recipient[at+1:])
}
if d.Domain == "" || len(d.Domain) > maxDomainLen {
return Deferral{}, false
}
d.Time = p.Time
errText := fallbackDeferralText(line)
canExtractOutboundIP := false
if m := hostBoundaryRe.FindStringSubmatchIndex(line); m != nil {
d.RemoteHost = line[m[2]:m[3]]
d.RemoteIP = parseIPLiteral(line[m[4]:m[5]])
errText = line[m[1]:]
canExtractOutboundIP = true
}
d.Provider = classifyMX(d.RemoteHost)
if d.Provider == "" {
d.Provider = inventory.ClassifyAddress(recipient)
}
d.SMTPCode = firstSMTPCode(errText)
if canExtractOutboundIP {
d.OutboundIP = firstIPv4(errText)
}
d.ReasonCode = classifyReason(errText)
d.Text = boundText(errText)
return d, true
}
// mxProviders maps provider MX host suffixes to the provider class. Google's
// aspmx hosts sit under google.com and googlemail.com, and Microsoft's under
// outlook.com, hosted business domains included
// (*.mail.protection.outlook.com).
var mxProviders = []struct {
suffix string
provider inventory.ProviderClass
}{
{"google.com", inventory.ProviderGmail},
{"googlemail.com", inventory.ProviderGmail},
{"outlook.com", inventory.ProviderOutlook},
{"yahoodns.net", inventory.ProviderYahoo},
}
// classifyMX returns the provider whose MX host deferred, or "" when the
// host is absent or no known provider's. The suffix must start a label, so
// a look-alike such as google.com.example.net does not match.
func classifyMX(host string) inventory.ProviderClass {
host = strings.TrimSuffix(strings.ToLower(host), ".")
for _, mx := range mxProviders {
if host == mx.suffix || strings.HasSuffix(host, "."+mx.suffix) {
return mx.provider
}
}
return ""
}
func fallbackDeferralText(line string) string {
if i := strings.Index(line, " defer ("); i >= 0 {
if j := strings.Index(line[i:], "):"); j >= 0 {
return strings.TrimSpace(line[i+j+2:])
}
}
return line
}
func parseIPLiteral(s string) string {
if len(s) > maxIPLen {
return ""
}
ip := net.ParseIP(s)
if ip == nil {
return ""
}
return s
}
func firstSMTPCode(s string) string {
for _, m := range smtpCodeRe.FindAllStringSubmatchIndex(s, -1) {
start, end := m[2], m[3]
if smtpCodeIsAddressFragment(s, start, end) {
continue
}
return s[start:end]
}
return ""
}
func smtpCodeIsAddressFragment(s string, start, end int) bool {
if start > 0 {
switch s[start-1] {
case '.', ':':
return true
}
}
if end < len(s) {
switch s[end] {
case '.', ':':
return true
}
}
return false
}
// firstIPv4 returns the first syntactically valid IPv4 address in s, or "".
func firstIPv4(s string) string {
for _, cand := range ipv4Re.FindAllString(s, -1) {
if ip := net.ParseIP(cand); ip != nil && ip.To4() != nil {
return cand
}
}
return ""
}
// classifyReason resolves a stable reason token: a bracketed provider code
// (e.g. TSS04) when present, otherwise a keyword label derived from the error
// text. Returns "" when nothing recognizable is found.
func classifyReason(errText string) string {
for _, m := range reasonRe.FindAllStringSubmatch(errText, -1) {
if validReasonCode(m[1]) {
return m[1]
}
}
return keywordReason(errText)
}
// keywordReason labels error text by keyword, or returns "". The labels after
// "blocked" say why a queued message waits rather than what a provider
// objects to; they come last so text an earlier label matches keeps it.
func keywordReason(errText string) string {
low := strings.ToLower(errText)
switch {
case strings.Contains(low, "spamhaus"):
return "spamhaus"
case strings.Contains(low, "unusual rate"), strings.Contains(low, "rate limit"),
strings.Contains(low, "too many"), strings.Contains(low, "unexpected volume"):
return "rate_limit"
case strings.Contains(low, "complaint"):
return "complaint"
case strings.Contains(low, "greylist"), strings.Contains(low, "grey-list"),
strings.Contains(low, "try again later"):
return "greylist"
case strings.Contains(low, "blocked"), strings.Contains(low, "blacklist"),
strings.Contains(low, "listed"):
return "blocked"
case strings.Contains(low, "mailbox full"), strings.Contains(low, "mailbox is full"),
strings.Contains(low, "over quota"), strings.Contains(low, "quota exceeded"):
return "mailbox_full"
case strings.Contains(low, "retry time not reached"):
return "retry_wait"
case strings.Contains(low, "no route to host"), strings.Contains(low, "network is unreachable"),
strings.Contains(low, "connection timed out"), strings.Contains(low, "host lookup did not complete"),
strings.Contains(low, "host not found"), strings.Contains(low, "non-existent host"),
strings.Contains(low, "unrouteable"):
return "host_not_found"
case strings.Contains(low, "refused"), strings.Contains(low, "rejected"),
strings.Contains(low, "denied"):
return "refused"
}
return ""
}
func validReasonCode(code string) bool {
upper := strings.ToUpper(code)
if strings.HasPrefix(upper, "TLS") {
return false
}
trailingDigits := 0
for i := len(code) - 1; i >= 0; i-- {
if code[i] < '0' || code[i] > '9' {
break
}
trailingDigits++
}
return trailingDigits >= 2
}
func boundText(s string) string {
s = strings.ToValidUTF8(s, "?")
s = strings.TrimSpace(whitespaceR.ReplaceAllString(s, " "))
if len(s) <= maxTextLen {
return s
}
// Truncate on a rune boundary: the deferral text is attacker-influenced,
// so a byte-position cut could split a multi-byte rune and store invalid
// UTF-8 that JSON would then mangle.
cut := maxTextLen
for cut > 0 && !utf8.RuneStart(s[cut]) {
cut--
}
return s[:cut]
}
// ReasonCount is a reason token and how often it occurred.
type ReasonCount struct {
Code string `json:"code"`
Count int `json:"count"`
}
// ProviderCount is a provider class and how often it appeared.
type ProviderCount struct {
Provider string `json:"provider"`
Count int `json:"count"`
}
// ProviderRollup aggregates deferrals to one provider class.
type ProviderRollup struct {
Provider string `json:"provider"`
Deferrals int `json:"deferrals"`
Reasons []ReasonCount `json:"reasons"`
LastSeen time.Time `json:"last_seen,omitzero"`
Sample string `json:"sample"`
}
// OutboundIPRollup aggregates deferrals affecting one of this server's sending
// IPs -- the reputation picture for that address.
type OutboundIPRollup struct {
IP string `json:"ip"`
Deferrals int `json:"deferrals"`
Providers []ProviderCount `json:"providers"`
Reasons []ReasonCount `json:"reasons"`
LastSeen time.Time `json:"last_seen,omitzero"`
}
// Report is the aggregated deferral picture over a window of log lines.
// Lines is how many lines were read. WindowStart and WindowEnd are the times
// of the first and last lines read that carry one, deferral or not; both are
// zero when none does.
type Report struct {
Deferrals int `json:"deferrals"`
Providers []ProviderRollup `json:"providers"`
OutboundIPs []OutboundIPRollup `json:"outbound_ips"`
Lines int `json:"lines"`
WindowStart time.Time `json:"window_start,omitzero"`
WindowEnd time.Time `json:"window_end,omitzero"`
}
// emptyReport returns a zero report with non-nil slices so it serializes as []
// rather than null.
func emptyReport() Report {
return Report{Providers: []ProviderRollup{}, OutboundIPs: []OutboundIPRollup{}}
}
// BuildReport parses every line and aggregates deferrals by provider and by
// outbound IP. Non-deferral lines are ignored.
func BuildReport(lines []string) Report {
provAgg := map[string]*provAccum{}
ipAgg := map[string]*ipAccum{}
deferrals := 0
var windowStart, windowEnd time.Time
for _, line := range lines {
p, rest, ok := eximlog.SplitPrefix(line, time.Local)
if !ok {
continue
}
if windowStart.IsZero() {
windowStart = p.Time
}
windowEnd = p.Time
d, ok := parseDeferral(line, p, rest)
if !ok {
continue
}
deferrals++
prov := string(d.Provider)
pa := provAgg[prov]
if pa == nil {
pa = &provAccum{reasons: map[string]int{}}
provAgg[prov] = pa
}
pa.add(d)
if d.OutboundIP != "" {
ia := ipAgg[d.OutboundIP]
if ia == nil {
ia = &ipAccum{providers: map[string]int{}, reasons: map[string]int{}}
ipAgg[d.OutboundIP] = ia
}
ia.add(d)
}
}
rep := emptyReport()
rep.Deferrals = deferrals
rep.Lines = len(lines)
rep.WindowStart, rep.WindowEnd = windowStart, windowEnd
for prov, pa := range provAgg {
rep.Providers = append(rep.Providers, ProviderRollup{
Provider: prov,
Deferrals: pa.count,
Reasons: sortedReasons(pa.reasons),
LastSeen: pa.lastSeen,
Sample: pa.sample,
})
}
for ip, ia := range ipAgg {
rep.OutboundIPs = append(rep.OutboundIPs, OutboundIPRollup{
IP: ip,
Deferrals: ia.count,
Providers: sortedProviders(ia.providers),
Reasons: sortedReasons(ia.reasons),
LastSeen: ia.lastSeen,
})
}
sort.Slice(rep.Providers, func(i, j int) bool {
if rep.Providers[i].Deferrals != rep.Providers[j].Deferrals {
return rep.Providers[i].Deferrals > rep.Providers[j].Deferrals
}
return rep.Providers[i].Provider < rep.Providers[j].Provider
})
sort.Slice(rep.OutboundIPs, func(i, j int) bool {
if rep.OutboundIPs[i].Deferrals != rep.OutboundIPs[j].Deferrals {
return rep.OutboundIPs[i].Deferrals > rep.OutboundIPs[j].Deferrals
}
return rep.OutboundIPs[i].IP < rep.OutboundIPs[j].IP
})
return rep
}
type provAccum struct {
count int
reasons map[string]int
lastSeen time.Time
sample string
}
func (p *provAccum) add(d Deferral) {
p.count++
if d.ReasonCode != "" {
p.reasons[d.ReasonCode]++
}
if d.Time.After(p.lastSeen) {
p.lastSeen = d.Time
}
if p.sample == "" {
p.sample = d.Text
}
}
type ipAccum struct {
count int
providers map[string]int
reasons map[string]int
lastSeen time.Time
}
func (a *ipAccum) add(d Deferral) {
a.count++
a.providers[string(d.Provider)]++
if d.ReasonCode != "" {
a.reasons[d.ReasonCode]++
}
if d.Time.After(a.lastSeen) {
a.lastSeen = d.Time
}
}
func sortedReasons(m map[string]int) []ReasonCount {
out := make([]ReasonCount, 0, len(m))
for code, n := range m {
out = append(out, ReasonCount{Code: code, Count: n})
}
sort.Slice(out, func(i, j int) bool {
if out[i].Count != out[j].Count {
return out[i].Count > out[j].Count
}
return out[i].Code < out[j].Code
})
return out
}
func sortedProviders(m map[string]int) []ProviderCount {
out := make([]ProviderCount, 0, len(m))
for prov, n := range m {
out = append(out, ProviderCount{Provider: prov, Count: n})
}
sort.Slice(out, func(i, j int) bool {
if out[i].Count != out[j].Count {
return out[i].Count > out[j].Count
}
return out[i].Provider < out[j].Provider
})
return out
}
package intel
import (
"errors"
"io/fs"
"os"
"path/filepath"
"regexp"
"strings"
"syscall"
"time"
)
// StuckReason says why a queued message has not left: a class to filter and
// colour by, plain words for the operator, and the bounded error text it was
// read from.
type StuckReason struct {
Class string `json:"class"`
Text string `json:"text"`
Raw string `json:"raw"`
}
const (
// msglogTailBytes bounds the msglog read. Exim appends every delivery
// attempt with the remote's reply, so the file grows for as long as the
// message stays queued; the newest lines carry the current reason.
msglogTailBytes = 8 << 10
msglogTailLines = 20
)
// stuckClasses maps a keywordReason label to its queue reason class.
var stuckClasses = map[string]string{
"rate_limit": "rate_limited",
"greylist": "greylisted",
"spamhaus": "blocked",
"blocked": "blocked",
"complaint": "blocked",
"mailbox_full": "mailbox_full",
"retry_wait": "retry_wait",
"host_not_found": "host_not_found",
"refused": "refused",
}
var stuckTexts = map[string]string{
"rate_limited": "rate limited",
"mailbox_full": "mailbox full",
"refused": "refused by the receiving server",
"host_not_found": "receiving server not found or unreachable",
"greylisted": "greylisted",
"blocked": "blocked by the receiving server",
"retry_wait": "retry time not reached",
"other": "other",
}
var noAttemptReason = StuckReason{Class: "none", Text: "no delivery attempt logged yet"}
// msglogNameRe matches what names a mailbox or host in a msglog line: <...>
// paths, "host [ip]" pairs with or without H=, bare addresses and quoted
// domains. Senders and recipient domains choose that text, so it is blanked
// before keyword classification. A quoted word without a dot is the remote's
// own reply wording and stays, unless Exim names it as the routing domain.
var msglogNameRe = regexp.MustCompile(`for any host for '[A-Za-z0-9.-]+'|<[^<>]*>|(?:H=)?[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+ \[(?:\d{1,3}(?:\.\d{1,3}){3}|[0-9A-Fa-f]*:[0-9A-Fa-f:.]+)\]|\S+@\S+|'[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+'`)
// ReadMsglogReason reads the tail of a queued message's exim msglog and says
// why the message is stuck, returning the last lines of that tail with it. A
// message without a msglog has had no delivery attempt yet. A msglog that is
// a symlink or not a regular file is refused with an error and never read.
func ReadMsglogReason(spoolRoot, id string) (StuckReason, []string, error) {
if !ValidMessageID(id) {
return StuckReason{}, nil, errors.New("invalid exim message id")
}
dir := filepath.Join(spoolRoot, "msglog")
// With split_spool_directory exim files a message under the sixth ID
// character; without it every msglog sits in one directory.
for _, path := range []string{filepath.Join(dir, id[5:6], id), filepath.Join(dir, id)} {
lines, err := readMsglogTail(path)
if errors.Is(err, fs.ErrNotExist) {
continue
}
if err != nil {
return noAttemptReason, nil, err
}
reason := msglogReason(lines)
if len(lines) > msglogTailLines {
lines = lines[len(lines)-msglogTailLines:]
}
return reason, lines, nil
}
return noAttemptReason, nil, nil
}
func readMsglogTail(path string) ([]string, error) {
// #nosec G304 -- path joins the spool root with an ID that passed
// ValidMessageID. O_NOFOLLOW refuses a planted symlink, O_NONBLOCK keeps a
// fifo from blocking the open, and readFileTail refuses non-regular files.
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
return readFileTail(f, msglogTailBytes)
}
// msglogReason reads the reason from the newest delivery attempt. An attempt
// is the newest error line, the address lines logged with it, and the host
// lines just before them: the address line can repeat only the host, so the
// host line may carry the reason. An earlier attempt never explains a newer
// one; when nothing in the newest attempt is recognized, it is other.
func msglogReason(lines []string) StuckReason {
i := len(lines) - 1
for ; i >= 0; i-- {
_, msg := splitMsglogTime(lines[i])
if info, ok := strings.CutPrefix(msg, "*** Frozen"); ok {
return frozenReason(info, lines[:i])
}
if msglogErrorLine(msg) {
break
}
}
if i < 0 {
return noAttemptReason
}
attemptTime, newest := splitMsglogTime(lines[i])
sawHost := false
for j := i; j >= 0; j-- {
at, msg := splitMsglogTime(lines[j])
if !msglogErrorLine(msg) {
break
}
if strings.HasPrefix(msg, "H=") {
sawHost = true
} else if sawHost || at != attemptTime {
break // an address line from an earlier attempt
}
errText := msglogErrText(msg)
if class, ok := stuckClasses[keywordReason(msglogNameRe.ReplaceAllString(errText, " "))]; ok {
return StuckReason{Class: class, Text: stuckTexts[class], Raw: boundText(errText)}
}
}
return StuckReason{Class: "other", Text: stuckTexts["other"], Raw: boundText(msglogErrText(newest))}
}
// frozenReason reports a frozen message with the cause exim wrote after
// "*** Frozen" and the newest error logged before it.
func frozenReason(info string, older []string) StuckReason {
info = strings.TrimSpace(info)
if strings.HasPrefix(info, "(") && strings.HasSuffix(info, ")") {
info = strings.TrimSpace(info[1 : len(info)-1])
}
r := StuckReason{Class: "frozen", Text: "frozen"}
if info != "" {
r.Text = boundText("frozen: " + info)
}
for i := len(older) - 1; i >= 0; i-- {
if _, msg := splitMsglogTime(older[i]); msglogErrorLine(msg) {
r.Raw = boundText(msglogErrText(msg))
break
}
}
return r
}
// splitMsglogTime splits the leading exim timestamp, if any, from the line.
func splitMsglogTime(line string) (stamp, msg string) {
n := len(eximTimeLayout)
if len(line) > n && line[n] == ' ' {
if _, err := time.Parse(eximTimeLayout, line[:n]); err == nil {
return line[:n], line[n+1:]
}
}
return "", line
}
// msglogErrorLine reports whether a msglog line records a failed attempt: a
// host-level error, an address deferral or an error reply. The arrival line
// never counts because the sender chooses its subject, and names are blanked
// so an address such as error@ cannot make a delivery line look failed.
func msglogErrorLine(msg string) bool {
if strings.HasPrefix(msg, "Received from ") {
return false
}
if strings.HasPrefix(msg, "H=") {
return true
}
text := msglogNameRe.ReplaceAllString(msg, " ")
return strings.Contains(text, " defer") || strings.Contains(strings.ToLower(text), "error")
}
// msglogErrText cuts a msglog line down to the error: from "SMTP error" when
// present, else from the deferral keyword, else the whole line. The result
// still holds addresses and host names; classification blanks them first.
func msglogErrText(msg string) string {
if i := strings.Index(msg, "SMTP error"); i >= 0 {
return msg[i:]
}
if i := strings.Index(msg, " defer"); i >= 0 {
return msg[i+1:]
}
return msg
}
package intel
import (
"context"
"errors"
"fmt"
"math"
"net/mail"
"os/exec"
"regexp"
"sort"
"strconv"
"strings"
"time"
"unicode"
"github.com/pidginhost/csm/internal/systemdrun"
)
// QueueComposition is the makeup of the exim queue: how much is real mail still
// trying to deliver versus null-sender bounce backscatter, how much is frozen,
// and which recipients are stuck the most.
type QueueComposition struct {
Total int `json:"total"`
Bounce int `json:"bounce"` // null-sender <> messages (backscatter)
Real int `json:"real"`
Frozen int `json:"frozen"`
FlushableBackscatter int `json:"flushable_backscatter"` // frozen AND null-sender: safe to flush
// OldestAgeSeconds is the age of the oldest queued message; left out
// when the queue is empty.
OldestAgeSeconds *int `json:"oldest_age_seconds,omitempty"`
TopRecipients []RecipientCount `json:"top_recipients"`
// Unreadable counts messages exim could not read. Those without an
// envelope count as neither real mail nor backscatter.
Unreadable int `json:"unreadable"`
}
// RecipientCount is a recipient address and how many queued messages target it.
type RecipientCount struct {
Address string `json:"address"`
Count int `json:"count"`
}
// QueueMessage is one message in the exim queue as `exim -bp` lists it.
type QueueMessage struct {
ID string `json:"id"`
Arrived time.Time `json:"arrived"`
AgeSeconds int `json:"age_seconds"`
// Size is in bytes, read back from the rounded size exim prints.
Size int64 `json:"size"`
Sender string `json:"sender"` // without angle brackets; "" for a bounce
LocalUser string `json:"local_user"` // the "(user)" marker, "" if absent
Frozen bool `json:"frozen"`
Bounce bool `json:"bounce"` // null sender <>
// Recipients holds the first undelivered recipients: at most
// maxQueueRecipients in a list, maxDetailRecipients in a detail.
Recipients []string `json:"recipients"`
RecipientCount int `json:"recipient_count"` // every undelivered recipient
Delivered int `json:"delivered"` // recipients exim marks D
// Unreadable is why exim could not read the message, such as "spool
// format error"; "" for a readable message. Without an envelope the
// sender is "" and Bounce is false, and only removal works.
Unreadable string `json:"unreadable,omitempty"`
}
const (
topRecipientLimit = 10
maxQueueRecipients = 50
maxDetailRecipients = 1000
// Queue headers are padded for age alignment. Recipient and continuation
// lines are indented much deeper and must not be parsed as new messages.
maxQueueHeaderIndent = 4
)
var (
// A queue header line: "<age> <size> <msgid> <sender> [(login)] [*** frozen ***]",
// where exim adds "(login)" when an untrusted caller set the sender. The
// login is also accepted before the sender. Accept both the legacy 6-6-2
// message id and the longer base62 form exim 4.97+ emits (6-11-4, e.g.
// "1wVR8E-0000000C9po-1DDg").
queueMsgIDRe = regexp.MustCompile(`^[0-9A-Za-z]{6}-(?:[0-9A-Za-z]{6}-[0-9A-Za-z]{2}|[0-9A-Za-z]{11}-[0-9A-Za-z]{4})$`)
queueAgeRe = regexp.MustCompile(`^\d+[smhdw]$`)
queueSizeRe = regexp.MustCompile(`(?i)^\d+(?:\.\d+)?[kmgt]?$`)
)
var queueSizeUnits = map[byte]float64{'k': 1 << 10, 'm': 1 << 20, 'g': 1 << 30, 't': 1 << 40}
// base62Digits is the alphabet exim encodes message IDs in.
const base62Digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
var (
eximLookPath = exec.LookPath
eximRun = func(ctx context.Context, name string, args ...string) ([]byte, error) {
// #nosec G204 -- name is either the resolved systemd-run path or Exim,
// and args contain fixed flags plus validated Exim message IDs.
return exec.CommandContext(ctx, name, args...).Output()
}
)
// ValidMessageID reports whether id is an exim message ID in either format.
// It gates every ID that reaches exim argv or a spool path: the pattern is
// anchored and admits only base62 groups, so no leading dash, path separator
// or whitespace can pass.
func ValidMessageID(id string) bool {
return queueMsgIDRe.MatchString(id)
}
// ArrivalFromID decodes the arrival time exim encodes in the first six
// characters of a message ID: seconds since the epoch in base 62. The legacy
// and the exim 4.97+ formats share that prefix.
func ArrivalFromID(id string) (time.Time, bool) {
if !ValidMessageID(id) {
return time.Time{}, false
}
var secs int64
for i := 0; i < 6; i++ {
secs = secs*62 + int64(strings.IndexByte(base62Digits, id[i]))
}
return time.Unix(secs, 0).UTC(), true
}
// ParseQueue parses `exim -bp` output into a composition summary. A message
// exim could not read counts as unreadable; one without an envelope counts as
// neither real mail nor backscatter, and none is flushable.
func ParseQueue(out string) (QueueComposition, error) {
comp := QueueComposition{TopRecipients: []RecipientCount{}}
recipients := map[string]int{}
oldestSeconds := -1
msgs, err := parseQueueEntries(out, math.MaxInt)
if err != nil {
return comp, err
}
for _, m := range msgs {
comp.Total++
if m.Bounce {
comp.Bounce++
} else if m.Sender != "" {
comp.Real++
}
if m.Unreadable != "" {
comp.Unreadable++
}
if m.Frozen {
comp.Frozen++
}
if m.Bounce && m.Frozen && m.Unreadable == "" {
comp.FlushableBackscatter++
}
if m.AgeSeconds > oldestSeconds {
oldestSeconds = m.AgeSeconds
oldest := m.AgeSeconds
comp.OldestAgeSeconds = &oldest
}
for _, addr := range m.Recipients {
recipients[addr]++
}
}
comp.TopRecipients = topRecipients(recipients)
return comp, nil
}
// ParseQueueList parses `exim -bp` output into one entry per queued message,
// keeping the first maxQueueRecipients undelivered recipients of each. Output
// that holds text but no queue entry is an error, not an empty queue.
func ParseQueueList(out []byte) ([]QueueMessage, error) {
return parseQueueMessages(out, maxQueueRecipients)
}
// ParseQueueDetail parses `exim -bp <id>` output for one message, keeping the
// first maxDetailRecipients undelivered recipients. Anything but exactly one
// message is an error.
func ParseQueueDetail(out []byte) (QueueMessage, error) {
msgs, err := parseQueueMessages(out, maxDetailRecipients)
if err != nil {
return QueueMessage{}, err
}
if len(msgs) != 1 {
return QueueMessage{}, fmt.Errorf("exim -bp listed %d messages, want 1", len(msgs))
}
return msgs[0], nil
}
func parseQueueMessages(out []byte, keep int) ([]QueueMessage, error) {
text := string(out)
msgs, err := parseQueueEntries(text, keep)
if err != nil {
return nil, err
}
if len(msgs) == 0 {
return []QueueMessage{}, nil
}
for i := range msgs {
m := &msgs[i]
m.Arrived, _ = ArrivalFromID(m.ID)
if m.Recipients == nil {
m.Recipients = []string{}
}
}
return msgs, nil
}
// errSpoolUnreadable means exim reported a spool error under no message it
// listed, so the listing cannot be read as the whole queue.
var errSpoolUnreadable = errors.New("exim -bp reported a spool error for no listed message")
// spoolDataMissing marks a message exim listed with a blank size: it could
// not stat the data file.
const spoolDataMissing = "spool data file missing"
// parseQueueEntries walks `exim -bp` output into messages, keeping the first
// keep undelivered recipients of each and counting all of them. Recipient
// lines are indented deeper than headers; any other line ends the current
// message so stray text cannot attach to it. A message exim could not read
// is listed with why: exim prints the error right below its line.
func parseQueueEntries(out string, keep int) ([]QueueMessage, error) {
var msgs []QueueMessage
current := -1
sawEntry := false
// afterHeader is set while the previous line was current's header, and
// bareID while it was a message ID exim printed without an envelope.
afterHeader, bareID := false, ""
for _, line := range strings.Split(out, "\n") {
trimmed := strings.TrimSpace(line)
prevHeader, prevID := afterHeader, bareID
afterHeader, bareID = false, ""
if text, frozen, ok := spoolErrorLine(trimmed); ok {
switch {
case prevID != "" && trimmed == eximNoSuchMessage:
// Delivered or removed while exim listed the queue.
current = -1
case prevID != "":
m := QueueMessage{ID: prevID, Unreadable: text}
// exim prints no age without the envelope; the ID dates it.
if at, ok := ArrivalFromID(prevID); ok {
m.AgeSeconds = max(0, int(time.Since(at)/time.Second))
}
msgs = append(msgs, m)
current = -1
case prevHeader:
// exim read the envelope and still lists the recipients.
msgs[current].Unreadable = text
msgs[current].Frozen = msgs[current].Frozen || frozen
default:
return nil, errSpoolUnreadable
}
continue
}
if prevID != "" {
return nil, errors.New("exim -bp omitted the spool error after a message ID")
}
if m, ok := parseQueueHeader(line); ok {
msgs = append(msgs, m)
current = len(msgs) - 1
afterHeader = true
sawEntry = true
continue
}
if id, ok := queueBareID(line); ok {
bareID, current = id, -1
sawEntry = true
continue
}
if current < 0 {
continue
}
if trimmed == "" {
continue
}
if queueHeaderCandidate(line) || queueHeaderIndent(line) <= maxQueueHeaderIndent || !queueRecipientLine(line) {
current = -1
continue
}
m := &msgs[current]
// Exim prefixes an already-delivered recipient with "D"; it is not stuck.
if addr, delivered := strings.CutPrefix(trimmed, "D "); delivered {
if _, ok := queueRecipientAddress(strings.TrimSpace(addr)); ok {
m.Delivered++
}
continue
}
if addr, ok := queueRecipientAddress(trimmed); ok {
m.RecipientCount++
if len(m.Recipients) < keep {
m.Recipients = append(m.Recipients, addr)
}
}
}
if bareID != "" {
return nil, errors.New("exim -bp omitted the spool error after a message ID")
}
if !sawEntry && strings.TrimSpace(out) != "" {
return nil, errors.New("exim -bp output holds no queue entry")
}
return msgs, nil
}
// spoolErrorLine reads the error exim prints below a message it could not
// read, such as "*** spool format error: size=37 ***", with the frozen mark
// exim may add after it. A recipient cannot pose as one: an address holding
// spaces is quoted, so its line starts with a quote.
func spoolErrorLine(trimmed string) (text string, frozen, ok bool) {
rest, found := strings.CutPrefix(trimmed, "*** spool ")
end := strings.Index(rest, " ***")
if !found || end < 0 {
return "", false, false
}
switch strings.TrimSpace(rest[end+len(" ***"):]) {
case "":
case "*** frozen ***":
frozen = true
default:
return "", false, false
}
return boundText("spool " + rest[:end]), frozen, true
}
// queueBareID reads the line exim prints for a message whose envelope it
// could not read: the ID alone, after a blank size.
func queueBareID(line string) (string, bool) {
// The five-character blank size and its separator occupy six spaces.
// Recipient indentation must not be mistaken for a missing envelope.
id, ok := strings.CutPrefix(line, " ")
if ok && queueMsgIDRe.MatchString(id) {
return id, true
}
return "", false
}
func parseQueueHeader(line string) (QueueMessage, bool) {
if queueHeaderIndent(line) > maxQueueHeaderIndent {
return QueueMessage{}, false
}
age, rest := cutQueueField(line)
size, rest := cutQueueField(rest)
var id string
if queueMsgIDRe.MatchString(size) {
// exim leaves the size blank when it cannot stat the data file.
id, size = size, ""
} else {
id, rest = cutQueueField(rest)
}
if !queueAgeRe.MatchString(age) || (size != "" && !queueSizeRe.MatchString(size)) || !queueMsgIDRe.MatchString(id) {
return QueueMessage{}, false
}
m := QueueMessage{ID: id, AgeSeconds: AgeToSeconds(age)}
if size == "" {
m.Unreadable = spoolDataMissing
} else {
m.Size = queueSizeBytes(size)
}
rest = strings.TrimSpace(rest)
if field, after := cutQueueField(rest); queueLocalUserField(field) {
m.LocalUser = field[1 : len(field)-1]
rest = strings.TrimSpace(after)
}
var sender string
if strings.HasPrefix(rest, "<") {
// The sender may quote spaces or ">" in its local part. Exim's own
// markers after it never contain ">", so the last one closes it.
end := strings.LastIndexByte(rest, '>')
if end < 0 {
return QueueMessage{}, false
}
sender, rest = rest[1:end], rest[end+1:]
m.Bounce = sender == ""
} else {
sender, rest = cutQueueField(rest)
if sender == "" {
return QueueMessage{}, false
}
}
m.Sender = sender
marks := strings.Fields(rest)
if len(marks) > 0 && m.LocalUser == "" && queueLocalUserField(marks[0]) {
m.LocalUser = marks[0][1 : len(marks[0])-1]
marks = marks[1:]
}
if len(marks) == 3 && marks[0] == "***" && marks[1] == "frozen" && marks[2] == "***" {
m.Frozen = true
marks = nil
}
if len(marks) != 0 {
return QueueMessage{}, false
}
return m, true
}
// cutQueueField returns the first whitespace-separated field of s and the
// text after it.
func cutQueueField(s string) (field, rest string) {
s = strings.TrimLeftFunc(s, unicode.IsSpace)
if i := strings.IndexFunc(s, unicode.IsSpace); i >= 0 {
return s[:i], s[i:]
}
return s, ""
}
// queueSizeBytes converts a size token that matched queueSizeRe ("800",
// "2.9K", "1.2M") to bytes. Exim rounds what it prints, so the result is the
// size the token shows, not the exact spool size.
func queueSizeBytes(token string) int64 {
token = strings.ToLower(token)
unit := 1.0
if u, ok := queueSizeUnits[token[len(token)-1]]; ok {
unit = u
token = token[:len(token)-1]
}
// The token matched queueSizeRe, so the only possible error is a range
// overflow, which returns +Inf and is clamped below.
n, _ := strconv.ParseFloat(token, 64)
// float64(math.MaxInt64) rounds up to 2^63, which int64 cannot hold.
if size := n * unit; size < math.MaxInt64 {
return int64(size)
}
return math.MaxInt64
}
func queueHeaderCandidate(line string) bool {
if queueHeaderIndent(line) > maxQueueHeaderIndent {
return false
}
fields := strings.Fields(line)
return (len(fields) == 4 || len(fields) == 7) && queueMsgIDRe.MatchString(fields[2])
}
func queueRecipientLine(line string) bool {
return len(line) > 0 && (line[0] == ' ' || line[0] == '\t')
}
func queueRecipientAddress(trimmed string) (string, bool) {
addr := trimmed
if strings.HasPrefix(addr, "<") && strings.HasSuffix(addr, ">") {
addr = addr[1 : len(addr)-1]
}
if len(addr) > maxAddressLen || !strings.Contains(addr, "@") {
return "", false
}
if parsed, err := mail.ParseAddress(addr); err != nil || parsed.Name != "" {
return "", false
}
return addr, true
}
func queueLocalUserField(field string) bool {
if len(field) < 3 || field[0] != '(' || field[len(field)-1] != ')' {
return false
}
return !strings.ContainsAny(field[1:len(field)-1], "() \t\r\n")
}
func queueHeaderIndent(line string) int {
for i := 0; i < len(line); i++ {
switch line[i] {
case ' ':
continue
case '\t':
return maxQueueHeaderIndent + 1
default:
return i
}
}
return len(line)
}
// AgeToSeconds converts an exim age token (e.g. "25m", "4d") to seconds. An
// unrecognized token returns 0.
func AgeToSeconds(age string) int {
if len(age) < 2 {
return 0
}
n, err := strconv.Atoi(age[:len(age)-1])
if err != nil {
return 0
}
switch age[len(age)-1] {
case 's':
return n
case 'm':
return n * 60
case 'h':
return n * 3600
case 'd':
return n * 86400
case 'w':
return n * 604800
}
return 0
}
func topRecipients(m map[string]int) []RecipientCount {
out := make([]RecipientCount, 0, len(m))
for addr, n := range m {
out = append(out, RecipientCount{Address: addr, Count: n})
}
sort.Slice(out, func(i, j int) bool {
if out[i].Count != out[j].Count {
return out[i].Count > out[j].Count
}
return out[i].Address < out[j].Address
})
if len(out) > topRecipientLimit {
out = out[:topRecipientLimit]
}
return out
}
// QueueReporter produces the queue composition for the host.
type QueueReporter interface {
Composition() (QueueComposition, error)
}
// EmptyQueueReporter yields an empty composition. It stands in on platforms
// with no exim queue (non-cPanel).
type EmptyQueueReporter struct{}
func (EmptyQueueReporter) Composition() (QueueComposition, error) {
return QueueComposition{TopRecipients: []RecipientCount{}}, nil
}
// EximQueueSource runs `exim -bp` and parses the result.
type EximQueueSource struct {
run func() ([]byte, error)
}
// NewEximQueueSource returns a source that reads the live exim queue.
func NewEximQueueSource() *EximQueueSource {
return &EximQueueSource{run: runEximBp}
}
// Composition lists the queue and summarizes it. exim failing to list it is
// an error: an empty composition would read as an empty queue.
func (s *EximQueueSource) Composition() (QueueComposition, error) {
out, err := s.run()
if err != nil {
return QueueComposition{TopRecipients: []RecipientCount{}}, err
}
return ParseQueue(string(out))
}
func runEximBp() ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return runEximCommand(ctx, 10*time.Second, "-bp")
}
func runEximCommand(ctx context.Context, runtimeMax time.Duration, args ...string) ([]byte, error) {
eximPath, err := eximLookPath("exim")
if err != nil {
return nil, err
}
return systemdrun.Run(ctx, eximLookPath, eximRun, systemdrun.Options{
Pipe: true,
RuntimeMax: runtimeMax,
}, eximPath, args...)
}
package intel
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/pidginhost/csm/internal/systemdrun"
)
// ErrNotQueued means exim no longer holds the message: it was delivered,
// returned to its sender or removed.
var ErrNotQueued = errors.New("message is not in the queue")
// Outcomes of a queue action, read from the queue after the action ran.
const (
OutcomeRemoved = "removed"
OutcomeBounced = "bounced"
OutcomeAttemptStarted = "attempt_started"
OutcomeThawed = "thawed"
OutcomeGone = "gone"
OutcomeUnchanged = "unchanged"
)
// queueActionFlags maps each per-message action to its exim option.
var queueActionFlags = map[string]string{
"retry": "-M", // delivery attempt now; thaws a frozen message first
"bounce": "-Mg", // give up and return the message to its sender
"delete": "-Mrm", // remove without telling the sender
"thaw": "-Mt",
}
// ValidQueueAction reports whether action names a per-message queue action.
func ValidQueueAction(action string) bool {
_, ok := queueActionFlags[action]
return ok
}
// QueueAdmin reads single queued messages and runs the per-message actions an
// administrator can take on them.
type QueueAdmin interface {
// Message lists one message with exim -bp <id>. It returns ErrNotQueued
// when exim does not hold a message with that ID.
Message(id string) (QueueMessage, error)
// Reason reads why the message is stuck from its msglog, with the last
// lines of that log.
Reason(id string) (StuckReason, []string, error)
// Act runs one action on the message and returns the outcome the queue
// shows afterwards. With OutcomeUnchanged an error means the action
// failed; with any other outcome it is what exim reported while the
// queue shows the change. A message that left the queue while exim
// reported an error is OutcomeGone, unless exim confirmed removal of
// an unreadable message.
Act(action, id string) (string, error)
}
// eximNoSuchMessage is what exim -bp <id> prints below the message line when
// the message's spool header does not exist.
const eximNoSuchMessage = "*** spool read error: No such file or directory ***"
const (
queueReadTimeout = 10 * time.Second
queueActionTimeout = 30 * time.Second
)
// EximQueueAdmin runs exim as transient units forked by PID 1 and reads
// msglogs under the exim spool root.
type EximQueueAdmin struct {
spoolRoot string
lookPath systemdrun.LookPathFunc
run systemdrun.RunnerFunc
}
// NewEximQueueAdmin returns an admin for the exim queue under spoolRoot that
// resolves and runs commands through lookPath and run.
func NewEximQueueAdmin(spoolRoot string, lookPath systemdrun.LookPathFunc, run systemdrun.RunnerFunc) *EximQueueAdmin {
return &EximQueueAdmin{spoolRoot: spoolRoot, lookPath: lookPath, run: run}
}
func (a *EximQueueAdmin) Message(id string) (QueueMessage, error) {
if !ValidMessageID(id) {
return QueueMessage{}, errors.New("invalid exim message id")
}
// exim aborts inside csm's sandbox even for a read, so the read never
// runs there either.
out, err := a.exim(systemdrun.Options{Pipe: true, Required: true, RuntimeMax: queueReadTimeout}, queueReadTimeout, "-bp", id)
// The reply, not the exit status, says the message is missing: exim
// reports it on stdout whatever status it exits with.
lines := strings.Split(strings.TrimRight(string(out), "\r\n"), "\n")
if len(lines) == 2 && strings.TrimSpace(lines[1]) == eximNoSuchMessage {
if missingID, ok := queueBareID(lines[0]); ok && missingID == id {
return QueueMessage{}, ErrNotQueued
}
}
if err != nil {
return QueueMessage{}, err
}
// An empty reply is an error: it does not say the message is gone.
m, err := ParseQueueDetail(out)
if err != nil {
return QueueMessage{}, err
}
if m.ID != id {
return QueueMessage{}, ErrNotQueued
}
return m, nil
}
func (a *EximQueueAdmin) Reason(id string) (StuckReason, []string, error) {
return ReadMsglogReason(a.spoolRoot, id)
}
// Act checks that the message is queued, runs the action and reads the
// queue again. As with FlushBackscatter, the queue decides the outcome; a
// removal or bounce requires a successful command, except when exim confirms
// removal of an unreadable message despite its error exit.
func (a *EximQueueAdmin) Act(action, id string) (string, error) {
flag, ok := queueActionFlags[action]
if !ok {
return OutcomeUnchanged, fmt.Errorf("unknown queue action %q", action)
}
before, err := a.Message(id)
if errors.Is(err, ErrNotQueued) {
return OutcomeGone, nil
}
if err != nil {
return OutcomeUnchanged, fmt.Errorf("read the message before acting: %w", err)
}
if before.Unreadable != "" && action != "delete" {
// exim refuses every other action on a message it cannot read.
return OutcomeUnchanged, fmt.Errorf("exim cannot read this message (%s); only delete works on it", before.Unreadable)
}
if action == "thaw" && !before.Frozen {
// A stale list, auto_thaw or another admin: nothing to thaw.
return OutcomeUnchanged, nil
}
if action == "retry" {
// A delivery attempt lasts as long as the receiving servers make it,
// and stopping it midway can deliver the message twice. The unit
// gets no runtime limit and nobody waits for it.
if _, err = a.exim(systemdrun.Options{Required: true, NoWait: true}, queueActionTimeout, flag, id); err != nil {
return OutcomeUnchanged, err
}
return OutcomeAttemptStarted, nil
}
out, runErr := a.exim(systemdrun.Options{Pipe: true, Required: true, RuntimeMax: queueActionTimeout}, queueActionTimeout, flag, id)
if errors.Is(runErr, systemdrun.ErrUnavailable) {
return OutcomeUnchanged, runErr
}
if runErr != nil {
// The text reaches the audit log, so both parts are bounded.
msg := "exim " + flag + ": " + boundText(runErr.Error())
if text := boundText(string(out)); text != "" {
msg += ": " + text
}
runErr = errors.New(msg)
}
after, err := a.Message(id)
switch {
case errors.Is(err, ErrNotQueued) && runErr == nil && action == "delete":
return OutcomeRemoved, nil
case errors.Is(err, ErrNotQueued) && before.Unreadable != "" && action == "delete" && eximRemovalConfirmed(out, id):
// Exim exits 1 after removing an unreadable message. Its completion
// line distinguishes that from a failed command while another
// caller removed the message or repaired it for delivery.
return OutcomeRemoved, runErr
case errors.Is(err, ErrNotQueued) && runErr == nil && action == "bounce":
return OutcomeBounced, nil
case errors.Is(err, ErrNotQueued):
// Delivered or removed by someone else while it was being thawed,
// or while exim failed the action: -Mrm reports "locked" when a
// delivery holds the message, and the delivery can then take it.
return OutcomeGone, runErr
case err != nil:
// %v, not %w: the action ran, so an ErrUnavailable from the read
// must not read as "nothing ran".
return OutcomeUnchanged, errors.Join(runErr, fmt.Errorf("the outcome could not be read back: %v", err))
case action == "thaw" && before.Frozen && !after.Frozen:
return OutcomeThawed, runErr
}
if runErr == nil {
runErr = errors.New("exim left the message as it was")
}
return OutcomeUnchanged, runErr
}
func eximRemovalConfirmed(out []byte, id string) bool {
confirmation := "Message " + id + " has been removed"
for line := range strings.SplitSeq(string(out), "\n") {
if line == confirmation || line == confirmation+" or did not exist" {
return true
}
}
return false
}
func (a *EximQueueAdmin) exim(opt systemdrun.Options, timeout time.Duration, args ...string) ([]byte, error) {
eximPath, err := a.lookPath("exim")
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return systemdrun.Run(ctx, a.lookPath, a.run, opt, eximPath, args...)
}
package intel
import (
"bytes"
"fmt"
"io"
"os"
"strings"
"github.com/pidginhost/csm/internal/platform"
)
// Default read bounds: enough recent history to see a throttle pattern without
// reading an unbounded multi-gigabyte log.
const (
defaultTailBytes = 8 << 20 // 8 MiB
defaultTailLines = 20000
)
// Reporter produces a deferral Report for the host.
type Reporter interface {
Report() (Report, error)
}
// EmptyReporter yields an empty report. It stands in on platforms with no exim
// log (non-cPanel) until their adapters land.
type EmptyReporter struct{}
func (EmptyReporter) Report() (Report, error) { return emptyReport(), nil }
// EximSource reads the tail of exim_mainlog and builds a deferral report.
type EximSource struct {
path string
tailBytes int64
tailLines int
readTail func(path string, maxBytes int64, maxLines int) []string
}
// NewEximSource returns a source reading the standard exim_mainlog location.
func NewEximSource() *EximSource {
return &EximSource{
path: platform.EximMainLog,
tailBytes: defaultTailBytes,
tailLines: defaultTailLines,
readTail: tailLines,
}
}
// Report reads the recent tail of the log and aggregates deferrals. A missing
// or unreadable log yields an empty report, not an error: no log just means no
// observed deferrals, and this is a read-only visibility surface.
func (s *EximSource) Report() (Report, error) {
return BuildReport(s.readTail(s.path, s.tailBytes, s.tailLines)), nil
}
// tailLines returns up to maxLines trailing lines of the file, reading at most
// the last maxBytes so a huge log never loads whole into memory.
func tailLines(path string, maxBytes int64, maxLines int) []string {
if maxBytes <= 0 || maxLines <= 0 {
return nil
}
f, err := os.Open(path) // #nosec G304 -- fixed exim_mainlog path, operator-scoped.
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
lines, err := readFileTail(f, maxBytes)
if err != nil {
return nil
}
if len(lines) > maxLines {
lines = lines[len(lines)-maxLines:]
}
return lines
}
// readFileTail reads at most the last maxBytes of a regular file, seeking past
// the rest, and splits them into lines. A partial first line (from the
// byte-window cut) is dropped.
func readFileTail(f *os.File, maxBytes int64) ([]string, error) {
info, err := f.Stat()
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() {
return nil, fmt.Errorf("%s is not a regular file", f.Name())
}
offset := info.Size() - maxBytes
if offset > 0 {
if _, seekErr := f.Seek(offset, io.SeekStart); seekErr != nil {
return nil, seekErr
}
}
// The limit holds the bound even if the file grew after Stat.
data, err := io.ReadAll(io.LimitReader(f, maxBytes))
if err != nil {
return nil, err
}
if offset > 0 {
nl := bytes.IndexByte(data, '\n')
if nl < 0 {
return nil, nil
}
data = data[nl+1:]
}
text := strings.TrimSuffix(string(data), "\n")
if text == "" {
return nil, nil
}
return strings.Split(text, "\n"), nil
}
package inventory
import (
"path/filepath"
"sort"
"strings"
)
// FS is the minimal filesystem surface the enumerator needs. Injected so the
// cPanel source can be tested against fixture directories without root.
type FS interface {
Glob(pattern string) ([]string, error)
ReadFile(name string) ([]byte, error)
}
// Source enumerates the forwarders configured on a host.
type Source interface {
Forwarders() ([]Forwarder, error)
}
// EmptySource reports no forwarders. It stands in on platforms whose
// enumeration is not wired yet (non-cPanel), so callers always hold a usable
// Source instead of a nil.
type EmptySource struct{}
func (EmptySource) Forwarders() ([]Forwarder, error) { return []Forwarder{}, nil }
// CPanelSource reads forwarders from cPanel's /etc/valiases directory, with
// local domains from /etc/localdomains and /etc/virtualdomains and owners from
// /etc/userdomains.
type CPanelSource struct {
fs FS
valiasGlob string
localDomainsPath string
virtualDomainsPath string
userDomainsPath string
}
// NewCPanelSource returns a source reading the standard cPanel locations.
func NewCPanelSource() *CPanelSource {
return &CPanelSource{
fs: osFS{},
valiasGlob: "/etc/valiases/*",
localDomainsPath: "/etc/localdomains",
virtualDomainsPath: "/etc/virtualdomains",
userDomainsPath: "/etc/userdomains",
}
}
// Forwarders enumerates every forwarder across all hosted domains. A missing
// or unreadable valias file is skipped, not fatal: partial inventory beats no
// inventory on a server with thousands of domains.
func (s *CPanelSource) Forwarders() ([]Forwarder, error) {
localDomains := s.loadLocalDomains()
owners := s.loadOwners()
files, err := s.fs.Glob(s.valiasGlob)
if err != nil {
return nil, err
}
var out []Forwarder
for _, path := range files {
domain := normalizeDomain(filepath.Base(path))
content, err := s.fs.ReadFile(path)
if err != nil {
continue
}
owner := owners[domain]
for _, line := range strings.Split(string(content), "\n") {
fwd, ok := parseForwarderLine(domain, line, localDomains)
if !ok {
continue
}
fwd.Owner = owner
out = append(out, fwd)
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Source < out[j].Source })
return out, nil
}
// loadLocalDomains reads cPanel's local-domain files into a normalized set.
// Returns an empty set when the files are unavailable, which makes every
// destination classify as external -- the safe direction for a reputation tool
// (over-report external, never hide it).
func (s *CPanelSource) loadLocalDomains() map[string]bool {
domains := make(map[string]bool)
for _, path := range []string{s.localDomainsPath, s.virtualDomainsPath} {
if path == "" {
continue
}
content, err := s.fs.ReadFile(path)
if err != nil {
continue
}
for _, line := range strings.Split(string(content), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
domain := configDomain(line)
if domain != "" {
domains[domain] = true
}
}
}
return domains
}
// loadOwners reads /etc/userdomains ("domain: user") into a domain->owner map.
func (s *CPanelSource) loadOwners() map[string]string {
owners := make(map[string]string)
content, err := s.fs.ReadFile(s.userDomainsPath)
if err != nil {
return owners
}
for _, line := range strings.Split(string(content), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
idx := strings.IndexByte(line, ':')
if idx <= 0 {
continue
}
domain := configDomain(line[:idx])
owner := strings.TrimSpace(line[idx+1:])
if domain != "" && owner != "" {
owners[domain] = owner
}
}
return owners
}
func configDomain(line string) string {
line = strings.TrimSpace(line)
if idx := strings.IndexByte(line, ':'); idx >= 0 {
line = strings.TrimSpace(line[:idx])
}
domain := normalizeDomain(line)
if domain == "" || strings.ContainsAny(domain, " \t\r\n/:\\") {
return ""
}
return domain
}
package inventory
import (
"os"
"path/filepath"
)
// osFS is the production FS backed by the real filesystem.
type osFS struct{}
func (osFS) Glob(pattern string) ([]string, error) { return filepath.Glob(pattern) }
// ReadFile reads a file. Paths come from a fixed glob of operator-owned mail
// config directories, not from untrusted input.
func (osFS) ReadFile(name string) ([]byte, error) {
return os.ReadFile(name) // #nosec G304 -- name is a valias path from a fixed glob, operator-scoped.
}
// Package inventory enumerates mail forwarders on a host and classifies their
// destinations, so operators can see which accounts relay mail off-server and
// to which providers. It is the canonical home for forwarder-domain logic in
// CSM. Enumeration is platform-abstracted: cPanel reads /etc/valiases, other
// hosts read /etc/aliases, ~/.forward, or postfix virtual maps (wired
// incrementally). Reading the filesystem is injected so the parsing logic is
// testable without root or a live mail server.
package inventory
import (
"strings"
"golang.org/x/net/publicsuffix"
)
// ProviderClass labels a forwarder destination by where it delivers. Free
// providers (Yahoo/Gmail/Outlook) are split out because forwarding spam to
// them is what degrades a server's outbound reputation; local stays on-server.
type ProviderClass string
const (
ProviderLocal ProviderClass = "local"
ProviderYahoo ProviderClass = "yahoo"
ProviderGmail ProviderClass = "gmail"
ProviderOutlook ProviderClass = "outlook"
ProviderExternal ProviderClass = "external"
)
// freeProviderExact maps known free-provider mail domains to their class.
// Lowercase keys; lookups lowercase the input.
var freeProviderExact = map[string]ProviderClass{
"gmail.com": ProviderGmail,
"googlemail.com": ProviderGmail,
"rocketmail.com": ProviderYahoo,
"yahoo.ca": ProviderYahoo,
"yahoo.co.in": ProviderYahoo,
"yahoo.co.jp": ProviderYahoo,
"yahoo.co.uk": ProviderYahoo,
"yahoo.com": ProviderYahoo,
"yahoo.com.au": ProviderYahoo,
"yahoo.com.br": ProviderYahoo,
"yahoo.com.mx": ProviderYahoo,
"yahoo.de": ProviderYahoo,
"yahoo.es": ProviderYahoo,
"yahoo.fr": ProviderYahoo,
"yahoo.it": ProviderYahoo,
"yahoo.ro": ProviderYahoo,
"ymail.com": ProviderYahoo,
"hotmail.co.uk": ProviderOutlook,
"hotmail.com": ProviderOutlook,
"hotmail.de": ProviderOutlook,
"hotmail.fr": ProviderOutlook,
"live.co.uk": ProviderOutlook,
"live.com": ProviderOutlook,
"live.com.au": ProviderOutlook,
"live.de": ProviderOutlook,
"live.fr": ProviderOutlook,
"live.it": ProviderOutlook,
"live.ro": ProviderOutlook,
"msn.com": ProviderOutlook,
"outlook.com": ProviderOutlook,
"outlook.de": ProviderOutlook,
}
// Destination is one resolved target of a forwarder.
type Destination struct {
Address string `json:"address"`
Domain string `json:"domain"`
Provider ProviderClass `json:"provider"`
}
// Forwarder is a single source address and everything it relays to.
type Forwarder struct {
Source string `json:"source"` // local_part@domain
Domain string `json:"domain"` // hosting domain
Owner string `json:"owner"` // panel account, "" if unknown
Destinations []Destination `json:"destinations"` // address targets only
KeepLocal bool `json:"keep_local"` // also delivers to a local mailbox
ForwardOnly bool `json:"forward_only"` // only remote targets, no local copy
}
// HasExternal reports whether any destination leaves the server.
func (f Forwarder) HasExternal() bool {
for _, d := range f.Destinations {
if d.Provider != ProviderLocal {
return true
}
}
return false
}
// HasFreeProvider reports whether any destination is a free-provider mailbox
// (the reputation-risk case).
func (f Forwarder) HasFreeProvider() bool {
for _, d := range f.Destinations {
switch d.Provider {
case ProviderYahoo, ProviderGmail, ProviderOutlook:
return true
}
}
return false
}
// ClassifyAddress returns the provider class of a mail address with no
// local-domain context. Use it where the address is known to be a remote
// recipient (e.g. a deferral target parsed from exim_mainlog), so a bare
// free-provider domain classifies as that provider rather than local.
func ClassifyAddress(addr string) ProviderClass {
return classifyProvider(addr, nil)
}
// classifyProvider returns the provider class of a destination address.
// localDomains are the domains hosted on this server (lowercased keys).
func classifyProvider(addr string, localDomains map[string]bool) ProviderClass {
addr = strings.TrimSpace(addr)
at := strings.LastIndexByte(addr, '@')
if at < 0 || at >= len(addr)-1 {
// No domain: a bare local part is delivered to the local mailbox.
return ProviderLocal
}
domain := normalizeDomain(addr[at+1:])
if localDomains[domain] {
return ProviderLocal
}
if c, ok := freeProviderExact[domain]; ok {
return c
}
if registered, ok := registeredDomain(domain); ok {
if c, ok := freeProviderExact[registered]; ok {
return c
}
}
return ProviderExternal
}
func registeredDomain(domain string) (string, bool) {
registered, err := publicsuffix.EffectiveTLDPlusOne(domain)
if err != nil {
return "", false
}
return normalizeDomain(registered), true
}
func normalizeDomain(domain string) string {
return strings.TrimSuffix(strings.ToLower(strings.TrimSpace(domain)), ".")
}
// parseForwarderLine parses one alias/valias line ("local_part: dest[, dest]")
// for the given hosting domain. Returns ok=false for blanks, comments,
// malformed lines, and non-address forwarders (pipes, :fail:, :blackhole:,
// /dev/null) -- those are not mail relayed to a mailbox and carry no
// reputation risk. A line is still returned for purely local aliases so the
// inventory is complete; callers filter on HasExternal as needed.
func parseForwarderLine(domain, line string, localDomains map[string]bool) (Forwarder, bool) {
domain = normalizeDomain(domain)
if domain == "" {
return Forwarder{}, false
}
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
return Forwarder{}, false
}
colon := strings.IndexByte(line, ':')
if colon < 0 {
return Forwarder{}, false
}
localPart := strings.TrimSpace(line[:colon])
rest := strings.TrimSpace(line[colon+1:])
if localPart == "" || rest == "" {
return Forwarder{}, false
}
// cPanel keys a forwarder by its whole address; a bare local part
// belongs to the file's domain.
source := localPart + "@" + domain
if at := strings.LastIndexByte(localPart, '@'); at >= 0 {
source = localPart[:at] + "@" + normalizeDomain(localPart[at+1:])
}
fwd := Forwarder{
Source: source,
Domain: domain,
}
for _, raw := range strings.Split(rest, ",") {
dest := strings.TrimSpace(raw)
if dest == "" || !isAddressDestination(dest) {
// Pipe / :fail: / :blackhole: / /dev/null / autoresponder:
// not an address relay. Skip the target but keep the record.
continue
}
fwd.Destinations = append(fwd.Destinations, newDestination(dest, localDomains))
}
if len(fwd.Destinations) == 0 {
return Forwarder{}, false
}
localCount := 0
for _, d := range fwd.Destinations {
if d.Provider == ProviderLocal {
localCount++
}
}
fwd.KeepLocal = localCount > 0
fwd.ForwardOnly = localCount == 0
return fwd, true
}
// isAddressDestination reports whether a valias destination is a mailbox
// address (as opposed to a pipe, discard, fail, or file delivery).
func isAddressDestination(dest string) bool {
dest = strings.TrimSpace(dest)
if quotedLocalPartAddress(dest) {
return true
}
if len(dest) >= 2 && dest[0] == '"' && dest[len(dest)-1] == '"' {
dest = strings.TrimSpace(dest[1 : len(dest)-1])
}
switch {
case dest == "":
return false
case strings.HasPrefix(dest, "|"): // pipe to a program
return false
case strings.HasPrefix(dest, ":"): // :fail:, :blackhole:, :defer:
return false
case strings.HasPrefix(dest, "/"): // file / /dev/null
return false
case strings.HasPrefix(dest, "\""): // malformed quote or quoted directive
return false
}
return true
}
func quotedLocalPartAddress(dest string) bool {
if !strings.HasPrefix(dest, "\"") {
return false
}
closing := strings.IndexByte(dest[1:], '"')
if closing < 0 {
return false
}
closing++ // convert offset in dest[1:] to index in dest
return closing+1 < len(dest) && dest[closing+1] == '@'
}
func newDestination(addr string, localDomains map[string]bool) Destination {
addr = normalizeAddressDestination(addr)
domain := ""
if at := strings.LastIndexByte(addr, '@'); at >= 0 && at < len(addr)-1 {
domain = normalizeDomain(addr[at+1:])
}
return Destination{
Address: addr,
Domain: domain,
Provider: classifyProvider(addr, localDomains),
}
}
func normalizeAddressDestination(addr string) string {
addr = strings.TrimSpace(addr)
if len(addr) >= 2 && addr[0] == '"' && addr[len(addr)-1] == '"' {
inner := strings.TrimSpace(addr[1 : len(addr)-1])
if strings.Contains(inner, "@") {
return inner
}
}
return addr
}
package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"strings"
"time"
"github.com/pidginhost/csm/internal/systemdrun"
)
// uapiPath is where cPanel ships uapi; the /usr/bin symlink is not present on
// every host and the service PATH is minimal.
const uapiPath = "/usr/local/cpanel/bin/uapi"
// uapiTimeout bounds one uapi call. uapi loads cPanel's Perl stack and the
// account's data on every invocation, so several seconds is normal.
const uapiTimeout = 30 * time.Second
// CPanelRemover deletes forwarders through cPanel's UAPI as the account that
// owns the domain, so cPanel updates its own caches along with the alias file.
type CPanelRemover struct {
lookPath systemdrun.LookPathFunc
run systemdrun.RunnerFunc
}
// NewCPanelRemover returns a remover that runs uapi with run, outside the
// daemon's sandbox through systemd-run.
func NewCPanelRemover(lookPath systemdrun.LookPathFunc, run systemdrun.RunnerFunc) *CPanelRemover {
return &CPanelRemover{lookPath: lookPath, run: run}
}
// Remove deletes one destination from forwarder f. f.Owner must name the
// cPanel account. uapi writes under the account home and into cPanel's
// caches, which the sandbox does not grant, so it runs only as a transient
// unit: inside the sandbox it could change the alias file and then fail.
func (r *CPanelRemover) Remove(f Forwarder, dest string) error {
ctx, cancel := context.WithTimeout(context.Background(), uapiTimeout)
defer cancel()
out, err := systemdrun.Run(ctx, r.lookPath, r.run,
systemdrun.Options{Pipe: true, Required: true, RuntimeMax: uapiTimeout},
uapiPath, "--output=json", "--user="+f.Owner, "Email", "delete_forwarder",
"address="+uapiValue(f.Source), "forwarder="+uapiValue(dest))
if errors.Is(err, systemdrun.ErrUnavailable) {
return err
}
return uapiReplyError(out, err)
}
// uapiValue encodes v for uapi's command line, which URI-decodes every value.
// A space goes in as %20, which reads the same whether or not the decoder
// takes + for a space.
func uapiValue(v string) string {
return strings.ReplaceAll(url.QueryEscape(v), "+", "%20")
}
// uapiReplyError preserves cPanel's own failure message. A success reply
// cannot override a failed command: systemd-run may fail after forwarding
// the output, leaving completion uncertain.
func uapiReplyError(out []byte, runErr error) error {
var reply struct {
Result struct {
Status int `json:"status"`
Errors []string `json:"errors"`
} `json:"result"`
}
if err := json.Unmarshal(out, &reply); err != nil {
text := strings.TrimSpace(string(out))
if len(text) > 200 {
text = text[:200]
}
if runErr != nil {
return fmt.Errorf("uapi: %w: %s", runErr, text)
}
return fmt.Errorf("unreadable uapi reply: %s", text)
}
if reply.Result.Status == 1 {
if runErr != nil {
return fmt.Errorf("uapi execution failed: %w", runErr)
}
return nil
}
if len(reply.Result.Errors) == 0 {
return errors.New("uapi reported failure without an error message")
}
return errors.New(strings.Join(reply.Result.Errors, "; "))
}
// Package policy is the single source of truth for the forward-guard hold
// decision: given the signals observed for a message, should the external
// forward copy be held? The same Verdict function feeds both the dry-run
// "would-hold" accounting and (in Phase 2) the generated MTA rule, so the two
// can never drift apart.
//
// The verdict is layered: any enabled signal that matches holds the message.
// Holding is conservative-by-omission -- a signal whose toggle is off, or a
// message with no matching signal, is never held.
package policy
// MessageMeta is the set of signals known about a single forwarded message.
// All fields default false (unknown == not flagged), so a partially-populated
// meta can only ever reduce the chance of a hold, never invent one.
type MessageMeta struct {
NullSender bool // envelope sender is <> (bounce/backscatter)
SpamFlagged bool // carries an X-Spam-Status: Yes spam verdict
MalwareHit bool // ClamAV / YARA-X matched
SenderIPBad bool // sender IP is in the CSM attack DB / reputation
SPFFail bool
DKIMFail bool
DMARCFail bool
}
// HoldSignals toggles which layered signals are allowed to hold a message.
// Each is individually switchable so operators can roll out one signal at a time.
type HoldSignals struct {
BounceBackscatter bool
SpamFlagged bool
Malware bool
BadSenderIP bool
AuthFail bool
}
// Config is the forward-guard policy input. Enabled is the master switch; when
// off, Verdict never holds regardless of signals. DryRun does not affect the
// verdict itself -- it tells callers whether to enforce the hold or only
// account for it -- so it lives here for callers but is not read by Verdict.
type Config struct {
Enabled bool
DryRun bool
HoldSignals HoldSignals
}
// Verdict reports whether a message's external-forward copy should be held and
// the matching reason codes (in a fixed, deterministic order). Reasons are
// stable identifiers safe to surface in the UI, log, and generated MTA rule.
func Verdict(meta MessageMeta, cfg Config) (hold bool, reasons []string) {
if !cfg.Enabled {
return false, nil
}
sig := cfg.HoldSignals
// Fixed evaluation order -> deterministic reason slice (no map iteration).
if sig.AuthFail && meta.SPFFail && meta.DKIMFail && meta.DMARCFail {
reasons = append(reasons, "auth_fail")
}
if sig.BadSenderIP && meta.SenderIPBad {
reasons = append(reasons, "bad_sender_ip")
}
if sig.BounceBackscatter && meta.NullSender {
reasons = append(reasons, "bounce_backscatter")
}
if sig.Malware && meta.MalwareHit {
reasons = append(reasons, "malware")
}
if sig.SpamFlagged && meta.SpamFlagged {
reasons = append(reasons, "spam_flagged")
}
return len(reasons) > 0, reasons
}
// Package quarantine is the CSM-owned Maildir that holds external forward
// copies the forward-guard decided to withhold. The exim transport (Phase 2
// Slice C) appends held copies here with X-CSM-* control headers; CSM lists
// them, releases (re-injects to the original external recipient) or deletes,
// and prunes by age. CSM is never in the live delivery path -- exim writes the
// file, CSM only acts on it afterwards.
package quarantine
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"io/fs"
"mime"
"net/mail"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
"unicode"
"github.com/pidginhost/csm/internal/safepath"
"github.com/pidginhost/csm/internal/systemdrun"
)
// Control headers the exim transport adds and CSM parses. They are stripped
// before a message is re-injected so they never leak to the recipient.
const (
hdrForwarder = "X-CSM-Forwarder"
hdrRecipient = "X-CSM-Recipient"
hdrSender = "X-CSM-Sender"
hdrReasons = "X-CSM-Reasons"
hdrPrefix = "X-CSM-"
)
// HeldMessage is the operator-facing view of one held forward copy.
type HeldMessage struct {
ID string `json:"id"`
Forwarder string `json:"forwarder"` // local source address that forwarded
Recipient string `json:"recipient"` // external destination that was held
Sender string `json:"sender"` // envelope sender ("" = null-sender bounce)
Reasons []string `json:"reasons"`
Subject string `json:"subject"`
HeldAt time.Time `json:"held_at"`
Size int64 `json:"size"`
}
// Quarantine manages the held-forward Maildir.
type Quarantine struct {
base string
counter atomic.Uint64
sendmail func(sender, recipient string, body []byte) error
lookPath systemdrun.LookPathFunc
run systemdrun.InputRunnerFunc
}
// New returns a quarantine rooted at dir (a Maildir; new/cur/tmp are created on
// demand). Release re-injects through the platform sendmail, run as a
// transient unit (see reinject).
func New(dir string) *Quarantine {
q := &Quarantine{base: dir, lookPath: exec.LookPath, run: runWithInput}
q.sendmail = q.reinject
return q
}
func (q *Quarantine) sub(name string) string { return filepath.Join(q.base, name) }
// Hold writes a held copy into the Maildir with the X-CSM-* control headers and
// returns its id (the Maildir filename). In production exim's appendfile writes
// these files; Hold produces the same layout for CSM-side paths and tests: the
// control headers follow the message's own header fields, as exim's
// headers_add appends them.
func (q *Quarantine) Hold(m HeldMessage, body []byte) (string, error) {
for _, d := range []string{"tmp", "new", "cur"} {
if err := os.MkdirAll(q.sub(d), 0700); err != nil {
return "", fmt.Errorf("creating maildir %s: %w", d, err)
}
}
// cut is where the header fields end: the start of the blank separator
// line, or the end of a message that has none.
cut := headerBlockEnd(body)
if start := bytes.LastIndexByte(body[:max(cut-1, 0)], '\n') + 1; cut > 0 && isBlankLine(body[start:cut]) {
cut = start
}
var buf bytes.Buffer
buf.Write(body[:cut])
if cut > 0 && body[cut-1] != '\n' {
buf.WriteString("\r\n")
}
fmt.Fprintf(&buf, "%s: %s\r\n", hdrForwarder, headerValue(m.Forwarder))
fmt.Fprintf(&buf, "%s: %s\r\n", hdrRecipient, headerValue(m.Recipient))
fmt.Fprintf(&buf, "%s: %s\r\n", hdrSender, headerValue(m.Sender))
fmt.Fprintf(&buf, "%s: %s\r\n", hdrReasons, headerValue(strings.Join(m.Reasons, ",")))
buf.Write(body[cut:])
id := fmt.Sprintf("%d.%d.csm", time.Now().UnixNano(), q.counter.Add(1))
tmp := filepath.Join(q.sub("tmp"), id)
if err := os.WriteFile(tmp, buf.Bytes(), 0600); err != nil { // #nosec G306 -- 0600 is intended
return "", fmt.Errorf("writing held message: %w", err)
}
dst := filepath.Join(q.sub("new"), id)
if err := os.Rename(tmp, dst); err != nil {
_ = os.Remove(tmp)
return "", fmt.Errorf("committing held message: %w", err)
}
return id, nil
}
// List returns every held message with parsed metadata. A missing Maildir is
// not an error: it just means nothing has been held.
func (q *Quarantine) List() ([]HeldMessage, error) {
out := []HeldMessage{}
for _, dir := range []string{"new", "cur"} {
entries, err := os.ReadDir(q.sub(dir))
if err != nil {
if os.IsNotExist(err) {
continue
}
return nil, err
}
for _, e := range entries {
if e.IsDir() {
continue
}
m, err := q.read(dir, e.Name())
if err != nil {
continue // skip unreadable/partial entries rather than fail the whole list
}
out = append(out, m)
}
}
sort.Slice(out, func(i, j int) bool { return out[i].HeldAt.Before(out[j].HeldAt) })
return out, nil
}
// maxHeaderBlock bounds the header block CSM reads from a held copy: exim's
// default header_maxsize (1 MiB) for the message's own fields, plus room for
// the control headers the transport appends after them. maxSubjectLen bounds
// the subject the list shows.
const (
maxHeaderBlock = 1<<20 + 8<<10
maxSubjectLen = 160
)
var errHeaderBlockTooLarge = errors.New("held message header block is larger than CSM reads")
// readHeaderBlock reads lines up to and including the first blank line, or
// to the end of a message with no body. A longer block than maxHeaderBlock is
// refused rather than cut: the control headers come last, so a cut block
// could hold only copies the message itself carried.
func readHeaderBlock(r io.Reader) ([]byte, error) {
br := bufio.NewReader(io.LimitReader(r, maxHeaderBlock+1))
var block []byte
for {
line, err := br.ReadBytes('\n')
block = append(block, line...)
if len(block) > maxHeaderBlock {
return nil, errHeaderBlockTooLarge
}
if err == io.EOF || err == nil && isBlankLine(line) {
return block, nil
}
if err != nil {
return nil, err
}
}
}
func (q *Quarantine) read(dir, id string) (HeldMessage, error) {
f, info, err := openMessageFile(filepath.Join(q.sub(dir), id))
if err != nil {
return HeldMessage{}, err
}
defer f.Close()
header, err := readHeaderBlock(f)
if err != nil {
return HeldMessage{}, err
}
hdr := parseControlHeaders(header)
return HeldMessage{
ID: id,
Forwarder: hdr[hdrForwarder],
Recipient: hdr[hdrRecipient],
Sender: hdr[hdrSender],
Reasons: splitReasons(hdr[hdrReasons]),
Subject: heldSubject(header),
HeldAt: info.ModTime(),
Size: info.Size(),
}, nil
}
// heldSubject returns the first Subject field of a header block, unfolded,
// as CleanSubject leaves it.
func heldSubject(header []byte) string {
var raw strings.Builder
found := false
for _, line := range strings.Split(string(header), "\n") {
line = strings.TrimRight(line, "\r")
folded := line != "" && (line[0] == ' ' || line[0] == '\t')
if found {
if !folded {
break
}
raw.WriteString(line)
continue
}
name, value, ok := strings.Cut(line, ":")
if ok && !folded && strings.EqualFold(strings.TrimRight(name, " \t"), "Subject") {
raw.WriteString(value)
found = true
}
}
return CleanSubject(raw.String())
}
// CleanSubject readies a mail subject for the web UI: RFC 2047 encoded words
// decoded, control characters turned into spaces, bidi and other format
// characters (which can reverse or hide text) dropped, runs of space
// collapsed, and cut to maxSubjectLen runes. The subject is the sender's
// text. A charset the decoder does not know leaves the words as sent.
func CleanSubject(subject string) string {
if decoded, err := new(mime.WordDecoder).DecodeHeader(subject); err == nil {
subject = decoded
}
subject = strings.Join(strings.Fields(strings.Map(func(r rune) rune {
switch {
case unicode.IsControl(r):
return ' '
case unicode.Is(unicode.Bidi_Control, r), unicode.Is(unicode.Cf, r):
return -1
}
return r
}, subject)), " ")
if r := []rune(subject); len(r) > maxSubjectLen {
subject = string(r[:maxSubjectLen])
}
return subject
}
// Release re-injects the held copy to its original external recipient (operator
// decided it was a false positive), then removes it. The copy is claimed
// first, so of two releases or a release and a delete only one acts; the
// other finds it not held. It is removed only after a successful
// re-injection and put back otherwise, so a sendmail failure never loses mail.
func (q *Quarantine) Release(id string) error {
claimed, held, err := q.claim(id)
if err != nil {
return err
}
if err := q.reinjectClaimed(claimed); err != nil {
if back := os.Rename(claimed, held); back != nil {
return errors.Join(err, fmt.Errorf("returning held message %q: %w", id, back))
}
return err
}
return os.Remove(claimed)
}
func (q *Quarantine) reinjectClaimed(path string) error {
data, _, err := readMessageFile(path)
if err != nil {
return err
}
header, err := readHeaderBlock(bytes.NewReader(data))
if err != nil {
return err
}
hdr := parseControlHeaders(header)
sender, hasSender := hdr[hdrSender]
recipient := hdr[hdrRecipient]
if !hasSender || sender != "" && !singleEnvelopeAddress(sender) || !singleEnvelopeAddress(recipient) {
return errors.New("held message does not record a safe single sender and recipient")
}
clean := stripControlHeaders(data)
if err := q.sendmail(sender, recipient, clean); err != nil {
return fmt.Errorf("re-injecting held message: %w", err)
}
return nil
}
// claim moves a held copy into tmp/ under a name of its own and returns
// where it is now and where it was held. The rename is atomic, so one
// caller wins and the others find the copy not held.
func (q *Quarantine) claim(id string) (claimed, held string, err error) {
held = q.pathOf(id)
if held == "" {
return "", "", errNotHeld(id)
}
if err := os.MkdirAll(q.sub("tmp"), 0700); err != nil {
return "", "", err
}
claimed = filepath.Join(q.sub("tmp"), fmt.Sprintf("%s.claim.%d.%d", filepath.Base(held), time.Now().UnixNano(), q.counter.Add(1)))
if err := os.Rename(held, claimed); err != nil {
if errors.Is(err, fs.ErrNotExist) {
return "", "", errNotHeld(id)
}
return "", "", err
}
return claimed, held, nil
}
// staleClaimAge is past any claim a live Release or Delete still holds: a
// release waits at most sendmailTimeout for sendmail.
const staleClaimAge = 5 * time.Minute
// RestoreStaleClaims holds again every copy that a Release or Delete claimed
// at least staleClaimAge before now and never finished with, as when the
// daemon died mid-release; such a release may already have been delivered.
// A claim whose id is held again stays in tmp/. It returns how many copies
// were restored.
func (q *Quarantine) RestoreStaleClaims(now time.Time) (int, error) {
entries, err := os.ReadDir(q.sub("tmp"))
if os.IsNotExist(err) {
return 0, nil
}
if err != nil {
return 0, err
}
restored := 0
var errs []error
for _, e := range entries {
id, at, ok := parseClaimName(e.Name())
if !ok || !e.Type().IsRegular() || now.Sub(at) < staleClaimAge {
continue
}
if err := q.restoreClaim(e.Name(), id); err != nil {
if !errors.Is(err, fs.ErrExist) && !errors.Is(err, fs.ErrNotExist) {
errs = append(errs, err)
}
continue
}
restored++
}
return restored, errors.Join(errs...)
}
func (q *Quarantine) restoreClaim(name, id string) error {
// List and locate also read cur/: restoring into new/ must not shadow
// another held copy with the same ID there.
if _, err := os.Lstat(filepath.Join(q.sub("cur"), id)); err == nil {
return fs.ErrExist
} else if !errors.Is(err, fs.ErrNotExist) {
return err
}
if err := os.MkdirAll(q.sub("new"), 0700); err != nil {
return err
}
src, err := safepath.OpenDir(q.sub("tmp"))
if err != nil {
return err
}
defer func() { _ = src.Close() }()
dst, err := safepath.OpenDir(q.sub("new"))
if err != nil {
return err
}
defer func() { _ = dst.Close() }()
// Publish and consume the claim atomically without replacing a held
// copy. Link followed by unlink leaves a second recoverable copy if the
// daemon stops, or another restorer races a release between the two.
return src.RenameTo(name, dst, id)
}
// parseClaimName reads the held id and the claim time from a name claim gave
// a copy; exim's own files in tmp/ do not parse.
func parseClaimName(name string) (id string, at time.Time, ok bool) {
i := strings.LastIndex(name, ".claim.")
if i <= 0 {
return "", time.Time{}, false
}
stamp, seq, found := strings.Cut(name[i+len(".claim."):], ".")
nanos, err := strconv.ParseInt(stamp, 10, 64)
if !found || err != nil || nanos <= 0 {
return "", time.Time{}, false
}
if _, err := strconv.ParseUint(seq, 10, 64); err != nil {
return "", time.Time{}, false
}
return name[:i], time.Unix(0, nanos), true
}
func errNotHeld(id string) error {
return fmt.Errorf("held message %q not found", id)
}
// Exim splits command-line recipients on commas, including dequoted local
// parts from its control headers. Check both the recorded and parsed forms
// so quoting cannot hide a list separator, whitespace or an option prefix.
func singleEnvelopeAddress(value string) bool {
unsafe := func(s string) bool {
return strings.HasPrefix(s, "-") || strings.ContainsAny(s, ",<>") ||
strings.ContainsFunc(s, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) })
}
if value == "" || unsafe(value) {
return false
}
addr, err := mail.ParseAddress(value)
return err == nil && addr.Name == "" && !unsafe(addr.Address)
}
// Delete discards a held copy without delivering it. It claims the copy as
// Release does, so a copy being released is not held for Delete.
func (q *Quarantine) Delete(id string) error {
claimed, _, err := q.claim(id)
if err != nil {
return err
}
return os.Remove(claimed)
}
// PruneOlderThan removes held copies older than maxAge and returns how many were
// removed.
func (q *Quarantine) PruneOlderThan(maxAge time.Duration) (int, error) {
cutoff := time.Now().Add(-maxAge)
removed := 0
for _, dir := range []string{"new", "cur"} {
entries, err := os.ReadDir(q.sub(dir))
if err != nil {
if os.IsNotExist(err) {
continue
}
return removed, err
}
for _, e := range entries {
if e.IsDir() {
continue
}
info, err := e.Info()
if err != nil {
continue
}
if info.Mode().IsRegular() && info.ModTime().Before(cutoff) {
if err := os.Remove(filepath.Join(q.sub(dir), e.Name())); err == nil {
removed++
}
}
}
}
return removed, nil
}
// CountsByForwarder returns how many held copies each forwarder produced.
func (q *Quarantine) CountsByForwarder() (map[string]int, error) {
msgs, err := q.List()
if err != nil {
return nil, err
}
counts := make(map[string]int)
for _, m := range msgs {
counts[m.Forwarder]++
}
return counts, nil
}
// pathOf returns the on-disk path of a held message id, or "" if absent.
func (q *Quarantine) pathOf(id string) string {
id = filepath.Base(id) // defend against traversal in a caller-supplied id
if id == "" || id == "." || id == ".." || id == string(filepath.Separator) {
return ""
}
for _, d := range []string{"new", "cur"} {
p := filepath.Join(q.sub(d), id)
info, err := os.Lstat(p)
if err != nil {
continue
}
if info.Mode().IsRegular() {
return p
}
}
return ""
}
func readMessageFile(path string) ([]byte, os.FileInfo, error) {
f, info, err := openMessageFile(path)
if err != nil {
return nil, nil, err
}
defer f.Close()
data, err := io.ReadAll(f)
if err != nil {
return nil, nil, err
}
return data, info, nil
}
// openMessageFile opens a held copy for reading. The caller closes it.
func openMessageFile(path string) (*os.File, os.FileInfo, error) {
// #nosec G304 -- path is a located Maildir entry under the CSM-owned base;
// O_NOFOLLOW rejects symlink entries and symlink swaps before reading.
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
if err != nil {
return nil, nil, err
}
info, err := f.Stat()
if err != nil {
_ = f.Close()
return nil, nil, err
}
if !info.Mode().IsRegular() {
_ = f.Close()
return nil, nil, fmt.Errorf("held message %q is not a regular file", filepath.Base(path))
}
return f, info, nil
}
// headerValue collapses a control-header value to a single safe line (no CR/LF
// so a hostile address cannot inject extra headers).
func headerValue(v string) string {
v = strings.ReplaceAll(v, "\r", "")
v = strings.ReplaceAll(v, "\n", "")
return strings.TrimSpace(v)
}
func splitReasons(v string) []string {
if strings.TrimSpace(v) == "" {
return nil
}
parts := strings.Split(v, ",")
out := make([]string, 0, len(parts))
for _, p := range parts {
if p = strings.TrimSpace(p); p != "" {
out = append(out, p)
}
}
return out
}
// headerBlockEnd returns the index just past the first blank line (the
// header/body separator), or len(data) if there is no blank line. A leading
// blank line means the header block is empty -- the rest is body. Defining the
// boundary by the first empty line (rather than searching for "\n\n") keeps
// parsing and stripping consistent on degenerate inputs.
func headerBlockEnd(data []byte) int {
off := 0
for off < len(data) {
nl := bytes.IndexByte(data[off:], '\n')
if nl < 0 {
return len(data) // no terminating newline: all headers, no body
}
if isBlankLine(data[off : off+nl+1]) {
return off + nl + 1
}
off += nl + 1
}
return len(data)
}
// isBlankLine reports whether line, with its line ending, is the blank line
// that ends a header block.
func isBlankLine(line []byte) bool {
return len(bytes.TrimRight(line, "\r\n")) == 0
}
// parseControlHeaders returns the X-CSM-* fields of the header block that
// starts data. A message can carry its own X-CSM-* fields; exim appends the
// real ones after every field the message brought, so the last occurrence
// of each wins. A folded continuation line is never a field.
func parseControlHeaders(data []byte) map[string]string {
out := map[string]string{}
header := data[:headerBlockEnd(data)]
for _, line := range strings.Split(string(header), "\n") {
line = strings.TrimRight(line, "\r")
colon := strings.IndexByte(line, ':')
if colon < 0 || line[0] == ' ' || line[0] == '\t' {
continue
}
key, ok := canonicalControlHeader(line[:colon])
if !ok {
continue
}
out[key] = headerValue(line[colon+1:])
}
return out
}
func canonicalControlHeader(name string) (string, bool) {
name = strings.TrimSpace(name)
switch {
case strings.EqualFold(name, hdrForwarder):
return hdrForwarder, true
case strings.EqualFold(name, hdrRecipient):
return hdrRecipient, true
case strings.EqualFold(name, hdrSender):
return hdrSender, true
case strings.EqualFold(name, hdrReasons):
return hdrReasons, true
case hasControlHeaderPrefix(name):
return name, true
default:
return "", false
}
}
func hasControlHeaderPrefix(line string) bool {
return len(line) >= len(hdrPrefix) && strings.EqualFold(line[:len(hdrPrefix)], hdrPrefix)
}
// stripControlHeaders removes every X-CSM-* header line, leaving the original
// message intact for re-injection. Only the header lines before the
// header/body separator are filtered; the separator and body are preserved
// byte-for-byte so the recipient sees exactly the original message.
func stripControlHeaders(data []byte) []byte {
var kept bytes.Buffer
rest := data
droppingControl := false
for len(rest) > 0 {
nl := bytes.IndexByte(rest, '\n')
var line []byte
if nl < 0 {
line, rest = rest, nil
} else {
line, rest = rest[:nl+1], rest[nl+1:]
}
trimmed := strings.TrimRight(string(line), "\r\n")
if trimmed == "" {
// Blank line ends the header block; keep it and the body verbatim.
kept.Write(line)
kept.Write(rest)
return kept.Bytes()
}
if line[0] == ' ' || line[0] == '\t' {
if droppingControl {
continue
}
kept.Write(line)
continue
}
droppingControl = hasControlHeaderPrefix(trimmed)
if droppingControl {
continue
}
kept.Write(line)
}
return kept.Bytes()
}
const (
sendmailPath = "/usr/sbin/sendmail"
// sendmailTimeout bounds a re-injection, the wait and the unit alike.
sendmailTimeout = 30 * time.Second
)
// reinject hands body to the platform sendmail for one recipient. exim
// cannot open its logs inside csm.service's ProtectSystem=strict sandbox and
// aborts there, possibly after queueing the copy, so sendmail runs as a
// transient unit or not at all: without systemd-run the release fails and
// the copy stays held. exim queues the message, forks its delivery and
// exits; KillMode=process keeps the unit's stop from killing that delivery,
// so released mail goes out at once rather than at the next queue run.
func (q *Quarantine) reinject(sender, recipient string, body []byte) error {
ctx, cancel := context.WithTimeout(context.Background(), sendmailTimeout)
defer cancel()
sender = headerValue(sender)
recipient = headerValue(recipient)
// -i: do not treat lone "." as end; -f: envelope sender ("" yields null
// sender); -- terminates options so a hostile recipient cannot be a flag.
_, err := systemdrun.RunInput(ctx, q.lookPath, q.run,
systemdrun.Options{Required: true, RuntimeMax: sendmailTimeout, KillMode: "process"},
body, sendmailPath, "-i", "-f", sender, "--", recipient)
return err
}
func runWithInput(ctx context.Context, input []byte, name string, args ...string) ([]byte, error) {
// #nosec G204 -- name is the resolved systemd-run path; the envelope
// addresses follow fixed flags, the recipient after --.
cmd := exec.CommandContext(ctx, name, args...)
cmd.Stdin = bytes.NewReader(input)
return cmd.Output()
}
package maillog
import (
"errors"
"fmt"
"os"
"github.com/pidginhost/csm/internal/config"
)
// New returns the mail-log Reader appropriate for cfg.Source. A "platform"
// default file path is supplied by the caller (computed from
// internal/platform.Detect()). Pass an empty string to skip the platform
// default - useful in tests.
//
// auto - try file (must exist); fall back to journal if file missing
// but units are configured.
// file - error if the file doesn't exist.
// journal - error if the journal reader is unavailable (default builds).
func New(cfg config.MailLogsConfig, platformDefaultFile string, queue *Queue) (Reader, error) {
path := cfg.File
if path == "" {
path = platformDefaultFile
}
switch cfg.Source {
case "file":
if _, err := os.Stat(path); err != nil {
return nil, fmt.Errorf("mail_logs.source=file but %s: %w", path, err)
}
return NewFileReader(path, queue), nil
case "journal":
if len(cfg.Units) == 0 {
return nil, fmt.Errorf("mail_logs.source=journal requires units")
}
return NewJournalReader(cfg.Units, queue), nil
case "auto":
if path != "" {
if _, err := os.Stat(path); err == nil {
return NewFileReader(path, queue), nil
}
}
if len(cfg.Units) == 0 {
return nil, errors.New("mail_logs.source=auto: log file not found and no units configured for journal fallback")
}
return NewJournalReader(cfg.Units, queue), nil
default:
return nil, fmt.Errorf("mail_logs.source=%q: unknown", cfg.Source)
}
}
package maillog
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
)
// maxLogLineBytes caps a single mail-log line. Real syslog lines top
// out around 8 KB; 64 KB is generous yet bounded. Without this cap a
// malformed source could ship a multi-gigabyte "line" and turn the
// reader into an OOM vector.
const maxLogLineBytes = 64 * 1024
// defaultGoneGrace is how long the source path must stay missing before
// the reader declares the source gone. Long enough to ride out a
// logrotate create-delay (rename old -> create new), short enough that an
// operator notices a real syslog->journald migration quickly.
const defaultGoneGrace = 90 * time.Second
// FileReader tails a single log file. It uses a 2-second polling loop
// because rsyslog/syslog-ng don't reliably trigger inotify events on
// every line written, and periodic path re-stat checks for log rotation.
//
// On context cancel the reader closes the output channel and returns.
type FileReader struct {
path string
queue *Queue
// resolve, when set, names the file to follow, so the reader moves to a
// newer dated file when the source starts one.
resolve func() (string, error)
// onGone, when set, fires once when the source path has been missing
// continuously for goneGrace. A FileReader whose path vanishes mid-run
// (e.g. a syslog->journald migration) otherwise tails a dead fd
// silently; the callback lets the daemon surface a finding and mark the
// watcher unhealthy. onRestored fires after the path returns and the
// reader can use it again. Resolved readers also report persistent
// failures to resolve or open their next target.
onGone func(error)
onRestored func()
goneGrace time.Duration
nowFn func() time.Time
// gone-tracking state, touched only by the single loop goroutine.
firstMissing time.Time
goneFired bool
restoreReady bool
}
// NewFileReader constructs a FileReader for the given path.
func NewFileReader(path string, queue *Queue) *FileReader {
return &FileReader{path: path, queue: queue, goneGrace: defaultGoneGrace, nowFn: time.Now}
}
// NewResolvedFileReader constructs a FileReader that follows the file
// resolve names, such as the newest of a set of dated logs. It starts at the
// end of the file resolve names now, and moves to a file resolve later names
// only after reading the current one to its end. A resolve error is returned
// unchanged so the supervisor reports the source unavailable.
func NewResolvedFileReader(resolve func() (string, error), queue *Queue) (*FileReader, error) {
path, err := resolve()
if err != nil {
return nil, err
}
r := NewFileReader(path, queue)
r.resolve = resolve
return r, nil
}
// switchTarget names the newer file a resolved reader moves to. Only a read
// that stopped, at the end of the current file or on an error, may move: the
// minute ticker can fire while the old file still holds unread lines. The
// reader moves only forward, to a name that sorts after the current one: if
// the current file is removed, an older one must not be replayed from its
// start.
func (r *FileReader) switchTarget(atEOF bool) (string, error) {
if r.resolve == nil || !atEOF {
return "", nil
}
next, err := r.resolve()
if err != nil {
return "", err
}
if filepath.Base(next) <= filepath.Base(r.path) {
return "", nil
}
return next, nil
}
// SetOnGone installs a callback invoked once when the source path has been
// missing, or a resolved target unavailable, for longer than the grace period.
// Must be called before Run.
func (r *FileReader) SetOnGone(fn func(error)) { r.onGone = fn }
// SetOnRestored installs a callback invoked once after a previously-gone
// source path returns and the reader is using it again. Must be called
// before Run.
func (r *FileReader) SetOnRestored(fn func()) { r.onRestored = fn }
// recordStat advances the missing-source state from a stat or resolution result.
// It fires onGone once after the path is missing past the grace period and
// arms the restore callback when the path returns.
func (r *FileReader) recordStat(missing bool, missErr error) {
if !missing {
if !r.firstMissing.IsZero() && r.goneFired {
r.restoreReady = true
}
r.firstMissing = time.Time{}
return
}
now := r.nowFn()
if r.firstMissing.IsZero() {
r.firstMissing = now
}
if !r.goneFired && !r.restoreReady && now.Sub(r.firstMissing) >= r.goneGrace {
r.goneFired = true
if r.onGone != nil {
r.onGone(missErr)
}
}
}
func (r *FileReader) recordRestored() {
if !r.restoreReady {
return
}
r.restoreReady = false
r.goneFired = false
if r.onRestored != nil {
r.onRestored()
}
}
// Run starts the polling loop and returns the line channel. Returns an
// error only when the path can't be opened at all; runtime errors during
// polling are best-effort logged via stderr but do not stop the reader.
func (r *FileReader) Run(ctx context.Context) (<-chan Line, error) {
input, err := r.open()
if err != nil {
return nil, fmt.Errorf("open %s: %w", r.path, err)
}
// A usable replacement retires the old source's current failure while
// retaining its historical uncertainty and delivery loss evidence.
r.queue.journal.outcome(false, false)
out := r.queue.channel()
go r.loop(ctx, out, input)
return out, nil
}
// Temporary EOF does not finish a log record. Both its bounded prefix and
// discard state must survive until the newline or a change of file generation.
type pendingLogLine struct {
data strings.Builder
truncated bool
onRead func(int, int, error)
}
func (p *pendingLogLine) reset() {
p.data.Reset()
p.truncated = false
}
func (p *pendingLogLine) read(ctx context.Context, r *bufio.Reader, maxBytes int) (string, bool, error) {
for {
if err := ctx.Err(); err != nil {
return "", false, err
}
chunk, err := r.ReadSlice('\n')
if p.onRead != nil {
p.onRead(len(chunk), r.Buffered(), err)
}
if len(chunk) > 0 {
switch {
case p.truncated:
// drain remainder to align on next newline
case p.data.Len()+len(chunk) <= maxBytes:
p.data.Write(chunk)
default:
if room := maxBytes - p.data.Len(); room > 0 {
p.data.Write(chunk[:room])
}
p.truncated = true
}
}
if errors.Is(err, bufio.ErrBufferFull) {
continue
}
if err != nil {
return "", false, err
}
line, truncated := p.data.String(), p.truncated
p.reset()
return line, truncated, nil
}
}
func (r *FileReader) open() (*mailFileInput, error) {
return r.openAt(0, io.SeekEnd)
}
func (r *FileReader) openRotated() (*mailFileInput, error) {
return r.openAt(0, io.SeekStart)
}
func (r *FileReader) openAt(offset int64, whence int) (*mailFileInput, error) {
f, err := os.Open(r.path) // #nosec G304 -- operator-supplied log path
if err != nil {
return nil, err
}
position, seekErr := f.Seek(offset, whence)
if seekErr != nil {
_ = f.Close()
return nil, seekErr
}
st, err := f.Stat()
if err != nil {
_ = f.Close()
return nil, err
}
unframed := false
if position > 0 {
// Only a newline before the start offset makes the next byte a
// record start.
var before [1]byte
_, err := f.ReadAt(before[:], position-1)
if errors.Is(err, io.EOF) {
// The file shrank after Seek. Rewind now, before it can regrow
// past the stale offset and conceal the truncation from polling.
position, err = f.Seek(0, io.SeekStart)
} else if err == nil {
unframed = before[0] != '\n'
}
if err != nil {
_ = f.Close()
return nil, err
}
}
dev, ino := fileIdentity(st)
return &mailFileInput{file: f, reader: bufio.NewReader(f), dev: dev, ino: ino, offset: position, size: st.Size(), unframed: unframed}, nil
}
func (r *FileReader) loop(ctx context.Context, out chan<- Line, input *mailFileInput) {
defer close(out)
f, reader, lastDev, lastIno, unframed := input.file, input.reader, input.dev, input.ino, input.unframed
source := r.queue.file.attach(input, true)
sampleCtx, stopSample := context.WithCancel(ctx)
sampleDone := make(chan struct{})
go r.queue.file.sampleLoop(sampleCtx, sampleDone)
normal := false
defer func() {
defer r.queue.file.finish()
defer func() { r.queue.discardFile(source, bufferedMailRecords(reader)) }()
stopSample()
if !normal {
r.queue.file.outcome(fileExitFault, true)
}
r.queue.file.operation()
// The sampler still owns its descriptor until the last Stat returns.
// Closing first can turn normal shutdown into a false source failure.
<-sampleDone
closed := false
defer func() {
if !closed {
r.queue.file.outcome(fileExitFault, true)
}
}()
if err := f.Close(); err != nil {
r.queue.file.outcome(fileCloseFault, true)
}
closed = true
}()
poll := time.NewTicker(2 * time.Second)
defer poll.Stop()
// Rotation safety-net: even if every poll tick finds zero EOFs (a
// continuously-active log), still re-stat once per minute so a
// rotation that happens during a sustained write burst is caught
// without waiting for the next idle period.
rotate := time.NewTicker(time.Minute)
defer rotate.Stop()
pending := pendingLogLine{onRead: func(n, buffered int, err error) { r.queue.file.consume(source, n, buffered, err) }}
rewindOnTruncate := func() {
buffered := bufferedMailRecords(reader)
reset, err := rewindTruncatedFile(f, reader)
r.queue.file.outcome(fileCursorFault, err != nil)
if err != nil {
fmt.Fprintf(os.Stderr, "maillog file_reader %s rewind: %v\n", r.path, err)
} else if reset {
r.queue.discardFile(source, buffered)
source = r.queue.file.attach(&mailFileInput{file: f}, false)
pending.reset()
unframed = false
}
}
switchTo := func(next *mailFileInput) {
_ = f.Close()
r.queue.discardFile(source, bufferedMailRecords(reader))
f, reader, lastDev, lastIno, unframed = next.file, next.reader, next.dev, next.ino, next.unframed
source = r.queue.file.attach(next, true)
pending.reset()
r.recordRestored()
}
reopenOnRotate := func(atEOF bool) {
// A resolved source moves only after draining its old file. A stat
// of that old file must not clear an unavailable new target either.
if r.resolve != nil && !atEOF {
return
}
path, resolveErr := r.switchTarget(atEOF)
if resolveErr != nil {
r.queue.file.outcome(fileOpenFault, true)
r.recordStat(true, resolveErr)
return
}
if path != "" {
prev := r.path
r.path = path
next, err := r.openRotated()
if err != nil {
r.path = prev
r.queue.file.outcome(fileOpenFault, true)
r.recordStat(true, err)
fmt.Fprintf(os.Stderr, "maillog file_reader %s open: %v\n", path, err)
return
}
r.recordStat(false, nil)
switchTo(next)
return
}
st, err := os.Stat(r.path)
if err != nil {
// Track persistent disappearance so a source that vanishes
// mid-run (syslog->journald migration) surfaces instead of
// tailing a dead fd in silence.
r.recordStat(r.resolve != nil || os.IsNotExist(err), err)
return
}
r.recordStat(false, nil)
if dev, ino := fileIdentity(st); dev == lastDev && ino == lastIno {
r.queue.file.outcome(fileOpenFault, false)
rewindOnTruncate()
r.recordRestored()
return
}
next, err := r.openRotated()
if err != nil {
r.queue.file.outcome(fileOpenFault, true)
fmt.Fprintf(os.Stderr, "maillog file_reader %s reopen: %v\n", r.path, err)
return
}
switchTo(next)
}
for {
select {
case <-ctx.Done():
normal = true
return
case <-poll.C:
r.queue.file.operation()
rewindOnTruncate()
for {
line, truncated, err := pending.read(ctx, reader, maxLogLineBytes)
if err != nil {
if ctx.Err() != nil {
normal = true
return
}
r.queue.file.outcome(fileReadFault, !errors.Is(err, io.EOF))
// Tight rotation detection: every time the reader
// hits EOF or any I/O error we re-stat the path so a
// post-rotate log is picked up by the next poll tick
// rather than waiting for the safety-net ticker.
reopenOnRotate(true)
break
}
r.queue.file.outcome(fileReadFault, false)
// This line ends here, so the next one starts a record.
framed := !unframed
unframed = false
if truncated {
r.queue.lose()
r.queue.file.complete(source)
fmt.Fprintf(os.Stderr, "maillog file_reader %s: oversized line skipped at %d bytes\n", r.path, maxLogLineBytes)
continue
}
var position Position
if framed {
position = filePosition(lastDev, lastIno, r.queue.file.settledAt(source), line)
}
if !r.queue.sendFile(ctx, out, Line{Source: "file", Message: line, Position: position}, source) {
normal = true
return
}
}
r.queue.file.idle()
case <-rotate.C:
r.queue.file.operation()
reopenOnRotate(false)
r.queue.file.idle()
}
}
}
func rewindTruncatedFile(f mailLogFile, reader *bufio.Reader) (bool, error) {
st, err := f.Stat()
if err != nil {
return false, err
}
offset, err := f.Seek(0, io.SeekCurrent)
if err != nil || st.Size() >= offset {
return false, err
}
if _, err := f.Seek(0, io.SeekStart); err != nil {
return false, err
}
// Read-ahead bytes belong to the generation that was truncated. Compare
// against the descriptor position so those bytes cannot conceal shrinkage.
reader.Reset(f)
return true, nil
}
// filePosition names a file line by the file and the line's offset and bytes,
// as every reader of the file names it.
func filePosition(dev, ino uint64, offset int64, line string) Position {
return Position{Stream: alert.FileStream(dev, ino), Cursor: alert.RecordCursor(offset, line), ObservedAt: time.Now()}
}
package maillog
import (
"bufio"
"bytes"
"context"
"errors"
"io"
"os"
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
type mailLogFile interface {
io.Reader
io.Seeker
Stat() (os.FileInfo, error)
Close() error
}
type mailFileInput struct {
file mailLogFile
reader *bufio.Reader
dev, ino uint64
offset, size int64
// unframed: offset is inside a line that began before this open.
unframed bool
}
// One identity per open or rewind prevents a late Stat from joining bytes from
// different file generations. All mutable generation fields use the owner lock.
type fileSourceGeneration struct {
file mailLogFile
read, settled, end int64
physical int64
revision uint64
eof, selected bool
known, invalid bool
lagAt time.Time
}
type fileSourceQueue struct {
mu sync.Mutex
seen bool
failures fileSourceFault
sampleFailed, uncertain bool
current *fileSourceGeneration
readerAt, sampleAt time.Time
}
type fileSourceFault uint8
const (
fileReadFault fileSourceFault = 1 << iota
fileCursorFault
fileOpenFault
fileCloseFault
fileExitFault
)
func (s *fileSourceQueue) attach(input *mailFileInput, known bool) *fileSourceGeneration {
s.mu.Lock()
defer s.mu.Unlock()
g := &fileSourceGeneration{file: input.file, read: input.offset, settled: input.offset, physical: input.offset, end: input.size, known: known}
g.invalid = input.size < input.offset
s.current, s.seen = g, true
s.failures, s.sampleFailed = 0, false
s.readerAt = time.Now()
g.refreshLag(s.readerAt, false)
return g
}
func (g *fileSourceGeneration) refreshLag(now time.Time, progress bool) {
if g.end <= g.settled || g.eof && g.end <= g.read {
g.lagAt = time.Time{}
} else if progress || g.lagAt.IsZero() {
g.lagAt = now
}
}
func (s *fileSourceQueue) operation() {
s.mu.Lock()
s.readerAt = time.Now()
s.mu.Unlock()
}
func (s *fileSourceQueue) idle() {
s.mu.Lock()
s.readerAt = time.Time{}
s.mu.Unlock()
}
func (s *fileSourceQueue) outcome(fault fileSourceFault, failed bool) {
s.mu.Lock()
if failed {
s.failures |= fault
s.uncertain = true
} else {
s.failures &^= fault
}
s.mu.Unlock()
}
func (s *fileSourceQueue) consume(g *fileSourceGeneration, n, buffered int, err error) {
s.mu.Lock()
defer s.mu.Unlock()
now := time.Now()
g.read += int64(n)
g.physical = g.read + int64(buffered)
g.end = max(g.end, g.physical)
g.revision++
g.eof, g.selected = errors.Is(err, io.EOF), err == nil
if g.eof && !g.invalid {
g.end, g.known = g.read, true
}
if n > 0 {
s.readerAt = now
}
g.refreshLag(now, n > 0)
}
// settledAt is the offset where the line being read starts: everything before
// it was delivered or discarded.
func (s *fileSourceQueue) settledAt(g *fileSourceGeneration) int64 {
s.mu.Lock()
defer s.mu.Unlock()
return g.settled
}
func (s *fileSourceQueue) complete(g *fileSourceGeneration) {
s.mu.Lock()
defer s.mu.Unlock()
g.settled, g.selected = g.read, false
g.revision++
s.readerAt = time.Now()
g.refreshLag(s.readerAt, true)
}
func (q *Queue) discardFile(g *fileSourceGeneration, bufferedRecords int) {
s := &q.file
s.mu.Lock()
defer s.mu.Unlock()
if g.selected {
bufferedRecords++
}
if bufferedRecords > 0 {
q.health.Lose(time.Now(), uint64(bufferedRecords))
}
s.current = nil
// Disk bytes do not establish record boundaries, and writers can append
// until close. Retain uncertainty without manufacturing a record count.
s.uncertain = true
}
func (s *fileSourceQueue) finish() {
s.mu.Lock()
s.readerAt = time.Time{}
s.mu.Unlock()
}
func bufferedMailRecords(reader *bufio.Reader) int {
// Peek only bytes already in memory; this must not read a closing source.
data, _ := reader.Peek(reader.Buffered())
return bytes.Count(data, []byte{'\n'})
}
func (s *fileSourceQueue) sample() {
s.mu.Lock()
g := s.current
if g == nil {
s.mu.Unlock()
return
}
revision := g.revision
s.sampleAt = time.Now()
s.mu.Unlock()
info, err := g.file.Stat()
s.mu.Lock()
defer s.mu.Unlock()
s.sampleAt = time.Time{}
if s.current != g {
return
}
s.sampleFailed = err != nil
s.uncertain = s.uncertain || err != nil
if err != nil || g.revision != revision {
return
}
if info.Size() < g.physical {
g.invalid, s.uncertain = true, true
} else if !g.invalid {
g.end, g.known = info.Size(), true
}
g.refreshLag(time.Now(), false)
}
func (s *fileSourceQueue) sampleLoop(ctx context.Context, done chan<- struct{}) {
defer close(done)
poll := time.NewTicker(2 * time.Second)
defer poll.Stop()
for {
select {
case <-ctx.Done():
return
case <-poll.C:
s.sample()
}
}
}
func (s *fileSourceQueue) snapshot(now time.Time) (queuehealth.Status, bool) {
s.mu.Lock()
defer s.mu.Unlock()
row := queuehealth.Status{Status: "ok", DepthUnit: "bytes", CapacityUnavailable: true, DroppedLowerBound: s.uncertain, LagBasis: "consumer_progress"}
if g := s.current; g != nil {
row.DepthUnavailable = !g.known || g.invalid || s.sampleFailed
if !row.DepthUnavailable {
row.Depth = int(max(0, g.end-g.settled))
}
if !g.lagAt.IsZero() {
row.LagSeconds = max(0, now.Sub(g.lagAt).Seconds())
}
} else {
// No descriptor left to measure. Zero bytes would be an invented
// reading of a source that is no longer open.
row.DepthUnavailable = true
}
for _, at := range []time.Time{s.readerAt, s.sampleAt} {
if !at.IsZero() {
row.ProcessingSeconds = max(row.ProcessingSeconds, now.Sub(at).Seconds())
}
}
switch {
case s.failures != 0 || s.sampleFailed:
row.Status, row.Reason = "degraded", "source_io"
case row.ProcessingSeconds >= time.Minute.Seconds():
row.Status, row.Reason = "degraded", "processing_lag"
case row.LagSeconds >= time.Minute.Seconds():
row.Status, row.Reason = "degraded", "consumer_stalled"
}
return row, s.seen
}
//go:build unix
package maillog
import (
"os"
"syscall"
)
// fileIdentity is the device and inode that name a log file.
func fileIdentity(fi os.FileInfo) (dev, ino uint64) {
if st, ok := fi.Sys().(*syscall.Stat_t); ok {
return uint64(st.Dev), st.Ino // #nosec G115 -- device IDs are non-negative on supported Unix hosts
}
return 0, 0
}
package maillog
import "time"
// JournalLine builds the Line for one journal entry. Its message takes the
// shape syslog writes to a mail log file, "<time> <identifier>[<pid>]: <text>",
// so the mail parsers treat file and journal input alike. They find the
// program tag at the first ": ", which MESSAGE alone does not carry; building
// it from the entry's fields keeps client text in the message from supplying
// it. Identifier and pid fall back the way journalctl's short output does.
func JournalLine(fields map[string]string, pos Position) Line {
msg := fields["MESSAGE"]
ident := fields["SYSLOG_IDENTIFIER"]
if ident == "" {
ident = fields["_COMM"]
}
if ident != "" {
pid := fields["_PID"]
if pid == "" {
pid = fields["SYSLOG_PID"]
}
if pid != "" {
ident += "[" + pid + "]"
}
msg = ident + ": " + msg
}
// The time bounds how far apart two lines can be and still pair.
if !pos.ObservedAt.IsZero() {
msg = pos.ObservedAt.Format(time.RFC3339Nano) + " " + msg
}
return Line{Source: "journal", Unit: fields["_SYSTEMD_UNIT"], Message: msg, Position: pos}
}
//go:build !linux || !journal
package maillog
import (
"context"
"errors"
)
// JournalReader is a no-op stub on builds without the `journal` tag.
// The factory (T4) uses this to produce a clear error rather than
// silently downgrading to file mode when the operator explicitly asked
// for journald.
type JournalReader struct{}
// NewJournalReader satisfies the same constructor signature as the
// linux+journal build, so the factory and tests compile identically
// on default builds.
func NewJournalReader(_ []string, _ *Queue) *JournalReader { return &JournalReader{} }
func JournalSupported() bool { return false }
// ErrJournalUnsupported is returned when the build was produced without
// the `journal` tag (default builds).
var ErrJournalUnsupported = errors.New("journal reader not compiled in (build with JOURNAL=1)")
// Run returns ErrJournalUnsupported immediately on stub builds.
func (*JournalReader) Run(_ context.Context) (<-chan Line, error) {
return nil, ErrJournalUnsupported
}
package maillog
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
// The journal cursor cannot measure its unread backlog. Track actual reader
// progress and the one selected entry without taking the journal's I/O locks.
type journalSourceQueue struct {
mu sync.Mutex
seen, active, selected bool
failed, uncertain bool
at time.Time
}
func (s *journalSourceQueue) outcome(failed, uncertain bool) {
s.mu.Lock()
s.failed = failed
s.uncertain = s.uncertain || uncertain
s.mu.Unlock()
}
func (s *journalSourceQueue) snapshot(now time.Time) (queuehealth.Status, bool) {
s.mu.Lock()
defer s.mu.Unlock()
// The cursor exposes no unread count and no waiting age, so the row
// measures the current operation and says so instead of reporting a
// backlog of zero.
row := queuehealth.Status{Status: "ok", DepthUnavailable: true, CapacityUnavailable: true, DroppedLowerBound: s.uncertain, LagBasis: "unavailable"}
if s.selected {
row.InFlight = 1
}
if s.active {
row.ProcessingSeconds = max(0, now.Sub(s.at).Seconds())
}
switch {
case s.failed:
row.Status, row.Reason = "degraded", "source_io"
case row.ProcessingSeconds >= time.Minute.Seconds():
row.Status, row.Reason = "degraded", "processing_lag"
}
return row, s.seen
}
package maillog
import (
"crypto/sha256"
"encoding/hex"
"time"
)
// Position names where a mail log line was read: the stream (one generation
// of the log file, or the journal), the line's place in it, and when it was
// observed. An empty Stream means the reader has no position for the line.
type Position struct {
Stream string
Cursor string
ObservedAt time.Time
}
// journalCursor encodes a journald cursor as a bounded token. systemd treats
// cursors as opaque, and a busy host's cursor outgrows the evidence bound, so
// a hash of the whole cursor names the entry.
func journalCursor(raw string) string {
if raw == "" {
return ""
}
sum := sha256.Sum256([]byte(raw))
return "jc1:" + hex.EncodeToString(sum[:16])
}
package maillog
import (
"context"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
const queueCapacity = 64
// Queue retains delivery health across reader replacements and source changes.
// The supervisor finishes the old reader before starting its replacement.
type Queue struct {
health *queuehealth.Tracker
journal journalSourceQueue
file fileSourceQueue
}
func NewQueue() *Queue {
return &Queue{health: queuehealth.New(queueCapacity, time.Minute)}
}
func (q *Queue) QueueStatuses(now time.Time) map[string]queuehealth.Status {
rows := map[string]queuehealth.Status{"delivery": q.health.Snapshot(now)}
if journal, seen := q.journal.snapshot(now); seen {
rows["journal_source"] = journal
}
if file, seen := q.file.snapshot(now); seen {
rows["file_source"] = file
}
return rows
}
func (q *Queue) channel() chan Line { return make(chan Line, queueCapacity) }
func (q *Queue) send(ctx context.Context, out chan<- Line, line Line) bool {
line.ticket = q.health.Begin(time.Now())
return q.sendTracked(ctx, out, line)
}
func (q *Queue) sendFile(ctx context.Context, out chan<- Line, line Line, source *fileSourceGeneration) bool {
line.ticket = q.health.Begin(time.Now())
q.file.complete(source)
return q.sendTracked(ctx, out, line)
}
func (q *Queue) sendTracked(ctx context.Context, out chan<- Line, line Line) bool {
select {
case out <- line:
return true
case <-ctx.Done():
line.reject()
return false
}
}
func (q *Queue) lose() { q.health.Lose(time.Now(), 1) }
func (line Line) reject() { line.ticket.Reject(time.Now()) }
// Process accounts for delivery through the entire consumer callback. False
// means delivery was abandoned. A panic is counted and continues to the owner.
func (line Line) Process(consume func(Line) bool) (completed bool) {
ticket := line.ticket
line.ticket = queuehealth.Ticket{}
ticket.Start(time.Now())
defer func() {
if completed {
ticket.Finish(time.Now())
} else {
ticket.Reject(time.Now())
}
}()
return consume(line)
}
package maillog
import (
"context"
"errors"
"sync"
"time"
)
// Supervise retries source selection and attachment until ctx is canceled or
// consume returns false. status receives nil only after successful attachment,
// and an error while unavailable. Each old reader finishes before its replacement
// starts, so source migration cannot count the same event through two readers.
func Supervise(ctx context.Context, factory func() (Reader, error), status func(error), consume func(Line) bool) {
delay := time.Second
var statusMu sync.Mutex
reported, failed := false, false
lastFailure := ""
report := func(err error) {
// File disappearance must change health even while consume is blocked.
// Serialize that callback with attachment and retry status updates.
statusMu.Lock()
defer statusMu.Unlock()
failure := ""
if err != nil {
failure = err.Error()
}
if reported && failed == (err != nil) && lastFailure == failure {
return
}
reported, failed, lastFailure = true, err != nil, failure
status(err)
}
ready := func() {
delay = time.Second
report(nil)
}
for ctx.Err() == nil {
reader, err := factory()
if err == nil {
err = consumeReader(ctx, reader, ready, report, consume)
}
if ctx.Err() != nil || errors.Is(err, context.Canceled) {
return
}
report(err)
timer := time.NewTimer(delay)
select {
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
delay = min(delay*2, 30*time.Second)
}
}
func consumeReader(ctx context.Context, reader Reader, ready func(), unavailable func(error), consume func(Line) bool) error {
readerCtx, cancel := context.WithCancel(ctx)
defer cancel()
gone := make(chan error, 1)
if file, ok := reader.(*FileReader); ok {
file.SetOnGone(func(err error) {
unavailable(err)
select {
case gone <- err:
default:
}
})
}
lines, err := reader.Run(readerCtx)
if err != nil {
return err
}
defer func() {
cancel()
for line := range lines {
line.reject()
}
}()
if err := ctx.Err(); err != nil {
return err
}
ready()
for {
select {
case <-ctx.Done():
return ctx.Err()
case err := <-gone:
cancel()
for line := range lines {
if ctxErr := ctx.Err(); ctxErr != nil {
line.reject()
return ctxErr
}
if !line.Process(consume) {
return context.Canceled
}
}
return err
case line, ok := <-lines:
if !ok {
return errors.New("mail log reader stopped")
}
if !line.Process(consume) {
return context.Canceled
}
}
}
}
// Package mailranges maintains an atomic in-memory map of mail-provider IP
// ranges used to exempt shared-source ranges (carrier CGNAT, mail providers)
// from firewall DoS heuristics. The package is self-contained: it embeds a
// seed snapshot so the binary ships with a usable fallback and holds no
// references to any identity-verification or bot-allowlist logic.
package mailranges
import (
_ "embed"
"encoding/json"
"errors"
"net"
"os"
"strings"
"sync/atomic"
"time"
)
// embeddedSnapshot is the seed snapshot compiled into the binary, used when
// the on-disk cache cannot be read or parsed. TestGenerateSnapshot refreshes
// the source from live SPF records when explicitly enabled; a build alone
// does not refresh it.
//
//go:embed snapshot.json
var embeddedSnapshot []byte
// cacheFile is the shared JSON schema for both the on-disk cache and the
// embedded snapshot. The same decoder handles both paths.
type cacheFile struct {
RefreshedAt int64 `json:"refreshed_at"`
Providers map[string][]string `json:"providers"`
}
// providerMap is the concrete type stored in providerAtom. atomic.Value
// requires the same concrete type on every Store call; a named type satisfies
// that without an extra allocation.
type providerMap map[string][]*net.IPNet
var (
providerAtom atomic.Value // stores providerMap; nil load = never published
lastRefreshAt atomic.Int64 // Unix timestamp; 0 = never published
)
// PublishProviderSnapshot installs a new provider snapshot atomically. The
// input is deep-copied before storing so callers may modify m after returning.
// A nil or empty m publishes an empty map (clears the effective set).
func PublishProviderSnapshot(m map[string][]*net.IPNet) {
cp := make(providerMap, len(m))
for k, v := range m {
nets := make([]*net.IPNet, len(v))
for i, n := range v {
nets[i] = cloneIPNet(n)
}
cp[k] = nets
}
providerAtom.Store(cp)
}
// ProviderNets returns a flat slice of all current provider nets across every
// provider. Each call returns a freshly allocated deep copy so callers may
// inspect or modify the slice and its elements without affecting the store.
func ProviderNets() []*net.IPNet {
m := loadProviderMap()
var out []*net.IPNet
for _, nets := range m {
for _, n := range nets {
out = append(out, cloneIPNet(n))
}
}
return out
}
// ProviderSnapshot returns the current provider map as a deep copy keyed by
// provider name. The returned map and its *net.IPNet values are independent of
// the atomic store; mutations do not propagate back.
func ProviderSnapshot() map[string][]*net.IPNet {
m := loadProviderMap()
out := make(map[string][]*net.IPNet, len(m))
for k, v := range m {
nets := make([]*net.IPNet, len(v))
for i, n := range v {
nets[i] = cloneIPNet(n)
}
out[k] = nets
}
return out
}
// LastRefresh returns when PublishProviderSnapshot was last called (via
// LoadCache or an external refresh), or the zero time if it never has been.
func LastRefresh() time.Time {
ts := lastRefreshAt.Load()
if ts == 0 {
return time.Time{}
}
return time.Unix(ts, 0)
}
// LoadCache reads the on-disk provider range cache at path and publishes it.
// On any read or parse failure it falls back to embeddedSnapshot. If both
// the on-disk cache and the embedded snapshot fail to parse, LoadCache
// publishes an empty provider map and returns the error so the caller can log
// it. A missing file is a normal first-run condition and is not an error when
// the embedded snapshot parses successfully.
func LoadCache(path string) error {
data, readErr := os.ReadFile(path) // #nosec G304 -- daemon-owned state path
if readErr == nil {
m, ts, err := parseCacheData(data)
if err == nil {
PublishProviderSnapshot(m)
lastRefreshAt.Store(ts)
return nil
}
// on-disk cache unreadable; fall through to embedded snapshot
}
m, ts, embErr := parseCacheData(embeddedSnapshot)
if embErr != nil {
// Both sources failed; publish an empty map so readers get a safe zero value.
PublishProviderSnapshot(nil)
return embErr
}
PublishProviderSnapshot(m)
lastRefreshAt.Store(ts)
return nil
}
// parseCacheData decodes the shared cacheFile JSON format into a provider map.
// Malformed entries make the whole cache unusable so LoadCache can fall back to
// the embedded snapshot instead of publishing a narrowed partial set.
func parseCacheData(data []byte) (map[string][]*net.IPNet, int64, error) {
var c cacheFile
if err := json.Unmarshal(data, &c); err != nil {
return nil, 0, err
}
m := make(map[string][]*net.IPNet, len(c.Providers))
total := 0
for provider, strs := range c.Providers {
for _, s := range strs {
_, n, err := net.ParseCIDR(strings.TrimSpace(s))
if err != nil {
return nil, 0, err
}
m[provider] = append(m[provider], n)
total++
}
}
if total == 0 {
return nil, 0, errors.New("mailranges: cache has no usable provider prefixes")
}
return m, c.RefreshedAt, nil
}
func loadProviderMap() providerMap {
v := providerAtom.Load()
if v == nil {
return nil
}
return v.(providerMap) //nolint:forcetypeassert -- only providerMap is ever stored
}
// cloneIPNet returns a deep copy of n. Both the IP and Mask byte slices are
// freshly allocated so mutations to the original do not affect the clone.
func cloneIPNet(n *net.IPNet) *net.IPNet {
if n == nil {
return nil
}
ip := make(net.IP, len(n.IP))
copy(ip, n.IP)
mask := make(net.IPMask, len(n.Mask))
copy(mask, n.Mask)
return &net.IPNet{IP: ip, Mask: mask}
}
package mailranges
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"sort"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/atomicio"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/metrics"
)
// Providers maps provider name to the SPF root domain that covers all
// outbound mail ranges for that provider. Only the two major mail providers
// whose shared IPs are relevant for DoS-exempt range building are included.
// No JSON feed URLs, ASN feeds, or CIDR lists; only SPF root domains.
var Providers = map[string]string{
"google": "_spf.google.com",
"microsoft": "spf.protection.outlook.com",
}
// staleCacheThreshold is how old the last-good cache must be before a failed
// refresh emits a warning so operators know the data may be outdated.
const staleCacheThreshold = 7 * 24 * time.Hour
// Package-level atomics for metrics. Separate from providerAtom and
// lastRefreshAt (defined in mailranges.go) to keep each concern isolated.
var (
mailrangesRefreshTotal atomic.Uint64 // number of refreshes with >= 1 success
mailrangesPrefixes atomic.Int64 // total prefix count after last successful refresh
staleCacheWarnings atomic.Uint64 // stale-cache warnings emitted (test-observable)
)
// RegisterMailrangesMetrics binds the mailranges counters and gauges to reg.
// Production callers pass metrics.Default(); tests pass metrics.NewRegistry()
// to keep registration isolated. Idempotent per-registry (the underlying
// RegisterCounterFunc/RegisterGaugeFunc panic on duplicate names; callers
// must not register the same registry twice).
func RegisterMailrangesMetrics(reg *metrics.Registry) {
reg.RegisterCounterFunc(
"csm_mailranges_refresh_total",
"Total provider range refreshes with at least one successful SPF resolve.",
func() float64 { return float64(mailrangesRefreshTotal.Load()) },
)
reg.RegisterGaugeFunc(
"csm_mailranges_prefixes",
"Current number of mail-provider prefixes across all providers.",
func() float64 { return float64(mailrangesPrefixes.Load()) },
)
reg.RegisterGaugeFunc(
"csm_mailranges_last_success_timestamp_seconds",
"Unix timestamp of the last successful provider range refresh.",
func() float64 { return float64(lastRefreshAt.Load()) },
)
}
// Refresh resolves every provider in Providers using r. For each provider that
// fails, the provider's previous last-good ranges are kept in the merged map.
// When at least one provider resolves successfully, the merged map is serialized
// and written atomically to cachePath, then published as the active snapshot and
// metrics are updated. The active snapshot is only updated after the write
// succeeds so a torn write cannot narrow the effective set.
//
// Returns (total prefixes across the merged map, nil) on full success.
// Returns (total, joinedErr) when some providers fail but at least one
// succeeds, where joinedErr wraps every failed provider's error.
// Returns (0, joinedErr) when all providers fail or the cache cannot be
// written; in the all-failure case, if the existing cache is older than 7 days,
// a stale-cache warning is emitted.
func Refresh(ctx context.Context, r Resolver, cachePath string) (int, error) {
// Take a snapshot of the current active provider map to use as the
// last-good baseline. Providers that fail this cycle keep their entry.
prev := ProviderSnapshot()
// Prime the merged map with last-good values. Successful resolves overwrite.
merged := make(map[string][]*net.IPNet, len(Providers))
for name := range Providers {
if nets, ok := prev[name]; ok {
merged[name] = nets
}
}
var errs []error
successCount := 0
for name, root := range Providers {
nets, err := ResolveSPF(ctx, r, root)
if err != nil {
csmlog.Warn("mailranges: provider SPF resolve failed",
"provider", name, "root", root, "err", err)
errs = append(errs, fmt.Errorf("provider %q: %w", name, err))
continue
}
if len(nets) == 0 {
err := fmt.Errorf("spf: no usable public prefixes")
csmlog.Warn("mailranges: provider SPF resolve failed",
"provider", name, "root", root, "err", err)
errs = append(errs, fmt.Errorf("provider %q: %w", name, err))
continue
}
merged[name] = nets
successCount++
}
// All providers failed: do not write or publish; warn if the cache is stale.
if successCount == 0 {
ts := lastRefreshAt.Load()
if ts != 0 && time.Since(time.Unix(ts, 0)) > staleCacheThreshold {
staleCacheWarnings.Add(1)
csmlog.Warn("mailranges: provider refresh failed; cache is stale",
"age_hours", int(time.Since(time.Unix(ts, 0)).Hours()))
}
return 0, errors.Join(errs...)
}
// Count total prefixes across all merged providers (successful + last-good).
total := 0
for _, nets := range merged {
total += len(nets)
}
// Build the on-disk cache payload. Sort each provider's slice for stable output.
cf := cacheFile{
RefreshedAt: time.Now().Unix(),
Providers: make(map[string][]string, len(merged)),
}
for name, nets := range merged {
strs := make([]string, 0, len(nets))
for _, n := range nets {
strs = append(strs, n.String())
}
sort.Strings(strs)
cf.Providers[name] = strs
}
data, err := json.Marshal(cf)
if err != nil {
return 0, err
}
// Atomic write first. Do not touch the active snapshot if the write fails.
if err := atomicio.AtomicWrite(cachePath, 0o600, data); err != nil {
return 0, err
}
// Publish only after the write commits.
PublishProviderSnapshot(merged)
lastRefreshAt.Store(cf.RefreshedAt)
// Update metrics only when at least one provider resolved successfully.
mailrangesRefreshTotal.Add(1)
mailrangesPrefixes.Store(int64(total))
// errs is non-nil only on partial failure; errors.Join(nil...) returns nil
// for the full-success path.
return total, errors.Join(errs...)
}
package mailranges
import (
"context"
"fmt"
"net"
"slices"
"strings"
)
// Resolver is the DNS lookup interface used by ResolveSPF and
// ResolveSPFPolicy. The standard library's net.Resolver satisfies this
// interface; tests supply a fake.
type Resolver interface {
LookupTXT(ctx context.Context, name string) ([]string, error)
}
// maxSPFDepth is the maximum recursion depth for include/redirect chains.
// Chains deeper than this are rejected; a well-formed SPF record never needs
// more than a handful of levels, and deeper chains are a sign of misconfiguration
// or an attempt to exhaust resolver resources.
const maxSPFDepth = 10
// maxSPFLookups bounds total DNS TXT lookups during one resolution.
// Depth alone does not bound a record that fans out to many unique includes.
const maxSPFLookups = 64
// nonPublicCIDRs lists finite (non-/0) reserved ranges that must never appear
// in a mail-provider SPF record. Default routes (0.0.0.0/0, ::/0) are handled
// separately because they contain every address and would incorrectly reject all
// public prefixes if tested with the Contains-based overlap check.
var nonPublicCIDRs = func() []*net.IPNet {
ranges := []string{
"10.0.0.0/8", // RFC 1918 private
"172.16.0.0/12", // RFC 1918 private
"192.168.0.0/16", // RFC 1918 private
"100.64.0.0/10", // RFC 6598 Shared Address Space
"127.0.0.0/8", // loopback (IPv4)
"::1/128", // loopback (IPv6)
"169.254.0.0/16", // link-local (IPv4)
"fe80::/10", // link-local (IPv6)
"fc00::/7", // ULA (unique-local)
"192.0.2.0/24", // RFC 5737 TEST-NET-1 (documentation)
"198.51.100.0/24", // RFC 5737 TEST-NET-2 (documentation)
"203.0.113.0/24", // RFC 5737 TEST-NET-3 (documentation)
"2001:db8::/32", // RFC 3849 (documentation)
}
cidrs := make([]*net.IPNet, 0, len(ranges))
for _, r := range ranges {
_, n, err := net.ParseCIDR(r)
if err != nil {
// All entries are compile-time constants; a parse error is a bug.
panic(fmt.Sprintf("mailranges: bad nonPublicCIDR %q: %v", r, err))
}
cidrs = append(cidrs, n)
}
return cidrs
}()
// isPublicPrefix reports whether n is routable public address space. It
// returns false for:
// - Any default route (prefix length 0), either IPv4 or IPv6.
// - Any prefix whose network address falls within a reserved range.
// - Any prefix that is a supernet containing a reserved range's network address,
// so attackers cannot smuggle private space inside a wide covering prefix.
//
// IPv4-mapped IPv6 prefixes (::ffff:<private>/N) are caught by the existing
// IPv4 range checks because Go's net.Contains normalises IPv4-mapped addresses
// to their 4-byte form before comparing against 4-byte reserved ranges.
func isPublicPrefix(n *net.IPNet) bool {
// Reject default routes: mask with all zero bits in either address family.
ones, bits := n.Mask.Size()
if bits > 0 && ones == 0 {
return false
}
// Bidirectional overlap check against every finite reserved range.
// reserved.Contains(n.IP): n's network address is within the reserved range.
// n.Contains(reserved.IP): n is a supernet that contains a reserved range.
for _, reserved := range nonPublicCIDRs {
if reserved.Contains(n.IP) || n.Contains(reserved.IP) {
return false
}
}
return true
}
// spfRecord holds the result of parsing one SPF TXT string. It carries only
// the token types the resolvers care about. Outside strict mode all other
// mechanisms (a, mx, ptr, exists, qualifiers) are silently ignored because
// they reference the domain itself, not static CIDR blocks useful for
// DoS-exempt range building.
type spfRecord struct {
nets []*net.IPNet
includes []string
redirect string // at most one per record
ptrs []string // strict mode only, normalised by spfPTRDomain
}
// parseSPFRecord parses a single SPF TXT record string and returns its tokens.
// It makes no network calls; all recursion lives in resolveSPFRec. Records
// that do not begin with "v=spf1" are rejected. Malformed ip4:/ip6: CIDRs cause
// an error so callers fall back to the last-good provider set rather than
// silently omitting them. In strict mode a macro in an include or redirect
// target is an error, and the other pass-qualified terms go through
// strictSPFTerm; self is the name the record was published at, which a bare
// ptr refers to.
func parseSPFRecord(txt, self string, strict bool) (spfRecord, error) {
fields := strings.Fields(txt)
if len(fields) == 0 || !strings.EqualFold(fields[0], "v=spf1") {
return spfRecord{}, fmt.Errorf("spf: not a v=spf1 record")
}
var rec spfRecord
var hasRedirect bool
for _, tok := range fields[1:] {
rawLower := strings.ToLower(tok)
if strings.HasPrefix(rawLower, "redirect=") {
// Track presence with a separate flag: an empty first redirect=
// value must not let a second redirect= slip past the guard.
if hasRedirect {
return spfRecord{}, fmt.Errorf("spf: multiple redirect= directives in one record")
}
hasRedirect = true
rec.redirect = tok[9:]
if rec.redirect == "" {
return spfRecord{}, fmt.Errorf("spf: empty redirect= directive")
}
// A macro expands per message, so the name as written is not a
// record the provider publishes for all its senders.
if strict && strings.Contains(rec.redirect, "%") {
return spfRecord{}, fmt.Errorf("spf: macro in redirect target %q", rec.redirect)
}
continue
}
mech, pass, hadQualifier := spfMechanismToken(tok)
lower := strings.ToLower(mech)
if hadQualifier && strings.HasPrefix(lower, "redirect=") {
return spfRecord{}, fmt.Errorf("spf: redirect= directive cannot carry a qualifier")
}
if !pass {
continue
}
switch {
case strings.HasPrefix(lower, "ip4:"):
cidr := mech[4:]
n, err := parseSPFIPNet("ip4", cidr)
if err != nil {
return spfRecord{}, fmt.Errorf("spf: bad ip4 prefix %q: %w", cidr, err)
}
rec.nets = append(rec.nets, n)
case strings.HasPrefix(lower, "ip6:"):
cidr := mech[4:]
n, err := parseSPFIPNet("ip6", cidr)
if err != nil {
return spfRecord{}, fmt.Errorf("spf: bad ip6 prefix %q: %w", cidr, err)
}
rec.nets = append(rec.nets, n)
case strings.HasPrefix(lower, "include:"):
domain := mech[8:]
if domain == "" {
return spfRecord{}, fmt.Errorf("spf: empty include directive")
}
if strict && strings.Contains(domain, "%") {
return spfRecord{}, fmt.Errorf("spf: macro in include target %q", domain)
}
rec.includes = append(rec.includes, domain)
case strict:
ptr, err := strictSPFTerm(mech, self)
if err != nil {
return spfRecord{}, err
}
if ptr != "" {
rec.ptrs = append(rec.ptrs, ptr)
}
}
// Outside strict mode all other tokens (all, a, mx, ptr, exists,
// modifiers) are ignored.
}
return rec, nil
}
// strictSPFTerm vets a pass-qualified term that adds no prefix and returns the
// ptr domain it authorises, if any. A Policy lists only prefixes and ptr
// domains, so a, mx, exists, all or an unknown mechanism would authorise
// senders it cannot hold; they fail rather than yield a short policy.
// Modifiers such as exp= authorise nothing.
func strictSPFTerm(mech, self string) (string, error) {
if i := strings.IndexAny(mech, ":/="); i >= 0 && mech[i] == '=' {
return "", nil
}
lower := strings.ToLower(mech)
switch {
case lower == "ptr":
return spfPTRDomain(self)
case strings.HasPrefix(lower, "ptr:"):
return spfPTRDomain(mech[4:])
}
return "", fmt.Errorf("spf: unsupported mechanism %q in strict policy", mech)
}
// spfLabelChars are the bytes a ptr domain label may hold.
const spfLabelChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
// spfPTRDomain returns d in lower case without one trailing dot, or an error
// unless d is a plain ASCII host name. Callers match a ptr domain as a name
// suffix, so a macro or wildcard would never match what the record meant.
// RFC 7208 forbids an all-digit top label, which also keeps out an IPv4
// literal.
func spfPTRDomain(d string) (string, error) {
name := strings.TrimSuffix(d, ".")
labels := strings.Split(name, ".")
ok := len(name) <= 253 && strings.Trim(labels[len(labels)-1], "0123456789") != ""
for _, l := range labels {
ok = ok && len(l) >= 1 && len(l) <= 63 && strings.Trim(l, spfLabelChars) == ""
}
if !ok {
return "", fmt.Errorf("spf: invalid ptr domain %q", d)
}
return strings.ToLower(name), nil
}
// spfMechanismToken returns the mechanism token after SPF qualifier handling.
// Only pass mechanisms (no qualifier or explicit '+') can contribute ranges.
func spfMechanismToken(tok string) (mech string, pass bool, hadQualifier bool) {
if tok == "" {
return "", false, false
}
switch tok[0] {
case '+':
return tok[1:], true, true
case '-', '~', '?':
return tok[1:], false, true
default:
return tok, true, false
}
}
func parseSPFIPNet(family, value string) (*net.IPNet, error) {
value = strings.TrimSpace(value)
if value == "" {
return nil, fmt.Errorf("empty prefix")
}
if strings.Contains(value, "/") {
_, n, err := net.ParseCIDR(value)
if err != nil {
return nil, err
}
return normalizeSPFIPNetFamily(family, n)
}
ip := net.ParseIP(value)
if ip == nil {
return nil, fmt.Errorf("invalid IP address")
}
switch family {
case "ip4":
ip4 := ip.To4()
if ip4 == nil {
return nil, fmt.Errorf("not an IPv4 prefix")
}
return &net.IPNet{IP: ip4, Mask: net.CIDRMask(32, 32)}, nil
case "ip6":
if ip.To4() != nil {
return nil, fmt.Errorf("not an IPv6 prefix")
}
ip16 := ip.To16()
if ip16 == nil {
return nil, fmt.Errorf("not an IPv6 prefix")
}
return &net.IPNet{IP: ip16, Mask: net.CIDRMask(128, 128)}, nil
default:
return nil, fmt.Errorf("unknown SPF IP family %q", family)
}
}
func normalizeSPFIPNetFamily(family string, n *net.IPNet) (*net.IPNet, error) {
_, bits := n.Mask.Size()
switch family {
case "ip4":
ip4 := n.IP.To4()
if ip4 == nil || bits != 32 {
return nil, fmt.Errorf("not an IPv4 prefix")
}
n.IP = ip4
return n, nil
case "ip6":
if n.IP.To4() != nil || bits != 128 {
return nil, fmt.Errorf("not an IPv6 prefix")
}
ip16 := n.IP.To16()
if ip16 == nil {
return nil, fmt.Errorf("not an IPv6 prefix")
}
n.IP = ip16
return n, nil
default:
return nil, fmt.Errorf("unknown SPF IP family %q", family)
}
}
// ResolveSPF resolves the SPF record for root, following include: and redirect=
// directives recursively up to maxSPFDepth levels. It collects all ip4: and
// ip6: prefixes found across the chain, rejects non-public prefixes, detects
// loops, and de-duplicates the result.
//
// The function returns an error rather than partial results on any anomaly
// (loop, depth exceeded, malformed TXT, non-public prefix, malformed CIDR) so
// callers can keep the last-good set instead of publishing a poisoned one.
func ResolveSPF(ctx context.Context, r Resolver, root string) ([]*net.IPNet, error) {
st := &spfResolveState{
onPath: make(map[string]bool),
memo: make(map[string][]*net.IPNet),
}
nets, err := resolveSPFRec(ctx, r, root, st, 0)
if err != nil {
return nil, err
}
return dedupNets(nets), nil
}
// Policy is every sender an SPF tree authorises: prefixes and ptr: domains.
type Policy struct {
Nets []*net.IPNet
PTRDomains []string // lower case, no trailing dot, deduplicated, sorted
}
// ResolveSPFPolicy walks root like ResolveSPF, with the same prefix checks and
// bounds, but strictly. It also collects pass-qualified ptr domains, and it
// fails on any pass-qualified term whose senders a Policy cannot list (a, mx,
// exists, all, an unknown mechanism, a ptr macro or an invalid ptr domain), on
// a macro in an include or redirect target, and on a tree that authorises no
// sender, so a caller can treat the result as the provider's whole sender set.
func ResolveSPFPolicy(ctx context.Context, r Resolver, root string) (Policy, error) {
st := &spfResolveState{
strict: true,
onPath: make(map[string]bool),
memo: make(map[string][]*net.IPNet),
}
nets, err := resolveSPFRec(ctx, r, root, st, 0)
if err != nil {
return Policy{}, err
}
slices.Sort(st.ptrs)
p := Policy{Nets: dedupNets(nets), PTRDomains: slices.Compact(st.ptrs)}
if len(p.Nets) == 0 && len(p.PTRDomains) == 0 {
return Policy{}, fmt.Errorf("spf: %q authorises no senders", root)
}
return p, nil
}
// spfResolveState carries the per-resolution bookkeeping shared across the
// recursive walk. onPath holds the domains on the current DFS branch so a true
// ancestor cycle errors, while memo caches fully-resolved domains so a diamond
// (the same sub-domain reached via two different include branches) resolves once
// and is reused instead of being mistaken for a loop. ptrs gathers strict-mode
// ptr domains from every parsed record; a memo hit adds none because its record
// already did.
type spfResolveState struct {
strict bool
onPath map[string]bool
memo map[string][]*net.IPNet
ptrs []string
lookups int
}
// resolveSPFRec is the internal recursive worker for ResolveSPF and
// ResolveSPFPolicy. It returns the prefixes collected from domain's record and
// everything it transitively references; in strict mode it also adds their ptr
// domains to st.ptrs.
func resolveSPFRec(ctx context.Context, r Resolver, domain string, st *spfResolveState, depth int) ([]*net.IPNet, error) {
// A domain present on the current branch is a genuine ancestor cycle.
if st.onPath[domain] {
return nil, fmt.Errorf("spf: include/redirect loop detected at %q", domain)
}
// A fully-resolved domain reached again off-path is a diamond, not a loop;
// reuse its cached result without re-querying or re-counting depth.
if cached, ok := st.memo[domain]; ok {
return cached, nil
}
// Depth bounds only NEW resolutions; cached/cyclic cases are handled above.
if depth >= maxSPFDepth {
return nil, fmt.Errorf("spf: depth limit (%d) exceeded at %q", maxSPFDepth, domain)
}
if st.lookups >= maxSPFLookups {
return nil, fmt.Errorf("spf: DNS lookup limit (%d) exceeded at %q", maxSPFLookups, domain)
}
st.lookups++
// Check context before every network call so callers can abort the chain.
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
txts, err := r.LookupTXT(ctx, domain)
if err != nil {
return nil, fmt.Errorf("spf: lookup %q: %w", domain, err)
}
// Select exactly one SPF record. Match only an exact "v=spf1" or a
// "v=spf1 " prefix so a malformed "v=spf1foo..." string is not selected
// over a real later record.
var spfTxt string
for _, t := range txts {
if isSPFTXT(t) {
if spfTxt != "" {
return nil, fmt.Errorf("spf: multiple v=spf1 records for %q", domain)
}
spfTxt = t
}
}
if spfTxt == "" {
return nil, fmt.Errorf("spf: no v=spf1 record for %q", domain)
}
rec, err := parseSPFRecord(spfTxt, domain, st.strict)
if err != nil {
return nil, err
}
st.ptrs = append(st.ptrs, rec.ptrs...)
// Enter the branch; leave it on return so siblings can revisit shared nodes.
st.onPath[domain] = true
defer delete(st.onPath, domain)
// Validate and collect ip4:/ip6: prefixes before recursing so a bad record
// causes an immediate error rather than a partial result.
var collected []*net.IPNet
// Repeated includes and overlapping subtrees must not multiply the
// working set, even when memoization keeps the DNS lookup count small.
seenNets := make(map[string]bool)
collect := func(n *net.IPNet) {
key := n.String()
if !seenNets[key] {
seenNets[key] = true
collected = append(collected, n)
}
}
for _, n := range rec.nets {
if !isPublicPrefix(n) {
return nil, fmt.Errorf("spf: non-public prefix %s in record for %q", n, domain)
}
collect(n)
}
// Recurse into includes.
seenIncludes := make(map[string]bool)
for _, inc := range rec.includes {
if seenIncludes[inc] {
continue
}
seenIncludes[inc] = true
sub, err := resolveSPFRec(ctx, r, inc, st, depth+1)
if err != nil {
return nil, err
}
for _, n := range sub {
collect(n)
}
}
// Follow redirect= (at most one per record, enforced by parseSPFRecord).
if rec.redirect != "" {
sub, err := resolveSPFRec(ctx, r, rec.redirect, st, depth+1)
if err != nil {
return nil, err
}
for _, n := range sub {
collect(n)
}
}
// Cache only on full success so a partially-resolved domain is never reused.
st.memo[domain] = collected
return collected, nil
}
func isSPFTXT(txt string) bool {
lower := strings.ToLower(txt)
return lower == "v=spf1" || strings.HasPrefix(lower, "v=spf1 ")
}
// dedupNets returns nets with duplicate prefixes (by CIDR string) removed.
// Order of first occurrence is preserved.
func dedupNets(nets []*net.IPNet) []*net.IPNet {
if len(nets) == 0 {
return nil
}
seen := make(map[string]bool, len(nets))
out := make([]*net.IPNet, 0, len(nets))
for _, n := range nets {
key := n.String()
if !seen[key] {
seen[key] = true
out = append(out, n)
}
}
return out
}
package mailstats
import (
"cmp"
"fmt"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/antispam"
)
const (
// BucketSeconds is the bucket width. Every real zone offset is a
// multiple of 15 minutes, so the browser can build days in any zone.
BucketSeconds = 900
Retention = 30 * 24 * time.Hour
RingSize = 200
RingAge = 7 * 24 * time.Hour
scanJoinSize = 1024
bucketLimit = int(Retention / (BucketSeconds * time.Second))
)
// What a mail server's own spam filter did with a message. Unmatched is a
// verdict no delivery or reject could be paired with.
const (
FilterDelivered = "delivered"
FilterJunk = "junk"
FilterRejected = "reject"
FilterDiscarded = "discard"
FilterUnmatched = "unmatched"
filterMode = "stalwart"
)
// Span is a stretch of time.
type Span struct {
Start time.Time `json:"start"`
End time.Time `json:"end"`
}
// Totals sums a range. A dry-run tag or reject delivered the message
// unchanged, so it counts as clean and as would-tag or would-reject, never
// as tagged or rejected. A scan whose message was then rejected or
// quarantined as malware counts only as scanned. Rejects counts every
// reject per class; Temporary is the temporary subset. Delivered, Junk,
// FilterRejected, Discarded and Unmatched are mail filter verdicts, separate
// from the scan counts, including Rejected.
type Totals struct {
Scanned int `json:"scanned"`
Clean int `json:"clean"`
Tagged int `json:"tagged"`
Rejected int `json:"rejected"`
WouldTag int `json:"would_tag"`
WouldReject int `json:"would_reject"`
Unfiltered int `json:"unfiltered"`
ScanErrors int `json:"scan_errors"`
Rejects map[Class]int `json:"rejects"`
Temporary map[Class]int `json:"temporary"`
Quarantined int `json:"quarantined"`
LoginsSMTP int `json:"logins_smtp"`
LoginsMailbox int `json:"logins_mailbox"`
Delivered int `json:"delivered"`
Junk int `json:"junk"`
FilterRejected int `json:"filter_rejected"`
Discarded int `json:"discarded"`
Unmatched int `json:"unmatched"`
}
// Point is one bucket of the chart. T is the bucket start in unix seconds.
// A temporary reject defers the message rather than deciding it, so no
// series counts one. Delivered, Junk, FilterRejected and Discarded are
// mail filter verdicts, separate from the scan counts.
type Point struct {
T int64 `json:"t"`
Clean int `json:"clean"`
Tagged int `json:"tagged"`
Rejected int `json:"rejected"`
Malware int `json:"malware"`
Other int `json:"other"`
Delivered int `json:"delivered,omitempty"`
Junk int `json:"junk,omitempty"`
FilterRejected int `json:"filter_rejected,omitempty"`
Discarded int `json:"discarded,omitempty"`
}
// Overview is what the counter knows about [From, To). FilterRejectsSeen
// is set when any kept bucket, in the range or not, holds a mail filter
// reject or discard.
type Overview struct {
From time.Time `json:"from"`
To time.Time `json:"to"`
CountingSince time.Time `json:"counting_since,omitzero"`
UncountedMinutes int `json:"uncounted_minutes"`
Gaps []Span `json:"gaps"`
Totals Totals `json:"totals"`
Series []Point `json:"series"`
FilterRejectsSeen bool `json:"filter_rejects_seen"`
}
// SendersReport is the outgoing mail of [from, to) by hosting account.
// Total counts every message with a submitter, including the Unresolved
// ones whose submitter matched no account and that no row holds.
type SendersReport struct {
Accounts []AccountSends `json:"accounts"`
Total int `json:"total"`
Unresolved int `json:"unresolved"`
}
// AccountSends is the number of messages one hosting account sent.
type AccountSends struct {
Account string `json:"account"`
Messages int `json:"messages"`
}
// bucket holds the counts of one BucketSeconds window. Scans is keyed by
// mode + ":" + outcome, where outcome is the scan decision, "error", or
// "malware" once a malware reject or quarantine of the message followed,
// or by "stalwart:" + a Filter outcome for a mail filter verdict. Temporary
// is the subset of Rejects that were temporary. Sends is keyed by hosting
// account, "" for a submitter that matched none, and stays nil until the
// window has a send.
type bucket struct {
Scans map[string]int
Rejects map[Class]int
Temporary map[Class]int
LoginsSMTP int
LoginsMailbox int
Quarantined int
Sends map[string]int `json:",omitempty"`
}
// scanRef is what the join keeps of a message: the score of its successful
// scan and the bucket and Scans key that scan counted under. Key is empty
// while no scan is retained. Malware is set once a malware reject or
// quarantine of the message took its scan out of clean, or will.
type scanRef struct {
score int
bucket int64
key string
malware bool
}
// ringEntry is a reject in its class ring. Seq orders entries with the
// same time across rings by arrival.
type ringEntry struct {
Reject
Seq uint64 `json:"seq"`
}
// Counter counts mail decisions in memory by the time the log line gives.
type Counter struct {
now func() time.Time
// start is when New ran, or what SetStart set, and so the end of any
// stop Load records. Unlike since, Load never replaces it. Guarded by
// persistMu.
start time.Time
// persistMu serializes Load and Flush. A Flush that writes between
// Load's read and its merge, or two Flushes that write out of order,
// would store stale counts.
persistMu sync.Mutex
// loaded is set once Load has merged the stored counts. Until then a
// write would replace stored counts with live ones. Guarded by
// persistMu.
loaded bool
mu sync.Mutex
since time.Time
buckets map[int64]*bucket
dirty map[int64]bool // buckets changed since the last stored write
rings map[Class][]ringEntry
ringDirty map[Class]bool
seq uint64
gaps []Span // stretches CSM was stopped, by start
scans map[string]scanRef
scanIDs []string // FIFO of the keys of scans, oldest first
}
// New returns an empty counter that has counted since now().
func New(now func() time.Time) *Counter {
start := now()
return &Counter{
now: now,
start: start,
since: start,
buckets: map[int64]*bucket{},
dirty: map[int64]bool{},
rings: map[Class][]ringEntry{},
ringDirty: map[Class]bool{},
scans: map[string]scanRef{},
}
}
// SetStart sets when counting began, for a counter built before its log
// readers start, so the stop the first Load records ends there. It does
// nothing once a Load has succeeded.
func (c *Counter) SetStart(t time.Time) {
c.persistMu.Lock()
defer c.persistMu.Unlock()
if !c.loaded {
c.start = t
c.mu.Lock()
c.since = t
c.mu.Unlock()
}
}
// RecordScan counts one scan marker. A failed scan counts as "error".
func (c *Counter) RecordScan(at time.Time, s antispam.Scan) {
outcome := s.Decision
if !s.OK {
outcome = "error"
}
c.mu.Lock()
defer c.mu.Unlock()
b := c.bucketAt(at)
if b == nil {
return
}
key := s.Mode + ":" + outcome
if s.OK && s.MsgID != "" {
ref := scanRef{score: s.Score, bucket: bucketStart(at), key: key}
if prev := c.scans[s.MsgID]; prev.malware && prev.key == "" {
// Quarantined before the log watcher read this marker.
ref.key, ref.malware = s.Mode+":malware", true
}
c.rememberScan(s.MsgID, ref)
key = ref.key
}
b.Scans[key]++
}
// RecordFilterVerdict counts one message by what the mail server's spam
// filter did with it, one of the Filter outcomes. Unknown outcomes are ignored.
func (c *Counter) RecordFilterVerdict(at time.Time, outcome string) {
switch outcome {
case FilterDelivered, FilterJunk, FilterRejected, FilterDiscarded, FilterUnmatched:
default:
return
}
c.mu.Lock()
defer c.mu.Unlock()
if b := c.bucketAt(at); b != nil {
b.Scans[filterMode+":"+outcome]++
}
}
// RecordReject counts one reject and keeps it in its class ring when it is
// newer than RingAge. A spam reject whose message was scanned gets the scan
// score as its detail.
func (c *Counter) RecordReject(r Reject) {
c.mu.Lock()
defer c.mu.Unlock()
b := c.bucketAt(r.At)
if b == nil {
return
}
b.Rejects[r.Class]++
if r.Temporary {
b.Temporary[r.Class]++
} else if r.Class == ClassMalware {
c.moveToMalware(r.MsgID)
}
if r.At.Before(c.now().Add(-RingAge)) {
return
}
if ref, ok := c.scans[r.MsgID]; ok && r.Class == ClassSpam {
r.Detail = fmt.Sprintf("score %.1f", float64(ref.score)/10)
}
c.seq++
ring := c.rings[r.Class]
// Equal timestamps keep arrival order, including across classes.
entry := ringEntry{Reject: r, Seq: c.seq}
i, _ := slices.BinarySearchFunc(ring, entry, func(x, y ringEntry) int {
return cmp.Or(x.At.Compare(y.At), cmp.Compare(x.Seq, y.Seq))
})
ring = slices.Insert(ring, i, entry)
c.rings[r.Class] = slices.Delete(ring, 0, max(0, len(ring)-RingSize))
c.ringDirty[r.Class] = true
}
// RecordLoginFailure counts n failed logins, SMTP or mailbox.
func (c *Counter) RecordLoginFailure(at time.Time, mailbox bool, n int) {
if n <= 0 {
return
}
c.mu.Lock()
defer c.mu.Unlock()
b := c.bucketAt(at)
switch {
case b == nil:
case mailbox:
b.LoginsMailbox += n
default:
b.LoginsSMTP += n
}
}
// RecordQuarantine counts one message CSM quarantined. The spool watcher
// quarantines as soon as Exim accepts the message, often before the log
// watcher reads its scan marker; RecordScan then counts that scan as
// malware.
func (c *Counter) RecordQuarantine(at time.Time, msgID string) {
c.mu.Lock()
defer c.mu.Unlock()
b := c.bucketAt(at)
if b == nil {
return
}
b.Quarantined++
if _, ok := c.scans[msgID]; !ok && msgID != "" {
c.rememberScan(msgID, scanRef{malware: true})
return
}
c.moveToMalware(msgID)
}
// RecordSend counts one message a hosting account sent. An empty account
// is a submitter that matched no account.
func (c *Counter) RecordSend(at time.Time, account string) {
c.mu.Lock()
defer c.mu.Unlock()
b := c.bucketAt(at)
if b == nil {
return
}
if b.Sends == nil {
b.Sends = map[string]int{}
}
// The owner lookups return slices of a log line or a whole host file,
// and assigning to an existing key stores the new string. Copy on every
// send so a bucket never pins those buffers.
b.Sends[strings.Clone(account)]++
}
// moveToMalware recounts the scan of msgID as a malware outcome. Exim logs
// the scan marker before the antivirus rejects the message, and the spool
// watcher quarantines only accepted mail, so the scan counted the message
// as clean, tagged or rejected. A scan moves once; one the join no longer
// holds, or whose bucket is gone, stays as it is. The caller holds c.mu.
func (c *Counter) moveToMalware(msgID string) {
ref, ok := c.scans[msgID]
if !ok || ref.malware || ref.key == "" {
return
}
b := c.buckets[ref.bucket]
if b == nil || b.Scans[ref.key] == 0 {
return
}
mode, _, _ := strings.Cut(ref.key, ":")
b.Scans[ref.key]--
b.Scans[mode+":malware"]++
c.dirty[ref.bucket] = true
ref.malware = true
c.scans[msgID] = ref
}
// bucketAt returns the bucket at falls in, creating it, or nil when at is
// more than Retention old or, like LineTime, more than an hour ahead. The
// caller holds c.mu.
func (c *Counter) bucketAt(at time.Time) *bucket {
if now := c.now(); at.Before(now.Add(-Retention)) || at.Sub(now) > time.Hour {
return nil
}
k := bucketStart(at)
b := c.bucket(k)
if b != nil {
c.dirty[k] = true
}
return b
}
// bucketStart returns the start of the bucket at falls in, in unix seconds.
func bucketStart(at time.Time) int64 { return at.Unix() / BucketSeconds * BucketSeconds }
// bucket keeps the newest bucketLimit buckets even if persistence is stalled.
// A late record must not evict a newer bucket. The caller holds c.mu.
func (c *Counter) bucket(k int64) *bucket {
b := c.buckets[k]
if b == nil {
if len(c.buckets) == bucketLimit {
oldest := k
for start := range c.buckets {
oldest = min(oldest, start)
}
if oldest == k {
return nil
}
c.dropBucket(oldest)
}
b = &bucket{Scans: map[string]int{}, Rejects: map[Class]int{}, Temporary: map[Class]int{}}
c.buckets[k] = b
}
return b
}
// A late record can recreate a dropped bucket. Invalidate its scan counts
// so a later quarantine cannot subtract a different message. Keep scores
// and FIFO positions until the join evicts them. The caller holds c.mu.
func (c *Counter) dropBucket(k int64) {
delete(c.buckets, k)
delete(c.dirty, k)
for id, ref := range c.scans {
if ref.bucket == k {
ref.key = ""
c.scans[id] = ref
}
}
}
// rememberScan keeps the last scanJoinSize messages seen in a successful
// scan or a quarantine, by message ID. The caller holds c.mu.
func (c *Counter) rememberScan(msgID string, ref scanRef) {
if _, ok := c.scans[msgID]; !ok {
if len(c.scanIDs) == scanJoinSize {
delete(c.scans, c.scanIDs[0])
c.scanIDs = c.scanIDs[1:]
}
c.scanIDs = append(c.scanIDs, msgID)
}
c.scans[msgID] = ref
}
// Overview sums the buckets that start in [from, to).
func (c *Counter) Overview(from, to time.Time) Overview {
type keyed struct {
t int64
b bucket
}
var in []keyed
c.mu.Lock()
since, gaps := c.since, slices.Clone(c.gaps)
seen := false
for k, b := range c.buckets {
seen = seen || b.Scans[filterMode+":"+FilterRejected] > 0 || b.Scans[filterMode+":"+FilterDiscarded] > 0
if start := time.Unix(k, 0); !start.Before(from) && start.Before(to) {
cp := *b
cp.Scans, cp.Rejects, cp.Temporary = maps.Clone(b.Scans), maps.Clone(b.Rejects), maps.Clone(b.Temporary)
in = append(in, keyed{k, cp})
}
}
c.mu.Unlock()
slices.SortFunc(in, func(x, y keyed) int { return cmp.Compare(x.t, y.t) })
ov := Overview{From: from, To: to, CountingSince: since, FilterRejectsSeen: seen,
Totals: Totals{Rejects: map[Class]int{}, Temporary: map[Class]int{}}, Series: make([]Point, 0, len(in))}
ov.Gaps, ov.UncountedMinutes = clipGaps(gaps, from, to)
tot := &ov.Totals
for _, e := range in {
p := Point{T: e.t}
for key, n := range e.b.Scans {
mode, outcome, _ := strings.Cut(key, ":")
if mode == filterMode {
// The mail server's own filter decided these; CSM scanned none.
switch outcome {
case FilterDelivered:
p.Delivered += n
case FilterJunk:
p.Junk += n
case FilterRejected:
p.FilterRejected += n
case FilterDiscarded:
p.Discarded += n
case FilterUnmatched:
tot.Unmatched += n
}
continue
}
enforce := mode == "enforce"
tot.Scanned += n
switch {
case outcome == "error":
tot.ScanErrors += n
case outcome == "malware":
// Scanned only: the reject or quarantine counts it as malware.
case outcome == "tag" && enforce:
p.Tagged += n
case outcome == "reject" && enforce:
p.Rejected += n
default:
p.Clean += n
switch outcome {
case "unfiltered":
tot.Unfiltered += n
case "tag":
tot.WouldTag += n
case "reject":
tot.WouldReject += n
}
}
}
for class, n := range e.b.Temporary {
tot.Temporary[class] += n
}
for class, n := range e.b.Rejects {
tot.Rejects[class] += n
permanent := n - e.b.Temporary[class]
switch class {
case ClassSpam, ClassRateLimited:
case ClassMalware:
p.Malware += permanent
default:
p.Other += permanent
}
}
p.Malware += e.b.Quarantined
tot.Clean += p.Clean
tot.Tagged += p.Tagged
tot.Rejected += p.Rejected
tot.Delivered += p.Delivered
tot.Junk += p.Junk
tot.FilterRejected += p.FilterRejected
tot.Discarded += p.Discarded
tot.Quarantined += e.b.Quarantined
tot.LoginsSMTP += e.b.LoginsSMTP
tot.LoginsMailbox += e.b.LoginsMailbox
ov.Series = append(ov.Series, p)
}
return ov
}
// TopSenders sums the sends of the buckets that start in [from, to). Rows
// come most messages first, then by account; n > 0 keeps the first n rows
// and leaves the totals whole.
func (c *Counter) TopSenders(from, to time.Time, n int) SendersReport {
sums := map[string]int{}
c.mu.Lock()
for k, b := range c.buckets {
if start := time.Unix(k, 0); !start.Before(from) && start.Before(to) {
for account, m := range b.Sends {
sums[account] += m
}
}
}
c.mu.Unlock()
rep := SendersReport{Accounts: []AccountSends{}, Unresolved: sums[""]}
for account, m := range sums {
rep.Total += m
if account != "" {
rep.Accounts = append(rep.Accounts, AccountSends{account, m})
}
}
slices.SortFunc(rep.Accounts, func(x, y AccountSends) int {
return cmp.Or(cmp.Compare(y.Messages, x.Messages), strings.Compare(x.Account, y.Account))
})
if n > 0 && len(rep.Accounts) > n {
rep.Accounts = rep.Accounts[:n]
}
return rep
}
// Rejects returns up to limit ring entries of the given classes, every
// class when classes is empty, with At in [from, to) and newer than
// RingAge. The newest come first; entries with the same time come newest
// arrival first.
func (c *Counter) Rejects(from, to time.Time, classes []Class, limit int) []Reject {
if limit <= 0 {
return []Reject{}
}
var got []ringEntry
c.mu.Lock()
cutoff := c.now().Add(-RingAge)
for class, ring := range c.rings {
if len(classes) > 0 && !slices.Contains(classes, class) {
continue
}
for _, e := range ring {
if !e.At.Before(from) && e.At.Before(to) && !e.At.Before(cutoff) {
got = append(got, e)
}
}
}
c.mu.Unlock()
slices.SortFunc(got, func(x, y ringEntry) int {
return cmp.Or(y.At.Compare(x.At), cmp.Compare(y.Seq, x.Seq))
})
out := make([]Reject, 0, min(limit, len(got)))
for _, e := range got[:min(limit, len(got))] {
out = append(out, e.Reject)
}
return out
}
// Package mailstats counts mail decisions from log lines for the Email
// Security page.
package mailstats
import (
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/eximlog"
)
// LineTime reads the time a mail log line starts with: Exim's date and time
// with an optional zone, a classic syslog stamp, or RFC 3339. A time without
// a zone is read in loc. A line with no readable time, or a time more than
// an hour after fallback, gets fallback.
func LineTime(line string, loc *time.Location, fallback time.Time) time.Time {
t, ok := lineTime(line, loc, fallback)
if !ok || t.Sub(fallback) > time.Hour {
return fallback
}
return t
}
func lineTime(line string, loc *time.Location, fallback time.Time) (time.Time, bool) {
first, _, _ := strings.Cut(line, " ")
if t, err := time.Parse(time.RFC3339, first); err == nil {
return t, true
}
if t, ok := eximlog.LineTime(line, loc); ok {
return t, true
}
if len(line) >= 15 && (len(line) == 15 || line[15] == ' ') {
if t, err := time.ParseInLocation(time.Stamp, line[:15], loc); err == nil {
// A syslog stamp has no year. A stamp that lands more than a
// day ahead of fallback was written last year.
year := fallback.In(loc).Year()
t = time.Date(year, t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), 0, loc)
if t.Sub(fallback) > 24*time.Hour {
year--
}
// Parsing with the inferred year rejects February 29 in a
// non-leap year instead of normalizing it to March 1.
t, err = time.ParseInLocation("2006 "+time.Stamp, strconv.Itoa(year)+" "+line[:15], loc)
return t, err == nil
}
}
return time.Time{}, false
}
package mailstats
import (
"cmp"
"encoding/json"
"errors"
"maps"
"slices"
"time"
)
// Store keeps the counter across restarts. *store.DB implements it.
type Store interface {
PutMailStats(buckets map[int64][]byte, rejects map[string][]byte, pruneBefore int64) error
MailStats() (map[int64][]byte, map[string][]byte, error)
AddMailStatsGap(start, end int64) error
MailStatsGaps(since int64) ([][2]int64, error)
GetMetaString(key string) string
SetMetaString(key, val string) error
}
const (
metaSince = "mailstats:since"
metaFlushedAt = "mailstats:flushed_at"
)
// Load adds the stored counts to what the counter holds, since lines may
// already have been counted, restores the reject rings and gaps, and
// records the stretch from the last flush to the counter's start as a gap.
// Every stop is one: a clean shutdown flushes as it stops, and after a crash
// the lines since the last flush were lost. Ending it at the start rather
// than at now keeps a retried Load from counting time already counted. A
// corrupt entry, an unknown class or an entry out of range is skipped. A
// failed load leaves live counts unchanged and can be retried. Once a Load
// succeeds, later calls do nothing.
func (c *Counter) Load(db Store, now time.Time) error {
c.persistMu.Lock()
defer c.persistMu.Unlock()
if c.loaded {
return nil
}
stored, rings, err := db.MailStats()
if err != nil {
return err
}
pairs, err := db.MailStatsGaps(now.Add(-Retention).Unix())
if err != nil {
return err
}
gaps := make([]Span, 0, len(pairs)+1)
for _, p := range pairs {
gaps = append(gaps, Span{time.Unix(p[0], 0).UTC(), time.Unix(p[1], 0).UTC()})
}
end := c.start.UTC()
flushed, ferr := time.Parse(time.RFC3339, db.GetMetaString(metaFlushedAt))
newGap := ferr == nil && end.After(flushed)
if newGap {
// Gap storage has second precision. Round outward so a positive
// stop within one second survives both this load and later restarts.
flushed = flushed.UTC().Truncate(time.Second)
if rounded := end.Truncate(time.Second); rounded.Before(end) {
end = rounded.Add(time.Second)
}
gaps = append(gaps, Span{flushed, end})
}
since, serr := time.Parse(time.RFC3339, db.GetMetaString(metaSince))
buckets := decodeBuckets(stored, now)
restored := decodeRings(rings, now)
c.mu.Lock()
if serr != nil {
since = c.since.UTC().Truncate(time.Second)
}
c.mu.Unlock()
// Commit startup metadata before merging. On failure Flush must remain
// memory-only, and a retry must not add the stored counts twice.
var errs []error
if newGap {
errs = append(errs, db.AddMailStatsGap(flushed.Unix(), end.Unix()))
}
if serr != nil {
errs = append(errs, db.SetMetaString(metaSince, since.Format(time.RFC3339)))
}
if err := errors.Join(errs...); err != nil {
return err
}
c.mu.Lock()
c.since = since
for k, b := range buckets {
if live := c.bucket(k); live != nil {
live.add(b)
}
}
c.mergeRings(restored)
// Re-numbering changes ties across classes. Persist every restored ring,
// including ones whose entries all expired, so disk matches memory.
for class := range rings {
if slices.Contains(Classes, Class(class)) {
c.ringDirty[Class(class)] = true
}
}
c.gaps = append(c.gaps, gaps...)
slices.SortFunc(c.gaps, func(x, y Span) int { return x.Start.Compare(y.Start) })
c.mu.Unlock()
c.loaded = true
return nil
}
// Flush prunes the counter, writes the buckets and rings that changed since
// the last successful write, prunes the store past Retention and records
// the flush time. Before a Load has merged it only prunes memory, so the
// store is never overwritten with live counts alone. After a failed write
// the changes stay marked for the next Flush.
func (c *Counter) Flush(db Store, now time.Time) error {
c.persistMu.Lock()
defer c.persistMu.Unlock()
c.Prune(now)
if !c.loaded {
return nil
}
c.mu.Lock()
buckets := make(map[int64][]byte, len(c.dirty))
for k := range c.dirty {
buckets[k], _ = json.Marshal(c.buckets[k])
}
rejects := make(map[string][]byte, len(c.ringDirty))
for class := range c.ringDirty {
ring := c.rings[class]
if ring == nil {
ring = []ringEntry{}
}
rejects[string(class)], _ = json.Marshal(ring)
}
dirty, ringDirty := c.dirty, c.ringDirty
c.dirty, c.ringDirty = map[int64]bool{}, map[Class]bool{}
c.mu.Unlock()
if err := db.PutMailStats(buckets, rejects, now.Add(-Retention).Unix()); err != nil {
c.mu.Lock()
for k := range dirty {
if c.buckets[k] != nil {
c.dirty[k] = true
}
}
maps.Copy(c.ringDirty, ringDirty)
c.mu.Unlock()
return err
}
return db.SetMetaString(metaFlushedAt, now.UTC().Format(time.RFC3339Nano))
}
// Prune drops buckets past Retention, ring entries past RingAge and gaps
// that ended before Retention. Flush calls it; without a store, call it on
// a timer to keep the counter bounded.
func (c *Counter) Prune(now time.Time) {
cut, ringCut := now.Add(-Retention), now.Add(-RingAge)
c.mu.Lock()
defer c.mu.Unlock()
for k := range c.buckets {
if k < cut.Unix() {
c.dropBucket(k)
}
}
for class, ring := range c.rings {
if kept := slices.DeleteFunc(ring, func(e ringEntry) bool { return e.At.Before(ringCut) }); len(kept) < len(ring) {
c.rings[class] = kept
c.ringDirty[class] = true
}
}
c.gaps = slices.DeleteFunc(c.gaps, func(g Span) bool { return g.End.Before(cut) })
}
// decodeBuckets returns the stored buckets that decode, start on a bucket
// boundary and fall in the range the counter keeps at now.
func decodeBuckets(stored map[int64][]byte, now time.Time) map[int64]*bucket {
lo, hi := now.Add(-Retention).Unix(), now.Add(time.Hour).Unix()
out := map[int64]*bucket{}
for k, raw := range stored {
var b bucket
if k%BucketSeconds != 0 || k < lo || k > hi || json.Unmarshal(raw, &b) != nil {
continue
}
out[k] = &b
}
return out
}
// decodeRings returns the stored ring entries of known classes, in the ring
// of their own class, that are newer than RingAge at now.
func decodeRings(rings map[string][]byte, now time.Time) []ringEntry {
lo, hi := now.Add(-RingAge), now.Add(time.Hour)
var out []ringEntry
for key, raw := range rings {
class := Class(key)
var ring []ringEntry
if !slices.Contains(Classes, class) || json.Unmarshal(raw, &ring) != nil {
continue
}
for _, e := range ring {
if e.Class == class && !e.At.Before(lo) && !e.At.After(hi) {
out = append(out, e)
}
}
}
return out
}
// add adds o's counts to b.
func (b *bucket) add(o *bucket) {
for k, n := range o.Scans {
b.Scans[k] += n
}
for k, n := range o.Rejects {
b.Rejects[k] += n
}
for k, n := range o.Temporary {
b.Temporary[k] += n
}
b.LoginsSMTP += o.LoginsSMTP
b.LoginsMailbox += o.LoginsMailbox
b.Quarantined += o.Quarantined
for k, n := range o.Sends {
if b.Sends == nil {
b.Sends = map[string]int{}
}
b.Sends[k] += n
}
}
// mergeRings merges restored entries into the live rings. Restored entries
// arrived before live ones, so all are numbered again in that order; each
// class then keeps its newest RingSize by time. The caller holds c.mu.
func (c *Counter) mergeRings(restored []ringEntry) {
live := slices.Concat(slices.Collect(maps.Values(c.rings))...)
bySeq := func(x, y ringEntry) int { return cmp.Compare(x.Seq, y.Seq) }
slices.SortFunc(restored, bySeq)
slices.SortFunc(live, bySeq)
c.seq = 0
rings := map[Class][]ringEntry{}
for _, e := range slices.Concat(restored, live) {
c.seq++
e.Seq = c.seq
rings[e.Class] = append(rings[e.Class], e)
}
for class, ring := range rings {
slices.SortFunc(ring, func(x, y ringEntry) int { return cmp.Or(x.At.Compare(y.At), cmp.Compare(x.Seq, y.Seq)) })
rings[class] = ring[max(0, len(ring)-RingSize):]
}
c.rings = rings
}
// clipGaps clips gaps to [from, to), merges the ones that overlap and
// returns them with their length in minutes.
func clipGaps(gaps []Span, from, to time.Time) ([]Span, int) {
out := []Span{}
var total time.Duration
for _, g := range gaps {
if g.Start.Before(from) {
g.Start = from
}
if g.End.After(to) {
g.End = to
}
if !g.End.After(g.Start) {
continue
}
if n := len(out); n > 0 && !g.Start.After(out[n-1].End) {
if g.End.After(out[n-1].End) {
total += g.End.Sub(out[n-1].End)
out[n-1].End = g.End
}
continue
}
total += g.End.Sub(g.Start)
out = append(out, g)
}
if total > 0 {
// Even a short restart loses coverage; zero would hide it in the UI.
return out, max(1, int(total.Round(time.Minute)/time.Minute))
}
return out, 0
}
package mailstats
import (
"net"
"regexp"
"strings"
"time"
)
// postfixReplyCode matches the SMTP code and enhanced status that open a
// Postfix reject reply. A milter reply carries the status alone.
var postfixReplyCode = regexp.MustCompile(`^(?:[0-9]{3} )?(?:[0-9]\.[0-9]{1,3}\.[0-9]{1,3} )?`)
// ParsePostfixReject reads a Postfix log line for an SMTP command that
// smtpd or postscreen rejected, or a message a milter rejected. The line
// must come from one of those daemons and open with the reject keyword, so
// client text echoed into other lines is never a reject.
func ParsePostfixReject(line string, loc *time.Location, fallback time.Time) (Reject, bool) {
tagEnd := strings.Index(line, ": ")
if tagEnd < 0 || !isPostfixRejectTag(line[strings.LastIndexAny(line[:tagEnd], " \t")+1:tagEnd]) {
return Reject{}, false
}
msg := line[tagEnd+2:]
r := Reject{Class: ClassOther}
rest, smtp := strings.CutPrefix(msg, "NOQUEUE: reject: ")
if !smtp {
id, after, ok := strings.Cut(msg, ": milter-reject: ")
if !ok || !isQueueID(id) {
return Reject{}, false
}
// Before a queue file exists Postfix logs NOQUEUE in its place.
if rest = after; id != "NOQUEUE" {
r.MsgID = id
}
}
command, rest, ok := strings.Cut(rest, " from ")
if !ok || strings.Trim(command, "ABCDEFGHIJKLMNOPQRSTUVWXYZ-") != "" || command == "" {
return Reject{}, false
}
client, reply, ok := strings.Cut(rest, " ")
if !ok {
return Reject{}, false
}
if r.IP = postfixPeerIP(client); r.IP == "" {
return Reject{}, false
}
if smtp {
r.Command = command
}
// The first metadata marker bounds the reply, so envelope and HELO
// text never becomes a reason. Before MAIL, Postfix may log only the
// protocol and HELO. Multiple markers can also be an unquoted address
// posing as metadata, so that reply cannot be classified safely.
end, markers := len(reply), 0
for _, marker := range []string{"; from=<", "; to=<", "; proto=", "; helo=<"} {
if i := strings.Index(reply, marker); i >= 0 {
end = min(end, i)
markers += strings.Count(reply, marker)
}
}
r.Sender, r.Recipient = postfixEnvelope(reply[end:])
reply = reply[:end]
r.Temporary = strings.HasPrefix(reply, "4")
reason := strings.TrimPrefix(reply, postfixReplyCode.FindString(reply))
r.At = LineTime(line, loc, fallback)
r.Detail = detail(reason)
switch {
case markers > 1:
case !smtp:
r.Class = classifyMilterReject(reason, r.Temporary)
case strings.HasPrefix(reason, "<"):
r.Class = classifyPostfixNamed(reason)
case strings.HasPrefix(reason, "Service unavailable; ") && strings.Contains(reason, " blocked using "):
r.Class = ClassBlocklisted
}
return r, true
}
// isPostfixRejectTag reports a syslog tag of smtpd, postscreen or cleanup,
// under any Postfix instance or service name, with a numeric pid.
func isPostfixRejectTag(tag string) bool {
name, pid, ok := strings.Cut(strings.TrimSuffix(tag, "]"), "[")
if !ok || !strings.HasSuffix(tag, "]") || !strings.HasPrefix(name, "postfix") || pid == "" || strings.Trim(pid, "0123456789") != "" {
return false
}
switch name[strings.LastIndexByte(name, '/')+1:] {
case "smtpd", "postscreen", "cleanup":
return strings.Contains(name, "/")
}
return false
}
func isQueueID(s string) bool {
return s != "" && strings.Trim(s, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz") == ""
}
// postfixPeerIP returns the address of a "name[addr]:" or "[addr]:port:"
// client field, or "" when the field is not one.
func postfixPeerIP(client string) string {
client, ok := strings.CutSuffix(client, ":")
if !ok {
return ""
}
if i := strings.LastIndex(client, "]:"); i >= 0 && strings.Trim(client[i+2:], "0123456789") == "" {
client = client[:i+1]
}
open := strings.LastIndexByte(client, '[')
if open < 0 || !strings.HasSuffix(client, "]") {
return ""
}
ip := client[open+1 : len(client)-1]
if net.ParseIP(ip) == nil {
return ""
}
return ip
}
// postfixEnvelope accepts only a complete envelope, respecting quoted
// local parts. A marker inside an address is not a field boundary. If two
// envelopes fit, leave the addresses empty rather than display client text
// as a sender or recipient belonging to the reject.
func postfixEnvelope(s string) (sender, recipient string) {
found := false
for {
_, after, ok := strings.Cut(s, "; from=<")
if !ok {
return sender, recipient
}
s = after
n := unquotedIndex(after, '>')
if n < 0 {
continue
}
from, rest := after[:n], strings.TrimPrefix(after[n+1:], ",")
to := ""
if addr, ok := strings.CutPrefix(rest, " to=<"); ok {
n = unquotedIndex(addr, '>')
if n < 0 {
continue
}
to, rest = addr[:n], strings.TrimPrefix(addr[n+1:], ",")
}
if !strings.HasPrefix(rest, " proto=") {
continue
}
if found {
return "", ""
}
sender, recipient, found = from, to, true
}
}
// Postfix logs the rejected name unquoted. Every ">: " can therefore end
// that name, even one followed by an unfamiliar policy reason. Only agree
// on a class when every possible boundary gives the same result.
func classifyPostfixNamed(reason string) Class {
subject, rest, ok := strings.Cut(reason, ">: ")
if !ok || strings.ContainsAny(subject, " \t\r\n\"") {
return ClassOther
}
class := classifyPostfixReason(rest)
for {
_, rest, ok = strings.Cut(rest, ">: ")
if !ok {
return class
}
if classifyPostfixReason(rest) != class {
return ClassOther
}
}
}
func classifyPostfixReason(reason string) Class {
switch {
case strings.HasPrefix(reason, "Recipient address rejected: "):
tail := strings.TrimPrefix(reason, "Recipient address rejected: ")
for _, text := range []string{"User unknown", "undeliverable address", "unverified address"} {
if strings.HasPrefix(tail, text) {
return ClassUnknownRecipient
}
}
case strings.HasPrefix(reason, "Sender address rejected: "):
return ClassSenderVerify
case strings.HasPrefix(reason, "Helo command rejected: "):
return ClassBadHELO
case strings.HasPrefix(reason, "Relay access denied"):
return ClassRelayDenied
case strings.HasPrefix(reason, "Client host rejected: ") && strings.Contains(reason, "blocked using "):
return ClassBlocklisted
}
return ClassOther
}
// classifyMilterReject classes the reply a milter gave. The text is the
// filter's own: rspamd's default spam reply and its antivirus reply.
func classifyMilterReject(reason string, temporary bool) Class {
switch {
case temporary:
case reason == "Spam message rejected":
return ClassSpam
case strings.Contains(reason, ": virus found: "):
return ClassMalware
}
return ClassOther
}
package mailstats
import (
"regexp"
"slices"
"strings"
"time"
"unicode"
"github.com/pidginhost/csm/internal/eximlog"
)
// Class is the kind of reason an SMTP reject gave.
type Class string
const (
ClassSpam Class = "spam"
ClassMalware Class = "malware"
ClassUnknownRecipient Class = "unknown_recipient"
ClassSenderVerify Class = "sender_verify"
ClassBadHELO Class = "bad_helo"
ClassRelayDenied Class = "relay_denied"
ClassAuthRequired Class = "auth_required"
ClassSuspended Class = "suspended"
ClassMailboxFull Class = "mailbox_full"
ClassBlocklisted Class = "blocklisted"
ClassRateLimited Class = "rate_limited"
ClassOther Class = "other"
)
// Classes lists every class in display order.
var Classes = []Class{
ClassSpam, ClassMalware, ClassUnknownRecipient, ClassSenderVerify,
ClassBadHELO, ClassRelayDenied, ClassAuthRequired, ClassSuspended,
ClassMailboxFull, ClassBlocklisted, ClassRateLimited, ClassOther,
}
// Reject is one SMTP command an Exim ACL rejected.
type Reject struct {
At time.Time `json:"at"`
Class Class `json:"class"`
Temporary bool `json:"temporary,omitempty"`
IP string `json:"ip,omitempty"`
Sender string `json:"sender,omitempty"`
Recipient string `json:"recipient,omitempty"`
Detail string `json:"detail"`
MsgID string `json:"msg_id,omitempty"`
Command string `json:"-"` // SMTP command rejected, as RCPT; "" after DATA or at connect
}
// rejectVerb matches the verb Exim logs for an ACL reject: "rejected" or
// "temporarily rejected", then the ACL's where-name. The names are upper
// case except "after DATA", "during MIME ACL checks" and "connection", so
// reason text such as "Message rejected as spam" does not match.
var rejectVerb = regexp.MustCompile(` (temporarily )?rejected (?:[A-Z]|after |during |connection )`)
var smtpCode = regexp.MustCompile(`^[0-9]{3} (?:[0-9]\.[0-9]{1,3}\.[0-9]{1,3} )?`)
// rejectTexts are the stock reasons, prefix texts first. A prefix text
// opens the reason. A mid text, which cPanel writes after other words, can
// appear anywhere in it.
var rejectTexts = []struct {
class Class
text string
mid bool
}{
{ClassUnknownRecipient, "No Such User Here", false},
{ClassUnknownRecipient, "No such person at this address", false},
{ClassSenderVerify, "Sender verify failed", false},
{ClassSenderVerify, "Could not complete sender verify", false},
{ClassRelayDenied, "Rejected relay attempt", false},
{ClassRelayDenied, "relay not permitted", false},
{ClassAuthRequired, "SMTP AUTH is required", false},
{ClassMailboxFull, "Mailbox is full", false},
{ClassMalware, "contains a virus or other harmful content", true},
{ClassUnknownRecipient, "account or domain may not exist", true},
{ClassBadHELO, "Invalid HELO name", true},
{ClassSuspended, "has been suspended", true},
{ClassBlocklisted, "is in an RBL", true},
{ClassBlocklisted, "is listed", true},
{ClassRateLimited, "ratelimited", true},
}
// ParseReject reads an Exim log line for an SMTP command an ACL rejected.
// The verb must directly follow the peer and the fields Exim logs with it,
// so the same words in a HELO, address or reason cannot make a line a
// reject. Lines where sender text could pose as the reason get ClassOther.
func ParseReject(line string, loc *time.Location, fallback time.Time) (Reject, bool) {
h := strings.Index(line, " H=")
if h < 0 {
return Reject{}, false
}
msgID, ok := rejectPrefix(line[:h])
if !ok {
return Reject{}, false
}
peer := line[h+len(" H="):]
verbs := rejectVerb.FindAllStringSubmatchIndex(peer, -1)
if len(verbs) == 0 {
return Reject{}, false
}
boundary := rejectBoundary(peer, verbs)
if boundary < 0 {
return Reject{}, false
}
verb, fieldsEnd, sender := boundary, boundary, ""
// The sender is the last field before the verb, but for the A= field
// Exim adds for an authenticated client. A verb inside a quoted sender
// comes first, so the real one is looked for after the sender.
if strings.HasPrefix(peer[boundary:], " F=<") {
start := boundary + len(" F=<")
n := unquotedIndex(peer[start:], '>')
if n < 0 {
return Reject{}, false
}
verb = start + n + 1
if auth, ok := strings.CutPrefix(peer[verb:], " A="); ok {
sp := unquotedIndex(auth, ' ')
if sp <= 0 {
return Reject{}, false
}
verb += len(" A=") + sp
}
sender = peer[start : start+n]
}
i := slices.IndexFunc(verbs, func(m []int) bool { return m[0] == verb })
if i < 0 {
return Reject{}, false
}
v := verbs[i]
ip, end := readPeer(peer[:fieldsEnd])
if end == 0 {
return Reject{}, false
}
r := Reject{At: LineTime(line, loc, fallback), Class: ClassOther, Temporary: v[2] >= 0,
IP: ip, Sender: sender, MsgID: msgID}
after := strings.TrimPrefix(peer[v[0]:], " temporarily")[len(" rejected "):]
if word, _, _ := strings.Cut(after, " "); word[0] >= 'A' && word[0] <= 'Z' {
r.Command = word
}
cmd, addr, reason, known := wherePhrase(after)
if !known {
r.Detail = detail(after)
return r, true
}
// Remote ident is unquoted client text, a quote lets an address hold
// spaces, and a second verb means one of them is sender text.
spoofable := strings.Contains(sender+addr, `"`) || strings.Contains(peer[end:fieldsEnd], " U=") ||
slices.IndexFunc(verbs, func(m []int) bool { return m[0] >= end && m[0] != v[0] }) >= 0
switch cmd {
case "RCPT":
r.Recipient = addr
case "MAIL":
r.Sender = addr
}
senderText := peer[:len(peer)-len(reason)]
reason = strings.TrimPrefix(reason, smtpCode.FindString(reason))
r.Detail = detail(reason)
if !spoofable {
r.Class = classify(reason, r.Temporary, senderText)
}
return r, true
}
// The HELO and quoted TLS fields can contain both field and verb text.
// Locate the metadata boundary before interpreting either as Exim syntax.
func rejectBoundary(peer string, verbs [][]int) int {
depth, nextVerb := 0, 0
quoted := false
for i := 0; i < len(peer); i++ {
for nextVerb < len(verbs) && verbs[nextVerb][0] < i {
nextVerb++
}
if quoted {
switch peer[i] {
case '\\':
i++
case '"':
quoted = false
}
continue
}
if depth == 0 && (strings.HasPrefix(peer[i:], " F=<") ||
(nextVerb < len(verbs) && verbs[nextVerb][0] == i)) {
return i
}
switch peer[i] {
case '(':
depth++
case ')':
if depth == 0 {
return -1
}
depth--
case '"':
quoted = depth == 0
}
}
return -1
}
// rejectPrefix reads what Exim writes before the peer: date, time,
// optional zone and PID, and the message ID an after-DATA reject carries.
func rejectPrefix(prefix string) (msgID string, ok bool) {
if eximlog.MessagePrefix(prefix) {
f := strings.Fields(prefix)
return f[len(f)-1], true
}
// A placeholder ID lets MessagePrefix check a prefix that has none.
return "", eximlog.MessagePrefix(prefix + " 0-0-0")
}
// readPeer reads s, the H= value and the fields Exim logs after it up to
// the sender or the verb. end is the offset just past the client address,
// or 0 when s is not such a peer.
func readPeer(s string) (ip string, end int) {
ip, end = eximlog.HFieldClientIPAndEnd(s)
if end == 0 {
return "", 0
}
rest := s[end:]
if port, cut := strings.CutPrefix(rest, ":"); cut {
rest = strings.TrimLeft(port, "0123456789")
}
if after, ok := strings.CutPrefix(rest, " TFO"); ok {
rest = strings.TrimPrefix(after, "*")
}
for rest != "" {
if !fieldStart(rest) {
return "", 0
}
n := unquotedIndex(rest[1:], ' ')
if n < 0 {
break
}
rest = rest[1+n:]
}
return ip, end
}
// fieldStart reports whether s opens an Exim field: a space, a key of one
// to three letters, and "=".
func fieldStart(s string) bool {
eq := strings.IndexByte(s, '=')
if len(s) < 3 || s[0] != ' ' || eq < 2 || eq > 4 {
return false
}
for _, c := range []byte(s[1:eq]) {
if (c < 'A' || c > 'Z') && (c < 'a' || c > 'z') {
return false
}
}
return true
}
// wherePhrase splits the text after the verb into the command, its address
// and the reason. known is false for a where-name this package does not
// read.
func wherePhrase(s string) (cmd, addr, reason string, known bool) {
for _, w := range []string{"after DATA: ", `connection in "connect" ACL: `} {
if reason, ok := strings.CutPrefix(s, w); ok {
return "", "", reason, true
}
}
for _, cmd := range []string{"RCPT", "MAIL"} {
if v, ok := strings.CutPrefix(s, cmd+" <"); ok {
n := unquotedIndex(v, '>')
if n >= 0 && strings.HasPrefix(v[n+1:], ": ") {
return cmd, v[:n], v[n+len(">: "):], true
}
}
}
return "", "", "", false
}
// unquotedIndex returns the index of the first c in s outside a quoted
// string, or -1. A quoted local part can hold spaces and delimiters.
func unquotedIndex(s string, c byte) int {
quoted := false
for i := 0; i < len(s); i++ {
switch {
case quoted && s[i] == '\\':
i++
case s[i] == '"':
quoted = !quoted
case !quoted && s[i] == c:
return i
}
}
return -1
}
// classify matches a reason against the stock texts. A reason can repeat
// the HELO or an address, so a mid text that senderText, the line up to
// the reason, also holds may be the sender's and is skipped.
func classify(reason string, temporary bool, senderText string) Class {
if reason == "Message rejected as spam" {
return ClassSpam
}
senderText = strings.ToLower(senderText)
for _, t := range rejectTexts {
switch {
case t.class == ClassRateLimited && !temporary:
case !t.mid && strings.HasPrefix(reason, t.text):
return t.class
case t.mid && strings.Contains(reason, t.text) && !strings.Contains(senderText, strings.ToLower(t.text)):
return t.class
}
}
return ClassOther
}
// detail is s without control characters, cut to 120 runes.
func detail(s string) string {
s = strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return -1
}
return r
}, s)
if r := []rune(s); len(r) > 120 {
s = string(r[:120])
}
return s
}
// Package mcp serves a read-only Model Context Protocol endpoint for support
// agents. It speaks JSON-RPC 2.0 over the MCP Streamable HTTP transport for
// requests the Web UI listener has authenticated against mcp.tokens, and
// answers every tool call by dispatching a GET to an allowlisted API read
// in-process. It holds no route that changes anything.
package mcp
import (
"encoding/json"
)
// Protocol versions this server accepts, newest first. initialize answers
// the client's version when it is listed and the newest otherwise.
var protocolVersions = []string{"2025-06-18", "2025-03-26", "2024-11-05"}
// ServerName is the name sent in initialize.
const ServerName = "csm"
// JSON-RPC 2.0 error codes.
const (
codeParse = -32700
codeInvalidRequest = -32600
codeMethodNotFound = -32601
codeInvalidParams = -32602
codeInternal = -32603
)
type rpcRequest struct {
JSONRPC string `json:"jsonrpc"`
ID json.RawMessage `json:"id,omitempty"`
Method string `json:"method"`
Params json.RawMessage `json:"params,omitempty"`
}
// isNotification reports whether the request carries no id. A notification
// gets no response.
func (r rpcRequest) isNotification() bool {
return len(r.ID) == 0 || string(r.ID) == "null"
}
type rpcError struct {
Code int `json:"code"`
Message string `json:"message"`
Data any `json:"data,omitempty"`
}
type rpcResponse struct {
JSONRPC string `json:"jsonrpc"`
ID json.RawMessage `json:"id"`
Result any `json:"result,omitempty"`
Error *rpcError `json:"error,omitempty"`
}
func newResult(id json.RawMessage, result any) rpcResponse {
if len(id) == 0 {
id = json.RawMessage("null")
}
return rpcResponse{JSONRPC: "2.0", ID: id, Result: result}
}
func newError(id json.RawMessage, code int, message string) rpcResponse {
if len(id) == 0 {
id = json.RawMessage("null")
}
return rpcResponse{JSONRPC: "2.0", ID: id, Error: &rpcError{Code: code, Message: message}}
}
type initializeParams struct {
ProtocolVersion string `json:"protocolVersion"`
}
type toolsCallParams struct {
Name string `json:"name"`
Arguments json.RawMessage `json:"arguments,omitempty"`
}
// content is one MCP content block; this server sends text only.
type content struct {
Type string `json:"type"`
Text string `json:"text"`
}
type toolResult struct {
Content []content `json:"content"`
IsError bool `json:"isError,omitempty"`
}
func textResult(text string) toolResult {
return toolResult{Content: []content{{Type: "text", Text: text}}}
}
func errorResult(text string) toolResult {
return toolResult{Content: []content{{Type: "text", Text: text}}, IsError: true}
}
// toolAnnotations is the MCP hint block. Every tool here is read-only and
// idempotent; the dispatcher makes that true by construction.
type toolAnnotations struct {
Title string `json:"title,omitempty"`
ReadOnlyHint bool `json:"readOnlyHint"`
DestructiveHint bool `json:"destructiveHint"`
IdempotentHint bool `json:"idempotentHint"`
OpenWorldHint bool `json:"openWorldHint"`
}
type toolDescriptor struct {
Name string `json:"name"`
Description string `json:"description"`
InputSchema map[string]any `json:"inputSchema"`
Annotations toolAnnotations `json:"annotations"`
}
func supportedProtocol(v string) bool {
for _, p := range protocolVersions {
if p == v {
return true
}
}
return false
}
package mcp
import (
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"strings"
"time"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/obs"
)
const (
// maxBodyBytes bounds one JSON-RPC request body.
maxBodyBytes = 1 << 20
// maxConcurrentCalls bounds tool calls running at once, since several
// routes read the live host (queue listings, account walks).
maxConcurrentCalls = 4
// requestTimeout bounds one request, every call of a batch included;
// the dispatcher gets it as a context deadline. A deadline per call let
// a batch outlive the listener's write timeout.
requestTimeout = 30 * time.Second
// queueWait bounds how long a call waits for a free slot.
queueWait = 10 * time.Second
// maxBatch bounds one JSON-RPC batch.
maxBatch = 20
)
// Options wires a Server.
type Options struct {
// Dispatcher serves the allowlisted reads.
Dispatcher Dispatcher
// Version, Hostname and Platform are reported by csm_overview.
Version string
Hostname string
Platform map[string]any
// ConfigSummary answers csm_config_summary; nil makes the tool answer
// an error result.
ConfigSummary func() map[string]any
}
// Server speaks MCP over requests the Web UI listener has already
// authenticated; it holds no listener or credentials of its own.
type Server struct {
dispatcher Dispatcher
tools map[string]tool
descriptors []toolDescriptor
version string
hostname string
platform map[string]any
configSummary func() map[string]any
calls chan struct{}
}
// New builds a server for opts.
func New(opts Options) *Server {
s := &Server{
dispatcher: opts.Dispatcher,
version: opts.Version,
hostname: opts.Hostname,
platform: opts.Platform,
configSummary: opts.ConfigSummary,
calls: make(chan struct{}, maxConcurrentCalls),
}
s.tools, s.descriptors = toolTable(opts.Dispatcher)
return s
}
// Tools lists the tool names the server offers, for logs and tests.
func (s *Server) Tools() []string {
names := make([]string, 0, len(s.descriptors))
for _, d := range s.descriptors {
names = append(names, d.Name)
}
return names
}
// ServeMCP answers one POST of the Streamable HTTP transport. The caller
// has checked the method, origin and credential; caller names the
// credential for the call log.
func (s *Server) ServeMCP(w http.ResponseWriter, r *http.Request, caller string) {
if v := r.Header.Get("MCP-Protocol-Version"); v != "" && !supportedProtocol(v) {
writeError(w, http.StatusBadRequest, "unsupported MCP protocol version "+v)
return
}
body, err := io.ReadAll(io.LimitReader(r.Body, maxBodyBytes+1))
if err != nil {
writeError(w, http.StatusBadRequest, "cannot read request body")
return
}
if len(body) > maxBodyBytes {
writeError(w, http.StatusRequestEntityTooLarge, "request body too large")
return
}
requests, batch, perr := parseRequests(body)
if perr != nil {
writeJSON(w, http.StatusOK, newError(nil, perr.Code, perr.Message))
return
}
ctx, cancel := context.WithTimeout(r.Context(), requestTimeout)
defer cancel()
var responses []rpcResponse
for _, req := range requests {
resp, ok := s.dispatch(ctx, req, caller, r.RemoteAddr)
if ok {
responses = append(responses, resp)
}
}
if len(responses) == 0 {
w.WriteHeader(http.StatusAccepted)
return
}
if batch {
writeJSON(w, http.StatusOK, responses)
return
}
writeJSON(w, http.StatusOK, responses[0])
}
// parseRequests decodes one request or a batch. It answers the JSON-RPC
// error the spec names for malformed input.
func parseRequests(body []byte) ([]rpcRequest, bool, *rpcError) {
trimmed := strings.TrimSpace(string(body))
if trimmed == "" {
return nil, false, &rpcError{Code: codeInvalidRequest, Message: "empty request"}
}
if strings.HasPrefix(trimmed, "[") {
var reqs []rpcRequest
if err := json.Unmarshal(body, &reqs); err != nil {
return nil, true, &rpcError{Code: codeParse, Message: "parse error"}
}
if len(reqs) == 0 {
return nil, true, &rpcError{Code: codeInvalidRequest, Message: "empty batch"}
}
if len(reqs) > maxBatch {
return nil, true, &rpcError{Code: codeInvalidRequest, Message: fmt.Sprintf("batch larger than %d", maxBatch)}
}
return reqs, true, nil
}
var req rpcRequest
if err := json.Unmarshal(body, &req); err != nil {
return nil, false, &rpcError{Code: codeParse, Message: "parse error"}
}
return []rpcRequest{req}, false, nil
}
// dispatch answers one JSON-RPC request. ok is false for a notification.
func (s *Server) dispatch(ctx context.Context, req rpcRequest, tokenName, peer string) (rpcResponse, bool) {
if req.JSONRPC != "2.0" || req.Method == "" {
if req.isNotification() {
return rpcResponse{}, false
}
return newError(req.ID, codeInvalidRequest, "invalid request"), true
}
if strings.HasPrefix(req.Method, "notifications/") {
return rpcResponse{}, false
}
if req.isNotification() {
// A request without an id gets no answer under JSON-RPC 2.0.
return rpcResponse{}, false
}
switch req.Method {
case "initialize":
var p initializeParams
if len(req.Params) > 0 {
if err := json.Unmarshal(req.Params, &p); err != nil {
return newError(req.ID, codeInvalidParams, "invalid initialize params"), true
}
}
version := protocolVersions[0]
if supportedProtocol(p.ProtocolVersion) {
version = p.ProtocolVersion
}
return newResult(req.ID, map[string]any{
"protocolVersion": version,
"capabilities": map[string]any{"tools": map[string]any{"listChanged": false}},
"serverInfo": map[string]any{"name": ServerName, "version": s.version},
"instructions": instructions,
}), true
case "ping":
return newResult(req.ID, map[string]any{}), true
case "tools/list":
return newResult(req.ID, map[string]any{"tools": s.descriptors}), true
case "tools/call":
var p toolsCallParams
if err := json.Unmarshal(req.Params, &p); err != nil || p.Name == "" {
return newError(req.ID, codeInvalidParams, "tools/call needs a name"), true
}
if _, ok := s.tools[p.Name]; !ok {
return newError(req.ID, codeInvalidParams, "unknown tool "+p.Name), true
}
result, err := s.runCall(ctx, p, tokenName, peer)
if err != nil {
return newError(req.ID, codeInvalidParams, err.Error()), true
}
return newResult(req.ID, result), true
case "resources/list":
return newResult(req.ID, map[string]any{"resources": []any{}}), true
case "resources/templates/list":
return newResult(req.ID, map[string]any{"resourceTemplates": []any{}}), true
case "prompts/list":
return newResult(req.ID, map[string]any{"prompts": []any{}}), true
}
return newError(req.ID, codeMethodNotFound, "method not found: "+req.Method), true
}
// instructions is sent on initialize so a client model knows the surface.
const instructions = "Read-only view of the CSM security monitor on this server. " +
"Every tool reads; none can block, unblock, fix, scan or change settings. " +
"Start with csm_overview, use csm_lookup_ip for an address and csm_account for a hosting account. " +
"Lists answer {items, total, truncated}; page with the limit, offset or page arguments a tool declares."
// runCall runs one tool call under the concurrency cap and the request's
// deadline, and logs it.
func (s *Server) runCall(ctx context.Context, p toolsCallParams, tokenName, peer string) (toolResult, error) {
if err := ctx.Err(); err != nil {
return errorResult(err.Error()), nil
}
select {
case s.calls <- struct{}{}:
case <-ctx.Done():
return errorResult("request cancelled"), nil
case <-time.After(queueWait):
return errorResult("server busy; retry shortly"), nil
}
// A free slot and cancellation can become ready together. Do not start
// another batch member after the request has expired.
if err := ctx.Err(); err != nil {
<-s.calls
return errorResult(err.Error()), nil
}
type answer struct {
result toolResult
err error
}
done := make(chan answer, 1)
obs.SafeGo("mcp", func() {
// Some live-host reads cannot observe cancellation. Keep their slot
// until they finish so repeated timeouts cannot spawn unlimited work.
out := answer{result: errorResult("tool call failed")}
defer func() {
<-s.calls
done <- out
}()
out.result, out.err = s.loggedCall(ctx, p, tokenName, peer)
})
select {
case <-ctx.Done():
return errorResult("request cancelled"), nil
case out := <-done:
if err := ctx.Err(); err != nil {
return errorResult(err.Error()), nil
}
return out.result, out.err
}
}
func (s *Server) loggedCall(ctx context.Context, p toolsCallParams, tokenName, peer string) (toolResult, error) {
start := time.Now()
result, status, err := s.callTool(ctx, p.Name, p.Arguments, peer)
outcome := "ok"
switch {
case err != nil:
outcome = "invalid"
case result.IsError:
outcome = "error"
}
csmlog.Info("mcp tool call", "token", tokenName, "tool", p.Name, "peer", peerForLog(peer),
"status", status, "outcome", outcome, "duration_ms", time.Since(start).Milliseconds(),
"args", argsForLog(p.Arguments))
return result, err
}
func peerForLog(peer string) string {
if host, _, err := net.SplitHostPort(peer); err == nil {
return host
}
return peer
}
// argsForLog keeps the call arguments in the log, cut to a line.
func argsForLog(raw json.RawMessage) string {
s := strings.TrimSpace(string(raw))
if len(s) > 200 {
s = s[:200] + "…"
}
return s
}
func writeJSON(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
enc := json.NewEncoder(w)
enc.SetEscapeHTML(false)
_ = enc.Encode(v)
}
func writeError(w http.ResponseWriter, status int, msg string) {
writeJSON(w, status, map[string]string{"error": msg})
}
package mcp
import (
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
)
// PlatformFacts is what csm_overview reports about the host.
func PlatformFacts(info platform.Info) map[string]any {
return map[string]any{
"os": string(info.OS),
"os_version": info.OSVersion,
"panel": string(info.Panel),
"web_server": string(info.WebServer),
"mta": string(platform.DetectMTA()),
}
}
// ConfigSummary returns the non-secret configuration facts a support agent
// needs. It names each field it reads; it never walks the config struct, so
// a secret added to the configuration later cannot leak through it.
func ConfigSummary(cfg *config.Config, info platform.Info) map[string]any {
fw := map[string]any{"enabled": false}
if cfg.Firewall != nil {
fw = map[string]any{
"enabled": cfg.Firewall.Enabled,
"ipv6": cfg.Firewall.IPv6,
"tcp_in": ints(cfg.Firewall.TCPIn),
"udp_in": ints(cfg.Firewall.UDPIn),
"restricted_tcp": ints(cfg.Firewall.RestrictedTCP),
}
}
t := cfg.Thresholds
return map[string]any{
"hostname": cfg.Hostname,
"mode": cfg.Mode,
"observe_mode": cfg.ObserveMode(),
"state_path": cfg.StatePath,
"platform": PlatformFacts(info),
"webui_listen": cfg.WebUI.Listen,
"infra_ips": strs(cfg.InfraIPs),
"account_roots": strs(cfg.AccountRoots),
"disabled_checks": strs(cfg.DisabledChecks),
"mail_logs": map[string]any{"source": cfg.MailLogs.Source},
"firewall": fw,
"auto_response": map[string]any{
"enabled": cfg.AutoResponse.Enabled,
"dry_run": cfg.AutoResponseDryRunEnabled(),
"block_expiry": cfg.AutoResponse.BlockExpiry,
"max_blocks_per_hour": cfg.AutoResponse.MaxBlocksPerHour,
},
"protections": map[string]any{
"bpf_enforcement": cfg.BPFEnforcement.Enabled,
"challenge_pages": cfg.Challenge.Enabled,
"php_shield": cfg.PHPShield.Enabled,
"email_av": cfg.EmailAV.Enabled,
"forward_guard": cfg.EmailProtection.ForwardGuard.Enabled,
"antispam": cfg.EmailProtection.Antispam.Enabled,
"antispam_dry_run": cfg.EmailProtection.Antispam.DryRun,
"cloudflare": cfg.Cloudflare.Enabled,
"retention_sweeps": cfg.Retention.Enabled,
},
"signatures": map[string]any{
"rules_dir": cfg.Signatures.RulesDir,
"auto_update": cfg.Signatures.AutoUpdate,
},
"thresholds": map[string]any{
"smtp_bruteforce_threshold": t.SMTPBruteForceThreshold,
"smtp_bruteforce_window_min": t.SMTPBruteForceWindowMin,
"smtp_bruteforce_subnet_threshold": t.SMTPBruteForceSubnetThresh,
"mail_bruteforce_threshold": t.MailBruteForceThreshold,
"mail_bruteforce_window_min": t.MailBruteForceWindowMin,
"mail_bruteforce_subnet_threshold": t.MailBruteForceSubnetThresh,
"pam_bruteforce_threshold": t.PAMBruteforceThreshold,
"pam_bruteforce_window_min": t.PAMBruteforceWindowMin,
"multi_ip_login_threshold": t.MultiIPLoginThreshold,
"cred_stuffing_distinct_accounts": t.CredStuffingDistinctAccounts,
"mail_queue_warn": t.MailQueueWarn,
"mail_queue_crit": t.MailQueueCrit,
"deep_scan_interval_min": t.DeepScanIntervalMin,
},
}
}
func ints(v []int) []int {
if v == nil {
return []int{}
}
return append([]int(nil), v...)
}
func strs(v []string) []string {
if v == nil {
return []string{}
}
return append([]string(nil), v...)
}
package mcp
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
)
// Dispatcher serves one allowlisted API read in-process. webui.Server's
// ReadOnly dispatcher is the implementation; it always sends GET and
// refuses every path outside its table with an error.
type Dispatcher interface {
Dispatch(ctx context.Context, path string, query url.Values, peer string) (status int, contentType string, body []byte, err error)
Routes() []string
}
// param describes one query parameter a tool accepts. Values are passed to
// the route as given; the route validates them as it does for any client.
type param struct {
name, kind, desc string
required bool
}
var (
pLimit = param{"limit", "integer", "Maximum number of items to return.", false}
pOffset = param{"offset", "integer", "Number of items to skip (paging).", false}
pFrom = param{"from", "string", "Start of the range: a calendar date YYYY-MM-DD (server time zone) or an RFC 3339 instant.", false}
pTo = param{"to", "string", "End of the range: a calendar date (inclusive) or an RFC 3339 instant (exclusive).", false}
pSeverity = param{"severity", "string", "Severity filter: CRITICAL, HIGH or WARNING.", false}
pSearch = param{"search", "string", "Free-text filter on the message.", false}
pChecks = param{"checks", "string", "Comma-separated check names to include.", false}
pIP = param{"ip", "string", "IPv4 or IPv6 address.", true}
pAccount = param{"account", "string", "Hosting account (system user) name.", false}
pHours = param{"hours", "integer", "Window in hours.", false}
pPage = param{"page", "integer", "Page number, from 1 (paging).", false}
pWindow = param{"window", "string", "Time window: 1h, 6h or 24h (default).", false}
// pMinSeverity is a floor, unlike pSeverity's exact match.
pMinSeverity = param{"severity", "string", "Minimum severity: WARNING, HIGH or CRITICAL.", false}
)
// tool is one MCP tool backed by a route, or by a composite function that
// calls several routes through the same dispatcher.
type tool struct {
name, title, desc string
route string
params []param
composite func(ctx context.Context, s *Server, args map[string]string, peer string) (any, error)
}
// routeTools is the one-tool-per-route table. Names say what the tool reads;
// descriptions name the route so a reader can cross-check docs/src/api.md.
var routeTools = []tool{
{name: "csm_status", title: "Daemon status", desc: "Full health snapshot of the CSM daemon: version, uptime, watchers, severity counts, store and firewall health, queues, admission ledger, security posture. GET /api/v1/status.", route: "/api/v1/status"},
{name: "csm_health", title: "Daemon health", desc: "Daemon liveness: fanotify, log watchers, loaded rules, uptime. GET /api/v1/health.", route: "/api/v1/health"},
{name: "csm_capabilities", title: "Build capabilities", desc: "Feature tokens this build and host support. GET /api/v1/capabilities.", route: "/api/v1/capabilities"},
{name: "csm_components", title: "Component matrix", desc: "Watcher and component attachment, event and upstream freshness state. GET /api/v1/components.", route: "/api/v1/components"},
{name: "csm_challenge_stats", title: "Challenge routing stats", desc: "Challenge-page routing activity: pending, escalated, routed by check, recent routes. GET /api/v1/challenge/stats.", route: "/api/v1/challenge/stats"},
{name: "csm_scan_jobs", title: "Scan jobs", desc: "Queued, running and finished full-scan jobs. GET /api/v1/scan-jobs.", route: "/api/v1/scan-jobs"},
{name: "csm_scan_job", title: "Scan job detail", desc: "One scan job by id. GET /api/v1/scan-jobs/{id}.", route: "/api/v1/scan-jobs/{id}", params: []param{{"id", "string", "Scan job id.", true}}},
{name: "csm_findings", title: "Active findings", desc: "Current active findings. GET /api/v1/findings.", route: "/api/v1/findings"},
{name: "csm_findings_enriched", title: "Enriched findings", desc: "Active findings with GeoIP, account, fix availability and the evidence address auto-block acts on. Ordered by severity then newest when limit is set. GET /api/v1/findings/enriched.", route: "/api/v1/findings/enriched", params: []param{pLimit}},
{name: "csm_finding_detail", title: "Finding detail", desc: "One finding with first/last seen, related audit actions and related history. GET /api/v1/finding-detail.", route: "/api/v1/finding-detail", params: []param{{"check", "string", "Check name of the finding.", true}, {"message", "string", "Finding message.", false}, {"key", "string", "Finding key, when known.", false}}},
{name: "csm_history", title: "Finding history", desc: "Paginated finding history with filters. GET /api/v1/history.", route: "/api/v1/history", params: []param{pLimit, pOffset, pFrom, pTo, pSeverity, pSearch, pChecks, {"scope", "string", "email limits results to the email checks.", false}}},
{name: "csm_stats", title: "24h stats", desc: "24h severity counts, auto-response summary and accounts with active high or critical findings. GET /api/v1/stats.", route: "/api/v1/stats"},
{name: "csm_stats_trend", title: "30-day trend", desc: "Daily severity counts for the last 30 days. GET /api/v1/stats/trend.", route: "/api/v1/stats/trend", params: []param{{"days", "integer", "Number of days, 30 by default.", false}}},
{name: "csm_stats_timeline", title: "24h timeline", desc: "Hourly severity counts for the last 24 hours. GET /api/v1/stats/timeline.", route: "/api/v1/stats/timeline"},
{name: "csm_blocked_ips", title: "Blocked IPs", desc: "Addresses CSM blocked, with reason, origin and expiry. GET /api/v1/blocked-ips.", route: "/api/v1/blocked-ips"},
{name: "csm_quarantine", title: "Quarantined files", desc: "Quarantined files with metadata. GET /api/v1/quarantine.", route: "/api/v1/quarantine"},
{name: "csm_db_object_backups", title: "DB object backups", desc: "MySQL triggers, events, procedures and functions CSM removed and backed up. GET /api/v1/db-object-backups.", route: "/api/v1/db-object-backups"},
{name: "csm_accounts", title: "Hosting accounts", desc: "Accounts a server-wide scan covers. GET /api/v1/accounts.", route: "/api/v1/accounts"},
{name: "csm_account", title: "Account detail", desc: "One account's findings, quarantine and history. GET /api/v1/account.", route: "/api/v1/account", params: []param{{"name", "string", "Account (system user) name.", true}}},
{name: "csm_audit", title: "UI audit log", desc: "Newest operator actions taken through the Web UI and API, with actor. GET /api/v1/audit.", route: "/api/v1/audit"},
{name: "csm_incident_timeline", title: "Incident timeline", desc: "Timeline of findings and actions for an address or account over a window. GET /api/v1/incident.", route: "/api/v1/incident", params: []param{{"ip", "string", "Address to follow.", false}, pAccount, pHours}},
{name: "csm_incidents", title: "Incidents", desc: "Correlated incidents (open, contained, resolved). GET /api/v1/incidents.", route: "/api/v1/incidents", params: []param{pLimit, pOffset, {"status", "string", "Status filter: open, contained, resolved, dismissed, or active for open and contained.", false}}},
{name: "csm_incident_groups", title: "Incident groups", desc: "Incidents grouped for the dashboard. GET /api/v1/incidents/groups.", route: "/api/v1/incidents/groups"},
{name: "csm_incident", title: "Incident detail", desc: "One incident by id. GET /api/v1/incidents/{id}.", route: "/api/v1/incidents/{id}", params: []param{{"id", "string", "Incident id.", true}}},
{name: "csm_email_stats", title: "Mail stats", desc: "Mail queue size and mail protection counters. Runs the MTA queue count. GET /api/v1/email/stats.", route: "/api/v1/email/stats"},
{name: "csm_email_quarantine", title: "Mail quarantine", desc: "Messages the email antivirus quarantined. GET /api/v1/email/quarantine.", route: "/api/v1/email/quarantine"},
{name: "csm_email_av_status", title: "Email AV status", desc: "Email antivirus engine and watcher state. GET /api/v1/email/av/status.", route: "/api/v1/email/av/status"},
{name: "csm_email_checks", title: "Email check names", desc: "Check names that belong to mail protection. GET /api/v1/email/checks.", route: "/api/v1/email/checks"},
{name: "csm_email_groups", title: "Mail finding groups", desc: "Mail findings grouped by mailbox and kind over a range. GET /api/v1/email/groups.", route: "/api/v1/email/groups", params: []param{pFrom, pTo, pLimit}},
{name: "csm_email_overview", title: "Mail overview", desc: "Mail protection overview over a range. GET /api/v1/email/overview.", route: "/api/v1/email/overview", params: []param{pFrom, pTo}},
{name: "csm_email_rejects", title: "Mail rejects", desc: "Rejected inbound mail over a range. GET /api/v1/email/rejects.", route: "/api/v1/email/rejects", params: []param{pFrom, pTo, pLimit, {"class", "string", "Reject classes: content (default), all, or class names joined by commas.", false}}},
{name: "csm_email_senders", title: "Mail senders", desc: "Outbound sender profiles over a range. GET /api/v1/email/senders.", route: "/api/v1/email/senders", params: []param{pFrom, pTo}},
{name: "csm_email_rspamd", title: "rspamd stats", desc: "Antispam engine statistics from rspamd. GET /api/v1/email/rspamd.", route: "/api/v1/email/rspamd"},
{name: "csm_email_relay_abuse", title: "Relay abuse", desc: "Authenticated relay abuse over a range. GET /api/v1/email/relay-abuse.", route: "/api/v1/email/relay-abuse", params: []param{pFrom, pTo, pLimit}},
{name: "csm_email_forwarders", title: "Mail forwarders", desc: "Forwarders and filters audit with summary. GET /api/v1/email/forwarders.", route: "/api/v1/email/forwarders"},
{name: "csm_email_deferrals", title: "Mail deferrals", desc: "Deferred delivery summary. GET /api/v1/email/deferrals.", route: "/api/v1/email/deferrals"},
{name: "csm_email_queue_composition", title: "Mail queue composition", desc: "What the mail queue holds, by sender and class. GET /api/v1/email/queue-composition.", route: "/api/v1/email/queue-composition"},
{name: "csm_email_queue", title: "Mail queue", desc: "Messages in the MTA queue. Reads the live queue. GET /api/v1/email/queue.", route: "/api/v1/email/queue", params: []param{{"filter", "string", "Queue filter: all (default), frozen or backscatter.", false}, {"q", "string", "Text filter on the listed messages.", false}, pPage, pLimit}},
{name: "csm_email_held", title: "Held forward copies", desc: "Forward copies the forward guard is holding. GET /api/v1/email/held.", route: "/api/v1/email/held"},
{name: "csm_performance", title: "Performance", desc: "Performance findings and a host resource sample. GET /api/v1/performance.", route: "/api/v1/performance"},
{name: "csm_hardening", title: "Hardening report", desc: "Last stored hardening audit report. GET /api/v1/hardening.", route: "/api/v1/hardening"},
{name: "csm_threat_stats", title: "Threat stats", desc: "Attack database totals. GET /api/v1/threat/stats.", route: "/api/v1/threat/stats"},
{name: "csm_threat_top_attackers", title: "Top attackers", desc: "Most active attacking addresses. GET /api/v1/threat/top-attackers.", route: "/api/v1/threat/top-attackers", params: []param{pLimit, {"spam_only", "integer", "1 ranks spam senders by spam score instead of attackers.", false}}},
{name: "csm_threat_ip", title: "Threat record for IP", desc: "Everything CSM recorded about one address: attacks, reputation, block state. GET /api/v1/threat/ip.", route: "/api/v1/threat/ip", params: []param{pIP}},
{name: "csm_threat_events", title: "Attack events", desc: "Recent attack events for one address. GET /api/v1/threat/events.", route: "/api/v1/threat/events", params: []param{pIP, pLimit}},
{name: "csm_threat_db_stats", title: "Threat DB stats", desc: "Threat database sizes and freshness. GET /api/v1/threat/db-stats.", route: "/api/v1/threat/db-stats"},
{name: "csm_threat_whitelist", title: "Whitelist", desc: "Addresses on the runtime whitelist. GET /api/v1/threat/whitelist.", route: "/api/v1/threat/whitelist"},
{name: "csm_rules_status", title: "Rules status", desc: "Signature rule set status and update state. GET /api/v1/rules/status.", route: "/api/v1/rules/status"},
{name: "csm_rules_list", title: "Rules list", desc: "Loaded signature rules. GET /api/v1/rules/list.", route: "/api/v1/rules/list"},
{name: "csm_modsec_stats", title: "ModSecurity stats", desc: "WAF event totals. GET /api/v1/modsec/stats.", route: "/api/v1/modsec/stats", params: []param{pWindow, pMinSeverity}},
{name: "csm_modsec_blocks", title: "ModSecurity blocks", desc: "Addresses blocked from WAF correlation. GET /api/v1/modsec/blocks.", route: "/api/v1/modsec/blocks", params: []param{pWindow, pMinSeverity}},
{name: "csm_modsec_events", title: "ModSecurity events", desc: "Recent WAF events. GET /api/v1/modsec/events.", route: "/api/v1/modsec/events", params: []param{pLimit, pWindow, pMinSeverity}},
{name: "csm_modsec_rules", title: "ModSecurity rules", desc: "Bundled WAF rule configuration and per-domain coverage. GET /api/v1/modsec/rules.", route: "/api/v1/modsec/rules"},
{name: "csm_firewall_status", title: "Firewall status", desc: "nftables engine state: enabled, managed, sets, counts. GET /api/v1/firewall/status.", route: "/api/v1/firewall/status"},
{name: "csm_firewall_allowed", title: "Firewall allow list", desc: "Addresses allowed through the firewall. GET /api/v1/firewall/allowed.", route: "/api/v1/firewall/allowed"},
{name: "csm_firewall_audit", title: "Firewall audit", desc: "Firewall change audit log. GET /api/v1/firewall/audit.", route: "/api/v1/firewall/audit", params: []param{pLimit, {"action", "string", "Action filter, for example block or unblock.", false}, {"source", "string", "Source filter: web_ui, cli, auto_response, challenge, whitelist, dyndns or system.", false}, pSearch}},
{name: "csm_firewall_subnets", title: "Firewall subnets", desc: "Denied subnets. GET /api/v1/firewall/subnets.", route: "/api/v1/firewall/subnets"},
{name: "csm_firewall_check", title: "Firewall check IP", desc: "Whether one address is blocked, by what and until when, including cPHulk. GET /api/v1/firewall/check.", route: "/api/v1/firewall/check", params: []param{pIP}},
{name: "csm_firewall_rollback", title: "Firewall rollback", desc: "Pending timed firewall configuration rollback, if any. GET /api/v1/settings/firewall/rollback.", route: "/api/v1/settings/firewall/rollback"},
{name: "csm_geoip", title: "GeoIP lookup", desc: "Country, ASN and organisation of an address from the local GeoIP database. GET /api/v1/geoip.", route: "/api/v1/geoip", params: []param{pIP}},
}
// compositeTools answer a support question in one call by reading several
// routes through the same dispatcher.
var compositeTools = []tool{
{name: "csm_overview", title: "Server overview", desc: "One-call picture of this server: hostname, platform, CSM version and mode, daemon status, health and capabilities.", composite: overview},
{name: "csm_lookup_ip", title: "Look up an IP", desc: "Everything about one address in one call: firewall block state, blocked-list entry, threat record and GeoIP.", params: []param{pIP}, composite: lookupIP},
{name: "csm_config_summary", title: "Configuration summary", desc: "Non-secret configuration facts: mode, platform, listeners, which protections are enabled, thresholds that explain blocks. Secrets are never read.", composite: configSummary},
}
// toolTable builds the name -> tool map and the descriptors sent by
// tools/list, keeping only route tools the dispatcher serves.
func toolTable(d Dispatcher) (map[string]tool, []toolDescriptor) {
served := map[string]bool{}
if d != nil {
for _, r := range d.Routes() {
served[r] = true
}
}
byName := map[string]tool{}
var descriptors []toolDescriptor
add := func(t tool) {
byName[t.name] = t
descriptors = append(descriptors, toolDescriptor{
Name: t.name,
Description: t.desc,
InputSchema: inputSchema(t.params),
Annotations: toolAnnotations{Title: t.title, ReadOnlyHint: true, IdempotentHint: true},
})
}
for _, t := range routeTools {
if !served[t.route] {
continue
}
add(t)
}
for _, t := range compositeTools {
add(t)
}
sort.Slice(descriptors, func(i, j int) bool { return descriptors[i].Name < descriptors[j].Name })
return byName, descriptors
}
func inputSchema(params []param) map[string]any {
props := map[string]any{}
var required []string
for _, p := range params {
props[p.name] = map[string]any{"type": p.kind, "description": p.desc}
if p.required {
required = append(required, p.name)
}
}
schema := map[string]any{"type": "object", "properties": props, "additionalProperties": false}
if len(required) > 0 {
schema["required"] = required
}
return schema
}
// decodeArguments turns the JSON arguments into strings the routes read as
// query values. Only declared parameters are accepted; a number is
// formatted as the route expects.
func decodeArguments(t tool, raw json.RawMessage) (map[string]string, error) {
args := map[string]string{}
if len(raw) == 0 || string(raw) == "null" {
raw = json.RawMessage("{}")
}
var given map[string]any
if err := json.Unmarshal(raw, &given); err != nil {
return nil, fmt.Errorf("arguments must be an object: %v", err)
}
declared := map[string]param{}
for _, p := range t.params {
declared[p.name] = p
}
for name, v := range given {
p, ok := declared[name]
if !ok {
return nil, fmt.Errorf("unknown argument %q", name)
}
switch val := v.(type) {
case nil:
continue
case string:
if len(val) > 1024 {
return nil, fmt.Errorf("argument %q is longer than 1024 characters", name)
}
args[name] = val
case float64:
if p.kind == "integer" {
args[name] = strconv.FormatInt(int64(val), 10)
} else {
args[name] = strconv.FormatFloat(val, 'f', -1, 64)
}
case bool:
args[name] = strconv.FormatBool(val)
default:
return nil, fmt.Errorf("argument %q must be a scalar", name)
}
}
for _, p := range t.params {
if p.required && args[p.name] == "" {
return nil, fmt.Errorf("argument %q is required", p.name)
}
}
return args, nil
}
// errRouteFailed carries the API's own error message for a non-2xx answer.
type errRouteFailed struct {
status int
message string
}
func (e *errRouteFailed) Error() string {
return fmt.Sprintf("%s (HTTP %d)", e.message, e.status)
}
// read dispatches one route and decodes its JSON answer. A non-2xx status
// becomes errRouteFailed with the API's error text; a cut response is an
// error asking the caller to narrow the query.
func (s *Server) read(ctx context.Context, route string, args map[string]string, peer string) (json.RawMessage, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
path := route
query := url.Values{}
for name, val := range args {
if name == "id" && strings.Contains(route, "{id}") {
if val == "" || strings.ContainsAny(val, "/?#%") {
return nil, errors.New("id must be a single path segment")
}
path = strings.Replace(route, "{id}", url.PathEscape(val), 1)
continue
}
query.Set(name, val)
}
status, ctype, body, err := s.dispatcher.Dispatch(ctx, path, query, peer)
if err != nil {
if isTruncated(err) {
return nil, fmt.Errorf("response too large for one call; narrow it with limit, offset or a filter")
}
return nil, err
}
if status < 200 || status > 299 {
return nil, &errRouteFailed{status: status, message: apiErrorMessage(body, status)}
}
if !strings.HasPrefix(ctype, "application/json") || !json.Valid(body) {
return nil, fmt.Errorf("route %s answered %q, not JSON", route, ctype)
}
return json.RawMessage(body), nil
}
// isTruncated recognises a dispatcher size cut by the Truncated method the
// webui error carries, without importing that package.
func isTruncated(err error) bool {
var t interface{ Truncated() bool }
return errors.As(err, &t) && t.Truncated()
}
func apiErrorMessage(body []byte, status int) string {
var e struct {
Error string `json:"error"`
}
if json.Unmarshal(body, &e) == nil && e.Error != "" {
return e.Error
}
if text := http.StatusText(status); text != "" {
return text
}
return "request failed"
}
// callTool runs one tool and returns the MCP result. Route failures are
// tool errors (isError), never JSON-RPC errors, so the model can read them.
func (s *Server) callTool(ctx context.Context, name string, raw json.RawMessage, peer string) (toolResult, int, error) {
t, ok := s.tools[name]
if !ok {
return toolResult{}, 0, fmt.Errorf("unknown tool %q", name)
}
args, err := decodeArguments(t, raw)
if err != nil {
return toolResult{}, 0, err
}
var out any
if t.composite != nil {
out, err = t.composite(ctx, s, args, peer)
} else {
out, err = s.read(ctx, t.route, args, peer)
}
if err != nil {
var rf *errRouteFailed
if errors.As(err, &rf) {
return errorResult(rf.message), rf.status, nil
}
return errorResult(err.Error()), 0, nil
}
text, err := json.Marshal(out)
if err != nil {
return toolResult{}, 0, err
}
return textResult(string(text)), http.StatusOK, nil
}
// overview reads status, health and capabilities and adds host facts.
func overview(ctx context.Context, s *Server, _ map[string]string, peer string) (any, error) {
out := map[string]any{
"hostname": s.hostname,
"version": s.version,
"platform": s.platform,
}
for key, route := range map[string]string{
"status": "/api/v1/status",
"health": "/api/v1/health",
"capabilities": "/api/v1/capabilities",
} {
body, err := s.read(ctx, route, nil, peer)
if err != nil {
out[key+"_error"] = err.Error()
continue
}
out[key] = body
}
return out, nil
}
// lookupIP reads the four address routes; a route that fails reports its
// error next to the others instead of failing the whole lookup.
func lookupIP(ctx context.Context, s *Server, args map[string]string, peer string) (any, error) {
ip := args["ip"]
out := map[string]any{"ip": ip}
for key, route := range map[string]string{
"firewall": "/api/v1/firewall/check",
"threat": "/api/v1/threat/ip",
"geoip": "/api/v1/geoip",
} {
body, err := s.read(ctx, route, map[string]string{"ip": ip}, peer)
if err != nil {
out[key+"_error"] = err.Error()
continue
}
out[key] = body
}
if body, err := s.read(ctx, "/api/v1/blocked-ips", nil, peer); err != nil {
out["blocked_error"] = err.Error()
} else {
out["blocked"] = blockedEntryFor(body, ip)
}
return out, nil
}
// blockedEntryFor picks the blocked-ips item for ip, or null.
func blockedEntryFor(body json.RawMessage, ip string) any {
var list struct {
Items []map[string]any `json:"items"`
}
if json.Unmarshal(body, &list) != nil {
return nil
}
for _, item := range list.Items {
if v, _ := item["ip"].(string); v == ip {
return item
}
}
return nil
}
func configSummary(_ context.Context, s *Server, _ map[string]string, _ string) (any, error) {
if s.configSummary == nil {
return nil, errors.New("configuration summary not available")
}
return s.configSummary(), nil
}
// Package metrics is CSM's local OpenMetrics implementation. It exists
// so the daemon can expose a `/metrics` endpoint (ROADMAP item 4)
// without pulling in `github.com/prometheus/client_golang`, which
// would add ~20 transitive dependencies for the handful of counters,
// gauges, and histograms this project actually needs.
//
// The surface is intentionally narrow: Counter, Gauge, Histogram, and
// their labelled vector siblings. No summaries, no collectors, no
// custom exposition formats. Metric objects are safe for concurrent
// use; registration is idempotent.
package metrics
import (
"errors"
"fmt"
"io"
"math"
"sort"
"strings"
"sync"
"sync/atomic"
)
// metricType discriminates the OpenMetrics TYPE line.
type metricType string
const (
typeCounter metricType = "counter"
typeGauge metricType = "gauge"
typeHistogram metricType = "histogram"
)
// collectable is any metric that can write its exposition form to w.
// Kept unexported; registry drives it.
type collectable interface {
writeTo(w *bufferedWriter)
}
// Registry holds a set of metrics and a lazily-refreshed snapshot of
// callback-driven gauges. Scraping takes a read lock; registration
// takes a write lock. Both are short-lived.
type Registry struct {
mu sync.RWMutex
entries []registered
names map[string]struct{}
gaugeHooks []gaugeHook
counterHooks []counterHook
}
type registered struct {
name string
c collectable
}
type gaugeHook struct {
name string
help string
fn func() float64
}
type counterHook struct {
name string
help string
fn func() float64
}
// NewRegistry returns an empty registry.
func NewRegistry() *Registry {
return &Registry{names: map[string]struct{}{}}
}
// MustRegister panics if a metric of the same name is already
// registered. Daemons call this once at startup; a duplicate is a
// programming error.
func (r *Registry) MustRegister(name string, c Collector) {
r.mu.Lock()
defer r.mu.Unlock()
if _, dup := r.names[name]; dup {
panic(fmt.Sprintf("metrics: duplicate registration %q", name))
}
r.names[name] = struct{}{}
r.entries = append(r.entries, registered{name: name, c: c})
}
// RegisterGaugeFunc exposes a value produced by calling fn at scrape
// time. Useful for "ask the OS for the bbolt file size" metrics where
// caching the value would be wrong.
func (r *Registry) RegisterGaugeFunc(name, help string, fn func() float64) {
r.mu.Lock()
defer r.mu.Unlock()
if _, dup := r.names[name]; dup {
panic(fmt.Sprintf("metrics: duplicate registration %q", name))
}
r.names[name] = struct{}{}
r.gaugeHooks = append(r.gaugeHooks, gaugeHook{name: name, help: help, fn: fn})
}
// RegisterCounterFunc is the counter equivalent of RegisterGaugeFunc.
// Exposition must be monotonically non-decreasing across calls;
// callers are on the hook for that invariant.
func (r *Registry) RegisterCounterFunc(name, help string, fn func() float64) {
r.mu.Lock()
defer r.mu.Unlock()
if _, dup := r.names[name]; dup {
panic(fmt.Sprintf("metrics: duplicate registration %q", name))
}
r.names[name] = struct{}{}
r.counterHooks = append(r.counterHooks, counterHook{name: name, help: help, fn: fn})
}
// WriteOpenMetrics renders a scrape in the OpenMetrics text format.
// The output ends with the `# EOF` marker that Prometheus requires
// when served as `Content-Type: application/openmetrics-text`.
func (r *Registry) WriteOpenMetrics(w io.Writer) error {
r.mu.RLock()
defer r.mu.RUnlock()
bw := newBufferedWriter(w)
// Stable ordering matters for diffs and for human reading. Sort
// by name at scrape time; registration order is not stable across
// restarts because goroutines may register concurrently.
entries := make([]registered, len(r.entries))
copy(entries, r.entries)
sort.Slice(entries, func(i, j int) bool { return entries[i].name < entries[j].name })
for _, e := range entries {
e.c.writeTo(bw)
}
gauges := append([]gaugeHook(nil), r.gaugeHooks...)
sort.Slice(gauges, func(i, j int) bool { return gauges[i].name < gauges[j].name })
for _, h := range gauges {
bw.writeMeta(h.name, h.help, typeGauge)
bw.writeSample(h.name, nil, h.fn())
}
counters := append([]counterHook(nil), r.counterHooks...)
sort.Slice(counters, func(i, j int) bool { return counters[i].name < counters[j].name })
for _, h := range counters {
bw.writeMeta(h.name, h.help, typeCounter)
bw.writeSample(h.name, nil, h.fn())
}
bw.writeEOF()
return bw.err
}
// -----------------------------------------------------------------------
// Counter
// -----------------------------------------------------------------------
// Counter is a monotonically non-decreasing float value.
type Counter struct {
name string
help string
// Stored as bits of a float64 so Add can safely work on values
// that never need to fit into int64 (e.g., byte counts).
bits uint64
}
// NewCounter constructs an unregistered Counter. Register with
// Registry.MustRegister(name, c).
func NewCounter(name, help string) *Counter {
return &Counter{name: name, help: help}
}
// Add increments the counter by v. Panics on negative v; counters are
// monotonic by contract.
func (c *Counter) Add(v float64) {
if v < 0 {
panic(fmt.Sprintf("metrics: counter %q Add(%g): negative delta", c.name, v))
}
for {
old := atomic.LoadUint64(&c.bits)
newVal := math.Float64frombits(old) + v
if atomic.CompareAndSwapUint64(&c.bits, old, math.Float64bits(newVal)) {
return
}
}
}
// Inc adds 1.
func (c *Counter) Inc() { c.Add(1) }
// Value returns the current counter value. Useful in tests.
func (c *Counter) Value() float64 {
return math.Float64frombits(atomic.LoadUint64(&c.bits))
}
func (c *Counter) writeTo(w *bufferedWriter) {
w.writeMeta(c.name, c.help, typeCounter)
w.writeSample(c.name, nil, c.Value())
}
// -----------------------------------------------------------------------
// Gauge
// -----------------------------------------------------------------------
// Gauge is a point-in-time numeric value that can go up or down.
type Gauge struct {
name string
help string
bits uint64
}
// NewGauge constructs an unregistered Gauge.
func NewGauge(name, help string) *Gauge {
return &Gauge{name: name, help: help}
}
// Set replaces the gauge value.
func (g *Gauge) Set(v float64) {
atomic.StoreUint64(&g.bits, math.Float64bits(v))
}
// Add updates the gauge by v (may be negative).
func (g *Gauge) Add(v float64) {
for {
old := atomic.LoadUint64(&g.bits)
newVal := math.Float64frombits(old) + v
if atomic.CompareAndSwapUint64(&g.bits, old, math.Float64bits(newVal)) {
return
}
}
}
// Inc adds 1. Dec subtracts 1.
func (g *Gauge) Inc() { g.Add(1) }
func (g *Gauge) Dec() { g.Add(-1) }
// Value returns the current gauge value.
func (g *Gauge) Value() float64 {
return math.Float64frombits(atomic.LoadUint64(&g.bits))
}
func (g *Gauge) writeTo(w *bufferedWriter) {
w.writeMeta(g.name, g.help, typeGauge)
w.writeSample(g.name, nil, g.Value())
}
// -----------------------------------------------------------------------
// Histogram
// -----------------------------------------------------------------------
// Histogram is a cumulative histogram with fixed upper-bound buckets.
// Buckets must be strictly increasing; the implicit +Inf bucket is
// appended automatically.
type Histogram struct {
name string
help string
upper []float64
bucketCnt []uint64 // atomic counters per bucket (last entry is +Inf)
sum uint64 // atomic float64 bits
count uint64 // atomic total count
}
// NewHistogram constructs an unregistered Histogram. upperBounds must
// be strictly increasing; the +Inf bucket is implicit.
func NewHistogram(name, help string, upperBounds []float64) *Histogram {
for i := 1; i < len(upperBounds); i++ {
if upperBounds[i] <= upperBounds[i-1] {
panic(fmt.Sprintf("metrics: histogram %q bounds must be strictly increasing", name))
}
}
return &Histogram{
name: name,
help: help,
upper: append([]float64{}, upperBounds...),
bucketCnt: make([]uint64, len(upperBounds)+1), // one extra for +Inf
}
}
// Observe records a single sample.
func (h *Histogram) Observe(v float64) {
for i, up := range h.upper {
if v <= up {
atomic.AddUint64(&h.bucketCnt[i], 1)
}
}
// Always increment the +Inf bucket (cumulative semantics).
atomic.AddUint64(&h.bucketCnt[len(h.upper)], 1)
atomic.AddUint64(&h.count, 1)
for {
old := atomic.LoadUint64(&h.sum)
newSum := math.Float64frombits(old) + v
if atomic.CompareAndSwapUint64(&h.sum, old, math.Float64bits(newSum)) {
return
}
}
}
func (h *Histogram) writeTo(w *bufferedWriter) {
w.writeMeta(h.name, h.help, typeHistogram)
for i, up := range h.upper {
labels := []labelPair{{"le", formatFloat(up)}}
w.writeSample(h.name+"_bucket", labels, float64(atomic.LoadUint64(&h.bucketCnt[i])))
}
w.writeSample(h.name+"_bucket", []labelPair{{"le", "+Inf"}}, float64(atomic.LoadUint64(&h.bucketCnt[len(h.upper)])))
w.writeSample(h.name+"_sum", nil, math.Float64frombits(atomic.LoadUint64(&h.sum)))
w.writeSample(h.name+"_count", nil, float64(atomic.LoadUint64(&h.count)))
}
// -----------------------------------------------------------------------
// Labelled variants (vectors)
// -----------------------------------------------------------------------
// labelledVec is the label bookkeeping CounterVec, HistogramVec and GaugeVec
// share: one child per label-value combination, bounded by maxChildren.
type labelledVec[T any] struct {
kind string // "counter", "histogram" or "gauge"; names the vec in panics
name string
help string
labelKeys []string
mu sync.Mutex
children map[string]*T
keys []string // insertion-ordered; stable for scrape ordering within a vec
maxChildren int // 0 = unlimited
}
// defaultVecCardinalityCap bounds the per-vec child count. Operators
// can raise the limit per metric via SetMaxChildren. The cap exists
// because user-controlled labels (per-IP, per-domain) can otherwise
// grow the children map without bound and exhaust memory.
const defaultVecCardinalityCap = 1000
// overflowLabelValue collapses all label values past the cap into a
// single sentinel bucket so cardinality stays bounded.
const overflowLabelValue = "_overflow_"
func newLabelledVec[T any](kind, name, help string, labelKeys []string) labelledVec[T] {
if len(labelKeys) == 0 {
panic(fmt.Sprintf("metrics: %s vec %q needs at least one label key", kind, name))
}
return labelledVec[T]{
kind: kind,
name: name,
help: help,
labelKeys: append([]string{}, labelKeys...),
children: map[string]*T{},
maxChildren: defaultVecCardinalityCap,
}
}
// SetMaxChildren overrides the per-vec cardinality cap. Pass 0 to
// disable the cap entirely (only safe when label values come from a
// fixed enum the operator controls).
func (v *labelledVec[T]) SetMaxChildren(n int) {
v.mu.Lock()
defer v.mu.Unlock()
v.maxChildren = n
}
// ChildCount returns the current number of distinct label-value
// children, including the overflow sentinel if used. Exposed for
// tests and operator health checks.
func (v *labelledVec[T]) ChildCount() int {
v.mu.Lock()
defer v.mu.Unlock()
return len(v.children)
}
// child returns the child for the given label values, creating it with
// newChild. Values are identified by the concatenation of label values;
// caller supplies them in the same order as labelKeys. Once the cap is
// reached, additional label-value combinations collapse to a single
// "_overflow_" bucket so the map stays bounded.
func (v *labelledVec[T]) child(values []string, newChild func() *T) *T {
if len(values) != len(v.labelKeys) {
panic(fmt.Sprintf("metrics: %s vec %q: got %d label values, want %d", v.kind, v.name, len(values), len(v.labelKeys)))
}
key := joinLabelValues(values)
v.mu.Lock()
defer v.mu.Unlock()
if c, ok := v.children[key]; ok {
return c
}
if v.maxChildren > 0 && len(v.children) >= explicitChildLimit(v.maxChildren) {
key = joinLabelValues(overflowLabelValuesForArity(len(v.labelKeys)))
if c, ok := v.children[key]; ok {
return c
}
}
c := newChild()
v.children[key] = c
v.keys = append(v.keys, key)
return c
}
// each calls fn with the label pairs of every child, in sorted key order.
// The children are snapshotted under the lock and fn runs outside it.
func (v *labelledVec[T]) each(fn func(pairs []labelPair, child *T)) {
v.mu.Lock()
keys := append([]string(nil), v.keys...)
childMap := make(map[string]*T, len(v.children))
for k, c := range v.children {
childMap[k] = c
}
v.mu.Unlock()
sort.Strings(keys)
for _, k := range keys {
values := splitLabelValues(k)
pairs := make([]labelPair, len(v.labelKeys))
for i, lk := range v.labelKeys {
pairs[i] = labelPair{key: lk, value: values[i]}
}
fn(pairs, childMap[k])
}
}
// overflowLabelValuesForArity returns a values slice of length n with
// every entry set to the overflow sentinel. Used so the cap path
// produces a single canonical key regardless of arity.
func overflowLabelValuesForArity(n int) []string {
out := make([]string, n)
for i := range out {
out[i] = overflowLabelValue
}
return out
}
func explicitChildLimit(maxChildren int) int {
if maxChildren <= 1 {
return 0
}
return maxChildren - 1
}
// CounterVec is a family of counters indexed by a fixed set of label
// keys. Label values are provided per sample.
type CounterVec struct {
labelledVec[Counter]
}
// NewCounterVec constructs a vector counter. labelKeys must be non-
// empty; use NewCounter for an unlabelled counter.
func NewCounterVec(name, help string, labelKeys []string) *CounterVec {
return &CounterVec{labelledVec: newLabelledVec[Counter]("counter", name, help, labelKeys)}
}
// With returns the child counter for the given label values, in the
// same order as labelKeys from NewCounterVec.
func (cv *CounterVec) With(values ...string) *Counter {
return cv.child(values, func() *Counter { return &Counter{name: cv.name, help: cv.help} })
}
func (cv *CounterVec) writeTo(w *bufferedWriter) {
w.writeMeta(cv.name, cv.help, typeCounter)
cv.each(func(pairs []labelPair, c *Counter) {
w.writeSample(cv.name, pairs, c.Value())
})
}
// HistogramVec is the labelled variant of Histogram. All children
// share the same upper-bound set.
type HistogramVec struct {
labelledVec[Histogram]
upper []float64
}
// NewHistogramVec constructs a vector histogram.
func NewHistogramVec(name, help string, labelKeys []string, upperBounds []float64) *HistogramVec {
hv := &HistogramVec{
labelledVec: newLabelledVec[Histogram]("histogram", name, help, labelKeys),
upper: append([]float64{}, upperBounds...),
}
for i := 1; i < len(upperBounds); i++ {
if upperBounds[i] <= upperBounds[i-1] {
panic(fmt.Sprintf("metrics: histogram vec %q bounds must be strictly increasing", name))
}
}
return hv
}
// With returns the child histogram for the given label values.
func (hv *HistogramVec) With(values ...string) *Histogram {
return hv.child(values, func() *Histogram {
return &Histogram{
name: hv.name,
help: hv.help,
upper: hv.upper,
bucketCnt: make([]uint64, len(hv.upper)+1),
}
})
}
func (hv *HistogramVec) writeTo(w *bufferedWriter) {
w.writeMeta(hv.name, hv.help, typeHistogram)
hv.each(func(labelPairs []labelPair, h *Histogram) {
for i, up := range h.upper {
pairs := append([]labelPair(nil), labelPairs...)
pairs = append(pairs, labelPair{"le", formatFloat(up)})
w.writeSample(hv.name+"_bucket", pairs, float64(atomicLoad(&h.bucketCnt[i])))
}
pairsInf := append([]labelPair(nil), labelPairs...)
pairsInf = append(pairsInf, labelPair{"le", "+Inf"})
w.writeSample(hv.name+"_bucket", pairsInf, float64(atomicLoad(&h.bucketCnt[len(h.upper)])))
w.writeSample(hv.name+"_sum", labelPairs, math.Float64frombits(atomicLoad(&h.sum)))
w.writeSample(hv.name+"_count", labelPairs, float64(atomicLoad(&h.count)))
})
}
// atomicLoad is a small helper so the HistogramVec writeTo reads match
// the base Histogram's atomic semantics without pulling sync/atomic
// across every line.
func atomicLoad(p *uint64) uint64 { return atomic.LoadUint64(p) }
// GaugeVec is the labelled variant of Gauge.
type GaugeVec struct {
labelledVec[Gauge]
}
// NewGaugeVec constructs a vector gauge.
func NewGaugeVec(name, help string, labelKeys []string) *GaugeVec {
return &GaugeVec{labelledVec: newLabelledVec[Gauge]("gauge", name, help, labelKeys)}
}
// With returns the child gauge for the given label values.
func (gv *GaugeVec) With(values ...string) *Gauge {
return gv.child(values, func() *Gauge { return &Gauge{name: gv.name, help: gv.help} })
}
func (gv *GaugeVec) writeTo(w *bufferedWriter) {
w.writeMeta(gv.name, gv.help, typeGauge)
gv.each(func(pairs []labelPair, g *Gauge) {
w.writeSample(gv.name, pairs, g.Value())
})
}
// -----------------------------------------------------------------------
// Internal exposition
// -----------------------------------------------------------------------
type labelPair struct {
key, value string
}
type bufferedWriter struct {
w io.Writer
err error
}
func newBufferedWriter(w io.Writer) *bufferedWriter {
return &bufferedWriter{w: w}
}
func (bw *bufferedWriter) writef(format string, args ...any) {
if bw.err != nil {
return
}
if _, err := fmt.Fprintf(bw.w, format, args...); err != nil {
bw.err = err
}
}
func (bw *bufferedWriter) writeMeta(name, help string, typ metricType) {
bw.writef("# HELP %s %s\n", name, escapeHelp(help))
bw.writef("# TYPE %s %s\n", name, typ)
}
func (bw *bufferedWriter) writeSample(name string, labels []labelPair, value float64) {
var sb strings.Builder
sb.WriteString(name)
if len(labels) > 0 {
sb.WriteByte('{')
for i, p := range labels {
if i > 0 {
sb.WriteByte(',')
}
sb.WriteString(p.key)
sb.WriteString(`="`)
sb.WriteString(escapeLabel(p.value))
sb.WriteByte('"')
}
sb.WriteByte('}')
}
bw.writef("%s %s\n", sb.String(), formatFloat(value))
}
func (bw *bufferedWriter) writeEOF() {
bw.writef("# EOF\n")
}
// Label values are joined with an ASCII unit-separator so `a|b` and
// `ab|` do not collide. Label values themselves cannot contain 0x1F
// (we would reject it in validation); joinLabelValues panics if
// someone smuggles one in.
const labelSep = "\x1f"
func joinLabelValues(vs []string) string {
for _, v := range vs {
if strings.Contains(v, labelSep) {
panic("metrics: label value contains unit-separator")
}
}
return strings.Join(vs, labelSep)
}
func splitLabelValues(k string) []string {
return strings.Split(k, labelSep)
}
// formatFloat renders a float64 in OpenMetrics-friendly form: integer
// samples look integer, floats keep precision, NaN and infinities use
// the OpenMetrics tokens.
func formatFloat(v float64) string {
switch {
case math.IsNaN(v):
return "NaN"
case math.IsInf(v, 1):
return "+Inf"
case math.IsInf(v, -1):
return "-Inf"
case v == math.Trunc(v) && math.Abs(v) < 1e15:
return fmt.Sprintf("%d", int64(v))
default:
return fmt.Sprintf("%g", v)
}
}
// escapeHelp replaces characters that break the HELP line format
// (newline, backslash). Strings the caller supplies are not attacker-
// controlled (they are developer literals), but defensive escaping
// keeps the scrape well-formed even if a future contributor gets
// creative.
func escapeHelp(s string) string {
s = strings.ReplaceAll(s, `\`, `\\`)
s = strings.ReplaceAll(s, "\n", `\n`)
return s
}
// escapeLabel is stricter: OpenMetrics requires \\, \n, and \" inside
// double-quoted label values.
func escapeLabel(s string) string {
s = strings.ReplaceAll(s, `\`, `\\`)
s = strings.ReplaceAll(s, `"`, `\"`)
s = strings.ReplaceAll(s, "\n", `\n`)
return s
}
// ErrNotRegistered can be returned by callers that look up a metric by
// name without finding it. Currently unused internally; kept for the
// external helper surface so test doubles can standardise on it.
var ErrNotRegistered = errors.New("metrics: not registered")
// -----------------------------------------------------------------------
// Process-wide default registry
// -----------------------------------------------------------------------
// defaultRegistry is the Registry the daemon shares across packages.
// Tests that need isolation should construct their own NewRegistry().
var defaultRegistry = NewRegistry()
// Default returns the process-wide Registry.
func Default() *Registry { return defaultRegistry }
// MustRegister is shorthand for Default().MustRegister.
func MustRegister(name string, c Collector) { defaultRegistry.MustRegister(name, c) }
// RegisterGaugeFunc is shorthand for Default().RegisterGaugeFunc.
func RegisterGaugeFunc(name, help string, fn func() float64) {
defaultRegistry.RegisterGaugeFunc(name, help, fn)
}
// RegisterCounterFunc is shorthand for Default().RegisterCounterFunc.
func RegisterCounterFunc(name, help string, fn func() float64) {
defaultRegistry.RegisterCounterFunc(name, help, fn)
}
// WriteOpenMetrics is shorthand for Default().WriteOpenMetrics.
func WriteOpenMetrics(w io.Writer) error {
return defaultRegistry.WriteOpenMetrics(w)
}
// Collector is the type accepted by Registry.MustRegister. Exported
// so external packages have a name for the interface, even though the
// useful method on it is unexported (only metrics-package types can
// implement it, which is the intent).
type Collector = collectable
package metrics
import (
"runtime"
"sync"
)
var runtimeMetricsOnce sync.Once
// goRuntimeCollector emits Go runtime memory and scheduler stats. It reads
// runtime.ReadMemStats once per scrape (ReadMemStats briefly stops the world,
// so a single read for the whole family is deliberate -- do not split it into
// per-metric gauge hooks).
type goRuntimeCollector struct{}
func (goRuntimeCollector) writeTo(bw *bufferedWriter) {
var m runtime.MemStats
runtime.ReadMemStats(&m)
g := func(name, help string, v float64) {
bw.writeMeta(name, help, typeGauge)
bw.writeSample(name, nil, v)
}
g("go_memstats_heap_alloc_bytes", "Heap bytes allocated and still in use.", float64(m.HeapAlloc))
g("go_memstats_heap_inuse_bytes", "Heap bytes in in-use spans.", float64(m.HeapInuse))
g("go_memstats_heap_idle_bytes", "Heap bytes idle, waiting to be used.", float64(m.HeapIdle))
g("go_memstats_heap_released_bytes", "Heap bytes released to the OS.", float64(m.HeapReleased))
g("go_memstats_heap_sys_bytes", "Heap bytes obtained from the OS.", float64(m.HeapSys))
g("go_memstats_heap_objects", "Number of currently allocated heap objects.", float64(m.HeapObjects))
g("go_memstats_stack_inuse_bytes", "Bytes in use by the stack allocator.", float64(m.StackInuse))
g("go_memstats_sys_bytes", "Total bytes obtained from the OS.", float64(m.Sys))
g("go_memstats_next_gc_bytes", "Heap size target for the next GC cycle.", float64(m.NextGC))
g("go_memstats_gc_cpu_fraction", "Fraction of CPU time used by GC since program start.", m.GCCPUFraction)
g("go_goroutines", "Number of goroutines that currently exist.", float64(runtime.NumGoroutine()))
}
// RegisterRuntimeMetrics adds Go runtime memory/scheduler stats to the default
// registry so they appear on the /metrics endpoint. Idempotent: safe to call on
// every daemon start (repeated starts in a test binary would otherwise panic on
// the duplicate registration).
func RegisterRuntimeMetrics() {
runtimeMetricsOnce.Do(func() {
defaultRegistry.MustRegister("go_runtime_stats", goRuntimeCollector{})
})
}
package mime
import (
"archive/tar"
"archive/zip"
"bufio"
"bytes"
"compress/gzip"
"encoding/binary"
"errors"
"io"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/zipdir"
)
var errArchiveWalk = errors.New("archive walk limit exceeded")
// maxDirectoryStream is the most compressed data an empty directory entry
// may carry. Writers that deflate empty directories emit about 2 bytes; a
// larger stream would be inflated without spending the extraction budget.
const maxDirectoryStream = 64
func validateZIPDirectoryEntry(file *zip.File) error {
if file.UncompressedSize64 != 0 || file.CompressedSize64 > maxDirectoryStream {
return zip.ErrFormat
}
if file.CompressedSize64 == 0 {
return nil
}
// Some writers deflate empty directories. File.Open otherwise bypasses
// their decompressor, so validate their claimed empty content as a file.
entry := *file
entry.Name = strings.TrimRight(entry.Name, "/")
rc, err := entry.Open()
if err != nil {
return err
}
defer rc.Close()
var probe [1]byte
n, err := io.ReadFull(rc, probe[:])
if n == 0 && err == io.EOF {
return nil
}
return zip.ErrFormat
}
// validateZIPDirectory checks the actual listing before archive/zip allocates
// File objects. Its end record count is only a hint, even for ZIP64 archives.
func validateZIPDirectory(r io.ReaderAt, size int64, dir zipdir.Directory, walkCap int) error {
maxRecords := uint64(max(walkCap, 0)) // #nosec G115 -- clamped to nonnegative.
if dir.Records > maxRecords {
return errArchiveWalk
}
// archive/zip accepts a shorter comment than zipdir.Read does. A second
// end record inside the comment must not select an unchecked directory.
tail := make([]byte, size-dir.EndOffset)
if _, err := r.ReadAt(tail, dir.EndOffset); err != nil {
return err
}
for i := len(tail) - zipdir.EndLen; i > 0; i-- {
if binary.LittleEndian.Uint32(tail[i:]) == zipdir.EndSignature {
return zipdir.ErrFormat
}
}
// archive/zip may fall back to the declared offset when the computed
// base is positive. Do not let it read an unchecked earlier listing.
if dir.DeclaredOffset < dir.Offset {
var sig [4]byte
if _, err := r.ReadAt(sig[:], dir.DeclaredOffset); err != nil {
return err
}
if binary.LittleEndian.Uint32(sig[:]) == zipdir.HeaderSignature {
return zipdir.ErrFormat
}
}
var header [zipdir.HeaderLen]byte
offset, remaining := dir.Offset, dir.Size
var records uint64
for remaining > 0 {
if records >= maxRecords {
return errArchiveWalk
}
if remaining < int64(len(header)) {
return zipdir.ErrFormat
}
if _, err := r.ReadAt(header[:], offset); err != nil {
return err
}
if binary.LittleEndian.Uint32(header[:]) != zipdir.HeaderSignature {
return zipdir.ErrFormat
}
n := int64(len(header)) + int64(binary.LittleEndian.Uint16(header[28:30])) +
int64(binary.LittleEndian.Uint16(header[30:32])) + int64(binary.LittleEndian.Uint16(header[32:34]))
if n > remaining {
return zipdir.ErrFormat
}
offset += n
remaining -= n
records++
}
if records != dir.Records {
return zipdir.ErrFormat
}
return nil
}
const tarBlockSize = 512
// gzipStreamReader preserves concatenated gzip streams while bounding empty
// streams, which can consume work without yielding any decompressed bytes.
type gzipStreamReader struct {
compressed *bufio.Reader
gr *gzip.Reader
walkCap int
streams int
ended bool
}
func (r *gzipStreamReader) Read(p []byte) (int, error) {
for {
if r.ended {
if _, err := r.compressed.Peek(1); err != nil {
return 0, err
}
if r.streams >= r.walkCap {
return 0, errArchiveWalk
}
if err := r.gr.Reset(r.compressed); err != nil {
return 0, err
}
r.gr.Multistream(false)
r.streams++
r.ended = false
}
n, err := r.gr.Read(p)
if err == io.EOF {
r.ended = true
if n == 0 {
continue
}
err = nil
}
return n, err
}
}
// tarWalkReader counts physical headers, including the PAX and GNU metadata
// that tar.Reader.Next consumes internally. tar.Reader still owns validation
// and metadata interpretation. After Next, setMember supplies the effective
// payload size, which can differ from the raw header size in PAX archives.
type tarWalkReader struct {
r io.Reader
walkCap int
walked int
header [tarBlockSize]byte
pending []byte
remaining int64 // -1 passes reads through while Next resolves a file header
padding int64
mapBytes int64 // sparse-map bytes consumed by Next after the file header
}
func (r *tarWalkReader) Read(p []byte) (int, error) {
if len(p) == 0 {
return 0, nil
}
if len(r.pending) > 0 {
n := copy(p, r.pending)
r.pending = r.pending[n:]
return n, nil
}
if r.remaining < 0 {
n, err := r.r.Read(p)
r.mapBytes += int64(n)
return n, err
}
if r.remaining > 0 || r.padding > 0 {
left := &r.remaining
if *left == 0 {
left = &r.padding
}
if int64(len(p)) > *left {
p = p[:*left]
}
n, err := r.r.Read(p)
*left -= int64(n)
return n, err
}
if _, err := io.ReadFull(r.r, r.header[:]); err != nil {
return 0, err
}
if r.header != [tarBlockSize]byte{} {
if r.walked >= r.walkCap {
return 0, errArchiveWalk
}
r.walked++
switch r.header[156] {
case tar.TypeXHeader, tar.TypeXGlobalHeader, tar.TypeGNULongName, tar.TypeGNULongLink:
n, err := tarMetadataSize(r.header[124:136])
if err != nil {
return 0, err
}
r.remaining, r.padding = n, (tarBlockSize-n%tarBlockSize)%tarBlockSize
case tar.TypeGNUSparse:
// The extractor does not scan legacy sparse members. Reject them
// before Next can walk an unchecked chain of extension headers.
return 0, errors.New("unsupported sparse tar member")
default:
r.remaining = -1
r.mapBytes = 0
}
}
r.pending = r.header[:]
return r.Read(p)
}
func (r *tarWalkReader) setMember(hdr *tar.Header) error {
n := hdr.Size
for key := range hdr.PAXRecords {
if strings.HasPrefix(key, "GNU.sparse.") {
// Next replaces Size with the logical sparse size. The wire still
// carries the physical size, possibly overridden by PAX "size".
var err error
n, err = tarMetadataSize(r.header[124:136])
if size := hdr.PAXRecords["size"]; size != "" {
n, err = strconv.ParseInt(size, 10, 64)
}
if err != nil {
return err
}
break
}
}
switch hdr.Typeflag {
case tar.TypeLink, tar.TypeSymlink, tar.TypeChar, tar.TypeBlock, tar.TypeDir, tar.TypeFifo:
n = 0
case tar.TypeXGlobalHeader:
// Next has consumed the metadata body but its padding remains.
return nil
}
r.remaining, r.padding = n-r.mapBytes, (tarBlockSize-n%tarBlockSize)%tarBlockSize
return nil
}
func tarMetadataSize(field []byte) (int64, error) {
if field[0]&0x80 != 0 {
// GNU base-256 numbers are signed. Sizes must be nonnegative and fit
// int64, so the high marker and leading padding are the only high bits.
if field[0] != 0x80 || !bytes.Equal(field[1:4], []byte{0, 0, 0}) {
return 0, tar.ErrHeader
}
n := binary.BigEndian.Uint64(field[4:])
if n > 1<<63-1 {
return 0, tar.ErrHeader
}
return int64(n), nil // #nosec G115 -- bounded by MaxInt64 above.
}
s := strings.Trim(string(field), " \x00")
if s == "" {
return 0, nil
}
if i := strings.IndexByte(s, 0); i >= 0 {
s = s[:i]
}
n, err := strconv.ParseUint(s, 8, 63)
if err != nil {
return 0, tar.ErrHeader
}
return int64(n), nil // #nosec G115 -- ParseUint is limited to 63 bits.
}
// finishTar verifies the gzip trailer and permits only tar's zero padding
// after its end marker. Another archive or nonzero payload was not scanned.
func finishTar(b *budgetReader) error {
var buf [32 << 10]byte
for {
n, err := b.Read(buf[:])
if b.err != nil {
return b.err
}
for _, v := range buf[:n] {
if v != 0 {
return errors.New("unread data after tar end marker")
}
}
if err == io.EOF {
return nil
}
}
}
package mime
import (
"archive/tar"
"archive/zip"
"bufio"
"bytes"
"compress/gzip"
"errors"
"fmt"
"io"
"mime"
"mime/multipart"
"net/textproto"
"os"
"path/filepath"
"strings"
"unicode"
"unicode/utf8"
"github.com/pidginhost/csm/internal/zipdir"
)
// ExtractedPart represents a single extracted attachment.
type ExtractedPart struct {
Filename string
ContentType string
Size int64
// File holds the part's bytes. Its offset is unspecified: read it with
// io.NewSectionReader(File, 0, Size).
File *os.File
// TempPath names the file when the default sink staged it on disk, and
// is empty for any other sink.
TempPath string
Nested bool
ArchiveName string
}
// ExtractionResult holds all extracted parts and envelope metadata.
type ExtractionResult struct {
Parts []ExtractedPart
Partial bool
PartialReason string
EncryptedEntries []EncryptedArchiveEntry
// Report names are bounded separately from extraction: encrypted members
// beyond the reporting limit still must not cause delivery retries.
EncryptedEntriesOmitted int
Direction string
From string
To []string
Subject string
// Shared across multipart recursion so ambiguous wrappers cannot branch
// into exponentially many decoding passes.
transferVariants int
}
// EncryptedArchiveEntry names an archive member CSM cannot read because the
// entry is encrypted. This is deliberately not a partial extraction: a partial
// extraction is a limit or a fault that a later attempt may get past, while an
// encrypted member stays unreadable however many times delivery is retried.
type EncryptedArchiveEntry struct {
ArchiveName string
Filename string
}
// Limits controls resource bounds during extraction.
type Limits struct {
MaxAttachmentSize int64
MaxArchiveDepth int
MaxArchiveFiles int
MaxExtractionSize int64
// TempDir is the directory CreateTemp uses for extracted parts.
// Empty falls back to os.TempDir() (/tmp on Linux). Operators
// should set this to a daemon-owned 0700 path so extracted email
// attachments are not staged in a world-writable directory where
// another local uid can race the scanner via symlink swaps.
TempDir string
// PartSink creates the file each part is written to. Nil stages parts
// with os.CreateTemp in TempDir. The parser owns any returned file, even
// on error; a custom sink remains responsible for removing named files.
PartSink func(prefix string) (*os.File, error)
// MaxParts caps how many parts are staged; 0 means no cap. Each staged
// part holds an open file, so a process with a descriptor limit needs it.
MaxParts int
}
// MaxParts caps the parts one message stages. Each holds an open file
// until the scan ends, so the cap keeps a message of thousands of tiny
// attachments from exhausting the process's descriptors.
const MaxParts = 200
// DefaultLimits returns the default extraction limits.
func DefaultLimits() Limits {
return Limits{
MaxAttachmentSize: 25 * 1024 * 1024,
MaxArchiveDepth: 1,
MaxArchiveFiles: 50,
MaxExtractionSize: 100 * 1024 * 1024,
MaxParts: MaxParts,
}
}
// Close closes every part's file and removes the ones the default sink
// staged on disk. Repeated calls do nothing.
func (r *ExtractionResult) Close() {
for i := range r.Parts {
p := &r.Parts[i]
if p.File != nil {
_ = p.File.Close()
p.File = nil
}
if p.TempPath != "" {
_ = os.Remove(p.TempPath)
p.TempPath = ""
}
}
}
// ParseSpoolMessage parses an Exim spool message (-H and -D files) and
// extracts its attachments. The caller closes the result.
func ParseSpoolMessage(headerPath, bodyPath string, limits Limits) (*ExtractionResult, error) {
// #nosec G304 -- path is Exim -H file path from mail queue scanner walk.
header, err := os.Open(headerPath)
if err != nil {
return nil, fmt.Errorf("parsing header file: %w", err)
}
defer header.Close()
// #nosec G304 -- path is mail queue file path from scanner walk.
body, err := os.Open(bodyPath)
if err != nil {
return nil, fmt.Errorf("reading body file: %w", err)
}
defer body.Close()
return parseSpool(header, body, filepath.Base(bodyPath), limits)
}
// ParseSpoolFiles is ParseSpoolMessage over open -H and -D files. Both are
// read with positional reads only: a descriptor's offset may be shared with
// whoever opened it. The -D marker line is named after the message ID on
// the first line of -H.
func ParseSpoolFiles(header, body *os.File, limits Limits) (*ExtractionResult, error) {
return parseSpool(header, body, "", limits)
}
func parseSpool(header, body io.ReaderAt, bodyMarker string, limits Limits) (*ExtractionResult, error) {
headerData, err := io.ReadAll(io.NewSectionReader(header, 0, maxHeaderFileBytes+1))
if err != nil {
return nil, fmt.Errorf("parsing header file: %w", err)
}
if len(headerData) > maxHeaderFileBytes {
return &ExtractionResult{Partial: true, PartialReason: "message header exceeds parser memory budget"}, nil
}
envelope, hdrs := parseEximHeaderData(headerData)
if bodyMarker == "" {
bodyMarker = bodyMarkerFromHeader(headerData)
}
result := &ExtractionResult{
From: envelope.from,
To: envelope.to,
Subject: envelope.subject,
}
// Determine direction from Received headers
result.Direction = detectDirection(hdrs)
maxBodyBytes := bodyReadLimit(limits)
bodyData, partial, err := readBodyLimited(body, maxBodyBytes)
if err != nil {
return nil, fmt.Errorf("reading body file: %w", err)
}
if partial {
result.Partial = true
result.PartialReason = "message body exceeds parser memory budget"
return result, nil
}
// Exim -D files open with a "<message-id>-D" marker line; strip it before
// any body decode so single-part base64/QP payloads are not corrupted by
// the marker bytes.
bodyData = stripSpoolBodyMarker(bodyData, bodyMarker)
ct := hdrs.Get("Content-Type")
if ct == "" {
ct = "text/plain"
}
mediaType, params, parseErr := mime.ParseMediaType(ct)
if parseErr != nil {
// Unparseable content type - treat as plain text, no attachments
return result, nil //nolint:nilerr // fail-open by design
}
if strings.HasPrefix(mediaType, "multipart/") {
boundary := params["boundary"]
if boundary == "" {
return result, nil
}
var totalSize int64
cte := strings.ToLower(hdrs.Get("Content-Transfer-Encoding"))
extractEncodedMultipart(bytes.NewReader(bodyData), cte, boundary, limits, result, &totalSize, 0, 0)
} else if !strings.HasPrefix(mediaType, "text/") {
// Single-part non-text message (e.g. application/octet-stream,
// application/pdf, image/*). These are attachment-like payloads
// that must be scanned even without a multipart wrapper.
cte := strings.ToLower(hdrs.Get("Content-Transfer-Encoding"))
readers, _ := transferReaders(cte, bytes.NewReader(bodyData), result)
var totalSize int64
for _, reader := range readers {
decoded, truncated, decodeErr := decodeSinglePart(reader, limits.MaxAttachmentSize+1)
if decodeErr != nil {
// The decoded prefix is still scanned; see extractMultipartNested.
markPartial(result, fmt.Sprintf("could not decode single-part attachment: %v", decodeErr))
}
switch {
case len(decoded) == 0:
// Nothing decoded, nothing to scan.
case !truncated && int64(len(decoded)) <= limits.MaxAttachmentSize:
if int64(len(decoded)) > limits.MaxExtractionSize-totalSize {
markPartial(result, "total extraction size exceeds limit")
break
}
if partsFull(limits, result) {
break
}
tmpFile, tempPath, tmpErr := limits.createPart("csm-emailav-single-*")
if tmpErr == nil {
n, writeErr := tmpFile.Write(decoded)
if writeErr != nil || n != len(decoded) {
discardPart(tmpFile, tempPath)
markPartial(result, "could not stage single-part attachment for scanning")
} else {
totalSize += int64(n)
// Name the body the way a multipart attachment is
// named: Content-Type name, then the disposition's
// filename, so a disposition-named archive is unpacked.
rawFilename := params["name"]
if rawFilename == "" {
if _, dparams, err := mime.ParseMediaType(hdrs.Get("Content-Disposition")); err == nil {
rawFilename = dparams["filename"]
}
}
if rawFilename == "" {
rawFilename = "attachment"
}
filename := SanitizeAttachmentName(rawFilename)
result.Parts = append(result.Parts, ExtractedPart{
Filename: filename,
ContentType: mediaType,
Size: int64(len(decoded)),
File: tmpFile,
TempPath: tempPath,
})
expandArchive(tmpFile, int64(n), rawFilename, filename, limits, result, &totalSize, 0)
}
} else {
markPartial(result, "could not stage single-part attachment for scanning")
}
default:
result.Partial = true
result.PartialReason = "single-part attachment exceeds max size"
}
}
}
// text/* bodies are not attachments - skip
return result, nil
}
func bodyReadLimit(limits Limits) int64 {
limit := limits.MaxExtractionSize
if limit < limits.MaxAttachmentSize*2 {
limit = limits.MaxAttachmentSize * 2
}
if limit <= 0 {
limit = DefaultLimits().MaxExtractionSize
}
return limit
}
func readFileLimited(path string, limit int64) ([]byte, error) {
// #nosec G304 -- path is mail queue file path from scanner walk.
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
data, err := io.ReadAll(io.LimitReader(f, limit+1))
if err != nil {
return nil, err
}
if int64(len(data)) > limit {
return nil, fmt.Errorf("message body exceeds parser memory budget")
}
return data, nil
}
// readBodyLimited reads up to limit+1 bytes of the spool body from offset
// 0. Returns (data, partial=true, nil) when the body exceeds the limit so
// the caller can mark the result as partial. Reading the size from the
// same read closes the TOCTOU window a Stat-then-Open sequence had.
func readBodyLimited(r io.ReaderAt, limit int64) ([]byte, bool, error) {
data, err := io.ReadAll(io.NewSectionReader(r, 0, limit+1))
if err != nil {
return nil, false, err
}
if int64(len(data)) > limit {
return nil, true, nil
}
return data, false, nil
}
func markPartial(result *ExtractionResult, reason string) {
result.Partial = true
if result.PartialReason == "" {
result.PartialReason = reason
}
}
// decodeSinglePart returns the decoded body, whether it was cut at limit, and
// any decode error. On error the bytes decoded before it are still returned.
func decodeSinglePart(r io.Reader, limit int64) ([]byte, bool, error) {
decoded, err := io.ReadAll(io.LimitReader(r, limit+1))
if int64(len(decoded)) > limit {
return decoded[:limit], true, err
}
return decoded, false, err
}
// maxHeaderFileBytes bounds the Exim -H file read into memory. Exim's own
// header size limit keeps real mail far below it.
const maxHeaderFileBytes = 16 << 20
type envelope struct {
from string
to []string
subject string
}
// bodyMarkerFromHeader names the -D marker line after the message ID Exim
// writes on the first line of -H ("<id>-H"), or "" when that line is not
// one.
func bodyMarkerFromHeader(headerData []byte) string {
first, _, _ := bytes.Cut(headerData, []byte("\n"))
id, ok := strings.CutSuffix(string(bytes.TrimRight(first, "\r")), "-H")
if !ok || id == "" {
return ""
}
return id + "-D"
}
// parseEximHeaderData parses the bytes of an Exim -H spool file into the
// envelope fields and the RFC 5322 header set. The -H format is:
//
// line 1: <message-id>-H
// line 2: <envelope-user> <uid> <gid>
// <envelope metadata / options / recipient block>
// <blank line>
// <RFC headers, each prefixed "<byte-count><flag> ">
//
// Each header line carries a decimal byte count and a single flag character
// ('F' From, 'T' To, 'P' Received, 'R' Reply-To, '*' deleted, space for the
// rest); folded continuations start with whitespace. Real cPanel-Exim writes
// this format, so parsing bare "From:" lines (as an earlier version did) never
// matched and left every field empty. Parsing is fail-open: unparseable input
// yields empty headers, never an error.
func parseEximHeaderData(data []byte) (*envelope, textproto.MIMEHeader) {
env := &envelope{}
hdrs := make(textproto.MIMEHeader)
reconstructed := reconstructEximHeaderBytes(data)
tp := textproto.NewReader(bufio.NewReader(bytes.NewReader(reconstructed)))
// A malformed trailing line makes ReadMIMEHeader return an error alongside
// the headers it parsed before the fault; keep those and fail open.
parsed, _ := tp.ReadMIMEHeader()
if len(parsed) == 0 {
return env, hdrs // fail-open: no recognizable headers
}
hdrs = parsed
env.from = hdrs.Get("From")
env.subject = hdrs.Get("Subject")
if to := hdrs.Get("To"); to != "" {
for _, addr := range strings.Split(to, ",") {
env.to = append(env.to, strings.TrimSpace(addr))
}
}
return env, hdrs
}
// ParseSpoolMIMEHeaders returns the live RFC 5322 headers from Exim -H data.
// Folded values are unfolded and deleted headers are omitted, just as they
// are for attachment extraction.
func ParseSpoolMIMEHeaders(data []byte) textproto.MIMEHeader {
_, headers := parseEximHeaderData(data)
return headers
}
// reconstructEximHeaderBytes rebuilds a plain RFC 5322 header block from an
// Exim -H file by dropping the two leading metadata lines and the envelope
// preamble, then stripping the "<byte-count><flag> " prefix from each header
// line. Deleted headers (flag '*') and their folds are omitted; folded
// continuation lines are preserved verbatim so textproto can rejoin them.
func reconstructEximHeaderBytes(data []byte) []byte {
sc := bufio.NewScanner(bytes.NewReader(data))
// A single header line (DKIM/ARC signatures, long base64) can be large but
// is bounded by the -H file already in memory.
sc.Buffer(make([]byte, 0, 8192), len(data)+64)
var out bytes.Buffer
lineNum := 0
inHeaders := false
skippingDeleted := false
haveLiveHeader := false
for sc.Scan() {
line := sc.Text()
lineNum++
if lineNum <= 2 {
continue // message-id marker, then envelope-user line
}
if !inHeaders {
if line == "" {
inHeaders = true
}
continue // envelope metadata / recipient block
}
// Folded continuation: RFC 5322 lines starting with WSP belong to the
// preceding header.
if len(line) > 0 && (line[0] == ' ' || line[0] == '\t') {
if !skippingDeleted && haveLiveHeader {
out.WriteString(line)
out.WriteString("\r\n")
}
continue
}
rest, flag, ok := stripEximPrefix(line)
if !ok {
skippingDeleted = false
haveLiveHeader = false
continue // not a recognizable header start
}
if flag == '*' {
skippingDeleted = true // deleted header: skip it and its folds
haveLiveHeader = false
continue
}
if !isMIMEHeaderStart(rest) {
skippingDeleted = false
haveLiveHeader = false
continue
}
skippingDeleted = false
haveLiveHeader = true
out.WriteString(rest)
out.WriteString("\r\n")
}
out.WriteString("\r\n") // terminate the header block for ReadMIMEHeader
return out.Bytes()
}
// stripEximPrefix removes the leading "<byte-count><flag> " from an Exim -H
// header line and returns the remaining "Name: value" text plus the flag byte.
// ok is false when the line does not carry a valid prefix.
func stripEximPrefix(line string) (rest string, flag byte, ok bool) {
i := 0
for i < len(line) && line[i] >= '0' && line[i] <= '9' {
i++
}
if i == 0 || i+1 >= len(line) {
return "", 0, false
}
flag = line[i]
if !isEximFlagByte(flag) || line[i+1] != ' ' {
return "", 0, false
}
return line[i+2:], flag, true
}
func isMIMEHeaderStart(line string) bool {
colon := strings.IndexByte(line, ':')
if colon <= 0 {
return false
}
for i := 0; i < colon; i++ {
if !isMIMEHeaderFieldNameByte(line[i]) {
return false
}
}
return true
}
func isMIMEHeaderFieldNameByte(b byte) bool {
return (b >= '0' && b <= '9') ||
(b >= 'a' && b <= 'z') ||
(b >= 'A' && b <= 'Z') ||
b == '!' || b == '#' || b == '$' || b == '%' ||
b == '&' || b == '\'' || b == '*' || b == '+' ||
b == '-' || b == '.' || b == '^' || b == '_' ||
b == '`' || b == '|' || b == '~'
}
func isEximFlagByte(b byte) bool {
return b == ' ' || b == '*' ||
(b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z')
}
// stripSpoolBodyMarker removes the first line of an Exim -D body file when it
// is the "<message-id>-D" marker Exim always writes there. The marker equals
// the -D file's base name, so matching on that is exact: bodies that lack a
// marker (or callers that pass a raw body) are left untouched. Without this,
// single-part base64/QP payloads decode the marker bytes as content and fail.
func stripSpoolBodyMarker(bodyData []byte, marker string) []byte {
if marker == "" {
return bodyData
}
nl := bytes.IndexByte(bodyData, '\n')
var first []byte
if nl < 0 {
first = bodyData
} else {
first = bodyData[:nl]
}
if string(bytes.TrimRight(first, "\r")) != marker {
return bodyData
}
if nl < 0 {
return nil
}
return bodyData[nl+1:]
}
// detectDirection guesses inbound vs outbound from Received headers.
func detectDirection(hdrs textproto.MIMEHeader) string {
received := hdrs.Values("Received")
if len(received) == 0 {
return "outbound" // locally generated, no Received headers
}
// If the first (topmost) Received header contains "authenticated" it's outbound
first := strings.ToLower(received[0])
if strings.Contains(first, "(authenticated") || strings.Contains(first, "auth=") {
return "outbound"
}
return "inbound"
}
// maxMIMENestingDepth caps multipart-in-multipart recursion. Legitimate
// mail rarely nests beyond mixed > alternative > related; a crafted message
// can nest arbitrarily and would otherwise consume one stack frame per
// wrapper while hiding attachments below any scanner's patience.
const maxMIMENestingDepth = 16
// extractEncodedMultipart recursively walks MIME parts, extracting attachments.
// depth counts archive nesting (zip-in-zip), not MIME nesting: an archive
// attached five multipart levels down is still archive depth 0.
func extractEncodedMultipart(r io.Reader, cte, boundary string, limits Limits, result *ExtractionResult, totalSize *int64, depth, mimeDepth int) {
readers, readErr := transferReaders(cte, r, result)
if readErr != nil {
markPartial(result, fmt.Sprintf("could not decode multipart body: %v", readErr))
}
for _, reader := range readers {
body := &readErrRecorder{r: reader}
parseErr := extractMultipartNested(body, boundary, limits, result, totalSize, depth, mimeDepth)
// A closing MIME delimiter ends parsing before the transfer decoder has
// necessarily reported its error. Drain the bounded spool body (or this
// outer part only), including any epilogue, to finish the decoder.
_, _ = io.Copy(io.Discard, body)
if body.err != nil {
markPartial(result, fmt.Sprintf("could not decode multipart body: %v", body.err))
}
if parseErr != nil {
markPartial(result, fmt.Sprintf("could not parse multipart body: %v", parseErr))
}
}
}
func extractMultipartNested(r io.Reader, boundary string, limits Limits, result *ExtractionResult, totalSize *int64, depth, mimeDepth int) error {
if mimeDepth >= maxMIMENestingDepth {
result.Partial = true
if result.PartialReason == "" {
result.PartialReason = "MIME nesting exceeds depth limit"
}
return nil
}
mr := multipart.NewReader(r, boundary)
for {
part, err := mr.NextRawPart()
if err == io.EOF {
return nil
}
if err != nil {
return err
}
ct := part.Header.Get("Content-Type")
if ct == "" {
ct = "text/plain"
}
mediaType, params, _ := mime.ParseMediaType(ct)
cte := strings.ToLower(part.Header.Get("Content-Transfer-Encoding"))
// Recurse into nested multipart. RFC 2045 forbids encoding a
// multipart body, but a sender can still do it, so decode first.
if strings.HasPrefix(mediaType, "multipart/") {
if b := params["boundary"]; b != "" {
// Failure within one wrapper must not hide its outer siblings.
extractEncodedMultipart(part, cte, b, limits, result, totalSize, depth, mimeDepth+1)
}
continue
}
// Skip inline text bodies - only extract attachments
disp := part.Header.Get("Content-Disposition")
filename := part.FileName()
if filename == "" {
// Try Content-Disposition filename param
if disp != "" {
_, dparams, _ := mime.ParseMediaType(disp)
filename = dparams["filename"]
}
}
if filename == "" {
// Mail clients fall back to the Content-Type name parameter.
filename = params["name"]
}
if filename == "" {
// No filename and text/* content - this is a body part, skip
if strings.HasPrefix(mediaType, "text/") {
continue
}
// Non-text without filename - use generic name
filename = "unnamed_attachment"
}
rawFilename := filename
filename = SanitizeAttachmentName(filename)
readers, readErr := transferReaders(cte, part, result)
if readErr != nil {
markPartial(result, fmt.Sprintf("could not decode attachment %q: %v", filename, readErr))
}
for _, reader := range readers {
// Decode the part body based on Content-Transfer-Encoding
bodyReader := &readErrRecorder{r: reader}
// An empty attachment has nothing to scan. Staging it would use
// up the part cap and could hide a real attachment behind it.
body := bufio.NewReader(bodyReader)
if _, peekErr := body.Peek(1); peekErr != nil {
if bodyReader.err != nil {
markPartial(result, fmt.Sprintf("could not decode attachment %q: %v", filename, bodyReader.err))
}
continue
}
if partsFull(limits, result) {
return nil
}
tmpFile, tempPath, err := limits.createPart("csm-emailav-*")
if err != nil {
markPartial(result, "could not stage attachment for scanning")
return fmt.Errorf("creating temp file: %w", err)
}
limited := io.LimitReader(body, limits.MaxAttachmentSize+1)
n, err := io.Copy(tmpFile, limited)
if err != nil && err != bodyReader.err {
discardPart(tmpFile, tempPath)
markPartial(result, "could not stage attachment for scanning")
continue // fail-open: skip this part
}
if bodyReader.err != nil {
// Keep what decoded: a mail client shows those bytes, so they
// must be scanned even though the rest of the part is lost.
markPartial(result, fmt.Sprintf("could not decode attachment %q: %v", filename, bodyReader.err))
if n == 0 {
discardPart(tmpFile, tempPath)
continue
}
}
if n > limits.MaxAttachmentSize {
discardPart(tmpFile, tempPath)
result.Partial = true
result.PartialReason = fmt.Sprintf("attachment %q exceeds max size %d", filename, limits.MaxAttachmentSize)
continue
}
*totalSize += n
if *totalSize > limits.MaxExtractionSize {
discardPart(tmpFile, tempPath)
result.Partial = true
result.PartialReason = fmt.Sprintf("total extraction size exceeds %d bytes", limits.MaxExtractionSize)
return nil // stop extracting
}
result.Parts = append(result.Parts, ExtractedPart{
Filename: filename,
ContentType: mediaType,
Size: n,
File: tmpFile,
TempPath: tempPath,
})
expandArchive(tmpFile, n, rawFilename, filename, limits, result, totalSize, depth)
}
}
}
// expandArchive unpacks an archive found at depth (0 for an attachment, 1 for
// a member of an attached archive) while max_archive_depth allows it. Past the
// limit its members were never read, so the message is not fully scanned.
func expandArchive(f *os.File, size int64, rawName, safeName string, limits Limits, result *ExtractionResult, totalSize *int64, depth int) {
kind := archiveKindForAttachmentName(rawName)
if kind == "" {
return
}
if depth >= limits.MaxArchiveDepth {
markPartial(result, fmt.Sprintf("nested archive %q not expanded", safeName))
return
}
switch kind {
case "zip":
extractZIP(f, size, safeName, limits, result, totalSize, depth+1)
case "tar.gz":
extractTarGz(f, size, safeName, limits, result, totalSize, depth+1)
}
}
// maxZipDirectoryBytes bounds the central directory an attached zip may
// declare before any of it is read.
const maxZipDirectoryBytes = 16 << 20
// archiveWalkCap bounds how many entries or headers one archive walk visits,
// of any kind. max_archive_files bounds what is extracted; this bounds the
// work spent getting there.
func archiveWalkCap(limits Limits) int {
maxInt := int(^uint(0) >> 1)
if limits.MaxArchiveFiles > maxInt/10 {
return maxInt
}
return 10 * max(limits.MaxArchiveFiles, 1)
}
func markArchiveWalkCap(result *ExtractionResult, archiveName string, walkCap int) {
markPartial(result, fmt.Sprintf("archive %q lists more than %d entries", archiveName, walkCap))
}
func markExtractionBudget(result *ExtractionResult, limits Limits) {
result.Partial = true
result.PartialReason = fmt.Sprintf("total extraction size exceeds %d bytes", limits.MaxExtractionSize)
}
// errExtractionBudget stops a decompressing reader once the message's
// extraction budget is spent.
var errExtractionBudget = errors.New("extraction budget spent")
// budgetReader adds what it yields to the message's shared spent count and
// fails once that count passes limit, reading at most one byte past it.
// Nested archives add to the same count. The first error sticks.
type budgetReader struct {
r io.Reader
spent *int64
limit int64
err error
}
func (b *budgetReader) Read(p []byte) (int, error) {
if b.err != nil {
return 0, b.err
}
if *b.spent > b.limit {
b.err = errExtractionBudget
return 0, b.err
}
if remaining := b.limit - *b.spent; remaining < int64(len(p)) {
p = p[:remaining+1]
}
n, err := b.r.Read(p)
*b.spent += int64(n)
if *b.spent > b.limit {
err = errExtractionBudget
}
// io.ReadFull may suppress an error returned with its final bytes. Keep
// it visible to the archive driver even when tar.Next reports EOF.
if err != nil && err != io.EOF {
b.err = err
}
return n, err
}
// markArchiveUnopened records an attached archive whose members could not be
// reached at all.
func markArchiveUnopened(result *ExtractionResult, archiveName string, err error) {
markPartial(result, fmt.Sprintf("archive %q could not be opened: %v", archiveName, err))
}
// zipEncryptedFlag is bit 0 of the ZIP general purpose bit flag: the entry is
// encrypted. APPNOTE.TXT section 4.4.4.
const zipEncryptedFlag = 0x1
// extractZIP reads the archive from its staged part file, which stays open
// and owned by the result.
func extractZIP(part *os.File, size int64, archiveName string, limits Limits, result *ExtractionResult, totalSize *int64, depth int) {
// An archive that cannot be opened was not scanned inside: a mail client
// may still repair and open it, so it must never count as clean.
f := io.NewSectionReader(part, 0, size)
// Validate the listing itself before zip.NewReader allocates per entry;
// the sender controls both the directory and the end record's counts.
walkCap := archiveWalkCap(limits)
dir, err := zipdir.Read(f, size, maxZipDirectoryBytes)
if err != nil {
markArchiveUnopened(result, archiveName, err)
return
}
if err = validateZIPDirectory(f, size, dir, walkCap); err != nil {
if errors.Is(err, errArchiveWalk) {
markArchiveWalkCap(result, archiveName, walkCap)
} else {
markArchiveUnopened(result, archiveName, err)
}
return
}
zr, err := zip.NewReader(f, size)
if err != nil {
markArchiveUnopened(result, archiveName, err)
return
}
extracted, walked := 0, 0
for _, zf := range zr.File {
walked++
if walked > walkCap {
markArchiveWalkCap(result, archiveName, walkCap)
return
}
safeName := SanitizeAttachmentName(zf.Name)
// Bit 0 of the general purpose flag marks an encrypted entry. Both
// encryption schemes in the wild set it, which matters because they
// fail in different places otherwise: legacy ZipCrypto keeps the
// deflate method and fails during the read, while WinZip AES uses
// method 99 and fails when the entry is opened. Reading the flag
// catches both before either produces a misleading error.
if zf.Flags&zipEncryptedFlag != 0 {
if len(result.EncryptedEntries) < limits.MaxArchiveFiles {
result.EncryptedEntries = append(result.EncryptedEntries, EncryptedArchiveEntry{
ArchiveName: archiveName,
Filename: safeName,
})
} else {
result.EncryptedEntriesOmitted++
}
continue
}
if zf.FileInfo().IsDir() {
if dirErr := validateZIPDirectoryEntry(zf); dirErr != nil {
markPartial(result, fmt.Sprintf("directory %q in archive %q contains unread data", safeName, archiveName))
}
continue
}
if extracted >= limits.MaxArchiveFiles {
markPartial(result, fmt.Sprintf("archive %q exceeds max files %d", archiveName, limits.MaxArchiveFiles))
return
}
// A member costs whichever is larger, its compressed or its inflated
// size. Records can share one compressed stream that inflates to
// nothing, so the compressed size is charged before inflating.
before := *totalSize
compressed := int64(min(zf.CompressedSize64, uint64(limits.MaxExtractionSize)+1)) // #nosec G115 -- bounded by an int64 above.
if before+compressed > limits.MaxExtractionSize {
// Never opened, so nothing is spent; the budget is exhausted.
markExtractionBudget(result, limits)
return
}
rc, err := zf.Open()
if err != nil {
markPartial(result, fmt.Sprintf("could not decompress file %q in archive %q: %v", safeName, archiveName, err))
continue
}
if partsFull(limits, result) {
rc.Close()
return
}
tmpFile, tempPath, err := limits.createPart("csm-emailav-zip-*")
if err != nil {
rc.Close()
markPartial(result, fmt.Sprintf("could not stage file %q in archive %q for scanning: %v", safeName, archiveName, err))
continue
}
// Every decompressed byte spends the budget, kept or thrown away, so
// a bomb of oversize members cannot decompress without limit.
var inflated int64
budget := &budgetReader{r: rc, spent: &inflated, limit: limits.MaxExtractionSize - before}
limited := io.LimitReader(budget, limits.MaxAttachmentSize+1)
n, err := io.Copy(tmpFile, limited)
rc.Close()
*totalSize = before + max(compressed, inflated)
if errors.Is(err, errExtractionBudget) || *totalSize > limits.MaxExtractionSize {
discardPart(tmpFile, tempPath)
markExtractionBudget(result, limits)
return
}
if err != nil || n > limits.MaxAttachmentSize {
discardPart(tmpFile, tempPath)
if n > limits.MaxAttachmentSize {
result.Partial = true
result.PartialReason = fmt.Sprintf("file %q in archive exceeds max size", safeName)
} else {
markPartial(result, fmt.Sprintf("could not decompress file %q in archive %q: %v", safeName, archiveName, err))
}
continue
}
result.Parts = append(result.Parts, ExtractedPart{
Filename: safeName,
ContentType: "application/octet-stream",
Size: n,
File: tmpFile,
TempPath: tempPath,
Nested: true,
ArchiveName: archiveName,
})
extracted++
expandArchive(tmpFile, n, zf.Name, safeName, limits, result, totalSize, depth)
}
}
// extractTarGz reads the archive from its staged part file, which stays
// open and owned by the result.
func extractTarGz(part *os.File, size int64, archiveName string, limits Limits, result *ExtractionResult, totalSize *int64, depth int) {
compressed := bufio.NewReader(io.NewSectionReader(part, 0, size))
gr, err := gzip.NewReader(compressed)
if err != nil {
markArchiveUnopened(result, archiveName, err)
return
}
defer func() { _ = gr.Close() }()
// Count streams explicitly so empty streams cannot bypass either budget.
gr.Multistream(false)
// Everything the gzip stream yields spends the budget, including the
// remainder tar.Next reads through to skip an oversize member.
walkCap := archiveWalkCap(limits)
streams := &gzipStreamReader{compressed: compressed, gr: gr, walkCap: walkCap, streams: 1}
budget := &budgetReader{r: streams, spent: totalSize, limit: limits.MaxExtractionSize}
walk := &tarWalkReader{r: budget, walkCap: walkCap}
tr := tar.NewReader(walk)
extracted := 0
for {
hdr, err := tr.Next()
if budget.err != nil {
err = budget.err
}
if err == io.EOF {
err = finishTar(budget)
if err == nil {
return
}
}
if errors.Is(err, errExtractionBudget) {
markExtractionBudget(result, limits)
return
}
if errors.Is(err, errArchiveWalk) {
markArchiveWalkCap(result, archiveName, walkCap)
return
}
if err != nil {
// The members after a broken header were never read.
markPartial(result, fmt.Sprintf("archive %q could not be read: %v", archiveName, err))
return
}
if err = walk.setMember(hdr); err != nil {
markPartial(result, fmt.Sprintf("archive %q could not be read: %v", archiveName, err))
return
}
if hdr.Typeflag != tar.TypeReg {
if walk.remaining > 0 && hdr.Typeflag != tar.TypeXGlobalHeader {
markPartial(result, fmt.Sprintf("unsupported file %q in archive %q contains unread data", SanitizeAttachmentName(hdr.Name), archiveName))
}
continue
}
if extracted >= limits.MaxArchiveFiles {
result.Partial = true
result.PartialReason = fmt.Sprintf("archive %q exceeds max files %d", archiveName, limits.MaxArchiveFiles)
return
}
safeName := SanitizeAttachmentName(hdr.Name)
if partsFull(limits, result) {
return
}
tmpFile, tempPath, err := limits.createPart("csm-emailav-tgz-*")
if err != nil {
markPartial(result, fmt.Sprintf("could not stage file %q in archive %q for scanning: %v", safeName, archiveName, err))
continue
}
// Sparse tar readers synthesize hole bytes outside the gzip stream.
// Bound logical output too, then charge only the bytes the physical
// stream did not already spend.
before := *totalSize
var logicalN int64
logical := &budgetReader{r: tr, spent: &logicalN, limit: limits.MaxExtractionSize - before}
limited := io.LimitReader(logical, limits.MaxAttachmentSize+1)
n, err := io.Copy(tmpFile, limited)
*totalSize += max(logicalN-(*totalSize-before), 0)
if *totalSize > limits.MaxExtractionSize {
budget.err = errExtractionBudget
}
if budget.err != nil {
err = budget.err
}
if errors.Is(err, errExtractionBudget) {
discardPart(tmpFile, tempPath)
markExtractionBudget(result, limits)
return
}
if err != nil || n > limits.MaxAttachmentSize {
discardPart(tmpFile, tempPath)
if n > limits.MaxAttachmentSize {
result.Partial = true
result.PartialReason = fmt.Sprintf("file %q in archive exceeds max size", safeName)
} else {
markPartial(result, fmt.Sprintf("could not read file %q from archive %q: %v", safeName, archiveName, err))
}
continue
}
result.Parts = append(result.Parts, ExtractedPart{
Filename: safeName,
ContentType: "application/octet-stream",
Size: n,
File: tmpFile,
TempPath: tempPath,
Nested: true,
ArchiveName: archiveName,
})
extracted++
expandArchive(tmpFile, n, hdr.Name, safeName, limits, result, totalSize, depth)
}
}
// maxAttachmentNameBytes caps a name that reaches logs, alerts and the parse
// worker's replies.
const maxAttachmentNameBytes = 255
// SanitizeAttachmentName trims an attachment or archive-entry name to its
// base name, truncates at control characters, replaces invalid UTF-8 and
// caps it at 255 bytes before the name reaches logs, alerts, or JSON
// responses.
func SanitizeAttachmentName(name string) string {
name = strings.TrimSpace(TruncateUTF8(strings.ToValidUTF8(attachmentBaseName(name), "\uFFFD"), maxAttachmentNameBytes))
switch name {
case "", ".", "..", "/":
return "_unnamed_"
}
return name
}
func attachmentBaseName(name string) string {
name = strings.ReplaceAll(name, "\\", "/")
name = filepath.Base(name)
// Truncate at the first control character so a crafted entry
// like "good.txt\nFAKE-LOG-LINE" cannot smuggle a forged log
// record past the visible filename.
if i := strings.IndexFunc(name, unicode.IsControl); i >= 0 {
name = name[:i]
}
return name
}
// TruncateUTF8 cuts s to at most max bytes without splitting a rune.
func TruncateUTF8(s string, max int) string {
if len(s) <= max {
return s
}
cut := max
for cut > 0 && !utf8.RuneStart(s[cut]) {
cut--
}
return s[:cut]
}
// createPart opens the file for one staged part. tempPath is set only when
// the default sink put it on disk.
func (l Limits) createPart(prefix string) (f *os.File, tempPath string, err error) {
if l.PartSink != nil {
f, err = l.PartSink(prefix)
if err != nil && f != nil {
discardPart(f, "")
f = nil
}
return f, "", err
}
f, err = os.CreateTemp(l.TempDir, prefix)
if err != nil {
return nil, "", err
}
return f, f.Name(), nil
}
func discardPart(f *os.File, tempPath string) {
_ = f.Close()
if tempPath != "" {
_ = os.Remove(tempPath)
}
}
// partsFull reports, and records as partial, a message that already staged
// MaxParts parts.
func partsFull(limits Limits, result *ExtractionResult) bool {
if limits.MaxParts <= 0 || len(result.Parts) < limits.MaxParts {
return false
}
markPartial(result, fmt.Sprintf("message has more than %d attachments", limits.MaxParts))
return true
}
func archiveKindForAttachmentName(rawName string) string {
// The display cap can remove a real extension or leave a false one at
// the cut. Classify the full name with the same control-byte handling.
for _, name := range []string{strings.TrimSpace(attachmentBaseName(rawName)), archiveDetectionName(rawName)} {
lower := strings.ToLower(name)
switch {
case strings.HasSuffix(lower, ".zip"):
return "zip"
case strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz"):
return "tar.gz"
}
}
return ""
}
// archiveDetectionName removes controls instead of truncating so a
// filename like "payload\u0085.zip" still gets unpacked while the
// public filename remains log-safe.
func archiveDetectionName(name string) string {
name = strings.ReplaceAll(name, "\\", "/")
name = filepath.Base(name)
name = strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return -1
}
return r
}, name)
return strings.TrimSpace(name)
}
package mime
import (
"bytes"
"errors"
"io"
"strings"
)
// transferDecoder returns a reader that undoes a part's
// Content-Transfer-Encoding. Decoding is as lenient as mail clients are:
// anything a client would render as an attachment has to reach the scanners
// too, or a malformed encoding becomes a way to deliver unscanned content.
// Malformed input still decodes as far as clients decode it; the reader then
// reports an error so the part is marked incompletely scanned. Unknown
// encodings (7bit, 8bit, binary) pass through unchanged.
func transferDecoder(cte string, r io.Reader) io.Reader {
switch cte {
case "base64":
return &decodeReader{src: r, dec: &base64Decoder{}}
case "quoted-printable":
return &decodeReader{src: r, dec: &qpDecoder{}}
default:
return r
}
}
// transferReaders preserves client interpretations of malformed padding:
// consuming full quartets, ignoring misplaced padding, or stopping at valid
// padding. A long suffix can hide an otherwise readable archive, so scanning
// a continued decode cannot replace scanning the padded prefix. The source
// is a part of the already memory-bounded spool body.
func transferReaders(cte string, r io.Reader, result *ExtractionResult) ([]io.Reader, error) {
if cte != "base64" {
return []io.Reader{transferDecoder(cte, r)}, nil
}
body, err := io.ReadAll(r)
readers := []io.Reader{transferDecoder(cte, bytes.NewReader(body))}
var alternatives []io.Reader
if prefix := base64PaddedPrefix(body); prefix != nil {
alternatives = append(alternatives, transferDecoder(cte, bytes.NewReader(prefix)))
}
if ambiguousBase64Padding(body) {
alternatives = append(alternatives, &decodeReader{src: bytes.NewReader(body), dec: &base64AlphabetDecoder{}})
}
for _, alternative := range alternatives {
const maxTransferVariants = 16
if result.transferVariants >= maxTransferVariants {
markPartial(result, "transfer decoding exceeds alternate interpretation limit")
} else {
result.transferVariants++
readers = append(readers, alternative)
}
}
return readers, err
}
// Return a normally padded prefix only when meaningful data follows it.
func base64PaddedPrefix(body []byte) []byte {
slot := 0
padded := false
for i, b := range body {
_, alphabet := base64Value(b)
switch {
case b == '=':
if slot < 2 {
return nil
}
padded = true
case !alphabet:
continue
case padded:
return nil // the quartet itself has misplaced padding
}
slot++
if slot == 4 {
if padded {
for _, tail := range body[i+1:] {
if _, ok := base64Value(tail); ok || tail == '=' {
return body[:i+1]
}
}
return nil
}
slot = 0
}
}
return nil
}
func ambiguousBase64Padding(body []byte) bool {
slot, pads := 0, 0
for _, b := range body {
if b == '=' {
if slot < 2 {
return true
}
pads++
} else if _, ok := base64Value(b); !ok {
continue
} else if pads > 0 {
return true
}
slot++
if slot == 4 {
slot, pads = 0, 0
}
}
return false
}
type base64AlphabetDecoder struct{ base64Decoder }
func (d *base64AlphabetDecoder) feed(b byte, out []byte) []byte {
if b == '=' {
return out
}
return d.base64Decoder.feed(b, out)
}
var (
errBase64AfterPadding = errors.New("base64 data after padding")
errBase64Truncated = errors.New("base64 data ends with an incomplete byte")
errBase64Padding = errors.New("misplaced base64 padding")
)
// byteDecoder turns encoded bytes into decoded bytes one input byte at a
// time. finish flushes held state at end of input and reports malformed input
// that was nonetheless decoded.
type byteDecoder interface {
feed(b byte, out []byte) []byte
finish(out []byte) ([]byte, error)
}
// decodeReader drives a byteDecoder. Decoded output is drained before any
// error is returned, so a caller always sees every byte a client would.
type decodeReader struct {
src io.Reader
dec byteDecoder
buf [4096]byte
pending []byte
err error
}
func (d *decodeReader) Read(p []byte) (int, error) {
for len(d.pending) == 0 && d.err == nil {
n, err := d.src.Read(d.buf[:])
out := d.pending[:0]
for _, b := range d.buf[:n] {
out = d.dec.feed(b, out)
}
if err != nil {
var decodeErr error
out, decodeErr = d.dec.finish(out)
// A truncated MIME part returns UnexpectedEOF. Its buffered
// tail still carries bytes; preserve the source error too.
if err == io.EOF && decodeErr != nil {
err = decodeErr
}
}
d.pending = out
d.err = err
}
if len(d.pending) == 0 {
return 0, d.err
}
n := copy(p, d.pending)
d.pending = d.pending[n:]
return n, nil
}
// base64Decoder decodes quartet by quartet and ignores every byte outside
// the alphabet, as RFC 2045 section 6.8 requires. Padding occupies a slot
// in the quartet, even when misplaced. Resetting at '=' instead would shift
// all subsequent bytes compared with clients that consume full quartets.
type base64Decoder struct {
quad [4]byte
n int
padding int
badPadding bool
sawPad bool
afterPadData bool
}
func (d *base64Decoder) feed(b byte, out []byte) []byte {
v, ok := base64Value(b)
switch {
case b == '=':
if d.n < 2 {
d.badPadding = true
}
d.padding++
d.sawPad = true
case !ok:
return out
case d.sawPad:
d.afterPadData = true
}
d.quad[d.n] = v
d.n++
if d.n == 4 {
out = d.flush(out)
d.n = 0
d.padding = 0
}
return out
}
// Clients such as Thunderbird emit at least one byte even for a quartet
// containing three or four padding characters.
func (d *base64Decoder) flush(out []byte) []byte {
q := d.quad
out = append(out, q[0]<<2|q[1]>>4)
if d.padding < 2 {
out = append(out, q[1]<<4|q[2]>>2)
}
if d.padding == 0 {
out = append(out, q[2]<<6|q[3])
}
return out
}
func (d *base64Decoder) finish(out []byte) ([]byte, error) {
var err error
switch {
case d.n >= 2:
for d.n < 4 {
d.quad[d.n] = 0
d.n++
d.padding++
}
out = d.flush(out) // missing final padding
case d.n == 1:
err = errBase64Truncated
}
d.n = 0
if d.badPadding {
err = errBase64Padding
}
if d.afterPadData {
err = errBase64AfterPadding
}
return out, err
}
func base64Value(b byte) (byte, bool) {
switch {
case b >= 'A' && b <= 'Z':
return b - 'A', true
case b >= 'a' && b <= 'z':
return b - 'a' + 26, true
case b >= '0' && b <= '9':
return b - '0' + 52, true
case b == '+':
return 62, true
case b == '/':
return 63, true
}
return 0, false
}
// qpDecoder decodes quoted-printable without ever failing: an escape that is
// not two hex digits is kept literally, a soft line break may end in CR, LF or
// CRLF, raw control bytes pass through, and lines have no length limit.
// Whitespace before a line break is transport padding and is dropped
// (RFC 2045 section 6.7, rule 3).
type qpDecoder struct {
state qpState
hex1 byte
ws []byte // whitespace held until the next byte shows whether it ends a line
}
type qpState int
const (
qpText qpState = iota
qpEquals // saw '='
qpEqualsH1 // saw '=' and one hex digit
qpEqualsWS // saw '=' followed by whitespace: soft break if a line break follows
qpSoftCR // soft break ended in CR; swallow a following LF
)
func (d *qpDecoder) feed(b byte, out []byte) []byte {
switch d.state {
case qpEquals:
switch {
case isHexDigit(b):
d.hex1 = b
d.state = qpEqualsH1
return out
case b == ' ' || b == '\t':
d.ws = append(d.ws[:0], b)
d.state = qpEqualsWS
return out
case b == '\r':
d.state = qpSoftCR
return out
case b == '\n':
d.state = qpText
return out
}
out = append(out, '=')
d.state = qpText
case qpEqualsH1:
d.state = qpText
if isHexDigit(b) {
return append(out, hexValue(d.hex1)<<4|hexValue(b))
}
out = append(out, '=', d.hex1)
case qpEqualsWS:
switch b {
case ' ', '\t':
d.ws = append(d.ws, b)
return out
case '\r':
d.ws = d.ws[:0]
d.state = qpSoftCR
return out
case '\n':
d.ws = d.ws[:0]
d.state = qpText
return out
}
out = append(out, '=')
out = append(out, d.ws...)
d.ws = d.ws[:0]
d.state = qpText
case qpSoftCR:
d.state = qpText
if b == '\n' {
return out
}
}
switch b {
case '=':
out = append(out, d.ws...)
d.ws = d.ws[:0]
d.state = qpEquals
case ' ', '\t':
d.ws = append(d.ws, b)
case '\r', '\n':
d.ws = d.ws[:0]
out = append(out, b)
default:
out = append(out, d.ws...)
d.ws = d.ws[:0]
out = append(out, b)
}
return out
}
func (d *qpDecoder) finish(out []byte) ([]byte, error) {
switch d.state {
case qpEquals:
out = append(out, '=')
case qpEqualsH1:
out = append(out, '=', d.hex1)
}
// Trailing whitespace on the last line and a trailing "= " soft break
// carry no data.
d.ws = d.ws[:0]
d.state = qpText
return out, nil
}
func isHexDigit(b byte) bool {
return b >= '0' && b <= '9' || b >= 'A' && b <= 'F' || b >= 'a' && b <= 'f'
}
func hexValue(b byte) byte {
switch {
case b >= '0' && b <= '9':
return b - '0'
case b >= 'a' && b <= 'f':
return b - 'a' + 10
default:
return b - 'A' + 10
}
}
// readErrRecorder remembers the error its source returned, so a failed copy
// can be attributed to decoding rather than to writing the staged file.
type readErrRecorder struct {
r io.Reader
err error
}
func (rr *readErrRecorder) Read(p []byte) (int, error) {
n, err := rr.r.Read(p)
if err != nil && err != io.EOF {
rr.err = err
}
return n, err
}
// DecodeTransferVariants decodes data under Content-Transfer-Encoding cte the
// way attachment extraction does: leniently, and under every reading of
// ambiguous base64 a mail client could apply. Decode errors are ignored; each
// variant holds every byte that decoded.
func DecodeTransferVariants(cte string, data []byte) [][]byte {
readers, _ := transferReaders(strings.ToLower(strings.TrimSpace(cte)), bytes.NewReader(data), &ExtractionResult{})
variants := make([][]byte, 0, len(readers))
for _, r := range readers {
decoded, _ := io.ReadAll(r)
variants = append(variants, decoded)
}
return variants
}
package modsec
import "github.com/pidginhost/csm/internal/platform"
// RuleDirs returns the candidate directories where vendor ModSecurity rules
// live for the detected web server / panel combination. The list is ordered
// from most-specific to least-specific so callers walking the dirs encounter
// the operator's installed pack before any system fallback.
func RuleDirs(info platform.Info) []string {
var dirs []string
switch info.WebServer {
case platform.WSApache:
if info.IsDebianFamily() {
dirs = append(dirs,
"/etc/apache2/conf.d/modsec_vendor_configs/",
"/etc/modsecurity/",
"/usr/share/modsecurity-crs/rules/",
)
}
if info.IsRHELFamily() {
dirs = append(dirs,
"/etc/httpd/modsecurity.d/",
"/etc/httpd/modsecurity.d/activated_rules/",
"/usr/share/modsecurity-crs/rules/",
)
}
dirs = append(dirs, "/usr/local/apache/conf/modsec_vendor_configs/")
case platform.WSNginx:
dirs = append(dirs,
"/etc/nginx/modsec/",
"/etc/modsecurity/",
"/usr/share/modsecurity-crs/rules/",
)
}
// cPanel + LiteSpeed: cPanel's modsec_assemble job writes vendor rules
// into the apache2 tree even when the front-end is LiteSpeed. Without
// this branch the rule probe has no filesystem evidence during the
// window in which modsec_assemble itself is rewriting the tree.
if info.IsCPanel() && info.WebServer == platform.WSLiteSpeed {
dirs = append(dirs,
"/etc/apache2/conf.d/modsec_vendor_configs/",
"/usr/local/apache/conf/modsec_vendor_configs/",
)
}
return dirs
}
package modsec
import (
"net"
"regexp"
"strconv"
"strings"
"time"
)
// ErrorLogMessage requires the module marker directly after the server header.
// Other modules can log request text containing a complete ModSecurity message.
func ErrorLogMessage(line string) (string, bool) {
at, marker, isLiteSpeed := errorLogMarker(line)
if at < 0 {
return "", false
}
message := strings.TrimSpace(line[at+len(marker):])
if isLiteSpeed {
message = strings.TrimSpace(strings.TrimPrefix(message, modsecApacheMarker))
}
return message, isLiteSpeed
}
const (
modsecApacheMarker = "ModSecurity:"
modsecLiteSpeedMarker = "[MODSEC]"
)
// Each marker has its own header grammar. Message text cannot change which
// server wrote the header or extend it with another module's bracketed fields.
var (
errorLogPrefix = regexp.MustCompile(`^(?:` +
`[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} \[[a-z]+\] [0-9]+#[0-9]+: (?:\*[0-9]+ )?|` +
`(?:\[[A-Z][a-z]{2} [A-Z][a-z]{2} [ 0-9]?[0-9] [0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)? [0-9]{4}\][ \t]*)?` +
`(?:\[(?:security[23]:)?(?:emerg|alert|crit|error|warn|notice|info|debug|trace[1-8])\][ \t]*)?` +
`(?:\[pid [0-9]+(?::tid [0-9]+)?\][ \t]*)?` +
`)(?:\[client (?:[^\[\]\r\n]|\[[^\[\]\r\n]*\])*\][ \t]*)*$`)
liteSpeedErrorLogPrefix = regexp.MustCompile(`^` +
`(?:[0-9]{4}-[0-9]{2}-[0-9]{2}(?: [0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?)? )?` +
`(?:\[(?:DEBUG|INFO|NOTICE|WARN|WARNING|ERROR|CRIT|ALERT|EMERG)\][ \t]*)?` +
`(?:\[[0-9]+\][ \t]*)?(?:\[T[0-9]+\][ \t]*)?` +
`\[(?:[^\[\]\s:]+|\[[^\[\]\r\n]+\]):[^\[\]\s#]+#[^\[\]\r\n]+\][ \t]*$`)
)
// errorLogMarker finds the module marker that ends the server-written prefix
// of a ModSecurity log line, or -1.
func errorLogMarker(line string) (at int, marker string, isLiteSpeed bool) {
a := strings.Index(line, modsecApacheMarker)
l := strings.Index(line, modsecLiteSpeedMarker)
if l >= 0 && (a < 0 || l < a) {
at, marker, isLiteSpeed = l, modsecLiteSpeedMarker, true
} else {
at, marker = a, modsecApacheMarker
}
if at >= 0 {
prefix := errorLogPrefix
if isLiteSpeed {
prefix = liteSpeedErrorLogPrefix
}
if !prefix.MatchString(line[:at]) {
return -1, "", false
}
}
return at, marker, isLiteSpeed
}
// The time each server writes at the start of its error-log prefix.
var (
apacheErrorLogTime = regexp.MustCompile(`^\[([A-Z][a-z]{2} [A-Z][a-z]{2} [ 0-9]?[0-9] [0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)? [0-9]{4})\]`)
nginxErrorLogTime = regexp.MustCompile(`^([0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}) `)
liteSpeedErrorLogTime = regexp.MustCompile(`^([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?) `)
)
// ErrorLogTime reads the time the server wrote at the start of a ModSecurity
// log line's prefix, in the server's local time. A line without a validated
// prefix, or whose prefix has no time, has none.
func ErrorLogTime(line string) (time.Time, bool) {
at, _, isLiteSpeed := errorLogMarker(line)
if at < 0 {
return time.Time{}, false
}
prefix := line[:at]
re, layout := nginxErrorLogTime, "2006/01/02 15:04:05"
switch {
case isLiteSpeed:
re, layout = liteSpeedErrorLogTime, "2006-01-02 15:04:05"
case strings.HasPrefix(prefix, "["):
re, layout = apacheErrorLogTime, "Mon Jan 2 15:04:05 2006"
}
m := re.FindStringSubmatch(prefix)
if m == nil {
return time.Time{}, false
}
t, err := time.ParseInLocation(layout, m[1], time.Local)
return t, err == nil
}
// ErrorLogClientIP reads the client address from the server-written prefix
// of a ModSecurity log line. Rule data, the URI and the message after the
// marker are client text and never name the client.
func ErrorLogClientIP(line string) string {
at, _, isLiteSpeed := errorLogMarker(line)
if at < 0 {
return ""
}
prefix := line[:at]
if isLiteSpeed {
return liteSpeedClientIP(prefix)
}
beforeClient, rest, ok := strings.Cut(prefix, "[client ")
if !ok {
return ""
}
if strings.HasPrefix(rest, "[") {
// Bracketed IPv6 with a port: [client [2001:db8::5]:5555].
raw, _, closed := strings.Cut(rest, "] ")
if !closed {
raw, closed = strings.CutSuffix(rest, "]")
}
host, port, err := net.SplitHostPort(raw)
if !closed || err != nil {
return ""
}
if _, err := strconv.ParseUint(port, 10, 16); err != nil {
return ""
}
if ip := net.ParseIP(host); ip != nil {
return ip.String()
}
return ""
}
raw, _, ok := strings.Cut(rest, "]")
if !ok {
return ""
}
// Apache 2.4's module-qualified prefix identifies its unbracketed
// IP:port format. Legacy Apache and libmodsecurity log a bare IP;
// stripping a numeric IPv6 tail there would name a different host.
ip := net.ParseIP(raw)
apachePort := strings.Contains(beforeClient, "[security2:") || strings.Contains(beforeClient, "[security3:")
if ip != nil && (!apachePort || ip.To4() != nil) {
return ip.String()
}
if i := strings.LastIndexByte(raw, ':'); i > 0 {
if _, err := strconv.ParseUint(raw[i+1:], 10, 16); err == nil {
if hostIP := net.ParseIP(raw[:i]); hostIP != nil {
return hostIP.String()
}
}
}
if ip != nil {
return ip.String()
}
return ""
}
// liteSpeedClientIP accepts only the known IPv4 connection field. An unknown
// connection format must not let a later field choose another address.
func liteSpeedClientIP(prefix string) string {
for {
_, rest, ok := strings.Cut(prefix, "[")
if !ok {
return ""
}
field, tail, ok := strings.Cut(rest, "]")
if !ok {
return ""
}
prefix = tail
if !strings.ContainsAny(field, ":[#") {
continue
}
host, connection, ok := strings.Cut(field, ":")
if !ok {
return ""
}
ip := net.ParseIP(host)
if ip == nil || ip.To4() == nil {
return ""
}
port, connection, ok := strings.Cut(connection, "-")
if !ok {
return ""
}
if _, err := strconv.ParseUint(port, 10, 16); err != nil {
return ""
}
id, vhost, ok := strings.Cut(connection, "#")
if !ok || id == "" || vhost == "" {
return ""
}
return ip.String()
}
}
package modsec
import "strings"
// LogRule is the rule a ModSecurity error-log line names. Msg and Tags may
// contain client-controlled macro expansions and are not confidence evidence.
type LogRule struct {
ID, File, Msg, Hostname, URI string
Tags []string
}
// Only tags may repeat in the order both ModSecurity engines write fields.
// LiteSpeed writes the matched data as logdata where the engines write data.
var ruleFields = [...]string{"file", "line", "id", "rev", "msg", "data", "logdata", "severity", "ver",
"maturity", "accuracy", "tag", "hostname", "uri", "unique_id", "ref"}
// ErrorLogRule reads the rule fields of a ModSecurity error-log line. Both
// engines write client text before the fields: the matched variable's name,
// and in libmodsecurity 3 its value, which it also leaves unescaped in
// macro-expanded messages and the URI. Client text can add a field but never
// remove one the engine wrote, so repeated or out-of-order fields leave the
// rule unknowable and ok false. Unclosed fields are also unknown.
func ErrorLogRule(line string) (LogRule, bool) {
msg, isLiteSpeed := ErrorLogMessage(line)
type field struct {
name string
start, value int
}
var fields []field
last := -1
for off := 0; off < len(msg); {
pos := strings.IndexByte(msg[off:], '[')
if pos < 0 {
break
}
pos += off
off = pos + 1
for i, name := range ruleFields {
marker := "[" + name + ` "`
if !strings.HasPrefix(msg[pos:], marker) {
continue
}
if i < last || (i == last && name != "tag") {
return LogRule{}, false
}
last = i
off = pos + len(marker)
fields = append(fields, field{name: name, start: pos, value: off})
break
}
}
var rule LogRule
for i, f := range fields {
end := len(msg)
if i+1 < len(fields) {
end = fields[i+1].start
}
// The next field bounds the value. Quotes within msg/uri may be
// escaped by Apache or raw in v3; neither is a field delimiter.
value, closed := strings.CutSuffix(strings.TrimRight(msg[f.value:end], " \t"), `"]`)
nativeID := isLiteSpeed && strings.HasPrefix(msg, "mod_security rule ") && f.name == "id"
if nativeID || (i+1 == len(fields) && (f.name == "unique_id" || f.name == "ref")) {
// LiteSpeed's id is followed by its file location and action text.
// Terminal server fields can precede a connector's request suffix.
value, _, closed = strings.Cut(msg[f.value:], `"]`)
}
if !closed {
return LogRule{}, false
}
switch f.name {
case "id":
rule.ID = value
if isLiteSpeed {
if rest, ok := strings.CutPrefix(msg[f.value+len(value):], `"] at [`); ok {
if loc, _, ok := strings.Cut(rest, "]"); ok {
if colon := strings.LastIndexByte(loc, ':'); colon > 0 {
loc = loc[:colon]
}
rule.File = loc
}
}
}
case "file":
rule.File = value
case "msg":
rule.Msg = value
case "tag":
rule.Tags = append(rule.Tags, value)
case "hostname":
rule.Hostname = value
case "uri":
rule.URI = value
}
}
return rule, true
}
package modsec
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
)
// RuleTreeFingerprint summarises the rule files BuildRegistry would parse:
// their paths, contents and directory precedence. Content hashing catches
// vendor updates that preserve timestamps and changes behind .conf symlinks,
// while avoiding the rule parser's allocations and action extraction.
// An empty fingerprint means a read failed and must never skip a rebuild.
//
// present is false when none of the dirs exist. That is the signal that the
// resolved directories no longer describe this host (a web server swapped out,
// a vendor pack removed) and detection has to run again; it is not the same as
// a rule tree that exists and happens to be empty.
func RuleTreeFingerprint(dirs []string) (string, bool) {
digest := sha256.New()
present := false
complete := true
for _, dir := range dirs {
if dir == "" {
continue
}
if info, err := os.Stat(dir); err == nil && info.IsDir() {
present = true
}
fmt.Fprintf(digest, "dir:%q\n", dir)
err := walkRuleFiles(dir, func(path string, walkErr error) error {
if walkErr != nil {
return walkErr
}
fileDigest, err := readRuleFile(path, func(reader io.Reader) error {
_, err := io.Copy(io.Discard, reader)
return err
})
if err != nil {
return err
}
fmt.Fprintf(digest, "file:%q:%x\n", path, fileDigest)
return nil
})
if err != nil {
complete = false
}
}
if !complete {
return "", present
}
return hex.EncodeToString(digest.Sum(nil)), present
}
// Hash the same stream the consumer reads so a concurrent rewrite cannot
// associate parsed actions with a fingerprint of different file contents.
func readRuleFile(path string, consume func(io.Reader) error) ([]byte, error) {
f, err := openRuleFile(path)
if err != nil {
return nil, err
}
digest := sha256.New()
readErr := consume(&ruleFileReader{reader: io.TeeReader(f, digest)})
if err := errors.Join(readErr, f.Close()); err != nil {
return nil, err
}
return digest.Sum(nil), nil
}
// Tests inject read failures and appends at EOF through the file boundary.
var openRuleFile = func(path string) (io.ReadCloser, error) {
// #nosec G304 -- path comes from the operator's ModSec rule directories.
return os.Open(path)
}
// The parser and drain share one terminal result. Retrying an I/O error
// would misclassify an incomplete parse as cacheable. Reading past EOF
// could hash a concurrent append that the parser never saw.
type ruleFileReader struct {
reader io.Reader
err error
}
func (r *ruleFileReader) Read(p []byte) (int, error) {
if r.err != nil {
return 0, r.err
}
var n int
n, r.err = r.reader.Read(p)
return n, r.err
}
package modsec
import "strings"
// RuleMetadata contains the literal message and tags from a rule's action
// list. Macro references are removed: their names are not attack evidence,
// and their expanded values can be controlled by the request.
type RuleMetadata struct {
Message, Tags string
}
// Metadata returns configuration text, never macro-expanded log text. The
// value contains no mutable state and belongs to the same snapshot as Action.
func (r *Registry) Metadata(ruleID int) (RuleMetadata, bool) {
if r == nil {
return RuleMetadata{}, false
}
m, ok := r.metadata[ruleID]
return m, ok
}
func ruleMetadata(raw string) RuleMetadata {
// A chain's metadata belongs to its starter. Operators and chained
// conditions can contain msg/tag-like text without defining either.
line, _, _ := strings.Cut(raw, "\n")
var m RuleMetadata
var tags []string
for _, token := range tokenizeActionString(secRuleActions(line)) {
name, value, ok := strings.Cut(token, ":")
if !ok {
continue
}
name = strings.ToLower(strings.TrimSpace(name))
if name != "msg" && name != "tag" {
continue
}
value = strings.TrimSpace(value)
if strings.HasPrefix(value, "'") {
var rest string
value, rest, ok = configArgument(value)
if !ok || strings.TrimSpace(rest) != "" {
continue
}
}
value = literalMetadataText(value)
if name == "msg" {
m.Message = value
} else {
tags = append(tags, value)
}
}
m.Tags = strings.Join(tags, " ")
return m
}
func literalMetadataText(value string) string {
var literal strings.Builder
for {
before, rest, macro := strings.Cut(value, "%{")
literal.WriteString(before)
if !macro {
return literal.String()
}
// Keep the surrounding words separate rather than creating a keyword.
literal.WriteByte(' ')
_, value, _ = strings.Cut(rest, "}")
}
}
package modsec
import (
"bufio"
"fmt"
"io"
"log"
"os"
"sort"
"strconv"
"strings"
)
const overridesHeader = "# CSM ModSecurity Rule Overrides\n# Managed by CSM - do not edit manually.\n"
// WriteOverrides writes the overrides file with SecRuleRemoveById directives.
// Atomic write (tmp + rename). Sorts IDs for deterministic output.
func WriteOverrides(path string, disabledIDs []int) error {
sort.Ints(disabledIDs)
var sb strings.Builder
sb.WriteString(overridesHeader)
for _, id := range disabledIDs {
fmt.Fprintf(&sb, "SecRuleRemoveById %d\n", id)
}
tmpPath := path + ".tmp"
// #nosec G306 -- ModSec config files are read by the webserver process
// (Apache/nginx) which runs as a different user. 0640 lets root write
// and the webserver group read; world-read stays off.
if err := os.WriteFile(tmpPath, []byte(sb.String()), 0640); err != nil {
return fmt.Errorf("writing overrides tmp: %w", err)
}
if err := os.Rename(tmpPath, path); err != nil {
_ = os.Remove(tmpPath)
return fmt.Errorf("renaming overrides: %w", err)
}
return nil
}
// ReadOverrides reads the overrides file and returns disabled rule IDs.
// Returns empty list (not error) if the file does not exist.
func ReadOverrides(path string) ([]int, error) {
// #nosec G304 -- path is operator-configured ModSec overrides file.
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
defer f.Close()
var ids []int
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 0, 64*1024), maxModsecLineBytes)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if strings.HasPrefix(line, "SecRuleRemoveById ") {
idStr := strings.TrimPrefix(line, "SecRuleRemoveById ")
if id, err := strconv.Atoi(strings.TrimSpace(idStr)); err == nil && id >= 900000 && id <= 900999 {
ids = append(ids, id)
}
}
}
return ids, scanner.Err()
}
// ReadOverridesRaw reads the overrides file content for rollback purposes.
// Returns nil (not error) if the file does not exist.
func ReadOverridesRaw(path string) []byte {
// #nosec G304 -- path is operator-configured ModSec overrides file.
data, err := os.ReadFile(path)
if err != nil {
return nil
}
return data
}
// RestoreOverrides writes raw content back to the overrides file (for rollback).
// Uses atomic tmp+rename to prevent partial writes on crash.
func RestoreOverrides(path string, content []byte) error {
if content == nil {
// File didn't exist before - remove it
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("removing overrides during rollback: %w", err)
}
return nil
}
tmpPath := path + ".tmp"
// #nosec G306 -- see note in WriteOverrides: webserver-readable ModSec config.
if err := os.WriteFile(tmpPath, content, 0640); err != nil {
return fmt.Errorf("writing rollback tmp: %w", err)
}
if err := os.Rename(tmpPath, path); err != nil {
_ = os.Remove(tmpPath)
return fmt.Errorf("renaming rollback: %w", err)
}
return nil
}
// EnsureOverridesInclude appends an Include directive for the overrides file
// to a ModSecurity config file if not already present, and creates an empty
// overrides file if it doesn't exist. Idempotent: re-reads content under the
// write-open to avoid appending duplicate Include directives.
func EnsureOverridesInclude(rulesFile, overridesFile string) {
// Open for read+write to check-then-append atomically (same fd).
// #nosec G302 G304 -- webserver-readable ModSec rules file; 0640 means root
// can write and the webserver group can read. No world read.
f, err := os.OpenFile(rulesFile, os.O_RDWR|os.O_APPEND, 0640)
if err != nil {
return
}
data, err := io.ReadAll(f)
if err != nil {
_ = f.Close()
return
}
if !strings.Contains(string(data), overridesFile) {
if _, writeErr := fmt.Fprintf(f, "\n# CSM overrides - managed by CSM rule management\nInclude %s\n", overridesFile); writeErr != nil {
_ = f.Close()
return
}
}
// Close error here drops the appended Include directive, leaving the
// override file unreferenced -- worth surfacing instead of swallowing.
if closeErr := f.Close(); closeErr != nil {
log.Printf("modsec: overrides include close failed for %s: %v", rulesFile, closeErr)
}
// Create empty overrides file if it doesn't exist
if _, err := os.Stat(overridesFile); os.IsNotExist(err) {
// #nosec G306 -- see note in WriteOverrides: webserver-readable.
_ = os.WriteFile(overridesFile, []byte(overridesHeader), 0640)
}
}
package modsec
import (
"bufio"
"errors"
"fmt"
"io"
"os"
"regexp"
"strconv"
"strings"
)
// maxModsecLineBytes bounds a single logical line for the rule scanners. Far
// above any legitimate directive, but finite so a pathological file cannot
// drive an unbounded allocation.
const maxModsecLineBytes = 8 << 20 // 8 MiB
// Rule represents a parsed ModSecurity rule from the CSM custom config.
type Rule struct {
ID int // e.g. 900112
Description string // from msg:'...' field
Action string // disposition keyword: deny|drop|block|redirect|proxy|pause|allow|pass; "" if rule has only metadata (log, msg, ...) and inherits SecDefaultAction
StatusCode int // 403, 429, 0 (for pass)
Phase int // phase named in the action list, 1-5; 0 if none
Raw string // full rule text including chains
IsCounter bool // true if pass,nolog (bookkeeping rule, hidden in UI)
}
var (
reMsg = regexp.MustCompile(`msg:'([^']*)'`)
reStatus = regexp.MustCompile(`status:(\d+)`)
)
// defaultPhase is the phase ModSecurity uses for a rule or SecDefaultAction
// that does not name one.
const defaultPhase = 2
// dispositionPriority lists the ModSecurity action keywords that decide
// what happens to the request, ordered most-disruptive first. The first
// keyword present as a standalone token in the action string wins so an
// explicit disruptive action cannot be weakened by block inheritance.
// Metadata keywords (log, msg, severity, tag, ...) are
// intentionally excluded. A rule that carries only metadata has an empty
// Action; the registry leaves it out, so the LiteSpeed classifier treats it
// as unknown and defaults it to block. block itself names no disruptive
// action: the registry resolves it through SecDefaultAction.
var dispositionPriority = []string{
"deny", "drop", "redirect", "proxy", "pause", "block", "allow", "pass",
}
// dispositionSet is dispositionPriority as a lookup table. Pre-built for
// O(1) membership checks during action-string tokenisation.
var dispositionSet = func() map[string]struct{} {
m := make(map[string]struct{}, len(dispositionPriority))
for _, k := range dispositionPriority {
m[k] = struct{}{}
}
return m
}()
// ParseRulesFile reads a ModSecurity config file and extracts CSM-owned rules
// (IDs in 900000-900999). Use ParseRulesFileAll for the rule-action registry,
// which needs every rule including vendor packs.
func ParseRulesFile(path string) ([]Rule, error) {
all, err := ParseRulesFileAll(path)
if err != nil {
return nil, err
}
var csm []Rule
for _, r := range all {
if r.ID >= 900000 && r.ID <= 900999 {
csm = append(csm, r)
}
}
return csm, nil
}
// ParseRulesFileAll reads a ModSecurity config file and extracts every rule,
// regardless of ID range. Handles line continuations (\) and chained rules
// (chain action keyword). Vendor packs (Comodo, OWASP CRS, Imunify360) all
// use this entrypoint via the rule-action registry so the daemon can tell
// pass-action rules apart from deny rules.
func ParseRulesFileAll(path string) ([]Rule, error) {
// #nosec G304 -- path is operator-configured ModSec rules file.
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("opening rules file: %w", err)
}
defer f.Close()
rules, _, err := parseRules(f)
return rules, err
}
// parseRules returns the rules in a config file and the disruptive action
// each phase's SecDefaultAction in it declares. A later directive for the
// same phase replaces an earlier one.
func parseRules(reader io.Reader) ([]Rule, map[int]string, error) {
// Phase 1: Read lines, joining backslash continuations into logical lines.
var logicalLines []string
var current strings.Builder
appendCurrent := func(part string) error {
if current.Len()+len(part) > maxModsecLineBytes {
return fmt.Errorf("logical modsec line exceeds %d bytes", maxModsecLineBytes)
}
current.WriteString(part)
return nil
}
scanner := bufio.NewScanner(reader)
// Vendor packs (OWASP CRS, Comodo, Imunify360, cPanel modsec_assemble)
// ship assembled/minified directives that can exceed the default 64 KB
// token. Without a larger buffer Scan stops at ErrTooLong and the file's
// rules drop out of the action registry, where unknown IDs then default
// to "deny" and skew the modsec signal. Raise the ceiling so real files
// parse in full.
scanner.Buffer(make([]byte, 0, 64*1024), maxModsecLineBytes)
for scanner.Scan() {
raw := scanner.Text()
trimmed := strings.TrimSpace(raw)
if strings.HasSuffix(trimmed, "\\") {
// Continuation: strip trailing \ and keep accumulating.
if err := appendCurrent(strings.TrimSuffix(trimmed, "\\")); err != nil {
return nil, nil, err
}
if err := appendCurrent(" "); err != nil {
return nil, nil, err
}
continue
}
if err := appendCurrent(trimmed); err != nil {
return nil, nil, err
}
logicalLines = append(logicalLines, current.String())
current.Reset()
}
if current.Len() > 0 {
logicalLines = append(logicalLines, current.String())
}
if err := scanner.Err(); err != nil {
return nil, nil, err
}
// Phase 2: Group logical lines into blocks.
// Each block starts with a SecRule and may include chained SecRules.
// When a directive's action string contains "chain", the next SecRule
// is part of the same block.
var blocks []string
var block strings.Builder
chainPending := false
defaults := make(map[int]string)
var parseErr error
flushBlock := func() {
if block.Len() > 0 {
blocks = append(blocks, block.String())
block.Reset()
}
chainPending = false
}
for _, line := range logicalLines {
if rest, ok := directiveArgs(line, "SecDefaultAction"); ok {
arg, _, valid := configArgument(rest)
if !valid {
parseErr = errors.Join(parseErr, fmt.Errorf("malformed SecDefaultAction argument"))
continue
}
// SecDefaultAction must name a disruptive action other than
// block; ModSecurity refuses to load one that does not.
if action := pickDisposition(arg); action != "" && action != "block" {
phase := actionPhase(arg)
if phase == 0 {
phase = defaultPhase
}
defaults[phase] = action
}
continue
}
_, isRule := directiveArgs(line, "SecRule")
if isRule {
if block.Len() > 0 && !chainPending {
flushBlock()
}
} else if block.Len() == 0 {
continue // skip comments and blank lines outside blocks
}
if block.Len() > 0 || isRule {
block.WriteString(line)
block.WriteString("\n")
// Only update chainPending for SecRule lines - non-SecRule
// directives between chained rules must not reset the flag.
if isRule {
chainPending = hasChainAction(line)
}
}
}
flushBlock()
// Phase 3: Parse each block into a Rule.
var rules []Rule
for _, b := range blocks {
if r, ok := parseBlock(b); ok {
rules = append(rules, r)
}
}
return rules, defaults, parseErr
}
// hasChainAction reads only the SecRule action argument: a regex or message
// containing chain must not swallow the next independent rule.
func hasChainAction(line string) bool {
for _, token := range tokenizeActionString(secRuleActions(line)) {
if strings.EqualFold(token, "chain") {
return true
}
}
return false
}
func parseBlock(block string) (Rule, bool) {
line, _, _ := strings.Cut(block, "\n")
actions := secRuleActions(line)
id := 0
for _, token := range tokenizeActionString(actions) {
name, value, ok := strings.Cut(token, ":")
if ok && strings.EqualFold(strings.TrimSpace(name), "id") {
id, _ = strconv.Atoi(strings.Trim(strings.TrimSpace(value), `'"`))
break
}
}
if id <= 0 {
return Rule{}, false
}
r := Rule{
ID: id,
Raw: strings.TrimSpace(block),
Action: pickDisposition(actions),
Phase: actionPhase(actions),
}
if mm := reMsg.FindStringSubmatch(block); mm != nil {
r.Description = mm[1]
}
if r.Action == "pass" && strings.Contains(strings.ToLower(actions), "nolog") {
r.IsCounter = true
}
if sm := reStatus.FindStringSubmatch(block); sm != nil {
r.StatusCode, _ = strconv.Atoi(sm[1])
}
return r, true
}
// Rule identity and disposition come from the third argument, never from
// an id-like substring in the operator or a chained rule's actions.
func secRuleActions(line string) string {
rest, ok := directiveArgs(line, "SecRule")
if !ok {
return ""
}
var arg string
for range 3 {
arg, rest, ok = configArgument(rest)
if !ok {
return ""
}
}
return arg
}
func directiveArgs(line, directive string) (string, bool) {
if len(line) <= len(directive) || !strings.EqualFold(line[:len(directive)], directive) {
return "", false
}
rest := line[len(directive):]
if rest[0] != ' ' && rest[0] != '\t' {
return "", false
}
return strings.TrimLeft(rest, " \t"), true
}
// Decode the outer Apache quoting before tokenizing the action list. A
// doubled backslash can escape an inner apostrophe only after this layer
// is removed; otherwise a following deny could look like message text.
func configArgument(s string) (string, string, bool) {
s = strings.TrimLeft(s, " \t")
if s == "" {
return "", "", false
}
if quote := s[0]; quote == '\'' || quote == '"' {
for i := 1; i < len(s); i++ {
switch s[i] {
case '\\':
if i+1 < len(s) && (s[i+1] == quote || s[i+1] == '\\') {
i++
}
case quote:
return decodeConfigArgument(s[1:i], quote), s[i+1:], true
}
}
return "", "", false
}
if i := strings.IndexAny(s, " \t"); i >= 0 {
return decodeConfigArgument(s[:i], 0), s[i:], true
}
return decodeConfigArgument(s, 0), "", true
}
func decodeConfigArgument(s string, quote byte) string {
var decoded strings.Builder
for i := 0; i < len(s); i++ {
if s[i] == '\\' && i+1 < len(s) && (s[i+1] == '\\' || (quote != 0 && s[i+1] == quote)) {
i++
}
decoded.WriteByte(s[i])
}
return decoded.String()
}
// pickDisposition returns the ModSecurity disposition keyword present in
// the action string, preferring more-disruptive keywords when several are
// present (defensive: a rule labelled "deny" wins over a stray "allow").
// Returns "" if the action string carries only metadata and the rule
// therefore inherits SecDefaultAction.
func pickDisposition(actionStr string) string {
tokens := tokenizeActionString(actionStr)
seen := make(map[string]struct{}, len(tokens))
for _, t := range tokens {
name := t
if i := strings.IndexByte(name, ':'); i >= 0 {
name = name[:i]
}
name = strings.ToLower(strings.TrimSpace(name))
if _, ok := dispositionSet[name]; ok {
seen[name] = struct{}{}
}
}
for _, kw := range dispositionPriority {
if _, ok := seen[kw]; ok {
return kw
}
}
return ""
}
// actionPhase returns the phase an action list names, or 0 if it names none.
// ModSecurity also accepts request, response and logging for 2, 4 and 5.
func actionPhase(actionStr string) int {
phase := 0
for _, tok := range tokenizeActionString(actionStr) {
name, value, ok := strings.Cut(tok, ":")
if !ok || !strings.EqualFold(strings.TrimSpace(name), "phase") {
continue
}
switch v := strings.ToLower(strings.Trim(strings.TrimSpace(value), "'")); v {
case "request":
phase = 2
case "response":
phase = 4
case "logging":
phase = 5
default:
if n, err := strconv.Atoi(v); err == nil && n >= 1 && n <= 5 {
phase = n
}
}
}
return phase
}
// tokenizeActionString splits a ModSecurity action list on top-level commas
// while respecting single-quoted string values (msg:'foo, bar', logdata:'...'),
// where commas are part of the literal and must not be treated as token
// separators. Backslash-escaped quotes inside the literals are preserved.
func tokenizeActionString(s string) []string {
var out []string
var cur strings.Builder
inSingle := false
escape := false
for _, r := range s {
switch {
case escape:
cur.WriteRune(r)
escape = false
case r == '\\':
cur.WriteRune(r)
escape = true
case r == '\'':
inSingle = !inSingle
cur.WriteRune(r)
case r == ',' && !inSingle:
tok := strings.TrimSpace(cur.String())
if tok != "" {
out = append(out, tok)
}
cur.Reset()
default:
cur.WriteRune(r)
}
}
if tok := strings.TrimSpace(cur.String()); tok != "" {
out = append(out, tok)
}
return out
}
// IsBlockingAction reports whether an action causes the request to be denied
// or otherwise diverted away from normal processing. Used by the LiteSpeed
// log-line classifier - error_log records every match as "triggered!"
// regardless of action, so the action lookup is the only way to tell a real
// deny apart from a pass-action informational rule. redirect, proxy and
// pause are disruptive: the original request never reaches the upstream
// application as intended, so they are classified the same as deny. The
// registry keeps block only when incomplete setup files prevent resolution;
// it must remain blocking until the default can be determined.
func IsBlockingAction(action string) bool {
switch action {
case "deny", "drop", "block", "redirect", "proxy", "pause":
return true
}
return false
}
package modsec
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"sync/atomic"
)
// Registry maps parsed ModSecurity rule IDs with a decisive disposition to
// that action (deny, drop, redirect, proxy, pause, pass, allow). A block rule
// maps to the disruptive action it inherits from SecDefaultAction, or stays
// block when the applicable setup files could not be read or parsed. Rules
// with only metadata actions are intentionally absent so callers use the
// unknown-rule default. It is consulted by the LiteSpeed
// log-line classifier - error_log records every match as "triggered!"
// regardless of whether the rule's action denied the request, so the action
// lookup is the only signal that distinguishes a real deny from a noisy
// pass-action informational rule. Literal message and tag metadata is kept
// for every parsed rule, including those without a decisive disposition.
type Registry struct {
actions map[int]string
metadata map[int]RuleMetadata
fingerprint string
}
// Action returns the declared action for ruleID and whether it is known.
// An unknown ID is the safe default: callers should treat it as a potential
// block when they have a populated registry but not this specific rule.
func (r *Registry) Action(ruleID int) (action string, known bool) {
if r == nil {
return "", false
}
a, ok := r.actions[ruleID]
return a, ok
}
// Len returns the number of rules in the registry. Useful for startup
// telemetry: zero typically means rule directories are missing.
func (r *Registry) Len() int {
if r == nil {
return 0
}
return len(r.actions)
}
// Fingerprint identifies the exact file contents used to build the registry.
// Empty means the build was incomplete and must not be cached.
func (r *Registry) Fingerprint() string {
if r == nil {
return ""
}
return r.fingerprint
}
// BuildRegistry walks every directory in dirs (recursively), parses each
// .conf file, and returns a Registry mapping rule IDs
// to actions. Read and parse errors are returned alongside the usable rules;
// a vendor pack with one malformed file should not blank the whole registry,
// but only builds that read every file in full can be cached as unchanged.
//
// Precedence: dirs is treated as most-specific-first. Within a single
// directory, files are walked in lexical order and a duplicate rule ID
// uses last-write-wins, mirroring how ModSecurity itself resolves two
// SecRule directives that share an ID. Across directories, the first
// directory to define a rule keeps it - that way an operator override in
// /etc/apache2/conf.d/modsec_vendor_configs/ is not silently replaced by
// a stale system fallback in /usr/share/modsecurity-crs/rules/.
//
// A block rule resolves to the disruptive action of the SecDefaultAction
// for its phase, or pass, ModSecurity's own default, when none applies.
// ModSecurity applies a default to the rules loaded after it in the same
// configuration context. CSM does not read the Include directives that set
// load order, so a default applies to the directory tree that holds it: a
// pack's setup file governs the rules beside and below it, and the nearest
// declaring directory wins per phase. A default never reaches a sibling
// directory or another entry in dirs. A rule loaded before a default in the
// same tree still resolves to it.
func BuildRegistry(dirs []string) (*Registry, error) {
actions := make(map[int]string)
metadata := make(map[int]RuleMetadata)
claimed := make(map[int]struct{})
digest := sha256.New()
// Read failures and parse failures are not the same for caching. Bytes
// that could not be read leave the tree unknown, so the build must not be
// cached. A file that was read in full but cannot be parsed -- a vendor
// file past the line ceiling, a truncated rule -- produces the same
// result on every pass, so it must not force a reparse every refresh.
var readErr, parseErr error
for _, dir := range dirs {
if dir == "" {
continue
}
// Rules resolve after the walk, once every default in the tree is known.
var found []foundRule
defaults := make(map[string]map[int]string) // directory -> phase -> action
incomplete := make(map[string]bool)
fmt.Fprintf(digest, "dir:%q\n", dir)
walkErr := walkRuleFiles(dir, func(path string, walkErr error) error {
fileDir := filepath.Dir(path)
if walkErr != nil {
incomplete[fileDir] = true
return walkErr
}
var rules []Rule
var fileDefaults map[int]string
var fileParseErr error
fileDigest, err := readRuleFile(path, func(reader io.Reader) error {
rules, fileDefaults, fileParseErr = parseRules(reader)
// Read the rest even when the parser stopped early, so the
// digest always describes the whole file and matches what
// RuleTreeFingerprint computes for the same tree.
_, drainErr := io.Copy(io.Discard, reader)
return drainErr
})
if err != nil {
incomplete[fileDir] = true
return err
}
fmt.Fprintf(digest, "file:%q:%x\n", path, fileDigest)
if fileParseErr != nil {
incomplete[fileDir] = true
parseErr = errors.Join(parseErr, fmt.Errorf("%s: %w", path, fileParseErr))
}
for phase, action := range fileDefaults {
if defaults[fileDir] == nil {
defaults[fileDir] = make(map[int]string)
}
defaults[fileDir][phase] = action
}
for _, r := range rules {
found = append(found, foundRule{id: r.ID, action: r.Action, phase: r.Phase, dir: fileDir, metadata: ruleMetadata(r.Raw)})
}
return nil
})
perDirActions := make(map[int]string)
perDirMetadata := make(map[int]RuleMetadata)
perDirClaimed := make(map[int]struct{})
root := filepath.Clean(dir)
for _, r := range found {
perDirClaimed[r.id] = struct{}{}
perDirMetadata[r.id] = r.metadata
action := r.action
if action == "block" {
action = inheritedAction(defaults, incomplete, root, r.dir, r.phase)
}
if action != "" {
perDirActions[r.id] = action
} else {
delete(perDirActions, r.id)
}
}
// Promote the per-directory map into the global map only for IDs
// that no earlier (more-specific) directory has already claimed.
for id := range perDirClaimed {
if _, exists := claimed[id]; !exists {
claimed[id] = struct{}{}
metadata[id] = perDirMetadata[id]
action, hasAction := perDirActions[id]
if !hasAction {
continue
}
actions[id] = action
}
}
readErr = errors.Join(readErr, walkErr)
}
reg := &Registry{actions: actions, metadata: metadata}
if readErr == nil {
reg.fingerprint = hex.EncodeToString(digest.Sum(nil))
}
return reg, errors.Join(readErr, parseErr)
}
type foundRule struct {
id int
action string
phase int
dir string
metadata RuleMetadata
}
// inheritedAction returns the disruptive action of the nearest default for
// phase declared in dir or one of its parents up to root, or pass. Missing
// setup content could override a known default, so unresolved blocks stay
// conservative instead of being treated as known pass actions.
func inheritedAction(defaults map[string]map[int]string, incomplete map[string]bool, root, dir string, phase int) string {
if phase == 0 {
phase = defaultPhase
}
for {
if incomplete[dir] {
return "block"
}
if action, ok := defaults[dir][phase]; ok {
return action
}
parent := filepath.Dir(dir)
if dir == root || parent == dir {
return "pass"
}
dir = parent
}
}
// Both the parser and fingerprint must see the same paths and precedence.
// Follow linked packs using their logical paths so defaults stay scoped to
// the configured tree. Track ancestors, not all visited directories: two
// aliases of the same pack may inherit different defaults.
func walkRuleFiles(dir string, visit func(string, error) error) error {
var readErr error
visitFile := func(path string, err error) {
if visitErr := visit(path, err); visitErr != nil {
readErr = errors.Join(readErr, fmt.Errorf("%s: %w", path, visitErr))
}
}
var walk func(string, []fs.FileInfo)
walk = func(path string, ancestors []fs.FileInfo) {
info, err := os.Stat(path)
if err != nil {
// Candidate directories commonly do not exist on this platform.
// A failure inside an existing tree means the walk is incomplete.
if path != dir || !errors.Is(err, fs.ErrNotExist) {
visitFile(path, err)
}
return
}
if !info.IsDir() {
if strings.HasSuffix(strings.ToLower(filepath.Base(path)), ".conf") {
visitFile(path, nil)
}
return
}
for _, ancestor := range ancestors {
if os.SameFile(info, ancestor) {
visitFile(path, fmt.Errorf("directory symlink cycle"))
return
}
}
entries, err := os.ReadDir(path)
if err != nil {
visitFile(path, err)
return
}
ancestors = append(ancestors, info)
for _, entry := range entries {
walk(filepath.Join(path, entry.Name()), ancestors)
}
}
dir = filepath.Clean(dir)
walk(dir, nil)
return readErr
}
var globalRegistry atomic.Pointer[Registry]
// SetGlobal installs r as the daemon-wide registry. Callers are expected to
// rebuild and re-set on a refresh interval. Safe for concurrent use.
func SetGlobal(r *Registry) {
globalRegistry.Store(r)
}
// ReplaceGlobal installs r as the daemon-wide registry, EXCEPT when r is empty
// (or nil) while the currently-installed registry is non-empty: in that case
// the previous registry is kept and false is returned.
//
// The vendor rule tree is transiently empty or unreadable during cPanel's
// nightly modsec_assemble rewrite and during a boot-time web-server
// mis-detection window (a LiteSpeed host probed before lsws has finished
// starting resolves to the wrong rule directories). Replacing a populated
// registry with an empty one would discard known pass and deny actions until
// the next successful refresh.
//
// Refresh callers should use this instead of SetGlobal. Returns true if r was
// installed, false if the previous registry was kept.
func ReplaceGlobal(r *Registry) bool {
for {
prev := globalRegistry.Load()
if r == nil || r.Len() == 0 {
if prev != nil && prev.Len() > 0 {
return false
}
}
if globalRegistry.CompareAndSwap(prev, r) {
return true
}
}
}
// Global returns the currently installed registry, or nil if none has been
// set yet (e.g. during very early daemon startup, or in unit tests that
// did not seed one). Callers must nil-check.
func Global() *Registry {
return globalRegistry.Load()
}
// ResetGlobalForTest clears the global registry. Test-only helper.
func ResetGlobalForTest() {
globalRegistry.Store(nil)
}
package modsec
import (
"context"
"errors"
"fmt"
"os/exec"
"time"
)
const reloadTimeout = 30 * time.Second
// Reload executes the configured web server reload command.
// Returns combined stdout+stderr output and any error.
func Reload(command string) (string, error) {
if command == "" {
return "", errors.New("reload command is empty")
}
ctx, cancel := context.WithTimeout(context.Background(), reloadTimeout)
defer cancel()
// Run through shell to support compound commands, quoted paths, etc.
// #nosec G204 -- `command` is the operator-configured reload command
// from csm.yaml (e.g. "apachectl graceful"), loaded at daemon startup
// from a root-owned config. Not webui-settable.
cmd := exec.CommandContext(ctx, "sh", "-c", command)
out, err := cmd.CombinedOutput()
output := string(out)
if ctx.Err() == context.DeadlineExceeded {
return output, fmt.Errorf("reload timed out after %v", reloadTimeout)
}
if err != nil {
return output, fmt.Errorf("reload failed: %w (output: %s)", err, output)
}
return output, nil
}
// Package mysqlclient wraps the database/sql + go-sql-driver/mysql
// pair for the read-only queries CSM issues against host-local
// MySQL/MariaDB. It replaces the per-call `mysql -e <query>`
// shell-outs across CMS DB scans, performance metrics, and forensic
// dumps so the daemon no longer forks and tears down a child process
// (plus its libc/libmariadbclient relocations) every query.
//
// Two cred modes are supported:
//
// - Root: implicit, mirrors the historical `mysql` CLI behaviour
// that reads /root/.my.cnf [client] section when present. RootDB
// parses that file and returns a *sql.DB pointed at unix-socket
// auth, falling back to the mysql CLI's root socket-auth default
// when absent.
//
// - Per-account: the caller passes explicit user / password / host
// / dbname / port / socket via DSN. PerAccountQuery opens,
// runs, closes.
//
// Output shape mirrors the previous `mysql -N -B -e` shell-out: rows
// are returned as []string where each entry is the tab-joined column
// values of one result row, with MySQL batch-mode escaping applied.
// Existing scanner code paths consume this format unchanged.
package mysqlclient
import (
"bufio"
"context"
"database/sql"
"fmt"
"net"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/go-sql-driver/mysql"
)
// queryTimeout caps a single SELECT to a defensive 30 s -- well under
// the legacy 2 min CLI cmdTimeout but enough for any realistic CMS
// scan / SHOW STATUS / SHOW PROCESSLIST result.
const queryTimeout = 30 * time.Second
// Creds carries per-account database credentials. Mirrors wpDBCreds in
// internal/checks so callers can pass the same shape without an extra
// adapter type.
type Creds struct {
User string
Password string
Host string
Port int
Socket string
DBName string
}
// dsn returns a go-sql-driver/mysql connection string. Empty Host
// falls back to the Unix socket path the mysql CLI uses on cPanel
// hosts; this matches the historical default when -h was omitted.
func (c Creds) dsn() string {
cfg := mysql.NewConfig()
cfg.User = c.User
cfg.Passwd = c.Password
cfg.DBName = c.DBName
// Read timeouts mirror queryTimeout's budget.
cfg.Timeout = 5 * time.Second
cfg.ReadTimeout = queryTimeout
cfg.WriteTimeout = 5 * time.Second
cfg.Loc = time.Local
cfg.Net, cfg.Addr = c.networkAddr()
return cfg.FormatDSN()
}
func (c Creds) networkAddr() (string, string) {
host := strings.TrimSpace(c.Host)
socket := strings.TrimSpace(c.Socket)
if socket != "" && (host == "" || host == "localhost") {
return "unix", socket
}
if strings.HasPrefix(host, "/") {
return "unix", host
}
if _, socket, ok := splitHostSocket(host); ok {
return "unix", socket
}
if host == "" || host == "localhost" {
return "unix", defaultUnixSocket()
}
host, port := splitHostPort(host, c.Port)
return "tcp", net.JoinHostPort(host, strconv.Itoa(port))
}
func splitHostSocket(host string) (string, string, bool) {
idx := strings.LastIndex(host, ":/")
if idx < 0 || idx == len(host)-1 {
return "", "", false
}
return host[:idx], host[idx+1:], true
}
func splitHostPort(host string, fallbackPort int) (string, int) {
if h, p, err := net.SplitHostPort(host); err == nil {
if port, perr := strconv.Atoi(p); perr == nil {
return trimIPv6Brackets(h), port
}
}
if strings.Count(host, ":") == 1 {
h, p, _ := strings.Cut(host, ":")
if port, err := strconv.Atoi(p); err == nil {
return h, port
}
}
if fallbackPort == 0 {
fallbackPort = 3306
}
return trimIPv6Brackets(host), fallbackPort
}
func trimIPv6Brackets(host string) string {
return strings.TrimSuffix(strings.TrimPrefix(host, "["), "]")
}
// defaultUnixSocket returns the first known mysql socket path that
// exists on disk, falling back to the cPanel canonical location.
func defaultUnixSocket() string {
candidates := []string{
"/var/lib/mysql/mysql.sock",
"/tmp/mysql.sock",
"/var/run/mysqld/mysqld.sock",
}
for _, p := range candidates {
if _, err := os.Stat(p); err == nil {
return p
}
}
return "/var/lib/mysql/mysql.sock"
}
// PerAccountQuery opens a short-lived connection with the supplied
// credentials, runs the query, and returns each row as a tab-joined
// string. Empty result set returns (nil, nil). Errors include open,
// query, and scan failures.
//
// Tests can intercept via SetPerAccountQueryForTest.
func PerAccountQuery(ctx context.Context, creds Creds, query string, args ...any) ([]string, error) {
if fn := getPerAccountQueryMock(); fn != nil {
return fn(ctx, creds, query, args...)
}
db, err := sql.Open("mysql", creds.dsn())
if err != nil {
return nil, fmt.Errorf("mysqlclient: open: %w", err)
}
defer func() { _ = db.Close() }()
// Single short-lived call: cap idle pool to avoid leaking
// connections on per-account scans across many tenants.
db.SetMaxOpenConns(1)
db.SetMaxIdleConns(0)
db.SetConnMaxLifetime(queryTimeout)
return runQuery(ctx, db, query, args...)
}
// PerAccountQueryFunc is the signature SetPerAccountQueryForTest accepts.
type PerAccountQueryFunc func(ctx context.Context, creds Creds, query string, args ...any) ([]string, error)
var (
perAccountMockMu sync.RWMutex
perAccountMock PerAccountQueryFunc
)
// SetPerAccountQueryForTest installs an interceptor for
// PerAccountQuery. Pass nil to clear and restore the real database/sql
// path. Production code paths must NOT call this.
func SetPerAccountQueryForTest(fn PerAccountQueryFunc) {
perAccountMockMu.Lock()
defer perAccountMockMu.Unlock()
perAccountMock = fn
}
func getPerAccountQueryMock() PerAccountQueryFunc {
perAccountMockMu.RLock()
defer perAccountMockMu.RUnlock()
return perAccountMock
}
// runQuery is the shared execution path for any *sql.DB. Internal so
// the package can grow a RootDB-backed singleton later without
// duplicating the row-iteration code.
func runQuery(ctx context.Context, db *sql.DB, query string, args ...any) ([]string, error) {
cctx, cancel := context.WithTimeout(ctx, queryTimeout)
defer cancel()
rows, err := db.QueryContext(cctx, query, args...)
if err != nil {
return nil, fmt.Errorf("mysqlclient: query: %w", err)
}
defer func() { _ = rows.Close() }()
cols, err := rows.Columns()
if err != nil {
return nil, fmt.Errorf("mysqlclient: columns: %w", err)
}
out := make([]string, 0)
raw := make([]sql.NullString, len(cols))
scanArgs := make([]any, len(cols))
for i := range raw {
scanArgs[i] = &raw[i]
}
for rows.Next() {
if err := rows.Scan(scanArgs...); err != nil {
return nil, fmt.Errorf("mysqlclient: scan: %w", err)
}
parts := make([]string, len(cols))
for i, v := range raw {
if v.Valid {
parts[i] = mysqlBatchEscape(v.String)
} else {
// `mysql -N -B` prints NULL for SQL NULL; preserve
// that so legacy parsers that key off the literal
// see the same bytes.
parts[i] = "NULL"
}
}
out = append(out, strings.Join(parts, "\t"))
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("mysqlclient: iterate: %w", err)
}
return out, nil
}
func mysqlBatchEscape(s string) string {
var b strings.Builder
for i := 0; i < len(s); i++ {
switch s[i] {
case 0:
b.WriteString(`\0`)
case '\n':
b.WriteString(`\n`)
case '\r':
b.WriteString(`\r`)
case '\t':
b.WriteString(`\t`)
case '\\':
b.WriteString(`\\`)
default:
b.WriteByte(s[i])
}
}
return b.String()
}
// BatchUnescape is the exact inverse of mysqlBatchEscape (and of `mysql -N -B`
// batch-mode output): it turns the escaped sequences \0 \n \r \t \\ back into
// their raw bytes. Because the escaper backslash-escapes every literal
// backslash, a backslash in batch output is always the start of one of these
// sequences, so the inverse is unambiguous and round-trips byte-for-byte.
//
// Callers that write a value read from batch output back to the database must
// unescape first; otherwise a real newline persists as the two-byte literal
// "\n", corrupting the value and breaking PHP-serialized length prefixes.
func BatchUnescape(s string) string {
if !strings.ContainsRune(s, '\\') {
return s
}
var b strings.Builder
b.Grow(len(s))
for i := 0; i < len(s); i++ {
if s[i] != '\\' || i+1 >= len(s) {
b.WriteByte(s[i])
continue
}
i++
switch s[i] {
case '0':
b.WriteByte(0)
case 'n':
b.WriteByte('\n')
case 'r':
b.WriteByte('\r')
case 't':
b.WriteByte('\t')
case '\\':
b.WriteByte('\\')
default:
// Not a batch-escape sequence: keep both bytes verbatim.
b.WriteByte('\\')
b.WriteByte(s[i])
}
}
return b.String()
}
// --- Root creds via /root/.my.cnf ---------------------------------------
var (
rootMu sync.Mutex
rootDB *sql.DB
rootPath = "/root/.my.cnf"
)
// SetRootCnfPath overrides the [client] config path before the first
// RootDB() call. Tests set this to a tempdir-rooted .my.cnf.
func SetRootCnfPath(p string) {
rootMu.Lock()
defer rootMu.Unlock()
rootPath = p
// Drop any cached DB so the next RootDB() rebuilds from the new path.
if rootDB != nil {
_ = rootDB.Close()
rootDB = nil
}
}
// RootQuery runs a query as root using the credentials in
// /root/.my.cnf (or the path set via SetRootCnfPath). It mirrors the
// historical `mysql -N -B -e <query>` invocation that picked up
// /root/.my.cnf implicitly. The connection is pooled across calls.
//
// Tests can intercept via SetRootQueryForTest.
func RootQuery(ctx context.Context, query string, args ...any) ([]string, error) {
if fn := getRootQueryMock(); fn != nil {
return fn(ctx, "", query, args...)
}
db, err := RootDB()
if err != nil {
return nil, err
}
return runQuery(ctx, db, query, args...)
}
// RootExecSchema runs a non-SELECT statement against an explicit schema
// using root creds. Mirrors `mysql <schema> -e <stmt>`.
func RootExecSchema(ctx context.Context, schema, stmt string, args ...any) (int64, error) {
if fn := getRootExecMock(); fn != nil {
return fn(ctx, schema, stmt, args...)
}
creds, err := loadRootCreds()
if err != nil {
return 0, err
}
creds.DBName = schema
db, err := sql.Open("mysql", creds.dsn())
if err != nil {
return 0, fmt.Errorf("mysqlclient: open %s: %w", schema, err)
}
defer func() { _ = db.Close() }()
db.SetMaxOpenConns(1)
db.SetMaxIdleConns(0)
db.SetConnMaxLifetime(queryTimeout)
cctx, cancel := context.WithTimeout(ctx, queryTimeout)
defer cancel()
res, err := db.ExecContext(cctx, stmt, args...)
if err != nil {
return 0, fmt.Errorf("mysqlclient: exec %s: %w", schema, err)
}
rows, _ := res.RowsAffected()
return rows, nil
}
// RootQueryFunc is the signature SetRootQueryForTest accepts. schema is
// empty for RootQuery, non-empty for RootQuerySchema.
type RootQueryFunc func(ctx context.Context, schema, query string, args ...any) ([]string, error)
// RootExecFunc is the signature SetRootExecForTest accepts. schema is the
// one RootExecSchema was called with.
type RootExecFunc func(ctx context.Context, schema, stmt string, args ...any) (int64, error)
var (
rootQueryMockMu sync.RWMutex
rootQueryMock RootQueryFunc
rootExecMockMu sync.RWMutex
rootExecMock RootExecFunc
)
// SetRootQueryForTest installs an interceptor for RootQuery /
// RootQuerySchema. Pass nil to clear and restore the real database/sql
// path. Production code paths must NOT call this.
func SetRootQueryForTest(fn RootQueryFunc) {
rootQueryMockMu.Lock()
defer rootQueryMockMu.Unlock()
rootQueryMock = fn
}
// SetRootExecForTest installs an interceptor for RootExecSchema. Pass nil
// to clear and restore the real database/sql path. Production code paths
// must NOT call this.
func SetRootExecForTest(fn RootExecFunc) {
rootExecMockMu.Lock()
defer rootExecMockMu.Unlock()
rootExecMock = fn
}
func getRootQueryMock() RootQueryFunc {
rootQueryMockMu.RLock()
defer rootQueryMockMu.RUnlock()
return rootQueryMock
}
func getRootExecMock() RootExecFunc {
rootExecMockMu.RLock()
defer rootExecMockMu.RUnlock()
return rootExecMock
}
// RootDB returns the pooled root MySQL handle backed by /root/.my.cnf
// when present (or the path set via SetRootCnfPath), falling back to
// the mysql CLI's root socket-auth default. sql.Open does not contact
// the server until the first query.
func RootDB() (*sql.DB, error) {
return rootSingleton()
}
// RootQuerySchema runs a query against an explicit schema using root
// creds. Mirrors `mysql <schema> -e <query>`.
func RootQuerySchema(ctx context.Context, schema, query string, args ...any) ([]string, error) {
if fn := getRootQueryMock(); fn != nil {
return fn(ctx, schema, query, args...)
}
creds, err := loadRootCreds()
if err != nil {
return nil, err
}
creds.DBName = schema
return PerAccountQuery(ctx, creds, query, args...)
}
func rootSingleton() (*sql.DB, error) {
rootMu.Lock()
if rootDB != nil {
defer rootMu.Unlock()
return rootDB, nil
}
rootMu.Unlock()
creds, err := loadRootCreds()
if err != nil {
return nil, err
}
db, err := sql.Open("mysql", creds.dsn())
if err != nil {
return nil, fmt.Errorf("mysqlclient: root open: %w", err)
}
// Conservative pool: root scans run at most a few queries per
// minute, sharing one idle connection is plenty.
db.SetMaxOpenConns(2)
db.SetMaxIdleConns(1)
db.SetConnMaxLifetime(5 * time.Minute)
rootMu.Lock()
defer rootMu.Unlock()
if rootDB != nil {
_ = db.Close()
return rootDB, nil
}
rootDB = db
return rootDB, nil
}
// loadRootCreds parses /root/.my.cnf's [client] section (or the
// path set via SetRootCnfPath) when the file exists. The format is the
// same INI subset mysql_config_editor / the official client honour:
// section header in `[client]`, key=value pairs (values may be
// unquoted, single quoted, or double quoted; matching quote is
// stripped).
//
// Missing files and omitted user keys fall back to user=root with an
// empty password so Unix socket-auth setups keep matching `mysql -e`.
func loadRootCreds() (Creds, error) {
rootMu.Lock()
path := rootPath
rootMu.Unlock()
creds := Creds{User: "root"}
// #nosec G304 -- path is operator-configured at init time, not
// attacker-controlled.
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return creds, nil
}
return Creds{}, fmt.Errorf("mysqlclient: open %s: %w", path, err)
}
defer f.Close()
inClient := false
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 0, 4096), 1<<20)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") {
continue
}
if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
inClient = strings.EqualFold(line, "[client]") || strings.EqualFold(line, "[mysql]")
continue
}
if !inClient {
continue
}
key, val, ok := strings.Cut(line, "=")
if !ok {
continue
}
key = strings.TrimSpace(key)
val = unquote(strings.TrimSpace(val))
switch strings.ToLower(key) {
case "user":
if val != "" {
creds.User = val
}
case "password", "pass":
creds.Password = val
case "host":
creds.Host = val
case "port":
if p, perr := strconv.Atoi(val); perr == nil {
creds.Port = p
}
case "socket":
creds.Socket = val
}
}
if err := scanner.Err(); err != nil {
return Creds{}, fmt.Errorf("mysqlclient: read %s: %w", path, err)
}
return creds, nil
}
func unquote(s string) string {
if len(s) >= 2 {
first, last := s[0], s[len(s)-1]
if (first == '"' && last == '"') || (first == '\'' && last == '\'') {
return s[1 : len(s)-1]
}
}
return s
}
package netutil
import (
"net"
"net/netip"
"slices"
"sync"
"time"
)
// HostAddresses lists the addresses bound to this host's interfaces,
// sorted, including loopback. A lookup failure gives an empty list, as
// IsHostAddress fails open.
func HostAddresses() []netip.Addr {
hostAddrMu.Lock()
lookup := hostAddrLookup
hostAddrMu.Unlock()
ips, err := lookup()
if err != nil {
return nil
}
out := make([]netip.Addr, 0, len(ips))
for _, ip := range ips {
if a, ok := netip.AddrFromSlice(ip); ok {
out = append(out, a.Unmap())
}
}
slices.SortFunc(out, func(a, b netip.Addr) int { return a.Compare(b) })
return slices.Compact(out)
}
// hostAddrCacheTTL bounds how long a cached interface enumeration is reused.
// Alias IPs come and go on panel hosts, so the set cannot be read once at
// startup, but enumerating on every finding would be wasteful.
const hostAddrCacheTTL = 5 * time.Minute
// hostAddrLookup returns every IP bound to a local interface. Package-level so
// tests can inject deterministic addresses.
var hostAddrLookup = enumerateHostAddresses
var (
hostAddrMu sync.Mutex
hostAddrCache map[string]struct{}
hostAddrCachedAt time.Time
hostAddrCacheGood bool
hostAddrGeneration uint64
)
func enumerateHostAddresses() ([]net.IP, error) {
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil, err
}
out := make([]net.IP, 0, len(addrs))
for _, a := range addrs {
switch v := a.(type) {
case *net.IPNet:
out = append(out, v.IP)
case *net.IPAddr:
out = append(out, v.IP)
}
}
return out, nil
}
// IsHostAddress reports whether ip is an address bound to one of this host's
// own interfaces.
//
// Traffic a machine sends to itself is not an attack on itself. cPanel hosts
// proxy nginx to Apache over the machine's public address rather than
// loopback, so without this guard every proxied request counts as inbound
// traffic from the server, and a site that answers its own cron with an error
// drives the host's own address up the local threat score.
//
// Loopback and link-local addresses deliberately do not count: callers that
// want to accept local traffic outright test for that separately, and folding
// it in here would let a check discard findings that really did originate on
// the box. A lookup failure fails open (reports false) so a transient syscall
// error cannot suppress every finding.
func IsHostAddress(ip string) bool {
parsed := net.ParseIP(ip)
if parsed == nil {
return false
}
if parsed.IsLoopback() || parsed.IsLinkLocalUnicast() || parsed.IsLinkLocalMulticast() {
return false
}
set, ok := hostAddresses()
if !ok {
return false
}
_, found := set[hostAddrKey(parsed)]
return found
}
// hostAddrKey normalizes an address so the v4-mapped-v6 and non-canonical
// textual forms of the same address share one key.
func hostAddrKey(ip net.IP) string {
if v4 := ip.To4(); v4 != nil {
return v4.String()
}
return ip.String()
}
func hostAddresses() (map[string]struct{}, bool) {
hostAddrMu.Lock()
if hostAddrCacheGood && time.Since(hostAddrCachedAt) < hostAddrCacheTTL {
set := hostAddrCache
hostAddrMu.Unlock()
return set, true
}
lookup, generation := hostAddrLookup, hostAddrGeneration
hostAddrMu.Unlock()
ips, err := lookup()
if err != nil {
return nil, false
}
set := make(map[string]struct{}, len(ips))
for _, ip := range ips {
if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
continue
}
set[hostAddrKey(ip)] = struct{}{}
}
hostAddrMu.Lock()
defer hostAddrMu.Unlock()
// Replacing the source invalidates any lookup already in flight. Never
// let an old callback repopulate the replacement's cache.
if generation != hostAddrGeneration {
return nil, false
}
if hostAddrCacheGood && time.Since(hostAddrCachedAt) < hostAddrCacheTTL {
return hostAddrCache, true
}
hostAddrCache = set
hostAddrCachedAt = time.Now()
hostAddrCacheGood = true
return set, true
}
// SetHostAddressLookup swaps the interface enumeration and drops the cache,
// returning a function that restores the previous lookup. Tests in other
// packages use it to pin a deterministic address set.
func SetHostAddressLookup(fn func() ([]net.IP, error)) func() {
hostAddrMu.Lock()
prev := hostAddrLookup
hostAddrLookup = fn
hostAddrGeneration++
hostAddrCache = nil
hostAddrCacheGood = false
hostLocalCache = nil
hostLocalCacheGood = false
hostAddrMu.Unlock()
return func() {
hostAddrMu.Lock()
hostAddrLookup = prev
hostAddrGeneration++
hostAddrCache = nil
hostAddrCacheGood = false
hostLocalCache = nil
hostLocalCacheGood = false
hostAddrMu.Unlock()
}
}
package netutil
import (
"fmt"
"net"
"net/netip"
"slices"
"strings"
"time"
)
// staticHostLocal lists the ranges every process on a host can send from
// without looking at its interfaces: loopback and the unspecified addresses.
var staticHostLocal = []struct {
prefix netip.Prefix
label string
}{
{netip.MustParsePrefix("127.0.0.0/8"), "127.0.0.0/8"},
{netip.MustParsePrefix("::1/128"), "::1"},
{netip.MustParsePrefix("0.0.0.0/32"), "0.0.0.0"},
{netip.MustParsePrefix("::/128"), "::"},
// Prefixes shorter than /96 can cover mapped IPv4 without having a
// mapped network address. Every such range contains mapped loopback.
{netip.MustParsePrefix("::ffff:127.0.0.0/104"), "127.0.0.0/8"},
}
// The host-local cache keeps the last interface set that was read, unlike the
// IsHostAddress cache: callers deciding whether to trust a peer must not fall
// back to "not local" because one refresh failed. Guarded by hostAddrMu.
var (
hostLocalCache map[netip.Addr]struct{}
hostLocalCachedAt time.Time
hostLocalCacheGood bool
hostLocalNextLookup uint64
hostLocalCachedLookup uint64
)
// IsHostLocal reports whether ip is an address a process on this host can
// connect from: loopback, unspecified, or bound to one of its interfaces,
// link-local included. known is false only when the interface set has never
// been read, so a caller deciding trust can refuse rather than guess.
// Failed or out-of-order refreshes preserve the most recent successful set.
func IsHostLocal(ip net.IP) (local, known bool) {
addr, ok := netip.AddrFromSlice(ip)
if !ok {
return false, false
}
addr = addr.Unmap()
if isStaticHostLocal(addr) {
return true, true
}
set, ok := hostLocalAddrs()
if !ok {
return false, false
}
_, found := set[addr]
return found, true
}
// HostLocalProxyEntry returns the first trusted-proxy entry that equals,
// contains or overlaps an address of this host, and what it covers.
// Loopback and unspecified hits need no interface lookup; any other entry
// needs a fresh one, and a failed lookup is an error, not a pass.
// Overlaps include IPv4-mapped IPv6 ranges, even prefixes shorter than /96.
func HostLocalProxyEntry(entries []string) (entry, covered string, err error) {
type parsed struct {
raw string
prefix netip.Prefix
}
var rest []parsed
for _, e := range entries {
raw := strings.TrimSpace(e)
p, ok := entryPrefix(raw)
if !ok {
continue
}
for _, s := range staticHostLocal {
if p.Overlaps(s.prefix) {
return raw, s.label, nil
}
}
rest = append(rest, parsed{raw, p})
}
if len(rest) == 0 {
return "", "", nil
}
hostAddrMu.Lock()
lookup := hostAddrLookup
hostAddrMu.Unlock()
ips, err := lookup()
if err != nil {
return "", "", fmt.Errorf("reading this host's addresses: %w", err)
}
host := hostLocalSet(ips)
addrs := make([]netip.Addr, 0, len(host))
for a := range host {
addrs = append(addrs, a)
}
slices.SortFunc(addrs, func(a, b netip.Addr) int { return a.Compare(b) })
for _, p := range rest {
for _, a := range addrs {
if p.prefix.Contains(a) {
return p.raw, a.String(), nil
}
}
}
return "", "", nil
}
func isStaticHostLocal(addr netip.Addr) bool {
for _, s := range staticHostLocal {
if s.prefix.Contains(addr) {
return true
}
}
return false
}
// entryPrefix parses an address or CIDR the way trusted-proxy lists do.
func entryPrefix(raw string) (netip.Prefix, bool) {
n := ParseCIDROrIP(raw)
if n == nil {
return netip.Prefix{}, false
}
addr, ok := netip.AddrFromSlice(n.IP)
if !ok {
return netip.Prefix{}, false
}
ones, bits := n.Mask.Size()
if bits == 0 {
return netip.Prefix{}, false
}
if addr.Is4In6() && bits == 128 {
if ones < 96 {
return netip.Prefix{}, false
}
ones -= 96
}
addr = addr.Unmap()
p, err := addr.Prefix(ones)
if err != nil {
return netip.Prefix{}, false
}
return p, true
}
func hostLocalSet(ips []net.IP) map[netip.Addr]struct{} {
set := make(map[netip.Addr]struct{}, len(ips))
for _, ip := range ips {
if a, ok := netip.AddrFromSlice(ip); ok {
set[a.Unmap()] = struct{}{}
}
}
return set
}
func hostLocalAddrs() (map[netip.Addr]struct{}, bool) {
hostAddrMu.Lock()
if hostLocalCacheGood && time.Since(hostLocalCachedAt) < hostAddrCacheTTL {
set := hostLocalCache
hostAddrMu.Unlock()
return set, true
}
lookup, generation := hostAddrLookup, hostAddrGeneration
hostLocalNextLookup++
lookupID := hostLocalNextLookup
hostAddrMu.Unlock()
ips, err := lookup()
hostAddrMu.Lock()
defer hostAddrMu.Unlock()
// A lookup swapped in meanwhile owns the cache; never let this one
// repopulate it.
if err != nil || generation != hostAddrGeneration {
return hostLocalCache, hostLocalCacheGood
}
// A slower, earlier lookup must not discard addresses discovered by a
// later one, even if that newer result has since expired.
if lookupID < hostLocalCachedLookup {
return hostLocalCache, hostLocalCacheGood
}
hostLocalCache = hostLocalSet(ips)
hostLocalCachedAt = time.Now()
hostLocalCacheGood = true
hostLocalCachedLookup = lookupID
return hostLocalCache, true
}
package netutil
import (
"net"
"strings"
)
// ParseIPToken extracts an IP address from one whitespace-delimited log or
// message token: surrounding brackets and trailing punctuation are removed,
// and a trailing ':' separator is dropped only when the token does not
// already parse as an address, so the "::" that ends many IPv6 addresses
// survives. The result is the canonical text form; ok is false when no
// address is left. It replaces per-caller TrimRight cutsets that contained
// ':' and cut IPv6 addresses short without re-validating the remnant.
func ParseIPToken(token string) (string, bool) {
t := strings.Trim(strings.TrimSpace(token), ",;.!?()<>\"'")
if ip := parseIPLiteral(t); ip != nil {
return ip.String(), true
}
if host, _, err := net.SplitHostPort(t); err == nil {
if ip := net.ParseIP(strings.Trim(host, "[]")); ip != nil {
return ip.String(), true
}
}
for strings.HasSuffix(t, ":") {
t = strings.TrimSuffix(t, ":")
if ip := parseIPLiteral(t); ip != nil {
return ip.String(), true
}
}
return "", false
}
func parseIPLiteral(token string) net.IP {
if len(token) >= 2 && token[0] == '[' && token[len(token)-1] == ']' {
token = token[1 : len(token)-1]
}
return net.ParseIP(token)
}
// Package netutil holds the shared public-range guard used to validate
// operator- and vendor-supplied IP ranges. It lives in its own package so the
// special-use CIDR list and the public-IP predicate have a single definition
// instead of being copy-pasted across internal/config and internal/threatintel.
package netutil
import (
"net"
"strings"
)
// nonPublicSpecialUseNets are address blocks that are routable-looking but must
// never be treated as public crawler space: an allowlist entry inside any of
// them would let an attacker claim addresses CSM should always scan.
var nonPublicSpecialUseNets = mustParseCIDRs(
"0.0.0.0/8", // "this network"
"100.64.0.0/10", // carrier-grade NAT
"192.0.0.0/24", // IETF protocol assignments
"192.0.2.0/24", // documentation
"192.88.99.0/24", // deprecated 6to4 relay anycast
"198.18.0.0/15", // benchmarking
"198.51.100.0/24", // documentation
"203.0.113.0/24", // documentation
"240.0.0.0/4", // reserved
"100::/64", // discard-only
"2001:2::/48", // benchmarking
"2001:db8::/32", // documentation
"2002::/16", // 6to4
"64:ff9b::/96", // IPv4/IPv6 translation
"64:ff9b:1::/48", // IPv4/IPv6 translation
)
func mustParseCIDRs(cidrs ...string) []*net.IPNet {
out := make([]*net.IPNet, 0, len(cidrs))
for _, cidr := range cidrs {
_, n, err := net.ParseCIDR(cidr)
if err != nil {
panic(err)
}
out = append(out, n)
}
return out
}
// IPInAnyNet reports whether ip is contained in any of nets.
func IPInAnyNet(ip net.IP, nets []*net.IPNet) bool {
for _, n := range nets {
if n.Contains(ip) {
return true
}
}
return false
}
// IsPublicIP reports whether ip is a public, globally-routable unicast address:
// not private, link-local, multicast, or in any non-public special-use block
// (CGNAT, documentation, benchmarking, reserved, IPv6 special-use). A nil IP is
// never public.
func IsPublicIP(ip net.IP) bool {
return ip.IsGlobalUnicast() && !ip.IsPrivate() &&
!ip.IsLinkLocalUnicast() && !ip.IsLinkLocalMulticast() && !ip.IsMulticast() &&
!IPInAnyNet(ip, nonPublicSpecialUseNets)
}
// ParseCIDROrIP parses a CIDR or a bare IP, returning the normalized network. A
// bare IPv4 becomes a /32 and a bare IPv6 a /128. Returns nil on garbage.
func ParseCIDROrIP(s string) *net.IPNet {
s = strings.TrimSpace(s)
if _, n, err := net.ParseCIDR(s); err == nil {
return NormalizeIPNet(n)
}
if ip := net.ParseIP(s); ip != nil {
if v4 := ip.To4(); v4 != nil {
return &net.IPNet{IP: v4, Mask: net.CIDRMask(32, 32)}
}
return &net.IPNet{IP: ip, Mask: net.CIDRMask(128, 128)}
}
return nil
}
// NormalizeIPNet returns n with its IP masked to the network address and IPv4
// kept in 4-byte form. An IPv4-mapped IPv6 CIDR is reduced to its effective
// IPv4 prefix so Contains matches on the IPv4 form. Returns nil for a malformed
// mask/IP combination.
func NormalizeIPNet(n *net.IPNet) *net.IPNet {
if n == nil {
return nil
}
if v4 := n.IP.To4(); v4 != nil {
mask := n.Mask
if len(mask) == net.IPv6len {
if _, bits := mask.Size(); bits != net.IPv6len*8 {
return nil
}
// Go treats IPv4-mapped IPv6 CIDRs as IPv4 ranges for Contains.
// Keep validation and matching on that same effective prefix.
mask = net.IPMask(mask[12:])
}
if _, bits := mask.Size(); bits != net.IPv4len*8 {
return nil
}
return &net.IPNet{IP: v4.Mask(mask), Mask: mask}
}
ip := n.IP.To16()
if ip == nil {
return nil
}
if _, bits := n.Mask.Size(); bits != net.IPv6len*8 {
return nil
}
return &net.IPNet{IP: ip.Mask(n.Mask), Mask: n.Mask}
}
// Package obs centralises crash reporting and selective error capture via
// Sentry. Init is a one-shot called from the daemon entry point; after
// that, callers use Go/SafeGo to launch goroutines with panic recovery
// and Capture/CaptureMsg to forward selected errors.
//
// When Sentry is disabled or the DSN is empty, every function becomes a
// no-op wrapper with the same semantics as a plain `go func()` call,
// so guarded call sites work unchanged in tests and in operator
// builds that opt out of telemetry.
package obs
import (
"fmt"
"os"
"sync/atomic"
"time"
"github.com/getsentry/sentry-go"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/platform"
)
var enabled atomic.Bool
// flushTimeout bounds how long shutdown flushes block before giving up.
// Sentry enforces its own deadline on Flush; we pass this value so
// stuck HTTP calls don't hang systemd past TimeoutStopSec.
const flushTimeout = 2 * time.Second
// Init configures the Sentry SDK once at daemon startup. Returns nil if
// Sentry is disabled or the DSN is empty so callers can treat init as
// optional. Safe to call when cfg is nil.
func Init(cfg *config.Config, version, buildHash string) error {
if cfg == nil || !cfg.Sentry.Enabled || cfg.Sentry.DSN == "" {
return nil
}
env := cfg.Sentry.Environment
if env == "" {
env = "production"
}
rate := cfg.Sentry.SampleRate
if rate <= 0 {
rate = 1.0
}
hostname, _ := os.Hostname()
release := "csm@" + version
if buildHash != "" && buildHash != "unknown" {
release = release + "+" + buildHash
}
err := sentry.Init(sentry.ClientOptions{
Dsn: cfg.Sentry.DSN,
Environment: env,
Release: release,
ServerName: hostname,
SampleRate: rate,
TracesSampleRate: 0.0,
Debug: cfg.Sentry.Debug,
AttachStacktrace: true,
})
if err != nil {
return fmt.Errorf("sentry init: %w", err)
}
info := platform.Detect()
sentry.ConfigureScope(func(scope *sentry.Scope) {
scope.SetTag("os", string(info.OS))
if info.OSVersion != "" {
scope.SetTag("os_version", info.OSVersion)
}
if info.Panel != "" {
scope.SetTag("panel", string(info.Panel))
}
if info.WebServer != "" {
scope.SetTag("webserver", string(info.WebServer))
}
})
enabled.Store(true)
return nil
}
// Enabled reports whether Init succeeded and Sentry is live.
func Enabled() bool { return enabled.Load() }
// Flush waits for queued events to be sent before returning. Call from
// shutdown paths before exit. Safe to call when Sentry is disabled.
func Flush() {
if !enabled.Load() {
return
}
sentry.Flush(flushTimeout)
}
// Go launches fn in a new goroutine. A panic in fn is captured with the
// given component tag, the event is flushed, and the panic is
// re-raised so the existing crash-and-systemd-restart behavior is
// preserved. Use this for long-lived supervisor goroutines where a
// silent death would leave the daemon in a degraded state.
func Go(component string, fn func()) {
go func() {
defer func() {
if r := recover(); r != nil {
report(component, r)
panic(r)
}
}()
fn()
}()
}
// SafeGo is Go but swallows the panic after capture. Use this for
// per-request handlers (socket accept, HTTP request) where one bad
// input should not crash the whole daemon.
func SafeGo(component string, fn func()) {
go func() {
defer func() {
if r := recover(); r != nil {
report(component, r)
}
}()
fn()
}()
}
// Capture sends an error to Sentry with a component tag. No-op when
// disabled or err is nil. Reserve for unexpected states and invariant
// violations; expected-failure errors (permission denied, transient
// network) should stay out of Sentry to avoid noise.
func Capture(component string, err error) {
if !enabled.Load() || err == nil {
return
}
sentry.WithScope(func(scope *sentry.Scope) {
scope.SetTag("component", component)
sentry.CaptureException(err)
})
}
// CaptureMsg is Capture for string-only events (e.g. invariant
// violations without a wrapped error).
func CaptureMsg(component, msg string) {
if !enabled.Load() {
return
}
sentry.WithScope(func(scope *sentry.Scope) {
scope.SetTag("component", component)
sentry.CaptureMessage(msg)
})
}
func report(component string, r any) {
if !enabled.Load() {
return
}
sentry.WithScope(func(scope *sentry.Scope) {
scope.SetTag("component", component)
sentry.CurrentHub().Recover(r)
})
sentry.Flush(flushTimeout)
}
//go:build linux
package parseworker
import (
"bytes"
"encoding/json"
"fmt"
"os"
"strings"
"unicode"
"golang.org/x/sys/unix"
emime "github.com/pidginhost/csm/internal/mime"
"github.com/pidginhost/csm/internal/workerproc"
)
func extractMessage(req workerproc.Frame, fds []*os.File) (workerproc.Frame, []*os.File) {
if len(fds) != 2 {
return workerproc.Frame{Error: fmt.Sprintf("extract_message takes two descriptors, got %d", len(fds))}, nil
}
for _, f := range fds {
st, err := f.Stat()
if err != nil {
return workerproc.Frame{Error: err.Error()}, nil
}
if !st.Mode().IsRegular() {
return workerproc.Frame{Error: "extract_message descriptor is not a regular file"}, nil
}
}
var args ExtractArgs
dec := json.NewDecoder(bytes.NewReader(req.Payload))
dec.DisallowUnknownFields()
if err := dec.Decode(&args); err != nil {
return workerproc.Frame{Error: fmt.Sprintf("extract_message arguments: %v", err)}, nil
}
parsed, err := emime.ParseSpoolFiles(fds[0], fds[1], emime.Limits{
MaxAttachmentSize: args.MaxAttachmentSize,
MaxArchiveDepth: args.MaxArchiveDepth,
MaxArchiveFiles: args.MaxArchiveFiles,
MaxExtractionSize: args.MaxExtractionSize,
MaxParts: MaxParts,
PartSink: memfdPart,
})
if err != nil {
return extractResult(ExtractResult{Partial: true, PartialReason: cleanText(err.Error(), maxIncomplete)}), nil
}
res := ExtractResult{
Partial: parsed.Partial,
PartialReason: cleanText(parsed.PartialReason, maxIncomplete),
Direction: parsed.Direction,
From: cleanText(parsed.From, maxHeaderBytes),
Subject: cleanText(parsed.Subject, maxHeaderBytes),
}
for _, to := range parsed.To[:min(len(parsed.To), maxRecipients)] {
res.To = append(res.To, cleanText(to, maxHeaderBytes))
}
entries := make([]EncryptedEntry, 0, len(parsed.EncryptedEntries))
for _, e := range parsed.EncryptedEntries {
entries = append(entries, EncryptedEntry{ArchiveName: e.ArchiveName, Filename: e.Filename})
}
capEncrypted(&res, entries, parsed.EncryptedEntriesOmitted)
files := make([]*os.File, 0, len(parsed.Parts))
for _, p := range parsed.Parts {
if p.Size == 0 {
// Nothing to scan; a descriptor for it would only spend the budget.
_ = p.File.Close()
continue
}
if err := sealFile(p.File); err != nil {
parsed.Close()
return workerproc.Frame{Error: fmt.Sprintf("seal part: %v", err)}, nil
}
res.Parts = append(res.Parts, ExtractPart{
Filename: p.Filename,
ContentType: cleanText(p.ContentType, maxNameBytes),
Size: p.Size,
Nested: p.Nested,
ArchiveName: p.ArchiveName,
})
files = append(files, p.File)
}
return extractResult(res), files
}
// capEncrypted keeps at most MaxEncryptedEntries names and counts the rest.
func capEncrypted(res *ExtractResult, entries []EncryptedEntry, omitted int) {
keep := min(len(entries), MaxEncryptedEntries)
res.EncryptedEntries = entries[:keep]
res.EncryptedEntriesOmitted = omitted + len(entries) - keep
}
func memfdPart(string) (*os.File, error) {
fd, err := unix.MemfdCreate("csm-mail-part", unix.MFD_CLOEXEC|unix.MFD_ALLOW_SEALING)
if err != nil {
return nil, err
}
return os.NewFile(uintptr(fd), "csm-mail-part"), nil
}
// cleanText makes a header-derived string safe to send: no control
// characters but tab (the daemon refuses them, as they can forge log
// lines), valid UTF-8, at most max bytes. Encoded, such a string is at most
// twice its length, which is what keeps the worst-case reply in its frame.
func cleanText(s string, max int) string {
s = strings.Map(func(r rune) rune {
if r != '\t' && unicode.IsControl(r) {
return -1
}
return r
}, s)
return emime.TruncateUTF8(strings.ToValidUTF8(s, "\uFFFD"), max)
}
// extractResult encodes without HTML escaping: escaped, '<' alone would
// take six bytes and push a crafted reply past the frame cap.
func extractResult(r ExtractResult) workerproc.Frame {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
enc.SetEscapeHTML(false)
_ = enc.Encode(r) // Numbers, strings and bools cannot fail to encode.
return workerproc.Frame{Payload: bytes.TrimSuffix(buf.Bytes(), []byte("\n"))}
}
package parseworker
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strings"
"time"
"unicode"
"unicode/utf8"
emime "github.com/pidginhost/csm/internal/mime"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/workerproc"
)
// ExtractTimeout bounds one message's extraction in a worker: three times the
// measured p99 (1.14 s for 200 messages up to the 25 MiB attachment cap on a
// 2-vCPU host), rounded up. It stays under the 5 s hold budget on purpose: a
// hanging message gets its verdict before the hold answers, so tempfail and
// quarantine still apply in time, and the scan slot is freed quickly. A
// message that kills the worker has none of its attachments scanned and is
// deferred whatever email_av.fail_mode says, except in observe mode.
const ExtractTimeout = 4 * time.Second
// extractTimeout is a seam for tests.
var extractTimeout = ExtractTimeout
// ExtractorConfig configures the mail parse worker pool.
type ExtractorConfig struct {
// Size is the slot count; values below 1 mean 4.
Size int
Launch func(slot int) workerproc.Launcher
OnFailure func(reason string, err error)
OnJob func(outcome string, memory uint64)
Log func(string, ...any)
}
// Extractor extracts mail attachments in a pool of parse workers, one slot
// per caller goroutine.
type Extractor struct {
pool *workerproc.Pool
}
// NewExtractor starts no worker; each slot's first Extract does.
func NewExtractor(cfg ExtractorConfig) (*Extractor, error) {
size := cfg.Size
if size < 1 {
size = 4
}
pool, err := workerproc.NewPool(workerproc.PoolConfig{Size: size, Config: func(slot int) workerproc.Config {
return workerproc.Config{
Name: fmt.Sprintf("parse worker (mail %d)", slot),
Launch: cfg.Launch(slot),
Timeout: extractTimeout,
Limits: ExtractLimits,
Log: cfg.Log,
MaxJobs: 1000,
IdleTimeout: 10 * time.Minute,
OnFailure: cfg.OnFailure,
OnJob: cfg.OnJob,
// A refused extraction delivers mail unscanned under the open
// fail mode, so crafted mail must not be able to open the breaker.
BreakOnSpawnOnly: true,
}
}})
if err != nil {
return nil, err
}
return &Extractor{pool: pool}, nil
}
// Size is the slot count.
func (e *Extractor) Size() int { return e.pool.Size() }
// Extract parses the message from its open -H and -D files on slot's worker.
// The parts come back as sealed memfds the caller closes with the result.
// A reply that fails validation kills the worker; the error then matches
// workerproc.ErrWorkerFailed with reason protocol.
func (e *Extractor) Extract(ctx context.Context, slot int, header, body *os.File, limits emime.Limits) (*emime.ExtractionResult, error) {
payload, err := json.Marshal(ExtractArgs{
MaxAttachmentSize: limits.MaxAttachmentSize,
MaxArchiveDepth: limits.MaxArchiveDepth,
MaxArchiveFiles: limits.MaxArchiveFiles,
MaxExtractionSize: limits.MaxExtractionSize,
})
if err != nil {
return nil, err
}
var res ExtractResult
_, fds, err := e.pool.Call(ctx, slot, workerproc.Frame{Op: OpExtractMessage, Payload: payload}, []*os.File{header, body},
func(reply workerproc.Frame, fds []*os.File) (bool, error) {
res = ExtractResult{}
return false, checkExtract(reply.Payload, fds, limits, &res)
})
if err != nil {
return nil, err
}
out := &emime.ExtractionResult{
Partial: res.Partial,
PartialReason: res.PartialReason,
EncryptedEntriesOmitted: res.EncryptedEntriesOmitted,
Direction: res.Direction,
From: res.From,
To: res.To,
Subject: res.Subject,
}
for _, en := range res.EncryptedEntries {
out.EncryptedEntries = append(out.EncryptedEntries, emime.EncryptedArchiveEntry{ArchiveName: en.ArchiveName, Filename: en.Filename})
}
for i, p := range res.Parts {
out.Parts = append(out.Parts, emime.ExtractedPart{
Filename: p.Filename,
ContentType: p.ContentType,
Size: p.Size,
File: fds[i],
Nested: p.Nested,
ArchiveName: p.ArchiveName,
})
}
return out, nil
}
// checkExtract validates a worker's reply before anything uses it: the
// descriptors match the parts one for one, each is a sealed memfd of
// exactly the claimed, non-zero size within the caps, and every string is
// bounded, valid UTF-8 and in the form the worker's own sanitizing gives.
func checkExtract(payload []byte, fds []*os.File, limits emime.Limits, res *ExtractResult) error {
// encoding/json replaces invalid UTF-8 instead of rejecting it.
if !utf8.Valid(payload) {
return errors.New("extract_message reply is not UTF-8")
}
// Parts may be null for an empty result, but it must be present. Keep
// its raw value so a missing result cannot look like a clean message.
var wire struct {
*ExtractResult
Parts json.RawMessage `json:"parts"`
}
wire.ExtractResult = res
dec := json.NewDecoder(bytes.NewReader(payload))
dec.DisallowUnknownFields()
if err := dec.Decode(&wire); err != nil {
return fmt.Errorf("extract_message reply: %w", err)
}
if err := dec.Decode(new(json.RawMessage)); err != io.EOF {
return errors.New("extract_message reply has trailing data")
}
if len(wire.Parts) == 0 {
return errors.New("extract_message reply has no parts result")
}
parts := json.NewDecoder(bytes.NewReader(wire.Parts))
parts.DisallowUnknownFields()
if err := parts.Decode(&res.Parts); err != nil {
return fmt.Errorf("extract_message parts: %w", err)
}
if len(res.Parts) > MaxParts || len(res.Parts) != len(fds) {
return fmt.Errorf("reply lists %d parts with %d descriptors", len(res.Parts), len(fds))
}
var total int64
for i, p := range res.Parts {
if err := checkName(p.Filename, false); err != nil {
return fmt.Errorf("part %d name: %w", i, err)
}
if err := checkName(p.ArchiveName, true); err != nil {
return fmt.Errorf("part %d archive name: %w", i, err)
}
if err := checkText(p.ContentType, maxNameBytes); err != nil {
return fmt.Errorf("part %d content type: %w", i, err)
}
if p.Size <= 0 || p.Size > limits.MaxAttachmentSize {
return fmt.Errorf("part %d size %d outside 1..%d", i, p.Size, limits.MaxAttachmentSize)
}
if p.Size > limits.MaxExtractionSize-total {
return fmt.Errorf("parts total more than %d bytes", limits.MaxExtractionSize)
}
total += p.Size
if err := checkSealed(fds[i]); err != nil {
return fmt.Errorf("part %d: %w", i, err)
}
st, err := fds[i].Stat()
if err != nil {
return err
}
if !st.Mode().IsRegular() || st.Size() != p.Size {
return fmt.Errorf("part %d is %d bytes, reply says %d", i, st.Size(), p.Size)
}
// Seals and Stat also succeed on a write-only descriptor. Prove
// the scanner can read it before accepting any part of the reply.
var probe [1]byte
if _, err := fds[i].ReadAt(probe[:], 0); err != nil {
return fmt.Errorf("part %d is unreadable: %w", i, err)
}
}
if len(res.EncryptedEntries) > MaxEncryptedEntries || res.EncryptedEntriesOmitted < 0 {
return errors.New("reply names too many encrypted entries")
}
for _, en := range res.EncryptedEntries {
if checkName(en.ArchiveName, false) != nil || checkName(en.Filename, false) != nil {
return errors.New("encrypted entry name is not sanitized")
}
}
if err := checkText(res.PartialReason, maxIncomplete); err != nil {
return fmt.Errorf("partial reason: %w", err)
}
if len(res.To) > maxRecipients {
return fmt.Errorf("reply lists %d recipients", len(res.To))
}
for _, s := range append([]string{res.From, res.Subject}, res.To...) {
if err := checkText(s, maxHeaderBytes); err != nil {
return fmt.Errorf("header: %w", err)
}
}
switch res.Direction {
case "", "inbound", "outbound":
default:
return fmt.Errorf("direction %q", res.Direction)
}
return nil
}
// checkName requires a name exactly as SanitizeAttachmentName leaves it.
func checkName(name string, optional bool) error {
if optional && name == "" {
return nil
}
if !utf8.ValidString(name) || len(name) > maxNameBytes || emime.SanitizeAttachmentName(name) != name {
return errors.New("not a sanitized name")
}
return nil
}
// checkText bounds a string and refuses control characters other than tab,
// which could forge log lines where the string is printed.
func checkText(s string, max int) error {
if !utf8.ValidString(s) || len(s) > max {
return fmt.Errorf("invalid or longer than %d bytes", max)
}
if strings.IndexFunc(s, func(r rune) bool { return r != '\t' && unicode.IsControl(r) }) >= 0 {
return errors.New("contains control characters")
}
return nil
}
// Stop kills every worker and refuses later extractions.
func (e *Extractor) Stop() error { return e.pool.Stop() }
// QueueStatuses publishes one requests row for the pool.
func (e *Extractor) QueueStatuses(now time.Time) map[string]queuehealth.Status {
return e.pool.QueueStatuses(now)
}
// FileSizeLimit is the LimitFSIZE a mail worker needs: above the largest
// part it may write, never below the zip pool's 64 MiB.
func FileSizeLimit(maxAttachmentSize int64) uint64 {
limit := uint64(64 << 20)
if maxAttachmentSize > 0 {
limit = max(limit, uint64(maxAttachmentSize)+1<<20) // #nosec G115 -- positive, checked above.
}
return limit
}
package parseworker
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/workerproc"
)
// errIncomplete wraps a worker's report that it could not list the whole
// zip. The zip is to blame, not the worker.
var errIncomplete = errors.New("zip listing incomplete")
// ListerConfig configures a Lister.
type ListerConfig struct {
Launch workerproc.Launcher
Log func(string, ...any)
OnFailure func(reason string, err error)
OnJob func(outcome string, memory uint64)
}
// Lister lists tenant zips through one parse worker at a time.
type Lister struct {
proc *workerproc.Supervisor
}
// NewLister starts no worker; the first ListZip does.
func NewLister(cfg ListerConfig) (*Lister, error) {
proc, err := workerproc.New(workerproc.Config{
Name: "parse worker (zip)",
Launch: cfg.Launch,
Timeout: 30 * time.Second,
Limits: Limits,
Log: cfg.Log,
MaxJobs: 1000,
IdleTimeout: 10 * time.Minute,
OnFailure: cfg.OnFailure,
OnJob: cfg.OnJob,
})
if err != nil {
return nil, err
}
return &Lister{proc: proc}, nil
}
// ListZip sends zip to the worker and calls visit with each entry, in
// directory order. The worker's listing is checked in full before visit
// sees any of it; a listing that fails the check kills the worker and the
// error matches workerproc.ErrWorkerFailed. A zip the worker could not list
// completely is an error that is not a worker failure. visit's error comes
// back unwrapped.
func (l *Lister) ListZip(ctx context.Context, zip *os.File, size int64, maxDirBytes uint64, visit func(name string, dir bool) error) error {
payload, err := json.Marshal(ListZipArgs{MaxDirBytes: maxDirBytes})
if err != nil {
return err
}
var res ListZipResult
var listingSize int64
_, fds, err := l.proc.Call(ctx, workerproc.Frame{Op: OpListZip, Payload: payload}, []*os.File{zip},
func(reply workerproc.Frame, fds []*os.File) (bool, error) {
var wire struct {
Entries *uint64 `json:"entries"`
Incomplete string `json:"incomplete,omitempty"`
}
dec := json.NewDecoder(bytes.NewReader(reply.Payload))
dec.DisallowUnknownFields()
if decodeErr := dec.Decode(&wire); decodeErr != nil {
return false, fmt.Errorf("list_zip reply: %w", decodeErr)
}
if wire.Entries == nil {
return false, errors.New("list_zip reply has no entry count")
}
res = ListZipResult{Entries: *wire.Entries, Incomplete: wire.Incomplete}
if res.Incomplete != "" {
if len(fds) != 0 || res.Entries != 0 || len(res.Incomplete) > maxIncomplete {
return false, errors.New("malformed incomplete listing")
}
return false, nil
}
var checkErr error
listingSize, checkErr = checkListing(ctx, fds, size, maxDirBytes, res.Entries)
return false, checkErr
})
if err != nil {
return err
}
if res.Incomplete != "" {
return fmt.Errorf("%w: %s", errIncomplete, res.Incomplete)
}
defer func() { _ = fds[0].Close() }()
_, err = readEntries(ctx, io.NewSectionReader(fds[0], 0, listingSize), maxDirBytes, visit)
return err
}
// checkListing validates the worker's listing before anything trusts it: one
// immutable regular file, no larger than the zip (a record is 3 bytes plus
// the name, a central-directory record at least 46 plus the name), whole
// records only, exactly as many as the reply claims.
func checkListing(ctx context.Context, fds []*os.File, zipSize int64, maxDirBytes, entries uint64) (int64, error) {
if len(fds) != 1 {
return 0, fmt.Errorf("listing carries %d descriptors, want 1", len(fds))
}
// Seal checks precede even Stat: the size and every record must stay
// unchanged from validation through delivery to visit.
if err := checkSealed(fds[0]); err != nil {
return 0, err
}
st, err := fds[0].Stat()
if err != nil {
return 0, err
}
if !st.Mode().IsRegular() {
return 0, errors.New("listing is not a regular file")
}
if st.Size() > zipSize {
return 0, fmt.Errorf("listing of %d bytes exceeds the %d-byte zip", st.Size(), zipSize)
}
n, err := readEntries(ctx, io.NewSectionReader(fds[0], 0, st.Size()), maxDirBytes, func(string, bool) error { return nil })
if err != nil {
return 0, fmt.Errorf("listing: %w", err)
}
if n != entries {
return 0, fmt.Errorf("listing holds %d records, reply claims %d", n, entries)
}
return st.Size(), nil
}
// Stop kills the worker and refuses later listings.
func (l *Lister) Stop() error { return l.proc.Stop() }
// Stats reports the worker's spawn and breaker state.
func (l *Lister) Stats() workerproc.Stats { return l.proc.Stats() }
// QueueStatuses publishes the requests row.
func (l *Lister) QueueStatuses(now time.Time) map[string]queuehealth.Status {
return l.proc.QueueStatuses(now)
}
package parseworker
import (
"bufio"
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"github.com/pidginhost/csm/internal/zipdir"
)
// ListEntries walks a zip's central directory and calls visit for each entry
// in order. Errors, visit's included, come back as is.
func ListEntries(ctx context.Context, r io.ReaderAt, size int64, maxDirBytes uint64, visit func(name string, dir bool) error) error {
dir, err := zipdir.Read(r, size, maxDirBytes)
if err != nil {
return err
}
return zipdir.Walk(ctx, r, dir, func(e zipdir.Entry) error {
return visit(e.Name, e.Mode().IsDir())
})
}
// A listing record is one flag byte (bit 0: directory; any other bit is
// invalid), a 2-byte big-endian name length, and the name.
const flagDir = 1
func writeEntry(w io.Writer, name string, dir bool) error {
if len(name) > math.MaxUint16 {
return fmt.Errorf("parseworker: entry name of %d bytes exceeds %d", len(name), math.MaxUint16)
}
var head [3]byte
if dir {
head[0] = flagDir
}
binary.BigEndian.PutUint16(head[1:], uint16(len(name))) // #nosec G115 -- bounded above.
if _, err := w.Write(head[:]); err != nil {
return err
}
_, err := io.WriteString(w, name)
return err
}
// readEntries decodes records until a clean end of input and returns how
// many it read. A short record or an unknown flag bit is an error; a visit
// error stops the read and is returned unwrapped. maxDirBytes is the original
// request's directory budget, never a size supplied by the worker. The caller
// must provide a reader over the listing file, not a potentially blocking pipe.
func readEntries(ctx context.Context, r io.Reader, maxDirBytes uint64, visit func(name string, dir bool) error) (uint64, error) {
br := bufio.NewReader(r)
var n uint64
var head [3]byte
for {
if err := ctx.Err(); err != nil {
return n, err
}
if _, err := io.ReadFull(br, head[:1]); err != nil {
if errors.Is(err, io.EOF) {
return n, nil
}
return n, fmt.Errorf("parseworker: listing record %d: %w", n, err)
}
if head[0]&^flagDir != 0 {
return n, fmt.Errorf("parseworker: listing record %d has flags %#x", n, head[0])
}
if _, err := io.ReadFull(br, head[1:]); err != nil {
return n, fmt.Errorf("parseworker: listing record %d truncated: %w", n, noEOF(err))
}
nameLen := binary.BigEndian.Uint16(head[1:])
// Each record costs at least a central-directory header and its name.
// Charging that overhead also bounds floods of empty names from a
// compromised worker without rejecting any valid directory in budget.
cost := uint64(zipdir.HeaderLen) + uint64(nameLen)
if cost > maxDirBytes {
return n, fmt.Errorf("parseworker: listing record %d: %w", n, zipdir.ErrScanLimit)
}
maxDirBytes -= cost
name := make([]byte, nameLen)
if _, err := io.ReadFull(br, name); err != nil {
return n, fmt.Errorf("parseworker: listing record %d truncated: %w", n, noEOF(err))
}
n++
if err := visit(string(name), head[0]&flagDir != 0); err != nil {
return n, err
}
}
}
func noEOF(err error) error {
if errors.Is(err, io.EOF) {
return io.ErrUnexpectedEOF
}
return err
}
package parseworker
import (
"errors"
"fmt"
"os"
"golang.org/x/sys/unix"
)
const listingSeals = unix.F_SEAL_WRITE | unix.F_SEAL_GROW | unix.F_SEAL_SHRINK
// sealFile seals a memfd against writes, growth and shrinkage.
func sealFile(f *os.File) error {
_, err := unix.FcntlInt(f.Fd(), unix.F_ADD_SEALS, listingSeals)
return err
}
// checkSealed verifies a memfd carries the seals sealFile adds.
func checkSealed(f *os.File) error {
raw, err := f.SyscallConn()
if err != nil {
return err
}
var seals int
var sealErr error
if err := raw.Control(func(fd uintptr) {
seals, sealErr = unix.FcntlInt(fd, unix.F_GET_SEALS, 0)
}); err != nil {
return err
}
if sealErr != nil {
return fmt.Errorf("listing seals: %w", sealErr)
}
if seals&listingSeals != listingSeals {
return errors.New("listing is not sealed against writes and size changes")
}
return nil
}
//go:build linux
package parseworker
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"net"
"os"
"golang.org/x/sys/unix"
"github.com/pidginhost/csm/internal/workerproc"
)
// Serve answers parse requests on conn until the daemon closes it.
func Serve(ctx context.Context, conn *net.UnixConn) error {
return workerproc.Serve(ctx, conn, WorkerLimits, handle)
}
func handle(ctx context.Context, req workerproc.Frame, fds []*os.File) (workerproc.Frame, []*os.File) {
switch req.Op {
case workerproc.OpHello:
return workerproc.HelloReply(ProtocolVersion), nil
case OpListZip:
return listZip(ctx, req, fds)
case OpExtractMessage:
return extractMessage(req, fds)
default:
return workerproc.Frame{Error: fmt.Sprintf("unknown op %q", req.Op)}, nil
}
}
func listZip(ctx context.Context, req workerproc.Frame, fds []*os.File) (workerproc.Frame, []*os.File) {
if len(fds) != 1 {
return workerproc.Frame{Error: fmt.Sprintf("list_zip takes one descriptor, got %d", len(fds))}, nil
}
var args ListZipArgs
dec := json.NewDecoder(bytes.NewReader(req.Payload))
dec.DisallowUnknownFields()
if err := dec.Decode(&args); err != nil {
return workerproc.Frame{Error: fmt.Sprintf("list_zip arguments: %v", err)}, nil
}
st, err := fds[0].Stat()
if err != nil {
return workerproc.Frame{Error: err.Error()}, nil
}
if !st.Mode().IsRegular() {
return workerproc.Frame{Error: "list_zip descriptor is not a regular file"}, nil
}
fd, err := unix.MemfdCreate("csm-zip-listing", unix.MFD_CLOEXEC|unix.MFD_ALLOW_SEALING)
if err != nil {
return workerproc.Frame{Error: fmt.Sprintf("memfd: %v", err)}, nil
}
listing := os.NewFile(uintptr(fd), "csm-zip-listing")
w := bufio.NewWriterSize(listing, 64<<10)
var n uint64
err = ListEntries(ctx, fds[0], st.Size(), args.MaxDirBytes, func(name string, dir bool) error {
n++
return writeEntry(w, name, dir)
})
if err == nil {
err = w.Flush()
}
if err != nil {
_ = listing.Close()
reason := err.Error()
if len(reason) > maxIncomplete {
reason = reason[:maxIncomplete]
}
return result(ListZipResult{Incomplete: reason}), nil
}
if _, err := unix.FcntlInt(listing.Fd(), unix.F_ADD_SEALS, listingSeals); err != nil {
_ = listing.Close()
return workerproc.Frame{Error: fmt.Sprintf("seal listing: %v", err)}, nil
}
return result(ListZipResult{Entries: n}), []*os.File{listing}
}
func result(r ListZipResult) workerproc.Frame {
payload, _ := json.Marshal(r) // A number and a string cannot fail to marshal.
return workerproc.Frame{Payload: payload}
}
package parseworker
import (
"strconv"
"time"
"github.com/pidginhost/csm/internal/workerproc"
)
// DefaultMemoryLimit caps a parse worker's memory and swap together when
// nothing else is configured: the zip lister and the CLI always use it.
const DefaultMemoryLimit = 512 << 20
// UnitConfig is the locked-down unit a parse worker runs in: binary is the
// csm executable, prefix names the units, memoryLimit caps memory and swap,
// fileSizeLimit bounds what the worker may write into a memfd. The daemon
// and the CLI share it.
func UnitConfig(binary, prefix string, memoryLimit, fileSizeLimit uint64, log func(string, ...any)) workerproc.UnitConfig {
return workerproc.UnitConfig{
Prefix: prefix,
Command: binary,
Args: []string{"parse-worker", "--memory-limit=" + strconv.FormatUint(memoryLimit, 10)},
MemoryLimit: memoryLimit,
FileSizeLimit: fileSizeLimit,
RuntimeMax: time.Hour,
StartTimeout: 10 * time.Second,
Protocol: ProtocolVersion,
Limits: Limits,
Log: log,
}
}
package phptaint
import (
"fmt"
"strings"
"unicode"
"unicode/utf8"
)
// Display bounds. Attacker-controlled identifiers must not be able to
// produce unbounded findings or logs.
const (
maxSegmentBytes = 64
maxChainSegments = 32
chainHeadKeep = 16
chainTailKeep = 15
)
// sanitizeSegment renders one untrusted display segment: valid UTF-8,
// non-printing and invalid bytes become '?', truncation happens at a rune
// boundary and the marker counts into the cap.
func sanitizeSegment(s string) string {
clean, _ := sanitize(s, maxSegmentBytes)
return clean
}
// sanitizeReason bounds Report.Reason the same way.
func sanitizeReason(s string) string {
clean, _ := sanitize(s, MaxReasonBytes)
return clean
}
func sanitize(s string, maxBytes int) (string, bool) {
s = strings.ToValidUTF8(s, "?")
s = strings.Map(func(r rune) rune {
// IsPrint excludes format controls (including bidi overrides) and the
// Unicode line/paragraph separators as well as ordinary control bytes.
if !unicode.IsPrint(r) {
return '?'
}
return r
}, s)
if len(s) <= maxBytes {
return s, false
}
cut := maxBytes - 3
for cut > 0 && !utf8.RuneStart(s[cut]) {
cut--
}
return s[:cut] + "...", true
}
// truncateChain sanitizes and bounds every segment, then keeps a bounded head
// and tail with one marker naming any omitted segments.
func truncateChain(via []string) ([]string, bool) {
bounded := make([]string, len(via))
truncated := false
for i, segment := range via {
var cut bool
bounded[i], cut = sanitize(segment, maxSegmentBytes)
truncated = truncated || cut
}
if len(bounded) <= maxChainSegments {
return bounded, truncated
}
omitted := len(bounded) - chainHeadKeep - chainTailKeep
out := make([]string, 0, maxChainSegments)
out = append(out, bounded[:chainHeadKeep]...)
out = append(out, fmt.Sprintf("... %d segment(s) omitted ...", omitted))
out = append(out, bounded[len(bounded)-chainTailKeep:]...)
return out, true
}
package phptaint
import (
"sort"
"strings"
"sync/atomic"
"github.com/VKCOM/php-parser/pkg/ast"
"github.com/VKCOM/php-parser/pkg/visitor"
"github.com/VKCOM/php-parser/pkg/visitor/traverser"
)
// callSinks are code-execution sinks that appear as ordinary function calls,
// mapped to the index of the argument that gets executed. assert() evaluates
// its first argument on PHP 7, which hosts still run; create_function()
// evaluates the code in its second argument.
var callSinks = map[string]int{"assert": 0, "create_function": 1}
// alwaysBoolPredicates are PHP builtins whose return type is bool for every
// input. A call to one cannot hand assert() a string to execute.
var alwaysBoolPredicates = map[string]bool{
"is_array": true, "is_bool": true, "is_callable": true, "is_countable": true,
"is_double": true, "is_float": true, "is_int": true, "is_integer": true,
"is_iterable": true, "is_long": true, "is_null": true, "is_numeric": true,
"is_object": true, "is_resource": true, "is_scalar": true, "is_string": true,
"is_subclass_of": true, "is_a": true,
"array_key_exists": true, "in_array": true, "property_exists": true,
"method_exists": true, "function_exists": true, "class_exists": true,
"interface_exists": true, "defined": true, "file_exists": true,
"is_dir": true, "is_file": true, "is_readable": true, "is_writable": true,
"str_contains": true, "str_starts_with": true, "str_ends_with": true,
"ctype_digit": true, "ctype_alpha": true, "ctype_alnum": true,
}
// assertArgumentCouldBeString reports whether e's top-level shape permits a
// string value. This is exactly what decides whether assert(e) can execute
// code on any PHP version this analyzer targets: PHP 7 only ever evaluated a
// *string* argument as code, and PHP 8 removed the eval form entirely. A
// logical, comparison, identity, or instanceof expression can only ever
// produce a bool (or, for <=>, an int) -- never a string -- so it is not a
// code-execution sink regardless of what it compares.
//
// The check is shallow by design: only e's own node type is inspected, not
// its operands. assert($a && fetchCode()) must still be excluded, because
// the value actually passed to assert() is the bool && produces, not the
// string one of its operands would have produced on its own.
func assertArgumentCouldBeString(e ast.Vertex) bool {
// A call to a predicate returns a bool whatever its argument was, so the
// value assert() receives can never be a string. This stays shallow for
// the same reason as the operator cases: only the outermost node decides
// what assert() is actually handed.
if call, ok := e.(*ast.ExprFunctionCall); ok {
return !alwaysBoolPredicates[calleeName(call.Function)]
}
if _, ok := e.(*ast.ExprEmpty); ok {
return false
}
if _, ok := e.(*ast.ExprIsset); ok {
return false
}
switch e.(type) {
case *ast.ExprBinaryBooleanAnd, *ast.ExprBinaryBooleanOr,
*ast.ExprBinaryLogicalAnd, *ast.ExprBinaryLogicalOr, *ast.ExprBinaryLogicalXor,
*ast.ExprBooleanNot,
*ast.ExprBinaryEqual, *ast.ExprBinaryNotEqual,
*ast.ExprBinaryIdentical, *ast.ExprBinaryNotIdentical,
*ast.ExprBinaryGreater, *ast.ExprBinaryGreaterOrEqual,
*ast.ExprBinarySmaller, *ast.ExprBinarySmallerOrEqual,
*ast.ExprBinarySpaceship,
*ast.ExprInstanceOf:
return false
}
return true
}
// sinkSite is one code-execution construct and the expression it executes.
type sinkSite struct {
kind string
expr ast.Vertex
}
type callSite struct {
name string
node ast.Vertex
}
// scopeFacts is everything the taint pass needs from one lexical scope.
// Collection is one pass; the analysis reads it repeatedly.
type scopeFacts struct {
assigns []*ast.ExprAssign
references []*ast.ExprAssignReference
concats []*ast.ExprAssignConcat
returns []*ast.StmtReturn
funcs []*ast.StmtFunction
methods []*ast.StmtClassMethod
// closures and arrowFuncs hold every closure and arrow function found in
// this scope. Both are declarations in the same sense funcs and methods
// are: each gets its own entry in declarationTree (so its span is
// excluded from the enclosing scope) and its own per-scope analysis in
// analyze, exactly like a named function's body. Without both halves, a
// closure parameter or local reassignment that merely shares a name with
// an outer variable would either borrow that outer variable's taint (if
// only excluded, never analysed) or stop being examined for its own
// sinks (if only analysed, never excluded) -- see analyze's closure and
// arrow-function loops for the second half.
closures []*ast.ExprClosure
arrowFuncs []*ast.ExprArrowFunction
// classLikes holds every class, interface, trait, and enum declaration
// (named or anonymous) found in this scope. Their own contents are
// never read from here directly -- methods are already tracked in
// methods above -- this list exists only to supply declarationTree
// with the position span of the class body itself.
classLikes []ast.Vertex
sinks []sinkSite
callNodes []*ast.ExprFunctionCall
callSites []callSite
calls map[string]bool
vars map[string]bool
varNodes []*ast.ExprVariable
// propNodes holds every property-fetch node (both "->" and "?->") found
// in this scope, whether it appears as a read or as an assignment
// target. readVarNodes keys each one to the specific property it fetches
// (see assignedTargetKey) so a write to one property never taints a read
// of a different one or of the bare base object.
propNodes []ast.Vertex
writes []ast.Vertex
// fileWrites holds every file_put_contents(path, data) call in this
// scope. It is the bridge the fetch-write-include dropper relies on:
// remote content lands in a local file that an include then executes
// by path, with no variable ever carrying the taint into the sink.
fileWrites []fileWriteSite
precisionLoss map[string]bool
visited int
budgetExceeded bool
// declTreeCache memoizes declarationTree's result for this scopeFacts
// instance. Safe to cache: a scopeFacts is fully populated by the
// traversal in collectScope/collectAll/collectOwnStmts before any
// caller can reach it and is never mutated afterward, and each
// instance is freshly allocated per collection call, so nothing here
// carries state across separate Analyze invocations. This is what lets
// functionSummaries call f.declarationTree() again on the same f
// analyze already indexed without repeating the O(D log D) build.
declTreeCache *declTree
}
func newScopeFacts() *scopeFacts {
return &scopeFacts{
calls: map[string]bool{},
vars: map[string]bool{},
precisionLoss: map[string]bool{},
}
}
// count enforces the collection budget. Once exceeded, collection stops
// recording so a hostile file cannot grow analysis state without bound.
func (f *scopeFacts) count() bool {
if f.budgetExceeded {
return false
}
f.visited++
if f.visited > maxCollectedNodes {
f.budgetExceeded = true
return false
}
return true
}
// factVisitor implements only the node types the analysis needs; every other
// node type is a no-op inherited from visitor.Null.
type factVisitor struct {
visitor.Null
f *scopeFacts
functionAliases map[string]string
// exclude, when set, marks the position spans of nested declarations
// this scope must not absorb facts from. The library traverser recurses
// unconditionally through control-flow wrappers (if/while/switch/try/
// foreach/...), so a declaration nested inside one of those -- the
// WordPress `if (!function_exists(...))` guard is the common case -- is
// still reached by this same traversal even though it belongs to a
// separately analysed scope. Position-based exclusion catches it
// regardless of which wrapper (or how many, nested how deep) sits
// between this scope and the declaration; a filter keyed on the direct
// statement type of the top of the list cannot, because it only ever
// sees the wrapper, never what the wrapper contains.
exclude *spanIndex
}
// excluded reports whether n's position falls inside a nested declaration
// this scope must not record facts from. A node without a determinable
// position is not excluded: parsed nodes normally always have positions,
// and failing open (keep, don't drop) matches the conservative choice
// already made in readVarNodes for the same edge case -- an unrecordable
// position must never cost the enclosing scope one of its own facts.
func (v *factVisitor) excluded(n ast.Vertex) bool {
if v.exclude == nil {
return false
}
span, ok := spanOf(n)
if !ok {
return false
}
return v.exclude.contains(span)
}
func (v *factVisitor) StmtNamespace(*ast.StmtNamespace) {
v.functionAliases = nil
}
func (v *factVisitor) StmtUse(n *ast.StmtUseList) {
for _, useNode := range n.Uses {
use, ok := useNode.(*ast.StmtUse)
if ok {
v.addFunctionAlias(n.Type, nil, use)
}
}
}
func (v *factVisitor) StmtGroupUse(n *ast.StmtGroupUseList) {
prefix, _ := n.Prefix.(*ast.Name)
for _, useNode := range n.Uses {
use, ok := useNode.(*ast.StmtUse)
if ok {
v.addFunctionAlias(n.Type, prefix, use)
}
}
}
func (v *factVisitor) addFunctionAlias(listType ast.Vertex, prefix *ast.Name, use *ast.StmtUse) {
useType := listType
if use.Type != nil {
useType = use.Type
}
id, ok := useType.(*ast.Identifier)
if !ok || !strings.EqualFold(string(id.Value), "function") {
return
}
name, ok := use.Use.(*ast.Name)
if !ok || len(name.Parts) == 0 {
return
}
parts := make([]ast.Vertex, 0, len(name.Parts)+4)
if prefix != nil {
parts = append(parts, prefix.Parts...)
}
parts = append(parts, name.Parts...)
target := joinNameParts(parts)
alias := ""
if use.Alias != nil {
alias = calleeName(use.Alias)
} else if last, ok := name.Parts[len(name.Parts)-1].(*ast.NamePart); ok {
alias = strings.ToLower(string(last.Value))
}
if alias == "" || target == "" {
return
}
if v.functionAliases == nil {
v.functionAliases = map[string]string{}
}
v.functionAliases[alias] = target
}
func (v *factVisitor) ExprVariable(n *ast.ExprVariable) {
if !v.f.count() || v.excluded(n) {
return
}
name := varName(n.Name)
v.f.vars[name] = true
v.f.varNodes = append(v.f.varNodes, n)
if name == "" {
v.f.precisionLoss["variable-variable"] = true
}
}
// ExprPropertyFetch and ExprNullsafePropertyFetch record every "->"/"?->"
// property fetch this scope contains, whether it is a read or (for the
// non-nullsafe form; nullsafe cannot appear as a write target in valid PHP)
// an assignment target. The traverser still visits the embedded base
// variable (e.g. $this inside $this->body) as an ordinary ExprVariable
// regardless of this hook, which is what lets a genuinely whole-object
// taint keep reaching a property read -- see readVarNodes.
func (v *factVisitor) ExprPropertyFetch(n *ast.ExprPropertyFetch) {
if !v.f.count() || v.excluded(n) {
return
}
v.f.propNodes = append(v.f.propNodes, n)
}
func (v *factVisitor) ExprNullsafePropertyFetch(n *ast.ExprNullsafePropertyFetch) {
if !v.f.count() || v.excluded(n) {
return
}
v.f.propNodes = append(v.f.propNodes, n)
}
func (v *factVisitor) ExprFunctionCall(n *ast.ExprFunctionCall) {
if !v.f.count() || v.excluded(n) {
return
}
name, resolved := v.resolveFunctionAlias(n)
v.f.calls[name] = true
v.f.callNodes = append(v.f.callNodes, resolved)
v.f.callSites = append(v.f.callSites, callSite{name: name, node: n})
if name == "" {
v.f.precisionLoss["dynamic-call"] = true
}
// assert() and create_function() are ordinary calls in the grammar, not
// dedicated nodes, so they are recognised here rather than by node type.
// The executed argument differs per sink (assert's is first,
// create_function's is second), so the index is looked up rather than
// assumed to be the last argument. assert() additionally requires its
// argument to be capable of holding a string: see
// assertArgumentCouldBeString for why a boolean-shaped argument (a
// comparison, a logical operator, instanceof, ...) is not a
// code-execution sink on any PHP version this analyzer targets.
if sink, ok := callSinkSite(name, n); ok {
v.f.sinks = append(v.f.sinks, sink)
}
if w, ok := callFileWriteSite(name, n); ok {
v.f.fileWrites = append(v.f.fileWrites, w)
}
}
// fileWriteSite is one file_put_contents(path, data) call.
type fileWriteSite struct {
path ast.Vertex
data ast.Vertex
}
// fileWriteCalls write their second argument to the path in their first.
var fileWriteCalls = map[string]bool{"file_put_contents": true}
func callFileWriteSite(name string, node ast.Vertex) (fileWriteSite, bool) {
call, ok := node.(*ast.ExprFunctionCall)
if !ok || !fileWriteCalls[name] || len(call.Args) < 2 {
return fileWriteSite{}, false
}
pathArg, okPath := call.Args[0].(*ast.Argument)
dataArg, okData := call.Args[1].(*ast.Argument)
if !okPath || !okData {
return fileWriteSite{}, false
}
return fileWriteSite{path: pathArg.Expr, data: dataArg.Expr}, true
}
func callSinkSite(name string, node ast.Vertex) (sinkSite, bool) {
call, ok := node.(*ast.ExprFunctionCall)
if !ok {
return sinkSite{}, false
}
idx, ok := callSinks[name]
if !ok || len(call.Args) <= idx {
return sinkSite{}, false
}
arg, ok := call.Args[idx].(*ast.Argument)
if !ok || (name == "assert" && !assertArgumentCouldBeString(arg.Expr)) {
return sinkSite{}, false
}
return sinkSite{kind: name, expr: arg.Expr}, true
}
// resolveFunctionAlias resolves a call target through this scope's
// function-alias table (populated from `use function` imports) and reports
// the canonical name. When the call is aliased, it returns a detached copy
// of the call node whose Function names the canonical target, so downstream
// name lookups (sourceGrade, decoder checks) see the resolved name
// without the parsed tree itself ever being rewritten mid-traversal: the
// traverser reads call.Function right after visiting call, so mutating it in
// place would orphan the original alias-name subtree from the walk in
// progress. The copy shares Args and Position with the original node, so
// argument inspection and source-span correlation are unaffected.
func (v *factVisitor) resolveFunctionAlias(call *ast.ExprFunctionCall) (string, *ast.ExprFunctionCall) {
name := calleeName(call.Function)
plain, ok := call.Function.(*ast.Name)
if !ok || len(plain.Parts) != 1 {
return name, call
}
target, ok := v.functionAliases[name]
if !ok {
return name, call
}
parts := strings.Split(target, "\\")
canonical := make([]ast.Vertex, 0, len(parts))
for _, part := range parts {
canonical = append(canonical, &ast.NamePart{Value: []byte(part)})
}
resolved := *call
resolved.Function = &ast.NameFullyQualified{Parts: canonical}
return target, &resolved
}
func (v *factVisitor) ExprMethodCall(n *ast.ExprMethodCall) {
v.call(calleeName(n.Method), n)
}
func (v *factVisitor) ExprNullsafeMethodCall(n *ast.ExprNullsafeMethodCall) {
v.call(calleeName(n.Method), n)
}
func (v *factVisitor) ExprStaticCall(n *ast.ExprStaticCall) {
v.call(calleeName(n.Call), n)
}
func (v *factVisitor) call(name string, node ast.Vertex) {
if !v.f.count() || v.excluded(node) {
return
}
v.f.calls[name] = true
v.f.callSites = append(v.f.callSites, callSite{name: name, node: node})
if name == "" {
v.f.precisionLoss["dynamic-call"] = true
}
}
func (v *factVisitor) ExprAssign(n *ast.ExprAssign) {
if !v.f.count() || v.excluded(n) {
return
}
v.f.assigns = append(v.f.assigns, n)
v.f.writes = append(v.f.writes, n.Var)
}
func (v *factVisitor) ExprAssignReference(n *ast.ExprAssignReference) {
if !v.f.count() || v.excluded(n) {
return
}
v.f.references = append(v.f.references, n)
v.f.writes = append(v.f.writes, n.Var)
}
func (v *factVisitor) ExprAssignConcat(n *ast.ExprAssignConcat) {
if !v.f.count() || v.excluded(n) {
return
}
v.f.concats = append(v.f.concats, n)
}
func (v *factVisitor) StmtReturn(n *ast.StmtReturn) {
if !v.f.count() || v.excluded(n) {
return
}
v.f.returns = append(v.f.returns, n)
}
func (v *factVisitor) StmtFunction(n *ast.StmtFunction) {
if !v.f.count() || v.excluded(n) {
return
}
v.f.funcs = append(v.f.funcs, n)
}
func (v *factVisitor) StmtClassMethod(n *ast.StmtClassMethod) {
if !v.f.count() || v.excluded(n) {
return
}
v.f.methods = append(v.f.methods, n)
}
func (v *factVisitor) ExprClosure(n *ast.ExprClosure) {
if !v.f.count() || v.excluded(n) {
return
}
v.f.closures = append(v.f.closures, n)
}
func (v *factVisitor) ExprArrowFunction(n *ast.ExprArrowFunction) {
if !v.f.count() || v.excluded(n) {
return
}
v.f.arrowFuncs = append(v.f.arrowFuncs, n)
}
func (v *factVisitor) StmtClass(n *ast.StmtClass) { v.classLike(n) }
func (v *factVisitor) StmtInterface(n *ast.StmtInterface) { v.classLike(n) }
func (v *factVisitor) StmtTrait(n *ast.StmtTrait) { v.classLike(n) }
func (v *factVisitor) StmtEnum(n *ast.StmtEnum) { v.classLike(n) }
// classLike records a class, interface, trait, or enum declaration's span.
// n's Name is nil for an anonymous class (`new class { ... }`); the node
// itself still carries a real position, so anonymous classes are covered by
// the same declaration-span exclusion as named ones, with no separate case.
func (v *factVisitor) classLike(n ast.Vertex) {
if !v.f.count() || v.excluded(n) {
return
}
v.f.classLikes = append(v.f.classLikes, n)
}
func (v *factVisitor) ExprEval(n *ast.ExprEval) { v.sink("eval", n.Expr) }
func (v *factVisitor) ExprInclude(n *ast.ExprInclude) { v.sink("include", n.Expr) }
func (v *factVisitor) ExprIncludeOnce(n *ast.ExprIncludeOnce) { v.sink("include_once", n.Expr) }
func (v *factVisitor) ExprRequire(n *ast.ExprRequire) { v.sink("require", n.Expr) }
func (v *factVisitor) ExprRequireOnce(n *ast.ExprRequireOnce) { v.sink("require_once", n.Expr) }
func (v *factVisitor) sink(kind string, expr ast.Vertex) {
if !v.f.count() || v.excluded(expr) {
return
}
v.f.sinks = append(v.f.sinks, sinkSite{kind: kind, expr: expr})
}
// collectScope gathers facts from one subtree using the parser library's own
// traverser. The library traverser cannot be stopped mid-walk, so it visits
// every node in the subtree regardless of budget; scopeFacts.count() bounds
// the actual cost by making every visitor method a no-op once the budget is
// exceeded, so an over-budget file does no further per-node work beyond the
// traversal dispatch itself. That traverser was measured surviving 2,000,000
// levels of nesting without a Go stack overflow, so a hand-rolled iterative
// walker is not needed to protect against attacker-controlled AST depth.
func collectScope(n ast.Vertex) *scopeFacts {
f := newScopeFacts()
if n == nil {
return f
}
traverser.NewTraverser(&factVisitor{f: f}).Traverse(n)
return f
}
// collectTopLevel gathers facts from the whole file, excluding anything
// positioned inside a declaration named in exclude. Folding a declaration's
// body into the flat top-level map lets a variable local to it taint an
// unrelated top-level variable that merely shares its name, which reports
// clean code as malicious: names like $data, $content and $tmp recur
// constantly in real PHP. exclude is normally built by
// declarationTree(...).exclusionFor(nil) over the whole file, so every
// declaration in the file is excluded (there is no declaration whose own
// statements this call needs to keep).
func collectTopLevel(root ast.Vertex, exclude *spanIndex) *scopeFacts {
r, ok := root.(*ast.Root)
if !ok {
return collectScope(root)
}
return collectOwnStmts(r.Stmts, exclude)
}
// collectOwnStmts gathers facts from a statement list -- a function body, a
// method body, or the file's top-level statements -- excluding anything
// positioned inside a nested declaration named in exclude. This is the same
// cross-scope leak collectTopLevel guards against, one level deeper: a
// function can declare another function (or a class, including an
// anonymous one) in its own body, and that nested declaration gets its own
// entry in the whole-file funcs/methods inventory, analysed separately with
// its own taint state. Folding it into this scope's flat map too would let
// its local variables taint an identically-named local in the enclosing
// body.
//
// Filtering happens by position, not by skipping statements of a
// declaration type at the top of stmts, because the library traverser
// recurses unconditionally through control-flow wrappers: a function
// declared inside `if (!function_exists('f')) { function f() {...} }` --
// the standard WordPress conditional-declaration guard -- is not a direct
// member of stmts, so a type-based filter on stmts itself would miss it
// (and miss it again for every layer of if/while/switch/try/foreach it is
// wrapped in). Every statement is traversed unconditionally here; exclude,
// consulted per node inside factVisitor, is what actually drops facts whose
// position falls inside one of those nested declarations, regardless of
// how they are reached.
func collectOwnStmts(stmts []ast.Vertex, exclude *spanIndex) *scopeFacts {
f := newScopeFacts()
t := traverser.NewTraverser(&factVisitor{f: f, exclude: exclude})
for _, stmt := range stmts {
if stmt != nil {
t.Traverse(stmt)
}
}
return f
}
// methodStmts unwraps a class method's single body vertex into a statement
// list, the shape collectOwnStmts and collectAll both expect. A braced
// method body is *ast.StmtStmtList; an abstract or interface method has a
// nil Stmt.
func methodStmts(stmt ast.Vertex) []ast.Vertex {
switch v := stmt.(type) {
case nil:
return nil
case *ast.StmtStmtList:
return v.Stmts
default:
return []ast.Vertex{stmt}
}
}
// arrowFunctionBody wraps an arrow function's single implicit-return
// expression into the one-element statement list collectOwnStmts expects,
// the same shape methodStmts gives a method body. Unlike a closure (whose
// Stmts is already a real statement list), `fn($x) => expr` has no braces or
// statements to unwrap -- expr itself is both the body and the return value.
func arrowFunctionBody(n *ast.ExprArrowFunction) []ast.Vertex {
if n.Expr == nil {
return nil
}
return []ast.Vertex{n.Expr}
}
// closureCaptureNames names the outer bindings a closure's use() clause
// receives. Only Uses is consulted, never Params: a parameter that happens to
// share an outer variable's name is the closure's own binding and receives
// nothing from the enclosing scope, which is the shadowing case that scoping a
// closure's body exists to keep clean.
func closureCaptureNames(cl *ast.ExprClosure) map[string]bool {
names := make(map[string]bool, len(cl.Uses))
for _, u := range cl.Uses {
use, ok := u.(*ast.ExprClosureUse)
if !ok {
continue
}
v, ok := use.Var.(*ast.ExprVariable)
if !ok {
continue
}
if name := varName(v.Name); name != "" {
names[name] = true
}
}
return names
}
// paramNames names the plain-variable parameters of a parameter list. A
// parameter with any other shape is skipped rather than guessed at.
func paramNames(params []ast.Vertex) map[string]bool {
names := make(map[string]bool, len(params))
for _, p := range params {
param, ok := p.(*ast.Parameter)
if !ok {
continue
}
v, ok := param.Var.(*ast.ExprVariable)
if !ok {
continue
}
if name := varName(v.Name); name != "" {
names[name] = true
}
}
return names
}
// arrowCaptureNames names the outer bindings an arrow function receives.
// Unlike a closure, which lists them in a use() clause, fn(...) => expr
// captures by value every enclosing variable its body mentions. body.vars
// already lists every variable name the body reads, so subtracting the arrow
// function's own parameters leaves exactly the names that must have come from
// outside. Static arrows are the one exception: they still capture ordinary
// variables, but PHP deliberately does not bind $this to them. Over-inclusive
// by design: a name that is not in fact tainted in the enclosing scope simply
// produces no marker.
func arrowCaptureNames(af *ast.ExprArrowFunction, body *scopeFacts) map[string]bool {
params := paramNames(af.Params)
names := make(map[string]bool, len(body.vars))
for name := range body.vars {
if name != "" && !params[name] && (name != "this" || af.StaticTkn == nil) {
names[name] = true
}
}
return names
}
type nodeSpan struct {
start int
end int
}
type declarationSpan struct {
nodeSpan
node ast.Vertex
}
type namedNodeSpan struct {
nodeSpan
name string
node ast.Vertex
}
// resolvedCallIndex restores the canonical call names from a whole-file
// collection to independently collected lexical scopes. Namespace-level
// `use function` imports are outside a function or method body, so collecting
// that body alone cannot resolve its aliases. Exact source spans join the two
// views without importing variables, assignments, or calls from another
// scope.
type resolvedCallIndex struct {
functions map[nodeSpan]*ast.ExprFunctionCall
sites map[nodeSpan]callSite
}
func newResolvedCallIndex(f *scopeFacts) resolvedCallIndex {
index := resolvedCallIndex{
functions: make(map[nodeSpan]*ast.ExprFunctionCall, len(f.callNodes)),
sites: make(map[nodeSpan]callSite, len(f.callSites)),
}
for _, call := range f.callNodes {
if span, ok := spanOf(call); ok {
index.functions[span] = call
}
}
for _, call := range f.callSites {
if span, ok := spanOf(call.node); ok {
index.sites[span] = call
}
}
return index
}
func (index resolvedCallIndex) apply(f *scopeFacts) *scopeFacts {
out := *f
out.callNodes = append([]*ast.ExprFunctionCall(nil), f.callNodes...)
for i, call := range out.callNodes {
if span, ok := spanOf(call); ok {
if resolved, found := index.functions[span]; found {
out.callNodes[i] = resolved
}
}
}
out.callSites = append([]callSite(nil), f.callSites...)
for i, call := range out.callSites {
if span, ok := spanOf(call.node); ok {
if resolved, found := index.sites[span]; found {
out.callSites[i] = resolved
}
}
}
out.calls = make(map[string]bool, len(out.callSites))
for _, call := range out.callSites {
out.calls[call.name] = true
}
out.sinks = make([]sinkSite, 0, len(f.sinks))
for _, sink := range f.sinks {
if _, callSink := callSinks[sink.kind]; !callSink {
out.sinks = append(out.sinks, sink)
}
}
for _, call := range out.callSites {
if sink, ok := callSinkSite(call.name, call.node); ok {
out.sinks = append(out.sinks, sink)
}
}
return &out
}
// declTree links each declaration in a whole-file collection to the
// position spans of its IMMEDIATE child declarations only, plus the total
// declaration count. Built once per file by declarationTree, it lets every
// scope's exclusion index be produced by a map lookup instead of scanning
// and re-sorting the whole file's declaration list once per declaration.
type declTree struct {
// children maps a declaration node -- or nil, for the top-level scope --
// to the spans of its immediate child declarations.
children map[ast.Vertex][]nodeSpan
// parent is the reverse: each indexed declaration to the declaration
// immediately enclosing it, or the nil interface when the top-level
// scope encloses it. Captured during the same sweep that builds
// children, since the sweep already knows both ends of the edge. It
// answers the question children cannot: given a closure, which scope did
// it capture its outer bindings FROM.
parent map[ast.Vertex]ast.Vertex
// ordered holds the same declarations in source nesting order. Retaining
// the already-sorted sweep input lets capture analysis walk lexical scopes
// without rebuilding or re-sorting the attacker-controlled declaration
// list.
ordered []declarationSpan
// count is the total number of declarations indexed, checked against
// maxDeclarations before any per-scope work begins.
count int
}
// declTreeBuilds counts how many times declarationTree has actually
// performed the O(D log D) sort-and-sweep build, as opposed to returning an
// already-cached result. It exists solely so a same-package white-box test
// can observe that this count stays a small constant as declaration count
// grows -- the structural invariant this whole file's design establishes --
// rather than asserting elapsed wall-clock time, which is flaky under
// shared CI load and, worse, would not fail reliably if a per-declaration
// rebuild were reintroduced at a scale too small to visibly stall a test
// run. It is read-only from every caller's perspective (nothing in this
// package branches on its value, and it never influences a Report), so it
// is test-only observation of internal behaviour, not the kind of mutable
// process-global configuration or cross-call state this package's purity
// contract forbids.
var declTreeBuilds atomic.Int64
// declarationTree indexes every declaration recorded in f: functions,
// methods, classes/interfaces/traits/enums (anonymous classes included, via
// classLikes), closures, and arrow functions. f must come from an
// unfiltered, whole-file collection (collectScope(root)) so every
// declaration in the file is present, regardless of how deeply any of them
// is nested inside another or wrapped in control flow. The result is cached
// on f (see scopeFacts.declTreeCache), so calling this more than once on the
// same f -- analyze and functionSummaries both do, independently -- costs
// one build, not two.
//
// Sorting happens ONCE, over all D declarations together, rather than once
// per declaration: a per-declaration rebuild-and-sort of the whole span
// list costs O(D) work times D declarations, O(D^2 log D) overall, which is
// a real CPU-exhaustion surface against attacker-controlled PHP source (a
// file of trivial one-line function declarations reaches tens of thousands
// of them well within MaxSourceBytes and maxCollectedNodes). Declarations in
// a legitimately parsed file nest properly -- one is either fully disjoint
// from another or fully contained inside it, never a partial overlap -- so
// a single stack sweep over the sorted spans (the same interval-stack
// technique distributeOrigins already uses in taint.go) assigns each
// declaration to its immediate parent in one pass: pop any open declaration
// that has already closed before this one starts, and whatever remains on
// top of the stack (if anything) is the immediate parent.
//
// Indexing only immediate children, not every descendant, is what keeps the
// total cost linearithmic in D: a fact positioned inside a deeper
// descendant is already covered by its immediate parent's span, so nothing
// beyond direct children is ever needed to exclude an arbitrarily deep
// nested declaration (see exclusionFor). Each declaration contributes to
// exactly one parent's child list, so those lists sum to D across the whole
// file, and sorting each of them once, at exclusionFor time, costs at most
// D log D in total across every scope in the file.
func (f *scopeFacts) declarationTree() declTree {
if f.declTreeCache != nil {
return *f.declTreeCache
}
declTreeBuilds.Add(1)
nodes := make([]declarationSpan, 0, len(f.funcs)+len(f.methods)+len(f.classLikes)+len(f.closures)+len(f.arrowFuncs))
add := func(n ast.Vertex) {
if span, ok := spanOf(n); ok {
nodes = append(nodes, declarationSpan{nodeSpan: span, node: n})
}
}
for _, fn := range f.funcs {
add(fn)
}
for _, m := range f.methods {
add(m)
}
for _, c := range f.classLikes {
add(c)
}
for _, cl := range f.closures {
add(cl)
}
for _, af := range f.arrowFuncs {
add(af)
}
sort.Slice(nodes, func(i, j int) bool {
if nodes[i].start != nodes[j].start {
return nodes[i].start < nodes[j].start
}
return nodes[i].end > nodes[j].end
})
tree := declTree{
children: make(map[ast.Vertex][]nodeSpan, len(nodes)+1),
parent: make(map[ast.Vertex]ast.Vertex, len(nodes)),
ordered: nodes,
count: len(nodes),
}
stack := make([]int, 0, len(nodes))
for i, n := range nodes {
// A declaration's end position is EXCLUSIVE, so a declaration
// starting exactly where the previous one ends is its sibling, not
// its child. PHP allows `}function` with nothing between, and
// minifiers emit it, so `<` here would silently reparent every such
// pair and leak the second one's locals into the enclosing scope.
for len(stack) > 0 && nodes[stack[len(stack)-1]].end <= n.start {
stack = stack[:len(stack)-1]
}
var parent ast.Vertex // nil selects the top-level scope's own children
if len(stack) > 0 {
parent = nodes[stack[len(stack)-1]].node
}
tree.children[parent] = append(tree.children[parent], n.nodeSpan)
tree.parent[n.node] = parent
stack = append(stack, i)
}
f.declTreeCache = &tree
return tree
}
// exclusionFor builds the spanIndex scope self must exclude from its own
// collection: the spans of self's immediate child declarations only (self
// nil selects the top-level scope's own children). See declarationTree for
// why immediate children alone are sufficient regardless of nesting depth.
func (t declTree) exclusionFor(self ast.Vertex) spanIndex {
return newSpanIndex(t.children[self])
}
// withoutNestedDeclarationVars returns a view of f whose VARIABLE facts drop
// anything positioned inside one of the excluded declaration spans, leaving
// its CALL facts whole.
//
// A return expression is collected without excluding nested declarations, and
// deliberately so: that is what lets a callee invoked inside a closure within
// the return contribute a dependency edge and a summary. But the taint state
// that expression is graded against belongs to the ENCLOSING body, and a
// variable belonging to a nested declaration -- a closure's own local, an
// arrow function's parameter -- was never that state's variable. Names like
// $data and $content recur constantly in real PHP, so a bare name collision
// is otherwise enough to report a flow on a file where the two never meet.
//
// Nodes without position information are retained, matching the rest of this
// package: an unplaceable fact fails open, toward keeping a dependency rather
// than silently dropping one.
//
// This is a trade, not a pure win, and the losing side is worth naming. A
// closure that is immediately invoked really does yield what it captured, so
// `return (function() use ($tainted) { return $tainted; })();` stops being
// reported. Taint the closure PRODUCES by calling a source inside itself is
// still followed, because calls are not filtered; only taint carried IN by a
// capture is lost. Every such case still records closure-capture precision
// loss, so the reduction is reported rather than silent, and it buys the
// removal of a false positive that fires whenever a nested declaration's
// parameter merely shares a name with a tainted variable outside it.
func (f *scopeFacts) withoutNestedDeclarationVars(exclude *spanIndex) *scopeFacts {
// Most functions have no nested declarations. Avoid copying three fact
// slices on every return evaluation when the exclusion index cannot
// possibly remove anything.
if exclude == nil || len(exclude.spans) == 0 {
return f
}
keep := func(n ast.Vertex) bool {
span, ok := spanOf(n)
return !ok || !exclude.contains(span)
}
out := *f
out.varNodes = make([]*ast.ExprVariable, 0, len(f.varNodes))
for _, n := range f.varNodes {
if keep(n) {
out.varNodes = append(out.varNodes, n)
}
}
out.propNodes = make([]ast.Vertex, 0, len(f.propNodes))
for _, n := range f.propNodes {
if keep(n) {
out.propNodes = append(out.propNodes, n)
}
}
out.writes = make([]ast.Vertex, 0, len(f.writes))
for _, n := range f.writes {
if keep(n) {
out.writes = append(out.writes, n)
}
}
// vars feeds the evidence strings only, never a taint decision, but it is
// rebuilt from the surviving nodes anyway so a report never names a
// variable that belongs to a nested declaration and was excluded from the
// grading that produced the report.
out.vars = make(map[string]bool, len(out.varNodes))
for _, n := range out.varNodes {
if name := varName(n.Name); name != "" {
out.vars[name] = true
}
}
return &out
}
// readVarNodes returns variables whose value is read in this subtree. The
// parser visitor also visits assignment targets; those are writes, not
// inputs to the assignment expression, and must not borrow taint from an
// earlier assignment.
func (f *scopeFacts) readVarNodes() []namedNodeSpan {
writes := make([]nodeSpan, 0, len(f.writes))
for _, n := range f.writes {
if span, ok := spanOf(n); ok {
writes = append(writes, span)
}
}
sort.Slice(writes, func(i, j int) bool { return writes[i].start < writes[j].start })
vars := make([]namedNodeSpan, 0, len(f.varNodes)+len(f.propNodes))
reads := make([]namedNodeSpan, 0, len(f.varNodes)+len(f.propNodes))
for _, n := range f.varNodes {
name := varName(n.Name)
if name == "" {
continue
}
if span, ok := spanOf(n); ok {
vars = append(vars, namedNodeSpan{nodeSpan: span, name: name, node: n})
} else {
// Parsed nodes normally always have positions. If a future parser
// omits one, retain the conservative taint dependency.
reads = append(reads, namedNodeSpan{name: name, node: n})
}
}
// Property-fetch reads are keyed to the specific property chain (see
// assignedTargetKey), not the bare base variable, so a write to one
// property cannot leak into a read of a different one. A property whose
// name is not statically known is skipped here entirely rather than
// keyed to "": the embedded base variable visited above already carries
// the conservative base-variable dependency for that case, via
// assignedTargetKey's own fallback at write time.
for _, n := range f.propNodes {
key := assignedTargetKey(n)
if key == "" {
continue
}
if span, ok := spanOf(n); ok {
vars = append(vars, namedNodeSpan{nodeSpan: span, name: key, node: n})
} else {
reads = append(reads, namedNodeSpan{name: key, node: n})
}
}
sort.Slice(vars, func(i, j int) bool { return vars[i].start < vars[j].start })
writeIndex := 0
maxWriteEnd := -1
for _, variable := range vars {
for writeIndex < len(writes) && writes[writeIndex].start <= variable.start {
if writes[writeIndex].end > maxWriteEnd {
maxWriteEnd = writes[writeIndex].end
}
writeIndex++
}
if maxWriteEnd >= variable.end {
continue
}
reads = append(reads, variable)
}
return reads
}
func spanOf(n ast.Vertex) (nodeSpan, bool) {
if n == nil || n.GetPosition() == nil {
return nodeSpan{}, false
}
pos := n.GetPosition()
if pos.StartPos < 0 || pos.EndPos < pos.StartPos {
return nodeSpan{}, false
}
return nodeSpan{start: pos.StartPos, end: pos.EndPos}, true
}
// calleeName renders a call target as a lowercase name. PHP function names
// are case-insensitive; variable and dynamic targets yield "".
//
// Multi-part names, including *ast.NameRelative (a "namespace\name" call),
// are never collapsed onto their last segment: PHP resolves an unqualified
// call in the current namespace before falling back to the global one, so
// treating "Foo\curl_exec" as global curl_exec would invent false
// positives. *ast.NameRelative is therefore left unhandled and yields "";
// missing an explicitly namespaced wrapper is the accepted trade.
func calleeName(n ast.Vertex) string {
switch v := n.(type) {
case *ast.Name:
return joinNameParts(v.Parts)
case *ast.NameFullyQualified:
return joinNameParts(v.Parts)
case *ast.Identifier:
return strings.ToLower(string(v.Value))
}
return ""
}
func joinNameParts(parts []ast.Vertex) string {
out := make([]string, 0, len(parts))
for _, p := range parts {
if np, ok := p.(*ast.NamePart); ok {
out = append(out, string(np.Value))
}
}
return strings.ToLower(strings.Join(out, "\\"))
}
// varName renders a variable's name. The parser's T_VARIABLE token includes
// the leading '$' in Identifier.Value, so it is trimmed here to yield the
// bare name. Variable variables yield "" because their identity is not
// statically known.
func varName(n ast.Vertex) string {
if id, ok := n.(*ast.Identifier); ok {
return strings.TrimPrefix(string(id.Value), "$")
}
return ""
}
package phptaint
// grade is the analyzer's internal lattice element. The public Confidence is
// its first component and keeps its meaning; basis and offset only explain
// which proof produced that confidence, so a stronger flow must carry its own
// explanation rather than an older, weaker one.
type grade struct {
conf Confidence
basis Basis
offset int
}
func directGrade(c Confidence, b Basis) grade {
return grade{conf: c, basis: b, offset: -1}
}
// rankedBases is the fixed tie priority, strongest first. It is also
// the index order of gradeSet, so the two can never disagree.
var rankedBases = [...]Basis{
BasisAlwaysRemote,
BasisLiteral,
BasisDecoded,
BasisRequest,
BasisCallArgument,
BasisUnresolved,
}
// basisRank is the position of b in rankedBases; -1 means undefined.
func basisRank(b Basis) int {
for i, r := range rankedBases {
if r == b {
return i
}
}
return -1
}
// stronger orders by confidence, then basis priority, then the lowest
// resolution offset, with -1 (direct) lowest. The order is total over valid
// grades so every join is deterministic regardless of worklist order.
func (g grade) stronger(o grade) bool {
if g.conf != o.conf {
return g.conf > o.conf
}
if rg, ro := basisRank(g.basis), basisRank(o.basis); rg != ro {
return rg < ro
}
return g.offset < o.offset
}
// directGradeOf rebuilds the ordering key from a finished Result.
func directGradeOf(r Result) grade {
return grade{conf: r.Confidence, basis: r.Basis, offset: r.ResolutionOffset}
}
// confidenceLevels is the number of Confidence values, the second index of
// a gradeSet.
const confidenceLevels = int(ConfidenceCertain) + 1
// gradeSet is the value the fixpoints store for a variable, an assignment,
// an origin and a summary: for each (basis, confidence) pair, the lowest
// resolution offset any proof with that key reached. A single grade cannot
// be stored there, because the decoder upgrade is not monotone under
// stronger: High always-remote is weaker than Certain literal, yet after
// both are decoded Certain always-remote wins. Keying by basis alone is not
// enough either: (High, 3) is weaker than (Certain, 9) on one basis, yet
// after the upgrade offset 3 wins. With confidence in the key, the upgrade
// just moves each entry to its basis's Certain key and joins it there, which
// distributes over the join. The join keeps the lowest offset per key, there
// are at most len(rankedBases) x confidenceLevels keys, so every fixpoint
// over gradeSet still terminates and has one answer. The single strongest
// grade is chosen only when a Result is emitted.
type gradeSet struct {
entries [len(rankedBases)][confidenceLevels]gradeEntry
}
// gradeEntry is one key's lowest resolution offset, stored biased by
// entryBias so the zero value means absent and the zero gradeSet is the
// empty set. A resolution offset is -1 or a position inside the source,
// which MaxSourceBytes bounds far below int32. The solver keeps one gradeSet
// per variable, assignment output and summary, so the entry stays four
// bytes rather than a padded (bool, int) pair.
type gradeEntry int32
const entryBias = 2
// entryAt encodes offset. An offset outside [-1, MaxSourceBytes] is an
// analyzer defect: it panics, which the package boundary recovers into a
// visible coverage gap, rather than storing a value that decodes to a
// different proof.
func entryAt(offset int) gradeEntry {
if offset < -1 || offset > MaxSourceBytes {
panic("phptaint: resolution offset out of range")
}
return gradeEntry(offset + entryBias)
}
func (e gradeEntry) present() bool { return e != 0 }
func (e gradeEntry) offset() int { return int(e) - entryBias }
// joinEntry keeps the lower offset for one key and reports whether it grew.
// The bias preserves order, so present entries compare directly.
func joinEntry(cur *gradeEntry, e gradeEntry) bool {
if !e.present() || (cur.present() && *cur <= e) {
return false
}
*cur = e
return true
}
// setOf is the set holding one proof. An undefined basis or confidence is
// an analyzer defect: dropping it would silently turn a flow into no flow,
// so it panics instead, and the package boundary (recovered) reports the
// file as a StatusPanic coverage gap.
func setOf(g grade) gradeSet {
r := basisRank(g.basis)
if r < 0 {
panic("phptaint: undefined basis")
}
if int(g.conf) >= confidenceLevels {
panic("phptaint: undefined confidence")
}
var s gradeSet
s.entries[r][g.conf] = entryAt(g.offset)
return s
}
func (s gradeSet) isEmpty() bool {
return s == gradeSet{}
}
// add joins o into s per key and reports whether s grew: a key appeared or
// an existing key's offset dropped.
func (s *gradeSet) add(o gradeSet) bool {
grew := false
for b := range o.entries {
for c := range o.entries[b] {
if joinEntry(&s.entries[b][c], o.entries[b][c]) {
grew = true
}
}
}
return grew
}
// decoded applies the decoder upgrade to every proof: each entry moves to
// its basis's Certain key, and each basis still explains its own
// acquisition.
func (s gradeSet) decoded() gradeSet {
var out gradeSet
for b := range s.entries {
for c := range s.entries[b] {
joinEntry(&out.entries[b][ConfidenceCertain], s.entries[b][c])
}
}
return out
}
// strongest selects the proof a Result reports, by the stronger order. The
// empty set yields the zero grade; callers emit only non-empty sets.
func (s gradeSet) strongest() grade {
var best grade
found := false
for b := range s.entries {
for c, e := range s.entries[b] {
if !e.present() {
continue
}
g := grade{conf: Confidence(c), basis: rankedBases[b], offset: e.offset()}
if !found || g.stronger(best) {
best = g
}
found = true
}
}
return best
}
package phptaint
import (
"fmt"
"github.com/VKCOM/php-parser/pkg/ast"
"github.com/VKCOM/php-parser/pkg/conf"
"github.com/VKCOM/php-parser/pkg/errors"
"github.com/VKCOM/php-parser/pkg/parser"
"github.com/VKCOM/php-parser/pkg/version"
)
// parserVersion is the highest grammar this parser implements. Hosts run
// newer PHP; constructs beyond this ceiling recover into a partial tree and
// are accounted as coverage gaps rather than analysed.
const parserVersion = "8.1"
// parseSource returns the tree plus a status. StatusAnalyzed here means only
// that parsing completed cleanly; the data-flow pass decides the final status.
func parseSource(src []byte) (ast.Vertex, Status, string) {
ver, err := version.New(parserVersion)
if err != nil {
return nil, StatusParseError, "parser version unavailable"
}
var syntaxErrs int
var firstLine int
root, err := parser.Parse(src, conf.Config{
Version: ver,
ErrorHandlerFunc: func(e *errors.Error) {
syntaxErrs++
if firstLine == 0 && e != nil && e.Pos != nil {
firstLine = e.Pos.StartLine
}
},
})
switch {
case err != nil:
// parser.Parse currently returns only configuration errors here. Keep
// this diagnostic generic so a future parser error cannot expose input.
return nil, StatusParseError, "parser failed"
case root == nil:
return nil, StatusParseError, "parser produced no tree"
case syntaxErrs > 0:
reason := fmt.Sprintf("recovered from %d syntax error(s)", syntaxErrs)
if firstLine > 0 {
reason += fmt.Sprintf(" starting at line %d", firstLine)
}
return root, StatusPartialParse, sanitizeReason(reason)
}
return root, StatusAnalyzed, ""
}
// Package phptaint reports remotely-fetched content that reaches a PHP
// code-execution sink.
//
// It exists because binding an executed variable to a fetched one requires a
// backreference, which neither YARA-X nor Go's RE2 provides.
//
// The package is pure: bytes in, report out. It touches no filesystem,
// config, store, or process global that any caller can observe or that
// carries state across calls. The exceptions are declTreeBuilds and
// summaryBodyEvals, package-level counters incremented during analysis purely
// so same-package tests can assert structural invariants. Nothing in this
// package branches on either, so neither is observable to a caller; see their
// own doc comments.
package phptaint
import (
"context"
"sort"
"github.com/VKCOM/php-parser/pkg/ast"
)
// Status is the outcome of an analysis attempt. Callers must not infer a
// clean file from an empty result slice. StatusAnalyzed and StatusNotCandidate
// are the two completed outcomes; every other status is a coverage gap and must
// be accounted for as such rather than counted as a clean file.
type Status uint8
const (
// StatusNotCandidate means the pre-filter alone proved the content
// cannot contain a flow this analyzer reports.
StatusNotCandidate Status = iota
// StatusAnalyzed means the content parsed cleanly and the data-flow pass
// ran to completion.
StatusAnalyzed
StatusOversize
// StatusParseError means no usable tree was produced.
StatusParseError
// StatusPartialParse means the parser recovered from syntax errors and
// returned an incomplete tree. Analysing it would under-report silently,
// so it is a coverage gap rather than a result.
StatusPartialParse
StatusResourceLimit
StatusCanceled
StatusPanic
// StatusTimeout means analysis was still running when its deadline
// expired and the process running it was killed. It is distinct from
// StatusCanceled, which is the caller withdrawing: a timeout means the
// analyzer did not stop on its own, which for this package is the
// expected outcome of a parser loop rather than a surprise.
StatusTimeout
// StatusWorkerFailure means analysis could not be carried out because the
// isolated process failed -- it exited unexpectedly, could not be
// started, or its reply was unusable. The content was never examined.
StatusWorkerFailure
)
// String names the status for metrics labels and operator-facing text.
func (s Status) String() string {
switch s {
case StatusNotCandidate:
return "not_candidate"
case StatusAnalyzed:
return "analyzed"
case StatusOversize:
return "oversize"
case StatusParseError:
return "parse_error"
case StatusPartialParse:
return "partial_parse"
case StatusResourceLimit:
return "resource_limit"
case StatusCanceled:
return "canceled"
case StatusPanic:
return "panic"
case StatusTimeout:
return "timeout"
case StatusWorkerFailure:
return "worker_failure"
}
return "unknown"
}
// Confidence grades how firmly the source was shown to be remote.
type Confidence uint8
const (
// ConfidenceLow means the source function also reads local files and its
// argument could not be proven either way.
ConfidenceLow Confidence = iota
// ConfidenceHigh means the argument carries a remote scheme.
ConfidenceHigh
// ConfidenceCertain means a remote source additionally passed through a
// decoder before reaching the sink.
ConfidenceCertain
)
// String names the confidence for operator-facing text.
func (c Confidence) String() string {
switch c {
case ConfidenceLow:
return "low"
case ConfidenceHigh:
return "high"
case ConfidenceCertain:
return "certain"
}
return "unknown"
}
// Basis names how a result's source was shown to be remote, or that it was
// not. It is explanation for reviewers and for the reporting decision in
// internal/checks; nothing inside this package detects on it.
type Basis string
const (
// BasisAlwaysRemote: the acquiring call can only read over the network.
BasisAlwaysRemote Basis = "always-remote"
// BasisLiteral: the path argument's exact text carries a remote scheme.
BasisLiteral Basis = "literal"
// BasisDecoded: the scheme appears only after PHP escape or builtin decoding.
BasisDecoded Basis = "decoded"
// BasisRequest: a requester can supply the beginning of the path.
BasisRequest Basis = "request"
// BasisCallArgument: a same-file call site supplied the remote argument.
BasisCallArgument Basis = "call-argument"
// BasisUnresolved: the analyzer followed every modeled construct and
// could not decide the argument's locality.
BasisUnresolved Basis = "unresolved"
)
// Valid reports whether b is one of the defined bases.
func (b Basis) Valid() bool {
return basisRank(b) >= 0
}
// MaxSourceBytes bounds the complete source an analysis accepts. Callers
// should read one byte past it to tell an exact-limit file from a truncated
// prefix.
const MaxSourceBytes = 2 << 20
// MaxReasonBytes bounds Report.Reason. Parser diagnostics can contain
// attacker-controlled text, so reports retain only sanitized, bounded context.
const MaxReasonBytes = 256
// Analyzer budgets. These bound work regardless of the parser's own
// resilience: a Go stack overflow is fatal and cannot be recovered, so
// analyzer recursion is capped explicitly.
const (
// maxCollectedNodes bounds nodes of interest recorded per scope. Total
// traversal work is separately bounded by MaxSourceBytes.
maxCollectedNodes = 800_000
maxSummarizedFuncs = 2_000
maxAnalysisDepth = 256
// MaxEvidenceResults bounds returned evidence paths; TotalResults still
// reports the full count. It is exported so the worker protocol can reject
// a clean reply that silently omits evidence.
MaxEvidenceResults = 8
// maxDeclarations bounds how many functions/methods/classes one file's
// declaration tree is built from. declarationTree is linearithmic in
// this count (not quadratic -- see its doc comment), so this exists as
// defence in depth against a future regression rather than as the
// primary complexity fix, and is set far above any real single PHP
// file's declaration count.
maxDeclarations = 20_000
)
// Result is one remote-source-to-sink flow.
type Result struct {
// Source is the acquiring call, such as "curl_exec".
Source string
// Identifiers lists every distinct variable and call name that appears
// in the sink's own expression, sorted alphabetically. This is context
// for a reviewer reading the report, not a laundering path: it includes
// names that never carried the tainted value, and carries no ordering
// information about how the value actually moved.
Identifiers []string
// Sink names the executing construct, such as "eval".
Sink string
// Confidence grades how firmly the source was shown to be remote.
Confidence Confidence
// Basis explains how the source's locality was established. It never
// changes Confidence; see the Basis type.
Basis Basis
// ResolutionOffset is the zero-based byte offset of the outermost call
// site where a parameter relation was resolved, or -1 for a direct result.
ResolutionOffset int
}
// Report is the outcome of analysing one source file.
type Report struct {
Status Status
// Results is non-empty only when Status is StatusAnalyzed.
Results []Result
// TotalResults counts every distinct source-sink endpoint pair before
// evidence truncation. It is non-zero only when Status is StatusAnalyzed.
TotalResults int
// Reason carries a stable status label plus bounded sanitized context.
// It never contains source excerpts.
Reason string
// PrecisionLoss names constructs that defeat variable identity, such as
// variable variables or extract(). Recorded, never silently ignored.
PrecisionLoss []string
// EvidenceTruncated reports that displayed evidence was shortened.
EvidenceTruncated bool
}
// CoverageGap constructs an incomplete report through the same boundary used
// by Analyze. Supervisors use it for failures that happen outside the analyzer
// so error text cannot bypass the report's sanitizing and size bounds.
func CoverageGap(status Status, reason string) Report {
return finalizeReport(Report{Status: status, Reason: reason})
}
// recovered recovers a panic at the package boundary so a parser or analyzer
// defect degrades to a coverage gap instead of taking down the caller.
func recovered(fn func() Report) (report Report) {
defer func() {
if r := recover(); r != nil {
// A panic value may contain parser tokens or source text. The status
// is actionable without reflecting that attacker-controlled value.
report = Report{Status: StatusPanic, Reason: "recovered panic during analysis"}
}
report = finalizeReport(report)
}()
return fn()
}
// finalizeReport enforces the package boundary invariants in one place. Only a
// completed analysis may carry findings; every evidence segment leaving the
// package is printable and bounded even if an internal producer misses a cap.
func finalizeReport(report Report) Report {
if report.Status == StatusAnalyzed {
report.Reason = ""
for i := range report.Results {
var cutSource, cutSink, cutIdentifiers bool
report.Results[i].Source, cutSource = sanitize(report.Results[i].Source, maxSegmentBytes)
report.Results[i].Sink, cutSink = sanitize(report.Results[i].Sink, maxSegmentBytes)
report.Results[i].Identifiers, cutIdentifiers = truncateChain(report.Results[i].Identifiers)
report.EvidenceTruncated = report.EvidenceTruncated || cutSource || cutSink || cutIdentifiers
}
for i := range report.PrecisionLoss {
report.PrecisionLoss[i] = sanitizeSegment(report.PrecisionLoss[i])
}
return report
}
report.Results = nil
report.TotalResults = 0
report.PrecisionLoss = nil
report.EvidenceTruncated = false
if report.Status == StatusNotCandidate {
report.Reason = ""
return report
}
detail := sanitizeReason(report.Reason)
report.Reason = report.Status.String()
if detail != "" {
report.Reason += ": " + detail
}
report.Reason = sanitizeReason(report.Reason)
return report
}
// Analyze owns the size check, pre-filter, parse, and data-flow pass. It
// recovers a panic at the package boundary via the recovered helper.
func Analyze(ctx context.Context, src []byte) Report {
return recovered(func() Report { return analyze(ctx, src) })
}
func analyze(ctx context.Context, src []byte) Report {
if len(src) > MaxSourceBytes {
return Report{Status: StatusOversize, Reason: "source exceeds maximum analyzed size"}
}
if err := ctx.Err(); err != nil {
return Report{Status: StatusCanceled, Reason: err.Error()}
}
if !IsCandidate(src) {
return Report{Status: StatusNotCandidate}
}
root, status, reason := parseSource(src)
if status != StatusAnalyzed {
return Report{Status: status, Reason: reason}
}
if err := ctx.Err(); err != nil {
return Report{Status: StatusCanceled, Reason: err.Error()}
}
// all: whole-file inventory, used for the declaration list and the
// budget. top: the file's top-level statements only, collected below via
// collectTopLevel. These MUST stay separate. A single flat taint map over
// the whole file lets a function-local variable taint an unrelated
// top-level variable that merely shares its name, which fires on clean
// code -- names like $data and $content are ubiquitous in real PHP.
// Function and method bodies get their own state below.
all := collectScope(root)
if all.budgetExceeded {
return Report{Status: StatusResourceLimit, Reason: "collection budget exceeded"}
}
if err := ctx.Err(); err != nil {
return Report{Status: StatusCanceled, Reason: err.Error()}
}
// tree indexes every declaration in the file once (see declarationTree
// for why this must happen exactly once, not once per declaration), so
// each scope below can derive its own exclusion index by lookup.
// functionSummaries calls f.declarationTree() again on this same all;
// that hits scopeFacts' own cache rather than rebuilding. Check the
// defence-in-depth cap before any further work, including
// functionSummaries, so a file that trips it does no summarization work
// either. This is a coverage gap, not a silent skip: StatusResourceLimit
// means the file was not examined, matching the same contract already
// used for a collection-budget overrun above.
tree := all.declarationTree()
if tree.count > maxDeclarations {
return Report{Status: StatusResourceLimit, Reason: "too many declarations to analyze"}
}
if err := ctx.Err(); err != nil {
return Report{Status: StatusCanceled, Reason: err.Error()}
}
callIndex := newResolvedCallIndex(all)
summaries, loss, err := functionSummaries(ctx, all)
if err != nil {
if err == errSummaryLimit {
return Report{Status: StatusResourceLimit, Reason: err.Error()}
}
return Report{Status: StatusCanceled, Reason: err.Error()}
}
// Each scope below excludes every declaration nested inside it except,
// when the scope IS a declaration's own body, that declaration's own
// span -- otherwise a function would exclude its own statements from
// itself. exclusionFor derives this from tree by lookup rather than by
// rescanning the file's declarations for every scope.
topExclude := tree.exclusionFor(nil)
top := callIndex.apply(collectTopLevel(root, &topExclude))
// factsByScope keeps the facts each loop below already builds, keyed by the
// declaration they belong to (nil for the top level). Nothing here
// re-collects: this only retains what would otherwise be discarded, so a
// closure can later ask what its enclosing scope held.
factsByScope := map[ast.Vertex]*scopeFacts{nil: top}
flows, err := findFlows(ctx, top, summaries, callIndex, &topExclude)
if err != nil {
return Report{Status: StatusCanceled, Reason: err.Error()}
}
// droppedTaint tracks whether ANY scope assigns tainted content to a
// target this package cannot key (see hasUnresolvableTaintedTarget):
// method-call-chain, static-property, list()-destructuring, and
// variable-variable targets all silently drop the flow rather than
// risk a false positive by guessing. Checked per scope (top level, each
// function, each method) using the SAME scope-isolated facts already
// collected here for findFlows, so "tainted" means tainted within that
// specific scope's own taint state -- never a same-named variable
// leaking taint in from an unrelated scope. Direct call origins come from
// all so namespace-level `use function` aliases stay resolved inside
// separately collected declaration bodies; source positions still limit
// them to the exact RHS being checked.
droppedTaint, err := hasUnresolvableTaintedTarget(ctx, top, summaries)
if err != nil {
return Report{Status: StatusCanceled, Reason: err.Error()}
}
// Sinks inside function bodies count too, each against its own state.
// collectOwnStmts (not collectAll) skips any function/class/method/
// closure/arrow-function nested inside this body, for the same reason
// collectTopLevel skips declarations at the file level: a nested
// declaration's locals must not taint an identically-named local in the
// enclosing body.
analyzeScope := func(scope ast.Vertex, stmts []ast.Vertex) error {
if err := ctx.Err(); err != nil {
return err
}
exclude := tree.exclusionFor(scope)
body := callIndex.apply(collectOwnStmts(stmts, &exclude))
factsByScope[scope] = body
bodyFlows, err := findFlows(ctx, body, summaries, callIndex, &exclude)
if err != nil {
return err
}
flows = append(flows, bodyFlows...)
if !droppedTaint {
droppedTaint, err = hasUnresolvableTaintedTarget(ctx, body, summaries)
}
return err
}
for _, fn := range all.funcs {
if err := analyzeScope(fn, fn.Stmts); err != nil {
return Report{Status: StatusCanceled, Reason: err.Error()}
}
}
for _, m := range all.methods {
// StmtClassMethod carries ONE Stmt vertex, not a Stmts slice.
if err := analyzeScope(m, methodStmts(m.Stmt)); err != nil {
return Report{Status: StatusCanceled, Reason: err.Error()}
}
}
// Closures are the most common nested scope in real PHP (WordPress hooks,
// shutdown/callback registration, ...), so they get the exact same
// treatment as a named function's body: excluded from the enclosing
// scope above via declarationTree, analysed here in their own. Recording
// the span without this loop (or the reverse) would either let a closure
// local borrow an unrelated outer variable's taint or stop examining the
// closure's own sinks entirely -- see the closures field doc in facts.go.
for _, cl := range all.closures {
if err := analyzeScope(cl, cl.Stmts); err != nil {
return Report{Status: StatusCanceled, Reason: err.Error()}
}
}
// An arrow function's body is a single implicit-return expression rather
// than a statement list (see arrowFunctionBody), but it is the same kind
// of nested scope as a closure and needs the same two-sided treatment.
for _, af := range all.arrowFuncs {
if err := analyzeScope(af, arrowFunctionBody(af)); err != nil {
return Report{Status: StatusCanceled, Reason: err.Error()}
}
}
if err := ctx.Err(); err != nil {
return Report{Status: StatusCanceled, Reason: err.Error()}
}
capturedTaint, captureErr := hasDroppedCapture(ctx, all, tree, factsByScope, summaries)
if captureErr != nil {
return Report{Status: StatusCanceled, Reason: captureErr.Error()}
}
if droppedTaint {
loss = append(loss, "unresolvable-assign-target")
}
if capturedTaint {
loss = append(loss, "closure-capture")
}
if droppedTaint || capturedTaint {
// loss is already sorted (functionSummaries sorts it); re-sort once
// after appending rather than inserting in place, so this stays a
// single obviously-correct call regardless of where the appended
// markers fall alphabetically among the other markers.
sort.Strings(loss)
}
flows = dedupeAndSort(flows)
total := len(flows)
truncated := total > MaxEvidenceResults
flows = retainStrongestEvidence(flows)
results := make([]Result, len(flows))
for i, flow := range flows {
results[i] = flow.Result
truncated = truncated || flow.evidenceTruncated
}
return Report{
Status: StatusAnalyzed,
Results: results,
TotalResults: total,
PrecisionLoss: loss,
EvidenceTruncated: truncated,
}
}
package phptaint
import "bytes"
// sinkKeywords and sourceKeywords drive admission only. They are language
// constructs and library function names, never variable or file names.
var (
sinkKeywords = [][]byte{
[]byte("eval"), []byte("include"), []byte("require"),
[]byte("assert"), []byte("create_function"),
}
sourceKeywords = [][]byte{
[]byte("curl_exec"), []byte("curl_multi_getcontent"),
[]byte("file_get_contents"), []byte("fopen"), []byte("fread"),
[]byte("fgets"), []byte("stream_get_contents"), []byte("readfile"),
[]byte("wp_remote_retrieve_body"), []byte("wp_remote_get"),
[]byte("fsockopen"),
}
phpOpenTags = [][]byte{[]byte("<?php"), []byte("<?="), []byte("<?")}
)
// foldChunkBytes bounds the lowercased working copy containsAnyFold folds
// src into. Matching case-insensitively requires comparing against a
// lowercased view of the bytes somewhere; the previous implementation
// lowercased the entire file up front, which meant a full-size allocation
// and copy for every file scanned, paid even on the reject path -- the hot
// path for a daemon walking millions of files, most of which are not PHP at
// all. Folding a small fixed-size window at a time bounds that cost to a
// constant instead of MaxSourceBytes. 64KiB is also small enough that the
// compiler keeps the working buffer on the stack rather than the heap, so
// this costs zero allocations, not just a smaller one.
const foldChunkBytes = 64 << 10
// maxNeedleBytes is the longest literal in sinkKeywords, sourceKeywords, or
// phpOpenTags, computed rather than hand-maintained so a future longer
// keyword cannot silently undersize the window overlap below and reopen a
// false negative. Consecutive folded windows overlap by this many bytes
// minus one so a match straddling a window boundary is never missed.
var maxNeedleBytes = longestNeedle(sinkKeywords, sourceKeywords, phpOpenTags)
func longestNeedle(groups ...[][]byte) int {
max := 0
for _, group := range groups {
for _, n := range group {
if len(n) > max {
max = len(n)
}
}
}
return max
}
// MayBePHPSource reports whether content could be PHP at all, judged only by
// the presence of an open tag. It exists for callers that must decide
// something about a file they cannot analyze in full -- an oversize file, say
// -- and would otherwise have to guess.
//
// It is deliberately weaker than IsCandidate: no sink or source keyword is
// required, because a caller holding only a prefix cannot conclude anything
// from their absence. Judging by content rather than by name or extension is
// the point; a scanner that decided what to examine from a path would be
// telling an attacker where to hide.
func MayBePHPSource(prefix []byte) bool {
// PHP emits arbitrary bytes before its opening tag as inline HTML.
// A NUL or binary header cannot rule out executable PHP later on.
return containsAnyFold(prefix, phpOpenTags)
}
// IsCandidate is the cheap byte scan Analyze runs before parsing; content
// it rejects is StatusNotCandidate. No parser runs, so a caller that isolates
// Analyze in another process can apply it in its own process first and skip
// the round trip. It is intentionally over-inclusive; the AST pass decides
// whether a real flow exists. PHP
// function names and language constructs are case-insensitive (EVAL, Eval
// and eval all execute the same construct), so admission matches
// case-insensitively too, on pain of a false negative admitting less than
// the AST rules would. The open-tag check runs first so a file that never
// looks like PHP never pays for the sink/source keyword scans either.
func IsCandidate(src []byte) bool {
if !MayBePHPSource(src) {
return false
}
return containsAnyFold(src, sinkKeywords) && containsAnyFold(src, sourceKeywords)
}
// containsAnyFold reports whether any needle occurs in hay under ASCII case
// folding. It folds hay one bounded window at a time into a stack-local
// buffer -- never allocating a lowercased copy of the whole input -- and
// runs the standard library's optimized bytes.Contains against each folded
// window, so search speed stays close to a plain bytes.Contains scan.
// Windows overlap by maxNeedleBytes-1 bytes so a needle split across a
// window boundary still lands whole inside the next window.
func containsAnyFold(hay []byte, needles [][]byte) bool {
if len(hay) == 0 {
return false
}
overlap := maxNeedleBytes - 1
step := foldChunkBytes - overlap
var buf [foldChunkBytes]byte
for start := 0; start < len(hay); start += step {
end := start + foldChunkBytes
if end > len(hay) {
end = len(hay)
}
window := hay[start:end]
folded := buf[:len(window)]
for i, b := range window {
folded[i] = lowerASCII(b)
}
for _, n := range needles {
if bytes.Contains(folded, n) {
return true
}
}
if end == len(hay) {
break
}
}
return false
}
func lowerASCII(b byte) byte {
if 'A' <= b && b <= 'Z' {
return b + ('a' - 'A')
}
return b
}
package phptaint
import (
"strings"
"github.com/VKCOM/php-parser/pkg/ast"
"github.com/pidginhost/csm/internal/cms"
)
// buildLocalPathConstants collects every bootstrap path constant declared by
// the given CMS descriptors into a lookup set.
func buildLocalPathConstants(descriptors []cms.Descriptor) map[string]bool {
out := make(map[string]bool)
for _, d := range descriptors {
for _, c := range d.PathConstants {
out[c] = true
}
}
return out
}
// alwaysRemote functions can only acquire content over the network.
var alwaysRemote = map[string]bool{
"curl_exec": true,
"curl_multi_getcontent": true,
"wp_remote_get": true,
"wp_remote_retrieve_body": true,
"fsockopen": true,
}
// dualUse functions return either local or remote content, so their argument
// decides. fread, fgets and stream_get_contents are deliberately excluded:
// they take a stream resource, never a path, so their argument carries no
// locality signal. readfile is excluded because it returns a byte count, not
// the content it writes, so no fetched bytes can flow from its return value.
// For stream readers, the acquiring fopen or fsockopen call is already the
// source and its tainted handle carries through the expression.
var dualUse = map[string]bool{
"file_get_contents": true,
"fopen": true,
}
// remoteSchemes mark an argument as remote. Only php://input is intrinsically
// request-controlled; php://memory, php://temp and local php://filter resources
// are not remote sources. A filter around a remote resource still contains its
// nested remote scheme and is classified accordingly.
var remoteSchemes = []string{"http://", "https://", "ftp://", "ftps://", "php://input", "data://"}
// These are the specific PHP and CMS constructs whose result is known to be a
// local path. Arbitrary constants and calls remain undecidable: their runtime
// value can be a remote URL.
//
// Every supported CMS defines filesystem path constants during bootstrap, and
// all of them compile templates or caches by writing generated PHP under one
// of those paths and including it afterwards. Without the constant, that read
// is undecidable, the generated file looks remotely acquired, and the include
// reports as remote execution on a stock installation. The constants come
// from the supported CMS table so a CMS declared there cannot be missing here.
var (
localPathConstants = buildLocalPathConstants(cms.All())
localPathResults = map[string]bool{
"get_template_directory": true,
"realpath": true,
"sys_get_temp_dir": true,
}
pathTransforms = map[string]bool{
"dirname": true,
"plugin_dir_path": true,
}
)
type locality uint8
const (
localityUnknown locality = iota
localityLocal
localityRemote
)
// sourceGrade reports whether a call acquires remote content, how firmly
// that was shown, and on what basis.
func sourceGrade(call *ast.ExprFunctionCall) (grade, bool) {
name := calleeName(call.Function)
if alwaysRemote[name] {
return directGrade(ConfidenceHigh, BasisAlwaysRemote), true
}
if !dualUse[name] {
return grade{}, false
}
unresolved := directGrade(ConfidenceLow, BasisUnresolved)
if len(call.Args) == 0 {
return unresolved, true
}
arg, ok := call.Args[0].(*ast.Argument)
if !ok {
return unresolved, true
}
switch argLocality(arg.Expr) {
case localityLocal:
return grade{}, false
case localityRemote:
return directGrade(ConfidenceHigh, BasisLiteral), true
}
return unresolved, true
}
// argLocality classifies an argument by shape. Literal text and
// concatenations of literals are decidable; anything else is unknown.
func argLocality(arg ast.Vertex) locality {
text, decidable := staticText(arg)
lower := strings.ToLower(text)
for _, scheme := range remoteSchemes {
if strings.Contains(lower, scheme) {
return localityRemote
}
}
if !decidable {
return localityUnknown
}
if strings.HasPrefix(text, "/") || strings.HasPrefix(text, "./") ||
strings.HasPrefix(text, "../") {
return localityLocal
}
if text != "" {
return localityLocal
}
return localityUnknown
}
// staticText folds an expression to text when every part is statically known.
// Known fragments survive an undecidable concatenation so a literal scheme can
// still prove remoteness; arbitrary constants and calls stay undecidable.
//
// Recursion is depth-bounded because this is the one place the package
// recurses over attacker-controlled structure, and a Go stack overflow is
// fatal: recover() cannot catch it. Exceeding the bound yields "undecidable",
// which degrades to a reduced-confidence source rather than a wrong answer.
func staticText(n ast.Vertex) (string, bool) { return staticTextAt(n, 0) }
func staticTextAt(n ast.Vertex, depth int) (string, bool) {
if depth >= maxAnalysisDepth {
return "", false
}
switch v := n.(type) {
case *ast.ScalarString:
raw := string(v.Value)
if len(raw) < 2 || raw[0] != raw[len(raw)-1] || (raw[0] != '\'' && raw[0] != '"') {
return raw, false
}
text := raw[1 : len(raw)-1]
// Double-quoted PHP strings interpret hex, octal and other escapes.
// Keeping visible fragments lets a literal scheme still prove remote,
// but an escape makes the complete runtime text undecidable.
if raw[0] == '"' && strings.ContainsRune(text, '\\') {
return text, false
}
return text, true
case *ast.ExprBinaryConcat:
left, okL := staticTextAt(v.Left, depth+1)
right, okR := staticTextAt(v.Right, depth+1)
// Preserve known literal fragments even when another fragment is
// dynamic. Separate undecidable pieces so two fragments cannot invent
// a scheme across an unknown runtime value.
if !okL || !okR {
return left + "\x00" + right, false
}
return left + right, true
case *ast.ExprBrackets:
return staticTextAt(v.Expr, depth+1)
case *ast.ExprConstFetch:
name := calleeName(v.Const)
if localPathConstants[name] {
return name, true
}
return "", false
case *ast.ExprFunctionCall:
name := calleeName(v.Function)
if localPathResults[name] {
return name, true
}
if !pathTransforms[name] || len(v.Args) == 0 {
return "", false
}
arg, ok := v.Args[0].(*ast.Argument)
if !ok {
return "", false
}
return staticTextAt(arg.Expr, depth+1)
case *ast.ScalarMagicConstant:
return strings.ToLower(string(v.Value)), true
}
return "", false
}
package phptaint
import (
"context"
"errors"
"sort"
"sync/atomic"
"github.com/VKCOM/php-parser/pkg/ast"
)
// precisionLossMarkers name calls that defeat static variable identity.
// extract() and compact() move values between named variables and an array
// at runtime; call_user_func(_array) dispatches through a value rather than
// a lexical name. Their presence is recorded so a caller knows coverage was
// reduced, rather than the loss passing silently.
var precisionLossMarkers = map[string]string{
"extract": "extract",
"compact": "compact",
"call_user_func": "dynamic-call",
"call_user_func_array": "dynamic-call",
}
var errSummaryLimit = errors.New("too many function summaries to analyze")
type bodyKind uint8
const (
bodyFunction bodyKind = iota
bodyMethod
)
// summaryKey names one entry in a summaryTables namespace: which of the two
// tables it lives in, plus the bare name within that table.
type summaryKey struct {
kind bodyKind
name string
}
// callSiteSummaryKey reports the summaryKey a call site would resolve
// against, mirroring the node-type switch summaryTables.lookup uses. A call
// shape outside the three kinds facts.go records resolves to nothing, same
// as lookup itself.
func callSiteSummaryKey(call callSite) (summaryKey, bool) {
switch call.node.(type) {
case *ast.ExprFunctionCall:
return summaryKey{kind: bodyFunction, name: call.name}, true
case *ast.ExprMethodCall, *ast.ExprNullsafeMethodCall, *ast.ExprStaticCall:
return summaryKey{kind: bodyMethod, name: call.name}, true
}
return summaryKey{}, false
}
// summaryBodyEvals counts how many times the interprocedural worklist below
// actually re-evaluated a body (taintedLocals plus its return expressions),
// as opposed to a body sitting untouched in the queue. It exists so a
// same-package white-box test can observe that this count grows with the
// call graph's edge count, not with (declaration count) x (declaration
// count), which is the structural invariant a dependency worklist is
// supposed to buy over a round-robin sweep. Nothing in this package branches
// on its value, so this is test-only observation, not shared process state.
var summaryBodyEvals atomic.Int64
// funcBody pairs a summarizable function or method with its collected facts
// and which summary namespace its name belongs to. Facts are collected once
// up front: they do not change across fixpoint rounds, only the summary
// tables consulted while interpreting them do.
type funcBody struct {
name string
kind bodyKind
facts *scopeFacts
// calls is the whole-file resolved-call view, kept so each evaluation can
// re-collect the return expressions with namespace-level aliases intact.
calls resolvedCallIndex
// exclude holds the spans of the declarations nested inside this body,
// the same index its own facts were collected with. A return expression
// is collected WITHOUT it, so its calls still reach nested declarations,
// and then its variables are filtered THROUGH it, so a nested
// declaration's locals are never graded against this body's taint state.
exclude spanIndex
// returnKeys names the summaries reachable from this body's return
// expressions. Only the KEYS are kept, never the collected facts: a
// return expression is collected without excluding nested declarations,
// so for a body whose return lexically contains further declarations
// those facts cover the whole remaining subtree, and retaining one per
// body across the fixpoint is quadratic in nesting depth. Attacker-
// written input reaches a gigabyte of live heap that way for a few
// hundred KB of source. The keys are a few bytes each and are all the
// dependency edges need.
returnKeys []summaryKey
}
// dependencyKeys names every summary a single evaluation of this body can
// read: those reachable from its own statements, consulted while computing
// its local taint state, plus those reachable from each return expression,
// consulted while grading what it returns. The worklist's correctness rests
// on this being a SUPERSET of what an evaluation actually reads. If a body
// can read a summary it has no edge on, a later rise in that summary never
// wakes it, and the fixpoint's answer starts depending on the order bodies
// were queued in -- which is the order they were declared in, which an
// attacker writing the file chooses.
func (b funcBody) dependencyKeys() []summaryKey {
keys := make([]summaryKey, 0, len(b.facts.callSites)+len(b.returnKeys))
for _, call := range b.facts.callSites {
if key, ok := callSiteSummaryKey(call); ok {
keys = append(keys, key)
}
}
return append(keys, b.returnKeys...)
}
// functionSummaries reports which user-defined functions and methods return
// remotely-sourced data, computed to a fixpoint over the call graph, plus any
// precision-loss markers observed while collecting their bodies.
//
// Functions and methods are kept in separate namespaces (summaryTables): PHP
// allows a function and a method, or methods on unrelated classes, to share
// a bare name, and a single shared table would let a tainted method
// anywhere in the file poison every same-named plain function call site. A
// method name declared by more than one class is dropped from the method
// table entirely rather than resolved to any one class's definition - a
// deliberate false-negative trade recorded as "ambiguous-method" precision
// loss, because this analyzer's zero-false-positive bar makes guessing
// wrong more expensive than a visible gap.
//
// This is what lets the analyzer see the motivating shape: a fetch in one
// function and a sink in another, joined only by a call. Without a summary
// for the fetching function, that flow is invisible to a single-scope pass.
func functionSummaries(ctx context.Context, f *scopeFacts) (summaryTables, []string, error) {
bodies, loss, err := summaryBodies(ctx, f)
if err != nil {
return summaryTables{}, nil, err
}
tables, err := solveSummaries(ctx, bodies)
if err != nil {
return summaryTables{}, nil, err
}
names := make([]string, 0, len(loss))
for name := range loss {
names = append(names, name)
}
sort.Strings(names)
if err := ctx.Err(); err != nil {
return summaryTables{}, nil, err
}
return tables, names, nil
}
// summaryBodies collects every summarizable function and method body, with
// the precision loss observed while collecting them. Separated from the
// fixpoint below so the two can be exercised apart: a test can build the
// bodies for a file and then run its own reference fixpoint over them, which
// is what pins the worklist to the answer an exhaustive sweep would give.
func summaryBodies(ctx context.Context, f *scopeFacts) ([]funcBody, map[string]bool, error) {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
loss := map[string]bool{}
// The enclosing facts already cover every declaration body. Record loss
// from them before omitting ambiguous methods, because that filter can
// exclude the only body containing a precision-loss construct.
recordPrecisionLoss(f, loss)
// A single class cannot legally declare the same method name twice, so
// any name appearing more than once in this flat, whole-file list was
// declared by more than one class - ambiguous, without needing to track
// which class owns which method. Ambiguous methods do not consume the
// summary-body budget because they are deliberately omitted from the
// summary table regardless of that budget.
methodCounts := make(map[string]int, len(f.methods))
for _, m := range f.methods {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
methodCounts[calleeName(m.Name)]++
}
summarizable := len(f.funcs)
for _, count := range methodCounts {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
if count > 1 {
loss["ambiguous-method"] = true
continue
}
summarizable++
}
if summarizable > maxSummarizedFuncs {
return nil, nil, errSummaryLimit
}
// tree indexes every declaration in f (see declarationTree), so a
// function or method nested inside another (however deeply, however
// many if/while/switch/try/foreach wrappers it sits behind) is excluded
// from its enclosing body's own facts by lookup rather than by
// rescanning the file's declarations for every body, and so cannot
// pollute that enclosing declaration's own interprocedural summary. In
// production f is the same *scopeFacts analyze already indexed, so this
// hits declarationTree's own cache rather than rebuilding, and is
// already known to be within maxDeclarations here (analyze's cap check
// runs first).
tree := f.declarationTree()
callIndex := newResolvedCallIndex(f)
bodies := make([]funcBody, 0, summarizable)
for _, fn := range f.funcs {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
exclude := tree.exclusionFor(fn)
facts := callIndex.apply(collectOwnStmts(fn.Stmts, &exclude))
returnKeys, err := returnSummaryKeys(ctx, facts, callIndex)
if err != nil {
return nil, nil, err
}
bodies = append(bodies, funcBody{
name: calleeName(fn.Name), kind: bodyFunction, facts: facts,
calls: callIndex, exclude: exclude, returnKeys: returnKeys,
})
}
for _, m := range f.methods {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
name := calleeName(m.Name)
if methodCounts[name] > 1 {
continue
}
// StmtClassMethod carries ONE Stmt vertex (normally a StmtStmtList),
// unlike StmtFunction which carries a Stmts slice.
exclude := tree.exclusionFor(m)
facts := callIndex.apply(collectOwnStmts(methodStmts(m.Stmt), &exclude))
returnKeys, err := returnSummaryKeys(ctx, facts, callIndex)
if err != nil {
return nil, nil, err
}
bodies = append(bodies, funcBody{
name: name, kind: bodyMethod, facts: facts,
calls: callIndex, exclude: exclude, returnKeys: returnKeys,
})
}
// Recheck every included body using its independently collected facts.
// The enclosing collection has one aggregate node budget, so it can stop
// recording inside a declaration that was already discovered; the fresh
// per-body budget can still preserve that declaration's loss markers.
for _, b := range bodies {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
recordPrecisionLoss(b.facts, loss)
}
return bodies, loss, nil
}
// solveSummaries runs the interprocedural fixpoint over prebuilt bodies.
func solveSummaries(ctx context.Context, bodies []funcBody) (summaryTables, error) {
// Summaries only ever move from absent to present, or grow pointwise as
// a gradeSet, so this is a monotone fixpoint over a finite lattice (one
// entry per (basis, confidence) key, each holding the lowest offset drawn
// from this file's call sites) whose result does not depend on the order
// bodies are (re)evaluated in - only on eventually evaluating every body
// whose inputs changed since it was last evaluated. A body's inputs are
// exactly the summaries of the functions and methods its own facts call,
// so a worklist keyed on that call graph re-evaluates a body only when a
// callee it actually references just changed, instead of re-sweeping
// every body on every pass regardless of whether anything it depends on
// moved. Which summaries a body depends on is exactly what
// dependencyKeys reports, and that must stay a superset of what an
// evaluation reads, or the order-independence claimed above is simply
// false: edges drawn from a body's own statements alone miss a callee
// invoked from inside a closure within a return, because a body's facts
// exclude nested declarations while the collection used to grade its
// return expression does not.
//
// This is the same dependency-worklist shape solveAssignments
// already runs intraprocedurally in taint.go, applied one level up: a
// body here plays the role an assignment plays there, and a produced
// summary name plays the role a variable plays there. Termination
// follows from the lattice being finite - at most one entry per body
// name, each raised a bounded number of times - rather than from any
// iteration count, so it needs no cap sized to the input the way a
// round-robin sweep would.
produced := make(map[summaryKey]bool, len(bodies))
for _, b := range bodies {
if b.name == "" {
continue
}
produced[summaryKey{kind: b.kind, name: b.name}] = true
}
// dependents maps a produced key to the bodies whose own facts call it,
// i.e. the bodies to wake when that key's grade rises. Built once
// from each body's already-collected call sites, so this costs one pass
// over the call sites this file already gathered rather than a rescan
// per round.
dependents := make(map[summaryKey][]int)
queue := make([]int, 0, len(bodies))
queued := make([]bool, len(bodies))
for i, b := range bodies {
if err := ctx.Err(); err != nil {
return summaryTables{}, err
}
if b.name == "" {
continue
}
for _, key := range b.dependencyKeys() {
if !produced[key] {
continue
}
dependents[key] = append(dependents[key], i)
}
queue = append(queue, i)
queued[i] = true
}
enqueue := func(indices []int) {
for _, i := range indices {
if !queued[i] {
queued[i] = true
queue = append(queue, i)
}
}
}
tables := summaryTables{funcs: map[string]gradeSet{}, methods: map[string]gradeSet{}}
for head := 0; head < len(queue); head++ {
if err := ctx.Err(); err != nil {
return summaryTables{}, err
}
i := queue[head]
queued[i] = false
b := bodies[i]
best, found, err := evalBodySummary(ctx, b, tables)
if err != nil {
return summaryTables{}, err
}
if !found {
continue
}
target := tables.funcs
if b.kind == bodyMethod {
target = tables.methods
}
if cur, ok := target[b.name]; ok {
if !cur.add(best) {
continue
}
best = cur
}
target[b.name] = best
enqueue(dependents[summaryKey{kind: b.kind, name: b.name}])
}
return tables, nil
}
// evalBodySummary grades what one body returns against the summaries known so
// far: the joined proofs of every tainted return, plus whether anything
// tainted is returned at all. It reads
// only the summaries dependencyKeys reports, which is what lets the worklist
// wake exactly the bodies an update can affect.
func evalBodySummary(ctx context.Context, b funcBody, tables summaryTables) (gradeSet, bool, error) {
summaryBodyEvals.Add(1)
st := taintedLocals(b.facts, tables)
var best gradeSet
found := false
for _, ret := range b.facts.returns {
if err := ctx.Err(); err != nil {
return gradeSet{}, false, err
}
if ret.Expr == nil {
continue
}
retFacts := b.calls.apply(collectScope(ret.Expr)).withoutNestedDeclarationVars(&b.exclude)
c, tainted := exprTaintFacts(retFacts, st, tables)
if !tainted {
continue
}
best.add(c)
found = true
}
return best, found, nil
}
// returnSummaryKeys names every summary reachable from a body's return
// expressions. calls is the whole-file resolved-call view, not an index
// rebuilt from f: f excludes nested declarations, while a return expression
// can read through an invoked closure or arrow function inside one. The
// whole-file view is what preserves namespace-level aliases for those nested
// calls.
//
// Each collection is transient. Only the keys survive, because the collected
// facts of a return expression cover every declaration nested inside it, and
// holding one per body for the life of the fixpoint costs memory quadratic in
// nesting depth on input an attacker writes.
func returnSummaryKeys(
ctx context.Context, f *scopeFacts, calls resolvedCallIndex,
) ([]summaryKey, error) {
var keys []summaryKey
for _, ret := range f.returns {
if err := ctx.Err(); err != nil {
return nil, err
}
if ret.Expr == nil {
continue
}
for _, call := range calls.apply(collectScope(ret.Expr)).callSites {
if key, ok := callSiteSummaryKey(call); ok {
keys = append(keys, key)
}
}
}
return keys, nil
}
// recordPrecisionLoss folds one body's precision-loss facts into the running
// set. Variable-variables and dynamic calls are already flagged during
// collection (facts.go); this adds the calls whose names are known but whose
// effect on variable identity is not visible to the collector, such as
// extract() writing into caller-invisible names.
func recordPrecisionLoss(f *scopeFacts, loss map[string]bool) {
for marker := range f.precisionLoss {
loss[marker] = true
}
for name := range f.calls {
if marker, ok := precisionLossMarkers[name]; ok {
loss[marker] = true
}
}
}
package phptaint
import (
"context"
"sort"
"strings"
"github.com/VKCOM/php-parser/pkg/ast"
)
// decoders preserve taint and raise confidence: fetching plaintext and
// executing it has a small benign population, but fetching, decoding, then
// executing has effectively none.
var decoders = map[string]bool{
"base64_decode": true, "gzinflate": true, "gzuncompress": true,
"gzdecode": true, "str_rot13": true, "hex2bin": true,
"convert_uudecode": true, "unserialize": true, "pack": true,
}
// There is no passthrough allowlist: taint propagation is structural, not
// name-based. exprTaint marks any expression tainted if it references a
// tainted variable anywhere in its subtree, so trim($a), sprintf($a), and an
// unlisted some_helper($a) are all already covered without naming a single
// one of them. A name list here could only ever be narrower than that rule.
// taintState maps a variable name to every proof, one per (basis,
// confidence) key, with which it carries remote content. See gradeSet for
// why it is not a single grade.
type taintState map[string]gradeSet
// summaryTables holds interprocedural summaries in two namespaces so a
// function and a method that happen to share a name can never collide: PHP
// resolves f(), $obj->f(), and Class::f() through distinct call syntax, and
// callSite.node retains which syntax was used, so a lookup can always pick
// the namespace the call site itself selects. A single shared map would let
// a tainted method anywhere in the file poison every same-named plain
// function call site, which is false-positive-only but unacceptable given
// this analyzer's zero-false-positive bar against real WordPress/plugin code.
type summaryTables struct {
funcs map[string]gradeSet
methods map[string]gradeSet
}
// lookup resolves a call site's summary in the namespace its call syntax
// selects. A node shape outside the three call kinds facts.go records
// (should not occur) resolves to nothing rather than guessing a namespace.
func (s summaryTables) lookup(call callSite) (gradeSet, bool) {
switch call.node.(type) {
case *ast.ExprFunctionCall:
c, ok := s.funcs[call.name]
return c, ok
case *ast.ExprMethodCall, *ast.ExprNullsafeMethodCall, *ast.ExprStaticCall:
c, ok := s.methods[call.name]
return c, ok
}
return gradeSet{}, false
}
// raise joins g into name's proofs and reports whether they grew.
func (s taintState) raise(name string, g gradeSet) bool {
if name == "" || g.isEmpty() {
return false
}
cur, ok := s[name]
if !cur.add(g) && ok {
return false
}
s[name] = cur
return true
}
type taintAssignment struct {
target string
node ast.Vertex
rhs nodeSpan
origins []taintOrigin
}
// taintOrigin is one input of an assignment: a variable read, another
// assignment's output, or a fixed value from a source or summarized call.
// Only fixed origins carry proofs of their own, so those live out of line in
// compiledAssignments.fixed and the origin holds an index: an attacker picks
// how many origins a file has, and a gradeSet in every one of them tripled
// what analysis allocated.
type taintOrigin struct {
variable string
assignment int
// fixed is the 1-based index of this origin's value in
// compiledAssignments.fixed; 0 means the origin is not fixed.
fixed int32
decoded bool
}
// compiledAssignments is one scope's solver input. fixed holds each distinct
// fixed origin value once.
type compiledAssignments struct {
assignments []taintAssignment
fixed []gradeSet
}
// fixedValues interns fixed origin values for one compileAssignments call.
type fixedValues struct {
values []gradeSet
index map[gradeSet]int32
}
// ref returns the 1-based reference to set, adding it on first use.
func (v *fixedValues) ref(set gradeSet) int32 {
if i, ok := v.index[set]; ok {
return i
}
if v.index == nil {
v.index = make(map[gradeSet]int32)
}
v.values = append(v.values, set)
i := int32(len(v.values)) // #nosec G115 -- one value per collected call, and maxCollectedNodes bounds those far below int32
v.index[set] = i
return i
}
type positionedOrigin struct {
nodeSpan
origin taintOrigin
self int
}
type assignmentInterval struct {
nodeSpan
assignment int
}
// taintedLocals computes the tainted variables of one scope to a fixpoint. It
// is flow-insensitive: assignment order within the scope is not modelled. A
// dependency worklist reaches arbitrarily long local chains without borrowing
// the interprocedural summary-round limit or silently returning a partial state.
func taintedLocals(f *scopeFacts, summaries summaryTables) taintState {
compiled, ok := compileAssignments(f, summaries)
if !ok {
return taintedLocalsFallback(f, summaries)
}
return solveAssignments(compiled)
}
func compileAssignments(f *scopeFacts, summaries summaryTables) (compiledAssignments, bool) {
assignments := make([]taintAssignment, 0, len(f.assigns)+len(f.references)*2+len(f.concats))
for _, a := range f.assigns {
var ok bool
assignments, _, ok = appendAssignment(assignments, a, a.Var, a.Expr)
if !ok {
return compiledAssignments{}, false
}
}
for _, a := range f.references {
var ok bool
assignments, _, ok = appendAssignment(assignments, a, a.Var, a.Expr)
if !ok {
return compiledAssignments{}, false
}
}
for _, a := range f.concats {
var index int
var ok bool
assignments, index, ok = appendAssignment(assignments, a, a.Var, a.Expr)
if !ok {
return compiledAssignments{}, false
}
if index >= 0 {
assignments[index].origins = append(assignments[index].origins, taintOrigin{
variable: assignedTargetKey(a.Var), assignment: -1,
})
}
}
forwardCount := len(assignments)
var fixed fixedValues
positioned := make([]positionedOrigin, 0, len(f.varNodes)+len(f.callSites)+forwardCount)
for _, variable := range f.readVarNodes() {
span, ok := spanOf(variable.node)
if !ok {
return compiledAssignments{}, false
}
positioned = append(positioned, positionedOrigin{
nodeSpan: span,
origin: taintOrigin{variable: variable.name, assignment: -1},
self: -1,
})
}
for _, call := range f.callNodes {
if g, source := sourceGrade(call); source {
span, ok := spanOf(call)
if !ok {
return compiledAssignments{}, false
}
positioned = append(positioned, positionedOrigin{
nodeSpan: span,
origin: taintOrigin{
assignment: -1, fixed: fixed.ref(setOf(g)),
},
self: -1,
})
}
}
for _, call := range f.callSites {
set, summarized := summaries.lookup(call)
if !summarized {
continue
}
span, ok := spanOf(call.node)
if !ok {
return compiledAssignments{}, false
}
positioned = append(positioned, positionedOrigin{
nodeSpan: span,
origin: taintOrigin{
assignment: -1, fixed: fixed.ref(set),
},
self: -1,
})
}
for i := 0; i < forwardCount; i++ {
span, ok := spanOf(assignments[i].node)
if !ok {
return compiledAssignments{}, false
}
positioned = append(positioned, positionedOrigin{
nodeSpan: span,
origin: taintOrigin{assignment: i},
self: i,
})
}
decoderSpans := make([]nodeSpan, 0)
for _, call := range f.callNodes {
if !decoders[calleeName(call.Function)] {
continue
}
for _, input := range decoderInputs(call) {
if span, ok := spanOf(input); ok {
decoderSpans = append(decoderSpans, span)
} else {
return compiledAssignments{}, false
}
}
}
decoders := newSpanIndex(decoderSpans)
for i := range positioned {
positioned[i].origin.decoded = decoders.contains(positioned[i].nodeSpan)
}
distributeOrigins(assignments, positioned)
// PHP references alias both names. A later write through either side
// changes the other, so add the persistent reverse dependency after the
// concrete assignment expression has received its own RHS origins.
for _, reference := range f.references {
target := assignedTargetKey(reference.Expr)
source := assignedTargetKey(reference.Var)
if target == "" || source == "" {
continue
}
assignments = append(assignments, taintAssignment{
target: target,
origins: []taintOrigin{{
variable: source, assignment: -1,
}},
})
}
return compiledAssignments{assignments: assignments, fixed: fixed.values}, true
}
func appendAssignment(assignments []taintAssignment, node, target, expr ast.Vertex) ([]taintAssignment, int, bool) {
name := assignedTargetKey(target)
if name == "" {
return assignments, -1, true
}
rhs, ok := spanOf(expr)
if !ok {
return nil, -1, false
}
assignments = append(assignments, taintAssignment{target: name, node: node, rhs: rhs})
return assignments, len(assignments) - 1, true
}
func distributeOrigins(assignments []taintAssignment, origins []positionedOrigin) {
intervals := make([]assignmentInterval, 0, len(assignments))
for i := range assignments {
if assignments[i].node != nil {
intervals = append(intervals, assignmentInterval{nodeSpan: assignments[i].rhs, assignment: i})
}
}
sort.Slice(intervals, func(i, j int) bool {
if intervals[i].start != intervals[j].start {
return intervals[i].start < intervals[j].start
}
return intervals[i].end > intervals[j].end
})
sort.Slice(origins, func(i, j int) bool {
if origins[i].start != origins[j].start {
return origins[i].start < origins[j].start
}
return origins[i].end < origins[j].end
})
stack := make([]assignmentInterval, 0)
next := 0
for _, origin := range origins {
for next < len(intervals) && intervals[next].start <= origin.start {
interval := intervals[next]
for len(stack) > 0 && stack[len(stack)-1].end < interval.start {
stack = stack[:len(stack)-1]
}
stack = append(stack, interval)
next++
}
for len(stack) > 0 && stack[len(stack)-1].end < origin.end {
stack = stack[:len(stack)-1]
}
for i := len(stack) - 1; i >= 0; i-- {
interval := stack[i]
if interval.assignment == origin.self || interval.start > origin.start || interval.end < origin.end {
continue
}
assignments[interval.assignment].origins = append(assignments[interval.assignment].origins, origin.origin)
break
}
}
}
type spanIndex struct {
spans []nodeSpan
maxEnds []int
}
func newSpanIndex(spans []nodeSpan) spanIndex {
sort.Slice(spans, func(i, j int) bool { return spans[i].start < spans[j].start })
index := spanIndex{spans: spans, maxEnds: make([]int, len(spans))}
maxEnd := -1
for i, span := range spans {
if span.end > maxEnd {
maxEnd = span.end
}
index.maxEnds[i] = maxEnd
}
return index
}
func (index spanIndex) contains(span nodeSpan) bool {
i := sort.Search(len(index.spans), func(i int) bool { return index.spans[i].start > span.start }) - 1
return i >= 0 && index.maxEnds[i] >= span.end
}
func solveAssignments(compiled compiledAssignments) taintState {
assignments := compiled.assignments
st := taintState{}
variableDependents := make(map[string][]int)
assignmentDependents := make([][]int, len(assignments))
for i, assignment := range assignments {
for _, origin := range assignment.origins {
switch {
case origin.fixed > 0:
case origin.assignment >= 0:
assignmentDependents[origin.assignment] = append(assignmentDependents[origin.assignment], i)
case origin.variable != "":
variableDependents[origin.variable] = append(variableDependents[origin.variable], i)
}
}
}
queue := make([]int, len(assignments))
queued := make([]bool, len(assignments))
outputs := make([]gradeSet, len(assignments))
outputSet := make([]bool, len(assignments))
for i := range assignments {
queue[i] = i
queued[i] = true
}
enqueue := func(indices []int) {
for _, index := range indices {
if !queued[index] {
queued[index] = true
queue = append(queue, index)
}
}
}
for head := 0; head < len(queue); head++ {
i := queue[head]
queued[i] = false
var best gradeSet
found := false
for _, origin := range assignments[i].origins {
var value gradeSet
var active bool
switch {
case origin.fixed > 0:
value, active = compiled.fixed[origin.fixed-1], true
case origin.assignment >= 0:
value, active = outputs[origin.assignment], outputSet[origin.assignment]
default:
value, active = st[origin.variable]
}
if !active {
continue
}
if origin.decoded {
// Decoding fetched content raises the confidence of this
// origin's proofs only; each keeps its own basis, and an
// origin outside every decoder keeps its own confidence.
value = value.decoded()
}
best.add(value)
found = true
}
if !found {
continue
}
if outputSet[i] {
merged := outputs[i]
if !merged.add(best) {
continue
}
best = merged
}
outputs[i], outputSet[i] = best, true
enqueue(assignmentDependents[i])
if st.raise(assignments[i].target, best) {
enqueue(variableDependents[assignments[i].target])
}
}
return st
}
// taintedLocalsFallback is the oracle path: no compiled solver, just direct
// fixpoint iteration over the raw facts. Its round cap must terminate AND
// stay complete. A fixed cap (the original defect) is neither: a reverse-
// ordered assignment chain propagates taint exactly one hop per round, so a
// chain longer than the cap evades detection entirely. Capping at the number
// of assignment facts plus one is always enough, because no dependency chain
// in this scope can have more hops than this scope has assignments, and it
// still terminates because it is a fixed bound.
func taintedLocalsFallback(f *scopeFacts, summaries summaryTables) taintState {
st := taintState{}
maxRounds := len(f.assigns) + len(f.references) + len(f.concats) + 1
for round := 0; round < maxRounds; round++ {
changed := false
for _, assignment := range f.assigns {
if g, tainted := exprTaint(assignment.Expr, st, summaries); tainted {
changed = st.raise(assignedTargetKey(assignment.Var), g) || changed
}
}
for _, assignment := range f.references {
if g, tainted := exprTaint(assignment.Expr, st, summaries); tainted {
changed = st.raise(assignedTargetKey(assignment.Var), g) || changed
}
if g, tainted := exprTaint(assignment.Var, st, summaries); tainted {
changed = st.raise(assignedTargetKey(assignment.Expr), g) || changed
}
}
for _, assignment := range f.concats {
if g, tainted := exprTaint(assignment.Expr, st, summaries); tainted {
changed = st.raise(assignedTargetKey(assignment.Var), g) || changed
}
}
if !changed {
return st
}
}
return st
}
// assignedVarName returns the root local variable written by an assignment,
// flattening any array-element or property-fetch chain onto its base
// variable. This remains the array-element behavior (containers are
// over-approximated on purpose, and array keys are not tracked at all), and
// it is also assignedTargetKey's fallback for a property chain it cannot key
// more precisely -- see there for when that applies.
func assignedVarName(target ast.Vertex) string {
for target != nil {
switch n := target.(type) {
case *ast.ExprVariable:
return varName(n.Name)
case *ast.ExprArrayDimFetch:
target = n.Var
case *ast.ExprPropertyFetch:
target = n.Var
default:
return ""
}
}
return ""
}
// propertyFetchParts returns the base expression and the (possibly dynamic)
// property vertex of a property fetch, regardless of whether it used the
// regular ("->") or nullsafe ("?->") operator. Nullsafe fetches cannot
// appear as an assignment target in valid PHP, but a property that was
// WRITTEN through the regular operator must still be found by a later READ
// spelled with "?->", so both node types resolve through the one path
// assignedTargetKey uses for both directions.
func propertyFetchParts(n ast.Vertex) (base, prop ast.Vertex, ok bool) {
switch v := n.(type) {
case *ast.ExprPropertyFetch:
return v.Var, v.Prop, true
case *ast.ExprNullsafePropertyFetch:
return v.Var, v.Prop, true
}
return nil, nil, false
}
// assignedTargetKey returns the taint-state key an assignment target -- or,
// via readVarNodes, a later read of the identical expression shape --
// resolves to. A bare variable resolves to its own name.
//
// The WHOLE access chain is walked, in whatever order property fetches
// ("->"/"?->") and array-dim fetches ("[...]") appear and however deeply
// nested: each property fetch contributes its name to the compound key, and
// each array-dim fetch is transparent, unwrapped without contributing a
// segment of its own. That is what keeps array indices over-approximated
// (containers are over-approximated on purpose: $a->log[] = X and
// $a->log[3] = Y both key to "a->log" as a whole -- array keys are not
// tracked) while still scoping precisely to the property chain around them.
//
// This generality is load-bearing, not incidental: an earlier version
// only unwrapped property fetches, so an array-dim fetch sitting
// OUTERMOST over a property fetch ($this->log[] = <tainted>, one of the
// most common idioms in OO PHP -- logs, queues, error collections, caches)
// broke the walk immediately and fell back to the bare base variable,
// reintroducing the exact wholesale-$this false-positive class this key
// scoping exists to close. Handling only that one shape and stopping would
// leave the same gap for every other ordering ($a[0]->b, $a->b[0]->c[1],
// ...), so the loop below handles both access kinds generically, in any
// order, rather than adding a case for the reported shape and calling it
// done.
//
// A property fetch is different from a bare variable or array element:
// $this->body = <tainted> must NOT taint every later use of bare $this,
// because $this appears in nearly every method of a class and one tainted
// property would otherwise poison every sink in it. This is deliberately
// not special-cased to the name "$this": $obj->prop = <tainted> gets
// exactly the same treatment. The key scopes to
// the full static property path (so $this->a->b is distinct from both
// $this->a and $this->a->c), joining base and property names with "->", a
// sequence no PHP variable name can contain, so a compound key can never
// collide with a bare one.
//
// The walk is iterative, not recursive: an access chain's depth is
// attacker-controlled PHP source ($a->b->c->... or $a[0][0][0]...), and Go
// recursion over unbounded attacker-controlled depth is an unrecoverable
// stack overflow, the same hazard TestTaintHandlesDeeplyNestedAssignments
// guards elsewhere in this package.
//
// A property whose name is not statically known ($obj->$name = X) has no
// specific key to scope to, so the ENTIRE chain falls back to
// assignedVarName's base-variable over-approximation, rather than a
// partially-keyed name nothing else would ever match.
func assignedTargetKey(target ast.Vertex) string {
var props []string
node := target
for {
if base, prop, ok := propertyFetchParts(node); ok {
name := calleeName(prop)
if name == "" {
return assignedVarName(target)
}
props = append(props, name)
node = base
continue
}
if dim, ok := node.(*ast.ExprArrayDimFetch); ok {
node = dim.Var
continue
}
break
}
base := assignedVarName(node)
if base == "" || len(props) == 0 {
return base
}
for i, j := 0, len(props)-1; i < j; i, j = i+1, j-1 {
props[i], props[j] = props[j], props[i]
}
return base + "->" + strings.Join(props, "->")
}
// exprTaint reports whether an expression carries remote content, and with
// which proofs. It collects the subtree once and correlates decoder inputs by
// source positions, without recursing over the parsed structure.
func exprTaint(e ast.Vertex, st taintState, summaries summaryTables) (gradeSet, bool) {
if e == nil {
return gradeSet{}, false
}
return exprTaintFacts(collectScope(e), st, summaries)
}
func exprTaintFacts(sub *scopeFacts, st taintState, summaries summaryTables) (gradeSet, bool) {
// A decoder raises confidence to Certain only for the origins that
// passed through THAT decoder's own argument, not for anything else in
// the same expression: f(base64_decode($clean), $tainted) must stay at
// the source's own grade, because the decode never touched $tainted, and
// in base64_decode(a()) . b() only a()'s proofs become Certain. The
// upgrade changes confidence only: each source's basis still explains
// how its content was acquired.
decoderSpans := make([]nodeSpan, 0)
for _, call := range sub.callNodes {
if !decoders[calleeName(call.Function)] {
continue
}
for _, input := range decoderInputs(call) {
if span, ok := spanOf(input); ok {
decoderSpans = append(decoderSpans, span)
}
}
}
return activeTaint(sub, st, summaries, newSpanIndex(decoderSpans))
}
// decoderInputs returns only arguments that carry data through the decoder.
// Most decoder APIs consume their payload in argument zero. pack is the
// exception: argument zero is a format string and the values begin at one.
func decoderInputs(call *ast.ExprFunctionCall) []ast.Vertex {
start := 0
if calleeName(call.Function) == "pack" {
start = 1
}
if start >= len(call.Args) {
return nil
}
inputs := make([]ast.Vertex, 0, len(call.Args)-start)
end := start + 1
if start == 1 {
end = len(call.Args)
}
for _, argNode := range call.Args[start:end] {
if arg, ok := argNode.(*ast.Argument); ok {
inputs = append(inputs, arg.Expr)
}
}
return inputs
}
// flowResult retains whether its already-bounded display evidence was
// shortened. Keeping this beside the Result until deduplication is complete
// lets Analyze report truncation only for evidence that actually survives to
// the returned result slice.
type flowResult struct {
Result
evidenceTruncated bool
}
// capturableNames expands a scope's taint keys into the binding names a
// capture can actually spell. Taint on a property is keyed by its whole access
// path ("o->body"), but a closure captures the base variable ($o) and receives
// the property along with it, so the base name has to be markable too or a
// genuinely dropped capture is never recorded.
func capturableNames(st taintState) map[string]bool {
names := make(map[string]bool, len(st))
for key := range st {
names[key] = true
if base, _, found := strings.Cut(key, "->"); found {
names[base] = true
}
}
return names
}
// hasDroppedCapture reports whether any closure or arrow function receives an
// outer binding that was tainted in the scope it captured from.
//
// A closure's body is analysed as its own scope, which is what stops an
// unrelated same-named outer variable from firing on clean code. The cost is
// that a value the closure genuinely does receive through use(), or that an
// arrow function picks up implicitly, stops being tracked at the boundary.
// This package's contract is that a reduction in coverage is recorded rather
// than passed over, so the drop is reported as precision loss.
//
// The check is deliberately gated on the captured value ACTUALLY being
// tainted. Capturing is ordinary PHP and appears in roughly one in seven of
// the files this analyzer examines, so a marker raised on the shape alone
// would be noise an operator cannot act on. Gated this way it means exactly
// one thing: taint was dropped here.
//
// Enclosing states are computed lazily and memoised, so a file with no
// capturing closure pays nothing and a scope with several capturing children
// is solved once. Nothing is seeded INTO a closure: taint deliberately does
// not cross the boundary, it is only reported as lost.
func hasDroppedCapture(
ctx context.Context, all *scopeFacts, tree declTree,
factsByScope map[ast.Vertex]*scopeFacts, summaries summaryTables,
) (bool, error) {
if len(all.closures) == 0 && len(all.arrowFuncs) == 0 {
return false, nil
}
// Record only declarations that actually capture a name. Building this
// cheap structural index first preserves the important lazy path: a file
// containing only non-capturing declarations never solves another taint
// state merely to decide that there was no capture.
captures := make(map[ast.Vertex]map[string]bool)
for _, cl := range all.closures {
if err := ctx.Err(); err != nil {
return false, err
}
names := closureCaptureNames(cl)
// A non-static closure created in object context binds $this
// implicitly, so it appears in no use() clause while still carrying
// whatever the enclosing object holds.
if body := factsByScope[cl]; cl.StaticTkn == nil && body != nil && body.vars["this"] {
names["this"] = true
}
if len(names) > 0 {
captures[cl] = names
}
}
for _, af := range all.arrowFuncs {
if err := ctx.Err(); err != nil {
return false, err
}
body := factsByScope[af]
if body == nil {
continue
}
if names := arrowCaptureNames(af, body); len(names) > 0 {
captures[af] = names
}
}
// $this is the exception to an ordinary closure being a hard capture
// boundary. Every non-static closure created in object context is bound to
// that object, even when only a declaration nested inside the closure reads
// $this. The independently collected closure facts exclude that nested
// declaration, so surface the implicit capture on every non-static
// closure/arrow along the path. Static declarations and named lexical
// scopes remain hard boundaries. Stopping at an already surfaced node keeps
// the total walk linear when many captures share a deeply nested path.
thisForwarded := make(map[ast.Vertex]bool)
var thisSeeds []ast.Vertex
for node, names := range captures {
if names["this"] {
thisSeeds = append(thisSeeds, node)
}
}
for _, node := range thisSeeds {
for scope := tree.parent[node]; scope != nil && !thisForwarded[scope]; scope = tree.parent[scope] {
if err := ctx.Err(); err != nil {
return false, err
}
forwards := false
switch declaration := scope.(type) {
case *ast.ExprClosure:
forwards = declaration.StaticTkn == nil
case *ast.ExprArrowFunction:
forwards = declaration.StaticTkn == nil
}
if !forwards {
break
}
thisForwarded[scope] = true
names := captures[scope]
if names == nil {
names = map[string]bool{}
captures[scope] = names
}
names["this"] = true
}
}
if len(captures) == 0 {
return false, nil
}
// A capture nested directly in an arrow function also makes that arrow
// capture the same binding implicitly. Mark every enclosing arrow on a
// capture path, plus the first ordinary lexical scope that supplies the
// value. Stopping when an already-marked scope is reached makes the total
// walk linear in declaration count even for deeply nested arrow trees.
needed := make(map[ast.Vertex]bool, len(captures)+1)
for node := range captures {
scope := tree.parent[node]
for !needed[scope] {
needed[scope] = true
if _, arrow := scope.(*ast.ExprArrowFunction); !arrow {
break
}
scope = tree.parent[scope]
}
}
// Taint states remain lazy and are memoised by their lexical scope. A
// class body can enclose a declaration but has no local variable facts of
// its own, so its state is intentionally empty.
states := make(map[ast.Vertex]taintState, len(needed))
stateFor := func(scope ast.Vertex) (taintState, error) {
if st, solved := states[scope]; solved {
return st, nil
}
if err := ctx.Err(); err != nil {
return nil, err
}
var st taintState
if f := factsByScope[scope]; f != nil {
st = taintedLocals(f, summaries)
}
if err := ctx.Err(); err != nil {
return nil, err
}
states[scope] = st
return st, nil
}
// active is a marker-only view of which tainted bindings are available at
// the current declaration boundary. It is never passed to taintedLocals or
// findFlows and therefore never seeds a closure or arrow body. Arrow scopes
// inherit the view because PHP propagates captures through nested arrows;
// every other declaration starts a new lexical variable scope. Boundary
// IDs make those resets O(1), while per-name stacks make enter/leave O(1).
type binding struct {
boundary int
tainted bool
}
type frame struct {
end int
previousBoundary int
pushed []string
}
active := make(map[string][]binding)
boundary, nextBoundary := 0, 0
push := func(name string, tainted bool, pushed *[]string) {
active[name] = append(active[name], binding{boundary: boundary, tainted: tainted})
*pushed = append(*pushed, name)
}
pop := func(f frame) {
for i := len(f.pushed) - 1; i >= 0; i-- {
name := f.pushed[i]
stack := active[name]
if len(stack) == 1 {
delete(active, name)
} else {
active[name] = stack[:len(stack)-1]
}
}
boundary = f.previousBoundary
}
capturesActive := func(names map[string]bool) bool {
for name := range names {
stack := active[name]
if len(stack) == 0 {
continue
}
value := stack[len(stack)-1]
if value.boundary == boundary && value.tainted {
return true
}
}
return false
}
if needed[nil] {
st, err := stateFor(nil)
if err != nil {
return false, err
}
for name := range capturableNames(st) {
active[name] = []binding{{boundary: boundary, tainted: true}}
}
}
frames := make([]frame, 0, 8)
seen := make(map[ast.Vertex]bool, len(captures))
for _, declaration := range tree.ordered {
if err := ctx.Err(); err != nil {
return false, err
}
// Exclusive end position, matching declarationTree's sweep: a
// declaration starting exactly where a frame ends is outside it.
for len(frames) > 0 && frames[len(frames)-1].end <= declaration.start {
pop(frames[len(frames)-1])
frames = frames[:len(frames)-1]
}
if names := captures[declaration.node]; len(names) > 0 {
seen[declaration.node] = true
if capturesActive(names) {
if err := ctx.Err(); err != nil {
return false, err
}
return true, nil
}
}
f := frame{end: declaration.end, previousBoundary: boundary}
if needed[declaration.node] {
if af, arrow := declaration.node.(*ast.ExprArrowFunction); arrow {
// A static arrow still captures ordinary outer variables, but
// it neither receives $this nor forwards it to a declaration
// nested inside the arrow.
if af.StaticTkn != nil {
push("this", false, &f.pushed)
}
// Parameters are the arrow's own bindings and shadow an
// identically named capture from any enclosing scope.
for name := range paramNames(af.Params) {
push(name, false, &f.pushed)
}
} else {
nextBoundary++
boundary = nextBoundary
}
st, err := stateFor(declaration.node)
if err != nil {
return false, err
}
for name := range capturableNames(st) {
push(name, true, &f.pushed)
}
}
frames = append(frames, f)
}
// Parser-produced declaration nodes always carry positions and therefore
// appear in tree.ordered. Retain the conservative direct-scope behavior if
// a future parser version emits an unpositioned capture node.
for node, names := range captures {
if seen[node] {
continue
}
st, err := stateFor(tree.parent[node])
if err != nil {
return false, err
}
available := capturableNames(st)
for name := range names {
if available[name] {
return true, nil
}
}
}
if err := ctx.Err(); err != nil {
return false, err
}
return false, nil
}
// unresolvableAssignRHS returns the right-hand-side expression of every
// assignment (=, =&, .=) in f whose target assignedTargetKey cannot resolve
// to a taint-state key: a method-call result mid-chain (`$a->b()->c = X`),
// a static property (`Foo::$cache = X`), a list()/[] destructuring target,
// or a variable-variable base. This is a purely structural scan (no taint
// fixpoint), used to cheaply decide whether hasUnresolvableTaintedTarget
// needs to do any further work at all.
func unresolvableAssignRHS(ctx context.Context, f *scopeFacts) ([]ast.Vertex, error) {
var out []ast.Vertex
for _, a := range f.assigns {
if err := ctx.Err(); err != nil {
return nil, err
}
if assignedTargetKey(a.Var) == "" {
out = append(out, a.Expr)
}
}
for _, a := range f.references {
if err := ctx.Err(); err != nil {
return nil, err
}
if assignedTargetKey(a.Var) == "" {
out = append(out, a.Expr)
}
}
for _, a := range f.concats {
if err := ctx.Err(); err != nil {
return nil, err
}
if assignedTargetKey(a.Var) == "" {
out = append(out, a.Expr)
}
}
return out, nil
}
// hasUnresolvableTaintedTarget reports whether f contains an assignment
// whose target cannot be keyed (see unresolvableAssignRHS) AND whose
// right-hand side is actually tainted -- the only case that represents a
// real, silent loss of tracking. `list($a, $b) = ['x', 'y']` and
// `Foo::$cache = 'literal'` drop nothing at all and must not be flagged;
// `list($a, $b) = curl_exec($u)` and `Foo::$cache = curl_exec($u)`
// genuinely drop taint this package cannot track further and must be.
//
// An earlier version of this check fired on the unkeyable SHAPE alone,
// regardless of taint. Measured against the reference corpus that reported
// the marker on 38.75% of analyzed files -- overwhelmingly ordinary,
// completely benign idioms (list() destructuring, static properties used
// as singletons/caches) that were not dropping anything -- noise no
// operator could act on. Gating on the RHS's own taint is what makes the
// marker mean "we dropped taint here" instead of "this file uses PHP".
//
// f.assigns/f.references/f.concats are scanned for an unresolvable target
// FIRST (see unresolvableAssignRHS), before the per-scope taint fixpoint
// below runs at all, so the (relatively expensive) taintedLocals call is
// skipped entirely for the overwhelming majority of scopes that have no
// unresolvable target -- the same early-return findFlows already uses for
// f.sinks being empty. Active origins are then correlated against the RHS
// spans from f's existing facts. Besides avoiding a traversal per RHS, this
// preserves canonical names already resolved from `use function` aliases;
// recollecting an isolated RHS has no access to the enclosing alias imports.
func hasUnresolvableTaintedTarget(
ctx context.Context, f *scopeFacts, summaries summaryTables,
) (bool, error) {
if err := ctx.Err(); err != nil {
return false, err
}
rhs, err := unresolvableAssignRHS(ctx, f)
if err != nil {
return false, err
}
if len(rhs) == 0 {
return false, nil
}
rhsSpans := make([]nodeSpan, 0, len(rhs))
var unpositioned []ast.Vertex
for _, expr := range rhs {
if err := ctx.Err(); err != nil {
return false, err
}
if span, ok := spanOf(expr); ok {
rhsSpans = append(rhsSpans, span)
} else {
unpositioned = append(unpositioned, expr)
}
}
var rhsReads []namedNodeSpan
if len(rhsSpans) > 0 {
index := newSpanIndex(rhsSpans)
for _, call := range f.callNodes {
if err := ctx.Err(); err != nil {
return false, err
}
span, positioned := spanOf(call)
if positioned && index.contains(span) {
if _, source := sourceGrade(call); source {
return true, nil
}
}
}
for _, call := range f.callSites {
if err := ctx.Err(); err != nil {
return false, err
}
span, positioned := spanOf(call.node)
if positioned && index.contains(span) {
if _, summarized := summaries.lookup(call); summarized {
return true, nil
}
}
}
for _, variable := range f.readVarNodes() {
if err := ctx.Err(); err != nil {
return false, err
}
span, positioned := spanOf(variable.node)
if positioned && index.contains(span) {
rhsReads = append(rhsReads, variable)
}
}
}
// Literal RHS values and direct source/summary calls are already decided
// above. Only variable-carried taint (or a synthetic positionless AST)
// needs the full per-scope assignment fixpoint.
if len(rhsReads) == 0 && len(unpositioned) == 0 {
return false, nil
}
st := taintedLocals(f, summaries)
if err := ctx.Err(); err != nil {
return false, err
}
for _, variable := range rhsReads {
if _, tainted := st[variable.name]; tainted {
return true, nil
}
}
// Parser-produced nodes have positions. Keep the helper total for
// synthetic or future positionless ASTs without turning shape alone into
// a precision-loss marker.
for _, expr := range unpositioned {
if err := ctx.Err(); err != nil {
return false, err
}
if _, tainted := exprTaint(expr, st, summaries); tainted {
return true, nil
}
}
return false, nil
}
// findFlows reports each sink in a scope that receives remote content.
// exclude names the declarations nested inside this scope. A sink's argument
// expression is collected WITHOUT it, so a call reached only through a closure
// there is still followed -- `include array_map(function(){ return fetch(); },
// $r)[0]` really does receive what that closure returns. Its variables are then
// filtered THROUGH it, because they are graded against this scope's taint
// state and a nested declaration's own parameter or local was never part of
// it. Same split, and same reason, as the return expressions in summaries.go.
// wholeCalls is the resolved-call index built from the unfiltered whole-file
// facts: the enclosing f correctly excludes nested declarations, so an index
// rebuilt from f cannot resolve namespace-level aliases on the calls
// deliberately kept from those declarations.
func findFlows(
ctx context.Context, f *scopeFacts, summaries summaryTables,
wholeCalls resolvedCallIndex, exclude *spanIndex,
) ([]flowResult, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if len(f.sinks) == 0 {
return nil, nil
}
st := taintedLocals(f, summaries)
if err := ctx.Err(); err != nil {
return nil, err
}
written := taintedWritePaths(f, st, summaries, wholeCalls, exclude)
out := make([]flowResult, 0, len(f.sinks))
for _, s := range f.sinks {
if err := ctx.Err(); err != nil {
return nil, err
}
sub := wholeCalls.apply(collectScope(s.expr)).withoutNestedDeclarationVars(exclude)
set, tainted := exprTaintFacts(sub, st, summaries)
if !tainted {
// A file this scope wrote from remote content and now includes
// by the same path expression executes that content just as a
// tainted variable would: the file is the carrier.
if ev, ok := written[pathExprKey(s.expr)]; ok && includeSink(s.kind) {
identifiers, identifiersTruncated := identifiersFor(sub)
out = append(out, flowResult{
Result: Result{
Source: ev.source,
Identifiers: identifiers,
Sink: s.kind,
Confidence: ev.value.conf,
Basis: ev.value.basis,
ResolutionOffset: ev.value.offset,
},
evidenceTruncated: ev.truncated || identifiersTruncated,
})
}
continue
}
c := set.strongest()
source, sourceTruncated := sourceLabel(sub, st, summaries)
identifiers, identifiersTruncated := identifiersFor(sub)
out = append(out, flowResult{
Result: Result{
Source: source,
Identifiers: identifiers,
Sink: s.kind,
Confidence: c.conf,
Basis: c.basis,
ResolutionOffset: c.offset,
},
evidenceTruncated: sourceTruncated || identifiersTruncated,
})
}
return out, nil
}
// sourceLabel names the acquiring construct for evidence. It prefers a
// directly visible source call, then a taint-returning callee, then the
// tainted variable that carried the value in. Resolution goes through
// callSites (rather than the bare f.calls name set) so a summarized method
// is matched via the same call-syntax-scoped lookup taintedLocals uses,
// instead of guessing across the function/method namespaces.
func sourceLabel(sub *scopeFacts, st taintState, summaries summaryTables) (string, bool) {
for _, call := range sub.callNodes {
if _, ok := sourceGrade(call); ok {
return sanitize(calleeName(call.Function), maxSegmentBytes)
}
}
names := make([]string, 0, len(sub.callSites))
for _, call := range sub.callSites {
if _, ok := summaries.lookup(call); ok {
names = append(names, call.name)
}
}
sort.Strings(names)
if len(names) > 0 {
return sanitize(names[0], maxSegmentBytes)
}
vars := make([]string, 0, len(sub.vars)+len(sub.propNodes))
for name := range sub.vars {
if _, ok := st[name]; ok {
vars = append(vars, name)
}
}
// A property key (e.g. "this->body") is checked separately from the
// bare-variable loop above: the base variable's own name is not tainted
// by a property write, so a flow carried entirely by a specific property
// would otherwise fall through every branch above and report "unknown"
// instead of naming the property that actually carried it.
for _, n := range sub.propNodes {
key := assignedTargetKey(n)
if key == "" {
continue
}
if _, ok := st[key]; ok {
vars = append(vars, key)
}
}
sort.Strings(vars)
if len(vars) > 0 {
return sanitize("$"+vars[0], maxSegmentBytes)
}
return "unknown", false
}
// identifiersFor renders the sanitized, bounded Identifiers list for
// evidence: every distinct variable and call name appearing anywhere in the
// sink's own expression, tainted or not. This is not a laundering path --
// there is no ordering or filtering by whether a name actually carried the
// tainted value, only alphabetical sort for deterministic output.
func identifiersFor(sub *scopeFacts) ([]string, bool) {
segs := make([]string, 0, len(sub.vars)+len(sub.calls))
for name := range sub.vars {
if name != "" {
segs = append(segs, "$"+name)
}
}
for name := range sub.calls {
segs = append(segs, name)
}
sort.Strings(segs)
return truncateChain(segs)
}
// dedupeAndSort collapses flows that render the same display endpoint pair
// and orders results so repeated runs are byte-identical.
func dedupeAndSort(in []flowResult) []flowResult {
seen := map[string]int{}
out := make([]flowResult, 0, len(in))
for _, r := range in {
key := r.Source + "\x00" + r.Sink
if idx, ok := seen[key]; ok {
// The stronger flow replaces the whole grade, so a raised
// confidence always carries its own basis and resolution point.
cur := directGradeOf(out[idx].Result)
if next := directGradeOf(r.Result); next.stronger(cur) {
out[idx].Confidence = r.Confidence
out[idx].Basis = r.Basis
out[idx].ResolutionOffset = r.ResolutionOffset
}
continue
}
seen[key] = len(out)
out = append(out, r)
}
sort.Slice(out, func(i, j int) bool {
if out[i].Source != out[j].Source {
return out[i].Source < out[j].Source
}
if out[i].Sink != out[j].Sink {
return out[i].Sink < out[j].Sink
}
return strings.Join(out[i].Identifiers, ",") < strings.Join(out[j].Identifiers, ",")
})
return out
}
// retainStrongestEvidence applies the display cap without hiding the grade
// that downstream alerting must use. The input is endpoint-sorted, so when the
// first strongest flow falls beyond the cap, replacing the last retained flow
// with it keeps the returned evidence deterministic and sorted.
func retainStrongestEvidence(flows []flowResult) []flowResult {
if len(flows) <= MaxEvidenceResults {
return flows
}
strongest := 0
for i := 1; i < len(flows); i++ {
// Basis only explains a flow; it must not change the retained
// endpoints (and therefore the finding's identity) on a confidence tie.
if flows[i].Confidence > flows[strongest].Confidence {
strongest = i
}
}
retained := append([]flowResult(nil), flows[:MaxEvidenceResults]...)
if strongest >= MaxEvidenceResults {
retained[len(retained)-1] = flows[strongest]
}
return retained
}
// activeTaint joins the proofs of an already-collected subtree's source
// calls, summarized calls, and tainted variable reads. Each origin inside a
// decoder input span is upgraded on its own before the join, so correlation
// stays linearithmic rather than recursively recollecting every nested
// decoder argument, and a proof outside every decoder keeps its confidence.
// A variable name is looked up in a taintState in exactly five places in this
// package, and every one of them must be fed by facts collected WITH
// declaration exclusion, or a nested declaration's own parameter or local
// borrows an identically named outer variable's taint and reports a flow on
// clean code. Names like $data, $content and $url recur constantly in real
// PHP, so a bare collision is enough. The five, and what keeps each scoped:
//
// solveAssignments origins built from the scope's own facts
// hasUnresolvableTaintedTarget the scope's own facts
// sourceLabel a filtered sub (vars rebuilt to match)
// stateFor taintedLocals over a per-scope facts value
// activeTaint (below) see the three feeders named next
//
// activeTaint is reached only through exprTaintFacts, which has three
// feeders: evalBodySummary and findFlows, both of which collect their
// expression WITHOUT exclusion so that a call reached only through a nested
// closure is still followed, and then filter its variable side back through
// the scope's exclusion index; and exprTaint, whose two callers are both
// reachable only for an AST node carrying no position, which this parser does
// not produce.
//
// Anything new that grades an expression against a scope's taint state joins
// this list and needs the same treatment. Fixing one instance at a time does
// not work here: this defect was found and fixed three separate times -- in
// the summaries path, in the sink path, and in the capture walk -- before the
// enumeration above made it possible to say the class was closed rather than
// merely that no more instances had turned up.
func activeTaint(sub *scopeFacts, st taintState, summaries summaryTables, decoderArgs spanIndex) (gradeSet, bool) {
var best gradeSet
found := false
join := func(set gradeSet, node ast.Vertex) {
// An origin without a position cannot be placed inside a decoder
// input, so it keeps its own grade.
if span, ok := spanOf(node); ok && decoderArgs.contains(span) {
set = set.decoded()
}
best.add(set)
found = true
}
for _, call := range sub.callNodes {
if c, ok := sourceGrade(call); ok {
join(setOf(c), call)
}
}
for _, call := range sub.callSites {
if c, ok := summaries.lookup(call); ok {
join(c, call.node)
}
}
for _, variable := range sub.readVarNodes() {
if c, ok := st[variable.name]; ok {
join(c, variable.node)
}
}
return best, found
}
// writtenEvidence is what a tainted file write contributes to an include of
// the same path.
type writtenEvidence struct {
source string
value grade
truncated bool
}
// taintedWritePaths maps the path key of every file_put_contents whose data
// argument is tainted in this scope to the evidence of that taint.
func taintedWritePaths(
f *scopeFacts, st taintState, summaries summaryTables,
wholeCalls resolvedCallIndex, exclude *spanIndex,
) map[string]writtenEvidence {
if len(f.fileWrites) == 0 {
return nil
}
out := make(map[string]writtenEvidence, len(f.fileWrites))
for _, w := range f.fileWrites {
key := pathExprKey(w.path)
if key == "" {
continue
}
sub := wholeCalls.apply(collectScope(w.data)).withoutNestedDeclarationVars(exclude)
set, tainted := exprTaintFacts(sub, st, summaries)
if !tainted {
continue
}
c := set.strongest()
if prev, ok := out[key]; ok {
if !c.stronger(prev.value) {
continue
}
if c.conf == prev.value.conf {
// Keep the first strongest-confidence endpoint as before;
// only its explanation changes when another basis wins.
prev.value = c
out[key] = prev
continue
}
}
source, truncated := sourceLabel(sub, st, summaries)
out[key] = writtenEvidence{source: source, value: c, truncated: truncated}
}
return out
}
func includeSink(kind string) bool {
switch kind {
case "include", "include_once", "require", "require_once":
return true
}
return false
}
// pathExprKey renders a path expression into a canonical string so a write
// and an include of the same path match syntactically: literals, magic
// constants, constants, variables, concatenations and argument-less or
// keyable calls. Anything else (a property, an array element, an unresolved
// call) yields "" and never matches, so no guess can produce a flow.
func pathExprKey(n ast.Vertex) string { return pathExprKeyAt(n, 0) }
func pathExprKeyAt(n ast.Vertex, depth int) string {
if depth >= maxAnalysisDepth {
return ""
}
switch v := n.(type) {
case *ast.ScalarString:
return "s:" + string(v.Value)
case *ast.ScalarMagicConstant:
return "m:" + strings.ToLower(string(v.Value))
case *ast.ExprConstFetch:
return "c:" + strings.ToLower(calleeName(v.Const))
case *ast.ExprVariable:
name := varName(v.Name)
if name == "" {
return ""
}
return "v:" + name
case *ast.ExprBrackets:
return pathExprKeyAt(v.Expr, depth+1)
case *ast.ExprBinaryConcat:
left := pathExprKeyAt(v.Left, depth+1)
right := pathExprKeyAt(v.Right, depth+1)
if left == "" || right == "" {
return ""
}
return "(" + left + "." + right + ")"
case *ast.ExprFunctionCall:
name := calleeName(v.Function)
if name == "" {
return ""
}
parts := make([]string, 0, len(v.Args))
for _, a := range v.Args {
arg, ok := a.(*ast.Argument)
if !ok {
return ""
}
key := pathExprKeyAt(arg.Expr, depth+1)
if key == "" {
return ""
}
parts = append(parts, key)
}
return "f:" + strings.ToLower(name) + "(" + strings.Join(parts, ",") + ")"
}
return ""
}
// Package phptaintipc defines the wire protocol spoken between the CSM daemon
// and the supervised `csm phptaint-worker` child process.
//
// The worker exists because the PHP parser this analyzer depends on can enter
// an infinite loop on attacker-controlled input. Two such inputs are known and
// there is no reason to believe they are the only ones: it is a hand-written
// lexer in an unmaintained package, and the second class was found outside a
// search that had concluded the first was the only one. A loop is not a panic,
// so recover() cannot catch it, and the parser never checks context, so a
// deadline around the call cannot stop it either. The only thing that reliably
// stops it is killing the process it runs in -- which is what the supervisor on
// the other side of this protocol does, and why analysis runs out of process at
// all.
//
// Framing, transport and supervision are internal/workerproc's: frames travel
// over a socket pair inherited across fork, which needs no socket path and so
// no new writable directory under the daemon's ProtectSystem=strict. This
// package defines the payloads and how they are validated.
//
// Frames carry phptaint's own Report type rather than a restatement of it.
// Worker and daemon are the same binary, so the two definitions can never
// disagree, and a translation layer could only introduce drift. That matters
// more here than it looks: an unmapped status would decode as the zero value,
// which is StatusNotCandidate -- "this file is clean". A coverage gap silently
// becoming a clean result is the one failure this package must not have.
package phptaintipc
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/pidginhost/csm/internal/phptaint"
"github.com/pidginhost/csm/internal/workerproc"
)
// MaxFrameBytes caps a single request or response body. The largest legitimate
// frame is an analyze request carrying phptaint.MaxSourceBytes of source, which
// JSON base64-encodes at 4/3 expansion, plus envelope. Responses are far
// smaller: a Report's evidence is already bounded by phptaint itself.
const MaxFrameBytes = 8 << 20
// Op selects the handler on the worker side. Strings rather than iota ints so
// an unrecognised op is reported as such instead of silently matching whatever
// constant happens to share its number.
const (
OpAnalyze = "analyze"
OpPing = "ping"
)
// ProtocolVersion is what the worker answers to workerproc's hello. Bump it
// when a frame changes shape, so a worker from another build is refused.
const ProtocolVersion = 1
// ErrSourceTooLarge is returned when a source buffer exceeds what phptaint
// would analyze anyway. Rejecting it here keeps the ceiling in one place and
// avoids marshalling a multi-megabyte frame only to have the worker decline it.
var ErrSourceTooLarge = errors.New("phptaintipc: source exceeds maximum analyzed size")
// AnalyzeArgs carries the bytes to analyze. The daemon has already read the
// file and run the pre-filter, so only admitted content crosses the boundary,
// and the worker is never given a path to open on its own.
type AnalyzeArgs struct {
Source []byte `json:"source"`
}
// AnalyzeResult carries the completed report.
type AnalyzeResult struct {
Report phptaint.Report `json:"report"`
}
// PingResult answers a readiness probe.
type PingResult struct {
OK bool `json:"ok"`
}
// EncodePayload validates and marshals v into a frame under op. AnalyzeArgs is
// size-checked here so an oversize source fails before it is ever encoded.
func EncodePayload(op string, v any) (workerproc.Frame, error) {
if v == nil {
return workerproc.Frame{Op: op}, nil
}
if err := validatePayload(v); err != nil {
return workerproc.Frame{}, err
}
raw, err := json.Marshal(v)
if err != nil {
return workerproc.Frame{}, fmt.Errorf("phptaintipc: marshal %s payload: %w", op, err)
}
return workerproc.Frame{Op: op, Payload: raw}, nil
}
// DecodePayload unmarshals and validates a frame's payload into v.
func DecodePayload(f workerproc.Frame, v any) error {
if len(f.Payload) == 0 {
return errors.New("phptaintipc: frame has no payload")
}
// Decode the security-sensitive payloads into fresh values. A missing JSON
// field must not retain a prior request's source or a prior response's clean
// status when a caller reuses storage across frames.
switch out := v.(type) {
case *AnalyzeArgs:
if out == nil {
return errors.New("phptaintipc: nil AnalyzeArgs decode target")
}
if _, err := requiredJSONField(f.Payload, "source"); err != nil {
return err
}
var decoded AnalyzeArgs
if err := json.Unmarshal(f.Payload, &decoded); err != nil {
return fmt.Errorf("phptaintipc: unmarshal payload: %w", err)
}
if err := validatePayload(decoded); err != nil {
return err
}
*out = decoded
return nil
case *AnalyzeResult:
if out == nil {
return errors.New("phptaintipc: nil AnalyzeResult decode target")
}
reportJSON, err := requiredJSONField(f.Payload, "report")
if err != nil {
return err
}
statusJSON, err := requiredJSONField(reportJSON, "status")
if err != nil {
return err
}
var status *phptaint.Status
if err := json.Unmarshal(statusJSON, &status); err != nil {
return fmt.Errorf("phptaintipc: unmarshal report status: %w", err)
}
if status == nil {
return errors.New("phptaintipc: report status is null")
}
if err := requireResultEvidenceKeys(reportJSON); err != nil {
return err
}
var decoded AnalyzeResult
if err := json.Unmarshal(f.Payload, &decoded); err != nil {
return fmt.Errorf("phptaintipc: unmarshal payload: %w", err)
}
if decoded.Report.Status != *status {
return errors.New("phptaintipc: ambiguous report status")
}
if err := validatePayload(decoded); err != nil {
return err
}
*out = decoded
return nil
}
if err := json.Unmarshal(f.Payload, v); err != nil {
return fmt.Errorf("phptaintipc: unmarshal payload: %w", err)
}
return nil
}
func validatePayload(v any) error {
switch payload := v.(type) {
case AnalyzeArgs:
return validateSourceSize(payload.Source)
case *AnalyzeArgs:
if payload == nil {
return errors.New("phptaintipc: nil AnalyzeArgs payload")
}
return validateSourceSize(payload.Source)
case AnalyzeResult:
return validateReport(payload.Report)
case *AnalyzeResult:
if payload == nil {
return errors.New("phptaintipc: nil AnalyzeResult payload")
}
return validateReport(payload.Report)
}
return nil
}
func validateSourceSize(source []byte) error {
if len(source) > phptaint.MaxSourceBytes {
return fmt.Errorf("%w (%d > %d bytes)", ErrSourceTooLarge, len(source), phptaint.MaxSourceBytes)
}
return nil
}
func validateReport(report phptaint.Report) error {
if report.Status.String() == "unknown" {
return fmt.Errorf("phptaintipc: unknown report status %d", report.Status)
}
hasEvidence := len(report.Results) != 0 || report.TotalResults != 0 ||
len(report.PrecisionLoss) != 0 || report.EvidenceTruncated
switch report.Status {
case phptaint.StatusTimeout, phptaint.StatusWorkerFailure:
// These describe what the parent did to, or could not do with, the
// worker. A process that returned a reply cannot report either one.
return errors.New("phptaintipc: worker reply carries a supervisor-only status")
case phptaint.StatusNotCandidate:
if hasEvidence || report.Reason != "" {
return errors.New("phptaintipc: not-candidate report carries analysis data")
}
case phptaint.StatusAnalyzed:
if report.Reason != "" {
return errors.New("phptaintipc: analyzed report carries an error reason")
}
if report.TotalResults < 0 {
return errors.New("phptaintipc: analyzed report has inconsistent result counts")
}
expectedResults := report.TotalResults
if expectedResults > phptaint.MaxEvidenceResults {
expectedResults = phptaint.MaxEvidenceResults
}
if len(report.Results) != expectedResults {
return errors.New("phptaintipc: analyzed report has inconsistent result counts")
}
if report.TotalResults > len(report.Results) && !report.EvidenceTruncated {
return errors.New("phptaintipc: analyzed report omitted evidence without marking truncation")
}
for _, result := range report.Results {
if result.Confidence.String() == "unknown" {
return errors.New("phptaintipc: analyzed report has unknown confidence")
}
if !result.Basis.Valid() {
return errors.New("phptaintipc: analyzed report has missing or unknown basis")
}
if result.ResolutionOffset < -1 {
return errors.New("phptaintipc: analyzed report has an invalid resolution offset")
}
}
default:
if hasEvidence {
return errors.New("phptaintipc: incomplete report carries analysis data")
}
if report.Reason == "" {
return errors.New("phptaintipc: incomplete report has no reason")
}
}
if len(report.Reason) > phptaint.MaxReasonBytes {
return fmt.Errorf("phptaintipc: report reason is %d bytes, exceeds cap %d", len(report.Reason), phptaint.MaxReasonBytes)
}
return nil
}
// ValidateReportForSource applies the checks that need the submitted source:
// a resolution offset must point inside it. The parent calls it after
// DecodePayload, which has already applied every source-independent check.
func ValidateReportForSource(report phptaint.Report, sourceLen int) error {
if err := validateReport(report); err != nil {
return err
}
for _, result := range report.Results {
if result.ResolutionOffset >= sourceLen {
return fmt.Errorf("phptaintipc: resolution offset %d outside %d-byte source", result.ResolutionOffset, sourceLen)
}
}
return nil
}
// requireResultEvidenceKeys checks every result for its basis and resolution
// offset keys. A missing offset would decode to 0, a real position, so a reply
// from a worker that predates either field must fail rather than pose as
// evidence resolved at the first byte.
func requireResultEvidenceKeys(reportJSON json.RawMessage) error {
resultsJSON, ok, err := lookupJSONField(reportJSON, "results")
if err != nil {
return err
}
if !ok {
return nil
}
var results []json.RawMessage
if err := json.Unmarshal(resultsJSON, &results); err != nil {
return fmt.Errorf("phptaintipc: unmarshal report results: %w", err)
}
for _, result := range results {
if _, err := requiredJSONField(result, "basis"); err != nil {
return err
}
offsetJSON, err := requiredJSONField(result, "resolutionoffset")
if err != nil {
return err
}
var offset *int
if err := json.Unmarshal(offsetJSON, &offset); err != nil {
return fmt.Errorf("phptaintipc: unmarshal resolution offset: %w", err)
}
if offset == nil {
return errors.New("phptaintipc: result resolution offset is null")
}
}
return nil
}
func requiredJSONField(raw []byte, name string) (json.RawMessage, error) {
found, ok, err := lookupJSONField(raw, name)
if err != nil {
return nil, err
}
if !ok {
return nil, fmt.Errorf("phptaintipc: payload has no %s field", name)
}
return found, nil
}
// lookupJSONField finds name in a JSON object the way encoding/json matches
// struct fields, case-insensitively, and rejects more than one spelling so a
// reply cannot carry two values for one field.
func lookupJSONField(raw []byte, name string) (json.RawMessage, bool, error) {
// A map discards repeated keys, but struct decoding can merge their
// values. Inspect every occurrence so validation sees what decoding sees.
if !json.Valid(raw) {
return nil, false, fmt.Errorf("phptaintipc: invalid JSON inspecting %s field", name)
}
decoder := json.NewDecoder(bytes.NewReader(raw))
start, err := decoder.Token()
if err != nil || start != json.Delim('{') {
return nil, false, fmt.Errorf("phptaintipc: expected object inspecting %s field", name)
}
var found json.RawMessage
ok := false
for decoder.More() {
field, err := decoder.Token()
if err != nil {
return nil, false, fmt.Errorf("phptaintipc: inspect %s field: %w", name, err)
}
var value json.RawMessage
if err := decoder.Decode(&value); err != nil {
return nil, false, fmt.Errorf("phptaintipc: inspect %s value: %w", name, err)
}
if !strings.EqualFold(field.(string), name) {
continue
}
if ok {
return nil, false, fmt.Errorf("phptaintipc: payload has ambiguous %s fields", name)
}
found, ok = value, true
}
return found, ok, nil
}
package phptaintworker
import (
"context"
"errors"
"fmt"
"os"
"sync"
"time"
"github.com/pidginhost/csm/internal/phptaint"
"github.com/pidginhost/csm/internal/phptaintipc"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/workerproc"
)
// ConsecutiveFailureLimit is how many failed workers in a row are tolerated
// before spawning pauses for one cooldown; see workerproc.
const ConsecutiveFailureLimit = workerproc.ConsecutiveFailureLimit
// SupervisorConfig describes how to start the worker process.
type SupervisorConfig struct {
// Launch starts a worker. The daemon passes workerproc.UnitLauncher so
// the worker runs locked down; nil runs Command directly, which only
// tests do.
Launch workerproc.Launcher
// Command and Args start the worker when Launch is nil.
Command string
Args []string
Env []string
// Timeout bounds a single analysis. It must exceed the slowest legitimate
// file by a wide margin: every expiry costs a killed process, so a value
// tuned too tightly converts slow-but-fine files into coverage gaps.
Timeout time.Duration
// Log is optional.
Log func(string, ...any)
}
// Supervisor runs PHP taint analysis in a worker process. Containment, the
// breaker and request health are workerproc's; this type adds the pre-filter,
// the payload schema and the mapping from failures to coverage gaps.
type Supervisor struct {
cfg SupervisorConfig
proc *workerproc.Supervisor
mu sync.Mutex
mode string
}
// NewSupervisor validates config. It does not start a worker: the first
// analysis does, so a scan that never admits a candidate never forks.
func NewSupervisor(cfg SupervisorConfig) (*Supervisor, error) {
if cfg.Command == "" && cfg.Launch == nil {
return nil, errors.New("phptaintworker: no launcher or command")
}
if cfg.Timeout <= 0 {
return nil, errors.New("phptaintworker: timeout must be positive")
}
s := &Supervisor{cfg: cfg}
proc, err := workerproc.New(workerproc.Config{
Name: "phptaint worker",
Launch: s.launch,
Timeout: cfg.Timeout,
Limits: workerproc.Limits{MaxFrameBytes: phptaintipc.MaxFrameBytes},
Log: cfg.Log,
})
if err != nil {
return nil, err
}
s.proc = proc
return s, nil
}
func (s *Supervisor) launch(ctx context.Context) (workerproc.Process, error) {
if s.cfg.Launch != nil {
return s.cfg.Launch(ctx)
}
s.mu.Lock()
mode := s.mode
s.mu.Unlock()
env := s.cfg.Env
if mode != "" {
env = append(append([]string(nil), env...), "CSM_HELPER_WORKER="+mode)
}
return workerproc.ExecLauncher(s.cfg.Command, s.cfg.Args, env)(ctx)
}
// SetMode is a test seam for the helper child, which selects its behaviour
// from the environment. Production callers never use it.
func (s *Supervisor) SetMode(mode string) {
s.mu.Lock()
defer s.mu.Unlock()
s.mode = mode
}
// LastChildPID reports the pid of the most recently started worker.
func (s *Supervisor) LastChildPID() int { return s.proc.Stats().LastPID }
// SpawnCount reports how many workers have been started.
func (s *Supervisor) SpawnCount() int { return s.proc.Stats().Spawns }
// Stop shuts down the current worker.
func (s *Supervisor) Stop() error { return s.proc.Stop() }
// QueueStatuses does not take the supervisor lock or inspect child processes.
func (s *Supervisor) QueueStatuses(now time.Time) map[string]queuehealth.Status {
return s.proc.QueueStatuses(now)
}
// Analyze runs one analysis in the worker. It never returns a status a caller
// could read as clean unless the worker actually produced one, or unless the
// pre-filter the worker applies before parsing already rejects the content.
func (s *Supervisor) Analyze(ctx context.Context, src []byte) phptaint.Report {
// The deep scan hands every file it reads to this analyzer, and the
// worker serves one request at a time. Content the pre-filter rejects
// needs no parser, so answering it here keeps images and plain text from
// queuing behind real analyses or forking a worker. Size and
// cancellation are the answers the worker path gives first, so they
// still take precedence.
if len(src) <= phptaint.MaxSourceBytes && ctx.Err() == nil && !phptaint.IsCandidate(src) {
// Cancellation may arrive during the byte scan. The IPC path
// observes it while waiting for a reply; a local answer must too.
if err := ctx.Err(); err != nil {
return gap(phptaint.StatusCanceled, err.Error())
}
return phptaint.Report{Status: phptaint.StatusNotCandidate}
}
req, err := phptaintipc.EncodePayload(phptaintipc.OpAnalyze, phptaintipc.AnalyzeArgs{Source: src})
if err != nil {
// An oversize source is the caller's own ceiling, not a worker fault,
// so it must not count toward the breaker. It never reaches a worker,
// so it is the answer whatever state the worker side is in.
return gap(phptaint.StatusOversize, err.Error())
}
var report phptaint.Report
_, _, err = s.proc.Call(ctx, req, nil, func(reply workerproc.Frame, _ []*os.File) (bool, error) {
var out phptaintipc.AnalyzeResult
if decodeErr := phptaintipc.DecodePayload(reply, &out); decodeErr != nil {
return false, decodeErr
}
if validErr := phptaintipc.ValidateReportForSource(out.Report, len(src)); validErr != nil {
return false, validErr
}
report = out.Report
return report.Status == phptaint.StatusPanic, nil
})
switch {
case err == nil:
return report
case errors.Is(err, workerproc.ErrStopped):
return gap(phptaint.StatusWorkerFailure, "supervisor stopped")
case errors.Is(err, workerproc.ErrTimeout):
return gap(phptaint.StatusTimeout, fmt.Sprintf("analysis exceeded %s; worker killed", s.cfg.Timeout))
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
return gap(phptaint.StatusCanceled, err.Error())
default:
return gap(phptaint.StatusWorkerFailure, err.Error())
}
}
func gap(status phptaint.Status, reason string) phptaint.Report {
return phptaint.CoverageGap(status, reason)
}
package phptaintworker
import (
"strconv"
"time"
"github.com/pidginhost/csm/internal/phptaintipc"
"github.com/pidginhost/csm/internal/workerproc"
)
// DefaultMemoryLimit caps the worker's memory and swap together.
const DefaultMemoryLimit = 512 << 20
// UnitConfig is the locked-down unit the PHP taint worker runs in: binary is
// the csm executable, prefix names the units. The worker writes no files, so
// it may not grow any.
func UnitConfig(binary, prefix string, log func(string, ...any)) workerproc.UnitConfig {
return workerproc.UnitConfig{
Prefix: prefix,
Command: binary,
Args: []string{"phptaint-worker", "--memory-limit=" + strconv.Itoa(DefaultMemoryLimit)},
MemoryLimit: DefaultMemoryLimit,
FileSizeLimit: 0,
RuntimeMax: time.Hour,
StartTimeout: 10 * time.Second,
Protocol: phptaintipc.ProtocolVersion,
Limits: workerproc.Limits{MaxFrameBytes: phptaintipc.MaxFrameBytes},
Log: log,
}
}
// Package phptaintworker is the child side of PHP taint analysis.
//
// The analyzer's parser can enter an infinite loop on attacker-controlled
// input. Two such inputs are known, both found by accident rather than by
// audit, in a hand-written lexer that has been unmaintained since 2022. A loop
// is not a panic, so recover() cannot catch it, and the parser never checks
// context, so no deadline inside this process can stop it. Running the analysis
// here means the parent can stop it the only way that works: by killing this
// process.
//
// Consequences for anything written in this package: a request that never
// returns is an expected outcome, not a bug to defend against locally, and this
// side must stay simple enough that the parent's timeout is the sole liveness
// mechanism. Do not add an internal watchdog -- it would only ever fire on
// inputs the parent already handles, while giving a false impression that this
// process can rescue itself.
package phptaintworker
import (
"context"
"fmt"
"net"
"os"
"os/signal"
"github.com/pidginhost/csm/internal/phptaint"
"github.com/pidginhost/csm/internal/phptaintipc"
"github.com/pidginhost/csm/internal/workerproc"
)
// Serve answers analysis requests on conn, one at a time, with no state
// carried between them; workerproc.Serve owns the loop and its framing.
//
// A valid frame with a bad operation or payload is answered with an error and
// the loop continues. Dropping the process for that peer mistake would make the
// parent replace its worker repeatedly.
func Serve(ctx context.Context, conn *net.UnixConn) error {
// The daemon passes no descriptors to this worker, so none are accepted.
lim := workerproc.Limits{MaxFrameBytes: phptaintipc.MaxFrameBytes}
return workerproc.Serve(ctx, conn, lim, func(ctx context.Context, req workerproc.Frame, _ []*os.File) (workerproc.Frame, []*os.File) {
return handle(ctx, req), nil
})
}
func handle(ctx context.Context, req workerproc.Frame) workerproc.Frame {
switch req.Op {
case workerproc.OpHello:
return workerproc.HelloReply(phptaintipc.ProtocolVersion)
case phptaintipc.OpPing:
return reply(phptaintipc.OpPing, phptaintipc.PingResult{OK: true})
case phptaintipc.OpAnalyze:
var args phptaintipc.AnalyzeArgs
if err := phptaintipc.DecodePayload(req, &args); err != nil {
return workerproc.Frame{Error: err.Error()}
}
// Analyze already contains its own panic boundary and reports one as a
// coverage gap, so there is nothing to recover here.
report := phptaint.Analyze(ctx, args.Source)
return reply(phptaintipc.OpAnalyze, phptaintipc.AnalyzeResult{Report: report})
default:
return workerproc.Frame{Error: fmt.Sprintf("phptaintworker: unknown op %q", req.Op)}
}
}
func reply(op string, v any) workerproc.Frame {
frame, err := phptaintipc.EncodePayload(op, v)
if err != nil {
return workerproc.Frame{Error: err.Error()}
}
return frame
}
// signalIgnore makes a signal a no-op for this process. The supervisor tests
// use it to build a child that a catchable signal cannot stop, which is what a
// parser loop behaves like.
func signalIgnore(sig os.Signal) {
signal.Ignore(sig)
}
package platform
import "path/filepath"
// IsCPanelCentralAccessLog identifies the server logs whose paths carry
// cPanel's proxy routing. An arbitrary access_logs override can name a
// website log, so only the platform's own central paths and their aliases
// establish that provenance.
func (i Info) IsCPanelCentralAccessLog(path string) bool {
if i.Panel != PanelCPanel || path == "" {
return false
}
defaults := Info{OS: i.OS, Panel: i.Panel, WebServer: i.WebServer}
populatePaths(&defaults)
path = resolvedAccessLogPath(path)
for _, candidate := range defaults.AccessLogPaths {
if path == resolvedAccessLogPath(candidate) {
return true
}
}
return false
}
func resolvedAccessLogPath(path string) string {
absolute, err := filepath.Abs(path)
if err != nil {
return ""
}
path = absolute
if real, err := filepath.EvalSymlinks(path); err == nil {
return real
}
return path
}
package platform
import (
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"github.com/pidginhost/csm/internal/safepath"
)
// ValidateAccountRootPattern rejects paths that cannot define a confined
// content tree. Missing directories are allowed for later provisioning.
func ValidateAccountRootPattern(pattern string) error {
if !filepath.IsAbs(pattern) || filepath.Clean(pattern) != pattern {
return fmt.Errorf("account root must be an absolute, clean path: %q", pattern)
}
for _, char := range pattern {
if char < 0x20 || char == 0x7f || char == '\\' {
return fmt.Errorf("account root contains an unsupported character: %q", pattern)
}
}
parts := strings.Split(strings.TrimPrefix(pattern, "/"), "/")
if len(parts) < 2 || strings.IndexAny(pattern, "*?[") == 1 {
return fmt.Errorf("account root must select a content directory below a fixed top-level directory: %q", pattern)
}
if _, err := filepath.Match(pattern, ""); err != nil {
return fmt.Errorf("invalid account root pattern %q: %w", pattern, err)
}
return nil
}
// ResolveAccountRoots expands configured content directories without following
// symlinks. Unsafe directories are excluded and reported alongside valid roots,
// so a broken tenant tree does not block remediation of other accounts.
func ResolveAccountRoots(patterns []string) ([]string, error) {
var roots []string
var problems []error
for _, pattern := range patterns {
if err := ValidateAccountRootPattern(pattern); err != nil {
return nil, err
}
matches, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
for _, path := range matches {
info, err := os.Lstat(path)
if os.IsNotExist(err) {
continue
}
if err != nil {
problems = append(problems, err)
continue
}
if !info.IsDir() && info.Mode()&os.ModeSymlink == 0 {
continue
}
if err := validateAccountRootDirectory(path); err != nil {
problems = append(problems, err)
continue
}
roots = append(roots, path)
}
}
slices.Sort(roots)
return slices.Compact(roots), errors.Join(problems...)
}
func validateAccountRootDirectory(path string) error {
dir, err := safepath.OpenDirNoFollow(path)
if err != nil {
return fmt.Errorf("open account root %s without symlinks: %w", path, err)
}
return dir.Close()
}
package platform
// FTPServerInstalled reports whether the host has an FTP server that needs
// a passive port range: every cPanel host ships one, elsewhere pure-ftpd,
// proftpd or vsftpd must be installed. An installed server counts even when
// stopped, so its ports are open before it starts.
func FTPServerInstalled() bool {
if pathExists("/usr/local/cpanel/version") {
return true
}
// Plesk renames ProFTPD, and Debian's Pure-FTPd authentication packages
// install distinct server binaries instead of the vanilla executable.
return binaryInstalled([]string{
"pure-ftpd", "pure-ftpd-mysql", "pure-ftpd-postgresql", "pure-ftpd-ldap",
"proftpd", "in.proftpd", "vsftpd",
}, []string{
"/usr/sbin/pure-ftpd",
"/usr/sbin/pure-ftpd-mysql",
"/usr/sbin/pure-ftpd-postgresql",
"/usr/sbin/pure-ftpd-ldap",
"/usr/sbin/proftpd",
"/usr/sbin/in.proftpd",
"/usr/sbin/vsftpd",
})
}
package platform
import (
"context"
"os"
"os/exec"
"slices"
"time"
)
// EximMainLog is the Exim main log the daemon follows on cPanel hosts.
const EximMainLog = "/var/log/exim_mainlog"
// EximSpoolRoot is the Exim spool directory on cPanel hosts, holding input/
// and msglog/.
const EximSpoolRoot = "/var/spool/exim"
// EximLocalSkeleton is the full set of cPanel exim.conf.local section
// markers, written when a host has no exim.conf.local yet. cPanel's
// buildeximconf injects whatever follows each marker into exim.conf.
const EximLocalSkeleton = `@AUTH@
@BEGINACL@
@CONFIG@
@DIRECTOREND@
@DIRECTORMIDDLE@
@DIRECTORSTART@
@ENDACL@
@RETRYEND@
@RETRYSTART@
@REWRITE@
@ROUTEREND@
@ROUTERSTART@
@TRANSPORTEND@
@TRANSPORTMIDDLE@
@TRANSPORTSTART@
`
// MTAIdents lists local users and process basenames belonging to the
// host's Mail Transfer Agent stack. Direct SMTP egress detection uses
// this allowlist to skip legitimate local MTA traffic instead of
// path-allowlisting a directory.
type MTAIdents struct {
Users []string
SystemUsers []string // Names that hosted accounts may also claim; require a system UID.
Processes []string
}
// IsMTAUser reports whether name and uid identify a known MTA account.
// Match is exact and case-sensitive (Linux usernames are).
func (m MTAIdents) IsMTAUser(name string, uid uint32) bool {
return slices.Contains(m.Users, name) || (uid < 1000 && slices.Contains(m.SystemUsers, name))
}
// IsMTAProcess reports whether basename is one of the known MTA process
// basenames. Exact match; the caller passes comm or basename(exe), not
// a full path.
func (m MTAIdents) IsMTAProcess(basename string) bool {
for _, p := range m.Processes {
if p == basename {
return true
}
}
return false
}
// LocalMTAIdentities returns the MTA users and process basenames that
// should be considered legitimate on the detected platform. cPanel
// hosts get exim variants; non-cPanel hosts get the postfix/dovecot
// baseline plus Stalwart.
func LocalMTAIdentities(info Info) MTAIdents {
users := []string{
"mail",
"mailnull",
"postfix",
"dovecot",
"dovenull",
"mailman",
}
processes := []string{
"postfix",
"smtpd",
"smtp",
"qmgr",
"pickup",
"cleanup",
"local",
"dovecot",
"imap-login",
"pop3-login",
"lmtp",
}
var systemUsers []string
if info.IsCPanel() {
users = append(users, "exim")
processes = append(processes, "exim", "exim4")
} else {
// Hosting accounts can claim these names on non-cPanel hosts too.
systemUsers = []string{"stalwart", "stalwart-mail"}
}
return MTAIdents{Users: users, SystemUsers: systemUsers, Processes: processes}
}
// MTAKind identifies the host's Mail Transfer Agent.
type MTAKind string
const (
MTAUnknown MTAKind = ""
MTAExim MTAKind = "exim"
MTAPostfix MTAKind = "postfix"
// MTAStalwart is Stalwart mail server, which writes its own log and is
// detected only by its running service.
MTAStalwart MTAKind = "stalwart"
)
var (
mtaLookPath = exec.LookPath
mtaStat = os.Stat
mtaServiceActive = systemdMTAServiceActive
mtaServiceEnabled = systemdMTAServiceEnabled
)
// DetectMTA reports the delivery agent rather than whichever package happens
// to leave a binary on disk. cPanel is authoritative because it runs Exim
// while retaining Postfix binaries; elsewhere an active service wins.
func DetectMTA() MTAKind {
if pathExists("/usr/local/cpanel/version") {
return MTAExim
}
eximActive := mtaServiceActive("exim") || mtaServiceActive("exim4")
postfixActive := mtaServiceActive("postfix")
// When both are active, keep Exim-specific weaknesses visible instead of
// dropping the checks behind an ambiguous result.
if eximActive {
return MTAExim
}
if postfixActive {
return MTAPostfix
}
if mtaServiceActive("stalwart") || mtaServiceActive("stalwart-mail") {
return MTAStalwart
}
// CSM can start while Stalwart is still starting or restarting; an
// enabled unit still names the mail server once Exim and Postfix are
// not running.
if mtaServiceEnabled("stalwart") || mtaServiceEnabled("stalwart-mail") {
return MTAStalwart
}
eximInstalled := binaryInstalled([]string{"exim", "exim4"}, []string{
"/usr/sbin/exim",
"/usr/sbin/exim4",
})
postfixInstalled := binaryInstalled([]string{"postfix"}, []string{
"/usr/sbin/postfix",
"/usr/libexec/postfix/master",
"/usr/lib/postfix/sbin/master",
})
if eximInstalled == postfixInstalled {
return MTAUnknown
}
if eximInstalled {
return MTAExim
}
return MTAPostfix
}
// ServiceActive reports whether the systemd unit is active.
func ServiceActive(unit string) bool { return mtaServiceActive(unit) }
func pathExists(path string) bool {
_, err := mtaStat(path)
return err == nil
}
func systemdMTAServiceActive(unit string) bool {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
// #nosec G204 -- callers pass only the literal MTA unit names above.
return exec.CommandContext(ctx, "systemctl", "is-active", "--quiet", unit).Run() == nil
}
func systemdMTAServiceEnabled(unit string) bool {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
// #nosec G204 -- callers pass only the literal MTA unit names above.
return exec.CommandContext(ctx, "systemctl", "is-enabled", "--quiet", unit).Run() == nil
}
// binaryInstalled reports whether any of the named binaries is on PATH or at
// one of the absolute locations. The daemon runs with a minimal PATH, so
// the absolute probes carry most installs.
func binaryInstalled(binaries, paths []string) bool {
for _, b := range binaries {
if _, err := mtaLookPath(b); err == nil {
return true
}
}
for _, p := range paths {
info, err := mtaStat(p)
if err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0 {
return true
}
}
return false
}
// Package platform detects the host OS, control panel, and web server so
// CSM checks can pick the right config/log paths instead of hardcoding
// cPanel+Apache layouts.
package platform
import (
"bufio"
"os"
"os/exec"
"slices"
"strings"
"sync"
"sync/atomic"
)
type OSFamily string
const (
OSUnknown OSFamily = ""
OSUbuntu OSFamily = "ubuntu"
OSDebian OSFamily = "debian"
OSAlma OSFamily = "almalinux"
OSRocky OSFamily = "rocky"
OSCentOS OSFamily = "centos"
OSRHEL OSFamily = "rhel"
OSCloudLinux OSFamily = "cloudlinux"
)
type Panel string
const (
PanelNone Panel = ""
PanelCPanel Panel = "cpanel"
PanelPlesk Panel = "plesk"
PanelDA Panel = "directadmin"
)
type WebServer string
const (
WSNone WebServer = ""
WSApache WebServer = "apache"
WSNginx WebServer = "nginx"
WSLiteSpeed WebServer = "litespeed"
)
// Info holds everything a check needs to locate web server resources.
type Info struct {
OS OSFamily
OSVersion string
Panel Panel
WebServer WebServer
// Config locations for the detected web server.
ApacheConfigDir string // e.g. /etc/apache2 or /etc/httpd
NginxConfigDir string // e.g. /etc/nginx
// Candidate log files. Populated based on detected web server + OS.
AccessLogPaths []string
ErrorLogPaths []string
ModSecAuditLogPaths []string
// DomlogGlobs is the list of glob patterns used to enumerate
// per-vhost access logs. Populated per panel + OS in populatePaths.
DomlogGlobs []string
// Binary paths useful for reload/control.
ApacheBinary string
NginxBinary string
LiteSpeedBinary string
}
// IsCPanel is a convenience for checks that still need to gate cPanel-only
// behavior (WHM API calls, /home/*/public_html enumeration, exim log
// tailing, etc.) without re-detecting each time.
func (i Info) IsCPanel() bool { return i.Panel == PanelCPanel }
// LiteSpeedConfigPath is LiteSpeed's own server configuration.
const LiteSpeedConfigPath = "/usr/local/lsws/conf/httpd_config.xml"
// NginxConfigRoot returns the nginx configuration root: the detected or
// configured NginxConfigDir, else the packaged default.
func (i Info) NginxConfigRoot() string {
if i.NginxConfigDir != "" {
return i.NginxConfigDir
}
return "/etc/nginx"
}
// ApacheCompatibleConfigDir returns the Apache-style config root consumed by
// the active web server. cPanel LiteSpeed reads the EA4 Apache tree even when
// Apache itself is not the detected server, and older detection snapshots may
// therefore have no ApacheConfigDir populated.
func (i Info) ApacheCompatibleConfigDir() string {
if i.ApacheConfigDir != "" {
return i.ApacheConfigDir
}
if i.IsCPanel() {
return "/etc/apache2"
}
return ""
}
// IsRHELFamily reports whether the OS uses rpm/dnf and /etc/httpd style paths.
func (i Info) IsRHELFamily() bool {
switch i.OS {
case OSAlma, OSRocky, OSCentOS, OSRHEL, OSCloudLinux:
return true
}
return false
}
// IsDebianFamily reports whether the OS uses dpkg/apt and /etc/apache2 style paths.
func (i Info) IsDebianFamily() bool {
return i.OS == OSUbuntu || i.OS == OSDebian
}
// CronSpoolDir returns the directory holding per-user crontabs: cronie
// (RHEL family, cPanel) writes them directly under /var/spool/cron, while
// Debian's cron keeps them in /var/spool/cron/crontabs.
func (i Info) CronSpoolDir() string {
if i.IsDebianFamily() {
return "/var/spool/cron/crontabs"
}
return "/var/spool/cron"
}
// AccountHomeRoots returns the directories whose children are hosting
// accounts: /home on cPanel, DirectAdmin and plain hosts, /var/www/vhosts
// on Plesk. Every account-scoped check, remediation and re-check resolves
// "<root>/<account>" through this instead of assuming /home.
func (i Info) AccountHomeRoots() []string {
if i.Panel == PanelPlesk {
return []string{"/var/www/vhosts"}
}
return []string{"/home"}
}
// PanelToolRoots returns the directories holding the control panel's own
// root-run tooling. cPanel keeps its maintenance scripts and binaries under
// /usr/local/cpanel; a panel-less host has no such tree. Provenance checks
// use this to tell the panel's own nightly work from a process that merely
// named itself after it, so the path has to come from detection rather than
// being written into the detector.
func (i Info) PanelToolRoots() []string {
if i.Panel == PanelCPanel {
return []string{"/usr/local/cpanel"}
}
return nil
}
// WebServerUsers returns the account(s) the web server serves requests as
// on this platform: cPanel's nobody, DirectAdmin's apache (plus nobody for
// its suEXEC fallback), Plesk's distribution web user, and for a panel-less
// host the distribution user of the detected server. Checks that audit
// "the web server's" crontab or group ownership must use this instead of
// assuming cPanel's nobody.
func (i Info) WebServerUsers() []string {
switch i.Panel {
case PanelCPanel:
return []string{"nobody"}
case PanelDA:
return []string{"apache", "nobody"}
}
if i.WebServer == WSLiteSpeed {
return []string{"nobody"}
}
if i.IsDebianFamily() {
return []string{"www-data"}
}
if i.WebServer == WSNginx {
return []string{"nginx"}
}
return []string{"apache"}
}
// MailLogPath returns the platform-default mail log file. Empty string
// means "no file source available" (operator must use journal).
func (i Info) MailLogPath() string {
if i.IsDebianFamily() {
return "/var/log/mail.log"
}
return "/var/log/maillog"
}
// AuthLogPath returns the platform-default SSH/PAM auth log file.
// Debian-family uses /var/log/auth.log; RHEL-family uses /var/log/secure.
func (i Info) AuthLogPath() string {
if i.IsDebianFamily() {
return "/var/log/auth.log"
}
return "/var/log/secure"
}
// Overrides lets the operator override auto-detected values from csm.yaml.
// Any field left blank or nil falls back to the auto-detected value.
//
// Panel and WebServer use pointer types so callers can distinguish "leave
// auto-detected" (nil) from "explicitly override to none" (pointer to
// PanelNone / WSNone). The non-pointer string/slice fields use the
// zero-value-means-unset convention since they have no legitimate "none"
// value to override to.
type Overrides struct {
Panel *Panel
WebServer *WebServer
AccessLogPaths []string
ErrorLogPaths []string
ModSecAuditLogPaths []string
DomlogGlobs []string
ApacheConfigDir string
NginxConfigDir string
}
var (
// detectedPtr caches the merged Detect() result. Stored behind an atomic
// pointer (not a sync.Once + plain field) so Refresh can replace it after
// the daemon has already probed once -- a host that mis-detected its web
// server at boot (lsws not yet running) self-heals on a later refresh.
detectedPtr atomic.Pointer[Info]
detectMu sync.Mutex // serialises first Detect/Refresh with SetOverrides
overrideMu sync.Mutex
pendingOverride *Overrides
)
// SetOverrides installs config-supplied overrides to be merged into the next
// (and all subsequent) Detect() result. Call this once from daemon startup,
// BEFORE the first Detect() call, so the merged info is what every check
// sees. Subsequent SetOverrides calls before Detect() replace the previous
// override; calls after Detect() are no-ops and log a warning via the
// returned bool.
//
// Returns true if the override was installed, false if Detect() had already
// cached an un-overridden result.
func SetOverrides(o Overrides) bool {
detectMu.Lock()
defer detectMu.Unlock()
overrideMu.Lock()
defer overrideMu.Unlock()
// sync.Once has no public "was called" query, so detectedFlag is the
// guard that makes late override attempts a true no-op for both Detect
// and fresh re-probes. Re-installing the overrides that are already in
// force is not late: several startup paths install and then detect, and
// the daemon's own call must not read "already detected" as "lost".
if isDetected() {
return pendingOverride != nil && overridesEqual(*pendingOverride, o)
}
pendingOverride = &o
return true
}
func overridesEqual(a, b Overrides) bool {
return optionalEqual(a.Panel, b.Panel) &&
optionalEqual(a.WebServer, b.WebServer) &&
slices.Equal(a.AccessLogPaths, b.AccessLogPaths) &&
slices.Equal(a.ErrorLogPaths, b.ErrorLogPaths) &&
slices.Equal(a.ModSecAuditLogPaths, b.ModSecAuditLogPaths) &&
slices.Equal(a.DomlogGlobs, b.DomlogGlobs) &&
a.ApacheConfigDir == b.ApacheConfigDir &&
a.NginxConfigDir == b.NginxConfigDir
}
func optionalEqual[T comparable](a, b *T) bool {
if a == nil || b == nil {
return a == nil && b == nil
}
return *a == *b
}
// isDetected returns true if Detect() has already cached a result.
// Internal helper — uses a separate flag because sync.Once has no query API.
var detectedFlag atomic.Bool
func isDetected() bool { return detectedFlag.Load() }
// Detect inspects the host and returns platform info. The result is cached
// until the next Refresh — callers that need a fresh probe without touching
// the cache should use DetectFresh instead.
func Detect() Info {
if p := detectedPtr.Load(); p != nil {
return *p
}
detectMu.Lock()
defer detectMu.Unlock()
// Double-check: another goroutine may have populated the cache while we
// waited for the lock.
if p := detectedPtr.Load(); p != nil {
return *p
}
i := DetectFreshWithOverrides()
detectedPtr.Store(&i)
detectedFlag.Store(true)
return i
}
// Refresh re-runs detection (applying any operator overrides) and replaces the
// cached Detect() result. Intended for a periodic caller so a detection that
// was wrong at boot -- for example a cPanel+LiteSpeed host probed before lsws
// finished starting, which pins Apache for everything -- self-heals instead of
// staying wrong for the daemon's lifetime. An operator web_server.type pin
// still wins, exactly as it does through Detect(), because
// DetectFreshWithOverrides re-applies the pending override.
func Refresh() Info {
detectMu.Lock()
defer detectMu.Unlock()
i := DetectFreshWithOverrides()
detectedPtr.Store(&i)
detectedFlag.Store(true)
return i
}
// DetectFresh always re-runs detection, ignoring any cached result.
// Intended for tests and for operator-triggered rescan. Does not apply
// config overrides — use Detect() for the operator-visible view.
func DetectFresh() Info {
i := Info{}
detectOS(&i)
detectPanel(&i)
detectWebServer(&i)
populatePaths(&i)
return i
}
// DetectFreshWithOverrides re-runs detection (ignoring the one-shot Detect()
// cache) and merges in any operator-supplied overrides, returning the same
// view Detect() would produce but without the process-lifetime cache.
//
// Periodic re-probes (the ModSec rule-action registry refresh) use this so a
// detection that was wrong at boot -- for example a LiteSpeed host probed
// before lsws finished starting, which resolves to the wrong rule
// directories and yields an empty registry -- self-heals on the next refresh
// instead of staying wrong for the daemon's lifetime. A configured
// web_server.type override still wins, exactly as it does through Detect().
func DetectFreshWithOverrides() Info {
i := DetectFresh()
var o Overrides
hasOverride := false
overrideMu.Lock()
if pendingOverride != nil {
o = *pendingOverride
hasOverride = true
}
overrideMu.Unlock()
if hasOverride {
i = applyOverrides(i, o)
}
return i
}
// applyOverrides merges non-empty override fields into info. Always returns
// a new Info — never mutates the input. Paths are replaced, not appended:
// if the operator configured an explicit access-log list, the auto-detected
// list is discarded so operators have full control.
func applyOverrides(info Info, o Overrides) Info {
// Panel override must happen before path rebuild so populatePaths
// picks up the cPanel overlay (or drops it) correctly. Nil means
// "leave auto-detected"; a non-nil pointer always wins, even when it
// points at PanelNone, so operators can explicitly force a host to
// look panel-less.
if o.Panel != nil {
info.Panel = *o.Panel
}
if o.WebServer != nil {
// Web server type changed -- rebuild paths from scratch unless the
// operator also supplied path overrides below. Same nil-vs-pointer
// semantics as Panel: a pointer at WSNone forces "no web server"
// instead of being silently ignored.
info.WebServer = *o.WebServer
info.AccessLogPaths = nil
info.ErrorLogPaths = nil
info.ModSecAuditLogPaths = nil
populatePaths(&info)
}
// When Panel or WebServer changed, recompute DomlogGlobs so the globs
// stay consistent with the new panel/web-server combination. Explicit
// DomlogGlobs override below wins over this recompute.
if o.Panel != nil || o.WebServer != nil {
info.DomlogGlobs = nil
populateDomlogGlobs(&info)
}
if len(o.AccessLogPaths) > 0 {
info.AccessLogPaths = append([]string(nil), o.AccessLogPaths...)
}
if len(o.ErrorLogPaths) > 0 {
info.ErrorLogPaths = append([]string(nil), o.ErrorLogPaths...)
}
if len(o.ModSecAuditLogPaths) > 0 {
info.ModSecAuditLogPaths = append([]string(nil), o.ModSecAuditLogPaths...)
}
if len(o.DomlogGlobs) > 0 {
info.DomlogGlobs = append([]string(nil), o.DomlogGlobs...)
}
if o.ApacheConfigDir != "" {
info.ApacheConfigDir = o.ApacheConfigDir
}
if o.NginxConfigDir != "" {
info.NginxConfigDir = o.NginxConfigDir
}
return info
}
// ResetForTest clears the cached Detect() result so tests can re-run with
// different fixtures. Never call from production code.
func ResetForTest() {
detectMu.Lock()
detectedPtr.Store(nil)
detectedFlag.Store(false)
detectMu.Unlock()
overrideMu.Lock()
pendingOverride = nil
overrideMu.Unlock()
}
func detectOS(i *Info) {
f, err := os.Open("/etc/os-release")
if err != nil {
return
}
defer func() { _ = f.Close() }()
scanner := bufio.NewScanner(f)
var id, versionID string
for scanner.Scan() {
line := scanner.Text()
key, val, ok := strings.Cut(line, "=")
if !ok {
continue
}
val = strings.Trim(val, `"'`)
switch key {
case "ID":
id = strings.ToLower(val)
case "VERSION_ID":
versionID = val
}
}
i.OSVersion = versionID
switch id {
case "ubuntu":
i.OS = OSUbuntu
case "debian":
i.OS = OSDebian
case "almalinux":
i.OS = OSAlma
case "rocky":
i.OS = OSRocky
case "centos":
i.OS = OSCentOS
case "rhel":
i.OS = OSRHEL
case "cloudlinux":
i.OS = OSCloudLinux
}
}
func detectPanel(i *Info) {
if _, err := os.Stat("/usr/local/cpanel/version"); err == nil {
i.Panel = PanelCPanel
return
}
if _, err := os.Stat("/usr/local/psa/version"); err == nil {
i.Panel = PanelPlesk
return
}
if _, err := os.Stat("/usr/local/directadmin/directadmin"); err == nil {
i.Panel = PanelDA
return
}
}
func detectWebServer(i *Info) {
// Prefer the process that's actually running. Fall back to installed
// binaries if nothing is running yet (first boot, non-systemd env).
running := runningServices()
// Always record binary paths for reload/control, even if not primary.
if bin, err := exec.LookPath("nginx"); err == nil {
i.NginxBinary = bin
}
if bin, err := exec.LookPath("apache2"); err == nil {
i.ApacheBinary = bin
} else if bin, err := exec.LookPath("httpd"); err == nil {
i.ApacheBinary = bin
}
// cPanel compiles its own httpd under /usr/local/apache/bin/httpd,
// which isn't always in PATH for root under the CSM service unit.
if i.ApacheBinary == "" {
const cpHttpd = "/usr/local/apache/bin/httpd"
if _, err := os.Stat(cpHttpd); err == nil {
i.ApacheBinary = cpHttpd
}
}
// LiteSpeed ships lshttpd/litespeed under /usr/local/lsws/bin and is not
// in PATH under the service unit. A host that serves via LiteSpeed has
// this binary on disk even before the lsws service is up, so probing for
// it lets detection correct the boot-order misdetect below.
for _, p := range []string{"/usr/local/lsws/bin/lshttpd", "/usr/local/lsws/bin/litespeed"} {
if _, err := os.Stat(p); err == nil {
i.LiteSpeedBinary = p
break
}
}
if i.LiteSpeedBinary == "" {
if bin, err := exec.LookPath("lshttpd"); err == nil {
i.LiteSpeedBinary = bin
} else if bin, err := exec.LookPath("litespeed"); err == nil {
i.LiteSpeedBinary = bin
}
}
i.WebServer = selectWebServer(i.Panel, running, i.ApacheBinary != "", i.NginxBinary != "")
i.WebServer = liteSpeedBinaryFallback(i.WebServer, running, i.LiteSpeedBinary != "")
}
// liteSpeedBinaryFallback corrects the historical boot-order misdetect: when
// no web server process is running yet, selectWebServer falls back to a
// binary-only guess and (on cPanel, which always keeps an httpd binary) picks
// Apache. A LiteSpeed binary present on disk is the strongest signal that this
// host serves through LiteSpeed, so it outranks that binary-only fallback.
//
// It only overrides when nothing is actually running: a live Apache/Nginx
// process is trusted over an installed-but-idle lsws binary.
func liteSpeedBinaryFallback(ws WebServer, running map[string]bool, hasLiteSpeedBinary bool) WebServer {
if !hasLiteSpeedBinary || anyWebServerRunning(running) {
return ws
}
switch ws {
case WSApache, WSNginx, WSNone:
return WSLiteSpeed
default:
return ws
}
}
// anyWebServerRunning reports whether any known web server unit is active, so
// the binary-only fallbacks only fire when nothing is genuinely serving.
func anyWebServerRunning(running map[string]bool) bool {
for _, up := range running {
if up {
return true
}
}
return false
}
// runningServices returns which web server process units are currently
// active. Uses systemctl when available; falls back to checking /proc.
func runningServices() map[string]bool {
active := map[string]bool{}
for _, unit := range []string{"nginx", "apache2", "httpd", "litespeed", "lshttpd", "lsws"} {
// #nosec G204 -- systemctl hardcoded; unit iterates a literal slice.
cmd := exec.Command("systemctl", "is-active", "--quiet", unit)
if err := cmd.Run(); err == nil {
active[unit] = true
}
}
return active
}
func selectWebServer(panel Panel, running map[string]bool, hasApacheBinary, hasNginxBinary bool) WebServer {
apacheRunning := running["apache2"] || running["httpd"]
litespeedRunning := running["litespeed"] || running["lshttpd"] || running["lsws"]
nginxRunning := running["nginx"]
// cPanel commonly runs Nginx as a reverse proxy in front of Apache.
// Prefer the origin server logs when Apache is active so real-time
// access and ModSecurity watchers tail the paths cPanel actually writes.
if panel == PanelCPanel {
switch {
case litespeedRunning:
return WSLiteSpeed
case apacheRunning:
return WSApache
case nginxRunning:
return WSNginx
case hasApacheBinary:
return WSApache
case hasNginxBinary:
return WSNginx
default:
return WSNone
}
}
switch {
case nginxRunning:
return WSNginx
case apacheRunning:
return WSApache
case litespeedRunning:
return WSLiteSpeed
case hasApacheBinary:
return WSApache
case hasNginxBinary:
return WSNginx
default:
return WSNone
}
}
func populatePaths(i *Info) {
// Apache config dir. cPanel compiles Apache from source and installs
// under /usr/local/apache, separate from the OS package tree; that
// override wins over the distro default when cPanel is present.
switch {
case i.Panel == PanelCPanel:
i.ApacheConfigDir = "/etc/apache2"
if dirExists("/usr/local/apache/conf") {
i.ApacheConfigDir = "/usr/local/apache/conf"
}
case i.IsDebianFamily():
if dirExists("/etc/apache2") {
i.ApacheConfigDir = "/etc/apache2"
}
case i.IsRHELFamily():
if dirExists("/etc/httpd") {
i.ApacheConfigDir = "/etc/httpd"
}
}
if dirExists("/etc/nginx") {
i.NginxConfigDir = "/etc/nginx"
}
// Log paths: pick candidates based on detected web server and OS layout.
// We include ALL plausible locations so log watchers can try each;
// missing paths are handled upstream by the retry logic.
switch i.WebServer {
case WSApache:
if i.IsDebianFamily() {
i.AccessLogPaths = []string{"/var/log/apache2/access.log", "/var/log/apache2/other_vhosts_access.log"}
i.ErrorLogPaths = []string{"/var/log/apache2/error.log"}
i.ModSecAuditLogPaths = []string{"/var/log/apache2/modsec_audit.log"}
} else {
i.AccessLogPaths = []string{"/var/log/httpd/access_log"}
i.ErrorLogPaths = []string{"/var/log/httpd/error_log"}
i.ModSecAuditLogPaths = []string{"/var/log/httpd/modsec_audit.log"}
}
case WSNginx:
i.AccessLogPaths = []string{"/var/log/nginx/access.log"}
i.ErrorLogPaths = []string{"/var/log/nginx/error.log"}
i.ModSecAuditLogPaths = []string{"/var/log/nginx/modsec_audit.log"}
case WSLiteSpeed:
i.AccessLogPaths = []string{"/usr/local/lsws/logs/access.log"}
i.ErrorLogPaths = []string{"/usr/local/lsws/logs/error.log"}
i.ModSecAuditLogPaths = []string{"/usr/local/lsws/logs/auditmodsec.log"}
}
// cPanel overlays its own access/error logs on top of the OS defaults.
// The access logs are where EasyApache writes, and LiteSpeed on cPanel
// writes the same file. cpsrvd's /usr/local/cpanel/logs/access_log is not
// a web access log: panel and webmail traffic parsed as web traffic turns
// a customer's own session into attack findings against their address.
if i.Panel == PanelCPanel {
i.AccessLogPaths = append([]string{
"/usr/local/apache/logs/access_log",
"/var/log/apache2/access_log",
"/etc/apache2/logs/access_log",
}, i.AccessLogPaths...)
i.ErrorLogPaths = append([]string{
"/usr/local/apache/logs/error_log",
}, i.ErrorLogPaths...)
i.ModSecAuditLogPaths = append([]string{
"/usr/local/apache/logs/modsec_audit.log",
"/var/log/modsec_audit.log",
}, i.ModSecAuditLogPaths...)
}
populateDomlogGlobs(i)
}
// populateDomlogGlobs sets DomlogGlobs based on panel type and, for
// bare-metal installs, the web server + OS family. Panel takes
// precedence over web server because panel-specific layouts write
// per-vhost logs to panel-owned directories regardless of what web
// server is running underneath.
func populateDomlogGlobs(i *Info) {
switch i.Panel {
case PanelCPanel:
i.DomlogGlobs = []string{
"/home/*/access-logs/*-ssl_log",
"/home/*/access-logs/*_log",
}
case PanelPlesk:
i.DomlogGlobs = []string{
"/var/www/vhosts/*/logs/access_ssl_log",
"/var/www/vhosts/*/logs/access_log",
"/var/www/vhosts/*/logs/proxy_access_ssl_log",
}
case PanelDA:
i.DomlogGlobs = []string{"/var/log/httpd/domains/*.log"}
default:
switch i.WebServer {
case WSApache:
if i.IsDebianFamily() {
i.DomlogGlobs = []string{
"/var/log/apache2/*-access.log",
"/var/log/apache2/*_access.log",
}
} else if i.IsRHELFamily() {
i.DomlogGlobs = []string{
"/var/log/httpd/*-access_log",
"/var/log/httpd/*_access_log",
}
}
case WSNginx:
i.DomlogGlobs = []string{
"/var/log/nginx/*.access.log",
"/var/log/nginx/*-access.log",
}
}
}
}
func dirExists(p string) bool {
fi, err := os.Stat(p)
return err == nil && fi.IsDir()
}
// Package privops is the inventory of every CSM operation that needs privilege
// beyond reading its own files, or that writes outside CSM's own directories.
//
// It exists so an operator can answer three questions before granting a
// security daemon host-level access: what does it touch, what does each of
// those need root (or a capability) for, and how do I turn any of it off.
// `csm privileges` renders it, docs/src/capability-matrix.md ships the
// rendered form, and gates in internal/ci keep both in step with the systemd
// sandbox that actually constrains the daemon.
package privops
import (
"encoding/json"
"fmt"
"path"
"slices"
"sort"
"strings"
)
// Privilege is a capability or credential an operation needs.
type Privilege string
const (
// Root means the operation needs uid 0 rather than one capability: it
// reads or writes files owned by many different accounts, or it drives a
// panel tool that assumes root.
Root Privilege = "root"
// CapDACReadSearch is the read side of scanning every account's files.
CapDACReadSearch Privilege = "CAP_DAC_READ_SEARCH"
// CapSysAdmin covers fanotify and mount inspection.
CapSysAdmin Privilege = "CAP_SYS_ADMIN"
// CapNetAdmin covers nftables mutation.
CapNetAdmin Privilege = "CAP_NET_ADMIN"
// CapKill covers signalling processes owned by other accounts.
CapKill Privilege = "CAP_KILL"
// CapBPF covers loading and attaching BPF programs.
CapBPF Privilege = "CAP_BPF"
// CapPerfmon is required alongside CAP_BPF for tracing and LSM programs.
CapPerfmon Privilege = "CAP_PERFMON"
// CapSyslog permits reading a restricted kernel message buffer.
CapSyslog Privilege = "CAP_SYSLOG"
// CapAuditControl permits querying and loading kernel audit rules.
CapAuditControl Privilege = "CAP_AUDIT_CONTROL"
// CapSysModule permits loading or unloading kernel modules.
CapSysModule Privilege = "CAP_SYS_MODULE"
// CapLinuxImmutable permits changing the executable's immutable flag.
CapLinuxImmutable Privilege = "CAP_LINUX_IMMUTABLE"
// CapLease permits exclusive file leases across account-owned files.
CapLease Privilege = "CAP_LEASE"
// Unprivileged marks work CSM does inside its own directories.
Unprivileged Privilege = "none"
)
// KnownPrivileges lists every privilege an operation may declare.
func KnownPrivileges() []Privilege {
return []Privilege{Root, CapDACReadSearch, CapSysAdmin, CapNetAdmin, CapKill, CapBPF, CapPerfmon, CapSyslog, CapAuditControl, CapSysModule, CapLinuxImmutable, CapLease, Unprivileged}
}
// Trigger says who starts an operation.
type Trigger string
const (
// Automatic operations run on the daemon's own schedule or in response to
// a detection, with no operator in the loop.
Automatic Trigger = "automatic"
// Operator operations run only when someone issues a command or clicks a
// button. Not running the command is how they are turned off.
Operator Trigger = "operator"
)
// RiskTier is an operation's action-risk tier: what can go wrong if it runs
// on a wrong target. The zero value is unclassified and fails
// TestEveryOperationHasARiskTier.
type RiskTier uint8
const (
RiskUnclassified RiskTier = iota
// RiskObserve (tier 0) changes nothing outside CSM's own trees.
RiskObserve
// RiskPreview (tier 1) records a recommendation or dry-run decision only.
// No inventory row currently represents previews separately; rows carry
// their maximum live effect, even when an execution can be a dry run.
RiskPreview
// RiskReversible (tier 2) makes a low-risk host change such as attaching
// a probe, opening a challenge gate or holding mail. The tier alone
// does not promise automatic rollback or reversal of incidental writes.
RiskReversible
// RiskContain (tier 3) quarantines, blocks or denies one target.
RiskContain
// RiskDestructive (tier 4) signals processes, restarts or reloads services,
// or rewrites content or configuration, including an existing archive.
RiskDestructive
)
// Number is the tier as the safety model numbers it, 0 to 4, or -1 when the
// operation is unclassified or invalid.
func (r RiskTier) Number() int {
if r < RiskObserve || r > RiskDestructive {
return -1
}
return int(r) - 1
}
// MarshalJSON uses the same public tier number as the text and Markdown
// views. The internal zero value is an unclassified sentinel, not tier 0.
func (r RiskTier) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprint(r.Number())), nil
}
// UnmarshalJSON translates public tier numbers back to their internal values.
// JSON null leaves the destination unchanged, as it does for other scalars.
func (r *RiskTier) UnmarshalJSON(data []byte) error {
var number *int
if err := json.Unmarshal(data, &number); err != nil {
return err
}
if number == nil {
return nil
}
if *number < -1 || *number > RiskDestructive.Number() {
return fmt.Errorf("invalid risk tier %d", *number)
}
*r = RiskTier(*number + 1) // #nosec G115 -- public tiers -1 through 4 map to 0 through 5.
return nil
}
// SafetyContract describes current authority, identity, recovery and limits.
// It is inventory metadata, not an enforcement mechanism or a claim that
// the full action lifecycle is implemented. Remaining gaps stay explicit.
type SafetyContract struct {
// Authority is the evidence and opt-ins required before it may run.
Authority string
// Identity is how the target is revalidated immediately before the change.
Identity string
// Recovery is how the change is reversed, or what it cannot undo.
Recovery string
// Limit names current bounds and where they do not apply.
Limit string
}
// csmOwnedPrefixes are the trees CSM creates and manages for itself. Writing
// inside them is not a host change: an operator who removes CSM removes them.
var csmOwnedPrefixes = []string{
"/var/lib/csm",
"/opt/csm",
"/var/log/csm",
"/var/log/csm-php-shield",
"/etc/csm",
"/var/cache/csm",
"/var/run/csm",
}
// Op is one privileged or state-changing operation.
type Op struct {
// ID is stable and namespaced as <subsystem>.<action>. Operators and
// panel integrations may key on it.
ID string
// Subsystem groups related operations in the rendered matrix.
Subsystem string
// Summary is one line: what the operation does.
Summary string
// Privileges is what the operation needs from the kernel or from uid 0.
Privileges []Privilege
// Trigger says whether the daemon starts this on its own.
Trigger Trigger
// Writes lists what the operation writes while it runs, including writes
// made by a tool it invokes. Filesystem paths start with "/"; anything
// else is a resource written as <kind>:<name>. Empty means read-only.
Writes []string
// Unsandboxed marks operations outside the daemon's systemd sandbox:
// transient services or standalone CLI commands. Mixed operations must
// have separate rows for their in-daemon writes.
Unsandboxed bool
// DisableKey is the config key that stops the operation, and DisableValue
// the YAML value to give it. An empty key makes no claim of a config switch.
DisableKey string
DisableValue string
// DisableReason explains why an automatic operation cannot be stopped
// through config. It must not invent a switch that only stops some callers.
DisableReason string
// Audited reports whether the operation writes a record to the action
// log. False is not a claim that the operation is silent, only that it is
// not yet on that stream; the daemon log still carries it.
Audited bool
// Risk is the operation's action-risk tier.
Risk RiskTier
// Contract is the operation's safety contract; nil until that contract has
// been specified for the operation.
Contract *SafetyContract
// RecoveryGap names recovery work not covered by this inventory's
// contracts. Required for host-changing operations without a contract.
RecoveryGap string
// WithoutPrivilege says what an operator loses by withholding the
// privilege, so the matrix reads as a decision, not a demand.
WithoutPrivilege string
}
// ChangesHost reports whether the operation writes outside CSM's own trees.
func (o Op) ChangesHost() bool {
for _, w := range o.Writes {
if !strings.HasPrefix(w, "/") {
return true
}
if !csmOwned(w) {
return true
}
}
return false
}
func csmOwned(name string) bool {
name = path.Clean(name)
for _, prefix := range csmOwnedPrefixes {
if name == prefix || strings.HasPrefix(name, prefix+"/") {
return true
}
}
return false
}
// Operations returns the inventory, ordered by subsystem then ID.
func Operations() []Op {
ops := append([]Op(nil), operations...)
for i := range ops {
ops[i].Privileges = slices.Clone(ops[i].Privileges)
ops[i].Writes = slices.Clone(ops[i].Writes)
if ops[i].Contract != nil {
c := *ops[i].Contract
ops[i].Contract = &c
}
}
sort.Slice(ops, func(i, j int) bool {
if ops[i].Subsystem != ops[j].Subsystem {
return ops[i].Subsystem < ops[j].Subsystem
}
return ops[i].ID < ops[j].ID
})
return ops
}
// DisableInstruction is shared by terminal and documentation output.
func (o Op) DisableInstruction() string {
if o.DisableKey != "" {
return o.DisableKey + ": " + o.DisableValue
}
if o.Trigger == Operator {
return "do not run the command"
}
if o.DisableReason != "" {
return "not configurable: " + o.DisableReason
}
return "not configurable"
}
// Markdown renders the inventory as the table shipped in the docs.
func Markdown() string {
var b strings.Builder
b.WriteString("| Operation | Needs | Trigger | Risk tier | Writes | Turn it off | Action record | Without the privilege |\n")
b.WriteString("| --- | --- | --- | --- | --- | --- | --- | --- |\n")
for _, op := range Operations() {
privs := make([]string, 0, len(op.Privileges))
for _, p := range op.Privileges {
privs = append(privs, string(p))
}
writes := "nothing (read-only)"
if len(op.Writes) > 0 {
writes = strings.Join(op.Writes, ", ")
}
if op.Unsandboxed {
writes += " (outside the systemd sandbox)"
}
off := op.DisableInstruction()
if op.DisableKey != "" {
off = "`" + off + "`"
}
audited := "no"
if op.Audited {
audited = "yes"
}
fmt.Fprintf(&b, "| `%s`<br>%s | %s | %s | %d | %s | %s | %s | %s |\n",
op.ID, op.Summary, strings.Join(privs, ", "), op.Trigger, op.Risk.Number(), writes, off, audited, op.WithoutPrivilege)
}
return b.String()
}
package processctx
import (
"container/list"
"sync"
"sync/atomic"
"time"
)
// Cache is a bounded LRU cache of processEntry keyed by PID with a per-entry
// TTL. Reads and writes are safe for concurrent use. now is a seam for
// deterministic testing.
type Cache struct {
mu sync.Mutex
cap int
ttl time.Duration
ll *list.List // front = most-recently-used
index map[int]*list.Element // pid -> element holding *processEntry
now func() time.Time
evictions atomic.Uint64
ttlPurges atomic.Uint64
misses atomic.Uint64
}
// Stats is a snapshot of cache counters. Safe to call concurrently.
type Stats struct {
Entries int
Evictions uint64 // LRU evictions (cap exceeded)
TTLPurges uint64 // entries dropped because ttl expired on Get
Misses uint64 // Get returned no entry (includes ttl purges)
}
// NewCache returns a cache with the given hard cap and TTL.
func NewCache(cap int, ttl time.Duration) *Cache {
if cap <= 0 {
cap = 1
}
return &Cache{
cap: cap,
ttl: ttl,
ll: list.New(),
index: make(map[int]*list.Element, cap),
now: time.Now,
}
}
// Put inserts or updates an entry. lastTouch is set to now.
func (c *Cache) Put(e processEntry) {
c.mu.Lock()
defer c.mu.Unlock()
e.lastTouch = c.now()
if el, ok := c.index[e.PID]; ok {
el.Value = &e
c.ll.MoveToFront(el)
return
}
el := c.ll.PushFront(&e)
c.index[e.PID] = el
for c.ll.Len() > c.cap {
c.evictOldestLocked()
}
}
// Get returns the entry for pid if present and not TTL-expired. Touching
// an entry promotes it in LRU order.
func (c *Cache) Get(pid int) (processEntry, bool) {
c.mu.Lock()
defer c.mu.Unlock()
el, ok := c.index[pid]
if !ok {
c.misses.Add(1)
return processEntry{}, false
}
entry := el.Value.(*processEntry)
if c.ttl > 0 && c.now().Sub(entry.lastTouch) > c.ttl {
c.removeLocked(el)
c.ttlPurges.Add(1)
c.misses.Add(1)
return processEntry{}, false
}
entry.lastTouch = c.now()
c.ll.MoveToFront(el)
return *entry, true
}
// Len returns the number of live entries (without forcing TTL purge).
func (c *Cache) Len() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.ll.Len()
}
// Stats returns a counter snapshot.
func (c *Cache) Stats() Stats {
c.mu.Lock()
n := c.ll.Len()
c.mu.Unlock()
return Stats{
Entries: n,
Evictions: c.evictions.Load(),
TTLPurges: c.ttlPurges.Load(),
Misses: c.misses.Load(),
}
}
func (c *Cache) evictOldestLocked() {
el := c.ll.Back()
if el == nil {
return
}
c.removeLocked(el)
c.evictions.Add(1)
}
func (c *Cache) removeLocked(el *list.Element) {
entry := el.Value.(*processEntry)
delete(c.index, entry.PID)
c.ll.Remove(el)
}
// PutFromExec is a minimal constructor for callers that have only
// PID/UID/comm/exe from an exec event. UIDKnown is true even for UID 0.
func (c *Cache) PutFromExec(pid, ppid, uid int, comm, exe string) {
c.PutFromExecStartedAt(pid, ppid, uid, comm, exe, time.Time{})
}
// PutFromExecStartedAt is PutFromExec with a detector-supplied process start
// time for PID-reuse validation.
func (c *Cache) PutFromExecStartedAt(pid, ppid, uid int, comm, exe string, startedAt time.Time) {
c.Put(processEntry{PID: pid, PPID: ppid, UID: uid, UIDKnown: true, Comm: comm, Exe: exe, StartedAt: startedAt})
}
// PutFromProc inserts a fully populated entry from /proc-style data. The
// current enricher validates and writes inside processctx; this helper keeps a
// public constructor for tests and future non-daemon callers without exposing
// processEntry.
func (c *Cache) PutFromProc(pid, ppid, uid int, user, account, comm, exe string, cmdline []string) {
c.PutFromProcStartedAt(pid, ppid, uid, user, account, comm, exe, cmdline, time.Time{})
}
// PutFromProcStartedAt is PutFromProc with a known process start time.
func (c *Cache) PutFromProcStartedAt(pid, ppid, uid int, user, account, comm, exe string, cmdline []string, startedAt time.Time) {
c.Put(processEntry{
PID: pid, PPID: ppid, UID: uid, UIDKnown: true,
User: user, Account: account,
Comm: comm, Exe: exe, Cmdline: cmdline, StartedAt: startedAt, ProcRead: true,
})
}
package processctx
import (
"errors"
"maps"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
// procReader is the slice of ProcReader the Enricher needs. Allows fakes.
type procReader interface {
Read(pid int) (processEntry, error)
}
// EnrichRequest is the immutable event snapshot queued off the ring-buffer
// path. UID/Comm/StartedAt are used to reject stale PID reuse before caching
// /proc data.
type EnrichRequest struct {
PID int
UID int
UIDKnown bool
Comm string
StartedAt time.Time
}
type enrichWork struct {
req EnrichRequest
ticket queuehealth.Ticket
}
// IdentityResolver maps a process UID to username/account metadata. It must be
// cache-only in the common path. The daemon implementation uses
// checks.LookupUser's cached /etc/passwd reader and simple local account
// inference; it must not call NSS, LDAP, whmapi1, network services, or any
// blocking account enumerator from the enricher worker.
//
// Implementations SHOULD return within ~1ms in the common case. If a future
// implementation needs a backing data source that can block, refresh it in a
// separate cache outside the worker and have Resolve return ("", "") on cache
// miss rather than stalling the enrichment queue.
type IdentityResolver interface {
Resolve(uid int) (user, account string)
}
type noopResolver struct{}
func (noopResolver) Resolve(int) (string, string) { return "", "" }
// EnricherConfig sizes the worker pool and queue.
type EnricherConfig struct {
Workers int
QueueCap int
Resolver IdentityResolver
}
// EnricherStats is a snapshot of enricher counters.
type EnricherStats struct {
Enqueued uint64
Drops uint64
Reads uint64
Errors uint64
Stale uint64
}
// Enricher consumes PIDs and populates Cache from ProcReader.Read off the
// hot path. Enqueue is nonblocking. On overflow it drops the oldest queued
// request and records that drop, so the producer keeps moving and the queue
// favors fresher process snapshots.
type Enricher struct {
cache *Cache
reader procReader
resolver IdentityResolver
cfg EnricherConfig
queue chan enrichWork
queueStats *queuehealth.Tracker
admission sync.Mutex
wg sync.WaitGroup
stopCh chan struct{}
started bool
stopped bool
stopOnce sync.Once
enqueued atomic.Uint64
drops atomic.Uint64
reads atomic.Uint64
errors atomic.Uint64
stale atomic.Uint64
latencyMu sync.RWMutex
observeLatency func(float64)
}
// NewEnricher prepares a pool; Start launches its workers.
func NewEnricher(cache *Cache, reader procReader, cfg EnricherConfig) *Enricher {
if cfg.Workers <= 0 {
cfg.Workers = 2
}
if cfg.QueueCap <= 0 {
cfg.QueueCap = 1024
}
resolver := cfg.Resolver
if resolver == nil {
resolver = noopResolver{}
}
return &Enricher{
cache: cache,
reader: reader,
resolver: resolver,
cfg: cfg,
queue: make(chan enrichWork, cfg.QueueCap),
queueStats: queuehealth.New(cfg.QueueCap, time.Minute),
stopCh: make(chan struct{}),
}
}
// Start launches the worker goroutines. Idempotent.
func (e *Enricher) Start() {
e.admission.Lock()
defer e.admission.Unlock()
if e.started || e.stopped {
return
}
e.started = true
e.wg.Add(e.cfg.Workers)
for i := 0; i < e.cfg.Workers; i++ {
go e.worker()
}
}
// Stop signals workers and waits for them to exit. Safe to call multiple times.
//
// The pool cannot restart. Running reads finish; buffered requests are discarded
// and counted after workers relinquish ownership.
func (e *Enricher) Stop() {
e.stopOnce.Do(func() {
e.admission.Lock()
e.stopped = true
close(e.stopCh)
close(e.queue)
e.admission.Unlock()
e.wg.Wait()
for work := range e.queue {
work.ticket.Reject(time.Now())
e.drops.Add(1)
}
})
}
// Enqueue adds a request to the work queue. Returns false for an invalid PID
// or a stopped enricher. If the queue is full, the oldest pending request is
// dropped and the new one is queued.
func (e *Enricher) Enqueue(req EnrichRequest) bool {
e.admission.Lock()
defer e.admission.Unlock()
if req.PID <= 0 || e.stopped {
e.queueStats.Lose(time.Now(), 1)
e.drops.Add(1)
return false
}
work := enrichWork{req: req, ticket: e.queueStats.Begin(time.Now())}
select {
case e.queue <- work:
default:
select {
case oldest := <-e.queue:
oldest.ticket.Reject(time.Now())
e.drops.Add(1)
default:
}
// Other producers and close are excluded; consumers can only free slots.
e.queue <- work
}
e.enqueued.Add(1)
return true
}
func (e *Enricher) QueueStatuses(now time.Time) map[string]queuehealth.Status {
out := map[string]queuehealth.Status{"enrichment": e.queueStats.Snapshot(now)}
if reader, ok := e.reader.(interface {
QueueStatuses(time.Time) map[string]queuehealth.Status
}); ok {
maps.Copy(out, reader.QueueStatuses(now))
}
return out
}
// SetLatencyObserver installs an optional callback used by metrics.
func (e *Enricher) SetLatencyObserver(fn func(float64)) {
e.latencyMu.Lock()
defer e.latencyMu.Unlock()
e.observeLatency = fn
}
func (e *Enricher) observe(seconds float64) {
e.latencyMu.RLock()
fn := e.observeLatency
e.latencyMu.RUnlock()
if fn != nil {
fn(seconds)
}
}
func (e *Enricher) shouldCache(req EnrichRequest, entry processEntry) bool {
reqUIDKnown := req.UIDKnown || req.UID != 0
if reqUIDKnown {
if !entry.UIDKnown || req.UID != entry.UID {
return false
}
} else if !entry.UIDKnown {
return false
}
if req.Comm != "" && entry.Comm != req.Comm {
return false
}
// PID-reuse guard: when the detector supplied a process-start
// snapshot, the /proc-derived start time must be present and match
// within a small tolerance. A missing or mismatched /proc value means
// the enricher cannot prove it is looking at the same process.
if !processStartMatches(req.StartedAt, entry.StartedAt) {
return false
}
return true
}
// processStartTimeTolerance bounds the allowed clock skew between a
// detector's process-start snapshot and /proc/<pid>/stat's starttime. Five
// seconds covers clock granularity and slow pickup without letting a PID-reuse
// race slip past.
const processStartTimeTolerance = 5 * time.Second
func processStartMatches(want, got time.Time) bool {
if want.IsZero() {
return true
}
if got.IsZero() {
return false
}
diff := want.Sub(got)
if diff < 0 {
diff = -diff
}
return diff <= processStartTimeTolerance
}
func (e *Enricher) enrichIdentity(entry *processEntry) {
if !entry.UIDKnown {
return
}
user, account := e.resolver.Resolve(entry.UID)
entry.User = user
entry.Account = account
}
// Stats returns a counter snapshot.
func (e *Enricher) Stats() EnricherStats {
return EnricherStats{
Enqueued: e.enqueued.Load(),
Drops: e.drops.Load(),
Reads: e.reads.Load(),
Errors: e.errors.Load(),
Stale: e.stale.Load(),
}
}
func (e *Enricher) worker() {
defer e.wg.Done()
for {
select {
case <-e.stopCh:
return
default:
}
select {
case <-e.stopCh:
return
case work, ok := <-e.queue:
if !ok {
return
}
e.process(work)
}
}
}
func (e *Enricher) process(work enrichWork) {
start := time.Now()
work.ticket.Start(start)
completed := false
defer func() {
if completed {
work.ticket.Finish(time.Now())
} else {
work.ticket.Reject(time.Now())
}
}()
e.reads.Add(1)
entry, err := e.reader.Read(work.req.PID)
e.observe(time.Since(start).Seconds())
if err != nil {
if errors.Is(err, ErrProcessGone) {
completed = true
} else {
e.errors.Add(1)
}
return
}
if !e.shouldCache(work.req, entry) {
e.stale.Add(1)
completed = true
return
}
e.enrichIdentity(&entry)
e.cache.Put(entry)
completed = true
}
package processctx
import "time"
// Materialize walks the PPID chain starting at pid up to MaxParentDepth and
// returns a ProcessContext tree. Returns nil if pid is not in the cache.
// Cycle-safe: tracks visited PIDs.
func (c *Cache) Materialize(pid int) *ProcessContext {
root, ok := c.Get(pid)
if !ok {
return nil
}
return c.materializeFromRoot(root)
}
// MaterializeVerified returns a materialized context only when the cached root
// entry still matches the event snapshot. The bool return reports whether the
// root entry still needs an off-path /proc read.
func (c *Cache) MaterializeVerified(pid, uid int, uidKnown bool, comm string) (*ProcessContext, bool) {
root, ok := c.Get(pid)
if !ok {
return nil, false
}
if !matchesSnapshot(root, uid, uidKnown, comm) {
return nil, false
}
return c.materializeFromRoot(root), !root.ProcRead
}
// MaterializeVerifiedSnapshot returns a materialized context only when the
// cached root entry still matches the full detector snapshot, including the
// process start time when one was captured.
func (c *Cache) MaterializeVerifiedSnapshot(req EnrichRequest) (*ProcessContext, bool) {
root, ok := c.Get(req.PID)
if !ok {
return nil, false
}
if !matchesSnapshot(root, req.UID, req.UIDKnown, req.Comm) {
return nil, false
}
if !processStartMatches(req.StartedAt, root.StartedAt) {
return nil, false
}
return c.materializeFromRoot(root), !root.ProcRead
}
func (c *Cache) materializeFromRoot(root processEntry) *ProcessContext {
visited := map[int]bool{root.PID: true}
head := toContext(root)
cur := head
parentPID := root.PPID
for depth := 1; depth < MaxParentDepth; depth++ {
if parentPID <= 0 || visited[parentPID] {
break
}
entry, ok := c.Get(parentPID)
if !ok {
break
}
visited[entry.PID] = true
cur.Parent = toContext(entry)
cur = cur.Parent
parentPID = entry.PPID
}
return head
}
func matchesSnapshot(e processEntry, uid int, uidKnown bool, comm string) bool {
if uidKnown {
if !e.UIDKnown || e.UID != uid {
return false
}
}
if comm != "" && e.Comm != comm {
return false
}
return true
}
func toContext(e processEntry) *ProcessContext {
var startedAt *time.Time
if !e.StartedAt.IsZero() {
t := e.StartedAt
startedAt = &t
}
return &ProcessContext{
PID: e.PID,
PPID: e.PPID,
UID: e.UID,
User: e.User,
Account: e.Account,
Comm: e.Comm,
Exe: e.Exe,
ExeResolved: e.ProcRead && e.Exe != "",
Cmdline: append([]string(nil), e.Cmdline...),
StartedAt: startedAt,
}
}
package processctx
import "github.com/pidginhost/csm/internal/metrics"
// RegisterMetrics binds cache and enricher counters/gauges to reg. The
// registry argument allows tests to use a private registry. Production
// callers should pass metrics.Default().
func RegisterMetrics(reg *metrics.Registry, cache *Cache, enr *Enricher) {
reg.RegisterGaugeFunc(
"csm_process_context_cache_entries",
"Live process-context cache entries.",
func() float64 { return float64(cache.Stats().Entries) },
)
reg.RegisterCounterFunc(
"csm_process_context_cache_evictions_total",
"Process-context cache LRU evictions (cap exceeded).",
func() float64 { return float64(cache.Stats().Evictions) },
)
reg.RegisterCounterFunc(
"csm_process_context_cache_ttl_purges_total",
"Process-context cache entries dropped because TTL expired on lookup.",
func() float64 { return float64(cache.Stats().TTLPurges) },
)
reg.RegisterCounterFunc(
"csm_process_context_cache_misses_total",
"Process-context cache lookup misses (includes TTL purges).",
func() float64 { return float64(cache.Stats().Misses) },
)
reg.RegisterCounterFunc(
"csm_process_context_enrich_queue_drops_total",
"Process-context enrichment requests refused, evicted or abandoned at shutdown.",
func() float64 { return float64(enr.Stats().Drops) },
)
reg.RegisterCounterFunc(
"csm_process_context_enrich_reads_total",
"Process-context /proc reads attempted by enricher workers.",
func() float64 { return float64(enr.Stats().Reads) },
)
reg.RegisterCounterFunc(
"csm_process_context_enrich_errors_total",
"Process-context /proc read errors (excluding ProcessGone).",
func() float64 { return float64(enr.Stats().Errors) },
)
reg.RegisterCounterFunc(
"csm_process_context_enrich_stale_total",
"Process-context enrichment results rejected as stale PID reuse.",
func() float64 { return float64(enr.Stats().Stale) },
)
latency := metrics.NewHistogram(
"csm_process_context_enrich_latency_seconds",
"Process-context enrichment worker latency in seconds.",
[]float64{0.001, 0.005, 0.01, 0.05, 0.1, 1},
)
reg.MustRegister("csm_process_context_enrich_latency_seconds", latency)
enr.SetLatencyObserver(latency.Observe)
}
package processctx
import (
"errors"
"io/fs"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
var procReads = newProcReadPool(procReadConcurrency)
type procReadPool struct {
slots chan struct{}
stats *queuehealth.Tracker
}
func newProcReadPool(capacity int) *procReadPool {
return &procReadPool{
slots: make(chan struct{}, capacity),
stats: queuehealth.NewSharedCapacity(capacity, time.Minute),
}
}
type procReadWork struct {
pool *procReadPool
ticket queuehealth.Ticket
remaining atomic.Int32
failOnce sync.Once
}
func (p *procReadPool) acquire() *procReadWork {
ticket := p.stats.Begin(time.Now())
select {
case p.slots <- struct{}{}:
work := &procReadWork{pool: p, ticket: ticket}
work.remaining.Store(2)
return work
default:
ticket.Reject(time.Now())
return nil
}
}
func (w *procReadWork) fail() {
w.failOnce.Do(func() { w.pool.stats.Lose(time.Now(), 1) })
}
func (w *procReadWork) release() {
// A timeout finishes the caller, not the syscall. Conversely, a returned
// syscall still owns its result until the caller takes it or times out.
if w.remaining.Add(-1) == 0 {
w.ticket.Finish(time.Now())
<-w.pool.slots
}
}
type procReadResult[T any] struct {
value T
err error
}
func executeProcRead[T any](work *procReadWork, fn func() (T, error), out chan<- procReadResult[T]) {
work.ticket.Start(time.Now())
completed := false
defer func() {
if !completed {
work.fail()
}
work.release()
}()
value, err := fn()
if err != nil && !errors.Is(err, fs.ErrNotExist) {
work.fail()
}
out <- procReadResult[T]{value: value, err: err}
completed = true
}
func runProcReadWithDeadline[T any](pool *procReadPool, d time.Duration, fn func() (T, error)) (T, bool) {
if d <= 0 {
value, err := fn()
return value, err == nil
}
var zero T
work := pool.acquire()
if work == nil {
return zero, false
}
defer work.release()
out := make(chan procReadResult[T], 1)
go executeProcRead(work, fn, out)
timer := time.NewTimer(d)
defer timer.Stop()
select {
case result := <-out:
if result.err != nil {
return zero, false
}
return result.value, true
case <-timer.C:
work.fail()
return zero, false
}
}
// QueueStatuses reports deadline-bounded process reads. The row is advisory:
// an expired read costs one detail on a finding that is still raised, and a
// loaded host expires reads without losing protection work.
func (*ProcReader) QueueStatuses(now time.Time) map[string]queuehealth.Status {
status := procReads.stats.Snapshot(now)
status.Advisory = true
return map[string]queuehealth.Status{"proc_reads": status}
}
package processctx
import (
"bytes"
"errors"
"io/fs"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
// ErrProcessGone is returned by ProcReader.Read when the /proc/<pid> tree
// no longer exists. Callers must treat this as a soft miss, not an error
// finding - short-lived processes are expected.
var ErrProcessGone = errors.New("process gone")
// ProcReader reads /proc/<pid>/{status,cmdline,exe} with a per-file deadline.
type ProcReader struct {
root string
perFileDeadline time.Duration
}
// NewProcReader constructs a reader rooted at procRoot ("/proc" in production,
// a temp dir in tests). perFileDeadline bounds each individual file read.
func NewProcReader(procRoot string, perFileDeadline time.Duration) *ProcReader {
return &ProcReader{root: procRoot, perFileDeadline: perFileDeadline}
}
// Read returns a processEntry populated from /proc/<pid>. Fields that cannot
// be read within the deadline are left at their zero value; the caller still
// gets an entry with whatever was retrievable. Returns ErrProcessGone when
// the /proc/<pid> directory does not exist.
func (r *ProcReader) Read(pid int) (processEntry, error) {
dir := filepath.Join(r.root, strconv.Itoa(pid))
if _, err := os.Stat(dir); err != nil {
if errors.Is(err, fs.ErrNotExist) {
return processEntry{}, ErrProcessGone
}
return processEntry{}, err
}
e := processEntry{PID: pid, ProcRead: true}
if data, ok := readFileWithDeadline(filepath.Join(dir, "status"), r.perFileDeadline); ok {
e.PPID = parseStatusPPID(string(data))
e.UID, e.UIDKnown = parseStatusUIDKnown(string(data))
e.Comm = parseStatusName(string(data))
}
if data, ok := readFileWithDeadline(filepath.Join(dir, "cmdline"), r.perFileDeadline); ok {
e.Cmdline = parseCmdline(data)
}
if target, ok := readlinkWithDeadline(filepath.Join(dir, "exe"), r.perFileDeadline); ok {
e.Exe = target
}
if data, ok := readFileWithDeadline(filepath.Join(dir, "stat"), r.perFileDeadline); ok {
if t, ok := r.parseStartedAt(data); ok {
e.StartedAt = t
}
}
return e, nil
}
// ReadStartedAt returns only /proc/<pid>/stat's process start time. It lets
// detector hot paths capture a lightweight PID-reuse token without reading
// status, cmdline, exe, or identity data.
func (r *ProcReader) ReadStartedAt(pid int) (time.Time, bool) {
if pid <= 0 {
return time.Time{}, false
}
path := filepath.Join(r.root, strconv.Itoa(pid), "stat")
data, ok := readFileWithDeadline(path, r.perFileDeadline)
if !ok {
return time.Time{}, false
}
return r.parseStartedAt(data)
}
// procStatStartTime extracts field 22 of /proc/<pid>/stat (starttime in
// clock ticks since boot). Field positions are deterministic except
// that the second field (comm) is parenthesized and may contain
// arbitrary bytes including spaces -- so we anchor on the final ")"
// before splitting the rest.
func procStatStartTime(data []byte) (int64, bool) {
end := bytes.LastIndexByte(data, ')')
if end < 0 || end+1 >= len(data) {
return 0, false
}
rest := strings.TrimSpace(string(data[end+1:]))
fields := strings.Fields(rest)
// rest starts at field 3 (state); starttime is field 22, i.e. index 19 in rest.
const starttimeIdx = 19
if len(fields) <= starttimeIdx {
return 0, false
}
v, err := strconv.ParseInt(fields[starttimeIdx], 10, 64)
if err != nil || v < 0 {
return 0, false
}
return v, true
}
// parseStartedAt converts /proc/<pid>/stat's starttime field into an
// absolute time using the host's boot time. Returns (zero, false) on
// any parse or btime resolution failure so callers leave the field
// unset rather than emitting bogus timestamps.
func (r *ProcReader) parseStartedAt(stat []byte) (time.Time, bool) {
ticks, ok := procStatStartTime(stat)
if !ok {
return time.Time{}, false
}
boot, ok := r.bootTime()
if !ok {
return time.Time{}, false
}
hz := clockTicksPerSecond()
if hz <= 0 {
return time.Time{}, false
}
sec := ticks / hz
rem := ticks % hz
ns := rem * int64(time.Second) / hz
return boot.Add(time.Duration(sec)*time.Second + time.Duration(ns)), true
}
// readFileWithDeadline reads up to 4 KiB from path; returns (data, true) on
// success or (nil, false) on any error or deadline expiry. /proc files are
// small; using ReadFile keeps the normal path simple. The generic deadline
// helper is tested with an injected slow function instead of a FIFO because
// general filesystem opens cannot be cancelled safely on every platform.
func readFileWithDeadline(path string, d time.Duration) ([]byte, bool) {
return runBytesWithDeadline(d, func() ([]byte, error) {
// #nosec G304 -- path is constructed from ProcReader.root + numeric PID;
// callers only pass procfs entries under r.root.
data, err := os.ReadFile(path)
if len(data) > 4096 {
data = data[:4096]
}
return data, err
})
}
// procReadConcurrency bounds how many deadline-bound /proc reads run at once.
// A blocking syscall goroutine cannot be cancelled in Go, so a wedged /proc
// entry (NFS-backed, D-state) leaks its goroutine until the kernel returns --
// which may be never. The cap turns what was an unbounded leak under PID-reuse
// churn into a fixed ceiling: once it is reached, further reads fail fast
// instead of spawning more abandonable goroutines. A goroutine releases its
// slot only when its syscall finally returns, so genuinely-stuck reads keep
// their slot (correctly counting against the ceiling).
const procReadConcurrency = 64
func runBytesWithDeadline(d time.Duration, fn func() ([]byte, error)) ([]byte, bool) {
return runProcReadWithDeadline(procReads, d, fn)
}
// readlinkWithDeadline runs Readlink in a goroutine and gives up after d.
func readlinkWithDeadline(path string, d time.Duration) (string, bool) {
return runProcReadWithDeadline(procReads, d, func() (string, error) { return os.Readlink(path) })
}
func parseStatusName(s string) string {
for _, line := range strings.Split(s, "\n") {
if rest, ok := strings.CutPrefix(line, "Name:\t"); ok {
return strings.TrimSpace(rest)
}
}
return ""
}
func parseStatusPPID(s string) int {
for _, line := range strings.Split(s, "\n") {
if rest, ok := strings.CutPrefix(line, "PPid:\t"); ok {
v, _ := strconv.Atoi(strings.TrimSpace(rest))
return v
}
}
return 0
}
func parseStatusUID(s string) int {
uid, _ := parseStatusUIDKnown(s)
return uid
}
func parseStatusUIDKnown(s string) (int, bool) {
for _, line := range strings.Split(s, "\n") {
if rest, ok := strings.CutPrefix(line, "Uid:\t"); ok {
fields := strings.Fields(rest)
if len(fields) == 0 {
return 0, false
}
v, err := strconv.Atoi(fields[0])
return v, err == nil
}
}
return 0, false
}
func parseCmdline(b []byte) []string {
if len(b) == 0 {
return nil
}
parts := bytes.Split(b, []byte{0})
out := make([]string, 0, len(parts))
for _, p := range parts {
if len(p) == 0 {
continue
}
out = append(out, string(p))
}
if len(out) == 0 {
return nil
}
return sanitizeCmdline(out)
}
const maxCmdlineArgLen = 256
var sensitiveCmdlineKeys = []string{"password", "passwd", "secret", "token", "api_key", "apikey"}
func sanitizeCmdline(args []string) []string {
out := make([]string, 0, len(args))
redactNext := false
for _, arg := range args {
if redactNext {
out = append(out, "<redacted>")
redactNext = false
continue
}
lower := strings.ToLower(arg)
redacted := false
for _, key := range sensitiveCmdlineKeys {
if strings.Contains(lower, key+"=") {
prefix, _, _ := strings.Cut(arg, "=")
out = append(out, truncateCmdlineArg(prefix+"=<redacted>"))
redacted = true
break
}
if lower == "--"+key || lower == "-"+key {
out = append(out, truncateCmdlineArg(arg))
redactNext = true
redacted = true
break
}
}
if redacted {
continue
}
out = append(out, truncateCmdlineArg(arg))
}
return out
}
func truncateCmdlineArg(arg string) string {
if len(arg) <= maxCmdlineArgLen {
return arg
}
return arg[:maxCmdlineArgLen]
}
package processctx
import (
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
)
// bootTimeCache caches the host boot time per proc root.
// Boot time is invariant for the life of a kernel; rereading /proc/stat
// on every Read() would only spend syscalls for an unchanging value.
type bootTimeCache struct {
once sync.Once
t time.Time
ok bool
}
var procReaderBoot sync.Map // root -> *bootTimeCache
func (r *ProcReader) bootTime() (time.Time, bool) {
v, _ := procReaderBoot.LoadOrStore(r.root, &bootTimeCache{})
c := v.(*bootTimeCache)
c.once.Do(func() {
c.t, c.ok = readBootTime(filepath.Join(r.root, "stat"))
})
return c.t, c.ok
}
func readBootTime(statPath string) (time.Time, bool) {
// #nosec G304 -- statPath is procReader.root + "stat"; root is operator-pinned.
data, err := os.ReadFile(statPath)
if err != nil {
return time.Time{}, false
}
for _, line := range strings.Split(string(data), "\n") {
if rest, ok := strings.CutPrefix(line, "btime "); ok {
sec, err := strconv.ParseInt(strings.TrimSpace(rest), 10, 64)
if err != nil {
return time.Time{}, false
}
return time.Unix(sec, 0), true
}
}
return time.Time{}, false
}
// clockTicksPerSecondOverride lets tests inject a known _SC_CLK_TCK
// without calling sysconf on the host.
var clockTicksPerSecondOverride int64
func clockTicksPerSecond() int64 {
if clockTicksPerSecondOverride > 0 {
return clockTicksPerSecondOverride
}
return defaultClockTicksPerSecond
}
// defaultClockTicksPerSecond is 100 on every mainstream Linux kernel
// CSM runs on (CONFIG_HZ_100=y). Reading sysconf would require cgo or
// a build-tagged platform shim; the constant is correct for every
// distribution kernel we target. Tests that want a different value
// set clockTicksPerSecondOverride.
const defaultClockTicksPerSecond = 100
// Package processhandle signals verified Linux processes through kernel handles.
package processhandle
import (
"context"
"errors"
"fmt"
"math"
"syscall"
)
var ErrUnsupported = errors.New("safe process signaling requires kernel pidfd support")
// Signal acquires a process handle before running verify. Verification may read
// numeric procfs paths: exit checks surrounding it reject a recycled PID. The
// final signal uses the captured handle even if the process exits afterward.
func Signal(ctx context.Context, pid int, sig syscall.Signal, verify func() error) error {
if err := ctx.Err(); err != nil {
return err
}
if pid <= 1 || pid > math.MaxInt32 {
return fmt.Errorf("refuse to signal PID %d", pid)
}
handle, err := openHandle(pid)
if err != nil {
return err
}
defer handle.close()
if err := handle.alive(); err != nil {
return err
}
if err := verify(); err != nil {
return err
}
if err := handle.alive(); err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
return handle.signal(sig)
}
// Available reports whether this kernel can pin and signal a process handle.
// Probe each snapshot: descriptor pressure or service restrictions can change
// while the daemon runs, even though the kernel itself remains the same.
func Available() error {
return probe()
}
//go:build linux
package processhandle
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"strconv"
"syscall"
"golang.org/x/sys/unix"
)
var (
pidfdOpen = unix.PidfdOpen
pidfdPoll = unix.Poll
pidfdSend = unix.PidfdSendSignal
pidfdClose = unix.Close
procRoot = "/proc"
)
// maxProcStatBytes bounds the tenant-visible /proc/<pid>/stat read. The comm
// field is capped at 16 bytes by the kernel, so a real record is far smaller.
const maxProcStatBytes = 4096
// procfs reports whether fd is a /proc/<pid> directory descriptor rather than
// a descriptor returned by pidfd_open. Both pin one struct pid, so
// pidfd_send_signal cannot be redirected to a recycled PID through either.
type handle struct {
fd int
procfs bool
}
// openHandle pins the target process. pidfd_open needs Linux 5.3; the 4.18
// kernels on EL8 and CloudLinux 8 do not have it, but their pidfd_send_signal
// (Linux 5.1) accepts a /proc/<pid> directory descriptor, which pins the same
// struct pid. Falling back preserves the identity guarantee instead of
// disabling termination on every supported EL8 host.
func openHandle(pid int) (*handle, error) {
fd, err := pidfdOpen(pid, 0)
if err == nil {
return &handle{fd: fd}, nil
}
if !errors.Is(err, unix.ENOSYS) {
return nil, signalError("pidfd_open", err)
}
dirfd, dirErr := unix.Open(procRoot+"/"+strconv.Itoa(pid), unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
if dirErr != nil {
if errors.Is(dirErr, unix.ENOENT) || errors.Is(dirErr, unix.ESRCH) {
return nil, fmt.Errorf("open %s/%d: %w: %w", procRoot, pid, os.ErrProcessDone, dirErr)
}
return nil, fmt.Errorf("open %s/%d: %w (pidfd_open: %w)", procRoot, pid, dirErr, err)
}
return &handle{fd: dirfd, procfs: true}, nil
}
func (h *handle) close() { _ = pidfdClose(h.fd) }
func (h *handle) alive() error {
if h.procfs {
return h.procfsAlive()
}
// #nosec G115 -- Linux file descriptors are non-negative signed C ints.
fds := []unix.PollFd{{Fd: int32(h.fd), Events: unix.POLLIN}}
if _, err := pidfdPoll(fds, 0); err != nil {
return fmt.Errorf("poll process handle: %w", err)
}
if fds[0].Revents&(unix.POLLIN|unix.POLLHUP) != 0 {
return os.ErrProcessDone
}
if fds[0].Revents != 0 {
return fmt.Errorf("invalid process handle poll events: %#x", fds[0].Revents)
}
return nil
}
// procfsAlive reads the pinned directory's own stat file. A reaped process
// removes those entries, and an unreaped one reports a terminal state, so a
// replacement holding the same numeric PID is never mistaken for the target.
func (h *handle) procfsAlive() error {
fd, err := unix.Openat(h.fd, "stat", unix.O_RDONLY|unix.O_CLOEXEC, 0)
if err != nil {
if errors.Is(err, unix.ENOENT) || errors.Is(err, unix.ESRCH) {
return os.ErrProcessDone
}
return fmt.Errorf("open process state: %w", err)
}
// #nosec G115 -- Successful openat returns a non-negative file descriptor.
file := os.NewFile(uintptr(fd), "stat")
defer func() { _ = file.Close() }()
data, err := io.ReadAll(io.LimitReader(file, maxProcStatBytes))
if err != nil {
if errors.Is(err, unix.ESRCH) {
return os.ErrProcessDone
}
return fmt.Errorf("read process state: %w", err)
}
state, err := procStatState(data)
if err != nil {
return err
}
// Z: exited, awaiting reap. X/x: released. Neither can receive a signal.
if state == 'Z' || state == 'X' || state == 'x' {
return os.ErrProcessDone
}
return nil
}
// procStatState returns the third field of /proc/<pid>/stat. The second field
// is a process-controlled name in parentheses that may itself contain spaces
// and parentheses, so the scan starts at its final closing parenthesis.
func procStatState(data []byte) (byte, error) {
end := bytes.LastIndexByte(data, ')')
if end < 0 || end+2 >= len(data) || data[end+1] != ' ' {
return 0, errors.New("unreadable process state")
}
return data[end+2], nil
}
func (h *handle) signal(sig syscall.Signal) error {
if err := pidfdSend(h.fd, sig, nil, 0); err != nil {
return signalError("pidfd_send_signal", err)
}
return nil
}
func signalError(operation string, err error) error {
if errors.Is(err, unix.ENOSYS) {
return fmt.Errorf("%s: %w: %w", operation, ErrUnsupported, err)
}
if errors.Is(err, unix.ESRCH) {
return fmt.Errorf("%s: %w: %w", operation, os.ErrProcessDone, err)
}
return fmt.Errorf("%s: %w", operation, err)
}
// probe pins this process and asks the kernel to deliver signal 0, which
// validates the whole path (handle acquisition, liveness, delivery) without
// disturbing any process.
func probe() error {
handle, err := openHandle(os.Getpid())
if err != nil {
return err
}
defer handle.close()
if err := handle.alive(); err != nil {
return err
}
return handle.signal(0)
}
// Package quarantinefs makes recovery copies durable before callers change
// live files. Its paths must be under an operator-owned quarantine directory.
package quarantinefs
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"golang.org/x/sys/unix"
)
var (
copyContent = io.Copy
syncFile = (*os.File).Sync
closeFile = (*os.File).Close
)
// EnsureDir persists newly created directory entries, including missing parents.
func EnsureDir(path string, mode os.FileMode) error {
info, err := os.Stat(path)
if err == nil {
if !info.IsDir() {
return fmt.Errorf("quarantine path is not a directory: %s", path)
}
// Another quarantine transaction may have just created this entry.
return SyncDir(filepath.Dir(path))
}
if !os.IsNotExist(err) {
return err
}
parent := filepath.Dir(path)
if err := EnsureDir(parent, mode); err != nil {
return err
}
if err := os.Mkdir(path, mode); err != nil && !os.IsExist(err) {
return err
}
return SyncDir(parent)
}
// WriteExclusive never overwrites older evidence. Success includes the file's
// contents, its close result, and the containing directory entry reaching disk.
// The caller must first persist the containing directory with EnsureDir.
func WriteExclusive(path string, content io.Reader, mode os.FileMode) (err error) {
// #nosec G304 -- caller supplies a path under operator-owned quarantine; exclusive creation refuses existing files and symlinks.
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)
if err != nil {
return err
}
closed := false
defer func() {
if !closed {
err = errors.Join(err, closeFile(f))
}
if err != nil {
if removeErr := os.Remove(path); removeErr != nil && !os.IsNotExist(removeErr) {
err = errors.Join(err, fmt.Errorf("partial copy retained at %s: %w", path, removeErr))
}
}
}()
if _, err := copyContent(f, content); err != nil {
return fmt.Errorf("writing %s: %w", path, err)
}
if err := syncFile(f); err != nil {
return fmt.Errorf("syncing %s: %w", path, err)
}
closed = true
if err := closeFile(f); err != nil {
return fmt.Errorf("closing %s: %w", path, err)
}
return SyncDir(filepath.Dir(path))
}
// Store writes a private content copy and its sidecar before the caller may
// remove or replace the original. On failure the caller must retain the original.
func Store(path string, content io.Reader, metadata []byte, mode os.FileMode) error {
if err := EnsureDir(filepath.Dir(path), 0700); err != nil {
return err
}
if err := WriteExclusive(path, content, mode); err != nil {
return err
}
if err := WriteExclusive(path+".meta", bytes.NewReader(metadata), 0600); err != nil {
if removeErr := os.Remove(path); removeErr != nil && !os.IsNotExist(removeErr) {
return errors.Join(err, fmt.Errorf("copy retained at %s: %w", path, removeErr))
}
return fmt.Errorf("writing quarantine metadata: %w", err)
}
return nil
}
func SyncDir(path string) error {
// #nosec G304 -- caller supplies the containing directory of a quarantine or restored file, opened read-only to persist its entries.
dir, err := os.Open(path)
if err != nil {
return err
}
return errors.Join(syncFile(dir), closeFile(dir))
}
func SyncFilePath(path string) error {
// #nosec G304 -- caller supplies a quarantine or remediation path; no-follow and regular-file checks reject substituted links and special files.
f, err := os.OpenFile(path, os.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0)
if err != nil {
return err
}
info, err := f.Stat()
if err != nil {
return errors.Join(err, closeFile(f))
}
if !info.Mode().IsRegular() {
return errors.Join(fmt.Errorf("cannot sync non-regular file %s", path), closeFile(f))
}
return errors.Join(syncFile(f), closeFile(f))
}
// RemoveEvidence runs only after a restored copy and its directory are durable.
// A failed content removal leaves its metadata intact for another recovery attempt.
func RemoveEvidence(path, metaPath string) error {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("restored, but quarantine content remains at %s: %w", path, err)
}
if err := SyncDir(filepath.Dir(path)); err != nil {
return fmt.Errorf("restored, but quarantine removal is not durable; metadata retained at %s: %w", metaPath, err)
}
if err := os.Remove(metaPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("restored, but quarantine metadata remains at %s: %w", metaPath, err)
}
return SyncDir(filepath.Dir(metaPath))
}
package quarantinefs
import (
"errors"
"fmt"
"io/fs"
"os"
"golang.org/x/sys/unix"
)
// SyncTree persists a directory being moved without following symlinks inside
// it. expected binds the traversal to the directory admitted by the caller.
func SyncTree(path string, expected os.FileInfo) error {
root, err := os.OpenRoot(path)
if err != nil {
return err
}
defer func() { _ = root.Close() }()
info, err := root.Stat(".")
if err != nil {
return err
}
if !info.IsDir() || !os.SameFile(info, expected) {
return fmt.Errorf("quarantine directory changed before syncing")
}
var directories []string
err = fs.WalkDir(root.FS(), ".", func(name string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
directories = append(directories, name)
return nil
}
if entry.Type()&os.ModeSymlink != 0 {
return nil
}
file, openErr := root.OpenFile(name, os.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0)
if openErr != nil {
return openErr
}
info, statErr := file.Stat()
if statErr != nil || !info.Mode().IsRegular() {
_ = file.Close()
return fmt.Errorf("cannot sync non-regular quarantine entry %s: %v", name, statErr)
}
return errors.Join(syncFile(file), closeFile(file))
})
if err != nil {
return err
}
// Persist child contents before the directory entries that make them reachable.
for i := len(directories) - 1; i >= 0; i-- {
dir, openErr := root.OpenFile(directories[i], os.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW, 0)
if openErr != nil {
return openErr
}
if err := errors.Join(syncFile(dir), closeFile(dir)); err != nil {
return err
}
}
return nil
}
package queuehealth
import "time"
// Work retains accounting after a channel receive. The consumer starts its
// ticket, then finishes or rejects it after processing, including error paths.
type Work[T any] struct {
Value T
Ticket Ticket
}
// Process keeps filtered work accounted through the consumer's whole callback.
// A panicking consumer loses this item; the panic continues to its owner.
func (w Work[T]) Process(fn func(T)) {
w.Ticket.Start(time.Now())
completed := false
defer func() {
if completed {
w.Ticket.Finish(time.Now())
} else {
w.Ticket.Reject(time.Now())
}
}()
fn(w.Value)
completed = true
}
// Channel accounts for bounded waiting work and its consumers. Its owner
// closes it after all producers have stopped; stopped consumers must leave
// queued work to DiscardPending instead of silently abandoning it.
type Channel[T any] struct {
items chan Work[T]
tracker *Tracker
now func() time.Time
}
func NewChannel[T any](capacity int, maxLag time.Duration) *Channel[T] {
return &Channel[T]{
items: make(chan Work[T], capacity), tracker: New(capacity, maxLag), now: time.Now,
}
}
func (q *Channel[T]) Items() <-chan Work[T] { return q.items }
func (q *Channel[T]) TrySend(value T) bool {
work := Work[T]{Value: value, Ticket: q.tracker.Begin(q.now())}
select {
case q.items <- work:
return true
default:
work.Ticket.Reject(q.now())
return false
}
}
func (q *Channel[T]) Send(value T, stop <-chan struct{}) bool {
work := Work[T]{Value: value, Ticket: q.tracker.Begin(q.now())}
select {
case q.items <- work:
return true
case <-stop:
work.Ticket.Reject(q.now())
return false
}
}
func (q *Channel[T]) Close() { close(q.items) }
// DiscardPending runs after Close and after consumers have stopped. A work
// item already received by a consumer still belongs to that consumer.
func (q *Channel[T]) DiscardPending() {
for work := range q.items {
work.Ticket.Reject(q.now())
}
}
func (q *Channel[T]) Snapshot(now time.Time) Status { return q.tracker.Snapshot(now) }
// Lose counts work rejected before it could be represented by a queued item.
func (q *Channel[T]) Lose(count uint64) { q.tracker.Lose(q.now(), count) }
package queuehealth
import "time"
// MeasurementWindow is how long a queue must stay unmeasurable before that
// counts as a degradation. A kernel counter read can fail once, and positions
// loaded separately can produce one incoherent sample, without the queue
// having a problem.
const MeasurementWindow = 30 * time.Second
// Dwell reports whether a condition has held continuously for a window. The
// owner serializes its calls with the rest of its snapshot.
type Dwell struct {
since time.Time
}
func (d *Dwell) Held(now time.Time, condition bool, window time.Duration) bool {
if !condition {
d.since = time.Time{}
return false
}
if d.since.IsZero() {
d.since = now
}
return now.Sub(d.since) >= window
}
package queuehealth
import "fmt"
// Evidence labels sampled and unavailable measurements so notifications and
// doctor cannot present bytes as events or consumer stalls as event ages.
func (s Status) Evidence() string {
unit := s.DepthUnit
if unit == "" {
unit = "items"
}
pending, capacity := fmt.Sprint(s.Depth), fmt.Sprint(s.Capacity)
if s.DepthUnavailable {
pending = "unknown"
}
if s.CapacityUnavailable {
capacity = "unknown"
}
depth := pending + "/" + capacity
lag := fmt.Sprintf("lag=%.0fs", s.LagSeconds)
switch s.LagBasis {
case "observed_age":
lag = fmt.Sprintf("observed_lag=%.0fs", s.LagSeconds)
case "consumer_progress":
lag = fmt.Sprintf("consumer_stall=%.0fs", s.LagSeconds)
case "operation_progress":
lag = fmt.Sprintf("operation_stall=%.0fs", s.LagSeconds)
case "deferred_checkpoint":
lag = fmt.Sprintf("deferred_age=%.0fs", s.LagSeconds)
case "unavailable":
lag = "lag=unavailable"
}
dropped := fmt.Sprintf("dropped=%d", s.DroppedTotal)
if s.DroppedLowerBound {
dropped = fmt.Sprintf("dropped>=%d", s.DroppedTotal)
}
return fmt.Sprintf("depth=%s %s running=%d %s recent_drops=%d %s processing=%.0fs", depth, unit, s.InFlight, dropped, s.RecentDrops, lag, s.ProcessingSeconds)
}
package queuehealth
import (
"sort"
"time"
)
// reminderInterval bounds how often one queue can announce. It also bounds a
// flapping queue: a degradation that returns within the interval of the last
// announcement is carried in status but not notified again.
const reminderInterval = 5 * time.Minute
// Event records a degradation, a bounded reminder or a recovery. Healthy
// startup produces no event, and one queue cannot suppress another's change.
type Event struct {
Name string
Current Status
Recovered bool
}
// incident is one queue's notification state. It outlives the degradation so a
// queue that recovers and degrades again stays inside the same bound.
type incident struct {
announcedAt time.Time
announced bool
degraded bool
}
// Reporter belongs to one health loop. Polling status does not consume its
// state or the trackers' counters, so API traffic cannot swallow an alert.
type Reporter struct {
last map[string]incident
}
func (r *Reporter) Events(now time.Time, states map[string]Status) []Event {
if r.last == nil {
r.last = make(map[string]incident)
}
names := make([]string, 0, len(states))
for name := range states {
names = append(names, name)
}
sort.Strings(names)
var events []Event
for _, name := range names {
s := states[name]
state := r.last[name]
switch {
case s.Advisory:
// Best-effort work is visible in status and doctor only.
continue
case s.Status == "degraded":
if state.announcedAt.IsZero() || now.Sub(state.announcedAt) >= reminderInterval {
events = append(events, Event{Name: name, Current: s})
state.announcedAt, state.announced = now, true
} else if !state.degraded {
// A degradation the bound suppressed must not announce a
// recovery either, or the pair count is unchanged.
state.announced = false
}
state.degraded = true
r.last[name] = state
case state.degraded:
if state.announced {
events = append(events, Event{Name: name, Current: s, Recovered: true})
state.announced = false
}
state.degraded = false
r.last[name] = state
case !state.announcedAt.IsZero() && now.Sub(state.announcedAt) >= reminderInterval:
delete(r.last, name)
}
}
return events
}
package queuehealth
import (
"sync"
"time"
)
// Sampled measures opaque queues whose entries cannot carry tickets. Its lag
// is time since observed consumer progress while work remains, not the age of
// an unseen event. Owners sample depth and a monotonic consumption counter
// together; observing new arrivals alone must not reset a stalled consumer.
type Sampled struct {
mu sync.Mutex
capacity int
unit string
maxLag time.Duration
depth int
progress uint64
stalledSince time.Time
fullSince time.Time
losses *Tracker
}
func NewSampled(capacity int, unit string, maxLag time.Duration) *Sampled {
return &Sampled{capacity: capacity, unit: unit, maxLag: maxLag, losses: New(0, maxLag)}
}
func (q *Sampled) Observe(now time.Time, depth int, progress uint64) {
q.mu.Lock()
defer q.mu.Unlock()
if depth == 0 {
q.stalledSince = time.Time{}
} else if q.stalledSince.IsZero() || progress != q.progress {
q.stalledSince = now
}
if q.capacity > 0 && depth >= q.capacity {
if q.fullSince.IsZero() {
q.fullSince = now
}
} else {
q.fullSince = time.Time{}
}
q.depth, q.progress = depth, progress
}
func (q *Sampled) Lose(now time.Time, count uint64) { q.losses.Lose(now, count) }
// SnapshotUnavailable retains independently counted losses without deriving
// pressure from an occupancy reading the owner could not validate.
func (q *Sampled) SnapshotUnavailable(now time.Time) Status {
q.mu.Lock()
defer q.mu.Unlock()
s := q.losses.Snapshot(now)
s.Capacity, s.DepthUnit = q.capacity, q.unit
s.DepthUnavailable, s.LagBasis = true, "unavailable"
return s
}
func (q *Sampled) Snapshot(now time.Time) Status {
q.mu.Lock()
defer q.mu.Unlock()
s := q.losses.Snapshot(now)
s.Depth, s.Capacity, s.DepthUnit = q.depth, q.capacity, q.unit
s.LagBasis = "consumer_progress"
if !q.stalledSince.IsZero() {
s.LagSeconds = max(0, now.Sub(q.stalledSince).Seconds())
}
switch {
case s.LagSeconds >= q.maxLag.Seconds():
s.Reason = "consumer_stalled"
case !q.fullSince.IsZero() && now.Sub(q.fullSince) >= fullWindow:
s.Reason = "queue_full"
}
if s.Reason != "" {
s.Status = "degraded"
}
return s
}
// Package queuehealth measures waiting, running and lost work independently
// of the channel that carries findings about a protection failure.
package queuehealth
import (
"sync"
"time"
)
const (
dropWindow = time.Minute
fullWindow = 30 * time.Second
dropThreshold = 3
)
// Status is the queue evidence carried by status, the API and doctor.
// LagSeconds measures the oldest waiting item unless LagBasis names another
// measurement. ProcessingSeconds measures
// the oldest running item, so an empty queue cannot conceal a stuck worker.
// Advisory marks a queue whose work is best effort, so its degradation is
// reported but does not make the host degraded and raises no notification.
type Status struct {
Status string `json:"status"`
Reason string `json:"reason,omitempty"`
Advisory bool `json:"advisory,omitempty"`
Depth int `json:"depth"`
DepthUnit string `json:"depth_unit,omitempty"`
DepthUnavailable bool `json:"depth_unavailable,omitempty"`
LagBasis string `json:"lag_basis,omitempty"`
Capacity int `json:"capacity"`
CapacityUnavailable bool `json:"capacity_unavailable,omitempty"`
InFlight int `json:"in_flight"`
DroppedTotal uint64 `json:"dropped_total"`
DroppedLowerBound bool `json:"dropped_lower_bound,omitempty"`
RecentDrops uint64 `json:"recent_drops"`
LagSeconds float64 `json:"lag_seconds"`
ProcessingSeconds float64 `json:"processing_seconds"`
}
type work struct {
queued time.Time
started time.Time
}
type dropBucket struct {
second int64
count uint64
}
// Tracker accounts for a bounded queue and its workers. Every Begin must
// end in Finish or Reject, including cancellation and panic paths. Tracking
// starts before a send so a fast consumer cannot finish an unrecorded item.
// Its retained work is bounded by the queue and the producer/worker counts.
type Tracker struct {
mu sync.Mutex
capacity int
maxLag time.Duration
sharedCapacity bool
next uint64
pending map[uint64]work
waiting int
fullSince time.Time
heldSince time.Time
dropped uint64
drops [60]dropBucket
}
func New(capacity int, maxLag time.Duration) *Tracker {
return &Tracker{capacity: capacity, maxLag: maxLag, pending: make(map[uint64]work)}
}
// NewSharedCapacity measures queues whose running work still occupies
// admission slots. Ordinary channels free those slots when a consumer reads.
func NewSharedCapacity(capacity int, maxLag time.Duration) *Tracker {
q := New(capacity, maxLag)
q.sharedCapacity = true
return q
}
// Ticket follows one item from enqueue through processing. A zero ticket
// represents synchronous work that never entered a queue.
type Ticket struct {
tracker *Tracker
id uint64
}
func (q *Tracker) Begin(now time.Time) Ticket {
return q.BeginAt(now, now)
}
// BeginAt sets the waiting-age origin independently of admission. Earlier
// delays must not backdate saturation; a future origin defers lag until the
// work is eligible to run.
func (q *Tracker) BeginAt(queuedAt, now time.Time) Ticket {
q.mu.Lock()
defer q.mu.Unlock()
q.next++
q.pending[q.next] = work{queued: queuedAt}
q.waiting++
q.updateFull(now)
return Ticket{tracker: q, id: q.next}
}
func (t Ticket) Start(now time.Time) {
if t.tracker == nil {
return
}
q := t.tracker
q.mu.Lock()
defer q.mu.Unlock()
w := q.pending[t.id]
w.started = now
q.pending[t.id] = w
q.waiting--
q.updateFull(now)
}
// Requeue returns a running item to waiting without resetting its admission
// time. Repeated attempts must not conceal a backlog that never completes.
func (t Ticket) Requeue(now time.Time) {
if t.tracker == nil {
return
}
q := t.tracker
q.mu.Lock()
defer q.mu.Unlock()
w := q.pending[t.id]
w.started = time.Time{}
q.pending[t.id] = w
q.waiting++
q.updateFull(now)
}
// RetainQueuedAt preserves earlier eligibility when observations coalesce.
// A later observation must never postpone work that was already overdue.
func (t Ticket) RetainQueuedAt(queuedAt time.Time) {
if t.tracker == nil {
return
}
q := t.tracker
q.mu.Lock()
defer q.mu.Unlock()
w := q.pending[t.id]
if queuedAt.Before(w.queued) {
w.queued = queuedAt
q.pending[t.id] = w
}
}
// MergeRunning absorbs a distinct running ticket on the same tracker into
// this waiting ticket. The caller retains only this ticket. Keeping the
// older age and completing the running ticket atomically avoids inventing
// an available waiting slot while coalescing a retry with new work.
func (t Ticket) MergeRunning(running Ticket, now time.Time) {
if t.tracker == nil {
return
}
q := t.tracker
q.mu.Lock()
defer q.mu.Unlock()
w := q.pending[t.id]
if earlier := q.pending[running.id].queued; earlier.Before(w.queued) {
w.queued = earlier
}
q.pending[t.id] = w
q.finish(running.id, now)
}
func (t Ticket) Finish(now time.Time) {
if t.tracker == nil {
return
}
q := t.tracker
q.mu.Lock()
defer q.mu.Unlock()
q.finish(t.id, now)
}
func (t Ticket) Reject(now time.Time) {
if t.tracker == nil {
return
}
q := t.tracker
q.mu.Lock()
defer q.mu.Unlock()
q.finish(t.id, now)
q.lose(now, 1)
}
func (q *Tracker) finish(id uint64, now time.Time) {
if q.pending[id].started.IsZero() {
q.waiting--
}
delete(q.pending, id)
q.updateFull(now)
}
func (q *Tracker) updateFull(now time.Time) {
occupied := q.waiting
if q.sharedCapacity {
occupied = len(q.pending)
}
if q.capacity > 0 && occupied >= q.capacity {
if q.fullSince.IsZero() {
q.fullSince = now
}
} else {
q.fullSince = time.Time{}
}
}
// Hold parks the queue while its owner deliberately withholds the consumer,
// such as the startup baseline. Ages stop at the hold so intentional waiting
// cannot report a stall; losses still count.
func (q *Tracker) Hold(now time.Time) {
q.mu.Lock()
defer q.mu.Unlock()
if q.heldSince.IsZero() {
q.heldSince = now
}
}
// Release resumes measurement. The held time is removed from every age, so
// work parked by the hold is measured from the release.
func (q *Tracker) Release(now time.Time) {
q.mu.Lock()
defer q.mu.Unlock()
if q.heldSince.IsZero() {
return
}
shift := now.Sub(q.heldSince)
rebase := func(t time.Time) time.Time {
switch {
case t.IsZero():
return t
case t.After(now):
// Deferred work has not begun aging and keeps its eligibility.
return t
case t.Before(q.heldSince):
return t.Add(shift)
default:
return now
}
}
for id, w := range q.pending {
w.queued, w.started = rebase(w.queued), rebase(w.started)
q.pending[id] = w
}
q.fullSince = rebase(q.fullSince)
q.heldSince = time.Time{}
}
// Lose records upstream loss for which no userspace work item exists, such
// as a kernel overflow. The cumulative count is never drained by a reporter.
func (q *Tracker) Lose(now time.Time, count uint64) {
q.mu.Lock()
defer q.mu.Unlock()
q.lose(now, count)
}
func (q *Tracker) lose(now time.Time, count uint64) {
q.dropped += count
second := now.Unix()
b := &q.drops[uint64(second)%uint64(len(q.drops))] // #nosec G115 -- modulo index is bounded even for a pre-epoch clock
if b.second != second {
*b = dropBucket{second: second}
}
b.count += count
}
func (q *Tracker) Snapshot(now time.Time) Status {
q.mu.Lock()
defer q.mu.Unlock()
s := Status{Status: "ok", Capacity: q.capacity, Depth: q.waiting, InFlight: len(q.pending) - q.waiting, DroppedTotal: q.dropped}
// A held queue measures ages up to the hold; drops keep the real clock.
clock := now
if !q.heldSince.IsZero() && q.heldSince.Before(now) {
clock = q.heldSince
}
for _, w := range q.pending {
if w.started.IsZero() {
s.LagSeconds = max(s.LagSeconds, clock.Sub(w.queued).Seconds())
} else {
s.ProcessingSeconds = max(s.ProcessingSeconds, clock.Sub(w.started).Seconds())
}
}
second := now.Unix()
for _, b := range q.drops {
if b.second <= second && second-b.second < int64(dropWindow/time.Second) {
s.RecentDrops += b.count
}
}
switch {
case s.LagSeconds >= q.maxLag.Seconds():
s.Reason = "backlog_lag"
case s.ProcessingSeconds >= q.maxLag.Seconds():
s.Reason = "processing_lag"
case !q.fullSince.IsZero() && clock.Sub(q.fullSince) >= fullWindow:
s.Reason = "queue_full"
case s.RecentDrops >= dropThreshold:
s.Reason = "dropped_work"
}
if s.Reason != "" {
s.Status = "degraded"
}
return s
}
// Package redisinfo wraps the go-redis client for the few read-only
// INFO calls CSM needs (memory metrics, keyspace counts). It replaces
// the `redis-cli info <section>` shell-outs the performance UI used
// to issue per-poll, eliminating libc/libpthread fork churn on hosts
// with a busy metrics dashboard.
//
// The client is a lazy package-level singleton: first call opens a
// connection-pooled client against the local redis socket / TCP, all
// subsequent calls reuse it. No Close path because the daemon runs
// for the host's lifetime; go-redis cleans up on process exit.
//
// Connection target matches redis-cli's default behaviour:
//
// 127.0.0.1:6379, no password, db 0
//
// When REDISCLI_AUTH is set, the client uses it as the password so
// daemon environments that previously made redis-cli work keep working.
// Hosts running redis on a non-default socket can override by calling
// SetAddr before the first MemoryUsage / Keyspace call. Absolute
// paths are treated as Unix sockets.
package redisinfo
import (
"context"
"fmt"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/redis/go-redis/v9"
)
const defaultAddr = "127.0.0.1:6379"
var (
mu sync.Mutex
defaultDB *redis.Client
clientBuilt bool
addr = defaultAddr
password = ""
)
// SetAddr overrides the connection target before the singleton opens.
// Calls after the first MemoryUsage / Keyspace are ignored: the
// singleton has already been built. Tests can also use SetClientForTest
// to swap the singleton wholesale.
func SetAddr(a, pwd string) {
mu.Lock()
defer mu.Unlock()
if clientBuilt {
return
}
addr = a
password = pwd
}
// SetClientForTest replaces the singleton with a caller-supplied
// client (typically pointed at miniredis or a real test instance).
// Pass nil to clear the override and let the next call rebuild the
// real singleton.
func SetClientForTest(c *redis.Client) {
mu.Lock()
defer mu.Unlock()
defaultDB = c
clientBuilt = c != nil
}
func client() *redis.Client {
mu.Lock()
defer mu.Unlock()
if defaultDB == nil {
defaultDB = redis.NewClient(redisOptions(addr, password))
clientBuilt = true
}
return defaultDB
}
func redisOptions(target, pwd string) *redis.Options {
network := "tcp"
if strings.HasPrefix(target, "/") {
network = "unix"
}
if pwd == "" {
pwd = os.Getenv("REDISCLI_AUTH")
}
return &redis.Options{
Network: network,
Addr: target,
Password: pwd,
DB: 0,
// Fail fast when redis is absent: this client only serves the
// metrics dashboard, never a hot path. Default MaxRetries=3 +
// ReadTimeout=3s would block the perfMetrics sampler for >9s on
// a host without redis (a normal config -- CSM does not require
// redis to run).
DialTimeout: 500 * time.Millisecond,
ReadTimeout: 500 * time.Millisecond,
WriteTimeout: 500 * time.Millisecond,
MaxRetries: -1, // disable retries entirely
PoolTimeout: 500 * time.Millisecond,
PoolSize: 2,
}
}
// MemoryUsage returns the redis used_memory and maxmemory values
// from `INFO memory`, in bytes. Either may be zero if the server
// omits the field. err non-nil only on connection / protocol error.
//
// Tests can intercept via SetMemoryUsageForTest.
func MemoryUsage(ctx context.Context) (used, max uint64, err error) {
if fn := getMemoryUsageMock(); fn != nil {
return fn(ctx)
}
c := client()
if c == nil {
return 0, 0, fmt.Errorf("redisinfo: client not initialised")
}
raw, err := c.Info(ctx, "memory").Result()
if err != nil {
return 0, 0, err
}
used, max = parseMemoryInfo(raw)
return used, max, nil
}
// MemoryUsageFunc is the signature SetMemoryUsageForTest accepts.
type MemoryUsageFunc func(ctx context.Context) (used, max uint64, err error)
// KeyspaceStatsFunc is the signature SetKeyspaceStatsForTest accepts.
type KeyspaceStatsFunc func(ctx context.Context) (KeyspaceStat, error)
// ConfigGetFunc is the signature SetConfigGetForTest accepts.
type ConfigGetFunc func(ctx context.Context, name string) (string, error)
var (
mockMu sync.RWMutex
memMock MemoryUsageFunc
keyMock KeyspaceStatsFunc
configMock ConfigGetFunc
)
// SetMemoryUsageForTest installs an interceptor for MemoryUsage. Pass
// nil to clear. Production code paths must NOT call this.
func SetMemoryUsageForTest(fn MemoryUsageFunc) {
mockMu.Lock()
defer mockMu.Unlock()
memMock = fn
}
// SetKeyspaceStatsForTest installs an interceptor for KeyspaceStats
// (and Keyspace, which proxies to it). Pass nil to clear.
func SetKeyspaceStatsForTest(fn KeyspaceStatsFunc) {
mockMu.Lock()
defer mockMu.Unlock()
keyMock = fn
}
// SetConfigGetForTest installs an interceptor for ConfigGet. Pass nil
// to clear.
func SetConfigGetForTest(fn ConfigGetFunc) {
mockMu.Lock()
defer mockMu.Unlock()
configMock = fn
}
func getMemoryUsageMock() MemoryUsageFunc {
mockMu.RLock()
defer mockMu.RUnlock()
return memMock
}
func getKeyspaceStatsMock() KeyspaceStatsFunc {
mockMu.RLock()
defer mockMu.RUnlock()
return keyMock
}
func getConfigGetMock() ConfigGetFunc {
mockMu.RLock()
defer mockMu.RUnlock()
return configMock
}
func parseMemoryInfo(raw string) (used, max uint64) {
for _, line := range strings.Split(raw, "\n") {
line = strings.TrimSpace(line)
switch {
case strings.HasPrefix(line, "used_memory:"):
used, _ = strconv.ParseUint(strings.TrimSpace(strings.TrimPrefix(line, "used_memory:")), 10, 64)
case strings.HasPrefix(line, "maxmemory:"):
max, _ = strconv.ParseUint(strings.TrimSpace(strings.TrimPrefix(line, "maxmemory:")), 10, 64)
}
}
return used, max
}
// Keyspace returns the sum of `keys=N` across every db<n> line in
// `INFO keyspace`. err non-nil only on connection / protocol error.
func Keyspace(ctx context.Context) (int64, error) {
stats, err := KeyspaceStats(ctx)
if err != nil {
return 0, err
}
return stats.TotalKeys, nil
}
// KeyspaceStat is the aggregated breakdown of `INFO keyspace`. Keys
// counts all keys across all dbs; Expires counts the subset with a
// TTL applied.
type KeyspaceStat struct {
TotalKeys int64
TotalExpires int64
}
// KeyspaceStats returns the per-db sums from `INFO keyspace`.
//
// Tests can intercept via SetKeyspaceStatsForTest.
func KeyspaceStats(ctx context.Context) (KeyspaceStat, error) {
if fn := getKeyspaceStatsMock(); fn != nil {
return fn(ctx)
}
c := client()
if c == nil {
return KeyspaceStat{}, fmt.Errorf("redisinfo: client not initialised")
}
raw, err := c.Info(ctx, "keyspace").Result()
if err != nil {
return KeyspaceStat{}, err
}
return parseKeyspaceStats(raw), nil
}
// ConfigGet returns the value of a single CONFIG GET parameter (e.g.
// "maxmemory", "save", "maxmemory-policy"). Empty result returns
// ("", nil) so callers can distinguish "unset" from "connection error".
//
// Tests can intercept via SetConfigGetForTest.
func ConfigGet(ctx context.Context, name string) (string, error) {
if fn := getConfigGetMock(); fn != nil {
return fn(ctx, name)
}
c := client()
if c == nil {
return "", fmt.Errorf("redisinfo: client not initialised")
}
m, err := c.ConfigGet(ctx, name).Result()
if err != nil {
return "", err
}
if v, ok := m[name]; ok {
return v, nil
}
return "", nil
}
func parseKeyspaceInfo(raw string) int64 {
return parseKeyspaceStats(raw).TotalKeys
}
func parseKeyspaceStats(raw string) KeyspaceStat {
var stat KeyspaceStat
for _, line := range strings.Split(raw, "\n") {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, "db") {
continue
}
parts := strings.SplitN(line, ":", 2)
if len(parts) < 2 {
continue
}
for _, kv := range strings.Split(parts[1], ",") {
kv = strings.TrimSpace(kv)
eq := strings.IndexByte(kv, '=')
if eq < 0 {
continue
}
val, perr := strconv.ParseInt(kv[eq+1:], 10, 64)
if perr != nil {
continue
}
switch kv[:eq] {
case "keys":
stat.TotalKeys += val
case "expires":
stat.TotalExpires += val
}
}
}
return stat
}
package reporting
import (
"context"
"sync/atomic"
)
// Action is the node's policy for acting on central scored-set data. It is
// deliberately conservative: central data never hard-blocks on its own.
type Action string
const (
// ActionOff consumes the set for visibility only; never acts.
ActionOff Action = "off"
// ActionChallenge elevates suspicion / serves a challenge for listed IPs.
ActionChallenge Action = "challenge"
// ActionBlockIfLocalCorroborated hard-blocks only when this node also saw
// abuse from the IP and the distributed score meets the threshold.
ActionBlockIfLocalCorroborated Action = "block_if_local_corroborated"
)
var validActions = [...]Action{
ActionOff,
ActionChallenge,
ActionBlockIfLocalCorroborated,
}
// ValidActions returns every config action ParseAction recognizes.
func ValidActions() []Action {
return append([]Action(nil), validActions[:]...)
}
// IsValidAction reports whether s names a config action ParseAction recognizes.
func IsValidAction(s string) bool {
action := Action(s)
for _, valid := range validActions {
if action == valid {
return true
}
}
return false
}
// ParseAction maps a config string to an Action, defaulting to challenge for
// any unknown value (the safe default; never silently block).
func ParseAction(s string) Action {
if IsValidAction(s) {
return Action(s)
}
return ActionChallenge
}
// Decision is what the node should do about an IP given central data.
type Decision int
const (
// DecisionIgnore takes no central-driven action.
DecisionIgnore Decision = iota
// DecisionChallenge elevates suspicion / serves a challenge.
DecisionChallenge
// DecisionBlock hard-blocks (only reachable with local corroboration).
DecisionBlock
)
// DecisionInput is the per-IP context for a central-data decision.
type DecisionInput struct {
Found bool // IP present in the central scored-set
Score int // distributed score 0-100
Protected bool // firebreak: infra/CF/crawler/RFC5737/allowlist
LocallyCorroborated bool // this node independently observed abuse from the IP
}
// Decide returns the node action for an IP. Firebreaks always win: a protected
// IP is never acted on from central data. A central-only signal can at most
// challenge; a hard block requires local corroboration, the action policy, and
// the score meeting blockThreshold.
func Decide(in DecisionInput, action Action, blockThreshold int) Decision {
if in.Protected || !in.Found || action == ActionOff {
return DecisionIgnore
}
if action == ActionBlockIfLocalCorroborated && in.LocallyCorroborated && in.Score >= blockThreshold {
return DecisionBlock
}
return DecisionChallenge
}
// centralState bundles the current snapshot and its derived lookup set so both
// are swapped in a single atomic store; two separate pointers could be read in
// a torn intermediate state.
type centralState struct {
snapshot ScoredSnapshot
set *Set
}
// CentralStore holds the current verified scored-set for concurrent lookups and
// is refreshed by a Puller. The state is swapped atomically so readers on the
// block/challenge path never block on a refresh and never see a torn snapshot.
type CentralStore struct {
puller *Puller
state atomic.Pointer[centralState]
}
// NewCentralStore builds an empty store backed by puller.
func NewCentralStore(puller *Puller) *CentralStore {
cs := &CentralStore{puller: puller}
empty := ScoredSnapshot{}
cs.state.Store(¢ralState{snapshot: empty, set: NewSet(empty)})
return cs
}
// Lookup returns the scored entry for ip from the current set.
func (cs *CentralStore) Lookup(ip string) (ScoredEntry, bool) {
return cs.state.Load().set.Lookup(ip)
}
// Version returns the current set version.
func (cs *CentralStore) Version() uint64 { return cs.state.Load().set.Version() }
// Refresh pulls an update and swaps in the new state on change. On a version
// gap it retries once from a cold pull (since=0) so a node that fell behind the
// diff window recovers with a full snapshot. A lower version is rejected so a
// rolled-back or hostile endpoint cannot regress the cache.
func (cs *CentralStore) Refresh(ctx context.Context) error {
cur := cs.state.Load().snapshot
next, changed, err := cs.puller.Refresh(ctx, cur)
if err == ErrSetVersionGap {
next, changed, err = cs.puller.Refresh(ctx, ScoredSnapshot{})
}
if err != nil {
return err
}
if changed {
nextState := ¢ralState{snapshot: next, set: NewSet(next)}
for {
latest := cs.state.Load()
if next.Version < latest.snapshot.Version {
return ErrSetVersionGap
}
if next.Version == latest.snapshot.Version {
return nil
}
if cs.state.CompareAndSwap(latest, nextState) {
return nil
}
}
}
return nil
}
package reporting
import (
"context"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
// maxScoredSetBytes caps a pulled scored-set payload so a hostile or broken
// endpoint cannot exhaust memory.
const maxScoredSetBytes = 64 << 20 // 64 MiB
var (
// ErrPullStatus means the endpoint returned an unexpected HTTP status.
ErrPullStatus = errors.New("reporting: scored-set pull bad status")
// ErrPullBodyTooLarge means the endpoint returned more bytes than a node
// will verify and cache.
ErrPullBodyTooLarge = errors.New("reporting: scored-set pull body too large")
// ErrPullUnauthorized means the central service refused the node: it is not
// enrolled and approved, its key is revoked, its clock is off, or the pull
// was not signed.
ErrPullUnauthorized = errors.New("reporting: scored-set pull unauthorized")
)
// Puller fetches and verifies the signed scored-set from the central service.
// It pulls a full snapshot on a cold cache and a one-step diff thereafter,
// verifying the Ed25519 signature before applying anything.
type Puller struct {
client *http.Client
url string
pubHex string
// node signs each pull when set.
node *Target
now func() time.Time
}
// NewPuller builds a Puller for setURL, verifying against the central public
// key (hex). A nil client uses a default with a 30s timeout.
func NewPuller(client *http.Client, setURL, pubHex string) *Puller {
if client == nil {
client = &http.Client{Timeout: 30 * time.Second}
}
return &Puller{client: client, url: setURL, pubHex: pubHex}
}
// NewSignedPuller builds a Puller that signs every pull with node's Ed25519
// key, as the central abuse database requires. node must use TransportEd25519.
// A nil now uses time.Now.
func NewSignedPuller(client *http.Client, setURL, pubHex string, node Target, now func() time.Time) *Puller {
p := NewPuller(client, setURL, pubHex)
if now == nil {
now = time.Now
}
p.node, p.now = &node, now
return p
}
// Refresh fetches an update relative to current and returns the new snapshot.
// When the endpoint reports no change (304), it returns current with
// changed=false. A diff that does not apply onto current (version gap) falls
// back by returning an error so the caller retries with a full pull (since=0).
func (p *Puller) Refresh(ctx context.Context, current ScoredSnapshot) (ScoredSnapshot, bool, error) {
reqURL, err := pullURL(p.url, current.Version)
if err != nil {
return current, false, err
}
if validateErr := ValidateTargetURL(reqURL); validateErr != nil {
return current, false, validateErr
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
if err != nil {
return current, false, err
}
client := p.client
if p.node != nil {
// http.Client would turn URL credentials into Basic Authorization.
req.URL.User = nil
if signErr := p.sign(req); signErr != nil {
return current, false, signErr
}
// A redirect could disclose the signed headers to another endpoint.
// A copy keeps the shared client's own policy.
c := *p.client
c.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
client = &c
}
// The scored-set URL is operator-configured (reputation.central.set_url) and
// the response is Ed25519-verified before use; not attacker-controlled.
// #nosec G704 -- central set URL is operator config, signature-verified.
resp, err := client.Do(req)
if err != nil {
// url.Error quotes the request URL, query and any operator token included.
var ue *url.Error
if errors.As(err, &ue) {
err = ue.Err
}
return current, false, fmt.Errorf("reporting: scored-set pull: %w", err)
}
defer func() { _ = resp.Body.Close() }()
switch resp.StatusCode {
case http.StatusOK:
case http.StatusNotModified:
return current, false, nil
case http.StatusUnauthorized:
return current, false, ErrPullUnauthorized
default:
return current, false, ErrPullStatus
}
body, err := readScoredSetBody(resp.Body, maxScoredSetBytes)
if err != nil {
return current, false, err
}
sig, err := parseSetSignature(resp.Header.Get("X-CSM-Signature"))
if err != nil {
return current, false, err
}
switch resp.Header.Get("X-CSM-Kind") {
case "diff":
vd, err := OpenDiff(body, sig, p.pubHex)
if err != nil {
return current, false, err
}
next, err := ApplyDiff(current, vd)
if err != nil {
return current, false, err // version gap: caller retries full
}
return next, true, nil
default: // "snapshot" or unset
snap, err := OpenSnapshot(body, sig, p.pubHex)
if err != nil {
return current, false, err
}
return snap, true, nil
}
}
func readScoredSetBody(r io.Reader, limit int64) ([]byte, error) {
body, err := io.ReadAll(io.LimitReader(r, limit+1))
if err != nil {
return nil, err
}
if int64(len(body)) > limit {
return nil, ErrPullBodyTooLarge
}
return body, nil
}
// pullURL drops any configured since for a full pull and sets exactly one for
// a warm pull. Other query parameters are kept.
func pullURL(base string, version uint64) (string, error) {
u, err := url.Parse(base)
if err != nil {
// url.Error quotes the whole URL, which may carry an operator token.
return "", ErrInsecureURL
}
q, err := url.ParseQuery(u.RawQuery)
if err != nil {
return "", err
}
q.Del("since")
if version > 0 {
q.Set("since", strconv.FormatUint(version, 10))
}
u.RawQuery = q.Encode()
return u.String(), nil
}
// sign adds the five node headers for a bodiless GET. The report target's
// bearer token is not sent.
func (p *Puller) sign(req *http.Request) error {
nonce, err := newNonce()
if err != nil {
return err
}
env := Envelope{
NodeID: p.node.NodeID,
KeyID: p.node.KeyID,
Method: http.MethodGet,
Path: requestPath(req.URL),
BodyHash: HashBody(nil),
Timestamp: p.now().UTC().Unix(),
Nonce: nonce,
}
sig, err := SignEd25519(env, p.node.Ed25519Key)
if err != nil {
return err
}
req.Header.Set("X-CSM-Node", env.NodeID)
req.Header.Set("X-CSM-Key", env.KeyID)
req.Header.Set("X-CSM-Timestamp", strconv.FormatInt(env.Timestamp, 10))
req.Header.Set("X-CSM-Nonce", env.Nonce)
req.Header.Set("X-CSM-Signature", "ed25519="+hex.EncodeToString(sig))
return nil
}
func parseSetSignature(h string) ([]byte, error) {
scheme, hexSig, ok := strings.Cut(h, "=")
if !ok || scheme != "ed25519" {
return nil, ErrSetSignature
}
sig, err := hex.DecodeString(hexSig)
if err != nil {
return nil, ErrSetSignature
}
return sig, nil
}
package reporting
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"sync"
"time"
)
// Spooler is the production Reporter: it enqueues minimized reports to a durable
// spool and drains them to all configured targets on an interval, retrying from
// the spool when a collector is down. The daemon invokes persistence from its
// report worker so database writes do not block the alert path.
type Spooler struct {
spool *Spool
sender *Sender
targets map[string]Target
order []string
interval time.Duration
logf func(string, ...any)
now func() time.Time
drainMu sync.Mutex
retries map[string]targetRetry // guarded by drainMu
}
type targetRetry struct {
ready time.Time
backoff time.Duration
logAfter time.Time
}
var errRetryPending = errors.New("reporting: target waiting to retry")
// maxRequestedWait bounds a collector's Retry-After. Retry state lives in
// memory, so an unbounded value would park the target until a restart.
const maxRequestedWait = time.Hour
// A progressing collector may throttle normally. Preserve the failure for
// spool accounting without repeating a warning for each delivered batch.
type quietRetryError struct{ error }
func (e *quietRetryError) Unwrap() error { return e.error }
// NewSpooler builds a Spooler over a spool, a sender, and the configured
// targets. A zero interval defaults to one minute.
func NewSpooler(spool *Spool, sender *Sender, targets []Target, interval time.Duration) *Spooler {
if interval <= 0 {
interval = time.Minute
}
m := make(map[string]Target, len(targets))
order := make([]string, 0, len(targets))
seen := make(map[string]bool, len(targets))
for _, t := range targets {
m[t.Name] = t
if !seen[t.Name] {
order = append(order, t.Name)
seen[t.Name] = true
}
}
return &Spooler{spool: spool, sender: sender, targets: m, order: order, interval: interval,
logf: log.Printf, now: time.Now, retries: make(map[string]targetRetry)}
}
// Enqueue persists r for delivery to every configured target. Dropped-count
// from spool overflow is logged so a sustained outage is visible.
func (s *Spooler) Enqueue(r Report) error {
body, err := json.Marshal(r)
if err != nil {
return err
}
var failures []error
for _, name := range s.order {
dropped, err := s.spool.Enqueue(name, body)
if err != nil {
failures = append(failures, fmt.Errorf("spool enqueue for %s: %w", name, err))
continue
}
if dropped > 0 {
s.logf("reporting: spool over capacity, dropped %d oldest reports for %s", dropped, name)
}
}
return errors.Join(failures...)
}
// DrainOnce attempts one delivery pass over the spool and returns how long to
// wait before the next one. Retry deadlines belong to each target, so an
// earlier pass for another collector cannot shorten its requested wait.
func (s *Spooler) DrainOnce(ctx context.Context) time.Duration {
s.drainMu.Lock()
defer s.drainMu.Unlock()
progress := make(map[string]bool)
pending := make(map[string]time.Time)
_, err := s.spool.Drain(func(target string, body []byte) error {
t, ok := s.targets[target]
if !ok {
// Target removed from config: drop the item by reporting success.
return nil
}
retry := s.retries[target]
if s.now().Before(retry.ready) {
pending[target] = retry.ready
return errRetryPending
}
err := s.sender.Send(ctx, t, body)
if err == nil || errors.Is(err, ErrReportRefused) {
delete(s.retries, target)
progress[target] = true
return err
}
now := s.now()
delay, quiet := s.interval, false
var throttled *ThrottledError
if errors.As(err, &throttled) && throttled.RetryAfter > 0 {
// No progress across passes is an outage, even if the collector
// keeps asking for a short retry. Bound scans and warning volume.
backoff := min(s.interval, throttled.RetryAfter)
if retry.backoff > 0 {
backoff = max(backoff, retry.backoff+min(retry.backoff, s.interval-retry.backoff))
}
retry.backoff = backoff
delay = max(min(throttled.RetryAfter, maxRequestedWait), backoff)
quiet = progress[target] || now.Before(retry.logAfter)
if !quiet {
retry.logAfter = now.Add(s.interval)
}
} else {
retry.backoff = 0
retry.logAfter = time.Time{}
}
retry.ready = now.Add(delay)
s.retries[target] = retry
pending[target] = retry.ready
if quiet {
return &quietRetryError{err}
}
return err
})
next := s.interval
for _, ready := range pending {
next = min(next, max(0, ready.Sub(s.now())))
}
var failures []error
if joined, ok := err.(interface{ Unwrap() []error }); ok {
failures = joined.Unwrap()
} else if err != nil {
failures = []error{err}
}
var logged []error
for _, failure := range failures {
var quiet *quietRetryError
if errors.Is(failure, errRetryPending) || errors.As(failure, &quiet) {
continue
}
logged = append(logged, failure)
}
if err := errors.Join(logged...); err != nil {
// Failed reports remain queued; refused reports were dropped.
s.logf("reporting: delivery incomplete: %v", err)
}
return next
}
// Run drains the spool until ctx is cancelled, each pass when the previous
// one asked.
func (s *Spooler) Run(ctx context.Context) {
t := time.NewTimer(s.interval)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
t.Reset(s.DrainOnce(ctx))
}
}
}
package reporting
import (
"net"
"time"
"github.com/pidginhost/csm/internal/alert"
)
// Class is the public abuse classification sent on the wire. It is a closed set
// matching the central database's accepted classes.
type Class string
const (
ClassBruteforce Class = "bruteforce"
ClassPHPRelay Class = "php_relay"
// #nosec G101 -- abuse-class label, not a credential.
ClassCredentialStuffing Class = "credential_stuffing"
ClassBadASNEgress Class = "bad_asn_egress"
ClassSpam Class = "spam"
)
// checkClass maps a CSM finding check name to its public abuse class. Only
// confirmed-abuse checks that carry a source IP appear here; anything absent is
// never reported. This is the v1 reportable set (host_takeover is an incident
// Kind, not a finding check, and is added when the gate also taps incidents).
var checkClass = map[string]Class{
"pam_bruteforce": ClassBruteforce,
"wp_login_bruteforce": ClassBruteforce,
"xmlrpc_abuse": ClassBruteforce,
"ftp_bruteforce": ClassBruteforce,
"smtp_bruteforce": ClassBruteforce,
"mail_bruteforce": ClassBruteforce,
"admin_panel_bruteforce": ClassBruteforce,
"credential_stuffing": ClassCredentialStuffing,
"email_php_relay_abuse": ClassPHPRelay,
"bad_asn_outbound": ClassBadASNEgress,
"mail_spam_source": ClassSpam,
}
// classMinSeverity is the least severe finding a class reports.
func classMinSeverity(class Class) alert.Severity {
switch class {
case ClassBadASNEgress, ClassSpam:
return alert.High
}
return alert.Critical
}
// Classify returns the abuse class for a check name, if it is reportable.
func Classify(check string) (Class, bool) {
c, ok := checkClass[check]
return c, ok
}
// Report is the minimized payload sent for a confirmed-abuse IP. It carries no
// hostnames, accounts, mailboxes, or paths. The JSON shape matches the central
// ingest contract exactly.
type Report struct {
IP string `json:"ip"`
Class Class `json:"class"`
Count int `json:"count"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
}
// Gate decides whether a finding is reportable and, if so, the minimized report
// to send. Only findings at or above their class minimum severity whose check
// is an enabled abuse class and that carry a usable source IP are reported.
type Gate struct {
// Enabled is the set of classes the operator has turned on. Empty means
// none are reported.
Enabled map[Class]bool
// Protected is the firebreak: addresses that must never be reported as
// attackers (private and documentation space, infrastructure, Cloudflare
// edges, verified crawlers). The daemon supplies the same predicate it
// uses before acting on central intelligence. Nil protects nothing.
Protected func(net.IP) bool
// SharedSenders reports whether an address belongs to a shared mail
// service, which sends for many customers and must never be reported as
// a spam source. Only the spam class consults it, and while it is nil
// spam is refused. It runs on the alert path, so it must not do network
// I/O.
SharedSenders func(net.IP) bool
}
// Consider returns the minimized report for f, or ok=false when f must not be
// reported. The minimizer is deny-by-default: it copies only the IP, class,
// count, and timestamps, never tenant/domain/mailbox/path/process fields.
func (g Gate) Consider(f alert.Finding) (Report, bool) {
class, ok := Classify(f.Check)
if !ok || !g.Enabled[class] {
return Report{}, false
}
// Severity values grow more severe as they increase (Warning < High < Critical).
// Brute-force and relay classes need the Critical verdict; bad-ASN
// egress and spam are only ever emitted as High, so demanding Critical
// would make those classes dead options.
if f.Severity < classMinSeverity(class) {
return Report{}, false
}
ip := net.ParseIP(f.SourceIP)
if ip == nil {
return Report{}, false
}
if g.Protected != nil && g.Protected(ip) {
return Report{}, false
}
if class == ClassSpam && (g.SharedSenders == nil || g.SharedSenders(ip)) {
return Report{}, false
}
ts := f.Timestamp
if ts.IsZero() {
return Report{}, false
}
return Report{
IP: ip.String(),
Class: class,
Count: 1,
FirstSeen: ts.UTC(),
LastSeen: ts.UTC(),
}, true
}
// Reporter accepts minimized reports for later delivery, returning an error
// when persistence is incomplete. Call it outside the scan/alert hot path.
type Reporter interface {
Enqueue(Report) error
}
// Noop is the default Reporter; it discards reports. Used when reporting is
// disabled so call sites stay unconditional.
type Noop struct{}
// Enqueue discards r.
func (Noop) Enqueue(Report) error { return nil }
package reporting
import (
"bytes"
"crypto/ed25519"
"encoding/hex"
"encoding/json"
"errors"
"io"
"net"
"sort"
"time"
)
// This is the node consume side of the signed scored-set. The encoding here
// MUST stay byte-identical to the central publisher (csm-abuse-db
// internal/publish) so a node re-marshals a decoded payload to exactly the
// signed bytes. TestScoredSetGoldenCanonical pins the canonical form.
var (
// ErrSetSignature means the scored-set signature did not verify.
ErrSetSignature = errors.New("reporting: scored-set signature invalid")
// ErrSetInvalid means a decoded scored-set was malformed or noncanonical.
ErrSetInvalid = errors.New("reporting: scored-set invalid")
// ErrSetVersionGap means a diff does not apply onto the cached version.
ErrSetVersionGap = errors.New("reporting: scored-set version gap")
)
// ScoredEntry is one scored IP in the distributed set.
type ScoredEntry struct {
IP string `json:"ip"`
Score int `json:"score"`
Classes []Class `json:"classes"`
LastSeen time.Time `json:"last_seen"`
}
// ScoredSnapshot is the full scored-set at a version.
type ScoredSnapshot struct {
Version uint64 `json:"version"`
Entries []ScoredEntry `json:"entries"`
}
// ScoredDiff is an incremental update from FromVersion to ToVersion.
type ScoredDiff struct {
FromVersion uint64 `json:"from_version"`
ToVersion uint64 `json:"to_version"`
Added []ScoredEntry `json:"added"`
Removed []string `json:"removed"`
Changed []ScoredEntry `json:"changed"`
}
// VerifiedScoredDiff has passed Ed25519 verification and canonical decode.
// Its internals stay opaque so raw decoded bytes cannot be fed to ApplyDiff.
type VerifiedScoredDiff struct {
diff ScoredDiff
verified bool
}
func knownClass(c Class) bool {
switch c {
case ClassBruteforce, ClassPHPRelay, ClassCredentialStuffing, ClassBadASNEgress, ClassSpam:
return true
}
return false
}
func canonicalScoredClasses(in []Class) ([]Class, bool) {
if len(in) == 0 {
return nil, false
}
out := append([]Class(nil), in...)
sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
for i, c := range out {
if !knownClass(c) {
return nil, false
}
if i > 0 && c == out[i-1] {
return nil, false
}
}
return out, true
}
func canonicalScoredEntry(e ScoredEntry) (ScoredEntry, bool) {
ip := net.ParseIP(e.IP)
if ip == nil || e.Score < 0 || e.Score > 100 || e.LastSeen.IsZero() {
return ScoredEntry{}, false
}
classes, ok := canonicalScoredClasses(e.Classes)
if !ok {
return ScoredEntry{}, false
}
return ScoredEntry{IP: ip.String(), Score: e.Score, Classes: classes, LastSeen: e.LastSeen.UTC()}, true
}
func canonicalScoredEntries(in []ScoredEntry) ([]ScoredEntry, bool) {
out := make([]ScoredEntry, len(in))
seen := make(map[string]struct{}, len(in))
for i, e := range in {
ce, ok := canonicalScoredEntry(e)
if !ok {
return nil, false
}
if _, dup := seen[ce.IP]; dup {
return nil, false
}
seen[ce.IP] = struct{}{}
out[i] = ce
}
sort.Slice(out, func(i, j int) bool { return out[i].IP < out[j].IP })
return out, true
}
func canonicalRemovedIPs(in []string) ([]string, bool) {
out := make([]string, len(in))
seen := make(map[string]struct{}, len(in))
for i, ip := range in {
parsed := net.ParseIP(ip)
if parsed == nil {
return nil, false
}
canonical := parsed.String()
if _, dup := seen[canonical]; dup {
return nil, false
}
seen[canonical] = struct{}{}
out[i] = canonical
}
sort.Strings(out)
return out, true
}
func canonicalScoredDiff(d ScoredDiff) (ScoredDiff, bool) {
if d.ToVersion <= d.FromVersion {
return ScoredDiff{}, false
}
added, ok := canonicalScoredEntries(d.Added)
if !ok {
return ScoredDiff{}, false
}
changed, ok := canonicalScoredEntries(d.Changed)
if !ok {
return ScoredDiff{}, false
}
removed, ok := canonicalRemovedIPs(d.Removed)
if !ok {
return ScoredDiff{}, false
}
if !scoredDiffOperationsDistinct(added, changed, removed) {
return ScoredDiff{}, false
}
return ScoredDiff{
FromVersion: d.FromVersion,
ToVersion: d.ToVersion,
Added: added,
Removed: removed,
Changed: changed,
}, true
}
func scoredDiffOperationsDistinct(added, changed []ScoredEntry, removed []string) bool {
seen := make(map[string]struct{}, len(added)+len(changed)+len(removed))
for _, e := range added {
seen[e.IP] = struct{}{}
}
for _, e := range changed {
if _, dup := seen[e.IP]; dup {
return false
}
seen[e.IP] = struct{}{}
}
for _, ip := range removed {
if _, dup := seen[ip]; dup {
return false
}
seen[ip] = struct{}{}
}
return true
}
func validateDiffAgainstBase(base ScoredSnapshot, d ScoredDiff) bool {
m := indexScoredEntries(base.Entries)
for _, ip := range d.Removed {
if _, ok := m[ip]; !ok {
return false
}
}
for _, e := range d.Added {
if _, ok := m[e.IP]; ok {
return false
}
}
for _, e := range d.Changed {
if _, ok := m[e.IP]; !ok {
return false
}
}
return true
}
func indexScoredEntries(entries []ScoredEntry) map[string]ScoredEntry {
m := make(map[string]ScoredEntry, len(entries))
for _, e := range entries {
m[e.IP] = e
}
return m
}
// MarshalScoredSnapshot deterministically encodes s (for signature/re-marshal).
func MarshalScoredSnapshot(s ScoredSnapshot) ([]byte, bool) {
entries, ok := canonicalScoredEntries(s.Entries)
if !ok {
return nil, false
}
b, err := json.Marshal(ScoredSnapshot{Version: s.Version, Entries: entries})
if err != nil {
return nil, false
}
return b, true
}
// MarshalScoredDiff deterministically encodes d (for signature/re-marshal).
func MarshalScoredDiff(d ScoredDiff) ([]byte, bool) {
d, ok := canonicalScoredDiff(d)
if !ok {
return nil, false
}
b, err := json.Marshal(d)
if err != nil {
return nil, false
}
return b, true
}
// VerifyScoredSet checks sig over payload under pubHex (the central public key).
func VerifyScoredSet(payload, sig []byte, pubHex string) error {
pub, err := hex.DecodeString(pubHex)
if err != nil || len(pub) != ed25519.PublicKeySize {
return ErrSetSignature
}
if !ed25519.Verify(ed25519.PublicKey(pub), payload, sig) {
return ErrSetSignature
}
return nil
}
func decodeStrict(b []byte, v any) error {
dec := json.NewDecoder(bytes.NewReader(b))
dec.DisallowUnknownFields()
if err := dec.Decode(v); err != nil {
return err
}
var extra struct{}
if err := dec.Decode(&extra); err != io.EOF {
if err == nil {
return ErrSetInvalid
}
return err
}
return nil
}
// OpenSnapshot verifies sig over payload, then decodes and canonical-checks the
// snapshot. Verification happens before any structural trust is placed in the
// bytes.
func OpenSnapshot(payload, sig []byte, pubHex string) (ScoredSnapshot, error) {
if err := VerifyScoredSet(payload, sig, pubHex); err != nil {
return ScoredSnapshot{}, err
}
var s ScoredSnapshot
if err := decodeStrict(payload, &s); err != nil {
return ScoredSnapshot{}, ErrSetInvalid
}
// A published snapshot is always version >= 1; version 0 is the node's
// empty-cache sentinel and must never be accepted from the wire (it would
// let a hostile endpoint pin the node to perpetual cold pulls).
if s.Version == 0 {
return ScoredSnapshot{}, ErrSetInvalid
}
entries, ok := canonicalScoredEntries(s.Entries)
if !ok {
return ScoredSnapshot{}, ErrSetInvalid
}
s.Entries = entries
canon, ok := MarshalScoredSnapshot(s)
if !ok || !bytes.Equal(canon, payload) {
return ScoredSnapshot{}, ErrSetInvalid
}
return s, nil
}
// OpenDiff verifies sig over payload then decodes the diff.
func OpenDiff(payload, sig []byte, pubHex string) (VerifiedScoredDiff, error) {
if err := VerifyScoredSet(payload, sig, pubHex); err != nil {
return VerifiedScoredDiff{}, err
}
var d ScoredDiff
if err := decodeStrict(payload, &d); err != nil {
return VerifiedScoredDiff{}, ErrSetInvalid
}
d, ok := canonicalScoredDiff(d)
if !ok {
return VerifiedScoredDiff{}, ErrSetInvalid
}
canon, ok := MarshalScoredDiff(d)
if !ok || !bytes.Equal(canon, payload) {
return VerifiedScoredDiff{}, ErrSetInvalid
}
return VerifiedScoredDiff{diff: d, verified: true}, nil
}
// ApplyDiff applies a verified diff onto base, returning the resulting snapshot.
func ApplyDiff(base ScoredSnapshot, d VerifiedScoredDiff) (ScoredSnapshot, error) {
if !d.verified {
return ScoredSnapshot{}, ErrSetSignature
}
entries, ok := canonicalScoredEntries(base.Entries)
if !ok {
return ScoredSnapshot{}, ErrSetInvalid
}
base.Entries = entries
diff := d.diff
if diff.FromVersion != base.Version {
return ScoredSnapshot{}, ErrSetVersionGap
}
if !validateDiffAgainstBase(base, diff) {
return ScoredSnapshot{}, ErrSetInvalid
}
m := indexScoredEntries(base.Entries)
for _, ip := range diff.Removed {
delete(m, ip)
}
for _, e := range diff.Added {
m[e.IP] = e
}
for _, e := range diff.Changed {
m[e.IP] = e
}
out := make([]ScoredEntry, 0, len(m))
for _, e := range m {
out = append(out, e)
}
entries, ok = canonicalScoredEntries(out)
if !ok {
return ScoredSnapshot{}, ErrSetInvalid
}
return ScoredSnapshot{Version: diff.ToVersion, Entries: entries}, nil
}
// Set is an in-memory lookup over the current scored-set.
type Set struct {
version uint64
byIP map[string]ScoredEntry
}
// NewSet builds a lookup set from a snapshot.
func NewSet(s ScoredSnapshot) *Set {
m := make(map[string]ScoredEntry, len(s.Entries))
for _, e := range s.Entries {
if ce, ok := canonicalScoredEntry(e); ok {
e = ce
}
e.Classes = append([]Class(nil), e.Classes...)
m[e.IP] = e
}
return &Set{version: s.Version, byIP: m}
}
// Version returns the set's version.
func (s *Set) Version() uint64 { return s.version }
// Lookup returns the scored entry for ip, normalizing the textual form.
func (s *Set) Lookup(ip string) (ScoredEntry, bool) {
p := net.ParseIP(ip)
if p == nil {
return ScoredEntry{}, false
}
e, ok := s.byIP[p.String()]
if ok {
e.Classes = append([]Class(nil), e.Classes...)
}
return e, ok
}
// Len returns the number of scored IPs.
func (s *Set) Len() int { return len(s.byIP) }
package reporting
import (
"bytes"
"context"
"crypto/ed25519"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
// Transport selects how a report is signed for a target.
type Transport string
const (
// TransportEd25519 signs with an Ed25519 node key (federation / central DB).
TransportEd25519 Transport = "ed25519"
// TransportHMAC signs with a shared HMAC secret (private collector).
TransportHMAC Transport = "hmac"
)
// Target is one reporting destination.
type Target struct {
Name string
URL string
Transport Transport
NodeID string
KeyID string
// Ed25519Key is the node's private key for TransportEd25519.
Ed25519Key ed25519.PrivateKey
// HMACSecret is the shared secret for TransportHMAC.
HMACSecret []byte
// BearerToken is an optional Authorization bearer for HMAC collectors.
BearerToken string
}
var (
// ErrInsecureURL means a non-HTTPS target was configured for a non-loopback
// host. Reports and their auth context must not cross the network in clear.
ErrInsecureURL = errors.New("reporting: target URL must be https")
// ErrRejected means the collector rejected the report (non-2xx, non-conflict).
ErrRejected = errors.New("reporting: report rejected")
// ErrReportRefused marks a 422: the collector read this report and refused
// it, so resending the same body would be refused again. Other statuses can
// come from a proxy, a version mismatch, an outage, throttling, or the node
// not being approved yet, and hit every report alike.
ErrReportRefused = errors.New("reporting: report refused as invalid")
)
// ThrottledError is a 429 from the collector. RetryAfter is the wait its
// Retry-After asked for, or zero when the header is missing or invalid.
type ThrottledError struct {
RetryAfter time.Duration
}
func (e *ThrottledError) Error() string {
return fmt.Sprintf("status %d", http.StatusTooManyRequests)
}
func retryAfter(h http.Header, now time.Time) time.Duration {
value := strings.TrimSpace(h.Get("Retry-After"))
n, err := strconv.ParseUint(value, 10, 64)
if errors.Is(err, strconv.ErrRange) {
// ParseUint can stop at overflow before checking the suffix.
for _, digit := range value {
if digit < '0' || digit > '9' {
return 0
}
}
}
if err == nil || errors.Is(err, strconv.ErrRange) {
const longest = time.Duration(1<<63 - 1)
if n > uint64(longest/time.Second) {
return longest
}
return time.Duration(n) * time.Second
}
if deadline, err := http.ParseTime(value); err == nil {
return max(0, deadline.Sub(now))
}
return 0
}
// Sender delivers a signed report body to a target over HTTP.
type Sender struct {
client *http.Client
now func() time.Time
}
// NewSender builds a Sender. A nil client uses a default with a 15s timeout.
func NewSender(client *http.Client, now func() time.Time) *Sender {
if client == nil {
client = &http.Client{Timeout: 15 * time.Second}
}
if now == nil {
now = time.Now
}
return &Sender{client: client, now: now}
}
// ValidateTargetURL reports whether raw is allowed for report delivery without
// logging or returning the raw URL.
func ValidateTargetURL(raw string) error {
u, err := url.Parse(raw)
if err != nil || !secureURL(u) {
return ErrInsecureURL
}
return nil
}
// Send signs body for t and POSTs it. A 2xx is success; 409 Conflict (the
// collector already has this report) is treated as success. A 422 refusal of
// the report itself wraps ErrReportRefused and is not worth retrying. A 429
// wraps a ThrottledError. Other statuses and transport errors are failures the
// caller retries from the spool.
func (s *Sender) Send(ctx context.Context, t Target, body []byte) error {
u, err := url.Parse(t.URL)
if err != nil {
return fmt.Errorf("reporting: bad target url: %w", err)
}
if !secureURL(u) {
return ErrInsecureURL
}
nonce, err := newNonce()
if err != nil {
return err
}
env := Envelope{
NodeID: t.NodeID,
KeyID: t.KeyID,
Method: http.MethodPost,
Path: requestPath(u),
BodyHash: HashBody(body),
Timestamp: s.now().UTC().Unix(),
Nonce: nonce,
}
sig, scheme, err := s.sign(t, env)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, t.URL, bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-CSM-Node", env.NodeID)
req.Header.Set("X-CSM-Key", env.KeyID)
req.Header.Set("X-CSM-Timestamp", fmt.Sprintf("%d", env.Timestamp))
req.Header.Set("X-CSM-Nonce", env.Nonce)
req.Header.Set("X-CSM-Signature", scheme+"="+hex.EncodeToString(sig))
if t.BearerToken != "" {
req.Header.Set("Authorization", "Bearer "+t.BearerToken)
}
resp, err := s.do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
switch {
case resp.StatusCode >= 200 && resp.StatusCode < 300:
return nil
case resp.StatusCode == http.StatusConflict:
return nil // collector already recorded this report (replay/dup)
case resp.StatusCode == http.StatusUnprocessableEntity:
return fmt.Errorf("%w (%w): status %d", ErrRejected, ErrReportRefused, resp.StatusCode)
case resp.StatusCode == http.StatusTooManyRequests:
return fmt.Errorf("%w: %w", ErrRejected, &ThrottledError{RetryAfter: retryAfter(resp.Header, s.now())})
default:
return fmt.Errorf("%w: status %d", ErrRejected, resp.StatusCode)
}
}
func (s *Sender) do(req *http.Request) (*http.Response, error) {
client := *s.client
// The signature is bound to the configured URL path, and auth headers must
// not be replayed to a redirected endpoint.
client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error {
return http.ErrUseLastResponse
}
// The destination is an operator-configured report target, validated to
// HTTPS (or loopback HTTP) by secureURL; it is not attacker-controlled.
// #nosec G704 -- report target URL is operator config, scheme-validated.
return client.Do(req)
}
func (s *Sender) sign(t Target, env Envelope) (sig []byte, scheme string, err error) {
switch t.Transport {
case TransportEd25519:
sig, err = SignEd25519(env, t.Ed25519Key)
return sig, "ed25519", err
case TransportHMAC:
sig, err = SignHMAC(env, t.HMACSecret)
return sig, "sha256", err
default:
return nil, "", fmt.Errorf("reporting: unknown transport %q", t.Transport)
}
}
// secureURL requires https, except for loopback hosts (local collectors / tests).
func secureURL(u *url.URL) bool {
host := u.Hostname()
if host == "" {
return false
}
if u.Scheme == "https" {
return true
}
if u.Scheme == "http" {
if ip := net.ParseIP(host); ip != nil {
return ip.IsLoopback()
}
return strings.EqualFold(host, "localhost")
}
return false
}
func requestPath(u *url.URL) string {
if u.Path == "" {
return "/"
}
return u.Path
}
func newNonce() (string, error) {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
return "", err
}
return hex.EncodeToString(b[:]), nil
}
package reporting
import (
"bytes"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"sync"
"time"
bolt "go.etcd.io/bbolt"
)
// spoolItem is one queued report: the destination target name and the exact
// minimized body bytes to sign and send.
type spoolItem struct {
Target string `json:"t"`
Body []byte `json:"b"`
}
// Spool is a durable, bounded outbound queue for reports, backed by bbolt so a
// down collector or a daemon restart does not drop confirmed-abuse reports.
type Spool struct {
db *bolt.DB
bucket []byte
max int
drain sync.Mutex
mutation sync.Mutex
health spoolHealth
}
// NewSpool opens (or creates) a spool at path with a per-node entry cap.
func NewSpool(path, bucket string, max int) (*Spool, error) {
db, err := bolt.Open(path, 0o600, &bolt.Options{Timeout: 2 * time.Second})
if err != nil {
return nil, err
}
s := &Spool{db: db, bucket: []byte(bucket), max: max, health: newSpoolHealth(max)}
if err := db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists(s.bucket)
if err != nil {
return err
}
now := time.Now()
c := b.Cursor()
for k, _ := c.First(); k != nil; k, _ = c.Next() {
s.health.pending[string(k)] = &spoolWork{ticket: s.health.stats.Begin(now)}
}
return nil
}); err != nil {
_ = db.Close()
return nil, err
}
return s, nil
}
// Close releases the underlying database.
func (s *Spool) Close() error {
s.mutation.Lock()
defer s.mutation.Unlock()
return s.db.Close()
}
// Enqueue appends a report body destined for target. When the queue exceeds its
// cap, the oldest entries are dropped (FIFO) and the dropped count is returned
// so the caller can surface it; reports are best-effort under sustained outage.
func (s *Spool) Enqueue(target string, body []byte) (dropped int, err error) {
item := spoolItem{Target: target, Body: body}
enc, err := json.Marshal(item)
if err != nil {
return 0, err
}
s.mutation.Lock()
defer s.mutation.Unlock()
ticket := s.health.stats.Begin(time.Now())
ticket.Start(time.Now())
var key [8]byte
var evicted []string
err = s.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucket)
seq, seqErr := b.NextSequence()
if seqErr != nil {
return seqErr
}
binary.BigEndian.PutUint64(key[:], seq)
if e := b.Put(key[:], enc); e != nil {
return e
}
// Count current keys via the cursor; Bucket.Stats is not reliable for
// pending changes inside the same write transaction.
count := 0
c := b.Cursor()
for k, _ := c.First(); k != nil; k, _ = c.Next() {
count++
}
// Trim from the front (oldest keys sort first) until within cap.
for count > s.max {
tc := b.Cursor()
k, _ := tc.First()
if k == nil {
break
}
evicted = append(evicted, string(k))
if e := b.Delete(k); e != nil {
return e
}
count--
}
return nil
})
s.health.enqueueFailed.Store(err != nil)
if err != nil {
ticket.Reject(time.Now())
return 0, err
}
s.applyEnqueue(string(key[:]), ticket, evicted)
return len(evicted), nil
}
// Len returns the number of queued items.
func (s *Spool) Len() int {
n := 0
_ = s.db.View(func(tx *bolt.Tx) error {
n = tx.Bucket(s.bucket).Stats().KeyN
return nil
})
return n
}
// Drain processes queued items in FIFO order, calling send for each. An item is
// removed when send returns nil. A send error pauses that item's target for
// the rest of the pass: the item and the target's later items stay queued, in
// order, for the next pass, while other targets keep draining. An item the
// collector refused as invalid (ErrReportRefused) is dropped, since the same
// body would be refused again; it counts as lost unless an earlier send of it
// was acknowledged. A spool read or removal
// failure stops the pass. Drain returns how many items were delivered and the
// failures it met.
func (s *Spool) Drain(send func(target string, body []byte) error) (delivered int, err error) {
s.drain.Lock()
defer s.drain.Unlock()
paused := make(map[string]bool)
refused := make(map[string]int)
var failures []error
// A paused target contributes one failure per pass; refusals are counted
// per target so a burst of them stays one entry.
result := func(extra error) error {
errs := append(slices.Clone(failures), extra)
for _, target := range slices.Sorted(maps.Keys(refused)) {
errs = append(errs, fmt.Errorf("target %s: %d reports refused as invalid and dropped: %w",
target, refused[target], ErrReportRefused))
}
return errors.Join(errs...)
}
var after []byte
for {
key, item, work, err := s.next(after, paused)
if err != nil {
return delivered, result(err)
}
if work == nil {
// One target's success must not hide another target's stall.
s.health.sendFailed.Store(len(paused) > 0)
return delivered, result(nil)
}
after = key
sendErr, ioErr := s.deliver(key, item, work, send)
switch {
case sendErr == nil:
case errors.Is(sendErr, ErrReportRefused):
// A refused item the spool failed to remove is still queued.
if ioErr == nil {
refused[item.Target]++
}
default:
failures = append(failures, fmt.Errorf("target %s: %w", item.Target, sendErr))
paused[item.Target] = true
}
if ioErr != nil {
return delivered, result(ioErr)
}
if sendErr == nil {
delivered++
}
}
}
// next returns the first queued item after the key `after` (from the start
// when nil) whose target is not paused in this pass.
func (s *Spool) next(after []byte, paused map[string]bool) (key []byte, item spoolItem, work *spoolWork, err error) {
s.mutation.Lock()
defer s.mutation.Unlock()
empty := false
err = s.db.View(func(tx *bolt.Tx) error {
c := tx.Bucket(s.bucket).Cursor()
var k, v []byte
if after == nil {
k, v = c.First()
empty = k == nil
} else if k, v = c.Seek(after); k != nil && bytes.Equal(k, after) {
k, v = c.Next()
}
for ; k != nil; k, v = c.Next() {
var it spoolItem
if decodeErr := json.Unmarshal(v, &it); decodeErr != nil {
return decodeErr
}
if paused[it.Target] {
continue
}
key, item = append([]byte(nil), k...), it
return nil
}
return nil
})
s.health.readFailed.Store(err != nil)
if err != nil {
return nil, item, nil, err
}
if empty {
s.health.sendFailed.Store(false)
s.health.removeFailed.Store(false)
}
if key == nil {
return nil, item, nil, nil
}
work = s.health.pending[string(key)]
if work == nil {
// The spool outlives the process that wrote it. A record with no
// accounting still has to be delivered, so it is adopted here rather
// than taken as an invariant.
work = &spoolWork{ticket: s.health.stats.Begin(time.Now())}
s.health.pending[string(key)] = work
}
work.ticket.Start(time.Now())
s.health.active = work
return key, item, work, nil
}
// deliver sends one item. sendErr is the collector's answer when it did not
// accept the item; a refused item is still removed. ioErr is a spool failure.
func (s *Spool) deliver(key []byte, item spoolItem, work *spoolWork, send func(string, []byte) error) (sendErr, ioErr error) {
sent, refused, removed := false, false, false
defer func() { s.finishDelivery(work, sent, refused, removed) }()
if err := send(item.Target, item.Body); err != nil {
if !errors.Is(err, ErrReportRefused) {
return err, nil
}
refused, sendErr = true, err
} else {
sent = true
}
s.mutation.Lock()
defer s.mutation.Unlock()
if !work.evicted {
err := s.db.Update(func(tx *bolt.Tx) error {
return tx.Bucket(s.bucket).Delete(key)
})
s.health.removeFailed.Store(err != nil)
if err != nil {
return sendErr, err
}
delete(s.health.pending, string(key))
}
removed = true
return sendErr, nil
}
package reporting
import (
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
type spoolWork struct {
ticket queuehealth.Ticket
evicted bool
delivered bool
}
type spoolHealth struct {
stats *queuehealth.Tracker
pending map[string]*spoolWork // guarded by Spool.mutation
active *spoolWork
enqueueFailed atomic.Bool
readFailed atomic.Bool
removeFailed atomic.Bool
sendFailed atomic.Bool
}
func newSpoolHealth(capacity int) spoolHealth {
return spoolHealth{
// Delivery normally runs once a minute. Allow a complete retry interval
// before age alone declares the durable queue stalled.
stats: queuehealth.NewSharedCapacity(capacity, 2*time.Minute),
pending: make(map[string]*spoolWork),
}
}
// QueueStatuses uses memory only, including when a database write or sender
// stalls. Existing records are timed from open, not from an invented disk age.
func (s *Spool) QueueStatuses(now time.Time) map[string]queuehealth.Status {
status := s.health.stats.Snapshot(now)
status.LagBasis = "observed_age"
switch {
case s.health.enqueueFailed.Load() || s.health.readFailed.Load() || s.health.removeFailed.Load():
status.Status, status.Reason = "degraded", "spool_io"
case s.health.sendFailed.Load():
status.Status, status.Reason = "degraded", "delivery_failed"
}
return map[string]queuehealth.Status{"spool": status}
}
// applyEnqueue runs only after commit, under the mutation lock. An evicted
// record can still be owned by send; its outcome decides whether it was lost.
func (s *Spool) applyEnqueue(key string, ticket queuehealth.Ticket, evicted []string) {
now := time.Now()
ticket.Requeue(now)
s.health.pending[key] = &spoolWork{ticket: ticket}
for _, key := range evicted {
work := s.health.pending[key]
delete(s.health.pending, key)
switch work {
case nil:
// A record evicted from disk with no accounting cannot be
// attributed to a caller; count the report it carried as lost.
s.health.stats.Lose(now, 1)
case s.health.active:
work.evicted = true
default:
work.discard(now)
}
}
}
func (s *Spool) finishDelivery(work *spoolWork, sent, refused, removed bool) {
s.mutation.Lock()
defer s.mutation.Unlock()
if !sent && !refused {
s.health.sendFailed.Store(true)
}
now := time.Now()
// Database removal can fail after receipt. Later eviction or a failed
// retry must not turn that earlier acknowledgement into a lost report.
work.delivered = work.delivered || sent
switch {
case removed && refused:
// Lost unless an earlier attempt was acknowledged before a failed
// removal left this retry behind.
work.discard(now)
case removed:
work.ticket.Finish(now)
case work.evicted:
work.discard(now)
default:
work.ticket.Requeue(now)
}
s.health.active = nil
}
func (w *spoolWork) discard(now time.Time) {
if w.delivered {
w.ticket.Finish(now)
} else {
w.ticket.Reject(now)
}
}
// Package reporting is the node side of CSM abuse reporting (Layer A). It turns
// confirmed-abuse findings into minimized, signed reports for a central abuse
// database or a private collector.
//
// The signed-envelope wire format here MUST stay byte-identical to the central
// service's verifier (csm-abuse-db internal/envelope). It is duplicated rather
// than imported to keep this repo's build self-contained; the wire-format test
// pins the canonical bytes so any divergence fails the build.
package reporting
import (
"crypto/ed25519"
"crypto/hmac"
"crypto/sha256"
"encoding/binary"
"errors"
)
const (
bodyHashLen = sha256.Size
// maxFieldLen bounds each canonical field; matches the central verifier.
maxFieldLen = 1 << 16
)
// ErrInvalidEnvelope means the envelope is structurally unusable.
var ErrInvalidEnvelope = errors.New("reporting: invalid envelope")
// Envelope is the set of fields signed alongside a report body. Field order and
// encoding match the central verifier exactly.
type Envelope struct {
NodeID string
KeyID string
Method string
Path string
BodyHash []byte // SHA-256 of the report body, 32 bytes
Timestamp int64 // unix seconds, UTC
Nonce string
}
// HashBody returns the SHA-256 of body.
func HashBody(body []byte) []byte {
sum := sha256.Sum256(body)
return sum[:]
}
// canonical returns the deterministic, injective encoding signed over. Every
// variable-length field is length-prefixed with a 4-byte big-endian length, and
// the timestamp is appended as 8 big-endian bytes.
func (e Envelope) canonical() ([]byte, error) {
if len(e.BodyHash) != bodyHashLen {
return nil, ErrInvalidEnvelope
}
fields := [][]byte{
[]byte(e.NodeID),
[]byte(e.KeyID),
[]byte(e.Method),
[]byte(e.Path),
e.BodyHash,
[]byte(e.Nonce),
}
size := 8
for _, f := range fields {
if len(f) > maxFieldLen {
return nil, ErrInvalidEnvelope
}
size += 4 + len(f)
}
buf := make([]byte, 0, size)
var lenbuf [4]byte
for _, f := range fields {
// len(f) is bounded by maxFieldLen above, well under math.MaxUint32.
// #nosec G115 -- length validated <= maxFieldLen (64 KiB) above.
binary.BigEndian.PutUint32(lenbuf[:], uint32(len(f)))
buf = append(buf, lenbuf[:]...)
buf = append(buf, f...)
}
var ts [8]byte
// Lossless int64 -> uint64 bit reinterpretation for fixed-width encoding.
// #nosec G115 -- intentional bit-pattern reinterpret, not a narrowing cast.
binary.BigEndian.PutUint64(ts[:], uint64(e.Timestamp))
buf = append(buf, ts[:]...)
return buf, nil
}
// SignEd25519 signs the canonical envelope with an Ed25519 private key.
func SignEd25519(e Envelope, priv ed25519.PrivateKey) ([]byte, error) {
msg, err := e.canonical()
if err != nil {
return nil, err
}
if len(priv) != ed25519.PrivateKeySize {
return nil, ErrInvalidEnvelope
}
return ed25519.Sign(priv, msg), nil
}
// SignHMAC signs the canonical envelope with HMAC-SHA256 (private collector).
func SignHMAC(e Envelope, secret []byte) ([]byte, error) {
msg, err := e.canonical()
if err != nil {
return nil, err
}
if len(secret) == 0 {
return nil, ErrInvalidEnvelope
}
mac := hmac.New(sha256.New, secret)
_, _ = mac.Write(msg)
return mac.Sum(nil), nil
}
package responsereplay
import (
"errors"
"math/rand"
"net"
"slices"
"strings"
"time"
)
// Classifier supplies the decisions the live path takes from policy code this
// package cannot import. The replay command and the cross-check tests build
// it from the real registry wrappers.
type Classifier struct {
// Blockable reports whether a finding may drive a single-IP scan block.
Blockable func(Finding) bool
// ChallengeFirst reports whether the live path routes the finding to the
// challenge instead of blocking it.
ChallengeFirst func(Finding) bool
// SourceIP extracts the address the live path would block, or "".
SourceIP func(Finding) string
// ExemptBlock recognises a recorded block from a path outside the scan
// budget (challenge timeout, central intel, credential spray, incident).
ExemptBlock func(Finding) (ObservedBlock, bool)
}
// ObservedBlock is a recorded block the model applies as given.
type ObservedBlock struct {
IP string
TTL time.Duration
}
// LegacyConfig is the resolved configuration of the scan admission path.
// Callers resolve defaults first; a zero cap is invalid here, never
// "exhausted".
type LegacyConfig struct {
MaxPerHour int
DenyTempLimit int // 0 means unlimited
BlockTTL time.Duration
PendingBound int
PendingMaxAge time.Duration
// HourLocation is the zone the live clock formats the hour key in.
HourLocation *time.Location
Seed int64
}
// PendingEntry is a queued candidate. QueuedAt is zero until first queued.
type PendingEntry struct {
Finding Finding
IP string
QueuedAt time.Time
}
// TempEntry is one blocked address in firewall insertion order. A zero
// ExpiresAt is a permanent block, which never counts against the temporary
// limit and is never evicted.
type TempEntry struct {
IP string
BlockedAt time.Time
ExpiresAt time.Time
}
// LegacyState is the model's whole state, as the live path persists it.
type LegacyState struct {
HourKey string
BlocksThisHour int
Pending []PendingEntry
Entries []TempEntry
}
// BatchOutcome is what one step did. Blocked counts new scan blocks,
// ExemptBlocked new blocks from observed exempt paths. Requeued is the
// pending length after the step, not a sum. For every step
//
// prior pending + NewCandidates =
// Blocked + Requeued + AgedOut + Overflowed +
// InvalidPending + IneligiblePending + PendingSatisfied
type BatchOutcome struct {
Blocked, Requeued, AgedOut, Overflowed, Evicted, ExemptBlocked int
// ExemptObserved counts recorded exempt rows, including ones that found
// the address already blocked and changed nothing.
ExemptObserved int
// Eligible counts blockable rows. MissingIP, AlreadyBlocked and
// ChallengeSkipped are eligible rows that produced no candidate.
Eligible, MissingIP, ChallengeSkipped, AlreadyBlocked int
// NewCandidates counts new addresses after deduplication against the
// step and the queue. FirstQueued counts candidates queued for the
// first time; neither counts a requeue.
NewCandidates, FirstQueued int
InvalidPending, IneligiblePending, PendingSatisfied int
// BlockedIPs and EvictedIPs are for tests and internal accounting only;
// reports carry counts.
BlockedIPs, EvictedIPs []string
// QueueDelays holds, for each queued candidate that got blocked, the
// time from first queueing to the block. EvictionResidences holds, for
// each eviction, how long the victim had been blocked.
QueueDelays, EvictionResidences []time.Duration
PendingHighWater, LiveHighWater int
}
// Legacy models the live scan admission path: the hourly counter, the
// bounded pending queue with its age limit, the random order candidates are
// tried in, and the temporary deny limit's eviction. It is not the
// production code and makes no decision for it.
type Legacy struct {
cfg LegacyConfig
classes Classifier
state LegacyState
rng *rand.Rand
now time.Time
started bool
}
var (
errLegacyConfig = errors.New("replay: invalid legacy configuration")
errLegacyState = errors.New("replay: invalid initial state")
errTimeBackwards = errors.New("replay: batch earlier than the previous one")
)
// NewLegacy validates the configuration and copies the initial state.
func NewLegacy(cfg LegacyConfig, classes Classifier, initial LegacyState) (*Legacy, error) {
if cfg.MaxPerHour <= 0 || cfg.DenyTempLimit < 0 || cfg.BlockTTL <= 0 || cfg.PendingBound <= 0 ||
cfg.PendingMaxAge <= 0 || cfg.HourLocation == nil || classes.Blockable == nil ||
classes.ChallengeFirst == nil || classes.SourceIP == nil || classes.ExemptBlock == nil {
return nil, errLegacyConfig
}
// The live queue holds one entry per address and the live set one block
// per address; a state that breaks either cannot come from it.
queued := map[string]bool{}
for _, p := range initial.Pending {
key := normalizeIP(p.IP)
if key == "" {
key = p.IP
}
if queued[key] {
return nil, errLegacyState
}
queued[key] = true
}
for i, e := range initial.Entries {
if _, ok := canonicalKey(e.IP); !ok {
return nil, errLegacyState
}
for _, other := range initial.Entries[:i] {
if sameAddr(e.IP, other.IP) {
return nil, errLegacyState
}
}
}
return &Legacy{cfg: cfg, classes: classes, state: copyState(initial), rng: rand.New(rand.NewSource(cfg.Seed))}, nil // #nosec G404 -- reproducible replay order, not security
}
// Snapshot returns a deep copy of the current state.
func (l *Legacy) Snapshot() LegacyState { return copyState(l.state) }
func copyState(s LegacyState) LegacyState {
s.Pending = slices.Clone(s.Pending)
s.Entries = slices.Clone(s.Entries)
return s
}
// Blocked reports whether ip holds a block at now.
func (l *Legacy) Blocked(ip string, now time.Time) bool {
for _, e := range l.state.Entries {
if sameAddr(e.IP, ip) && (e.ExpiresAt.IsZero() || now.Before(e.ExpiresAt)) {
return true
}
}
return false
}
// Step applies one batch at batch.At.
func (l *Legacy) Step(batch Batch) (BatchOutcome, error) {
if l.started && batch.At.Before(l.now) {
return BatchOutcome{}, errTimeBackwards
}
now := batch.At
l.now = now
l.started = true
var out BatchOutcome
l.state.Entries = slices.DeleteFunc(l.state.Entries, func(e TempEntry) bool {
return !e.ExpiresAt.IsZero() && !e.ExpiresAt.After(now)
})
out.LiveHighWater = len(l.state.Entries)
if key := now.In(l.cfg.HourLocation).Format("2006-01-02T15"); key != l.state.HourKey {
l.state.HourKey = key
l.state.BlocksThisHour = 0
}
// Recorded exempt blocks are applied as given, before the scan stage:
// their order against a scan at the same instant is not recorded.
for _, f := range batch.Findings {
obs, ok := l.classes.ExemptBlock(f)
if !ok {
continue
}
out.ExemptObserved++
ip := normalizeIP(obs.IP)
if ip == "" || obs.TTL <= 0 || l.Blocked(ip, now) {
continue
}
l.insert(&out, ip, now, obs.TTL)
out.ExemptBlocked++
}
candidates := map[string]PendingEntry{}
for _, p := range l.state.Pending {
// The stored address is what the live drain checks; the message is
// not re-read.
ip := normalizeIP(p.IP)
switch {
case ip == "":
out.InvalidPending++
case !l.classes.Blockable(p.Finding):
out.IneligiblePending++
case !p.QueuedAt.IsZero() && now.Sub(p.QueuedAt) > l.cfg.PendingMaxAge:
out.AgedOut++
case l.Blocked(ip, now):
out.PendingSatisfied++
default:
p.IP = ip
candidates[ip] = p
}
}
l.state.Pending = nil
for _, f := range batch.Findings {
if !l.classes.Blockable(f) {
continue
}
out.Eligible++
ip := l.classes.SourceIP(f)
switch {
case ip == "":
out.MissingIP++
continue
case l.Blocked(ip, now):
out.AlreadyBlocked++
continue
case l.classes.ChallengeFirst(f):
out.ChallengeSkipped++
continue
}
// A repeat refreshes the evidence and keeps the queue time.
if existing, ok := candidates[ip]; ok {
existing.Finding = f
candidates[ip] = existing
continue
}
candidates[ip] = PendingEntry{Finding: f, IP: ip}
out.NewCandidates++
}
// The live path ranges over a Go map. Sorting the keys and shuffling
// them with the seeded generator gives a reproducible random order, not
// Go's own order or distribution.
order := make([]string, 0, len(candidates))
for ip := range candidates {
order = append(order, ip)
}
slices.Sort(order)
l.rng.Shuffle(len(order), func(i, j int) { order[i], order[j] = order[j], order[i] })
for _, ip := range order {
c := candidates[ip]
if l.state.BlocksThisHour >= l.cfg.MaxPerHour {
if c.QueuedAt.IsZero() {
c.QueuedAt = now
if len(l.state.Pending) < l.cfg.PendingBound {
out.FirstQueued++
}
}
if len(l.state.Pending) < l.cfg.PendingBound {
l.state.Pending = append(l.state.Pending, c)
out.PendingHighWater = max(out.PendingHighWater, len(l.state.Pending))
} else {
out.Overflowed++
}
continue
}
l.insert(&out, ip, now, l.cfg.BlockTTL)
l.state.BlocksThisHour++
out.Blocked++
out.BlockedIPs = append(out.BlockedIPs, ip)
if !c.QueuedAt.IsZero() {
out.QueueDelays = append(out.QueueDelays, now.Sub(c.QueuedAt))
}
}
out.Requeued = len(l.state.Pending)
return out, nil
}
// insert adds a temporary block, first evicting the soonest-expiring
// temporary entry when the limit is full, as the engine does.
func (l *Legacy) insert(out *BatchOutcome, ip string, now time.Time, ttl time.Duration) {
if l.cfg.DenyTempLimit > 0 && l.temporaryCount() >= l.cfg.DenyTempLimit {
if victim, ok := EvictionVictim(l.state.Entries, ip); ok {
i := slices.IndexFunc(l.state.Entries, func(e TempEntry) bool { return e.IP == victim })
out.Evicted++
out.EvictedIPs = append(out.EvictedIPs, victim)
out.EvictionResidences = append(out.EvictionResidences, now.Sub(l.state.Entries[i].BlockedAt))
l.state.Entries = slices.Delete(l.state.Entries, i, i+1)
}
}
l.state.Entries = append(l.state.Entries, TempEntry{IP: ip, BlockedAt: now, ExpiresAt: now.Add(ttl)})
out.LiveHighWater = max(out.LiveHighWater, len(l.state.Entries))
}
func (l *Legacy) temporaryCount() int {
n := 0
for _, e := range l.state.Entries {
if !e.ExpiresAt.IsZero() {
n++
}
}
return n
}
// EvictionVictim is the engine's rule for a full temporary limit: the first
// entry with the earliest expiry, skipping permanent entries and the address
// being blocked. Expired entries are not skipped; admission prunes them
// first. The firewall package's tests hold this to the engine's own helper.
func EvictionVictim(entries []TempEntry, exclude string) (string, bool) {
var best TempEntry
found := false
for _, e := range entries {
if e.ExpiresAt.IsZero() || sameAddr(e.IP, exclude) {
continue
}
if !found || e.ExpiresAt.Before(best.ExpiresAt) {
best, found = e, true
}
}
return best.IP, found
}
// normalizeIP is the live queue's address rule, step for step: surrounding
// space, a port and brackets are stripped, loopback and unspecified
// addresses are refused, and the result is the canonical text form.
func normalizeIP(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
if host, _, err := net.SplitHostPort(raw); err == nil {
raw = host
}
ip := net.ParseIP(strings.Trim(raw, "[]"))
if ip == nil || ip.IsLoopback() || ip.IsUnspecified() {
return ""
}
return ip.String()
}
// canonicalKey and sameAddr compare addresses exactly as the firewall
// does: by net.ParseIP's canonical text, so an IPv4-mapped IPv6 address is
// its IPv4 address and two spellings of one IPv6 address are equal.
func canonicalKey(s string) (string, bool) {
ip := net.ParseIP(s)
if ip == nil {
return "", false
}
return ip.String(), true
}
func sameAddr(a, b string) bool {
if a == b {
return true
}
x, ok := canonicalKey(a)
if !ok {
return false
}
y, ok := canonicalKey(b)
return ok && x == y
}
package responsereplay
import (
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"os"
"time"
)
// BundleManifest is the manifest the recording tool writes last, as the
// completion marker of an anonymized bundle. The recording tool's tests read
// what it writes back through ReadBundleManifest, so the two cannot drift.
type BundleManifest struct {
FormatVersion int `json:"format_version"`
Tool BundleTool `json:"tool"`
SaltFingerprint string `json:"salt_fingerprint"`
AddressMap string `json:"address_map"`
Addresses BundleAddresses `json:"addresses"`
Inputs []BundleFile `json:"inputs"`
Outputs []BundleFile `json:"outputs"`
InputManifest *BundleInventory `json:"input_manifest,omitempty"`
Join BundleJoin `json:"join"`
DroppedFields map[string]int `json:"dropped_fields"`
ActionResults map[string]int `json:"action_results"`
Coverage map[string]string `json:"coverage"`
}
type BundleTool struct {
Revision string `json:"revision"`
Dirty bool `json:"dirty"`
GoVersion string `json:"go_version"`
ModuleVersion string `json:"module_version"`
}
type BundleFile struct {
Kind string `json:"kind"`
Ordinal int `json:"ordinal"`
SHA256 string `json:"sha256"`
Records int `json:"records"`
MinTS *time.Time `json:"min_ts,omitempty"`
MaxTS *time.Time `json:"max_ts,omitempty"`
}
// BundleAddresses counts distinct addresses and the distinct pseudonyms
// they became, per family. Fewer pseudonyms than addresses means distinct
// addresses were merged, which replay cannot undo.
type BundleAddresses struct {
IPv4Addresses int `json:"ipv4_addresses"`
IPv4Pseudonyms int `json:"ipv4_pseudonyms"`
IPv6Addresses int `json:"ipv6_addresses"`
IPv6Pseudonyms int `json:"ipv6_pseudonyms"`
}
type BundleInventory struct {
SHA256 string `json:"sha256"`
Records int `json:"records"`
}
// BundleJoin is the recording tool's join counts. Matched means only that an
// action named a finding id present in the recording.
type BundleJoin struct {
FindingRows int `json:"finding_rows"`
UniqueFindingIDs int `json:"unique_finding_ids"`
DuplicateFindingRows int `json:"duplicate_finding_rows"`
FindingRowsWithoutID int `json:"finding_rows_without_id"`
FindingRowsUnstamped int `json:"finding_rows_unstamped"`
ActionRows int `json:"action_rows"`
ActionRowsWithFindingID int `json:"action_rows_with_finding_id"`
ActionRowsMatched int `json:"action_rows_matched"`
ActionRowsMissingFinding int `json:"action_rows_missing_finding"`
ActionRowsWithoutFindingID int `json:"action_rows_without_finding_id"`
DurableRows int `json:"durable_rows"`
DurableKeys int `json:"durable_keys"`
DurableIdenticalDuplicates int `json:"durable_identical_duplicates"`
DurableConflictingKeys int `json:"durable_conflicting_keys"`
FirewallRows int `json:"firewall_rows"`
}
var errBundle = errors.New("replay: manifest is not a valid recording bundle manifest")
// The closed vocabularies a manifest may carry. A report copies coverage and
// results, so nothing outside these reaches it.
var (
bundleCoverageKinds = map[string]bool{"findings": true, "actions": true, "firewall_audit": true, "ledger": true, "review": true, "firewall_id_join": true}
bundleCoverageValues = map[string]bool{"present": true, "absent": true, "not_recorded": true, "not_supplied": true, "unavailable": true, "not_applicable": true}
// The firewall's results, then the admission ledger's attempt steps
// and summary decisions.
bundleResults = map[string]bool{"applied": true, "dry_run": true, "failed": true, "refused": true, "verified": true, "unknown": true,
"reserved": true, "executing": true, "narrowed": true, "observe": true, "expired_unproven": true, "queued": true, "coalesced": true}
bundleKinds = map[string]bool{"findings": true, "actions": true, "firewall_audit": true}
)
// NonScanReasonPrefixes are the reasons automatic blocks from outside the
// scan budget record in their finding details: challenge timeout, central
// intel, credential spray and incident blocks.
var NonScanReasonPrefixes = []string{"challenge timeout: ", "central-intel (locally corroborated)", "CSM credential_spray: ", "CSM incident: "}
// ReadBundleManifest reads a recording manifest strictly and returns it with
// the SHA-256 of its exact bytes.
func ReadBundleManifest(path string) (BundleManifest, string, error) {
f, err := os.Open(path) // #nosec G304 -- operator-chosen manifest
if err != nil {
return BundleManifest{}, "", readError{0, errOpen}
}
defer f.Close()
data, err := io.ReadAll(io.LimitReader(f, maxLineBytes+1))
if err != nil {
return BundleManifest{}, "", readError{0, errRead}
}
var m BundleManifest
if err := decodeStrict(data, &m); err != nil {
return BundleManifest{}, "", err
}
if !validBundle(m) {
return BundleManifest{}, "", errBundle
}
sum := sha256.Sum256(data)
return m, hex.EncodeToString(sum[:]), nil
}
func validBundle(m BundleManifest) bool {
if m.FormatVersion != 1 {
return false
}
for _, files := range [][]BundleFile{m.Inputs, m.Outputs} {
for _, f := range files {
if !bundleKinds[f.Kind] || !sha256Hex(f.SHA256) || f.Records < 0 || f.Ordinal < 1 {
return false
}
}
}
if _, ok := m.FindingsOutput(); !ok {
return false
}
for k, v := range m.Coverage {
if !bundleCoverageKinds[k] || !bundleCoverageValues[v] {
return false
}
}
for k, count := range m.ActionResults {
if !bundleResults[k] || count < 0 {
return false
}
}
for _, count := range m.DroppedFields {
if count < 0 {
return false
}
}
ad := m.Addresses
if ad.IPv4Addresses < 0 || ad.IPv4Pseudonyms < 0 || ad.IPv6Addresses < 0 || ad.IPv6Pseudonyms < 0 ||
ad.IPv4Pseudonyms > ad.IPv4Addresses || ad.IPv6Pseudonyms > ad.IPv6Addresses {
return false
}
j := m.Join
for _, count := range []int{j.FindingRows, j.UniqueFindingIDs, j.DuplicateFindingRows, j.FindingRowsWithoutID, j.FindingRowsUnstamped,
j.ActionRows, j.ActionRowsWithFindingID, j.ActionRowsMatched, j.ActionRowsMissingFinding, j.ActionRowsWithoutFindingID,
j.DurableRows, j.DurableKeys, j.DurableIdenticalDuplicates, j.DurableConflictingKeys, j.FirewallRows} {
if count < 0 {
return false
}
}
return true
}
// FindingsOutput returns the bundle's single finding stream.
func (m BundleManifest) FindingsOutput() (BundleFile, bool) {
var found BundleFile
n := 0
for _, f := range m.Outputs {
if f.Kind == "findings" {
found = f
n++
}
}
return found, n == 1
}
func sha256Hex(s string) bool {
b, err := hex.DecodeString(s)
return err == nil && len(b) == sha256.Size && hex.EncodeToString(b) == s
}
package responsereplay
import (
"maps"
"math"
"slices"
"time"
)
// Distribution summarises samples by nearest rank: the p-th percentile is
// the ceil(p*N)-th smallest sample. Every statistic is nil when there are no
// samples; an empty population has no latency, not zero latency.
type Distribution struct {
Count int `json:"count"`
P50 *int64 `json:"p50,omitempty"`
P90 *int64 `json:"p90,omitempty"`
P99 *int64 `json:"p99,omitempty"`
Max *int64 `json:"max,omitempty"`
}
// Distribute summarises samples without reordering them.
func Distribute(samples []int64) Distribution {
if len(samples) == 0 {
return Distribution{}
}
sorted := slices.Sorted(slices.Values(samples))
rank := func(p float64) *int64 {
v := sorted[int(math.Ceil(p*float64(len(sorted))))-1]
return &v
}
return Distribution{Count: len(sorted), P50: rank(0.50), P90: rank(0.90), P99: rank(0.99), Max: rank(1)}
}
// HourlyCounts counts times into consecutive elapsed hours, from the hour
// holding first to the hour holding last. Hours with nothing in them are
// zero. Buckets are elapsed hours, so a repeated or skipped local hour
// neither merges nor drops one; admission's own counter still keys on the
// formatted local hour.
func HourlyCounts(times []time.Time, first, last time.Time) []int64 {
start := hourIndex(first)
counts := make([]int64, int(hourIndex(last)-start)+1)
for _, t := range times {
if i := int(hourIndex(t) - start); i >= 0 && i < len(counts) {
counts[i]++
}
}
return counts
}
// HourlyDistribution includes empty elapsed hours without allocating a sample
// per hour. Sparse recordings can span more than a time.Duration can hold.
func HourlyDistribution(times []time.Time, first, last time.Time) Distribution {
start, end := hourIndex(first), hourIndex(last)
occupied := map[int64]int64{}
for _, t := range times {
if hour := hourIndex(t); hour >= start && hour <= end {
occupied[hour]++
}
}
n := int(end-start) + 1
frequencies := map[int64]int{0: n - len(occupied)}
for _, count := range occupied {
frequencies[count]++
}
values := slices.Sorted(maps.Keys(frequencies))
rank := func(percent int) *int64 {
// Split the multiplication to avoid overflow for large populations.
remaining := n/100*percent + (n%100*percent+99)/100
for _, value := range values {
remaining -= frequencies[value]
if remaining <= 0 {
return &value
}
}
return nil
}
return Distribution{Count: n, P50: rank(50), P90: rank(90), P99: rank(99), Max: rank(100)}
}
func hourIndex(t time.Time) int64 {
// Truncate before dividing so pre-epoch partial hours round down too.
return t.Truncate(time.Hour).Unix() / 3600
}
package responsereplay
import (
"bufio"
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"reflect"
"sort"
"strconv"
"strings"
"sync"
"time"
"unicode/utf8"
)
// Finding is one recorded finding, reduced to what replay reads. There is no
// source address field: the audit schema has none, and replay extracts it
// from the message exactly as the live path does.
type Finding struct {
At time.Time
Check string
Severity string
Message string
Details string
FindingID string
// Ordinal is the 1-based row of the finding in its recording.
Ordinal int
}
// Batch is the findings recorded at one instant. It is inferred from equal
// timestamps: a recording holds no scan or dispatch boundaries, and it holds
// no empty scans at all.
type Batch struct {
At time.Time
Findings []Finding
}
// Recording is a read finding stream and its provenance.
type Recording struct {
Findings []Finding
// SHA256 covers the exact bytes read from disk, compressed or not.
SHA256 string
Rows int
// Unstamped counts rows with a zero timestamp. They have no replay
// position, so they are counted and left out.
Unstamped int
}
// replayError is a fixed refusal code; it never carries input bytes.
type replayError string
func (e replayError) Error() string { return string(e) }
const (
errOpen replayError = "cannot open"
errRead replayError = "read failed"
errGzip replayError = "not a complete gzip stream"
errVersion replayError = "unsupported version"
errMissingVersion replayError = "missing version"
errMissingTime replayError = "missing timestamp"
errNotObject replayError = "not one JSON object"
errSyntax replayError = "malformed JSON"
errUnknownField replayError = "unclassified field"
errDuplicateKey replayError = "repeated field"
errNull replayError = "null value"
errTrailing replayError = "data after the object"
errType replayError = "field has the wrong type"
errLineTooLong replayError = "line over the limit"
errTooLong replayError = "field over its length limit"
errTooMany replayError = "array over its element limit"
errTooDeep replayError = "nesting over the parent limit"
)
// readError places a code by line. Line 0 is the file as a whole.
type readError struct {
line int
code error
}
func (e readError) Error() string {
if e.line == 0 {
return fmt.Sprintf("recording: %v", e.code)
}
return fmt.Sprintf("recording line %d: %v", e.line, e.code)
}
func (e readError) Unwrap() error { return e.code }
// The same offline parser limits the recording tool applies.
const (
maxLineBytes = 16 << 20
maxTextBytes = 64 << 10
maxScalarBytes = 4 << 10
maxArrayItems = 1024
maxParentDepth = 32
)
const auditSchemaVersion = 1
// wireEvent mirrors the audit log's finding schema field for field; the
// external schema test fails when the real type changes. Version and
// timestamp are pointers so an absent key is told apart from a zero value.
type wireEvent struct {
V *int `json:"v"`
Timestamp *time.Time `json:"ts"`
FindingID string `json:"finding_id"`
Severity string `json:"severity"`
Check string `json:"check"`
Message string `json:"message"`
Details string `json:"details,omitempty"`
Context string `json:"context,omitempty"`
FilePath string `json:"file_path,omitempty"`
Hostname string `json:"hostname"`
TenantID string `json:"tenant_id,omitempty"`
Domain string `json:"domain,omitempty"`
Mailbox string `json:"mailbox,omitempty"`
Process *wireProcess `json:"process,omitempty"`
CIDRs []string `json:"cidrs,omitempty"`
}
type wireProcess struct {
PID int `json:"pid"`
PPID int `json:"ppid"`
UID int `json:"uid"`
User string `json:"user,omitempty"`
Account string `json:"account,omitempty"`
Comm string `json:"comm,omitempty"`
Exe string `json:"exe,omitempty"`
Cmdline []string `json:"cmdline,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"`
Parent *wireProcess `json:"parent,omitempty"`
}
// ReadFindings reads a plain or gzip JSONL recording to its end; reading
// through EOF is what surfaces a bad gzip checksum or trailer. Rows keep
// their recorded order.
func ReadFindings(path string) (Recording, error) {
fail := func(line int, code error) (Recording, error) { return Recording{}, readError{line, code} }
f, err := os.Open(path) // #nosec G304 -- operator-chosen recording
if err != nil {
return fail(0, errOpen)
}
defer f.Close()
h := sha256.New()
raw := io.TeeReader(f, h)
r := raw
compressed := strings.HasSuffix(path, ".gz")
if compressed {
zr, err := gzip.NewReader(raw)
if err != nil {
return fail(0, errGzip)
}
defer func() { _ = zr.Close() }()
r = zr
}
var rec Recording
sc := bufio.NewScanner(r)
sc.Buffer(make([]byte, 0, 1<<20), maxLineBytes+1)
line := 0
for sc.Scan() {
line++
// Only JSON whitespace separates rows; other space is part of one.
data := bytes.Trim(sc.Bytes(), " \t\r\n")
if len(data) == 0 {
continue
}
rec.Rows++
finding, stamped, err := decodeFinding(data)
if err != nil {
return fail(line, err)
}
if !stamped {
rec.Unstamped++
continue
}
finding.Ordinal = rec.Rows
rec.Findings = append(rec.Findings, finding)
}
if err := sc.Err(); err != nil {
switch {
case errors.Is(err, bufio.ErrTooLong):
return fail(line+1, errLineTooLong)
case compressed:
return fail(0, errGzip)
}
return fail(0, errRead)
}
rec.SHA256 = hex.EncodeToString(h.Sum(nil))
return rec, nil
}
// decodeFinding decodes one row. stamped is false for a zero timestamp.
func decodeFinding(data []byte) (Finding, bool, error) {
var e wireEvent
if err := decodeStrict(data, &e); err != nil {
return Finding{}, false, err
}
switch {
case e.V == nil:
return Finding{}, false, errMissingVersion
case *e.V != auditSchemaVersion:
return Finding{}, false, errVersion
case e.Timestamp == nil:
return Finding{}, false, errMissingTime
}
f := Finding{At: *e.Timestamp, Check: e.Check, Severity: e.Severity, Message: e.Message, Details: e.Details, FindingID: e.FindingID}
return f, !e.Timestamp.IsZero(), nil
}
// Batches groups findings by instant, in time order. Equal instants keep
// their recorded order whatever offset they were written with. The input is
// not modified.
func Batches(findings []Finding) []Batch {
sorted := append([]Finding(nil), findings...)
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].At.Before(sorted[j].At) })
var out []Batch
for _, f := range sorted {
if n := len(out); n > 0 && out[n-1].At.Equal(f.At) {
out[n-1].Findings = append(out[n-1].Findings, f)
continue
}
out = append(out, Batch{At: f.At, Findings: []Finding{f}})
}
return out
}
// decodeStrict decodes exactly one JSON object into v. Go's decoder matches
// keys case-insensitively, keeps the last of repeated keys and stops after
// the first value, so a token pass first checks every key against the exact
// field names of v and rejects duplicates, nulls, overlong values and
// anything after the object. The recording tool applies the same rules with
// its own copy: this package may import nothing outside the standard library.
func decodeStrict(data []byte, v any) error {
if len(data) > maxLineBytes {
return errLineTooLong
}
if !validJSONUnicode(data) {
return errSyntax
}
dec := json.NewDecoder(bytes.NewReader(data))
dec.UseNumber()
tok, err := dec.Token()
if delim, ok := tok.(json.Delim); err != nil || !ok || delim != '{' {
if err != nil && !errors.Is(err, io.EOF) {
return errSyntax
}
return errNotObject
}
if err := checkObject(dec, reflect.TypeOf(v).Elem(), map[reflect.Type]int{}); err != nil {
return err
}
if _, err := dec.Token(); !errors.Is(err, io.EOF) {
return errTrailing
}
strict := json.NewDecoder(bytes.NewReader(data))
strict.DisallowUnknownFields()
if err := strict.Decode(v); err != nil {
return errType
}
// More reports only whether a value follows in the current array or
// object; a second Decode is the check that nothing follows at all.
if err := strict.Decode(new(json.RawMessage)); !errors.Is(err, io.EOF) {
return errTrailing
}
return nil
}
var timeType = reflect.TypeFor[time.Time]()
var longTextFields = map[string]bool{"message": true, "details": true, "context": true, "file_path": true, "exe": true, "cmdline": true}
func checkObject(dec *json.Decoder, t reflect.Type, depth map[reflect.Type]int) error {
if t.Kind() == reflect.Map && t.Key().Kind() == reflect.String {
return checkMap(dec, t)
}
if t.Kind() != reflect.Struct || t == timeType {
return errType
}
if depth[t] > maxParentDepth {
return errTooDeep
}
depth[t]++
defer func() { depth[t]-- }()
fields := jsonFieldTypes(t)
seen := map[string]bool{}
for dec.More() {
tok, err := dec.Token()
if err != nil {
return errSyntax
}
key, _ := tok.(string)
field, ok := fields[key]
if !ok {
return errUnknownField
}
if seen[key] {
return errDuplicateKey
}
seen[key] = true
if err := checkValue(dec, field, key, depth); err != nil {
return err
}
}
if _, err := dec.Token(); err != nil {
return errSyntax
}
return nil
}
// checkMap reads a keyed counter object such as a manifest's coverage:
// keys are bounded and unique, values are scalars of the element type.
func checkMap(dec *json.Decoder, t reflect.Type) error {
seen := map[string]bool{}
for dec.More() {
tok, err := dec.Token()
if err != nil {
return errSyntax
}
key, _ := tok.(string)
if len(key) > maxScalarBytes {
return errTooLong
}
if seen[key] {
return errDuplicateKey
}
seen[key] = true
if err := checkValue(dec, t.Elem(), key, map[reflect.Type]int{}); err != nil {
return err
}
}
if _, err := dec.Token(); err != nil {
return errSyntax
}
return nil
}
func checkValue(dec *json.Decoder, t reflect.Type, key string, depth map[reflect.Type]int) error {
for t.Kind() == reflect.Pointer {
t = t.Elem()
}
tok, err := dec.Token()
if err != nil {
return errSyntax
}
limit := maxScalarBytes
if longTextFields[key] {
limit = maxTextBytes
}
switch tok := tok.(type) {
case nil:
return errNull
case json.Delim:
switch tok {
case '{':
return checkObject(dec, t, depth)
case '[':
if t.Kind() != reflect.Slice {
return errType
}
for n := 0; dec.More(); n++ {
if n == maxArrayItems {
return errTooMany
}
if err := checkValue(dec, t.Elem(), key, depth); err != nil {
return err
}
}
if _, err := dec.Token(); err != nil {
return errSyntax
}
return nil
}
return errSyntax
case string:
if t.Kind() != reflect.String && t != timeType {
return errType
}
if len(tok) > limit {
return errTooLong
}
if t == timeType {
stamp, err := time.Parse(time.RFC3339Nano, tok)
if err != nil {
return errType
}
// Go parses offsets its JSON encoder refuses to write back.
if _, err := stamp.MarshalJSON(); err != nil {
return errType
}
}
case json.Number:
switch t.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
reflect.Float32, reflect.Float64:
default:
return errType
}
if len(tok) > maxScalarBytes {
return errTooLong
}
case bool:
if t.Kind() != reflect.Bool {
return errType
}
}
return nil
}
// encoding/json replaces malformed UTF-8 and lone UTF-16 surrogates with
// U+FFFD. Distinct raw identifiers would then become the same join key.
// Check the original string bytes before either decoding pass loses them.
func validJSONUnicode(data []byte) bool {
if !utf8.Valid(data) {
return false
}
for i := 0; i < len(data); i++ {
if data[i] != '"' {
continue
}
for i++; i < len(data) && data[i] != '"'; i++ {
if data[i] != '\\' {
continue
}
i++
if i >= len(data) {
return false
}
if data[i] != 'u' {
continue
}
if i+4 >= len(data) {
return false
}
u, err := strconv.ParseUint(string(data[i+1:i+5]), 16, 16)
if err != nil || u >= 0xdc00 && u <= 0xdfff {
return false
}
i += 4
if u < 0xd800 || u > 0xdbff {
continue
}
if i+6 >= len(data) || data[i+1] != '\\' || data[i+2] != 'u' {
return false
}
low, err := strconv.ParseUint(string(data[i+3:i+7]), 16, 16)
if err != nil || low < 0xdc00 || low > 0xdfff {
return false
}
i += 6
}
}
return true
}
var fieldTypeCache sync.Map
func jsonFieldTypes(t reflect.Type) map[string]reflect.Type {
if cached, ok := fieldTypeCache.Load(t); ok {
return cached.(map[string]reflect.Type)
}
fields := map[string]reflect.Type{}
for i := range t.NumField() {
f := t.Field(i)
name, _, _ := strings.Cut(f.Tag.Get("json"), ",")
if !f.IsExported() || name == "-" {
continue
}
if name == "" {
name = f.Name
}
fields[name] = f.Type
}
fieldTypeCache.Store(t, fields)
return fields
}
// Package safepath pins directories for operations on tenant-controlled names.
package safepath
import (
"crypto/rand"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"golang.org/x/sys/unix"
)
// Dir owns an open directory. Operations accept single basenames only and
// never resolve a symlink, including in the final component.
type Dir struct {
file *os.File
}
// OpenDir opens an operator-controlled root. Its ancestors must be trusted;
// use OpenTarget to traverse anything beneath it controlled by an account.
func OpenDir(path string) (*Dir, error) {
// #nosec G304 -- this opens the operator-controlled anchor; all tenant components are traversed with descriptor-relative no-follow operations
f, err := os.OpenFile(path, os.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
if err != nil {
return nil, err
}
return &Dir{file: f}, nil
}
// OpenDirNoFollow pins every component from the filesystem root. Configured
// content roots can belong to tenants, including their ancestor directories.
func OpenDirNoFollow(path string) (*Dir, error) {
if !filepath.IsAbs(path) || filepath.Clean(path) != path {
return nil, fmt.Errorf("directory root must be an absolute clean path: %q", path)
}
root, err := OpenDir("/")
if err != nil {
return nil, err
}
if path == "/" {
return root, nil
}
defer func() { _ = root.Close() }()
return root.walk(strings.TrimPrefix(path, "/"), false)
}
func (d *Dir) Close() error { return d.file.Close() }
func (d *Dir) Sync() error { return d.file.Sync() }
func (d *Dir) Info() (os.FileInfo, error) { return d.file.Stat() }
func (d *Dir) ReadDir() ([]os.DirEntry, error) { return d.file.ReadDir(-1) }
// ReadDirN reads at most n entries; io.EOF follows the last batch.
func (d *Dir) ReadDirN(n int) ([]os.DirEntry, error) { return d.file.ReadDir(n) }
// Walk opens a descendant directory one no-follow component at a time.
func (d *Dir) Walk(relative string) (*Dir, error) { return d.walk(relative, false) }
// SameFile reports whether two pinned handles name the same directory.
func (d *Dir) SameFile(other *Dir) (bool, error) {
want, err := d.file.Stat()
if err != nil {
return false, err
}
got, err := other.file.Stat()
if err != nil {
return false, err
}
return os.SameFile(want, got), nil
}
func validName(name string) bool {
return name != "" && name != "." && name != ".." && !strings.ContainsAny(name, "/\x00")
}
// fileFD returns f's integer descriptor for a descriptor-relative syscall.
// Callers must keep f alive across the call.
func fileFD(f *os.File) int {
// #nosec G115 -- an open descriptor is a small non-negative value; os.File only exposes it as uintptr
return int(f.Fd())
}
// adoptFD wraps a descriptor produced by a syscall whose error was already checked.
func adoptFD(fd int, name string) *os.File {
// #nosec G115 -- a syscall that reported success returns a non-negative descriptor
return os.NewFile(uintptr(fd), name)
}
func (d *Dir) OpenFile(name string, flags int, mode os.FileMode) (*os.File, error) {
if !validName(name) {
return nil, fmt.Errorf("invalid basename %q", name)
}
fd, err := unix.Openat(fileFD(d.file), name, flags|unix.O_NOFOLLOW|unix.O_CLOEXEC|unix.O_NONBLOCK, uint32(mode.Perm()))
// The integer descriptor does not keep os.File's finalizer alive.
runtime.KeepAlive(d)
if err != nil {
return nil, &os.PathError{Op: "openat", Path: name, Err: err}
}
return adoptFD(fd, name), nil
}
// Readlink returns the target of the symlink name inside d without following it.
func (d *Dir) Readlink(name string) (string, error) {
if !validName(name) {
return "", fmt.Errorf("invalid basename %q", name)
}
buf := make([]byte, 4096)
n, err := unix.Readlinkat(fileFD(d.file), name, buf)
runtime.KeepAlive(d)
if err == nil && n == len(buf) {
err = unix.ENAMETOOLONG
}
if err != nil {
return "", &os.PathError{Op: "readlinkat", Path: name, Err: err}
}
return string(buf[:n]), nil
}
func (d *Dir) CreateTemp() (*os.File, error) {
return d.OpenFile(".csm-restore-"+rand.Text(), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
}
// CreatePrivateTemp isolates transaction names from writers of the parent.
// The parent may rename this directory, so callers must keep using its handle.
func (d *Dir) CreatePrivateTemp() (*Dir, string, error) {
name := ".csm-restore-" + rand.Text()
err := unix.Mkdirat(fileFD(d.file), name, 0700)
runtime.KeepAlive(d)
if err != nil {
return nil, "", err
}
f, err := d.OpenFile(name, os.O_RDONLY|unix.O_DIRECTORY, 0)
if err != nil {
return nil, "", err
}
var stat unix.Stat_t
err = unix.Fstat(fileFD(f), &stat)
runtime.KeepAlive(f)
if err != nil {
_ = f.Close()
return nil, "", err
}
if int(stat.Uid) != os.Geteuid() || stat.Mode&0077 != 0 {
_ = f.Close()
return nil, "", fmt.Errorf("private restore directory changed while opening")
}
return &Dir{file: f}, name, nil
}
// RemoveDir removes only an empty directory; it never traverses its contents.
func (d *Dir) RemoveDir(name string) error {
return d.unlink(name, unix.AT_REMOVEDIR)
}
func (d *Dir) Remove(name string) error {
return d.unlink(name, 0)
}
func (d *Dir) unlink(name string, flags int) error {
if !validName(name) {
return fmt.Errorf("invalid basename %q", name)
}
err := unix.Unlinkat(fileFD(d.file), name, flags)
runtime.KeepAlive(d)
if err != nil {
return &os.PathError{Op: "unlinkat", Path: name, Err: err}
}
return nil
}
// RenameTo never replaces an existing destination. ExchangeTo atomically
// swaps two existing names. Neither operation follows either name's symlink.
func (d *Dir) RenameTo(name string, dest *Dir, destName string) error {
return d.rename(name, dest, destName, false)
}
func (d *Dir) ExchangeTo(name string, dest *Dir, destName string) error {
return d.rename(name, dest, destName, true)
}
// ReplaceTo renames name over destName, replacing whatever entry is there.
// A symlink at destName is replaced, not followed.
func (d *Dir) ReplaceTo(name string, dest *Dir, destName string) error {
if !validName(name) || !validName(destName) {
return fmt.Errorf("invalid rename basenames %q, %q", name, destName)
}
err := unix.Renameat(fileFD(d.file), name, fileFD(dest.file), destName)
runtime.KeepAlive(d)
runtime.KeepAlive(dest)
if err != nil {
return &os.LinkError{Op: "renameat", Old: name, New: destName, Err: err}
}
return nil
}
func (d *Dir) rename(name string, dest *Dir, destName string, exchange bool) error {
if !validName(name) || !validName(destName) {
return fmt.Errorf("invalid rename basenames %q, %q", name, destName)
}
err := renameat(fileFD(d.file), name, fileFD(dest.file), destName, exchange)
runtime.KeepAlive(d)
runtime.KeepAlive(dest)
if err != nil {
return &os.LinkError{Op: "renameat", Old: name, New: destName, Err: err}
}
return nil
}
// Target keeps both the trusted root and the destination's parent open.
// Check detects a renamed parent; I/O uses Parent even if that check races.
type Target struct {
Parent *Dir
Name string
root *Dir
relDir string
}
func OpenTarget(rootPath, relative string, createParents bool) (*Target, error) {
if !filepath.IsLocal(relative) || filepath.Clean(relative) != relative || relative == "." {
return nil, fmt.Errorf("invalid relative restore path %q", relative)
}
root, err := openTargetRoot(rootPath)
if err != nil {
return nil, err
}
relDir := filepath.Dir(relative)
parent, err := root.walk(relDir, createParents)
if err != nil {
_ = root.Close()
return nil, err
}
return &Target{Parent: parent, Name: filepath.Base(relative), root: root, relDir: relDir}, nil
}
func (t *Target) Close() {
_ = t.Parent.Close()
_ = t.root.Close()
}
func (t *Target) Check() error {
current, err := t.root.walk(t.relDir, false)
if err != nil {
return fmt.Errorf("restore parent changed: %w", err)
}
defer func() { _ = current.Close() }()
want, err := t.Parent.file.Stat()
if err != nil {
return err
}
got, err := current.file.Stat()
if err != nil {
return err
}
if !os.SameFile(want, got) {
return fmt.Errorf("restore parent changed")
}
return nil
}
func (d *Dir) walk(relative string, create bool) (*Dir, error) {
fd, err := unix.Openat(fileFD(d.file), ".", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
runtime.KeepAlive(d)
if err != nil {
return nil, err
}
current := &Dir{file: adoptFD(fd, ".")}
if relative == "." {
return current, nil
}
for _, name := range strings.Split(relative, string(filepath.Separator)) {
if !validName(name) {
_ = current.Close()
return nil, fmt.Errorf("invalid directory component %q", name)
}
next, openErr := current.OpenFile(name, os.O_RDONLY|unix.O_DIRECTORY, 0)
if os.IsNotExist(openErr) && create {
// Public document roots need traversable parents. A concurrent
// creator is harmless only if the no-follow open accepts its inode.
mkdirErr := unix.Mkdirat(fileFD(current.file), name, 0755)
runtime.KeepAlive(current)
if mkdirErr != nil && mkdirErr != unix.EEXIST {
_ = current.Close()
return nil, mkdirErr
}
next, openErr = current.OpenFile(name, os.O_RDONLY|unix.O_DIRECTORY, 0)
}
// A concurrent restore may have created the directory but not yet
// persisted its entry. Each successful restore needs its own sync.
if openErr == nil && create {
if syncErr := current.Sync(); syncErr != nil {
_ = next.Close()
openErr = syncErr
}
}
_ = current.Close()
if openErr != nil {
return nil, openErr
}
current = &Dir{file: next}
}
return current, nil
}
//go:build linux
package safepath
import (
"os"
"runtime"
"syscall"
"time"
"golang.org/x/sys/unix"
)
// Stat pins metadata without requesting read access. Restoring the original
// mode can remove all read permissions before the final identity check.
func (d *Dir) Stat(name string) (os.FileInfo, error) {
f, err := d.OpenFile(name, unix.O_PATH, 0)
if err != nil {
return nil, err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return nil, err
}
if info.Mode()&os.ModeSymlink != 0 {
return nil, &os.PathError{Op: "fstatat", Path: name, Err: syscall.ELOOP}
}
return info, nil
}
type descriptorFileInfo struct {
name string
stat syscall.Stat_t
}
func (i *descriptorFileInfo) Name() string { return i.name }
func (i *descriptorFileInfo) Size() int64 { return i.stat.Size }
func (i *descriptorFileInfo) ModTime() time.Time { return time.Unix(i.stat.Mtim.Sec, i.stat.Mtim.Nsec) }
func (i *descriptorFileInfo) IsDir() bool { return i.Mode().IsDir() }
func (i *descriptorFileInfo) Sys() any { return &i.stat }
func (i *descriptorFileInfo) Mode() os.FileMode {
mode := os.FileMode(i.stat.Mode & 0777)
switch i.stat.Mode & syscall.S_IFMT {
case syscall.S_IFBLK:
mode |= os.ModeDevice
case syscall.S_IFCHR:
mode |= os.ModeDevice | os.ModeCharDevice
case syscall.S_IFDIR:
mode |= os.ModeDir
case syscall.S_IFIFO:
mode |= os.ModeNamedPipe
case syscall.S_IFLNK:
mode |= os.ModeSymlink
case syscall.S_IFSOCK:
mode |= os.ModeSocket
}
if i.stat.Mode&syscall.S_ISGID != 0 {
mode |= os.ModeSetgid
}
if i.stat.Mode&syscall.S_ISUID != 0 {
mode |= os.ModeSetuid
}
if i.stat.Mode&syscall.S_ISVTX != 0 {
mode |= os.ModeSticky
}
return mode
}
// Lstat reads a child entry without opening it. This remains usable while the
// child inode has an exclusive lease and still keeps lookup relative to the
// pinned directory descriptor.
func (d *Dir) Lstat(name string) (os.FileInfo, error) {
if !validName(name) {
return nil, &os.PathError{Op: "fstatat", Path: name, Err: syscall.EINVAL}
}
var raw unix.Stat_t
err := unix.Fstatat(fileFD(d.file), name, &raw, unix.AT_SYMLINK_NOFOLLOW)
runtime.KeepAlive(d)
if err != nil {
return nil, &os.PathError{Op: "fstatat", Path: name, Err: err}
}
stat := syscall.Stat_t{
Dev: raw.Dev, Ino: raw.Ino, Nlink: raw.Nlink, Mode: raw.Mode,
Uid: raw.Uid, Gid: raw.Gid, Rdev: raw.Rdev, Size: raw.Size,
Blksize: raw.Blksize, Blocks: raw.Blocks,
Atim: syscall.Timespec{Sec: raw.Atim.Sec, Nsec: raw.Atim.Nsec},
Mtim: syscall.Timespec{Sec: raw.Mtim.Sec, Nsec: raw.Mtim.Nsec},
Ctim: syscall.Timespec{Sec: raw.Ctim.Sec, Nsec: raw.Ctim.Nsec},
}
return &descriptorFileInfo{name: name, stat: stat}, nil
}
//go:build linux
package safepath
import (
"os"
"runtime"
"time"
"unsafe"
"golang.org/x/sys/unix"
)
// SetModTime changes only the pinned inode, preserving nanosecond precision.
func SetModTime(file *os.File, mtime time.Time) error {
ts, err := unix.TimeToTimespec(mtime)
if err != nil {
return err
}
times := [2]unix.Timespec{{Nsec: unix.UTIME_OMIT}, ts}
// This is the kernel's futimens ABI. A NULL pathname operates on the fd
// without the newer AT_EMPTY_PATH flag or resolving a tenant-owned name.
// #nosec G103 -- fixed timespec array passed directly to a synchronous fd-only syscall; no pointer arithmetic or user-controlled address.
_, _, errno := unix.Syscall6(unix.SYS_UTIMENSAT, file.Fd(), 0, uintptr(unsafe.Pointer(×[0])), 0, 0, 0)
runtime.KeepAlive(file)
if errno != 0 {
return &os.PathError{Op: "futimens", Path: file.Name(), Err: errno}
}
return nil
}
package safepath
import "golang.org/x/sys/unix"
func renameat(oldfd int, old string, newfd int, name string, exchange bool) error {
flags := uint(unix.RENAME_NOREPLACE)
if exchange {
flags = unix.RENAME_EXCHANGE
}
return unix.Renameat2(oldfd, old, newfd, name, flags)
}
//go:build linux
package safepath
func openTargetRoot(path string) (*Dir, error) { return OpenDirNoFollow(path) }
package safepath
import (
"path/filepath"
"strings"
)
// Within reports whether path is base or sits below it, comparing cleaned
// lexical paths: no symlinks are followed and nothing is read. The root
// directory contains every absolute path. An empty path is nowhere.
func Within(path, base string) bool {
if path == "" {
return false
}
path = filepath.Clean(path)
base = filepath.Clean(base)
if base == string(filepath.Separator) {
return filepath.IsAbs(path)
}
return path == base || strings.HasPrefix(path, base+string(filepath.Separator))
}
// Package sdnotify talks to the systemd notification socket. The daemon calls
// Ready when watchers are attached, Status to publish a one-line state visible
// in `systemctl status`, and Watchdog on a recurring ticker so systemd's
// WatchdogSec= keep-alive doesn't expire.
//
// Capture takes the systemd variables out of the process environment at
// startup and keeps them here. Everything the daemon executes afterwards --
// the PHP taint worker, every command a check shells out to -- would otherwise
// inherit NOTIFY_SOCKET and be able to write to it. The unit declares
// NotifyAccess=main, so each of those children produced a "notification
// message from PID ..., but reception only permitted for main PID" line in the
// journal. Scrubbing once at the source is what keeps a spawn site added later
// from reintroducing it.
//
// Every function is a no-op when no socket was captured (the daemon is not
// running under systemd; e.g. dev mode). That contract makes it safe to call
// these helpers unconditionally without runtime gates in the daemon code.
package sdnotify
import (
"net"
"os"
"strconv"
"sync"
"time"
)
// systemdEnv are the variables systemd passes to a service that must not reach
// any child process. Only the first two are used here; the rest describe
// socket activation this daemon does not use, and a child that read them would
// act on descriptors it was never given.
var systemdEnv = []string{
"NOTIFY_SOCKET",
"WATCHDOG_USEC",
"WATCHDOG_PID",
"LISTEN_FDS",
"LISTEN_PID",
"LISTEN_FDNAMES",
}
var (
mu sync.RWMutex
socket string
watchdogTimeout time.Duration
)
// Capture reads the systemd notification environment and removes it from the
// process environment. Called once, before the daemon spawns anything.
// Re-reading on a later call is intentional and lets tests drive it.
func Capture() {
addr := os.Getenv("NOTIFY_SOCKET")
timeout := parseWatchdogTimeout(os.Getenv("WATCHDOG_USEC"))
for _, name := range systemdEnv {
_ = os.Unsetenv(name)
}
mu.Lock()
socket, watchdogTimeout = addr, timeout
mu.Unlock()
}
func parseWatchdogTimeout(usec string) time.Duration {
if usec == "" {
return 0
}
parsed, err := strconv.ParseInt(usec, 10, 64)
if err != nil || parsed <= 0 {
return 0
}
return time.Duration(parsed) * time.Microsecond
}
// WatchdogTimeout reports the interval systemd expects keepalives within, and
// whether the unit configured one at all.
func WatchdogTimeout() (time.Duration, bool) {
mu.RLock()
defer mu.RUnlock()
return watchdogTimeout, watchdogTimeout > 0
}
// Enabled reports whether a notification socket was captured, i.e. whether
// this process is running under systemd with notifications configured.
func Enabled() bool {
mu.RLock()
defer mu.RUnlock()
return socket != ""
}
// notify sends one datagram to the captured socket. Returns (true, nil) when
// the notification was delivered, (false, nil) when no socket was captured, or
// (false, err) on a real I/O error.
func notify(state string) (bool, error) {
mu.RLock()
addr := socket
mu.RUnlock()
if addr == "" {
return false, nil
}
// A leading "@" marks an abstract socket name; the syscall layer turns it
// into the leading NUL byte the kernel expects.
conn, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: addr, Net: "unixgram"})
if err != nil {
return false, err
}
defer func() { _ = conn.Close() }()
// A bound socket can stop draining its queue. Do not let a notification
// hold startup, status updates or watchdog shutdown indefinitely.
if err := conn.SetWriteDeadline(time.Now().Add(time.Second)); err != nil {
return false, err
}
if _, err := conn.Write([]byte(state)); err != nil {
return false, err
}
return true, nil
}
// Ready signals systemd that the daemon has finished startup.
func Ready() (bool, error) {
return notify("READY=1")
}
// Reloading signals systemd that the daemon is reloading its config.
func Reloading() (bool, error) {
return notify("RELOADING=1")
}
// Status sets a single-line status string visible in `systemctl status csm`.
func Status(msg string) (bool, error) {
return notify("STATUS=" + msg)
}
// Watchdog pings the systemd watchdog. Required when the unit declares
// WatchdogSec=; without periodic pings systemd will restart the daemon
// after WatchdogSec elapses.
func Watchdog() (bool, error) {
return notify("WATCHDOG=1")
}
// Package selftest runs CSM's detection over a small bundle of samples whose
// verdicts are known, so an operator can see what the shipped rules catch
// without pointing the scanner at a production account.
//
// The bundle carries adversarial samples and benign controls. The controls are
// the half that matters when judging a scanner: a rule set that flags an
// ordinary WordPress plugin is worse than one that misses a shell.
//
// Every sample is stored base64-encoded and only decoded in memory. Endpoint
// antivirus on a developer machine, or on the server an operator runs this on,
// deletes files that look like web shells; a decoded copy on disk would make
// the bundle disappear.
package selftest
import (
"encoding/base64"
"fmt"
)
// Sample is one file with a known verdict.
type Sample struct {
Name string
// Ext is the extension the scanner is told about, including the dot.
Ext string
// Malicious is what the sample is, not what the rules do with it.
Malicious bool
Description string
// RealtimeGap and YaraGap record that the shipped rule set does not fire
// on this sample today. They are measurements, not permissions: the gates
// fail when a gap closes as well as when one opens, so closing one is a
// deliberate edit here rather than a silent change in behaviour.
//
// A gap is a gap in the signature engines only. Taint analysis, the
// behavioural checks and PHP Shield are separate layers and are not
// measured by this bundle.
RealtimeGap bool
YaraGap bool
// Encoded is the sample content, base64-encoded at rest.
Encoded string
}
// Content decodes the sample.
func (s Sample) Content() ([]byte, error) {
data, err := base64.StdEncoding.DecodeString(s.Encoded)
if err != nil {
return nil, fmt.Errorf("sample %s: %w", s.Name, err)
}
if len(data) == 0 {
return nil, fmt.Errorf("sample %s: empty content", s.Name)
}
return data, nil
}
// Result is one sample's outcome.
type Result struct {
Name string `json:"name"`
Description string `json:"description"`
Malicious bool `json:"malicious"`
Detected bool `json:"detected"`
KnownGap bool `json:"known_gap"`
Rules []string `json:"rules,omitempty"`
Pass bool `json:"pass"`
Error string `json:"error,omitempty"`
}
// ScanFunc is the detection under test: it returns the names of the rules that
// fired on the content, or an error if the scan could not complete.
type ScanFunc func(content []byte, ext string) ([]string, error)
// Engine names the rule set being measured, so a sample's recorded gap is
// compared against the engine that has it.
type Engine string
const (
// Realtime is the YAML rule set the real-time watchers use.
Realtime Engine = "realtime"
// Yara is the YARA-X rule set used by scheduled and email scanning. It is
// present only in builds compiled with the yara tag.
Yara Engine = "yara"
)
func (s Sample) gap(engine Engine) bool {
if engine == Yara {
return s.YaraGap
}
return s.RealtimeGap
}
// Run scans every sample with one engine and reports whether the outcome
// matched what is recorded for it. A sample that cannot be decoded is a
// failure, not a skip: a bundle that silently shrinks proves nothing.
func Run(engine Engine, scan ScanFunc) []Result {
results := make([]Result, 0, len(samples))
for _, sample := range samples {
result := Result{
Name: sample.Name,
Description: sample.Description,
Malicious: sample.Malicious,
KnownGap: sample.Malicious && sample.gap(engine),
}
content, err := sample.Content()
if err != nil {
result.Error = err.Error()
results = append(results, result)
continue
}
result.Rules, err = scan(content, sample.Ext)
if err != nil {
result.Error = err.Error()
results = append(results, result)
continue
}
result.Detected = len(result.Rules) > 0
result.Pass = result.Detected == (sample.Malicious && !result.KnownGap)
results = append(results, result)
}
return results
}
// Summary counts outcomes. The three failure kinds mean different things and
// are never added together: a newly missed sample is a regression, a flagged
// control is a false positive, and a recorded gap that now fires is good news
// that needs the bundle updated.
type Summary struct {
Detected int `json:"detected"`
KnownGaps int `json:"known_gaps"`
Clean int `json:"clean"`
Missed int `json:"missed"`
FalsePositives int `json:"false_positives"`
ClosedGaps int `json:"closed_gaps"`
Errors int `json:"errors"`
}
// Failed reports whether the run found anything that needs attention.
func (s Summary) Failed() bool {
return s == (Summary{}) || s.Errors > 0 || s.Missed > 0 || s.FalsePositives > 0 || s.ClosedGaps > 0
}
// Summarize counts the results.
func Summarize(results []Result) Summary {
var out Summary
for _, r := range results {
switch {
case r.Error != "":
out.Errors++
case r.KnownGap && r.Detected:
out.ClosedGaps++
case r.KnownGap:
out.KnownGaps++
case r.Malicious && r.Detected:
out.Detected++
case r.Malicious:
out.Missed++
case r.Detected:
out.FalsePositives++
default:
out.Clean++
}
}
return out
}
// Samples returns the bundle.
func Samples() []Sample { return append([]Sample(nil), samples...) }
// Package session owns browser-session identities and their storage contract.
package session
import (
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"time"
)
var (
ErrInvalid = errors.New("invalid or expired browser session")
ErrFull = errors.New("browser session capacity reached; revoke unused sessions")
)
// MaxSessions bounds persisted browser state. Active sessions are never evicted
// to admit a new login.
const MaxSessions = 128
// Record is internal state. HTTP adapters must expose a separate public view;
// neither the verifier nor the login credential fingerprint belongs in it.
type Record struct {
ID string `json:"id"`
Verifier string `json:"verifier"`
Credential string `json:"credential"`
Name string `json:"name"`
Created time.Time `json:"created"`
LastSeen time.Time `json:"last_seen"`
Expires time.Time `json:"expires"`
RemoteIP string `json:"remote_ip"`
UserAgent string `json:"user_agent"`
}
func (r Record) Valid(now time.Time, idle time.Duration) bool {
// Requests sample now before entering the repository. A newer request
// may have already committed activity by the time this one reads it.
return r.ID != "" && r.Verifier != "" && r.Credential != "" &&
!r.Created.IsZero() && !r.LastSeen.Before(r.Created) &&
now.Before(r.Expires) && now.Before(r.LastSeen.Add(idle))
}
// Repository methods are atomic relative to revocation. Access must never
// recreate a record deleted by a concurrent logout. A failed commit is an error.
type Repository interface {
ReplaceBrowserSession(Record, string, time.Time, time.Duration) error
AccessBrowserSession(string, time.Time, time.Duration, bool) (Record, error)
ListBrowserSessions(time.Time, time.Duration) ([]Record, error)
RevokeBrowserSession(string) error
ClearBrowserSessions() error
}
type Manager struct {
repo Repository
lifetime time.Duration
idle time.Duration
}
// New starts one browser-session authority. Restart invalidates every existing
// browser session; API credentials have an independent lifecycle.
func New(repo Repository, lifetime, idle time.Duration) (*Manager, error) {
if repo == nil || lifetime < time.Second || idle < time.Second || idle > lifetime {
return nil, errors.New("invalid browser session store or policy")
}
if err := repo.ClearBrowserSessions(); err != nil {
return nil, err
}
return &Manager{repo: repo, lifetime: lifetime, idle: idle}, nil
}
func Hash(secret string) string {
sum := sha256.Sum256([]byte(secret))
return hex.EncodeToString(sum[:])
}
func (m *Manager) Create(name, credential, previous, remoteIP, userAgent string, now time.Time) (string, Record, error) {
if name == "" || credential == "" {
return "", Record{}, ErrInvalid
}
var secretBytes [32]byte
if _, err := rand.Read(secretBytes[:]); err != nil {
return "", Record{}, err
}
var idBytes [16]byte
if _, err := rand.Read(idBytes[:]); err != nil {
return "", Record{}, err
}
secret := base64.RawURLEncoding.EncodeToString(secretBytes[:])
if len(userAgent) > 256 {
userAgent = userAgent[:256]
}
record := Record{ID: hex.EncodeToString(idBytes[:]), Verifier: Hash(secret), Credential: credential, Name: name,
Created: now, LastSeen: now, Expires: now.Add(m.lifetime), RemoteIP: remoteIP, UserAgent: userAgent}
previousHash := ""
if previous != "" {
previousHash = Hash(previous)
}
if err := m.repo.ReplaceBrowserSession(record, previousHash, now, m.idle); err != nil {
return "", Record{}, err
}
return secret, record, nil
}
func (m *Manager) Access(secret string, now time.Time, touch bool) (Record, error) {
if len(secret) != 43 {
return Record{}, ErrInvalid
}
return m.repo.AccessBrowserSession(Hash(secret), now, m.idle, touch)
}
func (m *Manager) List(now time.Time) ([]Record, error) {
return m.repo.ListBrowserSessions(now, m.idle)
}
func (m *Manager) Revoke(id string) error { return m.repo.RevokeBrowserSession(id) }
func (m *Manager) RevokeAll() error { return m.repo.ClearBrowserSessions() }
// Package sharedsenders keeps mail services that many unrelated senders share
// from being reported as spam sources. Each provider's sending hosts come from
// its published SPF tree: prefixes, and ptr: domains whose hosts must pass
// forward-confirmed reverse DNS. Every doubt counts as shared, so a guard that
// is not ready or a DNS failure never turns into a report.
package sharedsenders
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"maps"
"net"
"net/netip"
"os"
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"unicode/utf8"
"github.com/pidginhost/csm/internal/atomicio"
"github.com/pidginhost/csm/internal/mailranges"
)
// providers maps each shared mail service to the SPF root that lists all of
// its sending hosts.
var providers = map[string]string{
"amazonses": "amazonses.com",
"brevo": "spf.brevo.com",
"google": "_spf.google.com",
"icloud": "icloud.com",
"mailgun": "mailgun.org",
"microsoft": "spf.protection.outlook.com",
"sendgrid": "sendgrid.net",
"yahoo": "yahoo.com",
}
const (
cacheVersion = 1
maxCacheBytes = 1 << 20
maxNets = 4096
maxPTRDomains = 128
// maxAge is how long a provider's last good policy keeps the guard ready.
maxAge = 7 * 24 * time.Hour
// lookupBudget bounds all DNS work of one Shared call.
lookupBudget = 5 * time.Second
// maxPTRNames bounds the forward lookups one address can cost. No mail
// host needs more reverse names, so a longer answer fails closed.
maxPTRNames = 10
)
// Resolver is the DNS the guard uses. *net.Resolver satisfies it.
type Resolver interface {
LookupTXT(ctx context.Context, name string) ([]string, error)
LookupAddr(ctx context.Context, addr string) ([]string, error)
LookupIP(ctx context.Context, network, host string) ([]net.IP, error)
}
type cacheFile struct {
Version int `json:"version"`
Providers map[string]cacheProvider `json:"providers"`
}
type cacheProvider struct {
Root string `json:"root"`
ResolvedAt int64 `json:"resolved_at"`
Nets []string `json:"nets"`
PTR []string `json:"ptr"`
}
// provider is one provider's last good policy, as cached and as parsed.
// Published snapshots share it, so it is never modified.
type provider struct {
cache cacheProvider
nets []netip.Prefix
}
type snapshot map[string]provider
// Guard answers whether an address belongs to a shared mail service. Covers
// and Shared read an immutable snapshot, so they never wait for a refresh.
type Guard struct {
path string
// mu serialises Load and Refresh so the cache file and the published
// snapshot cannot be left from different runs.
mu sync.Mutex
snap atomic.Pointer[snapshot]
}
// New returns a guard that is not ready until Load or Refresh publishes every
// provider. cachePath is where Refresh persists policies and Load reads them.
func New(cachePath string) *Guard {
return &Guard{path: cachePath}
}
// NewForTest returns a guard holding policies, keyed by provider name, all
// resolved at resolvedAt. It has no cache file, so Refresh on it fails. It
// panics on an unknown provider name or a policy Load would reject.
func NewForTest(policies map[string]mailranges.Policy, resolvedAt time.Time) *Guard {
s := make(snapshot, len(policies))
for name, pol := range policies {
if _, ok := providers[name]; !ok {
panic(fmt.Sprintf("sharedsenders: unknown provider %q", name))
}
p, err := fromPolicy(name, pol, resolvedAt)
if err != nil {
panic(fmt.Sprintf("sharedsenders: provider %s: %v", name, err))
}
s[name] = p
}
g := &Guard{}
g.snap.Store(&s)
return g
}
// Load publishes the cache file. A missing file leaves the guard as it was.
// One bad entry fails the whole load, because skipping it would publish a
// provider with fewer senders than it has. Load ignores its time: Ready judges
// each provider's age, so a clock stepped back since the last refresh does not
// drop the cache.
func (g *Guard) Load(_ time.Time) error {
g.mu.Lock()
defer g.mu.Unlock()
f, err := os.Open(g.path) // #nosec G304 -- daemon-owned state path
if errors.Is(err, fs.ErrNotExist) {
return nil
}
if err != nil {
return fmt.Errorf("sharedsenders: read cache: %w", err)
}
defer f.Close()
data, err := io.ReadAll(io.LimitReader(f, maxCacheBytes+1))
if err != nil {
return fmt.Errorf("sharedsenders: read cache: %w", err)
}
s, err := parseCache(data)
if err != nil {
return err
}
g.snap.Store(&s)
return nil
}
// Refresh resolves every provider's SPF tree. A provider that fails keeps its
// last good policy and resolve time, and the error names it. The cache is
// written before the snapshot is published, and nothing is published when the
// write fails, so a restart never loads an older policy than one in use.
func (g *Guard) Refresh(ctx context.Context, r Resolver, now time.Time) error {
if g.path == "" {
return errors.New("sharedsenders: guard has no cache file")
}
g.mu.Lock()
defer g.mu.Unlock()
next := make(snapshot, len(providers))
maps.Copy(next, g.current())
var errs []error
for _, name := range slices.Sorted(maps.Keys(providers)) {
pol, err := mailranges.ResolveSPFPolicy(ctx, r, providers[name])
var p provider
if err == nil {
p, err = fromPolicy(name, pol, now)
}
if err != nil {
errs = append(errs, fmt.Errorf("sharedsenders: provider %s: %w", name, err))
continue
}
next[name] = p
}
data, err := encodeCache(next)
if err == nil {
err = atomicio.AtomicWrite(g.path, 0o600, data)
}
if err != nil {
return errors.Join(append(errs, fmt.Errorf("sharedsenders: write cache: %w", err))...)
}
g.snap.Store(&next)
return errors.Join(errs...)
}
// Ready reports whether every provider resolved within the last 7 days.
func (g *Guard) Ready(now time.Time) bool {
return g.current().ready(now)
}
// Covers reports whether ip must not be reported without asking DNS: the
// guard is not ready, or ip is in a provider prefix.
func (g *Guard) Covers(ip netip.Addr, now time.Time) bool {
return g.current().covers(ip.Unmap(), now)
}
// ProviderPrefix reports whether a ready guard places ip in a provider
// prefix. Readiness and membership must use one snapshot: a refresh can
// publish a policy dated after now, which Covers would treat as covering all.
func (g *Guard) ProviderPrefix(ip netip.Addr, now time.Time) bool {
s := g.current()
return ip.IsValid() && s.ready(now) && s.covers(ip.Unmap(), now)
}
// Shared reports whether ip belongs to a shared mail service: Covers is true,
// or a reverse name of ip is a provider ptr domain or under one and resolves
// back to ip. No PTR, or no matching name that resolves back, is not shared.
// Any other DNS failure, or more reverse names than it checks, is shared.
func (g *Guard) Shared(ctx context.Context, r Resolver, ip netip.Addr, now time.Time) bool {
s := g.current()
ip = ip.Unmap()
if s.covers(ip, now) {
return true
}
var domains []string
for _, p := range s {
domains = append(domains, p.cache.PTR...)
}
// No reverse name could match, so a DNS failure must not hide the sender.
if len(domains) == 0 {
return false
}
ctx, cancel := context.WithTimeout(ctx, lookupBudget)
defer cancel()
names, err := r.LookupAddr(ctx, ip.String())
if err != nil {
return !notFound(err)
}
// One forward lookup per name at most, so this also caps forward lookups.
if len(names) > maxPTRNames {
return true
}
for _, name := range names {
host, ok := ptrHost(name, domains)
if !ok {
continue
}
// Ask only for the sender's family: a combined lookup can hide a
// failure behind the other family's successful answer. Root the name
// so the resolver does not also try the host's search domains.
network := "ip6"
if ip.Is4() {
network = "ip4"
}
addrs, lookupErr := r.LookupIP(ctx, network, host+".")
if lookupErr != nil {
if notFound(lookupErr) {
continue
}
return true
}
for _, a := range addrs {
if fwd, valid := netip.AddrFromSlice(a); valid && fwd.Unmap() == ip {
return true
}
}
}
return false
}
func (g *Guard) current() snapshot {
if s := g.snap.Load(); s != nil {
return *s
}
return nil
}
func (s snapshot) ready(now time.Time) bool {
for name := range providers {
p, ok := s[name]
at := time.Unix(p.cache.ResolvedAt, 0)
if !ok || at.After(now) || now.Sub(at) > maxAge {
return false
}
}
return true
}
func (s snapshot) covers(ip netip.Addr, now time.Time) bool {
if !ip.IsValid() || !s.ready(now) {
return true
}
for _, p := range s {
for _, n := range p.nets {
if n.Contains(ip) {
return true
}
}
}
return false
}
// notFound reports whether err says the name does not exist. A not-found that
// is also temporary or a timeout is not a final answer, so it fails closed.
func notFound(err error) bool {
var dnsErr *net.DNSError
return errors.As(err, &dnsErr) && dnsErr.IsNotFound && !dnsErr.IsTemporary && !dnsErr.IsTimeout
}
// ptrHost returns name in lower case without its root dot when that is one of
// domains or a name under one. DNS ignores case only in ASCII, and Unicode
// case folding would map other bytes onto ASCII letters, so a name with any
// non-ASCII byte never matches.
func ptrHost(name string, domains []string) (string, bool) {
for i := 0; i < len(name); i++ {
if name[i] >= utf8.RuneSelf {
return "", false
}
}
host := strings.ToLower(strings.TrimSuffix(name, "."))
for _, d := range domains {
if host == d || strings.HasSuffix(host, "."+d) {
return host, true
}
}
return "", false
}
func fromPolicy(name string, pol mailranges.Policy, at time.Time) (provider, error) {
c := cacheProvider{Root: providers[name], ResolvedAt: at.Unix(), PTR: pol.PTRDomains}
for _, n := range pol.Nets {
c.Nets = append(c.Nets, n.String())
}
return parseProvider(name, c)
}
// parseProvider checks a provider entry against what Refresh writes. Prefixes
// are not checked for public address space: mailranges did that before the
// write, and an extra prefix can only stop a report, never cause one.
func parseProvider(name string, c cacheProvider) (provider, error) {
switch {
case c.Root != providers[name]:
return provider{}, fmt.Errorf("root %q, want %q", c.Root, providers[name])
case len(c.Nets) == 0 && len(c.PTR) == 0:
return provider{}, errors.New("empty sender policy")
case len(c.Nets) > maxNets:
return provider{}, fmt.Errorf("%d prefixes, over %d", len(c.Nets), maxNets)
case len(c.PTR) > maxPTRDomains:
return provider{}, fmt.Errorf("%d ptr domains, over %d", len(c.PTR), maxPTRDomains)
}
// Copies, so a published provider shares no slice with the caller, and
// never nil, so an empty list is written as [] rather than null.
c.Nets, c.PTR = append([]string{}, c.Nets...), append([]string{}, c.PTR...)
p := provider{cache: c, nets: make([]netip.Prefix, 0, len(c.Nets))}
for _, s := range c.Nets {
n, err := netip.ParsePrefix(s)
if err != nil || n != n.Masked() || n.Addr().Is4In6() {
return provider{}, fmt.Errorf("bad prefix %q", s)
}
p.nets = append(p.nets, n)
}
for _, d := range c.PTR {
if !validPTRDomain(d) {
return provider{}, fmt.Errorf("bad ptr domain %q", d)
}
}
return p, nil
}
// validPTRDomain reports whether d has the shape mailranges gives a ptr
// domain: a lower-case ASCII host name without a trailing dot whose top label
// is not all digits.
func validPTRDomain(d string) bool {
labels := strings.Split(d, ".")
if len(d) > 253 || strings.Trim(labels[len(labels)-1], "0123456789") == "" {
return false
}
for _, l := range labels {
if l == "" || len(l) > 63 || strings.Trim(l, "abcdefghijklmnopqrstuvwxyz0123456789-_") != "" {
return false
}
}
return true
}
func parseCache(data []byte) (snapshot, error) {
if len(data) > maxCacheBytes {
return nil, fmt.Errorf("sharedsenders: cache over %d bytes", maxCacheBytes)
}
var c cacheFile
if err := json.Unmarshal(data, &c); err != nil {
return nil, fmt.Errorf("sharedsenders: cache: %w", err)
}
if c.Version != cacheVersion {
return nil, fmt.Errorf("sharedsenders: cache version %d, want %d", c.Version, cacheVersion)
}
s := make(snapshot, len(providers))
for name, cp := range c.Providers {
if _, ok := providers[name]; !ok {
continue
}
p, err := parseProvider(name, cp)
if err != nil {
return nil, fmt.Errorf("sharedsenders: provider %s: %w", name, err)
}
s[name] = p
}
return s, nil
}
func encodeCache(s snapshot) ([]byte, error) {
c := cacheFile{Version: cacheVersion, Providers: make(map[string]cacheProvider, len(s))}
for name, p := range s {
c.Providers[name] = p.cache
}
data, err := json.Marshal(c)
if err == nil && len(data) > maxCacheBytes {
err = fmt.Errorf("cache of %d bytes, over %d", len(data), maxCacheBytes)
}
return data, err
}
package signatures
import (
"archive/zip"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/pidginhost/csm/internal/atomicio"
"github.com/pidginhost/csm/internal/yara"
)
const (
forgeReleasesURL = "https://api.github.com/repos/YARAHQ/yara-forge/releases/latest"
forgeHTTPTimeout = 30 * time.Second
forgeMaxZIPSize = 20 * 1024 * 1024
// forgeMaxYarSize caps the decompressed size of a single .yar entry. The
// compressed ZIP is bounded by forgeMaxZIPSize, but a zip bomb (or a
// compromised CDN / signing key) can encode a far larger decompressed
// payload; the full ruleset tier is a few MiB, so 64 MiB is generous.
forgeMaxYarSize = 64 * 1024 * 1024
)
var forgeTierAsset = map[string]string{
"core": "packages/core/yara-rules-core.yar",
"extended": "packages/extended/yara-rules-extended.yar",
"full": "packages/full/yara-rules-full.yar",
}
var forgeAtomicWrite = atomicio.AtomicWrite
// ForgeUpdate checks for a new YARA Forge release and downloads it if newer.
// A detached signature is fetched from the ZIP URL + ".sig" and verified
// against the raw ZIP content before extraction.
func ForgeUpdate(rulesDir, tier, currentVersion, signingKey string, disabledRules []string) (newVersion string, ruleCount int, err error) {
return ForgeUpdateFromURL(rulesDir, tier, currentVersion, signingKey, "", disabledRules, yara.TestCompile)
}
// ForgeUpdateFromURL is ForgeUpdate with an explicit signed ZIP source. The
// downloadURL may contain {tier} and {version}; the signature is fetched from
// the resolved ZIP URL plus ".sig". compileCheck compiles the downloaded tier
// before it replaces anything on disk; the daemon passes its compile worker so
// downloaded rules never compile in a root process. It is required: an
// omitted check must not fall back to compiling here.
func ForgeUpdateFromURL(rulesDir, tier, currentVersion, signingKey, downloadURL string, disabledRules []string, compileCheck func(string) error) (newVersion string, ruleCount int, err error) {
if compileCheck == nil {
return "", 0, errors.New("YARA Forge update: no compile check for the downloaded rules")
}
if _, ok := forgeTierAsset[tier]; !ok {
return "", 0, fmt.Errorf("unknown YARA Forge tier: %q (valid: core, extended, full)", tier)
}
if e := requireSigningKey(signingKey); e != nil {
return "", 0, e
}
if strings.TrimSpace(downloadURL) == "" {
return "", 0, fmt.Errorf("signatures.yara_forge.download_url is required: upstream YARA Forge does not publish CSM detached signatures")
}
latestTag, err := forgeResolveLatestTag(downloadURL, tier)
if err != nil {
return "", 0, fmt.Errorf("checking YARA Forge release: %w", err)
}
if latestTag == currentVersion {
return currentVersion, 0, nil
}
zipURL := forgeDownloadURL(downloadURL, tier, latestTag)
zipData, err := forgeDownload(zipURL)
if err != nil {
return "", 0, fmt.Errorf("downloading YARA Forge %s: %w", tier, err)
}
sig, err := fetchSignature(zipURL + ".sig")
if err != nil {
return "", 0, fmt.Errorf("YARA Forge signature verification required but failed: %w", err)
}
if e := VerifySignature(signingKey, zipData, sig); e != nil {
return "", 0, fmt.Errorf("YARA Forge signature invalid: %w", e)
}
assetPath := forgeTierAsset[tier]
yarContent, err := forgeExtractYar(zipData, assetPath)
if err != nil {
return "", 0, fmt.Errorf("extracting YARA Forge rules: %w", err)
}
yarContent = filterDisabledRules(yarContent, mergeDisabledRules(disabledRules))
ruleCount = countRules(yarContent)
outFile := filepath.Join(rulesDir, fmt.Sprintf("yara-forge-%s.yar", tier))
outFileExisted := false
if _, err := os.Stat(outFile); err == nil {
outFileExisted = true
} else if !os.IsNotExist(err) {
return "", 0, fmt.Errorf("checking existing Forge tier: %w", err)
}
if err := compileCheck(string(yarContent)); err != nil {
return "", 0, fmt.Errorf("YARA compilation test failed (keeping existing rules): %w", err)
}
if err := os.MkdirAll(rulesDir, 0700); err != nil {
return "", 0, fmt.Errorf("creating rules dir: %w", err)
}
if outFileExisted {
if err := removeInactiveForgeTiers(rulesDir, tier); err != nil {
return "", 0, err
}
if err := forgeAtomicWrite(outFile, 0600, yarContent); err != nil {
return "", 0, fmt.Errorf("installing rules: %w", err)
}
return latestTag, ruleCount, nil
}
if err := forgeAtomicWrite(outFile, 0600, yarContent); err != nil {
return "", 0, fmt.Errorf("installing rules: %w", err)
}
if err := removeInactiveForgeTiers(rulesDir, tier); err != nil {
return "", 0, err
}
return latestTag, ruleCount, nil
}
func removeInactiveForgeTiers(rulesDir, activeTier string) error {
for t := range forgeTierAsset {
if t == activeTier {
continue
}
path := filepath.Join(rulesDir, fmt.Sprintf("yara-forge-%s.yar", t))
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("removing inactive Forge tier %s: %w", t, err)
}
}
return nil
}
func forgeDownloadURL(tmpl, tier, version string) string {
url := strings.TrimSpace(tmpl)
url = strings.ReplaceAll(url, "{tier}", tier)
url = strings.ReplaceAll(url, "{version}", version)
return url
}
// errForgePointerAbsent signals that the mirror publishes no latest-version
// pointer (404 or a template without a {version} directory), so the caller
// should fall back to the upstream GitHub release tag.
var errForgePointerAbsent = errors.New("mirror latest pointer absent")
// forgeTagPattern constrains a version tag before it is interpolated into a
// download URL. YARA Forge tags are short date/version tokens (e.g. 20260705,
// v2026.04.11); restricting the charset stops a tampered pointer from
// redirecting the download via path traversal.
var forgeTagPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
func forgeValidTag(tag string) bool {
return forgeTagPattern.MatchString(strings.TrimSpace(tag))
}
// forgeLatestPointerURL derives the mirror's latest-version pointer URL from
// the download template. The pointer lives in the directory that holds the
// per-{version} subdirectories:
//
// https://host/csm/yara-forge/{version}/yara-forge-rules-{tier}.zip
// -> https://host/csm/yara-forge/latest
//
// Returns ("", false) when the template has no {version} path segment, in
// which case there is no version-scoped directory to point into.
func forgeLatestPointerURL(tmpl, tier string) (string, bool) {
tmpl = strings.ReplaceAll(strings.TrimSpace(tmpl), "{tier}", tier)
schemeEnd := strings.Index(tmpl, "://")
if schemeEnd < 0 {
return "", false
}
authorityStart := schemeEnd + len("://")
authorityEndRel := strings.IndexAny(tmpl[authorityStart:], "/?#")
if authorityEndRel < 0 {
return "", false
}
pathStart := authorityStart + authorityEndRel
if tmpl[pathStart] != '/' {
return "", false
}
pathEnd := len(tmpl)
if pathEndRel := strings.IndexAny(tmpl[pathStart:], "?#"); pathEndRel >= 0 {
pathEnd = pathStart + pathEndRel
}
const versionPlaceholder = "{version}"
for search := pathStart; search < pathEnd; {
rel := strings.Index(tmpl[search:pathEnd], versionPlaceholder)
if rel < 0 {
return "", false
}
i := search + rel
after := i + len(versionPlaceholder)
standaloneSegment := tmpl[i-1] == '/' && (after == pathEnd || tmpl[after] == '/')
if standaloneSegment {
base := tmpl[:i]
if strings.Contains(base, versionPlaceholder) {
return "", false
}
return base + "latest", true
}
search = after
}
return "", false
}
// forgeLatestTagFromMirror fetches the mirror's latest-version pointer and
// returns the tag it names. It returns errForgePointerAbsent when the pointer
// is not published so the caller can fall back to the GitHub release API.
func forgeLatestTagFromMirror(downloadURL, tier string) (string, error) {
pointerURL, ok := forgeLatestPointerURL(downloadURL, tier)
if !ok {
return "", errForgePointerAbsent
}
client := &http.Client{Timeout: forgeHTTPTimeout}
resp, err := client.Get(pointerURL)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return "", errForgePointerAbsent
}
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("mirror latest pointer returned %d", resp.StatusCode)
}
raw, err := io.ReadAll(io.LimitReader(resp.Body, 128))
if err != nil {
return "", fmt.Errorf("reading mirror latest pointer: %w", err)
}
tag := strings.TrimSpace(string(raw))
if !forgeValidTag(tag) {
return "", fmt.Errorf("mirror latest pointer has invalid tag %q", tag)
}
return tag, nil
}
// forgeResolveLatestTag resolves the newest downloadable version. It prefers
// the mirror's own latest pointer -- the mirror only holds versions it has
// signed and published, so resolving from it can never request a version the
// mirror lacks (the root cause of the release-day 404 gap when GitHub outran
// the weekly mirror sync). It falls back to the upstream GitHub release tag
// only when the mirror publishes no pointer, preserving prior behavior for
// older mirror layouts and download_url templates without a {version} segment.
func forgeResolveLatestTag(downloadURL, tier string) (string, error) {
tag, err := forgeLatestTagFromMirror(downloadURL, tier)
if err == nil {
return tag, nil
}
if !errors.Is(err, errForgePointerAbsent) {
return "", err
}
tag, err = forgeLatestTag()
if err != nil {
return "", err
}
if !forgeValidTag(tag) {
return "", fmt.Errorf("GitHub release has invalid tag %q", tag)
}
return tag, nil
}
func forgeLatestTag() (string, error) {
client := &http.Client{Timeout: forgeHTTPTimeout}
resp, err := client.Get(forgeReleasesURL)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("GitHub API returned %d", resp.StatusCode)
}
var release struct {
TagName string `json:"tag_name"`
}
if err := json.NewDecoder(io.LimitReader(resp.Body, 1024*1024)).Decode(&release); err != nil {
return "", fmt.Errorf("parsing release JSON: %w", err)
}
if release.TagName == "" {
return "", fmt.Errorf("empty tag_name in release")
}
return release.TagName, nil
}
func forgeDownload(url string) ([]byte, error) {
client := &http.Client{Timeout: forgeHTTPTimeout}
resp, err := client.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("download returned %d", resp.StatusCode)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, forgeMaxZIPSize))
if err != nil {
return nil, fmt.Errorf("reading response: %w", err)
}
return data, nil
}
func forgeExtractYar(zipData []byte, assetPath string) ([]byte, error) {
reader, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData)))
if err != nil {
return nil, fmt.Errorf("opening ZIP: %w", err)
}
for _, f := range reader.File {
if f.Name == assetPath {
rc, err := f.Open()
if err != nil {
return nil, fmt.Errorf("opening %s in ZIP: %w", assetPath, err)
}
defer rc.Close()
data, err := io.ReadAll(io.LimitReader(rc, forgeMaxYarSize+1))
if err != nil {
return nil, fmt.Errorf("reading %s: %w", assetPath, err)
}
if len(data) > forgeMaxYarSize {
return nil, fmt.Errorf("%s exceeds decompressed size limit of %d bytes", assetPath, forgeMaxYarSize)
}
return data, nil
}
}
return nil, fmt.Errorf("asset %s not found in ZIP", assetPath)
}
func filterDisabledRules(content []byte, disabled []string) []byte {
return yara.StripRules(content, disabled)
}
func extractRuleName(line string) string {
return yara.RuleNameFromLine(line)
}
func countRules(content []byte) int {
count := 0
for _, line := range strings.Split(string(content), "\n") {
trimmed := strings.TrimSpace(line)
if extractRuleName(trimmed) != "" {
count++
}
}
return count
}
package signatures
import "github.com/pidginhost/csm/internal/yara"
// forgeSuppressedRules returns the built-in Forge rule suppressions. The list
// and the stripping live in internal/yara so the compiler and the downloader
// agree on exactly which rules are active; see internal/yara/suppressed.go for
// the criteria an entry has to meet.
func forgeSuppressedRules() []string {
return yara.SuppressedRuleNames()
}
// mergeDisabledRules combines operator-configured disabled rule names with the
// built-in suppressions, dropping duplicates. Built-ins apply even when the
// operator configured nothing, so a clean install does not inherit a known
// false-positive flood.
func mergeDisabledRules(operator []string) []string {
builtin := forgeSuppressedRules()
seen := make(map[string]struct{}, len(operator)+len(builtin))
merged := make([]string, 0, len(operator)+len(builtin))
for _, name := range append(builtin, operator...) {
if name == "" {
continue
}
if _, dup := seen[name]; dup {
continue
}
seen[name] = struct{}{}
merged = append(merged, name)
}
return merged
}
package signatures
import (
"sync"
"sync/atomic"
)
var (
globalScanner *Scanner
globalOnce sync.Once
)
var cacheGeneration atomic.Uint64
// CacheGeneration changes on every rule load and global swap. The rule file
// version alone cannot retire verdicts after an edit to an existing rule.
func CacheGeneration() uint64 { return cacheGeneration.Load() }
// Init initializes the global scanner with rules from the given directory.
// Safe to call multiple times - only the first call takes effect.
// Call Reload() on the returned scanner to reload rules (e.g., on SIGHUP).
func Init(rulesDir string, disabled ...string) *Scanner {
globalOnce.Do(func() {
globalScanner = NewScanner(rulesDir, disabled...)
})
return globalScanner
}
// Global returns the global scanner, or nil if Init() hasn't been called.
func Global() *Scanner {
return globalScanner
}
// SetGlobal replaces the global scanner and returns the previous one.
//
// Init is guarded by a sync.Once, so only the first call in a process takes
// effect. That is right for the daemon, which initializes once, but it means
// a test helper that calls Init to install its own rules silently does
// nothing after the first test has run -- and its cleanup silently fails to
// restore anything. Tests that need to swap rules use this instead, and put
// the previous scanner back when they finish.
func SetGlobal(s *Scanner) *Scanner {
defer cacheGeneration.Add(1)
previous := globalScanner
globalScanner = s
return previous
}
package signatures
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"github.com/pidginhost/csm/internal/contenttype"
"gopkg.in/yaml.v3"
)
// Rule represents a single malware detection rule loaded from an external file.
type Rule struct {
Name string `yaml:"name"`
Description string `yaml:"description"`
Severity string `yaml:"severity"` // "critical", "high", "warning"
Category string `yaml:"category"` // "webshell", "backdoor", "phishing", "dropper", "exploit"
FileTypes []string `yaml:"file_types"` // [".php", ".html", "*"] - which extensions to scan
Patterns []string `yaml:"patterns"` // literal string patterns (case-insensitive match)
Regexes []string `yaml:"regexes"` // regex patterns (for complex matching)
RegexPairs []RegexPair `yaml:"regex_pairs"` // each nearby pair counts as one regex match
ExcludePatterns []string `yaml:"exclude_patterns"` // if any match, rule is skipped (false positive reduction)
ExcludeRegexes []string `yaml:"exclude_regexes"` // regex exclusions
MinMatch int `yaml:"min_match"` // minimum patterns that must match (default: 1)
RequireRegex bool `yaml:"require_regex"` // if true, at least one regex must match in addition to min_match
// MaxFileBytes skips the rule for content larger than this many bytes
// (0 = no limit). MaxFileBytesExemptRegexes retain high-confidence
// structural matches above the bound. This bounds weak heuristics by size
// without making padding an escape from stronger branches.
MaxFileBytes int `yaml:"max_file_bytes"`
MaxFileBytesExemptRegexes []string `yaml:"max_file_bytes_exempt_regexes"`
// Populated by compile().
compiledRegexes []*compiledRegex
compiledRegexPairs []compiledRegexPair
compiledExcludeRegexes []*compiledRegex
compiledMaxFileBytesExemptRegexes []*compiledRegex
}
// RuleFile is the top-level structure of a rules YAML file.
type RuleFile struct {
Version int `yaml:"version"`
Updated string `yaml:"updated"`
Rules []Rule `yaml:"rules"`
}
// Scanner holds compiled rules and provides file scanning.
type Scanner struct {
mu sync.RWMutex
rules []Rule
version int
rulesDir string
loadErr error
// disabled holds the rule names the operator switched off, and
// disabledUnmatched the subset that matched nothing in the loaded
// ruleset. A name nobody recognises is almost always a typo, and a
// typo here reads as "the rule is off" while it keeps firing.
disabled []string
disabledUnmatched []string
disabledCount int
}
// NewScanner creates a scanner that loads rules from the given directory.
// Returns a scanner with no rules if the directory doesn't exist (not an error).
// Any load error is retained (see LoadError) so a best-effort init does not
// hide a corrupt rules directory that silently disabled all detection.
// Rule names in disabled are not loaded. This is the same operator setting
// that filters YARA-Forge downloads, applied to the rules CSM ships, so a
// misfiring signature can be switched off without editing rule files on a
// production host.
func NewScanner(rulesDir string, disabled ...string) *Scanner {
s := &Scanner{rulesDir: rulesDir, disabled: normalizeDisabled(disabled)}
s.disabledUnmatched = append([]string(nil), s.disabled...)
_ = s.Reload() // best-effort load on init; error retained via LoadError()
return s
}
// normalizeDisabled lowercases and de-duplicates the configured names, and
// drops empty entries. Rule names in the shipped files are lowercase, and an
// operator who types one in mixed case means the same rule.
func normalizeDisabled(names []string) []string {
if len(names) == 0 {
return nil
}
seen := make(map[string]struct{}, len(names))
out := make([]string, 0, len(names))
for _, name := range names {
trimmed := strings.ToLower(strings.TrimSpace(name))
if trimmed == "" {
continue
}
if _, dup := seen[trimmed]; dup {
continue
}
seen[trimmed] = struct{}{}
out = append(out, trimmed)
}
return out
}
// DisabledRules returns the rule names this scanner was told to switch off.
func (s *Scanner) DisabledRules() []string {
s.mu.RLock()
defer s.mu.RUnlock()
return append([]string(nil), s.disabled...)
}
// DisabledRulesWithoutMatch returns the configured names that matched no rule
// in the last load attempt. Config validation surfaces these: silently accepting
// a name nobody recognises is how an operator ends up believing a rule is off.
func (s *Scanner) DisabledRulesWithoutMatch() []string {
s.mu.RLock()
defer s.mu.RUnlock()
return append([]string(nil), s.disabledUnmatched...)
}
// DisabledRuleCount counts rules omitted from the installed ruleset by config.
func (s *Scanner) DisabledRuleCount() int {
s.mu.RLock()
defer s.mu.RUnlock()
return s.disabledCount
}
// LoadError returns the error from the most recent Reload, or nil if the last
// load was clean. A non-nil value means at least one rule file failed to load
// (corrupt YAML, unreadable file, bad regex); the rules that did load are still
// installed. Callers that can alert should surface this loudly at startup.
func (s *Scanner) LoadError() error {
s.mu.RLock()
defer s.mu.RUnlock()
return s.loadErr
}
// Reload loads/reloads all .yml and .yaml rule files from the rules directory.
func (s *Scanner) Reload() error {
// After the swap, so a new generation never pairs with the old rules.
defer cacheGeneration.Add(1)
if s.rulesDir == "" {
s.setLoadErr(nil)
return nil
}
entries, err := os.ReadDir(s.rulesDir)
if err != nil {
if os.IsNotExist(err) {
s.setLoadErr(nil)
return nil // no rules dir = no rules, not an error
}
e := fmt.Errorf("reading rules dir %s: %w", s.rulesDir, err)
s.setLoadErr(e)
return e
}
var allRules []Rule
maxVersion := 0
fileCount := 0
disabledCount := 0
disabled := make(map[string]struct{}, len(s.disabled))
for _, name := range s.disabled {
disabled[name] = struct{}{}
}
disabledSeen := make(map[string]struct{}, len(disabled))
shared := make(map[string]*compiledRegex)
// One corrupt or unreadable file must not abort the whole load: an
// attacker or a fat-fingered operator dropping one bad file would
// otherwise silently disable every other signature. Bad files/rules are
// skipped and logged; their errors are aggregated and returned so a
// caller that can alert still sees the failure.
var loadErrs []error
for _, entry := range entries {
name := entry.Name()
if entry.IsDir() {
continue
}
ext := strings.ToLower(filepath.Ext(name))
if ext != ".yml" && ext != ".yaml" {
continue
}
fileCount++
path := filepath.Join(s.rulesDir, name)
// #nosec G304 -- filepath.Join under operator-configured rulesDir.
data, err := os.ReadFile(path)
if err != nil {
loadErrs = append(loadErrs, fmt.Errorf("reading %s: %w", path, err))
fmt.Fprintf(os.Stderr, "signatures: skipping %s: %v\n", path, err)
continue
}
var rf RuleFile
if err := yaml.Unmarshal(data, &rf); err != nil {
loadErrs = append(loadErrs, fmt.Errorf("parsing %s: %w", path, err))
fmt.Fprintf(os.Stderr, "signatures: skipping %s: %v\n", path, err)
continue
}
// Compile rules
rulesBeforeFile := len(allRules)
for i := range rf.Rules {
rule := &rf.Rules[i]
if _, off := disabled[strings.ToLower(rule.Name)]; off {
disabledSeen[strings.ToLower(rule.Name)] = struct{}{}
disabledCount++
continue
}
if err := rule.compileShared(shared); err != nil {
loadErrs = append(loadErrs, fmt.Errorf("compiling rule %q in %s: %w", rule.Name, path, err))
fmt.Fprintf(os.Stderr, "signatures: skipping rule %q in %s: %v\n", rule.Name, path, err)
continue
}
if rule.MinMatch == 0 {
rule.MinMatch = 1
}
allRules = append(allRules, *rule)
}
if len(allRules) > rulesBeforeFile && rf.Version > maxVersion {
maxVersion = rf.Version
}
}
if fileCount == 0 {
s.mu.RLock()
hadRules := len(s.rules) > 0
s.mu.RUnlock()
if hadRules {
e := fmt.Errorf("no signature rule files found in %s", s.rulesDir)
s.setLoadErr(e)
return e
}
s.setLoadErr(nil)
return nil
}
var unmatched []string
for _, name := range s.disabled {
if _, seen := disabledSeen[name]; !seen {
unmatched = append(unmatched, name)
}
}
// Keep the old set on failure, but publish a clean load that config
// intentionally emptied. Retaining old rules in that case scans a set
// that is no longer on disk.
if len(allRules) == 0 && (disabledCount == 0 || len(loadErrs) > 0) {
err := errors.Join(append(loadErrs, fmt.Errorf("no signature rules loaded from %s", s.rulesDir))...)
s.mu.Lock()
s.loadErr = err
s.disabledUnmatched = unmatched
s.mu.Unlock()
return err
}
s.mu.Lock()
s.rules = allRules
s.version = maxVersion
s.loadErr = errors.Join(loadErrs...)
s.disabledUnmatched = unmatched
s.disabledCount = disabledCount
s.mu.Unlock()
if len(disabledSeen) > 0 {
fmt.Fprintf(os.Stderr, "signatures: %d rule(s) disabled by configuration\n", len(disabledSeen))
}
fmt.Fprintf(os.Stderr, "signatures: loaded %d rules (version %d) from %s\n", len(allRules), maxVersion, s.rulesDir)
// Rules installed, but report any skipped files so the caller can alert.
return errors.Join(loadErrs...)
}
// setLoadErr records the outcome of a load under the write lock.
func (s *Scanner) setLoadErr(err error) {
s.mu.Lock()
s.loadErr = err
s.mu.Unlock()
}
// compile pre-compiles regex patterns for a rule.
func (r *Rule) compile() error {
return r.compileShared(make(map[string]*compiledRegex))
}
// compileShared compiles the rule, reusing a regex already compiled from the
// same source for an earlier rule in shared, so a scan evaluates it once.
func (r *Rule) compileShared(shared map[string]*compiledRegex) error {
if r.MaxFileBytes < 0 {
return fmt.Errorf("max_file_bytes must be non-negative")
}
var err error
if r.compiledRegexes, err = compileRuleRegexes(shared, r.Regexes, "invalid regex"); err != nil {
return err
}
r.compiledRegexPairs = nil
for _, pair := range r.RegexPairs {
if pair.First == "" || pair.Second == "" || pair.MaxDistance < 0 {
return fmt.Errorf("regex_pairs require two non-empty regexes and a non-negative max_distance")
}
compiled, compileErr := compileRuleRegexes(shared, []string{pair.First, pair.Second}, "invalid regex_pairs regex")
if compileErr != nil {
return compileErr
}
r.compiledRegexPairs = append(r.compiledRegexPairs, compiledRegexPair{
first: compiled[0], second: compiled[1], maxDistance: pair.MaxDistance,
})
}
if r.compiledExcludeRegexes, err = compileRuleRegexes(shared, r.ExcludeRegexes, "invalid exclude regex"); err != nil {
return err
}
if r.compiledMaxFileBytesExemptRegexes, err = compileRuleRegexes(shared, r.MaxFileBytesExemptRegexes, "invalid max_file_bytes_exempt_regex"); err != nil {
return err
}
return nil
}
func compileRuleRegexes(shared map[string]*compiledRegex, patterns []string, errLabel string) ([]*compiledRegex, error) {
var out []*compiledRegex
for _, pattern := range patterns {
src := "(?i)" + pattern // rule regexes are case-insensitive
cr, ok := shared[src]
if !ok {
re, err := regexp.Compile(src)
if err != nil {
return nil, fmt.Errorf("%s '%s': %w", errLabel, pattern, err)
}
cr = &compiledRegex{Regexp: re, gate: gateFor(src)}
shared[src] = cr
}
out = append(out, cr)
}
return out, nil
}
// Match represents a rule that matched a file.
type Match struct {
RuleName string
Description string
Severity string
Category string
Matched []string // which patterns matched
}
// ScanContent scans file content against loaded rules.
// fileExt should include the dot (e.g., ".php").
func (s *Scanner) ScanContent(content []byte, fileExt string) []Match {
return s.ScanContentWithSize(content, fileExt, int64(len(content)))
}
// ScanContentWithSize scans content while using contentSize as the complete
// snapshot size for per-rule bounds. Prefix-scanning callers should pass the
// size of the open file represented by the prefix; ordinary callers should use
// ScanContent. A size smaller than the supplied bytes is raised to len(content)
// so a bad caller cannot turn a bounded rule back on for oversized content.
func (s *Scanner) ScanContentWithSize(content []byte, fileExt string, contentSize int64) []Match {
s.mu.RLock()
defer s.mu.RUnlock()
if len(s.rules) == 0 {
return nil
}
extLower := strings.ToLower(fileExt)
// Only a file that is an archive by name as well as by magic is left to the
// extraction-time scan; PHP executes past any leading bytes, so magic alone
// must never switch the rules off for an executable name.
if contenttype.IsArchiveExt(extLower) && contenttype.IsCompressedArchive(content) {
return nil
}
if contentSize < int64(len(content)) {
contentSize = int64(len(content))
}
contentLower := strings.ToLower(string(content))
eval := newRegexEval(content)
var matches []Match
for _, rule := range s.rules {
// Check if this rule applies to this file type
if !ruleMatchesExt(rule, extLower) {
continue
}
// Check exclusions first - if any exclude pattern matches, skip this rule
excluded := false
for _, pattern := range rule.ExcludePatterns {
if strings.Contains(contentLower, strings.ToLower(pattern)) {
excluded = true
break
}
}
if !excluded {
for _, re := range rule.compiledExcludeRegexes {
if eval.match(re) {
excluded = true
break
}
}
}
if excluded {
continue
}
if rule.MaxFileBytes > 0 && contentSize > int64(rule.MaxFileBytes) {
exempt := false
for _, re := range rule.compiledMaxFileBytesExemptRegexes {
if eval.match(re) {
exempt = true
break
}
}
if !exempt {
continue
}
}
// Count pattern matches
var matched []string
regexMatched := false
for _, pattern := range rule.Patterns {
if strings.Contains(contentLower, strings.ToLower(pattern)) {
matched = append(matched, pattern)
}
}
for _, re := range rule.compiledRegexes {
if eval.match(re) {
matched = append(matched, re.String())
regexMatched = true
}
}
for _, pair := range rule.compiledRegexPairs {
if eval.matchPair(pair) {
matched = append(matched, fmt.Sprintf("%s near %s", pair.first.String(), pair.second.String()))
regexMatched = true
}
}
if len(matched) >= rule.MinMatch && (!rule.RequireRegex || regexMatched) {
matches = append(matches, Match{
RuleName: rule.Name,
Description: rule.Description,
Severity: rule.Severity,
Category: rule.Category,
Matched: matched,
})
}
}
return matches
}
// ScanFile reads a file and scans it against loaded rules.
func (s *Scanner) ScanFile(path string, maxBytes int) []Match {
s.mu.RLock()
ruleCount := len(s.rules)
s.mu.RUnlock()
if ruleCount == 0 {
return nil
}
if maxBytes <= 0 {
return nil
}
// #nosec G304 -- ScanFile's whole purpose is to scan a file on disk;
// `path` comes from the daemon's file index walker or a fanotify event.
f, err := os.Open(path)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
// ReadAll over a LimitReader, not a single Read into a pre-sized buffer:
// a bare f.Read can return a short count on the first call, which would
// hand only a prefix to the scanner and silently miss malware further
// into the file. LimitReader also makes a negative/huge maxBytes safe
// (no make([]byte, maxBytes) panic / over-allocation).
buf, err := io.ReadAll(io.LimitReader(f, int64(maxBytes)))
if err != nil || len(buf) == 0 {
return nil
}
contentSize := int64(len(buf))
if info, statErr := f.Stat(); statErr == nil && info.Size() > contentSize {
contentSize = info.Size()
}
ext := filepath.Ext(path)
return s.ScanContentWithSize(buf, ext, contentSize)
}
// RuleNames returns the names of the loaded rules.
func (s *Scanner) RuleNames() []string {
s.mu.RLock()
defer s.mu.RUnlock()
names := make([]string, 0, len(s.rules))
for _, r := range s.rules {
names = append(names, r.Name)
}
return names
}
// RuleCount returns the number of loaded rules.
func (s *Scanner) RuleCount() int {
s.mu.RLock()
defer s.mu.RUnlock()
return len(s.rules)
}
// Version returns the highest version number across loaded rule files.
func (s *Scanner) Version() int {
s.mu.RLock()
defer s.mu.RUnlock()
return s.version
}
// canonicalScanExt folds extensions that carry PHP source but are not the
// extension rules are written against. Every extension a stock PHP handler
// executes (.phtml, .pht, .php5 ...) must meet the same rules as .php, and so
// must ".phps": it is PHP source by definition -- the extension exists so a
// server can display it -- so a payload staged under it is still matched.
// Without this fold such a file is read and then compared against nothing,
// because every PHP rule declares file_types [".php"].
func canonicalScanExt(ext string) string {
ext = strings.ToLower(ext)
if ext == ".phps" || contenttype.IsExecutablePHPExt(ext) {
return ".php"
}
return ext
}
func ruleMatchesExt(rule Rule, ext string) bool {
if len(rule.FileTypes) == 0 {
return true // no filter = match all
}
ext = strings.ToLower(ext)
canonicalExt := canonicalScanExt(ext)
for _, ft := range rule.FileTypes {
ft = strings.ToLower(ft)
// The alias is one-way: PHP rules also inspect .phps source, while a
// deliberately .phps-only rule must not broaden to executable .php.
if ft == "*" || ft == ext || ft == canonicalExt {
return true
}
}
return false
}
package signatures
import (
"regexp"
"strings"
"github.com/pidginhost/csm/internal/contenttype"
)
// maxReferencedPayloadPaths bounds how many payload paths one finding carries.
// Findings travel through alert mail, the audit log and webhooks, so a file
// holding hundreds of include statements must not turn one alert into an
// unbounded message.
const maxReferencedPayloadPaths = 5
// referencedPayloadWindow is how far a quoted path may sit from the
// include/require keyword that consumes it. The keyword and the literal are
// usually in the same statement; a local assigned one line earlier and
// included on the next is the shape the 2026-09-17 loader used.
const referencedPayloadWindow = 300
// nonExecutablePayloadLiteral matches a quoted path whose extension belongs to
// an image, archive or opaque data file. The extension list mirrors the one in
// the backdoor_include_nonexecutable rules; source partials (.php, .html,
// .tpl, .txt, .svg) are deliberately absent, because including those is templating.
var nonExecutablePayloadLiteral = regexp.MustCompile(
`(?i)['"]([^'"\r\n]{1,240}\.(?:png|jp(?:eg?|g)|gif|bmp|ico|cur|webp|tiff?|zip|rar|tar|gz|bz2|7z|log|dat|bin|cache|bak|old|csv|pdf|woff2?|ttf|eot|otf))['"]`,
)
// includeKeyword matches a PHP inclusion keyword in statement position.
var includeKeyword = regexp.MustCompile(`(?i)(?:^|[^A-Za-z0-9_$>])(?:include|require)(?:_once)?[\s('$"]`)
// ReferencedPayloadPaths returns the non-executable files that PHP content
// pulls in through include or require, in the order they appear and without
// duplicates.
//
// A loader finding names the PHP file that changed. The payload lives
// somewhere else -- in the incident that prompted this, a picture in a plugin
// asset directory two levels away -- and survives a clean-up of the PHP alone,
// so the finding has to name it too.
//
// Association is positional. RE2 cannot prove that the local assigned a path
// is the same local an include consumes, so a quoted payload path counts when
// an inclusion keyword sits within referencedPayloadWindow bytes of it. The
// output is remediation context attached to a finding that already fired, not
// a detection signal.
func ReferencedPayloadPaths(content []byte) []string {
// Module loaders spell require and include too, and a JavaScript bundle
// pulling in a sprite is not a loader for a hidden payload.
if !contenttype.HasPHPOpenTag(content) {
return nil
}
keyword := includeKeyword.FindIndex(content)
if keyword == nil {
return nil
}
var paths []string
seen := make(map[string]bool)
// Advance both cursors monotonically. Materializing every match before
// enforcing the output cap wastes memory; restarting the keyword search
// for each repeated literal makes enrichment quadratic on hostile input.
for offset := 0; offset < len(content); {
literal := nonExecutablePayloadLiteral.FindSubmatchIndex(content[offset:])
if literal == nil {
break
}
for i := range literal {
literal[i] += offset
}
offset = literal[1]
for keyword[1] <= literal[0]-referencedPayloadWindow {
next := keyword[1]
keyword = includeKeyword.FindIndex(content[next:])
if keyword == nil {
return paths
}
keyword[0] += next
keyword[1] += next
}
if keyword[0] >= literal[1]+referencedPayloadWindow {
continue
}
path := string(content[literal[2]:literal[3]])
if seen[path] {
continue
}
seen[path] = true
paths = append(paths, path)
if len(paths) == maxReferencedPayloadPaths {
break
}
}
return paths
}
// ReferencedPayloadDetail renders ReferencedPayloadPaths as a line to append
// to a finding's details, or an empty string when the content pulls in no
// non-executable file. Callers concatenate it unconditionally.
func ReferencedPayloadDetail(content []byte) string {
paths := ReferencedPayloadPaths(content)
if len(paths) == 0 {
return ""
}
return "\nIncluded payload files: " + strings.Join(paths, ", ")
}
package signatures
import (
"regexp"
"regexp/syntax"
"slices"
"strings"
"unicode"
"unicode/utf8"
)
// minGateLiteral is the shortest literal worth testing before a regex. A
// one-byte literal occurs in nearly every file, so testing for it costs a pass
// over the content and filters nothing.
const minGateLiteral = 2
// compiledRegex is a rule regex paired with the literals that gate it.
// Identical sources across rules share one value, so a scan evaluates each
// distinct regex once.
type compiledRegex struct {
*regexp.Regexp
gate regexGate
}
// regexGate is a conjunction of literal sets: every match of its regex
// contains at least one literal of each set, once the content is folded by
// foldForGate. The rule regexes are case-insensitive, and Go's engine cannot
// use a literal to skip ahead under case folding, so each one walks the whole
// file; a regex with a set none of whose literals is present cannot match and
// need not run. A nil gate admits every input.
type regexGate []literalSet
// literalSet lists lowercase ASCII literals of which a match contains at
// least one.
type literalSet []string
// gateFor derives the gate for a regex source written in the syntax
// regexp.Compile accepts. Sets whose shortest literal is under
// minGateLiteral bytes are left out: dropping a set only weakens the gate.
func gateFor(src string) regexGate {
re, err := syntax.Parse(src, syntax.Perl)
if err != nil {
return nil
}
var gate regexGate
for _, set := range requiredSets(re) {
if shortest(set) < minGateLiteral {
continue
}
slices.Sort(set)
gate = append(gate, literalSet(slices.Compact(set)))
}
// Literals may contain any ASCII byte, so a joined string cannot
// distinguish a literal's bytes from separators between literals.
slices.SortFunc(gate, slices.Compare[literalSet])
return slices.CompactFunc(gate, slices.Equal[literalSet])
}
// requiredSets returns literal sets that every match of re satisfies, each
// by containing at least one of the set's literals; nil when none is known.
// A literal is the lowercased longest ASCII run of a literal node: a match
// of the node contains that run, rune for rune up to case folding.
// Non-ASCII runes end a run because their fold partners are not all
// representable in the folded content.
func requiredSets(re *syntax.Regexp) [][]string {
switch re.Op {
case syntax.OpLiteral:
if lit := longestASCIIRun(re.Rune); lit != "" {
return [][]string{{lit}}
}
return nil
case syntax.OpCapture, syntax.OpPlus:
return requiredSets(re.Sub[0])
case syntax.OpRepeat:
if re.Min >= 1 {
return requiredSets(re.Sub[0])
}
return nil
case syntax.OpConcat:
// Every part of a concatenation is present in a match, so every
// part's sets hold.
var all [][]string
for _, sub := range re.Sub {
all = append(all, requiredSets(sub)...)
}
return all
case syntax.OpAlternate:
// A match takes one branch, so the branches' sets cannot be
// required together. Each branch contributes its strongest set to
// one union, and a branch without any voids it.
var union []string
for _, sub := range re.Sub {
sets := requiredSets(sub)
if len(sets) == 0 {
return nil
}
best := sets[0]
for _, set := range sets[1:] {
if shortest(set) > shortest(best) {
best = set
}
}
union = append(union, best...)
}
return [][]string{union}
}
return nil
}
func longestASCIIRun(runes []rune) string {
best, start := "", 0
for i := 0; i <= len(runes); i++ {
if i < len(runes) && runes[i] < utf8.RuneSelf {
continue
}
if i-start > len(best) {
best = strings.ToLower(string(runes[start:i]))
}
start = i + 1
}
return best
}
func shortest(lits []string) int {
if len(lits) == 0 {
return 0
}
n := len(lits[0])
for _, lit := range lits[1:] {
n = min(n, len(lit))
}
return n
}
// admits reports whether folded content holds a literal of every set.
// seen memoizes literal presence for the scan in progress: many regexes
// share literals such as "eval" or "base64_decode".
func (g regexGate) admits(folded string, seen map[string]bool) bool {
for _, set := range g {
if !set.present(folded, seen) {
return false
}
}
return true
}
func (set literalSet) present(folded string, seen map[string]bool) bool {
for _, lit := range set {
found, known := seen[lit]
if !known {
found = strings.Contains(folded, lit)
seen[lit] = found
}
if found {
return true
}
}
return false
}
// asciiFoldOf maps each non-ASCII rune that Unicode case folding pairs with an
// ASCII letter to that letter in lower case: a case-insensitive "s" also
// matches the long s, and "k" the Kelvin sign.
var asciiFoldOf = func() map[rune]byte {
m := map[rune]byte{}
for c := 'a'; c <= 'z'; c++ {
for r := unicode.SimpleFold(c); r != c; r = unicode.SimpleFold(r) {
if r >= utf8.RuneSelf {
m[r] = byte(c)
}
}
}
return m
}()
// foldForGate lowercases ASCII letters and replaces each non-ASCII fold
// partner of an ASCII letter with that letter. Content is decoded exactly as
// the regex engine decodes it, an invalid byte standing alone, so a literal a
// regex matches is never split or hidden by the fold.
func foldForGate(content []byte) string {
var b strings.Builder
b.Grow(len(content))
for i := 0; i < len(content); {
c := content[i]
if c < utf8.RuneSelf {
if 'A' <= c && c <= 'Z' {
c += 'a' - 'A'
}
b.WriteByte(c)
i++
continue
}
r, width := utf8.DecodeRune(content[i:])
if ascii, ok := asciiFoldOf[r]; ok {
b.WriteByte(ascii)
} else {
b.Write(content[i : i+width])
}
i += width
}
return b.String()
}
// regexEval evaluates rule regexes for one scan. Each distinct regex runs at
// most once, and only when its gate admits the content. The content is folded
// on first use, so a scan that reaches no gated regex never pays for it.
type regexEval struct {
content []byte
folded string
didFold bool
seen map[string]bool
results map[*compiledRegex]bool
}
func newRegexEval(content []byte) *regexEval {
return ®exEval{
content: content,
seen: make(map[string]bool),
results: make(map[*compiledRegex]bool),
}
}
func (e *regexEval) match(cr *compiledRegex) bool {
if matched, done := e.results[cr]; done {
return matched
}
if len(cr.gate) > 0 && !e.didFold {
e.folded, e.didFold = foldForGate(e.content), true
}
matched := cr.gate.admits(e.folded, e.seen) && cr.Match(e.content)
e.results[cr] = matched
return matched
}
package signatures
// RegexPair contributes one regex match when the start offsets of its two
// expressions are within MaxDistance bytes, in either order.
type RegexPair struct {
First string `yaml:"first"`
Second string `yaml:"second"`
MaxDistance int `yaml:"max_distance"`
}
type compiledRegexPair struct {
first, second *compiledRegex
maxDistance int
}
func (e *regexEval) matchPair(pair compiledRegexPair) bool {
if !e.match(pair.first) || !e.match(pair.second) {
return false
}
first := pair.first.FindAllIndex(e.content, -1)
second := pair.second.FindAllIndex(e.content, -1)
// Both lists are sorted. Advancing the earlier match examines each offset
// once instead of testing every pair or running a wide-gap regex.
for i, j := 0, 0; i < len(first) && j < len(second); {
distance := first[i][0] - second[j][0]
if distance >= -pair.maxDistance && distance <= pair.maxDistance {
return true
}
if distance < 0 {
i++
} else {
j++
}
}
return false
}
package signatures
import (
"bytes"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"time"
"gopkg.in/yaml.v3"
"github.com/pidginhost/csm/internal/atomicio"
)
// ErrUpdateRollback marks a signed update refused by rollback protection.
var ErrUpdateRollback = errors.New("signed rules update refused")
// UpdateOptions controls operator-approved exceptions to rollback protection.
type UpdateOptions struct {
AllowRuleCountDecrease bool
}
// Update downloads the latest rules from the configured URL.
// Validates the downloaded rules before installing.
// A detached ed25519 signature is fetched from url+".sig" and verified
// before the rules are installed.
// Returns the number of rules loaded, or error.
func Update(rulesDir, url, signingKey string, options UpdateOptions) (int, error) {
if url == "" {
return 0, fmt.Errorf("no update URL configured (set signatures.update_url in csm.yaml)")
}
if err := requireSigningKey(signingKey); err != nil {
return 0, err
}
// Download
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Get(url)
if err != nil {
return 0, fmt.Errorf("downloading rules from %s: %w", url, err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != 200 {
return 0, fmt.Errorf("download failed: HTTP %d from %s", resp.StatusCode, url)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024)) // max 10MB
if err != nil {
return 0, fmt.Errorf("reading response: %w", err)
}
sig, err := fetchSignature(url + ".sig")
if err != nil {
return 0, fmt.Errorf("signature verification required but failed: %w", err)
}
if err := VerifySignature(signingKey, data, sig); err != nil {
return 0, fmt.Errorf("rules signature invalid: %w", err)
}
// Validate: must parse as valid YAML rules
var rf RuleFile
if err := yaml.Unmarshal(data, &rf); err != nil {
return 0, fmt.Errorf("invalid rules file: %w", err)
}
if len(rf.Rules) == 0 {
return 0, fmt.Errorf("rules file contains no rules")
}
// Validate each rule compiles
for _, rule := range rf.Rules {
if err := rule.compile(); err != nil {
return 0, fmt.Errorf("rule '%s' failed validation: %w", rule.Name, err)
}
}
destPath := filepath.Join(rulesDir, "malware.yml")
// The daemon rescans every file on the host when a rules file changes on
// disk, so re-downloading the installed ruleset must not rewrite it.
// #nosec G304 -- destPath is under the operator-configured rules dir.
if installed, err := os.ReadFile(destPath); err == nil && bytes.Equal(installed, data) {
return len(rf.Rules), nil
}
if err := refuseRollback(destPath, rf, options.AllowRuleCountDecrease); err != nil {
return 0, err
}
// Ensure rules directory exists
if err := os.MkdirAll(rulesDir, 0700); err != nil {
return 0, fmt.Errorf("creating rules dir: %w", err)
}
// Atomic write: write-temp, fsync, rename, dir-fsync. The daemon reloads
// these rules on the next tick, so a torn write must never be observable.
if err := atomicio.AtomicWrite(destPath, 0600, data); err != nil {
return 0, fmt.Errorf("installing rules: %w", err)
}
return len(rf.Rules), nil
}
// refuseRollback rejects a validly signed update that would move the
// installed ruleset backwards: an older version number is a replayed release,
// and a rule count that collapses to under half of what is installed is a
// stale mirror or a truncated publish rather than ordinary churn. Either
// would silently strip detection while reporting a successful update. A
// missing or unparsable installed file gives nothing to compare against and
// is not protected: the signed update is the recovery path out of that state.
func refuseRollback(destPath string, next RuleFile, allowRuleCountDecrease bool) error {
current, err := os.ReadFile(destPath) // #nosec G304 -- operator-configured rules dir.
if errors.Is(err, os.ErrNotExist) {
return nil
}
if err != nil {
return fmt.Errorf("reading installed rules: %w", err)
}
installed, ok := parseInstalledRules(current)
if !ok {
return nil
}
if next.Version < installed.Version {
return fmt.Errorf("%w: refusing rules downgrade: update is version %d, installed rules are version %d", ErrUpdateRollback, next.Version, installed.Version)
}
if !allowRuleCountDecrease && len(next.Rules)*2 < len(installed.Rules) {
return fmt.Errorf("%w: refusing rules rollback: update carries %d rules, installed rules carry %d", ErrUpdateRollback, len(next.Rules), len(installed.Rules))
}
return nil
}
// parseInstalledRules reports false for an unparsable or empty installed
// file: the states a signed update must be allowed to repair.
func parseInstalledRules(data []byte) (RuleFile, bool) {
var installed RuleFile
if yaml.Unmarshal(data, &installed) != nil || len(installed.Rules) == 0 {
return RuleFile{}, false
}
return installed, true
}
package signatures
import (
"crypto/ed25519"
"encoding/hex"
"fmt"
"io"
"net/http"
"time"
)
func requireSigningKey(signingKey string) error {
if signingKey == "" {
return fmt.Errorf("signatures.signing_key is required for remote rule updates")
}
return nil
}
// VerifySignature checks an ed25519 signature over data using a hex-encoded public key.
// Returns nil if the signature is valid.
func VerifySignature(pubKeyHex string, data, signature []byte) error {
pubKeyBytes, err := hex.DecodeString(pubKeyHex)
if err != nil {
return fmt.Errorf("invalid signing key (bad hex): %w", err)
}
if len(pubKeyBytes) != ed25519.PublicKeySize {
return fmt.Errorf("invalid signing key length: got %d bytes, want %d", len(pubKeyBytes), ed25519.PublicKeySize)
}
if len(signature) != ed25519.SignatureSize {
return fmt.Errorf("invalid signature length: got %d bytes, want %d", len(signature), ed25519.SignatureSize)
}
pubKey := ed25519.PublicKey(pubKeyBytes)
if !ed25519.Verify(pubKey, data, signature) {
return fmt.Errorf("signature verification failed")
}
return nil
}
// fetchSignature downloads a detached signature from url + ".sig".
func fetchSignature(sigURL string) ([]byte, error) {
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Get(sigURL)
if err != nil {
return nil, fmt.Errorf("downloading signature from %s: %w", sigURL, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("signature download returned HTTP %d from %s", resp.StatusCode, sigURL)
}
sig, err := io.ReadAll(io.LimitReader(resp.Body, 1024)) // ed25519 sig is 64 bytes
if err != nil {
return nil, fmt.Errorf("reading signature: %w", err)
}
return sig, nil
}
// Package sourcewalk supports source inventories in a checkout shared with tests.
package sourcewalk
import (
"errors"
"io/fs"
)
// IgnoreVanished lets a walk continue when another test removes a descendant.
// Root errors and errors returned by the visitor still fail the inventory.
func IgnoreVanished(root string, visit fs.WalkDirFunc) fs.WalkDirFunc {
return func(name string, entry fs.DirEntry, err error) error {
if name != root && errors.Is(err, fs.ErrNotExist) {
return nil
}
return visit(name, entry, err)
}
}
// Package spamscore is the spam score shared by CSM's spam evidence and the
// rspamd reputation source: verdict masses decay with a 7-day half-life and
// a prior mass keeps a few verdicts from saturating the score.
package spamscore
import (
"math"
"time"
)
const (
// PriorMass is smoothing added to the total so one verdict alone scores
// 33 and two fresh spam verdicts with no ham reach 50.
PriorMass = 2.0
// HalfLife halves a verdict's weight per week.
HalfLife = 7 * 24 * time.Hour
)
// Recency is the weight left after age; a zero or negative age counts in
// full.
func Recency(age time.Duration) float64 {
if age <= 0 {
return 1
}
return math.Exp2(-age.Hours() / HalfLife.Hours())
}
// Score is 100 * spam / (total + PriorMass), rounded; 0 without spam.
func Score(spam, total float64) int {
if spam <= 0 || total <= 0 {
return 0
}
return int(math.Round(100 * spam / (total + PriorMass)))
}
// Package sshdconf reads the effective sshd_config the way sshd itself does:
// Include directives are followed, Match blocks are ignored because their
// directives are connection-scoped, and most keywords keep first-match-wins
// semantics.
//
// Port is the exception. sshd accumulates every Port directive, so a config
// can name several listening ports; collapsing them into a single value hides
// ports and makes any firewall guard built on top of it wrong.
package sshdconf
import (
"bufio"
"crypto/sha256"
"encoding/hex"
"io"
"net"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
)
// DefaultPath is where sshd reads its configuration from.
const DefaultPath = "/etc/ssh/sshd_config"
// defaultPort is OpenSSH's compiled-in listen port, used when nothing in the
// config names one.
const defaultPort = 22
// maxIncludeDepth mirrors OpenSSH's own nesting limit. Without it a config
// that includes itself recurses until the stack overflows, which Go cannot
// recover from.
const maxIncludeDepth = 16
// maxLineBytes caps how much of one line is kept. No real directive comes
// close; the cap exists so a corrupt file cannot be read into the daemon's
// memory whole. The remainder of an over-long line is discarded and parsing
// continues with the next one.
const maxLineBytes = 64 * 1024
// compiledDefaults are the OpenSSH defaults for the keywords CSM reads.
var compiledDefaults = map[string]string{
"addressfamily": "any",
"protocol": "2",
"passwordauthentication": "yes",
"permitrootlogin": "prohibit-password",
"maxauthtries": "6",
"x11forwarding": "no",
"usedns": "no",
}
// FS is the file access parsing needs. It matches the corresponding subset of
// the internal/checks OS injector, so that package can pass its own hook and
// keep its tests fake-driven.
type FS interface {
Open(name string) (*os.File, error)
Glob(pattern string) ([]string, error)
}
// OSFS reads the real filesystem.
type OSFS struct{}
// #nosec G304 -- callers pass the operator-owned sshd config path.
func (OSFS) Open(name string) (*os.File, error) { return os.Open(name) }
// Glob expands an Include pattern.
func (OSFS) Glob(pattern string) ([]string, error) { return filepath.Glob(pattern) }
// Config is a parsed sshd_config.
type Config struct {
present bool
values map[string]string
files []string
digests [][sha256.Size]byte
ports []int
listenAddresses []listenAddress
}
type listenAddress struct {
host string
port int
}
// Parse reads path and every file it includes. A missing or unreadable root
// file yields a Config of pure compiled defaults with Present reporting false,
// so callers can tell "sshd runs on defaults" from "there is no sshd config
// here at all".
func Parse(fsys FS, path string) *Config {
c := &Config{values: make(map[string]string)}
c.present = c.parseFile(fsys, path, filepath.Dir(path), 0, make(map[string]struct{}), false)
return c
}
// Present reports whether the root config file was readable.
func (c *Config) Present() bool { return c.present }
// Files lists every file the parse read, root first then Includes in the
// order sshd would read them. Change detection has to cover all of them:
// a drop-in can flip a setting without touching the root file.
func (c *Config) Files() []string {
return append([]string(nil), c.files...)
}
// Digest identifies the file paths and exact bytes consumed by Parse, in
// read order. Hashing through the parser keeps the effective values and the
// change-detection digest on one coherent filesystem snapshot.
func (c *Config) Digest() string {
h := sha256.New()
for i, file := range c.files {
h.Write([]byte(file))
h.Write([]byte{0})
h.Write(c.digests[i][:])
h.Write([]byte{0})
}
return hex.EncodeToString(h.Sum(nil))
}
// Value returns the effective value of keyword, lowercased, falling back to
// the OpenSSH compiled default. Keywords with no shipped default return "".
func (c *Config) Value(keyword string) string {
key := strings.ToLower(keyword)
if v, ok := c.values[key]; ok {
return strings.ToLower(v)
}
return compiledDefaults[key]
}
// ListenPorts returns the sorted TCP ports sshd accepts connections on.
//
// Port applies only to ListenAddress entries that omit a port, so a config
// where every ListenAddress carries its own port never binds the Port value.
func (c *Config) ListenPorts() []int {
v4, v6 := c.listenPorts(false)
return sortedUnique(append(v4, v6...))
}
// RemoteListenPorts returns the IPv4 and IPv6 ports reachable beyond the
// local host. It honors AddressFamily and explicit ListenAddress directives,
// and excludes loopback-only listeners that an inbound firewall cannot cut
// off.
func (c *Config) RemoteListenPorts() (ipv4, ipv6 []int) {
return c.listenPorts(true)
}
// parseFile reads one config file, following Include directives relative to
// rootDir. It reports whether the file was readable.
func (c *Config) parseFile(fsys FS, path, rootDir string, depth int, seen map[string]struct{}, collectOnly bool) bool {
if depth > maxIncludeDepth {
return false
}
path = filepath.Clean(path)
// A glob may match its containing file, and mutually recursive globs can
// branch exponentially before the depth limit. Re-reading a file cannot
// change this parser's first-value or set-like accumulated results.
if _, parsed := seen[path]; parsed {
return false
}
seen[path] = struct{}{}
f, err := fsys.Open(path)
if err != nil {
return false
}
defer func() { _ = f.Close() }()
c.files = append(c.files, path)
c.digests = append(c.digests, [sha256.Size]byte{})
digestIndex := len(c.digests) - 1
contentHash := sha256.New()
defer func() { copy(c.digests[digestIndex][:], contentHash.Sum(nil)) }()
inMatch := false
reader := bufio.NewReader(io.TeeReader(f, contentHash))
for {
rawLine, more := readLine(reader)
if !more {
break
}
line := strings.TrimSpace(rawLine)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
keyword, args, ok := splitDirective(line)
if !ok {
continue
}
// A Match block runs to EOF in the effective global view. Its
// directives do not apply globally, but Include targets are still read
// by sshd and must be recorded for change detection.
if keyword == "match" {
inMatch = true
continue
}
if keyword == "include" {
// sshd resolves a relative Include against its config directory,
// not against the including file, so nested drop-ins agree with
// the top-level file.
for _, pattern := range args {
if !filepath.IsAbs(pattern) {
pattern = filepath.Join(rootDir, pattern)
}
matches, globErr := fsys.Glob(pattern)
if globErr != nil {
continue
}
for _, m := range matches {
c.parseFile(fsys, m, rootDir, depth+1, seen, collectOnly || inMatch)
}
}
continue
}
if inMatch || collectOnly {
continue
}
switch keyword {
case "port":
if port, valid := parsePort(args[0]); valid {
c.ports = append(c.ports, port)
}
case "listenaddress":
c.recordListenAddress(args[0])
default:
if _, exists := c.values[keyword]; !exists {
c.values[keyword] = args[0]
}
}
}
return true
}
// readLine returns the next line without its terminator, keeping at most
// maxLineBytes of it, and reports whether one was read. bufio.Reader.ReadLine
// hands back the line in buffer-sized pieces, so the discarded tail of an
// over-long line never accumulates.
func readLine(r *bufio.Reader) (string, bool) {
var line strings.Builder
for {
chunk, isPrefix, err := r.ReadLine()
if err != nil {
return line.String(), line.Len() > 0
}
if remaining := maxLineBytes - line.Len(); remaining > 0 {
if len(chunk) > remaining {
chunk = chunk[:remaining]
}
line.Write(chunk)
}
if !isPrefix {
return line.String(), true
}
}
}
// recordListenAddress records an address and its optional explicit port.
// Forms without a port inherit every Port directive.
func (c *Config) recordListenAddress(value string) {
host, portStr, err := net.SplitHostPort(value)
if err != nil {
c.listenAddresses = append(c.listenAddresses, listenAddress{host: strings.Trim(value, "[]")})
return
}
port, valid := parsePort(portStr)
if !valid {
c.listenAddresses = append(c.listenAddresses, listenAddress{host: host})
return
}
c.listenAddresses = append(c.listenAddresses, listenAddress{host: host, port: port})
}
// splitDirective splits a keyword from its OpenSSH-style argument vector.
// Quoting, basic escapes, and comments follow argv_split in OpenSSH.
func splitDirective(line string) (keyword string, args []string, ok bool) {
sep := strings.IndexFunc(line, func(r rune) bool {
return r == ' ' || r == '\t' || r == '='
})
if sep <= 0 {
return "", nil, false
}
keyword = strings.ToLower(line[:sep])
rest := strings.TrimLeft(line[sep:], " \t")
if strings.HasPrefix(rest, "=") {
rest = strings.TrimLeft(rest[1:], " \t")
}
args, ok = splitArguments(rest)
if !ok || len(args) == 0 {
return "", nil, false
}
return keyword, args, true
}
func splitArguments(s string) ([]string, bool) {
var args []string
for i := 0; i < len(s); {
for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
i++
}
if i == len(s) || s[i] == '#' {
break
}
var arg strings.Builder
var quote byte
for i < len(s) {
ch := s[i]
switch {
case ch == '\\' && i+1 < len(s) &&
(s[i+1] == '\'' || s[i+1] == '"' || s[i+1] == '\\' || (quote == 0 && s[i+1] == ' ')):
i++
arg.WriteByte(s[i])
case quote == 0 && (ch == ' ' || ch == '\t'):
i++
goto argumentDone
case quote == 0 && (ch == '\'' || ch == '"'):
quote = ch
case quote != 0 && ch == quote:
quote = 0
default:
arg.WriteByte(ch)
}
i++
}
argumentDone:
if quote != 0 {
return nil, false
}
args = append(args, arg.String())
}
return args, true
}
func parsePort(s string) (int, bool) {
port, err := strconv.Atoi(strings.TrimSpace(s))
if err == nil {
return port, port >= 1 && port <= 65535
}
port, err = net.LookupPort("tcp", s)
if err != nil || port < 1 || port > 65535 {
return 0, false
}
return port, true
}
func (c *Config) listenPorts(remoteOnly bool) (ipv4, ipv6 []int) {
base := c.ports
if len(base) == 0 {
base = []int{defaultPort}
}
allowV4 := c.Value("addressfamily") != "inet6"
allowV6 := c.Value("addressfamily") != "inet"
if len(c.listenAddresses) == 0 {
if allowV4 {
ipv4 = append(ipv4, base...)
}
if allowV6 {
ipv6 = append(ipv6, base...)
}
return sortedUnique(ipv4), sortedUnique(ipv6)
}
for _, listener := range c.listenAddresses {
isV4, isV6, loopback := addressFamilies(listener.host)
if remoteOnly && loopback {
continue
}
ports := base
if listener.port != 0 {
ports = []int{listener.port}
}
if allowV4 && isV4 {
ipv4 = append(ipv4, ports...)
}
if allowV6 && isV6 {
ipv6 = append(ipv6, ports...)
}
}
return sortedUnique(ipv4), sortedUnique(ipv6)
}
func addressFamilies(host string) (ipv4, ipv6, loopback bool) {
if host == "*" {
return true, true, false
}
canonicalHost := strings.TrimSuffix(strings.ToLower(host), ".")
if canonicalHost == "localhost" || strings.HasSuffix(canonicalHost, ".localhost") {
return true, true, true
}
if zone := strings.LastIndex(host, "%"); zone >= 0 {
host = host[:zone]
}
ip := net.ParseIP(host)
if ip == nil {
return true, true, false
}
if ip.To4() != nil {
return true, false, ip.IsLoopback()
}
return false, true, ip.IsLoopback()
}
func sortedUnique(ports []int) []int {
seen := make(map[int]struct{}, len(ports))
out := make([]int, 0, len(ports))
for _, p := range ports {
if _, dup := seen[p]; dup {
continue
}
seen[p] = struct{}{}
out = append(out, p)
}
sort.Ints(out)
return out
}
package stalwartapi
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
)
// Account is one Stalwart user account.
type Account struct {
ID, Name, Email string
// Admin is the built-in admin role. Custom roles are not resolved.
Admin bool
// SignInDisabled means the account's own permissions deny sign-in.
SignInDisabled bool
// Inherit means the account has no permission list of its own.
Inherit bool
}
// AccountsNamed lists the accounts whose local part is exactly name, in
// every domain.
func (c *Client) AccountsNamed(ctx context.Context, name string) ([]Account, error) {
ids, err := c.query(ctx, "x:Account/query", map[string]any{"name": name})
if err != nil || len(ids) == 0 {
return nil, err
}
var all []Account
for page := range slices.Chunk(ids, accountPage) {
accounts, err := c.accounts(ctx, page)
if err != nil {
return nil, err
}
all = append(all, accounts...)
}
return all, nil
}
// AccountByAddress finds the account that signs in as address: by its own
// address, or through a mailbox alias, a domain alias, or both. An address
// without a local part is not found. Case is ignored for ASCII letters only:
// a Unicode fold could map a crafted name onto another account. The own
// address is tried first, by local part alone, so it does not depend on
// reading the whole domain list.
func (c *Client) AccountByAddress(ctx context.Context, address string) (Account, bool, error) {
address = asciiLower(address)
at := strings.LastIndexByte(address, '@')
if at <= 0 {
return Account{}, false, nil
}
local, domainName := address[:at], address[at+1:]
list, err := c.AccountsNamed(ctx, local)
if err != nil {
return Account{}, false, err
}
for _, a := range list {
if asciiLower(a.Email) == address {
return a, true, nil
}
}
domains, err := c.Domains(ctx)
if err != nil {
return Account{}, false, err
}
i := slices.IndexFunc(domains, func(d Domain) bool {
return d.Name == domainName || slices.Contains(d.Aliases, domainName)
})
if i < 0 {
return Account{}, false, nil
}
domain := domains[i]
for _, a := range list {
if asciiLower(a.Email) == local+"@"+domain.Name {
return a, true, nil
}
}
return c.accountByAlias(ctx, local, domain.ID)
}
// accountPage is the most ids one query asks for and one get carries:
// Stalwart's limit on objects in a get.
const accountPage = 500
// accountByAlias walks every account for the one with the alias local in
// domainID. Stalwart cannot query aliases. An alias that is switched off
// still matches: it can be switched back on. A server may return fewer ids
// than asked for, so only an empty page ends the walk, and a page with no
// new id stops it with an error rather than looping.
func (c *Client) accountByAlias(ctx context.Context, local, domainID string) (Account, bool, error) {
seen := map[string]bool{}
for pos := 0; ; {
ids, err := c.queryArgs(ctx, "x:Account/query", map[string]any{"limit": accountPage, "position": pos})
if err != nil || len(ids) == 0 {
return Account{}, false, err
}
fresh := false
for _, id := range ids {
fresh = fresh || !seen[id]
seen[id] = true
}
if !fresh {
return Account{}, false, errors.New("stalwart x:Account/query: paging repeated a page")
}
raw, err := c.rawAccounts(ctx, ids)
if err != nil {
return Account{}, false, err
}
for _, r := range raw {
for _, alias := range r.Aliases {
if alias.Name == local && alias.DomainID == domainID {
return r.account(), true, nil
}
}
}
pos += len(ids)
}
}
// Account reads one account. An unknown id is not found, not an error.
func (c *Client) Account(ctx context.Context, id string) (Account, bool, error) {
list, err := c.accounts(ctx, []string{id})
if err != nil {
return Account{}, false, err
}
for _, a := range list {
if a.ID == id {
return a, true, nil
}
}
return Account{}, false, nil
}
// DisableSignIn denies the account sign-in. An account that inherits its
// permissions has no list to patch, so it gets a list holding only the denial.
func (c *Client) DisableSignIn(ctx context.Context, a Account) error {
if c.bearer {
owner, err := c.SessionAccount(ctx)
if err != nil {
return err
}
if owner == a.ID {
return errors.New("CSM cannot suspend its own Stalwart API key account; change its password in Stalwart")
}
}
patch := map[string]any{"permissions/disabledPermissions/authenticate": true}
if a.Inherit {
patch = map[string]any{"permissions": map[string]any{
"@type": "Merge",
"disabledPermissions": map[string]any{"authenticate": true},
}}
}
_, err := c.set(ctx, "x:Account/set", map[string]any{"update": map[string]any{a.ID: patch}})
return err
}
// EnableSignIn removes the sign-in denial from the account's permissions and
// leaves the rest of the list alone. A missing account, or one whose
// sign-ins are not denied, is left without a write.
func (c *Client) EnableSignIn(ctx context.Context, id string) error {
a, found, err := c.Account(ctx, id)
if err != nil || !found || a.Inherit || !a.SignInDisabled {
return err
}
_, err = c.set(ctx, "x:Account/set", map[string]any{"update": map[string]any{
id: map[string]any{"permissions/disabledPermissions/authenticate": nil},
}})
return err
}
// rawAccount is an account as x:Account/get returns it.
type rawAccount struct {
ID string `json:"id"`
Name string `json:"name"`
Email string `json:"emailAddress"`
Permissions struct {
Type string `json:"@type"`
Disabled map[string]bool `json:"disabledPermissions"`
} `json:"permissions"`
Roles struct {
Type string `json:"@type"`
} `json:"roles"`
Aliases map[string]struct {
Name string `json:"name"`
DomainID string `json:"domainId"`
} `json:"aliases"`
}
func (r rawAccount) account() Account {
return Account{
ID: r.ID,
Name: r.Name,
Email: r.Email,
Admin: r.Roles.Type == "Admin",
SignInDisabled: r.Permissions.Disabled["authenticate"],
Inherit: r.Permissions.Type == "Inherit",
}
}
// accountProperties are the fields rawAccount decodes. A whole account also
// carries its credentials, secrets included, which CSM must never read.
var accountProperties = []string{"id", "name", "emailAddress", "permissions", "roles", "aliases"}
func (c *Client) rawAccounts(ctx context.Context, ids []string) ([]rawAccount, error) {
const method = "x:Account/get"
out, err := c.call(ctx, method, map[string]any{"ids": ids, "properties": accountProperties})
if err != nil {
return nil, err
}
var res struct {
List []rawAccount `json:"list"`
}
// A decode error can quote the response, so it is not passed on.
if json.Unmarshal(out, &res) != nil || res.List == nil {
return nil, fmt.Errorf("stalwart %s: unexpected response", method)
}
return res.List, nil
}
func (c *Client) accounts(ctx context.Context, ids []string) ([]Account, error) {
raw, err := c.rawAccounts(ctx, ids)
if err != nil {
return nil, err
}
out := make([]Account, 0, len(raw))
for _, r := range raw {
out = append(out, r.account())
}
return out, nil
}
func asciiLower(s string) string {
b := []byte(s)
for i, ch := range b {
if 'A' <= ch && ch <= 'Z' {
b[i] = ch + 'a' - 'A'
}
}
return string(b)
}
package stalwartapi
import (
"context"
"errors"
"fmt"
"time"
)
// BlockedIP is one of Stalwart's own IP bans.
type BlockedIP struct {
ID string `json:"id"`
Address string `json:"address"`
Reason string `json:"reason"`
}
// BlockedIPs lists Stalwart's bans on exactly address.
func (c *Client) BlockedIPs(ctx context.Context, address string) ([]BlockedIP, error) {
ids, err := c.query(ctx, "x:BlockedIp/query", map[string]any{"address": address})
if err != nil || len(ids) == 0 {
return nil, err
}
var list []BlockedIP
if err := c.get(ctx, "x:BlockedIp/get", ids, &list); err != nil {
return nil, err
}
return list, nil
}
// ReloadBlockedIPs has Stalwart read its ban list again, which is when a
// lifted ban stops being enforced.
func (c *Client) ReloadBlockedIPs(ctx context.Context) error {
_, err := c.set(ctx, "x:Action/set", map[string]any{
"create": map[string]any{"r": map[string]any{"@type": "ReloadBlockedIps"}},
})
return err
}
// DeleteBlockedIPs lifts the bans with these ids, then has Stalwart reload
// its ban list so the change takes effect. A failed reload is an error.
// Once the destroy is answered, Stalwart no longer lists the bans but
// enforces them until a reload, and a retry finds no ids left to delete. So
// the reload runs even after a partial destroy, is not cancelled with ctx,
// and is tried a second time when it fails.
func (c *Client) DeleteBlockedIPs(ctx context.Context, ids []string) error {
if len(ids) == 0 {
return nil
}
_, err := c.set(ctx, "x:BlockedIp/set", map[string]any{"destroy": ids})
var me *MethodError
if err != nil && !errors.As(err, &me) && !errors.Is(err, errNotAcknowledged) {
return err
}
var reloadErr error
for range 2 {
reloadCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
reloadErr = c.ReloadBlockedIPs(reloadCtx)
cancel()
if reloadErr == nil {
return err
}
}
return errors.Join(err, fmt.Errorf("reloading Stalwart's ban list: %w", reloadErr))
}
// Package stalwartapi is a small client for the Stalwart 1.0 admin API: the
// JMAP "x:" methods CSM uses to read and change Stalwart's settings.
package stalwartapi
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/netip"
"net/url"
"slices"
"strings"
"sync"
"time"
)
// maxResponse bounds every response body CSM reads from Stalwart.
const maxResponse = 4 << 20
var using = []string{"urn:ietf:params:jmap:core", "urn:stalwart:jmap"}
// ErrUnauthorized means Stalwart refused the login: a wrong password, or a
// wrong or destroyed key.
var ErrUnauthorized = errors.New("stalwart refused the login")
// ErrForbidden means Stalwart accepted the login but refused this client
// address or a suspended owner.
var ErrForbidden = errors.New("stalwart refused access for this login")
// errNotAcknowledged is a /set answer that leaves a requested write out.
var errNotAcknowledged = errors.New("write was not acknowledged")
// MethodError is a JMAP method failure, or one failed entry of a set call.
type MethodError struct {
Method string
Type string
Description string
}
func (e *MethodError) Error() string {
if e.Description == "" {
return fmt.Sprintf("stalwart %s: %s", e.Method, e.Type)
}
return fmt.Sprintf("stalwart %s: %s: %s", e.Method, e.Type, e.Description)
}
// Client talks to one Stalwart server as one admin or one API key. Errors
// name the method and URL path only, never the login.
type Client struct {
base string
user string
password string
// bearer sends password as an API key; redaction treats both alike.
bearer bool
hc *http.Client
mu sync.Mutex
account string
}
// New returns a client for the Stalwart server at baseURL.
func New(baseURL, user, password string) *Client {
base := strings.TrimRight(baseURL, "/")
return &Client{
base: base,
user: user,
password: password,
hc: &http.Client{
Timeout: 30 * time.Second,
// The session and its redirects name Stalwart's public hostname.
// CSM talks to the address it was given and nowhere else.
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= 5 || req.URL.Host != via[0].URL.Host || req.URL.Scheme != via[0].URL.Scheme || req.URL.User != nil {
return http.ErrUseLastResponse
}
return nil
},
},
}
}
// NewWithPeerCheck verifies each local TCP peer before sending the login.
// Remote servers are authenticated by TLS. The check also runs on reconnects.
func NewWithPeerCheck(baseURL, user, password string, check func(context.Context, net.Conn) error) *Client {
c := New(baseURL, user, password)
if !LocalURL(baseURL) {
return c
}
transport := http.DefaultTransport.(*http.Transport).Clone()
// Local credentials must never be sent through an environment proxy.
transport.Proxy = nil
dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}
transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
conn, err := dialer.DialContext(ctx, network, address)
if err != nil {
return nil, err
}
if err := check(ctx, conn); err != nil {
_ = conn.Close()
return nil, err
}
return conn, nil
}
c.hc.Transport = transport
return c
}
// NewKey returns a client that logs in with an API key. check verifies each
// local TCP peer as in NewWithPeerCheck. The daemon builds one per call, so a
// local client's own transport keeps no idle connection behind.
func NewKey(baseURL, key string, check func(context.Context, net.Conn) error) *Client {
c := NewWithPeerCheck(baseURL, "", key, check)
c.bearer = true
if t, ok := c.hc.Transport.(*http.Transport); ok {
t.DisableKeepAlives = true
}
return c
}
func (c *Client) do(ctx context.Context, method, path string, body []byte) ([]byte, error) {
if c.bearer && !LocalURL(c.base) {
return nil, errors.New("stalwart API keys may only be sent to a loopback address")
}
if err := ValidateURL(c.base); err != nil {
return nil, err
}
var rd io.Reader
if body != nil {
rd = bytes.NewReader(body)
}
req, err := http.NewRequestWithContext(ctx, method, c.base+path, rd)
if err != nil {
return nil, fmt.Errorf("stalwart %s: bad API address", path)
}
if c.bearer {
req.Header.Set("Authorization", "Bearer "+c.password)
} else {
req.SetBasicAuth(c.user, c.password)
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := c.hc.Do(req)
if err != nil {
var ue *url.Error
if errors.As(err, &ue) {
err = ue.Err
}
return nil, fmt.Errorf("stalwart %s: %w", path, c.redactError(err))
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusUnauthorized {
return nil, fmt.Errorf("stalwart %s: %w", path, ErrUnauthorized)
}
if resp.StatusCode == http.StatusForbidden {
return nil, fmt.Errorf("stalwart %s: %w", path, ErrForbidden)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("stalwart %s: HTTP %d", path, resp.StatusCode)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponse+1))
if err != nil {
return nil, fmt.Errorf("stalwart %s: reading response: %w", path, c.redactError(err))
}
if len(data) > maxResponse {
return nil, fmt.Errorf("stalwart %s: response larger than %d bytes", path, maxResponse)
}
return data, nil
}
// SessionAccount returns the session's primary account id: the admin, or the
// key's owner. It is read once.
func (c *Client) SessionAccount(ctx context.Context) (string, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.account != "" {
return c.account, nil
}
const path = "/.well-known/jmap"
data, err := c.do(ctx, http.MethodGet, path, nil)
if err != nil {
return "", err
}
var session struct {
PrimaryAccounts map[string]string `json:"primaryAccounts"`
}
if err := json.Unmarshal(data, &session); err != nil {
return "", fmt.Errorf("stalwart %s: not a JMAP session", path)
}
id := session.PrimaryAccounts["urn:stalwart:jmap"]
if id == "" {
return "", fmt.Errorf("stalwart %s: no admin account; is this Stalwart 1.0 or later?", path)
}
c.account = id
return id, nil
}
// call sends one method call and returns its response arguments. An
// accountId in args replaces the session's account.
func (c *Client) call(ctx context.Context, method string, args map[string]any) (json.RawMessage, error) {
account, err := c.SessionAccount(ctx)
if err != nil {
return nil, err
}
full := map[string]any{"accountId": account}
for k, v := range args {
full[k] = v
}
body, err := json.Marshal(map[string]any{
"using": using,
"methodCalls": []any{[]any{method, full, "a"}},
})
if err != nil {
return nil, err
}
data, err := c.do(ctx, http.MethodPost, "/jmap/", body)
if err != nil {
return nil, err
}
var resp struct {
MethodResponses [][]json.RawMessage `json:"methodResponses"`
}
if err := json.Unmarshal(data, &resp); err != nil || len(resp.MethodResponses) != 1 || len(resp.MethodResponses[0]) != 3 {
return nil, fmt.Errorf("stalwart %s: unexpected response", method)
}
var name string
if err := json.Unmarshal(resp.MethodResponses[0][0], &name); err != nil {
return nil, fmt.Errorf("stalwart %s: unexpected response", method)
}
out := resp.MethodResponses[0][1]
if name == "error" {
me := &MethodError{Method: method}
var e struct {
Type string `json:"type"`
Description string `json:"description"`
}
_ = json.Unmarshal(out, &e)
me.Type, me.Description = Redact(e.Type, c.user, c.password), Redact(e.Description, c.user, c.password)
return nil, me
}
if name != method {
return nil, fmt.Errorf("stalwart %s: answered as %s", method, Redact(name, c.user, c.password))
}
return out, nil
}
// set sends a /set call and turns its first failed entry into a MethodError.
func (c *Client) set(ctx context.Context, method string, args map[string]any) (json.RawMessage, error) {
out, err := c.call(ctx, method, args)
if err != nil {
return nil, err
}
var res struct {
Created map[string]*struct{} `json:"created"`
Updated map[string]*struct{} `json:"updated"`
Destroyed []string `json:"destroyed"`
NotCreated map[string]setFailure `json:"notCreated"`
NotUpdated map[string]setFailure `json:"notUpdated"`
NotDestroyed map[string]setFailure `json:"notDestroyed"`
}
if err := json.Unmarshal(out, &res); err != nil {
return nil, fmt.Errorf("stalwart %s: unexpected response", method)
}
for _, failed := range []map[string]setFailure{res.NotCreated, res.NotUpdated, res.NotDestroyed} {
for _, f := range failed {
return nil, &MethodError{Method: method, Type: Redact(f.Type, c.user, c.password), Description: Redact(f.Description, c.user, c.password)}
}
}
for operation, acknowledged := range map[string]map[string]*struct{}{"create": res.Created, "update": res.Updated} {
if requested, ok := args[operation].(map[string]any); ok {
for id := range requested {
if value, ok := acknowledged[id]; !ok || (operation == "create" && value == nil) {
return nil, fmt.Errorf("stalwart %s: %w", method, errNotAcknowledged)
}
}
}
}
if requested, ok := args["destroy"].([]string); ok {
for _, id := range requested {
if !slices.Contains(res.Destroyed, id) {
return nil, fmt.Errorf("stalwart %s: %w", method, errNotAcknowledged)
}
}
}
return out, nil
}
type setFailure struct {
Type string `json:"type"`
Description string `json:"description"`
}
// query sends a /query call and returns the matching ids.
func (c *Client) query(ctx context.Context, method string, filter map[string]any) ([]string, error) {
return c.queryArgs(ctx, method, map[string]any{"filter": filter})
}
// queryArgs sends a /query call with args and returns the ids.
func (c *Client) queryArgs(ctx context.Context, method string, args map[string]any) ([]string, error) {
out, err := c.call(ctx, method, args)
if err != nil {
return nil, err
}
var res struct {
IDs []string `json:"ids"`
}
if err := json.Unmarshal(out, &res); err != nil || res.IDs == nil {
return nil, fmt.Errorf("stalwart %s: unexpected response", method)
}
return res.IDs, nil
}
// get sends a /get call and decodes its list into dst.
func (c *Client) get(ctx context.Context, method string, ids []string, dst any) error {
return c.getFor(ctx, "", method, ids, dst)
}
// getFor is get for objects that belong to account. An empty account is the
// session's.
func (c *Client) getFor(ctx context.Context, account, method string, ids []string, dst any) error {
args := map[string]any{"ids": nil}
if ids != nil {
args["ids"] = ids
}
if account != "" {
args["accountId"] = account
}
out, err := c.call(ctx, method, args)
if err != nil {
return err
}
var res struct {
List json.RawMessage `json:"list"`
}
if err := json.Unmarshal(out, &res); err != nil || len(res.List) == 0 || bytes.Equal(bytes.TrimSpace(res.List), []byte("null")) {
return fmt.Errorf("stalwart %s: unexpected response", method)
}
if err := json.Unmarshal(res.List, dst); err != nil {
return fmt.Errorf("stalwart %s: unexpected response: %w", method, c.redactError(err))
}
return nil
}
// getSingleton reads the one object of a singleton type into dst.
func (c *Client) getSingleton(ctx context.Context, method string, dst any) error {
var list []json.RawMessage
if err := c.get(ctx, method, []string{"singleton"}, &list); err != nil {
return err
}
if len(list) != 1 || bytes.Equal(bytes.TrimSpace(list[0]), []byte("null")) {
return fmt.Errorf("stalwart %s: settings object not found", method)
}
if err := json.Unmarshal(list[0], dst); err != nil {
return fmt.Errorf("stalwart %s: unexpected response: %w", method, c.redactError(err))
}
return nil
}
func (c *Client) updateSingleton(ctx context.Context, method string, patch map[string]any) error {
_, err := c.set(ctx, method, map[string]any{"update": map[string]any{"singleton": patch}})
return err
}
// Redact removes the login's password and its common wire encodings from
// diagnostics, including server-provided text.
func Redact(text, user, password string) string {
if password == "" {
return text
}
for _, value := range []string{base64.StdEncoding.EncodeToString([]byte(user + ":" + password)), url.QueryEscape(password), url.PathEscape(password), password} {
text = strings.ReplaceAll(text, value, "[redacted]")
}
return text
}
// LocalURL reports whether the API address explicitly names loopback. Only
// a local API can use this daemon's firewall as proof of listener protection.
func LocalURL(raw string) bool {
u, err := url.Parse(raw)
if err != nil {
return false
}
if strings.EqualFold(u.Hostname(), "localhost") {
return true
}
ip, err := netip.ParseAddr(u.Hostname())
return err == nil && ip.Unmap().IsLoopback()
}
// ValidateURL rejects credentials, queries and fragments in API bases, and requires TLS
// when Basic authentication leaves this host. Errors never echo the address.
func ValidateURL(raw string) error {
u, err := url.Parse(raw)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" {
return errors.New("stalwart API address must be an http or https address")
}
if u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" {
return errors.New("stalwart API address must not contain a login, query or fragment")
}
if u.Scheme == "http" && !LocalURL(raw) {
return errors.New("stalwart API address must use https unless it is on this host")
}
return nil
}
func (c *Client) redactError(err error) error {
message := Redact(err.Error(), c.user, c.password)
if message == err.Error() {
return err
}
return errors.New(message)
}
package stalwartapi
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"strings"
"syscall"
"github.com/pidginhost/csm/internal/atomicio"
)
// ErrNoKeyFile reports that the API key file does not exist.
var ErrNoKeyFile = errors.New("no Stalwart API key file")
// maxKeyFileBytes bounds the read: a key is 42 characters.
const maxKeyFileBytes = 256
// keyFileOwner is the uid that must own the key file.
var keyFileOwner uint32
// SetKeyFileOwnerForTest lets tests that do not run as root read key files
// they wrote.
func SetKeyFileOwnerForTest(uid uint32) (restore func()) {
old := keyFileOwner
keyFileOwner = uid
return func() { keyFileOwner = old }
}
// ReadKeyFile reads the API key from path. The file must be a regular file
// owned by root and closed to group and others; a final symlink is not
// followed. Errors name the path and never the content.
func ReadKeyFile(path string) (string, error) {
// #nosec G304 -- the operator's configured key path, opened read-only without following a final link.
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if errors.Is(err, fs.ErrNotExist) {
return "", fmt.Errorf("%s: %w", path, ErrNoKeyFile)
}
if err != nil {
return "", err
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil {
return "", err
}
if !info.Mode().IsRegular() {
return "", fmt.Errorf("%s: not a regular file", path)
}
if st, ok := info.Sys().(*syscall.Stat_t); !ok || st.Uid != keyFileOwner {
return "", fmt.Errorf("%s: must be owned by root", path)
}
if perm := info.Mode().Perm(); perm&0o077 != 0 {
return "", fmt.Errorf("%s: mode %04o is open to group or others, want 0600", path, perm)
}
data, err := io.ReadAll(io.LimitReader(f, maxKeyFileBytes+1))
if err != nil {
return "", fmt.Errorf("%s: %w", path, err)
}
if len(data) > maxKeyFileBytes {
return "", fmt.Errorf("%s: larger than %d bytes", path, maxKeyFileBytes)
}
key := strings.TrimSpace(string(data))
if key == "" {
return "", fmt.Errorf("%s: empty", path)
}
return key, nil
}
// WriteKeyFile stores secret at path with mode 0600, replacing any file
// there in one rename so a reader never sees a partial key.
func WriteKeyFile(path, secret string) error {
return atomicio.AtomicWrite(path, 0o600, []byte(secret))
}
package stalwartapi
import (
"context"
"encoding/json"
"errors"
"fmt"
)
// APIKey is one API key of an account. Its secret is never readable.
type APIKey struct {
ID string `json:"id"`
Description string `json:"description"`
}
// APIKeys lists the API keys of the account accountID.
func (c *Client) APIKeys(ctx context.Context, accountID string) ([]APIKey, error) {
var list []APIKey
if err := c.getFor(ctx, accountID, "x:ApiKey/get", nil, &list); err != nil {
return nil, err
}
return list, nil
}
// CreateAPIKey creates an API key for the account accountID that works only
// from allowedIPs. Stalwart returns the secret once, here.
func (c *Client) CreateAPIKey(ctx context.Context, accountID, description string, allowedIPs []string) (id, secret string, err error) {
const method = "x:ApiKey/set"
// An empty list may leave the key without an address limit.
if len(allowedIPs) == 0 {
return "", "", fmt.Errorf("stalwart %s: an API key needs at least one allowed address", method)
}
out, err := c.set(ctx, method, map[string]any{
"accountId": accountID,
"create": map[string]any{"k": map[string]any{
"description": description,
"allowedIps": trueSet(allowedIPs...),
}},
})
if err != nil {
// Creation diagnostics can contain a new secret we have not decoded.
// Preserve only known local error identities, never server text.
for _, known := range []error{ErrUnauthorized, ErrForbidden, context.Canceled, context.DeadlineExceeded} {
if errors.Is(err, known) {
return "", "", fmt.Errorf("stalwart %s: key creation failed: %w", method, known)
}
}
return "", "", fmt.Errorf("stalwart %s: key creation failed", method)
}
var res struct {
Created map[string]struct {
ID string `json:"id"`
Secret string `json:"secret"`
} `json:"created"`
}
// The response holds the secret, so no part of it goes into the error.
if json.Unmarshal(out, &res) != nil || res.Created["k"].ID == "" || res.Created["k"].Secret == "" {
return "", "", fmt.Errorf("stalwart %s: unexpected response", method)
}
return res.Created["k"].ID, res.Created["k"].Secret, nil
}
// DeleteAPIKeys destroys the account's API keys with these ids.
func (c *Client) DeleteAPIKeys(ctx context.Context, accountID string, ids []string) error {
_, err := c.set(ctx, "x:ApiKey/set", map[string]any{"accountId": accountID, "destroy": ids})
return err
}
package stalwartapi
import (
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"strings"
)
// lookupPage is Stalwart's limit on objects in one get and one set.
const lookupPage = 500
// LookupKey is one key of a Stalwart in-memory lookup list.
type LookupKey struct {
ID string `json:"id"`
Key string `json:"key"`
Glob bool `json:"isGlobPattern"`
}
// LookupKeys lists each key once, ending on an empty query page and rejecting pages without new IDs.
func (c *Client) LookupKeys(ctx context.Context, namespace string) ([]LookupKey, error) {
const method = "x:MemoryLookupKey/get"
var all []LookupKey
seen := map[string]bool{}
for pos := 0; ; {
ids, err := c.queryArgs(ctx, "x:MemoryLookupKey/query", map[string]any{
"filter": map[string]any{"namespace": namespace}, "limit": lookupPage, "position": pos,
})
if err != nil {
return nil, err
}
if len(ids) == 0 {
return all, nil
}
var fresh []string
for _, id := range ids {
if !seen[id] {
fresh = append(fresh, id)
seen[id] = true
}
}
if len(fresh) == 0 {
return nil, errors.New("stalwart x:MemoryLookupKey/query: paging repeated a page")
}
for page := range slices.Chunk(fresh, lookupPage) {
out, err := c.call(ctx, method, map[string]any{"ids": page, "properties": []string{"key", "isGlobPattern"}})
if err != nil {
return nil, err
}
var res struct {
List []LookupKey `json:"list"`
}
// A decode error can quote the response, so it is not passed on.
if json.Unmarshal(out, &res) != nil || res.List == nil {
return nil, fmt.Errorf("stalwart %s: unexpected response", method)
}
pending := make(map[string]bool, len(page))
for _, id := range page {
pending[id] = true
}
for _, key := range res.List {
if key.ID == "" || !pending[key.ID] {
return nil, fmt.Errorf("stalwart %s: unexpected response", method)
}
delete(pending, key.ID)
}
// Missing entries are allowed because keys can be deleted between query and get.
all = append(all, res.List...)
}
pos += len(ids)
}
}
// CreateLookupKeys adds keys, treating "*" as a glob; changes take effect after ReloadLookupStores.
func (c *Client) CreateLookupKeys(ctx context.Context, namespace string, keys []string) error {
for page := range slices.Chunk(keys, lookupPage) {
create := make(map[string]any, len(page))
for i, key := range page {
create["k"+strconv.Itoa(i)] = map[string]any{"namespace": namespace, "key": key, "isGlobPattern": strings.Contains(key, "*")}
}
if _, err := c.set(ctx, "x:MemoryLookupKey/set", map[string]any{"create": create}); err != nil {
return err
}
}
return nil
}
// DeleteLookupKeys removes keys by ID; changes take effect after ReloadLookupStores.
func (c *Client) DeleteLookupKeys(ctx context.Context, ids []string) error {
for page := range slices.Chunk(ids, lookupPage) {
if _, err := c.set(ctx, "x:MemoryLookupKey/set", map[string]any{"destroy": page}); err != nil {
return err
}
}
return nil
}
// ReloadLookupStores makes created and deleted lookup keys take effect.
func (c *Client) ReloadLookupStores(ctx context.Context) error {
_, err := c.set(ctx, "x:Action/set", map[string]any{
"create": map[string]any{"r": map[string]any{"@type": "ReloadLookupStores"}},
})
return err
}
// SpamRule tries Arms in order, taking the first match or Else when none match.
type SpamRule struct {
ID, Type string
Enabled bool
Arms []RuleArm
Else string
}
// RuleArm is one if/then pair of a rule's condition.
type RuleArm struct {
If string `json:"if"`
Then string `json:"then"`
}
// SpamRule reads the named rule; a missing rule is not an error.
func (c *Client) SpamRule(ctx context.Context, name string) (SpamRule, bool, error) {
const method = "x:SpamRule/get"
ids, err := c.query(ctx, "x:SpamRule/query", map[string]any{"name": name})
if err != nil || len(ids) == 0 {
return SpamRule{}, false, err
}
var list []struct {
ID string `json:"id"`
Type string `json:"@type"`
Enable bool `json:"enable"`
Condition struct {
Match map[string]RuleArm `json:"match"`
Else string `json:"else"`
} `json:"condition"`
}
if err := c.get(ctx, method, ids[:1], &list); err != nil {
return SpamRule{}, false, err
}
if len(list) == 0 {
return SpamRule{}, false, nil
}
raw := list[0]
type numbered struct {
n uint64
key string
}
var keys []numbered
for key := range raw.Condition.Match {
n, err := strconv.ParseUint(key, 10, 32)
if err != nil {
return SpamRule{}, false, fmt.Errorf("stalwart %s: unexpected response", method)
}
keys = append(keys, numbered{n, key})
}
slices.SortFunc(keys, func(a, b numbered) int { return cmp.Or(cmp.Compare(a.n, b.n), strings.Compare(a.key, b.key)) })
r := SpamRule{ID: raw.ID, Type: raw.Type, Enabled: raw.Enable, Else: raw.Condition.Else}
for _, k := range keys {
r.Arms = append(r.Arms, raw.Condition.Match[k.key])
}
return r, true, nil
}
// CreateSpamRule adds an enabled rule with r's condition and Stalwart's default priority.
func (c *Client) CreateSpamRule(ctx context.Context, name string, r SpamRule) error {
_, err := c.set(ctx, "x:SpamRule/set", map[string]any{"create": map[string]any{"r": map[string]any{
"@type": "Any", "name": name, "enable": true, "condition": ruleCondition(r.Arms, r.Else),
}}})
return err
}
// SetSpamRuleCondition replaces a rule's condition without changing its other fields.
func (c *Client) SetSpamRuleCondition(ctx context.Context, id string, arms []RuleArm, elseExpr string) error {
_, err := c.set(ctx, "x:SpamRule/set", map[string]any{"update": map[string]any{
id: map[string]any{"condition": ruleCondition(arms, elseExpr)},
}})
return err
}
func ruleCondition(arms []RuleArm, elseExpr string) map[string]any {
match := make(map[string]RuleArm, len(arms))
for i, arm := range arms {
match[strconv.Itoa(i)] = arm
}
return map[string]any{"match": match, "else": elseExpr}
}
// SpamTag is one Stalwart spam filter tag and its score.
type SpamTag struct {
ID string `json:"id"`
Type string `json:"@type"`
Tag string `json:"tag"`
Score float64 `json:"score"`
}
// SpamTag reads the tag named tag. A missing tag is not found, not an error.
func (c *Client) SpamTag(ctx context.Context, tag string) (SpamTag, bool, error) {
ids, err := c.query(ctx, "x:SpamTag/query", map[string]any{"tag": tag})
if err != nil || len(ids) == 0 {
return SpamTag{}, false, err
}
var list []SpamTag
if err := c.get(ctx, "x:SpamTag/get", ids[:1], &list); err != nil || len(list) == 0 {
return SpamTag{}, false, err
}
return list[0], true, nil
}
// CreateSpamTag adds a scoring tag named tag.
func (c *Client) CreateSpamTag(ctx context.Context, tag string, score float64) error {
_, err := c.set(ctx, "x:SpamTag/set", map[string]any{"create": map[string]any{
"t": map[string]any{"@type": "Score", "tag": tag, "score": score},
}})
return err
}
package stalwartapi
import (
"bufio"
"context"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
"net/netip"
"net/url"
"os"
"os/user"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
)
// stalwartServiceUsers are the accounts Stalwart's packages run it as.
var stalwartServiceUsers = []string{"stalwart", "stalwart-mail"}
// CheckListener refuses a local API address whose port is held by anyone
// but root or Stalwart's service user.
func CheckListener(rawURL string) error {
return checkStalwartListener("/proc/net", rawURL, lookupUIDs)
}
func lookupUIDs(names ...string) []uint32 {
var out []uint32
for _, name := range names {
u, err := user.Lookup(name)
if err != nil {
continue
}
if uid, err := strconv.ParseUint(u.Uid, 10, 32); err == nil {
out = append(out, uint32(uid))
}
}
return out
}
// checkStalwartListener runs before the login is sent to an address on
// this host. Any local user can listen on an unprivileged port while
// Stalwart is down and collect the login, so every listening socket on the
// API port must belong to root or Stalwart's service user. A remote address
// is protected by TLS instead.
func checkStalwartListener(procNet, rawURL string, uidsOf func(names ...string) []uint32) error {
if !LocalURL(rawURL) {
return nil
}
u, err := url.Parse(rawURL)
if err != nil {
return err
}
port := 80
if u.Scheme == "https" {
port = 443
}
if p := u.Port(); p != "" {
if port, err = strconv.Atoi(p); err != nil {
return fmt.Errorf("API port %q is not a number", p)
}
}
var owners []uint32
for _, name := range []string{"tcp", "tcp6"} {
f, err := os.Open(filepath.Join(procNet, name)) // #nosec G304 -- fixed /proc paths
if errors.Is(err, os.ErrNotExist) && name == "tcp6" {
continue // IPv6 may be disabled in the kernel.
}
if err != nil {
return fmt.Errorf("cannot inspect %s sockets: %w", name, err)
}
found, readErr := listenOwners(f, port)
_ = f.Close()
if readErr != nil {
return fmt.Errorf("cannot inspect %s sockets: %w", name, readErr)
}
owners = append(owners, found...)
}
if len(owners) == 0 {
return fmt.Errorf("nothing on this host listens on port %d; is Stalwart running?", port)
}
allowed := append([]uint32{0}, uidsOf(stalwartServiceUsers...)...)
for _, uid := range owners {
if !slices.Contains(allowed, uid) {
return fmt.Errorf("port %d is held by uid %d, not root or Stalwart's service user", port, uid)
}
}
return nil
}
// listenOwners returns the uids of listening TCP sockets on port, read from
// a /proc/net/tcp or tcp6 table.
func listenOwners(r io.Reader, port int) ([]uint32, error) {
var out []uint32
sc := bufio.NewScanner(r)
for sc.Scan() {
f := strings.Fields(sc.Text())
if len(f) < 8 || f[3] != "0A" { // 0A is TCP_LISTEN
continue
}
_, hexPort, ok := strings.Cut(f[1], ":")
if !ok {
continue
}
p, err := strconv.ParseUint(hexPort, 16, 16)
if err != nil || int(p) != port {
continue
}
uid, err := strconv.ParseUint(f[7], 10, 32)
if err != nil {
return nil, errors.New("invalid socket owner")
}
out = append(out, uint32(uid))
}
return out, sc.Err()
}
// CheckPeer verifies that root or Stalwart's service user accepted conn. A
// listener check alone races service restarts, so it checks the accepted
// side of this connection, not whichever socket now listens on its port.
func CheckPeer(ctx context.Context, conn net.Conn) error {
local, err := netip.ParseAddrPort(conn.LocalAddr().String())
if err != nil {
return errors.New("cannot identify local API connection")
}
remote, err := netip.ParseAddrPort(conn.RemoteAddr().String())
if err != nil || !remote.Addr().Unmap().IsLoopback() {
return errors.New("local API address did not connect to loopback")
}
return waitStalwartPeer(ctx, "/proc/net", local, remote, lookupUIDs)
}
func waitStalwartPeer(ctx context.Context, procNet string, local, remote netip.AddrPort, uidsOf func(...string) []uint32) error {
allowed := append([]uint32{0}, uidsOf(stalwartServiceUsers...)...)
ctx, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
for _, name := range []string{"tcp", "tcp6"} {
f, err := os.Open(filepath.Join(procNet, name)) // #nosec G304 -- fixed /proc paths
if errors.Is(err, os.ErrNotExist) && name == "tcp6" {
continue
}
if err != nil {
return fmt.Errorf("cannot inspect API peer: %w", err)
}
uid, found, readErr := stalwartPeerOwner(f, local, remote)
_ = f.Close()
if readErr != nil {
return fmt.Errorf("cannot inspect API peer: %w", readErr)
}
if found {
if !slices.Contains(allowed, uid) {
return fmt.Errorf("API peer is held by uid %d, not root or Stalwart's service user", uid)
}
return nil
}
}
// Until accept(), procfs can report uid 0 and inode 0 even for an
// untrusted listener. Never treat that placeholder as a root peer.
select {
case <-ctx.Done():
return fmt.Errorf("cannot verify accepted API peer: %w", ctx.Err())
case <-ticker.C:
}
}
}
func stalwartPeerOwner(r io.Reader, local, remote netip.AddrPort) (uint32, bool, error) {
sc := bufio.NewScanner(r)
for sc.Scan() {
f := strings.Fields(sc.Text())
if len(f) < 10 || f[3] != "01" {
continue
}
server, err := procSocketAddress(f[1])
if err != nil {
return 0, false, err
}
client, err := procSocketAddress(f[2])
if err != nil {
return 0, false, err
}
if server != netip.AddrPortFrom(remote.Addr().Unmap(), remote.Port()) ||
client != netip.AddrPortFrom(local.Addr().Unmap(), local.Port()) {
continue
}
inode, err := strconv.ParseUint(f[9], 10, 64)
if err != nil {
return 0, false, errors.New("invalid socket inode")
}
if inode == 0 {
continue
}
uid, err := strconv.ParseUint(f[7], 10, 32)
if err != nil {
return 0, false, errors.New("invalid socket owner")
}
return uint32(uid), true, nil
}
return 0, false, sc.Err()
}
func procSocketAddress(raw string) (netip.AddrPort, error) {
host, port, ok := strings.Cut(raw, ":")
if !ok {
return netip.AddrPort{}, errors.New("invalid socket address")
}
b, err := hex.DecodeString(host)
if err != nil || (len(b) != 4 && len(b) != 16) {
return netip.AddrPort{}, errors.New("invalid socket address")
}
// procfs renders each native-endian 32-bit word as hexadecimal.
for i := 0; i < len(b); i += 4 {
word := binary.BigEndian.Uint32(b[i : i+4])
binary.NativeEndian.PutUint32(b[i:i+4], word)
}
ip, _ := netip.AddrFromSlice(b)
p, err := strconv.ParseUint(port, 16, 16)
if err != nil {
return netip.AddrPort{}, errors.New("invalid socket port")
}
return netip.AddrPortFrom(ip.Unmap(), uint16(p)), nil
}
package stalwartapi
import (
"context"
"encoding/json"
"fmt"
"slices"
)
// Role is a Stalwart role: the permissions it enables or denies and
// the roles it includes, all sorted.
type Role struct {
ID, Description string
Enabled []string
Disabled []string
RoleIDs []string
}
// RoleByDescription finds the role whose description is exactly description.
func (c *Client) RoleByDescription(ctx context.Context, description string) (Role, bool, error) {
ids, err := c.query(ctx, "x:Role/query", map[string]any{"description": description})
if err != nil || len(ids) == 0 {
return Role{}, false, err
}
var list []struct {
ID string `json:"id"`
Description string `json:"description"`
Enabled map[string]bool `json:"enabledPermissions"`
Disabled map[string]bool `json:"disabledPermissions"`
RoleIDs map[string]bool `json:"roleIds"`
}
if err := c.get(ctx, "x:Role/get", ids, &list); err != nil {
return Role{}, false, err
}
// The query filter is not documented as exact, so the match is checked here.
for _, r := range list {
if r.Description == description {
return Role{ID: r.ID, Description: r.Description, Enabled: setKeys(r.Enabled), Disabled: setKeys(r.Disabled), RoleIDs: setKeys(r.RoleIDs)}, true, nil
}
}
return Role{}, false, nil
}
// SaveRole creates r when it has no id, and otherwise replaces its enabled
// and disabled permissions and included roles with exactly those of r. It returns the id.
func (c *Client) SaveRole(ctx context.Context, r Role) (string, error) {
if r.ID != "" {
_, err := c.set(ctx, "x:Role/set", map[string]any{"update": map[string]any{r.ID: map[string]any{
"enabledPermissions": trueSet(r.Enabled...),
"disabledPermissions": trueSet(r.Disabled...),
"roleIds": trueSet(r.RoleIDs...),
}}})
if err != nil {
return "", err
}
return r.ID, nil
}
return c.create(ctx, "x:Role/set", map[string]any{
"description": r.Description,
"enabledPermissions": trueSet(r.Enabled...),
"disabledPermissions": trueSet(r.Disabled...),
"roleIds": trueSet(r.RoleIDs...),
})
}
// Domain is one of Stalwart's mail domains.
type Domain struct {
ID string `json:"id"`
Name string `json:"name"`
Aliases []string `json:"aliases,omitempty"`
}
// Domains lists every mail domain with its aliases.
func (c *Client) Domains(ctx context.Context) ([]Domain, error) {
var raw []struct {
ID string `json:"id"`
Name string `json:"name"`
Aliases map[string]bool `json:"aliases"`
}
if err := c.get(ctx, "x:Domain/get", nil, &raw); err != nil {
return nil, err
}
list := make([]Domain, 0, len(raw))
for _, r := range raw {
d := Domain{ID: r.ID, Name: r.Name}
for alias, on := range r.Aliases {
if on {
d.Aliases = append(d.Aliases, alias)
}
}
slices.Sort(d.Aliases)
list = append(list, d)
}
return list, nil
}
// CreateAccount creates a user account with no password that holds only the
// role roleID, and returns its id. It can sign in only with an API key.
func (c *Client) CreateAccount(ctx context.Context, name, domainID, roleID string) (string, error) {
return c.create(ctx, "x:Account/set", map[string]any{
"@type": "User",
"name": name,
"domainId": domainID,
"roles": customRoles(roleID),
})
}
// AccountRoleIDs lists the custom roles the account holds, sorted. A built-in
// role is none. An unknown account is an error.
func (c *Client) AccountRoleIDs(ctx context.Context, id string) ([]string, error) {
const method = "x:Account/get"
var list []struct {
ID string `json:"id"`
Roles struct {
RoleIDs map[string]bool `json:"roleIds"`
} `json:"roles"`
}
out, err := c.call(ctx, method, map[string]any{"ids": []string{id}, "properties": []string{"id", "roles"}})
if err != nil {
return nil, err
}
var res struct {
List json.RawMessage `json:"list"`
}
if json.Unmarshal(out, &res) != nil || json.Unmarshal(res.List, &list) != nil {
return nil, fmt.Errorf("stalwart %s: unexpected response", method)
}
for _, a := range list {
if a.ID == id {
return setKeys(a.Roles.RoleIDs), nil
}
}
return nil, fmt.Errorf("stalwart %s: account not found", method)
}
// AccountCredentialTypes lists the kinds of the account's credentials, such
// as Password, AppPassword or ApiKey, sorted. The credentials carry secrets,
// so only their kinds are decoded. An unknown account is an error.
func (c *Client) AccountCredentialTypes(ctx context.Context, id string) ([]string, error) {
const method = "x:Account/get"
out, err := c.call(ctx, method, map[string]any{"ids": []string{id}, "properties": []string{"id", "credentials"}})
if err != nil {
return nil, err
}
var res struct {
List []struct {
ID string `json:"id"`
Credentials map[string]struct {
Type string `json:"@type"`
} `json:"credentials"`
} `json:"list"`
}
// A decode error can quote the response, so it is not passed on.
if json.Unmarshal(out, &res) != nil {
return nil, fmt.Errorf("stalwart %s: unexpected response", method)
}
for _, a := range res.List {
if a.ID != id {
continue
}
types := make([]string, 0, len(a.Credentials))
for _, cred := range a.Credentials {
types = append(types, cred.Type)
}
slices.Sort(types)
return types, nil
}
return nil, fmt.Errorf("stalwart %s: account not found", method)
}
// SetAccountRole makes roleID the account's only role.
func (c *Client) SetAccountRole(ctx context.Context, id, roleID string) error {
_, err := c.set(ctx, "x:Account/set", map[string]any{"update": map[string]any{id: map[string]any{"roles": customRoles(roleID)}}})
return err
}
func customRoles(roleID string) map[string]any {
return map[string]any{"@type": "Custom", "roleIds": trueSet(roleID)}
}
// create sends one /set create and returns the new object's id.
func (c *Client) create(ctx context.Context, method string, object map[string]any) (string, error) {
out, err := c.set(ctx, method, map[string]any{"create": map[string]any{"c": object}})
if err != nil {
return "", err
}
var res struct {
Created map[string]struct {
ID string `json:"id"`
} `json:"created"`
}
if err := json.Unmarshal(out, &res); err != nil || res.Created["c"].ID == "" {
return "", fmt.Errorf("stalwart %s: unexpected response", method)
}
return res.Created["c"].ID, nil
}
// trueSet builds the JMAP set-as-map of keys: each key mapped to true.
func trueSet(keys ...string) map[string]bool {
m := make(map[string]bool, len(keys))
for _, k := range keys {
m[k] = true
}
return m
}
package stalwartapi
import (
"context"
"encoding/json"
"fmt"
"net/netip"
"sort"
"strings"
"time"
)
// TracingLevel is one per-event log level override.
type TracingLevel struct {
ID string `json:"id"`
Event string `json:"event"`
Level string `json:"level"`
}
// TracingLevels lists the per-event log level overrides.
func (c *Client) TracingLevels(ctx context.Context) ([]TracingLevel, error) {
var list []TracingLevel
if err := c.get(ctx, "x:EventTracingLevel/get", nil, &list); err != nil {
return nil, err
}
return list, nil
}
// SetTracingLevel sets the log level of event: it updates the override id,
// or creates one when id is empty.
func (c *Client) SetTracingLevel(ctx context.Context, id, event, level string) error {
args := map[string]any{"update": map[string]any{id: map[string]any{"level": level}}}
if id == "" {
args = map[string]any{"create": map[string]any{"c": map[string]any{"event": event, "level": level}}}
}
_, err := c.set(ctx, "x:EventTracingLevel/set", args)
return err
}
// HTTPSettings is where the HTTP listener takes the client address from.
type HTTPSettings struct {
ClientIPSource string `json:"clientIpSource"`
ForwardedHops int `json:"forwardedHops"`
}
// HTTPSettings reads the HTTP client address settings.
func (c *Client) HTTPSettings(ctx context.Context) (HTTPSettings, error) {
var h HTTPSettings
err := c.getSingleton(ctx, "x:Http/get", &h)
return h, err
}
// SetHTTPSettings writes the HTTP client address settings.
func (c *Client) SetHTTPSettings(ctx context.Context, h HTTPSettings) error {
patch := map[string]any{"clientIpSource": h.ClientIPSource}
if h.ForwardedHops > 0 {
patch["forwardedHops"] = h.ForwardedHops
}
return c.updateSingleton(ctx, "x:Http/set", patch)
}
// BanPeriods are how long Stalwart's bans last. Nil means permanent.
type BanPeriods struct {
Auth, Abuse, Scan *time.Duration
}
// BanPeriods reads the authentication, abuse and scan ban periods.
func (c *Client) BanPeriods(ctx context.Context) (BanPeriods, error) {
var raw struct {
Auth *int64 `json:"authBanPeriod"`
Abuse *int64 `json:"abuseBanPeriod"`
Scan *int64 `json:"scanBanPeriod"`
}
if err := c.getSingleton(ctx, "x:Security/get", &raw); err != nil {
return BanPeriods{}, err
}
ms := func(v *int64) *time.Duration {
if v == nil {
return nil
}
d := time.Duration(*v) * time.Millisecond
return &d
}
return BanPeriods{Auth: ms(raw.Auth), Abuse: ms(raw.Abuse), Scan: ms(raw.Scan)}, nil
}
// SetBanPeriods sets the authentication, abuse and scan ban periods to d.
// The loiter ban is left alone.
func (c *Client) SetBanPeriods(ctx context.Context, d time.Duration) error {
ms := d.Milliseconds()
return c.updateSingleton(ctx, "x:Security/set", map[string]any{
"authBanPeriod": ms, "abuseBanPeriod": ms, "scanBanPeriod": ms,
})
}
// Listener is one network listener.
type Listener struct {
ID string
Name string
Protocol string
Bind []netip.AddrPort
// ProxyNetworks are the listener's own PROXY protocol networks.
ProxyNetworks []string
}
// Listeners lists the network listeners. An address CSM cannot read is an
// error, because the caller decides whether a listener is reachable from
// outside.
func (c *Client) Listeners(ctx context.Context) ([]Listener, error) {
var raw []struct {
ID string `json:"id"`
Name string `json:"name"`
Protocol string `json:"protocol"`
Bind map[string]bool `json:"bind"`
Proxy map[string]bool `json:"overrideProxyTrustedNetworks"`
}
const method = "x:NetworkListener/get"
if err := c.get(ctx, method, nil, &raw); err != nil {
return nil, err
}
out := make([]Listener, 0, len(raw))
for _, r := range raw {
l := Listener{ID: r.ID, Name: r.Name, Protocol: r.Protocol, ProxyNetworks: setKeys(r.Proxy)}
for _, b := range setKeys(r.Bind) {
ap, err := netip.ParseAddrPort(b)
if err != nil {
return nil, fmt.Errorf("stalwart %s: listener %q binds %q, not an address and port", method, Redact(r.Name, c.user, c.password), Redact(b, c.user, c.password))
}
l.Bind = append(l.Bind, ap)
}
sort.Slice(l.Bind, func(i, j int) bool { return l.Bind[i].Compare(l.Bind[j]) < 0 })
out = append(out, l)
}
return out, nil
}
// ProxyTrustedNetworks reads the server-wide PROXY protocol networks.
func (c *Client) ProxyTrustedNetworks(ctx context.Context) ([]string, error) {
var raw struct {
Networks map[string]bool `json:"proxyTrustedNetworks"`
}
if err := c.getSingleton(ctx, "x:SystemSettings/get", &raw); err != nil {
return nil, err
}
return setKeys(raw.Networks), nil
}
// ReloadSettings makes Stalwart apply its stored settings and returns the
// warnings it reports.
func (c *Client) ReloadSettings(ctx context.Context) ([]string, error) {
const method = "x:Action/set"
out, err := c.set(ctx, method, map[string]any{
"create": map[string]any{"r": map[string]any{"@type": "ReloadSettings"}},
})
if err != nil {
return nil, err
}
var res struct {
Created map[string]struct {
Warnings map[string]json.RawMessage `json:"warnings"`
} `json:"created"`
}
if err := json.Unmarshal(out, &res); err != nil {
return nil, fmt.Errorf("stalwart %s: unexpected response", method)
}
r, ok := res.Created["r"]
if !ok {
return nil, fmt.Errorf("stalwart %s: reload was not acknowledged", method)
}
warnings := make([]string, 0, len(r.Warnings))
for k, v := range r.Warnings {
var s string
if json.Unmarshal(v, &s) != nil {
s = string(v)
}
warnings = append(warnings, Redact(k+": "+s, c.user, c.password))
}
sort.Strings(warnings)
return warnings, nil
}
// setKeys returns the keys a JMAP set-as-map marks true, sorted.
func setKeys(m map[string]bool) []string {
var out []string
for k, v := range m {
if v {
out = append(out, k)
}
}
sort.Strings(out)
return out
}
// SpamSettings are the spam filter switch and its score thresholds. A score
// of 0 means that action is off.
type SpamSettings struct {
Enabled bool
ScoreSpam, ScoreReject, ScoreDiscard float64
}
// SpamSettings reads the spam filter switch and thresholds. Missing or null
// fields are errors, not disabled settings.
func (c *Client) SpamSettings(ctx context.Context) (SpamSettings, error) {
const method = "x:SpamSettings/get"
var raw struct {
Enable *bool `json:"enable"`
ScoreSpam *float64 `json:"scoreSpam"`
ScoreReject *float64 `json:"scoreReject"`
ScoreDiscard *float64 `json:"scoreDiscard"`
}
if err := c.getSingleton(ctx, method, &raw); err != nil {
return SpamSettings{}, err
}
if raw.Enable == nil || raw.ScoreSpam == nil || raw.ScoreReject == nil || raw.ScoreDiscard == nil {
return SpamSettings{}, fmt.Errorf("stalwart %s: unexpected response", method)
}
return SpamSettings{Enabled: *raw.Enable, ScoreSpam: *raw.ScoreSpam, ScoreReject: *raw.ScoreReject, ScoreDiscard: *raw.ScoreDiscard}, nil
}
// SpamFilterStageOff reports whether the SMTP data stage never runs the spam
// filter: no conditional rule and a plain false default. An unexpected settings
// shape is an error, not evidence that the filter is off.
func (c *Client) SpamFilterStageOff(ctx context.Context) (bool, error) {
const method = "x:MtaStageData/get"
var raw struct {
Filter struct {
Match map[string]json.RawMessage `json:"match"`
Else *string `json:"else"`
} `json:"enableSpamFilter"`
}
if err := c.getSingleton(ctx, method, &raw); err != nil {
return false, err
}
if raw.Filter.Match == nil || raw.Filter.Else == nil {
return false, fmt.Errorf("stalwart %s: unexpected response", method)
}
return len(raw.Filter.Match) == 0 && strings.TrimSpace(*raw.Filter.Else) == "false", nil
}
// SpamProbe is one message for Stalwart to classify without delivering it.
type SpamProbe struct{ Message, RemoteIP, EhloDomain, MailFrom, RcptTo string }
// SpamProbeResult is Stalwart's verdict on a probe. Tags are the names of
// the rules that matched, sorted. Server text has the admin password redacted.
type SpamProbeResult struct {
Score float64
Result string
Tags []string
}
// BlocklistProbe is a harmless message whose tags show whether Stalwart's
// DNS blocklist lookups are refused by its resolver. It comes from the
// address the daemon keeps on CSM's list, so it also shows that list matches.
var BlocklistProbe = SpamProbe{
Message: "From: sender@example.com\r\n" +
"To: postmaster@example.com\r\n" +
"Subject: CSM blocklist check\r\n" +
"Date: Sat, 14 Nov 2026 10:00:00 +0000\r\n" +
"Message-ID: <csm-blocklist-probe@example.com>\r\n" +
"\r\n" +
"This message checks the spam filter's blocklist lookups.\r\n",
RemoteIP: "192.0.2.255",
EhloDomain: "mx.example.com",
MailFrom: "sender@example.com",
RcptTo: "postmaster@example.com",
}
// ClassifySpam asks Stalwart to score p with its spam filter. An incomplete
// classification is an error, not a zero score or an empty set of matched tags.
func (c *Client) ClassifySpam(ctx context.Context, p SpamProbe) (SpamProbeResult, error) {
const method = "x:Action/set"
out, err := c.set(ctx, method, map[string]any{
"create": map[string]any{"c": map[string]any{
"@type": "ClassifySpam",
"message": p.Message,
"remoteIp": p.RemoteIP,
"ehloDomain": p.EhloDomain,
"isTls": false,
"mailFrom": p.MailFrom,
"rcptTo": map[string]bool{p.RcptTo: true},
}},
})
if err != nil {
return SpamProbeResult{}, err
}
var res struct {
Created map[string]struct {
Score *float64 `json:"score"`
Result string `json:"result"`
Tags map[string]json.RawMessage `json:"tags"`
} `json:"created"`
}
if err := json.Unmarshal(out, &res); err != nil {
return SpamProbeResult{}, fmt.Errorf("stalwart %s: unexpected response", method)
}
r, ok := res.Created["c"]
if !ok {
return SpamProbeResult{}, fmt.Errorf("stalwart %s: classification was not acknowledged", method)
}
if r.Score == nil || r.Result == "" || r.Tags == nil {
return SpamProbeResult{}, fmt.Errorf("stalwart %s: unexpected response", method)
}
tags := make([]string, 0, len(r.Tags))
for name := range r.Tags {
tags = append(tags, Redact(name, c.user, c.password))
}
sort.Strings(tags)
return SpamProbeResult{Score: *r.Score, Result: Redact(r.Result, c.user, c.password), Tags: tags}, nil
}
package stalwartlog
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
)
// NewestFile returns the newest "<prefix>.YYYY-MM-DD" regular file in dir.
// Stalwart starts one such file per UTC day, so the newest by name is the
// one being written. Files dated after tomorrow (UTC, relative to now) are
// skipped: Stalwart cannot be writing them, and a reader that only moves
// forward would stay on one. The error wraps fs.ErrNotExist when there is none.
func NewestFile(dir, prefix string, now time.Time) (string, error) {
entries, err := os.ReadDir(dir)
if err != nil {
return "", err
}
y, m, d := now.UTC().Date()
latest := time.Date(y, m, d+1, 0, 0, 0, 0, time.UTC)
newest := ""
for _, e := range entries {
if !e.Type().IsRegular() {
continue
}
date, ok := strings.CutPrefix(e.Name(), prefix+".")
if !ok {
continue
}
day, err := time.Parse("2006-01-02", date)
if err != nil || len(date) != len("2006-01-02") || day.After(latest) {
continue
}
if e.Name() > newest {
newest = e.Name()
}
}
if newest == "" {
return "", fmt.Errorf("no %s.YYYY-MM-DD file in %s: %w", prefix, dir, fs.ErrNotExist)
}
return filepath.Join(dir, newest), nil
}
// Package stalwartlog parses the text log Stalwart mail server 1.0 writes:
// one event per line, "<RFC3339> <LEVEL> <description> (<event.name>)
// key = value, ...". Values are quoted strings, lists of them, (null) or bare
// tokens. Session keys (listenerId, localPort, remoteIp, remotePort) come
// before any client-supplied value, and the first occurrence of a key wins,
// so a quoted value cannot supply an address.
package stalwartlog
import (
"net/netip"
"strconv"
"strings"
"time"
"unicode/utf8"
)
// Event is one parsed log line. Fields holds scalar values with quotes
// removed and escapes decoded; Lists holds list values. Sub-event values
// retain their full text, with nested keys kept out of both maps. A (null) value
// leaves its key absent from both.
type Event struct {
Time time.Time
Level string
Name string
Fields map[string]string
Lists map[string][]string
}
// Session is the connection an event happened on.
type Session struct {
Listener string
LocalPort uint16
RemoteIP netip.Addr
RemotePort uint16
}
// Parse reads one log line. It reports false for anything that is not a
// complete event in Stalwart's layout.
func Parse(line string) (Event, bool) {
line = strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r")
stamp, rest, ok := strings.Cut(line, " ")
if !ok {
return Event{}, false
}
at, err := time.Parse(time.RFC3339, stamp)
if err != nil {
return Event{}, false
}
level, rest, ok := strings.Cut(rest, " ")
if !ok || !isLevel(level) {
return Event{}, false
}
name, tail, ok := eventName(rest)
if !ok {
return Event{}, false
}
ev := Event{Time: at, Level: level, Name: name, Fields: map[string]string{}, Lists: map[string][]string{}}
if _, ok := parseFields(tail, &ev, 0, ""); !ok {
return Event{}, false
}
return ev, true
}
// Session returns the event's connection, or false when a port or the
// client address is missing or malformed.
func (e Event) Session() (Session, bool) {
local, ok := port(e.Fields["localPort"])
if !ok {
return Session{}, false
}
remote, ok := port(e.Fields["remotePort"])
if !ok {
return Session{}, false
}
ip, err := netip.ParseAddr(e.Fields["remoteIp"])
if err != nil || ip.Zone() != "" {
return Session{}, false
}
return Session{Listener: e.Fields["listenerId"], LocalPort: local, RemoteIP: ip.Unmap(), RemotePort: remote}, true
}
func port(s string) (uint16, bool) {
n, err := strconv.ParseUint(s, 10, 16)
if err != nil || n == 0 {
return 0, false
}
return uint16(n), true
}
func isLevel(s string) bool {
if s == "" {
return false
}
for i := 0; i < len(s); i++ {
if s[i] < 'A' || s[i] > 'Z' {
return false
}
}
return true
}
// eventName finds the first "(name)" after the description whose name is a
// dotted event name followed by a space or the end of the line. The
// description is Stalwart's fixed text, never client data.
func eventName(rest string) (name, tail string, ok bool) {
for i := 0; i < len(rest); i++ {
if rest[i] != '(' || (i > 0 && rest[i-1] != ' ') {
continue
}
end := strings.IndexByte(rest[i:], ')')
if end < 0 {
return "", "", false
}
end += i
candidate := rest[i+1 : end]
if !isEventName(candidate) {
continue
}
after := rest[end+1:]
if after == "" {
return candidate, "", true
}
if after[0] == ' ' {
return candidate, after[1:], true
}
}
return "", "", false
}
func isEventName(s string) bool {
dots := 0
prev := byte('.')
for i := 0; i < len(s); i++ {
c := s[i]
switch {
case c == '.':
if prev == '.' {
return false
}
dots++
case c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '-':
default:
return false
}
prev = c
}
return dots > 0 && prev != '.'
}
// maxNesting bounds how deep sub-event values may nest.
const maxNesting = 4
// parseFields reads "key = value, ..." up to closing, or to the end of s when
// closing is empty. Values go into ev, first key wins; a nil ev discards them,
// which is how keys inside a sub-event stay out of the top-level fields.
func parseFields(s string, ev *Event, depth int, closing string) (string, bool) {
if s == "" && closing == "" {
return "", true
}
stops := []string{", "}
if closing != "" {
stops = append(stops, closing)
}
seen := map[string]bool{}
for {
key := keyLen(s)
if key == 0 || !strings.HasPrefix(s[key:], " = ") {
return "", false
}
name := s[:key]
s = s[key+len(" = "):]
var (
scalar string
list []string
isList bool
null bool
ok bool
)
switch {
case strings.HasPrefix(s, `"`):
scalar, s, ok = quoted(s)
case strings.HasPrefix(s, "["):
list, s, ok = listValue(s, depth)
isList = true
case strings.HasPrefix(s, "(null)"):
s, null, ok = s[len("(null)"):], true, true
default:
if event, after, isEvent, valid := subEvent(s, depth); isEvent {
scalar, s, ok = event, after, valid
} else {
scalar, s, ok = bare(s, stops...)
}
}
if !ok {
return "", false
}
if ev != nil && !seen[name] {
seen[name] = true
switch {
case null:
case isList:
ev.Lists[name] = list
default:
ev.Fields[name] = scalar
}
}
if closing == "" && s == "" {
return "", true
}
if closing != "" {
if after, done := strings.CutPrefix(s, closing); done {
return after, true
}
}
var more bool
s, more = strings.CutPrefix(s, ", ")
if !more {
return "", false
}
}
}
// subEvent reads an event with a field block. Blockless events stay bare
// values, preserving the text callers already received. Nested keys are
// validated and discarded; the enclosing value retains its original text.
func subEvent(s string, depth int) (value, rest string, isEvent, ok bool) {
for i := 0; i < len(s); i++ {
// Never search subsequent values or quoted client text for an event.
if strings.ContainsRune(`"[]{}`, rune(s[i])) || strings.HasPrefix(s[i:], ", ") {
return "", "", false, false
}
if s[i] != '(' || i < 2 || s[i-1] != ' ' {
continue
}
end := strings.IndexByte(s[i:], ')')
if end < 0 {
return "", "", false, false
}
end += i
after := s[end+1:]
if !isEventName(s[i+1:end]) || !strings.HasPrefix(after, " {") {
continue
}
if depth >= maxNesting {
return "", "", true, false
}
if tail, empty := strings.CutPrefix(after, " {}"); empty {
return s[:len(s)-len(tail)], tail, true, true
}
if fields, block := strings.CutPrefix(after, " { "); block {
rest, ok = parseFields(fields, nil, depth+1, " }")
if ok {
return s[:len(s)-len(rest)], rest, true, true
}
}
return "", "", true, false
}
return "", "", false, false
}
func keyLen(s string) int {
n := 0
for n < len(s) {
c := s[n]
if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_' || c == '-' {
n++
continue
}
break
}
return n
}
// quoted reads a double-quoted string. A backslash escapes the next
// character; \n, \t, \r, \0 and \u{...} decode, others stand for themselves.
func quoted(s string) (value, rest string, ok bool) {
var b strings.Builder
for i := 1; i < len(s); i++ {
switch c := s[i]; c {
case '"':
return b.String(), s[i+1:], true
case '\\':
i++
if i >= len(s) {
return "", "", false
}
switch e := s[i]; e {
case 'n':
b.WriteByte('\n')
case 't':
b.WriteByte('\t')
case 'r':
b.WriteByte('\r')
case '0':
b.WriteByte(0)
case 'u':
r, n := unicodeEscape(s[i+1:])
if n == 0 {
b.WriteByte(e)
continue
}
b.WriteRune(r)
i += n
default:
b.WriteByte(e)
}
default:
b.WriteByte(c)
}
}
return "", "", false
}
// unicodeEscape decodes "{hex}" after a \u, returning the rune and the bytes
// consumed, or 0 consumed when s does not start with a valid escape.
func unicodeEscape(s string) (rune, int) {
if !strings.HasPrefix(s, "{") {
return 0, 0
}
end := strings.IndexByte(s, '}')
if end < 2 || end > 7 {
return 0, 0
}
v, err := strconv.ParseUint(s[1:end], 16, 32)
if err != nil || v > utf8.MaxRune {
return 0, 0
}
r := rune(v) // #nosec G115 -- bounded by utf8.MaxRune above
if !utf8.ValidRune(r) {
return 0, 0
}
return r, end + 1
}
// listValue reads ["a", "b"]. Elements are quoted strings, sub-events or
// bare tokens.
func listValue(s string, depth int) (list []string, rest string, ok bool) {
s = s[1:]
if after, empty := strings.CutPrefix(s, "]"); empty {
return []string{}, after, true
}
for {
var v string
if strings.HasPrefix(s, `"`) {
v, s, ok = quoted(s)
} else if name, after, isEvent, valid := subEvent(s, depth); isEvent {
v, s, ok = name, after, valid
} else {
v, s, ok = bare(s, ", ", "]")
}
if !ok {
return nil, "", false
}
list = append(list, v)
if after, done := strings.CutPrefix(s, "]"); done {
return list, after, true
}
var more bool
s, more = strings.CutPrefix(s, ", ")
if !more {
return nil, "", false
}
}
}
// bare reads an unquoted token up to the first stop sequence or the end.
// It must be non-empty and contain no quote.
func bare(s string, stops ...string) (value, rest string, ok bool) {
end := len(s)
for _, stop := range stops {
if i := strings.Index(s, stop); i >= 0 && i < end {
end = i
}
}
value = s[:end]
if value == "" || strings.ContainsRune(value, '"') {
return "", "", false
}
return value, s[end:], true
}
package stalwartlog
import (
"fmt"
"strconv"
"strings"
)
// Protocol is what a Stalwart listener serves, known from its port.
type Protocol string
const (
ProtoUnknown Protocol = ""
ProtoSMTP Protocol = "smtp"
ProtoIMAP Protocol = "imap"
ProtoPOP3 Protocol = "pop3"
ProtoSieve Protocol = "sieve"
ProtoHTTP Protocol = "http"
)
// Ports maps listener ports to protocols.
type Ports map[uint16]Protocol
// ParsePorts returns Stalwart's standard ports plus overrides written as
// "PORT:PROTOCOL".
func ParsePorts(overrides []string) (Ports, error) {
p := Ports{
25: ProtoSMTP, 465: ProtoSMTP, 587: ProtoSMTP,
143: ProtoIMAP, 993: ProtoIMAP,
110: ProtoPOP3, 995: ProtoPOP3,
4190: ProtoSieve,
80: ProtoHTTP, 443: ProtoHTTP, 8080: ProtoHTTP,
}
for _, entry := range overrides {
portText, name, ok := strings.Cut(strings.TrimSpace(entry), ":")
n, err := strconv.ParseUint(portText, 10, 16)
if !ok || err != nil || n == 0 {
return nil, fmt.Errorf("port mapping %q: want PORT:PROTOCOL with a port from 1 to 65535", entry)
}
proto := Protocol(strings.ToLower(name))
switch proto {
case ProtoSMTP, ProtoIMAP, ProtoPOP3, ProtoSieve, ProtoHTTP:
default:
return nil, fmt.Errorf("port mapping %q: protocol must be smtp, imap, pop3, sieve or http", entry)
}
p[uint16(n)] = proto
}
return p, nil
}
// Of returns the protocol served on port, or ProtoUnknown.
func (p Ports) Of(port uint16) Protocol { return p[port] }
// Package stalwartsetup works out what a Stalwart server needs before CSM can
// rely on it, and applies it. Inspect only reads; Apply writes the changes
// Inspect planned and reloads Stalwart's settings once.
package stalwartsetup
import (
"cmp"
"context"
"errors"
"fmt"
"slices"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/ipmap"
"github.com/pidginhost/csm/internal/stalwartapi"
)
// API is the part of the Stalwart admin API setup uses.
type API interface {
TracingLevels(ctx context.Context) ([]stalwartapi.TracingLevel, error)
SetTracingLevel(ctx context.Context, id, event, level string) error
HTTPSettings(ctx context.Context) (stalwartapi.HTTPSettings, error)
SetHTTPSettings(ctx context.Context, h stalwartapi.HTTPSettings) error
BanPeriods(ctx context.Context) (stalwartapi.BanPeriods, error)
SetBanPeriods(ctx context.Context, d time.Duration) error
Listeners(ctx context.Context) ([]stalwartapi.Listener, error)
ProxyTrustedNetworks(ctx context.Context) ([]string, error)
SpamSettings(ctx context.Context) (stalwartapi.SpamSettings, error)
SpamFilterStageOff(ctx context.Context) (bool, error)
ReloadSettings(ctx context.Context) ([]string, error)
RoleByDescription(ctx context.Context, description string) (stalwartapi.Role, bool, error)
SaveRole(ctx context.Context, r stalwartapi.Role) (string, error)
Domains(ctx context.Context) ([]stalwartapi.Domain, error)
AccountsNamed(ctx context.Context, name string) ([]stalwartapi.Account, error)
AccountRoleIDs(ctx context.Context, id string) ([]string, error)
AccountCredentialTypes(ctx context.Context, id string) ([]string, error)
CreateAccount(ctx context.Context, name, domainID, roleID string) (string, error)
SetAccountRole(ctx context.Context, id, roleID string) error
APIKeys(ctx context.Context, accountID string) ([]stalwartapi.APIKey, error)
CreateAPIKey(ctx context.Context, accountID, description string, allowedIPs []string) (id, secret string, err error)
DeleteAPIKeys(ctx context.Context, accountID string, ids []string) error
SpamRule(ctx context.Context, name string) (stalwartapi.SpamRule, bool, error)
CreateSpamRule(ctx context.Context, name string, r stalwartapi.SpamRule) error
SetSpamRuleCondition(ctx context.Context, id string, arms []stalwartapi.RuleArm, elseExpr string) error
SpamTag(ctx context.Context, tag string) (stalwartapi.SpamTag, bool, error)
CreateSpamTag(ctx context.Context, tag string, score float64) error
}
// Inputs is what CSM knows about itself.
type Inputs struct {
// BlockExpiry is CSM's block length; Stalwart's bans are set to it.
BlockExpiry time.Duration
// LockManaged is true when the running daemon manages the firewall.
LockManaged bool
// LockedPorts are the ports the running firewall locks to listed users.
LockedPorts []int
// WebUsers are the web server's users, for the lock line setup suggests.
WebUsers []string
// LocalAPI is true when the API address is on this host, the only place
// CSM's key works from.
LocalAPI bool
// AccountDomain names the domain for CSM's service account when Stalwart
// has several.
AccountDomain string
// KeyAccount is the account the key in CSM's key file logs in as; empty
// when there is no working key.
KeyAccount string
// WriteKey stores a new key in CSM's key file.
WriteKey func(secret string) error
// Observe is true in observe mode, where the daemon does not write CSM's
// address list to Stalwart, so doctor does not test it.
Observe bool
}
// State is a step's outcome.
type State string
const (
Done State = "done"
Change State = "change"
Skipped State = "skipped"
Unsafe State = "unsafe"
Warn State = "warn"
)
// Step is one setup step.
type Step struct {
Name string
State State
Detail string
apply func(ctx context.Context, api API) error
// noReload marks a step that changes no Stalwart setting.
noReload bool
}
// Plan is the setup steps in order.
type Plan struct {
Steps []Step
// AddressListOn is true when Stalwart's rule for CSM's address list
// exists and is switched on.
AddressListOn bool
pendingReload bool
}
const (
failedLoginEvent = "auth.failed"
spamVerdictEvent = "spam.classify"
stalwartUser = "stalwart"
)
// CSM's role, service account and API key in Stalwart.
const (
ResponseRole = "CSM response"
ResponseAccount = "csm-response"
ResponseKey = "CSM"
)
// ResponsePermissions are what CSM's key needs beyond the User role: suspend
// accounts, lift bans, keep CSM's address list in Stalwart, and read every
// setting setup and doctor read.
var ResponsePermissions = []string{
"authenticate", "sysAccountGet", "sysAccountQuery", "sysAccountUpdate",
"sysBlockedIpGet", "sysBlockedIpQuery", "sysBlockedIpDestroy",
"sysActionCreate", "actionReloadBlockedIps", "actionClassifySpam",
"sysEventTracingLevelGet", "sysEventTracingLevelQuery", "sysHttpGet",
"sysSecurityGet", "sysNetworkListenerGet", "sysNetworkListenerQuery",
"sysSystemSettingsGet", "sysSpamSettingsGet", "sysMtaStageDataGet",
"sysRoleGet", "sysRoleQuery", "sysDomainGet", "sysDomainQuery",
"sysMemoryLookupKeyGet", "sysMemoryLookupKeyQuery", "sysMemoryLookupKeyCreate", "sysMemoryLookupKeyDestroy",
"actionReloadLookupStores", "sysSpamRuleGet", "sysSpamRuleQuery", "sysSpamTagGet", "sysSpamTagQuery",
}
// keyAddresses limit CSM's key to this host.
var keyAddresses = []string{"127.0.0.1", "::1"}
// ErrNotReloaded means Stalwart stored the settings Apply wrote but did not
// reload them. A later Inspect reads them as done, so the caller must tell
// the operator to restart Stalwart.
var ErrNotReloaded = errors.New("settings stored but not reloaded")
// ErrSettingsWrite marks a failed write of a Stalwart setting: part of it may
// be stored and not reloaded.
var ErrSettingsWrite = errors.New("stalwart setting not written")
// Inspect reads Stalwart's settings and plans each step. It writes nothing.
func Inspect(ctx context.Context, api API, in Inputs) (Plan, error) {
levels, err := api.TracingLevels(ctx)
if err != nil {
return Plan{}, err
}
httpSettings, err := api.HTTPSettings(ctx)
if err != nil {
return Plan{}, err
}
listeners, err := api.Listeners(ctx)
if err != nil {
return Plan{}, err
}
bans, err := api.BanPeriods(ctx)
if err != nil {
return Plan{}, err
}
proxy, err := api.ProxyTrustedNetworks(ctx)
if err != nil {
return Plan{}, err
}
spamStep, err := inspectSpamFilter(ctx, api)
if err != nil {
return Plan{}, err
}
listStep, listOn, err := inspectAddressList(ctx, api)
if err != nil {
return Plan{}, err
}
response, err := inspectResponse(ctx, api, in)
if err != nil {
return Plan{}, err
}
return Plan{Steps: append([]Step{
failedLoginStep(levels),
clientAddressStep(httpSettings, listeners, in),
banExpiryStep(bans, in.BlockExpiry),
proxyStep(listeners, proxy),
tracingStep("spam verdict logging", spamVerdictEvent, levels, "; rejected and discarded spam is not logged until then"),
spamStep,
listStep,
}, response...), AddressListOn: listOn}, nil
}
// inspectResponse plans CSM's role, its service account and the account's
// key. The apply of each later step reads the ids an earlier one creates.
func inspectResponse(ctx context.Context, api API, in Inputs) ([]Step, error) {
steps := []Step{{Name: "response role"}, {Name: "response account"}, {Name: "response key"}}
role, account, key := &steps[0], &steps[1], &steps[2]
skip := func(from []Step, why string) []Step {
for i := range from {
from[i].State, from[i].Detail = Skipped, why
}
return steps
}
if !in.LocalAPI {
return skip(steps, "Stalwart's API address is not on this host, and CSM's key works only from this host"), nil
}
user, found, err := api.RoleByDescription(ctx, "User")
if err != nil {
return nil, err
}
if !found {
role.State = Warn
role.Detail = `Stalwart has no role named "User", which CSM's role must include for its key to change user accounts; restore it in Stalwart's admin interface`
return skip(steps[1:], "needs CSM's role"), nil
}
current, haveRole, err := api.RoleByDescription(ctx, ResponseRole)
if err != nil {
return nil, err
}
roleID := current.ID
var missing []string
for _, p := range ResponsePermissions {
if !slices.Contains(current.Enabled, p) {
missing = append(missing, p)
}
}
if !slices.Contains(current.RoleIDs, user.ID) {
missing = append(missing, "the User role")
}
switch {
case !haveRole:
role.State, role.Detail = Change, fmt.Sprintf("create role %q with CSM's permissions and the User role", ResponseRole)
case len(current.Disabled) > 0:
role.State, role.Detail = Change, fmt.Sprintf("role %q denies permissions; reset it to CSM's permissions and the User role", ResponseRole)
case len(missing) > 0:
role.State, role.Detail = Change, fmt.Sprintf("role %q lacks %s; reset it to CSM's permissions and the User role", ResponseRole, strings.Join(missing, ", "))
default:
role.State, role.Detail = Done, fmt.Sprintf("role %q holds CSM's permissions and the User role", ResponseRole)
}
if role.State == Change {
want := stalwartapi.Role{ID: current.ID, Description: ResponseRole, Enabled: slices.Clone(ResponsePermissions), RoleIDs: []string{user.ID}}
role.noReload = true
role.apply = func(ctx context.Context, api API) (err error) {
roleID, err = api.SaveRole(ctx, want)
return err
}
}
accounts, err := api.AccountsNamed(ctx, ResponseAccount)
if err != nil {
return nil, err
}
accountID, name := "", ""
switch len(accounts) {
case 0:
domain, why, domainErr := responseDomain(ctx, api, in.AccountDomain)
if domainErr != nil {
return nil, domainErr
}
if why != "" {
account.State, account.Detail = Skipped, why
return skip(steps[2:], "needs CSM's service account"), nil
}
name = ResponseAccount + "@" + domain.Name
account.State = Change
account.Detail = fmt.Sprintf("create account %s with only CSM's role and no password; it can receive mail at that address", name)
account.noReload = true
account.apply = func(ctx context.Context, api API) (err error) {
accountID, err = api.CreateAccount(ctx, ResponseAccount, domain.ID, roleID)
return err
}
case 1:
a := accounts[0]
accountID, name = a.ID, cmp.Or(a.Email, a.Name)
roles, rolesErr := api.AccountRoleIDs(ctx, a.ID)
if rolesErr != nil {
return nil, rolesErr
}
creds, credsErr := api.AccountCredentialTypes(ctx, a.ID)
if credsErr != nil {
return nil, credsErr
}
// The key's loopback limit does not cover a password, and a key
// for an account that cannot sign in never works: setup adopts an
// existing account only when it holds nothing of anyone else's.
ours := haveRole && slices.Equal(roles, []string{current.ID})
why := ""
switch {
case a.SignInDisabled:
why = fmt.Sprintf("sign-ins are stopped for %s; allow them in Stalwart or remove the account, then run setup", name)
case slices.Contains(creds, "Password") || slices.Contains(creds, "AppPassword"):
why = fmt.Sprintf("%s has a password, and CSM's account must have none; remove the password or the account, then run setup", name)
case slices.ContainsFunc(creds, func(k string) bool { return k != "ApiKey" }):
// A kind CSM does not know, or none, may sign in from anywhere.
why = fmt.Sprintf("%s has a sign-in credential CSM does not know; remove it or the account, then run setup", name)
case !ours && len(creds) > 0:
why = fmt.Sprintf("an account named %s exists that CSM did not make; remove or rename it, then run setup", name)
}
if why != "" {
account.State, account.Detail = Warn, why
return skip(steps[2:], "needs CSM's own service account"), nil
}
if ours {
account.State, account.Detail = Done, name+" holds only CSM's role and no password"
break
}
account.State, account.Detail = Change, name+" must hold only CSM's role; set it"
account.noReload = true
account.apply = func(ctx context.Context, api API) error {
return api.SetAccountRole(ctx, a.ID, roleID)
}
default:
names := make([]string, len(accounts))
for i, a := range accounts {
names[i] = cmp.Or(a.Email, a.Name)
}
account.State = Warn
account.Detail = fmt.Sprintf("%d accounts are named %s (%s); CSM needs exactly one, so delete the others in Stalwart's admin interface",
len(accounts), ResponseAccount, strings.Join(names, ", "))
return skip(steps[2:], "needs exactly one service account"), nil
}
// Stalwart does not list secrets, so with several keys CSM cannot tell
// which one its file holds; replacing them all removes the others.
var keys []stalwartapi.APIKey
if accountID != "" && in.KeyAccount == accountID {
if keys, err = api.APIKeys(ctx, accountID); err != nil {
return nil, err
}
if len(keys) == 1 {
key.State, key.Detail = Done, "CSM's key file holds the only key of "+name
return steps, nil
}
}
key.State = Change
switch {
case in.KeyAccount == "":
key.Detail = "CSM's key file holds no working key"
case in.KeyAccount != accountID:
key.Detail = "the key in CSM's key file belongs to another account"
case len(keys) == 0:
key.Detail = "Stalwart lists no key of " + name
default:
key.Detail = name + " has extra API keys besides the one in CSM's key file"
}
key.Detail += fmt.Sprintf("; create a key of %s that works only from this host, store it in the key file and delete the account's other keys", name)
key.noReload = true
writeKey := in.WriteKey
key.apply = func(ctx context.Context, api API) error {
return replaceKey(ctx, api, accountID, writeKey)
}
return steps, nil
}
// responseDomain picks the domain for CSM's service account. why says what
// is missing when there is no single choice.
func responseDomain(ctx context.Context, api API, want string) (d stalwartapi.Domain, why string, err error) {
domains, err := api.Domains(ctx)
if err != nil {
return d, "", err
}
if want != "" {
// Aliases do not count: the account's address is in the domain itself.
for _, d := range domains {
if asciiEqualFold(d.Name, want) {
return d, "", nil
}
}
return d, fmt.Sprintf("Stalwart has no domain named %s; name one of its domains with --account-domain", want), nil
}
switch len(domains) {
case 0:
return d, "Stalwart has no domain for CSM's service account; add one first", nil
case 1:
return domains[0], "", nil
}
return d, fmt.Sprintf("Stalwart has %d domains; choose the one for CSM's service account with --account-domain NAME", len(domains)), nil
}
// asciiEqualFold ignores ASCII case only: a Unicode fold maps other letters,
// such as the Kelvin sign, onto ASCII ones.
func asciiEqualFold(a, b string) bool {
if len(a) != len(b) {
return false
}
lower := func(c byte) byte {
if 'A' <= c && c <= 'Z' {
return c + 'a' - 'A'
}
return c
}
for i := 0; i < len(a); i++ {
if lower(a[i]) != lower(b[i]) {
return false
}
}
return true
}
// replaceKey creates a key of the account, stores it, then deletes the
// account's other keys. A key that could not be stored is deleted again, so
// the old key keeps working. The secret goes to write and nowhere else.
func replaceKey(ctx context.Context, api API, accountID string, write func(string) error) (result error) {
id, secret, err := api.CreateAPIKey(ctx, accountID, ResponseKey, keyAddresses)
if err != nil {
return err
}
defer func() {
if result != nil {
if message := stalwartapi.Redact(result.Error(), "", secret); message != result.Error() {
result = errors.New(message)
}
}
}()
if err = write(secret); err != nil {
if delErr := api.DeleteAPIKeys(ctx, accountID, []string{id}); delErr != nil {
return errors.Join(err, fmt.Errorf("deleting the key that was not stored: %w", delErr))
}
return err
}
keys, err := api.APIKeys(ctx, accountID)
if err == nil {
var old []string
for _, k := range keys {
if k.ID != id {
old = append(old, k.ID)
}
}
if len(old) > 0 {
err = api.DeleteAPIKeys(ctx, accountID, old)
}
}
if err != nil {
return fmt.Errorf("the new key is stored and works, but the account's older keys remain; delete them in Stalwart's admin interface: %w", err)
}
return nil
}
func inspectSpamFilter(ctx context.Context, api API) (Step, error) {
spam, settingsErr := api.SpamSettings(ctx)
if errors.Is(settingsErr, stalwartapi.ErrUnauthorized) {
return Step{}, settingsErr
}
stageOff, stageErr := api.SpamFilterStageOff(ctx)
if errors.Is(stageErr, stalwartapi.ErrUnauthorized) {
return Step{}, stageErr
}
// These reads only inform the operator. Their failure must not block
// unrelated setup changes or put server error text in the plan.
s := Step{Name: "spam filter", State: Warn, Detail: "Cannot read Stalwart's spam filter status; check it in Stalwart's admin interface"}
if settingsErr == nil && stageErr == nil {
s = spamFilterStep(spam, stageOff)
}
return s, nil
}
// AddressListRule is the Stalwart spam rule that scores senders on CSM's
// address list.
const AddressListRule = "CSM_IP"
// AddressListArms is the rule's condition: one arm per symbol, strongest
// first, so the first match is the strongest listing of the sender.
func AddressListArms() []stalwartapi.RuleArm {
var arms []stalwartapi.RuleArm
for _, s := range ipmap.Symbols() {
arms = append(arms, stalwartapi.RuleArm{
If: fmt.Sprintf("key_exists('%s', remote_ip) || (is_ipv6_addr(remote_ip) && key_exists('%s', ip_reverse_name(remote_ip)))",
ipmap.StalwartNamespace(s, false), ipmap.StalwartNamespace(s, true)),
Then: "'" + s.Name + "'",
})
}
return arms
}
// inspectAddressList plans the rule that has Stalwart score senders on CSM's
// address list, and its tags. on is true when the rule exists and is switched
// on. A refused login stops setup; any other failed read is a warning, so a
// key from before the address list does not block the other steps.
func inspectAddressList(ctx context.Context, api API) (Step, bool, error) {
s, on, err := addressListStep(ctx, api)
if errors.Is(err, stalwartapi.ErrUnauthorized) {
return Step{}, false, err
}
if err != nil {
s = Step{Name: "address list", State: Warn, Detail: "Cannot read Stalwart's spam rules; run csm stalwart setup with an admin login"}
}
return s, on, nil
}
// addressListStep reads the rule and its tags. The rule's own switch is the
// operator's: setup creates the rule on but never switches it back on, and
// never changes a tag's score.
func addressListStep(ctx context.Context, api API) (s Step, on bool, err error) {
s = Step{Name: "address list", State: Warn}
rule, found, err := api.SpamRule(ctx, AddressListRule)
if err != nil {
return s, false, err
}
switch {
case found && rule.Type != "Any":
// The type is server text, so the detail does not quote it.
s.Detail = "Stalwart has a rule named " + AddressListRule + " that CSM did not make; remove or rename it, then run setup"
return s, false, nil
case found && !rule.Enabled:
s.State, s.Detail = Done, "rule "+AddressListRule+" is switched off in Stalwart, so CSM's address list adds no score"
return s, false, nil
}
on = found
var scores, nonScoring, missing []string
var create []ipmap.Symbol
for _, sym := range ipmap.Symbols() {
tag, have, err := api.SpamTag(ctx, sym.Name)
if err != nil {
return s, on, err
}
if !have {
missing = append(missing, sym.Name)
create = append(create, sym)
continue
}
if tag.Type == "Score" {
scores = append(scores, sym.Name+" "+strconv.FormatFloat(tag.Score, 'f', -1, 64))
} else {
nonScoring = append(nonScoring, sym.Name)
}
}
arms := AddressListArms()
reset := found && (!slices.Equal(rule.Arms, arms) || rule.Else != "false")
switch {
case !found:
var strongest int
for _, sym := range ipmap.Symbols() {
strongest = max(strongest, sym.Tenths())
}
s.Detail = fmt.Sprintf("create rule %s: Stalwart adds up to %s to the spam score of senders on CSM's address list, so more of their mail goes to Junk, or is rejected or discarded at the scores set in Stalwart; switch the rule off in Stalwart to stop",
AddressListRule, strconv.FormatFloat(float64(strongest)/10, 'f', -1, 64))
case len(missing) > 0 || reset:
var parts []string
if len(missing) > 0 {
noun := "tag"
if len(missing) > 1 {
noun = "tags"
}
parts = append(parts, "create the missing "+noun+" "+strings.Join(missing, ", "))
}
if reset {
parts = append(parts, "reset the condition of rule "+AddressListRule+" to match CSM's address list")
}
s.Detail = strings.Join(parts, "; ")
default:
s.State, s.Detail = Done, "rule "+AddressListRule+" adds "+strings.Join(scores, ", ")+" to senders on CSM's address list"
if len(scores) == 0 {
s.Detail = "rule " + AddressListRule + " has no scoring tags"
}
if len(nonScoring) > 0 {
s.Detail += "; non-scoring tags: " + strings.Join(nonScoring, ", ")
}
return s, on, nil
}
s.State = Change
s.apply = func(ctx context.Context, api API) error {
// Created tags leave the list, so a retried Apply does not repeat them.
for len(create) > 0 {
if err := api.CreateSpamTag(ctx, create[0].Name, float64(create[0].Tenths())/10); err != nil {
return err
}
create = create[1:]
}
switch {
case !found:
return api.CreateSpamRule(ctx, AddressListRule, stalwartapi.SpamRule{Arms: arms, Else: "false"})
case reset:
return api.SetSpamRuleCondition(ctx, rule.ID, arms, "false")
}
return nil
}
return s, on, nil
}
// Apply makes each planned change in order, then reloads Stalwart's settings
// once. It makes no call when nothing changes. A failed write stops it, and
// the settings written before it are still reloaded: a later run reads them
// as done, so they must not be left stored but inactive.
func (p *Plan) Apply(ctx context.Context, api API) ([]string, error) {
var failed error
for i := range p.Steps {
s := &p.Steps[i]
if s.State != Change || s.apply == nil {
continue
}
if err := s.apply(ctx, api); err != nil {
failed = fmt.Errorf("%s: %w", s.Name, err)
if !s.noReload {
failed = settingsWriteError{failed}
}
break
}
if !s.noReload {
p.pendingReload = true
}
s.apply = nil
}
if !p.pendingReload {
return nil, failed
}
warnings, err := api.ReloadSettings(ctx)
if err != nil {
return nil, errors.Join(failed, fmt.Errorf("%w: %w", ErrNotReloaded, err))
}
p.pendingReload = false
return warnings, failed
}
// settingsWriteError adds ErrSettingsWrite to a failed settings step's error
// without changing its text.
type settingsWriteError struct{ error }
func (e settingsWriteError) Unwrap() []error { return []error{e.error, ErrSettingsWrite} }
func failedLoginStep(levels []stalwartapi.TracingLevel) Step {
return tracingStep("failed-login logging", failedLoginEvent, levels, "")
}
// tracingStep raises event to info unless its override already logs it; why
// ends the detail of a step that still has to change.
func tracingStep(name, event string, levels []stalwartapi.TracingLevel, why string) Step {
s := Step{Name: name}
id, level := "", ""
for _, l := range levels {
if l.Event == event {
id, level = l.ID, l.Level
break
}
}
switch level {
case "info", "warn", "error":
s.State, s.Detail = Done, fmt.Sprintf("%s is logged at %s", event, level)
return s
case "":
s.Detail = fmt.Sprintf("%s has no log level of its own; set it to info%s", event, why)
default:
s.Detail = fmt.Sprintf("%s is logged at %s; set it to info%s", event, level, why)
}
s.State = Change
s.apply = func(ctx context.Context, api API) error {
return api.SetTracingLevel(ctx, id, event, "info")
}
return s
}
// spamFilterStep only reports the filter's switch and action scores; the address list is a separate step.
func spamFilterStep(sp stalwartapi.SpamSettings, stageOff bool) Step {
s := Step{Name: "spam filter"}
if !sp.Enabled || stageOff {
s.State = Warn
s.Detail = "Stalwart's spam filter is off, so inbound mail is not filtered; switch it on in Stalwart's admin interface"
return s
}
// A score of 0 switches that action off.
act := func(verb string, score float64) string {
if score > 0 {
return verb + " at " + strconv.FormatFloat(score, 'f', -1, 64)
}
return "never " + verb
}
s.State = Done
s.Detail = "on; " + act("files spam to Junk", sp.ScoreSpam) + ", " + act("rejects", sp.ScoreReject) + ", " + act("discards", sp.ScoreDiscard)
return s
}
// clientAddressStep decides whether Stalwart's HTTP listeners may take the
// client address from X-Forwarded-For. That is only safe when every HTTP
// listener is reachable from this host alone and CSM's running firewall lets
// only the web server and Stalwart connect to it: otherwise anyone who can
// connect can claim any address.
func clientAddressStep(h stalwartapi.HTTPSettings, listeners []stalwartapi.Listener, in Inputs) Step {
s := Step{Name: "webmail client addresses"}
forwarded := h.ClientIPSource != "" && h.ClientIPSource != "peer"
var ports []int
var public []string
for _, l := range listeners {
if l.Protocol != "http" {
continue
}
if len(l.Bind) == 0 {
public = append(public, l.Name+" has no known bind address")
}
for _, b := range l.Bind {
if !b.Addr().Unmap().IsLoopback() {
public = append(public, fmt.Sprintf("%s binds %s", l.Name, b))
}
if p := int(b.Port()); !slices.Contains(ports, p) {
ports = append(ports, p)
}
}
}
if len(ports) == 0 && len(public) == 0 {
s.State, s.Detail = Done, "Stalwart has no HTTP listener"
return s
}
slices.Sort(ports)
reason := ""
switch {
case len(public) > 0:
reason = fmt.Sprintf("listener %s, reachable from other hosts; bind every HTTP listener to 127.0.0.1 behind the web server first",
strings.Join(public, ", listener "))
case !in.LockManaged || !allLocked(ports, in.LockedPorts):
state := "CSM's running firewall does not lock the webmail port"
if !in.LockManaged {
state = "CSM's daemon is not managing the firewall"
}
reason = fmt.Sprintf("%s; add %s under firewall in csm.yaml and restart CSM", state, lockLine(ports, in.WebUsers))
}
switch {
case reason != "" && forwarded:
s.State = Unsafe
s.Detail = fmt.Sprintf("Stalwart takes client addresses from %s, which anyone who can connect can forge: %s; until then set it back to peer in Stalwart",
h.ClientIPSource, reason)
case reason != "":
s.State, s.Detail = Skipped, reason
case h == safeForwarded:
s.State, s.Detail = Done, "client addresses come from X-Forwarded-For (1 hop) behind CSM's port lock"
default:
s.State = Change
s.Detail = "client addresses are the web server's; take them from X-Forwarded-For (1 hop). " + webServerHeader
if forwarded {
// More hops, or a header the web server does not set, lets the
// client choose the address Stalwart reads.
s.Detail = fmt.Sprintf("client addresses come from %s with %d hops, which a client can forge; take them from X-Forwarded-For (1 hop). %s",
h.ClientIPSource, h.ForwardedHops, webServerHeader)
}
s.apply = func(ctx context.Context, api API) error {
return api.SetHTTPSettings(ctx, safeForwarded)
}
}
return s
}
// webServerHeader is the condition setup cannot check: with one hop Stalwart
// reads the last X-Forwarded-For entry, which must be the web server's own.
const webServerHeader = "The web server must set the header itself, as nginx does with proxy_set_header X-Forwarded-For $remote_addr or $proxy_add_x_forwarded_for; without it a client chooses the address"
// safeForwarded takes the address the web server, the only hop, appends.
var safeForwarded = stalwartapi.HTTPSettings{ClientIPSource: "xForwardedFor", ForwardedHops: 1}
func allLocked(ports, locked []int) bool {
for _, p := range ports {
if !slices.Contains(locked, p) {
return false
}
}
return true
}
// lockLine is the csm.yaml line that locks ports to the web users and
// Stalwart.
func lockLine(ports []int, webUsers []string) string {
users := slices.Clone(webUsers)
if !slices.Contains(users, stalwartUser) {
users = append(users, stalwartUser)
}
entries := make([]string, len(ports))
for i, p := range ports {
entries[i] = strconv.Quote(strconv.Itoa(p) + ":" + strings.Join(users, ","))
}
return "local_port_locks: [" + strings.Join(entries, ", ") + "]"
}
func banExpiryStep(b stalwartapi.BanPeriods, want time.Duration) Step {
s := Step{Name: "ban expiry"}
if want <= 0 {
s.State, s.Detail = Skipped, "CSM has no block length"
return s
}
same := func(d *time.Duration) bool { return d != nil && *d == want }
if same(b.Auth) && same(b.Abuse) && same(b.Scan) {
s.State, s.Detail = Done, fmt.Sprintf("failed-login, abuse and scan bans last %s, as CSM's blocks do", fmtPeriod(&want))
return s
}
s.State = Change
s.Detail = fmt.Sprintf("failed-login bans last %s, abuse %s, scan %s; set them to CSM's block length, %s",
fmtPeriod(b.Auth), fmtPeriod(b.Abuse), fmtPeriod(b.Scan), fmtPeriod(&want))
s.apply = func(ctx context.Context, api API) error { return api.SetBanPeriods(ctx, want) }
return s
}
func fmtPeriod(d *time.Duration) string {
switch {
case d == nil:
return "permanent"
case *d%time.Hour == 0:
return strconv.FormatInt(int64(*d/time.Hour), 10) + "h"
case *d%time.Minute == 0:
return strconv.FormatInt(int64(*d/time.Minute), 10) + "m"
}
return d.String()
}
// proxyStep reports mail listeners that accept the PROXY protocol: their log
// addresses come from the proxy's header, which CSM does not handle yet.
func proxyStep(listeners []stalwartapi.Listener, global []string) Step {
s := Step{Name: "proxy protocol listeners"}
var names []string
for _, l := range listeners {
if l.Protocol == "http" {
continue
}
if len(l.ProxyNetworks) > 0 || len(global) > 0 {
names = append(names, l.Name)
}
}
if len(names) == 0 {
s.State, s.Detail = Done, "no mail listener accepts the PROXY protocol"
return s
}
s.State = Warn
s.Detail = fmt.Sprintf("%s accept the PROXY protocol; CSM still blocks the addresses they log, so make sure only your proxies are trusted",
strings.Join(names, ", "))
return s
}
package state
import (
"path/filepath"
"github.com/pidginhost/csm/internal/alert"
)
// BeginDelivery reserves eligible findings until finish is called with the
// successfully delivered subset. No store lock stays held during delivery.
// Receipts belong to the selected entry, so a scan clearing it while a send
// is in flight cannot be undone by that send's late completion. A finding
// with no entry, such as an action or a correlation-derived finding, gets
// one here.
//
// ponytail: action and correlation-derived findings exist only while
// produced. When the history reminder re-raises their source after more than
// 24h, they may be emailed once more. The upgrade path would be clearing
// derived rows through the latest merge.
func (s *Store) BeginDelivery(findings []alert.Finding) ([]alert.Finding, func([]alert.Finding)) {
s.mu.Lock()
defer s.mu.Unlock()
now := s.clock()
if s.emailPending == nil {
s.emailPending = make(map[string]*Entry)
}
type receipt struct {
entry *Entry
hash string
}
selected := make(map[string]receipt)
var eligible []alert.Finding
for _, f := range s.filterUnemailedLocked(findings, now) {
key := f.Key()
entry := s.entries[key]
if entry != nil && s.emailPending[key] == entry {
continue
}
if entry == nil {
entry = &Entry{Hash: f.Fingerprint(), FirstSeen: now, AlertSent: now}
s.entries[key] = entry
}
// Removal may clear state while delivery is in flight, before Update
// attaches the path or for findings that never reach Update.
if f.FilePath != "" {
entry.Path = filepath.Clean(f.FilePath)
}
entry.LastSeen = now
s.dirty = true
s.emailPending[key] = entry
selected[key] = receipt{entry: entry, hash: f.Fingerprint()}
eligible = append(eligible, f)
}
return eligible, func(delivered []alert.Finding) {
s.mu.Lock()
defer s.mu.Unlock()
for _, f := range delivered {
key := f.Key()
r, ok := selected[key]
if ok && s.entries[key] == r.entry && f.Fingerprint() == r.hash {
r.entry.Emailed = r.hash
s.dirty = true
}
}
for key, r := range selected {
if s.emailPending[key] == r.entry {
delete(s.emailPending, key)
}
}
}
}
package state
import (
"errors"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/store"
)
// AppendHistoryDurable requires the bbolt history store and returns write
// failures so a caller can acknowledge records only after they are saved.
func (s *Store) AppendHistoryDurable(findings []alert.Finding) error {
if len(findings) == 0 {
return nil
}
db := store.Global()
if db == nil {
return errors.New("the history database is not open")
}
if err := db.AppendHistory(findings); err != nil {
return err
}
recordFindings(findings)
return nil
}
package state
import (
"fmt"
"os"
"path/filepath"
"syscall"
)
const lockFileName = "csm.lock"
// LockFile provides file-based locking to prevent concurrent CSM runs.
type LockFile struct {
path string
file *os.File
}
// AcquireLock creates an exclusive lock. Returns error if already locked.
func AcquireLock(stateDir string) (*LockFile, error) {
lockPath := filepath.Join(stateDir, lockFileName)
// #nosec G304 -- filepath.Join under operator-configured stateDir.
f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, fmt.Errorf("opening lock file: %w", err)
}
// Try non-blocking exclusive lock
// #nosec G115 -- os.File.Fd returns uintptr but POSIX file descriptors
// are small non-negative ints (rlimit ~1024); int conversion is lossless.
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
_ = f.Close()
return nil, fmt.Errorf("another CSM instance is already running")
}
// Write PID for debugging
_ = f.Truncate(0)
fmt.Fprintf(f, "%d\n", os.Getpid())
return &LockFile{path: lockPath, file: f}, nil
}
// Release releases the lock.
func (l *LockFile) Release() {
if l.file != nil {
// #nosec G115 -- see AcquireLock: POSIX fd fits in int.
_ = syscall.Flock(int(l.file.Fd()), syscall.LOCK_UN)
_ = l.file.Close()
// A lock file left behind is inert: the lock is the flock on the
// open descriptor, which Close released above.
_ = os.Remove(l.path)
}
}
package state
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
)
// MigrateStateDir copies the contents of oldPath into newPath if and only if:
// (1) oldPath exists and is non-empty,
// (2) newPath does not exist or is empty,
// (3) the two paths are not the same.
// Returns (true, nil) when a copy occurred, (false, nil) when no migration
// was needed. Any I/O error is fatal — the caller (daemon startup) should
// abort rather than silently start with mixed state.
func MigrateStateDir(oldPath, newPath string) (bool, error) {
if oldPath == "" || newPath == "" || oldPath == newPath {
return false, nil
}
oldInfo, err := os.Stat(oldPath)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return false, nil
}
return false, fmt.Errorf("stat %s: %w", oldPath, err)
}
if !oldInfo.IsDir() {
return false, nil
}
oldEntries, err := os.ReadDir(oldPath)
if err != nil {
return false, fmt.Errorf("reading %s: %w", oldPath, err)
}
if len(oldEntries) == 0 {
return false, nil
}
if newEntries, err := os.ReadDir(newPath); err == nil && hasStateContent(newEntries) {
return false, nil // new dir non-empty: no migration
} else if err != nil && !errors.Is(err, fs.ErrNotExist) {
return false, fmt.Errorf("reading %s: %w", newPath, err)
}
if err := os.MkdirAll(newPath, 0o700); err != nil {
return false, fmt.Errorf("mkdir %s: %w", newPath, err)
}
copied := false
for _, e := range oldEntries {
if e.Name() == lockFileName {
continue
}
src := filepath.Join(oldPath, e.Name())
dst := filepath.Join(newPath, e.Name())
if err := copyEntry(src, dst); err != nil {
return false, fmt.Errorf("copying %s: %w", src, err)
}
copied = true
}
return copied, nil
}
func hasStateContent(entries []os.DirEntry) bool {
for _, entry := range entries {
if entry.Name() != lockFileName {
return true
}
}
return false
}
func copyEntry(src, dst string) error {
info, err := os.Stat(src)
if err != nil {
return err
}
if info.IsDir() {
if mkdirErr := os.MkdirAll(dst, info.Mode().Perm()); mkdirErr != nil {
return mkdirErr
}
entries, readDirErr := os.ReadDir(src)
if readDirErr != nil {
return readDirErr
}
for _, e := range entries {
if copyErr := copyEntry(filepath.Join(src, e.Name()), filepath.Join(dst, e.Name())); copyErr != nil {
return copyErr
}
}
return nil
}
// #nosec G304 -- src derived from os.ReadDir of a trusted state directory.
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
// #nosec G304 -- dst is new state directory plus an entry name from os.ReadDir.
out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode().Perm())
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
_ = out.Close()
return err
}
if err := out.Sync(); err != nil {
_ = out.Close()
return err
}
return out.Close()
}
package state
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"time"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/atomicio"
)
const pendingFindingsFile = "pending_findings.json"
var removePendingFindingsFile = os.Remove
// pendingFinding is the parked form of a finding. The public Finding JSON
// leaves out the spray targets, claims and observation, which the incident
// correlator and admission read; a replayed finding needs them to act the way
// the original would have. The subnets are public, and the storage-only copy
// stays so an older daemon replays them too. Delivery provenance stays
// process-local.
type pendingFinding struct {
alert.Finding
ResponseCIDRs []string `json:"response_cidrs,omitempty"`
ResponseSprayTargets []string `json:"response_spray_targets,omitempty"`
ResponseClaims []admission.Claim `json:"response_claims,omitempty"`
ResponseObservation *alert.Observation `json:"response_observation,omitempty"`
// ResponseAttribution keeps how the parser read the address.
ResponseAttribution admission.Attribution `json:"response_attribution,omitempty"`
// ResponseSprayConstituents keeps the addresses a spray counted.
ResponseSprayConstituents []alert.SprayConstituent `json:"response_spray_constituents,omitempty"`
// ResponseIntel keeps the intel a reputation finding rests on.
ResponseIntel *admission.IntelRef `json:"response_intel,omitempty"`
// ResponseCause distinguishes database-session responses from retired scores.
ResponseCause *alert.Cause `json:"response_cause,omitempty"`
// ResponseMailLoginFailure keeps mail reputation hits in mail incidents.
ResponseMailLoginFailure bool `json:"response_mail_login_failure,omitempty"`
// ResponseBlockTTL keeps a spray block's own length.
ResponseBlockTTL time.Duration `json:"response_block_ttl,omitempty"`
// ResponseLoginAccount keeps an FTP login's authenticated name for
// customer history.
ResponseLoginAccount string `json:"response_login_account,omitempty"`
// A restart must not turn an unexamined snapshot into fresh content evidence.
ScanCarryForward bool `json:"scan_carry_forward,omitempty"`
// Delayed file response must retain its identity guard across a restart.
ResponseFileIdentity string `json:"response_file_identity,omitempty"`
}
func toPendingRecords(findings []alert.Finding) []pendingFinding {
records := make([]pendingFinding, len(findings))
for i, f := range findings {
records[i] = pendingFinding{Finding: f, ResponseCIDRs: f.CIDRs, ResponseSprayTargets: f.SprayTargets, ResponseClaims: f.Claims,
ResponseSprayConstituents: f.SprayConstituents, ResponseCause: f.Cause, ResponseMailLoginFailure: f.MailLoginFailure,
ResponseBlockTTL: f.BlockTTL, ResponseLoginAccount: f.LoginAccount, ResponseAttribution: f.Attribution,
ScanCarryForward: f.ScanCarryForward, ResponseFileIdentity: f.FileResponseIdentity}
// An unencodable expiry costs only its intel, not the whole batch.
if f.Intel != nil {
if _, err := f.Intel.Expires.MarshalJSON(); err == nil {
records[i].ResponseIntel = f.Intel
}
}
// A log can carry a time the JSON encoder refuses. Dropping that
// observation costs one finding its provenance, not the whole batch.
if _, err := f.Observation.ObservedAt.MarshalJSON(); err == nil && f.Observation != (alert.Observation{}) {
observation := f.Observation
records[i].ResponseObservation = &observation
}
}
return records
}
func fromPendingRecords(records []pendingFinding) []alert.Finding {
if records == nil {
return nil
}
findings := make([]alert.Finding, len(records))
for i, r := range records {
findings[i] = r.Finding
if r.ResponseCIDRs != nil {
findings[i].CIDRs = r.ResponseCIDRs
}
findings[i].SprayTargets = r.ResponseSprayTargets
findings[i].Claims = r.ResponseClaims
findings[i].SprayConstituents = r.ResponseSprayConstituents
findings[i].Intel = r.ResponseIntel
findings[i].Cause = r.ResponseCause
findings[i].MailLoginFailure = r.ResponseMailLoginFailure
findings[i].BlockTTL = r.ResponseBlockTTL
findings[i].LoginAccount = r.ResponseLoginAccount
findings[i].Attribution = r.ResponseAttribution
findings[i].ScanCarryForward = r.ScanCarryForward
findings[i].FileResponseIdentity = r.ResponseFileIdentity
if r.ResponseObservation != nil {
findings[i].Observation = *r.ResponseObservation
}
}
return findings
}
// pendingFindingsMax bounds the parked batch; a stop during a flood keeps the
// newest findings rather than growing the file without limit.
const pendingFindingsMax = 10000
// AppendPendingFindings parks findings that were still queued for dispatch
// when the daemon stopped. The next start takes them back and runs them
// through the full dispatch pipeline, so a realtime-only finding raised in the
// last seconds before a restart still triggers its auto-response instead of
// surviving only as a history line nothing re-detects. Shutdown drains the
// channel twice, so this appends rather than replaces.
func (s *Store) AppendPendingFindings(findings []alert.Finding) error {
if len(findings) == 0 {
return nil
}
call := s.pendingHealth().begin(len(findings))
defer call.finish()
s.mu.Lock()
defer s.mu.Unlock()
defer call.settleIO()
call.start()
pending, err := s.readPendingLocked(call)
if err != nil {
return err
}
old := pendingIdentity(pending)
call.observe(old)
total := len(pending) + len(findings)
pending = append(pending, findings...)
if len(pending) > pendingFindingsMax {
pending = pending[len(pending)-pendingFindingsMax:]
}
next := pendingIdentity(pending)
ages := call.appendedAges(old.count, len(pending))
write := s.writePendingFile
if write == nil {
write = atomicio.AtomicWriteJSON
}
knownLoss := max(0, len(findings)-pendingFindingsMax)
if !next.valid {
knownLoss = len(findings)
}
call.offer(knownLoss)
err = write(filepath.Join(s.path, pendingFindingsFile), 0o600, toPendingRecords(pending))
if err == nil {
call.complete(next, total-len(pending), false, ages)
return nil
}
call.ioFailed()
actual, readErr := s.readPendingLocked(call)
if readErr != nil {
return err
}
image := pendingIdentity(actual)
switch {
case image.valid && image == old:
call.complete(image, len(findings), true, nil)
case image.valid && next.valid && image == next:
call.complete(image, total-len(pending), true, ages)
default:
call.unreadable(call.lost)
call.observe(image)
}
return err
}
// TakePendingFindings returns the parked findings and clears them, so a
// replay that itself gets interrupted cannot double-dispatch on the next start.
func (s *Store) TakePendingFindings() ([]alert.Finding, error) {
call := s.pendingHealth().begin(0)
defer call.finish()
findings, err := s.takePendingFindings(call)
if err == nil {
call.finishReplay()
}
return findings, err
}
// ReplayPendingFindings keeps the cleared batch owned through dispatch. The
// callback runs without the state lock; later shutdown appends stay independent.
func (s *Store) ReplayPendingFindings(consume func([]alert.Finding)) error {
call := s.pendingHealth().begin(0)
defer call.finish()
pending, err := s.takePendingFindings(call)
if err != nil {
return err
}
if len(pending) > 0 {
consume(pending)
}
call.finishReplay()
return nil
}
func (s *Store) takePendingFindings(call *pendingCall) ([]alert.Finding, error) {
s.mu.Lock()
defer s.mu.Unlock()
defer call.settleIO()
call.start()
pending, err := s.readPendingLocked(call)
if err != nil {
return nil, err
}
old := pendingIdentity(pending)
call.observe(old)
call.offer(0)
if err := removePendingFindingsFile(filepath.Join(s.path, pendingFindingsFile)); err != nil {
if os.IsNotExist(err) && len(pending) == 0 {
call.complete(pendingImage{valid: true}, 0, false, nil)
return nil, nil
}
call.ioFailed()
actual, readErr := s.readPendingLocked(call)
if readErr == nil {
image := pendingIdentity(actual)
switch {
case image.valid && image == old:
call.complete(image, 0, true, nil)
case len(actual) == 0:
call.complete(image, len(pending), true, nil)
default:
call.unreadable(0)
call.observe(image)
}
}
return nil, fmt.Errorf("clear pending findings: %w", err)
}
call.detach(len(pending))
return pending, nil
}
func (s *Store) readPendingLocked(call *pendingCall) ([]alert.Finding, error) {
read := s.readPendingFile
if read == nil {
read = os.ReadFile
}
data, err := read(filepath.Join(s.path, pendingFindingsFile))
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
call.failedRead()
return nil, fmt.Errorf("read pending findings: %w", err)
}
var records []pendingFinding
if err := json.Unmarshal(data, &records); err != nil {
call.failedRead()
return nil, fmt.Errorf("decode pending findings: %w", err)
}
return fromPendingRecords(records), nil
}
package state
import (
"crypto/sha256"
"encoding/json"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/queuehealth"
)
type pendingImage struct {
count int
digest [sha256.Size]byte
valid bool
}
// Compare complete serialized batches, including duplicate occurrences. A
// finding's dedup key does not identify its persisted payload.
func pendingIdentity(findings []alert.Finding) pendingImage {
if len(findings) == 0 {
return pendingImage{valid: true}
}
data, err := json.Marshal(toPendingRecords(findings))
if err != nil {
return pendingImage{count: len(findings)}
}
// JSON repairs invalid UTF-8, including log text cut inside a character.
// Hash the recovered payload so a successful readback has the same identity.
var persisted []pendingFinding
if decodeErr := json.Unmarshal(data, &persisted); decodeErr != nil {
return pendingImage{count: len(findings)}
}
data, err = json.Marshal(persisted)
return pendingImage{count: len(findings), digest: sha256.Sum256(data), valid: err == nil}
}
type pendingQueue struct {
mu sync.Mutex
disk pendingImage
known bool
arrivals []time.Time
calls map[*pendingCall]struct{}
losses *queuehealth.Tracker
stateIO bool
lowerBound bool
uncertainAt time.Time
}
type pendingCall struct {
queue *pendingQueue
queued, progress time.Time
incoming, replay, lost int
offered, settled bool
}
func (s *Store) pendingHealth() *pendingQueue {
s.pendingHealthOnce.Do(func() {
s.pendingQueue = &pendingQueue{calls: make(map[*pendingCall]struct{}), losses: queuehealth.New(pendingFindingsMax, time.Minute)}
})
return s.pendingQueue
}
func (s *Store) observePendingQueue() {
pending, err := s.readPendingLocked(nil)
image := pendingIdentity(pending)
q := s.pendingHealth()
q.mu.Lock()
defer q.mu.Unlock()
if err != nil {
q.unknown(true)
return
}
q.setDisk(image)
}
func (q *pendingQueue) begin(count int) *pendingCall {
c := &pendingCall{queue: q, queued: time.Now(), incoming: count}
q.mu.Lock()
q.calls[c] = struct{}{}
q.mu.Unlock()
return c
}
func (c *pendingCall) start() {
c.queue.mu.Lock()
c.progress = time.Now()
c.queue.mu.Unlock()
}
func (q *pendingQueue) setDisk(next pendingImage) {
if !q.known || !next.valid || q.disk != next {
q.arrivals = make([]time.Time, next.count)
now := time.Now()
for i := range q.arrivals {
q.arrivals[i] = now
}
}
q.disk = next
q.known = true
}
func (c *pendingCall) appendedAges(oldCount, kept int) []time.Time {
q := c.queue
q.mu.Lock()
defer q.mu.Unlock()
ages := make([]time.Time, kept)
start := oldCount + c.incoming - kept
for i := range ages {
if index := start + i; index < oldCount {
ages[i] = q.arrivals[index]
} else {
ages[i] = c.queued
}
}
return ages
}
func (q *pendingQueue) unknown(ioFailure bool) {
q.known = false
q.lowerBound = true
q.stateIO = ioFailure
q.uncertainAt = time.Now()
}
func (c *pendingCall) observe(image pendingImage) {
q := c.queue
q.mu.Lock()
defer q.mu.Unlock()
if q.known && q.disk != image {
q.lowerBound = true
q.uncertainAt = time.Now()
}
q.setDisk(image)
c.progress = time.Now()
}
func (c *pendingCall) lose(count int) {
if count > c.lost {
c.queue.losses.Lose(time.Now(), uint64(count-c.lost)) // #nosec G115 -- the guarded positive difference is bounded by the finding batch lengths.
c.lost = count
}
}
// Incoming findings outside the retained suffix cannot survive either the old
// or the replacement file. Publish that known loss before attempting I/O.
func (c *pendingCall) offer(knownLoss int) {
q := c.queue
q.mu.Lock()
defer q.mu.Unlock()
c.lose(knownLoss)
c.offered = true
c.progress = time.Now()
}
func (c *pendingCall) ioFailed() {
c.queue.mu.Lock()
c.queue.stateIO = true
c.progress = time.Now()
c.queue.mu.Unlock()
}
func (c *pendingCall) complete(image pendingImage, loss int, failed bool, arrivals []time.Time) {
q := c.queue
q.mu.Lock()
defer q.mu.Unlock()
q.setDisk(image)
if arrivals != nil {
q.arrivals = arrivals
}
q.stateIO = failed
c.lose(loss)
c.incoming = 0
c.settled = true
c.progress = time.Now()
}
func (c *pendingCall) unreadable(loss int) {
q := c.queue
q.mu.Lock()
defer q.mu.Unlock()
q.unknown(true)
c.lose(loss)
c.incoming = 0
c.settled = true
c.progress = time.Now()
}
func (c *pendingCall) failedRead() {
if c == nil {
return
}
loss := c.lost
if !c.offered {
loss = c.incoming
}
c.unreadable(loss)
}
// The state lock still belongs to this call. Settle an unreturned write or
// clear before a newer operation can publish its own confirmed disk state.
func (c *pendingCall) settleIO() {
q := c.queue
q.mu.Lock()
defer q.mu.Unlock()
if c.offered && !c.settled {
q.unknown(false)
c.incoming = 0
c.settled = true
}
}
func (c *pendingCall) detach(count int) {
q := c.queue
q.mu.Lock()
defer q.mu.Unlock()
q.setDisk(pendingImage{valid: true})
q.stateIO = false
c.offered = false
c.replay = count
c.progress = time.Now()
}
func (c *pendingCall) finishReplay() {
c.queue.mu.Lock()
c.settled = true
c.queue.mu.Unlock()
}
func (c *pendingCall) finish() {
q := c.queue
q.mu.Lock()
defer q.mu.Unlock()
if !c.settled {
switch {
case c.replay > 0:
// Dispatch may have partially completed. Do not invent an exact loss or
// requeue findings that the existing at-most-once policy already cleared.
q.lowerBound = true
q.uncertainAt = time.Now()
default:
c.lose(c.incoming)
}
}
delete(q.calls, c)
}
// QueueStatuses uses metadata only, independently of state locks, file I/O and
// the replay callback. Waiting for the next restart has no processing deadline.
func (s *Store) QueueStatuses(now time.Time) map[string]queuehealth.Status {
q := s.pendingHealth()
q.mu.Lock()
defer q.mu.Unlock()
row := q.losses.Snapshot(now)
row.DepthUnit = "findings"
row.LagBasis = "deferred_checkpoint"
row.DepthUnavailable = !q.known
if q.known {
row.Depth = q.disk.count
}
row.DroppedLowerBound = q.lowerBound
if row.Depth > 0 {
row.LagSeconds = max(0, now.Sub(q.arrivals[0]).Seconds())
}
op := queuehealth.Status{Status: "ok", CapacityUnavailable: true, DepthUnit: "operations", LagBasis: "operation_progress"}
for c := range q.calls {
row.InFlight += c.incoming + c.replay
if c.progress.IsZero() {
op.Depth++
op.LagSeconds = max(op.LagSeconds, now.Sub(c.queued).Seconds())
} else {
op.InFlight++
op.ProcessingSeconds = max(op.ProcessingSeconds, now.Sub(c.progress).Seconds())
}
}
switch {
case q.stateIO:
row.Status, row.Reason = "degraded", "state_io"
case !q.known:
row.Status, row.Reason = "degraded", "persistence_uncertain"
case !q.uncertainAt.IsZero() && now.Sub(q.uncertainAt) < time.Minute:
row.Status, row.Reason = "degraded", "persistence_uncertain"
}
switch {
case op.LagSeconds >= time.Minute.Seconds():
op.Status, op.Reason = "degraded", "backlog_lag"
case op.ProcessingSeconds >= time.Minute.Seconds():
op.Status, op.Reason = "degraded", "processing_lag"
}
return map[string]queuehealth.Status{"pending": row, "pending_operations": op}
}
package state
import "github.com/pidginhost/csm/internal/alert"
// ScanCoverage describes only units proved by a completed scan. Scope keys
// are scanner-issued identities, never file paths or parsed display text.
type ScanCoverage struct {
PreservePaths map[string]map[string]bool
CompletedScopes map[string]map[string]bool
// IncompleteChecks preserves unexamined findings at the active-set cap,
// including when the scanner could not complete any database scope.
IncompleteChecks map[string]bool
}
func (c *ScanCoverage) completed(f alert.Finding) bool {
return c != nil && f.CoverageScope != "" && c.CompletedScopes[f.Check][f.CoverageScope]
}
// unexamined identifies retained state that must not be evicted to make room
// for new findings from a completed scope in the same scanner.
func (c *ScanCoverage) unexamined(f alert.Finding) bool {
return c != nil && (c.IncompleteChecks[f.Check] || c.CompletedScopes[f.Check] != nil) && !c.completed(f)
}
package state
import (
"fmt"
"os"
"strings"
"github.com/pidginhost/csm/internal/alert"
)
// RearmAbsentDedupFindings forgets dismissals of resolved conditions, only for
// the finding names whose owner supplied a replacement scan result. Update
// cannot do this: its input may be an unrelated tier or realtime batch. An
// alerted, undismissed entry is kept: deep scans do not cover every file each
// cycle, so a condition that is absent for one run and back the next keeps its
// daily reminder instead of alerting on every return.
func (s *Store) RearmAbsentDedupFindings(checks []string, findings []alert.Finding) {
if len(checks) == 0 {
return
}
owners := make(map[string]bool, len(checks))
for _, check := range checks {
owners[check] = true
}
seen := make(map[string]bool, len(findings))
for _, f := range findings {
seen[f.Key()] = true
}
s.mu.Lock()
defer s.mu.Unlock()
changed := false
for key := range s.entries {
identity, pinned := strings.CutPrefix(key, "dedup:")
check, _, _ := strings.Cut(identity, ":")
if pinned && owners[check] && !seen[key] && s.entries[key].IsBaseline {
delete(s.entries, key)
changed = true
}
}
if changed {
s.dirty = true
if err := s.save(); err != nil {
fmt.Fprintf(os.Stderr, "state: error saving resolved scan conditions: %v\n", err)
}
}
}
// RearmFindings resets specific conditions after their producer proves recovery.
// Other producers sharing a finding name keep their own alert history.
func (s *Store) RearmFindings(keys []string) {
s.mu.Lock()
defer s.mu.Unlock()
changed := false
for _, key := range keys {
// Internal housekeeping keys are not conditions; a key from a request
// must not reset them.
if strings.HasPrefix(key, "_") {
continue
}
if s.deleteRawLocked(key) {
changed = true
}
}
if changed {
if err := s.save(); err != nil {
fmt.Fprintf(os.Stderr, "state: error saving recovered scan conditions: %v\n", err)
}
}
}
// RearmDismissedFindings re-arms specific conditions only where the operator
// dismissed them. An alerted, undismissed condition keeps its daily reminder.
func (s *Store) RearmDismissedFindings(keys []string) {
s.mu.Lock()
defer s.mu.Unlock()
changed := false
for _, key := range keys {
if entry, ok := s.entries[key]; ok && entry.IsBaseline {
delete(s.entries, key)
changed = true
}
}
if changed {
s.dirty = true
if err := s.save(); err != nil {
fmt.Fprintf(os.Stderr, "state: error saving resolved scan conditions: %v\n", err)
}
}
}
package state
import (
"crypto/rand"
"crypto/sha256"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/atomicio"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/metrics"
"github.com/pidginhost/csm/internal/store"
)
// findingsTotal counts every finding CSM records, partitioned by
// severity. Registered lazily (via sync.Once) the first time a finding
// lands so tests that open Stores without running a full daemon do not
// panic on duplicate registration.
var (
findingsTotal *metrics.CounterVec
findingsTotalOnce sync.Once
)
func ensureFindingsMetric() {
findingsTotalOnce.Do(func() {
findingsTotal = metrics.NewCounterVec(
"csm_findings_total",
"Findings recorded by CSM, partitioned by severity.",
[]string{"severity"},
)
metrics.MustRegister("csm_findings_total", findingsTotal)
})
}
func recordFindings(findings []alert.Finding) {
if len(findings) == 0 {
return
}
ensureFindingsMetric()
for _, f := range findings {
findingsTotal.With(f.Severity.String()).Inc()
}
}
type Store struct {
mu sync.RWMutex
path string
entries map[string]*Entry
dirty bool // true if state changed since last save
savedHash string // hash of last saved state
lastSave time.Time // start of the last save attempt
now func() time.Time // save clock; nil means time.Now
throttleReservations map[string]struct{}
emailPending map[string]*Entry
pendingHealthOnce sync.Once
pendingQueue *pendingQueue
readPendingFile func(string) ([]byte, error)
writePendingFile func(string, os.FileMode, any) error
// LatestFindings holds the full output of the most recent scan cycle.
// This is what the Findings page shows - "what's wrong right now" -
// separate from the alert dedup state above which controls "what to email."
latestMu sync.RWMutex
latestFindings []alert.Finding
latestDigest [sha256.Size]byte // digest of the last persisted latest_findings.json
latestDigestSet bool
latestScanTime time.Time
// suppressMu makes each change to the suppression rules one
// read-modify-write, so concurrent edits do not overwrite each other.
suppressMu sync.Mutex
}
type Entry struct {
// observation changes only when Update records evidence, not when an email completes.
observation uint64
Hash string `json:"hash"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
AlertSent time.Time `json:"alert_sent"`
IsBaseline bool `json:"is_baseline"`
DismissalID string `json:"dismissal_id,omitempty"`
// Emailed is the fingerprint alert email or the generic webhook last
// delivered under this key. Hash follows every observation; Emailed
// changes only on delivery.
Emailed string `json:"emailed,omitempty"`
// KeptAt is when the latest-findings merge last kept the condition
// listed without a new report, as for a check that skipped its run.
// LastSeen stays the last report, which the findings page shows.
KeptAt time.Time `json:"kept_at,omitzero"`
// ContentSHA256 is the file content a dismissal or baseline covers.
ContentSHA256 string `json:"content_sha256,omitempty"`
// Path is the file the finding names, so alert state about a file CSM
// removed can be cleared.
Path string `json:"path,omitempty"`
}
// aliveAt is the last time the condition was reported or kept listed. An
// entry older than 24h by this measure has cleared.
func (e *Entry) aliveAt() time.Time {
if e.KeptAt.After(e.LastSeen) {
return e.KeptAt
}
return e.LastSeen
}
// lapsingAcknowledgments names the checks whose explicit identity is an event
// condition, reported for as long as it lasts. An acknowledgment of one
// covers later reports only until the condition clears. Scan findings keep
// permanent acknowledgments: a producer that reports once a day or less would
// otherwise lose the dismissal on every cycle.
var lapsingAcknowledgments = map[string]bool{
"auto_block": true,
"email_av_degraded": true,
"self_deleting_dropper_realtime": true,
"sensitive_file_modified": true,
"smtp_probe_abuse": true,
}
// permanent reports a baseline or dismissal that never lapses.
func (e *Entry) permanent(key string) bool {
if !e.IsBaseline {
return false
}
identity, explicit := strings.CutPrefix(key, "dedup:")
check, _, _ := strings.Cut(identity, ":")
return !explicit || !lapsingAcknowledgments[check]
}
// cleared reports whether the condition behind an entry that can lapse has
// gone unseen for 24h.
func (e *Entry) cleared(key string, now time.Time) bool {
return !e.permanent(key) && now.Sub(e.aliveAt()) >= 24*time.Hour
}
func Open(path string) (*Store, error) {
if err := os.MkdirAll(path, 0700); err != nil {
return nil, fmt.Errorf("creating state dir: %w", err)
}
s := &Store{
path: path,
entries: make(map[string]*Entry),
throttleReservations: make(map[string]struct{}),
}
stateFile := filepath.Join(path, "state.json")
loadJSONWithBackup(stateFile, &s.entries)
if s.entries == nil {
s.entries = make(map[string]*Entry)
}
s.migrateEmailGate()
// Load latest findings from disk (survives restart)
latestFile := filepath.Join(path, "latest_findings.json")
var findings []alert.Finding
if loadJSONWithBackup(latestFile, &findings) {
s.latestFindings = findings
}
s.observePendingQueue()
return s, nil
}
// loadJSONWithBackup parses file into v. The backup copy is refreshed only
// after a successful parse, so a corrupt file can never overwrite the last
// good one; on a parse failure the backup is parsed instead. Copying the
// file over its backup before parsing, as this used to, destroyed the
// backup exactly when it was needed and silently reset the alert dedup
// state, re-alerting every known finding. Returns true when v was loaded
// from either source.
func loadJSONWithBackup(file string, v any) bool {
bakFile := file + ".bak"
// #nosec G304 -- operator-configured statePath + fixed filename.
data, err := os.ReadFile(file)
if err == nil {
if unmarshalErr := json.Unmarshal(data, v); unmarshalErr == nil {
// Refresh atomically so a crash while copying a good primary cannot
// destroy the last usable backup.
_ = atomicio.AtomicWrite(bakFile, 0o600, data)
return true
} else {
fmt.Fprintf(os.Stderr, "warning: failed to parse %s: %v (trying %s)\n", file, unmarshalErr, bakFile)
}
}
// #nosec G304 -- same fixed name with a .bak suffix.
bak, err := os.ReadFile(bakFile)
if err != nil {
return false
}
if unmarshalErr := json.Unmarshal(bak, v); unmarshalErr != nil {
fmt.Fprintf(os.Stderr, "warning: failed to parse %s: %v\n", bakFile, unmarshalErr)
return false
}
fmt.Fprintf(os.Stderr, "warning: restored %s from %s\n", file, bakFile)
return true
}
// stateFileWriter writes state.json; a seam for tests.
var stateFileWriter = atomicio.AtomicWrite
// saveInterval bounds how often Update and MarkAlerted rewrite state.json.
// They run on every alert batch and the file holds the whole dedup map, so
// saving on each batch would rewrite it several times a minute on a busy
// host. FlushDue, Flush and Close write what they leave pending.
const saveInterval = 60 * time.Second
func (s *Store) clock() time.Time {
if s.now == nil {
return time.Now()
}
return s.now()
}
// FlushDue writes pending changes once saveInterval has passed since the
// last save. Call it periodically so the last batch before a quiet spell
// still reaches disk.
func (s *Store) FlushDue(now time.Time) error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.dirty || now.Sub(s.lastSave) < saveInterval {
return nil
}
return s.save()
}
// Flush writes pending changes now.
func (s *Store) Flush() error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.dirty {
return nil
}
return s.save()
}
func (s *Store) Close() error {
return s.Flush()
}
func (s *Store) save() error {
// Even a fresh or reset store can persist observations before its first
// successful delivery. They must never be mistaken for pre-gate state.
s.setRawLocked(emailGateMetaKey, "1")
// A failed write counts too: a full disk is retried once per
// saveInterval, not on every alert batch.
s.lastSave = s.clock()
data, err := json.MarshalIndent(s.entries, "", " ")
if err != nil {
return err
}
// Skip write if content hasn't changed
newHash := fmt.Sprintf("%x", sha256.Sum256(data))
if newHash == s.savedHash {
s.dirty = false
return nil
}
// Atomic write with fsync. Power loss between rename and a dir
// fsync can otherwise leave the file truncated; the savedHash gate
// above then "skips" the next attempt because the in-memory hash is
// unchanged, so corruption persists across restarts.
stateFile := filepath.Join(s.path, "state.json")
// Older releases wrote through a fixed <path>.tmp; clear one left by a
// crash so it cannot be mistaken for live state.
if err := os.Remove(stateFile + ".tmp"); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove stale tmp: %w", err)
}
if err := stateFileWriter(stateFile, 0o600, data); err != nil {
return err
}
s.savedHash = newHash
s.dirty = false
return nil
}
func findingKey(f alert.Finding) string {
return f.Key()
}
func findingHash(f alert.Finding) string {
return f.Fingerprint()
}
func (s *Store) FilterNew(findings []alert.Finding) []alert.Finding {
s.mu.RLock()
defer s.mu.RUnlock()
var newFindings []alert.Finding
now := s.clock()
for _, f := range findings {
key := findingKey(f)
hash := findingHash(f)
entry, exists := s.entries[key]
if !exists || entry.cleared(key, now) {
newFindings = append(newFindings, f)
continue
}
if entry.IsBaseline && entry.Hash == hash {
continue
}
if entry.Hash != hash {
newFindings = append(newFindings, f)
continue
}
// Same finding, check if we should re-alert (state expiry)
if !entry.AlertSent.IsZero() && time.Since(entry.AlertSent) > 24*time.Hour {
newFindings = append(newFindings, f)
}
}
return newFindings
}
// emailGateMetaKey marks alert state whose entries record email delivery.
const emailGateMetaKey = "__email_gate"
// migrateEmailGate treats every condition alerted before the email gate
// existed as delivered, muted and cap-held ones included. Without it an
// upgrade would email every known condition at once. Every save writes the
// marker, so state from a gated build never migrates, and a fresh store has
// nothing to migrate.
func (s *Store) migrateEmailGate() {
if len(s.entries) == 0 {
return
}
if _, ok := s.entries[emailGateMetaKey]; ok {
return
}
for key, entry := range s.entries {
if !entry.AlertSent.IsZero() && !entry.IsBaseline && !strings.HasPrefix(key, "_") {
entry.Emailed = entry.Hash
}
}
s.setRawLocked(emailGateMetaKey, "1")
}
// filterUnemailedLocked returns the findings alert email and the generic
// webhook have not delivered with their current content. Unlike FilterNew it
// has no daily reminder: a finding is offered again when its content changes,
// when a covering scan cleared it, or when it was neither reported nor kept
// listed for 24h. An entry that old is deleted here, so a return needs no
// Update sweep first. Held findings refresh LastSeen, so an action or derived
// finding that never reaches Update keeps its receipt while it is still
// produced.
func (s *Store) filterUnemailedLocked(findings []alert.Finding, now time.Time) []alert.Finding {
var out []alert.Finding
for _, f := range findings {
hash := findingHash(f)
key := findingKey(f)
entry, ok := s.entries[key]
if ok && entry.cleared(key, now) {
delete(s.entries, key)
s.dirty = true
ok = false
}
held := ok && (entry.Emailed == hash || (entry.IsBaseline && entry.Hash == hash))
if !held {
out = append(out, f)
continue
}
entry.LastSeen = now
s.dirty = true
}
return out
}
// MarkKept records that the latest-findings merge kept these keys listed. An
// owner that skips or does not finish a run keeps its rows without reporting
// them, and a listed condition must not count as cleared after 24h, neither
// for the email gate nor for the Update sweep. An entry already 24h stale is
// not revived: after downtime the next report emails once.
func (s *Store) MarkKept(keys []string) {
if len(keys) == 0 {
return
}
s.mu.Lock()
defer s.mu.Unlock()
now := s.clock()
for _, key := range keys {
entry, ok := s.entries[key]
// Permanent baselines need no liveness, so retaining them writes nothing.
if !ok || entry.permanent(key) || entry.cleared(key, now) {
continue
}
entry.KeptAt = now
s.dirty = true
}
}
// ForgetCleared deletes the alert state of findings a covering scan no longer
// reports, so a return is new again. Dismissals and internal keys stay.
func (s *Store) ForgetCleared(findings []alert.Finding) {
if len(findings) == 0 {
return
}
s.mu.Lock()
defer s.mu.Unlock()
for _, f := range findings {
key := findingKey(f)
if strings.HasPrefix(key, "_") {
continue
}
if entry, ok := s.entries[key]; ok && !entry.IsBaseline {
delete(s.entries, key)
s.dirty = true
}
}
}
// ReconcileDismissedContent ends a dismissal or baseline once its finding
// reports different file content, and records the content of one that has
// none yet (a dismissal from before content was recorded, or made while the
// hash was unknown). It must run before FilterNew and delivery see the batch.
// The entry is deleted rather than un-dismissed so no AlertSent or Emailed
// receipt survives: the changed file alerts as a new detection. A finding
// without a content hash, or carried from an unexamined path, proves nothing
// and changes nothing. The returned batch keeps, for an ended dismissal, only
// the observations of new content, so deduplication alerts on those.
func (s *Store) ReconcileDismissedContent(findings []alert.Finding) []alert.Finding {
s.mu.Lock()
defer s.mu.Unlock()
var ended map[string]string
for _, f := range findings {
if f.ContentSHA256 == "" || f.ScanCarryForward {
continue
}
key := findingKey(f)
if strings.HasPrefix(key, "_") {
continue
}
entry, ok := s.entries[key]
if !ok || !entry.IsBaseline || entry.ContentSHA256 == f.ContentSHA256 {
continue
}
if entry.ContentSHA256 == "" {
entry.ContentSHA256 = f.ContentSHA256
} else {
if ended == nil {
ended = make(map[string]string)
}
ended[key] = entry.ContentSHA256
delete(s.entries, key)
fmt.Fprintf(os.Stderr, "state: content changed, dismissal ended: %q\n", key)
}
s.dirty = true
}
if ended == nil {
return findings
}
kept := make([]alert.Finding, 0, len(findings))
for _, f := range findings {
if old, ok := ended[findingKey(f)]; ok && (f.ContentSHA256 == "" || f.ScanCarryForward || f.ContentSHA256 == old) {
continue
}
kept = append(kept, f)
}
return kept
}
// ForgetFileAlerts deletes the alert state of every finding about path after
// CSM removed or cleaned that file, so a report of the file coming back is new
// again. Dismissals and internal keys stay.
func (s *Store) ForgetFileAlerts(path string) {
if path == "" {
return
}
path = filepath.Clean(path)
s.mu.Lock()
defer s.mu.Unlock()
for key, entry := range s.entries {
if entry.Path != path || entry.IsBaseline || strings.HasPrefix(key, "_") {
continue
}
delete(s.entries, key)
s.dirty = true
}
}
func (s *Store) Update(findings []alert.Finding) {
s.mu.Lock()
defer s.mu.Unlock()
s.dirty = true
now := s.clock()
seen := make(map[string]bool)
for _, f := range findings {
key := findingKey(f)
hash := findingHash(f)
seen[key] = true
entry, exists := s.entries[key]
if exists && entry.cleared(key, now) {
exists = false
}
if !exists {
entry = &Entry{
observation: 1,
Hash: hash,
FirstSeen: now,
LastSeen: now,
AlertSent: now,
}
s.entries[key] = entry
} else {
entry.observation++
entry.Hash = hash
entry.LastSeen = now
if entry.AlertSent.IsZero() {
entry.AlertSent = now
}
}
if f.FilePath != "" {
entry.Path = filepath.Clean(f.FilePath)
}
}
// Clean up entries that are no longer found. Keys with a leading
// underscore are internal housekeeping written via SetRaw (throttles,
// per-file content-hash baselines, sentinel flags) and never appear in
// the findings stream, so the !seen branch would always evict them and
// silently re-arm one-shot detectors that gate on their presence.
for key, entry := range s.entries {
if strings.HasPrefix(key, "_") {
continue
}
if !seen[key] && entry.cleared(key, now) {
delete(s.entries, key)
}
}
}
// MarkAlerted refreshes AlertSent on each finding's entry so the 24-hour
// dedup window restarts. Call after dispatch with the slice that came back
// from FilterNew. Without this, any finding that survives past the 24-hour
// expiry branch in FilterNew re-emits on every subsequent tick because
// Update only sets AlertSent when an entry is first created.
func (s *Store) MarkAlerted(findings []alert.Finding) {
if len(findings) == 0 {
return
}
s.mu.Lock()
defer s.mu.Unlock()
now := s.clock()
changed := false
for _, f := range findings {
key := findingKey(f)
if entry, ok := s.entries[key]; ok {
entry.AlertSent = now
changed = true
}
}
if !changed {
return
}
s.dirty = true
}
func (s *Store) SetBaseline(findings []alert.Finding) {
s.mu.Lock()
defer s.mu.Unlock()
s.dirty = true
var baselineAt *Entry
if entry, ok := s.entries[baselineAtMetaKey]; ok {
entryCopy := *entry
baselineAt = &entryCopy
}
previous := s.entries
s.entries = make(map[string]*Entry)
now := time.Now()
for _, f := range findings {
key := findingKey(f)
hash := findingHash(f)
content := f.ContentSHA256
if content == "" {
// Missing evidence must not erase known content, including when
// another result in this baseline already supplied it.
entry := s.entries[key]
if entry == nil {
entry = previous[key]
}
if entry != nil {
content = entry.ContentSHA256
}
}
s.entries[key] = &Entry{
Hash: hash,
FirstSeen: now,
LastSeen: now,
IsBaseline: true,
ContentSHA256: content,
}
}
if baselineAt != nil {
s.entries[baselineAtMetaKey] = baselineAt
}
if err := s.save(); err != nil {
fmt.Fprintf(os.Stderr, "state: error saving baseline: %v\n", err)
}
}
// ShouldRunThrottled reports whether the throttle window for checkName has
// elapsed, consuming the slot when it allows. Use only when the throttled
// work cannot fail or time out after scheduling; otherwise pair the
// read-only ThrottleAllows probe with MarkThrottledRan on completion so a
// failed run does not forfeit its slot.
func (s *Store) ShouldRunThrottled(checkName string, intervalMin int) bool {
s.mu.Lock()
defer s.mu.Unlock()
key := throttleKey(checkName)
entry, exists := s.entries[key]
if !exists {
s.entries[key] = &Entry{LastSeen: time.Now()}
s.dirty = true
return true
}
if time.Since(entry.LastSeen) >= time.Duration(intervalMin)*time.Minute {
entry.LastSeen = time.Now()
s.dirty = true
return true
}
return false
}
// ThrottleAllows reports whether the throttle window for checkName has
// elapsed WITHOUT consuming the slot. Callers stamp the slot with
// MarkThrottledRan only after the work completes, so a check that times
// out keeps its slot and may retry on the next cycle instead of waiting
// out the full window with nothing stored.
func (s *Store) ThrottleAllows(checkName string, intervalMin int) bool {
s.mu.RLock()
defer s.mu.RUnlock()
entry, exists := s.entries[throttleKey(checkName)]
if !exists {
return true
}
return time.Since(entry.LastSeen) >= time.Duration(intervalMin)*time.Minute
}
// ReserveThrottle reports whether the throttle window allows checkName to
// start now and reserves the slot for this process until the caller either
// marks the run complete or releases the reservation. The reservation keeps
// concurrent scans from launching the same expensive check before the first
// one has had a chance to stamp its successful run.
func (s *Store) ReserveThrottle(checkName string, intervalMin int) bool {
s.mu.Lock()
defer s.mu.Unlock()
key := throttleKey(checkName)
s.ensureThrottleReservationsLocked()
if _, running := s.throttleReservations[key]; running {
return false
}
entry, exists := s.entries[key]
if exists && time.Since(entry.LastSeen) < time.Duration(intervalMin)*time.Minute {
return false
}
s.throttleReservations[key] = struct{}{}
return true
}
// MarkThrottledRan stamps the throttle slot for checkName. Call only after
// the throttled work actually completed, never at scheduling time.
func (s *Store) MarkThrottledRan(checkName string) {
s.mu.Lock()
defer s.mu.Unlock()
key := throttleKey(checkName)
if s.throttleReservations != nil {
delete(s.throttleReservations, key)
}
if entry, exists := s.entries[key]; exists {
entry.LastSeen = time.Now()
} else {
s.entries[key] = &Entry{LastSeen: time.Now()}
}
s.dirty = true
}
// ReleaseThrottle clears an in-process throttle reservation without stamping
// a successful run. Use when the reserved work times out or is interrupted.
func (s *Store) ReleaseThrottle(checkName string) {
s.mu.Lock()
defer s.mu.Unlock()
if s.throttleReservations != nil {
delete(s.throttleReservations, throttleKey(checkName))
}
}
func (s *Store) ensureThrottleReservationsLocked() {
if s.throttleReservations == nil {
s.throttleReservations = make(map[string]struct{})
}
}
func throttleKey(checkName string) string {
return "_throttle:" + checkName
}
func (s *Store) GetRaw(key string) (string, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
entry, ok := s.entries[key]
if !ok {
return "", false
}
return entry.Hash, true
}
func (s *Store) SetRaw(key, value string) {
s.mu.Lock()
defer s.mu.Unlock()
s.setRawLocked(key, value)
}
// DeleteRaw drops a raw housekeeping entry. Underscore-prefixed keys are exempt
// from the sweeper, so a caller that no longer wants one has to say so.
func (s *Store) DeleteRaw(key string) {
s.mu.Lock()
defer s.mu.Unlock()
s.deleteRawLocked(key)
}
// DeleteRawAndSave drops a raw housekeeping entry and persists the deletion
// before returning.
func (s *Store) DeleteRawAndSave(key string) error {
s.mu.Lock()
defer s.mu.Unlock()
changed := s.deleteRawLocked(key)
if !changed && !s.dirty {
return nil
}
return s.save()
}
func (s *Store) deleteRawLocked(key string) bool {
if _, ok := s.entries[key]; !ok {
return false
}
delete(s.entries, key)
s.dirty = true
return true
}
// SetRawAndSave stores a raw housekeeping value and immediately persists the
// state file. Use it for cursors where a completed scan must survive restart
// even when no finding is emitted later in the cycle.
func (s *Store) SetRawAndSave(key, value string) error {
s.mu.Lock()
defer s.mu.Unlock()
changed := s.setRawLocked(key, value)
if !changed && !s.dirty {
return nil
}
return s.save()
}
// ClaimRawTimestamp atomically claims a persisted time window. It returns
// true for the first claim and after interval has elapsed. A future stored
// timestamp can result from a backward wall-clock step; rebase it to now so
// the window does not stay suppressed until the old clock value catches up.
func (s *Store) ClaimRawTimestamp(key string, now time.Time, interval time.Duration) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
now = now.Round(0)
if entry, ok := s.entries[key]; ok {
if previous, err := time.Parse(time.RFC3339Nano, entry.Hash); err == nil {
previous = previous.Round(0)
if previous.After(now) {
entry.Hash = now.Format(time.RFC3339Nano)
entry.LastSeen = now
s.dirty = true
return false, s.save()
}
if now.Sub(previous) < interval {
return false, nil
}
}
}
s.setRawLocked(key, now.Format(time.RFC3339Nano))
return true, s.save()
}
func (s *Store) setRawLocked(key, value string) bool {
if s.entries == nil {
// A zero-value Store (tests, ad-hoc callers) must not panic on its
// first write; it simply has nothing persisted yet.
s.entries = make(map[string]*Entry)
}
entry, exists := s.entries[key]
if !exists {
s.entries[key] = &Entry{
Hash: value,
FirstSeen: time.Now(),
LastSeen: time.Now(),
}
s.dirty = true
return true
}
if entry.Hash != value {
entry.Hash = value
entry.LastSeen = time.Now()
s.dirty = true
return true
}
return false
}
// AppendHistory writes findings to the state DB. Without an open DB the
// findings are not kept: the daemon refuses to start without one.
func (s *Store) AppendHistory(findings []alert.Finding) {
if len(findings) == 0 {
return
}
recordFindings(findings)
if db := store.Global(); db != nil {
if err := db.AppendHistory(findings); err != nil {
fmt.Fprintf(os.Stderr, "store: append history: %v\n", err)
}
}
}
// Entries returns a snapshot copy of current state entries (thread-safe).
func (s *Store) Entries() map[string]*Entry {
s.mu.RLock()
defer s.mu.RUnlock()
copy := make(map[string]*Entry, len(s.entries))
for k, v := range s.entries {
if k[0] == '_' {
continue // skip internal keys
}
entryCopy := *v
copy[k] = &entryCopy
}
return copy
}
// Acknowledged reports whether a baseline or dismissal still covers f.
func (s *Store) Acknowledged(f alert.Finding) bool {
s.mu.RLock()
defer s.mu.RUnlock()
key := findingKey(f)
e, ok := s.entries[key]
return ok && e.IsBaseline && e.Hash == findingHash(f) && !e.cleared(key, s.clock())
}
// EntryForKey returns the dedup entry for a finding key (check:message).
func (s *Store) EntryForKey(key string) (Entry, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
e, ok := s.entries[key]
if !ok {
return Entry{}, false
}
return *e, true
}
// ReadHistory reads the last limit entries, starting at offset.
// Returns the findings (newest first) and total count.
func (s *Store) ReadHistory(limit, offset int) ([]alert.Finding, int) {
if db := store.Global(); db != nil {
return db.ReadHistory(limit, offset)
}
return nil, 0
}
// ReadHistoryFiltered reads matching history entries newest-first.
// Bounds accept server-local calendar days or RFC 3339 instants. Invalid
// bounds are ignored before they reach the lexicographic bbolt key filter.
func (s *Store) ReadHistoryFiltered(limit, offset int, from, to string, severity int, search string) ([]alert.Finding, int) {
return s.ReadHistoryFilteredWithChecks(limit, offset, from, to, severity, search, nil)
}
// ReadHistoryFilteredWithChecks reads matching history entries newest-first,
// optionally constrained to an exact check-name set.
func (s *Store) ReadHistoryFilteredWithChecks(
limit, offset int,
from, to string,
severity int,
search string,
checks map[string]bool,
) ([]alert.Finding, int) {
// An unreadable bound is dropped; the web UI rejects one before it gets
// here.
fromFilter := ""
toFilter := ""
if _, err := store.ParseHistoryBound(from, false); err == nil {
fromFilter = from
}
if _, err := store.ParseHistoryBound(to, true); err == nil {
toFilter = to
}
if db := store.Global(); db != nil {
return db.ReadHistoryFilteredWithChecks(limit, offset, fromFilter, toFilter, severity, search, checks)
}
return nil, 0
}
// ReadHistorySince returns all findings since the given time, newest-first.
func (s *Store) ReadHistorySince(since time.Time) []alert.Finding {
if db := store.Global(); db != nil {
return db.ReadHistorySince(since)
}
return nil
}
// HistoryMark changes whenever history does.
func (s *Store) HistoryMark() string {
if db := store.Global(); db != nil {
return db.HistoryMark()
}
return ""
}
// LatestByCheck returns the timestamp of the newest history entry of every
// check.
func (s *Store) LatestByCheck() map[string]time.Time {
if db := store.Global(); db != nil {
return db.LatestByCheck()
}
return map[string]time.Time{}
}
// SearchHistorySince returns up to limit matching findings since the given
// time, newest-first.
func (s *Store) SearchHistorySince(since time.Time, limit int, match func(alert.Finding) bool) []alert.Finding {
if limit <= 0 {
return nil
}
if db := store.Global(); db != nil {
return db.SearchHistorySince(since, limit, match)
}
return nil
}
// AggregateByHour returns 24 hourly severity buckets for the last 24 hours.
func (s *Store) AggregateByHour() []store.HourBucket {
if db := store.Global(); db != nil {
return db.AggregateByHour()
}
return nil
}
// AggregateByDay returns 30 daily severity buckets for the last 30 days.
func (s *Store) AggregateByDay() []store.DayBucket {
if db := store.Global(); db != nil {
return db.AggregateByDay()
}
return nil
}
// AggregateByDayN returns `days` daily severity buckets (oldest first),
// clamped by the underlying store's retention window.
func (s *Store) AggregateByDayN(days int) []store.DayBucket {
if db := store.Global(); db != nil {
return db.AggregateByDayN(days)
}
return nil
}
// SetLatestFindings merges scan results into the current findings set.
// Called by the daemon after each periodic scan completes. Merges rather
// than replaces - critical scan results coexist with deep scan results.
// Use ClearLatestFindings() + SetLatestFindings() for a full replace.
func (s *Store) SetLatestFindings(findings []alert.Finding) {
s.latestMu.Lock()
defer s.latestMu.Unlock()
// Build map of existing findings by key
existing := make(map[string]alert.Finding)
for _, f := range s.latestFindings {
existing[f.Key()] = f
}
// Merge new findings (update existing, add new)
for _, f := range findings {
existing[f.Key()] = f // newer overwrites older
}
s.latestFindings = orderAndCapLatest(existing)
s.latestScanTime = time.Now()
s.persistLatestLocked()
}
// PurgeAndMergeFindings atomically removes findings matching the given check
// names and then merges the new findings. This prevents a race window where
// concurrent readers could see findings with perf checks missing.
func (s *Store) PurgeAndMergeFindings(purgeChecks []string, findings []alert.Finding) {
s.PurgeAndMergeFindingsDerived(purgeChecks, findings, nil, nil)
}
// PurgeAndMergeFindingsDerived is PurgeAndMergeFindings followed, under the
// same lock and before the single persist, by a second purge-and-merge of
// derivedChecks with derive(merged): the correlation findings a tier cycle
// rebuilds from the merged set. One file write per cycle instead of two.
func (s *Store) PurgeAndMergeFindingsDerived(purgeChecks []string, findings []alert.Finding, derivedChecks []string, derive func([]alert.Finding) []alert.Finding) {
s.PurgeAndMergeFindingsDerivedWithCoverage(purgeChecks, findings, nil, derivedChecks, derive)
}
// PurgeAndMergeFindingsDerivedWithGaps preserves current findings for path
// aliases captured when the completed scan observed a coverage gap. Callers
// must supply both lexical and resolved aliases they accepted at scan time;
// preservation is evaluated under latestMu without resolving them again.
func (s *Store) PurgeAndMergeFindingsDerivedWithGaps(purgeChecks []string, findings []alert.Finding, preserveAliasesByCheck map[string]map[string]bool, derivedChecks []string, derive func([]alert.Finding) []alert.Finding) {
s.PurgeAndMergeFindingsDerivedWithCoverage(purgeChecks, findings, &ScanCoverage{PreservePaths: preserveAliasesByCheck}, derivedChecks, derive)
}
// PurgeAndMergeFindingsDerivedWithCoverage applies file gaps and completed
// scanner scopes against current state in the same transaction as correlation.
// It returns the current rows a purged check or a completed scope removed:
// rows that were preserved, automatically demoted or reported again in either
// pass are not retired, and neither are rows the active-set cap dropped.
// The keys of every other current row are returned as kept: those
// conditions are still present.
func (s *Store) PurgeAndMergeFindingsDerivedWithCoverage(purgeChecks []string, findings []alert.Finding, coverage *ScanCoverage, derivedChecks []string, derive func([]alert.Finding) []alert.Finding) (retired []alert.Finding, kept []string) {
s.latestMu.Lock()
defer s.latestMu.Unlock()
current := s.latestFindings
merged, retired := purgeAndMergeLatestWithCoverage(current, purgeChecks, findings, coverage, true)
if derive != nil {
derived := derive(append([]alert.Finding(nil), merged...))
var derivedRetired []alert.Finding
merged, derivedRetired = purgeAndMergeLatestWithCoverage(merged, derivedChecks, derived, coverage, false)
rederived := findingsByKey(derived)
n := 0
for _, f := range retired {
if _, ok := rederived[f.Key()]; !ok {
retired[n] = f
n++
}
}
retired = append(retired[:n], derivedRetired...)
}
gone := findingsByKey(retired)
kept = make([]string, 0, len(current))
for _, f := range current {
key := f.Key()
if _, ok := gone[key]; !ok {
kept = append(kept, key)
}
}
s.latestFindings = merged
s.latestScanTime = time.Now()
s.persistLatestLocked()
return retired, kept
}
// earliestObservation resolves the first-seen time a re-reported finding
// keeps. The stored row wins because it was there first; a row written before
// FirstSeen existed contributes its Timestamp instead, so upgrading does not
// reset the history of every long-lived finding. A finding nobody stored
// before starts from its own timestamp.
func earliestObservation(stored, reported alert.Finding) time.Time {
candidates := []time.Time{stored.FirstSeen, stored.Timestamp, reported.FirstSeen, reported.Timestamp}
var earliest time.Time
for _, t := range candidates {
if t.IsZero() {
continue
}
if earliest.IsZero() || t.Before(earliest) {
earliest = t
}
}
return earliest
}
// purgeAndMergeLatest drops findings owned by purgeChecks (and the timeout
// findings those runners produced), merges findings by key, and returns the
// ordered, capped result.
func purgeAndMergeLatest(current []alert.Finding, purgeChecks []string, findings []alert.Finding, preservePathsByCheck map[string]map[string]bool) []alert.Finding {
merged, _ := purgeAndMergeLatestWithCoverage(current, purgeChecks, findings, &ScanCoverage{PreservePaths: preservePathsByCheck}, true)
return merged
}
// purgeAndMergeLatestWithCoverage also returns the current rows it retired:
// those neither kept nor reported again, measured before the cap.
func purgeAndMergeLatestWithCoverage(current []alert.Finding, purgeChecks []string, findings []alert.Finding, coverage *ScanCoverage, retireScopes bool) ([]alert.Finding, []alert.Finding) {
if coverage == nil {
coverage = &ScanCoverage{}
}
preservePathsByCheck := coverage.PreservePaths
remove := make(map[string]bool, len(purgeChecks))
for _, c := range purgeChecks {
remove[c] = true
}
existing := make(map[string]alert.Finding, len(current)+len(findings))
// Owner replacement removes old rows from the output, but a key reported
// again in this merge must inherit its pre-purge observation. This history
// lasts only for this merge; resolved keys leave no tombstone behind.
observations := make(map[string]time.Time, len(current)+len(findings))
preserveAliases := normalizedPreservePathAliases(preservePathsByCheck)
preservationActive := preservePathsByCheck != nil
mismatchedCarryKeys := make(map[string]struct{})
holdChecks := make(map[string]bool)
if preservationActive {
for _, f := range findings {
if !f.ScanCarryForward {
continue
}
if pathMatchesPreservedAliases(f.FilePath, preserveAliases[f.Check]) {
continue
}
// Carry-forward and preservation metadata must describe the same
// scope. If they ever disagree, retain the whole owner rather than
// choosing between a purge and a stale resurrection.
holdChecks[f.Check] = true
mismatchedCarryKeys[f.Key()] = struct{}{}
}
}
protectedKeys := make(map[string]struct{})
scopeProtectedKeys := make(map[string]struct{})
for _, f := range current {
key := f.Key()
observations[key] = earliestObservation(alert.Finding{}, f)
// A finding this package demoted is waiting on the re-verifier, which
// reads the file, not on a scan that merely did not raise it again.
// Purging it here would discard the demotion state and let the same
// file come back at full severity on the next detection, so the
// demotion has to outlive a negative scan. A fresh finding normally
// replaces it below; a protected snapshot stays authoritative against a
// colliding finding from another path.
preserved := holdChecks[f.Check] || pathMatchesPreservedAliases(f.FilePath, preserveAliases[f.Check])
retire := shouldPurgeLatestFinding(f, remove) || (retireScopes && coverage.completed(f))
if preserved || isAutomaticallyDemotedFinding(f) || !retire {
existing[key] = f
if coverage.unexamined(f) {
scopeProtectedKeys[key] = struct{}{}
}
if preserved {
protectedKeys[key] = struct{}{}
}
}
}
for _, f := range findings {
// Marked carry-forward state came from the scanner's earlier snapshot.
// The current set under latestMu is authoritative: overwriting it would
// undo a concurrent update, and inserting it when absent would resurrect
// a concurrent dismissal. A genuinely fresh detection has no marker.
if preservationActive && f.ScanCarryForward {
continue
}
key := f.Key()
if _, staleSnapshot := mismatchedCarryKeys[key]; staleSnapshot {
continue
}
if _, currentIsAuthoritative := protectedKeys[key]; currentIsAuthoritative &&
!pathMatchesPreservedAliases(f.FilePath, preserveAliases[f.Check]) {
continue
}
f.ScanCarryForward = false
f.FirstSeen = earliestObservation(alert.Finding{FirstSeen: observations[key]}, f)
observations[key] = f.FirstSeen
existing[key] = f
if pathMatchesPreservedAliases(f.FilePath, preserveAliases[f.Check]) {
protectedKeys[key] = struct{}{}
}
}
// Protect only previously admitted unexamined findings. New partial
// detections compete for the remaining slots; protecting them too would
// let a persistently incomplete scanner grow the active set without bound.
// Refreshes of retained keys keep their protection and original first-seen.
for key := range scopeProtectedKeys {
protectedKeys[key] = struct{}{}
}
var retired []alert.Finding
for _, f := range current {
if _, ok := existing[f.Key()]; !ok {
retired = append(retired, f)
}
}
return orderAndCapLatestPreserving(existing, protectedKeys), retired
}
func normalizedPreservePathAliases(pathsByCheck map[string]map[string]bool) map[string]map[string]struct{} {
if len(pathsByCheck) == 0 {
return nil
}
aliasesByCheck := make(map[string]map[string]struct{}, len(pathsByCheck))
for check, paths := range pathsByCheck {
if len(paths) == 0 {
continue
}
aliases := make(map[string]struct{}, len(paths)*2)
for path := range paths {
// The scanner already captured both lexical and symlink-resolved
// identities at gap time. Re-resolving here would let a symlink
// retarget between scan and merge change which file is protected.
if alias := normalizedLatestFindingPath(path); alias != "" {
aliases[alias] = struct{}{}
}
}
aliasesByCheck[check] = aliases
}
return aliasesByCheck
}
func pathMatchesPreservedAliases(path string, preserved map[string]struct{}) bool {
if path == "" || len(preserved) == 0 {
return false
}
for _, alias := range latestFindingPathAliases(path) {
if _, ok := preserved[alias]; ok {
return true
}
}
return false
}
func latestFindingPathAliases(path string) []string {
if path == "" {
return nil
}
lexical := filepath.Clean(path)
if absolute, err := filepath.Abs(lexical); err == nil {
lexical = filepath.Clean(absolute)
}
aliases := []string{lexical}
if real, err := filepath.EvalSymlinks(lexical); err == nil {
real = filepath.Clean(real)
if real != lexical {
aliases = append(aliases, real)
}
}
return aliases
}
func normalizedLatestFindingPath(path string) string {
if path == "" {
return ""
}
lexical := filepath.Clean(path)
if absolute, err := filepath.Abs(lexical); err == nil {
lexical = filepath.Clean(absolute)
}
return lexical
}
// latestFindingsCap bounds the active set to keep memory and the persisted
// file bounded; the ordering below decides what the cap keeps.
const latestFindingsCap = 15000
// orderAndCapLatest flattens a keyed set into a deterministic order:
// severity first, then most recent, then key. Map iteration order used to
// decide both the file bytes and which findings a full set dropped.
func orderAndCapLatest(existing map[string]alert.Finding) []alert.Finding {
return orderAndCapLatestPreserving(existing, nil)
}
func orderAndCapLatestPreserving(existing map[string]alert.Finding, protected map[string]struct{}) []alert.Finding {
merged := make([]alert.Finding, 0, len(existing))
for _, f := range existing {
merged = append(merged, f)
}
sort.Slice(merged, func(i, j int) bool {
if merged[i].Severity != merged[j].Severity {
return merged[i].Severity > merged[j].Severity
}
if !merged[i].Timestamp.Equal(merged[j].Timestamp) {
return merged[i].Timestamp.After(merged[j].Timestamp)
}
return merged[i].Key() < merged[j].Key()
})
if len(merged) <= latestFindingsCap {
return merged
}
protectedCount := 0
for _, f := range merged {
if _, ok := protected[f.Key()]; ok {
protectedCount++
}
}
if protectedCount == 0 {
return merged[:latestFindingsCap]
}
// A preservation transaction must not retire an unexamined finding merely
// because newly merged, higher-severity findings reached the normal cap.
// Keep every protected identity, then fill the remaining slots in the same
// deterministic priority order. In the pathological case that protected
// state alone exceeds the cap, the coverage invariant wins temporarily.
unprotectedSlots := latestFindingsCap - protectedCount
if unprotectedSlots < 0 {
unprotectedSlots = 0
}
capped := make([]alert.Finding, 0, max(latestFindingsCap, protectedCount))
for _, f := range merged {
if _, ok := protected[f.Key()]; ok {
capped = append(capped, f)
continue
}
if unprotectedSlots > 0 {
capped = append(capped, f)
unprotectedSlots--
}
}
return capped
}
// latestFindingsWriter writes the persisted file; a seam for tests.
var latestFindingsWriter = atomicio.AtomicWrite
// persistLatestLocked writes latest_findings.json when its content changed
// since the last write. Callers hold latestMu. Compact JSON: the file is
// read back by this process only.
func (s *Store) persistLatestLocked() {
data, err := json.Marshal(s.latestFindings)
if err != nil {
return
}
digest := sha256.Sum256(data)
if s.latestDigestSet && digest == s.latestDigest {
return
}
path := filepath.Join(s.path, "latest_findings.json")
// Older releases wrote through a fixed <path>.tmp; clear one left by a
// crash so it cannot be mistaken for live state.
if err := os.Remove(path + ".tmp"); err != nil && !os.IsNotExist(err) {
return
}
if err := latestFindingsWriter(path, 0o600, data); err != nil {
return
}
s.latestDigest = digest
s.latestDigestSet = true
}
func shouldPurgeLatestFinding(f alert.Finding, remove map[string]bool) bool {
if remove[f.Check] {
return true
}
if f.Check != "check_timeout" {
return false
}
runner, ok := timeoutFindingRunner(f.Message)
return ok && remove[runner]
}
func timeoutFindingRunner(msg string) (string, bool) {
rest, ok := strings.CutPrefix(msg, "Check '")
if !ok {
return "", false
}
runner, _, ok := strings.Cut(rest, "'")
return runner, ok && runner != ""
}
// ClearLatestFindings removes all findings from the latest set.
// Use before SetLatestFindings for a full replace (e.g. initial scan).
func (s *Store) ClearLatestFindings() {
s.latestMu.Lock()
defer s.latestMu.Unlock()
s.latestFindings = nil
}
// LatestFindings returns the full results of the most recent scan.
// This is what the Findings page shows - "what's wrong right now."
func (s *Store) LatestFindings() []alert.Finding {
s.latestMu.RLock()
defer s.latestMu.RUnlock()
result := make([]alert.Finding, len(s.latestFindings))
copy(result, s.latestFindings)
return result
}
// LatestScanTime returns when the last scan completed.
func (s *Store) LatestScanTime() time.Time {
s.latestMu.RLock()
defer s.latestMu.RUnlock()
return s.latestScanTime
}
const baselineAtMetaKey = "__baseline_at"
// EnsureBaseline records the first-start timestamp the first time it is
// called against a fresh state directory. Subsequent calls preserve the
// original value so reinstalls / upgrades do not reset the baseline. Safe
// to call from the daemon boot path on every start.
func (s *Store) EnsureBaseline(now time.Time) {
if _, ok := s.GetRaw(baselineAtMetaKey); ok {
return
}
s.SetRaw(baselineAtMetaKey, now.UTC().Format(time.RFC3339Nano))
}
// BaselineAt returns the persisted baseline timestamp, or the zero time
// when EnsureBaseline has not been called yet.
func (s *Store) BaselineAt() time.Time {
raw, ok := s.GetRaw(baselineAtMetaKey)
if !ok {
return time.Time{}
}
t, err := time.Parse(time.RFC3339Nano, raw)
if err != nil {
return time.Time{}
}
return t
}
// DismissLatestFinding removes a finding from the latest scan results.
func (s *Store) DismissLatestFinding(key string) {
s.latestMu.Lock()
defer s.latestMu.Unlock()
var filtered []alert.Finding
for _, f := range s.latestFindings {
if f.Key() != key {
filtered = append(filtered, f)
}
}
s.latestFindings = filtered
s.persistLatestLocked()
}
// DemoteLatestFinding conditionally lowers a finding's severity in the latest
// scan results. The expected snapshot prevents a completed scan or realtime
// alert from being overwritten by an older re-verification result.
//
// Check, Message and Details never change. Finding.Key() hashes those fields,
// so an explanation written into the finding would orphan every dismissal,
// suppression and alert-dedup entry already keyed to it. DemotedFrom records
// only the severity needed to reverse the operation.
func (s *Store) DemoteLatestFinding(expected alert.Finding, severity alert.Severity) bool {
if severity != alert.Warning || expected.Severity < alert.High || expected.Severity > alert.Critical {
return false
}
s.latestMu.Lock()
defer s.latestMu.Unlock()
for i := range s.latestFindings {
current := &s.latestFindings[i]
if current.Key() != expected.Key() || current.Severity <= severity {
continue
}
if !sameLatestFindingSnapshot(*current, expected) {
return false
}
current.DemotedFrom = current.Severity
current.Severity = severity
s.latestFindings = orderAndCapLatest(findingsByKey(s.latestFindings))
s.persistLatestLocked()
return true
}
return false
}
// PrepareFindingResolution captures the listing and alert observation before a
// fix or re-check starts. Call the returned function only after recovery is
// proven. New evidence or a later operator decision invalidates the capture.
// Delivery receipts are deliberately excluded: finishing an email does not
// change the evidence, and deleting its entry invalidates any pending receipt.
func (s *Store) PrepareFindingResolution(expected alert.Finding) func() bool {
s.latestMu.RLock()
defer s.latestMu.RUnlock()
s.mu.RLock()
defer s.mu.RUnlock()
key := expected.Key()
if expected.Check == "" || strings.HasPrefix(key, "_") {
return func() bool { return false }
}
listed := false
for _, current := range s.latestFindings {
if current.Key() != key {
continue
}
if !sameLatestFindingSnapshot(current, expected) {
return func() bool { return false }
}
listed = true
break
}
entry := s.entries[key]
// An unlisted producer may have alert state before it has a UI row. It
// must not resolve someone else's evidence or an operator dismissal.
if !listed && entry != nil && (entry.Hash != findingHash(expected) || entry.IsBaseline) {
return func() bool { return false }
}
var observation uint64
var baseline bool
var dismissal string
if entry != nil {
observation, baseline, dismissal = entry.observation, entry.IsBaseline, entry.DismissalID
}
return func() bool {
s.latestMu.Lock()
defer s.latestMu.Unlock()
s.mu.Lock()
defer s.mu.Unlock()
currentEntry := s.entries[key]
// BeginDelivery may create an entry while remediation runs. Unlike an
// Update observation it has no evidence revision and may still be cleared.
deliveryOnly := entry == nil && currentEntry != nil && currentEntry.observation == 0 && currentEntry.Hash == findingHash(expected)
if (currentEntry != entry && !deliveryOnly) || (currentEntry != nil &&
(currentEntry.observation != observation || currentEntry.IsBaseline != baseline || currentEntry.DismissalID != dismissal)) {
return false
}
index := -1
for i, current := range s.latestFindings {
if current.Key() != key {
continue
}
if !listed || !sameLatestFindingSnapshot(current, expected) {
return false
}
index = i
break
}
if listed && index == -1 {
return false
}
if s.deleteRawLocked(key) {
if err := s.save(); err != nil {
fmt.Fprintf(os.Stderr, "state: error saving resolved finding: %v\n", err)
}
}
if index != -1 {
s.latestFindings = append(s.latestFindings[:index], s.latestFindings[index+1:]...)
s.persistLatestLocked()
}
return true
}
}
// RestoreLatestFindingSeverity reverses an automatic demotion after the exact
// verifier that owns the finding reports the content as live again.
func (s *Store) RestoreLatestFindingSeverity(expected alert.Finding) bool {
s.latestMu.Lock()
defer s.latestMu.Unlock()
for i := range s.latestFindings {
current := &s.latestFindings[i]
if current.Key() != expected.Key() || !isAutomaticallyDemotedFinding(*current) {
continue
}
if !sameLatestFindingSnapshot(*current, expected) {
return false
}
current.Severity = current.DemotedFrom
current.DemotedFrom = alert.Warning
s.latestFindings = orderAndCapLatest(findingsByKey(s.latestFindings))
s.persistLatestLocked()
return true
}
return false
}
func isAutomaticallyDemotedFinding(f alert.Finding) bool {
return f.Severity == alert.Warning &&
f.DemotedFrom >= alert.High && f.DemotedFrom <= alert.Critical
}
// sameLatestFindingSnapshot reports whether the stored finding is still the one
// verification looked at. Any field the verifier's decision rested on must
// match, or a newer detection would be silently overwritten by a stale verdict.
func sameLatestFindingSnapshot(current, expected alert.Finding) bool {
if current.Key() != expected.Key() ||
current.Check != expected.Check ||
current.Message != expected.Message ||
current.Details != expected.Details ||
current.Severity != expected.Severity ||
!current.Timestamp.Equal(expected.Timestamp) ||
current.FilePath != expected.FilePath ||
current.ContentSHA256 != expected.ContentSHA256 ||
current.DetectLogic != expected.DetectLogic {
return false
}
return current.DemotedFrom == expected.DemotedFrom
}
func findingsByKey(findings []alert.Finding) map[string]alert.Finding {
keyed := make(map[string]alert.Finding, len(findings))
for _, finding := range findings {
keyed[finding.Key()] = finding
}
return keyed
}
// DismissFinding marks a finding as baseline (acknowledged/dismissed).
// It will no longer appear in active findings or trigger new alerts.
func (s *Store) DismissFinding(key string) {
s.mu.Lock()
defer s.mu.Unlock()
if entry, exists := s.entries[key]; exists {
entry.IsBaseline = true
entry.DismissalID = ""
s.dirty = true
}
}
// DismissUndo records what DismissFindingWithUndo changed, so UndoDismiss can
// return the finding to the state it had before the operator dismissed it.
type DismissUndo struct {
Key string `json:"key"`
// Identity and hash prevent an old undo from reversing a later decision
// or changing alert state for newer evidence under the same key.
DismissalID string `json:"dismissal_id"`
Hash string `json:"hash"`
ClearBaseline bool `json:"clear_baseline,omitempty"`
CreatedEntry bool `json:"created_entry,omitempty"`
Removed []alert.Finding `json:"removed,omitempty"`
// PrevContent is nil when an earlier baseline's content was left alone.
// Undo must preserve evidence that baseline adopts after the dismissal.
PrevContent *string `json:"prev_content,omitempty"`
}
// DismissFindingWithUndo marks a finding as baseline and removes it from the
// latest list as one operation. Latest-only findings also need alert state:
// realtime findings can reach the UI before the next scan updates dedup state.
// The dismissal covers the content of the listed row.
func (s *Store) DismissFindingWithUndo(key string) DismissUndo {
s.latestMu.Lock()
defer s.latestMu.Unlock()
s.mu.Lock()
defer s.mu.Unlock()
u, _ := s.dismissLocked(key, nil)
return u
}
// DismissFindingAsShown dismisses key only while every listed row for it
// still carries shown, the content hash the operator judged (empty for a
// finding without one). A row whose content changed since it was shown is
// refused with nothing changed, so new content is reviewed before it is
// dismissed.
func (s *Store) DismissFindingAsShown(key, shown string) (DismissUndo, bool) {
s.latestMu.Lock()
defer s.latestMu.Unlock()
s.mu.Lock()
defer s.mu.Unlock()
return s.dismissLocked(key, &shown)
}
// dismissLocked runs under latestMu and mu. A nil shown makes no claim.
func (s *Store) dismissLocked(key string, shown *string) (DismissUndo, bool) {
u := DismissUndo{Key: key}
var kept []alert.Finding
for _, f := range s.latestFindings {
if f.Key() == key {
if shown != nil && f.ContentSHA256 != *shown {
return DismissUndo{Key: key}, false
}
u.Removed = append(u.Removed, f)
continue
}
kept = append(kept, f)
}
var content string
switch {
case shown != nil:
content = *shown
case len(u.Removed) > 0:
content = u.Removed[0].ContentSHA256
}
entry := s.entries[key]
if entry == nil && len(u.Removed) > 0 {
now := time.Now()
entry = &Entry{Hash: findingHash(u.Removed[0]), FirstSeen: now, LastSeen: now}
s.entries[key] = entry
u.CreatedEntry = true
}
if entry != nil {
u.ClearBaseline = !entry.IsBaseline
if u.ClearBaseline || (content != "" && content != entry.ContentSHA256) {
previous := entry.ContentSHA256
u.PrevContent = &previous
}
entry.IsBaseline = true
if content != "" {
entry.ContentSHA256 = content
}
entry.DismissalID = rand.Text()
u.DismissalID, u.Hash = entry.DismissalID, entry.Hash
s.dirty = true
}
s.latestFindings = kept
s.persistLatestLocked()
return u, true
}
// UndoDismiss reverses only the dismissal that still owns the entry. A copy
// reported by a scan in the meantime is newer evidence and remains listed.
func (s *Store) UndoDismiss(u DismissUndo) bool {
s.latestMu.Lock()
defer s.latestMu.Unlock()
s.mu.Lock()
defer s.mu.Unlock()
entry := s.entries[u.Key]
if u.DismissalID == "" || entry == nil || entry.DismissalID != u.DismissalID || entry.Hash != u.Hash {
return false
}
entry.DismissalID = ""
if u.PrevContent != nil {
entry.ContentSHA256 = *u.PrevContent
}
if u.CreatedEntry {
delete(s.entries, u.Key)
} else if u.ClearBaseline {
entry.IsBaseline = false
}
s.dirty = true
if len(u.Removed) > 0 {
merged := findingsByKey(s.latestFindings)
for _, f := range u.Removed {
if _, present := merged[f.Key()]; !present {
merged[f.Key()] = f
}
}
s.latestFindings = orderAndCapLatest(merged)
s.persistLatestLocked()
}
return true
}
// ParseKey splits a state key "check:message" into its components.
func ParseKey(key string) (check, message string) {
for i := 0; i < len(key); i++ {
if key[i] == ':' {
return key[:i], key[i+1:]
}
}
return key, ""
}
// --- Suppression rules ---
// SuppressionRule defines a rule for suppressing specific findings.
type SuppressionRule struct {
ID string `json:"id"`
Check string `json:"check"`
PathPattern string `json:"path_pattern,omitempty"`
Reason string `json:"reason"`
CreatedAt time.Time `json:"created_at"`
}
// LoadSuppressions reads suppression rules from disk.
func (s *Store) LoadSuppressions() []SuppressionRule {
data, err := os.ReadFile(filepath.Join(s.path, "suppressions.json"))
if err != nil {
return nil
}
var rules []SuppressionRule
if err := json.Unmarshal(data, &rules); err != nil {
return nil
}
return rules
}
// SaveSuppressions writes suppression rules to disk atomically with fsync.
// Callers that change the existing rules use UpdateSuppressions instead.
func (s *Store) SaveSuppressions(rules []SuppressionRule) error {
return atomicio.AtomicWriteJSON(filepath.Join(s.path, "suppressions.json"), 0o600, rules)
}
// UpdateSuppressions applies fn to the stored rules and saves what it returns
// as one step. An error from fn leaves the stored rules unchanged.
func (s *Store) UpdateSuppressions(fn func([]SuppressionRule) ([]SuppressionRule, error)) error {
s.suppressMu.Lock()
defer s.suppressMu.Unlock()
rules, err := fn(s.LoadSuppressions())
if err != nil {
return err
}
return s.SaveSuppressions(rules)
}
// IsSuppressed checks if a finding matches any loaded suppression rule.
// Load rules once with LoadSuppressions() and pass them in to avoid
// re-reading the file for every finding.
func (s *Store) IsSuppressed(f alert.Finding, rules []SuppressionRule) bool {
for _, rule := range rules {
if config.CanonicalCheckName(f.Check) != config.CanonicalCheckName(rule.Check) {
continue
}
// If no path pattern, suppress all findings for this check type
if rule.PathPattern == "" {
return true
}
// Match against the finding's FilePath
if f.FilePath != "" {
if matched, _ := filepath.Match(rule.PathPattern, f.FilePath); matched {
return true
}
}
for _, candidate := range suppressionPathCandidates(f) {
if matched, _ := filepath.Match(rule.PathPattern, candidate); matched {
return true
}
}
}
return false
}
func suppressionPathCandidates(f alert.Finding) []string {
if f.FilePath != "" {
return []string{f.FilePath}
}
fields := strings.Fields(f.Message + " " + f.Details)
seen := make(map[string]bool)
var paths []string
for _, field := range fields {
field = strings.Trim(field, `"'():,;[]{}<>`)
if !strings.HasPrefix(field, "/") {
continue
}
candidate := filepath.Clean(field)
if candidate == "." || candidate == "/" || seen[candidate] {
continue
}
seen[candidate] = true
paths = append(paths, candidate)
}
return paths
}
package store
import (
"bytes"
"encoding/json"
"strings"
"time"
bolt "go.etcd.io/bbolt"
)
// AdminEmailEntry records that a given email was observed as a WordPress
// administrator on (Account, Schema). One email may carry multiple
// entries when the same person administers several customer sites --
// surfacing that overlap is the whole point of the bucket.
type AdminEmailEntry struct {
Account string `json:"account"`
Schema string `json:"schema"`
LastSeen time.Time `json:"last_seen"`
}
const adminEmailsBucket = "admin:emails"
// RecordAdminEmail upserts an observation that `email` is administrator
// on (account, schema) at `now`. Re-recording the same triple updates
// LastSeen without creating a duplicate row; recording the same email
// across a different (account, schema) appends to the owner list. The
// email is lowercased before storage so case-mismatched recordings
// collapse to a single key.
func (db *DB) RecordAdminEmail(email, account, schema string, now time.Time) error {
email = strings.ToLower(strings.TrimSpace(email))
if email == "" || account == "" {
return nil
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(adminEmailsBucket))
var entries []AdminEmailEntry
if v := b.Get([]byte(email)); v != nil {
if err := json.Unmarshal(v, &entries); err != nil {
// Corrupt entry: restart the list rather than fail the
// whole write -- we'd rather lose one stale observation
// than block detection on a malformed payload.
entries = nil
}
}
updated := false
for i := range entries {
if entries[i].Account == account && entries[i].Schema == schema {
entries[i].LastSeen = now
updated = true
break
}
}
if !updated {
entries = append(entries, AdminEmailEntry{
Account: account,
Schema: schema,
LastSeen: now,
})
}
payload, err := json.Marshal(entries)
if err != nil {
return err
}
return b.Put([]byte(email), payload)
})
}
// AdminEmailOwners returns the full list of (account, schema, last_seen)
// triples recorded for `email`. Returns an empty slice when the email
// is unknown.
func (db *DB) AdminEmailOwners(email string) ([]AdminEmailEntry, error) {
email = strings.ToLower(strings.TrimSpace(email))
var out []AdminEmailEntry
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(adminEmailsBucket))
v := b.Get([]byte(email))
if v == nil {
return nil
}
return json.Unmarshal(v, &out)
})
return out, err
}
// OverlappingAdminEmails returns every email whose owner list has at
// least `minAccounts` distinct accounts after stale entries (older than
// `retention`) are pruned.
//
// `minAccounts` smaller than 2 is clamped to 2 because a single-account
// observation is never an overlap by definition.
func (db *DB) OverlappingAdminEmails(minAccounts int, retention time.Duration) (map[string][]AdminEmailEntry, error) {
if minAccounts < 2 {
minAccounts = 2
}
cutoff := time.Now().Add(-retention)
out := map[string][]AdminEmailEntry{}
type pruneItem struct {
key []byte
original []byte
fresh []byte
}
var toPrune []pruneItem
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(adminEmailsBucket))
return b.ForEach(func(k, v []byte) error {
var entries []AdminEmailEntry
if err := json.Unmarshal(v, &entries); err != nil {
return nil //nolint:nilerr // skip corrupt entry
}
fresh := entries[:0]
seen := map[string]struct{}{}
for _, e := range entries {
if e.LastSeen.Before(cutoff) {
continue
}
key := e.Account + "|" + e.Schema
if _, dup := seen[key]; dup {
continue
}
seen[key] = struct{}{}
fresh = append(fresh, e)
}
distinct := map[string]struct{}{}
for _, e := range fresh {
distinct[e.Account] = struct{}{}
}
if len(distinct) >= minAccounts {
out[string(k)] = append([]AdminEmailEntry(nil), fresh...)
}
if len(fresh) == 0 {
toPrune = append(toPrune, pruneItem{key: append([]byte(nil), k...), original: append([]byte(nil), v...)})
} else if len(fresh) != len(entries) {
payload, err := json.Marshal(fresh)
if err != nil {
return err
}
toPrune = append(toPrune, pruneItem{
key: append([]byte(nil), k...),
original: append([]byte(nil), v...),
fresh: payload,
})
}
return nil
})
})
if err != nil || len(toPrune) == 0 {
return out, err
}
err = db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(adminEmailsBucket))
for _, item := range toPrune {
if !bytes.Equal(b.Get(item.key), item.original) {
continue
}
if len(item.fresh) == 0 {
if deleteErr := b.Delete(item.key); deleteErr != nil {
return deleteErr
}
continue
}
if putErr := b.Put(item.key, item.fresh); putErr != nil {
return putErr
}
}
return nil
})
return out, err
}
package store
import (
"encoding/binary"
"errors"
"fmt"
"slices"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
const (
admissionMetaBucket = "adm:meta"
admissionEvidenceBucket = "adm:evidence"
admissionReportsBucket = "adm:reports"
admissionCandidatesBucket = "adm:candidates"
admissionAttemptsBucket = "adm:attempts"
admissionQueueBucket = "adm:queue"
admissionQueueStateBucket = "adm:qstate"
admissionChargesBucket = "adm:charges"
admissionHistoryBucket = "adm:history"
admissionRetireBucket = "adm:retire"
admissionRefsBucket = "adm:evrefs"
admissionRingsBucket = "adm:rings"
admissionOutboxBucket = "adm:outbox"
admissionWindowsBucket = "adm:windows"
admissionEpisodesBucket = "adm:episodes"
admissionIntentsBucket = "adm:intents"
admissionSchemaVersion = 7
// admissionSchemaLinked is the schema of a ledger holding a candidate
// linked to the block that enforces it. The first linked candidate
// writes it; older builds refuse it rather than misread the record.
admissionSchemaLinked = 8
// admissionSchemaShadow is the schema of a ledger holding a dry-run
// ending or a candidate the panel's verdict allowed. Its first such
// record writes it, with the same buckets.
admissionSchemaShadow = 9
// admissionSchemaDisabled is the schema of a ledger holding a candidate
// refused because the configuration turned its response off.
admissionSchemaDisabled = 10
// admissionSchemaLive is the schema of a ledger whose live start ran:
// its marker, and the state it carried over.
admissionSchemaLive = 11
)
var (
// admissionSchemaOneBuckets are the buckets of the schema 1 layout.
// Schema 2 adds the queue buckets, schema 3 the charges bucket, schema
// 4 the storage buckets, schema 5 the outbox and outcome buckets, schema
// 6 the episode bucket and schema 7 the intent bucket.
admissionSchemaOneBuckets = []string{admissionMetaBucket, admissionEvidenceBucket, admissionReportsBucket, admissionCandidatesBucket, admissionAttemptsBucket}
admissionQueueBuckets = []string{admissionQueueBucket, admissionQueueStateBucket}
admissionSchemaTwoBuckets = append(append([]string(nil), admissionSchemaOneBuckets...), admissionQueueBuckets...)
admissionSchemaThreeBuckets = append(append([]string(nil), admissionSchemaTwoBuckets...), admissionChargesBucket)
admissionStorageBuckets = []string{admissionHistoryBucket, admissionRetireBucket, admissionRefsBucket, admissionRingsBucket}
admissionSchemaFourBuckets = append(append([]string(nil), admissionSchemaThreeBuckets...), admissionStorageBuckets...)
admissionOutboxBuckets = []string{admissionOutboxBucket, admissionWindowsBucket}
admissionSchemaFiveBuckets = append(append([]string(nil), admissionSchemaFourBuckets...), admissionOutboxBuckets...)
admissionSchemaSixBuckets = append(append([]string(nil), admissionSchemaFiveBuckets...), admissionEpisodesBucket)
admissionBuckets = append(append([]string(nil), admissionSchemaSixBuckets...), admissionIntentsBucket)
admissionSchemaKey = []byte("schema")
// admissionFeaturesKey holds the ledger's feature bits, eight bytes big
// endian. Every build before it refuses an unknown meta key, so a bit
// keeps a binary that predates its layout off the ledger without a
// schema value.
admissionFeaturesKey = []byte("features")
admissionClockKey = []byte("clock")
admissionClockPendingKey = []byte("clock_pending")
admissionTrackerKey = []byte("generations")
admissionInventoryKey = []byte("inventory")
admissionAmbiguousKey = []byte("ambiguous_domains")
)
// ErrAdmissionSchema reports admission buckets this build cannot read: an
// unknown schema, or data without a schema marker.
var ErrAdmissionSchema = errors.New("admission ledger schema is not supported")
// AdmissionLedger is the durable admission state, kept in the
// daemon's state database. The engine is its only owner: mutating calls
// serialize on one mutex and each runs in one write transaction, so an error
// leaves the ledger unchanged. Nothing constructs it in production yet.
type AdmissionLedger struct {
db *DB
reg *admission.Registry
mu sync.Mutex
now time.Time
// elapsed belongs to the same committed reading as now. Running work
// can record its outcome even when damage prevents a later reading.
elapsed time.Duration
// current is set only by a reading this handle recorded. A reopened
// handle, or one whose last reading was refused, keeps the stored
// high-water mark but admits nothing new: after a restart that mark can
// be hours old, and old evidence would read as fresh.
current bool
// degraded is set when the current reading was degraded: wall time alone
// then cannot end an episode.
degraded bool
// reserved is set once this handle granted a reservation: Settle then
// refuses, since it would end work this process still runs.
reserved bool
// live is set once the live start ran; guarded by mu.
live bool
// set is the temporary set outside the ledger's effect records, as
// the owner last read it (SetOccupants); nil leaves block work without
// a slot budget. Guarded by mu.
set *admission.SetOccupancy
// imports are the known automatic imports the owner last read from the
// firewall (SetImports), victims with no record of their own. Guarded
// by mu.
imports []admission.EvictionVictim
// waiting are the scopes with work waiting for a set slot in each
// partition at the last schedule cycle, which shares the set with them
// until the next. Guarded by mu.
waiting map[admission.Partition][]string
// executed holds the reading each attempt this handle ran was executed
// at, until its outcome: a verified block's episode ends from it, not
// from a later reading a wall step moved. A restart turns a running
// attempt unknown, so no verified outcome needs one it lost. Guarded
// by mu.
executed map[admission.ActionID]admission.EpisodeClock
// missed holds the blocks the last firewall answer no longer held,
// with the reading of that answer (ObserveHeld). Guarded by mu.
missed map[admission.ActionID]time.Time
// Two answered checks per batch let the tail progress even when
// the configured capacity exceeds one engine call. Guarded by mu.
heldAfter []byte
heldPass bool
// retireAfter continues bounded history sweeps past retained episodes.
// Published only after a successful tick; guarded by mu.
retireAfter []byte
// revalidated is set once this handle has checked every queued
// candidate at a current reading. A reopened ledger cannot know what
// changed while it was closed, so its first schedule checks them all.
revalidated bool
inv atomic.Pointer[ledgerInventory]
// previewsEnd is set while observe previews do not answer their
// episodes: a forecasting owner sets it.
previewsEnd atomic.Bool
// held is the recovery hold: how many intents name each target key.
// Every intent belongs to an unresolved attempt (opening proves it), so
// a counted key is held. Opening derives it once; heldDelta collects a
// write transaction's intent changes, applied only after it commits.
// Both are guarded by mu.
held map[string]int
heldDelta map[string]int
// onRefused learns of each queued candidate a later check ends, with
// the refusal, its cause and the ledger time it ended at, once the
// ending commits; refused collects a write transaction's endings until
// then. Both are guarded by mu.
onRefused func(admission.Candidate, error, time.Time)
refused []refusedEnding
// failBeforeCommit lets a test fail a write transaction after all of
// its writes, to prove that nothing is published.
failBeforeCommit func(op string) error
}
type ledgerInventory struct {
inv *admission.Inventory
ambiguous int
}
func corruptRecord(err error) error {
if errors.Is(err, admission.ErrCorruptRecord) {
return err
}
return fmt.Errorf("%w: %v", admission.ErrCorruptRecord, err)
}
func refusal(r admission.Reason, detail string) error {
return &admission.Error{Reason: r, Detail: detail}
}
// OpenAdmissionLedger opens the ledger on db, creating its buckets on first
// use and upgrading a schema 1 to 6 ledger in the same transaction. The
// registry must be sealed: the set of producers cannot change under a
// running ledger.
func OpenAdmissionLedger(db *DB, reg *admission.Registry) (*AdmissionLedger, error) {
if db == nil || reg == nil || !reg.Sealed() {
return nil, errors.New("admission ledger needs a database and a sealed producer registry")
}
l := &AdmissionLedger{db: db, reg: reg}
published := ledgerInventory{}
err := db.bolt.Update(func(tx *bolt.Tx) error {
present := func(names []string) (n int) {
for _, name := range names {
if tx.Bucket([]byte(name)) != nil {
n++
}
}
return n
}
if err := tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
if strings.HasPrefix(string(name), "adm:") && !slices.Contains(admissionBuckets, string(name)) && string(name) != admissionEffectsBucket {
return admission.ErrCorruptRecord
}
return nil
}); err != nil {
return err
}
existing := present(admissionBuckets)
if existing == 0 {
for _, name := range admissionBuckets {
if _, err := tx.CreateBucket([]byte(name)); err != nil {
return err
}
}
if err := initializeLedgerMeta(tx.Bucket([]byte(admissionMetaBucket))); err != nil {
return err
}
if err := initializeQueueState(tx.Bucket([]byte(admissionQueueStateBucket))); err != nil {
return err
}
if err := putCeilingState(tx, admission.CeilingState{Fill: true}); err != nil {
return err
}
if err := putStorageState(tx, admission.NewStorageState()); err != nil {
return err
}
if err := putFixedNotices(tx); err != nil {
return err
}
if err := startEpisodeSequence(tx); err != nil {
return err
}
} else {
meta := tx.Bucket([]byte(admissionMetaBucket))
if meta == nil {
return admission.ErrCorruptRecord
}
switch schema := meta.Get(admissionSchemaKey); {
case len(schema) == 1 && schema[0] == 1:
if existing != len(admissionSchemaOneBuckets) || present(admissionSchemaOneBuckets) != existing {
return admission.ErrCorruptRecord
}
if err := upgradeLedgerToSchemaTwo(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaThree(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaFour(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaFive(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaSix(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaSeven(tx); err != nil {
return err
}
case len(schema) == 1 && schema[0] == 2:
if existing != len(admissionSchemaTwoBuckets) || present(admissionSchemaTwoBuckets) != existing {
return admission.ErrCorruptRecord
}
if err := upgradeLedgerToSchemaThree(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaFour(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaFive(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaSix(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaSeven(tx); err != nil {
return err
}
case len(schema) == 1 && schema[0] == 3:
if existing != len(admissionSchemaThreeBuckets) || present(admissionSchemaThreeBuckets) != existing {
return admission.ErrCorruptRecord
}
if err := upgradeLedgerToSchemaFour(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaFive(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaSix(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaSeven(tx); err != nil {
return err
}
case len(schema) == 1 && schema[0] == 4:
if existing != len(admissionSchemaFourBuckets) || present(admissionSchemaFourBuckets) != existing {
return admission.ErrCorruptRecord
}
if err := upgradeLedgerToSchemaFive(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaSix(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaSeven(tx); err != nil {
return err
}
case len(schema) == 1 && schema[0] == 5:
if existing != len(admissionSchemaFiveBuckets) || present(admissionSchemaFiveBuckets) != existing {
return admission.ErrCorruptRecord
}
if err := upgradeLedgerToSchemaSix(tx); err != nil {
return err
}
if err := upgradeLedgerToSchemaSeven(tx); err != nil {
return err
}
case len(schema) == 1 && schema[0] == 6:
if existing != len(admissionSchemaSixBuckets) || present(admissionSchemaSixBuckets) != existing {
return admission.ErrCorruptRecord
}
if err := upgradeLedgerToSchemaSeven(tx); err != nil {
return err
}
case len(schema) == 1 && schema[0] >= admissionSchemaVersion && schema[0] <= admissionSchemaNewest:
if existing != len(admissionBuckets) {
return admission.ErrCorruptRecord
}
default:
return ErrAdmissionSchema
}
}
if err := validateStateKeys(tx); err != nil {
return err
}
if _, err := loadFeatures(tx); err != nil {
return err
}
_, live, liveErr := loadLiveStart(tx)
if liveErr != nil {
return liveErr
}
l.live = live
if _, err := loadQueueState(tx); err != nil {
return err
}
if _, err := loadQueueCounters(tx); err != nil {
return err
}
if _, err := loadScheduleState(tx); err != nil {
return err
}
if _, err := loadIngressState(tx); err != nil {
return err
}
ceiling, ceilingErr := loadCeiling(tx)
if ceilingErr != nil {
return ceilingErr
}
l.elapsed = ceiling.Elapsed
if _, err := loadStorage(tx); err != nil {
return err
}
if err := proveEpisodes(tx); err != nil {
return err
}
if err := proveEffects(tx); err != nil {
return err
}
held, err := proveIntents(tx)
if err != nil {
return err
}
l.held = held
meta := tx.Bucket([]byte(admissionMetaBucket))
c, err := loadLedgerClock(meta)
if err != nil {
return err
}
if err = proveOutcomes(tx, c.Now()); err != nil {
return err
}
l.now = c.Now()
published, err = loadLedgerInventory(meta)
return err
})
if err != nil {
return nil, err
}
l.inv.Store(&published)
return l, nil
}
// raiseSchema records, in tx, that the ledger holds a record only schema n
// or later reads. It never lowers the marker: a later schema's records stay
// unreadable to builds that predate it.
func raiseSchema(tx *bolt.Tx, n byte) error {
meta := tx.Bucket([]byte(admissionMetaBucket))
schema := meta.Get(admissionSchemaKey)
if len(schema) != 1 {
return admission.ErrCorruptRecord
}
if schema[0] >= n {
return nil
}
return meta.Put(admissionSchemaKey, []byte{n})
}
func initializeLedgerMeta(meta *bolt.Bucket) error {
inv, err := admission.NewInventory(nil, nil)
if err != nil {
return err
}
snapshot, err := inv.MarshalBinary()
if err != nil {
return err
}
tracker, err := admission.NewGenerations().MarshalBinary()
if err != nil {
return err
}
for _, entry := range []struct{ key, value []byte }{
{admissionSchemaKey, []byte{admissionSchemaVersion}},
{admissionClockPendingKey, []byte{1}},
{admissionTrackerKey, tracker},
{admissionInventoryKey, snapshot},
{admissionAmbiguousKey, []byte("0")},
} {
if err := meta.Put(entry.key, entry.value); err != nil {
return err
}
}
return nil
}
// A separate marker distinguishes a fresh clock from a lost checkpoint.
// The first tick removes it in the same transaction that writes the clock.
func loadLedgerClock(meta *bolt.Bucket) (admission.Clock, error) {
raw, pending := meta.Get(admissionClockKey), meta.Get(admissionClockPendingKey)
if raw == nil && len(pending) == 1 && pending[0] == 1 {
return admission.Clock{}, nil
}
if raw == nil || pending != nil {
return admission.Clock{}, admission.ErrCorruptRecord
}
return admission.UnmarshalClock(raw)
}
func loadLedgerTracker(meta *bolt.Bucket) (*admission.Generations, error) {
g := admission.NewGenerations()
if err := g.UnmarshalBinary(meta.Get(admissionTrackerKey)); err != nil {
return nil, corruptRecord(err)
}
return g, nil
}
func loadLedgerInventory(meta *bolt.Bucket) (ledgerInventory, error) {
raw := meta.Get(admissionInventoryKey)
inv, err := admission.UnmarshalInventory(raw)
if err != nil {
return ledgerInventory{}, err
}
g, err := loadLedgerTracker(meta)
if err != nil {
return ledgerInventory{}, err
}
if !inv.MatchesGenerations(g) {
return ledgerInventory{}, admission.ErrCorruptRecord
}
amb := string(meta.Get(admissionAmbiguousKey))
n, err := strconv.Atoi(amb)
if err != nil || n < 0 || strconv.Itoa(n) != amb {
return ledgerInventory{}, admission.ErrCorruptRecord
}
return ledgerInventory{inv: inv, ambiguous: n}, nil
}
func (l *AdmissionLedger) update(op string, fn func(tx *bolt.Tx) error) error {
l.heldDelta, l.refused = map[string]int{}, nil
defer func() { l.heldDelta, l.refused = nil, nil }()
if err := l.db.bolt.Update(func(tx *bolt.Tx) error {
if err := fn(tx); err != nil {
return err
}
if l.failBeforeCommit != nil {
return l.failBeforeCommit(op)
}
return nil
}); err != nil {
return err
}
for key, n := range l.heldDelta {
if l.held[key] += n; l.held[key] <= 0 {
delete(l.held, key)
}
}
if l.onRefused != nil {
for _, r := range l.refused {
l.onRefused(r.c, r.err, r.at)
}
}
return nil
}
// refusedEnding is a queued candidate a check ended, why, and when.
type refusedEnding struct {
c admission.Candidate
err error
at time.Time
}
// ObserveRefusals registers fn to learn of each queued candidate a later
// check refuses (revalidation, an inventory refresh, a configuration
// opt-out, the recheck of a schedule or preview pick) or that loses its
// queue position, with the refusal, its cause and the ledger time it ended
// at, after the ending commits. fn runs under the ledger lock and must not
// call it.
func (l *AdmissionLedger) ObserveRefusals(fn func(admission.Candidate, error, time.Time)) {
l.mu.Lock()
defer l.mu.Unlock()
l.onRefused = fn
}
// Tick records a clock reading. The high-water mark it persists is the only
// time the other calls use. The same transaction meters the ceiling and the
// history allowances, retires history at its target and drops the outcome
// buckets that have left their windows, so a crash can neither lose nor
// repeat the elapsed time it credits.
func (l *AdmissionLedger) Tick(r admission.ClockReading) (admission.ClockTick, error) {
l.mu.Lock()
defer l.mu.Unlock()
var tick admission.ClockTick
var elapsed time.Duration
var retireAfter []byte
err := l.update("tick", func(tx *bolt.Tx) error {
meta := tx.Bucket([]byte(admissionMetaBucket))
c, err := loadLedgerClock(meta)
if err != nil {
return err
}
next, t, err := c.Advance(r)
if err != nil {
return err
}
data, err := next.MarshalBinary()
if err != nil {
return err
}
tick = t
if err = meta.Delete(admissionClockPendingKey); err != nil {
return err
}
if err = meta.Put(admissionClockKey, data); err != nil {
return err
}
if elapsed, err = meterCeiling(tx, t); err != nil {
return err
}
if err = pruneOutcomes(tx, t.Now); err != nil {
return err
}
if err = sweepEffects(tx, t); err != nil {
return err
}
retireAfter, err = l.meterStorage(tx, t)
return err
})
if err != nil {
l.current = false
return admission.ClockTick{}, err
}
l.now, l.elapsed, l.current, l.degraded = tick.Now, elapsed, true, tick.Degraded
l.retireAfter = retireAfter
return tick, nil
}
// clock is the admission time for calls that admit or dispatch work. It
// needs a reading recorded through this handle.
func (l *AdmissionLedger) clock() (time.Time, error) {
if !l.current {
return time.Time{}, refusal(admission.ReasonEngineUnavailable, "admission clock has no current reading")
}
return l.now, nil
}
// recordedClock is the stored high-water mark. It is enough to record the
// outcome of work that is already running.
func (l *AdmissionLedger) recordedClock() (time.Time, error) {
if l.now.IsZero() {
return time.Time{}, refusal(admission.ReasonEngineUnavailable, "admission clock has no reading")
}
return l.now, nil
}
// RefreshInventory folds one complete observation into the persisted
// generation tracker and publishes the new inventory only after the
// transaction commits. A failed read or transaction keeps the old one.
func (l *AdmissionLedger) RefreshInventory(obs admission.InventoryObservation) error {
if obs.AmbiguousDomains < 0 {
return refusal(admission.ReasonInvalid, "ambiguous domain count is negative")
}
l.mu.Lock()
defer l.mu.Unlock()
var next ledgerInventory
err := l.update("inventory", func(tx *bolt.Tx) error {
meta := tx.Bucket([]byte(admissionMetaBucket))
if _, err := loadLedgerInventory(meta); err != nil {
return err
}
g, err := loadLedgerTracker(meta)
if err != nil {
return err
}
gens, err := g.Observe(obs.Accounts, obs.Incarnations)
if err != nil {
return refusal(admission.ReasonInvalid, "inventory observation is malformed")
}
inv, err := admission.NewInventory(gens, obs.Domains)
if err != nil {
return refusal(admission.ReasonInvalid, "inventory observation is malformed")
}
tracker, err := g.MarshalBinary()
if err != nil {
return err
}
snapshot, err := inv.MarshalBinary()
if err != nil {
return err
}
if err = meta.Put(admissionTrackerKey, tracker); err != nil {
return err
}
if err = meta.Put(admissionInventoryKey, snapshot); err != nil {
return err
}
next = ledgerInventory{inv: inv, ambiguous: obs.AmbiguousDomains}
if err = meta.Put(admissionAmbiguousKey, strconv.AppendInt(nil, int64(obs.AmbiguousDomains), 10)); err != nil {
return err
}
// Queued candidates whose roots name an account that is no longer
// current end with the inventory change that retired it.
q, err := openQueueWith(tx, l.reg, inv, l.now)
if err != nil {
return err
}
q.refused = &l.refused
if err = q.checkOwners(); err != nil {
return err
}
return q.flush()
})
if err != nil {
return err
}
l.inv.Store(&next)
return nil
}
// Inventory is the last committed inventory.
func (l *AdmissionLedger) Inventory() *admission.Inventory { return l.inv.Load().inv }
// AmbiguousDomains is the count from the last committed observation.
func (l *AdmissionLedger) AmbiguousDomains() int { return l.inv.Load().ambiguous }
// Ledger features. featureLive marks a ledger whose live start ran: a
// build that predates execution must not open it.
const (
featureLive uint64 = 1 << 0
knownFeatures = featureLive
)
// loadFeatures reads the feature bits, zero when none are recorded. A bit
// this binary does not know means a newer layout.
func loadFeatures(tx *bolt.Tx) (uint64, error) {
raw := tx.Bucket([]byte(admissionMetaBucket)).Get(admissionFeaturesKey)
if raw == nil {
return 0, nil
}
if len(raw) != 8 {
return 0, admission.ErrCorruptRecord
}
bits := binary.BigEndian.Uint64(raw)
if bits&^knownFeatures != 0 {
return 0, ErrAdmissionSchema
}
return bits, nil
}
// addFeature records bit beside the features already set.
func addFeature(tx *bolt.Tx, bit uint64) error {
bits, err := loadFeatures(tx)
if err != nil {
return err
}
return tx.Bucket([]byte(admissionMetaBucket)).Put(admissionFeaturesKey, binary.BigEndian.AppendUint64(nil, bits|bit))
}
// validateStateKeys refuses state from a layout this binary does not know.
// Record decoders separately prove the required values and missing rows.
func validateStateKeys(tx *bolt.Tx) error {
for _, state := range []struct {
name string
keys [][]byte
}{
{admissionMetaBucket, [][]byte{admissionSchemaKey, admissionClockKey, admissionClockPendingKey, admissionTrackerKey, admissionInventoryKey, admissionAmbiguousKey, admissionFeaturesKey}},
{admissionQueueStateBucket, [][]byte{queueStateKey, queueCountersKey, scheduleStateKey, ingressStateKey, ceilingStateKey, storageStateKey, episodeStateKey, liveStartKey, effectTallyKey}},
} {
b := tx.Bucket([]byte(state.name))
if b == nil {
continue
}
if err := b.ForEach(func(k, v []byte) error {
if v == nil || !slices.ContainsFunc(state.keys, func(key []byte) bool { return string(key) == string(k) }) {
return admission.ErrCorruptRecord
}
return nil
}); err != nil {
return err
}
}
return nil
}
package store
import (
"bytes"
"slices"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// futureAuditSlots is the rows an attempt in state s may still write: a
// reserved one its execution and its outcome, an executing one its outcome.
// It is the schema 5 rule, which the outbox upgrade applies.
func futureAuditSlots(s admission.State) uint64 {
switch s {
case admission.StateReserved:
return 2
case admission.StateExecuting:
return 1
}
return 0
}
// heldAuditSlots is the rows attempt a may still write: those of
// futureAuditSlots, and for an unknown outcome not yet retired, the row that
// retires it.
func heldAuditSlots(a admission.AttemptRecord) uint64 {
if a.State == admission.StateUnknown && !a.Retired {
return 1
}
return futureAuditSlots(a.State)
}
// auditStepMatches proves the row's intent and phase against its retained
// records. Retry support can grow, but roots paid for by an attempt stay.
func auditStepMatches(row admission.AuditRow, a admission.AttemptRecord, c admission.Candidate, now time.Time) bool {
if row.Attempt != a.Attempt || row.Lane != a.Lane || !row.ExpiresAt.Equal(a.ExpiresAt) ||
row.Kind != c.Key.Kind || row.Target != c.Key.Target || row.Check != c.Check || row.FindingID != c.FindingID ||
row.At.Before(a.Reserved) || row.At.After(now) {
return false
}
for _, root := range row.Roots {
if !slices.Contains(c.Roots, root) {
return false
}
}
switch {
case row.Disposition == admission.DispositionExpiredUnproven:
return a.State == admission.StateUnknown && a.Retired && !row.At.Before(a.ExpiresAt) && !row.At.Before(a.Finished)
case row.State == admission.StateReserved:
return row.At.Equal(a.Reserved)
case row.State == admission.StateExecuting:
// A preview never ran: no execution row can belong to it.
return a.State != admission.StateReserved && a.State != admission.StateObserved && row.At.Before(a.ExpiresAt) &&
(a.Finished.IsZero() || !row.At.After(a.Finished))
default:
return row.State == a.State && row.Disposition == a.Disposition && row.At.Equal(a.Finished)
}
}
// auditFollows allows acknowledged phases to be absent, but remaining rows
// must describe one unchanged intent and consecutive lifecycle transitions.
func auditFollows(previous, row admission.AuditRow) bool {
if row.At.Before(previous.At) || row.Tier != previous.Tier || !slices.Equal(row.Roots, previous.Roots) {
return false
}
steps := row.Transition - previous.Transition
switch previous.State {
case admission.StateReserved:
switch row.State {
case admission.StateExecuting, admission.StateObserved:
return steps == 1
case admission.StateFailed:
return steps == 1 || steps == 2
case admission.StateVerified, admission.StateUnknown:
return steps == 2
}
case admission.StateExecuting:
return row.State.Terminal() && steps == 1
}
return false
}
// writeAuditRow stores the row of the transition c is recording, to
// attempt a's state; the reservation already holds its slot. c is the
// candidate as the step leaves it, before its transition is counted.
func (q *queueTx) writeAuditRow(c admission.Candidate, a admission.AttemptRecord, tier admission.Tier) error {
c.Transitions++
row, err := admission.NewAuditRow(c, a, tier, q.now)
if err != nil {
return err
}
data, err := row.MarshalBinary()
if err != nil {
return err
}
outbox := q.tx.Bucket([]byte(admissionOutboxBucket))
if outbox.Get(row.Key()) != nil {
return admission.ErrCorruptRecord
}
return outbox.Put(row.Key(), data)
}
// adjustAuditSlots holds or returns audit slots in the transaction's
// storage state.
func (q *queueTx) adjustAuditSlots(hold, release uint64) error {
// A step that changes no slot needs no storage state: the outcome of
// running work is recorded even when that record is damaged.
if hold == 0 && release == 0 {
return nil
}
s, err := q.storageState()
if err != nil {
return err
}
next, err := s.HoldAudit(hold)
if err != nil {
return err
}
if next, err = next.ReleaseAudit(release); err != nil {
return err
}
*s, q.storageDirty = next, true
return nil
}
// auditPending reports whether any attempt of candidate id still has an
// unacknowledged row. The upgrade to schema 4 runs before the outbox
// exists, when no row can be pending.
func auditPending(tx *bolt.Tx, id admission.CandidateID) (bool, error) {
outbox := tx.Bucket([]byte(admissionOutboxBucket))
if outbox == nil {
return false, nil
}
cur := outbox.Cursor()
for seq := uint32(1); seq <= admission.MaxAttempts; seq++ {
a, err := admission.NewAttempt(id, seq)
if err != nil {
return false, err
}
prefix := admission.AuditPrefix(a.ID)
if k, _ := cur.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix) {
return true, nil
}
}
return false, nil
}
// PendingAudit returns up to limit unacknowledged audit rows in key order.
// The consumer delivers them idempotently by their ID and time and
// acknowledges each with its Ack through AckAudit.
func (l *AdmissionLedger) PendingAudit(limit int) ([]admission.AuditRow, error) {
var rows []admission.AuditRow
err := l.db.bolt.View(func(tx *bolt.Tx) error {
cur := tx.Bucket([]byte(admissionOutboxBucket)).Cursor()
for k, v := cur.Seek([]byte{'a'}); k != nil && k[0] == 'a' && len(rows) < limit; k, v = cur.Next() {
row, err := admission.UnmarshalAuditRow(v)
if err != nil {
return err
}
if !bytes.Equal(row.Key(), k) {
return admission.ErrCorruptRecord
}
rows = append(rows, row)
}
return nil
})
if err != nil {
return nil, err
}
return rows, nil
}
// AckAudit removes delivered rows and returns their slots in one
// transaction. A row already acknowledged, or one written at another time
// than its acknowledgement names, is skipped. Once an ended candidate's
// last row is acknowledged, its history may be retired: the same
// transaction writes its retirement keys.
func (l *AdmissionLedger) AckAudit(acks []admission.AuditAck) error {
for _, ack := range acks {
if _, err := admission.ParseActionID(string(ack.ID.Action)); err != nil || ack.ID.Transition == 0 {
return refusal(admission.ReasonInvalid, "audit acknowledgement names no row")
}
}
l.mu.Lock()
defer l.mu.Unlock()
return l.update("audit", func(tx *bolt.Tx) error {
q, err := l.openQueue(tx, l.now)
if err != nil {
return err
}
outbox := tx.Bucket([]byte(admissionOutboxBucket))
touched := map[admission.CandidateID]bool{}
var acked uint64
for _, ack := range acks {
raw := outbox.Get(ack.ID.Key())
if raw == nil {
continue
}
row, decodeErr := admission.UnmarshalAuditRow(raw)
if decodeErr != nil || row.ID() != ack.ID {
return admission.ErrCorruptRecord
}
if !row.At.Equal(ack.At) {
continue
}
if err = outbox.Delete(ack.ID.Key()); err != nil {
return err
}
touched[row.Attempt.Candidate] = true
acked++
}
if acked == 0 {
return nil
}
if err = q.adjustAuditSlots(0, acked); err != nil {
return err
}
for id := range touched {
pending, err := auditPending(tx, id)
if err != nil {
return err
}
if pending {
continue
}
h, found, err := loadHistoryEntry(tx, id)
if err != nil {
return err
}
if !found {
return admission.ErrCorruptRecord
}
if h.Released {
c, loadErr := loadCandidate(tx, id)
if loadErr != nil {
return loadErr
}
if err = proveReleasedHistory(tx, c, h); err != nil {
return err
}
}
if err = putHistoryEntry(tx, id, h); err != nil {
return err
}
}
return q.flush()
})
}
package store
import (
"errors"
"slices"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// errCandidateMissing is the refusal for a candidate the ledger does not
// hold. It is one value, so callers can tell it from other refusals.
var errCandidateMissing error = &admission.Error{Reason: admission.ReasonInvalid, Detail: "candidate is not queued"}
func loadCandidate(tx *bolt.Tx, id admission.CandidateID) (admission.Candidate, error) {
if _, err := admission.ParseCandidateID(string(id)); err != nil {
return admission.Candidate{}, err
}
raw := tx.Bucket([]byte(admissionCandidatesBucket)).Get([]byte(id))
if raw == nil {
return admission.Candidate{}, errCandidateMissing
}
c, err := admission.UnmarshalCandidate(raw)
if err != nil {
return admission.Candidate{}, err
}
if got, _ := c.ID(); got != id {
return admission.Candidate{}, admission.ErrCorruptRecord
}
return c, nil
}
func putCandidate(tx *bolt.Tx, c admission.Candidate) error {
id, err := c.ID()
if err != nil {
return err
}
data, err := c.MarshalBinary()
if err != nil {
return err
}
// A dry-run ending or a panel allow is unknown to schema 8 readers of
// the candidate and of the counts and notices written with it, and a
// disabled refusal to schema 9 readers.
switch {
case c.Reason == admission.ReasonDisabled:
err = raiseSchema(tx, admissionSchemaDisabled)
case c.State == admission.StateDryRun || c.Reason == admission.ReasonVerdictAllow:
err = raiseSchema(tx, admissionSchemaShadow)
}
if err != nil {
return err
}
return tx.Bucket([]byte(admissionCandidatesBucket)).Put([]byte(id), data)
}
// rootSet returns the request's roots sorted and unique, bounded by MaxRoots.
func rootSet(req admission.CandidateRequest) ([]admission.EvidenceID, error) {
if req.PreviewTTL < 0 {
return nil, refusal(admission.ReasonInvalid, "response lifetime is negative")
}
if len(req.Support) >= admission.MaxRoots {
return nil, refusal(admission.ReasonInvalid, "candidate has too many roots")
}
ids := append([]admission.EvidenceID{req.Primary}, req.Support...)
for _, id := range ids {
if _, err := admission.ParseEvidenceID(string(id)); err != nil {
return nil, err
}
}
slices.Sort(ids)
ids = slices.Compact(ids)
return ids, nil
}
// mergeRoots adds new roots to the existing ones while room remains; the
// candidate keeps its original roots and never grows past MaxRoots.
func mergeRoots(have, add []admission.EvidenceID) []admission.EvidenceID {
out := slices.Clone(have)
for _, id := range add {
if len(out) >= admission.MaxRoots {
break
}
if !slices.Contains(out, id) {
out = append(out, id)
}
}
slices.Sort(out)
return out
}
// scopeOwner assigns an account only when every root names the same current
// account generation. Any host or mixed-account root keeps the host scope.
func scopeOwner(inv *admission.Inventory, roots []admission.Evidence) (admission.Owner, error) {
owner, several, unknown := admission.HostOwner(), false, false
for _, e := range roots {
o := e.Owner()
if o.IsHost() {
unknown = true
continue
}
if !inv.Current(o) {
return admission.HostOwner(), refusal(admission.ReasonStaleIdentity, "root evidence names an account that is no longer current")
}
if !owner.IsHost() && owner != o {
several = true
}
owner = o
}
if several || unknown {
return admission.HostOwner(), nil
}
return owner, nil
}
// Enqueue queues a candidate built from published evidence. A request for a
// queued candidate coalesces its new roots without extending queue age,
// freshness or history. A reserved or executing candidate only acknowledges
// roots it already holds: its attempt's evidence is frozen until the attempt
// ends. A terminal candidate is never revived. Enqueue takes the request's
// episode as given and keeps no episode row: production queues only
// through EnqueueGroup, where the ledger assigns episodes.
func (l *AdmissionLedger) Enqueue(req admission.CandidateRequest) (admission.Candidate, bool, error) {
ids, err := rootSet(req)
if err != nil {
return admission.Candidate{}, false, err
}
key := admission.CandidateKey{Kind: req.Kind, Target: req.Target, Episode: req.Episode, Generation: req.Generation}
id, err := key.ID()
if err != nil {
return admission.Candidate{}, false, err
}
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return admission.Candidate{}, false, err
}
var out admission.Candidate
var created bool
if err := l.update("enqueue", func(tx *bolt.Tx) error {
q, txErr := l.openQueue(tx, now)
if txErr != nil {
return txErr
}
if out, created, txErr = l.enqueueTx(q, req, key, id, ids); txErr != nil {
return txErr
}
return q.flush()
}); err != nil {
return admission.Candidate{}, false, err
}
return out, created, nil
}
// enqueueTx queues or coalesces one request within q's transaction. The
// caller flushes q once its transaction's work is done.
func (l *AdmissionLedger) enqueueTx(q *queueTx, req admission.CandidateRequest, key admission.CandidateKey, id admission.CandidateID, ids []admission.EvidenceID) (admission.Candidate, bool, error) {
tx, now := q.tx, q.now
primary, err := loadEvidence(tx, l.reg, req.Primary)
if err != nil {
return admission.Candidate{}, false, err
}
if err = admission.BindTarget(req.Target, req.Entry, primary); err != nil {
return admission.Candidate{}, false, err
}
if req.Entry != 0 {
if err = l.reg.ValidateVia(req.Entry, primary); err != nil {
return admission.Candidate{}, false, err
}
}
cur, err := loadCandidate(tx, id)
switch {
case err == nil:
return l.coalesceTx(q, cur, ids)
case !errors.Is(err, errCandidateMissing):
return admission.Candidate{}, false, err
}
// Candidates whose deadlines passed give up their positions before a
// new one is placed.
if err = q.sweep(); err != nil {
return admission.Candidate{}, false, err
}
roots, err := loadRoots(tx, l.reg, ids)
if err != nil {
return admission.Candidate{}, false, err
}
assessment, err := admission.Assess(key.Target, roots, now)
if err != nil {
return admission.Candidate{}, false, err
}
owner, err := scopeOwner(l.Inventory(), roots)
if err != nil {
return admission.Candidate{}, false, err
}
entry := primary.Entry()
if req.Entry != 0 {
entry = req.Entry
}
c := admission.Candidate{
Key: key,
Scope: admission.Scope{Owner: owner, Effect: key.Kind.Effect()},
Entry: entry,
PreviewTTL: req.PreviewTTL,
Check: primary.Check(),
FindingID: primary.FindingID(),
Roots: ids,
FirstQueued: now,
AgeOut: minTime(now.Add(admission.QueueAgeLimit), assessment.EvidenceExpiry),
State: admission.StateQueued,
Transitions: 1,
}
if err = q.insert(id, c, entryFor(assessment)); err != nil {
return admission.Candidate{}, false, err
}
for _, root := range ids {
if err = q.name(root); err != nil {
return admission.Candidate{}, false, err
}
}
return c, true, putCandidate(tx, c)
}
// coalesceTx revalidates a repeated request against current policy and
// inventory, then adds its new roots to a queued candidate, rescopes it and
// records its new assessment. The request alone and the merged set must
// both assess for the target; queue age and age-out never move. An
// in-flight candidate's roots are frozen, so it only acknowledges roots it
// already holds.
func (l *AdmissionLedger) coalesceTx(q *queueTx, cur admission.Candidate, ids []admission.EvidenceID) (admission.Candidate, bool, error) {
tx, now := q.tx, q.now
if cur.State.Terminal() {
return admission.Candidate{}, false, admission.ErrCandidateTerminal
}
if cur.Attempts > 0 {
if _, err := currentAttempt(tx, cur); err != nil {
return admission.Candidate{}, false, err
}
}
if cur.State != admission.StateQueued {
for _, id := range ids {
if !slices.Contains(cur.Roots, id) {
return admission.Candidate{}, false, admission.ErrTransitionConflict
}
}
}
// Validate the bounded request even when no extra roots will fit.
supplied, err := loadRoots(tx, l.reg, ids)
if err != nil {
return admission.Candidate{}, false, err
}
if _, err = admission.Assess(cur.Key.Target, supplied, now); err != nil {
return admission.Candidate{}, false, err
}
if _, err = scopeOwner(l.Inventory(), supplied); err != nil {
return admission.Candidate{}, false, err
}
merged := mergeRoots(cur.Roots, ids)
roots, err := loadRoots(tx, l.reg, merged)
if err != nil {
return admission.Candidate{}, false, err
}
if err = validateCandidateEntry(l.reg, cur, roots); err != nil {
return admission.Candidate{}, false, err
}
if _, err = admission.Assess(cur.Key.Target, roots, now); err != nil {
return admission.Candidate{}, false, err
}
owner, err := scopeOwner(l.Inventory(), roots)
if err != nil {
return admission.Candidate{}, false, err
}
if cur.State != admission.StateQueued {
return cur, false, nil
}
if !now.Before(cur.AgeOut) {
return admission.Candidate{}, false, refusal(admission.ReasonStale, "candidate aged out of the queue")
}
if slices.Equal(merged, cur.Roots) && owner == cur.Scope.Owner {
return cur, false, nil
}
id, _ := cur.ID()
entry, err := loadQueueEntry(tx, id)
if err != nil {
return admission.Candidate{}, false, err
}
if err = q.remapHistoryRoots(cur, merged); err != nil {
return admission.Candidate{}, false, err
}
for _, root := range merged {
if !slices.Contains(cur.Roots, root) {
if err = q.name(root); err != nil {
return admission.Candidate{}, false, err
}
}
}
cur.Roots, cur.Scope.Owner = merged, owner
cur.Transitions++
if err = putCandidate(tx, cur); err != nil {
return admission.Candidate{}, false, err
}
// Mark the old assessment due inside this transaction. The sweep must
// see merged support before any candidate chooses a displacement victim.
if entry.Assessed() {
entry.NextChange = now
}
if err = putQueueEntry(tx, id, entry); err != nil {
return admission.Candidate{}, false, err
}
q.noteDeadlines(cur, entry)
if err = q.sweep(); err != nil {
return admission.Candidate{}, false, err
}
stored, err := loadCandidate(tx, id)
return stored, false, err
}
func loadRoots(tx *bolt.Tx, reg *admission.Registry, ids []admission.EvidenceID) ([]admission.Evidence, error) {
roots := make([]admission.Evidence, 0, len(ids))
for _, id := range ids {
e, err := loadEvidence(tx, reg, id)
if err != nil {
return nil, err
}
roots = append(roots, e)
}
return roots, nil
}
// validateCandidateEntry binds the retained entry and target to a root of
// the primary check, not whichever supporting root sorts first.
func validateCandidateEntry(reg *admission.Registry, c admission.Candidate, roots []admission.Evidence) error {
for _, e := range roots {
if e.Check() == c.Check {
if err := admission.BindTarget(c.Key.Target, c.Entry, e); err != nil {
return err
}
return reg.ValidateVia(c.Entry, e)
}
}
return admission.ErrCorruptRecord
}
// Candidate loads a candidate record.
func (l *AdmissionLedger) Candidate(id admission.CandidateID) (admission.Candidate, error) {
var c admission.Candidate
err := l.db.bolt.View(func(tx *bolt.Tx) error {
var err error
c, err = loadCandidate(tx, id)
return err
})
return c, err
}
func minTime(a, b time.Time) time.Time {
if b.Before(a) {
return b
}
return a
}
package store
import (
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
var ceilingStateKey = []byte("ceiling")
// upgradeLedgerToSchemaThree adds the ceiling to a schema 2 ledger inside
// the opening transaction. Its recent spend is unknown, so its buckets start
// empty rather than full; no charge is invented for earlier reservations.
// The upgrade that completes the chain records the schema.
func upgradeLedgerToSchemaThree(tx *bolt.Tx) error {
state := tx.Bucket([]byte(admissionQueueStateBucket))
if state.Get(ceilingStateKey) != nil || state.Bucket(ceilingStateKey) != nil {
return admission.ErrCorruptRecord
}
if _, err := tx.CreateBucket([]byte(admissionChargesBucket)); err != nil {
return err
}
return putCeilingState(tx, admission.CeilingState{})
}
func loadCeilingState(tx *bolt.Tx) (admission.CeilingState, error) {
raw := tx.Bucket([]byte(admissionQueueStateBucket)).Get(ceilingStateKey)
if raw == nil {
return admission.CeilingState{}, admission.ErrCorruptRecord
}
return admission.UnmarshalCeilingState(raw)
}
func putCeilingState(tx *bolt.Tx, s admission.CeilingState) error {
data, err := s.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionQueueStateBucket)).Put(ceilingStateKey, data)
}
// loadCeiling loads the ceiling and proves it against the retained charges:
// they must add up to each lane's recorded usage, and none can have been
// spent after the ceiling's own elapsed time.
func loadCeiling(tx *bolt.Tx) (admission.CeilingState, error) {
s, err := loadCeilingState(tx)
if err != nil {
return s, err
}
var general, reserved uint64
err = tx.Bucket([]byte(admissionChargesBucket)).ForEach(func(k, v []byte) error {
c, decodeErr := admission.UnmarshalCharge(k, v)
if decodeErr != nil {
return decodeErr
}
if c.Elapsed > s.Elapsed {
return admission.ErrCorruptRecord
}
if c.Lane == admission.LaneGeneral {
general += uint64(c.Cost)
} else {
reserved += uint64(c.Cost)
}
return nil
})
if err != nil {
return s, err
}
if general != uint64(s.General.Used) || reserved != uint64(s.Reserved.Used) {
return s, admission.ErrCorruptRecord
}
return s, nil
}
// meterCeiling credits a tick's elapsed time to the ceiling and releases the
// charges that have left the window, oldest first. Keys order charges by
// time and the ledger spends them at non-decreasing elapsed time, so the
// walk stops at the first charge still counted. Stopping can only keep a
// later charge longer, never release one early.
func meterCeiling(tx *bolt.Tx, tick admission.ClockTick) (time.Duration, error) {
s, err := loadCeiling(tx)
if err != nil {
return 0, err
}
if s, err = s.Advance(tick.Elapsed); err != nil {
return 0, err
}
var released [][]byte
cur := tx.Bucket([]byte(admissionChargesBucket)).Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
c, decodeErr := admission.UnmarshalCharge(k, v)
if decodeErr != nil {
return 0, decodeErr
}
if !c.Releasable(tick.Now, s.Elapsed) {
break
}
if s, err = s.Release(c.Lane, c.Cost); err != nil {
return 0, err
}
released = append(released, k)
}
for _, k := range released {
if err = tx.Bucket([]byte(admissionChargesBucket)).Delete(k); err != nil {
return 0, err
}
}
return s.Elapsed, putCeilingState(tx, s)
}
// SetCeiling records the effective hourly ceiling. The first limit of a new
// ledger fills each bucket to its cap; every later one, including the same
// limit after a restart, only clips saved credit to the new caps.
func (l *AdmissionLedger) SetCeiling(limit uint32) error {
l.mu.Lock()
defer l.mu.Unlock()
return l.update("ceiling", func(tx *bolt.Tx) error {
s, err := loadCeilingState(tx)
if err != nil {
return err
}
if s, err = s.SetLimit(limit); err != nil {
return err
}
return putCeilingState(tx, s)
})
}
// ImportLegacySpend takes a new ledger's first limit together with the
// legacy hourly counter's spend, in one transaction, so switching counters
// does not grant a fresh allowance for blocks already charged.
// A ledger that already has a limit refuses: its import is done, and a
// restart or rerun cannot create fresh credit.
func (l *AdmissionLedger) ImportLegacySpend(limit uint32, spend admission.LegacySpend) error {
l.mu.Lock()
defer l.mu.Unlock()
return l.update("ceiling", func(tx *bolt.Tx) error {
s, err := loadCeilingState(tx)
if err != nil {
return err
}
next, charges, err := s.Import(limit, spend)
if err != nil {
return err
}
if err = putCharges(tx, charges); err != nil {
return err
}
return putCeilingState(tx, next)
})
}
// putCharges stores new charges. An existing key is damage: overwriting
// it would leave usage the retained charges no longer prove.
func putCharges(tx *bolt.Tx, charges []admission.Charge) error {
bucket := tx.Bucket([]byte(admissionChargesBucket))
for _, c := range charges {
key, err := c.Key()
if err != nil {
return err
}
data, err := c.MarshalBinary()
if err != nil {
return err
}
if bucket.Get(key) != nil {
return admission.ErrCorruptRecord
}
if err = bucket.Put(key, data); err != nil {
return err
}
}
return nil
}
// Ceiling is the committed ceiling state.
func (l *AdmissionLedger) Ceiling() (admission.CeilingState, error) {
var s admission.CeilingState
err := l.db.bolt.View(func(tx *bolt.Tx) error {
var err error
s, err = loadCeilingState(tx)
return err
})
return s, err
}
// CeilingSnapshot is the committed ceiling with the charges it retains, in
// key order, read in one transaction and proven against each other. A
// dry-run forecast starts from it.
func (l *AdmissionLedger) CeilingSnapshot() (admission.CeilingState, []admission.Charge, error) {
var s admission.CeilingState
var charges []admission.Charge
err := l.db.bolt.View(func(tx *bolt.Tx) error {
var err error
if s, err = loadCeiling(tx); err != nil {
return err
}
charges, err = loadCharges(tx)
return err
})
if err != nil {
return admission.CeilingState{}, nil, err
}
return s, charges, nil
}
// ForecastSeed is the ceiling and the retained charges a forecast session
// starts from: the committed state without the charges of observe previews,
// which are comparison work, not responses. Their units leave the window
// and their credit returns, up to the bucket's cap. Charges of the imported
// legacy hour and of genuine attempts stay. The committed ceiling is not
// changed.
func (l *AdmissionLedger) ForecastSeed() (admission.CeilingState, []admission.Charge, error) {
var s admission.CeilingState
var kept []admission.Charge
err := l.db.bolt.View(func(tx *bolt.Tx) error {
var err error
if s, err = loadCeiling(tx); err != nil {
return err
}
charges, err := loadCharges(tx)
if err != nil {
return err
}
attempts := tx.Bucket([]byte(admissionAttemptsBucket))
for _, c := range charges {
// A legacy hour's charge names no recorded attempt.
if attempts.Get([]byte(c.Action)) != nil {
a, decodeErr := loadAttempt(tx, c.Action)
if decodeErr != nil {
return decodeErr
}
candidate, candidateErr := loadCandidate(tx, a.Attempt.Candidate)
if candidateErr != nil || !c.At.Equal(a.Reserved) || c.Lane != a.Lane || c.Cost != candidate.Key.Kind.CeilingCost() {
return admission.ErrCorruptRecord
}
if a.State == admission.StateObserved {
if s, err = s.Uncharge(c.Lane, c.Cost); err != nil {
return err
}
continue
}
}
kept = append(kept, c)
}
return nil
})
if err != nil {
return admission.CeilingState{}, nil, err
}
return s, kept, nil
}
// laneFits rechecks a queued candidate before reservation because its pick
// was made in an earlier transaction. Any qualifying candidate may use the
// general lane; the direct and corroborated lanes also require their
// respective assessments.
func laneFits(q *queueTx, lc liveCandidate, lane admission.Lane) error {
if !lane.Valid() {
return refusal(admission.ReasonInvalid, "reservation names no lane")
}
a, reason, cause, err := q.check(lc, true)
switch {
case err != nil:
return err
case reason != 0:
return &admission.Error{Reason: reason, Cause: cause, Detail: "candidate no longer qualifies for a response"}
case lane == admission.LaneDirect && !a.DirectC3, lane == admission.LaneCorroborated && !a.Corroborated:
return admission.ErrLaneIneligible
}
return nil
}
// chargeTx spends cost units of lane's ceiling budget for an attempt and
// records the charge, in the reservation's own transaction.
func chargeTx(tx *bolt.Tx, now time.Time, action admission.ActionID, lane admission.Lane, cost uint32) error {
s, err := loadCeilingState(tx)
if err != nil {
return err
}
if s, err = s.Charge(lane, cost); err != nil {
return err
}
c := admission.Charge{At: now, Action: action, Lane: lane, Cost: cost, Elapsed: s.Elapsed}
key, err := c.Key()
if err != nil {
return err
}
data, err := c.MarshalBinary()
if err != nil {
return err
}
charges := tx.Bucket([]byte(admissionChargesBucket))
// Overwriting a stored charge would leave usage the retained charges no
// longer prove, and the next tick would refuse it.
if charges.Get(key) != nil {
return admission.ErrCorruptRecord
}
if err = charges.Put(key, data); err != nil {
return err
}
return putCeilingState(tx, s)
}
// loadCharges is every retained charge in key order, oldest first.
func loadCharges(tx *bolt.Tx) ([]admission.Charge, error) {
var out []admission.Charge
err := tx.Bucket([]byte(admissionChargesBucket)).ForEach(func(k, v []byte) error {
c, err := admission.UnmarshalCharge(k, v)
if err != nil {
return err
}
out = append(out, c)
return nil
})
return out, err
}
// ImportedLegacySpend reads the retained legacy charges for startup status.
// A charge of a retained attempt must match it. A charge whose attempt has
// retired is no legacy spend: after a reboot a charge waits for its
// remaining elapsed time, while history retires by wall time.
func (l *AdmissionLedger) ImportedLegacySpend() (admission.LegacySpend, error) {
var spend admission.LegacySpend
err := l.db.bolt.View(func(tx *bolt.Tx) error {
if _, err := loadCeiling(tx); err != nil {
return err
}
charges, err := loadCharges(tx)
if err != nil {
return err
}
var idsAt time.Time
ids := make(map[admission.ActionID]bool)
for _, c := range charges {
if tx.Bucket([]byte(admissionAttemptsBucket)).Get([]byte(c.Action)) != nil {
a, err := loadAttempt(tx, c.Action)
if err != nil {
return corruptRecord(err)
}
candidate, err := loadCandidate(tx, a.Attempt.Candidate)
if err != nil || !c.At.Equal(a.Reserved) || c.Lane != a.Lane || c.Cost != candidate.Key.Kind.CeilingCost() {
return admission.ErrCorruptRecord
}
continue
}
if !c.At.Equal(idsAt) {
// Charges are time-ordered; prior timestamps never recur.
clear(ids)
for seq := uint32(1); seq <= (admission.MaxCeiling+admission.MaxMemberCost-1)/admission.MaxMemberCost+1; seq++ {
ids[admission.LegacyActionID(c.At, seq)] = true
}
idsAt = c.At
}
if !ids[c.Action] {
continue
}
// One import charged every legacy unit at one time.
if spend.Units != 0 && !spend.At.Equal(c.At) {
return admission.ErrCorruptRecord
}
spend.At = c.At
spend.Units += c.Cost
}
return nil
})
if err != nil {
return admission.LegacySpend{}, err
}
return spend, err
}
package store
import (
"errors"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// errNothingDisabled rolls back a refusal pass that ended nothing, so a
// turn whose policy allows its whole queue commits no write.
var errNothingDisabled = errors.New("no queued response is disabled")
// RefuseDisabled ends queued responses the current policy forbids before
// capacity scheduling. Like queue revalidation, it visits at most the
// durable queue bound. It cannot touch in-flight attempts or let disabled
// work consume the turns or capacity of allowed work. All endings commit
// together, including their counters and schema marker; a pass that ends
// nothing commits nothing.
// allows may load a candidate's roots through its second argument, which
// returns nil if they cannot be read. Both callbacks run synchronously
// under the ledger lock and write transaction; neither may reenter the
// ledger or retain the loader after allows returns.
func (l *AdmissionLedger) RefuseDisabled(allows func(admission.Candidate, func() []admission.Evidence) bool) error {
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return err
}
updateErr := l.update("disabled", func(tx *bolt.Tx) error {
q, err := l.openQueue(tx, now)
if err != nil {
return err
}
live, err := q.live()
if err != nil {
return err
}
ended := false
for _, lc := range live {
roots := func() []admission.Evidence {
e, loadErr := loadRoots(tx, l.reg, lc.c.Roots)
if loadErr != nil {
return nil
}
return e
}
if lc.c.State == admission.StateQueued && !allows(lc.c, roots) {
if err = q.refuse(lc, admission.ReasonDisabled, 0); err != nil {
return err
}
ended = true
}
}
if !ended {
return errNothingDisabled
}
return q.flush()
})
if errors.Is(updateErr, errNothingDisabled) {
return nil
}
return updateErr
}
package store
import (
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
const (
// admissionEffectsBucket holds one record per automatic block that
// holds a temporary set slot, keyed by expiry.
admissionEffectsBucket = "adm:effects"
// admissionSchemaEffects is the schema of a ledger holding effect
// records: their bucket and their tally. The first record writes it;
// older builds refuse the ledger rather than forget the slots its
// blocks hold.
admissionSchemaEffects = 12
// admissionSchemaEffectGone is the schema of a ledger whose effect
// records may be gone: older builds would read a gone record as one
// that still holds its slot.
admissionSchemaEffectGone = 14
// admissionSchemaNewest is the newest schema this build reads.
admissionSchemaNewest = admissionSchemaEpisodeRetained
)
// effectTallyKey holds the effect records' tally in the queue state bucket.
var effectTallyKey = []byte("effects")
// effectLayout reports whether the effect records exist and refuses a
// layout schema rules out: none below 12, bucket and tally at 12, optional
// at 13, and required again at 14. At 13 and 14 they need the live start.
func effectLayout(tx *bolt.Tx, schema byte) (bool, error) {
state := tx.Bucket([]byte(admissionQueueStateBucket))
if state == nil {
return false, admission.ErrCorruptRecord
}
present := tx.Bucket([]byte(admissionEffectsBucket)) != nil
tallied := state.Get(effectTallyKey) != nil
var ok bool
switch {
case schema < admissionSchemaEffects:
ok = !present && !tallied
case schema == admissionSchemaEffects:
ok = present && tallied
case schema == admissionSchemaEpisodeElapsed:
ok = present == tallied && (!present || state.Get(liveStartKey) != nil)
default:
ok = present && tallied && state.Get(liveStartKey) != nil
}
if !ok {
return false, admission.ErrCorruptRecord
}
return present, nil
}
// loadEffectTally is the stored tally; zero before the first record.
func loadEffectTally(tx *bolt.Tx) (admission.EffectTally, error) {
raw := tx.Bucket([]byte(admissionQueueStateBucket)).Get(effectTallyKey)
if raw == nil {
return admission.EffectTally{}, nil
}
return admission.UnmarshalEffectTally(raw)
}
func putEffectTally(tx *bolt.Tx, t admission.EffectTally) error {
data, err := t.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionQueueStateBucket)).Put(effectTallyKey, data)
}
// effectKey is the key of the record attempt a holds, if any.
func effectKey(a admission.AttemptRecord) ([]byte, error) {
return admission.EffectRecordKey(a.Attempt.ID, a.ExpiresAt)
}
// holdSlot records that reserved attempt a of block c holds a temporary
// set slot in the partition of its lane or, when it evicts, of its victim,
// which it claims in the same transaction (the slot keeps its
// partition). The first record creates the bucket and raises the schema.
func (q *queueTx) holdSlot(c admission.Candidate, a admission.AttemptRecord, e admission.QueueEntry, victim admission.EvictionVictim) error {
rec := admission.EffectRecord{
Action: a.Attempt.ID, Target: c.Key.Target, Partition: admission.PartitionOf(a.Lane), Lane: a.Lane,
Tier: e.Tier, Scope: c.Scope.Key(), Direct: e.Direct, Corroborated: e.Corroborated, ExpiresAt: a.ExpiresAt,
}
if victim != (admission.EvictionVictim{}) {
rec.Partition, rec.Victim = victim.Partition, admission.EffectRef{Action: victim.Action, ExpiresAt: victim.ExpiresAt}
// An import has no record: this record alone holds the claim.
if !rec.Victim.Imported() {
if err := q.claimVictim(rec.Victim, rec.Action); err != nil {
return err
}
}
}
key, err := rec.Key()
if err != nil {
return err
}
data, err := rec.MarshalBinary()
if err != nil {
return err
}
b, err := q.tx.CreateBucketIfNotExists([]byte(admissionEffectsBucket))
if err != nil {
return err
}
if b.Get(key) != nil {
return admission.ErrCorruptRecord
}
if err = raiseSchema(q.tx, admissionSchemaEffects); err != nil {
return err
}
if err = addEffectTally(q.tx, rec.Partition, 1); err != nil {
return err
}
return b.Put(key, data)
}
// claimVictim names block as the pending evictor of the verified,
// unclaimed record ref names.
func (q *queueTx) claimVictim(ref admission.EffectRef, block admission.ActionID) error {
key, err := ref.Key()
if err != nil {
return err
}
b := q.tx.Bucket([]byte(admissionEffectsBucket))
if b == nil {
return admission.ErrCorruptRecord
}
raw := b.Get(key)
if raw == nil {
return admission.ErrCorruptRecord
}
rec, err := admission.UnmarshalEffectRecord(key, raw)
if err != nil {
return err
}
if !rec.Verified() || !rec.Holds() || rec.EvictedBy != "" {
return admission.ErrCorruptRecord
}
rec.EvictedBy = block
data, err := rec.MarshalBinary()
if err != nil {
return err
}
return b.Put(key, data)
}
// loadEffect is the record attempt a holds; false when it holds none.
func loadEffect(tx *bolt.Tx, a admission.AttemptRecord) (admission.EffectRecord, []byte, bool, error) {
b := tx.Bucket([]byte(admissionEffectsBucket))
if b == nil {
return admission.EffectRecord{}, nil, false, nil
}
key, err := effectKey(a)
if err != nil {
return admission.EffectRecord{}, nil, false, err
}
raw := b.Get(key)
if raw == nil {
return admission.EffectRecord{}, key, false, nil
}
rec, err := admission.UnmarshalEffectRecord(key, raw)
return rec, key, err == nil, err
}
// markApplied records the verified application of attempt a's block at
// admission time, with its recorded elapsed time as the proof a minimum
// hold needs, and what it did to the victim it claimed (v): a proven
// removal evicts the victim, whose record gives up its slot and stays,
// evicted, until its expiry; a spared victim returns to the pool. An
// attempt without a record (a preview, or one reserved by an earlier
// build) holds no slot.
func (q *queueTx) markApplied(a admission.AttemptRecord, v admission.VictimOutcome, elapsed time.Duration) error {
rec, key, found, err := loadEffect(q.tx, a)
if err != nil || !found {
return err
}
if rec.Victim != (admission.EffectRef{}) {
if err = q.endClaim(rec.Victim, rec.Action, v == admission.VictimRemoved); err != nil {
return err
}
}
rec.Applied, rec.Elapsed, rec.Victim = q.now, elapsed, admission.EffectRef{}
data, err := rec.MarshalBinary()
if err != nil {
return err
}
return q.tx.Bucket([]byte(admissionEffectsBucket)).Put(key, data)
}
// releaseSlot removes the record attempt a holds, if any, and gives back
// the claim on its victim.
func (q *queueTx) releaseSlot(a admission.AttemptRecord) error {
rec, key, found, err := loadEffect(q.tx, a)
if err != nil || !found {
return err
}
if rec.Victim != (admission.EffectRef{}) {
if err = q.endClaim(rec.Victim, rec.Action, false); err != nil {
return err
}
}
return deleteEffect(q.tx, key, rec)
}
// endClaim ends block's claim on the record ref names: evicted, it gives
// up its slot; otherwise it returns to the victim pool. A victim swept at
// its own expiry needs neither, and an import has no record: the firewall
// stops listing it once it is gone.
func (q *queueTx) endClaim(ref admission.EffectRef, block admission.ActionID, evicted bool) error {
if ref.Imported() {
return nil
}
key, err := ref.Key()
if err != nil {
return err
}
b := q.tx.Bucket([]byte(admissionEffectsBucket))
raw := b.Get(key)
if raw == nil {
return nil
}
rec, err := admission.UnmarshalEffectRecord(key, raw)
if err != nil {
return err
}
if rec.EvictedBy != block || rec.Evicted {
return admission.ErrCorruptRecord
}
if evicted {
if err = addEffectTally(q.tx, rec.Partition, -1); err != nil {
return err
}
if err = raiseSchema(q.tx, admissionSchemaEffectGone); err != nil {
return err
}
rec.Evicted, rec.Gone = true, q.now
} else {
rec.EvictedBy = ""
}
data, err := rec.MarshalBinary()
if err != nil {
return err
}
return b.Put(key, data)
}
func addEffectTally(tx *bolt.Tx, p admission.Partition, delta int) error {
tally, err := loadEffectTally(tx)
if err != nil {
return err
}
if tally, err = tally.Add(p, delta); err != nil {
return err
}
return putEffectTally(tx, tally)
}
// deleteEffect removes a record; one that still holds its slot leaves the
// tally, an evicted one left it at its eviction.
func deleteEffect(tx *bolt.Tx, key []byte, rec admission.EffectRecord) error {
if rec.Holds() {
if err := addEffectTally(tx, rec.Partition, -1); err != nil {
return err
}
}
return tx.Bucket([]byte(admissionEffectsBucket)).Delete(key)
}
// claimedVictim is the victim reserved attempt a's record claimed, as an
// intent names it; zero when it claimed none, the victim's record is gone
// at its own expiry, or the firewall no longer lists the import it
// claimed among imports. Execution clears a vanished claim in the same
// transaction so a v1 intent cannot hide a still-recorded removal.
func claimedVictim(tx *bolt.Tx, a admission.AttemptRecord, imports []admission.EvictionVictim) (admission.Victim, error) {
rec, ownKey, found, err := loadEffect(tx, a)
if err != nil || !found || rec.Victim == (admission.EffectRef{}) {
return admission.Victim{}, err
}
var raw []byte
var key []byte
if rec.Victim.Imported() {
for _, v := range imports {
if v.Action == rec.Victim.Action && v.ExpiresAt.Equal(rec.Victim.ExpiresAt) {
return admission.Victim{Action: v.Action, Target: v.Target, ExpiresAt: v.ExpiresAt}, nil
}
}
} else {
if key, err = rec.Victim.Key(); err != nil {
return admission.Victim{}, err
}
raw = tx.Bucket([]byte(admissionEffectsBucket)).Get(key)
}
if raw == nil {
rec.Victim = admission.EffectRef{}
data, encodeErr := rec.MarshalBinary()
if encodeErr != nil {
return admission.Victim{}, encodeErr
}
return admission.Victim{}, tx.Bucket([]byte(admissionEffectsBucket)).Put(ownKey, data)
}
victim, err := admission.UnmarshalEffectRecord(key, raw)
if err != nil {
return admission.Victim{}, err
}
return admission.Victim{Action: victim.Action, Target: victim.Target, ExpiresAt: victim.ExpiresAt}, nil
}
// sweepEffects removes the records whose expiry has passed at a trusted
// admission time now, soonest first. A degraded reading proves no expiry,
// as for an unknown outcome's retirement, so it removes nothing.
func sweepEffects(tx *bolt.Tx, tick admission.ClockTick) error {
b := tx.Bucket([]byte(admissionEffectsBucket))
if b == nil || tick.Degraded {
return nil
}
var expired [][]byte
var recs []admission.EffectRecord
cur := b.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
rec, err := admission.UnmarshalEffectRecord(k, v)
if err != nil {
return err
}
if tick.Now.Before(rec.ExpiresAt) {
break
}
expired, recs = append(expired, k), append(recs, rec)
}
for i, k := range expired {
if recs[i].Victim != (admission.EffectRef{}) {
a, err := loadAttempt(tx, recs[i].Action)
if err != nil {
return err
}
if a.State == admission.StateReserved {
q := queueTx{tx: tx}
if err := q.endClaim(recs[i].Victim, recs[i].Action, false); err != nil {
return err
}
}
}
if err := deleteEffect(tx, k, recs[i]); err != nil {
return err
}
}
return nil
}
// proveEffects checks that effect records exist exactly when
// the schema announces them, decode, and match their tally.
func proveEffects(tx *bolt.Tx) error {
b := tx.Bucket([]byte(admissionEffectsBucket))
schema := tx.Bucket([]byte(admissionMetaBucket)).Get(admissionSchemaKey)
if len(schema) != 1 || schema[0] < admissionSchemaVersion || schema[0] > admissionSchemaNewest {
return ErrAdmissionSchema
}
if present, err := effectLayout(tx, schema[0]); err != nil || !present {
return err
}
var counted admission.EffectTally
records := map[admission.ActionID]admission.EffectRecord{}
if err := b.ForEach(func(k, v []byte) error {
rec, err := admission.UnmarshalEffectRecord(k, v)
if err != nil {
return err
}
if _, seen := records[rec.Action]; seen {
return admission.ErrCorruptRecord
}
records[rec.Action] = rec
if !rec.Gone.IsZero() && schema[0] < admissionSchemaEffectGone {
return admission.ErrCorruptRecord
}
if rec.Holds() {
counted, err = counted.Add(rec.Partition, 1)
}
return err
}); err != nil {
return err
}
if !evictionsAgree(tx, records) {
return admission.ErrCorruptRecord
}
tally, err := loadEffectTally(tx)
if err != nil {
return err
}
if tally != counted {
return admission.ErrCorruptRecord
}
return nil
}
// evictionsAgree proves each eviction from both ends while both records
// remain (either may be swept at its own expiry first): a pending block
// names a victim it claimed, a claimed victim names a pending block that
// names it back, and an evicted victim names a verified block. Retained
// attempts distinguish a swept evictor from a prematurely missing one.
func evictionsAgree(tx *bolt.Tx, records map[admission.ActionID]admission.EffectRecord) bool {
clock, err := loadLedgerClock(tx.Bucket([]byte(admissionMetaBucket)))
if err != nil {
return false
}
claimedImports := map[admission.ActionID]bool{}
for _, rec := range records {
if rec.Victim.Imported() {
// No victim record can prove the import has only one claimant.
if claimedImports[rec.Victim.Action] {
return false
}
claimedImports[rec.Victim.Action] = true
if rec.Partition != admission.PartitionGeneral || !admission.ImportTier.Less(rec.Tier) {
return false
}
} else if rec.Victim.Action != "" {
victim, ok := records[rec.Victim.Action]
if ok {
if !victim.ExpiresAt.Equal(rec.Victim.ExpiresAt) || victim.EvictedBy != rec.Action || victim.Evicted || victim.Partition != rec.Partition || !evicts(rec, victim) || victim.Target == rec.Target {
return false
}
} else if clock.Now().Before(rec.Victim.ExpiresAt) {
return false
}
}
if rec.EvictedBy == "" {
continue
}
if block, ok := records[rec.EvictedBy]; ok {
claimed := block.Victim == admission.EffectRef{Action: rec.Action, ExpiresAt: rec.ExpiresAt}
if rec.Evicted != block.Verified() || rec.Evicted == claimed || rec.Partition != block.Partition || !evicts(block, rec) || rec.Target == block.Target {
return false
}
} else if raw := tx.Bucket([]byte(admissionAttemptsBucket)).Get([]byte(rec.EvictedBy)); raw != nil {
a, decodeErr := admission.UnmarshalAttempt(raw)
if decodeErr != nil || clock.Now().Before(a.ExpiresAt) || (rec.Evicted && a.State != admission.StateVerified) || (!rec.Evicted && a.State != admission.StateExecuting && a.State != admission.StateUnknown) {
return false
}
} else if clock.Now().Before(rec.Applied.Add(admission.HistoryRetention)) {
// An evictor cannot have finished before its victim was applied.
return false
}
}
return true
}
// evicts reports whether block may take victim's slot: a strictly lower
// victim, or an equal-tier one rotation took in the partition of the
// block's own lane (rotation never crosses partitions).
func evicts(block, victim admission.EffectRecord) bool {
return victim.Tier.Less(block.Tier) || (victim.Tier == block.Tier && admission.PartitionOf(block.Lane) == victim.Partition)
}
// effectsStatus reads the slots the records hold and the expiry of the
// first one.
func effectsStatus(tx *bolt.Tx, out *admission.EffectsStatus) error {
if err := proveEffects(tx); err != nil {
return err
}
tally, err := loadEffectTally(tx)
if err != nil {
return err
}
out.General, out.Reserved = tally.General, tally.Reserved
if b := tx.Bucket([]byte(admissionEffectsBucket)); b != nil {
cur := b.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
rec, decodeErr := admission.UnmarshalEffectRecord(k, v)
if decodeErr != nil {
return decodeErr
}
if rec.Holds() {
out.Next = rec.ExpiresAt
break
}
}
}
return nil
}
// A swept evictor still owns its execution intent until finishAttempt
// deletes it. End that claim before deletion as markApplied would (v for
// a verified block); ordinary records use the markApplied/releaseSlot
// paths instead.
func (q *queueTx) endSweptClaim(a admission.AttemptRecord, verified bool, v admission.VictimOutcome) error {
_, _, found, err := loadEffect(q.tx, a)
if err != nil || found {
return err
}
in, _, err := loadIntent(q.tx, a.Attempt.ID)
if err != nil || in.Victim == (admission.Victim{}) {
return err
}
return q.endClaim(admission.EffectRef{Action: in.Victim.Action, ExpiresAt: in.Victim.ExpiresAt}, a.Attempt.ID, verified && v == admission.VictimRemoved)
}
// Both records can expire independently. While either remains it must
// agree with the intent, including the victim address. Missing records
// are accepted only where expiry or history retention permits them.
func proveIntentVictim(tx *bolt.Tx, a admission.AttemptRecord, in admission.Intent) error {
rec, _, found, err := loadEffect(tx, a)
if err != nil {
return err
}
ref := admission.EffectRef{}
if in.Victim != (admission.Victim{}) {
ref = admission.EffectRef{Action: in.Victim.Action, ExpiresAt: in.Victim.ExpiresAt}
}
if found && rec.Victim != ref {
return admission.ErrCorruptRecord
}
b := tx.Bucket([]byte(admissionEffectsBucket))
if b == nil {
if ref != (admission.EffectRef{}) {
return admission.ErrCorruptRecord
}
return nil
}
// The member may have been swept first. Its claimed victim still
// proves that the intent cannot omit or substitute the removal.
if !found {
if err = b.ForEach(func(k, v []byte) error {
victim, decodeErr := admission.UnmarshalEffectRecord(k, v)
if decodeErr != nil {
return decodeErr
}
if victim.EvictedBy == a.Attempt.ID && (victim.Evicted || ref != (admission.EffectRef{Action: victim.Action, ExpiresAt: victim.ExpiresAt})) {
return admission.ErrCorruptRecord
}
return nil
}); err != nil {
return err
}
}
if ref == (admission.EffectRef{}) || ref.Imported() {
return nil
}
key, err := ref.Key()
if err != nil {
return err
}
if raw := b.Get(key); raw != nil {
victim, decodeErr := admission.UnmarshalEffectRecord(key, raw)
if decodeErr != nil || victim.Target != in.Victim.Target || victim.EvictedBy != a.Attempt.ID || victim.Evicted {
return admission.ErrCorruptRecord
}
} else {
clock, clockErr := loadLedgerClock(tx.Bucket([]byte(admissionMetaBucket)))
if clockErr != nil || clock.Now().Before(ref.ExpiresAt) {
return admission.ErrCorruptRecord
}
if raw := tx.Bucket([]byte(admissionAttemptsBucket)).Get([]byte(ref.Action)); raw != nil {
victim, decodeErr := admission.UnmarshalAttempt(raw)
if decodeErr != nil || victim.State != admission.StateVerified || !victim.ExpiresAt.Equal(ref.ExpiresAt) {
return admission.ErrCorruptRecord
}
c, loadErr := loadCandidate(tx, victim.Attempt.Candidate)
if loadErr != nil || c.Key.Target != in.Victim.Target || c.Key.Kind != admission.KindBlockIP {
return admission.ErrCorruptRecord
}
}
}
return nil
}
package store
import (
"crypto/rand"
"io"
"slices"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// admissionSchemaEpisodeElapsed is the schema of a ledger whose episode
// rows may carry an elapsed end. Every host writes episode rows, live or
// not, so unlike schemas 11 and 12 it records nothing of the live start
// or the effect records: at 13 those prove themselves.
const admissionSchemaEpisodeElapsed = 13
// admissionSchemaEpisodeRetained permits a verified episode whose
// candidate references are gone but whose boundary still answers, and a
// block line that names its latest verified generation.
const admissionSchemaEpisodeRetained = 15
// retainedShape reports whether e needs schema 15 to be read.
func retainedShape(e admission.Episode) bool {
return !e.HasCandidates() || slices.ContainsFunc(e.Lines, func(l admission.EpisodeLine) bool { return l.VerifiedGeneration > 0 })
}
// episodeStateKey holds the episode sequence in the queue state bucket.
var episodeStateKey = []byte("episodes")
// episodeNonce is where a new episode sequence reads its nonce.
var episodeNonce io.Reader = rand.Reader
func loadEpisodeSequence(tx *bolt.Tx) (admission.EpisodeSequence, error) {
raw := tx.Bucket([]byte(admissionQueueStateBucket)).Get(episodeStateKey)
if raw == nil {
return admission.EpisodeSequence{}, admission.ErrCorruptRecord
}
return admission.UnmarshalEpisodeSequence(raw)
}
func putEpisodeSequence(tx *bolt.Tx, s admission.EpisodeSequence) error {
data, err := s.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionQueueStateBucket)).Put(episodeStateKey, data)
}
// startEpisodeSequence gives a new or upgraded ledger its own sequence.
func startEpisodeSequence(tx *bolt.Tx) error {
s, err := admission.NewEpisodeSequence(episodeNonce)
if err != nil {
return err
}
return putEpisodeSequence(tx, s)
}
// upgradeLedgerToSchemaSix adds the episode bucket and sequence to a schema
// 5 ledger inside the opening transaction. Candidates queued before keep
// the episodes they were given; no episode row is invented for them.
func upgradeLedgerToSchemaSix(tx *bolt.Tx) error {
if _, err := tx.CreateBucket([]byte(admissionEpisodesBucket)); err != nil {
return err
}
return startEpisodeSequence(tx)
}
func loadEpisode(tx *bolt.Tx, key string) (admission.Episode, bool, error) {
raw := tx.Bucket([]byte(admissionEpisodesBucket)).Get([]byte(key))
if raw == nil {
return admission.Episode{}, false, nil
}
e, err := admission.UnmarshalEpisode(raw)
return e, true, err
}
func putEpisode(tx *bolt.Tx, key string, e admission.Episode) error {
data, err := e.MarshalBinary()
if err != nil {
// Placement chose the row; one it cannot store is damage, never a
// refused arrival that would keep a candidate without its row.
return corruptRecord(err)
}
if retainedShape(e) {
if err = raiseSchema(tx, admissionSchemaEpisodeRetained); err != nil {
return err
}
}
if e.EndElapsed > 0 {
if err = raiseSchema(tx, admissionSchemaEpisodeElapsed); err != nil {
return err
}
}
return tx.Bucket([]byte(admissionEpisodesBucket)).Put([]byte(key), data)
}
// episodeClock is admission time and the ceiling's elapsed time at this
// transaction's reading.
func (q *queueTx) episodeClock() (admission.EpisodeClock, error) {
s, err := loadCeilingState(q.tx)
return admission.EpisodeClock{Now: q.now, Elapsed: s.Elapsed}, err
}
// putEpisode anchors legacy rows even when a write changes only their
// lines. The raw encoder remains available for old-format fixtures.
func (q *queueTx) putEpisode(key string, e admission.Episode) error {
clock, err := q.episodeClock()
if err != nil {
return err
}
e, err = e.AnchorElapsed(clock)
if err != nil {
// A per-arrival refusal could commit candidate writes already
// made by placement without the matching episode boundary.
return corruptRecord(err)
}
return putEpisode(q.tx, key, e)
}
// proveEpisodes checks every episode row at open: it decodes under its
// target's canonical key, and each line names a stored candidate of that
// target, kind, episode and generation. The sequence must decode too.
// ParseTargetKey refuses a key that is not canonical.
func proveEpisodes(tx *bolt.Tx) error {
if _, err := loadEpisodeSequence(tx); err != nil {
return err
}
var hasVerifiedReference bool
if err := tx.Bucket([]byte(admissionEpisodesBucket)).ForEach(func(k, v []byte) error {
target, err := admission.ParseTargetKey(string(k), admission.Caps{IPv6: true})
if err != nil {
return admission.ErrCorruptRecord
}
e, err := admission.UnmarshalEpisode(v)
if err != nil {
return err
}
if retainedShape(e) {
schema := tx.Bucket([]byte(admissionMetaBucket)).Get(admissionSchemaKey)
if len(schema) != 1 || schema[0] < admissionSchemaEpisodeRetained {
return admission.ErrCorruptRecord
}
}
for _, line := range e.Lines {
hasVerifiedReference = hasVerifiedReference || line.VerifiedGeneration != 0
if _, _, err := loadVerifiedBlock(tx, e.ID, line, target); err != nil {
return err
}
if line.Candidate == "" {
continue
}
c, err := loadCandidate(tx, line.Candidate)
if err != nil {
return corruptRecord(err)
}
if c.Key != (admission.CandidateKey{Kind: line.Kind, Target: target, Episode: e.ID, Generation: line.Generation}) {
return admission.ErrCorruptRecord
}
}
return nil
}); err != nil {
return err
}
if !hasVerifiedReference {
return nil
}
// Forgotten returns can leave gaps between the current line and its
// latest verified block. Check retained history without assuming an
// unbroken generation chain or trusting the reference being proved.
return tx.Bucket([]byte(admissionCandidatesBucket)).ForEach(func(k, _ []byte) error {
c, err := loadCandidate(tx, admission.CandidateID(k))
if err != nil {
return corruptRecord(err)
}
if c.State != admission.StateVerified || (c.Key.Kind != admission.KindBlockIP && c.Key.Kind != admission.KindBlockSubnet) {
return nil
}
e, found, err := loadEpisode(tx, c.Key.Target.Key())
if err != nil || !found || e.ID != c.Key.Episode {
return err
}
if line, ok := e.Line(c.Key.Kind); ok && line.VerifiedGeneration != 0 && c.Key.Generation > line.VerifiedGeneration {
return admission.ErrCorruptRecord
}
return nil
})
}
// PreviewsAnswer sets whether an observe preview answers its episode's
// line as a verified effect does. Previews answer while admission only
// previews, so a comparison follows the episode. A forecasting owner turns
// them off: an address previewed before its session is forecast again on
// its next observation. Challenge lines always follow their previews,
// which no forecast models.
func (l *AdmissionLedger) PreviewsAnswer(on bool) { l.previewsEnd.Store(!on) }
// takeEpisode names a new episode from the ledger's sequence.
func (q *queueTx) takeEpisode() (admission.EpisodeID, error) {
s, err := loadEpisodeSequence(q.tx)
if err != nil {
return admission.EpisodeID{}, err
}
id, err := s.Take()
if err != nil {
return admission.EpisodeID{}, err
}
return id, putEpisodeSequence(q.tx, s)
}
// placeTx assigns an arrival its episode and generation from the target's
// episode row and queues or coalesces it. The row changes only
// when the arrival is queued or coalesced, or when it is refused because
// its kind's line already had an attempt: that observation still extends
// the episode. The engine, never the caller, chooses the episode.
func (l *AdmissionLedger) placeTx(q *queueTx, req admission.CandidateRequest, e admission.Evidence, ids []admission.EvidenceID) (admission.CandidateID, bool, error) {
// Assessment tolerates clock skew, but storing it as a frontier would
// make genuine observations stale or cross a boundary before its time.
if e.ObservedAt().After(q.now) {
return "", false, refusal(admission.ReasonInvalid, "episode observation is dated in the future")
}
rowKey := req.Target.Key()
cur, found, err := loadEpisode(q.tx, rowKey)
if err != nil {
return "", false, err
}
var at *admission.Episode
var states []admission.LineState
// gone is when the ledger learned the arriving kind's verified block
// left early; zero unless this arrival may re-admit it.
var gone time.Time
if found {
if q.started && !cur.Verified.IsZero() && req.Kind == admission.KindBlockIP {
if line, ok := cur.Line(req.Kind); ok && line.VerifiedGeneration == 0 {
if line.VerifiedGeneration, err = retainedVerifiedGeneration(q.tx, cur.ID, line, req.Target); err != nil {
return "", false, err
}
cur = cur.WithLine(line)
}
}
at = &cur
for _, line := range cur.Lines {
// Without previews answering, a remembered answer of a block
// line counts only for a verified block, which closes the episode.
ignore := l.previewsEnd.Load() && line.Kind != admission.KindChallenge
if line.Candidate == "" {
state := admission.LineEnded
if line.Answered && (!ignore || !cur.Verified.IsZero()) {
state = admission.LineAnswered
}
// A forgotten return still owes the verified block's gate.
var left time.Time
if state, left, err = q.readmit(cur, line, req, e, state, state == admission.LineAnswered); err != nil {
return "", false, err
}
if !left.IsZero() {
gone = left
}
states = append(states, state)
continue
}
c, loadErr := loadCandidate(q.tx, line.Candidate)
if loadErr != nil {
return "", false, corruptRecord(loadErr)
}
if c.Key != (admission.CandidateKey{Kind: line.Kind, Target: req.Target, Episode: cur.ID, Generation: line.Generation}) {
return "", false, admission.ErrCorruptRecord
}
retired := false
if c.State == admission.StateUnknown {
a, attemptErr := currentAttempt(q.tx, c)
if attemptErr != nil {
return "", false, attemptErr
}
retired = a.Retired
}
state := admission.LineStateOf(c, retired)
if ignore && c.State == admission.StateObserved {
state = admission.LineEnded
}
var left time.Time
if state, left, err = q.readmit(cur, line, req, e, state, c.State == admission.StateVerified); err != nil {
return "", false, err
}
if !left.IsZero() {
gone = left
}
states = append(states, state)
}
}
clock, err := q.episodeClock()
if err != nil {
return "", false, err
}
clock.Degraded = l.degraded
choice, err := admission.Place(at, states, req.Kind, e.ObservedAt(), clock, q.takeEpisode)
if err != nil {
return "", false, err
}
for _, root := range ids {
proof, proofErr := loadEvidence(q.tx, l.reg, root)
if proofErr != nil {
return "", false, proofErr
}
if proof.ObservedAt().Before(choice.Episode.Previous) || (!choice.Episode.PriorLast.IsZero() && !proof.ObservedAt().After(choice.Episode.PriorLast)) {
return "", false, refusal(admission.ReasonStale, "support belongs to an earlier episode")
}
if !gone.IsZero() && choice.Episode.ID == cur.ID && !proof.ObservedAt().After(gone) {
return "", false, refusal(admission.ReasonStale, "support predates the block's removal")
}
}
key := admission.CandidateKey{Kind: req.Kind, Target: req.Target, Episode: choice.Episode.ID, Generation: choice.Generation}
id, err := key.ID()
if err != nil {
return "", false, err
}
if choice.Answered {
if err = q.putEpisode(rowKey, choice.Episode); err != nil {
return "", false, err
}
return id, false, refusal(admission.ReasonExistingEffect, "the episode's response already has an attempt")
}
req.Episode, req.Generation = key.Episode, key.Generation
_, created, err := l.enqueueTx(q, req, key, id, ids)
if err != nil {
return id, false, err
}
next := admission.EpisodeLine{Kind: req.Kind, Generation: key.Generation, Candidate: id, Observed: e.ObservedAt()}
if line, ok := choice.Episode.Line(req.Kind); ok {
if line.Observed.After(next.Observed) {
next.Observed = line.Observed
}
// A later generation keeps the episode's latest verified block.
next.VerifiedGeneration = line.VerifiedGeneration
}
row := choice.Episode.WithLine(next)
return id, created, q.putEpisode(rowKey, row)
}
// retainedVerifiedGeneration anchors an older line on its next write.
// Only an unbroken chain proves which verified block was latest; a gap
// leaves the line answered, as it did before generations were recorded.
func retainedVerifiedGeneration(tx *bolt.Tx, episode admission.EpisodeID, line admission.EpisodeLine, target admission.Target) (uint32, error) {
key := admission.CandidateKey{Kind: line.Kind, Target: target, Episode: episode, Generation: line.Generation}
for ; key.Generation > 0; key.Generation-- {
id, err := key.ID()
if err != nil {
return 0, err
}
c, err := loadCandidate(tx, id)
if err == errCandidateMissing {
return 0, nil
}
if err != nil {
return 0, err
}
if c.State == admission.StateVerified {
return key.Generation, nil
}
}
return 0, nil
}
// VerifiedEnd is the original verified end, in admission time, of the
// episode candidate c belongs to when that episode is its target's
// current row and a verified block closed it: what a same-episode
// re-admission may last until. current distinguishes an
// unverified current row from a lost or replaced row, which cannot grant
// an older queued return a new lifetime.
func (l *AdmissionLedger) VerifiedEnd(c admission.Candidate) (end time.Time, current bool, err error) {
err = l.db.bolt.View(func(tx *bolt.Tx) error {
e, found, loadErr := loadEpisode(tx, c.Key.Target.Key())
if loadErr == nil && found && e.ID == c.Key.Episode {
end, current = e.Verified, true
}
return loadErr
})
return end, current, err
}
// readmit applies the evidence gate to the line of the arriving kind. A
// verified block that left before its episode's original end no longer
// answers a later attack of its kind: the attack is a charged re-admission
// of the same episode, never past that end and only
// on evidence observed after the latest removal, which gone
// returns. verified says the line answers through a verified block, its
// own or a forgotten return's. Queued and ended returns keep the gate; a
// line without proof keeps answering. Only a live ledger has effect
// records, so nothing changes on a host that previews or forecasts.
func (q *queueTx) readmit(cur admission.Episode, line admission.EpisodeLine, req admission.CandidateRequest, e admission.Evidence, state admission.LineState, verified bool) (admission.LineState, time.Time, error) {
if !q.started || cur.Verified.IsZero() || line.Kind != req.Kind || req.Kind != admission.KindBlockIP ||
(state != admission.LineQueued && state != admission.LineEnded && !verified) {
return state, time.Time{}, nil
}
left, err := q.verifiedGone(cur, line, req.Target)
if err != nil {
return state, time.Time{}, err
}
if !q.now.Before(cur.Verified) || left.IsZero() || !e.ObservedAt().After(left) {
return admission.LineAnswered, time.Time{}, nil
}
if verified {
state = admission.LineEnded
}
return state, left, nil
}
// verifiedGone is when the ledger learned the latest verified block of
// line's kind in episode cur left the kernel before its expiry, from that
// block's effect record; zero while it holds, or when no retained record
// says. History keeps that block until both episode ends pass.
func (q *queueTx) verifiedGone(cur admission.Episode, line admission.EpisodeLine, target admission.Target) (time.Time, error) {
c, found, err := loadVerifiedBlock(q.tx, cur.ID, line, target)
if err != nil || !found {
return time.Time{}, err
}
a, err := currentAttempt(q.tx, c)
if err != nil {
return time.Time{}, err
}
rec, _, found, err := loadEffect(q.tx, a)
if err != nil || !found {
return time.Time{}, err
}
return rec.Gone, nil
}
// A verified reference may outlive its history after both episode ends
// pass, but retained history must prove the reference's claimed outcome.
func loadVerifiedBlock(tx *bolt.Tx, episode admission.EpisodeID, line admission.EpisodeLine, target admission.Target) (admission.Candidate, bool, error) {
if line.VerifiedGeneration == 0 {
return admission.Candidate{}, false, nil
}
id, err := admission.CandidateKey{Kind: line.Kind, Target: target, Episode: episode, Generation: line.VerifiedGeneration}.ID()
if err != nil {
return admission.Candidate{}, false, err
}
c, err := loadCandidate(tx, id)
if err == errCandidateMissing {
return admission.Candidate{}, false, nil
}
if err != nil {
return admission.Candidate{}, false, err
}
if c.State != admission.StateVerified {
return admission.Candidate{}, false, admission.ErrCorruptRecord
}
return c, true, nil
}
// forgetLine clears a deleted candidate reference but retains the kind's
// generation and answer. A verified boundary survives the last reference;
// a retained predecessor's retirement collects it after both clocks pass.
func (q *queueTx) forgetLine(id admission.CandidateID, c admission.Candidate) error {
key := c.Key.Target.Key()
e, found, err := loadEpisode(q.tx, key)
if err != nil || !found {
return err
}
// Forgetting an unattempted return must not discard the earlier
// verified boundary. Its verified predecessor already pays to keep
// this row until expiry; that predecessor's retirement also collects
// an unreferenced row once both ends have passed.
answered := c.State == admission.StateVerified || c.State == admission.StateObserved ||
(c.Key.Kind == admission.KindBlockIP && !e.Verified.IsZero())
rest, had := e.WithoutCandidate(id, answered)
if !rest.HasCandidates() {
clock, clockErr := q.episodeClock()
if clockErr != nil {
return clockErr
}
// Only an answered address block line can hold the row (its codec
// refuses any other unreferenced shape); anything else goes as before.
line, ok := rest.Line(admission.KindBlockIP)
if rest.Verified.IsZero() || !ok || !line.Answered || (!clock.Now.Before(rest.Verified) && clock.Elapsed >= rest.EndElapsed) {
return q.tx.Bucket([]byte(admissionEpisodesBucket)).Delete([]byte(key))
}
}
if !had {
return nil
}
return q.putEpisode(key, rest)
}
// closeEpisode ends the episode of a verified block when its kernel element
// expires: its original expiry, less the lead of the engine's
// lease clock over admission time when it ran, on both admission
// and the ceiling's elapsed time, read on clock: the reading the block ran
// at. Other kinds, and
// a candidate of another episode than its target's row, change nothing. An
// episode queues no new generation of a kind while one is in flight, so
// the verified block is its kind's latest.
func (q *queueTx) closeEpisode(c admission.Candidate, lead time.Duration, clock admission.EpisodeClock) error {
if c.Key.Kind != admission.KindBlockIP && c.Key.Kind != admission.KindBlockSubnet {
return nil
}
key := c.Key.Target.Key()
e, found, err := loadEpisode(q.tx, key)
if err != nil || !found || e.ID != c.Key.Episode {
return err
}
if e, err = e.Close(c.ExpiresAt.Add(-max(lead, 0)), clock); err != nil {
return err
}
// The verified block is its kind's latest generation; the gate reads
// it even after later generations are retired. Only a live ledger
// gates re-admissions, so only it records one.
if line, ok := e.Line(c.Key.Kind); ok && q.started && line.Generation == c.Key.Generation {
line.VerifiedGeneration = c.Key.Generation
e = e.WithLine(line)
}
return putEpisode(q.tx, key, e)
}
package store
import (
"slices"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// loadStoredEvidence loads a published record as it was stored, without
// judging it against current policy.
func loadStoredEvidence(tx *bolt.Tx, id admission.EvidenceID) (admission.Evidence, error) {
if _, err := admission.ParseEvidenceID(string(id)); err != nil {
return admission.Evidence{}, err
}
raw := tx.Bucket([]byte(admissionEvidenceBucket)).Get([]byte(id))
if raw == nil {
return admission.Evidence{}, admission.ErrEvidenceUnpublished
}
e, err := admission.UnmarshalEvidence(raw)
if err != nil {
return admission.Evidence{}, corruptRecord(err)
}
if e.ID() != id {
return admission.Evidence{}, admission.ErrCorruptRecord
}
return e, nil
}
func loadEvidence(tx *bolt.Tx, reg *admission.Registry, id admission.EvidenceID) (admission.Evidence, error) {
e, err := loadStoredEvidence(tx, id)
if err != nil {
return e, err
}
// Revalidate on every use: a policy change applies to evidence minted
// before it.
if err := reg.Validate(e); err != nil {
return admission.Evidence{}, err
}
return e, nil
}
// PublishEvidence stores an immutable record. The same record again changes
// nothing; a different record under the same ID is refused and the stored
// original is kept.
func (l *AdmissionLedger) PublishEvidence(e admission.Evidence) (bool, error) {
if err := l.reg.Validate(e); err != nil {
return false, err
}
l.mu.Lock()
defer l.mu.Unlock()
published := false
err := l.update("publish", func(tx *bolt.Tx) error {
q, txErr := l.openQueue(tx, l.now)
if txErr != nil {
return txErr
}
if published, txErr = publishTx(q, l.reg, e); txErr != nil {
return txErr
}
return q.flush()
})
if err != nil {
return false, err
}
return published, nil
}
// publishTx stores e unless the same record is already there. A new record
// takes a loose position until a candidate names it.
func publishTx(q *queueTx, reg *admission.Registry, e admission.Evidence) (bool, error) {
if err := reg.Validate(e); err != nil {
return false, err
}
data, err := e.MarshalBinary()
if err != nil {
return false, err
}
id := []byte(e.ID())
b := q.tx.Bucket([]byte(admissionEvidenceBucket))
if raw := b.Get(id); raw != nil {
old, decodeErr := admission.UnmarshalEvidence(raw)
if decodeErr != nil {
return false, corruptRecord(decodeErr)
}
if old.ID() != e.ID() {
return false, admission.ErrCorruptRecord
}
if _, err = loadRefs(q.tx, e.ID()); err != nil {
return false, err
}
if !old.Equal(e) {
return false, old.Conflict(e)
}
return false, nil
}
// A surviving reference or report belongs to missing evidence, not to
// a new publication. Recreating it would lose its recorded ownership.
for _, name := range []string{admissionRefsBucket, admissionReportsBucket} {
owned := q.tx.Bucket([]byte(name))
if owned.Get(id) != nil || owned.Bucket(id) != nil {
return false, admission.ErrCorruptRecord
}
}
if err = b.Put(id, data); err != nil {
return false, err
}
return true, q.loosen(e.ID())
}
// LoadEvidence loads a published record and revalidates it.
func (l *AdmissionLedger) LoadEvidence(id admission.EvidenceID) (admission.Evidence, error) {
var e admission.Evidence
err := l.db.bolt.View(func(tx *bolt.Tx) error {
var err error
e, err = loadEvidence(tx, l.reg, id)
return err
})
return e, err
}
// loadReports loads the later reports of e. A stored record must name e,
// and it can never list e's own original finding.
func loadReports(tx *bolt.Tx, e admission.Evidence) (admission.ReportLinks, error) {
raw := tx.Bucket([]byte(admissionReportsBucket)).Get([]byte(e.ID()))
if raw == nil {
return admission.ReportLinks{Evidence: e.ID()}, nil
}
links, err := admission.UnmarshalReportLinks(raw)
if err != nil {
return admission.ReportLinks{}, err
}
if links.Evidence != e.ID() || slices.Contains(links.Links, e.FindingID()) {
return admission.ReportLinks{}, admission.ErrCorruptRecord
}
return links, nil
}
// LinkReport records a later finding that reported published evidence.
func (l *AdmissionLedger) LinkReport(id admission.EvidenceID, findingID string) error {
l.mu.Lock()
defer l.mu.Unlock()
return l.update("link", func(tx *bolt.Tx) error { return linkTx(tx, id, findingID) })
}
// linkTx links a report to stored evidence. Links are metadata: they do not
// depend on the evidence passing current policy.
func linkTx(tx *bolt.Tx, id admission.EvidenceID, findingID string) error {
original, err := loadStoredEvidence(tx, id)
if err != nil {
return err
}
links, err := loadReports(tx, original)
if err != nil {
return err
}
if findingID == original.FindingID() {
return nil
}
next, changed, err := links.Add(findingID)
if err != nil || !changed {
return err
}
data, err := next.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionReportsBucket)).Put([]byte(id), data)
}
// Reports returns the later findings linked to published evidence, whether
// or not it passes current policy.
func (l *AdmissionLedger) Reports(id admission.EvidenceID) ([]string, uint32, error) {
var links admission.ReportLinks
err := l.db.bolt.View(func(tx *bolt.Tx) error {
e, err := loadStoredEvidence(tx, id)
if err != nil {
return err
}
links, err = loadReports(tx, e)
return err
})
return links.Links, links.Dropped, err
}
package store
import (
"bytes"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// HeldEffect names a verified block the ledger counts as held, for the
// owner to ask the firewall whether it still is.
type HeldEffect struct {
Action admission.ActionID
Target admission.Target
ExpiresAt time.Time
}
// HeldEffects lists at most limit verified, held, unclaimed blocks in
// expiry order, wrapping after the last completed batch. ObserveHeld
// advances the cursor after two answered checks, so a held prefix cannot
// starve the tail when configured capacity exceeds one engine call.
func (l *AdmissionLedger) HeldEffects(limit int) ([]HeldEffect, error) {
if limit <= 0 {
return nil, refusal(admission.ReasonInvalid, "a list of held blocks needs a positive limit")
}
l.mu.Lock()
defer l.mu.Unlock()
var out []HeldEffect
err := l.db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(admissionEffectsBucket))
if b == nil {
return nil
}
cur := b.Cursor()
k, v := cur.First()
if len(l.heldAfter) != 0 {
k, v = cur.Seek(l.heldAfter)
if bytes.Equal(k, l.heldAfter) {
k, v = cur.Next()
}
if k == nil {
k, v = cur.First()
}
}
start := bytes.Clone(k)
for k != nil && len(out) < limit {
rec, err := admission.UnmarshalEffectRecord(k, v)
if err != nil {
return err
}
if rec.Verified() && rec.Holds() && rec.EvictedBy == "" {
out = append(out, HeldEffect{Action: rec.Action, Target: rec.Target, ExpiresAt: rec.ExpiresAt})
}
k, v = cur.Next()
if k == nil {
k, v = cur.First()
}
if bytes.Equal(k, start) {
break
}
}
return nil
})
if err != nil {
return nil, err
}
return out, nil
}
// ObserveHeld takes the firewall's answer for a list HeldEffects gave:
// held[i] says whether the block asked[i] names is still held. A block
// missed by two consecutive answers left the kernel before its expiry (an
// operator unblock, undo or flush, or a lost element), which needs no
// trusted clock: its record is gone from the first miss's
// reading and frees its slot. A record that changed in between is left to
// what changed it. It returns the blocks it marked, whose rows the owner
// retires; an error marks none and keeps all previous misses, accepting
// neither new misses nor held replies from the failed answer.
func (l *AdmissionLedger) ObserveHeld(asked []HeldEffect, held []bool) ([]HeldEffect, error) {
if len(asked) != len(held) {
return nil, refusal(admission.ReasonInvalid, "firewall answers do not match the held blocks")
}
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return nil, err
}
missed := make(map[admission.ActionID]time.Time)
var gone []HeldEffect
var since []time.Time
err = l.db.bolt.View(func(tx *bolt.Tx) error {
for i, h := range asked {
_, _, eligible, readErr := heldRecord(tx, h)
if readErr != nil {
return readErr
}
if !eligible || held[i] {
continue
}
if first, ok := l.missed[h.Action]; ok {
gone, since = append(gone, h), append(since, first)
} else {
missed[h.Action] = now
}
}
return nil
})
if err != nil {
return nil, err
}
var marked []HeldEffect
if len(gone) > 0 {
err = l.update("gone", func(tx *bolt.Tx) error {
marked = nil
for i, h := range gone {
ok, txErr := markGone(tx, h, since[i])
if txErr != nil {
return txErr
}
if ok {
marked = append(marked, h)
}
}
return nil
})
if err != nil {
return nil, err
}
}
l.missed = missed
l.heldPass = !l.heldPass
if !l.heldPass && len(asked) != 0 {
last := asked[len(asked)-1]
// Every key was validated before accepting this answer.
l.heldAfter, _ = admission.EffectRecordKey(last.Action, last.ExpiresAt)
}
return marked, nil
}
// markGone marks the record h names gone at since, if it is still a
// verified, held and unclaimed record of h's target.
func markGone(tx *bolt.Tx, h HeldEffect, since time.Time) (bool, error) {
rec, key, eligible, err := heldRecord(tx, h)
if err != nil || !eligible {
return false, err
}
if err = addEffectTally(tx, rec.Partition, -1); err != nil {
return false, err
}
rec.Gone = since
data, err := rec.MarshalBinary()
if err != nil {
return false, err
}
if err = raiseSchema(tx, admissionSchemaEffectGone); err != nil {
return false, err
}
return true, tx.Bucket([]byte(admissionEffectsBucket)).Put(key, data)
}
// heldRecord checks the identity before the answer counts as a miss.
// A pending, claimed, swept or already gone record contributes nothing.
func heldRecord(tx *bolt.Tx, h HeldEffect) (admission.EffectRecord, []byte, bool, error) {
key, err := admission.EffectRecordKey(h.Action, h.ExpiresAt)
if err != nil {
return admission.EffectRecord{}, nil, false, err
}
b := tx.Bucket([]byte(admissionEffectsBucket))
if b == nil {
return admission.EffectRecord{}, nil, false, nil
}
raw := b.Get(key)
if raw == nil {
return admission.EffectRecord{}, nil, false, nil
}
rec, err := admission.UnmarshalEffectRecord(key, raw)
if err != nil {
return admission.EffectRecord{}, nil, false, err
}
if rec.Target != h.Target {
return admission.EffectRecord{}, nil, false, refusal(admission.ReasonInvalid, "held answer names another target")
}
if !rec.Verified() || !rec.Holds() || rec.EvictedBy != "" {
return admission.EffectRecord{}, nil, false, nil
}
return rec, key, true, nil
}
package store
import (
"errors"
"math"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
func loadIngressState(tx *bolt.Tx) (admission.IngressState, error) {
raw := tx.Bucket([]byte(admissionQueueStateBucket)).Get(ingressStateKey)
if raw == nil {
return admission.IngressState{}, admission.ErrCorruptRecord
}
return admission.UnmarshalIngressState(raw)
}
func putIngressState(tx *bolt.Tx, s admission.IngressState) error {
data, err := s.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionQueueStateBucket)).Put(ingressStateKey, data)
}
// BeginIngress starts a new ingress generation. A generation still open
// ended without a clean close: whatever its ingress held was lost, and it
// is counted as interrupted.
func (l *AdmissionLedger) BeginIngress() (admission.IngressState, error) {
l.mu.Lock()
defer l.mu.Unlock()
var out admission.IngressState
err := l.update("ingress", func(tx *bolt.Tx) error {
s, err := loadIngressState(tx)
if err != nil {
return err
}
if s.Open {
s.Interrupted, s.Resumed = s.Interrupted+1, s.Generation+1
} else {
s.Resumed = 0
}
s.Generation++
s.Open, s.Persisted = true, 0
out = s
return putIngressState(tx, s)
})
if err != nil {
return admission.IngressState{}, err
}
return out, nil
}
// EndIngress closes the current generation cleanly.
func (l *AdmissionLedger) EndIngress() error {
l.mu.Lock()
defer l.mu.Unlock()
return l.update("ingress", func(tx *bolt.Tx) error {
s, err := loadIngressState(tx)
if err != nil || !s.Open {
return err
}
s.Open = false
return putIngressState(tx, s)
})
}
// EnqueueGroup persists a group of ingress arrivals in one transaction. Each
// arrival publishes its evidence, links its later reports and queues or
// coalesces its request. An arrival whose evidence differs from the stored
// record only in its finding is a later report: it is linked and queued
// against the original. A refused arrival is counted and returned in its
// result; any other error aborts the whole group.
func (l *AdmissionLedger) EnqueueGroup(arrivals []admission.Arrival, checkpoint *admission.IngressCheckpoint) ([]admission.ArrivalResult, int, error) {
if len(arrivals) > admission.MaxArrivalGroup {
return nil, 0, refusal(admission.ReasonInvalid, "arrival group is too large")
}
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return nil, 0, err
}
var out []admission.ArrivalResult
var revision int
err = l.update("group", func(tx *bolt.Tx) error {
ingress, txErr := loadIngressState(tx)
if txErr != nil {
return txErr
}
if !ingress.Open {
return refusal(admission.ReasonEngineUnavailable, "no ingress generation is open")
}
q, txErr := l.openQueue(tx, now)
if txErr != nil {
return txErr
}
if checkpoint != nil {
if txErr = checkpoint.Validate(ingress.Checkpoint, ingress.Generation); txErr != nil {
return txErr
}
if txErr = q.countCheckpoint(ingress.Checkpoint, checkpoint); txErr != nil {
return txErr
}
ingress.Checkpoint = checkpoint
}
// Empty groups must meet the same shared damage as arrivals. Drain
// uses one to distinguish a broken queue from a damaged arrival.
if _, txErr = q.queueView(); txErr != nil {
return txErr
}
if txErr = q.sweep(); txErr != nil {
return txErr
}
out = make([]admission.ArrivalResult, len(arrivals))
for i, a := range arrivals {
id, created, arriveErr := l.arriveTx(q, a)
out[i].Candidate, out[i].Created, out[i].Err = id, created, arriveErr
reason, refused := admission.ReasonOf(arriveErr)
if errors.Is(arriveErr, admission.ErrCandidateTerminal) || errors.Is(arriveErr, admission.ErrTransitionConflict) {
// Placement never lands on an ended or in-flight
// candidate; one that does is counted as an invalid
// refusal, never dropped.
reason, refused = admission.ReasonInvalid, true
}
switch {
case refused:
tier := arrivalTier(a, now)
if txErr = q.count(admission.EventRefused, reason, tier); txErr != nil {
return txErr
}
// A refused arrival names no stored candidate: its
// notice has its own check and family, and no example.
if kind := admission.GapNotice(admission.GapEnded, reason, 0, tier, false); kind != 0 {
key := admission.NoticeKey{Kind: kind, Reason: reason, Check: a.Evidence.Check(), Effect: a.Request.Kind.Effect()}
if txErr = q.raise(key, "", 0); txErr != nil {
return txErr
}
}
case arriveErr != nil:
return arriveErr
}
}
ingress.Persisted += uint64(len(arrivals))
if txErr = putIngressState(tx, ingress); txErr != nil {
return txErr
}
revision = tx.ID()
return q.flush()
})
if err != nil {
return nil, 0, err
}
return out, revision, nil
}
// countCheckpoint counts what the ingress decided since the previous
// checkpoint into the lifetime counters and the outcome windows and
// raises the notices of the
// Critical losses among it. They name no candidate: none was stored.
func (q *queueTx) countCheckpoint(previous, next *admission.IngressCheckpoint) error {
var old admission.QueueCounters
if previous != nil {
var err error
if old, err = admission.UnmarshalQueueCounters(previous.Counters); err != nil {
return err
}
}
counts, err := admission.UnmarshalQueueCounters(next.Counters)
if err != nil {
return err
}
for _, row := range counts.Rows() {
n := row.N - old.Count(row.Key)
if n == 0 {
continue
}
k, tier := row.Key, admission.Tier{Class: row.Key.Class, Severity: row.Key.Severity}
// No candidate is written for an ingress refusal, so its counts
// must raise the schema themselves before older readers see them.
if k.Reason == admission.ReasonDisabled {
if err = raiseSchema(q.tx, admissionSchemaDisabled); err != nil {
return err
}
}
// The lifetime counters count every queue drop, as the windows do.
if err = q.counters.AddCount(k, n); err != nil {
return err
}
q.countsDirty = true
if err = q.outcomes.AddCount(admission.QueueOutcome(k.Event, k.Reason, tier), n); err != nil {
return err
}
if kind := admission.GapNotice(admission.GapEnded, k.Reason, 0, tier, false); kind != 0 {
if err = q.raiseCount(admission.NoticeKey{Kind: kind, Reason: k.Reason}, n); err != nil {
return err
}
}
}
return nil
}
// arrivalTier is the tier the arrival's own evidence supports, for
// counting a refusal; zero when that evidence cannot be assessed.
func arrivalTier(a admission.Arrival, now time.Time) admission.Tier {
assessed, err := admission.Assess(a.Evidence.Target(), []admission.Evidence{a.Evidence}, now)
if err != nil {
return admission.Tier{}
}
return assessed.Tier
}
func (l *AdmissionLedger) arriveTx(q *queueTx, a admission.Arrival) (admission.CandidateID, bool, error) {
e := a.Evidence
if _, err := publishTx(q, l.reg, e); errors.Is(err, admission.ErrEvidenceConflict) {
stored, loadErr := loadEvidence(q.tx, l.reg, e.ID())
if loadErr != nil {
return "", false, loadErr
}
if !stored.SameExceptFinding(e) {
return "", false, err
}
// A report-only tail has already acknowledged the primary finding.
if !a.ReportsOnly {
if err = linkTx(q.tx, e.ID(), e.FindingID()); err != nil {
return "", false, err
}
}
// Report invariants refer to the immutable original finding, not
// the remint we just linked as a later report.
e = stored
} else if err != nil {
return "", false, err
}
for _, finding := range a.Reports {
if err := linkTx(q.tx, e.ID(), finding); err != nil {
return "", false, err
}
}
if a.Dropped != 0 {
links, err := loadReports(q.tx, e)
if err != nil {
return "", false, err
}
links.Dropped += min(a.Dropped, math.MaxUint32-links.Dropped)
data, err := links.MarshalBinary()
if err != nil {
return "", false, err
}
if err = q.tx.Bucket([]byte(admissionReportsBucket)).Put([]byte(e.ID()), data); err != nil {
return "", false, err
}
}
if a.ReportsOnly {
return "", false, nil
}
// The evidence and its acknowledged reports are stored before the
// request is judged, so a refused request cannot leave a later
// report-only tail an overflow count its links do not support.
if a.Request.Primary != a.Evidence.ID() {
return "", false, refusal(admission.ReasonInvalid, "arrival request does not name its evidence")
}
if !a.Request.Episode.IsZero() || a.Request.Generation != 0 {
return "", false, refusal(admission.ReasonInvalid, "an arrival names no episode: the ledger assigns it")
}
if err := admission.BindTarget(a.Request.Target, a.Request.Entry, e); err != nil {
return "", false, err
}
if a.Request.Entry != 0 {
if err := l.reg.ValidateVia(a.Request.Entry, e); err != nil {
return "", false, err
}
}
ids, err := rootSet(a.Request)
if err != nil {
return "", false, err
}
if err = admission.ValidateKindTarget(a.Request.Kind, a.Request.Target); err != nil {
return "", false, err
}
roots, err := loadRoots(q.tx, l.reg, ids)
if err != nil {
return "", false, err
}
if _, err = scopeOwner(l.Inventory(), roots); err != nil {
return "", false, err
}
// Evidence that cannot support its target now is no qualifying
// observation: it changes no episode.
if _, err = admission.Assess(a.Request.Target, []admission.Evidence{e}, q.now); err != nil {
return "", false, err
}
if _, err = admission.Assess(a.Request.Target, roots, q.now); err != nil {
return "", false, err
}
return l.placeTx(q, a.Request, e, ids)
}
// QueueSnapshot is the durable queue as the ingress sees it: every live
// position, the partition cursors, the admission time and the inventory.
// It needs a current reading, since the ingress assesses against its time.
func (l *AdmissionLedger) QueueSnapshot() (*admission.QueueSnapshot, error) {
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return nil, err
}
inv := l.Inventory()
snap := &admission.QueueSnapshot{Now: now, Inventory: inv}
err = l.db.bolt.View(func(tx *bolt.Tx) error {
q, txErr := openQueueWith(tx, l.reg, inv, now)
if txErr != nil {
return txErr
}
live, txErr := q.live()
if txErr != nil {
return txErr
}
for _, lc := range live {
snap.Items = append(snap.Items, queueItem(lc.id, lc.c, lc.entry))
}
snap.Cursors = q.state.Cursors
snap.Revision = tx.ID()
ingress, txErr := loadIngressState(tx)
if txErr != nil {
return txErr
}
snap.Generation, snap.Checkpoint = ingress.Generation, ingress.Checkpoint
return nil
})
if err != nil {
return nil, err
}
return snap, nil
}
package store
import (
"strings"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// upgradeLedgerToSchemaSeven adds the intent bucket to a schema 6 ledger
// inside the opening transaction. An attempt that ran without an outcome,
// or ended unknown, gets the intent its execution would have recorded, as a
// batch of its own: no effect it may have applied goes unrecorded. Each
// unknown outcome then holds what it holds under schema 7: its intent in
// the recovery reserve and the slot of the row that will retire it; opening
// proves the result. This upgrade completes the chain and records the
// schema.
func upgradeLedgerToSchemaSeven(tx *bolt.Tx) error {
if _, err := tx.CreateBucket([]byte(admissionIntentsBucket)); err != nil {
return err
}
s, loadErr := loadStorageState(tx)
if loadErr != nil {
return loadErr
}
if loadErr = tx.Bucket([]byte(admissionAttemptsBucket)).ForEach(func(_, v []byte) error {
a, err := admission.UnmarshalAttempt(v)
if err != nil {
return err
}
if a.State != admission.StateExecuting && a.State != admission.StateUnknown {
return nil
}
if a.State == admission.StateUnknown {
if s, err = s.HoldIntent(); err != nil {
return err
}
if s, err = s.HoldAudit(1); err != nil {
return err
}
}
c, err := loadCandidate(tx, a.Attempt.Candidate)
if err != nil {
return corruptRecord(err)
}
batch, err := admission.NewBatchID([]admission.ActionID{a.Attempt.ID})
if err != nil {
return err
}
return putIntent(tx, admission.Intent{Action: a.Attempt.ID, Batch: batch, Kind: c.Key.Kind, Target: c.Key.Target, ExpiresAt: a.ExpiresAt})
}); loadErr != nil {
return loadErr
}
if loadErr = putStorageState(tx, s); loadErr != nil {
return loadErr
}
return tx.Bucket([]byte(admissionMetaBucket)).Put(admissionSchemaKey, []byte{admissionSchemaVersion})
}
func loadIntent(tx *bolt.Tx, id admission.ActionID) (admission.Intent, bool, error) {
b := tx.Bucket([]byte(admissionIntentsBucket))
if b == nil {
return admission.Intent{}, false, admission.ErrCorruptRecord
}
raw := b.Get([]byte(id))
if raw == nil {
return admission.Intent{}, false, nil
}
in, err := admission.UnmarshalIntent(raw)
if err != nil {
return in, true, err
}
if in.Action != id {
return in, true, admission.ErrCorruptRecord
}
return in, true, nil
}
func putIntent(tx *bolt.Tx, in admission.Intent) error {
data, err := in.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionIntentsBucket)).Put([]byte(in.Action), data)
}
// deleteIntent removes the intent of an attempt whose outcome is settled.
// An attempt that ran always has one.
func deleteIntent(tx *bolt.Tx, id admission.ActionID) error {
_, err := removeIntent(tx, id)
return err
}
func removeIntent(tx *bolt.Tx, id admission.ActionID) (admission.Intent, error) {
in, found, err := loadIntent(tx, id)
if err != nil {
return in, corruptRecord(err)
}
if !found {
return in, admission.ErrCorruptRecord
}
return in, tx.Bucket([]byte(admissionIntentsBucket)).Delete([]byte(id))
}
// putIntent and deleteIntent on a transaction of the ledger's handle also
// record the change to the recovery hold, published when it commits.
func (q *queueTx) putIntent(in admission.Intent) error {
if err := putIntent(q.tx, in); err != nil {
return err
}
q.heldDelta[in.Target.Key()]++
return nil
}
func (q *queueTx) deleteIntent(id admission.ActionID) error {
in, err := removeIntent(q.tx, id)
if err != nil {
return err
}
q.heldDelta[in.Target.Key()]--
return nil
}
// unsettled reports whether attempt a's outcome is unresolved: it runs, or
// it ended unknown and recovery has not retired it.
func unsettled(a admission.AttemptRecord) bool {
return a.State == admission.StateExecuting || (a.State == admission.StateUnknown && !a.Retired)
}
// proveIntents checks every intent at open: it decodes under its own
// action, belongs to the current attempt of a stored candidate whose
// outcome is unresolved, and names that candidate's kind and target and the
// attempt's expiry. Every unresolved attempt has its intent. It returns the
// recovery hold those intents make.
func proveIntents(tx *bolt.Tx) (map[string]int, error) {
rows := map[admission.ActionID]bool{}
held := map[string]int{}
if err := tx.Bucket([]byte(admissionIntentsBucket)).ForEach(func(k, _ []byte) error {
id, err := admission.ParseActionID(string(k))
if err != nil {
return admission.ErrCorruptRecord
}
in, _, err := loadIntent(tx, id)
if err != nil {
return err
}
a, c, err := attemptAndCandidate(tx, id)
if err != nil {
return corruptRecord(err)
}
if !unsettled(a) || in.Kind != c.Key.Kind || in.Target != c.Key.Target || !in.ExpiresAt.Equal(a.ExpiresAt) {
return admission.ErrCorruptRecord
}
if err := proveIntentVictim(tx, a, in); err != nil {
return err
}
rows[id] = true
held[in.Target.Key()]++
return nil
}); err != nil {
return nil, err
}
return held, tx.Bucket([]byte(admissionAttemptsBucket)).ForEach(func(_, v []byte) error {
a, err := admission.UnmarshalAttempt(v)
if err != nil {
return err
}
if unsettled(a) && !rows[a.Attempt.ID] {
return admission.ErrCorruptRecord
}
return nil
})
}
// ExecuteBatch marks the reserved attempts of one batch as running and
// records each member's intent in the same transaction. See the Ledger
// interface.
func (l *AdmissionLedger) ExecuteBatch(ids []admission.ActionID) ([]admission.Intent, bool, error) {
intents, _, _, granted, err := l.execute(ids)
return intents, granted, err
}
// Execute runs one reserved attempt as a batch of its own and returns its
// records after the step.
func (l *AdmissionLedger) Execute(id admission.ActionID) (admission.Candidate, admission.AttemptRecord, bool, error) {
_, attempts, candidates, granted, err := l.execute([]admission.ActionID{id})
if err != nil {
return admission.Candidate{}, admission.AttemptRecord{}, false, err
}
return candidates[0], attempts[0], granted, nil
}
func (l *AdmissionLedger) execute(ids []admission.ActionID) ([]admission.Intent, []admission.AttemptRecord, []admission.Candidate, bool, error) {
batch, err := admission.NewBatchID(ids)
if err != nil {
return nil, nil, nil, false, err
}
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return nil, nil, nil, false, err
}
var intents []admission.Intent
var attempts []admission.AttemptRecord
var candidates []admission.Candidate
running := 0
err = l.update("execute", func(tx *bolt.Tx) error {
intents, attempts, candidates, running = nil, nil, nil, 0
q, txErr := l.openQueue(tx, now)
if txErr != nil {
return txErr
}
for _, id := range ids {
a, c, loadErr := attemptAndCandidate(tx, id)
if loadErr != nil {
return loadErr
}
in := admission.Intent{Action: id, Batch: batch, Kind: c.Key.Kind, Target: c.Key.Target, ExpiresAt: a.ExpiresAt}
switch {
case a.State == admission.StateReserved:
victim, victimErr := claimedVictim(tx, a, q.imports)
if victimErr != nil {
return victimErr
}
in.Victim = victim
case a.State == admission.StateExecuting:
var found bool
var intentErr error
in, found, intentErr = loadIntent(tx, id)
if intentErr != nil {
return corruptRecord(intentErr)
}
if !found {
return admission.ErrCorruptRecord
}
if in.Batch != batch {
return admission.ErrTransitionConflict
}
running++
case a.State != admission.StateReserved:
return admission.ErrTransitionConflict
}
intents, attempts, candidates = append(intents, in), append(attempts, a), append(candidates, c)
}
// Retired members can be minted again under the same action IDs,
// so a batch ID alone cannot prove every member ran. Check the
// whole batch before a running member's target hold can mask a
// conflict as a recovery deferral.
if running > 0 {
if running != len(ids) {
return admission.ErrTransitionConflict
}
return nil
}
for i, a := range attempts {
c := candidates[i]
if !now.Before(a.ExpiresAt) {
return refusal(admission.ReasonStale, "absolute expiry has passed")
}
held, holdErr := q.heldTarget(c.Key.Target)
if holdErr != nil {
return holdErr
}
if held {
return refusal(admission.ReasonPendingRecovery, "an unresolved attempt holds the target")
}
e, loadErr := loadQueueEntry(tx, a.Attempt.Candidate)
if loadErr != nil {
return loadErr
}
a.State, c.State = admission.StateExecuting, admission.StateExecuting
if txErr = q.writeAuditRow(c, a, e.Tier); txErr != nil {
return txErr
}
c.Transitions++
if txErr = putAttempt(tx, a); txErr != nil {
return txErr
}
if txErr = putCandidate(tx, c); txErr != nil {
return txErr
}
if txErr = q.putIntent(intents[i]); txErr != nil {
return txErr
}
attempts[i], candidates[i] = a, c
}
return q.flush()
})
if err != nil {
return nil, nil, nil, false, err
}
if running == 0 {
if l.executed == nil {
l.executed = map[admission.ActionID]admission.EpisodeClock{}
}
for _, a := range attempts {
l.executed[a.Attempt.ID] = admission.EpisodeClock{Now: now, Elapsed: l.elapsed}
}
}
return intents, attempts, candidates, running == 0, nil
}
// UnresolvedIntents reads, through the handle, the intent of every attempt
// whose outcome is unresolved, in action order: the owner retires the
// unknown ones once nothing of them can remain.
func (l *AdmissionLedger) UnresolvedIntents() ([]admission.Intent, error) {
return ReadUnresolvedIntents(l.db)
}
// ReadUnresolvedIntents reads the intent of every attempt whose outcome is
// unresolved, in action order, without a ledger handle: startup reads them
// before the firewall restores its state, whether or not the ledger owner
// started. A database without a ledger, or whose ledger predates intents,
// has none. An error means the ledger cannot be read; the caller then
// cannot tell a proven automatic entry from an unproven one.
func ReadUnresolvedIntents(db *DB) ([]admission.Intent, error) {
var out []admission.Intent
err := db.bolt.View(func(tx *bolt.Tx) error {
if current, err := currentLedger(tx); !current {
return err
}
return tx.Bucket([]byte(admissionIntentsBucket)).ForEach(func(k, _ []byte) error {
in, found, err := loadIntent(tx, admission.ActionID(k))
if err != nil {
return err
}
if !found {
return admission.ErrCorruptRecord
}
out = append(out, in)
return nil
})
})
if err != nil {
return nil, err
}
return out, nil
}
// currentLedger reports whether tx holds a ledger of the current schema for
// a read without a handle. A database without a ledger, or whose ledger
// predates intents, has nothing to read; one it cannot read is an error.
func currentLedger(tx *bolt.Tx) (bool, error) {
intents := tx.Bucket([]byte(admissionIntentsBucket))
meta := tx.Bucket([]byte(admissionMetaBucket))
if meta == nil {
return false, tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
if strings.HasPrefix(string(name), "adm:") {
return admission.ErrCorruptRecord
}
return nil
})
}
switch schema := meta.Get(admissionSchemaKey); {
case len(schema) == 1 && schema[0] >= 1 && schema[0] < admissionSchemaVersion:
if intents != nil {
return false, admission.ErrCorruptRecord
}
return false, nil
case len(schema) != 1 || schema[0] < admissionSchemaVersion || schema[0] > admissionSchemaNewest:
return false, ErrAdmissionSchema
case intents == nil:
return false, admission.ErrCorruptRecord
}
return true, nil
}
// ReadUnprovenBlockActions returns, in order, the actions among ids whose
// attempt the ledger records with anything but a verified outcome (it
// runs, ended unknown, retired or not, or failed) or whose verified block
// a proven removal evicted or the ledger marked gone. Startup marks their firewall rows unverified
// before the firewall restores its state, whether or not the ledger owner
// started. A name the ledger never recorded is not returned: no attempt
// claims its row. An error means the ledger cannot be read.
func ReadUnprovenBlockActions(db *DB, ids []string) ([]string, error) {
return readBlockActions(db, ids, func(tx *bolt.Tx, a admission.AttemptRecord) (bool, error) {
if a.State == admission.StateVerified {
return evictedBlock(tx, a)
}
return true, nil
})
}
// ReadSettledBlockActions returns, in order, the actions among ids whose
// attempt can have left nothing in the kernel: it failed, ended unknown
// and was retired after its expiry found nothing of it, or was verified
// and then evicted by a removal the applier proved or marked gone after
// two firewall answers found nothing of it. Startup removes their
// firewall rows; the other unproven ones stay held. A name the ledger
// never recorded is not returned. An error means the ledger cannot be read.
func ReadSettledBlockActions(db *DB, ids []string) ([]string, error) {
return readBlockActions(db, ids, func(tx *bolt.Tx, a admission.AttemptRecord) (bool, error) {
if a.State == admission.StateVerified {
return evictedBlock(tx, a)
}
return a.State == admission.StateFailed || (a.State == admission.StateUnknown && a.Retired), nil
})
}
// evictedBlock reports whether verified attempt a's block was evicted or
// marked gone: either way nothing it applied remains, and a restored row
// would hold an element whose slot the ledger no longer counts.
func evictedBlock(tx *bolt.Tx, a admission.AttemptRecord) (bool, error) {
// Startup reads before the owner proves the ledger. A damaged layout
// must not masquerade as a ledger without effect records.
b := tx.Bucket([]byte(admissionEffectsBucket))
schema := tx.Bucket([]byte(admissionMetaBucket)).Get(admissionSchemaKey)
if len(schema) != 1 || schema[0] < admissionSchemaVersion || schema[0] > admissionSchemaNewest {
return false, ErrAdmissionSchema
}
if _, err := effectLayout(tx, schema[0]); err != nil {
return false, err
}
rec, key, found, err := loadEffect(tx, a)
if err == nil && !found && b != nil && b.Bucket(key) != nil {
return false, admission.ErrCorruptRecord
}
if err == nil && found && !rec.Gone.IsZero() && schema[0] < admissionSchemaEffectGone {
return false, admission.ErrCorruptRecord
}
return found && (rec.Evicted || !rec.Gone.IsZero()), err
}
// readBlockActions returns, in order, the recorded actions among ids whose
// attempt keep accepts.
func readBlockActions(db *DB, ids []string, keep func(*bolt.Tx, admission.AttemptRecord) (bool, error)) ([]string, error) {
var out []string
err := db.bolt.View(func(tx *bolt.Tx) error {
if current, err := currentLedger(tx); !current {
return err
}
attempts := tx.Bucket([]byte(admissionAttemptsBucket))
if attempts == nil {
return admission.ErrCorruptRecord
}
for _, id := range ids {
if _, err := admission.ParseActionID(id); err != nil {
continue
}
raw := attempts.Get([]byte(id))
if raw == nil {
if attempts.Bucket([]byte(id)) != nil {
return admission.ErrCorruptRecord
}
continue
}
a, err := admission.UnmarshalAttempt(raw)
if err != nil {
return err
}
if string(a.Attempt.ID) != id {
return admission.ErrCorruptRecord
}
kept, err := keep(tx, a)
if err != nil {
return err
}
if kept {
out = append(out, id)
}
}
return nil
})
if err != nil {
return nil, err
}
return out, nil
}
// ReadAdmissionClock returns the admission clock the ledger saved, without
// opening a handle: the daemon reads it before the owner starts, to bound
// the blocks the firewall restores. A database without a current ledger
// returns the zero clock.
func ReadAdmissionClock(db *DB) (admission.Clock, error) {
var c admission.Clock
err := db.bolt.View(func(tx *bolt.Tx) error {
if current, err := currentLedger(tx); !current {
return err
}
var err error
c, err = loadLedgerClock(tx.Bucket([]byte(admissionMetaBucket)))
return err
})
if err != nil {
return admission.Clock{}, err
}
return c, nil
}
// RetireUnproven retires the uncertainty of an unknown attempt once nothing
// it may have applied can remain. The owner calls it when the
// attempt's absolute expiry has passed and it found no element of the
// attempt in a consistent snapshot; the ledger checks the expiry on a
// current, trusted clock, and waits until the outcome's rows are delivered
// so the event follows the outcome. The outcome stays unknown and counts
// as nothing new; nothing is dispatched again. The intent goes, the history
// and intent leave the recovery reserve for the allowance and the history
// may be retired after the review window, and an expired_unproven row
// records the event in the slot the outcome held. On an attempt already
// retired it is a no-op that reports false.
func (l *AdmissionLedger) RetireUnproven(id admission.ActionID) (admission.Candidate, admission.AttemptRecord, bool, error) {
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return admission.Candidate{}, admission.AttemptRecord{}, false, err
}
if l.degraded {
return admission.Candidate{}, admission.AttemptRecord{}, false, refusal(admission.ReasonEngineUnavailable, "a degraded clock reading cannot prove an expiry")
}
var c admission.Candidate
var a admission.AttemptRecord
var granted bool
err = l.update("retire", func(tx *bolt.Tx) error {
var txErr error
if a, c, txErr = attemptAndCandidate(tx, id); txErr != nil {
return txErr
}
switch {
case a.State != admission.StateUnknown:
return admission.ErrTransitionConflict
case a.Retired:
return nil
case now.Before(a.ExpiresAt):
return admission.ErrNotReady
}
pending, txErr := auditPending(tx, a.Attempt.Candidate)
if txErr != nil {
return txErr
}
if pending {
return admission.ErrNotReady
}
q, txErr := l.openQueue(tx, now)
if txErr != nil {
return txErr
}
if txErr = q.deleteIntent(id); txErr != nil {
return txErr
}
h, found, txErr := loadHistoryEntry(tx, a.Attempt.Candidate)
if txErr != nil {
return txErr
}
if !found || !h.Pinned {
return admission.ErrCorruptRecord
}
s, txErr := q.storageState()
if txErr != nil {
return txErr
}
if *s, txErr = s.UnpinHistory(h.General, h.Reserved); txErr != nil {
return txErr
}
q.storageDirty = true
h.Pinned = false
h.Eligible, _ = admission.HistoryTimes(h.Ended, c.ExpiresAt, admission.HistoryUntilExpiry(c))
a.Retired = true
// The event row names the retirement; the attempt keeps its
// unknown outcome. Its slot was held since that outcome.
event := a
event.Disposition = admission.DispositionExpiredUnproven
if txErr = q.writeAuditRow(c, event, admission.Tier{}); txErr != nil {
return txErr
}
c.Transitions++
if txErr = putHistoryEntry(tx, a.Attempt.Candidate, h); txErr != nil {
return txErr
}
if txErr = putAttempt(tx, a); txErr != nil {
return txErr
}
if txErr = putCandidate(tx, c); txErr != nil {
return txErr
}
granted = true
return q.flush()
})
if err != nil {
return admission.Candidate{}, admission.AttemptRecord{}, false, err
}
return c, a, granted, nil
}
// heldTarget derives the exact-key recovery hold independently of the
// current episode row, which may no longer name earlier live work (I1).
// Every unresolved attempt holds its target: an unknown outcome until
// recovery retires it, and a running attempt, which a crash before Settle or
// a Finish that failed to commit can leave behind, until its outcome is
// recorded. A transaction of the ledger's handle reads the handle's index
// as it stood when the transaction began; one without it derives the hold
// from the intents and their attempts once. Neither sees the transaction's
// own intent changes; none of their callers' lookups changes one.
func (q *queueTx) heldTarget(target admission.Target) (bool, error) {
intents := q.tx.Bucket([]byte(admissionIntentsBucket))
if intents == nil {
return false, admission.ErrCorruptRecord
}
if q.held != nil {
return q.held[target.Key()] > 0, nil
}
if q.heldTargets == nil {
targets := map[string]bool{}
if err := intents.ForEach(func(k, v []byte) error {
in, err := admission.UnmarshalIntent(v)
if err != nil {
return err
}
if in.Action != admission.ActionID(k) {
return admission.ErrCorruptRecord
}
a, err := loadAttempt(q.tx, in.Action)
if err != nil {
return corruptRecord(err)
}
if unsettled(a) {
targets[in.Target.Key()] = true
}
return nil
}); err != nil {
return false, err
}
q.heldTargets = targets
}
return q.heldTargets[target.Key()], nil
}
package store
import (
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// liveStartKey holds the live start record in the queue state bucket.
var liveStartKey = []byte("live_start")
func loadLiveStart(tx *bolt.Tx) (admission.LiveStart, bool, error) {
raw := tx.Bucket([]byte(admissionQueueStateBucket)).Get(liveStartKey)
schema := tx.Bucket([]byte(admissionMetaBucket)).Get(admissionSchemaKey)
if len(schema) != 1 || schema[0] < admissionSchemaVersion || schema[0] > admissionSchemaNewest {
return admission.LiveStart{}, false, ErrAdmissionSchema
}
// Only schema 13 can precede or follow the live start. Schema 14
// requires it again because only live effect records can turn gone.
if schema[0] != admissionSchemaEpisodeElapsed && (schema[0] >= admissionSchemaLive) != (raw != nil) {
return admission.LiveStart{}, false, admission.ErrCorruptRecord
}
if raw == nil {
return admission.LiveStart{}, false, nil
}
s, err := admission.UnmarshalLiveStart(raw)
return s, err == nil, err
}
// LiveStarted reports whether the ledger's live start ran.
func (l *AdmissionLedger) LiveStarted() bool {
l.mu.Lock()
defer l.mu.Unlock()
return l.live
}
// StartLive runs once, when the owner first executes, in one transaction:
// it carries the greater of the legacy hour's count and the charges
// retained in its hour, leaves pending legacy entries alone, ends earlier
// observe previews' answers, clears their notices, makes their history
// eligible, and records the start with the live feature bit, which older
// builds refuse. It also ends the block_ip work the preview period left
// queued.
// A ledger that has started refuses with ErrTransitionConflict.
// It needs a reading this handle recorded.
func (l *AdmissionLedger) StartLive(h admission.LegacyHandover) error {
l.mu.Lock()
defer l.mu.Unlock()
if l.live {
return admission.ErrTransitionConflict
}
now, clockErr := l.clock()
if clockErr != nil {
return clockErr
}
err := l.update("live start", func(tx *bolt.Tx) error {
if _, found, loadErr := loadLiveStart(tx); loadErr != nil || found {
if found {
return admission.ErrTransitionConflict
}
return loadErr
}
q, err := l.openQueue(tx, now)
if err != nil {
return err
}
if err = q.endPreviewQueue(); err != nil {
return err
}
if err = q.endPreviewAnswers(); err != nil {
return err
}
if err = q.clearNotices(); err != nil {
return err
}
if err = q.releasePreviewHistory(); err != nil {
return err
}
if err = carryLegacySpend(tx, h.Spend); err != nil {
return err
}
data, err := admission.LiveStart{At: now}.MarshalBinary()
if err != nil {
return err
}
if err = tx.Bucket([]byte(admissionQueueStateBucket)).Put(liveStartKey, data); err != nil {
return err
}
if err = raiseSchema(tx, admissionSchemaLive); err != nil {
return err
}
// Ledgers already at a later every-host schema gain nothing from the
// raise; the feature bit is what earlier builds refuse.
if err = addFeature(tx, featureLive); err != nil {
return err
}
return q.flush()
})
if err == nil {
l.live = true
}
return err
}
// carryLegacySpend charges what the legacy hour counted beyond the charges
// retained in it: the window that ends at the hour's end, which may include
// the first import's charges for the same hour.
func carryLegacySpend(tx *bolt.Tx, spend admission.LegacySpend) error {
if err := keepPreLiveCharges(tx); err != nil {
return err
}
s, err := loadCeiling(tx)
if err != nil {
return err
}
charges, err := loadCharges(tx)
if err != nil {
return err
}
var retained uint32
from := spend.At.Add(-admission.CeilingWindow)
for _, c := range charges {
if c.At.After(from) && !c.At.After(spend.At) {
retained += c.Cost
}
}
next, carried, err := s.CarryLegacy(spend, retained)
if err != nil {
return err
}
if err = putCharges(tx, carried); err != nil {
return err
}
return putCeilingState(tx, next)
}
// endPreviewQueue ends the block_ip work the preview period left queued,
// deferred or not: legacy blocking answered every block_ip finding while
// admission previewed. It ends silently, as disabled, with no new attempt
// or charge; prior attempts remain, and work of other kinds stays queued.
func (q *queueTx) endPreviewQueue() error {
live, err := q.live()
if err != nil {
return err
}
for _, lc := range live {
if lc.c.State == admission.StateQueued && lc.c.Key.Kind == admission.KindBlockIP {
if err = q.refuse(lc, admission.ReasonDisabled, 0); err != nil {
return err
}
}
}
return nil
}
// endPreviewAnswers makes placement's forecast rule for preview answers
// durable (section 8, row 4): a block line a preview answered ends, so the
// next observation queues a new generation, and a row no candidate anchors
// any longer is deleted, so its target's next observation opens a new
// episode. Challenge lines keep their previews' answers, and a remembered
// answer of a verified block keeps its episode's verified end.
func (q *queueTx) endPreviewAnswers() error {
episodes := q.tx.Bucket([]byte(admissionEpisodesBucket))
type change struct {
key string
row admission.Episode
}
var changes []change
err := episodes.ForEach(func(k, v []byte) error {
e, err := admission.UnmarshalEpisode(v)
if err != nil {
return err
}
ended := false
for i, line := range e.Lines {
if line.Kind == admission.KindChallenge {
continue
}
if line.Candidate == "" {
if line.Answered && e.Verified.IsZero() {
e.Lines[i].Answered, ended = false, true
}
continue
}
c, err := loadCandidate(q.tx, line.Candidate)
if err != nil {
return corruptRecord(err)
}
if c.State == admission.StateObserved {
e.Lines[i].Candidate, ended = "", true
}
}
if ended {
changes = append(changes, change{string(k), e})
}
return nil
})
if err != nil {
return err
}
for _, c := range changes {
if !c.row.HasCandidates() {
err = episodes.Delete([]byte(c.key))
} else {
err = q.putEpisode(c.key, c.row)
}
if err != nil {
return err
}
}
return nil
}
// clearNotices removes every notice record raised while admission only
// previewed, delivered or not: their gaps were legacy blocking's to close,
// and a preview delivery must not delay a key's first live notice. The
// fixed records stay, empty.
func (q *queueTx) clearNotices() error {
outbox := q.tx.Bucket([]byte(admissionOutboxBucket))
var quiet, records [][]byte
cur := outbox.Cursor()
for k, _ := cur.Seek([]byte{'n'}); k != nil && k[0] == 'n'; k, _ = cur.Next() {
records = append(records, k)
}
for k, _ := cur.Seek([]byte{'q'}); k != nil && k[0] == 'q'; k, _ = cur.Next() {
quiet = append(quiet, k)
}
for _, k := range quiet {
if err := outbox.Delete(k); err != nil {
return err
}
}
for _, kb := range records {
key, err := admission.ParseNoticeKey(kb)
if err != nil {
return err
}
if key.Fixed() {
if err = putNoticeRecord(q.tx, admission.NewNoticeRecord(key)); err != nil {
return err
}
continue
}
if err = outbox.Delete(kb); err != nil {
return err
}
s, err := q.storageState()
if err != nil {
return err
}
if *s, err = s.RemoveNotice(); err != nil {
return err
}
q.storageDirty = true
}
return nil
}
// releasePreviewHistory makes the history of every ended observe preview
// eligible now (design C7): a preview carries no review or undo, so
// pressure retires it before genuine history. Its index keys move with it;
// one whose audit rows wait gets them on acknowledgement.
func (q *queueTx) releasePreviewHistory() error {
var ids []admission.CandidateID
err := q.tx.Bucket([]byte(admissionHistoryBucket)).ForEach(func(k, v []byte) error {
h, err := admission.UnmarshalHistoryEntry(v)
if err != nil {
return err
}
if h.Ended.IsZero() || h.Pinned || !h.Eligible.After(q.now) {
return nil
}
c, err := loadCandidate(q.tx, admission.CandidateID(k))
if err != nil {
return corruptRecord(err)
}
if !admission.HistoryUntilExpiry(c) {
ids = append(ids, admission.CandidateID(k))
}
return nil
})
if err != nil {
return err
}
retire := q.tx.Bucket([]byte(admissionRetireBucket))
for _, id := range ids {
h, _, err := loadHistoryEntry(q.tx, id)
if err != nil {
return err
}
keys, err := retireKeysOf(q.tx, id, h)
if err != nil {
return err
}
for _, k := range keys {
if err = retire.Delete(k); err != nil {
return err
}
}
h.Released, h.Eligible = true, q.now
if err = putHistoryEntry(q.tx, id, h); err != nil {
return err
}
}
return nil
}
// keepPreLiveCharges makes the greater-of balance survive later forecasts
// and early preview-history retirement. Preserve each charge's lane and
// wall/elapsed deadlines; the handover must not grant a refund.
func keepPreLiveCharges(tx *bolt.Tx) error {
charges, err := loadCharges(tx)
if err != nil {
return err
}
attempts := tx.Bucket([]byte(admissionAttemptsBucket))
bucket := tx.Bucket([]byte(admissionChargesBucket))
for _, c := range charges {
if attempts.Get([]byte(c.Action)) == nil {
continue
}
a, err := loadAttempt(tx, c.Action)
if err != nil {
return err
}
candidate, err := loadCandidate(tx, a.Attempt.Candidate)
if err != nil || !c.At.Equal(a.Reserved) || c.Lane != a.Lane || c.Cost != candidate.Key.Kind.CeilingCost() {
return admission.ErrCorruptRecord
}
if a.State != admission.StateObserved {
continue
}
key, err := c.Key()
if err != nil {
return err
}
if err = bucket.Delete(key); err != nil {
return err
}
c.Action = admission.LiveStartPreviewActionID(c.Action)
if err = putCharges(tx, []admission.Charge{c}); err != nil {
return err
}
}
return nil
}
package store
import (
"errors"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// quietRemovalsPerTick bounds the quiet notice records one Tick removes.
const quietRemovalsPerTick = 64
func loadNoticeRecord(tx *bolt.Tx, k admission.NoticeKey) (admission.NoticeRecord, bool, error) {
kb, err := k.Bytes()
if err != nil {
return admission.NoticeRecord{}, false, err
}
raw := tx.Bucket([]byte(admissionOutboxBucket)).Get(kb)
if raw == nil {
return admission.NoticeRecord{}, false, nil
}
r, err := admission.UnmarshalNoticeRecord(raw)
if err != nil || r.Key != k {
return admission.NoticeRecord{}, false, admission.ErrCorruptRecord
}
return r, true, nil
}
// storeNoticeRecord writes r after old, keeping the quiet index in step:
// a record is indexed while all its events are delivered.
func storeNoticeRecord(tx *bolt.Tx, old, r admission.NoticeRecord) error {
outbox := tx.Bucket([]byte(admissionOutboxBucket))
if at := old.QuietAt(); !at.IsZero() {
k, err := admission.QuietKey(at, old.Key)
if err != nil {
return err
}
if outbox.Get(k) == nil {
return admission.ErrCorruptRecord
}
if err = outbox.Delete(k); err != nil {
return err
}
}
if at := r.QuietAt(); !at.IsZero() {
k, err := admission.QuietKey(at, r.Key)
if err != nil {
return err
}
if err = outbox.Put(k, []byte{}); err != nil {
return err
}
}
return putNoticeRecord(tx, r)
}
// raise records one notice event under key: in its own record, or, when a
// new key finds no room in the notice share, in its kind's overflow
// record. A Critical event also counts in the Critical summary, except a
// partial containment: its response ran. A zero candidate names no example.
func (q *queueTx) raise(key admission.NoticeKey, id admission.CandidateID, transitions uint32) error {
return q.notify(key, func(r admission.NoticeRecord) (admission.NoticeRecord, error) {
return r.Add(q.now, id, transitions)
})
}
// raiseCount records n events under key without examples, as raise does.
func (q *queueTx) raiseCount(key admission.NoticeKey, n uint64) error {
return q.notify(key, func(r admission.NoticeRecord) (admission.NoticeRecord, error) {
return r.AddCount(q.now, n)
})
}
func (q *queueTx) notify(key admission.NoticeKey, add func(admission.NoticeRecord) (admission.NoticeRecord, error)) error {
r, found, err := loadNoticeRecord(q.tx, key)
if err != nil {
return err
}
if !found {
if key.Fixed() {
return admission.ErrCorruptRecord
}
// A damaged storage record cannot pay for a new record, but the
// transition may still be one allowed over it: the event then counts in the fixed overflow record.
allocated, allocErr := q.allocateNotice()
if allocErr != nil && !errors.Is(allocErr, admission.ErrCorruptRecord) {
return allocErr
}
if allocated {
r = admission.NewNoticeRecord(key)
} else if r, found, err = loadNoticeRecord(q.tx, admission.OverflowKey(key.Kind)); err != nil || !found {
return admission.ErrCorruptRecord
}
}
next, err := add(r)
if err != nil {
return err
}
if err = storeNoticeRecord(q.tx, r, next); err != nil {
return err
}
if !key.Kind.Critical() || key.Partial() {
return nil
}
return q.notify(admission.NoticeKey{Kind: admission.NoticeCriticalSummary}, add)
}
// allocateNotice charges a new notice record to the notice share when the
// share and the reserve's room both allow it. The share is checked first:
// the room walks every live candidate.
func (q *queueTx) allocateNotice() (bool, error) {
s, err := q.storageState()
if err != nil {
return false, err
}
next, ok := s.AddNotice()
if !ok {
return false, nil
}
room, err := q.recoveryRoom()
if err != nil || room < admission.NoticeSlotBytes {
return false, err
}
*s, q.storageDirty = next, true
return true, nil
}
// noticeCheck is the check of c's highest-severity root, the lexically
// first on a tie: the check that made the work as severe as it is.
func noticeCheck(tx *bolt.Tx, c admission.Candidate) (string, error) {
var check string
var sev admission.Severity
for _, id := range c.Roots {
e, err := loadStoredEvidence(tx, id)
if err != nil {
return "", corruptRecord(err)
}
if e.Severity() > sev || (e.Severity() == sev && e.Check() < check) {
check, sev = e.Check(), e.Severity()
}
}
return check, nil
}
// partial raises the partial-containment record of applied block work
// whose root is a Critical destination: the block drops replies from the
// address, but outbound-only traffic to it still leaves.
func (q *queueTx) partial(d admission.Disposition, c admission.Candidate, id admission.CandidateID, transitions uint32) error {
if d != admission.DispositionApplied {
return nil
}
for _, root := range c.Roots {
e, err := loadStoredEvidence(q.tx, root)
if err != nil {
return corruptRecord(err)
}
if e.Family().Destination() && e.Severity() == admission.SeverityCritical {
return q.raise(admission.PartialContainmentKey(e.Check(), c.Key.Kind.Effect()), id, transitions)
}
}
return nil
}
// gap raises the notice an event of candidate c raises, if any.
// transitions is c's count after the event.
func (q *queueTx) gap(event admission.GapEvent, reason admission.Reason, outcome admission.Disposition, entry admission.QueueEntry, id admission.CandidateID, c admission.Candidate, transitions uint32) error {
kind := admission.GapNotice(event, reason, outcome, entry.Tier, q.started)
if kind == 0 {
return nil
}
check, err := noticeCheck(q.tx, c)
if err != nil {
return err
}
return q.raise(admission.NoticeKey{Kind: kind, Reason: reason, Outcome: outcome, Check: check, Effect: c.Key.Kind.Effect()}, id, transitions)
}
// removeQuietNotices removes keyed records whose events were all delivered
// an interval ago, oldest first, at most quietRemovalsPerTick.
func (q *queueTx) removeQuietNotices() error {
outbox := q.tx.Bucket([]byte(admissionOutboxBucket))
cur := outbox.Cursor()
for n := 0; n < quietRemovalsPerTick; n++ {
k, _ := cur.Seek([]byte{'q'})
if k == nil || k[0] != 'q' {
return nil
}
at, key, err := admission.ParseQuietKey(k)
if err != nil {
return err
}
if at.After(q.now) {
return nil
}
r, found, err := loadNoticeRecord(q.tx, key)
if err != nil {
return err
}
if !found || !r.QuietAt().Equal(at) {
return admission.ErrCorruptRecord
}
kb, _ := key.Bytes()
if err = cur.Delete(); err != nil {
return err
}
if err = outbox.Delete(kb); err != nil {
return err
}
s, err := q.storageState()
if err != nil {
return err
}
if *s, err = s.RemoveNotice(); err != nil {
return err
}
q.storageDirty = true
}
return nil
}
// PendingNotices returns the notice records due at the stored admission
// time: a key at most once an hour, each summary at most once a minute.
// The sender delivers them and acknowledges what it delivered.
func (l *AdmissionLedger) PendingNotices() ([]admission.NoticeRecord, error) {
var due []admission.NoticeRecord
err := l.db.bolt.View(func(tx *bolt.Tx) error {
clock, err := loadLedgerClock(tx.Bucket([]byte(admissionMetaBucket)))
if err != nil {
return err
}
cur := tx.Bucket([]byte(admissionOutboxBucket)).Cursor()
for k, v := cur.Seek([]byte{'n'}); k != nil && k[0] == 'n'; k, v = cur.Next() {
key, err := admission.ParseNoticeKey(k)
if err != nil {
return err
}
r, err := admission.UnmarshalNoticeRecord(v)
if err != nil || r.Key != key {
return admission.ErrCorruptRecord
}
if r.Due(clock.Now()) {
due = append(due, r)
}
}
return nil
})
if err != nil {
return nil, err
}
return due, nil
}
// AckNotices records deliveries in one transaction, each covering the
// count it names; events that arrived after the read stay pending. A
// record already removed is skipped.
func (l *AdmissionLedger) AckNotices(acks []admission.NoticeAck) error {
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.recordedClock()
if err != nil {
return err
}
return l.update("notices", func(tx *bolt.Tx) error {
for _, a := range acks {
r, found, err := loadNoticeRecord(tx, a.Key)
if err != nil {
return err
}
if !found || !r.First.Equal(a.First) {
continue
}
next, err := r.Ack(a.Count, now)
if err != nil {
return err
}
if err = storeNoticeRecord(tx, r, next); err != nil {
return err
}
}
return nil
})
}
package store
import (
"bytes"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
func putNoticeRecord(tx *bolt.Tx, r admission.NoticeRecord) error {
k, err := r.Key.Bytes()
if err != nil {
return err
}
data, err := r.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionOutboxBucket)).Put(k, data)
}
// putFixedNotices writes the empty records every ledger keeps: one
// overflow record per keyed kind and the two summaries.
func putFixedNotices(tx *bolt.Tx) error {
for _, k := range admission.FixedNoticeKeys() {
if err := putNoticeRecord(tx, admission.NewNoticeRecord(k)); err != nil {
return err
}
}
return nil
}
// upgradeLedgerToSchemaFive adds the outbox and the outcome buckets to a
// schema 4 ledger inside the opening transaction. The outbox starts with
// the fixed notice records, charged to the reserve, and holds the slots of
// the rows outstanding attempts may still write; no row or outcome is
// invented for earlier transitions.
func upgradeLedgerToSchemaFive(tx *bolt.Tx) error {
for _, name := range admissionOutboxBuckets {
if _, err := tx.CreateBucket([]byte(name)); err != nil {
return err
}
}
s, err := loadStorageState(tx)
if err != nil {
return err
}
s.NoticeRecords = admission.FixedNotices
// Outstanding attempts will write the rows of their remaining steps.
if err = tx.Bucket([]byte(admissionAttemptsBucket)).ForEach(func(_, v []byte) error {
a, decodeErr := admission.UnmarshalAttempt(v)
if decodeErr != nil {
return decodeErr
}
s.AuditSlots += futureAuditSlots(a.State)
return nil
}); err != nil {
return err
}
if err = putStorageState(tx, s); err != nil {
return err
}
return putFixedNotices(tx)
}
// proveOutbox checks every outbox row against the storage record: each
// notice record decodes under its own key and the fixed ones exist, each
// audit row decodes under its own key and belongs to a recorded attempt
// and a transition its candidate made, and the record counts exactly the
// rows stored and the slots outstanding attempts hold.
func proveOutbox(tx *bolt.Tx, s admission.StorageState) error {
clock, clockErr := loadLedgerClock(tx.Bucket([]byte(admissionMetaBucket)))
if clockErr != nil {
return clockErr
}
now := clock.Now()
fixed := map[admission.NoticeKey]bool{}
for _, k := range admission.FixedNoticeKeys() {
fixed[k] = true
}
var notices, rows uint64
last := map[admission.ActionID]admission.AuditRow{}
type transitions struct{ first, last uint32 }
attempts := map[admission.CandidateID][admission.MaxAttempts]transitions{}
quiet := map[admission.NoticeKey]time.Time{}
var quietRecords int
outbox := tx.Bucket([]byte(admissionOutboxBucket))
err := outbox.ForEach(func(k, v []byte) error {
if len(k) == 0 || outbox.Bucket(k) != nil {
return admission.ErrCorruptRecord
}
switch k[0] {
case 'q':
at, key, err := admission.ParseQuietKey(k)
if err != nil || len(v) != 0 {
return admission.ErrCorruptRecord
}
if _, dup := quiet[key]; dup {
return admission.ErrCorruptRecord
}
quiet[key] = at
case 'n':
key, err := admission.ParseNoticeKey(k)
if err != nil {
return err
}
r, err := admission.UnmarshalNoticeRecord(v)
if err != nil || r.Key != key {
return admission.ErrCorruptRecord
}
if r.Count > 0 && (now.IsZero() || r.Last.After(now) || r.Sent.After(now)) {
return admission.ErrCorruptRecord
}
delete(fixed, key)
if !r.QuietAt().IsZero() {
quietRecords++
}
notices++
case 'a':
row, err := admission.UnmarshalAuditRow(v)
if err != nil || !bytes.Equal(row.Key(), k) {
return admission.ErrCorruptRecord
}
a, err := loadAttempt(tx, row.Attempt.ID)
if err != nil {
return corruptRecord(err)
}
c, err := loadCandidate(tx, row.Attempt.Candidate)
if err != nil || row.Transition > c.Transitions {
return admission.ErrCorruptRecord
}
if now.IsZero() || !auditStepMatches(row, a, c, now) {
return admission.ErrCorruptRecord
}
if previous, found := last[row.Attempt.ID]; found && !auditFollows(previous, row) {
return admission.ErrCorruptRecord
}
last[row.Attempt.ID] = row
// Attempt IDs sort independently of lifecycle order. Their
// transition ranges must still be disjoint and follow retries.
bounds := attempts[row.Attempt.Candidate]
for seq := uint32(1); seq <= admission.MaxAttempts; seq++ {
bound := bounds[seq-1]
if bound.first != 0 && ((seq < row.Attempt.Seq && bound.last >= row.Transition) ||
(seq > row.Attempt.Seq && bound.first <= row.Transition)) {
return admission.ErrCorruptRecord
}
}
bound := &bounds[row.Attempt.Seq-1]
if bound.first == 0 {
bound.first = row.Transition
}
bound.last = row.Transition
attempts[row.Attempt.Candidate] = bounds
rows++
default:
return admission.ErrCorruptRecord
}
return nil
})
if err != nil {
return err
}
// The quiet index names exactly the records whose events were all
// delivered, at the time each becomes quiet.
if len(quiet) != quietRecords {
return admission.ErrCorruptRecord
}
for key, at := range quiet {
r, found, loadErr := loadNoticeRecord(tx, key)
if loadErr != nil || !found || !r.QuietAt().Equal(at) {
return admission.ErrCorruptRecord
}
}
// Each attempt holds a slot for every row it wrote and every row it may
// still write, and has written no more rows than steps it took.
future := uint64(0)
if err = tx.Bucket([]byte(admissionAttemptsBucket)).ForEach(func(_, v []byte) error {
a, decodeErr := admission.UnmarshalAttempt(v)
if decodeErr != nil {
return decodeErr
}
// Delivery removes the event row, but cannot remove the clock's
// proof that retirement followed the expiry and the outcome.
if a.Retired && (now.Before(a.ExpiresAt) || now.Before(a.Finished)) {
return admission.ErrCorruptRecord
}
future += heldAuditSlots(a)
return nil
}); err != nil {
return err
}
if len(fixed) != 0 || notices != s.NoticeRecords || rows+future != s.AuditSlots {
return admission.ErrCorruptRecord
}
return nil
}
// proveOutcomes checks every outcome bucket: it decodes, and it lies in
// its span's window at the stored clock. Before the first reading there is
// none.
func proveOutcomes(tx *bolt.Tx, now time.Time) error {
return tx.Bucket([]byte(admissionWindowsBucket)).ForEach(func(k, v []byte) error {
span, start, err := admission.ParseSpanKey(k)
if err != nil {
return err
}
if now.IsZero() || start.Before(span.Oldest(now)) || start.After(span.Start(now)) {
return admission.ErrCorruptRecord
}
_, err = admission.UnmarshalOutcomeCounts(v)
return err
})
}
// flushOutcomes adds the transaction's counted events to the bucket of
// each span that holds the transaction's time.
func (q *queueTx) flushOutcomes() error {
if len(q.outcomes.Rows()) == 0 {
return nil
}
windows := q.tx.Bucket([]byte(admissionWindowsBucket))
for _, span := range admission.Spans() {
k := span.Key(span.Start(q.now))
var c admission.OutcomeCounts
if raw := windows.Get(k); raw != nil {
var err error
if c, err = admission.UnmarshalOutcomeCounts(raw); err != nil {
return err
}
}
c.Merge(q.outcomes)
data, err := c.MarshalBinary()
if err != nil {
return err
}
if err = windows.Put(k, data); err != nil {
return err
}
}
return nil
}
// pruneOutcomes drops the buckets that have left their span's window at
// now. Each span keeps a bounded number of buckets, so this is bounded.
func pruneOutcomes(tx *bolt.Tx, now time.Time) error {
cur := tx.Bucket([]byte(admissionWindowsBucket)).Cursor()
for _, span := range admission.Spans() {
oldest := span.Key(span.Oldest(now))
prefix := oldest[:2]
for k, _ := cur.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix) && bytes.Compare(k, oldest) < 0; k, _ = cur.Seek(prefix) {
if err := cur.Delete(); err != nil {
return err
}
}
}
return nil
}
package store
import (
"errors"
"sort"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
var (
queueStateKey = []byte("queue")
queueCountersKey = []byte("counters")
scheduleStateKey = []byte("schedule")
ingressStateKey = []byte("ingress")
)
func initializeQueueState(b *bolt.Bucket) error {
for _, rec := range []struct {
key []byte
value interface{ MarshalBinary() ([]byte, error) }
}{
{queueStateKey, admission.QueueState{}},
{queueCountersKey, admission.QueueCounters{}},
{scheduleStateKey, admission.ScheduleState{}},
{ingressStateKey, admission.IngressState{}},
} {
data, err := rec.value.MarshalBinary()
if err != nil {
return err
}
if err = b.Put(rec.key, data); err != nil {
return err
}
}
return nil
}
// upgradeLedgerToSchemaTwo adds the queue buckets to a schema 1 ledger
// inside the opening transaction. Every live candidate gets an unassessed
// entry holding a general position; the first current reading assesses and
// places it. A damaged candidate or attempt history refuses the upgrade,
// and the transaction leaves the schema 1 ledger exactly as it was. The
// upgrade that completes the chain records the schema.
func upgradeLedgerToSchemaTwo(tx *bolt.Tx) error {
for _, name := range admissionQueueBuckets {
if _, err := tx.CreateBucket([]byte(name)); err != nil {
return err
}
}
queue := tx.Bucket([]byte(admissionQueueBucket))
live := 0
var nextSweep time.Time
err := tx.Bucket([]byte(admissionCandidatesBucket)).ForEach(func(k, v []byte) error {
c, err := admission.UnmarshalCandidate(v)
if err != nil {
return err
}
if id, _ := c.ID(); string(id) != string(k) {
return admission.ErrCorruptRecord
}
if c.State.Terminal() {
return nil
}
// The queue reads a live candidate's attempt history on every walk,
// so the upgrade proves it now rather than commit an unusable queue.
if c.Attempts > 0 {
if _, err = currentAttempt(tx, c); err != nil {
return err
}
}
live++
if c.State == admission.StateQueued && (nextSweep.IsZero() || c.FirstQueued.Before(nextSweep)) {
nextSweep = c.FirstQueued
}
// Eligibility is unknown until a current reading. Every imported
// candidate must fit the general partition without borrowing reserve.
if live > admission.PartitionGeneral.DurableCapacity() {
return refusal(admission.ReasonQueueOverflow, "legacy queue exceeds unassessed capacity")
}
data, err := admission.QueueEntry{Partition: admission.PartitionGeneral}.MarshalBinary()
if err != nil {
return err
}
return queue.Put(k, data)
})
if err != nil {
return err
}
if err := initializeQueueState(tx.Bucket([]byte(admissionQueueStateBucket))); err != nil {
return err
}
// Unassessed queued work is due at the first current reading, even if
// an arrival reaches the ledger before an explicit recovery sweep.
return putQueueState(tx, admission.QueueState{NextSweep: nextSweep})
}
func loadQueueState(tx *bolt.Tx) (admission.QueueState, error) {
raw := tx.Bucket([]byte(admissionQueueStateBucket)).Get(queueStateKey)
if raw == nil {
return admission.QueueState{}, admission.ErrCorruptRecord
}
return admission.UnmarshalQueueState(raw)
}
func loadQueueCounters(tx *bolt.Tx) (admission.QueueCounters, error) {
raw := tx.Bucket([]byte(admissionQueueStateBucket)).Get(queueCountersKey)
if raw == nil {
return admission.QueueCounters{}, admission.ErrCorruptRecord
}
return admission.UnmarshalQueueCounters(raw)
}
// loadQueueEntry loads the entry of a live candidate. A live candidate
// without one, or an entry without its candidate, is a damaged ledger.
func loadQueueEntry(tx *bolt.Tx, id admission.CandidateID) (admission.QueueEntry, error) {
raw := tx.Bucket([]byte(admissionQueueBucket)).Get([]byte(id))
if raw == nil {
return admission.QueueEntry{}, admission.ErrCorruptRecord
}
return admission.UnmarshalQueueEntry(raw)
}
func putQueueState(tx *bolt.Tx, s admission.QueueState) error {
data, err := s.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionQueueStateBucket)).Put(queueStateKey, data)
}
func putQueueCounters(tx *bolt.Tx, q admission.QueueCounters) error {
data, err := q.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionQueueStateBucket)).Put(queueCountersKey, data)
}
func putQueueEntry(tx *bolt.Tx, id admission.CandidateID, q admission.QueueEntry) error {
data, err := q.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionQueueBucket)).Put([]byte(id), data)
}
// queueTx is the queue bookkeeping of one write transaction. State and
// counters load once, the view is built on first use, and flush writes back
// what changed, so every queue change commits with the candidate records it
// depends on.
type queueTx struct {
tx *bolt.Tx
reg *admission.Registry
inv *admission.Inventory
now time.Time
state admission.QueueState
counters admission.QueueCounters
view *admission.QueueView
// seq orders the arrivals decided in this transaction.
seq uint64
stateDirty, countsDirty bool
// storage is loaded on first use; flush trims its rings and writes it
// back if it changed.
storage admission.StorageState
storageLoaded, storageDirty bool
// outcomes are the events this transaction counts into the outcome
// windows when it flushes.
outcomes admission.OutcomeCounts
// held is the handle's recovery hold when the transaction began and
// heldDelta its pending changes (see AdmissionLedger.held). Without
// them, heldTargets is derived once from the intents, never episode
// rows.
held, heldDelta map[string]int
heldTargets map[string]bool
// refused collects the candidates refuse ends, for the handle to report
// once the transaction commits (AdmissionLedger.refused); nil reports
// nothing.
refused *[]refusedEnding
// started is the handle's live start (AdmissionLedger.live) and set
// the handle's set occupancy (AdmissionLedger.set). degraded is the
// handle's last reading (AdmissionLedger.degraded): no record rotates
// on it.
started, degraded bool
set *admission.SetOccupancy
imports []admission.EvictionVictim
// waiting is the handle's waiting scopes (AdmissionLedger.waiting) or a
// schedule cycle's own; shared is set once slots shared the set.
waiting map[admission.Partition][]string
shared bool
// executed is the handle's execute readings (AdmissionLedger.executed).
executed map[admission.ActionID]admission.EpisodeClock
}
func (l *AdmissionLedger) openQueue(tx *bolt.Tx, now time.Time) (*queueTx, error) {
q, err := openQueueWith(tx, l.reg, l.Inventory(), now)
if err != nil {
return nil, err
}
q.held, q.heldDelta, q.started, q.degraded, q.set, q.imports, q.waiting = l.held, l.heldDelta, l.live, l.degraded, l.set, l.imports, l.waiting
q.executed, q.refused = l.executed, &l.refused
return q, nil
}
func openQueueWith(tx *bolt.Tx, reg *admission.Registry, inv *admission.Inventory, now time.Time) (*queueTx, error) {
state, err := loadQueueState(tx)
if err != nil {
return nil, err
}
counters, err := loadQueueCounters(tx)
if err != nil {
return nil, err
}
return &queueTx{tx: tx, reg: reg, inv: inv, now: now, state: state, counters: counters}, nil
}
func (q *queueTx) flush() error {
if q.view != nil && q.view.Cursors() != q.state.Cursors {
q.state.Cursors, q.stateDirty = q.view.Cursors(), true
}
if q.stateDirty {
if err := putQueueState(q.tx, q.state); err != nil {
return err
}
}
if q.countsDirty {
if err := putQueueCounters(q.tx, q.counters); err != nil {
return err
}
}
if err := q.flushOutcomes(); err != nil {
return err
}
return q.flushStorage()
}
func queueItem(id admission.CandidateID, c admission.Candidate, e admission.QueueEntry) admission.QueueItem {
return admission.QueueItem{
Key: string(id), Scope: c.Scope.Key(), Partition: e.Partition, Tier: e.Tier, Eligible: e.Eligible(),
Queued: c.FirstQueued, Fixed: c.State != admission.StateQueued || !e.Assessed(),
}
}
type liveCandidate struct {
id admission.CandidateID
c admission.Candidate
entry admission.QueueEntry
// fresh is set by markFresh for work whose episode had no verified
// response.
fresh bool
}
// live loads every live candidate with its entry. An entry must name a
// live candidate: an ended or missing one is a damaged ledger.
func (q *queueTx) live() ([]liveCandidate, error) {
var out []liveCandidate
err := q.tx.Bucket([]byte(admissionQueueBucket)).ForEach(func(k, raw []byte) error {
entry, err := admission.UnmarshalQueueEntry(raw)
if err != nil {
return err
}
id := admission.CandidateID(k)
c, err := loadCandidate(q.tx, id)
if err != nil || c.State.Terminal() {
return admission.ErrCorruptRecord
}
if c.Attempts > 0 {
if _, err = currentAttempt(q.tx, c); err != nil {
return err
}
}
out = append(out, liveCandidate{id: id, c: c, entry: entry})
return nil
})
return out, err
}
func (q *queueTx) queueView() (*admission.QueueView, error) {
if q.view != nil {
return q.view, nil
}
all, err := q.live()
if err != nil {
return nil, err
}
v := admission.NewDurableQueueView(q.state.Cursors)
for _, lc := range all {
if err := v.Add(queueItem(lc.id, lc.c, lc.entry)); err != nil {
return nil, admission.ErrCorruptRecord
}
}
q.view = v
return v, nil
}
func (q *queueTx) count(event admission.QueueEvent, reason admission.Reason, tier admission.Tier) error {
q.countsDirty = true
if err := q.counters.Add(admission.CountKey{Event: event, Reason: reason, Class: tier.Class, Severity: tier.Severity}); err != nil {
return err
}
return q.outcomes.Add(admission.QueueOutcome(event, reason, tier))
}
// noteDeadlines brings the next sweep forward to the earliest time the
// queued candidate may need attention without a new report.
func (q *queueTx) noteDeadlines(c admission.Candidate, e admission.QueueEntry) {
due := c.AgeOut
if c.Attempts > 0 && c.ExpiresAt.Before(due) {
due = c.ExpiresAt
}
if !e.Assessed() {
due = q.now
} else if e.NextChange.Before(due) {
due = e.NextChange
}
if q.state.NextSweep.IsZero() || due.Before(q.state.NextSweep) {
q.state.NextSweep, q.stateDirty = due, true
}
}
// release deletes a candidate's entry when it leaves the queue for good and
// counts why. A queue event with no reason counts nothing.
func (q *queueTx) release(id admission.CandidateID, e admission.QueueEntry, event admission.QueueEvent, reason admission.Reason) error {
if err := q.tx.Bucket([]byte(admissionQueueBucket)).Delete([]byte(id)); err != nil {
return err
}
if q.view != nil {
q.view.Remove(string(id))
}
if event == 0 {
return nil
}
return q.count(event, reason, e.Tier)
}
// end ends a queued candidate for reason and releases its position.
func (q *queueTx) end(lc liveCandidate, reason admission.Reason) error {
state, ok := terminalFor[reason.Disposition()]
if !ok || lc.c.State != admission.StateQueued || !admission.CanTransition(lc.c.State, state) {
return admission.ErrTransitionConflict
}
c := lc.c
if err := q.trimUnreservedRoots(&c); err != nil {
return err
}
c.State, c.Disposition, c.Reason, c.NotBefore = state, reason.Disposition(), reason, time.Time{}
c.Transitions++
if err := putCandidate(q.tx, c); err != nil {
return err
}
if err := q.release(lc.id, lc.entry, admission.EventEnded, reason); err != nil {
return err
}
if err := q.gap(admission.GapEnded, reason, 0, lc.entry, lc.id, c, c.Transitions); err != nil {
return err
}
return q.ended(lc.id, c)
}
// refuse ends a queued candidate a check refused and keeps the refusal, with
// its cause, for the handle to report once the transaction commits.
func (q *queueTx) refuse(lc liveCandidate, reason admission.Reason, cause admission.Cause) error {
if err := q.end(lc, reason); err != nil {
return err
}
if q.refused != nil {
*q.refused = append(*q.refused, refusedEnding{c: lc.c, err: &admission.Error{Reason: reason, Cause: cause, Detail: "candidate no longer qualifies for a response"}, at: q.now})
}
return nil
}
// endByKey ends the queued candidate a view decision displaced and reports
// it as refused, with no attribution cause.
func (q *queueTx) endByKey(key string, reason admission.Reason) error {
id := admission.CandidateID(key)
c, err := loadCandidate(q.tx, id)
if err != nil {
return err
}
e, err := loadQueueEntry(q.tx, id)
if err != nil {
return err
}
return q.refuse(liveCandidate{id: id, c: c, entry: e}, reason, 0)
}
// entryFor turns an assessment into the entry fields it decides.
func entryFor(a admission.Assessment) admission.QueueEntry {
return admission.QueueEntry{Tier: a.Tier, Direct: a.DirectC3, Corroborated: a.Corroborated, NextChange: a.NextChange}
}
// insert decides a queue position for a new candidate and records its
// entry. A displaced candidate ends as queue overflow in the same
// transaction; a refused arrival is a queue overflow refusal.
func (q *queueTx) insert(id admission.CandidateID, c admission.Candidate, e admission.QueueEntry) error {
v, err := q.queueView()
if err != nil {
return err
}
q.seq++
it := queueItem(id, c, e)
it.Seq = q.seq
placed, ok := v.Admit(it)
if !ok {
return refusal(admission.ReasonQueueOverflow, "queue has no position for the candidate")
}
if placed.Displaced {
if err := q.endByKey(placed.Victim.Key, admission.ReasonQueueOverflow); err != nil {
return err
}
}
e.Partition = placed.Partition
q.noteDeadlines(c, e)
return putQueueEntry(q.tx, id, e)
}
// reassess records a new assessment for a queued candidate and returns the
// candidate as stored. One that lost its reserved eligibility leaves the
// reserved partition and is placed again among general work, where it can
// displace newer work or, finding no position, end as queue overflow. One
// that gained eligibility moves to a free reserved position.
func (q *queueTx) reassess(lc liveCandidate, next admission.QueueEntry) (admission.Candidate, error) {
v, err := q.queueView()
if err != nil {
return lc.c, err
}
next.Partition = lc.entry.Partition
v.Remove(string(lc.id))
switch {
case lc.entry.Partition == admission.PartitionReserved && !next.Eligible():
placed, ok := v.Admit(queueItem(lc.id, lc.c, next))
if !ok {
lc.entry = next
if err = q.refuse(lc, admission.ReasonQueueOverflow, 0); err != nil {
return lc.c, err
}
return loadCandidate(q.tx, lc.id)
}
if placed.Displaced {
if err = q.endByKey(placed.Victim.Key, admission.ReasonQueueOverflow); err != nil {
return lc.c, err
}
}
next.Partition = placed.Partition
case next.Eligible() && v.Room(admission.PartitionReserved):
next.Partition = admission.PartitionReserved
err = v.Add(queueItem(lc.id, lc.c, next))
default:
err = v.Add(queueItem(lc.id, lc.c, next))
}
if err != nil {
return lc.c, err
}
q.noteDeadlines(lc.c, next)
return lc.c, putQueueEntry(q.tx, lc.id, next)
}
// check revalidates a queued candidate against current policy and
// inventory and, when timed, assesses it at q.now. A refusal is the reason
// the candidate must end and, for an attribution refusal, its cause; any
// other error is returned. A root that is no longer published is damage,
// not a refusal: evidence is never removed.
func (q *queueTx) check(lc liveCandidate, timed bool) (admission.Assessment, admission.Reason, admission.Cause, error) {
c := lc.c
roots, err := loadRoots(q.tx, q.reg, c.Roots)
if errors.Is(err, admission.ErrEvidenceUnpublished) {
return admission.Assessment{}, 0, 0, admission.ErrCorruptRecord
}
if _, refused := admission.ReasonOf(err); err != nil && !refused {
return admission.Assessment{}, 0, 0, err
}
if timed && (!q.now.Before(c.AgeOut) || (c.Attempts > 0 && !q.now.Before(c.ExpiresAt))) {
return admission.Assessment{}, admission.ReasonStale, 0, nil
}
if err == nil {
err = validateCandidateEntry(q.reg, c, roots)
}
if err == nil {
_, err = scopeOwner(q.inv, roots)
}
var a admission.Assessment
if err == nil && timed {
a, err = admission.Assess(c.Key.Target, roots, q.now)
}
if reason, refused := admission.ReasonOf(err); refused {
return admission.Assessment{}, reason, admission.CauseOf(err), nil
}
return a, 0, 0, err
}
// revalidate checks queued candidates in queue order, oldest first. With
// all, every one is checked against policy, inventory and time; otherwise
// only those whose deadline has passed or that were never assessed. It then
// recomputes the next sweep from what is still queued.
func (q *queueTx) revalidate(all bool) error {
live, err := q.live()
if err != nil {
return err
}
if _, err = q.queueView(); err != nil {
return err
}
sort.Slice(live, func(i, j int) bool {
if !live[i].c.FirstQueued.Equal(live[j].c.FirstQueued) {
return live[i].c.FirstQueued.Before(live[j].c.FirstQueued)
}
return live[i].id < live[j].id
})
type assessedCandidate struct {
liveCandidate
next admission.QueueEntry
}
var staying, moving []assessedCandidate
for _, lc := range live {
if lc.c.State != admission.StateQueued {
continue
}
due := !lc.entry.Assessed() || !q.now.Before(lc.entry.NextChange) || !q.now.Before(lc.c.AgeOut) ||
(lc.c.Attempts > 0 && !q.now.Before(lc.c.ExpiresAt))
if !all && !due {
continue
}
a, reason, cause, checkErr := q.check(lc, true)
if checkErr != nil {
return checkErr
}
if reason != 0 {
if err = q.refuse(lc, reason, cause); err != nil {
return err
}
continue
}
next := entryFor(a)
if lc.entry.Partition == admission.PartitionReserved && !next.Eligible() {
q.view.Remove(string(lc.id))
moving = append(moving, assessedCandidate{lc, next})
} else {
staying = append(staying, assessedCandidate{lc, next})
}
}
// Release invalid work and refresh every retained tier before choosing
// victims. Otherwise an older demotion competes with expired occupancy
// or yesterday's priority, and can be dropped despite available room.
for _, pending := range append(staying, moving...) {
if _, err = q.reassess(pending.liveCandidate, pending.next); err != nil {
return err
}
}
return q.resweep()
}
// resweep recomputes the next sweep from the queued candidates.
func (q *queueTx) resweep() error {
live, err := q.live()
if err != nil {
return err
}
q.state.NextSweep, q.stateDirty = time.Time{}, true
for _, lc := range live {
if lc.c.State == admission.StateQueued {
q.noteDeadlines(lc.c, lc.entry)
}
}
return nil
}
// sweep revalidates the candidates whose deadlines have passed, if any.
func (q *queueTx) sweep() error {
if q.state.NextSweep.IsZero() || q.now.Before(q.state.NextSweep) {
return nil
}
return q.revalidate(false)
}
// checkOwners ends every queued candidate whose roots no longer resolve to
// a current owner or pass current policy. It needs no clock.
func (q *queueTx) checkOwners() error {
live, err := q.live()
if err != nil {
return err
}
for _, lc := range live {
if lc.c.State != admission.StateQueued {
continue
}
_, reason, cause, err := q.check(lc, false)
if err != nil {
return err
}
if reason != 0 {
if err = q.refuse(lc, reason, cause); err != nil {
return err
}
}
}
return nil
}
// Revalidate checks every queued candidate against current policy,
// inventory and the admission clock. A candidate that no longer qualifies
// ends with the refusal's reason; the others take their new assessment,
// which can move them between partitions.
func (l *AdmissionLedger) Revalidate() error {
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return err
}
err = l.update("revalidate", func(tx *bolt.Tx) error {
q, txErr := l.openQueue(tx, now)
if txErr != nil {
return txErr
}
if txErr = q.revalidate(true); txErr != nil {
return txErr
}
return q.flush()
})
if err == nil {
l.revalidated = true
}
return err
}
func loadScheduleState(tx *bolt.Tx) (admission.ScheduleState, error) {
raw := tx.Bucket([]byte(admissionQueueStateBucket)).Get(scheduleStateKey)
if raw == nil {
return admission.ScheduleState{}, admission.ErrCorruptRecord
}
return admission.UnmarshalScheduleState(raw)
}
func putScheduleState(tx *bolt.Tx, s admission.ScheduleState) error {
data, err := s.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionQueueStateBucket)).Put(scheduleStateKey, data)
}
// unitCost is the fair cost of one queued address, prefix or service tuple.
// Ceiling demand comes from the candidate kind.
const unitCost = 1
// Schedule picks the next candidates to serve under lim and records the
// scheduler's new position; the picks stay queued until the engine
// reserves or ends them. Each lane serves at most what the ceiling lets it
// charge and its history allowance can take now; work that would not fit
// the recovery reserve waits. The first schedule of a reopened ledger
// checks every queued candidate first; later ones end the candidates whose
// deadlines passed. Each pick is revalidated before it is returned: one
// that no longer qualifies ends, and its turn goes to the next candidate.
// Pending candidates wait on an outside answer, such as the panel's
// verdict: they are passed over as a retry in its backoff is, never picked
// or deferred, and their scopes keep their turns. An allowed candidate is
// picked as uncharged work whatever the budgets, and so is work of a kind
// a live ledger only previews.
func (l *AdmissionLedger) Schedule(lim admission.ScheduleLimits, answers map[admission.CandidateID]admission.Answer) ([]admission.Pick, error) {
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return nil, err
}
var picks []admission.Pick
var waiting map[admission.Partition][]string
if err = l.update("schedule", func(tx *bolt.Tx) error {
q, txErr := l.openQueue(tx, now)
if txErr != nil {
return txErr
}
if picks, txErr = l.scheduleTx(q, lim, answers); txErr != nil {
return txErr
}
waiting = q.waiting
return q.flush()
}); err != nil {
return nil, err
}
l.revalidated, l.waiting = true, waiting
return picks, nil
}
func (l *AdmissionLedger) scheduleTx(q *queueTx, lim admission.ScheduleLimits, answers map[admission.CandidateID]admission.Answer) ([]admission.Pick, error) {
// Revalidate directly rather than through sweep: an upgraded ledger's
// unassessed candidates are due before any deadline is recorded.
if err := q.revalidate(!l.revalidated); err != nil {
return nil, err
}
st, err := loadScheduleState(q.tx)
if err != nil {
return nil, err
}
lim, _, err = q.scheduleLimits(lim)
if err != nil {
return nil, err
}
live, err := q.live()
if err != nil {
return nil, err
}
items, byID, err := q.scheduleItems(live, lim.RecoveryBytes, answers)
if err != nil {
return nil, err
}
q.waiting = waitingScopes(items, byID)
slots, err := q.slots()
if err != nil {
return nil, err
}
// Each cycle under pressure moves the share cursors once, before any
// pick, so the reservations of its picks share the set as it did.
if q.shared {
if next := slots.Advance(); next != q.state.SetCursors {
q.state.SetCursors, q.stateDirty = next, true
}
}
if err = q.markFresh(byID); err != nil {
return nil, err
}
// A pick that fails revalidation ends, and the schedule is computed
// again from the stored position without it, so its turn is not spent.
// So is the schedule after a pick that earlier picks left no room: work
// that finds no room on any lane is not ready to the scheduler.
sched := slotReady(items, byID, slots)
for {
picks, next, schedErr := admission.Schedule(sched, st, limitBySlots(lim, slots))
if schedErr != nil {
return nil, schedErr
}
unfit := map[admission.CandidateID]admission.Lane{}
room := slots.Clone()
for _, p := range picks {
for _, it := range sched {
if it.ID == p.ID {
if _, ok := room.Take(p.Lane, demandOf(it, byID[p.ID])); !ok {
unfit[p.ID] = p.Lane
}
}
}
}
refused := map[admission.CandidateID]bool{}
for _, p := range picks {
_, reason, cause, checkErr := q.check(byID[p.ID], true)
if checkErr != nil {
return nil, checkErr
}
if reason == 0 {
continue
}
if checkErr = q.refuse(byID[p.ID], reason, cause); checkErr != nil {
return nil, checkErr
}
refused[p.ID] = true
}
if len(refused) == 0 && len(unfit) == 0 {
if err = q.deferUnserved(items, byID, picks, lim, slots, answers); err != nil {
return nil, err
}
return picks, putScheduleState(q.tx, next)
}
kept, keptSched := items[:0:0], sched[:0:0]
for i, it := range items {
if !refused[it.ID] {
kept = append(kept, it)
if lane, blocked := unfit[it.ID]; blocked {
if lane == admission.LaneGeneral {
sched[i].BlockedGeneral = true
} else {
sched[i].BlockedReserved = true
}
}
keptSched = append(keptSched, sched[i])
}
}
items, sched = kept, keptSched
if len(refused) > 0 {
// Refused picks consumed no room. Reconsider every surviving
// lane against the original pool and fair position.
sched = slotReady(items, byID, slots)
}
}
}
// previewReservedMembers bounds the reserved lane's share of a preview
// selection: an unlimited reserved lane could starve general work, and the
// bound is not ceiling credit.
const previewReservedMembers = 13
// SchedulePreview selects up to MaxBatchMembers queued candidates for a
// dry-run forecast and records the scheduler's new position. Selection is
// not admission: the live ceiling and history budgets do not bound it, and
// it reserves, charges and defers nothing for capacity; the forecast ends
// each pick. Every candidate weighs one fair unit and no history. A
// challenge's comparison preview spends live history, so a challenge the
// live history budget cannot pay for is passed over as pending work is,
// and one only its reserved lane can pay for is served on that lane; on a
// ledger that has started live its preview spends none and it is served
// as any other work. The
// reserved lane alternates direct and corroborated work for at most 13
// members, then the general lane serves the rest by class and severity;
// reserved-eligible work competes there too. Revalidation, age-out, retry
// backoff and recovery holds gate it as they gate Schedule, a held target
// is deferred as pending recovery, a stale live budget wait clears, and
// pending work is passed over.
func (l *AdmissionLedger) SchedulePreview(answers map[admission.CandidateID]admission.Answer) ([]admission.Pick, error) {
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return nil, err
}
var picks []admission.Pick
if err = l.update("preview", func(tx *bolt.Tx) error {
q, txErr := l.openQueue(tx, now)
if txErr != nil {
return txErr
}
if picks, txErr = l.previewTx(q, answers); txErr != nil {
return txErr
}
return q.flush()
}); err != nil {
return nil, err
}
l.revalidated = true
return picks, nil
}
func (l *AdmissionLedger) previewTx(q *queueTx, answers map[admission.CandidateID]admission.Answer) ([]admission.Pick, error) {
if err := q.revalidate(!l.revalidated); err != nil {
return nil, err
}
st, err := loadScheduleState(q.tx)
if err != nil {
return nil, err
}
live, err := q.live()
if err != nil {
return nil, err
}
if err = q.checkPreviewStorage(live); err != nil {
return nil, err
}
byID := map[admission.CandidateID]liveCandidate{}
held := map[admission.CandidateID]bool{}
var history liveHistory
var items []admission.ScheduleItem
for _, lc := range live {
if lc.c.State != admission.StateQueued {
continue
}
h, heldErr := q.heldTarget(lc.c.Key.Target)
if heldErr != nil {
return nil, heldErr
}
byID[lc.id], held[lc.id] = lc, h
item := admission.ScheduleItem{
ID: lc.id, Scope: lc.c.Scope.Key(), Tier: lc.entry.Tier, Direct: lc.entry.Direct, Corroborated: lc.entry.Corroborated,
Queued: lc.c.FirstQueued, Cost: unitCost, CeilingCost: unitCost,
Ready: !h && !q.now.Before(lc.c.NotBefore) && answers[lc.id] != admission.AnswerPending,
}
// A challenge preview spends history only until the ledger starts
// live; after that it ends uncharged and needs none.
if lc.c.Key.Kind == admission.KindChallenge && !q.started {
_, fits, general, reserved, judgeErr := history.challenge(q, lc)
if judgeErr != nil {
return nil, judgeErr
}
// Not ready: passed over without a wait or a fair turn.
item.Ready = item.Ready && fits && (general || reserved)
if !reserved {
item.Direct, item.Corroborated = false, false
}
}
items = append(items, item)
}
lim := admission.ScheduleLimits{General: admission.MaxBatchMembers, Reserved: previewReservedMembers, Members: admission.MaxBatchMembers}
// As in a schedule, a pick that fails revalidation ends and the
// selection is computed again from the stored position without it.
for {
picks, next, schedErr := admission.Schedule(items, st, lim)
if schedErr != nil {
return nil, schedErr
}
refused := map[admission.CandidateID]bool{}
remaining := history
skipped := map[admission.CandidateID]bool{}
for i, p := range picks {
_, reason, cause, checkErr := q.check(byID[p.ID], true)
if checkErr != nil {
return nil, checkErr
}
if reason == 0 {
if byID[p.ID].c.Key.Kind == admission.KindChallenge && !q.started {
lane, fits, budgetErr := remaining.take(q, byID[p.ID], p)
if budgetErr != nil {
return nil, budgetErr
}
if !fits {
// Recompute from the original fair position without
// spending a turn on an unaffordable comparison.
skipped[p.ID] = true
} else {
picks[i].Lane = lane
}
}
continue
}
if checkErr = q.refuse(byID[p.ID], reason, cause); checkErr != nil {
return nil, checkErr
}
refused[p.ID] = true
}
if len(refused) == 0 && len(skipped) == 0 {
if err = q.deferHeld(items, byID, held, answers); err != nil {
return nil, err
}
return picks, putScheduleState(q.tx, next)
}
kept := items[:0:0]
for _, it := range items {
if !refused[it.ID] {
if skipped[it.ID] {
it.Ready = false
}
kept = append(kept, it)
}
}
items = kept
}
}
// liveHistory is the live history budget a challenge's comparison preview
// spends, read once for a selection or a wake: what the recovery reserve
// has left and what each lane may charge now.
type liveHistory struct {
loaded bool
room, general, reserved uint64
}
// take accounts for earlier challenge picks in this selection, without
// charging the ledger or changing their unit fair-turn weight.
func (h *liveHistory) take(q *queueTx, lc liveCandidate, p admission.Pick) (admission.Lane, bool, error) {
bytes, recovery, fits, err := q.historyNeed(lc, h.room)
if err != nil || !fits {
return 0, false, err
}
lanes := []admission.Lane{p.Lane}
if p.Lane != admission.LaneGeneral {
lanes = append(lanes, admission.LaneGeneral)
} else if p.Reserved != 0 {
lanes = append(lanes, p.Reserved)
}
for _, lane := range lanes {
budget := &h.general
if lane != admission.LaneGeneral {
budget = &h.reserved
}
if uint64(bytes) <= *budget {
*budget -= uint64(bytes)
h.room -= uint64(recovery)
return lane, true, nil
}
}
return 0, false, nil
}
// challenge judges queued challenge lc as a schedule judges its item: the
// history bytes its comparison preview charges, whether the recovery
// reserve takes it, and whether the general lane and, when lc is eligible
// for one, its reserved lane can pay for it now.
func (h *liveHistory) challenge(q *queueTx, lc liveCandidate) (bytes uint32, fits, general, reserved bool, err error) {
if !h.loaded {
if h.room, err = q.recoveryRoom(); err != nil {
return 0, false, false, false, err
}
if h.general, h.reserved, err = q.historyBudgets(); err != nil {
return 0, false, false, false, err
}
h.loaded = true
}
if bytes, _, fits, err = q.historyNeed(lc, h.room); err != nil {
return 0, false, false, false, err
}
eligible := lc.entry.Direct || lc.entry.Corroborated
return bytes, fits, uint64(bytes) <= h.general, eligible && uint64(bytes) <= h.reserved, nil
}
// Preview ignores capacity pressure, not damaged accounting. Read the same
// ceiling, storage and live-history records as live scheduling without granting
// any history or recovery budget.
func (q *queueTx) checkPreviewStorage(live []liveCandidate) error {
if _, err := loadCeilingState(q.tx); err != nil {
return err
}
if _, err := q.storageState(); err != nil {
return err
}
for _, lc := range live {
if _, _, _, err := q.historyNeed(lc, 0); err != nil {
return err
}
}
return nil
}
// deferHeld records the real holds a preview selection passed over, as a
// schedule records them: work an unresolved attempt holds waits as pending
// recovery once its retry backoff is over. A selection judges no live
// budget, so it records no other reason, and a live ceiling or storage
// share wait left on other work is stale: it clears as a schedule clears
// a resolved one. Pending work keeps its reason.
func (q *queueTx) deferHeld(items []admission.ScheduleItem, byID map[admission.CandidateID]liveCandidate, held map[admission.CandidateID]bool, answers map[admission.CandidateID]admission.Answer) error {
for _, it := range items {
lc := byID[it.ID]
if answers[it.ID] == admission.AnswerPending {
continue
}
c := lc.c
switch {
case held[it.ID] && !q.now.Before(c.NotBefore):
if c.Reason == admission.ReasonPendingRecovery {
continue
}
if err := q.deferred(lc.id, &c, lc.entry, admission.ReasonPendingRecovery); err != nil {
return err
}
case c.Reason == admission.ReasonCeiling, c.Reason == admission.ReasonStorageShare:
c.Reason = 0
default:
continue
}
c.Transitions++
if err := putCandidate(q.tx, c); err != nil {
return err
}
}
return nil
}
// PreviewWake is the earliest admission time at which a dry-run forecast
// has work without a new report: now while selectable work waits, or a
// retry's backoff end, a queued deadline, or, until the ledger starts
// live, when the live history budget can pay for a challenge the
// selection passes over. Other live budgets never delay it, and pending
// work wakes only at its deadlines. ok is false when nothing waits.
func (l *AdmissionLedger) PreviewWake(answers map[admission.CandidateID]admission.Answer) (time.Time, bool, error) {
// The lock guards the recovery hold this read shares with writers.
l.mu.Lock()
defer l.mu.Unlock()
var wake time.Time
err := l.db.bolt.View(func(tx *bolt.Tx) error {
clock, err := loadLedgerClock(tx.Bucket([]byte(admissionMetaBucket)))
if err != nil {
return err
}
now := clock.Now()
q, err := openQueueWith(tx, l.reg, l.Inventory(), now)
if err != nil {
return err
}
q.held, q.started = l.held, l.live
live, err := q.live()
if err != nil {
return err
}
if err = q.checkPreviewStorage(live); err != nil {
return err
}
q.state.NextSweep = time.Time{}
earliest := func(t time.Time) {
if q.state.NextSweep.IsZero() || t.Before(q.state.NextSweep) {
q.state.NextSweep = t
}
}
var history liveHistory
for _, lc := range live {
if lc.c.State != admission.StateQueued {
continue
}
q.noteDeadlines(lc.c, lc.entry)
if answers[lc.id] == admission.AnswerPending {
continue
}
held, heldErr := q.heldTarget(lc.c.Key.Target)
if heldErr != nil {
return heldErr
}
switch {
case held:
case lc.c.NotBefore.After(now):
earliest(lc.c.NotBefore)
case lc.c.Key.Kind == admission.KindChallenge && !q.started:
if err = q.challengeWake(&history, lc, earliest); err != nil {
return err
}
default:
earliest(now)
}
}
wake = q.state.NextSweep
if !wake.IsZero() && wake.Before(now) {
wake = now
}
return nil
})
return wake, !wake.IsZero(), err
}
// challengeWake notes when queued challenge lc can be selected: now when
// the live history budget can pay for its comparison preview, otherwise
// when a lane it may use can. Like a schedule's wake, it notes nothing
// while the recovery reserve cannot take it.
func (q *queueTx) challengeWake(history *liveHistory, lc liveCandidate, earliest func(time.Time)) error {
bytes, fits, general, reserved, err := history.challenge(q, lc)
switch {
case err != nil:
return err
case !fits:
return nil
case general || reserved:
earliest(q.now)
return nil
}
storage, err := q.storageState()
if err != nil {
return err
}
lanes := []admission.Lane{admission.LaneGeneral}
if lc.entry.Direct || lc.entry.Corroborated {
lanes = append(lanes, admission.LaneDirect)
}
for _, lane := range lanes {
room := history.general
if lane != admission.LaneGeneral {
room = history.reserved
}
at, ok, wakeErr := q.historyWake(storage, lane, room, bytes, 0)
if wakeErr != nil {
return wakeErr
}
if ok {
earliest(at)
}
}
return nil
}
// deferUnserved records why ready work the schedule left queued waits: its
// lanes ran out of ceiling budget, of free set slots in the partitions
// they occupy or of history budget, or its details do not fit the recovery
// reserve. Work passed over for the member bound or another
// scope's fair turn waits without a reason, and so does a retry in its
// backoff. A changed reason is one transition, counted and announced once,
// as Defer records it. Pending work keeps its reason, and so does set_full
// work while the ledger has no set to judge it by: the engine refused it.
func (q *queueTx) deferUnserved(items []admission.ScheduleItem, byID map[admission.CandidateID]liveCandidate, picks []admission.Pick, lim admission.ScheduleLimits, slots *admission.SlotLimits, answers map[admission.CandidateID]admission.Answer) error {
picked := make(map[admission.CandidateID]bool, len(picks))
general, reserved := lim.General, lim.Reserved
generalBytes, reservedBytes := lim.GeneralBytes, lim.ReservedBytes
rest := slots.Clone()
for _, p := range picks {
picked[p.ID] = true
if p.Lane == admission.LaneGeneral {
general, generalBytes = general-p.Cost, generalBytes-uint64(p.Bytes)
} else {
reserved, reservedBytes = reserved-p.Cost, reservedBytes-uint64(p.Bytes)
}
for _, it := range items {
if it.ID == p.ID {
rest.Take(p.Lane, demandOf(it, byID[p.ID]))
}
}
}
recovery := lim.RecoveryBytes
for _, it := range items {
if picked[it.ID] {
recovery -= uint64(it.Recovery)
}
}
for _, it := range items {
if answers[it.ID] == admission.AnswerPending {
continue
}
lc := byID[it.ID]
if lc.c.Reason == admission.ReasonSetFull && slots == nil {
continue
}
eligible := it.Direct || it.Corroborated
generalCeiling := general >= it.CeilingCost
reservedCeiling := eligible && reserved >= it.CeilingCost
generalSet, reservedSet := roomOn(rest, it, lc)
generalRoom := generalCeiling && generalSet
reservedRoom := reservedCeiling && reservedSet
held, err := q.heldTarget(lc.c.Key.Target)
if err != nil {
return err
}
var reason admission.Reason
switch {
case picked[it.ID]:
case q.now.Before(lc.c.NotBefore):
case held:
reason = admission.ReasonPendingRecovery
case !generalCeiling && !reservedCeiling:
reason = admission.ReasonCeiling
case !generalRoom && !reservedRoom:
reason = admission.ReasonSetFull
case (!generalRoom || generalBytes < uint64(it.Bytes)) && (!reservedRoom || reservedBytes < uint64(it.Bytes)):
reason = admission.ReasonStorageShare
case uint64(it.Recovery) > recovery:
reason = admission.ReasonPendingRecovery
}
if lc.c.Reason == reason {
continue
}
c := lc.c
if reason == 0 {
switch c.Reason {
case admission.ReasonCeiling, admission.ReasonSetFull, admission.ReasonStorageShare, admission.ReasonPendingRecovery:
default:
continue
}
c.Reason = 0
} else {
if err := q.deferred(lc.id, &c, lc.entry, reason); err != nil {
return err
}
}
c.Transitions++
if err := putCandidate(q.tx, c); err != nil {
return err
}
}
return nil
}
// scheduleLimits lowers the caller's lane bounds to what the ceiling can
// charge now and sets each lane's history budget.
func (q *queueTx) scheduleLimits(lim admission.ScheduleLimits) (admission.ScheduleLimits, admission.CeilingState, error) {
ceiling, err := loadCeilingState(q.tx)
if err != nil {
return lim, ceiling, err
}
lim.General = min(lim.General, ceiling.Budget(admission.LaneGeneral))
lim.Reserved = min(lim.Reserved, ceiling.Budget(admission.LaneDirect))
lim.RecoveryBytes, err = q.recoveryRoom()
if err != nil {
return lim, ceiling, err
}
lim.GeneralBytes, lim.ReservedBytes, err = q.historyBudgets()
return lim, ceiling, err
}
// scheduleItems are the queued candidates as the scheduler sees them, each
// with the history a reservation of it would charge now. One whose details
// would not fit the recovery reserve is not ready: it waits for recovery to
// settle unresolved outcomes. Nor is pending work. An allowed candidate
// ends without a charge: it needs no ceiling, history or recovery room.
func (q *queueTx) scheduleItems(live []liveCandidate, room uint64, answers map[admission.CandidateID]admission.Answer) ([]admission.ScheduleItem, map[admission.CandidateID]liveCandidate, error) {
byID := map[admission.CandidateID]liveCandidate{}
var items []admission.ScheduleItem
for _, lc := range live {
if lc.c.State != admission.StateQueued {
continue
}
bytes, recovery, fits, err := q.historyNeed(lc, room)
if err != nil {
return nil, nil, err
}
held, err := q.heldTarget(lc.c.Key.Target)
if err != nil {
return nil, nil, err
}
byID[lc.id] = lc
cost := lc.c.Key.Kind.CeilingCost()
// On a live ledger only block_ip work runs; a preview of any other
// kind spends nothing, so it takes no lane's room from work that does.
if answers[lc.id] == admission.AnswerAllow || (q.started && lc.c.Key.Kind != admission.KindBlockIP) {
cost, bytes, recovery, fits = 0, 0, 0, true
}
items = append(items, admission.ScheduleItem{
ID: lc.id, Scope: lc.c.Scope.Key(), Tier: lc.entry.Tier, Direct: lc.entry.Direct,
Corroborated: lc.entry.Corroborated, Queued: lc.c.FirstQueued, Cost: unitCost, CeilingCost: cost, Bytes: bytes, Recovery: recovery,
Ready: fits && !held && !q.now.Before(lc.c.NotBefore) && answers[lc.id] != admission.AnswerPending,
})
}
return items, byID, nil
}
// NextWake is the earliest admission time at which queued work changes
// without a new report: a retry wait ends, a queued deadline passes, or
// ready work gains ceiling budget, a set slot or history budget on a lane
// it can use. Work waiting for a set slot on either eligible lane wakes
// at the next future effect expiry or setRecheck, whichever is sooner.
// It is the stored admission time when a schedule would serve ready work
// already. The engine's timer ticks and schedules then. ok is false when
// nothing waits. Pending work, as Schedule passes it over, wakes only at
// its deadlines: its answer brings its next turn.
func (l *AdmissionLedger) NextWake(answers map[admission.CandidateID]admission.Answer) (time.Time, bool, error) {
// The lock guards the recovery hold this read shares with writers.
l.mu.Lock()
defer l.mu.Unlock()
var wake time.Time
err := l.db.bolt.View(func(tx *bolt.Tx) error {
clock, err := loadLedgerClock(tx.Bucket([]byte(admissionMetaBucket)))
if err != nil {
return err
}
q, err := openQueueWith(tx, l.reg, l.Inventory(), clock.Now())
if err != nil {
return err
}
q.held, q.started, q.degraded, q.set, q.imports = l.held, l.live, l.degraded, l.set, l.imports
live, err := q.live()
if err != nil {
return err
}
// Stored sweep times may outlive a released position. Derive the
// wake from live queued work in this same read transaction.
q.state.NextSweep = time.Time{}
for _, lc := range live {
if lc.c.State == admission.StateQueued {
q.noteDeadlines(lc.c, lc.entry)
}
}
wake = q.state.NextSweep
earliest := func(t time.Time) {
if wake.IsZero() || t.Before(wake) {
wake = t
}
}
lim, ceiling, err := q.scheduleLimits(admission.ScheduleLimits{General: admission.MaxCeiling, Reserved: admission.MaxCeiling, Members: admission.MaxBatchMembers})
if err != nil {
return err
}
items, byID, err := q.scheduleItems(live, lim.RecoveryBytes, answers)
if err != nil {
return err
}
q.waiting = waitingScopes(items, byID)
slots, err := q.slots()
if err != nil {
return err
}
// The next cycle moves the share cursors first; so does its probe.
if q.shared {
slots.Advance()
}
if err = q.markFresh(byID); err != nil {
return err
}
// A due sweep must assess imported entries before the scheduler
// can use their tiers. Overdue work needs attention now.
if !wake.IsZero() && !wake.After(clock.Now()) {
wake = clock.Now()
return nil
}
var general, reserved, full bool
for _, it := range items {
// A retry cannot become ready at its backoff while recovery
// space is unavailable, nor while it waits on an outside
// answer. Its validity deadlines still apply.
if uint64(it.Recovery) > lim.RecoveryBytes || answers[it.ID] == admission.AnswerPending {
continue
}
if nb := byID[it.ID].c.NotBefore; nb.After(clock.Now()) {
earliest(nb)
}
if it.Ready {
general, reserved = true, reserved || it.Direct || it.Corroborated
// Either lane can free a slot before the other earns enough
// ceiling or history credit to use its existing room.
generalSet, reservedSet := roomOn(slots, it, byID[it.ID])
full = full || !generalSet || (reservedLane(it) != 0 && !reservedSet)
}
}
if !general {
return nil
}
st, err := loadScheduleState(tx)
if err != nil {
return err
}
// Work no lane has room for is not ready, so a victim it cannot
// take never wakes the owner at once.
items = slotReady(items, byID, slots)
picks, _, err := admission.Schedule(items, st, limitBySlots(lim, slots))
if err != nil {
return err
}
if len(picks) > 0 {
earliest(clock.Now())
return nil
}
if full {
earliest(clock.Now().Add(setRecheck))
at, ok, soonErr := soonestEffect(tx, clock.Now())
if soonErr != nil {
return soonErr
}
if ok {
earliest(at)
}
}
storage, err := q.storageState()
if err != nil {
return err
}
// Only a lane without ceiling budget needs the window's charges.
var charges []admission.Charge
for _, lane := range []admission.Lane{admission.LaneGeneral, admission.LaneDirect} {
if lane == admission.LaneDirect && !reserved {
continue
}
probes := []struct {
units uint32
wait time.Duration
}{{ceiling.Budget(lane), 0}}
if ceiling.Budget(lane) == 0 {
if charges == nil {
if charges, err = loadCharges(tx); err != nil {
return err
}
}
if d, ok := ceiling.UntilBudget(lane, charges, clock.Now()); ok {
// Every charged ledger member needs one unit. Its next
// turn can differ from the challenge borrowing it now.
probes = append(probes, struct {
units uint32
wait time.Duration
}{unitCost, d})
}
}
for _, budget := range probes {
// Probe the actual next head with a full byte budget. Its
// earned turn, rather than a fixed quantum, sets the timer.
probe := lim
probe.Members = 1
room := lim.GeneralBytes
if lane == admission.LaneGeneral {
probe.General = budget.units
probe.Reserved, probe.ReservedBytes, probe.GeneralBytes = 0, 0, admission.MaxHistoryBytes
} else {
probe.Reserved = budget.units
probe.General, probe.GeneralBytes, probe.ReservedBytes = 0, 0, admission.MaxHistoryBytes
room = lim.ReservedBytes
}
var laneItems []admission.ScheduleItem
for _, item := range items {
if lane == admission.LaneGeneral {
item.Direct, item.Corroborated = false, false
} else if !item.Direct && !item.Corroborated {
continue
}
laneItems = append(laneItems, item)
}
heads, _, probeErr := admission.Schedule(laneItems, st, limitBySlots(probe, slots))
if probeErr != nil {
return probeErr
}
if len(heads) > 0 {
at, ok, wakeErr := q.historyWake(storage, lane, room, heads[0].Bytes, budget.wait)
if wakeErr != nil {
return wakeErr
}
if ok {
earliest(at)
}
}
}
}
return nil
})
return wake, !wake.IsZero(), err
}
// historyWake is when lane can pay need bytes of history, no sooner than
// after from now. room is what the lane may charge now. When its allowance
// has room for need once retirable history is retired, that is when its
// credit covers need, and nothing when no wait is needed; otherwise it is
// when its oldest history that may not be retired yet becomes retirable.
func (q *queueTx) historyWake(storage *admission.StorageState, lane admission.Lane, room uint64, need uint32, after time.Duration) (time.Time, bool, error) {
if room < uint64(need) {
// Recompute room without the credit cap, including an
// upgrade's excess, before choosing credit or retirement.
copy := *storage
full := admission.NewStorageState()
copy.General.Credit, copy.Reserved.Credit = full.General.Credit, full.Reserved.Credit
size, _ := admission.HistoryLanes()
used := copy.General.Used
if lane != admission.LaneGeneral {
_, size = admission.HistoryLanes()
used = copy.Reserved.Used
}
r, err := q.retirable(lane, used-min(used, size-uint64(need)))
if err != nil {
return time.Time{}, false, err
}
room = copy.HistoryBudget(lane, r)
}
if room >= uint64(need) {
wait := after
if d, ok := storage.UntilHistoryCost(lane, need); ok {
wait = max(wait, d)
}
return q.now.Add(wait), wait > 0, nil
}
at, ok, err := q.nextRetirable(lane)
if err != nil || !ok {
return time.Time{}, false, err
}
if ceilingAt := q.now.Add(after); at.Before(ceilingAt) {
at = ceilingAt
}
return at, true, nil
}
package store
import (
"fmt"
"os"
"runtime/debug"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// CheckLedgerRestore returns ErrLedgerBehind when putting the database at
// archive, or the firewall state saved beside it, in place of the database
// at target would move a ledger that started live back: the archive's
// ledger never started live or its clock is behind. An older ledger would
// spend an hour's charges again and run the live start a second time from
// stale blocks. A target whose ledger never started live is never refused,
// so recovery onto a fresh host keeps working. A target this binary cannot
// read is not refused either: restoring over it is the repair. An archive
// ledger it cannot read counts as behind. Neither file needs a ledger.
func CheckLedgerRestore(archive, target string) error {
if live, at, err := readLedgerPosition(target, true); err == nil && live {
return checkArchiveLedger(archive, at)
}
return nil
}
func checkArchiveLedger(archive string, targetClock time.Time) error {
live, clock, err := readLedgerPosition(archive, false)
switch {
case err != nil:
return fmt.Errorf("%w: it cannot be read (%v); use --force to restore it anyway", ErrLedgerBehind, err)
case !live:
return fmt.Errorf("%w: it never started live; use --force to restore it anyway", ErrLedgerBehind)
case clock.Before(targetClock):
return fmt.Errorf("%w: its clock reads %s, this host's %s; use --force to restore it anyway",
ErrLedgerBehind, clock.Format(time.RFC3339Nano), targetClock.Format(time.RFC3339Nano))
}
return nil
}
// readLedgerPosition reads, without a ledger handle, whether the database
// at path holds a ledger that started live and its admission clock. A
// missing file or one without a current ledger has not started.
func readLedgerPosition(path string, allowLeakedPages bool) (live bool, at time.Time, err error) {
// bbolt reports a damaged page by panicking. A database that cannot be
// read is an answer here, not a crash: restoring over it is the repair.
defer func() {
if r := recover(); r != nil {
live, at, err = false, time.Time{}, fmt.Errorf("reading the ledger database: %v", r)
}
}()
// Check the file type and page structure before opening bbolt. A pipe
// blocks before the lock timeout and page cycles never finish.
err = verifyStoppedDatabase(path, allowLeakedPages)
if os.IsNotExist(err) {
return false, time.Time{}, nil
}
if err != nil {
return false, time.Time{}, err
}
db, err := bolt.Open(path, 0o600, &bolt.Options{ReadOnly: true, Timeout: time.Second})
if err != nil {
return false, time.Time{}, err
}
defer func() { _ = db.Close() }()
// Truncated mapped pages raise memory faults rather than bbolt panics.
// Limit fault recovery to this goroutine's read and restore its setting.
defer debug.SetPanicOnFault(debug.SetPanicOnFault(true))
var clock admission.Clock
err = db.View(func(tx *bolt.Tx) error {
if current, currentErr := currentLedger(tx); !current {
return currentErr
}
if tx.Bucket([]byte(admissionQueueStateBucket)) == nil {
return admission.ErrCorruptRecord
}
if _, featuresErr := loadFeatures(tx); featuresErr != nil {
return featuresErr
}
var viewErr error
if _, live, viewErr = loadLiveStart(tx); viewErr != nil {
return viewErr
}
clock, viewErr = loadLedgerClock(tx.Bucket([]byte(admissionMetaBucket)))
return viewErr
})
return live, clock.Now(), err
}
package store
import (
"slices"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// setRecheck is the longest that block work a full set defers sleeps
// before the set is read again: an operator unblock, an expired operator or
// legacy block or a raised limit frees a slot no ledger record names.
const setRecheck = time.Minute
// SetOccupants records the temporary set outside the ledger's effect
// records, as the firewall last showed it: the configured limit, the
// entries admission never evicts (Unclaimed) and the known automatic
// imports (General). Schedules, reservations and wakes then budget block
// work by the free slots it leaves once the ledger's own records are added.
// nil leaves block work without a slot budget, as an owner
// without a firewall engine runs it.
func (l *AdmissionLedger) SetOccupants(base *admission.SetOccupancy) {
l.mu.Lock()
defer l.mu.Unlock()
if base != nil {
copied := *base
base = &copied
}
l.set = base
}
// SetImports records the known automatic imports the firewall last listed:
// (C2, High) victims in the general partition with no record of
// their own, which SetOccupants counts as General. Entries that are not
// such an import are dropped. A block that claims one names it in its own
// record until its outcome.
func (l *AdmissionLedger) SetImports(imports []admission.EvictionVictim) {
l.mu.Lock()
defer l.mu.Unlock()
l.imports = nil
for _, v := range imports {
if v.Imported() {
l.imports = append(l.imports, v)
}
}
}
// HasChargedWork reports whether a queued candidate that charges the
// ceiling, so takes a set slot, is past its retry wait: the only work a
// read of the set can budget.
func (l *AdmissionLedger) HasChargedWork() (bool, error) {
l.mu.Lock()
defer l.mu.Unlock()
var found bool
err := l.db.bolt.View(func(tx *bolt.Tx) error {
clock, err := loadLedgerClock(tx.Bucket([]byte(admissionMetaBucket)))
if err != nil {
return err
}
q, err := openQueueWith(tx, l.reg, l.Inventory(), clock.Now())
if err != nil {
return err
}
live, err := q.live()
for _, lc := range live {
found = found || (lc.c.State == admission.StateQueued && lc.c.Key.Kind.CeilingCost() > 0 && !q.now.Before(lc.c.NotBefore))
}
return err
})
return found, err
}
// EffectTally is the set slots the ledger's own records hold in each
// partition.
func (l *AdmissionLedger) EffectTally() (admission.EffectTally, error) {
l.mu.Lock()
defer l.mu.Unlock()
var tally admission.EffectTally
err := l.db.bolt.View(func(tx *bolt.Tx) (err error) {
tally, err = loadEffectTally(tx)
return err
})
return tally, err
}
// slots is the set budget of this transaction: nil without a set, else
// the free slots the handle's set and the ledger's records leave, with the
// verified records and imports strictly higher work may evict once the
// general partition is full (no work evicts while it has a free slot). A
// live ledger on a trusted clock also shares each partition among its
// scopes, so fresh work may rotate an equal-tier record whose hold passed.
func (q *queueTx) slots() (*admission.SlotLimits, error) {
if q.set == nil {
return nil, nil
}
tally, err := loadEffectTally(q.tx)
if err != nil {
return nil, err
}
base := *q.set
// The bounded list can omit imports, but every listed import still
// occupies a general slot. Leave invalid counts for tally validation.
if base.General >= 0 {
base.General = max(base.General, len(q.imports))
}
occ, err := tally.Occupy(base)
if err != nil {
return nil, err
}
var holding map[admission.Partition]map[string]int
slots, err := occ.Limits(func() ([]admission.EvictionVictim, error) {
rotates, holdErr := q.holdTest()
if holdErr != nil {
return nil, holdErr
}
pool, scopes, poolErr := victimPool(q.tx, q.now, rotates, q.imports)
if rotates != nil {
holding = scopes
}
return pool, poolErr
})
if err == nil && holding != nil {
slots.Share(holding, q.waiting, q.state.SetCursors)
q.shared = true
}
return slots, err
}
// waitingScopes are the scopes of queued work that would hold a set
// slot, by the partitions it may occupy: general, and reserved too for
// work eligible for a reserved lane. Such work shares the set with the
// scopes holding it (active scopes include eligible waiting
// work); a retry's wait does not end its claim.
func waitingScopes(items []admission.ScheduleItem, byID map[admission.CandidateID]liveCandidate) map[admission.Partition][]string {
out := map[admission.Partition][]string{}
for _, it := range items {
if !demandOf(it, byID[it.ID]).Evicts {
continue
}
out[admission.PartitionGeneral] = append(out[admission.PartitionGeneral], it.Scope)
if reservedLane(it) != 0 {
out[admission.PartitionReserved] = append(out[admission.PartitionReserved], it.Scope)
}
}
for p := range out {
slices.Sort(out[p])
out[p] = slices.Compact(out[p])
}
return out
}
// holdTest reports whether a record's rotation hold passed: nil, so no
// record rotates, before the live start or on a degraded reading
// (wall time alone proves nothing then).
func (q *queueTx) holdTest() (func(admission.EffectRecord) bool, error) {
if !q.started || q.degraded {
return nil, nil
}
ceiling, err := loadCeilingState(q.tx)
if err != nil {
return nil, err
}
return func(rec admission.EffectRecord) bool { return rec.HoldPassed(q.now, ceiling.Elapsed) }, nil
}
// victimPool is what strictly higher work may evict and fresh equal-tier
// work may rotate: records verified, holding their slot and
// claimed by no pending block, each with its scope and application and
// rotatable when rotates passes it, and the unexpired imports no block's
// record names. holding counts the slots each scope's unexpired,
// unclaimed records hold in each partition, pending ones included. The
// full-set path walks the records once; free-slot admission skips it.
func victimPool(tx *bolt.Tx, now time.Time, rotates func(admission.EffectRecord) bool, imports []admission.EvictionVictim) ([]admission.EvictionVictim, map[admission.Partition]map[string]int, error) {
var pool []admission.EvictionVictim
holding := map[admission.Partition]map[string]int{}
claimed := map[admission.ActionID]bool{}
if b := tx.Bucket([]byte(admissionEffectsBucket)); b != nil {
err := b.ForEach(func(k, v []byte) error {
rec, err := admission.UnmarshalEffectRecord(k, v)
if err != nil {
return err
}
if rec.Victim.Imported() {
claimed[rec.Victim.Action] = true
}
if !rec.Holds() || rec.EvictedBy != "" || !rec.ExpiresAt.After(now) {
return nil
}
if holding[rec.Partition] == nil {
holding[rec.Partition] = map[string]int{}
}
holding[rec.Partition][rec.Scope]++
if rec.Verified() {
pool = append(pool, admission.EvictionVictim{Action: rec.Action, Target: rec.Target, Tier: rec.Tier, Partition: rec.Partition, ExpiresAt: rec.ExpiresAt,
Scope: rec.Scope, Applied: rec.Applied, Rotatable: rotates != nil && rotates(rec)})
}
return nil
})
if err != nil {
return nil, nil, err
}
}
for _, v := range imports {
if !claimed[v.Action] && v.ExpiresAt.After(now) {
pool = append(pool, v)
}
}
return pool, holding, nil
}
// fresh reports whether c's episode has had no verified response, the
// only work that may rotate an equal-tier block. Work whose
// episode row is missing or names another episode is not fresh: an
// unknown history never proves a new episode.
func (q *queueTx) fresh(c admission.Candidate) (bool, error) {
if !q.started {
return false, nil
}
return freshEpisode(q.tx, c)
}
// freshEpisode reports whether c is block_ip work whose target's episode
// row names its episode and records no verified response.
func freshEpisode(tx *bolt.Tx, c admission.Candidate) (bool, error) {
if c.Key.Kind != admission.KindBlockIP {
return false, nil
}
e, found, err := loadEpisode(tx, c.Key.Target.Key())
return found && e.ID == c.Key.Episode && e.Verified.IsZero(), err
}
// Fresh reports whether c's episode row lets it rotate an equal-tier
// block as a live ledger decides it, on any ledger: a dry-run
// forecast asks it for its picks.
func (l *AdmissionLedger) Fresh(c admission.Candidate) (bool, error) {
var fresh bool
err := l.db.bolt.View(func(tx *bolt.Tx) error {
var err error
fresh, err = freshEpisode(tx, c)
return err
})
return fresh, err
}
// markFresh notes which of the live candidates byID are fresh. Only a
// shared set needs it: nothing rotates otherwise.
func (q *queueTx) markFresh(byID map[admission.CandidateID]liveCandidate) error {
if !q.shared {
return nil
}
for id, lc := range byID {
var err error
if lc.fresh, err = q.fresh(lc.c); err != nil {
return err
}
byID[id] = lc
}
return nil
}
// limitBySlots lowers lane budgets to free slots or victims under
// pressure; readiness and contention checks decide each pick's fit.
func limitBySlots(lim admission.ScheduleLimits, slots *admission.SlotLimits) admission.ScheduleLimits {
lim.General, lim.Reserved = min(lim.General, slots.Budget(admission.LaneGeneral)), min(lim.Reserved, slots.Budget(admission.LaneDirect))
return lim
}
// demandOf is what schedule item it of candidate lc asks of the set: only
// one-unit block_ip work may evict, never the block of its own address,
// and only fresh work may rotate.
func demandOf(it admission.ScheduleItem, lc liveCandidate) admission.SlotDemand {
return admission.SlotDemand{Tier: it.Tier, Cost: it.CeilingCost, Evicts: lc.c.Key.Kind == admission.KindBlockIP && it.CeilingCost == 1,
Reserved: it.Direct || it.Corroborated, Target: lc.c.Key.Target, Scope: it.Scope, Fresh: lc.fresh}
}
// reservedLane is the reserved lane item it is eligible for; zero for
// general-only work.
func reservedLane(it admission.ScheduleItem) admission.Lane {
switch {
case it.Direct:
return admission.LaneDirect
case it.Corroborated:
return admission.LaneCorroborated
}
return 0
}
// roomOn reports whether item it finds room in slots on its general lane
// and on its reserved lane, if any.
func roomOn(slots *admission.SlotLimits, it admission.ScheduleItem, lc liveCandidate) (general, reserved bool) {
d := demandOf(it, lc)
_, general = slots.Room(admission.LaneGeneral, d)
if lane := reservedLane(it); lane != 0 {
_, reserved = slots.Room(lane, d)
}
return general, reserved
}
// slotReady refines lane readiness without changing the overall Ready
// control: slot pressure preserves saved turns, while backoff releases
// a saved ceiling turn through the scheduler's existing rule.
func slotReady(items []admission.ScheduleItem, byID map[admission.CandidateID]liveCandidate, slots *admission.SlotLimits) []admission.ScheduleItem {
out := slices.Clone(items)
for i := range out {
general, reserved := roomOn(slots, out[i], byID[out[i].ID])
out[i].BlockedGeneral, out[i].BlockedReserved = !general, !reserved
}
return out
}
// soonestEffect is the first future effect expiry. A degraded reading can
// retain past expiries; they must neither spin the timer nor hide a future one.
func soonestEffect(tx *bolt.Tx, now time.Time) (time.Time, bool, error) {
if err := proveEffects(tx); err != nil {
return time.Time{}, false, err
}
if b := tx.Bucket([]byte(admissionEffectsBucket)); b != nil {
cur := b.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
rec, err := admission.UnmarshalEffectRecord(k, v)
if err != nil {
return time.Time{}, false, err
}
if rec.Holds() && rec.ExpiresAt.After(now) {
return rec.ExpiresAt, true, nil
}
}
}
return time.Time{}, false, nil
}
package store
import (
"errors"
"sort"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// Status reads the ledger for status and doctor: one read
// transaction, without the mutex or a current clock, so it answers while
// another call runs and while the ledger cannot be written. Each section
// is read on its own; a damaged record fails only its own section.
func (l *AdmissionLedger) Status() admission.LedgerStatus {
var s admission.LedgerStatus
err := l.db.bolt.View(func(tx *bolt.Tx) error {
section := func(read func() error, buckets ...string) string {
for _, name := range buckets {
if tx.Bucket([]byte(name)) == nil {
return admission.ErrCorruptRecord.Error()
}
}
if err := read(); err != nil {
return err.Error()
}
return ""
}
var now time.Time
s.Clock.Error = section(func() error {
clock, err := loadLedgerClock(tx.Bucket([]byte(admissionMetaBucket)))
if err != nil {
return err
}
now = clock.Now()
s.Clock.Now = now
return nil
}, admissionMetaBucket)
s.Queue.Error = section(func() error { return queueStatus(tx, &s.Queue) }, admissionQueueBucket, admissionCandidatesBucket)
s.Counters.Error = section(func() error {
counters, loadErr := loadQueueCounters(tx)
s.Counters.Rows = admission.CountRows(counters.Rows())
return loadErr
}, admissionQueueStateBucket)
s.Outcomes.Error = section(func() error {
if s.Clock.Error != "" {
return errors.New("the admission clock is unreadable")
}
return outcomesStatus(tx, now, &s.Outcomes)
}, admissionWindowsBucket)
s.Ingress.Error = section(func() error {
in, loadErr := loadIngressState(tx)
s.Ingress = admission.IngressSection{Generation: in.Generation, Open: in.Open, Persisted: in.Persisted, Interrupted: in.Interrupted, Resumed: in.Resumed}
return loadErr
}, admissionQueueStateBucket)
s.Ceiling.Error = section(func() error { return ceilingStatus(tx, now, &s.Ceiling) }, admissionQueueStateBucket, admissionChargesBucket)
s.Storage.Error = section(func() error { return storageStatus(tx, &s.Storage) }, admissionQueueStateBucket, admissionAttemptsBucket, admissionHistoryBucket)
s.Outbox.Error = section(func() error { return outboxStatus(tx, &s.Outbox) }, admissionOutboxBucket)
s.Notices.Error = section(func() error { return noticesStatus(tx, &s.Notices) }, admissionOutboxBucket)
s.Unresolved.Error = section(func() error { return unresolvedStatus(tx, &s.Unresolved) }, admissionIntentsBucket, admissionAttemptsBucket, admissionCandidatesBucket)
s.Effects.Error = section(func() error { return effectsStatus(tx, &s.Effects) }, admissionMetaBucket, admissionQueueStateBucket)
return nil
})
if err != nil {
for _, section := range []*string{
&s.Clock.Error, &s.Queue.Error, &s.Counters.Error, &s.Outcomes.Error, &s.Ingress.Error,
&s.Ceiling.Error, &s.Storage.Error, &s.Outbox.Error, &s.Notices.Error, &s.Unresolved.Error, &s.Effects.Error,
} {
*section = err.Error()
}
}
return s
}
func queueStatus(tx *bolt.Tx, out *admission.QueueStatus) error {
occupancy := map[admission.QueueOccupancy]int{}
err := tx.Bucket([]byte(admissionQueueBucket)).ForEach(func(k, v []byte) error {
e, err := admission.UnmarshalQueueEntry(v)
if err != nil {
return err
}
c, err := loadCandidate(tx, admission.CandidateID(k))
if err != nil {
return corruptRecord(err)
}
switch c.State {
case admission.StateQueued:
out.Queued++
if c.Attempts > 0 {
out.Retrying++
}
if out.Oldest.IsZero() || c.FirstQueued.Before(out.Oldest) {
out.Oldest = c.FirstQueued
}
case admission.StateReserved:
out.Reserved++
case admission.StateExecuting:
out.Executing++
default:
return admission.ErrCorruptRecord
}
occupancy[admission.QueueOccupancy{Kind: c.Key.Kind.String(), Partition: e.Partition.String()}]++
return nil
})
for o, n := range occupancy {
o.Count = n
out.Occupancy = append(out.Occupancy, o)
}
sort.Slice(out.Occupancy, func(i, j int) bool {
a, b := out.Occupancy[i], out.Occupancy[j]
if a.Kind != b.Kind {
return a.Kind < b.Kind
}
return a.Partition < b.Partition
})
return err
}
// outcomesStatus sums each span's buckets inside its window at now.
func outcomesStatus(tx *bolt.Tx, now time.Time, out *admission.OutcomesStatus) error {
if err := proveOutcomes(tx, now); err != nil {
return err
}
windows := tx.Bucket([]byte(admissionWindowsBucket))
for _, w := range []struct {
span admission.Span
rows *[]admission.OutcomeStatusRow
}{{admission.SpanFiveMinutes, &out.Hour}, {admission.SpanHour, &out.Day}, {admission.SpanDay, &out.Month}} {
var sum admission.OutcomeCounts
cur := windows.Cursor()
oldest := w.span.Key(w.span.Oldest(now))
for k, v := cur.Seek(oldest); k != nil && k[0] == oldest[0] && k[1] == oldest[1]; k, v = cur.Next() {
if _, _, err := admission.ParseSpanKey(k); err != nil {
return err
}
c, err := admission.UnmarshalOutcomeCounts(v)
if err != nil {
return err
}
sum.Merge(c)
}
*w.rows = admission.OutcomeRows(sum)
}
return nil
}
func ceilingStatus(tx *bolt.Tx, now time.Time, out *admission.CeilingStatus) error {
c, err := loadCeiling(tx)
if err != nil {
return err
}
general, reserved := admission.CeilingLanes(c.Limit)
out.Limit = c.Limit
out.General = admission.LaneStatus{Size: general, Used: c.General.Used, Credit: c.General.Units()}
out.Reserved = admission.LaneStatus{Size: reserved, Used: c.Reserved.Used, Credit: c.Reserved.Units()}
var charges []admission.Charge
for _, lane := range []struct {
lane admission.Lane
out *admission.LaneStatus
}{{admission.LaneGeneral, &out.General}, {admission.LaneDirect, &out.Reserved}} {
if c.Budget(lane.lane) > 0 || now.IsZero() {
continue
}
if charges == nil {
if charges, err = loadCharges(tx); err != nil {
return err
}
}
if d, ok := c.UntilBudget(lane.lane, charges, now); ok {
lane.out.Next = d
}
}
return nil
}
// storageStatus reads the storage record, the outstanding attempts'
// holds and the retained review horizon.
func storageStatus(tx *bolt.Tx, out *admission.StorageStatus) error {
s, err := loadStorageState(tx)
if err != nil {
return err
}
allowance := func(l admission.Lane, m admission.HistoryMeter, size uint64) admission.AllowanceStatus {
return admission.AllowanceStatus{Size: size, Used: m.Used, Credit: m.Bytes(), Room: s.HistoryRoom(l)}
}
general, reserved := admission.HistoryLanes()
out.General = allowance(admission.LaneGeneral, s.General, general)
out.Reserved = allowance(admission.LaneDirect, s.Reserved, reserved)
out.Pinned, out.Ended, out.Loose = s.Recovery, s.Ended.Count, s.Loose.Count
// Outstanding attempts are read from the attempts, not the queue, so
// a damaged queue record cannot hide the reserve.
err = tx.Bucket([]byte(admissionAttemptsBucket)).ForEach(func(k, v []byte) error {
a, decodeErr := admission.UnmarshalAttempt(v)
if decodeErr != nil {
return decodeErr
}
if string(k) != string(a.Attempt.ID) {
return admission.ErrCorruptRecord
}
if a.State != admission.StateReserved && a.State != admission.StateExecuting {
return nil
}
h, found, loadErr := loadHistoryEntry(tx, a.Attempt.Candidate)
if loadErr != nil || !found {
return admission.ErrCorruptRecord
}
out.Outstanding += uint64(h.Charged()) + admission.UnknownHoldBytes
return nil
})
if err != nil {
return err
}
if held := s.Recovery + s.OutboxBytes() + out.Outstanding; s.Recovery < admission.RecoveryReserveBytes && held < admission.RecoveryReserveBytes {
out.RecoveryRoom = admission.RecoveryReserveBytes - held
}
return tx.Bucket([]byte(admissionHistoryBucket)).ForEach(func(k, v []byte) error {
if _, err := admission.ParseCandidateID(string(k)); err != nil {
return admission.ErrCorruptRecord
}
h, err := admission.UnmarshalHistoryEntry(v)
if err != nil {
return err
}
if !h.Ended.IsZero() && (out.ReviewHorizon.IsZero() || h.Ended.Before(out.ReviewHorizon)) {
out.ReviewHorizon = h.Ended
}
return nil
})
}
// outboxStatus counts the rows and records the outbox holds, from the
// outbox itself, so a damaged storage record does not hide them.
func outboxStatus(tx *bolt.Tx, out *admission.OutboxStatus) error {
if err := tx.Bucket([]byte(admissionOutboxBucket)).ForEach(func(k, v []byte) error {
switch k[0] {
case 'a':
if row, err := admission.UnmarshalAuditRow(v); err != nil || string(row.Key()) != string(k) {
return admission.ErrCorruptRecord
}
out.AuditRows++
case 'n':
out.NoticeRecords++
case 'q':
// Notice records and their quiet indexes are proved in notices.
default:
return admission.ErrCorruptRecord
}
return nil
}); err != nil {
return err
}
out.AuditBytes = out.AuditRows * admission.AuditSlotBytes
out.NoticeBytes = out.NoticeRecords * admission.NoticeSlotBytes
return nil
}
func noticesStatus(tx *bolt.Tx, out *admission.NoticesStatus) error {
for _, key := range admission.FixedNoticeKeys() {
if _, found, err := loadNoticeRecord(tx, key); err != nil || !found {
return admission.ErrCorruptRecord
}
}
cur := tx.Bucket([]byte(admissionOutboxBucket)).Cursor()
quiet := map[admission.NoticeKey]time.Time{}
for k, v := cur.Seek([]byte{'n'}); k != nil && k[0] == 'n'; k, v = cur.Next() {
key, err := admission.ParseNoticeKey(k)
if err != nil {
return err
}
r, err := admission.UnmarshalNoticeRecord(v)
if err != nil || r.Key != key {
return admission.ErrCorruptRecord
}
if key.Kind == admission.NoticeCriticalSummary {
out.LastCriticalGap = r.Last
}
if r.Count > 0 {
out.Records = append(out.Records, admission.NoticeRow(r))
}
if at := r.QuietAt(); !at.IsZero() {
quiet[key] = at
}
}
for k, v := cur.Seek([]byte{'q'}); k != nil && k[0] == 'q'; k, v = cur.Next() {
at, key, err := admission.ParseQuietKey(k)
if err != nil || v == nil || len(v) != 0 || !quiet[key].Equal(at) {
return admission.ErrCorruptRecord
}
delete(quiet, key)
}
if len(quiet) != 0 {
return admission.ErrCorruptRecord
}
return nil
}
// unresolvedStatus counts the attempts the intents name, by whether they
// run or ended unknown.
func unresolvedStatus(tx *bolt.Tx, out *admission.UnresolvedStatus) error {
// Intents alone cannot reveal a missing row or a retired attempt that
// incorrectly retained its row. Prove both directions before counting.
if _, err := proveIntents(tx); err != nil {
return err
}
return tx.Bucket([]byte(admissionIntentsBucket)).ForEach(func(k, _ []byte) error {
in, _, err := loadIntent(tx, admission.ActionID(k))
if err != nil {
return err
}
a, err := loadAttempt(tx, in.Action)
if err != nil {
return corruptRecord(err)
}
switch a.State {
case admission.StateExecuting:
out.Running++
case admission.StateUnknown:
out.Unknown++
if out.Retires.IsZero() || a.ExpiresAt.Before(out.Retires) {
out.Retires = a.ExpiresAt
}
default:
return admission.ErrCorruptRecord
}
return nil
})
}
package store
import (
"bytes"
"encoding/binary"
"errors"
"slices"
"sort"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
var storageStateKey = []byte("storage")
// Ring kinds of the rings bucket. An entry's key is its kind and its
// big-endian position; its value is the ID it keeps.
const (
ringEnded = 'e'
ringLoose = 'l'
)
func ringKey(kind byte, pos uint64) []byte {
return binary.BigEndian.AppendUint64([]byte{kind}, pos)
}
func loadStorageState(tx *bolt.Tx) (admission.StorageState, error) {
raw := tx.Bucket([]byte(admissionQueueStateBucket)).Get(storageStateKey)
if raw == nil {
return admission.StorageState{}, admission.ErrCorruptRecord
}
return admission.UnmarshalStorageState(raw)
}
func putStorageState(tx *bolt.Tx, s admission.StorageState) error {
data, err := s.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionQueueStateBucket)).Put(storageStateKey, data)
}
// historyCostOf is the history cost of c from the stored size of each root.
func historyCostOf(tx *bolt.Tx, c admission.Candidate) (uint32, error) {
evidence := tx.Bucket([]byte(admissionEvidenceBucket))
sizes := make([]int, len(c.Roots))
for i, id := range c.Roots {
raw := evidence.Get([]byte(id))
if raw == nil {
return 0, admission.ErrCorruptRecord
}
sizes[i] = len(raw)
}
return admission.HistoryCost(c, sizes)
}
// putHistoryEntry stores an entry with its retirement keys. An ended
// entry gets them only once its candidate's audit rows are acknowledged.
func putHistoryEntry(tx *bolt.Tx, id admission.CandidateID, h admission.HistoryEntry) error {
data, err := h.MarshalBinary()
if err != nil {
return err
}
keys, err := retireKeysOf(tx, id, h)
if err != nil {
return err
}
retire := tx.Bucket([]byte(admissionRetireBucket))
for _, k := range keys {
if err = retire.Put(k, nil); err != nil {
return err
}
}
return tx.Bucket([]byte(admissionHistoryBucket)).Put([]byte(id), data)
}
// retireKeysOf is the retirement keys entry h of candidate id holds now:
// none while any of its audit rows waits for acknowledgement.
func retireKeysOf(tx *bolt.Tx, id admission.CandidateID, h admission.HistoryEntry) ([][]byte, error) {
keys, err := h.RetireKeys(id)
if err != nil || len(keys) == 0 {
return keys, err
}
if pending, err := auditPending(tx, id); err != nil || pending {
return nil, err
}
return keys, nil
}
// upgradeLedgerToSchemaFour adds storage accounting to a schema 3 ledger
// inside the opening transaction. Every stored candidate's roots are
// counted. A candidate that ended before any attempt joins the ended ring
// in queue order. One with an attempt gets a history entry charged its
// history cost on the allowance of its latest attempt; an unresolved one is
// pinned in the recovery reserve. Evidence no candidate names joins the
// loose ring in observation order. Each ring keeps its newest entries. The
// allowances start without credit, since recent spend is unknown, and may
// start over their size. The upgrade that completes the chain records the
// schema.
func upgradeLedgerToSchemaFour(tx *bolt.Tx) error {
if err := validateUnownedRows(tx); err != nil {
return err
}
state := tx.Bucket([]byte(admissionQueueStateBucket))
if state.Get(storageStateKey) != nil || state.Bucket(storageStateKey) != nil {
return admission.ErrCorruptRecord
}
for _, name := range admissionStorageBuckets {
if _, err := tx.CreateBucket([]byte(name)); err != nil {
return err
}
}
clock, err := loadLedgerClock(tx.Bucket([]byte(admissionMetaBucket)))
if err != nil {
return err
}
var s admission.StorageState
refs := map[admission.EvidenceID]uint32{}
type ending struct {
id admission.CandidateID
c admission.Candidate
}
var endings []ending
err = tx.Bucket([]byte(admissionCandidatesBucket)).ForEach(func(k, v []byte) error {
c, walkErr := admission.UnmarshalCandidate(v)
if walkErr != nil {
return walkErr
}
id, _ := c.ID()
if string(id) != string(k) {
return admission.ErrCorruptRecord
}
for _, root := range c.Roots {
refs[root]++
}
if c.Attempts == 0 {
if c.State.Terminal() {
endings = append(endings, ending{id, c})
}
return nil
}
last, walkErr := currentAttempt(tx, c)
if walkErr != nil {
return walkErr
}
cost, walkErr := historyCostOf(tx, c)
if walkErr != nil {
return walkErr
}
h := admission.HistoryEntry{RootMask: (1 << len(c.Roots)) - 1}
if last.Lane == 0 || last.Lane == admission.LaneGeneral {
h.General, s.General.Used = cost, s.General.Used+uint64(cost)
} else {
h.Reserved, s.Reserved.Used = cost, s.Reserved.Used+uint64(cost)
}
if c.State.Terminal() {
// An outcome ended when its attempt finished. A queue ending
// after a failed attempt left no time: count it from now.
h.Ended = clock.Now()
if last.State == c.State {
h.Ended = last.Finished
}
if c.State == admission.StateUnknown {
h.Pinned = true
if s, walkErr = s.PinHistory(h.General, h.Reserved); walkErr != nil {
return walkErr
}
} else {
h.Eligible, _ = admission.HistoryTimes(h.Ended, c.ExpiresAt, admission.HistoryUntilExpiry(c))
}
}
return putHistoryEntry(tx, id, h)
})
if err != nil {
return err
}
// Prove every root before pruning old candidates can erase its count.
for root := range refs {
if tx.Bucket([]byte(admissionEvidenceBucket)).Get([]byte(root)) == nil {
return admission.ErrCorruptRecord
}
}
sort.Slice(endings, func(i, j int) bool {
if !endings[i].c.FirstQueued.Equal(endings[j].c.FirstQueued) {
return endings[i].c.FirstQueued.Before(endings[j].c.FirstQueued)
}
return endings[i].id < endings[j].id
})
rings := tx.Bucket([]byte(admissionRingsBucket))
candidates := tx.Bucket([]byte(admissionCandidatesBucket))
for i, e := range endings {
if len(endings)-i > admission.MaxEndedCandidates {
for _, root := range e.c.Roots {
refs[root]--
}
if err = candidates.Delete([]byte(e.id)); err != nil {
return err
}
continue
}
var pos uint64
s.Ended, pos = s.Ended.Push()
if err = rings.Put(ringKey(ringEnded, pos), []byte(e.id)); err != nil {
return err
}
}
type looseEvidence struct {
id admission.EvidenceID
at time.Time
}
var loose []looseEvidence
var expired []admission.EvidenceID
evidence, references := tx.Bucket([]byte(admissionEvidenceBucket)), tx.Bucket([]byte(admissionRefsBucket))
err = evidence.ForEach(func(k, v []byte) error {
e, walkErr := admission.UnmarshalEvidence(v)
if walkErr != nil {
return corruptRecord(walkErr)
}
id := admission.EvidenceID(k)
if e.ID() != id {
return admission.ErrCorruptRecord
}
n, named := refs[id]
delete(refs, id)
if !named || n == 0 {
if named && !clock.Now().Before(e.ObservedAt().Add(admission.SupportLookback)) {
expired = append(expired, id)
return nil
}
loose = append(loose, looseEvidence{id, e.ObservedAt()})
return nil
}
data, walkErr := admission.EvidenceRefs{Refs: n}.MarshalBinary()
if walkErr != nil {
return walkErr
}
return references.Put(k, data)
})
if err != nil {
return err
}
for _, id := range expired {
if err = deleteEvidence(tx, id); err != nil {
return err
}
}
sort.Slice(loose, func(i, j int) bool {
if !loose[i].at.Equal(loose[j].at) {
return loose[i].at.Before(loose[j].at)
}
return loose[i].id < loose[j].id
})
reports := tx.Bucket([]byte(admissionReportsBucket))
for i, e := range loose {
if len(loose)-i > admission.MaxLooseEvidence {
if err = evidence.Delete([]byte(e.id)); err != nil {
return err
}
if err = reports.Delete([]byte(e.id)); err != nil {
return err
}
continue
}
var pos uint64
s.Loose, pos = s.Loose.Push()
if err = rings.Put(ringKey(ringLoose, pos), []byte(e.id)); err != nil {
return err
}
data, encErr := admission.EvidenceRefs{Loose: pos}.MarshalBinary()
if encErr != nil {
return encErr
}
if err = references.Put([]byte(e.id), data); err != nil {
return err
}
}
return putStorageState(tx, s)
}
// loadStorage loads the storage state and proves it against the stored
// entries: the history entries must add up to each allowance's usage and
// the recovery reserve, and each ring must hold as many entries as it
// counts, at positions it has handed out.
func loadStorage(tx *bolt.Tx) (admission.StorageState, error) {
s, err := loadStorageState(tx)
if err != nil {
return s, err
}
var general, reserved, recovery uint64
err = tx.Bucket([]byte(admissionHistoryBucket)).ForEach(func(k, v []byte) error {
if _, parseErr := admission.ParseCandidateID(string(k)); parseErr != nil {
return admission.ErrCorruptRecord
}
h, decodeErr := admission.UnmarshalHistoryEntry(v)
switch {
case decodeErr != nil:
return decodeErr
case h.Pinned:
recovery += uint64(h.Charged()) + admission.MaxIntentBytes
default:
general += uint64(h.General)
reserved += uint64(h.Reserved)
}
return nil
})
if err != nil {
return s, err
}
if general != s.General.Used || reserved != s.Reserved.Used || recovery != s.Recovery {
return s, admission.ErrCorruptRecord
}
if err = proveOutbox(tx, s); err != nil {
return s, err
}
var ended, loose admission.RingState
err = tx.Bucket([]byte(admissionRingsBucket)).ForEach(func(k, v []byte) error {
if len(k) != 9 || v == nil {
return admission.ErrCorruptRecord
}
pos := binary.BigEndian.Uint64(k[1:])
if pos == 0 {
return admission.ErrCorruptRecord
}
switch k[0] {
case ringEnded:
ended.Count++
ended.Last = max(ended.Last, pos)
case ringLoose:
loose.Count++
loose.Last = max(loose.Last, pos)
default:
return admission.ErrCorruptRecord
}
return nil
})
if err != nil {
return s, err
}
if ended.Count != s.Ended.Count || ended.Last > s.Ended.Last || loose.Count != s.Loose.Count || loose.Last > s.Loose.Last {
return s, admission.ErrCorruptRecord
}
return s, proveStorageLinks(tx)
}
// Storage is the committed storage state.
func (l *AdmissionLedger) Storage() (admission.StorageState, error) {
var s admission.StorageState
err := l.db.bolt.View(func(tx *bolt.Tx) error {
var err error
s, err = loadStorageState(tx)
return err
})
return s, err
}
// storageState is the transaction's storage state, loaded on first use.
func (q *queueTx) storageState() (*admission.StorageState, error) {
if !q.storageLoaded {
s, err := loadStorageState(q.tx)
if err != nil {
return nil, err
}
q.storage, q.storageLoaded = s, true
}
return &q.storage, nil
}
func loadRefs(tx *bolt.Tx, id admission.EvidenceID) (admission.EvidenceRefs, error) {
raw := tx.Bucket([]byte(admissionRefsBucket)).Get([]byte(id))
if raw == nil {
return admission.EvidenceRefs{}, admission.ErrCorruptRecord
}
return admission.UnmarshalEvidenceRefs(raw)
}
func putRefs(tx *bolt.Tx, id admission.EvidenceID, r admission.EvidenceRefs) error {
data, err := r.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionRefsBucket)).Put([]byte(id), data)
}
// loosen gives evidence no candidate names the next loose position.
func (q *queueTx) loosen(id admission.EvidenceID) error {
s, err := q.storageState()
if err != nil {
return err
}
var pos uint64
s.Loose, pos = s.Loose.Push()
q.storageDirty = true
if err = q.tx.Bucket([]byte(admissionRingsBucket)).Put(ringKey(ringLoose, pos), []byte(id)); err != nil {
return err
}
return putRefs(q.tx, id, admission.EvidenceRefs{Loose: pos})
}
// name records one more stored candidate naming id as a root. Loose
// evidence leaves its ring position.
func (q *queueTx) name(id admission.EvidenceID) error {
r, err := loadRefs(q.tx, id)
if err != nil {
return err
}
if r.Loose != 0 {
rings := q.tx.Bucket([]byte(admissionRingsBucket))
key := ringKey(ringLoose, r.Loose)
if string(rings.Get(key)) != string(id) {
return admission.ErrCorruptRecord
}
s, err := q.storageState()
if err != nil {
return err
}
if s.Loose, err = s.Loose.Remove(); err != nil {
return err
}
q.storageDirty = true
if err = rings.Delete(key); err != nil {
return err
}
r = admission.EvidenceRefs{}
}
r.Refs++
return putRefs(q.tx, id, r)
}
// unname records that one stored candidate no longer names id. Evidence no
// candidate names keeps a loose position while it could still support a
// candidate, and is removed once it cannot.
func (q *queueTx) unname(id admission.EvidenceID) error {
r, err := loadRefs(q.tx, id)
if err != nil {
return err
}
if r.Refs == 0 {
return admission.ErrCorruptRecord
}
if r.Refs--; r.Refs > 0 {
return putRefs(q.tx, id, r)
}
e, err := loadStoredEvidence(q.tx, id)
if errors.Is(err, admission.ErrEvidenceUnpublished) {
return admission.ErrCorruptRecord
}
if err != nil {
return err
}
if q.now.Before(e.ObservedAt().Add(admission.SupportLookback)) {
return q.loosen(id)
}
return deleteEvidence(q.tx, id)
}
// ended records a candidate that has just ended. One that ended before any
// attempt is not history: it takes the next ended position, and only the
// newest are kept. An admitted one's history is dated: it may be retired
// after the review window and, if verified, its effect's lifetime; an
// unresolved outcome is pinned in the recovery reserve instead.
func (q *queueTx) ended(id admission.CandidateID, c admission.Candidate) error {
if c.Attempts > 0 {
h, found, err := loadHistoryEntry(q.tx, id)
if err != nil {
return err
}
if !found || !h.Ended.IsZero() {
return admission.ErrCorruptRecord
}
h.Ended = q.now
if c.State == admission.StateUnknown {
s, err := q.storageState()
if err != nil {
return err
}
if *s, err = s.PinHistory(h.General, h.Reserved); err != nil {
return err
}
if *s, err = s.HoldIntent(); err != nil {
return err
}
h.Pinned, q.storageDirty = true, true
} else {
h.Eligible, _ = admission.HistoryTimes(q.now, c.ExpiresAt, admission.HistoryUntilExpiry(c))
}
return putHistoryEntry(q.tx, id, h)
}
s, err := q.storageState()
if err != nil {
return err
}
var pos uint64
s.Ended, pos = s.Ended.Push()
q.storageDirty = true
return q.tx.Bucket([]byte(admissionRingsBucket)).Put(ringKey(ringEnded, pos), []byte(id))
}
// deleteEvidence removes a record no candidate names, with its report
// links and reference count.
func deleteEvidence(tx *bolt.Tx, id admission.EvidenceID) error {
// Prove the records before removal so eviction cannot conceal damage.
e, err := loadStoredEvidence(tx, id)
if err != nil {
return corruptRecord(err)
}
if _, err = loadReports(tx, e); err != nil {
return err
}
for _, name := range []string{admissionEvidenceBucket, admissionReportsBucket, admissionRefsBucket} {
if err := tx.Bucket([]byte(name)).Delete([]byte(id)); err != nil {
return err
}
}
return nil
}
// flushStorage keeps each ring within its bound, oldest out first, and
// writes the state back if it changed. It runs when the transaction's
// work is done, so a record named late in the transaction is kept.
func (q *queueTx) flushStorage() error {
if !q.storageDirty {
return nil
}
s := &q.storage
cur := q.tx.Bucket([]byte(admissionRingsBucket)).Cursor()
// Removed endings can loosen their roots, so they go first.
for s.Ended.Count > admission.MaxEndedCandidates {
k, v := cur.Seek([]byte{ringEnded})
if len(k) != 9 || k[0] != ringEnded || v == nil || binary.BigEndian.Uint64(k[1:]) == 0 {
return admission.ErrCorruptRecord
}
id := admission.CandidateID(v)
c, err := loadCandidate(q.tx, id)
if errors.Is(err, errCandidateMissing) || (err == nil && (!c.State.Terminal() || c.Attempts != 0)) {
return admission.ErrCorruptRecord
}
if err != nil {
return err
}
if err = cur.Delete(); err != nil {
return err
}
if err = q.tx.Bucket([]byte(admissionCandidatesBucket)).Delete([]byte(id)); err != nil {
return err
}
if err = q.forgetLine(id, c); err != nil {
return err
}
for _, root := range c.Roots {
if err = q.unname(root); err != nil {
return err
}
}
if s.Ended, err = s.Ended.Remove(); err != nil {
return err
}
}
for s.Loose.Count > admission.MaxLooseEvidence {
k, v := cur.Seek([]byte{ringLoose})
if len(k) != 9 || k[0] != ringLoose || v == nil || binary.BigEndian.Uint64(k[1:]) == 0 {
return admission.ErrCorruptRecord
}
id := admission.EvidenceID(v)
r, err := loadRefs(q.tx, id)
if err != nil {
return err
}
if r.Loose != binary.BigEndian.Uint64(k[1:]) {
return admission.ErrCorruptRecord
}
if err = cur.Delete(); err != nil {
return err
}
if err = deleteEvidence(q.tx, id); err != nil {
return err
}
if s.Loose, err = s.Loose.Remove(); err != nil {
return err
}
}
q.storageDirty = false
return putStorageState(q.tx, *s)
}
func loadHistoryEntry(tx *bolt.Tx, id admission.CandidateID) (admission.HistoryEntry, bool, error) {
raw := tx.Bucket([]byte(admissionHistoryBucket)).Get([]byte(id))
if raw == nil {
return admission.HistoryEntry{}, false, nil
}
h, err := admission.UnmarshalHistoryEntry(raw)
return h, true, err
}
// chargeHistory charges an admitted candidate's history to the allowance of
// the lane it was admitted on: what its history cost grew by since its last
// reservation, from the lane's credit and within its allowance. A
// reservation is refused while the candidate, if its outcome were
// unresolved, and its attempt's audit rows would not fit the recovery
// reserve.
func (q *queueTx) chargeHistory(id admission.CandidateID, c admission.Candidate, lane admission.Lane) error {
cost, err := historyCostOf(q.tx, c)
if err != nil {
return err
}
h, found, err := loadHistoryEntry(q.tx, id)
if err != nil {
return err
}
if c.Attempts > 1 && !found || found && !h.Ended.IsZero() {
return admission.ErrCorruptRecord
}
if room, roomErr := q.recoveryRoom(); roomErr != nil {
return roomErr
} else if uint64(max(cost, h.Charged()))+admission.AttemptRecoveryBytes > room {
err = refusal(admission.ReasonPendingRecovery, "outstanding outcomes fill the recovery reserve")
return err
}
return q.growHistory(id, c, h, lane, cost)
}
// chargeLink charges an admitted candidate's history for the link it now
// carries, on the lane its last attempt was admitted on (the general one
// for an attempt reserved before lanes were kept).
func (q *queueTx) chargeLink(id admission.CandidateID, c admission.Candidate) error {
// The caller counts termination after this charge; cost validation
// needs the complete ending even when the failed attempt never ran.
c.Transitions++
cost, err := historyCostOf(q.tx, c)
if err != nil {
return err
}
h, found, err := loadHistoryEntry(q.tx, id)
if err != nil {
return err
}
if !found || !h.Ended.IsZero() {
return admission.ErrCorruptRecord
}
a, err := currentAttempt(q.tx, c)
if err != nil {
return err
}
lane := a.Lane
if lane == 0 {
lane = admission.LaneGeneral
}
return q.growHistory(id, c, h, lane, cost)
}
// growHistory raises entry h of admitted candidate c to cost on lane: what
// its history grew by, from the lane's credit and within its allowance.
func (q *queueTx) growHistory(id admission.CandidateID, c admission.Candidate, h admission.HistoryEntry, lane admission.Lane, cost uint32) error {
if cost <= h.Charged() {
return nil
}
s, err := q.storageState()
if err != nil {
return err
}
grown := cost - h.Charged()
// A charge the credit cannot cover is refused after this, and the
// whole transaction, retirements included, rolls back.
if err = q.retireForRoom(lane, uint64(grown)); err != nil {
return err
}
next, err := s.ChargeHistory(lane, grown)
if err != nil {
return err
}
if lane == admission.LaneGeneral {
h.General += grown
} else {
h.Reserved += grown
}
h.RootMask = (1 << len(c.Roots)) - 1
h.LegacyCost = false
if err = putHistoryEntry(q.tx, id, h); err != nil {
return err
}
*s, q.storageDirty = next, true
return nil
}
// retireForRoom retires ended history of lane's allowance that may be
// retired, oldest first, until the allowance has room for need bytes or
// nothing more may be retired.
func (q *queueTx) retireForRoom(lane admission.Lane, need uint64) error {
s, err := q.storageState()
if err != nil {
return err
}
kind := retireKind(lane)
cur := q.tx.Bucket([]byte(admissionRetireBucket)).Cursor()
k, _ := cur.Seek([]byte{kind})
for s.HistoryRoom(lane) < need {
if k == nil || k[0] != kind {
return nil
}
_, at, id, err := admission.ParseRetireKey(k)
if err != nil {
return err
}
if at.After(q.now) {
return nil
}
key := bytes.Clone(k)
if err = q.retire(id, key); err != nil {
return err
}
// A live episode can keep this history. Advance past it; a
// deletion instead leaves Seek at the next remaining entry.
k, _ = cur.Seek(key)
if bytes.Equal(k, key) {
k, _ = cur.Next()
}
}
return nil
}
// retirementsPerTick bounds the retirements one Tick makes, so a backlog
// after downtime clears over several ticks instead of in one long clock
// transaction. Pressure retires what it needs at once.
const retirementsPerTick = 64
// retireAtTarget examines at most retirementsPerTick entries after the
// previous batch. A retained episode must not hide later eligible history.
// A nil continuation starts the next pass at the oldest target.
func (q *queueTx) retireAtTarget(after []byte) ([]byte, error) {
cur := q.tx.Bucket([]byte(admissionRetireBucket)).Cursor()
k, _ := cur.Seek([]byte{admission.RetireTarget})
if len(after) != 0 {
k, _ = cur.Seek(after)
if bytes.Equal(k, after) {
k, _ = cur.Next()
}
}
for n := 0; n < retirementsPerTick; n++ {
if k == nil || k[0] != admission.RetireTarget {
return nil, nil
}
_, at, id, err := admission.ParseRetireKey(k)
if err != nil {
return nil, err
}
if at.After(q.now) {
return nil, nil
}
key := bytes.Clone(k)
if err = q.retire(id, key); err != nil {
return nil, err
}
after = key
// A live episode can keep this history. Advance past it; a
// deletion instead leaves Seek at the next remaining entry.
k, _ = cur.Seek(key)
if bytes.Equal(k, key) {
k, _ = cur.Next()
}
}
return after, nil
}
// meterStorage credits a tick's elapsed time to the history allowances and
// retires the history whose target has passed, in the clock's transaction.
func (l *AdmissionLedger) meterStorage(tx *bolt.Tx, tick admission.ClockTick) ([]byte, error) {
q, err := openQueueWith(tx, l.reg, l.Inventory(), tick.Now)
if err != nil {
return nil, err
}
s, err := q.storageState()
if err != nil {
return nil, err
}
if *s, err = s.Advance(tick.Elapsed); err != nil {
return nil, err
}
q.storageDirty = true
after, err := q.retireAtTarget(l.retireAfter)
if err != nil {
return nil, err
}
if err = q.removeQuietNotices(); err != nil {
return nil, err
}
return after, q.flushStorage()
}
// retirementTime keeps history that pays for a verified episode until
// both boundaries pass. Scheduling and deletion must use the same gate.
func (q *queueTx) retirementTime(c admission.Candidate, at time.Time) (time.Time, error) {
if c.State != admission.StateVerified || (c.Key.Kind != admission.KindBlockIP && c.Key.Kind != admission.KindBlockSubnet) {
return at, nil
}
e, found, err := loadEpisode(q.tx, c.Key.Target.Key())
if err != nil || !found || e.ID != c.Key.Episode || e.Verified.IsZero() {
return at, err
}
clock, err := q.episodeClock()
if err != nil {
return time.Time{}, err
}
left := max(e.Verified.Sub(clock.Now), e.EndElapsed-clock.Elapsed)
if left > 0 && clock.Now.Add(left).After(at) {
at = clock.Now.Add(left)
}
return at, nil
}
// retire removes an ended candidate's details: its record, its attempts,
// its history entry and index keys. Its bytes return to their allowances
// and its roots are released. key is the index key that led here; it must
// be one of the entry's own.
func (q *queueTx) retire(id admission.CandidateID, key []byte) error {
h, found, err := loadHistoryEntry(q.tx, id)
if err != nil {
return err
}
if !found {
return admission.ErrCorruptRecord
}
c, err := loadCandidate(q.tx, id)
if errors.Is(err, errCandidateMissing) || (err == nil && (!c.State.Terminal() || c.Attempts == 0)) {
return admission.ErrCorruptRecord
}
if err != nil {
return err
}
keys, err := h.RetireKeys(id)
if err != nil {
return err
}
if !slices.ContainsFunc(keys, func(k []byte) bool { return bytes.Equal(k, key) }) {
return admission.ErrCorruptRecord
}
if err = proveReleasedHistory(q.tx, c, h); err != nil {
return err
}
// Deletion must not conceal missing or inconsistent attempt history.
if _, err = currentAttempt(q.tx, c); err != nil {
return err
}
at, err := q.retirementTime(c, q.now)
if err != nil || at.After(q.now) {
return err
}
retireIndex := q.tx.Bucket([]byte(admissionRetireBucket))
for _, k := range keys {
if retireIndex.Get(k) == nil {
return admission.ErrCorruptRecord
}
if err = retireIndex.Delete(k); err != nil {
return err
}
}
attempts := q.tx.Bucket([]byte(admissionAttemptsBucket))
for seq := uint32(1); seq <= c.Attempts; seq++ {
a, idErr := admission.NewAttempt(id, seq)
if idErr != nil {
return idErr
}
if err = attempts.Delete([]byte(a.ID)); err != nil {
return err
}
}
for _, name := range []string{admissionHistoryBucket, admissionCandidatesBucket} {
if err = q.tx.Bucket([]byte(name)).Delete([]byte(id)); err != nil {
return err
}
}
if err = q.forgetLine(id, c); err != nil {
return err
}
for _, root := range c.Roots {
if err = q.unname(root); err != nil {
return err
}
}
s, err := q.storageState()
if err != nil {
return err
}
if *s, err = s.ReleaseHistory(h.General, h.Reserved); err != nil {
return err
}
q.storageDirty = true
return nil
}
// historyNeed is what reserving lc now would charge its lane's history,
// the reserve room the reservation needs, and whether that fits.
func (q *queueTx) historyNeed(lc liveCandidate, room uint64) (uint32, uint32, bool, error) {
cost, err := historyCostOf(q.tx, lc.c)
if err != nil {
return 0, 0, false, err
}
h, found, err := loadHistoryEntry(q.tx, lc.id)
if err != nil {
return 0, 0, false, err
}
if lc.c.Attempts > 0 && !found {
return 0, 0, false, admission.ErrCorruptRecord
}
recovery := max(cost, h.Charged()) + admission.AttemptRecoveryBytes
fits := uint64(recovery) <= room
if cost <= h.Charged() {
return 0, recovery, fits, nil
}
return cost - h.Charged(), recovery, fits, nil
}
func retireKind(lane admission.Lane) byte {
if lane == admission.LaneGeneral {
return admission.RetireGeneral
}
return admission.RetireReserved
}
// retirable is the bytes of lane's allowance that ended history which may
// be retired now holds, counted until they reach limit.
func (q *queueTx) retirable(lane admission.Lane, limit uint64) (uint64, error) {
kind := retireKind(lane)
var sum uint64
cur := q.tx.Bucket([]byte(admissionRetireBucket)).Cursor()
for k, _ := cur.Seek([]byte{kind}); k != nil && k[0] == kind && sum < limit; k, _ = cur.Next() {
_, at, id, err := admission.ParseRetireKey(k)
if err != nil {
return 0, err
}
if at.After(q.now) {
break
}
h, found, err := loadHistoryEntry(q.tx, id)
if err != nil {
return 0, err
}
if !found {
return 0, admission.ErrCorruptRecord
}
c, err := loadCandidate(q.tx, id)
if err != nil {
return 0, corruptRecord(err)
}
at, err = q.retirementTime(c, at)
if err != nil {
return 0, err
}
if at.After(q.now) {
continue
}
if lane == admission.LaneGeneral {
sum += uint64(h.General)
} else {
sum += uint64(h.Reserved)
}
}
return sum, nil
}
// historyBudgets are the history bytes each lane may charge now: its
// credit, bounded by the room its allowance has once retirable history is
// retired.
func (q *queueTx) historyBudgets() (general, reserved uint64, err error) {
s, err := q.storageState()
if err != nil {
return 0, 0, err
}
budget := func(lane admission.Lane) (uint64, error) {
size, _ := admission.HistoryLanes()
used := s.General.Used
if lane != admission.LaneGeneral {
_, size = admission.HistoryLanes()
used = s.Reserved.Used
}
// Subtract first so even a grandfathered overfull allowance cannot wrap.
need := used - min(used, size-s.HistoryCredit(lane))
r, walkErr := q.retirable(lane, need)
return s.HistoryBudget(lane, r), walkErr
}
if general, err = budget(admission.LaneGeneral); err != nil {
return 0, 0, err
}
reserved, err = budget(admission.LaneDirect)
return general, reserved, err
}
// nextRetirable is the earliest future retirement of lane's history,
// including entries whose wall eligibility precedes their episode's end.
func (q *queueTx) nextRetirable(lane admission.Lane) (time.Time, bool, error) {
kind := retireKind(lane)
var next time.Time
cur := q.tx.Bucket([]byte(admissionRetireBucket)).Cursor()
for k, _ := cur.Seek([]byte{kind}); k != nil && k[0] == kind; k, _ = cur.Next() {
_, at, id, err := admission.ParseRetireKey(k)
if err != nil {
return time.Time{}, false, err
}
if !next.IsZero() && !at.Before(next) {
break
}
c, err := loadCandidate(q.tx, id)
if err != nil {
return time.Time{}, false, corruptRecord(err)
}
at, err = q.retirementTime(c, at)
if err != nil {
return time.Time{}, false, err
}
if at.After(q.now) && (next.IsZero() || at.Before(next)) {
next = at
}
}
return next, !next.IsZero(), nil
}
// validateUnownedRows checks rows that candidate and evidence walks do not
// necessarily reach. An upgrade must not silently retain or discard them.
func validateUnownedRows(tx *bolt.Tx) error {
if err := tx.Bucket([]byte(admissionReportsBucket)).ForEach(func(k, v []byte) error {
if v == nil {
return admission.ErrCorruptRecord
}
raw := tx.Bucket([]byte(admissionEvidenceBucket)).Get(k)
e, err := admission.UnmarshalEvidence(raw)
if err != nil || string(e.ID()) != string(k) {
return admission.ErrCorruptRecord
}
_, err = loadReports(tx, e)
return err
}); err != nil {
return err
}
return tx.Bucket([]byte(admissionAttemptsBucket)).ForEach(func(k, v []byte) error {
a, err := admission.UnmarshalAttempt(v)
if err != nil || string(a.Attempt.ID) != string(k) {
return admission.ErrCorruptRecord
}
c, err := loadCandidate(tx, a.Attempt.Candidate)
if err != nil || a.Attempt.Seq > c.Attempts {
return admission.ErrCorruptRecord
}
return nil
})
}
// recoveryRoom is what the recovery and outbox reserve has left after
// pinned history and the outbox. It also holds space for every outstanding
// attempt's history and intent: finishing several such attempts as unknown
// cannot overbook it.
func (q *queueTx) recoveryRoom() (uint64, error) {
s, err := q.storageState()
if err != nil {
return 0, err
}
if s.Recovery >= admission.RecoveryReserveBytes || s.OutboxBytes() >= admission.RecoveryReserveBytes-s.Recovery {
return 0, nil
}
room := admission.RecoveryReserveBytes - s.Recovery - s.OutboxBytes()
live, err := q.live()
if err != nil {
return 0, err
}
for _, lc := range live {
if lc.c.State != admission.StateReserved && lc.c.State != admission.StateExecuting {
continue
}
h, found, loadErr := loadHistoryEntry(q.tx, lc.id)
if loadErr != nil {
return 0, loadErr
}
if !found {
return 0, admission.ErrCorruptRecord
}
hold := uint64(h.Charged()) + admission.UnknownHoldBytes
if hold > room {
return 0, nil
}
room -= hold
}
return room, nil
}
// remapHistoryRoots preserves the paid roots' identity when sorting new
// retry support changes their positions in the candidate.
func (q *queueTx) remapHistoryRoots(c admission.Candidate, merged []admission.EvidenceID) error {
if c.Attempts == 0 {
return nil
}
id, _ := c.ID()
h, found, err := loadHistoryEntry(q.tx, id)
if err != nil {
return err
}
if !found || h.RootMask == 0 || h.RootMask>>len(c.Roots) != 0 {
return admission.ErrCorruptRecord
}
var mask uint32
for i, root := range c.Roots {
if h.RootMask&(1<<i) != 0 {
mask |= 1 << slices.Index(merged, root)
}
}
h.RootMask = mask
return putHistoryEntry(q.tx, id, h)
}
// trimUnreservedRoots returns retry-only support to the loose ring when a
// queued candidate ends without another reservation. Paid evidence stays.
func (q *queueTx) trimUnreservedRoots(c *admission.Candidate) error {
if c.Attempts == 0 {
return nil
}
id, _ := c.ID()
h, found, err := loadHistoryEntry(q.tx, id)
if err != nil {
return err
}
if !found || h.RootMask == 0 || h.RootMask>>len(c.Roots) != 0 {
return admission.ErrCorruptRecord
}
var roots []admission.EvidenceID
for i, root := range c.Roots {
if h.RootMask&(1<<i) != 0 {
roots = append(roots, root)
} else if err = q.unname(root); err != nil {
return err
}
}
c.Roots = roots
h.RootMask = (1 << len(roots)) - 1
return putHistoryEntry(q.tx, id, h)
}
// proveStorageLinks checks ownership as well as totals. No uncharged row,
// missing reference or foreign retirement key may survive a reopen.
func proveStorageLinks(tx *bolt.Tx) error {
if err := validateUnownedRows(tx); err != nil {
return err
}
refs := map[admission.EvidenceID]uint32{}
history := map[string]bool{}
ended := map[string]bool{}
queued := map[string]bool{}
keys := map[string]bool{}
if err := tx.Bucket([]byte(admissionCandidatesBucket)).ForEach(func(k, _ []byte) error {
c, err := loadCandidate(tx, admission.CandidateID(k))
if err != nil {
return corruptRecord(err)
}
for _, root := range c.Roots {
refs[root]++
}
if !c.State.Terminal() {
queued[string(k)] = true
}
if c.Attempts == 0 {
if c.State.Terminal() {
ended[string(k)] = true
}
return nil
}
last, err := currentAttempt(tx, c)
if err != nil {
return err
}
h, found, err := loadHistoryEntry(tx, admission.CandidateID(k))
if err != nil {
return err
}
if !found || h.RootMask == 0 || h.RootMask>>len(c.Roots) != 0 || c.State.Terminal() == h.Ended.IsZero() || h.Pinned != (c.State == admission.StateUnknown && !last.Retired) {
return admission.ErrCorruptRecord
}
if c.State != admission.StateQueued && h.RootMask != (1<<len(c.Roots))-1 {
return admission.ErrCorruptRecord
}
paid := c
paid.Roots = nil
for i, root := range c.Roots {
if h.RootMask&(1<<i) != 0 {
paid.Roots = append(paid.Roots, root)
}
}
cost, err := historyCostOf(tx, paid)
if err != nil {
return admission.ErrCorruptRecord
}
if h.LegacyCost {
// Upgrading keeps old history charges and invents no episode
// rows. Earlier schema 6 writers also used version 1, so a
// candidate whose episode row exists still pays its allowance.
episode, exists, episodeErr := loadEpisode(tx, c.Key.Target.Key())
if episodeErr != nil {
return episodeErr
}
if !exists || episode.ID != c.Key.Episode {
cost -= admission.MaxEpisodeBytes
}
}
if h.Charged() < cost {
return admission.ErrCorruptRecord
}
if c.State.Terminal() {
if last.State == c.State && !h.Ended.Equal(last.Finished) {
return admission.ErrCorruptRecord
}
if !h.Pinned {
// Endings before the row rule kept only verified history
// until expiry; such an entry keeps its time. The live
// start released a preview's history at its own time; a
// ledger without one has none.
eligible, _ := admission.HistoryTimes(h.Ended, c.ExpiresAt, admission.HistoryUntilExpiry(c))
earlier, _ := admission.HistoryTimes(h.Ended, c.ExpiresAt, c.State == admission.StateVerified)
switch {
case h.Released:
if proofErr := proveReleasedHistory(tx, c, h); proofErr != nil {
return proofErr
}
case !h.Eligible.Equal(eligible) && !h.Eligible.Equal(earlier):
return admission.ErrCorruptRecord
}
}
}
history[string(k)] = true
retire, err := retireKeysOf(tx, admission.CandidateID(k), h)
if err != nil {
return err
}
for _, key := range retire {
keys[string(key)] = true
}
return nil
}); err != nil {
return err
}
for _, check := range []struct {
name string
want map[string]bool
}{
{admissionHistoryBucket, history}, {admissionRetireBucket, keys}, {admissionQueueBucket, queued},
} {
b := tx.Bucket([]byte(check.name))
if err := b.ForEach(func(k, v []byte) error {
if !check.want[string(k)] || b.Bucket(k) != nil {
return admission.ErrCorruptRecord
}
if check.name == admissionRetireBucket && len(v) != 0 {
return admission.ErrCorruptRecord
}
if check.name == admissionQueueBucket {
if _, err := admission.UnmarshalQueueEntry(v); err != nil {
return err
}
}
delete(check.want, string(k))
return nil
}); err != nil {
return err
}
if len(check.want) != 0 {
return admission.ErrCorruptRecord
}
}
loose := map[admission.EvidenceID]uint64{}
if err := tx.Bucket([]byte(admissionRingsBucket)).ForEach(func(k, v []byte) error {
if len(k) != 9 {
return admission.ErrCorruptRecord
}
switch k[0] {
case ringEnded:
if !ended[string(v)] {
return admission.ErrCorruptRecord
}
delete(ended, string(v))
case ringLoose:
id := admission.EvidenceID(v)
if refs[id] != 0 || loose[id] != 0 {
return admission.ErrCorruptRecord
}
loose[id] = binary.BigEndian.Uint64(k[1:])
default:
return admission.ErrCorruptRecord
}
return nil
}); err != nil {
return err
}
if len(ended) != 0 {
return admission.ErrCorruptRecord
}
evidence := map[string]bool{}
if err := tx.Bucket([]byte(admissionEvidenceBucket)).ForEach(func(k, v []byte) error {
e, err := admission.UnmarshalEvidence(v)
if err != nil || string(e.ID()) != string(k) {
return admission.ErrCorruptRecord
}
r, err := loadRefs(tx, e.ID())
if err != nil {
return err
}
if r.Refs != refs[e.ID()] || r.Loose != loose[e.ID()] {
return admission.ErrCorruptRecord
}
delete(refs, e.ID())
delete(loose, e.ID())
evidence[string(k)] = true
return nil
}); err != nil {
return err
}
if len(refs) != 0 || len(loose) != 0 {
return admission.ErrCorruptRecord
}
return tx.Bucket([]byte(admissionRefsBucket)).ForEach(func(k, _ []byte) error {
if !evidence[string(k)] {
return admission.ErrCorruptRecord
}
return nil
})
}
// proveReleasedHistory protects mutation paths that consume a retirement
// time after open. A valid checksum alone cannot waive genuine retention.
func proveReleasedHistory(tx *bolt.Tx, c admission.Candidate, h admission.HistoryEntry) error {
if !h.Released {
return nil
}
start, found, err := loadLiveStart(tx)
if err != nil {
return err
}
if !found || admission.HistoryUntilExpiry(c) || !h.Eligible.Equal(start.At) {
return admission.ErrCorruptRecord
}
return nil
}
package store
import (
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
var _ admission.Ledger = (*AdmissionLedger)(nil)
func loadAttempt(tx *bolt.Tx, id admission.ActionID) (admission.AttemptRecord, error) {
if _, err := admission.ParseActionID(string(id)); err != nil {
return admission.AttemptRecord{}, err
}
raw := tx.Bucket([]byte(admissionAttemptsBucket)).Get([]byte(id))
if raw == nil {
return admission.AttemptRecord{}, refusal(admission.ReasonInvalid, "attempt is not recorded")
}
a, err := admission.UnmarshalAttempt(raw)
if err != nil {
return admission.AttemptRecord{}, err
}
if a.Attempt.ID != id {
return admission.AttemptRecord{}, admission.ErrCorruptRecord
}
return a, nil
}
func putAttempt(tx *bolt.Tx, a admission.AttemptRecord) error {
data, err := a.MarshalBinary()
if err != nil {
return err
}
return tx.Bucket([]byte(admissionAttemptsBucket)).Put([]byte(a.Attempt.ID), data)
}
// Attempt loads an attempt record.
func (l *AdmissionLedger) Attempt(id admission.ActionID) (admission.AttemptRecord, error) {
var a admission.AttemptRecord
err := l.db.bolt.View(func(tx *bolt.Tx) error {
var err error
a, err = loadAttempt(tx, id)
return err
})
return a, err
}
// modify applies fn to one candidate in a write transaction. fn reports
// whether it changed the record; only a change bumps Transitions. Queue
// bookkeeping fn changes commits with the candidate.
func (l *AdmissionLedger) modify(op string, id admission.CandidateID, fn func(q *queueTx, c *admission.Candidate) (bool, error)) (admission.Candidate, error) {
l.mu.Lock()
defer l.mu.Unlock()
var out admission.Candidate
err := l.update(op, func(tx *bolt.Tx) error {
c, err := loadCandidate(tx, id)
if err != nil {
return err
}
if c.Attempts > 0 {
if _, err = currentAttempt(tx, c); err != nil {
return err
}
}
q, err := l.openQueue(tx, l.now)
if err != nil {
return err
}
from := c.State
changed, err := fn(q, &c)
if err != nil {
return err
}
out = c
if !changed {
return nil
}
if !admission.CanTransition(from, c.State) {
return admission.ErrTransitionConflict
}
out.Transitions++
if err = putCandidate(tx, out); err != nil {
return err
}
return q.flush()
})
if err != nil {
return admission.Candidate{}, err
}
return out, nil
}
// Defer records a changed deferral reason on a queued candidate. The same
// reason again is not a new transition, so polling never inflates counts.
func (l *AdmissionLedger) Defer(id admission.CandidateID, reason admission.Reason) (admission.Candidate, error) {
if reason.Disposition() != admission.DispositionDeferred {
return admission.Candidate{}, refusal(admission.ReasonInvalid, "reason is not a deferral")
}
return l.modify("defer", id, func(q *queueTx, c *admission.Candidate) (bool, error) {
switch {
case c.State.Terminal():
return false, admission.ErrCandidateTerminal
case c.State != admission.StateQueued:
return false, admission.ErrTransitionConflict
case c.Reason == reason:
return false, nil
}
e, err := loadQueueEntry(q.tx, id)
if err != nil {
return false, err
}
return true, q.deferred(id, c, e, reason)
})
}
// deferred records reason on queued candidate c, counts the deferral and
// raises its notice. The caller counts the transition and stores c.
func (q *queueTx) deferred(id admission.CandidateID, c *admission.Candidate, e admission.QueueEntry, reason admission.Reason) error {
c.Reason = reason
if err := q.count(admission.EventDeferred, reason, e.Tier); err != nil {
return err
}
return q.gap(admission.GapDeferred, reason, 0, e, id, *c, c.Transitions+1)
}
var terminalFor = map[admission.Disposition]admission.State{
admission.DispositionRefused: admission.StateRefused,
admission.DispositionWithheld: admission.StateWithheld,
admission.DispositionDropped: admission.StateDropped,
}
// Terminate ends a queued candidate as refused, withheld or dropped and
// releases its queue position. The same ending again is a no-op; a
// candidate that already ended otherwise refuses like every other call on
// an ended candidate, and an in-flight one is a conflict.
func (l *AdmissionLedger) Terminate(id admission.CandidateID, reason admission.Reason) (admission.Candidate, error) {
d := reason.Disposition()
state, ok := terminalFor[d]
if !ok {
return admission.Candidate{}, refusal(admission.ReasonInvalid, "reason does not end a candidate")
}
return l.modify("terminate", id, func(q *queueTx, c *admission.Candidate) (bool, error) {
switch {
case c.State == state && c.Reason == reason:
return false, nil
case c.State.Terminal():
return false, admission.ErrCandidateTerminal
case c.State != admission.StateQueued:
return false, admission.ErrTransitionConflict
}
return true, q.terminate(id, c, state, reason, admission.EffectLink{})
})
}
// TerminateExisting ends a queued candidate refused as an existing effect,
// linked to the block that enforces it. The same ending again
// is a no-op; any other ending refuses like every call on an ended
// candidate. An admitted candidate's history pays for the link.
func (l *AdmissionLedger) TerminateExisting(id admission.CandidateID, link admission.EffectLink) (admission.Candidate, error) {
if link == (admission.EffectLink{}) {
return admission.Candidate{}, refusal(admission.ReasonInvalid, "an existing effect needs the block that enforces it")
}
return l.modify("terminate", id, func(q *queueTx, c *admission.Candidate) (bool, error) {
switch {
case c.State == admission.StateRefused && c.Reason == admission.ReasonExistingEffect && c.Link.Effect == link.Effect && c.Link.ExpiresAt.Equal(link.ExpiresAt):
return false, nil
case c.State.Terminal():
return false, admission.ErrCandidateTerminal
case c.State != admission.StateQueued:
return false, admission.ErrTransitionConflict
}
return true, q.terminate(id, c, admission.StateRefused, admission.ReasonExistingEffect, link)
})
}
// terminate ends queued candidate c as reason, linked when link is set,
// and releases its queue position. The caller counts the transition.
func (q *queueTx) terminate(id admission.CandidateID, c *admission.Candidate, state admission.State, reason admission.Reason, link admission.EffectLink) error {
e, err := loadQueueEntry(q.tx, id)
if err != nil {
return err
}
if err = q.trimUnreservedRoots(c); err != nil {
return err
}
c.State, c.Disposition, c.Reason, c.NotBefore, c.Link = state, reason.Disposition(), reason, time.Time{}, link
if link != (admission.EffectLink{}) {
if c.Attempts > 0 {
if err = q.chargeLink(id, *c); err != nil {
return err
}
}
// Older builds cannot read the linked record: refuse them the ledger.
if err = raiseSchema(q.tx, admissionSchemaLinked); err != nil {
return err
}
}
if err = q.release(id, e, admission.EventEnded, reason); err != nil {
return err
}
if err = q.gap(admission.GapEnded, reason, 0, e, id, *c, c.Transitions+1); err != nil {
return err
}
return q.ended(id, *c)
}
// EndDryRun ends a queued candidate as a dry run: dry-run mode
// forecast it instead of running it. Nothing is reserved, charged or
// allocated; a retry keeps its failed attempts, their charges and the
// history they paid for, which ends. The ending counts one dry-run
// outcome and no queue event, raises no notice and writes no audit row,
// and the episode stays open to a later observation's next generation.
// The work must be ready as for a reservation: a retry waits out its
// backoff, a held target waits for recovery and stale work is refused.
// changed is false for the same ending again; any other ended candidate
// is terminal, and one in flight is a conflict.
func (l *AdmissionLedger) EndDryRun(id admission.CandidateID) (admission.Candidate, bool, error) {
var changed bool
c, err := l.modify("dry_run", id, func(q *queueTx, c *admission.Candidate) (bool, error) {
switch {
case c.State == admission.StateDryRun:
return false, nil
case c.State.Terminal():
return false, admission.ErrCandidateTerminal
case c.State != admission.StateQueued:
return false, admission.ErrTransitionConflict
}
if _, err := l.clock(); err != nil {
return false, err
}
if err := q.ready(*c); err != nil {
return false, err
}
changed = true
return true, q.endDryRun(id, c)
})
if err != nil {
return admission.Candidate{}, false, err
}
return c, changed, nil
}
// ready refuses queued candidate c as a reservation of it would before
// charging anything: during its retry backoff, once stale, or while an
// unresolved attempt holds its target.
func (q *queueTx) ready(c admission.Candidate) error {
switch {
case q.now.Before(c.NotBefore):
return admission.ErrNotReady
case !q.now.Before(c.AgeOut):
return refusal(admission.ReasonStale, "candidate aged out of the queue")
case c.Attempts > 0 && !q.now.Before(c.ExpiresAt):
return refusal(admission.ReasonStale, "absolute expiry has passed")
}
held, err := q.heldTarget(c.Key.Target)
if err != nil {
return err
}
if held {
return refusal(admission.ReasonPendingRecovery, "an unresolved attempt holds the target")
}
return nil
}
// endDryRun ends queued candidate c as a dry run and releases its queue
// position without a queue event: its one count is the dry-run outcome.
// The caller counts the transition.
func (q *queueTx) endDryRun(id admission.CandidateID, c *admission.Candidate) error {
e, err := loadQueueEntry(q.tx, id)
if err != nil {
return err
}
if err = q.trimUnreservedRoots(c); err != nil {
return err
}
c.State, c.Disposition, c.Reason, c.NotBefore = admission.StateDryRun, admission.DispositionDryRun, 0, time.Time{}
if err = q.release(id, e, 0, 0); err != nil {
return err
}
if err = q.outcomes.Add(admission.AttemptOutcome(admission.DispositionDryRun, e.Tier)); err != nil {
return err
}
return q.ended(id, *c)
}
// Reserve admits the next attempt of a queued candidate on lane and reports
// true. The first reservation fixes the absolute expiry, which must be in
// the future; retries keep it and refuse once it has passed. Every attempt
// is charged to its lane, retries included, and the charge commits with the
// attempt, as does the history the candidate's details may keep. A
// candidate that is already reserved or executing returns its current
// attempt and false, so recovery reuses the attempt ID without being
// granted or charged anything.
func (l *AdmissionLedger) Reserve(id admission.CandidateID, lane admission.Lane, expiresAt time.Time) (admission.Candidate, admission.AttemptRecord, bool, error) {
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return admission.Candidate{}, admission.AttemptRecord{}, false, err
}
var c admission.Candidate
var a admission.AttemptRecord
var granted bool
var victim admission.EvictionVictim
if err := l.update("reserve", func(tx *bolt.Tx) error {
q, txErr := l.openQueue(tx, now)
if txErr != nil {
return txErr
}
if c, a, granted, txErr = l.reserveTx(q, id, lane, expiresAt, true, &victim); txErr != nil {
return txErr
}
return q.flush()
}); err != nil {
return admission.Candidate{}, admission.AttemptRecord{}, false, err
}
l.reserved = l.reserved || granted
return c, a, granted, nil
}
// reserveTx reserves the next attempt of candidate id. With hold, a block
// a live ledger reserves holds its temporary set slot from now on; only a
// live owner executes, so no record predates the live start. victim
// receives the block whose slot the reservation takes, zero for a free
// slot; a readback finds none.
func (l *AdmissionLedger) reserveTx(q *queueTx, id admission.CandidateID, lane admission.Lane, expiresAt time.Time, hold bool, victim *admission.EvictionVictim) (admission.Candidate, admission.AttemptRecord, bool, error) {
tx, now := q.tx, q.now
c, err := loadCandidate(tx, id)
if err != nil {
return c, admission.AttemptRecord{}, false, err
}
var current admission.AttemptRecord
if c.Attempts > 0 {
current, err = currentAttempt(tx, c)
if err != nil {
return c, admission.AttemptRecord{}, false, err
}
}
switch {
case c.State == admission.StateReserved || c.State == admission.StateExecuting:
if (!expiresAt.IsZero() && !expiresAt.Equal(c.ExpiresAt)) || (lane != 0 && lane != current.Lane) {
return c, admission.AttemptRecord{}, false, admission.ErrTransitionConflict
}
return c, current, false, nil
case c.State.Terminal():
return c, admission.AttemptRecord{}, false, admission.ErrCandidateTerminal
case now.Before(c.NotBefore):
return c, admission.AttemptRecord{}, false, admission.ErrNotReady
case !now.Before(c.AgeOut):
return c, admission.AttemptRecord{}, false, refusal(admission.ReasonStale, "candidate aged out of the queue")
case c.Attempts >= admission.MaxAttempts:
return c, admission.AttemptRecord{}, false, admission.ErrCorruptRecord
}
held, holdErr := q.heldTarget(c.Key.Target)
if holdErr != nil {
return c, admission.AttemptRecord{}, false, holdErr
}
if held {
return c, admission.AttemptRecord{}, false, refusal(admission.ReasonPendingRecovery, "an unresolved attempt holds the target")
}
if c.Attempts == 0 {
if !expiresAt.After(now) {
return c, admission.AttemptRecord{}, false, refusal(admission.ReasonInvalid, "absolute expiry must be in the future")
}
c.ExpiresAt = expiresAt
} else {
if !expiresAt.IsZero() && !expiresAt.Equal(c.ExpiresAt) {
return c, admission.AttemptRecord{}, false, admission.ErrTransitionConflict
}
if !now.Before(c.ExpiresAt) {
return c, admission.AttemptRecord{}, false, refusal(admission.ReasonStale, "absolute expiry has passed")
}
}
// A queued candidate holds a queue position; one without an entry is
// invisible to capacity and fairness.
entry, err := loadQueueEntry(tx, id)
if err != nil {
return c, admission.AttemptRecord{}, false, err
}
if err = laneFits(q, liveCandidate{id: id, c: c, entry: entry}, lane); err != nil {
return c, admission.AttemptRecord{}, false, err
}
// The slots are rechecked under the lock, before anything is charged:
// a free slot of the lane's partition, or under pressure a victim the
// block strictly outranks or, for fresh work, the equal-tier block
// rotation gives it.
if cost := c.Key.Kind.CeilingCost(); cost > 0 {
slots, slotErr := q.slots()
if slotErr != nil {
return c, admission.AttemptRecord{}, false, slotErr
}
fresh, freshErr := q.fresh(c)
if freshErr != nil {
return c, admission.AttemptRecord{}, false, freshErr
}
d := admission.SlotDemand{Tier: entry.Tier, Cost: cost, Evicts: c.Key.Kind == admission.KindBlockIP && cost == 1, Reserved: entry.Direct || entry.Corroborated, Target: c.Key.Target,
Scope: c.Scope.Key(), Fresh: fresh}
var ok bool
if *victim, ok = slots.Room(lane, d); !ok {
return c, admission.AttemptRecord{}, false, refusal(admission.ReasonSetFull, "the temporary set has no free slot for the lane")
}
}
next, err := admission.NewAttempt(id, c.Attempts+1)
if err != nil {
return c, admission.AttemptRecord{}, false, err
}
if tx.Bucket([]byte(admissionAttemptsBucket)).Get([]byte(next.ID)) != nil {
return c, admission.AttemptRecord{}, false, admission.ErrCorruptRecord
}
if !admission.CanTransition(c.State, admission.StateReserved) {
return c, admission.AttemptRecord{}, false, admission.ErrTransitionConflict
}
if cost := c.Key.Kind.CeilingCost(); cost > 0 {
if err = chargeTx(tx, now, next.ID, lane, cost); err != nil {
return c, admission.AttemptRecord{}, false, err
}
}
a := admission.AttemptRecord{Attempt: next, State: admission.StateReserved, ExpiresAt: c.ExpiresAt, Reserved: now, Lane: lane}
c.State, c.Attempts, c.Reason, c.NotBefore = admission.StateReserved, next.Seq, 0, time.Time{}
c.Transitions++
if err = q.chargeHistory(id, c, lane); err != nil {
return c, admission.AttemptRecord{}, false, err
}
// The reservation holds a slot for each row its attempt can write and
// writes the first, its own.
if err = q.adjustAuditSlots(admission.AuditStepsPerAttempt, 0); err != nil {
return c, admission.AttemptRecord{}, false, err
}
written := c
written.Transitions--
if err = q.writeAuditRow(written, a, entry.Tier); err != nil {
return c, admission.AttemptRecord{}, false, err
}
if err := putAttempt(tx, a); err != nil {
return c, a, false, err
}
// Notice allocation must see this attempt's recovery hold.
if err := putCandidate(tx, c); err != nil {
return c, a, false, err
}
if hold && q.started && c.Key.Kind == admission.KindBlockIP {
if err := q.holdSlot(c, a, entry, *victim); err != nil {
return c, a, false, err
}
// A rotation means the set is too small for the attack.
if *victim != (admission.EvictionVictim{}) && victim.Tier == entry.Tier {
check, checkErr := noticeCheck(tx, c)
if checkErr != nil {
return c, a, false, checkErr
}
key := admission.NoticeKey{Kind: admission.NoticeCapacity, Reason: admission.ReasonSetFull, Check: check, Effect: c.Key.Kind.Effect()}
if err := q.raise(key, id, c.Transitions); err != nil {
return c, a, false, err
}
}
}
return c, a, true, nil
}
// currentAttempt proves the whole bounded predecessor chain and its link
// to the candidate. A readable row alone is not proof of admissible history.
func currentAttempt(tx *bolt.Tx, c admission.Candidate) (admission.AttemptRecord, error) {
id, err := c.ID()
if err != nil {
return admission.AttemptRecord{}, admission.ErrCorruptRecord
}
var prev admission.AttemptRecord
for seq := uint32(1); seq <= c.Attempts; seq++ {
identity, err := admission.NewAttempt(id, seq)
if err != nil {
return admission.AttemptRecord{}, admission.ErrCorruptRecord
}
a, err := loadAttempt(tx, identity.ID)
if err != nil || a.Attempt != identity || !a.ExpiresAt.Equal(c.ExpiresAt) || a.Reserved.Before(c.FirstQueued) || !a.Reserved.Before(c.AgeOut) {
return admission.AttemptRecord{}, admission.ErrCorruptRecord
}
if seq > 1 && (prev.State != admission.StateFailed || a.Reserved.Before(prev.Finished.Add(admission.RetryBackoff(seq-1)))) {
return admission.AttemptRecord{}, admission.ErrCorruptRecord
}
prev = a
}
valid := false
switch c.State {
case admission.StateQueued:
valid = prev.State == admission.StateFailed && c.NotBefore.Equal(prev.Finished.Add(admission.RetryBackoff(c.Attempts)))
case admission.StateRefused, admission.StateWithheld, admission.StateDropped, admission.StateDryRun:
valid = prev.State == admission.StateFailed
default:
valid = prev.State == c.State && prev.Disposition == c.Disposition
}
if c.Attempts == 0 || !valid {
return admission.AttemptRecord{}, admission.ErrCorruptRecord
}
return prev, nil
}
// Historical outcomes can be acknowledged again, but only the current
// attempt may make a new transition.
func attemptAndCandidate(tx *bolt.Tx, id admission.ActionID) (admission.AttemptRecord, admission.Candidate, error) {
a, err := loadAttempt(tx, id)
if err != nil {
return a, admission.Candidate{}, err
}
c, err := loadCandidate(tx, a.Attempt.Candidate)
if err != nil {
return a, c, err
}
if _, err := currentAttempt(tx, c); err != nil {
return a, c, err
}
if a.Attempt.Seq > c.Attempts {
return a, c, admission.ErrCorruptRecord
}
return a, c, nil
}
// attemptStep runs one attempt transition and reports whether it changed
// anything. now is l.clock for a step that dispatches work and
// l.recordedClock for one that only records an outcome.
func (l *AdmissionLedger) attemptStep(op string, id admission.ActionID, now func() (time.Time, error), fn func(q *queueTx, a *admission.AttemptRecord, c *admission.Candidate) (bool, error)) (admission.Candidate, admission.AttemptRecord, bool, error) {
l.mu.Lock()
defer l.mu.Unlock()
at, err := now()
if err != nil {
return admission.Candidate{}, admission.AttemptRecord{}, false, err
}
var outC admission.Candidate
var outA admission.AttemptRecord
var changed bool
err = l.update(op, func(tx *bolt.Tx) error {
a, c, loadErr := attemptAndCandidate(tx, id)
if loadErr != nil {
return loadErr
}
q, loadErr := l.openQueue(tx, at)
if loadErr != nil {
return loadErr
}
from := c.State
stepChanged, stepErr := fn(q, &a, &c)
if stepErr != nil {
return stepErr
}
outC, outA = c, a
if !stepChanged {
return nil
}
if !admission.CanTransition(from, c.State) {
return admission.ErrTransitionConflict
}
c.Transitions++
outC, changed = c, true
if putErr := putAttempt(tx, a); putErr != nil {
return putErr
}
if putErr := putCandidate(tx, c); putErr != nil {
return putErr
}
return q.flush()
})
if err != nil {
return admission.Candidate{}, admission.AttemptRecord{}, false, err
}
if outA.State != admission.StateExecuting {
delete(l.executed, id)
}
return outC, outA, changed, nil
}
// Finish records the outcome of the candidate's current attempt. Applied and
// narrowed need a running attempt, and so does unknown: an attempt that never
// ran is known not to have applied, so it can only fail. A proven failure
// with attempts left returns the candidate to the queue after a backoff; an
// unknown outcome ends it without a retry. An ended candidate releases its
// queue position in the same transaction. A preview is recorded by Observe.
func (l *AdmissionLedger) Finish(id admission.ActionID, d admission.Disposition) (admission.Candidate, admission.AttemptRecord, error) {
return l.FinishOnLease(id, d, 0)
}
// FinishOnLease is Finish for an attempt whose expiry was set on the
// engine's lease clock, lead ahead of admission time when it ran. A
// verified block's episode ends when its kernel element expires in
// admission time, its expiry less the lead; the block keeps
// its expiry. A lead at or below zero ends the episode at the expiry
// itself, never later.
func (l *AdmissionLedger) FinishOnLease(id admission.ActionID, d admission.Disposition, lead time.Duration) (admission.Candidate, admission.AttemptRecord, error) {
return l.FinishEviction(id, d, admission.VictimRemoved, lead)
}
// FinishEviction is FinishOnLease with what the attempt did to the victim
// it claimed. Only a verified block's claim depends on it: a removal the
// applier proved evicts the victim and a victim it never removed returns
// to the pool. An unproven removal is an unknown outcome, never a verified
// one. A failure always returns the victim; an unknown outcome keeps it.
func (l *AdmissionLedger) FinishEviction(id admission.ActionID, d admission.Disposition, v admission.VictimOutcome, lead time.Duration) (admission.Candidate, admission.AttemptRecord, error) {
switch d {
case admission.DispositionApplied, admission.DispositionNarrowed, admission.DispositionFailed, admission.DispositionUnknown:
default:
return admission.Candidate{}, admission.AttemptRecord{}, refusal(admission.ReasonInvalid, "disposition is not an attempt outcome")
}
switch v {
case admission.VictimRemoved, admission.VictimSpared:
case admission.VictimUnproven:
if d == admission.DispositionApplied || d == admission.DispositionNarrowed {
return admission.Candidate{}, admission.AttemptRecord{}, refusal(admission.ReasonInvalid, "an unproven removal is an unknown outcome")
}
default:
return admission.Candidate{}, admission.AttemptRecord{}, refusal(admission.ReasonInvalid, "no victim outcome")
}
cand, att, _, err := l.attemptStep("finish", id, l.recordedClock, func(q *queueTx, a *admission.AttemptRecord, c *admission.Candidate) (bool, error) {
return q.finishAttempt(a, c, d, v, lead, l.elapsed)
})
return cand, att, err
}
// FinishExisting records that the current attempt found its effect already
// enforced: the attempt fails, keeping its charge, and with
// attempts left the pick ends refused as an existing effect, linked to the
// block, in the same transaction, so it is never retried against that
// block. On the last attempt the pick fails as with Finish. The same
// ending again is a no-op; a historical attempt or a different terminal
// link is a conflict. Expiry equality compares instants, not locations.
func (l *AdmissionLedger) FinishExisting(id admission.ActionID, link admission.EffectLink) (admission.Candidate, admission.AttemptRecord, error) {
if link == (admission.EffectLink{}) {
return admission.Candidate{}, admission.AttemptRecord{}, refusal(admission.ReasonInvalid, "an existing effect needs the block that enforces it")
}
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.recordedClock()
if err != nil {
return admission.Candidate{}, admission.AttemptRecord{}, err
}
var outC admission.Candidate
var outA admission.AttemptRecord
err = l.update("finish", func(tx *bolt.Tx) error {
a, c, loadErr := attemptAndCandidate(tx, id)
if loadErr != nil {
return loadErr
}
// A delayed result must not end a later retry of this candidate.
if a.Attempt.Seq != c.Attempts {
return admission.ErrTransitionConflict
}
if c.State.Terminal() && c.State != admission.StateFailed &&
(c.State != admission.StateRefused || c.Reason != admission.ReasonExistingEffect ||
c.Link.Effect != link.Effect || !c.Link.ExpiresAt.Equal(link.ExpiresAt)) {
return admission.ErrCandidateTerminal
}
q, loadErr := l.openQueue(tx, now)
if loadErr != nil {
return loadErr
}
from := c.State
changed, stepErr := q.finishAttempt(&a, &c, admission.DispositionFailed, admission.VictimSpared, 0, l.elapsed)
if stepErr != nil {
return stepErr
}
if changed {
if !admission.CanTransition(from, c.State) {
return admission.ErrTransitionConflict
}
c.Transitions++
// The ending below reads the finished attempt.
if putErr := putAttempt(tx, a); putErr != nil {
return putErr
}
}
if c.State == admission.StateQueued {
if stepErr = q.terminate(a.Attempt.Candidate, &c, admission.StateRefused, admission.ReasonExistingEffect, link); stepErr != nil {
return stepErr
}
c.Transitions++
changed = true
}
outC, outA = c, a
if !changed {
return nil
}
if putErr := putCandidate(tx, c); putErr != nil {
return putErr
}
return q.flush()
})
if err != nil {
return admission.Candidate{}, admission.AttemptRecord{}, err
}
delete(l.executed, id)
return outC, outA, nil
}
// Settle ends every attempt a previous process left outstanding, in one
// transaction. A reserved attempt has no intent, so it dispatched nothing:
// it fails, keeping its charge, and its candidate retries after the
// backoff while attempts remain. A running attempt may have applied its
// effect: it ends unknown and keeps its intent. Nothing is dispatched
// again. Called after this handle reserved anything, it would end that
// work too, so it refuses: the owner settles once, before its first
// reservation.
func (l *AdmissionLedger) Settle() (failed, unknown int, err error) {
l.mu.Lock()
defer l.mu.Unlock()
if l.reserved {
return 0, 0, admission.ErrTransitionConflict
}
err = l.update("settle", func(tx *bolt.Tx) error {
failed, unknown = 0, 0
var outstanding []admission.ActionID
if walkErr := tx.Bucket([]byte(admissionAttemptsBucket)).ForEach(func(_, v []byte) error {
a, decodeErr := admission.UnmarshalAttempt(v)
if decodeErr == nil && (a.State == admission.StateReserved || a.State == admission.StateExecuting) {
outstanding = append(outstanding, a.Attempt.ID)
}
return decodeErr
}); walkErr != nil {
return walkErr
}
// A new ledger has neither attempts nor a stored clock yet.
if len(outstanding) == 0 {
return nil
}
now, txErr := l.recordedClock()
if txErr != nil {
return txErr
}
q, txErr := l.openQueue(tx, now)
if txErr != nil {
return txErr
}
for _, id := range outstanding {
a, c, loadErr := attemptAndCandidate(tx, id)
if loadErr != nil {
return loadErr
}
d := admission.DispositionFailed
if a.State == admission.StateExecuting {
d = admission.DispositionUnknown
unknown++
} else {
failed++
}
if _, txErr = q.finishAttempt(&a, &c, d, admission.VictimSpared, 0, l.elapsed); txErr != nil {
return txErr
}
c.Transitions++
if txErr = putAttempt(tx, a); txErr != nil {
return txErr
}
if txErr = putCandidate(tx, c); txErr != nil {
return txErr
}
}
return q.flush()
})
if err != nil {
return 0, 0, err
}
clear(l.executed)
return failed, unknown, nil
}
// Observe reserves the next attempt of a queued candidate on lane as
// Reserve does, charge and history included, and ends it in the same
// transaction as an observe preview: the attempt never runs, applies
// nothing and ends no episode. One transaction leaves no reserved preview
// behind a crash. A candidate already reserved or running is a readback,
// as with Reserve: nothing is granted or changed.
func (l *AdmissionLedger) Observe(id admission.CandidateID, lane admission.Lane, expiresAt time.Time) (admission.Candidate, admission.AttemptRecord, bool, error) {
c, a, granted, _, err := l.ObserveVictim(id, lane, expiresAt)
return c, a, granted, err
}
// ObserveVictim is Observe that also reports the block whose slot the
// preview took: a strictly lower-tier block, or on a live ledger an
// equal-tier block rotation gives it; zero for a free slot or a readback.
// The preview claims nothing, so later previews may name the same block.
func (l *AdmissionLedger) ObserveVictim(id admission.CandidateID, lane admission.Lane, expiresAt time.Time) (admission.Candidate, admission.AttemptRecord, bool, admission.EvictionVictim, error) {
l.mu.Lock()
defer l.mu.Unlock()
now, err := l.clock()
if err != nil {
return admission.Candidate{}, admission.AttemptRecord{}, false, admission.EvictionVictim{}, err
}
var c admission.Candidate
var a admission.AttemptRecord
var granted bool
var victim admission.EvictionVictim
if err := l.update("observe", func(tx *bolt.Tx) error {
q, txErr := l.openQueue(tx, now)
if txErr != nil {
return txErr
}
if c, a, granted, txErr = l.reserveTx(q, id, lane, expiresAt, false, &victim); txErr != nil || !granted {
return txErr
}
from := c.State
if _, txErr = q.finishAttempt(&a, &c, admission.DispositionObserve, admission.VictimSpared, 0, l.elapsed); txErr != nil {
return txErr
}
if !admission.CanTransition(from, c.State) {
return admission.ErrTransitionConflict
}
c.Transitions++
if txErr = putAttempt(tx, a); txErr != nil {
return txErr
}
if txErr = putCandidate(tx, c); txErr != nil {
return txErr
}
return q.flush()
}); err != nil {
return admission.Candidate{}, admission.AttemptRecord{}, false, admission.EvictionVictim{}, err
}
return c, a, granted, victim, nil
}
// finishAttempt records outcome d of the candidate's current attempt a and
// reports whether it changed anything. The caller counts the transition
// and stores both records.
func (q *queueTx) finishAttempt(a *admission.AttemptRecord, c *admission.Candidate, d admission.Disposition, v admission.VictimOutcome, lead, elapsed time.Duration) (bool, error) {
var state admission.State
switch d {
case admission.DispositionApplied, admission.DispositionNarrowed:
state = admission.StateVerified
case admission.DispositionFailed:
state = admission.StateFailed
case admission.DispositionUnknown:
state = admission.StateUnknown
case admission.DispositionObserve:
state = admission.StateObserved
}
// An unknown readback still needs its recovery intent until retirement.
if d == admission.DispositionUnknown && unsettled(*a) {
if _, found, err := loadIntent(q.tx, a.Attempt.ID); err != nil || !found {
return false, admission.ErrCorruptRecord
}
}
if a.State.Terminal() {
if a.Disposition == d {
return false, nil
}
return false, admission.ErrTransitionConflict
}
e, err := loadQueueEntry(q.tx, a.Attempt.Candidate)
if err != nil {
return false, err
}
// The candidate moves in step with its current attempt, so the
// lifecycle table decides which outcomes a reserved attempt allows.
from := c.State
if state != admission.StateUnknown {
if err = q.endSweptClaim(*a, state == admission.StateVerified, v); err != nil {
return false, err
}
}
// An attempt that never ran returns the slot of its execution row. One
// that ran keeps its intent only while its outcome is unknown.
var unused uint64
if a.State == admission.StateReserved {
unused = 1
} else if d != admission.DispositionUnknown {
if err = q.deleteIntent(a.Attempt.ID); err != nil {
return false, err
}
}
a.State, a.Disposition, a.Finished = state, d, q.now
// A verified block keeps its slot until its expiry, and so does an
// unknown one, which may hold it; any other outcome gives it back.
switch state {
case admission.StateVerified:
err = q.markApplied(*a, v, elapsed)
case admission.StateUnknown:
default:
err = q.releaseSlot(*a)
}
if err != nil {
return false, err
}
if err = q.outcomes.Add(admission.AttemptOutcome(d, e.Tier)); err != nil {
return false, err
}
switch {
case d == admission.DispositionFailed && c.Attempts < admission.MaxAttempts:
c.State, c.NotBefore = admission.StateQueued, q.now.Add(admission.RetryBackoff(c.Attempts))
default:
c.State, c.Disposition = state, d
}
// A row cannot record an outcome its attempt never reached, so the
// table is asked before the row is written.
if !admission.CanTransition(from, c.State) {
return false, admission.ErrTransitionConflict
}
// The row goes first: the ending below writes retirement keys
// only when no row of the candidate is pending.
if err = q.writeAuditRow(*c, *a, e.Tier); err != nil {
return false, err
}
// An unknown outcome holds the slot of the row that will retire it.
var retirement uint64
if d == admission.DispositionUnknown {
retirement = 1
}
if err = q.adjustAuditSlots(retirement, unused); err != nil {
return false, err
}
if c.State == admission.StateQueued {
q.noteDeadlines(*c, e)
return true, nil
}
if err = q.release(a.Attempt.Candidate, e, 0, 0); err != nil {
return false, err
}
// Pin the ending before allocating its notice: release removed
// the outstanding hold, and the notice cannot spend those bytes.
if err = q.ended(a.Attempt.Candidate, *c); err != nil {
return false, err
}
if c.State == admission.StateVerified {
// The block ran at its Execute reading. Without one, finishing
// uses the last committed reading, even if the ceiling is now
// unreadable and new work has stopped.
clock := admission.EpisodeClock{Now: q.now, Elapsed: elapsed}
if ran, ok := q.executed[a.Attempt.ID]; ok {
clock = ran
}
if err = q.closeEpisode(*c, lead, clock); err != nil {
return false, err
}
err = q.raise(admission.NoticeKey{Kind: admission.NoticeAppliedSummary}, a.Attempt.Candidate, c.Transitions+1)
if err == nil {
err = q.partial(d, *c, a.Attempt.Candidate, c.Transitions+1)
}
} else {
// A preview is intentional non-enforcement: it raises no gap.
err = q.gap(admission.GapOutcome, 0, d, e, a.Attempt.Candidate, *c, c.Transitions+1)
}
return true, err
}
package store
import (
"encoding/json"
"time"
"github.com/pidginhost/csm/internal/alert"
bolt "go.etcd.io/bbolt"
)
// SeverityBucket holds aggregated counts by severity.
type SeverityBucket struct {
Critical int `json:"critical"`
High int `json:"high"`
Warning int `json:"warning"`
Total int `json:"total"`
}
// HourBucket is a SeverityBucket for the hour that begins at Start.
type HourBucket struct {
Start time.Time `json:"start"`
SeverityBucket
}
// DayBucket is a SeverityBucket keyed by date.
type DayBucket struct {
Date string `json:"date"`
SeverityBucket
}
// AggregateByHour returns 24 hourly buckets (oldest first) for the last 24 hours.
// It seeks directly to the start key in bbolt, scanning only the relevant range.
func (db *DB) AggregateByHour() []HourBucket {
now := time.Now()
currentHour := now.Truncate(time.Hour)
cutoff := currentHour.Add(-23 * time.Hour)
// Map: hours-ago (0=current, 23=oldest) → counts
counts := make(map[int]*SeverityBucket, 24)
for i := 0; i < 24; i++ {
counts[i] = &SeverityBucket{}
}
seekPrefix := timeKeyLowerBound(cutoff)
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("history"))
if b == nil {
return nil
}
c := b.Cursor()
// Seek to the earliest key that could be in our 24h window.
for k, v := c.Seek([]byte(seekPrefix)); k != nil; k, v = c.Next() {
var f alert.Finding
if err := json.Unmarshal(v, &f); err != nil {
continue
}
if f.Timestamp.Before(cutoff) {
continue
}
if f.Timestamp.After(now) {
continue
}
fHour := f.Timestamp.Truncate(time.Hour)
hoursAgo := int(currentHour.Sub(fHour).Hours())
if hoursAgo < 0 || hoursAgo >= 24 {
continue
}
bucket := counts[hoursAgo]
bucket.Total++
switch f.Severity {
case alert.Critical:
bucket.Critical++
case alert.High:
bucket.High++
case alert.Warning:
bucket.Warning++
}
}
return nil
})
// Build result oldest→newest (23h ago → 0h ago)
result := make([]HourBucket, 24)
for i := 0; i < 24; i++ {
hoursAgo := 23 - i
t := currentHour.Add(-time.Duration(hoursAgo) * time.Hour)
result[i] = HourBucket{
Start: t,
SeverityBucket: *counts[hoursAgo],
}
}
return result
}
// ReadHistorySince returns all findings since the given time, using bbolt cursor
// seeking for efficiency. Results are newest-first.
func (db *DB) ReadHistorySince(since time.Time) []alert.Finding {
seekPrefix := timeKeyLowerBound(since)
var results []alert.Finding
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("history"))
if b == nil {
return nil
}
c := b.Cursor()
for k, v := c.Seek([]byte(seekPrefix)); k != nil; k, v = c.Next() {
var f alert.Finding
if err := json.Unmarshal(v, &f); err != nil {
continue
}
results = append(results, f)
}
return nil
})
// Reverse to newest-first
for i, j := 0, len(results)-1; i < j; i, j = i+1, j-1 {
results[i], results[j] = results[j], results[i]
}
return results
}
// SearchHistorySince returns up to limit findings since the given time,
// newest-first. The matcher runs while the bbolt cursor walks backward so
// callers that only need a bounded result set do not materialize the whole
// time window first.
func (db *DB) SearchHistorySince(since time.Time, limit int, match func(alert.Finding) bool) []alert.Finding {
if limit <= 0 {
return nil
}
cutoffKey := timeKeyLowerBound(since)
var results []alert.Finding
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("history"))
if b == nil {
return nil
}
c := b.Cursor()
for k, v := c.Last(); k != nil && len(results) < limit; k, v = c.Prev() {
if string(k) < cutoffKey {
break
}
var f alert.Finding
if err := json.Unmarshal(v, &f); err != nil {
continue
}
if match != nil && !match(f) {
continue
}
results = append(results, f)
}
return nil
})
return results
}
// AggregateByDay returns 30 daily buckets (oldest first) for the last 30 days.
// Reads from the pre-aggregated stats:daily bucket so the trend chart is
// not affected by history pruning.
func (db *DB) AggregateByDay() []DayBucket {
return db.AggregateByDayN(30)
}
// AggregateByDayN returns `days` daily buckets (oldest first) ending today.
// Days outside [1, dailyRetentionDays] are clamped to that range. Days with
// no recorded findings are returned as zero-value buckets.
func (db *DB) AggregateByDayN(days int) []DayBucket {
if days < 1 {
days = 1
}
if days > dailyRetentionDays {
days = dailyRetentionDays
}
// Hard cap so a future bump to dailyRetentionDays cannot accidentally
// drive a huge slice allocation here. Daily aggregation over more than
// a decade is not a real workload for this surface.
const aggregateBucketHardCap = 4096
if days > aggregateBucketHardCap {
days = aggregateBucketHardCap
}
now := time.Now()
today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local)
cutoff := today.AddDate(0, 0, -(days - 1))
buckets := make([]DayBucket, days)
for i := 0; i < days; i++ {
d := cutoff.AddDate(0, 0, i)
buckets[i] = DayBucket{Date: d.Format("2006-01-02")}
}
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucketStatsDaily))
if b == nil {
return nil
}
for i := range buckets {
v := b.Get([]byte(buckets[i].Date))
if v == nil {
continue
}
var sb SeverityBucket
if err := json.Unmarshal(v, &sb); err != nil {
continue
}
buckets[i].SeverityBucket = sb
}
return nil
})
return buckets
}
package store
import (
"errors"
"fmt"
"os"
"slices"
"time"
bolt "go.etcd.io/bbolt"
)
// Learning ledger buckets for antispam learning. Only the learning
// engine writes them, through LearnUpdate.
const (
LearnMetaBucket = "antispam:learn:meta"
LearnClustersBucket = "antispam:learn:clusters"
LearnAliasesBucket = "antispam:learn:aliases"
LearnTargetsBucket = "antispam:learn:targets"
LearnIntentsBucket = "antispam:learn:intents"
LearnOpsBucket = "antispam:learn:ops"
LearnUsageBucket = "antispam:learn:usage"
LearnCursorBucket = "antispam:learn:cursor"
)
// LearnBuckets lists every learning ledger bucket.
var LearnBuckets = []string{
LearnMetaBucket, LearnClustersBucket, LearnAliasesBucket, LearnTargetsBucket,
LearnIntentsBucket, LearnOpsBucket, LearnUsageBucket, LearnCursorBucket,
}
// Spam folder watcher buckets: folder identities, baselines,
// scheduler cursors and seen message identities. They share LearnTx and
// leave exports with the ledger.
const (
LearnWatchFoldersBucket = "antispam:learnwatch:folders"
LearnWatchSeenBucket = "antispam:learnwatch:seen"
)
// WatchBuckets lists every watcher bucket.
var WatchBuckets = []string{LearnWatchFoldersBucket, LearnWatchSeenBucket}
// LearnTx is a bbolt transaction limited to the learning and watcher buckets.
type LearnTx struct {
tx *bolt.Tx
}
func (t LearnTx) bucket(name string) (*bolt.Bucket, error) {
if !slices.Contains(LearnBuckets, name) && !slices.Contains(WatchBuckets, name) {
return nil, fmt.Errorf("learning: bucket %q is not a learning bucket", name)
}
b := t.tx.Bucket([]byte(name))
if b == nil {
return nil, fmt.Errorf("learning: bucket %q missing", name)
}
return b, nil
}
// Get returns a copy of the value at key, nil when absent.
func (t LearnTx) Get(bucket string, key []byte) ([]byte, error) {
b, err := t.bucket(bucket)
if err != nil {
return nil, err
}
if v := b.Get(key); v != nil {
return append([]byte{}, v...), nil
}
return nil, nil
}
// Put stores value at key.
func (t LearnTx) Put(bucket string, key, value []byte) error {
b, err := t.bucket(bucket)
if err != nil {
return err
}
if len(key) == 0 {
return errors.New("learning: empty key")
}
return b.Put(append([]byte(nil), key...), append([]byte(nil), value...))
}
// Delete removes key; a missing key is not an error.
func (t LearnTx) Delete(bucket string, key []byte) error {
b, err := t.bucket(bucket)
if err != nil {
return err
}
return b.Delete(key)
}
// ForEach visits every key in order. fn must not keep k or v.
func (t LearnTx) ForEach(bucket string, fn func(k, v []byte) error) error {
b, err := t.bucket(bucket)
if err != nil {
return err
}
return b.ForEach(fn)
}
// Scan visits keys strictly after after (from the first key when after is
// empty) in order until fn returns false. fn must not keep k or v.
func (t LearnTx) Scan(bucket string, after []byte, fn func(k, v []byte) (bool, error)) error {
b, err := t.bucket(bucket)
if err != nil {
return err
}
c := b.Cursor()
k, v := c.First()
if len(after) > 0 {
k, v = c.Seek(after)
if k != nil && string(k) == string(after) {
k, v = c.Next()
}
}
for ; k != nil; k, v = c.Next() {
more, err := fn(k, v)
if err != nil || !more {
return err
}
}
return nil
}
// NextSequence returns the bucket's next sequence number. Numbers are never
// reused, even after their records are deleted.
func (t LearnTx) NextSequence(bucket string) (uint64, error) {
b, err := t.bucket(bucket)
if err != nil {
return 0, err
}
return b.NextSequence()
}
// LearnUpdate runs fn in one read-write transaction; an error rolls it back.
func (db *DB) LearnUpdate(fn func(LearnTx) error) error {
return db.bolt.Update(func(tx *bolt.Tx) error { return fn(LearnTx{tx: tx}) })
}
// LearnView runs fn in one read-only transaction.
func (db *DB) LearnView(fn func(LearnTx) error) error {
return db.bolt.View(func(tx *bolt.Tx) error { return fn(LearnTx{tx: tx}) })
}
// clearLearning drops every learning and watcher record from a private
// snapshot.
func (db *DB) clearLearning() error {
return db.bolt.Update(func(tx *bolt.Tx) error {
for _, name := range slices.Concat(LearnBuckets, WatchBuckets) {
if tx.Bucket([]byte(name)) == nil {
continue
}
if err := tx.DeleteBucket([]byte(name)); err != nil {
return err
}
}
return nil
})
}
// ResetLearning empties every learning and watcher bucket of the live
// database in one transaction, for a fresh learning namespace.
func (db *DB) ResetLearning() error {
return db.bolt.Update(func(tx *bolt.Tx) error {
for _, name := range slices.Concat(LearnBuckets, WatchBuckets) {
if tx.Bucket([]byte(name)) != nil {
if err := tx.DeleteBucket([]byte(name)); err != nil {
return err
}
}
if _, err := tx.CreateBucket([]byte(name)); err != nil {
return err
}
}
return nil
})
}
// LearningStatePresent reports whether the database file at path holds any
// learning ledger record, opening it read-only so the file is unchanged.
// A missing file holds none.
func LearningStatePresent(path string) (bool, error) {
if _, err := os.Stat(path); os.IsNotExist(err) {
return false, nil
}
b, err := bolt.Open(path, 0o600, &bolt.Options{ReadOnly: true, Timeout: time.Second})
if err != nil {
return false, err
}
defer func() { _ = b.Close() }()
present := false
err = b.View(func(tx *bolt.Tx) error {
for _, name := range LearnBuckets {
if bucket := tx.Bucket([]byte(name)); bucket != nil {
if k, _ := bucket.Cursor().First(); k != nil {
present = true
return nil
}
}
}
return nil
})
return present, err
}
package store
import bolt "go.etcd.io/bbolt"
// ResetLearnWatch cancels unsent reports and starts new folder baselines
// after explicit disable. The learning ledger and contribution guards stay.
func (db *DB) ResetLearnWatch() error {
return db.bolt.Update(func(tx *bolt.Tx) error {
for _, name := range WatchBuckets {
if err := tx.DeleteBucket([]byte(name)); err != nil {
return err
}
if _, err := tx.CreateBucket([]byte(name)); err != nil {
return err
}
}
return nil
})
}
package store
import (
"archive/tar"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"syscall"
"time"
"github.com/klauspost/compress/zstd"
bolt "go.etcd.io/bbolt"
)
// writeFileNoFollow is os.WriteFile that refuses to write through a
// symlink standing at path.
//
// #nosec G304 G703 -- path is the export destination the operator named.
func writeFileNoFollow(path string, data []byte, perm os.FileMode) error {
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|syscall.O_NOFOLLOW, perm)
if err != nil {
return err
}
if _, err := f.Write(data); err != nil {
_ = f.Close()
return err
}
return f.Close()
}
// ArchiveSchemaVersion is the on-wire schema for backup archives. Bump
// when the manifest layout or contents shape changes incompatibly. Old
// CSM binaries refuse archives newer than the version they understand.
const ArchiveSchemaVersion = 1
// Standard entry names inside the tar.
const (
manifestEntry = "manifest.json"
bboltSnapshotEntry = "bbolt.snapshot"
stateEntryPrefix = "state/"
rulesEntryPrefix = "rules/"
stateLockFileName = "csm.lock"
)
var transientStatePaths = []string{
"firewall/confirm_pending",
"firewall/rollback.nft",
"firewall/rollback.nft.state.json",
"firewall/rollback.nft.config.json",
"firewall/rollback.sh",
}
// Sentinel errors so callers can branch on the failure mode instead of
// matching strings.
var (
ErrSchemaVersionTooNew = errors.New("archive schema version is newer than this binary supports")
ErrPlatformMismatch = errors.New("archive source platform does not match current host")
ErrManifestMissing = errors.New("archive does not contain manifest.json")
ErrCorruptArchive = errors.New("archive is corrupt or not a CSM backup")
ErrLedgerBehind = errors.New("the archive's admission ledger is behind this host's live ledger")
)
// Manifest is the JSON header at the top of every archive.
type Manifest struct {
SchemaVersion int `json:"schema_version"`
CSMVersion string `json:"csm_version"`
SourceHostname string `json:"source_hostname"`
SourcePlatform map[string]string `json:"source_platform"`
ExportTS time.Time `json:"export_ts"`
Contents []string `json:"contents"`
BboltBuckets []string `json:"bbolt_buckets,omitempty"`
BboltSHA256 string `json:"bbolt_sha256,omitempty"`
}
// ExportOptions configures Export.
type ExportOptions struct {
StatePath string // /var/lib/csm/state, source for state JSON files
RulesPath string // /opt/csm/rules, source for signature cache (empty -> skip)
DstPath string // .csmbak file to create
Manifest Manifest // caller fills CSMVersion/SourceHostname/SourcePlatform; rest filled here
}
// ExportResult summarises a successful export.
type ExportResult struct {
Path string
Bytes int64
ArchiveSHA256 string
BboltSHA256 string
}
// ImportOptions configures Import.
type ImportOptions struct {
SrcPath string
StatePath string
RulesPath string
Only string // "all" | "baseline" | "firewall"
ForcePlatformMismatch bool
Force bool // restore a ledger behind this host's live one
CurrentPlatform map[string]string // for the mismatch check
}
// ImportResult summarises a successful import.
type ImportResult struct {
Manifest Manifest
BucketsRestored []string
StateFiles int
RulesFiles int
}
// Export writes a tar+zstd archive containing a bbolt snapshot, the
// state directory, and (optionally) the signature-rules directory. The
// daemon is the single source of truth for paths; the caller fills the
// manifest with hostname/version/platform.
func (db *DB) Export(opts ExportOptions) (*ExportResult, error) {
if opts.DstPath == "" {
return nil, errors.New("DstPath is empty")
}
if opts.StatePath == "" {
return nil, errors.New("StatePath is empty")
}
man := opts.Manifest
if man.SchemaVersion == 0 {
man.SchemaVersion = ArchiveSchemaVersion
}
if man.ExportTS.IsZero() {
man.ExportTS = time.Now().UTC()
}
man.Contents = []string{"bbolt", "state"}
if opts.RulesPath != "" {
man.Contents = append(man.Contents, "rules")
}
// Snapshot bbolt to a temp file in the same directory so we can hash it
// and stream it into the tar without holding a long bolt transaction.
snapDir := filepath.Dir(opts.DstPath)
snap, err := os.CreateTemp(snapDir, "csm-export-bbolt-*.snap")
if err != nil {
return nil, fmt.Errorf("creating bbolt snapshot temp: %w", err)
}
snapPath := snap.Name()
defer func() { _ = os.Remove(snapPath) }()
err = db.bolt.View(func(tx *bolt.Tx) error {
_, werr := tx.WriteTo(snap)
return werr
})
if err != nil {
_ = snap.Close()
return nil, fmt.Errorf("bbolt snapshot: %w", err)
}
if err = snap.Close(); err != nil {
return nil, fmt.Errorf("closing bbolt snapshot: %w", err)
}
if err = DisarmBrowserSessionsSnapshot(snapPath); err != nil {
return nil, err
}
if err = DisarmFirewallRollbackSnapshot(snapPath); err != nil {
return nil, err
}
// Describe the sanitized snapshot, not the live database: a full import
// reports these buckets as restored.
man.BboltBuckets, err = listSnapshotBuckets(snapPath)
if err != nil {
return nil, err
}
man.BboltSHA256, err = sha256File(snapPath)
if err != nil {
return nil, fmt.Errorf("hashing bbolt snapshot: %w", err)
}
// Build the archive on disk; hash it as we write. Close is called
// explicitly below so any close error after fsync is surfaced --
// silently dropping it would mean the operator gets "export
// succeeded" for a file that may not be fully persisted.
// O_NOFOLLOW: a local account that gets to create the destination name
// first must not have the daemon write the archive through a symlink
// into a file it can read.
out, err := os.OpenFile(opts.DstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|syscall.O_NOFOLLOW, 0600)
if err != nil {
return nil, fmt.Errorf("creating archive: %w", err)
}
// closed gates the deferred Close so the explicit Close below
// can return its error without a double-close. success gates the
// cleanup of the partial archive + companion file: any
// error-return path between OpenFile and the final return removes
// the half-written files so the operator does not mistake them
// for a usable backup.
closed := false
success := false
defer func() {
if !closed {
_ = out.Close()
}
if !success {
_ = os.Remove(opts.DstPath)
_ = os.Remove(opts.DstPath + ".sha256")
}
}()
archHash := sha256.New()
mw := io.MultiWriter(out, archHash)
zw, err := zstd.NewWriter(mw, zstd.WithEncoderLevel(zstd.SpeedDefault))
if err != nil {
return nil, fmt.Errorf("zstd writer: %w", err)
}
tw := tar.NewWriter(zw)
// 1. manifest first so a streaming reader sees schema info before payload.
manBytes, err := json.MarshalIndent(man, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal manifest: %w", err)
}
if err = writeTarFile(tw, manifestEntry, manBytes, man.ExportTS); err != nil {
return nil, err
}
// 2. bbolt snapshot (already on disk).
if err = streamFileToTar(tw, bboltSnapshotEntry, snapPath, man.ExportTS); err != nil {
return nil, err
}
// 3. state files (skip runtime-owned files captured separately or not at all).
stateSkip := append([]string{"csm.db", stateLockFileName, "exports"}, transientStatePaths...)
if err = walkDirIntoTar(tw, opts.StatePath, stateEntryPrefix, stateSkip, man.ExportTS); err != nil {
return nil, err
}
// 4. rules files (optional).
if opts.RulesPath != "" {
if err = walkDirIntoTar(tw, opts.RulesPath, rulesEntryPrefix, nil, man.ExportTS); err != nil {
return nil, err
}
}
if err = tw.Close(); err != nil {
return nil, fmt.Errorf("closing tar: %w", err)
}
if err = zw.Close(); err != nil {
return nil, fmt.Errorf("closing zstd: %w", err)
}
if err = out.Sync(); err != nil {
return nil, fmt.Errorf("fsync archive: %w", err)
}
if err = out.Close(); err != nil {
return nil, fmt.Errorf("closing archive: %w", err)
}
closed = true
info, err := os.Stat(opts.DstPath)
if err != nil {
return nil, fmt.Errorf("stat archive: %w", err)
}
archiveSHA := hex.EncodeToString(archHash.Sum(nil))
// Write companion .sha256 file alongside for operator verification.
// O_NOFOLLOW for the same reason as the archive itself: the companion
// can truncate whatever a planted symlink points at.
companion := opts.DstPath + ".sha256"
companionLine := fmt.Sprintf("%s %s\n", archiveSHA, filepath.Base(opts.DstPath))
if err = writeFileNoFollow(companion, []byte(companionLine), 0600); err != nil {
return nil, fmt.Errorf("writing companion sha256: %w", err)
}
success = true
return &ExportResult{
Path: opts.DstPath,
Bytes: info.Size(),
ArchiveSHA256: archiveSHA,
BboltSHA256: man.BboltSHA256,
}, nil
}
// Import unpacks an archive into the target state and rules paths. Live
// daemons must be stopped first; callers enforce that before invoking, and
// hold the state lock throughout: the ledger comparison opens the target
// database read-only and nothing may write it meanwhile.
func Import(opts ImportOptions) (*ImportResult, error) {
if opts.SrcPath == "" {
return nil, errors.New("SrcPath is empty")
}
if opts.StatePath == "" {
return nil, errors.New("StatePath is empty")
}
only := opts.Only
if only == "" {
only = "all"
}
switch only {
case "all", "baseline", "firewall":
default:
return nil, fmt.Errorf("invalid Only value %q (want all|baseline|firewall)", only)
}
in, err := os.Open(opts.SrcPath)
if err != nil {
return nil, fmt.Errorf("opening archive: %w", err)
}
defer in.Close()
zr, err := zstd.NewReader(in)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrCorruptArchive, err)
}
defer zr.Close()
tr := tar.NewReader(zr)
// Manifest must come first.
hdr, err := tr.Next()
if err != nil {
return nil, fmt.Errorf("%w: reading first entry: %v", ErrCorruptArchive, err)
}
if hdr.Name != manifestEntry {
return nil, fmt.Errorf("%w: first entry is %q, want %q", ErrManifestMissing, hdr.Name, manifestEntry)
}
manBytes, err := io.ReadAll(tr)
if err != nil {
return nil, fmt.Errorf("%w: reading manifest: %v", ErrCorruptArchive, err)
}
var man Manifest
if err = json.Unmarshal(manBytes, &man); err != nil {
return nil, fmt.Errorf("%w: parsing manifest: %v", ErrCorruptArchive, err)
}
if man.SchemaVersion > ArchiveSchemaVersion {
return nil, fmt.Errorf("%w: archive=%d binary=%d", ErrSchemaVersionTooNew, man.SchemaVersion, ArchiveSchemaVersion)
}
if !opts.ForcePlatformMismatch && !platformMatches(man.SourcePlatform, opts.CurrentPlatform) {
return nil, fmt.Errorf("%w: archive=%v current=%v (use --force-platform-mismatch to override)", ErrPlatformMismatch, man.SourcePlatform, opts.CurrentPlatform)
}
// Stage every payload into a temp dir; commit only after a complete
// successful read so a half-imported state is impossible.
stage, err := os.MkdirTemp(filepath.Dir(opts.StatePath), "csm-import-stage-*")
if err != nil {
return nil, fmt.Errorf("creating staging dir: %w", err)
}
defer func() { _ = os.RemoveAll(stage) }()
stagedBbolt := ""
stagedState := []string{}
stagedRules := []string{}
for {
nextHdr, nextErr := tr.Next()
if nextErr == io.EOF {
break
}
if nextErr != nil {
return nil, fmt.Errorf("%w: reading entry: %v", ErrCorruptArchive, nextErr)
}
if nextHdr.Typeflag != tar.TypeReg {
continue
}
clean := filepath.Clean(nextHdr.Name)
if strings.HasPrefix(clean, "..") || filepath.IsAbs(clean) {
return nil, fmt.Errorf("%w: unsafe entry name %q", ErrCorruptArchive, nextHdr.Name)
}
dst := filepath.Join(stage, clean)
// Defence in depth against zip-slip: confirm the joined path still
// resolves inside the staging dir even after symlinks / unusual
// path elements in the archive header.
rel, relErr := filepath.Rel(stage, dst)
if relErr != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
return nil, fmt.Errorf("%w: unsafe entry name %q", ErrCorruptArchive, nextHdr.Name)
}
if isTransientStateArchiveEntry(clean) {
continue
}
// Only the snapshot may replace csm.db. Treating it as a state file
// would let a baseline import replace the ledger without checking it.
if clean == stateEntryPrefix+"csm.db" {
return nil, fmt.Errorf("%w: database must be stored as %s", ErrCorruptArchive, bboltSnapshotEntry)
}
if mkErr := os.MkdirAll(filepath.Dir(dst), 0700); mkErr != nil {
return nil, fmt.Errorf("staging dir: %w", mkErr)
}
// #nosec G304 -- dst is filepath.Join(stage, clean) where stage is a freshly created MkdirTemp under StatePath's parent and clean has been validated three lines above (no ".." prefix, not absolute). Cannot escape the staging dir.
f, openErr := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if openErr != nil {
return nil, fmt.Errorf("staging file: %w", openErr)
}
// nextHdr.Size bound caps bytes copied so a hostile archive can't
// drive the stage dir to fill the filesystem.
if _, copyErr := io.CopyN(f, tr, nextHdr.Size); copyErr != nil {
_ = f.Close()
return nil, fmt.Errorf("staging copy %s: %w", clean, copyErr)
}
if closeErr := f.Close(); closeErr != nil {
return nil, fmt.Errorf("closing staged file: %w", closeErr)
}
switch {
case clean == bboltSnapshotEntry:
stagedBbolt = dst
case strings.HasPrefix(clean, stateEntryPrefix):
stagedState = append(stagedState, clean)
case strings.HasPrefix(clean, rulesEntryPrefix):
stagedRules = append(stagedRules, clean)
}
}
if (only == "all" || only == "firewall") && stagedBbolt == "" {
return nil, fmt.Errorf("%w: %s import needs bbolt snapshot in archive", ErrCorruptArchive, only)
}
baselineFirewall := only == "baseline" && slices.Contains(stagedState, stateEntryPrefix+"firewall/state.json")
// Verify the staged bbolt snapshot against the hash the manifest recorded
// at export time before any bbolt-consuming import applies payloads.
// zstd's frame CRC catches transport corruption, but a snapshot that was
// altered before compression would otherwise be promoted over csm.db
// unchecked. Baselines carrying firewall state also use this snapshot
// for the ledger guard. Empty hash means a pre-hash archive; skip rather
// than reject.
if (only == "all" || only == "firewall" || baselineFirewall) && stagedBbolt != "" && man.BboltSHA256 != "" {
gotHash, hashErr := sha256File(stagedBbolt)
if hashErr != nil {
return nil, fmt.Errorf("hashing staged bbolt snapshot: %w", hashErr)
}
if gotHash != man.BboltSHA256 {
return nil, fmt.Errorf("%w: bbolt snapshot hash mismatch (archive manifest %s, staged %s)", ErrCorruptArchive, man.BboltSHA256, gotHash)
}
}
if (only == "all" || only == "firewall") && stagedBbolt != "" {
if err := DisarmBrowserSessionsSnapshot(stagedBbolt); err != nil {
return nil, err
}
if err := DisarmFirewallRollbackSnapshot(stagedBbolt); err != nil {
return nil, err
}
}
// The firewall part merges only fw:* buckets; the others replace the
// ledger or put the firewall state saved beside it next to this one.
if !opts.Force && (only == "all" || baselineFirewall) {
if err := CheckLedgerRestore(stagedBbolt, filepath.Join(opts.StatePath, "csm.db")); err != nil {
return nil, err
}
}
res := &ImportResult{Manifest: man}
if only == "all" {
// Older archives can list session buckets removed during sanitization.
// Read the staged snapshot before applying any files to the destination.
restored, listErr := listSnapshotBuckets(stagedBbolt)
if listErr != nil {
return nil, listErr
}
res.BucketsRestored = restored
}
// Apply state files (always, unless caller filtered everything out).
if only == "all" || only == "baseline" {
if err := os.MkdirAll(opts.StatePath, 0700); err != nil {
return nil, fmt.Errorf("creating state path: %w", err)
}
for _, rel := range stagedState {
src := filepath.Join(stage, rel)
dst := filepath.Join(opts.StatePath, strings.TrimPrefix(rel, stateEntryPrefix))
if err := atomicReplace(src, dst); err != nil {
return nil, fmt.Errorf("restoring %s: %w", rel, err)
}
res.StateFiles++
}
}
// Apply rules files (only=all only; baseline and firewall skip rules).
if only == "all" && opts.RulesPath != "" {
if err := os.MkdirAll(opts.RulesPath, 0700); err != nil {
return nil, fmt.Errorf("creating rules path: %w", err)
}
for _, rel := range stagedRules {
src := filepath.Join(stage, rel)
dst := filepath.Join(opts.RulesPath, strings.TrimPrefix(rel, rulesEntryPrefix))
if err := atomicReplace(src, dst); err != nil {
return nil, fmt.Errorf("restoring %s: %w", rel, err)
}
res.RulesFiles++
}
}
// Apply bbolt:
// only=all -> wholesale rename the snapshot over csm.db
// only=firewall -> open snapshot read-only and copy fw:* buckets
// into the target bbolt
// only=baseline -> skip bbolt entirely
switch only {
case "all":
target := filepath.Join(opts.StatePath, "csm.db")
if err := atomicReplace(stagedBbolt, target); err != nil {
return nil, fmt.Errorf("restoring csm.db: %w", err)
}
case "firewall":
restored, err := mergeFirewallBucketsFromSnapshot(stagedBbolt, opts.StatePath)
if err != nil {
return nil, fmt.Errorf("merging firewall buckets: %w", err)
}
res.BucketsRestored = restored
case "baseline":
// no bbolt work
}
return res, nil
}
func isTransientStateArchiveEntry(name string) bool {
rel, ok := strings.CutPrefix(name, stateEntryPrefix)
if !ok {
return false
}
if rel == stateLockFileName || rel == "exports" || strings.HasPrefix(rel, "exports/") {
return true
}
for _, transient := range transientStatePaths {
if rel == transient {
return true
}
}
return false
}
// platformMatches compares the archive's stored platform map against the
// current host. Empty maps match anything (used when a caller hasn't
// supplied detection results, e.g., in some test paths).
func platformMatches(a, b map[string]string) bool {
if len(a) == 0 || len(b) == 0 {
return true
}
keys := []string{"os", "panel", "webserver"}
for _, k := range keys {
if a[k] != b[k] {
return false
}
}
return true
}
// listSnapshotBuckets returns the bucket names actually present in a
// snapshot file (not the static bucketNames slice; migrations may have
// removed some, and export sanitization removes others).
func listSnapshotBuckets(path string) ([]string, error) {
if err := verifySnapshot(path); err != nil {
return nil, err
}
snapshot, err := bolt.Open(path, 0600, &bolt.Options{Timeout: time.Second, ReadOnly: true})
if err != nil {
return nil, fmt.Errorf("opening bbolt snapshot: %w", err)
}
out := []string{}
err = snapshot.View(func(tx *bolt.Tx) error {
return tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
out = append(out, string(name))
return nil
})
})
if closeErr := snapshot.Close(); err == nil && closeErr != nil {
err = fmt.Errorf("closing bbolt snapshot: %w", closeErr)
}
if err != nil {
return nil, fmt.Errorf("listing bbolt snapshot buckets: %w", err)
}
sort.Strings(out)
return out, nil
}
func writeTarFile(tw *tar.Writer, name string, data []byte, modTime time.Time) error {
hdr := &tar.Header{
Name: name,
Mode: 0600,
Size: int64(len(data)),
ModTime: modTime,
}
if err := tw.WriteHeader(hdr); err != nil {
return fmt.Errorf("tar header %s: %w", name, err)
}
if _, err := tw.Write(data); err != nil {
return fmt.Errorf("tar write %s: %w", name, err)
}
return nil
}
// streamFileToTar copies srcPath into the tar under name. There is a
// theoretical Stat-then-Copy race when the source file is replaced
// between f.Stat() and io.Copy() -- the tar writer would then see a
// mismatched byte count. In practice the only files this function
// reads on a live daemon are state JSON written via atomic-replace
// (so a mid-write read sees either the old or the new full content)
// and the bbolt snapshot (already a frozen copy on disk). The risk
// is bounded enough to live with for v1.
// sha256File returns the lowercase hex SHA-256 of a file, matching the
// encoding Export records in Manifest.BboltSHA256.
func sha256File(path string) (string, error) {
// #nosec G304 -- path is the staged bbolt snapshot inside the import
// staging dir (MkdirTemp under StatePath), not attacker-controlled.
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func streamFileToTar(tw *tar.Writer, name, srcPath string, modTime time.Time) error {
// #nosec G304 -- srcPath is supplied by Export's caller (the daemon's control handler, sourced from cfg.StatePath / cfg.Signatures.RulesDir) or is the bbolt snapshot path created via os.CreateTemp earlier in Export. Both are root-controlled; csm runs as root via systemd. Not attacker-controlled.
f, err := os.Open(srcPath)
if err != nil {
return fmt.Errorf("open %s: %w", srcPath, err)
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return fmt.Errorf("stat %s: %w", srcPath, err)
}
hdr := &tar.Header{
Name: name,
Mode: 0600,
Size: info.Size(),
ModTime: modTime,
}
if err := tw.WriteHeader(hdr); err != nil {
return fmt.Errorf("tar header %s: %w", name, err)
}
if _, err := io.Copy(tw, f); err != nil {
return fmt.Errorf("tar copy %s: %w", name, err)
}
return nil
}
// walkDirIntoTar streams every regular file under srcDir into the tar
// under entryPrefix, skipping relative paths in the skip list. A skipped
// directory excludes its complete tree.
func walkDirIntoTar(tw *tar.Writer, srcDir, entryPrefix string, skip []string, modTime time.Time) error {
skipSet := map[string]bool{}
for _, s := range skip {
skipSet[s] = true
}
err := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(srcDir, path)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
if skipSet[rel] {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if info.IsDir() {
return nil
}
entry := entryPrefix + filepath.ToSlash(rel)
return streamFileToTar(tw, entry, path, modTime)
})
if err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// atomicReplace renames src over dst, ensuring the parent directory is
// fsync'd so the rename survives a crash.
func atomicReplace(src, dst string) error {
if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil {
return err
}
if err := os.Rename(src, dst); err != nil {
return err
}
parent, err := os.Open(filepath.Dir(dst))
if err != nil {
return err
}
defer parent.Close()
return parent.Sync()
}
func isFirewallBucket(name string) bool {
return strings.HasPrefix(name, "fw:")
}
package store
import (
"fmt"
"os"
"path/filepath"
"sort"
"time"
"github.com/pidginhost/csm/internal/firewall"
bolt "go.etcd.io/bbolt"
)
// The durable firewall journal and its committed snapshot validate each
// other, so they move as one unit and are never merged key by key with
// another database's.
func firewallDurableBucket(name string) bool {
switch name {
case firewallActionsBucket, firewallBudgetBucket, firewallAuditBucket, firewallActionIndexBucket, firewallBudgetIndexBucket,
firewallActionHistoryBucket, firewallMemberIndexBucket, firewallBatchMembersBucket, firewallSnapshotBucket:
return true
}
for _, rows := range firewallSnapshotBuckets {
if name == rows {
return true
}
}
return false
}
// firewallImportJournal reports whether tx holds a durable journal or
// snapshot. It refuses one that cannot be read or still has unresolved or
// undelivered work: an import must not replace or carry recovery evidence.
func firewallImportJournal(tx *bolt.Tx) (bool, error) {
index, err := readFirewallJournalIndex(tx)
if err != nil {
return true, err
}
if index.PendingID != "" || len(index.Audit) > 0 {
return true, fmt.Errorf("%w: firewall journal has outstanding work", firewall.ErrStateConflict)
}
if tx.Bucket([]byte(firewallSnapshotBucket)) == nil {
if index.Initialized {
return true, fmt.Errorf("%w: firewall journal has no snapshot", firewall.ErrStateCorrupt)
}
return false, nil
}
meta, err := readFirewallSnapshotMeta(tx)
if err != nil {
return true, err
}
rows, err := readFirewallSnapshotRows(tx, meta)
if err != nil {
return true, err
}
if _, err = decodeFirewallSnapshot(rows, meta); err != nil {
return true, err
}
return true, nil
}
// Firewall buckets contain flat rows. Nested data must neither be copied as
// an empty value nor silently discarded when replacing a target journal.
func validateFirewallImportBuckets(tx *bolt.Tx) error {
return tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
if !isFirewallBucket(string(name)) {
return nil
}
return bucket.ForEach(func(_, value []byte) error {
if value == nil {
return fmt.Errorf("%w: nested firewall bucket", firewall.ErrStateCorrupt)
}
return nil
})
})
}
// mergeFirewallBucketsFromSnapshot copies the snapshot's fw:* buckets into
// the target bbolt at statePath/csm.db in one transaction. A durable journal
// in the snapshot replaces the target's as a whole; a target journal the
// snapshot lacks is kept. Every other firewall bucket merges key by key,
// keeping destination-only keys. Returns the bucket names copied.
func mergeFirewallBucketsFromSnapshot(snapshotPath, statePath string) ([]string, error) {
if err := verifySnapshot(snapshotPath); err != nil {
return nil, err
}
src, err := bolt.Open(snapshotPath, 0600, &bolt.Options{Timeout: 5 * time.Second, ReadOnly: true})
if err != nil {
return nil, fmt.Errorf("opening snapshot: %w", err)
}
defer func() { _ = src.Close() }()
var merged []string
err = src.View(func(stx *bolt.Tx) error {
incoming, archiveErr := firewallImportJournal(stx)
if archiveErr != nil {
return fmt.Errorf("archive firewall journal: %w", archiveErr)
}
if shapeErr := validateFirewallImportBuckets(stx); shapeErr != nil {
return shapeErr
}
// Refuse the source before creating a destination; general store
// startup would commit unrelated initialization before validation.
if mkdirErr := os.MkdirAll(statePath, 0700); mkdirErr != nil {
return fmt.Errorf("creating target directory: %w", mkdirErr)
}
target := filepath.Join(statePath, "csm.db")
if verifyErr := VerifyDatabase(target); verifyErr != nil {
return fmt.Errorf("target database: %w", verifyErr)
}
// bbolt otherwise commits during Open if the file has no persisted
// freelist. Defer that write to the import's validated transaction.
dst, openErr := bolt.Open(target, 0600, &bolt.Options{Timeout: 5 * time.Second, NoFreelistSync: true})
if openErr != nil {
return fmt.Errorf("opening target: %w", openErr)
}
defer func() { _ = dst.Close() }()
dst.NoFreelistSync = false
return dst.Update(func(dtx *bolt.Tx) error {
local, localErr := firewallImportJournal(dtx)
if localErr != nil {
return fmt.Errorf("local firewall journal: %w", localErr)
}
if shapeErr := validateFirewallImportBuckets(dtx); shapeErr != nil {
return fmt.Errorf("local firewall buckets: %w", shapeErr)
}
if incoming {
// Collect names first: deleting while visiting the root cursor
// can skip buckets after the tree changes.
var names [][]byte
if listErr := dtx.ForEach(func(name []byte, _ *bolt.Bucket) error {
if firewallDurableBucket(string(name)) {
names = append(names, append([]byte(nil), name...))
}
return nil
}); listErr != nil {
return listErr
}
for _, name := range names {
if deleteErr := dtx.DeleteBucket(name); deleteErr != nil {
return deleteErr
}
}
}
if mergeErr := stx.ForEach(func(name []byte, sb *bolt.Bucket) error {
if !isFirewallBucket(string(name)) || local && !incoming && firewallDurableBucket(string(name)) {
return nil
}
bucket, createErr := dtx.CreateBucketIfNotExists(name)
if createErr != nil {
return createErr
}
if copyErr := sb.ForEach(func(k, v []byte) error {
return bucket.Put(append([]byte(nil), k...), append([]byte(nil), v...))
}); copyErr != nil {
return copyErr
}
merged = append(merged, string(name))
return nil
}); mergeErr != nil {
return mergeErr
}
// Tentative config rollback is process-lifetime state. Disarm it
// in this transaction so a late refusal cannot leave a partial import.
if rollback := dtx.Bucket([]byte(fwRollbackBucket)); rollback != nil {
return rollback.Delete([]byte(fwRollbackKey))
}
return nil
})
})
if err != nil {
return nil, err
}
sort.Strings(merged)
return merged, nil
}
package store
import (
"bytes"
"encoding/json"
"fmt"
"time"
bolt "go.etcd.io/bbolt"
)
// maxAttackEvents is the maximum number of attack events to retain.
// It is a var (not const) so tests can override it.
var maxAttackEvents = 100_000
// AttackEvent is the store-layer representation of an attack event.
type AttackEvent struct {
Timestamp time.Time `json:"timestamp"`
IP string `json:"ip"`
AttackType string `json:"attack_type"`
CheckName string `json:"check_name"`
Severity int `json:"severity"`
Account string `json:"account,omitempty"`
Message string `json:"message,omitempty"`
}
// MailEvidence is what CSM learned about an address as a mail sender:
// mail login attacks and the spam verdicts of its scanned messages.
// Masses are decayed to MassAt.
type MailEvidence struct {
AuthAttacks int `json:"auth_attacks,omitempty"`
LastAuthAttack time.Time `json:"last_auth_attack,omitzero"`
Spam int `json:"spam,omitempty"`
Ham int `json:"ham,omitempty"`
SpamMass float64 `json:"spam_mass,omitempty"`
HamMass float64 `json:"ham_mass,omitempty"`
MassAt time.Time `json:"mass_at,omitzero"`
LastSpam time.Time `json:"last_spam,omitzero"`
}
// IPRecord is the store-layer representation of an IP attack record.
type IPRecord struct {
IP string `json:"ip"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
EventCount int `json:"event_count"`
AttackCounts map[string]int `json:"attack_counts,omitempty"`
Accounts map[string]int `json:"accounts,omitempty"`
AuthSuccessAccounts map[string]int `json:"auth_success_accounts,omitempty"`
ThreatScore int `json:"threat_score"`
AutoBlocked bool `json:"auto_blocked,omitempty"`
Mail *MailEvidence `json:"mail,omitempty"`
// NeedsRewrite marks retired persisted state even when rescoring leaves
// the score unchanged. It is load metadata, never part of the record.
NeedsRewrite bool `json:"-"`
}
func (r *IPRecord) UnmarshalJSON(data []byte) error {
type record IPRecord
var decoded record
if err := json.Unmarshal(data, &decoded); err != nil {
return err
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(data, &fields); err != nil {
return err
}
for _, key := range []string{"brute_force_window_start", "brute_force_window_count", "brute_force_sustained_at"} {
if _, ok := fields[key]; ok {
decoded.NeedsRewrite = true
break
}
}
*r = IPRecord(decoded)
return nil
}
// RecordAttackEvent inserts an attack event into both the primary bucket
// (attacks:events, keyed by TimeKey) and the secondary index bucket
// (attacks:events:ip, keyed by IP/TimeKey). It increments the event counter
// and prunes oldest entries if the count exceeds maxAttackEvents.
func (db *DB) RecordAttackEvent(event AttackEvent, counter int) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
primary := tx.Bucket([]byte("attacks:events"))
writer := newTimeKeyWriter(primary)
secondary := tx.Bucket([]byte("attacks:events:ip"))
key := TimeKey(event.Timestamp, counter)
val, err := json.Marshal(event)
if err != nil {
return err
}
// Callers restart the counter for each batch. Reserve an unused
// primary key in this write transaction so an index never loses its event.
for primary.Get([]byte(key)) != nil {
counter++
key = TimeKey(event.Timestamp, counter)
}
if err := writer.put([]byte(key), val); err != nil {
return err
}
writer.settle()
// The index only needs the key; the event itself lives in the
// primary bucket.
secondaryKey := event.IP + "/" + key
if err := secondary.Put([]byte(secondaryKey), []byte{}); err != nil {
return err
}
if err := incrCounter(tx, "attacks:events:count", 1); err != nil {
return err
}
// Prune oldest entries if count exceeds maxAttackEvents.
meta := tx.Bucket([]byte("meta"))
var count int
if v := meta.Get([]byte("attacks:events:count")); v != nil {
_, _ = fmt.Sscanf(string(v), "%d", &count)
}
if count > maxAttackEvents {
excess := count - maxAttackEvents
c := primary.Cursor()
k, v := c.First()
for ; k != nil && excess > 0; excess-- {
// Unmarshal to get the IP for secondary index cleanup.
var ev AttackEvent
if err := json.Unmarshal(v, &ev); err != nil {
return err
}
secKey := ev.IP + "/" + string(k)
if err := secondary.Delete([]byte(secKey)); err != nil {
return err
}
if err := c.Delete(); err != nil {
return err
}
// Re-seek after delete (bbolt cursor behavior).
k, v = c.First()
}
if err := setCounter(tx, "attacks:events:count", maxAttackEvents); err != nil {
return err
}
}
return nil
})
}
// QueryAttackEvents returns up to limit attack events for the given IP,
// newest-first. It walks the secondary index backwards from the end of the
// IP's key range and resolves each entry in the primary bucket.
func (db *DB) QueryAttackEvents(ip string, limit int) []AttackEvent {
var results []AttackEvent
prefix := []byte(ip + "/")
_ = db.bolt.View(func(tx *bolt.Tx) error {
primary := tx.Bucket([]byte("attacks:events"))
c := tx.Bucket([]byte("attacks:events:ip")).Cursor()
var v []byte
k, _ := c.Seek(append(append([]byte(nil), prefix...), 0xff))
if k == nil {
k, v = c.Last()
} else {
k, v = c.Prev()
}
for ; k != nil && bytes.HasPrefix(k, prefix) && len(results) < limit; k, v = c.Prev() {
if ev, ok := resolveIndexedAttackEvent(primary, ip, k[len(prefix):], v); ok {
results = append(results, ev)
}
}
return nil
})
return results
}
// resolveIndexedAttackEvent reads the event an index entry points at. Index
// entries written by earlier builds carry their own copy of the event, which
// is used when the primary row is gone or now holds another address's event.
func resolveIndexedAttackEvent(primary *bolt.Bucket, ip string, timeKey, indexValue []byte) (AttackEvent, bool) {
if raw := primary.Get(timeKey); raw != nil {
var ev AttackEvent
if json.Unmarshal(raw, &ev) == nil && ev.IP == ip {
return ev, true
}
}
if len(indexValue) > 0 {
var ev AttackEvent
if json.Unmarshal(indexValue, &ev) == nil && ev.IP == ip {
return ev, true
}
}
return AttackEvent{}, false
}
// IPRecordTx writes IP records inside one UpdateIPRecords transaction.
type IPRecordTx struct{ b *bolt.Bucket }
// UpdateIPRecords runs fn in one transaction on the attacks:records bucket,
// so a batch of saves and deletes costs one sync however large it is. A
// record that Save or Delete refuses leaves the rest of the batch intact;
// an error returned by fn discards the whole batch.
func (db *DB) UpdateIPRecords(fn func(*IPRecordTx) error) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
return fn(&IPRecordTx{b: tx.Bucket([]byte("attacks:records"))})
})
}
// Save stores record keyed by its IP.
func (w *IPRecordTx) Save(record IPRecord) error {
val, err := json.Marshal(record)
if err != nil {
return err
}
return w.b.Put([]byte(record.IP), val)
}
// Delete removes the record stored for ip.
func (w *IPRecordTx) Delete(ip string) error {
return w.b.Delete([]byte(ip))
}
// SaveIPRecord stores an IP record in the attacks:records bucket, keyed by IP.
func (db *DB) SaveIPRecord(record IPRecord) error {
return db.UpdateIPRecords(func(w *IPRecordTx) error { return w.Save(record) })
}
// LoadIPRecord retrieves an IP record from the attacks:records bucket.
// Returns the record and true if found, or a zero value and false if not.
func (db *DB) LoadIPRecord(ip string) (IPRecord, bool) {
var record IPRecord
var found bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("attacks:records"))
v := b.Get([]byte(ip))
if v == nil {
return nil
}
if json.Unmarshal(v, &record) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
found = true
return nil
})
return record, found
}
// LoadAllIPRecords returns all IP records from the attacks:records bucket.
func (db *DB) LoadAllIPRecords() map[string]*IPRecord {
records := make(map[string]*IPRecord)
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("attacks:records"))
return b.ForEach(func(k, v []byte) error {
var record IPRecord
if json.Unmarshal(v, &record) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
records[string(k)] = &record
return nil
})
})
return records
}
// DeleteIPRecord removes an IP record from the attacks:records bucket.
func (db *DB) DeleteIPRecord(ip string) error {
return db.UpdateIPRecords(func(w *IPRecordTx) error { return w.Delete(ip) })
}
// ReadAllAttackEvents returns all attack events from the primary bucket.
// Used for stats computation (hourly/daily bucketing).
func (db *DB) ReadAllAttackEvents() []AttackEvent {
var events []AttackEvent
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("attacks:events"))
return b.ForEach(func(k, v []byte) error {
var ev AttackEvent
if json.Unmarshal(v, &ev) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
events = append(events, ev)
return nil
})
})
return events
}
package store
import (
"bytes"
"encoding/binary"
"net"
"time"
bolt "go.etcd.io/bbolt"
)
// botVerifyEntry layout:
//
// byte 0 verified flag (1 byte)
// bytes 1..8 expiry as unix nanos (8 bytes big-endian)
//
// Key format:
//
// <bot-bytes> 0x00 <ip-bytes>
//
// IP is stored in its 16-byte form (IPv4 promoted via To16). The
// 0x00 separator allows the bot name to be any non-null string while
// keeping keys byte-sortable within a single bucket.
func botVerifyKey(ip net.IP, bot string) []byte {
ipBytes := ip.To16()
if ipBytes == nil {
return nil
}
out := make([]byte, 0, len(bot)+1+16)
out = append(out, []byte(bot)...)
out = append(out, 0x00)
out = append(out, ipBytes...)
return out
}
// PutBotVerify stores a PTR+forward-A verification result with an
// explicit expiry. A verified=false entry means the IP failed rDNS
// and will emit http_ua_spoof on the next scan that sees it with the
// same bot UA.
func (db *DB) PutBotVerify(ip net.IP, bot string, verified bool, expiresAt time.Time) error {
key := botVerifyKey(ip, bot)
if key == nil {
return nil
}
var val [9]byte
if verified {
val[0] = 1
}
binary.BigEndian.PutUint64(val[1:], uint64(expiresAt.UnixNano())) // #nosec G115 -- unix nano stored as bit pattern; sign is irrelevant for expiry comparison
return db.bolt.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("botverify"))
if err != nil {
return err
}
if records := tx.Bucket([]byte(botVerifyUnverifiableBucket)); records != nil {
if err := records.Delete(key); err != nil {
return err
}
}
return b.Put(key, val[:])
})
}
// botVerifyUnverifiableBucket holds sources whose claimed bot identity had no
// PTR record, keyed like the verdict cache. The value is the observation time
// as unix nanos; the verifier derives retry suppression and history from it
// and sweeps records that can no longer matter. It lives apart from the
// verdict bucket because a missing PTR proves nothing about identity, and a
// build that predates this bucket must never read it as a spoof verdict.
const botVerifyUnverifiableBucket = "botverify_unverifiable"
// PutBotVerifyUnverifiable records that ip had no PTR for bot at observedAt.
func (db *DB) PutBotVerifyUnverifiable(ip net.IP, bot string, observedAt time.Time) error {
key := botVerifyKey(ip, bot)
if key == nil {
return nil
}
var val [8]byte
binary.BigEndian.PutUint64(val[:], uint64(observedAt.UnixNano())) // #nosec G115 -- unix nano stored as bit pattern; sign is irrelevant for time comparison
return db.bolt.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(botVerifyUnverifiableBucket))
if err != nil {
return err
}
return b.Put(key, val[:])
})
}
// BotVerifyUnverifiable returns when ip was recorded without a PTR for bot.
// Reads never write, so they cannot extend a record or race a bucket reset.
func (db *DB) BotVerifyUnverifiable(ip net.IP, bot string) (observedAt time.Time, ok bool) {
key := botVerifyKey(ip, bot)
if key == nil {
return time.Time{}, false
}
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(botVerifyUnverifiableBucket))
if b == nil {
return nil
}
if val := b.Get(key); len(val) == 8 {
observedAt, ok = unixNanoBits(val), true
}
return nil
})
return observedAt, ok
}
// SweepBotVerifyUnverifiable removes records observed before cutoff and
// returns how many it removed.
func (db *DB) SweepBotVerifyUnverifiable(cutoff time.Time) (int, error) {
removed := 0
err := db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(botVerifyUnverifiableBucket))
if b == nil {
return nil
}
var stale [][]byte
if err := b.ForEach(func(k, v []byte) error {
if len(v) != 8 || unixNanoBits(v).Before(cutoff) {
stale = append(stale, append([]byte(nil), k...))
}
return nil
}); err != nil {
return err
}
for _, k := range stale {
if err := b.Delete(k); err != nil {
return err
}
}
removed = len(stale)
return nil
})
if err != nil {
return 0, err
}
return removed, nil
}
func unixNanoBits(val []byte) time.Time {
return time.Unix(0, int64(binary.BigEndian.Uint64(val))) // #nosec G115 -- reinterpret stored bit pattern as signed nanos
}
// EnsureBotVerifyLogicVersion compares the stored cache logic version
// with version and, on mismatch (or when no marker exists yet), drops
// the botverify bucket and its missing-PTR records, then records the new
// version. The marker lives in the "meta" bucket under
// botverify:logic_version. Returns true when the bucket was dropped.
//
// Use this from daemon startup so that any change to the verifier
// logic (BotDomains suffix list, ClaimedBotFromUA mapping, etc.)
// automatically invalidates entries written under the old rules.
// Operators do not need to know about the cache.
func (db *DB) EnsureBotVerifyLogicVersion(version int) (bool, error) {
var dropped bool
err := db.bolt.Update(func(tx *bolt.Tx) error {
meta, mErr := tx.CreateBucketIfNotExists([]byte("meta"))
if mErr != nil {
return mErr
}
current := uint64(version) // #nosec G115 -- logic version is a small positive internal constant
stored := ^uint64(0)
if raw := meta.Get([]byte("botverify:logic_version")); len(raw) == 8 {
stored = binary.BigEndian.Uint64(raw)
}
if stored == current {
return nil
}
if _, dErr := resetBotVerifyBuckets(tx); dErr != nil {
return dErr
}
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], current)
if pErr := meta.Put([]byte("botverify:logic_version"), buf[:]); pErr != nil {
return pErr
}
dropped = true
return nil
})
if err != nil {
return false, err
}
return dropped, nil
}
// ResetBotVerify drops every cached PTR+forward-A result and no-PTR record.
// Returns the number of entries cleared. Use after a verifier-logic upgrade that
// would invalidate prior negative cache entries (e.g., a domain suffix
// fix that turns prior false-spoof entries into positives). Safe to
// call when the bucket is missing or empty.
func (db *DB) ResetBotVerify() (int, error) {
var cleared int
err := db.bolt.Update(func(tx *bolt.Tx) error {
var err error
cleared, err = resetBotVerifyBuckets(tx)
return err
})
if err != nil {
return 0, err
}
return cleared, nil
}
// resetBotVerifyBuckets empties the verdict cache and its no-PTR records
// together, so no retry suppression outlives the rules that produced it.
func resetBotVerifyBuckets(tx *bolt.Tx) (int, error) {
cleared := 0
for _, name := range []string{"botverify", botVerifyUnverifiableBucket} {
if b := tx.Bucket([]byte(name)); b != nil {
cleared += b.Stats().KeyN
if err := tx.DeleteBucket([]byte(name)); err != nil {
return 0, err
}
}
}
for _, name := range []string{"botverify", botVerifyUnverifiableBucket} {
if _, err := tx.CreateBucket([]byte(name)); err != nil {
return 0, err
}
}
return cleared, nil
}
// GetBotVerify returns (verified, valid). valid=false means no
// non-expired entry exists; the caller should treat the IP as
// unverified and (optionally) enqueue an async verify job.
func (db *DB) GetBotVerify(ip net.IP, bot string) (verified, valid bool) {
key := botVerifyKey(ip, bot)
if key == nil {
return false, false
}
var stored []byte
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("botverify"))
if b == nil {
return nil
}
val := b.Get(key)
if len(val) != 9 {
return nil
}
stored = append([]byte(nil), val...)
exp := time.Unix(0, int64(binary.BigEndian.Uint64(val[1:]))) // #nosec G115 -- reinterpret stored bit pattern as signed nanos
if time.Now().After(exp) {
return nil
}
verified = val[0] == 1
valid = true
return nil
})
if valid || len(stored) != 9 {
return verified, valid
}
_ = db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("botverify"))
current := b.Get(key)
if !bytes.Equal(current, stored) {
return nil
}
exp := time.Unix(0, int64(binary.BigEndian.Uint64(current[1:]))) // #nosec G115 -- reinterpret stored bit pattern as signed nanos
if time.Now().After(exp) {
return b.Delete(key)
}
return nil
})
return verified, valid
}
package store
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"time"
"github.com/pidginhost/csm/internal/session"
bolt "go.etcd.io/bbolt"
)
const browserSessionsBucket = "browser_sessions"
func (db *DB) ReplaceBrowserSession(rec session.Record, previous string, now time.Time, idle time.Duration) error {
if !rec.Valid(now, idle) {
return session.ErrInvalid
}
raw, err := json.Marshal(rec)
if err != nil {
return err
}
return boltUpdate(db.bolt, func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(browserSessionsBucket))
if err != nil {
return err
}
if previous != "" {
if _, readErr := readBrowserSession(tx, previous, now, idle); readErr != nil {
return readErr
}
}
c := b.Cursor()
count := 0
for k, v := c.First(); k != nil; k, v = c.Next() {
var old session.Record
if err := json.Unmarshal(v, &old); err != nil {
return fmt.Errorf("decode browser session: %w", err)
}
if string(k) == previous || !old.Valid(now, idle) {
if err := c.Delete(); err != nil {
return err
}
continue
}
count++
}
if count >= session.MaxSessions {
return session.ErrFull
}
if b.Get([]byte(rec.Verifier)) != nil {
return errors.New("browser session identity collision")
}
return b.Put([]byte(rec.Verifier), raw)
})
}
func readBrowserSession(tx *bolt.Tx, key string, now time.Time, idle time.Duration) (session.Record, error) {
b := tx.Bucket([]byte(browserSessionsBucket))
if b == nil {
return session.Record{}, session.ErrInvalid
}
raw := b.Get([]byte(key))
if raw == nil {
return session.Record{}, session.ErrInvalid
}
var rec session.Record
if err := json.Unmarshal(raw, &rec); err != nil {
return session.Record{}, fmt.Errorf("decode browser session: %w", err)
}
if rec.Verifier != key || !rec.Valid(now, idle) {
return session.Record{}, session.ErrInvalid
}
return rec, nil
}
func (db *DB) AccessBrowserSession(key string, now time.Time, idle time.Duration, touch bool) (session.Record, error) {
var rec session.Record
err := db.bolt.View(func(tx *bolt.Tx) error {
var readErr error
rec, readErr = readBrowserSession(tx, key, now, idle)
return readErr
})
if err != nil || !touch {
return rec, err
}
// Bound write amplification from dashboard polling. Expiry remains based on
// the last committed activity, so this can only shorten the idle window.
interval := min(30*time.Second, idle/4)
if now.Sub(rec.LastSeen) < interval {
return rec, nil
}
err = boltUpdate(db.bolt, func(tx *bolt.Tx) error {
current, readErr := readBrowserSession(tx, key, now, idle)
if readErr != nil {
return readErr
}
// Another request may have touched the session since our read.
// Preserve its newer activity and the existing write interval.
if now.Sub(current.LastSeen) < interval {
rec = current
return nil
}
current.LastSeen = now
raw, marshalErr := json.Marshal(current)
if marshalErr != nil {
return marshalErr
}
if putErr := tx.Bucket([]byte(browserSessionsBucket)).Put([]byte(key), raw); putErr != nil {
return putErr
}
rec = current
return nil
})
if err != nil {
return session.Record{}, err
}
return rec, nil
}
func (db *DB) ListBrowserSessions(now time.Time, idle time.Duration) ([]session.Record, error) {
records := make([]session.Record, 0)
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(browserSessionsBucket))
if b == nil {
return nil
}
return b.ForEach(func(k, v []byte) error {
var rec session.Record
if err := json.Unmarshal(v, &rec); err != nil {
return fmt.Errorf("decode browser session: %w", err)
}
if rec.Verifier != string(k) {
return errors.New("browser session identity mismatch")
}
if rec.Valid(now, idle) {
records = append(records, rec)
}
return nil
})
})
if err != nil {
return nil, err
}
sort.Slice(records, func(i, j int) bool { return records[i].Created.After(records[j].Created) })
return records, nil
}
func (db *DB) RevokeBrowserSession(id string) error {
return boltUpdate(db.bolt, func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(browserSessionsBucket))
if b == nil {
return nil
}
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var rec session.Record
if err := json.Unmarshal(v, &rec); err != nil {
return fmt.Errorf("decode browser session: %w", err)
}
if rec.ID == id {
return c.Delete()
}
}
return nil
})
}
func (db *DB) ClearBrowserSessions() error {
return boltUpdate(db.bolt, func(tx *bolt.Tx) error {
if tx.Bucket([]byte(browserSessionsBucket)) == nil {
return nil
}
return tx.DeleteBucket([]byte(browserSessionsBucket))
})
}
// DisarmBrowserSessionsSnapshot strips session verifiers and the antispam
// learning ledger from a private, stopped snapshot. The ledger cannot be
// restored without its anchor bodies and hash keys, and an old one resumed
// against live storage would steer audits back to stale targets. It must
// never be called on the daemon's live database.
func DisarmBrowserSessionsSnapshot(path string) error {
if err := verifySnapshot(path); err != nil {
return err
}
file, err := os.CreateTemp(filepath.Dir(path), ".session-free-*")
if err != nil {
return err
}
cleanPath := file.Name()
defer func() { _ = os.Remove(cleanPath) }()
if err = file.Close(); err != nil {
return err
}
err = func() (result error) {
snapshot, openErr := bolt.Open(path, 0600, &bolt.Options{Timeout: time.Second})
if openErr != nil {
return openErr
}
defer func() { result = errors.Join(result, snapshot.Close()) }()
db := &DB{bolt: snapshot, path: path}
if clearErr := db.ClearBrowserSessions(); clearErr != nil {
return clearErr
}
if clearErr := db.clearLearning(); clearErr != nil {
return clearErr
}
// Compact even if the bucket was already absent: earlier revocations
// may have left metadata in free pages. Only live records are copied.
_, _, compactErr := db.CompactInto(cleanPath, 16*1024*1024)
return compactErr
}()
if err != nil {
return err
}
return os.Rename(cleanPath, path)
}
package store
import (
"encoding/binary"
"fmt"
"net"
"time"
bolt "go.etcd.io/bbolt"
)
// challengePassBucket maps an address (16-byte form) to when it last passed
// a challenge, as big-endian unix nanos.
const challengePassBucket = "challenge_passes"
// PutChallengePass records that ip passed a challenge at at.
func (db *DB) PutChallengePass(ip string, at time.Time) error {
parsed := net.ParseIP(ip)
if parsed == nil {
return fmt.Errorf("challenge pass: invalid address %q", ip)
}
var val [8]byte
binary.BigEndian.PutUint64(val[:], uint64(at.UnixNano())) // #nosec G115 -- unix nanos stored as a bit pattern and read back the same way
return db.bolt.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(challengePassBucket))
if err != nil {
return err
}
key := parsed.To16()
// Completions persist after releasing the challenge-list lock, so
// a slower earlier completion must not shorten a later pass.
if prev := b.Get(key); len(prev) == 8 {
previous := time.Unix(0, int64(binary.BigEndian.Uint64(prev))) // #nosec G115 -- stored unix nanos bit pattern
if !at.After(previous) {
return nil
}
}
return b.Put(key, val[:])
})
}
// ChallengePasses returns the passes recorded after cutoff and deletes the
// older ones.
func (db *DB) ChallengePasses(cutoff time.Time) (map[string]time.Time, error) {
out := make(map[string]time.Time)
err := db.bolt.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(challengePassBucket))
if err != nil {
return err
}
var stale [][]byte
err = b.ForEach(func(k, v []byte) error {
if len(k) != net.IPv6len || len(v) != 8 {
stale = append(stale, append([]byte(nil), k...))
return nil
}
at := time.Unix(0, int64(binary.BigEndian.Uint64(v))) // #nosec G115 -- reverses the bit pattern PutChallengePass wrote
if !at.After(cutoff) {
stale = append(stale, append([]byte(nil), k...))
return nil
}
out[net.IP(k).String()] = at
return nil
})
if err != nil {
return err
}
for _, k := range stale {
if err := b.Delete(k); err != nil {
return err
}
}
return nil
})
return out, err
}
package store
import bolt "go.etcd.io/bbolt"
const contentLogicVersionKey = "content:logic_version"
// ContentLogicVersionChanged reports whether token differs from the last
// completed finding re-verification sweep. Reading and committing the marker
// are separate so a crash or shutdown during the sweep causes a retry.
func (db *DB) ContentLogicVersionChanged(token string) (bool, error) {
var changed bool
err := db.bolt.View(func(tx *bolt.Tx) error {
meta := tx.Bucket([]byte("meta"))
if meta == nil {
changed = true
return nil
}
changed = string(meta.Get([]byte(contentLogicVersionKey))) != token
return nil
})
if err != nil {
return false, err
}
return changed, nil
}
// SetContentLogicVersion records token after a finding re-verification sweep
// completes, preventing another run until one of its logic versions changes.
func (db *DB) SetContentLogicVersion(token string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
meta, err := tx.CreateBucketIfNotExists([]byte("meta"))
if err != nil {
return err
}
return meta.Put([]byte(contentLogicVersionKey), []byte(token))
})
}
package store
import (
"bytes"
bolt "go.etcd.io/bbolt"
)
// dailySummaryKey holds the daily alert summary window in the meta bucket.
const dailySummaryKey = "alerts:daily_summary"
// LoadDailySummary returns the saved daily alert summary, or nil when none
// was saved.
func (db *DB) LoadDailySummary() ([]byte, error) {
var out []byte
err := db.bolt.View(func(tx *bolt.Tx) error {
out = bytes.Clone(tx.Bucket([]byte("meta")).Get([]byte(dailySummaryKey)))
return nil
})
return out, err
}
// SaveDailySummary replaces the saved daily alert summary.
func (db *DB) SaveDailySummary(b []byte) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
return tx.Bucket([]byte("meta")).Put([]byte(dailySummaryKey), b)
})
}
package store
import (
"bytes"
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"time"
bolt "go.etcd.io/bbolt"
)
// Bucket names - all buckets are created on Open().
var bucketNames = []string{
"history",
"attacks:records",
"attacks:events",
"attacks:events:ip",
"threats",
"threats:whitelist",
"fw:blocked",
"fw:allowed",
"fw:subnets",
"fw:port_allowed",
"reputation",
mailGoodSourceBucket,
eximFrozenSeenBucket,
mailStatsBucket,
mailRejectsBucket,
mailGapsBucket,
spamSourceBucket,
mailSprayBucket,
knownCustomersBucket,
"plugins",
"plugins:sites",
"meta",
"email:geo",
"email:fwd",
senderProfileBucket,
"db_object_backups",
"sig_watch",
bucketStatsDaily,
bucketLatestByCheck,
"phprelay:meta",
"phprelay:msgindex",
"phprelay:ignore",
"phprelay:settings",
"incidents",
"fw:rollback",
adminEmailsBucket,
mailboxSuspensionsBucket,
"botverify",
challengePassBucket,
botVerifyUnverifiableBucket,
prefsBucket,
"scan_jobs",
"scan_job_findings",
scanCursorBucket,
LearnMetaBucket,
LearnClustersBucket,
LearnAliasesBucket,
LearnTargetsBucket,
LearnIntentsBucket,
LearnOpsBucket,
LearnUsageBucket,
LearnCursorBucket,
LearnWatchFoldersBucket,
LearnWatchSeenBucket,
}
// DB wraps a bbolt database.
type DB struct {
bolt *bolt.DB
path string
}
var (
globalDB *DB
globalMu sync.Mutex
ensureOnce sync.Once
ensureErr error
)
// Global returns the singleton DB instance.
func Global() *DB {
globalMu.Lock()
defer globalMu.Unlock()
return globalDB
}
// SetGlobal sets the singleton DB instance.
func SetGlobal(db *DB) {
globalMu.Lock()
globalDB = db
globalMu.Unlock()
}
// WriteTxID returns the id of the most recently committed write
// transaction. bbolt bumps the id once per committed write transaction,
// so two readings bracket a code path and their difference counts its
// write commits.
func (db *DB) WriteTxID() int {
var id int
_ = db.bolt.View(func(tx *bolt.Tx) error {
id = tx.ID()
return nil
})
return id
}
// EnsureOpen opens the store if not already open. Safe to call from any CLI path.
// First call opens the DB; subsequent calls return immediately.
func EnsureOpen(statePath string) error {
ensureOnce.Do(func() {
db, err := Open(statePath)
if err != nil {
ensureErr = err
return
}
SetGlobal(db)
})
return ensureErr
}
// Open opens or creates the bbolt database at {statePath}/csm.db.
// Creates all buckets if they don't exist. Runs migration if needed.
func Open(statePath string) (*DB, error) {
dbPath := filepath.Join(statePath, "csm.db")
if err := os.MkdirAll(statePath, 0700); err != nil {
return nil, fmt.Errorf("creating state dir: %w", err)
}
bdb, err := bolt.Open(dbPath, 0600, &bolt.Options{Timeout: 5 * time.Second})
if err != nil {
return nil, fmt.Errorf("opening bbolt: %w", err)
}
// Create all buckets
err = bdb.Update(func(tx *bolt.Tx) error {
// No time-index buckets means this database cannot contain legacy
// wall-clock keys. Mark it canonical in the same transaction that
// creates the buckets, avoiding an extra startup write on new stores.
freshTimeKeyStore := tx.Bucket([]byte("history")) == nil &&
tx.Bucket([]byte("attacks:events")) == nil &&
tx.Bucket([]byte("attacks:events:ip")) == nil
for _, name := range bucketNames {
if _, berr := tx.CreateBucketIfNotExists([]byte(name)); berr != nil {
return fmt.Errorf("creating bucket %s: %w", name, berr)
}
}
// Initialise phprelay schema_version on first open. Stored as
// 8-byte big-endian uint64 so future migrations can read/compare
// it consistently.
meta := tx.Bucket([]byte("phprelay:meta"))
if meta.Get([]byte("schema_version")) == nil {
if perr := meta.Put([]byte("schema_version"), []byte{0, 0, 0, 0, 0, 0, 0, 1}); perr != nil {
return fmt.Errorf("init phprelay schema_version: %w", perr)
}
}
if freshTimeKeyStore {
if markerErr := tx.Bucket([]byte("meta")).Put(
[]byte(timeKeyUTCMarker), []byte(timeKeyMigrationDone),
); markerErr != nil {
return fmt.Errorf("init UTC time-key marker: %w", markerErr)
}
}
return nil
})
if err != nil {
_ = bdb.Close()
return nil, err
}
db := &DB{bolt: bdb, path: dbPath}
// Run migration if needed. A failed migration does not set the "migrated"
// sentinel, so proceeding would boot the daemon on partial security state
// and retry the same broken migration every restart. Fail loud instead.
if err := db.migrateIfNeeded(statePath); err != nil {
_ = bdb.Close()
return nil, fmt.Errorf("store migration: %w", err)
}
// Older releases formatted keys in whatever zone each producer supplied.
// Canonicalize those persisted keys before any key-based reads or pruning.
if err := db.migrateTimeKeysToUTC(); err != nil {
_ = bdb.Close()
return nil, fmt.Errorf("time-key migration: %w", err)
}
// One-time backfill of stats:daily from existing history. Runs on
// hosts upgrading from a build that pre-dates the stats:daily bucket;
// no-op afterwards thanks to a meta sentinel.
if err := db.BackfillStatsDaily(); err != nil {
fmt.Fprintf(os.Stderr, "store: stats:daily backfill warning: %v\n", err)
}
if err := db.BackfillLatestByCheck(); err != nil {
fmt.Fprintf(os.Stderr, "store: stats:latest_by_check backfill warning: %v\n", err)
}
if err := db.seedDefaultModSecNoEscalateRules(); err != nil {
fmt.Fprintf(os.Stderr, "store: ModSecurity no-escalate seed warning: %v\n", err)
}
return db, nil
}
const (
modsecNoEscalateSeededKey = "modsec:no_escalate_seeded"
modsecNoEscalateBrowserSeededKey = "modsec:no_escalate_seeded_900134"
defaultModSecNoEscalateWPEnumerationID = 900112
defaultModSecNoEscalateBrowserUploadID = 900134
)
func (db *DB) seedDefaultModSecNoEscalateRules() error {
return db.bolt.Update(func(tx *bolt.Tx) error {
meta := tx.Bucket([]byte("meta"))
if meta.Get([]byte(modsecNoEscalateSeededKey)) != nil && meta.Get([]byte(modsecNoEscalateBrowserSeededKey)) != nil {
return nil
}
var ids []int
if v := meta.Get([]byte(modsecNoEscalateKey)); v != nil {
if err := json.Unmarshal(v, &ids); err != nil {
return err
}
} else if meta.Get([]byte(modsecNoEscalateSeededKey)) == nil {
// WordPress user enumeration is blocked at the HTTP layer only.
ids = append(ids, defaultModSecNoEscalateWPEnumerationID)
}
if meta.Get([]byte(modsecNoEscalateBrowserSeededKey)) == nil && !slices.Contains(ids, defaultModSecNoEscalateBrowserUploadID) {
// A third-party page can steer browser-sent upload progress
// denies, so they must not ban the visitor. Added once, also on
// upgrade; an operator can turn the escalation back on.
ids = append(ids, defaultModSecNoEscalateBrowserUploadID)
}
val, err := json.Marshal(ids)
if err != nil {
return err
}
if err := meta.Put([]byte(modsecNoEscalateKey), val); err != nil {
return err
}
if err := meta.Put([]byte(modsecNoEscalateSeededKey), []byte("1")); err != nil {
return err
}
return meta.Put([]byte(modsecNoEscalateBrowserSeededKey), []byte("1"))
})
}
// Close closes the bbolt database.
func (db *DB) Close() error {
if db.bolt == nil {
return nil
}
return db.bolt.Close()
}
// Path returns the on-disk path of the bbolt database file.
func (db *DB) Path() string {
return db.path
}
// HasBucket reports whether a top-level bucket named name exists in db.
func (db *DB) HasBucket(name string) bool {
found := false
_ = db.bolt.View(func(tx *bolt.Tx) error {
if tx.Bucket([]byte(name)) != nil {
found = true
}
return nil
})
return found
}
// timeKeyFillPercent packs mostly ordered writes to buckets keyed by TimeKey.
// Their split tail pages are rarely written again, so bbolt's default of 0.5
// would leave them half empty.
const timeKeyFillPercent = 0.9
// timeKeyWriter picks the split point for one write transaction. bbolt applies
// a bucket's FillPercent to every page it splits at commit, so the choice is
// per transaction rather than per key. A few delayed keys cost less than half
// empty tail pages; when most keys land before the stored tail, the balanced
// default leaves room in the pages they split. Also balance when delayed
// records carry at least half the inserted bytes: a minority of large records
// can otherwise leave sparse pages throughout the backfill.
type timeKeyWriter struct {
bucket *bolt.Bucket
tail []byte
older int
total int
olderBytes int
totalBytes int
}
func newTimeKeyWriter(b *bolt.Bucket) *timeKeyWriter {
tail, _ := b.Cursor().Last()
return &timeKeyWriter{bucket: b, tail: bytes.Clone(tail)}
}
func (w *timeKeyWriter) put(key, value []byte) error {
size := len(key) + len(value)
w.total++
w.totalBytes += size
if w.tail != nil && bytes.Compare(key, w.tail) <= 0 {
w.older++
w.olderBytes += size
}
return w.bucket.Put(key, value)
}
// settle sets the split point once every key of the transaction is known. It
// must run before the transaction commits.
func (w *timeKeyWriter) settle() {
w.bucket.FillPercent = timeKeyFillPercent
if w.older*2 > w.total || (w.olderBytes > 0 && w.olderBytes >= w.totalBytes-w.olderBytes) {
w.bucket.FillPercent = bolt.DefaultFillPercent
}
}
// TimeKey produces a fixed-width 28-byte key for chronological ordering.
// Format: YYYYMMDDHHmmssNNNNNNNNN-CCCC
// Lexicographic order equals chronological order.
//
// Keys use UTC so ordering is stable across producer zones, host timezone
// changes, and repeated local hours during daylight-saving transitions.
func TimeKey(t time.Time, counter int) string {
t = t.UTC()
return fmt.Sprintf("%04d%02d%02d%02d%02d%02d%09d-%04d",
t.Year(), t.Month(), t.Day(),
t.Hour(), t.Minute(), t.Second(),
t.Nanosecond(), counter)
}
// ParseTimeKeyPrefix converts a date string "YYYY-MM-DD" to a seek prefix "YYYYMMDD".
func ParseTimeKeyPrefix(date string) string {
if len(date) == 10 && date[4] == '-' && date[7] == '-' {
return date[:4] + date[5:7] + date[8:10]
}
return date
}
// getCounter reads a counter from the meta bucket. Returns 0 if not found.
// HistoryCount returns the number of findings in the history bucket.
func (db *DB) HistoryCount() int {
return db.getCounter("history:count")
}
func (db *DB) getCounter(key string) int {
var count int
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
if v := b.Get([]byte(key)); v != nil {
_, _ = fmt.Sscanf(string(v), "%d", &count)
}
return nil
})
return count
}
// setCounter writes a counter to the meta bucket within an existing transaction.
func setCounter(tx *bolt.Tx, key string, count int) error {
b := tx.Bucket([]byte("meta"))
return b.Put([]byte(key), []byte(fmt.Sprintf("%d", count)))
}
// IsHealthy returns true if the bbolt file is open and all required buckets exist.
func (db *DB) IsHealthy() bool {
if db == nil || db.bolt == nil {
return false
}
err := db.bolt.View(func(tx *bolt.Tx) error {
for _, name := range []string{"history", "fw:blocked", "meta"} {
if tx.Bucket([]byte(name)) == nil {
return fmt.Errorf("bucket missing: %s", name)
}
}
return nil
})
return err == nil
}
// SizeBytes returns the on-disk size of the bbolt database file. Returns 0 if unavailable.
func (db *DB) SizeBytes() int64 {
if db == nil || db.path == "" {
return 0
}
info, err := os.Stat(db.path)
if err != nil {
return 0
}
return info.Size()
}
// incrCounter increments a counter within an existing transaction.
func incrCounter(tx *bolt.Tx, key string, delta int) error {
b := tx.Bucket([]byte("meta"))
var current int
if v := b.Get([]byte(key)); v != nil {
fmt.Sscanf(string(v), "%d", ¤t)
}
return b.Put([]byte(key), []byte(fmt.Sprintf("%d", current+delta)))
}
type dryRunBlockRecord struct {
IP string `json:"ip"`
Reason string `json:"reason"`
TimeoutSec int `json:"timeout_sec"`
}
// RecordDryRunBlock appends a dry-run-block record to the "dry_run_blocks"
// bucket. Called by the firewall engine when auto_response.dry_run is active
// so operators can review "what would have been blocked" before going live.
func (db *DB) RecordDryRunBlock(ip, reason string, timeout time.Duration) {
if db == nil || db.bolt == nil {
return
}
// Log-derived reasons can carry raw control bytes; the JSON encoder
// emits escape forms that dry-run readers can decode.
payload := dryRunBlockRecord{
IP: ip,
Reason: reason,
TimeoutSec: int(timeout.Seconds()),
}
val, err := json.Marshal(payload)
if err != nil {
return
}
_ = db.bolt.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(dryRunBlocksBucket))
if err != nil {
return err
}
key := []byte(time.Now().UTC().Format(time.RFC3339Nano) + ":" + ip)
return b.Put(key, val)
})
}
// PurgeAllDryRunBlocks deletes every record from the dry_run_blocks
// bucket and returns the number removed. Called when the operator
// flips auto_response.dry_run from true to false so /api/v1/status no
// longer reports a stale count from the previous dry-run window. A
// later periodic prune handles the slow accumulation case via
// PurgeDryRunBlocksOlderThan.
func (db *DB) PurgeAllDryRunBlocks() int {
if db == nil || db.bolt == nil {
return 0
}
removed := 0
_ = db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(dryRunBlocksBucket))
if b == nil {
return nil
}
removed = b.Stats().KeyN
return tx.DeleteBucket([]byte(dryRunBlocksBucket))
})
return removed
}
// PurgeDryRunBlocksOlderThan removes every dry_run_blocks record
// whose timestamp prefix is strictly older than cutoff. Returns the
// number removed. Key format is "<RFC3339Nano>:<ip>"; entries with a
// key that does not parse as a timestamp are left in place so a
// future key-format change does not silently drop records.
func (db *DB) PurgeDryRunBlocksOlderThan(cutoff time.Time) int {
if db == nil || db.bolt == nil {
return 0
}
removed := 0
_ = db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(dryRunBlocksBucket))
if b == nil {
return nil
}
var stale [][]byte
_ = b.ForEach(func(k, _ []byte) error {
s := string(k)
// RecordDryRunBlock writes UTC timestamps which always
// end in `Z`, so the first `Z:` reliably separates the
// timestamp from the IP (the IP itself can contain
// colons in v6 form, ruling out a naive first-colon
// split).
idx := strings.Index(s, "Z:")
if idx < 0 {
return nil
}
ts, err := time.Parse(time.RFC3339Nano, s[:idx+1])
if err != nil {
// Forward-compat: unrecognised key format is left
// in place rather than treated as stale, so a
// future schema change does not silently drop
// records during the rolling upgrade window.
return nil //nolint:nilerr // intentional skip
}
if ts.Before(cutoff) {
keyCopy := append([]byte(nil), k...)
stale = append(stale, keyCopy)
}
return nil
})
for _, k := range stale {
if err := b.Delete(k); err == nil {
removed++
}
}
return nil
})
return removed
}
// DryRunBlocksCount returns the number of recorded dry-run block entries.
func (db *DB) DryRunBlocksCount() int {
if db == nil || db.bolt == nil {
return 0
}
count := 0
_ = db.bolt.View(func(tx *bolt.Tx) error {
if b := tx.Bucket([]byte(dryRunBlocksBucket)); b != nil {
count = b.Stats().KeyN
}
return nil
})
return count
}
// migrateIfNeeded checks for the meta:migrated key and runs migration if absent.
func (db *DB) migrateIfNeeded(statePath string) error {
var migrated bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
if b.Get([]byte("migrated")) != nil {
migrated = true
}
return nil
})
if migrated {
return nil
}
return db.runMigration(statePath)
}
package store
import (
"encoding/json"
"fmt"
"time"
bolt "go.etcd.io/bbolt"
)
// DBObjectBackup is the persisted record of a SHOW CREATE captured
// before a manual `csm db-clean drop-object`. The CREATE SQL is the
// backup -- replaying it restores the object verbatim. Fields are
// public so the cleanup-history UI can render them without a
// separate API.
type DBObjectBackup struct {
Account string `json:"account"`
Schema string `json:"schema"`
Kind string `json:"kind"` // trigger | event | procedure | function
Name string `json:"name"`
CreateSQL string `json:"create_sql"`
DroppedAt time.Time `json:"dropped_at"`
DroppedBy string `json:"dropped_by"` // operator login or "csm" for daemon-driven
FindingID string `json:"finding_id,omitempty"`
RestoredAt time.Time `json:"restored_at,omitempty"`
}
// PutDBObjectBackup writes one backup record. Key shape:
// `<account>:<schema>:<kind>:<name>:<unix_nanos>` so multiple drops of
// the same object name (e.g., re-creates by an attacker) each get
// their own record.
func (db *DB) PutDBObjectBackup(b DBObjectBackup) error {
if b.Account == "" || b.Schema == "" || b.Kind == "" || b.Name == "" {
return fmt.Errorf("PutDBObjectBackup: account/schema/kind/name all required")
}
if b.DroppedAt.IsZero() {
b.DroppedAt = time.Now().UTC()
}
key := fmt.Sprintf("%s:%s:%s:%s:%d",
b.Account, b.Schema, b.Kind, b.Name, b.DroppedAt.UnixNano())
payload, err := json.Marshal(b)
if err != nil {
return fmt.Errorf("marshal backup: %w", err)
}
return db.bolt.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte("db_object_backups"))
if bucket == nil {
return fmt.Errorf("db_object_backups bucket missing (store not migrated)")
}
return bucket.Put([]byte(key), payload)
})
}
// ListDBObjectBackups returns every record for the given account, in
// insertion order. Used by the CLI's listing path and cleanup-history UI.
func (db *DB) ListDBObjectBackups(account string) ([]DBObjectBackup, error) {
var out []DBObjectBackup
prefix := []byte(account + ":")
err := db.bolt.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte("db_object_backups"))
if bucket == nil {
return nil
}
c := bucket.Cursor()
for k, v := c.Seek(prefix); k != nil && hasPrefix(k, prefix); k, v = c.Next() {
var b DBObjectBackup
if err := json.Unmarshal(v, &b); err != nil {
continue
}
out = append(out, b)
}
return nil
})
return out, err
}
// GetDBObjectBackupByKey fetches a single record by its exact bbolt
// key. Returns ok=false (not an error) when the key is missing,
// matching the lookup-then-act flow callers use.
func (db *DB) GetDBObjectBackupByKey(key string) (DBObjectBackup, bool, error) {
var rec DBObjectBackup
var found bool
err := db.bolt.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte("db_object_backups"))
if bucket == nil {
return nil
}
raw := bucket.Get([]byte(key))
if raw == nil {
return nil
}
if err := json.Unmarshal(raw, &rec); err != nil {
return err
}
found = true
return nil
})
return rec, found, err
}
// MarkDBObjectBackupRestored records that a backup has been replayed. The
// backup row stays in place for audit and future manual inspection, but the
// WebUI can stop offering repeat restore actions for that exact archive.
func (db *DB) MarkDBObjectBackupRestored(key string, restoredAt time.Time) error {
if restoredAt.IsZero() {
restoredAt = time.Now().UTC()
}
return db.bolt.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte("db_object_backups"))
if bucket == nil {
return fmt.Errorf("db_object_backups bucket missing (store not migrated)")
}
raw := bucket.Get([]byte(key))
if raw == nil {
return nil
}
var rec DBObjectBackup
if err := json.Unmarshal(raw, &rec); err != nil {
return err
}
rec.RestoredAt = restoredAt.UTC()
payload, err := json.Marshal(rec)
if err != nil {
return fmt.Errorf("marshal backup: %w", err)
}
return bucket.Put([]byte(key), payload)
})
}
// ListDBObjectBackupsAll returns every record in the bucket,
// regardless of account, in insertion order. Used by the webui
// cleanup-history listing where the operator browses across all
// accounts at once.
func (db *DB) ListDBObjectBackupsAll() ([]DBObjectBackup, []string, error) {
var records []DBObjectBackup
var keys []string
err := db.bolt.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte("db_object_backups"))
if bucket == nil {
return nil
}
return bucket.ForEach(func(k, v []byte) error {
var b DBObjectBackup
// Skip malformed rows silently; returning the unmarshal
// error from ForEach would abort the entire iteration,
// which is the wrong choice when one bad row shouldn't
// hide every other operator's history.
if json.Unmarshal(v, &b) == nil {
records = append(records, b)
keys = append(keys, string(k))
}
return nil
})
})
return records, keys, err
}
func hasPrefix(b, prefix []byte) bool {
if len(b) < len(prefix) {
return false
}
for i, c := range prefix {
if b[i] != c {
return false
}
}
return true
}
package store
import (
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/netip"
"strings"
"time"
"github.com/pidginhost/csm/internal/admission"
bolt "go.etcd.io/bbolt"
)
// dryRunBlocksBucket holds examples of responses a dry run would have run:
// the legacy firewall writer's or, while an admission owner runs on the
// firewall engine, only the owner's forecast examples. Its count is a count of the
// retained examples, not of addresses or predictions.
const dryRunBlocksBucket = "dry_run_blocks"
// Bounds of the forecast examples.
const (
maxDryRunBlocks = 20000
// maxDryRunBatch bounds one write: one turn's examples.
maxDryRunBatch = admission.MaxBatchMembers
// maxDryRunMaintenance bounds the records one maintenance step visits
// or deletes, cap trimming included.
maxDryRunMaintenance = 256
maxDryRunKeyBytes = 128
maxDryRunValueBytes = 1024
)
// dryRunKeyTime is the fixed-width UTC time that leads a forecast example's
// key, so keys sort by time within the format.
const dryRunKeyTime = "2006-01-02T15:04:05.000000000Z"
// ErrDryRunBlocksFull refuses a batch that cannot fit without evicting
// examples belonging to a session in that batch.
var ErrDryRunBlocksFull = errors.New("dry-run examples have no room")
var errNoStore = errors.New("the state database is not open")
// DryRunBlock is one example of a forecast pick that passed the modelled
// capacity checks: a preview, never a record of an applied block.
type DryRunBlock struct {
// At is when the forecast predicted it.
At time.Time
// IP is the canonical address and Check the registered check that
// asked for the response.
IP, Check string
Candidate admission.CandidateID
// Session is the forecast session, 32 lowercase hex digits.
Session string
Kind admission.Kind
// ExpiresAt is when the response would have ended.
ExpiresAt time.Time
}
type dryRunExampleRecord struct {
IP string `json:"ip"`
Reason string `json:"reason"`
TimeoutSec int64 `json:"timeout_sec"`
CandidateID string `json:"candidate_id"`
SessionID string `json:"session_id"`
Kind string `json:"kind"`
ExpiresAt string `json:"expires_at"`
}
// encode validates b and returns its key and value.
func (b DryRunBlock) encode() (key, value []byte, err error) {
at := b.At.UTC()
addr, parseErr := netip.ParseAddr(b.IP)
switch {
case b.At.IsZero() || at.Year() < 1 || at.Year() > 9999:
return nil, nil, errors.New("example time is out of range")
case parseErr != nil || addr.String() != b.IP || addr.Zone() != "" || addr.Is4In6():
return nil, nil, errors.New("example address is not canonical")
case !checkText(b.Check):
return nil, nil, errors.New("example reason is not a check name")
case !lowerHexText(b.Session, 32):
return nil, nil, errors.New("example session is malformed")
case !b.Kind.Valid():
return nil, nil, errors.New("example kind is unknown")
case !b.ExpiresAt.After(b.At):
return nil, nil, errors.New("example has no lifetime left")
case b.ExpiresAt.UTC().Year() > 9999:
return nil, nil, errors.New("example expiry is out of range")
}
if _, err = admission.ParseCandidateID(string(b.Candidate)); err != nil {
return nil, nil, err
}
key = []byte(at.Format(dryRunKeyTime) + ":" + b.IP + ":" + string(b.Candidate))
value, err = json.Marshal(dryRunExampleRecord{
IP: b.IP, Reason: b.Check, TimeoutSec: int64(b.ExpiresAt.Sub(b.At) / time.Second),
CandidateID: string(b.Candidate), SessionID: b.Session, Kind: b.Kind.String(),
ExpiresAt: b.ExpiresAt.UTC().Format(time.RFC3339Nano),
})
switch {
case err != nil:
return nil, nil, err
case len(key) > maxDryRunKeyBytes || len(value) > maxDryRunValueBytes:
return nil, nil, fmt.Errorf("example is %d+%d bytes, above its bound", len(key), len(value))
}
return key, value, nil
}
// checkText reports whether s has the form of a registered check name.
func checkText(s string) bool {
return s != "" && strings.Trim(s, "abcdefghijklmnopqrstuvwxyz0123456789_") == ""
}
func lowerHexText(s string, n int) bool {
_, err := hex.DecodeString(s)
return len(s) == n && err == nil && strings.ToLower(s) == s
}
// RecordDryRunBlocks writes one turn's forecast examples in one
// transaction, at most one batch. Every record is validated before anything
// is written, and a bad one refuses the batch. A full bucket makes room by
// evicting earlier sessions' first keys in key order, at most the batch's
// size. A bucket above its cap, or without enough evictable records,
// refuses the batch with ErrDryRunBlocksFull.
func (db *DB) RecordDryRunBlocks(blocks []DryRunBlock) error {
if db == nil || db.bolt == nil {
return errNoStore
}
if len(blocks) > maxDryRunBatch {
return fmt.Errorf("%d examples exceed one batch", len(blocks))
}
keys, values := make([][]byte, len(blocks)), make([][]byte, len(blocks))
keep := make(map[string]bool)
for i, b := range blocks {
var err error
if keys[i], values[i], err = b.encode(); err != nil {
return err
}
keep[b.Session] = true
}
return db.bolt.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists([]byte(dryRunBlocksBucket))
if err != nil {
return err
}
held := bucket.Stats().KeyN
if held > maxDryRunBlocks {
return ErrDryRunBlocksFull
}
fresh := map[string]bool{}
incoming := map[string]bool{}
for _, k := range keys {
incoming[string(k)] = true
if bucket.Get(k) == nil {
fresh[string(k)] = true
}
}
var evict [][]byte
c := bucket.Cursor()
needed := held + len(fresh) - maxDryRunBlocks
for k, v := c.First(); k != nil && len(evict) < needed; k, v = c.Next() {
if !incoming[string(k)] && !keep[dryRunSession(v)] {
evict = append(evict, append([]byte(nil), k...))
}
}
if len(evict) < needed {
return ErrDryRunBlocksFull
}
for _, k := range evict {
if err = bucket.Delete(k); err != nil {
return err
}
}
for i, k := range keys {
if err = bucket.Put(k, values[i]); err != nil {
return err
}
}
return nil
})
}
// DryRunMaintenance is one bounded step of the pass that keeps the examples
// within their bounds.
type DryRunMaintenance struct {
// Cursor is the last key the pass visited; nil starts a pass.
Cursor []byte
// Cutoff removes records whose key time is strictly before it. A key
// whose time does not parse stays for the cap and purges to remove.
Cutoff time.Time
// Purge removes every record of a session other than Keep, legacy and
// unreadable records included; with Keep empty it removes all.
Purge bool
Keep string
}
// DryRunMaintained is what one step did.
type DryRunMaintained struct {
// Next resumes the pass; Done is set once it has visited the last key.
Next []byte
Done bool
// Visited counts the records the step read, Removed those it deleted.
Visited, Removed int
}
// MaintainDryRunBlocks runs one step of the maintenance pass: at most 256
// records visited or deleted, resuming the pass after Cursor. Cap eviction
// and purging preserve Keep even if its timestamps precede the cutoff.
// Ordinary time pruning still applies outside a purge. A read decides
// whether anything is due, so a step that removes nothing writes nothing.
// A failed step changes nothing; resume from the same cursor.
func (db *DB) MaintainDryRunBlocks(step DryRunMaintenance) (DryRunMaintained, error) {
if db == nil || db.bolt == nil {
return DryRunMaintained{}, errNoStore
}
var res DryRunMaintained
var victims [][]byte
plan := func(tx *bolt.Tx) error { //nolint:unparam // bolt's View fixes the signature
res, victims = DryRunMaintained{Next: step.Cursor}, nil
bucket := tx.Bucket([]byte(dryRunBlocksBucket))
if bucket == nil {
res.Done, res.Next = true, nil
return nil
}
c := bucket.Cursor()
over := bucket.Stats().KeyN - maxDryRunBlocks
k, v := c.First()
if step.Cursor != nil {
if k, v = c.Seek(step.Cursor); k != nil && string(k) == string(step.Cursor) {
k, v = c.Next()
}
}
for ; k != nil && res.Visited < maxDryRunMaintenance; k, v = c.Next() {
res.Visited++
res.Next = append([]byte(nil), k...)
kept := step.Keep != "" && dryRunSession(v) == step.Keep
remove := !kept && (over > 0 || step.Purge)
if !step.Purge && dryRunKeyBefore(k, step.Cutoff) {
remove = true
}
if remove {
victims = append(victims, res.Next)
over--
}
}
if k == nil {
res.Done, res.Next = true, nil
}
return nil
}
if err := db.bolt.View(plan); err != nil || len(victims) == 0 {
return res, err
}
err := db.bolt.Update(func(tx *bolt.Tx) error {
if err := plan(tx); err != nil {
return err
}
bucket := tx.Bucket([]byte(dryRunBlocksBucket))
for _, k := range victims {
if err := bucket.Delete(k); err != nil {
return err
}
}
return nil
})
if err != nil {
return DryRunMaintained{}, err
}
res.Removed = len(victims)
return res, nil
}
// dryRunKeyBefore reports whether key's time, legacy or forecast form,
// parses and is strictly before cutoff.
func dryRunKeyBefore(key []byte, cutoff time.Time) bool {
at, ok := dryRunKeyAt(string(key))
return ok && at.Before(cutoff)
}
// dryRunKeyAt parses the UTC time that leads a key. Both writers end it
// with Z, and an IPv6 address after it contains colons, so the first "Z:"
// ends the time.
func dryRunKeyAt(key string) (time.Time, bool) {
end := strings.Index(key, "Z:")
if end < 0 {
return time.Time{}, false
}
at, err := time.Parse(time.RFC3339Nano, key[:end+1])
return at, err == nil
}
// dryRunSession is a record's forecast session, empty for a legacy or
// unreadable one.
func dryRunSession(value []byte) string {
var rec struct {
SessionID string `json:"session_id"`
}
if len(value) > maxDryRunValueBytes || json.Unmarshal(value, &rec) != nil {
return ""
}
return rec.SessionID
}
package store
import (
"encoding/json"
"time"
bolt "go.etcd.io/bbolt"
)
// GeoHistory tracks the countries from which a mailbox has logged in.
type GeoHistory struct {
Countries map[string]int64 `json:"countries"`
LoginCount int `json:"login_count"`
}
// SetGeoHistory stores geo login history for a mailbox in the email:geo bucket.
func (db *DB) SetGeoHistory(mailbox string, h GeoHistory) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("email:geo"))
val, err := json.Marshal(h)
if err != nil {
return err
}
return b.Put([]byte(mailbox), val)
})
}
// GetGeoHistory retrieves geo login history for a mailbox.
// Returns the entry and true if found, or a zero value and false if not.
func (db *DB) GetGeoHistory(mailbox string) (GeoHistory, bool) {
var h GeoHistory
var found bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("email:geo"))
v := b.Get([]byte(mailbox))
if v == nil {
return nil
}
if json.Unmarshal(v, &h) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
found = true
return nil
})
return h, found
}
// SetForwarderHash stores a forwarder config hash in the email:fwd bucket.
func (db *DB) SetForwarderHash(key, hash string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("email:fwd"))
return b.Put([]byte(key), []byte(hash))
})
}
// GetForwarderHash retrieves a forwarder config hash.
// Returns the hash and true if found, or an empty string and false if not.
func (db *DB) GetForwarderHash(key string) (string, bool) {
var hash string
var found bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("email:fwd"))
v := b.Get([]byte(key))
if v == nil {
return nil
}
hash = string(v)
found = true
return nil
})
return hash, found
}
// GetEmailPWLastRefresh reads the last email password-check refresh timestamp
// from the meta bucket. Returns the zero time if not set.
func (db *DB) GetEmailPWLastRefresh() time.Time {
var t time.Time
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
v := b.Get([]byte("email:pw_last_refresh"))
if v == nil {
return nil
}
parsed, err := time.Parse(time.RFC3339, string(v))
if err != nil {
return nil //nolint:nilerr // skip corrupt entry
}
t = parsed
return nil
})
return t
}
// SetEmailPWLastRefresh writes the email password-check refresh timestamp
// to the meta bucket.
func (db *DB) SetEmailPWLastRefresh(t time.Time) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
return b.Put([]byte("email:pw_last_refresh"), []byte(t.Format(time.RFC3339)))
})
}
// GetMetaString reads a string value from the meta bucket.
// Returns an empty string if the key is not found.
func (db *DB) GetMetaString(key string) string {
val, _ := db.ReadMetaString(key)
return val
}
// ReadMetaString distinguishes an absent key from a failed read for callers
// that must not treat unavailable metadata as permission to repeat an action.
func (db *DB) ReadMetaString(key string) (string, error) {
var val string
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
v := b.Get([]byte(key))
if v == nil {
return nil
}
val = string(v)
return nil
})
return val, err
}
// SetMetaString writes a string value to the meta bucket.
func (db *DB) SetMetaString(key, val string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
return b.Put([]byte(key), []byte(val))
})
}
package store
import (
"errors"
"time"
bolt "go.etcd.io/bbolt"
bolterrors "go.etcd.io/bbolt/errors"
)
const eximFrozenSeenBucket = "exim:frozen_seen"
// SaveEximFrozenSeen replaces the persisted frozen-message dedup snapshot:
// queue ID -> last time the daemon saw the message frozen. Persisting it lets
// a restart recognise messages it already reported, so the first queue run
// after a restart does not re-report every message that is still frozen.
// The bucket is rewritten whole, so IDs the daemon has dropped disappear.
func (db *DB) SaveEximFrozenSeen(seen map[string]time.Time) error {
encoded := make(map[string][]byte, len(seen))
for id, lastSeen := range seen {
if id == "" {
continue
}
val, err := lastSeen.UTC().MarshalText()
if err != nil {
return err
}
encoded[id] = val
}
return db.bolt.Update(func(tx *bolt.Tx) error {
if err := tx.DeleteBucket([]byte(eximFrozenSeenBucket)); err != nil && !errors.Is(err, bolterrors.ErrBucketNotFound) {
return err
}
b, err := tx.CreateBucket([]byte(eximFrozenSeenBucket))
if err != nil {
return err
}
for id, val := range encoded {
if err := b.Put([]byte(id), val); err != nil {
return err
}
}
return nil
})
}
// LoadEximFrozenSeen returns the persisted frozen-message dedup snapshot.
// Returns an empty map when nothing has been persisted yet.
func (db *DB) LoadEximFrozenSeen() (map[string]time.Time, error) {
out := make(map[string]time.Time)
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(eximFrozenSeenBucket))
if b == nil {
return nil
}
return b.ForEach(func(k, v []byte) error {
var lastSeen time.Time
if lastSeen.UnmarshalText(v) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
out[string(k)] = lastSeen
return nil
})
})
return out, err
}
package store
import (
"time"
bolt "go.etcd.io/bbolt"
)
// externalScriptsBucket remembers every external <script src> host seen in
// a site's wp_options, keyed "<site>|<option>|<host>" -> RFC3339 first-seen
// time, plus "<site>|" -> baseline completion time. The structural
// classifier only flags attacker markers, so a loader on an unremarkable
// HTTPS host is invisible to it; remembering hosts lets the scan report a
// host the first time it appears after the site's baseline, once.
const externalScriptsBucket = "db:external_scripts"
func externalScriptKey(site, option, host string) []byte {
return []byte(site + "|" + option + "|" + host)
}
func externalScriptBaselineKey(site string) []byte {
return []byte(site + "|")
}
// MarkExternalScriptSeen records host for (site, option) and reports whether
// it is new. Nothing is new until FinishExternalScriptBaseline has run for
// the site: the first scan records what is already there without reporting
// it, the way file_index treats its first pass.
func (db *DB) MarkExternalScriptSeen(site, option, host string, now time.Time) (bool, error) {
if db == nil || db.bolt == nil {
return false, nil
}
isNew := false
err := db.bolt.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(externalScriptsBucket))
if err != nil {
return err
}
key := externalScriptKey(site, option, host)
if b.Get(key) != nil {
return nil
}
isNew = b.Get(externalScriptBaselineKey(site)) != nil
return b.Put(key, []byte(now.UTC().Format(time.RFC3339)))
})
return isNew, err
}
// FinishExternalScriptBaseline marks the site's first scan as complete, so
// hosts recorded from now on count as new.
func (db *DB) FinishExternalScriptBaseline(site string, now time.Time) error {
if db == nil || db.bolt == nil {
return nil
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(externalScriptsBucket))
if err != nil {
return err
}
key := externalScriptBaselineKey(site)
if b.Get(key) != nil {
return nil
}
return b.Put(key, []byte(now.UTC().Format(time.RFC3339)))
})
}
package store
import (
"encoding/json"
"fmt"
"time"
"github.com/pidginhost/csm/internal/firewall"
bolt "go.etcd.io/bbolt"
)
// FWBlockedEntry represents an IP blocked by the firewall.
type FWBlockedEntry struct {
IP string `json:"ip"`
Reason string `json:"reason"`
Source string `json:"source,omitempty"`
BlockedAt time.Time `json:"blocked_at"`
ExpiresAt time.Time `json:"expires_at"` // zero = permanent
}
// FWAllowedEntry represents an IP explicitly allowed through the firewall.
type FWAllowedEntry struct {
IP string `json:"ip"`
Reason string `json:"reason"`
Source string `json:"source,omitempty"`
Port int `json:"port"` // 0 = all ports
ExpiresAt time.Time `json:"expires_at"` // zero = permanent
}
// FWSubnetEntry represents a subnet added to the firewall.
type FWSubnetEntry struct {
CIDR string `json:"cidr"`
Reason string `json:"reason"`
Source string `json:"source,omitempty"`
AddedAt time.Time `json:"added_at"`
}
// FWPortAllowEntry represents a per-IP port allow rule.
type FWPortAllowEntry struct {
Key string `json:"key"` // IP:port/proto
IP string `json:"ip"`
Port int `json:"port"`
Proto string `json:"proto"`
Reason string `json:"reason"`
Source string `json:"source,omitempty"`
}
// FirewallState holds the full state across all 4 firewall buckets.
type FirewallState struct {
Blocked []FWBlockedEntry
Allowed []FWAllowedEntry
Subnets []FWSubnetEntry
PortAllowed []FWPortAllowEntry
}
// portAllowKey returns the composite key "IP:port/proto".
func portAllowKey(ip string, port int, proto string) string {
return fmt.Sprintf("%s:%d/%s", ip, port, proto)
}
// BlockIP adds an IP to the fw:blocked bucket.
func (db *DB) BlockIP(ip, reason string, expiresAt time.Time) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("fw:blocked"))
entry := FWBlockedEntry{
IP: ip,
Reason: reason,
Source: firewall.InferProvenance("block", reason),
BlockedAt: time.Now(),
ExpiresAt: expiresAt,
}
val, err := json.Marshal(entry)
if err != nil {
return err
}
return b.Put([]byte(ip), val)
})
}
// UnblockIP removes an IP from the fw:blocked bucket.
func (db *DB) UnblockIP(ip string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("fw:blocked"))
return b.Delete([]byte(ip))
})
}
// GetBlockedIP looks up a blocked IP. Returns false if not found or expired.
func (db *DB) GetBlockedIP(ip string) (FWBlockedEntry, bool) {
var entry FWBlockedEntry
var found bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("fw:blocked"))
v := b.Get([]byte(ip))
if v == nil {
return nil
}
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
// Filter expired entries (zero ExpiresAt = permanent).
if !entry.ExpiresAt.IsZero() && !entry.ExpiresAt.After(time.Now()) {
return nil
}
found = true
return nil
})
return entry, found
}
// AllowIP adds an IP to the fw:allowed bucket.
func (db *DB) AllowIP(ip, reason string, expiresAt time.Time) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("fw:allowed"))
entry := FWAllowedEntry{
IP: ip,
Reason: reason,
Source: firewall.InferProvenance("allow", reason),
ExpiresAt: expiresAt,
}
val, err := json.Marshal(entry)
if err != nil {
return err
}
return b.Put([]byte(ip), val)
})
}
// RemoveAllow removes an IP from the fw:allowed bucket.
func (db *DB) RemoveAllow(ip string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("fw:allowed"))
return b.Delete([]byte(ip))
})
}
// AddSubnet adds a CIDR to the fw:subnets bucket.
func (db *DB) AddSubnet(cidr, reason string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("fw:subnets"))
entry := FWSubnetEntry{
CIDR: cidr,
Reason: reason,
Source: firewall.InferProvenance("block_subnet", reason),
AddedAt: time.Now(),
}
val, err := json.Marshal(entry)
if err != nil {
return err
}
return b.Put([]byte(cidr), val)
})
}
// RemoveSubnet removes a CIDR from the fw:subnets bucket.
func (db *DB) RemoveSubnet(cidr string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("fw:subnets"))
return b.Delete([]byte(cidr))
})
}
// AddPortAllow adds a per-IP port allow rule to the fw:port_allowed bucket.
func (db *DB) AddPortAllow(ip string, port int, proto, reason string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("fw:port_allowed"))
key := portAllowKey(ip, port, proto)
entry := FWPortAllowEntry{
Key: key,
IP: ip,
Port: port,
Proto: proto,
Reason: reason,
Source: firewall.InferProvenance("allow_port", reason),
}
val, err := json.Marshal(entry)
if err != nil {
return err
}
return b.Put([]byte(key), val)
})
}
// RemovePortAllow removes a per-IP port allow rule from the fw:port_allowed bucket.
func (db *DB) RemovePortAllow(ip string, port int, proto string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("fw:port_allowed"))
key := portAllowKey(ip, port, proto)
return b.Delete([]byte(key))
})
}
// ListPortAllows returns all entries in the fw:port_allowed bucket.
func (db *DB) ListPortAllows() []FWPortAllowEntry {
var entries []FWPortAllowEntry
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("fw:port_allowed"))
return b.ForEach(func(k, v []byte) error {
var entry FWPortAllowEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
entries = append(entries, entry)
return nil
})
})
return entries
}
// LoadFirewallState reads all 4 firewall buckets and assembles a FirewallState.
// Expired blocked and allowed entries are filtered out.
func (db *DB) LoadFirewallState() FirewallState {
var state FirewallState
now := time.Now()
_ = db.bolt.View(func(tx *bolt.Tx) error {
// fw:blocked - filter expired
blocked := tx.Bucket([]byte("fw:blocked"))
_ = blocked.ForEach(func(k, v []byte) error {
var entry FWBlockedEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
if !entry.ExpiresAt.IsZero() && !entry.ExpiresAt.After(now) {
return nil // expired
}
state.Blocked = append(state.Blocked, entry)
return nil
})
// fw:allowed - filter expired
allowed := tx.Bucket([]byte("fw:allowed"))
_ = allowed.ForEach(func(k, v []byte) error {
var entry FWAllowedEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
if !entry.ExpiresAt.IsZero() && !entry.ExpiresAt.After(now) {
return nil // expired
}
state.Allowed = append(state.Allowed, entry)
return nil
})
// fw:subnets
subnets := tx.Bucket([]byte("fw:subnets"))
_ = subnets.ForEach(func(k, v []byte) error {
var entry FWSubnetEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
state.Subnets = append(state.Subnets, entry)
return nil
})
// fw:port_allowed
portAllowed := tx.Bucket([]byte("fw:port_allowed"))
_ = portAllowed.ForEach(func(k, v []byte) error {
var entry FWPortAllowEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
state.PortAllowed = append(state.PortAllowed, entry)
return nil
})
return nil
})
return state
}
package store
import (
"bytes"
"encoding/binary"
"fmt"
"time"
"github.com/pidginhost/csm/internal/firewall"
bolt "go.etcd.io/bbolt"
)
// Terminal actions are indexed by outcome time so retention reads an ordered
// list without decoding the evidence each record carries.
const firewallActionHistoryBucket = "fw:action_history"
type firewallActionRetentionCaps struct {
// Actions and Bytes bound the retained outcome history. Bytes matters
// because one record keeps two complete states plus kernel evidence, so a
// host with a large blocked set writes far more per action than a small one.
Actions int
Bytes uint64
// BudgetWindows bounds the hourly scan counters. Only the current window is
// ever read, and the inventory is validated on every admission.
BudgetWindows int
}
// Retention sweeps are opt-in, so on a default install these caps are the only
// bound on journal growth. They apply on the write path, not on a timer.
var firewallActionRetention = firewallActionRetentionCaps{Actions: 1000, Bytes: 64 << 20, BudgetWindows: 48}
func firewallActionHistoryKey(at time.Time, id string) []byte {
nanoseconds := at.UnixNano()
if nanoseconds < 0 {
nanoseconds = 0
}
return []byte(fmt.Sprintf("%020d\x00%s", nanoseconds, id))
}
func firewallActionHistoryID(key []byte) (string, bool) {
separator := bytes.IndexByte(key, 0)
if separator < 0 || separator+1 >= len(key) {
return "", false
}
return string(key[separator+1:]), true
}
type firewallActionHistoryEntry struct {
key []byte
id string
size uint64
}
// readFirewallActionHistory returns retained outcomes oldest first. A history
// key without its action record is corruption: both are written in one
// transaction, so recovery must not silently accept a half-deleted record.
func readFirewallActionHistory(tx *bolt.Tx) ([]firewallActionHistoryEntry, uint64, error) {
b := tx.Bucket([]byte(firewallActionHistoryBucket))
if b == nil {
return nil, 0, nil
}
actions := tx.Bucket([]byte(firewallActionsBucket))
var entries []firewallActionHistoryEntry
var total uint64
cursor := b.Cursor()
for key, raw := cursor.First(); key != nil; key, raw = cursor.Next() {
id, ok := firewallActionHistoryID(key)
if !ok || len(raw) != 8 {
return nil, 0, fmt.Errorf("%w: firewall action history entry", firewall.ErrStateCorrupt)
}
if actions == nil || actions.Get([]byte(id)) == nil {
return nil, 0, fmt.Errorf("%w: firewall action history without record", firewall.ErrStateCorrupt)
}
// The index stores the action row size. Audit rows are counted live
// so existing indexes also include every retained evidence copy.
size := binary.BigEndian.Uint64(raw) + firewallActionAuditSize(tx, id)
total += size
entries = append(entries, firewallActionHistoryEntry{key: bytes.Clone(key), id: id, size: size})
}
return entries, total, nil
}
// Older journals predate the outcome index. Build it atomically before the
// first retention operation, including undelivered outcomes for later pruning.
func initializeFirewallActionHistory(tx *bolt.Tx) error {
if tx.Bucket([]byte(firewallActionHistoryBucket)) != nil {
return nil
}
actions := tx.Bucket([]byte(firewallActionsBucket))
if actions == nil {
return nil
}
if _, err := tx.CreateBucketIfNotExists([]byte(firewallActionHistoryBucket)); err != nil {
return err
}
return actions.ForEach(func(key, raw []byte) error {
a, err := readFirewallAction(tx, string(key))
if err != nil {
return err
}
if firewallActionOutstanding(a) {
return nil
}
return recordFirewallActionHistory(tx, a, len(raw))
})
}
// Audit keys have a fixed-width version suffix. A prefix alone also matches
// other valid request IDs containing NUL, so match the complete key length.
func firewallActionAuditSize(tx *bolt.Tx, id string) uint64 {
var size uint64
if b := tx.Bucket([]byte(firewallAuditBucket)); b != nil {
prefix := append([]byte(id), 0)
cursor := b.Cursor()
for key, raw := cursor.Seek(prefix); key != nil && bytes.HasPrefix(key, prefix); key, raw = cursor.Next() {
if len(key) == len(prefix)+20 {
size += uint64(len(raw))
}
}
}
return size
}
func recordFirewallActionHistory(tx *bolt.Tx, a firewall.FirewallAction, size int) error {
if err := initializeFirewallActionHistory(tx); err != nil {
return err
}
b := tx.Bucket([]byte(firewallActionHistoryBucket))
var encoded [8]byte
binary.BigEndian.PutUint64(encoded[:], uint64(size)) // #nosec G115 -- a stored record length is never negative.
return b.Put(firewallActionHistoryKey(a.UpdatedAt, a.Request.ID), encoded[:])
}
// updateFirewallActionHistorySize keeps byte accounting honest after an
// acknowledgement rewrites a retained record.
func updateFirewallActionHistorySize(tx *bolt.Tx, a firewall.FirewallAction, size int) error {
if err := initializeFirewallActionHistory(tx); err != nil {
return err
}
b := tx.Bucket([]byte(firewallActionHistoryBucket))
if b == nil {
return nil
}
key := firewallActionHistoryKey(a.UpdatedAt, a.Request.ID)
if b.Get(key) == nil {
return nil
}
var encoded [8]byte
binary.BigEndian.PutUint64(encoded[:], uint64(size)) // #nosec G115 -- a stored record length is never negative.
return b.Put(key, encoded[:])
}
func undeliveredFirewallAuditIDs(index firewallJournalIndex) map[string]bool {
undelivered := make(map[string]bool, len(index.Audit))
for _, ref := range index.Audit {
undelivered[ref.ID] = true
}
return undelivered
}
// deleteFirewallAction removes one retained outcome with every event that
// refers to it. Undelivered outcomes are never passed here.
func deleteFirewallAction(tx *bolt.Tx, entry firewallActionHistoryEntry) error {
if err := releaseFirewallBatchMembers(tx, entry.id); err != nil {
return err
}
if b := tx.Bucket([]byte(firewallAuditBucket)); b != nil {
prefix := append([]byte(entry.id), 0)
// Collect first: acknowledgement rewrites an audit leaf in this
// transaction, and a bbolt cursor that deletes and then steps with
// Next skips keys in a bucket already written by the transaction.
var keys [][]byte
cursor := b.Cursor()
for key, _ := cursor.Seek(prefix); key != nil && bytes.HasPrefix(key, prefix); key, _ = cursor.Next() {
if len(key) == len(prefix)+20 {
keys = append(keys, append([]byte(nil), key...))
}
}
for _, key := range keys {
if err := b.Delete(key); err != nil {
return err
}
}
}
if b := tx.Bucket([]byte(firewallActionsBucket)); b != nil {
if err := b.Delete([]byte(entry.id)); err != nil {
return err
}
}
return tx.Bucket([]byte(firewallActionHistoryBucket)).Delete(entry.key)
}
// pruneFirewallActionHistory enforces the retained-outcome caps inside the
// transaction that recorded the newest outcome. The newest record and any
// record with undelivered audit are kept whatever the caps say.
func pruneFirewallActionHistory(tx *bolt.Tx, index firewallJournalIndex) error {
if err := initializeFirewallActionHistory(tx); err != nil {
return err
}
entries, total, err := readFirewallActionHistory(tx)
if err != nil {
return err
}
undelivered := undeliveredFirewallAuditIDs(index)
count := len(entries)
for _, entry := range entries[:max(len(entries)-1, 0)] {
if count <= firewallActionRetention.Actions && total <= firewallActionRetention.Bytes {
return nil
}
if undelivered[entry.id] {
continue
}
if err := deleteFirewallAction(tx, entry); err != nil {
return err
}
count--
total -= entry.size
}
return nil
}
// SweepFirewallActionsOlderThan deletes settled outcomes whose audit has been
// delivered and whose result is older than cutoff. Pending actions and
// undelivered outcomes are retained regardless of age: recovery still needs
// them. Deleting an outcome ends the undo window for that action.
func (db *DB) SweepFirewallActionsOlderThan(cutoff time.Time) (int, error) {
var deleted int
err := boltUpdate(db.bolt, func(tx *bolt.Tx) error {
deleted = 0
index, err := readFirewallJournalIndex(tx)
if err != nil {
return err
}
if initErr := initializeFirewallActionHistory(tx); initErr != nil {
return initErr
}
entries, _, err := readFirewallActionHistory(tx)
if err != nil {
return err
}
undelivered := undeliveredFirewallAuditIDs(index)
boundary := firewallActionHistoryKey(cutoff, "")
for _, entry := range entries {
if bytes.Compare(entry.key, boundary) >= 0 {
break
}
if undelivered[entry.id] {
continue
}
if err := deleteFirewallAction(tx, entry); err != nil {
return err
}
deleted++
}
return sweepFirewallScanBudget(tx, cutoff)
})
if err != nil {
return 0, err
}
return deleted, nil
}
// sweepFirewallScanBudget drops hourly counters the admission path can no
// longer charge. The inventory and its rows are updated together so a missing
// counter keeps meaning corruption.
func sweepFirewallScanBudget(tx *bolt.Tx, cutoff time.Time) error {
inventory, err := readFirewallBudgetInventory(tx)
if err != nil {
return err
}
boundary := cutoff.UTC().Format(firewallScanWindowLayout)
keep := inventory.Windows[:0:0]
for _, window := range inventory.Windows {
if window >= boundary {
keep = append(keep, window)
continue
}
if err := deleteFirewallScanBudgetWindow(tx, window); err != nil {
return err
}
inventory.PrunedThrough = max(inventory.PrunedThrough, window)
}
if len(keep) == len(inventory.Windows) {
return nil
}
inventory.Windows = keep
return writeFirewallBudgetInventory(tx, inventory)
}
func deleteFirewallScanBudgetWindow(tx *bolt.Tx, window string) error {
b := tx.Bucket([]byte(firewallBudgetBucket))
if b == nil {
return nil
}
return b.Delete([]byte(window))
}
// pruneFirewallScanBudgetWindows keeps the newest windows only. The current
// window may move backwards after a clock correction. Keep a durable boundary
// so admission refuses discarded windows instead of resetting their charges.
func pruneFirewallScanBudgetWindows(tx *bolt.Tx, inventory firewallBudgetInventory) (firewallBudgetInventory, error) {
if len(inventory.Windows) <= firewallActionRetention.BudgetWindows {
return inventory, nil
}
excess := len(inventory.Windows) - firewallActionRetention.BudgetWindows
for _, window := range inventory.Windows[:excess] {
if err := deleteFirewallScanBudgetWindow(tx, window); err != nil {
return inventory, err
}
}
inventory.PrunedThrough = max(inventory.PrunedThrough, inventory.Windows[excess-1])
inventory.Windows = append(inventory.Windows[:0:0], inventory.Windows[excess:]...)
return inventory, writeFirewallBudgetInventory(tx, inventory)
}
package store
import (
"bytes"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"math"
"net/netip"
"slices"
"strings"
"time"
"unicode/utf8"
"github.com/pidginhost/csm/internal/firewall"
bolt "go.etcd.io/bbolt"
)
const firewallActionsBucket = "fw:actions"
const firewallBudgetBucket = "fw:scan_budget"
const firewallAuditBucket = "fw:action_audit"
const firewallActionIndexBucket = "fw:action_index"
const firewallBudgetIndexBucket = "fw:budget_index"
// A batch of several members indexes each member ID to its batch, and the
// batch to its members so retention can release them without decoding the
// record. A one-member action needs neither: its only ID is its key.
const firewallMemberIndexBucket = "fw:member_index"
const firewallBatchMembersBucket = "fw:batch_members"
const firewallScanWindowLayout = "2006-01-02T15"
// An action with a manifest is stored as version 2. One without stays version
// 1, the only format earlier builds read, so its encoding never changes.
const firewallManifestVersion = 2
var _ firewall.ActionStore = (*DB)(nil)
// ErrFirewallActionMissing distinguishes an unknown request from a damaged
// journal. Neither is permission to execute an unrecorded mutation.
var ErrFirewallActionMissing = firewall.ErrActionMissing
func firewallActionPending(phase string) bool {
return phase == "planned" || phase == "executing" || phase == "applied" || phase == "unknown"
}
// firewallActionOutstanding reports whether a still holds the pending
// interlock: a pending phase that recovery has not retired.
func firewallActionOutstanding(a firewall.FirewallAction) bool {
return firewallActionPending(a.Phase) && !a.Retired
}
func validateFirewallAdmission(a firewall.FirewallAction) error {
if a.Request.ID == "" || len(a.Request.ID) > 128 || a.Request.Operation == "" || a.Request.Actor == "" || a.CreatedAt.IsZero() || a.Revision == 0 || a.Retired && a.Phase != "unknown" {
return errors.New("invalid firewall action admission")
}
for _, value := range []string{a.Request.ID, a.Request.Operation, a.Request.Target, a.Request.Reason, a.Request.Actor, a.Request.ActorDetail, a.Request.Source, a.Request.FindingID, a.Request.IncidentID, a.Request.UndoOf, a.Detail} {
if !utf8.ValidString(value) {
return errors.New("firewall action contains invalid Unicode")
}
}
for _, at := range []time.Time{a.CreatedAt, a.UpdatedAt} {
_, offset := at.Zone()
if offset%60 != 0 {
return errors.New("firewall action time cannot round-trip")
}
}
if err := validateFirewallBudget(a.Request.Source, a.Budget); err != nil {
return err
}
return validateFirewallManifest(a)
}
func validateFirewallBudget(source string, budget *firewall.ScanAdmission) error {
if budget == nil {
return nil
}
if source != "scan" || budget.Limit <= 0 {
return errors.New("invalid firewall scan admission")
}
if _, err := time.Parse(firewallScanWindowLayout, budget.Window); err != nil {
return errors.New("invalid firewall scan window")
}
return nil
}
var errInvalidFirewallManifest = errors.New("invalid firewall batch manifest")
// A manifest names each member once. One member repeats its header and
// charge. A larger batch holds automatic temporary blocks under a header that
// carries no target, link or charge of its own. Only a block removes rows.
func validateFirewallManifest(a firewall.FirewallAction) error {
if a.Members == nil {
return nil
}
if len(a.Members) == 0 || len(a.Members) > firewall.MaxActionMembers {
return errInvalidFirewallManifest
}
header := a.Request
if len(a.Members) == 1 {
if a.Members[0].Request != header || !sameFirewallBudget(a.Members[0].Budget, a.Budget) {
return errInvalidFirewallManifest
}
} else if header.Operation != "block" || !header.Automatic || header.Target != "" || header.Reason != "" || header.FindingID != "" || header.IncidentID != "" || header.UndoOf != "" || header.TTL != 0 || a.Budget != nil {
return errInvalidFirewallManifest
}
ids, targets, victims := map[string]bool{header.ID: true}, map[string]bool{}, map[string]bool{}
limits := map[string]int{}
for i, m := range a.Members {
r := m.Request
if i > 0 || len(a.Members) > 1 {
if r.ID == "" || len(r.ID) > 128 || ids[r.ID] || r.Operation != "block" || !r.Automatic || r.TTL <= 0 || !canonicalFirewallAddress(r.Target) || targets[r.Target] || r.UndoOf != "" || r.Actor != header.Actor || r.ActorDetail != header.ActorDetail || r.Source != header.Source {
return errInvalidFirewallManifest
}
for _, value := range []string{r.ID, r.Target, r.Reason, r.FindingID, r.IncidentID} {
if !utf8.ValidString(value) {
return errInvalidFirewallManifest
}
}
}
// A one-member target may be any spelling; victims compare by address.
ids[r.ID], targets[canonicalFirewallTarget(r.Target)] = true, true
if err := validateFirewallBudget(r.Source, m.Budget); err != nil {
return err
}
if m.Budget != nil {
if limit, seen := limits[m.Budget.Window]; seen && limit != m.Budget.Limit {
return errInvalidFirewallManifest
}
limits[m.Budget.Window] = m.Budget.Limit
}
if len(m.Victims) > 0 && r.Operation != "block" {
return errInvalidFirewallManifest
}
for _, v := range m.Victims {
if !canonicalFirewallAddress(v.IP) || v.ExpiresAt.IsZero() || victims[v.IP] || !validFirewallRowEncoding(v) {
return errInvalidFirewallManifest
}
victims[v.IP] = true
}
}
for ip := range victims {
if targets[ip] {
return errInvalidFirewallManifest
}
}
if len(a.Members) > 1 {
return validateFirewallBatchStates(a)
}
return validateFirewallVictims(a)
}
// canonicalFirewallAddress accepts one address in its only spelling, so no
// two members or victims can name the same address differently.
func canonicalFirewallAddress(ip string) bool {
addr, err := netip.ParseAddr(ip)
return err == nil && addr.Zone() == "" && addr.Unmap().String() == ip
}
func sameFirewallBudget(a, b *firewall.ScanAdmission) bool {
return a == b || a != nil && b != nil && *a == *b
}
// Journal envelopes detect valid-JSON corruption before recovery or typed undo
// interprets historical evidence. Checksums cover the exact stored payload.
type firewallJournalEnvelope struct {
Version uint64 `json:"version"`
Payload json.RawMessage `json:"payload"`
SHA256 string `json:"sha256"`
}
func encodeFirewallJournal(value any) ([]byte, error) {
return encodeFirewallJournalVersion(value, 1)
}
func encodeFirewallJournalVersion(value any, version uint64) ([]byte, error) {
payload, err := json.Marshal(value)
if err != nil {
return nil, err
}
digest := sha256.Sum256(payload)
return json.Marshal(firewallJournalEnvelope{Version: version, Payload: payload, SHA256: fmt.Sprintf("%x", digest)})
}
func encodeFirewallActionRecord(a firewall.FirewallAction) ([]byte, error) {
if a.Members == nil {
return encodeFirewallJournal(a)
}
return encodeFirewallJournalVersion(a, firewallManifestVersion)
}
func decodeFirewallJournal(raw []byte) ([]byte, error) {
payload, _, err := decodeFirewallJournalVersion(raw, 1)
if err != nil {
return nil, err
}
if !validFirewallJSONText(raw) {
return nil, firewall.ErrStateCorrupt
}
return payload, nil
}
// Only the version belongs to every schema. Newer records may change any
// other field, including its type or text encoding.
func decodeFirewallVersion(raw []byte, newest uint64) (uint64, error) {
var header struct {
Version uint64 `json:"version"`
}
if json.Unmarshal(raw, &header) != nil || header.Version == 0 {
return 0, firewall.ErrStateCorrupt
}
if header.Version > newest {
return 0, firewall.ErrStateUnsupported
}
return header.Version, nil
}
// Text validation belongs to the reader, after any nested version checks:
// a supported audit envelope can contain an unsupported action envelope.
func decodeFirewallJournalVersion(raw []byte, newest uint64) ([]byte, uint64, error) {
version, err := decodeFirewallVersion(raw, newest)
if err != nil {
return nil, 0, err
}
var envelope firewallJournalEnvelope
if json.Unmarshal(raw, &envelope) != nil || !bytes.HasPrefix(bytes.TrimSpace(envelope.Payload), []byte("{")) {
return nil, 0, firewall.ErrStateCorrupt
}
digest := sha256.Sum256(envelope.Payload)
if envelope.SHA256 != fmt.Sprintf("%x", digest) {
return nil, 0, firewall.ErrStateCorrupt
}
return envelope.Payload, version, nil
}
func decodeFirewallAction(raw []byte) (firewall.FirewallAction, error) {
var a firewall.FirewallAction
if len(raw) == 0 {
return a, ErrFirewallActionMissing
}
payload, version, err := decodeFirewallJournalVersion(raw, firewallManifestVersion)
if err != nil {
return firewall.FirewallAction{}, err
}
if !validFirewallJSONText(raw) {
return firewall.FirewallAction{}, firewall.ErrStateCorrupt
}
raw = payload
var evidence struct {
Before json.RawMessage `json:"before"`
After json.RawMessage `json:"after"`
Members json.RawMessage `json:"members"`
}
if json.Unmarshal(raw, &evidence) != nil || !bytes.HasPrefix(bytes.TrimSpace(evidence.Before), []byte("{")) || !bytes.HasPrefix(bytes.TrimSpace(evidence.After), []byte("{")) {
return firewall.FirewallAction{}, firewall.ErrStateCorrupt
}
if json.Unmarshal(raw, &a) != nil || validateFirewallAdmission(a) != nil || a.UpdatedAt.IsZero() || a.AuditVersion == 0 || a.AuditAck > a.AuditVersion {
return firewall.FirewallAction{}, firewall.ErrStateCorrupt
}
// Version 1 never carries a manifest and version 2 always does, so a
// record is never read as a different batch than the one written.
if (version == 1) != (evidence.Members == nil) || version == firewallManifestVersion && len(a.Members) == 0 {
return firewall.FirewallAction{}, firewall.ErrStateCorrupt
}
if !firewallActionPending(a.Phase) && a.Phase != "verified" && a.Phase != "failed" {
return firewall.FirewallAction{}, firewall.ErrStateCorrupt
}
if _, _, err := encodeFirewallSnapshot(a.Before); err != nil {
return firewall.FirewallAction{}, fmt.Errorf("%w: action before state: %v", firewall.ErrStateCorrupt, err)
}
if _, _, err := encodeFirewallSnapshot(a.After); err != nil {
return firewall.FirewallAction{}, fmt.Errorf("%w: action after state: %v", firewall.ErrStateCorrupt, err)
}
return a, nil
}
func readFirewallAction(tx *bolt.Tx, id string) (firewall.FirewallAction, error) {
b := tx.Bucket([]byte(firewallActionsBucket))
if b == nil {
return firewall.FirewallAction{}, ErrFirewallActionMissing
}
a, err := decodeFirewallAction(b.Get([]byte(id)))
if err == nil && a.Request.ID != id {
return firewall.FirewallAction{}, firewall.ErrStateCorrupt
}
return a, err
}
func writeFirewallAction(tx *bolt.Tx, a firewall.FirewallAction) (int, error) {
raw, err := encodeFirewallActionRecord(a)
if err != nil {
return 0, err
}
b, err := tx.CreateBucketIfNotExists([]byte(firewallActionsBucket))
if err != nil {
return 0, err
}
return len(raw), b.Put([]byte(a.Request.ID), raw)
}
// Only outstanding work is indexed. Retained history is validated when read for
// inspection or undo, so admission does not decode every historical snapshot.
type firewallAuditReference struct {
ID string `json:"id"`
Version uint64 `json:"version"`
}
type firewallJournalIndex struct {
Initialized bool `json:"initialized"`
PendingID string `json:"pending_id"`
Audit []firewallAuditReference `json:"audit"`
}
func readFirewallJournalIndex(tx *bolt.Tx) (firewallJournalIndex, error) {
var index firewallJournalIndex
b := tx.Bucket([]byte(firewallActionIndexBucket))
if b == nil {
for _, name := range []string{firewallActionsBucket, firewallAuditBucket, firewallBudgetBucket, firewallBudgetIndexBucket, firewallActionHistoryBucket, firewallMemberIndexBucket, firewallBatchMembersBucket} {
if tx.Bucket([]byte(name)) != nil {
return index, firewall.ErrStateCorrupt
}
}
return index, nil
}
payload, err := decodeFirewallJournal(b.Get([]byte("index")))
if err != nil {
return index, err
}
var required struct {
PendingID *string `json:"pending_id"`
Audit json.RawMessage `json:"audit"`
}
if json.Unmarshal(payload, &required) != nil || required.PendingID == nil || !bytes.HasPrefix(bytes.TrimSpace(required.Audit), []byte("[")) {
return firewallJournalIndex{}, firewall.ErrStateCorrupt
}
if json.Unmarshal(payload, &index) != nil || !index.Initialized || tx.Bucket([]byte(firewallActionsBucket)) == nil {
return firewallJournalIndex{}, firewall.ErrStateCorrupt
}
seen := make(map[firewallAuditReference]bool, len(index.Audit))
for _, ref := range index.Audit {
if ref.ID == "" || ref.Version == 0 || seen[ref] {
return firewallJournalIndex{}, firewall.ErrStateCorrupt
}
seen[ref] = true
}
return index, nil
}
func writeFirewallJournalIndex(tx *bolt.Tx, index firewallJournalIndex) error {
index.Initialized = true
if index.Audit == nil {
index.Audit = []firewallAuditReference{}
}
raw, err := encodeFirewallJournal(index)
if err != nil {
return err
}
b, err := tx.CreateBucketIfNotExists([]byte(firewallActionIndexBucket))
if err != nil {
return err
}
return b.Put([]byte("index"), raw)
}
func indexedPendingFirewallAction(tx *bolt.Tx, index firewallJournalIndex) (firewall.FirewallAction, error) {
if index.PendingID == "" {
return firewall.FirewallAction{}, nil
}
a, err := readFirewallAction(tx, index.PendingID)
if errors.Is(err, firewall.ErrStateUnsupported) {
return firewall.FirewallAction{}, err
}
if err != nil || !firewallActionOutstanding(a) {
return firewall.FirewallAction{}, firewall.ErrStateCorrupt
}
return a, nil
}
func refusePendingFirewallActions(tx *bolt.Tx) error {
index, err := readFirewallJournalIndex(tx)
if err != nil {
return err
}
a, err := indexedPendingFirewallAction(tx, index)
if err != nil {
return err
}
if a.Request.ID != "" {
return fmt.Errorf("%w: action %s requires recovery", firewall.ErrStateConflict, a.Request.ID)
}
return nil
}
type firewallScanBudget struct {
Window string `json:"window"`
Count *int `json:"count"`
}
// Keep the inventory separate from pending action metadata: recovery need not
// decode previously used budget windows. Missing charged counters are corruption.
type firewallBudgetInventory struct {
Initialized bool `json:"initialized"`
Windows []string `json:"windows"`
// PrunedThrough prevents discarded charges from reopening after a clock
// correction. Omitted in journals written before budget retention.
PrunedThrough string `json:"pruned_through,omitempty"`
}
func readFirewallBudgetInventory(tx *bolt.Tx) (firewallBudgetInventory, error) {
var inventory firewallBudgetInventory
b := tx.Bucket([]byte(firewallBudgetIndexBucket))
if b == nil {
for _, name := range []string{firewallActionsBucket, firewallAuditBucket, firewallBudgetBucket, firewallActionIndexBucket, firewallActionHistoryBucket, firewallMemberIndexBucket, firewallBatchMembersBucket} {
if tx.Bucket([]byte(name)) != nil {
return inventory, firewall.ErrStateCorrupt
}
}
return inventory, nil
}
payload, err := decodeFirewallJournal(b.Get([]byte("index")))
if err != nil {
return inventory, err
}
if json.Unmarshal(payload, &inventory) != nil || !inventory.Initialized || inventory.Windows == nil {
return firewallBudgetInventory{}, firewall.ErrStateCorrupt
}
if inventory.PrunedThrough != "" {
if _, err := time.Parse(firewallScanWindowLayout, inventory.PrunedThrough); err != nil {
return firewallBudgetInventory{}, firewall.ErrStateCorrupt
}
}
for i, window := range inventory.Windows {
if window <= inventory.PrunedThrough {
return firewallBudgetInventory{}, firewall.ErrStateCorrupt
}
if _, err := time.Parse(firewallScanWindowLayout, window); err != nil {
return firewallBudgetInventory{}, firewall.ErrStateCorrupt
}
if i > 0 && inventory.Windows[i-1] >= window {
return firewallBudgetInventory{}, firewall.ErrStateCorrupt
}
}
return inventory, nil
}
func writeFirewallBudgetInventory(tx *bolt.Tx, inventory firewallBudgetInventory) error {
inventory.Initialized = true
if inventory.Windows == nil {
inventory.Windows = []string{}
}
raw, err := encodeFirewallJournal(inventory)
if err != nil {
return err
}
b, err := tx.CreateBucketIfNotExists([]byte(firewallBudgetIndexBucket))
if err != nil {
return err
}
return b.Put([]byte("index"), raw)
}
func readFirewallScanBudget(tx *bolt.Tx, window string) (int, error) {
inventory, err := readFirewallBudgetInventory(tx)
if err != nil {
return 0, err
}
return readFirewallScanBudgetCount(tx, window, inventory)
}
func readFirewallScanBudgetCount(tx *bolt.Tx, window string, inventory firewallBudgetInventory) (int, error) {
_, known := slices.BinarySearch(inventory.Windows, window)
b := tx.Bucket([]byte(firewallBudgetBucket))
if b == nil && len(inventory.Windows) > 0 {
return 0, firewall.ErrStateCorrupt
}
if b != nil && b.Bucket([]byte(window)) != nil {
return 0, firewall.ErrStateCorrupt
}
if b == nil || b.Get([]byte(window)) == nil {
if known {
return 0, firewall.ErrStateCorrupt
}
return 0, nil
}
if !known {
return 0, firewall.ErrStateCorrupt
}
payload, err := decodeFirewallJournal(b.Get([]byte(window)))
if err != nil {
return 0, err
}
var budget firewallScanBudget
if json.Unmarshal(payload, &budget) != nil || budget.Window != window || budget.Count == nil || *budget.Count <= 0 {
return 0, firewall.ErrStateCorrupt
}
if _, err := time.Parse(firewallScanWindowLayout, budget.Window); err != nil {
return 0, firewall.ErrStateCorrupt
}
return *budget.Count, nil
}
// chargeFirewallScanBudget charges count units to one hourly window, or
// refuses the whole batch when they do not fit under its limit.
func chargeFirewallScanBudget(tx *bolt.Tx, window string, limit, count int) error {
inventory, err := readFirewallBudgetInventory(tx)
if err != nil {
return err
}
if window <= inventory.PrunedThrough {
return firewall.ErrScanBudget
}
charged, err := readFirewallScanBudgetCount(tx, window, inventory)
if err != nil {
return err
}
if count > limit-charged {
return firewall.ErrScanBudget
}
b, err := tx.CreateBucketIfNotExists([]byte(firewallBudgetBucket))
if err != nil {
return err
}
charged += count
raw, err := encodeFirewallJournal(firewallScanBudget{Window: window, Count: &charged})
if err != nil {
return err
}
if err := b.Put([]byte(window), raw); err != nil {
return err
}
if position, known := slices.BinarySearch(inventory.Windows, window); !known {
inventory.Windows = slices.Insert(inventory.Windows, position, window)
if err := writeFirewallBudgetInventory(tx, inventory); err != nil {
return err
}
if _, err := pruneFirewallScanBudgetWindows(tx, inventory); err != nil {
return err
}
}
return nil
}
type firewallScanBudgetCharge struct {
window string
limit int
count int
}
// firewallBatchCharges counts the charged members per window, in window order.
func firewallBatchCharges(a firewall.FirewallAction) []firewallScanBudgetCharge {
var charges []firewallScanBudgetCharge
for _, m := range a.ActionMembers() {
if m.Budget == nil {
continue
}
i := slices.IndexFunc(charges, func(c firewallScanBudgetCharge) bool { return c.window == m.Budget.Window })
if i < 0 {
charges = append(charges, firewallScanBudgetCharge{window: m.Budget.Window, limit: m.Budget.Limit})
i = len(charges) - 1
}
charges[i].count++
}
slices.SortFunc(charges, func(x, y firewallScanBudgetCharge) int { return strings.Compare(x.window, y.window) })
return charges
}
type firewallMemberIndexEntry struct {
Batch string `json:"batch"`
}
type firewallBatchMembers struct {
Members []string `json:"members"`
}
// firewallBatchMemberIDs returns the member IDs a batch indexes: none for one
// member, whose ID is the batch's own.
func firewallBatchMemberIDs(a firewall.FirewallAction) []string {
if len(a.Members) < 2 {
return nil
}
ids := make([]string, len(a.Members))
for i, m := range a.Members {
ids[i] = m.Request.ID
}
return ids
}
// refuseReusedFirewallIDs keeps every ID in exactly one batch: no admitted
// batch may name the new header or a new member, in any role.
func refuseReusedFirewallIDs(tx *bolt.Tx, a firewall.FirewallAction) error {
index := tx.Bucket([]byte(firewallMemberIndexBucket))
actions := tx.Bucket([]byte(firewallActionsBucket))
for i, id := range append([]string{a.Request.ID}, firewallBatchMemberIDs(a)...) {
if index != nil && index.Bucket([]byte(id)) != nil || actions != nil && actions.Bucket([]byte(id)) != nil {
return firewall.ErrStateCorrupt
}
if index != nil && index.Get([]byte(id)) != nil || i > 0 && actions != nil && actions.Get([]byte(id)) != nil {
return fmt.Errorf("%w: member ID reused", firewall.ErrStateConflict)
}
}
return nil
}
func indexFirewallBatchMembers(tx *bolt.Tx, a firewall.FirewallAction) error {
ids := firewallBatchMemberIDs(a)
if ids == nil {
return nil
}
index, err := tx.CreateBucketIfNotExists([]byte(firewallMemberIndexBucket))
if err != nil {
return err
}
entry, err := encodeFirewallJournal(firewallMemberIndexEntry{Batch: a.Request.ID})
if err != nil {
return err
}
for _, id := range ids {
if putErr := index.Put([]byte(id), entry); putErr != nil {
return putErr
}
}
batches, err := tx.CreateBucketIfNotExists([]byte(firewallBatchMembersBucket))
if err != nil {
return err
}
raw, err := encodeFirewallJournal(firewallBatchMembers{Members: ids})
if err != nil {
return err
}
return batches.Put([]byte(a.Request.ID), raw)
}
// releaseFirewallBatchMembers deletes a removed batch's member index. Every
// entry must still name that batch: anything else is damage, not permission
// to free an ID another batch holds.
func releaseFirewallBatchMembers(tx *bolt.Tx, id string) error {
batches := tx.Bucket([]byte(firewallBatchMembersBucket))
if batches != nil && batches.Bucket([]byte(id)) != nil {
return firewall.ErrStateCorrupt
}
if batches == nil || batches.Get([]byte(id)) == nil {
return nil
}
payload, err := decodeFirewallJournal(batches.Get([]byte(id)))
if err != nil {
return err
}
var list firewallBatchMembers
if json.Unmarshal(payload, &list) != nil || len(list.Members) < 2 {
return firewall.ErrStateCorrupt
}
index := tx.Bucket([]byte(firewallMemberIndexBucket))
for _, member := range list.Members {
if index == nil {
return firewall.ErrStateCorrupt
}
entryRaw, err := decodeFirewallJournal(index.Get([]byte(member)))
if errors.Is(err, firewall.ErrStateUnsupported) {
return err
}
var entry firewallMemberIndexEntry
if err != nil || json.Unmarshal(entryRaw, &entry) != nil || entry.Batch != id {
return firewall.ErrStateCorrupt
}
if err := index.Delete([]byte(member)); err != nil {
return err
}
}
return batches.Delete([]byte(id))
}
func (db *DB) ReadFirewallScanBudget(window string) (int, error) {
var count int
err := db.bolt.View(func(tx *bolt.Tx) error {
var err error
count, err = readFirewallScanBudget(tx, window)
return err
})
return count, err
}
// AdmitFirewallAction commits the recovery evidence and accepted scan charge
// together. The complete committed state stays at Before until verification.
func (db *DB) AdmitFirewallAction(in firewall.FirewallAction) (result firewall.FirewallAction, fresh bool, err error) {
defer func() { recordFirewallWriteError(err) }()
if validationErr := validateFirewallAdmission(in); validationErr != nil {
return result, false, validationErr
}
if _, _, err = encodeFirewallSnapshot(in.Before); err != nil {
return result, false, err
}
if _, _, err = encodeFirewallSnapshot(in.After); err != nil {
return result, false, err
}
before, err := json.Marshal(in.Before)
if err != nil {
return result, false, err
}
err = db.updateFirewallSnapshot(1, func(tx *bolt.Tx) error {
index, indexErr := readFirewallJournalIndex(tx)
if indexErr != nil {
return indexErr
}
pending, pendingErr := indexedPendingFirewallAction(tx, index)
if pendingErr != nil {
return pendingErr
}
existing, readErr := readFirewallAction(tx, in.Request.ID)
if readErr == nil {
if !existing.SameManifest(in) {
return fmt.Errorf("%w: request ID reused", firewall.ErrStateConflict)
}
result = existing
return nil
}
if !errors.Is(readErr, ErrFirewallActionMissing) {
return readErr
}
if reuseErr := refuseReusedFirewallIDs(tx, in); reuseErr != nil {
return reuseErr
}
if pending.Request.ID != "" {
return fmt.Errorf("%w: action %s requires recovery", firewall.ErrStateConflict, pending.Request.ID)
}
meta, txErr := readFirewallSnapshotMeta(tx)
if txErr != nil {
return txErr
}
if meta.Revision != in.Revision {
return firewall.ErrStateConflict
}
rows, txErr := readFirewallSnapshotRows(tx, meta)
if txErr != nil {
return txErr
}
state, txErr := decodeFirewallSnapshot(rows, meta)
if txErr != nil {
return txErr
}
current, txErr := json.Marshal(state)
if txErr != nil {
return txErr
}
if !bytes.Equal(current, before) {
return fmt.Errorf("%w: action before state differs", firewall.ErrStateConflict)
}
if !index.Initialized {
if inventoryErr := writeFirewallBudgetInventory(tx, firewallBudgetInventory{}); inventoryErr != nil {
return inventoryErr
}
}
for _, charge := range firewallBatchCharges(in) {
if chargeErr := chargeFirewallScanBudget(tx, charge.window, charge.limit, charge.count); chargeErr != nil {
return chargeErr
}
}
in.Phase = "planned"
in.UpdatedAt = in.CreatedAt
in.Detail = ""
in.AuditVersion, in.AuditAck = 1, 0
if _, writeErr := writeFirewallAction(tx, in); writeErr != nil {
return writeErr
}
if indexErr := indexFirewallBatchMembers(tx, in); indexErr != nil {
return indexErr
}
index.PendingID = in.Request.ID
if writeErr := writeFirewallJournalIndex(tx, index); writeErr != nil {
return writeErr
}
// Decode to detach every slice from caller-owned input.
result, txErr = readFirewallAction(tx, in.Request.ID)
fresh = txErr == nil
return txErr
})
if err != nil {
return firewall.FirewallAction{}, false, err
}
return result, fresh, nil
}
func (db *DB) ReadFirewallAction(id string) (firewall.FirewallAction, error) {
var a firewall.FirewallAction
err := db.bolt.View(func(tx *bolt.Tx) error {
var err error
a, err = readFirewallAction(tx, id)
return err
})
if err != nil {
return firewall.FirewallAction{}, err
}
return a, nil
}
// ReadFirewallBatchMember returns the batch that indexes member id. An ID no
// batch indexes is missing; an entry naming a batch that does not list the
// member is damage.
func (db *DB) ReadFirewallBatchMember(id string) (firewall.FirewallAction, error) {
var a firewall.FirewallAction
err := db.bolt.View(func(tx *bolt.Tx) error {
index := tx.Bucket([]byte(firewallMemberIndexBucket))
if index != nil && index.Bucket([]byte(id)) != nil {
return firewall.ErrStateCorrupt
}
if index == nil || index.Get([]byte(id)) == nil {
return ErrFirewallActionMissing
}
payload, err := decodeFirewallJournal(index.Get([]byte(id)))
if err != nil {
return err
}
var entry firewallMemberIndexEntry
if json.Unmarshal(payload, &entry) != nil {
return firewall.ErrStateCorrupt
}
a, err = readFirewallAction(tx, entry.Batch)
if errors.Is(err, ErrFirewallActionMissing) || err == nil && !slices.Contains(firewallBatchMemberIDs(a), id) {
return firewall.ErrStateCorrupt
}
return err
})
if err != nil {
return firewall.FirewallAction{}, err
}
return a, nil
}
func (db *DB) PendingFirewallActions() ([]firewall.FirewallAction, error) {
var pending []firewall.FirewallAction
err := db.bolt.View(func(tx *bolt.Tx) error {
index, err := readFirewallJournalIndex(tx)
if err != nil {
return err
}
a, err := indexedPendingFirewallAction(tx, index)
if err != nil {
return err
}
if a.Request.ID != "" {
pending = append(pending, a)
}
return nil
})
if err != nil {
return nil, err
}
return pending, nil
}
func firewallAuditPhase(phase string) bool {
return phase == "unknown" || phase == "verified" || phase == "failed"
}
// Each outcome event keeps its original payload. Acknowledgement changes only
// the envelope; retaining it permits exact, idempotent acknowledgements.
type firewallAuditEvent struct {
Action json.RawMessage `json:"action"`
Acknowledged bool `json:"acknowledged"`
}
func firewallAuditKey(id string, version uint64) []byte {
return []byte(fmt.Sprintf("%s\x00%020d", id, version))
}
func decodeFirewallAuditEvent(key, raw []byte) (firewallAuditEvent, firewall.FirewallAction, error) {
var event firewallAuditEvent
payload, _, err := decodeFirewallJournalVersion(raw, 1)
if err != nil {
return event, firewall.FirewallAction{}, err
}
if json.Unmarshal(payload, &event) != nil {
return event, firewall.FirewallAction{}, firewall.ErrStateCorrupt
}
a, err := decodeFirewallAction(event.Action)
if errors.Is(err, firewall.ErrStateUnsupported) {
return event, firewall.FirewallAction{}, err
}
if err != nil || !validFirewallJSONText(raw) || !firewallAuditPhase(a.Phase) || !bytes.Equal(key, firewallAuditKey(a.Request.ID, a.AuditVersion)) {
return event, firewall.FirewallAction{}, firewall.ErrStateCorrupt
}
return event, a, nil
}
func writeFirewallAuditEvent(tx *bolt.Tx, a firewall.FirewallAction) error {
if !firewallAuditPhase(a.Phase) {
return nil
}
action, err := encodeFirewallActionRecord(a)
if err != nil {
return err
}
raw, err := encodeFirewallJournal(firewallAuditEvent{Action: action})
if err != nil {
return err
}
b, err := tx.CreateBucketIfNotExists([]byte(firewallAuditBucket))
if err != nil {
return err
}
key := firewallAuditKey(a.Request.ID, a.AuditVersion)
if b.Get(key) != nil || b.Bucket(key) != nil {
return firewall.ErrStateConflict
}
return b.Put(key, raw)
}
func (db *DB) FirewallAuditPending() ([]firewall.FirewallAction, error) {
var pending []firewall.FirewallAction
err := db.bolt.View(func(tx *bolt.Tx) error {
index, err := readFirewallJournalIndex(tx)
if err != nil {
return err
}
b := tx.Bucket([]byte(firewallAuditBucket))
for _, ref := range index.Audit {
if b == nil {
return firewall.ErrStateCorrupt
}
key := firewallAuditKey(ref.ID, ref.Version)
event, a, decodeErr := decodeFirewallAuditEvent(key, b.Get(key))
if errors.Is(decodeErr, firewall.ErrStateUnsupported) {
return decodeErr
}
if decodeErr != nil || event.Acknowledged {
return firewall.ErrStateCorrupt
}
current, readErr := readFirewallAction(tx, ref.ID)
if errors.Is(readErr, firewall.ErrStateUnsupported) {
return readErr
}
if readErr != nil || !current.SameManifest(a) || a.AuditVersion > current.AuditVersion {
return firewall.ErrStateCorrupt
}
pending = append(pending, a)
}
return nil
})
if err != nil {
return nil, err
}
return pending, nil
}
func validFirewallActionTransition(from, to string) bool {
if !firewallActionPending(from) {
return false
}
switch to {
case "executing":
return from == "planned"
case "applied":
return from == "executing"
case "unknown", "verified", "failed":
return true
}
return false
}
func validateFirewallActionBase(tx *bolt.Tx, a firewall.FirewallAction) error {
meta, err := readFirewallSnapshotMeta(tx)
if err != nil {
return err
}
if meta.Revision != a.Revision {
return firewall.ErrStateConflict
}
rows, err := readFirewallSnapshotRows(tx, meta)
if err != nil {
return err
}
state, err := decodeFirewallSnapshot(rows, meta)
if err != nil {
return err
}
current, err := json.Marshal(state)
if err != nil {
return err
}
before, err := json.Marshal(a.Before)
if err != nil {
return err
}
if !bytes.Equal(current, before) {
return fmt.Errorf("%w: action before evidence differs from committed state", firewall.ErrStateCorrupt)
}
return nil
}
func (db *DB) TransitionFirewallAction(id, phase, detail string, at time.Time) (result firewall.FirewallAction, resultErr error) {
defer func() { recordFirewallWriteError(resultErr) }()
resultErr = db.updateFirewallSnapshot(1, func(tx *bolt.Tx) error {
index, indexErr := readFirewallJournalIndex(tx)
if indexErr != nil {
return indexErr
}
a, err := readFirewallAction(tx, id)
if err != nil {
return err
}
if firewallActionOutstanding(a) {
if index.PendingID != id {
return firewall.ErrStateCorrupt
}
if baseErr := validateFirewallActionBase(tx, a); baseErr != nil {
return baseErr
}
}
if a.Retired {
return fmt.Errorf("%w: retired action cannot transition", firewall.ErrStateConflict)
}
if a.Phase == phase && a.Detail == detail {
result = a
return nil
}
if !validFirewallActionTransition(a.Phase, phase) || at.IsZero() || a.AuditVersion == math.MaxUint64 {
return fmt.Errorf("%w: invalid action transition", firewall.ErrStateConflict)
}
if phase == "verified" || phase == "failed" {
state := a.Before
if phase == "verified" {
state = a.After
}
if publishErr := publishFirewallState(tx, a, state); publishErr != nil {
return publishErr
}
}
a.Phase, a.Detail, a.UpdatedAt = phase, detail, at
result, err = commitFirewallTransition(tx, index, a)
return err
})
if resultErr != nil {
return firewall.FirewallAction{}, resultErr
}
return result, nil
}
// publishFirewallState commits state as the revision after a's own.
func publishFirewallState(tx *bolt.Tx, a firewall.FirewallAction, state firewall.FirewallState) error {
meta, rows, err := encodeFirewallSnapshot(state)
if err != nil {
return err
}
if a.Revision == math.MaxUint64 {
return firewall.ErrStateConflict
}
meta.Revision = a.Revision + 1
raw, err := json.Marshal(meta)
if err != nil {
return err
}
return replaceFirewallSnapshot(tx, a.Revision, meta, rows, raw)
}
// commitFirewallTransition writes a's next version and its audit event, and
// once a no longer holds the pending interlock frees it and indexes a as
// retained history.
func commitFirewallTransition(tx *bolt.Tx, index firewallJournalIndex, a firewall.FirewallAction) (firewall.FirewallAction, error) {
if err := validateFirewallAdmission(a); err != nil {
return firewall.FirewallAction{}, err
}
a.AuditVersion++
size, err := writeFirewallAction(tx, a)
if err != nil {
return firewall.FirewallAction{}, err
}
if err := writeFirewallAuditEvent(tx, a); err != nil {
return firewall.FirewallAction{}, err
}
settled := !firewallActionOutstanding(a)
if settled {
index.PendingID = ""
}
if firewallAuditPhase(a.Phase) {
index.Audit = append(index.Audit, firewallAuditReference{ID: a.Request.ID, Version: a.AuditVersion})
}
if err := writeFirewallJournalIndex(tx, index); err != nil {
return firewall.FirewallAction{}, err
}
if settled {
if err := recordFirewallActionHistory(tx, a, size); err != nil {
return firewall.FirewallAction{}, err
}
if err := pruneFirewallActionHistory(tx, index); err != nil {
return firewall.FirewallAction{}, err
}
}
return a, nil
}
// RetireFirewallAction retires the uncertainty of an unknown block action
// once nothing it may have applied can remain. The
// caller has proved that the kernel holds the retired state at at; the store
// checks that every effect has expired by then and that the unknown
// outcome's audit was delivered, so the event follows the outcome. In one
// transaction it commits the rows Before and After agree on as the next
// revision, frees the pending interlock and records the event. The outcome
// stays unknown and its charges stay spent. On an action already retired
// it changes nothing.
func (db *DB) RetireFirewallAction(id string, at time.Time) (result firewall.FirewallAction, resultErr error) {
defer func() { recordFirewallWriteError(resultErr) }()
resultErr = db.updateFirewallSnapshot(1, func(tx *bolt.Tx) error {
index, err := readFirewallJournalIndex(tx)
if err != nil {
return err
}
a, err := readFirewallAction(tx, id)
if err != nil {
return err
}
if a.Retired {
result = a
return nil
}
if a.Phase != "unknown" {
return fmt.Errorf("%w: action %s is not an unknown outcome", firewall.ErrStateConflict, id)
}
if index.PendingID != id {
return firewall.ErrStateCorrupt
}
if baseErr := validateFirewallActionBase(tx, a); baseErr != nil {
return baseErr
}
if slices.ContainsFunc(index.Audit, func(ref firewallAuditReference) bool { return ref.ID == id }) {
return fmt.Errorf("%w: action %s outcome audit pending", firewall.ErrStateConflict, id)
}
if a.AuditAck != a.AuditVersion {
return fmt.Errorf("%w: action %s outcome audit acknowledgement missing", firewall.ErrStateCorrupt, id)
}
state, ok := firewall.RetiredActionState(a, at)
if !ok || a.AuditVersion == math.MaxUint64 {
return fmt.Errorf("%w: action %s has an effect that can still be live", firewall.ErrStateConflict, id)
}
if publishErr := publishFirewallState(tx, a, state); publishErr != nil {
return publishErr
}
a.Retired, a.UpdatedAt = true, at
result, err = commitFirewallTransition(tx, index, a)
return err
})
if resultErr != nil {
return firewall.FirewallAction{}, resultErr
}
return result, nil
}
func (db *DB) AcknowledgeFirewallAudit(id string, version uint64) (err error) {
defer func() { recordFirewallWriteError(err) }()
return db.updateFirewallSnapshot(1, func(tx *bolt.Tx) error {
index, indexErr := readFirewallJournalIndex(tx)
if indexErr != nil {
return indexErr
}
position := -1
for i, ref := range index.Audit {
if ref.ID == id && ref.Version == version {
position = i
break
}
}
a, err := readFirewallAction(tx, id)
if err != nil {
return err
}
if version == 0 || version > a.AuditVersion {
return firewall.ErrStateConflict
}
b := tx.Bucket([]byte(firewallAuditBucket))
key := firewallAuditKey(id, version)
if b == nil || b.Get(key) == nil {
if position >= 0 {
return firewall.ErrStateCorrupt
}
return firewall.ErrStateConflict
}
event, eventAction, err := decodeFirewallAuditEvent(key, b.Get(key))
if err != nil {
return err
}
if !eventAction.SameManifest(a) {
return firewall.ErrStateCorrupt
}
if event.Acknowledged {
if position >= 0 {
return firewall.ErrStateCorrupt
}
return nil
}
if position < 0 {
return firewall.ErrStateCorrupt
}
event.Acknowledged = true
raw, err := encodeFirewallJournal(event)
if err != nil {
return err
}
if err := b.Put(key, raw); err != nil {
return err
}
a.AuditAck = max(a.AuditAck, version)
size, writeErr := writeFirewallAction(tx, a)
if writeErr != nil {
return writeErr
}
if err := updateFirewallActionHistorySize(tx, a, size); err != nil {
return err
}
index.Audit = append(index.Audit[:position], index.Audit[position+1:]...)
if err := writeFirewallJournalIndex(tx, index); err != nil {
return err
}
return pruneFirewallActionHistory(tx, index)
})
}
package store
import (
"encoding/json"
"net/netip"
"github.com/pidginhost/csm/internal/firewall"
)
// validateFirewallBatchStates holds a batch of two or more members to the
// only change it may make: each member adds its own row, and every other row
// stays as it was. Nothing evicts for a batch, so only a verified row that
// had expired when the batch was planned may be missing from its After state.
func validateFirewallBatchStates(a firewall.FirewallAction) error {
if !sameFirewallRows(a.Before.BlockedNet, a.After.BlockedNet) || !sameFirewallRows(a.Before.Allowed, a.After.Allowed) || !sameFirewallRows(a.Before.PortAllowed, a.After.PortAllowed) {
return errInvalidFirewallManifest
}
kept := map[string]int{}
beforeRows := make([]string, 0, len(a.Before.Blocked))
for _, row := range a.Before.Blocked {
raw, err := json.Marshal(row)
if err != nil {
return errInvalidFirewallManifest
}
key := string(raw)
kept[key]++
beforeRows = append(beforeRows, key)
}
members := map[string]firewall.ActionRequest{}
targets := map[string]bool{}
seen := map[string]bool{}
for _, m := range a.Members {
members[m.Request.ID] = m.Request
targets[m.Request.Target] = true
}
nextBefore := 0
for _, row := range a.After.Blocked {
raw, err := json.Marshal(row)
if err != nil {
return errInvalidFirewallManifest
}
if kept[string(raw)] > 0 {
// Address aliases can carry different decisions. Their order
// determines which decision the engine's lookups select.
for nextBefore < len(beforeRows) && beforeRows[nextBefore] != string(raw) {
if !firewallRowExpired(a.Before.Blocked[nextBefore], a) {
return errInvalidFirewallManifest
}
nextBefore++
}
if nextBefore == len(beforeRows) {
return errInvalidFirewallManifest
}
nextBefore++
kept[string(raw)]--
if !firewallRowExpired(row, a) && targets[canonicalFirewallTarget(row.IP)] {
return errInvalidFirewallManifest
}
continue
}
m, ok := members[row.ActionID]
if seen[row.ActionID] {
return errInvalidFirewallManifest
}
if !ok || row.IP != m.Target || row.Reason != m.Reason || row.Unverified || !row.BlockedAt.Equal(a.CreatedAt) || !row.ExpiresAt.Equal(row.BlockedAt.Add(m.TTL)) || !row.ExpiresAt.After(a.CreatedAt) {
return errInvalidFirewallManifest
}
if row.LoginSpray != m.LoginSpray || (row.LoginSpray && !row.Automatic) {
return errInvalidFirewallManifest
}
seen[row.ActionID] = true
}
if len(seen) != len(members) {
return errInvalidFirewallManifest
}
for _, row := range a.Before.Blocked[nextBefore:] {
if !firewallRowExpired(row, a) {
return errInvalidFirewallManifest
}
}
return nil
}
// validateFirewallVictims holds a one-member action to the victims it
// records. Undo restores a victim from this record, so each must be exactly
// the one row that named its address before the action, and no row may name
// that address after it.
func validateFirewallVictims(a firewall.FirewallAction) error {
for _, v := range a.Members[0].Victims {
if v.Unverified {
return errInvalidFirewallManifest
}
want, err := json.Marshal(v)
if err != nil {
return errInvalidFirewallManifest
}
held, same := 0, false
for _, row := range a.Before.Blocked {
if canonicalFirewallTarget(row.IP) != v.IP {
continue
}
held++
raw, err := json.Marshal(row)
same = same || err == nil && string(raw) == string(want)
}
if held != 1 || !same {
return errInvalidFirewallManifest
}
for _, row := range a.After.Blocked {
if canonicalFirewallTarget(row.IP) == v.IP {
return errInvalidFirewallManifest
}
}
}
return nil
}
func firewallRowExpired(row firewall.BlockedEntry, a firewall.FirewallAction) bool {
// Expiry cannot settle an unknown kernel effect; its owner must do so.
return !row.Unverified && !row.ExpiresAt.IsZero() && !row.ExpiresAt.After(a.CreatedAt)
}
// canonicalFirewallTarget compares a stored row's address with a member's
// canonical target, whatever spelling an older row used.
func canonicalFirewallTarget(ip string) string {
if addr, err := netip.ParseAddr(ip); err == nil {
return addr.Unmap().String()
}
return ip
}
func sameFirewallRows[T any](a, b []T) bool {
if len(a) == 0 || len(b) == 0 {
return len(a) == len(b)
}
left, err := json.Marshal(a)
if err != nil {
return false
}
right, err := json.Marshal(b)
return err == nil && string(left) == string(right)
}
package store
import (
"encoding/json"
"fmt"
"time"
bolt "go.etcd.io/bbolt"
)
// FirewallRollback is a pending tentative-apply record. The previous
// csm.yaml bytes are stashed verbatim so a recovery path can restore the
// file byte-for-byte without re-rendering through any encoder. Hashes are
// recorded so the daemon can sanity-check the on-disk file matches what
// was applied before deciding to revert.
type FirewallRollback struct {
PrevYAML []byte `json:"prev_yaml"`
PrevHash string `json:"prev_hash"`
NewHash string `json:"new_hash"`
AppliedAt time.Time `json:"applied_at"`
ExpiresAt time.Time `json:"expires_at"`
AppliedBy string `json:"applied_by"`
}
const fwRollbackBucket = "fw:rollback"
const fwRollbackKey = "pending"
// SaveFirewallRollback writes a pending rollback record. Overwrites any
// existing pending entry; callers must clear or revert the previous one
// first if that matters for their flow.
func (db *DB) SaveFirewallRollback(rb FirewallRollback) error {
val, err := json.Marshal(rb)
if err != nil {
return err
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(fwRollbackBucket))
return b.Put([]byte(fwRollbackKey), val)
})
}
// GetFirewallRollback returns the pending rollback or (zero, false) if
// none. The bool distinguishes "no record" from a zero-valued record.
// A bbolt unmarshal failure is treated as "no usable record" so the
// daemon can skip a corrupt entry instead of refusing to start.
func (db *DB) GetFirewallRollback() (FirewallRollback, bool) {
var rb FirewallRollback
found := false
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(fwRollbackBucket))
val := b.Get([]byte(fwRollbackKey))
if val == nil {
return nil
}
if uerr := json.Unmarshal(val, &rb); uerr != nil {
// Corrupt record: leave found=false so the caller treats
// it as "no pending rollback" rather than panicking.
return nil //nolint:nilerr // swallowing is the intent; see comment.
}
found = true
return nil
})
return rb, found
}
// ClearFirewallRollback drops the pending rollback. Idempotent: deleting
// a non-existent key is not an error in bbolt.
func (db *DB) ClearFirewallRollback() error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(fwRollbackBucket))
return b.Delete([]byte(fwRollbackKey))
})
}
// DisarmFirewallRollbackSnapshot removes process-lifetime rollback intent
// from a stopped, private bbolt snapshot. Backup and restore artifacts keep
// durable firewall state, but must not make an old tentative configuration
// write active again on a later daemon start.
func DisarmFirewallRollbackSnapshot(path string) error {
if err := verifySnapshot(path); err != nil {
return err
}
snapshot, err := bolt.Open(path, 0o600, nil)
if err != nil {
return fmt.Errorf("opening state snapshot: %w", err)
}
pending := false
if err := snapshot.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(fwRollbackBucket))
pending = bucket != nil && bucket.Get([]byte(fwRollbackKey)) != nil
return nil
}); err != nil {
_ = snapshot.Close()
return fmt.Errorf("checking firewall rollback snapshot: %w", err)
}
if !pending {
if err := snapshot.Close(); err != nil {
return fmt.Errorf("closing state snapshot: %w", err)
}
return nil
}
if err := snapshot.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(fwRollbackBucket))
if bucket == nil {
return nil
}
return bucket.Delete([]byte(fwRollbackKey))
}); err != nil {
_ = snapshot.Close()
return fmt.Errorf("disarming firewall rollback snapshot: %w", err)
}
if err := snapshot.Close(); err != nil {
return fmt.Errorf("closing state snapshot: %w", err)
}
return nil
}
package store
import (
"bytes"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
"fmt"
"math"
"strconv"
"time"
"unicode/utf8"
"github.com/pidginhost/csm/internal/firewall"
bolt "go.etcd.io/bbolt"
)
var _ firewall.StateStore = (*DB)(nil)
const firewallSnapshotBucket = "fw:state"
const firewallSnapshotKey = "snapshot"
const firewallSnapshotVersion = 1
// Fixed collection order is part of the versioned storage format. Metadata
// lives under fw:* so a future atomic firewall restore can include it.
var firewallSnapshotBuckets = [...]string{"fw:blocked", "fw:subnets", "fw:allowed", "fw:port_allowed"}
type firewallCollection struct {
// Keys preserve engine order, including distinct entries for the same IP.
// Nil and empty keys preserve nil and empty domain slices respectively.
Keys []string `json:"keys"`
Digest string `json:"sha256"`
}
type firewallSnapshotMeta struct {
Version int `json:"version"`
Revision uint64 `json:"revision"`
Collections []firewallCollection `json:"collections"`
}
type firewallSnapshotRows [4][][]byte
// ReadFirewallState copies a single consistent snapshot before decoding domain
// values. No bbolt-owned bytes or partially decoded state escape this method.
func (db *DB) ReadFirewallState() (result firewall.FirewallState, revision uint64, err error) {
started := time.Now()
defer func() {
firewallReadDuration.Observe(time.Since(started).Seconds())
if err != nil {
firewallReadFailures.Inc()
}
}()
var meta firewallSnapshotMeta
var rows firewallSnapshotRows
err = db.bolt.View(func(tx *bolt.Tx) error {
var readErr error
meta, readErr = readFirewallSnapshotMeta(tx)
if readErr != nil {
return readErr
}
rows, readErr = readFirewallSnapshotRows(tx, meta)
return readErr
})
if err != nil {
return firewall.FirewallState{}, 0, err
}
state, err := decodeFirewallSnapshot(rows, meta)
if err != nil {
return firewall.FirewallState{}, 0, err
}
return state, meta.Revision, nil
}
// ReplaceFirewallState encodes outside the writer lock and publishes state and
// revision together. Existing runtime callers continue using their current
// backend; this API is not an implicit migration or a cutover marker.
func (db *DB) ReplaceFirewallState(expectedRevision uint64, state firewall.FirewallState) (revision uint64, err error) {
defer func() { recordFirewallWriteError(err) }()
if expectedRevision == math.MaxUint64 {
return 0, fmt.Errorf("%w: revision exhausted", firewall.ErrStateConflict)
}
meta, rows, err := encodeFirewallSnapshot(state)
if err != nil {
return 0, err
}
meta.Revision = expectedRevision + 1
encoded, err := json.Marshal(meta)
if err != nil {
return 0, err
}
err = db.updateFirewallSnapshot(len(state.Blocked)+len(state.BlockedNet)+len(state.Allowed)+len(state.PortAllowed), func(tx *bolt.Tx) error {
if pendingErr := refusePendingFirewallActions(tx); pendingErr != nil {
return pendingErr
}
return replaceFirewallSnapshot(tx, expectedRevision, meta, rows, encoded)
})
if err != nil {
return 0, err
}
return meta.Revision, nil
}
// replaceFirewallSnapshot is private so future action admission can share this
// transaction without exporting CRUD or transaction callbacks to domain code.
func replaceFirewallSnapshot(tx *bolt.Tx, expected uint64, next firewallSnapshotMeta, rows firewallSnapshotRows, encoded []byte) error {
current, metaErr := readFirewallSnapshotMeta(tx)
if metaErr == firewall.ErrStateUninitialized {
if expected != 0 {
return firewall.ErrStateConflict
}
} else {
if metaErr != nil {
return metaErr
}
if current.Revision != expected {
return firewall.ErrStateConflict
}
// An out-of-band legacy bucket write cannot silently bypass the revision.
currentRows, readErr := readFirewallSnapshotRows(tx, current)
if readErr != nil {
return readErr
}
if _, err := decodeFirewallSnapshot(currentRows, current); err != nil {
return err
}
}
for i, name := range firewallSnapshotBuckets {
if tx.Bucket([]byte(name)) != nil {
if err := tx.DeleteBucket([]byte(name)); err != nil {
return err
}
}
b, err := tx.CreateBucket([]byte(name))
if err != nil {
return err
}
for j, key := range next.Collections[i].Keys {
if err := b.Put([]byte(key), rows[i][j]); err != nil {
return err
}
}
}
b, err := tx.CreateBucketIfNotExists([]byte(firewallSnapshotBucket))
if err != nil {
return err
}
return b.Put([]byte(firewallSnapshotKey), encoded)
}
func readFirewallSnapshotMeta(tx *bolt.Tx) (firewallSnapshotMeta, error) {
var meta firewallSnapshotMeta
b := tx.Bucket([]byte(firewallSnapshotBucket))
if b == nil {
return meta, firewall.ErrStateUninitialized
}
raw := b.Get([]byte(firewallSnapshotKey))
if raw == nil {
return meta, fmt.Errorf("%w: missing metadata", firewall.ErrStateCorrupt)
}
if _, err := decodeFirewallVersion(raw, firewallSnapshotVersion); err != nil {
return meta, err
}
if err := json.Unmarshal(raw, &meta); err != nil {
return meta, fmt.Errorf("%w: metadata decode: %v", firewall.ErrStateCorrupt, err)
}
if meta.Version != firewallSnapshotVersion || meta.Revision == 0 || len(meta.Collections) != len(firewallSnapshotBuckets) {
return meta, fmt.Errorf("%w: metadata version, revision or collections", firewall.ErrStateCorrupt)
}
return meta, nil
}
func readFirewallSnapshotRows(tx *bolt.Tx, meta firewallSnapshotMeta) (firewallSnapshotRows, error) {
var rows firewallSnapshotRows
for i, name := range firewallSnapshotBuckets {
b := tx.Bucket([]byte(name))
if b == nil {
return rows, fmt.Errorf("%w: missing collection %s", firewall.ErrStateCorrupt, name)
}
collection := meta.Collections[i]
expected := make(map[string]struct{}, len(collection.Keys))
for _, key := range collection.Keys {
if _, exists := expected[key]; exists || key == "" {
return rows, fmt.Errorf("%w: invalid collection keys", firewall.ErrStateCorrupt)
}
expected[key] = struct{}{}
raw := b.Get([]byte(key))
if raw == nil {
return rows, fmt.Errorf("%w: missing row in %s", firewall.ErrStateCorrupt, name)
}
rows[i] = append(rows[i], bytes.Clone(raw))
}
if err := b.ForEach(func(k, v []byte) error {
if _, ok := expected[string(k)]; !ok || v == nil {
return fmt.Errorf("%w: unexpected row in %s", firewall.ErrStateCorrupt, name)
}
return nil
}); err != nil {
return rows, err
}
if digestFirewallCollection(collection.Keys, rows[i]) != collection.Digest {
return rows, fmt.Errorf("%w: changed collection %s", firewall.ErrStateCorrupt, name)
}
}
return rows, nil
}
func digestFirewallCollection(keys []string, rows [][]byte) string {
hash := sha256.New()
var size [8]byte
for i, key := range keys {
binary.BigEndian.PutUint64(size[:], uint64(len(key)))
_, _ = hash.Write(size[:])
_, _ = hash.Write([]byte(key))
binary.BigEndian.PutUint64(size[:], uint64(len(rows[i])))
_, _ = hash.Write(size[:])
_, _ = hash.Write(rows[i])
}
return hex.EncodeToString(hash.Sum(nil))
}
func encodeFirewallSnapshot(state firewall.FirewallState) (firewallSnapshotMeta, firewallSnapshotRows, error) {
meta := firewallSnapshotMeta{Version: firewallSnapshotVersion, Collections: make([]firewallCollection, 4)}
var rows firewallSnapshotRows
var err error
meta.Collections[0], rows[0], err = encodeFirewallCollection(state.Blocked, func(e firewall.BlockedEntry) string { return e.IP })
if err != nil {
return meta, rows, err
}
meta.Collections[1], rows[1], err = encodeFirewallCollection(state.BlockedNet, func(e firewall.SubnetEntry) string { return e.CIDR })
if err != nil {
return meta, rows, err
}
meta.Collections[2], rows[2], err = encodeFirewallCollection(state.Allowed, func(e firewall.AllowedEntry) string { return e.IP })
if err != nil {
return meta, rows, err
}
meta.Collections[3], rows[3], err = encodeFirewallCollection(state.PortAllowed, func(e firewall.PortAllowEntry) string { return portAllowKey(e.IP, e.Port, e.Proto) })
return meta, rows, err
}
func encodeFirewallCollection[T any](entries []T, identity func(T) string) (firewallCollection, [][]byte, error) {
var collection firewallCollection
var rows [][]byte
if entries != nil {
collection.Keys = make([]string, 0, len(entries))
}
used := make(map[string]bool, len(entries))
for i, entry := range entries {
key := identity(entry)
if key == "" || used[key] {
key = fmt.Sprintf("\x00%d", i)
}
if used[key] {
return collection, nil, fmt.Errorf("invalid firewall row identity")
}
used[key] = true
if !validFirewallRowEncoding(entry) {
return collection, nil, fmt.Errorf("firewall row cannot be encoded losslessly")
}
row, err := json.Marshal(entry)
if err != nil {
return collection, nil, fmt.Errorf("encode firewall row: %w", err)
}
collection.Keys = append(collection.Keys, key)
rows = append(rows, row)
}
collection.Digest = digestFirewallCollection(collection.Keys, rows)
return collection, rows, nil
}
func decodeFirewallSnapshot(rows firewallSnapshotRows, meta firewallSnapshotMeta) (firewall.FirewallState, error) {
var state firewall.FirewallState
var err error
state.Blocked, err = decodeFirewallCollection[firewall.BlockedEntry](rows[0], meta.Collections[0].Keys != nil)
if err != nil {
return firewall.FirewallState{}, err
}
state.BlockedNet, err = decodeFirewallCollection[firewall.SubnetEntry](rows[1], meta.Collections[1].Keys != nil)
if err != nil {
return firewall.FirewallState{}, err
}
state.Allowed, err = decodeFirewallCollection[firewall.AllowedEntry](rows[2], meta.Collections[2].Keys != nil)
if err != nil {
return firewall.FirewallState{}, err
}
state.PortAllowed, err = decodeFirewallCollection[firewall.PortAllowEntry](rows[3], meta.Collections[3].Keys != nil)
if err != nil {
return firewall.FirewallState{}, err
}
return state, nil
}
func decodeFirewallCollection[T any](rows [][]byte, nonNil bool) ([]T, error) {
var entries []T
if nonNil {
entries = make([]T, 0, len(rows))
}
for _, row := range rows {
var entry T
// A matching checksum does not make malformed text decodable. JSON
// silently replaces invalid Unicode, which would alter stored evidence.
if !validFirewallJSONText(row) {
return nil, fmt.Errorf("%w: row is not valid Unicode", firewall.ErrStateCorrupt)
}
trimmed := bytes.TrimSpace(row)
if len(trimmed) == 0 || trimmed[0] != '{' {
return nil, fmt.Errorf("%w: row is not an object", firewall.ErrStateCorrupt)
}
if err := json.Unmarshal(row, &entry); err != nil {
return nil, fmt.Errorf("%w: row decode: %v", firewall.ErrStateCorrupt, err)
}
entries = append(entries, entry)
}
return entries, nil
}
// The JSON decoder checks syntax but accepts invalid UTF-8 and unpaired UTF-16
// surrogate escapes. Check those first without rejecting literal backslashes
// or valid surrogate pairs. Non-Unicode escapes are left to the JSON decoder.
func validFirewallJSONText(raw []byte) bool {
if !utf8.Valid(raw) {
return false
}
for i := 0; i < len(raw); i++ {
if raw[i] != '\\' {
continue
}
i++
if i >= len(raw) {
return false
}
if raw[i] != 'u' {
continue
}
if i+4 >= len(raw) {
return false
}
code, err := strconv.ParseUint(string(raw[i+1:i+5]), 16, 16)
if err != nil || code >= 0xdc00 && code <= 0xdfff {
return false
}
i += 4
if code < 0xd800 || code > 0xdbff {
continue
}
if i+6 >= len(raw) || raw[i+1] != '\\' || raw[i+2] != 'u' {
return false
}
low, err := strconv.ParseUint(string(raw[i+3:i+7]), 16, 16)
if err != nil || low < 0xdc00 || low > 0xdfff {
return false
}
i += 6
}
return true
}
// encoding/json replaces invalid UTF-8 and rounds timezone offsets to minutes.
// Reject either loss before a successful write can acknowledge altered evidence.
func validFirewallRowEncoding(entry any) bool {
var fields []string
var times []time.Time
switch e := entry.(type) {
case firewall.BlockedEntry:
times = []time.Time{e.BlockedAt, e.ExpiresAt}
fields = []string{e.IP, e.Reason, e.Source, e.ActionID}
case firewall.SubnetEntry:
times = []time.Time{e.BlockedAt, e.ExpiresAt}
fields = []string{e.CIDR, e.Reason, e.Source}
case firewall.AllowedEntry:
times = []time.Time{e.ExpiresAt}
fields = []string{e.IP, e.Reason, e.Source}
case firewall.PortAllowEntry:
fields = []string{e.IP, e.Proto, e.Reason, e.Source}
default:
return false
}
for _, timestamp := range times {
_, offset := timestamp.Zone()
if offset%60 != 0 {
return false
}
}
for _, field := range fields {
if !utf8.ValidString(field) {
return false
}
}
return true
}
package store
import (
"errors"
"fmt"
"time"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/metrics"
bolt "go.etcd.io/bbolt"
)
// These collectors have no labels: paths, addresses and caller identities never
// allocate time series. They cover only the new firewall snapshot contract.
var (
firewallWriteWait = metrics.NewHistogram("csm_storage_firewall_write_wait_seconds", "Firewall writer lock wait.", []float64{.001, .01, .1, 1, 10})
firewallWriteTransaction = metrics.NewHistogram("csm_storage_firewall_write_transaction_seconds", "Firewall write transaction through commit or rollback, excluding writer wait.", []float64{.001, .01, .1, 1, 10})
firewallReadDuration = metrics.NewHistogram("csm_storage_firewall_read_seconds", "Firewall snapshot read including copying and decoding.", []float64{.001, .01, .1, 1, 10})
firewallBatchRows = metrics.NewHistogram("csm_storage_firewall_batch_rows", "Rows in attempted firewall snapshot transactions.", []float64{0, 100, 1000, 10000, 100000})
firewallPendingWrites = metrics.NewGauge("csm_storage_firewall_pending_writes", "Firewall writes waiting or executing.")
firewallWriteFailures = metrics.NewCounter("csm_storage_firewall_write_failures_total", "Firewall replacements returning errors, including admission refusals.")
firewallReadFailures = metrics.NewCounter("csm_storage_firewall_read_failures_total", "Firewall reads returning errors.")
firewallCommitFailures = metrics.NewCounter("csm_storage_firewall_commit_failures_total", "Firewall transactions accepted by the callback without a confirmed commit.")
firewallConflicts = metrics.NewCounter("csm_storage_firewall_conflicts_total", "Firewall replacements refused due to revision conflicts.")
)
func init() {
metrics.MustRegister("csm_storage_firewall_write_wait_seconds", firewallWriteWait)
metrics.MustRegister("csm_storage_firewall_write_transaction_seconds", firewallWriteTransaction)
metrics.MustRegister("csm_storage_firewall_read_seconds", firewallReadDuration)
metrics.MustRegister("csm_storage_firewall_batch_rows", firewallBatchRows)
metrics.MustRegister("csm_storage_firewall_pending_writes", firewallPendingWrites)
metrics.MustRegister("csm_storage_firewall_write_failures_total", firewallWriteFailures)
metrics.MustRegister("csm_storage_firewall_read_failures_total", firewallReadFailures)
metrics.MustRegister("csm_storage_firewall_commit_failures_total", firewallCommitFailures)
metrics.MustRegister("csm_storage_firewall_conflicts_total", firewallConflicts)
}
func recordFirewallWriteError(err error) {
if err != nil {
firewallWriteFailures.Inc()
}
if errors.Is(err, firewall.ErrStateConflict) {
firewallConflicts.Inc()
}
}
func (db *DB) updateFirewallSnapshot(rowCount int, fn func(*bolt.Tx) error) error {
firewallBatchRows.Observe(float64(rowCount))
firewallPendingWrites.Inc()
defer firewallPendingWrites.Dec()
waiting := time.Now()
var acquired time.Time
accepted := false
err := boltUpdate(db.bolt, func(tx *bolt.Tx) error {
acquired = time.Now()
firewallWriteWait.Observe(acquired.Sub(waiting).Seconds())
callbackErr := fn(tx)
accepted = callbackErr == nil
return callbackErr
})
if acquired.IsZero() {
// Begin failed: elapsed time belongs to acquisition, not a transaction.
firewallWriteWait.Observe(time.Since(waiting).Seconds())
} else {
firewallWriteTransaction.Observe(time.Since(acquired).Seconds())
}
if err != nil && accepted {
firewallCommitFailures.Inc()
// bbolt can return a sync error after publishing its new metadata.
// An accepted callback plus an error is not proof of rollback.
return fmt.Errorf("%w: %w", firewall.ErrStateCommitUncertain, err)
}
return err
}
package store
import (
"encoding/json"
"time"
bolt "go.etcd.io/bbolt"
)
// AuditResult represents a single hardening check result.
type AuditResult struct {
Category string `json:"category"`
Name string `json:"name"`
Title string `json:"title"`
Status string `json:"status"`
Message string `json:"message"`
Fix string `json:"fix,omitempty"`
}
// AuditReport is the full result of a hardening audit run.
type AuditReport struct {
Timestamp time.Time `json:"timestamp,omitzero"`
ServerType string `json:"server_type"`
Results []AuditResult `json:"results"`
Score int `json:"score"`
Total int `json:"total"`
}
const hardeningReportKey = "hardening:report"
// SaveHardeningReport persists the latest audit report in the meta bucket.
func (db *DB) SaveHardeningReport(report *AuditReport) error {
data, err := json.Marshal(report)
if err != nil {
return err
}
return db.bolt.Update(func(tx *bolt.Tx) error {
return tx.Bucket([]byte("meta")).Put([]byte(hardeningReportKey), data)
})
}
// LoadHardeningReport retrieves the latest audit report from the meta bucket.
// Returns a zero-value report (nil Results) if no report has been saved yet.
func (db *DB) LoadHardeningReport() (*AuditReport, error) {
var report AuditReport
err := db.bolt.View(func(tx *bolt.Tx) error {
v := tx.Bucket([]byte("meta")).Get([]byte(hardeningReportKey))
if v == nil {
return nil
}
return json.Unmarshal(v, &report)
})
return &report, err
}
package store
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"time"
"unicode/utf8"
"github.com/pidginhost/csm/internal/alert"
bolt "go.etcd.io/bbolt"
)
// maxHistoryEntries is the maximum number of history entries to retain.
// It is a var (not const) so tests can override it.
var maxHistoryEntries = 100_000
// AppendHistory inserts findings into the history bucket with TimeKey keys.
// It increments the history:count counter and prunes oldest entries if the
// count exceeds maxHistoryEntries.
func (db *DB) AppendHistory(findings []alert.Finding) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("history"))
writer := newTimeKeyWriter(b)
for i, f := range findings {
val, err := json.Marshal(alert.SanitizeFinding(f))
if err != nil {
return err
}
key := nextHistoryKey(b, f.Timestamp, i)
if err := writer.put([]byte(key), val); err != nil {
return err
}
// Same transaction as the history insert: either both land or
// neither does, so the daily aggregate can never drift.
if err := incrStatsDaily(tx, f.Timestamp, f.Severity); err != nil {
return err
}
if err := bumpLatestByCheck(tx, f.Check, f.Timestamp); err != nil {
return err
}
}
writer.settle()
if err := incrCounter(tx, "history:count", len(findings)); err != nil {
return err
}
// Cheap sweep against the bounded stats:daily bucket. Done here
// (rather than on a timer) so the daily-aggregate path has a
// single owner.
if len(findings) > 0 {
if err := bumpHistoryRevision(tx); err != nil {
return err
}
if err := pruneStatsDaily(tx, time.Now()); err != nil {
return err
}
}
// Prune oldest entries if count exceeds maxHistoryEntries.
meta := tx.Bucket([]byte("meta"))
var count int
if v := meta.Get([]byte("history:count")); v != nil {
_, _ = fmt.Sscanf(string(v), "%d", &count)
}
if count > maxHistoryEntries {
excess := count - maxHistoryEntries
c := b.Cursor()
k, _ := c.First()
for ; k != nil && excess > 0; excess-- {
// Delete() moves the cursor to the next item, so we
// must NOT call c.Next() after it.
if err := c.Delete(); err != nil {
return err
}
k, _ = c.First()
}
if err := setCounter(tx, "history:count", maxHistoryEntries); err != nil {
return err
}
}
return nil
})
}
func nextHistoryKey(b *bolt.Bucket, timestamp time.Time, start int) string {
for counter := start; ; counter++ {
key := TimeKey(timestamp, counter)
// Shutdown drains can persist separate batches whose findings carry
// the same detector timestamp. Probe instead of overwriting history.
if b.Get([]byte(key)) == nil {
return key
}
}
}
func bumpHistoryRevision(tx *bolt.Tx) error {
return incrCounter(tx, "history:revision", 1)
}
// HistoryMark changes with history mutations, even when retention and a later
// append reuse the same keys or a migration changes only interior keys.
func (db *DB) HistoryMark() string {
var mark string
_ = db.bolt.View(func(tx *bolt.Tx) error {
mark = string(tx.Bucket([]byte("meta")).Get([]byte("history:revision")))
return nil
})
return mark
}
// ReadHistory reads findings from the history bucket, newest-first.
// It returns up to limit findings starting at offset, plus the total count.
func (db *DB) ReadHistory(limit, offset int) ([]alert.Finding, int) {
total := db.getCounter("history:count")
var results []alert.Finding
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("history"))
c := b.Cursor()
// Skip offset entries from the end (newest first).
skipped := 0
k, v := c.Last()
for ; k != nil && skipped < offset; k, v = c.Prev() {
skipped++
}
// Collect up to limit entries.
for ; k != nil && len(results) < limit; k, v = c.Prev() {
var f alert.Finding
if err := json.Unmarshal(v, &f); err == nil {
results = append(results, f)
}
}
return nil
})
return results, total
}
// ReadHistoryFiltered reads findings with optional filtering.
// Parameters:
// - from, to: calendar dates or RFC 3339 instants (empty to skip)
// - severity: filter by severity level (-1 for no filter)
// - search: case-insensitive substring match on check/message/details (empty to skip)
func (db *DB) ReadHistoryFiltered(limit, offset int, from, to string, severity int, search string) ([]alert.Finding, int) {
return db.ReadHistoryFilteredWithChecks(limit, offset, from, to, severity, search, nil)
}
// ParseHistoryBound reads one end of a history date range. A calendar date
// (YYYY-MM-DD) names a server-local day: as a start it is that day's
// midnight, as an end the next day's, so the whole day is included. An RFC
// 3339 instant is used as given; as an end it is exclusive. An empty bound
// is the zero time.
func ParseHistoryBound(s string, end bool) (time.Time, error) {
return parseHistoryBoundIn(s, end, time.Local)
}
func parseHistoryBoundIn(s string, end bool, loc *time.Location) (time.Time, error) {
s = strings.TrimSpace(s)
if s == "" {
return time.Time{}, nil
}
if t, err := time.Parse(time.RFC3339, s); err == nil {
return t, nil
}
// Parse the calendar in UTC so a missing local midnight cannot normalize
// the date into the preceding day before we choose the exclusive end.
day, err := time.Parse("2006-01-02", s)
if err != nil {
return time.Time{}, fmt.Errorf("date %q is neither YYYY-MM-DD nor RFC 3339", s)
}
if end {
day = day.AddDate(0, 0, 1)
}
// Walk the surrounding zone intervals to find the earliest instant of
// the day. A repeated midnight uses its first occurrence; a skipped
// midnight (or date) starts at the transition into the next valid time.
for at := day.Add(-48 * time.Hour).In(loc); ; {
_, offset := at.Zone()
candidate := day.Add(-time.Duration(offset) * time.Second).In(loc)
if candidate.Before(at) {
candidate = at
}
_, zoneEnd := at.ZoneBounds()
if zoneEnd.IsZero() || candidate.Before(zoneEnd) {
return candidate, nil
}
at = zoneEnd
}
}
// decodeHistoryEntry decodes one stored finding; a var so tests can count
// decodes.
var decodeHistoryEntry = func(v []byte, f *alert.Finding) error { return json.Unmarshal(v, f) }
// ReadHistoryFilteredWithChecks reads findings with optional filters, including
// an exact check-name set when checks is non-nil.
func (db *DB) ReadHistoryFilteredWithChecks(
limit, offset int,
from, to string,
severity int,
search string,
checks map[string]bool,
) ([]alert.Finding, int) {
return db.readHistoryFilteredIn(limit, offset, from, to, severity, search, checks, time.Local)
}
func (db *DB) readHistoryFilteredIn(
limit, offset int,
from, to string,
severity int,
search string,
checks map[string]bool,
loc *time.Location,
) ([]alert.Finding, int) {
var results []alert.Finding
matched := 0
searchLower := strings.ToLower(search)
from, to = strings.TrimSpace(from), strings.TrimSpace(to)
mayMatch := historyPrefilter(severity, searchLower, checks)
var fromPrefix, toPrefix string
if from != "" {
if fromTime, err := parseHistoryBoundIn(from, false, loc); err == nil {
fromPrefix = timeKeyLowerBound(fromTime)
} else {
fromPrefix = ParseTimeKeyPrefix(from)
}
}
if to != "" {
if toTime, err := parseHistoryBoundIn(to, true, loc); err == nil {
// An exclusive upper bound: the next local midnight for a
// date, the instant itself for an RFC 3339 end.
toPrefix = timeKeyLowerBound(toTime)
} else {
toPrefix = ParseTimeKeyPrefix(to) + "99"
}
}
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("history"))
c := b.Cursor()
// Seek to the first key at the exclusive upper bound, then step back so the
// descending walk starts at the newest in-range entry. Without this the
// loop walked (and skipped) every entry newer than `to`, which is O(N)
// of the whole bucket when querying an old range on a large history.
k, v := c.Last()
if toPrefix != "" {
if sk, _ := c.Seek([]byte(toPrefix)); sk != nil {
// Seek lands on the first key >= toPrefix (out of range);
// the previous key is the newest in-range entry.
k, v = c.Prev()
} else {
// All keys are below toPrefix; Last() is already in range.
k, v = c.Last()
}
}
for ; k != nil; k, v = c.Prev() {
key := string(k)
// Defensive: anything still above the upper bound is out of range.
if toPrefix != "" && key >= toPrefix {
continue
}
// Time-range: if key is below fromPrefix, all remaining are older - stop.
if fromPrefix != "" && key < fromPrefix {
break
}
if !mayMatch(v) {
continue
}
var f alert.Finding
if err := decodeHistoryEntry(v, &f); err != nil {
continue
}
// Severity filter.
if severity >= 0 && int(f.Severity) != severity {
continue
}
// Exact check-name filter.
if checks != nil && !checks[f.Check] {
continue
}
// Search filter.
if search != "" && !containsLower(f.Check, searchLower) &&
!containsLower(f.Message, searchLower) &&
!containsLower(f.Details, searchLower) {
continue
}
matched++
if matched > offset && len(results) < limit {
results = append(results, f)
}
}
return nil
})
return results, matched
}
// historyPrefilter returns a test on a stored entry's JSON that is false only
// when the entry cannot pass the severity, check or search filter, so the
// walk that counts matches decodes only plausible entries. Each part applies
// only when the text it looks for is stored verbatim: JSON escapes quotes,
// backslashes, control characters, <, > and &, so a search for those is
// left to the decoded check.
func historyPrefilter(severity int, searchLower string, checks map[string]bool) func([]byte) bool {
var sevNeedles [][]byte
if severity >= 0 {
sevNeedles = [][]byte{
[]byte(fmt.Sprintf(`"severity":%d,`, severity)),
[]byte(fmt.Sprintf(`"severity":%d}`, severity)),
}
}
var checkNeedles [][]byte
activeChecks := 0
for _, enabled := range checks {
if enabled {
activeChecks++
}
}
if checks != nil && activeChecks == 0 {
return func([]byte) bool { return false }
}
for name, enabled := range checks {
if !enabled {
continue
}
// The empty check also matches an absent or null field.
if name == "" || !storedVerbatim(name) {
checkNeedles = nil
break
}
checkNeedles = append(checkNeedles, []byte(`"check":"`+name+`"`))
}
var searchNeedle []byte
if searchLower != "" && storedVerbatim(searchLower) {
searchNeedle = []byte(searchLower)
}
return func(v []byte) bool {
if sevNeedles != nil && !bytes.Contains(v, sevNeedles[0]) && !bytes.Contains(v, sevNeedles[1]) {
// Zero also accepts omitted/null fields and negative zero. Keep
// the fast rejection for ordinary nonzero severity values.
if severity == 0 && (!bytes.Contains(v, []byte(`"severity":`)) ||
bytes.Contains(v, []byte(`"severity":null`)) || bytes.Contains(v, []byte(`"severity":-0`))) {
return true
}
return !plainHistoryJSON(v)
}
if checks != nil && checkNeedles != nil && !containsAnyBytes(v, checkNeedles) {
return !plainHistoryJSON(v)
}
if searchNeedle != nil && !bytes.Contains(bytes.ToLower(v), searchNeedle) {
return !plainHistoryJSON(v)
}
return true
}
}
// Raw needles are conclusive only for unescaped compact JSON with canonical
// field names. Other valid encodings (including case-insensitive JSON keys)
// are left to the decoder. This is an eligibility check, not a JSON validator.
func plainHistoryJSON(v []byte) bool {
if len(v) < 2 || v[0] != '{' || v[len(v)-1] != '}' {
return false
}
start := -1
for i, b := range v {
if b == '\\' {
return false
}
if b == '"' {
if start < 0 {
start = i + 1
} else {
if i+1 < len(v) && v[i+1] == ':' {
for _, c := range v[start:i] {
if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '_' {
return false
}
}
}
start = -1
}
} else if start < 0 && (b == ' ' || b == '\t' || b == '\n' || b == '\r') {
return false
}
}
return true
}
// storedVerbatim reports whether encoding/json writes s unchanged inside a
// JSON string.
func storedVerbatim(s string) bool {
for _, r := range s {
if r < 0x20 || r == '"' || r == '\\' || r == '<' || r == '>' || r == '&' || r == '\u2028' || r == '\u2029' || r == utf8.RuneError {
return false
}
}
return true
}
func containsAnyBytes(v []byte, needles [][]byte) bool {
for _, n := range needles {
if bytes.Contains(v, n) {
return true
}
}
return false
}
// substr must already be lowercase.
func containsLower(s, substr string) bool {
return strings.Contains(strings.ToLower(s), substr)
}
package store
import (
"encoding/json"
"errors"
"fmt"
"sort"
"time"
bolt "go.etcd.io/bbolt"
"github.com/pidginhost/csm/internal/incident"
csmlog "github.com/pidginhost/csm/internal/log"
)
const incidentsBucket = "incidents"
// Bound rows inspected as well as deleted: retained or corrupt rows must
// not turn a sweep into one long transaction on bbolt's shared writer.
const incidentCompactionBatchSize = 256
// SaveIncident persists an incident, overwriting any prior record with
// the same ID. Caller is responsible for setting UpdatedAt before
// invoking; this method just writes.
func (db *DB) SaveIncident(inc incident.Incident) error {
data, err := json.Marshal(inc)
if err != nil {
return err
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(incidentsBucket))
return b.Put([]byte(inc.ID), data)
})
}
// GetIncident returns (incident, true, nil) if found, (zero, false, nil)
// if not, (zero, false, err) on store error.
func (db *DB) GetIncident(id string) (incident.Incident, bool, error) {
var (
inc incident.Incident
found bool
)
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(incidentsBucket))
v := b.Get([]byte(id))
if v == nil {
return nil
}
if err := json.Unmarshal(v, &inc); err != nil {
return err
}
if err := validateIncidentRow(id, inc); err != nil {
return err
}
found = true
return nil
})
return inc, found, err
}
// ListIncidents returns every stored incident, newest UpdatedAt first.
// Rows that fail JSON decode or storage invariants are skipped with a
// warn log so the rest of the bucket is still restorable. Aborting on
// the first bad row would leave the daemon with no restored incidents
// at startup.
func (db *DB) ListIncidents() ([]incident.Incident, error) {
var out []incident.Incident
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(incidentsBucket))
return b.ForEach(func(k, v []byte) error {
inc, ok := decodeIncidentRow(k, v)
if !ok {
return nil
}
out = append(out, inc)
return nil
})
})
if err != nil {
return nil, err
}
sort.Slice(out, func(i, j int) bool {
return out[i].UpdatedAt.After(out[j].UpdatedAt)
})
return out, nil
}
// ListIncidentsByStatus returns incidents matching the requested status,
// newest UpdatedAt first.
func (db *DB) ListIncidentsByStatus(status incident.Status) ([]incident.Incident, error) {
all, err := db.ListIncidents()
if err != nil {
return nil, err
}
out := all[:0]
for _, inc := range all {
if inc.Status == status {
out = append(out, inc)
}
}
return out, nil
}
// CompactIncidents removes resolved/dismissed incidents that have outlived
// their retention. Open and Contained incidents are never pruned regardless
// of age. Each transaction inspects a bounded batch, letting other store
// writers run during a large backlog. On error the count includes only
// committed deletions from preceding batches.
func (db *DB) CompactIncidents(now time.Time, retention incident.ClosedRetention) (int, error) {
pruned := 0
var next []byte
for {
n, resume, err := db.compactIncidentsBatch(now, retention, next)
pruned += n
if err != nil || resume == nil {
return pruned, err
}
next = resume
}
}
// The resume key is inclusive and copied before the transaction ends.
// Selection and deletion share a transaction so a concurrent operator
// update cannot be deleted using a stale retention decision.
func (db *DB) compactIncidentsBatch(now time.Time, retention incident.ClosedRetention, start []byte) (int, []byte, error) {
var next []byte
var toDelete [][]byte
err := db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(incidentsBucket))
cursor := b.Cursor()
k, v := cursor.First()
if start != nil {
k, v = cursor.Seek(start)
}
for scanned := 0; k != nil && scanned < incidentCompactionBatchSize; scanned++ {
inc, ok := decodeIncidentRow(k, v)
if ok && retention.Expired(inc, now) {
toDelete = append(toDelete, append([]byte(nil), k...))
}
k, v = cursor.Next()
}
if k != nil {
next = append([]byte(nil), k...)
}
for _, k := range toDelete {
if err := b.Delete(k); err != nil {
return err
}
}
return nil
})
if err != nil {
return 0, nil, err
}
return len(toDelete), next, nil
}
func decodeIncidentRow(k, v []byte) (incident.Incident, bool) {
rowID := string(k)
var inc incident.Incident
if err := json.Unmarshal(v, &inc); err != nil {
warnSkippedIncidentRow(rowID, err)
return incident.Incident{}, false
}
if err := validateIncidentRow(rowID, inc); err != nil {
warnSkippedIncidentRow(rowID, err)
return incident.Incident{}, false
}
return inc, true
}
func validateIncidentRow(rowID string, inc incident.Incident) error {
if rowID == "" {
return errors.New("empty row key")
}
if inc.ID == "" {
return errors.New("empty incident id")
}
if inc.ID != rowID {
return fmt.Errorf("incident id %q does not match row key %q", inc.ID, rowID)
}
if !incidentStatusValid(inc.Status) {
return fmt.Errorf("invalid status %q", inc.Status)
}
return nil
}
func incidentStatusValid(status incident.Status) bool {
switch status {
case incident.StatusOpen, incident.StatusContained, incident.StatusResolved, incident.StatusDismissed:
return true
default:
return false
}
}
func warnSkippedIncidentRow(rowID string, err error) {
csmlog.Warn("store: skipping corrupt incident row",
"bucket", incidentsBucket, "id", rowID, "err", err)
}
package store
import (
"container/heap"
"encoding/json"
"time"
bolt "go.etcd.io/bbolt"
)
// knownCustomersBucket maps an address to the last successful customer login
// seen from it.
const knownCustomersBucket = "customer:logins"
// maxKnownCustomerRecords bounds a load: the daemon's set and as many again
// waiting for deletion.
const maxKnownCustomerRecords = 100000
// A record names one address and holds one bounded mailbox name. Anything
// larger is damage and is not decoded.
const (
maxKnownCustomerKeyBytes = 64
maxKnownCustomerValueBytes = 2048
)
// KnownCustomer is the last successful customer login from one address.
// Syslog marks a login read from syslog rather than from a log only the
// service writes.
type KnownCustomer struct {
Kind string `json:"kind"`
Account string `json:"account,omitempty"`
At time.Time `json:"at"`
Syslog bool `json:"syslog,omitempty"`
}
// SaveKnownCustomers writes upserts and removes deletes in one transaction.
func (db *DB) SaveKnownCustomers(upserts map[string]KnownCustomer, deletes []string) error {
if len(upserts)+len(deletes) == 0 {
return nil
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(knownCustomersBucket))
if err != nil {
return err
}
for _, ip := range deletes {
if ip == "" {
continue
}
if err := b.Delete([]byte(ip)); err != nil {
return err
}
}
for ip, c := range upserts {
if ip == "" {
continue
}
val, err := json.Marshal(c)
if err != nil {
return err
}
if err := b.Put([]byte(ip), val); err != nil {
return err
}
}
return nil
})
}
// LoadKnownCustomers returns the newest readable records, at most
// maxKnownCustomerRecords. Validating addresses and ages is the caller's job.
// Malformed, oversized and older records are deleted, so the bucket stays
// bounded; the records are still returned when that deletion fails.
func (db *DB) LoadKnownCustomers() (map[string]KnownCustomer, error) {
var kept knownCustomerHeap
var drop [][]byte
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(knownCustomersBucket))
if b == nil {
return nil
}
return b.ForEach(func(k, v []byte) error {
c, ok := decodeKnownCustomer(k, v)
if !ok {
drop = append(drop, append([]byte(nil), k...))
return nil
}
heap.Push(&kept, knownCustomerRecord{string(k), c})
if kept.Len() > maxKnownCustomerRecords {
drop = append(drop, []byte(heap.Pop(&kept).(knownCustomerRecord).ip))
}
return nil
})
})
if err != nil {
return nil, err
}
out := make(map[string]KnownCustomer, kept.Len())
for _, r := range kept {
out[r.ip] = r.c
}
if len(drop) > 0 {
err = db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(knownCustomersBucket))
for _, k := range drop {
if derr := b.Delete(k); derr != nil {
return derr
}
}
return nil
})
}
return out, err
}
// decodeKnownCustomer reads one record, refusing one too large to be ours.
func decodeKnownCustomer(k, v []byte) (KnownCustomer, bool) {
var c KnownCustomer
if len(k) > maxKnownCustomerKeyBytes || len(v) > maxKnownCustomerValueBytes {
return c, false
}
return c, json.Unmarshal(v, &c) == nil
}
type knownCustomerRecord struct {
ip string
c KnownCustomer
}
// knownCustomerHeap keeps the oldest record on top, so a full heap drops it.
type knownCustomerHeap []knownCustomerRecord
func (h knownCustomerHeap) Len() int { return len(h) }
func (h knownCustomerHeap) Less(i, j int) bool {
if c := h[i].c.At.Compare(h[j].c.At); c != 0 {
return c < 0
}
return h[i].ip < h[j].ip
}
func (h knownCustomerHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *knownCustomerHeap) Push(x any) { *h = append(*h, x.(knownCustomerRecord)) }
func (h *knownCustomerHeap) Pop() any {
old := *h
last := old[len(old)-1]
*h = old[:len(old)-1]
return last
}
package store
import (
"encoding/json"
"fmt"
"time"
"github.com/pidginhost/csm/internal/alert"
bolt "go.etcd.io/bbolt"
)
// stats:latest_by_check maps a check name to the timestamp of its newest
// history entry. AppendHistory updates it in the same transaction, so asking
// when a check last fired reads one small bucket instead of decoding up to
// maxHistoryEntries findings.
const (
bucketLatestByCheck = "stats:latest_by_check"
metaLatestByCheckBackfilled = "stats:latest_by_check:backfilled"
)
// bumpLatestByCheck records t for check unless a newer time is already
// stored. It runs inside the caller's write transaction.
func bumpLatestByCheck(tx *bolt.Tx, check string, t time.Time) error {
if check == "" {
return nil
}
b := tx.Bucket([]byte(bucketLatestByCheck))
if b == nil {
return fmt.Errorf("bucket %s missing", bucketLatestByCheck)
}
if v := b.Get([]byte(check)); v != nil {
if prev, err := time.Parse(time.RFC3339Nano, string(v)); err == nil && !t.After(prev) {
return nil
}
}
return b.Put([]byte(check), []byte(t.UTC().Format(time.RFC3339Nano)))
}
// LatestByCheck returns the timestamp of the newest history entry of every
// check that has written history.
func (db *DB) LatestByCheck() map[string]time.Time {
out := map[string]time.Time{}
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucketLatestByCheck))
if b == nil {
return nil
}
return b.ForEach(func(k, v []byte) error {
if t, err := time.Parse(time.RFC3339Nano, string(v)); err == nil {
out[string(k)] = t
}
return nil
})
})
return out
}
// BackfillLatestByCheck seeds stats:latest_by_check from history once, on
// the first start after an upgrade. A meta sentinel makes it a no-op after.
func (db *DB) BackfillLatestByCheck() error {
var done bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
done = tx.Bucket([]byte("meta")).Get([]byte(metaLatestByCheckBackfilled)) != nil
return nil
})
if done {
return nil
}
latest := map[string]time.Time{}
if err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("history"))
if b == nil {
return nil
}
return b.ForEach(func(_, v []byte) error {
// A corrupt entry is skipped, as every history reader does.
var f alert.Finding
if json.Unmarshal(v, &f) == nil && f.Check != "" && f.Timestamp.After(latest[f.Check]) {
latest[f.Check] = f.Timestamp
}
return nil
})
}); err != nil {
return err
}
return db.bolt.Update(func(tx *bolt.Tx) error {
for check, t := range latest {
if err := bumpLatestByCheck(tx, check, t); err != nil {
return err
}
}
return tx.Bucket([]byte("meta")).Put([]byte(metaLatestByCheckBackfilled), []byte(time.Now().UTC().Format(time.RFC3339)))
})
}
package store
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
bolt "go.etcd.io/bbolt"
)
// Suspension kinds and the parts a mail server applied.
const (
SuspensionKindMailbox = "mailbox" // one mailbox's logins and/or outgoing mail
SuspensionKindAccount = "account" // a hosting account's outgoing mail hold
AppliedSuspendLogin = "suspend_login"
AppliedSuspendOutgoing = "suspend_outgoing"
AppliedHoldOutgoing = "hold_outgoing"
AppliedDisableSignIn = "disable_signin"
)
const mailboxSuspensionsBucket = "mail:suspensions"
// mailboxSuspensionRetention keeps a released row visible for context before
// the next write prunes it.
const mailboxSuspensionRetention = 30 * 24 * time.Hour
// ErrMailboxSuspensionNotFound reports an ID with no stored suspension.
var ErrMailboxSuspensionNotFound = errors.New("mailbox suspension not found")
// MailboxSuspension records one suspension CSM made, so it can be listed and
// released. Subject is the identity the trigger named: a mailbox, a bare
// hosting account, or a domain whose owner was held.
type MailboxSuspension struct {
ID string `json:"id"`
Backend string `json:"backend"`
Subject string `json:"subject"`
Kind string `json:"kind"`
Account string `json:"account,omitempty"`
Ref string `json:"ref,omitempty"` // the mail server's id for what was suspended
Applied []string `json:"applied"`
Check string `json:"check"`
Reason string `json:"reason"`
SuspendedAt time.Time `json:"suspended_at"`
ReleasedAt *time.Time `json:"released_at,omitempty"`
ReleasedBy string `json:"released_by,omitempty"`
}
// MailboxSuspensionID is the stable key of a subject on one mail server.
// Mail identities compare case-insensitively.
func MailboxSuspensionID(backend, subject string) string {
sum := sha256.Sum256([]byte(backend + "\x00" + strings.ToLower(subject)))
return hex.EncodeToString(sum[:8])
}
// PutMailboxSuspension upserts s by its ID and prunes released rows older
// than the retention period.
func (db *DB) PutMailboxSuspension(s MailboxSuspension) error {
payload, err := json.Marshal(s)
if err != nil {
return err
}
cutoff := time.Now().Add(-mailboxSuspensionRetention)
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(mailboxSuspensionsBucket))
var stale [][]byte
if err := b.ForEach(func(k, v []byte) error {
var old MailboxSuspension
if json.Unmarshal(v, &old) == nil && old.ReleasedAt != nil && old.ReleasedAt.Before(cutoff) {
stale = append(stale, append([]byte(nil), k...))
}
return nil
}); err != nil {
return err
}
for _, k := range stale {
if err := b.Delete(k); err != nil {
return err
}
}
return b.Put([]byte(s.ID), payload)
})
}
// GetMailboxSuspension returns the suspension stored under id.
func (db *DB) GetMailboxSuspension(id string) (MailboxSuspension, bool, error) {
var s MailboxSuspension
var found bool
err := db.bolt.View(func(tx *bolt.Tx) error {
v := tx.Bucket([]byte(mailboxSuspensionsBucket)).Get([]byte(id))
if v == nil {
return nil
}
found = true
return json.Unmarshal(v, &s)
})
return s, found, err
}
// ListMailboxSuspensions returns every stored suspension, newest first.
func (db *DB) ListMailboxSuspensions() ([]MailboxSuspension, error) {
var out []MailboxSuspension
err := db.bolt.View(func(tx *bolt.Tx) error {
return tx.Bucket([]byte(mailboxSuspensionsBucket)).ForEach(func(k, v []byte) error {
var s MailboxSuspension
if err := json.Unmarshal(v, &s); err != nil {
return fmt.Errorf("mailbox suspension %s: %w", k, err)
}
out = append(out, s)
return nil
})
})
sort.SliceStable(out, func(i, j int) bool { return out[i].SuspendedAt.After(out[j].SuspendedAt) })
return out, err
}
// MarkMailboxSuspensionReleased stamps the release time and who released it.
func (db *DB) MarkMailboxSuspensionReleased(id string, at time.Time, by string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(mailboxSuspensionsBucket))
v := b.Get([]byte(id))
if v == nil {
return fmt.Errorf("%w: %s", ErrMailboxSuspensionNotFound, id)
}
var s MailboxSuspension
if err := json.Unmarshal(v, &s); err != nil {
return err
}
s.ReleasedAt, s.ReleasedBy = &at, by
payload, err := json.Marshal(s)
if err != nil {
return err
}
return b.Put([]byte(id), payload)
})
}
package store
import (
"encoding/json"
"errors"
"time"
bolt "go.etcd.io/bbolt"
bolterrors "go.etcd.io/bbolt/errors"
)
const mailGoodSourceBucket = "mail:good_source"
// GoodSourcePair is the persisted established-sender window for one (IP,
// mailbox): the earliest and most recent successful auth. Persisting it lets
// established good standing survive a daemon restart so the post-restart
// cold-start does not re-open the mail brute-force false-positive window.
type GoodSourcePair struct {
First time.Time `json:"first"`
Last time.Time `json:"last"`
}
// SaveMailGoodSource replaces the persisted mail good-source snapshot with data.
// Snapshot semantics: the bucket is fully rewritten so IPs no longer present are
// dropped, keeping the persisted set in step with the live tracker.
func (db *DB) SaveMailGoodSource(data map[string]map[string]GoodSourcePair) error {
encoded := make(map[string][]byte, len(data))
for ip, accts := range data {
if ip == "" || len(accts) == 0 {
continue
}
val, err := json.Marshal(accts)
if err != nil {
return err
}
encoded[ip] = val
}
return db.bolt.Update(func(tx *bolt.Tx) error {
if err := tx.DeleteBucket([]byte(mailGoodSourceBucket)); err != nil && !errors.Is(err, bolterrors.ErrBucketNotFound) {
return err
}
b, err := tx.CreateBucket([]byte(mailGoodSourceBucket))
if err != nil {
return err
}
for ip, val := range encoded {
if perr := b.Put([]byte(ip), val); perr != nil {
return perr
}
}
return nil
})
}
// LoadMailGoodSource returns the persisted mail good-source snapshot keyed by
// IP. Returns an empty map when nothing has been persisted yet.
func (db *DB) LoadMailGoodSource() (map[string]map[string]GoodSourcePair, error) {
out := make(map[string]map[string]GoodSourcePair)
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(mailGoodSourceBucket))
if b == nil {
return nil
}
return b.ForEach(func(k, v []byte) error {
var accts map[string]GoodSourcePair
if json.Unmarshal(v, &accts) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
out[string(k)] = accts
return nil
})
})
return out, err
}
package store
import (
"bytes"
"encoding/json"
"errors"
"time"
bolt "go.etcd.io/bbolt"
bolterrors "go.etcd.io/bbolt/errors"
)
const mailSprayBucket = "mail:spray"
// Key prefixes in mailSprayBucket: one entry per evidence IP, per trusted
// IP, per corroborated mailbox name, and per address blocked for spraying.
const (
mailSprayIPPrefix = "ip:"
mailSprayTrustPrefix = "trust:"
mailSprayNamePrefix = "name:"
mailSprayBlockPrefix = "blk:"
)
// MailSprayFailure is one failed mailbox name of one IP.
type MailSprayFailure struct {
First time.Time `json:"first"`
Last time.Time `json:"last"`
Count int `json:"count"`
Exist uint8 `json:"exist,omitzero"`
}
// MailSpraySource is the verdict cadence of one address.
type MailSpraySource struct {
SuppressedUntil time.Time `json:"suppressed_until,omitzero"`
CompromiseUntil time.Time `json:"compromise_until,omitzero"`
}
// MailSprayIP is the persisted failure evidence of one IPv4 address or IPv6
// /64.
type MailSprayIP struct {
Failed map[string]MailSprayFailure `json:"failed,omitempty"`
WeakAlerted map[string]time.Time `json:"weak_alerted,omitempty"`
Revoked map[string]time.Time `json:"revoked,omitempty"` // cracked mailboxes that cannot earn trust
Sources map[string]MailSpraySource `json:"sources,omitempty"` // by address
Untracked time.Time `json:"untracked,omitzero"`
RevokedOverflow time.Time `json:"revoked_overflow,omitzero"`
VerdictAt time.Time `json:"verdict_at,omitzero"`
ObservedAt time.Time `json:"observed_at,omitzero"` // last observe mode block verdict
ObservedNames int `json:"observed_names,omitzero"` // untrusted names it listed
LastSeen time.Time `json:"last_seen"`
}
// MailSprayState is the persisted mail login spray memory. Keeping it across
// restarts stops a nightly restart from wiping a week of slow spray evidence
// and the clean-login trust that exempts real users.
type MailSprayState struct {
IPs map[string]MailSprayIP // failure evidence by IPv4 address or IPv6 /64
Trust map[string]map[string]GoodSourcePair // clean logins by IP, then mailbox
Networks map[string]map[string]time.Time // missing name to the networks that tried it
Blocked map[string]time.Time // spray block end by address, never by /64
}
// MailSprayDelta is a change to the persisted spray memory: entries to
// write, keyed as in MailSprayState, and keys to delete.
type MailSprayDelta struct {
Put MailSprayState
DeleteIPs []string
DeleteTrust []string
DeleteNames []string
DeleteBlocked []string
}
// Empty reports whether the delta changes nothing.
func (d MailSprayDelta) Empty() bool {
return len(d.Put.IPs)+len(d.Put.Trust)+len(d.Put.Networks)+len(d.Put.Blocked)+
len(d.DeleteIPs)+len(d.DeleteTrust)+len(d.DeleteNames)+len(d.DeleteBlocked) == 0
}
// SaveMailSpray replaces the persisted spray memory with st. The bucket is
// rewritten in one transaction so entries the detector has dropped do not
// come back at the next load.
func (db *DB) SaveMailSpray(st MailSprayState) error {
encoded, err := encodeMailSpray(st)
if err != nil {
return err
}
return db.bolt.Update(func(tx *bolt.Tx) error {
if err := tx.DeleteBucket([]byte(mailSprayBucket)); err != nil && !errors.Is(err, bolterrors.ErrBucketNotFound) {
return err
}
b, err := tx.CreateBucket([]byte(mailSprayBucket))
if err != nil {
return err
}
return putMailSpray(b, encoded)
})
}
// ApplyMailSprayDelta writes and deletes only the delta's keys in one
// transaction, leaving the rest of the bucket in place.
func (db *DB) ApplyMailSprayDelta(d MailSprayDelta) error {
encoded, err := encodeMailSpray(d.Put)
if err != nil {
return err
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(mailSprayBucket))
if err != nil {
return err
}
for prefix, keys := range map[string][]string{
mailSprayIPPrefix: d.DeleteIPs,
mailSprayTrustPrefix: d.DeleteTrust,
mailSprayNamePrefix: d.DeleteNames,
mailSprayBlockPrefix: d.DeleteBlocked,
} {
for _, k := range keys {
if k == "" {
continue
}
if derr := b.Delete([]byte(prefix + k)); derr != nil {
return derr
}
}
}
return putMailSpray(b, encoded)
})
}
func encodeMailSpray(st MailSprayState) (map[string][]byte, error) {
encoded := make(map[string][]byte, len(st.IPs)+len(st.Trust)+len(st.Networks)+len(st.Blocked))
put := func(prefix, key string, v any) error {
if key == "" {
return nil
}
val, err := json.Marshal(v)
if err != nil {
return err
}
encoded[prefix+key] = val
return nil
}
for ip, e := range st.IPs {
if err := put(mailSprayIPPrefix, ip, e); err != nil {
return nil, err
}
}
for ip, t := range st.Trust {
if err := put(mailSprayTrustPrefix, ip, t); err != nil {
return nil, err
}
}
for name, n := range st.Networks {
if err := put(mailSprayNamePrefix, name, n); err != nil {
return nil, err
}
}
for ip, end := range st.Blocked {
if err := put(mailSprayBlockPrefix, ip, end); err != nil {
return nil, err
}
}
return encoded, nil
}
func putMailSpray(b *bolt.Bucket, encoded map[string][]byte) error {
for k, val := range encoded {
if err := b.Put([]byte(k), val); err != nil {
return err
}
}
return nil
}
// LoadMailSpray visits one saved entry at a time: trust, evidence,
// corroboration, then block markers.
// maxEntryBytes bounds JSON decoding before allocation. Malformed and oversized
// entries are skipped so one damaged entry cannot prevent the rest loading.
// The visitor must not write to the store while the read transaction is open.
func (db *DB) LoadMailSpray(maxEntryBytes int, visit func(MailSprayState)) error {
return db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(mailSprayBucket))
if b == nil {
return nil
}
for _, prefix := range []string{mailSprayTrustPrefix, mailSprayIPPrefix, mailSprayNamePrefix, mailSprayBlockPrefix} {
c := b.Cursor()
for k, v := c.Seek([]byte(prefix)); k != nil && bytes.HasPrefix(k, []byte(prefix)); k, v = c.Next() {
if len(k) == len(prefix) || len(k) > maxEntryBytes || len(v) > maxEntryBytes {
continue
}
key := string(k[len(prefix):])
switch prefix {
case mailSprayIPPrefix:
var e MailSprayIP
if json.Unmarshal(v, &e) == nil {
visit(MailSprayState{IPs: map[string]MailSprayIP{key: e}})
}
case mailSprayTrustPrefix:
var t map[string]GoodSourcePair
if json.Unmarshal(v, &t) == nil {
visit(MailSprayState{Trust: map[string]map[string]GoodSourcePair{key: t}})
}
case mailSprayNamePrefix:
var n map[string]time.Time
if json.Unmarshal(v, &n) == nil {
visit(MailSprayState{Networks: map[string]map[string]time.Time{key: n}})
}
case mailSprayBlockPrefix:
var end time.Time
if json.Unmarshal(v, &end) == nil {
visit(MailSprayState{Blocked: map[string]time.Time{key: end}})
}
}
}
}
return nil
})
}
package store
import (
"bytes"
"encoding/binary"
bolt "go.etcd.io/bbolt"
)
// The mail decision counter of the Email Security page. Stats and gap keys
// are unix seconds; a gap is keyed by its end, so pruning by key drops only
// gaps that ended before the cutoff. Ring keys are reject classes.
const (
mailStatsBucket = "mail:stats"
mailRejectsBucket = "mail:rejects"
mailGapsBucket = "mail:gaps"
)
func unixKey(sec int64) []byte {
return binary.BigEndian.AppendUint64(nil, uint64(sec)) // #nosec G115 -- unix seconds since 1970 are non-negative, so key order is time order
}
func keyUnix(k []byte) int64 {
return int64(binary.BigEndian.Uint64(k)) // #nosec G115 -- reverses unixKey
}
// PutMailStats writes the given counter buckets and reject rings, each ring
// replacing the class's stored ring, and deletes stats buckets and gaps
// keyed below pruneBefore.
func (db *DB) PutMailStats(buckets map[int64][]byte, rejects map[string][]byte, pruneBefore int64) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
stats := tx.Bucket([]byte(mailStatsBucket))
for k, v := range buckets {
if err := stats.Put(unixKey(k), v); err != nil {
return err
}
}
rings := tx.Bucket([]byte(mailRejectsBucket))
for class, v := range rejects {
if err := rings.Put([]byte(class), v); err != nil {
return err
}
}
cutoff := unixKey(pruneBefore)
for _, b := range []*bolt.Bucket{stats, tx.Bucket([]byte(mailGapsBucket))} {
// Re-seek after each delete: Next skips keys once this
// transaction has written to the bucket.
c := b.Cursor()
for k, _ := c.First(); k != nil && bytes.Compare(k, cutoff) < 0; k, _ = c.First() {
if err := c.Delete(); err != nil {
return err
}
}
}
return nil
})
}
// MailStats returns the stored counter buckets by start and reject rings by
// class.
func (db *DB) MailStats() (buckets map[int64][]byte, rejects map[string][]byte, err error) {
buckets, rejects = map[int64][]byte{}, map[string][]byte{}
err = db.bolt.View(func(tx *bolt.Tx) error {
if statsErr := tx.Bucket([]byte(mailStatsBucket)).ForEach(func(k, v []byte) error {
if len(k) == 8 {
buckets[keyUnix(k)] = bytes.Clone(v)
}
return nil
}); statsErr != nil {
return statsErr
}
return tx.Bucket([]byte(mailRejectsBucket)).ForEach(func(k, v []byte) error {
rejects[string(k)] = bytes.Clone(v)
return nil
})
})
return buckets, rejects, err
}
// AddMailStatsGap stores a stretch the counter did not see.
func (db *DB) AddMailStatsGap(start, end int64) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
gaps := tx.Bucket([]byte(mailGapsBucket))
key := unixKey(end)
// Rapid restarts can end in the same stored second. A later stop
// must not shorten the outage already recorded under that key.
if old := gaps.Get(key); len(old) == 8 {
start = min(start, keyUnix(old))
}
return gaps.Put(key, unixKey(start))
})
}
// MailStatsGaps returns the stored gaps that end after since as
// [start, end] pairs, oldest first.
func (db *DB) MailStatsGaps(since int64) ([][2]int64, error) {
var gaps [][2]int64
err := db.bolt.View(func(tx *bolt.Tx) error {
c := tx.Bucket([]byte(mailGapsBucket)).Cursor()
for k, v := c.Seek(unixKey(since + 1)); k != nil; k, v = c.Next() {
if len(k) == 8 && len(v) == 8 {
gaps = append(gaps, [2]int64{keyUnix(v), keyUnix(k)})
}
}
return nil
})
return gaps, err
}
package store
import (
"bufio"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
bolt "go.etcd.io/bbolt"
)
// All os.Open / os.ReadFile calls below take paths derived from the
// operator-configured statePath (root-owned /opt/csm or /var/lib/csm
// by default) joined with fixed filenames. gosec G304 suppressions on
// each line refer back to this package-level trust model.
func (db *DB) runMigration(statePath string) error {
fmt.Fprintf(os.Stderr, "store: migrating flat files to bbolt...\n")
var errs []string
if err := db.migrateHistory(statePath); err != nil {
errs = append(errs, fmt.Sprintf("history: %v", err))
}
if err := db.migrateAttackDB(statePath); err != nil {
errs = append(errs, fmt.Sprintf("attackdb: %v", err))
}
if err := db.migrateThreatDB(statePath); err != nil {
errs = append(errs, fmt.Sprintf("threatdb: %v", err))
}
if err := db.migrateReputation(statePath); err != nil {
errs = append(errs, fmt.Sprintf("reputation: %v", err))
}
if len(errs) > 0 {
return fmt.Errorf("partial migration: %s", strings.Join(errs, "; "))
}
_ = db.bolt.Update(func(tx *bolt.Tx) error {
return tx.Bucket([]byte("meta")).Put([]byte("migrated"), []byte(time.Now().Format(time.RFC3339)))
})
fmt.Fprintf(os.Stderr, "store: migration complete\n")
return nil
}
func (db *DB) migrateHistory(statePath string) error {
path := filepath.Join(statePath, "history.jsonl")
if _, err := os.Stat(path); err != nil {
return nil
}
// #nosec G304 -- see runMigration trust note.
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
var findings []alert.Finding
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 1024*1024), 1024*1024)
for scanner.Scan() {
var finding alert.Finding
if err := json.Unmarshal(scanner.Bytes(), &finding); err != nil {
continue
}
findings = append(findings, finding)
}
if len(findings) > 0 {
if err := db.AppendHistory(findings); err != nil {
return err
}
}
renameToBackup(path)
fmt.Fprintf(os.Stderr, "store: migrated %d history entries\n", len(findings))
return nil
}
func (db *DB) migrateAttackDB(statePath string) error {
dbDir := filepath.Join(statePath, "attack_db")
// records.json
recordsPath := filepath.Join(dbDir, "records.json")
if _, err := os.Stat(recordsPath); err == nil {
// #nosec G304 -- see runMigration trust note.
data, err := os.ReadFile(recordsPath)
if err != nil {
return err
}
var records map[string]*IPRecord
if err := json.Unmarshal(data, &records); err != nil {
return err
}
// One transaction: a sync per record takes minutes on a large file.
if err := db.UpdateIPRecords(func(w *IPRecordTx) error {
for _, r := range records {
if saveErr := w.Save(*r); saveErr != nil {
return saveErr
}
}
return nil
}); err != nil {
return err
}
renameToBackup(recordsPath)
fmt.Fprintf(os.Stderr, "store: migrated %d attack records\n", len(records))
}
// events.jsonl
eventsPath := filepath.Join(dbDir, "events.jsonl")
if _, err := os.Stat(eventsPath); err == nil {
// #nosec G304 -- see runMigration trust note.
f, err := os.Open(eventsPath)
if err != nil {
return err
}
defer f.Close()
counter := 0
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 1024*1024), 1024*1024)
for scanner.Scan() {
var event AttackEvent
if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
continue
}
if err := db.RecordAttackEvent(event, counter); err != nil {
return err
}
counter++
}
renameToBackup(eventsPath)
fmt.Fprintf(os.Stderr, "store: migrated %d attack events\n", counter)
}
return nil
}
func (db *DB) migrateThreatDB(statePath string) error {
dbDir := filepath.Join(statePath, "threat_db")
// permanent.txt
permPath := filepath.Join(dbDir, "permanent.txt")
if _, err := os.Stat(permPath); err == nil {
// #nosec G304 -- see runMigration trust note.
data, err := os.ReadFile(permPath)
if err != nil {
return err
}
count := 0
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
parts := strings.SplitN(line, " # ", 2)
ip := strings.TrimSpace(parts[0])
reason := ""
if len(parts) > 1 {
reason = strings.TrimSpace(parts[1])
}
if ip != "" {
// Migrated rows keep an empty Source so Expired can still
// classify them by reason text: the flat file mixes operator
// adds with old temp auto-block writes, and stamping them all
// as operator would make the auto-block ones permanent.
if err := db.putThreatEntry(PermanentBlockEntry{
IP: ip,
Reason: reason,
BlockedAt: time.Now(),
}); err != nil {
return err
}
count++
}
}
renameToBackup(permPath)
fmt.Fprintf(os.Stderr, "store: migrated %d permanent blocks\n", count)
}
// whitelist.txt
wlPath := filepath.Join(dbDir, "whitelist.txt")
if _, err := os.Stat(wlPath); err == nil {
// #nosec G304 -- see runMigration trust note.
data, err := os.ReadFile(wlPath)
if err != nil {
return err
}
count := 0
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
parts := strings.Fields(line)
ip := parts[0]
permanent := true
var expiresAt time.Time
for _, p := range parts[1:] {
if strings.HasPrefix(p, "expires=") {
t, err := time.Parse(time.RFC3339, strings.TrimPrefix(p, "expires="))
if err == nil {
expiresAt = t
permanent = false
}
}
if p == "permanent" {
permanent = true
}
}
if err := db.AddWhitelistEntry(ip, expiresAt, permanent); err != nil {
return err
}
count++
}
renameToBackup(wlPath)
fmt.Fprintf(os.Stderr, "store: migrated %d whitelist entries\n", count)
}
return nil
}
func (db *DB) migrateReputation(statePath string) error {
path := filepath.Join(statePath, "reputation_cache.json")
if _, serr := os.Stat(path); serr != nil {
return nil //nolint:nilerr // file does not exist, skip migration
}
// #nosec G304 -- see runMigration trust note.
data, err := os.ReadFile(path)
if err != nil {
return err
}
type rawCache struct {
Entries map[string]*ReputationEntry `json:"entries"`
}
var cache rawCache
if err := json.Unmarshal(data, &cache); err != nil {
return err
}
count := 0
for ip, entry := range cache.Entries {
if err := db.SetReputation(ip, *entry); err != nil {
return err
}
count++
}
renameToBackup(path)
fmt.Fprintf(os.Stderr, "store: migrated %d reputation entries\n", count)
return nil
}
func renameToBackup(path string) {
_ = os.Rename(path, path+".bak")
}
package store
import (
"bytes"
"encoding/json"
"fmt"
"strconv"
"time"
bolt "go.etcd.io/bbolt"
)
const modsecNoEscalateKey = "modsec:no_escalate_rules"
// GetModSecNoEscalateRules returns the set of ModSecurity rule IDs that should
// NOT escalate to nftables firewall blocks. Stored in the meta bucket.
// An unreadable list is not an empty list: callers must not authorize firewall
// escalation or report all rules enabled when the operator's choices are unknown.
func (db *DB) GetModSecNoEscalateRules() (map[int]bool, error) {
rules := make(map[int]bool)
err := db.bolt.View(func(tx *bolt.Tx) error {
v := tx.Bucket([]byte("meta")).Get([]byte(modsecNoEscalateKey))
if v == nil {
return nil
}
var ids []int
if err := json.Unmarshal(v, &ids); err != nil {
return err
}
for _, id := range ids {
rules[id] = true
}
return nil
})
if err != nil {
return nil, fmt.Errorf("read ModSecurity escalation exclusions: %w", err)
}
return rules, nil
}
// SetModSecNoEscalateRules stores the set of rule IDs that should not escalate.
func (db *DB) SetModSecNoEscalateRules(rules map[int]bool) error {
var ids []int
for id := range rules {
ids = append(ids, id)
}
return db.bolt.Update(func(tx *bolt.Tx) error {
val, err := json.Marshal(ids)
if err != nil {
return err
}
return tx.Bucket([]byte("meta")).Put([]byte(modsecNoEscalateKey), val)
})
}
// AddModSecNoEscalateRule atomically adds a single rule ID to the no-escalate set.
// Read-modify-write happens in a single bbolt Update transaction to prevent races.
func (db *DB) AddModSecNoEscalateRule(ruleID int) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
var ids []int
if v := b.Get([]byte(modsecNoEscalateKey)); v != nil {
if err := json.Unmarshal(v, &ids); err != nil {
return fmt.Errorf("read ModSecurity escalation exclusions: %w", err)
}
}
for _, id := range ids {
if id == ruleID {
return nil // already present
}
}
ids = append(ids, ruleID)
val, err := json.Marshal(ids)
if err != nil {
return err
}
return b.Put([]byte(modsecNoEscalateKey), val)
})
}
// RemoveModSecNoEscalateRule atomically removes a single rule ID from the no-escalate set.
func (db *DB) RemoveModSecNoEscalateRule(ruleID int) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
var ids []int
if v := b.Get([]byte(modsecNoEscalateKey)); v != nil {
if err := json.Unmarshal(v, &ids); err != nil {
return fmt.Errorf("read ModSecurity escalation exclusions: %w", err)
}
}
var filtered []int
for _, id := range ids {
if id != ruleID {
filtered = append(filtered, id)
}
}
val, err := json.Marshal(filtered)
if err != nil {
return err
}
return b.Put([]byte(modsecNoEscalateKey), val)
})
}
// RuleHitStats holds hit count and last-hit time for a ModSecurity rule.
type RuleHitStats struct {
Hits int `json:"hits"`
LastHit time.Time `json:"last_hit"`
}
type ruleHitData struct {
Buckets map[string]int `json:"buckets"` // key: "YYYYMMDDHH" → count
LastHit time.Time `json:"last_hit"`
}
type modSecPruneItem struct {
key []byte
original []byte
data ruleHitData
remove bool
}
func modsecHitKey(ruleID int) string {
return fmt.Sprintf("modsec:hits:%d", ruleID)
}
func hourBucket(t time.Time) string {
return fmt.Sprintf("%04d%02d%02d%02d", t.Year(), t.Month(), t.Day(), t.Hour())
}
// IncrModSecRuleHit increments the hit counter for a rule ID in the current hour bucket.
func (db *DB) IncrModSecRuleHit(ruleID int, timestamp time.Time) {
key := modsecHitKey(ruleID)
bucket := hourBucket(timestamp)
_ = db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
var data ruleHitData
if v := b.Get([]byte(key)); v != nil {
if json.Unmarshal(v, &data) != nil {
data = ruleHitData{Buckets: make(map[string]int)}
}
} else {
data = ruleHitData{Buckets: make(map[string]int)}
}
data.Buckets[bucket]++
data.LastHit = timestamp
val, err := json.Marshal(data)
if err != nil {
return err
}
return b.Put([]byte(key), val)
})
}
// GetModSecRuleHits returns hit counts and last-hit timestamps for all rules
// within the last 24 hours. Prunes buckets older than 24h.
// Note: hourly bucket granularity means the window is 24h +/- 1h at boundaries.
func (db *DB) GetModSecRuleHits() map[int]RuleHitStats {
result := make(map[int]RuleHitStats)
cutoff := time.Now().Add(-24 * time.Hour)
cutoffBucket := hourBucket(cutoff)
prefix := []byte("modsec:hits:")
var toPrune []modSecPruneItem
// Read pass stays read-only so the common (nothing-to-prune) call never
// commits a write transaction. Pruning runs in a separate Update only when
// stale buckets were found.
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
c := b.Cursor()
for k, v := c.Seek(prefix); k != nil && len(k) >= len(prefix) && string(k[:len(prefix)]) == string(prefix); k, v = c.Next() {
idStr := string(k[len(prefix):])
ruleID, err := strconv.Atoi(idStr)
if err != nil {
continue
}
var data ruleHitData
if json.Unmarshal(v, &data) != nil {
continue
}
total := 0
needsPrune := false
for bk, count := range data.Buckets {
if bk >= cutoffBucket {
total += count
} else {
needsPrune = true
}
}
if needsPrune {
// bbolt keys/values are only valid inside the tx; deep copy both.
toPrune = append(toPrune, modSecPruneItem{
key: append([]byte(nil), k...),
original: append([]byte(nil), v...),
data: data,
remove: total == 0,
})
}
if total > 0 {
result[ruleID] = RuleHitStats{
Hits: total,
LastHit: data.LastHit,
}
}
}
return nil
})
if len(toPrune) == 0 {
return result
}
_ = db.bolt.Update(func(tx *bolt.Tx) error {
return pruneModSecRuleHits(tx.Bucket([]byte("meta")), toPrune, cutoffBucket)
})
return result
}
func pruneModSecRuleHits(bucket *bolt.Bucket, items []modSecPruneItem, cutoffBucket string) error {
for _, item := range items {
// A concurrent hit may have rewritten the row after the read pass.
// Leave it for the next read rather than clobbering the fresh count.
if !bytes.Equal(bucket.Get(item.key), item.original) {
continue
}
if item.remove {
if err := bucket.Delete(item.key); err != nil {
return err
}
continue
}
for bk := range item.data.Buckets {
if bk < cutoffBucket {
delete(item.data.Buckets, bk)
}
}
val, err := json.Marshal(item.data)
if err != nil {
return err
}
if err := bucket.Put(item.key, val); err != nil {
return err
}
}
return nil
}
package store
import (
"errors"
"fmt"
bolt "go.etcd.io/bbolt"
)
// PHPRelayKV is a single key/value pair for batched writes.
type PHPRelayKV struct {
Key []byte
Value []byte
}
// allowed php_relay buckets. Validated on every helper to prevent the
// daemon from accidentally writing into other buckets through these
// generic wrappers.
var phpRelayBucketAllowlist = map[string]struct{}{
"phprelay:meta": {},
"phprelay:msgindex": {},
"phprelay:ignore": {},
"phprelay:settings": {},
"phprelay:baseline": {}, // Stage 3
}
func phpRelayCheckBucket(name string) error {
if _, ok := phpRelayBucketAllowlist[name]; !ok {
return fmt.Errorf("php_relay: bucket %q is not in the allowlist", name)
}
return nil
}
// PHPRelayPut writes a single key/value into the named php_relay bucket.
func (db *DB) PHPRelayPut(bucket, key string, value []byte) error {
if err := phpRelayCheckBucket(bucket); err != nil {
return err
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucket))
if b == nil {
return fmt.Errorf("bucket %q missing", bucket)
}
return b.Put([]byte(key), append([]byte(nil), value...))
})
}
// PHPRelayGet reads a single value. ok=false when the key is absent.
func (db *DB) PHPRelayGet(bucket, key string) ([]byte, bool, error) {
if err := phpRelayCheckBucket(bucket); err != nil {
return nil, false, err
}
var out []byte
var found bool
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucket))
if b == nil {
return fmt.Errorf("bucket %q missing", bucket)
}
v := b.Get([]byte(key))
if v == nil {
return nil
}
out = append([]byte(nil), v...)
found = true
return nil
})
return out, found, err
}
// PHPRelayDelete removes a key. Missing keys are not an error.
func (db *DB) PHPRelayDelete(bucket, key string) error {
if err := phpRelayCheckBucket(bucket); err != nil {
return err
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucket))
if b == nil {
return fmt.Errorf("bucket %q missing", bucket)
}
return b.Delete([]byte(key))
})
}
// PHPRelayPutBatch writes many key/value pairs in a single bbolt
// transaction. Used by the msgIndexPersister to keep IOPS bounded.
// Returns on the first encode/put error; partial commits are visible
// only at transaction boundary.
func (db *DB) PHPRelayPutBatch(bucket string, ops []PHPRelayKV) error {
if err := phpRelayCheckBucket(bucket); err != nil {
return err
}
if len(ops) == 0 {
return nil
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucket))
if b == nil {
return fmt.Errorf("bucket %q missing", bucket)
}
for _, kv := range ops {
if len(kv.Key) == 0 {
return errors.New("php_relay: empty key")
}
if err := b.Put(append([]byte(nil), kv.Key...), append([]byte(nil), kv.Value...)); err != nil {
return err
}
}
return nil
})
}
// PHPRelaySweep iterates the bucket and deletes every key for which
// shouldDelete returns true. Decoding is the caller's responsibility.
// Returns the number of deletions.
func (db *DB) PHPRelaySweep(bucket string, shouldDelete func(key, value []byte) bool) (int, error) {
if err := phpRelayCheckBucket(bucket); err != nil {
return 0, err
}
n := 0
err := db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucket))
if b == nil {
return fmt.Errorf("bucket %q missing", bucket)
}
var toDelete [][]byte
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if shouldDelete(k, v) {
toDelete = append(toDelete, append([]byte(nil), k...))
}
}
for _, k := range toDelete {
if err := b.Delete(k); err != nil {
return err
}
}
n = len(toDelete)
return nil
})
return n, err
}
// PHPRelayList returns a copy of every key/value in the bucket. Used at
// daemon start to restore the in-memory ignoreList.
func (db *DB) PHPRelayList(bucket string) (map[string][]byte, error) {
if err := phpRelayCheckBucket(bucket); err != nil {
return nil, err
}
out := make(map[string][]byte)
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucket))
if b == nil {
return fmt.Errorf("bucket %q missing", bucket)
}
return b.ForEach(func(k, v []byte) error {
out[string(k)] = append([]byte(nil), v...)
return nil
})
})
return out, err
}
package store
import (
"encoding/json"
"time"
bolt "go.etcd.io/bbolt"
)
// PluginInfo holds cached metadata for a WordPress plugin from the API.
type PluginInfo struct {
LatestVersion string `json:"latest_version"`
TestedUpTo string `json:"tested_up_to"`
LastChecked int64 `json:"last_checked_unix"`
}
// SitePluginEntry describes a single plugin installed on a WordPress site.
type SitePluginEntry struct {
Slug string `json:"slug"`
Name string `json:"name"`
Status string `json:"status"`
InstalledVersion string `json:"installed_version"`
UpdateVersion string `json:"update_version"`
}
// SitePlugins holds the full plugin inventory for a WordPress installation.
type SitePlugins struct {
Account string `json:"account"`
Domain string `json:"domain"`
Plugins []SitePluginEntry `json:"plugins"`
}
// SetPluginInfo stores plugin metadata keyed by slug in the plugins bucket.
func (db *DB) SetPluginInfo(slug string, info PluginInfo) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("plugins"))
val, err := json.Marshal(info)
if err != nil {
return err
}
return b.Put([]byte(slug), val)
})
}
// GetPluginInfo retrieves plugin metadata for the given slug.
// Returns the entry and true if found, or a zero value and false if not.
func (db *DB) GetPluginInfo(slug string) (PluginInfo, bool) {
var info PluginInfo
var found bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("plugins"))
v := b.Get([]byte(slug))
if v == nil {
return nil
}
if json.Unmarshal(v, &info) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
found = true
return nil
})
return info, found
}
// SetSitePlugins stores the plugin inventory for a WordPress installation
// keyed by its filesystem path in the plugins:sites bucket.
func (db *DB) SetSitePlugins(wpPath string, site SitePlugins) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("plugins:sites"))
val, err := json.Marshal(site)
if err != nil {
return err
}
return b.Put([]byte(wpPath), val)
})
}
// GetSitePlugins retrieves the plugin inventory for a WordPress installation.
// Returns the entry and true if found, or a zero value and false if not.
func (db *DB) GetSitePlugins(wpPath string) (SitePlugins, bool) {
var site SitePlugins
var found bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("plugins:sites"))
v := b.Get([]byte(wpPath))
if v == nil {
return nil
}
if json.Unmarshal(v, &site) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
found = true
return nil
})
return site, found
}
// DeleteSitePlugins removes the plugin inventory for a WordPress installation.
func (db *DB) DeleteSitePlugins(wpPath string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("plugins:sites"))
return b.Delete([]byte(wpPath))
})
}
// AllSitePlugins returns all site plugin inventories keyed by WordPress path.
func (db *DB) AllSitePlugins() map[string]SitePlugins {
entries := make(map[string]SitePlugins)
_ = db.bolt.View(func(tx *bolt.Tx) error {
return tx.Bucket([]byte("plugins:sites")).ForEach(func(k, v []byte) error {
var s SitePlugins
if json.Unmarshal(v, &s) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
entries[string(k)] = s
return nil
})
})
return entries
}
// GetPluginRefreshTime reads the last plugin refresh timestamp from the meta bucket.
// Returns the zero time if not set.
func (db *DB) GetPluginRefreshTime() time.Time {
var t time.Time
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
v := b.Get([]byte("plugins:last_refresh"))
if v == nil {
return nil
}
parsed, err := time.Parse(time.RFC3339, string(v))
if err != nil {
return nil //nolint:nilerr // skip corrupt entry
}
t = parsed
return nil
})
return t
}
// SetPluginRefreshTime writes the plugin refresh timestamp to the meta bucket.
func (db *DB) SetPluginRefreshTime(t time.Time) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
return b.Put([]byte("plugins:last_refresh"), []byte(t.Format(time.RFC3339)))
})
}
package store
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"time"
bolt "go.etcd.io/bbolt"
)
// prefsBucket holds per-operator preference blobs:
// - "<opkey>:user" -> user settings JSON (density, timezone, etc.)
// - "<opkey>:views:<page>" -> saved filter views JSON
// - "<opkey>:table:<tableID>" -> per-table column visibility JSON
// - "<opkey>:undo:<seq>" -> bulk-action undo entry (sequence is
// big-endian uint64 of unix nano, so prefix iteration is chronological)
//
// opkey is a SHA-256 hex of the operator's auth token, computed at request
// time by the webui layer. The store never sees the token itself.
const prefsBucket = "prefs:operator"
// MaxPrefBlobSize caps the size of any single preference blob. Large enough
// for saved views with dozens of params, small enough to keep abuse bounded.
const MaxPrefBlobSize = 64 * 1024
// MaxUndoEntries caps how many undo entries one operator may have queued at
// once. Older entries fall out as new ones are recorded.
const MaxUndoEntries = 32
// UndoTTL is how long an undo entry remains valid. Matches the banner timeout
// the UI advertises so an operator can never undo an action whose banner has
// already disappeared.
const UndoTTL = 30 * time.Second
// ErrPrefBlobTooLarge is returned when a preference blob exceeds MaxPrefBlobSize.
var ErrPrefBlobTooLarge = errors.New("preference blob too large")
func prefsKey(opkey, ns string) []byte {
return []byte(opkey + ":" + ns)
}
func undoKey(opkey string, seq uint64) []byte {
out := make([]byte, 0, len(opkey)+6+8)
out = append(out, opkey...)
out = append(out, ':', 'u', 'n', 'd', 'o', ':')
var seqBE [8]byte
binary.BigEndian.PutUint64(seqBE[:], seq)
return append(out, seqBE[:]...)
}
func undoKeyPrefix(opkey string) []byte {
return []byte(opkey + ":undo:")
}
// GetOperatorPref returns the raw JSON blob stored at (opkey, ns). Returns
// nil with nil error when no entry exists.
func (db *DB) GetOperatorPref(opkey, ns string) ([]byte, error) {
if db == nil || db.bolt == nil {
return nil, errors.New("store unavailable")
}
if opkey == "" || ns == "" {
return nil, errors.New("opkey and namespace required")
}
var out []byte
err := db.bolt.View(func(tx *bolt.Tx) error {
b, err := bucketOrCreate(tx, prefsBucket)
if err != nil {
return err
}
v := b.Get(prefsKey(opkey, ns))
if v == nil {
return nil
}
// Bolt invalidates the slice after the tx ends; copy out.
out = append([]byte(nil), v...)
return nil
})
return out, err
}
// PutOperatorPref writes the JSON blob to (opkey, ns). Rejects payloads above
// MaxPrefBlobSize.
func (db *DB) PutOperatorPref(opkey, ns string, data []byte) error {
if db == nil || db.bolt == nil {
return errors.New("store unavailable")
}
if opkey == "" || ns == "" {
return errors.New("opkey and namespace required")
}
if len(data) > MaxPrefBlobSize {
return ErrPrefBlobTooLarge
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b, err := bucketOrCreate(tx, prefsBucket)
if err != nil {
return err
}
return b.Put(prefsKey(opkey, ns), data)
})
}
// DeleteOperatorPref removes the blob at (opkey, ns). No error if absent.
func (db *DB) DeleteOperatorPref(opkey, ns string) error {
if db == nil || db.bolt == nil {
return errors.New("store unavailable")
}
if opkey == "" || ns == "" {
return errors.New("opkey and namespace required")
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b, err := bucketOrCreate(tx, prefsBucket)
if err != nil {
return err
}
return b.Delete(prefsKey(opkey, ns))
})
}
// UndoEntry is one record in the bulk-action undo queue.
type UndoEntry struct {
Targets []string `json:"targets,omitempty"` // IPs whose later edits invalidate this action
ID string `json:"id"` // matches the seq encoded in the bbolt key
RecordedAt time.Time `json:"recorded_at"` // wall time the entry was written
Action string `json:"action"` // e.g. "threat_bulk_block"
Inverse string `json:"inverse"` // inverse action key the runner will dispatch
Payload []byte `json:"payload"` // opaque JSON the runner understands
Summary string `json:"summary"` // human-readable label for the banner
}
// AppendUndoEntry queues an undo entry for opkey. The entry's ID and
// RecordedAt are filled in. Prunes expired entries and trims the queue to
// MaxUndoEntries before writing. Returns the saved entry.
func (db *DB) AppendUndoEntry(opkey string, e UndoEntry) (UndoEntry, error) {
if db == nil || db.bolt == nil {
return UndoEntry{}, errors.New("store unavailable")
}
if opkey == "" {
return UndoEntry{}, errors.New("opkey required")
}
if e.Inverse == "" {
return UndoEntry{}, errors.New("inverse action required")
}
now := time.Now().UTC()
seq := uint64(now.UnixNano())
e.ID = fmt.Sprintf("%016x", seq)
e.RecordedAt = now
raw, err := encodeUndoEntry(e)
if err != nil {
return UndoEntry{}, err
}
if len(raw) > MaxPrefBlobSize {
return UndoEntry{}, ErrPrefBlobTooLarge
}
err = db.bolt.Update(func(tx *bolt.Tx) error {
b, berr := bucketOrCreate(tx, prefsBucket)
if berr != nil {
return berr
}
if perr := pruneOperatorUndo(b, opkey, now); perr != nil {
return perr
}
return b.Put(undoKey(opkey, seq), raw)
})
if err != nil {
return UndoEntry{}, err
}
return e, nil
}
// LatestUndoEntry returns the most recent non-expired undo entry for opkey,
// or (zero, false, nil) if none exists.
func (db *DB) LatestUndoEntry(opkey string) (UndoEntry, bool, error) {
if db == nil || db.bolt == nil {
return UndoEntry{}, false, errors.New("store unavailable")
}
if opkey == "" {
return UndoEntry{}, false, errors.New("opkey required")
}
now := time.Now().UTC()
var entry UndoEntry
var found bool
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(prefsBucket))
if b == nil {
return nil
}
c := b.Cursor()
prefix := undoKeyPrefix(opkey)
// Seek to first key strictly greater than the prefix, then walk
// backwards. Newest entries sort last because the suffix is the
// big-endian nano timestamp.
for k, v := c.Last(); k != nil; k, v = c.Prev() {
if !bytes.HasPrefix(k, prefix) {
if bytes.Compare(k, prefix) < 0 {
return nil
}
continue
}
e, err := decodeUndoEntry(v)
if err != nil {
continue
}
if now.Sub(e.RecordedAt) > UndoTTL {
return nil
}
entry = e
found = true
return nil
}
return nil
})
return entry, found, err
}
// ConsumeUndoEntry removes the entry identified by id from opkey's queue and
// returns the decoded value. Returns (zero, false, nil) if the entry has
// already expired or never existed.
func (db *DB) ConsumeUndoEntry(opkey, id string) (UndoEntry, bool, error) {
if db == nil || db.bolt == nil {
return UndoEntry{}, false, errors.New("store unavailable")
}
if opkey == "" || id == "" {
return UndoEntry{}, false, errors.New("opkey and id required")
}
now := time.Now().UTC()
var entry UndoEntry
var found bool
err := db.bolt.Update(func(tx *bolt.Tx) error {
b, err := bucketOrCreate(tx, prefsBucket)
if err != nil {
return err
}
if perr := pruneOperatorUndo(b, opkey, now); perr != nil {
return perr
}
c := b.Cursor()
prefix := undoKeyPrefix(opkey)
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
e, derr := decodeUndoEntry(v)
if derr != nil {
continue
}
if e.ID != id {
continue
}
if now.Sub(e.RecordedAt) > UndoTTL {
return b.Delete(k)
}
entry = e
found = true
return b.Delete(k)
}
return nil
})
return entry, found, err
}
// PurgeOperatorUndo drops every undo entry for opkey (used when an operator
// logs out or when tests need to reset state).
func (db *DB) PurgeOperatorUndo(opkey string) error {
if db == nil || db.bolt == nil {
return errors.New("store unavailable")
}
if opkey == "" {
return errors.New("opkey required")
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(prefsBucket))
if b == nil {
return nil
}
var keys [][]byte
prefix := undoKeyPrefix(opkey)
c := b.Cursor()
for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = c.Next() {
keys = append(keys, append([]byte(nil), k...))
}
for _, k := range keys {
if err := b.Delete(k); err != nil {
return err
}
}
return nil
})
}
// pruneOperatorUndo drops expired entries and trims the queue to
// MaxUndoEntries. Must run inside a writable transaction.
func pruneOperatorUndo(b *bolt.Bucket, opkey string, now time.Time) error {
prefix := undoKeyPrefix(opkey)
type kv struct {
key []byte
entry UndoEntry
}
var kept []kv
var expired [][]byte
c := b.Cursor()
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
e, err := decodeUndoEntry(v)
if err != nil {
expired = append(expired, append([]byte(nil), k...))
continue
}
if now.Sub(e.RecordedAt) > UndoTTL {
expired = append(expired, append([]byte(nil), k...))
continue
}
kept = append(kept, kv{key: append([]byte(nil), k...), entry: e})
}
for _, k := range expired {
if err := b.Delete(k); err != nil {
return err
}
}
// kept is already in chronological order because keys sort by big-endian
// nano suffix. Trim oldest first.
if len(kept) >= MaxUndoEntries {
drop := len(kept) - MaxUndoEntries + 1
for i := 0; i < drop; i++ {
if err := b.Delete(kept[i].key); err != nil {
return err
}
}
}
return nil
}
func bucketOrCreate(tx *bolt.Tx, name string) (*bolt.Bucket, error) {
if tx.Writable() {
return tx.CreateBucketIfNotExists([]byte(name))
}
if b := tx.Bucket([]byte(name)); b != nil {
return b, nil
}
return nil, fmt.Errorf("bucket %s not initialised", name)
}
// InvalidateUndoTargets retires actions superseded by a later decision about
// any of their IPs, across operators. Invalidate the whole action so a stale
// bulk undo cannot silently restore only part of an operator's decision.
func (db *DB) InvalidateUndoTargets(ips []string) error {
return db.bolt.Update(func(tx *bolt.Tx) error { return invalidateUndoTargets(tx, ips) })
}
func invalidateUndoTargets(tx *bolt.Tx, ips []string) error {
b := tx.Bucket([]byte(prefsBucket))
if b == nil {
return nil
}
targets := make(map[string]bool, len(ips))
for _, ip := range ips {
targets[ip] = true
}
var keys [][]byte
if err := b.ForEach(func(k, v []byte) error {
if !bytes.Contains(k, []byte(":undo:")) {
return nil
}
entry, err := decodeUndoEntry(v)
if err != nil {
return nil //nolint:nilerr // Corrupt entries cannot execute.
}
for _, ip := range entry.Targets {
if targets[ip] {
keys = append(keys, append([]byte(nil), k...))
break
}
}
return nil
}); err != nil {
return err
}
for _, key := range keys {
if err := b.Delete(key); err != nil {
return err
}
}
return nil
}
package store
import "encoding/json"
func encodeUndoEntry(e UndoEntry) ([]byte, error) {
return json.Marshal(e)
}
func decodeUndoEntry(raw []byte) (UndoEntry, error) {
var e UndoEntry
if err := json.Unmarshal(raw, &e); err != nil {
return UndoEntry{}, err
}
return e, nil
}
package store
import (
"bytes"
"fmt"
"time"
bolt "go.etcd.io/bbolt"
)
// RewriteUndoEntryRecordedAt rewrites the RecordedAt field of an undo
// entry identified by id. Used by tests that need to age an entry past the
// TTL window without sleeping for real.
func RewriteUndoEntryRecordedAt(db *DB, id string, at time.Time) error {
if db == nil || db.bolt == nil {
return fmt.Errorf("store unavailable")
}
if id == "" {
return fmt.Errorf("id required")
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(prefsBucket))
if b == nil {
return fmt.Errorf("prefs bucket missing")
}
c := b.Cursor()
needle := []byte(":undo:")
for k, v := c.First(); k != nil; k, v = c.Next() {
if !bytes.Contains(k, needle) {
continue
}
e, err := decodeUndoEntry(v)
if err != nil {
continue
}
if e.ID != id {
continue
}
e.RecordedAt = at
raw, err := encodeUndoEntry(e)
if err != nil {
return err
}
return b.Put(k, raw)
}
return fmt.Errorf("entry %s not found", id)
})
}
package store
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
bolt "go.etcd.io/bbolt"
)
// Meta-bucket keys for AbuseIPDB quota accounting. Persisted so enforcement
// survives daemon restarts and spans across 10-minute scan cycles.
const (
abuseQuotaExhaustedKey = "abuse:quota_exhausted_until"
abuseDailyCountPrefix = "abuse:daily_count:" // + YYYY-MM-DD in UTC
)
// ReputationEntry holds the cached reputation data for an IP address.
type ReputationEntry struct {
Score int `json:"score"`
Category string `json:"category"`
CheckedAt time.Time `json:"checked_at"`
}
// SetReputation stores a reputation entry for the given IP.
func (db *DB) SetReputation(ip string, entry ReputationEntry) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("reputation"))
val, err := json.Marshal(entry)
if err != nil {
return err
}
return b.Put([]byte(ip), val)
})
}
// ApplyReputationChanges stores and deletes reputation entries in one write
// transaction.
func (db *DB) ApplyReputationChanges(upserts map[string]ReputationEntry, deletes map[string]bool) error {
if len(upserts) == 0 && len(deletes) == 0 {
return nil
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("reputation"))
for ip := range deletes {
if err := b.Delete([]byte(ip)); err != nil {
return err
}
}
for ip, entry := range upserts {
val, err := json.Marshal(entry)
if err != nil {
return err
}
if err := b.Put([]byte(ip), val); err != nil {
return err
}
}
return nil
})
}
// GetReputation retrieves a reputation entry for the given IP.
// Returns the entry and true if found, or a zero value and false if not.
func (db *DB) GetReputation(ip string) (ReputationEntry, bool) {
var entry ReputationEntry
var found bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("reputation"))
v := b.Get([]byte(ip))
if v == nil {
return nil
}
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
found = true
return nil
})
return entry, found
}
// CleanExpiredReputation deletes entries older than maxAge.
// Uses a collect-then-delete pattern because bbolt does not allow mutation
// during ForEach iteration. Returns the count of entries removed.
func (db *DB) CleanExpiredReputation(maxAge time.Duration) int {
var removed int
cutoff := time.Now().Add(-maxAge)
updateErr := boltUpdate(db.bolt, func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("reputation"))
// Collect keys to delete.
var toDelete [][]byte
_ = b.ForEach(func(k, v []byte) error {
var entry ReputationEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
if entry.CheckedAt.Before(cutoff) {
keyCopy := make([]byte, len(k))
copy(keyCopy, k)
toDelete = append(toDelete, keyCopy)
}
return nil
})
// Delete collected keys.
for _, k := range toDelete {
if err := b.Delete(k); err != nil {
return err
}
removed++
}
return nil
})
return committedCount("reputation", updateErr, removed)
}
// AllReputation returns all reputation entries keyed by IP.
func (db *DB) AllReputation() map[string]ReputationEntry {
entries := make(map[string]ReputationEntry)
_ = db.bolt.View(func(tx *bolt.Tx) error {
return tx.Bucket([]byte("reputation")).ForEach(func(k, v []byte) error {
var e ReputationEntry
if json.Unmarshal(v, &e) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
entries[string(k)] = e
return nil
})
})
return entries
}
// SetAbuseQuotaExhaustedUntil records the time at which the AbuseIPDB
// quota is expected to reset. While now < t, callers should skip API
// queries. The daemon re-reads this on every cycle so the flag survives
// restarts and multi-hour backoffs.
func (db *DB) SetAbuseQuotaExhaustedUntil(t time.Time) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
return tx.Bucket([]byte("meta")).Put(
[]byte(abuseQuotaExhaustedKey),
[]byte(t.UTC().Format(time.RFC3339)),
)
})
}
// AbuseQuotaExhaustedUntil returns the persisted quota-reset timestamp,
// or zero time if none is recorded (or the stored value is unparseable).
func (db *DB) AbuseQuotaExhaustedUntil() time.Time {
var ts time.Time
_ = db.bolt.View(func(tx *bolt.Tx) error {
v := tx.Bucket([]byte("meta")).Get([]byte(abuseQuotaExhaustedKey))
if v == nil {
return nil
}
parsed, err := time.Parse(time.RFC3339, string(v))
if err != nil {
return nil //nolint:nilerr // skip corrupt entry
}
ts = parsed
return nil
})
return ts
}
// IncrementAbuseQueryCount bumps and returns the AbuseIPDB query counter
// for the given UTC date (YYYY-MM-DD). Used as a daily circuit breaker.
func (db *DB) IncrementAbuseQueryCount(utcDate string) int {
var count int
_ = db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
pruneAbuseDailyCounts(b, utcDate)
key := []byte(abuseDailyCountPrefix + utcDate)
if v := b.Get(key); v != nil {
_, _ = fmt.Sscanf(string(v), "%d", &count)
}
count++
return b.Put(key, []byte(fmt.Sprintf("%d", count)))
})
return count
}
// ReserveAbuseQuerySlots atomically reserves up to requested AbuseIPDB
// query slots for utcDate without increasing the daily counter beyond max.
func (db *DB) ReserveAbuseQuerySlots(utcDate string, requested, max int) int {
if requested <= 0 || max <= 0 {
return 0
}
var reserved int
_ = db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
pruneAbuseDailyCounts(b, utcDate)
key := []byte(abuseDailyCountPrefix + utcDate)
count := 0
if v := b.Get(key); v != nil {
_, _ = fmt.Sscanf(string(v), "%d", &count)
}
if count >= max {
return nil
}
remaining := max - count
reserved = requested
if reserved > remaining {
reserved = remaining
}
count += reserved
return b.Put(key, []byte(fmt.Sprintf("%d", count)))
})
return reserved
}
func pruneAbuseDailyCounts(bucket *bolt.Bucket, keepDate string) {
prefix := []byte(abuseDailyCountPrefix)
cutoff, err := time.Parse("2006-01-02", keepDate)
if err != nil {
return
}
oldest := cutoff.AddDate(0, 0, -1)
cursor := bucket.Cursor()
for key, _ := cursor.Seek(prefix); key != nil && strings.HasPrefix(string(key), abuseDailyCountPrefix); key, _ = cursor.Next() {
date := strings.TrimPrefix(string(key), abuseDailyCountPrefix)
parsed, parseErr := time.Parse("2006-01-02", date)
if parseErr != nil || parsed.Before(oldest) {
_ = cursor.Delete()
}
}
}
// AbuseQueryCount returns the AbuseIPDB query count for the given UTC date.
func (db *DB) AbuseQueryCount(utcDate string) int {
var count int
_ = db.bolt.View(func(tx *bolt.Tx) error {
v := tx.Bucket([]byte("meta")).Get([]byte(abuseDailyCountPrefix + utcDate))
if v != nil {
_, _ = fmt.Sscanf(string(v), "%d", &count)
}
return nil
})
return count
}
// EnforceReputationCap ensures the reputation bucket has at most max entries.
// If the count exceeds max, the oldest entries (by CheckedAt) are deleted.
// Returns the count of entries removed.
func (db *DB) EnforceReputationCap(max int) int {
var removed int
updateErr := boltUpdate(db.bolt, func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("reputation"))
// Collect all entries with their keys.
type keyed struct {
key []byte
checkedAt time.Time
}
var all []keyed
_ = b.ForEach(func(k, v []byte) error {
var entry ReputationEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
keyCopy := make([]byte, len(k))
copy(keyCopy, k)
all = append(all, keyed{key: keyCopy, checkedAt: entry.CheckedAt})
return nil
})
if len(all) <= max {
return nil
}
// Sort by CheckedAt ascending (oldest first).
sort.Slice(all, func(i, j int) bool {
return all[i].checkedAt.Before(all[j].checkedAt)
})
// Delete the oldest entries beyond the cap.
excess := len(all) - max
for i := 0; i < excess; i++ {
if err := b.Delete(all[i].key); err != nil {
return err
}
removed++
}
return nil
})
return committedCount("reputation cap", updateErr, removed)
}
package store
import (
"encoding/json"
"errors"
"fmt"
"os"
"time"
bolt "go.etcd.io/bbolt"
)
// timeKeyLowerBound computes the lexicographic lower bound of any TimeKey
// produced for timestamp t. Any TimeKey whose stored time is strictly
// earlier than t sorts before this string; any TimeKey at or after t sorts
// at or above it. Matches the format in TimeKey().
func timeKeyLowerBound(t time.Time) string {
t = t.UTC()
return fmt.Sprintf("%04d%02d%02d%02d%02d%02d%09d-0000",
t.Year(), t.Month(), t.Day(),
t.Hour(), t.Minute(), t.Second(),
t.Nanosecond())
}
// SweepHistoryOlderThan deletes history entries whose TimeKey is strictly
// older than cutoff. Returns the number of entries deleted. All work runs
// in a single bbolt transaction so the UI never sees a half-swept state;
// callers pick cutoffs that keep the batch bounded.
func (db *DB) SweepHistoryOlderThan(cutoff time.Time) (int, error) {
cutoffKey := timeKeyLowerBound(cutoff)
var deleted int
err := db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("history"))
if b == nil {
return nil
}
c := b.Cursor()
for k, _ := c.First(); k != nil && string(k) < cutoffKey; k, _ = c.First() {
if err := c.Delete(); err != nil {
return err
}
deleted++
}
if deleted == 0 {
return nil
}
if err := bumpHistoryRevision(tx); err != nil {
return err
}
// Decrement the history:count counter without letting it underflow.
current := 0
if v := tx.Bucket([]byte("meta")).Get([]byte("history:count")); v != nil {
_, _ = fmt.Sscanf(string(v), "%d", ¤t)
}
newCount := current - deleted
if newCount < 0 {
newCount = 0
}
return setCounter(tx, "history:count", newCount)
})
return deleted, err
}
// SweepAttackEventsOlderThan deletes attacks:events entries older than
// cutoff and the matching entries from the attacks:events:ip secondary
// index. Returns the number of primary-bucket entries deleted.
func (db *DB) SweepAttackEventsOlderThan(cutoff time.Time) (int, error) {
cutoffKey := timeKeyLowerBound(cutoff)
var deleted int
err := db.bolt.Update(func(tx *bolt.Tx) error {
primary := tx.Bucket([]byte("attacks:events"))
secondary := tx.Bucket([]byte("attacks:events:ip"))
if primary == nil {
return nil
}
c := primary.Cursor()
for k, v := c.First(); k != nil && string(k) < cutoffKey; k, v = c.First() {
// The secondary index is keyed "<ip>/<TimeKey>", so we need
// the event's IP to prune it.
var ev AttackEvent
if err := json.Unmarshal(v, &ev); err == nil && secondary != nil {
secKey := []byte(ev.IP + "/" + string(k))
if err := secondary.Delete(secKey); err != nil {
return err
}
}
if err := c.Delete(); err != nil {
return err
}
deleted++
}
if deleted == 0 {
return nil
}
current := 0
if v := tx.Bucket([]byte("meta")).Get([]byte("attacks:events:count")); v != nil {
_, _ = fmt.Sscanf(string(v), "%d", ¤t)
}
newCount := current - deleted
if newCount < 0 {
newCount = 0
}
return setCounter(tx, "attacks:events:count", newCount)
})
return deleted, err
}
// Size returns the on-disk size of the bbolt file in bytes. bbolt does
// not shrink the file on delete; compare Size() before and after a
// CompactInto call to see how much space would be reclaimed.
func (db *DB) Size() (int64, error) {
info, err := os.Stat(db.path)
if err != nil {
return 0, err
}
return info.Size(), nil
}
// CompactionDue reports whether the state db is worth compacting: large enough
// that the slack matters AND fragmented enough that a compaction would reclaim
// a meaningful fraction. minSizeMB and fillRatio come from Retention config;
// non-positive values disable the check. freeBytes above sizeBytes is clamped
// (used=0) rather than producing a negative fill. Startup compaction and the
// daemon's compaction hint share it so the hint never promises a compaction
// the next start will skip.
func CompactionDue(sizeBytes, freeBytes int64, minSizeMB int, fillRatio float64) bool {
if minSizeMB <= 0 || fillRatio <= 0 || sizeBytes <= 0 {
return false
}
if sizeBytes < int64(minSizeMB)*1024*1024 {
return false
}
used := sizeBytes - freeBytes
if used < 0 {
used = 0
}
fill := float64(used) / float64(sizeBytes)
return fill < fillRatio
}
// FreeBytes returns the number of bytes held by free and pending pages in the
// bbolt freelist -- the space a compaction would reclaim. bbolt never shrinks
// the file on delete, so a large FreeBytes relative to Size means the on-disk
// file is mostly slack and is worth compacting.
func (db *DB) FreeBytes() (int64, error) {
// FreeAlloc includes free and pending pages using the database's page
// size. Stats holds bbolt's statistics lock; Info reads the mmap without
// locking and can race with remapping while the live database grows.
return int64(db.bolt.Stats().FreeAlloc), nil
}
// CompactInto snapshots the live DB into a fresh bbolt file at dstPath
// using bolt.Compact. Returns the source size and the compacted size
// (both in bytes).
//
// Correctness: bolt.Compact runs a View transaction on src for the
// duration of the walk, so concurrent Update calls on src will either
// land before the walk begins (captured in the snapshot) or after it
// completes (not in the snapshot). It is the caller's job to quiesce
// writers between the CompactInto call and the file rename+reopen that
// promotes the new file; otherwise post-snapshot writes are silently
// dropped during the swap.
//
// txMaxSize caps per-transaction bytes written to the destination (see
// bolt.Compact docs). Zero means "one transaction for the whole copy",
// which is the fastest path for DBs that comfortably fit in memory.
func (db *DB) CompactInto(dstPath string, txMaxSize int64) (srcSize, dstSize int64, err error) {
if dstPath == "" {
return 0, 0, errors.New("dst path is empty")
}
// Snapshot the src size up front; if bolt.Compact mutates src in ways
// we didn't anticipate, a concurrent reader still sees consistent
// numbers.
srcInfo, statErr := os.Stat(db.path)
if statErr != nil {
return 0, 0, fmt.Errorf("stat src: %w", statErr)
}
srcSize = srcInfo.Size()
dst, err := bolt.Open(dstPath, 0600, &bolt.Options{Timeout: 5 * time.Second})
if err != nil {
// bolt.Open may have created a zero-byte file before failing; clean it up.
_ = os.Remove(dstPath)
return srcSize, 0, fmt.Errorf("opening dst: %w", err)
}
compactErr := bolt.Compact(dst, db.bolt, txMaxSize)
if closeErr := dst.Close(); closeErr != nil && compactErr == nil {
compactErr = fmt.Errorf("closing dst: %w", closeErr)
}
if compactErr != nil {
_ = os.Remove(dstPath)
return srcSize, 0, compactErr
}
dstInfo, err := os.Stat(dstPath)
if err != nil {
return srcSize, 0, fmt.Errorf("stat dst: %w", err)
}
return srcSize, dstInfo.Size(), nil
}
// SweepReputationOlderThan deletes reputation entries whose CheckedAt is
// strictly older than cutoff. The bucket is keyed by IP and not by time,
// so the sweep inspects each value; malformed rows are skipped rather than
// aborting the sweep.
func (db *DB) SweepReputationOlderThan(cutoff time.Time) (int, error) {
var deleted int
err := db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("reputation"))
if b == nil {
return nil
}
var stale [][]byte
if err := b.ForEach(func(k, v []byte) error {
var e ReputationEntry
// Malformed rows are skipped: a bad Unmarshal here must not
// abort the sweep for the rest of the bucket. Expressed as
// "proceed only when the row parses and is stale" so the
// happy path stays on the left of the guard.
if err := json.Unmarshal(v, &e); err == nil && e.CheckedAt.Before(cutoff) {
// Copy k because the slice is only valid for the
// duration of the callback.
stale = append(stale, append([]byte(nil), k...))
}
return nil
}); err != nil {
return err
}
for _, k := range stale {
if err := b.Delete(k); err != nil {
return err
}
deleted++
}
return nil
})
return deleted, err
}
package store
import (
"encoding/json"
"fmt"
"time"
bolt "go.etcd.io/bbolt"
)
const scanCursorBucket = "scan_cursor"
// ScanCursorRecord is the rolling-coverage cursor for one (account, check).
type ScanCursorRecord struct {
Account string `json:"account"`
Check string `json:"check"`
LastPath string `json:"last_path"` // last path-sorted candidate scanned
WrappedAt time.Time `json:"wrapped_at"` // when the cursor last wrapped to start
LastFullCycleTS time.Time `json:"last_full_cycle_ts"` // when a full sweep last completed
// Signature rescan progress (deep YARA cursor only). RulesGeneration is
// the queued rescan the walk is covering, RulesFrom is where the walk
// stood when it took that generation, and RulesWrapped is set once the
// walk has since wrapped to the start. RulesRootsHash ties the lap to its
// resolved roots so a new root behind the cursor cannot be cleared unseen.
RulesGeneration uint64 `json:"rules_generation,omitempty"`
RulesFrom string `json:"rules_from,omitempty"`
RulesWrapped bool `json:"rules_wrapped,omitempty"`
RulesRootsHash string `json:"rules_roots_hash,omitempty"`
// ChangedThrough is the lower ctime bound for priority scans, not proof
// that older files were read: those remain the rolling window's job.
// A lap visits paths once up to ChangedEnd. Errors retain the old bound
// after the lap, so unreadable files get another chance on the next lap.
ChangedThrough time.Time `json:"changed_through,omitempty"`
ChangedFrom string `json:"changed_from,omitempty"`
ChangedEnd string `json:"changed_end,omitempty"`
ChangedLapStart time.Time `json:"changed_lap_start,omitempty"`
ChangedLapFailed bool `json:"changed_lap_failed,omitempty"`
// ServedAt is a logical ordering timestamp, independent of clock steps.
ChangedServedAt time.Time `json:"changed_served_at,omitempty"`
// Wall and boot clocks allow a backward clock step to rewind the ctime
// bound even when the next scan's wall time exceeds the previous one.
ChangedClock time.Time `json:"changed_clock,omitempty"`
ChangedUptime float64 `json:"changed_uptime,omitempty"`
ChangedBootID string `json:"changed_boot_id,omitempty"`
}
// scanCursorKey builds the bucket key for a (account, check) pair.
// Format: "<account>/<check>".
func scanCursorKey(account, check string) []byte {
return []byte(account + "/" + check)
}
// GetScanCursor retrieves the cursor record for (account, check).
// ok=false when absent (no error).
func (db *DB) GetScanCursor(account, check string) (ScanCursorRecord, bool, error) {
var rec ScanCursorRecord
var found bool
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(scanCursorBucket))
if b == nil {
return fmt.Errorf("bucket %q missing", scanCursorBucket)
}
v := b.Get(scanCursorKey(account, check))
if v == nil {
return nil
}
found = true
return json.Unmarshal(v, &rec)
})
if err != nil {
return ScanCursorRecord{}, false, err
}
return rec, found, nil
}
// PutScanCursor creates or replaces the cursor record for rec.Account/rec.Check.
func (db *DB) PutScanCursor(rec ScanCursorRecord) error {
val, err := json.Marshal(rec)
if err != nil {
return fmt.Errorf("scancursor: marshal: %w", err)
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(scanCursorBucket))
if b == nil {
return fmt.Errorf("bucket %q missing", scanCursorBucket)
}
return b.Put(scanCursorKey(rec.Account, rec.Check), val)
})
}
package store
import (
"bytes"
"encoding/json"
"fmt"
"sort"
"time"
bolt "go.etcd.io/bbolt"
"github.com/pidginhost/csm/internal/alert"
)
const (
scanJobsBucket = "scan_jobs"
scanJobFindingsBucket = "scan_job_findings"
)
// ScanJobRecord holds the metadata for a single full-scan job.
type ScanJobRecord struct {
ID string `json:"id"`
Scope string `json:"scope"`
Target string `json:"target"`
State string `json:"state"`
Created time.Time `json:"created"`
Started time.Time `json:"started,omitzero"`
Finished time.Time `json:"finished,omitzero"`
FilesScanned int `json:"files_scanned,omitempty"`
FilesEst int `json:"files_est,omitempty"`
FindingCount int `json:"finding_count,omitempty"`
// FindingsStored is how many findings were actually persisted. It equals
// FindingCount unless the per-job cap truncated the tail, in which case
// FindingsTruncated is set and the UI shows "showing first N of M".
FindingsStored int `json:"findings_stored,omitempty"`
FindingsTruncated bool `json:"findings_truncated,omitempty"`
// Progress fields for scope="all" jobs. Zero/empty for account-scope jobs.
AccountsTotal int `json:"accounts_total,omitempty"`
AccountsDone int `json:"accounts_done,omitempty"`
CurrentAccount string `json:"current_account,omitempty"`
Options map[string]any `json:"options,omitempty"`
Error string `json:"error,omitempty"`
}
// PutScanJob creates or replaces a scan-job record.
func (db *DB) PutScanJob(rec ScanJobRecord) error {
val, err := json.Marshal(rec)
if err != nil {
return fmt.Errorf("scanjobs: marshal: %w", err)
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(scanJobsBucket))
if b == nil {
return fmt.Errorf("bucket %q missing", scanJobsBucket)
}
return b.Put([]byte(rec.ID), val)
})
}
// GetScanJob retrieves a single scan-job record. ok=false when the ID is absent.
func (db *DB) GetScanJob(id string) (ScanJobRecord, bool, error) {
var rec ScanJobRecord
var found bool
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(scanJobsBucket))
if b == nil {
return fmt.Errorf("bucket %q missing", scanJobsBucket)
}
v := b.Get([]byte(id))
if v == nil {
return nil
}
found = true
return json.Unmarshal(v, &rec)
})
if err != nil {
return ScanJobRecord{}, false, err
}
return rec, found, nil
}
// ListScanJobs returns all scan-job records ordered newest-first (by Created,
// with ID as a deterministic tiebreaker for equal timestamps).
func (db *DB) ListScanJobs() ([]ScanJobRecord, error) {
var jobs []ScanJobRecord
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(scanJobsBucket))
if b == nil {
return fmt.Errorf("bucket %q missing", scanJobsBucket)
}
return b.ForEach(func(_, v []byte) error {
var rec ScanJobRecord
if err := json.Unmarshal(v, &rec); err != nil {
return err
}
jobs = append(jobs, rec)
return nil
})
})
if err != nil {
return nil, err
}
sort.Slice(jobs, func(i, j int) bool {
ci, cj := jobs[i].Created, jobs[j].Created
if ci.Equal(cj) {
return jobs[i].ID > jobs[j].ID
}
return ci.After(cj)
})
return jobs, nil
}
// findingKey builds the bucket key for a scan-job finding.
// Format: "<job_id>/<zero-padded-seq>" using 8 decimal digits for seq so
// lexicographic order equals insertion order and prefix scans work cleanly.
func findingKey(jobID string, seq int) []byte {
return []byte(fmt.Sprintf("%s/%08d", jobID, seq))
}
// findingPrefix returns the prefix used to address all findings for a job.
func findingPrefix(jobID string) []byte {
return []byte(jobID + "/")
}
// AppendScanJobFinding persists a single finding for the given job.
// seq must be unique within the job (caller supplies a monotonic counter).
// Using many small values rather than one growing blob enables pagination
// via cursor seeks without loading the whole finding list into memory.
func (db *DB) AppendScanJobFinding(id string, seq int, f alert.Finding) error {
return db.AppendScanJobFindings(id, seq, []alert.Finding{f})
}
// AppendScanJobFindings persists a batch of findings for the given job in a
// single write transaction. Keys are seq, seq+1, ... seq+len-1. Batching
// amortizes bbolt's per-commit fsync across the whole slice instead of paying
// one fsync per finding, which dominated the write cost of a large scan.
func (db *DB) AppendScanJobFindings(id string, seq int, findings []alert.Finding) error {
if len(findings) == 0 {
return nil
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(scanJobFindingsBucket))
if b == nil {
return fmt.Errorf("bucket %q missing", scanJobFindingsBucket)
}
for i, f := range findings {
val, err := json.Marshal(f)
if err != nil {
return fmt.Errorf("scanjobs: marshal finding: %w", err)
}
if err := b.Put(findingKey(id, seq+i), val); err != nil {
return err
}
}
return nil
})
}
// ListScanJobFindings returns a paginated slice of findings for a job.
// total is the count of ALL findings for the job (ignoring offset/limit).
// offset and limit follow the usual slice semantics; limit=0 returns all.
func (db *DB) ListScanJobFindings(id string, offset, limit int) ([]alert.Finding, int, error) {
prefix := findingPrefix(id)
var findings []alert.Finding
total := 0
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(scanJobFindingsBucket))
if b == nil {
return fmt.Errorf("bucket %q missing", scanJobFindingsBucket)
}
c := b.Cursor()
pos := 0
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
total++
if pos >= offset && (limit == 0 || len(findings) < limit) {
var f alert.Finding
if err := json.Unmarshal(v, &f); err != nil {
return err
}
findings = append(findings, f)
}
pos++
}
return nil
})
if err != nil {
return nil, 0, err
}
return findings, total, nil
}
// PruneScanJobs enforces two independent retention limits and removes the
// finding rows of every pruned job. Returns the number of jobs pruned.
//
// - keepJobs bounds how many job records are retained.
// - maxTotalFindings bounds the cumulative finding rows across retained jobs
// (0 disables the volume cap). A single scan can emit tens of thousands of
// findings, so a job-count cap alone lets retained findings grow without
// bound; the volume cap keeps the state file in check.
//
// Jobs are ranked newest-first (Created desc, ID tiebreaker). The newest job is
// always kept even if it alone exceeds the volume cap, so retention never wipes
// the most recent result. The whole operation runs in one bolt.Update so a
// concurrent insert cannot make the decision stale.
func (db *DB) PruneScanJobs(keepJobs, maxTotalFindings int) (int, error) {
if keepJobs < 0 {
keepJobs = 0
}
pruned := 0
err := db.bolt.Update(func(tx *bolt.Tx) error {
jb := tx.Bucket([]byte(scanJobsBucket))
if jb == nil {
return fmt.Errorf("bucket %q missing", scanJobsBucket)
}
fb := tx.Bucket([]byte(scanJobFindingsBucket))
if fb == nil {
return fmt.Errorf("bucket %q missing", scanJobFindingsBucket)
}
// Collect all job records from the bucket (reads are allowed inside Update).
var jobs []ScanJobRecord
if err := jb.ForEach(func(_, v []byte) error {
var rec ScanJobRecord
if err := json.Unmarshal(v, &rec); err != nil {
return err
}
jobs = append(jobs, rec)
return nil
}); err != nil {
return err
}
// Sort newest-first: same ordering as ListScanJobs.
sort.Slice(jobs, func(i, j int) bool {
ci, cj := jobs[i].Created, jobs[j].Created
if ci.Equal(cj) {
return jobs[i].ID > jobs[j].ID
}
return ci.After(cj)
})
kept := 0
keptFindings := 0
var delErr error
for _, rec := range jobs {
// Retention applies to finished jobs only. A queued or running
// job is older than the job that just completed, so counting or
// deleting it here would drop work the scheduler still owns.
if rec.State == "queued" || rec.State == "running" {
continue
}
overJobCount := kept >= keepJobs
if overJobCount {
if delErr = deleteScanJobAndFindings(jb, fb, rec.ID); delErr != nil {
return delErr
}
pruned++
continue
}
jobFindings := 0
if maxTotalFindings > 0 {
// Count only far enough to decide whether this job fits. For
// the newest job, which is always kept, a saturated count is
// enough to force later non-empty jobs over the volume cap.
remaining := maxTotalFindings - keptFindings
if kept == 0 {
remaining = maxTotalFindings
}
if remaining < 0 {
remaining = 0
}
jobFindings = countFindingRowsUpTo(fb, rec.ID, remaining+1)
if kept >= 1 && jobFindings > 0 && keptFindings+jobFindings > maxTotalFindings {
if delErr = deleteScanJobAndFindings(jb, fb, rec.ID); delErr != nil {
return delErr
}
pruned++
continue
}
}
kept++
keptFindings += jobFindings
}
return nil
})
if err != nil {
return 0, err
}
return pruned, nil
}
// countFindingRowsUpTo counts finding rows stored for jobID, stopping early
// after limit rows when limit > 0. The volume-cap retention path only needs to
// know whether a job crosses a threshold, not its exact size above that point.
func countFindingRowsUpTo(fb *bolt.Bucket, jobID string, limit int) int {
prefix := findingPrefix(jobID)
n := 0
c := fb.Cursor()
for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = c.Next() {
n++
if limit > 0 && n >= limit {
break
}
}
return n
}
// deleteScanJobAndFindings removes a job record and all of its finding rows.
// Keys are collected before deletion because the cursor cannot be mutated mid-iteration.
func deleteScanJobAndFindings(jb, fb *bolt.Bucket, jobID string) error {
if err := jb.Delete([]byte(jobID)); err != nil {
return err
}
prefix := findingPrefix(jobID)
c := fb.Cursor()
var stale [][]byte
for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = c.Next() {
stale = append(stale, append([]byte(nil), k...))
}
for _, k := range stale {
if err := fb.Delete(k); err != nil {
return err
}
}
return nil
}
package store
import (
"encoding/json"
"strings"
bolt "go.etcd.io/bbolt"
)
// senderProfileBucket holds one SenderProfile per authenticated mail account.
const senderProfileBucket = "email:sender"
// SenderDay summarises one UTC day of authenticated sends for a mailbox. The
// distinct sets are capped by the writer, so a counted value is a floor once
// the cap is reached.
type SenderDay struct {
Sends int `json:"sends"`
// IPs are the distinct source addresses seen.
IPs []string `json:"ips,omitempty"`
// MaxHourIPs is the most distinct source addresses seen inside any one
// rolling hour of the day.
MaxHourIPs int `json:"max_hour_ips"`
// Countries are the distinct source countries resolved for the day.
Countries []string `json:"countries,omitempty"`
// Recipients are the distinct envelope recipients, lower-cased.
Recipients []string `json:"recipients,omitempty"`
// SingleRecipients excludes bulk announcements from individual fan-out.
SingleRecipients []string `json:"single_recipients,omitempty"`
}
// SenderProfile is a mailbox's recent sending history keyed by UTC day
// ("2006-01-02"). The realtime detector compares today against the prior
// days to judge whether a change in sources or fan-out is the owner's habit
// or someone else's.
type SenderProfile struct {
Days map[string]*SenderDay `json:"days"`
}
// SenderProfileKeys returns the stored spellings of a mailbox. Authentication
// logs can vary in case while cPanel suspends and releases the same mailbox.
func (db *DB) SenderProfileKeys(mailbox string) ([]string, error) {
var keys []string
err := db.bolt.View(func(tx *bolt.Tx) error {
return tx.Bucket([]byte(senderProfileBucket)).ForEach(func(k, _ []byte) error {
if strings.EqualFold(string(k), mailbox) {
keys = append(keys, string(k))
}
return nil
})
})
return keys, err
}
// SetSenderProfile stores the sending profile for a mailbox.
func (db *DB) SetSenderProfile(mailbox string, p SenderProfile) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
val, err := json.Marshal(p)
if err != nil {
return err
}
return tx.Bucket([]byte(senderProfileBucket)).Put([]byte(mailbox), val)
})
}
// GetSenderProfile retrieves the sending profile for a mailbox. A missing
// profile is empty; unreadable history is an error, never a fresh baseline.
func (db *DB) GetSenderProfile(mailbox string) (SenderProfile, error) {
var p SenderProfile
err := db.bolt.View(func(tx *bolt.Tx) error {
v := tx.Bucket([]byte(senderProfileBucket)).Get([]byte(mailbox))
if v == nil {
return nil
}
return json.Unmarshal(v, &p)
})
if err != nil {
return SenderProfile{}, err
}
return p, nil
}
// PruneSenderProfiles expires old days even for mailboxes that never send
// again. Empty profiles are removed instead of retaining recipient history.
// The walk only collects; bbolt leaves a cursor undefined once the bucket it
// traverses is written, so every write happens after the walk.
func (db *DB) PruneSenderProfiles(oldest string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(senderProfileBucket))
var expired [][]byte
rewritten := map[string][]byte{}
cursor := bucket.Cursor()
for key, value := cursor.First(); key != nil; key, value = cursor.Next() {
var p SenderProfile
if err := json.Unmarshal(value, &p); err != nil {
return err
}
changed := false
for day := range p.Days {
if day < oldest {
delete(p.Days, day)
changed = true
}
}
switch {
case len(p.Days) == 0:
expired = append(expired, append([]byte(nil), key...))
case changed:
encoded, err := json.Marshal(p)
if err != nil {
return err
}
rewritten[string(key)] = encoded
}
}
for _, key := range expired {
if err := bucket.Delete(key); err != nil {
return err
}
}
for key, encoded := range rewritten {
if err := bucket.Put([]byte(key), encoded); err != nil {
return err
}
}
return nil
})
}
package store
import (
"encoding/json"
"errors"
"fmt"
"math"
"time"
bolt "go.etcd.io/bbolt"
)
// Persistence helpers for the signature-update watcher. The daemon stores
// what it last saw of each signature file in bbolt so a restart does not
// trigger a phantom rescan -- without this the in-memory map starts empty
// after every restart and every file looks new.
const (
sigWatchKey = "last_mtimes"
sigRescanKey = "rescan"
)
// ErrSignatureRescanCorrupt means the queue cannot establish whether a sweep
// is owed. The watcher repairs it by conservatively queuing a fresh sweep.
var ErrSignatureRescanCorrupt = errors.New("corrupt signature rescan record")
// SignatureFileState is what the watcher last saw of one rules file. SHA256
// is empty when the content was never hashed, and Size is -1 when unknown:
// both hold for entries recorded before content hashes were tracked.
type SignatureFileState struct {
Mtime time.Time `json:"mtime"`
Size int64 `json:"size"`
SHA256 string `json:"sha256,omitempty"`
}
// GetSignatureFiles returns the persisted watcher state. Empty (not nil) when
// nothing has been persisted yet. A map written before content hashes were
// tracked holds bare mtimes; its entries come back with Size -1 and no hash.
func (db *DB) GetSignatureFiles() (map[string]SignatureFileState, error) {
out := map[string]SignatureFileState{}
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("sig_watch"))
if b == nil {
return nil
}
raw := b.Get([]byte(sigWatchKey))
if len(raw) == 0 {
return nil
}
if err := json.Unmarshal(raw, &out); err == nil {
return nil
}
var legacy map[string]time.Time
if err := json.Unmarshal(raw, &legacy); err != nil {
return err
}
out = make(map[string]SignatureFileState, len(legacy))
for path, mtime := range legacy {
out[path] = SignatureFileState{Mtime: mtime, Size: -1}
}
return nil
})
return out, err
}
// PutSignatureFiles overwrites the persisted watcher state. Removed files
// must disappear from the store, so the whole map is replaced.
func (db *DB) PutSignatureFiles(m map[string]SignatureFileState) error {
_, err := db.putSignatureFiles(m, false)
return err
}
// PutSignatureFilesWithRescan writes the watcher state and queues a rescan
// in one transaction, so a restart cannot keep the new rule state while
// losing the rescan it calls for. It returns the generation that only the
// pass covering this queue may clear.
func (db *DB) PutSignatureFilesWithRescan(m map[string]SignatureFileState) (uint64, error) {
return db.putSignatureFiles(m, true)
}
// signatureRescan is the queued rescan. Generation keeps counting after a
// clear, so a pass can never clear a queue armed after it read its own.
type signatureRescan struct {
Generation uint64 `json:"generation"`
Pending bool `json:"pending"`
}
func (db *DB) putSignatureFiles(m map[string]SignatureFileState, rescan bool) (uint64, error) {
payload, marshalErr := json.Marshal(m)
if marshalErr != nil {
return 0, marshalErr
}
var gen uint64
update := func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("sig_watch"))
if b == nil {
return errors.New("sig_watch bucket missing (store not migrated)")
}
if err := b.Put([]byte(sigWatchKey), payload); err != nil {
return err
}
if !rescan {
return nil
}
q, err := readSignatureRescan(b)
if err != nil && !errors.Is(err, ErrSignatureRescanCorrupt) {
return err
}
// Keep the generation outside the JSON record too, so repairing it
// cannot reuse an in-flight sweep's token. The transaction ID also
// seeds stores written before the bucket sequence was used.
transactionID := uint64(tx.ID()) // #nosec G115 -- bbolt transaction IDs are positive.
previous := max(b.Sequence(), q.Generation, transactionID)
if previous == math.MaxUint64 {
return errors.New("signature rescan generation exhausted")
}
q.Generation = previous + 1
if err := b.SetSequence(q.Generation); err != nil {
return err
}
q.Pending = true
gen = q.Generation
return writeSignatureRescan(b, q)
}
if err := db.bolt.Update(update); err != nil {
return 0, err
}
return gen, nil
}
// SignatureRescanPending returns the generation of the queued rescan,
// or 0 when none is queued.
func (db *DB) SignatureRescanPending() (uint64, error) {
var q signatureRescan
var sequence uint64
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("sig_watch"))
if b == nil {
return nil
}
var err error
q, err = readSignatureRescan(b)
sequence = b.Sequence()
return err
})
if err != nil {
return 0, err
}
// Preserve legacy tokens before handing them to a sweep. Compaction can
// lower the transaction ID, so it alone cannot fence a repaired queue.
if q.Generation > sequence {
if err := db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("sig_watch"))
if b == nil {
return errors.New("sig_watch bucket missing (store not migrated)")
}
return b.SetSequence(max(b.Sequence(), q.Generation))
}); err != nil {
return 0, err
}
}
if !q.Pending {
return 0, nil
}
return q.Generation, nil
}
// ClearSignatureRescan removes the queued rescan when gen is still the
// queued generation, and reports whether it did.
func (db *DB) ClearSignatureRescan(gen uint64) (bool, error) {
cleared := false
err := db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("sig_watch"))
if b == nil {
return errors.New("sig_watch bucket missing (store not migrated)")
}
q, err := readSignatureRescan(b)
if err != nil || !q.Pending || q.Generation != gen {
return err
}
q.Pending = false
cleared = true
return writeSignatureRescan(b, q)
})
if err != nil {
return false, err
}
return cleared, nil
}
func readSignatureRescan(b *bolt.Bucket) (signatureRescan, error) {
raw := b.Get([]byte(sigRescanKey))
if raw == nil {
return signatureRescan{}, nil
}
var record struct {
Generation uint64 `json:"generation"`
Pending *bool `json:"pending"`
}
if err := json.Unmarshal(raw, &record); err != nil {
return signatureRescan{}, fmt.Errorf("%w: %v", ErrSignatureRescanCorrupt, err)
}
if record.Generation == 0 || record.Pending == nil {
return signatureRescan{}, ErrSignatureRescanCorrupt
}
return signatureRescan{Generation: record.Generation, Pending: *record.Pending}, nil
}
func writeSignatureRescan(b *bolt.Bucket, q signatureRescan) error {
raw, err := json.Marshal(q)
if err != nil {
return err
}
return b.Put([]byte(sigRescanKey), raw)
}
package store
import (
"bufio"
"bytes"
"encoding/binary"
"errors"
"fmt"
"hash/fnv"
"io"
"math"
"os"
bolt "go.etcd.io/bbolt"
)
// bbolt's version 2 disk format uses native byte order. Decode it with
// bounded reads: its mmap cursors and Tx.Check can fault, recurse forever,
// or panic on library-owned goroutines when page references are damaged.
const (
snapshotPageHeader = 16
snapshotMetaSize = 64
snapshotBranch = 1
snapshotLeaf = 2
snapshotFreelist = 16
)
// verifySnapshot only accepts private, stopped snapshots. Callers keep
// them private until the subsequent bbolt read or mutation completes.
func verifySnapshot(path string) error {
return verifyStoppedDatabase(path, false)
}
func verifyStoppedDatabase(path string, allowLeakedPages bool) error {
info, err := os.Stat(path)
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return fmt.Errorf("bbolt snapshot is not a regular file")
}
// #nosec G304 -- the caller holds the state lock or owns the private snapshot.
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
err = checkSnapshot(f, info.Size())
if allowLeakedPages && errors.Is(err, errLeakedPages) {
return nil
}
if err != nil {
return fmt.Errorf("bbolt snapshot is damaged: %w", err)
}
return nil
}
var errLeakedPages = errors.New("snapshot has unreachable unfreed pages")
// VerifyDatabase checks a stopped live database as verifySnapshot checks a
// snapshot, but accepts leaked pages: they waste space, and bbolt reads and
// writes around them. A missing file is a new database.
func VerifyDatabase(path string) error {
err := verifyStoppedDatabase(path, true)
if os.IsNotExist(err) {
// A dangling link is an existing target. A writable bbolt open
// would create its missing referent, possibly outside the state dir.
if _, linkErr := os.Lstat(path); os.IsNotExist(linkErr) {
return nil
}
}
return err
}
type snapshotMeta struct {
pageSize int64
root, freelist, pages, txid uint64
}
func readSnapshotMeta(r io.ReaderAt, offset int64) (snapshotMeta, bool) {
var data [snapshotPageHeader + snapshotMetaSize]byte
if _, err := r.ReadAt(data[:], offset); err != nil {
return snapshotMeta{}, false
}
m := data[snapshotPageHeader:]
order := binary.NativeEndian
h := fnv.New64a()
_, _ = h.Write(m[:56])
if order.Uint32(m) != 0xED0CDAED || order.Uint32(m[4:]) != 2 || order.Uint64(m[56:]) != h.Sum64() {
return snapshotMeta{}, false
}
return snapshotMeta{
pageSize: int64(order.Uint32(m[8:])), root: order.Uint64(m[16:]),
freelist: order.Uint64(m[32:]), pages: order.Uint64(m[40:]), txid: order.Uint64(m[48:]),
}, true
}
func checkSnapshot(r io.ReaderAt, size int64) error {
// bbolt probes the first 4 KiB for the page size. A shorter file can
// pass the metadata checksum while bbolt uses a different page size.
if size < 4096 {
return fmt.Errorf("database too short for bbolt metadata")
}
m, valid := readSnapshotMeta(r, 0)
if !valid {
// Match bbolt's recovery of the page size from the second meta.
for offset := int64(1024); offset <= 1024<<14 && offset < size-1024; offset *= 2 {
if m, valid = readSnapshotMeta(r, offset); valid {
if m.pageSize != offset {
return fmt.Errorf("misplaced snapshot recovery metadata")
}
break
}
}
}
if !valid || m.pageSize < snapshotPageHeader+snapshotMetaSize || m.pageSize > size/2 {
return fmt.Errorf("invalid snapshot metadata")
}
other, otherValid := readSnapshotMeta(r, m.pageSize)
if otherValid && other.pageSize != m.pageSize {
return fmt.Errorf("inconsistent snapshot page sizes")
}
if otherValid && other.txid > m.txid {
m = other
}
// The next write must supersede this transaction.
if m.txid == math.MaxUint64 {
return fmt.Errorf("snapshot transaction ID exhausted")
}
if m.pages < 2 || m.pages > math.MaxInt64 || int64(m.pages) > size/m.pageSize {
return fmt.Errorf("snapshot pages exceed file bounds")
}
v := snapshotVerifier{reader: r, meta: m, used: make(map[uint64]bool)}
if m.freelist != ^uint64(0) {
if err := v.freePages(); err != nil {
return err
}
}
work := []snapshotTree{{id: m.root, root: true}}
for len(work) > 0 {
last := len(work) - 1
tree := work[last]
work[last] = snapshotTree{}
work = work[:last]
children, err := v.treePage(tree)
if err != nil {
return err
}
work = append(work, children...)
}
if m.freelist != ^uint64(0) && uint64(len(v.used)) != m.pages-2 {
return errLeakedPages
}
return nil
}
type snapshotVerifier struct {
reader io.ReaderAt
meta snapshotMeta
used map[uint64]bool
}
// claim rejects both cycles and a live page overlapping another allocation
// or the freelist. Unused pages need not contain a valid page header.
func (v *snapshotVerifier) claim(id uint64) error {
if id < 2 || id >= v.meta.pages || v.used[id] {
return fmt.Errorf("invalid or multiply referenced snapshot page %d", id)
}
v.used[id] = true
return nil
}
func (v *snapshotVerifier) page(id uint64) (*io.SectionReader, error) {
if err := v.claim(id); err != nil {
return nil, err
}
var header [snapshotPageHeader]byte
// #nosec G115 -- claim bounds id below pages, which is at most file size / page size.
offset := int64(id) * v.meta.pageSize
if _, err := v.reader.ReadAt(header[:], offset); err != nil {
return nil, err
}
order := binary.NativeEndian
span := uint64(order.Uint32(header[12:])) + 1
if order.Uint64(header[:]) != id || span > v.meta.pages-id {
return nil, fmt.Errorf("invalid snapshot page %d header", id)
}
for i := uint64(1); i < span; i++ {
if err := v.claim(id + i); err != nil {
return nil, err
}
}
// span is a uint32 overflow count plus one, so it fits in int64.
return io.NewSectionReader(v.reader, offset, (int64(order.Uint32(header[12:]))+1)*v.meta.pageSize), nil
}
func (v *snapshotVerifier) freePages() error {
data, err := v.page(v.meta.freelist)
if err != nil {
return err
}
order := binary.NativeEndian
var header [snapshotPageHeader]byte
if _, err := io.ReadFull(data, header[:]); err != nil {
return err
}
if order.Uint16(header[8:]) != snapshotFreelist {
return fmt.Errorf("invalid snapshot freelist type")
}
count := uint64(order.Uint16(header[10:]))
remaining := data.Size() - snapshotPageHeader
ids := bufio.NewReader(data)
var id [8]byte
if count == 0xffff {
if _, err := io.ReadFull(ids, id[:]); err != nil {
return err
}
count = order.Uint64(id[:])
remaining -= 8
}
if count > math.MaxInt64 || int64(count) > remaining/8 {
return fmt.Errorf("snapshot freelist exceeds page bounds")
}
for i := uint64(0); i < count; i++ {
if _, err := io.ReadFull(ids, id[:]); err != nil {
return err
}
if err := v.claim(order.Uint64(id[:])); err != nil {
return err
}
}
return nil
}
type snapshotTree struct {
id uint64
inline *io.SectionReader
min, max []byte
depth int
leafDepth *int // shared within a single branched bucket
root bool // the top-level tree may contain only bucket entries
}
func (v *snapshotVerifier) treePage(tree snapshotTree) ([]snapshotTree, error) {
data := tree.inline
if data == nil {
var err error
data, err = v.page(tree.id)
if err != nil {
return nil, err
}
}
if data.Size() < snapshotPageHeader {
return nil, fmt.Errorf("short inline snapshot page")
}
order := binary.NativeEndian
var header [snapshotPageHeader]byte
if _, err := data.ReadAt(header[:], 0); err != nil {
return nil, err
}
flags, count := order.Uint16(header[8:]), int64(order.Uint16(header[10:]))
if flags != snapshotLeaf && flags != snapshotBranch || flags == snapshotBranch && count < 2 || tree.min != nil && count == 0 {
return nil, fmt.Errorf("invalid snapshot tree page type or count")
}
if tree.inline != nil && (order.Uint64(header[:]) != 0 || flags != snapshotLeaf || order.Uint32(header[12:]) != 0) {
return nil, fmt.Errorf("invalid inline snapshot page")
}
// bbolt rebalances siblings as nodes of the same kind.
if flags == snapshotBranch && tree.leafDepth == nil {
tree.leafDepth = new(int)
}
if flags == snapshotLeaf && tree.leafDepth != nil {
if *tree.leafDepth != 0 && *tree.leafDepth != tree.depth {
return nil, fmt.Errorf("inconsistent snapshot leaf depth")
}
*tree.leafDepth = tree.depth
}
// Both branch and leaf elements have a 16-byte header.
end := int64(snapshotPageHeader) + count*16
if end > data.Size() {
return nil, fmt.Errorf("snapshot elements exceed page bounds")
}
var children []snapshotTree
var previous []byte
elements := bufio.NewReader(io.NewSectionReader(data, snapshotPageHeader, count*16))
for i := int64(0); i < count; i++ {
at := int64(snapshotPageHeader) + i*16
var element [16]byte
if _, err := io.ReadFull(elements, element[:]); err != nil {
return nil, err
}
var pos, keySize, valueSize int64
if flags == snapshotLeaf {
pos, keySize, valueSize = int64(order.Uint32(element[4:])), int64(order.Uint32(element[8:])), int64(order.Uint32(element[12:]))
} else {
pos, keySize = int64(order.Uint32(element[:])), int64(order.Uint32(element[4:]))
}
start, stop := at+pos, at+pos+keySize+valueSize
if start < end || stop > data.Size() || keySize == 0 || keySize > bolt.MaxKeySize || valueSize > bolt.MaxValueSize {
return nil, fmt.Errorf("snapshot key or value exceeds page bounds")
}
end = stop
key := make([]byte, keySize)
if _, err := data.ReadAt(key, start); err != nil {
return nil, err
}
// bbolt uses a child's first key to update its parent on spill.
if i == 0 && tree.min != nil && !bytes.Equal(key, tree.min) {
return nil, fmt.Errorf("snapshot branch key differs from its child")
}
if i > 0 && bytes.Compare(previous, key) >= 0 || tree.max != nil && bytes.Compare(key, tree.max) >= 0 {
return nil, fmt.Errorf("snapshot keys are out of order")
}
previous = key
if flags == snapshotBranch {
if len(children) > 0 {
children[len(children)-1].max = key
}
children = append(children, snapshotTree{
id: order.Uint64(element[8:]), min: key, max: tree.max, root: tree.root,
depth: tree.depth + 1, leafDepth: tree.leafDepth,
})
continue
}
entryFlags := order.Uint32(element[:])
if entryFlags != 0 && entryFlags != 1 || tree.root && entryFlags != 1 {
return nil, fmt.Errorf("invalid snapshot bucket entry")
}
if entryFlags == 0 {
continue
}
// bbolt never inlines a bucket containing subbuckets.
if valueSize < 16 || tree.inline != nil {
return nil, fmt.Errorf("invalid snapshot bucket header")
}
var bucket [16]byte
if _, err := data.ReadAt(bucket[:], start+keySize); err != nil {
return nil, err
}
child := snapshotTree{id: order.Uint64(bucket[:])}
if child.id == 0 {
// bbolt may copy the entire bucket value to align its header.
// Its writer inlines at most a quarter page after the header.
if valueSize-16 > v.meta.pageSize/4 {
return nil, fmt.Errorf("oversized inline snapshot bucket")
}
child.inline = io.NewSectionReader(data, start+keySize+16, valueSize-16)
} else if valueSize != 16 {
return nil, fmt.Errorf("invalid external snapshot bucket size")
}
children = append(children, child)
}
if tree.inline != nil && end != data.Size() {
return nil, fmt.Errorf("invalid inline snapshot page size")
}
return children, nil
}
package store
import (
"encoding/json"
"errors"
"time"
bolt "go.etcd.io/bbolt"
bolterrors "go.etcd.io/bbolt/errors"
)
const spamSourceBucket = "mail:spam_source"
// SpamSource is the persisted spam source tracker state for one IP. Keeping it
// across restarts stops a daemon restart from resetting the reject occasions,
// the clean-mail window and the report throttle.
type SpamSource struct {
Occasions []time.Time `json:"occasions,omitempty"` // counted reject occasions, oldest first
LastClean time.Time `json:"last_clean,omitzero"`
LastFinding time.Time `json:"last_finding,omitzero"`
Seen time.Time `json:"seen"`
}
// SaveSpamSources replaces the persisted tracker state with m. The bucket is
// rewritten in one transaction so IPs the tracker has dropped do not come back
// at the next load.
func (db *DB) SaveSpamSources(m map[string]SpamSource) error {
encoded := make(map[string][]byte, len(m))
for ip, src := range m {
if ip == "" {
continue
}
val, err := json.Marshal(src)
if err != nil {
return err
}
encoded[ip] = val
}
return db.bolt.Update(func(tx *bolt.Tx) error {
if err := tx.DeleteBucket([]byte(spamSourceBucket)); err != nil && !errors.Is(err, bolterrors.ErrBucketNotFound) {
return err
}
b, err := tx.CreateBucket([]byte(spamSourceBucket))
if err != nil {
return err
}
for ip, val := range encoded {
if perr := b.Put([]byte(ip), val); perr != nil {
return perr
}
}
return nil
})
}
// LoadSpamSources returns the persisted tracker state keyed by IP, or an empty
// map when nothing has been saved yet. An entry that does not decode or has no
// observation time is skipped: failing the load would keep the detector off on
// every restart, while skipping loses only that sender's history.
func (db *DB) LoadSpamSources() (map[string]SpamSource, error) {
out := make(map[string]SpamSource)
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(spamSourceBucket))
if b == nil {
return nil
}
return b.ForEach(func(k, v []byte) error {
var src SpamSource
if json.Unmarshal(v, &src) != nil || src.Seen.IsZero() {
return nil //nolint:nilerr // skip corrupt entry
}
out[string(k)] = src
return nil
})
})
return out, err
}
package store
import (
"encoding/json"
"fmt"
"time"
"github.com/pidginhost/csm/internal/alert"
bolt "go.etcd.io/bbolt"
)
// stats:daily holds pre-aggregated SeverityBucket counters keyed by
// "YYYY-MM-DD" date. It is updated atomically with every history insert
// so the 30-day trend chart is decoupled from history pruning. The
// bucket has at most dailyRetentionDays rows and grows by ~50 bytes/day.
const (
bucketStatsDaily = "stats:daily"
metaStatsDailyBackfilled = "stats:daily:backfilled"
)
// dailyRetentionDays caps how far back stats:daily keeps per-day rows.
// Var (not const) so tests can override.
var dailyRetentionDays = 365
// incrStatsDaily increments the per-severity counters for a single
// finding's date inside an existing bbolt write transaction. The caller
// owns the transaction; this helper does not commit.
func incrStatsDaily(tx *bolt.Tx, t time.Time, sev alert.Severity) error {
b := tx.Bucket([]byte(bucketStatsDaily))
if b == nil {
return fmt.Errorf("bucket %s missing", bucketStatsDaily)
}
key := []byte(t.Format("2006-01-02"))
var sb SeverityBucket
if v := b.Get(key); v != nil {
if err := json.Unmarshal(v, &sb); err != nil {
// Corrupted entry - reset rather than refusing to record.
sb = SeverityBucket{}
}
}
sb.Total++
switch sev {
case alert.Critical:
sb.Critical++
case alert.High:
sb.High++
case alert.Warning:
sb.Warning++
}
val, err := json.Marshal(sb)
if err != nil {
return err
}
return b.Put(key, val)
}
// pruneStatsDaily deletes stats:daily rows older than dailyRetentionDays.
// Cheap because the bucket is bounded to ~dailyRetentionDays entries and
// keys sort lexicographically as YYYY-MM-DD.
func pruneStatsDaily(tx *bolt.Tx, now time.Time) error {
b := tx.Bucket([]byte(bucketStatsDaily))
if b == nil {
return nil
}
cutoff := now.AddDate(0, 0, -(dailyRetentionDays - 1)).Format("2006-01-02")
// Collect first: the caller has just written today's row in this
// transaction, and a bbolt cursor that deletes and then steps with Next
// skips keys in a bucket already written by the transaction.
var stale [][]byte
c := b.Cursor()
for k, _ := c.First(); k != nil && string(k) < cutoff; k, _ = c.Next() {
stale = append(stale, append([]byte(nil), k...))
}
for _, k := range stale {
if err := b.Delete(k); err != nil {
return err
}
}
return nil
}
// BackfillStatsDaily seeds stats:daily from the history bucket on first
// run after upgrade. Idempotent: a meta sentinel ensures it only runs
// once. Safe on hosts where the meta:migrated sentinel was set before
// stats:daily existed.
func (db *DB) BackfillStatsDaily() error {
var alreadyDone bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
if v := tx.Bucket([]byte("meta")).Get([]byte(metaStatsDailyBackfilled)); v != nil {
alreadyDone = true
}
return nil
})
if alreadyDone {
return nil
}
// Read history in a View transaction and aggregate in memory so we
// don't hold the write lock while scanning potentially large history.
type counts struct {
c, h, w, total int
}
perDay := make(map[string]*counts)
err := db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("history"))
if b == nil {
return nil
}
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var f alert.Finding
if err := json.Unmarshal(v, &f); err != nil {
continue
}
key := f.Timestamp.Format("2006-01-02")
cnt, ok := perDay[key]
if !ok {
cnt = &counts{}
perDay[key] = cnt
}
cnt.total++
switch f.Severity {
case alert.Critical:
cnt.c++
case alert.High:
cnt.h++
case alert.Warning:
cnt.w++
}
}
return nil
})
if err != nil {
return err
}
// Apply the aggregated counts and set the sentinel atomically. We
// merge into existing rows (additive) so the operation stays
// idempotent if the sentinel write somehow gets lost mid-flight.
return db.bolt.Update(func(tx *bolt.Tx) error {
// Re-check sentinel inside the write transaction in case another
// process raced ahead of us.
if v := tx.Bucket([]byte("meta")).Get([]byte(metaStatsDailyBackfilled)); v != nil {
return nil
}
b := tx.Bucket([]byte(bucketStatsDaily))
if b == nil {
return fmt.Errorf("bucket %s missing", bucketStatsDaily)
}
for key, cnt := range perDay {
var sb SeverityBucket
if v := b.Get([]byte(key)); v != nil {
if err := json.Unmarshal(v, &sb); err != nil {
sb = SeverityBucket{}
}
}
sb.Critical += cnt.c
sb.High += cnt.h
sb.Warning += cnt.w
sb.Total += cnt.total
val, mErr := json.Marshal(sb)
if mErr != nil {
return mErr
}
if pErr := b.Put([]byte(key), val); pErr != nil {
return pErr
}
}
return tx.Bucket([]byte("meta")).Put(
[]byte(metaStatsDailyBackfilled),
[]byte(time.Now().Format(time.RFC3339)),
)
})
}
package store
import (
"encoding/json"
"fmt"
"strings"
"time"
bolt "go.etcd.io/bbolt"
)
// Threat entry sources. Rows written before source tagging existed carry
// an empty Source; Expired classifies those by their reason text.
const (
ThreatSourceOperator = "operator"
ThreatSourceAutoBlock = "autoblock"
)
// legacyOperatorReasonPrefixes identifies pre-source-tagging rows added by a
// human. The Web UI manual/bulk block handlers are the only operator-facing
// writers this bucket ever had and they always used these exact reason
// strings; flat-file migration appended a "[YYYY-MM-DD]" suffix, hence the
// prefix match. Every other legacy row came from the temporary auto-block
// path, which historically wrote no-expiry rows that re-flagged the IP on
// every access after the block lapsed (permablock loop).
var legacyOperatorReasonPrefixes = []string{
"Manually blocked via CSM Web UI",
"Bulk blocked via CSM Web UI",
"Permanently blocked via CSM Web UI",
"Bulk permanently blocked via CSM Web UI",
}
// PermanentBlockEntry represents an IP blocked by the threat system.
// A zero ExpiresAt on an operator row means the row never expires. Auto-block
// rows must carry a real expiry; a zero expiry is treated as lapsed so it
// cannot become permanent reputation evidence.
type PermanentBlockEntry struct {
IP string `json:"ip"`
Reason string `json:"reason"`
BlockedAt time.Time `json:"blocked_at"`
Source string `json:"source,omitempty"`
ExpiresAt time.Time `json:"expires_at,omitzero"`
}
// Expired reports whether the entry should no longer count as a live
// threat. Legacy rows (no source, no expiry) that do not match a known
// operator reason are treated as expired: they were written by the old
// temporary auto-block path and keeping them alive re-creates the
// permablock loop on upgraded hosts.
func (e PermanentBlockEntry) Expired(now time.Time) bool {
if e.Source == ThreatSourceAutoBlock && e.ExpiresAt.IsZero() {
return true
}
if !e.ExpiresAt.IsZero() {
return !e.ExpiresAt.After(now)
}
if e.Source != "" {
return false
}
for _, prefix := range legacyOperatorReasonPrefixes {
if strings.HasPrefix(e.Reason, prefix) {
return false
}
}
return true
}
// WhitelistEntry represents an IP that should bypass threat checks.
type WhitelistEntry struct {
IP string `json:"ip"`
ExpiresAt time.Time `json:"expires_at"`
Permanent bool `json:"permanent"`
}
// AddPermanentBlock adds an IP to the block list as a never-expiring
// operator entry. Only increments threats:count if the key is new.
// Overwrites any temp entry for the same IP: an explicit operator block
// upgrades it to permanent.
func (db *DB) AddPermanentBlock(ip, reason string) error {
return db.putThreatEntry(PermanentBlockEntry{
IP: ip,
Reason: reason,
BlockedAt: time.Now(),
Source: ThreatSourceOperator,
})
}
// AddTempBlock records an auto-blocked IP with an expiry matching the
// firewall block, so the entry lapses together with the block instead of
// flagging the IP forever. A zero expiresAt is deliberately ignored: an
// auto-block-sourced threat row must never become permanent evidence.
// Never downgrades an existing permanent row, and never shortens a longer
// temp expiry already on file. Only increments threats:count if the key is
// new.
func (db *DB) AddTempBlock(ip, reason string, expiresAt time.Time) error {
return db.addExpiringThreat(ip, reason, ThreatSourceAutoBlock, expiresAt)
}
// AddOperatorTempBlock records a timed operator block (for example the Web
// UI 24h block) with the same expiry as the firewall block. The row carries
// the operator source but lapses with the block, so a mistaken timed block
// cannot turn into permanent reputation evidence. Same guards as
// AddTempBlock: zero expiry ignored, permanent rows never downgraded, longer
// live expiries never shortened.
func (db *DB) AddOperatorTempBlock(ip, reason string, expiresAt time.Time) error {
return db.addExpiringThreat(ip, reason, ThreatSourceOperator, expiresAt)
}
// AddAppliedTempBlock records one admission outcome without rewriting a repeat.
func (db *DB) AddAppliedTempBlock(ip, reason string, expiresAt time.Time) error {
return db.AddAppliedTempBlocks([]PermanentBlockEntry{{IP: ip, Reason: reason, ExpiresAt: expiresAt}})
}
// AddAppliedTempBlocks commits one delivery batch together. No-op rows do not
// change their timestamp, counters or undo entries created after the block.
func (db *DB) AddAppliedTempBlocks(entries []PermanentBlockEntry) error {
if len(entries) == 0 {
return nil
}
return db.bolt.Update(func(tx *bolt.Tx) error {
for _, entry := range entries {
if !time.Now().Before(entry.ExpiresAt) {
continue
}
if err := addExpiringThreatTx(tx, entry.IP, entry.Reason, ThreatSourceAutoBlock, entry.ExpiresAt, true); err != nil {
return err
}
}
return nil
})
}
func (db *DB) addExpiringThreat(ip, reason, source string, expiresAt time.Time) error {
if expiresAt.IsZero() {
return nil
}
return db.bolt.Update(func(tx *bolt.Tx) error {
return addExpiringThreatTx(tx, ip, reason, source, expiresAt, false)
})
}
func addExpiringThreatTx(tx *bolt.Tx, ip, reason, source string, expiresAt time.Time, replay bool) error {
if !replay {
if err := invalidateUndoTargets(tx, []string{ip}); err != nil {
return err
}
}
b := tx.Bucket([]byte("threats"))
entry := PermanentBlockEntry{
IP: ip,
Reason: reason,
BlockedAt: time.Now(),
Source: source,
ExpiresAt: expiresAt,
}
existing := b.Get([]byte(ip))
if existing != nil {
var cur PermanentBlockEntry
decodeErr := json.Unmarshal(existing, &cur)
if replay && decodeErr == nil && cur.Source == source && cur.Reason == reason && cur.ExpiresAt.Equal(expiresAt) {
return nil
}
if decodeErr == nil && !cur.Expired(time.Now()) {
if cur.ExpiresAt.IsZero() {
return nil
}
if cur.ExpiresAt.After(expiresAt) {
return nil
}
}
}
if replay {
if err := invalidateUndoTargets(tx, []string{ip}); err != nil {
return err
}
}
val, err := json.Marshal(entry)
if err != nil {
return err
}
if err := b.Put([]byte(ip), val); err != nil {
return err
}
if existing == nil {
return incrCounter(tx, "threats:count", 1)
}
return nil
}
// putThreatEntry writes an entry as-is. Only increments threats:count if
// the key is new. Migration uses it directly so flat-file rows keep their
// empty Source and stay subject to legacy classification in Expired;
// routing them through AddPermanentBlock would stamp them as operator rows
// and bless historical auto-block poison as permanent.
func (db *DB) putThreatEntry(entry PermanentBlockEntry) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
if err := invalidateUndoTargets(tx, []string{entry.IP}); err != nil {
return err
}
b := tx.Bucket([]byte("threats"))
isNew := b.Get([]byte(entry.IP)) == nil
val, err := json.Marshal(entry)
if err != nil {
return err
}
if err := b.Put([]byte(entry.IP), val); err != nil {
return err
}
if isNew {
return incrCounter(tx, "threats:count", 1)
}
return nil
})
}
// PruneExpiredThreats deletes threat entries whose lifetime has lapsed,
// including legacy no-source auto-block rows (see Expired). Returns the
// count removed. Collect-then-delete because bbolt forbids mutation during
// ForEach iteration.
func (db *DB) PruneExpiredThreats() int {
var removed int
now := time.Now()
updateErr := boltUpdate(db.bolt, func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("threats"))
var toDelete [][]byte
_ = b.ForEach(func(k, v []byte) error {
var entry PermanentBlockEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
if entry.Expired(now) {
keyCopy := make([]byte, len(k))
copy(keyCopy, k)
toDelete = append(toDelete, keyCopy)
}
return nil
})
for _, k := range toDelete {
if err := b.Delete(k); err != nil {
return err
}
removed++
}
if removed == 0 {
return nil
}
// Clamp instead of blind decrement: a bulk prune of legacy rows
// surfaces any historical counter drift at scale, and the counter
// must never go negative.
current := 0
if v := tx.Bucket([]byte("meta")).Get([]byte("threats:count")); v != nil {
_, _ = fmt.Sscanf(string(v), "%d", ¤t)
}
newCount := current - removed
if newCount < 0 {
newCount = 0
}
return setCounter(tx, "threats:count", newCount)
})
return committedCount("threats", updateErr, removed)
}
// RemovePermanentBlock removes an IP from the permanent block list and decrements the count.
func (db *DB) RemovePermanentBlock(ip string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
if err := invalidateUndoTargets(tx, []string{ip}); err != nil {
return err
}
b := tx.Bucket([]byte("threats"))
if b.Get([]byte(ip)) == nil {
return nil
}
if err := b.Delete([]byte(ip)); err != nil {
return err
}
return incrCounter(tx, "threats:count", -1)
})
}
// TiedToFirewallBlock reports whether the entry only lives as long as a
// firewall block: auto-block rows and any row carrying an expiry (timed
// operator blocks). Never-expiring operator rows and legacy no-source rows
// are standalone evidence.
func (e PermanentBlockEntry) TiedToFirewallBlock() bool {
return e.Source == ThreatSourceAutoBlock || !e.ExpiresAt.IsZero()
}
// RemoveTemporaryBlock deletes the threat row for ip only when it is tied to
// a firewall block (see TiedToFirewallBlock). Never-expiring operator rows
// and legacy no-source rows are left untouched, so a firewall-only unblock
// never silently clears an operator's deliberate permanent block. Stale
// timed rows would otherwise keep re-flagging the IP via ip_reputation.
// Returns whether a row was removed. The read and delete run in one
// transaction so a concurrent upgrade to a permanent row cannot be clobbered.
func (db *DB) RemoveTemporaryBlock(ip string) (bool, error) {
removed := false
err := db.bolt.Update(func(tx *bolt.Tx) error {
if err := invalidateUndoTargets(tx, []string{ip}); err != nil {
return err
}
b := tx.Bucket([]byte("threats"))
v := b.Get([]byte(ip))
if v == nil {
return nil
}
var entry PermanentBlockEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
if !entry.TiedToFirewallBlock() {
return nil
}
if err := b.Delete([]byte(ip)); err != nil {
return err
}
removed = true
return incrCounter(tx, "threats:count", -1)
})
return removed, err
}
// GetPermanentBlock looks up a permanent block entry by IP.
// Returns the entry and true if found, or a zero value and false if not.
func (db *DB) GetPermanentBlock(ip string) (PermanentBlockEntry, bool) {
var entry PermanentBlockEntry
var found bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("threats"))
v := b.Get([]byte(ip))
if v == nil {
return nil
}
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
found = true
return nil
})
return entry, found
}
// AllPermanentBlocks returns all entries in the permanent block list
// (including expired ones - callers filter via Expired).
func (db *DB) AllPermanentBlocks() []PermanentBlockEntry {
var entries []PermanentBlockEntry
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("threats"))
return b.ForEach(func(k, v []byte) error {
var entry PermanentBlockEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
entries = append(entries, entry)
return nil
})
})
return entries
}
// AddWhitelistEntry adds an IP to the whitelist.
func (db *DB) AddWhitelistEntry(ip string, expiresAt time.Time, permanent bool) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
if err := invalidateUndoTargets(tx, []string{ip}); err != nil {
return err
}
b := tx.Bucket([]byte("threats:whitelist"))
entry := WhitelistEntry{
IP: ip,
ExpiresAt: expiresAt,
Permanent: permanent,
}
val, err := json.Marshal(entry)
if err != nil {
return err
}
return b.Put([]byte(ip), val)
})
}
// RemoveWhitelistEntry removes an IP from the whitelist.
func (db *DB) RemoveWhitelistEntry(ip string) error {
return db.bolt.Update(func(tx *bolt.Tx) error {
if err := invalidateUndoTargets(tx, []string{ip}); err != nil {
return err
}
b := tx.Bucket([]byte("threats:whitelist"))
return b.Delete([]byte(ip))
})
}
// IsWhitelisted checks if an IP is whitelisted and not expired.
func (db *DB) IsWhitelisted(ip string) bool {
var whitelisted bool
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("threats:whitelist"))
v := b.Get([]byte(ip))
if v == nil {
return nil
}
var entry WhitelistEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
if entry.Permanent || entry.ExpiresAt.After(time.Now()) {
whitelisted = true
}
return nil
})
return whitelisted
}
// ListWhitelist returns all whitelist entries (including expired - caller filters).
func (db *DB) ListWhitelist() []WhitelistEntry {
var entries []WhitelistEntry
_ = db.bolt.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("threats:whitelist"))
return b.ForEach(func(k, v []byte) error {
var entry WhitelistEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
entries = append(entries, entry)
return nil
})
})
return entries
}
// PruneExpiredWhitelist deletes expired non-permanent whitelist entries.
// Returns the count of entries removed. Uses a collect-then-delete pattern
// because bbolt does not allow mutation during ForEach iteration.
func (db *DB) PruneExpiredWhitelist() int {
var removed int
now := time.Now()
updateErr := boltUpdate(db.bolt, func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("threats:whitelist"))
// Collect keys to delete.
var toDelete [][]byte
_ = b.ForEach(func(k, v []byte) error {
var entry WhitelistEntry
if json.Unmarshal(v, &entry) != nil {
return nil //nolint:nilerr // skip corrupt entry
}
if !entry.Permanent && !entry.ExpiresAt.After(now) {
keyCopy := make([]byte, len(k))
copy(keyCopy, k)
toDelete = append(toDelete, keyCopy)
}
return nil
})
// Delete collected keys.
for _, k := range toDelete {
if err := b.Delete(k); err != nil {
return err
}
removed++
}
return nil
})
return committedCount("whitelist", updateErr, removed)
}
package store
import (
"bytes"
"encoding/json"
"fmt"
"strconv"
"time"
"github.com/pidginhost/csm/internal/alert"
bolt "go.etcd.io/bbolt"
)
const (
timeKeyUTCMarker = "time_keys:utc_v1"
timeKeyHistoryTemp = "time_keys:migrate:history"
timeKeyAttacksTemp = "time_keys:migrate:attacks"
timeKeyAttacksIPTemp = "time_keys:migrate:attacks_ip"
timeKeyMigrationDone = "1"
historyBucketName = "history"
attackEventsBucketName = "attacks:events"
attackEventsIPBucket = "attacks:events:ip"
)
// migrateTimeKeysToUTC rewrites keys created by releases that formatted the
// wall clock from each timestamp's own location. Values retain the authoritative
// instant, so the rewrite can safely canonicalize mixed local/UTC stores. The
// buckets and marker are replaced in one transaction: a failed migration leaves
// the original indexes intact and is retried on the next Open.
func (db *DB) migrateTimeKeysToUTC() error {
var done bool
if err := db.bolt.View(func(tx *bolt.Tx) error {
meta := tx.Bucket([]byte("meta"))
done = meta != nil && string(meta.Get([]byte(timeKeyUTCMarker))) == timeKeyMigrationDone
return nil
}); err != nil {
return err
}
if done {
return nil
}
return db.bolt.Update(func(tx *bolt.Tx) error {
meta := tx.Bucket([]byte("meta"))
if string(meta.Get([]byte(timeKeyUTCMarker))) == timeKeyMigrationDone {
return nil
}
if err := migrateHistoryTimeKeys(tx); err != nil {
return fmt.Errorf("history: %w", err)
}
if err := migrateAttackTimeKeys(tx); err != nil {
return fmt.Errorf("attack events: %w", err)
}
return meta.Put([]byte(timeKeyUTCMarker), []byte(timeKeyMigrationDone))
})
}
func migrateHistoryTimeKeys(tx *bolt.Tx) error {
src := tx.Bucket([]byte(historyBucketName))
if bucketIsEmpty(src) {
return nil
}
tmp, err := freshMigrationBucket(tx, timeKeyHistoryTemp)
if err != nil {
return err
}
if src != nil {
c := src.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var finding alert.Finding
if decodeErr := json.Unmarshal(v, &finding); decodeErr != nil {
if _, putErr := putPreservedValue(tmp, k, v); putErr != nil {
return putErr
}
continue
}
if _, putErr := putMigratedTimeValue(tmp, finding.Timestamp, timeKeyCounter(k), v); putErr != nil {
return putErr
}
}
}
if err := replaceBucketFromTemp(tx, historyBucketName, timeKeyHistoryTemp); err != nil {
return err
}
return bumpHistoryRevision(tx)
}
func migrateAttackTimeKeys(tx *bolt.Tx) error {
primary := tx.Bucket([]byte(attackEventsBucketName))
secondary := tx.Bucket([]byte(attackEventsIPBucket))
if bucketIsEmpty(primary) && bucketIsEmpty(secondary) {
return nil
}
tmpPrimary, err := freshMigrationBucket(tx, timeKeyAttacksTemp)
if err != nil {
return err
}
keyMap := make(map[string]string)
if primary != nil {
c := primary.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
oldKey := string(k)
var event AttackEvent
var newKey string
var putErr error
if decodeErr := json.Unmarshal(v, &event); decodeErr != nil {
newKey, putErr = putPreservedValue(tmpPrimary, k, v)
} else {
newKey, putErr = putMigratedTimeValue(tmpPrimary, event.Timestamp, timeKeyCounter(k), v)
}
if putErr != nil {
return putErr
}
keyMap[oldKey] = newKey
}
}
if replaceErr := replaceBucketFromTemp(tx, attackEventsBucketName, timeKeyAttacksTemp); replaceErr != nil {
return replaceErr
}
tmpSecondary, err := freshMigrationBucket(tx, timeKeyAttacksIPTemp)
if err != nil {
return err
}
if secondary != nil {
c := secondary.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
sep := bytes.LastIndexByte(k, '/')
if sep >= 0 {
if newTimeKey, ok := keyMap[string(k[sep+1:])]; ok {
newKey := string(k[:sep+1]) + newTimeKey
if _, err := putPreservedValue(tmpSecondary, []byte(newKey), v); err != nil {
return err
}
continue
}
}
if _, err := putPreservedValue(tmpSecondary, k, v); err != nil {
return err
}
}
}
return replaceBucketFromTemp(tx, attackEventsIPBucket, timeKeyAttacksIPTemp)
}
func bucketIsEmpty(b *bolt.Bucket) bool {
if b == nil {
return true
}
k, _ := b.Cursor().First()
return k == nil
}
func freshMigrationBucket(tx *bolt.Tx, name string) (*bolt.Bucket, error) {
if tx.Bucket([]byte(name)) != nil {
if err := tx.DeleteBucket([]byte(name)); err != nil {
return nil, err
}
}
return tx.CreateBucket([]byte(name))
}
func replaceBucketFromTemp(tx *bolt.Tx, name, tempName string) error {
if tx.Bucket([]byte(name)) != nil {
if err := tx.DeleteBucket([]byte(name)); err != nil {
return err
}
}
dst, err := tx.CreateBucket([]byte(name))
if err != nil {
return err
}
if name == historyBucketName || name == attackEventsBucketName {
// The temporary bucket's cursor yields sorted keys even when
// canonicalizing mixed time zones changed their original order.
dst.FillPercent = timeKeyFillPercent
}
tmp := tx.Bucket([]byte(tempName))
if tmp != nil {
if err := tmp.ForEach(func(k, v []byte) error {
return dst.Put(k, v)
}); err != nil {
return err
}
}
return tx.DeleteBucket([]byte(tempName))
}
func putMigratedTimeValue(b *bolt.Bucket, timestamp time.Time, start int, value []byte) (string, error) {
for counter := start; ; counter++ {
key := TimeKey(timestamp, counter)
if b.Get([]byte(key)) == nil {
return key, b.Put([]byte(key), value)
}
}
}
func putPreservedValue(b *bolt.Bucket, key, value []byte) (string, error) {
candidate := string(key)
for suffix := 0; b.Get([]byte(candidate)) != nil; suffix++ {
candidate = fmt.Sprintf("%s~%04d", key, suffix)
}
return candidate, b.Put([]byte(candidate), value)
}
func timeKeyCounter(key []byte) int {
sep := bytes.LastIndexByte(key, '-')
if sep < 0 || sep == len(key)-1 {
return 0
}
counter, err := strconv.Atoi(string(key[sep+1:]))
if err != nil || counter < 0 {
return 0
}
return counter
}
package store
import (
"fmt"
"os"
bolt "go.etcd.io/bbolt"
)
// boltUpdate runs fn in a read-write transaction. A variable so tests can
// make the commit fail after fn has run, which is the failure the prune
// helpers must report honestly.
var boltUpdate = func(b *bolt.DB, fn func(*bolt.Tx) error) error {
return b.Update(fn)
}
// committedCount turns a per-transaction deletion count into what actually
// happened: a transaction that failed to commit removed nothing, whatever
// the loop inside it counted.
func committedCount(what string, err error, removed int) int {
if err != nil {
fmt.Fprintf(os.Stderr, "store: %s prune failed to commit, %d row(s) kept: %v\n", what, removed, err)
return 0
}
return removed
}
package store
import (
"encoding/json"
"fmt"
"time"
bolt "go.etcd.io/bbolt"
)
// WPVerificationResult describes a completed attempt without storing command
// output, which may contain account credentials or arbitrary PHP output.
type WPVerificationResult struct {
State string `json:"state"`
Reason string `json:"reason,omitempty"`
}
// WPVerificationRecord retains the last result and a bounded failure streak.
type WPVerificationRecord struct {
WPVerificationResult
Account string `json:"account,omitempty"`
ObservedAt time.Time `json:"observed_at"`
AttemptAt time.Time `json:"attempt_at"`
Failures int `json:"failures"`
// Keep the preceding attempt so overlapping scans can finish out of order
// without inventing or losing a consecutive failure.
PreviousAttemptAt time.Time `json:"previous_attempt_at,omitzero"`
PreviousState string `json:"previous_state,omitempty"`
// AbsentAt fences off attempts from before this installation was removed.
AbsentAt time.Time `json:"absent_at,omitzero"`
}
type wpVerificationState struct {
Rows map[string]WPVerificationRecord `json:"rows"`
// Completed discovery watermarks prevent a late scan from resurrecting
// installations removed by a newer full or account-scoped discovery.
Discovery map[string]time.Time `json:"discovery"`
}
func wpVerificationKey(kind string) ([]byte, error) {
if kind != "core" && kind != "plugins" {
return nil, fmt.Errorf("unknown WordPress verification kind %q", kind)
}
return []byte("wp_verification:" + kind), nil
}
func readWPVerification(b *bolt.Bucket, key []byte) (wpVerificationState, error) {
s := wpVerificationState{}
if data := b.Get(key); data != nil {
if err := json.Unmarshal(data, &s); err != nil {
return s, fmt.Errorf("decode WordPress verification state: %w", err)
}
}
if s.Rows == nil {
s.Rows = make(map[string]WPVerificationRecord)
}
if s.Discovery == nil {
s.Discovery = make(map[string]time.Time)
}
return s, nil
}
// UpdateWPVerification atomically merges one scan's actual attempts. Discovery
// alone never increments failures. Repeated consumers of the same scan share
// at, so they cannot turn one failed attempt into a persistent failure.
func (db *DB) UpdateWPVerification(kind string, at time.Time, scope string, discovered map[string]string, results map[string]WPVerificationResult, complete bool) error {
key, err := wpVerificationKey(kind)
if err != nil {
return err
}
if at.IsZero() {
return fmt.Errorf("WordPress verification requires a scan time")
}
return db.bolt.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("meta"))
s, err := readWPVerification(b, key)
if err != nil {
return err
}
for path, account := range discovered {
if scope != "" && account != scope {
continue
}
row, exists := s.Rows[path]
if !exists {
row.AbsentAt = s.Discovery[""]
if s.Discovery[account].After(row.AbsentAt) {
row.AbsentAt = s.Discovery[account]
}
}
if row.AbsentAt.After(at) || (row.Account != account && row.ObservedAt.After(at)) {
continue
}
if !row.ObservedAt.After(at) {
row.Account, row.ObservedAt = account, at
}
if result, ok := results[path]; ok {
if attemptErr := row.recordAttempt(at, result); attemptErr != nil {
return attemptErr
}
}
s.Rows[path] = row
}
if complete && !s.Discovery[""].After(at) && !s.Discovery[scope].After(at) {
for path, row := range s.Rows {
if _, found := discovered[path]; !found && (scope == "" || row.Account == scope) && !row.ObservedAt.After(at) {
delete(s.Rows, path)
}
}
if scope == "" {
for account, prior := range s.Discovery {
if !prior.After(at) {
delete(s.Discovery, account)
}
}
}
s.Discovery[scope] = at
}
data, err := json.Marshal(s)
if err != nil {
return err
}
return b.Put(key, data)
})
}
func (row *WPVerificationRecord) recordAttempt(at time.Time, result WPVerificationResult) error {
if at.Equal(row.AttemptAt) || !at.After(row.PreviousAttemptAt) {
return nil
}
switch result.State {
case "verified", "modified", "not_wordpress", "unverified":
default:
return fmt.Errorf("invalid WordPress verification result %q", result.State)
}
if at.After(row.AttemptAt) {
row.PreviousAttemptAt, row.PreviousState = row.AttemptAt, row.State
row.WPVerificationResult, row.AttemptAt = result, at
} else {
row.PreviousAttemptAt, row.PreviousState = at, result.State
}
row.Failures = 0
if row.State == "unverified" {
row.Failures = 1
if row.PreviousState == "unverified" {
row.Failures = 2
}
}
return nil
}
// WPVerification reports persisted coverage, returning read errors rather than
// presenting missing evidence as a clean empty inventory.
func (db *DB) WPVerification(kind string) (map[string]WPVerificationRecord, error) {
key, err := wpVerificationKey(kind)
if err != nil {
return nil, err
}
var rows map[string]WPVerificationRecord
err = db.bolt.View(func(tx *bolt.Tx) error {
s, readErr := readWPVerification(tx.Bucket([]byte("meta")), key)
rows = s.Rows
return readErr
})
return rows, err
}
package systemdrun
import (
"context"
"errors"
"fmt"
)
// LookPathFunc resolves an executable, returning a non-nil error when it
// cannot safely be used. Callers inject their own so tests do not depend on
// the host.
type LookPathFunc func(file string) (string, error)
// RunnerFunc executes a command and returns its output. Callers inject the
// runner their package already uses, which keeps command execution behind one
// abstraction per package instead of two.
type RunnerFunc func(ctx context.Context, name string, args ...string) ([]byte, error)
// InputRunnerFunc is a RunnerFunc that also writes input to the command's
// stdin.
type InputRunnerFunc func(ctx context.Context, input []byte, name string, args ...string) ([]byte, error)
// Run executes name/args as a transient unit forked by PID 1, so the command
// escapes the caller's service sandbox, and returns its output.
//
// When systemd-run cannot be used -- lookup fails, or a harmless probe fails
// while the caller's context remains active -- the command runs directly
// instead. That is no worse than not having the wrapper at all, and it is the
// only honest option: giving up would disable the command entirely on a host
// where it might still work. Options.Required turns both cases into
// ErrUnavailable, for commands that would half-apply inside the sandbox.
//
// The caller's command is attempted at most once. Its exit status and output
// cannot be told apart from systemd-run's own (--wait and --pipe propagate the
// unit's status and output verbatim), so a failure is never retried: running it
// again would repeat side effects like freezing or removing queued mail, and
// would mask the real error.
func Run(ctx context.Context, lookPath LookPathFunc, run RunnerFunc, opt Options, name string, args ...string) ([]byte, error) {
return runUnit(ctx, lookPath, func(ctx context.Context, _ []byte, name string, args ...string) ([]byte, error) {
return run(ctx, name, args...)
}, opt, nil, name, args...)
}
// RunInput is Run with input on the command's stdin. systemd-run hands the
// unit its own stdin only with --pipe, so RunInput always pipes. The
// availability probe gets no input.
func RunInput(ctx context.Context, lookPath LookPathFunc, run InputRunnerFunc, opt Options, input []byte, name string, args ...string) ([]byte, error) {
opt.Pipe, opt.NoWait = true, false
return runUnit(ctx, lookPath, run, opt, input, name, args...)
}
func runUnit(ctx context.Context, lookPath LookPathFunc, run InputRunnerFunc, opt Options, input []byte, name string, args ...string) ([]byte, error) {
systemdRunPath, lookupErr := lookPath("systemd-run")
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
}
if lookupErr != nil || systemdRunPath == "" {
if opt.Required {
if lookupErr != nil {
return nil, fmt.Errorf("%w: %w", ErrUnavailable, lookupErr)
}
return nil, ErrUnavailable
}
return run(ctx, input, name, args...)
}
// Probe with a command that has no side effects, so the decision to fall
// back is made before the caller's command has had a chance to run.
probeName, probeArgs := Argv(systemdRunPath, opt, probeCommand)
_, probeErr := run(ctx, nil, probeName, probeArgs...)
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
}
if probeErr != nil {
if opt.Required {
return nil, fmt.Errorf("%w: %w", ErrUnavailable, probeErr)
}
return run(ctx, input, name, args...)
}
wrappedName, wrappedArgs := Argv(systemdRunPath, opt, name, args...)
return run(ctx, input, wrappedName, wrappedArgs...)
}
// ErrUnavailable means lookup or the availability probe failed for a Required
// command, which was not attempted. Wrapped command failures do not carry this
// guarantee because the command may already have changed state.
var ErrUnavailable = errors.New("systemd-run cannot start a transient unit")
// probeCommand is the no-op used to test whether systemd-run can start a unit.
// /bin/true is part of coreutils, which systemd itself depends on.
const probeCommand = "/bin/true"
// Package systemdrun builds argv for running a command outside the calling
// service's systemd sandbox.
//
// CSM's daemon runs under ProtectSystem=strict with a narrow ReadWritePaths
// allow-list. Heavyweight system tools write paths that allow-list cannot
// reasonably enumerate, and some refuse to start at all when a path they need
// is read-only. Handing such a command to systemd-run makes PID 1 fork it as a
// transient unit, outside csm.service's mount namespace and seccomp filters.
//
// Never use --scope for this: scope mode wraps a child of the sandboxed
// process, so it inherits exactly the restrictions the wrapper exists to
// escape.
package systemdrun
import (
"fmt"
"strings"
"time"
)
// Options tunes how the transient unit is run.
type Options struct {
// Pipe connects the unit's stdio to the caller so its output can be
// captured. Without it the unit's stdout goes to the journal and the
// caller reads nothing. --pipe already implies waiting for the unit.
Pipe bool
// RuntimeMax bounds the unit's lifetime via RuntimeMaxSec. Zero omits the
// property, leaving the unit unbounded.
RuntimeMax time.Duration
// Required makes Run return ErrUnavailable instead of running the command
// directly, for commands that must never run inside the caller's sandbox.
Required bool
// NoWait returns as soon as the unit has started, with neither --pipe nor
// --wait, for commands whose run time the caller must not wait out. It
// overrides Pipe: the unit's output goes to the journal, and an error
// means only that the unit could not be started.
NoWait bool
// KillMode sets the unit's KillMode property; empty keeps systemd's
// default, control-group, which stops every process the command forked
// when its main process ends. "process" stops only the main process,
// for a command that forks work meant to outlive it.
KillMode string
// Unit names the transient unit, so the caller can stop it by name.
// Empty lets systemd pick one.
Unit string
// Properties are extra unit properties, one --property each.
Properties []string
}
// Argv returns the argv that runs name/args as a transient unit. When
// systemdRunPath is empty -- systemd-run is not installed, so there is no
// sandbox to escape either -- the command's own argv is returned unchanged.
func Argv(systemdRunPath string, opt Options, name string, args ...string) (string, []string) {
if systemdRunPath == "" {
return name, args
}
flags := []string{"--quiet", "--collect"}
switch {
case opt.NoWait:
case opt.Pipe:
flags = append(flags, "--pipe")
default:
flags = append(flags, "--wait")
}
if opt.RuntimeMax > 0 {
flags = append(flags, "--property=RuntimeMaxSec="+formatRuntimeMax(opt.RuntimeMax))
}
if opt.KillMode != "" {
flags = append(flags, "--property=KillMode="+opt.KillMode)
}
if opt.Unit != "" {
flags = append(flags, "--unit="+opt.Unit)
}
for _, p := range opt.Properties {
flags = append(flags, "--property="+p)
}
flags = append(flags, "--")
flags = append(flags, name)
// PID 1 expands dollar expressions even though no shell is involved.
// Doubling dollars preserves literal argv on older systemd versions that
// do not support --expand-environment=no. Direct execution needs no escape.
for _, arg := range args {
flags = append(flags, strings.ReplaceAll(arg, "$", "$$"))
}
return systemdRunPath, flags
}
func formatRuntimeMax(d time.Duration) string {
// systemd time spans have microsecond granularity. Round a positive
// sub-microsecond duration up so it never becomes the special zero value,
// then render fractional seconds without float rounding.
microseconds := d / time.Microsecond
if d%time.Microsecond != 0 {
microseconds++
}
seconds := microseconds / 1_000_000
fraction := microseconds % 1_000_000
if fraction == 0 {
return fmt.Sprintf("%ds", seconds)
}
fractionText := strings.TrimRight(fmt.Sprintf("%06d", fraction), "0")
return fmt.Sprintf("%d.%ss", seconds, fractionText)
}
package threat
import (
"encoding/json"
"os"
"path/filepath"
"time"
"github.com/pidginhost/csm/internal/attackdb"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/store"
)
// IPIntelligence is the complete picture of an IP from all sources.
type IPIntelligence struct {
IP string `json:"ip"`
// Internal attack history
AttackRecord *attackdb.IPRecord `json:"attack_record,omitempty"`
LocalScore int `json:"local_score"`
// SpamScore is the record's spam evidence score.
SpamScore int `json:"spam_score,omitempty"`
// ThreatDB (feeds + permanent blocklist)
InThreatDB bool `json:"in_threat_db"`
ThreatDBSource string `json:"threat_db_source,omitempty"`
// ThreatDBPermanent marks local evidence that never lapses, so an
// unblocked IP keeps scoring as malicious and is flagged again on its
// next sighting. ThreatDBExpiresAt is set when the evidence lapses
// with the block that recorded it.
ThreatDBPermanent bool `json:"threat_db_permanent,omitempty"`
ThreatDBExpiresAt *time.Time `json:"threat_db_expires_at,omitempty"`
// AbuseIPDB cache
AbuseScore int `json:"abuse_score"`
AbuseCategory string `json:"abuse_category,omitempty"`
// Firewall state
CurrentlyBlocked bool `json:"currently_blocked"`
BlockReason string `json:"block_reason,omitempty"`
BlockedAt *time.Time `json:"blocked_at,omitempty"`
BlockExpiresAt *time.Time `json:"block_expires_at,omitempty"`
BlockPermanent bool `json:"block_permanent,omitempty"`
// GeoIP (populated by API layer, not by Lookup)
Country string `json:"country,omitempty"`
CountryName string `json:"country_name,omitempty"`
City string `json:"city,omitempty"`
ASN uint `json:"asn,omitempty"`
ASOrg string `json:"as_org,omitempty"`
Network string `json:"network,omitempty"`
// Composite
UnifiedScore int `json:"unified_score"`
Verdict string `json:"verdict"` // "clean", "suspicious", "malicious", "blocked"
}
// Lookup returns the full intelligence picture for an IP.
// All reads are local (no network calls). Pre-loads shared state files
// once (same as LookupBatch) to avoid re-reading per field.
func Lookup(ip, statePath string) *IPIntelligence {
abuseCache := loadFullAbuseCache(statePath)
blockMap := loadFullBlockState(statePath)
intel := &IPIntelligence{
IP: ip,
AbuseScore: -1, // not cached
}
// 1. Attack DB
if adb := attackdb.Global(); adb != nil {
if rec := adb.LookupIP(ip); rec != nil {
intel.AttackRecord = rec
intel.LocalScore = rec.ThreatScore
intel.SpamScore = attackdb.SpamScore(rec.Mail, time.Now())
}
}
// 2. ThreatDB (feeds + permanent)
if tdb := checks.GetThreatDB(); tdb != nil {
applyThreatDBMatch(intel, tdb)
}
// 3. AbuseIPDB from pre-loaded cache
if entry, ok := abuseCache[ip]; ok {
intel.AbuseScore = entry.Score
intel.AbuseCategory = entry.Category
}
// 4. Block state from pre-loaded map
applyBlockState(intel, blockMap)
computeVerdict(intel)
return intel
}
// LookupBatch returns intelligence for multiple IPs efficiently.
// Pre-loads shared state files once instead of per-IP.
func LookupBatch(ips []string, statePath string) []*IPIntelligence {
results := make([]*IPIntelligence, len(ips))
abuseCache := loadFullAbuseCache(statePath)
blockMap := loadFullBlockState(statePath)
for i, ip := range ips {
intel := &IPIntelligence{
IP: ip,
AbuseScore: -1,
}
// Attack DB
if adb := attackdb.Global(); adb != nil {
if rec := adb.LookupIP(ip); rec != nil {
intel.AttackRecord = rec
intel.LocalScore = rec.ThreatScore
intel.SpamScore = attackdb.SpamScore(rec.Mail, time.Now())
}
}
// ThreatDB
if tdb := checks.GetThreatDB(); tdb != nil {
applyThreatDBMatch(intel, tdb)
}
// AbuseIPDB from pre-loaded cache
if entry, ok := abuseCache[ip]; ok {
intel.AbuseScore = entry.Score
intel.AbuseCategory = entry.Category
}
// Block state from pre-loaded map
applyBlockState(intel, blockMap)
computeVerdict(intel)
results[i] = intel
}
return results
}
// applyThreatDBMatch copies the threat-DB match and its lifetime onto intel.
func applyThreatDBMatch(intel *IPIntelligence, tdb *checks.ThreatDB) {
match, found := tdb.LookupMatch(intel.IP)
if !found {
return
}
intel.InThreatDB = true
intel.ThreatDBSource = match.Source
intel.ThreatDBPermanent = match.Permanent
if !match.ExpiresAt.IsZero() {
t := match.ExpiresAt
intel.ThreatDBExpiresAt = &t
}
}
func computeVerdict(intel *IPIntelligence) {
intel.UnifiedScore = intel.LocalScore
if intel.AbuseScore > intel.UnifiedScore {
intel.UnifiedScore = intel.AbuseScore
}
if intel.InThreatDB && intel.UnifiedScore < 100 {
intel.UnifiedScore = 100
}
switch {
case intel.CurrentlyBlocked:
intel.Verdict = "blocked"
case intel.UnifiedScore >= 80:
intel.Verdict = "malicious"
case intel.UnifiedScore >= 40:
intel.Verdict = "suspicious"
default:
intel.Verdict = "clean"
}
}
func applyBlockState(intel *IPIntelligence, blockMap map[string]*blockEntry) {
bs, ok := blockMap[intel.IP]
if !ok {
return
}
intel.CurrentlyBlocked = true
intel.BlockReason = bs.reason
intel.BlockPermanent = bs.permanent
if !bs.blockedAt.IsZero() {
t := bs.blockedAt
intel.BlockedAt = &t
}
if !bs.expiresAt.IsZero() && bs.expiresAt.Year() > 1 {
t := bs.expiresAt
intel.BlockExpiresAt = &t
}
}
// --- AbuseIPDB cache reader ---
type abuseEntry struct {
Score int `json:"score"`
Category string `json:"category"`
}
func loadFullAbuseCache(statePath string) map[string]*abuseEntry {
result := make(map[string]*abuseEntry)
sixHoursAgo := time.Now().Add(-6 * time.Hour)
// Try bbolt store first - after migration the flat file is renamed to .bak.
if sdb := store.Global(); sdb != nil {
for ip, entry := range sdb.AllReputation() {
if entry.CheckedAt.Before(sixHoursAgo) || entry.Score < 0 {
continue // expired or error sentinel
}
result[ip] = &abuseEntry{Score: entry.Score, Category: entry.Category}
}
return result
}
// Fallback: flat-file JSON (pre-migration).
type cacheEntry struct {
Score int `json:"score"`
Category string `json:"category"`
CheckedAt time.Time `json:"checked_at"`
}
type cacheFile struct {
Entries map[string]*cacheEntry `json:"entries"`
}
// #nosec G304 -- filepath.Join under operator-configured statePath.
data, err := os.ReadFile(filepath.Join(statePath, "reputation_cache.json"))
if err != nil {
return result
}
var cf cacheFile
if json.Unmarshal(data, &cf) != nil || cf.Entries == nil {
return result
}
for ip, entry := range cf.Entries {
if entry.CheckedAt.Before(sixHoursAgo) || entry.Score < 0 {
continue // expired or error sentinel
}
result[ip] = &abuseEntry{Score: entry.Score, Category: entry.Category}
}
return result
}
// --- Firewall block state reader ---
type blockEntry struct {
reason string
blockedAt time.Time
expiresAt time.Time
permanent bool
}
func loadFullBlockState(statePath string) map[string]*blockEntry {
result := make(map[string]*blockEntry)
now := time.Now()
// Read the authoritative firewall engine state (flat-file state.json).
// The bbolt fw:blocked bucket is written only at migration, so it would
// return a stale snapshot rather than the live block set.
if fwState, err := firewall.LoadState(statePath); err == nil && fwState != nil {
for _, entry := range fwState.Blocked {
perm := entry.ExpiresAt.IsZero() || entry.ExpiresAt.Year() <= 1
result[entry.IP] = &blockEntry{
reason: entry.Reason,
blockedAt: entry.BlockedAt,
expiresAt: entry.ExpiresAt,
permanent: perm,
}
}
}
// A present engine state wins even when its display is empty. Legacy
// bookkeeping cannot prove that a held-back or removed block is enforced.
if _, err := os.Stat(filepath.Join(statePath, "firewall", "state.json")); !os.IsNotExist(err) {
return result
}
// CSM blocked_ips.json (legacy fallback when engine state is absent)
type csmEntry struct {
IP string `json:"ip"`
Reason string `json:"reason"`
BlockedAt time.Time `json:"blocked_at"`
ExpiresAt time.Time `json:"expires_at"`
}
type csmFile struct {
IPs []csmEntry `json:"ips"`
}
// #nosec G304 -- filepath.Join under operator-configured statePath.
if data, err := os.ReadFile(filepath.Join(statePath, "blocked_ips.json")); err == nil {
var cf csmFile
if json.Unmarshal(data, &cf) == nil {
for _, entry := range cf.IPs {
if entry.ExpiresAt.IsZero() || now.Before(entry.ExpiresAt) {
if _, exists := result[entry.IP]; !exists {
perm := entry.ExpiresAt.IsZero() || entry.ExpiresAt.Year() <= 1
result[entry.IP] = &blockEntry{
reason: entry.Reason,
blockedAt: entry.BlockedAt,
expiresAt: entry.ExpiresAt,
permanent: perm,
}
}
}
}
}
}
return result
}
// Package threatintel -- bot allowlist + verification.
//
// botallowlist.go owns the embedded static IP CIDR ranges and the UA
// substring -> claimed-bot mapping. The static snapshots are a fast positive
// allow path: if the source IP falls inside a published bot range, skip the
// request without rDNS. The embedded snapshots are a trusted fallback; the
// bot-ranges auto-updater (botranges_update.go) refreshes them at runtime.
package threatintel
import (
_ "embed"
"encoding/json"
"net"
"strings"
"sync"
)
//go:embed embed/googlebot.json
var googlebotJSON []byte
//go:embed embed/bingbot.json
var bingbotJSON []byte
//go:embed embed/applebot.json
var applebotJSON []byte
// AI crawlers publish IP ranges rather than crawler reverse DNS, so they ship
// as embedded snapshots and are kept current by the auto-updater. OpenAI's
// three feeds (GPTBot, ChatGPT-User, OAI-SearchBot) all verify "gptbot".
//
//go:embed embed/openai-gptbot.json
var openaiGPTBotJSON []byte
//go:embed embed/openai-chatgpt-user.json
var openaiChatGPTUserJSON []byte
//go:embed embed/openai-searchbot.json
var openaiSearchBotJSON []byte
//go:embed embed/perplexitybot.json
var perplexitybotJSON []byte
// Anthropic publishes one combined crawler feed (ClaudeBot, Claude-User,
// Claude-SearchBot) at claude.com/crawling/bots.json. It documents IP-list
// verification, not reverse DNS, so the snapshot is the authoritative source.
//
//go:embed embed/claudebot.json
var claudebotJSON []byte
// BotRanges holds the parsed allowlist data, indexed by claimed-bot
// identity ("googlebot", "bingbot", "applebot").
type BotRanges struct {
byBot map[string][]*net.IPNet
}
type embedFile struct {
Prefixes []struct {
IPv4 string `json:"ipv4Prefix"`
IPv6 string `json:"ipv6Prefix"`
} `json:"prefixes"`
}
var (
defaultRanges *BotRanges
rangesOnce sync.Once
)
// DefaultRanges parses the embedded snapshots once and returns the
// global BotRanges. Safe to call concurrently.
func DefaultRanges() *BotRanges {
rangesOnce.Do(func() {
defaultRanges = &BotRanges{byBot: map[string][]*net.IPNet{}}
// A slice (not a map) so several feeds can share one identity:
// OpenAI's three crawler feeds all populate "gptbot".
for _, src := range []struct {
bot string
raw []byte
}{
{"googlebot", googlebotJSON},
{"bingbot", bingbotJSON},
{"applebot", applebotJSON},
{"gptbot", openaiGPTBotJSON},
{"gptbot", openaiChatGPTUserJSON},
{"gptbot", openaiSearchBotJSON},
{"perplexitybot", perplexitybotJSON},
{"claudebot", claudebotJSON},
} {
bot, raw := src.bot, src.raw
var f embedFile
if err := json.Unmarshal(raw, &f); err != nil {
continue
}
for _, p := range f.Prefixes {
if cidr := p.IPv4; cidr != "" {
if _, n, err := net.ParseCIDR(cidr); err == nil {
defaultRanges.byBot[bot] = append(defaultRanges.byBot[bot], n)
}
}
if cidr := p.IPv6; cidr != "" {
if _, n, err := net.ParseCIDR(cidr); err == nil {
defaultRanges.byBot[bot] = append(defaultRanges.byBot[bot], n)
}
}
}
}
})
return defaultRanges
}
// IPInBot reports whether the given IP falls inside the static range
// of the given bot identity.
func (r *BotRanges) IPInBot(ip net.IP, bot string) bool {
if ip == nil {
return false
}
for _, n := range r.byBot[bot] {
if n.Contains(ip) {
return true
}
}
for _, n := range fetchedRangesFor(bot) {
if n.Contains(ip) {
return true
}
}
return false
}
// IPInAnyBot reports whether the IP falls inside any published crawler
// range (Googlebot/Bingbot/Applebot snapshots). Unlike IPInBot it needs
// no claimed-UA, so callers that only have an IP (e.g. the incident
// correlator's whitelist backstop) can recognise a verified-crawler
// address. Deliberately covers only crawlers that publish authoritative
// IP ranges -- CDN edge ranges are NOT included, because legitimate and
// malicious traffic share a CDN's egress IPs and whitelisting them would
// hide attacks proxied through the CDN.
func (r *BotRanges) IPInAnyBot(ip net.IP) bool {
if ip == nil {
return false
}
for _, nets := range r.byBot {
for _, n := range nets {
if n.Contains(ip) {
return true
}
}
}
for _, nets := range fetchedRangesSnapshot() {
for _, n := range nets {
if n.Contains(ip) {
return true
}
}
}
return false
}
// AICrawlerRangePrefixCounts returns the active prefix count for each built-in
// AI crawler that has a vendor range feed. Embedded snapshots are counted along
// with any fetched overlay so status views never report "none" while the
// packaged fallback ranges are still active.
func AICrawlerRangePrefixCounts() map[string]int {
sourceBots := map[string]struct{}{}
for _, src := range DefaultRangeSources() {
sourceBots[src.Bot] = struct{}{}
}
seen := map[string]map[string]struct{}{}
add := func(bot string, nets []*net.IPNet) {
if len(nets) == 0 {
return
}
if seen[bot] == nil {
seen[bot] = map[string]struct{}{}
}
for _, n := range nets {
if n != nil {
seen[bot][n.String()] = struct{}{}
}
}
}
defaults := DefaultRanges()
for bot := range sourceBots {
add(bot, defaults.byBot[bot])
}
for bot, nets := range fetchedRangesSnapshot() {
add(bot, nets)
}
out := make(map[string]int, len(seen))
for bot, prefixes := range seen {
out[bot] = len(prefixes)
}
return out
}
// ClaimedBotFromUA returns the lower-case bot identity if the UA looks
// like a known bot. Empty string otherwise. Identities match BotDomains
// keys in botverify.go so the async verifier can look up the right
// DNS suffix list.
func ClaimedBotFromUA(ua string) string {
low := strings.ToLower(ua)
switch {
case strings.Contains(low, "googlebot"):
return "googlebot"
case strings.Contains(low, "bingbot"):
return "bingbot"
case strings.Contains(low, "applebot"):
return "applebot"
// Appendix A bots: no published static IP range.
case strings.Contains(low, "duckduckbot"):
return "duckduckbot"
case strings.Contains(low, "amazonbot"):
return "amazonbot"
case strings.Contains(low, "gptbot"),
strings.Contains(low, "chatgpt-user"),
strings.Contains(low, "oai-searchbot"):
return "gptbot"
case strings.Contains(low, "claudebot"),
strings.Contains(low, "claude-user"),
strings.Contains(low, "claude-searchbot"):
return "claudebot"
case strings.Contains(low, "perplexitybot"):
return "perplexitybot"
case strings.Contains(low, "meta-externalagent"),
strings.Contains(low, "meta-webindexer"),
strings.Contains(low, "facebookexternalhit"):
return "facebookbot"
case strings.Contains(low, "bravebot"):
return "bravebot"
// SEO backlink crawlers: no published static IP range, rDNS-verified.
case strings.Contains(low, "seranking"):
return "seranking"
default:
// Operator-configured bots (reputation.verified_bots) extend the
// built-in set without a code change.
return OperatorBotFromUA(low)
}
}
package threatintel
import (
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os"
"sort"
"strings"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/atomicio"
"github.com/pidginhost/csm/internal/netutil"
)
// fetchedRanges is the runtime-updatable overlay of vendor-published IP ranges,
// keyed by bot identity. It augments the embedded snapshots in DefaultRanges so
// crawler allowlists refresh without a new release. Swapped wholesale; scan-path
// readers load the pointer lock-free.
var fetchedRanges atomic.Pointer[map[string][]*net.IPNet]
// PublishFetchedRanges installs the runtime range overlay. nil clears it.
func PublishFetchedRanges(m map[string][]*net.IPNet) {
if len(m) == 0 {
fetchedRanges.Store(nil)
setLastRefreshUnix(0)
return
}
cp := make(map[string][]*net.IPNet, len(m))
for k, v := range m {
cp[k] = append([]*net.IPNet(nil), v...)
}
fetchedRanges.Store(&cp)
}
func fetchedRangesFor(bot string) []*net.IPNet {
p := fetchedRanges.Load()
if p == nil {
return nil
}
return (*p)[bot]
}
func fetchedRangesSnapshot() map[string][]*net.IPNet {
p := fetchedRanges.Load()
if p == nil {
return nil
}
return *p
}
// FetchedRangesSnapshot returns a shallow copy of the current overlay for
// inspection or merge by the updater.
func FetchedRangesSnapshot() map[string][]*net.IPNet {
snap := fetchedRangesSnapshot()
out := make(map[string][]*net.IPNet, len(snap))
for k, v := range snap {
out[k] = append([]*net.IPNet(nil), v...)
}
return out
}
// lastRefreshUnix is the Unix time of the last successful vendor fetch. It is
// persisted in the cache file so a restart restores the genuine fetch time
// rather than resetting it. Zero means never refreshed.
var lastRefreshUnix atomic.Int64
func setLastRefreshUnix(ts int64) { lastRefreshUnix.Store(ts) }
// LastFetchedRangesRefresh returns when the AI-crawler ranges were last fetched
// from the vendor feeds, or the zero time if they never have been. The web UI
// surfaces this so operators can see whether the auto-updater is keeping the
// snapshot current.
func LastFetchedRangesRefresh() time.Time {
ts := lastRefreshUnix.Load()
if ts == 0 {
return time.Time{}
}
return time.Unix(ts, 0)
}
// RefreshFetchedRanges fetches every source, persists the merged overlay to
// cachePath (skipped when empty), and publishes it. The first successful feed
// for a bot identity replaces that bot's previous overlay; later successful
// feeds for the same identity append. A bot whose every feed fails keeps its
// previous overlay, so a full vendor outage never narrows the allowlist.
// Returns the number of bot identities refreshed.
func RefreshFetchedRanges(ctx context.Context, client *http.Client, sources []RangeSource, cachePath string) (int, error) {
merged := FetchedRangesSnapshot()
updated := map[string]struct{}{}
var lastErr error
for _, src := range sources {
nets, err := FetchRange(ctx, client, src.URL)
if err != nil {
lastErr = err
continue
}
if _, ok := updated[src.Bot]; !ok {
merged[src.Bot] = nil
updated[src.Bot] = struct{}{}
}
merged[src.Bot] = append(merged[src.Bot], nets...)
}
if len(updated) == 0 {
return 0, lastErr
}
refreshedAt := time.Now().Unix()
if cachePath != "" {
if err := saveFetchedRanges(cachePath, merged, refreshedAt); err != nil {
return 0, err
}
}
PublishFetchedRanges(merged)
setLastRefreshUnix(refreshedAt)
return len(updated), nil
}
// RangeSource is one vendor IP-range feed mapped to a bot identity. Multiple
// sources may share an identity (OpenAI publishes GPTBot, ChatGPT-User and
// OAI-SearchBot separately; all verify the "gptbot" identity).
type RangeSource struct {
Bot string
URL string
}
// DefaultRangeSources are the vendor feeds the auto-updater refreshes. URLs are
// stable, vendor-published JSON in the {prefixes:[{ipv4Prefix|ipv6Prefix}]}
// shape. Anthropic's feed is one combined list for ClaudeBot, Claude-User, and
// Claude-SearchBot, all of which verify the "claudebot" identity.
func DefaultRangeSources() []RangeSource {
return []RangeSource{
{Bot: "gptbot", URL: "https://openai.com/gptbot.json"},
{Bot: "gptbot", URL: "https://openai.com/chatgpt-user.json"},
{Bot: "gptbot", URL: "https://openai.com/searchbot.json"},
{Bot: "perplexitybot", URL: "https://www.perplexity.ai/perplexitybot.json"},
{Bot: "claudebot", URL: "https://claude.com/crawling/bots.json"},
}
}
const (
maxRangeBytes = 4 << 20 // 4 MiB; vendor feeds are well under this
maxPrefixesPerFeed = 100000 // sanity cap against a runaway feed
)
// ParseRangeJSON parses a vendor range feed and returns valid, public,
// suitably-narrow CIDRs. Unparseable, over-broad, or non-public entries are
// dropped (not errors) so one bad row cannot poison the feed, and the same
// guards as operator-configured ranges stop a compromised or mistaken feed from
// allowlisting the whole internet.
func ParseRangeJSON(data []byte) ([]*net.IPNet, error) {
var f embedFile
if err := json.Unmarshal(data, &f); err != nil {
return nil, fmt.Errorf("bot range feed: %w", err)
}
var out []*net.IPNet
for _, p := range f.Prefixes {
for _, cidr := range []string{p.IPv4, p.IPv6} {
if cidr == "" {
continue
}
_, n, err := net.ParseCIDR(strings.TrimSpace(cidr))
if err != nil {
continue
}
n = netutil.NormalizeIPNet(n)
if n == nil || !operatorBotIPRangeAllowed(n) {
continue
}
out = append(out, n)
if len(out) >= maxPrefixesPerFeed {
return out, nil
}
}
}
return out, nil
}
// rangeFetchClient serves FetchRange callers that pass no client. A feed host
// that accepts the connection and never answers must not hang the updater,
// so it carries a timeout, unlike http.DefaultClient.
var rangeFetchClient = &http.Client{Timeout: 30 * time.Second}
// FetchRange downloads and parses one vendor range feed.
func FetchRange(ctx context.Context, client *http.Client, url string) ([]*net.IPNet, error) {
if client == nil {
client = rangeFetchClient
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("bot range feed %s: HTTP %d", url, resp.StatusCode)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, maxRangeBytes+1))
if err != nil {
return nil, err
}
if len(data) > maxRangeBytes {
return nil, fmt.Errorf("bot range feed %s: exceeds %d bytes", url, maxRangeBytes)
}
nets, err := ParseRangeJSON(data)
if err != nil {
return nil, err
}
if len(nets) == 0 {
return nil, fmt.Errorf("bot range feed %s: no valid prefixes", url)
}
return nets, nil
}
// rangeCacheFile is the on-disk persistence shape for the fetched overlay.
type rangeCacheFile struct {
Bots map[string][]string `json:"bots"`
// RefreshedAt is the Unix time the cache was written, i.e. when the ranges
// were last fetched. Persisted so the last-refresh time survives a restart.
RefreshedAt int64 `json:"refreshed_at,omitempty"`
}
// SaveFetchedRanges persists the overlay so a restart keeps the last-good feed
// until the next refresh completes. The write time is stamped as the
// last-refresh time.
func SaveFetchedRanges(path string, m map[string][]*net.IPNet) error {
return saveFetchedRanges(path, m, time.Now().Unix())
}
func saveFetchedRanges(path string, m map[string][]*net.IPNet, refreshedAt int64) error {
c := rangeCacheFile{Bots: map[string][]string{}, RefreshedAt: refreshedAt}
for bot, nets := range m {
strs := make([]string, 0, len(nets))
for _, n := range nets {
strs = append(strs, n.String())
}
sort.Strings(strs)
c.Bots[bot] = strs
}
data, err := json.Marshal(c)
if err != nil {
return err
}
return atomicio.AtomicWrite(path, 0o600, data)
}
// LoadFetchedRanges reads a previously saved overlay and publishes it. A
// missing file is not an error (first run). Entries are re-validated on load.
func LoadFetchedRanges(path string) error {
return loadFetchedRanges(path, true)
}
// LoadFetchedRangesRequired is the control-command variant: by the time the
// daemon is asked to reload, the CLI must already have written a cache file.
func LoadFetchedRangesRequired(path string) error {
return loadFetchedRanges(path, false)
}
func loadFetchedRanges(path string, allowMissing bool) error {
data, err := os.ReadFile(path) // #nosec G304 -- daemon-owned state path
if err != nil {
if allowMissing && os.IsNotExist(err) {
return nil
}
return err
}
var c rangeCacheFile
if err := json.Unmarshal(data, &c); err != nil {
return err
}
m := make(map[string][]*net.IPNet, len(c.Bots))
for bot, strs := range c.Bots {
for _, s := range strs {
_, n, err := net.ParseCIDR(s)
if err != nil {
continue
}
if n = netutil.NormalizeIPNet(n); n != nil && operatorBotIPRangeAllowed(n) {
m[bot] = append(m[bot], n)
}
}
}
PublishFetchedRanges(m)
setLastRefreshUnix(c.RefreshedAt)
return nil
}
package threatintel
import (
"context"
"errors"
"net"
"slices"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
type resolver interface {
LookupAddr(ctx context.Context, ip string) ([]string, error)
LookupIP(ctx context.Context, network, host string) ([]net.IP, error)
}
// verifier owns one resolver + a domain suffix list per bot identity.
// One verifier per bot identity in practice; tests construct directly.
type verifier struct {
res resolver
domains []string // lower-case suffix list, e.g. "googlebot.com"
}
func newVerifier(r resolver, domains []string) *verifier {
low := make([]string, len(domains))
for i, d := range domains {
low[i] = strings.ToLower(d)
}
return &verifier{res: r, domains: low}
}
// LogicVersion identifies the current shape of the bot-verifier logic
// (BotDomains suffix list, ClaimedBotFromUA mapping, no-PTR semantics).
// Bump this whenever a change here would invalidate cache entries
// written by an older build -- for example, adding a new domain suffix
// that turns prior negatives into positives, or adding a new UA -> bot
// identity mapping. The daemon calls store.DB.EnsureBotVerifyLogicVersion
// at startup with this value; a mismatch wipes the botverify bucket so
// the next scan re-verifies every IP under the new rules.
const LogicVersion = 5
// ErrUnverifiable signals that the resolver returned no usable PTR for
// the source IP, so the verifier cannot prove or disprove the claimed
// bot identity. Callers treat this as fail-open: do not cache a verdict or
// flag as spoof. Genuine spoof signals -- PTR present but outside the
// bot's domain suffix list, or forward-confirm mismatch -- still return
// (false, nil).
var ErrUnverifiable = errors.New("bot verify: no PTR record for source IP")
// verify performs Google's official PTR + forward-A method. Returns
// (true, nil) on success, (false, nil) on a definitive negative
// (PTR resolves but does not belong to the claimed bot's domain, or
// forward-A fails to round-trip the IP), (false, ErrUnverifiable) when
// the IP has no PTR at all, and (false, err) on context cancellation
// or transient resolver failure. Both error paths cause the async
// worker to skip the verdict write so unverifiable IPs do not get pinned
// as spoof for the TTL window.
func (v *verifier) verify(ctx context.Context, ip net.IP) (bool, error) {
names, err := v.res.LookupAddr(ctx, ip.String())
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return false, ctxErr
}
if isDNSNotFound(err) {
return false, ErrUnverifiable
}
return false, err
}
if len(names) == 0 {
return false, ErrUnverifiable
}
matched := ""
for _, n := range names {
ln := strings.ToLower(strings.TrimSuffix(n, "."))
for _, suf := range v.domains {
if strings.HasSuffix(ln, "."+suf) || ln == suf {
matched = ln
break
}
}
if matched != "" {
break
}
}
if matched == "" {
return false, nil
}
addrs, err := v.res.LookupIP(ctx, "ip", matched)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return false, ctxErr
}
if isDNSNotFound(err) {
return false, nil
}
return false, err
}
for _, a := range addrs {
if a.Equal(ip) {
return true, nil
}
}
return false, nil
}
func isDNSNotFound(err error) bool {
var dnsErr *net.DNSError
return errors.As(err, &dnsErr) && dnsErr.IsNotFound && !dnsErr.IsTemporary && !dnsErr.IsTimeout
}
// AsyncBotVerifier runs PTR+forward-A verify in a single background
// goroutine, deduplicating in-flight jobs. Result writes through the
// put callback (store.DB.PutBotVerify); reads happen from the scan
// hot path via store.DB.GetBotVerify with no goroutine.
type AsyncBotVerifier struct {
mu sync.Mutex
inflight map[string]time.Time
attempts map[string]botVerifyAttempt
ch chan verifyJob
v map[string]*verifier // bot identity -> verifier; guarded by mu
res resolver // retained so SetOperatorEntries can rebuild v
put func(net.IP, string, bool, time.Time) error
// unverifiable holds no-PTR results. A missing PTR rarely changes between
// scans, so repeat claims wait for the record to lapse instead of queuing
// the same lookup on every scan and crowding out crawlers not yet checked.
unverifiable UnverifiableRecords
// unverifiableSwept tracks sweep attempts, including failures, and is
// owned by the single worker that writes records.
unverifiableSwept time.Time
stats *queuehealth.Tracker
stop <-chan struct{}
closed bool
}
// Attempt history prevents each unresolved retry from renewing the initial
// grace. It is bounded by queue capacity. New sources can replace the oldest
// completed attempt after its initial cooldown, so repeated failures cannot
// reserve every slot for the full cache TTL. A live job or newly granted grace
// is never evicted; retries cannot slide that reservation indefinitely.
type botVerifyAttempt struct {
retryAfter time.Time
retainUntil time.Time
expiresAt time.Time
}
type verifyJob struct {
IP net.IP
Bot string
ticket queuehealth.Ticket
}
// BotDomains maps built-in claimed-bot identities to DNS suffixes for
// reverse and forward DNS verification. The HTTP classifier checks static
// ranges first and uses this verifier for addresses outside those ranges.
var BotDomains = map[string][]string{
"googlebot": {"googlebot.com", "google.com"},
"bingbot": {"search.msn.com"},
"applebot": {"applebot.apple.com", "apple.com"},
"duckduckbot": {"duckduckgo.com"},
"amazonbot": {"amazonbot.amazon", "amazon.com", "developer.amazon.com"},
"gptbot": {"openai.com"},
"claudebot": {"anthropic.com"},
"perplexitybot": {"perplexity.ai"},
"facebookbot": {"fbsv.net", "tfbnw.net", "facebook.com"},
"bravebot": {"brave.com"},
"seranking": {"seranking.com"},
}
// UnverifiableRecords persists when a claimed bot identity had no PTR. The
// verifier derives retry suppression and attempt history from that time and
// sweeps records older than the history. store.DB implements it.
type UnverifiableRecords interface {
PutBotVerifyUnverifiable(ip net.IP, bot string, observedAt time.Time) error
BotVerifyUnverifiable(ip net.IP, bot string) (observedAt time.Time, ok bool)
SweepBotVerifyUnverifiable(cutoff time.Time) (int, error)
}
// NewAsyncBotVerifier constructs an async verifier backed by the
// system resolver. put is store.DB.PutBotVerify or a test seam; records is
// the store, or nil to keep no-PTR results in retry history only.
func NewAsyncBotVerifier(put func(net.IP, string, bool, time.Time) error, records UnverifiableRecords) *AsyncBotVerifier {
res := net.DefaultResolver
a := &AsyncBotVerifier{
inflight: make(map[string]time.Time),
ch: make(chan verifyJob, 256),
v: make(map[string]*verifier),
res: res,
put: put,
unverifiable: records,
stats: queuehealth.New(256, time.Minute),
}
for bot, domains := range BotDomains {
a.v[bot] = newVerifier(res, domains)
}
return a
}
// SetOperatorEntries rebuilds the per-bot verifier set from the built-in
// BotDomains plus operator-configured entries. An operator entry naming a
// built-in extends that bot's suffix list; a new name adds its own verifier.
// Safe to call after Run has started (SIGHUP reload): v is swapped under mu,
// which the worker also holds when reading it.
func (a *AsyncBotVerifier) SetOperatorEntries(entries []BotEntry) {
entries = normalizeBotEntries(entries, false)
m := make(map[string]*verifier, len(BotDomains)+len(entries))
for bot, domains := range BotDomains {
m[bot] = newVerifier(a.res, domains)
}
for _, e := range entries {
if len(e.RDNSSuffixes) == 0 {
continue
}
if existing, ok := m[e.Name]; ok {
merged := append(append([]string(nil), existing.domains...), e.RDNSSuffixes...)
m[e.Name] = newVerifier(a.res, merged)
} else {
m[e.Name] = newVerifier(a.res, e.RDNSSuffixes)
}
}
a.mu.Lock()
a.v = m
a.mu.Unlock()
}
// Enqueue reports whether a job is queued or already in flight. Unsupported
// identities and unavailable capacity never receive pending treatment, and a
// source with a live no-PTR record is not queued again until it lapses.
func (a *AsyncBotVerifier) Enqueue(ip net.IP, bot string) bool {
key := bot + "|" + ip.String()
a.mu.Lock()
defer a.mu.Unlock()
// Serialize the record read with finish: a worker that writes after this
// read must still be in flight when we decide whether to admit a retry.
var recorded bool
if a.unverifiable != nil {
if observed, ok := a.unverifiable.BotVerifyUnverifiable(ip, bot); ok {
now := time.Now()
if !now.After(observed.Add(botVerifyUnverifiableTTL)) {
return false
}
// A lapsed record still counts as an attempt for as long as
// in-memory history would, so a restart or eviction cannot
// grant fresh pending treatment.
recorded = now.Before(observed.Add(botVerifyCacheTTL))
}
}
if a.closed {
a.stats.Lose(time.Now(), 1)
return false
}
select {
case <-a.stop:
a.stats.Lose(time.Now(), 1)
return false
default:
}
if _, ok := a.inflight[key]; ok {
return true
}
if ip == nil || a.v[bot] == nil {
return false
}
now := time.Now()
for attemptKey, previous := range a.attempts {
_, live := a.inflight[attemptKey]
if !live && !now.Before(previous.expiresAt) && !now.Before(previous.retryAfter) {
delete(a.attempts, attemptKey)
}
}
attempt, attempted := a.attempts[key]
if attempted && now.Before(attempt.retryAfter) {
return false
}
var replaceKey string
if !attempted && len(a.attempts) >= cap(a.ch) {
var oldest time.Time
for attemptKey, previous := range a.attempts {
if _, live := a.inflight[attemptKey]; live || now.Before(previous.retainUntil) {
continue
}
if replaceKey == "" || previous.expiresAt.Before(oldest) {
replaceKey = attemptKey
oldest = previous.expiresAt
}
}
if replaceKey == "" {
a.stats.Lose(now, 1)
return false
}
}
var pendingUntil time.Time
if !attempted && !recorded {
pendingUntil = now.Add(botVerifyTimeout)
}
a.inflight[key] = pendingUntil
// The caller may reuse its IP buffer as soon as admission returns. The
// queued lookup and its dedup key must retain the same address.
job := verifyJob{IP: slices.Clone(ip), Bot: bot, ticket: a.stats.Begin(time.Now())}
select {
case a.ch <- job:
if !attempted {
if a.attempts == nil {
a.attempts = make(map[string]botVerifyAttempt)
}
delete(a.attempts, replaceKey)
a.attempts[key] = botVerifyAttempt{
retainUntil: now.Add(botVerifyRetryDelay),
expiresAt: now.Add(botVerifyCacheTTL),
}
}
return true
default:
job.ticket.Reject(time.Now())
delete(a.inflight, key)
return false
}
}
// Pending is true only while an admitted job is live and its initial grace
// has not expired. Queue wait counts against the same bound as a DNS lookup.
func (a *AsyncBotVerifier) Pending(ip net.IP, bot string) bool {
a.mu.Lock()
defer a.mu.Unlock()
if a.closed || a.v[bot] == nil {
return false
}
select {
case <-a.stop:
return false
default:
}
until, ok := a.inflight[bot+"|"+ip.String()]
return ok && time.Now().Before(until)
}
const (
botVerifyTimeout = 5 * time.Second
botVerifyRetryDelay = time.Minute
botVerifyCacheTTL = 24 * time.Hour
// Shorter than a verdict: a crawler that gains a PTR is verified within
// the hour, while a stable no-PTR source costs one lookup per hour.
botVerifyUnverifiableTTL = time.Hour
)
func (a *AsyncBotVerifier) QueueStatuses(now time.Time) map[string]queuehealth.Status {
return map[string]queuehealth.Status{"requests": a.stats.Snapshot(now)}
}
// Run processes the queue until stopCh closes. Runs as a single
// goroutine so DNS calls are serialised; volume is bounded by the
// inflight dedup map so bursts do not launch unbounded goroutines.
//
// Closing stopCh cancels the parent context, so any in-flight verify
// returns from its DNS lookup immediately rather than holding the Run
// goroutine for the per-job 5s timeout.
func (a *AsyncBotVerifier) Run(stopCh <-chan struct{}) {
a.mu.Lock()
a.stop = stopCh
a.mu.Unlock()
ctx, cancel := context.WithCancel(context.Background())
bridge := make(chan struct{})
go func() {
defer close(bridge)
select {
case <-stopCh:
cancel()
case <-ctx.Done():
}
}()
defer func() {
cancel()
<-bridge
a.mu.Lock()
a.closed = true
close(a.ch)
a.mu.Unlock()
for job := range a.ch {
a.finish(job, false, false)
}
}()
for {
select {
case <-stopCh:
return
default:
}
select {
case <-ctx.Done():
return
case job := <-a.ch:
select {
case <-stopCh:
a.finish(job, false, false)
return
default:
}
a.processWithContext(ctx, job)
}
}
}
func (a *AsyncBotVerifier) process(job verifyJob) {
a.processWithContext(context.Background(), job)
}
func (a *AsyncBotVerifier) processWithContext(parent context.Context, job verifyJob) {
job.ticket.Start(time.Now())
completed, cached := false, false
defer func() { a.finish(job, completed, cached) }()
a.mu.Lock()
v, ok := a.v[job.Bot]
a.mu.Unlock()
if !ok {
completed = true
return
}
ctx, cancel := context.WithTimeout(parent, botVerifyTimeout)
defer cancel()
result, err := v.verify(ctx, job.IP)
cancel()
if err != nil {
completed = errors.Is(err, ErrUnverifiable) && a.recordUnverifiable(job)
return
}
if a.put == nil {
completed = true
return
}
cached = a.put(job.IP, job.Bot, result, time.Now().Add(botVerifyCacheTTL)) == nil
completed = cached
}
// recordUnverifiable reports whether a no-PTR result is settled. It is not a
// verdict, so a lapsed record prevents fresh pending grace while within the
// history window. An unwritten record leaves the work unaccounted for.
func (a *AsyncBotVerifier) recordUnverifiable(job verifyJob) bool {
if a.unverifiable == nil {
return true
}
now := time.Now()
if a.unverifiable.PutBotVerifyUnverifiable(job.IP, job.Bot, now) != nil {
return false
}
// Records past the attempt history affect nothing. Sweeping after a write
// bounds the bucket by recent no-PTR volume; once an hour keeps the scan
// off the per-result path. Failures also wait an hour, so a failing large
// transaction cannot hold up every subsequent result write.
if now.Sub(a.unverifiableSwept) >= botVerifyUnverifiableTTL {
a.unverifiableSwept = now
_, _ = a.unverifiable.SweepBotVerifyUnverifiable(now.Add(-botVerifyCacheTTL))
}
return true
}
func (a *AsyncBotVerifier) finish(job verifyJob, completed, cached bool) {
a.mu.Lock()
defer a.mu.Unlock()
key := job.Bot + "|" + job.IP.String()
delete(a.inflight, key)
if cached {
delete(a.attempts, key)
} else if attempt, tracked := a.attempts[key]; tracked {
attempt.retryAfter = time.Now().Add(botVerifyRetryDelay)
a.attempts[key] = attempt
}
if completed {
job.ticket.Finish(time.Now())
} else {
job.ticket.Reject(time.Now())
}
}
package threatintel
import (
"sync"
"sync/atomic"
"github.com/pidginhost/csm/internal/metrics"
)
var upstreamMetrics = struct {
mu sync.Mutex
registered map[*metrics.Registry]struct{}
active atomic.Pointer[UpstreamSource]
cacheHitsTotal atomic.Int64
cacheMissesTotal atomic.Int64
backendFailuresTotal atomic.Int64
}{
registered: make(map[*metrics.Registry]struct{}),
}
// RegisterUpstreamMetrics binds upstream counters to reg so operators
// can observe cache effectiveness and upstream health.
// Production callers pass metrics.Default(); tests pass an isolated
// registry. Idempotent per registry because reputation checks rebuild
// the upstream source every cycle.
func RegisterUpstreamMetrics(reg *metrics.Registry, src *UpstreamSource) {
if reg == nil || src == nil {
return
}
upstreamMetrics.active.Store(src)
upstreamMetrics.mu.Lock()
if _, ok := upstreamMetrics.registered[reg]; ok {
upstreamMetrics.mu.Unlock()
return
}
upstreamMetrics.registered[reg] = struct{}{}
upstreamMetrics.mu.Unlock()
registerUpstreamMetricsLocked(reg)
}
// ClearUpstreamMetricsSource clears the source used by the breaker
// gauge when upstream reputation is disabled by a hot-reloaded config.
func ClearUpstreamMetricsSource() {
upstreamMetrics.active.Store(nil)
}
func registerUpstreamMetricsLocked(reg *metrics.Registry) {
reg.RegisterCounterFunc(
"csm_threatintel_cache_hits_total",
"Upstream threat-intel cache hits.",
func() float64 {
return float64(upstreamMetrics.cacheHitsTotal.Load())
},
)
reg.RegisterCounterFunc(
"csm_threatintel_cache_misses_total",
"Upstream threat-intel lookups not served from the local cache.",
func() float64 {
return float64(upstreamMetrics.cacheMissesTotal.Load())
},
)
reg.RegisterCounterFunc(
"csm_threatintel_backend_failures_total",
"Upstream threat-intel backend failures (network, 4xx, 5xx, malformed body).",
func() float64 {
return float64(upstreamMetrics.backendFailuresTotal.Load())
},
)
reg.RegisterGaugeFunc(
"csm_threatintel_breaker_open",
"Circuit breaker for the upstream source; 1 when open (calls refused), 0 when closed or half-open.",
func() float64 {
src := activeUpstreamMetricsSource()
if src != nil && src.BreakerOpen() {
return 1
}
return 0
},
)
}
func activeUpstreamMetricsSource() *UpstreamSource {
return upstreamMetrics.active.Load()
}
func resetUpstreamMetricsForTest() {
upstreamMetrics.active.Store(nil)
upstreamMetrics.cacheHitsTotal.Store(0)
upstreamMetrics.cacheMissesTotal.Store(0)
upstreamMetrics.backendFailuresTotal.Store(0)
}
package threatintel
import (
"hash/fnv"
"net"
"sort"
"strconv"
"strings"
"sync/atomic"
"github.com/pidginhost/csm/internal/netutil"
)
// BotEntry is an operator-configured verified bot: claimed-UA substrings
// mapped to the rDNS suffixes or IP ranges that confirm it. These are additive
// on top of the built-in allowlist (BotDomains / the ClaimedBotFromUA switch /
// the embedded static IP ranges); operators extend coverage for crawlers CSM
// does not ship without a code change.
type BotEntry struct {
Name string
UASubstrings []string
RDNSSuffixes []string
// IPRanges are CIDRs (or single IPs) for bots that verify by address
// rather than reverse DNS -- AI agents that publish ranges and have no
// crawler-domain rDNS. Membership is checked synchronously on the scan
// path, so these need no async PTR lookup.
IPRanges []string
}
// operatorBots holds the active operator list. Swapped wholesale on reload;
// readers run on the scan hot path so the pointer load must stay lock-free.
var operatorBots atomic.Pointer[[]BotEntry]
// operatorNets is the parsed IP-range index (bot name -> CIDRs), kept in sync
// with operatorBots. Stored before operatorBots on update so a reader that
// sees a new bot name already finds its ranges.
var operatorNets atomic.Pointer[map[string][]*net.IPNet]
// SetOperatorBots installs the operator-configured bot list. Names,
// substrings, and suffixes are lower-cased and trimmed (leading dots on
// suffixes dropped) so matching is case-insensitive and consistent with the
// built-in tables. Entries without a name or any UA substring are skipped:
// the UA substring is what links a request to the identity.
func SetOperatorBots(entries []BotEntry) {
norm := normalizeBotEntries(entries, true)
nets := make(map[string][]*net.IPNet)
for _, e := range norm {
for _, r := range e.IPRanges {
if n := netutil.ParseCIDROrIP(r); n != nil && operatorBotIPRangeAllowed(n) {
nets[e.Name] = append(nets[e.Name], n)
}
}
}
operatorNets.Store(&nets)
operatorBots.Store(&norm)
}
// operatorBotIPRangeAllowed rejects ranges too broad to be a real crawler fleet
// and any non-public address space, so the allowlist cannot be turned into a
// blanket detection bypass. The public-space test is shared with the config
// validator via internal/netutil.
func operatorBotIPRangeAllowed(n *net.IPNet) bool {
ones, bits := n.Mask.Size()
if bits == 32 && ones < 16 {
return false
}
if bits == 128 && ones < 32 {
return false
}
return netutil.IsPublicIP(n.IP)
}
// IPInAnyVerifiedBotRange reports whether ip belongs to any verified-bot IP
// range: the built-in/auto-updated crawler snapshots plus operator
// reputation.verified_bots IP-range entries. Unlike the UA-keyed lookups it
// needs only an address, so the firewall auto-block guard can recognise a
// published crawler from the finding IP alone. rDNS-only bots have no range
// here and are handled on the request path, not by this address check.
func IPInAnyVerifiedBotRange(ip net.IP) bool {
if ip == nil {
return false
}
if DefaultRanges().IPInAnyBot(ip) {
return true
}
p := operatorNets.Load()
if p == nil {
return false
}
for _, nets := range *p {
for _, n := range nets {
if n.Contains(ip) {
return true
}
}
}
return false
}
// OperatorBotIPVerified reports whether ip falls in any IP range configured
// for the named operator bot. Synchronous; no DNS. This is how AI agents that
// publish address ranges (PerplexityBot, GPTBot, ClaudeBot) are confirmed.
func OperatorBotIPVerified(name string, ip net.IP) bool {
if name == "" || ip == nil {
return false
}
p := operatorNets.Load()
if p == nil {
return false
}
for _, n := range (*p)[name] {
if n.Contains(ip) {
return true
}
}
return false
}
func normalizeBotEntries(entries []BotEntry, requireUA bool) []BotEntry {
norm := make([]BotEntry, 0, len(entries))
for _, e := range entries {
ne := BotEntry{Name: normalizeBotName(e.Name)}
seenUA := map[string]struct{}{}
for _, raw := range e.UASubstrings {
s := normalizeUASubstring(raw)
if s == "" {
continue
}
if _, ok := seenUA[s]; ok {
continue
}
seenUA[s] = struct{}{}
ne.UASubstrings = append(ne.UASubstrings, s)
}
seenSuffix := map[string]struct{}{}
for _, raw := range e.RDNSSuffixes {
d := normalizeSuffix(raw)
if d == "" {
continue
}
if _, ok := seenSuffix[d]; ok {
continue
}
seenSuffix[d] = struct{}{}
ne.RDNSSuffixes = append(ne.RDNSSuffixes, d)
}
seenRange := map[string]struct{}{}
for _, raw := range e.IPRanges {
n := netutil.ParseCIDROrIP(raw)
if n == nil {
continue
}
r := n.String()
if _, ok := seenRange[r]; ok {
continue
}
seenRange[r] = struct{}{}
ne.IPRanges = append(ne.IPRanges, r)
}
if ne.Name == "" || (requireUA && len(ne.UASubstrings) == 0) {
continue
}
norm = append(norm, ne)
}
return norm
}
func normalizeBotName(name string) string {
return strings.ToLower(strings.TrimSpace(name))
}
func normalizeUASubstring(s string) string {
return strings.ToLower(strings.TrimSpace(s))
}
func normalizeSuffix(d string) string {
return strings.TrimPrefix(strings.ToLower(strings.TrimSpace(d)), ".")
}
func operatorBotEntries() []BotEntry {
p := operatorBots.Load()
if p == nil {
return nil
}
return *p
}
// OperatorBotFromUA returns the operator bot identity whose UA substring
// matches ua, or "" if none. Built-in identities are checked first by
// ClaimedBotFromUA; this is the fallback.
func OperatorBotFromUA(lowUA string) string {
lowUA = strings.ToLower(lowUA)
for _, e := range operatorBotEntries() {
for _, s := range e.UASubstrings {
if strings.Contains(lowUA, s) {
return e.Name
}
}
}
return ""
}
func operatorDomains(name string) []string {
for _, e := range operatorBotEntries() {
if e.Name == name {
return e.RDNSSuffixes
}
}
return nil
}
// OperatorBotsCacheVersion folds the operator bot list into the base cache
// logic version. Changing verified_bots therefore changes the stamp the
// daemon hands EnsureBotVerifyLogicVersion, which drops the PTR-verdict cache
// so a previously-spoofed IP is re-checked under the new suffixes instead of
// staying pinned for the cache TTL. The hash is order-independent so the same
// set in a different file order yields the same stamp.
func OperatorBotsCacheVersion(base int, entries []BotEntry) int {
entries = normalizeBotEntries(entries, false)
lines := make([]string, 0, len(entries))
for _, e := range entries {
subs := append([]string(nil), e.UASubstrings...)
sufs := append([]string(nil), e.RDNSSuffixes...)
rngs := append([]string(nil), e.IPRanges...)
sort.Strings(subs)
sort.Strings(sufs)
sort.Strings(rngs)
lines = append(lines, e.Name+
"\x00"+strings.Join(subs, ",")+
"\x00"+strings.Join(sufs, ",")+
"\x00"+strings.Join(rngs, ","))
}
sort.Strings(lines)
h := fnv.New64a()
_, _ = h.Write([]byte(strconv.Itoa(base)))
for _, l := range lines {
_, _ = h.Write([]byte{0x01})
_, _ = h.Write([]byte(l))
}
return int(h.Sum64() & 0x7fffffff)
}
package threatintel
import (
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/pidginhost/csm/internal/spamscore"
)
const rspamdMaxHistoryBytes = 2 << 20
// RspamdSource queries rspamd's rolling history and returns a score
// 0..100 derived only from rows matching the requested IP.
//
// Token resolution reads the process environment at Score time. External
// environment changes require a daemon restart.
type RspamdSource struct {
url string
token string // static token from config (may be empty)
tokenEnv string // env var name to consult at query time
client *http.Client
}
func NewRspamdSource(url, token, tokenEnv string) *RspamdSource {
return &RspamdSource{
url: url,
token: token,
tokenEnv: tokenEnv,
client: &http.Client{Timeout: 5 * time.Second},
}
}
func (s *RspamdSource) Name() string { return "rspamd" }
// resolveToken reads the env var (if set) at query time, falling back to
// the static token. External environment changes require a daemon restart.
func (s *RspamdSource) resolveToken() string {
if s.tokenEnv != "" {
if v := os.Getenv(s.tokenEnv); v != "" {
return v
}
}
return s.token
}
type rspamdHistoryResp struct {
Rows []rspamdHistoryRow `json:"rows"`
History []rspamdHistoryRow `json:"history"`
Data []rspamdHistoryRow `json:"data"`
}
func (r rspamdHistoryResp) entries() []rspamdHistoryRow {
out := make([]rspamdHistoryRow, 0, len(r.Rows)+len(r.History)+len(r.Data))
out = append(out, r.Rows...)
out = append(out, r.History...)
out = append(out, r.Data...)
return out
}
type rspamdHistoryRow struct {
IP string
Action string
Score float64
UnixTime float64
}
func (r *rspamdHistoryRow) UnmarshalJSON(data []byte) error {
var raw map[string]json.RawMessage
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
r.IP = firstJSONString(raw, "ip", "sender_ip", "client_ip")
r.Action = firstJSONString(raw, "action", "metric_action")
r.Score = firstJSONFloat(raw, "score")
r.UnixTime = firstJSONFloat(raw, "unix_time")
return nil
}
func firstJSONString(raw map[string]json.RawMessage, names ...string) string {
for _, name := range names {
v, ok := raw[name]
if !ok {
continue
}
var s string
if err := json.Unmarshal(v, &s); err == nil {
return s
}
}
return ""
}
func firstJSONFloat(raw map[string]json.RawMessage, names ...string) float64 {
for _, name := range names {
v, ok := raw[name]
if !ok {
continue
}
var f float64
if err := json.Unmarshal(v, &f); err == nil {
return f
}
}
return 0
}
// Score sends a GET to <url>/history and scores only history rows for ip.
func (s *RspamdSource) Score(ctx context.Context, ip string) (int, error) {
endpoint, err := s.historyEndpoint()
if err != nil {
return 0, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return 0, err
}
if tok := s.resolveToken(); tok != "" {
req.Header.Set("Password", tok)
}
resp, err := s.client.Do(req)
if err != nil {
return 0, fmt.Errorf("rspamd: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return 0, fmt.Errorf("rspamd HTTP %d", resp.StatusCode)
}
rows, err := decodeRspamdHistory(resp.Body)
if err != nil {
return 0, fmt.Errorf("rspamd decode: %w", err)
}
return scoreRspamdHistory(rows, ip, time.Now()), nil
}
func (s *RspamdSource) historyEndpoint() (string, error) {
u, err := url.Parse(s.url)
if err != nil {
return "", err
}
if u.Scheme == "" || u.Host == "" {
return "", fmt.Errorf("rspamd URL must include scheme and host")
}
u.Path = strings.TrimRight(u.Path, "/") + "/history"
u.RawQuery = ""
return u.String(), nil
}
func decodeRspamdHistory(r io.Reader) ([]rspamdHistoryRow, error) {
var raw json.RawMessage
if err := json.NewDecoder(io.LimitReader(r, rspamdMaxHistoryBytes)).Decode(&raw); err != nil {
return nil, err
}
trimmed := strings.TrimSpace(string(raw))
if strings.HasPrefix(trimmed, "[") {
var rows []rspamdHistoryRow
if err := json.Unmarshal(raw, &rows); err != nil {
return nil, err
}
return rows, nil
}
var history rspamdHistoryResp
if err := json.Unmarshal(raw, &history); err != nil {
return nil, err
}
return history.entries(), nil
}
// scoreRspamdHistory converts an IP's history rows into a 0..100
// confidence that the IP is a spam source. The score is the recency-
// weighted proportion of definitive spam verdicts among classifiable
// delivered/spam verdicts, smoothed by spamscore.PriorMass. Delivered ham
// therefore dilutes the score toward 0 instead of accumulating it: a
// correspondent MTA that sends mostly legitimate mail stays near 0 no
// matter how much it sends, while an IP whose recent traffic is
// predominantly rejected climbs toward 100.
func scoreRspamdHistory(rows []rspamdHistoryRow, ip string, now time.Time) int {
want := normalizeIP(ip)
var spamMass, totalMass float64
for _, row := range rows {
if normalizeIP(row.IP) != want {
continue
}
spamWeight, counts := rspamdActionSignal(row.Action)
if !counts {
continue
}
recency := rspamdRecencyFactor(row.UnixTime, now)
spamMass += spamWeight * recency
totalMass += recency
}
return spamscore.Score(spamMass, totalMass)
}
// rspamdActionSignal maps an rspamd action onto per-message spam mass
// in [0,1] and reports whether the row is classifiable as ham or spam.
// "no action" is delivered ham; greylist and soft reject are tempfail
// flow control that fires on first contact from every unknown sender
// and on rate limits, so they are neutral instead of ham. Genuinely
// spammy retries earn reject, quarantine, discard, add-header, or
// rewrite-subject rows, which do count. The numeric rspamd score is
// ignored on purpose: the action already is rspamd's calibrated
// thresholding of that score.
func rspamdActionSignal(action string) (float64, bool) {
switch strings.ToLower(strings.TrimSpace(action)) {
case "reject", "discard", "quarantine", "spam":
return 1.0, true
case "add header", "rewrite subject", "probable spam":
return 0.7, true
case "no action", "clean":
return 0, true
default:
return 0, false
}
}
// rspamdRecencyFactor halves a row's weight per spamscore.HalfLife, so
// the score tracks recent behaviour; rspamd's rolling history can span
// months on quiet servers. Rows without a parseable unix_time count at full weight rather than
// being dropped: rspamd's rolling history is bounded, so undated rows
// are treated as current.
func rspamdRecencyFactor(unixTime float64, now time.Time) float64 {
if unixTime <= 0 {
return 1
}
return spamscore.Recency(now.Sub(time.Unix(int64(unixTime), 0)))
}
func normalizeIP(ip string) string {
ip = strings.TrimSpace(ip)
parsed := net.ParseIP(ip)
if parsed == nil {
return ip
}
return parsed.String()
}
// Package threatintel defines a pluggable interface for IP reputation
// providers and an Aggregator that combines their scores. CSM uses this
// to consult AbuseIPDB plus optional rspamd / upstream sources without
// hardcoding multiple lookup paths in the reputation check.
package threatintel
import "context"
// Source is a single scoring provider. Score returns 0..100 (higher is
// worse). A source that has no opinion on the IP must return 0, nil -
// that score is excluded from the aggregator's average. Errors are
// per-source and non-fatal at the aggregator level (other sources still
// run).
type Source interface {
Name() string
Score(ctx context.Context, ip string) (int, error)
}
// Aggregator runs every registered source and averages their non-zero scores.
type Aggregator struct {
sources []Source
}
// NewAggregator constructs an empty Aggregator.
func NewAggregator() *Aggregator { return &Aggregator{} }
// Register adds a Source to the aggregator. Order matters only for the
// `Sources` field of Result (which preserves registration order); the
// aggregated score is order-independent.
func (a *Aggregator) Register(s Source) { a.sources = append(a.sources, s) }
// Result holds the aggregated value and per-source breakdown.
type Result struct {
AggregatedScore int `json:"aggregated_score"`
Sources map[string]int `json:"sources"`
}
// Score queries every registered source. Per-source errors are swallowed
// (the source contributes "no signal"). The aggregated score is the mean
// of non-zero scores; if every source returned 0 (or errored), the
// aggregated score is 0.
func (a *Aggregator) Score(ctx context.Context, ip string) (Result, error) {
out := Result{Sources: map[string]int{}}
sum, n := 0, 0
for _, s := range a.sources {
score, err := s.Score(ctx, ip)
if err != nil {
continue
}
out.Sources[s.Name()] = score
if score > 0 {
sum += score
n++
}
}
if n > 0 {
out.AggregatedScore = sum / n
}
return out, nil
}
package threatintel
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"sync"
"sync/atomic"
"time"
)
const (
upstreamMaxResponseBytes = 1 << 20
// defaultMaxCacheEntries caps the per-IP score cache so a sustained
// flood of unique attacker IPs cannot grow the map without bound.
// Once the cap is reached, expired entries are pruned first; if the
// cap is still exceeded the oldest entry (by expires) is evicted.
defaultMaxCacheEntries = 10000
// defaultBreakerTrip is the number of consecutive upstream failures
// after which the source short-circuits subsequent Score calls.
defaultBreakerTrip = 5
// defaultBreakerCooldown is how long the breaker stays open before
// allowing a probe call through.
defaultBreakerCooldown = 60 * time.Second
)
// UpstreamConfig configures the HTTP threat-intel client. TokenEnv reads
// the process environment before each HTTP request. External environment
// changes require a daemon restart.
type UpstreamConfig struct {
URL string
Token string
TokenEnv string
CacheTTL time.Duration
Timeout time.Duration
}
// UpstreamSource queries a panel-side TI cache. The wire contract is
// documented in docs/src/upstream-threat-intel-contract.md.
//
// GET <URL>/lookup?ip=<ip>
// Authorization: Bearer <token> (omitted if no token resolved)
//
// 200 OK
// {"ip":"1.2.3.4","score":75,"source":"upstream","ttl_sec":900}
//
// Errors of any flavour (network, 4xx, 5xx, malformed JSON) propagate
// up - the aggregator treats them as "no signal" rather than fatal.
type UpstreamSource struct {
cfg UpstreamConfig
client *http.Client
mu sync.RWMutex
cache map[string]upstreamEntry
// maxCacheEntries caps the in-memory score cache. Exposed for tests
// that want to validate eviction without staging 10k entries.
maxCacheEntries int
// breakerMu guards the circuit-breaker state below.
breakerMu sync.Mutex
consecutiveErrors int
breakerOpenedAt time.Time
breakerProbe bool
breakerTrip int
breakerCooldown time.Duration
// Per-source counters back MetricsSnapshot. Package-level metrics use
// process-wide counters because reputation checks rebuild this source.
cacheHitsTotal atomic.Int64
cacheMissesTotal atomic.Int64
backendFailuresTotal atomic.Int64
}
type upstreamEntry struct {
score int
expires time.Time
}
// upstreamResponse mirrors the documented panel response shape.
type upstreamResponse struct {
IP string `json:"ip"`
Score int `json:"score"`
Source string `json:"source,omitempty"`
TTLSec int `json:"ttl_sec,omitempty"`
}
func NewUpstreamSource(cfg UpstreamConfig) *UpstreamSource {
if cfg.Timeout == 0 {
cfg.Timeout = 5 * time.Second
}
if cfg.CacheTTL == 0 {
cfg.CacheTTL = 15 * time.Minute
}
return &UpstreamSource{
cfg: cfg,
client: &http.Client{Timeout: cfg.Timeout},
cache: make(map[string]upstreamEntry),
maxCacheEntries: defaultMaxCacheEntries,
breakerTrip: defaultBreakerTrip,
breakerCooldown: defaultBreakerCooldown,
}
}
func (u *UpstreamSource) Name() string { return "upstream" }
// resolveToken reads TokenEnv (if set) at query time, falling back to the
// static token. External environment changes require a daemon restart.
func (u *UpstreamSource) resolveToken() string {
if u.cfg.TokenEnv != "" {
if v := os.Getenv(u.cfg.TokenEnv); v != "" {
return v
}
}
return u.cfg.Token
}
func (u *UpstreamSource) Score(ctx context.Context, ip string) (int, error) {
if v, ok := u.cacheGet(ip); ok {
u.cacheHitsTotal.Add(1)
upstreamMetrics.cacheHitsTotal.Add(1)
return v, nil
}
u.cacheMissesTotal.Add(1)
upstreamMetrics.cacheMissesTotal.Add(1)
if open, until := u.breakerOpen(); open {
if until.IsZero() {
return 0, fmt.Errorf("upstream breaker probe already running")
}
return 0, fmt.Errorf("upstream breaker open for %s", time.Until(until).Round(time.Second))
}
endpoint, err := u.lookupEndpoint(ip)
if err != nil {
return 0, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return 0, err
}
if tok := u.resolveToken(); tok != "" {
req.Header.Set("Authorization", "Bearer "+tok)
}
req.Header.Set("Accept", "application/json")
resp, err := u.client.Do(req)
if err != nil {
u.breakerObserve(false)
return 0, fmt.Errorf("upstream request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
u.breakerObserve(false)
fmt.Fprintf(os.Stderr, "upstream threat-intel: HTTP %d for %s\n", resp.StatusCode, ip)
return 0, fmt.Errorf("upstream HTTP %d", resp.StatusCode)
}
var body upstreamResponse
if err := json.NewDecoder(io.LimitReader(resp.Body, upstreamMaxResponseBytes)).Decode(&body); err != nil {
u.breakerObserve(false)
return 0, fmt.Errorf("upstream decode: %w", err)
}
if normalizeIP(body.IP) != normalizeIP(ip) {
u.breakerObserve(false)
return 0, fmt.Errorf("upstream response ip mismatch: got %q want %q", body.IP, ip)
}
if body.Score < 0 || body.Score > 100 {
u.breakerObserve(false)
return 0, fmt.Errorf("upstream score out of range: %d", body.Score)
}
u.breakerObserve(true)
// The response may shorten the cache lifetime, never extend it past the
// operator's cache_ttl: an unbounded ttl_sec pinned a score for as long
// as the daemon ran.
ttl := u.cfg.CacheTTL
if body.TTLSec > 0 {
if responseTTL := time.Duration(body.TTLSec) * time.Second; responseTTL < ttl {
ttl = responseTTL
}
}
u.cachePut(ip, body.Score, ttl)
return body.Score, nil
}
// breakerOpen reports whether the circuit breaker is currently open.
// When open and the cooldown has elapsed, the breaker transitions to a
// half-open state by clearing the timestamp so one probe call may pass.
func (u *UpstreamSource) breakerOpen() (bool, time.Time) {
u.breakerMu.Lock()
defer u.breakerMu.Unlock()
if u.breakerOpenedAt.IsZero() {
return false, time.Time{}
}
until := u.breakerOpenedAt.Add(u.breakerCooldown)
if time.Now().Before(until) {
return true, until
}
if u.breakerProbe {
return true, time.Time{}
}
u.breakerProbe = true
return false, time.Time{}
}
func (u *UpstreamSource) resetBreakerLocked() {
u.consecutiveErrors = 0
u.breakerProbe = false
u.breakerOpenedAt = time.Time{}
}
func (u *UpstreamSource) recordBreakerFailureLocked(now time.Time) {
u.breakerProbe = false
u.consecutiveErrors++
if u.breakerTrip > 0 && u.consecutiveErrors >= u.breakerTrip {
u.breakerOpenedAt = now
}
}
func (u *UpstreamSource) closeExpiredBreakerLocked(now time.Time) {
if !u.breakerOpenedAt.IsZero() && now.Sub(u.breakerOpenedAt) >= u.breakerCooldown && !u.breakerProbe {
u.breakerOpenedAt = time.Time{}
}
}
// breakerObserve records the outcome of an upstream call so the
// breaker can trip after enough consecutive failures or reset on a
// successful response.
func (u *UpstreamSource) breakerObserve(success bool) {
u.breakerMu.Lock()
defer u.breakerMu.Unlock()
if success {
u.resetBreakerLocked()
return
}
u.backendFailuresTotal.Add(1)
upstreamMetrics.backendFailuresTotal.Add(1)
now := time.Now()
u.closeExpiredBreakerLocked(now)
u.recordBreakerFailureLocked(now)
}
// MetricsSnapshot returns this source's cache-hit, cache-miss, and
// backend-failure counters. Safe to call from any goroutine.
func (u *UpstreamSource) MetricsSnapshot() (cacheHits, cacheMisses, backendFailures int64) {
return u.cacheHitsTotal.Load(), u.cacheMissesTotal.Load(), u.backendFailuresTotal.Load()
}
// BreakerOpen reports whether the circuit breaker is currently in the
// open state (refusing calls). Read-only.
func (u *UpstreamSource) BreakerOpen() bool {
u.breakerMu.Lock()
defer u.breakerMu.Unlock()
if u.breakerOpenedAt.IsZero() {
return false
}
return time.Now().Before(u.breakerOpenedAt.Add(u.breakerCooldown))
}
func (u *UpstreamSource) lookupEndpoint(ip string) (string, error) {
endpoint, err := url.Parse(u.cfg.URL)
if err != nil {
return "", fmt.Errorf("parsing upstream URL: %w", err)
}
if endpoint.Scheme != "http" && endpoint.Scheme != "https" {
return "", fmt.Errorf("upstream URL must use http or https")
}
if endpoint.Host == "" {
return "", fmt.Errorf("upstream URL must include host")
}
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/lookup"
endpoint.Fragment = ""
q := endpoint.Query()
q.Set("ip", normalizeIP(ip))
endpoint.RawQuery = q.Encode()
return endpoint.String(), nil
}
func (u *UpstreamSource) cacheGet(ip string) (int, bool) {
u.mu.RLock()
defer u.mu.RUnlock()
e, ok := u.cache[ip]
if !ok || time.Now().After(e.expires) {
return 0, false
}
return e.score, true
}
func (u *UpstreamSource) cachePut(ip string, score int, ttl time.Duration) {
u.mu.Lock()
defer u.mu.Unlock()
u.cache[ip] = upstreamEntry{score: score, expires: time.Now().Add(ttl)}
u.evictLocked()
}
// evictLocked drops expired entries first, then evicts the oldest
// (smallest expires) until size is within maxCacheEntries. Caller
// must hold u.mu.
func (u *UpstreamSource) evictLocked() {
if u.maxCacheEntries <= 0 || len(u.cache) <= u.maxCacheEntries {
return
}
now := time.Now()
for k, e := range u.cache {
if now.After(e.expires) {
delete(u.cache, k)
}
}
for len(u.cache) > u.maxCacheEntries {
var oldestKey string
var oldestAt time.Time
first := true
for k, e := range u.cache {
if first || e.expires.Before(oldestAt) {
oldestKey = k
oldestAt = e.expires
first = false
}
}
delete(u.cache, oldestKey)
}
}
// cacheLen is exposed for tests to inspect cache size without exposing
// the underlying map.
func (u *UpstreamSource) cacheLen() int {
u.mu.RLock()
defer u.mu.RUnlock()
return len(u.cache)
}
package updatecheck
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
)
const defaultGitHubReleasesURL = "https://api.github.com/repos/pidginhost/csm/releases/latest"
// fetchGitHubLatest returns the newest tagged release version, with
// any leading "v" stripped. Pre-releases are skipped via the standard
// /releases/latest endpoint, which already excludes them.
func fetchGitHubLatest(ctx context.Context, hc *http.Client, url string) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return "", err
}
req.Header.Set("Accept", "application/vnd.github+json")
req.Header.Set("User-Agent", "csm-update-check")
resp, err := hc.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
return "", fmt.Errorf("github releases: status %d", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return "", err
}
var payload struct {
TagName string `json:"tag_name"`
}
if err := json.Unmarshal(body, &payload); err != nil {
return "", err
}
tag := strings.TrimSpace(payload.TagName)
if tag == "" {
return "", fmt.Errorf("github releases: empty tag_name")
}
return strings.TrimPrefix(tag, "v"), nil
}
package updatecheck
import (
"context"
"errors"
"fmt"
"os/exec"
"reflect"
"runtime"
"sort"
"strings"
"time"
)
// AptProbe queries `apt-cache policy <pkg>` and returns the candidate
// version. Returns an error when apt-cache is missing, the package is
// unknown, or the candidate is "(none)".
func AptProbe(packageName string) PackageProbe {
return func(ctx context.Context) (string, error) {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "apt-cache", "policy", packageName) // #nosec G204 -- packageName is operator-controlled config, not attacker input
out, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("apt-cache policy: %w", err)
}
return parseAptPolicy(string(out))
}
}
// DnfProbe queries `dnf --quiet repoquery --queryformat=%{version}
// <pkg>` and returns the highest version line. Returns an error when
// dnf is missing or returns no rows.
func DnfProbe(packageName string) PackageProbe {
return func(ctx context.Context) (string, error) {
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "dnf", "--quiet", "repoquery", "--queryformat=%{version}\n", packageName) // #nosec G204 -- packageName is operator-controlled config, not attacker input
out, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("dnf repoquery: %w", err)
}
return parseDnfRepoquery(string(out))
}
}
func parseAptPolicy(out string) (string, error) {
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, "Candidate:") {
continue
}
v := strings.TrimSpace(strings.TrimPrefix(line, "Candidate:"))
if v == "" || v == "(none)" {
return "", errors.New("apt-cache policy: candidate (none)")
}
return aptStripEpochRevision(v), nil
}
return "", errors.New("apt-cache policy: no candidate line")
}
func parseDnfRepoquery(out string) (string, error) {
versions := []string{}
for _, line := range strings.Split(out, "\n") {
v := strings.TrimSpace(line)
if v == "" {
continue
}
versions = append(versions, v)
}
if len(versions) == 0 {
return "", errors.New("dnf repoquery: no versions")
}
sort.Slice(versions, func(i, j int) bool { return isNewer(versions[i], versions[j]) })
return versions[0], nil
}
// aptStripEpochRevision drops the optional "EPOCH:" prefix and
// "-DEBIAN_REVISION" suffix added by the apt versioning scheme so the
// returned string can be compared to a plain semver tag.
func aptStripEpochRevision(v string) string {
if i := strings.Index(v, ":"); i >= 0 {
v = v[i+1:]
}
if i := strings.Index(v, "-"); i >= 0 {
v = v[:i]
}
return v
}
// pkgSourceLabel best-effort labels a probe as "apt" or "dnf" by name.
// Unknown probes get "package".
func pkgSourceLabel(p PackageProbe) string {
if p == nil {
return "package"
}
name := runtime.FuncForPC(reflect.ValueOf(p).Pointer()).Name()
switch {
case strings.Contains(name, "AptProbe"):
return "apt"
case strings.Contains(name, "DnfProbe"):
return "dnf"
default:
return "package"
}
}
package updatecheck
import (
"strconv"
"strings"
)
// isNewer returns true when a is strictly greater than b under
// dot-separated numeric ordering. "dev" or empty current always
// loses (a real release is always newer than "dev"). Current-version
// strings produced by git describe, such as 3.0.0-12-gabcdef0, compare
// as newer than their base tag but older than the next tagged release.
func isNewer(a, b string) bool {
a = strings.TrimPrefix(strings.TrimSpace(a), "v")
b = strings.TrimPrefix(strings.TrimSpace(b), "v")
if a == "" {
return false
}
if b == "" || b == "dev" {
return true
}
if base, ok := gitDescribeBase(b); ok {
return isNewer(a, base)
}
ap := strings.Split(a, ".")
bp := strings.Split(b, ".")
for i := 0; i < len(ap) || i < len(bp); i++ {
var av, bv string
if i < len(ap) {
av = ap[i]
}
if i < len(bp) {
bv = bp[i]
}
ai, aErr := strconv.Atoi(av)
bi, bErr := strconv.Atoi(bv)
switch {
case aErr == nil && bErr == nil:
if ai != bi {
return ai > bi
}
case aErr == nil && bErr != nil:
return true
case aErr != nil && bErr == nil:
return false
default:
if av != bv {
return av > bv
}
}
}
return false
}
func gitDescribeBase(v string) (string, bool) {
parts := strings.Split(v, "-")
if len(parts) < 3 {
return "", false
}
if _, err := strconv.Atoi(parts[1]); err != nil {
return "", false
}
if !strings.HasPrefix(parts[2], "g") || !isNumericDotted(parts[0]) {
return "", false
}
return parts[0], true
}
func isNumericDotted(v string) bool {
parts := strings.Split(v, ".")
if len(parts) < 2 {
return false
}
for _, p := range parts {
if p == "" {
return false
}
if _, err := strconv.Atoi(p); err != nil {
return false
}
}
return true
}
// Package updatecheck polls upstream release channels and tells the
// daemon whether a newer CSM version is available so the Web UI can
// surface a banner. It never fetches binaries or modifies the running
// install. Operators upgrade through their normal channel
// (apt, dnf, install.sh, deploy pipeline).
//
// Two sources, tried in order:
//
// 1. GitHub Releases API ("https://api.github.com/repos/pidginhost/csm/releases/latest").
// 2. apt-cache policy or dnf repoquery against the OS package, used
// when the GitHub call fails (network blocked, rate-limited, etc.).
//
// The package never panics on a transient network or exec failure --
// it records the error in Info.Err and keeps the previous successful
// result in place so the banner does not flicker on a single bad poll.
package updatecheck
import (
"context"
"net/http"
"sync/atomic"
"time"
)
// Info is the cached result surfaced to /api/v1/status. Zero value
// means "no check has completed yet"; CheckedAt.IsZero() is the
// canonical signal.
type Info struct {
LatestVersion string `json:"latest_version,omitempty"`
Available bool `json:"available"`
Source string `json:"source,omitempty"` // "github" | "apt" | "dnf"
CheckedAt time.Time `json:"checked_at,omitempty"`
Err string `json:"err,omitempty"`
}
// PackageProbe queries the OS package manager for the highest
// available version of the configured package. Implementations
// must respect ctx for cancellation and timeout.
type PackageProbe func(ctx context.Context) (string, error)
// Options configures a Checker. Zero-valued fields take safe defaults.
type Options struct {
// CurrentVersion is the running daemon's version string ("dev" is
// treated as "always older than any tagged release").
CurrentVersion string
// Interval is how often the checker polls upstream. Clamped to a
// minimum of 1h to avoid hammering the GitHub API.
Interval time.Duration
// GitHubAPIURL overrides the default GitHub releases URL. Used by
// tests; production should leave this empty.
GitHubAPIURL string
// HTTPClient is the HTTP client used for the GitHub request. nil
// gets a sane default with a 15s timeout.
HTTPClient *http.Client
// PackageProbe is the apt/dnf fallback. nil disables fallback.
PackageProbe PackageProbe
// Now lets tests inject a clock. Defaults to time.Now.
Now func() time.Time
// LogErr receives non-fatal probe errors. Optional; nil silences.
LogErr func(source string, err error)
}
// Checker holds polling state. Safe for concurrent reads of Latest()
// while the goroutine started by Run is updating the cache.
type Checker struct {
opts Options
cache atomic.Pointer[Info]
}
// New builds a Checker. Validate the options here so Run can rely on
// invariants without re-checking each tick.
func New(opts Options) *Checker {
if opts.Interval <= 0 {
opts.Interval = 24 * time.Hour
}
if opts.Interval < time.Hour {
opts.Interval = time.Hour
}
if opts.GitHubAPIURL == "" {
opts.GitHubAPIURL = defaultGitHubReleasesURL
}
if opts.HTTPClient == nil {
opts.HTTPClient = &http.Client{Timeout: 15 * time.Second}
}
if opts.Now == nil {
opts.Now = time.Now
}
c := &Checker{opts: opts}
c.cache.Store(&Info{})
return c
}
// Latest returns the most recent successful poll plus any error from
// the most recent attempt. The returned value is safe to mutate.
func (c *Checker) Latest() Info {
if v := c.cache.Load(); v != nil {
return *v
}
return Info{}
}
// CheckOnce runs a single poll synchronously and returns the result.
// Run uses this internally; tests call it directly to avoid the ticker.
func (c *Checker) CheckOnce(ctx context.Context) Info {
now := c.opts.Now()
latest, err := fetchGitHubLatest(ctx, c.opts.HTTPClient, c.opts.GitHubAPIURL)
source := "github"
if err != nil {
if c.opts.LogErr != nil {
c.opts.LogErr("github", err)
}
if c.opts.PackageProbe != nil {
pkgVer, pkgErr := c.opts.PackageProbe(ctx)
if pkgErr == nil {
latest = pkgVer
source = pkgSourceLabel(c.opts.PackageProbe)
err = nil
} else {
if c.opts.LogErr != nil {
c.opts.LogErr("package", pkgErr)
}
err = pkgErr
}
}
}
info := Info{CheckedAt: now}
if err != nil {
// Preserve the last good LatestVersion so the banner does
// not flicker on a single bad poll.
prev := c.Latest()
info.LatestVersion = prev.LatestVersion
info.Available = prev.Available
info.Source = prev.Source
info.Err = err.Error()
} else {
info.LatestVersion = latest
info.Source = source
info.Available = isNewer(latest, c.opts.CurrentVersion)
}
c.cache.Store(&info)
return info
}
// Run polls on the configured interval until ctx is cancelled. It
// performs an initial check after a 5-minute warm-up so daemon
// startup is not blocked on outbound HTTP.
func (c *Checker) Run(ctx context.Context) {
select {
case <-ctx.Done():
return
case <-time.After(5 * time.Minute):
}
c.CheckOnce(ctx)
t := time.NewTicker(c.opts.Interval)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
c.CheckOnce(ctx)
}
}
}
// Package verdict implements an HMAC-signed HTTP client for the
// auto_response.verdict_callback hook. CSM POSTs a Request to the panel
// URL before each automatic block; the panel's Response is advisory
// (block / allow / "" -> block default; tenant_id is logged). Errors are
// fail-open: the caller (firewall.Engine.BlockIP) proceeds with the
// default block on any callback failure.
package verdict
import (
"bytes"
"context"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"strings"
"time"
)
const (
verdictMaxResponseBytes = 64 << 10 // 64 KB - verdict response is small JSON
// verdictMaxResponseSkew bounds how far the panel's reply timestamp
// may drift from CSM's clock. Anything older or further into the
// future is treated as a replayed or forged response.
verdictMaxResponseSkew = 5 * time.Minute
)
// Config configures the verdict callback client. HMACSecretEnv reads the
// process environment once per exchange. External environment changes
// require a daemon restart.
//
// RequireResponseSignature controls whether the panel must sign its
// response body with the same HMAC scheme used on the request
// (X-CSM-Signature header) and echo the request nonce + timestamp.
// Default is true: when a secret is configured, CSM rejects unsigned
// or forged responses to prevent an on-path attacker from silently
// downgrading a block to "allow". Set the pointer to a false value
// only during phpanel-side rollouts that have not yet implemented
// response signing. Even on the opt-out path, replay protection is
// best-effort enforced: if the panel does echo nonce or timestamp,
// they must match; a panel that echoes neither still works (legacy
// shape). When no HMAC secret is configured at all, signature and
// replay checks are skipped because there is no key to verify against.
//
// AllowUnsigned is the runtime form of
// auto_response.verdict_callback.allow_unsigned. By default, an unsigned
// "allow" response is rejected because it would disable the block. Set true
// only for configs that explicitly opted in to an unsigned rollout.
type Config struct {
URL string
HMACSecret string
HMACSecretEnv string
RequireResponseSignature *bool
AllowUnsigned bool
Timeout time.Duration
}
// ErrorClass is the fixed class of a failed exchange. A caller reports the
// class, never the error text, which can carry a URL or panel text.
type ErrorClass string
const (
ClassNonce ErrorClass = "nonce_generation"
ClassTimeout ErrorClass = "timeout"
ClassTransport ErrorClass = "transport"
ClassHTTPStatus ErrorClass = "http_status"
ClassBodySize ErrorClass = "body_size"
ClassDecode ErrorClass = "decode"
ClassSignature ErrorClass = "signature"
ClassReplay ErrorClass = "replay"
ClassConfiguration ErrorClass = "configuration"
ClassProtocol ErrorClass = "protocol"
)
type classedError struct {
class ErrorClass
err error
}
func (e *classedError) Error() string { return e.err.Error() }
func (e *classedError) Unwrap() error { return e.err }
// ClassOf is the class of an error Ask returned, or empty for any other.
func ClassOf(err error) ErrorClass {
var c *classedError
if errors.As(err, &c) {
return c.class
}
return ""
}
func fail(class ErrorClass, err error) (Response, error) {
return Response{}, &classedError{class: class, err: err}
}
// exchangeClass classes a failure to send the request or read its reply.
func exchangeClass(err error) ErrorClass {
var ne net.Error
if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &ne) && ne.Timeout()) {
return ClassTimeout
}
return ClassTransport
}
// requireResponseSig returns the effective response-signature requirement,
// defaulting to true (secure by default) when the operator did not set
// an explicit value.
func (c Config) requireResponseSig() bool {
if c.RequireResponseSignature == nil {
return true
}
return *c.RequireResponseSignature
}
// Request is what CSM asks the panel about. Ask sets Nonce and Timestamp
// for every exchange. They bind each request to its own reply so an
// attacker cannot replay an old "allow" verdict.
type Request struct {
IP string `json:"ip"`
Reason string `json:"reason"`
Severity string `json:"severity,omitempty"`
Source string `json:"source,omitempty"` // "auto_response" | "manual"
Nonce string `json:"nonce,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
}
// Response is what the panel may answer.
//
// verdict = "block" - CSM proceeds with its default action.
// verdict = "allow" - CSM logs the verdict but does NOT block.
// verdict = "" or missing - equivalent to "block" (default).
// tenant_id = optional attribution string CSM logs alongside the decision.
// nonce = MUST equal the Request.Nonce when present. Required when
// response signing is in effect; defeats replay of captured replies.
// timestamp = unix seconds the panel produced the reply. MUST be within
// verdictMaxResponseSkew of CSM's clock when present.
type Response struct {
Verdict string `json:"verdict,omitempty"`
TenantID string `json:"tenant_id,omitempty"`
Note string `json:"note,omitempty"`
Nonce string `json:"nonce,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
}
// Client posts each block decision to the configured URL and reads the
// (advisory) response. Timeouts and 5xx are returned as errors - the
// caller decides whether to fail open (allow CSM to proceed) or closed.
type Client struct {
cfg Config
client *http.Client
}
func New(cfg Config) *Client {
if cfg.Timeout == 0 {
cfg.Timeout = 2 * time.Second
}
return &Client{cfg: cfg, client: &http.Client{Timeout: cfg.Timeout}}
}
// resolveSecret reads HMACSecretEnv at call time, falling back to the
// static secret. External environment changes require a daemon restart.
func (c *Client) resolveSecret() string {
if c.cfg.HMACSecretEnv != "" {
if v := os.Getenv(c.cfg.HMACSecretEnv); v != "" {
return v
}
}
return c.cfg.HMACSecret
}
// verifyResponseSignature checks that header carries a well-formed
// X-CSM-Signature header (sha256=<hex>) over body, computed with secret.
// Returns a descriptive error on missing, malformed, or mismatched values
// using a constant-time comparison.
func verifyResponseSignature(secret string, body []byte, header string) error {
if header == "" {
return fmt.Errorf("verdict callback response missing signature header (X-CSM-Signature)")
}
const prefix = "sha256="
if !strings.HasPrefix(header, prefix) {
return fmt.Errorf("verdict callback response signature has unsupported algorithm")
}
gotHex := strings.TrimPrefix(header, prefix)
got, err := hex.DecodeString(gotHex)
if err != nil {
return fmt.Errorf("verdict callback response signature is not valid hex")
}
mac := hmac.New(sha256.New, []byte(secret))
mac.Write(body)
want := mac.Sum(nil)
if !hmac.Equal(got, want) {
return fmt.Errorf("verdict callback response signature mismatch")
}
return nil
}
// Ask POSTs the request, returns the panel's response or an error.
func (c *Client) Ask(ctx context.Context, req Request) (Response, error) {
// Defense-in-depth URL check (config validation already ran at load,
// this re-check defends against misconfiguration via cfg corruption).
rawURL := strings.TrimSpace(c.cfg.URL)
if rawURL == "" {
return fail(ClassConfiguration, fmt.Errorf("verdict callback URL not configured"))
}
parsed, err := url.Parse(rawURL)
if err != nil {
return fail(ClassConfiguration, fmt.Errorf("verdict callback URL parse: %w", err))
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return fail(ClassConfiguration, fmt.Errorf("verdict callback URL must use http or https"))
}
if parsed.Host == "" {
return fail(ClassConfiguration, fmt.Errorf("verdict callback URL must include host"))
}
nonce, err := newNonce()
if err != nil {
return fail(ClassNonce, fmt.Errorf("verdict callback nonce generation: %w", err))
}
req.Nonce = nonce
req.Timestamp = time.Now().Unix()
body, err := json.Marshal(req)
if err != nil {
return fail(ClassProtocol, err)
}
secret := c.resolveSecret()
r, err := http.NewRequestWithContext(ctx, http.MethodPost, rawURL, bytes.NewReader(body))
if err != nil {
return fail(ClassConfiguration, err)
}
r.Header.Set("Content-Type", "application/json")
r.Header.Set("Accept", "application/json")
if secret != "" {
mac := hmac.New(sha256.New, []byte(secret))
mac.Write(body)
r.Header.Set("X-CSM-Signature", "sha256="+hex.EncodeToString(mac.Sum(nil)))
}
resp, err := c.client.Do(r)
if err != nil {
return fail(exchangeClass(err), fmt.Errorf("verdict callback: %w", err))
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
// Return the IP in the error so the caller logs it through the
// structured audit path instead of leaking it to raw stderr.
return fail(ClassHTTPStatus, fmt.Errorf("verdict callback HTTP %d for %s", resp.StatusCode, req.IP))
}
data, err := io.ReadAll(io.LimitReader(resp.Body, verdictMaxResponseBytes+1))
if err != nil {
return fail(exchangeClass(err), fmt.Errorf("verdict callback read: %w", err))
}
if int64(len(data)) > verdictMaxResponseBytes {
return fail(ClassBodySize, fmt.Errorf("verdict callback response exceeds %d bytes", verdictMaxResponseBytes))
}
if secret != "" && c.cfg.requireResponseSig() {
// Verify the panel signed its response with the same secret used
// on the request. Without this check a network attacker could
// downgrade block to allow on every call. Rejecting the reply keeps
// the engine on its default block path.
if err := verifyResponseSignature(secret, data, resp.Header.Get("X-CSM-Signature")); err != nil {
return fail(ClassSignature, err)
}
}
strictReplay := secret != "" && c.cfg.requireResponseSig()
if strings.TrimSpace(string(data)) == "" {
return Response{}, nil
}
var out Response
dec := json.NewDecoder(bytes.NewReader(data))
if err := dec.Decode(&out); err != nil {
return fail(ClassDecode, fmt.Errorf("verdict callback decode: %w", err))
}
var trailing struct{}
if err := dec.Decode(&trailing); err != io.EOF {
return fail(ClassDecode, fmt.Errorf("verdict callback decode: trailing JSON"))
}
// Validate response shape. Unknown verdict strings are rejected
// defensively rather than silently treated as "block".
if out.Verdict != "" && out.Verdict != "block" && out.Verdict != "allow" {
return fail(ClassProtocol, fmt.Errorf("verdict callback returned unknown verdict %q", out.Verdict))
}
// Fail closed on an unsigned allow. Without a secret there is no replay or
// signature protection (the checks below are gated on secret != ""), so an
// on-path attacker could return "allow" on every call and silently disable
// auto-blocking. Returning an error keeps the engine on its default block
// path unless the config explicitly opted in to unsigned callback verdicts.
if secret == "" && out.Verdict == "allow" && !c.cfg.AllowUnsigned {
return fail(ClassSignature, fmt.Errorf("verdict callback: refusing unsigned allow (no HMAC secret configured)"))
}
// Replay protection runs whenever a secret is configured. Strict
// mode (response signing required) demands the panel echo nonce and
// timestamp. Best-effort mode (signing opt-out) still enforces what
// the panel did echo, so a captured stale reply with a wrong nonce
// or a long-expired timestamp is rejected even when the operator
// has not yet enabled response signing on the panel side.
if secret != "" {
if out.Nonce != "" {
if subtle.ConstantTimeCompare([]byte(out.Nonce), []byte(req.Nonce)) != 1 {
return fail(ClassReplay, fmt.Errorf("verdict callback response nonce mismatch"))
}
} else if strictReplay {
return fail(ClassReplay, fmt.Errorf("verdict callback response missing nonce"))
}
if out.Timestamp != 0 {
drift := time.Since(time.Unix(out.Timestamp, 0))
if drift < 0 {
drift = -drift
}
if drift > verdictMaxResponseSkew {
return fail(ClassReplay, fmt.Errorf("verdict callback response timestamp drift %s exceeds %s", drift, verdictMaxResponseSkew))
}
} else if strictReplay {
return fail(ClassReplay, fmt.Errorf("verdict callback response missing timestamp"))
}
// An "allow" is the only verdict an on-path attacker gains from
// forging. In best-effort mode (signing not required) the replay
// checks above only fire when the panel echoed a nonce or timestamp,
// so an attacker can strip both to slip an unbound allow through.
// Require an allow to carry at least one replay binding; otherwise
// treat it like the no-secret case and refuse, keeping the engine on
// its default block path.
if out.Verdict == "allow" && out.Nonce == "" && out.Timestamp == 0 {
return fail(ClassReplay, fmt.Errorf("verdict callback: refusing allow with no replay binding (nonce/timestamp)"))
}
}
return out, nil
}
// newNonce returns a fresh 128-bit hex nonce. crypto/rand is the only
// acceptable source: math/rand would let an attacker who observes one
// nonce predict the next and craft a replay reply in advance.
func newNonce() (string, error) {
var buf [16]byte
if _, err := io.ReadFull(rand.Reader, buf[:]); err != nil {
return "", err
}
return hex.EncodeToString(buf[:]), nil
}
package webui
import (
"net/http"
"path/filepath"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
)
func (s *Server) handleAccount(w http.ResponseWriter, r *http.Request) {
name := r.URL.Query().Get("name")
if err := validateAccountName(name); err != nil {
http.Redirect(w, r, "/findings", http.StatusFound)
return
}
s.renderTemplate(w, r, "account.html", map[string]string{
"Hostname": s.cfg.Hostname,
"AccountName": name,
"HomeDir": checks.AccountHomeDirIn(s.accountRoots(), name),
})
}
// accountPathPrefixes returns "<root>/<name>/" for every account root, the
// prefixes a path inside the account's home starts with.
func (s *Server) accountPathPrefixes(name string) []string {
roots := s.accountRoots()
out := make([]string, 0, len(roots))
for _, root := range roots {
out = append(out, filepath.Join(root, name)+"/")
// Findings and quarantine metadata can carry the resolved path even
// after the file has been moved away. Resolve the root, not the file.
if resolved, err := filepath.EvalSymlinks(root); err == nil && resolved != filepath.Clean(root) {
out = append(out, filepath.Join(resolved, name)+"/")
}
}
return out
}
func pathHasAnyPrefix(path string, prefixes []string) bool {
for _, prefix := range prefixes {
if strings.HasPrefix(path, prefix) {
return true
}
}
return false
}
func containsAny(s string, subs []string) bool {
for _, sub := range subs {
if strings.Contains(s, sub) {
return true
}
}
return false
}
func accountFindingMatches(f alert.Finding, name string, prefixes []string) bool {
for _, owner := range []string{f.TenantID, f.CPUser} {
if owner = strings.TrimSpace(owner); owner != "" {
return owner == name
}
}
return containsAny(f.Message, prefixes) || containsAny(f.Details, prefixes) || pathHasAnyPrefix(f.FilePath, prefixes)
}
func (s *Server) apiAccountDetail(w http.ResponseWriter, r *http.Request) {
name := r.URL.Query().Get("name")
if err := validateAccountName(name); err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
homePrefixes := s.accountPathPrefixes(name)
// Current findings for this account
type findingView struct {
Severity string `json:"severity"`
Check string `json:"check"`
Message string `json:"message"`
HasFix bool `json:"has_fix"`
}
var accountFindings []findingView
latest := s.store.LatestFindings()
for _, f := range latest {
if !operatorFacingCheck(f.Check) {
continue
}
if accountFindingMatches(f, name, homePrefixes) {
accountFindings = append(accountFindings, findingView{
Severity: f.Severity.String(),
Check: f.Check,
Message: f.Message,
HasFix: checks.HasFix(f.Check),
})
}
}
// Quarantined files for this account
type qEntry struct {
ID string `json:"id"`
OriginalPath string `json:"original_path"`
Size int64 `json:"size"`
Reason string `json:"reason"`
}
var quarantined []qEntry
rootMetas := listMetaFiles(quarantineDir)
preCleanMetas := listMetaFiles(filepath.Join(quarantineDir, "pre_clean"))
metas := rootMetas
metas = append(metas, preCleanMetas...)
for _, metaPath := range metas {
meta, err := readQuarantineMeta(metaPath)
if err != nil {
continue
}
if pathHasAnyPrefix(meta.OriginalPath, homePrefixes) {
id := strings.TrimSuffix(filepath.Base(metaPath), ".meta")
quarantined = append(quarantined, qEntry{
ID: id, OriginalPath: meta.OriginalPath, Size: meta.Size, Reason: meta.Reason,
})
}
}
// Recent history for this account (last 100 matching entries)
allHistory, _ := s.store.ReadHistory(2000, 0)
type histEntry struct {
Severity string `json:"severity"`
Check string `json:"check"`
Message string `json:"message"`
Timestamp time.Time `json:"timestamp"`
}
var history []histEntry
for _, f := range allHistory {
if len(history) >= 100 {
break
}
if accountFindingMatches(f, name, homePrefixes) {
history = append(history, histEntry{
Severity: f.Severity.String(), Check: f.Check, Message: f.Message,
Timestamp: f.Timestamp.UTC(),
})
}
}
writeJSON(w, map[string]interface{}{
"account": name,
"findings": accountFindings,
"quarantined": quarantined,
"history": history,
"whm_url": "https://" + s.cfg.Hostname + ":2087/scripts/domainsdata?user=" + name,
})
}
package webui
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"log"
"net"
"net/http"
"net/netip"
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
"unicode"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/health"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
var reIPReputation = regexp.MustCompile(`Known malicious IP accessing server: (\S+) \((.+)\)`)
// apiStatus returns the daemon's full health snapshot as JSON. Backward
// compatible with prior callers: every field they consumed (hostname,
// uptime, started_at, rules_loaded, scan_running, last_scan_time) is
// still present, with new fields added alongside.
func (s *Server) apiStatus(w http.ResponseWriter, _ *http.Request) {
provider := s.provider
s.scanMu.Lock()
scanning := s.scanRunning
s.scanMu.Unlock()
if s.scanInProgress != nil && s.scanInProgress() {
scanning = true
}
if provider == nil {
// No daemon-side provider installed (test harness). Fall back to
// the legacy minimal payload so existing UI code keeps working.
resp := map[string]interface{}{
"hostname": s.cfg.Hostname,
"uptime_seconds": int64(time.Since(s.startTime).Seconds()),
"started_at": s.startTime.UTC(),
"started_at_token": daemonStartToken(s.startTime),
"rules_loaded": s.signatureCount(),
"scan_running": scanning,
"status": "down",
}
if s.store != nil {
if last := s.store.LatestScanTime(); !last.IsZero() {
resp["last_scan_time"] = last.UTC()
}
}
writeJSON(w, resp)
return
}
snap := health.Build(provider, s.version, health.Capabilities())
resp := map[string]interface{}{
"hostname": snap.Hostname,
"version": snap.Version,
"uptime_seconds": snap.UptimeSec,
"started_at": snap.StartedAt.UTC(),
"started_at_token": daemonStartToken(snap.StartedAt),
"rules_loaded": s.signatureCount(),
"scan_running": scanning,
"blocklist_size": snap.BlocklistSize,
"incidents_open": snap.IncidentsOpen,
"bpf_enforcement_active": snap.BPFEnforcementActive,
"history_count": snap.HistoryCount,
"severities": snap.Severities,
"watchers": snap.Watchers,
"store_healthy": snap.StoreHealthy,
"store_size_mb": snap.StoreSizeMB,
"config_hash": snap.ConfigHash,
"binary_hash": snap.BinaryHash,
"capabilities": snap.Capabilities,
"backends": snap.Backends,
"dry_run_blocks": snap.DryRunBlocks,
"automation": snap.Automation,
"mode": snap.Mode,
"status": snap.OverallStatus(),
}
// latest_scan mirrors the health.Snapshot JSON tag and is the canonical
// name; last_scan_time is the legacy key kept for older clients (the
// cPHulk dashboard). A time that is not set is left out.
if !snap.LatestScan.IsZero() {
resp["latest_scan"] = snap.LatestScan.UTC()
resp["last_scan_time"] = snap.LatestScan.UTC()
}
if !snap.BaselineAt.IsZero() {
resp["baseline_at"] = snap.BaselineAt.UTC()
}
// security_posture is the threat-aware badge signal, distinct from
// "status" (which stays operational: is the daemon alive and attached).
// A daemon can be operationally "ok" while sitting on open critical
// incidents, so the dashboard pill must fold in incident severity rather
// than report "Healthy" next to thousands of criticals.
openBySev := map[string]int{}
if s.incidentCorrelator != nil {
openBySev = s.incidentCorrelator.OpenCountsBySeverity()
}
resp["incidents_open_by_severity"] = openBySev
resp["security_posture"] = securityPosture(operationalProblems(s.signatureCount(), snap), openBySev["critical"], openBySev["high"])
if !snap.Update.CheckedAt.IsZero() {
resp["update"] = snap.Update
}
// Present only after the daemon has merged an active set; absence means
// "not observed yet", not "clean".
if snap.CorrelationAttribution != nil {
resp["correlation_attribution"] = snap.CorrelationAttribution
}
if len(snap.Queues) != 0 {
resp["queues"] = snap.Queues
}
if len(snap.WordPressVerification) != 0 {
resp["wordpress_verification"] = snap.WordPressVerification
}
if snap.Admission != nil {
resp["admission"] = snap.Admission
}
if snap.AntispamLearning != nil {
resp["antispam_learning"] = snap.AntispamLearning
}
writeJSON(w, resp)
}
// operationalProblems counts daemon faults that should prevent a healthy
// posture even when there are no active high-severity incidents.
func operationalProblems(sigCount int, snap health.Snapshot) int {
problems := 0
if sigCount == 0 {
problems++
}
if !snap.StoreHealthy {
problems++
}
if !snap.AllWatchersAttached() {
problems++
}
for _, q := range snap.Queues {
if q.Status == "degraded" && !q.Advisory {
problems++
break
}
}
return problems
}
// securityPosture collapses operational faults and active-incident severity
// into the dashboard badge tier:
//
// - "critical" if any active critical incident exists, or the daemon has
// multiple operational problems.
// - "warning" if any active high incident exists, or exactly one operational
// problem.
// - "healthy" otherwise.
//
// Incident severity takes precedence over operational faults so a fully
// attached daemon still reads "critical" while critical incidents are active.
func securityPosture(opProblems, openCritical, openHigh int) string {
if openCritical > 0 || opProblems >= 2 {
return "critical"
}
if openHigh > 0 || opProblems == 1 {
return "warning"
}
return "healthy"
}
// apiCapabilities returns the static feature-flag list for this build.
func (s *Server) apiCapabilities(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, map[string]interface{}{
"capabilities": health.Capabilities(),
"version": s.version,
})
}
// apiFindings returns current scan results - "what's wrong right now."
func (s *Server) apiFindings(w http.ResponseWriter, _ *http.Request) {
latest := s.store.LatestFindings()
type entryView struct {
Severity string `json:"severity"`
Check string `json:"check"`
Message string `json:"message"`
Details string `json:"details,omitempty"`
Context string `json:"context,omitempty"`
Time time.Time `json:"time"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
HasFix bool `json:"has_fix"`
}
suppressions := s.store.LoadSuppressions()
var result []entryView
for _, f := range latest {
if !operatorFacingCheck(f.Check) {
continue
}
// Skip suppressed findings
if s.store.IsSuppressed(f, suppressions) {
continue
}
firstSeen := f.Timestamp
lastSeen := f.Timestamp
if entry, ok := s.store.EntryForKey(f.Key()); ok {
firstSeen = entry.FirstSeen
lastSeen = entry.LastSeen
}
result = append(result, entryView{
Severity: f.Severity.String(),
Check: f.Check,
Message: f.Message,
Details: f.Details,
Context: f.Context,
Time: f.Timestamp.UTC(),
FirstSeen: firstSeen.UTC(),
LastSeen: lastSeen.UTC(),
HasFix: checks.HasFix(f.Check),
})
}
writeAll(w, result)
}
// enrichedFinding is the JSON response type for the enriched findings endpoint.
type enrichedFinding struct {
Key string `json:"key"`
Severity string `json:"severity"`
Check string `json:"check"`
Message string `json:"message"`
Details string `json:"details,omitempty"`
Context string `json:"context,omitempty"`
FilePath string `json:"file_path,omitempty"`
Account string `json:"account,omitempty"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
HasFix bool `json:"has_fix"`
HasVerify bool `json:"has_verify"`
FixDesc string `json:"fix_desc,omitempty"`
ContentSHA256 string `json:"content_sha256,omitempty"`
// BlockIP is the attacker address an operator may block from this
// finding; empty for checks that do not report one.
BlockIP string `json:"block_ip,omitempty"`
}
// dedupIPReputation groups ip_reputation findings by IP, merging sources and
// promoting to the highest severity. Non-ip_reputation findings pass through unchanged.
func dedupIPReputation(items []enrichedFinding) []enrichedFinding {
type ipGroup struct {
entry enrichedFinding
sources []string
}
ipGroups := make(map[string]*ipGroup)
var ipOrder []string
var result []enrichedFinding
for _, item := range items {
if item.Check != "ip_reputation" {
result = append(result, item)
continue
}
m := reIPReputation.FindStringSubmatch(item.Message)
if m == nil {
result = append(result, item)
continue
}
ip, source := m[1], m[2]
if g, ok := ipGroups[ip]; ok {
g.sources = append(g.sources, source)
if item.FirstSeen.Before(g.entry.FirstSeen) {
g.entry.FirstSeen = item.FirstSeen
}
if item.LastSeen.After(g.entry.LastSeen) {
g.entry.LastSeen = item.LastSeen
}
if severityRank(item.Severity) > severityRank(g.entry.Severity) {
g.entry.Severity = item.Severity
}
} else {
ipGroups[ip] = &ipGroup{
entry: item,
sources: []string{source},
}
ipOrder = append(ipOrder, ip)
}
}
for _, ip := range ipOrder {
g := ipGroups[ip]
sort.Strings(g.sources)
g.entry.Message = fmt.Sprintf("Known malicious IP accessing server: %s (%s)", ip, strings.Join(g.sources, ", "))
result = append(result, g.entry)
}
return result
}
// apiFindingsEnriched returns findings with IP dedup, account extraction, and severity counts.
func (s *Server) apiFindingsEnriched(w http.ResponseWriter, r *http.Request) {
latest := s.store.LatestFindings()
suppressions := s.store.LoadSuppressions()
items := make([]enrichedFinding, 0)
for _, f := range latest {
if !operatorFacingCheck(f.Check) {
continue
}
if s.store.IsSuppressed(f, suppressions) {
continue
}
firstSeen := f.Timestamp
lastSeen := f.Timestamp
if entry, ok := s.store.EntryForKey(f.Key()); ok {
firstSeen = entry.FirstSeen
lastSeen = entry.LastSeen
}
items = append(items, enrichedFinding{
Key: f.Key(),
Severity: severityLabel(f.Severity),
Check: f.Check,
Message: f.Message,
Details: f.Details,
Context: f.Context,
FilePath: f.FilePath,
Account: extractAccountFromFinding(f),
FirstSeen: firstSeen.UTC(),
LastSeen: lastSeen.UTC(),
HasFix: checks.HasFix(f.Check),
HasVerify: checks.CanVerify(f.Check),
FixDesc: checks.FixDescription(f.Check, f.Message, f.FilePath),
ContentSHA256: f.ContentSHA256,
BlockIP: checks.ManualBlockIP(f),
})
}
items = dedupIPReputation(items)
version := enrichedFindingsVersion(items)
if r.URL.Query().Get("fields") == "version" {
writeJSON(w, map[string]interface{}{"version": version, "total": len(items)})
return
}
var critCount, highCount, warnCount int
for _, item := range items {
switch item.Severity {
case "CRITICAL":
critCount++
case "HIGH":
highCount++
default:
warnCount++
}
}
checkTypeSet := make(map[string]bool)
accountSet := make(map[string]bool)
for _, item := range items {
checkTypeSet[item.Check] = true
if item.Account != "" {
accountSet[item.Account] = true
}
}
checkTypes := make([]string, 0, len(checkTypeSet))
for ct := range checkTypeSet {
checkTypes = append(checkTypes, ct)
}
sort.Strings(checkTypes)
accounts := make([]string, 0, len(accountSet))
for a := range accountSet {
accounts = append(accounts, a)
}
sort.Strings(accounts)
extra := map[string]interface{}{
"check_types": checkTypes,
"accounts": accounts,
"critical_count": critCount,
"high_count": highCount,
"warning_count": warnCount,
"version": version,
}
if limit := queryInt(r, "limit", 0); limit > 0 {
sortEnrichedBySeverity(items)
writeCapped(w, items, len(items), limit, extra)
return
}
extra["total"] = len(items)
writeItems(w, items, extra)
}
// sortEnrichedBySeverity orders findings most severe first, newest first
// within a severity, so a limited list keeps the ones that matter.
func sortEnrichedBySeverity(items []enrichedFinding) {
rank := map[string]int{"CRITICAL": 3, "HIGH": 2}
sort.SliceStable(items, func(i, j int) bool {
if ri, rj := rank[items[i].Severity], rank[items[j].Severity]; ri != rj {
return ri > rj
}
return items[i].LastSeen.After(items[j].LastSeen)
})
}
// enrichedFindingsVersion changes when the listed findings, their severities
// or their file content do. A client that polls only to learn whether the
// list changed asks for ?fields=version and compares. ip_reputation rows are identified by
// their message, which carries the merged sources.
func enrichedFindingsVersion(items []enrichedFinding) string {
ids := make([]string, 0, len(items))
for _, f := range items {
id := f.Key
if f.Check == "ip_reputation" {
id = f.Check + ":" + f.Message
}
ids = append(ids, id+"|"+f.Severity+"|"+f.ContentSHA256)
}
sort.Strings(ids)
h := sha256.New()
for _, id := range ids {
h.Write([]byte(id))
h.Write([]byte{0})
}
return hex.EncodeToString(h.Sum(nil))[:16]
}
// apiHistory returns paginated finding history.
// Supports optional filtering via "from", "to" (YYYY-MM-DD or RFC 3339), and "severity" (a label or 0/1/2) query params.
func (s *Server) apiHistory(w http.ResponseWriter, r *http.Request) {
limit := queryInt(r, "limit", 50)
if limit > 5000 {
limit = 5000
}
offset := queryInt(r, "offset", 0)
q, ok := parseHistoryQuery(w, r)
if !ok {
return
}
findings, total := s.readHistoryPage(q, limit, offset)
writeItems(w, withAccountIP(findings), map[string]interface{}{
"total": total,
"limit": limit,
"offset": offset,
"truncated": historyPageTruncated(total, offset, len(findings)),
})
}
// historyQuery is the filter set /api/v1/history and its CSV export share.
type historyQuery struct {
from, to, search string
severity int // -1 for any
checks map[string]bool
}
func (q historyQuery) filtered() bool {
return q.from != "" || q.to != "" || q.severity >= 0 || q.search != "" || q.checks != nil
}
// parseHistoryQuery reads the history filters. scope=email selects the email
// check list unless checks names the checks. An unreadable date is a 400,
// written here, and ok is false.
func parseHistoryQuery(w http.ResponseWriter, r *http.Request) (historyQuery, bool) {
v := r.URL.Query()
if _, _, ok := historyRangeQuery(w, v, time.Time{}, time.Time{}); !ok {
return historyQuery{}, false
}
q := historyQuery{from: v.Get("from"), to: v.Get("to"), search: v.Get("search"), severity: -1}
if sev, ok := parseSeverity(v.Get("severity")); ok {
q.severity = int(sev)
}
if checksStr := v.Get("checks"); checksStr != "" {
q.checks = make(map[string]bool)
for _, c := range strings.Split(checksStr, ",") {
if c = strings.TrimSpace(c); c != "" {
q.checks[c] = true
}
}
}
if q.checks == nil && v.Get("scope") == "email" {
q.checks = emailCheckSet()
}
return q, true
}
func (s *Server) readHistoryPage(q historyQuery, limit, offset int) ([]alert.Finding, int) {
if !q.filtered() {
return s.store.ReadHistory(limit, offset)
}
return s.store.ReadHistoryFilteredWithChecks(limit, offset, q.from, q.to, q.severity, q.search, q.checks)
}
// historyPageTruncated reports whether matches exist past the returned page.
// total counts every match, so a client can page on through offset.
func historyPageTruncated(total, offset, returned int) bool {
return total > offset+returned
}
// apiFinding is a stored finding as the API sends it: its severity, and the
// one it was demoted from, are labels. The embedded Finding promotes its
// other JSON fields; these shallower fields take the severity keys.
type apiFinding struct {
alert.Finding
Severity string `json:"severity"`
DemotedFrom string `json:"demoted_from,omitempty"`
}
func toAPIFinding(f alert.Finding) apiFinding {
a := apiFinding{Finding: f, Severity: f.Severity.String()}
// Demotion only lowers a severity, so WARNING, the zero level, is never
// the one a finding was demoted from.
if f.DemotedFrom > alert.Warning {
a.DemotedFrom = f.DemotedFrom.String()
}
return a
}
func toAPIFindings(findings []alert.Finding) []apiFinding {
out := make([]apiFinding, len(findings))
for i, f := range findings {
out[i] = toAPIFinding(f)
}
return out
}
// historyFinding is an apiFinding with the normalized account and remote IP,
// so clients render structured fields instead of regex-scraping the
// human-readable message. It embeds the exported Finding, not apiFinding:
// apiValue cannot reach into an unexported embedded struct.
type historyFinding struct {
alert.Finding
Severity string `json:"severity"`
DemotedFrom string `json:"demoted_from,omitempty"`
Account string `json:"account,omitempty"`
IP string `json:"ip,omitempty"`
}
func withAccountIP(findings []alert.Finding) []historyFinding {
out := make([]historyFinding, len(findings))
for i, f := range findings {
a := toAPIFinding(f)
out[i] = historyFinding{Finding: f, Severity: a.Severity, DemotedFrom: a.DemotedFrom,
Account: findingAccount(f), IP: findingIP(f)}
}
return out
}
// parseSeverity reads a severity filter: a label in any case, or the 0/1/2
// level older callers send. ok is false for anything else.
func parseSeverity(text string) (alert.Severity, bool) {
switch strings.ToUpper(strings.TrimSpace(text)) {
case "WARNING", "0":
return alert.Warning, true
case "HIGH", "1":
return alert.High, true
case "CRITICAL", "2":
return alert.Critical, true
}
return 0, false
}
// findingAccount returns the account/mailbox attribution for a finding,
// preferring structured fields over legacy text extraction so the value
// survives message wording changes.
func findingAccount(f alert.Finding) string {
for _, account := range []string{f.Mailbox, f.TenantID, f.CPUser} {
if account = strings.TrimSpace(account); account != "" {
return account
}
}
if acct := legacyEmailAccount(f); acct != "" {
return acct
}
if acct := extractAccountFromFinding(f); acct != "" {
return acct
}
return strings.TrimSpace(f.Domain)
}
func findingIP(f alert.Finding) string {
if ip := strings.TrimSpace(alert.AttackerAddress(f)); ip != "" {
return ip
}
if !isEmailHistoryCheck(f.Check) {
return ""
}
for _, s := range []string{f.Message, f.Details} {
if ip := firstIPToken(s); ip != "" {
return ip
}
}
return ""
}
func legacyEmailAccount(f alert.Finding) string {
if !isEmailHistoryCheck(f.Check) {
return ""
}
for _, source := range []string{f.Message, f.Details} {
for _, prefix := range []string{"for ", "Account ", "account ", "Sender "} {
if token := tokenAfter(source, prefix); strings.Contains(token, "@") {
return token
}
}
if token := tokenAfter(source, "set_id="); token != "" {
return token
}
for _, prefix := range []string{"Domain ", "Domain: ", "domain ", "domain: "} {
if token := tokenAfter(source, prefix); token != "" {
return token
}
}
}
return ""
}
func isEmailHistoryCheck(check string) bool {
return emailKindForCheck(check) != "" ||
strings.HasPrefix(check, "email_") ||
strings.HasPrefix(check, "mail_") ||
strings.HasPrefix(check, "smtp_")
}
func tokenAfter(s, prefix string) string {
idx := strings.Index(s, prefix)
if idx < 0 {
return ""
}
rest := s[idx+len(prefix):]
if end := strings.IndexAny(rest, " \n\t,"); end >= 0 {
rest = rest[:end]
}
return cleanHistoryToken(rest)
}
func cleanHistoryToken(s string) string {
s = strings.TrimSpace(s)
s = strings.Trim(s, `"'<>[]()`)
s = strings.TrimRight(s, ".,;:")
return s
}
func firstIPToken(s string) string {
for _, raw := range strings.FieldsFunc(s, func(r rune) bool {
if unicode.IsSpace(r) {
return true
}
switch r {
case ',', ';', '(', ')':
return true
default:
return false
}
}) {
if ip := normalizeHistoryIPToken(raw); ip != "" {
return ip
}
}
return ""
}
func normalizeHistoryIPToken(raw string) string {
raw = strings.Trim(raw, `"'<>`)
if ip := parseHistoryIP(raw); ip != "" {
return ip
}
if eq := strings.LastIndexByte(raw, '='); eq >= 0 && eq < len(raw)-1 {
if ip := normalizeHistoryIPToken(raw[eq+1:]); ip != "" {
return ip
}
}
candidate := strings.TrimRight(raw, ".,;:")
if host, _, err := net.SplitHostPort(candidate); err == nil {
if ip := parseHistoryIP(host); ip != "" {
return ip
}
}
if strings.HasPrefix(candidate, "[") {
if end := strings.IndexByte(candidate, ']'); end > 1 {
if ip := parseHistoryIP(candidate[1:end]); ip != "" {
return ip
}
}
}
return parseHistoryIP(candidate)
}
func parseHistoryIP(raw string) string {
raw = strings.TrimSpace(strings.Trim(raw, "[]"))
if raw == "" {
return ""
}
if addr, err := netip.ParseAddr(raw); err == nil {
return addr.String()
}
if prefix, err := netip.ParsePrefix(raw); err == nil {
return prefix.String()
}
return ""
}
// var (not const) so tests can redirect to t.TempDir(). Production
// callers must not mutate at runtime.
var quarantineDir = "/opt/csm/quarantine"
// apiQuarantine lists quarantined files with metadata.
func (s *Server) apiQuarantine(w http.ResponseWriter, _ *http.Request) {
type quarantineEntry struct {
ID string `json:"id"`
Kind string `json:"kind"`
OriginalPath string `json:"original_path"`
Size int64 `json:"size"`
QuarantineAt time.Time `json:"quarantined_at,omitzero"`
Reason string `json:"reason"`
LiveState string `json:"live_state"`
OriginalModTime time.Time `json:"original_mtime,omitzero"`
}
var entries []quarantineEntry
// Scan both root quarantine dir and pre_clean subdirectory
rootMetas := listMetaFiles(quarantineDir)
preCleanMetas := listMetaFiles(filepath.Join(quarantineDir, "pre_clean"))
metaFiles := rootMetas
metaFiles = append(metaFiles, preCleanMetas...)
for _, metaFile := range metaFiles {
meta, err := readQuarantineMeta(metaFile)
if err != nil {
continue
}
// Hide entries whose original has been restored byte-identical to
// the archive: the archive is redundant and the UI should reflect
// the live filesystem, not the quarantine history. Divergence
// (missing, different size, different content) keeps the entry
// visible -- the operator still has to reconcile it.
archivePath := strings.TrimSuffix(metaFile, ".meta")
liveState := quarantineLiveState(archivePath, meta.OriginalPath)
if liveState == "restored_identical" {
continue
}
kind := "quarantine"
if strings.HasPrefix(quarantineEntryID(metaFile), preCleanQuarantineIDPrefix) {
kind = "pre_clean"
}
entries = append(entries, quarantineEntry{
ID: quarantineEntryID(metaFile),
Kind: kind,
OriginalPath: meta.OriginalPath,
Size: meta.Size,
QuarantineAt: meta.QuarantineAt.UTC(),
Reason: meta.Reason,
LiveState: liveState,
OriginalModTime: meta.OriginalModTime.UTC(),
})
}
// Sort newest first
sort.Slice(entries, func(i, j int) bool {
if entries[i].QuarantineAt.Equal(entries[j].QuarantineAt) {
return entries[i].ID < entries[j].ID
}
return entries[i].QuarantineAt.After(entries[j].QuarantineAt)
})
writeAll(w, entries)
}
// apiStats returns severity counts and per-check breakdown for the last 24
// hours. The summary is shared with the dashboard page and recomputed only
// when history changes.
func (s *Server) apiStats(w http.ResponseWriter, _ *http.Request) {
sum := s.statsSummary24h()
result := map[string]interface{}{
"last_24h": map[string]interface{}{
"critical": sum.critical,
"high": sum.high,
"warning": sum.warning,
"total": sum.critical + sum.high + sum.warning,
},
"by_check": sum.byCheck,
"accounts_at_risk": s.accountsAtRisk(),
"auto_response": map[string]int{
"blocked": sum.autoBlocked,
"quarantined": sum.autoQuarantined,
"killed": sum.autoKilled,
},
"top_accounts": sum.topAccounts,
"brute_force": sum.bruteForce,
}
if !sum.lastCritical.IsZero() {
result["last_critical"] = sum.lastCritical.UTC()
}
writeJSON(w, result)
}
func buildBruteForceSummary(ips map[string]int, types map[string]int) map[string]interface{} {
// Top attacker IPs
type ipCount struct {
IP string `json:"ip"`
Count int `json:"count"`
}
var topIPs []ipCount
for ip, count := range ips {
topIPs = append(topIPs, ipCount{ip, count})
}
sort.Slice(topIPs, func(i, j int) bool {
return topIPs[i].Count > topIPs[j].Count
})
if len(topIPs) > 10 {
topIPs = topIPs[:10]
}
total := 0
for _, v := range types {
total += v
}
return map[string]interface{}{
"total_attacks": total,
"unique_ips": len(ips),
"wp_login_count": types["wp-login"],
"xmlrpc_count": types["xmlrpc"] + types["xmlrpc-modsec"],
"top_ips": topIPs,
}
}
// apiStatsTrend returns daily finding counts by severity for the trend
// chart. Accepts optional ?days=N (default 30, clamped to the store's
// retention window).
func (s *Server) apiStatsTrend(w http.ResponseWriter, r *http.Request) {
days := 30
if v := r.URL.Query().Get("days"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
days = n
}
}
writeAll(w, s.store.AggregateByDayN(days))
}
// apiStatsTimeline returns 24 hourly buckets for the findings timeline chart.
// Uses efficient bbolt cursor seeking instead of loading all findings into memory.
func (s *Server) apiStatsTimeline(w http.ResponseWriter, _ *http.Request) {
buckets, _ := s.timelineMemo.get(s.store.HistoryMark(), func() any {
return s.store.AggregateByHour()
}).([]store.HourBucket)
writeAll(w, buckets)
}
// apiHealth returns daemon health status.
func (s *Server) apiHealth(w http.ResponseWriter, _ *http.Request) {
health := map[string]interface{}{
"daemon_mode": true,
"uptime_seconds": int(time.Since(s.startTime).Seconds()),
"rules_loaded": s.signatureCount(),
"fanotify": s.fanotifyRunning(),
"log_watchers": s.logWatchersRunning(),
}
writeJSON(w, health)
}
// historyCSVMax bounds one CSV export; the history filters narrow it to reach
// older entries.
const historyCSVMax = 5000
// apiHistoryCSV exports the newest history entries matching the History
// filters as a CSV download.
func (s *Server) apiHistoryCSV(w http.ResponseWriter, r *http.Request) {
q, ok := parseHistoryQuery(w, r)
if !ok {
return
}
findings, _ := s.readHistoryPage(q, historyCSVMax, 0)
w.Header().Set("Content-Type", "text/csv")
w.Header().Set("Content-Disposition", "attachment; filename=csm-history.csv")
// CSV header
fmt.Fprintf(w, "Timestamp,Severity,Check,Message,Details\n")
for _, f := range findings {
sev := "WARNING"
switch f.Severity {
case alert.Critical:
sev = "CRITICAL"
case alert.High:
sev = "HIGH"
}
// Escape CSV fields
msg := csvEscape(f.Message)
details := csvEscape(f.Details)
fmt.Fprintf(w, "%s,%s,%s,%s,%s\n",
f.Timestamp.Format(time.RFC3339), sev, f.Check, msg, details)
}
}
// csvEscape quotes a field for CSV and neutralises spreadsheet formula
// triggers. Finding text is attacker-chosen (a filename, a User-Agent, a
// mailbox): a field starting with "=", "+", "-", "@", a tab or a carriage
// return became a live formula when the export was opened in a spreadsheet.
// Such fields are prefixed with a single quote, the convention spreadsheets
// use to force text, and quoted so the prefix survives.
func csvEscape(s string) string {
if s != "" && strings.ContainsAny(s[:1], "=+-@\t\r") {
s = "'" + s
}
if strings.ContainsAny(s, ",\"\n\r'") {
return "\"" + strings.ReplaceAll(s, "\"", "\"\"") + "\""
}
return s
}
// safeLogString renders a caller-controlled string into a log entry without
// allowing embedded CR/LF/control bytes to forge a separate log line.
func safeLogString(s string) string { return strconv.Quote(s) }
func daemonStartToken(start time.Time) string {
return start.UTC().Format(time.RFC3339Nano)
}
func (s *Server) daemonStartToken() string {
if s.provider != nil {
return daemonStartToken(s.provider.StartedAt())
}
return daemonStartToken(s.startTime)
}
// apiFix applies a known remediation action for a finding.
// POST /api/v1/fix body: {"check": "check_type", "message": "...", "details": "..."}
func (s *Server) apiFix(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
Check string `json:"check"`
Message string `json:"message"`
Details string `json:"details"`
FilePath string `json:"file_path"`
Key string `json:"key"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.Check == "" || req.Message == "" {
writeJSONError(w, "check and message are required", http.StatusBadRequest)
return
}
if !checks.HasFix(req.Check) {
writeJSONError(w, "no automated fix available for this check type", http.StatusBadRequest)
return
}
message, details, filePath, stored, err := s.fixTargetFromStore(req.Key, req.Check, req.Message, req.Details, req.FilePath)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
resolve := s.store.PrepareFindingResolution(stored)
result := s.applyFix(r.Context(), req.Check, message, details, filePath)
// A successful fix removes the evidence: drop the listing and re-arm the
// alert, so a reinfection at the same path is reported.
if result.Success {
resolve()
s.auditLog(r, "fix", req.Check, result.Action)
}
writeRemediation(w, result)
}
// apiVerifyFinding re-checks whether a finding's condition still holds against
// the live filesystem. It dismisses a resolved finding, lowers an inert
// replacement to Warning, or restores an earlier automatic demotion. This lets
// an operator confirm a manual fix immediately instead of waiting for the next
// scan, and is the "Re-check" action behind a finding row.
// POST /api/v1/verify-finding body: {"check":"...","message":"...","details":"...","file_path":"...","key":"..."}
func (s *Server) apiVerifyFinding(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req verifyFindingRequest
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.Check == "" || req.Message == "" {
writeJSONError(w, "check and message are required", http.StatusBadRequest)
return
}
in, _, stored, found := s.verifyFindingInput(req)
resolve := s.store.PrepareFindingResolution(stored)
in.Context = r.Context()
response := verifyFindingResponse{OK: true, VerifyResult: s.verifyFinding(in)}
switch {
case response.Checked && response.Resolved:
if found && resolve() {
s.auditLog(r, "verify-resolved", req.Check, response.Detail)
}
// A severity change rewrites the stored finding, so it needs the exact
// snapshot the verifier read; a request that could not be matched to one
// leaves the finding alone rather than guessing which it meant.
case found && checks.ShouldRestoreSeverity(stored, response.VerifyResult):
if s.store.RestoreLatestFindingSeverity(stored) {
response.SeverityChange = "restored"
s.auditLog(r, "verify-restored", req.Check, response.Detail)
}
case found && checks.ShouldDemoteSeverity(stored, response.VerifyResult):
if s.store.DemoteLatestFinding(stored, alert.Warning) {
response.SeverityChange = "demoted"
s.auditLog(r, "verify-demoted", req.Check, response.Detail)
}
}
writeJSON(w, response)
}
// verifyFindingResponse distinguishes the verifier's recommendation from a
// state change the store actually accepted. Demote remains a verdict: it can be
// true for an already-demoted finding or after a concurrent scan replaced the
// snapshot, neither of which means this request changed the stored severity.
type verifyFindingResponse struct {
OK bool `json:"ok"`
checks.VerifyResult
SeverityChange string `json:"severity_change,omitempty"`
}
type verifyFindingRequest struct {
Check string `json:"check"`
Message string `json:"message"`
Details string `json:"details"`
FilePath string `json:"file_path"`
ContentSHA256 string `json:"content_sha256"`
Key string `json:"key"`
}
// verifyFindingInput builds the verifier input, and returns the stored finding
// it was built from so a caller applying a severity change can pass the exact
// snapshot the verifier saw.
func (s *Server) verifyFindingInput(req verifyFindingRequest) (checks.VerifyInput, string, alert.Finding, bool) {
in := checks.VerifyInput{
Check: req.Check, Message: req.Message, Details: req.Details,
Path: req.FilePath,
}
f, ok := s.latestFindingForVerify(req.Key, req.Check, req.Message)
if !ok {
return in, req.Key, alert.Finding{}, false
}
in.Message = f.Message
in.Details = f.Details
in.Path = f.FilePath
in.ContentSHA256 = f.ContentSHA256
in.DetectLogic = f.DetectLogic
return in, f.Key(), f, true
}
func (s *Server) latestFindingForVerify(key, check, message string) (alert.Finding, bool) {
var matched alert.Finding
found := false
for _, f := range s.store.LatestFindings() {
if f.Check != check || f.Message != message {
continue
}
if key != "" {
if f.Key() == key {
return f, true
}
continue
}
if found {
return alert.Finding{}, false
}
matched = f
found = true
}
return matched, found
}
const bulkFixBodyMax = 64 * 1024
// apiBulkFix applies fixes to multiple findings at once.
// POST /api/v1/fix-bulk body: [{"check":"...", "message":"...", "details":"..."}, ...]
func (s *Server) apiBulkFix(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var reqs []struct {
Check string `json:"check"`
Message string `json:"message"`
Details string `json:"details"`
FilePath string `json:"file_path"`
Key string `json:"key"`
}
if err := decodeJSONBodyLimited(w, r, bulkFixBodyMax, &reqs); err != nil {
writeJSONError(w, "invalid request body", http.StatusBadRequest)
return
}
if len(reqs) == 0 {
writeJSONError(w, "At least one fix is required", http.StatusBadRequest)
return
}
results := make([]bulkFixItem, 0, len(reqs))
for _, req := range reqs {
if !checks.HasFix(req.Check) {
results = append(results, bulkFixItem{Check: req.Check, Error: fmt.Sprintf("no fix for %s", req.Check)})
continue
}
message, details, filePath, stored, err := s.fixTargetFromStore(req.Key, req.Check, req.Message, req.Details, req.FilePath)
if err != nil {
results = append(results, bulkFixItem{Check: req.Check, Error: err.Error()})
continue
}
resolve := s.store.PrepareFindingResolution(stored)
result := s.applyFix(r.Context(), req.Check, message, details, filePath)
if result.Success {
resolve()
s.auditLog(r, "fix", req.Check, result.Action)
}
results = append(results, bulkFixItem{
Check: req.Check, OK: result.Success, Action: result.Action,
Description: result.Description, Error: result.Error, Reverted: result.Reverted,
})
}
succeeded := 0
for _, item := range results {
if item.OK {
succeeded++
}
}
fields := map[string]interface{}{
"results": results,
"total": len(results),
"succeeded": succeeded,
"failed": len(results) - succeeded,
}
if succeeded == 0 {
fields["error"] = "No fix applied"
writeJSONStatus(w, http.StatusUnprocessableEntity, fields)
return
}
writeOK(w, fields)
}
// bulkFixItem is one fix of a bulk request: which check, whether it applied,
// and what it did or why it did not.
type bulkFixItem struct {
Check string `json:"check"`
OK bool `json:"ok"`
Action string `json:"action,omitempty"`
Description string `json:"description,omitempty"`
Error string `json:"error,omitempty"`
Reverted bool `json:"reverted,omitempty"`
}
// bulkItemFailure names one item of a batch that did not apply and why.
type bulkItemFailure struct {
Item string `json:"item"`
Error string `json:"error"`
}
// apiAccounts returns the account names for the scan dropdown: the accounts a
// server-wide scan covers.
//
//nolint:unused // registered via mux.Handle in server.go
func (s *Server) apiAccounts(w http.ResponseWriter, _ *http.Request) {
accounts, err := s.scanAccounts(s.liveCfg())
if err != nil {
writeJSONError(w, "Could not list accounts", http.StatusInternalServerError)
return
}
writeAll(w, accounts)
}
// --- Action endpoints ---
// apiBlockIP blocks an IP via the firewall engine.
// POST /api/v1/block-ip body: {"ip": "1.2.3.4", "reason": "..."}
func (s *Server) apiBlockIP(w http.ResponseWriter, r *http.Request) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IP string `json:"ip"`
Reason string `json:"reason"`
Duration string `json:"duration"`
// IncidentID, when set, notes the block on that incident so the
// timeline shows an operator acted. Optional: the firewall action is
// the point, the note is bookkeeping.
IncidentID json.RawMessage `json:"incident_id"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil {
writeJSONError(w, "Invalid request body", http.StatusBadRequest)
return
}
if req.IP == "" {
writeJSONError(w, "IP is required", http.StatusBadRequest)
return
}
parsedIP, err := parseAndValidateIP(req.IP)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
// Audit, incident and threat records key on the canonical spelling.
req.IP = parsedIP.String()
if req.Reason == "" {
req.Reason = "Blocked via CSM Web UI"
}
dur, err := parseDuration(req.Duration)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
if s.blocker == nil {
writeJSONError(w, "Firewall engine not available", http.StatusServiceUnavailable)
return
}
// Operator-initiated: bypass auto_response.dry_run gate.
if err := blockIPForOperator(s.blocker, req.IP, req.Reason, dur); err != nil {
writeJSONError(w, fmt.Sprintf("Block failed: %v", err), http.StatusInternalServerError)
return
}
s.auditLog(r, "block_ip", req.IP, req.Reason)
// A failure to annotate must not turn a successful block into an error:
// the address is blocked either way, and a stale incident id is the
// operator's tab being out of date, not a fault worth refusing.
var incidentID string
if json.Unmarshal(req.IncidentID, &incidentID) == nil && incidentID != "" && s.incidentCorrelator != nil {
if err := s.incidentCorrelator.RecordOperatorBlock(incidentID, req.IP, dur); err != nil {
log.Printf("webui: could not note an operator block on incident %s: %v",
safeLogString(incidentID), err)
}
}
resp := map[string]interface{}{"ip": req.IP}
// The input chain accepts Cloudflare edges on 80/443 before the blocked
// drop, so a block of a covered IP does not stop its web traffic.
if cc, ok := s.blocker.(cloudflareChecker); ok && cc.CloudflareCovers(req.IP) {
resp["warning"] = firewall.CloudflareCoverageWarning
}
writeOK(w, resp)
}
// apiUnblockIP removes an IP from the firewall + cphulk.
// POST /api/v1/unblock-ip body: {"ip": "1.2.3.4"}
func (s *Server) apiUnblockIP(w http.ResponseWriter, r *http.Request) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IP string `json:"ip"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.IP == "" {
writeJSONError(w, "IP is required", http.StatusBadRequest)
return
}
parsedIP, err := parseAndValidateIP(req.IP)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
req.IP = parsedIP.String()
if s.blocker == nil {
writeJSONError(w, "Firewall engine not available", http.StatusServiceUnavailable)
return
}
if err := s.blocker.UnblockIP(req.IP); err != nil {
writeJSONError(w, fmt.Sprintf("Unblock failed: %v", err), http.StatusInternalServerError)
return
}
checks.DropAutoBlockThreatRow(req.IP)
// Also flush from cphulk (cPanel brute force detector); best effort,
// the unblock is what was asked.
_ = flushCphulk(req.IP)
s.auditLog(r, "unblock_ip", req.IP, "manual unblock via UI")
writeOK(w, map[string]interface{}{"ip": req.IP})
}
// apiUnblockBulk unblocks multiple IPs at once.
func (s *Server) apiUnblockBulk(w http.ResponseWriter, r *http.Request) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IPs []string `json:"ips"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || len(req.IPs) == 0 {
writeJSONError(w, "IPs array is required", http.StatusBadRequest)
return
}
// 500 fits the largest Blocks-table page size (250 plus All) so one UI
// selection round-trips as a single request and a single undo token.
if len(req.IPs) > 500 {
writeJSONError(w, "IPs must be 1-500 items", http.StatusBadRequest)
return
}
if s.blocker == nil {
writeJSONError(w, "Firewall engine not available", http.StatusServiceUnavailable)
return
}
priorBlocks := make(map[string]firewall.BlockedEntry)
seen := make(map[string]bool, len(req.IPs))
succeeded := 0
unblocked := make([]string, 0, len(req.IPs))
failed := []bulkItemFailure{}
removedThreats := make([]undoThreatRow, 0, len(req.IPs))
for _, ip := range req.IPs {
parsed, err := parseAndValidateIP(ip)
if err != nil {
failed = append(failed, bulkItemFailure{Item: ip, Error: err.Error()})
continue
}
ip = parsed.String()
if seen[ip] {
continue
}
seen[ip] = true
before, err := s.unblockIPForUndo(ip)
if err != nil {
failed = append(failed, bulkItemFailure{Item: ip, Error: err.Error()})
continue
}
if before != nil {
priorBlocks[ip] = *before
}
if row, ok := captureUndoThreatRow(ip, true); ok {
removedThreats = append(removedThreats, row)
}
checks.DropAutoBlockThreatRow(ip)
s.auditLog(r, "unblock_ip", ip, "bulk unblock via UI")
unblocked = append(unblocked, ip)
succeeded++
}
_ = flushCphulkIPs(unblocked) // best effort: the unblocks are what was asked
var undoToken string
if succeeded > 0 {
undoToken = s.recordUndoEntry(r, "firewall_bulk_unblock", undoInverseFirewallUnblock,
fmt.Sprintf("Unblocked %d IPs", succeeded),
undoPayloadIPs{
IPs: unblocked,
RestoreThreats: removedThreats,
BlockSnapshot: true,
RestoreBlocks: priorBlocks,
})
}
fields := map[string]interface{}{
"total": len(req.IPs),
"succeeded": succeeded,
"failed": failed,
}
if succeeded == 0 {
fields["error"] = "No address was unblocked"
writeJSONStatus(w, http.StatusUnprocessableEntity, fields)
return
}
if undoToken != "" {
fields["undo_token"] = undoToken
}
writeOK(w, fields)
}
// blockedEntry is a raw blocked IP record from firewall state.
type blockedEntry struct {
IP string `json:"ip"`
Reason string `json:"reason"`
Source string `json:"source,omitempty"`
BlockedAt time.Time `json:"blocked_at"`
ExpiresAt time.Time `json:"expires_at"`
}
type blockedView struct {
IP string `json:"ip"`
Reason string `json:"reason"`
Source string `json:"source"`
BlockedAt time.Time `json:"blocked_at,omitzero"`
// ExpiresAt is left out for a permanent block.
ExpiresAt time.Time `json:"expires_at,omitzero"`
}
func formatBlockedView(b blockedEntry) (blockedView, bool) {
if !b.ExpiresAt.IsZero() && time.Now().After(b.ExpiresAt) {
return blockedView{}, false // expired
}
view := blockedView{
IP: b.IP,
Reason: b.Reason,
Source: b.Source,
BlockedAt: b.BlockedAt.UTC(),
ExpiresAt: b.ExpiresAt.UTC(),
}
if view.Source == "" {
view.Source = firewall.InferProvenance("block", b.Reason)
}
return view, true
}
// apiBlockedIPs returns the list of currently blocked IPs.
func (s *Server) apiBlockedIPs(w http.ResponseWriter, _ *http.Request) {
result := []blockedView{}
fwFile := filepath.Join(s.cfg.StatePath, "firewall", "state.json")
_, fwStatErr := os.Stat(fwFile) // #nosec G304 -- filepath.Join under operator-configured StatePath.
fwState, fwErr := firewall.LoadState(s.cfg.StatePath)
if fwErr != nil && fwStatErr == nil {
// The engine state exists but cannot be read: an empty list would
// tell the operator nothing is blocked.
writeJSONError(w, "Firewall state unavailable", http.StatusInternalServerError)
return
}
if fwErr == nil && fwState != nil {
for _, entry := range fwState.Blocked {
b := blockedEntry{
IP: entry.IP,
Reason: entry.Reason,
Source: entry.Source,
BlockedAt: entry.BlockedAt,
ExpiresAt: entry.ExpiresAt,
}
if view, ok := formatBlockedView(b); ok {
result = append(result, view)
}
}
// A present engine state file wins even when empty. blocked_ips.json
// is only a legacy fallback when the engine file does not exist.
if fwStatErr == nil || len(fwState.Blocked) > 0 {
writeAll(w, result)
return
}
}
// Fall back to blocked_ips.json (legacy)
stateFile := filepath.Join(s.cfg.StatePath, "blocked_ips.json")
// #nosec G304 -- filepath.Join under operator-configured StatePath.
data, err := os.ReadFile(stateFile)
if os.IsNotExist(err) {
writeAll(w, result)
return
}
if err != nil {
writeJSONError(w, "Firewall state unavailable", http.StatusInternalServerError)
return
}
var blockState struct {
IPs []blockedEntry `json:"ips"`
}
if err := json.Unmarshal(data, &blockState); err != nil {
writeJSONError(w, "Firewall state unavailable", http.StatusInternalServerError)
return
}
for _, b := range blockState.IPs {
if view, ok := formatBlockedView(b); ok {
result = append(result, view)
}
}
writeAll(w, result)
}
// apiDismissFinding marks a finding as baseline (acknowledged/dismissed).
// POST /api/v1/dismiss body: {"key": "check:message"}
// dismissBulkMax bounds one dismiss request. The whole request is one undo
// entry, so a larger selection must be narrowed rather than split.
const dismissBulkMax = 500
func (s *Server) apiDismissFinding(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
Key string `json:"key"`
Keys []string `json:"keys"`
// Contents maps a key to the content hash its row showed (empty for
// none). A listed key whose content changed since is not dismissed.
Contents map[string]*string `json:"contents"`
}
if err := decodeJSONBodyLimited(w, r, 1<<20, &req); err != nil {
writeJSONError(w, "Invalid request body", http.StatusBadRequest)
return
}
keys := req.Keys
switch {
case req.Key != "" && len(req.Keys) > 0:
writeJSONError(w, "Send key or keys, not both", http.StatusBadRequest)
return
case req.Key != "":
keys = []string{req.Key}
case len(keys) == 0:
writeJSONError(w, "Key is required", http.StatusBadRequest)
return
case len(keys) > dismissBulkMax:
writeJSONError(w, fmt.Sprintf("At most %d findings per request", dismissBulkMax), http.StatusBadRequest)
return
}
uniqueKeys := make([]string, 0, len(keys))
seenKeys := make(map[string]bool, len(keys))
for _, key := range keys {
if key == "" {
writeJSONError(w, "Key is required", http.StatusBadRequest)
return
}
if !seenKeys[key] {
seenKeys[key] = true
uniqueKeys = append(uniqueKeys, key)
}
}
keys = uniqueKeys
for _, shown := range req.Contents {
if shown == nil || (*shown != "" && !isContentSHA256(*shown)) {
writeJSONError(w, "Invalid content hash", http.StatusBadRequest)
return
}
}
undos := make([]state.DismissUndo, 0, len(keys))
applied := make([]string, 0, len(keys))
var refused []string
for _, key := range keys {
var u state.DismissUndo
if shown, claimed := req.Contents[key]; claimed {
var ok bool
if u, ok = s.store.DismissFindingAsShown(key, *shown); !ok {
refused = append(refused, key)
continue
}
} else {
u = s.store.DismissFindingWithUndo(key)
}
undos = append(undos, u)
applied = append(applied, key)
s.auditLog(r, "dismiss", key, "")
}
if len(keys) == 1 && len(refused) == 1 {
writeJSONError(w, "The file changed since the page loaded. Review the finding again.", http.StatusConflict)
return
}
resp := map[string]interface{}{"count": len(applied)}
if len(refused) > 0 {
resp["refused"] = refused
}
summary := fmt.Sprintf("Dismissed %d findings", len(applied))
if len(applied) == 1 {
check, _ := state.ParseKey(applied[0])
summary = "Dismissed " + check + " finding"
}
if len(keys) == 1 {
resp["key"] = keys[0]
}
if len(applied) > 0 {
if token := s.recordUndoEntry(r, "dismiss", undoInverseFindingUndismiss, summary,
undoPayloadIPs{Dismissals: undos}); token != "" {
resp["undo_token"] = token
}
}
writeOK(w, resp)
}
// isContentSHA256 reports a finding content hash: 64 lowercase hex characters.
func isContentSHA256(s string) bool {
b, err := hex.DecodeString(s)
return err == nil && len(b) == sha256.Size && hex.EncodeToString(b) == s
}
// apiQuarantinePreview returns the first 8KB of a quarantined file for inspection.
func (s *Server) apiQuarantinePreview(w http.ResponseWriter, r *http.Request) {
entry, err := resolveQuarantineEntry(r.URL.Query().Get("id"))
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
info, err := os.Stat(entry.ItemPath)
if err != nil {
writeJSONError(w, "not found", http.StatusNotFound)
return
}
if info.IsDir() {
writeJSON(w, map[string]interface{}{
"id": entry.ID, "is_dir": true,
"preview": "[directory - content preview not available]",
})
return
}
f, err := os.Open(entry.ItemPath)
if err != nil {
writeJSONError(w, "cannot read file", http.StatusInternalServerError)
return
}
defer f.Close()
buf := make([]byte, 8192)
n, _ := f.Read(buf)
writeJSON(w, map[string]interface{}{
"id": entry.ID,
"preview": string(buf[:n]),
"truncated": info.Size() > 8192,
"total_size": info.Size(),
})
}
// quarantineBulkDeleteMax bounds the files one bulk-delete request removes.
// The UI sends larger selections as several requests of this size.
const quarantineBulkDeleteMax = 100
// apiQuarantineBulkDelete permanently removes quarantined files and their metadata.
// removeQuarantineItem deletes one quarantined file or directory. Tests
// replace it to exercise a deletion the filesystem refuses.
var removeQuarantineItem = os.RemoveAll
func (s *Server) apiQuarantineBulkDelete(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IDs []string `json:"ids"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil {
writeJSONError(w, "Invalid request body", http.StatusBadRequest)
return
}
if len(req.IDs) == 0 || len(req.IDs) > quarantineBulkDeleteMax {
writeJSONError(w, fmt.Sprintf("IDs must be 1-%d items", quarantineBulkDeleteMax), http.StatusBadRequest)
return
}
count := 0
deleted := []string{}
failed := []string{}
for _, id := range req.IDs {
entry, err := resolveQuarantineEntry(id)
if err != nil || !quarantineEntryDeletable(entry) {
failed = append(failed, id)
continue
}
if _, statErr := os.Lstat(entry.ItemPath); statErr == nil {
if err := removeQuarantineItem(entry.ItemPath); err != nil {
// Keep the sidecar: the list is built from sidecars, so the
// archive stays visible and the delete can be retried.
log.Printf("webui: failed to delete quarantined %s: %v", safeLogString(entry.ItemPath), err)
failed = append(failed, id)
continue
}
count++
} else if !os.IsNotExist(statErr) {
failed = append(failed, id)
continue
}
if err := os.Remove(entry.MetaPath); err != nil && !os.IsNotExist(err) {
log.Printf("webui: failed to remove quarantine meta %s: %v", safeLogString(entry.MetaPath), err)
}
deleted = append(deleted, id)
}
details := "deleted: " + strings.Join(deleted, ", ")
if len(failed) > 0 {
details += "; failed: " + strings.Join(failed, ", ")
}
s.auditLog(r, "quarantine_bulk_delete", fmt.Sprintf("%d files", count), details)
if len(deleted) == 0 {
writeJSONStatus(w, http.StatusUnprocessableEntity, map[string]interface{}{
"error": "No file was deleted", "count": 0, "failed": failed,
})
return
}
writeOK(w, map[string]interface{}{"count": count, "failed": failed})
}
// apiTestAlert sends a test finding through all configured alert channels.
func (s *Server) apiTestAlert(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
cfg := s.liveCfg()
if !cfg.Alerts.Email.Enabled && !cfg.Alerts.Webhook.Enabled {
writeJSONError(w, "No alert channel is enabled", http.StatusServiceUnavailable)
return
}
testFinding := []alert.Finding{{
Severity: alert.Warning,
Check: "test_alert",
Message: "Test alert from CSM Web UI",
Details: fmt.Sprintf("Sent by admin at %s", time.Now().Format("2006-01-02 15:04:05")),
Timestamp: time.Now(),
}}
err := alert.Dispatch(cfg, testFinding)
if err != nil {
// The request was fine; the alert channel behind the daemon failed.
writeJSONError(w, "Alert delivery failed: "+err.Error(), http.StatusBadGateway)
return
}
s.auditLog(r, "test_alert", "notification", "sent test alert")
writeOK(w, nil)
}
// apiScanAccount runs an on-demand scan for a single cPanel account.
// POST /api/v1/scan-account body: {"account": "username"}
func (s *Server) apiScanAccount(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
Account string `json:"account"`
}
if err := decodeJSONBodyLimited(w, r, 32*1024, &req); err != nil || req.Account == "" {
writeJSONError(w, "Account name is required", http.StatusBadRequest)
return
}
if err := validateAccountName(req.Account); err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
// Rate limit: only one scan at a time
if !s.acquireScan() {
writeJSONError(w, "A scan is already in progress. Please wait.", http.StatusTooManyRequests)
return
}
defer s.releaseScan()
// Extend the write deadline for this long-running request.
// Account scans can take several minutes; the default WriteTimeout
// causes ERR_HTTP2_PROTOCOL_ERROR in browsers when it fires mid-stream.
rc := http.NewResponseController(w)
_ = rc.SetWriteDeadline(time.Now().Add(longRequestTimeout))
start := time.Now()
findings := checks.RunAccountScan(s.liveCfg(), s.store, req.Account)
elapsed := time.Since(start).Round(time.Millisecond)
s.auditLog(r, "scan_account", req.Account, fmt.Sprintf("%d findings in %s", len(findings), elapsed))
writeOK(w, map[string]interface{}{
"account": req.Account,
"count": len(findings),
"elapsed_seconds": elapsed.Seconds(),
})
}
// parseModeString converts a permission string like "-rw-r--r--" to os.FileMode.
func parseModeString(s string) os.FileMode {
if len(s) < 10 {
return 0644
}
var mode os.FileMode
perms := s[len(s)-9:] // last 9 chars: "rwxr-xr-x"
bits := []os.FileMode{
0400, 0200, 0100, // owner r/w/x
0040, 0020, 0010, // group r/w/x
0004, 0002, 0001, // other r/w/x
}
for i, b := range bits {
if i < len(perms) && perms[i] != '-' {
mode |= b
}
}
for _, flag := range s[:len(s)-9] {
switch flag {
case 'u':
mode |= os.ModeSetuid
case 'g':
mode |= os.ModeSetgid
case 't':
mode |= os.ModeSticky
}
}
return mode
}
// flushCphulk removes brute-force login history for an IP from cPanel's cphulk.
// Callers should pre-validate `ip` with parseAndValidateIP. This function
// re-validates as defense-in-depth so a future caller that forgets cannot
// expose a shell-execution surface even if exec.Command itself does not
// invoke a shell.
func flushCphulk(ip string) error {
return flushCphulkIPs([]string{ip})
}
// flushCphulkIPs uses the WHM API's indexed array arguments so a bulk
// firewall action starts one whmapi1 process instead of one per address.
// It returns whmapi1's failure, including whmapi1 not being installed, so
// a caller reports the flush only when it ran.
func flushCphulkIPs(ips []string) error {
args := []string{"flush_cphulk_login_history_for_ips"}
valid := 0
for _, ip := range ips {
parsed, err := parseRecordedIP(ip)
if err != nil {
continue
}
param := "ip"
if valid > 0 {
param = fmt.Sprintf("ip-%d", valid)
}
args = append(args, param+"="+parsed.String())
valid++
}
if valid == 0 {
return nil
}
// #nosec G204 -- whmapi1 is fixed and every argument value is parsed as
// an IP above. exec.Command passes arguments directly without a shell.
if _, err := exec.Command("whmapi1", args...).Output(); err != nil {
return fmt.Errorf("whmapi1: %w", err)
}
return nil
}
// apiExport returns a JSON bundle of exportable state.
func (s *Server) apiExport(w http.ResponseWriter, _ *http.Request) {
// Collect suppressions
suppressions := s.store.LoadSuppressions()
if suppressions == nil {
suppressions = []state.SuppressionRule{}
}
// Collect whitelist
var whitelist []checks.WhitelistIP
if tdb := checks.GetThreatDB(); tdb != nil {
whitelist = tdb.WhitelistedIPs()
}
if whitelist == nil {
whitelist = []checks.WhitelistIP{}
}
bundle := map[string]interface{}{
"exported_at": time.Now().UTC(),
"hostname": s.cfg.Hostname,
"suppressions": suppressions,
"whitelist": whitelist,
}
w.Header().Set("Content-Disposition", "attachment; filename=csm-state-export.json")
writeJSON(w, bundle)
}
// apiImport merges an exported state bundle into the current state.
func (s *Server) apiImport(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
// The bundle is what /api/v1/export writes; the decoder refuses unknown
// fields, so every field the export carries is named here.
var bundle struct {
ExportedAt string `json:"exported_at"`
Hostname string `json:"hostname"`
Suppressions []state.SuppressionRule `json:"suppressions"`
Whitelist []checks.WhitelistIP `json:"whitelist"`
}
if err := decodeJSONBodyLimited(w, r, 512*1024, &bundle); err != nil {
writeJSONError(w, "invalid JSON body", http.StatusBadRequest)
return
}
imported, skipped := 0, 0
warning := ""
// Merge suppressions (dedup by ID)
if len(bundle.Suppressions) > 0 {
err := s.store.UpdateSuppressions(func(existing []state.SuppressionRule) ([]state.SuppressionRule, error) {
existingIDs := make(map[string]bool)
for _, rule := range existing {
existingIDs[rule.ID] = true
}
for _, rule := range bundle.Suppressions {
// Same contract as a rule added through the UI: a check name
// and a valid glob are required (otherwise the rule
// suppresses nothing and only clutters the list), and every
// rule needs an ID or it can never be deleted from the UI.
if !suppressionCheckName.MatchString(rule.Check) {
skipped++
continue
}
if rule.PathPattern != "" {
if _, err := filepath.Match(rule.PathPattern, ""); err != nil {
skipped++
continue
}
}
if rule.ID == "" {
rule.ID = newSuppressionID()
}
if rule.CreatedAt.IsZero() {
rule.CreatedAt = time.Now()
}
if !existingIDs[rule.ID] {
existingIDs[rule.ID] = true
existing = append(existing, rule)
imported++
}
}
return existing, nil
})
if err != nil {
writeJSONError(w, fmt.Sprintf("failed to save suppressions: %v", err), http.StatusInternalServerError)
return
}
}
// Merge whitelist IPs
if len(bundle.Whitelist) > 0 {
tdb := checks.GetThreatDB()
if tdb == nil {
skipped += len(bundle.Whitelist)
warning = "The threat database is not available; whitelist entries were not imported."
} else {
existingSet := make(map[string]bool)
for _, w := range tdb.WhitelistedIPs() {
existingSet[w.IP] = true
}
now := time.Now()
for _, entry := range bundle.Whitelist {
// Validate imported IPs like every interactive route does: an
// unvalidated bundle could otherwise poison the threat DB /
// firewall allow-list with malformed or attacker-chosen entries
// (whitelisting bypasses blocking). Use the canonical form.
ip, err := parseAndValidateIP(entry.IP)
// Entries from the configuration file are managed there, and an
// expired temporary entry has nothing left to import.
if err != nil || entry.Configured || (entry.ExpiresAt != nil && !entry.ExpiresAt.After(now)) {
skipped++
continue
}
canonical := ip.String()
if existingSet[canonical] {
continue
}
// A temporary entry stays temporary, with its remaining time.
if entry.ExpiresAt != nil {
tdb.TempWhitelist(canonical, entry.ExpiresAt.Sub(now))
} else {
tdb.AddWhitelist(canonical)
}
existingSet[canonical] = true
imported++
}
}
}
s.auditLog(r, "import", "state", fmt.Sprintf("imported %d items, skipped %d", imported, skipped))
resp := map[string]interface{}{"imported": imported, "skipped": skipped}
if warning != "" {
resp["warning"] = warning
}
writeOK(w, resp)
}
// apiFindingDetail returns detail about a specific finding including related actions.
func (s *Server) apiFindingDetail(w http.ResponseWriter, r *http.Request) {
check := r.URL.Query().Get("check")
message := r.URL.Query().Get("message")
if check == "" {
writeJSONError(w, "check is required", http.StatusBadRequest)
return
}
// Alert state is keyed by Finding.Key(), which folds in a hash of the
// details (and the source IP for IP-keyed checks); "check:message" is
// only the key of a finding with neither. Resolve the stored finding
// first so findings with details get their first/last-seen times.
key := check + ":" + message
if f, ok := s.latestFindingForVerify(r.URL.Query().Get("key"), check, message); ok {
key = f.Key()
}
// Get state entry for this finding (first/last seen)
var firstSeen, lastSeen time.Time
if entry, ok := s.store.EntryForKey(key); ok {
firstSeen = entry.FirstSeen.UTC()
lastSeen = entry.LastSeen.UTC()
}
// Search audit log for related actions
actions := s.searchAuditEntries(check, 20)
// Search history for related findings (same check type, last 50)
allHistory, _ := s.store.ReadHistory(2000, 0)
type histEntry struct {
Severity string `json:"severity"`
Check string `json:"check"`
Message string `json:"message"`
Timestamp time.Time `json:"timestamp"`
}
var related []histEntry
for _, f := range allHistory {
if len(related) >= 50 {
break
}
if f.Check == check {
related = append(related, histEntry{
Severity: f.Severity.String(),
Check: f.Check,
Message: f.Message,
Timestamp: f.Timestamp.UTC(),
})
}
}
detail := map[string]interface{}{
"check": check,
"message": message,
"actions": actions,
"related": related,
}
if !firstSeen.IsZero() {
detail["first_seen"] = firstSeen
detail["last_seen"] = lastSeen
}
writeJSON(w, detail)
}
// extractAccountFromFinding returns the cPanel account a finding belongs to:
// the owner the check recorded (TenantID, or CPUser for mail relay), else a
// /home/{user}/ path in the message, details or file path, else "Account: "
// / "user: " in the details field (used by login checks).
func extractAccountFromFinding(f alert.Finding) string {
for _, owner := range []string{f.TenantID, f.CPUser} {
if owner = strings.TrimSpace(owner); owner != "" {
return owner
}
}
if f.FilePath == "" && (f.Check == "wp_core_unverified" || f.Check == "wp_plugin_inventory_unverified") {
// Collapsed coverage warnings carry an account only when every
// installation shares it. Their bounded path sample cannot establish
// ownership, even when it happens to show just one account.
return ""
}
for _, s := range []string{f.Message, f.Details, f.FilePath} {
if idx := strings.Index(s, "/home/"); idx >= 0 {
rest := s[idx+6:]
if end := strings.IndexByte(rest, '/'); end > 0 {
return rest[:end]
}
}
}
for _, prefix := range []string{"Account: ", "user: "} {
if idx := strings.Index(f.Details, prefix); idx >= 0 {
rest := f.Details[idx+len(prefix):]
end := strings.IndexAny(rest, " \n\t,")
if end > 0 {
return rest[:end]
}
if len(rest) > 0 {
return rest
}
}
}
return ""
}
func writeJSONError(w http.ResponseWriter, message string, code int) {
writeJSONStatus(w, code, map[string]string{"error": message})
}
// writeJSON sends compact JSON: indentation added about a third to large
// lists such as findings and history, and nothing reads it but code.
func writeJSON(w http.ResponseWriter, data interface{}) {
writeJSONStatus(w, http.StatusOK, data)
}
// writeJSONStatus sends data as JSON with the given status code.
func writeJSONStatus(w http.ResponseWriter, code int, data interface{}) {
body, err := apiValue(data)
var encoded []byte
if err == nil {
encoded, err = json.Marshal(body)
}
if err != nil {
code = http.StatusInternalServerError
encoded, _ = json.Marshal(map[string]string{"error": err.Error()})
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
_, _ = w.Write(append(encoded, '\n'))
}
// durationSeconds reads Go duration text such as "24h" as seconds. ok is
// false when the text is empty or not a duration.
func durationSeconds(text string) (float64, bool) {
d, err := time.ParseDuration(text)
if err != nil {
return 0, false
}
return d.Seconds(), true
}
// writeItems answers a collection: {"items": [...]} plus extra, which holds
// "total" when the handler counted the matches, the paging keys and side
// data. A nil list goes out as [] so an empty collection is never null.
func writeItems[T any](w http.ResponseWriter, items []T, extra map[string]interface{}) {
if items == nil {
items = []T{}
}
body := make(map[string]interface{}, len(extra)+1)
for k, v := range extra {
body[k] = v
}
body["items"] = items
writeJSON(w, body)
}
// writeAll answers a collection that holds every match, with their count.
func writeAll[T any](w http.ResponseWriter, items []T) {
writeItems(w, items, map[string]interface{}{"total": len(items)})
}
// writeCapped answers a collection cut to limit items out of total matches.
func writeCapped[T any](w http.ResponseWriter, items []T, total, limit int, extra map[string]interface{}) {
if extra == nil {
extra = map[string]interface{}{}
}
if len(items) > limit {
items = items[:limit]
}
extra["total"] = total
extra["offset"] = 0
extra["limit"] = limit
extra["truncated"] = total > len(items) || extra["truncated"] == true
writeItems(w, items, extra)
}
// writeOK answers a successful action: "ok": true plus the action's fields.
func writeOK(w http.ResponseWriter, fields map[string]interface{}) {
writeOKStatus(w, http.StatusOK, fields)
}
// writeOKStatus is writeOK with another 2xx status, such as 202 for work
// that continues after the response.
func writeOKStatus(w http.ResponseWriter, code int, fields map[string]interface{}) {
body := make(map[string]interface{}, len(fields)+1)
for k, v := range fields {
body[k] = v
}
body["ok"] = true
writeJSONStatus(w, code, body)
}
// writeRemediation answers one fix. A fix that did not apply is an error:
// 422 when the target was not eligible and left unchanged, 500 when applying
// it failed.
func writeRemediation(w http.ResponseWriter, res checks.RemediationResult) {
if !res.Success {
msg := res.Error
if msg == "" {
msg = "The fix did not apply"
}
code := http.StatusInternalServerError
if res.Refused {
code = http.StatusUnprocessableEntity
}
body := map[string]interface{}{"error": msg}
if res.Action != "" {
body["action"] = res.Action
}
writeJSONStatus(w, code, body)
return
}
fields := map[string]interface{}{"action": res.Action, "description": res.Description}
if res.Reverted {
fields["reverted"] = true
}
writeOK(w, fields)
}
// writeRequestError answers a failure from middleware that guards both API
// and page routes: JSON under /api/, plain text elsewhere.
func writeRequestError(w http.ResponseWriter, r *http.Request, msg string, code int) {
if strings.HasPrefix(r.URL.Path, "/api/") {
writeJSONError(w, msg, code)
return
}
http.Error(w, msg, code)
}
// apiNotFound answers every /api/ path no route matches. Without it the
// page catch-all served the dashboard HTML with 200 to API clients.
// Unauthenticated callers get 401, as for a real route.
func (s *Server) apiNotFound(w http.ResponseWriter, r *http.Request) {
if s.refuseSpentBearer(w, r) {
return
}
if !s.tokenHasScope(r, "read") {
writeJSONError(w, "Unauthorized", http.StatusUnauthorized)
return
}
writeJSONError(w, "Not found", http.StatusNotFound)
}
// queryInt reads a non-negative integer query parameter. A missing, negative
// or non-numeric value gives defaultVal.
func queryInt(r *http.Request, key string, defaultVal int) int {
val := r.URL.Query().Get(key)
if val == "" {
return defaultVal
}
n, err := strconv.Atoi(val)
if err != nil || n < 0 {
return defaultVal
}
return n
}
package webui
import (
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/pidginhost/csm/internal/alert"
)
// sseWriteTimeout caps how long each SSE write is allowed to block on a
// slow or stuck client. It must stay below the daemon's WebUI shutdown
// budget so an in-flight flush cannot outlive graceful shutdown.
const sseWriteTimeout = 3 * time.Second
// apiEvents streams findings to the client over Server-Sent Events. A
// subscriber connects once and receives a `data: {...}\n\n` block per
// finding plus a periodic `: keepalive\n\n` comment line every 25s so
// intermediate proxies don't time the connection out. Auth is checked
// by the upstream requireRead middleware.
func (s *Server) apiEvents(w http.ResponseWriter, r *http.Request) {
s.mu.RLock()
bus := s.findingBus
s.mu.RUnlock()
if bus == nil {
writeJSONError(w, "event bus not available", http.StatusServiceUnavailable)
return
}
if _, ok := w.(http.Flusher); !ok {
writeJSONError(w, "streaming unsupported", http.StatusInternalServerError)
return
}
// Reserve a subscriber slot before sending any stream headers so a flood of
// connections cannot exhaust daemon memory. Done up front so the cap can be
// reported as a clean 503 rather than mid-stream.
sub, ok := bus.TrySubscribe()
if !ok {
writeJSONError(w, "too many event stream subscribers", http.StatusServiceUnavailable)
return
}
shutdownDone := s.pruneDone
// The middleware authenticated the supplied credentials. Remember their
// presence, not their current validity: expiry between middleware and
// this handler must still select an authentication recheck.
_, cookieErr := r.Cookie("csm_auth")
authenticatedStream := cookieErr == nil || carriesBearer(r)
streamStopped := func() bool {
if r.Context().Err() != nil {
return true
}
// Keep the middleware's cookie-or-bearer policy, without extending
// an idle session on this passive stream.
if authenticatedStream {
if _, ok := s.cookieSessionToken(r, "read", false); !ok && !s.isBearerAuth(r) {
return true
}
}
select {
case <-shutdownDone:
return true
default:
return false
}
}
defer func() {
if streamStopped() {
bus.Unsubscribe(sub)
} else {
bus.Abort(sub)
}
}()
rc := http.NewResponseController(w)
setWriteDeadline := func() error {
return rc.SetWriteDeadline(time.Now().Add(sseWriteTimeout))
}
if err := setWriteDeadline(); err != nil {
writeJSONError(w, "streaming write deadlines unsupported", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no") // nginx won't buffer
writeFrame := func(format string, args ...any) error {
if err := setWriteDeadline(); err != nil {
return err
}
if _, err := fmt.Fprintf(w, format, args...); err != nil {
return err
}
return rc.Flush()
}
// Initial flush establishes the connection and proxies see the headers
// before the first event. Bound it by the same write deadline.
if err := writeFrame(""); err != nil {
return
}
keepalive := time.NewTicker(25 * time.Second)
defer keepalive.Stop()
for {
if streamStopped() {
return
}
select {
case <-r.Context().Done():
return
case <-shutdownDone:
return
case <-keepalive.C:
if streamStopped() {
return
}
if err := writeFrame(": keepalive\n\n"); err != nil {
return
}
case delivery, ok := <-sub.Events():
if !ok {
return
}
encodingFailed := false
err := delivery.Process(func(f alert.Finding) error {
if streamStopped() {
return nil
}
wire, err := apiValue(toAPIFinding(f))
if err != nil {
encodingFailed = true
return err
}
body, err := json.Marshal(wire)
if err != nil {
encodingFailed = true
return err
}
err = writeFrame("data: %s\n\n", body)
// Closing a tab can interrupt its write. Once demand is withdrawn,
// that cancellation must not become a host delivery failure.
if err != nil && streamStopped() {
return nil
}
return err
})
if err != nil && !encodingFailed {
return
}
}
}
}
package webui
import (
"fmt"
"path/filepath"
"github.com/pidginhost/csm/internal/alert"
)
// fixTargetFromStore pins a fix request to the stored finding it names. The
// returned snapshot is the evidence a successful fix may resolve.
// Only when the finding is no longer in the latest set does the client input
// stand on its own, bounded by the remediation roots as before.
func (s *Server) fixTargetFromStore(key, check, message, details, filePath string) (string, string, string, alert.Finding, error) {
latest := s.store.LatestFindings()
if key != "" {
for _, f := range latest {
if f.Key() != key {
continue
}
if f.Check != check {
return "", "", "", alert.Finding{}, fmt.Errorf("check does not match the stored finding")
}
return storedFixTarget(f, filePath)
}
}
var matched alert.Finding
found := false
for _, f := range latest {
if f.Check != check || f.Message != message {
continue
}
if found {
return "", "", "", alert.Finding{}, fmt.Errorf("finding key is required for an ambiguous fix target")
}
matched = f
found = true
}
if !found {
return message, details, filePath, alert.Finding{}, nil
}
return storedFixTarget(matched, filePath)
}
func storedFixTarget(f alert.Finding, requestedPath string) (string, string, string, alert.Finding, error) {
// Older findings have no FilePath. Their stored message remains the
// authority; a caller-supplied path must not replace it.
if f.FilePath != "" && requestedPath != "" && filepath.Clean(requestedPath) != filepath.Clean(f.FilePath) {
return "", "", "", alert.Finding{}, fmt.Errorf("file_path does not match the stored finding")
}
return f.Message, f.Details, f.FilePath, f, nil
}
package webui
import (
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/quarantinefs"
"github.com/pidginhost/csm/internal/safepath"
)
// apiQuarantineRestore restores a quarantined file to its original location.
// POST /api/v1/quarantine-restore body: {"id": "filename"}
// forgetRemovedFile lets restore tests observe the ledger update.
var forgetRemovedFile = checks.ForgetRemovedFile
func (s *Server) apiQuarantineRestore(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
ID string `json:"id"`
}
if err := decodeJSONBodyLimited(w, r, 16*1024, &req); err != nil || req.ID == "" {
writeJSONError(w, "ID is required", http.StatusBadRequest)
return
}
entry, err := resolveQuarantineEntry(req.ID)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
if s.resumeQuarantinedSpoolMessage(w, r, entry) {
return
}
if !quarantineEntryDeletable(entry) {
writeJSONError(w, "Quarantine entry not found", http.StatusNotFound)
return
}
metaData, err := os.ReadFile(entry.MetaPath)
if err != nil {
writeJSONError(w, "Quarantine entry not found", http.StatusNotFound)
return
}
var meta checks.QuarantineMeta
if unmarshalErr := json.Unmarshal(metaData, &meta); unmarshalErr != nil {
writeJSONError(w, "Invalid metadata", http.StatusInternalServerError)
return
}
if meta.SpoolDir != "" {
s.restoreQuarantinedSpoolMessage(w, r, entry, meta)
return
}
roots, rootErr := quarantineRootsForConfig(s.cfg)
restorePath, err := validateQuarantineRestorePath(meta.OriginalPath, roots)
if err != nil {
writeJSONError(w, errors.Join(err, rootErr).Error(), http.StatusBadRequest)
return
}
if quarantineRestoreAfterValidateForTest != nil {
quarantineRestoreAfterValidateForTest(restorePath)
}
target, err := openQuarantineRestoreTarget(restorePath, roots, meta.RestoreAction == "")
if err != nil {
writeJSONError(w, fmt.Sprintf("Cannot open restore destination: %v", err), http.StatusConflict)
return
}
defer target.Close()
// Check if quarantined item is a directory or file
quarInfo, err := os.Lstat(entry.ItemPath)
if err != nil {
writeJSONError(w, fmt.Sprintf("Cannot stat quarantined file: %v", err), http.StatusInternalServerError)
return
}
if quarInfo.Mode()&os.ModeSymlink != 0 {
writeJSONError(w, "Cannot restore symlink quarantine entry", http.StatusInternalServerError)
return
}
// Parse original mode from metadata (format: "-rw-r--r--" or "drwxr-xr-x")
restoredMode := os.FileMode(0644)
if meta.Mode != "" && len(meta.Mode) >= 10 {
restoredMode = parseModeString(meta.Mode)
}
// Before the write: the restore must not read as the removed file
// coming back.
forgetRemovedFile(restorePath)
if meta.RestoreAction != "" {
if filepath.Clean(filepath.Dir(entry.ItemPath)) != filepath.Join(quarantineDir, "pre_clean") {
writeJSONError(w, "Virtual-patch backups must come from pre_clean", http.StatusBadRequest)
return
}
if quarInfo.IsDir() {
writeJSONError(w, "Invalid virtual-patch backup", http.StatusInternalServerError)
return
}
if err := checks.RestoreVirtualPatchBackup(entry.ItemPath, target, meta); err != nil {
if errors.Is(err, checks.ErrVirtualPatchRestoreConflict) {
writeJSONError(w, err.Error(), http.StatusConflict)
return
}
writeJSONError(w, fmt.Sprintf("Cannot restore virtual-patch backup: %v", err), http.StatusInternalServerError)
return
}
if err := removeRestoredQuarantineEvidence(entry.ItemPath, entry.MetaPath); err != nil {
writeJSONError(w, err.Error(), http.StatusInternalServerError)
return
}
s.auditLog(r, "restore", restorePath, "virtual-patch restore")
writeOK(w, map[string]interface{}{
"path": restorePath,
"warning": "Virtual-patch reverted. Re-scan recommended.",
})
return
}
if err := target.Check(); err != nil {
writeJSONError(w, err.Error(), http.StatusConflict)
return
}
if quarInfo.IsDir() {
if err := restoreQuarantineDirectory(entry.ItemPath, target, restoredMode, meta); err != nil {
writeJSONError(w, fmt.Sprintf("Cannot restore directory: %v", err), http.StatusConflict)
return
}
} else {
if err := restoreQuarantineFile(entry.ItemPath, restorePath, target, restoredMode, meta); err != nil {
writeRestoreError(w, err)
return
}
}
if err := removeRestoredQuarantineEvidence(entry.ItemPath, entry.MetaPath); err != nil {
writeJSONError(w, err.Error(), http.StatusInternalServerError)
return
}
s.auditLog(r, "restore", restorePath, "quarantine restore")
writeOK(w, map[string]interface{}{
"path": restorePath,
"warning": "File restored to original location. Re-scan recommended.",
})
}
// restoreStatusError carries the HTTP status a restore failure answers with.
type restoreStatusError struct {
status int
msg string
}
func (e restoreStatusError) Error() string { return e.msg }
func restoreFailed(status int, format string, args ...any) error {
return restoreStatusError{status: status, msg: fmt.Sprintf(format, args...)}
}
func writeRestoreError(w http.ResponseWriter, err error) {
var statusErr restoreStatusError
if errors.As(err, &statusErr) {
writeJSONError(w, statusErr.msg, statusErr.status)
return
}
writeJSONError(w, err.Error(), http.StatusInternalServerError)
}
// restoreQuarantineFile copies a quarantined regular file to target with
// its recorded owner, mode and modification time. The destination must not
// exist; on failure nothing is left at target and the quarantine copy stays.
func restoreQuarantineFile(itemPath, restorePath string, target *safepath.Target, restoredMode os.FileMode, meta checks.QuarantineMeta) error {
// #nosec G304 -- itemPath comes from resolveQuarantineEntry under the operator-owned quarantine directory.
src, readErr := os.Open(itemPath)
if readErr != nil {
return restoreFailed(http.StatusInternalServerError, "Cannot read quarantined file: %v", readErr)
}
defer src.Close()
// Allocate cleanup space before creating the destination. A copy may
// fail because the filesystem is full, when mkdir can fail as well.
stage, stageName, stageErr := target.Parent.CreatePrivateTemp()
if stageErr != nil {
return restoreFailed(http.StatusInternalServerError, "Cannot stage restored file: %v", stageErr)
}
defer func() {
_ = stage.Close()
_ = target.Parent.RemoveDir(stageName)
}()
const stagedName = "restore"
dst, createErr := stage.OpenFile(stagedName, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
if createErr != nil {
return restoreFailed(http.StatusInternalServerError, "Cannot create restored file: %v", createErr)
}
// Even a failed first stat can be cleaned safely inside this private
// directory; no tenant can substitute a different file under its name.
defer func() { _ = stage.Remove(stagedName) }()
defer dst.Close()
createdInfo, statErr := statQuarantineCreatedFile(dst)
if statErr != nil {
return restoreFailed(http.StatusInternalServerError, "Cannot stat restored file: %v", statErr)
}
// Keep the inode pinned after dst.Close, preventing inode reuse from
// making a foreign file pass cleanup's identity check.
guard, guardErr := stage.OpenFile(stagedName, os.O_RDONLY, 0)
if guardErr != nil {
return restoreFailed(http.StatusInternalServerError, "Cannot pin restored file; quarantine retained")
}
defer guard.Close()
if err := target.Check(); err != nil {
return restoreFailed(http.StatusConflict, "Cannot restore - destination changed during restore")
}
if err := stage.RenameTo(stagedName, target.Parent, target.Name); err != nil {
return restoreFailed(http.StatusConflict, "Cannot restore - file already exists at original path: %v", err)
}
keepDestination := false
defer func() {
if !keepDestination {
if err := discardQuarantineRestore(target, createdInfo, stage, stageName); err != nil {
log.Printf("webui: restore cleanup failed: %v", err)
}
}
}()
if quarantineRestoreAfterCreateForTest != nil {
quarantineRestoreAfterCreateForTest(restorePath)
}
if _, err := ensureOpenFileStillAtTarget(dst, target); err != nil {
_ = src.Close()
return restoreFailed(http.StatusConflict, "Cannot restore - destination changed during restore")
}
_, copyErr := io.Copy(dst, src)
if closeErr := src.Close(); copyErr == nil && closeErr != nil {
copyErr = closeErr
}
if copyErr != nil {
return restoreFailed(http.StatusInternalServerError, "Cannot write restored file: %v", copyErr)
}
if quarantineRestoreBeforeFinalizeForTest != nil {
quarantineRestoreBeforeFinalizeForTest(restorePath)
}
if _, err := ensureOpenFileStillAtTarget(dst, target); err != nil {
return restoreFailed(http.StatusConflict, "Cannot restore - destination changed during restore")
}
if err := dst.Chown(meta.Owner, meta.Group); err != nil {
return restoreFailed(http.StatusInternalServerError, "Cannot restore file ownership; quarantine retained: %v", err)
}
if err := dst.Chmod(restoredMode); err != nil {
return restoreFailed(http.StatusInternalServerError, "Cannot restore file mode: %v", err)
}
if !meta.OriginalModTime.IsZero() {
if err := restoreQuarantineModTime(dst, meta.OriginalModTime); err != nil {
return restoreFailed(http.StatusInternalServerError, "Cannot restore modification time; quarantine retained: %v", err)
}
}
restoredInfo, err := ensureOpenFileStillAtTarget(dst, target)
if err != nil {
return restoreFailed(http.StatusConflict, "Cannot restore - destination changed during restore")
}
if err := syncQuarantineRestoredFile(dst); err != nil {
return restoreFailed(http.StatusInternalServerError, "Restored file could not be synced; quarantine retained: %v", err)
}
if err := dst.Close(); err != nil {
return restoreFailed(http.StatusInternalServerError, "Cannot write restored file: %v", err)
}
if err := ensureTargetStillNamesInfo(target, restoredInfo); err != nil {
return restoreFailed(http.StatusConflict, "Cannot restore - destination changed during restore")
}
if err := syncQuarantineRestoredParent(target.Parent); err != nil {
return restoreFailed(http.StatusInternalServerError, "Restored directory could not be synced; quarantine retained: %v", err)
}
if err := ensureTargetStillNamesInfo(target, restoredInfo); err != nil {
return restoreFailed(http.StatusConflict, "Cannot restore - destination changed during restore")
}
keepDestination = true
return nil
}
// quarantineRestoreAfterCreateForTest lets race tests replace the path
// after O_EXCL creation; nil in production.
var quarantineRestoreAfterCreateForTest func(string)
var quarantineRestoreAfterValidateForTest func(string)
var quarantineRestoreBeforeFinalizeForTest func(string)
var quarantineRestoreBeforeDiscardForTest func()
var syncQuarantineRestoredFile = (*os.File).Sync
var statQuarantineCreatedFile = (*os.File).Stat
var syncQuarantineRestoredParent = (*safepath.Dir).Sync
var removeRestoredQuarantineEvidence = quarantinefs.RemoveEvidence
var restoreQuarantineModTime = safepath.SetModTime
// Isolate the name before testing its inode: checking and then unlinking in
// a tenant-writable parent would allow a replacement between those operations.
// Cleanup uses the pinned parent even if its original pathname was renamed.
func discardQuarantineRestore(target *safepath.Target, want os.FileInfo, stage *safepath.Dir, stageName string) error {
if quarantineRestoreBeforeDiscardForTest != nil {
quarantineRestoreBeforeDiscardForTest()
}
const name = "discard"
if err := target.Parent.RenameTo(target.Name, stage, name); err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
got, statErr := stage.Stat(name)
if statErr != nil || !os.SameFile(want, got) {
if err := stage.RenameTo(name, target.Parent, target.Name); err != nil {
return fmt.Errorf("destination changed; displaced entry retained in %s: %w", stageName, err)
}
return nil
}
if err := stage.Remove(name); err != nil {
return err
}
return target.Parent.Sync()
}
func ensureOpenFileStillAtTarget(f *os.File, target *safepath.Target) (os.FileInfo, error) {
fileInfo, err := f.Stat()
if err != nil {
return nil, fmt.Errorf("cannot stat restored file handle: %w", err)
}
if err := ensureTargetStillNamesInfo(target, fileInfo); err != nil {
return nil, err
}
return fileInfo, nil
}
func ensureTargetStillNamesInfo(target *safepath.Target, fileInfo os.FileInfo) error {
if err := target.Check(); err != nil {
return err
}
pathInfo, err := target.Parent.Stat(target.Name)
if err != nil {
return fmt.Errorf("cannot stat restored file path: %w", err)
}
if !os.SameFile(fileInfo, pathInfo) {
return fmt.Errorf("restore destination changed during restore")
}
return nil
}
func openQuarantineRestoreTarget(path string, roots []string, createParents bool) (*safepath.Target, error) {
var root string
for _, base := range roots {
if isPathUnder(path, base) && len(base) > len(root) {
root = base
}
}
if root == "" {
return nil, fmt.Errorf("restore path is outside the allowed restore roots")
}
relative, err := filepath.Rel(root, path)
if err != nil {
return nil, err
}
return safepath.OpenTarget(root, relative, createParents)
}
func restoreQuarantineDirectory(path string, target *safepath.Target, mode os.FileMode, meta checks.QuarantineMeta) error {
// Quarantine is daemon-owned. Both sides of the rename still use pinned
// parents so destination swaps cannot redirect the transaction.
source, err := safepath.OpenDir(filepath.Dir(path))
if err != nil {
return err
}
defer func() { _ = source.Close() }()
name := filepath.Base(path)
dir, err := source.OpenFile(name, os.O_RDONLY, 0)
if err != nil {
return err
}
defer dir.Close()
info, err := dir.Stat()
if err != nil {
return err
}
if !info.IsDir() {
return fmt.Errorf("quarantine entry is no longer a directory")
}
if err := quarantinefs.SyncTree(path, info); err != nil {
return err
}
if err := dir.Chown(meta.Owner, meta.Group); err != nil {
return err
}
if err := dir.Chmod(mode); err != nil {
return err
}
if !meta.OriginalModTime.IsZero() {
if err := restoreQuarantineModTime(dir, meta.OriginalModTime); err != nil {
return err
}
}
if err := syncQuarantineRestoredFile(dir); err != nil {
return err
}
if err := dir.Close(); err != nil {
return err
}
if err := target.Check(); err != nil {
return err
}
if err := source.RenameTo(name, target.Parent, target.Name); err != nil {
return err
}
if quarantineRestoreAfterDirectoryMoveForTest != nil {
quarantineRestoreAfterDirectoryMoveForTest()
}
if err := ensureTargetStillNamesInfo(target, info); err != nil {
if rollbackErr := rollbackQuarantineDirectory(source, name, target, info); rollbackErr != nil {
return fmt.Errorf("%w; restoring quarantine entry failed: %v", err, rollbackErr)
}
return err
}
if err := syncQuarantineRestoredParent(target.Parent); err != nil {
return fmt.Errorf("directory moved to restore destination but sync failed; quarantine metadata retained: %w", err)
}
if err := source.Sync(); err != nil {
return fmt.Errorf("directory restored but quarantine removal is not durable; metadata retained: %w", err)
}
return ensureTargetStillNamesInfo(target, info)
}
var quarantineRestoreAfterDirectoryMoveForTest func()
// Rollback must identify the isolated inode, not a name a tenant can replace
// between validation and rename. Foreign entries go back without overwriting.
func rollbackQuarantineDirectory(source *safepath.Dir, name string, target *safepath.Target, want os.FileInfo) error {
stage, stageName, err := source.CreatePrivateTemp()
if err != nil {
return err
}
defer func() {
_ = stage.Close()
_ = source.RemoveDir(stageName)
}()
const displaced = "displaced"
if err := target.Parent.RenameTo(target.Name, stage, displaced); err != nil {
return err
}
got, statErr := stage.Stat(displaced)
if statErr != nil || !os.SameFile(want, got) {
if err := stage.RenameTo(displaced, target.Parent, target.Name); err != nil {
return fmt.Errorf("destination changed; displaced entry retained in %s: %w", stageName, err)
}
return fmt.Errorf("destination changed; quarantine directory was moved by another writer")
}
if err := stage.RenameTo(displaced, source, name); err != nil {
return fmt.Errorf("quarantine directory retained in %s: %w", stageName, err)
}
return nil
}
package webui
import (
"bytes"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/quarantinefs"
"github.com/pidginhost/csm/internal/safepath"
)
// Exclusive file creation protects individual destinations, but a message's
// two entries and its progress record must also be handled as one operation.
var spoolRestoreMu sync.Mutex
type spoolRestoreState struct {
Stage string `json:"stage"`
Header checks.QuarantineMeta `json:"header"`
Body *checks.QuarantineMeta `json:"body,omitempty"`
}
func spoolRestoreBase(entry quarantineEntryRef) string {
for _, suffix := range []string{"-H", "-D"} {
if base, ok := strings.CutSuffix(entry.ItemPath, suffix); ok {
return base
}
}
return ""
}
func readSpoolRestoreState(base string) (*spoolRestoreState, error) {
// #nosec G304 -- base comes from resolveQuarantineEntry under the operator-owned quarantine directory.
data, err := os.ReadFile(base + ".restore")
if err != nil {
return nil, err
}
var state spoolRestoreState
if err := json.Unmarshal(data, &state); err != nil {
return nil, err
}
if filepath.Base(state.Stage) != state.Stage || !strings.HasPrefix(state.Stage, ".csm-restore-") {
return nil, fmt.Errorf("invalid mail restore staging directory")
}
return &state, nil
}
// Cleanup can fail after either sidecar has been removed. The durable progress
// record lets either original ID finish cleanup without requeueing the message.
func (s *Server) resumeQuarantinedSpoolMessage(w http.ResponseWriter, r *http.Request, entry quarantineEntryRef) bool {
base := spoolRestoreBase(entry)
if base == "" {
return false
}
spoolRestoreMu.Lock()
defer spoolRestoreMu.Unlock()
state, err := readSpoolRestoreState(base)
if os.IsNotExist(err) {
return false
}
if err != nil {
writeRestoreError(w, err)
} else {
s.finishQuarantinedSpoolMessage(w, r, base, state)
}
return true
}
func (s *Server) restoreQuarantinedSpoolMessage(w http.ResponseWriter, r *http.Request, entry quarantineEntryRef, meta checks.QuarantineMeta) {
spoolRestoreMu.Lock()
defer spoolRestoreMu.Unlock()
spoolDir, suffix, err := checks.QuarantinedSpoolFile(meta)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
base, ok := strings.CutSuffix(entry.ItemPath, suffix)
if !ok {
writeJSONError(w, "Quarantined spool file does not match its entry", http.StatusBadRequest)
return
}
if state, err := readSpoolRestoreState(base); err == nil {
s.finishQuarantinedSpoolMessage(w, r, base, state)
return
} else if !os.IsNotExist(err) {
writeRestoreError(w, err)
return
}
if !quarantineEntryDeletable(entry) {
writeJSONError(w, "Quarantine entry not found", http.StatusNotFound)
return
}
state := &spoolRestoreState{}
for _, want := range []string{"-D", "-H"} {
other, err := readQuarantineMeta(base + want + ".meta")
if errors.Is(err, os.ErrNotExist) && want == "-D" {
// A partial quarantine can leave the original body in the spool.
// An orphaned archive is not proof that the body was restored.
if _, statErr := os.Lstat(base + want); os.IsNotExist(statErr) {
continue
}
}
if err != nil {
writeRestoreError(w, err)
return
}
otherDir, otherSuffix, err := checks.QuarantinedSpoolFile(*other)
if err != nil || otherDir != spoolDir || otherSuffix != want || other.MessageID != meta.MessageID {
writeJSONError(w, "Quarantined header and body belong to different messages", http.StatusConflict)
return
}
if want == "-D" {
state.Body = other
} else {
state.Header = *other
}
}
for _, suffix := range []string{"-D", "-H"} {
if _, err := validateQuarantineRestorePath(filepath.Join(spoolDir, meta.MessageID+suffix), []string{spoolDir}); err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
}
if err := prepareSpoolRestore(base, spoolDir, state); err != nil {
writeRestoreError(w, err)
return
}
s.finishQuarantinedSpoolMessage(w, r, base, state)
}
func prepareSpoolRestore(base, spoolDir string, state *spoolRestoreState) error {
target, targetErr := safepath.OpenTarget(spoolDir, state.Header.MessageID+"-H", false)
if targetErr != nil {
return targetErr
}
defer target.Close()
if err := requireSpoolHeaderAbsent(target); err != nil {
return err
}
if state.Body == nil {
if err := checkRestoredSpoolBody(target.Parent, state.Header.MessageID, ""); err != nil {
return err
}
}
stage, name, stageErr := target.Parent.CreatePrivateTemp()
if stageErr != nil {
return stageErr
}
state.Stage = name
retained := false
defer func() {
if !retained {
_ = stage.Remove("-D")
_ = stage.Remove("-H")
}
_ = stage.Close()
if !retained {
_ = target.Parent.RemoveDir(name)
}
}()
for _, suffix := range []string{"-D", "-H"} {
meta := &state.Header
if suffix == "-D" {
meta = state.Body
}
if meta == nil {
continue
}
info, err := os.Lstat(base + suffix)
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return fmt.Errorf("quarantined spool file is not a regular file")
}
staged, err := safepath.OpenTarget(spoolDir, filepath.Join(name, suffix), false)
if err != nil {
return err
}
mode := os.FileMode(0640)
if len(meta.Mode) >= 10 {
mode = parseModeString(meta.Mode)
}
err = restoreQuarantineFile(base+suffix, meta.OriginalPath, staged, mode, *meta)
staged.Close()
if err != nil {
return err
}
}
if err := target.Parent.Sync(); err != nil {
return err
}
data, encodeErr := json.Marshal(state)
if encodeErr != nil {
return encodeErr
}
writeErr := quarantinefs.WriteExclusive(base+".restore", bytes.NewReader(data), 0600)
// A directory sync failure can leave a valid record. Its staged files
// must survive so the next request can resume it.
_, statErr := os.Lstat(base + ".restore")
retained = statErr == nil
return writeErr
}
func requireSpoolHeaderAbsent(target *safepath.Target) error {
if _, err := target.Parent.Stat(target.Name); !os.IsNotExist(err) {
return restoreFailed(http.StatusConflict, "Cannot restore - mail header already exists or cannot be inspected")
}
return target.Check()
}
func (s *Server) finishQuarantinedSpoolMessage(w http.ResponseWriter, r *http.Request, base string, state *spoolRestoreState) {
if err := publishSpoolRestore(base, state); err != nil {
writeRestoreError(w, err)
return
}
messagePath := strings.TrimSuffix(state.Header.OriginalPath, "-H")
s.auditLog(r, "restore", messagePath, "quarantined mail message returned to the queue")
writeOK(w, map[string]interface{}{
"path": messagePath,
"warning": "Message returned to the mail queue; it is delivered on the next queue run.",
})
}
func publishSpoolRestore(base string, state *spoolRestoreState) error {
spoolDir, suffix, checkErr := checks.QuarantinedSpoolFile(state.Header)
if checkErr != nil || suffix != "-H" {
return fmt.Errorf("invalid mail restore header metadata")
}
if state.Body != nil {
dir, suffix, err := checks.QuarantinedSpoolFile(*state.Body)
if err != nil || dir != spoolDir || suffix != "-D" || state.Body.MessageID != state.Header.MessageID {
return fmt.Errorf("invalid mail restore body metadata")
}
}
target, err := safepath.OpenTarget(spoolDir, state.Header.MessageID+"-H", false)
if err != nil {
return err
}
defer target.Close()
stage, err := target.Parent.Walk(state.Stage)
if os.IsNotExist(err) && spoolRestoreEvidenceRemoved(base, state) {
return removeSpoolRestoreRecord(base)
}
if err != nil {
return err
}
defer func() { _ = stage.Close() }()
// The record and both staged files must be durable before any publication,
// including a retry after an earlier directory sync failed.
if err := quarantinefs.SyncDir(filepath.Dir(base)); err != nil {
return err
}
if _, headerErr := stage.Stat("-H"); headerErr == nil {
if err := requireSpoolHeaderAbsent(target); err != nil {
return err
}
if state.Body != nil {
if _, bodyErr := stage.Stat("-D"); bodyErr == nil {
if err := stage.RenameTo("-D", target.Parent, state.Header.MessageID+"-D"); err != nil {
return restoreFailed(http.StatusConflict, "Cannot publish restored body: %v", err)
}
} else if !os.IsNotExist(bodyErr) {
return bodyErr
}
}
archive := ""
if state.Body != nil {
archive = base + "-D"
}
if err := checkRestoredSpoolBody(target.Parent, state.Header.MessageID, archive); err != nil {
return err
}
if err := syncSpoolPublication(target.Parent, stage); err != nil {
return err
}
if err := stage.RenameTo("-H", target.Parent, target.Name); err != nil {
return restoreFailed(http.StatusConflict, "Cannot publish restored header: %v", err)
}
} else if !os.IsNotExist(headerErr) {
return headerErr
}
// A missing staged header means publication already happened. Exim may
// have consumed the live message; never infer that it needs requeueing.
if err := syncSpoolPublication(target.Parent, stage); err != nil {
return err
}
for _, suffix := range []string{"-D", "-H"} {
if suffix == "-D" && state.Body == nil {
continue
}
if err := removeRestoredQuarantineEvidence(base+suffix, base+suffix+".meta"); err != nil {
return fmt.Errorf("message returned to queue; retry to finish quarantine cleanup: %w", err)
}
}
if err := target.Parent.RemoveDir(state.Stage); err != nil {
return err
}
if err := target.Parent.Sync(); err != nil {
return err
}
return removeSpoolRestoreRecord(base)
}
func syncSpoolPublication(parent, stage *safepath.Dir) error {
return errors.Join(syncQuarantineRestoredParent(parent), stage.Sync())
}
func checkRestoredSpoolBody(parent *safepath.Dir, messageID, archive string) error {
body, err := parent.OpenFile(messageID+"-D", os.O_RDONLY, 0)
if err != nil {
return fmt.Errorf("cannot restore header without its body: %w", err)
}
defer body.Close()
info, err := body.Stat()
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return fmt.Errorf("mail body is not a regular file")
}
if archive != "" {
// #nosec G304 -- archive is the paired body under the operator-owned quarantine directory, derived from resolveQuarantineEntry.
source, err := os.Open(archive)
if err != nil {
return err
}
defer source.Close()
liveHash, savedHash := sha256.New(), sha256.New()
if _, err := io.Copy(liveHash, body); err != nil {
return err
}
if _, err := io.Copy(savedHash, source); err != nil {
return err
}
if !bytes.Equal(liveHash.Sum(nil), savedHash.Sum(nil)) {
return restoreFailed(http.StatusConflict, "Restored mail body changed; quarantine retained")
}
}
return nil
}
func spoolRestoreEvidenceRemoved(base string, state *spoolRestoreState) bool {
for _, suffix := range []string{"-D", "-H"} {
if suffix == "-D" && state.Body == nil {
continue
}
for _, path := range []string{base + suffix, base + suffix + ".meta"} {
if _, err := os.Lstat(path); !os.IsNotExist(err) {
return false
}
}
}
return true
}
func removeSpoolRestoreRecord(base string) error {
if err := os.Remove(base + ".restore"); err != nil {
return err
}
return quarantinefs.SyncDir(filepath.Dir(base))
}
package webui
import (
"encoding/json"
"errors"
"fmt"
"reflect"
"sync"
"time"
)
// apiValue returns a response the way the API sends it:
//
// - every instant is in UTC. Stores stamp times in the host's zone, and
// older records keep whatever zone they were saved in.
// - an empty list is [] and an empty map {}, never null. A nil pointer
// still means "unknown" and stays null; byte slices and raw JSON keep
// their own encoding.
//
// It works on a copy reached through exported fields, pointers, slices,
// arrays, maps and interfaces. The input is never modified: it may share
// slices and pointers with a live store, so every container on the way to
// a change is copied before anything in it changes.
//
// The copy keeps concrete types and value versus pointer placement, so
// encoding/json renders it exactly as it would the original, custom
// marshalers included. Map keys are left alone; two distinct instants must
// not collapse into one key.
var timeType = reflect.TypeFor[time.Time]()
// errResponseTooDeep refuses a value nested past maxResponseDepth, which only
// a cycle reaches.
var errResponseTooDeep = errors.New("response nests too deep to encode")
const maxResponseDepth = 64
// errUnreachableField refuses a response type that embeds an unexported
// struct holding times or lists. encoding/json promotes its fields, but
// reflection cannot set them, so they would go out unconverted.
var errUnreachableField = errors.New("response type embeds an unexported struct apiValue cannot reach")
// normalizeCache records, per type, whether apiValue may change a value of
// it: it can reach a time.Time, a list or a map.
var normalizeCache sync.Map // reflect.Type -> bool
// emptySlices holds one empty slice per type. A slice with no capacity
// shares no storage, so every response can use the same one.
var emptySlices sync.Map // reflect.Type -> reflect.Value
func emptySlice(t reflect.Type) reflect.Value {
if cached, ok := emptySlices.Load(t); ok {
return cached.(reflect.Value)
}
empty := reflect.MakeSlice(t, 0, 0)
emptySlices.Store(t, empty)
return empty
}
func needsNormalizing(t reflect.Type) bool {
if cached, ok := normalizeCache.Load(t); ok {
return cached.(bool)
}
needs := needsNormalizingIn(t, map[reflect.Type]bool{})
normalizeCache.Store(t, needs)
return needs
}
// needsNormalizingIn answers needsNormalizing for one type. A type met again
// during its own walk adds nothing its other fields do not already show, so
// a recursive type such as a process parent chain terminates.
func needsNormalizingIn(t reflect.Type, seen map[reflect.Type]bool) bool {
if t == timeType {
return true
}
if seen[t] {
return false
}
seen[t] = true
switch t.Kind() {
case reflect.Interface, reflect.Map:
// An interface may hold anything; a map may be nil.
return true
case reflect.Slice:
// A byte slice is base64 or raw JSON, where nil has its own meaning.
return t.Elem().Kind() != reflect.Uint8
case reflect.Pointer, reflect.Array:
return needsNormalizingIn(t.Elem(), seen)
case reflect.Struct:
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
if f.Tag.Get("json") == "-" && !hasJSONMarshaler(t) {
continue
}
if (f.IsExported() || f.Anonymous) && needsNormalizingIn(f.Type, seen) {
return true
}
}
}
return false
}
var jsonMarshalerType = reflect.TypeFor[json.Marshaler]()
func hasJSONMarshaler(t reflect.Type) bool {
return t.Implements(jsonMarshalerType) || reflect.PointerTo(t).Implements(jsonMarshalerType)
}
var normalizeStructCache sync.Map // reflect.Type -> []int
// History pages repeat the same finding and process types thousands of
// times. Resolve tags and reachable fields once per type, not per row.
func normalizingFields(t reflect.Type) []int {
if cached, ok := normalizeStructCache.Load(t); ok {
return cached.([]int)
}
custom := hasJSONMarshaler(t)
var fields []int
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
if (!f.IsExported() && !f.Anonymous) || (f.Tag.Get("json") == "-" && !custom) {
continue
}
if needsNormalizing(f.Type) {
fields = append(fields, i)
}
}
normalizeStructCache.Store(t, fields)
return fields
}
func apiValue(v any) (any, error) {
rv := reflect.ValueOf(v)
if !rv.IsValid() || !needsNormalizing(rv.Type()) {
return v, nil
}
out := reflect.New(rv.Type()).Elem()
out.Set(rv)
if err := normalizeInPlace(out, 0); err != nil {
return nil, err
}
return out.Interface(), nil
}
// normalizeInPlace normalizes v, an addressable value the caller owns. A
// pointer, slice, map or interface in v still points at shared data, so each
// is replaced by a copy before anything under it changes.
func normalizeInPlace(v reflect.Value, depth int) error {
t := v.Type()
if t == timeType {
p := v.Addr().Interface().(*time.Time)
*p = p.UTC()
return nil
}
if !needsNormalizing(t) {
return nil
}
if depth > maxResponseDepth {
return errResponseTooDeep
}
switch t.Kind() {
case reflect.Pointer:
if v.IsNil() {
return nil
}
p := reflect.New(t.Elem())
p.Elem().Set(v.Elem())
if err := normalizeInPlace(p.Elem(), depth+1); err != nil {
return err
}
v.Set(p)
case reflect.Interface:
if v.IsNil() {
return nil
}
elem := v.Elem()
if !needsNormalizing(elem.Type()) {
return nil
}
c := reflect.New(elem.Type()).Elem()
c.Set(elem)
if err := normalizeInPlace(c, depth+1); err != nil {
return err
}
v.Set(c)
case reflect.Slice:
if v.IsNil() {
v.Set(emptySlice(t))
return nil
}
if !needsNormalizing(t.Elem()) {
return nil
}
c := reflect.MakeSlice(t, v.Len(), v.Len())
reflect.Copy(c, v)
for i := 0; i < c.Len(); i++ {
if err := normalizeInPlace(c.Index(i), depth+1); err != nil {
return err
}
}
v.Set(c)
case reflect.Array:
// An array is a value; v already holds its own copy.
for i := 0; i < v.Len(); i++ {
if err := normalizeInPlace(v.Index(i), depth+1); err != nil {
return err
}
}
case reflect.Map:
if v.IsNil() {
v.Set(reflect.MakeMap(t))
return nil
}
if !needsNormalizing(t.Elem()) {
return nil
}
c := reflect.MakeMapWithSize(t, v.Len())
iter := v.MapRange()
for iter.Next() {
val := reflect.New(t.Elem()).Elem()
val.Set(iter.Value())
if err := normalizeInPlace(val, depth+1); err != nil {
return err
}
c.SetMapIndex(iter.Key(), val)
}
v.Set(c)
case reflect.Struct:
for _, index := range normalizingFields(t) {
f := v.Field(index)
if !f.CanSet() {
return fmt.Errorf("%w: %s in %s", errUnreachableField, f.Type(), t)
}
if err := normalizeInPlace(f, depth+1); err != nil {
return err
}
}
}
return nil
}
package webui
import (
"bytes"
"context"
"encoding/json"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
const (
uiAuditFile = "ui_audit.jsonl"
maxUIAuditSize = 10 * 1024 * 1024 // 10 MB
)
// UIAuditEntry records a UI action for compliance and accountability.
type UIAuditEntry struct {
Timestamp time.Time `json:"timestamp"`
Action string `json:"action"` // block, unblock, dismiss, fix, whitelist, etc.
Target string `json:"target"` // IP, finding key, file path
Details string `json:"details,omitempty"` // extra context
SourceIP string `json:"source_ip,omitempty"` // admin's IP
// Actor is the name of the credential that acted, and Via says whether
// it came as an API token or a browser login.
Actor string `json:"actor,omitempty"`
Via string `json:"via,omitempty"`
}
// auditLog records a UI action to the audit log, attributed to the
// credential behind r.
func (s *Server) auditLog(r *http.Request, action, target, details string) {
actor, via := s.requestActor(r)
s.auditLogAs(r, actor, via, action, target, details)
}
// auditLogAs records a UI action for an actor resolved by the caller. Login
// has no session yet, and logout or revocation ends the session that made
// the request, so those handlers name the actor themselves.
func (s *Server) auditLogAs(r *http.Request, actor, via, action, target, details string) {
entry := UIAuditEntry{
Timestamp: time.Now(),
Action: action,
Target: target,
Details: details,
SourceIP: extractClientIP(r),
Actor: actor,
Via: via,
}
path := filepath.Join(s.cfg.StatePath, uiAuditFile)
data, err := json.Marshal(entry)
if err != nil {
return
}
data = append(data, '\n')
// Rotation and append are one step: two writers that both saw an
// oversized log would otherwise rotate twice and rename the fresh file
// over the archived history.
s.auditMu.Lock()
defer s.auditMu.Unlock()
if info, statErr := os.Stat(path); statErr == nil && info.Size() > maxUIAuditSize {
if renameErr := os.Rename(path, path+".1"); renameErr != nil {
log.Printf("webui: audit rotation failed for %s: %v", path, renameErr)
}
}
// #nosec G304 -- filepath.Join under operator-configured StatePath.
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
log.Printf("webui: audit open failed for %s: %v", path, err)
return
}
if _, err := f.Write(data); err != nil {
_ = f.Close()
log.Printf("webui: audit write failed for %s: %v", path, err)
return
}
if err := f.Close(); err != nil {
log.Printf("webui: audit close failed for %s: %v", path, err)
}
}
type auditActorKey struct{}
type auditActor struct{ name, via, scope string }
func withAuditActor(r *http.Request, actor, via, scope string) *http.Request {
return r.WithContext(context.WithValue(r.Context(), auditActorKey{}, auditActor{name: actor, via: via, scope: scope}))
}
// requestActor uses the identity captured at authorization so an action
// finishing after logout or expiry keeps its actor. Direct callers resolve
// against startup credentials without refreshing session activity.
func (s *Server) requestActor(r *http.Request) (actor, via string) {
if r == nil {
return "", ""
}
if actor, ok := r.Context().Value(auditActorKey{}).(auditActor); ok {
return actor.name, actor.via
}
// Match authorization's cookie-first order, including credential binding.
if tok, ok := s.cookieSessionCredential(r, "admin", false); ok {
return tok.Name, "browser"
}
if tok, ok := s.bearerCredentialWithScope(r, "admin"); ok {
return tok.Name, "api"
}
return "", ""
}
func extractClientIP(r *http.Request) string {
// Use RemoteAddr directly - XFF is trivially spoofable and this is
// a security audit log, so we only trust the TCP connection source.
return clientIPKey(r.RemoteAddr)
}
// readUIAuditLog returns the last N audit entries, newest first (all when
// limit is 0). It reads the log from the end, so asking for the newest few
// does not parse up to maxUIAuditSize of older entries.
func readUIAuditLog(statePath string, limit int) []UIAuditEntry {
path := filepath.Join(statePath, uiAuditFile)
// #nosec G304 -- filepath.Join under operator-configured statePath.
f, err := os.Open(path)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil {
return nil
}
return tailAuditEntries(f, info.Size(), limit)
}
// tailAuditEntries parses the audit lines in r[0:size] newest first and stops
// once limit entries are found (0 means all). Lines that are blank or not an
// entry are skipped; one line may be as long as the log (bulk undo targets).
func tailAuditEntries(r io.ReaderAt, size int64, limit int) []UIAuditEntry {
var out []UIAuditEntry
done := func(line []byte) bool {
line = bytes.TrimRight(line, "\r")
if len(line) == 0 {
return false
}
var entry UIAuditEntry
if json.Unmarshal(line, &entry) == nil {
out = append(out, entry)
}
return limit > 0 && len(out) >= limit
}
// head holds the bytes before the earliest newline seen so far: the
// unfinished start of a line. Chunks grow with it, so a long line is
// read in a logarithmic number of steps.
var head []byte
end := size
for end > 0 {
n := int64(64 * 1024)
if int64(len(head)) > n {
n = int64(len(head))
}
start := max(end-n, 0)
data := make([]byte, end-start, end-start+int64(len(head)))
if _, err := r.ReadAt(data, start); err != nil && err != io.EOF {
return out
}
data = append(data, head...)
for {
i := bytes.LastIndexByte(data, '\n')
if i < 0 {
break
}
if done(data[i+1:]) {
return out
}
data = data[:i]
}
head = data
end = start
}
done(head)
return out
}
// searchAuditEntries returns audit entries whose target or details contain the search string.
func (s *Server) searchAuditEntries(search string, limit int) []UIAuditEntry {
if search == "" || limit <= 0 {
return nil
}
readLimit := limit * 10
if readLimit > 5000 {
readLimit = 5000
}
all := readUIAuditLog(s.cfg.StatePath, readLimit)
searchLower := strings.ToLower(search)
var matched []UIAuditEntry
for _, e := range all {
if strings.Contains(strings.ToLower(e.Target), searchLower) ||
strings.Contains(strings.ToLower(e.Details), searchLower) ||
strings.Contains(strings.ToLower(e.Action), searchLower) {
matched = append(matched, e)
if len(matched) >= limit {
break
}
}
}
return matched
}
func (s *Server) handleAudit(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "audit.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
// uiAuditPageLimit is how many of the newest UI audit entries the API returns.
const uiAuditPageLimit = 200
// GET /api/v1/audit - return the newest UI audit log entries
func (s *Server) apiUIAudit(w http.ResponseWriter, r *http.Request) {
// One entry past the limit tells whether older entries were left out.
entries := readUIAuditLog(s.cfg.StatePath, uiAuditPageLimit+1)
truncated := len(entries) > uiAuditPageLimit
if truncated {
entries = entries[:uiAuditPageLimit]
}
writeItems(w, entries, map[string]interface{}{"offset": 0, "limit": uiAuditPageLimit, "truncated": truncated})
}
package webui
import (
"crypto/subtle"
"errors"
"log"
"net"
"net/http"
"net/netip"
"strings"
"time"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/session"
)
// --- Authentication ---
// tokenHasScope reports whether the credentials in r grant at least the
// requested scope. "read" is granted by read and admin tokens, "unban" by
// unban and admin tokens, and "admin" only by admin-scope tokens.
// Constant-time compare against every configured token. Browser sessions
// bind to the named administrator credential that created them; the API
// credential itself is never accepted from a cookie.
func (s *Server) tokenHasScope(r *http.Request, want string) bool {
// Browser cookie session
if _, ok := s.cookieTokenWithScope(r, want); ok {
return true
}
// Bearer token
_, ok := s.bearerTokenWithScope(r, want)
return ok
}
func (s *Server) cookieTokenWithScope(r *http.Request, want string) (string, bool) {
return s.cookieSessionToken(r, want, sessionActivity(r))
}
// sessionActivity reports whether a request is the operator's own activity,
// which extends an idle browser session: a page load, or an API call the UI
// marks with X-CSM-Active because it followed the operator's input. Timer
// polls and the event stream do not, so a page left open still reaches the
// idle timeout.
func sessionActivity(r *http.Request) bool {
if r.URL.Path == "/metrics" || r.URL.Path == "/api/v1/events" {
return false
}
if strings.HasPrefix(r.URL.Path, "/api/") {
return r.Header.Get("X-CSM-Active") == "1"
}
if r.Method != http.MethodGet {
return false
}
// Fetches can poll HTML pages too. Only navigation counts as a page
// load; older browsers identify it by Accept instead of Fetch Metadata.
if mode := r.Header.Get("Sec-Fetch-Mode"); mode != "" {
return mode == "navigate"
}
return strings.Contains(r.Header.Get("Accept"), "text/html")
}
func (s *Server) cookieSessionToken(r *http.Request, want string, touch bool) (string, bool) {
tok, ok := s.cookieSessionCredential(r, want, touch)
return tok.Token, ok
}
func (s *Server) cookieSessionCredential(r *http.Request, want string, touch bool) (config.WebUIToken, bool) {
c, err := r.Cookie("csm_auth")
if err != nil {
return config.WebUIToken{}, false
}
// Check the source before touching, so a cookie used from outside the
// credential's addresses cannot keep its session from idling out.
tok, ok := s.sessionCredential(c.Value, want, false)
if !ok || !tok.AcceptsFrom(requestAddr(r)) {
return config.WebUIToken{}, false
}
if touch {
return s.sessionCredential(c.Value, want, true)
}
return tok, true
}
// requestAddr is the TCP peer. Credential decisions never trust forwarded
// headers.
func requestAddr(r *http.Request) netip.Addr {
addr, _ := netip.ParseAddr(clientIPKey(r.RemoteAddr))
return addr
}
func (s *Server) sessionCredential(secret, want string, touch bool) (config.WebUIToken, bool) {
if s.sessions == nil {
return config.WebUIToken{}, false
}
rec, err := s.sessions.Access(secret, s.sessionNow(), touch)
if err != nil {
return config.WebUIToken{}, false
}
for _, tok := range s.cfg.WebUI.Tokens {
if tok.Name == rec.Name && session.Hash(tok.Token) == rec.Credential && tok.Scope == "admin" && webUITokenAllows(tok, want, s.sessionNow()) {
return tok, true
}
}
// A removed, rotated or downgraded login credential cannot leave a
// browser session active, even if a caller changes config in place.
_ = s.sessions.Revoke(rec.ID)
return config.WebUIToken{}, false
}
func (s *Server) bearerTokenWithScope(r *http.Request, want string) (string, bool) {
tok, ok := s.bearerCredentialWithScope(r, want)
return tok.Token, ok
}
func (s *Server) bearerCredentialWithScope(r *http.Request, want string) (config.WebUIToken, bool) {
auth := r.Header.Get("Authorization")
if !strings.HasPrefix(auth, "Bearer ") {
return config.WebUIToken{}, false
}
supplied := strings.TrimPrefix(auth, "Bearer ")
if supplied == "" {
return config.WebUIToken{}, false
}
for _, tok := range s.cfg.WebUI.Tokens {
if webUITokenMatches(supplied, tok) && webUITokenAllows(tok, want, s.sessionNow()) && tok.AcceptsFrom(requestAddr(r)) {
return tok, true
}
}
return config.WebUIToken{}, false
}
func webUITokenMatches(supplied string, tok config.WebUIToken) bool {
return supplied != "" &&
tok.Token != "" &&
subtle.ConstantTimeCompare([]byte(supplied), []byte(tok.Token)) == 1
}
func webUITokenAllows(tok config.WebUIToken, want string, now time.Time) bool {
if !config.WebUITokenCurrent(tok.IssuedAt, tok.ExpiresAt, now) {
return false
}
switch want {
case "read":
return tok.Scope == "read" || tok.Scope == "admin"
case "unban":
return tok.Scope == "unban" || tok.Scope == "admin"
case "admin":
return tok.Scope == "admin"
default:
return false
}
}
// credentialAttemptsPerMinute bounds credential guesses per client. Login
// form submissions and failed bearer tokens share it: both guess the same
// secrets, and the API budget of hundreds of requests a minute would let a
// token be guessed at API speed.
const credentialAttemptsPerMinute = 5
// recentCredentialAttempts returns the client's attempts inside the last
// minute. The caller holds loginMu.
func (s *Server) recentCredentialAttempts(ip string, now time.Time) []time.Time {
var recent []time.Time
for _, t := range s.loginAttempts[ip] {
if now.Sub(t) < time.Minute {
recent = append(recent, t)
}
}
return recent
}
// takeCredentialAttempt records one attempt and reports whether the client
// was still inside its budget. Check and record share one critical section so
// concurrent guesses cannot overshoot it.
func (s *Server) takeCredentialAttempt(ip string) bool {
s.loginMu.Lock()
defer s.loginMu.Unlock()
now := time.Now()
recent := s.recentCredentialAttempts(ip, now)
if len(recent) >= credentialAttemptsPerMinute {
return false
}
if _, tracked := s.loginAttempts[ip]; !tracked {
boundRateLimitMap(s.loginAttempts, now.Add(-time.Minute))
}
s.loginAttempts[ip] = append(recent, now)
return true
}
func carriesBearer(r *http.Request) bool {
return strings.HasPrefix(r.Header.Get("Authorization"), "Bearer ")
}
// knownWebUIBearer recognizes UI and metrics credentials for guess
// accounting only. Each endpoint must still enforce its own scope.
func (s *Server) knownWebUIBearer(r *http.Request) bool {
_, known := s.bearerCredentialWithScope(r, "read")
if !known {
_, known = s.bearerCredentialWithScope(r, "unban")
}
return known || s.metricsBearerMatches(r)
}
// refuseSpentBearer answers a request that presents a bearer token from a
// client with no attempts left. It runs before the token is compared, so a
// blocked client learns nothing from the response, right token or wrong.
// The budget check, credential comparison and failed-attempt charge share a
// critical section; otherwise concurrent requests can all pass the check.
func (s *Server) refuseSpentBearer(w http.ResponseWriter, r *http.Request) bool {
if !carriesBearer(r) {
return false
}
ip := rateLimitKey(r.RemoteAddr)
s.loginMu.Lock()
now := time.Now()
recent := s.recentCredentialAttempts(ip, now)
spent := len(recent) >= credentialAttemptsPerMinute
if !spent {
known := s.knownWebUIBearer(r)
if !known {
_, known = mcpTokenFor(s.cfg.MCP.Tokens, strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "))
}
// A valid credential outside its scope is not a guess, including
// the independently rotatable metrics and MCP credentials.
if !known {
if _, tracked := s.loginAttempts[ip]; !tracked {
boundRateLimitMap(s.loginAttempts, now.Add(-time.Minute))
}
s.loginAttempts[ip] = append(recent, now)
}
}
s.loginMu.Unlock()
if !spent {
return false
}
if strings.HasPrefix(r.URL.Path, "/api/") {
writeJSONError(w, "Too many failed token attempts", http.StatusTooManyRequests)
} else {
http.Error(w, "Too many failed token attempts", http.StatusTooManyRequests)
}
return true
}
func (s *Server) requireAuth(next http.Handler) http.Handler {
return s.requireScope("admin", next)
}
// requireScope admits browser sessions, which are always admin, and bearer
// tokens that grant want.
func (s *Server) requireScope(want string, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if s.refuseSpentBearer(w, r) {
return
}
if tok, ok := s.cookieSessionCredential(r, want, sessionActivity(r)); ok {
next.ServeHTTP(w, withAuditActor(r, tok.Name, "browser", tok.Scope))
return
}
if tok, ok := s.bearerCredentialWithScope(r, want); ok {
next.ServeHTTP(w, withAuditActor(r, tok.Name, "api", tok.Scope))
return
}
// API calls get 401 JSON; browser requests get redirect to login
if strings.HasPrefix(r.URL.Path, "/api/") {
writeJSONError(w, "Unauthorized", http.StatusUnauthorized)
return
}
http.Redirect(w, r, "/login", http.StatusFound)
})
}
func (s *Server) requireRead(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if s.refuseSpentBearer(w, r) {
return
}
if s.tokenHasScope(r, "read") {
if r.Method != http.MethodGet {
w.Header().Set("Allow", http.MethodGet)
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
next.ServeHTTP(w, r)
return
}
// API calls get 401 JSON; browser requests get redirect to login
if strings.HasPrefix(r.URL.Path, "/api/") {
writeJSONError(w, "Unauthorized", http.StatusUnauthorized)
return
}
http.Redirect(w, r, "/login", http.StatusFound)
})
}
// adminCredential uses the scope captured at authorization so expiry while
// reading a request body cannot change the release kind. Direct callers
// resolve credentials without extending the session.
func (s *Server) adminCredential(r *http.Request) bool {
if actor, ok := r.Context().Value(auditActorKey{}).(auditActor); ok {
return actor.scope == "admin"
}
if _, ok := s.cookieSessionCredential(r, "admin", false); ok {
return true
}
_, ok := s.bearerCredentialWithScope(r, "admin")
return ok
}
// isAuthenticated is a thin shim used by handleLogin and metrics_api.
// New callers should prefer tokenHasScope directly.
func (s *Server) isAuthenticated(r *http.Request) bool {
return s.tokenHasScope(r, "admin")
}
// clientIPKey strips the port from a net/http RemoteAddr for use as a
// per-client rate-limit key, handling bracketed IPv6 ([::1]:443 -> ::1).
// Falls back to the raw value when there is no host:port to split, so a
// missing port never collapses distinct clients onto one key.
func clientIPKey(remoteAddr string) string {
if host, _, err := net.SplitHostPort(remoteAddr); err == nil {
return host
}
return remoteAddr
}
// rateLimitKey is the client a rate limit counts: the IPv4 address, or the
// /64 of an IPv6 address, since one IPv6 client is routed a whole /64 and can
// rotate addresses inside it.
func rateLimitKey(remoteAddr string) string {
host := clientIPKey(remoteAddr)
ip, err := netip.ParseAddr(host)
if err != nil {
return host
}
ip = ip.WithZone("").Unmap()
if ip.Is4() {
return ip.String()
}
return netip.PrefixFrom(ip, 64).Masked().String()
}
func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet && s.refuseSpentBearer(w, r) {
return
}
// Redirect already-authenticated users to dashboard
if r.Method == http.MethodGet && s.isAuthenticated(r) {
http.Redirect(w, r, "/dashboard", http.StatusFound)
return
}
if r.Method == http.MethodGet {
s.renderTemplate(w, r, "login.html", nil)
return
}
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
// One credential budget per client (IPv4 address or IPv6 /64), shared
// with failed API tokens.
if !s.takeCredentialAttempt(rateLimitKey(r.RemoteAddr)) {
http.Error(w, "Too many login attempts", http.StatusTooManyRequests)
return
}
r.Body = http.MaxBytesReader(w, r.Body, 4096)
if err := r.ParseForm(); err != nil {
http.Error(w, "Invalid login request", http.StatusBadRequest)
return
}
token := r.PostForm.Get("token")
// Only admin-scope tokens may log in via the browser form.
var loginName string
if token != "" {
for _, tok := range s.cfg.WebUI.Tokens {
if webUITokenMatches(token, tok) && webUITokenAllows(tok, "admin", s.sessionNow()) && tok.AcceptsFrom(requestAddr(r)) {
loginName = tok.Name
break
}
}
}
if loginName == "" {
// Failed logins go to the daemon log, not the UI audit trail: they
// are unauthenticated, and addresses from a whole network could
// otherwise rotate operator history out of the audit file.
// #nosec G706 -- RemoteAddr is the TCP peer address net/http sets, not request content.
log.Printf("webui: failed browser login from %s", safeLogString(clientIPKey(r.RemoteAddr)))
s.renderTemplate(w, r, "login.html", map[string]string{"Error": "Invalid token"})
return
}
if s.sessions == nil {
http.Error(w, "Session store unavailable", http.StatusServiceUnavailable)
return
}
previous := ""
if c, err := r.Cookie("csm_auth"); err == nil {
// Reauthentication already proved the new login credential. Look up
// the old session without touching it, and retain storage errors so
// a failed lookup cannot silently turn rotation into a fresh login.
_, err := s.sessions.Access(c.Value, s.sessionNow(), false)
if err != nil && !errors.Is(err, session.ErrInvalid) {
http.Error(w, "Session store unavailable", http.StatusServiceUnavailable)
return
}
if err == nil {
previous = c.Value
}
}
secret, record, err := s.sessions.Create(loginName, session.Hash(token), previous, clientIPKey(r.RemoteAddr), r.UserAgent(), s.sessionNow())
if errors.Is(err, session.ErrFull) {
// The Sessions page needs a login, so name the recovery paths here.
http.Error(w, "Browser session limit reached. Wait for idle sessions to expire, or revoke sessions with an admin API token.", http.StatusServiceUnavailable)
return
}
if err != nil {
http.Error(w, "Cannot create browser session", http.StatusServiceUnavailable)
return
}
http.SetCookie(w, &http.Cookie{
Name: "csm_auth",
Value: secret,
Path: "/",
HttpOnly: true,
Secure: true,
SameSite: http.SameSiteStrictMode,
MaxAge: int(record.Expires.Sub(record.Created).Seconds()),
Expires: record.Expires,
})
s.auditLogAs(r, loginName, "browser", "login", loginName, "browser session started")
http.Redirect(w, r, "/dashboard", http.StatusFound)
}
// --- Logout ---
func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if c, err := r.Cookie("csm_auth"); err == nil && s.sessions != nil {
rec, err := s.sessions.Access(c.Value, s.sessionNow(), false)
if err != nil && !errors.Is(err, session.ErrInvalid) {
http.Error(w, "Cannot revoke browser session", http.StatusServiceUnavailable)
return
}
if err == nil {
if err = s.sessions.Revoke(rec.ID); err != nil {
http.Error(w, "Cannot revoke browser session", http.StatusServiceUnavailable)
return
}
s.auditLogAs(r, rec.Name, "browser", "logout", rec.Name, "browser session ended")
}
}
clearBrowserCookie(w)
http.Redirect(w, r, "/login", http.StatusFound)
}
func clearBrowserCookie(w http.ResponseWriter) {
http.SetCookie(w, &http.Cookie{
Name: "csm_auth",
Value: "",
Path: "/",
HttpOnly: true,
Secure: true,
SameSite: http.SameSiteStrictMode,
MaxAge: -1, // delete cookie
})
}
package webui
import (
"fmt"
"log"
"net"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/store"
)
func (s *Server) blockIPPreservingLifetime(ip, reason string, ttl time.Duration) error {
if ttl > 0 {
if guarded, ok := s.blocker.(lifetimeKeepingBlocker); ok {
err := guarded.BlockIPForcePreserveLifetime(ip, reason, ttl)
checks.ObserveOperatorBlock(err, checks.BlockSourceWebUI)
return err
}
// Test blockers and integrations without the engine extension still
// honor the persisted lifetime. The live engine checks under its lock.
blocks, err := snapshotFirewallBlocks(s.cfg.StatePath)
if err != nil {
return err
}
if entry, ok := blocks[ip]; ok {
if entry.ExpiresAt.IsZero() {
return firewall.ErrPermanentBlock
}
if time.Until(entry.ExpiresAt) > ttl {
return firewall.ErrLongerBlock
}
}
}
return blockIPForOperator(s.blocker, ip, reason, ttl)
}
func snapshotFirewallBlocks(statePath string) (map[string]firewall.BlockedEntry, error) {
state, err := firewall.LoadState(statePath)
if err != nil {
return nil, err
}
blocks := make(map[string]firewall.BlockedEntry, len(state.Blocked))
for _, entry := range state.Blocked {
if ip := net.ParseIP(entry.IP); ip != nil {
entry.IP = ip.String()
blocks[entry.IP] = entry
}
}
return blocks, nil
}
func invalidateIPUndo(ip string) {
if db := store.Global(); db != nil {
if err := db.InvalidateUndoTargets([]string{ip}); err != nil {
log.Printf("webui: invalidate IP undo: %v", err)
}
}
}
func threatRowsForIP(rows []undoThreatRow, ip string) []undoThreatRow {
for _, row := range rows {
if row.IP == ip {
return []undoThreatRow{row}
}
}
return nil
}
// undoSnapshotBlocks restores each original deadline, never a fresh 24h
// window. Later firewall decisions invalidate the snapshot even if they
// originated outside the Web UI and did not touch the threat database.
func (s *Server) undoSnapshotBlocks(payload undoPayloadIPs, clearEvidence bool) (int, error) {
if s.blocker == nil {
return 0, fmt.Errorf("firewall engine not available")
}
current, err := snapshotFirewallBlocks(s.cfg.StatePath)
if err != nil {
return 0, err
}
for _, ip := range payload.IPs {
got, exists := current[ip]
want, expected := payload.ExpectedBlocks[ip]
if exists != expected || (exists && !firewall.SameBlockedEntry(got, want)) {
return 0, firewall.ErrBlockChanged
}
}
count := 0
for _, ip := range payload.IPs {
if _, err := parseAndValidateIP(ip); err != nil {
continue
}
prior, hadPrior := payload.RestoreBlocks[ip]
ttl := time.Duration(0)
if hadPrior && !prior.ExpiresAt.IsZero() {
ttl = time.Until(prior.ExpiresAt)
if ttl <= 0 {
hadPrior = false
}
}
if !hadPrior && !clearEvidence {
continue
}
if restorer, ok := s.blocker.(blockRestorer); ok {
var expected, restore *firewall.BlockedEntry
if entry, exists := payload.ExpectedBlocks[ip]; exists {
expected = &entry
}
if hadPrior {
restore = &prior
}
err := restorer.RestoreBlockIfUnchanged(ip, expected, restore)
if hadPrior {
checks.ObserveOperatorBlock(err, checks.BlockSourceWebUI)
}
if err != nil {
continue
}
} else if hadPrior {
if err := blockIPForOperator(s.blocker, ip, prior.Reason, ttl); err != nil {
continue
}
} else if err := s.blocker.UnblockIP(ip); err != nil {
continue
}
if clearEvidence {
if tdb := checks.GetThreatDB(); tdb != nil {
tdb.RemovePermanent(ip)
}
}
restoreUndoThreatRows(threatRowsForIP(payload.RestoreThreats, ip))
if clearEvidence && !hadPrior {
_ = flushCphulk(ip) // best effort
}
count++
}
return count, nil
}
func (s *Server) blockIPForUndo(ip, reason string, ttl time.Duration) (*firewall.BlockedEntry, *firewall.BlockedEntry, error) {
if blocker, ok := s.blocker.(undoableBlocker); ok {
before, after, err := blocker.BlockIPForUndo(ip, reason, ttl)
checks.ObserveOperatorBlock(err, checks.BlockSourceWebUI)
return before, after, err
}
before, err := snapshotFirewallBlocks(s.cfg.StatePath)
if err != nil {
return nil, nil, err
}
if err = s.blockIPPreservingLifetime(ip, reason, ttl); err != nil {
return nil, nil, err
}
after, err := snapshotFirewallBlocks(s.cfg.StatePath)
return blockPointer(before, ip), blockPointer(after, ip), err
}
func (s *Server) unblockIPForUndo(ip string) (*firewall.BlockedEntry, error) {
if blocker, ok := s.blocker.(undoableUnblocker); ok {
return blocker.UnblockIPForUndo(ip)
}
before, err := snapshotFirewallBlocks(s.cfg.StatePath)
if err != nil {
return nil, err
}
if err := s.blocker.UnblockIP(ip); err != nil {
return nil, err
}
return blockPointer(before, ip), nil
}
func blockPointer(blocks map[string]firewall.BlockedEntry, ip string) *firewall.BlockedEntry {
if entry, exists := blocks[ip]; exists {
return &entry
}
return nil
}
package webui
import (
"net/http"
"github.com/pidginhost/csm/internal/checks"
)
// apiChallengeStats returns challenge-routing activity for the web UI: the live
// pending count, how many challenge timeouts escalated to a hard block, the
// cumulative routes per source check since daemon start, and the most recent
// routes. Read-only; safe for read-scope tokens.
func (s *Server) apiChallengeStats(w http.ResponseWriter, _ *http.Request) {
stats := checks.ChallengeUIStats()
resp := map[string]interface{}{
"routed_by_check": stats.RoutedByCheck,
"recent": stats.Recent,
"pending": 0,
"escalated": 0,
"passed": 0,
}
if s.provider != nil {
automation := s.provider.AutomationStatus()
resp["pending"] = automation.ChallengePending
resp["escalated"] = automation.ChallengeEscalated
resp["passed"] = automation.ChallengePassed
}
writeJSON(w, resp)
}
package webui
import (
"net/http"
"sort"
"time"
"github.com/pidginhost/csm/internal/health"
)
// componentsProvider is the optional capability surface the daemon
// exposes for /api/v1/components. Tests and the API-only fallback path
// can omit it; the handler degrades to attached/unknown.
type componentsProvider interface {
WatcherStatuses() map[string]bool
WatcherChangedAt() map[string]time.Time
}
// componentsUpstreamProvider is the optional capability surface the
// daemon adds when it has per-watcher upstream probes wired. Returning
// nil / absent for a watcher means "no probe, do not flag deaf".
type componentsUpstreamProvider interface {
WatcherUpstream() map[string]health.UpstreamResult
}
// componentRow is the JSON shape returned per watcher.
type componentRow struct {
Name string `json:"name"`
Label string `json:"label"`
Status string `json:"status"` // "ok" | "degraded" | "deaf" | "idle" | "unknown"
Attached bool `json:"attached"`
ChangedAt time.Time `json:"changed_at,omitzero"`
LastEventAt time.Time `json:"last_event_at,omitzero"`
LastEventCheck string `json:"last_event_check,omitempty"`
UpstreamFresh *bool `json:"upstream_fresh,omitempty"`
UpstreamReason string `json:"upstream_reason,omitempty"`
UpstreamSeenAt time.Time `json:"upstream_seen_at,omitzero"`
}
// componentLabels maps the short watcher name to the operator-facing label.
// Watchers not in the map render with their raw key.
var componentLabels = map[string]string{
"fanotify": "Fanotify (filesystem)",
"audit": "Auditd",
"modsec": "ModSecurity audit",
"afalg": "AF_ALG kernel monitor",
"phprelay": "PHP relay watcher",
"maillog": "Mail log",
"stalwart_log": "Stalwart log",
"email_av_spool": "Email AV spool",
"forwarder": "Forwarder watcher",
"pamlistener": "PAM listener",
"php_shield": "PHP Shield event log",
"connection": "Connection tracker",
"exec": "Exec monitor",
"sensitive": "Sensitive file monitor",
"accesslog": "Access log",
"dovecot_log": "Dovecot log",
"exim_mainlog": "Exim mainlog",
"cpanel_access_log": "cPanel access log",
"yara_worker": "YARA-X worker",
}
// componentCheckOrigin maps a finding Check name back to the watcher that
// emits it. Only the entries with a clear single-source origin are listed;
// finding names reused by periodic or retroactive scans intentionally have
// no entry so they do not advance a watcher's "last event" clock.
var componentCheckOrigin = map[string]string{
"cgi_backdoor_realtime": "fanotify",
"cgi_suspicious_location_realtime": "fanotify",
"credential_log_realtime": "fanotify",
"email_auth_failure_realtime": "maillog",
"email_av_degraded": "email_av_spool",
"email_av_encrypted_archive": "email_av_spool",
"email_av_isolation_unavailable": "email_av_spool",
"email_av_parse_error": "email_av_spool",
"email_av_parse_worker_failed": "email_av_spool",
"email_av_parse_worker_rejected": "email_av_spool",
"email_av_parse_worker_source": "email_av_spool",
"email_av_quarantine_error": "email_av_spool",
"email_av_source_backlog": "email_av_spool",
"email_av_timeout": "email_av_spool",
"email_compromised_account": "maillog",
"email_credential_leak": "maillog",
"email_dkim_failure": "maillog",
"email_malware": "email_av_spool",
"email_php_form_relay": "phprelay",
"email_php_form_relay_observed": "phprelay",
"email_php_form_relay_script": "phprelay",
"email_php_form_relay_table_full": "phprelay",
"email_php_relay_action_dry_run": "phprelay",
"email_php_relay_action_failed": "phprelay",
"email_php_relay_action_skipped": "phprelay",
"email_php_relay_abuse": "phprelay",
"email_php_relay_account_volume_capped": "phprelay",
"email_php_relay_cpanel_limit_unreadable": "phprelay",
"email_php_relay_disabled": "phprelay",
"email_php_relay_inotify_overflow": "phprelay",
"email_php_relay_inotify_overflow_recovered": "phprelay",
"email_php_relay_msgindex_persist_failed": "phprelay",
"email_php_relay_no_exim": "phprelay",
"email_php_relay_overflow_scan_truncated": "phprelay",
"email_php_relay_path2b_disabled": "phprelay",
"email_php_relay_policies_reload": "phprelay",
"email_php_relay_rate_limit_hit": "phprelay",
"email_php_relay_sweep_failed": "phprelay",
"email_php_relay_watcher_failed": "phprelay",
"email_defer_fail_governor": "maillog",
"email_rate_critical": "maillog",
"email_rate_warning": "maillog",
"email_spam_outbreak": "maillog",
"email_spf_rejection": "maillog",
"executable_in_config_realtime": "fanotify",
"executable_in_tmp_realtime": "fanotify",
"exim_frozen_realtime": "maillog",
"fanotify_overflow": "fanotify",
"htaccess_injection_realtime": "fanotify",
"removed_file_restored_realtime": "fanotify",
"self_deleting_dropper_realtime": "fanotify",
"self_deleting_dropper_overflow": "fanotify",
"mail_account_compromised": "maillog",
"mail_account_spray": "maillog",
"mail_auth_backend_degraded": "maillog",
"mail_bruteforce": "maillog",
"mail_bruteforce_suspected": "maillog",
"mail_log_source_unavailable": "maillog",
"mail_subnet_spray": "maillog",
"modsec_block_escalation": "modsec",
"modsec_block_realtime": "modsec",
"modsec_classifier_gap": "modsec",
"modsec_csm_block_escalation": "modsec",
"modsec_low_confidence_burst": "modsec",
"modsec_warning_realtime": "modsec",
"obfuscated_php_realtime": "fanotify",
"credential_stuffing": "pamlistener",
"pam_bruteforce": "pamlistener",
"pam_login": "pamlistener",
"phishing_kit_realtime": "fanotify",
"phishing_realtime": "fanotify",
"php_shield_block": "php_shield",
"php_shield_eval": "php_shield",
"php_shield_webshell": "php_shield",
"php_config_realtime": "fanotify",
"php_dropper_realtime": "fanotify",
"php_in_sensitive_dir_realtime": "fanotify",
"php_in_uploads_realtime": "fanotify",
"signature_match_realtime": "fanotify",
"smtp_account_spray": "maillog",
"smtp_bruteforce": "maillog",
"smtp_probe_abuse": "maillog",
"smtp_subnet_spray": "maillog",
"stalwart_ban": "stalwart_log",
"stalwart_log_degraded": "stalwart_log",
"stalwart_webmail_bruteforce": "stalwart_log",
"webshell_content_realtime": "fanotify",
"webshell_realtime": "fanotify",
"yara_match_realtime": "fanotify",
"yara_match_scheduled": "scheduled",
"yara_realtime_scan_error": "fanotify",
}
// apiComponents returns one row per registered watcher with its live
// state, time since last state change, and the most recent finding it
// emitted within a 7-day lookback. Drives the dashboard component
// matrix.
func (s *Server) apiComponents(w http.ResponseWriter, _ *http.Request) {
cp, _ := s.provider.(componentsProvider)
if cp == nil {
writeAll(w, []componentRow{})
return
}
statuses := cp.WatcherStatuses()
changed := cp.WatcherChangedAt()
lastEvents := s.lastEventByWatcher(7 * 24 * time.Hour)
var upstream map[string]health.UpstreamResult
if up, ok := s.provider.(componentsUpstreamProvider); ok {
upstream = up.WatcherUpstream()
}
rows := make([]componentRow, 0, len(statuses))
for name, attached := range statuses {
row := componentRow{
Name: name,
Label: componentLabel(name),
Attached: attached,
}
if t, ok := changed[name]; ok {
row.ChangedAt = t.UTC()
}
if ev, ok := lastEvents[name]; ok && !ev.at.IsZero() {
row.LastEventAt = ev.at.UTC()
row.LastEventCheck = ev.check
}
var upstreamFresh *bool
if up, ok := upstream[name]; ok {
fresh := up.Fresh
upstreamFresh = &fresh
row.UpstreamFresh = upstreamFresh
row.UpstreamReason = up.Reason
row.UpstreamSeenAt = up.LastActivity.UTC()
}
row.Status = componentStatus(attached, lastEvents[name].at, upstreamFresh)
rows = append(rows, row)
}
sort.Slice(rows, func(i, j int) bool {
if rows[i].Status != rows[j].Status {
return componentStatusRank(rows[i].Status) < componentStatusRank(rows[j].Status)
}
return rows[i].Label < rows[j].Label
})
writeAll(w, rows)
}
type watcherEvent struct {
at time.Time
check string
}
// lastEventByWatcher returns the most recent finding per known watcher key
// within the lookback window. It reads the store's per-check index of newest
// timestamps rather than decoding the window's history on every poll.
// Checks not in componentCheckOrigin are skipped so periodic-scan output
// does not get attributed to a real-time watcher.
func (s *Server) lastEventByWatcher(window time.Duration) map[string]watcherEvent {
out := map[string]watcherEvent{}
if s.store == nil {
return out
}
since := time.Now().Add(-window)
for check, at := range s.store.LatestByCheck() {
watcher, ok := componentCheckOrigin[check]
if !ok || at.Before(since) {
continue
}
// Map order is random; equal times resolve by check name.
if cur, exists := out[watcher]; exists && (cur.at.After(at) || cur.at.Equal(at) && cur.check < check) {
continue
}
out[watcher] = watcherEvent{at: at, check: check}
}
// Also fold in the latest scan set so freshly-emitted findings appear
// before they have rolled into history.
for _, f := range s.store.LatestFindings() {
if f.Timestamp.Before(since) {
continue
}
watcher, ok := componentCheckOrigin[f.Check]
if !ok {
continue
}
if cur, exists := out[watcher]; exists && !cur.at.Before(f.Timestamp) {
continue
}
out[watcher] = watcherEvent{at: f.Timestamp, check: f.Check}
}
return out
}
func componentLabel(name string) string {
if l, ok := componentLabels[name]; ok {
return l
}
return name
}
// componentStatus collapses the per-row state into a UI bucket.
// - degraded: watcher detached (attempted setup, failed or fell off)
// - deaf: attached but the upstream feeding it has gone silent
// (probe registered, returned Fresh=false). Operator action needed
// before this watcher will ever produce events again.
// - ok: attached AND has produced at least one event recently
// - idle: attached, no events in window, and either no probe is
// wired or the probe still confirms the upstream is alive
// - unknown: not attached and no record either way (reserved)
func componentStatus(attached bool, lastEvent time.Time, upstreamFresh *bool) string {
if !attached {
return "degraded"
}
if upstreamFresh != nil && !*upstreamFresh {
return "deaf"
}
if lastEvent.IsZero() {
return "idle"
}
return "ok"
}
func componentStatusRank(status string) int {
switch status {
case "degraded":
return 0
case "deaf":
return 1
case "idle":
return 2
case "ok":
return 3
default:
return 4
}
}
package webui
import (
"net/http"
"sort"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/store"
)
// Cleanup-history handlers for the db_object_backups bbolt bucket.
// htaccess pre_clean backups already surface through the existing
// /api/v1/quarantine listing (their .meta sidecars now match the
// JSON QuarantineMeta shape). The bbolt-backed db_object_backups
// bucket needs its own list + restore endpoints because the data
// lives outside the filesystem-quarantine flow.
//
// All handlers are registered behind requireAuth in server.go;
// the restore handler additionally requires CSRF.
// dbObjectBackupEntry is the JSON shape returned to the cleanup-
// history UI. The Key field is opaque to the UI -- it round-trips
// to the restore endpoint as-is so the lookup is a single bbolt
// Get, not a multi-field reconstruction.
type dbObjectBackupEntry struct {
Key string `json:"key"`
Account string `json:"account"`
Schema string `json:"schema"`
Kind string `json:"kind"`
Name string `json:"name"`
DroppedAt time.Time `json:"dropped_at"`
DroppedBy string `json:"dropped_by"`
FindingID string `json:"finding_id,omitempty"`
BodyBytes int `json:"body_bytes"` // length of CreateSQL; surfaced for size hint
RestoredAt time.Time `json:"restored_at,omitzero"`
Restored bool `json:"restored"`
}
const dbObjectBackupPreviewBytes = 8 * 1024
// apiDBObjectBackups returns every record in the bucket, newest
// first by DroppedAt. The full CreateSQL is intentionally NOT
// returned in the listing -- those payloads can be large and the
// listing is meant for browse-and-pick. The preview endpoint returns
// one bounded CREATE SQL payload on demand.
func (s *Server) apiDBObjectBackups(w http.ResponseWriter, _ *http.Request) {
sdb := store.Global()
if sdb == nil {
writeAll(w, []dbObjectBackupEntry{})
return
}
records, keys, err := sdb.ListDBObjectBackupsAll()
if err != nil {
writeJSONError(w, "failed to list backups: "+err.Error(), http.StatusInternalServerError)
return
}
out := make([]dbObjectBackupEntry, 0, len(records))
for i, r := range records {
entry := dbObjectBackupEntry{
Key: keys[i],
Account: r.Account,
Schema: r.Schema,
Kind: r.Kind,
Name: r.Name,
DroppedAt: r.DroppedAt.UTC(),
DroppedBy: r.DroppedBy,
FindingID: r.FindingID,
BodyBytes: len(r.CreateSQL),
}
if !r.RestoredAt.IsZero() {
entry.Restored = true
entry.RestoredAt = r.RestoredAt.UTC()
}
out = append(out, entry)
}
// Newest first -- the bbolt key embeds unix-nanos so a string
// sort already produces chronological-by-drop-time when
// reversed. Doing it in Go keeps the contract explicit.
sortDBObjectBackupsNewestFirst(out)
writeAll(w, out)
}
// apiDBObjectBackupPreview returns a bounded CREATE SQL preview for one
// backup. Full restore still round-trips only the opaque key.
func (s *Server) apiDBObjectBackupPreview(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
key := r.URL.Query().Get("key")
if key == "" {
writeJSONError(w, "key is required", http.StatusBadRequest)
return
}
sdb := store.Global()
if sdb == nil {
writeJSONError(w, "bbolt store not available", http.StatusServiceUnavailable)
return
}
rec, ok, err := sdb.GetDBObjectBackupByKey(key)
if err != nil {
writeJSONError(w, "failed to read backup: "+err.Error(), http.StatusInternalServerError)
return
}
if !ok {
writeJSONError(w, "backup not found", http.StatusNotFound)
return
}
preview := rec.CreateSQL
truncated := false
if len(preview) > dbObjectBackupPreviewBytes {
preview = preview[:dbObjectBackupPreviewBytes]
truncated = true
}
writeJSON(w, map[string]any{
"key": key,
"account": rec.Account,
"schema": rec.Schema,
"kind": rec.Kind,
"name": rec.Name,
"preview": preview,
"truncated": truncated,
"total_size": len(rec.CreateSQL),
})
}
// apiDBObjectBackupRestore re-executes the captured CREATE SQL.
// POST body: {"key": "<bbolt key>"}. The handler delegates to
// checks.RestoreDBObjectBackup; CSRF is enforced upstream in
// server.go's requireCSRF wrapper.
func (s *Server) apiDBObjectBackupRestore(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
Key string `json:"key"`
}
if err := decodeJSONBodyLimited(w, r, 16*1024, &req); err != nil || req.Key == "" {
writeJSONError(w, "key is required", http.StatusBadRequest)
return
}
// Tell a missing store or backup apart from a restore that failed.
sdb := store.Global()
if sdb == nil {
writeJSONError(w, "bbolt store not available", http.StatusServiceUnavailable)
return
}
if _, found, err := sdb.GetDBObjectBackupByKey(req.Key); err != nil {
writeJSONError(w, "looking up backup: "+err.Error(), http.StatusInternalServerError)
return
} else if !found {
writeJSONError(w, "backup not found (may have been pruned)", http.StatusNotFound)
return
}
result := checks.RestoreDBObjectBackup(req.Key)
if !result.Success {
writeJSONError(w, result.Message, http.StatusInternalServerError)
return
}
s.auditLog(r, "db_object_restore", req.Key, result.Message)
details := result.Details
if details == nil {
details = []string{}
}
writeOK(w, map[string]interface{}{
"message": result.Message,
"details": details,
})
}
// sortDBObjectBackupsNewestFirst sorts in place by DroppedAt
// descending. Local helper rather than relying on sort.Slice so
// the comparator is unambiguous in code review.
func sortDBObjectBackupsNewestFirst(entries []dbObjectBackupEntry) {
// Stable, so backups dropped at the same instant keep their store order.
sort.SliceStable(entries, func(i, j int) bool {
return entries[i].DroppedAt.After(entries[j].DroppedAt)
})
}
package webui
import (
"net/http"
"github.com/pidginhost/csm/internal/mailfwd/intel"
"github.com/pidginhost/csm/internal/platform"
)
// selectDeferralReporter picks the deferral-intel source for the host. Only
// cPanel/exim is wired; other platforms get the empty reporter until their
// adapters land (Phase 3).
func selectDeferralReporter() intel.Reporter {
if cpanelMailBackend(platform.Detect()) {
return intel.NewEximSource()
}
return intel.EmptyReporter{}
}
// apiEmailDeferrals handles GET /api/v1/email/deferrals and returns the
// outbound-deferral picture parsed from exim_mainlog: per-provider deferral
// rollup and per-outbound-IP reputation with stated reason codes.
func (s *Server) apiEmailDeferrals(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if s.deferralReporter == nil {
writeJSON(w, intel.Report{
Providers: []intel.ProviderRollup{},
OutboundIPs: []intel.OutboundIPRollup{},
})
return
}
// The report reads up to 8 MiB of exim_mainlog, and the page asks for
// it on every load and every minute while its tab is open, so one read
// serves memoMaxAge. History does not change it: the mark stays fixed.
// The memo is its own, so other memoized queries cannot evict it.
res := s.deferralsMemo.get("", func() any {
rep, err := s.deferralReporter.Report()
return deferralsResult{rep, err}
}).(deferralsResult)
if res.err != nil {
writeJSONError(w, "Failed to read deferral log", http.StatusInternalServerError)
return
}
writeJSON(w, res.rep)
}
// deferralsResult is one memoized deferrals read. The exim source answers
// a missing or unreadable log with an empty report, never an error.
type deferralsResult struct {
rep intel.Report
err error
}
package webui
import (
"context"
"io"
"net/http"
"os"
"os/exec"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/emailav"
"github.com/pidginhost/csm/internal/mailfwd/intel"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/systemdrun"
"github.com/pidginhost/csm/internal/yara"
)
type emailStatsResponse struct {
QueueSize int `json:"queue_size"`
// QueueUnavailable marks queue_size as meaningless because the depth could
// not be read. Without it a failed probe looks like an empty queue.
QueueUnavailable bool `json:"queue_unavailable"`
QueueWarn int `json:"queue_warn"`
QueueCrit int `json:"queue_crit"`
FrozenCount int `json:"frozen_count"`
// OldestAgeSeconds is the age of the oldest queued message; left out
// when the queue is empty or could not be read.
OldestAgeSeconds *int `json:"oldest_age_seconds,omitempty"`
SMTPBlock bool `json:"smtp_block"`
SMTPAllowUsers []string `json:"smtp_allow_users"`
SMTPPorts []int `json:"smtp_ports"`
PortFlood []portFloodEntry `json:"port_flood"`
TopSenders []senderEntry `json:"top_senders"`
}
type portFloodEntry struct {
Port int `json:"port"`
Proto string `json:"proto"`
Hits int `json:"hits"`
Seconds int `json:"window_seconds"`
}
type senderEntry struct {
Domain string `json:"domain"`
Count int `json:"count"`
}
func (s *Server) apiEmailStats(w http.ResponseWriter, _ *http.Request) {
cfg := s.liveCfg()
resp := emailStatsResponse{
QueueWarn: cfg.Thresholds.MailQueueWarn,
QueueCrit: cfg.Thresholds.MailQueueCrit,
}
q := queueSnapshot(time.Now())
resp.QueueSize, resp.QueueUnavailable = q.Size, q.Unavailable
resp.FrozenCount, resp.OldestAgeSeconds = q.Frozen, q.OldestAgeSeconds
// Firewall config
fw := cfg.Firewall
resp.SMTPBlock = fw.SMTPBlock
resp.SMTPAllowUsers = fw.SMTPAllowUsers
if resp.SMTPAllowUsers == nil {
resp.SMTPAllowUsers = []string{}
}
resp.SMTPPorts = fw.SMTPPorts
if resp.SMTPPorts == nil {
resp.SMTPPorts = []int{}
}
// Port flood rules for SMTP ports only
smtpPorts := map[int]bool{25: true, 465: true, 587: true}
for _, pf := range fw.PortFlood {
if smtpPorts[pf.Port] {
resp.PortFlood = append(resp.PortFlood, portFloodEntry{
Port: pf.Port,
Proto: pf.Proto,
Hits: pf.Hits,
Seconds: pf.Seconds,
})
}
}
if resp.PortFlood == nil {
resp.PortFlood = []portFloodEntry{}
}
// Top senders from exim_mainlog
resp.TopSenders = topMailSenders(500, 10)
if resp.TopSenders == nil {
resp.TopSenders = []senderEntry{}
}
writeJSON(w, resp)
}
// apiEmailQuarantineList handles GET /api/v1/email/quarantine and returns all
// quarantined email messages, or an empty array if the quarantine is not configured.
func (s *Server) apiEmailQuarantineList(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
quarantine := s.emailQuarantineHandle()
if quarantine == nil {
writeAll(w, []emailav.QuarantineMetadata{})
return
}
msgs, err := quarantine.ListMessages()
if err != nil {
writeJSONError(w, "Failed to list quarantine", http.StatusInternalServerError)
return
}
writeAll(w, msgs)
}
// apiEmailQuarantineAction handles GET, POST (release), and DELETE operations on
// individual quarantined messages at /api/v1/email/quarantine/{msgID}.
func (s *Server) apiEmailQuarantineAction(w http.ResponseWriter, r *http.Request) {
// Extract everything after the prefix, e.g. "abc123" or "abc123/release".
tail := strings.TrimPrefix(r.URL.Path, "/api/v1/email/quarantine/")
if tail == "" {
writeJSONError(w, "Missing message ID", http.StatusBadRequest)
return
}
parts := strings.SplitN(tail, "/", 2)
msgID := parts[0]
action := ""
if len(parts) == 2 {
action = parts[1]
}
if err := validateEximMessageID(msgID); err != nil {
writeJSONError(w, "Invalid message ID: "+err.Error(), http.StatusBadRequest)
return
}
quarantine := s.emailQuarantineHandle()
if quarantine == nil {
writeJSONError(w, "Email quarantine not configured", http.StatusServiceUnavailable)
return
}
switch r.Method {
case http.MethodGet:
if action != "" {
writeJSONError(w, "Unknown action", http.StatusBadRequest)
return
}
meta, err := quarantine.GetMessage(msgID)
if err != nil {
writeJSONError(w, "Message not found", http.StatusNotFound)
return
}
writeJSON(w, meta)
case http.MethodPost:
if action != "release" {
writeJSONError(w, "Unknown action; use /release", http.StatusBadRequest)
return
}
if err := quarantine.ReleaseMessage(msgID); err != nil {
writeJSONError(w, "Failed to release message: "+err.Error(), http.StatusInternalServerError)
return
}
s.auditLog(r, "email_quarantine_release", msgID, "released to the mail queue")
writeOK(w, map[string]interface{}{"message_id": msgID})
case http.MethodDelete:
if action != "" {
writeJSONError(w, "Unknown action", http.StatusBadRequest)
return
}
if err := quarantine.DeleteMessage(msgID); err != nil {
writeJSONError(w, "Failed to delete message: "+err.Error(), http.StatusInternalServerError)
return
}
s.auditLog(r, "email_quarantine_delete", msgID, "deleted permanently")
writeOK(w, map[string]interface{}{"message_id": msgID})
default:
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
}
}
// Seams so the AV status handler can be tested without a live clamd.
var (
resolveClamdSocket = config.ResolveClamdSocket
clamdSocketAvailable = func(path string) bool {
return emailav.NewClamdScanner(path).Available()
}
)
type emailAVStatusResponse struct {
Enabled bool `json:"enabled"`
ClamdAvailable bool `json:"clamd_available"`
ClamdSocket string `json:"clamd_socket"`
YaraXAvailable bool `json:"yarax_available"`
YaraXRuleCount int `json:"yarax_rule_count"`
WatcherMode string `json:"watcher_mode"`
Quarantined int `json:"quarantined"`
}
// apiEmailAVStatus handles GET /api/v1/email/av/status and returns the current
// state of the email AV subsystem (ClamAV, YARA-X, quarantine count, watcher mode).
func (s *Server) apiEmailAVStatus(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
cfg := s.liveCfg()
resp := emailAVStatusResponse{
Enabled: cfg.EmailAV.Enabled,
}
// ClamAV availability. Resolve the socket the same way the daemon's
// scanner does: when the configured path is not answering the daemon
// falls back to a discovered one and mail really is being scanned, so
// probing the configured path here would report the subsystem down over
// a working scanner. Report the socket actually in use, not the stale
// setting, or the page sends the operator after the wrong thing.
clamdSocket, _ := resolveClamdSocket(cfg.EmailAV.ClamdSocket)
if clamdSocket == "" {
// Nothing configured and nothing discovered: name the documented
// default so the card points somewhere rather than showing a blank.
clamdSocket = config.DefaultClamdSocket
}
resp.ClamdSocket = clamdSocket
resp.ClamdAvailable = clamdSocketAvailable(clamdSocket)
// YARA-X availability and rule count. Active() covers both the
// in-process scanner and the out-of-process worker so this card
// reports the real rule count under either backend.
resp.YaraXAvailable = yara.Available()
if b := yara.Active(); b != nil {
resp.YaraXRuleCount = b.RuleCount()
}
// Watcher mode (set by daemon on startup).
resp.WatcherMode = s.emailAVMode()
if resp.WatcherMode == "" {
resp.WatcherMode = "disabled"
}
// Count of currently quarantined messages.
if quarantine := s.emailQuarantineHandle(); quarantine != nil {
msgs, err := quarantine.ListMessages()
if err == nil {
resp.Quarantined = len(msgs)
}
}
writeJSON(w, resp)
}
// eximLookPath and eximRun are injection points so tests can drive the queue
// probes without a live exim.
var (
eximLookPath = exec.LookPath
eximRun = func(ctx context.Context, name string, args ...string) ([]byte, error) {
// #nosec G204 -- name is the resolved systemd-run path or the fixed Exim command.
return exec.CommandContext(ctx, name, args...).Output()
}
)
// runExim executes an exim query as a transient unit forked by PID 1. The web
// server runs inside csm.service's ProtectSystem=strict sandbox, where /var/log
// is read-only; exim opens its main log for append even for a read-only query
// and aborts when it cannot, so a direct call returns nothing at all.
func runExim(timeout time.Duration, args ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return systemdrun.Run(ctx, eximLookPath, eximRun, systemdrun.Options{
Pipe: true,
RuntimeMax: timeout,
}, "exim", args...)
}
// eximQueueSize returns the current Exim mail queue count. The second result is
// false when the depth could not be read: reporting zero there is
// indistinguishable from a healthy empty queue.
func eximQueueSize() (int, bool) {
out, err := runExim(5*time.Second, "-bpc")
if err != nil {
return 0, false
}
n, err := strconv.Atoi(strings.TrimSpace(string(out)))
if err != nil {
return 0, false
}
return n, true
}
// eximQueueDetails lists the queue with `exim -bp`, oldest first. ok is false
// when the listing could not be read or parsed.
func eximQueueDetails() ([]intel.QueueMessage, bool) {
out, err := runExim(10*time.Second, "-bp")
if err != nil {
return nil, false
}
msgs, err := intel.ParseQueueList(out)
if err != nil {
return nil, false
}
sort.SliceStable(msgs, func(i, j int) bool {
if !msgs[i].Arrived.Equal(msgs[j].Arrived) {
return msgs[i].Arrived.Before(msgs[j].Arrived)
}
return msgs[i].ID < msgs[j].ID
})
return msgs, true
}
// emailQueueState is the live queue the Email page shows.
type emailQueueState struct {
Size int `json:"size"`
// Unavailable marks size as meaningless because the depth could not be
// read.
Unavailable bool `json:"unavailable"`
Warn int `json:"warn"`
Crit int `json:"crit"`
Frozen int `json:"frozen"`
// OldestAgeSeconds is left out when the queue is empty, holds no dated
// message, or the detail is unavailable.
OldestAgeSeconds *int `json:"oldest_age_seconds,omitempty"`
// DetailUnavailable marks frozen and oldest as unknown: no listing has
// succeeded within queueDetailMaxAge.
DetailUnavailable bool `json:"detail_unavailable"`
}
const (
// queueDetailTTL spaces queue listings. exim -bp reads every queued
// message, which takes seconds on a deep queue, while the Email page
// polls; the depth from exim -bpc stays live.
queueDetailTTL = time.Minute
// queueDetailMaxAge is how long the last good listing is served while
// newer ones fail. A deep queue is when listings time out and when a
// stale frozen count misleads most.
queueDetailMaxAge = 10 * time.Minute
)
// queueDetail caches the queue listing the overview and the queue list share.
// Its lock is held while a listing runs, so concurrent polls wait for one
// listing instead of each starting their own.
var queueDetail struct {
sync.Mutex
state queueDetailState
}
type queueDetailState struct {
tried time.Time // last listing attempt
listed time.Time // last successful listing, or the empty count
// emptied marks listed as an exim -bpc that found the queue empty
// rather than a listing. The overview has no detail for mail queued
// since; the queue list serves the empty queue.
emptied bool
// refill marks a positive count that disproved a known empty result.
// The queue list then lists without waiting out queueDetailTTL; any
// listing clears the mark, so each refill costs one extra listing.
refill bool
msgs []intel.QueueMessage // oldest first; never modified once stored
frozen int
// oldestQueued is when the oldest listed message was queued, the age
// the listing gave counted back from listed; zero when it gave none.
oldestQueued time.Time
}
// refreshQueueListing lists the queue when the last attempt, failed or not,
// is queueDetailTTL old. The caller holds queueDetail.
func refreshQueueListing(now time.Time) {
d := &queueDetail.state
if !d.tried.IsZero() && now.Sub(d.tried) < queueDetailTTL {
return
}
d.tried = now
msgs, ok := eximQueueDetails()
if !ok {
if len(d.msgs) == 0 {
// Mail may have arrived since the empty result. A failed read
// cannot confirm that the queue is still empty.
*d = queueDetailState{tried: now}
}
return
}
*d = queueDetailState{tried: now, listed: now, msgs: msgs}
oldest := -1
for _, m := range msgs {
if m.Frozen {
d.frozen++
}
oldest = max(oldest, m.AgeSeconds)
}
if oldest >= 0 {
d.oldestQueued = now.Add(-time.Duration(oldest) * time.Second)
}
}
// staleQueueListing reports whether the cached listing is too old to serve.
// The caller holds queueDetail.
func staleQueueListing(now time.Time) bool {
d := &queueDetail.state
return d.listed.IsZero() || now.Sub(d.listed) > queueDetailMaxAge
}
// oldestQueuedSeconds ages the oldest listed message to now. The caller holds
// queueDetail.
func oldestQueuedSeconds(now time.Time) *int {
d := &queueDetail.state
if d.oldestQueued.IsZero() {
return nil
}
// A request that waited on the lock can hold a now from before the
// listing; age from the listing then, never below what it showed.
at := now
if at.Before(d.listed) {
at = d.listed
}
secs := int(at.Sub(d.oldestQueued) / time.Second)
return &secs
}
// expireQueueListing makes the next caller list the queue again, after an
// action changed it.
func expireQueueListing() {
queueDetail.Lock()
defer queueDetail.Unlock()
queueDetail.state.tried = time.Time{}
}
// queueSnapshot reads the queue depth on every call and lists the queue at
// most once per queueDetailTTL, failed attempts included. Frozen and oldest
// come from the last good listing, with oldest aged to now, for up to
// queueDetailMaxAge; past that DetailUnavailable is set. An empty depth
// drops the listing, which describes mail that is gone. Only a failed depth
// marks the queue unavailable. The caller fills Warn and Crit.
func queueSnapshot(now time.Time) emailQueueState {
size, ok := eximQueueSize()
q := emailQueueState{Size: size, Unavailable: !ok}
queueDetail.Lock()
defer queueDetail.Unlock()
d := &queueDetail.state
if ok && size == 0 {
*d = queueDetailState{tried: d.tried, listed: now, emptied: true}
return q
}
if ok && size > 0 && !d.listed.IsZero() && len(d.msgs) == 0 {
// The count disproves the cached empty result, even when another
// listing is not due yet. The overview keeps the retry interval;
// the queue list lists at once.
*d = queueDetailState{tried: d.tried, refill: true}
}
refreshQueueListing(now)
if d.emptied || staleQueueListing(now) {
q.DetailUnavailable = true
return q
}
q.Frozen = d.frozen
q.OldestAgeSeconds = oldestQueuedSeconds(now)
return q
}
// topMailSenders parses the last N lines of exim_mainlog and returns the
// top K sender domains by outbound message count.
func topMailSenders(tailLines, topK int) []senderEntry {
f, err := os.Open(platform.EximMainLog)
if err != nil {
return nil
}
defer f.Close()
// Read tail of file
info, _ := f.Stat()
var data []byte
if info != nil && info.Size() > 256*1024 {
if _, err := f.Seek(-256*1024, 2); err != nil {
return nil
}
data, _ = io.ReadAll(f)
} else {
data, _ = io.ReadAll(f)
}
lines := strings.Split(string(data), "\n")
// Take last N lines
if len(lines) > tailLines {
lines = lines[len(lines)-tailLines:]
}
counts := make(map[string]int)
for _, line := range lines {
if domain := checks.ArrivalSenderDomain(line); domain != "" {
counts[domain]++
}
}
var entries []senderEntry
for domain, count := range counts {
entries = append(entries, senderEntry{Domain: domain, Count: count})
}
sort.Slice(entries, func(i, j int) bool {
return entries[i].Count > entries[j].Count
})
if len(entries) > topK {
entries = entries[:topK]
}
return entries
}
package webui
import "net/http"
// emailCheck is one check the Email page shows. Kind is its group on the
// workbench; "" lists the check in Findings but never groups it.
type emailCheck struct {
Name string `json:"name"`
Kind string `json:"kind"`
}
// emailChecks is the one list of email checks. TestEmailCheckListCoversRegistry
// fails when a registered check in the Email & Phishing category, or one in
// any category named mail_, smtp_, email_, exim_ or antispam_, is in neither
// emailChecks nor emailCheckExclusions.
var emailChecks = []emailCheck{
{"email_compromised_account", "compromised_account"},
{"email_credential_leak", "compromised_account"},
{"email_weak_password", "compromised_account"},
{"mail_account_compromised", "compromised_account"},
{"email_mailbox_response", "compromised_account"},
{"email_pipe_forwarder", "compromised_account"},
{"email_suspicious_forwarder", "compromised_account"},
{"email_spam_outbreak", "spam_outbreak"},
{"email_rate_critical", "spam_outbreak"},
{"email_rate_warning", "spam_outbreak"},
{"email_php_relay_abuse", "spam_outbreak"},
{"email_php_form_relay", "spam_outbreak"},
{"email_php_form_relay_observed", "spam_outbreak"},
{"email_php_form_relay_script", "spam_outbreak"},
{"email_php_relay_action_failed", "spam_outbreak"},
{"email_php_relay_rate_limit_hit", "spam_outbreak"},
{"email_cloud_relay_abuse", "spam_outbreak"},
{"email_auth_failure_realtime", "auth_failure"},
{"email_suspicious_geo", "auth_failure"},
{"mail_bruteforce", "auth_failure"},
{"mail_bruteforce_suspected", "auth_failure"},
{"mail_subnet_spray", "auth_failure"},
{"mail_account_spray", "auth_failure"},
{"smtp_bruteforce", "auth_failure"},
{"smtp_subnet_spray", "auth_failure"},
{"smtp_account_spray", "auth_failure"},
{"smtp_probe_abuse", "auth_failure"},
{"stalwart_ban", "auth_failure"},
{"stalwart_webmail_bruteforce", "auth_failure"},
{"email_malware", "malware"},
{"email_phishing_content", "malware"},
{"email_av_degraded", "malware"},
{"email_av_encrypted_archive", "malware"},
{"email_av_timeout", "malware"},
{"email_av_parse_error", "malware"},
{"email_av_parse_worker_failed", "malware"},
{"email_av_parse_worker_rejected", "malware"},
{"email_av_parse_worker_source", "malware"},
{"email_av_source_backlog", "malware"},
{"email_av_isolation_unavailable", "malware"},
{"email_av_quarantine_error", "malware"},
{"mail_per_account", "queue_alert"},
{"mail_queue", "queue_alert"},
{"mail_queue_unavailable", "queue_alert"},
{"email_defer_fail_governor", "queue_alert"},
{"exim_frozen_realtime", "queue_alert"},
{"email_dkim_failure", ""},
{"email_spf_rejection", ""},
{"email_filter_blackhole", ""},
{"email_filter_exfil", ""},
{"email_filter_forwarder", ""},
{"email_filter_pipe", ""},
{"email_mail_filters", ""},
{"email_password_audit_incomplete", ""},
{"mail_auth_backend_degraded", ""},
{"mail_log_source_unavailable", ""},
{"stalwart_log_degraded", ""},
{"stalwart_address_list", ""},
{"stalwart_spam_filter_off", ""},
{"mail_spam_source", ""},
{"shared_senders_stale", ""},
{"spam_evidence_evicted", ""},
{"antispam_apply_failed", ""},
{"antispam_controller_open", ""},
{"antispam_file_replaced", ""},
{"antispam_feed", ""},
{"antispam_ip_map", ""},
{"antispam_learning", ""},
{"antispam_precondition", ""},
{"antispam_rspamd_down", ""},
{"antispam_rspamd_recovered", ""},
{"antispam_scan_error", ""},
{"email_av_hold_bypass", ""},
{"email_av_late_verdict", ""},
{"email_av_queue_overflow", ""},
{"email_av_scan_error", ""},
{"email_av_scanner_panic", ""},
{"email_php_form_relay_table_full", ""},
{"email_php_relay_account_volume_capped", ""},
{"email_php_relay_action_dry_run", ""},
{"email_php_relay_action_skipped", ""},
{"email_php_relay_cpanel_limit_unreadable", ""},
{"email_php_relay_disabled", ""},
{"email_php_relay_inotify_overflow", ""},
{"email_php_relay_inotify_overflow_recovered", ""},
{"email_php_relay_msgindex_persist_failed", ""},
{"email_php_relay_no_exim", ""},
{"email_php_relay_overflow_scan_truncated", ""},
{"email_php_relay_path2b_disabled", ""},
{"email_php_relay_policies_reload", ""},
{"email_php_relay_sweep_failed", ""},
{"email_php_relay_watcher_failed", ""},
}
// emailCheckExclusions are Email & Phishing checks that find phishing kits,
// pages and harvested credential logs on websites, not mail. The Email page
// leaves them out.
var emailCheckExclusions = []string{
"credential_log_realtime",
"phishing_credential_log",
"phishing_directory",
"phishing_iframe",
"phishing_kit_archive",
"phishing_kit_archive_unscannable",
"phishing_kit_realtime",
"phishing_page",
"phishing_php",
"phishing_realtime",
"phishing_redirector",
}
var emailCheckKinds = func() map[string]string {
kinds := make(map[string]string, len(emailChecks))
for _, c := range emailChecks {
kinds[c.Name] = c.Kind
}
return kinds
}()
// emailCheckSet returns a new set of the listed check names, for a history
// filter.
func emailCheckSet() map[string]bool {
set := make(map[string]bool, len(emailChecks))
for _, c := range emailChecks {
set[c.Name] = true
}
return set
}
// apiEmailChecks answers the email check list with each check's group kind.
func (s *Server) apiEmailChecks(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, map[string][]emailCheck{"checks": emailChecks})
}
package webui
import (
"net/http"
"net/netip"
"net/url"
"slices"
"sort"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/store"
)
// emailGroupsScanCap is the hard upper bound on matching findings retained per
// /api/v1/email/groups call. Bounded reads keep the workbench cheap on
// hosts that store thousands of mail-related findings per day.
const emailGroupsScanCap = 5000
// emailGroupsDefaultLimit / Max bound the number of grouped rows returned
// to the operator UI. The plan caps the email first viewport at ~250 nodes
// so 200 is the highest useful ceiling.
const (
emailGroupsDefaultLimit = 50
emailGroupsMaxLimit = 200
)
type emailGroup struct {
Kind string `json:"kind"`
Severity string `json:"severity"`
level alert.Severity
Title string `json:"title"`
Subject string `json:"subject"`
Count int `json:"count"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
Summary string `json:"summary"`
SampleFindings []apiFinding `json:"sample_findings"`
IPs []string `json:"ips,omitempty"`
TopIPs []string `json:"top_ips,omitempty"`
Domains []string `json:"domains,omitempty"`
MessageIDs []string `json:"message_ids,omitempty"`
// BlockState is all, some, none or no_ip: how many of IPs are under an
// enforced block now. It is empty when the firewall state is unreadable.
BlockState string `json:"block_state"`
// Handled marks an auth failure group whose every finding is answered
// by its blocked attackers. Compromise evidence remains actionable.
Handled bool `json:"handled"`
// blockInsufficient covers missing addresses, successful logins, and
// distributed attacks that name only their latest source.
blockInsufficient bool
}
type emailGroupsResponse struct {
Groups []emailGroup `json:"items"`
Total int `json:"total"`
Offset int `json:"offset"`
Limit int `json:"limit"`
From time.Time `json:"from"`
To time.Time `json:"to"`
Scanned int `json:"scanned"`
Truncated bool `json:"truncated"`
// HandledCount counts handled groups across all of Total.
HandledCount int `json:"handled_count"`
// BlocksUnavailable is set when the firewall state cannot be read. No
// group is handled then: an unknown block must not hide a group.
BlocksUnavailable bool `json:"blocks_unavailable"`
historyMark string
}
// emailKindForCheck maps an alert check name to its email-workbench group
// kind. Returns "" when the check is not grouped and the finding should be
// skipped by /api/v1/email/groups.
func emailKindForCheck(check string) string {
return emailCheckKinds[check]
}
// emailGroupKey is the dedup key used to merge findings into a single
// grouped action row. Different kinds prefer different identity fields:
// auth failures cluster by mailbox/IP, spam/malware/compromised by mailbox
// or domain, and queue alerts by check name.
func emailGroupKey(kind string, f alert.Finding) string {
mailbox := strings.ToLower(strings.TrimSpace(f.Mailbox))
domain := strings.ToLower(strings.TrimSpace(f.Domain))
switch kind {
case "auth_failure":
if mailbox != "" {
return "mailbox:" + mailbox
}
if ip := alert.AttackerAddress(f); ip != "" {
return "ip:" + ip
}
if domain != "" {
return "domain:" + domain
}
return "auth:unknown"
case "queue_alert":
return "queue:" + f.Check
default:
if mailbox != "" {
return kind + ":mailbox:" + mailbox
}
if domain != "" {
return kind + ":domain:" + domain
}
if ip := alert.AttackerAddress(f); ip != "" {
return kind + ":ip:" + ip
}
// Fall back to message text so two distinct payloads with no
// identity fields still produce two groups instead of collapsing.
return kind + ":msg:" + strings.TrimSpace(f.Message)
}
}
// emailGroupTitle renders the human-readable identifier for a grouped row.
// Prefers mailbox > domain > source IP > message text. Queue alerts have
// hard-coded labels because their finding text varies by host.
func emailGroupTitle(kind string, f alert.Finding) string {
if kind == "queue_alert" {
switch f.Check {
case "mail_queue":
return "Mail queue threshold"
case "mail_queue_unavailable":
return "Mail queue unavailable"
case "mail_per_account":
return "Per-account mail volume"
case "exim_frozen_realtime":
return "Frozen mail queue"
}
}
if f.Mailbox != "" {
return f.Mailbox
}
if f.Domain != "" {
return f.Domain
}
if ip := alert.AttackerAddress(f); ip != "" {
return ip
}
return strings.TrimSpace(f.Message)
}
// emailGroupSubject describes the identity dimension behind the group --
// "mailbox", "domain", "ip", or "queue" -- so the UI can pick the right
// detail-panel tabs without re-reading the raw findings.
func emailGroupSubject(kind string, f alert.Finding) string {
if kind == "queue_alert" {
return "queue"
}
if f.Mailbox != "" {
return "mailbox"
}
if f.Domain != "" {
return "domain"
}
if alert.AttackerAddress(f) != "" {
return "ip"
}
return "unknown"
}
// buildEmailGroups walks the supplied findings (already bounded), merges
// matching findings into grouped rows, and returns the result sorted by
// severity (desc) then last-seen (desc). Pure function -- the HTTP
// handler is a thin wrapper so tests can drive grouping directly.
func buildEmailGroups(findings []alert.Finding, from, to time.Time, kindFilter string) []emailGroup {
type aggregator struct {
group *emailGroup
ipCounts map[string]int
domainSet map[string]struct{}
msgIDSet map[string]struct{}
samples []alert.Finding // newest-first
}
groups := make(map[string]*aggregator)
order := make([]string, 0)
for _, f := range findings {
ts := f.Timestamp
if !from.IsZero() && ts.Before(from) {
continue
}
if !to.IsZero() && ts.After(to) {
continue
}
kind := emailKindForCheck(f.Check)
if kind == "" {
continue
}
if kindFilter != "" && kindFilter != kind {
continue
}
key := emailGroupKey(kind, f)
agg, ok := groups[key]
if !ok {
agg = &aggregator{
group: &emailGroup{
Kind: kind,
level: f.Severity,
Title: emailGroupTitle(kind, f),
Subject: emailGroupSubject(kind, f),
FirstSeen: ts.UTC(),
LastSeen: ts.UTC(),
},
ipCounts: make(map[string]int),
domainSet: make(map[string]struct{}),
msgIDSet: make(map[string]struct{}),
}
groups[key] = agg
order = append(order, key)
}
agg.group.Count++
if f.Severity > agg.group.level {
agg.group.level = f.Severity
}
if ts.Before(agg.group.FirstSeen) {
agg.group.FirstSeen = ts.UTC()
}
if ts.After(agg.group.LastSeen) {
agg.group.LastSeen = ts.UTC()
}
if ip := alert.AttackerAddress(f); ip != "" {
agg.ipCounts[ip]++
} else {
agg.group.blockInsufficient = true
}
// Subnet sprays are checked against coverage of their entire CIDR.
// Other findings follow the existing block-response policy.
if f.Check != "mail_subnet_spray" && f.Check != "smtp_subnet_spray" && !checks.IPResponseAnswersFinding(nil, f, true) {
agg.group.blockInsufficient = true
}
if f.Domain != "" {
agg.domainSet[strings.ToLower(f.Domain)] = struct{}{}
}
for _, id := range f.MsgIDs {
if id != "" {
agg.msgIDSet[id] = struct{}{}
}
}
// Keep up to 3 most recent samples (assumes input is newest-first).
if len(agg.samples) < 3 {
agg.samples = append(agg.samples, f)
}
}
out := make([]emailGroup, 0, len(order))
for _, key := range order {
agg := groups[key]
g := agg.group
// Compose summary text: count + identity + IP/domain hint.
hint := ""
if g.Kind == "auth_failure" && len(agg.ipCounts) > 0 {
hint = " from " + plural(len(agg.ipCounts), "IP")
} else if len(agg.domainSet) > 1 {
hint = " across " + plural(len(agg.domainSet), "domain")
}
g.Summary = plural(g.Count, "event") + hint
g.Severity = g.level.String()
g.SampleFindings = toAPIFindings(agg.samples)
if len(agg.ipCounts) > 0 {
g.IPs = sortedKeys(agg.ipCounts)
g.TopIPs = topKeysByCount(agg.ipCounts, 5)
}
if len(agg.domainSet) > 0 {
g.Domains = sortedSetKeys(agg.domainSet)
}
if len(agg.msgIDSet) > 0 {
g.MessageIDs = sortedSetKeys(agg.msgIDSet)
if len(g.MessageIDs) > 10 {
g.MessageIDs = g.MessageIDs[:10]
}
}
out = append(out, *g)
}
sort.SliceStable(out, func(i, j int) bool {
if out[i].level != out[j].level {
return out[i].level > out[j].level
}
if out[i].Count != out[j].Count {
return out[i].Count > out[j].Count
}
return out[i].LastSeen.After(out[j].LastSeen)
})
return out
}
func plural(n int, label string) string {
if n == 1 {
return "1 " + label
}
return itoa(n) + " " + label + "s"
}
func itoa(n int) string {
// Avoid pulling strconv just for this hot path; keeps the helper inline.
if n == 0 {
return "0"
}
neg := n < 0
if neg {
n = -n
}
var buf [20]byte
i := len(buf)
for n > 0 {
i--
buf[i] = byte('0' + n%10)
n /= 10
}
if neg {
i--
buf[i] = '-'
}
return string(buf[i:])
}
func sortedKeys(m map[string]int) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
func sortedSetKeys(m map[string]struct{}) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// topKeysByCount returns up to k entries from m sorted by descending count
// (ties broken alphabetically) so the UI shows the dominant attackers
// first.
func topKeysByCount(m map[string]int, k int) []string {
type entry struct {
key string
count int
}
entries := make([]entry, 0, len(m))
for key, c := range m {
entries = append(entries, entry{key, c})
}
sort.SliceStable(entries, func(i, j int) bool {
if entries[i].count != entries[j].count {
return entries[i].count > entries[j].count
}
return entries[i].key < entries[j].key
})
if k < len(entries) {
entries = entries[:k]
}
out := make([]string, len(entries))
for i, e := range entries {
out[i] = e.key
}
return out
}
// historyRangeQuery reads the from and to parameters the history endpoints
// share, with the meaning store.ParseHistoryBound gives them: to is
// exclusive. A missing bound takes its default. An unreadable one is a 400,
// written here, and ok is false.
func historyRangeQuery(w http.ResponseWriter, q url.Values, defFrom, defTo time.Time) (from, to time.Time, ok bool) {
from, err := store.ParseHistoryBound(q.Get("from"), false)
if err != nil {
writeJSONError(w, "Invalid from: use YYYY-MM-DD or an RFC 3339 time", http.StatusBadRequest)
return from, to, false
}
to, err = store.ParseHistoryBound(q.Get("to"), true)
if err != nil {
writeJSONError(w, "Invalid to: use YYYY-MM-DD or an RFC 3339 time", http.StatusBadRequest)
return from, to, false
}
if from.IsZero() {
from = defFrom
}
if to.IsZero() {
to = defTo
}
return from, to, true
}
// apiEmailGroups handles GET /api/v1/email/groups. Returns server-side
// grouped action rows for the email workbench. Read-scope tokens may
// call this endpoint -- it does not mutate state.
func (s *Server) apiEmailGroups(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
q := r.URL.Query()
limit := queryInt(r, "limit", emailGroupsDefaultLimit)
if limit <= 0 || limit > emailGroupsMaxLimit {
limit = emailGroupsDefaultLimit
}
now := time.Now()
from, to, ok := emailRange(w, q, now)
if !ok {
return
}
kindFilter := q.Get("kind")
// The memo holds every group. Block state changes while history does
// not, so marking, ordering and the limit happen on each request, on a
// copy of the shared groups.
key := "groups?" + q.Encode()
compute := func() any { return s.buildEmailGroupsResponse(from, to, kindFilter) }
resp := s.emailMemo(key, compute).(emailGroupsResponse)
// A throttled result can miss a new attacker, so it cannot mark groups
// handled. Any check's finding moves the mark; recompute rather than
// show every handled group as needing attention.
if s.store != nil && resp.historyMark != s.store.HistoryMark() {
resp = s.emailMemos.memo(key).fresh(s.store.HistoryMark(), compute).(emailGroupsResponse)
}
resp.Groups = slices.Clone(resp.Groups)
if ips, nets, err := activeBlocks(s.cfg.StatePath, now); err != nil {
resp.BlocksUnavailable = true
} else {
annotateEmailGroups(resp.Groups, ips, nets)
}
incomplete := resp.Truncated || s.store != nil && resp.historyMark != s.store.HistoryMark()
for i := range resp.Groups {
g := &resp.Groups[i]
// Before the response limit is applied, Truncated means history
// was cut short. History that changed after the result was built
// can also hold new attackers.
if incomplete {
g.Handled = false
}
if g.Handled {
resp.HandledCount++
}
}
sort.SliceStable(resp.Groups, func(i, j int) bool {
return !resp.Groups[i].Handled && resp.Groups[j].Handled
})
if len(resp.Groups) > limit {
resp.Truncated = true
resp.Groups = resp.Groups[:limit]
}
resp.Limit = limit
writeJSON(w, resp)
}
// activeBlocks returns the addresses and subnets under an enforced block at
// now. Dry-run blocks are never written to the firewall state.
func activeBlocks(statePath string, now time.Time) (ips map[netip.Addr]bool, nets []netip.Prefix, err error) {
st, err := firewall.LoadState(statePath)
if err != nil {
return nil, nil, err
}
ips = make(map[netip.Addr]bool, len(st.Blocked))
for _, b := range st.Blocked {
if !blockActive(b.Unverified, b.ExpiresAt, now) {
continue
}
if ip, err := netip.ParseAddr(b.IP); err == nil {
ips[ip.Unmap()] = true
}
}
for _, n := range st.BlockedNet {
if !blockActive(false, n.ExpiresAt, now) {
continue
}
if p, err := netip.ParsePrefix(n.CIDR); err == nil {
nets = append(nets, p)
}
}
return ips, nets, nil
}
// annotateEmailGroups sets each group's block state, and marks an auth
// failure group handled when every finding names its attacker and every one
// of its IPs is blocked.
func annotateEmailGroups(groups []emailGroup, ips map[netip.Addr]bool, nets []netip.Prefix) {
for i := range groups {
g := &groups[i]
blocked := 0
for _, s := range g.IPs {
if entryBlocked(s, ips, nets) {
blocked++
}
}
switch {
case len(g.IPs) == 0:
g.BlockState = "no_ip"
case blocked == len(g.IPs):
g.BlockState = "all"
case blocked > 0:
g.BlockState = "some"
default:
g.BlockState = "none"
}
g.Handled = g.Kind == "auth_failure" && !g.blockInsufficient && g.BlockState == "all"
}
}
// entryBlocked reports whether a group's IP entry is under an active block.
// An address is covered by a direct block or a subnet block holding it. A
// subnet, which a subnet spray names, is covered only by a subnet block
// holding all of it. Anything else is never blocked.
func entryBlocked(s string, ips map[netip.Addr]bool, nets []netip.Prefix) bool {
if ip, err := netip.ParseAddr(s); err == nil {
ip = ip.Unmap()
return ips[ip] || slices.ContainsFunc(nets, func(p netip.Prefix) bool { return p.Contains(ip) })
}
if sub, err := netip.ParsePrefix(s); err == nil {
return slices.ContainsFunc(nets, func(p netip.Prefix) bool {
return p.Bits() <= sub.Bits() && p.Contains(sub.Masked().Addr())
})
}
return false
}
func (s *Server) buildEmailGroupsResponse(from, to time.Time, kindFilter string) emailGroupsResponse {
var findings []alert.Finding
var mark string
if s.store != nil {
mark = s.store.HistoryMark()
// Filter while walking history so unrelated findings, or findings
// newer than the requested range, never use up the scan budget.
findings = s.store.SearchHistorySince(from, emailGroupsScanCap+1, func(f alert.Finding) bool {
if !f.Timestamp.Before(to) {
return false
}
kind := emailKindForCheck(f.Check)
return kind != "" && (kindFilter == "" || kind == kindFilter)
})
}
truncated := false
if len(findings) > emailGroupsScanCap {
findings = findings[:emailGroupsScanCap]
truncated = true
}
scanned := len(findings)
groups := buildEmailGroups(findings, from, to, kindFilter)
return emailGroupsResponse{
Groups: groups,
Total: len(groups),
From: from.UTC(),
To: to.UTC(),
Scanned: scanned,
Truncated: truncated,
historyMark: mark,
}
}
// emailMemo reuses an email workbench result for the same query while
// history is unchanged; the page polls these every minute.
func (s *Server) emailMemo(key string, compute func() any) any {
if s.store == nil {
return compute()
}
return s.emailMemos.memo(key).get(s.store.HistoryMark(), compute)
}
// rangeMemoKey keys an email memo on a range cut to the minute. The page asks
// up to now, so a key holding the exact times never matched a later request;
// requests in the same minute now share one history walk.
func rangeMemoKey(name string, from, to time.Time) string {
return name + "?from=" + strconv.FormatInt(from.Truncate(time.Minute).Unix(), 10) +
"&to=" + strconv.FormatInt(to.Truncate(time.Minute).Unix(), 10)
}
package webui
import (
"net/http"
"net/url"
"strings"
"time"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/mailstats"
)
// emailBlockCounts counts the active mail-related IP blocks, and those of
// them placed within the requested range.
type emailBlockCounts struct {
Active int `json:"active"`
InRange int `json:"in_range"`
// Unavailable marks the counts as meaningless because the firewall
// state could not be read. A missing state file is not an error.
Unavailable bool `json:"unavailable"`
}
// emailOverviewResponse is the Email page's headline numbers and chart. The
// counter's fields stay at the top level.
type emailOverviewResponse struct {
mailstats.Overview
Mode string `json:"mode"` // enforce | dry-run | off
SMTPRestricted bool `json:"smtp_restricted"`
Queue emailQueueState `json:"queue"`
Blocks emailBlockCounts `json:"blocks"`
}
// mailBlockKeywords mark a block reason as mail-related.
var mailBlockKeywords = []string{"mail", "smtp", "spam", "phish", "mailer"}
// blockActive reports whether a block is in force at now. An unverified row
// was never proven in the kernel, and a zero expiry is permanent. Subnet
// blocks have no unverified state and pass false.
func blockActive(unverified bool, expires, now time.Time) bool {
return !unverified && (expires.IsZero() || expires.After(now))
}
func mailRelatedReason(reason string) bool {
reason = strings.ToLower(reason)
for _, kw := range mailBlockKeywords {
if strings.Contains(reason, kw) {
return true
}
}
return false
}
// mailBlockCounts counts the active mail-related IP blocks in the firewall
// state, and those placed in [from, to). A state that cannot be read is
// unavailable: zero would tell the operator nothing is blocked.
func mailBlockCounts(statePath string, from, to, now time.Time) emailBlockCounts {
var counts emailBlockCounts
st, err := firewall.LoadState(statePath)
if err != nil {
counts.Unavailable = true
return counts
}
for _, b := range st.Blocked {
if !blockActive(b.Unverified, b.ExpiresAt, now) || !mailRelatedReason(b.Reason) {
continue
}
counts.Active++
if !b.BlockedAt.Before(from) && b.BlockedAt.Before(to) {
counts.InRange++
}
}
return counts
}
func antispamMode(cfg *config.Config) string {
switch as := cfg.EmailProtection.Antispam; {
case !as.Enabled:
return "off"
case as.DryRun:
return "dry-run"
}
return "enforce"
}
// emailRange reads from/to for the Email page's counted views: the last
// 24 h by default, the 24 h before to when only to is given, and to now
// when only from is. A reversed range is swapped.
func emailRange(w http.ResponseWriter, q url.Values, now time.Time) (from, to time.Time, ok bool) {
from, to, ok = historyRangeQuery(w, q, time.Time{}, now)
if !ok {
return from, to, false
}
if from.IsZero() {
from = to.Add(-24 * time.Hour)
}
if to.Before(from) {
from, to = to, from
}
return from, to, true
}
// apiEmailOverview handles GET /api/v1/email/overview: the mail decision
// counts for from/to (see emailRange), the live queue and the mail-related
// firewall blocks.
func (s *Server) apiEmailOverview(w http.ResponseWriter, r *http.Request) {
now := time.Now()
from, to, ok := emailRange(w, r.URL.Query(), now)
if !ok {
return
}
cfg := s.liveCfg()
resp := emailOverviewResponse{
Overview: mailstats.Overview{From: from, To: to},
Mode: antispamMode(cfg),
SMTPRestricted: cfg.Firewall != nil && cfg.Firewall.SMTPBlock,
Queue: queueSnapshot(now),
Blocks: mailBlockCounts(s.cfg.StatePath, from, to, now),
}
if c := s.mailStatsCounter(); c != nil {
resp.Overview = c.Overview(from, to)
}
resp.Queue.Warn, resp.Queue.Crit = cfg.Thresholds.MailQueueWarn, cfg.Thresholds.MailQueueCrit
writeJSON(w, resp)
}
package webui
import (
"errors"
"net/http"
"strings"
"time"
"github.com/pidginhost/csm/internal/mailfwd/intel"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/systemdrun"
)
const (
queueListDefaultLimit = 25
queueListMaxLimit = 100
// queueReasonTTL is how long a message's msglog reason is reused.
queueReasonTTL = time.Minute
// queueUnavailableText answers the queue routes on hosts without exim.
queueUnavailableText = "Mail queue is not available on this host"
)
// selectQueueAdmin picks the per-message queue executor. Only cPanel/exim is
// wired; other platforms answer the queue routes as unavailable.
func selectQueueAdmin() intel.QueueAdmin {
if cpanelMailBackend(platform.Detect()) {
return intel.NewEximQueueAdmin(platform.EximSpoolRoot, eximLookPath, eximRun)
}
return nil
}
// queueItem is a queued message with why it is stuck.
type queueItem struct {
intel.QueueMessage
Reason intel.StuckReason `json:"reason"`
}
// queueMessageDetail is one queued message with the last lines of its msglog.
type queueMessageDetail struct {
intel.QueueMessage
Reason intel.StuckReason `json:"reason"`
Msglog []string `json:"msglog"`
}
type cachedQueueReason struct {
read time.Time
reason intel.StuckReason
}
// apiEmailQueueList handles GET /api/v1/email/queue: one page of the queue,
// oldest first, filtered by filter=all|frozen|backscatter and q (a sender or
// recipient substring). queued, frozen, backscatter and oldest_seconds
// describe the whole queue; total counts the matches; listed_at is when the
// listing ran. The listing is the cached one the overview uses, and msglogs
// are read only for the page.
func (s *Server) apiEmailQueueList(w http.ResponseWriter, r *http.Request) {
if s.queueAdmin == nil {
writeJSONError(w, queueUnavailableText, http.StatusServiceUnavailable)
return
}
query := r.URL.Query()
filter := query.Get("filter")
switch filter {
case "":
filter = "all"
case "all", "frozen", "backscatter":
default:
writeJSONError(w, "Invalid filter: use all, frozen or backscatter", http.StatusBadRequest)
return
}
search := strings.ToLower(strings.TrimSpace(query.Get("q")))
page := max(queryInt(r, "page", 1), 1)
limit := queryInt(r, "limit", queueListDefaultLimit)
if limit <= 0 {
limit = queueListDefaultLimit
}
limit = min(limit, queueListMaxLimit)
now := time.Now()
queueDetail.Lock()
if queueDetail.state.refill {
// The card already counts mail queued since an empty listing; list
// it now rather than answer that the queue is unknown until the
// next listing is due.
queueDetail.state.tried = time.Time{}
}
refreshQueueListing(now)
stale := staleQueueListing(now)
msgs, frozen, oldest, listed := queueDetail.state.msgs, queueDetail.state.frozen, oldestQueuedSeconds(now), queueDetail.state.listed
queueDetail.Unlock()
if stale {
// No count goes out: any number here would read as a queue size.
writeItems(w, []queueItem{}, map[string]interface{}{"page": page, "limit": limit, "unavailable": true})
return
}
backscatter := 0
var matched []intel.QueueMessage
for _, m := range msgs {
if m.Bounce {
backscatter++
}
if queueMessageMatches(m, filter, search) {
matched = append(matched, m)
}
}
start := len(matched)
if page-1 <= len(matched)/limit {
start = min((page-1)*limit, len(matched))
}
end := min(start+limit, len(matched))
extra := map[string]interface{}{
"total": len(matched),
"queued": len(msgs),
"page": page,
"limit": limit,
"frozen": frozen,
"backscatter": backscatter,
"unavailable": false,
"listed_at": listed.UTC(),
}
if oldest != nil {
extra["oldest_seconds"] = *oldest
}
writeItems(w, s.pageReasons(now, matched[start:end]), extra)
}
func queueMessageMatches(m intel.QueueMessage, filter, search string) bool {
if (filter == "frozen" && !m.Frozen) || (filter == "backscatter" && !m.Bounce) {
return false
}
if search == "" || strings.Contains(strings.ToLower(m.Sender), search) {
return true
}
for _, rcpt := range m.Recipients {
if strings.Contains(strings.ToLower(rcpt), search) {
return true
}
}
return false
}
// pageReasons pairs each message of a list page with why it is stuck. A
// msglog is reused for queueReasonTTL while cached. The cache holds at most
// queueListMaxLimit reasons, retaining the current page before older pages.
func (s *Server) pageReasons(now time.Time, msgs []intel.QueueMessage) []queueItem {
s.queueReasonMu.Lock()
defer s.queueReasonMu.Unlock()
for id, c := range s.queueReasons {
if now.Sub(c.read) >= queueReasonTTL {
delete(s.queueReasons, id)
}
}
if s.queueReasons == nil {
s.queueReasons = map[string]cachedQueueReason{}
}
pageIDs := make(map[string]bool, len(msgs))
for _, m := range msgs {
pageIDs[m.ID] = true
}
items := make([]queueItem, len(msgs))
for i, m := range msgs {
c, ok := s.queueReasons[m.ID]
if !ok {
if len(s.queueReasons) == queueListMaxLimit {
var oldestID string
var oldest time.Time
for id, entry := range s.queueReasons {
if !pageIDs[id] && (oldestID == "" || entry.read.Before(oldest)) {
oldestID, oldest = id, entry.read
}
}
delete(s.queueReasons, oldestID)
}
c.read = now
c.reason, _ = s.msglogReason(m.ID)
s.queueReasons[m.ID] = c
}
items[i] = queueItem{QueueMessage: m, Reason: queueReason(m, c.reason)}
}
return items
}
// msglogReason reads a message's msglog. A log CSM refuses to read, such as a
// symlink, is reported as unreadable rather than as no attempt.
func (s *Server) msglogReason(id string) (intel.StuckReason, []string) {
reason, lines, err := s.queueAdmin.Reason(id)
if err != nil {
return intel.StuckReason{Class: "other", Text: "delivery log unreadable", Raw: err.Error()}, nil
}
return reason, lines
}
// queueReason corrects what the msglog cannot know: a message exim cannot
// read is stuck for that reason, and a message thawed since it froze is
// waiting for its next delivery attempt.
func queueReason(m intel.QueueMessage, r intel.StuckReason) intel.StuckReason {
if m.Unreadable != "" {
return intel.StuckReason{Class: "unreadable", Text: "exim cannot read this message", Raw: m.Unreadable}
}
if r.Class == "frozen" && !m.Frozen {
r.Class, r.Text = "retry_wait", "thawed, waiting for the next delivery attempt"
}
return r
}
// apiEmailQueueMessage handles GET /api/v1/email/queue/{id}: one queued
// message with all its undelivered recipients, why it is stuck and the last
// lines of its msglog, read live.
func (s *Server) apiEmailQueueMessage(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
if !intel.ValidMessageID(id) {
writeJSONError(w, "Invalid message ID", http.StatusBadRequest)
return
}
if s.queueAdmin == nil {
writeJSONError(w, queueUnavailableText, http.StatusServiceUnavailable)
return
}
m, err := s.queueAdmin.Message(id)
if errors.Is(err, intel.ErrNotQueued) {
writeJSONError(w, "Message is not in the queue", http.StatusNotFound)
return
}
if errors.Is(err, systemdrun.ErrUnavailable) {
writeJSONError(w, "Reading the queue needs systemd-run: "+err.Error(), http.StatusServiceUnavailable)
return
}
if err != nil {
writeJSONError(w, "Failed to read the message from the queue: "+err.Error(), http.StatusInternalServerError)
return
}
reason, lines := s.msglogReason(id)
if lines == nil {
lines = []string{}
}
writeJSON(w, queueMessageDetail{QueueMessage: m, Reason: queueReason(m, reason), Msglog: lines})
}
// apiEmailQueueAction handles POST /api/v1/email/queue/{id}/{action} for
// retry, bounce, delete and thaw. The answer carries the outcome the queue
// shows afterwards, and every attempt is audit-logged with it and with any
// error exim reported, including for work the queue shows done.
func (s *Server) apiEmailQueueAction(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
id, action := r.PathValue("id"), r.PathValue("action")
if !intel.ValidMessageID(id) {
writeJSONError(w, "Invalid message ID", http.StatusBadRequest)
return
}
if !intel.ValidQueueAction(action) {
writeJSONError(w, "Unknown action: use retry, bounce, delete or thaw", http.StatusBadRequest)
return
}
if s.queueAdmin == nil {
writeJSONError(w, queueUnavailableText, http.StatusServiceUnavailable)
return
}
outcome, err := s.queueAdmin.Act(action, id)
details := "outcome: " + outcome
if err != nil {
details += "; " + err.Error()
}
s.auditLog(r, "email_queue_"+action, id, details)
s.queueReasonMu.Lock()
delete(s.queueReasons, id)
s.queueReasonMu.Unlock()
expireQueueListing()
switch {
case outcome == intel.OutcomeGone:
writeJSONStatus(w, http.StatusConflict, map[string]string{"error": "Message is no longer in the queue", "outcome": outcome})
case outcome != intel.OutcomeUnchanged || err == nil:
writeOK(w, map[string]interface{}{"outcome": outcome})
case errors.Is(err, systemdrun.ErrUnavailable):
writeJSONStatus(w, http.StatusServiceUnavailable, map[string]string{"error": "Queue actions need systemd-run: " + err.Error(), "outcome": outcome})
default:
writeJSONStatus(w, http.StatusInternalServerError, map[string]string{"error": "Queue action failed: " + err.Error(), "outcome": outcome})
}
}
package webui
import (
"net/http"
"slices"
"strings"
"time"
"github.com/pidginhost/csm/internal/mailstats"
)
const (
emailRejectsDefaultLimit = 50
emailRejectsMaxLimit = 200
)
// contentRejectClasses are the rejects the message content earned. They
// are what class=content, the default, lists.
var contentRejectClasses = []mailstats.Class{mailstats.ClassSpam, mailstats.ClassMalware}
// rejectClasses resolves the class parameter: content, all (nil), or
// class names joined by commas. ok is false for anything else.
func rejectClasses(class string) (classes []mailstats.Class, ok bool) {
switch class {
case "content":
return contentRejectClasses, true
case "all":
return nil, true
}
for name := range strings.SplitSeq(class, ",") {
c := mailstats.Class(name)
if !slices.Contains(mailstats.Classes, c) {
return nil, false
}
classes = append(classes, c)
}
return classes, true
}
// permanentRejects counts the permanent rejects of every class in an
// overview: its rejects less its temporary ones.
func permanentRejects(t mailstats.Totals) map[mailstats.Class]int {
out := make(map[mailstats.Class]int, len(mailstats.Classes))
for _, c := range mailstats.Classes {
out[c] = t.Rejects[c] - t.Temporary[c]
}
return out
}
// apiEmailRejects handles GET /api/v1/email/rejects: the newest rejects the
// mail decision counter keeps for from/to (see emailRange), and the
// permanent rejects of every class in the range. Each entry names a sender
// and a recipient, so the route is admin only.
func (s *Server) apiEmailRejects(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
class := q.Get("class")
if class == "" {
class = "content"
}
classes, ok := rejectClasses(class)
if !ok {
writeJSONError(w, "Invalid class: use content, all or reject class names joined by commas", http.StatusBadRequest)
return
}
limit := queryInt(r, "limit", emailRejectsDefaultLimit)
if limit <= 0 {
limit = emailRejectsDefaultLimit
}
limit = min(limit, emailRejectsMaxLimit)
from, to, ok := emailRange(w, q, time.Now())
if !ok {
return
}
var items []mailstats.Reject
var totals mailstats.Totals
if c := s.mailStatsCounter(); c != nil {
items = c.Rejects(from, to, classes, limit)
totals = c.Overview(from, to).Totals
}
writeItems(w, items, map[string]interface{}{
"class": class,
"limit": limit,
"kept_days": int(mailstats.RingAge / (24 * time.Hour)),
"totals": permanentRejects(totals),
})
}
package webui
import (
"cmp"
"context"
"net/http"
"time"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/antispam/hostfacts"
csmlog "github.com/pidginhost/csm/internal/log"
)
// rspamd's numbers are cached this long, and one read of its controller
// may take at most rspamdFetchTimeout.
const (
rspamdStatsTTL = 60 * time.Second
rspamdFetchTimeout = 10 * time.Second
)
// Fixed reasons for the rspamd card. A read error is never shown: it can
// name the controller's address.
const (
rspamdNoAnswer = "rspamd's controller did not answer"
rspamdNotChecked = "rspamd's controller has not been checked yet"
rspamdOff = "antispam is off"
)
// readRspamdStats reads rspamd's controller; tests replace it.
var readRspamdStats = hostfacts.ReadRspamdStats
// emailRspamdResponse is the Email page's rspamd card. Status and mode are
// current; stats or the reason there are none can be up to rspamdStatsTTL
// old, as FetchedAt says.
type emailRspamdResponse struct {
Status string `json:"status"` // up | down | off
Mode string `json:"mode"` // enforce | dry-run | off
Version string `json:"version,omitempty"`
Available bool `json:"available"`
Reason string `json:"reason,omitempty"`
Stats *antispam.RspamdStats `json:"stats,omitempty"`
FetchedAt time.Time `json:"fetched_at"`
}
// rspamdFetch is one read of the controller at ep: stats, or the reason
// there are none.
type rspamdFetch struct {
ep antispam.StatsEndpoint
stats *antispam.RspamdStats
reason string
at time.Time
}
// apiEmailRspamd handles GET /api/v1/email/rspamd: whether rspamd is up, the
// antispam mode, and rspamd's own counters since it started.
func (s *Server) apiEmailRspamd(w http.ResponseWriter, r *http.Request) {
cfg := s.liveCfg()
resp := emailRspamdResponse{Status: "off", Mode: antispamMode(cfg), FetchedAt: time.Now()}
if !cfg.EmailProtection.Antispam.Enabled {
resp.Reason = rspamdOff
writeJSON(w, resp)
return
}
var ep antispam.StatsEndpoint
var reason string
var down bool
if s.rspamdSource != nil {
ep, reason, down = s.rspamdSource()
}
resp.Status = "up"
if down {
resp.Status = "down"
}
resp.Version = ep.Version
if ep.Address == "" {
resp.Reason = cmp.Or(reason, rspamdNotChecked)
writeJSON(w, resp)
return
}
f := s.rspamdStats(r.Context(), ep)
resp.Available, resp.Stats, resp.Reason, resp.FetchedAt = f.stats != nil, f.stats, f.reason, f.at
if f.stats != nil {
resp.Version = cmp.Or(f.stats.Version, ep.Version)
}
writeJSON(w, resp)
}
// rspamdStats returns the cached read of ep, reading the controller again
// once the cache is older than rspamdStatsTTL or holds another endpoint.
// The lock is held across the read so a burst of requests reads once. The
// read outlives a client that goes away, so its result is about rspamd and
// can be cached for the requests waiting on it.
func (s *Server) rspamdStats(ctx context.Context, ep antispam.StatsEndpoint) rspamdFetch {
s.rspamdMu.Lock()
defer s.rspamdMu.Unlock()
if c := s.rspamdCache; c.ep == ep && time.Since(c.at) < rspamdStatsTTL {
return c
}
readCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), rspamdFetchTimeout)
defer cancel()
stats, err := readRspamdStats(readCtx, ep)
f := rspamdFetch{ep: ep, at: time.Now()}
if err != nil {
// The read's errors are fixed text: no address, URL or password.
csmlog.Warn("email page could not read rspamd's controller", "err", err)
f.reason = rspamdNoAnswer
} else {
f.stats = &stats
}
s.rspamdCache = f
return f
}
package webui
import (
"net/http"
"time"
"github.com/pidginhost/csm/internal/mailstats"
)
// emailSendersLimit bounds the account rows; the totals stay whole.
const emailSendersLimit = 100
// emailSendersResponse is the outgoing mail of a range by hosting account,
// how many outbound abuse events the range holds, and how many of them name
// each account, by kind.
type emailSendersResponse struct {
mailstats.SendersReport
AbuseEvents int `json:"abuse_events"`
AbuseByAccount map[string]map[string]int `json:"abuse_by_account"`
// AbuseTruncated says the scan cap cut AbuseEvents.
AbuseTruncated bool `json:"abuse_events_truncated"`
}
// apiEmailSenders handles GET /api/v1/email/senders: the messages each
// hosting account sent in from/to (see emailRange), busiest first, and the
// count of the outbound abuse events /api/v1/email/relay-abuse lists for the
// same range, under the same scan cap and memo, with those naming an account
// counted per account and kind. The rows name accounts, so the route is admin
// only.
func (s *Server) apiEmailSenders(w http.ResponseWriter, r *http.Request) {
from, to, ok := emailRange(w, r.URL.Query(), time.Now())
if !ok {
return
}
resp := emailSendersResponse{SendersReport: mailstats.SendersReport{Accounts: []mailstats.AccountSends{}}}
if c := s.mailStatsCounter(); c != nil {
resp.SendersReport = c.TopSenders(from, to, emailSendersLimit)
}
abuse := s.buildRelayAbuseResponse(from, to, 1)
resp.AbuseEvents, resp.AbuseByAccount, resp.AbuseTruncated = abuse.Total, abuse.byAccount, abuse.TotalTruncated
writeJSON(w, resp)
}
package webui
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/platform"
)
// featureTTL is how long a platform snapshot is reused. Detecting the mail
// server asks systemd, and a mail server can start after CSM does, so the
// snapshot is neither taken per request nor kept for the process lifetime.
const featureTTL = 5 * time.Minute
// cpanelMailBackend reports whether the cPanel/Exim adapters behind the mail
// queue, held copies, forwarders, deferrals and suspensions can run here. The
// API backends and the UI flags share it so a tab never shows without its API.
func cpanelMailBackend(info platform.Info) bool { return info.IsCPanel() }
// uiFeatures maps the detected panel and mail server to the named features
// the Web UI renders. Templates, the settings schema and the page scripts
// all read these names instead of testing the platform themselves.
func uiFeatures(info platform.Info, mta platform.MTAKind) map[string]bool {
cp := cpanelMailBackend(info)
return map[string]bool{
"cpanel": info.IsCPanel(),
"mail": cp || mta != platform.MTAUnknown,
"mailQueue": cp,
"mailHeld": cp,
"mailForwarders": cp,
"mailDeferrals": cp,
"mailSuspensions": cp || mta == platform.MTAStalwart,
"emailAV": cp,
"cphulk": info.IsCPanel(),
"stalwartFilter": mta == platform.MTAStalwart,
}
}
// featureSnapshot is one platform detection and the features derived from it.
type featureSnapshot struct {
at time.Time
features map[string]bool
}
// hostPlatform detects this host's panel and mail server. The package tests
// replace it so no test depends on what the build host runs.
var hostPlatform = func() (platform.Info, platform.MTAKind) {
return platform.Detect(), platform.DetectMTA()
}
type featureState struct {
// detectPlatform and featureNow are nil outside tests.
detectPlatform func() (platform.Info, platform.MTAKind)
featureNow func() time.Time
featureMu sync.Mutex
featureCache featureSnapshot
// featureRefreshing is set while one caller re-detects; the others
// keep the previous snapshot instead of waiting on systemd.
featureRefreshing bool
}
func (s *Server) platformSnapshot() featureSnapshot {
now := time.Now()
if s.featureNow != nil {
now = s.featureNow()
}
s.featureMu.Lock()
c := s.featureCache
if c.features != nil && (now.Sub(c.at) < featureTTL || s.featureRefreshing) {
s.featureMu.Unlock()
return c
}
s.featureRefreshing = c.features != nil
s.featureMu.Unlock()
// Detection runs unlocked: it can wait on systemd for seconds.
var info platform.Info
var mta platform.MTAKind
if s.detectPlatform != nil {
info, mta = s.detectPlatform()
} else {
info, mta = hostPlatform()
}
snap := featureSnapshot{at: now, features: uiFeatures(info, mta)}
s.featureMu.Lock()
s.featureCache = snap
s.featureRefreshing = false
s.featureMu.Unlock()
return snap
}
// features returns the feature flags for this host. The map is shared;
// callers must not modify it.
func (s *Server) features() map[string]bool { return s.platformSnapshot().features }
// feature reports whether the named feature is available on this host.
// Unknown names are off.
func (s *Server) feature(name string) bool { return s.features()[name] }
// filterSettingsSection returns sec as a host with the given features sees
// it: ok is false when the section itself needs a missing feature, and
// fields that need one are left out. The schema is not modified.
func filterSettingsSection(sec SettingsSection, features map[string]bool) (SettingsSection, bool) {
if sec.Feature != "" && !features[sec.Feature] {
return SettingsSection{}, false
}
fields := make([]SettingsField, 0, len(sec.Fields))
for _, f := range sec.Fields {
if f.Feature == "" || features[f.Feature] {
fields = append(fields, f)
}
}
sec.Fields = fields
return sec, true
}
// settingsSections lists the settings sections this host shows.
func (s *Server) settingsSections() []SettingsSection {
features := s.features()
all := AllSettingsSections()
out := make([]SettingsSection, 0, len(all))
for _, sec := range all {
if f, ok := filterSettingsSection(sec, features); ok {
out = append(out, f)
}
}
return out
}
// lookupSettingsSection returns a section as this host shows it. Reads and
// saves both go through it, so a hidden field is neither returned nor
// accepted, and its value on disk stays as it is.
func (s *Server) lookupSettingsSection(id string) (SettingsSection, bool) {
sec, ok := LookupSettingsSection(id)
if !ok {
return SettingsSection{}, false
}
return filterSettingsSection(sec, s.features())
}
package webui
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"os/exec"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
csmlog "github.com/pidginhost/csm/internal/log"
"github.com/pidginhost/csm/internal/platform"
)
type firewallAllowView struct {
IP string `json:"ip"`
Reason string `json:"reason"`
Source string `json:"source"`
// ExpiresAt is left out for a permanent rule.
ExpiresAt time.Time `json:"expires_at,omitzero"`
}
type firewallPortAllowView struct {
IP string `json:"ip"`
Port int `json:"port"`
Proto string `json:"proto"`
Reason string `json:"reason"`
Source string `json:"source"`
}
const cphulkFirewallCheckTimeout = 8 * time.Second
var firewallCheckCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) {
// #nosec G204 -- command names are fixed by trusted call sites; HTTP input
// is parsed as an IP and passed as an execve argument without shell expansion.
return exec.CommandContext(ctx, name, args...).Output()
}
// apiFirewallStatus returns the firewall engine configuration and state summary.
func (s *Server) apiFirewallStatus(w http.ResponseWriter, _ *http.Request) {
cfg := config.EffectiveFirewallConfig(s.liveCfg())
state, err := firewall.LoadState(s.cfg.StatePath)
if err != nil {
writeJSONError(w, "firewall state unavailable (corrupt state file)", http.StatusInternalServerError)
return
}
now := time.Now()
blockedPermanent := 0
blockedTemporary := 0
for _, entry := range state.Blocked {
if entry.ExpiresAt.IsZero() {
blockedPermanent++
continue
}
if now.Before(entry.ExpiresAt) {
blockedTemporary++
}
}
allowPermanent := 0
allowTemporary := 0
for _, entry := range state.Allowed {
if entry.ExpiresAt.IsZero() {
allowPermanent++
continue
}
if now.Before(entry.ExpiresAt) {
allowTemporary++
}
}
portAllowCount := len(state.PortAllowed)
for _, rule := range cfg.TCPInAllow {
portAllowCount += len(rule.Sources)
}
result := map[string]interface{}{
"enabled": cfg.Enabled,
"ipv6": cfg.IPv6,
"tcp_in": cfg.TCPIn,
"tcp_out": cfg.TCPOut,
"udp_in": cfg.UDPIn,
"udp_out": cfg.UDPOut,
"restricted_tcp": cfg.RestrictedTCP,
"passive_ftp": [2]int{cfg.PassiveFTPStart, cfg.PassiveFTPEnd},
"conn_rate_limit": cfg.ConnRateLimit,
"conn_limit": cfg.ConnLimit,
"syn_flood_protection": cfg.SYNFloodProtection,
"udp_flood": cfg.UDPFlood,
"smtp_block": cfg.SMTPBlock,
"log_dropped": cfg.LogDropped,
"deny_ip_limit": cfg.DenyIPLimit,
"blocked_count": blockedPermanent + blockedTemporary,
"blocked_net_count": len(state.BlockedNet),
"blocked_permanent": blockedPermanent,
"blocked_temporary": blockedTemporary,
"allowed_count": allowPermanent + allowTemporary,
"allow_permanent": allowPermanent,
"allow_temporary": allowTemporary,
"port_allow_count": portAllowCount,
"infra_ips": cfg.InfraIPs,
"infra_count": len(cfg.InfraIPs),
"port_flood_rules": len(cfg.PortFlood),
"country_block": cfg.CountryBlock,
"dyndns_hosts": cfg.DynDNSHosts,
}
writeJSON(w, result)
}
// apiFirewallAllowed returns active firewall allow rules and port exceptions.
func (s *Server) apiFirewallAllowed(w http.ResponseWriter, _ *http.Request) {
state, err := firewall.LoadState(s.cfg.StatePath)
if err != nil {
writeJSONError(w, "firewall state unavailable (corrupt state file)", http.StatusInternalServerError)
return
}
now := time.Now()
allowed := make([]firewallAllowView, 0, len(state.Allowed))
for _, entry := range state.Allowed {
if !entry.ExpiresAt.IsZero() && !now.Before(entry.ExpiresAt) {
continue
}
view := firewallAllowView{
IP: entry.IP,
Reason: entry.Reason,
Source: entry.Source,
ExpiresAt: entry.ExpiresAt.UTC(),
}
if view.Source == "" {
view.Source = firewall.InferProvenance("allow", entry.Reason)
}
allowed = append(allowed, view)
}
sort.Slice(allowed, func(i, j int) bool {
return allowed[i].IP < allowed[j].IP
})
portAllowed := make([]firewallPortAllowView, 0, len(state.PortAllowed))
for _, entry := range state.PortAllowed {
portAllowed = append(portAllowed, firewallPortAllowView{
IP: entry.IP,
Port: entry.Port,
Proto: entry.Proto,
Reason: entry.Reason,
Source: entry.Source,
})
if portAllowed[len(portAllowed)-1].Source == "" {
portAllowed[len(portAllowed)-1].Source = firewall.InferProvenance("allow_port", entry.Reason)
}
}
// Ports opened from config are exceptions too; state holds only the
// ones added with `csm firewall allow-port`.
for _, rule := range config.EffectiveFirewallConfig(s.liveCfg()).TCPInAllow {
for _, source := range rule.Sources {
portAllowed = append(portAllowed, firewallPortAllowView{
IP: source,
Port: rule.Port,
Proto: "tcp",
Reason: "Declared in firewall.tcp_in_allow",
Source: firewall.SourceConfig,
})
}
}
sort.Slice(portAllowed, func(i, j int) bool {
if portAllowed[i].IP != portAllowed[j].IP {
return portAllowed[i].IP < portAllowed[j].IP
}
if portAllowed[i].Port != portAllowed[j].Port {
return portAllowed[i].Port < portAllowed[j].Port
}
return portAllowed[i].Proto < portAllowed[j].Proto
})
writeJSON(w, map[string]interface{}{
"allowed": allowed,
"port_allowed": portAllowed,
})
}
// apiFirewallAllowIP adds a firewall allow rule, temporary when duration > 0.
func (s *Server) apiFirewallAllowIP(w http.ResponseWriter, r *http.Request) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IP string `json:"ip"`
Reason string `json:"reason"`
Duration string `json:"duration"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.IP == "" {
writeJSONError(w, "IP is required", http.StatusBadRequest)
return
}
parsedIP, err := parseAllowIP(req.IP)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
// Audit, incident and threat records key on the canonical spelling.
req.IP = parsedIP.String()
if req.Reason == "" {
req.Reason = "Allowed via CSM Web UI"
}
dur, err := parseDuration(req.Duration)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
if dur > 0 {
allower, ok := s.blocker.(ipTempAllower)
if !ok || allower == nil {
writeJSONError(w, "Firewall allow rules are not available", http.StatusServiceUnavailable)
return
}
if err := allower.TempAllowIP(req.IP, req.Reason, dur); err != nil {
writeJSONError(w, fmt.Sprintf("Allow failed: %v", err), http.StatusInternalServerError)
return
}
s.auditLog(r, "firewall_allow", req.IP, fmt.Sprintf("temporary allow %s: %s", dur, req.Reason))
writeOK(w, map[string]interface{}{"ip": req.IP, "temporary": true})
return
}
allower, ok := s.blocker.(ipAllower)
if !ok || allower == nil {
writeJSONError(w, "Firewall allow rules are not available", http.StatusServiceUnavailable)
return
}
if err := allower.AllowIP(req.IP, req.Reason); err != nil {
writeJSONError(w, fmt.Sprintf("Allow failed: %v", err), http.StatusInternalServerError)
return
}
s.auditLog(r, "firewall_allow", req.IP, "permanent allow: "+req.Reason)
writeOK(w, map[string]interface{}{"ip": req.IP, "temporary": false})
}
// apiFirewallRemoveAllow removes a firewall allow rule.
func (s *Server) apiFirewallRemoveAllow(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IP string `json:"ip"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.IP == "" {
writeJSONError(w, "IP is required", http.StatusBadRequest)
return
}
parsedIP, err := parseAndValidateIP(req.IP)
if err != nil {
writeJSONError(w, fmt.Sprintf("invalid IP address: %s", req.IP), http.StatusBadRequest)
return
}
// Audit, incident and threat records key on the canonical spelling.
req.IP = parsedIP.String()
allower, ok := s.blocker.(allowRemover)
if !ok || allower == nil {
writeJSONError(w, "Firewall allow rules are not available", http.StatusServiceUnavailable)
return
}
if err := allower.RemoveAllowIP(req.IP); err != nil {
writeJSONError(w, fmt.Sprintf("Remove failed: %v", err), http.StatusInternalServerError)
return
}
s.auditLog(r, "firewall_remove_allow", req.IP, "removed allow rule")
writeOK(w, map[string]interface{}{"ip": req.IP})
}
// apiFirewallAudit returns recent firewall audit log entries.
func (s *Server) apiFirewallAudit(w http.ResponseWriter, r *http.Request) {
limit := queryInt(r, "limit", 100)
// Filters apply to the whole log and the limit to what they matched, so a
// search reaches entries older than the newest page.
entries := firewall.ReadAuditLog(s.cfg.StatePath, 0)
type auditView struct {
Timestamp time.Time `json:"timestamp"`
Action string `json:"action"`
IP string `json:"ip"`
Reason string `json:"reason"`
Source string `json:"source"`
// DurationSeconds is the block or allow lifetime; left out when
// the action had none.
DurationSeconds float64 `json:"duration_seconds,omitempty"`
}
search := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("search")))
actionFilter := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("action")))
sourceFilter := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("source")))
var result []auditView
for _, e := range entries {
source := e.Source
if source == "" {
source = firewall.InferProvenance(e.Action, e.Reason)
}
if actionFilter != "" && strings.ToLower(e.Action) != actionFilter {
continue
}
if sourceFilter != "" && strings.ToLower(source) != sourceFilter {
continue
}
if search != "" {
haystack := strings.ToLower(strings.Join([]string{e.Action, e.IP, e.Reason, source}, " "))
if !strings.Contains(haystack, search) {
continue
}
}
view := auditView{
Timestamp: e.Timestamp.UTC(),
Action: e.Action,
IP: e.IP,
Reason: e.Reason,
Source: source,
}
// The log stores the lifetime as Go duration text such as "24h0m0s".
if secs, ok := durationSeconds(e.Duration); ok {
view.DurationSeconds = secs
}
result = append(result, view)
}
if limit == 0 {
writeAll(w, result)
return
}
// The newest entries are last in the log.
total := len(result)
if total > limit {
result = result[total-limit:]
}
writeCapped(w, result, total, limit, nil)
}
// apiFirewallSubnets returns currently blocked subnets.
func (s *Server) apiFirewallSubnets(w http.ResponseWriter, _ *http.Request) {
state, err := firewall.LoadState(s.cfg.StatePath)
if err != nil {
writeJSONError(w, "firewall state unavailable (corrupt state file)", http.StatusInternalServerError)
return
}
type subnetView struct {
CIDR string `json:"cidr"`
Reason string `json:"reason"`
Source string `json:"source"`
BlockedAt time.Time `json:"blocked_at,omitzero"`
// ExpiresAt is left out for a permanent block.
ExpiresAt time.Time `json:"expires_at,omitzero"`
}
var result []subnetView
for _, sn := range state.BlockedNet {
v := subnetView{
CIDR: sn.CIDR,
Reason: sn.Reason,
Source: sn.Source,
BlockedAt: sn.BlockedAt.UTC(),
ExpiresAt: sn.ExpiresAt.UTC(),
}
if v.Source == "" {
v.Source = firewall.InferProvenance("block_subnet", sn.Reason)
}
result = append(result, v)
}
writeAll(w, result)
}
// apiFirewallDenySubnet blocks a subnet via the firewall engine.
func (s *Server) apiFirewallDenySubnet(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
CIDR string `json:"cidr"`
Reason string `json:"reason"`
Duration string `json:"duration"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.CIDR == "" {
writeJSONError(w, "CIDR is required", http.StatusBadRequest)
return
}
if _, err := validateCIDR(req.CIDR); err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
if req.Reason == "" {
req.Reason = "Blocked via CSM Web UI"
}
dur, err := parseDuration(req.Duration)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
sb, ok := s.blocker.(subnetBlocker)
if !ok || sb == nil {
writeJSONError(w, "Firewall engine not available", http.StatusServiceUnavailable)
return
}
if err := sb.BlockSubnet(req.CIDR, req.Reason, dur); err != nil {
writeJSONError(w, fmt.Sprintf("Block failed: %v", err), http.StatusInternalServerError)
return
}
lifetime := "permanent"
if dur > 0 {
lifetime = dur.String()
}
s.auditLog(r, "firewall_deny_subnet", req.CIDR, lifetime+": "+req.Reason)
writeOK(w, map[string]interface{}{"cidr": req.CIDR})
}
// apiFirewallRemoveSubnet removes a subnet block.
func (s *Server) apiFirewallRemoveSubnet(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
CIDR string `json:"cidr"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.CIDR == "" {
writeJSONError(w, "CIDR is required", http.StatusBadRequest)
return
}
if _, _, err := net.ParseCIDR(req.CIDR); err != nil {
writeJSONError(w, "Invalid CIDR notation", http.StatusBadRequest)
return
}
sb, ok := s.blocker.(subnetUnblocker)
if !ok || sb == nil {
writeJSONError(w, "Firewall engine not available", http.StatusServiceUnavailable)
return
}
if err := sb.UnblockSubnet(req.CIDR); err != nil {
writeJSONError(w, fmt.Sprintf("Remove failed: %v", err), http.StatusInternalServerError)
return
}
s.auditLog(r, "firewall_remove_subnet", req.CIDR, "removed subnet block")
writeOK(w, map[string]interface{}{"cidr": req.CIDR})
}
// apiFirewallFlush clears all blocked IPs.
func (s *Server) apiFirewallFlush(w http.ResponseWriter, r *http.Request) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
fb, ok := s.blocker.(blockFlusher)
if !ok || fb == nil {
writeJSONError(w, "Firewall engine not available", http.StatusServiceUnavailable)
return
}
result, err := checks.FlushAutoBlockState(s.cfg.StatePath, fb.FlushBlocked)
if result.SnapshotErr != nil {
csmlog.Warn("web firewall flush could not snapshot persisted blocks", "err", result.SnapshotErr)
}
if err != nil {
if result.Flushed {
writeJSONError(w, fmt.Sprintf("Firewall flushed but auto-block cleanup failed: %v", err), http.StatusInternalServerError)
} else {
writeJSONError(w, fmt.Sprintf("Flush failed: %v", err), http.StatusInternalServerError)
}
return
}
s.auditLog(r, "firewall_flush", "blocked set", fmt.Sprintf("flushed: %v", result.Flushed))
writeOK(w, nil)
}
// apiFirewallFlushCphulk clears cPHulk login history for one IP without touching firewall state.
func (s *Server) apiFirewallFlushCphulk(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IP string `json:"ip"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.IP == "" {
writeJSONError(w, "IP is required", http.StatusBadRequest)
return
}
parsedIP, err := parseAndValidateIP(req.IP)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
// Audit, incident and threat records key on the canonical spelling.
req.IP = parsedIP.String()
if err := flushCphulk(req.IP); err != nil {
writeJSONError(w, "Could not clear cPHulk login history: "+err.Error(), http.StatusInternalServerError)
return
}
s.auditLog(r, "cphulk_clear", req.IP, "cleared cPHulk login history")
writeOK(w, map[string]interface{}{"ip": req.IP})
}
// apiFirewallCheck checks if an IP is blocked in CSM or cphulk.
// GET /api/v1/firewall/check?ip=1.2.3.4
// phclient calls this route and reads success, permanent, temporary and
// cphulk (the cpanel-service shape it replaced), so the body keeps
// "success": true next to the fields; it is a deprecated alias:
//
// {"success": true, "ip": "1.2.3.4", "permanent": "reason or null", "temporary": "reason or null", "cphulk": true/false}
//
// Failures are error statuses like every other route; phclient treats any
// non-2xx as a failed call.
func (s *Server) apiFirewallCheck(w http.ResponseWriter, r *http.Request) {
ip := r.URL.Query().Get("ip")
if ip == "" || net.ParseIP(ip) == nil {
writeJSONError(w, "The ip is not valid or it was not set.", http.StatusBadRequest)
return
}
result := map[string]interface{}{
"success": true,
"ip": ip,
"permanent": nil,
"temporary": nil,
"cphulk": false,
}
// Check CSM firewall state
state, err := firewall.LoadState(s.cfg.StatePath)
if err != nil {
// A corrupt state file means we cannot tell whether the IP is
// blocked; report failure rather than a misleading "not blocked".
writeJSONError(w, "Firewall state unavailable", http.StatusInternalServerError)
return
}
now := time.Now()
// Compare as parsed addresses: an IPv6 block saved in one spelling
// must be found when queried in another.
queried := net.ParseIP(ip)
for _, b := range state.Blocked {
if entryIP := net.ParseIP(b.IP); entryIP != nil && entryIP.Equal(queried) {
if b.ExpiresAt.IsZero() {
result["permanent"] = b.Reason
} else if now.Before(b.ExpiresAt) {
// temporary keeps its text form for existing callers;
// expires_at is the instant.
result["temporary"] = fmt.Sprintf("%s (expires in %s)", b.Reason,
time.Until(b.ExpiresAt).Truncate(time.Minute))
result["expires_at"] = b.ExpiresAt.UTC()
}
}
}
// Check blocked subnets
parsedIP := net.ParseIP(ip)
for _, sn := range state.BlockedNet {
_, network, err := net.ParseCIDR(sn.CIDR)
if err == nil && network.Contains(parsedIP) {
result["permanent"] = fmt.Sprintf("Subnet block: %s - %s", sn.CIDR, sn.Reason)
}
}
// Check cphulk (cPanel brute force detector) - read-only check.
if platform.Detect().IsCPanel() && cphulkTempBanContainsIP(r.Context(), ip) {
result["cphulk"] = true
} else {
cphulkOut, cphulkErr := runFirewallCheckCommand(r.Context(), "whmapi1", "read_cphulk_records",
"list_name=black", "--output=json")
if cphulkErr == nil && cphulkBlocksIP(cphulkOut, ip) {
result["cphulk"] = true
}
}
writeJSON(w, result)
}
// cphulkBlocksIP scopes the match to cPHulk record IP fields. A raw token
// search would still match unrelated strings such as operator notes.
func cphulkBlocksIP(jsonOut []byte, ip string) bool {
if ip == "" {
return false
}
var payload struct {
Data struct {
Records []map[string]json.RawMessage `json:"records"`
} `json:"data"`
}
if err := json.Unmarshal(jsonOut, &payload); err != nil {
return false
}
for _, record := range payload.Data.Records {
for key, raw := range record {
if !isCphulkRecordIPField(key) {
continue
}
var value string
if err := json.Unmarshal(raw, &value); err != nil {
continue
}
if value == ip {
return true
}
}
}
return false
}
func isCphulkRecordIPField(key string) bool {
switch strings.ToLower(key) {
case "ip", "ip_address":
return true
default:
return false
}
}
func runFirewallCheckCommand(parent context.Context, name string, args ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(parent, cphulkFirewallCheckTimeout)
defer cancel()
return firewallCheckCommandOutput(ctx, name, args...)
}
// cPHulk brute-force temp bans live in the cphulk-TempBan nftables set, not in
// read_cphulk_records, so ask nftables for exact element membership instead of
// scanning a textual set dump where one IP could match unrelated text.
func cphulkTempBanContainsIP(parent context.Context, ip string) bool {
lookupIP, ok := cphulkTempBanLookupIP(ip)
if !ok {
return false
}
_, err := runFirewallCheckCommand(parent, "nft", "get", "element",
"inet", "filter", "cphulk-TempBan", "{", lookupIP, "}")
return err == nil
}
func cphulkTempBanLookupIP(ip string) (string, bool) {
parsed := net.ParseIP(strings.TrimSpace(ip))
if parsed == nil {
return "", false
}
if ip4 := parsed.To4(); ip4 != nil {
return net.IP(ip4).String(), true
}
return parsed.String(), true
}
// apiFirewallUnban unblocks an IP from CSM + cphulk in one call.
// POST /api/v1/firewall/unban body: {"ip": "1.2.3.4"}
func (s *Server) apiFirewallUnban(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IP string `json:"ip"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.IP == "" {
writeJSONError(w, "The ip is not valid or it was not set.", http.StatusBadRequest)
return
}
parsedIP, err := parseAndValidateIP(req.IP)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
// Audit, incident and threat records key on the canonical spelling.
req.IP = parsedIP.String()
// 1. Unblock from CSM firewall (individual IP). An unban-scope token is
// a hosting panel's self-service unban: it lifts the block but keeps the
// address's mail failure history, so renewed spraying is blocked again.
// Only an admin credential makes this an operator release.
if s.blocker != nil {
var unblockErr error
if s.adminCredential(r) {
unblockErr = s.blocker.UnblockIP(req.IP)
} else if u, ok := s.blocker.(selfUnbanner); ok {
unblockErr = u.UnbanIP(req.IP)
} else {
writeJSONError(w, "Self-service unban is not supported", http.StatusNotImplemented)
return
}
if unblockErr != nil {
writeJSONError(w, "Failed to unblock IP: "+unblockErr.Error(), http.StatusInternalServerError)
return
}
}
checks.DropAutoBlockThreatRow(req.IP)
// 2. Also remove covering subnet blocks that CSM added automatically. A
// subnet an operator blocked, or whose automatic ownership is unproven,
// stays: one address's unban must not reopen it, whoever asks.
// A corrupt state file only costs us the subnet sweep; the IP unblock
// above already ran, so skip this step rather than fail the whole unban.
state, stateErr := firewall.LoadState(s.cfg.StatePath)
subnetRemoved, subnetKept := "", ""
if stateErr == nil && state != nil {
for _, sn := range state.BlockedNet {
_, network, err := net.ParseCIDR(sn.CIDR)
if err != nil || !network.Contains(parsedIP) {
continue
}
if !sn.Automatic {
if subnetKept == "" {
subnetKept = sn.CIDR
}
continue
}
sb, ok := s.blocker.(conditionalSubnetUnblocker)
if !ok {
writeJSONError(w, "Safe subnet removal is not supported: "+sn.CIDR, http.StatusNotImplemented)
return
}
if err := sb.UnblockSubnetIfUnchanged(sn); err != nil {
message := "Failed to remove subnet " + sn.CIDR + ": " + err.Error()
if errors.Is(err, firewall.ErrBlockChanged) {
message = "Subnet " + sn.CIDR + " changed during unban; retry the request"
}
writeJSONError(w, message, http.StatusInternalServerError)
return
}
if subnetRemoved == "" {
subnetRemoved = sn.CIDR
}
}
}
// 3. Flush from cphulk
_ = flushCphulk(req.IP) // best effort: the unblock is what was asked
// "success" is the deprecated alias phclient reads; see apiFirewallCheck.
result := map[string]interface{}{"ok": true, "success": true, "ip": req.IP}
details := "unblock, clear auto-block state, flush cPHulk"
if subnetRemoved != "" {
result["subnet_removed"] = subnetRemoved
details += ", removed subnet " + subnetRemoved
}
if subnetKept != "" {
result["subnet_kept"] = subnetKept
details += ", kept subnet " + subnetKept
}
s.auditLog(r, "firewall_unban", req.IP, details)
writeJSON(w, result)
}
package webui
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"time"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall/rollback"
"github.com/pidginhost/csm/internal/integrity"
"github.com/pidginhost/csm/internal/obs"
)
// apiFirewallTentativeApply handles POST /api/v1/settings/firewall/tentative-apply.
// Body shape mirrors the regular settings POST plus an optional
// timeout_min field (1..30, default 5). The handler runs the same
// change validation as the normal save path, snapshots the previous
// csm.yaml bytes into bbolt, writes the new file, and triggers a
// daemon restart. The rollback manager arms an in-process timer; if
// the operator does not POST /confirm before the deadline the daemon
// restores the snapshot and restarts itself.
func (s *Server) apiFirewallTentativeApply(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
// This handler rewrites csm.yaml like the settings and verified-bots
// saves do; it shares their lock so a concurrent save cannot be lost.
configMu := integrity.ConfigWriteMutex()
configMu.Lock()
defer configMu.Unlock()
mgr := rollback.Global()
if mgr == nil {
writeJSONError(w, "rollback manager not available", http.StatusServiceUnavailable)
return
}
if mgr.Status().Pending {
writeJSONError(w, "a firewall rollback is already pending; confirm or revert first", http.StatusConflict)
return
}
section, ok := LookupSettingsSection("firewall")
if !ok {
writeJSONError(w, "firewall section not registered", http.StatusInternalServerError)
return
}
ifMatch := r.Header.Get("If-Match")
if ifMatch == "" {
writeJSONError(w, "If-Match header required", http.StatusBadRequest)
return
}
var body struct {
Changes map[string]json.RawMessage `json:"changes"`
TimeoutMin int `json:"timeout_min"`
}
if err := decodeJSONBodyLimited(w, r, 256*1024, &body); err != nil {
writeJSONError(w, "invalid body: "+err.Error(), http.StatusBadRequest)
return
}
diskBytes, err := os.ReadFile(s.cfg.ConfigFile) // #nosec G304 -- operator-supplied config path
if err != nil {
writeJSONError(w, "read config: "+err.Error(), http.StatusInternalServerError)
return
}
disk, err := config.LoadBytes(diskBytes)
if err != nil {
writeJSONError(w, "parse config: "+err.Error(), http.StatusInternalServerError)
return
}
disk.ConfigFile = s.cfg.ConfigFile
disk.ConfigDir = s.cfg.ConfigDir
if disk.Integrity.ConfigHash != ifMatch {
writeJSONError(w, "config changed on disk, reload", http.StatusPreconditionFailed)
return
}
if rejectIfConfDirChanged(w, s.cfg.ConfigDir, disk) {
return
}
clone := *disk
if disk.Firewall != nil {
fw := *disk.Firewall
clone.Firewall = &fw
}
yamlChanges, errs := buildChangeSet(section, &clone, body.Changes)
if len(errs) > 0 {
writeValidationErrors(w, errs)
return
}
validationResults := append(config.Validate(&clone), config.ValidateDeepSection(&clone, section.ID)...)
fieldErrors, warnings := splitValidationResults(validationResults)
if len(fieldErrors) > 0 {
writeValidationErrors(w, fieldErrors)
return
}
localizeValidationFields(warnings, "firewall")
edited, err := config.YAMLEdit(diskBytes, yamlChanges)
if err != nil {
writeJSONError(w, "yaml edit: "+err.Error(), http.StatusInternalServerError)
return
}
// Stage rollback BEFORE the on-disk write so a crash between the two
// leaves the snapshot recoverable: if there is no new file on disk
// yet, the snapshot is a no-op revert. The reverse order would let
// the daemon come back to the new config with no rollback record and
// no way to undo without operator intervention.
timeout := time.Duration(body.TimeoutMin) * time.Minute
st, err := mgr.Apply(diskBytes, edited, timeout, extractClientIP(r))
if err != nil {
writeJSONError(w, "stage rollback: "+err.Error(), http.StatusInternalServerError)
return
}
if err := integrity.SignAndSavePreserving(s.cfg.ConfigFile, s.cfg.ConfigDir, edited, &clone, disk.Integrity.BinaryHash); err != nil {
// Best-effort cleanup: the snapshot is now misleading because
// the on-disk file never changed. Drop it so the operator does
// not see a phantom pending rollback in the UI.
_ = mgr.AbortApplyIfCurrent(st)
writeJSONError(w, "save: "+err.Error(), http.StatusInternalServerError)
return
}
s.auditLog(r, "settings-tentative-apply", "firewall", auditDetailsFor(section, body.Changes))
// Defer the restart so the response flushes first; otherwise the
// client sees a connection reset and cannot read the rollback ETA
// it needs to drive the countdown banner.
s.scheduleDaemonRestart(250 * time.Millisecond)
if warnings == nil {
warnings = []fieldError{}
}
applied := changedFieldList(body.Changes, section)
// The daemon restarts to apply a firewall change, so every applied field
// waits for it; the shape matches the settings save response.
writeOK(w, map[string]interface{}{
"warnings": warnings,
"rollback": st,
"new_etag": clone.Integrity.ConfigHash,
"applied": applied,
"requires_restart": applied,
"pending_restart": true,
})
}
// changedFieldList returns the dotted YAML paths of the keys in changes
// scoped to the section. Used in the response so the UI knows which
// fields to highlight as pending.
func changedFieldList(changes map[string]json.RawMessage, section SettingsSection) []string {
out := make([]string, 0, len(changes))
for k := range changes {
if k == "" {
out = append(out, section.YAMLPath)
continue
}
out = append(out, section.YAMLPath+"."+k)
}
return out
}
// apiFirewallRollbackStatus returns the pending rollback record, if any.
// Read endpoint, no CSRF; safe to poll for the countdown banner.
func (s *Server) apiFirewallRollbackStatus(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
mgr := rollback.Global()
if mgr == nil {
writeJSON(w, rollback.Status{})
return
}
writeJSON(w, mgr.Status())
}
func rejectConfigWriteDuringRollback(w http.ResponseWriter) bool {
mgr := rollback.Global()
if mgr == nil || !mgr.Status().Pending {
return false
}
writeJSONError(w, "a firewall rollback is pending; confirm or revert it before saving other settings", http.StatusConflict)
return true
}
// apiFirewallRollbackConfirm handles POST .../confirm. Drops the snapshot;
// the new config stays.
func (s *Server) apiFirewallRollbackConfirm(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
mgr := rollback.Global()
if mgr == nil {
writeJSONError(w, "rollback manager not available", http.StatusServiceUnavailable)
return
}
st := mgr.Status()
if !st.Pending {
writeJSONError(w, "no pending rollback", http.StatusConflict)
return
}
if err := mgr.ConfirmIfCurrent(st); err != nil {
writeJSONError(w, "confirm: "+err.Error(), http.StatusInternalServerError)
return
}
s.auditLog(r, "settings-rollback-confirm", "firewall", "")
writeOK(w, nil)
}
// apiFirewallRollbackRevert handles POST .../revert. Restores the
// snapshot to disk and triggers a daemon restart. Returns 200 with the
// pre-revert status; the actual restart happens on a goroutine so the
// response can flush.
func (s *Server) apiFirewallRollbackRevert(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
mgr := rollback.Global()
if mgr == nil {
writeJSONError(w, "rollback manager not available", http.StatusServiceUnavailable)
return
}
st := mgr.Status()
if !st.Pending {
writeJSONError(w, "no pending rollback", http.StatusConflict)
return
}
s.scheduleRollbackRevert(mgr, st, 30*time.Second)
s.auditLog(r, "settings-rollback-revert", "firewall", "")
// The revert and restart run after the response.
writeOKStatus(w, http.StatusAccepted, nil)
}
// scheduleDaemonRestart fires restartDaemon after delay in a supervised
// goroutine. The select on pruneDone lets the goroutine exit cleanly if
// the server begins shutdown during the pre-restart delay, so an
// operator-initiated stop is not chased by a phantom restart.
func (s *Server) scheduleDaemonRestart(delay time.Duration) {
obs.SafeGo("webui-daemon-restart", func() {
select {
case <-s.pruneDone:
return
case <-time.After(delay):
}
if _, err := s.restartDaemon(); err != nil {
fmt.Fprintf(os.Stderr, "webui: daemon restart failed: %v\n", err)
}
})
}
// scheduleRollbackRevert runs the revert in a supervised goroutine with
// a hard timeout. If shutdown already started before the worker runs, it
// does not begin a new revert; once started, the revert owns its restart
// context so the restart it triggers cannot cancel itself via Shutdown.
func (s *Server) scheduleRollbackRevert(mgr *rollback.Manager, expected rollback.Status, timeout time.Duration) {
obs.SafeGo("webui-rollback-revert", func() {
select {
case <-s.pruneDone:
return
default:
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
if err := mgr.RevertIfCurrent(ctx, expected); err != nil {
fmt.Fprintf(os.Stderr, "webui: rollback revert failed: %v\n", err)
}
})
}
package webui
import (
"errors"
"net/http"
"sort"
"strings"
"github.com/pidginhost/csm/internal/mailfwd/inventory"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/systemdrun"
)
// forwarderDestination is one resolved target of a forwarder, as served to the
// UI. Provider is the inventory class string (local/yahoo/gmail/outlook/external)
// the table renders as a badge.
type forwarderDestination struct {
Address string `json:"address"`
Domain string `json:"domain"`
Provider string `json:"provider"`
}
// forwarderEntry is a single source address and everything it relays to.
type forwarderEntry struct {
Source string `json:"source"`
Domain string `json:"domain"`
Owner string `json:"owner"`
Destinations []forwarderDestination `json:"destinations"`
Providers []string `json:"providers"` // distinct destination classes, sorted
KeepLocal bool `json:"keep_local"`
ForwardOnly bool `json:"forward_only"`
HasExternal bool `json:"has_external"`
HasFreeProvider bool `json:"has_free_provider"`
}
// forwardersSummary is the page-header rollup: how many forwarders exist and
// how many carry reputation risk (leave the server / target a free provider).
type forwardersSummary struct {
Total int `json:"total"`
External int `json:"external"`
FreeProvider int `json:"free_provider"`
}
type forwardersResponse struct {
Forwarders []forwarderEntry `json:"items"`
Total int `json:"total"`
Summary forwardersSummary `json:"summary"`
}
// selectForwarderSource picks the inventory source for the host. Only cPanel
// enumeration is wired; other platforms get the empty source until their
// adapters land (Phase 3).
func selectForwarderSource() inventory.Source {
if cpanelMailBackend(platform.Detect()) {
return inventory.NewCPanelSource()
}
return inventory.EmptySource{}
}
// selectForwarderRemover picks how forwarders are deleted. Only cPanel is
// wired; elsewhere the delete route answers as unavailable. eximLookPath and
// eximRun run any command, not only exim.
func selectForwarderRemover() *inventory.CPanelRemover {
if cpanelMailBackend(platform.Detect()) {
return inventory.NewCPanelRemover(eximLookPath, eximRun)
}
return nil
}
// apiEmailForwarders handles GET /api/v1/email/forwarders and returns the host's
// forwarder inventory: each source, its destinations with provider class, owner,
// and whether it keeps a local copy or forwards only.
func (s *Server) apiEmailForwarders(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
resp := forwardersResponse{Forwarders: []forwarderEntry{}}
if s.forwarderSource == nil {
writeJSON(w, resp)
return
}
fwds, err := s.forwarderSource.Forwarders()
if err != nil {
writeJSONError(w, "Failed to enumerate forwarders", http.StatusInternalServerError)
return
}
for _, f := range fwds {
resp.Forwarders = append(resp.Forwarders, toForwarderEntry(f))
resp.Summary.Total++
resp.Total++
if f.HasExternal() {
resp.Summary.External++
}
if f.HasFreeProvider() {
resp.Summary.FreeProvider++
}
}
writeJSON(w, resp)
}
func toForwarderEntry(f inventory.Forwarder) forwarderEntry {
dests := make([]forwarderDestination, 0, len(f.Destinations))
seen := make(map[string]bool, len(f.Destinations))
providers := make([]string, 0, len(f.Destinations))
for _, d := range f.Destinations {
dests = append(dests, forwarderDestination{
Address: d.Address,
Domain: d.Domain,
Provider: string(d.Provider),
})
if p := string(d.Provider); !seen[p] {
seen[p] = true
providers = append(providers, p)
}
}
sort.Strings(providers)
return forwarderEntry{
Source: f.Source,
Domain: f.Domain,
Owner: f.Owner,
Destinations: dests,
Providers: providers,
KeepLocal: f.KeepLocal,
ForwardOnly: f.ForwardOnly,
HasExternal: f.HasExternal(),
HasFreeProvider: f.HasFreeProvider(),
}
}
// apiEmailForwarderDelete handles POST /api/v1/email/forwarders/delete: it
// removes one destination of one forwarder. The pair must be in the current
// inventory, which also names the account uapi runs as, so a request can
// never pick the account or reach an address the list does not show. Every
// attempt that reaches cPanel is audit-logged with its outcome.
func (s *Server) apiEmailForwarderDelete(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
Source string `json:"source"`
Destination string `json:"destination"`
}
if err := decodeJSONBodyLimited(w, r, 4096, &req); err != nil || req.Source == "" || req.Destination == "" {
writeJSONError(w, "Body must name a source and a destination", http.StatusBadRequest)
return
}
if s.forwarderRemover == nil || s.forwarderSource == nil {
writeJSONError(w, "Forwarders cannot be changed on this host", http.StatusServiceUnavailable)
return
}
fwds, err := s.forwarderSource.Forwarders()
if err != nil {
writeJSONError(w, "Failed to enumerate forwarders", http.StatusInternalServerError)
return
}
f, ok := findForwarderDestination(fwds, req.Source, req.Destination)
switch {
case !ok:
writeJSONError(w, "No such forwarder; reload the list", http.StatusNotFound)
return
case strings.HasPrefix(f.Source, "*@"):
writeJSONError(w, "The default address is changed in cPanel", http.StatusBadRequest)
return
case f.Owner == "":
writeJSONError(w, "No cPanel account owns "+f.Domain, http.StatusConflict)
return
}
err = s.forwarderRemover.Remove(f, req.Destination)
details := "destination: " + req.Destination + "; owner: " + f.Owner
if err != nil {
details += "; error: " + err.Error()
}
s.auditLog(r, "email_forwarder_delete", f.Source, details)
switch {
case err == nil:
writeOK(w, nil)
case errors.Is(err, systemdrun.ErrUnavailable):
writeJSONError(w, "Deleting forwarders needs systemd-run: "+err.Error(), http.StatusServiceUnavailable)
default:
writeJSONError(w, "Forwarder delete failed: "+err.Error(), http.StatusInternalServerError)
}
}
// findForwarderDestination returns the forwarder whose source and one of whose
// destinations match exactly.
func findForwarderDestination(fwds []inventory.Forwarder, source, dest string) (inventory.Forwarder, bool) {
for _, f := range fwds {
if f.Source != source {
continue
}
for _, d := range f.Destinations {
if d.Address == dest {
return f, true
}
}
}
return inventory.Forwarder{}, false
}
package webui
import (
"net"
"net/http"
"github.com/pidginhost/csm/internal/geoip"
)
// SetGeoIPDB sets the GeoIP database for IP lookups.
func (s *Server) SetGeoIPDB(db *geoip.DB) {
s.geoIPDB.Store(db)
}
// apiGeoIPLookup returns geolocation info for an IP.
// GET /api/v1/geoip?ip=1.2.3.4 - fast local lookup (country + ASN)
// GET /api/v1/geoip?ip=1.2.3.4&detail=1 - includes RDAP org/ISP (may take 1-3s)
func (s *Server) apiGeoIPLookup(w http.ResponseWriter, r *http.Request) {
ip := r.URL.Query().Get("ip")
if ip == "" {
writeJSONError(w, "ip parameter required", http.StatusBadRequest)
return
}
if net.ParseIP(ip) == nil {
writeJSONError(w, "invalid IP address", http.StatusBadRequest)
return
}
db := s.geoIPDB.Load()
if db == nil {
writeJSONError(w, "GeoIP databases not loaded", http.StatusServiceUnavailable)
return
}
var info geoip.Info
if r.URL.Query().Get("detail") == "1" {
info = db.LookupWithRDAP(ip)
} else {
info = db.Lookup(ip)
}
writeJSON(w, info)
}
// apiGeoIPBatch returns geolocation info for multiple IPs.
// POST /api/v1/geoip/batch body: {"ips": ["1.2.3.4", "5.6.7.8"]}
func (s *Server) apiGeoIPBatch(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IPs []string `json:"ips"`
}
if err := decodeJSONBodyLimited(w, r, 32*1024, &req); err != nil {
writeJSONError(w, "invalid request body", http.StatusBadRequest)
return
}
if len(req.IPs) > 500 {
writeJSONError(w, "maximum 500 IPs per request", http.StatusBadRequest)
return
}
type geoResult struct {
Country string `json:"country"`
CountryName string `json:"country_name"`
City string `json:"city"`
ASOrg string `json:"as_org"`
Error string `json:"error,omitempty"`
}
db := s.geoIPDB.Load()
results := make(map[string]geoResult, len(req.IPs))
for _, ip := range req.IPs {
if net.ParseIP(ip) == nil {
results[ip] = geoResult{Error: "invalid IP format"}
continue
}
if db == nil {
results[ip] = geoResult{Error: "GeoIP database not loaded"}
continue
}
info := db.Lookup(ip)
results[ip] = geoResult{
Country: info.Country,
CountryName: info.CountryName,
City: info.City,
ASOrg: info.ASOrg,
}
}
writeJSON(w, map[string]interface{}{"results": results})
}
package webui
import (
"bytes"
"fmt"
"html/template"
"net/http"
"os"
"time"
"github.com/pidginhost/csm/internal/checks"
)
func (s *Server) renderTemplate(w http.ResponseWriter, r *http.Request, name string, data interface{}) {
base := s.templates[name]
if base == nil {
fmt.Fprintf(os.Stderr, "[webui] template %s missing\n", name)
http.Error(w, "template not found", http.StatusInternalServerError)
return
}
// The CSRF token belongs to the browser session loading the page, so
// each render binds it on a clone of the parsed template.
tmpl, err := base.Clone()
if err != nil {
fmt.Fprintf(os.Stderr, "[webui] template %s clone error: %v\n", name, err)
http.Error(w, "template render error", http.StatusInternalServerError)
return
}
token := s.csrfTokenFor(r)
tmpl.Funcs(template.FuncMap{"csrfToken": func() string { return token }})
// Render into a buffer first so an execution error can still surface as a
// 500 — html/template streams directly to its writer, and once any byte
// has been flushed the status header is locked in.
var buf bytes.Buffer
if err := tmpl.ExecuteTemplate(&buf, name, data); err != nil {
fmt.Fprintf(os.Stderr, "[webui] template %s error: %v\n", name, err)
http.Error(w, "template render error", http.StatusInternalServerError)
return
}
if _, err := w.Write(buf.Bytes()); err != nil {
fmt.Fprintf(os.Stderr, "[webui] template %s write error: %v\n", name, err)
}
}
type dashboardData struct {
Hostname string
Uptime string
Critical int
High int
Warning int
Total int
SigCount int
FanotifyActive bool
LogWatchers int
LastCriticalAgo string
LastCriticalISO string // RFC3339 of most recent critical, "" if none (so relative time can tick client-side)
RecentFindings []historyEntry
}
type historyEntry struct {
Severity string
SevClass string
Check string
Message string
Details string
Timestamp string
TimestampISO string // RFC3339 for JS comparison
TimeAgo string
HasFix bool
FixDesc string
Key string // canonical dedup key (matches alert.Finding.Key())
}
type quarantineData struct {
Hostname string
Files []quarantineEntry
}
type quarantineEntry struct {
ID string
OriginalPath string
Size int64
QuarantineAt string
Reason string
}
func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) {
sum := s.statsSummary24h()
recent := make([]historyEntry, 0, len(sum.recent))
for _, f := range sum.recent {
recent = append(recent, historyEntry{
Severity: severityLabel(f.Severity),
SevClass: severityClass(f.Severity),
Check: f.Check,
Message: f.Message,
Details: f.Details,
Timestamp: f.Timestamp.Format("15:04:05"),
TimestampISO: f.Timestamp.Format(time.RFC3339),
TimeAgo: timeAgo(f.Timestamp),
HasFix: checks.HasFix(f.Check),
FixDesc: checks.FixDescription(f.Check, f.Message, f.FilePath),
Key: f.Key(),
})
}
lastCriticalAgo, lastCriticalISO := "None", ""
if !sum.lastCritical.IsZero() {
lastCriticalAgo = timeAgo(sum.lastCritical)
lastCriticalISO = sum.lastCritical.Format(time.RFC3339)
}
data := dashboardData{
Hostname: s.cfg.Hostname,
Uptime: time.Since(s.startTime).Round(time.Second).String(),
Critical: sum.critical,
High: sum.high,
Warning: sum.warning,
Total: sum.critical + sum.high + sum.warning,
SigCount: s.signatureCount(),
FanotifyActive: s.fanotifyRunning(),
LogWatchers: s.logWatchersRunning(),
LastCriticalAgo: lastCriticalAgo,
LastCriticalISO: lastCriticalISO,
RecentFindings: recent,
}
s.renderTemplate(w, r, "dashboard.html", data)
}
func (s *Server) handleFindings(w http.ResponseWriter, r *http.Request) {
// Findings page is now JS-driven - enriched API provides data
s.renderTemplate(w, r, "findings.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
func (s *Server) handleHistoryRedirect(w http.ResponseWriter, r *http.Request) {
// History is now a tab on the findings page - redirect for backward compat
target := "/findings?tab=history"
if qs := r.URL.RawQuery; qs != "" {
target = "/findings?tab=history&" + qs
}
// #nosec G710 -- target always starts with the fixed same-origin
// /findings path; the incoming query can only add parameters.
http.Redirect(w, r, target, http.StatusFound)
}
// handleBlockedRedirect sends the Firewall page's old address to the page.
func (s *Server) handleBlockedRedirect(w http.ResponseWriter, r *http.Request) {
target := "/firewall"
if qs := r.URL.RawQuery; qs != "" {
target += "?" + qs
}
// #nosec G710 -- target always starts with the fixed same-origin
// /firewall path; the incoming query can only add parameters.
http.Redirect(w, r, target, http.StatusFound)
}
func (s *Server) handleQuarantine(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "quarantine.html", quarantineData{
Hostname: s.cfg.Hostname,
})
}
func (s *Server) handleCleanupHistory(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "cleanup-history.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
func (s *Server) handleFirewall(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "firewall.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
func (s *Server) handleEmail(w http.ResponseWriter, r *http.Request) {
// No mail server here: the sidebar has no Email entry, so an old
// bookmark lands on the dashboard.
if !s.feature("mail") {
http.Redirect(w, r, "/dashboard", http.StatusSeeOther)
return
}
s.renderTemplate(w, r, "email.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
func (s *Server) handleSettings(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "settings.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
package webui
import (
"net/http"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/store"
)
// apiHardening returns the last stored audit report (GET).
func (s *Server) apiHardening(w http.ResponseWriter, _ *http.Request) {
db := store.Global()
if db == nil {
writeJSON(w, &store.AuditReport{})
return
}
report, err := db.LoadHardeningReport()
if err != nil {
writeJSONError(w, "failed to load report: "+err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, report)
}
// apiHardeningRun runs the audit, stores the result, and returns it (POST only).
func (s *Server) apiHardeningRun(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if !s.acquireScan() {
writeJSONError(w, "A scan is already in progress. Please wait.", http.StatusConflict)
return
}
defer s.releaseScan()
// The audit can outlast the server's WriteTimeout; extend, never shorten.
rc := http.NewResponseController(w)
_ = rc.SetWriteDeadline(time.Now().Add(longRequestTimeout))
report := checks.RunHardeningAudit(s.liveCfg())
s.auditLog(r, "hardening_run", "server", "hardening audit run")
if db := store.Global(); db != nil {
if err := db.SaveHardeningReport(report); err != nil {
writeJSONError(w, "audit completed but failed to save report: "+err.Error(), http.StatusInternalServerError)
return
}
}
// The fresh report, with the action's ok flag next to its fields.
writeJSON(w, struct {
OK bool `json:"ok"`
*store.AuditReport
}{true, report})
}
// handleHardening renders the hardening audit page.
func (s *Server) handleHardening(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "hardening.html", nil)
}
package webui
import (
"net/http"
"regexp"
"strings"
"github.com/pidginhost/csm/internal/mailfwd/adapter"
"github.com/pidginhost/csm/internal/mailfwd/quarantine"
"github.com/pidginhost/csm/internal/platform"
)
// heldIDRe bounds a held-message id to a Maildir filename shape. The store also
// defends (filepath.Base + regular-file check), but validating at the handler
// boundary rejects traversal/control input before it reaches the filesystem.
var heldIDRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,128}$`)
// heldForwardStore is the held-forward quarantine surface the webui needs.
// *quarantine.Quarantine satisfies it; tests use a fake.
type heldForwardStore interface {
List() ([]quarantine.HeldMessage, error)
Release(id string) error
Delete(id string) error
}
const forwardQuarantineDir = adapter.QuarantineDir
// selectForwardHeld returns the held-forward store for the host. Only
// cPanel/exim writes held copies; other platforms have none.
func selectForwardHeld() heldForwardStore {
if cpanelMailBackend(platform.Detect()) {
return quarantine.New(forwardQuarantineDir)
}
return nil
}
// apiEmailHeldList handles GET /api/v1/email/held and returns the forward
// copies the guard has held.
func (s *Server) apiEmailHeldList(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if s.forwardHeld == nil {
writeAll(w, []quarantine.HeldMessage{})
return
}
msgs, err := s.forwardHeld.List()
if err != nil {
writeJSONError(w, "Failed to list held forwards", http.StatusInternalServerError)
return
}
writeAll(w, msgs)
}
// apiEmailHeldAction handles POST /api/v1/email/held/{id}/release (re-inject the
// held copy to its external recipient) and DELETE /api/v1/email/held/{id}
// (discard). Both mutate, so they run under auth + CSRF and are audit-logged.
func (s *Server) apiEmailHeldAction(w http.ResponseWriter, r *http.Request) {
tail := strings.TrimPrefix(r.URL.Path, "/api/v1/email/held/")
if tail == "" {
writeJSONError(w, "Missing held message ID", http.StatusBadRequest)
return
}
parts := strings.SplitN(tail, "/", 2)
id := parts[0]
action := ""
if len(parts) == 2 {
action = parts[1]
}
if !heldIDRe.MatchString(id) || strings.Contains(id, "..") {
writeJSONError(w, "Invalid held message ID", http.StatusBadRequest)
return
}
if s.forwardHeld == nil {
writeJSONError(w, "Forward guard not available on this host", http.StatusServiceUnavailable)
return
}
switch r.Method {
case http.MethodPost:
if action != "release" {
writeJSONError(w, "Unknown action; use /release", http.StatusBadRequest)
return
}
if err := s.forwardHeld.Release(id); err != nil {
writeJSONError(w, "Failed to release held forward: "+err.Error(), http.StatusInternalServerError)
return
}
s.auditLog(r, "email_held_release", id, "re-injected held forward copy to its external recipient")
writeOK(w, map[string]interface{}{"id": id})
case http.MethodDelete:
if action != "" {
writeJSONError(w, "Unknown action", http.StatusBadRequest)
return
}
if err := s.forwardHeld.Delete(id); err != nil {
writeJSONError(w, "Failed to delete held forward: "+err.Error(), http.StatusInternalServerError)
return
}
s.auditLog(r, "email_held_delete", id, "deleted held forward copy")
writeOK(w, map[string]interface{}{"id": id})
default:
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
}
}
package webui
import (
"bytes"
"encoding/json"
"fmt"
"html/template"
"net"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/integrity"
"github.com/pidginhost/csm/internal/platform"
"github.com/pidginhost/csm/internal/safepath"
)
// jsonForScript marshals v to JSON and returns it as template.JS suitable
// for direct substitution into a <script> block. json.Marshal already
// escapes < > & U+2028 U+2029 to \uXXXX form, so the output cannot break
// out of the surrounding <script> tag or trigger JS line-terminator
// parsing quirks. On marshal failure the fallback is the JS literal
// "null" so the enclosing template still parses.
func jsonForScript(v interface{}) template.JS {
b, err := json.Marshal(v)
if err != nil {
return template.JS("null")
}
// Defense-in-depth: Go's json.Marshal has historically escaped these
// by default, but an explicit pass guarantees the contract even if
// that default ever changes or the input arrived pre-encoded.
b = bytes.ReplaceAll(b, []byte("<"), []byte(`\u003c`))
b = bytes.ReplaceAll(b, []byte(">"), []byte(`\u003e`))
b = bytes.ReplaceAll(b, []byte("&"), []byte(`\u0026`))
b = bytes.ReplaceAll(b, []byte("\u2028"), []byte(`\u2028`))
b = bytes.ReplaceAll(b, []byte("\u2029"), []byte(`\u2029`))
// #nosec G203 -- Output is JSON bytes with HTML/JS-dangerous codepoints
// escaped above; safe to hand to html/template as JS.
return template.JS(b)
}
func decodeJSONBodyLimited(w http.ResponseWriter, r *http.Request, limit int64, dst interface{}) error {
if limit <= 0 {
limit = 64 * 1024
}
r.Body = http.MaxBytesReader(w, r.Body, limit)
dec := json.NewDecoder(r.Body)
dec.DisallowUnknownFields()
if err := dec.Decode(dst); err != nil {
return err
}
if dec.More() {
return fmt.Errorf("request body must contain a single JSON value")
}
return nil
}
// validateEximMessageID rejects message ids that would be unsafe to
// interpolate into filesystem paths. Real Exim ids are of the form
// 6-6-2 on Exim 4.96 and older, and 6-11-4 on Exim 4.97 and newer. The
// validator accepts anything with the same character class so callers can use
// shorter fixtures in tests while still blocking `..`, `/`, `\`, dots, NUL
// bytes, and other shell / path-traversal metacharacters even if the inner
// Quarantine layer's filepath.Base() guard regresses.
func validateEximMessageID(id string) error {
if id == "" {
return fmt.Errorf("message id is required")
}
if len(id) > 32 {
return fmt.Errorf("message id too long (%d chars, max 32)", len(id))
}
for _, c := range id {
if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') && c != '-' {
return fmt.Errorf("message id contains invalid character: %q", c)
}
}
return nil
}
// mustBeWithin resolves candidate against root and reports whether the
// result still lives inside root after symlink and `..` resolution.
// Returns the cleaned absolute path on success. Defense-in-depth for any
// handler that takes a user-supplied path fragment, joins it under a
// trusted root, and hands the result to os.Remove / os.RemoveAll / etc.
func mustBeWithin(root, candidate string) (string, error) {
if root == "" {
return "", fmt.Errorf("root is required")
}
absRoot, err := filepath.Abs(filepath.Clean(root))
if err != nil {
return "", fmt.Errorf("resolve root: %w", err)
}
absRoot, err = filepath.EvalSymlinks(absRoot)
if err != nil {
return "", fmt.Errorf("resolve root symlinks: %w", err)
}
abs, err := resolvePathUnderRoot(absRoot, rootRelativeCandidate(candidate))
if err != nil {
return "", err
}
if !isPathWithin(abs, absRoot) {
return "", fmt.Errorf("path %q escapes root %q", candidate, absRoot)
}
return abs, nil
}
func rootRelativeCandidate(candidate string) string {
if volume := filepath.VolumeName(candidate); volume != "" {
candidate = strings.TrimPrefix(candidate, volume)
}
candidate = strings.TrimLeft(candidate, string(filepath.Separator))
if candidate == "" {
return "."
}
return candidate
}
func resolvePathUnderRoot(root, rel string) (string, error) {
parts := pathParts(rel)
current := root
symlinkCount := 0
for i := 0; i < len(parts); i++ {
part := parts[i]
switch part {
case ".", "":
continue
case "..":
current = filepath.Dir(current)
if !isPathWithin(current, root) {
return "", fmt.Errorf("path %q escapes root %q", rel, root)
}
continue
}
next := filepath.Join(current, part)
info, err := os.Lstat(next)
if err != nil {
if os.IsNotExist(err) {
current = next
continue
}
return "", fmt.Errorf("stat candidate: %w", err)
}
if info.Mode()&os.ModeSymlink == 0 {
current = next
continue
}
symlinkCount++
if symlinkCount > 255 {
return "", fmt.Errorf("too many symlinks resolving %q", rel)
}
target, err := os.Readlink(next)
if err != nil {
return "", fmt.Errorf("read symlink: %w", err)
}
if !filepath.IsAbs(target) {
target = filepath.Join(filepath.Dir(next), target)
}
target = filepath.Clean(target)
targetRel, err := filepath.Rel(root, target)
if err != nil {
return "", fmt.Errorf("resolve symlink target: %w", err)
}
nextParts := append([]string{}, pathParts(targetRel)...)
nextParts = append(nextParts, parts[i+1:]...)
parts = nextParts
current = root
i = -1
}
return filepath.Clean(current), nil
}
func pathParts(path string) []string {
if path == "" || path == "." {
return nil
}
raw := strings.Split(path, string(filepath.Separator))
parts := make([]string, 0, len(raw))
for _, part := range raw {
if part != "" && part != "." {
parts = append(parts, part)
}
}
return parts
}
// validateAccountName checks that name is a valid cPanel account name:
// 1-64 characters, alphanumeric and underscore only.
func validateAccountName(name string) error {
if name == "" {
return fmt.Errorf("account name is required")
}
if len(name) > 64 {
return fmt.Errorf("account name too long (%d chars, max 64)", len(name))
}
if (name[0] < 'a' || name[0] > 'z') && (name[0] < 'A' || name[0] > 'Z') {
return fmt.Errorf("account name must start with a letter")
}
for _, c := range name {
if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') && c != '_' {
return fmt.Errorf("account name contains invalid character: %c", c)
}
}
return nil
}
// parseAndValidateIP parses an IP string and rejects non-routable addresses
// (loopback, private RFC 1918, link-local, multicast, unspecified, broadcast).
// RFC 5737 documentation ranges (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24)
// are intentionally allowed.
func parseAndValidateIP(s string) (net.IP, error) {
ip, err := parseRecordedIP(s)
if err != nil {
return nil, err
}
if ip.IsPrivate() {
return nil, fmt.Errorf("private address not allowed: %s", strings.TrimSpace(s))
}
return ip, nil
}
// parseAllowIP validates an address for an allow action. A private address is
// internal infrastructure: an allow rule does not stop CSM from flagging it,
// infra_ips does.
func parseAllowIP(s string) (net.IP, error) {
ip, err := parseRecordedIP(s)
if err != nil {
return nil, err
}
if ip.IsPrivate() {
return nil, fmt.Errorf("private address not allowed: %s; list internal addresses in infra_ips and restart CSM", strings.TrimSpace(s))
}
return ip, nil
}
// parseRecordedIP accepts any address CSM can record as an attack source:
// the routable ones parseAndValidateIP accepts plus private addresses, which
// internal hosts missing from infra_ips get recorded under.
func parseRecordedIP(s string) (net.IP, error) {
s = strings.TrimSpace(s)
if s == "" {
return nil, fmt.Errorf("IP address is required")
}
ip := net.ParseIP(s)
if ip == nil {
return nil, fmt.Errorf("invalid IP address: %s", s)
}
if ip.IsLoopback() {
return nil, fmt.Errorf("loopback address not allowed: %s", s)
}
if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
return nil, fmt.Errorf("link-local address not allowed: %s", s)
}
if ip.IsMulticast() {
return nil, fmt.Errorf("multicast address not allowed: %s", s)
}
if ip.IsUnspecified() {
return nil, fmt.Errorf("unspecified address not allowed: %s", s)
}
// Broadcast: 255.255.255.255
if ip.Equal(net.IPv4bcast) {
return nil, fmt.Errorf("broadcast address not allowed: %s", s)
}
return ip, nil
}
// validateCIDR parses a CIDR string and rejects overly broad prefixes
// (/0 through /7; minimum allowed is /8).
func validateCIDR(s string) (*net.IPNet, error) {
if s == "" {
return nil, fmt.Errorf("CIDR is required")
}
_, ipNet, err := net.ParseCIDR(s)
if err != nil {
return nil, fmt.Errorf("invalid CIDR: %w", err)
}
ones, bits := ipNet.Mask.Size()
minPrefix := 8
if bits == 128 { // IPv6
minPrefix = 32
}
if ones < minPrefix {
return nil, fmt.Errorf("CIDR prefix /%d is too broad (minimum /%d)", ones, minPrefix)
}
return ipNet, nil
}
// parseDuration parses a human-friendly duration string from the web UI.
// Supported formats: "24h", "7d", "30d", "0" (permanent), "" (permanent).
// Zero means permanent in the block/allow contexts that consume this.
// Unparseable, negative, or out-of-range input is an error to keep a typo or
// integer overflow from silently becoming a permanent rule.
func parseDuration(s string) (time.Duration, error) {
s = strings.TrimSpace(s)
if s == "" || s == "0" {
return 0, nil
}
if strings.HasPrefix(s, "-") {
return 0, fmt.Errorf("invalid duration %q", s)
}
if strings.HasSuffix(s, "d") {
num := strings.TrimSuffix(s, "d")
if num == "" {
return 0, fmt.Errorf("invalid duration %q", s)
}
for _, c := range num {
if c < '0' || c > '9' {
return 0, fmt.Errorf("invalid duration %q", s)
}
}
days, err := strconv.ParseUint(num, 10, 64)
const maxDuration = time.Duration(1<<63 - 1)
if err != nil || days > uint64(maxDuration/(24*time.Hour)) {
return 0, fmt.Errorf("invalid duration %q", s)
}
return time.Duration(days) * 24 * time.Hour, nil
}
d, err := time.ParseDuration(s)
if err != nil || d == 0 && strings.ContainsAny(s, "123456789") {
return 0, fmt.Errorf("invalid duration %q", s)
}
return d, nil
}
// operatorFacingCheck reports whether findings of a check belong in the
// finding lists operators act on. auto_response and auto_block record what
// CSM already did; check_timeout and health describe CSM itself.
func operatorFacingCheck(check string) bool {
switch check {
case "auto_response", "auto_block", "check_timeout", "health":
return false
}
return true
}
// isPathUnder returns true if the cleaned path is strictly under the base
// directory. It prevents path traversal via ".." and prefix tricks
// (e.g., /home/username is not under /home/user).
func isPathUnder(path, base string) bool {
cleanPath := filepath.Clean(path)
cleanBase := filepath.Clean(base)
// Ensure base ends with separator so "/home/user" doesn't match "/home/username"
prefix := cleanBase + string(filepath.Separator)
return strings.HasPrefix(cleanPath, prefix)
}
func isPathWithin(path, base string) bool {
return safepath.Within(path, base)
}
// hashLiveStateFile hashes a file for quarantineLiveState; a var so tests
// can count reads.
var hashLiveStateFile = integrity.HashFile
// liveStates remembers how a quarantine archive compared with its live file,
// keyed by both files' change keys, so the list does not hash every pair on
// every request. It is cleared when it grows past liveStatesMax entries.
var (
liveStatesMu sync.Mutex
liveStates = map[string]liveStateResult{}
)
const liveStatesMax = 10000
type liveStateResult struct {
keys string
state string
}
func quarantineLiveState(archivePath, originalPath string) string {
origInfo, err := os.Stat(originalPath)
if os.IsNotExist(err) {
return "original_missing"
}
if err != nil {
return "unknown"
}
if !origInfo.Mode().IsRegular() {
return "original_not_file"
}
archInfo, err := os.Stat(archivePath)
if os.IsNotExist(err) {
return "archive_missing"
}
if err != nil {
return "unknown"
}
if !archInfo.Mode().IsRegular() {
return "archive_not_file"
}
if origInfo.Size() != archInfo.Size() {
return "live_differs"
}
pair := archivePath + "\x00" + originalPath
keys := integrity.FileChangeKey(origInfo) + "|" + integrity.FileChangeKey(archInfo)
liveStatesMu.Lock()
cached, ok := liveStates[pair]
liveStatesMu.Unlock()
if ok && cached.keys == keys {
return cached.state
}
origHash, err := hashLiveStateFile(originalPath)
if err != nil {
return "unknown"
}
archHash, err := hashLiveStateFile(archivePath)
if err != nil {
return "unknown"
}
state := "live_differs"
if origHash == archHash {
state = "restored_identical"
}
liveStatesMu.Lock()
if len(liveStates) >= liveStatesMax {
liveStates = map[string]liveStateResult{}
}
liveStates[pair] = liveStateResult{keys: keys, state: state}
liveStatesMu.Unlock()
return state
}
const preCleanQuarantineIDPrefix = "pre_clean:"
type quarantineEntryRef struct {
ID string
ItemPath string
MetaPath string
}
func quarantineEntryID(metaPath string) string {
id := strings.TrimSuffix(filepath.Base(metaPath), ".meta")
if filepath.Clean(filepath.Dir(metaPath)) == filepath.Join(quarantineDir, "pre_clean") {
return preCleanQuarantineIDPrefix + id
}
return id
}
// reservedQuarantineNames are subtrees of the quarantine root that are not
// entries: the pre-clean backups and the email quarantine. An ID naming one
// of them resolves to the subtree itself and must never be deleted.
var reservedQuarantineNames = map[string]bool{"pre_clean": true, "email": true}
// quarantineEntryDeletable reports whether an ID resolved to a real
// quarantine entry: not a reserved subtree, and carrying the metadata
// sidecar every quarantined item is written with.
func quarantineEntryDeletable(entry quarantineEntryRef) bool {
if reservedQuarantineNames[filepath.Base(entry.ItemPath)] {
return false
}
_, err := os.Stat(entry.MetaPath)
return err == nil
}
func resolveQuarantineEntry(id string) (quarantineEntryRef, error) {
rawID := strings.TrimSpace(id)
if rawID == "" {
return quarantineEntryRef{}, fmt.Errorf("quarantine ID is required")
}
baseDir := quarantineDir
name := rawID
if strings.HasPrefix(rawID, preCleanQuarantineIDPrefix) {
baseDir = filepath.Join(quarantineDir, "pre_clean")
name = strings.TrimPrefix(rawID, preCleanQuarantineIDPrefix)
}
name = filepath.Base(name)
if name == "" || name == "." || name == ".." {
return quarantineEntryRef{}, fmt.Errorf("invalid quarantine ID")
}
itemPath := filepath.Join(baseDir, name)
if !isPathWithin(itemPath, baseDir) {
return quarantineEntryRef{}, fmt.Errorf("invalid quarantine ID")
}
return quarantineEntryRef{
ID: rawID,
ItemPath: itemPath,
MetaPath: itemPath + ".meta",
}, nil
}
// readQuarantineMeta reads and parses a quarantine .meta JSON file.
func readQuarantineMeta(metaPath string) (*checks.QuarantineMeta, error) {
// #nosec G304 -- metaPath is constructed by resolveQuarantineEntry under
// the quarantine base dir with filepath.Base applied to the ID.
data, err := os.ReadFile(metaPath)
if err != nil {
return nil, fmt.Errorf("read quarantine meta: %w", err)
}
var meta checks.QuarantineMeta
if err := json.Unmarshal(data, &meta); err != nil {
return nil, fmt.Errorf("parse quarantine meta: %w", err)
}
return &meta, nil
}
// listMetaFiles returns the full paths of all .meta files in dir (non-recursive).
// Returns nil on any error (e.g., directory does not exist).
func listMetaFiles(dir string) []string {
entries, err := os.ReadDir(dir)
if err != nil {
return nil
}
var metas []string
for _, e := range entries {
if e.IsDir() {
continue
}
if strings.HasSuffix(e.Name(), ".meta") {
metas = append(metas, filepath.Join(dir, e.Name()))
}
}
return metas
}
// Tests redirect platform and scratch roots without changing the config scope.
var quarantineRestoreRoots []string
func quarantineRootsForConfig(cfg *config.Config) ([]string, error) {
roots := quarantineRestoreRoots
if roots == nil {
roots = append(platform.Detect().AccountHomeRoots(), "/tmp", "/dev/shm", "/var/tmp")
}
roots = append([]string(nil), roots...)
// Only platform-owned roots can have aliases (for example /tmp on a
// development host). Snapshot them before adding tenant-owned roots.
for _, root := range roots {
if resolved, err := filepath.EvalSymlinks(root); err == nil && resolved != root {
roots = append(roots, resolved)
}
}
if cfg != nil {
configured, err := platform.ResolveAccountRoots(cfg.AccountRoots)
return append(roots, configured...), err
}
return roots, nil
}
func validateQuarantineRestorePath(path string, roots []string) (string, error) {
cleanPath := filepath.Clean(strings.TrimSpace(path))
if cleanPath == "" {
return "", fmt.Errorf("restore path is required")
}
if !filepath.IsAbs(cleanPath) {
return "", fmt.Errorf("restore path must be absolute")
}
if !pathWithinAny(cleanPath, roots) {
return "", fmt.Errorf("restore path is outside the allowed restore roots: %s", cleanPath)
}
ancestor, err := nearestExistingAncestor(cleanPath)
if err != nil {
return "", err
}
resolvedAncestor, err := filepath.EvalSymlinks(ancestor)
if err != nil {
return "", fmt.Errorf("cannot validate restore path: %w", err)
}
if !pathWithinAny(resolvedAncestor, roots) {
return "", fmt.Errorf("restore path escapes the allowed restore roots: %s", cleanPath)
}
if accountRoot := homeAccountRoot(cleanPath); accountRoot != "" && !isPathWithin(resolvedAncestor, accountRoot) {
return "", fmt.Errorf("restore path escapes the account boundary: %s", cleanPath)
}
return cleanPath, nil
}
func pathWithinAny(path string, bases []string) bool {
for _, base := range bases {
if isPathWithin(path, base) {
return true
}
}
return false
}
func nearestExistingAncestor(path string) (string, error) {
current := filepath.Clean(path)
for {
if _, err := os.Lstat(current); err == nil {
return current, nil
} else if !os.IsNotExist(err) {
return "", fmt.Errorf("cannot stat restore path: %w", err)
}
parent := filepath.Dir(current)
if parent == current {
return "", fmt.Errorf("restore path has no existing ancestor: %s", path)
}
current = parent
}
}
func homeAccountRoot(path string) string {
clean := filepath.Clean(path)
for _, root := range platform.Detect().AccountHomeRoots() {
rest, found := strings.CutPrefix(clean, filepath.Clean(root)+string(filepath.Separator))
if !found {
continue
}
account, tail, inside := strings.Cut(rest, string(filepath.Separator))
if inside && tail != "" {
return filepath.Join(root, account)
}
}
return ""
}
package webui
import (
"sync"
"time"
)
// Results computed from a window of recent history are reused between polls.
// A result is kept while history is unchanged, for at most memoMaxAge so time
// windows keep moving on a quiet host; when history changes it is recomputed,
// but not more often than memoMinAge, so a busy host walks history at most
// once per memoMinAge however many dashboards are open. Vars so tests can
// force a recompute.
var (
memoMinAge = 15 * time.Second
memoMaxAge = time.Minute
)
// historyMemo holds one memoised result. Concurrent callers of a stale
// result share one computation.
type historyMemo struct {
mu sync.Mutex
valid bool
mark string
at time.Time
value any
computes int // how many times value was computed; read by tests
}
func (m *historyMemo) get(mark string, compute func() any) any {
return m.getWithin(mark, memoMinAge, compute)
}
// fresh is get without the memoMinAge throttle, for a caller that cannot
// use a result built before mark. Callers at the same mark still share one
// computation.
func (m *historyMemo) fresh(mark string, compute func() any) any {
return m.getWithin(mark, 0, compute)
}
func (m *historyMemo) getWithin(mark string, minAge time.Duration, compute func() any) any {
m.mu.Lock()
defer m.mu.Unlock()
age := time.Since(m.at)
if m.valid && age < memoMaxAge && (mark == m.mark || age < minAge) {
return m.value
}
m.value = compute()
m.computes++
m.mark = mark
m.at = time.Now()
m.valid = true
return m.value
}
// historyMemos is a small keyed set of memos for results that depend on
// request parameters, such as the email date range. The oldest entry is
// dropped when the set is full.
type historyMemos struct {
mu sync.Mutex
items map[string]*historyMemo
order []string
}
const historyMemosMax = 32
func (ms *historyMemos) memo(key string) *historyMemo {
ms.mu.Lock()
defer ms.mu.Unlock()
if m, ok := ms.items[key]; ok {
return m
}
if ms.items == nil {
ms.items = map[string]*historyMemo{}
}
if len(ms.order) >= historyMemosMax {
delete(ms.items, ms.order[0])
ms.order = ms.order[1:]
}
m := &historyMemo{}
ms.items[key] = m
ms.order = append(ms.order, key)
return m
}
package webui
import (
"errors"
"fmt"
"net/http"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/incident"
)
type timelineEvent struct {
Timestamp time.Time `json:"timestamp"`
Type string `json:"type"` // "finding", "action", "block"
Severity string `json:"severity,omitempty"` // a finding's label; actions have none
Summary string `json:"summary"`
Details string `json:"details,omitempty"`
Source string `json:"source"` // "history", "audit", "firewall"
}
const incidentTimelineEventLimit = 200
// incidentSnapshotScanCap bounds how many incidents the timeline walk
// will inspect from the correlator snapshot per request. Correlators on
// hot hosts can hold thousands of open + recently-closed incidents and
// each one carries a full event timeline; without this cap a single
// /api/v1/incident call can walk hundreds of thousands of timeline
// events before paginating down to incidentTimelineEventLimit. The cap
// is generous (most timelines hit the 200-event ceiling well before the
// 1000-incident ceiling) but bounds worst-case wall time and memory.
const incidentSnapshotScanCap = 1000
func (s *Server) handleIncident(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "incident.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
func (s *Server) apiIncident(w http.ResponseWriter, r *http.Request) {
ip := r.URL.Query().Get("ip")
account := r.URL.Query().Get("account")
if ip == "" && account == "" {
writeJSONError(w, "ip or account parameter is required", http.StatusBadRequest)
return
}
hours := queryInt(r, "hours", 72)
if hours > 720 {
hours = 720
} // max 30 days
cutoff := time.Now().Add(-time.Duration(hours) * time.Hour)
var events []timelineEvent
// Build search terms
var searchTerms []string
if ip != "" {
searchTerms = append(searchTerms, ip)
}
if account != "" {
searchTerms = append(searchTerms, "/home/"+account+"/", account)
}
dedup := make(map[string]struct{})
dedupKey := func(t time.Time, summary string) string {
return t.UTC().Format(time.RFC3339Nano) + "|" + summary
}
matchesHistoryQuery := func(f alert.Finding) bool {
matched := false
for _, term := range searchTerms {
if strings.Contains(f.Message, term) || strings.Contains(f.Details, term) {
matched = true
break
}
}
if !matched {
return false
}
summary := f.Check + ": " + f.Message
key := dedupKey(f.Timestamp, summary)
if _, seen := dedup[key]; seen {
return false
}
dedup[key] = struct{}{}
return true
}
// Search newest-first and stop once the timeline has enough matching
// history rows. Busy hosts can retain large 30-day windows, so response
// size alone is not a safe bound for the read path.
allHistory := s.store.SearchHistorySince(cutoff, incidentTimelineEventLimit+1, matchesHistoryQuery)
historyCapped := len(allHistory) > incidentTimelineEventLimit
if historyCapped {
// The extra row only tells that more exist; forget it so an
// incident event it duplicates is still listed.
dropped := allHistory[incidentTimelineEventLimit]
delete(dedup, dedupKey(dropped.Timestamp, dropped.Check+": "+dropped.Message))
allHistory = allHistory[:incidentTimelineEventLimit]
}
for _, f := range allHistory {
summary := f.Check + ": " + f.Message
events = append(events, timelineEvent{
Timestamp: f.Timestamp.UTC(),
Type: "finding",
Severity: f.Severity.String(),
Summary: summary,
Details: f.Details,
Source: "history",
})
}
// Fold in events from the incident correlator. The finding history
// bucket rotates aggressively on busy hosts so a Critical incident
// from two days ago may have no surviving history row, but the
// incident object still carries the full timeline. Walk every
// incident, match by RemoteIP for IP queries or by Account / Mailbox /
// Domain for account queries, and emit each matching timeline event.
truncated := historyCapped
if s.incidentCorrelator != nil {
snap, totalIncidents := s.incidentCorrelator.SnapshotPageStatuses(nil, 0, incidentSnapshotScanCap)
truncated = truncated || totalIncidents > len(snap)
for _, inc := range snap {
incMatches := incidentMatchesAccount(inc, account)
for _, ev := range inc.Timeline {
if ev.Time.Before(cutoff) {
continue
}
match := false
if ip != "" && ev.RemoteIP == ip {
match = true
}
if !match && incMatches {
match = true
}
if !match {
continue
}
summary := ev.Check
if ev.Message != "" {
if summary != "" {
summary += ": "
}
summary += ev.Message
}
key := dedupKey(ev.Time, summary)
if _, seen := dedup[key]; seen {
continue
}
dedup[key] = struct{}{}
events = append(events, timelineEvent{
Timestamp: ev.Time.UTC(),
Type: "finding",
Severity: inc.Severity.String(),
Summary: summary,
Details: "From incident " + inc.ID + " (" + string(inc.Kind) + ", " + string(inc.Status) + ")",
Source: "incident:" + inc.ID,
})
}
}
}
// Search UI audit log
const auditScanLimit = 500
auditEntries := readUIAuditLog(s.cfg.StatePath, auditScanLimit+1)
if len(auditEntries) > auditScanLimit {
truncated = true
auditEntries = auditEntries[:auditScanLimit]
}
for _, a := range auditEntries {
if a.Timestamp.Before(cutoff) {
continue
}
matched := false
for _, term := range searchTerms {
if strings.Contains(a.Target, term) || strings.Contains(a.Details, term) {
matched = true
break
}
}
if !matched {
continue
}
events = append(events, timelineEvent{
Timestamp: a.Timestamp.UTC(),
Type: "action",
Summary: a.Action + ": " + a.Target,
Details: a.Details,
Source: "audit",
})
}
// Newest first, compared as instants.
sort.SliceStable(events, func(i, j int) bool {
return events[i].Timestamp.After(events[j].Timestamp)
})
total := len(events)
if total > incidentTimelineEventLimit {
events = events[:incidentTimelineEventLimit]
truncated = true
}
if truncated {
w.Header().Set("X-CSM-Truncated", "1")
}
writeItems(w, events, map[string]interface{}{
"total": total,
"offset": 0,
"limit": incidentTimelineEventLimit,
"query_ip": ip,
"query_account": account,
"window_seconds": hours * 3600,
"truncated": truncated,
})
}
// incidentMatchesAccount reports whether an incident's identity fields
// match the account search term. Empty account never matches so an
// IP-only query does not pull in unrelated incidents.
func incidentMatchesAccount(inc incident.Incident, account string) bool {
if account == "" {
return false
}
return inc.Account == account || inc.Mailbox == account || inc.Domain == account
}
// maxIncidentPageSize caps the page size a client may request so a
// misbehaving consumer cannot OOM the daemon by asking for the whole
// world in one round-trip. The web UI's default page size is well
// below this; the ceiling exists for defense in depth.
const maxIncidentPageSize = 500
// defaultIncidentPageSize is the page size when the client passes no
// limit. Tuned to fit comfortably on one screen.
const defaultIncidentPageSize = 50
// apiIncidentList serves GET /api/v1/incidents as one page:
// {"items":[...], "total":N, "offset":N, "limit":N, "status":"..."}.
// limit defaults to defaultIncidentPageSize; total counts every incident
// the status filter matches, so a client pages on through offset.
//
// status accepts the four incident statuses (open/contained/resolved/dismissed)
// plus the UI-only convenience "active" that means
// open+contained. An empty string means all statuses. Anything else is
// rejected with 400 Bad Request rather than silently widening to all,
// which would hide a typo like ?status=opn.
func (s *Server) apiIncidentList(w http.ResponseWriter, r *http.Request) {
statusParam := r.URL.Query().Get("status")
statuses, err := parseIncidentStatusFilter(statusParam)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
limit := queryInt(r, "limit", defaultIncidentPageSize)
if limit <= 0 {
limit = defaultIncidentPageSize
}
if limit > maxIncidentPageSize {
limit = maxIncidentPageSize
}
offset := queryInt(r, "offset", 0)
if offset < 0 {
offset = 0
}
var items []incident.Incident
total := 0
if s.incidentCorrelator != nil {
items, total = s.incidentPage(statuses, offset, limit)
}
writeItems(w, items, map[string]any{
"total": total,
"offset": offset,
"limit": limit,
"status": statusParam,
"truncated": historyPageTruncated(total, offset, len(items)),
})
}
// parseIncidentStatusFilter validates the status query parameter and
// returns the set of statuses it expands to. Empty input means "all";
// "active" is the UI-only convenience for open+contained.
func parseIncidentStatusFilter(s string) ([]incident.Status, error) {
switch s {
case "":
return nil, nil
case "active":
return []incident.Status{incident.StatusOpen, incident.StatusContained}, nil
case string(incident.StatusOpen),
string(incident.StatusContained),
string(incident.StatusResolved),
string(incident.StatusDismissed):
return []incident.Status{incident.Status(s)}, nil
}
return nil, fmt.Errorf("invalid status %q", s)
}
// incidentPage returns a status-filtered page. The "active" filter
// expands to open+contained and is handled by the correlator in one
// sorted pass so pagination stays stable across statuses.
func (s *Server) incidentPage(statuses []incident.Status, offset, limit int) ([]incident.Incident, int) {
return s.incidentCorrelator.SnapshotPageStatuses(statuses, offset, limit)
}
// apiIncidentShow serves GET /api/v1/incidents/<id>. 404 if not found.
func (s *Server) apiIncidentShow(w http.ResponseWriter, r *http.Request) {
id := strings.TrimPrefix(r.URL.Path, "/api/v1/incidents/")
id = strings.TrimSuffix(id, "/")
if id == "" || s.incidentCorrelator == nil {
writeJSONError(w, "Incident not found", http.StatusNotFound)
return
}
inc, ok := s.incidentCorrelator.Get(id)
if !ok {
writeJSONError(w, "Incident not found", http.StatusNotFound)
return
}
writeJSON(w, inc)
}
// apiIncidentStatus serves POST /api/v1/incidents/<id>/status. Body
// {"status": "resolved", "details": "..."}.
func (s *Server) apiIncidentStatus(w http.ResponseWriter, r *http.Request) {
id := strings.TrimPrefix(r.URL.Path, "/api/v1/incidents/")
id = strings.TrimSuffix(id, "/status")
id = strings.Trim(id, "/")
var body struct {
Status string `json:"status"`
Details string `json:"details"`
}
// Cap the request body like every other mutating handler; a bare
// json.NewDecoder(r.Body) would buffer an unbounded body into memory.
if err := decodeJSONBodyLimited(w, r, 16*1024, &body); err != nil {
writeJSONError(w, "Invalid request body", http.StatusBadRequest)
return
}
if s.incidentCorrelator == nil {
writeJSONError(w, "Incidents are not enabled", http.StatusServiceUnavailable)
return
}
if err := s.incidentCorrelator.SetStatus(id, incident.Status(body.Status), body.Details); err != nil {
code := http.StatusBadRequest
if errors.Is(err, incident.ErrIncidentNotFound) {
code = http.StatusNotFound
}
writeJSONError(w, err.Error(), code)
return
}
s.auditLog(r, "incident_status", id, strings.TrimSpace(body.Status+" "+body.Details))
writeJSON(w, map[string]interface{}{"ok": true})
}
// apiIncidentRouter dispatches /api/v1/incidents/<id>[...] sub-paths.
// POST .../status -> apiIncidentStatus; GET .../<id> -> apiIncidentShow.
func (s *Server) apiIncidentRouter(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/status") {
s.apiIncidentStatus(w, r)
return
}
s.apiIncidentShow(w, r)
}
package webui
import (
"net/http"
"strconv"
"strings"
"github.com/pidginhost/csm/internal/incident"
)
// incidentGroupsDefaultLimit / Max bound how many group rows the UI
// receives. A typical busy production host emits ~12 attacker-IP rows;
// the cap of 200 leaves headroom for the long tail without unbounded
// payload size.
const (
incidentGroupsDefaultLimit = 50
incidentGroupsMaxLimit = 200
)
// apiIncidentGroups handles GET /api/v1/incidents/groups. Buckets the
// in-memory incident snapshot by (kind, source) and returns rolled-up
// group rows. Read-scope eligible; never mutates state.
func (s *Server) apiIncidentGroups(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
q := r.URL.Query()
limit := queryInt(r, "limit", incidentGroupsDefaultLimit)
if limit <= 0 || limit > incidentGroupsMaxLimit {
limit = incidentGroupsDefaultLimit
}
offset := queryInt(r, "offset", 0)
if offset < 0 {
offset = 0
}
filter := incident.GroupFilter{
Kind: incident.Kind(strings.TrimSpace(q.Get("kind"))),
Offset: offset,
MaxGroups: limit,
}
status := strings.ToLower(strings.TrimSpace(q.Get("status")))
switch status {
case "", "active":
// Default surface: open + contained, the UI's primary tab.
filter.StatusSet = []incident.Status{incident.StatusOpen, incident.StatusContained}
case "all":
// No status filter; the operator wants the full picture.
case string(incident.StatusOpen),
string(incident.StatusContained),
string(incident.StatusResolved),
string(incident.StatusDismissed):
filter.StatusSet = []incident.Status{incident.Status(status)}
default:
writeJSONError(w, "unknown status: "+strconv.Quote(q.Get("status")), http.StatusBadRequest)
return
}
var resp incident.GroupsResponse
if s.incidentCorrelator != nil {
resp = incident.BuildGroups(s.incidentCorrelator.Snapshot(), filter)
}
writeItems(w, resp.Groups, map[string]interface{}{
"total": resp.TotalGroups,
"offset": offset,
"limit": limit,
"scanned_incidents": resp.ScannedIncidents,
"truncated": resp.Truncated || historyPageTruncated(resp.TotalGroups, offset, len(resp.Groups)),
"scan_truncated": resp.Truncated,
})
}
package webui
import (
"errors"
"net/http"
"regexp"
"strings"
"github.com/pidginhost/csm/internal/store"
)
// MailboxSuspensionRow is a suspension CSM made, with the mail server's live
// state when it could be read.
type MailboxSuspensionRow struct {
store.MailboxSuspension
Active bool `json:"active"`
StatusUnknown bool `json:"status_unknown,omitempty"`
}
// MailboxSuspensions lists and releases the suspensions CSM made. The daemon
// implements it; tests use a fake.
type MailboxSuspensions interface {
ListSuspensions() ([]MailboxSuspensionRow, error)
ReleaseSuspension(id, actor string) error
}
// ErrSuspensionReleased reports a release of a suspension already lifted.
var ErrSuspensionReleased = errors.New("suspension already released")
var suspensionIDRe = regexp.MustCompile(`^[0-9a-f]{16}$`)
// SetMailboxSuspensions installs the daemon's suspension service.
func (s *Server) SetMailboxSuspensions(m MailboxSuspensions) {
s.mailboxSuspensions = m
}
// apiEmailSuspensionsList handles GET /api/v1/email/suspensions.
func (s *Server) apiEmailSuspensionsList(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if s.mailboxSuspensions == nil {
writeAll(w, []MailboxSuspensionRow{})
return
}
rows, err := s.mailboxSuspensions.ListSuspensions()
if err != nil {
writeJSONError(w, "Failed to list suspended mailboxes", http.StatusInternalServerError)
return
}
writeAll(w, rows)
}
// apiEmailSuspensionAction handles POST /api/v1/email/suspensions/{id}/release.
// It changes the mail server, so it runs under auth + CSRF and is audit-logged.
func (s *Server) apiEmailSuspensionAction(w http.ResponseWriter, r *http.Request) {
tail := strings.TrimPrefix(r.URL.Path, "/api/v1/email/suspensions/")
id, action, _ := strings.Cut(tail, "/")
if !suspensionIDRe.MatchString(id) {
writeJSONError(w, "Invalid suspension ID", http.StatusBadRequest)
return
}
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if action != "release" {
writeJSONError(w, "Unknown action; use /release", http.StatusBadRequest)
return
}
if s.mailboxSuspensions == nil {
writeJSONError(w, "Mailbox suspensions are not available on this host", http.StatusServiceUnavailable)
return
}
actor, _ := s.requestActor(r)
if err := s.mailboxSuspensions.ReleaseSuspension(id, actor); err != nil {
switch {
case errors.Is(err, store.ErrMailboxSuspensionNotFound):
writeJSONError(w, "Suspension not found", http.StatusNotFound)
case errors.Is(err, ErrSuspensionReleased):
writeJSONError(w, "Suspension already released", http.StatusConflict)
default:
writeJSONError(w, "Release failed: "+err.Error(), http.StatusInternalServerError)
}
return
}
s.auditLog(r, "email_mailbox_release", id, "released a mailbox suspension")
writeOK(w, map[string]any{"id": id})
}
package webui
import (
"crypto/subtle"
"log"
"net/http"
"strings"
"time"
"github.com/pidginhost/csm/internal/config"
)
// mcpPath is where the read-only MCP endpoint answers on the Web UI
// listener.
const mcpPath = "/mcp"
// MCPServer answers MCP requests the Web UI has authenticated. caller names
// the mcp.tokens credential.
type MCPServer interface {
ServeMCP(w http.ResponseWriter, r *http.Request, caller string)
}
// SetMCP attaches the read-only MCP endpoint. Call it before Start.
func (s *Server) SetMCP(m MCPServer) { s.mcp = m }
// serveMCP admits only mcp.tokens credentials: a Web UI token or browser
// session never opens it, and an MCP token opens nothing else. Unknown
// tokens draw on the same per-client guess budget as the login form and
// API bearer tokens.
func (s *Server) serveMCP(w http.ResponseWriter, r *http.Request) {
if s.mcp == nil {
writeJSONError(w, "Not found", http.StatusNotFound)
return
}
if r.Method != http.MethodPost {
// No server-initiated stream and no sessions: the transport allows 405.
w.Header().Set("Allow", http.MethodPost)
writeJSONError(w, "This endpoint accepts POST only", http.StatusMethodNotAllowed)
return
}
tok, status, msg := s.mcpCredential(r)
if status != 0 {
// A spent budget is not logged: a flood would fill the log.
if status != http.StatusTooManyRequests {
// #nosec G706 -- RemoteAddr is the TCP peer address net/http sets and the name comes from csm.yaml, not request content.
log.Printf("mcp: refused token %q from %s: %s", tok.Name, safeLogString(clientIPKey(r.RemoteAddr)), msg)
}
w.Header().Set("WWW-Authenticate", `Bearer realm="csm-mcp"`)
writeJSONError(w, msg, status)
return
}
s.mcp.ServeMCP(w, r, tok.Name)
}
// mcpCredential returns the mcp.tokens entry the request presents and, when
// it is refused, the status and message; a known entry is returned even
// then so the refusal can name it. Budget check, comparison and the charge
// for an unknown token share one critical section, as in refuseSpentBearer.
// A known token outside its dates or addresses is not a guess and is not
// charged.
func (s *Server) mcpCredential(r *http.Request) (config.MCPToken, int, string) {
supplied, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
if !ok || supplied == "" {
return config.MCPToken{}, http.StatusUnauthorized, "Bearer token required"
}
ip := rateLimitKey(r.RemoteAddr)
s.loginMu.Lock()
now := time.Now()
recent := s.recentCredentialAttempts(ip, now)
if len(recent) >= credentialAttemptsPerMinute {
s.loginMu.Unlock()
return config.MCPToken{}, http.StatusTooManyRequests, "Too many failed token attempts"
}
tok, known := mcpTokenFor(s.cfg.MCP.Tokens, supplied)
if !known && !s.knownWebUIBearer(r) {
if _, tracked := s.loginAttempts[ip]; !tracked {
boundRateLimitMap(s.loginAttempts, now.Add(-time.Minute))
}
s.loginAttempts[ip] = append(recent, now)
}
s.loginMu.Unlock()
switch {
case !known:
return config.MCPToken{}, http.StatusUnauthorized, "Invalid token"
case !tok.Current(now):
return tok, http.StatusUnauthorized, "Token expired or not yet valid"
case !tok.AcceptsFrom(requestAddr(r)):
return tok, http.StatusUnauthorized, "Token not accepted from this address"
}
return tok, 0, ""
}
// mcpTokenFor compares supplied against every entry in constant time, so
// timing does not tell which entry matched.
func mcpTokenFor(tokens []config.MCPToken, supplied string) (config.MCPToken, bool) {
var found config.MCPToken
matched := false
for _, tok := range tokens {
if tok.Token != "" && subtle.ConstantTimeCompare([]byte(tok.Token), []byte(supplied)) == 1 && !matched {
found, matched = tok, true
}
}
return found, matched
}
package webui
import (
"crypto/subtle"
"fmt"
"net/http"
"os"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/metrics"
)
// handleMetrics serves Prometheus text exposition from the process
// default metrics registry. ROADMAP item 4.
//
// Auth policy (checked in isMetricsAuthenticated):
//
// - If `webui.metrics_token` is set in the live config, a matching
// `Authorization: Bearer` header unlocks the endpoint. The token
// is read from config.Active() per request so a SIGHUP rotation
// (the field is tagged `hotreload:"safe"`) takes effect without
// a restart.
// - As a fallback, a valid UI session cookie or the UI AuthToken
// Bearer is accepted so the dashboard can self-scrape without a
// second credential.
//
// No CSRF required: metrics is read-only and idempotent.
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if s.refuseSpentBearer(w, r) {
return
}
if !s.isMetricsAuthenticated(r) {
w.Header().Set("WWW-Authenticate", `Bearer realm="csm-metrics"`)
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
if r.Method == http.MethodHead {
return
}
if err := metrics.WriteOpenMetrics(w); err != nil {
// The response body is already partially written; there is no
// meaningful HTTP status to flip. Drop a server log line.
fmt.Fprintf(os.Stderr, "webui: metrics WriteOpenMetrics: %v\n", err)
}
}
func (s *Server) isMetricsAuthenticated(r *http.Request) bool {
return s.metricsBearerMatches(r) || s.isAuthenticated(r)
}
func (s *Server) metricsBearerMatches(r *http.Request) bool {
// Read the metrics token from config.Active() when available so a
// SIGHUP-driven rotation of webui.metrics_token takes effect on
// the next request. MetricsToken is tagged `hotreload:"safe"` for
// this reason, even though its WebUI parent is restart-required
// for the listener/TLS/auth-token fields. Fall back to s.cfg on
// the cold-start window before SetActive is called.
cfg := s.cfg
if live := config.Active(); live != nil {
cfg = live
}
tok := cfg.WebUI.MetricsToken
if tok != "" && config.WebUITokenCurrent(cfg.WebUI.MetricsTokenIssuedAt, cfg.WebUI.MetricsTokenExpiresAt, s.sessionNow()) {
if auth := r.Header.Get("Authorization"); len(auth) > 7 && auth[:7] == "Bearer " {
if subtle.ConstantTimeCompare([]byte(auth[7:]), []byte(tok)) == 1 {
return true
}
}
}
return false
}
package webui
import (
"net"
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/modsec"
"github.com/pidginhost/csm/internal/store"
)
func (s *Server) handleModSec(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "modsec.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
// modsecBlockView is an aggregated view of blocks per IP+rule. Phase 8.4
// extends the response with first_seen / last_seen_iso (RFC3339), top_uris,
// domain_count, and sample_events so the workbench can drive the detail
// panel without a second round trip. The extra fields are additive; legacy
// field names keep their JSON keys.
type modsecBlockView struct {
IP string `json:"ip"`
Country string `json:"country"`
CountryName string `json:"country_name"`
RuleID string `json:"rule_id"`
Description string `json:"description"`
Domains string `json:"domains"`
DomainList []string `json:"domain_list,omitempty"`
DomainCount int `json:"domain_count"`
Hits int `json:"hits"`
LastSeen time.Time `json:"last_seen,omitzero"`
FirstSeen time.Time `json:"first_seen,omitzero"`
TopURIs []string `json:"top_uris"`
SampleEvents []modsecSampleEvent `json:"sample_events"`
Escalated bool `json:"escalated"`
}
// modsecSampleEvent is a compact per-IP event included in the grouped
// blocks response so the UI can show recent activity without a second
// call to /api/v1/modsec/events.
type modsecSampleEvent struct {
Time time.Time `json:"time"`
RuleID string `json:"rule_id"`
Hostname string `json:"hostname"`
URI string `json:"uri"`
Severity string `json:"severity"`
}
// modsecEventView is a single ModSecurity event. The UI renders Time in the
// operator's time zone and sorts on it.
type modsecEventView struct {
Time time.Time `json:"time"`
IP string `json:"ip"`
Country string `json:"country"`
RuleID string `json:"rule_id"`
Hostname string `json:"hostname"`
URI string `json:"uri"`
Severity string `json:"severity"`
}
// apiModSecStats returns 24h summary stats for ModSecurity blocks.
func (s *Server) apiModSecStats(w http.ResponseWriter, r *http.Request) {
findings := deduplicateModSecFindings(s.modsecFindings(r))
uniqueIPs := make(map[string]bool)
ruleCounts := make(map[string]int)
escalated := 0
for _, f := range findings {
ip := extractModSecIP(f)
if ip != "" {
uniqueIPs[ip] = true
}
rule := extractModSecRule(f)
if rule != "" {
ruleCounts[rule]++
}
if isModSecEscalation(f.Check) {
escalated++
}
}
topRule := "--"
topCount := 0
for rule, count := range ruleCounts {
if count > topCount {
topCount = count
topRule = rule
}
}
writeJSON(w, map[string]interface{}{
"total": len(findings),
"unique_ips": len(uniqueIPs),
"escalated": escalated,
"top_rule": topRule,
})
}
const (
// modsecFindingsScanCap bounds the 24h history read before per-IP
// aggregation starts. The aggregate map has its own cap below, but
// the handler still needs a findings cap for hosts seeing one or two
// hot IP+rule pairs millions of times.
modsecFindingsScanCap = 10000
// modsecBlocksMaxAggregates caps the IP+rule aggregation map so a host
// with millions of unique ModSec rule hits cannot OOM the daemon by
// asking for /api/v1/modsec/blocks. Existing aggregates keep updating
// after the cap is reached; new IP+rule keys past the cap are dropped
// silently with the X-CSM-Truncated response header set so monitoring
// can flag the condition. Default sized for ~50 MB peak: 50000 entries
// times a few hundred bytes per aggregate.
modsecBlocksMaxAggregates = 50000
)
// apiModSecBlocks returns aggregated blocks per IP+rule for the last 24h.
func (s *Server) apiModSecBlocks(w http.ResponseWriter, r *http.Request) {
findings, truncated := s.modsecFindingsWithTruncation(r)
findings = deduplicateModSecFindings(findings)
type blockAgg struct {
ip string
ruleID string
description string
domains map[string]bool
uriCounts map[string]int
hits int
firstSeen time.Time
lastSeen time.Time
escalated bool
samples []modsecSampleEvent // newest-first, capped at 3
}
byBlock := make(map[string]*blockAgg)
// byIP indexes the aggregates per address, so marking escalated
// addresses does not scan every aggregate once per address.
byIP := make(map[string][]*blockAgg)
escalatedIPs := make(map[string]bool)
blockKey := func(ip, rule string) string {
return ip + "\x00" + rule
}
for _, f := range findings {
if isModSecEscalation(f.Check) {
ip := extractModSecIP(f)
if ip != "" {
escalatedIPs[ip] = true
}
continue
}
ip := extractModSecIP(f)
if ip == "" {
continue
}
rule := extractModSecRule(f)
desc := extractModSecDescription(f)
domain := extractModSecHostname(f)
uri := extractModSecURI(f)
key := blockKey(ip, rule)
agg, ok := byBlock[key]
if !ok {
if len(byBlock) >= modsecBlocksMaxAggregates {
truncated = true
continue
}
agg = &blockAgg{
ip: ip,
ruleID: rule,
description: desc,
domains: make(map[string]bool),
uriCounts: make(map[string]int),
firstSeen: f.Timestamp,
}
byBlock[key] = agg
byIP[ip] = append(byIP[ip], agg)
}
agg.hits++
if agg.firstSeen.IsZero() || f.Timestamp.Before(agg.firstSeen) {
agg.firstSeen = f.Timestamp
}
if f.Timestamp.After(agg.lastSeen) {
agg.lastSeen = f.Timestamp
if rule != "" {
agg.ruleID = rule
}
if desc != "" {
agg.description = desc
}
}
if agg.description == "" && desc != "" {
agg.description = desc
}
// Skip server IPs and empty hostnames - only show actual domain names.
// ModSecurity logs the server IP as hostname when the request doesn't
// match a specific vhost (e.g. direct IP access, SNI mismatch).
if domain != "" && !looksLikeIP(domain) {
agg.domains[domain] = true
}
if uri != "" {
agg.uriCounts[uri]++
}
if len(agg.samples) < 3 {
agg.samples = append(agg.samples, modsecSampleEvent{
Time: f.Timestamp.UTC(),
RuleID: rule,
Hostname: domain,
URI: uri,
Severity: f.Severity.String(),
})
}
}
for ip := range escalatedIPs {
for _, agg := range byIP[ip] {
agg.escalated = true
}
if len(byIP[ip]) == 0 {
if len(byBlock) >= modsecBlocksMaxAggregates {
truncated = true
continue
}
byBlock[blockKey(ip, "")] = &blockAgg{
ip: ip,
escalated: true,
domains: make(map[string]bool),
uriCounts: make(map[string]int),
}
}
}
var result []modsecBlockView
for _, agg := range byBlock {
if agg.hits == 0 && !agg.escalated {
continue
}
var domainList []string
for d := range agg.domains {
domainList = append(domainList, d)
}
sort.Strings(domainList)
domains := strings.Join(domainList, ", ")
if len(domains) > 80 {
domains = domains[:77] + "..."
}
topURIs := topKeysByCount(agg.uriCounts, 5)
country, countryName := s.modsecCountryOf(agg.ip)
result = append(result, modsecBlockView{
IP: agg.ip,
Country: country,
CountryName: countryName,
RuleID: agg.ruleID,
Description: agg.description,
Domains: domains,
DomainList: domainList,
DomainCount: len(agg.domains),
Hits: agg.hits,
LastSeen: agg.lastSeen.UTC(),
FirstSeen: agg.firstSeen.UTC(),
TopURIs: topURIs,
SampleEvents: agg.samples,
Escalated: agg.escalated,
})
}
sort.Slice(result, func(i, j int) bool {
if result[i].Hits != result[j].Hits {
return result[i].Hits > result[j].Hits
}
if result[i].Escalated != result[j].Escalated {
return result[i].Escalated
}
if !result[i].LastSeen.Equal(result[j].LastSeen) {
return result[i].LastSeen.After(result[j].LastSeen)
}
if result[i].IP != result[j].IP {
return result[i].IP < result[j].IP
}
return result[i].RuleID < result[j].RuleID
})
if truncated {
w.Header().Set("X-CSM-Truncated", "1")
}
writeItems(w, result, map[string]interface{}{
"total": len(result), "offset": 0, "limit": modsecBlocksMaxAggregates, "truncated": truncated,
})
}
// apiModSecEvents returns the most recent individual ModSecurity events.
func (s *Server) apiModSecEvents(w http.ResponseWriter, r *http.Request) {
limit := 100
if l := r.URL.Query().Get("limit"); l != "" {
if n, err := strconv.Atoi(l); err == nil && n > 0 && n <= 500 {
limit = n
}
}
findings, truncated := s.modsecFindingsWithTruncation(r)
findings = deduplicateModSecFindings(findings)
result := make([]modsecEventView, 0, limit)
total := 0
for _, f := range findings {
if isModSecEscalation(f.Check) {
continue
}
total++
if len(result) >= limit {
continue
}
ip := extractModSecIP(f)
country, _ := s.modsecCountryOf(ip)
result = append(result, modsecEventView{
Time: f.Timestamp.UTC(),
IP: ip,
Country: country,
RuleID: extractModSecRule(f),
Hostname: extractModSecHostname(f),
URI: extractModSecURI(f),
Severity: f.Severity.String(),
})
}
writeCapped(w, result, total, limit, map[string]interface{}{"truncated": truncated})
}
// deduplicateModSecFindings merges Apache + LiteSpeed duplicate events.
// Both log the same block within the same second - keep one with merged fields.
func deduplicateModSecFindings(findings []alert.Finding) []alert.Finding {
type dedupKey struct {
second time.Time
ip string
rule string
escalation bool
}
seen := make(map[dedupKey]int) // key → index in result
var result []alert.Finding
for _, f := range findings {
ip := extractModSecIP(f)
rule := extractModSecRule(f)
ts := f.Timestamp.UTC().Truncate(time.Second)
// An escalation and its HTTP event share fields but both must survive:
// the blocks view uses one for ban status and the other for hit counts.
key := dedupKey{second: ts, ip: ip, rule: rule, escalation: isModSecEscalation(f.Check)}
if idx, ok := seen[key]; ok {
// Merge richer details into existing entry
existing := &result[idx]
if extractModSecHostname(f) != "" && extractModSecHostname(*existing) == "" {
existing.Details = f.Details
}
} else {
seen[key] = len(result)
result = append(result, f)
}
}
return result
}
func isModSecEscalation(check string) bool {
return check == "modsec_block_escalation" || check == "modsec_csm_block_escalation"
}
// modsecWindow maps the optional window query param to a lookback duration.
// Missing or unrecognized values fall back to 24h, which also bounds the read
// since history retention and the scan cap assume roughly a day.
func modsecWindow(r *http.Request) time.Duration {
switch r.URL.Query().Get("window") {
case "1h":
return time.Hour
case "6h":
return 6 * time.Hour
default:
return 24 * time.Hour
}
}
// modsecSeverityFilter maps the optional severity query param to a minimum
// severity. ok is false when no (or an unrecognized) filter is requested, in
// which case all severities pass.
func modsecSeverityFilter(r *http.Request) (alert.Severity, bool) {
return parseSeverity(r.URL.Query().Get("severity"))
}
// modsecCountryOf resolves an IP to its ISO country code and full name via the
// loaded GeoIP database. Returns empty strings when the IP is empty or no
// GeoIP database is loaded.
func (s *Server) modsecCountryOf(ip string) (string, string) {
if ip == "" {
return "", ""
}
db := s.geoIPDB.Load()
if db == nil {
return "", ""
}
info := db.Lookup(ip)
return info.Country, info.CountryName
}
// modsecFindings returns modsec findings for the request window and severity
// filter.
func (s *Server) modsecFindings(r *http.Request) []alert.Finding {
findings, _ := s.modsecFindingsWithTruncation(r)
return findings
}
func (s *Server) modsecFindingsWithTruncation(r *http.Request) ([]alert.Finding, bool) {
db := store.Global()
if db == nil {
return nil, false
}
cutoff := time.Now().Add(-modsecWindow(r))
minSev, hasSev := modsecSeverityFilter(r)
findings := db.SearchHistorySince(cutoff, modsecFindingsScanCap+1, func(f alert.Finding) bool {
if !strings.HasPrefix(f.Check, "modsec_") {
return false
}
if hasSev && f.Severity < minSev {
return false
}
return true
})
if len(findings) > modsecFindingsScanCap {
return findings[:modsecFindingsScanCap], true
}
return findings, false
}
// --- Field extraction from finding Details ---
// Details format: "Rule: NNNN\nMessage: ...\nHostname: ...\nURI: ..."
func extractModSecIP(f alert.Finding) string {
if f.SourceIP != "" {
if ip := net.ParseIP(f.SourceIP); ip != nil {
return ip.String()
}
return ""
}
// Older findings may carry only the raw log. Display text includes
// request data, so it cannot supply an address when the source is unknown.
raw := f.Details
if strings.HasPrefix(raw, "Rule: ") {
raw = extractDetailField(raw, "Raw: ")
}
return modsec.ErrorLogClientIP(raw)
}
func extractModSecRule(f alert.Finding) string {
return modsecDetailField(f, "Rule: ", func(r modsec.LogRule) string { return r.ID })
}
// modsecDetailField reads a field the realtime handler stored. Structured
// details keep the raw line too, whose request text can imitate any field, so
// only complete raw-line details from older findings are parsed. A truncated
// prefix can lose the duplicate fields that reveal an imitation.
func modsecDetailField(f alert.Finding, prefix string, field func(modsec.LogRule) string) string {
if v := extractDetailField(f.Details, prefix); v != "" || strings.HasPrefix(f.Details, "Rule: ") {
return v
}
if strings.HasSuffix(f.Details, "...") {
return ""
}
rule, _ := modsec.ErrorLogRule(f.Details)
return field(rule)
}
// csmRuleDescriptions provides fallback descriptions for known ModSecurity
// rules.
// LiteSpeed error logs omit the [msg "..."] field, so the log-extracted
// description is often empty. This map ensures the UI always shows a
// meaningful description for rules we define ourselves and common vendor rules.
var csmRuleDescriptions = map[string]string{
"900001": "Blocked LEVIATHAN CGI extension access",
"900002": "Blocked LEVIATHAN directory access",
"900003": "Blocked PHP execution in uploads directory",
"900004": "Blocked PHP execution in languages directory",
"900005": "Blocked direct wp-config.php access",
"900007": "XML-RPC rate limit exceeded",
"900008": "Blocked known webshell filename access",
"900009": "Blocked GSocket User-Agent",
"900100": "WP-Automatic SQLi (CVE-2024-27956)",
"900101": "LayerSlider SQLi (CVE-2024-2879)",
"900102": "Really Simple Security auth bypass (CVE-2024-10924)",
"900103": "LiteSpeed Cache directory traversal (CVE-2024-4345)",
"900104": "Ultimate Member SQLi (CVE-2024-1071)",
"900105": "Backup Migration RCE (CVE-2023-6553)",
"900106": "GiveWP object injection (CVE-2024-5932)",
"900107": "WP File Manager arbitrary upload (CVE-2024-3400)",
"900110": "PHP object injection attempt",
"900111": "Blocked PHP in wp-content/upgrade",
"900112": "WordPress user enumeration blocked",
"900113": "wp-login brute force rate limit",
"900114": "wp-login brute force rate limit",
"900115": "Blocked .env file access",
"900116": "Blocked scanner probe",
"900120": "Blocked wp-coder preview endpoint",
"900121": "Blocked wp-coder attributes endpoint",
"900122": "Blocked wp2shell exploit tool User-Agent",
"900123": "REST batch request counter",
"900124": "REST batch endpoint rate limit",
"900125": "Blocked wp2shell tool fingerprint",
"900126": "Blocked wp2shell tool fingerprint (encoded)",
"900127": "Ultimate Member privilege escalation (CVE-2023-3460)",
// Comodo WAF (CWAF) common rules. Rule IDs in the 21xxxx range are
// from the Comodo vendor ruleset (e.g. /etc/apache2/conf.d/
// modsec_vendor_configs/comodo_litespeed/), NOT from OWASP CRS.
// They were previously mislabeled as "OWASP:" here.
"210710": "Comodo WAF: Request content type is not allowed by policy",
"210381": "Comodo WAF: URL encoding abuse attack attempt",
"214930": "Comodo WAF: Inbound anomaly score threshold exceeded",
"214940": "Comodo WAF: Outbound anomaly score threshold exceeded",
"218420": "Comodo WAF: Request content type restriction",
// OWASP CRS common rules. IDs in the 9xxxxx range are the standard
// OWASP CRS 3.x schema (920xxx protocol, 930xxx LFI, 941xxx XSS,
// 942xxx SQLi).
"920170": "OWASP: Validate GET/HEAD request",
"920420": "OWASP: Request content type is not allowed by policy",
"920600": "OWASP: Illegal Accept header",
"930100": "OWASP: Path traversal attack",
"930110": "OWASP: Path traversal attack",
"930120": "OWASP: OS file access attempt",
"941100": "OWASP: XSS attack detected via libinjection",
"941160": "OWASP: XSS Filter - Category 1",
"942100": "OWASP: SQL injection attack detected via libinjection",
}
func extractModSecDescription(f alert.Finding) string {
if v := modsecDetailField(f, "Message: ", func(r modsec.LogRule) string { return r.Msg }); v != "" {
return v
}
// Fallback: use static description for CSM custom rules when the log
// format (e.g. LiteSpeed) doesn't include the [msg "..."] field.
rule := extractModSecRule(f)
if desc, ok := csmRuleDescriptions[rule]; ok {
return desc
}
return ""
}
func extractModSecHostname(f alert.Finding) string {
return modsecDetailField(f, "Hostname: ", func(r modsec.LogRule) string { return r.Hostname })
}
func extractModSecURI(f alert.Finding) string {
return modsecDetailField(f, "URI: ", func(r modsec.LogRule) string { return r.URI })
}
func extractDetailField(details, prefix string) string {
for _, line := range strings.Split(details, "\n") {
if strings.HasPrefix(line, prefix) {
return strings.TrimPrefix(line, prefix)
}
}
return ""
}
// looksLikeIP returns true if the string looks like an IP address (not a domain).
func looksLikeIP(s string) bool {
if len(s) < 7 {
return false
}
for _, c := range s {
if c != '.' && (c < '0' || c > '9') {
return false
}
}
return strings.Count(s, ".") == 3
}
package webui
import (
"fmt"
"net/http"
"os"
"sort"
"strconv"
"time"
"github.com/pidginhost/csm/internal/modsec"
"github.com/pidginhost/csm/internal/store"
)
func validateModSecDisabledRules(allRules []modsec.Rule, disabled []int) error {
knownIDs := make(map[int]bool)
counterIDs := make(map[int]bool)
for _, rule := range allRules {
knownIDs[rule.ID] = true
if rule.IsCounter {
counterIDs[rule.ID] = true
}
}
for _, id := range disabled {
if !knownIDs[id] {
return fmt.Errorf("rule ID %d is not a known CSM rule", id)
}
if counterIDs[id] {
return fmt.Errorf("rule ID %d is a protected bookkeeping rule", id)
}
}
return nil
}
func (s *Server) handleModSecRules(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "modsec-rules.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
// GET /api/v1/modsec/rules - list all CSM rules with status
func (s *Server) apiModSecRules(w http.ResponseWriter, _ *http.Request) {
cfg := s.liveCfg().ModSec
// Check all three config fields
var missing []string
if cfg.RulesFile == "" {
missing = append(missing, "rules_file")
}
if cfg.OverridesFile == "" {
missing = append(missing, "overrides_file")
}
if cfg.ReloadCommand == "" {
missing = append(missing, "reload_command")
}
if len(missing) > 0 {
writeItems(w, []struct{}{}, map[string]interface{}{
"total": 0,
"configured": false,
"missing": missing,
})
return
}
// Parse rules from config file
allRules, err := modsec.ParseRulesFile(cfg.RulesFile)
if err != nil {
fmt.Fprintf(os.Stderr, "modsec: parse rules failed: %v\n", err)
writeJSONError(w, "Failed to parse rules file", http.StatusInternalServerError)
return
}
// Read disabled IDs from overrides file
disabledIDs, _ := modsec.ReadOverrides(cfg.OverridesFile)
disabledSet := make(map[int]bool)
for _, id := range disabledIDs {
disabledSet[id] = true
}
// Read escalation exclusions and hit counts from store
var noEscalate map[int]bool
var hits map[int]store.RuleHitStats
if db := store.Global(); db != nil {
noEscalate, err = db.GetModSecNoEscalateRules()
if err != nil {
writeJSONError(w, "Failed to read escalation exclusions", http.StatusInternalServerError)
return
}
hits = db.GetModSecRuleHits()
}
// Build response - filter out counter rules
type ruleView struct {
ID int `json:"id"`
Description string `json:"description"`
Action string `json:"action"`
StatusCode int `json:"status_code"`
Phase int `json:"phase"`
Enabled bool `json:"enabled"`
Escalate bool `json:"escalate"`
Hits24h int `json:"hits_24h"`
LastHit time.Time `json:"last_hit,omitzero"`
}
var rules []ruleView
for _, r := range allRules {
if r.IsCounter {
continue // hide bookkeeping rules
}
rv := ruleView{
ID: r.ID,
Description: r.Description,
Action: r.Action,
StatusCode: r.StatusCode,
Phase: r.Phase,
Enabled: !disabledSet[r.ID],
Escalate: !noEscalate[r.ID],
}
if h, ok := hits[r.ID]; ok {
rv.Hits24h = h.Hits
rv.LastHit = h.LastHit.UTC()
}
rules = append(rules, rv)
}
active := 0
for _, r := range rules {
if r.Enabled {
active++
}
}
writeItems(w, rules, map[string]interface{}{
"total": len(rules),
"active": active,
"disabled": disabledIDs,
"configured": true,
})
}
// POST /api/v1/modsec/rules/apply - write overrides and reload
func (s *Server) apiModSecRulesApply(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
// Serialize apply operations - the write+reload+rollback sequence
// must not interleave with concurrent applies.
s.modSecApplyMu.Lock()
defer s.modSecApplyMu.Unlock()
cfg := s.liveCfg().ModSec
if cfg.RulesFile == "" || cfg.OverridesFile == "" || cfg.ReloadCommand == "" {
writeJSONError(w, "ModSecurity not configured", http.StatusBadRequest)
return
}
var req struct {
Disabled []int `json:"disabled"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil {
writeJSONError(w, "Invalid request body", http.StatusBadRequest)
return
}
// Validate: only allow disabling known CSM rule IDs from the parsed rules file
allRules, err := modsec.ParseRulesFile(cfg.RulesFile)
if err != nil {
fmt.Fprintf(os.Stderr, "modsec: parse rules failed: %v\n", err)
writeJSONError(w, "Failed to parse rules file", http.StatusInternalServerError)
return
}
if err := validateModSecDisabledRules(allRules, req.Disabled); err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
// Save previous state for rollback
previousContent := modsec.ReadOverridesRaw(cfg.OverridesFile)
// Write new overrides
if writeErr := modsec.WriteOverrides(cfg.OverridesFile, req.Disabled); writeErr != nil {
fmt.Fprintf(os.Stderr, "modsec: write overrides failed: %v\n", writeErr)
writeJSONError(w, "Failed to write overrides", http.StatusInternalServerError)
return
}
// Reload web server
output, reloadErr := modsec.Reload(cfg.ReloadCommand)
if reloadErr != nil {
// Rollback on failure
rollbackErr := modsec.RestoreOverrides(cfg.OverridesFile, previousContent)
outcome := "Web server reload failed, changes rolled back"
if rollbackErr != nil {
outcome = "Web server reload failed; rollback failed, check the overrides before reloading"
fmt.Fprintf(os.Stderr, "modsec: rollback failed: %v\n", rollbackErr)
}
fmt.Fprintf(os.Stderr, "modsec: reload failed: %v\noutput: %s\n", reloadErr, output)
// Truncate output for client - may contain sensitive system paths
clientOutput := output
if len(clientOutput) > 500 {
clientOutput = clientOutput[:500] + "... (truncated)"
}
s.auditLog(r, "modsec_rules_apply_failed", "overrides", outcome)
writeJSONStatus(w, http.StatusInternalServerError, map[string]interface{}{
"error": outcome,
"reload_output": clientOutput,
"rolled_back": rollbackErr == nil,
})
return
}
s.auditLog(r, "modsec_rules_apply", "overrides", fmt.Sprintf("disabled rules: %v", req.Disabled))
writeOK(w, map[string]interface{}{
"disabled_count": len(req.Disabled),
})
}
// GET /api/v1/modsec/rules/escalation - rule IDs excluded from escalation
// POST /api/v1/modsec/rules/escalation - toggle escalation for a single rule
func (s *Server) apiModSecRulesEscalation(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
db := store.Global()
if db == nil {
writeJSONError(w, "Store not available", http.StatusInternalServerError)
return
}
if r.Method == http.MethodGet {
rules, err := db.GetModSecNoEscalateRules()
if err != nil {
writeJSONError(w, "Failed to read escalation exclusions", http.StatusInternalServerError)
return
}
ids := []int{}
for id := range rules {
ids = append(ids, id)
}
sort.Ints(ids)
writeAll(w, ids)
return
}
var req struct {
RuleID int `json:"rule_id"`
Escalate bool `json:"escalate"`
}
if err := decodeJSONBodyLimited(w, r, 4*1024, &req); err != nil {
writeJSONError(w, "Invalid request body", http.StatusBadRequest)
return
}
// Validate: only accept known CSM rule IDs (900000-900999)
if req.RuleID < 900000 || req.RuleID > 900999 {
writeJSONError(w, "Rule ID must be a CSM custom rule (900000-900999)", http.StatusBadRequest)
return
}
var err error
if req.Escalate {
err = db.RemoveModSecNoEscalateRule(req.RuleID)
} else {
err = db.AddModSecNoEscalateRule(req.RuleID)
}
if err != nil {
fmt.Fprintf(os.Stderr, "modsec: escalation update failed: %v\n", err)
writeJSONError(w, "Failed to update escalation setting", http.StatusInternalServerError)
return
}
setting := "escalation off"
if req.Escalate {
setting = "escalation on"
}
s.auditLog(r, "modsec_rule_escalation", strconv.Itoa(req.RuleID), setting)
writeOK(w, map[string]interface{}{"rule_id": req.RuleID, "escalate": req.Escalate})
}
package webui
import (
"bufio"
"context"
"fmt"
"net/http"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/mysqlclient"
"github.com/pidginhost/csm/internal/redisinfo"
)
// --- Local response types ---
type perfResponse struct {
Metrics *perfMetrics `json:"metrics"`
Findings []perfFindingView `json:"findings"`
}
type perfMetrics struct {
LoadAvg [3]float64 `json:"load_avg"`
CPUCores int `json:"cpu_cores"`
MemTotalMB uint64 `json:"mem_total_mb"`
MemUsedMB uint64 `json:"mem_used_mb"`
MemAvailMB uint64 `json:"mem_avail_mb"`
SwapTotalMB uint64 `json:"swap_total_mb"`
SwapUsedMB uint64 `json:"swap_used_mb"`
PHPProcs int `json:"php_procs_total"`
TopPHPUsers []userProcs `json:"top_php_users"`
// MySQL telemetry is best-effort. Both fields are nil when csm could
// not read the server's process status or the mysql client failed (no /root/.my.cnf,
// no socket auth, mysqld absent). The webui renders "n/a" in that case
// so operators can tell "MySQL is idle" from "we couldn't ask".
// Redis follows the same rule: the memory fields are nil when INFO
// memory failed, RedisKeys when INFO keyspace failed.
MySQLMemMB *uint64 `json:"mysql_mem_mb"`
MySQLConns *int `json:"mysql_conns"`
RedisMemMB *uint64 `json:"redis_mem_mb"`
RedisMaxMB *uint64 `json:"redis_maxmem_mb"`
RedisKeys *int64 `json:"redis_keys"`
UptimeSeconds int64 `json:"uptime_seconds"`
}
type userProcs struct {
User string `json:"user"`
Count int `json:"count"`
}
type perfFindingView struct {
Severity string `json:"severity"`
level alert.Severity
Check string `json:"check"`
Message string `json:"message"`
Details string `json:"details,omitempty"`
Key string `json:"key"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
}
// --- Cached values ---
var (
perfCoresOnce sync.Once
perfCoresCache int
perfUIDMapOnce sync.Once
perfUIDMapCache map[string]string
)
// cachedCores reads /proc/cpuinfo once and counts "processor\t" lines.
func cachedCores() int {
perfCoresOnce.Do(func() {
f, err := os.Open("/proc/cpuinfo")
if err != nil {
perfCoresCache = 1
return
}
defer func() { _ = f.Close() }()
count := 0
scanner := bufio.NewScanner(f)
for scanner.Scan() {
if strings.HasPrefix(scanner.Text(), "processor\t") {
count++
}
}
if count == 0 {
count = 1
}
perfCoresCache = count
})
return perfCoresCache
}
// cachedUID resolves a UID string to a username via /etc/passwd, cached.
func cachedUID(uid string) string {
perfUIDMapOnce.Do(func() {
perfUIDMapCache = make(map[string]string)
data, err := os.ReadFile("/etc/passwd")
if err != nil {
return
}
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Split(line, ":")
if len(fields) >= 3 {
perfUIDMapCache[fields[2]] = fields[0]
}
}
})
if name, ok := perfUIDMapCache[uid]; ok {
return name
}
return uid
}
// --- Metrics sampler ---
// runCmdQuick runs a command with a 5-second timeout. All call sites pass
// constant binary names (mysql, redis-cli, etc.) and literal argument
// lists — no HTTP-controlled input reaches this function.
func runCmdQuick(name string, args ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
// #nosec G204 -- see function-level comment: constant names/args only.
out, err := exec.CommandContext(ctx, name, args...).Output()
if ctx.Err() == context.DeadlineExceeded {
return nil, fmt.Errorf("command timed out: %s", name)
}
return out, err
}
// isPHPWorkerCmdline reports whether a /proc cmdline belongs to a PHP process
// that serves requests.
//
// Two forms exist across the supported stacks: LiteSpeed spawns lsphp, and
// cPanel EA4 on Apache runs php-fpm, whose workers retitle themselves
// "php-fpm: pool <name>" and run as the account user. Matching only lsphp
// reported zero PHP activity on every Apache host.
//
// The php-fpm master is excluded on purpose: it runs as root and serves no
// requests, so counting it would attribute per-account load to root.
func isPHPWorkerCmdline(cmdline string) bool {
if strings.HasPrefix(cmdline, "php-fpm: pool ") {
return true
}
if strings.HasPrefix(cmdline, "php-fpm: master") {
return false
}
return strings.Contains(cmdline, "lsphp")
}
// isMySQLServerCmdline reports whether a /proc cmdline is the database server
// itself, as opposed to a client, a wrapper script, or a backup tool.
//
// The pid file used to be read from a hardcoded path, which does not exist on
// the cPanel/MariaDB hosts CSM primarily targets: MariaDB writes
// /var/lib/mysql/<host>.pid instead. Matching the process avoids maintaining a
// list of per-distribution pid paths.
func isMySQLServerCmdline(cmdline string) bool {
fields := strings.Fields(cmdline)
if len(fields) == 0 {
return false
}
base := filepath.Base(fields[0])
// A shell running mysqld_safe has the shell as argv[0]; neither it nor the
// wrapper is the server process.
return base == "mysqld" || base == "mariadbd"
}
// mysqlServerRSSMB returns the resident set size of the running database
// server in MB, or nil when no server process is found.
func mysqlServerRSSMB() *uint64 {
cmdlinePaths, _ := filepath.Glob("/proc/[0-9]*/cmdline")
for _, cmdPath := range cmdlinePaths {
// #nosec G304 -- cmdPath from /proc/*/cmdline glob; kernel pseudo-FS.
data, err := os.ReadFile(cmdPath)
if err != nil {
continue
}
if !isMySQLServerCmdline(strings.ReplaceAll(string(data), "\x00", " ")) {
continue
}
// #nosec G304 -- /proc/<pid>/status; kernel pseudo-FS.
statusData, _ := os.ReadFile(filepath.Join(filepath.Dir(cmdPath), "status"))
for _, line := range strings.Split(string(statusData), "\n") {
if !strings.HasPrefix(line, "VmRSS:") {
continue
}
fields := strings.Fields(line)
if len(fields) >= 2 {
if kb, perr := strconv.ParseUint(fields[1], 10, 64); perr == nil {
mb := kb / 1024
return &mb
}
}
break
}
}
return nil
}
// sampleMetrics gathers live system metrics and returns a populated perfMetrics.
func sampleMetrics() *perfMetrics {
m := &perfMetrics{}
// Load averages
if data, err := os.ReadFile("/proc/loadavg"); err == nil {
fields := strings.Fields(string(data))
if len(fields) >= 3 {
for i := 0; i < 3; i++ {
v, _ := strconv.ParseFloat(fields[i], 64)
m.LoadAvg[i] = v
}
}
}
// CPU cores
m.CPUCores = cachedCores()
// Memory from /proc/meminfo
{
f, err := os.Open("/proc/meminfo")
if err == nil {
var memTotal, memAvail, memFree, memBuffers, memCached, swapTotal, swapFree uint64
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
val, _ := strconv.ParseUint(fields[1], 10, 64)
switch fields[0] {
case "MemTotal:":
memTotal = val
case "MemAvailable:":
memAvail = val
case "MemFree:":
memFree = val
case "Buffers:":
memBuffers = val
case "Cached:":
memCached = val
case "SwapTotal:":
swapTotal = val
case "SwapFree:":
swapFree = val
}
}
_ = f.Close()
m.MemTotalMB = memTotal / 1024
m.MemAvailMB = memAvail / 1024
// Used = Total - Free - Buffers - Cached
used := memTotal
if memFree+memBuffers+memCached <= memTotal {
used = memTotal - memFree - memBuffers - memCached
}
m.MemUsedMB = used / 1024
m.SwapTotalMB = swapTotal / 1024
if swapFree <= swapTotal {
m.SwapUsedMB = (swapTotal - swapFree) / 1024
}
}
}
// PHP processes: scan /proc/*/cmdline for PHP request workers.
{
cmdlinePaths, _ := filepath.Glob("/proc/[0-9]*/cmdline")
userCounts := make(map[string]int)
total := 0
for _, cmdPath := range cmdlinePaths {
// #nosec G304 -- cmdPath from /proc/*/cmdline glob; kernel pseudo-FS.
data, err := os.ReadFile(cmdPath)
if err != nil {
continue
}
cmdStr := strings.ReplaceAll(string(data), "\x00", " ")
if !isPHPWorkerCmdline(cmdStr) {
continue
}
pid := filepath.Base(filepath.Dir(cmdPath))
// #nosec G304 -- /proc/<pid>/status; kernel pseudo-FS, pid from /proc glob.
statusData, _ := os.ReadFile(filepath.Join("/proc", pid, "status"))
uid := ""
for _, line := range strings.Split(string(statusData), "\n") {
if strings.HasPrefix(line, "Uid:\t") {
f := strings.Fields(strings.TrimPrefix(line, "Uid:\t"))
if len(f) > 0 {
uid = f[0]
}
break
}
}
if uid == "" {
uid = "unknown"
}
username := cachedUID(uid)
userCounts[username]++
total++
}
m.PHPProcs = total
// Build sorted top-10 list
type up struct {
user string
count int
}
var ups []up
for u, c := range userCounts {
ups = append(ups, up{u, c})
}
sort.Slice(ups, func(i, j int) bool {
return ups[i].count > ups[j].count
})
if len(ups) > 10 {
ups = ups[:10]
}
m.TopPHPUsers = make([]userProcs, len(ups))
for i, u := range ups {
m.TopPHPUsers[i] = userProcs{User: u.user, Count: u.count}
}
}
// MySQL: PID -> VmRSS, plus Threads_connected. Both fields stay nil
// when the lookup fails so the webui can show "n/a" instead of a
// misleading 0.
{
m.MySQLMemMB = mysqlServerRSSMB()
// Connection count. mysqlclient open returns nil on auth failure,
// missing socket, or absent server -- in every such case we
// leave MySQLConns nil rather than reporting a fake 0.
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
rows, err := mysqlclient.RootQuery(ctx, "SHOW STATUS LIKE 'Threads_connected'")
cancel()
if err == nil && len(rows) > 0 {
fields := strings.Fields(rows[0])
if len(fields) >= 2 {
if n, perr := strconv.Atoi(fields[1]); perr == nil {
m.MySQLConns = &n
}
}
}
}
// Redis: memory + keyspace via in-process client (no redis-cli fork).
{
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if used, max, err := redisinfo.MemoryUsage(ctx); err == nil {
usedMB, maxMB := used/(1024*1024), max/(1024*1024)
m.RedisMemMB, m.RedisMaxMB = &usedMB, &maxMB
}
if total, err := redisinfo.Keyspace(ctx); err == nil {
m.RedisKeys = &total
}
}
// Uptime from /proc/uptime
{
data, err := os.ReadFile("/proc/uptime")
if err == nil {
fields := strings.Fields(string(data))
if len(fields) >= 1 {
secs, _ := strconv.ParseFloat(fields[0], 64)
m.UptimeSeconds = int64(secs)
}
}
}
return m
}
// perfSampleTTL is how long a metrics sample is served before the next
// request takes a new one. Var so tests can force a fresh sample.
var perfSampleTTL = 10 * time.Second
// perfSample is one metrics sample and when it was taken.
type perfSample struct {
metrics *perfMetrics
at time.Time
}
func (s *Server) storePerfSample(m *perfMetrics, at time.Time) {
s.perfSample.Store(&perfSample{metrics: m, at: at})
}
func (s *Server) freshPerfSample() (*perfMetrics, bool) {
p := s.perfSample.Load()
if p == nil || time.Since(p.at) >= perfSampleTTL {
return nil, false
}
return p.metrics, true
}
// currentPerfMetrics returns a sample no older than perfSampleTTL, taking
// one if needed. Requests that arrive while a sample is being taken wait for
// it instead of sampling again.
func (s *Server) currentPerfMetrics() *perfMetrics {
if m, ok := s.freshPerfSample(); ok {
return m
}
s.perfMu.Lock()
defer s.perfMu.Unlock()
if m, ok := s.freshPerfSample(); ok {
return m
}
sample := s.samplePerf
if sample == nil {
sample = sampleMetrics
}
m := sample()
s.storePerfSample(m, time.Now())
return m
}
// apiPerformance returns the latest performance snapshot plus perf_ findings.
func (s *Server) apiPerformance(w http.ResponseWriter, r *http.Request) {
limit := queryInt(r, "limit", 100)
if limit > 500 {
limit = 500
}
metrics := s.currentPerfMetrics()
latest := s.store.LatestFindings()
suppressions := s.store.LoadSuppressions()
var views []perfFindingView
for _, f := range latest {
if !strings.HasPrefix(f.Check, "perf_") {
continue
}
if s.store.IsSuppressed(f, suppressions) {
continue
}
firstSeen := f.Timestamp
lastSeen := f.Timestamp
if entry, ok := s.store.EntryForKey(f.Key()); ok {
if entry.IsBaseline && entry.DismissalID != "" && entry.Hash == f.Fingerprint() {
continue
}
firstSeen = entry.FirstSeen
lastSeen = entry.LastSeen
}
key := f.Key()
views = append(views, perfFindingView{
Severity: f.Severity.String(),
level: f.Severity,
Check: f.Check,
Message: f.Message,
Details: f.Details,
Key: key,
FirstSeen: firstSeen.UTC(),
LastSeen: lastSeen.UTC(),
})
}
// Sort by severity descending
sort.Slice(views, func(i, j int) bool {
return views[i].level > views[j].level
})
if len(views) > limit {
views = views[:limit]
}
writeJSON(w, perfResponse{
Metrics: metrics,
Findings: views,
})
}
// apiPerfFixErrorLog truncates an account-owned error_log identified by
// the perf_error_logs finding. Admin scope; CSRF enforced at the route.
func (s *Server) apiPerfFixErrorLog(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
Path string `json:"path"`
Key string `json:"key"`
}
if err := decodeJSONBodyLimited(w, r, 1<<14, &req); err != nil {
writeJSONError(w, "invalid request body", http.StatusBadRequest)
return
}
if req.Path == "" {
writeJSONError(w, "path is required", http.StatusBadRequest)
return
}
resolve := s.preparePerfResolution(req.Key, "perf_error_logs", req.Path)
res := checks.FixErrorLogBloatInRoots(req.Path, checks.ResolveValidatedDocroots(s.liveCfg()))
if !res.Success {
writeRemediation(w, res)
return
}
resolve()
s.auditLog(r, "perf_fix_error_log", req.Path, res.Description)
writeRemediation(w, res)
}
// apiPerfFixDisplayErrors disables display_errors in an account-owned
// .user.ini / php.ini / .htaccess identified by the perf_wp_config
// finding's Details field. Admin scope; CSRF enforced at the route.
//
//nolint:dupl // mirrors apiPerfFixErrorLog; separate handlers keep audit actions explicit.
func (s *Server) apiPerfFixDisplayErrors(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
Path string `json:"path"`
Key string `json:"key"`
}
if err := decodeJSONBodyLimited(w, r, 1<<14, &req); err != nil {
writeJSONError(w, "invalid request body", http.StatusBadRequest)
return
}
if req.Path == "" {
writeJSONError(w, "path is required", http.StatusBadRequest)
return
}
resolve := s.preparePerfResolution(req.Key, "perf_wp_config", req.Path)
res := checks.FixDisplayErrorsOnInRoots(req.Path, s.perfFixAllowedRoots())
if !res.Success {
writeRemediation(w, res)
return
}
resolve()
s.auditLog(r, "perf_fix_display_errors", req.Path, res.Description)
writeRemediation(w, res)
}
// apiPerfFixWPCron disables WP-Cron in an account-owned wp-config.php
// identified by a perf_wp_cron finding and installs a per-user system cron
// that runs wp-cron.php. Admin scope; CSRF enforced at the route.
func (s *Server) apiPerfFixWPCron(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
Path string `json:"path"`
Key string `json:"key"`
}
if err := decodeJSONBodyLimited(w, r, 1<<14, &req); err != nil {
writeJSONError(w, "invalid request body", http.StatusBadRequest)
return
}
if req.Path == "" {
writeJSONError(w, "path is required", http.StatusBadRequest)
return
}
cfg := s.liveCfg()
options := checks.WPCronFixOptions{}
if cfg != nil {
options.IntervalMinutes = cfg.Performance.WPCronFix.IntervalMinutes
options.PHPBin = cfg.Performance.WPCronFix.PHPBin
}
resolve := s.preparePerfResolution(req.Key, "perf_wp_cron", req.Path)
res := checks.FixDisableWPCronInRoots(req.Path, checks.ResolveValidatedDocroots(cfg), options)
if !res.Success {
writeRemediation(w, res)
return
}
resolve()
s.auditLog(r, "perf_fix_wp_cron", req.Path, res.Description)
writeRemediation(w, res)
}
func (s *Server) perfFixAllowedRoots() []string {
cfg := s.liveCfg()
if cfg == nil {
return []string{"/home"}
}
return checks.ResolveWebRoots(cfg)
}
func (s *Server) preparePerfResolution(key, check, path string) func() bool {
key = strings.TrimSpace(key)
for _, f := range s.store.LatestFindings() {
if f.Key() != key || f.Check != check {
continue
}
// These producers carry the target in text, not FilePath. Match the
// complete target field so fixing one file cannot resolve another.
matches := false
switch check {
case "perf_error_logs":
matches = f.Message == "Bloated error_log: "+path
case "perf_wp_config":
matches = strings.HasPrefix(f.Message, "display_errors enabled in production for ") && f.Details == "File: "+path+", Value: On"
case "perf_wp_cron":
matches = f.Details == "File: "+path+" - add define('DISABLE_WP_CRON', true); and use a real cron job instead"
}
if matches {
return s.store.PrepareFindingResolution(f)
}
}
return func() bool { return false }
}
// handlePerformance renders the performance dashboard page.
func (s *Server) handlePerformance(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "performance.html", nil)
}
package webui
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"net/http"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/store"
)
var errNoStore = errors.New("store unavailable")
func nowUnix() int64 { return time.Now().UTC().Unix() }
// operatorKey returns a SHA-256 hex of the auth credential carried by r. It
// is used as the per-operator partition key for the preferences store. The
// store never sees the raw token; only its hash.
//
// Returns "" when no admin credential is present (handlers below run after
// requireAuth, so this normally only happens in tests that bypass middleware).
func (s *Server) operatorKey(r *http.Request) string {
if bearer, ok := s.bearerTokenWithScope(r, "admin"); ok {
return hashOperatorToken(bearer)
}
if cookie, ok := s.cookieTokenWithScope(r, "admin"); ok {
return hashOperatorToken(cookie)
}
return ""
}
func hashOperatorToken(raw string) string {
sum := sha256.Sum256([]byte(raw))
return hex.EncodeToString(sum[:])
}
const (
prefsNamespaceUser = "user"
prefsNamespaceViews = "views"
)
// userPrefsBlob is the JSON document the client posts to /api/v1/prefs/user.
// Fields are validated and clipped to a small enum where appropriate so an
// attacker cannot smuggle arbitrary template data into the layout.
type userPrefsBlob struct {
Density string `json:"density"`
Timezone string `json:"timezone"`
AutoRefresh string `json:"auto_refresh"`
TableColumns map[string][]string `json:"table_columns,omitempty"`
}
func sanitizeUserPrefs(in userPrefsBlob) userPrefsBlob {
out := userPrefsBlob{}
switch in.Density {
case "compact", "comfortable":
out.Density = in.Density
}
switch in.Timezone {
case "server", "local":
out.Timezone = in.Timezone
default:
// Allow IANA-shaped strings (e.g. "Europe/Bucharest"). Reject anything
// containing whitespace or control characters; the value gets reflected
// in JS where Intl.DateTimeFormat will reject malformed zones anyway.
if isIANAish(in.Timezone) {
out.Timezone = in.Timezone
}
}
switch in.AutoRefresh {
case "on", "off":
out.AutoRefresh = in.AutoRefresh
}
if len(in.TableColumns) > 0 {
cleaned := make(map[string][]string, len(in.TableColumns))
for k, vals := range in.TableColumns {
if !isSimpleIdent(k) || len(vals) > 64 {
continue
}
var v []string
for _, name := range vals {
if isSimpleIdent(name) {
v = append(v, name)
}
}
cleaned[k] = v
}
if len(cleaned) > 0 {
out.TableColumns = cleaned
}
}
return out
}
func isIANAish(s string) bool {
if s == "" || len(s) > 64 {
return false
}
for _, r := range s {
switch {
case r >= 'A' && r <= 'Z':
case r >= 'a' && r <= 'z':
case r >= '0' && r <= '9':
case r == '_' || r == '+' || r == '-' || r == '/':
default:
return false
}
}
return true
}
func isSimpleIdent(s string) bool {
if s == "" || len(s) > 64 {
return false
}
for _, r := range s {
switch {
case r >= 'A' && r <= 'Z':
case r >= 'a' && r <= 'z':
case r >= '0' && r <= '9':
case r == '_' || r == '-' || r == '.':
default:
return false
}
}
return true
}
// apiPrefsUser handles GET and PUT for the operator's user-pref blob.
func (s *Server) apiPrefsUser(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
s.handleGetUserPrefs(w, r)
case http.MethodPut:
s.handlePutUserPrefs(w, r)
default:
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
}
}
func (s *Server) handleGetUserPrefs(w http.ResponseWriter, r *http.Request) {
opkey := s.operatorKey(r)
if opkey == "" {
writeJSONError(w, "Unauthenticated", http.StatusUnauthorized)
return
}
sdb := store.Global()
if sdb == nil {
writeJSON(w, userPrefsBlob{})
return
}
raw, err := sdb.GetOperatorPref(opkey, prefsNamespaceUser)
if err != nil {
writeJSONError(w, "Store error", http.StatusInternalServerError)
return
}
if raw == nil {
writeJSON(w, userPrefsBlob{})
return
}
var blob userPrefsBlob
if err := json.Unmarshal(raw, &blob); err != nil {
writeJSON(w, userPrefsBlob{})
return
}
writeJSON(w, sanitizeUserPrefs(blob))
}
func (s *Server) handlePutUserPrefs(w http.ResponseWriter, r *http.Request) {
opkey := s.operatorKey(r)
if opkey == "" {
writeJSONError(w, "Unauthenticated", http.StatusUnauthorized)
return
}
var blob userPrefsBlob
if err := decodeJSONBodyLimited(w, r, store.MaxPrefBlobSize, &blob); err != nil {
writeJSONError(w, "Invalid request body", http.StatusBadRequest)
return
}
clean := sanitizeUserPrefs(blob)
raw, err := json.Marshal(clean)
if err != nil {
writeJSONError(w, "Encoding failed", http.StatusInternalServerError)
return
}
sdb := store.Global()
if sdb == nil {
writeJSONError(w, "Store unavailable", http.StatusServiceUnavailable)
return
}
if err := sdb.PutOperatorPref(opkey, prefsNamespaceUser, raw); err != nil {
writeJSONError(w, "Store error", http.StatusInternalServerError)
return
}
// The stored preferences, with the action's ok flag next to them.
stored := map[string]interface{}{
"density": clean.Density,
"timezone": clean.Timezone,
"auto_refresh": clean.AutoRefresh,
}
if len(clean.TableColumns) > 0 {
stored["table_columns"] = clean.TableColumns
}
writeOK(w, stored)
}
// savedView represents one user-named filter combination for a page.
type savedView struct {
Name string `json:"name"`
Page string `json:"page"`
Params map[string]string `json:"params"`
Updated int64 `json:"updated"`
}
// savedViewResponse is a saved view as the API sends it. The store keeps
// updated as Unix seconds; the API sends it as an instant like every time.
type savedViewResponse struct {
Name string `json:"name"`
Page string `json:"page"`
Params map[string]string `json:"params"`
Updated time.Time `json:"updated,omitzero"`
}
const maxSavedViewsPerOperator = 200
// apiPrefsViews handles list (GET), upsert (PUT), and delete (DELETE) of
// saved filter views.
func (s *Server) apiPrefsViews(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
s.handleListSavedViews(w, r)
case http.MethodPut:
s.handlePutSavedView(w, r)
case http.MethodDelete:
s.handleDeleteSavedView(w, r)
default:
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
}
}
func (s *Server) loadSavedViews(opkey string) []savedView {
sdb := store.Global()
if sdb == nil {
return nil
}
raw, err := sdb.GetOperatorPref(opkey, prefsNamespaceViews)
if err != nil || raw == nil {
return nil
}
var views []savedView
if err := json.Unmarshal(raw, &views); err != nil {
return nil
}
return views
}
func (s *Server) saveSavedViews(opkey string, views []savedView) error {
sort.SliceStable(views, func(i, j int) bool {
if views[i].Page != views[j].Page {
return views[i].Page < views[j].Page
}
return views[i].Name < views[j].Name
})
raw, err := json.Marshal(views)
if err != nil {
return err
}
sdb := store.Global()
if sdb == nil {
return errNoStore
}
return sdb.PutOperatorPref(opkey, prefsNamespaceViews, raw)
}
func (s *Server) handleListSavedViews(w http.ResponseWriter, r *http.Request) {
opkey := s.operatorKey(r)
if opkey == "" {
writeJSONError(w, "Unauthenticated", http.StatusUnauthorized)
return
}
page := strings.TrimSpace(r.URL.Query().Get("page"))
views := s.loadSavedViews(opkey)
out := make([]savedViewResponse, 0, len(views))
for _, v := range views {
if page != "" && v.Page != page {
continue
}
view := savedViewResponse{Name: v.Name, Page: v.Page, Params: v.Params}
if v.Updated > 0 {
view.Updated = time.Unix(v.Updated, 0).UTC()
}
out = append(out, view)
}
writeAll(w, out)
}
func (s *Server) handlePutSavedView(w http.ResponseWriter, r *http.Request) {
opkey := s.operatorKey(r)
if opkey == "" {
writeJSONError(w, "Unauthenticated", http.StatusUnauthorized)
return
}
var body struct {
Name string `json:"name"`
Page string `json:"page"`
Params map[string]string `json:"params"`
}
if err := decodeJSONBodyLimited(w, r, store.MaxPrefBlobSize, &body); err != nil {
writeJSONError(w, "Invalid request body", http.StatusBadRequest)
return
}
body.Name = strings.TrimSpace(body.Name)
body.Page = strings.TrimSpace(body.Page)
if !isSimpleIdent(body.Page) {
writeJSONError(w, "Invalid page", http.StatusBadRequest)
return
}
if body.Name == "" || len(body.Name) > 80 {
writeJSONError(w, "Name must be 1-80 characters", http.StatusBadRequest)
return
}
if !isPrintableLabel(body.Name) {
writeJSONError(w, "Name contains invalid characters", http.StatusBadRequest)
return
}
if len(body.Params) > 32 {
writeJSONError(w, "Too many params", http.StatusBadRequest)
return
}
cleanParams := make(map[string]string, len(body.Params))
for k, v := range body.Params {
if !isSimpleIdent(k) || len(v) > 256 {
writeJSONError(w, "Invalid param", http.StatusBadRequest)
return
}
cleanParams[k] = v
}
views := s.loadSavedViews(opkey)
now := nowUnix()
updated := false
for i := range views {
if views[i].Page == body.Page && views[i].Name == body.Name {
views[i].Params = cleanParams
views[i].Updated = now
updated = true
break
}
}
if !updated {
if len(views) >= maxSavedViewsPerOperator {
writeJSONError(w, "Saved view limit reached", http.StatusBadRequest)
return
}
views = append(views, savedView{
Name: body.Name,
Page: body.Page,
Params: cleanParams,
Updated: now,
})
}
if err := s.saveSavedViews(opkey, views); err != nil {
writeJSONError(w, "Store error", http.StatusInternalServerError)
return
}
writeOK(w, nil)
}
func (s *Server) handleDeleteSavedView(w http.ResponseWriter, r *http.Request) {
opkey := s.operatorKey(r)
if opkey == "" {
writeJSONError(w, "Unauthenticated", http.StatusUnauthorized)
return
}
var body struct {
Name string `json:"name"`
Page string `json:"page"`
}
if err := decodeJSONBodyLimited(w, r, 8*1024, &body); err != nil {
writeJSONError(w, "Invalid request body", http.StatusBadRequest)
return
}
views := s.loadSavedViews(opkey)
out := views[:0]
removed := false
for _, v := range views {
if v.Page == body.Page && v.Name == body.Name {
removed = true
continue
}
out = append(out, v)
}
if !removed {
writeJSONError(w, "View not found", http.StatusNotFound)
return
}
if err := s.saveSavedViews(opkey, out); err != nil {
writeJSONError(w, "Store error", http.StatusInternalServerError)
return
}
writeOK(w, nil)
}
func isPrintableLabel(s string) bool {
for _, r := range s {
if r < 0x20 || r == 0x7f {
return false
}
}
return true
}
package webui
import (
"fmt"
"net/http"
"github.com/pidginhost/csm/internal/mailfwd/intel"
"github.com/pidginhost/csm/internal/platform"
)
// selectQueueReporter picks the queue-composition source for the host. Only
// cPanel/exim is wired; other platforms get the empty reporter.
func selectQueueReporter() intel.QueueReporter {
if cpanelMailBackend(platform.Detect()) {
return intel.NewEximQueueSource()
}
return intel.EmptyQueueReporter{}
}
// selectQueueFlusher picks the backscatter-flush executor for the host. Only
// cPanel/exim is wired; other platforms expose the route as unavailable.
func selectQueueFlusher() intel.QueueFlusher {
if cpanelMailBackend(platform.Detect()) {
return intel.NewEximQueueFlusher()
}
return nil
}
// apiEmailFlushBackscatter handles POST /api/v1/email/queue/flush-backscatter.
// It removes only frozen null-sender messages -- undeliverable bounce
// backscatter -- from the exim queue. Mutating, so it runs under auth + CSRF
// and is audit-logged.
func (s *Server) apiEmailFlushBackscatter(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if s.queueFlusher == nil {
writeJSONError(w, "Mail queue flush not available on this host", http.StatusServiceUnavailable)
return
}
res, err := s.queueFlusher.FlushBackscatter()
// The attempt must be recorded even when the verification read fails and
// cannot confirm a removal. A target can also leave through concurrent
// delivery, so the audit records observed queue state without attributing
// every disappearance to the removal command.
targeted := res.Targeted
if targeted > 0 {
s.auditLog(r, "email_flush_backscatter", "mail-queue",
fmt.Sprintf("removal requested for %d frozen null-sender message(s); %d confirmed no longer queued afterward",
targeted, res.Removed))
}
if err != nil {
message := fmt.Sprintf("Failed to flush backscatter: %v", err)
if targeted > 0 {
message = fmt.Sprintf("Flush incomplete: %d of %d targeted message(s) confirmed no longer queued: %v",
res.Removed, targeted, err)
}
writeJSONStatus(w, http.StatusInternalServerError, map[string]interface{}{"error": message, "removed": res.Removed})
return
}
writeOK(w, map[string]interface{}{"removed": res.Removed})
}
// apiEmailQueueComposition handles GET /api/v1/email/queue-composition and
// returns the makeup of the exim queue: real mail vs null-sender bounce
// backscatter, frozen count, oldest age, and the most-stuck recipients.
func (s *Server) apiEmailQueueComposition(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if s.queueReporter == nil {
writeJSON(w, intel.QueueComposition{TopRecipients: []intel.RecipientCount{}})
return
}
comp, err := s.queueReporter.Composition()
if err != nil {
writeJSONError(w, "Failed to read mail queue", http.StatusInternalServerError)
return
}
writeJSON(w, comp)
}
package webui
import (
"sort"
"sync/atomic"
"time"
)
// apiRateLimitMaxIPs caps how many source addresses the unauthenticated
// per-IP rate-limit maps hold. The five-minute prune bounded them over time
// but not between prunes: a scan from many addresses grew them without
// limit in the meantime.
const apiRateLimitMaxIPs = 10000
// rateLimitSweeps counts full passes over a rate-limit map; tests read it.
var rateLimitSweeps atomic.Int64
// boundRateLimitMap keeps m under apiRateLimitMaxIPs before a new address
// is inserted. Caller holds the map's mutex. A full map is swept once down
// to nine tenths of the ceiling: stale entries (no hit after cutoff) go
// first, then the addresses with the oldest last hit. Sweeping one address
// at a time scanned the whole map for every new address, so a flood of new
// sources cost a full scan per request under the lock.
func boundRateLimitMap(m map[string][]time.Time, cutoff time.Time) {
if len(m) < apiRateLimitMaxIPs {
return
}
rateLimitSweeps.Add(1)
type lastHit struct {
ip string
last time.Time
}
live := make([]lastHit, 0, len(m))
for ip, hits := range m {
if len(hits) == 0 || !hits[len(hits)-1].After(cutoff) {
delete(m, ip)
continue
}
live = append(live, lastHit{ip, hits[len(hits)-1]})
}
keep := apiRateLimitMaxIPs - apiRateLimitMaxIPs/10
if len(live) <= keep {
return
}
sort.Slice(live, func(i, j int) bool { return live[i].last.Before(live[j].last) })
for _, h := range live[:len(live)-keep] {
delete(m, h.ip)
}
}
package webui
import (
"bytes"
"context"
"errors"
"net/http"
"net/url"
"sort"
"strings"
)
// ReadOnlyRoute is one GET route the read-only dispatcher can reach. The
// table is the whole surface: a path that is not listed cannot be served,
// whatever the caller sends.
type ReadOnlyRoute struct {
// Pattern is the http.ServeMux pattern, with {id} where the public
// mux takes a path value or a prefix.
Pattern string
handler http.HandlerFunc
}
// readOnlyRoutes lists every API read the MCP endpoint may dispatch. Keep
// it to GET handlers that answer JSON; TestReadOnlyRoutesAreRegisteredReads
// checks each one against the public route registrations.
func (s *Server) readOnlyRoutes() []ReadOnlyRoute {
return []ReadOnlyRoute{
{"/api/v1/status", s.apiStatus},
{"/api/v1/health", s.apiHealth},
{"/api/v1/capabilities", s.apiCapabilities},
{"/api/v1/components", s.apiComponents},
{"/api/v1/challenge/stats", s.apiChallengeStats},
{"/api/v1/scan-jobs", s.apiScanJobsList},
{"/api/v1/scan-jobs/{id}", s.apiScanJobsRouter},
{"/api/v1/findings", s.apiFindings},
{"/api/v1/findings/enriched", s.apiFindingsEnriched},
{"/api/v1/finding-detail", s.apiFindingDetail},
{"/api/v1/history", s.apiHistory},
{"/api/v1/stats", s.apiStats},
{"/api/v1/stats/trend", s.apiStatsTrend},
{"/api/v1/stats/timeline", s.apiStatsTimeline},
{"/api/v1/blocked-ips", s.apiBlockedIPs},
{"/api/v1/quarantine", s.apiQuarantine},
{"/api/v1/db-object-backups", s.apiDBObjectBackups},
{"/api/v1/accounts", s.apiAccounts},
{"/api/v1/account", s.apiAccountDetail},
{"/api/v1/audit", s.apiUIAudit},
{"/api/v1/incident", s.apiIncident},
{"/api/v1/incidents", s.apiIncidentList},
{"/api/v1/incidents/groups", s.apiIncidentGroups},
{"/api/v1/incidents/{id}", s.apiIncidentShow},
{"/api/v1/email/stats", s.apiEmailStats},
{"/api/v1/email/quarantine", s.apiEmailQuarantineList},
{"/api/v1/email/av/status", s.apiEmailAVStatus},
{"/api/v1/email/checks", s.apiEmailChecks},
{"/api/v1/email/groups", s.apiEmailGroups},
{"/api/v1/email/overview", s.apiEmailOverview},
{"/api/v1/email/rejects", s.apiEmailRejects},
{"/api/v1/email/senders", s.apiEmailSenders},
{"/api/v1/email/rspamd", s.apiEmailRspamd},
{"/api/v1/email/relay-abuse", s.apiEmailRelayAbuse},
{"/api/v1/email/forwarders", s.apiEmailForwarders},
{"/api/v1/email/deferrals", s.apiEmailDeferrals},
{"/api/v1/email/queue-composition", s.apiEmailQueueComposition},
{"/api/v1/email/queue", s.apiEmailQueueList},
{"/api/v1/email/held", s.apiEmailHeldList},
{"/api/v1/performance", s.apiPerformance},
{"/api/v1/hardening", s.apiHardening},
{"/api/v1/threat/stats", s.apiThreatStats},
{"/api/v1/threat/top-attackers", s.apiThreatTopAttackers},
{"/api/v1/threat/ip", s.apiThreatIP},
{"/api/v1/threat/events", s.apiThreatEvents},
{"/api/v1/threat/db-stats", s.apiThreatDBStats},
{"/api/v1/threat/whitelist", s.apiThreatWhitelist},
{"/api/v1/rules/status", s.apiRulesStatus},
{"/api/v1/rules/list", s.apiRulesList},
{"/api/v1/modsec/stats", s.apiModSecStats},
{"/api/v1/modsec/blocks", s.apiModSecBlocks},
{"/api/v1/modsec/events", s.apiModSecEvents},
{"/api/v1/modsec/rules", s.apiModSecRules},
{"/api/v1/firewall/status", s.apiFirewallStatus},
{"/api/v1/firewall/allowed", s.apiFirewallAllowed},
{"/api/v1/firewall/audit", s.apiFirewallAudit},
{"/api/v1/firewall/subnets", s.apiFirewallSubnets},
{"/api/v1/firewall/check", s.apiFirewallCheck},
{"/api/v1/settings/firewall/rollback", s.apiFirewallRollbackStatus},
{"/api/v1/geoip", s.apiGeoIPLookup},
}
}
// ReadOnlyDispatcher serves the read-only route table in-process. It holds
// a private ServeMux with nothing but those routes, each behind getOnly,
// and builds every request itself with the GET method, so no caller input
// can select a method or a route outside the table.
type ReadOnlyDispatcher struct {
routes *http.ServeMux
patterns []string
maxBody int
}
// ReadOnlyMaxBody caps one dispatched response. Larger answers are cut and
// reported as truncated; callers narrow the query with the route's own
// limit and filter parameters.
const ReadOnlyMaxBody = 4 << 20
// ErrReadOnlyRoute is returned for a path outside the read-only table.
var ErrReadOnlyRoute = errors.New("not a read-only route")
// ReadOnly returns the dispatcher for this server. It is safe to call once
// the server is constructed; the handlers read the same live wiring the
// HTTPS listener uses.
func (s *Server) ReadOnly() *ReadOnlyDispatcher {
d := &ReadOnlyDispatcher{routes: http.NewServeMux(), maxBody: ReadOnlyMaxBody}
for _, route := range s.readOnlyRoutes() {
d.routes.Handle(route.Pattern, getOnly(route.handler))
d.patterns = append(d.patterns, route.Pattern)
}
sort.Strings(d.patterns)
return d
}
// Routes lists the patterns the dispatcher serves, sorted.
func (d *ReadOnlyDispatcher) Routes() []string {
return append([]string(nil), d.patterns...)
}
// Serves reports whether path (without a query) is inside the table. A
// pattern with {id} matches one non-empty path segment in its place.
func (d *ReadOnlyDispatcher) Serves(path string) bool {
if path == "" || strings.Contains(path, "?") || strings.Contains(path, "..") {
return false
}
for _, pattern := range d.patterns {
if readOnlyPatternMatches(pattern, path) {
return true
}
}
return false
}
func readOnlyPatternMatches(pattern, path string) bool {
prefix, _, wild := strings.Cut(pattern, "{id}")
if !wild {
return pattern == path
}
rest, ok := strings.CutPrefix(path, prefix)
return ok && rest != "" && !strings.Contains(rest, "/")
}
// Dispatch serves GET path?query to the matching read-only handler and
// returns the status, content type and body it wrote. peer is recorded as
// the request's RemoteAddr for handlers and logs that name the caller. A
// path outside the table answers ErrReadOnlyRoute without touching any
// handler.
func (d *ReadOnlyDispatcher) Dispatch(ctx context.Context, path string, query url.Values, peer string) (int, string, []byte, error) {
if !d.Serves(path) {
return 0, "", nil, ErrReadOnlyRoute
}
target := &url.URL{Path: path, RawQuery: query.Encode()}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil)
if err != nil {
return 0, "", nil, err
}
req.RemoteAddr = peer
req.Header.Set("Accept", "application/json")
rec := &readOnlyRecorder{header: http.Header{}, status: http.StatusOK, limit: d.maxBody}
d.routes.ServeHTTP(rec, req)
body := rec.body.Bytes()
if rec.truncated {
return rec.status, rec.header.Get("Content-Type"), body, errReadOnlyTruncated{}
}
return rec.status, rec.header.Get("Content-Type"), body, nil
}
// errReadOnlyTruncated reports a response cut at ReadOnlyMaxBody. Callers
// in other packages recognise it through its Truncated method.
type errReadOnlyTruncated struct{}
func (errReadOnlyTruncated) Error() string {
return "response larger than the read-only dispatch limit"
}
// Truncated marks the error as a size cut rather than a handler failure.
func (errReadOnlyTruncated) Truncated() bool { return true }
// readOnlyRecorder captures a handler's answer in memory, up to limit.
type readOnlyRecorder struct {
header http.Header
status int
body bytes.Buffer
limit int
truncated bool
}
func (r *readOnlyRecorder) Header() http.Header { return r.header }
func (r *readOnlyRecorder) WriteHeader(status int) { r.status = status }
func (r *readOnlyRecorder) Write(p []byte) (int, error) {
room := r.limit - r.body.Len()
if room <= 0 {
r.truncated = true
return len(p), nil
}
if len(p) > room {
r.body.Write(p[:room])
r.truncated = true
return len(p), nil
}
return r.body.Write(p)
}
// Flush satisfies handlers that stream; the recorder is already in memory.
func (r *readOnlyRecorder) Flush() {}
package webui
import (
"net/http"
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
)
type relayAbuseResponse struct {
Entries []relayAbuseEntry `json:"items"`
Total int `json:"total"`
Offset int `json:"offset"`
Limit int `json:"limit"`
From time.Time `json:"from"`
To time.Time `json:"to"`
Truncated bool `json:"truncated"`
// TotalTruncated says the scan cap cut Total, so the range can hold
// more events. Truncated is also set when only limit cut the list.
TotalTruncated bool `json:"total_truncated"`
// byAccount counts the events of the range that name an account, by
// account and kind, before limit cuts the list.
byAccount map[string]map[string]int
}
type relayAbuseEntry struct {
Path string `json:"path"`
PathLabel string `json:"path_label"`
Severity string `json:"severity"`
SourceIP string `json:"source_ip,omitempty"`
CPUser string `json:"cp_user,omitempty"`
TriggerCount int `json:"trigger_count"`
DetectedAt time.Time `json:"detected_at"`
Sites []relaySiteEntry `json:"sites"`
MsgSample []string `json:"msg_sample,omitempty"`
}
type relaySiteEntry struct {
Site string `json:"site"`
Script string `json:"script"`
Hits int `json:"hits"`
LastSeen time.Time `json:"last_seen"`
SampleSubject string `json:"sample_subject,omitempty"`
}
const relayAbuseDefaultLimit = 20
const relayAbuseMaxLimit = 100
// relayAbuseMemo names the history walk the relay-abuse list and the senders
// abuse count share. The limit only cuts the answer, so it is not part of the
// key.
const relayAbuseMemo = "relay-abuse"
func relayPathLabel(path string) string {
switch path {
case "fanout":
return "Spam outbreak (IP fanout)"
case "volume":
return "High volume script"
case "header":
return "Suspicious headers"
case "volume_account":
return "High volume account"
case "":
return "Unknown path"
default:
return path
}
}
// apiEmailRelayAbuse handles GET /api/v1/email/relay-abuse. Read-only.
// Reads email_php_relay_abuse findings from persisted history (the realtime
// dispatch path does not populate LatestFindings).
func (s *Server) apiEmailRelayAbuse(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
q := r.URL.Query()
limit := queryInt(r, "limit", relayAbuseDefaultLimit)
if limit <= 0 || limit > relayAbuseMaxLimit {
limit = relayAbuseDefaultLimit
}
from, to, ok := emailRange(w, q, time.Now())
if !ok {
return
}
writeJSON(w, s.buildRelayAbuseResponse(from, to, limit))
}
func (s *Server) buildRelayAbuseResponse(from, to time.Time, limit int) relayAbuseResponse {
resp := relayAbuseResponse{
Entries: []relayAbuseEntry{},
Limit: limit,
From: from.UTC(),
To: to.UTC(),
byAccount: map[string]map[string]int{},
}
if s.store == nil {
return resp
}
key := rangeMemoKey(relayAbuseMemo, from, to)
snapshot := s.emailMemo(key, func() any {
return s.relayAbuseHistory(from.Truncate(time.Minute), to.Truncate(time.Minute).Add(time.Minute))
}).([]relayAbuseEntry)
if len(snapshot) > emailGroupsScanCap {
// A partial minute-wide snapshot may omit matches behind newer
// findings outside the exact range. Apply its bounds during the
// history walk instead, before spending the match budget.
exactKey := key + "&exact-from=" + from.UTC().Format(time.RFC3339Nano) + "&exact-to=" + to.UTC().Format(time.RFC3339Nano)
snapshot = s.emailMemo(exactKey, func() any {
return s.relayAbuseHistory(from, to)
}).([]relayAbuseEntry)
}
// Memoized rows are shared by concurrent requests. Filter into a new
// slice before sorting, retaining each request's precise range.
rows := make([]relayAbuseEntry, 0, len(snapshot))
for _, row := range snapshot {
if !row.DetectedAt.Before(from) && row.DetectedAt.Before(to) {
rows = append(rows, row)
}
}
if len(rows) > emailGroupsScanCap {
rows = rows[:emailGroupsScanCap]
resp.Truncated, resp.TotalTruncated = true, true
}
resp.Total = len(rows)
for _, row := range rows {
if row.CPUser == "" {
continue
}
kinds := resp.byAccount[row.CPUser]
if kinds == nil {
kinds = map[string]int{}
resp.byAccount[row.CPUser] = kinds
}
kinds[row.PathLabel]++
}
sort.SliceStable(rows, func(i, j int) bool {
if !rows[i].DetectedAt.Equal(rows[j].DetectedAt) {
return rows[i].DetectedAt.After(rows[j].DetectedAt)
}
if rows[i].Path != rows[j].Path {
return rows[i].Path < rows[j].Path
}
if rows[i].SourceIP != rows[j].SourceIP {
return rows[i].SourceIP < rows[j].SourceIP
}
return rows[i].CPUser < rows[j].CPUser
})
if len(rows) > limit {
resp.Truncated = true
rows = rows[:limit]
}
resp.Entries = rows
return resp
}
func (s *Server) relayAbuseHistory(from, to time.Time) []relayAbuseEntry {
findings := s.store.SearchHistorySince(from, emailGroupsScanCap+1, func(f alert.Finding) bool {
return f.Check == "email_php_relay_abuse" && f.Timestamp.Before(to)
})
rows := make([]relayAbuseEntry, 0, len(findings))
for _, finding := range findings {
rows = append(rows, toRelayAbuseEntry(finding))
}
return rows
}
func toRelayAbuseEntry(f alert.Finding) relayAbuseEntry {
e := relayAbuseEntry{
Path: f.Path,
PathLabel: relayPathLabel(f.Path),
Severity: f.Severity.String(),
SourceIP: f.SourceIP,
CPUser: f.CPUser,
TriggerCount: relayTriggerCount(f),
DetectedAt: f.Timestamp,
Sites: []relaySiteEntry{},
MsgSample: f.MsgIDs,
}
for _, h := range f.RelayBreakdown {
site, script := splitScriptKey(h.ScriptKey)
e.Sites = append(e.Sites, relaySiteEntry{
Site: site,
Script: script,
Hits: h.Hits,
LastSeen: h.LastSeen,
SampleSubject: h.SampleSubject,
})
}
return e
}
func relayTriggerCount(f alert.Finding) int {
if f.RelayTotal > 0 {
return f.RelayTotal
}
sum := 0
for _, h := range f.RelayBreakdown {
if h.Hits > 0 {
sum += h.Hits
}
}
if sum > 0 {
return sum
}
return len(f.MsgIDs)
}
// splitScriptKey splits a "host:/path" script key into host and path. The key
// is built as host + ":" + path with path starting at "/", so the delimiter is
// the ":/" boundary. Splitting there (not the first colon) keeps host:port and
// IPv6-literal hosts intact. A key without ":/" yields ("", key) so the row
// still renders.
func splitScriptKey(k string) (site, script string) {
if i := strings.Index(k, ":/"); i >= 0 {
return k[:i], k[i+1:]
}
return "", k
}
package webui
import (
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/pidginhost/csm/internal/signatures"
"github.com/pidginhost/csm/internal/yara"
)
func (s *Server) handleRules(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "rules.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
// GET /api/v1/rules/status
func (s *Server) apiRulesStatus(w http.ResponseWriter, _ *http.Request) {
cfg := s.liveCfg()
yamlCount := 0
yamlVersion := 0
if scanner := signatures.Global(); scanner != nil {
yamlCount = scanner.RuleCount()
yamlVersion = scanner.Version()
}
yaraCount := 0
if b := yara.Active(); b != nil {
yaraCount = b.RuleCount()
}
result := map[string]interface{}{
"yaml_rules": yamlCount,
"yara_rules": yaraCount,
"yara_available": yara.Available(),
"yaml_version": yamlVersion,
"rules_dir": cfg.Signatures.RulesDir,
"auto_update": cfg.Signatures.UpdateURL != "",
"update_url": cfg.Signatures.UpdateURL,
}
if secs, ok := durationSeconds(cfg.Signatures.UpdateInterval); ok {
result["update_interval_seconds"] = secs
}
writeJSON(w, result)
}
// GET /api/v1/rules/list
func (s *Server) apiRulesList(w http.ResponseWriter, _ *http.Request) {
rulesDir := s.liveCfg().Signatures.RulesDir
type ruleFileInfo struct {
Name string `json:"name"`
Type string `json:"type"` // "yaml" or "yara"
Size int64 `json:"size"`
}
var files []ruleFileInfo
entries, err := os.ReadDir(rulesDir)
if err != nil {
if os.IsNotExist(err) {
writeAll(w, files)
return
}
writeJSONError(w, fmt.Sprintf("reading rules directory: %v", err), http.StatusInternalServerError)
return
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
ext := strings.ToLower(filepath.Ext(name))
var fileType string
switch ext {
case ".yml", ".yaml":
fileType = "yaml"
case ".yar", ".yara":
fileType = "yara"
default:
continue // skip non-rule files
}
info, err := entry.Info()
if err != nil {
continue
}
files = append(files, ruleFileInfo{
Name: name,
Type: fileType,
Size: info.Size(),
})
}
writeAll(w, files)
}
// POST /api/v1/rules/reload
func (s *Server) apiRulesReload(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var yamlErr, yaraErr error
yamlCount := 0
yaraCount := 0
if scanner := signatures.Global(); scanner != nil {
yamlErr = scanner.Reload()
yamlCount = scanner.RuleCount()
// Update the cached sig count shown in dashboard/status
s.SetSigCount(yamlCount)
}
if b := yara.Active(); b != nil {
yaraErr = b.Reload()
yaraCount = b.RuleCount()
}
var errors []string
if yamlErr != nil {
errors = append(errors, fmt.Sprintf("YAML reload: %v", yamlErr))
}
if yaraErr != nil {
errors = append(errors, fmt.Sprintf("YARA reload: %v", yaraErr))
}
s.auditLog(r, "rules_reload", "signatures", fmt.Sprintf("errors: %d", len(errors)))
result := map[string]interface{}{
"yaml_rules": yamlCount,
"yara_rules": yaraCount,
}
if len(errors) > 0 {
result["error"] = strings.Join(errors, "; ")
result["errors"] = errors
writeJSONStatus(w, http.StatusInternalServerError, result)
return
}
writeOK(w, result)
}
package webui
import (
"fmt"
"net/http"
"strings"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/control"
"github.com/pidginhost/csm/internal/store"
)
// ScanJobController is the subset of the daemon's ScanJobManager the WebUI
// needs to enqueue and cancel full-scan jobs. The daemon injects a concrete
// *ScanJobManager via SetScanJobs; nil until wired (POST endpoints → 503).
type ScanJobController interface {
Enqueue(scope, target string, opts checks.AccountScanOptions, quarantine bool) (string, error)
Cancel(id string) error
}
// apiScanJobsList handles GET /api/v1/scan-jobs.
// Returns all scan-job records, newest-first.
func (s *Server) apiScanJobsList(w http.ResponseWriter, _ *http.Request) {
db := store.Global()
if db == nil {
writeJSONError(w, "store unavailable", http.StatusServiceUnavailable)
return
}
jobs, err := db.ListScanJobs()
if err != nil {
writeJSONError(w, "failed to list scan jobs: "+err.Error(), http.StatusInternalServerError)
return
}
writeAll(w, jobs)
}
// apiScanJobsRouter handles /api/v1/scan-jobs/{id} and
// /api/v1/scan-jobs/{id}/findings.
//
// - GET /api/v1/scan-jobs/{id} → job detail
// - GET /api/v1/scan-jobs/{id}/findings → paginated findings
//
// Only GET is accepted; other methods return 405.
func (s *Server) apiScanJobsRouter(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
db := store.Global()
if db == nil {
writeJSONError(w, "store unavailable", http.StatusServiceUnavailable)
return
}
tail := strings.TrimPrefix(r.URL.Path, "/api/v1/scan-jobs/")
if tail == "" {
writeJSONError(w, "scan job id required", http.StatusBadRequest)
return
}
parts := strings.SplitN(tail, "/", 2)
id := parts[0]
sub := ""
if len(parts) == 2 {
sub = parts[1]
}
switch sub {
case "":
s.apiScanJobDetail(w, r, db, id)
case "findings":
s.apiScanJobFindings(w, r, db, id)
default:
writeJSONError(w, "not found", http.StatusNotFound)
}
}
// apiScanJobDetail handles GET /api/v1/scan-jobs/{id}.
func (s *Server) apiScanJobDetail(w http.ResponseWriter, _ *http.Request, db *store.DB, id string) {
rec, ok, err := db.GetScanJob(id)
if err != nil {
writeJSONError(w, "failed to get scan job: "+err.Error(), http.StatusInternalServerError)
return
}
if !ok {
writeJSONError(w, "scan job not found", http.StatusNotFound)
return
}
writeJSON(w, map[string]any{"job": rec})
}
// A full-server scan job can record tens of thousands of findings, so the
// findings endpoint pages them.
const (
scanJobFindingsDefaultLimit = 500
scanJobFindingsMaxLimit = 5000
)
// apiScanJobFindings handles GET /api/v1/scan-jobs/{id}/findings.
// Query params: offset (default 0), limit (default 500, at most 5000).
func (s *Server) apiScanJobFindings(w http.ResponseWriter, r *http.Request, db *store.DB, id string) {
offset := queryInt(r, "offset", 0)
limit := queryInt(r, "limit", scanJobFindingsDefaultLimit)
if limit <= 0 {
limit = scanJobFindingsDefaultLimit
}
if limit > scanJobFindingsMaxLimit {
limit = scanJobFindingsMaxLimit
}
findings, total, err := db.ListScanJobFindings(id, offset, limit)
if err != nil {
writeJSONError(w, "failed to list findings: "+err.Error(), http.StatusInternalServerError)
return
}
writeItems(w, toAPIFindings(findings), map[string]any{
"job_id": id,
"total": total,
"offset": offset,
"limit": limit,
"truncated": offset+len(findings) < total,
})
}
// scanJobEnqueueBody is the JSON request body for POST /api/v1/scan-jobs.
type scanJobEnqueueBody struct {
Scope string `json:"scope"`
Target string `json:"target"`
RespectIgnores bool `json:"respect_ignores"`
Quarantine bool `json:"quarantine"`
}
// apiScanJobsEnqueue handles POST /api/v1/scan-jobs.
// Requires admin auth + CSRF (enforced by the mux registration).
func (s *Server) apiScanJobsEnqueue(w http.ResponseWriter, r *http.Request) {
if s.scanJobs == nil {
writeJSONError(w, "scan job manager not available", http.StatusServiceUnavailable)
return
}
var body scanJobEnqueueBody
if err := decodeJSONBodyLimited(w, r, 4*1024, &body); err != nil {
writeJSONError(w, "invalid request body: "+err.Error(), http.StatusBadRequest)
return
}
opts := checks.FullScanOptions(s.liveCfg(), body.RespectIgnores)
switch body.Scope {
case "account":
if body.Target == "" || !control.ValidScanAccountTarget(body.Target) {
writeJSONError(w, "invalid or missing account target", http.StatusBadRequest)
return
}
id, err := s.scanJobs.Enqueue("account", body.Target, opts, body.Quarantine)
if err != nil {
msg := "enqueue failed: " + err.Error()
status := http.StatusInternalServerError
if strings.Contains(err.Error(), "queue is full") {
status = http.StatusConflict
}
writeJSONError(w, msg, status)
return
}
s.auditLog(r, "scan_job_enqueue", body.Target, fmt.Sprintf("job %s, account scan, quarantine=%v", id, body.Quarantine))
// The scan runs after the response: 202 with the job to poll.
writeOKStatus(w, http.StatusAccepted, map[string]interface{}{"job_id": id, "state": "queued"})
case "all":
if body.Quarantine {
writeJSONError(w, "quarantine is not supported with scope \"all\"", http.StatusBadRequest)
return
}
id, err := s.scanJobs.Enqueue("all", "all", opts, false)
if err != nil {
msg := "enqueue failed: " + err.Error()
status := http.StatusInternalServerError
if strings.Contains(err.Error(), "queue is full") {
status = http.StatusConflict
}
writeJSONError(w, msg, status)
return
}
s.auditLog(r, "scan_job_enqueue", "all", fmt.Sprintf("job %s, full scan", id))
// The scan runs after the response: 202 with the job to poll.
writeOKStatus(w, http.StatusAccepted, map[string]interface{}{"job_id": id, "state": "queued"})
default:
writeJSONError(w, "unsupported scope: must be \"account\" or \"all\"", http.StatusBadRequest)
}
}
// apiScanJobsCancel handles POST /api/v1/scan-jobs/{id}/cancel.
// The path arriving here is everything after /api/v1/scan-jobs/, e.g.
// "sj-abc123/cancel". The tail must end exactly in "/<id>/cancel".
// Requires admin auth + CSRF (enforced by the mux registration).
func (s *Server) apiScanJobsCancel(w http.ResponseWriter, r *http.Request) {
if s.scanJobs == nil {
writeJSONError(w, "scan job manager not available", http.StatusServiceUnavailable)
return
}
// Path: /api/v1/scan-jobs/{id}/cancel
tail := strings.TrimPrefix(r.URL.Path, "/api/v1/scan-jobs/")
if !strings.HasSuffix(tail, "/cancel") {
writeJSONError(w, "not found", http.StatusNotFound)
return
}
id := strings.TrimSuffix(tail, "/cancel")
if id == "" {
writeJSONError(w, "scan job id required", http.StatusNotFound)
return
}
if err := s.scanJobs.Cancel(id); err != nil {
msg := err.Error()
status := http.StatusConflict
if strings.Contains(msg, "not found") {
status = http.StatusNotFound
}
writeJSONError(w, "cancel failed: "+msg, status)
return
}
s.auditLog(r, "scan_job_cancel", id, "cancel requested")
// The job stops after the response: 202 with its state.
writeOKStatus(w, http.StatusAccepted, map[string]interface{}{"job_id": id, "state": "canceling"})
}
package webui
import (
"context"
"crypto/hmac"
"crypto/sha256"
"crypto/subtle"
"crypto/tls"
"encoding/hex"
"fmt"
"html/template"
"io/fs"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/antispam"
"github.com/pidginhost/csm/internal/attackdb"
"github.com/pidginhost/csm/internal/broadcast"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/emailav"
"github.com/pidginhost/csm/internal/geoip"
"github.com/pidginhost/csm/internal/health"
"github.com/pidginhost/csm/internal/incident"
"github.com/pidginhost/csm/internal/mailfwd/intel"
"github.com/pidginhost/csm/internal/mailfwd/inventory"
"github.com/pidginhost/csm/internal/mailstats"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/session"
"github.com/pidginhost/csm/internal/state"
persiststore "github.com/pidginhost/csm/internal/store"
)
// IPBlocker abstracts the firewall engine for block/unblock operations.
type IPBlocker interface {
BlockIP(ip string, reason string, timeout time.Duration) error
UnblockIP(ip string) error
}
// blockIPForOperator calls BlockIPForce when the blocker supports it (engine
// on live systems), otherwise falls back to BlockIP (test stubs). This ensures
// operator-initiated blocks from the Web UI are never silenced by dry_run.
// Every attempt reports into the shared firewall outcome metric.
func blockIPForOperator(b IPBlocker, ip, reason string, timeout time.Duration) error {
var err error
if fb, ok := b.(forceBlocker); ok {
err = fb.BlockIPForce(ip, reason, timeout)
} else {
err = b.BlockIP(ip, reason, timeout)
}
checks.ObserveOperatorBlock(err, checks.BlockSourceWebUI)
return err
}
// noListDir wraps an http.FileSystem so http.FileServer cannot serve a
// directory index. Opening a directory returns fs.ErrNotExist, which the
// FileServer turns into a 404, while individual files are served normally.
// This keeps the unauthenticated /static/ assets reachable for the login
// page without letting anyone enumerate the shipped file set.
type noListDir struct{ fs http.FileSystem }
func (d noListDir) Open(name string) (http.File, error) {
f, err := d.fs.Open(name)
if err != nil {
return nil, err
}
info, err := f.Stat()
if err != nil {
_ = f.Close()
return nil, err
}
if info.IsDir() {
_ = f.Close()
return nil, fs.ErrNotExist
}
return f, nil
}
// serverWriteTimeout bounds a response. Handlers that run longer (account
// scans, the hardening audit) extend their own deadline to longRequestTimeout.
const (
serverWriteTimeout = 300 * time.Second
longRequestTimeout = 10 * time.Minute
)
// Server is the web UI HTTP server. Serves API always; serves HTML pages
// and static files only if the UI directory exists on disk.
type Server struct {
sessions *session.Manager
sessionNow func() time.Time
cfg *config.Config
store *state.Store
httpSrv *http.Server
templates map[string]*template.Template
hasUI bool // true if UI directory with templates exists
uiDir string // path to UI directory on disk
staticDir string // uiDir/static
assets assetVersions
startTime time.Time
sigCount int // loaded signature rule count
fanotifyActive func() bool // live daemon reader; nil outside the daemon
logWatcherCount func() int // live daemon reader; nil outside the daemon
blocker IPBlocker
geoIPDB atomic.Pointer[geoip.DB]
// emailQuarantine, emailAVWatcherMode and the mail decision counter
// mailStats are installed by the daemon after the listener already
// serves; request goroutines read them, so they are held atomically and
// read through the accessors below.
emailQuarantine atomic.Pointer[emailav.Quarantine]
emailAVWatcherMode atomic.Value // string
mailStats atomic.Pointer[mailstats.Counter]
forwarderSource inventory.Source
forwarderRemover *inventory.CPanelRemover // nil off cPanel
deferralReporter intel.Reporter
queueReporter intel.QueueReporter
queueFlusher intel.QueueFlusher
queueAdmin intel.QueueAdmin
forwardHeld heldForwardStore
mailboxSuspensions MailboxSuspensions
mcp MCPServer
version string
// queueReasons caches msglog reasons for the queue list by message ID.
queueReasonMu sync.Mutex
queueReasons map[string]cachedQueueReason
// Host metrics are sampled when the Performance page asks, at most once
// per perfSampleTTL; perfMu makes concurrent requests share a sample.
perfSample atomic.Pointer[perfSample]
perfMu sync.Mutex
samplePerf func() *perfMetrics
incidentCorrelator *incident.Correlator
// Rate limiting
threatActionMu sync.Mutex // serialize operator block, clear and undo decisions
loginMu sync.Mutex
loginAttempts map[string][]time.Time
apiMu sync.Mutex
apiRequests map[string][]time.Time // per-IP API rate limiting
scanMu sync.Mutex
scanRunning bool // only one scan at a time
auditMu sync.Mutex // serializes UI audit rotation and appends
modSecApplyMu sync.Mutex // serializes modsec rules apply (write+reload+rollback)
sigCountMu sync.RWMutex
// settingsSaveHook runs before reading or changing a settings save's
// config, including attempts that fail or require a restart.
settingsSaveHook func()
// configAppliedHook runs after a settings save has published a new live
// config, so the daemon can push safe-field values into the subsystems
// that cache them. nil in handler tests.
configAppliedHook func()
// verifyFinding is per server so handler tests can inject a verdict without
// replacing process-wide behavior while another server is handling a request.
verifyFinding func(checks.VerifyInput) checks.VerifyResult
// applyFix is per server for the same reason: handler tests observe a
// fix outcome without touching the host.
applyFix func(ctx context.Context, check, message, details string, filePath ...string) checks.RemediationResult
// scanInProgress reports scans the UI did not start; per server so
// handler tests can hold one without running checks.
scanInProgress func() bool
// Results computed from recent history, reused while it is unchanged.
statsMemo historyMemo
timelineMemo historyMemo
deferralsMemo historyMemo
emailMemos historyMemos
// rspamdSource is set before Start. rspamdMu guards rspamdCache and
// is held across a controller read.
rspamdSource func() (ep antispam.StatsEndpoint, reason string, down bool)
rspamdMu sync.Mutex
rspamdCache rspamdFetch
// accountRoots and scanAccounts are the platform's account inventory;
// per server so handler tests can supply their own tree.
accountRoots func() []string
scanAccounts func(*config.Config) ([]string, error)
// featureState caches the platform features the UI renders.
featureState
provider health.Provider // set by Daemon when it starts the WebUI
mu sync.RWMutex
findingBus *broadcast.Bus // set by Daemon via SetFindingBus
// Graceful shutdown signal for background goroutines and streaming handlers.
// shutdownOnce makes Shutdown idempotent; closing pruneDone twice panics.
pruneDone chan struct{}
shutdownOnce sync.Once
// restartDaemon is called by apiSettingsRestart. Tests override this.
restartDaemon func() (output []byte, err error)
// verifiedBotsReloader pushes a saved verified_bots list into the live
// bot registry + verifier without a restart. Set by the Daemon; nil in tests.
verifiedBotsReloader func() error
// scanJobs is the full-scan job manager injected by the Daemon via
// SetScanJobs. Nil until wired; POST scan-job handlers return 503 when nil.
scanJobs ScanJobController
}
// liveCfg returns the configuration a handler should act on: the last
// reloaded one, or the startup snapshot before any reload. The server keeps
// s.cfg for restart-required wiring (listener, TLS, tokens, state paths);
// thresholds, firewall policy, scan options and alert routing change under a
// SIGHUP and must be read here, or the UI shows and applies stale settings.
func (s *Server) liveCfg() *config.Config {
if live := config.Active(); live != nil {
return live
}
return s.cfg
}
// New creates a new web UI server.
func New(cfg *config.Config, store *state.Store) (*Server, error) {
s := &Server{
cfg: cfg,
store: store,
startTime: time.Now(),
loginAttempts: make(map[string][]time.Time),
apiRequests: make(map[string][]time.Time),
pruneDone: make(chan struct{}),
forwarderSource: selectForwarderSource(),
forwarderRemover: selectForwarderRemover(),
deferralReporter: selectDeferralReporter(),
queueReporter: selectQueueReporter(),
queueFlusher: selectQueueFlusher(),
queueAdmin: selectQueueAdmin(),
forwardHeld: selectForwardHeld(),
verifyFinding: checks.VerifyFindingInput,
applyFix: checks.ApplyFix,
scanInProgress: checks.ScanInProgress,
samplePerf: sampleMetrics,
accountRoots: checks.AccountHomeRoots,
scanAccounts: checks.EnumerateScanAccounts,
}
lifetime, idle, err := cfg.BrowserSessionDurations()
if err != nil {
return nil, err
}
s.sessionNow = time.Now
if db := persiststore.Global(); db != nil {
s.sessions, err = session.New(db, lifetime, idle)
if err != nil {
return nil, fmt.Errorf("initialize browser sessions: %w", err)
}
}
// Check if UI directory exists on disk
s.uiDir = cfg.WebUI.UIDir
if s.uiDir == "" {
s.uiDir = "/opt/csm/ui"
}
funcMap := s.templateFuncs()
// Try to load templates from disk
templateDir := filepath.Join(s.uiDir, "templates")
staticDir := filepath.Join(s.uiDir, "static")
s.staticDir = staticDir
if _, err := os.Stat(templateDir); err == nil {
s.templates = make(map[string]*template.Template)
layoutPath := filepath.Join(templateDir, "layout.html")
for _, page := range []string{"dashboard", "findings", "quarantine", "cleanup-history", "firewall", "modsec", "modsec-rules", "verified-bots", "threat", "rules", "audit", "account", "incident", "email", "performance", "hardening", "settings", "sessions"} {
pagePath := filepath.Join(templateDir, page+".html")
t, err := template.New(page+".html").Funcs(funcMap).ParseFiles(layoutPath, pagePath)
if err != nil {
return nil, fmt.Errorf("parsing template %s from %s: %w", page, templateDir, err)
}
s.templates[page+".html"] = t
}
loginPath := filepath.Join(templateDir, "login.html")
loginTmpl, err := template.New("login.html").Funcs(funcMap).ParseFiles(loginPath)
if err != nil {
return nil, fmt.Errorf("parsing login template: %w", err)
}
s.templates["login.html"] = loginTmpl
s.hasUI = true
fmt.Fprintf(os.Stderr, "WebUI: loaded templates from %s\n", templateDir)
} else {
fmt.Fprintf(os.Stderr, "WebUI: UI directory not found at %s - running in API-only mode\n", s.uiDir)
}
// Set up routes
mux := http.NewServeMux()
// Static files and HTML pages - only if UI directory exists
if s.hasUI {
// Static assets must stay reachable pre-auth (the login page loads its
// own CSS/JS), so they are not behind requireAuth. They must not be
// enumerable, though: noListDir makes directory requests 404 instead of
// returning an index listing of every shipped file.
mux.Handle("/static/", s.staticHandler(staticDir))
mux.HandleFunc("/login", s.handleLogin)
mux.Handle("/", s.requireAuth(http.HandlerFunc(s.handleDashboard)))
mux.Handle("/dashboard", s.requireAuth(http.HandlerFunc(s.handleDashboard)))
mux.Handle("/findings", s.requireAuth(http.HandlerFunc(s.handleFindings)))
mux.Handle("/history", s.requireAuth(http.HandlerFunc(s.handleHistoryRedirect)))
mux.Handle("/quarantine", s.requireAuth(http.HandlerFunc(s.handleQuarantine)))
mux.Handle("/cleanup-history", s.requireAuth(http.HandlerFunc(s.handleCleanupHistory)))
mux.Handle("/blocked", s.requireAuth(http.HandlerFunc(s.handleBlockedRedirect)))
mux.Handle("/firewall", s.requireAuth(http.HandlerFunc(s.handleFirewall)))
mux.Handle("/threat", s.requireAuth(http.HandlerFunc(s.handleThreat)))
mux.Handle("/rules", s.requireAuth(http.HandlerFunc(s.handleRules)))
mux.Handle("/audit", s.requireAuth(http.HandlerFunc(s.handleAudit)))
mux.Handle("/account", s.requireAuth(http.HandlerFunc(s.handleAccount)))
mux.Handle("/incident", s.requireAuth(http.HandlerFunc(s.handleIncident)))
mux.Handle("/email", s.requireAuth(http.HandlerFunc(s.handleEmail)))
mux.Handle("/performance", s.requireAuth(http.HandlerFunc(s.handlePerformance)))
mux.Handle("/hardening", s.requireAuth(http.HandlerFunc(s.handleHardening)))
mux.Handle("/settings", s.requireAuth(http.HandlerFunc(s.handleSettings)))
mux.Handle("GET /sessions", s.requireAuth(http.HandlerFunc(s.handleSessions)))
mux.Handle("POST /sessions/revoke", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.handleSessionRevoke))))
mux.Handle("/modsec", s.requireAuth(http.HandlerFunc(s.handleModSec)))
mux.Handle("/modsec/rules", s.requireAuth(http.HandlerFunc(s.handleModSecRules)))
mux.Handle("/verified-bots", s.requireAuth(http.HandlerFunc(s.handleVerifiedBots)))
}
// Auth-protected API - read (read-scope tokens accepted)
// Any /api/ path no route below matches; see apiNotFound.
mux.Handle("/api/", http.HandlerFunc(s.apiNotFound))
mux.Handle("/api/v1/events", s.requireRead(http.HandlerFunc(s.apiEvents)))
mux.Handle("/api/v1/status", s.requireRead(http.HandlerFunc(s.apiStatus)))
mux.Handle("/api/v1/challenge/stats", s.requireRead(http.HandlerFunc(s.apiChallengeStats)))
// Read-only MCP endpoint; answers 404 until SetMCP attaches a server.
mux.HandleFunc(mcpPath, s.serveMCP)
mux.Handle("/api/v1/scan-jobs", s.requireRead(http.HandlerFunc(s.apiScanJobsList)))
mux.Handle("/api/v1/scan-jobs/", s.requireRead(http.HandlerFunc(s.apiScanJobsRouter)))
// POST /api/v1/scan-jobs — enqueue a new full-scan job (admin + CSRF).
// Go 1.22 method-qualified patterns take precedence over the unqualified
// prefix above, so GET /api/v1/scan-jobs still goes to the read handler.
mux.Handle("POST /api/v1/scan-jobs", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiScanJobsEnqueue))))
// POST /api/v1/scan-jobs/{id}/cancel — cancel a queued or running job.
mux.Handle("POST /api/v1/scan-jobs/", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiScanJobsCancel))))
mux.Handle("/api/v1/findings", s.requireRead(http.HandlerFunc(s.apiFindings)))
mux.Handle("/api/v1/findings/enriched", s.requireRead(http.HandlerFunc(s.apiFindingsEnriched)))
mux.Handle("/api/v1/history", s.requireRead(http.HandlerFunc(s.apiHistory)))
mux.Handle("/api/v1/stats", s.requireRead(http.HandlerFunc(s.apiStats)))
mux.Handle("/api/v1/stats/trend", s.requireRead(http.HandlerFunc(s.apiStatsTrend)))
mux.Handle("/api/v1/stats/timeline", s.requireRead(http.HandlerFunc(s.apiStatsTimeline)))
mux.Handle("/api/v1/blocked-ips", s.requireRead(http.HandlerFunc(s.apiBlockedIPs)))
mux.Handle("/api/v1/capabilities", s.requireRead(http.HandlerFunc(s.apiCapabilities)))
mux.Handle("/api/v1/health", s.requireRead(http.HandlerFunc(s.apiHealth)))
mux.Handle("/api/v1/components", s.requireRead(http.HandlerFunc(s.apiComponents)))
// Auth-protected API - admin-only reads (data with write-adjacent sensitivity)
mux.Handle("/api/v1/quarantine", s.requireAuth(getOnly(http.HandlerFunc(s.apiQuarantine))))
mux.Handle("/api/v1/modsec/stats", s.requireRead(http.HandlerFunc(s.apiModSecStats)))
mux.Handle("/api/v1/modsec/blocks", s.requireRead(http.HandlerFunc(s.apiModSecBlocks)))
mux.Handle("/api/v1/modsec/events", s.requireRead(http.HandlerFunc(s.apiModSecEvents)))
mux.Handle("/api/v1/modsec/rules", s.requireAuth(getOnly(http.HandlerFunc(s.apiModSecRules))))
mux.Handle("/api/v1/modsec/rules/apply", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiModSecRulesApply))))
mux.Handle("/api/v1/modsec/rules/escalation", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiModSecRulesEscalation))))
mux.Handle("/api/v1/verified-bots", s.requireAuth(http.HandlerFunc(s.apiVerifiedBots)))
mux.Handle("/api/v1/verified-bots/apply", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiVerifiedBotsApply))))
mux.Handle("/api/v1/accounts", s.requireAuth(getOnly(http.HandlerFunc(s.apiAccounts))))
mux.Handle("/api/v1/account", s.requireAuth(getOnly(http.HandlerFunc(s.apiAccountDetail))))
mux.Handle("/api/v1/history/csv", s.requireAuth(getOnly(http.HandlerFunc(s.apiHistoryCSV))))
mux.Handle("/api/v1/export", s.requireAuth(getOnly(http.HandlerFunc(s.apiExport))))
mux.Handle("/api/v1/import", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiImport))))
mux.Handle("/api/v1/incident", s.requireAuth(getOnly(http.HandlerFunc(s.apiIncident))))
// Admin-scope on both routes: ServeMux cannot disambiguate by HTTP method,
// so the POST .../status mutator forces admin; reads under the same prefix
// inherit it (admin is a superset of read). The sub-path also runs CSRF
// because the router can dispatch POST .../status; requireCSRF only acts
// on unsafe methods so GET .../<id> still passes through.
mux.Handle("/api/v1/incidents", s.requireAuth(getOnly(http.HandlerFunc(s.apiIncidentList))))
mux.Handle("/api/v1/incidents/groups", s.requireRead(http.HandlerFunc(s.apiIncidentGroups)))
mux.Handle("/api/v1/incidents/", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiIncidentRouter))))
mux.Handle("/api/v1/email/stats", s.requireAuth(getOnly(http.HandlerFunc(s.apiEmailStats))))
mux.Handle("/api/v1/email/quarantine", s.requireAuth(http.HandlerFunc(s.apiEmailQuarantineList)))
mux.Handle("/api/v1/email/quarantine/", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiEmailQuarantineAction))))
mux.Handle("/api/v1/email/av/status", s.requireAuth(http.HandlerFunc(s.apiEmailAVStatus)))
mux.Handle("/api/v1/email/checks", s.requireRead(http.HandlerFunc(s.apiEmailChecks)))
mux.Handle("/api/v1/email/groups", s.requireRead(http.HandlerFunc(s.apiEmailGroups)))
mux.Handle("/api/v1/email/overview", s.requireRead(http.HandlerFunc(s.apiEmailOverview)))
mux.Handle("/api/v1/email/rejects", s.requireAuth(getOnly(http.HandlerFunc(s.apiEmailRejects))))
mux.Handle("/api/v1/email/senders", s.requireAuth(getOnly(http.HandlerFunc(s.apiEmailSenders))))
mux.Handle("/api/v1/email/rspamd", s.requireRead(http.HandlerFunc(s.apiEmailRspamd)))
mux.Handle("/api/v1/email/relay-abuse", s.requireRead(http.HandlerFunc(s.apiEmailRelayAbuse)))
mux.Handle("/api/v1/email/forwarders", s.requireRead(http.HandlerFunc(s.apiEmailForwarders)))
mux.Handle("/api/v1/email/forwarders/delete", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiEmailForwarderDelete))))
mux.Handle("/api/v1/email/deferrals", s.requireRead(http.HandlerFunc(s.apiEmailDeferrals)))
mux.Handle("/api/v1/email/queue-composition", s.requireRead(http.HandlerFunc(s.apiEmailQueueComposition)))
mux.Handle("/api/v1/email/queue/flush-backscatter", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiEmailFlushBackscatter))))
mux.Handle("/api/v1/email/queue", s.requireAuth(getOnly(http.HandlerFunc(s.apiEmailQueueList))))
mux.Handle("/api/v1/email/queue/{id}", s.requireAuth(getOnly(http.HandlerFunc(s.apiEmailQueueMessage))))
mux.Handle("/api/v1/email/queue/{id}/{action}", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiEmailQueueAction))))
mux.Handle("/api/v1/email/held", s.requireAuth(http.HandlerFunc(s.apiEmailHeldList)))
mux.Handle("/api/v1/email/held/", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiEmailHeldAction))))
mux.Handle("/api/v1/email/suspensions", s.requireAuth(http.HandlerFunc(s.apiEmailSuspensionsList)))
mux.Handle("/api/v1/email/suspensions/", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiEmailSuspensionAction))))
mux.Handle("/api/v1/performance", s.requireAuth(getOnly(http.HandlerFunc(s.apiPerformance))))
mux.Handle("/api/v1/perf/fix-error-log", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiPerfFixErrorLog))))
mux.Handle("/api/v1/perf/fix-display-errors", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiPerfFixDisplayErrors))))
mux.Handle("/api/v1/perf/fix-wp-cron", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiPerfFixWPCron))))
mux.Handle("/api/v1/hardening", s.requireAuth(getOnly(http.HandlerFunc(s.apiHardening))))
mux.Handle("/api/v1/hardening/run", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiHardeningRun))))
// Threat Intelligence API
mux.Handle("/api/v1/threat/stats", s.requireAuth(getOnly(http.HandlerFunc(s.apiThreatStats))))
mux.Handle("/api/v1/threat/top-attackers", s.requireAuth(getOnly(http.HandlerFunc(s.apiThreatTopAttackers))))
mux.Handle("/api/v1/threat/ip", s.requireAuth(getOnly(http.HandlerFunc(s.apiThreatIP))))
mux.Handle("/api/v1/threat/events", s.requireAuth(getOnly(http.HandlerFunc(s.apiThreatEvents))))
mux.Handle("/api/v1/threat/db-stats", s.requireAuth(getOnly(http.HandlerFunc(s.apiThreatDBStats))))
mux.Handle("/api/v1/audit", s.requireAuth(getOnly(http.HandlerFunc(s.apiUIAudit))))
mux.Handle("/api/v1/finding-detail", s.requireAuth(getOnly(http.HandlerFunc(s.apiFindingDetail))))
mux.Handle("/api/v1/threat/whitelist-ip", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiThreatWhitelistIP))))
mux.Handle("/api/v1/threat/whitelist", s.requireAuth(getOnly(http.HandlerFunc(s.apiThreatWhitelist))))
mux.Handle("/api/v1/threat/unwhitelist-ip", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiThreatUnwhitelistIP))))
mux.Handle("/api/v1/threat/block-ip", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiThreatBlockIP))))
mux.Handle("/api/v1/threat/block-ip-permanent", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiThreatBlockIPPermanent))))
mux.Handle("/api/v1/threat/clear-ip", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiThreatClearIP))))
mux.Handle("/api/v1/threat/temp-whitelist-ip", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiThreatTempWhitelistIP))))
mux.Handle("/api/v1/threat/bulk-action", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiThreatBulkAction))))
// Rules API
mux.Handle("/api/v1/rules/status", s.requireAuth(getOnly(http.HandlerFunc(s.apiRulesStatus))))
mux.Handle("/api/v1/rules/list", s.requireAuth(getOnly(http.HandlerFunc(s.apiRulesList))))
mux.Handle("/api/v1/rules/reload", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiRulesReload))))
// Suppressions API
mux.Handle("/api/v1/suppressions", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiSuppressions))))
// Firewall API
mux.Handle("/api/v1/firewall/status", s.requireAuth(getOnly(http.HandlerFunc(s.apiFirewallStatus))))
mux.Handle("/api/v1/firewall/allowed", s.requireAuth(getOnly(http.HandlerFunc(s.apiFirewallAllowed))))
mux.Handle("/api/v1/firewall/audit", s.requireAuth(getOnly(http.HandlerFunc(s.apiFirewallAudit))))
mux.Handle("/api/v1/firewall/subnets", s.requireAuth(getOnly(http.HandlerFunc(s.apiFirewallSubnets))))
mux.Handle("/api/v1/firewall/check", s.requireScope("unban", getOnly(http.HandlerFunc(s.apiFirewallCheck))))
// Settings API
mux.Handle("/api/v1/settings/restart", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiSettingsRestart))))
mux.Handle("/api/v1/settings/firewall/tentative-apply", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiFirewallTentativeApply))))
mux.Handle("/api/v1/settings/firewall/confirm", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiFirewallRollbackConfirm))))
mux.Handle("/api/v1/settings/firewall/revert", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiFirewallRollbackRevert))))
mux.Handle("/api/v1/settings/firewall/rollback", s.requireAuth(http.HandlerFunc(s.apiFirewallRollbackStatus)))
mux.Handle("/api/v1/settings", s.requireAuth(http.HandlerFunc(s.apiSettingsSections)))
mux.Handle("/api/v1/settings/", s.requireAuth(http.HandlerFunc(s.apiSettings)))
// GeoIP API
mux.Handle("/api/v1/geoip", s.requireAuth(getOnly(http.HandlerFunc(s.apiGeoIPLookup))))
mux.Handle("/api/v1/geoip/batch", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiGeoIPBatch))))
// Auth-protected API - actions (with CSRF validation)
mux.Handle("/api/v1/fix", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiFix))))
mux.Handle("/api/v1/verify-finding", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiVerifyFinding))))
mux.Handle("/api/v1/fix-bulk", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiBulkFix))))
mux.Handle("/api/v1/scan-account", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiScanAccount))))
mux.Handle("/api/v1/test-alert", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiTestAlert))))
mux.Handle("/api/v1/block-ip", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiBlockIP))))
mux.Handle("/api/v1/unblock-ip", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiUnblockIP))))
mux.Handle("/api/v1/unblock-bulk", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiUnblockBulk))))
mux.Handle("/api/v1/dismiss", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiDismissFinding))))
mux.Handle("/api/v1/quarantine-preview", s.requireAuth(getOnly(http.HandlerFunc(s.apiQuarantinePreview))))
mux.Handle("/api/v1/quarantine-restore", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiQuarantineRestore))))
mux.Handle("/api/v1/quarantine/bulk-delete", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiQuarantineBulkDelete))))
mux.Handle("/api/v1/db-object-backups", s.requireAuth(getOnly(http.HandlerFunc(s.apiDBObjectBackups))))
mux.Handle("/api/v1/db-object-backup-preview", s.requireAuth(http.HandlerFunc(s.apiDBObjectBackupPreview)))
mux.Handle("/api/v1/db-object-backup-restore", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiDBObjectBackupRestore))))
mux.Handle("/api/v1/firewall/deny-subnet", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiFirewallDenySubnet))))
mux.Handle("/api/v1/firewall/allow-ip", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiFirewallAllowIP))))
mux.Handle("/api/v1/firewall/remove-allow", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiFirewallRemoveAllow))))
mux.Handle("/api/v1/firewall/remove-subnet", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiFirewallRemoveSubnet))))
mux.Handle("/api/v1/firewall/cphulk-clear", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiFirewallFlushCphulk))))
mux.Handle("/api/v1/firewall/flush", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiFirewallFlush))))
mux.Handle("/api/v1/firewall/unban", s.requireScope("unban", s.requireCSRF(http.HandlerFunc(s.apiFirewallUnban))))
// Operator preferences (P5.2 saved views, P5.4 user prefs).
mux.Handle("/api/v1/prefs/user", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiPrefsUser))))
mux.Handle("/api/v1/prefs/views", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiPrefsViews))))
// Bulk-action undo (P5.3).
mux.Handle("/api/v1/undo/pending", s.requireAuth(http.HandlerFunc(s.apiUndoPending)))
mux.Handle("/api/v1/undo/run", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiUndoRun))))
// Session management requires admin scope; browser mutations require CSRF.
mux.Handle("/api/v1/sessions", s.requireAuth(http.HandlerFunc(s.apiSessions)))
mux.Handle("DELETE /api/v1/sessions", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiSessions))))
mux.Handle("/api/v1/sessions/", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.apiSessions))))
mux.Handle("/logout", s.requireAuth(s.requireCSRF(http.HandlerFunc(s.handleLogout))))
// /metrics (ROADMAP item 4) has its own auth: the handler accepts
// cfg.WebUI.MetricsToken as a dedicated Bearer token so Prometheus
// scrapers get a credential that does not also unlock the UI, and
// falls back to the existing AuthToken/session path so the UI can
// self-scrape. No CSRF: read-only endpoint.
mux.HandleFunc("/metrics", s.handleMetrics)
s.httpSrv = &http.Server{
Addr: cfg.WebUI.Listen,
Handler: s.securityHeaders(mux),
ReadHeaderTimeout: 10 * time.Second, // slowloris protection
ReadTimeout: 30 * time.Second, // max time to read full request
WriteTimeout: serverWriteTimeout,
IdleTimeout: 120 * time.Second,
MaxHeaderBytes: 1 << 20, // 1MB
TLSConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
// Disable HTTP/2: Go's HTTP/2 implementation applies WriteTimeout
// to the entire connection, not per-stream. Long-running handlers
// (account scans ~5min) cause ERR_HTTP2_PROTOCOL_ERROR in browsers
// when the timeout fires. HTTP/1.1 handles per-request deadlines
// correctly via ResponseController.SetWriteDeadline.
NextProtos: []string{"http/1.1"},
CipherSuites: []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
},
},
}
s.restartDaemon = defaultRestartDaemon
return s, nil
}
// pruneLoginAttempts periodically cleans up stale rate-limit entries.
// It returns when s.pruneDone is closed.
func (s *Server) pruneLoginAttempts() {
ticker := time.NewTicker(5 * time.Minute)
defer ticker.Stop()
for {
select {
case <-s.pruneDone:
return
case <-ticker.C:
s.loginMu.Lock()
cutoff := time.Now().Add(-time.Minute)
for ip, attempts := range s.loginAttempts {
var recent []time.Time
for _, t := range attempts {
if t.After(cutoff) {
recent = append(recent, t)
}
}
if len(recent) == 0 {
delete(s.loginAttempts, ip)
} else {
s.loginAttempts[ip] = recent
}
}
s.loginMu.Unlock()
// Also prune API rate-limit entries
s.apiMu.Lock()
for ip, reqs := range s.apiRequests {
var recent []time.Time
for _, t := range reqs {
if t.After(cutoff) {
recent = append(recent, t)
}
}
if len(recent) == 0 {
delete(s.apiRequests, ip)
} else {
s.apiRequests[ip] = recent
}
}
s.apiMu.Unlock()
}
}
}
// Start starts the HTTPS server. Blocks until shutdown.
func (s *Server) Start() error {
certPath := s.cfg.WebUI.TLSCert
keyPath := s.cfg.WebUI.TLSKey
if certPath == "" && keyPath == "" {
certPath = filepath.Join(s.cfg.StatePath, "webui.crt")
keyPath = filepath.Join(s.cfg.StatePath, "webui.key")
if err := EnsureTLSCert(certPath, keyPath, s.cfg.Hostname); err != nil {
return fmt.Errorf("TLS cert setup: %w", err)
}
} else if err := renewTLSCert(certPath, keyPath); err != nil {
return fmt.Errorf("TLS cert renewal: %w", err)
}
certs, err := newCertReloader(certPath, keyPath)
if err != nil {
return fmt.Errorf("TLS cert load: %w", err)
}
s.httpSrv.TLSConfig.GetCertificate = certs.GetCertificate
// A listener failure ends these workers just as Shutdown does.
defer s.shutdownOnce.Do(func() { close(s.pruneDone) })
obs.Go("webui-prune-logins", s.pruneLoginAttempts)
obs.Go("webui-cert-renewal", func() { s.renewCertLoop(certPath, keyPath) })
fmt.Fprintf(os.Stderr, "WebUI listening on https://%s\n", s.cfg.WebUI.Listen)
return s.httpSrv.ListenAndServeTLS("", "")
}
// renewCertLoop checks the generated certificate daily and renews it before
// it expires; the listener picks up the new files on the next handshake.
func (s *Server) renewCertLoop(certPath, keyPath string) {
ticker := time.NewTicker(24 * time.Hour)
defer ticker.Stop()
for {
select {
case <-s.pruneDone:
return
case <-ticker.C:
if err := renewTLSCert(certPath, keyPath); err != nil {
fmt.Fprintf(os.Stderr, "webui: TLS certificate renewal: %v\n", err)
}
}
}
}
// Shutdown gracefully stops the server. Safe to call more than once;
// the underlying pruneDone close is guarded so duplicate shutdown does
// not panic.
func (s *Server) Shutdown(ctx context.Context) error {
s.shutdownOnce.Do(func() { close(s.pruneDone) })
if s.httpSrv == nil {
return nil
}
return s.httpSrv.Shutdown(ctx)
}
// canonicalAllowedOrigin returns the single CORS origin the web UI
// will accept on /api/ requests. Built from cfg.Hostname plus the
// listen port so a forged HTTP Host header cannot redirect the check.
func (s *Server) canonicalAllowedOrigin() string {
host := canonicalOriginHost(s.cfg.Hostname)
port := webUIListenPort(s.cfg.WebUI.Listen)
if port != "" && port != "443" {
if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
host = host[1 : len(host)-1]
}
host = net.JoinHostPort(host, port)
}
return "https://" + host
}
func canonicalOriginHost(host string) string {
host = strings.TrimSpace(host)
if strings.HasPrefix(host, "[") {
if end := strings.Index(host, "]"); end > 0 {
if ip := net.ParseIP(host[1:end]); ip != nil {
if ip4 := ip.To4(); ip4 != nil {
return ip4.String()
}
return "[" + ip.String() + "]"
}
}
}
if ip := net.ParseIP(host); ip != nil {
if ip4 := ip.To4(); ip4 != nil {
return ip4.String()
}
return "[" + ip.String() + "]"
}
return strings.ToLower(host)
}
func webUIListenPort(listen string) string {
if _, port, err := net.SplitHostPort(listen); err == nil {
return port
}
if idx := strings.LastIndex(listen, ":"); idx >= 0 {
return listen[idx+1:]
}
return ""
}
// originAllowed reports whether a browser Origin may make credentialed
// requests. A loopback origin (an SSH tunnel on any local port) is trusted
// only when it is the origin the request was sent to: host is the request's
// Host. Another local service in the same browser shares the Web UI's
// cookies, since cookies ignore the port, and must not be trusted.
func (s *Server) originAllowed(origin, host string) bool {
u, err := url.Parse(origin)
if err != nil || !originHeaderURL(u) || !strings.EqualFold(u.Scheme, "https") {
return false
}
if isLoopbackOriginHost(u.Hostname()) {
return host != "" && sameOrigin(origin, "https://"+host)
}
if sameOrigin(origin, s.canonicalAllowedOrigin()) {
return true
}
for _, listed := range s.liveCfg().WebUI.AllowedOrigins {
if sameOrigin(origin, listed) {
return true
}
}
return false
}
func isLoopbackOriginHost(host string) bool {
if strings.EqualFold(host, "localhost") {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
func sameOrigin(got, want string) bool {
gotURL, err := url.Parse(got)
if err != nil || !originHeaderURL(gotURL) {
return false
}
wantURL, err := url.Parse(want)
if err != nil || !originHeaderURL(wantURL) {
return false
}
return strings.EqualFold(gotURL.Scheme, wantURL.Scheme) &&
strings.EqualFold(gotURL.Hostname(), wantURL.Hostname()) &&
originPort(gotURL) == originPort(wantURL)
}
func originHeaderURL(u *url.URL) bool {
return u.Scheme != "" && u.Host != "" && u.User == nil &&
u.Path == "" && u.RawQuery == "" && u.Fragment == ""
}
func originPort(u *url.URL) string {
if port := u.Port(); port != "" {
return port
}
switch strings.ToLower(u.Scheme) {
case "https":
return "443"
case "http":
return "80"
default:
return ""
}
}
// SetSigCount sets the loaded signature count for the status API.
func (s *Server) SetSigCount(count int) {
s.sigCountMu.Lock()
s.sigCount = count
s.sigCountMu.Unlock()
}
func (s *Server) signatureCount() int {
s.sigCountMu.RLock()
defer s.sigCountMu.RUnlock()
return s.sigCount
}
// HasUI returns true if UI templates were loaded from disk.
func (s *Server) HasUI() bool {
return s.hasUI
}
// SetIPBlocker sets the firewall engine for block/unblock operations.
func (s *Server) SetIPBlocker(b IPBlocker) {
s.blocker = b
}
// SetHealthInfo installs the daemon readers the health API and dashboard
// call on each request. Log watchers can start after the web UI, so a
// snapshot taken at startup went stale.
func (s *Server) SetHealthInfo(fanotifyActive func() bool, logWatchers func() int) {
s.fanotifyActive = fanotifyActive
s.logWatcherCount = logWatchers
}
func (s *Server) fanotifyRunning() bool {
return s.fanotifyActive != nil && s.fanotifyActive()
}
func (s *Server) logWatchersRunning() int {
if s.logWatcherCount == nil {
return 0
}
return s.logWatcherCount()
}
// SetEmailQuarantine sets the email quarantine for the email AV API endpoints.
func (s *Server) SetEmailQuarantine(q *emailav.Quarantine) {
s.emailQuarantine.Store(q)
}
// SetEmailAVWatcherMode sets the watcher mode string for the email AV status API.
func (s *Server) SetEmailAVWatcherMode(mode string) {
s.emailAVWatcherMode.Store(mode)
}
// emailQuarantineHandle returns the installed email quarantine, or nil
// before the daemon installs one.
func (s *Server) emailQuarantineHandle() *emailav.Quarantine {
return s.emailQuarantine.Load()
}
// SetMailStats installs the mail decision counter the Email Security page
// reads.
func (s *Server) SetMailStats(c *mailstats.Counter) {
s.mailStats.Store(c)
}
// mailStatsCounter returns the installed mail decision counter, or nil
// before the daemon installs one.
func (s *Server) mailStatsCounter() *mailstats.Counter {
return s.mailStats.Load()
}
// SetRspamdSource installs the daemon's answer to where rspamd's
// controller can be read without a password, from the last completed
// antispam pass, and why not when it cannot. The down flag comes from live
// probe health, independent of dismissed findings. Call it before Start.
func (s *Server) SetRspamdSource(fn func() (antispam.StatsEndpoint, string, bool)) {
s.rspamdSource = fn
}
// emailAVMode returns the installed AV watcher mode, "" before it is set.
func (s *Server) emailAVMode() string {
if v, ok := s.emailAVWatcherMode.Load().(string); ok {
return v
}
return ""
}
// SetSettingsSaveHook registers the host refresh for each settings save
// attempt. Call it before Start.
func (s *Server) SetSettingsSaveHook(fn func()) {
s.settingsSaveHook = fn
}
// SetVersion sets the application version for display in the UI.
// SetConfigAppliedHook registers the callback the Daemon uses to apply a
// config the web UI just published, the same steps a SIGHUP reload runs.
func (s *Server) SetConfigAppliedHook(fn func()) {
s.configAppliedHook = fn
}
func (s *Server) SetVersion(v string) {
s.version = v
}
// SetHealthProvider installs the daemon's health provider. The webui
// constructs without one so unit tests can run without a daemon; the
// daemon must call this before any request hits /api/v1/status.
func (s *Server) SetHealthProvider(p health.Provider) {
s.provider = p
}
// SetFindingBus installs the broadcaster the SSE event stream subscribes
// to. The webui constructs without one so unit tests work without a
// daemon; the daemon must call this before any request hits /api/v1/events.
func (s *Server) SetFindingBus(bus *broadcast.Bus) {
s.mu.Lock()
defer s.mu.Unlock()
s.findingBus = bus
}
// SetIncidentCorrelator wires the incident correlator. Called once at
// startup; treated as immutable after first set.
func (s *Server) SetIncidentCorrelator(c *incident.Correlator) {
s.incidentCorrelator = c
}
// SetScanJobs wires the full-scan job manager so POST /api/v1/scan-jobs and
// POST /api/v1/scan-jobs/{id}/cancel can enqueue and cancel jobs through the
// daemon's single-job queue. Must be called before the server starts
// accepting requests; POST handlers return 503 when nil.
func (s *Server) SetScanJobs(c ScanJobController) {
s.scanJobs = c
}
// csmConfig returns the feature-flag map used by the frontend. Templates
// render it through jsonForScript.
func (s *Server) csmConfig() map[string]interface{} {
cfg := s.liveCfg()
return map[string]interface{}{
"version": s.version,
// What this host supports; pages hide the rest.
"features": s.features(),
"emailAV": cfg.EmailAV.Enabled,
"firewall": cfg.Firewall != nil && cfg.Firewall.Enabled,
"autoResponse": cfg.AutoResponse.Enabled,
"threatIntel": cfg.Reputation.AbuseIPDBKey != "",
"signatures": cfg.Signatures.RulesDir != "",
"challenge": cfg.Challenge.Difficulty > 0,
"fanotify": s.fanotifyRunning(),
"hostname": s.cfg.Hostname,
// Attack types (brute_force, waf_block, ...) group findings in the
// attack database; their labels come from there.
"attackTypes": attackdb.AttackTypeLabels(),
// #nosec G101 -- Not credentials. This is a lookup from
// finding check name (webshell, email_credential_leak, etc.) to
// the human-readable label rendered in the UI.
"checkNames": map[string]string{
"webshell": "Web Shell",
"cpanel_login": "cPanel Login",
"perf_load": "Load",
"perf_php_processes": "PHP Processes",
"perf_memory": "Memory",
"perf_php_handler": "PHP Handler",
"perf_mysql_config": "MySQL Config",
"perf_redis_config": "Redis Config",
"perf_error_logs": "Error Logs",
"perf_wp_config": "WP Config",
"perf_wp_transients": "WP Transients",
"perf_wp_cron": "WP Cron",
"perf_wp_loopback": "WP Loopback",
"perf_wp_myisam": "WP MyISAM Tables",
"integrity": "Integrity",
"db_siteurl_hijack": "DB URL Hijack",
"db_siteurl_invalid": "DB Invalid Site Address",
"db_options_injection": "DB Options Injection",
"db_post_injection": "DB Post Injection",
"db_spam_injection": "DB Spam Injection",
"db_rogue_admin": "DB Rogue Admin",
"db_suspicious_admin_email": "DB Suspicious Admin",
"mail_queue": "Mail Queue",
"mail_queue_unavailable": "Mail Queue Unavailable",
"mail_per_account": "Mail Volume",
"email_phishing_content": "Email Phishing",
"email_malware": "Email Malware",
"email_compromised_account": "Compromised Account",
"email_cloud_relay_abuse": "Cloud-Relay Credential Abuse",
"email_spam_outbreak": "Spam Outbreak",
"email_defer_fail_governor": "Defer/Fail Governor",
"email_credential_leak": "Credential Leak",
"email_auth_failure_realtime": "Auth Failure",
"smtp_bruteforce": "SMTP Brute Force",
"smtp_subnet_spray": "SMTP Subnet Spray",
"smtp_account_spray": "SMTP Account Spray",
"mail_bruteforce": "Mail Brute Force",
"mail_bruteforce_suspected": "Mail Auth Advisory",
"mail_subnet_spray": "Mail Subnet Spray",
"mail_auth_backend_degraded": "Mail Auth Backend Degraded",
"admin_panel_bruteforce": "Admin Panel Brute Force",
"mail_account_spray": "Mail Account Spray",
"mail_account_compromised": "Mail Account Compromised",
"http_asn_crawl": "Distributed ASN Crawl",
"http_request_flood": "HTTP Request Flood",
"http_scanner_profile": "HTTP Scanner Profile",
"http_claimed_bot_unverified": "Unverified Claimed Bot",
"http_ua_spoof": "HTTP UA Spoof",
"http_distributed_flood": "Distributed HTTP Flood",
"webshell_caller": "Webshell Caller",
"webshell_path_public": "Flagged File Requested Publicly",
"exim_frozen_realtime": "Frozen Message",
"email_suspicious_geo": "Suspicious Geo Login",
"email_rate_critical": "Email Rate Critical",
"email_rate_warning": "Email Rate Warning",
"email_dkim_failure": "DKIM Failure",
"email_spf_rejection": "SPF Rejection",
"email_pipe_forwarder": "Pipe Forwarder",
"email_suspicious_forwarder": "Suspicious Forwarder",
"cpanel_login_realtime": "cPanel Login",
"cpanel_password_purge_realtime": "Password Purge",
"ssh_login_unknown_ip": "SSH Login",
"pam_login": "PAM Login",
"pam_bruteforce": "PAM Brute Force",
"modsec_block_escalation": "ModSec Escalation",
"modsec_csm_block_escalation": "ModSec Escalation",
"whm_password_change_noninfra": "WHM Password Change",
"password_hijack_confirmed": "Password Hijack",
},
}
}
// --- Template helpers ---
func severityClass(sev alert.Severity) string {
switch sev {
case alert.Critical:
return "critical"
case alert.High:
return "high"
case alert.Warning:
return "warning"
default:
return "secondary"
}
}
func severityLabel(sev alert.Severity) string {
switch sev {
case alert.Critical:
return "CRITICAL"
case alert.High:
return "HIGH"
case alert.Warning:
return "WARNING"
default:
return "INFO"
}
}
// severityRank returns a numeric rank for severity labels (higher = more severe).
func severityRank(label string) int {
switch label {
case "CRITICAL":
return 3
case "HIGH":
return 2
case "WARNING":
return 1
default:
return 0
}
}
func timeAgo(t time.Time) string {
d := time.Since(t)
switch {
case d < time.Minute:
return "just now"
case d < time.Hour:
return fmt.Sprintf("%dm ago", int(d.Minutes()))
case d < 24*time.Hour:
return fmt.Sprintf("%dh ago", int(d.Hours()))
default:
return fmt.Sprintf("%dd ago", int(d.Hours()/24))
}
}
func formatTime(t time.Time) string {
return t.Format("2006-01-02 15:04:05")
}
// isoTime renders an instant for a <time datetime> attribute, which the
// page rewrites in the operator's time zone.
func isoTime(t time.Time) string {
return t.UTC().Format(time.RFC3339)
}
func (s *Server) templateFuncs() template.FuncMap {
return template.FuncMap{
"severityClass": severityClass,
"severityLabel": severityLabel,
"timeAgo": timeAgo,
"formatTime": formatTime,
"isoTime": isoTime,
"asset": s.assetURL,
"csrfToken": func() string { return "" }, // per request, see renderTemplate
"feature": s.feature,
"csmConfig": func() template.JS { return jsonForScript(s.csmConfig()) },
"json": jsonForScript,
"multiply": func(a, b int) int { return a * b },
"add": func(a, b int) int { return a + b },
"subtract": func(a, b int) int { return a - b },
"divisibleBy": func(a, b int) bool { return b != 0 && a%b == 0 },
"serverTimeZone": serverTimeZoneName,
"serverUTCOffset": serverUTCOffsetMinutes,
}
}
// --- Security headers middleware ---
func (s *Server) securityHeaders(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Frame-Options", "DENY")
w.Header().Set("X-Content-Type-Options", "nosniff")
// Turn the legacy XSS auditor off: where browsers still honour it,
// "1; mode=block" can be abused to suppress page scripts.
w.Header().Set("X-XSS-Protection", "0")
w.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains")
w.Header().Set("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self'; connect-src 'self'; img-src 'self' data:; font-src 'self'; object-src 'none'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'")
w.Header().Set("Permissions-Policy", "camera=(), microphone=(), geolocation=(), payment=()")
w.Header().Set("Cache-Control", "no-store")
// CORS/origin validation: reject cross-origin API requests.
// Non-loopback origins must be configured; loopback origins must
// match Host so another local service cannot borrow UI cookies.
// Browser logout and session revocation change server state like API
// writes. Login stays reachable: it already requires the credential.
if strings.HasPrefix(r.URL.Path, "/api/") || r.URL.Path == mcpPath || r.URL.Path == "/logout" || strings.HasPrefix(r.URL.Path, "/sessions") {
origin := r.Header.Get("Origin")
if origin != "" {
if !s.originAllowed(origin, r.Host) {
writeRequestError(w, r, "Cross-origin request blocked", http.StatusForbidden)
return
}
w.Header().Set("Access-Control-Allow-Origin", origin)
w.Header().Set("Access-Control-Allow-Credentials", "true")
w.Header().Add("Vary", "Origin")
}
// A preflight from an unknown origin was refused above. An
// allowed origin learns the methods and headers the API takes,
// or the browser never sends the credentialed write.
if r.Method == "OPTIONS" {
if origin != "" {
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
requested := r.Header.Get("Access-Control-Request-Headers")
if requested == "" {
requested = "Authorization, Content-Type, If-Match, X-CSRF-Token"
}
w.Header().Set("Access-Control-Allow-Headers", requested)
w.Header().Set("Access-Control-Max-Age", "600")
}
w.WriteHeader(http.StatusNoContent)
return
}
}
// API rate limiting: 600 requests per minute per IP. /metrics and
// /mcp take bearer tokens too, so guessing them shares the budget.
if strings.HasPrefix(r.URL.Path, "/api/") || r.URL.Path == "/metrics" || r.URL.Path == mcpPath {
ip := rateLimitKey(r.RemoteAddr)
s.apiMu.Lock()
now := time.Now()
cutoff := now.Add(-time.Minute)
var recent []time.Time
for _, t := range s.apiRequests[ip] {
if t.After(cutoff) {
recent = append(recent, t)
}
}
if len(recent) >= 600 {
s.apiMu.Unlock()
writeRequestError(w, r, "Rate limit exceeded", http.StatusTooManyRequests)
return
}
if _, tracked := s.apiRequests[ip]; !tracked {
boundRateLimitMap(s.apiRequests, cutoff)
}
s.apiRequests[ip] = append(recent, now)
s.apiMu.Unlock()
}
next.ServeHTTP(w, r)
})
}
// getOnly answers every method but GET with a JSON 405, for read routes
// behind requireAuth; requireRead does the same for its routes.
func getOnly(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.Header().Set("Allow", http.MethodGet)
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
next.ServeHTTP(w, r)
})
}
// --- CSRF protection ---
// csrfTokenForSession derives the CSRF token of one browser session from its
// own active admin secret and cookie secret. Each session gets its
// own token and a new login a new one; knowing a token reveals neither
// secret. Expiry of another administrator's credential cannot invalidate it.
func (s *Server) csrfTokenForSession(sessionSecret string) string {
secret := s.csrfSecret(sessionSecret)
if secret == "" || sessionSecret == "" {
return ""
}
mac := hmac.New(sha256.New, []byte(secret))
fmt.Fprintf(mac, "csm-csrf-v2:%s", sessionSecret)
return hex.EncodeToString(mac.Sum(nil))[:32]
}
// csrfTokenFor is the CSRF token of the browser session r carries.
func (s *Server) csrfTokenFor(r *http.Request) string {
c, err := r.Cookie("csm_auth")
if err != nil {
return ""
}
return s.csrfTokenForSession(c.Value)
}
func (s *Server) csrfSecret(sessionSecret string) string {
tok, _ := s.sessionCredential(sessionSecret, "admin", false)
return tok.Token
}
// validateCSRF enforces the browser-session CSRF boundary on state-changing
// routes. Bearer-authenticated requests skip the check because cross-origin
// browser requests cannot attach the Authorization header without script access
// to the bearer token.
func (s *Server) validateCSRF(r *http.Request) bool {
if !isUnsafeCSRFMethod(r.Method) {
return true // only validate state-changing methods
}
// Skip CSRF only when the bearer token itself grants admin writes. A
// read-scope bearer presented alongside an admin cookie must not turn
// the cookie-authenticated request into a CSRF-exempt API call.
// CSRF protection is only needed for cookie-based browser sessions.
if s.isAdminBearerAuth(r) {
return true
}
// A scoped bearer is exempt only where the route's middleware accepted
// that bearer, never beside a browser session that authorized the write.
if actor, ok := r.Context().Value(auditActorKey{}).(auditActor); ok && actor.via == "api" {
return true
}
expected := s.csrfTokenFor(r)
// A request without an admin secret or a browser session cannot prove
// it came from a page of that session, so the mutating path stays closed.
if expected == "" {
return false
}
// Check header (API calls from JS use this)
if token := r.Header.Get("X-CSRF-Token"); token != "" {
return subtle.ConstantTimeCompare([]byte(token), []byte(expected)) == 1
}
// Check form field (traditional form posts). Only the body counts: a
// token in the query string would end up in URLs, logs and Referer.
if token := r.PostFormValue("csrf_token"); token != "" {
return subtle.ConstantTimeCompare([]byte(token), []byte(expected)) == 1
}
return false
}
// requireCSRF wraps a handler to validate CSRF on POST, PUT, PATCH, and DELETE
// requests. PUT joined the unsafe set when /api/v1/prefs/user landed; the
// existing list pre-dates that endpoint.
func (s *Server) requireCSRF(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Skip CSRF for admin Bearer token auth. API-to-API callers do not
// need CSRF protection, but read-scope bearer tokens never authorize
// mutating handlers on their own.
if isUnsafeCSRFMethod(r.Method) && !s.isAdminBearerAuth(r) && !s.validateCSRF(r) {
writeRequestError(w, r, "Invalid CSRF token", http.StatusForbidden)
return
}
next.ServeHTTP(w, r)
})
}
func isUnsafeCSRFMethod(method string) bool {
switch method {
case http.MethodPost, http.MethodPut, http.MethodDelete, http.MethodPatch:
return true
default:
return false
}
}
func (s *Server) isBearerAuth(r *http.Request) bool {
_, ok := s.bearerTokenWithScope(r, "read")
return ok
}
func (s *Server) isAdminBearerAuth(r *http.Request) bool {
_, ok := s.bearerTokenWithScope(r, "admin")
return ok
}
// --- Scan rate limiting ---
// acquireScan tries to start a scan. Returns false if a scan is already running.
func (s *Server) acquireScan() bool {
s.scanMu.Lock()
defer s.scanMu.Unlock()
if s.scanRunning {
return false
}
s.scanRunning = true
return true
}
func (s *Server) releaseScan() {
s.scanMu.Lock()
s.scanRunning = false
s.scanMu.Unlock()
}
package webui
import (
"os"
"strings"
"time"
)
// Sources of the host's time zone name. Tests replace them.
var (
timeZoneEnv = func() (string, bool) { return os.LookupEnv("TZ") }
readTimeZoneFile = func() ([]byte, error) { return os.ReadFile("/etc/timezone") }
readLocaltimeLink = func() (string, error) { return os.Readlink("/etc/localtime") }
)
// serverTimeZoneName returns the host's IANA time zone name, or "" when it
// cannot be determined. The browser needs the name to show "server time";
// Go only knows the loaded zone as "Local".
func serverTimeZoneName() string {
valid := func(name string) string {
name = strings.TrimSpace(name)
if name == "" || name == "Local" {
return ""
}
if _, err := time.LoadLocation(name); err != nil {
return ""
}
return name
}
if env, set := timeZoneEnv(); set {
// An explicit TZ controls Go's Local even when it is empty or
// unreadable; host files must not replace that choice.
env = strings.TrimPrefix(env, ":")
if env == "" {
return "UTC"
}
if i := strings.Index(env, "zoneinfo/"); i >= 0 {
env = env[i+len("zoneinfo/"):]
}
return valid(env)
}
if target, err := readLocaltimeLink(); err == nil {
if i := strings.Index(target, "zoneinfo/"); i >= 0 {
if name := valid(target[i+len("zoneinfo/"):]); name != "" {
return name
}
}
}
if b, err := readTimeZoneFile(); err == nil {
if name := valid(string(b)); name != "" {
return name
}
}
return ""
}
// serverUTCOffsetMinutes is the fallback when the zone has no known name:
// the current offset, which is right until the next daylight-saving change.
func serverUTCOffsetMinutes() int {
_, offset := time.Now().Zone()
return offset / 60
}
package webui
import (
"net/http"
"strings"
"time"
"github.com/pidginhost/csm/internal/session"
)
type browserSessionView struct {
ID string `json:"id"`
Name string `json:"name"`
Created time.Time `json:"created"`
LastSeen time.Time `json:"last_seen"`
Expires time.Time `json:"expires"`
RemoteIP string `json:"remote_ip"`
UserAgent string `json:"user_agent"`
Current bool `json:"current"`
}
func (s *Server) browserSessionViews(r *http.Request) ([]browserSessionView, error) {
if s.sessions == nil {
return nil, errNoStore
}
records, err := s.sessions.List(s.sessionNow())
if err != nil {
return nil, err
}
current := ""
if cookie, err := r.Cookie("csm_auth"); err == nil {
current = session.Hash(cookie.Value)
}
views := make([]browserSessionView, 0, len(records))
for _, rec := range records {
views = append(views, browserSessionView{ID: rec.ID, Name: rec.Name, Created: rec.Created,
LastSeen: rec.LastSeen, Expires: rec.Expires, RemoteIP: rec.RemoteIP, UserAgent: rec.UserAgent, Current: rec.Verifier == current})
}
return views, nil
}
func (s *Server) apiSessions(w http.ResponseWriter, r *http.Request) {
id := strings.TrimPrefix(r.URL.Path, "/api/v1/sessions")
if id != "" {
id = strings.TrimPrefix(id, "/")
if !validSessionID(id) {
writeJSONError(w, "Session not found", http.StatusNotFound)
return
}
}
switch r.Method {
case http.MethodGet:
if id != "" {
writeJSONError(w, "Session not found", http.StatusNotFound)
return
}
views, err := s.browserSessionViews(r)
if err != nil {
writeJSONError(w, "Session store unavailable", http.StatusServiceUnavailable)
return
}
writeAll(w, views)
case http.MethodDelete:
if s.sessions == nil {
writeJSONError(w, "Session store unavailable", http.StatusServiceUnavailable)
return
}
actor, via := s.requestActor(r)
if id != "" && !s.sessionExists(id) {
writeJSONError(w, "Session not found", http.StatusNotFound)
return
}
var err error
if id == "" {
err = s.sessions.RevokeAll()
} else {
err = s.sessions.Revoke(id)
}
if err != nil {
writeJSONError(w, "Cannot revoke browser session", http.StatusServiceUnavailable)
return
}
s.auditSessionRevoke(r, actor, via, id)
if id == "" {
clearBrowserCookie(w)
}
writeOK(w, nil)
default:
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
}
}
// sessionExists reports whether id names an active browser session, so
// revoking an unknown one answers 404 instead of a success that did nothing.
func (s *Server) sessionExists(id string) bool {
records, err := s.sessions.List(s.sessionNow())
if err != nil {
return false
}
for _, rec := range records {
if rec.ID == id {
return true
}
}
return false
}
func validSessionID(id string) bool {
if len(id) != 32 {
return false
}
for _, ch := range id {
if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') {
return false
}
}
return true
}
func (s *Server) handleSessions(w http.ResponseWriter, r *http.Request) {
views, err := s.browserSessionViews(r)
if err != nil {
http.Error(w, "Session store unavailable", http.StatusServiceUnavailable)
return
}
s.renderTemplate(w, r, "sessions.html", map[string]any{"Sessions": views})
}
func (s *Server) handleSessionRevoke(w http.ResponseWriter, r *http.Request) {
if s.sessions == nil {
http.Error(w, "Session store unavailable", http.StatusServiceUnavailable)
return
}
r.Body = http.MaxBytesReader(w, r.Body, 4096)
if err := r.ParseForm(); err != nil {
http.Error(w, "Invalid request", http.StatusBadRequest)
return
}
id := r.PostForm.Get("id")
actor, via := s.requestActor(r)
var err error
switch {
case id == "all":
err = s.sessions.RevokeAll()
case validSessionID(id):
err = s.sessions.Revoke(id)
default:
http.Error(w, "Invalid session", http.StatusBadRequest)
return
}
if err != nil {
http.Error(w, "Cannot revoke browser session", http.StatusServiceUnavailable)
return
}
target := id
if id == "all" {
target = ""
}
s.auditSessionRevoke(r, actor, via, target)
if id == "all" {
clearBrowserCookie(w)
http.Redirect(w, r, "/login", http.StatusSeeOther)
return
}
http.Redirect(w, r, "/sessions", http.StatusSeeOther)
}
// auditSessionRevoke records a revocation for the actor resolved before it,
// since revoking the caller's own session also ends its attribution. An
// empty id means every browser session.
func (s *Server) auditSessionRevoke(r *http.Request, actor, via, id string) {
if id == "" {
s.auditLogAs(r, actor, via, "session_revoke_all", "browser sessions", "every browser session logged out")
return
}
s.auditLogAs(r, actor, via, "session_revoke", id, "browser session revoked")
}
package webui
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"os/exec"
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/integrity"
"gopkg.in/yaml.v3"
)
const settingsURLPrefix = "/api/v1/settings/"
type pendingSettingsSection struct {
ID string `json:"id"`
Title string `json:"title"`
}
func pendingRestartSections(live, disk *config.Config) []pendingSettingsSection {
if live == nil || disk == nil {
return nil
}
seen := map[string]struct{}{}
for _, c := range config.Diff(live, disk) {
if c.Tag == config.TagSafe {
continue
}
for _, section := range settingsSections {
if c.Field == section.YAMLPath || strings.HasPrefix(c.Field, section.YAMLPath+".") {
seen[section.ID] = struct{}{}
break
}
}
}
if len(seen) == 0 {
return nil
}
out := make([]pendingSettingsSection, 0, len(seen))
for _, section := range settingsSections {
if _, ok := seen[section.ID]; ok {
out = append(out, pendingSettingsSection{ID: section.ID, Title: section.Title})
}
}
return out
}
func cloneConfigForSettingsApply(src *config.Config) config.Config {
clone := *src
if src.Firewall != nil {
fw := *src.Firewall
clone.Firewall = &fw
}
return clone
}
func copySettingsChangeValues(dst, src *config.Config, section SettingsSection, changes map[string]json.RawMessage) error {
for key := range changes {
path := []string{section.YAMLPath}
if key != "" {
path = append(path, strings.Split(key, ".")...)
}
if err := copyConfigPathValue(dst, src, path); err != nil {
return err
}
if section.YAMLPath == "auto_response" && key == "max_blocks_per_hour" {
dst.AutoResponse.MaxBlocksPerHourDefaulted = src.AutoResponse.MaxBlocksPerHourDefaulted
}
}
return nil
}
func copyConfigPathValue(dst, src *config.Config, path []string) error {
dstValue := reflect.ValueOf(dst).Elem()
srcValue := reflect.ValueOf(src).Elem()
for i, key := range path {
if srcValue.Kind() == reflect.Pointer {
if srcValue.IsNil() {
return fmt.Errorf("path %v: source element %d is nil", path, i)
}
if dstValue.IsNil() {
dstValue.Set(reflect.New(dstValue.Type().Elem()))
}
srcValue = srcValue.Elem()
dstValue = dstValue.Elem()
}
field, ok := fieldByYAMLTag(srcValue.Type(), key)
if !ok {
return fmt.Errorf("no yaml field %q under %s", key, strings.Join(path[:i], "."))
}
srcValue = srcValue.FieldByIndex(field.Index)
dstValue = dstValue.FieldByIndex(field.Index)
}
dstValue.Set(srcValue)
return nil
}
func (s *Server) apiSettingsSections(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
writeJSON(w, map[string]interface{}{
"groups": SectionGroupOrder,
"sections": s.settingsSections(),
})
}
func (s *Server) apiSettings(w http.ResponseWriter, r *http.Request) {
// This prefix serves both GET (read settings) and POST (update); CSRF is
// enforced only on the mutating POST path.
switch r.Method {
case http.MethodGet:
s.apiSettingsGet(w, r)
case http.MethodPost:
s.requireCSRF(http.HandlerFunc(s.apiSettingsPost)).ServeHTTP(w, r)
default:
writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
}
}
func (s *Server) apiSettingsGet(w http.ResponseWriter, r *http.Request) {
sectionID := strings.TrimPrefix(r.URL.Path, settingsURLPrefix)
if sectionID == "" || strings.Contains(sectionID, "/") {
writeJSONError(w, "section required", http.StatusBadRequest)
return
}
if sectionID == "restart" {
writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
section, ok := s.lookupSettingsSection(sectionID)
if !ok {
writeJSONError(w, "unknown section", http.StatusNotFound)
return
}
resolveFieldOptions(§ion)
diskBytes, err := os.ReadFile(s.cfg.ConfigFile) // #nosec G304 -- operator-configured config path
if err != nil {
writeJSONError(w, "read config: "+err.Error(), http.StatusInternalServerError)
return
}
mergedBytes, err := config.MergeBytesWithDir(diskBytes, s.cfg.ConfigDir)
if err != nil {
writeJSONError(w, "load config: "+err.Error(), http.StatusInternalServerError)
return
}
disk, err := config.LoadBytes(mergedBytes)
if err != nil {
writeJSONError(w, "load config: "+err.Error(), http.StatusInternalServerError)
return
}
disk.ConfigFile = s.cfg.ConfigFile
disk.ConfigDir = s.cfg.ConfigDir
redacted := config.Redact(disk)
values, err := extractSectionValues(mergedBytes, redacted, section)
if err != nil {
writeJSONError(w, "extract: "+err.Error(), http.StatusInternalServerError)
return
}
// A field this host hides is left out of the values too.
if full, ok := LookupSettingsSection(sectionID); ok {
for _, f := range full.Fields {
if lookupSchemaField(section, f.YAMLPath) == nil {
delete(values, f.YAMLPath)
}
}
}
var pendingFields []string
var pendingSections []pendingSettingsSection
if live := config.Active(); live != nil {
diff := config.Diff(live, disk)
for _, c := range diff {
if c.Tag != config.TagSafe && (c.Field == section.YAMLPath || strings.HasPrefix(c.Field, section.YAMLPath+".")) {
pendingFields = append(pendingFields, c.Field)
}
}
pendingSections = pendingRestartSections(live, disk)
}
etag := integrity.HashConfigStableBytes(diskBytes)
w.Header().Set("ETag", etag)
writeJSON(w, map[string]interface{}{
"section": section,
"values": values,
"etag": etag,
"pending_restart": len(pendingFields) > 0,
"pending_fields": pendingFields,
"pending_sections": pendingSections,
})
}
func extractSectionValues(rawBytes []byte, effectiveCfg *config.Config, section SettingsSection) (map[string]interface{}, error) {
effective, err := extractSectionEffectiveValues(effectiveCfg, section)
if err != nil {
return nil, err
}
raw, err := extractSectionRawValues(rawBytes, section)
if err != nil {
return nil, err
}
values := make(map[string]interface{}, len(effective))
for k, v := range effective {
values[k] = v
}
overlayNullableState(section, values, raw)
return values, nil
}
func extractSectionEffectiveValues(cfg *config.Config, section SettingsSection) (map[string]interface{}, error) {
var wrapper map[string]interface{}
data, err := yaml.Marshal(cfg)
if err != nil {
return nil, err
}
if err := yaml.Unmarshal(data, &wrapper); err != nil {
return nil, err
}
raw, ok := wrapper[section.YAMLPath]
if !ok {
return map[string]interface{}{}, nil
}
if _, isMap := raw.(map[string]interface{}); !isMap {
return map[string]interface{}{section.YAMLPath: raw}, nil
}
return raw.(map[string]interface{}), nil
}
func extractSectionRawValues(rawBytes []byte, section SettingsSection) (map[string]interface{}, error) {
var wrapper map[string]interface{}
if err := yaml.Unmarshal(rawBytes, &wrapper); err != nil {
return nil, err
}
raw, ok := wrapper[section.YAMLPath]
if !ok {
return map[string]interface{}{}, nil
}
if _, isMap := raw.(map[string]interface{}); !isMap {
return map[string]interface{}{section.YAMLPath: raw}, nil
}
return raw.(map[string]interface{}), nil
}
func overlayNullableState(section SettingsSection, values, raw map[string]interface{}) {
for _, field := range section.Fields {
if !field.Nullable {
continue
}
// All v1 nullable fields are direct children of the section.
if strings.Contains(field.YAMLPath, ".") {
continue
}
if v, ok := raw[field.YAMLPath]; ok {
values[field.YAMLPath] = v
continue
}
values[field.YAMLPath] = nil
}
}
func (s *Server) apiSettingsPost(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
sectionID := strings.TrimPrefix(r.URL.Path, settingsURLPrefix)
if sectionID == "" || strings.Contains(sectionID, "/") {
writeJSONError(w, "section required", http.StatusBadRequest)
return
}
section, ok := s.lookupSettingsSection(sectionID)
if !ok {
writeJSONError(w, "unknown section", http.StatusNotFound)
return
}
ifMatch := r.Header.Get("If-Match")
if ifMatch == "" {
writeJSONError(w, "If-Match header required", http.StatusBadRequest)
return
}
var body struct {
Changes map[string]json.RawMessage `json:"changes"`
}
if err := decodeJSONBodyLimited(w, r, 256*1024, &body); err != nil {
writeJSONError(w, "invalid body: "+err.Error(), http.StatusBadRequest)
return
}
configMu := integrity.ConfigWriteMutex()
configMu.Lock()
defer configMu.Unlock()
if s.settingsSaveHook != nil {
s.settingsSaveHook()
}
if rejectConfigWriteDuringRollback(w) {
return
}
diskBytes, err := os.ReadFile(s.cfg.ConfigFile) // #nosec G304 -- operator-supplied config path
if err != nil {
writeJSONError(w, "read config: "+err.Error(), http.StatusInternalServerError)
return
}
disk, err := config.LoadBytes(diskBytes)
if err != nil {
writeJSONError(w, "parse config: "+err.Error(), http.StatusInternalServerError)
return
}
disk.ConfigFile = s.cfg.ConfigFile
disk.ConfigDir = s.cfg.ConfigDir
if integrity.HashConfigStableBytes(diskBytes) != ifMatch {
writeJSONError(w, "config changed on disk, reload", http.StatusPreconditionFailed)
return
}
if rejectIfConfDirChanged(w, s.cfg.ConfigDir, disk) {
return
}
effectiveDisk, err := config.LoadBytesWithDir(diskBytes, s.cfg.ConfigDir)
if err != nil {
writeJSONError(w, "load merged config: "+err.Error(), http.StatusInternalServerError)
return
}
effectiveDisk.ConfigFile = s.cfg.ConfigFile
clone := cloneConfigForSettingsApply(disk)
yamlChanges, errs := buildChangeSet(section, &clone, body.Changes)
if len(errs) > 0 {
writeValidationErrors(w, errs)
return
}
// Validate the merged result before the edited YAML goes back through
// Load. This looks like a duplicate of the check further down, but Load
// rejects some combinations itself and returns a plain error, which loses
// the field attribution the dashboard needs to mark the offending input.
effectiveCandidate := cloneConfigForSettingsApply(effectiveDisk)
if _, errs := buildChangeSet(section, &effectiveCandidate, body.Changes); len(errs) > 0 {
writeValidationErrors(w, errs)
return
}
fieldErrors, _ := splitValidationResults(config.Validate(&effectiveCandidate))
if len(fieldErrors) > 0 {
writeValidationErrors(w, fieldErrors)
return
}
edited, err := config.YAMLEdit(diskBytes, yamlChanges)
if err != nil {
writeJSONError(w, "yaml edit: "+err.Error(), http.StatusInternalServerError)
return
}
effectiveClone, err := config.LoadBytesWithDir(edited, s.cfg.ConfigDir)
if err != nil {
writeJSONError(w, "load edited merged config: "+err.Error(), http.StatusUnprocessableEntity)
return
}
effectiveClone.ConfigFile = s.cfg.ConfigFile
// Drop-ins can replace the entered credential. Validate the actual
// destination/credential pair before either disk or live config changes.
rebindErrs, err := credentialRebindErrors(section, effectiveDisk, effectiveClone, body.Changes)
if err != nil {
writeJSONError(w, "read merged settings: "+err.Error(), http.StatusInternalServerError)
return
}
if len(rebindErrs) > 0 {
writeValidationErrors(w, rebindErrs)
return
}
validationResults := append(config.Validate(effectiveClone), config.ValidateDeepSection(effectiveClone, section.ID)...)
fieldErrors, warnings := splitValidationResults(validationResults)
if len(fieldErrors) > 0 {
writeValidationErrors(w, fieldErrors)
return
}
if section.ID == "firewall" {
localizeValidationFields(warnings, "firewall")
}
diff := config.Diff(effectiveDisk, effectiveClone)
var restartFields []string
for _, c := range diff {
if c.Tag != config.TagSafe {
restartFields = append(restartFields, c.Field)
}
}
var liveCandidate *config.Config
if len(restartFields) == 0 {
if live := config.Active(); live != nil {
candidate := cloneConfigForSettingsApply(live)
if err := copySettingsChangeValues(&candidate, effectiveClone, section, body.Changes); err != nil {
writeJSONError(w, "prepare live config: "+err.Error(), http.StatusInternalServerError)
return
}
liveCandidate = &candidate
} else {
candidate := cloneConfigForSettingsApply(effectiveClone)
liveCandidate = &candidate
}
// Pending restart settings remain active in this candidate. A valid
// disk config can still disable the last channel of a running digest.
// Check the existing invariants before changing disk or live state.
liveErrors, _ := splitValidationResults(config.Validate(liveCandidate))
if len(liveErrors) > 0 {
for i := range liveErrors {
liveErrors[i].Message += "; restart to apply pending settings first"
}
writeValidationErrors(w, liveErrors)
return
}
}
if err := integrity.SignAndSavePreserving(s.cfg.ConfigFile, s.cfg.ConfigDir, edited, &clone, disk.Integrity.BinaryHash); err != nil {
writeJSONError(w, "save: "+err.Error(), http.StatusInternalServerError)
return
}
newETag := clone.Integrity.ConfigHash
newIntegrity := clone.Integrity
effectiveClone.Integrity = newIntegrity
if liveCandidate != nil {
applySignedIntegrityState(liveCandidate, &clone)
config.SetActive(liveCandidate)
if s.configAppliedHook != nil {
s.configAppliedHook()
}
} else if live := config.Active(); live != nil {
livePatched := *live
applySignedIntegrityState(&livePatched, &clone)
config.SetActive(&livePatched)
}
applied := []string{}
for _, c := range diff {
applied = append(applied, c.Field)
}
s.auditLog(r, "settings-save", sectionID, auditDetailsFor(section, body.Changes))
if restartFields == nil {
restartFields = []string{}
}
if warnings == nil {
warnings = []fieldError{}
}
pending := pendingRestartSections(config.Active(), effectiveClone)
if pending == nil {
pending = []pendingSettingsSection{}
}
writeOK(w, map[string]interface{}{
"applied": applied,
"requires_restart": restartFields,
"pending_restart": len(restartFields) > 0,
"pending_sections": pending,
"warnings": warnings,
"new_etag": newETag,
})
}
// applySignedIntegrityState keeps the conf.d digest and the main-config
// exemption policy that selected it together when an API save updates only a
// subset of the running config. Pairing a new hash with an older list makes
// the next periodic Verify report a false mismatch.
func applySignedIntegrityState(dst, signedMain *config.Config) {
dst.Integrity = signedMain.Integrity
dst.ConfD = signedMain.ConfD
}
// rejectIfConfDirChanged refuses to bless a save when the drop-ins on disk no
// longer match what disk (the main config as last signed) recorded. The
// exemption list of that same on-disk config decides which fragments count.
func rejectIfConfDirChanged(w http.ResponseWriter, confDir string, disk *config.Config) bool {
currentHash, err := integrity.HashConfDir(confDir, disk.ConfD.IntegrityExempt)
if err != nil {
writeJSONError(w, "hash conf.d: "+err.Error(), http.StatusInternalServerError)
return true
}
if currentHash != disk.Integrity.ConfdHash {
writeJSONError(w, "conf.d changed on disk, reload", http.StatusPreconditionFailed)
return true
}
return false
}
type fieldError struct {
Field string `json:"field"`
Message string `json:"message"`
}
func splitValidationResults(results []config.ValidationResult) (errs []fieldError, warnings []fieldError) {
for _, v := range results {
if v.Level == "error" {
errs = append(errs, fieldError{Field: v.Field, Message: v.Message})
continue
}
if v.Level == "warn" {
warnings = append(warnings, fieldError{Field: v.Field, Message: v.Message})
}
}
return errs, warnings
}
func localizeValidationFields(results []fieldError, section string) {
prefix := section + "."
for i := range results {
results[i].Field = strings.TrimPrefix(results[i].Field, prefix)
}
}
// writeValidationErrors answers 422 with the one error message every
// failure carries and the per-field problems next to it.
func writeValidationErrors(w http.ResponseWriter, errs []fieldError) {
msg := "Invalid values"
if len(errs) == 1 {
msg = errs[0].Field + ": " + errs[0].Message
}
writeJSONStatus(w, http.StatusUnprocessableEntity, map[string]interface{}{"error": msg, "errors": errs})
}
const fileOnlyFieldMessage = "Change this in csm.yaml. The web UI cannot set commands, file paths, sockets or environment variable names."
// credentialRebindErrors refuses a URL change that would send the stored
// credential of that URL to a new address unless the same save enters the
// credential again. A credential read from an environment variable cannot be
// re-entered here, so its address can only change in csm.yaml.
func credentialRebindErrors(section SettingsSection, current, candidate *config.Config, changes map[string]json.RawMessage) ([]fieldError, error) {
var errs []fieldError
var before, after map[string]interface{}
for _, field := range section.Fields {
if field.CredentialField == "" {
continue
}
if before == nil {
var err error
before, err = extractSectionEffectiveValues(current, section)
if err != nil {
return nil, err
}
after, err = extractSectionEffectiveValues(candidate, section)
if err != nil {
return nil, err
}
}
if settingsStringAt(before, field.YAMLPath) == settingsStringAt(after, field.YAMLPath) {
continue
}
if env := settingsStringAt(after, field.CredentialEnvField); env != "" {
errs = append(errs, fieldError{Field: field.YAMLPath, Message: "The credential for this address is read from environment variable " + env + ". Change the address in csm.yaml."})
continue
}
secret := settingsStringAt(after, field.CredentialField)
if secret == "" {
continue
}
if entered := enteredSecret(changes[field.CredentialField]); entered == "" || entered != secret {
errs = append(errs, fieldError{Field: field.YAMLPath, Message: "Enter the effective credential again when changing this address. If a conf.d drop-in overrides it, change the address and credential in the configuration files."})
}
}
return errs, nil
}
// settingsStringAt returns the string at a dotted path inside a section's
// effective values, or "" when the path is absent or not a string.
func settingsStringAt(values map[string]interface{}, dotted string) string {
var cur interface{} = values
for _, part := range strings.Split(dotted, ".") {
m, ok := cur.(map[string]interface{})
if !ok {
return ""
}
cur = m[part]
}
s, _ := cur.(string)
return s
}
// enteredSecret returns a secret the request actually supplies: absent, empty
// and the redaction placeholder all mean "keep the stored value".
func enteredSecret(raw json.RawMessage) string {
if raw == nil {
return ""
}
var v string
if err := json.Unmarshal(raw, &v); err != nil || v == config.RedactedValue || strings.TrimSpace(v) == "" {
return ""
}
return v
}
func buildChangeSet(section SettingsSection, clone *config.Config, changes map[string]json.RawMessage) ([]config.YAMLChange, []fieldError) {
var out []config.YAMLChange
var errs []fieldError
for key, raw := range changes {
field := lookupSchemaField(section, key)
if field == nil {
errs = append(errs, fieldError{Field: key, Message: "unknown field"})
continue
}
if field.FileOnly {
errs = append(errs, fieldError{Field: key, Message: fileOnlyFieldMessage})
continue
}
if field.Secret {
var sv string
if err := json.Unmarshal(raw, &sv); err == nil && sv == config.RedactedValue {
continue
}
}
if field.Type == "[]enum" {
if field.OptionsSource == "disabled_check_names" || field.OptionsSource == "check_names" {
var err error
raw, err = normaliseDisabledCheckNamesRaw(raw)
if err != nil {
errs = append(errs, fieldError{Field: key, Message: "decode: " + err.Error()})
continue
}
}
if badValues, ok := validateEnumArray(field, raw); !ok {
for _, bv := range badValues {
errs = append(errs, fieldError{Field: key, Message: "unknown value: " + bv})
}
continue
}
}
if field.Type == "enum" {
if badValue, ok := validateEnumScalar(field, raw); !ok {
errs = append(errs, fieldError{Field: key, Message: "unknown value: " + badValue})
continue
}
}
if field.Type == "[]int" {
normalised, badValues, perr := normaliseIntArray(field, raw)
if perr != nil {
errs = append(errs, fieldError{Field: key, Message: perr.Error()})
continue
}
if len(badValues) > 0 {
for _, bv := range badValues {
errs = append(errs, fieldError{Field: key, Message: "invalid value: " + bv})
}
continue
}
raw = normalised
}
// For float fields, coerce JSON string -> JSON number so the downstream
// json.Unmarshal into *float64 (in applyToClone) succeeds.
if field.Type == "float" {
if normalised, ok := coerceFloatRaw(raw); ok {
raw = normalised
} else {
errs = append(errs, fieldError{Field: key, Message: "decode: expected float"})
continue
}
}
fullPath := section.YAMLPath
if key != "" {
fullPath = section.YAMLPath + "." + key
}
decoded, err := decodeJSONForYAML(raw, field)
if err != nil {
errs = append(errs, fieldError{Field: key, Message: "decode: " + err.Error()})
continue
}
out = append(out, config.YAMLChange{Path: strings.Split(fullPath, "."), Value: decoded})
if err := applyToClone(clone, strings.Split(fullPath, "."), raw); err != nil {
errs = append(errs, fieldError{Field: key, Message: err.Error()})
}
}
return out, errs
}
func validateEnumScalar(field *SettingsField, raw json.RawMessage) (bad string, ok bool) {
var value string
if err := json.Unmarshal(raw, &value); err != nil {
return "(not a string)", false
}
resolved := resolvedOptionsForField(field)
if len(resolved) == 0 {
return "", true
}
for _, opt := range resolved {
if value == opt {
return "", true
}
}
return value, false
}
func normaliseDisabledCheckNamesRaw(raw json.RawMessage) (json.RawMessage, error) {
var values []string
if err := json.Unmarshal(raw, &values); err != nil {
return nil, fmt.Errorf("expected string array")
}
seen := make(map[string]struct{}, len(values))
out := make([]string, 0, len(values))
for _, value := range values {
value = config.CanonicalCheckName(strings.TrimSpace(value))
if value == "" {
continue
}
if _, ok := seen[value]; ok {
continue
}
seen[value] = struct{}{}
out = append(out, value)
}
encoded, err := json.Marshal(out)
if err != nil {
return nil, err
}
return encoded, nil
}
// validateEnumArray checks that raw is a JSON array of strings, each of
// which appears in the field's resolved Options. Returns the slice of
// unknown values and ok=false when any value is out-of-set. An empty
// array is allowed (clears the list). Calls resolveFieldOptions-style
// logic by inspecting OptionsSource + Options directly so validation
// doesn't depend on GET ordering.
func validateEnumArray(field *SettingsField, raw json.RawMessage) (bad []string, ok bool) {
var values []string
if err := json.Unmarshal(raw, &values); err != nil {
return []string{"(not a string array)"}, false
}
resolved := resolvedOptionsForField(field)
if len(resolved) == 0 {
return nil, true
}
allowed := make(map[string]struct{}, len(resolved))
for _, v := range resolved {
allowed[v] = struct{}{}
}
seen := map[string]struct{}{}
for _, v := range values {
if _, dup := seen[v]; dup {
continue
}
seen[v] = struct{}{}
if _, okk := allowed[v]; !okk {
bad = append(bad, v)
}
}
return bad, len(bad) == 0
}
// resolvedOptionsForField returns the flat list of allowed values for a
// []enum field, whether it uses static Options or an OptionsSource. Keeps
// POST-side validation independent of the GET-time mutation.
func resolvedOptionsForField(field *SettingsField) []string {
tmp := &SettingsField{Type: field.Type, OptionsSource: field.OptionsSource}
switch field.OptionsSource {
case "check_names":
applyCheckNameOptions(tmp)
case "disabled_check_names":
// Validation accepts a wider set than the UI dropdown (which lists
// public finding names only): the scheduler also honors compatibility
// runner IDs, so rejecting them here would break existing configs.
return disabledCheckValidationOptions()
case "geoip_editions":
applyGeoIPEditionOptions(tmp)
}
if len(tmp.Options) > 0 {
return tmp.Options
}
if len(field.Options) > 0 {
return field.Options
}
return tmp.Options
}
// normaliseIntArray parses raw as a JSON array of integers (or strings that
// parse as integers), enforces field.Min/Max as the per-element bound (default
// 1..65535 for port-list semantics when both are nil), deduplicates, and
// returns a JSON array of distinct ascending integers. Returns the offending
// raw values as badValues when any element is outside the allowed range; the
// parse error path is reserved for malformed JSON.
func normaliseIntArray(field *SettingsField, raw json.RawMessage) (json.RawMessage, []string, error) {
var items []json.RawMessage
if err := json.Unmarshal(raw, &items); err != nil {
return nil, nil, fmt.Errorf("expected array of ints: %s", err.Error())
}
minV := int64(1)
maxV := int64(65535)
if field.Min != nil {
minV = *field.Min
}
if field.Max != nil {
maxV = *field.Max
}
seen := make(map[int64]struct{}, len(items))
out := make([]int64, 0, len(items))
var bad []string
for _, item := range items {
var n int64
if err := json.Unmarshal(item, &n); err != nil {
var s string
if serr := json.Unmarshal(item, &s); serr != nil {
bad = append(bad, string(item))
continue
}
s = strings.TrimSpace(s)
if s == "" {
continue
}
parsed, perr := strconv.ParseInt(s, 10, 64)
if perr != nil {
bad = append(bad, s)
continue
}
n = parsed
}
if n < minV || n > maxV {
bad = append(bad, strconv.FormatInt(n, 10))
continue
}
if _, dup := seen[n]; dup {
continue
}
seen[n] = struct{}{}
out = append(out, n)
}
if len(bad) > 0 {
return nil, bad, nil
}
sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
encoded, err := json.Marshal(out)
if err != nil {
return nil, nil, err
}
return encoded, nil, nil
}
// coerceFloatRaw returns a JSON-number representation of raw if raw is either
// a JSON number already or a JSON string that parses to float64. The second
// return is false if neither form is valid.
func coerceFloatRaw(raw json.RawMessage) (json.RawMessage, bool) {
var asNum float64
if err := json.Unmarshal(raw, &asNum); err == nil {
b, _ := json.Marshal(asNum)
return b, true
}
var asStr string
if err := json.Unmarshal(raw, &asStr); err == nil {
f, err := strconv.ParseFloat(asStr, 64)
if err != nil {
return nil, false
}
b, _ := json.Marshal(f)
return b, true
}
return nil, false
}
func lookupSchemaField(section SettingsSection, key string) *SettingsField {
for i := range section.Fields {
if section.Fields[i].YAMLPath == key {
return §ion.Fields[i]
}
}
return nil
}
func decodeJSONForYAML(raw json.RawMessage, field *SettingsField) (interface{}, error) {
if string(raw) == "null" {
if !field.Nullable {
return nil, fmt.Errorf("null is only allowed for nullable fields")
}
return nil, nil
}
if field.Type == "float" {
// Accept either a JSON number or a JSON string containing a number.
var asNum float64
if err := json.Unmarshal(raw, &asNum); err == nil {
return asNum, nil
}
var asStr string
if err := json.Unmarshal(raw, &asStr); err == nil {
f, perr := strconv.ParseFloat(asStr, 64)
if perr != nil {
return nil, fmt.Errorf("not a float: %q", asStr)
}
return f, nil
}
return nil, fmt.Errorf("expected float, got %s", string(raw))
}
var v interface{}
if err := json.Unmarshal(raw, &v); err != nil {
return nil, err
}
return v, nil
}
func applyToClone(cfg *config.Config, path []string, raw json.RawMessage) error {
v := reflect.ValueOf(cfg).Elem()
for i, key := range path {
if v.Kind() == reflect.Pointer {
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
v = v.Elem()
}
if v.Kind() != reflect.Struct {
return fmt.Errorf("path %v: element %d is not a struct", path, i)
}
field, ok := fieldByYAMLTag(v.Type(), key)
if !ok {
return fmt.Errorf("no yaml field %q under %s", key, strings.Join(path[:i], "."))
}
v = v.FieldByIndex(field.Index)
}
ptr := reflect.New(v.Type())
if err := json.Unmarshal(raw, ptr.Interface()); err != nil {
return fmt.Errorf("unmarshal into %s: %w", v.Type(), err)
}
v.Set(ptr.Elem())
return nil
}
func fieldByYAMLTag(t reflect.Type, yamlName string) (reflect.StructField, bool) {
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
tag := f.Tag.Get("yaml")
if tag == "" {
continue
}
name := tag
if idx := strings.IndexByte(tag, ','); idx >= 0 {
name = tag[:idx]
}
if name == yamlName {
return f, true
}
}
return reflect.StructField{}, false
}
func auditDetailsFor(section SettingsSection, changes map[string]json.RawMessage) string {
redacted := make(map[string]interface{}, len(changes))
for k, raw := range changes {
if field := lookupSchemaField(section, k); field != nil && field.Secret {
redacted[k] = "***"
continue
}
var v interface{}
_ = json.Unmarshal(raw, &v)
redacted[k] = v
}
b, _ := json.Marshal(redacted)
return string(b)
}
// defaultRestartDaemon is the production implementation. Tests override
// s.restartDaemon with a fake.
func defaultRestartDaemon() ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// #nosec G204 -- fixed argv, no operator input interpolated.
cmd := exec.CommandContext(ctx, "systemctl", "restart", "csm")
return cmd.CombinedOutput()
}
// settingsRestartDelay is how long the restart is deferred after the 202 is
// written, so the acknowledgement reaches the client before systemctl restart
// SIGTERMs this process. Overridden in tests to keep them fast.
var settingsRestartDelay = 250 * time.Millisecond
// apiSettingsRestart handles POST /api/v1/settings/restart. It schedules the
// restart asynchronously and returns 202 immediately: `systemctl restart csm`
// SIGTERMs this very process, so a synchronous restart cannot send a clean
// response -- the call returns "signal: terminated" and the old handler turned
// that into a spurious 500 even though the restart had succeeded. The frontend
// treats 202 as "restart issued" and polls for the daemon to return; a failed
// restart surfaces as the daemon not coming back, and the error is logged
// server-side by scheduleDaemonRestart.
func (s *Server) apiSettingsRestart(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
s.auditLog(r, "settings-restart", "daemon", "")
writeOKStatus(w, http.StatusAccepted, map[string]interface{}{
"started_at_token": s.daemonStartToken(),
})
if flusher, ok := w.(http.Flusher); ok {
flusher.Flush()
}
s.scheduleDaemonRestart(settingsRestartDelay)
}
package webui
import (
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/geoip"
)
// resolveFieldOptions populates Options / OptionGroups for any []enum field
// that declares an OptionsSource. Called once per GET /api/v1/settings/:id
// so the UI always sees a fresh list (for check_names this matters if the
// registry grows over time).
//
// The section's Fields slice header points into the package-level
// settingsSections backing array. Writing into it directly would race
// across concurrent requests. Copy the slice first so mutations are local
// to this request.
func resolveFieldOptions(section *SettingsSection) {
fields := make([]SettingsField, len(section.Fields))
copy(fields, section.Fields)
section.Fields = fields
for i := range section.Fields {
f := §ion.Fields[i]
if f.OptionsSource == "" {
continue
}
switch f.OptionsSource {
case "check_names":
applyCheckNameOptions(f)
case "disabled_check_names":
applyDisabledCheckNameOptions(f)
case "geoip_editions":
applyGeoIPEditionOptions(f)
}
}
}
func applyCheckNameOptions(f *SettingsField) {
infos := checks.PublicCheckInfos()
byCategory := make(map[string][]string)
var order []string
for _, info := range infos {
if _, ok := byCategory[info.Category]; !ok {
order = append(order, info.Category)
}
byCategory[info.Category] = append(byCategory[info.Category], info.Name)
}
groups := make([]OptionGroup, 0, len(order))
flat := make([]string, 0, len(infos))
for _, cat := range order {
groups = append(groups, OptionGroup{Label: cat, Values: byCategory[cat]})
flat = append(flat, byCategory[cat]...)
}
f.Options = flat
f.OptionGroups = groups
}
func applyDisabledCheckNameOptions(f *SettingsField) {
allowed := make(map[string]struct{})
for _, name := range checks.DisabledCheckNames() {
allowed[name] = struct{}{}
}
infos := checks.PublicCheckInfos()
byCategory := make(map[string][]string)
var order []string
for _, info := range infos {
if _, ok := allowed[info.Name]; !ok {
continue
}
if _, ok := byCategory[info.Category]; !ok {
order = append(order, info.Category)
}
byCategory[info.Category] = append(byCategory[info.Category], info.Name)
}
groups := make([]OptionGroup, 0, len(order))
flat := make([]string, 0, len(allowed))
for _, cat := range order {
groups = append(groups, OptionGroup{Label: cat, Values: byCategory[cat]})
flat = append(flat, byCategory[cat]...)
}
f.Options = flat
f.OptionGroups = groups
}
// disabledCheckValidationOptions returns the values a POST to the scheduled-scan
// disabled_checks list may contain: public finding names plus the compatibility
// runner IDs the scheduler still honors. Wider than the UI option set so saving
// an existing config that disables a check by runner ID does not get rejected.
func disabledCheckValidationOptions() []string {
return checks.DisabledCheckConfigNames()
}
func applyGeoIPEditionOptions(f *SettingsField) {
free, commercial := geoip.KnownEditions()
flat := append([]string{}, free...)
flat = append(flat, commercial...)
f.Options = flat
f.OptionGroups = []OptionGroup{
{Label: "GeoLite2 (free)", Values: free},
{Label: "GeoIP2 (paid)", Values: commercial},
}
}
package webui
import (
"fmt"
"github.com/pidginhost/csm/internal/admission"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/firewall"
)
// OptionGroup is an ordered label + values pair used to render grouped
// multi-select options (e.g. "Authentication & Login" → [cpanel_login, ...]).
type OptionGroup struct {
Label string `json:"label"`
Values []string `json:"values"`
}
// SettingsField describes a single editable leaf within a settings
// section. YAMLPath is the dotted key path relative to the section's
// YAMLPath. For example inside the Alerts section, the field with
// YAMLPath "email.enabled" has full path "alerts.email.enabled".
//
// For Type "[]enum" fields, Options and/or OptionGroups are resolved at
// request time. A field may either declare a static Options list or set
// OptionsSource to have the handler populate Options/OptionGroups from a
// registry ("check_names", "geoip_editions").
type SettingsField struct {
YAMLPath string `json:"yaml_path"`
Type string `json:"type"`
Label string `json:"label"`
Help string `json:"help,omitempty"`
Secret bool `json:"secret,omitempty"`
Nullable bool `json:"nullable,omitempty"`
Min *int64 `json:"min,omitempty"`
Max *int64 `json:"max,omitempty"`
Options []string `json:"options,omitempty"`
OptionGroups []OptionGroup `json:"option_groups,omitempty"`
OptionsSource string `json:"options_source,omitempty"`
Placeholder string `json:"placeholder,omitempty"`
// FieldGroup is the inner subdivider label rendered as a fieldset
// inside a single section (e.g. firewall fields split into Access
// ports / Rate limits / Logging). Empty string means the field
// renders ungrouped under the section's flat grid.
FieldGroup string `json:"field_group,omitempty"`
// Feature names the platform feature (see uiFeatures) this field needs.
// A host without it does not show the field and cannot set it; the
// value in csm.yaml is left alone. Empty means every host.
Feature string `json:"-"`
// FileOnly fields are shown but can only be changed in csm.yaml. They
// name a command, executable, path, socket or environment variable the
// root daemon acts on, so a browser session must not be able to set them.
FileOnly bool `json:"file_only,omitempty"`
// CredentialField and CredentialEnvField name the secret this URL field
// sends to its address. A web UI change of the URL must re-enter that
// secret, so a session cannot point a stored credential at a host of its
// choosing; a secret read from the environment can only move in csm.yaml.
CredentialField string `json:"-"`
CredentialEnvField string `json:"-"`
}
// SettingsSection groups the fields of one top-level Config sub-tree.
// YAMLPath is the root key in csm.yaml (e.g. "auto_response"). ID is
// the URL-path identifier used by the API. Restart is a UI hint based
// on the current hotreload struct tag; final safe-vs-restart authority
// comes from config.Diff at runtime. Icon is a Tabler icon suffix (e.g.
// "bell" for "ti ti-bell"); Group is the nav category the section lives
// in ("Alerting", "Detection", "Integrations", "Ops").
type SettingsSection struct {
ID string `json:"id"`
Title string `json:"title"`
YAMLPath string `json:"yaml_path"`
Restart bool `json:"restart_hint"`
ReloadTag string `json:"reload_tag,omitempty"`
Icon string `json:"icon,omitempty"`
Group string `json:"group,omitempty"`
// Feature hides the whole section on hosts without it; see
// SettingsField.Feature.
Feature string `json:"-"`
Fields []SettingsField `json:"fields"`
}
// Section groups for the sidebar. Order here defines order in the UI.
const (
SectionGroupAlerting = "Alerting"
SectionGroupDetection = "Detection"
SectionGroupFirewall = "Firewall"
SectionGroupIntegrations = "Integrations"
SectionGroupOps = "Operations"
)
// SectionGroupOrder is the display order of sidebar group headers.
var SectionGroupOrder = []string{
SectionGroupAlerting,
SectionGroupDetection,
SectionGroupFirewall,
SectionGroupIntegrations,
SectionGroupOps,
}
// Field-group labels used to subdivide large sections. Reuse via
// constants so Phase 6 does not scatter free strings through the
// schema and the static test can lock them in.
const (
FieldGroupAccessPorts = "Access ports"
FieldGroupIPv6 = "IPv6"
FieldGroupRateLimits = "Rate limits"
FieldGroupFloodProtection = "Flood protection"
FieldGroupGeoDynDNS = "Geo and DynDNS"
FieldGroupSMTPControls = "SMTP controls"
FieldGroupLogging = "Logging"
FieldGroupLimits = "Limits"
FieldGroupScanIntervals = "Scan intervals"
FieldGroupWebBruteForce = "Web brute force" // #nosec G101 -- UI label, not a credential.
FieldGroupMailBruteForce = "Mail brute force"
FieldGroupSMTPBruteForce = "SMTP brute force"
FieldGroupAccountSpray = "Account spray"
FieldGroupStateRetention = "State retention"
FieldGroupAbuseReporting = "Abuse reporting"
FieldGroupCentralDB = "Central database"
FieldGroupBlockDigest = "Block digest"
)
func int64p(v int64) *int64 { return &v }
var settingsSections = []SettingsSection{
{
ID: "alerts",
Title: "Alerts",
YAMLPath: "alerts",
Icon: "bell",
Group: SectionGroupAlerting,
Restart: false,
Fields: []SettingsField{
{YAMLPath: "email.enabled", Type: "bool", Label: "Email alerts enabled"},
{YAMLPath: "email.to", Type: "[]string", Label: "Recipients", Help: "One email address per line"},
{YAMLPath: "email.from", Type: "string", Label: "From address"},
{YAMLPath: "email.smtp", Type: "string", Label: "SMTP server", Placeholder: "smtp.example.com:587"},
{YAMLPath: "email.disabled_checks", Type: "[]enum", Label: "Disabled check names", OptionsSource: "check_names", Help: "Findings with these check names never trigger email alerts."},
{YAMLPath: "webhook.enabled", Type: "bool", Label: "Webhook alerts enabled"},
{YAMLPath: "webhook.url", Type: "string", Label: "Webhook URL"},
{YAMLPath: "webhook.type", Type: "enum", Label: "Webhook type", Options: []string{"slack", "discord", "generic", "phpanel"}},
{YAMLPath: "webhook.hmac_secret", Type: "string", Label: "Webhook HMAC secret", Secret: true},
{YAMLPath: "webhook.hmac_secret_env", Type: "string", Label: "Webhook HMAC secret env", FileOnly: true},
{YAMLPath: "heartbeat.enabled", Type: "bool", Label: "Heartbeat enabled"},
{YAMLPath: "heartbeat.url", Type: "string", Label: "Heartbeat URL"},
{YAMLPath: "max_per_hour", Type: "int", Label: "Max alerts per hour", Min: int64p(0), Max: int64p(10000)},
{YAMLPath: "block_digest.enabled", Type: "bool", Label: "Watch blocks by country", Help: "Adds watched-country block counts to the daily summary and allows live block alerts.", FieldGroup: FieldGroupBlockDigest},
{YAMLPath: "block_digest.countries", Type: "[]string", Label: "Watched countries", Help: "Two-letter country codes, one per line. Empty falls back to trusted countries, then all countries.", FieldGroup: FieldGroupBlockDigest},
{YAMLPath: "block_digest.interval", Type: "string", Label: "Summary save interval", Placeholder: "1h", Help: "How often the daily summary is saved and watched blocks fold into it, and how long a live alert for one address waits before it can repeat. Below 1m runs as 1m.", FieldGroup: FieldGroupBlockDigest},
{YAMLPath: "block_digest.live", Type: "bool", Label: "Send live block alerts", Help: "Also send one alert per qualifying watched-country block.", FieldGroup: FieldGroupBlockDigest},
{YAMLPath: "block_digest.send_on", Type: "enum", Label: "Live alert policy", Options: []string{"any", "customer"}, Help: "Live alerts only. With customer, only blocks classed as a likely customer alert.", FieldGroup: FieldGroupBlockDigest},
{YAMLPath: "block_digest.channel", Type: "enum", Label: "Live alert channel", Options: []string{"", "email", "webhook"}, Help: "Live alerts only. Empty uses every enabled alert channel. The daily summary uses every enabled channel except a phpanel webhook.", FieldGroup: FieldGroupBlockDigest},
{YAMLPath: "block_digest.min_block", Type: "int", Label: "Minimum watched blocks", Min: int64p(0), Max: int64p(100000), Help: "Watched blocks needed to show the summary section. 0 always shows it.", FieldGroup: FieldGroupBlockDigest},
},
},
{
ID: "thresholds",
Title: "Thresholds",
YAMLPath: "thresholds",
Icon: "adjustments",
Group: SectionGroupAlerting,
Restart: false,
Fields: []SettingsField{
{YAMLPath: "mail_queue_warn", Type: "int", Label: "Mail queue warn", Min: int64p(0), FieldGroup: FieldGroupScanIntervals},
{YAMLPath: "mail_queue_crit", Type: "int", Label: "Mail queue critical", Min: int64p(0), FieldGroup: FieldGroupScanIntervals},
{YAMLPath: "deep_scan_interval_min", Type: "int", Label: "Deep scan interval (min)", Min: int64p(1), FieldGroup: FieldGroupScanIntervals},
{YAMLPath: "multi_ip_login_threshold", Type: "int", Label: "Multi-IP login threshold", Min: int64p(1), FieldGroup: FieldGroupAccountSpray},
{YAMLPath: "multi_ip_login_window_min", Type: "int", Label: "Multi-IP login window (min)", Min: int64p(1), FieldGroup: FieldGroupAccountSpray},
{YAMLPath: "cred_stuffing_distinct_accounts", Type: "int", Label: "Credential stuffing distinct accounts", Min: int64p(2), Max: int64p(200), FieldGroup: FieldGroupAccountSpray, Help: "Distinct failed accounts from one source IP inside the auth window before credential_stuffing fires. Default 5."},
{YAMLPath: "pam_bruteforce_threshold", Type: "int", Label: "PAM brute-force threshold", Min: int64p(2), Max: int64p(1000), FieldGroup: FieldGroupAccountSpray, Help: "PAM authentication failures from one source IP inside the window before pam_bruteforce fires and the address is auto-blocked. Default 5."},
{YAMLPath: "pam_bruteforce_window_min", Type: "int", Label: "PAM brute-force window (min)", Min: int64p(1), Max: int64p(1440), FieldGroup: FieldGroupAccountSpray, Help: "Minutes over which PAM failures from one source IP are counted. Default 10."},
{YAMLPath: "plugin_check_interval_min", Type: "int", Label: "Plugin check interval (min)", Min: int64p(1), FieldGroup: FieldGroupScanIntervals},
{YAMLPath: "brute_force_window", Type: "int", Label: "Brute force window", Min: int64p(1), FieldGroup: FieldGroupWebBruteForce},
{YAMLPath: "domlog_max_files", Type: "int", Label: "Domlog max files", Min: int64p(1), Max: int64p(100000), FieldGroup: FieldGroupWebBruteForce},
{YAMLPath: "domlog_tail_lines", Type: "int", Label: "Domlog tail lines", Min: int64p(10), Max: int64p(100000), FieldGroup: FieldGroupWebBruteForce, Help: "Trailing lines tailed from each per-domain access log per WP brute-force cycle. Default 500 covers ~10 minutes of traffic on a busy site."},
{YAMLPath: "domlog_max_age_min", Type: "int", Label: "Domlog max age (min)", Min: int64p(1), Max: int64p(1440), FieldGroup: FieldGroupWebBruteForce, Help: "Skip per-domain access logs untouched for this many minutes. Default 30. Raise on low-traffic hosts where a slow-burn dictionary attack against a quiet domain still needs to fall inside the freshness window."},
{YAMLPath: "mail_log_tail_lines", Type: "int", Label: "Mail log tail lines", Min: int64p(10), Max: int64p(100000), FieldGroup: FieldGroupMailBruteForce, Help: "Trailing lines of /var/log/exim_mainlog read by the per-account mail rate scanner. Default 500. Raise on busy mail hosts where a single account's spam burst spreads across more than 500 lines per cycle."},
{YAMLPath: "syslog_messages_tail_lines", Type: "int", Label: "Syslog messages tail lines", Min: int64p(10), Max: int64p(100000), FieldGroup: FieldGroupScanIntervals, Help: "Trailing lines of /var/log/messages read only by the legacy direct-check FTP path. The daemon's FTP brute-force detector follows the log forward-only and uses ftp_fail_window_min instead. Default 200."},
{YAMLPath: "ftp_fail_window_min", Type: "int", Label: "FTP fail window (min)", Min: int64p(1), Max: int64p(1440), FieldGroup: FieldGroupScanIntervals, Help: "Sliding window in minutes over which the FTP brute-force detector accumulates per-IP pure-ftpd auth failures before raising ftp_bruteforce. Default 30."},
{YAMLPath: "exposed_file_scan_depth", Type: "int", Label: "Exposed-file scan depth", Min: int64p(1), Max: int64p(int64(config.MaxExposedFileScanDepth)), FieldGroup: FieldGroupLimits, Help: "Directory levels below each web document root inspected for downloadable sensitive files. Default 2; maximum 10."},
{YAMLPath: "account_scan_max_files", Type: "int", Label: "Account scan max files", Min: int64p(1), Max: int64p(100000), FieldGroup: FieldGroupLimits, Help: "Per-cycle cap for account and mail-domain scanner paths. Default 10000. Raise on very large multi-tenant hosts."},
{YAMLPath: "crontab_base64_blob_max_bytes", Type: "int", Label: "Crontab base64 blob max bytes", Min: int64p(1024), Max: int64p(1048576), FieldGroup: FieldGroupLimits, Help: "Encoded-byte cap for one crontab base64 candidate before decoded-content matching. Default 16384. Must be a multiple of 4."},
{YAMLPath: "http_flood_threshold", Type: "int", Label: "HTTP flood threshold", Min: int64p(0), FieldGroup: FieldGroupWebBruteForce, Help: "Per-IP requests per window that emits http_request_flood. 0 disables. Sample baseline first."},
{YAMLPath: "http_flood_window_min", Type: "int", Label: "HTTP flood window (min)", Min: int64p(1), FieldGroup: FieldGroupWebBruteForce},
{YAMLPath: "http_ua_spoof_threshold", Type: "int", Label: "UA spoof threshold", Min: int64p(1), FieldGroup: FieldGroupWebBruteForce},
{YAMLPath: "xmlrpc_threshold", Type: "int", Label: "XML-RPC threshold", Min: int64p(0), FieldGroup: FieldGroupWebBruteForce, Help: "Per-IP POST /xmlrpc.php count before access-log xmlrpc_abuse. 0 disables."},
{YAMLPath: "http_distributed_min_ips", Type: "int", Label: "Distributed HTTP min IPs", Min: int64p(0), FieldGroup: FieldGroupWebBruteForce, Help: "Distinct already-abusive source IPs per vhost before the distributed HTTP flood rollup fires. 0 disables."},
{YAMLPath: "http_scanner_min_requests", Type: "int", Label: "URL scanner min requests", Min: int64p(0), FieldGroup: FieldGroupWebBruteForce, Help: "Per-IP request volume before http_scanner_profile evaluates. 0 disables."},
{YAMLPath: "http_scanner_error_pct", Type: "int", Label: "URL scanner error percent", Min: int64p(1), Max: int64p(100), FieldGroup: FieldGroupWebBruteForce, Help: "Minimum percentage of requests that must return a probe-error status. Default 90."},
{YAMLPath: "http_scanner_min_distinct_paths", Type: "int", Label: "URL scanner distinct paths", Min: int64p(1), Max: int64p(int64(config.HTTPScannerMaxDistinctPaths)), FieldGroup: FieldGroupWebBruteForce, Help: "Minimum distinct error paths after query strings are stripped. Default 10."},
{YAMLPath: "http_scanner_status_codes", Type: "[]int", Label: "URL scanner status codes", Min: int64p(100), Max: int64p(599), FieldGroup: FieldGroupWebBruteForce, Help: "HTTP statuses counted as probe errors. Default 404 and 403; add redirects only when legitimate traffic cannot be redirect-heavy."},
{YAMLPath: "smtp_bruteforce_threshold", Type: "int", Label: "SMTP bruteforce threshold", Min: int64p(1), FieldGroup: FieldGroupSMTPBruteForce},
{YAMLPath: "smtp_bruteforce_window_min", Type: "int", Label: "SMTP bruteforce window (min)", Min: int64p(1), FieldGroup: FieldGroupSMTPBruteForce},
{YAMLPath: "smtp_bruteforce_suppress_min", Type: "int", Label: "SMTP bruteforce suppress (min)", Min: int64p(1), FieldGroup: FieldGroupSMTPBruteForce},
{YAMLPath: "smtp_bruteforce_subnet_threshold", Type: "int", Label: "SMTP bruteforce /24 threshold", Min: int64p(1), FieldGroup: FieldGroupSMTPBruteForce},
{YAMLPath: "smtp_account_spray_threshold", Type: "int", Label: "SMTP account spray threshold", Min: int64p(1), FieldGroup: FieldGroupAccountSpray},
{YAMLPath: "smtp_bruteforce_max_tracked", Type: "int", Label: "SMTP bruteforce max tracked", Min: int64p(100), FieldGroup: FieldGroupStateRetention},
{YAMLPath: "smtp_bruteforce_slow_threshold", Type: "int", Label: "SMTP slow bruteforce threshold", Min: int64p(0), Max: int64p(int64(config.SlowBruteMaxThreshold)), FieldGroup: FieldGroupSMTPBruteForce, Help: "Failed auths from one IP across at least three mailboxes inside the slow window; 10 distinct mailboxes trigger on breadth alone. Default 40; 0 disables; nonzero values must be at least 3."},
{YAMLPath: "smtp_bruteforce_slow_window_min", Type: "int", Label: "SMTP slow bruteforce window (min)", Min: int64p(1), Max: int64p(int64(config.SlowBruteMaxWindowMin)), FieldGroup: FieldGroupSMTPBruteForce, Help: "Long-horizon sliding window for paced SMTP auth failures. Default 360; maximum one week."},
{YAMLPath: "mail_bruteforce_threshold", Type: "int", Label: "Mail bruteforce threshold", Min: int64p(1), FieldGroup: FieldGroupMailBruteForce},
{YAMLPath: "mail_bruteforce_window_min", Type: "int", Label: "Mail bruteforce window (min)", Min: int64p(1), FieldGroup: FieldGroupMailBruteForce},
{YAMLPath: "mail_bruteforce_suppress_min", Type: "int", Label: "Mail bruteforce suppress (min)", Min: int64p(1), FieldGroup: FieldGroupMailBruteForce},
{YAMLPath: "mail_bruteforce_subnet_threshold", Type: "int", Label: "Mail bruteforce /24 threshold", Min: int64p(1), FieldGroup: FieldGroupMailBruteForce},
{YAMLPath: "mail_account_spray_threshold", Type: "int", Label: "Mail account spray threshold", Min: int64p(1), FieldGroup: FieldGroupAccountSpray},
{YAMLPath: "mail_bruteforce_max_tracked", Type: "int", Label: "Mail bruteforce max tracked", Min: int64p(100), FieldGroup: FieldGroupStateRetention},
{YAMLPath: "mail_bruteforce_slow_threshold", Type: "int", Label: "Mail slow bruteforce threshold", Min: int64p(0), Max: int64p(int64(config.SlowBruteMaxThreshold)), FieldGroup: FieldGroupMailBruteForce, Help: "Failed auths from one IP across at least three mailboxes inside the slow window; 10 distinct mailboxes trigger on breadth alone. Default 40; 0 disables; nonzero values must be at least 3."},
{YAMLPath: "mail_bruteforce_slow_window_min", Type: "int", Label: "Mail slow bruteforce window (min)", Min: int64p(1), Max: int64p(int64(config.SlowBruteMaxWindowMin)), FieldGroup: FieldGroupMailBruteForce, Help: "Long-horizon sliding window for paced IMAP, POP3, and ManageSieve auth failures. Default 360; maximum one week."},
{YAMLPath: "mail_spray_mode", Type: "enum", Label: "Mail login spray mode", Options: []string{"off", "observe", "enforce"}, FieldGroup: FieldGroupMailBruteForce, Help: "Long-memory SMTP and IMAP/POP3 login spray rules. observe (default) reports findings without blocking; enforce blocks; off stops the rules but still records failures and clean logins."},
{YAMLPath: "mail_spray_window_min", Type: "int", Label: "Mail login spray window (min)", Min: int64p(60), Max: int64p(int64(config.SlowBruteMaxWindowMin)), FieldGroup: FieldGroupMailBruteForce, Help: "How long failed and clean logins are remembered per IP. Default 10080 (one week)."},
{YAMLPath: "mail_spray_breadth_mailboxes", Type: "int", Label: "Mail login spray mailboxes", Min: int64p(0), Max: int64p(10), FieldGroup: FieldGroupMailBruteForce, Help: "An IP is flagged once it has failed on this many mailboxes, the current one included, not counting those it has logged in to cleanly. Default 2; 0 disables; nonzero values must be at least 2."},
{YAMLPath: "mail_spray_missing_networks", Type: "int", Label: "Mail login spray missing-name networks", Min: int64p(0), Max: int64p(50), FieldGroup: FieldGroupMailBruteForce, Help: "A failure on a non-existent mailbox is flagged once this many networks, the current one included, have tried that mailbox. Default 3; 0 disables."},
{YAMLPath: "mail_spray_block_expiry", Type: "string", Label: "Mail login spray block length", Placeholder: config.DefaultMailSprayBlockExpiry, FieldGroup: FieldGroupMailBruteForce, Help: fmt.Sprintf("How long an enforce mode spray block lasts, from 24h to 2160h. Default %s (30 days). Other blocks keep the auto-response block expiry.", config.DefaultMailSprayBlockExpiry)},
{YAMLPath: "mail_brute_account_key", Type: "string", Label: "Mail brute account-key extractor", FieldGroup: FieldGroupMailBruteForce, Placeholder: "builtin:dovecot-user", Help: "How to derive the account from a dovecot/postfix log line: builtin:dovecot-user (default), builtin:postfix-sasl, or regex:<pattern> where group 1 is the account."},
},
},
{
ID: "mail_logs",
Title: "Mail logs",
YAMLPath: "mail_logs",
Icon: "mail",
Group: SectionGroupDetection,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "source", Type: "enum", Label: "Log source", Options: []string{"auto", "file", "journal"}, Help: "auto: try platform default file then fall back to journal. file: require log file. journal: read systemd-journald (needs journal build tag)."},
{YAMLPath: "file", Type: "string", Label: "Log file override", Placeholder: "/var/log/maillog", Help: "Override the platform-default file path. Leave blank to keep the default.", FileOnly: true},
{YAMLPath: "units", Type: "[]string", Label: "Journal units", Help: "Systemd units to match when source=journal. One per line (e.g. postfix, dovecot)."},
},
},
{
ID: "stalwart",
Title: "Stalwart mail server",
YAMLPath: "stalwart",
Icon: "mail",
Group: SectionGroupDetection,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "log_dir", Type: "string", Label: "Log directory", Placeholder: "/var/log/stalwart", Help: "Where Stalwart writes its daily log files.", FileOnly: true},
{YAMLPath: "log_prefix", Type: "string", Label: "Log file prefix", Placeholder: "stalwart.log", Help: "Name of the daily log files before the date suffix."},
{YAMLPath: "ports", Type: "[]string", Label: "Listener ports", Help: "Listener ports CSM cannot map by default, as PORT:PROTOCOL with protocol smtp, imap, pop3, sieve or http. One per line (e.g. 10465:smtp)."},
{YAMLPath: "api_url", Type: "string", Label: "Admin API address", Placeholder: "http://127.0.0.1:8080", Help: "Where csm stalwart setup reaches Stalwart's admin API. The login comes from the environment, never from this file.", FileOnly: true},
{YAMLPath: "api_key_file", Type: "string", Label: "API key file", Placeholder: "/etc/csm/stalwart-api.key", Help: "File holding the API key CSM acts on Stalwart with. It must be a regular file owned by root with no group or other access.", FileOnly: true},
},
},
{
ID: "suppressions",
Title: "Suppressions",
YAMLPath: "suppressions",
Icon: "volume-off",
Group: SectionGroupAlerting,
Restart: false,
Fields: []SettingsField{
{YAMLPath: "upcp_window_start", Type: "string", Label: "UPCP window start (HH:MM)"},
{YAMLPath: "upcp_window_end", Type: "string", Label: "UPCP window end (HH:MM)"},
{YAMLPath: "known_api_tokens", Type: "[]string", Label: "Known API tokens (hashed)", Feature: "cpanel"},
{YAMLPath: "ignore_paths", Type: "[]string", Label: "Ignore paths"},
{YAMLPath: "suppress_webmail_alerts", Type: "bool", Label: "Suppress webmail login alerts", Feature: "cpanel"},
{YAMLPath: "suppress_cpanel_login_alerts", Type: "bool", Label: "Suppress cPanel login alerts", Feature: "cpanel"},
{YAMLPath: "trusted_countries", Type: "[]string", Label: "Trusted countries (ISO 3166-1 alpha-2)"},
},
},
{
ID: "auto_response",
Title: "Auto-Response",
YAMLPath: "auto_response",
Icon: "bolt",
Group: SectionGroupDetection,
Restart: false,
Fields: []SettingsField{
{YAMLPath: "enabled", Type: "bool", Label: "Auto-response enabled"},
{YAMLPath: "kill_processes", Type: "bool", Label: "Kill malicious processes"},
{YAMLPath: "quarantine_files", Type: "bool", Label: "Quarantine malicious files"},
{YAMLPath: "block_ips", Type: "bool", Label: "Block attacker IPs"},
{YAMLPath: "block_expiry", Type: "string", Label: "Block expiry", Placeholder: config.DefaultBlockExpiry, Help: fmt.Sprintf("Positive temporary-block duration. Default %s. Admission previews and dry-run forecasts cap new automatic lifetimes at this duration. Retries keep their first expiry. Existing blocking keeps its own lifetimes.", config.DefaultBlockExpiry)},
{YAMLPath: "max_blocks_per_hour", Type: "int", Label: "Max IP blocks per hour", Min: int64p(0), Help: fmt.Sprintf("Hourly budget for scan-driven IP blocks; challenge, central and incident blocks are not limited. 0 uses the defaults: %d for scan-driven blocks and %d for the admission ledger, which caps any value at %d.", config.DefaultMaxBlocksPerHour, config.DefaultAdmissionCeiling, admission.MaxCeiling)},
{YAMLPath: "enforce_permissions", Type: "bool", Label: "Auto-chmod 644 world/group-writable PHP"},
{YAMLPath: "fix_wp_cron", Type: "bool", Label: "Auto-disable WP-Cron + install system cron", Help: "On perf_wp_cron findings, edit wp-config.php and add a per-user cron. Tune interval/php under Performance."},
{YAMLPath: "block_cpanel_logins", Type: "bool", Label: "Block on cPanel/webmail login alerts", Feature: "cpanel"},
{YAMLPath: "http_scanner_action", Type: "enum", Label: "URL scanner response", Options: []string{"challenge", "block"}, Help: "Response to http_scanner_profile findings. challenge routes the IP to the PoW challenge when the challenge subsystem is enabled, falling back to a block when it is not; block always hard-blocks."},
{YAMLPath: "netblock", Type: "bool", Label: "Auto-block /24 on threshold"},
{YAMLPath: "netblock_threshold", Type: "int", Label: "Netblock threshold", Min: int64p(config.MinBlockEscalationCount), Help: "Blocked addresses in one IPv4 /24 or IPv6 /64 before the subnet itself is blocked. Default 3."},
{YAMLPath: "netblock_window", Type: "string", Label: "Netblock window", Placeholder: "168h", Help: "Positive window in which blocked addresses count toward the netblock threshold, including blocks that already expired. Default 168h."},
{YAMLPath: "permblock", Type: "bool", Label: "Auto-promote to permanent"},
{YAMLPath: "permblock_count", Type: "int", Label: "Temp blocks before permanent", Min: int64p(config.MinBlockEscalationCount), Help: "Temporary blocks inside the window before the address is blocked permanently. Default 4."},
{YAMLPath: "permblock_interval", Type: "string", Label: "Permblock window", Placeholder: "24h", Help: "Positive window for counting temporary blocks. Default 24h."},
{YAMLPath: "clean_database", Type: "bool", Label: "Auto-clean DB injections"},
{YAMLPath: "virtual_patch_exposed_files", Type: "enum", Label: "Virtual-patch web-exposed files", Options: []string{"off", "manual", "auto"}, Help: "Write .htaccess 'Require all denied' rules for confirmed web-exposed files. off: detect only. manual: apply via `csm virtual-patch`. auto: apply each scan except warning-only sample SQL (gated by dry_run)."},
{YAMLPath: "dry_run", Type: "bool", Label: "Auto-response dry run", Help: "Preview automatic IP blocks and web-exposed-file virtual patches without enforcing them."},
{YAMLPath: "verdict_callback.enabled", Type: "bool", Label: "Verdict callback hook"},
{YAMLPath: "verdict_callback.url", Type: "string", Label: "Verdict callback URL"},
{YAMLPath: "verdict_callback.hmac_secret", Type: "string", Label: "Verdict callback HMAC secret", Secret: true},
{YAMLPath: "verdict_callback.hmac_secret_env", Type: "string", Label: "Verdict callback HMAC secret env", FileOnly: true},
{YAMLPath: "verdict_callback.allow_unsigned", Type: "bool", Label: "Allow unsigned verdict callback"},
{YAMLPath: "verdict_callback.require_response_signature", Type: "bool", Label: "Require signed verdict response"},
{YAMLPath: "verdict_callback.timeout_sec", Type: "int", Label: "Verdict callback timeout (sec)", Min: int64p(1), Max: int64p(30)},
},
},
{
ID: "reputation",
Title: "Reputation",
YAMLPath: "reputation",
Icon: "shield-check",
Group: SectionGroupIntegrations,
Restart: false,
Fields: []SettingsField{
{YAMLPath: "abuseipdb_key", Type: "string", Label: "AbuseIPDB API key", Secret: true},
{YAMLPath: "whitelist", Type: "[]string", Label: "Whitelisted IPs", Help: "Never flagged as malicious"},
{YAMLPath: "bot_verify_enabled", Type: "bool", Label: "Verify search-engine bots via rDNS", Nullable: true},
{YAMLPath: "rspamd.enabled", Type: "bool", Label: "Rspamd threat-intel"},
// #nosec G101 -- names of the config fields that hold the credential, not a credential.
{YAMLPath: "rspamd.url", Type: "string", Label: "Rspamd controller URL", CredentialField: "rspamd.token", CredentialEnvField: "rspamd.token_env"},
{YAMLPath: "rspamd.token", Type: "string", Label: "Rspamd controller password", Secret: true},
{YAMLPath: "rspamd.token_env", Type: "string", Label: "Rspamd password env var", FileOnly: true},
{YAMLPath: "upstream.enabled", Type: "bool", Label: "Upstream threat-intel cache"},
// #nosec G101 -- names of the config fields that hold the credential, not a credential.
{YAMLPath: "upstream.url", Type: "string", Label: "Upstream URL", CredentialField: "upstream.token", CredentialEnvField: "upstream.token_env"},
{YAMLPath: "upstream.token", Type: "string", Label: "Upstream bearer token", Secret: true},
{YAMLPath: "upstream.token_env", Type: "string", Label: "Upstream token env var", FileOnly: true},
{YAMLPath: "upstream.cache_ttl_min", Type: "int", Label: "Upstream cache TTL (min)", Min: int64p(1), Max: int64p(1440)},
{YAMLPath: "upstream.timeout_sec", Type: "int", Label: "Upstream request timeout (sec)", Min: int64p(1), Max: int64p(60)},
{YAMLPath: "report.enabled", Type: "bool", Label: "Report confirmed abuse", Help: "Send signed, minimized reports for confirmed-abuse IPs. Targets are configured in YAML.", FieldGroup: FieldGroupAbuseReporting},
{YAMLPath: "report.classes", Type: "[]string", Label: "Reported classes", Help: "bruteforce, php_relay, credential_stuffing, bad_asn_egress, spam", FieldGroup: FieldGroupAbuseReporting},
{YAMLPath: "report.spool_max", Type: "int", Label: "Report spool size", Min: int64p(0), FieldGroup: FieldGroupAbuseReporting},
{YAMLPath: "central.enabled", Type: "bool", Label: "Consume central abuse database", FieldGroup: FieldGroupCentralDB},
{YAMLPath: "central.set_url", Type: "string", Label: "Scored-set URL", FieldGroup: FieldGroupCentralDB},
{YAMLPath: "central.pubkey_env", Type: "string", Label: "Central public-key env var", FieldGroup: FieldGroupCentralDB, FileOnly: true},
{YAMLPath: "central.auth_target", Type: "string", Label: "Signing report target", Help: "Name of the report target whose node key signs downloads; required by the central abuse database", FieldGroup: FieldGroupCentralDB},
{YAMLPath: "central.action", Type: "string", Label: "Action on listed IPs", Help: "off | challenge | block_if_local_corroborated", FieldGroup: FieldGroupCentralDB},
{YAMLPath: "central.block_threshold", Type: "int", Label: "Block score threshold", Min: int64p(0), Max: int64p(100), FieldGroup: FieldGroupCentralDB},
{YAMLPath: "central.refresh_interval", Type: "string", Label: "Refresh interval", Help: "e.g. 6h", FieldGroup: FieldGroupCentralDB},
},
},
{
ID: "email_protection",
Title: "Email Protection",
YAMLPath: "email_protection",
Icon: "mail-shield",
Group: SectionGroupDetection,
Restart: false,
Fields: []SettingsField{
{YAMLPath: "password_check_interval_min", Type: "int", Label: "Password check interval (min)", Min: int64p(1)},
{YAMLPath: "high_volume_senders", Type: "[]string", Label: "High-volume senders"},
{YAMLPath: "rate_warn_threshold", Type: "int", Label: "Rate warn threshold", Min: int64p(1)},
{YAMLPath: "rate_crit_threshold", Type: "int", Label: "Rate critical threshold", Min: int64p(1)},
{YAMLPath: "rate_window_min", Type: "int", Label: "Rate window (min)", Min: int64p(1)},
{YAMLPath: "known_forwarders", Type: "[]string", Label: "Known forwarders"},
},
},
{
ID: "challenge",
Title: "Challenge",
YAMLPath: "challenge",
Icon: "user-question",
Group: SectionGroupDetection,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "enabled", Type: "bool", Label: "Challenge pages enabled"},
{YAMLPath: "listen_port", Type: "int", Label: "Listen port", Min: int64p(1), Max: int64p(65535)},
{YAMLPath: "difficulty", Type: "int", Label: "PoW difficulty (0-5)", Min: int64p(0), Max: int64p(5)},
{YAMLPath: "trusted_proxies", Type: "[]string", Label: "Trusted proxy IPs"},
// challenge.secret is auto-generated at daemon startup; intentionally
// omitted so the UI cannot overwrite or leak the HMAC key.
},
},
{
ID: "php_shield",
Title: "PHP Shield",
YAMLPath: "php_shield",
Icon: "brand-php",
Group: SectionGroupDetection,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "enabled", Type: "bool", Label: "PHP Shield enabled"},
},
},
{
ID: "detection",
Title: "Detection",
YAMLPath: "detection",
Icon: "radar",
Group: SectionGroupDetection,
Restart: false,
Fields: []SettingsField{
{YAMLPath: "placement_backend", Type: "enum", Label: "Moved and linked file backend", Options: []string{"auto", "bpf", "legacy", "none"}, Help: "How realtime protection sees files renamed or hard-linked into a site. auto tries BPF, then fanotify (kernel 5.9 or newer), then none. bpf requires BPF. legacy uses fanotify only. none leaves those files to the deep scan. Takes effect after a restart."},
},
},
{
ID: "signatures",
Title: "Signatures",
YAMLPath: "signatures",
Icon: "scan",
Group: SectionGroupDetection,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "auto_update", Type: "bool", Label: "Auto-update rules"},
{YAMLPath: "update_interval", Type: "string", Label: "Update interval", Placeholder: "24h"},
{YAMLPath: "yara_forge.enabled", Type: "bool", Label: "YARA-Forge enabled"},
{YAMLPath: "yara_forge.tier", Type: "enum", Label: "YARA-Forge tier", Options: []string{"core", "extended", "full"}},
{YAMLPath: "yara_forge.update_interval", Type: "string", Label: "YARA-Forge interval", Placeholder: "168h"},
{YAMLPath: "yara_forge.download_url", Type: "string", Label: "YARA-Forge signed ZIP URL"},
{YAMLPath: "disabled_rules", Type: "[]string", Label: "Disabled rule names"},
{YAMLPath: "yara_worker_enabled", Type: "bool", Label: "Run YARA-X in a locked-down worker"},
},
},
{
ID: "email_av",
Title: "Email AV",
YAMLPath: "email_av",
Feature: "emailAV",
Icon: "virus",
Group: SectionGroupDetection,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "enabled", Type: "bool", Label: "Email AV enabled"},
{YAMLPath: "clamd_socket", Type: "string", Label: "clamd socket", FileOnly: true},
{YAMLPath: "scan_timeout", Type: "string", Label: "Scan timeout", Placeholder: "30s"},
{YAMLPath: "max_attachment_size", Type: "int", Label: "Max attachment bytes", Min: int64p(1024)},
{YAMLPath: "max_archive_depth", Type: "int", Label: "Max archive depth", Min: int64p(0)},
{YAMLPath: "max_archive_files", Type: "int", Label: "Max archive files", Min: int64p(1)},
{YAMLPath: "max_extraction_size", Type: "int", Label: "Max extraction bytes", Min: int64p(1024)},
{YAMLPath: "quarantine_infected", Type: "bool", Label: "Quarantine infected"},
{YAMLPath: "scan_concurrency", Type: "int", Label: "Scan concurrency", Min: int64p(1), Max: int64p(64)},
},
},
{
ID: "modsec",
Title: "ModSecurity",
YAMLPath: "modsec",
Icon: "shield-lock",
Group: SectionGroupDetection,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "rules_file", Type: "string", Label: "Rules file path", FileOnly: true},
{YAMLPath: "overrides_file", Type: "string", Label: "Overrides file path", FileOnly: true},
{YAMLPath: "reload_command", Type: "string", Label: "Reload command", FileOnly: true},
},
},
{
ID: "performance",
Title: "Performance",
YAMLPath: "performance",
Icon: "activity",
Group: SectionGroupOps,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "enabled", Type: "bool", Label: "Performance checks", Nullable: true, Help: "Leave unset to inherit default (on)"},
{YAMLPath: "load_high_multiplier", Type: "float", Label: "Load high multiplier"},
{YAMLPath: "load_critical_multiplier", Type: "float", Label: "Load critical multiplier"},
{YAMLPath: "php_process_warn_per_user", Type: "int", Label: "PHP process warn per user", Min: int64p(1)},
{YAMLPath: "php_process_critical_total_multiplier", Type: "int", Label: "PHP process crit multiplier", Min: int64p(1)},
{YAMLPath: "error_log_warn_size_mb", Type: "int", Label: "Error log warn size (MB)", Min: int64p(1)},
{YAMLPath: "error_log_critical_size_mb", Type: "int", Label: "Error log critical size (MB)", Min: int64p(1), Help: "A bloated error_log over this size is High and alerts; smaller ones only show on the Performance page. Default 1024."},
{YAMLPath: "mysql_join_buffer_max_mb", Type: "int", Label: "MySQL join buffer max (MB)", Min: int64p(1)},
{YAMLPath: "mysql_wait_timeout_max", Type: "int", Label: "MySQL wait timeout max (s)", Min: int64p(1)},
{YAMLPath: "mysql_max_connections_per_user", Type: "int", Label: "MySQL max connections per user", Min: int64p(1)},
{YAMLPath: "redis_bgsave_min_interval", Type: "int", Label: "Redis bgsave min interval (s)", Min: int64p(1)},
{YAMLPath: "redis_large_dataset_gb", Type: "int", Label: "Redis large dataset (GB)", Min: int64p(1)},
{YAMLPath: "wp_memory_limit_max_mb", Type: "int", Label: "WP memory limit max (MB)", Min: int64p(32)},
{YAMLPath: "wp_transient_warn_mb", Type: "int", Label: "WP transient warn (MB)", Min: int64p(1)},
{YAMLPath: "wp_transient_critical_mb", Type: "int", Label: "WP transient critical (MB)", Min: int64p(1)},
{YAMLPath: "wp_cron_fix.interval_minutes", Type: "int", Label: "WP-Cron fix: system cron interval (min)", Min: int64p(1), Max: int64p(60), Help: "How often the installed system cron runs wp-cron.php. Only bounds task latency; WordPress keeps its own event schedule. Default 15."},
{YAMLPath: "wp_cron_fix.php_bin", Type: "string", Label: "WP-Cron fix: PHP binary", Placeholder: "/usr/local/bin/php", Help: "Overrides the cron interpreter for every site. Leave empty to use the unambiguous cPanel MultiPHP version, then auto-detect.", FileOnly: true},
},
},
{
ID: "cloudflare",
Title: "Cloudflare",
YAMLPath: "cloudflare",
Icon: "cloud",
Group: SectionGroupIntegrations,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "enabled", Type: "bool", Label: "Cloudflare integration"},
{YAMLPath: "refresh_hours", Type: "int", Label: "Refresh interval (hours)", Min: int64p(1), Max: int64p(168)},
},
},
{
ID: "geoip",
Title: "GeoIP",
YAMLPath: "geoip",
Icon: "world",
Group: SectionGroupIntegrations,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "account_id", Type: "string", Label: "MaxMind account ID"},
{YAMLPath: "license_key", Type: "string", Label: "MaxMind license key", Secret: true},
{YAMLPath: "editions", Type: "[]enum", Label: "Database editions", OptionsSource: "geoip_editions", Help: "Which MaxMind databases to download. GeoLite2-* are free; GeoIP2-* require a paid subscription."},
{YAMLPath: "auto_update", Type: "bool", Label: "Auto-update databases", Nullable: true},
{YAMLPath: "update_interval", Type: "string", Label: "Update interval", Placeholder: "24h"},
},
},
{
ID: "infra_ips",
Title: "Infra IPs",
YAMLPath: "infra_ips",
Icon: "server",
Group: SectionGroupOps,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "", Type: "[]string", Label: "Trusted infra IPs and CIDRs"},
},
},
{
ID: "disabled_checks",
Title: "Disabled checks",
YAMLPath: "disabled_checks",
Icon: "x-circle",
Group: SectionGroupOps,
Restart: false,
Fields: []SettingsField{
{YAMLPath: "", Type: "[]enum", Label: "Skip scheduled check runners", OptionsSource: "disabled_check_names", Help: "Selecting a finding name skips the scheduled check runner or runners that emit it, including sibling findings from the same runner. Realtime findings are not affected. For email-only suppression, use Alerts > Disabled check names instead."},
},
},
{
ID: "sentry",
Title: "Sentry",
YAMLPath: "sentry",
Icon: "bug",
Group: SectionGroupIntegrations,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "enabled", Type: "bool", Label: "Sentry enabled"},
{YAMLPath: "dsn", Type: "string", Label: "Sentry DSN", Secret: true},
{YAMLPath: "environment", Type: "string", Label: "Environment", Placeholder: "production"},
{YAMLPath: "sample_rate", Type: "float", Label: "Sample rate (0 to 1.0)"},
{YAMLPath: "debug", Type: "bool", Label: "Debug logging"},
},
},
{
ID: "firewall",
Title: "Firewall",
YAMLPath: "firewall",
Icon: "shield-lock",
Group: SectionGroupFirewall,
Restart: true,
Fields: []SettingsField{
{YAMLPath: "enabled", Type: "bool", Label: "Firewall enabled", Help: "Activates the nftables-based firewall on next daemon restart. Verify port lists below before enabling to avoid lockout.", FieldGroup: FieldGroupAccessPorts},
{YAMLPath: "ipv6", Type: "bool", Label: "IPv6 dual-stack", FieldGroup: FieldGroupIPv6},
{YAMLPath: "tcp_in", Type: "[]int", Label: "Inbound TCP ports", Help: "SSH (22) is intentionally not in the default; add it explicitly if sshd listens on 22. WebUI port must be present or you will lose remote access on restart.", FieldGroup: FieldGroupAccessPorts},
{YAMLPath: "tcp_out", Type: "[]int", Label: "Outbound TCP ports", FieldGroup: FieldGroupAccessPorts},
{YAMLPath: "udp_in", Type: "[]int", Label: "Inbound UDP ports", FieldGroup: FieldGroupAccessPorts},
{YAMLPath: "udp_out", Type: "[]int", Label: "Outbound UDP ports", Help: "Includes 6277/24441 by default for SpamAssassin DCC/Pyzor; do not remove unless rspamd-only.", FieldGroup: FieldGroupAccessPorts},
{YAMLPath: "tcp6_in", Type: "[]int", Label: "Inbound TCP6 ports", Help: "Empty inherits tcp_in. Only set when IPv6 should differ from IPv4.", FieldGroup: FieldGroupIPv6},
{YAMLPath: "tcp6_out", Type: "[]int", Label: "Outbound TCP6 ports", Help: "Empty inherits tcp_out.", FieldGroup: FieldGroupIPv6},
{YAMLPath: "udp6_in", Type: "[]int", Label: "Inbound UDP6 ports", Help: "Empty inherits udp_in.", FieldGroup: FieldGroupIPv6},
{YAMLPath: "udp6_out", Type: "[]int", Label: "Outbound UDP6 ports", Help: "Empty inherits udp_out.", FieldGroup: FieldGroupIPv6},
{YAMLPath: "restricted_tcp", Type: "[]int", Label: "Restricted TCP (infra-only)", Help: "Reachable only from infra_ips. Manage infra_ips in its own section.", FieldGroup: FieldGroupAccessPorts},
{YAMLPath: "required_tcp_out", Type: "[]int", Label: "Required outbound TCP ports", Help: "Checked only, never added to the policy: validation warns when an effective outbound family omits one. Integrations declare the ports they dial in their own conf.d fragment.", FieldGroup: FieldGroupAccessPorts},
{YAMLPath: "passive_ftp_start", Type: "int", Label: "Passive FTP range start", Min: int64p(1024), Max: int64p(65535), FieldGroup: FieldGroupAccessPorts},
{YAMLPath: "passive_ftp_end", Type: "int", Label: "Passive FTP range end", Min: int64p(1024), Max: int64p(65535), FieldGroup: FieldGroupAccessPorts},
{YAMLPath: "drop_nolog", Type: "[]int", Label: "Silent-drop ports", Help: "Dropped without logging to keep scanner noise out of the log.", FieldGroup: FieldGroupLogging},
{YAMLPath: "conn_rate_limit", Type: "int", Label: "Conn rate limit (per IP/min)", Min: int64p(0), Max: int64p(100000), Help: "0 disables. 200 tolerates shared CGNAT egress. IPv6 metered per /64.", FieldGroup: FieldGroupRateLimits},
{YAMLPath: "conn_limit", Type: "int", Label: "Concurrent connections per IPv4", Min: int64p(0), Max: int64p(100000), Help: "0 disables. IPv4 only (no IPv6).", FieldGroup: FieldGroupRateLimits},
{YAMLPath: "syn_flood_protection", Type: "bool", Label: "SYN flood protection", Help: "Dual-stack; IPv6 metered per /64.", FieldGroup: FieldGroupFloodProtection},
{YAMLPath: "udp_flood", Type: "bool", Label: "UDP flood protection", Help: "Dual-stack; IPv6 metered per /64.", FieldGroup: FieldGroupFloodProtection},
{YAMLPath: "udp_flood_rate", Type: "int", Label: "UDP packets/sec", Min: int64p(1), Max: int64p(100000), Help: "Per source; IPv6 metered per /64.", FieldGroup: FieldGroupFloodProtection},
{YAMLPath: "udp_flood_burst", Type: "int", Label: "UDP burst allowance", Min: int64p(1), Max: int64p(1000000), Help: "Per source; IPv6 metered per /64.", FieldGroup: FieldGroupFloodProtection},
{YAMLPath: "deny_ip_limit", Type: "int", Label: "Permanent block cap", Min: int64p(0), Max: int64p(1000000), Help: fmt.Sprintf("0 = unlimited. Default %d.", firewall.DefaultConfig().DenyIPLimit), FieldGroup: FieldGroupLimits},
{YAMLPath: "deny_temp_ip_limit", Type: "int", Label: "Temporary block cap", Min: int64p(0), Max: int64p(1000000), Help: fmt.Sprintf("0 = unlimited for existing blocking; automatic admission then plans for %d slots. Default %d.", admission.TemporaryCapacity, firewall.DefaultConfig().DenyTempIPLimit), FieldGroup: FieldGroupLimits},
{YAMLPath: "country_block", Type: "[]string", Label: "Country block (ISO-3166)", Help: "Two-letter codes, one per line.", FieldGroup: FieldGroupGeoDynDNS},
{YAMLPath: "country_db_path", Type: "string", Label: "Country DB path override", Placeholder: "(default: <state_path>/geoip)", FieldGroup: FieldGroupGeoDynDNS, FileOnly: true},
{YAMLPath: "dyndns_hosts", Type: "[]string", Label: "DynDNS hosts", Help: "Resolved every 5 minutes and merged into the trusted set.", FieldGroup: FieldGroupGeoDynDNS},
{YAMLPath: "smtp_block", Type: "bool", Label: "Block outbound SMTP", Help: "When enabled, only smtp_allow_users may originate outbound mail. Verify allow list first.", FieldGroup: FieldGroupSMTPControls},
{YAMLPath: "smtp_allow_users", Type: "[]string", Label: "SMTP allow users", Help: "root is always allowed.", FieldGroup: FieldGroupSMTPControls},
{YAMLPath: "smtp_ports", Type: "[]int", Label: "SMTP ports", FieldGroup: FieldGroupSMTPControls},
{YAMLPath: "local_port_locks", Type: "[]string", Label: "Local port locks", Help: "PORT:user,user. Only these users and root may open new connections to the port on this host.", FieldGroup: FieldGroupSMTPControls},
{YAMLPath: "log_dropped", Type: "bool", Label: "Log dropped packets", FieldGroup: FieldGroupLogging},
{YAMLPath: "log_rate", Type: "int", Label: "Log entries per minute", Min: int64p(0), Max: int64p(10000), FieldGroup: FieldGroupLogging},
},
},
}
// SettingsSectionIDs returns the ordered list of section IDs.
func SettingsSectionIDs() []string {
out := make([]string, 0, len(settingsSections))
for _, s := range settingsSections {
out = append(out, s.ID)
}
return out
}
// LookupSettingsSection returns the section with the given ID.
func LookupSettingsSection(id string) (SettingsSection, bool) {
for _, s := range settingsSections {
if s.ID == id {
return withReloadPolicy(s), true
}
}
return SettingsSection{}, false
}
// AllSettingsSections returns the list of sections. Intended for
// read-only consumers such as the dashboard navigation.
func AllSettingsSections() []SettingsSection {
out := make([]SettingsSection, len(settingsSections))
for i, s := range settingsSections {
out[i] = withReloadPolicy(s)
}
return out
}
func withReloadPolicy(section SettingsSection) SettingsSection {
for _, policy := range config.HotReloadManifest() {
if policy.Field == section.YAMLPath {
section.Restart = policy.RestartRequired
section.ReloadTag = policy.Tag
break
}
}
return section
}
package webui
import (
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"io"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/pidginhost/csm/internal/integrity"
)
// Static files are linked with a content version (/static/js/app.js?v=...).
// A request carrying the current version may be cached for a year, since an
// upgrade changes the version and so the URL; any other request revalidates.
const (
staticCacheVersioned = "public, max-age=31536000, immutable"
staticCacheOther = "no-cache"
)
// assetVersions caches versions by file identity and change metadata, so an
// upgrade that preserves modification times still gets new URLs.
type assetVersions struct {
mu sync.Mutex
byPath map[string]assetVersion
}
type assetVersion struct {
info os.FileInfo
key string
version string
}
func (a *assetVersions) version(file string) string {
// #nosec G304 -- file is a cleaned path under the UI static directory.
f, err := os.Open(file)
if err != nil {
return ""
}
defer func() { _ = f.Close() }()
return a.versionOf(file, f)
}
func (a *assetVersions) versionOf(file string, f http.File) string {
info, err := f.Stat()
if err != nil || !info.Mode().IsRegular() {
return ""
}
key := integrity.FileChangeKey(info)
a.mu.Lock()
cached, ok := a.byPath[file]
a.mu.Unlock()
if ok && cached.key == key && os.SameFile(cached.info, info) {
return cached.version
}
h := sha256.New()
_, hashErr := io.Copy(h, f)
_, seekErr := f.Seek(0, io.SeekStart)
if hashErr != nil || seekErr != nil {
return ""
}
after, err := f.Stat()
if err != nil || integrity.FileChangeKey(after) != key || !os.SameFile(info, after) {
return ""
}
v := hex.EncodeToString(h.Sum(nil))[:12]
a.mu.Lock()
if a.byPath == nil {
a.byPath = map[string]assetVersion{}
}
a.byPath[file] = assetVersion{info: info, key: key, version: v}
a.mu.Unlock()
return v
}
// staticFile maps a URL path below /static/ to a file in dir. path.Clean
// keeps it inside dir.
func staticFile(dir, rel string) string {
return filepath.Join(dir, filepath.FromSlash(path.Clean("/"+rel)))
}
// assetURL is the versioned URL of a file below the static directory, for
// templates. A file that cannot be read gets the plain URL.
func (s *Server) assetURL(rel string) string {
u := "/static/" + strings.TrimPrefix(rel, "/")
if v := s.assets.version(staticFile(s.staticDir, rel)); v != "" {
u += "?v=" + v
}
return u
}
// gzipTypes are the static file types worth compressing; fonts in woff2 and
// images are compressed already.
var gzipTypes = map[string]bool{".js": true, ".css": true, ".svg": true, ".json": true, ".map": true, ".ttf": true}
// Hash and serve the same open file. A pathname can be replaced by an upgrade
// between checking its version and opening it in FileServer.
type versionedStaticFS struct {
http.FileSystem
dir string
versions *assetVersions
requested string
header http.Header
}
func (fs versionedStaticFS) Open(name string) (http.File, error) {
f, err := fs.FileSystem.Open(name)
if err == nil && fs.requested != "" && fs.requested == fs.versions.versionOf(staticFile(fs.dir, name), f) {
fs.header.Set("Cache-Control", staticCacheVersioned)
}
return f, err
}
func (s *Server) staticHandler(dir string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rel := strings.TrimPrefix(r.URL.Path, "/static/")
w.Header().Set("Cache-Control", staticCacheOther)
files := http.StripPrefix("/static/", http.FileServer(versionedStaticFS{
FileSystem: noListDir{http.Dir(dir)}, dir: dir, versions: &s.assets,
requested: r.URL.Query().Get("v"), header: w.Header(),
}))
if !gzipTypes[strings.ToLower(path.Ext(rel))] {
files.ServeHTTP(w, r)
return
}
w.Header().Add("Vary", "Accept-Encoding")
// A range is a byte range of the file; it cannot be served encoded.
if r.Header.Get("Range") != "" || !acceptsGzip(r) {
files.ServeHTTP(w, r)
return
}
gw := &gzipResponseWriter{ResponseWriter: w, head: r.Method == http.MethodHead}
defer gw.close()
files.ServeHTTP(gw, r)
})
}
func acceptsGzip(r *http.Request) bool {
wildcard := false
for _, part := range strings.Split(strings.Join(r.Header.Values("Accept-Encoding"), ","), ",") {
params := strings.Split(part, ";")
enc := strings.TrimSpace(params[0])
quality := 1.0
for _, param := range params[1:] {
name, value, _ := strings.Cut(strings.TrimSpace(param), "=")
if strings.EqualFold(strings.TrimSpace(name), "q") {
var err error
quality, err = strconv.ParseFloat(strings.TrimSpace(value), 64)
if err != nil {
quality = 0
}
}
}
accepted := quality > 0 && quality <= 1
if strings.EqualFold(enc, "gzip") {
return accepted
}
if enc == "*" {
wildcard = accepted
}
}
return wildcard
}
// gzipResponseWriter compresses a 200 response body. Other statuses (304,
// errors) pass through untouched.
type gzipResponseWriter struct {
http.ResponseWriter
zw *gzip.Writer
head bool
wroteHeader bool
}
func (g *gzipResponseWriter) WriteHeader(code int) {
if g.wroteHeader {
return
}
g.wroteHeader = true
if code == http.StatusOK && g.Header().Get("Content-Encoding") == "" {
g.Header().Del("Content-Length")
g.Header().Set("Content-Encoding", "gzip")
if !g.head {
g.zw = gzip.NewWriter(g.ResponseWriter)
}
}
g.ResponseWriter.WriteHeader(code)
}
func (g *gzipResponseWriter) Write(p []byte) (int, error) {
if !g.wroteHeader {
g.WriteHeader(http.StatusOK)
}
if g.head {
return len(p), nil
}
if g.zw != nil {
return g.zw.Write(p)
}
return g.ResponseWriter.Write(p)
}
func (g *gzipResponseWriter) close() {
if g.zw != nil {
_ = g.zw.Close()
}
}
package webui
import (
"sort"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/checks"
)
// statsSummary is what the dashboard shows about the last 24 hours of
// history. The stats API and the dashboard page share it; it is recomputed
// only when history changes (see historyMemo).
type statsSummary struct {
critical, high, warning int
byCheck map[string]int
lastCritical time.Time
autoBlocked, autoQuarantined, autoKilled int
topAccounts []accountCount
bruteForce map[string]interface{}
recent []alert.Finding // newest non-internal findings, at most 10
}
type accountCount struct {
Account string `json:"account"`
Count int `json:"count"`
}
type atRiskAccount struct {
Account string `json:"account"`
Severity string `json:"severity"`
Findings int `json:"findings"`
}
func (s *Server) statsSummary24h() *statsSummary {
return s.statsMemo.get(s.store.HistoryMark(), func() any {
return summarizeHistory(s.store.ReadHistorySince(time.Now().Add(-24 * time.Hour)))
}).(*statsSummary)
}
// summarizeHistory computes the summary from findings, newest first.
func summarizeHistory(findings []alert.Finding) *statsSummary {
critical, high, warning := 0, 0, 0
byCheck := make(map[string]int)
for _, f := range findings {
switch f.Severity {
case alert.Critical:
critical++
case alert.High:
high++
case alert.Warning:
warning++
}
byCheck[f.Check]++
}
// Most recent critical finding, for "time since last critical"
// (findings are newest-first from ReadHistorySince).
var lastCritical time.Time
for _, f := range findings {
if f.Severity == alert.Critical {
lastCritical = f.Timestamp
break
}
}
// The dashboard's live feed: the newest findings, without internal checks.
var recent []alert.Finding
for _, f := range findings {
if len(recent) == 10 {
break
}
if !operatorFacingCheck(f.Check) {
continue
}
recent = append(recent, f)
}
// Auto-response summary: count actions by type in 24h
autoBlocked, autoQuarantined, autoKilled := 0, 0, 0
// Top targeted accounts
accountHits := make(map[string]int)
// Brute force summary
bruteForceIPs := make(map[string]int) // IP -> total attempts
bruteForceTypes := make(map[string]int) // "wp-login" / "xmlrpc" -> count
for _, f := range findings {
// Extract account from finding path/message
acct := extractAccountFromFinding(f)
if acct != "" {
accountHits[acct]++
}
// Count auto-response actions
switch f.Check {
case "auto_block":
autoBlocked++
case "auto_response":
if strings.Contains(f.Message, "quarantin") {
autoQuarantined++
} else if strings.Contains(f.Message, "kill") || strings.Contains(f.Message, "Kill") {
autoKilled++
}
case "wp_login_bruteforce":
bruteForceTypes["wp-login"]++
if ip := checks.ExtractIPFromFinding(f); ip != "" {
bruteForceIPs[ip]++
}
case "xmlrpc_abuse":
bruteForceTypes["xmlrpc"]++
if ip := checks.ExtractIPFromFinding(f); ip != "" {
bruteForceIPs[ip]++
}
case "modsec_block_escalation", "modsec_csm_block_escalation":
if strings.Contains(f.Message, "xmlrpc") || strings.Contains(f.Message, "900006") || strings.Contains(f.Message, "900007") {
bruteForceTypes["xmlrpc-modsec"]++
}
}
}
// Top targeted accounts (by finding count)
var topAccounts []accountCount
for acct, count := range accountHits {
topAccounts = append(topAccounts, accountCount{acct, count})
}
sort.Slice(topAccounts, func(i, j int) bool {
return topAccounts[i].Count > topAccounts[j].Count
})
if len(topAccounts) > 5 {
topAccounts = topAccounts[:5]
}
return &statsSummary{
critical: critical,
high: high,
warning: warning,
byCheck: byCheck,
lastCritical: lastCritical,
autoBlocked: autoBlocked,
autoQuarantined: autoQuarantined,
autoKilled: autoKilled,
topAccounts: topAccounts,
bruteForce: buildBruteForceSummary(bruteForceIPs, bruteForceTypes),
recent: recent,
}
}
// accountsAtRisk lists the accounts with an active critical or high finding,
// worst first. It reads the latest findings the way the Findings page does,
// not history, so a fixed or dismissed finding drops off at once.
func (s *Server) accountsAtRisk() []atRiskAccount {
type risk struct {
account string
worst alert.Severity
findings int
}
byAccount := make(map[string]*risk)
suppressions := s.store.LoadSuppressions()
for _, f := range s.store.LatestFindings() {
if !operatorFacingCheck(f.Check) || s.store.IsSuppressed(f, suppressions) {
continue
}
acct := extractAccountFromFinding(f)
if acct == "" {
continue
}
r := byAccount[acct]
if r == nil {
r = &risk{account: acct, worst: f.Severity}
byAccount[acct] = r
}
r.findings++
if f.Severity > r.worst {
r.worst = f.Severity
}
}
var list []*risk
for _, r := range byAccount {
if r.worst >= alert.High {
list = append(list, r)
}
}
sort.Slice(list, func(i, j int) bool {
if list[i].worst != list[j].worst {
return list[i].worst > list[j].worst
}
if list[i].findings != list[j].findings {
return list[i].findings > list[j].findings
}
return list[i].account < list[j].account
})
if len(list) > 50 {
list = list[:50]
}
out := make([]atRiskAccount, 0, len(list))
for _, r := range list {
out = append(out, atRiskAccount{Account: r.account, Severity: r.worst.String(), Findings: r.findings})
}
return out
}
package webui
import "fmt"
// subnetCoverer is implemented by the firewall engine: it reports the
// blocked subnet, if any, that still covers an address.
type subnetCoverer interface {
BlockedSubnetCovering(ip string) (string, bool)
}
// coveringSubnetWarning explains why an unblocked or allowed address may
// still be dropped: the chain drops @blocked_nets before it accepts
// @allowed_ips, so a covering blocked subnet overrides both actions. Empty
// when nothing covers the address or the blocker cannot tell.
func coveringSubnetWarning(blocker IPBlocker, ip string) string {
cov, ok := blocker.(subnetCoverer)
if !ok {
return ""
}
cidr, covered := cov.BlockedSubnetCovering(ip)
if !covered {
return ""
}
return fmt.Sprintf("still dropped by blocked subnet %s; unblock that subnet to restore access", cidr)
}
package webui
import (
"crypto/rand"
"encoding/hex"
"time"
)
// newSuppressionID returns a fresh opaque rule ID. Shared by the UI add path
// and the import path so an imported rule without an ID can be deleted like
// any other.
func newSuppressionID() string {
b := make([]byte, 8)
if _, err := rand.Read(b); err != nil {
return "sup-" + hex.EncodeToString([]byte(time.Now().UTC().Format("20060102150405.000000000")))
}
return hex.EncodeToString(b)
}
package webui
import (
"net/http"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/state"
)
// suppressionCheckName matches the check names findings carry. A rule
// matches a finding's check exactly, so a glob or free text would be saved
// as a rule that matches nothing.
var suppressionCheckName = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.:-]{0,127}$`)
// knownCheck reports whether name is a registered check or the check of a
// current finding. Checks from other subsystems may be missing from the
// registry, so an unknown name is a warning, not an error.
func (s *Server) knownCheck(name string) bool {
for _, known := range checks.AllCheckNames() {
if known == name {
return true
}
}
for _, f := range s.store.LatestFindings() {
if f.Check == name {
return true
}
}
return false
}
func (s *Server) apiSuppressions(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
writeAll(w, s.store.LoadSuppressions())
case http.MethodPost:
var req struct {
Check string `json:"check"`
PathPattern string `json:"path_pattern"`
// AllPaths is the explicit opt-in for a rule without a path
// pattern, which hides every finding of the check and stops its
// remediation.
AllPaths bool `json:"all_paths"`
Reason string `json:"reason"`
}
if err := decodeJSONBodyLimited(w, r, 32*1024, &req); err != nil || req.Check == "" {
writeJSONError(w, "check field is required", http.StatusBadRequest)
return
}
if !suppressionCheckName.MatchString(req.Check) {
writeJSONError(w, "check must be a check name such as webshell; patterns and spaces are not allowed", http.StatusBadRequest)
return
}
req.PathPattern = strings.TrimSpace(req.PathPattern)
switch {
case req.PathPattern == "" && !req.AllPaths:
writeJSONError(w, "path_pattern is required; set all_paths to suppress every finding of this check", http.StatusBadRequest)
return
case req.PathPattern != "" && req.AllPaths:
writeJSONError(w, "path_pattern and all_paths are mutually exclusive", http.StatusBadRequest)
return
case req.PathPattern != "":
if _, err := filepath.Match(req.PathPattern, ""); err != nil {
writeJSONError(w, "path_pattern is not a valid glob: "+err.Error(), http.StatusBadRequest)
return
}
}
id := newSuppressionID()
err := s.store.UpdateSuppressions(func(rules []state.SuppressionRule) ([]state.SuppressionRule, error) {
return append(rules, state.SuppressionRule{
ID: id,
Check: req.Check,
PathPattern: req.PathPattern,
Reason: req.Reason,
CreatedAt: time.Now(),
}), nil
})
if err != nil {
writeJSONError(w, "failed to save suppression: "+err.Error(), http.StatusInternalServerError)
return
}
scope := "pattern: " + req.PathPattern
if req.AllPaths {
scope = "all paths"
}
s.auditLog(r, "suppress", req.Check, scope)
resp := map[string]interface{}{"id": id}
if !s.knownCheck(req.Check) {
resp["warning"] = "No known check is named " + req.Check + "; the rule matches nothing until a finding with that check appears."
}
writeOK(w, resp)
case http.MethodDelete:
var req struct {
ID string `json:"id"`
}
if err := decodeJSONBodyLimited(w, r, 16*1024, &req); err != nil || req.ID == "" {
writeJSONError(w, "id is required", http.StatusBadRequest)
return
}
found := false
err := s.store.UpdateSuppressions(func(rules []state.SuppressionRule) ([]state.SuppressionRule, error) {
var filtered []state.SuppressionRule
for _, rule := range rules {
if rule.ID != req.ID {
filtered = append(filtered, rule)
} else {
found = true
}
}
return filtered, nil
})
if err != nil {
writeJSONError(w, "failed to save suppressions: "+err.Error(), http.StatusInternalServerError)
return
}
if !found {
writeJSONError(w, "Suppression rule not found", http.StatusNotFound)
return
}
s.auditLog(r, "unsuppress", req.ID, "removed suppression rule")
writeOK(w, map[string]interface{}{"id": req.ID})
default:
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
}
}
package webui
import (
"errors"
"fmt"
"net"
"net/http"
"strings"
"time"
"github.com/pidginhost/csm/internal/alert"
"github.com/pidginhost/csm/internal/attackdb"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/threat"
)
func (s *Server) handleThreat(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "threat.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
// GET /api/v1/threat/stats
func (s *Server) apiThreatStats(w http.ResponseWriter, r *http.Request) {
adb := attackdb.Global()
if adb == nil {
writeJSONError(w, "attack database not initialized", http.StatusServiceUnavailable)
return
}
writeJSON(w, adb.Stats())
}
// GET /api/v1/threat/top-attackers?limit=25[&spam_only=1]
func (s *Server) apiThreatTopAttackers(w http.ResponseWriter, r *http.Request) {
limit := queryInt(r, "limit", 25)
if limit > 200 {
limit = 200
}
adb := attackdb.Global()
if adb == nil {
writeItems(w, []struct{}{}, map[string]interface{}{"offset": 0, "limit": limit, "truncated": false})
return
}
// One record past the limit tells whether the list was cut. Spam-only
// records score 0, so the spam filter lists them by spam score instead.
now := time.Now()
var recs []*attackdb.IPRecord
if r.URL.Query().Get("spam_only") == "1" {
recs = adb.TopSpamSenders(limit+1, now)
} else {
recs = adb.TopAttackers(limit + 1)
}
truncated := len(recs) > limit
if truncated {
recs = recs[:limit]
}
// Enrich with unified intelligence
ips := make([]string, len(recs))
for i, rec := range recs {
ips[i] = rec.IP
}
intels := threat.LookupBatch(ips, s.cfg.StatePath)
type enriched struct {
*attackdb.IPRecord
UnifiedScore int `json:"unified_score"`
Verdict string `json:"verdict"`
AbuseScore int `json:"abuse_score"`
InThreatDB bool `json:"in_threat_db"`
Blocked bool `json:"currently_blocked"`
Country string `json:"country,omitempty"`
ASOrg string `json:"as_org,omitempty"`
SpamScore int `json:"spam_score,omitempty"`
}
results := make([]enriched, len(recs))
for i, rec := range recs {
results[i] = enriched{
IPRecord: rec,
UnifiedScore: intels[i].UnifiedScore,
Verdict: intels[i].Verdict,
AbuseScore: intels[i].AbuseScore,
InThreatDB: intels[i].InThreatDB,
Blocked: intels[i].CurrentlyBlocked,
SpamScore: attackdb.SpamScore(rec.Mail, now),
}
if gdb := s.geoIPDB.Load(); gdb != nil {
geo := gdb.Lookup(rec.IP)
results[i].Country = geo.Country
results[i].ASOrg = geo.ASOrg
}
}
writeItems(w, results, map[string]interface{}{"offset": 0, "limit": limit, "truncated": truncated})
}
// GET /api/v1/threat/ip?ip=1.2.3.4
func (s *Server) apiThreatIP(w http.ResponseWriter, r *http.Request) {
parsed := net.ParseIP(r.URL.Query().Get("ip"))
if parsed == nil {
writeJSONError(w, "invalid or missing ip parameter", http.StatusBadRequest)
return
}
ip := parsed.String()
intel := threat.Lookup(ip, s.cfg.StatePath)
// Enrich with GeoIP data if available
if gdb := s.geoIPDB.Load(); gdb != nil {
geo := gdb.Lookup(ip)
intel.Country = geo.Country
intel.CountryName = geo.CountryName
intel.City = geo.City
intel.ASN = geo.ASN
intel.ASOrg = geo.ASOrg
intel.Network = geo.Network
}
writeJSON(w, intel)
}
// threatEventView is an attack event as the API sends it: sev is a label.
type threatEventView struct {
attackdb.Event
Severity string `json:"sev"`
}
// GET /api/v1/threat/events?ip=1.2.3.4&limit=50
func (s *Server) apiThreatEvents(w http.ResponseWriter, r *http.Request) {
ip := r.URL.Query().Get("ip")
if ip == "" || net.ParseIP(ip) == nil {
writeJSONError(w, "invalid or missing ip parameter", http.StatusBadRequest)
return
}
limit := queryInt(r, "limit", 50)
if limit > 500 {
limit = 500
}
adb := attackdb.Global()
if adb == nil {
writeItems(w, []threatEventView{}, map[string]interface{}{"offset": 0, "limit": limit, "truncated": false})
return
}
// One event past the limit tells whether older events were left out.
events := adb.QueryEvents(ip, limit+1)
truncated := len(events) > limit
if truncated {
events = events[:limit]
}
views := make([]threatEventView, len(events))
for i, ev := range events {
views[i] = threatEventView{Event: ev, Severity: alert.Severity(ev.Severity).String()}
}
writeItems(w, views, map[string]interface{}{"offset": 0, "limit": limit, "truncated": truncated})
}
// GET /api/v1/threat/db-stats
func (s *Server) apiThreatDBStats(w http.ResponseWriter, r *http.Request) {
result := make(map[string]interface{})
if tdb := checks.GetThreatDB(); tdb != nil {
result["threat_db"] = tdb.Stats()
}
if adb := attackdb.Global(); adb != nil {
result["attack_db"] = map[string]interface{}{
"total_ips": adb.TotalIPs(),
"top_line": adb.FormatTopLine(),
}
}
writeJSON(w, result)
}
// POST /api/v1/threat/whitelist-ip - mark an IP as a known customer
// Unblocks, removes from threat DB + attack DB, adds to whitelist.
func (s *Server) apiThreatWhitelistIP(w http.ResponseWriter, r *http.Request) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != "POST" {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IP string `json:"ip"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.IP == "" {
writeJSONError(w, "IP is required", http.StatusBadRequest)
return
}
parsedIP, err := parseAllowIP(req.IP)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
// Every downstream key (firewall, threat DB, attack DB, audit) uses the
// canonical form; a padded or upper-case spelling validated but then
// matched nothing and still answered 200.
req.IP = parsedIP.String()
actions, releaseErr := s.releaseIP(req.IP, ipRelease{allow: releaseAllow, reason: "CSM whitelist: customer IP"})
warning := ""
if s.blocker != nil {
warning = coveringSubnetWarning(s.blocker, req.IP)
}
detail := "permanent whitelist"
if warning != "" {
detail += "; " + warning
}
s.auditLog(r, "whitelist_ip", req.IP, detail)
if releaseErr != nil {
writeJSONError(w, "IP action incomplete: "+releaseErr.Error(), http.StatusInternalServerError)
return
}
resp := map[string]interface{}{
"ip": req.IP,
"actions": actions,
}
if warning != "" {
resp["warning"] = warning
}
writeOK(w, resp)
}
// GET /api/v1/threat/whitelist - list all whitelisted IPs
func (s *Server) apiThreatWhitelist(w http.ResponseWriter, r *http.Request) {
tdb := checks.GetThreatDB()
if tdb == nil {
writeAll(w, []checks.WhitelistIP{})
return
}
writeAll(w, tdb.WhitelistedIPs())
}
// POST /api/v1/threat/unwhitelist-ip - remove an IP from the whitelist
func (s *Server) apiThreatUnwhitelistIP(w http.ResponseWriter, r *http.Request) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != "POST" {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IP string `json:"ip"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.IP == "" {
writeJSONError(w, "IP is required", http.StatusBadRequest)
return
}
parsedIP, err := parseAndValidateIP(req.IP)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
// Every downstream key (firewall, threat DB, attack DB, audit) uses the
// canonical form; a padded or upper-case spelling validated but then
// matched nothing and still answered 200.
req.IP = parsedIP.String()
if tdb := checks.GetThreatDB(); tdb != nil {
if tdb.IsConfigWhitelisted(req.IP) {
writeJSONError(w, "IP is managed by reputation.whitelist; edit the config and reload", http.StatusConflict)
return
}
tdb.RemoveWhitelist(req.IP)
}
// Also remove from firewall allow list
if s.blocker != nil {
if remover, ok := s.blocker.(allowRemover); ok {
if err := remover.RemoveAllowIP(req.IP); err != nil {
s.auditLog(r, "unwhitelist_ip_failed", req.IP, err.Error())
writeJSONError(w, "Remove firewall allow rule: "+err.Error(), http.StatusInternalServerError)
return
}
}
}
s.auditLog(r, "unwhitelist_ip", req.IP, "removed from runtime whitelist and firewall allow list")
writeOK(w, map[string]interface{}{"ip": req.IP})
}
// manualBlockTTL is the lifetime of the Web UI "Block (24h)" action. The
// threat evidence it records carries the same expiry, so the address stops
// counting as malicious when the firewall block lapses.
const manualBlockTTL = 24 * time.Hour
const (
manualBlockReason = "Manually blocked via CSM Web UI"
manualPermBlockReason = "Permanently blocked via CSM Web UI"
bulkBlockReason = "Bulk blocked via CSM Web UI"
bulkPermanentBlockReason = "Bulk permanently blocked via CSM Web UI"
)
// POST /api/v1/threat/block-ip - manually block an IP for 24 hours.
func (s *Server) apiThreatBlockIP(w http.ResponseWriter, r *http.Request) {
s.operatorBlockIP(w, r, false)
}
// POST /api/v1/threat/block-ip-permanent - block an IP with no expiry.
// Permanence is chosen by the authenticated operator action alone: no
// request field can turn the 24h block into a permanent one.
func (s *Server) apiThreatBlockIPPermanent(w http.ResponseWriter, r *http.Request) {
s.operatorBlockIP(w, r, true)
}
func (s *Server) operatorBlockIP(w http.ResponseWriter, r *http.Request, permanent bool) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != "POST" {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IP string `json:"ip"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.IP == "" {
writeJSONError(w, "IP is required", http.StatusBadRequest)
return
}
parsedIP, err := parseAndValidateIP(req.IP)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
// Every downstream key (firewall, threat DB, attack DB, audit) uses the
// canonical form; a padded or upper-case spelling validated but then
// matched nothing and still answered 200.
req.IP = parsedIP.String()
reason := manualBlockReason
ttl := manualBlockTTL
if permanent {
reason = manualPermBlockReason
ttl = 0
}
var actions []string
// 1. Block in firewall. A zero timeout is a permanent firewall block.
if s.blocker == nil {
writeJSONError(w, "firewall engine not available", http.StatusServiceUnavailable)
return
}
// Operator-initiated: bypass auto_response.dry_run gate.
if err := s.blockIPPreservingLifetime(req.IP, reason, ttl); err != nil {
status := http.StatusInternalServerError
if errors.Is(err, firewall.ErrPermanentBlock) || errors.Is(err, firewall.ErrLongerBlock) {
status = http.StatusConflict
}
writeJSONError(w, fmt.Sprintf("block failed: %v", err), status)
return
}
invalidateIPUndo(req.IP)
if permanent {
actions = append(actions, "blocked in firewall permanently")
} else {
actions = append(actions, "blocked in firewall for 24h")
}
// 2. Record threat evidence with the same lifetime as the block.
if tdb := checks.GetThreatDB(); tdb != nil {
if permanent {
tdb.AddPermanent(req.IP, reason)
actions = append(actions, "added to threat DB permanently")
} else {
tdb.AddOperatorTemporary(req.IP, reason, ttl)
actions = append(actions, "added to threat DB for 24h")
}
}
// 3. Record in attack DB
if adb := attackdb.Global(); adb != nil {
adb.MarkBlocked(req.IP)
actions = append(actions, "recorded in attack DB")
}
if permanent {
s.auditLog(r, "block_ip_permanent", req.IP, "manual permanent block")
} else {
s.auditLog(r, "block_ip", req.IP, "manual block 24h")
}
writeOK(w, map[string]interface{}{
"ip": req.IP,
"permanent": permanent,
"actions": actions,
})
}
// POST /api/v1/threat/clear-ip - unblock + clear from all DBs without whitelisting.
// For dynamic IP customers: one-time cleanup, IP can be re-blocked later.
// ipRelease says how releaseIP lets an address go. The zero value unblocks
// and forgets it without allowing it (Unblock & Clear).
type ipRelease struct {
allow releaseAllowMode
ttl time.Duration // for releaseTempAllow
reason string // firewall allow rule comment
}
type releaseAllowMode int
const (
releaseNoAllow releaseAllowMode = iota
releaseAllow
releaseTempAllow
)
// releaseIP is the shared tail of whitelist, temporary whitelist, clear and
// bulk whitelist: unblock the address in the firewall, allow it as rel says,
// drop it from the threat database's permanent list (whitelisting it there
// too when allowed), and forget it in the attack database, all inside the
// netblock-history lock so the history cleanup is one operator action with
// the firewall change. cPHulk's login history is flushed after. It returns
// the steps that took effect and any failed required steps. Other changes
// may have happened on error, so callers still audit.
func (s *Server) releaseIP(ip string, rel ipRelease) ([]string, error) {
var actions []string
var failures []error
hours := int(rel.ttl / time.Hour)
err := checks.ForgetNetblockHistory(s.cfg.StatePath, ip, func() {
if s.blocker != nil {
if err := s.blocker.UnblockIP(ip); err == nil {
actions = append(actions, "unblocked from firewall")
} else {
failures = append(failures, fmt.Errorf("unblock firewall: %w", err))
}
switch rel.allow {
case releaseAllow:
if allower, ok := s.blocker.(ipAllower); ok {
if err := allower.AllowIP(ip, rel.reason); err == nil {
actions = append(actions, "added to firewall allow list")
} else {
failures = append(failures, fmt.Errorf("allow in firewall: %w", err))
}
}
case releaseTempAllow:
if allower, ok := s.blocker.(ipTempAllower); ok {
if err := allower.TempAllowIP(ip, rel.reason, rel.ttl); err == nil {
actions = append(actions, fmt.Sprintf("temp allowed in firewall for %dh", hours))
} else {
failures = append(failures, fmt.Errorf("temporarily allow in firewall: %w", err))
}
}
}
}
if tdb := checks.GetThreatDB(); tdb != nil {
tdb.RemovePermanent(ip)
switch rel.allow {
case releaseAllow:
tdb.AddWhitelist(ip)
actions = append(actions, "removed from threat DB, added to whitelist")
case releaseTempAllow:
tdb.TempWhitelist(ip, rel.ttl)
actions = append(actions, fmt.Sprintf("temp whitelisted for %dh", hours))
default:
actions = append(actions, "removed from threat DB")
}
}
if adb := attackdb.Global(); adb != nil {
// Historical imports can retain equivalent address spellings.
adb.ForgetIP(net.ParseIP(ip))
actions = append(actions, "removed from attack DB")
}
})
if err == nil {
actions = append(actions, "removed from subnet block history")
} else {
failures = append(failures, fmt.Errorf("subnet history cleanup failed: %w", err))
}
if flushCphulk(ip) == nil {
actions = append(actions, "flushed cPanel login history")
}
return actions, errors.Join(failures...)
}
func (s *Server) apiThreatClearIP(w http.ResponseWriter, r *http.Request) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != "POST" {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IP string `json:"ip"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.IP == "" {
writeJSONError(w, "IP is required", http.StatusBadRequest)
return
}
parsedIP, err := parseRecordedIP(req.IP)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
// Every downstream key (firewall, threat DB, attack DB, audit) uses the
// canonical form; a padded or upper-case spelling validated but then
// matched nothing and still answered 200.
req.IP = parsedIP.String()
actions, releaseErr := s.releaseIP(req.IP, ipRelease{})
s.auditLog(r, "clear_ip", req.IP, "unblock & clear")
if releaseErr != nil {
writeJSONError(w, "IP action incomplete: "+releaseErr.Error(), http.StatusInternalServerError)
return
}
writeOK(w, map[string]interface{}{
"ip": req.IP,
"actions": actions,
})
}
// POST /api/v1/threat/temp-whitelist-ip - whitelist for a specified duration.
func (s *Server) apiThreatTempWhitelistIP(w http.ResponseWriter, r *http.Request) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != "POST" {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IP string `json:"ip"`
Hours int `json:"hours"` // default 24
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil || req.IP == "" {
writeJSONError(w, "IP is required", http.StatusBadRequest)
return
}
parsedIP, err := parseAllowIP(req.IP)
if err != nil {
writeJSONError(w, err.Error(), http.StatusBadRequest)
return
}
// Every downstream key (firewall, threat DB, attack DB, audit) uses the
// canonical form; a padded or upper-case spelling validated but then
// matched nothing and still answered 200.
req.IP = parsedIP.String()
if req.Hours <= 0 {
req.Hours = 24
}
if req.Hours > 168 { // max 7 days
req.Hours = 168
}
ttl := time.Duration(req.Hours) * time.Hour
actions, releaseErr := s.releaseIP(req.IP, ipRelease{allow: releaseTempAllow, ttl: ttl, reason: "CSM temp whitelist"})
s.auditLog(r, "temp_whitelist_ip", req.IP, fmt.Sprintf("%dh temp whitelist", req.Hours))
if releaseErr != nil {
writeJSONError(w, "IP action incomplete: "+releaseErr.Error(), http.StatusInternalServerError)
return
}
writeOK(w, map[string]interface{}{
"ip": req.IP,
"duration_seconds": ttl.Seconds(),
"actions": actions,
})
}
// threatBulkActionMax bounds the addresses one bulk threat action changes. Each
// request returns one undo token, so the UI refuses larger selections instead
// of splitting them.
const threatBulkActionMax = 100
// POST /api/v1/threat/bulk-action - block or whitelist multiple IPs at once.
func (s *Server) apiThreatBulkAction(w http.ResponseWriter, r *http.Request) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
IPs []string `json:"ips"`
Action string `json:"action"`
}
if err := decodeJSONBodyLimited(w, r, 64*1024, &req); err != nil {
writeJSONError(w, "Invalid request body", http.StatusBadRequest)
return
}
if len(req.IPs) == 0 || len(req.IPs) > threatBulkActionMax {
writeJSONError(w, fmt.Sprintf("IPs must be 1-%d items", threatBulkActionMax), http.StatusBadRequest)
return
}
blockAction := req.Action == "block" || req.Action == "block_permanent"
if !blockAction && req.Action != "whitelist" {
writeJSONError(w, "Action must be 'block', 'block_permanent' or 'whitelist'", http.StatusBadRequest)
return
}
if blockAction && s.blocker == nil {
writeJSONError(w, "firewall engine not available", http.StatusServiceUnavailable)
return
}
// Permanence follows the operator action, never a per-IP request field.
permanent := req.Action == "block_permanent"
blockReason := bulkBlockReason
blockTTL := manualBlockTTL
if permanent {
blockReason = bulkPermanentBlockReason
blockTTL = 0
}
priorBlocks := make(map[string]firewall.BlockedEntry)
expectedBlocks := make(map[string]firewall.BlockedEntry)
seen := make(map[string]bool, len(req.IPs))
count := 0
succeeded := make([]string, 0, len(req.IPs))
var removedThreats []undoThreatRow
var warnings []string
parseIP := parseAllowIP
if blockAction {
parseIP = parseAndValidateIP
}
for _, ipStr := range req.IPs {
parsedIP, err := parseIP(ipStr)
if err != nil {
warnings = append(warnings, ipStr+": "+err.Error())
continue
}
ipStr = parsedIP.String()
if seen[ipStr] {
continue
}
seen[ipStr] = true
switch {
case blockAction:
// Mirror the single-IP block flow.
// Operator-initiated bulk block: bypass auto_response.dry_run gate.
before, after, err := s.blockIPForUndo(ipStr, blockReason, blockTTL)
if err != nil {
warnings = append(warnings, ipStr+": "+err.Error())
continue
}
if before != nil {
priorBlocks[ipStr] = *before
}
if after != nil {
expectedBlocks[ipStr] = *after
}
invalidateIPUndo(ipStr)
// Capture whatever evidence is already on file so the undo
// restores it instead of dropping an older permanent row this
// block did not create.
if row, ok := captureUndoThreatRow(ipStr, false); ok {
removedThreats = append(removedThreats, row)
}
if tdb := checks.GetThreatDB(); tdb != nil {
if permanent {
tdb.AddPermanent(ipStr, blockReason)
} else {
tdb.AddOperatorTemporary(ipStr, blockReason, blockTTL)
}
}
if adb := attackdb.Global(); adb != nil {
adb.MarkBlocked(ipStr)
}
succeeded = append(succeeded, ipStr)
count++
default:
// Capture the live threat row before dropping it so an undo can
// restore it exactly, preserving source/expiry, instead of
// leaving a whitelisted attacker with no threat record.
row, hadThreat := captureUndoThreatRow(ipStr, false)
if _, err := s.releaseIP(ipStr, ipRelease{allow: releaseAllow, reason: "CSM bulk whitelist"}); err != nil {
warnings = append(warnings, ipStr+": "+err.Error())
continue
}
if hadThreat {
removedThreats = append(removedThreats, row)
}
if s.blocker != nil {
if warning := coveringSubnetWarning(s.blocker, ipStr); warning != "" {
warnings = append(warnings, ipStr+": "+warning)
}
}
succeeded = append(succeeded, ipStr)
count++
}
}
auditDetail := ""
if blockAction {
auditDetail = "24h block"
if permanent {
auditDetail = "permanent block"
}
auditDetail += ": "
}
auditDetail += strings.Join(succeeded, ", ")
s.auditLog(r, "threat_bulk_"+req.Action, fmt.Sprintf("%d IPs", count), auditDetail)
var undoToken string
if count > 0 {
inverse := undoInverseThreatBlock
summary := fmt.Sprintf("Blocked %d IPs", count)
action := "threat_bulk_block"
switch {
case permanent:
summary = fmt.Sprintf("Permanently blocked %d IPs", count)
action = "threat_bulk_block_permanent"
case !blockAction:
inverse = undoInverseThreatWhitelist
summary = fmt.Sprintf("Whitelisted %d IPs", count)
action = "threat_bulk_whitelist"
}
undoToken = s.recordUndoEntry(r, action, inverse, summary,
undoPayloadIPs{IPs: succeeded, RestoreThreats: removedThreats, BlockSnapshot: blockAction, RestoreBlocks: priorBlocks, ExpectedBlocks: expectedBlocks})
}
if warnings == nil {
warnings = []string{}
}
fields := map[string]interface{}{
"count": count,
"permanent": permanent,
"warnings": warnings,
}
if count == 0 {
// No address completed: an error, with each address's reason.
msg := "No address action completed"
if len(warnings) > 0 {
msg += ": " + strings.Join(warnings, "; ")
}
fields["error"] = msg
writeJSONStatus(w, http.StatusUnprocessableEntity, fields)
return
}
if undoToken != "" {
fields["undo_token"] = undoToken
}
writeOK(w, fields)
}
// writeJSON is defined in api.go
package webui
import (
"bytes"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"math/big"
"net"
"os"
"path/filepath"
"sync"
"time"
"github.com/pidginhost/csm/internal/integrity"
)
// selfSignedValidity is how long a generated certificate is valid; a var so
// tests can generate one close to expiry.
var selfSignedValidity = 365 * 24 * time.Hour
// Inject write failures without relying on permissions that root bypasses.
var writeTLSFile = writeFileReplace
// renewBefore is how close to expiry a generated certificate is replaced.
const renewBefore = 30 * 24 * time.Hour
// selfSignedOrganization marks the certificates EnsureTLSCert generates.
const selfSignedOrganization = "CSM Security Monitor"
// EnsureTLSCert generates a self-signed ECDSA P-256 certificate if the cert
// or key file doesn't exist, and renews one it generated earlier when it
// expires within renewBefore. Any other certificate, such as one the
// operator installed, is left alone. Includes localhost and the server
// hostname in the certificate SANs.
func EnsureTLSCert(certPath, keyPath string, extraNames ...string) error {
if fileExists(certPath) && fileExists(keyPath) {
return renewTLSCert(certPath, keyPath)
}
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return fmt.Errorf("generating key: %w", err)
}
serial, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
// Use first extra name (hostname) as CN, fall back to localhost
cn := "localhost"
if len(extraNames) > 0 && extraNames[0] != "" {
cn = extraNames[0]
}
template := &x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{
Organization: []string{selfSignedOrganization},
CommonName: cn,
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(selfSignedValidity),
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
DNSNames: buildDNSNames(extraNames),
IPAddresses: buildIPList(extraNames),
}
certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
if err != nil {
return fmt.Errorf("creating certificate: %w", err)
}
keyDER, err := x509.MarshalECPrivateKey(key)
if err != nil {
return fmt.Errorf("marshaling key: %w", err)
}
// This path creates the initial pair. Renewal retains its key so a
// failed certificate replacement cannot break the pair on disk.
if err := writeTLSFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})); err != nil {
return fmt.Errorf("writing key: %w", err)
}
if err := writeTLSFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})); err != nil {
return fmt.Errorf("writing cert: %w", err)
}
return nil
}
// renewTLSCert never generates missing files: an operator may temporarily
// remove either file while replacing their certificate.
func renewTLSCert(certPath, keyPath string) error {
// Read and validate the same certificate that will be renewed, even
// when an operator replaces the files during this check.
// #nosec G304 -- paths are operator-configured TLS files.
certPEM, err := os.ReadFile(certPath)
if err != nil {
return err
}
block, prefix, suffix := leafCertificateBlock(certPEM)
if block == nil {
return nil
}
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return fmt.Errorf("parsing certificate for renewal: %w", err)
}
if !ownCertExpiring(cert) {
return nil
}
// #nosec G304 -- paths are operator-configured TLS files.
keyPEM, err := os.ReadFile(keyPath)
if err != nil {
return err
}
pair, err := tls.X509KeyPair(certPEM, keyPEM)
if err != nil {
return fmt.Errorf("loading certificate for renewal: %w", err)
}
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
if err != nil {
return err
}
cert.SerialNumber = serial
cert.NotBefore = time.Now()
cert.NotAfter = cert.NotBefore.Add(selfSignedValidity)
der, err := x509.CreateCertificate(rand.Reader, cert, cert, cert.PublicKey, pair.PrivateKey)
if err != nil {
return fmt.Errorf("renewing certificate: %w", err)
}
renewed := append(bytes.Clone(prefix), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...)
renewed = append(renewed, suffix...)
return writeTLSFile(certPath, renewed)
}
// leafCertificateBlock returns the certificate the TLS stack serves: the
// first CERTIFICATE block. A combined file, such as cPanel's service
// certificate, carries the private key ahead of it. The surrounding bytes
// must survive renewal because they may contain the key and chain.
func leafCertificateBlock(data []byte) (block *pem.Block, prefix, suffix []byte) {
rest := data
for {
block, rest = pem.Decode(rest)
if block == nil {
return nil, nil, nil
}
if block.Type == "CERTIFICATE" {
// Decode skips leading text and malformed blocks. Its accepted
// BEGIN marker is the last one before the consumed block's end.
end := len(data) - len(rest)
start := bytes.LastIndex(data[:end], []byte("-----BEGIN CERTIFICATE-----"))
return block, data[:start], rest
}
}
}
// ownCertExpiring recognizes CSM's self-signed certificates near expiry.
// A matching issuer name alone does not prove a self-signature.
func ownCertExpiring(cert *x509.Certificate) bool {
if !bytes.Equal(cert.RawIssuer, cert.RawSubject) {
return false
}
if len(cert.Subject.Organization) != 1 || cert.Subject.Organization[0] != selfSignedOrganization {
return false
}
if err := cert.CheckSignature(cert.SignatureAlgorithm, cert.RawTBSCertificate, cert.Signature); err != nil {
return false
}
return time.Until(cert.NotAfter) < renewBefore
}
// writeFileReplace writes data to a private temporary file beside path and
// renames it over path.
func writeFileReplace(path string, data []byte) error {
tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
if err != nil {
return err
}
defer func() { _ = os.Remove(tmp.Name()) }()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), path)
}
// certReloader serves the certificate on disk and loads it again when the
// files change, so a renewed or replaced certificate needs no restart.
type certReloader struct {
certPath, keyPath string
mu sync.Mutex
cert *tls.Certificate
stamp string
}
func newCertReloader(certPath, keyPath string) (*certReloader, error) {
r := &certReloader{certPath: certPath, keyPath: keyPath}
if _, err := r.GetCertificate(nil); err != nil {
return nil, err
}
return r, nil
}
// GetCertificate is a tls.Config.GetCertificate. A pair that does not load
// (mid-replacement) keeps the previous certificate in service.
func (r *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
stamp := fileStamp(r.certPath) + "|" + fileStamp(r.keyPath)
r.mu.Lock()
defer r.mu.Unlock()
if r.cert != nil && stamp == r.stamp {
return r.cert, nil
}
cert, err := tls.LoadX509KeyPair(r.certPath, r.keyPath)
if err != nil {
if r.cert != nil {
return r.cert, nil
}
return nil, err
}
r.cert, r.stamp = &cert, stamp
return r.cert, nil
}
func fileStamp(path string) string {
info, err := os.Stat(path)
if err != nil {
return ""
}
return integrity.FileChangeKey(info)
}
func buildDNSNames(extra []string) []string {
names := []string{"localhost"}
for _, n := range extra {
if net.ParseIP(n) == nil { // not an IP - it's a hostname
names = append(names, n)
}
}
return names
}
func buildIPList(extra []string) []net.IP {
ips := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}
for _, n := range extra {
if ip := net.ParseIP(n); ip != nil {
ips = append(ips, ip)
}
}
return ips
}
func fileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
package webui
import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"strings"
"time"
"github.com/pidginhost/csm/internal/checks"
"github.com/pidginhost/csm/internal/firewall"
"github.com/pidginhost/csm/internal/state"
"github.com/pidginhost/csm/internal/store"
)
// Recognised inverse-action keys. Each handler that records an undo entry
// sets one of these on the entry; apiUndoRun dispatches based on the value.
const (
undoInverseThreatBlock = "threat_bulk_unblock"
undoInverseThreatUnblock = "threat_bulk_block"
undoInverseThreatWhitelist = "threat_bulk_unwhitelist"
undoInverseThreatUnwhitelist = "threat_bulk_whitelist"
undoInverseFirewallUnblock = "firewall_bulk_reblock"
undoInverseFindingUndismiss = "finding_undismiss"
)
// maxUndoPayloadSize bounds decompression of persisted data while leaving
// room for the threat-row snapshot behind the 500-item firewall action.
const maxUndoPayloadSize = 4 * 1024 * 1024
// undoPayloadIPs is the payload schema for every undo entry we currently
// generate: a list of IPs plus an optional reason and timeout. Future undo
// kinds can add their own payload structs alongside this one.
type undoPayloadIPs struct {
BlockSnapshot bool `json:"block_snapshot,omitempty"`
RestoreBlocks map[string]firewall.BlockedEntry `json:"restore_blocks,omitempty"`
ExpectedBlocks map[string]firewall.BlockedEntry `json:"expected_blocks,omitempty"`
IPs []string `json:"ips"`
Reason string `json:"reason,omitempty"`
Timeout string `json:"timeout,omitempty"` // ParseDuration-compatible
// RestoreThreats carries the threat-DB rows a bulk action removed so the
// matching undo can put them back exactly.
RestoreThreats []undoThreatRow `json:"restore_threats,omitempty"`
// Dismissals carries what a finding dismissal changed so its undo can
// list the finding again and re-arm its alerts.
Dismissals []state.DismissUndo `json:"dismissals,omitempty"`
}
// undoThreatRow captures a removed threat-DB row's identity so undo can
// restore it with the same source and expiry, instead of resurrecting an
// auto-block row as a never-expiring operator block (or vice versa).
type undoThreatRow struct {
IP string `json:"ip"`
Reason string `json:"reason"`
Source string `json:"source,omitempty"`
ExpiresAt time.Time `json:"expires_at,omitzero"`
}
// recordUndoEntry persists an undo entry for the operator who issued r and
// returns the new entry's ID. The ID lets the calling handler surface the
// undo token to the client in the same response. Any store error is logged
// and swallowed so a bulk action never fails just because the undo queue
// could not be encoded or written.
func (s *Server) recordUndoEntry(r *http.Request, action, inverse, summary string, payload undoPayloadIPs) string {
if r == nil {
return ""
}
opkey := s.operatorKey(r)
if opkey == "" {
return ""
}
sdb := store.Global()
if sdb == nil {
return ""
}
raw, err := encodeUndoPayload(payload)
if err != nil {
log.Printf("webui: encode undo entry: %v", err)
return ""
}
entry, err := sdb.AppendUndoEntry(opkey, store.UndoEntry{
Targets: undoTargets(payload),
Action: action,
Inverse: inverse,
Payload: raw,
Summary: summary,
})
if err != nil {
log.Printf("webui: record undo entry: %v", err)
return ""
}
return entry.ID
}
// undoTargets names what an undo entry acts on: its IPs, or the finding keys
// of a dismissal.
func undoTargets(payload undoPayloadIPs) []string {
if len(payload.IPs) > 0 {
return payload.IPs
}
keys := make([]string, 0, len(payload.Dismissals))
for _, d := range payload.Dismissals {
keys = append(keys, d.Key)
}
return keys
}
func encodeUndoPayload(payload undoPayloadIPs) ([]byte, error) {
raw, err := json.Marshal(payload)
if err != nil {
return nil, err
}
if len(raw) > maxUndoPayloadSize {
return nil, fmt.Errorf("undo payload exceeds %d bytes", maxUndoPayloadSize)
}
var compressed bytes.Buffer
zw, err := gzip.NewWriterLevel(&compressed, gzip.BestSpeed)
if err != nil {
return nil, err
}
if _, err := zw.Write(raw); err != nil {
_ = zw.Close()
return nil, err
}
if err := zw.Close(); err != nil {
return nil, err
}
if compressed.Len() < len(raw) {
return compressed.Bytes(), nil
}
return raw, nil
}
func decodeUndoPayload(raw []byte, payload *undoPayloadIPs) error {
if len(raw) < 2 || raw[0] != 0x1f || raw[1] != 0x8b {
return json.Unmarshal(raw, payload)
}
zr, err := gzip.NewReader(bytes.NewReader(raw))
if err != nil {
return err
}
decoded, readErr := io.ReadAll(io.LimitReader(zr, maxUndoPayloadSize+1))
closeErr := zr.Close()
if readErr != nil {
return readErr
}
if closeErr != nil {
return closeErr
}
if len(decoded) > maxUndoPayloadSize {
return fmt.Errorf("undo payload exceeds %d bytes", maxUndoPayloadSize)
}
return json.Unmarshal(decoded, payload)
}
// undoPendingView is the JSON shape returned to the client. The payload is
// stripped because the client only needs identity + summary to render the
// banner; the server keeps the payload for the actual undo run.
type undoPendingView struct {
ID string `json:"id"`
Action string `json:"action"`
Inverse string `json:"inverse"`
Summary string `json:"summary"`
RecordedAt time.Time `json:"recorded_at"`
ExpiresAt time.Time `json:"expires_at"`
}
// apiUndoPending returns the latest non-expired undo entry for the operator,
// or an empty object when no entry is queued.
func (s *Server) apiUndoPending(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
opkey := s.operatorKey(r)
if opkey == "" {
writeJSONError(w, "Unauthenticated", http.StatusUnauthorized)
return
}
sdb := store.Global()
if sdb == nil {
writeJSON(w, map[string]interface{}{})
return
}
entry, ok, err := sdb.LatestUndoEntry(opkey)
if err != nil {
writeJSONError(w, "Store error", http.StatusInternalServerError)
return
}
if !ok {
writeJSON(w, map[string]interface{}{})
return
}
writeJSON(w, undoPendingView{
ID: entry.ID,
Action: entry.Action,
Inverse: entry.Inverse,
Summary: entry.Summary,
RecordedAt: entry.RecordedAt,
ExpiresAt: entry.RecordedAt.Add(store.UndoTTL),
})
}
type undoRunRequest struct {
ID string `json:"id"`
}
type undoRunResponse struct {
OK bool `json:"ok"`
Action string `json:"action"`
Inverse string `json:"inverse"`
Count int `json:"count"`
}
// apiUndoRun consumes the named undo entry (or the most recent one when id
// is empty) and dispatches its inverse. Each successful undo also writes a
// "undo_<original>" audit entry so the trail records the reversal.
func (s *Server) apiUndoRun(w http.ResponseWriter, r *http.Request) {
s.threatActionMu.Lock()
defer s.threatActionMu.Unlock()
if r.Method != http.MethodPost {
writeJSONError(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
opkey := s.operatorKey(r)
if opkey == "" {
writeJSONError(w, "Unauthenticated", http.StatusUnauthorized)
return
}
var req undoRunRequest
if err := decodeJSONBodyLimited(w, r, 4*1024, &req); err != nil {
writeJSONError(w, "Invalid request body", http.StatusBadRequest)
return
}
sdb := store.Global()
if sdb == nil {
writeJSONError(w, "Store unavailable", http.StatusServiceUnavailable)
return
}
var (
entry store.UndoEntry
ok bool
err error
)
if req.ID == "" {
entry, ok, err = sdb.LatestUndoEntry(opkey)
if err == nil && ok {
entry, ok, err = sdb.ConsumeUndoEntry(opkey, entry.ID)
}
} else {
entry, ok, err = sdb.ConsumeUndoEntry(opkey, req.ID)
}
if err != nil {
writeJSONError(w, "Store error", http.StatusInternalServerError)
return
}
if !ok {
writeJSONError(w, "Undo window expired", http.StatusGone)
return
}
targets := strings.Join(entry.Targets, ", ")
resp, runErr := s.runUndoEntry(entry)
if runErr != nil {
// The entry is consumed and the inverse may have run part way, so the
// attempt is recorded even though it failed.
s.auditLog(r, "undo_"+entry.Action+"_failed", targets, entry.Summary+": "+runErr.Error())
writeJSONError(w, runErr.Error(), http.StatusInternalServerError)
return
}
s.auditLog(r, "undo_"+entry.Action, fmt.Sprintf("%d items", resp.Count), entry.Summary+": "+targets)
writeJSON(w, resp)
}
func (s *Server) runUndoEntry(entry store.UndoEntry) (undoRunResponse, error) {
var payload undoPayloadIPs
if len(entry.Payload) > 0 {
if err := decodeUndoPayload(entry.Payload, &payload); err != nil {
return undoRunResponse{}, fmt.Errorf("decode payload: %w", err)
}
}
resp := undoRunResponse{
OK: true,
Action: entry.Action,
Inverse: entry.Inverse,
}
switch entry.Inverse {
case undoInverseThreatBlock:
// Original action blocked IPs; inverse unblocks them and puts back
// the evidence that was on file before the block.
count, err := s.undoBulkBlock(payload)
if err != nil {
return undoRunResponse{}, err
}
resp.Count = count
case undoInverseThreatUnblock:
// Original unblocked IPs; inverse re-blocks them with the saved reason.
// The payload is self-written state, so a bad timeout falls back to
// the default instead of failing the undo.
timeout, terr := parseDuration(payload.Timeout)
if terr != nil || timeout == 0 {
timeout = 24 * time.Hour
}
reason := payload.Reason
if reason == "" {
reason = "Undo: re-block via CSM Web UI"
}
count, err := s.undoBulkReblock(payload.IPs, reason, timeout)
if err != nil {
return undoRunResponse{}, err
}
resp.Count = count
case undoInverseThreatWhitelist:
resp.Count = s.undoBulkWhitelist(payload)
case undoInverseThreatUnwhitelist:
resp.Count = s.undoBulkUnwhitelist(payload.IPs)
case undoInverseFirewallUnblock:
if payload.BlockSnapshot {
count, err := s.undoSnapshotBlocks(payload, false)
if err != nil {
return undoRunResponse{}, err
}
resp.Count = count
break
}
reason := payload.Reason
if reason == "" {
reason = "Undo: re-block via CSM Web UI"
}
timeout, terr := parseDuration(payload.Timeout)
if terr != nil || timeout == 0 {
timeout = 24 * time.Hour
}
count, err := s.undoBulkReblock(payload.IPs, reason, timeout)
if err != nil {
return undoRunResponse{}, err
}
restoreUndoThreatRows(payload.RestoreThreats)
resp.Count = count
case undoInverseFindingUndismiss:
for _, d := range payload.Dismissals {
if s.store.UndoDismiss(d) {
resp.Count++
}
}
// Save now, once for all keys: a dismissal already on disk would
// otherwise survive a crash and suppress the restored finding. A
// failed write stays pending for the next due flush.
if resp.Count > 0 {
if err := s.store.Flush(); err != nil {
log.Printf("webui: save undo of dismissal: %v", err)
}
}
default:
return undoRunResponse{}, fmt.Errorf("unknown inverse action %q", entry.Inverse)
}
return resp, nil
}
// captureUndoThreatRow snapshots the live threat row for ip. With
// temporaryOnly set it captures only rows tied to a firewall block, which is
// exactly the set a firewall unblock drops.
func captureUndoThreatRow(ip string, temporaryOnly bool) (undoThreatRow, bool) {
sdb := store.Global()
if sdb == nil {
return undoThreatRow{}, false
}
entry, ok := sdb.GetPermanentBlock(ip)
if !ok || entry.Expired(time.Now()) {
return undoThreatRow{}, false
}
if temporaryOnly && !entry.TiedToFirewallBlock() {
return undoThreatRow{}, false
}
return undoThreatRow{
IP: entry.IP,
Reason: entry.Reason,
Source: entry.Source,
ExpiresAt: entry.ExpiresAt,
}, true
}
func restoreUndoThreatRows(rows []undoThreatRow) {
tdb := checks.GetThreatDB()
if tdb == nil {
return
}
now := time.Now()
for _, row := range rows {
if _, err := parseAndValidateIP(row.IP); err != nil {
continue
}
if shouldRestoreUndoThreatAsPermanent(row, now) {
tdb.AddPermanent(row.IP, row.Reason)
continue
}
if row.ExpiresAt.IsZero() {
continue
}
ttl := time.Until(row.ExpiresAt)
if ttl <= 0 {
continue // already lapsed; nothing worth restoring
}
// A timed operator block keeps its operator source: restoring it as
// auto-block evidence would let the auto-block cleanup paths delete
// an operator's deliberate block.
if row.Source == store.ThreatSourceOperator {
tdb.AddOperatorTemporary(row.IP, row.Reason, ttl)
continue
}
tdb.AddTemporary(row.IP, row.Reason, ttl)
}
}
func shouldRestoreUndoThreatAsPermanent(row undoThreatRow, now time.Time) bool {
if row.Source == store.ThreatSourceOperator {
return row.ExpiresAt.IsZero()
}
if row.Source != "" || !row.ExpiresAt.IsZero() {
return false
}
legacy := store.PermanentBlockEntry{Reason: row.Reason}
return !legacy.Expired(now)
}
func (s *Server) undoBulkBlock(payload undoPayloadIPs) (int, error) {
if payload.BlockSnapshot {
return s.undoSnapshotBlocks(payload, true)
}
if s.blocker == nil {
return 0, fmt.Errorf("firewall engine not available")
}
count := 0
for _, ip := range payload.IPs {
if _, err := parseAndValidateIP(ip); err != nil {
continue
}
if err := s.blocker.UnblockIP(ip); err != nil {
continue
}
if tdb := checks.GetThreatDB(); tdb != nil {
tdb.RemovePermanent(ip)
}
restoreUndoThreatRows(threatRowsForIP(payload.RestoreThreats, ip))
_ = flushCphulk(ip) // best effort
count++
}
return count, nil
}
func (s *Server) undoBulkReblock(ips []string, reason string, timeout time.Duration) (int, error) {
if s.blocker == nil {
return 0, fmt.Errorf("firewall engine not available")
}
count := 0
for _, ip := range ips {
if _, err := parseAndValidateIP(ip); err != nil {
continue
}
if err := blockIPForOperator(s.blocker, ip, reason, timeout); err != nil {
continue
}
count++
}
return count, nil
}
func (s *Server) undoBulkWhitelist(payload undoPayloadIPs) int {
count := 0
for _, ip := range payload.IPs {
if _, err := parseAndValidateIP(ip); err != nil {
continue
}
if tdb := checks.GetThreatDB(); tdb != nil {
tdb.RemoveWhitelist(ip)
}
// The bulk whitelist added a firewall allow rule; leaving it lets a
// mis-whitelisted attacker bypass every future block indefinitely.
if s.blocker != nil {
if remover, ok := s.blocker.(allowRemover); ok {
_ = remover.RemoveAllowIP(ip)
}
}
count++
}
// Restore the threat rows the whitelist removed. RemoveWhitelist ran for
// every IP above, so the whitelist no longer suppresses these adds.
restoreUndoThreatRows(payload.RestoreThreats)
return count
}
func (s *Server) undoBulkUnwhitelist(ips []string) int {
count := 0
for _, ip := range ips {
if _, err := parseAndValidateIP(ip); err != nil {
continue
}
if tdb := checks.GetThreatDB(); tdb != nil {
tdb.AddWhitelist(ip)
}
count++
}
return count
}
package webui
import (
"net/http"
"os"
"strings"
"github.com/pidginhost/csm/internal/config"
"github.com/pidginhost/csm/internal/integrity"
"github.com/pidginhost/csm/internal/threatintel"
)
// SetVerifiedBotsReloader registers the callback the Daemon uses to push a
// saved verified_bots list into the live registry + rDNS verifier, so edits
// take effect without a restart. nil-safe: tests leave it unset.
func (s *Server) SetVerifiedBotsReloader(fn func() error) {
s.verifiedBotsReloader = fn
}
func (s *Server) handleVerifiedBots(w http.ResponseWriter, r *http.Request) {
s.renderTemplate(w, r, "verified-bots.html", map[string]string{
"Hostname": s.cfg.Hostname,
})
}
// apiVerifiedBots (GET /api/v1/verified-bots) returns the configured list plus
// the config etag for optimistic locking on save.
func (s *Server) apiVerifiedBots(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
diskBytes, err := os.ReadFile(s.cfg.ConfigFile) // #nosec G304 -- operator-supplied config path
if err != nil {
writeJSONError(w, "read config: "+err.Error(), http.StatusInternalServerError)
return
}
disk, err := config.LoadBytes(diskBytes)
if err != nil {
writeJSONError(w, "parse config: "+err.Error(), http.StatusInternalServerError)
return
}
bots := disk.Reputation.VerifiedBots
writeItems(w, bots, map[string]interface{}{
"total": len(bots),
"etag": disk.Integrity.ConfigHash,
"bot_ranges": botRangesSummary(disk),
})
}
// botRangesSummary is the read-only view of the built-in AI-crawler ranges the
// Verified Bots page shows: the configured auto-update posture plus the live
// per-bot prefix counts and last-refresh time from the active overlay.
func botRangesSummary(disk *config.Config) map[string]interface{} {
summary := map[string]interface{}{
"auto_update": disk.BotRangesAutoUpdate(),
"prefixes": threatintel.AICrawlerRangePrefixCounts(),
}
if secs, ok := durationSeconds(disk.Reputation.BotRanges.UpdateInterval); ok {
summary["update_interval_seconds"] = secs
}
if ts := threatintel.LastFetchedRangesRefresh(); !ts.IsZero() {
summary["last_refresh"] = ts
}
return summary
}
// apiVerifiedBotsApply (POST /api/v1/verified-bots/apply) validates and
// persists the whole verified_bots list to csm.yaml, then applies it live.
// The list is validated exactly as a config load would, so the same
// shared-hosting/over-broad-range guards apply here as on disk.
func (s *Server) apiVerifiedBotsApply(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
// Serialize read-validate-write-reload so concurrent saves cannot race.
configMu := integrity.ConfigWriteMutex()
configMu.Lock()
defer configMu.Unlock()
if rejectConfigWriteDuringRollback(w) {
return
}
ifMatch := r.Header.Get("If-Match")
if ifMatch == "" {
writeJSONError(w, "If-Match header required", http.StatusBadRequest)
return
}
var body struct {
Bots *[]config.VerifiedBot `json:"bots"`
}
if err := decodeJSONBodyLimited(w, r, 256*1024, &body); err != nil {
writeJSONError(w, "invalid body: "+err.Error(), http.StatusBadRequest)
return
}
if body.Bots == nil {
writeJSONError(w, "bots is required", http.StatusBadRequest)
return
}
bots := normalizeVerifiedBotsForSave(*body.Bots)
diskBytes, err := os.ReadFile(s.cfg.ConfigFile) // #nosec G304 -- operator-supplied config path
if err != nil {
writeJSONError(w, "read config: "+err.Error(), http.StatusInternalServerError)
return
}
disk, err := config.LoadBytes(diskBytes)
if err != nil {
writeJSONError(w, "parse config: "+err.Error(), http.StatusInternalServerError)
return
}
disk.ConfigFile = s.cfg.ConfigFile
disk.ConfigDir = s.cfg.ConfigDir
if disk.Integrity.ConfigHash != ifMatch {
writeJSONError(w, "config changed on disk, reload", http.StatusPreconditionFailed)
return
}
if rejectIfConfDirChanged(w, s.cfg.ConfigDir, disk) {
return
}
clone := cloneConfigForSettingsApply(disk)
clone.Reputation.VerifiedBots = bots
var verr []fieldError
for _, v := range config.Validate(&clone) {
if v.Level == "error" && strings.HasPrefix(v.Field, "reputation.verified_bots") {
verr = append(verr, fieldError{Field: v.Field, Message: v.Message})
}
}
if len(verr) > 0 {
writeValidationErrors(w, verr)
return
}
// YAMLEdit block-renders []interface{} via yaml.Marshal; a typed
// []config.VerifiedBot is not recognized, so wrap each entry.
botsVal := make([]interface{}, len(bots))
for i, b := range bots {
botsVal[i] = b
}
edited, err := config.YAMLEdit(diskBytes, []config.YAMLChange{
{Path: []string{"reputation", "verified_bots"}, Value: botsVal},
})
if err != nil {
writeJSONError(w, "yaml edit: "+err.Error(), http.StatusInternalServerError)
return
}
if err := integrity.SignAndSavePreserving(s.cfg.ConfigFile, s.cfg.ConfigDir, edited, &clone, disk.Integrity.BinaryHash); err != nil {
writeJSONError(w, "save: "+err.Error(), http.StatusInternalServerError)
return
}
s.auditLog(r, "verified_bots_save", "reputation.verified_bots", "csm.yaml rewritten")
newIntegrity := clone.Integrity
// reputation is a hot-reload-safe section: apply to the live config now.
if live := config.Active(); live != nil {
liveClone := *live
liveClone.Reputation.VerifiedBots = bots
applySignedIntegrityState(&liveClone, &clone)
config.SetActive(&liveClone)
} else {
config.SetActive(&clone)
}
// Push the new list into the running registry + verifier (no restart).
if s.verifiedBotsReloader != nil {
_ = s.verifiedBotsReloader()
}
writeJSON(w, map[string]interface{}{
"ok": true,
"count": len(bots),
"new_etag": newIntegrity.ConfigHash,
})
}
func normalizeVerifiedBotsForSave(in []config.VerifiedBot) []config.VerifiedBot {
out := make([]config.VerifiedBot, len(in))
copy(out, in)
for i := range out {
out[i].UASubstrings = nilIfEmptyStrings(out[i].UASubstrings)
out[i].RDNSSuffixes = nilIfEmptyStrings(out[i].RDNSSuffixes)
out[i].IPRanges = nilIfEmptyStrings(out[i].IPRanges)
}
return out
}
func nilIfEmptyStrings(v []string) []string {
if len(v) == 0 {
return nil
}
return v
}
// Package workerproc runs CSM's out-of-process workers: code that parses
// attacker-controlled input and can loop, crash or exhaust memory, so the
// daemon runs it in a child process it can kill.
//
// Daemon and worker share one Unix stream socket. A frame is a 4-byte
// big-endian length followed by a JSON envelope, and may carry descriptors
// with SCM_RIGHTS. The length prefix comes from the peer and is checked
// against a cap before any buffer is allocated.
package workerproc
import (
"bytes"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"os"
"syscall"
"golang.org/x/sys/unix"
)
// maxSendFDs is the kernel's SCM_MAX_FD: the most descriptors one sendmsg
// can carry.
const maxSendFDs = 253
// Frame is the envelope. A request carries an Op and a typed Payload; a
// response leaves Op empty and carries either a Payload or an Error.
type Frame struct {
Op string `json:"op,omitempty"`
Payload json.RawMessage `json:"payload,omitempty"`
Error string `json:"error,omitempty"`
}
// Limits bounds what one received frame may carry. MaxFrameBytes caps the
// body; MaxFDs caps descriptors, and 0 accepts none.
type Limits struct {
MaxFrameBytes int
MaxFDs int
}
// protocolError distinguishes a rejected peer frame from socket failure
// without changing the diagnostic text or hiding its underlying error.
type protocolError struct{ err error }
func (e protocolError) Error() string { return e.err.Error() }
func (e protocolError) Unwrap() error { return e.err }
// WriteFrame writes one length-prefixed frame.
func WriteFrame(w io.Writer, f Frame, maxBytes int) error {
body, err := marshalBody(f, maxBytes)
if err != nil {
return err
}
if err := writeFull(w, withHeader(body)); err != nil {
return fmt.Errorf("workerproc: write frame: %w", err)
}
return nil
}
// ReadFrame reads one length-prefixed frame.
//
// The length prefix comes from the peer, so it is checked against the cap
// BEFORE any buffer is allocated. Reading the declared size first and
// validating afterwards would let a four-byte write ask for gigabytes.
func ReadFrame(r io.Reader, maxBytes int) (Frame, error) {
var hdr [4]byte
if _, err := io.ReadFull(r, hdr[:]); err != nil {
return Frame{}, fmt.Errorf("workerproc: read header: %w", err)
}
size := binary.BigEndian.Uint32(hdr[:])
if int64(size) > int64(maxBytes) {
return Frame{}, protocolError{fmt.Errorf("workerproc: declared frame size %d exceeds cap %d", size, maxBytes)}
}
body := make([]byte, size)
if _, err := io.ReadFull(r, body); err != nil {
if errors.Is(err, io.EOF) {
// Only an end before the header is a clean end of stream.
err = io.ErrUnexpectedEOF
}
return Frame{}, fmt.Errorf("workerproc: read body: %w", err)
}
var f Frame
if err := json.Unmarshal(body, &f); err != nil {
return Frame{}, protocolError{fmt.Errorf("workerproc: unmarshal frame: %w", err)}
}
return f, nil
}
// SendFrame writes one frame to conn with fds attached. The caller keeps
// ownership of fds.
func SendFrame(conn *net.UnixConn, f Frame, fds []*os.File, maxBytes int) error {
body, err := marshalBody(f, maxBytes)
if err != nil {
return err
}
return sendBody(conn, body, fds)
}
// sendBody sends an already marshalled, already capped body.
func sendBody(conn *net.UnixConn, body []byte, fds []*os.File) error {
if len(fds) > maxSendFDs {
return fmt.Errorf("workerproc: %d descriptors exceed the kernel limit of %d", len(fds), maxSendFDs)
}
buf := withHeader(body)
n := 0
err := withRawFDs(fds, func(raw []int) error {
var oob []byte
if len(raw) > 0 {
oob = syscall.UnixRights(raw...)
}
// WriteMsgUnix is a single sendmsg, which a stream socket may cut
// short once its buffer fills. The descriptors went with the first
// byte; the rest of the frame follows as plain writes.
var writeErr error
n, _, writeErr = conn.WriteMsgUnix(buf, oob, nil)
return writeErr
})
if err == nil && n < len(buf) {
err = writeFull(conn, buf[n:])
}
if err != nil {
return fmt.Errorf("workerproc: send frame: %w", err)
}
return nil
}
// ReceiveFrame reads one frame from conn with any descriptors that came with
// it. On error every received descriptor is closed; on success the caller
// owns the returned files. After an error the connection must be closed:
// unread bytes and descriptors may remain from the rejected frame.
func ReceiveFrame(conn *net.UnixConn, lim Limits) (Frame, []*os.File, error) {
r := &rightsReader{
conn: conn,
limit: lim.MaxFDs,
oob: make([]byte, syscall.CmsgSpace(4*min(max(lim.MaxFDs, 1), maxSendFDs))),
}
f, err := ReadFrame(r, lim.MaxFrameBytes)
if err == nil {
// ReadFull drops an error returned with the last bytes it needed.
err = r.err
}
if err != nil {
closeFiles(r.files)
return Frame{}, nil, err
}
return f, r.files, nil
}
// rightsReader reads a stream socket and collects the SCM_RIGHTS descriptors
// that arrive along the way. A sender attaches descriptors to a frame's first
// byte, so they arrive on the first read; any later ones, or any past the
// limit, come from a peer trying to pile descriptors up and are closed at
// once.
type rightsReader struct {
conn *net.UnixConn
oob []byte
limit int
reads int
files []*os.File
err error
}
func (r *rightsReader) Read(p []byte) (int, error) {
if r.err != nil {
// Reject at once rather than wait for body bytes a hostile peer may
// never send.
return 0, r.err
}
n, oobn, flags, _, err := r.conn.ReadMsgUnix(p, r.oob)
r.reads++
if oobn > 0 {
r.collect(r.oob[:oobn])
}
if flags&syscall.MSG_CTRUNC != 0 && r.err == nil {
// Not a protocol error: the kernel also truncates when this process
// is out of descriptors, so it is no proof of a hostile worker.
r.err = errors.New("workerproc: descriptors truncated in transit")
}
if r.err != nil {
return n, r.err
}
if errors.Is(err, io.EOF) {
// net wraps EOF in an OpError; io.ReadFull only recognises it bare,
// and needs it to tell a clean end from a truncated frame.
err = io.EOF
}
return n, err
}
func (r *rightsReader) collect(oob []byte) {
// Parse one message at a time so a malformed later message cannot hide
// descriptors recvmsg already installed and that still need closing.
for len(oob) >= unix.CmsgLen(0) {
hdr, data, rest, err := unix.ParseOneSocketControlMessage(oob)
if err != nil {
r.fail(fmt.Errorf("workerproc: parse control message: %w", err))
return
}
oob = rest
if hdr.Level != unix.SOL_SOCKET || hdr.Type != unix.SCM_RIGHTS {
continue
}
fds, err := unix.ParseUnixRights(&unix.SocketControlMessage{Header: hdr, Data: data})
if err != nil {
r.fail(fmt.Errorf("workerproc: parse descriptors: %w", err))
continue
}
for _, fd := range fds {
switch {
case r.reads > 1:
r.fail(errors.New("workerproc: descriptors arrived after the start of a frame"))
case len(r.files) >= r.limit:
r.fail(fmt.Errorf("workerproc: frame carries more than %d descriptors", r.limit))
default:
r.files = append(r.files, os.NewFile(uintptr(fd), "received"))
continue
}
_ = unix.Close(fd)
}
}
}
func (r *rightsReader) fail(err error) {
if r.err == nil {
r.err = protocolError{err}
}
}
// marshalBody encodes without HTML escaping. Frames are never HTML, and
// escaping grows '<', '>' and '&' sixfold: a peer-influenced payload could
// otherwise be pushed past its frame cap. That covers a json.RawMessage
// payload too, which json.Marshal would escape again.
func marshalBody(f Frame, maxBytes int) ([]byte, error) {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
enc.SetEscapeHTML(false)
if err := enc.Encode(f); err != nil {
return nil, fmt.Errorf("workerproc: marshal frame: %w", err)
}
body := bytes.TrimSuffix(buf.Bytes(), []byte("\n"))
if len(body) > maxBytes {
return nil, fmt.Errorf("workerproc: frame body %d bytes exceeds cap %d", len(body), maxBytes)
}
return body, nil
}
func withHeader(body []byte) []byte {
buf := make([]byte, 4+len(body))
// #nosec G115 -- every caller has bounded len(body) by a frame cap far below 4 GiB.
binary.BigEndian.PutUint32(buf, uint32(len(body)))
copy(buf[4:], body)
return buf
}
func writeFull(w io.Writer, p []byte) error {
for len(p) != 0 {
n, err := w.Write(p)
if err != nil {
return err
}
if n <= 0 || n > len(p) {
return io.ErrShortWrite
}
p = p[n:]
}
return nil
}
func closeFiles(files []*os.File) {
for _, f := range files {
_ = f.Close()
}
}
func dupFiles(files []*os.File) ([]*os.File, error) {
if len(files) > maxSendFDs {
return nil, fmt.Errorf("workerproc: %d descriptors exceed the kernel limit of %d", len(files), maxSendFDs)
}
owned := make([]*os.File, 0, len(files))
for _, f := range files {
var fd int
err := withRawFDs([]*os.File{f}, func(raw []int) error {
var dupErr error
fd, dupErr = dupFD(raw[0])
return dupErr
})
if err != nil {
closeFiles(owned)
return nil, fmt.Errorf("workerproc: duplicate request descriptor: %w", err)
}
owned = append(owned, os.NewFile(uintptr(fd), "worker-request"))
}
return owned, nil
}
// dupFD is a seam so tests can exhaust descriptors on demand.
var dupFD = func(fd int) (int, error) { return unix.FcntlInt(uintptr(fd), unix.F_DUPFD_CLOEXEC, 0) }
// withRawFDs runs fn with the raw descriptors of files, each held open for
// the duration. It never calls Fd: that switches the descriptor to blocking
// mode, and O_NONBLOCK lives on the open file description the caller shares,
// so even a duplicate would leave the caller's pipe without deadlines.
func withRawFDs(files []*os.File, fn func([]int) error) error {
raw := make([]int, 0, len(files))
var hold func(i int) error
hold = func(i int) error {
if i == len(files) {
return fn(raw)
}
rc, err := files[i].SyscallConn()
if err != nil {
return err
}
var inner error
if err := rc.Control(func(fd uintptr) {
raw = append(raw, int(fd)) // #nosec G115 -- a descriptor always fits in int.
inner = hold(i + 1)
}); err != nil {
return err
}
return inner
}
return hold(0)
}
package workerproc
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"sync"
"time"
)
// OpHello is the first request on every worker connection. A worker built
// from a different version speaks a different protocol and must not get
// real requests.
const OpHello = "hello"
// Hello is the payload of a hello request and of its reply.
type Hello struct {
Protocol int `json:"protocol"`
}
// HelloReply is a worker's answer to OpHello.
func HelloReply(protocol int) Frame {
payload, _ := json.Marshal(Hello{Protocol: protocol}) // An int field cannot fail to marshal.
return Frame{Payload: payload}
}
// Handshake sends hello and checks the worker answers with protocol. The
// exchange honors ctx's cancellation and deadline, clearing the connection
// deadline afterwards. A reply with an op, an Error or any descriptor is
// refused. The caller must close the connection after a failed handshake.
func Handshake(ctx context.Context, conn *net.UnixConn, lim Limits, protocol int) (retErr error) {
if err := ctx.Err(); err != nil {
return err
}
if deadline, ok := ctx.Deadline(); ok {
if err := conn.SetDeadline(deadline); err != nil {
return fmt.Errorf("workerproc: hello: %w", err)
}
}
var canceled sync.WaitGroup
canceled.Add(1)
stop := context.AfterFunc(ctx, func() {
defer canceled.Done()
_ = conn.SetDeadline(time.Now())
})
defer func() {
// A cancellation callback already running must finish before the
// deadline is cleared, or it could poison the next exchange.
if !stop() {
canceled.Wait()
}
_ = conn.SetDeadline(time.Time{})
if ctxErr := ctx.Err(); ctxErr != nil {
retErr = ctxErr
}
}()
payload, err := json.Marshal(Hello{Protocol: protocol})
if err != nil {
return fmt.Errorf("workerproc: hello: %w", err)
}
if sendErr := SendFrame(conn, Frame{Op: OpHello, Payload: payload}, nil, lim.MaxFrameBytes); sendErr != nil {
return fmt.Errorf("workerproc: hello: %w", sendErr)
}
// MaxFDs 0: ReceiveFrame closes and refuses any descriptor.
reply, _, err := ReceiveFrame(conn, Limits{MaxFrameBytes: lim.MaxFrameBytes})
if err != nil {
return fmt.Errorf("workerproc: hello reply: %w", err)
}
switch {
case reply.Error != "":
return fmt.Errorf("workerproc: hello refused: %s", reply.Error)
case reply.Op != "":
return errors.New("workerproc: hello reply carries an op")
}
var h Hello
if err := json.Unmarshal(reply.Payload, &h); err != nil {
return fmt.Errorf("workerproc: hello reply: %w", err)
}
if h.Protocol != protocol {
return fmt.Errorf("workerproc: worker speaks protocol %d, want %d", h.Protocol, protocol)
}
return nil
}
package workerproc
import (
"context"
"errors"
"fmt"
"net"
"os"
"os/exec"
"syscall"
)
// Process is a started worker.
type Process interface {
// Conn is the parent's end of the worker socket.
Conn() *net.UnixConn
// Kill sends SIGKILL and never blocks.
Kill() error
// Done is closed once the process has exited and been reaped.
Done() <-chan struct{}
// PID is for logs and Stats only.
PID() int
// Retire is asked before reusing a worker and after each served
// request; true means the worker should take no more.
Retire() bool
}
// Launcher starts one worker.
type Launcher func(ctx context.Context) (Process, error)
// ExecLauncher starts command as a child process whose fd 0 and fd 1 are both
// its end of one Unix socket pair. ctx is checked before starting and not
// held: ending the request must not kill the worker, because whether a
// worker dies is the supervisor's decision.
func ExecLauncher(command string, args, env []string) Launcher {
return func(ctx context.Context) (Process, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
conn, childFile, err := workerSocket()
if err != nil {
return nil, err
}
defer childFile.Close()
cmd := exec.Command(command, args...) // #nosec G204 -- command comes from the daemon's own config, not from scanned content.
cmd.Env = env
cmd.Stdin = childFile
cmd.Stdout = childFile
if err := cmd.Start(); err != nil {
_ = conn.Close()
return nil, fmt.Errorf("workerproc: start worker: %w", err)
}
p := &execProcess{cmd: cmd, conn: conn, done: make(chan struct{})}
go func() {
_ = cmd.Wait()
close(p.done)
}()
return p, nil
}
}
type execProcess struct {
cmd *exec.Cmd
conn *net.UnixConn
done chan struct{}
}
func (p *execProcess) Conn() *net.UnixConn { return p.conn }
func (p *execProcess) Done() <-chan struct{} { return p.done }
func (p *execProcess) PID() int { return p.cmd.Process.Pid }
func (p *execProcess) Retire() bool { return false }
func (p *execProcess) Kill() error {
if err := p.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) {
return err
}
return nil
}
// workerSocket returns the parent's connection and the child's end of a new
// socket pair. The caller closes childFile once the child has it.
func workerSocket() (*net.UnixConn, *os.File, error) {
pair, err := socketpair()
if err != nil {
return nil, nil, fmt.Errorf("workerproc: socketpair: %w", err)
}
parentFile := os.NewFile(uintptr(pair[0]), "worker-parent")
childFile := os.NewFile(uintptr(pair[1]), "worker-child")
c, err := net.FileConn(parentFile)
_ = parentFile.Close()
if err != nil {
_ = childFile.Close()
return nil, nil, fmt.Errorf("workerproc: worker socket: %w", err)
}
return c.(*net.UnixConn), childFile, nil
}
// socketpair returns a close-on-exec Unix stream pair. Holding ForkLock keeps
// a concurrent exec from inheriting the descriptors before they are marked;
// SOCK_CLOEXEC would do this atomically, but it does not exist on every
// platform the packages are built on.
func socketpair() ([2]int, error) {
syscall.ForkLock.RLock()
defer syscall.ForkLock.RUnlock()
pair, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
if err != nil {
return [2]int{}, err
}
syscall.CloseOnExec(pair[0])
syscall.CloseOnExec(pair[1])
return pair, nil
}
package workerproc
import (
"context"
"errors"
"fmt"
"os"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
// PoolConfig describes a fixed set of worker slots.
type PoolConfig struct {
Size int
// Config gives slot 0..Size-1 its supervisor configuration.
Config func(slot int) Config
}
// Pool is a fixed set of supervisors, one per slot. Callers own slots: a
// slot is used by one goroutine, so nobody waits on another caller's worker.
// Memory a worker hands back (memfds) stays charged to that worker's cgroup,
// so a caller that holds its last reply while it works must keep using the
// same slot, never one whose cgroup another caller is filling.
type Pool struct {
slots []*Supervisor
}
// NewPool starts no worker; each slot's first Call does.
func NewPool(cfg PoolConfig) (*Pool, error) {
if cfg.Size < 1 {
return nil, errors.New("workerproc: pool size must be positive")
}
p := &Pool{slots: make([]*Supervisor, cfg.Size)}
for slot := range p.slots {
s, err := New(cfg.Config(slot))
if err != nil {
return nil, fmt.Errorf("workerproc: pool slot %d: %w", slot, err)
}
p.slots[slot] = s
}
return p, nil
}
// Size is the number of slots.
func (p *Pool) Size() int { return len(p.slots) }
// Call runs req on slot's worker. slot must be in 0..Size-1.
func (p *Pool) Call(ctx context.Context, slot int, req Frame, fds []*os.File, check Check) (Frame, []*os.File, error) {
return p.slots[slot].Call(ctx, req, fds, check)
}
// Stop stops every slot.
func (p *Pool) Stop() error {
for _, s := range p.slots {
_ = s.Stop()
}
return nil
}
// Stats is each slot's supervisor state, by slot.
func (p *Pool) Stats() []Stats {
out := make([]Stats, len(p.slots))
for slot, s := range p.slots {
out[slot] = s.Stats()
}
return out
}
// QueueStatuses folds the slots into one requests row.
func (p *Pool) QueueStatuses(now time.Time) map[string]queuehealth.Status {
total := queuehealth.Status{Status: "ok", Capacity: len(p.slots)}
for _, s := range p.slots {
row := s.QueueStatuses(now)["requests"]
total.Depth += row.Depth
total.InFlight += row.InFlight
total.DroppedTotal += row.DroppedTotal
total.RecentDrops += row.RecentDrops
total.DroppedLowerBound = total.DroppedLowerBound || row.DroppedLowerBound
total.LagSeconds = max(total.LagSeconds, row.LagSeconds)
total.ProcessingSeconds = max(total.ProcessingSeconds, row.ProcessingSeconds)
if row.Status != "ok" && total.Status == "ok" {
total.Status, total.Reason = row.Status, row.Reason
}
}
return map[string]queuehealth.Status{"requests": total}
}
package workerproc
import (
"sync"
"time"
"github.com/pidginhost/csm/internal/queuehealth"
)
type requestQueue struct {
mu sync.Mutex
pending map[*requestWork]struct{}
losses *queuehealth.Tracker
}
type requestWork struct {
queue *requestQueue
queued, started, phaseDeadline, rpcDeadline time.Time
callerDone, rpcOutstanding, awaitingRPC, failed bool
}
func newRequestQueue() *requestQueue {
return &requestQueue{pending: make(map[*requestWork]struct{}), losses: queuehealth.New(0, time.Minute)}
}
func (q *requestQueue) begin() *requestWork {
w := &requestWork{queue: q, queued: time.Now()}
q.mu.Lock()
q.pending[w] = struct{}{}
q.mu.Unlock()
return w
}
func (w *requestWork) start() {
w.queue.mu.Lock()
w.started = time.Now()
w.phaseDeadline = w.started.Add(time.Minute)
w.queue.mu.Unlock()
}
func (w *requestWork) beginRPC(deadline time.Time) {
w.queue.mu.Lock()
w.rpcOutstanding = true
w.awaitingRPC = true
w.rpcDeadline, w.phaseDeadline = deadline, deadline
w.queue.mu.Unlock()
}
func (w *requestWork) progress() {
w.queue.mu.Lock()
w.awaitingRPC = false
w.phaseDeadline = time.Now().Add(time.Minute)
w.queue.mu.Unlock()
}
func (w *requestWork) finishRPC(completed bool) {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
w.rpcOutstanding = false
if w.awaitingRPC {
w.awaitingRPC = false
w.phaseDeadline = time.Now().Add(time.Minute)
}
if !completed {
w.failLocked()
}
w.releaseLocked()
}
func (w *requestWork) finishCaller(completed bool) {
q := w.queue
q.mu.Lock()
defer q.mu.Unlock()
w.callerDone = true
if !completed {
w.failLocked()
}
w.releaseLocked()
}
func (w *requestWork) releaseLocked() {
// Closing the socket normally releases the RPC goroutine. A caller that
// already returned must not hide an operation still blocked in I/O.
if w.callerDone && !w.rpcOutstanding {
delete(w.queue.pending, w)
}
}
func (w *requestWork) fail() {
w.queue.mu.Lock()
w.failLocked()
w.queue.mu.Unlock()
}
func (w *requestWork) failLocked() {
if !w.failed {
w.failed = true
w.queue.losses.Lose(time.Now(), 1)
}
}
// QueueStatuses does not take the supervisor lock or inspect child processes.
func (s *Supervisor) QueueStatuses(now time.Time) map[string]queuehealth.Status {
q := s.health
q.mu.Lock()
defer q.mu.Unlock()
status := q.losses.Snapshot(now)
status.CapacityUnavailable = true
var waitingLate, runningLate bool
for w := range q.pending {
if w.started.IsZero() {
status.Depth++
status.LagSeconds = max(status.LagSeconds, now.Sub(w.queued).Seconds())
waitingLate = waitingLate || now.Sub(w.queued) >= time.Minute
} else {
status.InFlight++
status.ProcessingSeconds = max(status.ProcessingSeconds, now.Sub(w.started).Seconds())
runningLate = runningLate || (!w.callerDone && !now.Before(w.phaseDeadline)) || (w.rpcOutstanding && !now.Before(w.rpcDeadline))
}
}
switch {
case waitingLate:
status.Reason = "backlog_lag"
case runningLate:
status.Reason = "processing_lag"
}
if status.Reason != "" {
status.Status = "degraded"
}
return map[string]queuehealth.Status{"requests": status}
}
package workerproc
import (
"context"
"errors"
"fmt"
"io"
"net"
"os"
"syscall"
)
// Handler answers one request. It owns neither the request descriptors,
// which Serve closes after it returns, nor the reply descriptors, which
// Serve closes after sending them.
type Handler func(ctx context.Context, req Frame, fds []*os.File) (Frame, []*os.File)
// Serve answers request frames on conn until the parent closes it. It checks
// ctx between frames, not from inside the handler; the parent process remains
// the sole liveness boundary. It is the whole worker loop: one request at a
// time, no concurrency, no state carried between requests.
//
// A reply never carries an op: the parent matches replies by order on a
// single-flight connection, and echoing the op would invite a reader to match
// on it instead. Broken framing ends the loop because a trustworthy boundary
// for the next request no longer exists.
func Serve(ctx context.Context, conn *net.UnixConn, lim Limits, h Handler) error {
for {
if err := ctx.Err(); err != nil {
return err
}
req, fds, err := ReceiveFrame(conn, lim)
if err != nil {
if errors.Is(err, io.EOF) {
return nil
}
return err
}
resp, respFDs := h(ctx, req, fds)
closeFiles(fds)
resp.Op = ""
err = SendFrame(conn, resp, respFDs, lim.MaxFrameBytes)
closeFiles(respFDs)
if err != nil {
return err
}
}
}
// ChildConn returns the connection a launcher handed this worker on fd 0.
// A worker started by hand has a terminal or a pipe there, and reading frames
// from it would only produce garbage, so anything but a socket is refused.
func ChildConn() (*net.UnixConn, error) {
var st syscall.Stat_t
if err := syscall.Fstat(0, &st); err != nil {
return nil, fmt.Errorf("workerproc: fd 0: %w", err)
}
if st.Mode&syscall.S_IFMT != syscall.S_IFSOCK {
return nil, errors.New("workerproc: fd 0 is not a socket; workers are started by the daemon")
}
c, err := net.FileConn(os.Stdin)
if err != nil {
return nil, fmt.Errorf("workerproc: fd 0: %w", err)
}
uc, ok := c.(*net.UnixConn)
if !ok {
_ = c.Close()
return nil, errors.New("workerproc: fd 0 is not a Unix socket")
}
return uc, nil
}
package workerproc
import (
"sync"
"time"
)
const (
// StrikeLimit is how many worker failures within StrikeTTL block an
// input. An input that kills a worker every time would otherwise cost a
// worker on every scan, and keep its check incomplete forever.
StrikeLimit = 2
// StrikeTTL is how long a failure counts against an input.
StrikeTTL = 24 * time.Hour
// maxStrikeKeys bounds memory; when full, the input whose last failure
// is oldest is forgotten.
maxStrikeKeys = 1024
)
// Strikes remembers inputs, by key, that made a worker fail.
type Strikes struct {
mu sync.Mutex
entries map[string]strike
now func() time.Time
limit int
ttl time.Duration
}
type strike struct {
count int
last time.Time
}
// NewStrikes returns an empty record that blocks after StrikeLimit failures
// within StrikeTTL.
func NewStrikes() *Strikes { return NewStrikesWith(StrikeLimit, StrikeTTL) }
// NewStrikesWith returns an empty record that blocks a key after limit
// failures, each within ttl of the one before.
func NewStrikesWith(limit int, ttl time.Duration) *Strikes {
return &Strikes{entries: make(map[string]strike), now: time.Now, limit: limit, ttl: ttl}
}
// Record counts one worker failure on key.
func (s *Strikes) Record(key string) {
s.mu.Lock()
defer s.mu.Unlock()
now := s.now()
e, ok := s.entries[key]
if ok && now.Sub(e.last) > s.ttl {
e = strike{}
}
if !ok && len(s.entries) >= maxStrikeKeys {
// A worker failure is rare enough to scan this bounded table.
oldest := ""
for k, v := range s.entries {
if oldest == "" || v.last.Before(s.entries[oldest].last) {
oldest = k
}
}
delete(s.entries, oldest)
}
e.count++
e.last = now
s.entries[key] = e
}
// Blocked reports the limit of failures on key reached, the last within the
// TTL.
func (s *Strikes) Blocked(key string) bool {
s.mu.Lock()
defer s.mu.Unlock()
e, ok := s.entries[key]
return ok && e.count >= s.limit && s.now().Sub(e.last) <= s.ttl
}
// Len is how many inputs have unexpired failures. Dropping expired entries
// lets callers avoid hashing every input once the failure window has passed.
func (s *Strikes) Len() int {
s.mu.Lock()
defer s.mu.Unlock()
now := s.now()
for key, entry := range s.entries {
if now.Sub(entry.last) > s.ttl {
delete(s.entries, key)
}
}
return len(s.entries)
}
package workerproc
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"net"
"os"
"sync"
"syscall"
"time"
)
// ConsecutiveFailureLimit is how many failed workers in a row are tolerated
// before the supervisor pauses spawning replacements for one cooldown.
//
// It is a constant rather than a setting because nothing an operator can
// observe would tell them a better value: the right number depends on how the
// parser fails, not on the host. It is also harmful in both directions -- too
// low and one transient failure blinds the rest of a scan, too high and a
// crafted account gets the exec storm the limit exists to prevent. When it
// trips, the coverage gap says so, so the condition stays visible even though
// the threshold is not tunable.
const ConsecutiveFailureLimit = 3
// BreakerCooldown is how long the supervisor refuses to spawn replacements
// after ConsecutiveFailureLimit failures in a row, before it tries once more.
//
// The breaker bounds an exec storm; it must not switch the detector off. With
// no way back it would be a kill switch any tenant could throw with a handful
// of crafted files, disabling analysis across a shared host for the life of
// the daemon -- worse than not deploying the detector, because an operator
// would believe it was running. One trial per cooldown keeps the storm bounded
// (a hostile account costs one timeout per cooldown, not one per file) while
// letting a host recover without an operator having to notice and restart.
//
// The cost of each retry lands on the shared deep-scan budget, which the YARA
// and JavaScript consumers walk too, so a longer cooldown buys their coverage
// back on a host that stays broken.
const BreakerCooldown = 60 * time.Second
// reapTimeout bounds how long killLocked waits for a SIGKILLed child while
// holding the supervisor lock.
const reapTimeout = 10 * time.Second
var (
ErrStopped = errors.New("supervisor stopped")
ErrBreakerOpen = errors.New("worker failing repeatedly")
ErrTimeout = errors.New("deadline exceeded; worker killed")
ErrSpawn = errors.New("worker did not start")
ErrFrameTooLarge = errors.New("request exceeds frame cap")
// ErrWorkerFailed marks a failure the worker caused: a timeout, a broken
// socket, an Error frame, or a reply the caller rejected. Errors that
// carry it have their own text; this sentinel only classifies them.
ErrWorkerFailed = errors.New("worker failed")
)
// Failure reasons: why a worker was given up on.
const (
ReasonTimeout = "timeout"
ReasonProtocol = "protocol"
ReasonCrash = "crash"
ReasonOOM = "oom"
ReasonSpawn = "spawn"
)
// Job outcomes reported to Config.OnJob besides the failure reasons.
const (
OutcomeOK = "ok"
OutcomeRefused = "refused"
)
// memoryReporter is implemented by processes that know their memory use.
type memoryReporter interface{ MemoryUsed() uint64 }
// failure tags a counted failure with its reason without changing its text,
// which callers already report as coverage-gap reasons. Only round-trip
// failures match ErrWorkerFailed; a failed launch is ErrSpawn.
type failure struct {
err error
reason string
}
func (e failure) Error() string { return e.err.Error() }
func (e failure) Unwrap() error { return e.err }
func (e failure) Is(target error) bool { return target == ErrWorkerFailed && e.reason != ReasonSpawn }
// FailureReason is the reason a failed Call gave up on its worker, or "" for
// an error that is not a counted failure: cancellation, a stopped
// supervisor, the breaker, the daemon's own descriptor exhaustion.
func FailureReason(err error) string {
var f failure
if errors.As(err, &f) {
return f.reason
}
return ""
}
// oomReporter is implemented by processes that can tell a memory kill from
// a crash.
type oomReporter interface{ OOMKilled() bool }
// localExhaustion reports the daemon running out of descriptors. That is not
// the worker's fault, and outside load can push the daemon there, so it must
// not count toward the breaker. Before a request starts it leaves a healthy
// worker running; a mid-frame failure still requires retiring the socket.
func localExhaustion(err error) bool {
return errors.Is(err, syscall.EMFILE) || errors.Is(err, syscall.ENFILE)
}
// Config describes one kind of worker.
type Config struct {
// Name prefixes errors and log lines, e.g. "phptaint worker".
Name string
Launch Launcher
// Timeout bounds a single request. It must exceed the slowest legitimate
// request by a wide margin: every expiry costs a killed process.
Timeout time.Duration
// Limits applies to replies. A request over MaxFrameBytes is refused.
Limits Limits
// Log is optional.
Log func(string, ...any)
// MaxJobs retires a worker after that many requests; 0 means no limit.
MaxJobs int
// IdleTimeout stops a worker left unused that long; 0 means never.
IdleTimeout time.Duration
// OnFailure is told of every counted failure, with the supervisor lock
// held: it must not block. Optional.
OnFailure func(reason string, err error)
// OnJob is told the outcome of every request that reached the breaker:
// OutcomeOK with the worker's memory use after the job (0 when
// unknown), a failure reason, or OutcomeRefused when the breaker
// refused it. Cancellation, a stopped supervisor and the daemon's own
// descriptor exhaustion are not reported. It runs with the supervisor
// lock held and must not block. Optional.
OnJob func(outcome string, memory uint64)
// BreakOnSpawnOnly lets only failed starts open the breaker. It is for
// workers whose callers cannot skip a refused request: there the breaker
// would let hostile input that kills a few workers switch the worker off
// for unrelated requests. Each request then costs at most one spawn.
BreakOnSpawnOnly bool
}
// Check validates a reply. err rejects it: the worker is killed and the
// failure counts toward the breaker. lost marks a valid reply that reports
// work not done; it counts toward request health only.
type Check func(reply Frame, fds []*os.File) (lost bool, err error)
// Stats is a point-in-time view for tests and status pages.
type Stats struct {
Spawns, LastPID, ConsecutiveFailures int
BreakerOpenedAt time.Time
BreakerOpen bool
// Running reports a worker process attached.
Running bool
// Kills counts counted failures by reason.
Kills map[string]uint64
}
// Supervisor runs one worker process at a time and guarantees that a request
// which does not return kills the process it was running in.
//
// The guarantee is the point. A deadline on its own is not containment: the
// call returns to the caller while the child keeps spinning on a core, and the
// next request starts another one. That is the failure mode of a timeout
// without a kill.
//
// A worker that misses its deadline, or fails in any other way, is poisoned:
// killed, reaped, and never used again. Replacement is lazy, so a scan with no
// further requests pays nothing for the last request's failure.
type Supervisor struct {
cfg Config
health *requestQueue
mu sync.Mutex
child *child
lastPID int
spawns int
consecutive int
// openedAt is when the breaker last opened; the zero value means closed.
openedAt time.Time
stopped bool
// jobs counts requests served by the current child.
jobs int
lastUse time.Time
idle *time.Timer
kills map[string]uint64
// now is a seam so the cooldown can be exercised without sleeping.
now func() time.Time
}
type child struct {
proc Process
conn transport
}
// transport is the parent's side of a worker connection. Tests replace it.
type transport interface {
send(body []byte, fds []*os.File) error
receive() (Frame, []*os.File, error)
close() error
}
type unixTransport struct {
conn *net.UnixConn
lim Limits
}
func (t unixTransport) send(body []byte, fds []*os.File) error { return sendBody(t.conn, body, fds) }
func (t unixTransport) receive() (Frame, []*os.File, error) { return ReceiveFrame(t.conn, t.lim) }
func (t unixTransport) close() error { return t.conn.Close() }
// New validates cfg. It does not start a worker: the first request does, so
// a scan that never needs one never forks.
func New(cfg Config) (*Supervisor, error) {
if cfg.Launch == nil {
return nil, errors.New("workerproc: nil launcher")
}
if cfg.Timeout <= 0 {
return nil, errors.New("workerproc: timeout must be positive")
}
if cfg.Limits.MaxFrameBytes <= 0 {
return nil, errors.New("workerproc: frame cap must be positive")
}
return &Supervisor{cfg: cfg, now: time.Now, health: newRequestQueue()}, nil
}
// Stop shuts down the current worker and refuses later requests.
func (s *Supervisor) Stop() error {
s.mu.Lock()
defer s.mu.Unlock()
s.stopped = true
s.killLocked()
return nil
}
// Stats reports spawn and breaker state.
func (s *Supervisor) Stats() Stats {
s.mu.Lock()
defer s.mu.Unlock()
return Stats{
Spawns: s.spawns,
LastPID: s.lastPID,
ConsecutiveFailures: s.consecutive,
BreakerOpenedAt: s.openedAt,
BreakerOpen: s.breakerOpenLocked(),
Running: s.child != nil,
Kills: maps.Clone(s.kills),
}
}
// Call sends req with fds to the worker and returns its reply. The caller
// keeps ownership of fds and owns the returned descriptors. It may close or
// reuse fds after Call returns, even if a canceled send has not yet finished.
// A reply with an Error or an Op set, or one check rejects, is a worker failure.
func (s *Supervisor) Call(ctx context.Context, req Frame, fds []*os.File, check Check) (Frame, []*os.File, error) {
return s.CallGuarded(ctx, req, fds, check, nil, nil)
}
// CallGuarded runs before and after under the same lock as Call. This lets
// callers refuse inputs based on preceding requests, and record the result
// before the next request checks that state. A before error refuses work
// without starting or killing a worker; after runs only if before succeeds.
// Neither callback may call methods that take the supervisor lock.
func (s *Supervisor) CallGuarded(ctx context.Context, req Frame, fds []*os.File, check Check, before func() error, after func(error)) (Frame, []*os.File, error) {
work := s.health.begin()
completed := false
defer func() { work.finishCaller(completed) }()
s.mu.Lock()
defer s.mu.Unlock()
work.start()
if s.stopped {
completed = true
return Frame{}, nil, ErrStopped
}
if err := ctx.Err(); err != nil {
completed = true
return Frame{}, nil, err
}
if before != nil {
if err := before(); err != nil {
work.fail()
completed = true
return Frame{}, nil, err
}
}
reply, replyFDs, err := s.callLocked(ctx, req, fds, check, work)
if after != nil {
after(err)
}
completed = true
return reply, replyFDs, err
}
// armIdleLocked restarts the idle countdown after a served request. The timer
// cannot stop a worker mid-request: stopIdle needs the lock Call holds.
func (s *Supervisor) armIdleLocked() {
if s.cfg.IdleTimeout <= 0 || s.stopped || s.child == nil {
return
}
s.lastUse = s.now()
if s.idle == nil {
s.idle = time.AfterFunc(s.cfg.IdleTimeout, s.stopIdle)
return
}
s.idle.Reset(s.cfg.IdleTimeout)
}
func (s *Supervisor) stopIdle() {
s.mu.Lock()
defer s.mu.Unlock()
// A request that ran while the timer fired has moved lastUse on and
// re-armed the timer.
if s.stopped || s.child == nil || s.now().Sub(s.lastUse) < s.cfg.IdleTimeout {
return
}
s.logf("%s %d idle", s.cfg.Name, s.child.proc.PID())
s.killLocked()
}
func (s *Supervisor) callLocked(ctx context.Context, req Frame, fds []*os.File, check Check, work *requestWork) (Frame, []*os.File, error) {
if s.stopped {
return Frame{}, nil, ErrStopped
}
if ctxErr := ctx.Err(); ctxErr != nil {
return Frame{}, nil, ctxErr
}
// Marshalled once: these are the bytes the worker gets.
body, err := json.Marshal(req)
if err != nil {
return Frame{}, nil, fmt.Errorf("%s: marshal request: %w", s.cfg.Name, err)
}
if len(body) > s.cfg.Limits.MaxFrameBytes {
// The caller's own ceiling, not a worker fault, so it must not count
// toward the breaker.
return Frame{}, nil, fmt.Errorf("%w: %d > %d bytes", ErrFrameTooLarge, len(body), s.cfg.Limits.MaxFrameBytes)
}
if ctxErr := ctx.Err(); ctxErr != nil {
return Frame{}, nil, ctxErr
}
// Duplicated before any launch or send, so that running out of
// descriptors is seen as the daemon's condition, not the worker's. The
// copies also outlive a canceled call: the send goroutine owns them.
ownedFDs, err := dupFiles(fds)
if err != nil {
work.fail()
return Frame{}, nil, fmt.Errorf("%s: %w", s.cfg.Name, err)
}
handedOff := false
defer func() {
if !handedOff {
closeFiles(ownedFDs)
}
}()
if ctxErr := ctx.Err(); ctxErr != nil {
return Frame{}, nil, ctxErr
}
if s.breakerOpenLocked() {
work.fail()
s.reportJobLocked(OutcomeRefused, 0)
return Frame{}, nil, fmt.Errorf("%w: %d failures in a row, next attempt after %s",
ErrBreakerOpen, s.consecutive, BreakerCooldown)
}
if s.child != nil {
select {
case <-s.child.proc.Done():
// It ended on its own: a lifetime cap, a watchdog, the kernel.
// Not this request's fault, and nothing to count against it.
s.logf("%s %d exited", s.cfg.Name, s.child.proc.PID())
s.killLocked()
default:
// An idle worker may have reached its retirement deadline
// since the last reply. Replace it before sending new input.
if s.child.proc.Retire() {
s.logf("%s %d retired", s.cfg.Name, s.child.proc.PID())
s.killLocked()
}
}
}
if s.child == nil {
proc, launchErr := s.cfg.Launch(ctx)
if launchErr != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
// The caller withdrew during the launch: not a worker fault,
// though a half-open trial has been consumed. Only the
// caller's own context counts; a launcher timing out on its
// own is a failed start.
s.rearmBreakerLocked()
return Frame{}, nil, ctxErr
}
work.fail()
if localExhaustion(launchErr) {
return Frame{}, nil, fmt.Errorf("%w: %w", ErrSpawn, launchErr)
}
// The breaker bounds how often this logs: three in a row, then
// one trial per cooldown.
s.logf("%s did not start: %q", s.cfg.Name, launchErr)
err := failure{err: fmt.Errorf("%w: %v", ErrSpawn, launchErr), reason: ReasonSpawn}
s.recordFailureLocked(err)
return Frame{}, nil, err
}
s.child = &child{proc: proc, conn: unixTransport{conn: proc.Conn(), lim: s.cfg.Limits}}
s.jobs = 0
s.spawns++
s.lastPID = proc.PID()
if s.cfg.BreakOnSpawnOnly {
// Input can kill this worker next, but its successful start has
// already proved that the preceding startup outage is over.
s.consecutive = 0
}
}
handedOff = true
reply, replyFDs, lost, reason, err := s.roundTripLocked(ctx, body, ownedFDs, check, work)
if err != nil {
if reason == ReasonCrash && !localExhaustion(err) {
// Asked before the kill below: the unit's cgroup, which holds
// the evidence, goes away with it.
if r, ok := s.child.proc.(oomReporter); ok && r.OOMKilled() {
reason = ReasonOOM
}
}
// A transport or validator can return its own context error. Only
// the caller withdrawing exempts a failure from the breaker.
canceled := ctx.Err() != nil && (errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded))
if !canceled {
work.fail()
}
// The child is not trusted after any failure: it may be mid-parse and
// spinning, or it may have left a partial frame in the socket that
// would desynchronise every later request.
s.killLocked()
if localExhaustion(err) {
// The socket may be mid-frame, so retire it, but never charge
// the input for descriptor pressure in this process.
s.rearmBreakerLocked()
return Frame{}, nil, err
}
if canceled {
// The caller withdrew, so this is not a worker fault and must not
// count toward the breaker. It did consume the trial, though: the
// cooldown has to be re-armed or a caller that cancels can spend
// one spawn per request while the breaker is nominally open.
s.rearmBreakerLocked()
return Frame{}, nil, err
}
f := failure{err: err, reason: reason}
s.recordFailureLocked(f)
return Frame{}, nil, f
}
s.consecutive = 0
if lost {
work.fail()
}
// Retiring a healthy worker is not a failure: the reply stands and
// nothing counts toward the breaker.
s.jobs++
// Retire reads the worker's memory, so ask it first, every time.
retire := s.child.proc.Retire() || (s.cfg.MaxJobs > 0 && s.jobs >= s.cfg.MaxJobs)
var memory uint64
if m, ok := s.child.proc.(memoryReporter); ok {
memory = m.MemoryUsed()
}
s.reportJobLocked(OutcomeOK, memory)
if retire {
s.logf("%s %d retired", s.cfg.Name, s.child.proc.PID())
s.killLocked()
}
s.armIdleLocked()
return reply, replyFDs, nil
}
// recordFailureLocked counts a worker failure and, once the run of failures
// reaches the limit, restarts the cooldown. Stamping on every failure at or
// past the limit -- a failed trial request included -- is what makes a
// persistently broken host retry once per cooldown instead of once per file.
func (s *Supervisor) recordFailureLocked(f failure) {
if !s.cfg.BreakOnSpawnOnly || f.reason == ReasonSpawn {
s.consecutive++
s.rearmBreakerLocked()
}
if s.kills == nil {
s.kills = make(map[string]uint64)
}
s.kills[f.reason]++
if s.cfg.OnFailure != nil {
s.cfg.OnFailure(f.reason, f)
}
s.reportJobLocked(f.reason, 0)
}
func (s *Supervisor) reportJobLocked(outcome string, memory uint64) {
if s.cfg.OnJob != nil {
s.cfg.OnJob(outcome, memory)
}
}
// rearmBreakerLocked restarts the cooldown when the breaker is at or past its
// limit. openedAt is never cleared on success: breakerOpenLocked reads it only
// once consecutive has reached the limit, and every path that reaches the limit
// stamps it first, so a stale value cannot be observed.
func (s *Supervisor) rearmBreakerLocked() {
if s.consecutive >= ConsecutiveFailureLimit {
s.openedAt = s.now()
}
}
// breakerOpenLocked reports whether this request must be refused without
// spawning. Once the cooldown elapses the breaker goes half-open: the request
// is let through as a trial, which either resets the counter on success or
// restarts the cooldown on failure.
func (s *Supervisor) breakerOpenLocked() bool {
if s.consecutive < ConsecutiveFailureLimit {
return false
}
return s.now().Sub(s.openedAt) < BreakerCooldown
}
// roundTripLocked writes one request and waits for its reply, bounded by the
// configured timeout. The socket round trip runs on its own goroutine because
// the child may stop before reading the complete request or never answer. The
// goroutine ends when killLocked closes the socket; a reply it reads after
// the caller gave up has its descriptors closed there. The goroutine owns
// ownedFDs and closes them once sent: cancellation can return before it
// reaches sendmsg, so they must not be the caller's files.
func (s *Supervisor) roundTripLocked(ctx context.Context, body []byte, ownedFDs []*os.File, check Check, work *requestWork) (Frame, []*os.File, bool, string, error) {
c := s.child
type result struct {
frame Frame
fds []*os.File
err error
}
deadline := time.Now().Add(s.cfg.Timeout)
timer := time.NewTimer(s.cfg.Timeout)
defer timer.Stop()
if parent, ok := ctx.Deadline(); ok && parent.Before(deadline) {
deadline = parent
}
work.beginRPC(deadline)
defer work.progress()
results := make(chan result)
abandon := make(chan struct{})
defer close(abandon)
go func() {
var res result
completed := false
func() {
// The socket work is over once the reply is in hand, so request
// health stops charging it to the RPC deadline before the
// handoff, not after.
defer func() { work.finishRPC(completed) }()
sendErr := func() error {
defer closeFiles(ownedFDs)
return c.conn.send(body, ownedFDs)
}()
if sendErr != nil {
res.err = fmt.Errorf("write request: %w", sendErr)
} else if res.frame, res.fds, res.err = c.conn.receive(); res.err != nil {
res.err = fmt.Errorf("read reply: %w", res.err)
}
completed = true
}()
select {
case results <- res:
case <-abandon:
closeFiles(res.fds)
}
}()
select {
case <-timer.C:
return Frame{}, nil, false, ReasonTimeout, fmt.Errorf("%s: %w after %s", s.cfg.Name, ErrTimeout, s.cfg.Timeout)
case <-ctx.Done():
return Frame{}, nil, false, "", ctx.Err()
case res := <-results:
work.progress()
defer func() { closeFiles(res.fds) }()
if res.err != nil {
reason := ReasonCrash
var invalid protocolError
if errors.As(res.err, &invalid) {
reason = ReasonProtocol
}
return Frame{}, nil, false, reason, fmt.Errorf("%s: %w", s.cfg.Name, res.err)
}
var err error
lost := false
switch {
case res.frame.Error != "":
err = fmt.Errorf("%s: worker: %q", s.cfg.Name, res.frame.Error)
case res.frame.Op != "":
err = fmt.Errorf("%s: response carries op %q", s.cfg.Name, res.frame.Op)
default:
if lost, err = check(res.frame, res.fds); err != nil {
err = fmt.Errorf("%s: %w", s.cfg.Name, err)
}
}
if err != nil {
return Frame{}, nil, false, ReasonProtocol, err
}
replyFDs := res.fds
res.fds = nil // Ownership passes to the caller only on success.
return res.frame, replyFDs, lost, "", nil
}
}
// killLocked terminates the current worker and waits for it to be reaped.
//
// SIGKILL, not SIGTERM: the process this exists to stop is spinning inside a
// parser loop and never returns to a point where a catchable signal would be
// handled. Closing the socket unblocks the round-trip goroutine so it cannot
// outlive the child.
func (s *Supervisor) killLocked() {
if s.idle != nil {
s.idle.Stop()
s.idle = nil
}
c := s.child
if c == nil {
return
}
s.child = nil
_ = c.proc.Kill()
_ = c.conn.close()
// Bounded, because this runs while the supervisor lock is held: an
// unbounded wait would turn one unreapable child into a frozen supervisor
// for every later request. SIGKILL cannot be caught or ignored, so the
// only way to reach the timeout is a process wedged in uninterruptible
// sleep. Giving up on the reap leaks one process; blocking here would
// stop the scan, and the scan is what protects the host.
select {
case <-c.proc.Done():
case <-time.After(reapTimeout):
s.logf("%s %d did not exit after SIGKILL", s.cfg.Name, c.proc.PID())
return
}
// Stop is the only caller with stopped set: a worker ended on purpose,
// not one killed for a fault or a retirement already logged.
ended := "killed"
if s.stopped {
ended = "stopped"
}
s.logf("%s %d %s", s.cfg.Name, c.proc.PID(), ended)
}
func (s *Supervisor) logf(format string, args ...any) {
if s.cfg.Log != nil {
s.cfg.Log(format, args...)
}
}
package workerproc
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"unicode"
"github.com/pidginhost/csm/internal/systemdrun"
)
// UnitConfig describes workers run as locked-down transient systemd units.
type UnitConfig struct {
// Prefix names the units: Prefix + "-" + a generation number. Each
// supervisor needs a distinct prefix because it owns the slice's count.
Prefix string
Command string
Args []string
// MemoryLimit caps memory and swap together, in bytes.
MemoryLimit uint64
// FileSizeLimit is LimitFSIZE. It bounds the worker's memfds too, so it
// must exceed the largest reply a worker writes.
FileSizeLimit uint64
RuntimeMax time.Duration
// StartTimeout bounds unit start, hello and the memory cap together.
StartTimeout time.Duration
Protocol int
// Limits applies to the hello reply.
Limits Limits
Log func(string, ...any)
// InaccessiblePaths are hidden from the worker on top of ProtectHome:
// account homes on other volumes. Paths that are not absolute and clean,
// or that hold spaces, quotes, backslashes, percent signs or control
// bytes, are dropped.
InaccessiblePaths []string
// AllowExecutableMemory turns MemoryDenyWriteExecute off, for YARA-X:
// wasmtime maps its compiled rules executable and cannot load them
// otherwise. Every other worker keeps the denial.
AllowExecutableMemory bool
hooks *unitHooks // tests only; nil means the real system
}
// unitHooks are the system calls the launcher makes, replaced in tests.
type unitHooks struct {
systemdRun string
configureSlice func(ctx context.Context, name string) error
mainPID func(ctx context.Context, p *unitProcess) (int, error)
killUnit func(ctx context.Context, p *unitProcess) error
procRoot, cgroupRoot string
pollInterval time.Duration
}
// unitKillTimeout bounds one systemctl kill.
const unitKillTimeout = 5 * time.Second
// unitOutputDrainTimeout lets final diagnostics drain without waiting on
// a writer inherited by a process that outlives systemd-run.
const unitOutputDrainTimeout = time.Second
// oomWatchGrace bounds how long a dead worker's OOM watch may still report.
const oomWatchGrace = time.Second
// unitGeneration numbers units. Seeded with the start time so a restarted
// daemon does not reuse the name of a unit still being torn down.
var unitGeneration = func() *atomic.Int64 {
var g atomic.Int64
g.Store(time.Now().UnixNano())
return &g
}()
// lockdownProperties are the unit properties every worker runs under: no
// network, no host files, no privileges, bounded memory, tasks and files.
func lockdownProperties(cfg UnitConfig) []string {
memory := strconv.FormatUint(cfg.MemoryLimit, 10)
denyWriteExecute := "MemoryDenyWriteExecute=yes"
if cfg.AllowExecutableMemory {
denyWriteExecute = "MemoryDenyWriteExecute=no"
}
props := []string{
"DynamicUser=yes",
"CapabilityBoundingSet=",
"NoNewPrivileges=yes",
"PrivateNetwork=yes",
"RestrictAddressFamilies=AF_UNIX",
"ProtectSystem=strict",
"ProtectHome=yes",
"PrivateTmp=yes",
"PrivateDevices=yes",
"ProtectKernelTunables=yes",
"ProtectKernelModules=yes",
"ProtectControlGroups=yes",
"RestrictNamespaces=yes",
"RestrictRealtime=yes",
denyWriteExecute,
"UMask=0077",
"SystemCallFilter=@system-service",
// In place of LockPersonality=yes, which systemd-run on systemd 239
// rejects as an unknown assignment. Denying personality(2) outright
// is stricter, and no worker needs it.
"SystemCallFilter=~personality",
"SystemCallErrorNumber=EPERM",
"MemoryAccounting=yes",
"MemoryLimit=" + memory,
"MemoryMax=" + memory,
"MemorySwapMax=0",
"TasksMax=64",
"LimitNOFILE=256",
"LimitCORE=0",
"LimitFSIZE=" + strconv.FormatUint(cfg.FileSizeLimit, 10),
// The containing slice gets the same weight before launch.
"CPUWeight=50",
// Prefer a hostile parser as the victim under host memory pressure.
"OOMScoreAdjust=1000",
"Slice=" + unitSlice(cfg.Prefix),
}
paths, _ := inaccessiblePaths(cfg.InaccessiblePaths)
for _, p := range paths {
// "-": a path missing on this host is not an error.
props = append(props, "InaccessiblePaths=-"+p)
}
return props
}
// inaccessiblePaths returns the usable paths sorted and without duplicates,
// and the ones it dropped.
func inaccessiblePaths(in []string) (keep, dropped []string) {
seen := make(map[string]bool, len(in))
for _, p := range in {
if p == "/" || !filepath.IsAbs(p) || filepath.Clean(p) != p || strings.ContainsAny(p, " \t\n\"'\\%") || strings.ContainsFunc(p, unicode.IsControl) {
dropped = append(dropped, p)
continue
}
if !seen[p] {
seen[p] = true
keep = append(keep, p)
}
}
sort.Strings(keep)
return keep, dropped
}
// unitSlice is the slice a prefix's units run in: one per prefix, directly
// under system.slice, so workers share system.slice's resources. On cgroup
// v2, work is admitted only once earlier units are gone; memory.events then
// counts the current unit's OOM kills and outlives its own cgroup.
func unitSlice(prefix string) string {
return "system-" + strings.ReplaceAll(prefix, "-", `\x2d`) + ".slice"
}
// UnitLauncher starts each worker as its own transient unit through
// `systemd-run --pipe`, whose stdin and stdout are the worker's socket. A
// worker is handed out only after hello and verification of its own memory
// cgroup's cap and usage; there is no fallback to direct execution.
func UnitLauncher(cfg UnitConfig) Launcher {
h := unitHooks{}
if cfg.hooks != nil {
h = *cfg.hooks
}
if h.mainPID == nil {
h.mainPID = systemctlMainPID
}
if h.configureSlice == nil {
h.configureSlice = systemctlConfigureSlice
}
if h.killUnit == nil {
h.killUnit = systemctlKill
}
if h.procRoot == "" {
h.procRoot = "/proc"
}
if h.cgroupRoot == "" {
h.cgroupRoot = "/sys/fs/cgroup"
}
if h.pollInterval <= 0 {
h.pollInterval = 500 * time.Millisecond
}
if _, dropped := inaccessiblePaths(cfg.InaccessiblePaths); len(dropped) > 0 && cfg.Log != nil {
cfg.Log("%s: not hiding unusable paths %q", cfg.Prefix, dropped)
}
return func(ctx context.Context) (Process, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
p, err := startUnit(ctx, cfg, &h)
if err != nil {
return nil, fmt.Errorf("workerproc: isolation unavailable: %w", err)
}
return p, nil
}
}
func startUnit(ctx context.Context, cfg UnitConfig, h *unitHooks) (*unitProcess, error) {
startCtx, cancel := context.WithTimeout(ctx, cfg.StartTimeout)
defer cancel()
path := h.systemdRun
if path == "" {
var err error
if path, err = exec.LookPath("systemd-run"); err != nil {
return nil, err
}
}
// CPU weights compare siblings. A weight on the only service in a
// default-weight slice would let it outweigh csm.service instead.
if err := h.configureSlice(startCtx, unitSlice(cfg.Prefix)); err != nil {
return nil, fmt.Errorf("configure worker slice: %w", err)
}
conn, childFile, err := workerSocket()
if err != nil {
return nil, err
}
unit := cfg.Prefix + "-" + strconv.FormatInt(unitGeneration.Add(1), 10)
name, args := systemdrun.Argv(path, systemdrun.Options{
Pipe: true,
RuntimeMax: cfg.RuntimeMax,
Unit: unit,
Properties: lockdownProperties(cfg),
}, cfg.Command, cfg.Args...)
// Not CommandContext: ending the request must not kill the worker.
cmd := exec.Command(name, args...) // #nosec G204 -- systemd-run with fixed flags and the daemon's own binary.
// The worker's output is untrusted: it reaches the log quoted and capped.
stderrR, stderrW, err := os.Pipe()
if err != nil {
_ = childFile.Close()
_ = conn.Close()
return nil, err
}
cmd.Stdin = childFile
cmd.Stdout = childFile
cmd.Stderr = stderrW
err = cmd.Start()
// Closed at once, not deferred: while the daemon holds the child's end,
// a systemd-run that exits early leaves the hello waiting for its
// deadline instead of reading end of file.
_ = childFile.Close()
_ = stderrW.Close()
if err != nil {
_ = stderrR.Close()
_ = conn.Close()
return nil, fmt.Errorf("start systemd-run: %w", err)
}
logf := cfg.Log
if logf == nil {
logf = func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) }
}
go func() {
defer func() { _ = stderrR.Close() }()
logWorkerOutput(stderrR, unit, logf)
}()
p := &unitProcess{unit: unit, cmd: cmd, conn: conn, done: make(chan struct{}), killUnit: h.killUnit, log: cfg.Log}
go func() {
_ = cmd.Wait()
// The unit belongs to PID 1 and can outlive the systemd-run client.
_ = p.Kill()
// EOF is not guaranteed if another process still holds the writer.
// Allow buffered diagnostics through, then release the reader without
// delaying Done and its watchdog shutdown. A closed reader needs no deadline.
_ = stderrR.SetReadDeadline(time.Now().Add(unitOutputDrainTimeout))
close(p.done)
}()
if err := p.start(startCtx, cfg, h); err != nil {
_ = p.Kill()
_ = conn.Close()
select {
case <-p.done:
case <-time.After(reapTimeout):
p.logf("unit %s did not stop", unit)
}
return nil, fmt.Errorf("unit %s: %w", unit, err)
}
return p, nil
}
// start finishes a launch: hello, then the main PID, which only systemd
// reports, never the worker.
func (p *unitProcess) start(ctx context.Context, cfg UnitConfig, h *unitHooks) error {
stop := context.AfterFunc(ctx, func() { _ = p.conn.Close() })
defer stop()
if err := Handshake(ctx, p.conn, cfg.Limits, cfg.Protocol); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
return err
}
pid, err := h.mainPID(ctx, p)
if err != nil {
return fmt.Errorf("main PID: %w", err)
}
if pid <= 0 {
return errors.New("unit has no main process")
}
p.pid = pid
cg, found, err := findMemoryCgroup(h.procRoot, h.cgroupRoot, pid, p.unit)
if err != nil {
return fmt.Errorf("cannot cap worker memory: %w", err)
}
if !found {
// Polling the main process alone would miss any child it forks.
return errors.New("cannot cap worker memory: no memory cgroup for the unit")
}
if capErr := cg.checkRAMLimit(cfg.MemoryLimit); capErr != nil {
return fmt.Errorf("cannot cap worker memory: %w", capErr)
}
if capErr := cg.capSwap(cfg.MemoryLimit, h.procRoot); capErr != nil {
return fmt.Errorf("cannot cap worker memory: %w", capErr)
}
p.cg = cg
if cg.v2 {
// The unit's own counters vanish with its cgroup before the daemon
// sees the worker die; the slice's hierarchical ones stay.
if dir := filepath.Dir(cg.dir); filepath.Base(dir) == unitSlice(cfg.Prefix) {
// Reaping systemd-run is not proof that its service is gone:
// killing the unit is asynchronous and can fail or time out.
if waitErr := waitForUnitSlice(ctx, dir, filepath.Base(cg.dir), h.pollInterval); waitErr != nil {
return fmt.Errorf("wait for worker slice: %w", waitErr)
}
slice := memoryCgroup{dir: dir, v2: true}
if n, serr := slice.oomKills(); serr == nil {
p.slice, p.sliceOOMKills = &slice, n
}
}
if p.slice == nil {
p.logf("unit %s: memory kills will read as crashes: no slice counter", p.unit)
}
} else if w, werr := cg.watchOOM(); werr != nil {
p.logf("unit %s: memory kills will read as crashes: %v", p.unit, werr)
} else {
p.oom = w
go func() {
// The watch ends with the cgroup; this bounds it if the kernel
// never reports the removal.
<-p.done
time.Sleep(oomWatchGrace)
w.stop()
}()
}
p.usage = func() (uint64, error) { return cg.usage(h.procRoot) }
used, err := p.usage()
if err != nil {
return fmt.Errorf("cannot measure worker memory: %w", err)
}
if used > cfg.MemoryLimit {
return fmt.Errorf("worker memory %d over cap %d", used, cfg.MemoryLimit)
}
// Stop the startup cancellation hook before handing out the socket.
// If cancellation won, the callback may still be closing it.
stop()
if err := ctx.Err(); err != nil {
return err
}
p.limit = cfg.MemoryLimit
if cfg.RuntimeMax > 0 {
p.retireAt = time.Now().Add(cfg.RuntimeMax - cfg.RuntimeMax/4)
}
go p.watch(h.pollInterval)
return nil
}
func waitForUnitSlice(ctx context.Context, dir, unit string, interval time.Duration) error {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
if err := ctx.Err(); err != nil {
return err
}
entries, err := os.ReadDir(dir)
if err != nil {
return err
}
found, shared := false, false
for _, entry := range entries {
if !entry.IsDir() {
continue
}
if entry.Name() == unit {
found = true
} else {
shared = true
}
}
if !found {
return errors.New("worker cgroup disappeared")
}
if !shared {
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
}
}
// watch kills the worker once its memory and swap pass the cap, in case the
// kernel's own limit did not take effect. It fails closed: usage that cannot
// be read while the worker is alive kills it too.
func (p *unitProcess) watch(interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-p.done:
return
case <-ticker.C:
}
if p.killing.Load() {
// Already being killed: its cgroup can vanish before systemd-run
// exits, and that gap is not a fault to report.
return
}
used, err := p.usage()
if err == nil && used <= p.limit {
continue
}
if err != nil && localExhaustion(err) {
// The daemon is out of descriptors, not the worker out of
// line; the kernel cap was verified at start.
continue
}
select {
case <-p.done:
return // Gone already: its counters vanished with it.
default:
}
if p.requestKill(err == nil) {
if err != nil {
p.logf("unit %s: cannot read memory, killing: %v", p.unit, err)
} else {
p.logf("unit %s: memory %d over cap %d, killing", p.unit, used, p.limit)
}
}
return
}
}
// unitProcess is one worker unit. Done follows the systemd-run client, which
// exits when the unit ends.
type unitProcess struct {
unit string
cmd *exec.Cmd
conn *net.UnixConn
done chan struct{}
pid int
killUnit func(ctx context.Context, p *unitProcess) error
kill sync.Once
killing atomic.Bool
log func(string, ...any)
// usage reads memory and swap in use; limit is the cap it is held to.
// lastUsed is what Retire last read.
usage func() (uint64, error)
limit uint64
lastUsed atomic.Uint64
// retireAt is three quarters into RuntimeMaxSec; zero means no cap.
retireAt time.Time
// cg is the unit's memory cgroup; memoryKill records the watchdog
// killing the worker for memory.
cg memoryCgroup
memoryKill atomic.Bool
// oom follows the kernel's OOM kills in cg on cgroup v1; slice and
// sliceOOMKills, the count at start, read them on v2. Without either a
// memory kill reads as a crash.
oom *oomWatch
slice *memoryCgroup
sliceOOMKills uint64
}
func (p *unitProcess) Conn() *net.UnixConn { return p.conn }
func (p *unitProcess) Done() <-chan struct{} { return p.done }
func (p *unitProcess) PID() int { return p.pid }
// Retire asks for a fresh worker once this one holds more than three
// quarters of its cap after a job, or its usage cannot be read, or it is
// three quarters into its lifetime cap: systemd ending it mid-request would
// charge that request's input with a worker failure.
func (p *unitProcess) Retire() bool {
p.lastUsed.Store(0)
if !p.retireAt.IsZero() && time.Now().After(p.retireAt) {
return true
}
if p.usage == nil {
return false
}
used, err := p.usage()
if err == nil {
p.lastUsed.Store(used)
}
return err != nil || used > p.limit/4*3
}
// MemoryUsed is the memory and swap in use when Retire last read it, or 0.
func (p *unitProcess) MemoryUsed() uint64 { return p.lastUsed.Load() }
// Kill asks systemd to SIGKILL the unit and returns at once: the supervisor
// calls it holding its lock, and a hung systemctl must not freeze it. The
// worker is killed by unit name, never by a PID it could have influenced.
func (p *unitProcess) Kill() error {
p.requestKill(false)
return nil
}
// requestKill records only the winning cause, before closing the socket can
// make the supervisor ask for an OOM verdict. A watchdog read racing with a
// deliberate kill must neither relabel that kill nor log another one.
func (p *unitProcess) requestKill(memory bool) bool {
requested := false
p.kill.Do(func() {
requested = true
p.killing.Store(true)
p.memoryKill.Store(memory)
// A failed watchdog check must prevent further requests even while
// systemctl is stalled or fails to reach the manager.
_ = p.conn.Close()
go func() {
ctx, cancel := context.WithTimeout(context.Background(), unitKillTimeout)
defer cancel()
if err := p.killUnit(ctx, p); err != nil {
p.logf("kill unit %s: %v", p.unit, err)
}
}()
})
return requested
}
// OOMKilled reports a memory kill: by the watchdog, or by the kernel as the
// v2 slice recorded it or the v1 OOM watch saw it. The unit's own counters
// are a fallback while its cgroup still exists. On v1 it waits briefly for
// the watch's verdict, since the removal notification may still be pending.
func (p *unitProcess) OOMKilled() bool {
if p.memoryKill.Load() {
return true
}
if p.slice != nil {
if n, err := p.slice.oomKills(); err == nil && n > p.sliceOOMKills {
return true
}
}
if p.oom != nil && p.oom.wait(oomWatchGrace) {
return true
}
n, err := p.cg.oomKills()
return err == nil && n > 0
}
func (p *unitProcess) logf(format string, args ...any) {
if p.log != nil {
p.log(format, args...)
}
}
func systemctlMainPID(ctx context.Context, p *unitProcess) (int, error) {
out, err := exec.CommandContext(ctx, "systemctl", "show", "-p", "MainPID", "--value", p.unit+".service").Output() // #nosec G204 -- the unit name is ours.
if err != nil {
return 0, err
}
return strconv.Atoi(strings.TrimSpace(string(out)))
}
func systemctlConfigureSlice(ctx context.Context, name string) error {
out, err := exec.CommandContext(ctx, "systemctl", "set-property", "--runtime", name, "CPUWeight=50").CombinedOutput() // #nosec G204 -- the slice name comes from the caller's unit prefix.
if err != nil {
return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
func systemctlKill(ctx context.Context, p *unitProcess) error {
out, err := exec.CommandContext(ctx, "systemctl", "kill", "--signal=SIGKILL", p.unit+".service").CombinedOutput() // #nosec G204 -- the unit name is ours.
if err != nil && !strings.Contains(string(out), "not loaded") {
// "not loaded": the unit already ended, which is what Kill wants.
return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
// CheckUnitCommand reports why a unit's dynamic user could not run path: the
// file must be executable by others, which needs no read permission for a
// compiled binary, and every directory above it searchable by others.
func CheckUnitCommand(path string) error {
if !filepath.IsAbs(path) {
return fmt.Errorf("workerproc: unit command %q is not an absolute path", path)
}
st, err := os.Stat(path)
if err != nil {
return err
}
if !st.Mode().IsRegular() || st.Mode().Perm()&0o001 == 0 {
return fmt.Errorf("workerproc: %s is mode %04o; worker units need it executable by others, such as 0711", path, st.Mode().Perm())
}
for dir := filepath.Dir(path); ; dir = filepath.Dir(dir) {
st, err := os.Stat(dir)
if err != nil {
return err
}
if st.Mode().Perm()&0o001 == 0 {
return fmt.Errorf("workerproc: %s is mode %04o; worker units need it searchable by others", dir, st.Mode().Perm())
}
if dir == filepath.Dir(dir) {
return nil
}
}
}
// StopUnits SIGKILLs every unit whose name starts with prefix, for leftovers
// of a daemon that died without stopping its workers.
func StopUnits(ctx context.Context, prefix string) error {
out, err := exec.CommandContext(ctx, "systemctl", "kill", "--signal=SIGKILL", "--", prefix+"*.service").CombinedOutput() // #nosec G204 -- the prefix is a constant of the caller.
if err != nil {
return fmt.Errorf("workerproc: stop units %s*: %w: %s", prefix, err, strings.TrimSpace(string(out)))
}
return nil
}
// Bounds on worker output in the daemon's log, per unit.
const (
maxWorkerLogLine = 512
maxWorkerLogBytes = 64 << 10
maxWorkerLogLines = 256
)
// logWorkerOutput logs r line by line, quoted, each line cut to
// maxWorkerLogLine bytes, until maxWorkerLogLines lines or maxWorkerLogBytes
// of log entries are spent; then it says so once and drains r to end of
// file. An entry costs what it adds to the log, so tiny lines cannot turn
// the byte budget into tens of thousands of journal entries. Empty lines are
// not logged but still cost their newline.
func logWorkerOutput(r io.Reader, unit string, logf func(string, ...any)) {
br := bufio.NewReader(r)
budget, lines := maxWorkerLogBytes, 0
for {
line, err := readCappedLine(br, maxWorkerLogLine)
if len(line) > 0 || err == nil {
entry := ""
cost := 1
if len(line) > 0 {
entry = fmt.Sprintf("%s: %q", unit, line)
cost = len(entry) + 1
}
if cost > budget || (entry != "" && lines >= maxWorkerLogLines) {
logf("%s: further output dropped", unit)
_, _ = io.Copy(io.Discard, br)
return
}
budget -= cost
if entry != "" {
lines++
logf("%s", entry)
}
}
if err != nil {
return
}
}
}
// readCappedLine reads one line and returns at most max bytes of it, without
// the newline. The rest of a longer line is read and discarded.
func readCappedLine(br *bufio.Reader, max int) ([]byte, error) {
var line []byte
for {
chunk, err := br.ReadSlice('\n')
chunk = bytes.TrimSuffix(chunk, []byte("\n"))
if room := max - len(line); room > 0 {
line = append(line, chunk[:min(room, len(chunk))]...)
}
if err != bufio.ErrBufferFull {
return line, err
}
}
}
package workerproc
import (
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"time"
)
// memoryCgroup is a worker unit's own memory cgroup directory.
type memoryCgroup struct {
dir string
v2 bool
}
// findMemoryCgroup reads the cgroup of pid from procRoot. A cgroup v1 memory
// controller line wins; otherwise the unified (v2) line is used. found is
// false when neither names a directory under cgroupRoot.
func findMemoryCgroup(procRoot, cgroupRoot string, pid int, unit string) (memoryCgroup, bool, error) {
data, err := os.ReadFile(filepath.Join(procRoot, strconv.Itoa(pid), "cgroup")) // #nosec G304 -- procfs path built from a PID systemd reported.
if err != nil {
return memoryCgroup{}, false, err
}
var unified string
for _, line := range strings.Split(string(data), "\n") {
parts := strings.SplitN(line, ":", 3)
if len(parts) != 3 {
continue
}
if parts[0] == "0" && parts[1] == "" {
unified = parts[2]
continue
}
for _, controller := range strings.Split(parts[1], ",") {
if controller != "memory" {
continue
}
if err := checkUnitCgroup(parts[2], unit); err != nil {
return memoryCgroup{}, false, err
}
if dir := filepath.Join(cgroupRoot, "memory", parts[2]); isDir(dir) {
return memoryCgroup{dir: dir}, true, nil
}
return memoryCgroup{}, false, errors.New("memory cgroup directory unavailable")
}
}
if unified != "" {
if err := checkUnitCgroup(unified, unit); err != nil {
return memoryCgroup{}, false, err
}
if dir := filepath.Join(cgroupRoot, unified); isDir(dir) {
return memoryCgroup{dir: dir, v2: true}, true, nil
}
}
return memoryCgroup{}, false, nil
}
func checkUnitCgroup(path, unit string) error {
// A PID can exit and be reused between systemctl and the procfs read.
// Never change the limits of a different unit or a shared parent group.
if !filepath.IsAbs(path) || filepath.Clean(path) != path || filepath.Base(path) != unit+".service" {
return fmt.Errorf("memory cgroup %q does not belong to unit %s", path, unit)
}
return nil
}
func (c memoryCgroup) checkRAMLimit(limit uint64) error {
name := "memory.limit_in_bytes"
if c.v2 {
name = "memory.max"
}
got, err := readCgroupUint(filepath.Join(c.dir, name))
if err != nil {
return err
}
if got == 0 || got > limit {
return fmt.Errorf("RAM limit %d exceeds worker cap %d or is zero", got, limit)
}
return nil
}
func isDir(path string) bool {
st, err := os.Stat(path) // #nosec G703 -- a cgroup path the kernel reported for a PID systemd reported.
return err == nil && st.IsDir()
}
// capSwap makes the cgroup's memory cap cover swap too. On cgroup v1
// MemoryLimit caps RAM only: a worker over it swaps instead of dying, and a
// host with plenty of swap never stops it. Each write is read back, because a
// cap that silently did not take effect is no cap.
func (c memoryCgroup) capSwap(limit uint64, procRoot string) error {
if c.v2 {
swapMax := filepath.Join(c.dir, "memory.swap.max")
current, err := readCgroupValue(swapMax)
if errors.Is(err, os.ErrNotExist) {
// No swap accounting: swap cannot be charged to the unit, which
// is safe only while the host has no swap at all.
return noSwap(procRoot)
}
if err != nil || current == "0" {
return err
}
return writeCgroupValue(swapMax, "0", func(got string) bool { return got == "0" })
}
memsw := filepath.Join(c.dir, "memory.memsw.limit_in_bytes")
if _, err := os.Stat(memsw); err == nil {
return writeCgroupValue(memsw, strconv.FormatUint(limit, 10), func(got string) bool {
// The kernel may round down to a page; anything above the cap,
// such as the unlimited default, means the write did not land.
n, parseErr := strconv.ParseUint(got, 10, 64)
return parseErr == nil && n > 0 && n <= limit
})
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
// No swap accounting: keep the cgroup out of swap instead.
return writeCgroupValue(filepath.Join(c.dir, "memory.swappiness"), "0", func(got string) bool { return got == "0" })
}
// usage is the cgroup's memory and swap in use. It counts memfd pages
// charged to the unit, which VmRSS misses.
func (c memoryCgroup) usage(procRoot string) (uint64, error) {
if c.v2 {
used, err := readCgroupUint(filepath.Join(c.dir, "memory.current"))
if err != nil {
return 0, err
}
swap, err := readCgroupUint(filepath.Join(c.dir, "memory.swap.current"))
if errors.Is(err, os.ErrNotExist) {
// Swap can be enabled after launch. Without accounting, the
// no-swap condition must hold for the worker's whole lifetime.
return used, noSwap(procRoot)
}
return used + swap, err
}
used, err := readCgroupUint(filepath.Join(c.dir, "memory.memsw.usage_in_bytes"))
if errors.Is(err, os.ErrNotExist) {
return readCgroupUint(filepath.Join(c.dir, "memory.usage_in_bytes"))
}
return used, err
}
// oomKills is how many processes the kernel OOM-killed in the cgroup.
func (c memoryCgroup) oomKills() (uint64, error) {
name := "memory.oom_control"
if c.v2 {
name = "memory.events"
}
data, err := os.ReadFile(filepath.Join(c.dir, name)) // #nosec G304 -- a file in the worker unit's own cgroup.
if err != nil {
return 0, err
}
for _, line := range strings.Split(string(data), "\n") {
if value, ok := strings.CutPrefix(line, "oom_kill "); ok {
return strconv.ParseUint(strings.TrimSpace(value), 10, 64)
}
}
return 0, nil
}
// noSwap fails when any swap area is active.
func noSwap(procRoot string) error {
data, err := os.ReadFile(filepath.Join(procRoot, "swaps")) // #nosec G304 -- procfs.
if err != nil {
return err
}
// The first line is a header; each further line is one swap area.
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
if len(lines) > 1 {
return errors.New("swap is on and the kernel does not account it per cgroup")
}
return nil
}
func readCgroupValue(path string) (string, error) {
b, err := os.ReadFile(path) // #nosec G304 -- a file in the worker unit's own cgroup.
return strings.TrimSpace(string(b)), err
}
func readCgroupUint(path string) (uint64, error) {
v, err := readCgroupValue(path)
if err != nil {
return 0, err
}
return strconv.ParseUint(v, 10, 64)
}
// writeCgroupValue writes value without creating the file, then checks the
// value read back.
func writeCgroupValue(path, value string, ok func(string) bool) error {
f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0) // #nosec G304 -- a file in the worker unit's own cgroup.
if err != nil {
return err
}
_, err = f.WriteString(value)
if closeErr := f.Close(); err == nil {
err = closeErr
}
if err != nil {
return err
}
got, err := readCgroupValue(path)
if err != nil {
return err
}
if !ok(got) {
return fmt.Errorf("%s reads %q after writing %s", path, got, value)
}
return nil
}
// oomWatch follows the kernel's OOM notifications for one worker cgroup. The
// daemon sees a worker die only once systemd-run exits, after systemd has
// removed the unit's cgroup and its oom_kill counter with it, so the kill
// has to be read at the kernel's notification, while the cgroup exists.
type oomWatch struct {
seen atomic.Bool
done chan struct{}
stop func()
}
// wait returns whether an OOM kill was seen, after the watch ends or d.
func (w *oomWatch) wait(d time.Duration) bool {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-w.done:
case <-timer.C:
}
return w.seen.Load()
}
//go:build linux
package workerproc
import (
"encoding/binary"
"fmt"
"io"
"os"
"path/filepath"
"golang.org/x/sys/unix"
)
// watchOOM follows a cgroup v1 memory cgroup's OOM notifications through an
// eventfd registered for memory.oom_control. On v2 the unit's slice counts
// the kills instead (unitSlice). After cgroup removal the caller bounds any
// remaining wait with stop.
func (c memoryCgroup) watchOOM() (*oomWatch, error) {
return c.watchOOMv1()
}
func (c memoryCgroup) watchOOMv1() (*oomWatch, error) {
efd, err := unix.Eventfd(0, unix.EFD_CLOEXEC|unix.EFD_NONBLOCK)
if err != nil {
return nil, fmt.Errorf("oom eventfd: %w", err)
}
ev := os.NewFile(uintptr(efd), "oom eventfd") // #nosec G115 -- eventfd returned a non-negative descriptor.
ctl, err := os.Open(filepath.Join(c.dir, "memory.oom_control"))
if err != nil {
_ = ev.Close()
return nil, err
}
// The kernel takes its own references at registration.
reg := fmt.Sprintf("%d %d", efd, ctl.Fd())
err = os.WriteFile(filepath.Join(c.dir, "cgroup.event_control"), []byte(reg), 0)
_ = ctl.Close()
if err != nil {
_ = ev.Close()
return nil, fmt.Errorf("register oom eventfd: %w", err)
}
w := &oomWatch{done: make(chan struct{}), stop: func() { _ = ev.Close() }}
go func() {
defer close(w.done)
w.readOOMv1(ev, func() bool { return isDir(c.dir) })
}()
return w, nil
}
func (w *oomWatch) readOOMv1(ev io.Reader, alive func() bool) {
var buf [8]byte
var count uint64
for {
if _, err := ev.Read(buf[:]); err != nil {
return
}
count += binary.NativeEndian.Uint64(buf[:])
if oomFromV1Event(count, alive()) {
w.seen.Store(true)
return
}
// The directory may disappear between read and stat, before its
// removal notification arrives. Keep reading until a second event
// proves an OOM or the post-exit grace period closes the eventfd.
}
}
// oomFromV1Event reads a cgroup v1 OOM eventfd wake-up. The eventfd fires at
// an OOM and once more at cgroup removal, which the kernel signals after the
// directory is gone. While the directory exists, any wake-up is an OOM; once
// it is gone, a total count above one means an OOM came before the removal.
func oomFromV1Event(count uint64, alive bool) bool {
return alive || count > 1
}
package wpcheck
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"golang.org/x/sys/unix"
)
type Cache struct {
mu sync.RWMutex
statePath string
checksums map[string]map[string]string // core: "<version>:<locale>" -> relPath -> MD5
pluginChecksums map[string]map[string]string // plugins: "plugin:<slug>:<version>" -> relPath -> SHA256
roots map[string]rootEntry
fetching map[string]bool
// Retain admission times after completion so fast responses and repeated
// misses cannot turn tenant-written release names into unlimited fetches.
coreFetchAfter map[string]time.Time
// pluginNotFoundUntil records slug+version pairs that wordpress.org
// returned 404 for, paired with the absolute time at which the
// suppression expires. Plugins hosted outside wp.org (paid forks,
// custom internal plugins) would otherwise re-arm the 4-attempt
// retry cycle on every cache miss. The TTL ensures wp.org adding
// a plugin later still gets picked up.
pluginNotFoundUntil map[string]time.Time
// stopCh, when non-nil and closed, signals pending retry timers to
// drop their scheduled fetch instead of firing. Wired by the daemon
// to the FileMonitor stopCh so checksum-retry chains do not survive
// daemon shutdown.
stopMu sync.RWMutex
stopCh <-chan struct{}
}
type rootEntry struct {
version string
locale string
}
func NewCache(statePath string) *Cache {
c := &Cache{
statePath: statePath,
checksums: make(map[string]map[string]string),
pluginChecksums: make(map[string]map[string]string),
roots: make(map[string]rootEntry),
fetching: make(map[string]bool),
coreFetchAfter: make(map[string]time.Time),
pluginNotFoundUntil: make(map[string]time.Time),
}
c.loadFromDisk()
return c
}
func cacheKey(version, locale string) string {
return version + ":" + locale
}
func diskFilename(version, locale string) string {
return version + "_" + locale + ".json"
}
func (c *Cache) loadFromDisk() {
dir := filepath.Join(c.statePath, "wp-checksums")
entries, err := os.ReadDir(dir)
if err != nil {
return
}
for _, entry := range entries {
name := entry.Name()
if entry.IsDir() || !strings.HasSuffix(name, ".json") {
continue
}
// #nosec G304 -- dir is {statePath}/wp-checksums; name comes from
// our own os.ReadDir of that same dir.
data, err := os.ReadFile(filepath.Join(dir, name))
if err != nil {
continue
}
checksums, err := ParseChecksumResponse(data)
if err != nil {
continue
}
base := strings.TrimSuffix(name, ".json")
parts := strings.SplitN(base, "_", 2)
if len(parts) != 2 {
continue
}
c.checksums[cacheKey(parts[0], parts[1])] = checksums
}
}
// PersistChecksums writes checksum data to disk atomically (tmpfile + rename)
// and populates the in-memory cache. The file is written to {statePath}/wp-checksums/.
func (c *Cache) PersistChecksums(version, locale string, rawJSON []byte, checksums map[string]string) error {
dir := filepath.Join(c.statePath, "wp-checksums")
if err := os.MkdirAll(dir, 0700); err != nil {
return fmt.Errorf("creating wp-checksums dir: %w", err)
}
filename := diskFilename(version, locale)
tmpPath := filepath.Join(dir, filename+".tmp")
finalPath := filepath.Join(dir, filename)
if err := os.WriteFile(tmpPath, rawJSON, 0600); err != nil {
return fmt.Errorf("writing temp file: %w", err)
}
if err := os.Rename(tmpPath, finalPath); err != nil {
_ = os.Remove(tmpPath)
return fmt.Errorf("renaming to final: %w", err)
}
c.mu.Lock()
c.checksums[cacheKey(version, locale)] = checksums
c.mu.Unlock()
return nil
}
func (c *Cache) lookupChecksum(version, locale, relativePath string) (string, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
versionMap, ok := c.checksums[cacheKey(version, locale)]
if !ok {
return "", false
}
md5hex, ok := versionMap[relativePath]
return md5hex, ok
}
func (c *Cache) hasChecksums(version, locale string) bool {
c.mu.RLock()
ok := c.checksums[cacheKey(version, locale)] != nil
c.mu.RUnlock()
return ok
}
func (c *Cache) getRoot(root string) (version, locale string, ok bool) {
c.mu.RLock()
entry, ok := c.roots[root]
c.mu.RUnlock()
if !ok {
return "", "", false
}
return entry.version, entry.locale, true
}
func (c *Cache) setRoot(root, version, locale string) {
c.mu.Lock()
c.roots[root] = rootEntry{version: version, locale: locale}
c.mu.Unlock()
}
func (c *Cache) invalidateRoot(root string) {
c.mu.Lock()
delete(c.roots, root)
c.mu.Unlock()
}
const (
coreFetchMaxPending = 8
coreFetchHistoryMax = 64
coreFetchCooldown = time.Hour
)
func (c *Cache) startBackgroundFetch(version, locale string) {
if c.isStopped() {
return
}
key := cacheKey(version, locale)
c.mu.Lock()
if c.fetching[key] || c.checksums[key] != nil {
c.mu.Unlock()
return
}
now := time.Now()
pending := 0
for oldKey, until := range c.coreFetchAfter {
// Only core admissions consume this budget. Plugin requests retain
// their in-flight flags for deduplication, without limiting core.
if c.fetching[oldKey] {
pending++
} else if !now.Before(until) {
delete(c.coreFetchAfter, oldKey)
}
}
if pending >= coreFetchMaxPending || now.Before(c.coreFetchAfter[key]) || len(c.coreFetchAfter) >= coreFetchHistoryMax {
c.mu.Unlock()
return
}
c.coreFetchAfter[key] = now.Add(coreFetchCooldown)
c.fetching[key] = true
c.mu.Unlock()
go c.fetchWithRetry(version, locale, 0)
}
func (c *Cache) fetchWithRetry(version, locale string, attempt int) {
backoffs := []time.Duration{1 * time.Minute, 5 * time.Minute, 15 * time.Minute, 1 * time.Hour}
key := cacheKey(version, locale)
if c.isStopped() {
c.clearFetching(key)
return
}
rawJSON, checksums, err := FetchChecksums(version, locale)
if err != nil {
if attempt >= len(backoffs) {
c.mu.Lock()
c.coreFetchAfter[key] = time.Now().Add(coreFetchCooldown)
delete(c.fetching, key)
c.mu.Unlock()
fmt.Fprintf(os.Stderr, "wpcheck: core fetch abandoned for WP %s (%s) after %d attempts: %v\n",
version, locale, attempt+1, err)
return
}
delay := backoffs[attempt]
fmt.Fprintf(os.Stderr, "wpcheck: fetch failed for WP %s (%s), retry in %v: %v\n",
version, locale, delay, err)
c.scheduleRetry(delay, func() {
c.fetchWithRetry(version, locale, attempt+1)
}, func() {
c.clearFetching(key)
})
return
}
if err := c.PersistChecksums(version, locale, rawJSON, checksums); err != nil {
fmt.Fprintf(os.Stderr, "wpcheck: persist failed for WP %s (%s): %v\n", version, locale, err)
}
c.clearFetching(key)
fmt.Fprintf(os.Stderr, "wpcheck: cached %d checksums for WP %s (%s)\n", len(checksums), version, locale)
}
const maxFileSize = 2 << 20
// readCompleteFileForHash returns a stable-size snapshot of a regular file.
// Hash verification must never accept only a prefix: if a known-good file is
// exactly maxFileSize bytes, an attacker could otherwise append a payload that
// the one-shot bounded Pread silently ignores.
func readCompleteFileForHash(fd int) []byte {
var before unix.Stat_t
if err := unix.Fstat(fd, &before); err != nil || before.Size <= 0 || before.Size > maxFileSize {
return nil
}
if before.Mode&unix.S_IFMT != unix.S_IFREG {
return nil
}
data := make([]byte, int(before.Size))
offset := 0
interrupts := 0
for offset < len(data) {
n, err := unix.Pread(fd, data[offset:], int64(offset))
if n > 0 {
offset += n
interrupts = 0
}
if err != nil && !errors.Is(err, unix.EINTR) {
return nil
}
if n == 0 {
if !errors.Is(err, unix.EINTR) {
return nil
}
interrupts++
if interrupts > 100 {
return nil
}
}
}
var after unix.Stat_t
if err := unix.Fstat(fd, &after); err != nil || before.Dev != after.Dev ||
before.Ino != after.Ino || before.Size != after.Size {
return nil
}
return data
}
package wpcheck
import (
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
)
// wpRootLevelFiles are filenames that only exist at the WP installation root.
// index.php is excluded - it requires a secondary check (version.php must exist).
var wpRootLevelFiles = map[string]bool{
"wp-activate.php": true,
"wp-blog-header.php": true,
"wp-comments-post.php": true,
"wp-config-sample.php": true,
"wp-cron.php": true,
"wp-links-opml.php": true,
"wp-load.php": true,
"wp-login.php": true,
"wp-mail.php": true,
"wp-settings.php": true,
"wp-signup.php": true,
"wp-trackback.php": true,
"xmlrpc.php": true,
}
// DetectWPRoot returns the WordPress installation root directory for a file path,
// or empty string if the path is not inside a WP core location.
//
// Detection methods:
// - Path contains /wp-includes/ or /wp-admin/ → root is everything before that segment
// - Filename is a known root-level WP file → root is the parent directory
// - Filename is index.php and version.php exists in wp-includes/ → root is the parent directory
func DetectWPRoot(path string) string {
// Check for /wp-includes/ or /wp-admin/ in path
for _, marker := range []string{"/wp-includes/", "/wp-admin/"} {
if idx := strings.Index(path, marker); idx >= 0 {
return path[:idx]
}
}
// Check for direct wp-includes or wp-admin (file directly inside)
dir := filepath.Dir(path)
base := filepath.Base(dir)
if base == "wp-includes" || base == "wp-admin" {
return filepath.Dir(dir)
}
// Check for known root-level WP files
name := filepath.Base(path)
if wpRootLevelFiles[name] {
return dir
}
// Special case: index.php requires version.php to confirm WP root
if name == "index.php" {
versionPath := filepath.Join(dir, "wp-includes", "version.php")
if _, err := os.Stat(versionPath); err == nil {
return dir
}
}
return ""
}
// RelativePath computes the path of a file relative to the WP root.
// Returns empty string if the file is not under root.
func RelativePath(root, path string) string {
rel, err := filepath.Rel(root, path)
if err != nil || strings.HasPrefix(rel, "..") {
return ""
}
return rel
}
var (
reVersion = regexp.MustCompile(`\$wp_version\s*=\s*'([^']+)'`)
reLocale = regexp.MustCompile(`\$wp_local_package\s*=\s*'([^']+)'`)
// version.php is tenant-writable and its strings name a root-written
// cache file and a query string, so both are held to the shapes
// WordPress actually ships: "6.5", "6.5.2", "6.7-RC1", "6.8-alpha-59245"
// and locales such as "en_US", "de_DE_formal", "ary".
reValidVersion = regexp.MustCompile(`^[0-9]+(?:\.[0-9]+){0,3}(?:-[A-Za-z0-9]+(?:[.-][A-Za-z0-9]+)*)?$`)
reValidLocale = regexp.MustCompile(`^[a-z]{2,3}(?:_[A-Za-z0-9]{2,10}){0,2}$`)
)
// Release identifiers are retained per candidate and used in request URLs
// and cache filenames. Bound their bytes as well as the number of lookups.
const maxCoreVersionLength = 64
// ParseVersionContent extracts the WP version and locale from version.php content.
// Locale defaults to "en_US" if $wp_local_package is not present. A version or
// locale outside the shapes WordPress ships is an error: the install is then
// treated as unverifiable rather than letting tenant-chosen text reach the
// checksum cache path or the API query.
func ParseVersionContent(data []byte) (version, locale string, err error) {
m := reVersion.FindSubmatch(data)
if m == nil {
return "", "", errors.New("wp_version not found in version.php")
}
if len(m[1]) > maxCoreVersionLength {
return "", "", errors.New("wp_version in version.php is too long")
}
version = string(m[1])
if !reValidVersion.MatchString(version) {
return "", "", fmt.Errorf("wp_version %q in version.php is not a WordPress version string", version)
}
locale = "en_US"
if lm := reLocale.FindSubmatch(data); lm != nil {
locale = string(lm[1])
}
if !reValidLocale.MatchString(locale) {
return "", "", fmt.Errorf("wp_local_package %q in version.php is not a WordPress locale", locale)
}
return version, locale, nil
}
// ReadVersionFile reads and parses {root}/wp-includes/version.php.
func ReadVersionFile(root string) (version, locale string, err error) {
// #nosec G304 -- path derived from a WordPress install root discovered
// by the scanner under configured /home/*/public_html paths.
data, err := os.ReadFile(filepath.Join(root, "wp-includes", "version.php"))
if err != nil {
return "", "", err
}
return ParseVersionContent(data)
}
package wpcheck
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"time"
)
var httpClient = &http.Client{Timeout: 10 * time.Second}
// checksumAPIURL returns the WordPress.org checksum API URL for a version and
// locale. Both values are validated at parse time; escaping them here keeps
// the query shape independent of that validation.
func checksumAPIURL(version, locale string) string {
return fmt.Sprintf("https://api.wordpress.org/core/checksums/1.0/?version=%s&locale=%s", url.QueryEscape(version), url.QueryEscape(locale))
}
// checksumResponse is the JSON structure returned by the WP checksum API.
type checksumResponse struct {
Checksums map[string]string `json:"checksums"`
}
// ParseChecksumResponse parses the JSON response from the WP checksum API.
// Returns a map of relative_path -> md5_hex.
func ParseChecksumResponse(data []byte) (map[string]string, error) {
var resp checksumResponse
if err := json.Unmarshal(data, &resp); err != nil {
return nil, fmt.Errorf("invalid JSON: %w", err)
}
if len(resp.Checksums) == 0 {
return nil, errors.New("empty or missing checksums in response")
}
return resp.Checksums, nil
}
// FetchChecksums fetches official checksums from api.wordpress.org for a given
// version and locale. Returns the raw response body and parsed checksums.
func FetchChecksums(version, locale string) (rawJSON []byte, checksums map[string]string, err error) {
url := checksumAPIURL(version, locale)
resp, err := httpClient.Get(url)
if err != nil {
return nil, nil, fmt.Errorf("HTTP request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("HTTP %d from %s", resp.StatusCode, url)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) // 2MB limit
if err != nil {
return nil, nil, fmt.Errorf("reading response: %w", err)
}
checksums, err = ParseChecksumResponse(body)
if err != nil {
return nil, nil, err
}
return body, checksums, nil
}
package wpcheck
import (
"crypto/subtle"
"encoding/hex"
)
const maxHexDigestLength = 64
func constantTimeHexDigestEqual(actualDigest []byte, expectedHex string) bool {
actualHexLen := len(actualDigest) * 2
if actualHexLen == 0 || actualHexLen > maxHexDigestLength {
return false
}
var actual [maxHexDigestLength]byte
var expected [maxHexDigestLength]byte
hex.Encode(actual[:actualHexLen], actualDigest)
if len(expectedHex) >= actualHexLen {
copy(expected[:actualHexLen], expectedHex[:actualHexLen])
} else {
copy(expected[:actualHexLen], expectedHex)
}
lengthEqual := 0
if len(expectedHex) == actualHexLen {
lengthEqual = 1
}
digestEqual := subtle.ConstantTimeCompare(actual[:actualHexLen], expected[:actualHexLen])
return lengthEqual&digestEqual == 1
}
package wpcheck
import (
"archive/zip"
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"golang.org/x/sys/unix"
)
// Plugin verification mirrors the core-file verification path: when an
// installed or update-staged plugin file matches the hash we computed from the
// plugin's official wordpress.org ZIP, signature/YARA rule matches on it are
// false positives and should not fire.
const (
pluginsSegment = "/wp-content/plugins/"
upgradeSegment = "/wp-content/upgrade/"
)
// DetectPluginRoot returns the plugin root directory and slug for a path that
// sits inside a plugin, either installed under /wp-content/plugins/<slug>/ or
// staged by an in-progress update under /wp-content/upgrade/<package>/<slug>/.
// Returns empty strings if the path is not inside a plugin.
func DetectPluginRoot(path string) (root, slug string) {
if root, slug := detectInstalledPluginRoot(path); root != "" {
return root, slug
}
return detectStagedPluginRoot(path)
}
func detectInstalledPluginRoot(path string) (root, slug string) {
idx := strings.Index(path, pluginsSegment)
if idx < 0 {
return "", ""
}
tail := path[idx+len(pluginsSegment):]
slashIdx := strings.Index(tail, "/")
if slashIdx <= 0 {
return "", ""
}
slug = tail[:slashIdx]
if !safePluginPathComponent(slug) {
return "", ""
}
root = path[:idx+len(pluginsSegment)] + slug
return root, slug
}
// detectStagedPluginRoot resolves the layout WordPress unpacks an update into:
// wp-content/upgrade/<package>/<slug>/<rest>, moved into wp-content/plugins/
// only once the install succeeds. The staged tree carries the same files and
// plugin header, so the per-file hash comparison against the official ZIP also
// works before the move. Without this, a routine plugin update leaves every
// one of its files unverifiable while it is staged.
func detectStagedPluginRoot(path string) (root, slug string) {
idx := strings.Index(path, upgradeSegment)
if idx < 0 {
return "", ""
}
pkg, tail, ok := strings.Cut(path[idx+len(upgradeSegment):], "/")
if !ok || !safePluginPathComponent(pkg) {
return "", ""
}
slug, tail, ok = strings.Cut(tail, "/")
// A package directory with no file below <slug>/ is not a staged plugin.
if !ok || tail == "" || !safePluginPathComponent(slug) {
return "", ""
}
return path[:idx+len(upgradeSegment)] + pkg + "/" + slug, slug
}
// safePluginPathComponent rejects the components that would let a crafted path
// resolve a root outside the directory it appears to name.
func safePluginPathComponent(name string) bool {
return name != "" && name != "." && name != ".."
}
const (
pluginHeaderReadLimit = 8192
maxPluginRootEntries = 256
)
var (
rePluginNameHeader = regexp.MustCompile(`(?im)^[ \t/*#@]*Plugin Name:[ \t]*[^ \t\r\n]`)
rePluginVersionHeader = regexp.MustCompile(`(?im)^[ \t/*#@]*Version:[ \t]*([^\s]+)`)
)
// ReadPluginVersion extracts the Version: header from the plugin's main
// file. Most plugins use <slug>.php, but WordPress permits any root-level PHP
// filename. The fallback directory scan is bounded and requires Plugin Name:
// as well as Version: so theme and core staging trees fail closed.
func ReadPluginVersion(pluginRoot, slug string) (string, error) {
if !safePluginPathComponent(slug) {
return "", errors.New("invalid plugin slug")
}
preferredName := slug + ".php"
preferredPath := filepath.Join(pluginRoot, preferredName)
version, found, err := readPluginVersionHeader(preferredPath)
if err == nil && found {
return version, nil
}
if err != nil && !errors.Is(err, os.ErrNotExist) {
return "", err
}
// #nosec G304 -- pluginRoot is derived from a path the scanner received
// from fanotify under a recognized plugin or update-staging layout.
dir, err := os.OpenFile(pluginRoot, os.O_RDONLY|unix.O_NONBLOCK|unix.O_DIRECTORY, 0)
if err != nil {
return "", err
}
defer func() { _ = dir.Close() }()
entries, err := dir.ReadDir(maxPluginRootEntries + 1)
if err != nil && !errors.Is(err, io.EOF) {
return "", fmt.Errorf("reading plugin root: %w", err)
}
if len(entries) > maxPluginRootEntries {
return "", fmt.Errorf("plugin root has more than %d entries", maxPluginRootEntries)
}
versions := make(map[string]struct{})
for _, entry := range entries {
if entry.Name() == preferredName || filepath.Ext(entry.Name()) != ".php" {
continue
}
info, infoErr := entry.Info()
if infoErr != nil {
return "", fmt.Errorf("stat plugin entry %s: %w", entry.Name(), infoErr)
}
if !info.Mode().IsRegular() {
continue
}
candidatePath := filepath.Join(pluginRoot, entry.Name())
candidateVersion, candidateFound, readErr := readPluginVersionHeader(candidatePath)
if readErr != nil {
return "", readErr
}
if candidateFound {
versions[candidateVersion] = struct{}{}
}
}
if len(versions) == 0 {
return "", errors.New("plugin header not found in root PHP files")
}
if len(versions) > 1 {
return "", errors.New("ambiguous plugin versions in root PHP files")
}
for candidateVersion := range versions {
return candidateVersion, nil
}
return "", errors.New("plugin version unavailable")
}
func readPluginVersionHeader(path string) (version string, found bool, err error) {
buf, err := readPackageHeader(path)
if err != nil {
return "", false, err
}
if !rePluginNameHeader.Match(buf) {
return "", false, nil
}
m := rePluginVersionHeader.FindSubmatch(buf)
if m == nil {
return "", false, nil
}
return string(m[1]), true, nil
}
// Header paths are account writable. Nonblocking opens followed by an fd stat
// reject FIFOs and devices, including ones reached through symlinks or swapped
// into place just before the open, without pinning an analyzer worker.
func readPackageHeader(path string) ([]byte, error) {
// #nosec G304 -- callers construct path from a scanner-derived root and a
// safe immediate child. Reads are bounded to WordPress's 8 KiB header limit.
f, err := os.OpenFile(path, os.O_RDONLY|unix.O_NONBLOCK, 0)
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() {
return nil, errors.New("package header is not a regular file")
}
buf, err := io.ReadAll(io.LimitReader(f, pluginHeaderReadLimit))
if err != nil {
return nil, fmt.Errorf("reading package header: %w", err)
}
return buf, nil
}
// pluginZipURL returns the canonical wordpress.org download URL for a given
// plugin slug and version.
func pluginZipURL(slug, version string) string {
return fmt.Sprintf("https://downloads.wordpress.org/plugin/%s.%s.zip", slug, version)
}
// FetchPluginChecksums downloads the plugin ZIP from wordpress.org,
// extracts each file, and returns a map of relative path -> SHA256 hex.
// The returned paths are relative to the plugin root (the leading
// "<slug>/" prefix from the ZIP entries is stripped).
func FetchPluginChecksums(slug, version string) (map[string]string, error) {
return fetchPluginChecksumsFromURL(pluginZipURL(slug, version), slug)
}
const maxPluginZipBytes = 100 << 20 // 100 MB ceiling
// ErrPluginNotInWPOrg is returned when wordpress.org responds with HTTP 404
// for a plugin slug+version. Plugins that are not in the wp.org repository
// (paid forks, custom internal plugins, slugs that simply do not exist) need
// to be distinguished from transient errors so the cache can suppress
// further fetch attempts for a TTL.
//
// 5xx responses, network errors, and malformed responses are NOT this
// error - those keep their normal retry behaviour because the plugin may
// still exist in the catalogue and wp.org may simply be having an outage.
var ErrPluginNotInWPOrg = errors.New("plugin not in wordpress.org repository")
func fetchPluginChecksumsFromURL(url, slug string) (map[string]string, error) {
resp, err := httpClient.Get(url) //nolint:gosec,bodyclose // httpClient has a timeout; body is closed below.
if err != nil {
return nil, fmt.Errorf("plugin zip GET failed: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusNotFound {
return nil, fmt.Errorf("plugin zip HTTP 404 from %s: %w", url, ErrPluginNotInWPOrg)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("plugin zip HTTP %d from %s", resp.StatusCode, url)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxPluginZipBytes))
if err != nil {
return nil, fmt.Errorf("reading plugin zip: %w", err)
}
zr, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
if err != nil {
return nil, fmt.Errorf("opening plugin zip: %w", err)
}
out := make(map[string]string, len(zr.File))
prefix := slug + "/"
for _, zf := range zr.File {
if zf.FileInfo().IsDir() {
continue
}
name := zf.Name
if !strings.HasPrefix(name, prefix) {
// Malformed ZIP (e.g. nested into a differently-named folder).
// Skip; callers detect partial results by checking cache emptiness.
continue
}
rel := filepath.Clean(strings.TrimPrefix(name, prefix))
// Reject path-traversal and absolute paths: a crafted ZIP entry
// named "<slug>/../../etc/passwd" would otherwise land in the
// checksum map. Defense-in-depth against a compromised CDN.
if rel == "." || strings.HasPrefix(rel, "..") || strings.HasPrefix(rel, "/") {
continue
}
rc, err := zf.Open()
if err != nil {
return nil, fmt.Errorf("opening zip entry %s: %w", name, err)
}
// Cap decompressed size per entry. Without this, a zip-bomb whose
// compressed body fits under maxPluginZipBytes can still exhaust
// memory during io.Copy. +1 lets us detect overflow.
limited := io.LimitReader(rc, maxPluginZipBytes+1)
h := sha256.New()
nCopied, err := io.Copy(h, limited)
_ = rc.Close()
if err != nil {
return nil, fmt.Errorf("hashing zip entry %s: %w", name, err)
}
if nCopied > maxPluginZipBytes {
return nil, fmt.Errorf("zip entry %s exceeds per-entry size cap", name)
}
out[rel] = hex.EncodeToString(h.Sum(nil))
}
if len(out) == 0 {
return nil, errors.New("plugin zip yielded no checksums")
}
return out, nil
}
// --- Cache plugin support ------------------------------------------------
func pluginKey(slug, version string) string {
// Account-written plugin headers must not alias core version/locale keys
// in the shared in-flight map.
return "plugin:" + slug + ":" + version
}
func (c *Cache) setPluginChecksums(slug, version string, checksums map[string]string) {
c.mu.Lock()
defer c.mu.Unlock()
if c.pluginChecksums == nil {
c.pluginChecksums = make(map[string]map[string]string)
}
c.pluginChecksums[pluginKey(slug, version)] = checksums
}
func (c *Cache) lookupPluginChecksum(slug, version, relPath string) (string, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
m, ok := c.pluginChecksums[pluginKey(slug, version)]
if !ok {
return "", false
}
h, ok := m[relPath]
return h, ok
}
func (c *Cache) hasPluginChecksums(slug, version string) bool {
c.mu.RLock()
_, ok := c.pluginChecksums[pluginKey(slug, version)]
c.mu.RUnlock()
return ok
}
// pluginNotFoundTTL bounds how long a wp.org 404 outcome suppresses
// re-fetches for the same slug+version. After expiry the next cache miss
// retries normally, so a plugin that wp.org publishes later will be
// picked up. 72 hours strikes a balance between not flooding wp.org with
// requests for non-existent plugins and propagating corrections in
// reasonable time.
const pluginNotFoundTTL = 72 * time.Hour
// markPluginNotFound records a wp.org 404 outcome so subsequent fetches
// short-circuit until ttl elapses. Caller passes ttl explicitly so tests
// can shorten or invert it; production code should use pluginNotFoundTTL.
func (c *Cache) markPluginNotFound(slug, version string, ttl time.Duration) {
key := pluginKey(slug, version)
c.mu.Lock()
if c.pluginNotFoundUntil == nil {
c.pluginNotFoundUntil = make(map[string]time.Time)
}
now := time.Now()
for oldKey, until := range c.pluginNotFoundUntil {
if !now.Before(until) {
delete(c.pluginNotFoundUntil, oldKey)
}
}
c.pluginNotFoundUntil[key] = now.Add(ttl)
c.mu.Unlock()
}
// isPluginNotFound reports whether an unexpired wp.org 404 marker exists
// for slug+version. Markers are scoped to slug+version so a fork of a
// plugin under a new version number that DOES exist on wp.org is still
// fetched.
func (c *Cache) isPluginNotFound(slug, version string) bool {
key := pluginKey(slug, version)
c.mu.RLock()
until, ok := c.pluginNotFoundUntil[key]
c.mu.RUnlock()
if !ok {
return false
}
return time.Now().Before(until)
}
func (c *Cache) startBackgroundPluginFetch(slug, version string) {
if c.isStopped() {
return
}
key := pluginKey(slug, version)
c.mu.Lock()
if c.fetching == nil {
c.fetching = make(map[string]bool)
}
// Recheck both outcomes under the admission lock: another request may
// have finished after the caller's cache lookup.
if c.fetching[key] || c.pluginChecksums[key] != nil || time.Now().Before(c.pluginNotFoundUntil[key]) {
c.mu.Unlock()
return
}
c.fetching[key] = true
c.mu.Unlock()
go c.fetchPluginWithRetry(slug, version, 0)
}
// fetchPluginWithRetry mirrors the core-checksum fetchWithRetry: the
// fetching flag stays set across retries so cache-miss events for the
// same slug/version do not spawn new goroutines. On exhaustion the flag
// is cleared so a future event can retry fresh.
//
// Special case: an HTTP 404 from wordpress.org is treated as a definitive
// "this plugin is not in the wp.org repository" signal. We mark the
// slug+version not-found for pluginNotFoundTTL and skip the retry cycle
// entirely. Network errors and 5xx responses keep their normal retry
// behaviour - those are transient.
func (c *Cache) fetchPluginWithRetry(slug, version string, attempt int) {
backoffs := []time.Duration{1 * time.Minute, 5 * time.Minute, 15 * time.Minute, 1 * time.Hour}
key := pluginKey(slug, version)
if c.isStopped() {
c.clearFetching(key)
return
}
checksums, err := FetchPluginChecksums(slug, version)
if err == nil {
c.setPluginChecksums(slug, version, checksums)
c.clearFetching(key)
fmt.Fprintf(os.Stderr, "wpcheck: cached %d checksums for plugin %s %s\n", len(checksums), slug, version)
return
}
if errors.Is(err, ErrPluginNotInWPOrg) {
c.markPluginNotFound(slug, version, pluginNotFoundTTL)
c.clearFetching(key)
fmt.Fprintf(os.Stderr, "wpcheck: plugin %s %s not in wp.org repository, suppressing retries for %s\n",
slug, version, pluginNotFoundTTL)
return
}
if attempt >= len(backoffs) {
c.clearFetching(key)
fmt.Fprintf(os.Stderr, "wpcheck: plugin fetch abandoned for %s %s after %d attempts: %v\n",
slug, version, attempt+1, err)
return
}
delay := backoffs[attempt]
fmt.Fprintf(os.Stderr, "wpcheck: plugin fetch failed for %s %s, retry in %v: %v\n",
slug, version, delay, err)
c.scheduleRetry(delay, func() {
c.fetchPluginWithRetry(slug, version, attempt+1)
}, func() {
c.clearFetching(key)
})
}
package wpcheck
import (
"time"
)
// SetStopCh wires a daemon-level cancellation channel into the cache.
// Closing the channel signals every pending checksum-retry timer to drop
// the scheduled fetch instead of firing. Safe to call once after NewCache,
// before any fetches start.
func (c *Cache) SetStopCh(stop <-chan struct{}) {
c.stopMu.Lock()
c.stopCh = stop
c.stopMu.Unlock()
}
func (c *Cache) currentStopCh() <-chan struct{} {
c.stopMu.RLock()
defer c.stopMu.RUnlock()
return c.stopCh
}
func (c *Cache) isStopped() bool {
return stopClosed(c.currentStopCh())
}
func stopClosed(stop <-chan struct{}) bool {
if stop == nil {
return false
}
select {
case <-stop:
return true
default:
return false
}
}
func (c *Cache) clearFetching(key string) {
c.mu.Lock()
delete(c.fetching, key)
c.mu.Unlock()
}
// scheduleRetry runs fn after delay unless the cache's stop channel
// closes first. Always returns immediately. Cancellation is necessary
// because the longest checksum-retry backoff (1 hour) survives daemon
// shutdown otherwise, causing wp.org fetches against torn-down state.
func (c *Cache) scheduleRetry(delay time.Duration, fn func(), onCancel func()) {
stop := c.currentStopCh()
go func() {
if stopClosed(stop) {
runCancel(onCancel)
return
}
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-timer.C:
if stopClosed(stop) {
runCancel(onCancel)
return
}
fn()
case <-stop:
runCancel(onCancel)
}
}()
}
func runCancel(onCancel func()) {
if onCancel != nil {
onCancel()
}
}
package wpcheck
import (
"crypto/md5" // #nosec G501 -- wordpress.org publishes MD5 digests for core files
"crypto/sha256"
"encoding/hex"
"io"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"golang.org/x/sys/unix"
)
// PackageKind names the WordPress package family a path belongs to.
type PackageKind int
const (
KindNone PackageKind = iota
KindCore
KindPlugin
KindTheme
)
// Verdict is the outcome of locating a file inside a WordPress package and,
// when wordpress.org publishes checksums for that package version, comparing
// the file against them. The first four values describe the package; the
// last three are comparison results.
type Verdict int
const (
// VerdictUnknown: the path is not inside a recognised package.
VerdictUnknown Verdict = iota
// VerdictNoVersion: the package is recognised but its version header is
// not readable. During an unpack the header file may simply not exist
// yet, so a caller should describe the path again later.
VerdictNoVersion
// VerdictPending: the checksums for this package version are being
// fetched. A digest taken now can be verified once they arrive.
VerdictPending
// VerdictUnavailable: wordpress.org has no checksums for this package
// version. Themes, premium plugins and private forks land here.
VerdictUnavailable
// VerdictReady: the checksums are cached and Verify can compare a digest.
VerdictReady
VerdictVerified
// VerdictMismatch: the digest differs from the official file, or the
// official package has no file at this relative path.
VerdictMismatch
// VerdictUnverifiable: checksums are cached but the file could not be
// hashed (over the size cap, or not a regular file).
VerdictUnverifiable
)
func (v Verdict) String() string {
switch v {
case VerdictUnknown:
return "unknown"
case VerdictNoVersion:
return "no-version"
case VerdictPending:
return "pending"
case VerdictUnavailable:
return "unavailable"
case VerdictReady:
return "ready"
case VerdictVerified:
return "verified"
case VerdictMismatch:
return "mismatch"
case VerdictUnverifiable:
return "unverifiable"
}
return "verdict(" + strconv.Itoa(int(v)) + ")"
}
// Verification identifies one file inside a WordPress package. Root is the
// package root on disk, Rel the path below it as the checksum manifest keys
// it. Digest holds the hex digest of the file content in the algorithm
// wordpress.org publishes for the kind: SHA-256 for plugins, MD5 for core.
type Verification struct {
Verdict Verdict
Kind PackageKind
Root string
Slug string
Version string
Locale string
Rel string
Digest string
// Staged pins a retained digest to this release instead of retrying it
// against a later installed-core header. RootInfo identifies the package
// directory observed while reading the header, before content analysis;
// Root can name its installed location after a staged plugin was moved.
Staged bool
RootInfo os.FileInfo
}
var reThemeVersionHeader = regexp.MustCompile(`(?im)^[ \t/*#@]*Version:[ \t]*([^\s]+)`)
var reThemeNameHeader = regexp.MustCompile(`(?im)^[ \t/*#@]*Theme Name:[ \t]*[^ \t\r\n]`)
// Describe locates path inside a WordPress core, plugin or theme tree and
// reports whether official checksums for that package version are cached,
// being fetched, or do not exist. It reads package headers only, never the
// file at path, and starts a background fetch on a cache miss.
func (c *Cache) Describe(path string) Verification {
// The outer package owns every manifest path, including bundled plugins
// in a core ZIP and plugin files with core-like names or subdirectories.
stagedRoot, stagedSlug := detectStagedPluginRoot(path)
installedRoot, installedSlug := detectInstalledPluginRoot(path)
if installedRoot != "" && (stagedRoot == "" || len(installedRoot) < len(stagedRoot)) {
return c.describePluginRoot(installedRoot, installedSlug, path)
}
if root, slug := stagedRoot, stagedSlug; root != "" {
return c.describeStagedRoot(root, slug, path)
}
core := c.describeCore(path)
if core.Verdict != VerdictUnknown && core.Verdict != VerdictNoVersion {
return core
}
if theme := describeTheme(path); theme.Verdict != VerdictUnknown {
return theme
}
return core
}
func (c *Cache) describeStagedRoot(root, slug, path string) Verification {
before, beforeErr := os.Lstat(root)
var v Verification
if slug == "wordpress" {
v = c.describeCoreRoot(root, path, true)
} else {
v = c.describePluginRoot(root, slug, path)
}
v.Staged = true
after, afterErr := os.Lstat(root)
if beforeErr == nil && afterErr == nil && before.IsDir() && os.SameFile(before, after) {
v.RootInfo = before
} else {
// A header from a replacement tree cannot identify this event. Keep
// its kind and relative path so its digest can still be retained.
v.Version, v.Locale = "", ""
v.Verdict = VerdictNoVersion
}
return v
}
func (c *Cache) describeCore(path string) Verification {
root := DetectWPRoot(path)
return c.describeCoreRoot(root, path, false)
}
func (c *Cache) describeCoreRoot(root, path string, staged bool) Verification {
if root == "" {
return Verification{Verdict: VerdictUnknown}
}
rel := RelativePath(root, path)
if rel == "" {
return Verification{Verdict: VerdictUnknown}
}
v := Verification{Kind: KindCore, Root: root, Rel: rel}
if rel == filepath.Join("wp-includes", "version.php") {
c.invalidateRoot(root)
}
// A staging pathname can be reused while its next header is still
// missing. Only installed roots may reuse a cached version.
var version, locale string
var ok bool
if !staged {
version, locale, ok = c.getRoot(root)
}
if !ok {
var err error
version, locale, err = ReadVersionFile(root)
if err != nil {
v.Verdict = VerdictNoVersion
return v
}
if !staged {
c.setRoot(root, version, locale)
}
}
v.Version, v.Locale = version, locale
if c.hasChecksums(version, locale) {
v.Verdict = VerdictReady
return v
}
c.startBackgroundFetch(version, locale)
v.Verdict = VerdictPending
return v
}
func (c *Cache) describePluginRoot(root, slug, path string) Verification {
rel := RelativePath(root, path)
if rel == "" {
return Verification{Verdict: VerdictUnknown}
}
v := Verification{Kind: KindPlugin, Root: root, Slug: slug, Rel: rel}
version, err := ReadPluginVersion(root, slug)
if err != nil || version == "" {
// The same staged layout carries themes. A style.css with a theme
// header settles the kind, and a theme has no checksum source.
if theme := describeThemeRoot(root, slug, rel); theme.Verdict != VerdictUnknown {
return theme
}
v.Verdict = VerdictNoVersion
return v
}
v.Version = version
switch {
case c.hasPluginChecksums(slug, version):
v.Verdict = VerdictReady
case c.isPluginNotFound(slug, version):
v.Verdict = VerdictUnavailable
default:
c.startBackgroundPluginFetch(slug, version)
v.Verdict = VerdictPending
}
return v
}
// describeTheme handles the installed theme layout. Staged themes share the
// staged plugin layout and are recognised from describePluginRoot.
func describeTheme(path string) Verification {
const themesSegment = "/wp-content/themes/"
idx := strings.Index(path, themesSegment)
if idx < 0 {
return Verification{Verdict: VerdictUnknown}
}
slug, rel, ok := strings.Cut(path[idx+len(themesSegment):], "/")
if !ok || rel == "" || !safePluginPathComponent(slug) {
return Verification{Verdict: VerdictUnknown}
}
return describeThemeRoot(path[:idx+len(themesSegment)]+slug, slug, rel)
}
func describeThemeRoot(root, slug, rel string) Verification {
buf, err := readPackageHeader(filepath.Join(root, "style.css"))
if err != nil || !reThemeNameHeader.Match(buf) {
return Verification{Verdict: VerdictUnknown}
}
v := Verification{Verdict: VerdictUnavailable, Kind: KindTheme, Root: root, Slug: slug, Rel: rel}
if m := reThemeVersionHeader.FindSubmatch(buf); m != nil {
v.Version = string(m[1])
}
return v
}
// Digest hashes the complete content behind fd with the algorithm
// wordpress.org publishes for kind. It returns "" when the file cannot be
// hashed whole, so a partial read can never verify as stock.
func (c *Cache) Digest(kind PackageKind, fd int) string {
switch kind {
case KindPlugin, KindCore:
default:
return ""
}
data := readCompleteFileForHash(fd)
if data == nil {
return ""
}
if kind == KindPlugin {
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:])
}
// #nosec G401 -- MD5 is required here: wordpress.org ships MD5 digests
// as the canonical integrity reference for core files. We compare
// against their published values, not derive authority from the hash.
sum := md5.Sum(data)
return hex.EncodeToString(sum[:])
}
// Verify compares v.Digest against the official checksum for v. A
// description taken while the fetch was still running resolves here once
// the checksums are cached, so callers can hash at event time and compare
// later without holding the file open.
func (c *Cache) Verify(v Verification) Verdict {
switch v.Kind {
case KindPlugin:
return c.verifyPlugin(v)
case KindCore:
return c.verifyCore(v)
}
return v.Verdict
}
func (c *Cache) verifyPlugin(v Verification) Verdict {
if v.Version == "" {
return VerdictNoVersion
}
if !c.hasPluginChecksums(v.Slug, v.Version) {
if c.isPluginNotFound(v.Slug, v.Version) {
return VerdictUnavailable
}
c.startBackgroundPluginFetch(v.Slug, v.Version)
return VerdictPending
}
if v.Digest == "" {
return VerdictUnverifiable
}
expected, ok := c.lookupPluginChecksum(v.Slug, v.Version, v.Rel)
if !ok {
return VerdictMismatch
}
if hexDigestEqual(v.Digest, expected) {
return VerdictVerified
}
return VerdictMismatch
}
func (c *Cache) verifyCore(v Verification) Verdict {
if v.Version == "" {
return VerdictNoVersion
}
if !c.hasChecksums(v.Version, v.Locale) {
c.startBackgroundFetch(v.Version, v.Locale)
return VerdictPending
}
if v.Digest == "" {
return VerdictUnverifiable
}
if expected, ok := c.lookupChecksum(v.Version, v.Locale, v.Rel); ok && hexDigestEqual(v.Digest, expected) {
return VerdictVerified
}
if v.Staged {
return VerdictMismatch
}
// A core update rewrites version.php part-way through. A file that fails
// against the version cached for this root may be stock for the version
// the root now declares.
c.invalidateRoot(v.Root)
version, locale, err := ReadVersionFile(v.Root)
if err != nil || (version == v.Version && locale == v.Locale) {
return VerdictMismatch
}
c.setRoot(v.Root, version, locale)
if !c.hasChecksums(version, locale) {
c.startBackgroundFetch(version, locale)
return VerdictPending
}
if expected, ok := c.lookupChecksum(version, locale, v.Rel); ok && hexDigestEqual(v.Digest, expected) {
return VerdictVerified
}
return VerdictMismatch
}
// VerifyFile describes path, hashes fd when the package kind has a checksum
// source, and resolves the comparison when the checksums are already cached.
func (c *Cache) VerifyFile(fd int, path string) Verification {
v := c.Describe(path)
if v.Staged && v.Kind == KindPlugin && v.Verdict == VerdictNoVersion {
if moved, ok := c.describeMovedStagedPlugin(fd, v); ok {
v = moved
}
}
switch v.Verdict {
case VerdictNoVersion, VerdictPending, VerdictReady:
v.Digest = c.Digest(v.Kind, fd)
}
if v.Verdict == VerdictReady {
v.Verdict = c.Verify(v)
}
return v
}
// describeMovedStagedPlugin identifies a staged plugin file that WordPress
// renamed into wp-content/plugins/ before the event was analyzed. The staging
// path is gone by then, but the event descriptor still holds the file. The
// installed header names its release only while the installed path is that
// same file: a copy, or an older release left by a failed update, says nothing
// about the bytes the event wrote.
func (c *Cache) describeMovedStagedPlugin(fd int, staged Verification) (Verification, bool) {
idx := strings.Index(staged.Root, upgradeSegment)
if idx < 0 {
return Verification{}, false
}
root := staged.Root[:idx] + pluginsSegment + staged.Slug
installed := filepath.Join(root, staged.Rel)
var event unix.Stat_t
if err := unix.Fstat(fd, &event); err != nil {
return Verification{}, false
}
// Pin the directory during the header read as well as the event file.
// Hard links can preserve a file across a tree swap, and an unpinned
// directory inode can be reused before the final identity check.
// #nosec G304 -- root is derived from the recognized staged plugin path.
dir, err := os.OpenFile(root, os.O_RDONLY|unix.O_NONBLOCK|unix.O_DIRECTORY|unix.O_NOFOLLOW, 0)
if err != nil {
return Verification{}, false
}
defer func() { _ = dir.Close() }()
before, err := dir.Stat()
if err != nil || !pathIsFile(installed, event) {
return Verification{}, false
}
v := c.describePluginRoot(root, staged.Slug, installed)
after, err := os.Lstat(root)
if v.Kind != KindPlugin || v.Version == "" || err != nil || !after.IsDir() ||
!os.SameFile(before, after) || !pathIsFile(installed, event) {
return Verification{}, false
}
v.Staged, v.RootInfo = true, before
return v, true
}
func pathIsFile(path string, file unix.Stat_t) bool {
var st unix.Stat_t
return unix.Lstat(path, &st) == nil && st.Dev == file.Dev && st.Ino == file.Ino
}
func hexDigestEqual(digest, expected string) bool {
raw, err := hex.DecodeString(digest)
if err != nil {
return false
}
return constantTimeHexDigestEqual(raw, expected)
}
// resolve hashes fd when the checksums for v are cached and returns the
// comparison, or v's own verdict when there is nothing to compare against.
func (c *Cache) resolve(fd int, v Verification) Verdict {
if v.Verdict != VerdictReady {
return v.Verdict
}
v.Digest = c.Digest(v.Kind, fd)
return c.Verify(v)
}
// IsVerifiedCoreFile reports whether path is an unmodified file of the
// WordPress core version its install declares.
func (c *Cache) IsVerifiedCoreFile(fd int, path string) bool {
v := c.Describe(path)
return v.Kind == KindCore && c.resolve(fd, v) == VerdictVerified
}
// IsVerifiedPluginFile reports whether path is an unmodified file of the
// wordpress.org release its plugin header declares, installed or staged.
func (c *Cache) IsVerifiedPluginFile(fd int, path string) bool {
v := c.Describe(path)
return v.Kind == KindPlugin && c.resolve(fd, v) == VerdictVerified
}
// PackageListing reports whether the official release of the WordPress core
// or wordpress.org plugin that path belongs to ships a file at that path.
// known is true only when that release's manifest is cached and the header
// naming the release, version.php or the plugin's main file, is the release's
// own copy: the account can edit either. Theme files, staged upgrades and core
// paths outside wp-admin and wp-includes, where sites add their own files, are
// never known. Only cached manifests are consulted; nothing is fetched.
func (c *Cache) PackageListing(path string) (listed, known bool) {
if root, _ := detectStagedPluginRoot(path); root != "" {
return false, false
}
if root, slug := detectInstalledPluginRoot(path); root != "" {
main := slug + ".php"
data, ok := readAuthenticHeader(filepath.Join(root, main))
if !ok {
return false, false
}
header := data[:min(len(data), pluginHeaderReadLimit)]
m := rePluginVersionHeader.FindSubmatch(header)
if m == nil || !rePluginNameHeader.Match(header) {
return false, false
}
version := string(m[1])
sum := sha256.Sum256(data)
// Authenticate and test membership against one cache state. A refresh
// must not turn a missing replacement manifest into a known absence.
c.mu.RLock()
defer c.mu.RUnlock()
manifest := c.pluginChecksums[pluginKey(slug, version)]
want, ok := manifest[main]
if !ok || !hexDigestEqual(hex.EncodeToString(sum[:]), want) {
return false, false
}
_, listed = manifest[RelativePath(root, path)]
return listed, true
}
if strings.Contains(path, "/wp-content/themes/") {
return false, false
}
root := DetectWPRoot(path)
if root == "" {
return false, false
}
rel := RelativePath(root, path)
if !strings.HasPrefix(rel, "wp-admin/") && !strings.HasPrefix(rel, "wp-includes/") {
return false, false
}
data, ok := readAuthenticHeader(filepath.Join(root, "wp-includes", "version.php"))
if !ok {
return false, false
}
version, locale, err := ParseVersionContent(data)
if err != nil {
return false, false
}
sum := md5.Sum(data) // #nosec G401 -- wordpress.org publishes MD5 digests for core files
c.mu.RLock()
defer c.mu.RUnlock()
manifest := c.checksums[cacheKey(version, locale)]
want, ok := manifest["wp-includes/version.php"]
if !ok || !hexDigestEqual(hex.EncodeToString(sum[:]), want) {
return false, false
}
_, listed = manifest[rel]
return listed, true
}
// authenticHeaderMax bounds a header file read whole for hashing. Release
// copies of version.php and plugin main files are far smaller.
const authenticHeaderMax = 1 << 20
// readAuthenticHeader reads a whole header file, refusing special files and
// anything over authenticHeaderMax. The caller parses and hashes the same
// bytes, so a file swapped between the two cannot pass.
func readAuthenticHeader(path string) ([]byte, bool) {
// #nosec G304 -- path is a header under a scanner-derived package root.
f, err := os.OpenFile(path, os.O_RDONLY|unix.O_NONBLOCK, 0)
if err != nil {
return nil, false
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil || !info.Mode().IsRegular() || info.Size() > authenticHeaderMax {
return nil, false
}
data, err := io.ReadAll(io.LimitReader(f, authenticHeaderMax+1))
if err != nil || len(data) > authenticHeaderMax {
return nil, false
}
return data, true
}
package yara
import (
"errors"
"fmt"
"io"
"os"
"sync/atomic"
"github.com/pidginhost/csm/internal/contenttype"
)
// Backend is the consumable scanning surface shared by the in-process
// *Scanner and out-of-process process supervisor. Callers should depend
// on this interface (via Active()) so they keep working when the daemon
// switches backends at startup. String-valued rule metadata travels on
// Match.Meta, so adapters that historically reached for the compiled
// *yara_x.Rules object (e.g. emailav) now work uniformly under both
// backends -- see internal/emailav/yarax.go.
type Backend interface {
ScanBytes(data []byte) []Match
RuleCount() int
Reload() error
}
// CheckedScanner is an optional capability on a Backend. ScanBytesChecked
// reports a scan failure (worker down, descriptor creation failure, or a
// transport error) distinctly from "no matches". Plain ScanBytes cannot: it
// returns nil for both a clean file and a failed scan, so a caller with a
// fail-closed policy -- email AV, finding re-check -- would auto-clear a file
// it never actually scanned. Both production backends implement it; the
// in-process stub returns a nil error because it cannot fail this way.
type CheckedScanner interface {
ScanBytesChecked(data []byte) ([]Match, error)
}
// CheckedFDScanner scans an open file the backend reads itself, such as a
// mail part memfd passed to the YARA worker.
type CheckedFDScanner interface {
ScanFDChecked(f *os.File, size int64) ([]Match, error)
}
// ScanFileDescriptorChecked scans the first size bytes of f, failing closed:
// a backend that cannot take the descriptor returns an error, never a clean
// result. A backend without the capability (in-process YARA) gets the bytes.
// No name goes with them, so the archive-by-name skip does not apply.
func ScanFileDescriptorChecked(b Backend, f *os.File, size int64) ([]Match, error) {
if size <= 0 {
return nil, fmt.Errorf("yara: invalid part size: %d", size)
}
if b == nil {
return nil, errors.New("yara: backend unavailable")
}
if fs, ok := b.(CheckedFDScanner); ok {
return fs.ScanFDChecked(f, size)
}
data, err := io.ReadAll(io.NewSectionReader(f, 0, size))
if err != nil {
return nil, fmt.Errorf("yara: read part: %w", err)
}
if int64(len(data)) != size {
return nil, fmt.Errorf("yara: read part: got %d of %d bytes: %w", len(data), size, io.ErrUnexpectedEOF)
}
return ScanBytesChecked(b, "", data)
}
// ScanBytesChecked scans data, read from the file called name, via b,
// surfacing a scan error when b supports the CheckedScanner capability.
// Backends without it fall back to the error-free ScanBytes. Callers that must
// fail closed on an unscannable payload should use this instead of
// Backend.ScanBytes.
//
// The archive policy lives here, at the backend-agnostic boundary, because it
// needs the name: raw archive bytes are not scannable and their stored entries
// trip rules with spurious tokens, so a file that is an archive by name and by
// magic is left to the extraction-time scan. Magic alone is not enough -- PHP
// executes past any leading bytes -- so an executable name is always scanned.
func ScanBytesChecked(b Backend, name string, data []byte) ([]Match, error) {
if b == nil {
return nil, errors.New("yara: backend unavailable")
}
if contenttype.IsArchiveFile(name, data) {
return nil, nil
}
if cs, ok := b.(CheckedScanner); ok {
return cs.ScanBytesChecked(data)
}
return b.ScanBytes(data), nil
}
var activeBackend atomic.Pointer[backendHolder]
var cacheGeneration atomic.Uint64
// CacheGeneration changes when rules or the active backend change. Rule
// counts alone cannot invalidate verdicts after an edit to an existing rule.
func CacheGeneration() uint64 { return cacheGeneration.Load() }
// InvalidateCaches retires verdicts after loading rules, including a worker
// restart that may load different on-disk content without an explicit reload.
func InvalidateCaches() { cacheGeneration.Add(1) }
type backendHolder struct{ b Backend }
// Active returns the configured scanning backend. When SetActive has
// not been called, it falls back to the in-process singleton Global().
// Returns a nil interface if neither is available (e.g. a !yara build
// with no supervisor wired up); callers must nil-check.
func Active() Backend {
if h := activeBackend.Load(); h != nil && h.b != nil {
return h.b
}
if g := Global(); g != nil {
return g
}
return nil
}
// SetActive installs a scanning backend. Calling with nil clears the
// override and restores the Global() fallback. Safe to call at any
// time; reads in-flight see the prior backend finish and the next read
// sees the new one.
func SetActive(b Backend) {
defer InvalidateCaches()
if b == nil {
activeBackend.Store(nil)
return
}
activeBackend.Store(&backendHolder{b: b})
}
package yara
import (
"fmt"
"os"
"sync"
)
var (
globalScanner *Scanner
globalOnce sync.Once
)
// Init initializes the global YARA-X scanner.
// Returns nil scanner if YARA-X is not compiled in or no rules found.
func Init(rulesDir string, disabled ...string) *Scanner {
if !Available() {
return nil
}
globalOnce.Do(func() {
s, err := NewScanner(rulesDir, disabled...)
if err != nil {
fmt.Fprintf(os.Stderr, "yara: init error: %v\n", err)
return
}
globalScanner = s
})
return globalScanner
}
// Global returns the global YARA-X scanner, or nil if not initialized.
func Global() *Scanner {
return globalScanner
}
package yara
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"syscall"
)
// validateRulesDir refuses to compile YARA rules from a directory or
// file whose ownership or permissions would let a non-root non-self
// account drop a rule that disables detection. Mirrors the same trust
// rules CSM applies to /etc/csm/conf.d: only root or the running
// process may own the dir or any rule file, and group/world write bits
// are refused. A missing directory is a no-op so an operator who has
// not installed YARA rules yet does not see startup failures.
func validateRulesDir(dir string) error {
if dir == "" {
return nil
}
info, err := os.Stat(dir)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return fmt.Errorf("rules dir stat: %w", err)
}
if !info.IsDir() {
return fmt.Errorf("rules dir is not a directory: %s", dir)
}
if trustErr := checkYaraEntryTrust(dir, info); trustErr != nil {
return trustErr
}
entries, err := os.ReadDir(dir)
if err != nil {
return fmt.Errorf("rules dir read: %w", err)
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
ext := strings.ToLower(filepath.Ext(entry.Name()))
if ext != ".yar" && ext != ".yara" {
continue
}
path := filepath.Join(dir, entry.Name())
fileInfo, err := os.Lstat(path)
if err != nil {
return fmt.Errorf("rule file stat %s: %w", path, err)
}
if fileInfo.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("rule file is a symlink: %s", path)
}
if !fileInfo.Mode().IsRegular() {
return fmt.Errorf("rule file is not a regular file: %s", path)
}
if trustErr := checkYaraEntryTrust(path, fileInfo); trustErr != nil {
return trustErr
}
}
return nil
}
// checkYaraEntryTrust enforces the perm + ownership trust rules on a
// single path. Used for both the rules dir itself and each rule file.
func checkYaraEntryTrust(path string, info os.FileInfo) error {
if mode := info.Mode().Perm(); mode&0022 != 0 {
return fmt.Errorf("rules path %s has unsafe mode %04o (group or world writable); set 0750 or stricter", path, mode)
}
sys, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return nil
}
selfUID := uint32(os.Geteuid()) // #nosec G115 -- Linux uid_t is uint32; os.Geteuid returns a non-negative process uid.
if sys.Uid != 0 && sys.Uid != selfUID {
return fmt.Errorf("rules path %s owner uid=%d is neither root nor process uid=%d; refusing to load untrusted rules", path, sys.Uid, selfUID)
}
return nil
}
//go:build !yara
package yara
import "io"
// Scanner is a no-op stub when YARA-X is not compiled in.
type Scanner struct{}
// Match represents a YARA rule that matched. Meta carries string-valued
// rule metadata pulled from `rule.Metadata()` (see yarax build); under
// this stub no rule ever matches, so Meta is never populated.
type Match struct {
RuleName string
Meta map[string]string
}
// NewScanner returns nil when YARA-X is not available.
func NewScanner(_ string, _ ...string) (*Scanner, error) {
return nil, nil
}
// DisabledRules returns nothing without YARA-X; no rules are compiled.
func (s *Scanner) DisabledRules() []string { return nil }
// DisabledRuleCount returns zero without YARA-X.
func (s *Scanner) DisabledRuleCount() int { return 0 }
// Reload is a no-op without YARA-X.
func (s *Scanner) Reload() error { return nil }
// ScanBytes returns nil without YARA-X.
func (s *Scanner) ScanBytes(_ []byte) []Match { return nil }
// ScanBytesChecked returns nil without YARA-X; there is no engine to fail.
func (s *Scanner) ScanBytesChecked(_ []byte) ([]Match, error) { return nil, nil }
// RuleCount returns 0 without YARA-X.
func (s *Scanner) RuleCount() int { return 0 }
// GlobalRules returns nil without YARA-X (no compiled rules available).
func (s *Scanner) GlobalRules() interface{} { return nil }
// Available returns false (YARA-X is not compiled in).
func Available() bool {
return false
}
// TestCompile is a no-op when YARA-X is not compiled in.
func TestCompile(source string) error {
return nil
}
// Rules is an empty stand-in: without YARA-X nothing compiles or loads.
type Rules struct{}
// CompileSources returns ErrNoEngine without YARA-X.
func CompileSources(_ []Source) (*Rules, error) { return nil, ErrNoEngine }
// CompileIsolatedSources returns ErrNoEngine without YARA-X.
func CompileIsolatedSources(_ []Source) (*Rules, error) { return nil, ErrNoEngine }
// LoadRules returns ErrNoEngine without YARA-X.
func LoadRules(_ io.Reader) (*Rules, error) { return nil, ErrNoEngine }
// Count returns 0 without YARA-X.
func (r *Rules) Count() int { return 0 }
// Destroy is a no-op without YARA-X.
func (r *Rules) Destroy() {}
// WriteTo returns ErrNoEngine without YARA-X.
func (r *Rules) WriteTo(_ io.Writer) (int64, error) { return 0, ErrNoEngine }
// Scan returns ErrNoEngine without YARA-X.
func (r *Rules) Scan(_ []byte) ([]Match, error) { return nil, ErrNoEngine }
package yara
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
)
// ErrNoEngine is returned by the compile and load entry points of a build
// without YARA-X.
var ErrNoEngine = errors.New("yara: YARA-X is not compiled into this build")
// Source is the text of one rule file, ready to compile. Name is the file's
// base name, so compile errors say which file is broken.
type Source struct {
Name string
Text []byte
}
// ReadSources reads the .yar and .yara files in rulesDir in name order, with
// the built-in suppressions and the operator's disabled rules stripped, and
// counts the rules stripped. A missing or unset directory is an empty rule
// set, not an error. Reading needs no engine, so the daemon can prepare rule
// text that a separate process compiles.
func ReadSources(rulesDir string, disabled []string) ([]Source, int, error) {
if rulesDir == "" {
return nil, 0, nil
}
// Refuse to read rules from a directory or file that a third party could
// overwrite. An attacker who can drop a blank rule silently disables
// every detection downstream of YARA.
if err := validateRulesDir(rulesDir); err != nil {
return nil, 0, err
}
entries, err := os.ReadDir(rulesDir)
if err != nil {
if os.IsNotExist(err) {
return nil, 0, nil
}
return nil, 0, fmt.Errorf("reading YARA rules dir: %w", err)
}
names := strippedRuleNames(disabled)
var sources []Source
disabledCount := 0
for _, entry := range entries {
name := entry.Name()
if entry.IsDir() {
continue
}
ext := strings.ToLower(filepath.Ext(name))
if ext != ".yar" && ext != ".yara" {
continue
}
path := filepath.Join(rulesDir, name)
data, err := os.ReadFile(path) // #nosec G304 -- ReadDir supplies a basename in the operator-selected directory; rule files are trust-checked above.
if err != nil {
return nil, 0, fmt.Errorf("reading %s: %w", path, err)
}
// Strip at compile time, not only at download time: a tier already on
// disk would otherwise keep firing until the next weekly update. The
// operator's own list rides along, so switching off a shipped rule
// does not mean editing rule files on a production host.
filtered, removed := stripRules(data, names)
disabledCount += removed
sources = append(sources, Source{Name: name, Text: filtered})
}
return sources, disabledCount, nil
}
// strippedRuleNames merges the built-in suppressions with the operator's
// disabled list, dropping blanks and duplicates.
func strippedRuleNames(disabled []string) []string {
builtin := SuppressedRuleNames()
seen := make(map[string]struct{}, len(builtin)+len(disabled))
out := make([]string, 0, len(builtin)+len(disabled))
for _, name := range append(builtin, disabled...) {
trimmed := strings.TrimSpace(name)
if trimmed == "" {
continue
}
if _, dup := seen[trimmed]; dup {
continue
}
seen[trimmed] = struct{}{}
out = append(out, trimmed)
}
return out
}
package yara
import "strings"
// YARA Forge aggregates community rulesets of mixed quality. Most are useful,
// but a rule whose condition ordinary code satisfies fires on every account on
// a shared host at once, and that volume hides real detections. Those rules are
// stripped both when a tier is downloaded and when rules are compiled, so a
// tier already on disk stops firing at the next reload instead of at the next
// weekly update.
//
// A rule belongs here only when its condition -- not merely its subject -- is
// unsound, and only when CSM detects the same technique itself. Suppressing a
// rule is never a substitute for detection: pair every entry with a rule in
// configs/malware.yar.
var suppressedRuleNames = []string{
// Detects HTML smuggling by requiring exactly one "payload marker", but the
// marker set mixes base64 file headers with the generic `.charCodeAt(i)^`
// XOR idiom. Any minified bundle that offers a client-side download and
// hashes a string satisfies it with nothing smuggled, which reported stock
// analytics plugins and cached page assets as malware delivery.
// Replaced by html_smuggling_payload, which requires the encoded header.
"ELCEEF_HTML_Smuggling_A",
// Detects a Windows IIS native-module family, but its condition is
// `native_module_private and 2 of ($i*) or 5 of them`. The trailing arm
// drops the native-module guard, so five generic strings decide the
// match -- and four of them are crawler user agents (Baiduspider,
// 360Spider, Sogou, YisouSpider) that any PHP bot-filter table lists
// alongside HTTP_X_FORWARDED_FOR. Stock analytics plugins matched.
// Replaced by seo_cloak_group14_ioc, which binds the campaign markers to
// native-module or campaign-infrastructure context and drops the user-agent
// arm.
"ESET_IIS_Group14",
}
// SuppressedRuleNames returns the built-in rule suppressions.
func SuppressedRuleNames() []string {
out := make([]string, len(suppressedRuleNames))
copy(out, suppressedRuleNames)
return out
}
// StripRules removes the named rules, bodies included, from YARA source text.
// Unknown names are ignored so a suppression list may name rules that a given
// tier does not ship.
func StripRules(content []byte, names []string) []byte {
filtered, _ := stripRules(content, names)
return filtered
}
// stripRules also counts removed declarations so a deliberately empty set
// remains distinguishable from a missing or empty rule file.
func stripRules(content []byte, names []string) ([]byte, int) {
if len(names) == 0 {
return content, 0
}
drop := make(map[string]bool, len(names))
for _, name := range names {
if name = strings.ToLower(strings.TrimSpace(name)); name != "" {
drop[name] = true
}
}
if len(drop) == 0 {
return content, 0
}
var result []byte
kept := 0
removed := 0
for _, rule := range sourceRules(content) {
if drop[strings.ToLower(rule.name)] {
removed++
result = append(result, content[kept:rule.start]...)
kept = rule.end
}
}
if kept == 0 {
return content, 0
}
return append(result, content[kept:]...), removed
}
type sourceRule struct {
name string
start, end int
}
// RuleNames returns declarations outside comments and literals. Validation
// uses the same boundaries as stripping, including modifiers and compact files.
func RuleNames(content []byte) []string {
var names []string
for _, rule := range sourceRules(content) {
names = append(names, rule.name)
}
return names
}
// sourceRules locates complete rule bodies without interpreting conditions.
// Counting source braces or removing whole lines can silently erase a neighbor:
// braces also occur in strings, comments, regexes and hex patterns.
func sourceRules(content []byte) []sourceRule {
var rules []sourceRule
start := -1
for pos := 0; pos < len(content); {
token, from, end := yaraToken(content, pos)
pos = end
if token == "private" || token == "global" {
if start < 0 {
start = from
}
continue
}
if token != "rule" {
start = -1
continue
}
if start < 0 {
start = from
}
name, _, next := yaraToken(content, pos)
pos = next
if !yaraIdentifier(name) {
break
}
token, _, pos = yaraToken(content, pos)
if token == ":" {
token, _, pos = yaraToken(content, pos)
if !yaraIdentifier(token) {
break
}
for yaraIdentifier(token) {
token, _, pos = yaraToken(content, pos)
}
}
if token != "{" {
break
}
depth := 1
for pos < len(content) && depth > 0 {
token, _, pos = yaraToken(content, pos)
switch token {
case "{":
depth++
case "}":
depth--
}
}
// Leave malformed source intact so compilation still reports it.
if depth != 0 {
break
}
rules = append(rules, sourceRule{name: name, start: start, end: pos})
start = -1
}
return rules
}
// yaraToken skips trivia and consumes quoted/regex literals as single tokens.
// YARA uses backslash for division, so slash always starts a regex or comment.
func yaraToken(src []byte, pos int) (token string, start, end int) {
for pos < len(src) {
switch src[pos] {
case ' ', '\t', '\r', '\n', '\f', '\v':
pos++
continue
case '/':
if pos+1 < len(src) && src[pos+1] == '/' {
pos += 2
for pos < len(src) && src[pos] != '\n' && src[pos] != '\r' {
pos++
}
continue
}
if pos+1 < len(src) && src[pos+1] == '*' {
pos += 2
for pos+1 < len(src) && (src[pos] != '*' || src[pos+1] != '/') {
pos++
}
pos = min(pos+2, len(src))
continue
}
}
break
}
start = pos
if pos == len(src) {
return "", pos, pos
}
ch := src[pos]
pos++
if ch == '"' || ch == '/' {
inClass := false
for pos < len(src) {
c := src[pos]
pos++
if c == '\\' {
pos = min(pos+1, len(src))
continue
}
if ch == '/' {
switch c {
case '[':
inClass = true
case ']':
inClass = false
}
}
if c == ch && !inClass {
break
}
}
return string(src[start:pos]), start, pos
}
if yaraIdent(ch) {
for pos < len(src) && yaraIdent(src[pos]) {
pos++
}
}
return string(src[start:pos]), start, pos
}
func yaraIdent(ch byte) bool {
return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch >= '0' && ch <= '9' || ch == '_'
}
func yaraIdentifier(token string) bool {
// A following declaration must not become part of a malformed tag list.
if token == "rule" || token == "private" || token == "global" {
return false
}
if token == "" || token[0] >= '0' && token[0] <= '9' {
return false
}
for i := range len(token) {
if !yaraIdent(token[i]) {
return false
}
}
return true
}
// RuleNameFromLine returns the rule name declared on a source line, or "" when
// the line does not open a rule.
func RuleNameFromLine(line string) string {
s := strings.TrimPrefix(line, "private ")
if !strings.HasPrefix(s, "rule ") {
return ""
}
s = s[5:]
for i, ch := range s {
if ch == ' ' || ch == '\t' || ch == ':' || ch == '{' {
return s[:i]
}
}
return s
}
package yaraipc
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"unicode/utf8"
)
// DecodeScanReply decodes a scan_fd reply payload. It requires an explicit
// matches array, including for a clean scan, and refuses unknown fields and
// trailing data: an empty or unrelated reply must not stand in for a
// completed scan. A clean scan returns an empty, non-nil slice.
func DecodeScanReply(payload []byte) ([]Match, error) {
if !utf8.Valid(payload) {
return nil, errors.New("yaraipc: invalid scan_fd reply")
}
var res scanFDResult
dec := json.NewDecoder(bytes.NewReader(payload))
dec.DisallowUnknownFields()
if err := dec.Decode(&res); err != nil {
return nil, fmt.Errorf("yaraipc: decode scan_fd reply: %w", err)
}
if res.Matches == nil {
return nil, errors.New("yaraipc: scan_fd reply has no matches array")
}
if err := dec.Decode(new(json.RawMessage)); err != io.EOF {
return nil, errors.New("yaraipc: scan_fd reply has trailing data")
}
return res.Matches, nil
}
// ReopenReadOnly opens a second, read-only description of the regular file f,
// so a worker that receives it cannot write the file or move the caller's
// offset, and the caller's flags stay as they were.
func ReopenReadOnly(f *os.File) (*os.File, error) {
if f == nil {
return nil, errors.New("yaraipc: no descriptor")
}
st, err := f.Stat()
if err != nil {
return nil, err
}
if !st.Mode().IsRegular() {
return nil, errors.New("descriptor is not a regular file")
}
rc, err := f.SyscallConn()
if err != nil {
return nil, err
}
var ro *os.File
var openErr error
// Control keeps the source open across the reopen even if the scan's
// owner closes it after a timeout. Fd neither pins it nor preserves flags.
if err := rc.Control(func(fd uintptr) {
// #nosec G304 -- reopens a descriptor this process holds open inside Control.
ro, openErr = os.OpenFile(fmt.Sprintf("/proc/self/fd/%d", fd), os.O_RDONLY, 0)
}); err != nil {
return nil, err
}
return ro, openErr
}
//go:build linux
package yaraipc
import (
"fmt"
"io"
"os"
"golang.org/x/sys/unix"
)
// memfdCreate is a seam so tests can make memfd creation fail.
var memfdCreate = unix.MemfdCreate
// SealedMemfd copies data into a memory-backed file and seals it, so the
// worker that receives it can read the bytes but cannot change them.
func SealedMemfd(name string, data []byte) (*os.File, error) {
f, err := newMemfd(name)
if err != nil {
return nil, err
}
if _, err = f.Write(data); err == nil {
err = sealFile(f)
}
if err != nil {
_ = f.Close()
return nil, fmt.Errorf("yaraipc: fill %s memfd: %w", name, err)
}
return f, nil
}
// CopySealedMemfd copies the first size bytes of src into a new sealed memfd
// owned by this process, so nothing another process kept open can change it.
// A source shorter than size is an error.
func CopySealedMemfd(name string, src *os.File, size int64) (*os.File, error) {
f, err := newMemfd(name)
if err != nil {
return nil, err
}
n, err := io.Copy(f, io.NewSectionReader(src, 0, size))
if err == nil && n != size {
err = fmt.Errorf("copied %d of %d bytes: %w", n, size, io.ErrUnexpectedEOF)
}
if err == nil {
err = sealFile(f)
}
if err != nil {
_ = f.Close()
return nil, fmt.Errorf("yaraipc: copy %s memfd: %w", name, err)
}
return f, nil
}
func newMemfd(name string) (*os.File, error) {
fd, err := memfdCreate(name, unix.MFD_CLOEXEC|unix.MFD_ALLOW_SEALING)
if err != nil {
return nil, fmt.Errorf("yaraipc: create %s memfd: %w", name, err)
}
return os.NewFile(uintptr(fd), name), nil // #nosec G115 -- memfd_create returned a valid descriptor.
}
func sealFile(f *os.File) error {
rc, err := f.SyscallConn()
if err != nil {
return err
}
var sealErr error
if err = rc.Control(func(fd uintptr) {
_, sealErr = unix.FcntlInt(fd, unix.F_ADD_SEALS, unix.F_SEAL_WRITE|unix.F_SEAL_GROW|unix.F_SEAL_SHRINK|unix.F_SEAL_SEAL)
}); err != nil {
return err
}
return sealErr
}
package yaraworker
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"sync"
"sync/atomic"
"time"
"golang.org/x/sync/semaphore"
"github.com/pidginhost/csm/internal/obs"
"github.com/pidginhost/csm/internal/queuehealth"
"github.com/pidginhost/csm/internal/workerproc"
"github.com/pidginhost/csm/internal/yara"
"github.com/pidginhost/csm/internal/yaraipc"
)
// BackendConfig describes the daemon side of out-of-process YARA.
type BackendConfig struct {
// ScanLaunch starts the scan worker and CompileLaunch a compile worker.
// The daemon passes locked-down unit launchers.
ScanLaunch, CompileLaunch workerproc.Launcher
RulesDir string
DisabledRules []string
// ScanTimeout bounds one scan, LoadTimeout a fresh worker loading the
// compiled rules, CompileTimeout one compile.
ScanTimeout, LoadTimeout, CompileTimeout time.Duration
// OnFailure is told of every counted scan worker failure, and OnJob of
// every scan worker request outcome. Both run under the worker
// supervisor's lock and must not block.
OnFailure func(reason string, err error)
OnJob func(outcome string, memory uint64)
Log func(string, ...any)
// WarmAfterReload starts a scan worker in the background after every
// successful reload, so the rules load off the scan path.
WarmAfterReload bool
}
// Backend is the YARA backend the daemon installs with yara.SetActive. It
// reads rule text, has a short-lived compile worker compile it, keeps the
// compiled rules in a sealed memfd, and loads them into each scan worker it
// starts: a worker restart reloads, it never recompiles. Every scan reaches
// the worker as a descriptor, and a scan that did not run is an error.
type Backend struct {
cfg BackendConfig
proc *workerproc.Supervisor
ctx context.Context
stop context.CancelFunc
// compileGate runs one compile at a time: each can need a gigabyte.
// Waiting reloads must be able to cancel without stopping its owner.
compileGate *semaphore.Weighted
mu sync.Mutex
rules *compiledRules
compileErr string
stopped bool
// gen numbers rule sets. A worker loaded with an older one retires.
gen atomic.Uint64
// strikes remembers, by SHA-256, content that made the scan worker fail
// under the current rules.
strikes atomic.Pointer[workerproc.Strikes]
// loadedStrikes follows the rules actually loaded by launch, even when
// a reload lands during admission. Only the supervisor's lock protects it.
loadedStrikes *workerproc.Strikes
scanErrs scanErrLog
}
// compiledRules is one compiled rule set, held by the daemon.
type compiledRules struct {
file *os.File
count int
gen uint64
}
// NewBackend validates cfg. It compiles nothing and starts no worker.
func NewBackend(cfg BackendConfig) (*Backend, error) {
if cfg.ScanLaunch == nil || cfg.CompileLaunch == nil {
return nil, errors.New("yaraworker: scan and compile launchers are required")
}
if cfg.ScanTimeout <= 0 || cfg.LoadTimeout <= 0 || cfg.CompileTimeout <= 0 {
return nil, errors.New("yaraworker: timeouts must be positive")
}
// Scans fail closed until a reload compiles the rules or finds none to
// compile; a reload that ends any other way leaves this in place.
b := &Backend{cfg: cfg, compileGate: semaphore.NewWeighted(1), compileErr: "no YARA rules compiled yet"}
proc, err := workerproc.New(workerproc.Config{
Name: "yara worker",
Launch: b.launch,
Timeout: cfg.ScanTimeout,
// MaxFDs 0: a scan reply carrying a descriptor is a protocol failure.
Limits: workerproc.Limits{MaxFrameBytes: yaraipc.MaxFrameBytes},
Log: cfg.Log,
OnFailure: cfg.OnFailure,
OnJob: cfg.OnJob,
// A file that kills the worker must not switch YARA off for every
// other file on the host: only a failed start counts toward the
// breaker, so hostile content costs one restart per scan.
BreakOnSpawnOnly: true,
})
if err != nil {
return nil, err
}
b.proc = proc
b.ctx, b.stop = context.WithCancel(context.Background())
b.strikes.Store(workerproc.NewStrikes())
return b, nil
}
// Reload compiles the rules directory in a compile worker and, on success,
// makes the result the rules every later scan uses. On failure the current
// rules stay; with none loaded the failure becomes CompileError and scans
// fail closed. An error that workerproc.FailureReason classifies came from
// the compile worker, not the rules.
func (b *Backend) Reload() error {
return b.ReloadContext(context.Background())
}
// ReloadContext lets startup retries cancel a compile as soon as shutdown
// begins, before the daemon finishes draining its other workers.
func (b *Backend) ReloadContext(ctx context.Context) error {
if err := b.compileGate.Acquire(ctx, 1); err != nil {
return err
}
defer b.compileGate.Release(1)
if b.isStopped() {
return workerproc.ErrStopped
}
if err := ctx.Err(); err != nil {
return err
}
sources, disabled, err := yara.ReadSources(b.cfg.RulesDir, b.cfg.DisabledRules)
if err != nil {
return b.failReload(err)
}
if len(sources) > yaraipc.MaxSources {
return b.failReload(fmt.Errorf("%d YARA rule files in %s, more than the %d one compile accepts; merge rule files or set signatures.yara_worker_enabled: false", len(sources), b.cfg.RulesDir, yaraipc.MaxSources))
}
if len(sources) == 0 {
b.mu.Lock()
defer b.mu.Unlock()
if b.stopped {
return workerproc.ErrStopped
}
if b.rules != nil {
return fmt.Errorf("no YARA rule files found in %s", b.cfg.RulesDir)
}
b.compileErr = ""
return nil
}
f, count, err := b.compile(ctx, sources, true)
if err == nil && count == 0 && disabled == 0 {
_ = f.Close()
err = fmt.Errorf("no YARA rules compiled from %s", b.cfg.RulesDir)
}
if err != nil {
return b.failReload(err)
}
b.mu.Lock()
if b.stopped || ctx.Err() != nil {
b.mu.Unlock()
_ = f.Close()
if err := ctx.Err(); err != nil {
return err
}
return workerproc.ErrStopped
}
old := b.rules
b.rules = &compiledRules{file: f, count: count, gen: b.gen.Add(1)}
b.compileErr = ""
// The new rules get a new chance at content that failed the old ones.
b.strikes.Store(workerproc.NewStrikes())
b.mu.Unlock()
if old != nil {
_ = old.file.Close()
}
yara.InvalidateCaches()
if b.cfg.WarmAfterReload && count > 0 {
// A failed warm-up is reported through OnFailure like any request.
obs.Go("yara-worker-warm", func() { _ = b.Warm() })
}
return nil
}
func (b *Backend) failReload(err error) error {
b.mu.Lock()
defer b.mu.Unlock()
if b.stopped {
return workerproc.ErrStopped
}
if b.rules == nil {
b.compileErr = err.Error()
}
return err
}
// TestCompile compiles source alone, without touching the scanning rules,
// so a downloaded rule file can be checked before it is installed.
func (b *Backend) TestCompile(source string) error {
if err := b.compileGate.Acquire(b.ctx, 1); err != nil {
return workerproc.ErrStopped
}
defer b.compileGate.Release(1)
if b.isStopped() {
return workerproc.ErrStopped
}
_, count, err := b.compile(context.Background(), []yara.Source{{Name: "source", Text: []byte(source)}}, false)
if err != nil {
return err
}
if count == 0 {
return errors.New("no rules compiled from source")
}
return nil
}
// copySealedMemfd is a seam for tests that count rule copies.
var copySealedMemfd = yaraipc.CopySealedMemfd
// compile runs one compile worker over sources and returns the rule count.
// With keep it also returns the compiled rules copied into a memfd this
// process owns and sealed.
func (b *Backend) compile(ctx context.Context, sources []yara.Source, keep bool) (*os.File, int, error) {
if b.isStopped() {
return nil, 0, workerproc.ErrStopped
}
ctx, cancel := context.WithCancel(ctx)
stop := context.AfterFunc(b.ctx, cancel)
defer func() { stop(); cancel() }()
if err := ctx.Err(); err != nil {
return nil, 0, err
}
var args yaraipc.CompileArgs
var text bytes.Buffer
for _, src := range sources {
args.Sources = append(args.Sources, yaraipc.SourceSpan{Name: src.Name, Size: int64(len(src.Text))})
text.Write(src.Text)
}
input, err := yaraipc.SealedMemfd("csm-yara-sources", text.Bytes())
if err != nil {
return nil, 0, err
}
defer func() { _ = input.Close() }()
payload, err := json.Marshal(args)
if err != nil {
return nil, 0, err
}
sup, err := workerproc.New(workerproc.Config{
Name: "yara compile worker",
Launch: b.cfg.CompileLaunch,
Timeout: b.cfg.CompileTimeout,
Limits: workerproc.Limits{MaxFrameBytes: yaraipc.MaxFrameBytes, MaxFDs: 1},
Log: b.cfg.Log,
})
if err != nil {
return nil, 0, err
}
defer func() { _ = sup.Stop() }()
var res struct {
RuleCount *int `json:"rule_count"`
CompileError string `json:"compile_error,omitempty"`
}
var size int64
_, fds, err := sup.Call(ctx, workerproc.Frame{Op: yaraipc.OpCompile, Payload: payload}, []*os.File{input},
func(reply workerproc.Frame, fds []*os.File) (bool, error) {
if decodeErr := decodeArgs(reply.Payload, &res); decodeErr != nil {
return false, fmt.Errorf("compile reply: %w", decodeErr)
}
if res.RuleCount == nil || *res.RuleCount < 0 {
return false, errors.New("compile reply: missing or invalid rule count")
}
if res.CompileError != "" {
if len(fds) != 0 || *res.RuleCount != 0 {
return false, errors.New("compile reply: a failed compile returned rules")
}
return false, nil
}
if len(fds) != 1 {
return false, fmt.Errorf("compile reply: %d descriptors, want 1", len(fds))
}
st, statErr := fds[0].Stat()
if statErr != nil {
return false, fmt.Errorf("compile reply: %w", statErr)
}
if !st.Mode().IsRegular() || st.Size() <= 0 || st.Size() > yaraipc.MaxSerializedRules {
return false, fmt.Errorf("compile reply: compiled rules of %d bytes, want 1 to %d", st.Size(), yaraipc.MaxSerializedRules)
}
size = st.Size()
return false, nil
})
if err != nil {
return nil, 0, compileFailure(err)
}
if res.CompileError != "" {
return nil, 0, errors.New(res.CompileError)
}
defer func() { _ = fds[0].Close() }()
if !keep {
return nil, *res.RuleCount, nil
}
kept, err := copySealedMemfd("csm-yara-rules", fds[0], size)
if err != nil {
return nil, 0, err
}
return kept, *res.RuleCount, nil
}
// compileFailure says why a compile worker failed. Running out of memory
// under the compile worker's fixed cap comes from the rule set, so it is
// returned as a plain error that callers treat like broken rules, not as a
// worker outage to retry. Other failures keep their classification and name
// their reason.
func compileFailure(err error) error {
switch reason := workerproc.FailureReason(err); reason {
case "":
return err
case workerproc.ReasonOOM:
return fmt.Errorf("the YARA rule set needs more memory than the compile worker may use (%v); trim the rules or set signatures.yara_worker_enabled: false", err)
default:
return fmt.Errorf("%w (%s)", err, reason)
}
}
// launch starts a scan worker and loads the current rules into it before the
// supervisor may send it a scan.
func (b *Backend) launch(ctx context.Context) (workerproc.Process, error) {
b.mu.Lock()
rules := b.rules
strikes := b.strikes.Load()
var ro *os.File
var err error
if rules != nil {
// Under the lock: a reload closes the file it replaces.
ro, err = yaraipc.ReopenReadOnly(rules.file)
}
b.mu.Unlock()
if rules == nil {
return nil, errors.New("no compiled YARA rules")
}
if err != nil {
return nil, fmt.Errorf("compiled YARA rules: %w", err)
}
defer func() { _ = ro.Close() }()
p, err := b.cfg.ScanLaunch(ctx)
if err != nil {
return nil, err
}
if err := b.loadRules(ctx, p, ro, rules.count); err != nil {
// Asked before the kill: the unit's cgroup holds the evidence.
if r, ok := p.(interface{ OOMKilled() bool }); ok && r.OOMKilled() {
err = fmt.Errorf("%w (out of memory)", err)
}
_ = p.Kill()
_ = p.Conn().Close()
return nil, err
}
b.loadedStrikes = strikes
return &rulesProcess{Process: p, gen: rules.gen, current: &b.gen}, nil
}
func (b *Backend) loadRules(ctx context.Context, p workerproc.Process, rules *os.File, count int) (retErr error) {
conn := p.Conn()
deadline := time.Now().Add(b.cfg.LoadTimeout)
if d, ok := ctx.Deadline(); ok && d.Before(deadline) {
deadline = d
}
if err := conn.SetDeadline(deadline); err != nil {
return fmt.Errorf("load rules: %w", err)
}
var canceled sync.WaitGroup
canceled.Add(1)
stop := context.AfterFunc(ctx, func() {
defer canceled.Done()
_ = conn.SetDeadline(time.Now())
})
defer func() {
// A cancellation callback already running must finish before the
// deadline is cleared, or it could poison the first scan.
if !stop() {
canceled.Wait()
}
_ = conn.SetDeadline(time.Time{})
if ctxErr := ctx.Err(); ctxErr != nil {
retErr = ctxErr
}
}()
if err := workerproc.SendFrame(conn, workerproc.Frame{Op: yaraipc.OpLoad}, []*os.File{rules}, yaraipc.MaxFrameBytes); err != nil {
return fmt.Errorf("load rules: %w", err)
}
// MaxFDs 0: ReceiveFrame closes and refuses any descriptor.
reply, _, err := workerproc.ReceiveFrame(conn, workerproc.Limits{MaxFrameBytes: yaraipc.MaxFrameBytes})
if err != nil {
return fmt.Errorf("load rules: %w", err)
}
if reply.Error != "" || reply.Op != "" {
return fmt.Errorf("load rules: worker refused: %q", reply.Error)
}
loaded, err := decodeRuleCount(reply.Payload)
if err != nil {
return fmt.Errorf("load rules: %w", err)
}
if loaded != count {
return fmt.Errorf("load rules: worker loaded %d rules, want %d", loaded, count)
}
return nil
}
func decodeRuleCount(payload json.RawMessage) (int, error) {
var res struct {
RuleCount *int `json:"rule_count"`
}
if err := decodeArgs(payload, &res); err != nil {
return 0, err
}
if res.RuleCount == nil || *res.RuleCount < 0 {
return 0, errors.New("missing or invalid rule count")
}
return *res.RuleCount, nil
}
// rulesProcess is a scan worker that retires once newer rules exist. It
// passes on the optional memory interfaces workerproc looks for, which
// embedding the Process interface alone would hide.
type rulesProcess struct {
workerproc.Process
gen uint64
current *atomic.Uint64
}
func (p *rulesProcess) Retire() bool {
return p.gen != p.current.Load() || p.Process.Retire()
}
func (p *rulesProcess) MemoryUsed() uint64 {
if r, ok := p.Process.(interface{ MemoryUsed() uint64 }); ok {
return r.MemoryUsed()
}
return 0
}
func (p *rulesProcess) OOMKilled() bool {
if r, ok := p.Process.(interface{ OOMKilled() bool }); ok {
return r.OOMKilled()
}
return false
}
// ScanBytes flattens a failed scan to no matches; callers that must fail
// closed use ScanBytesChecked.
func (b *Backend) ScanBytes(data []byte) []yara.Match {
m, _ := b.ScanBytesChecked(data)
return m
}
// ScanBytesChecked scans bytes the daemon holds; they reach the worker as a
// sealed memfd. With no rules, or a set whose every rule is disabled, it
// answers clean without a worker, as the in-process scanner does, unless the
// rules failed to compile.
func (b *Backend) ScanBytesChecked(data []byte) ([]yara.Match, error) {
rules, compileErr, stopped := b.state()
if stopped {
return nil, workerproc.ErrStopped
}
if len(data) == 0 {
return nil, nil
}
if rules == nil || rules.count == 0 {
if compileErr == "" {
return nil, nil
}
return nil, fmt.Errorf("yara scanner unavailable: %s", compileErr)
}
f, err := yaraipc.SealedMemfd("csm-yara-scan", data)
if err != nil {
b.logScanErr(err)
return nil, err
}
defer func() { _ = f.Close() }()
return b.scan(f, len(data))
}
// ScanFDChecked scans the first size bytes of f, such as a mail part. The
// worker gets its own read-only open. It requires at least one compiled rule.
func (b *Backend) ScanFDChecked(f *os.File, size int64) ([]yara.Match, error) {
if size <= 0 || size > int64(int(^uint(0)>>1)) {
return nil, fmt.Errorf("yaraworker: invalid descriptor scan size: %d", size)
}
rules, compileErr, stopped := b.state()
if stopped {
return nil, workerproc.ErrStopped
}
if rules == nil || rules.count == 0 {
if compileErr == "" {
compileErr = "no compiled rules"
}
return nil, fmt.Errorf("yara scanner unavailable: %s", compileErr)
}
ro, err := yaraipc.ReopenReadOnly(f)
if err != nil {
err = fmt.Errorf("yaraworker: reopen read-only: %w", err)
b.logScanErr(err)
return nil, err
}
defer func() { _ = ro.Close() }()
return b.scan(ro, int(size))
}
// errStruckOut refuses content that kept making the scan worker fail: each
// attempt would cost a worker restart, a rules load and up to a timeout.
var errStruckOut = errors.New("yaraworker: not scanned: this content made the YARA worker fail repeatedly")
func (b *Backend) scan(f *os.File, size int) ([]yara.Match, error) {
key := ""
// Both callbacks run in the supervisor's request order. Checking before
// joining its queue would let queued copies bypass newly recorded strikes.
before := func() error {
strikes := b.strikes.Load()
if strikes.Len() > 0 {
var err error
if key, err = contentKey(f, size); err != nil {
return fmt.Errorf("yaraworker: hash content: %w", err)
}
if strikes.Blocked(key) {
return errStruckOut
}
}
return nil
}
after := func(err error) {
if errors.Is(err, workerproc.ErrWorkerFailed) {
if key == "" {
key, _ = contentKey(f, size)
}
if key != "" {
b.loadedStrikes.Record(key)
}
}
}
payload, err := json.Marshal(yaraipc.ScanFDArgs{MaxBytes: size})
if err != nil {
return nil, err
}
var matches []yaraipc.Match
_, _, err = b.proc.CallGuarded(b.ctx, workerproc.Frame{Op: yaraipc.OpScanFD, Payload: payload}, []*os.File{f},
func(reply workerproc.Frame, _ []*os.File) (bool, error) {
m, decodeErr := yaraipc.DecodeScanReply(reply.Payload)
matches = m
return false, decodeErr
}, before, after)
if err != nil {
b.logScanErr(err)
return nil, err
}
return toYaraMatches(matches), nil
}
func contentKey(f *os.File, size int) (string, error) {
h := sha256.New()
if _, err := io.CopyN(h, io.NewSectionReader(f, 0, int64(size)), int64(size)); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// Warm starts the scan worker and loads the rules now, so the first scan
// after a compile does not wait for the load. Without rules it does nothing.
func (b *Backend) Warm() error {
if rules, _, _ := b.state(); rules == nil || rules.count == 0 {
return nil
}
_, _, err := b.proc.Call(b.ctx, workerproc.Frame{Op: yaraipc.OpPing}, nil,
func(reply workerproc.Frame, _ []*os.File) (bool, error) {
_, decodeErr := decodeRuleCount(reply.Payload)
return false, decodeErr
})
return err
}
// RuleCount is the number of compiled rules scans use.
func (b *Backend) RuleCount() int {
if rules, _, _ := b.state(); rules != nil {
return rules.count
}
return 0
}
// CompileError is why no rules are loaded, or "" when rules are loaded or
// the rule set is empty.
func (b *Backend) CompileError() string {
_, compileErr, _ := b.state()
return compileErr
}
// Stop cancels compilation and in-flight scans, kills the scan worker, and
// refuses later work.
func (b *Backend) Stop() error {
b.mu.Lock()
b.stopped = true
b.stop()
b.mu.Unlock()
err := b.proc.Stop()
// A compile owns its short-lived supervisor until it returns. Wait for
// its canceled call to kill and reap that worker before finishing stop.
_ = b.compileGate.Acquire(context.Background(), 1)
defer b.compileGate.Release(1)
b.mu.Lock()
rules := b.rules
b.rules = nil
b.mu.Unlock()
if rules != nil {
_ = rules.file.Close()
}
return err
}
// Stats is the scan worker supervisor's state.
func (b *Backend) Stats() workerproc.Stats { return b.proc.Stats() }
// QueueStatuses reports the scan worker's request queue.
func (b *Backend) QueueStatuses(now time.Time) map[string]queuehealth.Status {
return b.proc.QueueStatuses(now)
}
// Snapshot shutdown and rules together: Stop clears the rules, which must
// not be mistaken for the empty-directory exception in a byte scan.
func (b *Backend) state() (*compiledRules, string, bool) {
b.mu.Lock()
defer b.mu.Unlock()
return b.rules, b.compileErr, b.stopped
}
func (b *Backend) current() *compiledRules {
rules, _, _ := b.state()
return rules
}
func (b *Backend) isStopped() bool {
b.mu.Lock()
defer b.mu.Unlock()
return b.stopped
}
func (b *Backend) logScanErr(err error) { b.scanErrs.log(b.logf, err) }
func (b *Backend) logf(format string, args ...any) {
if b.cfg.Log != nil {
b.cfg.Log(format, args...)
}
}
func toYaraMatches(in []yaraipc.Match) []yara.Match {
if len(in) == 0 {
return nil
}
out := make([]yara.Match, len(in))
for i := range in {
out[i] = yara.Match{RuleName: in[i].RuleName, Meta: in[i].Meta}
}
return out
}
package yaraworker
import (
"sync"
"time"
)
// scanErrLog writes a scan failure at most once per scanErrLogWindow per
// distinct message, reporting how many identical failures were folded into
// the gap so the volume stays visible. A worker that cannot load its rules
// fails identically for every file, so the same message would otherwise be
// written once per scanned file. Tracking is per message rather than per
// most-recent, so two failures alternating cannot defeat suppression.
type scanErrLog struct {
mu sync.Mutex
seen map[string]*scanErrRecord
overflow scanErrRecord
}
// scanErrRecord tracks one distinct failure message: when it was last
// written, and how many identical failures happened since.
type scanErrRecord struct {
at time.Time
suppressed int
}
// scanErrMaxTracked bounds the distinct-message table so a failure carrying
// unique text (a path, an offset) cannot grow it without limit.
const scanErrMaxTracked = 32
// scanErrLogWindow bounds how often one recurring scan failure is written.
// Long enough that a broken rules directory cannot flood the journal, short
// enough that a persistent fault keeps reappearing.
const scanErrLogWindow = time.Minute
func (l *scanErrLog) log(logf func(string, ...any), err error) {
msg := err.Error()
now := time.Now()
l.mu.Lock()
if l.seen == nil {
l.seen = make(map[string]*scanErrRecord)
}
rec, ok := l.seen[msg]
overflow := !ok && len(l.seen) >= scanErrMaxTracked
if overflow {
// Preserve known recurring failures. Resetting the table lets a stream
// of unique offsets or paths disable throttling for every message.
rec = &l.overflow
ok = !rec.at.IsZero()
}
if ok && now.Sub(rec.at) < scanErrLogWindow {
rec.suppressed++
l.mu.Unlock()
return
}
if !ok && !overflow {
rec = &scanErrRecord{}
l.seen[msg] = rec
}
suppressed := rec.suppressed
rec.at = now
rec.suppressed = 0
l.mu.Unlock()
switch {
case overflow:
logf("scan: %v (%d additional scan failures suppressed)", err, suppressed)
case suppressed > 0:
logf("scan: %v (%d identical failures suppressed)", err, suppressed)
default:
logf("scan: %v", err)
}
}
package yaraworker
import (
"strconv"
"time"
"github.com/pidginhost/csm/internal/workerproc"
"github.com/pidginhost/csm/internal/yaraipc"
)
// Unit name prefixes. Startup kills units left by a daemon that died,
// matching on the prefix plus "-".
const (
ScanUnitPrefix = "csm-yara-worker"
CompileUnitPrefix = "csm-yara-compile-worker"
)
// ScanMemoryBase is the scan worker's memory cap before the largest content
// it may be handed: the loaded rules (179 MB for Forge full) plus headroom.
// The daemon adds the larger of its file and mail part size caps, because
// the worker reads a whole scan into memory.
const ScanMemoryBase = 512 << 20
// CompileMemoryLimit caps the compile worker. Compiling Forge full peaks at
// about 640 MB.
const CompileMemoryLimit = 1 << 30
// ScanUnitConfig is the locked-down unit the scan worker runs in. It writes
// no files. A relaunch reloads the rules, so it lives long: every restart
// costs a load of up to a few seconds.
func ScanUnitConfig(binary string, memoryLimit uint64, log func(string, ...any)) workerproc.UnitConfig {
return unitConfig(binary, ScanUnitPrefix, memoryLimit, 0, 24*time.Hour, log)
}
// CompileUnitConfig is the locked-down unit one compile runs in. Its only
// file is the memfd holding the compiled rules.
func CompileUnitConfig(binary string, log func(string, ...any)) workerproc.UnitConfig {
return unitConfig(binary, CompileUnitPrefix, CompileMemoryLimit, yaraipc.MaxSerializedRules, 5*time.Minute, log)
}
func unitConfig(binary, prefix string, memoryLimit, fileSizeLimit uint64, runtimeMax time.Duration, log func(string, ...any)) workerproc.UnitConfig {
return workerproc.UnitConfig{
Prefix: prefix,
Command: binary,
Args: []string{"yara-worker", "--memory-limit=" + strconv.FormatUint(memoryLimit, 10)},
MemoryLimit: memoryLimit,
FileSizeLimit: fileSizeLimit,
RuntimeMax: runtimeMax,
StartTimeout: 10 * time.Second,
Protocol: yaraipc.ProtocolVersion,
Limits: workerproc.Limits{MaxFrameBytes: yaraipc.MaxFrameBytes},
Log: log,
AllowExecutableMemory: true,
}
}
package yaraworker
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"os"
"strings"
"unicode/utf8"
"github.com/pidginhost/csm/internal/workerproc"
"github.com/pidginhost/csm/internal/yara"
"github.com/pidginhost/csm/internal/yaraipc"
)
// Engine compiles rule text and loads compiled rules. DefaultEngine is
// YARA-X; tests substitute a fake because plain builds have no engine.
type Engine interface {
Compile(sources []yara.Source) (Rules, error)
Load(r io.Reader) (Rules, error)
}
// Rules is a compiled rule set owned by the worker until Destroy.
type Rules interface {
Count() int
Destroy()
WriteTo(w io.Writer) (int64, error)
Scan(data []byte) ([]yara.Match, error)
}
// DefaultEngine is YARA-X. In a build without it every call returns
// yara.ErrNoEngine.
func DefaultEngine() Engine { return yaraEngine{} }
type yaraEngine struct{}
func (yaraEngine) Compile(sources []yara.Source) (Rules, error) {
r, err := yara.CompileIsolatedSources(sources)
if err != nil {
return nil, err
}
return r, nil
}
func (yaraEngine) Load(rd io.Reader) (Rules, error) {
r, err := yara.LoadRules(rd)
if err != nil {
return nil, err
}
return r, nil
}
// maxSourceName bounds a rule file name in a compile request.
const maxSourceName = 255
// Serve is the `csm yara-worker` loop on the socket its launcher handed it.
// The same worker compiles (in the short-lived compile unit) and scans (in
// the scan unit): it reads only descriptors it is given, never a path, and
// keeps the rules it loaded for the life of the connection.
func Serve(ctx context.Context, conn *net.UnixConn, eng Engine) error {
w := &worker{eng: eng}
defer w.close()
return workerproc.Serve(ctx, conn, workerproc.Limits{MaxFrameBytes: yaraipc.MaxFrameBytes, MaxFDs: 1}, w.handle)
}
type worker struct {
eng Engine
rules Rules
}
func (w *worker) close() {
if w.rules != nil {
w.rules.Destroy()
w.rules = nil
}
}
// fdCount is how many request descriptors each op takes.
var fdCount = map[string]int{
workerproc.OpHello: 0,
yaraipc.OpPing: 0,
yaraipc.OpCompile: 1,
yaraipc.OpLoad: 1,
yaraipc.OpScanFD: 1,
}
func (w *worker) handle(_ context.Context, req workerproc.Frame, fds []*os.File) (workerproc.Frame, []*os.File) {
if req.Error != "" {
return fail("request carries an error")
}
want, known := fdCount[req.Op]
if !known {
return fail("unknown op %q", req.Op)
}
if len(fds) != want {
return fail("%s takes %d descriptors, got %d", req.Op, want, len(fds))
}
if (req.Op == yaraipc.OpLoad || req.Op == yaraipc.OpPing) && len(req.Payload) != 0 {
if err := decodeArgs(req.Payload, &struct{}{}); err != nil {
return fail("%s: %v", req.Op, err)
}
}
switch req.Op {
case workerproc.OpHello:
var hello workerproc.Hello
if err := decodeArgs(req.Payload, &hello); err != nil {
return fail("hello: %v", err)
}
if hello.Protocol != yaraipc.ProtocolVersion {
return fail("hello: unsupported protocol %d", hello.Protocol)
}
return workerproc.HelloReply(yaraipc.ProtocolVersion), nil
case yaraipc.OpCompile:
return w.compile(req, fds[0])
case yaraipc.OpLoad:
return w.load(fds[0])
case yaraipc.OpPing:
if w.rules == nil {
return fail("no rules loaded")
}
return reply(yaraipc.LoadResult{RuleCount: w.rules.Count()}), nil
default:
return w.scan(req, fds[0])
}
}
func (w *worker) compile(req workerproc.Frame, f *os.File) (workerproc.Frame, []*os.File) {
var args yaraipc.CompileArgs
if err := decodeArgs(req.Payload, &args); err != nil {
return fail("compile: %v", err)
}
size, err := regularSize(f)
if err != nil {
return fail("compile: %v", err)
}
if err = checkSpans(args.Sources, size); err != nil {
return fail("compile: %v", err)
}
sources := make([]yara.Source, 0, len(args.Sources))
var off int64
for _, span := range args.Sources {
text := make([]byte, span.Size)
if _, err = io.ReadFull(io.NewSectionReader(f, off, span.Size), text); err != nil {
return fail("compile: read %s: %v", span.Name, err)
}
off += span.Size
sources = append(sources, yara.Source{Name: span.Name, Text: text})
}
rules, err := w.eng.Compile(sources)
if err != nil {
// The operator's rules are broken, not the worker.
return reply(yaraipc.CompileResult{CompileError: boundedDiagnostic(err.Error())}), nil
}
defer rules.Destroy()
var buf serializedRulesBuffer
if _, err = rules.WriteTo(&buf); errors.Is(err, errRulesTooLarge) {
// Like a syntax error, a rule set too big to hand over is a
// property of the rules, not a worker failure to retry.
return reply(yaraipc.CompileResult{CompileError: fmt.Sprintf(
"compiled rules exceed the %d MiB limit; trim the rules or set signatures.yara_worker_enabled: false",
yaraipc.MaxSerializedRules>>20)}), nil
} else if err != nil {
return fail("compile: serialize: %v", err)
}
out, err := yaraipc.SealedMemfd("csm-yara-rules", buf.buf.Bytes())
if err != nil {
return fail("compile: %v", err)
}
return reply(yaraipc.CompileResult{RuleCount: rules.Count()}), []*os.File{out}
}
var errRulesTooLarge = errors.New("compiled rules exceed size limit")
type serializedRulesBuffer struct{ buf bytes.Buffer }
func (b *serializedRulesBuffer) Write(p []byte) (int, error) {
if len(p) > yaraipc.MaxSerializedRules-b.buf.Len() {
return 0, errRulesTooLarge
}
return b.buf.Write(p)
}
// checkSpans requires named spans that exactly cover size bytes.
func checkSpans(spans []yaraipc.SourceSpan, size int64) error {
if len(spans) == 0 || len(spans) > yaraipc.MaxSources {
return fmt.Errorf("%d sources, want 1 to %d", len(spans), yaraipc.MaxSources)
}
var total int64
for _, s := range spans {
if s.Name == "" || len(s.Name) > maxSourceName || strings.ContainsAny(s.Name, "/\x00") {
return fmt.Errorf("invalid source name %q", s.Name)
}
if s.Size < 0 || s.Size > size-total {
return fmt.Errorf("source %s does not fit the descriptor", s.Name)
}
total += s.Size
}
if total != size {
return fmt.Errorf("sources cover %d of %d bytes", total, size)
}
return nil
}
func (w *worker) load(f *os.File) (workerproc.Frame, []*os.File) {
size, err := regularSize(f)
if err != nil {
return fail("load: %v", err)
}
if size > yaraipc.MaxSerializedRules {
return fail("load: compiled rules exceed size limit")
}
rules, err := w.eng.Load(io.NewSectionReader(f, 0, size))
if err != nil {
return fail("load: %v", err)
}
w.close()
w.rules = rules
return reply(yaraipc.LoadResult{RuleCount: rules.Count()}), nil
}
func (w *worker) scan(req workerproc.Frame, f *os.File) (workerproc.Frame, []*os.File) {
if w.rules == nil || w.rules.Count() == 0 {
return fail("no rules loaded")
}
var args yaraipc.ScanFDArgs
if err := decodeArgs(req.Payload, &args); err != nil {
return fail("scan_fd: %v", err)
}
if args.MaxBytes <= 0 {
return fail("invalid descriptor scan size: %d", args.MaxBytes)
}
size, err := regularSize(f)
if err != nil {
return fail("scan_fd: %v", err)
}
if int64(args.MaxBytes) > size {
return fail("read descriptor: %d bytes, want %d", size, args.MaxBytes)
}
// One buffer of the exact size: the memory cap allows one copy.
data := make([]byte, args.MaxBytes)
if _, err = io.ReadFull(io.NewSectionReader(f, 0, int64(args.MaxBytes)), data); err != nil {
return fail("read descriptor: %v", err)
}
matches, err := w.rules.Scan(data)
if err != nil {
return fail("scan: %v", err)
}
out := make([]yaraipc.Match, 0, len(matches))
for _, m := range matches {
out = append(out, yaraipc.Match{RuleName: m.RuleName, Meta: m.Meta})
}
return reply(struct {
Matches []yaraipc.Match `json:"matches"`
}{out}), nil
}
func regularSize(f *os.File) (int64, error) {
st, err := f.Stat()
if err != nil {
return 0, err
}
if !st.Mode().IsRegular() {
return 0, errors.New("descriptor is not a regular file")
}
return st.Size(), nil
}
func decodeArgs(payload json.RawMessage, out any) error {
payload = bytes.Trim(payload, " \t\r\n")
if len(payload) == 0 || payload[0] != '{' || !utf8.Valid(payload) {
return errors.New("payload must be a UTF-8 JSON object")
}
dec := json.NewDecoder(bytes.NewReader(payload))
dec.DisallowUnknownFields()
if err := dec.Decode(out); err != nil {
return err
}
if err := dec.Decode(new(json.RawMessage)); err != io.EOF {
return errors.New("payload has trailing data")
}
return nil
}
func reply(v any) workerproc.Frame {
payload, err := json.Marshal(v)
if err != nil {
return workerproc.Frame{Error: "marshal reply: " + err.Error()}
}
return workerproc.Frame{Payload: payload}
}
func fail(format string, args ...any) (workerproc.Frame, []*os.File) {
return workerproc.Frame{Error: boundedDiagnostic(fmt.Sprintf(format, args...))}, nil
}
func boundedDiagnostic(s string) string {
// JSON can expand each byte to six characters. Reserve framing space so
// even a diagnostic quoting a huge source line remains a result, not a
// transport failure that would count against worker health.
const limit = (yaraipc.MaxFrameBytes - 128) / 6
if len(s) > limit {
return strings.ToValidUTF8(s[:limit], "?") + "..."
}
return s
}
// Package zipdir locates a zip archive's central directory from its
// end-of-central-directory record, zip64 included, without reading any entry
// data. Callers use the record count and directory size to refuse an archive
// before allocating per-entry metadata for it.
package zipdir
import (
"archive/zip"
"bufio"
"context"
"encoding/binary"
"errors"
"io"
"io/fs"
"strings"
)
// Record layout constants from APPNOTE.TXT.
const (
HeaderLen = 46
EndLen = 22
End64Len = 56
End64LocatorLen = 20
MaxCommentLen = 1<<16 - 1
HeaderSignature = 0x02014b50
EndSignature = 0x06054b50
End64Signature = 0x06064b50
End64LocatorSign = 0x07064b50
)
var (
ErrFormat = errors.New("invalid zip central directory")
ErrScanLimit = errors.New("zip central directory exceeds scan limit")
)
// Directory locates a zip's central directory.
type Directory struct {
Offset int64
Size int64
Records uint64
// DeclaredOffset and EndOffset let callers check that another ZIP parser
// will select the same directory, including for self-extracting archives.
DeclaredOffset int64
EndOffset int64
}
// Read finds the end-of-central-directory record (zip64 included) and
// returns where the directory is and how many records it claims. A directory
// larger than maxDirBytes is ErrScanLimit. Nothing is decompressed and only
// the tail and the zip64 records are read.
func Read(r io.ReaderAt, size int64, maxDirBytes uint64) (Directory, error) {
if size < EndLen {
return Directory{}, ErrFormat
}
tailLen := int64(EndLen + MaxCommentLen)
if tailLen > size {
tailLen = size
}
tail := make([]byte, int(tailLen))
if err := readAtFull(r, tail, size-tailLen); err != nil {
return Directory{}, err
}
endIndex := findEnd(tail)
if endIndex < 0 {
return Directory{}, ErrFormat
}
endOffset := size - tailLen + int64(endIndex)
end := tail[endIndex : endIndex+EndLen]
if binary.LittleEndian.Uint16(end[4:6]) != 0 ||
binary.LittleEndian.Uint16(end[6:8]) != 0 {
return Directory{}, ErrFormat
}
recordsThisDisk := uint64(binary.LittleEndian.Uint16(end[8:10]))
records := uint64(binary.LittleEndian.Uint16(end[10:12]))
directorySize := uint64(binary.LittleEndian.Uint32(end[12:16]))
directoryOffset := uint64(binary.LittleEndian.Uint32(end[16:20]))
if recordsThisDisk != records {
return Directory{}, ErrFormat
}
directoryEndOffset := endOffset
if records == 0xffff || directorySize == 0xffffffff || directoryOffset == 0xffffffff {
var err error
directoryEndOffset, records, directorySize, directoryOffset, err = readZip64(r, endOffset)
if err != nil {
return Directory{}, err
}
}
if directorySize > maxDirBytes {
return Directory{}, ErrScanLimit
}
directorySize64, sizeOK := toOffset(directorySize)
directoryOffset64, offsetOK := toOffset(directoryOffset)
if !sizeOK || !offsetOK || directoryEndOffset < 0 ||
directorySize64 > directoryEndOffset || directoryOffset64 > directoryEndOffset {
return Directory{}, ErrFormat
}
if records > directorySize/HeaderLen {
return Directory{}, ErrFormat
}
return Directory{
Offset: directoryEndOffset - directorySize64,
Size: directorySize64,
Records: records,
DeclaredOffset: directoryOffset64,
EndOffset: endOffset,
}, nil
}
// findEnd returns the end record whose comment reaches the end of tail. A
// record found that way cannot be a decoy inside another record's comment.
// Without one, it falls back to the record nearest the end whose comment
// still fits, which is the record archive/zip opens, but only when nothing
// but padding follows it: a trailing line break or NUL fill reads the same in
// every unzip, while other bytes could carry a decoy a lenient one would pick.
func findEnd(tail []byte) int {
fallback := -1
for i := len(tail) - EndLen; i >= 0; i-- {
if binary.LittleEndian.Uint32(tail[i:i+4]) != EndSignature {
continue
}
commentLen := int(binary.LittleEndian.Uint16(tail[i+20 : i+22]))
switch end := i + EndLen + commentLen; {
case end == len(tail):
return i
case end < len(tail) && fallback < 0 && onlyPadding(tail[end:]):
fallback = i
}
}
return fallback
}
func onlyPadding(b []byte) bool {
for _, c := range b {
switch c {
case 0, ' ', '\t', '\r', '\n':
default:
return false
}
}
return true
}
func readZip64(r io.ReaderAt, endOffset int64) (end int64, records, size, offset uint64, err error) {
locatorOffset := endOffset - End64LocatorLen
if locatorOffset < 0 {
return 0, 0, 0, 0, ErrFormat
}
var locator [End64LocatorLen]byte
if err := readAtFull(r, locator[:], locatorOffset); err != nil {
return 0, 0, 0, 0, err
}
if binary.LittleEndian.Uint32(locator[0:4]) != End64LocatorSign ||
binary.LittleEndian.Uint32(locator[4:8]) != 0 ||
binary.LittleEndian.Uint32(locator[16:20]) != 1 {
return 0, 0, 0, 0, ErrFormat
}
zip64Offset := binary.LittleEndian.Uint64(locator[8:16])
zip64Offset64, ok := toOffset(zip64Offset)
if !ok || zip64Offset64 > locatorOffset {
return 0, 0, 0, 0, ErrFormat
}
var record [End64Len]byte
if err := readAtFull(r, record[:], zip64Offset64); err != nil {
return 0, 0, 0, 0, err
}
if binary.LittleEndian.Uint32(record[0:4]) != End64Signature ||
binary.LittleEndian.Uint64(record[4:12]) < End64Len-12 ||
binary.LittleEndian.Uint32(record[16:20]) != 0 ||
binary.LittleEndian.Uint32(record[20:24]) != 0 {
return 0, 0, 0, 0, ErrFormat
}
recordsThisDisk := binary.LittleEndian.Uint64(record[24:32])
records = binary.LittleEndian.Uint64(record[32:40])
if recordsThisDisk != records {
return 0, 0, 0, 0, ErrFormat
}
return zip64Offset64, records,
binary.LittleEndian.Uint64(record[40:48]),
binary.LittleEndian.Uint64(record[48:56]), nil
}
func toOffset(value uint64) (int64, bool) {
if value > 1<<63-1 {
return 0, false
}
return int64(value), true // #nosec G115 -- the MaxInt64 bound is checked above.
}
func readAtFull(r io.ReaderAt, dst []byte, offset int64) error {
n, err := r.ReadAt(dst, offset)
if n != len(dst) {
if errors.Is(err, io.EOF) {
return ErrFormat
}
if err != nil {
return err
}
return ErrFormat
}
if err != nil && !errors.Is(err, io.EOF) {
return err
}
return nil
}
// Entry is one central-directory record: its name and the attributes that
// say what kind of file it is. Nothing of the entry's data is read.
type Entry struct {
Name string
CreatorVersion uint16
ExternalAttrs uint32
}
// Mode interprets the record's attributes the way archive/zip does.
func (e Entry) Mode() fs.FileMode {
h := zip.FileHeader{
Name: strings.ReplaceAll(e.Name, `\`, "/"),
CreatorVersion: e.CreatorVersion,
ExternalAttrs: e.ExternalAttrs,
}
return h.Mode()
}
// Walk streams the central directory record by record, holding one record at
// a time, and calls visit for each. It checks ctx before every 256th record.
// A visit error stops the walk and is returned as is. A directory that holds
// a different number of records or bytes than dir claims is ErrFormat.
func Walk(ctx context.Context, r io.ReaderAt, dir Directory, visit func(Entry) error) error {
reader := bufio.NewReader(io.NewSectionReader(r, dir.Offset, dir.Size))
var header [HeaderLen]byte
remaining := dir.Size
for records := uint64(0); records < dir.Records; records++ {
if records%256 == 0 {
if err := ctx.Err(); err != nil {
return err
}
}
if remaining < HeaderLen {
return ErrFormat
}
if _, err := io.ReadFull(reader, header[:]); err != nil {
return readError(err)
}
if binary.LittleEndian.Uint32(header[:4]) != HeaderSignature {
return ErrFormat
}
nameLen := int(binary.LittleEndian.Uint16(header[28:30]))
extraLen := int64(binary.LittleEndian.Uint16(header[30:32]))
commentLen := int64(binary.LittleEndian.Uint16(header[32:34]))
remaining -= HeaderLen + int64(nameLen) + extraLen + commentLen
if remaining < 0 {
return ErrFormat
}
name := make([]byte, nameLen)
if _, err := io.ReadFull(reader, name); err != nil {
return readError(err)
}
if _, err := io.CopyN(io.Discard, reader, extraLen+commentLen); err != nil {
return readError(err)
}
if err := visit(Entry{
Name: string(name),
CreatorVersion: binary.LittleEndian.Uint16(header[4:6]),
ExternalAttrs: binary.LittleEndian.Uint32(header[38:42]),
}); err != nil {
return err
}
}
if remaining == 0 {
return ctx.Err()
}
// APPNOTE permits one digital signature after the file records. Anything
// else is unread directory content, even when the record count matches.
var signature [6]byte
if remaining < int64(len(signature)) {
return ErrFormat
}
if _, err := io.ReadFull(reader, signature[:]); err != nil {
return readError(err)
}
length := int64(binary.LittleEndian.Uint16(signature[4:6]))
if binary.LittleEndian.Uint32(signature[:4]) != 0x05054b50 || remaining != 6+length {
return ErrFormat
}
if _, err := io.CopyN(io.Discard, reader, length); err != nil {
return readError(err)
}
return ctx.Err()
}
func readError(err error) error {
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return ErrFormat
}
return err
}